def launcher(logger,port,lauchfrequency,maxruntime):
xmlrpcclient = xmlrpclib.ServerProxy('http://localhost:' + str(port))
maxseconds = maxruntime*60
time.sleep(3) #to allow jobqserver to start
while True:
time.sleep(lauchfrequency)
job = xmlrpcclient.getjob()
if job: #0 means nothing to launch
jobnumber = job[1]
task_to_run = job[2]
# Start a timer thread for maxruntime error
timer_thread = threading.Timer(maxseconds,maxruntimeerror,args=(logger,maxruntime,jobnumber,task_to_run))
timer_thread.start()
try:
starttime = datetime.datetime.now()
logger.info(u'Starting job %(job)s',job=jobnumber)
result = subprocess.call(task_to_run,stdin=open(os.devnull,'r'),stdout=open(os.devnull,'w'),stderr=open(os.devnull,'w'))
time_taken = datetime.timedelta(seconds=(datetime.datetime.now() - starttime).seconds)
logger.info(u'Finished job %(job)s, elapsed time %(time_taken)s, result %(result)s',job=jobnumber,time_taken=time_taken,result=result)
except Exception, msg:
logger.error(u'Error starting job %(job)s: %(msg)s',job=jobnumber,msg=msg)
botslib.sendbotserrorreport(u'[Bots Job Queue] - Error starting job',
u'Error starting job %s:\n %s\n\n %s' % (jobnumber,task_to_run,msg))
timer_thread.cancel()
def generate_report(stuff2evaluate):
for results in botslib.query('''SELECT idta,lastopen,lasterror,lastok,lastdone,
send,processerrors,ts,lastreceived,type,status
FROM report
WHERE idta=%(rootidta)s''',
{'rootidta':stuff2evaluate}):
break
else:
raise botslib.PanicError(_(u'In generate report: could not find report?'))
subject = _(u'[Bots Error Report] %(time)s')%{'time':str(results['ts'])[:16]}
reporttext = _(u'Bots Report; type: %(type)s, time: %(time)s\n')%{'type':results['type'],'time':str(results['ts'])[:19]}
reporttext += _(u' %d files received/processed in run.\n')%(results['lastreceived'])
if results['lastdone']:
reporttext += _(u' %d files without errors,\n')%(results['lastdone'])
if results['lasterror']:
subject += _(u'; %d file errors')%(results['lasterror'])
reporttext += _(u' %d files with errors,\n')%(results['lasterror'])
if results['lastok']:
subject += _(u'; %d files stuck')%(results['lastok'])
reporttext += _(u' %d files got stuck,\n')%(results['lastok'])
if results['lastopen']:
subject += _(u'; %d system errors')%(results['lastopen'])
reporttext += _(u' %d system errors,\n')%(results['lastopen'])
if results['processerrors']:
subject += _(u'; %d process errors')%(results['processerrors'])
reporttext += _(u' %d errors in processes.\n')%(results['processerrors'])
reporttext += _(u' %d files send in run.\n')%(results['send'])
botsglobal.logger.info(reporttext)
# sendreportifprocesserror allows blocking of email reports for process errors
if (results['lasterror'] or results['lastopen'] or results['lastok'] or
(results['processerrors'] and botsglobal.ini.getboolean('settings','sendreportifprocesserror',True))):
botslib.sendbotserrorreport(subject,reporttext)
return int(results['status']) #return report status: 0 (no error) or 1 (error)
def launcher(logger,port,lauchfrequency,maxruntime):
xmlrpcclient = xmlrpclib.ServerProxy(u'http://localhost:' + unicode(port))
maxseconds = maxruntime*60
time.sleep(3) #to allow jobqserver to start
while True:
time.sleep(lauchfrequency)
job = xmlrpcclient.getjob()
if job: #0 means nothing to launch
jobnumber = job[1]
task_to_run = job[2]
# Start a timer thread for maxruntime error
timer_thread = threading.Timer(maxseconds,maxruntimeerror,args=(logger,maxruntime,jobnumber,task_to_run))
timer_thread.start()
try:
starttime = datetime.datetime.now()
logger.info(u'Starting job %(job)s',{'job':jobnumber})
result = subprocess.call(task_to_run,stdin=open(os.devnull,'r'),stdout=open(os.devnull,'w'),stderr=open(os.devnull,'w'))
time_taken = datetime.timedelta(seconds=(datetime.datetime.now() - starttime).seconds)
logger.info(u'Finished job %(job)s, elapsed time %(time_taken)s, result %(result)s',{'job':jobnumber,'time_taken':time_taken,'result':result})
except Exception as msg:
logger.error(u'Error starting job %(job)s: %(msg)s',{'job':jobnumber,'msg':msg})
botslib.sendbotserrorreport(u'[Bots Job Queue] - Error starting job',
u'Error starting job %(job)s:\n %(task)s\n\n %(msg)s' % {'job':jobnumber,'task':task_to_run,'msg':msg})
timer_thread.cancel()
def email_error_report(rootidtaofrun):
for results in botslib.query('''SELECT idta,lastopen,lasterror,lastok,lastdone,
send,processerrors,ts,lastreceived,type,status
FROM report
WHERE idta=%(rootidtaofrun)s''',
{'rootidtaofrun':rootidtaofrun}):
break
else:
raise botslib.PanicError(_(u'In generate report: could not find report?'))
subject = _(u'[Bots Error Report] %(time)s')%{'time':str(results['ts'])[:16]}
reporttext = _(u'Bots Report; type: %(type)s, time: %(time)s\n')%{'type':results['type'],'time':str(results['ts'])[:19]}
reporttext += _(u' %d files received/processed in run.\n')%(results['lastreceived'])
if results['lastdone']:
reporttext += _(u' %d files without errors,\n')%(results['lastdone'])
if results['lasterror']:
subject += _(u'; %d file errors')%(results['lasterror'])
reporttext += _(u' %d files with errors,\n')%(results['lasterror'])
if results['lastok']:
subject += _(u'; %d files stuck')%(results['lastok'])
reporttext += _(u' %d files got stuck,\n')%(results['lastok'])
if results['lastopen']:
subject += _(u'; %d system errors')%(results['lastopen'])
reporttext += _(u' %d system errors,\n')%(results['lastopen'])
if results['processerrors']:
subject += _(u'; %d process errors')%(results['processerrors'])
reporttext += _(u' %d errors in processes.\n')%(results['processerrors'])
reporttext += _(u' %d files send in run.\n')%(results['send'])
botsglobal.logger.info(reporttext) #log the report texts
# only send email report if there are errors.
# sendreportifprocesserror (in bots.ini): no email reports if only process errors
if results['lasterror'] or results['lastopen'] or results['lastok'] or (results['processerrors'] and botsglobal.ini.getboolean('settings','sendreportifprocesserror',True)):
# Include details about process errors in the email report; if debug is True: includes trace
if results['processerrors']:
for row in botslib.query('''SELECT idroute,fromchannel,tochannel,errortext
FROM ta
WHERE idta>=%(rootidtaofrun)s
AND status=%(status)s
AND statust=%(statust)s ''',
{'rootidtaofrun':rootidtaofrun,'status':PROCESS,'statust':ERROR}):
reporttext += '\nProcess error:\n'
for key in row.keys():
reporttext += '%s: %s\n' % (key,row[key])
# Include details about file errors in the email report; if debug is True: includes trace
if results['lasterror'] or results['lastopen'] or results['lastok']:
for row in botslib.query('''SELECT idroute,frompartner,fromchannel,topartner,tochannel,errortext,infilename
FROM filereport
WHERE idta>%(rootidtaofrun)s
AND statust!=%(statust)s ''',
{'rootidtaofrun':rootidtaofrun,'statust':DONE}):
reporttext += '\nFile error:\n'
for key in row.keys():
reporttext += '%s: %s\n' % (key,row[key])
botslib.sendbotserrorreport(subject,reporttext)
return int(results['status']) #return report status: 0 (no error) or 1 (error)
def email_error_report(rootidtaofrun):
for results in botslib.query('''SELECT idta,lastopen,lasterror,lastok,lastdone,
send,processerrors,ts,lastreceived,type,status
FROM report
WHERE idta=%(rootidtaofrun)s''',
{'rootidtaofrun':rootidtaofrun}):
break
else:
raise botslib.PanicError(_(u'In generate report: could not find report?'))
subject = _(u'[Bots Error Report] %(time)s')%{'time':unicode(results['ts'])[:16]}
reporttext = _(u'Bots Report; type: %(type)s, time: %(time)s\n')%{'type':results['type'],'time':unicode(results['ts'])[:19]}
reporttext += _(u' %d files received/processed in run.\n')%(results['lastreceived'])
if results['lastdone']:
reporttext += _(u' %d files without errors,\n')%(results['lastdone'])
if results['lasterror']:
subject += _(u'; %d file errors')%(results['lasterror'])
reporttext += _(u' %d files with errors,\n')%(results['lasterror'])
if results['lastok']:
subject += _(u'; %d files stuck')%(results['lastok'])
reporttext += _(u' %d files got stuck,\n')%(results['lastok'])
if results['lastopen']:
subject += _(u'; %d system errors')%(results['lastopen'])
reporttext += _(u' %d system errors,\n')%(results['lastopen'])
if results['processerrors']:
subject += _(u'; %d process errors')%(results['processerrors'])
reporttext += _(u' %d errors in processes.\n')%(results['processerrors'])
reporttext += _(u' %d files send in run.\n')%(results['send'])
botsglobal.logger.info(reporttext) #log the report texts
# only send email report if there are errors.
# sendreportifprocesserror (in bots.ini): no email reports if only process errors
if results['lasterror'] or results['lastopen'] or results['lastok'] or (results['processerrors'] and botsglobal.ini.getboolean('settings','sendreportifprocesserror',True)):
# Include details about process errors in the email report; if debug is True: includes trace
if results['processerrors']:
for row in botslib.query('''SELECT idroute,fromchannel,tochannel,errortext
FROM ta
WHERE idta>=%(rootidtaofrun)s
AND status=%(status)s
AND statust=%(statust)s ''',
{'rootidtaofrun':rootidtaofrun,'status':PROCESS,'statust':ERROR}):
reporttext += '\nProcess error:\n'
for key in row.keys():
reporttext += '%s: %s\n' % (key,row[key])
# Include details about file errors in the email report; if debug is True: includes trace
if results['lasterror'] or results['lastopen'] or results['lastok']:
for row in botslib.query('''SELECT idroute,frompartner,fromchannel,topartner,tochannel,errortext,infilename
FROM filereport
WHERE idta>%(rootidtaofrun)s
AND statust!=%(statust)s ''',
{'rootidtaofrun':rootidtaofrun,'statust':DONE}):
reporttext += '\nFile error:\n'
for key in row.keys():
reporttext += '%s: %s\n' % (key,row[key])
botslib.sendbotserrorreport(subject,reporttext)
return int(results['status']) #return report status: 0 (no error) or 1 (error)
def generate_report(stuff2evaluate):
for results in botslib.query(
'''SELECT idta,lastopen,lasterror,lastok,lastdone,
send,processerrors,ts,lastreceived,type,status
FROM report
WHERE idta=%(rootidta)s''',
{'rootidta': stuff2evaluate}):
break
else:
raise botslib.PanicError(
_(u'In generate report: could not find report?'))
subject = _(u'[Bots Error Report] %(time)s') % {
'time': str(results['ts'])[:16]
}
reporttext = _(u'Bots Report; type: %(type)s, time: %(time)s\n') % {
'type': results['type'],
'time': str(results['ts'])[:19]
}
reporttext += _(u' %d files received/processed in run.\n') % (
results['lastreceived'])
if results['lastdone']:
reporttext += _(u' %d files without errors,\n') % (
results['lastdone'])
if results['lasterror']:
subject += _(u'; %d file errors') % (results['lasterror'])
reporttext += _(u' %d files with errors,\n') % (
results['lasterror'])
if results['lastok']:
subject += _(u'; %d files stuck') % (results['lastok'])
reporttext += _(u' %d files got stuck,\n') % (results['lastok'])
if results['lastopen']:
subject += _(u'; %d system errors') % (results['lastopen'])
reporttext += _(u' %d system errors,\n') % (results['lastopen'])
if results['processerrors']:
subject += _(u'; %d process errors') % (results['processerrors'])
reporttext += _(u' %d errors in processes.\n') % (
results['processerrors'])
reporttext += _(u' %d files send in run.\n') % (results['send'])
botsglobal.logger.info(reporttext)
# sendreportifprocesserror allows blocking of email reports for process errors
if (results['lasterror'] or results['lastopen'] or results['lastok']
or (results['processerrors'] and botsglobal.ini.getboolean(
'settings', 'sendreportifprocesserror', True))):
botslib.sendbotserrorreport(subject, reporttext)
return int(
results['status']) #return report status: 0 (no error) or 1 (error)
def generate_report(stuff2evaluate):
for results in botslib.query(
"""SELECT idta,lastopen,lasterror,lastok,lastdone,
send,processerrors,ts,lastreceived,type,status
FROM report
WHERE idta=%(rootidta)s""",
{"rootidta": stuff2evaluate},
):
break
else:
raise botslib.PanicError(_(u"In generate report: could not find report?"))
subject = _(u"[Bots Error Report] %(time)s") % {"time": str(results["ts"])[:16]}
reporttext = _(u"Bots Report; type: %(type)s, time: %(time)s\n") % {
"type": results["type"],
"time": str(results["ts"])[:19],
}
reporttext += _(u" %d files received/processed in run.\n") % (results["lastreceived"])
if results["lastdone"]:
reporttext += _(u" %d files without errors,\n") % (results["lastdone"])
if results["lasterror"]:
subject += _(u"; %d file errors") % (results["lasterror"])
reporttext += _(u" %d files with errors,\n") % (results["lasterror"])
if results["lastok"]:
subject += _(u"; %d files stuck") % (results["lastok"])
reporttext += _(u" %d files got stuck,\n") % (results["lastok"])
if results["lastopen"]:
subject += _(u"; %d system errors") % (results["lastopen"])
reporttext += _(u" %d system errors,\n") % (results["lastopen"])
if results["processerrors"]:
subject += _(u"; %d process errors") % (results["processerrors"])
reporttext += _(u" %d errors in processes.\n") % (results["processerrors"])
reporttext += _(u" %d files send in run.\n") % (results["send"])
botsglobal.logger.info(reporttext)
# sendreportifprocesserror allows blocking of email reports for process errors
if (
results["lasterror"]
or results["lastopen"]
or results["lastok"]
or (results["processerrors"] and botsglobal.ini.getboolean("settings", "sendreportifprocesserror", True))
):
botslib.sendbotserrorreport(subject, reporttext)
return int(results["status"]) # return report status: 0 (no error) or 1 (error)
def start():
#exit codes:
# 0: OK, no errors
# 1: (system) errors
# 2: bots ran OK, but there are errors/process errors in the run
# 3: Database is locked, but "maxruntime" has not been exceeded.
#********command line arguments**************************
commandspossible = [
'--new', '--retry', '--retransmit', '--cleanup', '--crashrecovery',
'--retrycommunication', '--automaticretrycommunication'
]
commandstorun = []
routestorun = [] #list with routes to run
configdir = 'config'
for arg in sys.argv[1:]:
if not arg:
continue
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Configuration directory indicated, but no directory name.'
sys.exit(1)
elif arg in commandspossible:
commandstorun.append(arg)
elif arg in ["?", "/?"] or arg.startswith('-'):
showusage()
sys.exit(0)
else: #pick up names of routes to run
routestorun.append(arg)
if not commandstorun: #if no command on command line, use new (default)
commandstorun = ['--new']
#**************init general: find locating of bots, configfiles, init paths etc.****************
botsinit.generalinit(configdir)
#set current working directory to botspath
#~ old_current_directory = os.getcwdu()
os.chdir(botsglobal.ini.get('directories', 'botspath'))
#**************initialise logging******************************
try:
botsinit.initenginelogging()
except:
print _('Error in initialising logging system.')
traceback.print_exc()
sys.exit(1)
else:
atexit.register(logging.shutdown)
for key, value in botslib.botsinfo(): #log start info
botsglobal.logger.info(u'%s: "%s".', key, value)
#**************connect to database**********************************
try:
botsinit.connect()
except:
botsglobal.logger.exception(
_(u'Could not connect to database. Database settings are in bots/config/settings.py.'
))
sys.exit(1)
else:
botsglobal.logger.info(_(u'Connected to database.'))
atexit.register(botsglobal.db.close)
#initialise user exits for the whole bots-engine (this script file)
try:
userscript, scriptname = botslib.botsimport('routescripts',
'botsengine')
except ImportError:
userscript = scriptname = None
#**************handle database lock****************************************
#try to set a lock on the database; if this is not possible, the database is already locked. Either:
#1 another instance bots bots-engine is (still) running
#2 or bots-engine had a severe crash.
#What to do?
#first: check ts of database lock. If below a certain value (set in bots.ini) we assume an other instance is running. Exit quietly - no errors, no logging.
# else: Warn user, give advise on what to do. gather data: nr files in, errors.
#next: warn with report & logging. advise a crashrecovery.
if not botslib.set_database_lock():
if '--crashrecovery' in commandstorun: #user starts recovery operation; the databaselock is ignored; the databaselock is unlocked when routes have run.
commandstorun = ['--crashrecovery'] #is an exclusive option!
else:
#when scheduling bots it is possible that the last run is still running. Check if maxruntime has passed:
vanaf = datetime.datetime.today() - datetime.timedelta(
minutes=botsglobal.ini.getint('settings', 'maxruntime', 60))
for row in botslib.query(
'''SELECT ts FROM mutex WHERE ts < %(vanaf)s ''',
{'vanaf': vanaf}):
warn = _(
u'!Bots database is locked!\nBots-engine has ended in an unexpected way during the last run.\nThis happens, but is very very rare.\nPossible causes: bots-engine terminated by user, system crash, power-down, etc.\nA forced retry of the last run is advised; bots will (try to) repair the last run.'
)
botsglobal.logger.critical(warn)
botslib.sendbotserrorreport(
_(u'[Bots severe error]Database is locked'), warn)
#add: count errors etc.
sys.exit(1)
else: #maxruntime has not passed. Exit silently, nothing reported
botsglobal.logger.info(
_(u'Database is locked, but "maxruntime" has not been exceeded.'
))
sys.exit(3)
else:
if '--crashrecovery' in commandstorun: #user starts recovery operation but there is no databaselock.
warn = _(
u'User started a forced retry of the last run.\nOnly use this when the database is locked.\nThe database was not locked (database is OK).\nSo Bots has done nothing now.'
)
botsglobal.logger.error(warn)
botslib.sendbotserrorreport(
_(u'[Bots Error Report] User started a forced retry of last run, but this was not needed'
), warn)
botslib.remove_database_lock()
sys.exit(1)
#*************get list of routes to run****************************************
#~ raise Exception('locked database') #for testing database lock: abort, database will be locked
if routestorun:
botsglobal.logger.info(u'Run routes from command line: "%s".',
str(routestorun))
else: # no routes from command line parameters: fetch all active routes from database
for row in botslib.query(
'''SELECT DISTINCT idroute
FROM routes
WHERE active=%(active)s
AND (notindefaultrun=%(notindefaultrun)s OR notindefaultrun IS NULL)
ORDER BY idroute ''', {
'active': True,
'notindefaultrun': False
}):
routestorun.append(row['idroute'])
botsglobal.logger.info(_(u'Run active routes from database: "%s".'),
str(routestorun))
#routestorun is now either a list with routes from commandline, or the list of active routes for the routes table in the db.
#**************run the routes for retry, retransmit and new runs*************************************
try:
#commandstorun determines the type(s) of run
#routes to run is a listof the routes that are runs (for each command to run
#botsglobal.incommunicate is used to control if there is communication in; only 'new' incommunicates.
#botsglobal.minta4query controls which ta's are queried by the routes.
#stuff2evaluate controls what is evaluated in automatic maintenance.
errorinrun = 0 #detect if there has been some error. Only used for good exit() code
botsglobal.incommunicate = False
if '--crashrecovery' in commandstorun:
botsglobal.logger.info(_(u'Run crash recovery.'))
stuff2evaluate = botslib.set_minta4query_crashrecovery()
if stuff2evaluate:
router.routedispatcher(routestorun)
errorinrun += automaticmaintenance.evaluate(
'--crashrecovery', stuff2evaluate)
else:
botsglobal.logger.info(
_(u'No retry of the last run - there was no last run.'))
if userscript and hasattr(userscript, 'postcrashrecovery'):
botslib.runscript(userscript,
scriptname,
'postcrashrecovery',
routestorun=routestorun)
if '--retrycommunication' in commandstorun:
botsglobal.logger.info(_(u'Run communication retry.'))
stuff2evaluate = router.routedispatcher(routestorun,
'--retrycommunication')
if stuff2evaluate:
errorinrun += automaticmaintenance.evaluate(
'--retrycommunication', stuff2evaluate)
else:
botsglobal.logger.info(
_(u'Run recommunicate: nothing to recommunicate.'))
if userscript and hasattr(userscript, 'postretrycommunication'):
botslib.runscript(userscript,
scriptname,
'postretrycommunication',
routestorun=routestorun)
if '--automaticretrycommunication' in commandstorun:
botsglobal.logger.info(_(u'Run automatic communication retry.'))
stuff2evaluate = router.routedispatcher(
routestorun, '--automaticretrycommunication')
if stuff2evaluate:
errorinrun += automaticmaintenance.evaluate(
'--automaticretrycommunication', stuff2evaluate)
else:
botsglobal.logger.info(
_(u'Run automatic recommunicate: nothing to recommunicate.'
))
if userscript and hasattr(userscript,
'postautomaticretrycommunication'):
botslib.runscript(userscript,
scriptname,
'postautomaticretrycommunication',
routestorun=routestorun)
if '--retry' in commandstorun:
botsglobal.logger.info(u'Run retry.')
stuff2evaluate = router.routedispatcher(routestorun, '--retry')
if stuff2evaluate:
errorinrun += automaticmaintenance.evaluate(
'--retry', stuff2evaluate)
else:
botsglobal.logger.info(_(u'Run retry: nothing to retry.'))
if userscript and hasattr(userscript, 'postretry'):
botslib.runscript(userscript,
scriptname,
'postretry',
routestorun=routestorun)
if '--retransmit' in commandstorun:
botsglobal.logger.info(u'Run retransmit.')
stuff2evaluate = router.routedispatcher(routestorun,
'--retransmit')
if stuff2evaluate:
errorinrun += automaticmaintenance.evaluate(
'--retransmit', stuff2evaluate)
else:
botsglobal.logger.info(
_(u'Run retransmit: nothing to retransmit.'))
if userscript and hasattr(userscript, 'postretransmit'):
botslib.runscript(userscript,
scriptname,
'postretransmit',
routestorun=routestorun)
if '--new' in commandstorun:
botsglobal.logger.info('Run new.')
botsglobal.incommunicate = True
botsglobal.minta4query = 0 #meaning: reset. the actual value is set later (in routedispatcher)
stuff2evaluate = router.routedispatcher(routestorun)
errorinrun += automaticmaintenance.evaluate(
'--new', stuff2evaluate)
if userscript and hasattr(userscript, 'postnewrun'):
botslib.runscript(userscript,
scriptname,
'postnewrun',
routestorun=routestorun)
if '--cleanup' in commandstorun or botsglobal.ini.get(
'settings', 'whencleanup', 'always') == 'always':
botsglobal.logger.debug(u'Do cleanup.')
cleanup.cleanup()
botslib.remove_database_lock()
except Exception, e:
botsglobal.logger.exception(
_(u'Severe error in bots system:\n%s') %
(e)) #of course this 'should' not happen.
sys.exit(1)
def maxruntimeerror(logger,maxruntime,jobnumber,task_to_run):
logger.error(u'Job %(job)s exceeded maxruntime of %(maxruntime)s minutes',job=jobnumber,maxruntime=maxruntime)
botslib.sendbotserrorreport(u'[Bots Job Queue] - Job exceeded maximum runtime',
u'Job %s exceeded maxruntime of %s minutes:\n %s' % (jobnumber,maxruntime,task_to_run))
def start():
''' sysexit codes:
0: OK, no errors
1: (system) errors incl parsing of command line arguments
2: bots ran OK, but there are errors/process errors in the run
3: Database is locked, but "maxruntime" has not been exceeded.
'''
#NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
#********command line arguments**************************
usage = '''
This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
Does the actual translations and communications; it's the workhorse. It does not have a fancy interface.
Usage:
%(name)s [run-options] [config-option] [routes]
Run-options (can be combined):
--new receive new edi files (default: if no run-option given: run as new).
--resend resend as indicated by user.
--rereceive rereceive as indicated by user.
--automaticretrycommunication - automatically retry outgoing communication.
--cleanup remove older data from database.
Config-option:
-c<directory> directory for configuration files (default: config).
Routes: list of routes to run. Default: all active routes (in the database)
'''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version}
configdir = 'config'
commandspossible = ['--automaticretrycommunication','--resend','--rereceive','--new']
commandstorun = []
routestorun = [] #list with routes to run
do_cleanup_parameter = False
for arg in sys.argv[1:]:
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Error: configuration directory indicated, but no directory name.'
sys.exit(1)
elif arg in commandspossible:
commandstorun.append(arg)
elif arg == '--cleanup':
do_cleanup_parameter = True
elif arg in ["?", "/?",'-h', '--help'] or arg.startswith('-'):
print usage
sys.exit(0)
else: #pick up names of routes to run
routestorun.append(arg)
if not commandstorun and not do_cleanup_parameter: #if no command on command line, use new (default)
commandstorun = ['--new']
commandstorun = [command[2:] for command in commandspossible if command in commandstorun] #sort commands
#***********end handling command line arguments**************************
botsinit.generalinit(configdir) #find locating of bots, configfiles, init paths etc.
#set working directory to bots installation. advantage: when using relative paths it is clear that this point paths within bots installation.
os.chdir(botsglobal.ini.get('directories','botspath'))
#**************check if another instance of bots-engine is running/if port is free******************************
try:
engine_socket = botslib.check_if_other_engine_is_running()
except socket.error:
sys.exit(3)
else:
atexit.register(engine_socket.close)
#**************initialise logging******************************
process_name = 'engine'
botsglobal.logger = botsinit.initenginelogging(process_name)
atexit.register(logging.shutdown)
for key,value in botslib.botsinfo(): #log info about environement, versions, etc
botsglobal.logger.info(u'%(key)s: "%(value)s".',{'key':key,'value':value})
#**************connect to database**********************************
try:
botsinit.connect()
except Exception as msg:
botsglobal.logger.exception(_(u'Could not connect to database. Database settings are in bots/config/settings.py. Error: "%(msg)s".'),{'msg':msg})
sys.exit(1)
else:
botsglobal.logger.info(_(u'Connected to database.'))
atexit.register(botsglobal.db.close)
#************initialise user exits for the whole bots-engine*************************
try:
userscript,scriptname = botslib.botsimport('routescripts','botsengine')
except ImportError: #userscript is not there; other errors like syntax errors are not catched
userscript = scriptname = None
#***acceptance tests: initialiase acceptance user script******************************
acceptance_userscript = acceptance_scriptname = None
if botsglobal.ini.getboolean('acceptance','runacceptancetest',False):
botsglobal.logger.info(_(u'This run is an acceptance test - as indicated in option "runacceptancetest" in bots.ini.'))
try:
acceptance_userscript,acceptance_scriptname = botslib.botsimport('routescripts','bots_acceptancetest')
except ImportError:
botsglobal.logger.info(_(u'In acceptance test there is no script file "bots_acceptancetest.py" to check the results of the acceptance test.'))
#**************handle database lock****************************************
#set a lock on the database; if not possible, the database is locked: an earlier instance of bots-engine was terminated unexpectedly.
if not botslib.set_database_lock():
#for SQLite: do a integrity check on the database
if botsglobal.settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3':
cursor = botsglobal.db.execute('''PRAGMA integrity_check''')
result = cursor.fetchone()
if result[0] != u'ok':
warn = _(u'!Bots database is locked!\n'\
'Bots did an integrity check on the database, but database was not OK.\n'\
'Manual action is needed!\n'\
'Bots has stopped processing EDI files.')
botsglobal.logger.critical(warn)
botslib.sendbotserrorreport(_(u'[Bots severe error]Database is damaged'),warn)
sys.exit(1)
warn = _(u'!Bots database is locked!\n'\
'Bots-engine has ended in an unexpected way during the last run.\n'\
'Most likely causes: sudden power-down, system crash, problems with disk I/O, bots-engine terminated by user, etc.\n'
'Bots will do an automatic crash recovery now.')
botsglobal.logger.critical(warn)
botslib.sendbotserrorreport(_(u'[Bots severe error]Database is locked'),warn)
commandstorun.insert(0,'crashrecovery') #there is a database lock. Add a crashrecovery as first command to run.
atexit.register(botslib.remove_database_lock)
#**************run the routes**********************************************
#commandstorun determines the type(s) of run. eg: ['automaticretrycommunication','new']
try:
botslib.prepare_confirmrules()
#in acceptance tests: run a user script before running eg to clean output directories******************************
botslib.tryrunscript(acceptance_userscript,acceptance_scriptname,'pretest',routestorun=routestorun)
botslib.tryrunscript(userscript,scriptname,'pre',commandstorun=commandstorun,routestorun=routestorun)
errorinrun = 0 #detect if there has been some error. Only used for correct exit() code
first_command_2_run = True
for command in commandstorun:
#if multiple commands in run: reports etc are based on timestamp; so there needs to be at least one second between these runs.
if first_command_2_run:
first_command_2_run = False
else:
time.sleep(1)
botsglobal.logger.info(_(u'Run "%(command)s".'),{'command':command})
#************get list of routes to run*******************************
if routestorun:
use_routestorun = routestorun[:]
botsglobal.logger.info(_(u'Run routes from command line: "%(routes)s".'),{'routes':str(use_routestorun)})
elif command == 'new': #fetch all active routes from database unless 'not in default run' or not active.
use_routestorun = []
for row in botslib.query('''SELECT DISTINCT idroute
FROM routes
WHERE active=%(active)s
AND (notindefaultrun=%(notindefaultrun)s OR notindefaultrun IS NULL)
ORDER BY idroute ''',
{'active':True,'notindefaultrun':False}):
use_routestorun.append(row['idroute'])
botsglobal.logger.info(_(u'Run active routes from database that are in default run: "%(routes)s".'),{'routes':str(use_routestorun)})
else: #for command other than 'new': use all active routes.
use_routestorun = []
for row in botslib.query('''SELECT DISTINCT idroute
FROM routes
WHERE active=%(active)s
ORDER BY idroute ''',
{'active':True}):
use_routestorun.append(row['idroute'])
botsglobal.logger.info(_(u'Run all active routes from database: "%(routes)s".'),{'routes':str(use_routestorun)})
#************run routes for this command******************************
botslib.tryrunscript(userscript,scriptname,'pre' + command,routestorun=use_routestorun)
errorinrun += router.rundispatcher(command,use_routestorun)
botslib.tryrunscript(userscript,scriptname,'post' + command,routestorun=use_routestorun)
#*********finished running routes for this command****************************
#*********finished all commands****************************************
botslib.tryrunscript(userscript,scriptname,'post',commandstorun=commandstorun,routestorun=routestorun)
try: #in acceptance tests: run a user script. no good reporting of errors/results in post-test script. Reason: this is after automaticmaintence.
botslib.tryrunscript(acceptance_userscript,acceptance_scriptname,'posttest',routestorun=use_routestorun)
except Exception as msg:
print str(msg)
cleanup.cleanup(do_cleanup_parameter,userscript,scriptname)
except Exception as msg:
botsglobal.logger.exception(_(u'Severe error in bots system:\n%(msg)s'),{'msg':str(msg)}) #of course this 'should' not happen.
sys.exit(1)
else:
if errorinrun:
sys.exit(2) #indicate: error(s) in run(s)
else:
sys.exit(0) #OK
def start():
''' sysexit codes:
0: OK, no errors
1: (system) errors incl parsing of command line arguments
2: bots ran OK, but there are errors/process errors in the run
3: Database is locked, but "maxruntime" has not been exceeded.
'''
#NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
#********command line arguments**************************
usage = '''
This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
Does the actual translations and communications; it's the workhorse. It does not have a fancy interface.
Usage:
%(name)s [run-options] [config-option] [routes]
Run-options (can be combined):
--new receive new edi files (default: if no run-option given: run as new).
--resend resend as indicated by user.
--rereceive rereceive as indicated by user.
--automaticretrycommunication - automatically retry outgoing communication.
--cleanup remove older data from database.
Config-option:
-c<directory> directory for configuration files (default: config).
Routes: list of routes to run. Default: all active routes (in the database)
''' % {
'name': os.path.basename(sys.argv[0]),
'version': botsglobal.version
}
configdir = 'config'
commandspossible = [
'--automaticretrycommunication', '--resend', '--rereceive', '--new'
]
commandstorun = []
routestorun = [] #list with routes to run
do_cleanup_parameter = False
for arg in sys.argv[1:]:
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Error: configuration directory indicated, but no directory name.'
sys.exit(1)
elif arg in commandspossible:
commandstorun.append(arg)
elif arg == '--cleanup':
do_cleanup_parameter = True
elif arg in ["?", "/?", '-h', '--help'] or arg.startswith('-'):
print usage
sys.exit(0)
else: #pick up names of routes to run
routestorun.append(arg)
if not commandstorun and not do_cleanup_parameter: #if no command on command line, use new (default)
commandstorun = ['--new']
commandstorun = [
command[2:] for command in commandspossible if command in commandstorun
] #sort commands
#***********end handling command line arguments**************************
botsinit.generalinit(
configdir) #find locating of bots, configfiles, init paths etc.
#set working directory to bots installation. advantage: when using relative paths it is clear that this point paths within bots installation.
os.chdir(botsglobal.ini.get('directories', 'botspath'))
#**************check if another instance of bots-engine is running/if port is free******************************
try:
engine_socket = botslib.check_if_other_engine_is_running()
except socket.error:
sys.exit(3)
else:
atexit.register(engine_socket.close)
#**************initialise logging******************************
process_name = 'engine'
botsglobal.logger = botsinit.initenginelogging(process_name)
atexit.register(logging.shutdown)
for key, value in botslib.botsinfo(
): #log info about environement, versions, etc
botsglobal.logger.info(u'%(key)s: "%(value)s".', {
'key': key,
'value': value
})
#**************connect to database**********************************
try:
botsinit.connect()
except Exception as msg:
botsglobal.logger.exception(
_(u'Could not connect to database. Database settings are in bots/config/settings.py. Error: "%(msg)s".'
), {'msg': msg})
sys.exit(1)
else:
botsglobal.logger.info(_(u'Connected to database.'))
atexit.register(botsglobal.db.close)
#************initialise user exits for the whole bots-engine*************************
try:
userscript, scriptname = botslib.botsimport('routescripts',
'botsengine')
except botslib.BotsImportError: #userscript is not there; other errors like syntax errors are not catched
userscript = scriptname = None
#***acceptance tests: initialiase acceptance user script******************************
acceptance_userscript = acceptance_scriptname = None
if botsglobal.ini.getboolean('acceptance', 'runacceptancetest', False):
botsglobal.logger.info(
_(u'This run is an acceptance test - as indicated in option "runacceptancetest" in bots.ini.'
))
try:
acceptance_userscript, acceptance_scriptname = botslib.botsimport(
'routescripts', 'bots_acceptancetest')
except botslib.BotsImportError:
botsglobal.logger.info(
_(u'In acceptance test there is no script file "bots_acceptancetest.py" to check the results of the acceptance test.'
))
#**************handle database lock****************************************
#set a lock on the database; if not possible, the database is locked: an earlier instance of bots-engine was terminated unexpectedly.
if not botslib.set_database_lock():
#for SQLite: do a integrity check on the database
if botsglobal.settings.DATABASES['default'][
'ENGINE'] == 'django.db.backends.sqlite3':
cursor = botsglobal.db.execute('''PRAGMA integrity_check''')
result = cursor.fetchone()
if result[0] != u'ok':
warn = _(u'!Bots database is locked!\n'\
'Bots did an integrity check on the database, but database was not OK.\n'\
'Manual action is needed!\n'\
'Bots has stopped processing EDI files.')
botsglobal.logger.critical(warn)
botslib.sendbotserrorreport(
_(u'[Bots severe error]Database is damaged'), warn)
sys.exit(1)
warn = _(u'!Bots database is locked!\n'\
'Bots-engine has ended in an unexpected way during the last run.\n'\
'Most likely causes: sudden power-down, system crash, problems with disk I/O, bots-engine terminated by user, etc.\n'
'Bots will do an automatic crash recovery now.')
botsglobal.logger.critical(warn)
botslib.sendbotserrorreport(
_(u'[Bots severe error]Database is locked'), warn)
commandstorun.insert(
0, 'crashrecovery'
) #there is a database lock. Add a crashrecovery as first command to run.
atexit.register(botslib.remove_database_lock)
warnings.simplefilter('error', UnicodeWarning)
#**************run the routes**********************************************
#commandstorun determines the type(s) of run. eg: ['automaticretrycommunication','new']
try:
botslib.prepare_confirmrules()
#in acceptance tests: run a user script before running eg to clean output directories******************************
botslib.tryrunscript(acceptance_userscript,
acceptance_scriptname,
'pretest',
routestorun=routestorun)
botslib.tryrunscript(userscript,
scriptname,
'pre',
commandstorun=commandstorun,
routestorun=routestorun)
errorinrun = 0 #detect if there has been some error. Only used for correct exit() code
first_command_2_run = True
for command in commandstorun:
#if multiple commands in run: reports etc are based on timestamp; so there needs to be at least one second between these runs.
if first_command_2_run:
first_command_2_run = False
else:
time.sleep(1)
botsglobal.logger.info(_(u'Run "%(command)s".'),
{'command': command})
#************get list of routes to run*******************************
if routestorun:
use_routestorun = routestorun[:]
botsglobal.logger.info(
_(u'Run routes from command line: "%(routes)s".'),
{'routes': unicode(use_routestorun)})
elif command == 'new': #fetch all active routes from database unless 'not in default run' or not active.
use_routestorun = []
for row in botslib.query(
'''SELECT DISTINCT idroute
FROM routes
WHERE active=%(active)s
AND (notindefaultrun=%(notindefaultrun)s OR notindefaultrun IS NULL)
ORDER BY idroute ''', {
'active': True,
'notindefaultrun': False
}):
use_routestorun.append(row['idroute'])
botsglobal.logger.info(
_(u'Run active routes from database that are in default run: "%(routes)s".'
), {'routes': unicode(use_routestorun)})
else: #for command other than 'new': use all active routes.
use_routestorun = []
for row in botslib.query(
'''SELECT DISTINCT idroute
FROM routes
WHERE active=%(active)s
ORDER BY idroute ''',
{'active': True}):
use_routestorun.append(row['idroute'])
botsglobal.logger.info(
_(u'Run all active routes from database: "%(routes)s".'),
{'routes': unicode(use_routestorun)})
#************run routes for this command******************************
botslib.tryrunscript(userscript,
scriptname,
'pre' + command,
routestorun=use_routestorun)
errorinrun += router.rundispatcher(command, use_routestorun)
botslib.tryrunscript(userscript,
scriptname,
'post' + command,
routestorun=use_routestorun)
#*********finished running routes for this command****************************
#*********finished all commands****************************************
botslib.tryrunscript(userscript,
scriptname,
'post',
commandstorun=commandstorun,
routestorun=routestorun)
try: #in acceptance tests: run a user script. no good reporting of errors/results in post-test script. Reason: this is after automaticmaintence.
botslib.tryrunscript(acceptance_userscript,
acceptance_scriptname,
'posttest',
routestorun=use_routestorun)
except Exception as msg:
print unicode(msg)
cleanup.cleanup(do_cleanup_parameter, userscript, scriptname)
except Exception as msg:
botsglobal.logger.exception(
_(u'Severe error in bots system:\n%(msg)s'),
{'msg': unicode(msg)}) #of course this 'should' not happen.
sys.exit(1)
else:
if errorinrun:
sys.exit(2) #indicate: error(s) in run(s)
else:
sys.exit(0) #OK
def start():
#exit codes:
# 0: OK, no errors
# 1: (system) errors
# 2: bots ran OK, but there are errors/process errors in the run
# 3: Database is locked, but "maxruntime" has not been exceeded.
#********command line arguments**************************
commandspossible = ['--new','--retry','--retransmit','--cleanup','--crashrecovery','--retrycommunication','--automaticretrycommunication']
commandstorun = []
routestorun = [] #list with routes to run
configdir = 'config'
for arg in sys.argv[1:]:
if not arg:
continue
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Configuration directory indicated, but no directory name.'
sys.exit(1)
elif arg in commandspossible:
commandstorun.append(arg)
elif arg in ["?", "/?"] or arg.startswith('-'):
showusage()
sys.exit(0)
else: #pick up names of routes to run
routestorun.append(arg)
if not commandstorun: #if no command on command line, use new (default)
commandstorun = ['--new']
#**************init general: find locating of bots, configfiles, init paths etc.****************
botsinit.generalinit(configdir)
#set current working directory to botspath
#~ old_current_directory = os.getcwdu()
os.chdir(botsglobal.ini.get('directories','botspath'))
#**************initialise logging******************************
try:
botsinit.initenginelogging()
except:
print _('Error in initialising logging system.')
traceback.print_exc()
sys.exit(1)
else:
atexit.register(logging.shutdown)
for key,value in botslib.botsinfo(): #log start info
botsglobal.logger.info(u'%s: "%s".',key,value)
#**************connect to database**********************************
try:
botsinit.connect()
except:
botsglobal.logger.exception(_(u'Could not connect to database. Database settings are in bots/config/settings.py.'))
sys.exit(1)
else:
botsglobal.logger.info(_(u'Connected to database.'))
atexit.register(botsglobal.db.close)
#initialise user exits for the whole bots-engine (this script file)
try:
userscript,scriptname = botslib.botsimport('routescripts','botsengine')
except ImportError:
userscript = scriptname = None
#**************handle database lock****************************************
#try to set a lock on the database; if this is not possible, the database is already locked. Either:
#1 another instance bots bots-engine is (still) running
#2 or bots-engine had a severe crash.
#What to do?
#first: check ts of database lock. If below a certain value (set in bots.ini) we assume an other instance is running. Exit quietly - no errors, no logging.
# else: Warn user, give advise on what to do. gather data: nr files in, errors.
#next: warn with report & logging. advise a crashrecovery.
if not botslib.set_database_lock():
if '--crashrecovery' in commandstorun: #user starts recovery operation; the databaselock is ignored; the databaselock is unlocked when routes have run.
commandstorun = ['--crashrecovery'] #is an exclusive option!
else:
#when scheduling bots it is possible that the last run is still running. Check if maxruntime has passed:
vanaf = datetime.datetime.today() - datetime.timedelta(minutes=botsglobal.ini.getint('settings','maxruntime',60))
for row in botslib.query('''SELECT ts FROM mutex WHERE ts < %(vanaf)s ''',{'vanaf':vanaf}):
warn = _(u'!Bots database is locked!\nBots-engine has ended in an unexpected way during the last run.\nThis happens, but is very very rare.\nPossible causes: bots-engine terminated by user, system crash, power-down, etc.\nA forced retry of the last run is advised; bots will (try to) repair the last run.')
botsglobal.logger.critical(warn)
botslib.sendbotserrorreport(_(u'[Bots severe error]Database is locked'),warn)
#add: count errors etc.
sys.exit(1)
else: #maxruntime has not passed. Exit silently, nothing reported
botsglobal.logger.info(_(u'Database is locked, but "maxruntime" has not been exceeded.'))
sys.exit(3)
else:
if '--crashrecovery' in commandstorun: #user starts recovery operation but there is no databaselock.
warn = _(u'User started a forced retry of the last run.\nOnly use this when the database is locked.\nThe database was not locked (database is OK).\nSo Bots has done nothing now.')
botsglobal.logger.error(warn)
botslib.sendbotserrorreport(_(u'[Bots Error Report] User started a forced retry of last run, but this was not needed'),warn)
botslib.remove_database_lock()
sys.exit(1)
#*************get list of routes to run****************************************
#~ raise Exception('locked database') #for testing database lock: abort, database will be locked
if routestorun:
botsglobal.logger.info(u'Run routes from command line: "%s".',str(routestorun))
else: # no routes from command line parameters: fetch all active routes from database
for row in botslib.query('''SELECT DISTINCT idroute
FROM routes
WHERE active=%(active)s
AND (notindefaultrun=%(notindefaultrun)s OR notindefaultrun IS NULL)
ORDER BY idroute ''',
{'active':True,'notindefaultrun':False}):
routestorun.append(row['idroute'])
botsglobal.logger.info(_(u'Run active routes from database: "%s".'),str(routestorun))
#routestorun is now either a list with routes from commandline, or the list of active routes for the routes table in the db.
#**************run the routes for retry, retransmit and new runs*************************************
try:
#commandstorun determines the type(s) of run
#routes to run is a listof the routes that are runs (for each command to run
#botsglobal.incommunicate is used to control if there is communication in; only 'new' incommunicates.
#botsglobal.minta4query controls which ta's are queried by the routes.
#stuff2evaluate controls what is evaluated in automatic maintenance.
errorinrun = 0 #detect if there has been some error. Only used for good exit() code
botsglobal.incommunicate = False
if '--crashrecovery' in commandstorun:
botsglobal.logger.info(_(u'Run crash recovery.'))
stuff2evaluate = botslib.set_minta4query_crashrecovery()
if stuff2evaluate:
router.routedispatcher(routestorun)
errorinrun += automaticmaintenance.evaluate('--crashrecovery',stuff2evaluate)
else:
botsglobal.logger.info(_(u'No retry of the last run - there was no last run.'))
if userscript and hasattr(userscript,'postcrashrecovery'):
botslib.runscript(userscript,scriptname,'postcrashrecovery',routestorun=routestorun)
if '--retrycommunication' in commandstorun:
botsglobal.logger.info(_(u'Run communication retry.'))
stuff2evaluate = router.routedispatcher(routestorun,'--retrycommunication')
if stuff2evaluate:
errorinrun += automaticmaintenance.evaluate('--retrycommunication',stuff2evaluate)
else:
botsglobal.logger.info(_(u'Run recommunicate: nothing to recommunicate.'))
if userscript and hasattr(userscript,'postretrycommunication'):
botslib.runscript(userscript,scriptname,'postretrycommunication',routestorun=routestorun)
if '--automaticretrycommunication' in commandstorun:
botsglobal.logger.info(_(u'Run automatic communication retry.'))
stuff2evaluate = router.routedispatcher(routestorun,'--automaticretrycommunication')
if stuff2evaluate:
errorinrun += automaticmaintenance.evaluate('--automaticretrycommunication',stuff2evaluate)
else:
botsglobal.logger.info(_(u'Run automatic recommunicate: nothing to recommunicate.'))
if userscript and hasattr(userscript,'postautomaticretrycommunication'):
botslib.runscript(userscript,scriptname,'postautomaticretrycommunication',routestorun=routestorun)
if '--retry' in commandstorun:
botsglobal.logger.info(u'Run retry.')
stuff2evaluate = router.routedispatcher(routestorun,'--retry')
if stuff2evaluate:
errorinrun += automaticmaintenance.evaluate('--retry',stuff2evaluate)
else:
botsglobal.logger.info(_(u'Run retry: nothing to retry.'))
if userscript and hasattr(userscript,'postretry'):
botslib.runscript(userscript,scriptname,'postretry',routestorun=routestorun)
if '--retransmit' in commandstorun:
botsglobal.logger.info(u'Run retransmit.')
stuff2evaluate = router.routedispatcher(routestorun,'--retransmit')
if stuff2evaluate:
errorinrun += automaticmaintenance.evaluate('--retransmit',stuff2evaluate)
else:
botsglobal.logger.info(_(u'Run retransmit: nothing to retransmit.'))
if userscript and hasattr(userscript,'postretransmit'):
botslib.runscript(userscript,scriptname,'postretransmit',routestorun=routestorun)
if '--new' in commandstorun:
botsglobal.logger.info('Run new.')
botsglobal.incommunicate = True
botsglobal.minta4query = 0 #meaning: reset. the actual value is set later (in routedispatcher)
stuff2evaluate = router.routedispatcher(routestorun)
errorinrun += automaticmaintenance.evaluate('--new',stuff2evaluate)
if userscript and hasattr(userscript,'postnewrun'):
botslib.runscript(userscript,scriptname,'postnewrun',routestorun=routestorun)
if '--cleanup' in commandstorun or botsglobal.ini.get('settings','whencleanup','always')=='always':
botsglobal.logger.debug(u'Do cleanup.')
cleanup.cleanup()
botslib.remove_database_lock()
except Exception,e:
botsglobal.logger.exception(_(u'Severe error in bots system:\n%s')%(e)) #of course this 'should' not happen.
sys.exit(1)
botsglobal.logger.info(_(u'In acceptance test there is no script file "bots_acceptancetest.py" to check the results of the acceptance test.'))
#**************handle database lock****************************************
#set a lock on the database; if not possible, the database is locked: an earlier instance of bots-engine was terminated unexpectedly.
if not botslib.set_database_lock():
#for SQLite: do a integrity check on the database
if botsglobal.settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3':
cursor = botsglobal.db.execute('''PRAGMA integrity_check''')
result = cursor.fetchone()
if result[0] != u'ok':
warn = _(u'!Bots database is locked!\n'\
'Bots did an integrity check on the database, but database was not OK.\n'\
'Manual action is needed!\n'\
'Bots has stopped processing EDI files.')
botsglobal.logger.critical(warn)
botslib.sendbotserrorreport(_(u'[Bots severe error]Database is damaged'),warn)
sys.exit(1)
warn = _(u'!Bots database is locked!\n'\
'Bots-engine has ended in an unexpected way during the last run.\n'\
'Most likely causes: sudden power-down, system crash, problems with disk I/O, bots-engine terminated by user, etc.\n'
'Bots will do an automatic crash recovery now.')
botsglobal.logger.critical(warn)
botslib.sendbotserrorreport(_(u'[Bots severe error]Database is locked'),warn)
commandstorun.insert(0,'crashrecovery') #there is a database lock. Add a crashrecovery as first command to run.
atexit.register(botslib.remove_database_lock)
#**************run the routes**********************************************
#commandstorun determines the type(s) of run. eg: ['automaticretrycommunication','new']
#for each command: run all routes
# for each route: run all seq
try:
#in acceptance tests: run a user script before running eg to clean output directories******************************
def maxruntimeerror(logger,maxruntime,jobnumber,task_to_run):
logger.error(u'Job %(job)s exceeded maxruntime of %(maxruntime)s minutes',{'job':jobnumber,'maxruntime':maxruntime})
botslib.sendbotserrorreport(u'[Bots Job Queue] - Job exceeded maximum runtime',
u'Job %(job)s exceeded maxruntime of %(maxruntime)s minutes:\n %(task)s' % {'job':jobnumber,'maxruntime':maxruntime,'task':task_to_run})
def maxruntimeerror(logger,maxruntime,jobnumber,task_to_run):
logger.error(u'Job %(job)s exceeded maxruntime of %(maxruntime)s minutes',{'job':jobnumber,'maxruntime':maxruntime})
botslib.sendbotserrorreport(u'[Bots Job Queue] - Job exceeded maximum runtime',
u'Job %(job)s exceeded maxruntime of %(maxruntime)s minutes:\n %(task)s' % {'job':jobnumber,'maxruntime':maxruntime,'task':task_to_run})