def launcher(logger,port,lauchfrequency,maxruntime): xmlrpcclient = xmlrpclib.ServerProxy('http://localhost:' + str(port)) maxseconds = maxruntime*60 time.sleep(3) #to allow jobqserver to start while True: time.sleep(lauchfrequency) job = xmlrpcclient.getjob() if job: #0 means nothing to launch jobnumber = job[1] task_to_run = job[2] # Start a timer thread for maxruntime error timer_thread = threading.Timer(maxseconds,maxruntimeerror,args=(logger,maxruntime,jobnumber,task_to_run)) timer_thread.start() try: starttime = datetime.datetime.now() logger.info(u'Starting job %(job)s',job=jobnumber) result = subprocess.call(task_to_run,stdin=open(os.devnull,'r'),stdout=open(os.devnull,'w'),stderr=open(os.devnull,'w')) time_taken = datetime.timedelta(seconds=(datetime.datetime.now() - starttime).seconds) logger.info(u'Finished job %(job)s, elapsed time %(time_taken)s, result %(result)s',job=jobnumber,time_taken=time_taken,result=result) except Exception, msg: logger.error(u'Error starting job %(job)s: %(msg)s',job=jobnumber,msg=msg) botslib.sendbotserrorreport(u'[Bots Job Queue] - Error starting job', u'Error starting job %s:\n %s\n\n %s' % (jobnumber,task_to_run,msg)) timer_thread.cancel()
def generate_report(stuff2evaluate): for results in botslib.query('''SELECT idta,lastopen,lasterror,lastok,lastdone, send,processerrors,ts,lastreceived,type,status FROM report WHERE idta=%(rootidta)s''', {'rootidta':stuff2evaluate}): break else: raise botslib.PanicError(_(u'In generate report: could not find report?')) subject = _(u'[Bots Error Report] %(time)s')%{'time':str(results['ts'])[:16]} reporttext = _(u'Bots Report; type: %(type)s, time: %(time)s\n')%{'type':results['type'],'time':str(results['ts'])[:19]} reporttext += _(u' %d files received/processed in run.\n')%(results['lastreceived']) if results['lastdone']: reporttext += _(u' %d files without errors,\n')%(results['lastdone']) if results['lasterror']: subject += _(u'; %d file errors')%(results['lasterror']) reporttext += _(u' %d files with errors,\n')%(results['lasterror']) if results['lastok']: subject += _(u'; %d files stuck')%(results['lastok']) reporttext += _(u' %d files got stuck,\n')%(results['lastok']) if results['lastopen']: subject += _(u'; %d system errors')%(results['lastopen']) reporttext += _(u' %d system errors,\n')%(results['lastopen']) if results['processerrors']: subject += _(u'; %d process errors')%(results['processerrors']) reporttext += _(u' %d errors in processes.\n')%(results['processerrors']) reporttext += _(u' %d files send in run.\n')%(results['send']) botsglobal.logger.info(reporttext) # sendreportifprocesserror allows blocking of email reports for process errors if (results['lasterror'] or results['lastopen'] or results['lastok'] or (results['processerrors'] and botsglobal.ini.getboolean('settings','sendreportifprocesserror',True))): botslib.sendbotserrorreport(subject,reporttext) return int(results['status']) #return report status: 0 (no error) or 1 (error)
def launcher(logger,port,lauchfrequency,maxruntime): xmlrpcclient = xmlrpclib.ServerProxy(u'http://localhost:' + unicode(port)) maxseconds = maxruntime*60 time.sleep(3) #to allow jobqserver to start while True: time.sleep(lauchfrequency) job = xmlrpcclient.getjob() if job: #0 means nothing to launch jobnumber = job[1] task_to_run = job[2] # Start a timer thread for maxruntime error timer_thread = threading.Timer(maxseconds,maxruntimeerror,args=(logger,maxruntime,jobnumber,task_to_run)) timer_thread.start() try: starttime = datetime.datetime.now() logger.info(u'Starting job %(job)s',{'job':jobnumber}) result = subprocess.call(task_to_run,stdin=open(os.devnull,'r'),stdout=open(os.devnull,'w'),stderr=open(os.devnull,'w')) time_taken = datetime.timedelta(seconds=(datetime.datetime.now() - starttime).seconds) logger.info(u'Finished job %(job)s, elapsed time %(time_taken)s, result %(result)s',{'job':jobnumber,'time_taken':time_taken,'result':result}) except Exception as msg: logger.error(u'Error starting job %(job)s: %(msg)s',{'job':jobnumber,'msg':msg}) botslib.sendbotserrorreport(u'[Bots Job Queue] - Error starting job', u'Error starting job %(job)s:\n %(task)s\n\n %(msg)s' % {'job':jobnumber,'task':task_to_run,'msg':msg}) timer_thread.cancel()
def email_error_report(rootidtaofrun): for results in botslib.query('''SELECT idta,lastopen,lasterror,lastok,lastdone, send,processerrors,ts,lastreceived,type,status FROM report WHERE idta=%(rootidtaofrun)s''', {'rootidtaofrun':rootidtaofrun}): break else: raise botslib.PanicError(_(u'In generate report: could not find report?')) subject = _(u'[Bots Error Report] %(time)s')%{'time':str(results['ts'])[:16]} reporttext = _(u'Bots Report; type: %(type)s, time: %(time)s\n')%{'type':results['type'],'time':str(results['ts'])[:19]} reporttext += _(u' %d files received/processed in run.\n')%(results['lastreceived']) if results['lastdone']: reporttext += _(u' %d files without errors,\n')%(results['lastdone']) if results['lasterror']: subject += _(u'; %d file errors')%(results['lasterror']) reporttext += _(u' %d files with errors,\n')%(results['lasterror']) if results['lastok']: subject += _(u'; %d files stuck')%(results['lastok']) reporttext += _(u' %d files got stuck,\n')%(results['lastok']) if results['lastopen']: subject += _(u'; %d system errors')%(results['lastopen']) reporttext += _(u' %d system errors,\n')%(results['lastopen']) if results['processerrors']: subject += _(u'; %d process errors')%(results['processerrors']) reporttext += _(u' %d errors in processes.\n')%(results['processerrors']) reporttext += _(u' %d files send in run.\n')%(results['send']) botsglobal.logger.info(reporttext) #log the report texts # only send email report if there are errors. # sendreportifprocesserror (in bots.ini): no email reports if only process errors if results['lasterror'] or results['lastopen'] or results['lastok'] or (results['processerrors'] and botsglobal.ini.getboolean('settings','sendreportifprocesserror',True)): # Include details about process errors in the email report; if debug is True: includes trace if results['processerrors']: for row in botslib.query('''SELECT idroute,fromchannel,tochannel,errortext FROM ta WHERE idta>=%(rootidtaofrun)s AND status=%(status)s AND statust=%(statust)s ''', {'rootidtaofrun':rootidtaofrun,'status':PROCESS,'statust':ERROR}): reporttext += '\nProcess error:\n' for key in row.keys(): reporttext += '%s: %s\n' % (key,row[key]) # Include details about file errors in the email report; if debug is True: includes trace if results['lasterror'] or results['lastopen'] or results['lastok']: for row in botslib.query('''SELECT idroute,frompartner,fromchannel,topartner,tochannel,errortext,infilename FROM filereport WHERE idta>%(rootidtaofrun)s AND statust!=%(statust)s ''', {'rootidtaofrun':rootidtaofrun,'statust':DONE}): reporttext += '\nFile error:\n' for key in row.keys(): reporttext += '%s: %s\n' % (key,row[key]) botslib.sendbotserrorreport(subject,reporttext) return int(results['status']) #return report status: 0 (no error) or 1 (error)
def email_error_report(rootidtaofrun): for results in botslib.query('''SELECT idta,lastopen,lasterror,lastok,lastdone, send,processerrors,ts,lastreceived,type,status FROM report WHERE idta=%(rootidtaofrun)s''', {'rootidtaofrun':rootidtaofrun}): break else: raise botslib.PanicError(_(u'In generate report: could not find report?')) subject = _(u'[Bots Error Report] %(time)s')%{'time':unicode(results['ts'])[:16]} reporttext = _(u'Bots Report; type: %(type)s, time: %(time)s\n')%{'type':results['type'],'time':unicode(results['ts'])[:19]} reporttext += _(u' %d files received/processed in run.\n')%(results['lastreceived']) if results['lastdone']: reporttext += _(u' %d files without errors,\n')%(results['lastdone']) if results['lasterror']: subject += _(u'; %d file errors')%(results['lasterror']) reporttext += _(u' %d files with errors,\n')%(results['lasterror']) if results['lastok']: subject += _(u'; %d files stuck')%(results['lastok']) reporttext += _(u' %d files got stuck,\n')%(results['lastok']) if results['lastopen']: subject += _(u'; %d system errors')%(results['lastopen']) reporttext += _(u' %d system errors,\n')%(results['lastopen']) if results['processerrors']: subject += _(u'; %d process errors')%(results['processerrors']) reporttext += _(u' %d errors in processes.\n')%(results['processerrors']) reporttext += _(u' %d files send in run.\n')%(results['send']) botsglobal.logger.info(reporttext) #log the report texts # only send email report if there are errors. # sendreportifprocesserror (in bots.ini): no email reports if only process errors if results['lasterror'] or results['lastopen'] or results['lastok'] or (results['processerrors'] and botsglobal.ini.getboolean('settings','sendreportifprocesserror',True)): # Include details about process errors in the email report; if debug is True: includes trace if results['processerrors']: for row in botslib.query('''SELECT idroute,fromchannel,tochannel,errortext FROM ta WHERE idta>=%(rootidtaofrun)s AND status=%(status)s AND statust=%(statust)s ''', {'rootidtaofrun':rootidtaofrun,'status':PROCESS,'statust':ERROR}): reporttext += '\nProcess error:\n' for key in row.keys(): reporttext += '%s: %s\n' % (key,row[key]) # Include details about file errors in the email report; if debug is True: includes trace if results['lasterror'] or results['lastopen'] or results['lastok']: for row in botslib.query('''SELECT idroute,frompartner,fromchannel,topartner,tochannel,errortext,infilename FROM filereport WHERE idta>%(rootidtaofrun)s AND statust!=%(statust)s ''', {'rootidtaofrun':rootidtaofrun,'statust':DONE}): reporttext += '\nFile error:\n' for key in row.keys(): reporttext += '%s: %s\n' % (key,row[key]) botslib.sendbotserrorreport(subject,reporttext) return int(results['status']) #return report status: 0 (no error) or 1 (error)
def generate_report(stuff2evaluate): for results in botslib.query( '''SELECT idta,lastopen,lasterror,lastok,lastdone, send,processerrors,ts,lastreceived,type,status FROM report WHERE idta=%(rootidta)s''', {'rootidta': stuff2evaluate}): break else: raise botslib.PanicError( _(u'In generate report: could not find report?')) subject = _(u'[Bots Error Report] %(time)s') % { 'time': str(results['ts'])[:16] } reporttext = _(u'Bots Report; type: %(type)s, time: %(time)s\n') % { 'type': results['type'], 'time': str(results['ts'])[:19] } reporttext += _(u' %d files received/processed in run.\n') % ( results['lastreceived']) if results['lastdone']: reporttext += _(u' %d files without errors,\n') % ( results['lastdone']) if results['lasterror']: subject += _(u'; %d file errors') % (results['lasterror']) reporttext += _(u' %d files with errors,\n') % ( results['lasterror']) if results['lastok']: subject += _(u'; %d files stuck') % (results['lastok']) reporttext += _(u' %d files got stuck,\n') % (results['lastok']) if results['lastopen']: subject += _(u'; %d system errors') % (results['lastopen']) reporttext += _(u' %d system errors,\n') % (results['lastopen']) if results['processerrors']: subject += _(u'; %d process errors') % (results['processerrors']) reporttext += _(u' %d errors in processes.\n') % ( results['processerrors']) reporttext += _(u' %d files send in run.\n') % (results['send']) botsglobal.logger.info(reporttext) # sendreportifprocesserror allows blocking of email reports for process errors if (results['lasterror'] or results['lastopen'] or results['lastok'] or (results['processerrors'] and botsglobal.ini.getboolean( 'settings', 'sendreportifprocesserror', True))): botslib.sendbotserrorreport(subject, reporttext) return int( results['status']) #return report status: 0 (no error) or 1 (error)
def generate_report(stuff2evaluate): for results in botslib.query( """SELECT idta,lastopen,lasterror,lastok,lastdone, send,processerrors,ts,lastreceived,type,status FROM report WHERE idta=%(rootidta)s""", {"rootidta": stuff2evaluate}, ): break else: raise botslib.PanicError(_(u"In generate report: could not find report?")) subject = _(u"[Bots Error Report] %(time)s") % {"time": str(results["ts"])[:16]} reporttext = _(u"Bots Report; type: %(type)s, time: %(time)s\n") % { "type": results["type"], "time": str(results["ts"])[:19], } reporttext += _(u" %d files received/processed in run.\n") % (results["lastreceived"]) if results["lastdone"]: reporttext += _(u" %d files without errors,\n") % (results["lastdone"]) if results["lasterror"]: subject += _(u"; %d file errors") % (results["lasterror"]) reporttext += _(u" %d files with errors,\n") % (results["lasterror"]) if results["lastok"]: subject += _(u"; %d files stuck") % (results["lastok"]) reporttext += _(u" %d files got stuck,\n") % (results["lastok"]) if results["lastopen"]: subject += _(u"; %d system errors") % (results["lastopen"]) reporttext += _(u" %d system errors,\n") % (results["lastopen"]) if results["processerrors"]: subject += _(u"; %d process errors") % (results["processerrors"]) reporttext += _(u" %d errors in processes.\n") % (results["processerrors"]) reporttext += _(u" %d files send in run.\n") % (results["send"]) botsglobal.logger.info(reporttext) # sendreportifprocesserror allows blocking of email reports for process errors if ( results["lasterror"] or results["lastopen"] or results["lastok"] or (results["processerrors"] and botsglobal.ini.getboolean("settings", "sendreportifprocesserror", True)) ): botslib.sendbotserrorreport(subject, reporttext) return int(results["status"]) # return report status: 0 (no error) or 1 (error)
def start(): #exit codes: # 0: OK, no errors # 1: (system) errors # 2: bots ran OK, but there are errors/process errors in the run # 3: Database is locked, but "maxruntime" has not been exceeded. #********command line arguments************************** commandspossible = [ '--new', '--retry', '--retransmit', '--cleanup', '--crashrecovery', '--retrycommunication', '--automaticretrycommunication' ] commandstorun = [] routestorun = [] #list with routes to run configdir = 'config' for arg in sys.argv[1:]: if not arg: continue if arg.startswith('-c'): configdir = arg[2:] if not configdir: print 'Configuration directory indicated, but no directory name.' sys.exit(1) elif arg in commandspossible: commandstorun.append(arg) elif arg in ["?", "/?"] or arg.startswith('-'): showusage() sys.exit(0) else: #pick up names of routes to run routestorun.append(arg) if not commandstorun: #if no command on command line, use new (default) commandstorun = ['--new'] #**************init general: find locating of bots, configfiles, init paths etc.**************** botsinit.generalinit(configdir) #set current working directory to botspath #~ old_current_directory = os.getcwdu() os.chdir(botsglobal.ini.get('directories', 'botspath')) #**************initialise logging****************************** try: botsinit.initenginelogging() except: print _('Error in initialising logging system.') traceback.print_exc() sys.exit(1) else: atexit.register(logging.shutdown) for key, value in botslib.botsinfo(): #log start info botsglobal.logger.info(u'%s: "%s".', key, value) #**************connect to database********************************** try: botsinit.connect() except: botsglobal.logger.exception( _(u'Could not connect to database. Database settings are in bots/config/settings.py.' )) sys.exit(1) else: botsglobal.logger.info(_(u'Connected to database.')) atexit.register(botsglobal.db.close) #initialise user exits for the whole bots-engine (this script file) try: userscript, scriptname = botslib.botsimport('routescripts', 'botsengine') except ImportError: userscript = scriptname = None #**************handle database lock**************************************** #try to set a lock on the database; if this is not possible, the database is already locked. Either: #1 another instance bots bots-engine is (still) running #2 or bots-engine had a severe crash. #What to do? #first: check ts of database lock. If below a certain value (set in bots.ini) we assume an other instance is running. Exit quietly - no errors, no logging. # else: Warn user, give advise on what to do. gather data: nr files in, errors. #next: warn with report & logging. advise a crashrecovery. if not botslib.set_database_lock(): if '--crashrecovery' in commandstorun: #user starts recovery operation; the databaselock is ignored; the databaselock is unlocked when routes have run. commandstorun = ['--crashrecovery'] #is an exclusive option! else: #when scheduling bots it is possible that the last run is still running. Check if maxruntime has passed: vanaf = datetime.datetime.today() - datetime.timedelta( minutes=botsglobal.ini.getint('settings', 'maxruntime', 60)) for row in botslib.query( '''SELECT ts FROM mutex WHERE ts < %(vanaf)s ''', {'vanaf': vanaf}): warn = _( u'!Bots database is locked!\nBots-engine has ended in an unexpected way during the last run.\nThis happens, but is very very rare.\nPossible causes: bots-engine terminated by user, system crash, power-down, etc.\nA forced retry of the last run is advised; bots will (try to) repair the last run.' ) botsglobal.logger.critical(warn) botslib.sendbotserrorreport( _(u'[Bots severe error]Database is locked'), warn) #add: count errors etc. sys.exit(1) else: #maxruntime has not passed. Exit silently, nothing reported botsglobal.logger.info( _(u'Database is locked, but "maxruntime" has not been exceeded.' )) sys.exit(3) else: if '--crashrecovery' in commandstorun: #user starts recovery operation but there is no databaselock. warn = _( u'User started a forced retry of the last run.\nOnly use this when the database is locked.\nThe database was not locked (database is OK).\nSo Bots has done nothing now.' ) botsglobal.logger.error(warn) botslib.sendbotserrorreport( _(u'[Bots Error Report] User started a forced retry of last run, but this was not needed' ), warn) botslib.remove_database_lock() sys.exit(1) #*************get list of routes to run**************************************** #~ raise Exception('locked database') #for testing database lock: abort, database will be locked if routestorun: botsglobal.logger.info(u'Run routes from command line: "%s".', str(routestorun)) else: # no routes from command line parameters: fetch all active routes from database for row in botslib.query( '''SELECT DISTINCT idroute FROM routes WHERE active=%(active)s AND (notindefaultrun=%(notindefaultrun)s OR notindefaultrun IS NULL) ORDER BY idroute ''', { 'active': True, 'notindefaultrun': False }): routestorun.append(row['idroute']) botsglobal.logger.info(_(u'Run active routes from database: "%s".'), str(routestorun)) #routestorun is now either a list with routes from commandline, or the list of active routes for the routes table in the db. #**************run the routes for retry, retransmit and new runs************************************* try: #commandstorun determines the type(s) of run #routes to run is a listof the routes that are runs (for each command to run #botsglobal.incommunicate is used to control if there is communication in; only 'new' incommunicates. #botsglobal.minta4query controls which ta's are queried by the routes. #stuff2evaluate controls what is evaluated in automatic maintenance. errorinrun = 0 #detect if there has been some error. Only used for good exit() code botsglobal.incommunicate = False if '--crashrecovery' in commandstorun: botsglobal.logger.info(_(u'Run crash recovery.')) stuff2evaluate = botslib.set_minta4query_crashrecovery() if stuff2evaluate: router.routedispatcher(routestorun) errorinrun += automaticmaintenance.evaluate( '--crashrecovery', stuff2evaluate) else: botsglobal.logger.info( _(u'No retry of the last run - there was no last run.')) if userscript and hasattr(userscript, 'postcrashrecovery'): botslib.runscript(userscript, scriptname, 'postcrashrecovery', routestorun=routestorun) if '--retrycommunication' in commandstorun: botsglobal.logger.info(_(u'Run communication retry.')) stuff2evaluate = router.routedispatcher(routestorun, '--retrycommunication') if stuff2evaluate: errorinrun += automaticmaintenance.evaluate( '--retrycommunication', stuff2evaluate) else: botsglobal.logger.info( _(u'Run recommunicate: nothing to recommunicate.')) if userscript and hasattr(userscript, 'postretrycommunication'): botslib.runscript(userscript, scriptname, 'postretrycommunication', routestorun=routestorun) if '--automaticretrycommunication' in commandstorun: botsglobal.logger.info(_(u'Run automatic communication retry.')) stuff2evaluate = router.routedispatcher( routestorun, '--automaticretrycommunication') if stuff2evaluate: errorinrun += automaticmaintenance.evaluate( '--automaticretrycommunication', stuff2evaluate) else: botsglobal.logger.info( _(u'Run automatic recommunicate: nothing to recommunicate.' )) if userscript and hasattr(userscript, 'postautomaticretrycommunication'): botslib.runscript(userscript, scriptname, 'postautomaticretrycommunication', routestorun=routestorun) if '--retry' in commandstorun: botsglobal.logger.info(u'Run retry.') stuff2evaluate = router.routedispatcher(routestorun, '--retry') if stuff2evaluate: errorinrun += automaticmaintenance.evaluate( '--retry', stuff2evaluate) else: botsglobal.logger.info(_(u'Run retry: nothing to retry.')) if userscript and hasattr(userscript, 'postretry'): botslib.runscript(userscript, scriptname, 'postretry', routestorun=routestorun) if '--retransmit' in commandstorun: botsglobal.logger.info(u'Run retransmit.') stuff2evaluate = router.routedispatcher(routestorun, '--retransmit') if stuff2evaluate: errorinrun += automaticmaintenance.evaluate( '--retransmit', stuff2evaluate) else: botsglobal.logger.info( _(u'Run retransmit: nothing to retransmit.')) if userscript and hasattr(userscript, 'postretransmit'): botslib.runscript(userscript, scriptname, 'postretransmit', routestorun=routestorun) if '--new' in commandstorun: botsglobal.logger.info('Run new.') botsglobal.incommunicate = True botsglobal.minta4query = 0 #meaning: reset. the actual value is set later (in routedispatcher) stuff2evaluate = router.routedispatcher(routestorun) errorinrun += automaticmaintenance.evaluate( '--new', stuff2evaluate) if userscript and hasattr(userscript, 'postnewrun'): botslib.runscript(userscript, scriptname, 'postnewrun', routestorun=routestorun) if '--cleanup' in commandstorun or botsglobal.ini.get( 'settings', 'whencleanup', 'always') == 'always': botsglobal.logger.debug(u'Do cleanup.') cleanup.cleanup() botslib.remove_database_lock() except Exception, e: botsglobal.logger.exception( _(u'Severe error in bots system:\n%s') % (e)) #of course this 'should' not happen. sys.exit(1)
def maxruntimeerror(logger,maxruntime,jobnumber,task_to_run): logger.error(u'Job %(job)s exceeded maxruntime of %(maxruntime)s minutes',job=jobnumber,maxruntime=maxruntime) botslib.sendbotserrorreport(u'[Bots Job Queue] - Job exceeded maximum runtime', u'Job %s exceeded maxruntime of %s minutes:\n %s' % (jobnumber,maxruntime,task_to_run))
def start(): ''' sysexit codes: 0: OK, no errors 1: (system) errors incl parsing of command line arguments 2: bots ran OK, but there are errors/process errors in the run 3: Database is locked, but "maxruntime" has not been exceeded. ''' #NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start. #********command line arguments************************** usage = ''' This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net). Does the actual translations and communications; it's the workhorse. It does not have a fancy interface. Usage: %(name)s [run-options] [config-option] [routes] Run-options (can be combined): --new receive new edi files (default: if no run-option given: run as new). --resend resend as indicated by user. --rereceive rereceive as indicated by user. --automaticretrycommunication - automatically retry outgoing communication. --cleanup remove older data from database. Config-option: -c<directory> directory for configuration files (default: config). Routes: list of routes to run. Default: all active routes (in the database) '''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version} configdir = 'config' commandspossible = ['--automaticretrycommunication','--resend','--rereceive','--new'] commandstorun = [] routestorun = [] #list with routes to run do_cleanup_parameter = False for arg in sys.argv[1:]: if arg.startswith('-c'): configdir = arg[2:] if not configdir: print 'Error: configuration directory indicated, but no directory name.' sys.exit(1) elif arg in commandspossible: commandstorun.append(arg) elif arg == '--cleanup': do_cleanup_parameter = True elif arg in ["?", "/?",'-h', '--help'] or arg.startswith('-'): print usage sys.exit(0) else: #pick up names of routes to run routestorun.append(arg) if not commandstorun and not do_cleanup_parameter: #if no command on command line, use new (default) commandstorun = ['--new'] commandstorun = [command[2:] for command in commandspossible if command in commandstorun] #sort commands #***********end handling command line arguments************************** botsinit.generalinit(configdir) #find locating of bots, configfiles, init paths etc. #set working directory to bots installation. advantage: when using relative paths it is clear that this point paths within bots installation. os.chdir(botsglobal.ini.get('directories','botspath')) #**************check if another instance of bots-engine is running/if port is free****************************** try: engine_socket = botslib.check_if_other_engine_is_running() except socket.error: sys.exit(3) else: atexit.register(engine_socket.close) #**************initialise logging****************************** process_name = 'engine' botsglobal.logger = botsinit.initenginelogging(process_name) atexit.register(logging.shutdown) for key,value in botslib.botsinfo(): #log info about environement, versions, etc botsglobal.logger.info(u'%(key)s: "%(value)s".',{'key':key,'value':value}) #**************connect to database********************************** try: botsinit.connect() except Exception as msg: botsglobal.logger.exception(_(u'Could not connect to database. Database settings are in bots/config/settings.py. Error: "%(msg)s".'),{'msg':msg}) sys.exit(1) else: botsglobal.logger.info(_(u'Connected to database.')) atexit.register(botsglobal.db.close) #************initialise user exits for the whole bots-engine************************* try: userscript,scriptname = botslib.botsimport('routescripts','botsengine') except ImportError: #userscript is not there; other errors like syntax errors are not catched userscript = scriptname = None #***acceptance tests: initialiase acceptance user script****************************** acceptance_userscript = acceptance_scriptname = None if botsglobal.ini.getboolean('acceptance','runacceptancetest',False): botsglobal.logger.info(_(u'This run is an acceptance test - as indicated in option "runacceptancetest" in bots.ini.')) try: acceptance_userscript,acceptance_scriptname = botslib.botsimport('routescripts','bots_acceptancetest') except ImportError: botsglobal.logger.info(_(u'In acceptance test there is no script file "bots_acceptancetest.py" to check the results of the acceptance test.')) #**************handle database lock**************************************** #set a lock on the database; if not possible, the database is locked: an earlier instance of bots-engine was terminated unexpectedly. if not botslib.set_database_lock(): #for SQLite: do a integrity check on the database if botsglobal.settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3': cursor = botsglobal.db.execute('''PRAGMA integrity_check''') result = cursor.fetchone() if result[0] != u'ok': warn = _(u'!Bots database is locked!\n'\ 'Bots did an integrity check on the database, but database was not OK.\n'\ 'Manual action is needed!\n'\ 'Bots has stopped processing EDI files.') botsglobal.logger.critical(warn) botslib.sendbotserrorreport(_(u'[Bots severe error]Database is damaged'),warn) sys.exit(1) warn = _(u'!Bots database is locked!\n'\ 'Bots-engine has ended in an unexpected way during the last run.\n'\ 'Most likely causes: sudden power-down, system crash, problems with disk I/O, bots-engine terminated by user, etc.\n' 'Bots will do an automatic crash recovery now.') botsglobal.logger.critical(warn) botslib.sendbotserrorreport(_(u'[Bots severe error]Database is locked'),warn) commandstorun.insert(0,'crashrecovery') #there is a database lock. Add a crashrecovery as first command to run. atexit.register(botslib.remove_database_lock) #**************run the routes********************************************** #commandstorun determines the type(s) of run. eg: ['automaticretrycommunication','new'] try: botslib.prepare_confirmrules() #in acceptance tests: run a user script before running eg to clean output directories****************************** botslib.tryrunscript(acceptance_userscript,acceptance_scriptname,'pretest',routestorun=routestorun) botslib.tryrunscript(userscript,scriptname,'pre',commandstorun=commandstorun,routestorun=routestorun) errorinrun = 0 #detect if there has been some error. Only used for correct exit() code first_command_2_run = True for command in commandstorun: #if multiple commands in run: reports etc are based on timestamp; so there needs to be at least one second between these runs. if first_command_2_run: first_command_2_run = False else: time.sleep(1) botsglobal.logger.info(_(u'Run "%(command)s".'),{'command':command}) #************get list of routes to run******************************* if routestorun: use_routestorun = routestorun[:] botsglobal.logger.info(_(u'Run routes from command line: "%(routes)s".'),{'routes':str(use_routestorun)}) elif command == 'new': #fetch all active routes from database unless 'not in default run' or not active. use_routestorun = [] for row in botslib.query('''SELECT DISTINCT idroute FROM routes WHERE active=%(active)s AND (notindefaultrun=%(notindefaultrun)s OR notindefaultrun IS NULL) ORDER BY idroute ''', {'active':True,'notindefaultrun':False}): use_routestorun.append(row['idroute']) botsglobal.logger.info(_(u'Run active routes from database that are in default run: "%(routes)s".'),{'routes':str(use_routestorun)}) else: #for command other than 'new': use all active routes. use_routestorun = [] for row in botslib.query('''SELECT DISTINCT idroute FROM routes WHERE active=%(active)s ORDER BY idroute ''', {'active':True}): use_routestorun.append(row['idroute']) botsglobal.logger.info(_(u'Run all active routes from database: "%(routes)s".'),{'routes':str(use_routestorun)}) #************run routes for this command****************************** botslib.tryrunscript(userscript,scriptname,'pre' + command,routestorun=use_routestorun) errorinrun += router.rundispatcher(command,use_routestorun) botslib.tryrunscript(userscript,scriptname,'post' + command,routestorun=use_routestorun) #*********finished running routes for this command**************************** #*********finished all commands**************************************** botslib.tryrunscript(userscript,scriptname,'post',commandstorun=commandstorun,routestorun=routestorun) try: #in acceptance tests: run a user script. no good reporting of errors/results in post-test script. Reason: this is after automaticmaintence. botslib.tryrunscript(acceptance_userscript,acceptance_scriptname,'posttest',routestorun=use_routestorun) except Exception as msg: print str(msg) cleanup.cleanup(do_cleanup_parameter,userscript,scriptname) except Exception as msg: botsglobal.logger.exception(_(u'Severe error in bots system:\n%(msg)s'),{'msg':str(msg)}) #of course this 'should' not happen. sys.exit(1) else: if errorinrun: sys.exit(2) #indicate: error(s) in run(s) else: sys.exit(0) #OK
def start(): ''' sysexit codes: 0: OK, no errors 1: (system) errors incl parsing of command line arguments 2: bots ran OK, but there are errors/process errors in the run 3: Database is locked, but "maxruntime" has not been exceeded. ''' #NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start. #********command line arguments************************** usage = ''' This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net). Does the actual translations and communications; it's the workhorse. It does not have a fancy interface. Usage: %(name)s [run-options] [config-option] [routes] Run-options (can be combined): --new receive new edi files (default: if no run-option given: run as new). --resend resend as indicated by user. --rereceive rereceive as indicated by user. --automaticretrycommunication - automatically retry outgoing communication. --cleanup remove older data from database. Config-option: -c<directory> directory for configuration files (default: config). Routes: list of routes to run. Default: all active routes (in the database) ''' % { 'name': os.path.basename(sys.argv[0]), 'version': botsglobal.version } configdir = 'config' commandspossible = [ '--automaticretrycommunication', '--resend', '--rereceive', '--new' ] commandstorun = [] routestorun = [] #list with routes to run do_cleanup_parameter = False for arg in sys.argv[1:]: if arg.startswith('-c'): configdir = arg[2:] if not configdir: print 'Error: configuration directory indicated, but no directory name.' sys.exit(1) elif arg in commandspossible: commandstorun.append(arg) elif arg == '--cleanup': do_cleanup_parameter = True elif arg in ["?", "/?", '-h', '--help'] or arg.startswith('-'): print usage sys.exit(0) else: #pick up names of routes to run routestorun.append(arg) if not commandstorun and not do_cleanup_parameter: #if no command on command line, use new (default) commandstorun = ['--new'] commandstorun = [ command[2:] for command in commandspossible if command in commandstorun ] #sort commands #***********end handling command line arguments************************** botsinit.generalinit( configdir) #find locating of bots, configfiles, init paths etc. #set working directory to bots installation. advantage: when using relative paths it is clear that this point paths within bots installation. os.chdir(botsglobal.ini.get('directories', 'botspath')) #**************check if another instance of bots-engine is running/if port is free****************************** try: engine_socket = botslib.check_if_other_engine_is_running() except socket.error: sys.exit(3) else: atexit.register(engine_socket.close) #**************initialise logging****************************** process_name = 'engine' botsglobal.logger = botsinit.initenginelogging(process_name) atexit.register(logging.shutdown) for key, value in botslib.botsinfo( ): #log info about environement, versions, etc botsglobal.logger.info(u'%(key)s: "%(value)s".', { 'key': key, 'value': value }) #**************connect to database********************************** try: botsinit.connect() except Exception as msg: botsglobal.logger.exception( _(u'Could not connect to database. Database settings are in bots/config/settings.py. Error: "%(msg)s".' ), {'msg': msg}) sys.exit(1) else: botsglobal.logger.info(_(u'Connected to database.')) atexit.register(botsglobal.db.close) #************initialise user exits for the whole bots-engine************************* try: userscript, scriptname = botslib.botsimport('routescripts', 'botsengine') except botslib.BotsImportError: #userscript is not there; other errors like syntax errors are not catched userscript = scriptname = None #***acceptance tests: initialiase acceptance user script****************************** acceptance_userscript = acceptance_scriptname = None if botsglobal.ini.getboolean('acceptance', 'runacceptancetest', False): botsglobal.logger.info( _(u'This run is an acceptance test - as indicated in option "runacceptancetest" in bots.ini.' )) try: acceptance_userscript, acceptance_scriptname = botslib.botsimport( 'routescripts', 'bots_acceptancetest') except botslib.BotsImportError: botsglobal.logger.info( _(u'In acceptance test there is no script file "bots_acceptancetest.py" to check the results of the acceptance test.' )) #**************handle database lock**************************************** #set a lock on the database; if not possible, the database is locked: an earlier instance of bots-engine was terminated unexpectedly. if not botslib.set_database_lock(): #for SQLite: do a integrity check on the database if botsglobal.settings.DATABASES['default'][ 'ENGINE'] == 'django.db.backends.sqlite3': cursor = botsglobal.db.execute('''PRAGMA integrity_check''') result = cursor.fetchone() if result[0] != u'ok': warn = _(u'!Bots database is locked!\n'\ 'Bots did an integrity check on the database, but database was not OK.\n'\ 'Manual action is needed!\n'\ 'Bots has stopped processing EDI files.') botsglobal.logger.critical(warn) botslib.sendbotserrorreport( _(u'[Bots severe error]Database is damaged'), warn) sys.exit(1) warn = _(u'!Bots database is locked!\n'\ 'Bots-engine has ended in an unexpected way during the last run.\n'\ 'Most likely causes: sudden power-down, system crash, problems with disk I/O, bots-engine terminated by user, etc.\n' 'Bots will do an automatic crash recovery now.') botsglobal.logger.critical(warn) botslib.sendbotserrorreport( _(u'[Bots severe error]Database is locked'), warn) commandstorun.insert( 0, 'crashrecovery' ) #there is a database lock. Add a crashrecovery as first command to run. atexit.register(botslib.remove_database_lock) warnings.simplefilter('error', UnicodeWarning) #**************run the routes********************************************** #commandstorun determines the type(s) of run. eg: ['automaticretrycommunication','new'] try: botslib.prepare_confirmrules() #in acceptance tests: run a user script before running eg to clean output directories****************************** botslib.tryrunscript(acceptance_userscript, acceptance_scriptname, 'pretest', routestorun=routestorun) botslib.tryrunscript(userscript, scriptname, 'pre', commandstorun=commandstorun, routestorun=routestorun) errorinrun = 0 #detect if there has been some error. Only used for correct exit() code first_command_2_run = True for command in commandstorun: #if multiple commands in run: reports etc are based on timestamp; so there needs to be at least one second between these runs. if first_command_2_run: first_command_2_run = False else: time.sleep(1) botsglobal.logger.info(_(u'Run "%(command)s".'), {'command': command}) #************get list of routes to run******************************* if routestorun: use_routestorun = routestorun[:] botsglobal.logger.info( _(u'Run routes from command line: "%(routes)s".'), {'routes': unicode(use_routestorun)}) elif command == 'new': #fetch all active routes from database unless 'not in default run' or not active. use_routestorun = [] for row in botslib.query( '''SELECT DISTINCT idroute FROM routes WHERE active=%(active)s AND (notindefaultrun=%(notindefaultrun)s OR notindefaultrun IS NULL) ORDER BY idroute ''', { 'active': True, 'notindefaultrun': False }): use_routestorun.append(row['idroute']) botsglobal.logger.info( _(u'Run active routes from database that are in default run: "%(routes)s".' ), {'routes': unicode(use_routestorun)}) else: #for command other than 'new': use all active routes. use_routestorun = [] for row in botslib.query( '''SELECT DISTINCT idroute FROM routes WHERE active=%(active)s ORDER BY idroute ''', {'active': True}): use_routestorun.append(row['idroute']) botsglobal.logger.info( _(u'Run all active routes from database: "%(routes)s".'), {'routes': unicode(use_routestorun)}) #************run routes for this command****************************** botslib.tryrunscript(userscript, scriptname, 'pre' + command, routestorun=use_routestorun) errorinrun += router.rundispatcher(command, use_routestorun) botslib.tryrunscript(userscript, scriptname, 'post' + command, routestorun=use_routestorun) #*********finished running routes for this command**************************** #*********finished all commands**************************************** botslib.tryrunscript(userscript, scriptname, 'post', commandstorun=commandstorun, routestorun=routestorun) try: #in acceptance tests: run a user script. no good reporting of errors/results in post-test script. Reason: this is after automaticmaintence. botslib.tryrunscript(acceptance_userscript, acceptance_scriptname, 'posttest', routestorun=use_routestorun) except Exception as msg: print unicode(msg) cleanup.cleanup(do_cleanup_parameter, userscript, scriptname) except Exception as msg: botsglobal.logger.exception( _(u'Severe error in bots system:\n%(msg)s'), {'msg': unicode(msg)}) #of course this 'should' not happen. sys.exit(1) else: if errorinrun: sys.exit(2) #indicate: error(s) in run(s) else: sys.exit(0) #OK
def start(): #exit codes: # 0: OK, no errors # 1: (system) errors # 2: bots ran OK, but there are errors/process errors in the run # 3: Database is locked, but "maxruntime" has not been exceeded. #********command line arguments************************** commandspossible = ['--new','--retry','--retransmit','--cleanup','--crashrecovery','--retrycommunication','--automaticretrycommunication'] commandstorun = [] routestorun = [] #list with routes to run configdir = 'config' for arg in sys.argv[1:]: if not arg: continue if arg.startswith('-c'): configdir = arg[2:] if not configdir: print 'Configuration directory indicated, but no directory name.' sys.exit(1) elif arg in commandspossible: commandstorun.append(arg) elif arg in ["?", "/?"] or arg.startswith('-'): showusage() sys.exit(0) else: #pick up names of routes to run routestorun.append(arg) if not commandstorun: #if no command on command line, use new (default) commandstorun = ['--new'] #**************init general: find locating of bots, configfiles, init paths etc.**************** botsinit.generalinit(configdir) #set current working directory to botspath #~ old_current_directory = os.getcwdu() os.chdir(botsglobal.ini.get('directories','botspath')) #**************initialise logging****************************** try: botsinit.initenginelogging() except: print _('Error in initialising logging system.') traceback.print_exc() sys.exit(1) else: atexit.register(logging.shutdown) for key,value in botslib.botsinfo(): #log start info botsglobal.logger.info(u'%s: "%s".',key,value) #**************connect to database********************************** try: botsinit.connect() except: botsglobal.logger.exception(_(u'Could not connect to database. Database settings are in bots/config/settings.py.')) sys.exit(1) else: botsglobal.logger.info(_(u'Connected to database.')) atexit.register(botsglobal.db.close) #initialise user exits for the whole bots-engine (this script file) try: userscript,scriptname = botslib.botsimport('routescripts','botsengine') except ImportError: userscript = scriptname = None #**************handle database lock**************************************** #try to set a lock on the database; if this is not possible, the database is already locked. Either: #1 another instance bots bots-engine is (still) running #2 or bots-engine had a severe crash. #What to do? #first: check ts of database lock. If below a certain value (set in bots.ini) we assume an other instance is running. Exit quietly - no errors, no logging. # else: Warn user, give advise on what to do. gather data: nr files in, errors. #next: warn with report & logging. advise a crashrecovery. if not botslib.set_database_lock(): if '--crashrecovery' in commandstorun: #user starts recovery operation; the databaselock is ignored; the databaselock is unlocked when routes have run. commandstorun = ['--crashrecovery'] #is an exclusive option! else: #when scheduling bots it is possible that the last run is still running. Check if maxruntime has passed: vanaf = datetime.datetime.today() - datetime.timedelta(minutes=botsglobal.ini.getint('settings','maxruntime',60)) for row in botslib.query('''SELECT ts FROM mutex WHERE ts < %(vanaf)s ''',{'vanaf':vanaf}): warn = _(u'!Bots database is locked!\nBots-engine has ended in an unexpected way during the last run.\nThis happens, but is very very rare.\nPossible causes: bots-engine terminated by user, system crash, power-down, etc.\nA forced retry of the last run is advised; bots will (try to) repair the last run.') botsglobal.logger.critical(warn) botslib.sendbotserrorreport(_(u'[Bots severe error]Database is locked'),warn) #add: count errors etc. sys.exit(1) else: #maxruntime has not passed. Exit silently, nothing reported botsglobal.logger.info(_(u'Database is locked, but "maxruntime" has not been exceeded.')) sys.exit(3) else: if '--crashrecovery' in commandstorun: #user starts recovery operation but there is no databaselock. warn = _(u'User started a forced retry of the last run.\nOnly use this when the database is locked.\nThe database was not locked (database is OK).\nSo Bots has done nothing now.') botsglobal.logger.error(warn) botslib.sendbotserrorreport(_(u'[Bots Error Report] User started a forced retry of last run, but this was not needed'),warn) botslib.remove_database_lock() sys.exit(1) #*************get list of routes to run**************************************** #~ raise Exception('locked database') #for testing database lock: abort, database will be locked if routestorun: botsglobal.logger.info(u'Run routes from command line: "%s".',str(routestorun)) else: # no routes from command line parameters: fetch all active routes from database for row in botslib.query('''SELECT DISTINCT idroute FROM routes WHERE active=%(active)s AND (notindefaultrun=%(notindefaultrun)s OR notindefaultrun IS NULL) ORDER BY idroute ''', {'active':True,'notindefaultrun':False}): routestorun.append(row['idroute']) botsglobal.logger.info(_(u'Run active routes from database: "%s".'),str(routestorun)) #routestorun is now either a list with routes from commandline, or the list of active routes for the routes table in the db. #**************run the routes for retry, retransmit and new runs************************************* try: #commandstorun determines the type(s) of run #routes to run is a listof the routes that are runs (for each command to run #botsglobal.incommunicate is used to control if there is communication in; only 'new' incommunicates. #botsglobal.minta4query controls which ta's are queried by the routes. #stuff2evaluate controls what is evaluated in automatic maintenance. errorinrun = 0 #detect if there has been some error. Only used for good exit() code botsglobal.incommunicate = False if '--crashrecovery' in commandstorun: botsglobal.logger.info(_(u'Run crash recovery.')) stuff2evaluate = botslib.set_minta4query_crashrecovery() if stuff2evaluate: router.routedispatcher(routestorun) errorinrun += automaticmaintenance.evaluate('--crashrecovery',stuff2evaluate) else: botsglobal.logger.info(_(u'No retry of the last run - there was no last run.')) if userscript and hasattr(userscript,'postcrashrecovery'): botslib.runscript(userscript,scriptname,'postcrashrecovery',routestorun=routestorun) if '--retrycommunication' in commandstorun: botsglobal.logger.info(_(u'Run communication retry.')) stuff2evaluate = router.routedispatcher(routestorun,'--retrycommunication') if stuff2evaluate: errorinrun += automaticmaintenance.evaluate('--retrycommunication',stuff2evaluate) else: botsglobal.logger.info(_(u'Run recommunicate: nothing to recommunicate.')) if userscript and hasattr(userscript,'postretrycommunication'): botslib.runscript(userscript,scriptname,'postretrycommunication',routestorun=routestorun) if '--automaticretrycommunication' in commandstorun: botsglobal.logger.info(_(u'Run automatic communication retry.')) stuff2evaluate = router.routedispatcher(routestorun,'--automaticretrycommunication') if stuff2evaluate: errorinrun += automaticmaintenance.evaluate('--automaticretrycommunication',stuff2evaluate) else: botsglobal.logger.info(_(u'Run automatic recommunicate: nothing to recommunicate.')) if userscript and hasattr(userscript,'postautomaticretrycommunication'): botslib.runscript(userscript,scriptname,'postautomaticretrycommunication',routestorun=routestorun) if '--retry' in commandstorun: botsglobal.logger.info(u'Run retry.') stuff2evaluate = router.routedispatcher(routestorun,'--retry') if stuff2evaluate: errorinrun += automaticmaintenance.evaluate('--retry',stuff2evaluate) else: botsglobal.logger.info(_(u'Run retry: nothing to retry.')) if userscript and hasattr(userscript,'postretry'): botslib.runscript(userscript,scriptname,'postretry',routestorun=routestorun) if '--retransmit' in commandstorun: botsglobal.logger.info(u'Run retransmit.') stuff2evaluate = router.routedispatcher(routestorun,'--retransmit') if stuff2evaluate: errorinrun += automaticmaintenance.evaluate('--retransmit',stuff2evaluate) else: botsglobal.logger.info(_(u'Run retransmit: nothing to retransmit.')) if userscript and hasattr(userscript,'postretransmit'): botslib.runscript(userscript,scriptname,'postretransmit',routestorun=routestorun) if '--new' in commandstorun: botsglobal.logger.info('Run new.') botsglobal.incommunicate = True botsglobal.minta4query = 0 #meaning: reset. the actual value is set later (in routedispatcher) stuff2evaluate = router.routedispatcher(routestorun) errorinrun += automaticmaintenance.evaluate('--new',stuff2evaluate) if userscript and hasattr(userscript,'postnewrun'): botslib.runscript(userscript,scriptname,'postnewrun',routestorun=routestorun) if '--cleanup' in commandstorun or botsglobal.ini.get('settings','whencleanup','always')=='always': botsglobal.logger.debug(u'Do cleanup.') cleanup.cleanup() botslib.remove_database_lock() except Exception,e: botsglobal.logger.exception(_(u'Severe error in bots system:\n%s')%(e)) #of course this 'should' not happen. sys.exit(1)
botsglobal.logger.info(_(u'In acceptance test there is no script file "bots_acceptancetest.py" to check the results of the acceptance test.')) #**************handle database lock**************************************** #set a lock on the database; if not possible, the database is locked: an earlier instance of bots-engine was terminated unexpectedly. if not botslib.set_database_lock(): #for SQLite: do a integrity check on the database if botsglobal.settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3': cursor = botsglobal.db.execute('''PRAGMA integrity_check''') result = cursor.fetchone() if result[0] != u'ok': warn = _(u'!Bots database is locked!\n'\ 'Bots did an integrity check on the database, but database was not OK.\n'\ 'Manual action is needed!\n'\ 'Bots has stopped processing EDI files.') botsglobal.logger.critical(warn) botslib.sendbotserrorreport(_(u'[Bots severe error]Database is damaged'),warn) sys.exit(1) warn = _(u'!Bots database is locked!\n'\ 'Bots-engine has ended in an unexpected way during the last run.\n'\ 'Most likely causes: sudden power-down, system crash, problems with disk I/O, bots-engine terminated by user, etc.\n' 'Bots will do an automatic crash recovery now.') botsglobal.logger.critical(warn) botslib.sendbotserrorreport(_(u'[Bots severe error]Database is locked'),warn) commandstorun.insert(0,'crashrecovery') #there is a database lock. Add a crashrecovery as first command to run. atexit.register(botslib.remove_database_lock) #**************run the routes********************************************** #commandstorun determines the type(s) of run. eg: ['automaticretrycommunication','new'] #for each command: run all routes # for each route: run all seq try: #in acceptance tests: run a user script before running eg to clean output directories******************************
def maxruntimeerror(logger,maxruntime,jobnumber,task_to_run): logger.error(u'Job %(job)s exceeded maxruntime of %(maxruntime)s minutes',{'job':jobnumber,'maxruntime':maxruntime}) botslib.sendbotserrorreport(u'[Bots Job Queue] - Job exceeded maximum runtime', u'Job %(job)s exceeded maxruntime of %(maxruntime)s minutes:\n %(task)s' % {'job':jobnumber,'maxruntime':maxruntime,'task':task_to_run})