def evaluate(self): try: return automaticmaintenance.evaluate(self.command,self.get_minta4query()) except: botsglobal.logger.exception(_(u'Error in automatic maintenance.')) return 1
def start(): #exit codes: # 0: OK, no errors # 1: (system) errors # 2: bots ran OK, but there are errors/process errors in the run # 3: Database is locked, but "maxruntime" has not been exceeded. #********command line arguments************************** commandspossible = [ '--new', '--retry', '--retransmit', '--cleanup', '--crashrecovery', '--retrycommunication', '--automaticretrycommunication' ] commandstorun = [] routestorun = [] #list with routes to run configdir = 'config' for arg in sys.argv[1:]: if not arg: continue if arg.startswith('-c'): configdir = arg[2:] if not configdir: print 'Configuration directory indicated, but no directory name.' sys.exit(1) elif arg in commandspossible: commandstorun.append(arg) elif arg in ["?", "/?"] or arg.startswith('-'): showusage() sys.exit(0) else: #pick up names of routes to run routestorun.append(arg) if not commandstorun: #if no command on command line, use new (default) commandstorun = ['--new'] #**************init general: find locating of bots, configfiles, init paths etc.**************** botsinit.generalinit(configdir) #set current working directory to botspath #~ old_current_directory = os.getcwdu() os.chdir(botsglobal.ini.get('directories', 'botspath')) #**************initialise logging****************************** try: botsinit.initenginelogging() except: print _('Error in initialising logging system.') traceback.print_exc() sys.exit(1) else: atexit.register(logging.shutdown) for key, value in botslib.botsinfo(): #log start info botsglobal.logger.info(u'%s: "%s".', key, value) #**************connect to database********************************** try: botsinit.connect() except: botsglobal.logger.exception( _(u'Could not connect to database. Database settings are in bots/config/settings.py.' )) sys.exit(1) else: botsglobal.logger.info(_(u'Connected to database.')) atexit.register(botsglobal.db.close) #initialise user exits for the whole bots-engine (this script file) try: userscript, scriptname = botslib.botsimport('routescripts', 'botsengine') except ImportError: userscript = scriptname = None #**************handle database lock**************************************** #try to set a lock on the database; if this is not possible, the database is already locked. Either: #1 another instance bots bots-engine is (still) running #2 or bots-engine had a severe crash. #What to do? #first: check ts of database lock. If below a certain value (set in bots.ini) we assume an other instance is running. Exit quietly - no errors, no logging. # else: Warn user, give advise on what to do. gather data: nr files in, errors. #next: warn with report & logging. advise a crashrecovery. if not botslib.set_database_lock(): if '--crashrecovery' in commandstorun: #user starts recovery operation; the databaselock is ignored; the databaselock is unlocked when routes have run. commandstorun = ['--crashrecovery'] #is an exclusive option! else: #when scheduling bots it is possible that the last run is still running. Check if maxruntime has passed: vanaf = datetime.datetime.today() - datetime.timedelta( minutes=botsglobal.ini.getint('settings', 'maxruntime', 60)) for row in botslib.query( '''SELECT ts FROM mutex WHERE ts < %(vanaf)s ''', {'vanaf': vanaf}): warn = _( u'!Bots database is locked!\nBots-engine has ended in an unexpected way during the last run.\nThis happens, but is very very rare.\nPossible causes: bots-engine terminated by user, system crash, power-down, etc.\nA forced retry of the last run is advised; bots will (try to) repair the last run.' ) botsglobal.logger.critical(warn) botslib.sendbotserrorreport( _(u'[Bots severe error]Database is locked'), warn) #add: count errors etc. sys.exit(1) else: #maxruntime has not passed. Exit silently, nothing reported botsglobal.logger.info( _(u'Database is locked, but "maxruntime" has not been exceeded.' )) sys.exit(3) else: if '--crashrecovery' in commandstorun: #user starts recovery operation but there is no databaselock. warn = _( u'User started a forced retry of the last run.\nOnly use this when the database is locked.\nThe database was not locked (database is OK).\nSo Bots has done nothing now.' ) botsglobal.logger.error(warn) botslib.sendbotserrorreport( _(u'[Bots Error Report] User started a forced retry of last run, but this was not needed' ), warn) botslib.remove_database_lock() sys.exit(1) #*************get list of routes to run**************************************** #~ raise Exception('locked database') #for testing database lock: abort, database will be locked if routestorun: botsglobal.logger.info(u'Run routes from command line: "%s".', str(routestorun)) else: # no routes from command line parameters: fetch all active routes from database for row in botslib.query( '''SELECT DISTINCT idroute FROM routes WHERE active=%(active)s AND (notindefaultrun=%(notindefaultrun)s OR notindefaultrun IS NULL) ORDER BY idroute ''', { 'active': True, 'notindefaultrun': False }): routestorun.append(row['idroute']) botsglobal.logger.info(_(u'Run active routes from database: "%s".'), str(routestorun)) #routestorun is now either a list with routes from commandline, or the list of active routes for the routes table in the db. #**************run the routes for retry, retransmit and new runs************************************* try: #commandstorun determines the type(s) of run #routes to run is a listof the routes that are runs (for each command to run #botsglobal.incommunicate is used to control if there is communication in; only 'new' incommunicates. #botsglobal.minta4query controls which ta's are queried by the routes. #stuff2evaluate controls what is evaluated in automatic maintenance. errorinrun = 0 #detect if there has been some error. Only used for good exit() code botsglobal.incommunicate = False if '--crashrecovery' in commandstorun: botsglobal.logger.info(_(u'Run crash recovery.')) stuff2evaluate = botslib.set_minta4query_crashrecovery() if stuff2evaluate: router.routedispatcher(routestorun) errorinrun += automaticmaintenance.evaluate( '--crashrecovery', stuff2evaluate) else: botsglobal.logger.info( _(u'No retry of the last run - there was no last run.')) if userscript and hasattr(userscript, 'postcrashrecovery'): botslib.runscript(userscript, scriptname, 'postcrashrecovery', routestorun=routestorun) if '--retrycommunication' in commandstorun: botsglobal.logger.info(_(u'Run communication retry.')) stuff2evaluate = router.routedispatcher(routestorun, '--retrycommunication') if stuff2evaluate: errorinrun += automaticmaintenance.evaluate( '--retrycommunication', stuff2evaluate) else: botsglobal.logger.info( _(u'Run recommunicate: nothing to recommunicate.')) if userscript and hasattr(userscript, 'postretrycommunication'): botslib.runscript(userscript, scriptname, 'postretrycommunication', routestorun=routestorun) if '--automaticretrycommunication' in commandstorun: botsglobal.logger.info(_(u'Run automatic communication retry.')) stuff2evaluate = router.routedispatcher( routestorun, '--automaticretrycommunication') if stuff2evaluate: errorinrun += automaticmaintenance.evaluate( '--automaticretrycommunication', stuff2evaluate) else: botsglobal.logger.info( _(u'Run automatic recommunicate: nothing to recommunicate.' )) if userscript and hasattr(userscript, 'postautomaticretrycommunication'): botslib.runscript(userscript, scriptname, 'postautomaticretrycommunication', routestorun=routestorun) if '--retry' in commandstorun: botsglobal.logger.info(u'Run retry.') stuff2evaluate = router.routedispatcher(routestorun, '--retry') if stuff2evaluate: errorinrun += automaticmaintenance.evaluate( '--retry', stuff2evaluate) else: botsglobal.logger.info(_(u'Run retry: nothing to retry.')) if userscript and hasattr(userscript, 'postretry'): botslib.runscript(userscript, scriptname, 'postretry', routestorun=routestorun) if '--retransmit' in commandstorun: botsglobal.logger.info(u'Run retransmit.') stuff2evaluate = router.routedispatcher(routestorun, '--retransmit') if stuff2evaluate: errorinrun += automaticmaintenance.evaluate( '--retransmit', stuff2evaluate) else: botsglobal.logger.info( _(u'Run retransmit: nothing to retransmit.')) if userscript and hasattr(userscript, 'postretransmit'): botslib.runscript(userscript, scriptname, 'postretransmit', routestorun=routestorun) if '--new' in commandstorun: botsglobal.logger.info('Run new.') botsglobal.incommunicate = True botsglobal.minta4query = 0 #meaning: reset. the actual value is set later (in routedispatcher) stuff2evaluate = router.routedispatcher(routestorun) errorinrun += automaticmaintenance.evaluate( '--new', stuff2evaluate) if userscript and hasattr(userscript, 'postnewrun'): botslib.runscript(userscript, scriptname, 'postnewrun', routestorun=routestorun) if '--cleanup' in commandstorun or botsglobal.ini.get( 'settings', 'whencleanup', 'always') == 'always': botsglobal.logger.debug(u'Do cleanup.') cleanup.cleanup() botslib.remove_database_lock() except Exception, e: botsglobal.logger.exception( _(u'Severe error in bots system:\n%s') % (e)) #of course this 'should' not happen. sys.exit(1)
def start(): #exit codes: # 0: OK, no errors # 1: (system) errors # 2: bots ran OK, but there are errors/process errors in the run # 3: Database is locked, but "maxruntime" has not been exceeded. #********command line arguments************************** commandspossible = ['--new','--retry','--retransmit','--cleanup','--crashrecovery','--retrycommunication','--automaticretrycommunication'] commandstorun = [] routestorun = [] #list with routes to run configdir = 'config' for arg in sys.argv[1:]: if not arg: continue if arg.startswith('-c'): configdir = arg[2:] if not configdir: print 'Configuration directory indicated, but no directory name.' sys.exit(1) elif arg in commandspossible: commandstorun.append(arg) elif arg in ["?", "/?"] or arg.startswith('-'): showusage() sys.exit(0) else: #pick up names of routes to run routestorun.append(arg) if not commandstorun: #if no command on command line, use new (default) commandstorun = ['--new'] #**************init general: find locating of bots, configfiles, init paths etc.**************** botsinit.generalinit(configdir) #set current working directory to botspath #~ old_current_directory = os.getcwdu() os.chdir(botsglobal.ini.get('directories','botspath')) #**************initialise logging****************************** try: botsinit.initenginelogging() except: print _('Error in initialising logging system.') traceback.print_exc() sys.exit(1) else: atexit.register(logging.shutdown) for key,value in botslib.botsinfo(): #log start info botsglobal.logger.info(u'%s: "%s".',key,value) #**************connect to database********************************** try: botsinit.connect() except: botsglobal.logger.exception(_(u'Could not connect to database. Database settings are in bots/config/settings.py.')) sys.exit(1) else: botsglobal.logger.info(_(u'Connected to database.')) atexit.register(botsglobal.db.close) #initialise user exits for the whole bots-engine (this script file) try: userscript,scriptname = botslib.botsimport('routescripts','botsengine') except ImportError: userscript = scriptname = None #**************handle database lock**************************************** #try to set a lock on the database; if this is not possible, the database is already locked. Either: #1 another instance bots bots-engine is (still) running #2 or bots-engine had a severe crash. #What to do? #first: check ts of database lock. If below a certain value (set in bots.ini) we assume an other instance is running. Exit quietly - no errors, no logging. # else: Warn user, give advise on what to do. gather data: nr files in, errors. #next: warn with report & logging. advise a crashrecovery. if not botslib.set_database_lock(): if '--crashrecovery' in commandstorun: #user starts recovery operation; the databaselock is ignored; the databaselock is unlocked when routes have run. commandstorun = ['--crashrecovery'] #is an exclusive option! else: #when scheduling bots it is possible that the last run is still running. Check if maxruntime has passed: vanaf = datetime.datetime.today() - datetime.timedelta(minutes=botsglobal.ini.getint('settings','maxruntime',60)) for row in botslib.query('''SELECT ts FROM mutex WHERE ts < %(vanaf)s ''',{'vanaf':vanaf}): warn = _(u'!Bots database is locked!\nBots-engine has ended in an unexpected way during the last run.\nThis happens, but is very very rare.\nPossible causes: bots-engine terminated by user, system crash, power-down, etc.\nA forced retry of the last run is advised; bots will (try to) repair the last run.') botsglobal.logger.critical(warn) botslib.sendbotserrorreport(_(u'[Bots severe error]Database is locked'),warn) #add: count errors etc. sys.exit(1) else: #maxruntime has not passed. Exit silently, nothing reported botsglobal.logger.info(_(u'Database is locked, but "maxruntime" has not been exceeded.')) sys.exit(3) else: if '--crashrecovery' in commandstorun: #user starts recovery operation but there is no databaselock. warn = _(u'User started a forced retry of the last run.\nOnly use this when the database is locked.\nThe database was not locked (database is OK).\nSo Bots has done nothing now.') botsglobal.logger.error(warn) botslib.sendbotserrorreport(_(u'[Bots Error Report] User started a forced retry of last run, but this was not needed'),warn) botslib.remove_database_lock() sys.exit(1) #*************get list of routes to run**************************************** #~ raise Exception('locked database') #for testing database lock: abort, database will be locked if routestorun: botsglobal.logger.info(u'Run routes from command line: "%s".',str(routestorun)) else: # no routes from command line parameters: fetch all active routes from database for row in botslib.query('''SELECT DISTINCT idroute FROM routes WHERE active=%(active)s AND (notindefaultrun=%(notindefaultrun)s OR notindefaultrun IS NULL) ORDER BY idroute ''', {'active':True,'notindefaultrun':False}): routestorun.append(row['idroute']) botsglobal.logger.info(_(u'Run active routes from database: "%s".'),str(routestorun)) #routestorun is now either a list with routes from commandline, or the list of active routes for the routes table in the db. #**************run the routes for retry, retransmit and new runs************************************* try: #commandstorun determines the type(s) of run #routes to run is a listof the routes that are runs (for each command to run #botsglobal.incommunicate is used to control if there is communication in; only 'new' incommunicates. #botsglobal.minta4query controls which ta's are queried by the routes. #stuff2evaluate controls what is evaluated in automatic maintenance. errorinrun = 0 #detect if there has been some error. Only used for good exit() code botsglobal.incommunicate = False if '--crashrecovery' in commandstorun: botsglobal.logger.info(_(u'Run crash recovery.')) stuff2evaluate = botslib.set_minta4query_crashrecovery() if stuff2evaluate: router.routedispatcher(routestorun) errorinrun += automaticmaintenance.evaluate('--crashrecovery',stuff2evaluate) else: botsglobal.logger.info(_(u'No retry of the last run - there was no last run.')) if userscript and hasattr(userscript,'postcrashrecovery'): botslib.runscript(userscript,scriptname,'postcrashrecovery',routestorun=routestorun) if '--retrycommunication' in commandstorun: botsglobal.logger.info(_(u'Run communication retry.')) stuff2evaluate = router.routedispatcher(routestorun,'--retrycommunication') if stuff2evaluate: errorinrun += automaticmaintenance.evaluate('--retrycommunication',stuff2evaluate) else: botsglobal.logger.info(_(u'Run recommunicate: nothing to recommunicate.')) if userscript and hasattr(userscript,'postretrycommunication'): botslib.runscript(userscript,scriptname,'postretrycommunication',routestorun=routestorun) if '--automaticretrycommunication' in commandstorun: botsglobal.logger.info(_(u'Run automatic communication retry.')) stuff2evaluate = router.routedispatcher(routestorun,'--automaticretrycommunication') if stuff2evaluate: errorinrun += automaticmaintenance.evaluate('--automaticretrycommunication',stuff2evaluate) else: botsglobal.logger.info(_(u'Run automatic recommunicate: nothing to recommunicate.')) if userscript and hasattr(userscript,'postautomaticretrycommunication'): botslib.runscript(userscript,scriptname,'postautomaticretrycommunication',routestorun=routestorun) if '--retry' in commandstorun: botsglobal.logger.info(u'Run retry.') stuff2evaluate = router.routedispatcher(routestorun,'--retry') if stuff2evaluate: errorinrun += automaticmaintenance.evaluate('--retry',stuff2evaluate) else: botsglobal.logger.info(_(u'Run retry: nothing to retry.')) if userscript and hasattr(userscript,'postretry'): botslib.runscript(userscript,scriptname,'postretry',routestorun=routestorun) if '--retransmit' in commandstorun: botsglobal.logger.info(u'Run retransmit.') stuff2evaluate = router.routedispatcher(routestorun,'--retransmit') if stuff2evaluate: errorinrun += automaticmaintenance.evaluate('--retransmit',stuff2evaluate) else: botsglobal.logger.info(_(u'Run retransmit: nothing to retransmit.')) if userscript and hasattr(userscript,'postretransmit'): botslib.runscript(userscript,scriptname,'postretransmit',routestorun=routestorun) if '--new' in commandstorun: botsglobal.logger.info('Run new.') botsglobal.incommunicate = True botsglobal.minta4query = 0 #meaning: reset. the actual value is set later (in routedispatcher) stuff2evaluate = router.routedispatcher(routestorun) errorinrun += automaticmaintenance.evaluate('--new',stuff2evaluate) if userscript and hasattr(userscript,'postnewrun'): botslib.runscript(userscript,scriptname,'postnewrun',routestorun=routestorun) if '--cleanup' in commandstorun or botsglobal.ini.get('settings','whencleanup','always')=='always': botsglobal.logger.debug(u'Do cleanup.') cleanup.cleanup() botslib.remove_database_lock() except Exception,e: botsglobal.logger.exception(_(u'Severe error in bots system:\n%s')%(e)) #of course this 'should' not happen. sys.exit(1)