def evaluate(self):
try:
return automaticmaintenance.evaluate(self.command,self.get_minta4query())
except:
botsglobal.logger.exception(_(u'Error in automatic maintenance.'))
return 1
def start():
#exit codes:
# 0: OK, no errors
# 1: (system) errors
# 2: bots ran OK, but there are errors/process errors in the run
# 3: Database is locked, but "maxruntime" has not been exceeded.
#********command line arguments**************************
commandspossible = [
'--new', '--retry', '--retransmit', '--cleanup', '--crashrecovery',
'--retrycommunication', '--automaticretrycommunication'
]
commandstorun = []
routestorun = [] #list with routes to run
configdir = 'config'
for arg in sys.argv[1:]:
if not arg:
continue
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Configuration directory indicated, but no directory name.'
sys.exit(1)
elif arg in commandspossible:
commandstorun.append(arg)
elif arg in ["?", "/?"] or arg.startswith('-'):
showusage()
sys.exit(0)
else: #pick up names of routes to run
routestorun.append(arg)
if not commandstorun: #if no command on command line, use new (default)
commandstorun = ['--new']
#**************init general: find locating of bots, configfiles, init paths etc.****************
botsinit.generalinit(configdir)
#set current working directory to botspath
#~ old_current_directory = os.getcwdu()
os.chdir(botsglobal.ini.get('directories', 'botspath'))
#**************initialise logging******************************
try:
botsinit.initenginelogging()
except:
print _('Error in initialising logging system.')
traceback.print_exc()
sys.exit(1)
else:
atexit.register(logging.shutdown)
for key, value in botslib.botsinfo(): #log start info
botsglobal.logger.info(u'%s: "%s".', key, value)
#**************connect to database**********************************
try:
botsinit.connect()
except:
botsglobal.logger.exception(
_(u'Could not connect to database. Database settings are in bots/config/settings.py.'
))
sys.exit(1)
else:
botsglobal.logger.info(_(u'Connected to database.'))
atexit.register(botsglobal.db.close)
#initialise user exits for the whole bots-engine (this script file)
try:
userscript, scriptname = botslib.botsimport('routescripts',
'botsengine')
except ImportError:
userscript = scriptname = None
#**************handle database lock****************************************
#try to set a lock on the database; if this is not possible, the database is already locked. Either:
#1 another instance bots bots-engine is (still) running
#2 or bots-engine had a severe crash.
#What to do?
#first: check ts of database lock. If below a certain value (set in bots.ini) we assume an other instance is running. Exit quietly - no errors, no logging.
# else: Warn user, give advise on what to do. gather data: nr files in, errors.
#next: warn with report & logging. advise a crashrecovery.
if not botslib.set_database_lock():
if '--crashrecovery' in commandstorun: #user starts recovery operation; the databaselock is ignored; the databaselock is unlocked when routes have run.
commandstorun = ['--crashrecovery'] #is an exclusive option!
else:
#when scheduling bots it is possible that the last run is still running. Check if maxruntime has passed:
vanaf = datetime.datetime.today() - datetime.timedelta(
minutes=botsglobal.ini.getint('settings', 'maxruntime', 60))
for row in botslib.query(
'''SELECT ts FROM mutex WHERE ts < %(vanaf)s ''',
{'vanaf': vanaf}):
warn = _(
u'!Bots database is locked!\nBots-engine has ended in an unexpected way during the last run.\nThis happens, but is very very rare.\nPossible causes: bots-engine terminated by user, system crash, power-down, etc.\nA forced retry of the last run is advised; bots will (try to) repair the last run.'
)
botsglobal.logger.critical(warn)
botslib.sendbotserrorreport(
_(u'[Bots severe error]Database is locked'), warn)
#add: count errors etc.
sys.exit(1)
else: #maxruntime has not passed. Exit silently, nothing reported
botsglobal.logger.info(
_(u'Database is locked, but "maxruntime" has not been exceeded.'
))
sys.exit(3)
else:
if '--crashrecovery' in commandstorun: #user starts recovery operation but there is no databaselock.
warn = _(
u'User started a forced retry of the last run.\nOnly use this when the database is locked.\nThe database was not locked (database is OK).\nSo Bots has done nothing now.'
)
botsglobal.logger.error(warn)
botslib.sendbotserrorreport(
_(u'[Bots Error Report] User started a forced retry of last run, but this was not needed'
), warn)
botslib.remove_database_lock()
sys.exit(1)
#*************get list of routes to run****************************************
#~ raise Exception('locked database') #for testing database lock: abort, database will be locked
if routestorun:
botsglobal.logger.info(u'Run routes from command line: "%s".',
str(routestorun))
else: # no routes from command line parameters: fetch all active routes from database
for row in botslib.query(
'''SELECT DISTINCT idroute
FROM routes
WHERE active=%(active)s
AND (notindefaultrun=%(notindefaultrun)s OR notindefaultrun IS NULL)
ORDER BY idroute ''', {
'active': True,
'notindefaultrun': False
}):
routestorun.append(row['idroute'])
botsglobal.logger.info(_(u'Run active routes from database: "%s".'),
str(routestorun))
#routestorun is now either a list with routes from commandline, or the list of active routes for the routes table in the db.
#**************run the routes for retry, retransmit and new runs*************************************
try:
#commandstorun determines the type(s) of run
#routes to run is a listof the routes that are runs (for each command to run
#botsglobal.incommunicate is used to control if there is communication in; only 'new' incommunicates.
#botsglobal.minta4query controls which ta's are queried by the routes.
#stuff2evaluate controls what is evaluated in automatic maintenance.
errorinrun = 0 #detect if there has been some error. Only used for good exit() code
botsglobal.incommunicate = False
if '--crashrecovery' in commandstorun:
botsglobal.logger.info(_(u'Run crash recovery.'))
stuff2evaluate = botslib.set_minta4query_crashrecovery()
if stuff2evaluate:
router.routedispatcher(routestorun)
errorinrun += automaticmaintenance.evaluate(
'--crashrecovery', stuff2evaluate)
else:
botsglobal.logger.info(
_(u'No retry of the last run - there was no last run.'))
if userscript and hasattr(userscript, 'postcrashrecovery'):
botslib.runscript(userscript,
scriptname,
'postcrashrecovery',
routestorun=routestorun)
if '--retrycommunication' in commandstorun:
botsglobal.logger.info(_(u'Run communication retry.'))
stuff2evaluate = router.routedispatcher(routestorun,
'--retrycommunication')
if stuff2evaluate:
errorinrun += automaticmaintenance.evaluate(
'--retrycommunication', stuff2evaluate)
else:
botsglobal.logger.info(
_(u'Run recommunicate: nothing to recommunicate.'))
if userscript and hasattr(userscript, 'postretrycommunication'):
botslib.runscript(userscript,
scriptname,
'postretrycommunication',
routestorun=routestorun)
if '--automaticretrycommunication' in commandstorun:
botsglobal.logger.info(_(u'Run automatic communication retry.'))
stuff2evaluate = router.routedispatcher(
routestorun, '--automaticretrycommunication')
if stuff2evaluate:
errorinrun += automaticmaintenance.evaluate(
'--automaticretrycommunication', stuff2evaluate)
else:
botsglobal.logger.info(
_(u'Run automatic recommunicate: nothing to recommunicate.'
))
if userscript and hasattr(userscript,
'postautomaticretrycommunication'):
botslib.runscript(userscript,
scriptname,
'postautomaticretrycommunication',
routestorun=routestorun)
if '--retry' in commandstorun:
botsglobal.logger.info(u'Run retry.')
stuff2evaluate = router.routedispatcher(routestorun, '--retry')
if stuff2evaluate:
errorinrun += automaticmaintenance.evaluate(
'--retry', stuff2evaluate)
else:
botsglobal.logger.info(_(u'Run retry: nothing to retry.'))
if userscript and hasattr(userscript, 'postretry'):
botslib.runscript(userscript,
scriptname,
'postretry',
routestorun=routestorun)
if '--retransmit' in commandstorun:
botsglobal.logger.info(u'Run retransmit.')
stuff2evaluate = router.routedispatcher(routestorun,
'--retransmit')
if stuff2evaluate:
errorinrun += automaticmaintenance.evaluate(
'--retransmit', stuff2evaluate)
else:
botsglobal.logger.info(
_(u'Run retransmit: nothing to retransmit.'))
if userscript and hasattr(userscript, 'postretransmit'):
botslib.runscript(userscript,
scriptname,
'postretransmit',
routestorun=routestorun)
if '--new' in commandstorun:
botsglobal.logger.info('Run new.')
botsglobal.incommunicate = True
botsglobal.minta4query = 0 #meaning: reset. the actual value is set later (in routedispatcher)
stuff2evaluate = router.routedispatcher(routestorun)
errorinrun += automaticmaintenance.evaluate(
'--new', stuff2evaluate)
if userscript and hasattr(userscript, 'postnewrun'):
botslib.runscript(userscript,
scriptname,
'postnewrun',
routestorun=routestorun)
if '--cleanup' in commandstorun or botsglobal.ini.get(
'settings', 'whencleanup', 'always') == 'always':
botsglobal.logger.debug(u'Do cleanup.')
cleanup.cleanup()
botslib.remove_database_lock()
except Exception, e:
botsglobal.logger.exception(
_(u'Severe error in bots system:\n%s') %
(e)) #of course this 'should' not happen.
sys.exit(1)
def start():
#exit codes:
# 0: OK, no errors
# 1: (system) errors
# 2: bots ran OK, but there are errors/process errors in the run
# 3: Database is locked, but "maxruntime" has not been exceeded.
#********command line arguments**************************
commandspossible = ['--new','--retry','--retransmit','--cleanup','--crashrecovery','--retrycommunication','--automaticretrycommunication']
commandstorun = []
routestorun = [] #list with routes to run
configdir = 'config'
for arg in sys.argv[1:]:
if not arg:
continue
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Configuration directory indicated, but no directory name.'
sys.exit(1)
elif arg in commandspossible:
commandstorun.append(arg)
elif arg in ["?", "/?"] or arg.startswith('-'):
showusage()
sys.exit(0)
else: #pick up names of routes to run
routestorun.append(arg)
if not commandstorun: #if no command on command line, use new (default)
commandstorun = ['--new']
#**************init general: find locating of bots, configfiles, init paths etc.****************
botsinit.generalinit(configdir)
#set current working directory to botspath
#~ old_current_directory = os.getcwdu()
os.chdir(botsglobal.ini.get('directories','botspath'))
#**************initialise logging******************************
try:
botsinit.initenginelogging()
except:
print _('Error in initialising logging system.')
traceback.print_exc()
sys.exit(1)
else:
atexit.register(logging.shutdown)
for key,value in botslib.botsinfo(): #log start info
botsglobal.logger.info(u'%s: "%s".',key,value)
#**************connect to database**********************************
try:
botsinit.connect()
except:
botsglobal.logger.exception(_(u'Could not connect to database. Database settings are in bots/config/settings.py.'))
sys.exit(1)
else:
botsglobal.logger.info(_(u'Connected to database.'))
atexit.register(botsglobal.db.close)
#initialise user exits for the whole bots-engine (this script file)
try:
userscript,scriptname = botslib.botsimport('routescripts','botsengine')
except ImportError:
userscript = scriptname = None
#**************handle database lock****************************************
#try to set a lock on the database; if this is not possible, the database is already locked. Either:
#1 another instance bots bots-engine is (still) running
#2 or bots-engine had a severe crash.
#What to do?
#first: check ts of database lock. If below a certain value (set in bots.ini) we assume an other instance is running. Exit quietly - no errors, no logging.
# else: Warn user, give advise on what to do. gather data: nr files in, errors.
#next: warn with report & logging. advise a crashrecovery.
if not botslib.set_database_lock():
if '--crashrecovery' in commandstorun: #user starts recovery operation; the databaselock is ignored; the databaselock is unlocked when routes have run.
commandstorun = ['--crashrecovery'] #is an exclusive option!
else:
#when scheduling bots it is possible that the last run is still running. Check if maxruntime has passed:
vanaf = datetime.datetime.today() - datetime.timedelta(minutes=botsglobal.ini.getint('settings','maxruntime',60))
for row in botslib.query('''SELECT ts FROM mutex WHERE ts < %(vanaf)s ''',{'vanaf':vanaf}):
warn = _(u'!Bots database is locked!\nBots-engine has ended in an unexpected way during the last run.\nThis happens, but is very very rare.\nPossible causes: bots-engine terminated by user, system crash, power-down, etc.\nA forced retry of the last run is advised; bots will (try to) repair the last run.')
botsglobal.logger.critical(warn)
botslib.sendbotserrorreport(_(u'[Bots severe error]Database is locked'),warn)
#add: count errors etc.
sys.exit(1)
else: #maxruntime has not passed. Exit silently, nothing reported
botsglobal.logger.info(_(u'Database is locked, but "maxruntime" has not been exceeded.'))
sys.exit(3)
else:
if '--crashrecovery' in commandstorun: #user starts recovery operation but there is no databaselock.
warn = _(u'User started a forced retry of the last run.\nOnly use this when the database is locked.\nThe database was not locked (database is OK).\nSo Bots has done nothing now.')
botsglobal.logger.error(warn)
botslib.sendbotserrorreport(_(u'[Bots Error Report] User started a forced retry of last run, but this was not needed'),warn)
botslib.remove_database_lock()
sys.exit(1)
#*************get list of routes to run****************************************
#~ raise Exception('locked database') #for testing database lock: abort, database will be locked
if routestorun:
botsglobal.logger.info(u'Run routes from command line: "%s".',str(routestorun))
else: # no routes from command line parameters: fetch all active routes from database
for row in botslib.query('''SELECT DISTINCT idroute
FROM routes
WHERE active=%(active)s
AND (notindefaultrun=%(notindefaultrun)s OR notindefaultrun IS NULL)
ORDER BY idroute ''',
{'active':True,'notindefaultrun':False}):
routestorun.append(row['idroute'])
botsglobal.logger.info(_(u'Run active routes from database: "%s".'),str(routestorun))
#routestorun is now either a list with routes from commandline, or the list of active routes for the routes table in the db.
#**************run the routes for retry, retransmit and new runs*************************************
try:
#commandstorun determines the type(s) of run
#routes to run is a listof the routes that are runs (for each command to run
#botsglobal.incommunicate is used to control if there is communication in; only 'new' incommunicates.
#botsglobal.minta4query controls which ta's are queried by the routes.
#stuff2evaluate controls what is evaluated in automatic maintenance.
errorinrun = 0 #detect if there has been some error. Only used for good exit() code
botsglobal.incommunicate = False
if '--crashrecovery' in commandstorun:
botsglobal.logger.info(_(u'Run crash recovery.'))
stuff2evaluate = botslib.set_minta4query_crashrecovery()
if stuff2evaluate:
router.routedispatcher(routestorun)
errorinrun += automaticmaintenance.evaluate('--crashrecovery',stuff2evaluate)
else:
botsglobal.logger.info(_(u'No retry of the last run - there was no last run.'))
if userscript and hasattr(userscript,'postcrashrecovery'):
botslib.runscript(userscript,scriptname,'postcrashrecovery',routestorun=routestorun)
if '--retrycommunication' in commandstorun:
botsglobal.logger.info(_(u'Run communication retry.'))
stuff2evaluate = router.routedispatcher(routestorun,'--retrycommunication')
if stuff2evaluate:
errorinrun += automaticmaintenance.evaluate('--retrycommunication',stuff2evaluate)
else:
botsglobal.logger.info(_(u'Run recommunicate: nothing to recommunicate.'))
if userscript and hasattr(userscript,'postretrycommunication'):
botslib.runscript(userscript,scriptname,'postretrycommunication',routestorun=routestorun)
if '--automaticretrycommunication' in commandstorun:
botsglobal.logger.info(_(u'Run automatic communication retry.'))
stuff2evaluate = router.routedispatcher(routestorun,'--automaticretrycommunication')
if stuff2evaluate:
errorinrun += automaticmaintenance.evaluate('--automaticretrycommunication',stuff2evaluate)
else:
botsglobal.logger.info(_(u'Run automatic recommunicate: nothing to recommunicate.'))
if userscript and hasattr(userscript,'postautomaticretrycommunication'):
botslib.runscript(userscript,scriptname,'postautomaticretrycommunication',routestorun=routestorun)
if '--retry' in commandstorun:
botsglobal.logger.info(u'Run retry.')
stuff2evaluate = router.routedispatcher(routestorun,'--retry')
if stuff2evaluate:
errorinrun += automaticmaintenance.evaluate('--retry',stuff2evaluate)
else:
botsglobal.logger.info(_(u'Run retry: nothing to retry.'))
if userscript and hasattr(userscript,'postretry'):
botslib.runscript(userscript,scriptname,'postretry',routestorun=routestorun)
if '--retransmit' in commandstorun:
botsglobal.logger.info(u'Run retransmit.')
stuff2evaluate = router.routedispatcher(routestorun,'--retransmit')
if stuff2evaluate:
errorinrun += automaticmaintenance.evaluate('--retransmit',stuff2evaluate)
else:
botsglobal.logger.info(_(u'Run retransmit: nothing to retransmit.'))
if userscript and hasattr(userscript,'postretransmit'):
botslib.runscript(userscript,scriptname,'postretransmit',routestorun=routestorun)
if '--new' in commandstorun:
botsglobal.logger.info('Run new.')
botsglobal.incommunicate = True
botsglobal.minta4query = 0 #meaning: reset. the actual value is set later (in routedispatcher)
stuff2evaluate = router.routedispatcher(routestorun)
errorinrun += automaticmaintenance.evaluate('--new',stuff2evaluate)
if userscript and hasattr(userscript,'postnewrun'):
botslib.runscript(userscript,scriptname,'postnewrun',routestorun=routestorun)
if '--cleanup' in commandstorun or botsglobal.ini.get('settings','whencleanup','always')=='always':
botsglobal.logger.debug(u'Do cleanup.')
cleanup.cleanup()
botslib.remove_database_lock()
except Exception,e:
botsglobal.logger.exception(_(u'Severe error in bots system:\n%s')%(e)) #of course this 'should' not happen.
sys.exit(1)
