Beispiel #1
0
def launcher(logger,port,lauchfrequency,maxruntime):
    xmlrpcclient = xmlrpclib.ServerProxy('http://localhost:' + str(port))
    maxseconds = maxruntime*60
    time.sleep(3)   #to allow jobqserver to start
    while True:
        time.sleep(lauchfrequency)
        job = xmlrpcclient.getjob()

        if job:       #0 means nothing to launch
            jobnumber = job[1]
            task_to_run = job[2]
            # Start a timer thread for maxruntime error
            timer_thread = threading.Timer(maxseconds,maxruntimeerror,args=(logger,maxruntime,jobnumber,task_to_run))
            timer_thread.start()
            try:
                starttime = datetime.datetime.now()
                logger.info(u'Starting job %(job)s',job=jobnumber)
                result = subprocess.call(task_to_run,stdin=open(os.devnull,'r'),stdout=open(os.devnull,'w'),stderr=open(os.devnull,'w'))
                time_taken = datetime.timedelta(seconds=(datetime.datetime.now() - starttime).seconds)
                logger.info(u'Finished job %(job)s, elapsed time %(time_taken)s, result %(result)s',job=jobnumber,time_taken=time_taken,result=result)
            except Exception, msg:
                logger.error(u'Error starting job %(job)s: %(msg)s',job=jobnumber,msg=msg)
                botslib.sendbotserrorreport(u'[Bots Job Queue] - Error starting job',
                                            u'Error starting job %s:\n %s\n\n %s' % (jobnumber,task_to_run,msg))
            timer_thread.cancel()
def generate_report(stuff2evaluate):
    for results in botslib.query('''SELECT idta,lastopen,lasterror,lastok,lastdone,
                                            send,processerrors,ts,lastreceived,type,status
                                    FROM report
                                    WHERE idta=%(rootidta)s''',
                                    {'rootidta':stuff2evaluate}):
        break
    else:
        raise botslib.PanicError(_(u'In generate report: could not find report?'))
    subject = _(u'[Bots Error Report] %(time)s')%{'time':str(results['ts'])[:16]}
    reporttext = _(u'Bots Report; type: %(type)s, time: %(time)s\n')%{'type':results['type'],'time':str(results['ts'])[:19]}
    reporttext += _(u'    %d files received/processed in run.\n')%(results['lastreceived'])
    if results['lastdone']:
        reporttext += _(u'    %d files without errors,\n')%(results['lastdone'])
    if results['lasterror']:
        subject += _(u'; %d file errors')%(results['lasterror'])
        reporttext += _(u'    %d files with errors,\n')%(results['lasterror'])
    if results['lastok']:
        subject += _(u'; %d files stuck')%(results['lastok'])
        reporttext += _(u'    %d files got stuck,\n')%(results['lastok'])
    if results['lastopen']:
        subject += _(u'; %d system errors')%(results['lastopen'])
        reporttext += _(u'    %d system errors,\n')%(results['lastopen'])
    if results['processerrors']:
        subject += _(u'; %d process errors')%(results['processerrors'])
        reporttext += _(u'    %d errors in processes.\n')%(results['processerrors'])
    reporttext += _(u'    %d files send in run.\n')%(results['send'])
    
    botsglobal.logger.info(reporttext)
    # sendreportifprocesserror allows blocking of email reports for process errors
    if (results['lasterror'] or results['lastopen'] or results['lastok'] or
       (results['processerrors'] and botsglobal.ini.getboolean('settings','sendreportifprocesserror',True))):
        botslib.sendbotserrorreport(subject,reporttext)
    return int(results['status'])    #return report status: 0 (no error) or 1 (error)
def launcher(logger,port,lauchfrequency,maxruntime):
    xmlrpcclient = xmlrpclib.ServerProxy(u'http://localhost:' + unicode(port))
    maxseconds = maxruntime*60
    time.sleep(3)   #to allow jobqserver to start
    while True:
        time.sleep(lauchfrequency)
        job = xmlrpcclient.getjob()

        if job:       #0 means nothing to launch
            jobnumber = job[1]
            task_to_run = job[2]
            # Start a timer thread for maxruntime error
            timer_thread = threading.Timer(maxseconds,maxruntimeerror,args=(logger,maxruntime,jobnumber,task_to_run))
            timer_thread.start()
            try:
                starttime = datetime.datetime.now()
                logger.info(u'Starting job %(job)s',{'job':jobnumber})
                result = subprocess.call(task_to_run,stdin=open(os.devnull,'r'),stdout=open(os.devnull,'w'),stderr=open(os.devnull,'w'))
                time_taken = datetime.timedelta(seconds=(datetime.datetime.now() - starttime).seconds)
                logger.info(u'Finished job %(job)s, elapsed time %(time_taken)s, result %(result)s',{'job':jobnumber,'time_taken':time_taken,'result':result})
            except Exception as msg:
                logger.error(u'Error starting job %(job)s: %(msg)s',{'job':jobnumber,'msg':msg})
                botslib.sendbotserrorreport(u'[Bots Job Queue] - Error starting job',
                                            u'Error starting job %(job)s:\n %(task)s\n\n %(msg)s' % {'job':jobnumber,'task':task_to_run,'msg':msg})
            timer_thread.cancel()
Beispiel #4
0
def email_error_report(rootidtaofrun):
    for results in botslib.query('''SELECT idta,lastopen,lasterror,lastok,lastdone,
                                            send,processerrors,ts,lastreceived,type,status
                                    FROM report
                                    WHERE idta=%(rootidtaofrun)s''',
                                    {'rootidtaofrun':rootidtaofrun}):
        break
    else:
        raise botslib.PanicError(_(u'In generate report: could not find report?'))
    subject = _(u'[Bots Error Report] %(time)s')%{'time':str(results['ts'])[:16]}
    reporttext = _(u'Bots Report; type: %(type)s, time: %(time)s\n')%{'type':results['type'],'time':str(results['ts'])[:19]}
    reporttext += _(u'    %d files received/processed in run.\n')%(results['lastreceived'])
    if results['lastdone']:
        reporttext += _(u'    %d files without errors,\n')%(results['lastdone'])
    if results['lasterror']:
        subject += _(u'; %d file errors')%(results['lasterror'])
        reporttext += _(u'    %d files with errors,\n')%(results['lasterror'])
    if results['lastok']:
        subject += _(u'; %d files stuck')%(results['lastok'])
        reporttext += _(u'    %d files got stuck,\n')%(results['lastok'])
    if results['lastopen']:
        subject += _(u'; %d system errors')%(results['lastopen'])
        reporttext += _(u'    %d system errors,\n')%(results['lastopen'])
    if results['processerrors']:
        subject += _(u'; %d process errors')%(results['processerrors'])
        reporttext += _(u'    %d errors in processes.\n')%(results['processerrors'])
    reporttext += _(u'    %d files send in run.\n')%(results['send'])

    botsglobal.logger.info(reporttext)      #log the report texts
    # only send email report if there are errors.
    # sendreportifprocesserror (in bots.ini): no email reports if only process errors
    if results['lasterror'] or results['lastopen'] or results['lastok'] or (results['processerrors'] and botsglobal.ini.getboolean('settings','sendreportifprocesserror',True)):

        # Include details about process errors in the email report; if debug is True: includes trace
        if results['processerrors']:
            for row in botslib.query('''SELECT idroute,fromchannel,tochannel,errortext
                                        FROM ta
                                        WHERE idta>=%(rootidtaofrun)s
                                        AND status=%(status)s
                                        AND statust=%(statust)s ''',
                                        {'rootidtaofrun':rootidtaofrun,'status':PROCESS,'statust':ERROR}):
                reporttext += '\nProcess error:\n'
                for key in row.keys():
                    reporttext += '%s: %s\n' % (key,row[key])
        # Include details about file errors in the email report; if debug is True: includes trace
        if results['lasterror'] or results['lastopen'] or results['lastok']:
            for row in botslib.query('''SELECT idroute,frompartner,fromchannel,topartner,tochannel,errortext,infilename
                                        FROM filereport
                                        WHERE idta>%(rootidtaofrun)s
                                        AND statust!=%(statust)s ''',
                                        {'rootidtaofrun':rootidtaofrun,'statust':DONE}):
                reporttext += '\nFile error:\n'
                for key in row.keys():
                    reporttext += '%s: %s\n' % (key,row[key])

        botslib.sendbotserrorreport(subject,reporttext)

    return int(results['status'])    #return report status: 0 (no error) or 1 (error)
def email_error_report(rootidtaofrun):
    for results in botslib.query('''SELECT idta,lastopen,lasterror,lastok,lastdone,
                                            send,processerrors,ts,lastreceived,type,status
                                    FROM report
                                    WHERE idta=%(rootidtaofrun)s''',
                                    {'rootidtaofrun':rootidtaofrun}):
        break
    else:
        raise botslib.PanicError(_(u'In generate report: could not find report?'))
    subject = _(u'[Bots Error Report] %(time)s')%{'time':unicode(results['ts'])[:16]}
    reporttext = _(u'Bots Report; type: %(type)s, time: %(time)s\n')%{'type':results['type'],'time':unicode(results['ts'])[:19]}
    reporttext += _(u'    %d files received/processed in run.\n')%(results['lastreceived'])
    if results['lastdone']:
        reporttext += _(u'    %d files without errors,\n')%(results['lastdone'])
    if results['lasterror']:
        subject += _(u'; %d file errors')%(results['lasterror'])
        reporttext += _(u'    %d files with errors,\n')%(results['lasterror'])
    if results['lastok']:
        subject += _(u'; %d files stuck')%(results['lastok'])
        reporttext += _(u'    %d files got stuck,\n')%(results['lastok'])
    if results['lastopen']:
        subject += _(u'; %d system errors')%(results['lastopen'])
        reporttext += _(u'    %d system errors,\n')%(results['lastopen'])
    if results['processerrors']:
        subject += _(u'; %d process errors')%(results['processerrors'])
        reporttext += _(u'    %d errors in processes.\n')%(results['processerrors'])
    reporttext += _(u'    %d files send in run.\n')%(results['send'])

    botsglobal.logger.info(reporttext)      #log the report texts
    # only send email report if there are errors.
    # sendreportifprocesserror (in bots.ini): no email reports if only process errors
    if results['lasterror'] or results['lastopen'] or results['lastok'] or (results['processerrors'] and botsglobal.ini.getboolean('settings','sendreportifprocesserror',True)):

        # Include details about process errors in the email report; if debug is True: includes trace
        if results['processerrors']:
            for row in botslib.query('''SELECT idroute,fromchannel,tochannel,errortext
                                        FROM ta
                                        WHERE idta>=%(rootidtaofrun)s
                                        AND status=%(status)s
                                        AND statust=%(statust)s ''',
                                        {'rootidtaofrun':rootidtaofrun,'status':PROCESS,'statust':ERROR}):
                reporttext += '\nProcess error:\n'
                for key in row.keys():
                    reporttext += '%s: %s\n' % (key,row[key])
        # Include details about file errors in the email report; if debug is True: includes trace
        if results['lasterror'] or results['lastopen'] or results['lastok']:
            for row in botslib.query('''SELECT idroute,frompartner,fromchannel,topartner,tochannel,errortext,infilename
                                        FROM filereport
                                        WHERE idta>%(rootidtaofrun)s
                                        AND statust!=%(statust)s ''',
                                        {'rootidtaofrun':rootidtaofrun,'statust':DONE}):
                reporttext += '\nFile error:\n'
                for key in row.keys():
                    reporttext += '%s: %s\n' % (key,row[key])

        botslib.sendbotserrorreport(subject,reporttext)

    return int(results['status'])    #return report status: 0 (no error) or 1 (error)
def generate_report(stuff2evaluate):
    for results in botslib.query(
            '''SELECT idta,lastopen,lasterror,lastok,lastdone,
                                            send,processerrors,ts,lastreceived,type,status
                                    FROM report
                                    WHERE idta=%(rootidta)s''',
        {'rootidta': stuff2evaluate}):
        break
    else:
        raise botslib.PanicError(
            _(u'In generate report: could not find report?'))
    subject = _(u'[Bots Error Report] %(time)s') % {
        'time': str(results['ts'])[:16]
    }
    reporttext = _(u'Bots Report; type: %(type)s, time: %(time)s\n') % {
        'type': results['type'],
        'time': str(results['ts'])[:19]
    }
    reporttext += _(u'    %d files received/processed in run.\n') % (
        results['lastreceived'])
    if results['lastdone']:
        reporttext += _(u'    %d files without errors,\n') % (
            results['lastdone'])
    if results['lasterror']:
        subject += _(u'; %d file errors') % (results['lasterror'])
        reporttext += _(u'    %d files with errors,\n') % (
            results['lasterror'])
    if results['lastok']:
        subject += _(u'; %d files stuck') % (results['lastok'])
        reporttext += _(u'    %d files got stuck,\n') % (results['lastok'])
    if results['lastopen']:
        subject += _(u'; %d system errors') % (results['lastopen'])
        reporttext += _(u'    %d system errors,\n') % (results['lastopen'])
    if results['processerrors']:
        subject += _(u'; %d process errors') % (results['processerrors'])
        reporttext += _(u'    %d errors in processes.\n') % (
            results['processerrors'])
    reporttext += _(u'    %d files send in run.\n') % (results['send'])

    botsglobal.logger.info(reporttext)
    # sendreportifprocesserror allows blocking of email reports for process errors
    if (results['lasterror'] or results['lastopen'] or results['lastok']
            or (results['processerrors'] and botsglobal.ini.getboolean(
                'settings', 'sendreportifprocesserror', True))):
        botslib.sendbotserrorreport(subject, reporttext)
    return int(
        results['status'])  #return report status: 0 (no error) or 1 (error)
def generate_report(stuff2evaluate):
    for results in botslib.query(
        """SELECT idta,lastopen,lasterror,lastok,lastdone,
                                            send,processerrors,ts,lastreceived,type,status
                                    FROM report
                                    WHERE idta=%(rootidta)s""",
        {"rootidta": stuff2evaluate},
    ):
        break
    else:
        raise botslib.PanicError(_(u"In generate report: could not find report?"))
    subject = _(u"[Bots Error Report] %(time)s") % {"time": str(results["ts"])[:16]}
    reporttext = _(u"Bots Report; type: %(type)s, time: %(time)s\n") % {
        "type": results["type"],
        "time": str(results["ts"])[:19],
    }
    reporttext += _(u"    %d files received/processed in run.\n") % (results["lastreceived"])
    if results["lastdone"]:
        reporttext += _(u"    %d files without errors,\n") % (results["lastdone"])
    if results["lasterror"]:
        subject += _(u"; %d file errors") % (results["lasterror"])
        reporttext += _(u"    %d files with errors,\n") % (results["lasterror"])
    if results["lastok"]:
        subject += _(u"; %d files stuck") % (results["lastok"])
        reporttext += _(u"    %d files got stuck,\n") % (results["lastok"])
    if results["lastopen"]:
        subject += _(u"; %d system errors") % (results["lastopen"])
        reporttext += _(u"    %d system errors,\n") % (results["lastopen"])
    if results["processerrors"]:
        subject += _(u"; %d process errors") % (results["processerrors"])
        reporttext += _(u"    %d errors in processes.\n") % (results["processerrors"])
    reporttext += _(u"    %d files send in run.\n") % (results["send"])

    botsglobal.logger.info(reporttext)
    # sendreportifprocesserror allows blocking of email reports for process errors
    if (
        results["lasterror"]
        or results["lastopen"]
        or results["lastok"]
        or (results["processerrors"] and botsglobal.ini.getboolean("settings", "sendreportifprocesserror", True))
    ):
        botslib.sendbotserrorreport(subject, reporttext)
    return int(results["status"])  # return report status: 0 (no error) or 1 (error)
Beispiel #8
0
def start():
    #exit codes:
    # 0: OK, no errors
    # 1: (system) errors
    # 2: bots ran OK, but there are errors/process errors  in the run
    # 3: Database is locked, but "maxruntime" has not been exceeded.
    #********command line arguments**************************
    commandspossible = [
        '--new', '--retry', '--retransmit', '--cleanup', '--crashrecovery',
        '--retrycommunication', '--automaticretrycommunication'
    ]
    commandstorun = []
    routestorun = []  #list with routes to run
    configdir = 'config'
    for arg in sys.argv[1:]:
        if not arg:
            continue
        if arg.startswith('-c'):
            configdir = arg[2:]
            if not configdir:
                print 'Configuration directory indicated, but no directory name.'
                sys.exit(1)
        elif arg in commandspossible:
            commandstorun.append(arg)
        elif arg in ["?", "/?"] or arg.startswith('-'):
            showusage()
            sys.exit(0)
        else:  #pick up names of routes to run
            routestorun.append(arg)
    if not commandstorun:  #if no command on command line, use new (default)
        commandstorun = ['--new']
    #**************init general: find locating of bots, configfiles, init paths etc.****************
    botsinit.generalinit(configdir)
    #set current working directory to botspath
    #~ old_current_directory = os.getcwdu()
    os.chdir(botsglobal.ini.get('directories', 'botspath'))
    #**************initialise logging******************************
    try:
        botsinit.initenginelogging()
    except:
        print _('Error in initialising logging system.')
        traceback.print_exc()
        sys.exit(1)
    else:
        atexit.register(logging.shutdown)

    for key, value in botslib.botsinfo():  #log start info
        botsglobal.logger.info(u'%s: "%s".', key, value)
    #**************connect to database**********************************
    try:
        botsinit.connect()
    except:
        botsglobal.logger.exception(
            _(u'Could not connect to database. Database settings are in bots/config/settings.py.'
              ))
        sys.exit(1)
    else:
        botsglobal.logger.info(_(u'Connected to database.'))
        atexit.register(botsglobal.db.close)
    #initialise user exits for the whole bots-engine (this script file)
    try:
        userscript, scriptname = botslib.botsimport('routescripts',
                                                    'botsengine')
    except ImportError:
        userscript = scriptname = None

    #**************handle database lock****************************************
    #try to set a lock on the database; if this is not possible, the database is already locked. Either:
    #1 another instance bots bots-engine is (still) running
    #2 or bots-engine had a severe crash.
    #What to do?
    #first: check ts of database lock. If below a certain value (set in bots.ini) we assume an other instance is running. Exit quietly - no errors, no logging.
    #                                  else: Warn user, give advise on what to do. gather data: nr files in, errors.
    #next:  warn with report & logging. advise a crashrecovery.
    if not botslib.set_database_lock():
        if '--crashrecovery' in commandstorun:  #user starts recovery operation; the databaselock is ignored; the databaselock is unlocked when routes have run.
            commandstorun = ['--crashrecovery']  #is an exclusive option!
        else:
            #when scheduling bots it is possible that the last run is still running. Check if maxruntime has passed:
            vanaf = datetime.datetime.today() - datetime.timedelta(
                minutes=botsglobal.ini.getint('settings', 'maxruntime', 60))
            for row in botslib.query(
                    '''SELECT ts FROM mutex WHERE ts < %(vanaf)s ''',
                {'vanaf': vanaf}):
                warn = _(
                    u'!Bots database is locked!\nBots-engine has ended in an unexpected way during the last run.\nThis happens, but is very very rare.\nPossible causes: bots-engine terminated by user, system crash, power-down, etc.\nA forced retry of the last run is advised; bots will (try to) repair the last run.'
                )
                botsglobal.logger.critical(warn)
                botslib.sendbotserrorreport(
                    _(u'[Bots severe error]Database is locked'), warn)
                #add: count errors etc.
                sys.exit(1)
            else:  #maxruntime has not passed. Exit silently, nothing reported
                botsglobal.logger.info(
                    _(u'Database is locked, but "maxruntime" has not been exceeded.'
                      ))
                sys.exit(3)
    else:
        if '--crashrecovery' in commandstorun:  #user starts recovery operation but there is no databaselock.
            warn = _(
                u'User started a forced retry of the last run.\nOnly use this when the database is locked.\nThe database was not locked (database is OK).\nSo Bots has done nothing now.'
            )
            botsglobal.logger.error(warn)
            botslib.sendbotserrorreport(
                _(u'[Bots Error Report] User started a forced retry of last run, but this was not needed'
                  ), warn)
            botslib.remove_database_lock()
            sys.exit(1)

    #*************get list of routes to run****************************************
    #~ raise Exception('locked database')       #for testing database lock: abort, database will be locked
    if routestorun:
        botsglobal.logger.info(u'Run routes from command line: "%s".',
                               str(routestorun))
    else:  # no routes from command line parameters: fetch all active routes from database
        for row in botslib.query(
                '''SELECT DISTINCT idroute
                                    FROM routes
                                    WHERE active=%(active)s 
                                    AND (notindefaultrun=%(notindefaultrun)s OR notindefaultrun IS NULL)
                                    ORDER BY idroute ''', {
                    'active': True,
                    'notindefaultrun': False
                }):
            routestorun.append(row['idroute'])
        botsglobal.logger.info(_(u'Run active routes from database: "%s".'),
                               str(routestorun))
    #routestorun is now either a list with routes from commandline, or the list of active routes for the routes table in the db.
    #**************run the routes for retry, retransmit and new runs*************************************
    try:
        #commandstorun determines the type(s) of run
        #routes to run is a listof the routes that are runs (for each command to run
        #botsglobal.incommunicate is used to control if there is communication in; only 'new' incommunicates.
        #botsglobal.minta4query controls which ta's are queried by the routes.
        #stuff2evaluate controls what is evaluated in automatic maintenance.
        errorinrun = 0  #detect if there has been some error. Only used for good exit() code
        botsglobal.incommunicate = False
        if '--crashrecovery' in commandstorun:
            botsglobal.logger.info(_(u'Run crash recovery.'))
            stuff2evaluate = botslib.set_minta4query_crashrecovery()
            if stuff2evaluate:
                router.routedispatcher(routestorun)
                errorinrun += automaticmaintenance.evaluate(
                    '--crashrecovery', stuff2evaluate)
            else:
                botsglobal.logger.info(
                    _(u'No retry of the last run - there was no last run.'))
            if userscript and hasattr(userscript, 'postcrashrecovery'):
                botslib.runscript(userscript,
                                  scriptname,
                                  'postcrashrecovery',
                                  routestorun=routestorun)
        if '--retrycommunication' in commandstorun:
            botsglobal.logger.info(_(u'Run communication retry.'))
            stuff2evaluate = router.routedispatcher(routestorun,
                                                    '--retrycommunication')
            if stuff2evaluate:
                errorinrun += automaticmaintenance.evaluate(
                    '--retrycommunication', stuff2evaluate)
            else:
                botsglobal.logger.info(
                    _(u'Run recommunicate: nothing to recommunicate.'))
            if userscript and hasattr(userscript, 'postretrycommunication'):
                botslib.runscript(userscript,
                                  scriptname,
                                  'postretrycommunication',
                                  routestorun=routestorun)
        if '--automaticretrycommunication' in commandstorun:
            botsglobal.logger.info(_(u'Run automatic communication retry.'))
            stuff2evaluate = router.routedispatcher(
                routestorun, '--automaticretrycommunication')
            if stuff2evaluate:
                errorinrun += automaticmaintenance.evaluate(
                    '--automaticretrycommunication', stuff2evaluate)
            else:
                botsglobal.logger.info(
                    _(u'Run automatic recommunicate: nothing to recommunicate.'
                      ))
            if userscript and hasattr(userscript,
                                      'postautomaticretrycommunication'):
                botslib.runscript(userscript,
                                  scriptname,
                                  'postautomaticretrycommunication',
                                  routestorun=routestorun)
        if '--retry' in commandstorun:
            botsglobal.logger.info(u'Run retry.')
            stuff2evaluate = router.routedispatcher(routestorun, '--retry')
            if stuff2evaluate:
                errorinrun += automaticmaintenance.evaluate(
                    '--retry', stuff2evaluate)
            else:
                botsglobal.logger.info(_(u'Run retry: nothing to retry.'))
            if userscript and hasattr(userscript, 'postretry'):
                botslib.runscript(userscript,
                                  scriptname,
                                  'postretry',
                                  routestorun=routestorun)
        if '--retransmit' in commandstorun:
            botsglobal.logger.info(u'Run retransmit.')
            stuff2evaluate = router.routedispatcher(routestorun,
                                                    '--retransmit')
            if stuff2evaluate:
                errorinrun += automaticmaintenance.evaluate(
                    '--retransmit', stuff2evaluate)
            else:
                botsglobal.logger.info(
                    _(u'Run retransmit: nothing to retransmit.'))
            if userscript and hasattr(userscript, 'postretransmit'):
                botslib.runscript(userscript,
                                  scriptname,
                                  'postretransmit',
                                  routestorun=routestorun)
        if '--new' in commandstorun:
            botsglobal.logger.info('Run new.')
            botsglobal.incommunicate = True
            botsglobal.minta4query = 0  #meaning: reset. the actual value is set later (in routedispatcher)
            stuff2evaluate = router.routedispatcher(routestorun)
            errorinrun += automaticmaintenance.evaluate(
                '--new', stuff2evaluate)
            if userscript and hasattr(userscript, 'postnewrun'):
                botslib.runscript(userscript,
                                  scriptname,
                                  'postnewrun',
                                  routestorun=routestorun)
        if '--cleanup' in commandstorun or botsglobal.ini.get(
                'settings', 'whencleanup', 'always') == 'always':
            botsglobal.logger.debug(u'Do cleanup.')
            cleanup.cleanup()
        botslib.remove_database_lock()
    except Exception, e:
        botsglobal.logger.exception(
            _(u'Severe error in bots system:\n%s') %
            (e))  #of course this 'should' not happen.
        sys.exit(1)
Beispiel #9
0
def maxruntimeerror(logger,maxruntime,jobnumber,task_to_run):
    logger.error(u'Job %(job)s exceeded maxruntime of %(maxruntime)s minutes',job=jobnumber,maxruntime=maxruntime)
    botslib.sendbotserrorreport(u'[Bots Job Queue] - Job exceeded maximum runtime',
                                u'Job %s exceeded maxruntime of %s minutes:\n %s' % (jobnumber,maxruntime,task_to_run))
Beispiel #10
0
def start():
    ''' sysexit codes:
        0: OK, no errors
        1: (system) errors incl parsing of command line arguments
        2: bots ran OK, but there are errors/process errors  in the run
        3: Database is locked, but "maxruntime" has not been exceeded.
    '''
    #NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
    #********command line arguments**************************
    usage = '''
    This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
    Does the actual translations and communications; it's the workhorse. It does not have a fancy interface.

    Usage:
        %(name)s  [run-options] [config-option] [routes]
    Run-options (can be combined):
        --new                receive new edi files (default: if no run-option given: run as new).
        --resend             resend as indicated by user.
        --rereceive          rereceive as indicated by user.
        --automaticretrycommunication - automatically retry outgoing communication.
        --cleanup            remove older data from database.
    Config-option:
        -c<directory>        directory for configuration files (default: config).
    Routes: list of routes to run. Default: all active routes (in the database)

    '''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version}
    configdir = 'config'
    commandspossible = ['--automaticretrycommunication','--resend','--rereceive','--new']
    commandstorun = []
    routestorun = []    #list with routes to run
    do_cleanup_parameter = False
    for arg in sys.argv[1:]:
        if arg.startswith('-c'):
            configdir = arg[2:]
            if not configdir:
                print 'Error: configuration directory indicated, but no directory name.'
                sys.exit(1)
        elif arg in commandspossible:
            commandstorun.append(arg)
        elif arg == '--cleanup':
            do_cleanup_parameter = True
        elif arg in ["?", "/?",'-h', '--help'] or arg.startswith('-'):
            print usage
            sys.exit(0)
        else:   #pick up names of routes to run
            routestorun.append(arg)
    if not commandstorun and not do_cleanup_parameter:   #if no command on command line, use new (default)
        commandstorun = ['--new']
    commandstorun = [command[2:] for command in commandspossible if command in commandstorun]   #sort commands
    #***********end handling command line arguments**************************
    
    botsinit.generalinit(configdir)     #find locating of bots, configfiles, init paths etc.
    #set working directory to bots installation. advantage: when using relative paths it is clear that this point paths within bots installation. 
    os.chdir(botsglobal.ini.get('directories','botspath'))

    #**************check if another instance of bots-engine is running/if port is free******************************
    try:
        engine_socket = botslib.check_if_other_engine_is_running()
    except socket.error:
        sys.exit(3)
    else:
        atexit.register(engine_socket.close)

    #**************initialise logging******************************
    process_name = 'engine'
    botsglobal.logger = botsinit.initenginelogging(process_name)
    atexit.register(logging.shutdown)
    for key,value in botslib.botsinfo():    #log info about environement, versions, etc
        botsglobal.logger.info(u'%(key)s: "%(value)s".',{'key':key,'value':value})

    #**************connect to database**********************************
    try:
        botsinit.connect()
    except Exception as msg:
        botsglobal.logger.exception(_(u'Could not connect to database. Database settings are in bots/config/settings.py. Error: "%(msg)s".'),{'msg':msg})
        sys.exit(1)
    else:
        botsglobal.logger.info(_(u'Connected to database.'))
        atexit.register(botsglobal.db.close)
    #************initialise user exits for the whole bots-engine*************************
    try:
        userscript,scriptname = botslib.botsimport('routescripts','botsengine')
    except ImportError:      #userscript is not there; other errors like syntax errors are not catched
        userscript = scriptname = None
    #***acceptance tests: initialiase acceptance user script******************************
    acceptance_userscript = acceptance_scriptname = None
    if botsglobal.ini.getboolean('acceptance','runacceptancetest',False):
        botsglobal.logger.info(_(u'This run is an acceptance test - as indicated in option "runacceptancetest" in bots.ini.'))
        try:
            acceptance_userscript,acceptance_scriptname = botslib.botsimport('routescripts','bots_acceptancetest')
        except ImportError:
            botsglobal.logger.info(_(u'In acceptance test there is no script file "bots_acceptancetest.py" to check the results of the acceptance test.'))

    #**************handle database lock****************************************
    #set a lock on the database; if not possible, the database is locked: an earlier instance of bots-engine was terminated unexpectedly.
    if not botslib.set_database_lock():
        #for SQLite: do a integrity check on the database
        if botsglobal.settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3':
            cursor = botsglobal.db.execute('''PRAGMA integrity_check''')
            result = cursor.fetchone()
            if result[0] != u'ok':
                warn =  _(u'!Bots database is locked!\n'\
                            'Bots did an integrity check on the database, but database was not OK.\n'\
                            'Manual action is needed!\n'\
                            'Bots has stopped processing EDI files.')
                botsglobal.logger.critical(warn)
                botslib.sendbotserrorreport(_(u'[Bots severe error]Database is damaged'),warn)
                sys.exit(1)
        warn =  _(u'!Bots database is locked!\n'\
                    'Bots-engine has ended in an unexpected way during the last run.\n'\
                    'Most likely causes: sudden power-down, system crash, problems with disk I/O, bots-engine terminated by user, etc.\n'
                    'Bots will do an automatic crash recovery now.')
        botsglobal.logger.critical(warn)
        botslib.sendbotserrorreport(_(u'[Bots severe error]Database is locked'),warn)
        commandstorun.insert(0,'crashrecovery')         #there is a database lock. Add a crashrecovery as first command to run.
    atexit.register(botslib.remove_database_lock)
    #**************run the routes**********************************************
    #commandstorun determines the type(s) of run. eg: ['automaticretrycommunication','new']
    try:
        botslib.prepare_confirmrules()
        #in acceptance tests: run a user script before running eg to clean output directories******************************
        botslib.tryrunscript(acceptance_userscript,acceptance_scriptname,'pretest',routestorun=routestorun)
        botslib.tryrunscript(userscript,scriptname,'pre',commandstorun=commandstorun,routestorun=routestorun)
        errorinrun = 0      #detect if there has been some error. Only used for correct exit() code
        first_command_2_run = True
        for command in commandstorun:
            #if multiple commands in run: reports etc are based on timestamp; so there needs to be at least one second between these runs.
            if first_command_2_run:
                first_command_2_run = False
            else:
                time.sleep(1) 
            botsglobal.logger.info(_(u'Run "%(command)s".'),{'command':command})
            #************get list of routes to run*******************************
            if routestorun:
                use_routestorun = routestorun[:]
                botsglobal.logger.info(_(u'Run routes from command line: "%(routes)s".'),{'routes':str(use_routestorun)})
            elif command == 'new':  #fetch all active routes from database unless 'not in default run' or not active.
                use_routestorun = []
                for row in botslib.query('''SELECT DISTINCT idroute
                                            FROM routes
                                            WHERE active=%(active)s
                                            AND (notindefaultrun=%(notindefaultrun)s OR notindefaultrun IS NULL)
                                            ORDER BY idroute ''',
                                            {'active':True,'notindefaultrun':False}):
                    use_routestorun.append(row['idroute'])
                botsglobal.logger.info(_(u'Run active routes from database that are in default run: "%(routes)s".'),{'routes':str(use_routestorun)})
            else:   #for command other than 'new': use all active routes.
                use_routestorun = []
                for row in botslib.query('''SELECT DISTINCT idroute
                                            FROM routes
                                            WHERE active=%(active)s
                                            ORDER BY idroute ''',
                                            {'active':True}):
                    use_routestorun.append(row['idroute'])
                botsglobal.logger.info(_(u'Run all active routes from database: "%(routes)s".'),{'routes':str(use_routestorun)})
            #************run routes for this command******************************
            botslib.tryrunscript(userscript,scriptname,'pre' + command,routestorun=use_routestorun)
            errorinrun += router.rundispatcher(command,use_routestorun)
            botslib.tryrunscript(userscript,scriptname,'post' + command,routestorun=use_routestorun)
            #*********finished running routes for this command****************************
        #*********finished all commands****************************************
        botslib.tryrunscript(userscript,scriptname,'post',commandstorun=commandstorun,routestorun=routestorun)
        try:    #in acceptance tests: run a user script. no good reporting of errors/results in post-test script. Reason: this is after automaticmaintence.
            botslib.tryrunscript(acceptance_userscript,acceptance_scriptname,'posttest',routestorun=use_routestorun)
        except Exception as msg:
            print str(msg)
        
        cleanup.cleanup(do_cleanup_parameter,userscript,scriptname)
    except Exception as msg:
        botsglobal.logger.exception(_(u'Severe error in bots system:\n%(msg)s'),{'msg':str(msg)})    #of course this 'should' not happen.
        sys.exit(1)
    else:
        if errorinrun:
            sys.exit(2) #indicate: error(s) in run(s)
        else:
            sys.exit(0) #OK
Beispiel #11
0
def start():
    ''' sysexit codes:
        0: OK, no errors
        1: (system) errors incl parsing of command line arguments
        2: bots ran OK, but there are errors/process errors  in the run
        3: Database is locked, but "maxruntime" has not been exceeded.
    '''
    #NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
    #********command line arguments**************************
    usage = '''
    This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
    Does the actual translations and communications; it's the workhorse. It does not have a fancy interface.

    Usage:
        %(name)s  [run-options] [config-option] [routes]
    Run-options (can be combined):
        --new                receive new edi files (default: if no run-option given: run as new).
        --resend             resend as indicated by user.
        --rereceive          rereceive as indicated by user.
        --automaticretrycommunication - automatically retry outgoing communication.
        --cleanup            remove older data from database.
    Config-option:
        -c<directory>        directory for configuration files (default: config).
    Routes: list of routes to run. Default: all active routes (in the database)

    ''' % {
        'name': os.path.basename(sys.argv[0]),
        'version': botsglobal.version
    }
    configdir = 'config'
    commandspossible = [
        '--automaticretrycommunication', '--resend', '--rereceive', '--new'
    ]
    commandstorun = []
    routestorun = []  #list with routes to run
    do_cleanup_parameter = False
    for arg in sys.argv[1:]:
        if arg.startswith('-c'):
            configdir = arg[2:]
            if not configdir:
                print 'Error: configuration directory indicated, but no directory name.'
                sys.exit(1)
        elif arg in commandspossible:
            commandstorun.append(arg)
        elif arg == '--cleanup':
            do_cleanup_parameter = True
        elif arg in ["?", "/?", '-h', '--help'] or arg.startswith('-'):
            print usage
            sys.exit(0)
        else:  #pick up names of routes to run
            routestorun.append(arg)
    if not commandstorun and not do_cleanup_parameter:  #if no command on command line, use new (default)
        commandstorun = ['--new']
    commandstorun = [
        command[2:] for command in commandspossible if command in commandstorun
    ]  #sort commands
    #***********end handling command line arguments**************************

    botsinit.generalinit(
        configdir)  #find locating of bots, configfiles, init paths etc.
    #set working directory to bots installation. advantage: when using relative paths it is clear that this point paths within bots installation.
    os.chdir(botsglobal.ini.get('directories', 'botspath'))

    #**************check if another instance of bots-engine is running/if port is free******************************
    try:
        engine_socket = botslib.check_if_other_engine_is_running()
    except socket.error:
        sys.exit(3)
    else:
        atexit.register(engine_socket.close)

    #**************initialise logging******************************
    process_name = 'engine'
    botsglobal.logger = botsinit.initenginelogging(process_name)
    atexit.register(logging.shutdown)
    for key, value in botslib.botsinfo(
    ):  #log info about environement, versions, etc
        botsglobal.logger.info(u'%(key)s: "%(value)s".', {
            'key': key,
            'value': value
        })

    #**************connect to database**********************************
    try:
        botsinit.connect()
    except Exception as msg:
        botsglobal.logger.exception(
            _(u'Could not connect to database. Database settings are in bots/config/settings.py. Error: "%(msg)s".'
              ), {'msg': msg})
        sys.exit(1)
    else:
        botsglobal.logger.info(_(u'Connected to database.'))
        atexit.register(botsglobal.db.close)
    #************initialise user exits for the whole bots-engine*************************
    try:
        userscript, scriptname = botslib.botsimport('routescripts',
                                                    'botsengine')
    except botslib.BotsImportError:  #userscript is not there; other errors like syntax errors are not catched
        userscript = scriptname = None
    #***acceptance tests: initialiase acceptance user script******************************
    acceptance_userscript = acceptance_scriptname = None
    if botsglobal.ini.getboolean('acceptance', 'runacceptancetest', False):
        botsglobal.logger.info(
            _(u'This run is an acceptance test - as indicated in option "runacceptancetest" in bots.ini.'
              ))
        try:
            acceptance_userscript, acceptance_scriptname = botslib.botsimport(
                'routescripts', 'bots_acceptancetest')
        except botslib.BotsImportError:
            botsglobal.logger.info(
                _(u'In acceptance test there is no script file "bots_acceptancetest.py" to check the results of the acceptance test.'
                  ))

    #**************handle database lock****************************************
    #set a lock on the database; if not possible, the database is locked: an earlier instance of bots-engine was terminated unexpectedly.
    if not botslib.set_database_lock():
        #for SQLite: do a integrity check on the database
        if botsglobal.settings.DATABASES['default'][
                'ENGINE'] == 'django.db.backends.sqlite3':
            cursor = botsglobal.db.execute('''PRAGMA integrity_check''')
            result = cursor.fetchone()
            if result[0] != u'ok':
                warn =  _(u'!Bots database is locked!\n'\
                            'Bots did an integrity check on the database, but database was not OK.\n'\
                            'Manual action is needed!\n'\
                            'Bots has stopped processing EDI files.')
                botsglobal.logger.critical(warn)
                botslib.sendbotserrorreport(
                    _(u'[Bots severe error]Database is damaged'), warn)
                sys.exit(1)
        warn =  _(u'!Bots database is locked!\n'\
                    'Bots-engine has ended in an unexpected way during the last run.\n'\
                    'Most likely causes: sudden power-down, system crash, problems with disk I/O, bots-engine terminated by user, etc.\n'
                    'Bots will do an automatic crash recovery now.')
        botsglobal.logger.critical(warn)
        botslib.sendbotserrorreport(
            _(u'[Bots severe error]Database is locked'), warn)
        commandstorun.insert(
            0, 'crashrecovery'
        )  #there is a database lock. Add a crashrecovery as first command to run.
    atexit.register(botslib.remove_database_lock)

    warnings.simplefilter('error', UnicodeWarning)

    #**************run the routes**********************************************
    #commandstorun determines the type(s) of run. eg: ['automaticretrycommunication','new']
    try:
        botslib.prepare_confirmrules()
        #in acceptance tests: run a user script before running eg to clean output directories******************************
        botslib.tryrunscript(acceptance_userscript,
                             acceptance_scriptname,
                             'pretest',
                             routestorun=routestorun)
        botslib.tryrunscript(userscript,
                             scriptname,
                             'pre',
                             commandstorun=commandstorun,
                             routestorun=routestorun)
        errorinrun = 0  #detect if there has been some error. Only used for correct exit() code
        first_command_2_run = True
        for command in commandstorun:
            #if multiple commands in run: reports etc are based on timestamp; so there needs to be at least one second between these runs.
            if first_command_2_run:
                first_command_2_run = False
            else:
                time.sleep(1)
            botsglobal.logger.info(_(u'Run "%(command)s".'),
                                   {'command': command})
            #************get list of routes to run*******************************
            if routestorun:
                use_routestorun = routestorun[:]
                botsglobal.logger.info(
                    _(u'Run routes from command line: "%(routes)s".'),
                    {'routes': unicode(use_routestorun)})
            elif command == 'new':  #fetch all active routes from database unless 'not in default run' or not active.
                use_routestorun = []
                for row in botslib.query(
                        '''SELECT DISTINCT idroute
                                            FROM routes
                                            WHERE active=%(active)s
                                            AND (notindefaultrun=%(notindefaultrun)s OR notindefaultrun IS NULL)
                                            ORDER BY idroute ''', {
                            'active': True,
                            'notindefaultrun': False
                        }):
                    use_routestorun.append(row['idroute'])
                botsglobal.logger.info(
                    _(u'Run active routes from database that are in default run: "%(routes)s".'
                      ), {'routes': unicode(use_routestorun)})
            else:  #for command other than 'new': use all active routes.
                use_routestorun = []
                for row in botslib.query(
                        '''SELECT DISTINCT idroute
                                            FROM routes
                                            WHERE active=%(active)s
                                            ORDER BY idroute ''',
                    {'active': True}):
                    use_routestorun.append(row['idroute'])
                botsglobal.logger.info(
                    _(u'Run all active routes from database: "%(routes)s".'),
                    {'routes': unicode(use_routestorun)})
            #************run routes for this command******************************
            botslib.tryrunscript(userscript,
                                 scriptname,
                                 'pre' + command,
                                 routestorun=use_routestorun)
            errorinrun += router.rundispatcher(command, use_routestorun)
            botslib.tryrunscript(userscript,
                                 scriptname,
                                 'post' + command,
                                 routestorun=use_routestorun)
            #*********finished running routes for this command****************************
        #*********finished all commands****************************************
        botslib.tryrunscript(userscript,
                             scriptname,
                             'post',
                             commandstorun=commandstorun,
                             routestorun=routestorun)
        try:  #in acceptance tests: run a user script. no good reporting of errors/results in post-test script. Reason: this is after automaticmaintence.
            botslib.tryrunscript(acceptance_userscript,
                                 acceptance_scriptname,
                                 'posttest',
                                 routestorun=use_routestorun)
        except Exception as msg:
            print unicode(msg)

        cleanup.cleanup(do_cleanup_parameter, userscript, scriptname)
    except Exception as msg:
        botsglobal.logger.exception(
            _(u'Severe error in bots system:\n%(msg)s'),
            {'msg': unicode(msg)})  #of course this 'should' not happen.
        sys.exit(1)
    else:
        if errorinrun:
            sys.exit(2)  #indicate: error(s) in run(s)
        else:
            sys.exit(0)  #OK
Beispiel #12
0
def start():
    #exit codes:
    # 0: OK, no errors
    # 1: (system) errors
    # 2: bots ran OK, but there are errors/process errors  in the run
    # 3: Database is locked, but "maxruntime" has not been exceeded.
    #********command line arguments**************************
    commandspossible = ['--new','--retry','--retransmit','--cleanup','--crashrecovery','--retrycommunication','--automaticretrycommunication']
    commandstorun = []
    routestorun = []    #list with routes to run
    configdir = 'config'
    for arg in sys.argv[1:]:
        if not arg:
            continue
        if arg.startswith('-c'):
            configdir = arg[2:]
            if not configdir:
                print 'Configuration directory indicated, but no directory name.'
                sys.exit(1)
        elif arg in commandspossible:
            commandstorun.append(arg)
        elif arg in ["?", "/?"] or arg.startswith('-'):
            showusage()
            sys.exit(0)
        else:   #pick up names of routes to run
            routestorun.append(arg)
    if not commandstorun:   #if no command on command line, use new (default)
        commandstorun = ['--new']
    #**************init general: find locating of bots, configfiles, init paths etc.****************
    botsinit.generalinit(configdir)
    #set current working directory to botspath
    #~ old_current_directory = os.getcwdu()
    os.chdir(botsglobal.ini.get('directories','botspath'))
    #**************initialise logging******************************
    try:
        botsinit.initenginelogging()
    except:
        print _('Error in initialising logging system.')
        traceback.print_exc()
        sys.exit(1)
    else:
        atexit.register(logging.shutdown)
        
    for key,value in botslib.botsinfo():    #log start info
        botsglobal.logger.info(u'%s: "%s".',key,value)
    #**************connect to database**********************************
    try:
        botsinit.connect() 
    except:
        botsglobal.logger.exception(_(u'Could not connect to database. Database settings are in bots/config/settings.py.'))
        sys.exit(1)
    else:
        botsglobal.logger.info(_(u'Connected to database.'))
        atexit.register(botsglobal.db.close)
    #initialise user exits for the whole bots-engine (this script file)
    try:
        userscript,scriptname = botslib.botsimport('routescripts','botsengine')
    except ImportError:
        userscript = scriptname = None
        
    #**************handle database lock****************************************
    #try to set a lock on the database; if this is not possible, the database is already locked. Either:
    #1 another instance bots bots-engine is (still) running
    #2 or bots-engine had a severe crash.
    #What to do? 
    #first: check ts of database lock. If below a certain value (set in bots.ini) we assume an other instance is running. Exit quietly - no errors, no logging.
    #                                  else: Warn user, give advise on what to do. gather data: nr files in, errors.
    #next:  warn with report & logging. advise a crashrecovery.
    if not botslib.set_database_lock():
        if '--crashrecovery' in commandstorun:    #user starts recovery operation; the databaselock is ignored; the databaselock is unlocked when routes have run.
            commandstorun = ['--crashrecovery']  #is an exclusive option!
        else:
            #when scheduling bots it is possible that the last run is still running. Check if maxruntime has passed:
            vanaf = datetime.datetime.today() - datetime.timedelta(minutes=botsglobal.ini.getint('settings','maxruntime',60))
            for row in botslib.query('''SELECT ts FROM mutex WHERE ts < %(vanaf)s ''',{'vanaf':vanaf}):
                warn = _(u'!Bots database is locked!\nBots-engine has ended in an unexpected way during the last run.\nThis happens, but is very very rare.\nPossible causes: bots-engine terminated by user, system crash, power-down, etc.\nA forced retry of the last run is advised; bots will (try to) repair the last run.')
                botsglobal.logger.critical(warn)
                botslib.sendbotserrorreport(_(u'[Bots severe error]Database is locked'),warn)
                #add: count errors etc.
                sys.exit(1)
            else:   #maxruntime has not passed. Exit silently, nothing reported
                botsglobal.logger.info(_(u'Database is locked, but "maxruntime" has not been exceeded.'))
                sys.exit(3)
    else:
        if '--crashrecovery' in commandstorun:    #user starts recovery operation but there is no databaselock.
            warn = _(u'User started a forced retry of the last run.\nOnly use this when the database is locked.\nThe database was not locked (database is OK).\nSo Bots has done nothing now.')
            botsglobal.logger.error(warn)
            botslib.sendbotserrorreport(_(u'[Bots Error Report] User started a forced retry of last run, but this was not needed'),warn)
            botslib.remove_database_lock()
            sys.exit(1)
            
    #*************get list of routes to run****************************************
    #~ raise Exception('locked database')       #for testing database lock: abort, database will be locked
    if routestorun: 
        botsglobal.logger.info(u'Run routes from command line: "%s".',str(routestorun))
    else:   # no routes from command line parameters: fetch all active routes from database
        for row in botslib.query('''SELECT DISTINCT idroute
                                    FROM routes
                                    WHERE active=%(active)s 
                                    AND (notindefaultrun=%(notindefaultrun)s OR notindefaultrun IS NULL)
                                    ORDER BY idroute ''',
                                    {'active':True,'notindefaultrun':False}):
            routestorun.append(row['idroute'])
        botsglobal.logger.info(_(u'Run active routes from database: "%s".'),str(routestorun))
    #routestorun is now either a list with routes from commandline, or the list of active routes for the routes table in the db.
    #**************run the routes for retry, retransmit and new runs*************************************
    try: 
        #commandstorun determines the type(s) of run
        #routes to run is a listof the routes that are runs (for each command to run
        #botsglobal.incommunicate is used to control if there is communication in; only 'new' incommunicates.
        #botsglobal.minta4query controls which ta's are queried by the routes.
        #stuff2evaluate controls what is evaluated in automatic maintenance.
        errorinrun = 0      #detect if there has been some error. Only used for good exit() code
        botsglobal.incommunicate = False
        if '--crashrecovery' in commandstorun:
            botsglobal.logger.info(_(u'Run crash recovery.'))
            stuff2evaluate = botslib.set_minta4query_crashrecovery()
            if stuff2evaluate:
                router.routedispatcher(routestorun)
                errorinrun +=  automaticmaintenance.evaluate('--crashrecovery',stuff2evaluate)
            else:
                botsglobal.logger.info(_(u'No retry of the last run - there was no last run.'))
            if userscript and hasattr(userscript,'postcrashrecovery'):
                botslib.runscript(userscript,scriptname,'postcrashrecovery',routestorun=routestorun)
        if '--retrycommunication' in commandstorun:
            botsglobal.logger.info(_(u'Run communication retry.'))
            stuff2evaluate = router.routedispatcher(routestorun,'--retrycommunication')
            if stuff2evaluate:
                errorinrun +=  automaticmaintenance.evaluate('--retrycommunication',stuff2evaluate)
            else:
                botsglobal.logger.info(_(u'Run recommunicate: nothing to recommunicate.'))
            if userscript and hasattr(userscript,'postretrycommunication'):
                botslib.runscript(userscript,scriptname,'postretrycommunication',routestorun=routestorun)
        if '--automaticretrycommunication' in commandstorun:
            botsglobal.logger.info(_(u'Run automatic communication retry.'))
            stuff2evaluate = router.routedispatcher(routestorun,'--automaticretrycommunication')
            if stuff2evaluate:
                errorinrun +=  automaticmaintenance.evaluate('--automaticretrycommunication',stuff2evaluate)
            else:
                botsglobal.logger.info(_(u'Run automatic recommunicate: nothing to recommunicate.'))
            if userscript and hasattr(userscript,'postautomaticretrycommunication'):
                botslib.runscript(userscript,scriptname,'postautomaticretrycommunication',routestorun=routestorun)
        if '--retry' in commandstorun:
            botsglobal.logger.info(u'Run retry.')
            stuff2evaluate = router.routedispatcher(routestorun,'--retry')
            if stuff2evaluate:
                errorinrun +=  automaticmaintenance.evaluate('--retry',stuff2evaluate)
            else:
                botsglobal.logger.info(_(u'Run retry: nothing to retry.'))
            if userscript and hasattr(userscript,'postretry'):
                botslib.runscript(userscript,scriptname,'postretry',routestorun=routestorun)
        if '--retransmit' in commandstorun:
            botsglobal.logger.info(u'Run retransmit.')
            stuff2evaluate = router.routedispatcher(routestorun,'--retransmit')
            if stuff2evaluate:
                errorinrun +=  automaticmaintenance.evaluate('--retransmit',stuff2evaluate)
            else:
                botsglobal.logger.info(_(u'Run retransmit: nothing to retransmit.'))
            if userscript and hasattr(userscript,'postretransmit'):
                botslib.runscript(userscript,scriptname,'postretransmit',routestorun=routestorun)
        if '--new' in commandstorun:
            botsglobal.logger.info('Run new.')
            botsglobal.incommunicate = True
            botsglobal.minta4query = 0  #meaning: reset. the actual value is set later (in routedispatcher)
            stuff2evaluate = router.routedispatcher(routestorun)
            errorinrun +=  automaticmaintenance.evaluate('--new',stuff2evaluate)
            if userscript and hasattr(userscript,'postnewrun'):
                botslib.runscript(userscript,scriptname,'postnewrun',routestorun=routestorun)
        if '--cleanup' in commandstorun or botsglobal.ini.get('settings','whencleanup','always')=='always':
            botsglobal.logger.debug(u'Do cleanup.')
            cleanup.cleanup()
        botslib.remove_database_lock()
    except Exception,e:
        botsglobal.logger.exception(_(u'Severe error in bots system:\n%s')%(e))    #of course this 'should' not happen. 
        sys.exit(1)
Beispiel #13
0
            botsglobal.logger.info(_(u'In acceptance test there is no script file "bots_acceptancetest.py" to check the results of the acceptance test.'))

    #**************handle database lock****************************************
    #set a lock on the database; if not possible, the database is locked: an earlier instance of bots-engine was terminated unexpectedly.
    if not botslib.set_database_lock():
        #for SQLite: do a integrity check on the database
        if botsglobal.settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3':
            cursor = botsglobal.db.execute('''PRAGMA integrity_check''')
            result = cursor.fetchone()
            if result[0] != u'ok':
                warn =  _(u'!Bots database is locked!\n'\
                            'Bots did an integrity check on the database, but database was not OK.\n'\
                            'Manual action is needed!\n'\
                            'Bots has stopped processing EDI files.')
                botsglobal.logger.critical(warn)
                botslib.sendbotserrorreport(_(u'[Bots severe error]Database is damaged'),warn)
                sys.exit(1)
        warn =  _(u'!Bots database is locked!\n'\
                    'Bots-engine has ended in an unexpected way during the last run.\n'\
                    'Most likely causes: sudden power-down, system crash, problems with disk I/O, bots-engine terminated by user, etc.\n'
                    'Bots will do an automatic crash recovery now.')
        botsglobal.logger.critical(warn)
        botslib.sendbotserrorreport(_(u'[Bots severe error]Database is locked'),warn)
        commandstorun.insert(0,'crashrecovery')         #there is a database lock. Add a crashrecovery as first command to run.
    atexit.register(botslib.remove_database_lock)
    #**************run the routes**********************************************
    #commandstorun determines the type(s) of run. eg: ['automaticretrycommunication','new']
    #for each command: run all routes
    #    for each route: run all seq
    try:
        #in acceptance tests: run a user script before running eg to clean output directories******************************
Beispiel #14
0
def maxruntimeerror(logger,maxruntime,jobnumber,task_to_run):
    logger.error(u'Job %(job)s exceeded maxruntime of %(maxruntime)s minutes',{'job':jobnumber,'maxruntime':maxruntime})
    botslib.sendbotserrorreport(u'[Bots Job Queue] - Job exceeded maximum runtime',
                                u'Job %(job)s exceeded maxruntime of %(maxruntime)s minutes:\n %(task)s' % {'job':jobnumber,'maxruntime':maxruntime,'task':task_to_run})
def maxruntimeerror(logger,maxruntime,jobnumber,task_to_run):
    logger.error(u'Job %(job)s exceeded maxruntime of %(maxruntime)s minutes',{'job':jobnumber,'maxruntime':maxruntime})
    botslib.sendbotserrorreport(u'[Bots Job Queue] - Job exceeded maximum runtime',
                                u'Job %(job)s exceeded maxruntime of %(maxruntime)s minutes:\n %(task)s' % {'job':jobnumber,'maxruntime':maxruntime,'task':task_to_run})