Exemplo n.º 1
0
def start():
    #********command line arguments**************************
    editype = ''
    messagetype = ''
    configdir = 'config'
    for arg in sys.argv[1:]:
        if not arg:
            continue
        if arg.startswith('-c'):
            configdir = arg[2:]
            if not configdir:
                print '    !!Indicated Bots should use specific .ini file but no file name was given.'
                showusage()
        elif arg in ["?", "/?"] or arg.startswith('-'):
            showusage()
        else:
            if not editype:
                editype = arg
            else:
                messagetype = arg
    if not (editype and messagetype):
        print '    !!Both editype and messagetype are required.'
        showusage()
    #********end handling command line arguments**************************

    try:
        botsinit.generalinit(configdir)
        botsinit.initenginelogging()
        grammar.grammarread(editype, messagetype)
    except:
        print 'Found error in grammar:'
        print botslib.txtexc()
    else:
        print 'OK - no error found in grammar'
Exemplo n.º 2
0
def start():
    #NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
    #***command line arguments**************************
    usage = '''
    This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
    A utility to generate the index file of a plugin; this can be seen as a database dump of the configuration.
    This is eg useful for version control.
    Usage:
        %(name)s  -c<directory>
    Options:
        -c<directory>   directory for configuration files (default: config).
        
    '''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version}
    configdir = 'config'
    for arg in sys.argv[1:]:
        if arg.startswith('-c'):
            configdir = arg[2:]
            if not configdir:
                print 'Error: configuration directory indicated, but no directory name.'
                sys.exit(1)
        else:
            print usage
            sys.exit(0)
    #***end handling command line arguments**************************
    botsinit.generalinit(configdir)     #find locating of bots, configfiles, init paths etc.
    import pluglib              #import here, import at start of file gives error; first initialize.
    usersys = botsglobal.ini.get('directories','usersysabs')
    index_filename = os.path.join(usersys,'index.py')
    dummy_for_cleaned_data = {'databaseconfiguration':True,'umlists':True,'databasetransactions':False}
    pluglib.make_index(dummy_for_cleaned_data,index_filename)
Exemplo n.º 3
0
def startmulti(grammardir, editype):
    ''' specialized tool for bulk checking of grammars while developing botsgrammars 
        grammardir: directory with gramars (eg bots/usersys/grammars/edifact)
        editype: eg edifact
    '''
    configdir = 'config'
    botsinit.generalinit(
        configdir)  #find locating of bots, configfiles, init paths etc.
    process_name = 'grammarcheck'
    botsglobal.logger = botsinit.initenginelogging(process_name)
    atexit.register(logging.shutdown)

    for filename in glob.iglob(grammardir):
        filename_basename = os.path.basename(filename)
        if filename_basename in ['__init__.py', 'envelope.py']:
            continue
        if filename_basename.startswith(
                'edifact') or filename_basename.startswith(
                    'records') or filename_basename.endswith('records.py'):
            continue
        if filename_basename.endswith('pyc'):
            continue
        filename_noextension = os.path.splitext(filename_basename)[0]
        try:
            grammar.grammarread(editype, filename_noextension)
        except:
            print botslib.txtexc()
            print '\n'
        else:
            print 'OK - no error found in grammar', filename, '\n'
Exemplo n.º 4
0
def start():
    #NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
    #***command line arguments**************************
    usage = '''
    This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
    A utility to generate the index file of a plugin; this can be seen as a database dump of the configuration.
    This is eg useful for version control.
    Usage:
        %(name)s  -c<directory>
    Options:
        -c<directory>   directory for configuration files (default: config).
        
    '''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version}
    configdir = 'config'
    for arg in sys.argv[1:]:
        if arg.startswith('-c'):
            configdir = arg[2:]
            if not configdir:
                print 'Error: configuration directory indicated, but no directory name.'
                sys.exit(1)
        else:
            print usage
            sys.exit(0)
    #***end handling command line arguments**************************
    botsinit.generalinit(configdir)     #find locating of bots, configfiles, init paths etc.
    import pluglib              #import here, import at start of file gives error; first initialize.
    usersys = botsglobal.ini.get('directories','usersysabs')
    index_filename = os.path.join(usersys,'index.py')
    dummy_for_cleaned_data = {'databaseconfiguration':True,'umlists':botsglobal.ini.getboolean('settings','codelists_in_plugin',True),'databasetransactions':False}
    pluglib.make_index(dummy_for_cleaned_data,index_filename)
Exemplo n.º 5
0
def start():
    #NOTE bots is always on PYTHONPATH!!! - otherwise it will not start.
    #***command line arguments**************************
    configdir = 'config'
    for arg in sys.argv[1:]:
        if not arg:
            continue
        if arg.startswith('-c'):
            configdir = arg[2:]
            if not configdir:
                print 'Configuration directory indicated, but no directory name.'
                sys.exit(1)
        elif arg in ["?", "/?"] or arg.startswith('-'):
            showusage()
        else:
            showusage()
    
    #***init general: find locating of bots, configfiles, init paths etc.***********************
    botsinit.generalinit(configdir)

    #***initialise logging. This logging only contains the logging from bots-webserver, not from cherrypy.
    botsglobal.logger = logging.getLogger('bots-webserver')
    botsglobal.logger.setLevel(logging.DEBUG)
    h = TimedRotatingFileHandler(botslib.join(botsglobal.ini.get('directories','logging'),'webserver.log'), backupCount=10)
    fileformat = logging.Formatter("%(asctime)s %(levelname)-8s: %(message)s",'%Y%m%d %H:%M:%S')
    h.setFormatter(fileformat)
    botsglobal.logger.addHandler(h)
    
    #***init cherrypy as webserver*********************************************
    #global configuration for cherrypy
    cherrypy.config.update({'global': {'log.screen': False, 'server.environment': botsglobal.ini.get('webserver','environment','production')}})
    #cherrypy handling of static files
    conf = {'/': {'tools.staticdir.on' : True,'tools.staticdir.dir' : 'media' ,'tools.staticdir.root': botsglobal.ini.get('directories','botspath')}}
    servestaticfiles = cherrypy.tree.mount(None, '/media', conf)    #None: no cherrypy application (as this only serves static files)
    #cherrypy handling of django
    servedjango = WSGIHandler()     #was: servedjango = AdminMediaHandler(WSGIHandler())  but django does not need the AdminMediaHandler in this setup. is much faster.
    #cherrypy uses a dispatcher in order to handle the serving of static files and django.
    dispatcher = wsgiserver.WSGIPathInfoDispatcher({'/': servedjango, '/media': servestaticfiles})
    botswebserver = wsgiserver.CherryPyWSGIServer(bind_addr=('0.0.0.0', botsglobal.ini.getint('webserver','port',8080)), wsgi_app=dispatcher, server_name=botsglobal.ini.get('webserver','name','bots-webserver'))
    botsglobal.logger.info(_(u'Bots web-server started.'))
    #handle ssl: cherrypy < 3.2 always uses pyOpenssl. cherrypy >= 3.2 uses python buildin ssl (python >= 2.6 has buildin support for ssl).
    ssl_certificate = botsglobal.ini.get('webserver','ssl_certificate',None)
    ssl_private_key = botsglobal.ini.get('webserver','ssl_private_key',None)
    if ssl_certificate and ssl_private_key:
        if cherrypy.__version__ >= '3.2.0':
            adapter_class = wsgiserver.get_ssl_adapter_class('builtin')
            botswebserver.ssl_adapter = adapter_class(ssl_certificate,ssl_private_key)
        else:
            #but: pyOpenssl should be there!
            botswebserver.ssl_certificate = ssl_certificate
            botswebserver.ssl_private_key = ssl_private_key
        botsglobal.logger.info(_(u'Bots web-server uses ssl (https).'))
    else:
        botsglobal.logger.info(_(u'Bots web-server uses plain http (no ssl).'))
    
    #***start the cherrypy webserver.
    try:
        botswebserver.start()
    except KeyboardInterrupt:
        botswebserver.stop()
Exemplo n.º 6
0
def start(configdir):
    botsinit.generalinit(
        configdir)  #find locating of bots, configfiles, init paths etc.
    process_name = 'apache_webserver_' + configdir
    botsglobal.logger = botsinit.initserverlogging(
        process_name
    )  #initialise file-logging for web-server. This logging only contains the logging from bots-webserver.
Exemplo n.º 7
0
def startmulti(grammardir,editype):
    ''' specialized tool for bulk checking of grammars while developing botsgrammars 
        grammardir: directory with gramars (eg bots/usersys/grammars/edifact)
        editype: eg edifact
    '''
    configdir = 'config'
    botsinit.generalinit(configdir)     #find locating of bots, configfiles, init paths etc.
    process_name = 'grammarcheck'
    botsglobal.logger = botsinit.initenginelogging(process_name)
    atexit.register(logging.shutdown)
    
    for filename in glob.iglob(grammardir):
        filename_basename = os.path.basename(filename)
        if filename_basename in ['__init__.py','envelope.py']:
            continue
        if filename_basename.startswith('edifact') or filename_basename.startswith('records') or filename_basename.endswith('records.py'):
            continue
        if filename_basename.endswith('pyc'):
            continue
        filename_noextension = os.path.splitext(filename_basename)[0]
        try:
            grammar.grammarread(editype,filename_noextension)
        except:
            print botslib.txtexc()
            print '\n'
        else:
            print 'OK - no error found in grammar',filename,'\n'
Exemplo n.º 8
0
def start():
    #********command line arguments**************************
    editype =''
    messagetype = ''
    configdir = 'config'
    for arg in sys.argv[1:]:
        if not arg:
            continue
        if arg.startswith('-c'):
            configdir = arg[2:]
            if not configdir:
                print '    !!Indicated Bots should use specific .ini file but no file name was given.'
                showusage()
        elif arg in ["?", "/?"] or arg.startswith('-'):
            showusage()
        else:
            if not editype:
                editype = arg
            else:
                messagetype = arg
    if not (editype and messagetype):
        print '    !!Both editype and messagetype are required.'
        showusage()
    #********end handling command line arguments**************************

    try:
        botsinit.generalinit(configdir)
        botsinit.initenginelogging()
        grammar.grammarread(editype,messagetype)
    except:
        print 'Found error in grammar:'
        print botslib.txtexc()
    else:
        print 'OK - no error found in grammar'
Exemplo n.º 9
0
def start():
    #NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
    #********command line arguments**************************
    usage = '''
    This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
    Checks a Bots grammar. Same checks are used as in translations with bots-engine. Searches for grammar in 
    regular place: bots/usersys/grammars/<editype>/<messagetype>.py  (even if a path is passed).
    
    Usage:  %(name)s  -c<directory> <editype> <messagetype>
       or   %(name)s  -c<directory> <path to grammar>
    Options:
        -c<directory>   directory for configuration files (default: config).
    Examples:
        %(name)s -cconfig  edifact  ORDERSD96AUNEAN008
        %(name)s -cconfig  C:/python27/lib/site-packages/bots/usersys/grammars/edifact/ORDERSD96AUNEAN008.py
        
    '''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version}
    configdir = 'config'
    editype =''
    messagetype = ''
    for arg in sys.argv[1:]:
        if arg.startswith('-c'):
            configdir = arg[2:]
            if not configdir:
                print 'Error: configuration directory indicated, but no directory name.'
                sys.exit(1)
        elif arg in ["?", "/?",'-h', '--help'] or arg.startswith('-'):
            print usage
            sys.exit(0)
        else:
            if os.path.isfile(arg):
                p1,p2 = os.path.split(arg)
                editype = os.path.basename(p1)
                messagetype,ext = os.path.splitext(p2)
                messagetype = str(messagetype)
                print 'grammarcheck',editype,messagetype
            elif not editype:
                editype = arg
            else:
                messagetype = arg
    if not (editype and messagetype):
        print 'Error: both editype and messagetype, or a file path, are required.'
        sys.exit(1)
    #***end handling command line arguments**************************
    botsinit.generalinit(configdir)     #find locating of bots, configfiles, init paths etc.
    process_name = 'grammarcheck'
    botsglobal.logger = botsinit.initenginelogging(process_name)
    atexit.register(logging.shutdown)

    try:
        grammar.grammarread(editype,messagetype)
    except:
        print 'Found error in grammar: ',botslib.txtexc()
        sys.exit(1)
    else:
        print 'OK - no error found in grammar'
        sys.exit(0)
Exemplo n.º 10
0
def start():
    #NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
    #***command line arguments**************************
    usage = '''
    This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
    Server program that ensures only a single bots-engine runs at any time, and no engine run requests are 
    lost/discarded. Each request goes to a queue and is run in sequence when the previous run completes. 
    Use of the job queue is optional and must be configured in bots.ini (jobqueue section, enabled = True).
    Usage:
        %(name)s  -c<directory>
    Options:
        -c<directory>   directory for configuration files (default: config).
        
    '''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version}
    configdir = 'config'
    for arg in sys.argv[1:]:
        if arg.startswith('-c'):
            configdir = arg[2:]
            if not configdir:
                print 'Error: configuration directory indicated, but no directory name.'
                sys.exit(1)
        else:
            print usage
            sys.exit(0)
    #***end handling command line arguments**************************
    botsinit.generalinit(configdir)     #find locating of bots, configfiles, init paths etc.
    if not botsglobal.ini.getboolean('jobqueue','enabled',False):
        print 'Error: bots jobqueue cannot start; not enabled in %s/bots.ini' % configdir
        sys.exit(1)
    process_name = 'jobqueue'
    logger = botsinit.initserverlogging(process_name)
    logger.log(25,u'Bots %(process_name)s started.',{'process_name':process_name})
    logger.log(25,u'Bots %(process_name)s configdir: "%(configdir)s".',{'process_name':process_name,'configdir':botsglobal.ini.get('directories','config')})
    port = botsglobal.ini.getint('jobqueue','port',28082)
    logger.log(25,u'Bots %(process_name)s listens for xmlrpc at port: "%(port)s".',{'process_name':process_name,'port':port})

    #start launcher thread
    lauchfrequency = botsglobal.ini.getint('jobqueue','lauchfrequency',5)
    maxruntime = botsglobal.ini.getint('settings','maxruntime',60)
    launcher_thread = threading.Thread(name='launcher', target=launcher, args=(logger,port,lauchfrequency,maxruntime))
    launcher_thread.daemon = True
    launcher_thread.start()
    logger.info(u'Jobqueue launcher started.')

    #the main thread is the xmlrpc server: all adding, getting etc for jobqueue is done via xmlrpc.
    logger.info(u'Jobqueue server started.')
    server = SimpleXMLRPCServer(('localhost', port),logRequests=False)        
    server.register_instance(Jobqueue(logger))
    try:
        server.serve_forever()
    except KeyboardInterrupt:
        pass
    
    sys.exit(0)
Exemplo n.º 11
0
def start():
    # NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
    # ***command line arguments**************************
    # if config (-c option) is before job argument, will be used as config-dir of job2queue it self and not for job
    # if config (-c option) is after job argument, use both as config-dir of job2queue and as -c option of job.
    # if config (-c option) is before and after job argument, use only the after...could change that but seems not to be useful.
    usage = """
    This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
    Places a job in the bots jobqueue. Bots jobqueue takes care of correct processing of jobs.    
    Usage:
        %(name)s  [-c<directory>] [-p<priority>] job [job-parameters]
    Options:
        -c<directory>   directory for configuration files (default: config).
        -p<priority>    priority of job, 1-9 (default: 5, highest priority is 1).
    Example of usage:
        %(name)s bots-engine.py
        %(name)s python2.7 /usr/local/bin/bots-engine.py
        %(name)s -p1 python2.7 /usr/local/bin/bots-engine.py -cconfig2 myroute
        
    """ % {
        "name": os.path.basename(sys.argv[0]),
        "version": botsglobal.version,
    }
    configdir = "config"  # default value
    priority = 5  # default value
    task_args = []
    for arg in sys.argv[1:]:
        if arg.startswith("-c"):
            configdir = arg[2:]
            if not configdir:
                print "Error: configuration directory indicated, but no directory name."
                sys.exit(1)
            if task_args:
                task_args.append(arg)
        elif arg.startswith("-p"):
            try:
                priority = int(arg[2:])
            except:
                print "Error: priority should be numeric (1=highest, 9=lowest)."
                sys.exit(1)
        elif arg in ["?", "/?", "-h", "--help"]:
            print usage
            sys.exit(0)
        else:
            task_args.append(arg)
    # ***end handling command line arguments**************************
    botsinit.generalinit(configdir)  # needed to read config
    if not botsglobal.ini.getboolean("jobqueue", "enabled", False):
        print "Error: bots jobqueue cannot start; not enabled in %s/bots.ini" % configdir
        sys.exit(1)

    terug = send_job_to_jobqueue(task_args, priority)
    print JOBQUEUEMESSAGE2TXT[terug]
    sys.exit(terug)
Exemplo n.º 12
0
def start():
    #NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
    #***command line arguments**************************
    usage = '''
    This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
    Server program that ensures only a single bots-engine runs at any time, and no engine run requests are 
    lost/discarded. Each request goes to a queue and is run in sequence when the previous run completes. 
    Use of the job queue is optional and must be configured in bots.ini (jobqueue section, enabled = True).
    Usage:
        %(name)s  -c<directory>
    Options:
        -c<directory>   directory for configuration files (default: config).
        
    '''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version}
    configdir = 'config'
    for arg in sys.argv[1:]:
        if arg.startswith('-c'):
            configdir = arg[2:]
            if not configdir:
                print 'Error: configuration directory indicated, but no directory name.'
                sys.exit(1)
        else:
            print usage
            sys.exit(0)
    #***end handling command line arguments**************************
    botsinit.generalinit(configdir)     #find locating of bots, configfiles, init paths etc.
    if not botsglobal.ini.getboolean('jobqueue','enabled',False):
        print 'Error: bots jobqueue cannot start; not enabled in %s/bots.ini' % configdir
        sys.exit(1)
    process_name = 'jobqueue'
    logger = botsinit.initserverlogging(process_name)
    logger.log(25,u'Bots %(process_name)s started.',process_name=process_name)
    logger.log(25,u'Bots %(process_name)s configdir: "%(configdir)s".',process_name=process_name,configdir=botsglobal.ini.get('directories','config'))
    port = botsglobal.ini.getint('jobqueue','port',28082)
    logger.log(25,u'Bots %(process_name)s listens for xmlrpc at port: "%(port)s".',process_name=process_name,port=port)

    #start launcher thread
    lauchfrequency = botsglobal.ini.getint('jobqueue','lauchfrequency',5)
    maxruntime = botsglobal.ini.getint('settings','maxruntime',60)
    launcher_thread = threading.Thread(name='launcher', target=launcher, args=(logger,port,lauchfrequency,maxruntime))
    launcher_thread.daemon = True
    launcher_thread.start()
    logger.info(u'Jobqueue launcher started.')

    # this main thread is the jobqserver (the xmlrpc server for handling jobqueue)
    logger.info(u'Jobqueue server started.')
    server = SimpleXMLRPCServer(('localhost', port),logRequests=False)        
    server.register_instance(Jobqueue(logger))
    try:
        server.serve_forever()
    except KeyboardInterrupt:
        pass
    
    sys.exit(0)
Exemplo n.º 13
0
def start():
    #NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
    #***command line arguments**************************
    #if config (-c option) is before job argument, will be used as config-dir of job2queue it self and not for job
    #if config (-c option) is after job argument, use both as config-dir of job2queue and as -c option of job.
    #if config (-c option) is before and after job argument, use only the after...could change that but seems not to be useful.
    usage = '''
    This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
    Places a job in the bots jobqueue. Bots jobqueue takes care of correct processing of jobs.    
    Usage:
        %(name)s  [-c<directory>] [-p<priority>] job [job-parameters]
    Options:
        -c<directory>   directory for configuration files (default: config).
        -p<priority>    priority of job, 1-9 (default: 5, highest priority is 1).
    Example of usage:
        %(name)s bots-engine.py
        %(name)s python2.7 /usr/local/bin/bots-engine.py
        %(name)s -p1 python2.7 /usr/local/bin/bots-engine.py -cconfig2 myroute
        
    ''' % {
        'name': os.path.basename(sys.argv[0]),
        'version': botsglobal.version
    }
    configdir = 'config'  #default value
    priority = 5  #default value
    task_args = []
    for arg in sys.argv[1:]:
        if arg.startswith('-c'):
            configdir = arg[2:]
            if not configdir:
                print 'Error: configuration directory indicated, but no directory name.'
                sys.exit(1)
            if task_args:
                task_args.append(arg)
        elif arg.startswith('-p'):
            try:
                priority = int(arg[2:])
            except:
                print 'Error: priority should be numeric (1=highest, 9=lowest).'
                sys.exit(1)
        elif arg in ["?", "/?", '-h', '--help']:
            print usage
            sys.exit(0)
        else:
            task_args.append(arg)
    #***end handling command line arguments**************************
    botsinit.generalinit(configdir)  #needed to read config
    if not botsglobal.ini.getboolean('jobqueue', 'enabled', False):
        print 'Error: bots jobqueue cannot start; not enabled in %s/bots.ini' % configdir
        sys.exit(1)

    terug = send_job_to_jobqueue(task_args, priority)
    print JOBQUEUEMESSAGE2TXT[terug]
    sys.exit(terug)
Exemplo n.º 14
0
def start():
    # NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
    # ***command line arguments**************************
    usage = '''
    This is "%(name)s", developed by Jagged Peak Inc.
    A utility to read a plugin. Enter a comma separated list of plugin files.
    Usage:
        %(name)s -c<config_directory> -d<plugin_directory> <pluginfile(s)>
    Options:
        -c<directory>   directory for configuration files (default: config).
        -d<directory>   directory for plugin file(s) default: D:/BOTS/plugins
        
    ''' % {
        'name': os.path.basename(sys.argv[0]),
        'version': botsglobal.version
    }

    configdir = 'config'
    plugindir = 'D:/BOTS/plugins'
    pluginfiles = ''

    # Handle cmd arguments
    for arg in sys.argv[1:]:
        if arg.startswith('-c'):
            configdir = arg[2:]
            if not configdir:
                print 'Error: configuration directory indicated, but no directory name.'
                sys.exit(1)
            if arg.startswith('-d'):
                plugindir = arg[2:]
            if not configdir:
                print 'Error: plugin directory indicated, but no directory name.'
                sys.exit(1)
        else:
            pluginfiles = arg

    if not pluginfiles:
        print 'Plugin file was not entered. See usage'
        print usage
        sys.exit(1)

    # Init required vars and libs
    botsinit.generalinit(
        configdir)  # find locating of bots, configfiles, init paths etc.
    process_name = 'plug_read'
    botsglobal.logger = botsinit.initserverlogging(process_name)
    import pluglib  # import here, import at start of file gives error; first initialize.

    # Call read plugin for each file
    pluginfiles_list = pluginfiles.split(',')
    for pluginfile in pluginfiles_list:
        plugin_file_path = os.path.join(plugindir, pluginfile)
        pluglib.read_plugin(plugin_file_path)
Exemplo n.º 15
0
def start():
    #********command line arguments**************************
    usage = '''
    This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
    Updates existing bots database to version %(version)s

    Usage:
        %(name)s  [config-option]
    Options:
        -c<directory>        directory for configuration files (default: config).

    '''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version}
    configdir = 'config'
    for arg in sys.argv[1:]:
        if arg.startswith('-c'):
            configdir = arg[2:]
            if not configdir:
                print 'Error: configuration directory indicated, but no directory name.'
                sys.exit(3)
        else:   #pick up names of routes to run
            print usage
            sys.exit(0)
    #***end handling command line arguments**************************
    botsinit.generalinit(configdir)     #find locating of bots, configfiles, init paths etc.

    #**************check if another instance of bots-engine is running/if port is free******************************
    try:
        engine_socket = botslib.check_if_other_engine_is_running()
    except socket.error:
        sys.exit(3)
    else:
        atexit.register(engine_socket.close)

    #**************initialise logging******************************
    process_name = 'updatedatabase'
    botsglobal.logger = botsinit.initenginelogging(process_name)
    atexit.register(logging.shutdown)
    for key,value in botslib.botsinfo():    #log info about environement, versions, etc
        botsglobal.logger.info(u'%(key)s: "%(value)s".',{'key':key,'value':value})

    #**************connect to database**********************************
    try:
        botsinit.connect()
    except Exception,msg:
        botsglobal.logger.exception(_(u'Could not connect to database. Database settings are in bots/config/settings.py. Error: "%(msg)s".'),{'msg':msg})
        sys.exit(3)
Exemplo n.º 16
0
def startmulti(grammardir,editype):
    ''' used in seperate tool for bulk checking of gramamrs while developing edifact->botsgramamrs '''
    import glob
    botsinit.generalinit('config')
    botsinit.initenginelogging()
    for filename in glob.glob(grammardir):
        filename_basename = os.path.basename(filename)
        filename_noextension = os.path.splitext(filename_basename)[0]
        if filename_basename in ['__init__.py']:
            continue
        if filename_basename.startswith('edifact'):
            continue
        if filename_basename.startswith('records') or filename_basename.endswith('records.py'):
            continue
        try:
            grammar.grammarread(editype,filename_noextension)
        except:
            #~ print 'Found error in grammar:',filename
            print botslib.txtexc()
            print '\n'
        else:
            print 'OK - no error found in grammar',filename,'\n'
Exemplo n.º 17
0
def startmulti(grammardir, editype):
    ''' used in seperate tool for bulk checking of gramamrs while developing edifact->botsgramamrs '''
    import glob
    botsinit.generalinit('config')
    botsinit.initenginelogging()
    for filename in glob.glob(grammardir):
        filename_basename = os.path.basename(filename)
        filename_noextension = os.path.splitext(filename_basename)[0]
        if filename_basename in ['__init__.py']:
            continue
        if filename_basename.startswith('edifact'):
            continue
        if filename_basename.startswith(
                'records') or filename_basename.endswith('records.py'):
            continue
        try:
            grammar.grammarread(editype, filename_noextension)
        except:
            #~ print 'Found error in grammar:',filename
            print botslib.txtexc()
            print '\n'
        else:
            print 'OK - no error found in grammar', filename, '\n'
Exemplo n.º 18
0
def start():
    #exit codes:
    # 0: OK, no errors
    # 1: (system) errors
    # 2: bots ran OK, but there are errors/process errors  in the run
    # 3: Database is locked, but "maxruntime" has not been exceeded.
    #********command line arguments**************************
    commandspossible = [
        '--new', '--retry', '--retransmit', '--cleanup', '--crashrecovery',
        '--retrycommunication', '--automaticretrycommunication'
    ]
    commandstorun = []
    routestorun = []  #list with routes to run
    configdir = 'config'
    for arg in sys.argv[1:]:
        if not arg:
            continue
        if arg.startswith('-c'):
            configdir = arg[2:]
            if not configdir:
                print 'Configuration directory indicated, but no directory name.'
                sys.exit(1)
        elif arg in commandspossible:
            commandstorun.append(arg)
        elif arg in ["?", "/?"] or arg.startswith('-'):
            showusage()
            sys.exit(0)
        else:  #pick up names of routes to run
            routestorun.append(arg)
    if not commandstorun:  #if no command on command line, use new (default)
        commandstorun = ['--new']
    #**************init general: find locating of bots, configfiles, init paths etc.****************
    botsinit.generalinit(configdir)
    #set current working directory to botspath
    #~ old_current_directory = os.getcwdu()
    os.chdir(botsglobal.ini.get('directories', 'botspath'))
    #**************initialise logging******************************
    try:
        botsinit.initenginelogging()
    except:
        print _('Error in initialising logging system.')
        traceback.print_exc()
        sys.exit(1)
    else:
        atexit.register(logging.shutdown)

    for key, value in botslib.botsinfo():  #log start info
        botsglobal.logger.info(u'%s: "%s".', key, value)
    #**************connect to database**********************************
    try:
        botsinit.connect()
    except:
        botsglobal.logger.exception(
            _(u'Could not connect to database. Database settings are in bots/config/settings.py.'
              ))
        sys.exit(1)
    else:
        botsglobal.logger.info(_(u'Connected to database.'))
        atexit.register(botsglobal.db.close)
    #initialise user exits for the whole bots-engine (this script file)
    try:
        userscript, scriptname = botslib.botsimport('routescripts',
                                                    'botsengine')
    except ImportError:
        userscript = scriptname = None

    #**************handle database lock****************************************
    #try to set a lock on the database; if this is not possible, the database is already locked. Either:
    #1 another instance bots bots-engine is (still) running
    #2 or bots-engine had a severe crash.
    #What to do?
    #first: check ts of database lock. If below a certain value (set in bots.ini) we assume an other instance is running. Exit quietly - no errors, no logging.
    #                                  else: Warn user, give advise on what to do. gather data: nr files in, errors.
    #next:  warn with report & logging. advise a crashrecovery.
    if not botslib.set_database_lock():
        if '--crashrecovery' in commandstorun:  #user starts recovery operation; the databaselock is ignored; the databaselock is unlocked when routes have run.
            commandstorun = ['--crashrecovery']  #is an exclusive option!
        else:
            #when scheduling bots it is possible that the last run is still running. Check if maxruntime has passed:
            vanaf = datetime.datetime.today() - datetime.timedelta(
                minutes=botsglobal.ini.getint('settings', 'maxruntime', 60))
            for row in botslib.query(
                    '''SELECT ts FROM mutex WHERE ts < %(vanaf)s ''',
                {'vanaf': vanaf}):
                warn = _(
                    u'!Bots database is locked!\nBots-engine has ended in an unexpected way during the last run.\nThis happens, but is very very rare.\nPossible causes: bots-engine terminated by user, system crash, power-down, etc.\nA forced retry of the last run is advised; bots will (try to) repair the last run.'
                )
                botsglobal.logger.critical(warn)
                botslib.sendbotserrorreport(
                    _(u'[Bots severe error]Database is locked'), warn)
                #add: count errors etc.
                sys.exit(1)
            else:  #maxruntime has not passed. Exit silently, nothing reported
                botsglobal.logger.info(
                    _(u'Database is locked, but "maxruntime" has not been exceeded.'
                      ))
                sys.exit(3)
    else:
        if '--crashrecovery' in commandstorun:  #user starts recovery operation but there is no databaselock.
            warn = _(
                u'User started a forced retry of the last run.\nOnly use this when the database is locked.\nThe database was not locked (database is OK).\nSo Bots has done nothing now.'
            )
            botsglobal.logger.error(warn)
            botslib.sendbotserrorreport(
                _(u'[Bots Error Report] User started a forced retry of last run, but this was not needed'
                  ), warn)
            botslib.remove_database_lock()
            sys.exit(1)

    #*************get list of routes to run****************************************
    #~ raise Exception('locked database')       #for testing database lock: abort, database will be locked
    if routestorun:
        botsglobal.logger.info(u'Run routes from command line: "%s".',
                               str(routestorun))
    else:  # no routes from command line parameters: fetch all active routes from database
        for row in botslib.query(
                '''SELECT DISTINCT idroute
                                    FROM routes
                                    WHERE active=%(active)s 
                                    AND (notindefaultrun=%(notindefaultrun)s OR notindefaultrun IS NULL)
                                    ORDER BY idroute ''', {
                    'active': True,
                    'notindefaultrun': False
                }):
            routestorun.append(row['idroute'])
        botsglobal.logger.info(_(u'Run active routes from database: "%s".'),
                               str(routestorun))
    #routestorun is now either a list with routes from commandline, or the list of active routes for the routes table in the db.
    #**************run the routes for retry, retransmit and new runs*************************************
    try:
        #commandstorun determines the type(s) of run
        #routes to run is a listof the routes that are runs (for each command to run
        #botsglobal.incommunicate is used to control if there is communication in; only 'new' incommunicates.
        #botsglobal.minta4query controls which ta's are queried by the routes.
        #stuff2evaluate controls what is evaluated in automatic maintenance.
        errorinrun = 0  #detect if there has been some error. Only used for good exit() code
        botsglobal.incommunicate = False
        if '--crashrecovery' in commandstorun:
            botsglobal.logger.info(_(u'Run crash recovery.'))
            stuff2evaluate = botslib.set_minta4query_crashrecovery()
            if stuff2evaluate:
                router.routedispatcher(routestorun)
                errorinrun += automaticmaintenance.evaluate(
                    '--crashrecovery', stuff2evaluate)
            else:
                botsglobal.logger.info(
                    _(u'No retry of the last run - there was no last run.'))
            if userscript and hasattr(userscript, 'postcrashrecovery'):
                botslib.runscript(userscript,
                                  scriptname,
                                  'postcrashrecovery',
                                  routestorun=routestorun)
        if '--retrycommunication' in commandstorun:
            botsglobal.logger.info(_(u'Run communication retry.'))
            stuff2evaluate = router.routedispatcher(routestorun,
                                                    '--retrycommunication')
            if stuff2evaluate:
                errorinrun += automaticmaintenance.evaluate(
                    '--retrycommunication', stuff2evaluate)
            else:
                botsglobal.logger.info(
                    _(u'Run recommunicate: nothing to recommunicate.'))
            if userscript and hasattr(userscript, 'postretrycommunication'):
                botslib.runscript(userscript,
                                  scriptname,
                                  'postretrycommunication',
                                  routestorun=routestorun)
        if '--automaticretrycommunication' in commandstorun:
            botsglobal.logger.info(_(u'Run automatic communication retry.'))
            stuff2evaluate = router.routedispatcher(
                routestorun, '--automaticretrycommunication')
            if stuff2evaluate:
                errorinrun += automaticmaintenance.evaluate(
                    '--automaticretrycommunication', stuff2evaluate)
            else:
                botsglobal.logger.info(
                    _(u'Run automatic recommunicate: nothing to recommunicate.'
                      ))
            if userscript and hasattr(userscript,
                                      'postautomaticretrycommunication'):
                botslib.runscript(userscript,
                                  scriptname,
                                  'postautomaticretrycommunication',
                                  routestorun=routestorun)
        if '--retry' in commandstorun:
            botsglobal.logger.info(u'Run retry.')
            stuff2evaluate = router.routedispatcher(routestorun, '--retry')
            if stuff2evaluate:
                errorinrun += automaticmaintenance.evaluate(
                    '--retry', stuff2evaluate)
            else:
                botsglobal.logger.info(_(u'Run retry: nothing to retry.'))
            if userscript and hasattr(userscript, 'postretry'):
                botslib.runscript(userscript,
                                  scriptname,
                                  'postretry',
                                  routestorun=routestorun)
        if '--retransmit' in commandstorun:
            botsglobal.logger.info(u'Run retransmit.')
            stuff2evaluate = router.routedispatcher(routestorun,
                                                    '--retransmit')
            if stuff2evaluate:
                errorinrun += automaticmaintenance.evaluate(
                    '--retransmit', stuff2evaluate)
            else:
                botsglobal.logger.info(
                    _(u'Run retransmit: nothing to retransmit.'))
            if userscript and hasattr(userscript, 'postretransmit'):
                botslib.runscript(userscript,
                                  scriptname,
                                  'postretransmit',
                                  routestorun=routestorun)
        if '--new' in commandstorun:
            botsglobal.logger.info('Run new.')
            botsglobal.incommunicate = True
            botsglobal.minta4query = 0  #meaning: reset. the actual value is set later (in routedispatcher)
            stuff2evaluate = router.routedispatcher(routestorun)
            errorinrun += automaticmaintenance.evaluate(
                '--new', stuff2evaluate)
            if userscript and hasattr(userscript, 'postnewrun'):
                botslib.runscript(userscript,
                                  scriptname,
                                  'postnewrun',
                                  routestorun=routestorun)
        if '--cleanup' in commandstorun or botsglobal.ini.get(
                'settings', 'whencleanup', 'always') == 'always':
            botsglobal.logger.debug(u'Do cleanup.')
            cleanup.cleanup()
        botslib.remove_database_lock()
    except Exception, e:
        botsglobal.logger.exception(
            _(u'Severe error in bots system:\n%s') %
            (e))  #of course this 'should' not happen.
        sys.exit(1)
Exemplo n.º 19
0
def start():
    #NOTE bots is always on PYTHONPATH!!! - otherwise it will not start.
    #********command line arguments**************************
    configdir = 'config'
    for arg in sys.argv[1:]:
        if not arg:
            continue
        if arg.startswith('-c'):
            configdir = arg[2:]
            if not configdir:
                print 'Configuration directory indicated, but no directory name.'
                sys.exit(1)
        elif arg in ["?", "/?"] or arg.startswith('-'):
            showusage()
        else:
            showusage()
    
    #init general: find locating of bots, configfiles, init paths etc.***********************
    botsinit.generalinit(configdir)

    #init cherrypy; only needed for webserver. *********************************************
    cherrypy.config.update({'global': { 'tools.staticdir.root': botsglobal.ini.get('directories','botspath'),
                                        'server.socket_host' : "0.0.0.0",       #to what IP addresses should be server. 0.0.0.0: all. See cherrypy docs
                                        'server.socket_port': botsglobal.ini.getint('webserver','port',8080),
                                        'server.environment': botsglobal.ini.get('webserver','environment','development'),    # development production
                                        'log.screen': False,
                                        #~ 'log.error_file': '',    #set later to rotating log file
                                        #~ 'log.access_file': '',    #set later to rotating log file
                                        }})
    conf = {'/': {'tools.staticdir.on' : True,'tools.staticdir.dir' : 'media' }}
            #~ '/favicon.ico': {'tools.staticfile.on': True,'tools.staticfile.filename': '/home/hje/botsup-django/bots/media/images/favicon.ico'}}
    cherrypy.tree.graft(AdminMediaHandler(WSGIHandler()), '/')
    myroot = Root()
    myappl = cherrypy.tree.mount(myroot, '/media', conf)    #myappl is needed to set logging 



    botsglobal.logger = logging.getLogger('webserver')
    botsglobal.logger.setLevel(logging.DEBUG)
    h = logging.handlers.TimedRotatingFileHandler(botslib.join(botsglobal.ini.get('directories','logging'),'webserver.log'),when='midnight', backupCount=10)
    fileformat = logging.Formatter("%(asctime)s %(levelname)-8s %(name)s : %(message)s",'%Y%m%d %H:%M:%S')
    h.setFormatter(fileformat)
    # add rotating file handler to main logger
    botsglobal.logger.addHandler(h)
    



    # Make RotatingFileHandler for the error log.
    #~ h = logging.handlers.TimedRotatingFileHandler(botslib.join(botsglobal.ini.get('directories','logging'),'webserver.log'),when='midnight', backupCount=10)
    #~ fileformat = logging.Formatter("%(asctime)s %(levelname)-8s %(name)s : %(message)s",'%Y%m%d %H:%M:%S')
    #~ h.setLevel(logging.INFO)
    #~ h.setFormatter(fileformat)
    #~ myappl.log.error_log.addHandler(h)

    # MakeRotatingFileHandler for the access log.
    #~ h = logging.handlers.TimedRotatingFileHandler(os.path.normpath(os.path.join(botsglobal.ini.get('directories','botspath'), 'botssys/logging/webserver.log')),when='midnight', backupCount=10)
    #~ h.setLevel(logging.DEBUG)
    #~ myappl.log.access_log.addHandler(h)
    #~ botsglobal.logger = myappl.log.access_log 
    
    #write start info to cherrypy log********************************************
    botsglobal.logger.info(_(u'Bots web server started.'))
    botsglobal.logger.info(_(u'Python version: "%s".'),sys.version)
    botsglobal.logger.info(_(u'Django version: "%s".'),django.VERSION)
    
    #start cherrypy *********************************************************************
    cherrypy.engine.start()
    cherrypy.engine.block()
Exemplo n.º 20
0
def start():
    #********command line arguments**************************
    usage = '''
    This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
    Creates a grammar from an xml file.'
    Usage:'
        %(name)s  -c<directory>  <xml_file>  <xml_grammar_file>
    Options:
        -c<directory>      directory for configuration files (default: config).
        -a                 all xml elements as records
        <xml_file>         name of the xml file to read
        <xml_grammar_file> name of the grammar file to write
    
    '''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version}
    configdir = 'config'
    edifile =''
    botsgrammarfilename = ''
    allrecords = False
    for arg in sys.argv[1:]:
        if arg.startswith('-c'):
            configdir = arg[2:]
            if not configdir:
                print 'Error: configuration directory indicated, but no directory name.'
                sys.exit(1)
        elif arg.startswith('-a'):
            allrecords = True
        elif arg in ["?", "/?",'-h', '--help'] or arg.startswith('-'):
            print usage
            sys.exit(0)
        else:
            if not edifile:
                edifile = arg
            else:
                botsgrammarfilename = arg
    if not edifile or not botsgrammarfilename:
        print 'Error: both edifile and grammarfile are required.'
        sys.exit(0)
    #***end handling command line arguments**************************
    botsinit.generalinit(configdir)     #find locating of bots, configfiles, init paths etc.
    process_name = 'xml2botsgrammar'
    botsglobal.logger = botsinit.initenginelogging(process_name)
    atexit.register(logging.shutdown)

    targetNamespace = ''
    #*******************************************************************
    #***add classes for handling editype xml to inmessage
    #*******************************************************************
    if allrecords:
        #~ editype = 'xmlforgrammar_allrecords'
        inmessage.xmlforgrammar = xmlforgrammar_allrecords
    else:
        #~ editype = 'xmlforgrammar' 
        inmessage.xmlforgrammar = xmlforgrammar
    #make inmessage object: read the xml file
    inn = inmessage.parse_edi_file(editype='xmlforgrammar',messagetype='',filename=edifile)
    inn.checkforerrorlist() #no exception if infile has been lexed and parsed OK else raises an error
    #make outmessage object; nothing is 'filled' yet. In mapping tree is filled; nothing is written to file.
    out = outmessage.outmessage_init(editype='xmlnocheck',messagetype='',filename='',divtext='',topartner='')    
    
    #***mapping: make 'normalised' out-tree suited for writing as a grammar***************************************************
    mpath_root = [OrderedDict({'BOTSID':inn.root.record['BOTSID'],'BOTSIDnr':'1'})] #handle root
    out.put(*mpath_root)
    map_writefields(out,inn.root,mpath_root)
    
    #walk tree; write results to out-tree
    mpath_start = []
    for node_instance,mpath in map_treewalker(inn.root,mpath_start):
        mpath.append(OrderedDict({'BOTSID':node_instance.record['BOTSID']}))
        if out.get(*mpath) is None:     #if node does not exist: write it.
            out.put(*mpath)
        map_writefields(out,node_instance,mpath)
    #***mapping is done 

    #***convert out-tree to grammar
    structure = []
    recorddefs = {}
    tree2grammar(out.root,structure,recorddefs)

    #***write grammar to file
    grammar2file(botsgrammarfilename,structure,recorddefs,targetNamespace)
Exemplo n.º 21
0
def start():
    ''' sysexit codes:
        0: OK, no errors
        1: (system) errors incl parsing of command line arguments
        2: bots ran OK, but there are errors/process errors  in the run
        3: Database is locked, but "maxruntime" has not been exceeded.
    '''
    #NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
    #********command line arguments**************************
    usage = '''
    This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
    Does the actual translations and communications; it's the workhorse. It does not have a fancy interface.

    Usage:
        %(name)s  [run-options] [config-option] [routes]
    Run-options (can be combined):
        --new                receive new edi files (default: if no run-option given: run as new).
        --resend             resend as indicated by user.
        --rereceive          rereceive as indicated by user.
        --automaticretrycommunication - automatically retry outgoing communication.
        --cleanup            remove older data from database.
    Config-option:
        -c<directory>        directory for configuration files (default: config).
    Routes: list of routes to run. Default: all active routes (in the database)

    '''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version}
    configdir = 'config'
    commandspossible = ['--automaticretrycommunication','--resend','--rereceive','--new']
    commandstorun = []
    routestorun = []    #list with routes to run
    do_cleanup_parameter = False
    for arg in sys.argv[1:]:
        if arg.startswith('-c'):
            configdir = arg[2:]
            if not configdir:
                print 'Error: configuration directory indicated, but no directory name.'
                sys.exit(1)
        elif arg in commandspossible:
            commandstorun.append(arg)
        elif arg == '--cleanup':
            do_cleanup_parameter = True
        elif arg in ["?", "/?",'-h', '--help'] or arg.startswith('-'):
            print usage
            sys.exit(0)
        else:   #pick up names of routes to run
            routestorun.append(arg)
    if not commandstorun and not do_cleanup_parameter:   #if no command on command line, use new (default)
        commandstorun = ['--new']
    commandstorun = [command[2:] for command in commandspossible if command in commandstorun]   #sort commands
    #***********end handling command line arguments**************************
    
    botsinit.generalinit(configdir)     #find locating of bots, configfiles, init paths etc.
    #set working directory to bots installation. advantage: when using relative paths it is clear that this point paths within bots installation. 
    os.chdir(botsglobal.ini.get('directories','botspath'))

    #**************check if another instance of bots-engine is running/if port is free******************************
    try:
        engine_socket = botslib.check_if_other_engine_is_running()
    except socket.error:
        sys.exit(3)
    else:
        atexit.register(engine_socket.close)

    #**************initialise logging******************************
    process_name = 'engine'
    botsglobal.logger = botsinit.initenginelogging(process_name)
    atexit.register(logging.shutdown)
    for key,value in botslib.botsinfo():    #log info about environement, versions, etc
        botsglobal.logger.info(u'%(key)s: "%(value)s".',{'key':key,'value':value})

    #**************connect to database**********************************
    try:
        botsinit.connect()
    except Exception as msg:
        botsglobal.logger.exception(_(u'Could not connect to database. Database settings are in bots/config/settings.py. Error: "%(msg)s".'),{'msg':msg})
        sys.exit(1)
    else:
        botsglobal.logger.info(_(u'Connected to database.'))
        atexit.register(botsglobal.db.close)
    #************initialise user exits for the whole bots-engine*************************
    try:
        userscript,scriptname = botslib.botsimport('routescripts','botsengine')
    except ImportError:      #userscript is not there; other errors like syntax errors are not catched
        userscript = scriptname = None
    #***acceptance tests: initialiase acceptance user script******************************
    acceptance_userscript = acceptance_scriptname = None
    if botsglobal.ini.getboolean('acceptance','runacceptancetest',False):
        botsglobal.logger.info(_(u'This run is an acceptance test - as indicated in option "runacceptancetest" in bots.ini.'))
        try:
            acceptance_userscript,acceptance_scriptname = botslib.botsimport('routescripts','bots_acceptancetest')
        except ImportError:
            botsglobal.logger.info(_(u'In acceptance test there is no script file "bots_acceptancetest.py" to check the results of the acceptance test.'))

    #**************handle database lock****************************************
    #set a lock on the database; if not possible, the database is locked: an earlier instance of bots-engine was terminated unexpectedly.
    if not botslib.set_database_lock():
        #for SQLite: do a integrity check on the database
        if botsglobal.settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3':
            cursor = botsglobal.db.execute('''PRAGMA integrity_check''')
            result = cursor.fetchone()
            if result[0] != u'ok':
                warn =  _(u'!Bots database is locked!\n'\
                            'Bots did an integrity check on the database, but database was not OK.\n'\
                            'Manual action is needed!\n'\
                            'Bots has stopped processing EDI files.')
                botsglobal.logger.critical(warn)
                botslib.sendbotserrorreport(_(u'[Bots severe error]Database is damaged'),warn)
                sys.exit(1)
        warn =  _(u'!Bots database is locked!\n'\
                    'Bots-engine has ended in an unexpected way during the last run.\n'\
                    'Most likely causes: sudden power-down, system crash, problems with disk I/O, bots-engine terminated by user, etc.\n'
                    'Bots will do an automatic crash recovery now.')
        botsglobal.logger.critical(warn)
        botslib.sendbotserrorreport(_(u'[Bots severe error]Database is locked'),warn)
        commandstorun.insert(0,'crashrecovery')         #there is a database lock. Add a crashrecovery as first command to run.
    atexit.register(botslib.remove_database_lock)
    #**************run the routes**********************************************
    #commandstorun determines the type(s) of run. eg: ['automaticretrycommunication','new']
    try:
        botslib.prepare_confirmrules()
        #in acceptance tests: run a user script before running eg to clean output directories******************************
        botslib.tryrunscript(acceptance_userscript,acceptance_scriptname,'pretest',routestorun=routestorun)
        botslib.tryrunscript(userscript,scriptname,'pre',commandstorun=commandstorun,routestorun=routestorun)
        errorinrun = 0      #detect if there has been some error. Only used for correct exit() code
        first_command_2_run = True
        for command in commandstorun:
            #if multiple commands in run: reports etc are based on timestamp; so there needs to be at least one second between these runs.
            if first_command_2_run:
                first_command_2_run = False
            else:
                time.sleep(1) 
            botsglobal.logger.info(_(u'Run "%(command)s".'),{'command':command})
            #************get list of routes to run*******************************
            if routestorun:
                use_routestorun = routestorun[:]
                botsglobal.logger.info(_(u'Run routes from command line: "%(routes)s".'),{'routes':str(use_routestorun)})
            elif command == 'new':  #fetch all active routes from database unless 'not in default run' or not active.
                use_routestorun = []
                for row in botslib.query('''SELECT DISTINCT idroute
                                            FROM routes
                                            WHERE active=%(active)s
                                            AND (notindefaultrun=%(notindefaultrun)s OR notindefaultrun IS NULL)
                                            ORDER BY idroute ''',
                                            {'active':True,'notindefaultrun':False}):
                    use_routestorun.append(row['idroute'])
                botsglobal.logger.info(_(u'Run active routes from database that are in default run: "%(routes)s".'),{'routes':str(use_routestorun)})
            else:   #for command other than 'new': use all active routes.
                use_routestorun = []
                for row in botslib.query('''SELECT DISTINCT idroute
                                            FROM routes
                                            WHERE active=%(active)s
                                            ORDER BY idroute ''',
                                            {'active':True}):
                    use_routestorun.append(row['idroute'])
                botsglobal.logger.info(_(u'Run all active routes from database: "%(routes)s".'),{'routes':str(use_routestorun)})
            #************run routes for this command******************************
            botslib.tryrunscript(userscript,scriptname,'pre' + command,routestorun=use_routestorun)
            errorinrun += router.rundispatcher(command,use_routestorun)
            botslib.tryrunscript(userscript,scriptname,'post' + command,routestorun=use_routestorun)
            #*********finished running routes for this command****************************
        #*********finished all commands****************************************
        botslib.tryrunscript(userscript,scriptname,'post',commandstorun=commandstorun,routestorun=routestorun)
        try:    #in acceptance tests: run a user script. no good reporting of errors/results in post-test script. Reason: this is after automaticmaintence.
            botslib.tryrunscript(acceptance_userscript,acceptance_scriptname,'posttest',routestorun=use_routestorun)
        except Exception as msg:
            print str(msg)
        
        cleanup.cleanup(do_cleanup_parameter,userscript,scriptname)
    except Exception as msg:
        botsglobal.logger.exception(_(u'Severe error in bots system:\n%(msg)s'),{'msg':str(msg)})    #of course this 'should' not happen.
        sys.exit(1)
    else:
        if errorinrun:
            sys.exit(2) #indicate: error(s) in run(s)
        else:
            sys.exit(0) #OK
Exemplo n.º 22
0
def start():
    #exit codes:
    # 0: OK, no errors
    # 1: (system) errors
    # 2: bots ran OK, but there are errors/process errors  in the run
    # 3: Database is locked, but "maxruntime" has not been exceeded.
    #********command line arguments**************************
    commandspossible = ['--new','--retry','--retransmit','--cleanup','--crashrecovery','--retrycommunication','--automaticretrycommunication']
    commandstorun = []
    routestorun = []    #list with routes to run
    configdir = 'config'
    for arg in sys.argv[1:]:
        if not arg:
            continue
        if arg.startswith('-c'):
            configdir = arg[2:]
            if not configdir:
                print 'Configuration directory indicated, but no directory name.'
                sys.exit(1)
        elif arg in commandspossible:
            commandstorun.append(arg)
        elif arg in ["?", "/?"] or arg.startswith('-'):
            showusage()
            sys.exit(0)
        else:   #pick up names of routes to run
            routestorun.append(arg)
    if not commandstorun:   #if no command on command line, use new (default)
        commandstorun = ['--new']
    #**************init general: find locating of bots, configfiles, init paths etc.****************
    botsinit.generalinit(configdir)
    #set current working directory to botspath
    #~ old_current_directory = os.getcwdu()
    os.chdir(botsglobal.ini.get('directories','botspath'))
    #**************initialise logging******************************
    try:
        botsinit.initenginelogging()
    except:
        print _('Error in initialising logging system.')
        traceback.print_exc()
        sys.exit(1)
    else:
        atexit.register(logging.shutdown)
        
    for key,value in botslib.botsinfo():    #log start info
        botsglobal.logger.info(u'%s: "%s".',key,value)
    #**************connect to database**********************************
    try:
        botsinit.connect() 
    except:
        botsglobal.logger.exception(_(u'Could not connect to database. Database settings are in bots/config/settings.py.'))
        sys.exit(1)
    else:
        botsglobal.logger.info(_(u'Connected to database.'))
        atexit.register(botsglobal.db.close)
    #initialise user exits for the whole bots-engine (this script file)
    try:
        userscript,scriptname = botslib.botsimport('routescripts','botsengine')
    except ImportError:
        userscript = scriptname = None
        
    #**************handle database lock****************************************
    #try to set a lock on the database; if this is not possible, the database is already locked. Either:
    #1 another instance bots bots-engine is (still) running
    #2 or bots-engine had a severe crash.
    #What to do? 
    #first: check ts of database lock. If below a certain value (set in bots.ini) we assume an other instance is running. Exit quietly - no errors, no logging.
    #                                  else: Warn user, give advise on what to do. gather data: nr files in, errors.
    #next:  warn with report & logging. advise a crashrecovery.
    if not botslib.set_database_lock():
        if '--crashrecovery' in commandstorun:    #user starts recovery operation; the databaselock is ignored; the databaselock is unlocked when routes have run.
            commandstorun = ['--crashrecovery']  #is an exclusive option!
        else:
            #when scheduling bots it is possible that the last run is still running. Check if maxruntime has passed:
            vanaf = datetime.datetime.today() - datetime.timedelta(minutes=botsglobal.ini.getint('settings','maxruntime',60))
            for row in botslib.query('''SELECT ts FROM mutex WHERE ts < %(vanaf)s ''',{'vanaf':vanaf}):
                warn = _(u'!Bots database is locked!\nBots-engine has ended in an unexpected way during the last run.\nThis happens, but is very very rare.\nPossible causes: bots-engine terminated by user, system crash, power-down, etc.\nA forced retry of the last run is advised; bots will (try to) repair the last run.')
                botsglobal.logger.critical(warn)
                botslib.sendbotserrorreport(_(u'[Bots severe error]Database is locked'),warn)
                #add: count errors etc.
                sys.exit(1)
            else:   #maxruntime has not passed. Exit silently, nothing reported
                botsglobal.logger.info(_(u'Database is locked, but "maxruntime" has not been exceeded.'))
                sys.exit(3)
    else:
        if '--crashrecovery' in commandstorun:    #user starts recovery operation but there is no databaselock.
            warn = _(u'User started a forced retry of the last run.\nOnly use this when the database is locked.\nThe database was not locked (database is OK).\nSo Bots has done nothing now.')
            botsglobal.logger.error(warn)
            botslib.sendbotserrorreport(_(u'[Bots Error Report] User started a forced retry of last run, but this was not needed'),warn)
            botslib.remove_database_lock()
            sys.exit(1)
            
    #*************get list of routes to run****************************************
    #~ raise Exception('locked database')       #for testing database lock: abort, database will be locked
    if routestorun: 
        botsglobal.logger.info(u'Run routes from command line: "%s".',str(routestorun))
    else:   # no routes from command line parameters: fetch all active routes from database
        for row in botslib.query('''SELECT DISTINCT idroute
                                    FROM routes
                                    WHERE active=%(active)s 
                                    AND (notindefaultrun=%(notindefaultrun)s OR notindefaultrun IS NULL)
                                    ORDER BY idroute ''',
                                    {'active':True,'notindefaultrun':False}):
            routestorun.append(row['idroute'])
        botsglobal.logger.info(_(u'Run active routes from database: "%s".'),str(routestorun))
    #routestorun is now either a list with routes from commandline, or the list of active routes for the routes table in the db.
    #**************run the routes for retry, retransmit and new runs*************************************
    try: 
        #commandstorun determines the type(s) of run
        #routes to run is a listof the routes that are runs (for each command to run
        #botsglobal.incommunicate is used to control if there is communication in; only 'new' incommunicates.
        #botsglobal.minta4query controls which ta's are queried by the routes.
        #stuff2evaluate controls what is evaluated in automatic maintenance.
        errorinrun = 0      #detect if there has been some error. Only used for good exit() code
        botsglobal.incommunicate = False
        if '--crashrecovery' in commandstorun:
            botsglobal.logger.info(_(u'Run crash recovery.'))
            stuff2evaluate = botslib.set_minta4query_crashrecovery()
            if stuff2evaluate:
                router.routedispatcher(routestorun)
                errorinrun +=  automaticmaintenance.evaluate('--crashrecovery',stuff2evaluate)
            else:
                botsglobal.logger.info(_(u'No retry of the last run - there was no last run.'))
            if userscript and hasattr(userscript,'postcrashrecovery'):
                botslib.runscript(userscript,scriptname,'postcrashrecovery',routestorun=routestorun)
        if '--retrycommunication' in commandstorun:
            botsglobal.logger.info(_(u'Run communication retry.'))
            stuff2evaluate = router.routedispatcher(routestorun,'--retrycommunication')
            if stuff2evaluate:
                errorinrun +=  automaticmaintenance.evaluate('--retrycommunication',stuff2evaluate)
            else:
                botsglobal.logger.info(_(u'Run recommunicate: nothing to recommunicate.'))
            if userscript and hasattr(userscript,'postretrycommunication'):
                botslib.runscript(userscript,scriptname,'postretrycommunication',routestorun=routestorun)
        if '--automaticretrycommunication' in commandstorun:
            botsglobal.logger.info(_(u'Run automatic communication retry.'))
            stuff2evaluate = router.routedispatcher(routestorun,'--automaticretrycommunication')
            if stuff2evaluate:
                errorinrun +=  automaticmaintenance.evaluate('--automaticretrycommunication',stuff2evaluate)
            else:
                botsglobal.logger.info(_(u'Run automatic recommunicate: nothing to recommunicate.'))
            if userscript and hasattr(userscript,'postautomaticretrycommunication'):
                botslib.runscript(userscript,scriptname,'postautomaticretrycommunication',routestorun=routestorun)
        if '--retry' in commandstorun:
            botsglobal.logger.info(u'Run retry.')
            stuff2evaluate = router.routedispatcher(routestorun,'--retry')
            if stuff2evaluate:
                errorinrun +=  automaticmaintenance.evaluate('--retry',stuff2evaluate)
            else:
                botsglobal.logger.info(_(u'Run retry: nothing to retry.'))
            if userscript and hasattr(userscript,'postretry'):
                botslib.runscript(userscript,scriptname,'postretry',routestorun=routestorun)
        if '--retransmit' in commandstorun:
            botsglobal.logger.info(u'Run retransmit.')
            stuff2evaluate = router.routedispatcher(routestorun,'--retransmit')
            if stuff2evaluate:
                errorinrun +=  automaticmaintenance.evaluate('--retransmit',stuff2evaluate)
            else:
                botsglobal.logger.info(_(u'Run retransmit: nothing to retransmit.'))
            if userscript and hasattr(userscript,'postretransmit'):
                botslib.runscript(userscript,scriptname,'postretransmit',routestorun=routestorun)
        if '--new' in commandstorun:
            botsglobal.logger.info('Run new.')
            botsglobal.incommunicate = True
            botsglobal.minta4query = 0  #meaning: reset. the actual value is set later (in routedispatcher)
            stuff2evaluate = router.routedispatcher(routestorun)
            errorinrun +=  automaticmaintenance.evaluate('--new',stuff2evaluate)
            if userscript and hasattr(userscript,'postnewrun'):
                botslib.runscript(userscript,scriptname,'postnewrun',routestorun=routestorun)
        if '--cleanup' in commandstorun or botsglobal.ini.get('settings','whencleanup','always')=='always':
            botsglobal.logger.debug(u'Do cleanup.')
            cleanup.cleanup()
        botslib.remove_database_lock()
    except Exception,e:
        botsglobal.logger.exception(_(u'Severe error in bots system:\n%s')%(e))    #of course this 'should' not happen. 
        sys.exit(1)
Exemplo n.º 23
0
def start():
    #********command line arguments**************************
    edifile =''
    grammarfile = ''
    configdir = 'config'
    for arg in sys.argv[1:]:
        if not arg:
            continue
        if arg.startswith('-c'):
            configdir = arg[2:]
            if not configdir:
                print '    !!Indicated Bots should use specific .ini file but no file name was given.'
                showusage()
        elif arg in ["?", "/?"] or arg.startswith('-'):
            showusage()
        else:
            if not edifile:
                edifile = arg
            else:
                grammarfile = arg
    if not (edifile and grammarfile):
        print '    !!Both edifile and grammarfile are required.'
        showusage()

    #********end handling command line arguments**************************
    editype='xmlnocheck'
    messagetype='xmlnocheckxxxtemporaryforxml2grammar'
    mpath = []
    
    botsinit.generalinit(configdir)
    os.chdir(botsglobal.ini.get('directories','botspath'))
    botsinit.initenginelogging()
    
    #the xml file is parsed as an xmlnocheck message....so a (temp) xmlnocheck grammar is needed....without content... this file is not removed....
    tmpgrammarfile = botslib.join(botsglobal.ini.get('directories','usersysabs'),'grammars',editype,messagetype+'.py')
    f = open(tmpgrammarfile,'w')
    f.close()

    inn = inmessage.edifromfile(editype=editype,messagetype=messagetype,filename=edifile)
    #~ inn.root.display()
    out = outmessage.outmessage_init(editype=editype,messagetype=messagetype,filename='botssys/infile/unitnode/output/inisout03.edi',divtext='',topartner='')    #make outmessage object
    
    #handle root
    rootmpath = [{'BOTSID':inn.root.record['BOTSID']}]
    out.put(*rootmpath)
    writefields(out,inn.root,rootmpath)
    #walk tree; write results to out-tree
    for node,mpath in treewalker(inn.root,mpath):
        mpath.append({'BOTSID':node.record['BOTSID']})
        if out.get(*mpath) is None:
            out.put(*mpath)
        writefields(out,node,mpath)
        
    #~ out.root.display()
    
    #out-tree is finished; represents ' normalised' tree suited for writing as a grammar
    structure = []
    recorddefs = {}
    tree2grammar(out.root,structure,recorddefs)
    #~ for key,value in recorddefs.items():
        #~ print key,value
        #~ print '\n'
    sortedstructurelist = structure2list(structure)
    recorddefsstring = recorddefs2string(recorddefs,sortedstructurelist)
    structurestring = structure2string(structure)
    
    #write grammar file
    grammar = open(grammarfile,'wb')
    grammar.write('#grammar automatically generated by bots open source edi software.')
    grammar.write('\n')
    grammar.write('from bots.botsconfig import *')
    grammar.write('\n\n')
    grammar.write('syntax = {}')
    grammar.write('\n\n')
    grammar.write('structure = [\n%s]\n'%(structurestring))
    grammar.write('\n\n')
    grammar.write('recorddefs = %s'%(recorddefsstring))
    grammar.write('\n\n')
    grammar.close()
    print 'grammar file is written',grammarfile
Exemplo n.º 24
0
def start():
    #********command line arguments**************************
    usage = '''
    This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
    Updates existing bots database to version %(version)s

    Usage:
        %(name)s  [config-option]
    Options:
        -c<directory>        directory for configuration files (default: config).

    '''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version}
    configdir = 'config'
    for arg in sys.argv[1:]:
        if arg.startswith('-c'):
            configdir = arg[2:]
            if not configdir:
                print 'Error: configuration directory indicated, but no directory name.'
                sys.exit(3)
        else:   #pick up names of routes to run
            print usage
            sys.exit(0)
    #***end handling command line arguments**************************
    botsinit.generalinit(configdir)     #find locating of bots, configfiles, init paths etc.

    #**************check if another instance of bots-engine is running/if port is free******************************
    try:
        engine_socket = botslib.check_if_other_engine_is_running()
    except socket.error:
        sys.exit(3)
    else:
        atexit.register(engine_socket.close)

    #**************initialise logging******************************
    process_name = 'updatedatabase'
    botsglobal.logger = botsinit.initenginelogging(process_name)
    atexit.register(logging.shutdown)
    for key,value in botslib.botsinfo():    #log info about environement, versions, etc
        botsglobal.logger.info(u'%(key)s: "%(value)s".',{'key':key,'value':value})

    #**************connect to database**********************************
    try:
        botsinit.connect()
    except Exception as msg:
        botsglobal.logger.exception(_(u'Could not connect to database. Database settings are in bots/config/settings.py. Error: "%(msg)s".'),{'msg':msg})
        sys.exit(3)
    else:
        botsglobal.logger.info(_(u'Connected to database.'))
        atexit.register(botsglobal.db.close)

    #**************handle database lock****************************************
    #set a lock on the database; if not possible, the database is locked: an earlier instance of bots-engine was terminated unexpectedly.
    if not botslib.set_database_lock():
        warn =  _(u'!Bots database is locked!\n'\
                    'Bots-engine has ended in an unexpected way during the last run.\n'\
                    'Most likely causes: sudden power-down, system crash, problems with disk I/O, bots-engine terminated by user, etc.')
        botsglobal.logger.critical(warn)
        sys.exit(3)
    atexit.register(botslib.remove_database_lock)

    if botsglobal.settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3':
        terug = sqlite3()
    elif botsglobal.settings.DATABASES['default']['ENGINE'] == 'django.db.backends.mysql':
        terug = mysql()
    elif botsglobal.settings.DATABASES['default']['ENGINE'] == 'django.db.backends.postgresql_psycopg2':
        terug = postgresql_psycopg2()
    
    sys.exit(terug)
Exemplo n.º 25
0
def start(configdir):
    botsinit.generalinit(configdir)     #find locating of bots, configfiles, init paths etc.
    process_name = 'apache_webserver_' + configdir
    botsglobal.logger = botsinit.initserverlogging(process_name)    #initialise file-logging for web-server. This logging only contains the logging from bots-webserver.
Exemplo n.º 26
0
def start():
    #********command line arguments**************************
    usage = '''
    This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
    Creates a grammar from an xml file.'
    Usage:'
        %(name)s  -c<directory>  <xml_file>  <xml_grammar_file>
    Options:
        -c<directory>      directory for configuration files (default: config).
        <xml_file>         name of the xml file to read
        <xml_grammar_file> name of the grammar file to write
    
    '''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version}
    configdir = 'config'
    edifile =''
    grammarfile = ''
    for arg in sys.argv[1:]:
        if arg.startswith('-c'):
            configdir = arg[2:]
            if not configdir:
                print 'Error: configuration directory indicated, but no directory name.'
                sys.exit(1)
        elif arg in ["?", "/?",'-h', '--help'] or arg.startswith('-'):
            print usage
            sys.exit(0)
        else:
            if not edifile:
                edifile = arg
            else:
                grammarfile = arg
    if not edifile or not grammarfile:
        print 'Error: both edifile and grammarfile are required.'
        sys.exit(0)
    #***end handling command line arguments**************************
    botsinit.generalinit(configdir)     #find locating of bots, configfiles, init paths etc.
    process_name = 'xml2botsgrammar'
    botsglobal.logger = botsinit.initenginelogging(process_name)
    atexit.register(logging.shutdown)
    
    #the xml file is parsed as an xmlnocheck message
    editype = 'xmlnocheck'
    messagetype = 'xmlnocheckxxxtemporaryforxml2grammar'
    mpath = []
    #a (temp) xmlnocheck grammar is needed (but needs not actual content. This file is not removed.
    tmpgrammarfile = botslib.join(botsglobal.ini.get('directories','usersysabs'),'grammars',editype,messagetype+'.py')
    filehandler = open(tmpgrammarfile,'w')
    filehandler.close()
    
    #make inmessage object: read the xml file
    inn = inmessage.parse_edi_file(editype=editype,messagetype=messagetype,filename=edifile,remove_empties_from_xml=False)
    #make outmessage object; nothing is 'filled' yet.
    out = outmessage.outmessage_init(editype=editype,messagetype=messagetype,filename='botssys/infile/unitnode/output/inisout03.edi',divtext='',topartner='')    
    
    #***do the mapping***************************************************
    #handle root
    rootmpath = [{'BOTSID':inn.root.record['BOTSID'],'BOTSIDnr':'1'}]
    out.put(*rootmpath)
    map_writefields(out,inn.root,rootmpath)
    #walk tree; write results to out-tree
    for node_instance,mpath in map_treewalker(inn.root,mpath):
        mpath.append({'BOTSID':node_instance.record['BOTSID']})
        if out.get(*mpath) is None:
            out.put(*mpath)
        map_writefields(out,node_instance,mpath)

    #***mapping is done; out-tree is finished; represents 'normalised' tree suited for writing as a grammar
    structure = []
    recorddefs = {}
    tree2grammar(out.root,structure,recorddefs)
    #~ for key,value in recorddefs.items():
        #~ print key,value
        #~ print '\n'
    sortedstructurelist = structure2list(structure)
    recorddefsstring = recorddefs2string(recorddefs,sortedstructurelist)
    structurestring = structure2string(structure)

    #write grammar file
    grammar = open(grammarfile,'wb')
    grammar.write('#grammar automatically generated by bots open source edi translator.')
    grammar.write('\n')
    grammar.write('from bots.botsconfig import *')
    grammar.write('\n\n')
    grammar.write('syntax = {}')
    grammar.write('\n\n')
    grammar.write('structure = [\n%s]\n'%(structurestring))
    grammar.write('\n\n')
    grammar.write('recorddefs = %s'%(recorddefsstring))
    grammar.write('\n\n')
    grammar.close()
    print 'grammar file is written:',grammarfile
Exemplo n.º 27
0
def start():
    # NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
    # ***command line arguments**************************
    usage = """
    This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
    A utility to generate the index file of a plugin; this can be seen as a database dump of the configuration.
    This is eg useful for version control.
    Usage:
        %(name)s  -c<directory>
    Options:
        -c<directory>   directory for configuration files (default: config).
        
    """ % {
        "name": os.path.basename(sys.argv[0]),
        "version": botsglobal.version,
    }
    configdir = "config"
    for arg in sys.argv[1:]:
        if arg.startswith("-c"):
            configdir = arg[2:]
            if not configdir:
                print "Error: configuration directory indicated, but no directory name."
                sys.exit(1)
        else:
            print usage
            sys.exit(0)
    # ***end handling command line arguments**************************
    botsinit.generalinit(configdir)  # find locating of bots, configfiles, init paths etc.
    if not botsglobal.ini.getboolean("jobqueue", "enabled", False):
        print "Error: bots jobqueue cannot start; not enabled in %s/bots.ini" % configdir
        sys.exit(1)
    process_name = "dirmonitor"
    logger = botsinit.initserverlogging(process_name)
    logger.log(25, u"Bots %(process_name)s started.", {"process_name": process_name})
    logger.log(
        25,
        u'Bots %(process_name)s configdir: "%(configdir)s".',
        {"process_name": process_name, "configdir": botsglobal.ini.get("directories", "config")},
    )

    botsenginepath = os.path.join(
        os.path.dirname(os.path.abspath(sys.argv[0])), "bots-engine.py"
    )  # get path to bots-engine
    cond = threading.Condition()
    tasks = set()
    dir_watch_data = []
    for section in botsglobal.ini.sections():
        if section.startswith("dirmonitor") and section[len("dirmonitor") :]:
            dir_watch_data.append({})
            dir_watch_data[-1]["path"] = botsglobal.ini.get(section, "path")
            dir_watch_data[-1]["rec"] = botsglobal.ini.getboolean(section, "recursive", False)
            dir_watch_data[-1]["filemask"] = botsglobal.ini.get(section, "filemask", "*")
            dir_watch_data[-1]["route"] = botsglobal.ini.get(section, "route", "")
    if not dir_watch_data:
        logger.error(u"Nothing to watch!")
        sys.exit(0)

    if os.name == "nt":
        # for windows: start a thread per directory watcher
        for dir_watch in dir_watch_data:
            dir_watch_thread = threading.Thread(target=windows_event_handler, args=(logger, dir_watch, cond, tasks))
            dir_watch_thread.daemon = True  # do not wait for thread when exiting
            dir_watch_thread.start()
    else:
        # for linux: one watch-thread, but multiple watches.
        dir_watch_thread = threading.Thread(target=linux_event_handler, args=(logger, dir_watch_data, cond, tasks))
        dir_watch_thread.daemon = True  # do not wait for thread when exiting
        dir_watch_thread.start()

    # this main thread get the results from the watch-thread(s).
    logger.info(u"Bots %(process_name)s started.", {"process_name": process_name})
    active_receiving = False
    timeout = 2.0
    cond.acquire()
    while True:
        # this functions as a buffer: all events go into set tasks.
        # the tasks are fired to jobqueue after TIMOUT sec.
        # this is to avoid firing to many tasks to jobqueue; events typically come in bursts.
        # is value of timeout is larger, reaction times are slower...but less tasks are fired to jobqueue.
        # in itself this is not a problem, as jobqueue will alos discard duplicate jobs.
        # 2 sec seems to e a good value: reasonable quick, not to nervous.
        cond.wait(timeout=timeout)  # get back when results, or after timeout sec
        if tasks:
            if not active_receiving:  # first request (after tasks have been  fired, or startup of dirmonitor)
                active_receiving = True
                last_time = time.time()
            else:  # active receiving events
                current_time = time.time()
                if current_time - last_time >= timeout:  # cond.wait returned probably because of a timeout
                    try:
                        for task in tasks:
                            logger.info(
                                u'Send to queue "%(path)s %(config)s %(task)s".',
                                {"path": botsenginepath, "config": "-c" + configdir, "task": task},
                            )
                            job2queue.send_job_to_jobqueue([sys.executable, botsenginepath, "-c" + configdir, task])
                    except Exception as msg:
                        logger.info(u'Error in running task: "%(msg)s".', {"msg": msg})
                    tasks.clear()
                    active_receiving = False
                else:  # cond.wait returned probably because of a timeout
                    logger.debug(u"time difference to small.")
                    last_time = current_time
    cond.release()
    sys.exit(0)
Exemplo n.º 28
0
def start():
    #NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
    #***command line arguments**************************
    usage = '''
    This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
    A utility to generate the index file of a plugin; this can be seen as a database dump of the configuration.
    This is eg useful for version control.
    Usage:
        %(name)s  -c<directory>
    Options:
        -c<directory>   directory for configuration files (default: config).
        
    ''' % {
        'name': os.path.basename(sys.argv[0]),
        'version': botsglobal.version
    }
    configdir = 'config'
    for arg in sys.argv[1:]:
        if arg.startswith('-c'):
            configdir = arg[2:]
            if not configdir:
                print 'Error: configuration directory indicated, but no directory name.'
                sys.exit(1)
        else:
            print usage
            sys.exit(0)
    #***end handling command line arguments**************************
    botsinit.generalinit(
        configdir)  #find locating of bots, configfiles, init paths etc.
    if not botsglobal.ini.getboolean('jobqueue', 'enabled', False):
        print 'Error: bots jobqueue cannot start; not enabled in %s/bots.ini' % configdir
        sys.exit(1)
    process_name = 'dirmonitor'
    logger = botsinit.initserverlogging(process_name)
    logger.log(25, u'Bots %(process_name)s started.',
               {'process_name': process_name})
    logger.log(
        25, u'Bots %(process_name)s configdir: "%(configdir)s".', {
            'process_name': process_name,
            'configdir': botsglobal.ini.get('directories', 'config')
        })

    botsenginepath = os.path.join(os.path.dirname(os.path.abspath(
        sys.argv[0])), 'bots-engine.py')  #get path to bots-engine
    cond = threading.Condition()
    tasks = set()
    dir_watch_data = []
    for section in botsglobal.ini.sections():
        if section.startswith('dirmonitor') and section[len('dirmonitor'):]:
            dir_watch_data.append({})
            dir_watch_data[-1]['path'] = botsglobal.ini.get(section, 'path')
            dir_watch_data[-1]['rec'] = botsglobal.ini.getboolean(
                section, 'recursive', False)
            dir_watch_data[-1]['filemask'] = botsglobal.ini.get(
                section, 'filemask', '*')
            dir_watch_data[-1]['route'] = botsglobal.ini.get(
                section, 'route', '')
    if not dir_watch_data:
        logger.error(u'Nothing to watch!')
        sys.exit(0)

    if os.name == 'nt':
        #for windows: start a thread per directory watcher
        for dir_watch in dir_watch_data:
            dir_watch_thread = threading.Thread(target=windows_event_handler,
                                                args=(logger, dir_watch, cond,
                                                      tasks))
            dir_watch_thread.daemon = True  #do not wait for thread when exiting
            dir_watch_thread.start()
    else:
        #for linux: one watch-thread, but multiple watches.
        dir_watch_thread = threading.Thread(target=linux_event_handler,
                                            args=(logger, dir_watch_data, cond,
                                                  tasks))
        dir_watch_thread.daemon = True  #do not wait for thread when exiting
        dir_watch_thread.start()

    # this main thread get the results from the watch-thread(s).
    logger.info(u'Bots %(process_name)s started.',
                {'process_name': process_name})
    active_receiving = False
    timeout = 2.0
    cond.acquire()
    while True:
        #this functions as a buffer: all events go into set tasks.
        #the tasks are fired to jobqueue after TIMOUT sec.
        #this is to avoid firing to many tasks to jobqueue; events typically come in bursts.
        #is value of timeout is larger, reaction times are slower...but less tasks are fired to jobqueue.
        #in itself this is not a problem, as jobqueue will alos discard duplicate jobs.
        #2 sec seems to e a good value: reasonable quick, not to nervous.
        cond.wait(
            timeout=timeout)  #get back when results, or after timeout sec
        if tasks:
            if not active_receiving:  #first request (after tasks have been  fired, or startup of dirmonitor)
                active_receiving = True
                last_time = time.time()
            else:  #active receiving events
                current_time = time.time()
                if current_time - last_time >= timeout:  #cond.wait returned probably because of a timeout
                    try:
                        for task in tasks:
                            logger.info(
                                u'Send to queue "%(path)s %(config)s %(task)s".',
                                {
                                    'path': botsenginepath,
                                    'config': '-c' + configdir,
                                    'task': task
                                })
                            job2queue.send_job_to_jobqueue([
                                sys.executable, botsenginepath,
                                '-c' + configdir, task
                            ])
                    except Exception as msg:
                        logger.info(u'Error in running task: "%(msg)s".',
                                    {'msg': msg})
                    tasks.clear()
                    active_receiving = False
                else:  #cond.wait returned probably because of a timeout
                    logger.debug(u'time difference to small.')
                    last_time = current_time
    cond.release()
    sys.exit(0)
Exemplo n.º 29
0
def start():
    #NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
    #********command line arguments**************************
    usage = '''
    This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
    Checks a Bots grammar. Same checks are used as in translations with bots-engine. Searches for grammar in 
    regular place: bots/usersys/grammars/<editype>/<messagetype>.py  (even if a path is passed).
    
    Usage:  %(name)s  -c<directory> <editype> <messagetype>
       or   %(name)s  -c<directory> <path to grammar>
    Options:
        -c<directory>   directory for configuration files (default: config).
    Examples:
        %(name)s -cconfig  edifact  ORDERSD96AUNEAN008
        %(name)s -cconfig  C:/python27/lib/site-packages/bots/usersys/grammars/edifact/ORDERSD96AUNEAN008.py
        
    ''' % {
        'name': os.path.basename(sys.argv[0]),
        'version': botsglobal.version
    }
    configdir = 'config'
    editype = ''
    messagetype = ''
    for arg in sys.argv[1:]:
        if arg.startswith('-c'):
            configdir = arg[2:]
            if not configdir:
                print 'Error: configuration directory indicated, but no directory name.'
                sys.exit(1)
        elif arg in ["?", "/?", '-h', '--help'] or arg.startswith('-'):
            print usage
            sys.exit(0)
        else:
            if os.path.isfile(arg):
                p1, p2 = os.path.split(arg)
                editype = os.path.basename(p1)
                messagetype, ext = os.path.splitext(p2)
                messagetype = unicode(messagetype)
                print 'grammarcheck', editype, messagetype
            elif not editype:
                editype = arg
            else:
                messagetype = arg
    if not (editype and messagetype):
        print 'Error: both editype and messagetype, or a file path, are required.'
        sys.exit(1)
    #***end handling command line arguments**************************
    botsinit.generalinit(
        configdir)  #find locating of bots, configfiles, init paths etc.
    process_name = 'grammarcheck'
    botsglobal.logger = botsinit.initenginelogging(process_name)
    atexit.register(logging.shutdown)

    try:
        grammar.grammarread(editype, messagetype)
    except:
        print 'Found error in grammar: ', botslib.txtexc()
        sys.exit(1)
    else:
        print 'OK - no error found in grammar'
        sys.exit(0)
Exemplo n.º 30
0
def start():
    ''' sysexit codes:
        0: OK, no errors
        1: (system) errors incl parsing of command line arguments
        2: bots ran OK, but there are errors/process errors  in the run
        3: Database is locked, but "maxruntime" has not been exceeded.
    '''
    #NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
    #********command line arguments**************************
    usage = '''
    This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
    Does the actual translations and communications; it's the workhorse. It does not have a fancy interface.

    Usage:
        %(name)s  [config-option]
    Config-option:
        -c<directory>        directory for configuration files (default: config).

    '''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version}
    configdir = 'config'
    for arg in sys.argv[1:]:
        if arg.startswith('-c'):
            configdir = arg[2:]
            if not configdir:
                print 'Error: configuration directory indicated, but no directory name.'
                sys.exit(1)
        elif arg in ["?", "/?",'-h', '--help'] or arg.startswith('-'):
            print usage
            sys.exit(0)
    #***********end handling command line arguments**************************
    
    botsinit.generalinit(configdir)     #find locating of bots, configfiles, init paths etc.
    #set working directory to bots installation. advantage: when using relative paths it is clear that this point paths within bots installation. 
    os.chdir(botsglobal.ini.get('directories','botspath'))

    #**************initialise logging******************************
    process_name = 'engine2'
    botsglobal.logger = botsinit.initenginelogging(process_name)
    atexit.register(logging.shutdown)
    for key,value in botslib.botsinfo():    #log info about environement, versions, etc
        botsglobal.logger.info(u'%(key)s: "%(value)s".',{'key':key,'value':value})

    #**************connect to database**********************************
    try:
        botsinit.connect()
    except Exception as msg:
        botsglobal.logger.exception(_(u'Could not connect to database. Database settings are in bots/config/settings.py. Error: "%(msg)s".'),{'msg':msg})
        sys.exit(1)
    else:
        botsglobal.logger.info(_(u'Connected to database.'))
        atexit.register(botsglobal.db.close)

    warnings.simplefilter('error', UnicodeWarning)
        
    #import global scripts for bots-engine
    try:
        userscript,scriptname = botslib.botsimport('routescripts','botsengine')
    except botslib.BotsImportError:      #userscript is not there; other errors like syntax errors are not catched
        userscript = scriptname = None
    #***acceptance tests: initialiase acceptance user script******************************
    acceptance_userscript = acceptance_scriptname = None
    if botsglobal.ini.getboolean('acceptance','runacceptancetest',False):
        botsglobal.logger.info(_(u'This run is an acceptance test - as indicated in option "runacceptancetest" in bots.ini.'))
        try:
            acceptance_userscript,acceptance_scriptname = botslib.botsimport('routescripts','bots_acceptancetest')
        except botslib.BotsImportError:
            botsglobal.logger.info(_(u'In acceptance test there is no script file "bots_acceptancetest.py" to check the results of the acceptance test.'))

    try:
        #~ botslib.prepare_confirmrules()
        #in acceptance tests: run a user script before running eg to clean output directories******************************
        botslib.tryrunscript(acceptance_userscript,acceptance_scriptname,'pretest')
        botslib.tryrunscript(userscript,scriptname,'pre')
        errorinrun = engine2_run()
    except Exception as msg:
        botsglobal.logger.exception(_(u'Severe error in bots system:\n%(msg)s'),{'msg':unicode(msg)})    #of course this 'should' not happen.
        sys.exit(1)
    else:
        if errorinrun:
            sys.exit(2) #indicate: error(s) in run(s)
        else:
            sys.exit(0) #OK
Exemplo n.º 31
0
def start():
    #********command line arguments**************************
    usage = '''
    This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
    Updates existing bots database to version %(version)s

    Usage:
        %(name)s  [config-option]
    Options:
        -c<directory>        directory for configuration files (default: config).

    ''' % {
        'name': os.path.basename(sys.argv[0]),
        'version': botsglobal.version
    }
    configdir = 'config'
    for arg in sys.argv[1:]:
        if arg.startswith('-c'):
            configdir = arg[2:]
            if not configdir:
                print 'Error: configuration directory indicated, but no directory name.'
                sys.exit(3)
        else:  #pick up names of routes to run
            print usage
            sys.exit(0)
    #***end handling command line arguments**************************
    botsinit.generalinit(
        configdir)  #find locating of bots, configfiles, init paths etc.

    #**************check if another instance of bots-engine is running/if port is free******************************
    try:
        engine_socket = botslib.check_if_other_engine_is_running()
    except socket.error:
        sys.exit(3)
    else:
        atexit.register(engine_socket.close)

    #**************initialise logging******************************
    process_name = 'updatedatabase'
    botsglobal.logger = botsinit.initenginelogging(process_name)
    atexit.register(logging.shutdown)
    for key, value in botslib.botsinfo(
    ):  #log info about environement, versions, etc
        botsglobal.logger.info(u'%(key)s: "%(value)s".', {
            'key': key,
            'value': value
        })

    #**************connect to database**********************************
    try:
        botsinit.connect()
    except Exception as msg:
        botsglobal.logger.exception(
            _(u'Could not connect to database. Database settings are in bots/config/settings.py. Error: "%(msg)s".'
              ), {'msg': msg})
        sys.exit(3)
    else:
        botsglobal.logger.info(_(u'Connected to database.'))
        atexit.register(botsglobal.db.close)

    #**************handle database lock****************************************
    #set a lock on the database; if not possible, the database is locked: an earlier instance of bots-engine was terminated unexpectedly.
    if not botslib.set_database_lock():
        warn =  _(u'!Bots database is locked!\n'\
                    'Bots-engine has ended in an unexpected way during the last run.\n'\
                    'Most likely causes: sudden power-down, system crash, problems with disk I/O, bots-engine terminated by user, etc.')
        botsglobal.logger.critical(warn)
        sys.exit(3)
    atexit.register(botslib.remove_database_lock)

    if botsglobal.settings.DATABASES['default'][
            'ENGINE'] == 'django.db.backends.sqlite3':
        terug = sqlite3()
    elif botsglobal.settings.DATABASES['default'][
            'ENGINE'] == 'django.db.backends.mysql':
        terug = mysql()
    elif botsglobal.settings.DATABASES['default'][
            'ENGINE'] == 'django.db.backends.postgresql_psycopg2':
        terug = postgresql_psycopg2()

    sys.exit(terug)
Exemplo n.º 32
0
def start():
    #NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
    #***command line arguments**************************
    usage = '''
    This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
    The %(name)s is the web server for bots; the interface (bots-monitor) can be accessed in a 
    browser, eg 'http://localhost:8080'.
    Usage:
        %(name)s  -c<directory>
    Options:
        -c<directory>   directory for configuration files (default: config).
    
    ''' % {
        'name': os.path.basename(sys.argv[0]),
        'version': botsglobal.version
    }
    configdir = 'config'
    for arg in sys.argv[1:]:
        if arg.startswith('-c'):
            configdir = arg[2:]
            if not configdir:
                print 'Error: configuration directory indicated, but no directory name.'
                sys.exit(1)
        else:
            print usage
            sys.exit(0)
    #***end handling command line arguments**************************
    botsinit.generalinit(
        configdir)  #find locating of bots, configfiles, init paths etc.
    process_name = 'webserver'
    botsglobal.logger = botsinit.initserverlogging(
        process_name
    )  #initialise file-logging for web-server. This logging only contains the logging from bots-webserver, not from cherrypy.

    #***init cherrypy as webserver*********************************************
    #global configuration for cherrypy
    cherrypy.config.update({
        'global': {
            'log.screen':
            False,
            'server.environment':
            botsglobal.ini.get('webserver', 'environment', 'production')
        }
    })
    #cherrypy handling of static files
    conf = {
        '/': {
            'tools.staticdir.on': True,
            'tools.staticdir.dir': 'media',
            'tools.staticdir.root': botsglobal.ini.get('directories',
                                                       'botspath')
        }
    }
    servestaticfiles = cherrypy.tree.mount(
        None, '/media', conf
    )  #None: no cherrypy application (as this only serves static files)
    #cherrypy handling of django
    servedjango = WSGIHandler(
    )  #was: servedjango = AdminMediaHandler(WSGIHandler())  but django does not need the AdminMediaHandler in this setup. is much faster.
    #cherrypy uses a dispatcher in order to handle the serving of static files and django.
    dispatcher = wsgiserver.WSGIPathInfoDispatcher({
        '/': servedjango,
        '/media': servestaticfiles
    })
    botswebserver = wsgiserver.CherryPyWSGIServer(
        bind_addr=('0.0.0.0', botsglobal.ini.getint('webserver', 'port',
                                                    8080)),
        wsgi_app=dispatcher,
        server_name=botsglobal.ini.get('webserver', 'name', 'bots-webserver'))
    botsglobal.logger.log(25, _(u'Bots %(process_name)s started.'),
                          {'process_name': process_name})
    botsglobal.logger.log(
        25, _(u'Bots %(process_name)s configdir: "%(configdir)s".'), {
            'process_name': process_name,
            'configdir': botsglobal.ini.get('directories', 'config')
        })
    botsglobal.logger.log(
        25, _(u'Bots %(process_name)s serving at port: "%(port)s".'), {
            'process_name': process_name,
            'port': botsglobal.ini.getint('webserver', 'port', 8080)
        })
    #handle ssl: cherrypy < 3.2 always uses pyOpenssl. cherrypy >= 3.2 uses python buildin ssl (python >= 2.6 has buildin support for ssl).
    ssl_certificate = botsglobal.ini.get('webserver', 'ssl_certificate', None)
    ssl_private_key = botsglobal.ini.get('webserver', 'ssl_private_key', None)
    if ssl_certificate and ssl_private_key:
        if cherrypy.__version__ >= '3.2.0':
            adapter_class = wsgiserver.get_ssl_adapter_class('builtin')
            botswebserver.ssl_adapter = adapter_class(ssl_certificate,
                                                      ssl_private_key)
        else:
            #but: pyOpenssl should be there!
            botswebserver.ssl_certificate = ssl_certificate
            botswebserver.ssl_private_key = ssl_private_key
        botsglobal.logger.log(25,
                              _(u'Bots %(process_name)s uses ssl (https).'),
                              {'process_name': process_name})
    else:
        botsglobal.logger.log(
            25, _(u'Bots %(process_name)s uses plain http (no ssl).'),
            {'process_name': process_name})

    #***start the cherrypy webserver.************************************************
    try:
        botswebserver.start()
    except KeyboardInterrupt:
        botswebserver.stop()
Exemplo n.º 33
0
def start():
    ''' sysexit codes:
        0: OK, no errors
        1: (system) errors incl parsing of command line arguments
        2: bots ran OK, but there are errors/process errors  in the run
        3: Database is locked, but "maxruntime" has not been exceeded.
    '''
    #NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
    #********command line arguments**************************
    usage = '''
    This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
    Does the actual translations and communications; it's the workhorse. It does not have a fancy interface.

    Usage:
        %(name)s  [config-option]
    Config-option:
        -c<directory>        directory for configuration files (default: config).

    ''' % {
        'name': os.path.basename(sys.argv[0]),
        'version': botsglobal.version
    }
    configdir = 'config'
    for arg in sys.argv[1:]:
        if arg.startswith('-c'):
            configdir = arg[2:]
            if not configdir:
                print 'Error: configuration directory indicated, but no directory name.'
                sys.exit(1)
        elif arg in ["?", "/?", '-h', '--help'] or arg.startswith('-'):
            print usage
            sys.exit(0)
    #***********end handling command line arguments**************************

    botsinit.generalinit(
        configdir)  #find locating of bots, configfiles, init paths etc.
    #set working directory to bots installation. advantage: when using relative paths it is clear that this point paths within bots installation.
    os.chdir(botsglobal.ini.get('directories', 'botspath'))

    #**************initialise logging******************************
    process_name = 'engine2'
    botsglobal.logger = botsinit.initenginelogging(process_name)
    atexit.register(logging.shutdown)
    for key, value in botslib.botsinfo(
    ):  #log info about environement, versions, etc
        botsglobal.logger.info(u'%(key)s: "%(value)s".', {
            'key': key,
            'value': value
        })

    #**************connect to database**********************************
    try:
        botsinit.connect()
    except Exception as msg:
        botsglobal.logger.exception(
            _(u'Could not connect to database. Database settings are in bots/config/settings.py. Error: "%(msg)s".'
              ), {'msg': msg})
        sys.exit(1)
    else:
        botsglobal.logger.info(_(u'Connected to database.'))
        atexit.register(botsglobal.db.close)

    warnings.simplefilter('error', UnicodeWarning)

    #import global scripts for bots-engine
    try:
        userscript, scriptname = botslib.botsimport('routescripts',
                                                    'botsengine')
    except botslib.BotsImportError:  #userscript is not there; other errors like syntax errors are not catched
        userscript = scriptname = None
    #***acceptance tests: initialiase acceptance user script******************************
    acceptance_userscript = acceptance_scriptname = None
    if botsglobal.ini.getboolean('acceptance', 'runacceptancetest', False):
        botsglobal.logger.info(
            _(u'This run is an acceptance test - as indicated in option "runacceptancetest" in bots.ini.'
              ))
        try:
            acceptance_userscript, acceptance_scriptname = botslib.botsimport(
                'routescripts', 'bots_acceptancetest')
        except botslib.BotsImportError:
            botsglobal.logger.info(
                _(u'In acceptance test there is no script file "bots_acceptancetest.py" to check the results of the acceptance test.'
                  ))

    try:
        #~ botslib.prepare_confirmrules()
        #in acceptance tests: run a user script before running eg to clean output directories******************************
        botslib.tryrunscript(acceptance_userscript, acceptance_scriptname,
                             'pretest')
        botslib.tryrunscript(userscript, scriptname, 'pre')
        errorinrun = engine2_run()
    except Exception as msg:
        botsglobal.logger.exception(
            _(u'Severe error in bots system:\n%(msg)s'),
            {'msg': unicode(msg)})  #of course this 'should' not happen.
        sys.exit(1)
    else:
        if errorinrun:
            sys.exit(2)  #indicate: error(s) in run(s)
        else:
            sys.exit(0)  #OK
Exemplo n.º 34
0
def start():
    ''' sysexit codes:
        0: OK, no errors
        1: (system) errors incl parsing of command line arguments
        2: bots ran OK, but there are errors/process errors  in the run
        3: Database is locked, but "maxruntime" has not been exceeded.
    '''
    #NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
    #********command line arguments**************************
    usage = '''
    This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
    Does the actual translations and communications; it's the workhorse. It does not have a fancy interface.

    Usage:
        %(name)s  [run-options] [config-option] [routes]
    Run-options (can be combined):
        --new                receive new edi files (default: if no run-option given: run as new).
        --resend             resend as indicated by user.
        --rereceive          rereceive as indicated by user.
        --automaticretrycommunication - automatically retry outgoing communication.
        --cleanup            remove older data from database.
    Config-option:
        -c<directory>        directory for configuration files (default: config).
    Routes: list of routes to run. Default: all active routes (in the database)

    '''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version}
    configdir = 'config'
    commandspossible = ['--automaticretrycommunication','--resend','--rereceive','--new']
    commandstorun = []
    routestorun = []    #list with routes to run
    do_cleanup_parameter = False
    for arg in sys.argv[1:]:
        if arg.startswith('-c'):
            configdir = arg[2:]
            if not configdir:
                print 'Error: configuration directory indicated, but no directory name.'
                sys.exit(1)
        elif arg in commandspossible:
            commandstorun.append(arg)
        elif arg == '--cleanup':
            do_cleanup_parameter = True
        elif arg in ["?", "/?",'-h', '--help'] or arg.startswith('-'):
            print usage
            sys.exit(0)
        else:   #pick up names of routes to run
            routestorun.append(arg)
    if not commandstorun and not do_cleanup_parameter:   #if no command on command line, use new (default)
        commandstorun = ['--new']
    commandstorun = [command[2:] for command in commandspossible if command in commandstorun]   #sort commands
    #***end handling command line arguments**************************
    botsinit.generalinit(configdir)     #find locating of bots, configfiles, init paths etc.
    #set working directory to bots installation. 
    #possible advantage: when using relative paths it is clear that this point paths within bots installation. 
    #most of time not needed: 
    #1. in production: do not use relative paths within bots directory
    #2. in eg incoming messages path name is used via botslib.join, which makes an absulute path...
    #use this as bots2.* always ad this; avoid breaking.
    os.chdir(botsglobal.ini.get('directories','botspath'))

    #**************check if another instance of bots-engine is running/if port is free******************************
    try:
        engine_socket = botslib.check_if_other_engine_is_running()
    except socket.error:
        sys.exit(3)
    else:
        atexit.register(engine_socket.close)

    #**************initialise logging******************************
    process_name = 'engine'
    botsglobal.logger = botsinit.initenginelogging(process_name)
    atexit.register(logging.shutdown)
    for key,value in botslib.botsinfo():    #log info about environement, versions, etc
        botsglobal.logger.info(u'%(key)s: "%(value)s".',{'key':key,'value':value})

    #**************connect to database**********************************
    try:
        botsinit.connect()
    except Exception,msg:
        botsglobal.logger.exception(_(u'Could not connect to database. Database settings are in bots/config/settings.py. Error: "%(msg)s".'),{'msg':msg})
        sys.exit(1)
Exemplo n.º 35
0
def start():
    ''' sysexit codes:
        0: OK, no errors
        1: (system) errors incl parsing of command line arguments
        2: bots ran OK, but there are errors/process errors  in the run
        3: Database is locked, but "maxruntime" has not been exceeded.
    '''
    #NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
    #********command line arguments**************************
    usage = '''
    This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
    Does the actual translations and communications; it's the workhorse. It does not have a fancy interface.

    Usage:
        %(name)s  [run-options] [config-option] [routes]
    Run-options (can be combined):
        --new                receive new edi files (default: if no run-option given: run as new).
        --resend             resend as indicated by user.
        --rereceive          rereceive as indicated by user.
        --automaticretrycommunication - automatically retry outgoing communication.
        --cleanup            remove older data from database.
    Config-option:
        -c<directory>        directory for configuration files (default: config).
    Routes: list of routes to run. Default: all active routes (in the database)

    ''' % {
        'name': os.path.basename(sys.argv[0]),
        'version': botsglobal.version
    }
    configdir = 'config'
    commandspossible = [
        '--automaticretrycommunication', '--resend', '--rereceive', '--new'
    ]
    commandstorun = []
    routestorun = []  #list with routes to run
    do_cleanup_parameter = False
    for arg in sys.argv[1:]:
        if arg.startswith('-c'):
            configdir = arg[2:]
            if not configdir:
                print 'Error: configuration directory indicated, but no directory name.'
                sys.exit(1)
        elif arg in commandspossible:
            commandstorun.append(arg)
        elif arg == '--cleanup':
            do_cleanup_parameter = True
        elif arg in ["?", "/?", '-h', '--help'] or arg.startswith('-'):
            print usage
            sys.exit(0)
        else:  #pick up names of routes to run
            routestorun.append(arg)
    if not commandstorun and not do_cleanup_parameter:  #if no command on command line, use new (default)
        commandstorun = ['--new']
    commandstorun = [
        command[2:] for command in commandspossible if command in commandstorun
    ]  #sort commands
    #***********end handling command line arguments**************************

    botsinit.generalinit(
        configdir)  #find locating of bots, configfiles, init paths etc.
    #set working directory to bots installation. advantage: when using relative paths it is clear that this point paths within bots installation.
    os.chdir(botsglobal.ini.get('directories', 'botspath'))

    #**************check if another instance of bots-engine is running/if port is free******************************
    try:
        engine_socket = botslib.check_if_other_engine_is_running()
    except socket.error:
        sys.exit(3)
    else:
        atexit.register(engine_socket.close)

    #**************initialise logging******************************
    process_name = 'engine'
    botsglobal.logger = botsinit.initenginelogging(process_name)
    atexit.register(logging.shutdown)
    for key, value in botslib.botsinfo(
    ):  #log info about environement, versions, etc
        botsglobal.logger.info(u'%(key)s: "%(value)s".', {
            'key': key,
            'value': value
        })

    #**************connect to database**********************************
    try:
        botsinit.connect()
    except Exception as msg:
        botsglobal.logger.exception(
            _(u'Could not connect to database. Database settings are in bots/config/settings.py. Error: "%(msg)s".'
              ), {'msg': msg})
        sys.exit(1)
    else:
        botsglobal.logger.info(_(u'Connected to database.'))
        atexit.register(botsglobal.db.close)
    #************initialise user exits for the whole bots-engine*************************
    try:
        userscript, scriptname = botslib.botsimport('routescripts',
                                                    'botsengine')
    except botslib.BotsImportError:  #userscript is not there; other errors like syntax errors are not catched
        userscript = scriptname = None
    #***acceptance tests: initialiase acceptance user script******************************
    acceptance_userscript = acceptance_scriptname = None
    if botsglobal.ini.getboolean('acceptance', 'runacceptancetest', False):
        botsglobal.logger.info(
            _(u'This run is an acceptance test - as indicated in option "runacceptancetest" in bots.ini.'
              ))
        try:
            acceptance_userscript, acceptance_scriptname = botslib.botsimport(
                'routescripts', 'bots_acceptancetest')
        except botslib.BotsImportError:
            botsglobal.logger.info(
                _(u'In acceptance test there is no script file "bots_acceptancetest.py" to check the results of the acceptance test.'
                  ))

    #**************handle database lock****************************************
    #set a lock on the database; if not possible, the database is locked: an earlier instance of bots-engine was terminated unexpectedly.
    if not botslib.set_database_lock():
        #for SQLite: do a integrity check on the database
        if botsglobal.settings.DATABASES['default'][
                'ENGINE'] == 'django.db.backends.sqlite3':
            cursor = botsglobal.db.execute('''PRAGMA integrity_check''')
            result = cursor.fetchone()
            if result[0] != u'ok':
                warn =  _(u'!Bots database is locked!\n'\
                            'Bots did an integrity check on the database, but database was not OK.\n'\
                            'Manual action is needed!\n'\
                            'Bots has stopped processing EDI files.')
                botsglobal.logger.critical(warn)
                botslib.sendbotserrorreport(
                    _(u'[Bots severe error]Database is damaged'), warn)
                sys.exit(1)
        warn =  _(u'!Bots database is locked!\n'\
                    'Bots-engine has ended in an unexpected way during the last run.\n'\
                    'Most likely causes: sudden power-down, system crash, problems with disk I/O, bots-engine terminated by user, etc.\n'
                    'Bots will do an automatic crash recovery now.')
        botsglobal.logger.critical(warn)
        botslib.sendbotserrorreport(
            _(u'[Bots severe error]Database is locked'), warn)
        commandstorun.insert(
            0, 'crashrecovery'
        )  #there is a database lock. Add a crashrecovery as first command to run.
    atexit.register(botslib.remove_database_lock)

    warnings.simplefilter('error', UnicodeWarning)

    #**************run the routes**********************************************
    #commandstorun determines the type(s) of run. eg: ['automaticretrycommunication','new']
    try:
        botslib.prepare_confirmrules()
        #in acceptance tests: run a user script before running eg to clean output directories******************************
        botslib.tryrunscript(acceptance_userscript,
                             acceptance_scriptname,
                             'pretest',
                             routestorun=routestorun)
        botslib.tryrunscript(userscript,
                             scriptname,
                             'pre',
                             commandstorun=commandstorun,
                             routestorun=routestorun)
        errorinrun = 0  #detect if there has been some error. Only used for correct exit() code
        first_command_2_run = True
        for command in commandstorun:
            #if multiple commands in run: reports etc are based on timestamp; so there needs to be at least one second between these runs.
            if first_command_2_run:
                first_command_2_run = False
            else:
                time.sleep(1)
            botsglobal.logger.info(_(u'Run "%(command)s".'),
                                   {'command': command})
            #************get list of routes to run*******************************
            if routestorun:
                use_routestorun = routestorun[:]
                botsglobal.logger.info(
                    _(u'Run routes from command line: "%(routes)s".'),
                    {'routes': unicode(use_routestorun)})
            elif command == 'new':  #fetch all active routes from database unless 'not in default run' or not active.
                use_routestorun = []
                for row in botslib.query(
                        '''SELECT DISTINCT idroute
                                            FROM routes
                                            WHERE active=%(active)s
                                            AND (notindefaultrun=%(notindefaultrun)s OR notindefaultrun IS NULL)
                                            ORDER BY idroute ''', {
                            'active': True,
                            'notindefaultrun': False
                        }):
                    use_routestorun.append(row['idroute'])
                botsglobal.logger.info(
                    _(u'Run active routes from database that are in default run: "%(routes)s".'
                      ), {'routes': unicode(use_routestorun)})
            else:  #for command other than 'new': use all active routes.
                use_routestorun = []
                for row in botslib.query(
                        '''SELECT DISTINCT idroute
                                            FROM routes
                                            WHERE active=%(active)s
                                            ORDER BY idroute ''',
                    {'active': True}):
                    use_routestorun.append(row['idroute'])
                botsglobal.logger.info(
                    _(u'Run all active routes from database: "%(routes)s".'),
                    {'routes': unicode(use_routestorun)})
            #************run routes for this command******************************
            botslib.tryrunscript(userscript,
                                 scriptname,
                                 'pre' + command,
                                 routestorun=use_routestorun)
            errorinrun += router.rundispatcher(command, use_routestorun)
            botslib.tryrunscript(userscript,
                                 scriptname,
                                 'post' + command,
                                 routestorun=use_routestorun)
            #*********finished running routes for this command****************************
        #*********finished all commands****************************************
        botslib.tryrunscript(userscript,
                             scriptname,
                             'post',
                             commandstorun=commandstorun,
                             routestorun=routestorun)
        try:  #in acceptance tests: run a user script. no good reporting of errors/results in post-test script. Reason: this is after automaticmaintence.
            botslib.tryrunscript(acceptance_userscript,
                                 acceptance_scriptname,
                                 'posttest',
                                 routestorun=use_routestorun)
        except Exception as msg:
            print unicode(msg)

        cleanup.cleanup(do_cleanup_parameter, userscript, scriptname)
    except Exception as msg:
        botsglobal.logger.exception(
            _(u'Severe error in bots system:\n%(msg)s'),
            {'msg': unicode(msg)})  #of course this 'should' not happen.
        sys.exit(1)
    else:
        if errorinrun:
            sys.exit(2)  #indicate: error(s) in run(s)
        else:
            sys.exit(0)  #OK
Exemplo n.º 36
0
def start():
    #NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
    #***command line arguments**************************
    usage = '''
    This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
    The %(name)s is the web server for bots; the interface (bots-monitor) can be accessed in a 
    browser, eg 'http://localhost:8080'.
    Usage:
        %(name)s  -c<directory>
    Options:
        -c<directory>   directory for configuration files (default: config).
    
    '''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version}
    configdir = 'config'
    for arg in sys.argv[1:]:
        if arg.startswith('-c'):
            configdir = arg[2:]
            if not configdir:
                print 'Error: configuration directory indicated, but no directory name.'
                sys.exit(1)
        else:
            print usage
            sys.exit(0)
    #***end handling command line arguments**************************
    botsinit.generalinit(configdir)     #find locating of bots, configfiles, init paths etc.
    process_name = 'webserver'
    botsglobal.logger = botsinit.initserverlogging(process_name)    #initialise file-logging for web-server. This logging only contains the logging from bots-webserver, not from cherrypy.

    #***init cherrypy as webserver*********************************************
    #global configuration for cherrypy
    cherrypy.config.update({'global': {'log.screen': False, 'server.environment': botsglobal.ini.get('webserver','environment','production')}})
    #cherrypy handling of static files
    conf = {'/': {'tools.staticdir.on' : True,'tools.staticdir.dir' : 'media' ,'tools.staticdir.root': botsglobal.ini.get('directories','botspath')}}
    servestaticfiles = cherrypy.tree.mount(None, '/media', conf)    #None: no cherrypy application (as this only serves static files)
    #cherrypy handling of django
    servedjango = WSGIHandler()     #was: servedjango = AdminMediaHandler(WSGIHandler())  but django does not need the AdminMediaHandler in this setup. is much faster.
    #cherrypy uses a dispatcher in order to handle the serving of static files and django.
    dispatcher = wsgiserver.WSGIPathInfoDispatcher({'/': servedjango, '/media': servestaticfiles})
    botswebserver = wsgiserver.CherryPyWSGIServer(bind_addr=('0.0.0.0', botsglobal.ini.getint('webserver','port',8080)), wsgi_app=dispatcher, server_name=botsglobal.ini.get('webserver','name','bots-webserver'))
    botsglobal.logger.log(25,_(u'Bots %(process_name)s started.'),
                                {'process_name':process_name})
    botsglobal.logger.log(25,_(u'Bots %(process_name)s configdir: "%(configdir)s".'),
                                {'process_name':process_name, 'configdir':botsglobal.ini.get('directories','config')})
    botsglobal.logger.log(25,_(u'Bots %(process_name)s serving at port: "%(port)s".'),
                                {'process_name':process_name,'port':botsglobal.ini.getint('webserver','port',8080)})
    #handle ssl: cherrypy < 3.2 always uses pyOpenssl. cherrypy >= 3.2 uses python buildin ssl (python >= 2.6 has buildin support for ssl).
    ssl_certificate = botsglobal.ini.get('webserver','ssl_certificate',None)
    ssl_private_key = botsglobal.ini.get('webserver','ssl_private_key',None)
    if ssl_certificate and ssl_private_key:
        if cherrypy.__version__ >= '3.2.0':
            adapter_class = wsgiserver.get_ssl_adapter_class('builtin')
            botswebserver.ssl_adapter = adapter_class(ssl_certificate,ssl_private_key)
        else:
            #but: pyOpenssl should be there!
            botswebserver.ssl_certificate = ssl_certificate
            botswebserver.ssl_private_key = ssl_private_key
        botsglobal.logger.log(25,_(u'Bots %(process_name)s uses ssl (https).'),{'process_name':process_name})
    else:
        botsglobal.logger.log(25,_(u'Bots %(process_name)s uses plain http (no ssl).'),{'process_name':process_name})

    #***start the cherrypy webserver.************************************************
    try:
        botswebserver.start()
    except KeyboardInterrupt:
        botswebserver.stop()