def start(): #********command line arguments************************** editype = '' messagetype = '' configdir = 'config' for arg in sys.argv[1:]: if not arg: continue if arg.startswith('-c'): configdir = arg[2:] if not configdir: print ' !!Indicated Bots should use specific .ini file but no file name was given.' showusage() elif arg in ["?", "/?"] or arg.startswith('-'): showusage() else: if not editype: editype = arg else: messagetype = arg if not (editype and messagetype): print ' !!Both editype and messagetype are required.' showusage() #********end handling command line arguments************************** try: botsinit.generalinit(configdir) botsinit.initenginelogging() grammar.grammarread(editype, messagetype) except: print 'Found error in grammar:' print botslib.txtexc() else: print 'OK - no error found in grammar'
def start(): #NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start. #***command line arguments************************** usage = ''' This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net). A utility to generate the index file of a plugin; this can be seen as a database dump of the configuration. This is eg useful for version control. Usage: %(name)s -c<directory> Options: -c<directory> directory for configuration files (default: config). '''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version} configdir = 'config' for arg in sys.argv[1:]: if arg.startswith('-c'): configdir = arg[2:] if not configdir: print 'Error: configuration directory indicated, but no directory name.' sys.exit(1) else: print usage sys.exit(0) #***end handling command line arguments************************** botsinit.generalinit(configdir) #find locating of bots, configfiles, init paths etc. import pluglib #import here, import at start of file gives error; first initialize. usersys = botsglobal.ini.get('directories','usersysabs') index_filename = os.path.join(usersys,'index.py') dummy_for_cleaned_data = {'databaseconfiguration':True,'umlists':True,'databasetransactions':False} pluglib.make_index(dummy_for_cleaned_data,index_filename)
def startmulti(grammardir, editype): ''' specialized tool for bulk checking of grammars while developing botsgrammars grammardir: directory with gramars (eg bots/usersys/grammars/edifact) editype: eg edifact ''' configdir = 'config' botsinit.generalinit( configdir) #find locating of bots, configfiles, init paths etc. process_name = 'grammarcheck' botsglobal.logger = botsinit.initenginelogging(process_name) atexit.register(logging.shutdown) for filename in glob.iglob(grammardir): filename_basename = os.path.basename(filename) if filename_basename in ['__init__.py', 'envelope.py']: continue if filename_basename.startswith( 'edifact') or filename_basename.startswith( 'records') or filename_basename.endswith('records.py'): continue if filename_basename.endswith('pyc'): continue filename_noextension = os.path.splitext(filename_basename)[0] try: grammar.grammarread(editype, filename_noextension) except: print botslib.txtexc() print '\n' else: print 'OK - no error found in grammar', filename, '\n'
def start(): #NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start. #***command line arguments************************** usage = ''' This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net). A utility to generate the index file of a plugin; this can be seen as a database dump of the configuration. This is eg useful for version control. Usage: %(name)s -c<directory> Options: -c<directory> directory for configuration files (default: config). '''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version} configdir = 'config' for arg in sys.argv[1:]: if arg.startswith('-c'): configdir = arg[2:] if not configdir: print 'Error: configuration directory indicated, but no directory name.' sys.exit(1) else: print usage sys.exit(0) #***end handling command line arguments************************** botsinit.generalinit(configdir) #find locating of bots, configfiles, init paths etc. import pluglib #import here, import at start of file gives error; first initialize. usersys = botsglobal.ini.get('directories','usersysabs') index_filename = os.path.join(usersys,'index.py') dummy_for_cleaned_data = {'databaseconfiguration':True,'umlists':botsglobal.ini.getboolean('settings','codelists_in_plugin',True),'databasetransactions':False} pluglib.make_index(dummy_for_cleaned_data,index_filename)
def start(): #NOTE bots is always on PYTHONPATH!!! - otherwise it will not start. #***command line arguments************************** configdir = 'config' for arg in sys.argv[1:]: if not arg: continue if arg.startswith('-c'): configdir = arg[2:] if not configdir: print 'Configuration directory indicated, but no directory name.' sys.exit(1) elif arg in ["?", "/?"] or arg.startswith('-'): showusage() else: showusage() #***init general: find locating of bots, configfiles, init paths etc.*********************** botsinit.generalinit(configdir) #***initialise logging. This logging only contains the logging from bots-webserver, not from cherrypy. botsglobal.logger = logging.getLogger('bots-webserver') botsglobal.logger.setLevel(logging.DEBUG) h = TimedRotatingFileHandler(botslib.join(botsglobal.ini.get('directories','logging'),'webserver.log'), backupCount=10) fileformat = logging.Formatter("%(asctime)s %(levelname)-8s: %(message)s",'%Y%m%d %H:%M:%S') h.setFormatter(fileformat) botsglobal.logger.addHandler(h) #***init cherrypy as webserver********************************************* #global configuration for cherrypy cherrypy.config.update({'global': {'log.screen': False, 'server.environment': botsglobal.ini.get('webserver','environment','production')}}) #cherrypy handling of static files conf = {'/': {'tools.staticdir.on' : True,'tools.staticdir.dir' : 'media' ,'tools.staticdir.root': botsglobal.ini.get('directories','botspath')}} servestaticfiles = cherrypy.tree.mount(None, '/media', conf) #None: no cherrypy application (as this only serves static files) #cherrypy handling of django servedjango = WSGIHandler() #was: servedjango = AdminMediaHandler(WSGIHandler()) but django does not need the AdminMediaHandler in this setup. is much faster. #cherrypy uses a dispatcher in order to handle the serving of static files and django. dispatcher = wsgiserver.WSGIPathInfoDispatcher({'/': servedjango, '/media': servestaticfiles}) botswebserver = wsgiserver.CherryPyWSGIServer(bind_addr=('0.0.0.0', botsglobal.ini.getint('webserver','port',8080)), wsgi_app=dispatcher, server_name=botsglobal.ini.get('webserver','name','bots-webserver')) botsglobal.logger.info(_(u'Bots web-server started.')) #handle ssl: cherrypy < 3.2 always uses pyOpenssl. cherrypy >= 3.2 uses python buildin ssl (python >= 2.6 has buildin support for ssl). ssl_certificate = botsglobal.ini.get('webserver','ssl_certificate',None) ssl_private_key = botsglobal.ini.get('webserver','ssl_private_key',None) if ssl_certificate and ssl_private_key: if cherrypy.__version__ >= '3.2.0': adapter_class = wsgiserver.get_ssl_adapter_class('builtin') botswebserver.ssl_adapter = adapter_class(ssl_certificate,ssl_private_key) else: #but: pyOpenssl should be there! botswebserver.ssl_certificate = ssl_certificate botswebserver.ssl_private_key = ssl_private_key botsglobal.logger.info(_(u'Bots web-server uses ssl (https).')) else: botsglobal.logger.info(_(u'Bots web-server uses plain http (no ssl).')) #***start the cherrypy webserver. try: botswebserver.start() except KeyboardInterrupt: botswebserver.stop()
def start(configdir): botsinit.generalinit( configdir) #find locating of bots, configfiles, init paths etc. process_name = 'apache_webserver_' + configdir botsglobal.logger = botsinit.initserverlogging( process_name ) #initialise file-logging for web-server. This logging only contains the logging from bots-webserver.
def startmulti(grammardir,editype): ''' specialized tool for bulk checking of grammars while developing botsgrammars grammardir: directory with gramars (eg bots/usersys/grammars/edifact) editype: eg edifact ''' configdir = 'config' botsinit.generalinit(configdir) #find locating of bots, configfiles, init paths etc. process_name = 'grammarcheck' botsglobal.logger = botsinit.initenginelogging(process_name) atexit.register(logging.shutdown) for filename in glob.iglob(grammardir): filename_basename = os.path.basename(filename) if filename_basename in ['__init__.py','envelope.py']: continue if filename_basename.startswith('edifact') or filename_basename.startswith('records') or filename_basename.endswith('records.py'): continue if filename_basename.endswith('pyc'): continue filename_noextension = os.path.splitext(filename_basename)[0] try: grammar.grammarread(editype,filename_noextension) except: print botslib.txtexc() print '\n' else: print 'OK - no error found in grammar',filename,'\n'
def start(): #********command line arguments************************** editype ='' messagetype = '' configdir = 'config' for arg in sys.argv[1:]: if not arg: continue if arg.startswith('-c'): configdir = arg[2:] if not configdir: print ' !!Indicated Bots should use specific .ini file but no file name was given.' showusage() elif arg in ["?", "/?"] or arg.startswith('-'): showusage() else: if not editype: editype = arg else: messagetype = arg if not (editype and messagetype): print ' !!Both editype and messagetype are required.' showusage() #********end handling command line arguments************************** try: botsinit.generalinit(configdir) botsinit.initenginelogging() grammar.grammarread(editype,messagetype) except: print 'Found error in grammar:' print botslib.txtexc() else: print 'OK - no error found in grammar'
def start(): #NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start. #********command line arguments************************** usage = ''' This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net). Checks a Bots grammar. Same checks are used as in translations with bots-engine. Searches for grammar in regular place: bots/usersys/grammars/<editype>/<messagetype>.py (even if a path is passed). Usage: %(name)s -c<directory> <editype> <messagetype> or %(name)s -c<directory> <path to grammar> Options: -c<directory> directory for configuration files (default: config). Examples: %(name)s -cconfig edifact ORDERSD96AUNEAN008 %(name)s -cconfig C:/python27/lib/site-packages/bots/usersys/grammars/edifact/ORDERSD96AUNEAN008.py '''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version} configdir = 'config' editype ='' messagetype = '' for arg in sys.argv[1:]: if arg.startswith('-c'): configdir = arg[2:] if not configdir: print 'Error: configuration directory indicated, but no directory name.' sys.exit(1) elif arg in ["?", "/?",'-h', '--help'] or arg.startswith('-'): print usage sys.exit(0) else: if os.path.isfile(arg): p1,p2 = os.path.split(arg) editype = os.path.basename(p1) messagetype,ext = os.path.splitext(p2) messagetype = str(messagetype) print 'grammarcheck',editype,messagetype elif not editype: editype = arg else: messagetype = arg if not (editype and messagetype): print 'Error: both editype and messagetype, or a file path, are required.' sys.exit(1) #***end handling command line arguments************************** botsinit.generalinit(configdir) #find locating of bots, configfiles, init paths etc. process_name = 'grammarcheck' botsglobal.logger = botsinit.initenginelogging(process_name) atexit.register(logging.shutdown) try: grammar.grammarread(editype,messagetype) except: print 'Found error in grammar: ',botslib.txtexc() sys.exit(1) else: print 'OK - no error found in grammar' sys.exit(0)
def start(): #NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start. #***command line arguments************************** usage = ''' This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net). Server program that ensures only a single bots-engine runs at any time, and no engine run requests are lost/discarded. Each request goes to a queue and is run in sequence when the previous run completes. Use of the job queue is optional and must be configured in bots.ini (jobqueue section, enabled = True). Usage: %(name)s -c<directory> Options: -c<directory> directory for configuration files (default: config). '''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version} configdir = 'config' for arg in sys.argv[1:]: if arg.startswith('-c'): configdir = arg[2:] if not configdir: print 'Error: configuration directory indicated, but no directory name.' sys.exit(1) else: print usage sys.exit(0) #***end handling command line arguments************************** botsinit.generalinit(configdir) #find locating of bots, configfiles, init paths etc. if not botsglobal.ini.getboolean('jobqueue','enabled',False): print 'Error: bots jobqueue cannot start; not enabled in %s/bots.ini' % configdir sys.exit(1) process_name = 'jobqueue' logger = botsinit.initserverlogging(process_name) logger.log(25,u'Bots %(process_name)s started.',{'process_name':process_name}) logger.log(25,u'Bots %(process_name)s configdir: "%(configdir)s".',{'process_name':process_name,'configdir':botsglobal.ini.get('directories','config')}) port = botsglobal.ini.getint('jobqueue','port',28082) logger.log(25,u'Bots %(process_name)s listens for xmlrpc at port: "%(port)s".',{'process_name':process_name,'port':port}) #start launcher thread lauchfrequency = botsglobal.ini.getint('jobqueue','lauchfrequency',5) maxruntime = botsglobal.ini.getint('settings','maxruntime',60) launcher_thread = threading.Thread(name='launcher', target=launcher, args=(logger,port,lauchfrequency,maxruntime)) launcher_thread.daemon = True launcher_thread.start() logger.info(u'Jobqueue launcher started.') #the main thread is the xmlrpc server: all adding, getting etc for jobqueue is done via xmlrpc. logger.info(u'Jobqueue server started.') server = SimpleXMLRPCServer(('localhost', port),logRequests=False) server.register_instance(Jobqueue(logger)) try: server.serve_forever() except KeyboardInterrupt: pass sys.exit(0)
def start(): # NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start. # ***command line arguments************************** # if config (-c option) is before job argument, will be used as config-dir of job2queue it self and not for job # if config (-c option) is after job argument, use both as config-dir of job2queue and as -c option of job. # if config (-c option) is before and after job argument, use only the after...could change that but seems not to be useful. usage = """ This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net). Places a job in the bots jobqueue. Bots jobqueue takes care of correct processing of jobs. Usage: %(name)s [-c<directory>] [-p<priority>] job [job-parameters] Options: -c<directory> directory for configuration files (default: config). -p<priority> priority of job, 1-9 (default: 5, highest priority is 1). Example of usage: %(name)s bots-engine.py %(name)s python2.7 /usr/local/bin/bots-engine.py %(name)s -p1 python2.7 /usr/local/bin/bots-engine.py -cconfig2 myroute """ % { "name": os.path.basename(sys.argv[0]), "version": botsglobal.version, } configdir = "config" # default value priority = 5 # default value task_args = [] for arg in sys.argv[1:]: if arg.startswith("-c"): configdir = arg[2:] if not configdir: print "Error: configuration directory indicated, but no directory name." sys.exit(1) if task_args: task_args.append(arg) elif arg.startswith("-p"): try: priority = int(arg[2:]) except: print "Error: priority should be numeric (1=highest, 9=lowest)." sys.exit(1) elif arg in ["?", "/?", "-h", "--help"]: print usage sys.exit(0) else: task_args.append(arg) # ***end handling command line arguments************************** botsinit.generalinit(configdir) # needed to read config if not botsglobal.ini.getboolean("jobqueue", "enabled", False): print "Error: bots jobqueue cannot start; not enabled in %s/bots.ini" % configdir sys.exit(1) terug = send_job_to_jobqueue(task_args, priority) print JOBQUEUEMESSAGE2TXT[terug] sys.exit(terug)
def start(): #NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start. #***command line arguments************************** usage = ''' This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net). Server program that ensures only a single bots-engine runs at any time, and no engine run requests are lost/discarded. Each request goes to a queue and is run in sequence when the previous run completes. Use of the job queue is optional and must be configured in bots.ini (jobqueue section, enabled = True). Usage: %(name)s -c<directory> Options: -c<directory> directory for configuration files (default: config). '''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version} configdir = 'config' for arg in sys.argv[1:]: if arg.startswith('-c'): configdir = arg[2:] if not configdir: print 'Error: configuration directory indicated, but no directory name.' sys.exit(1) else: print usage sys.exit(0) #***end handling command line arguments************************** botsinit.generalinit(configdir) #find locating of bots, configfiles, init paths etc. if not botsglobal.ini.getboolean('jobqueue','enabled',False): print 'Error: bots jobqueue cannot start; not enabled in %s/bots.ini' % configdir sys.exit(1) process_name = 'jobqueue' logger = botsinit.initserverlogging(process_name) logger.log(25,u'Bots %(process_name)s started.',process_name=process_name) logger.log(25,u'Bots %(process_name)s configdir: "%(configdir)s".',process_name=process_name,configdir=botsglobal.ini.get('directories','config')) port = botsglobal.ini.getint('jobqueue','port',28082) logger.log(25,u'Bots %(process_name)s listens for xmlrpc at port: "%(port)s".',process_name=process_name,port=port) #start launcher thread lauchfrequency = botsglobal.ini.getint('jobqueue','lauchfrequency',5) maxruntime = botsglobal.ini.getint('settings','maxruntime',60) launcher_thread = threading.Thread(name='launcher', target=launcher, args=(logger,port,lauchfrequency,maxruntime)) launcher_thread.daemon = True launcher_thread.start() logger.info(u'Jobqueue launcher started.') # this main thread is the jobqserver (the xmlrpc server for handling jobqueue) logger.info(u'Jobqueue server started.') server = SimpleXMLRPCServer(('localhost', port),logRequests=False) server.register_instance(Jobqueue(logger)) try: server.serve_forever() except KeyboardInterrupt: pass sys.exit(0)
def start(): #NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start. #***command line arguments************************** #if config (-c option) is before job argument, will be used as config-dir of job2queue it self and not for job #if config (-c option) is after job argument, use both as config-dir of job2queue and as -c option of job. #if config (-c option) is before and after job argument, use only the after...could change that but seems not to be useful. usage = ''' This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net). Places a job in the bots jobqueue. Bots jobqueue takes care of correct processing of jobs. Usage: %(name)s [-c<directory>] [-p<priority>] job [job-parameters] Options: -c<directory> directory for configuration files (default: config). -p<priority> priority of job, 1-9 (default: 5, highest priority is 1). Example of usage: %(name)s bots-engine.py %(name)s python2.7 /usr/local/bin/bots-engine.py %(name)s -p1 python2.7 /usr/local/bin/bots-engine.py -cconfig2 myroute ''' % { 'name': os.path.basename(sys.argv[0]), 'version': botsglobal.version } configdir = 'config' #default value priority = 5 #default value task_args = [] for arg in sys.argv[1:]: if arg.startswith('-c'): configdir = arg[2:] if not configdir: print 'Error: configuration directory indicated, but no directory name.' sys.exit(1) if task_args: task_args.append(arg) elif arg.startswith('-p'): try: priority = int(arg[2:]) except: print 'Error: priority should be numeric (1=highest, 9=lowest).' sys.exit(1) elif arg in ["?", "/?", '-h', '--help']: print usage sys.exit(0) else: task_args.append(arg) #***end handling command line arguments************************** botsinit.generalinit(configdir) #needed to read config if not botsglobal.ini.getboolean('jobqueue', 'enabled', False): print 'Error: bots jobqueue cannot start; not enabled in %s/bots.ini' % configdir sys.exit(1) terug = send_job_to_jobqueue(task_args, priority) print JOBQUEUEMESSAGE2TXT[terug] sys.exit(terug)
def start(): # NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start. # ***command line arguments************************** usage = ''' This is "%(name)s", developed by Jagged Peak Inc. A utility to read a plugin. Enter a comma separated list of plugin files. Usage: %(name)s -c<config_directory> -d<plugin_directory> <pluginfile(s)> Options: -c<directory> directory for configuration files (default: config). -d<directory> directory for plugin file(s) default: D:/BOTS/plugins ''' % { 'name': os.path.basename(sys.argv[0]), 'version': botsglobal.version } configdir = 'config' plugindir = 'D:/BOTS/plugins' pluginfiles = '' # Handle cmd arguments for arg in sys.argv[1:]: if arg.startswith('-c'): configdir = arg[2:] if not configdir: print 'Error: configuration directory indicated, but no directory name.' sys.exit(1) if arg.startswith('-d'): plugindir = arg[2:] if not configdir: print 'Error: plugin directory indicated, but no directory name.' sys.exit(1) else: pluginfiles = arg if not pluginfiles: print 'Plugin file was not entered. See usage' print usage sys.exit(1) # Init required vars and libs botsinit.generalinit( configdir) # find locating of bots, configfiles, init paths etc. process_name = 'plug_read' botsglobal.logger = botsinit.initserverlogging(process_name) import pluglib # import here, import at start of file gives error; first initialize. # Call read plugin for each file pluginfiles_list = pluginfiles.split(',') for pluginfile in pluginfiles_list: plugin_file_path = os.path.join(plugindir, pluginfile) pluglib.read_plugin(plugin_file_path)
def start(): #********command line arguments************************** usage = ''' This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net). Updates existing bots database to version %(version)s Usage: %(name)s [config-option] Options: -c<directory> directory for configuration files (default: config). '''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version} configdir = 'config' for arg in sys.argv[1:]: if arg.startswith('-c'): configdir = arg[2:] if not configdir: print 'Error: configuration directory indicated, but no directory name.' sys.exit(3) else: #pick up names of routes to run print usage sys.exit(0) #***end handling command line arguments************************** botsinit.generalinit(configdir) #find locating of bots, configfiles, init paths etc. #**************check if another instance of bots-engine is running/if port is free****************************** try: engine_socket = botslib.check_if_other_engine_is_running() except socket.error: sys.exit(3) else: atexit.register(engine_socket.close) #**************initialise logging****************************** process_name = 'updatedatabase' botsglobal.logger = botsinit.initenginelogging(process_name) atexit.register(logging.shutdown) for key,value in botslib.botsinfo(): #log info about environement, versions, etc botsglobal.logger.info(u'%(key)s: "%(value)s".',{'key':key,'value':value}) #**************connect to database********************************** try: botsinit.connect() except Exception,msg: botsglobal.logger.exception(_(u'Could not connect to database. Database settings are in bots/config/settings.py. Error: "%(msg)s".'),{'msg':msg}) sys.exit(3)
def startmulti(grammardir,editype): ''' used in seperate tool for bulk checking of gramamrs while developing edifact->botsgramamrs ''' import glob botsinit.generalinit('config') botsinit.initenginelogging() for filename in glob.glob(grammardir): filename_basename = os.path.basename(filename) filename_noextension = os.path.splitext(filename_basename)[0] if filename_basename in ['__init__.py']: continue if filename_basename.startswith('edifact'): continue if filename_basename.startswith('records') or filename_basename.endswith('records.py'): continue try: grammar.grammarread(editype,filename_noextension) except: #~ print 'Found error in grammar:',filename print botslib.txtexc() print '\n' else: print 'OK - no error found in grammar',filename,'\n'
def startmulti(grammardir, editype): ''' used in seperate tool for bulk checking of gramamrs while developing edifact->botsgramamrs ''' import glob botsinit.generalinit('config') botsinit.initenginelogging() for filename in glob.glob(grammardir): filename_basename = os.path.basename(filename) filename_noextension = os.path.splitext(filename_basename)[0] if filename_basename in ['__init__.py']: continue if filename_basename.startswith('edifact'): continue if filename_basename.startswith( 'records') or filename_basename.endswith('records.py'): continue try: grammar.grammarread(editype, filename_noextension) except: #~ print 'Found error in grammar:',filename print botslib.txtexc() print '\n' else: print 'OK - no error found in grammar', filename, '\n'
def start(): #exit codes: # 0: OK, no errors # 1: (system) errors # 2: bots ran OK, but there are errors/process errors in the run # 3: Database is locked, but "maxruntime" has not been exceeded. #********command line arguments************************** commandspossible = [ '--new', '--retry', '--retransmit', '--cleanup', '--crashrecovery', '--retrycommunication', '--automaticretrycommunication' ] commandstorun = [] routestorun = [] #list with routes to run configdir = 'config' for arg in sys.argv[1:]: if not arg: continue if arg.startswith('-c'): configdir = arg[2:] if not configdir: print 'Configuration directory indicated, but no directory name.' sys.exit(1) elif arg in commandspossible: commandstorun.append(arg) elif arg in ["?", "/?"] or arg.startswith('-'): showusage() sys.exit(0) else: #pick up names of routes to run routestorun.append(arg) if not commandstorun: #if no command on command line, use new (default) commandstorun = ['--new'] #**************init general: find locating of bots, configfiles, init paths etc.**************** botsinit.generalinit(configdir) #set current working directory to botspath #~ old_current_directory = os.getcwdu() os.chdir(botsglobal.ini.get('directories', 'botspath')) #**************initialise logging****************************** try: botsinit.initenginelogging() except: print _('Error in initialising logging system.') traceback.print_exc() sys.exit(1) else: atexit.register(logging.shutdown) for key, value in botslib.botsinfo(): #log start info botsglobal.logger.info(u'%s: "%s".', key, value) #**************connect to database********************************** try: botsinit.connect() except: botsglobal.logger.exception( _(u'Could not connect to database. Database settings are in bots/config/settings.py.' )) sys.exit(1) else: botsglobal.logger.info(_(u'Connected to database.')) atexit.register(botsglobal.db.close) #initialise user exits for the whole bots-engine (this script file) try: userscript, scriptname = botslib.botsimport('routescripts', 'botsengine') except ImportError: userscript = scriptname = None #**************handle database lock**************************************** #try to set a lock on the database; if this is not possible, the database is already locked. Either: #1 another instance bots bots-engine is (still) running #2 or bots-engine had a severe crash. #What to do? #first: check ts of database lock. If below a certain value (set in bots.ini) we assume an other instance is running. Exit quietly - no errors, no logging. # else: Warn user, give advise on what to do. gather data: nr files in, errors. #next: warn with report & logging. advise a crashrecovery. if not botslib.set_database_lock(): if '--crashrecovery' in commandstorun: #user starts recovery operation; the databaselock is ignored; the databaselock is unlocked when routes have run. commandstorun = ['--crashrecovery'] #is an exclusive option! else: #when scheduling bots it is possible that the last run is still running. Check if maxruntime has passed: vanaf = datetime.datetime.today() - datetime.timedelta( minutes=botsglobal.ini.getint('settings', 'maxruntime', 60)) for row in botslib.query( '''SELECT ts FROM mutex WHERE ts < %(vanaf)s ''', {'vanaf': vanaf}): warn = _( u'!Bots database is locked!\nBots-engine has ended in an unexpected way during the last run.\nThis happens, but is very very rare.\nPossible causes: bots-engine terminated by user, system crash, power-down, etc.\nA forced retry of the last run is advised; bots will (try to) repair the last run.' ) botsglobal.logger.critical(warn) botslib.sendbotserrorreport( _(u'[Bots severe error]Database is locked'), warn) #add: count errors etc. sys.exit(1) else: #maxruntime has not passed. Exit silently, nothing reported botsglobal.logger.info( _(u'Database is locked, but "maxruntime" has not been exceeded.' )) sys.exit(3) else: if '--crashrecovery' in commandstorun: #user starts recovery operation but there is no databaselock. warn = _( u'User started a forced retry of the last run.\nOnly use this when the database is locked.\nThe database was not locked (database is OK).\nSo Bots has done nothing now.' ) botsglobal.logger.error(warn) botslib.sendbotserrorreport( _(u'[Bots Error Report] User started a forced retry of last run, but this was not needed' ), warn) botslib.remove_database_lock() sys.exit(1) #*************get list of routes to run**************************************** #~ raise Exception('locked database') #for testing database lock: abort, database will be locked if routestorun: botsglobal.logger.info(u'Run routes from command line: "%s".', str(routestorun)) else: # no routes from command line parameters: fetch all active routes from database for row in botslib.query( '''SELECT DISTINCT idroute FROM routes WHERE active=%(active)s AND (notindefaultrun=%(notindefaultrun)s OR notindefaultrun IS NULL) ORDER BY idroute ''', { 'active': True, 'notindefaultrun': False }): routestorun.append(row['idroute']) botsglobal.logger.info(_(u'Run active routes from database: "%s".'), str(routestorun)) #routestorun is now either a list with routes from commandline, or the list of active routes for the routes table in the db. #**************run the routes for retry, retransmit and new runs************************************* try: #commandstorun determines the type(s) of run #routes to run is a listof the routes that are runs (for each command to run #botsglobal.incommunicate is used to control if there is communication in; only 'new' incommunicates. #botsglobal.minta4query controls which ta's are queried by the routes. #stuff2evaluate controls what is evaluated in automatic maintenance. errorinrun = 0 #detect if there has been some error. Only used for good exit() code botsglobal.incommunicate = False if '--crashrecovery' in commandstorun: botsglobal.logger.info(_(u'Run crash recovery.')) stuff2evaluate = botslib.set_minta4query_crashrecovery() if stuff2evaluate: router.routedispatcher(routestorun) errorinrun += automaticmaintenance.evaluate( '--crashrecovery', stuff2evaluate) else: botsglobal.logger.info( _(u'No retry of the last run - there was no last run.')) if userscript and hasattr(userscript, 'postcrashrecovery'): botslib.runscript(userscript, scriptname, 'postcrashrecovery', routestorun=routestorun) if '--retrycommunication' in commandstorun: botsglobal.logger.info(_(u'Run communication retry.')) stuff2evaluate = router.routedispatcher(routestorun, '--retrycommunication') if stuff2evaluate: errorinrun += automaticmaintenance.evaluate( '--retrycommunication', stuff2evaluate) else: botsglobal.logger.info( _(u'Run recommunicate: nothing to recommunicate.')) if userscript and hasattr(userscript, 'postretrycommunication'): botslib.runscript(userscript, scriptname, 'postretrycommunication', routestorun=routestorun) if '--automaticretrycommunication' in commandstorun: botsglobal.logger.info(_(u'Run automatic communication retry.')) stuff2evaluate = router.routedispatcher( routestorun, '--automaticretrycommunication') if stuff2evaluate: errorinrun += automaticmaintenance.evaluate( '--automaticretrycommunication', stuff2evaluate) else: botsglobal.logger.info( _(u'Run automatic recommunicate: nothing to recommunicate.' )) if userscript and hasattr(userscript, 'postautomaticretrycommunication'): botslib.runscript(userscript, scriptname, 'postautomaticretrycommunication', routestorun=routestorun) if '--retry' in commandstorun: botsglobal.logger.info(u'Run retry.') stuff2evaluate = router.routedispatcher(routestorun, '--retry') if stuff2evaluate: errorinrun += automaticmaintenance.evaluate( '--retry', stuff2evaluate) else: botsglobal.logger.info(_(u'Run retry: nothing to retry.')) if userscript and hasattr(userscript, 'postretry'): botslib.runscript(userscript, scriptname, 'postretry', routestorun=routestorun) if '--retransmit' in commandstorun: botsglobal.logger.info(u'Run retransmit.') stuff2evaluate = router.routedispatcher(routestorun, '--retransmit') if stuff2evaluate: errorinrun += automaticmaintenance.evaluate( '--retransmit', stuff2evaluate) else: botsglobal.logger.info( _(u'Run retransmit: nothing to retransmit.')) if userscript and hasattr(userscript, 'postretransmit'): botslib.runscript(userscript, scriptname, 'postretransmit', routestorun=routestorun) if '--new' in commandstorun: botsglobal.logger.info('Run new.') botsglobal.incommunicate = True botsglobal.minta4query = 0 #meaning: reset. the actual value is set later (in routedispatcher) stuff2evaluate = router.routedispatcher(routestorun) errorinrun += automaticmaintenance.evaluate( '--new', stuff2evaluate) if userscript and hasattr(userscript, 'postnewrun'): botslib.runscript(userscript, scriptname, 'postnewrun', routestorun=routestorun) if '--cleanup' in commandstorun or botsglobal.ini.get( 'settings', 'whencleanup', 'always') == 'always': botsglobal.logger.debug(u'Do cleanup.') cleanup.cleanup() botslib.remove_database_lock() except Exception, e: botsglobal.logger.exception( _(u'Severe error in bots system:\n%s') % (e)) #of course this 'should' not happen. sys.exit(1)
def start(): #NOTE bots is always on PYTHONPATH!!! - otherwise it will not start. #********command line arguments************************** configdir = 'config' for arg in sys.argv[1:]: if not arg: continue if arg.startswith('-c'): configdir = arg[2:] if not configdir: print 'Configuration directory indicated, but no directory name.' sys.exit(1) elif arg in ["?", "/?"] or arg.startswith('-'): showusage() else: showusage() #init general: find locating of bots, configfiles, init paths etc.*********************** botsinit.generalinit(configdir) #init cherrypy; only needed for webserver. ********************************************* cherrypy.config.update({'global': { 'tools.staticdir.root': botsglobal.ini.get('directories','botspath'), 'server.socket_host' : "0.0.0.0", #to what IP addresses should be server. 0.0.0.0: all. See cherrypy docs 'server.socket_port': botsglobal.ini.getint('webserver','port',8080), 'server.environment': botsglobal.ini.get('webserver','environment','development'), # development production 'log.screen': False, #~ 'log.error_file': '', #set later to rotating log file #~ 'log.access_file': '', #set later to rotating log file }}) conf = {'/': {'tools.staticdir.on' : True,'tools.staticdir.dir' : 'media' }} #~ '/favicon.ico': {'tools.staticfile.on': True,'tools.staticfile.filename': '/home/hje/botsup-django/bots/media/images/favicon.ico'}} cherrypy.tree.graft(AdminMediaHandler(WSGIHandler()), '/') myroot = Root() myappl = cherrypy.tree.mount(myroot, '/media', conf) #myappl is needed to set logging botsglobal.logger = logging.getLogger('webserver') botsglobal.logger.setLevel(logging.DEBUG) h = logging.handlers.TimedRotatingFileHandler(botslib.join(botsglobal.ini.get('directories','logging'),'webserver.log'),when='midnight', backupCount=10) fileformat = logging.Formatter("%(asctime)s %(levelname)-8s %(name)s : %(message)s",'%Y%m%d %H:%M:%S') h.setFormatter(fileformat) # add rotating file handler to main logger botsglobal.logger.addHandler(h) # Make RotatingFileHandler for the error log. #~ h = logging.handlers.TimedRotatingFileHandler(botslib.join(botsglobal.ini.get('directories','logging'),'webserver.log'),when='midnight', backupCount=10) #~ fileformat = logging.Formatter("%(asctime)s %(levelname)-8s %(name)s : %(message)s",'%Y%m%d %H:%M:%S') #~ h.setLevel(logging.INFO) #~ h.setFormatter(fileformat) #~ myappl.log.error_log.addHandler(h) # MakeRotatingFileHandler for the access log. #~ h = logging.handlers.TimedRotatingFileHandler(os.path.normpath(os.path.join(botsglobal.ini.get('directories','botspath'), 'botssys/logging/webserver.log')),when='midnight', backupCount=10) #~ h.setLevel(logging.DEBUG) #~ myappl.log.access_log.addHandler(h) #~ botsglobal.logger = myappl.log.access_log #write start info to cherrypy log******************************************** botsglobal.logger.info(_(u'Bots web server started.')) botsglobal.logger.info(_(u'Python version: "%s".'),sys.version) botsglobal.logger.info(_(u'Django version: "%s".'),django.VERSION) #start cherrypy ********************************************************************* cherrypy.engine.start() cherrypy.engine.block()
def start(): #********command line arguments************************** usage = ''' This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net). Creates a grammar from an xml file.' Usage:' %(name)s -c<directory> <xml_file> <xml_grammar_file> Options: -c<directory> directory for configuration files (default: config). -a all xml elements as records <xml_file> name of the xml file to read <xml_grammar_file> name of the grammar file to write '''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version} configdir = 'config' edifile ='' botsgrammarfilename = '' allrecords = False for arg in sys.argv[1:]: if arg.startswith('-c'): configdir = arg[2:] if not configdir: print 'Error: configuration directory indicated, but no directory name.' sys.exit(1) elif arg.startswith('-a'): allrecords = True elif arg in ["?", "/?",'-h', '--help'] or arg.startswith('-'): print usage sys.exit(0) else: if not edifile: edifile = arg else: botsgrammarfilename = arg if not edifile or not botsgrammarfilename: print 'Error: both edifile and grammarfile are required.' sys.exit(0) #***end handling command line arguments************************** botsinit.generalinit(configdir) #find locating of bots, configfiles, init paths etc. process_name = 'xml2botsgrammar' botsglobal.logger = botsinit.initenginelogging(process_name) atexit.register(logging.shutdown) targetNamespace = '' #******************************************************************* #***add classes for handling editype xml to inmessage #******************************************************************* if allrecords: #~ editype = 'xmlforgrammar_allrecords' inmessage.xmlforgrammar = xmlforgrammar_allrecords else: #~ editype = 'xmlforgrammar' inmessage.xmlforgrammar = xmlforgrammar #make inmessage object: read the xml file inn = inmessage.parse_edi_file(editype='xmlforgrammar',messagetype='',filename=edifile) inn.checkforerrorlist() #no exception if infile has been lexed and parsed OK else raises an error #make outmessage object; nothing is 'filled' yet. In mapping tree is filled; nothing is written to file. out = outmessage.outmessage_init(editype='xmlnocheck',messagetype='',filename='',divtext='',topartner='') #***mapping: make 'normalised' out-tree suited for writing as a grammar*************************************************** mpath_root = [OrderedDict({'BOTSID':inn.root.record['BOTSID'],'BOTSIDnr':'1'})] #handle root out.put(*mpath_root) map_writefields(out,inn.root,mpath_root) #walk tree; write results to out-tree mpath_start = [] for node_instance,mpath in map_treewalker(inn.root,mpath_start): mpath.append(OrderedDict({'BOTSID':node_instance.record['BOTSID']})) if out.get(*mpath) is None: #if node does not exist: write it. out.put(*mpath) map_writefields(out,node_instance,mpath) #***mapping is done #***convert out-tree to grammar structure = [] recorddefs = {} tree2grammar(out.root,structure,recorddefs) #***write grammar to file grammar2file(botsgrammarfilename,structure,recorddefs,targetNamespace)
def start(): ''' sysexit codes: 0: OK, no errors 1: (system) errors incl parsing of command line arguments 2: bots ran OK, but there are errors/process errors in the run 3: Database is locked, but "maxruntime" has not been exceeded. ''' #NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start. #********command line arguments************************** usage = ''' This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net). Does the actual translations and communications; it's the workhorse. It does not have a fancy interface. Usage: %(name)s [run-options] [config-option] [routes] Run-options (can be combined): --new receive new edi files (default: if no run-option given: run as new). --resend resend as indicated by user. --rereceive rereceive as indicated by user. --automaticretrycommunication - automatically retry outgoing communication. --cleanup remove older data from database. Config-option: -c<directory> directory for configuration files (default: config). Routes: list of routes to run. Default: all active routes (in the database) '''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version} configdir = 'config' commandspossible = ['--automaticretrycommunication','--resend','--rereceive','--new'] commandstorun = [] routestorun = [] #list with routes to run do_cleanup_parameter = False for arg in sys.argv[1:]: if arg.startswith('-c'): configdir = arg[2:] if not configdir: print 'Error: configuration directory indicated, but no directory name.' sys.exit(1) elif arg in commandspossible: commandstorun.append(arg) elif arg == '--cleanup': do_cleanup_parameter = True elif arg in ["?", "/?",'-h', '--help'] or arg.startswith('-'): print usage sys.exit(0) else: #pick up names of routes to run routestorun.append(arg) if not commandstorun and not do_cleanup_parameter: #if no command on command line, use new (default) commandstorun = ['--new'] commandstorun = [command[2:] for command in commandspossible if command in commandstorun] #sort commands #***********end handling command line arguments************************** botsinit.generalinit(configdir) #find locating of bots, configfiles, init paths etc. #set working directory to bots installation. advantage: when using relative paths it is clear that this point paths within bots installation. os.chdir(botsglobal.ini.get('directories','botspath')) #**************check if another instance of bots-engine is running/if port is free****************************** try: engine_socket = botslib.check_if_other_engine_is_running() except socket.error: sys.exit(3) else: atexit.register(engine_socket.close) #**************initialise logging****************************** process_name = 'engine' botsglobal.logger = botsinit.initenginelogging(process_name) atexit.register(logging.shutdown) for key,value in botslib.botsinfo(): #log info about environement, versions, etc botsglobal.logger.info(u'%(key)s: "%(value)s".',{'key':key,'value':value}) #**************connect to database********************************** try: botsinit.connect() except Exception as msg: botsglobal.logger.exception(_(u'Could not connect to database. Database settings are in bots/config/settings.py. Error: "%(msg)s".'),{'msg':msg}) sys.exit(1) else: botsglobal.logger.info(_(u'Connected to database.')) atexit.register(botsglobal.db.close) #************initialise user exits for the whole bots-engine************************* try: userscript,scriptname = botslib.botsimport('routescripts','botsengine') except ImportError: #userscript is not there; other errors like syntax errors are not catched userscript = scriptname = None #***acceptance tests: initialiase acceptance user script****************************** acceptance_userscript = acceptance_scriptname = None if botsglobal.ini.getboolean('acceptance','runacceptancetest',False): botsglobal.logger.info(_(u'This run is an acceptance test - as indicated in option "runacceptancetest" in bots.ini.')) try: acceptance_userscript,acceptance_scriptname = botslib.botsimport('routescripts','bots_acceptancetest') except ImportError: botsglobal.logger.info(_(u'In acceptance test there is no script file "bots_acceptancetest.py" to check the results of the acceptance test.')) #**************handle database lock**************************************** #set a lock on the database; if not possible, the database is locked: an earlier instance of bots-engine was terminated unexpectedly. if not botslib.set_database_lock(): #for SQLite: do a integrity check on the database if botsglobal.settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3': cursor = botsglobal.db.execute('''PRAGMA integrity_check''') result = cursor.fetchone() if result[0] != u'ok': warn = _(u'!Bots database is locked!\n'\ 'Bots did an integrity check on the database, but database was not OK.\n'\ 'Manual action is needed!\n'\ 'Bots has stopped processing EDI files.') botsglobal.logger.critical(warn) botslib.sendbotserrorreport(_(u'[Bots severe error]Database is damaged'),warn) sys.exit(1) warn = _(u'!Bots database is locked!\n'\ 'Bots-engine has ended in an unexpected way during the last run.\n'\ 'Most likely causes: sudden power-down, system crash, problems with disk I/O, bots-engine terminated by user, etc.\n' 'Bots will do an automatic crash recovery now.') botsglobal.logger.critical(warn) botslib.sendbotserrorreport(_(u'[Bots severe error]Database is locked'),warn) commandstorun.insert(0,'crashrecovery') #there is a database lock. Add a crashrecovery as first command to run. atexit.register(botslib.remove_database_lock) #**************run the routes********************************************** #commandstorun determines the type(s) of run. eg: ['automaticretrycommunication','new'] try: botslib.prepare_confirmrules() #in acceptance tests: run a user script before running eg to clean output directories****************************** botslib.tryrunscript(acceptance_userscript,acceptance_scriptname,'pretest',routestorun=routestorun) botslib.tryrunscript(userscript,scriptname,'pre',commandstorun=commandstorun,routestorun=routestorun) errorinrun = 0 #detect if there has been some error. Only used for correct exit() code first_command_2_run = True for command in commandstorun: #if multiple commands in run: reports etc are based on timestamp; so there needs to be at least one second between these runs. if first_command_2_run: first_command_2_run = False else: time.sleep(1) botsglobal.logger.info(_(u'Run "%(command)s".'),{'command':command}) #************get list of routes to run******************************* if routestorun: use_routestorun = routestorun[:] botsglobal.logger.info(_(u'Run routes from command line: "%(routes)s".'),{'routes':str(use_routestorun)}) elif command == 'new': #fetch all active routes from database unless 'not in default run' or not active. use_routestorun = [] for row in botslib.query('''SELECT DISTINCT idroute FROM routes WHERE active=%(active)s AND (notindefaultrun=%(notindefaultrun)s OR notindefaultrun IS NULL) ORDER BY idroute ''', {'active':True,'notindefaultrun':False}): use_routestorun.append(row['idroute']) botsglobal.logger.info(_(u'Run active routes from database that are in default run: "%(routes)s".'),{'routes':str(use_routestorun)}) else: #for command other than 'new': use all active routes. use_routestorun = [] for row in botslib.query('''SELECT DISTINCT idroute FROM routes WHERE active=%(active)s ORDER BY idroute ''', {'active':True}): use_routestorun.append(row['idroute']) botsglobal.logger.info(_(u'Run all active routes from database: "%(routes)s".'),{'routes':str(use_routestorun)}) #************run routes for this command****************************** botslib.tryrunscript(userscript,scriptname,'pre' + command,routestorun=use_routestorun) errorinrun += router.rundispatcher(command,use_routestorun) botslib.tryrunscript(userscript,scriptname,'post' + command,routestorun=use_routestorun) #*********finished running routes for this command**************************** #*********finished all commands**************************************** botslib.tryrunscript(userscript,scriptname,'post',commandstorun=commandstorun,routestorun=routestorun) try: #in acceptance tests: run a user script. no good reporting of errors/results in post-test script. Reason: this is after automaticmaintence. botslib.tryrunscript(acceptance_userscript,acceptance_scriptname,'posttest',routestorun=use_routestorun) except Exception as msg: print str(msg) cleanup.cleanup(do_cleanup_parameter,userscript,scriptname) except Exception as msg: botsglobal.logger.exception(_(u'Severe error in bots system:\n%(msg)s'),{'msg':str(msg)}) #of course this 'should' not happen. sys.exit(1) else: if errorinrun: sys.exit(2) #indicate: error(s) in run(s) else: sys.exit(0) #OK
def start(): #exit codes: # 0: OK, no errors # 1: (system) errors # 2: bots ran OK, but there are errors/process errors in the run # 3: Database is locked, but "maxruntime" has not been exceeded. #********command line arguments************************** commandspossible = ['--new','--retry','--retransmit','--cleanup','--crashrecovery','--retrycommunication','--automaticretrycommunication'] commandstorun = [] routestorun = [] #list with routes to run configdir = 'config' for arg in sys.argv[1:]: if not arg: continue if arg.startswith('-c'): configdir = arg[2:] if not configdir: print 'Configuration directory indicated, but no directory name.' sys.exit(1) elif arg in commandspossible: commandstorun.append(arg) elif arg in ["?", "/?"] or arg.startswith('-'): showusage() sys.exit(0) else: #pick up names of routes to run routestorun.append(arg) if not commandstorun: #if no command on command line, use new (default) commandstorun = ['--new'] #**************init general: find locating of bots, configfiles, init paths etc.**************** botsinit.generalinit(configdir) #set current working directory to botspath #~ old_current_directory = os.getcwdu() os.chdir(botsglobal.ini.get('directories','botspath')) #**************initialise logging****************************** try: botsinit.initenginelogging() except: print _('Error in initialising logging system.') traceback.print_exc() sys.exit(1) else: atexit.register(logging.shutdown) for key,value in botslib.botsinfo(): #log start info botsglobal.logger.info(u'%s: "%s".',key,value) #**************connect to database********************************** try: botsinit.connect() except: botsglobal.logger.exception(_(u'Could not connect to database. Database settings are in bots/config/settings.py.')) sys.exit(1) else: botsglobal.logger.info(_(u'Connected to database.')) atexit.register(botsglobal.db.close) #initialise user exits for the whole bots-engine (this script file) try: userscript,scriptname = botslib.botsimport('routescripts','botsengine') except ImportError: userscript = scriptname = None #**************handle database lock**************************************** #try to set a lock on the database; if this is not possible, the database is already locked. Either: #1 another instance bots bots-engine is (still) running #2 or bots-engine had a severe crash. #What to do? #first: check ts of database lock. If below a certain value (set in bots.ini) we assume an other instance is running. Exit quietly - no errors, no logging. # else: Warn user, give advise on what to do. gather data: nr files in, errors. #next: warn with report & logging. advise a crashrecovery. if not botslib.set_database_lock(): if '--crashrecovery' in commandstorun: #user starts recovery operation; the databaselock is ignored; the databaselock is unlocked when routes have run. commandstorun = ['--crashrecovery'] #is an exclusive option! else: #when scheduling bots it is possible that the last run is still running. Check if maxruntime has passed: vanaf = datetime.datetime.today() - datetime.timedelta(minutes=botsglobal.ini.getint('settings','maxruntime',60)) for row in botslib.query('''SELECT ts FROM mutex WHERE ts < %(vanaf)s ''',{'vanaf':vanaf}): warn = _(u'!Bots database is locked!\nBots-engine has ended in an unexpected way during the last run.\nThis happens, but is very very rare.\nPossible causes: bots-engine terminated by user, system crash, power-down, etc.\nA forced retry of the last run is advised; bots will (try to) repair the last run.') botsglobal.logger.critical(warn) botslib.sendbotserrorreport(_(u'[Bots severe error]Database is locked'),warn) #add: count errors etc. sys.exit(1) else: #maxruntime has not passed. Exit silently, nothing reported botsglobal.logger.info(_(u'Database is locked, but "maxruntime" has not been exceeded.')) sys.exit(3) else: if '--crashrecovery' in commandstorun: #user starts recovery operation but there is no databaselock. warn = _(u'User started a forced retry of the last run.\nOnly use this when the database is locked.\nThe database was not locked (database is OK).\nSo Bots has done nothing now.') botsglobal.logger.error(warn) botslib.sendbotserrorreport(_(u'[Bots Error Report] User started a forced retry of last run, but this was not needed'),warn) botslib.remove_database_lock() sys.exit(1) #*************get list of routes to run**************************************** #~ raise Exception('locked database') #for testing database lock: abort, database will be locked if routestorun: botsglobal.logger.info(u'Run routes from command line: "%s".',str(routestorun)) else: # no routes from command line parameters: fetch all active routes from database for row in botslib.query('''SELECT DISTINCT idroute FROM routes WHERE active=%(active)s AND (notindefaultrun=%(notindefaultrun)s OR notindefaultrun IS NULL) ORDER BY idroute ''', {'active':True,'notindefaultrun':False}): routestorun.append(row['idroute']) botsglobal.logger.info(_(u'Run active routes from database: "%s".'),str(routestorun)) #routestorun is now either a list with routes from commandline, or the list of active routes for the routes table in the db. #**************run the routes for retry, retransmit and new runs************************************* try: #commandstorun determines the type(s) of run #routes to run is a listof the routes that are runs (for each command to run #botsglobal.incommunicate is used to control if there is communication in; only 'new' incommunicates. #botsglobal.minta4query controls which ta's are queried by the routes. #stuff2evaluate controls what is evaluated in automatic maintenance. errorinrun = 0 #detect if there has been some error. Only used for good exit() code botsglobal.incommunicate = False if '--crashrecovery' in commandstorun: botsglobal.logger.info(_(u'Run crash recovery.')) stuff2evaluate = botslib.set_minta4query_crashrecovery() if stuff2evaluate: router.routedispatcher(routestorun) errorinrun += automaticmaintenance.evaluate('--crashrecovery',stuff2evaluate) else: botsglobal.logger.info(_(u'No retry of the last run - there was no last run.')) if userscript and hasattr(userscript,'postcrashrecovery'): botslib.runscript(userscript,scriptname,'postcrashrecovery',routestorun=routestorun) if '--retrycommunication' in commandstorun: botsglobal.logger.info(_(u'Run communication retry.')) stuff2evaluate = router.routedispatcher(routestorun,'--retrycommunication') if stuff2evaluate: errorinrun += automaticmaintenance.evaluate('--retrycommunication',stuff2evaluate) else: botsglobal.logger.info(_(u'Run recommunicate: nothing to recommunicate.')) if userscript and hasattr(userscript,'postretrycommunication'): botslib.runscript(userscript,scriptname,'postretrycommunication',routestorun=routestorun) if '--automaticretrycommunication' in commandstorun: botsglobal.logger.info(_(u'Run automatic communication retry.')) stuff2evaluate = router.routedispatcher(routestorun,'--automaticretrycommunication') if stuff2evaluate: errorinrun += automaticmaintenance.evaluate('--automaticretrycommunication',stuff2evaluate) else: botsglobal.logger.info(_(u'Run automatic recommunicate: nothing to recommunicate.')) if userscript and hasattr(userscript,'postautomaticretrycommunication'): botslib.runscript(userscript,scriptname,'postautomaticretrycommunication',routestorun=routestorun) if '--retry' in commandstorun: botsglobal.logger.info(u'Run retry.') stuff2evaluate = router.routedispatcher(routestorun,'--retry') if stuff2evaluate: errorinrun += automaticmaintenance.evaluate('--retry',stuff2evaluate) else: botsglobal.logger.info(_(u'Run retry: nothing to retry.')) if userscript and hasattr(userscript,'postretry'): botslib.runscript(userscript,scriptname,'postretry',routestorun=routestorun) if '--retransmit' in commandstorun: botsglobal.logger.info(u'Run retransmit.') stuff2evaluate = router.routedispatcher(routestorun,'--retransmit') if stuff2evaluate: errorinrun += automaticmaintenance.evaluate('--retransmit',stuff2evaluate) else: botsglobal.logger.info(_(u'Run retransmit: nothing to retransmit.')) if userscript and hasattr(userscript,'postretransmit'): botslib.runscript(userscript,scriptname,'postretransmit',routestorun=routestorun) if '--new' in commandstorun: botsglobal.logger.info('Run new.') botsglobal.incommunicate = True botsglobal.minta4query = 0 #meaning: reset. the actual value is set later (in routedispatcher) stuff2evaluate = router.routedispatcher(routestorun) errorinrun += automaticmaintenance.evaluate('--new',stuff2evaluate) if userscript and hasattr(userscript,'postnewrun'): botslib.runscript(userscript,scriptname,'postnewrun',routestorun=routestorun) if '--cleanup' in commandstorun or botsglobal.ini.get('settings','whencleanup','always')=='always': botsglobal.logger.debug(u'Do cleanup.') cleanup.cleanup() botslib.remove_database_lock() except Exception,e: botsglobal.logger.exception(_(u'Severe error in bots system:\n%s')%(e)) #of course this 'should' not happen. sys.exit(1)
def start(): #********command line arguments************************** edifile ='' grammarfile = '' configdir = 'config' for arg in sys.argv[1:]: if not arg: continue if arg.startswith('-c'): configdir = arg[2:] if not configdir: print ' !!Indicated Bots should use specific .ini file but no file name was given.' showusage() elif arg in ["?", "/?"] or arg.startswith('-'): showusage() else: if not edifile: edifile = arg else: grammarfile = arg if not (edifile and grammarfile): print ' !!Both edifile and grammarfile are required.' showusage() #********end handling command line arguments************************** editype='xmlnocheck' messagetype='xmlnocheckxxxtemporaryforxml2grammar' mpath = [] botsinit.generalinit(configdir) os.chdir(botsglobal.ini.get('directories','botspath')) botsinit.initenginelogging() #the xml file is parsed as an xmlnocheck message....so a (temp) xmlnocheck grammar is needed....without content... this file is not removed.... tmpgrammarfile = botslib.join(botsglobal.ini.get('directories','usersysabs'),'grammars',editype,messagetype+'.py') f = open(tmpgrammarfile,'w') f.close() inn = inmessage.edifromfile(editype=editype,messagetype=messagetype,filename=edifile) #~ inn.root.display() out = outmessage.outmessage_init(editype=editype,messagetype=messagetype,filename='botssys/infile/unitnode/output/inisout03.edi',divtext='',topartner='') #make outmessage object #handle root rootmpath = [{'BOTSID':inn.root.record['BOTSID']}] out.put(*rootmpath) writefields(out,inn.root,rootmpath) #walk tree; write results to out-tree for node,mpath in treewalker(inn.root,mpath): mpath.append({'BOTSID':node.record['BOTSID']}) if out.get(*mpath) is None: out.put(*mpath) writefields(out,node,mpath) #~ out.root.display() #out-tree is finished; represents ' normalised' tree suited for writing as a grammar structure = [] recorddefs = {} tree2grammar(out.root,structure,recorddefs) #~ for key,value in recorddefs.items(): #~ print key,value #~ print '\n' sortedstructurelist = structure2list(structure) recorddefsstring = recorddefs2string(recorddefs,sortedstructurelist) structurestring = structure2string(structure) #write grammar file grammar = open(grammarfile,'wb') grammar.write('#grammar automatically generated by bots open source edi software.') grammar.write('\n') grammar.write('from bots.botsconfig import *') grammar.write('\n\n') grammar.write('syntax = {}') grammar.write('\n\n') grammar.write('structure = [\n%s]\n'%(structurestring)) grammar.write('\n\n') grammar.write('recorddefs = %s'%(recorddefsstring)) grammar.write('\n\n') grammar.close() print 'grammar file is written',grammarfile
def start(): #********command line arguments************************** usage = ''' This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net). Updates existing bots database to version %(version)s Usage: %(name)s [config-option] Options: -c<directory> directory for configuration files (default: config). '''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version} configdir = 'config' for arg in sys.argv[1:]: if arg.startswith('-c'): configdir = arg[2:] if not configdir: print 'Error: configuration directory indicated, but no directory name.' sys.exit(3) else: #pick up names of routes to run print usage sys.exit(0) #***end handling command line arguments************************** botsinit.generalinit(configdir) #find locating of bots, configfiles, init paths etc. #**************check if another instance of bots-engine is running/if port is free****************************** try: engine_socket = botslib.check_if_other_engine_is_running() except socket.error: sys.exit(3) else: atexit.register(engine_socket.close) #**************initialise logging****************************** process_name = 'updatedatabase' botsglobal.logger = botsinit.initenginelogging(process_name) atexit.register(logging.shutdown) for key,value in botslib.botsinfo(): #log info about environement, versions, etc botsglobal.logger.info(u'%(key)s: "%(value)s".',{'key':key,'value':value}) #**************connect to database********************************** try: botsinit.connect() except Exception as msg: botsglobal.logger.exception(_(u'Could not connect to database. Database settings are in bots/config/settings.py. Error: "%(msg)s".'),{'msg':msg}) sys.exit(3) else: botsglobal.logger.info(_(u'Connected to database.')) atexit.register(botsglobal.db.close) #**************handle database lock**************************************** #set a lock on the database; if not possible, the database is locked: an earlier instance of bots-engine was terminated unexpectedly. if not botslib.set_database_lock(): warn = _(u'!Bots database is locked!\n'\ 'Bots-engine has ended in an unexpected way during the last run.\n'\ 'Most likely causes: sudden power-down, system crash, problems with disk I/O, bots-engine terminated by user, etc.') botsglobal.logger.critical(warn) sys.exit(3) atexit.register(botslib.remove_database_lock) if botsglobal.settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3': terug = sqlite3() elif botsglobal.settings.DATABASES['default']['ENGINE'] == 'django.db.backends.mysql': terug = mysql() elif botsglobal.settings.DATABASES['default']['ENGINE'] == 'django.db.backends.postgresql_psycopg2': terug = postgresql_psycopg2() sys.exit(terug)
def start(configdir): botsinit.generalinit(configdir) #find locating of bots, configfiles, init paths etc. process_name = 'apache_webserver_' + configdir botsglobal.logger = botsinit.initserverlogging(process_name) #initialise file-logging for web-server. This logging only contains the logging from bots-webserver.
def start(): #********command line arguments************************** usage = ''' This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net). Creates a grammar from an xml file.' Usage:' %(name)s -c<directory> <xml_file> <xml_grammar_file> Options: -c<directory> directory for configuration files (default: config). <xml_file> name of the xml file to read <xml_grammar_file> name of the grammar file to write '''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version} configdir = 'config' edifile ='' grammarfile = '' for arg in sys.argv[1:]: if arg.startswith('-c'): configdir = arg[2:] if not configdir: print 'Error: configuration directory indicated, but no directory name.' sys.exit(1) elif arg in ["?", "/?",'-h', '--help'] or arg.startswith('-'): print usage sys.exit(0) else: if not edifile: edifile = arg else: grammarfile = arg if not edifile or not grammarfile: print 'Error: both edifile and grammarfile are required.' sys.exit(0) #***end handling command line arguments************************** botsinit.generalinit(configdir) #find locating of bots, configfiles, init paths etc. process_name = 'xml2botsgrammar' botsglobal.logger = botsinit.initenginelogging(process_name) atexit.register(logging.shutdown) #the xml file is parsed as an xmlnocheck message editype = 'xmlnocheck' messagetype = 'xmlnocheckxxxtemporaryforxml2grammar' mpath = [] #a (temp) xmlnocheck grammar is needed (but needs not actual content. This file is not removed. tmpgrammarfile = botslib.join(botsglobal.ini.get('directories','usersysabs'),'grammars',editype,messagetype+'.py') filehandler = open(tmpgrammarfile,'w') filehandler.close() #make inmessage object: read the xml file inn = inmessage.parse_edi_file(editype=editype,messagetype=messagetype,filename=edifile,remove_empties_from_xml=False) #make outmessage object; nothing is 'filled' yet. out = outmessage.outmessage_init(editype=editype,messagetype=messagetype,filename='botssys/infile/unitnode/output/inisout03.edi',divtext='',topartner='') #***do the mapping*************************************************** #handle root rootmpath = [{'BOTSID':inn.root.record['BOTSID'],'BOTSIDnr':'1'}] out.put(*rootmpath) map_writefields(out,inn.root,rootmpath) #walk tree; write results to out-tree for node_instance,mpath in map_treewalker(inn.root,mpath): mpath.append({'BOTSID':node_instance.record['BOTSID']}) if out.get(*mpath) is None: out.put(*mpath) map_writefields(out,node_instance,mpath) #***mapping is done; out-tree is finished; represents 'normalised' tree suited for writing as a grammar structure = [] recorddefs = {} tree2grammar(out.root,structure,recorddefs) #~ for key,value in recorddefs.items(): #~ print key,value #~ print '\n' sortedstructurelist = structure2list(structure) recorddefsstring = recorddefs2string(recorddefs,sortedstructurelist) structurestring = structure2string(structure) #write grammar file grammar = open(grammarfile,'wb') grammar.write('#grammar automatically generated by bots open source edi translator.') grammar.write('\n') grammar.write('from bots.botsconfig import *') grammar.write('\n\n') grammar.write('syntax = {}') grammar.write('\n\n') grammar.write('structure = [\n%s]\n'%(structurestring)) grammar.write('\n\n') grammar.write('recorddefs = %s'%(recorddefsstring)) grammar.write('\n\n') grammar.close() print 'grammar file is written:',grammarfile
def start(): # NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start. # ***command line arguments************************** usage = """ This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net). A utility to generate the index file of a plugin; this can be seen as a database dump of the configuration. This is eg useful for version control. Usage: %(name)s -c<directory> Options: -c<directory> directory for configuration files (default: config). """ % { "name": os.path.basename(sys.argv[0]), "version": botsglobal.version, } configdir = "config" for arg in sys.argv[1:]: if arg.startswith("-c"): configdir = arg[2:] if not configdir: print "Error: configuration directory indicated, but no directory name." sys.exit(1) else: print usage sys.exit(0) # ***end handling command line arguments************************** botsinit.generalinit(configdir) # find locating of bots, configfiles, init paths etc. if not botsglobal.ini.getboolean("jobqueue", "enabled", False): print "Error: bots jobqueue cannot start; not enabled in %s/bots.ini" % configdir sys.exit(1) process_name = "dirmonitor" logger = botsinit.initserverlogging(process_name) logger.log(25, u"Bots %(process_name)s started.", {"process_name": process_name}) logger.log( 25, u'Bots %(process_name)s configdir: "%(configdir)s".', {"process_name": process_name, "configdir": botsglobal.ini.get("directories", "config")}, ) botsenginepath = os.path.join( os.path.dirname(os.path.abspath(sys.argv[0])), "bots-engine.py" ) # get path to bots-engine cond = threading.Condition() tasks = set() dir_watch_data = [] for section in botsglobal.ini.sections(): if section.startswith("dirmonitor") and section[len("dirmonitor") :]: dir_watch_data.append({}) dir_watch_data[-1]["path"] = botsglobal.ini.get(section, "path") dir_watch_data[-1]["rec"] = botsglobal.ini.getboolean(section, "recursive", False) dir_watch_data[-1]["filemask"] = botsglobal.ini.get(section, "filemask", "*") dir_watch_data[-1]["route"] = botsglobal.ini.get(section, "route", "") if not dir_watch_data: logger.error(u"Nothing to watch!") sys.exit(0) if os.name == "nt": # for windows: start a thread per directory watcher for dir_watch in dir_watch_data: dir_watch_thread = threading.Thread(target=windows_event_handler, args=(logger, dir_watch, cond, tasks)) dir_watch_thread.daemon = True # do not wait for thread when exiting dir_watch_thread.start() else: # for linux: one watch-thread, but multiple watches. dir_watch_thread = threading.Thread(target=linux_event_handler, args=(logger, dir_watch_data, cond, tasks)) dir_watch_thread.daemon = True # do not wait for thread when exiting dir_watch_thread.start() # this main thread get the results from the watch-thread(s). logger.info(u"Bots %(process_name)s started.", {"process_name": process_name}) active_receiving = False timeout = 2.0 cond.acquire() while True: # this functions as a buffer: all events go into set tasks. # the tasks are fired to jobqueue after TIMOUT sec. # this is to avoid firing to many tasks to jobqueue; events typically come in bursts. # is value of timeout is larger, reaction times are slower...but less tasks are fired to jobqueue. # in itself this is not a problem, as jobqueue will alos discard duplicate jobs. # 2 sec seems to e a good value: reasonable quick, not to nervous. cond.wait(timeout=timeout) # get back when results, or after timeout sec if tasks: if not active_receiving: # first request (after tasks have been fired, or startup of dirmonitor) active_receiving = True last_time = time.time() else: # active receiving events current_time = time.time() if current_time - last_time >= timeout: # cond.wait returned probably because of a timeout try: for task in tasks: logger.info( u'Send to queue "%(path)s %(config)s %(task)s".', {"path": botsenginepath, "config": "-c" + configdir, "task": task}, ) job2queue.send_job_to_jobqueue([sys.executable, botsenginepath, "-c" + configdir, task]) except Exception as msg: logger.info(u'Error in running task: "%(msg)s".', {"msg": msg}) tasks.clear() active_receiving = False else: # cond.wait returned probably because of a timeout logger.debug(u"time difference to small.") last_time = current_time cond.release() sys.exit(0)
def start(): #NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start. #***command line arguments************************** usage = ''' This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net). A utility to generate the index file of a plugin; this can be seen as a database dump of the configuration. This is eg useful for version control. Usage: %(name)s -c<directory> Options: -c<directory> directory for configuration files (default: config). ''' % { 'name': os.path.basename(sys.argv[0]), 'version': botsglobal.version } configdir = 'config' for arg in sys.argv[1:]: if arg.startswith('-c'): configdir = arg[2:] if not configdir: print 'Error: configuration directory indicated, but no directory name.' sys.exit(1) else: print usage sys.exit(0) #***end handling command line arguments************************** botsinit.generalinit( configdir) #find locating of bots, configfiles, init paths etc. if not botsglobal.ini.getboolean('jobqueue', 'enabled', False): print 'Error: bots jobqueue cannot start; not enabled in %s/bots.ini' % configdir sys.exit(1) process_name = 'dirmonitor' logger = botsinit.initserverlogging(process_name) logger.log(25, u'Bots %(process_name)s started.', {'process_name': process_name}) logger.log( 25, u'Bots %(process_name)s configdir: "%(configdir)s".', { 'process_name': process_name, 'configdir': botsglobal.ini.get('directories', 'config') }) botsenginepath = os.path.join(os.path.dirname(os.path.abspath( sys.argv[0])), 'bots-engine.py') #get path to bots-engine cond = threading.Condition() tasks = set() dir_watch_data = [] for section in botsglobal.ini.sections(): if section.startswith('dirmonitor') and section[len('dirmonitor'):]: dir_watch_data.append({}) dir_watch_data[-1]['path'] = botsglobal.ini.get(section, 'path') dir_watch_data[-1]['rec'] = botsglobal.ini.getboolean( section, 'recursive', False) dir_watch_data[-1]['filemask'] = botsglobal.ini.get( section, 'filemask', '*') dir_watch_data[-1]['route'] = botsglobal.ini.get( section, 'route', '') if not dir_watch_data: logger.error(u'Nothing to watch!') sys.exit(0) if os.name == 'nt': #for windows: start a thread per directory watcher for dir_watch in dir_watch_data: dir_watch_thread = threading.Thread(target=windows_event_handler, args=(logger, dir_watch, cond, tasks)) dir_watch_thread.daemon = True #do not wait for thread when exiting dir_watch_thread.start() else: #for linux: one watch-thread, but multiple watches. dir_watch_thread = threading.Thread(target=linux_event_handler, args=(logger, dir_watch_data, cond, tasks)) dir_watch_thread.daemon = True #do not wait for thread when exiting dir_watch_thread.start() # this main thread get the results from the watch-thread(s). logger.info(u'Bots %(process_name)s started.', {'process_name': process_name}) active_receiving = False timeout = 2.0 cond.acquire() while True: #this functions as a buffer: all events go into set tasks. #the tasks are fired to jobqueue after TIMOUT sec. #this is to avoid firing to many tasks to jobqueue; events typically come in bursts. #is value of timeout is larger, reaction times are slower...but less tasks are fired to jobqueue. #in itself this is not a problem, as jobqueue will alos discard duplicate jobs. #2 sec seems to e a good value: reasonable quick, not to nervous. cond.wait( timeout=timeout) #get back when results, or after timeout sec if tasks: if not active_receiving: #first request (after tasks have been fired, or startup of dirmonitor) active_receiving = True last_time = time.time() else: #active receiving events current_time = time.time() if current_time - last_time >= timeout: #cond.wait returned probably because of a timeout try: for task in tasks: logger.info( u'Send to queue "%(path)s %(config)s %(task)s".', { 'path': botsenginepath, 'config': '-c' + configdir, 'task': task }) job2queue.send_job_to_jobqueue([ sys.executable, botsenginepath, '-c' + configdir, task ]) except Exception as msg: logger.info(u'Error in running task: "%(msg)s".', {'msg': msg}) tasks.clear() active_receiving = False else: #cond.wait returned probably because of a timeout logger.debug(u'time difference to small.') last_time = current_time cond.release() sys.exit(0)
def start(): #NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start. #********command line arguments************************** usage = ''' This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net). Checks a Bots grammar. Same checks are used as in translations with bots-engine. Searches for grammar in regular place: bots/usersys/grammars/<editype>/<messagetype>.py (even if a path is passed). Usage: %(name)s -c<directory> <editype> <messagetype> or %(name)s -c<directory> <path to grammar> Options: -c<directory> directory for configuration files (default: config). Examples: %(name)s -cconfig edifact ORDERSD96AUNEAN008 %(name)s -cconfig C:/python27/lib/site-packages/bots/usersys/grammars/edifact/ORDERSD96AUNEAN008.py ''' % { 'name': os.path.basename(sys.argv[0]), 'version': botsglobal.version } configdir = 'config' editype = '' messagetype = '' for arg in sys.argv[1:]: if arg.startswith('-c'): configdir = arg[2:] if not configdir: print 'Error: configuration directory indicated, but no directory name.' sys.exit(1) elif arg in ["?", "/?", '-h', '--help'] or arg.startswith('-'): print usage sys.exit(0) else: if os.path.isfile(arg): p1, p2 = os.path.split(arg) editype = os.path.basename(p1) messagetype, ext = os.path.splitext(p2) messagetype = unicode(messagetype) print 'grammarcheck', editype, messagetype elif not editype: editype = arg else: messagetype = arg if not (editype and messagetype): print 'Error: both editype and messagetype, or a file path, are required.' sys.exit(1) #***end handling command line arguments************************** botsinit.generalinit( configdir) #find locating of bots, configfiles, init paths etc. process_name = 'grammarcheck' botsglobal.logger = botsinit.initenginelogging(process_name) atexit.register(logging.shutdown) try: grammar.grammarread(editype, messagetype) except: print 'Found error in grammar: ', botslib.txtexc() sys.exit(1) else: print 'OK - no error found in grammar' sys.exit(0)
def start(): ''' sysexit codes: 0: OK, no errors 1: (system) errors incl parsing of command line arguments 2: bots ran OK, but there are errors/process errors in the run 3: Database is locked, but "maxruntime" has not been exceeded. ''' #NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start. #********command line arguments************************** usage = ''' This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net). Does the actual translations and communications; it's the workhorse. It does not have a fancy interface. Usage: %(name)s [config-option] Config-option: -c<directory> directory for configuration files (default: config). '''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version} configdir = 'config' for arg in sys.argv[1:]: if arg.startswith('-c'): configdir = arg[2:] if not configdir: print 'Error: configuration directory indicated, but no directory name.' sys.exit(1) elif arg in ["?", "/?",'-h', '--help'] or arg.startswith('-'): print usage sys.exit(0) #***********end handling command line arguments************************** botsinit.generalinit(configdir) #find locating of bots, configfiles, init paths etc. #set working directory to bots installation. advantage: when using relative paths it is clear that this point paths within bots installation. os.chdir(botsglobal.ini.get('directories','botspath')) #**************initialise logging****************************** process_name = 'engine2' botsglobal.logger = botsinit.initenginelogging(process_name) atexit.register(logging.shutdown) for key,value in botslib.botsinfo(): #log info about environement, versions, etc botsglobal.logger.info(u'%(key)s: "%(value)s".',{'key':key,'value':value}) #**************connect to database********************************** try: botsinit.connect() except Exception as msg: botsglobal.logger.exception(_(u'Could not connect to database. Database settings are in bots/config/settings.py. Error: "%(msg)s".'),{'msg':msg}) sys.exit(1) else: botsglobal.logger.info(_(u'Connected to database.')) atexit.register(botsglobal.db.close) warnings.simplefilter('error', UnicodeWarning) #import global scripts for bots-engine try: userscript,scriptname = botslib.botsimport('routescripts','botsengine') except botslib.BotsImportError: #userscript is not there; other errors like syntax errors are not catched userscript = scriptname = None #***acceptance tests: initialiase acceptance user script****************************** acceptance_userscript = acceptance_scriptname = None if botsglobal.ini.getboolean('acceptance','runacceptancetest',False): botsglobal.logger.info(_(u'This run is an acceptance test - as indicated in option "runacceptancetest" in bots.ini.')) try: acceptance_userscript,acceptance_scriptname = botslib.botsimport('routescripts','bots_acceptancetest') except botslib.BotsImportError: botsglobal.logger.info(_(u'In acceptance test there is no script file "bots_acceptancetest.py" to check the results of the acceptance test.')) try: #~ botslib.prepare_confirmrules() #in acceptance tests: run a user script before running eg to clean output directories****************************** botslib.tryrunscript(acceptance_userscript,acceptance_scriptname,'pretest') botslib.tryrunscript(userscript,scriptname,'pre') errorinrun = engine2_run() except Exception as msg: botsglobal.logger.exception(_(u'Severe error in bots system:\n%(msg)s'),{'msg':unicode(msg)}) #of course this 'should' not happen. sys.exit(1) else: if errorinrun: sys.exit(2) #indicate: error(s) in run(s) else: sys.exit(0) #OK
def start(): #********command line arguments************************** usage = ''' This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net). Updates existing bots database to version %(version)s Usage: %(name)s [config-option] Options: -c<directory> directory for configuration files (default: config). ''' % { 'name': os.path.basename(sys.argv[0]), 'version': botsglobal.version } configdir = 'config' for arg in sys.argv[1:]: if arg.startswith('-c'): configdir = arg[2:] if not configdir: print 'Error: configuration directory indicated, but no directory name.' sys.exit(3) else: #pick up names of routes to run print usage sys.exit(0) #***end handling command line arguments************************** botsinit.generalinit( configdir) #find locating of bots, configfiles, init paths etc. #**************check if another instance of bots-engine is running/if port is free****************************** try: engine_socket = botslib.check_if_other_engine_is_running() except socket.error: sys.exit(3) else: atexit.register(engine_socket.close) #**************initialise logging****************************** process_name = 'updatedatabase' botsglobal.logger = botsinit.initenginelogging(process_name) atexit.register(logging.shutdown) for key, value in botslib.botsinfo( ): #log info about environement, versions, etc botsglobal.logger.info(u'%(key)s: "%(value)s".', { 'key': key, 'value': value }) #**************connect to database********************************** try: botsinit.connect() except Exception as msg: botsglobal.logger.exception( _(u'Could not connect to database. Database settings are in bots/config/settings.py. Error: "%(msg)s".' ), {'msg': msg}) sys.exit(3) else: botsglobal.logger.info(_(u'Connected to database.')) atexit.register(botsglobal.db.close) #**************handle database lock**************************************** #set a lock on the database; if not possible, the database is locked: an earlier instance of bots-engine was terminated unexpectedly. if not botslib.set_database_lock(): warn = _(u'!Bots database is locked!\n'\ 'Bots-engine has ended in an unexpected way during the last run.\n'\ 'Most likely causes: sudden power-down, system crash, problems with disk I/O, bots-engine terminated by user, etc.') botsglobal.logger.critical(warn) sys.exit(3) atexit.register(botslib.remove_database_lock) if botsglobal.settings.DATABASES['default'][ 'ENGINE'] == 'django.db.backends.sqlite3': terug = sqlite3() elif botsglobal.settings.DATABASES['default'][ 'ENGINE'] == 'django.db.backends.mysql': terug = mysql() elif botsglobal.settings.DATABASES['default'][ 'ENGINE'] == 'django.db.backends.postgresql_psycopg2': terug = postgresql_psycopg2() sys.exit(terug)
def start(): #NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start. #***command line arguments************************** usage = ''' This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net). The %(name)s is the web server for bots; the interface (bots-monitor) can be accessed in a browser, eg 'http://localhost:8080'. Usage: %(name)s -c<directory> Options: -c<directory> directory for configuration files (default: config). ''' % { 'name': os.path.basename(sys.argv[0]), 'version': botsglobal.version } configdir = 'config' for arg in sys.argv[1:]: if arg.startswith('-c'): configdir = arg[2:] if not configdir: print 'Error: configuration directory indicated, but no directory name.' sys.exit(1) else: print usage sys.exit(0) #***end handling command line arguments************************** botsinit.generalinit( configdir) #find locating of bots, configfiles, init paths etc. process_name = 'webserver' botsglobal.logger = botsinit.initserverlogging( process_name ) #initialise file-logging for web-server. This logging only contains the logging from bots-webserver, not from cherrypy. #***init cherrypy as webserver********************************************* #global configuration for cherrypy cherrypy.config.update({ 'global': { 'log.screen': False, 'server.environment': botsglobal.ini.get('webserver', 'environment', 'production') } }) #cherrypy handling of static files conf = { '/': { 'tools.staticdir.on': True, 'tools.staticdir.dir': 'media', 'tools.staticdir.root': botsglobal.ini.get('directories', 'botspath') } } servestaticfiles = cherrypy.tree.mount( None, '/media', conf ) #None: no cherrypy application (as this only serves static files) #cherrypy handling of django servedjango = WSGIHandler( ) #was: servedjango = AdminMediaHandler(WSGIHandler()) but django does not need the AdminMediaHandler in this setup. is much faster. #cherrypy uses a dispatcher in order to handle the serving of static files and django. dispatcher = wsgiserver.WSGIPathInfoDispatcher({ '/': servedjango, '/media': servestaticfiles }) botswebserver = wsgiserver.CherryPyWSGIServer( bind_addr=('0.0.0.0', botsglobal.ini.getint('webserver', 'port', 8080)), wsgi_app=dispatcher, server_name=botsglobal.ini.get('webserver', 'name', 'bots-webserver')) botsglobal.logger.log(25, _(u'Bots %(process_name)s started.'), {'process_name': process_name}) botsglobal.logger.log( 25, _(u'Bots %(process_name)s configdir: "%(configdir)s".'), { 'process_name': process_name, 'configdir': botsglobal.ini.get('directories', 'config') }) botsglobal.logger.log( 25, _(u'Bots %(process_name)s serving at port: "%(port)s".'), { 'process_name': process_name, 'port': botsglobal.ini.getint('webserver', 'port', 8080) }) #handle ssl: cherrypy < 3.2 always uses pyOpenssl. cherrypy >= 3.2 uses python buildin ssl (python >= 2.6 has buildin support for ssl). ssl_certificate = botsglobal.ini.get('webserver', 'ssl_certificate', None) ssl_private_key = botsglobal.ini.get('webserver', 'ssl_private_key', None) if ssl_certificate and ssl_private_key: if cherrypy.__version__ >= '3.2.0': adapter_class = wsgiserver.get_ssl_adapter_class('builtin') botswebserver.ssl_adapter = adapter_class(ssl_certificate, ssl_private_key) else: #but: pyOpenssl should be there! botswebserver.ssl_certificate = ssl_certificate botswebserver.ssl_private_key = ssl_private_key botsglobal.logger.log(25, _(u'Bots %(process_name)s uses ssl (https).'), {'process_name': process_name}) else: botsglobal.logger.log( 25, _(u'Bots %(process_name)s uses plain http (no ssl).'), {'process_name': process_name}) #***start the cherrypy webserver.************************************************ try: botswebserver.start() except KeyboardInterrupt: botswebserver.stop()
def start(): ''' sysexit codes: 0: OK, no errors 1: (system) errors incl parsing of command line arguments 2: bots ran OK, but there are errors/process errors in the run 3: Database is locked, but "maxruntime" has not been exceeded. ''' #NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start. #********command line arguments************************** usage = ''' This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net). Does the actual translations and communications; it's the workhorse. It does not have a fancy interface. Usage: %(name)s [config-option] Config-option: -c<directory> directory for configuration files (default: config). ''' % { 'name': os.path.basename(sys.argv[0]), 'version': botsglobal.version } configdir = 'config' for arg in sys.argv[1:]: if arg.startswith('-c'): configdir = arg[2:] if not configdir: print 'Error: configuration directory indicated, but no directory name.' sys.exit(1) elif arg in ["?", "/?", '-h', '--help'] or arg.startswith('-'): print usage sys.exit(0) #***********end handling command line arguments************************** botsinit.generalinit( configdir) #find locating of bots, configfiles, init paths etc. #set working directory to bots installation. advantage: when using relative paths it is clear that this point paths within bots installation. os.chdir(botsglobal.ini.get('directories', 'botspath')) #**************initialise logging****************************** process_name = 'engine2' botsglobal.logger = botsinit.initenginelogging(process_name) atexit.register(logging.shutdown) for key, value in botslib.botsinfo( ): #log info about environement, versions, etc botsglobal.logger.info(u'%(key)s: "%(value)s".', { 'key': key, 'value': value }) #**************connect to database********************************** try: botsinit.connect() except Exception as msg: botsglobal.logger.exception( _(u'Could not connect to database. Database settings are in bots/config/settings.py. Error: "%(msg)s".' ), {'msg': msg}) sys.exit(1) else: botsglobal.logger.info(_(u'Connected to database.')) atexit.register(botsglobal.db.close) warnings.simplefilter('error', UnicodeWarning) #import global scripts for bots-engine try: userscript, scriptname = botslib.botsimport('routescripts', 'botsengine') except botslib.BotsImportError: #userscript is not there; other errors like syntax errors are not catched userscript = scriptname = None #***acceptance tests: initialiase acceptance user script****************************** acceptance_userscript = acceptance_scriptname = None if botsglobal.ini.getboolean('acceptance', 'runacceptancetest', False): botsglobal.logger.info( _(u'This run is an acceptance test - as indicated in option "runacceptancetest" in bots.ini.' )) try: acceptance_userscript, acceptance_scriptname = botslib.botsimport( 'routescripts', 'bots_acceptancetest') except botslib.BotsImportError: botsglobal.logger.info( _(u'In acceptance test there is no script file "bots_acceptancetest.py" to check the results of the acceptance test.' )) try: #~ botslib.prepare_confirmrules() #in acceptance tests: run a user script before running eg to clean output directories****************************** botslib.tryrunscript(acceptance_userscript, acceptance_scriptname, 'pretest') botslib.tryrunscript(userscript, scriptname, 'pre') errorinrun = engine2_run() except Exception as msg: botsglobal.logger.exception( _(u'Severe error in bots system:\n%(msg)s'), {'msg': unicode(msg)}) #of course this 'should' not happen. sys.exit(1) else: if errorinrun: sys.exit(2) #indicate: error(s) in run(s) else: sys.exit(0) #OK
def start(): ''' sysexit codes: 0: OK, no errors 1: (system) errors incl parsing of command line arguments 2: bots ran OK, but there are errors/process errors in the run 3: Database is locked, but "maxruntime" has not been exceeded. ''' #NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start. #********command line arguments************************** usage = ''' This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net). Does the actual translations and communications; it's the workhorse. It does not have a fancy interface. Usage: %(name)s [run-options] [config-option] [routes] Run-options (can be combined): --new receive new edi files (default: if no run-option given: run as new). --resend resend as indicated by user. --rereceive rereceive as indicated by user. --automaticretrycommunication - automatically retry outgoing communication. --cleanup remove older data from database. Config-option: -c<directory> directory for configuration files (default: config). Routes: list of routes to run. Default: all active routes (in the database) '''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version} configdir = 'config' commandspossible = ['--automaticretrycommunication','--resend','--rereceive','--new'] commandstorun = [] routestorun = [] #list with routes to run do_cleanup_parameter = False for arg in sys.argv[1:]: if arg.startswith('-c'): configdir = arg[2:] if not configdir: print 'Error: configuration directory indicated, but no directory name.' sys.exit(1) elif arg in commandspossible: commandstorun.append(arg) elif arg == '--cleanup': do_cleanup_parameter = True elif arg in ["?", "/?",'-h', '--help'] or arg.startswith('-'): print usage sys.exit(0) else: #pick up names of routes to run routestorun.append(arg) if not commandstorun and not do_cleanup_parameter: #if no command on command line, use new (default) commandstorun = ['--new'] commandstorun = [command[2:] for command in commandspossible if command in commandstorun] #sort commands #***end handling command line arguments************************** botsinit.generalinit(configdir) #find locating of bots, configfiles, init paths etc. #set working directory to bots installation. #possible advantage: when using relative paths it is clear that this point paths within bots installation. #most of time not needed: #1. in production: do not use relative paths within bots directory #2. in eg incoming messages path name is used via botslib.join, which makes an absulute path... #use this as bots2.* always ad this; avoid breaking. os.chdir(botsglobal.ini.get('directories','botspath')) #**************check if another instance of bots-engine is running/if port is free****************************** try: engine_socket = botslib.check_if_other_engine_is_running() except socket.error: sys.exit(3) else: atexit.register(engine_socket.close) #**************initialise logging****************************** process_name = 'engine' botsglobal.logger = botsinit.initenginelogging(process_name) atexit.register(logging.shutdown) for key,value in botslib.botsinfo(): #log info about environement, versions, etc botsglobal.logger.info(u'%(key)s: "%(value)s".',{'key':key,'value':value}) #**************connect to database********************************** try: botsinit.connect() except Exception,msg: botsglobal.logger.exception(_(u'Could not connect to database. Database settings are in bots/config/settings.py. Error: "%(msg)s".'),{'msg':msg}) sys.exit(1)
def start(): ''' sysexit codes: 0: OK, no errors 1: (system) errors incl parsing of command line arguments 2: bots ran OK, but there are errors/process errors in the run 3: Database is locked, but "maxruntime" has not been exceeded. ''' #NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start. #********command line arguments************************** usage = ''' This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net). Does the actual translations and communications; it's the workhorse. It does not have a fancy interface. Usage: %(name)s [run-options] [config-option] [routes] Run-options (can be combined): --new receive new edi files (default: if no run-option given: run as new). --resend resend as indicated by user. --rereceive rereceive as indicated by user. --automaticretrycommunication - automatically retry outgoing communication. --cleanup remove older data from database. Config-option: -c<directory> directory for configuration files (default: config). Routes: list of routes to run. Default: all active routes (in the database) ''' % { 'name': os.path.basename(sys.argv[0]), 'version': botsglobal.version } configdir = 'config' commandspossible = [ '--automaticretrycommunication', '--resend', '--rereceive', '--new' ] commandstorun = [] routestorun = [] #list with routes to run do_cleanup_parameter = False for arg in sys.argv[1:]: if arg.startswith('-c'): configdir = arg[2:] if not configdir: print 'Error: configuration directory indicated, but no directory name.' sys.exit(1) elif arg in commandspossible: commandstorun.append(arg) elif arg == '--cleanup': do_cleanup_parameter = True elif arg in ["?", "/?", '-h', '--help'] or arg.startswith('-'): print usage sys.exit(0) else: #pick up names of routes to run routestorun.append(arg) if not commandstorun and not do_cleanup_parameter: #if no command on command line, use new (default) commandstorun = ['--new'] commandstorun = [ command[2:] for command in commandspossible if command in commandstorun ] #sort commands #***********end handling command line arguments************************** botsinit.generalinit( configdir) #find locating of bots, configfiles, init paths etc. #set working directory to bots installation. advantage: when using relative paths it is clear that this point paths within bots installation. os.chdir(botsglobal.ini.get('directories', 'botspath')) #**************check if another instance of bots-engine is running/if port is free****************************** try: engine_socket = botslib.check_if_other_engine_is_running() except socket.error: sys.exit(3) else: atexit.register(engine_socket.close) #**************initialise logging****************************** process_name = 'engine' botsglobal.logger = botsinit.initenginelogging(process_name) atexit.register(logging.shutdown) for key, value in botslib.botsinfo( ): #log info about environement, versions, etc botsglobal.logger.info(u'%(key)s: "%(value)s".', { 'key': key, 'value': value }) #**************connect to database********************************** try: botsinit.connect() except Exception as msg: botsglobal.logger.exception( _(u'Could not connect to database. Database settings are in bots/config/settings.py. Error: "%(msg)s".' ), {'msg': msg}) sys.exit(1) else: botsglobal.logger.info(_(u'Connected to database.')) atexit.register(botsglobal.db.close) #************initialise user exits for the whole bots-engine************************* try: userscript, scriptname = botslib.botsimport('routescripts', 'botsengine') except botslib.BotsImportError: #userscript is not there; other errors like syntax errors are not catched userscript = scriptname = None #***acceptance tests: initialiase acceptance user script****************************** acceptance_userscript = acceptance_scriptname = None if botsglobal.ini.getboolean('acceptance', 'runacceptancetest', False): botsglobal.logger.info( _(u'This run is an acceptance test - as indicated in option "runacceptancetest" in bots.ini.' )) try: acceptance_userscript, acceptance_scriptname = botslib.botsimport( 'routescripts', 'bots_acceptancetest') except botslib.BotsImportError: botsglobal.logger.info( _(u'In acceptance test there is no script file "bots_acceptancetest.py" to check the results of the acceptance test.' )) #**************handle database lock**************************************** #set a lock on the database; if not possible, the database is locked: an earlier instance of bots-engine was terminated unexpectedly. if not botslib.set_database_lock(): #for SQLite: do a integrity check on the database if botsglobal.settings.DATABASES['default'][ 'ENGINE'] == 'django.db.backends.sqlite3': cursor = botsglobal.db.execute('''PRAGMA integrity_check''') result = cursor.fetchone() if result[0] != u'ok': warn = _(u'!Bots database is locked!\n'\ 'Bots did an integrity check on the database, but database was not OK.\n'\ 'Manual action is needed!\n'\ 'Bots has stopped processing EDI files.') botsglobal.logger.critical(warn) botslib.sendbotserrorreport( _(u'[Bots severe error]Database is damaged'), warn) sys.exit(1) warn = _(u'!Bots database is locked!\n'\ 'Bots-engine has ended in an unexpected way during the last run.\n'\ 'Most likely causes: sudden power-down, system crash, problems with disk I/O, bots-engine terminated by user, etc.\n' 'Bots will do an automatic crash recovery now.') botsglobal.logger.critical(warn) botslib.sendbotserrorreport( _(u'[Bots severe error]Database is locked'), warn) commandstorun.insert( 0, 'crashrecovery' ) #there is a database lock. Add a crashrecovery as first command to run. atexit.register(botslib.remove_database_lock) warnings.simplefilter('error', UnicodeWarning) #**************run the routes********************************************** #commandstorun determines the type(s) of run. eg: ['automaticretrycommunication','new'] try: botslib.prepare_confirmrules() #in acceptance tests: run a user script before running eg to clean output directories****************************** botslib.tryrunscript(acceptance_userscript, acceptance_scriptname, 'pretest', routestorun=routestorun) botslib.tryrunscript(userscript, scriptname, 'pre', commandstorun=commandstorun, routestorun=routestorun) errorinrun = 0 #detect if there has been some error. Only used for correct exit() code first_command_2_run = True for command in commandstorun: #if multiple commands in run: reports etc are based on timestamp; so there needs to be at least one second between these runs. if first_command_2_run: first_command_2_run = False else: time.sleep(1) botsglobal.logger.info(_(u'Run "%(command)s".'), {'command': command}) #************get list of routes to run******************************* if routestorun: use_routestorun = routestorun[:] botsglobal.logger.info( _(u'Run routes from command line: "%(routes)s".'), {'routes': unicode(use_routestorun)}) elif command == 'new': #fetch all active routes from database unless 'not in default run' or not active. use_routestorun = [] for row in botslib.query( '''SELECT DISTINCT idroute FROM routes WHERE active=%(active)s AND (notindefaultrun=%(notindefaultrun)s OR notindefaultrun IS NULL) ORDER BY idroute ''', { 'active': True, 'notindefaultrun': False }): use_routestorun.append(row['idroute']) botsglobal.logger.info( _(u'Run active routes from database that are in default run: "%(routes)s".' ), {'routes': unicode(use_routestorun)}) else: #for command other than 'new': use all active routes. use_routestorun = [] for row in botslib.query( '''SELECT DISTINCT idroute FROM routes WHERE active=%(active)s ORDER BY idroute ''', {'active': True}): use_routestorun.append(row['idroute']) botsglobal.logger.info( _(u'Run all active routes from database: "%(routes)s".'), {'routes': unicode(use_routestorun)}) #************run routes for this command****************************** botslib.tryrunscript(userscript, scriptname, 'pre' + command, routestorun=use_routestorun) errorinrun += router.rundispatcher(command, use_routestorun) botslib.tryrunscript(userscript, scriptname, 'post' + command, routestorun=use_routestorun) #*********finished running routes for this command**************************** #*********finished all commands**************************************** botslib.tryrunscript(userscript, scriptname, 'post', commandstorun=commandstorun, routestorun=routestorun) try: #in acceptance tests: run a user script. no good reporting of errors/results in post-test script. Reason: this is after automaticmaintence. botslib.tryrunscript(acceptance_userscript, acceptance_scriptname, 'posttest', routestorun=use_routestorun) except Exception as msg: print unicode(msg) cleanup.cleanup(do_cleanup_parameter, userscript, scriptname) except Exception as msg: botsglobal.logger.exception( _(u'Severe error in bots system:\n%(msg)s'), {'msg': unicode(msg)}) #of course this 'should' not happen. sys.exit(1) else: if errorinrun: sys.exit(2) #indicate: error(s) in run(s) else: sys.exit(0) #OK
def start(): #NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start. #***command line arguments************************** usage = ''' This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net). The %(name)s is the web server for bots; the interface (bots-monitor) can be accessed in a browser, eg 'http://localhost:8080'. Usage: %(name)s -c<directory> Options: -c<directory> directory for configuration files (default: config). '''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version} configdir = 'config' for arg in sys.argv[1:]: if arg.startswith('-c'): configdir = arg[2:] if not configdir: print 'Error: configuration directory indicated, but no directory name.' sys.exit(1) else: print usage sys.exit(0) #***end handling command line arguments************************** botsinit.generalinit(configdir) #find locating of bots, configfiles, init paths etc. process_name = 'webserver' botsglobal.logger = botsinit.initserverlogging(process_name) #initialise file-logging for web-server. This logging only contains the logging from bots-webserver, not from cherrypy. #***init cherrypy as webserver********************************************* #global configuration for cherrypy cherrypy.config.update({'global': {'log.screen': False, 'server.environment': botsglobal.ini.get('webserver','environment','production')}}) #cherrypy handling of static files conf = {'/': {'tools.staticdir.on' : True,'tools.staticdir.dir' : 'media' ,'tools.staticdir.root': botsglobal.ini.get('directories','botspath')}} servestaticfiles = cherrypy.tree.mount(None, '/media', conf) #None: no cherrypy application (as this only serves static files) #cherrypy handling of django servedjango = WSGIHandler() #was: servedjango = AdminMediaHandler(WSGIHandler()) but django does not need the AdminMediaHandler in this setup. is much faster. #cherrypy uses a dispatcher in order to handle the serving of static files and django. dispatcher = wsgiserver.WSGIPathInfoDispatcher({'/': servedjango, '/media': servestaticfiles}) botswebserver = wsgiserver.CherryPyWSGIServer(bind_addr=('0.0.0.0', botsglobal.ini.getint('webserver','port',8080)), wsgi_app=dispatcher, server_name=botsglobal.ini.get('webserver','name','bots-webserver')) botsglobal.logger.log(25,_(u'Bots %(process_name)s started.'), {'process_name':process_name}) botsglobal.logger.log(25,_(u'Bots %(process_name)s configdir: "%(configdir)s".'), {'process_name':process_name, 'configdir':botsglobal.ini.get('directories','config')}) botsglobal.logger.log(25,_(u'Bots %(process_name)s serving at port: "%(port)s".'), {'process_name':process_name,'port':botsglobal.ini.getint('webserver','port',8080)}) #handle ssl: cherrypy < 3.2 always uses pyOpenssl. cherrypy >= 3.2 uses python buildin ssl (python >= 2.6 has buildin support for ssl). ssl_certificate = botsglobal.ini.get('webserver','ssl_certificate',None) ssl_private_key = botsglobal.ini.get('webserver','ssl_private_key',None) if ssl_certificate and ssl_private_key: if cherrypy.__version__ >= '3.2.0': adapter_class = wsgiserver.get_ssl_adapter_class('builtin') botswebserver.ssl_adapter = adapter_class(ssl_certificate,ssl_private_key) else: #but: pyOpenssl should be there! botswebserver.ssl_certificate = ssl_certificate botswebserver.ssl_private_key = ssl_private_key botsglobal.logger.log(25,_(u'Bots %(process_name)s uses ssl (https).'),{'process_name':process_name}) else: botsglobal.logger.log(25,_(u'Bots %(process_name)s uses plain http (no ssl).'),{'process_name':process_name}) #***start the cherrypy webserver.************************************************ try: botswebserver.start() except KeyboardInterrupt: botswebserver.stop()