def start():
#********command line arguments**************************
editype = ''
messagetype = ''
configdir = 'config'
for arg in sys.argv[1:]:
if not arg:
continue
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print ' !!Indicated Bots should use specific .ini file but no file name was given.'
showusage()
elif arg in ["?", "/?"] or arg.startswith('-'):
showusage()
else:
if not editype:
editype = arg
else:
messagetype = arg
if not (editype and messagetype):
print ' !!Both editype and messagetype are required.'
showusage()
#********end handling command line arguments**************************
try:
botsinit.generalinit(configdir)
botsinit.initenginelogging()
grammar.grammarread(editype, messagetype)
except:
print 'Found error in grammar:'
print botslib.txtexc()
else:
print 'OK - no error found in grammar'
def start():
#NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
#***command line arguments**************************
usage = '''
This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
A utility to generate the index file of a plugin; this can be seen as a database dump of the configuration.
This is eg useful for version control.
Usage:
%(name)s -c<directory>
Options:
-c<directory> directory for configuration files (default: config).
'''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version}
configdir = 'config'
for arg in sys.argv[1:]:
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Error: configuration directory indicated, but no directory name.'
sys.exit(1)
else:
print usage
sys.exit(0)
#***end handling command line arguments**************************
botsinit.generalinit(configdir) #find locating of bots, configfiles, init paths etc.
import pluglib #import here, import at start of file gives error; first initialize.
usersys = botsglobal.ini.get('directories','usersysabs')
index_filename = os.path.join(usersys,'index.py')
dummy_for_cleaned_data = {'databaseconfiguration':True,'umlists':True,'databasetransactions':False}
pluglib.make_index(dummy_for_cleaned_data,index_filename)
def startmulti(grammardir, editype):
''' specialized tool for bulk checking of grammars while developing botsgrammars
grammardir: directory with gramars (eg bots/usersys/grammars/edifact)
editype: eg edifact
'''
configdir = 'config'
botsinit.generalinit(
configdir) #find locating of bots, configfiles, init paths etc.
process_name = 'grammarcheck'
botsglobal.logger = botsinit.initenginelogging(process_name)
atexit.register(logging.shutdown)
for filename in glob.iglob(grammardir):
filename_basename = os.path.basename(filename)
if filename_basename in ['__init__.py', 'envelope.py']:
continue
if filename_basename.startswith(
'edifact') or filename_basename.startswith(
'records') or filename_basename.endswith('records.py'):
continue
if filename_basename.endswith('pyc'):
continue
filename_noextension = os.path.splitext(filename_basename)[0]
try:
grammar.grammarread(editype, filename_noextension)
except:
print botslib.txtexc()
print '\n'
else:
print 'OK - no error found in grammar', filename, '\n'
def start():
#NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
#***command line arguments**************************
usage = '''
This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
A utility to generate the index file of a plugin; this can be seen as a database dump of the configuration.
This is eg useful for version control.
Usage:
%(name)s -c<directory>
Options:
-c<directory> directory for configuration files (default: config).
'''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version}
configdir = 'config'
for arg in sys.argv[1:]:
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Error: configuration directory indicated, but no directory name.'
sys.exit(1)
else:
print usage
sys.exit(0)
#***end handling command line arguments**************************
botsinit.generalinit(configdir) #find locating of bots, configfiles, init paths etc.
import pluglib #import here, import at start of file gives error; first initialize.
usersys = botsglobal.ini.get('directories','usersysabs')
index_filename = os.path.join(usersys,'index.py')
dummy_for_cleaned_data = {'databaseconfiguration':True,'umlists':botsglobal.ini.getboolean('settings','codelists_in_plugin',True),'databasetransactions':False}
pluglib.make_index(dummy_for_cleaned_data,index_filename)
def start():
#NOTE bots is always on PYTHONPATH!!! - otherwise it will not start.
#***command line arguments**************************
configdir = 'config'
for arg in sys.argv[1:]:
if not arg:
continue
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Configuration directory indicated, but no directory name.'
sys.exit(1)
elif arg in ["?", "/?"] or arg.startswith('-'):
showusage()
else:
showusage()
#***init general: find locating of bots, configfiles, init paths etc.***********************
botsinit.generalinit(configdir)
#***initialise logging. This logging only contains the logging from bots-webserver, not from cherrypy.
botsglobal.logger = logging.getLogger('bots-webserver')
botsglobal.logger.setLevel(logging.DEBUG)
h = TimedRotatingFileHandler(botslib.join(botsglobal.ini.get('directories','logging'),'webserver.log'), backupCount=10)
fileformat = logging.Formatter("%(asctime)s %(levelname)-8s: %(message)s",'%Y%m%d %H:%M:%S')
h.setFormatter(fileformat)
botsglobal.logger.addHandler(h)
#***init cherrypy as webserver*********************************************
#global configuration for cherrypy
cherrypy.config.update({'global': {'log.screen': False, 'server.environment': botsglobal.ini.get('webserver','environment','production')}})
#cherrypy handling of static files
conf = {'/': {'tools.staticdir.on' : True,'tools.staticdir.dir' : 'media' ,'tools.staticdir.root': botsglobal.ini.get('directories','botspath')}}
servestaticfiles = cherrypy.tree.mount(None, '/media', conf) #None: no cherrypy application (as this only serves static files)
#cherrypy handling of django
servedjango = WSGIHandler() #was: servedjango = AdminMediaHandler(WSGIHandler()) but django does not need the AdminMediaHandler in this setup. is much faster.
#cherrypy uses a dispatcher in order to handle the serving of static files and django.
dispatcher = wsgiserver.WSGIPathInfoDispatcher({'/': servedjango, '/media': servestaticfiles})
botswebserver = wsgiserver.CherryPyWSGIServer(bind_addr=('0.0.0.0', botsglobal.ini.getint('webserver','port',8080)), wsgi_app=dispatcher, server_name=botsglobal.ini.get('webserver','name','bots-webserver'))
botsglobal.logger.info(_(u'Bots web-server started.'))
#handle ssl: cherrypy < 3.2 always uses pyOpenssl. cherrypy >= 3.2 uses python buildin ssl (python >= 2.6 has buildin support for ssl).
ssl_certificate = botsglobal.ini.get('webserver','ssl_certificate',None)
ssl_private_key = botsglobal.ini.get('webserver','ssl_private_key',None)
if ssl_certificate and ssl_private_key:
if cherrypy.__version__ >= '3.2.0':
adapter_class = wsgiserver.get_ssl_adapter_class('builtin')
botswebserver.ssl_adapter = adapter_class(ssl_certificate,ssl_private_key)
else:
#but: pyOpenssl should be there!
botswebserver.ssl_certificate = ssl_certificate
botswebserver.ssl_private_key = ssl_private_key
botsglobal.logger.info(_(u'Bots web-server uses ssl (https).'))
else:
botsglobal.logger.info(_(u'Bots web-server uses plain http (no ssl).'))
#***start the cherrypy webserver.
try:
botswebserver.start()
except KeyboardInterrupt:
botswebserver.stop()
def start(configdir):
botsinit.generalinit(
configdir) #find locating of bots, configfiles, init paths etc.
process_name = 'apache_webserver_' + configdir
botsglobal.logger = botsinit.initserverlogging(
process_name
) #initialise file-logging for web-server. This logging only contains the logging from bots-webserver.
def startmulti(grammardir,editype):
''' specialized tool for bulk checking of grammars while developing botsgrammars
grammardir: directory with gramars (eg bots/usersys/grammars/edifact)
editype: eg edifact
'''
configdir = 'config'
botsinit.generalinit(configdir) #find locating of bots, configfiles, init paths etc.
process_name = 'grammarcheck'
botsglobal.logger = botsinit.initenginelogging(process_name)
atexit.register(logging.shutdown)
for filename in glob.iglob(grammardir):
filename_basename = os.path.basename(filename)
if filename_basename in ['__init__.py','envelope.py']:
continue
if filename_basename.startswith('edifact') or filename_basename.startswith('records') or filename_basename.endswith('records.py'):
continue
if filename_basename.endswith('pyc'):
continue
filename_noextension = os.path.splitext(filename_basename)[0]
try:
grammar.grammarread(editype,filename_noextension)
except:
print botslib.txtexc()
print '\n'
else:
print 'OK - no error found in grammar',filename,'\n'
def start():
#********command line arguments**************************
editype =''
messagetype = ''
configdir = 'config'
for arg in sys.argv[1:]:
if not arg:
continue
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print ' !!Indicated Bots should use specific .ini file but no file name was given.'
showusage()
elif arg in ["?", "/?"] or arg.startswith('-'):
showusage()
else:
if not editype:
editype = arg
else:
messagetype = arg
if not (editype and messagetype):
print ' !!Both editype and messagetype are required.'
showusage()
#********end handling command line arguments**************************
try:
botsinit.generalinit(configdir)
botsinit.initenginelogging()
grammar.grammarread(editype,messagetype)
except:
print 'Found error in grammar:'
print botslib.txtexc()
else:
print 'OK - no error found in grammar'
def start():
#NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
#********command line arguments**************************
usage = '''
This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
Checks a Bots grammar. Same checks are used as in translations with bots-engine. Searches for grammar in
regular place: bots/usersys/grammars/<editype>/<messagetype>.py (even if a path is passed).
Usage: %(name)s -c<directory> <editype> <messagetype>
or %(name)s -c<directory> <path to grammar>
Options:
-c<directory> directory for configuration files (default: config).
Examples:
%(name)s -cconfig edifact ORDERSD96AUNEAN008
%(name)s -cconfig C:/python27/lib/site-packages/bots/usersys/grammars/edifact/ORDERSD96AUNEAN008.py
'''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version}
configdir = 'config'
editype =''
messagetype = ''
for arg in sys.argv[1:]:
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Error: configuration directory indicated, but no directory name.'
sys.exit(1)
elif arg in ["?", "/?",'-h', '--help'] or arg.startswith('-'):
print usage
sys.exit(0)
else:
if os.path.isfile(arg):
p1,p2 = os.path.split(arg)
editype = os.path.basename(p1)
messagetype,ext = os.path.splitext(p2)
messagetype = str(messagetype)
print 'grammarcheck',editype,messagetype
elif not editype:
editype = arg
else:
messagetype = arg
if not (editype and messagetype):
print 'Error: both editype and messagetype, or a file path, are required.'
sys.exit(1)
#***end handling command line arguments**************************
botsinit.generalinit(configdir) #find locating of bots, configfiles, init paths etc.
process_name = 'grammarcheck'
botsglobal.logger = botsinit.initenginelogging(process_name)
atexit.register(logging.shutdown)
try:
grammar.grammarread(editype,messagetype)
except:
print 'Found error in grammar: ',botslib.txtexc()
sys.exit(1)
else:
print 'OK - no error found in grammar'
sys.exit(0)
def start():
#NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
#***command line arguments**************************
usage = '''
This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
Server program that ensures only a single bots-engine runs at any time, and no engine run requests are
lost/discarded. Each request goes to a queue and is run in sequence when the previous run completes.
Use of the job queue is optional and must be configured in bots.ini (jobqueue section, enabled = True).
Usage:
%(name)s -c<directory>
Options:
-c<directory> directory for configuration files (default: config).
'''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version}
configdir = 'config'
for arg in sys.argv[1:]:
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Error: configuration directory indicated, but no directory name.'
sys.exit(1)
else:
print usage
sys.exit(0)
#***end handling command line arguments**************************
botsinit.generalinit(configdir) #find locating of bots, configfiles, init paths etc.
if not botsglobal.ini.getboolean('jobqueue','enabled',False):
print 'Error: bots jobqueue cannot start; not enabled in %s/bots.ini' % configdir
sys.exit(1)
process_name = 'jobqueue'
logger = botsinit.initserverlogging(process_name)
logger.log(25,u'Bots %(process_name)s started.',{'process_name':process_name})
logger.log(25,u'Bots %(process_name)s configdir: "%(configdir)s".',{'process_name':process_name,'configdir':botsglobal.ini.get('directories','config')})
port = botsglobal.ini.getint('jobqueue','port',28082)
logger.log(25,u'Bots %(process_name)s listens for xmlrpc at port: "%(port)s".',{'process_name':process_name,'port':port})
#start launcher thread
lauchfrequency = botsglobal.ini.getint('jobqueue','lauchfrequency',5)
maxruntime = botsglobal.ini.getint('settings','maxruntime',60)
launcher_thread = threading.Thread(name='launcher', target=launcher, args=(logger,port,lauchfrequency,maxruntime))
launcher_thread.daemon = True
launcher_thread.start()
logger.info(u'Jobqueue launcher started.')
#the main thread is the xmlrpc server: all adding, getting etc for jobqueue is done via xmlrpc.
logger.info(u'Jobqueue server started.')
server = SimpleXMLRPCServer(('localhost', port),logRequests=False)
server.register_instance(Jobqueue(logger))
try:
server.serve_forever()
except KeyboardInterrupt:
pass
sys.exit(0)
def start():
# NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
# ***command line arguments**************************
# if config (-c option) is before job argument, will be used as config-dir of job2queue it self and not for job
# if config (-c option) is after job argument, use both as config-dir of job2queue and as -c option of job.
# if config (-c option) is before and after job argument, use only the after...could change that but seems not to be useful.
usage = """
This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
Places a job in the bots jobqueue. Bots jobqueue takes care of correct processing of jobs.
Usage:
%(name)s [-c<directory>] [-p<priority>] job [job-parameters]
Options:
-c<directory> directory for configuration files (default: config).
-p<priority> priority of job, 1-9 (default: 5, highest priority is 1).
Example of usage:
%(name)s bots-engine.py
%(name)s python2.7 /usr/local/bin/bots-engine.py
%(name)s -p1 python2.7 /usr/local/bin/bots-engine.py -cconfig2 myroute
""" % {
"name": os.path.basename(sys.argv[0]),
"version": botsglobal.version,
}
configdir = "config" # default value
priority = 5 # default value
task_args = []
for arg in sys.argv[1:]:
if arg.startswith("-c"):
configdir = arg[2:]
if not configdir:
print "Error: configuration directory indicated, but no directory name."
sys.exit(1)
if task_args:
task_args.append(arg)
elif arg.startswith("-p"):
try:
priority = int(arg[2:])
except:
print "Error: priority should be numeric (1=highest, 9=lowest)."
sys.exit(1)
elif arg in ["?", "/?", "-h", "--help"]:
print usage
sys.exit(0)
else:
task_args.append(arg)
# ***end handling command line arguments**************************
botsinit.generalinit(configdir) # needed to read config
if not botsglobal.ini.getboolean("jobqueue", "enabled", False):
print "Error: bots jobqueue cannot start; not enabled in %s/bots.ini" % configdir
sys.exit(1)
terug = send_job_to_jobqueue(task_args, priority)
print JOBQUEUEMESSAGE2TXT[terug]
sys.exit(terug)
def start():
#NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
#***command line arguments**************************
usage = '''
This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
Server program that ensures only a single bots-engine runs at any time, and no engine run requests are
lost/discarded. Each request goes to a queue and is run in sequence when the previous run completes.
Use of the job queue is optional and must be configured in bots.ini (jobqueue section, enabled = True).
Usage:
%(name)s -c<directory>
Options:
-c<directory> directory for configuration files (default: config).
'''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version}
configdir = 'config'
for arg in sys.argv[1:]:
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Error: configuration directory indicated, but no directory name.'
sys.exit(1)
else:
print usage
sys.exit(0)
#***end handling command line arguments**************************
botsinit.generalinit(configdir) #find locating of bots, configfiles, init paths etc.
if not botsglobal.ini.getboolean('jobqueue','enabled',False):
print 'Error: bots jobqueue cannot start; not enabled in %s/bots.ini' % configdir
sys.exit(1)
process_name = 'jobqueue'
logger = botsinit.initserverlogging(process_name)
logger.log(25,u'Bots %(process_name)s started.',process_name=process_name)
logger.log(25,u'Bots %(process_name)s configdir: "%(configdir)s".',process_name=process_name,configdir=botsglobal.ini.get('directories','config'))
port = botsglobal.ini.getint('jobqueue','port',28082)
logger.log(25,u'Bots %(process_name)s listens for xmlrpc at port: "%(port)s".',process_name=process_name,port=port)
#start launcher thread
lauchfrequency = botsglobal.ini.getint('jobqueue','lauchfrequency',5)
maxruntime = botsglobal.ini.getint('settings','maxruntime',60)
launcher_thread = threading.Thread(name='launcher', target=launcher, args=(logger,port,lauchfrequency,maxruntime))
launcher_thread.daemon = True
launcher_thread.start()
logger.info(u'Jobqueue launcher started.')
# this main thread is the jobqserver (the xmlrpc server for handling jobqueue)
logger.info(u'Jobqueue server started.')
server = SimpleXMLRPCServer(('localhost', port),logRequests=False)
server.register_instance(Jobqueue(logger))
try:
server.serve_forever()
except KeyboardInterrupt:
pass
sys.exit(0)
def start():
#NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
#***command line arguments**************************
#if config (-c option) is before job argument, will be used as config-dir of job2queue it self and not for job
#if config (-c option) is after job argument, use both as config-dir of job2queue and as -c option of job.
#if config (-c option) is before and after job argument, use only the after...could change that but seems not to be useful.
usage = '''
This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
Places a job in the bots jobqueue. Bots jobqueue takes care of correct processing of jobs.
Usage:
%(name)s [-c<directory>] [-p<priority>] job [job-parameters]
Options:
-c<directory> directory for configuration files (default: config).
-p<priority> priority of job, 1-9 (default: 5, highest priority is 1).
Example of usage:
%(name)s bots-engine.py
%(name)s python2.7 /usr/local/bin/bots-engine.py
%(name)s -p1 python2.7 /usr/local/bin/bots-engine.py -cconfig2 myroute
''' % {
'name': os.path.basename(sys.argv[0]),
'version': botsglobal.version
}
configdir = 'config' #default value
priority = 5 #default value
task_args = []
for arg in sys.argv[1:]:
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Error: configuration directory indicated, but no directory name.'
sys.exit(1)
if task_args:
task_args.append(arg)
elif arg.startswith('-p'):
try:
priority = int(arg[2:])
except:
print 'Error: priority should be numeric (1=highest, 9=lowest).'
sys.exit(1)
elif arg in ["?", "/?", '-h', '--help']:
print usage
sys.exit(0)
else:
task_args.append(arg)
#***end handling command line arguments**************************
botsinit.generalinit(configdir) #needed to read config
if not botsglobal.ini.getboolean('jobqueue', 'enabled', False):
print 'Error: bots jobqueue cannot start; not enabled in %s/bots.ini' % configdir
sys.exit(1)
terug = send_job_to_jobqueue(task_args, priority)
print JOBQUEUEMESSAGE2TXT[terug]
sys.exit(terug)
def start():
# NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
# ***command line arguments**************************
usage = '''
This is "%(name)s", developed by Jagged Peak Inc.
A utility to read a plugin. Enter a comma separated list of plugin files.
Usage:
%(name)s -c<config_directory> -d<plugin_directory> <pluginfile(s)>
Options:
-c<directory> directory for configuration files (default: config).
-d<directory> directory for plugin file(s) default: D:/BOTS/plugins
''' % {
'name': os.path.basename(sys.argv[0]),
'version': botsglobal.version
}
configdir = 'config'
plugindir = 'D:/BOTS/plugins'
pluginfiles = ''
# Handle cmd arguments
for arg in sys.argv[1:]:
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Error: configuration directory indicated, but no directory name.'
sys.exit(1)
if arg.startswith('-d'):
plugindir = arg[2:]
if not configdir:
print 'Error: plugin directory indicated, but no directory name.'
sys.exit(1)
else:
pluginfiles = arg
if not pluginfiles:
print 'Plugin file was not entered. See usage'
print usage
sys.exit(1)
# Init required vars and libs
botsinit.generalinit(
configdir) # find locating of bots, configfiles, init paths etc.
process_name = 'plug_read'
botsglobal.logger = botsinit.initserverlogging(process_name)
import pluglib # import here, import at start of file gives error; first initialize.
# Call read plugin for each file
pluginfiles_list = pluginfiles.split(',')
for pluginfile in pluginfiles_list:
plugin_file_path = os.path.join(plugindir, pluginfile)
pluglib.read_plugin(plugin_file_path)
def start():
#********command line arguments**************************
usage = '''
This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
Updates existing bots database to version %(version)s
Usage:
%(name)s [config-option]
Options:
-c<directory> directory for configuration files (default: config).
'''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version}
configdir = 'config'
for arg in sys.argv[1:]:
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Error: configuration directory indicated, but no directory name.'
sys.exit(3)
else: #pick up names of routes to run
print usage
sys.exit(0)
#***end handling command line arguments**************************
botsinit.generalinit(configdir) #find locating of bots, configfiles, init paths etc.
#**************check if another instance of bots-engine is running/if port is free******************************
try:
engine_socket = botslib.check_if_other_engine_is_running()
except socket.error:
sys.exit(3)
else:
atexit.register(engine_socket.close)
#**************initialise logging******************************
process_name = 'updatedatabase'
botsglobal.logger = botsinit.initenginelogging(process_name)
atexit.register(logging.shutdown)
for key,value in botslib.botsinfo(): #log info about environement, versions, etc
botsglobal.logger.info(u'%(key)s: "%(value)s".',{'key':key,'value':value})
#**************connect to database**********************************
try:
botsinit.connect()
except Exception,msg:
botsglobal.logger.exception(_(u'Could not connect to database. Database settings are in bots/config/settings.py. Error: "%(msg)s".'),{'msg':msg})
sys.exit(3)
def startmulti(grammardir,editype):
''' used in seperate tool for bulk checking of gramamrs while developing edifact->botsgramamrs '''
import glob
botsinit.generalinit('config')
botsinit.initenginelogging()
for filename in glob.glob(grammardir):
filename_basename = os.path.basename(filename)
filename_noextension = os.path.splitext(filename_basename)[0]
if filename_basename in ['__init__.py']:
continue
if filename_basename.startswith('edifact'):
continue
if filename_basename.startswith('records') or filename_basename.endswith('records.py'):
continue
try:
grammar.grammarread(editype,filename_noextension)
except:
#~ print 'Found error in grammar:',filename
print botslib.txtexc()
print '\n'
else:
print 'OK - no error found in grammar',filename,'\n'
def startmulti(grammardir, editype):
''' used in seperate tool for bulk checking of gramamrs while developing edifact->botsgramamrs '''
import glob
botsinit.generalinit('config')
botsinit.initenginelogging()
for filename in glob.glob(grammardir):
filename_basename = os.path.basename(filename)
filename_noextension = os.path.splitext(filename_basename)[0]
if filename_basename in ['__init__.py']:
continue
if filename_basename.startswith('edifact'):
continue
if filename_basename.startswith(
'records') or filename_basename.endswith('records.py'):
continue
try:
grammar.grammarread(editype, filename_noextension)
except:
#~ print 'Found error in grammar:',filename
print botslib.txtexc()
print '\n'
else:
print 'OK - no error found in grammar', filename, '\n'
def start():
#exit codes:
# 0: OK, no errors
# 1: (system) errors
# 2: bots ran OK, but there are errors/process errors in the run
# 3: Database is locked, but "maxruntime" has not been exceeded.
#********command line arguments**************************
commandspossible = [
'--new', '--retry', '--retransmit', '--cleanup', '--crashrecovery',
'--retrycommunication', '--automaticretrycommunication'
]
commandstorun = []
routestorun = [] #list with routes to run
configdir = 'config'
for arg in sys.argv[1:]:
if not arg:
continue
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Configuration directory indicated, but no directory name.'
sys.exit(1)
elif arg in commandspossible:
commandstorun.append(arg)
elif arg in ["?", "/?"] or arg.startswith('-'):
showusage()
sys.exit(0)
else: #pick up names of routes to run
routestorun.append(arg)
if not commandstorun: #if no command on command line, use new (default)
commandstorun = ['--new']
#**************init general: find locating of bots, configfiles, init paths etc.****************
botsinit.generalinit(configdir)
#set current working directory to botspath
#~ old_current_directory = os.getcwdu()
os.chdir(botsglobal.ini.get('directories', 'botspath'))
#**************initialise logging******************************
try:
botsinit.initenginelogging()
except:
print _('Error in initialising logging system.')
traceback.print_exc()
sys.exit(1)
else:
atexit.register(logging.shutdown)
for key, value in botslib.botsinfo(): #log start info
botsglobal.logger.info(u'%s: "%s".', key, value)
#**************connect to database**********************************
try:
botsinit.connect()
except:
botsglobal.logger.exception(
_(u'Could not connect to database. Database settings are in bots/config/settings.py.'
))
sys.exit(1)
else:
botsglobal.logger.info(_(u'Connected to database.'))
atexit.register(botsglobal.db.close)
#initialise user exits for the whole bots-engine (this script file)
try:
userscript, scriptname = botslib.botsimport('routescripts',
'botsengine')
except ImportError:
userscript = scriptname = None
#**************handle database lock****************************************
#try to set a lock on the database; if this is not possible, the database is already locked. Either:
#1 another instance bots bots-engine is (still) running
#2 or bots-engine had a severe crash.
#What to do?
#first: check ts of database lock. If below a certain value (set in bots.ini) we assume an other instance is running. Exit quietly - no errors, no logging.
# else: Warn user, give advise on what to do. gather data: nr files in, errors.
#next: warn with report & logging. advise a crashrecovery.
if not botslib.set_database_lock():
if '--crashrecovery' in commandstorun: #user starts recovery operation; the databaselock is ignored; the databaselock is unlocked when routes have run.
commandstorun = ['--crashrecovery'] #is an exclusive option!
else:
#when scheduling bots it is possible that the last run is still running. Check if maxruntime has passed:
vanaf = datetime.datetime.today() - datetime.timedelta(
minutes=botsglobal.ini.getint('settings', 'maxruntime', 60))
for row in botslib.query(
'''SELECT ts FROM mutex WHERE ts < %(vanaf)s ''',
{'vanaf': vanaf}):
warn = _(
u'!Bots database is locked!\nBots-engine has ended in an unexpected way during the last run.\nThis happens, but is very very rare.\nPossible causes: bots-engine terminated by user, system crash, power-down, etc.\nA forced retry of the last run is advised; bots will (try to) repair the last run.'
)
botsglobal.logger.critical(warn)
botslib.sendbotserrorreport(
_(u'[Bots severe error]Database is locked'), warn)
#add: count errors etc.
sys.exit(1)
else: #maxruntime has not passed. Exit silently, nothing reported
botsglobal.logger.info(
_(u'Database is locked, but "maxruntime" has not been exceeded.'
))
sys.exit(3)
else:
if '--crashrecovery' in commandstorun: #user starts recovery operation but there is no databaselock.
warn = _(
u'User started a forced retry of the last run.\nOnly use this when the database is locked.\nThe database was not locked (database is OK).\nSo Bots has done nothing now.'
)
botsglobal.logger.error(warn)
botslib.sendbotserrorreport(
_(u'[Bots Error Report] User started a forced retry of last run, but this was not needed'
), warn)
botslib.remove_database_lock()
sys.exit(1)
#*************get list of routes to run****************************************
#~ raise Exception('locked database') #for testing database lock: abort, database will be locked
if routestorun:
botsglobal.logger.info(u'Run routes from command line: "%s".',
str(routestorun))
else: # no routes from command line parameters: fetch all active routes from database
for row in botslib.query(
'''SELECT DISTINCT idroute
FROM routes
WHERE active=%(active)s
AND (notindefaultrun=%(notindefaultrun)s OR notindefaultrun IS NULL)
ORDER BY idroute ''', {
'active': True,
'notindefaultrun': False
}):
routestorun.append(row['idroute'])
botsglobal.logger.info(_(u'Run active routes from database: "%s".'),
str(routestorun))
#routestorun is now either a list with routes from commandline, or the list of active routes for the routes table in the db.
#**************run the routes for retry, retransmit and new runs*************************************
try:
#commandstorun determines the type(s) of run
#routes to run is a listof the routes that are runs (for each command to run
#botsglobal.incommunicate is used to control if there is communication in; only 'new' incommunicates.
#botsglobal.minta4query controls which ta's are queried by the routes.
#stuff2evaluate controls what is evaluated in automatic maintenance.
errorinrun = 0 #detect if there has been some error. Only used for good exit() code
botsglobal.incommunicate = False
if '--crashrecovery' in commandstorun:
botsglobal.logger.info(_(u'Run crash recovery.'))
stuff2evaluate = botslib.set_minta4query_crashrecovery()
if stuff2evaluate:
router.routedispatcher(routestorun)
errorinrun += automaticmaintenance.evaluate(
'--crashrecovery', stuff2evaluate)
else:
botsglobal.logger.info(
_(u'No retry of the last run - there was no last run.'))
if userscript and hasattr(userscript, 'postcrashrecovery'):
botslib.runscript(userscript,
scriptname,
'postcrashrecovery',
routestorun=routestorun)
if '--retrycommunication' in commandstorun:
botsglobal.logger.info(_(u'Run communication retry.'))
stuff2evaluate = router.routedispatcher(routestorun,
'--retrycommunication')
if stuff2evaluate:
errorinrun += automaticmaintenance.evaluate(
'--retrycommunication', stuff2evaluate)
else:
botsglobal.logger.info(
_(u'Run recommunicate: nothing to recommunicate.'))
if userscript and hasattr(userscript, 'postretrycommunication'):
botslib.runscript(userscript,
scriptname,
'postretrycommunication',
routestorun=routestorun)
if '--automaticretrycommunication' in commandstorun:
botsglobal.logger.info(_(u'Run automatic communication retry.'))
stuff2evaluate = router.routedispatcher(
routestorun, '--automaticretrycommunication')
if stuff2evaluate:
errorinrun += automaticmaintenance.evaluate(
'--automaticretrycommunication', stuff2evaluate)
else:
botsglobal.logger.info(
_(u'Run automatic recommunicate: nothing to recommunicate.'
))
if userscript and hasattr(userscript,
'postautomaticretrycommunication'):
botslib.runscript(userscript,
scriptname,
'postautomaticretrycommunication',
routestorun=routestorun)
if '--retry' in commandstorun:
botsglobal.logger.info(u'Run retry.')
stuff2evaluate = router.routedispatcher(routestorun, '--retry')
if stuff2evaluate:
errorinrun += automaticmaintenance.evaluate(
'--retry', stuff2evaluate)
else:
botsglobal.logger.info(_(u'Run retry: nothing to retry.'))
if userscript and hasattr(userscript, 'postretry'):
botslib.runscript(userscript,
scriptname,
'postretry',
routestorun=routestorun)
if '--retransmit' in commandstorun:
botsglobal.logger.info(u'Run retransmit.')
stuff2evaluate = router.routedispatcher(routestorun,
'--retransmit')
if stuff2evaluate:
errorinrun += automaticmaintenance.evaluate(
'--retransmit', stuff2evaluate)
else:
botsglobal.logger.info(
_(u'Run retransmit: nothing to retransmit.'))
if userscript and hasattr(userscript, 'postretransmit'):
botslib.runscript(userscript,
scriptname,
'postretransmit',
routestorun=routestorun)
if '--new' in commandstorun:
botsglobal.logger.info('Run new.')
botsglobal.incommunicate = True
botsglobal.minta4query = 0 #meaning: reset. the actual value is set later (in routedispatcher)
stuff2evaluate = router.routedispatcher(routestorun)
errorinrun += automaticmaintenance.evaluate(
'--new', stuff2evaluate)
if userscript and hasattr(userscript, 'postnewrun'):
botslib.runscript(userscript,
scriptname,
'postnewrun',
routestorun=routestorun)
if '--cleanup' in commandstorun or botsglobal.ini.get(
'settings', 'whencleanup', 'always') == 'always':
botsglobal.logger.debug(u'Do cleanup.')
cleanup.cleanup()
botslib.remove_database_lock()
except Exception, e:
botsglobal.logger.exception(
_(u'Severe error in bots system:\n%s') %
(e)) #of course this 'should' not happen.
sys.exit(1)
def start():
#NOTE bots is always on PYTHONPATH!!! - otherwise it will not start.
#********command line arguments**************************
configdir = 'config'
for arg in sys.argv[1:]:
if not arg:
continue
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Configuration directory indicated, but no directory name.'
sys.exit(1)
elif arg in ["?", "/?"] or arg.startswith('-'):
showusage()
else:
showusage()
#init general: find locating of bots, configfiles, init paths etc.***********************
botsinit.generalinit(configdir)
#init cherrypy; only needed for webserver. *********************************************
cherrypy.config.update({'global': { 'tools.staticdir.root': botsglobal.ini.get('directories','botspath'),
'server.socket_host' : "0.0.0.0", #to what IP addresses should be server. 0.0.0.0: all. See cherrypy docs
'server.socket_port': botsglobal.ini.getint('webserver','port',8080),
'server.environment': botsglobal.ini.get('webserver','environment','development'), # development production
'log.screen': False,
#~ 'log.error_file': '', #set later to rotating log file
#~ 'log.access_file': '', #set later to rotating log file
}})
conf = {'/': {'tools.staticdir.on' : True,'tools.staticdir.dir' : 'media' }}
#~ '/favicon.ico': {'tools.staticfile.on': True,'tools.staticfile.filename': '/home/hje/botsup-django/bots/media/images/favicon.ico'}}
cherrypy.tree.graft(AdminMediaHandler(WSGIHandler()), '/')
myroot = Root()
myappl = cherrypy.tree.mount(myroot, '/media', conf) #myappl is needed to set logging
botsglobal.logger = logging.getLogger('webserver')
botsglobal.logger.setLevel(logging.DEBUG)
h = logging.handlers.TimedRotatingFileHandler(botslib.join(botsglobal.ini.get('directories','logging'),'webserver.log'),when='midnight', backupCount=10)
fileformat = logging.Formatter("%(asctime)s %(levelname)-8s %(name)s : %(message)s",'%Y%m%d %H:%M:%S')
h.setFormatter(fileformat)
# add rotating file handler to main logger
botsglobal.logger.addHandler(h)
# Make RotatingFileHandler for the error log.
#~ h = logging.handlers.TimedRotatingFileHandler(botslib.join(botsglobal.ini.get('directories','logging'),'webserver.log'),when='midnight', backupCount=10)
#~ fileformat = logging.Formatter("%(asctime)s %(levelname)-8s %(name)s : %(message)s",'%Y%m%d %H:%M:%S')
#~ h.setLevel(logging.INFO)
#~ h.setFormatter(fileformat)
#~ myappl.log.error_log.addHandler(h)
# MakeRotatingFileHandler for the access log.
#~ h = logging.handlers.TimedRotatingFileHandler(os.path.normpath(os.path.join(botsglobal.ini.get('directories','botspath'), 'botssys/logging/webserver.log')),when='midnight', backupCount=10)
#~ h.setLevel(logging.DEBUG)
#~ myappl.log.access_log.addHandler(h)
#~ botsglobal.logger = myappl.log.access_log
#write start info to cherrypy log********************************************
botsglobal.logger.info(_(u'Bots web server started.'))
botsglobal.logger.info(_(u'Python version: "%s".'),sys.version)
botsglobal.logger.info(_(u'Django version: "%s".'),django.VERSION)
#start cherrypy *********************************************************************
cherrypy.engine.start()
cherrypy.engine.block()
def start():
#********command line arguments**************************
usage = '''
This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
Creates a grammar from an xml file.'
Usage:'
%(name)s -c<directory> <xml_file> <xml_grammar_file>
Options:
-c<directory> directory for configuration files (default: config).
-a all xml elements as records
<xml_file> name of the xml file to read
<xml_grammar_file> name of the grammar file to write
'''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version}
configdir = 'config'
edifile =''
botsgrammarfilename = ''
allrecords = False
for arg in sys.argv[1:]:
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Error: configuration directory indicated, but no directory name.'
sys.exit(1)
elif arg.startswith('-a'):
allrecords = True
elif arg in ["?", "/?",'-h', '--help'] or arg.startswith('-'):
print usage
sys.exit(0)
else:
if not edifile:
edifile = arg
else:
botsgrammarfilename = arg
if not edifile or not botsgrammarfilename:
print 'Error: both edifile and grammarfile are required.'
sys.exit(0)
#***end handling command line arguments**************************
botsinit.generalinit(configdir) #find locating of bots, configfiles, init paths etc.
process_name = 'xml2botsgrammar'
botsglobal.logger = botsinit.initenginelogging(process_name)
atexit.register(logging.shutdown)
targetNamespace = ''
#*******************************************************************
#***add classes for handling editype xml to inmessage
#*******************************************************************
if allrecords:
#~ editype = 'xmlforgrammar_allrecords'
inmessage.xmlforgrammar = xmlforgrammar_allrecords
else:
#~ editype = 'xmlforgrammar'
inmessage.xmlforgrammar = xmlforgrammar
#make inmessage object: read the xml file
inn = inmessage.parse_edi_file(editype='xmlforgrammar',messagetype='',filename=edifile)
inn.checkforerrorlist() #no exception if infile has been lexed and parsed OK else raises an error
#make outmessage object; nothing is 'filled' yet. In mapping tree is filled; nothing is written to file.
out = outmessage.outmessage_init(editype='xmlnocheck',messagetype='',filename='',divtext='',topartner='')
#***mapping: make 'normalised' out-tree suited for writing as a grammar***************************************************
mpath_root = [OrderedDict({'BOTSID':inn.root.record['BOTSID'],'BOTSIDnr':'1'})] #handle root
out.put(*mpath_root)
map_writefields(out,inn.root,mpath_root)
#walk tree; write results to out-tree
mpath_start = []
for node_instance,mpath in map_treewalker(inn.root,mpath_start):
mpath.append(OrderedDict({'BOTSID':node_instance.record['BOTSID']}))
if out.get(*mpath) is None: #if node does not exist: write it.
out.put(*mpath)
map_writefields(out,node_instance,mpath)
#***mapping is done
#***convert out-tree to grammar
structure = []
recorddefs = {}
tree2grammar(out.root,structure,recorddefs)
#***write grammar to file
grammar2file(botsgrammarfilename,structure,recorddefs,targetNamespace)
def start():
''' sysexit codes:
0: OK, no errors
1: (system) errors incl parsing of command line arguments
2: bots ran OK, but there are errors/process errors in the run
3: Database is locked, but "maxruntime" has not been exceeded.
'''
#NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
#********command line arguments**************************
usage = '''
This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
Does the actual translations and communications; it's the workhorse. It does not have a fancy interface.
Usage:
%(name)s [run-options] [config-option] [routes]
Run-options (can be combined):
--new receive new edi files (default: if no run-option given: run as new).
--resend resend as indicated by user.
--rereceive rereceive as indicated by user.
--automaticretrycommunication - automatically retry outgoing communication.
--cleanup remove older data from database.
Config-option:
-c<directory> directory for configuration files (default: config).
Routes: list of routes to run. Default: all active routes (in the database)
'''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version}
configdir = 'config'
commandspossible = ['--automaticretrycommunication','--resend','--rereceive','--new']
commandstorun = []
routestorun = [] #list with routes to run
do_cleanup_parameter = False
for arg in sys.argv[1:]:
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Error: configuration directory indicated, but no directory name.'
sys.exit(1)
elif arg in commandspossible:
commandstorun.append(arg)
elif arg == '--cleanup':
do_cleanup_parameter = True
elif arg in ["?", "/?",'-h', '--help'] or arg.startswith('-'):
print usage
sys.exit(0)
else: #pick up names of routes to run
routestorun.append(arg)
if not commandstorun and not do_cleanup_parameter: #if no command on command line, use new (default)
commandstorun = ['--new']
commandstorun = [command[2:] for command in commandspossible if command in commandstorun] #sort commands
#***********end handling command line arguments**************************
botsinit.generalinit(configdir) #find locating of bots, configfiles, init paths etc.
#set working directory to bots installation. advantage: when using relative paths it is clear that this point paths within bots installation.
os.chdir(botsglobal.ini.get('directories','botspath'))
#**************check if another instance of bots-engine is running/if port is free******************************
try:
engine_socket = botslib.check_if_other_engine_is_running()
except socket.error:
sys.exit(3)
else:
atexit.register(engine_socket.close)
#**************initialise logging******************************
process_name = 'engine'
botsglobal.logger = botsinit.initenginelogging(process_name)
atexit.register(logging.shutdown)
for key,value in botslib.botsinfo(): #log info about environement, versions, etc
botsglobal.logger.info(u'%(key)s: "%(value)s".',{'key':key,'value':value})
#**************connect to database**********************************
try:
botsinit.connect()
except Exception as msg:
botsglobal.logger.exception(_(u'Could not connect to database. Database settings are in bots/config/settings.py. Error: "%(msg)s".'),{'msg':msg})
sys.exit(1)
else:
botsglobal.logger.info(_(u'Connected to database.'))
atexit.register(botsglobal.db.close)
#************initialise user exits for the whole bots-engine*************************
try:
userscript,scriptname = botslib.botsimport('routescripts','botsengine')
except ImportError: #userscript is not there; other errors like syntax errors are not catched
userscript = scriptname = None
#***acceptance tests: initialiase acceptance user script******************************
acceptance_userscript = acceptance_scriptname = None
if botsglobal.ini.getboolean('acceptance','runacceptancetest',False):
botsglobal.logger.info(_(u'This run is an acceptance test - as indicated in option "runacceptancetest" in bots.ini.'))
try:
acceptance_userscript,acceptance_scriptname = botslib.botsimport('routescripts','bots_acceptancetest')
except ImportError:
botsglobal.logger.info(_(u'In acceptance test there is no script file "bots_acceptancetest.py" to check the results of the acceptance test.'))
#**************handle database lock****************************************
#set a lock on the database; if not possible, the database is locked: an earlier instance of bots-engine was terminated unexpectedly.
if not botslib.set_database_lock():
#for SQLite: do a integrity check on the database
if botsglobal.settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3':
cursor = botsglobal.db.execute('''PRAGMA integrity_check''')
result = cursor.fetchone()
if result[0] != u'ok':
warn = _(u'!Bots database is locked!\n'\
'Bots did an integrity check on the database, but database was not OK.\n'\
'Manual action is needed!\n'\
'Bots has stopped processing EDI files.')
botsglobal.logger.critical(warn)
botslib.sendbotserrorreport(_(u'[Bots severe error]Database is damaged'),warn)
sys.exit(1)
warn = _(u'!Bots database is locked!\n'\
'Bots-engine has ended in an unexpected way during the last run.\n'\
'Most likely causes: sudden power-down, system crash, problems with disk I/O, bots-engine terminated by user, etc.\n'
'Bots will do an automatic crash recovery now.')
botsglobal.logger.critical(warn)
botslib.sendbotserrorreport(_(u'[Bots severe error]Database is locked'),warn)
commandstorun.insert(0,'crashrecovery') #there is a database lock. Add a crashrecovery as first command to run.
atexit.register(botslib.remove_database_lock)
#**************run the routes**********************************************
#commandstorun determines the type(s) of run. eg: ['automaticretrycommunication','new']
try:
botslib.prepare_confirmrules()
#in acceptance tests: run a user script before running eg to clean output directories******************************
botslib.tryrunscript(acceptance_userscript,acceptance_scriptname,'pretest',routestorun=routestorun)
botslib.tryrunscript(userscript,scriptname,'pre',commandstorun=commandstorun,routestorun=routestorun)
errorinrun = 0 #detect if there has been some error. Only used for correct exit() code
first_command_2_run = True
for command in commandstorun:
#if multiple commands in run: reports etc are based on timestamp; so there needs to be at least one second between these runs.
if first_command_2_run:
first_command_2_run = False
else:
time.sleep(1)
botsglobal.logger.info(_(u'Run "%(command)s".'),{'command':command})
#************get list of routes to run*******************************
if routestorun:
use_routestorun = routestorun[:]
botsglobal.logger.info(_(u'Run routes from command line: "%(routes)s".'),{'routes':str(use_routestorun)})
elif command == 'new': #fetch all active routes from database unless 'not in default run' or not active.
use_routestorun = []
for row in botslib.query('''SELECT DISTINCT idroute
FROM routes
WHERE active=%(active)s
AND (notindefaultrun=%(notindefaultrun)s OR notindefaultrun IS NULL)
ORDER BY idroute ''',
{'active':True,'notindefaultrun':False}):
use_routestorun.append(row['idroute'])
botsglobal.logger.info(_(u'Run active routes from database that are in default run: "%(routes)s".'),{'routes':str(use_routestorun)})
else: #for command other than 'new': use all active routes.
use_routestorun = []
for row in botslib.query('''SELECT DISTINCT idroute
FROM routes
WHERE active=%(active)s
ORDER BY idroute ''',
{'active':True}):
use_routestorun.append(row['idroute'])
botsglobal.logger.info(_(u'Run all active routes from database: "%(routes)s".'),{'routes':str(use_routestorun)})
#************run routes for this command******************************
botslib.tryrunscript(userscript,scriptname,'pre' + command,routestorun=use_routestorun)
errorinrun += router.rundispatcher(command,use_routestorun)
botslib.tryrunscript(userscript,scriptname,'post' + command,routestorun=use_routestorun)
#*********finished running routes for this command****************************
#*********finished all commands****************************************
botslib.tryrunscript(userscript,scriptname,'post',commandstorun=commandstorun,routestorun=routestorun)
try: #in acceptance tests: run a user script. no good reporting of errors/results in post-test script. Reason: this is after automaticmaintence.
botslib.tryrunscript(acceptance_userscript,acceptance_scriptname,'posttest',routestorun=use_routestorun)
except Exception as msg:
print str(msg)
cleanup.cleanup(do_cleanup_parameter,userscript,scriptname)
except Exception as msg:
botsglobal.logger.exception(_(u'Severe error in bots system:\n%(msg)s'),{'msg':str(msg)}) #of course this 'should' not happen.
sys.exit(1)
else:
if errorinrun:
sys.exit(2) #indicate: error(s) in run(s)
else:
sys.exit(0) #OK
def start():
#exit codes:
# 0: OK, no errors
# 1: (system) errors
# 2: bots ran OK, but there are errors/process errors in the run
# 3: Database is locked, but "maxruntime" has not been exceeded.
#********command line arguments**************************
commandspossible = ['--new','--retry','--retransmit','--cleanup','--crashrecovery','--retrycommunication','--automaticretrycommunication']
commandstorun = []
routestorun = [] #list with routes to run
configdir = 'config'
for arg in sys.argv[1:]:
if not arg:
continue
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Configuration directory indicated, but no directory name.'
sys.exit(1)
elif arg in commandspossible:
commandstorun.append(arg)
elif arg in ["?", "/?"] or arg.startswith('-'):
showusage()
sys.exit(0)
else: #pick up names of routes to run
routestorun.append(arg)
if not commandstorun: #if no command on command line, use new (default)
commandstorun = ['--new']
#**************init general: find locating of bots, configfiles, init paths etc.****************
botsinit.generalinit(configdir)
#set current working directory to botspath
#~ old_current_directory = os.getcwdu()
os.chdir(botsglobal.ini.get('directories','botspath'))
#**************initialise logging******************************
try:
botsinit.initenginelogging()
except:
print _('Error in initialising logging system.')
traceback.print_exc()
sys.exit(1)
else:
atexit.register(logging.shutdown)
for key,value in botslib.botsinfo(): #log start info
botsglobal.logger.info(u'%s: "%s".',key,value)
#**************connect to database**********************************
try:
botsinit.connect()
except:
botsglobal.logger.exception(_(u'Could not connect to database. Database settings are in bots/config/settings.py.'))
sys.exit(1)
else:
botsglobal.logger.info(_(u'Connected to database.'))
atexit.register(botsglobal.db.close)
#initialise user exits for the whole bots-engine (this script file)
try:
userscript,scriptname = botslib.botsimport('routescripts','botsengine')
except ImportError:
userscript = scriptname = None
#**************handle database lock****************************************
#try to set a lock on the database; if this is not possible, the database is already locked. Either:
#1 another instance bots bots-engine is (still) running
#2 or bots-engine had a severe crash.
#What to do?
#first: check ts of database lock. If below a certain value (set in bots.ini) we assume an other instance is running. Exit quietly - no errors, no logging.
# else: Warn user, give advise on what to do. gather data: nr files in, errors.
#next: warn with report & logging. advise a crashrecovery.
if not botslib.set_database_lock():
if '--crashrecovery' in commandstorun: #user starts recovery operation; the databaselock is ignored; the databaselock is unlocked when routes have run.
commandstorun = ['--crashrecovery'] #is an exclusive option!
else:
#when scheduling bots it is possible that the last run is still running. Check if maxruntime has passed:
vanaf = datetime.datetime.today() - datetime.timedelta(minutes=botsglobal.ini.getint('settings','maxruntime',60))
for row in botslib.query('''SELECT ts FROM mutex WHERE ts < %(vanaf)s ''',{'vanaf':vanaf}):
warn = _(u'!Bots database is locked!\nBots-engine has ended in an unexpected way during the last run.\nThis happens, but is very very rare.\nPossible causes: bots-engine terminated by user, system crash, power-down, etc.\nA forced retry of the last run is advised; bots will (try to) repair the last run.')
botsglobal.logger.critical(warn)
botslib.sendbotserrorreport(_(u'[Bots severe error]Database is locked'),warn)
#add: count errors etc.
sys.exit(1)
else: #maxruntime has not passed. Exit silently, nothing reported
botsglobal.logger.info(_(u'Database is locked, but "maxruntime" has not been exceeded.'))
sys.exit(3)
else:
if '--crashrecovery' in commandstorun: #user starts recovery operation but there is no databaselock.
warn = _(u'User started a forced retry of the last run.\nOnly use this when the database is locked.\nThe database was not locked (database is OK).\nSo Bots has done nothing now.')
botsglobal.logger.error(warn)
botslib.sendbotserrorreport(_(u'[Bots Error Report] User started a forced retry of last run, but this was not needed'),warn)
botslib.remove_database_lock()
sys.exit(1)
#*************get list of routes to run****************************************
#~ raise Exception('locked database') #for testing database lock: abort, database will be locked
if routestorun:
botsglobal.logger.info(u'Run routes from command line: "%s".',str(routestorun))
else: # no routes from command line parameters: fetch all active routes from database
for row in botslib.query('''SELECT DISTINCT idroute
FROM routes
WHERE active=%(active)s
AND (notindefaultrun=%(notindefaultrun)s OR notindefaultrun IS NULL)
ORDER BY idroute ''',
{'active':True,'notindefaultrun':False}):
routestorun.append(row['idroute'])
botsglobal.logger.info(_(u'Run active routes from database: "%s".'),str(routestorun))
#routestorun is now either a list with routes from commandline, or the list of active routes for the routes table in the db.
#**************run the routes for retry, retransmit and new runs*************************************
try:
#commandstorun determines the type(s) of run
#routes to run is a listof the routes that are runs (for each command to run
#botsglobal.incommunicate is used to control if there is communication in; only 'new' incommunicates.
#botsglobal.minta4query controls which ta's are queried by the routes.
#stuff2evaluate controls what is evaluated in automatic maintenance.
errorinrun = 0 #detect if there has been some error. Only used for good exit() code
botsglobal.incommunicate = False
if '--crashrecovery' in commandstorun:
botsglobal.logger.info(_(u'Run crash recovery.'))
stuff2evaluate = botslib.set_minta4query_crashrecovery()
if stuff2evaluate:
router.routedispatcher(routestorun)
errorinrun += automaticmaintenance.evaluate('--crashrecovery',stuff2evaluate)
else:
botsglobal.logger.info(_(u'No retry of the last run - there was no last run.'))
if userscript and hasattr(userscript,'postcrashrecovery'):
botslib.runscript(userscript,scriptname,'postcrashrecovery',routestorun=routestorun)
if '--retrycommunication' in commandstorun:
botsglobal.logger.info(_(u'Run communication retry.'))
stuff2evaluate = router.routedispatcher(routestorun,'--retrycommunication')
if stuff2evaluate:
errorinrun += automaticmaintenance.evaluate('--retrycommunication',stuff2evaluate)
else:
botsglobal.logger.info(_(u'Run recommunicate: nothing to recommunicate.'))
if userscript and hasattr(userscript,'postretrycommunication'):
botslib.runscript(userscript,scriptname,'postretrycommunication',routestorun=routestorun)
if '--automaticretrycommunication' in commandstorun:
botsglobal.logger.info(_(u'Run automatic communication retry.'))
stuff2evaluate = router.routedispatcher(routestorun,'--automaticretrycommunication')
if stuff2evaluate:
errorinrun += automaticmaintenance.evaluate('--automaticretrycommunication',stuff2evaluate)
else:
botsglobal.logger.info(_(u'Run automatic recommunicate: nothing to recommunicate.'))
if userscript and hasattr(userscript,'postautomaticretrycommunication'):
botslib.runscript(userscript,scriptname,'postautomaticretrycommunication',routestorun=routestorun)
if '--retry' in commandstorun:
botsglobal.logger.info(u'Run retry.')
stuff2evaluate = router.routedispatcher(routestorun,'--retry')
if stuff2evaluate:
errorinrun += automaticmaintenance.evaluate('--retry',stuff2evaluate)
else:
botsglobal.logger.info(_(u'Run retry: nothing to retry.'))
if userscript and hasattr(userscript,'postretry'):
botslib.runscript(userscript,scriptname,'postretry',routestorun=routestorun)
if '--retransmit' in commandstorun:
botsglobal.logger.info(u'Run retransmit.')
stuff2evaluate = router.routedispatcher(routestorun,'--retransmit')
if stuff2evaluate:
errorinrun += automaticmaintenance.evaluate('--retransmit',stuff2evaluate)
else:
botsglobal.logger.info(_(u'Run retransmit: nothing to retransmit.'))
if userscript and hasattr(userscript,'postretransmit'):
botslib.runscript(userscript,scriptname,'postretransmit',routestorun=routestorun)
if '--new' in commandstorun:
botsglobal.logger.info('Run new.')
botsglobal.incommunicate = True
botsglobal.minta4query = 0 #meaning: reset. the actual value is set later (in routedispatcher)
stuff2evaluate = router.routedispatcher(routestorun)
errorinrun += automaticmaintenance.evaluate('--new',stuff2evaluate)
if userscript and hasattr(userscript,'postnewrun'):
botslib.runscript(userscript,scriptname,'postnewrun',routestorun=routestorun)
if '--cleanup' in commandstorun or botsglobal.ini.get('settings','whencleanup','always')=='always':
botsglobal.logger.debug(u'Do cleanup.')
cleanup.cleanup()
botslib.remove_database_lock()
except Exception,e:
botsglobal.logger.exception(_(u'Severe error in bots system:\n%s')%(e)) #of course this 'should' not happen.
sys.exit(1)
def start():
#********command line arguments**************************
edifile =''
grammarfile = ''
configdir = 'config'
for arg in sys.argv[1:]:
if not arg:
continue
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print ' !!Indicated Bots should use specific .ini file but no file name was given.'
showusage()
elif arg in ["?", "/?"] or arg.startswith('-'):
showusage()
else:
if not edifile:
edifile = arg
else:
grammarfile = arg
if not (edifile and grammarfile):
print ' !!Both edifile and grammarfile are required.'
showusage()
#********end handling command line arguments**************************
editype='xmlnocheck'
messagetype='xmlnocheckxxxtemporaryforxml2grammar'
mpath = []
botsinit.generalinit(configdir)
os.chdir(botsglobal.ini.get('directories','botspath'))
botsinit.initenginelogging()
#the xml file is parsed as an xmlnocheck message....so a (temp) xmlnocheck grammar is needed....without content... this file is not removed....
tmpgrammarfile = botslib.join(botsglobal.ini.get('directories','usersysabs'),'grammars',editype,messagetype+'.py')
f = open(tmpgrammarfile,'w')
f.close()
inn = inmessage.edifromfile(editype=editype,messagetype=messagetype,filename=edifile)
#~ inn.root.display()
out = outmessage.outmessage_init(editype=editype,messagetype=messagetype,filename='botssys/infile/unitnode/output/inisout03.edi',divtext='',topartner='') #make outmessage object
#handle root
rootmpath = [{'BOTSID':inn.root.record['BOTSID']}]
out.put(*rootmpath)
writefields(out,inn.root,rootmpath)
#walk tree; write results to out-tree
for node,mpath in treewalker(inn.root,mpath):
mpath.append({'BOTSID':node.record['BOTSID']})
if out.get(*mpath) is None:
out.put(*mpath)
writefields(out,node,mpath)
#~ out.root.display()
#out-tree is finished; represents ' normalised' tree suited for writing as a grammar
structure = []
recorddefs = {}
tree2grammar(out.root,structure,recorddefs)
#~ for key,value in recorddefs.items():
#~ print key,value
#~ print '\n'
sortedstructurelist = structure2list(structure)
recorddefsstring = recorddefs2string(recorddefs,sortedstructurelist)
structurestring = structure2string(structure)
#write grammar file
grammar = open(grammarfile,'wb')
grammar.write('#grammar automatically generated by bots open source edi software.')
grammar.write('\n')
grammar.write('from bots.botsconfig import *')
grammar.write('\n\n')
grammar.write('syntax = {}')
grammar.write('\n\n')
grammar.write('structure = [\n%s]\n'%(structurestring))
grammar.write('\n\n')
grammar.write('recorddefs = %s'%(recorddefsstring))
grammar.write('\n\n')
grammar.close()
print 'grammar file is written',grammarfile
def start():
#********command line arguments**************************
usage = '''
This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
Updates existing bots database to version %(version)s
Usage:
%(name)s [config-option]
Options:
-c<directory> directory for configuration files (default: config).
'''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version}
configdir = 'config'
for arg in sys.argv[1:]:
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Error: configuration directory indicated, but no directory name.'
sys.exit(3)
else: #pick up names of routes to run
print usage
sys.exit(0)
#***end handling command line arguments**************************
botsinit.generalinit(configdir) #find locating of bots, configfiles, init paths etc.
#**************check if another instance of bots-engine is running/if port is free******************************
try:
engine_socket = botslib.check_if_other_engine_is_running()
except socket.error:
sys.exit(3)
else:
atexit.register(engine_socket.close)
#**************initialise logging******************************
process_name = 'updatedatabase'
botsglobal.logger = botsinit.initenginelogging(process_name)
atexit.register(logging.shutdown)
for key,value in botslib.botsinfo(): #log info about environement, versions, etc
botsglobal.logger.info(u'%(key)s: "%(value)s".',{'key':key,'value':value})
#**************connect to database**********************************
try:
botsinit.connect()
except Exception as msg:
botsglobal.logger.exception(_(u'Could not connect to database. Database settings are in bots/config/settings.py. Error: "%(msg)s".'),{'msg':msg})
sys.exit(3)
else:
botsglobal.logger.info(_(u'Connected to database.'))
atexit.register(botsglobal.db.close)
#**************handle database lock****************************************
#set a lock on the database; if not possible, the database is locked: an earlier instance of bots-engine was terminated unexpectedly.
if not botslib.set_database_lock():
warn = _(u'!Bots database is locked!\n'\
'Bots-engine has ended in an unexpected way during the last run.\n'\
'Most likely causes: sudden power-down, system crash, problems with disk I/O, bots-engine terminated by user, etc.')
botsglobal.logger.critical(warn)
sys.exit(3)
atexit.register(botslib.remove_database_lock)
if botsglobal.settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3':
terug = sqlite3()
elif botsglobal.settings.DATABASES['default']['ENGINE'] == 'django.db.backends.mysql':
terug = mysql()
elif botsglobal.settings.DATABASES['default']['ENGINE'] == 'django.db.backends.postgresql_psycopg2':
terug = postgresql_psycopg2()
sys.exit(terug)
def start(configdir):
botsinit.generalinit(configdir) #find locating of bots, configfiles, init paths etc.
process_name = 'apache_webserver_' + configdir
botsglobal.logger = botsinit.initserverlogging(process_name) #initialise file-logging for web-server. This logging only contains the logging from bots-webserver.
def start():
#********command line arguments**************************
usage = '''
This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
Creates a grammar from an xml file.'
Usage:'
%(name)s -c<directory> <xml_file> <xml_grammar_file>
Options:
-c<directory> directory for configuration files (default: config).
<xml_file> name of the xml file to read
<xml_grammar_file> name of the grammar file to write
'''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version}
configdir = 'config'
edifile =''
grammarfile = ''
for arg in sys.argv[1:]:
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Error: configuration directory indicated, but no directory name.'
sys.exit(1)
elif arg in ["?", "/?",'-h', '--help'] or arg.startswith('-'):
print usage
sys.exit(0)
else:
if not edifile:
edifile = arg
else:
grammarfile = arg
if not edifile or not grammarfile:
print 'Error: both edifile and grammarfile are required.'
sys.exit(0)
#***end handling command line arguments**************************
botsinit.generalinit(configdir) #find locating of bots, configfiles, init paths etc.
process_name = 'xml2botsgrammar'
botsglobal.logger = botsinit.initenginelogging(process_name)
atexit.register(logging.shutdown)
#the xml file is parsed as an xmlnocheck message
editype = 'xmlnocheck'
messagetype = 'xmlnocheckxxxtemporaryforxml2grammar'
mpath = []
#a (temp) xmlnocheck grammar is needed (but needs not actual content. This file is not removed.
tmpgrammarfile = botslib.join(botsglobal.ini.get('directories','usersysabs'),'grammars',editype,messagetype+'.py')
filehandler = open(tmpgrammarfile,'w')
filehandler.close()
#make inmessage object: read the xml file
inn = inmessage.parse_edi_file(editype=editype,messagetype=messagetype,filename=edifile,remove_empties_from_xml=False)
#make outmessage object; nothing is 'filled' yet.
out = outmessage.outmessage_init(editype=editype,messagetype=messagetype,filename='botssys/infile/unitnode/output/inisout03.edi',divtext='',topartner='')
#***do the mapping***************************************************
#handle root
rootmpath = [{'BOTSID':inn.root.record['BOTSID'],'BOTSIDnr':'1'}]
out.put(*rootmpath)
map_writefields(out,inn.root,rootmpath)
#walk tree; write results to out-tree
for node_instance,mpath in map_treewalker(inn.root,mpath):
mpath.append({'BOTSID':node_instance.record['BOTSID']})
if out.get(*mpath) is None:
out.put(*mpath)
map_writefields(out,node_instance,mpath)
#***mapping is done; out-tree is finished; represents 'normalised' tree suited for writing as a grammar
structure = []
recorddefs = {}
tree2grammar(out.root,structure,recorddefs)
#~ for key,value in recorddefs.items():
#~ print key,value
#~ print '\n'
sortedstructurelist = structure2list(structure)
recorddefsstring = recorddefs2string(recorddefs,sortedstructurelist)
structurestring = structure2string(structure)
#write grammar file
grammar = open(grammarfile,'wb')
grammar.write('#grammar automatically generated by bots open source edi translator.')
grammar.write('\n')
grammar.write('from bots.botsconfig import *')
grammar.write('\n\n')
grammar.write('syntax = {}')
grammar.write('\n\n')
grammar.write('structure = [\n%s]\n'%(structurestring))
grammar.write('\n\n')
grammar.write('recorddefs = %s'%(recorddefsstring))
grammar.write('\n\n')
grammar.close()
print 'grammar file is written:',grammarfile
def start():
# NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
# ***command line arguments**************************
usage = """
This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
A utility to generate the index file of a plugin; this can be seen as a database dump of the configuration.
This is eg useful for version control.
Usage:
%(name)s -c<directory>
Options:
-c<directory> directory for configuration files (default: config).
""" % {
"name": os.path.basename(sys.argv[0]),
"version": botsglobal.version,
}
configdir = "config"
for arg in sys.argv[1:]:
if arg.startswith("-c"):
configdir = arg[2:]
if not configdir:
print "Error: configuration directory indicated, but no directory name."
sys.exit(1)
else:
print usage
sys.exit(0)
# ***end handling command line arguments**************************
botsinit.generalinit(configdir) # find locating of bots, configfiles, init paths etc.
if not botsglobal.ini.getboolean("jobqueue", "enabled", False):
print "Error: bots jobqueue cannot start; not enabled in %s/bots.ini" % configdir
sys.exit(1)
process_name = "dirmonitor"
logger = botsinit.initserverlogging(process_name)
logger.log(25, u"Bots %(process_name)s started.", {"process_name": process_name})
logger.log(
25,
u'Bots %(process_name)s configdir: "%(configdir)s".',
{"process_name": process_name, "configdir": botsglobal.ini.get("directories", "config")},
)
botsenginepath = os.path.join(
os.path.dirname(os.path.abspath(sys.argv[0])), "bots-engine.py"
) # get path to bots-engine
cond = threading.Condition()
tasks = set()
dir_watch_data = []
for section in botsglobal.ini.sections():
if section.startswith("dirmonitor") and section[len("dirmonitor") :]:
dir_watch_data.append({})
dir_watch_data[-1]["path"] = botsglobal.ini.get(section, "path")
dir_watch_data[-1]["rec"] = botsglobal.ini.getboolean(section, "recursive", False)
dir_watch_data[-1]["filemask"] = botsglobal.ini.get(section, "filemask", "*")
dir_watch_data[-1]["route"] = botsglobal.ini.get(section, "route", "")
if not dir_watch_data:
logger.error(u"Nothing to watch!")
sys.exit(0)
if os.name == "nt":
# for windows: start a thread per directory watcher
for dir_watch in dir_watch_data:
dir_watch_thread = threading.Thread(target=windows_event_handler, args=(logger, dir_watch, cond, tasks))
dir_watch_thread.daemon = True # do not wait for thread when exiting
dir_watch_thread.start()
else:
# for linux: one watch-thread, but multiple watches.
dir_watch_thread = threading.Thread(target=linux_event_handler, args=(logger, dir_watch_data, cond, tasks))
dir_watch_thread.daemon = True # do not wait for thread when exiting
dir_watch_thread.start()
# this main thread get the results from the watch-thread(s).
logger.info(u"Bots %(process_name)s started.", {"process_name": process_name})
active_receiving = False
timeout = 2.0
cond.acquire()
while True:
# this functions as a buffer: all events go into set tasks.
# the tasks are fired to jobqueue after TIMOUT sec.
# this is to avoid firing to many tasks to jobqueue; events typically come in bursts.
# is value of timeout is larger, reaction times are slower...but less tasks are fired to jobqueue.
# in itself this is not a problem, as jobqueue will alos discard duplicate jobs.
# 2 sec seems to e a good value: reasonable quick, not to nervous.
cond.wait(timeout=timeout) # get back when results, or after timeout sec
if tasks:
if not active_receiving: # first request (after tasks have been fired, or startup of dirmonitor)
active_receiving = True
last_time = time.time()
else: # active receiving events
current_time = time.time()
if current_time - last_time >= timeout: # cond.wait returned probably because of a timeout
try:
for task in tasks:
logger.info(
u'Send to queue "%(path)s %(config)s %(task)s".',
{"path": botsenginepath, "config": "-c" + configdir, "task": task},
)
job2queue.send_job_to_jobqueue([sys.executable, botsenginepath, "-c" + configdir, task])
except Exception as msg:
logger.info(u'Error in running task: "%(msg)s".', {"msg": msg})
tasks.clear()
active_receiving = False
else: # cond.wait returned probably because of a timeout
logger.debug(u"time difference to small.")
last_time = current_time
cond.release()
sys.exit(0)
def start():
#NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
#***command line arguments**************************
usage = '''
This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
A utility to generate the index file of a plugin; this can be seen as a database dump of the configuration.
This is eg useful for version control.
Usage:
%(name)s -c<directory>
Options:
-c<directory> directory for configuration files (default: config).
''' % {
'name': os.path.basename(sys.argv[0]),
'version': botsglobal.version
}
configdir = 'config'
for arg in sys.argv[1:]:
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Error: configuration directory indicated, but no directory name.'
sys.exit(1)
else:
print usage
sys.exit(0)
#***end handling command line arguments**************************
botsinit.generalinit(
configdir) #find locating of bots, configfiles, init paths etc.
if not botsglobal.ini.getboolean('jobqueue', 'enabled', False):
print 'Error: bots jobqueue cannot start; not enabled in %s/bots.ini' % configdir
sys.exit(1)
process_name = 'dirmonitor'
logger = botsinit.initserverlogging(process_name)
logger.log(25, u'Bots %(process_name)s started.',
{'process_name': process_name})
logger.log(
25, u'Bots %(process_name)s configdir: "%(configdir)s".', {
'process_name': process_name,
'configdir': botsglobal.ini.get('directories', 'config')
})
botsenginepath = os.path.join(os.path.dirname(os.path.abspath(
sys.argv[0])), 'bots-engine.py') #get path to bots-engine
cond = threading.Condition()
tasks = set()
dir_watch_data = []
for section in botsglobal.ini.sections():
if section.startswith('dirmonitor') and section[len('dirmonitor'):]:
dir_watch_data.append({})
dir_watch_data[-1]['path'] = botsglobal.ini.get(section, 'path')
dir_watch_data[-1]['rec'] = botsglobal.ini.getboolean(
section, 'recursive', False)
dir_watch_data[-1]['filemask'] = botsglobal.ini.get(
section, 'filemask', '*')
dir_watch_data[-1]['route'] = botsglobal.ini.get(
section, 'route', '')
if not dir_watch_data:
logger.error(u'Nothing to watch!')
sys.exit(0)
if os.name == 'nt':
#for windows: start a thread per directory watcher
for dir_watch in dir_watch_data:
dir_watch_thread = threading.Thread(target=windows_event_handler,
args=(logger, dir_watch, cond,
tasks))
dir_watch_thread.daemon = True #do not wait for thread when exiting
dir_watch_thread.start()
else:
#for linux: one watch-thread, but multiple watches.
dir_watch_thread = threading.Thread(target=linux_event_handler,
args=(logger, dir_watch_data, cond,
tasks))
dir_watch_thread.daemon = True #do not wait for thread when exiting
dir_watch_thread.start()
# this main thread get the results from the watch-thread(s).
logger.info(u'Bots %(process_name)s started.',
{'process_name': process_name})
active_receiving = False
timeout = 2.0
cond.acquire()
while True:
#this functions as a buffer: all events go into set tasks.
#the tasks are fired to jobqueue after TIMOUT sec.
#this is to avoid firing to many tasks to jobqueue; events typically come in bursts.
#is value of timeout is larger, reaction times are slower...but less tasks are fired to jobqueue.
#in itself this is not a problem, as jobqueue will alos discard duplicate jobs.
#2 sec seems to e a good value: reasonable quick, not to nervous.
cond.wait(
timeout=timeout) #get back when results, or after timeout sec
if tasks:
if not active_receiving: #first request (after tasks have been fired, or startup of dirmonitor)
active_receiving = True
last_time = time.time()
else: #active receiving events
current_time = time.time()
if current_time - last_time >= timeout: #cond.wait returned probably because of a timeout
try:
for task in tasks:
logger.info(
u'Send to queue "%(path)s %(config)s %(task)s".',
{
'path': botsenginepath,
'config': '-c' + configdir,
'task': task
})
job2queue.send_job_to_jobqueue([
sys.executable, botsenginepath,
'-c' + configdir, task
])
except Exception as msg:
logger.info(u'Error in running task: "%(msg)s".',
{'msg': msg})
tasks.clear()
active_receiving = False
else: #cond.wait returned probably because of a timeout
logger.debug(u'time difference to small.')
last_time = current_time
cond.release()
sys.exit(0)
def start():
#NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
#********command line arguments**************************
usage = '''
This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
Checks a Bots grammar. Same checks are used as in translations with bots-engine. Searches for grammar in
regular place: bots/usersys/grammars/<editype>/<messagetype>.py (even if a path is passed).
Usage: %(name)s -c<directory> <editype> <messagetype>
or %(name)s -c<directory> <path to grammar>
Options:
-c<directory> directory for configuration files (default: config).
Examples:
%(name)s -cconfig edifact ORDERSD96AUNEAN008
%(name)s -cconfig C:/python27/lib/site-packages/bots/usersys/grammars/edifact/ORDERSD96AUNEAN008.py
''' % {
'name': os.path.basename(sys.argv[0]),
'version': botsglobal.version
}
configdir = 'config'
editype = ''
messagetype = ''
for arg in sys.argv[1:]:
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Error: configuration directory indicated, but no directory name.'
sys.exit(1)
elif arg in ["?", "/?", '-h', '--help'] or arg.startswith('-'):
print usage
sys.exit(0)
else:
if os.path.isfile(arg):
p1, p2 = os.path.split(arg)
editype = os.path.basename(p1)
messagetype, ext = os.path.splitext(p2)
messagetype = unicode(messagetype)
print 'grammarcheck', editype, messagetype
elif not editype:
editype = arg
else:
messagetype = arg
if not (editype and messagetype):
print 'Error: both editype and messagetype, or a file path, are required.'
sys.exit(1)
#***end handling command line arguments**************************
botsinit.generalinit(
configdir) #find locating of bots, configfiles, init paths etc.
process_name = 'grammarcheck'
botsglobal.logger = botsinit.initenginelogging(process_name)
atexit.register(logging.shutdown)
try:
grammar.grammarread(editype, messagetype)
except:
print 'Found error in grammar: ', botslib.txtexc()
sys.exit(1)
else:
print 'OK - no error found in grammar'
sys.exit(0)
def start():
''' sysexit codes:
0: OK, no errors
1: (system) errors incl parsing of command line arguments
2: bots ran OK, but there are errors/process errors in the run
3: Database is locked, but "maxruntime" has not been exceeded.
'''
#NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
#********command line arguments**************************
usage = '''
This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
Does the actual translations and communications; it's the workhorse. It does not have a fancy interface.
Usage:
%(name)s [config-option]
Config-option:
-c<directory> directory for configuration files (default: config).
'''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version}
configdir = 'config'
for arg in sys.argv[1:]:
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Error: configuration directory indicated, but no directory name.'
sys.exit(1)
elif arg in ["?", "/?",'-h', '--help'] or arg.startswith('-'):
print usage
sys.exit(0)
#***********end handling command line arguments**************************
botsinit.generalinit(configdir) #find locating of bots, configfiles, init paths etc.
#set working directory to bots installation. advantage: when using relative paths it is clear that this point paths within bots installation.
os.chdir(botsglobal.ini.get('directories','botspath'))
#**************initialise logging******************************
process_name = 'engine2'
botsglobal.logger = botsinit.initenginelogging(process_name)
atexit.register(logging.shutdown)
for key,value in botslib.botsinfo(): #log info about environement, versions, etc
botsglobal.logger.info(u'%(key)s: "%(value)s".',{'key':key,'value':value})
#**************connect to database**********************************
try:
botsinit.connect()
except Exception as msg:
botsglobal.logger.exception(_(u'Could not connect to database. Database settings are in bots/config/settings.py. Error: "%(msg)s".'),{'msg':msg})
sys.exit(1)
else:
botsglobal.logger.info(_(u'Connected to database.'))
atexit.register(botsglobal.db.close)
warnings.simplefilter('error', UnicodeWarning)
#import global scripts for bots-engine
try:
userscript,scriptname = botslib.botsimport('routescripts','botsengine')
except botslib.BotsImportError: #userscript is not there; other errors like syntax errors are not catched
userscript = scriptname = None
#***acceptance tests: initialiase acceptance user script******************************
acceptance_userscript = acceptance_scriptname = None
if botsglobal.ini.getboolean('acceptance','runacceptancetest',False):
botsglobal.logger.info(_(u'This run is an acceptance test - as indicated in option "runacceptancetest" in bots.ini.'))
try:
acceptance_userscript,acceptance_scriptname = botslib.botsimport('routescripts','bots_acceptancetest')
except botslib.BotsImportError:
botsglobal.logger.info(_(u'In acceptance test there is no script file "bots_acceptancetest.py" to check the results of the acceptance test.'))
try:
#~ botslib.prepare_confirmrules()
#in acceptance tests: run a user script before running eg to clean output directories******************************
botslib.tryrunscript(acceptance_userscript,acceptance_scriptname,'pretest')
botslib.tryrunscript(userscript,scriptname,'pre')
errorinrun = engine2_run()
except Exception as msg:
botsglobal.logger.exception(_(u'Severe error in bots system:\n%(msg)s'),{'msg':unicode(msg)}) #of course this 'should' not happen.
sys.exit(1)
else:
if errorinrun:
sys.exit(2) #indicate: error(s) in run(s)
else:
sys.exit(0) #OK
def start():
#********command line arguments**************************
usage = '''
This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
Updates existing bots database to version %(version)s
Usage:
%(name)s [config-option]
Options:
-c<directory> directory for configuration files (default: config).
''' % {
'name': os.path.basename(sys.argv[0]),
'version': botsglobal.version
}
configdir = 'config'
for arg in sys.argv[1:]:
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Error: configuration directory indicated, but no directory name.'
sys.exit(3)
else: #pick up names of routes to run
print usage
sys.exit(0)
#***end handling command line arguments**************************
botsinit.generalinit(
configdir) #find locating of bots, configfiles, init paths etc.
#**************check if another instance of bots-engine is running/if port is free******************************
try:
engine_socket = botslib.check_if_other_engine_is_running()
except socket.error:
sys.exit(3)
else:
atexit.register(engine_socket.close)
#**************initialise logging******************************
process_name = 'updatedatabase'
botsglobal.logger = botsinit.initenginelogging(process_name)
atexit.register(logging.shutdown)
for key, value in botslib.botsinfo(
): #log info about environement, versions, etc
botsglobal.logger.info(u'%(key)s: "%(value)s".', {
'key': key,
'value': value
})
#**************connect to database**********************************
try:
botsinit.connect()
except Exception as msg:
botsglobal.logger.exception(
_(u'Could not connect to database. Database settings are in bots/config/settings.py. Error: "%(msg)s".'
), {'msg': msg})
sys.exit(3)
else:
botsglobal.logger.info(_(u'Connected to database.'))
atexit.register(botsglobal.db.close)
#**************handle database lock****************************************
#set a lock on the database; if not possible, the database is locked: an earlier instance of bots-engine was terminated unexpectedly.
if not botslib.set_database_lock():
warn = _(u'!Bots database is locked!\n'\
'Bots-engine has ended in an unexpected way during the last run.\n'\
'Most likely causes: sudden power-down, system crash, problems with disk I/O, bots-engine terminated by user, etc.')
botsglobal.logger.critical(warn)
sys.exit(3)
atexit.register(botslib.remove_database_lock)
if botsglobal.settings.DATABASES['default'][
'ENGINE'] == 'django.db.backends.sqlite3':
terug = sqlite3()
elif botsglobal.settings.DATABASES['default'][
'ENGINE'] == 'django.db.backends.mysql':
terug = mysql()
elif botsglobal.settings.DATABASES['default'][
'ENGINE'] == 'django.db.backends.postgresql_psycopg2':
terug = postgresql_psycopg2()
sys.exit(terug)
def start():
#NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
#***command line arguments**************************
usage = '''
This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
The %(name)s is the web server for bots; the interface (bots-monitor) can be accessed in a
browser, eg 'http://localhost:8080'.
Usage:
%(name)s -c<directory>
Options:
-c<directory> directory for configuration files (default: config).
''' % {
'name': os.path.basename(sys.argv[0]),
'version': botsglobal.version
}
configdir = 'config'
for arg in sys.argv[1:]:
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Error: configuration directory indicated, but no directory name.'
sys.exit(1)
else:
print usage
sys.exit(0)
#***end handling command line arguments**************************
botsinit.generalinit(
configdir) #find locating of bots, configfiles, init paths etc.
process_name = 'webserver'
botsglobal.logger = botsinit.initserverlogging(
process_name
) #initialise file-logging for web-server. This logging only contains the logging from bots-webserver, not from cherrypy.
#***init cherrypy as webserver*********************************************
#global configuration for cherrypy
cherrypy.config.update({
'global': {
'log.screen':
False,
'server.environment':
botsglobal.ini.get('webserver', 'environment', 'production')
}
})
#cherrypy handling of static files
conf = {
'/': {
'tools.staticdir.on': True,
'tools.staticdir.dir': 'media',
'tools.staticdir.root': botsglobal.ini.get('directories',
'botspath')
}
}
servestaticfiles = cherrypy.tree.mount(
None, '/media', conf
) #None: no cherrypy application (as this only serves static files)
#cherrypy handling of django
servedjango = WSGIHandler(
) #was: servedjango = AdminMediaHandler(WSGIHandler()) but django does not need the AdminMediaHandler in this setup. is much faster.
#cherrypy uses a dispatcher in order to handle the serving of static files and django.
dispatcher = wsgiserver.WSGIPathInfoDispatcher({
'/': servedjango,
'/media': servestaticfiles
})
botswebserver = wsgiserver.CherryPyWSGIServer(
bind_addr=('0.0.0.0', botsglobal.ini.getint('webserver', 'port',
8080)),
wsgi_app=dispatcher,
server_name=botsglobal.ini.get('webserver', 'name', 'bots-webserver'))
botsglobal.logger.log(25, _(u'Bots %(process_name)s started.'),
{'process_name': process_name})
botsglobal.logger.log(
25, _(u'Bots %(process_name)s configdir: "%(configdir)s".'), {
'process_name': process_name,
'configdir': botsglobal.ini.get('directories', 'config')
})
botsglobal.logger.log(
25, _(u'Bots %(process_name)s serving at port: "%(port)s".'), {
'process_name': process_name,
'port': botsglobal.ini.getint('webserver', 'port', 8080)
})
#handle ssl: cherrypy < 3.2 always uses pyOpenssl. cherrypy >= 3.2 uses python buildin ssl (python >= 2.6 has buildin support for ssl).
ssl_certificate = botsglobal.ini.get('webserver', 'ssl_certificate', None)
ssl_private_key = botsglobal.ini.get('webserver', 'ssl_private_key', None)
if ssl_certificate and ssl_private_key:
if cherrypy.__version__ >= '3.2.0':
adapter_class = wsgiserver.get_ssl_adapter_class('builtin')
botswebserver.ssl_adapter = adapter_class(ssl_certificate,
ssl_private_key)
else:
#but: pyOpenssl should be there!
botswebserver.ssl_certificate = ssl_certificate
botswebserver.ssl_private_key = ssl_private_key
botsglobal.logger.log(25,
_(u'Bots %(process_name)s uses ssl (https).'),
{'process_name': process_name})
else:
botsglobal.logger.log(
25, _(u'Bots %(process_name)s uses plain http (no ssl).'),
{'process_name': process_name})
#***start the cherrypy webserver.************************************************
try:
botswebserver.start()
except KeyboardInterrupt:
botswebserver.stop()
def start():
''' sysexit codes:
0: OK, no errors
1: (system) errors incl parsing of command line arguments
2: bots ran OK, but there are errors/process errors in the run
3: Database is locked, but "maxruntime" has not been exceeded.
'''
#NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
#********command line arguments**************************
usage = '''
This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
Does the actual translations and communications; it's the workhorse. It does not have a fancy interface.
Usage:
%(name)s [config-option]
Config-option:
-c<directory> directory for configuration files (default: config).
''' % {
'name': os.path.basename(sys.argv[0]),
'version': botsglobal.version
}
configdir = 'config'
for arg in sys.argv[1:]:
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Error: configuration directory indicated, but no directory name.'
sys.exit(1)
elif arg in ["?", "/?", '-h', '--help'] or arg.startswith('-'):
print usage
sys.exit(0)
#***********end handling command line arguments**************************
botsinit.generalinit(
configdir) #find locating of bots, configfiles, init paths etc.
#set working directory to bots installation. advantage: when using relative paths it is clear that this point paths within bots installation.
os.chdir(botsglobal.ini.get('directories', 'botspath'))
#**************initialise logging******************************
process_name = 'engine2'
botsglobal.logger = botsinit.initenginelogging(process_name)
atexit.register(logging.shutdown)
for key, value in botslib.botsinfo(
): #log info about environement, versions, etc
botsglobal.logger.info(u'%(key)s: "%(value)s".', {
'key': key,
'value': value
})
#**************connect to database**********************************
try:
botsinit.connect()
except Exception as msg:
botsglobal.logger.exception(
_(u'Could not connect to database. Database settings are in bots/config/settings.py. Error: "%(msg)s".'
), {'msg': msg})
sys.exit(1)
else:
botsglobal.logger.info(_(u'Connected to database.'))
atexit.register(botsglobal.db.close)
warnings.simplefilter('error', UnicodeWarning)
#import global scripts for bots-engine
try:
userscript, scriptname = botslib.botsimport('routescripts',
'botsengine')
except botslib.BotsImportError: #userscript is not there; other errors like syntax errors are not catched
userscript = scriptname = None
#***acceptance tests: initialiase acceptance user script******************************
acceptance_userscript = acceptance_scriptname = None
if botsglobal.ini.getboolean('acceptance', 'runacceptancetest', False):
botsglobal.logger.info(
_(u'This run is an acceptance test - as indicated in option "runacceptancetest" in bots.ini.'
))
try:
acceptance_userscript, acceptance_scriptname = botslib.botsimport(
'routescripts', 'bots_acceptancetest')
except botslib.BotsImportError:
botsglobal.logger.info(
_(u'In acceptance test there is no script file "bots_acceptancetest.py" to check the results of the acceptance test.'
))
try:
#~ botslib.prepare_confirmrules()
#in acceptance tests: run a user script before running eg to clean output directories******************************
botslib.tryrunscript(acceptance_userscript, acceptance_scriptname,
'pretest')
botslib.tryrunscript(userscript, scriptname, 'pre')
errorinrun = engine2_run()
except Exception as msg:
botsglobal.logger.exception(
_(u'Severe error in bots system:\n%(msg)s'),
{'msg': unicode(msg)}) #of course this 'should' not happen.
sys.exit(1)
else:
if errorinrun:
sys.exit(2) #indicate: error(s) in run(s)
else:
sys.exit(0) #OK
def start():
''' sysexit codes:
0: OK, no errors
1: (system) errors incl parsing of command line arguments
2: bots ran OK, but there are errors/process errors in the run
3: Database is locked, but "maxruntime" has not been exceeded.
'''
#NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
#********command line arguments**************************
usage = '''
This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
Does the actual translations and communications; it's the workhorse. It does not have a fancy interface.
Usage:
%(name)s [run-options] [config-option] [routes]
Run-options (can be combined):
--new receive new edi files (default: if no run-option given: run as new).
--resend resend as indicated by user.
--rereceive rereceive as indicated by user.
--automaticretrycommunication - automatically retry outgoing communication.
--cleanup remove older data from database.
Config-option:
-c<directory> directory for configuration files (default: config).
Routes: list of routes to run. Default: all active routes (in the database)
'''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version}
configdir = 'config'
commandspossible = ['--automaticretrycommunication','--resend','--rereceive','--new']
commandstorun = []
routestorun = [] #list with routes to run
do_cleanup_parameter = False
for arg in sys.argv[1:]:
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Error: configuration directory indicated, but no directory name.'
sys.exit(1)
elif arg in commandspossible:
commandstorun.append(arg)
elif arg == '--cleanup':
do_cleanup_parameter = True
elif arg in ["?", "/?",'-h', '--help'] or arg.startswith('-'):
print usage
sys.exit(0)
else: #pick up names of routes to run
routestorun.append(arg)
if not commandstorun and not do_cleanup_parameter: #if no command on command line, use new (default)
commandstorun = ['--new']
commandstorun = [command[2:] for command in commandspossible if command in commandstorun] #sort commands
#***end handling command line arguments**************************
botsinit.generalinit(configdir) #find locating of bots, configfiles, init paths etc.
#set working directory to bots installation.
#possible advantage: when using relative paths it is clear that this point paths within bots installation.
#most of time not needed:
#1. in production: do not use relative paths within bots directory
#2. in eg incoming messages path name is used via botslib.join, which makes an absulute path...
#use this as bots2.* always ad this; avoid breaking.
os.chdir(botsglobal.ini.get('directories','botspath'))
#**************check if another instance of bots-engine is running/if port is free******************************
try:
engine_socket = botslib.check_if_other_engine_is_running()
except socket.error:
sys.exit(3)
else:
atexit.register(engine_socket.close)
#**************initialise logging******************************
process_name = 'engine'
botsglobal.logger = botsinit.initenginelogging(process_name)
atexit.register(logging.shutdown)
for key,value in botslib.botsinfo(): #log info about environement, versions, etc
botsglobal.logger.info(u'%(key)s: "%(value)s".',{'key':key,'value':value})
#**************connect to database**********************************
try:
botsinit.connect()
except Exception,msg:
botsglobal.logger.exception(_(u'Could not connect to database. Database settings are in bots/config/settings.py. Error: "%(msg)s".'),{'msg':msg})
sys.exit(1)
def start():
''' sysexit codes:
0: OK, no errors
1: (system) errors incl parsing of command line arguments
2: bots ran OK, but there are errors/process errors in the run
3: Database is locked, but "maxruntime" has not been exceeded.
'''
#NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
#********command line arguments**************************
usage = '''
This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
Does the actual translations and communications; it's the workhorse. It does not have a fancy interface.
Usage:
%(name)s [run-options] [config-option] [routes]
Run-options (can be combined):
--new receive new edi files (default: if no run-option given: run as new).
--resend resend as indicated by user.
--rereceive rereceive as indicated by user.
--automaticretrycommunication - automatically retry outgoing communication.
--cleanup remove older data from database.
Config-option:
-c<directory> directory for configuration files (default: config).
Routes: list of routes to run. Default: all active routes (in the database)
''' % {
'name': os.path.basename(sys.argv[0]),
'version': botsglobal.version
}
configdir = 'config'
commandspossible = [
'--automaticretrycommunication', '--resend', '--rereceive', '--new'
]
commandstorun = []
routestorun = [] #list with routes to run
do_cleanup_parameter = False
for arg in sys.argv[1:]:
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Error: configuration directory indicated, but no directory name.'
sys.exit(1)
elif arg in commandspossible:
commandstorun.append(arg)
elif arg == '--cleanup':
do_cleanup_parameter = True
elif arg in ["?", "/?", '-h', '--help'] or arg.startswith('-'):
print usage
sys.exit(0)
else: #pick up names of routes to run
routestorun.append(arg)
if not commandstorun and not do_cleanup_parameter: #if no command on command line, use new (default)
commandstorun = ['--new']
commandstorun = [
command[2:] for command in commandspossible if command in commandstorun
] #sort commands
#***********end handling command line arguments**************************
botsinit.generalinit(
configdir) #find locating of bots, configfiles, init paths etc.
#set working directory to bots installation. advantage: when using relative paths it is clear that this point paths within bots installation.
os.chdir(botsglobal.ini.get('directories', 'botspath'))
#**************check if another instance of bots-engine is running/if port is free******************************
try:
engine_socket = botslib.check_if_other_engine_is_running()
except socket.error:
sys.exit(3)
else:
atexit.register(engine_socket.close)
#**************initialise logging******************************
process_name = 'engine'
botsglobal.logger = botsinit.initenginelogging(process_name)
atexit.register(logging.shutdown)
for key, value in botslib.botsinfo(
): #log info about environement, versions, etc
botsglobal.logger.info(u'%(key)s: "%(value)s".', {
'key': key,
'value': value
})
#**************connect to database**********************************
try:
botsinit.connect()
except Exception as msg:
botsglobal.logger.exception(
_(u'Could not connect to database. Database settings are in bots/config/settings.py. Error: "%(msg)s".'
), {'msg': msg})
sys.exit(1)
else:
botsglobal.logger.info(_(u'Connected to database.'))
atexit.register(botsglobal.db.close)
#************initialise user exits for the whole bots-engine*************************
try:
userscript, scriptname = botslib.botsimport('routescripts',
'botsengine')
except botslib.BotsImportError: #userscript is not there; other errors like syntax errors are not catched
userscript = scriptname = None
#***acceptance tests: initialiase acceptance user script******************************
acceptance_userscript = acceptance_scriptname = None
if botsglobal.ini.getboolean('acceptance', 'runacceptancetest', False):
botsglobal.logger.info(
_(u'This run is an acceptance test - as indicated in option "runacceptancetest" in bots.ini.'
))
try:
acceptance_userscript, acceptance_scriptname = botslib.botsimport(
'routescripts', 'bots_acceptancetest')
except botslib.BotsImportError:
botsglobal.logger.info(
_(u'In acceptance test there is no script file "bots_acceptancetest.py" to check the results of the acceptance test.'
))
#**************handle database lock****************************************
#set a lock on the database; if not possible, the database is locked: an earlier instance of bots-engine was terminated unexpectedly.
if not botslib.set_database_lock():
#for SQLite: do a integrity check on the database
if botsglobal.settings.DATABASES['default'][
'ENGINE'] == 'django.db.backends.sqlite3':
cursor = botsglobal.db.execute('''PRAGMA integrity_check''')
result = cursor.fetchone()
if result[0] != u'ok':
warn = _(u'!Bots database is locked!\n'\
'Bots did an integrity check on the database, but database was not OK.\n'\
'Manual action is needed!\n'\
'Bots has stopped processing EDI files.')
botsglobal.logger.critical(warn)
botslib.sendbotserrorreport(
_(u'[Bots severe error]Database is damaged'), warn)
sys.exit(1)
warn = _(u'!Bots database is locked!\n'\
'Bots-engine has ended in an unexpected way during the last run.\n'\
'Most likely causes: sudden power-down, system crash, problems with disk I/O, bots-engine terminated by user, etc.\n'
'Bots will do an automatic crash recovery now.')
botsglobal.logger.critical(warn)
botslib.sendbotserrorreport(
_(u'[Bots severe error]Database is locked'), warn)
commandstorun.insert(
0, 'crashrecovery'
) #there is a database lock. Add a crashrecovery as first command to run.
atexit.register(botslib.remove_database_lock)
warnings.simplefilter('error', UnicodeWarning)
#**************run the routes**********************************************
#commandstorun determines the type(s) of run. eg: ['automaticretrycommunication','new']
try:
botslib.prepare_confirmrules()
#in acceptance tests: run a user script before running eg to clean output directories******************************
botslib.tryrunscript(acceptance_userscript,
acceptance_scriptname,
'pretest',
routestorun=routestorun)
botslib.tryrunscript(userscript,
scriptname,
'pre',
commandstorun=commandstorun,
routestorun=routestorun)
errorinrun = 0 #detect if there has been some error. Only used for correct exit() code
first_command_2_run = True
for command in commandstorun:
#if multiple commands in run: reports etc are based on timestamp; so there needs to be at least one second between these runs.
if first_command_2_run:
first_command_2_run = False
else:
time.sleep(1)
botsglobal.logger.info(_(u'Run "%(command)s".'),
{'command': command})
#************get list of routes to run*******************************
if routestorun:
use_routestorun = routestorun[:]
botsglobal.logger.info(
_(u'Run routes from command line: "%(routes)s".'),
{'routes': unicode(use_routestorun)})
elif command == 'new': #fetch all active routes from database unless 'not in default run' or not active.
use_routestorun = []
for row in botslib.query(
'''SELECT DISTINCT idroute
FROM routes
WHERE active=%(active)s
AND (notindefaultrun=%(notindefaultrun)s OR notindefaultrun IS NULL)
ORDER BY idroute ''', {
'active': True,
'notindefaultrun': False
}):
use_routestorun.append(row['idroute'])
botsglobal.logger.info(
_(u'Run active routes from database that are in default run: "%(routes)s".'
), {'routes': unicode(use_routestorun)})
else: #for command other than 'new': use all active routes.
use_routestorun = []
for row in botslib.query(
'''SELECT DISTINCT idroute
FROM routes
WHERE active=%(active)s
ORDER BY idroute ''',
{'active': True}):
use_routestorun.append(row['idroute'])
botsglobal.logger.info(
_(u'Run all active routes from database: "%(routes)s".'),
{'routes': unicode(use_routestorun)})
#************run routes for this command******************************
botslib.tryrunscript(userscript,
scriptname,
'pre' + command,
routestorun=use_routestorun)
errorinrun += router.rundispatcher(command, use_routestorun)
botslib.tryrunscript(userscript,
scriptname,
'post' + command,
routestorun=use_routestorun)
#*********finished running routes for this command****************************
#*********finished all commands****************************************
botslib.tryrunscript(userscript,
scriptname,
'post',
commandstorun=commandstorun,
routestorun=routestorun)
try: #in acceptance tests: run a user script. no good reporting of errors/results in post-test script. Reason: this is after automaticmaintence.
botslib.tryrunscript(acceptance_userscript,
acceptance_scriptname,
'posttest',
routestorun=use_routestorun)
except Exception as msg:
print unicode(msg)
cleanup.cleanup(do_cleanup_parameter, userscript, scriptname)
except Exception as msg:
botsglobal.logger.exception(
_(u'Severe error in bots system:\n%(msg)s'),
{'msg': unicode(msg)}) #of course this 'should' not happen.
sys.exit(1)
else:
if errorinrun:
sys.exit(2) #indicate: error(s) in run(s)
else:
sys.exit(0) #OK
def start():
#NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
#***command line arguments**************************
usage = '''
This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
The %(name)s is the web server for bots; the interface (bots-monitor) can be accessed in a
browser, eg 'http://localhost:8080'.
Usage:
%(name)s -c<directory>
Options:
-c<directory> directory for configuration files (default: config).
'''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version}
configdir = 'config'
for arg in sys.argv[1:]:
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Error: configuration directory indicated, but no directory name.'
sys.exit(1)
else:
print usage
sys.exit(0)
#***end handling command line arguments**************************
botsinit.generalinit(configdir) #find locating of bots, configfiles, init paths etc.
process_name = 'webserver'
botsglobal.logger = botsinit.initserverlogging(process_name) #initialise file-logging for web-server. This logging only contains the logging from bots-webserver, not from cherrypy.
#***init cherrypy as webserver*********************************************
#global configuration for cherrypy
cherrypy.config.update({'global': {'log.screen': False, 'server.environment': botsglobal.ini.get('webserver','environment','production')}})
#cherrypy handling of static files
conf = {'/': {'tools.staticdir.on' : True,'tools.staticdir.dir' : 'media' ,'tools.staticdir.root': botsglobal.ini.get('directories','botspath')}}
servestaticfiles = cherrypy.tree.mount(None, '/media', conf) #None: no cherrypy application (as this only serves static files)
#cherrypy handling of django
servedjango = WSGIHandler() #was: servedjango = AdminMediaHandler(WSGIHandler()) but django does not need the AdminMediaHandler in this setup. is much faster.
#cherrypy uses a dispatcher in order to handle the serving of static files and django.
dispatcher = wsgiserver.WSGIPathInfoDispatcher({'/': servedjango, '/media': servestaticfiles})
botswebserver = wsgiserver.CherryPyWSGIServer(bind_addr=('0.0.0.0', botsglobal.ini.getint('webserver','port',8080)), wsgi_app=dispatcher, server_name=botsglobal.ini.get('webserver','name','bots-webserver'))
botsglobal.logger.log(25,_(u'Bots %(process_name)s started.'),
{'process_name':process_name})
botsglobal.logger.log(25,_(u'Bots %(process_name)s configdir: "%(configdir)s".'),
{'process_name':process_name, 'configdir':botsglobal.ini.get('directories','config')})
botsglobal.logger.log(25,_(u'Bots %(process_name)s serving at port: "%(port)s".'),
{'process_name':process_name,'port':botsglobal.ini.getint('webserver','port',8080)})
#handle ssl: cherrypy < 3.2 always uses pyOpenssl. cherrypy >= 3.2 uses python buildin ssl (python >= 2.6 has buildin support for ssl).
ssl_certificate = botsglobal.ini.get('webserver','ssl_certificate',None)
ssl_private_key = botsglobal.ini.get('webserver','ssl_private_key',None)
if ssl_certificate and ssl_private_key:
if cherrypy.__version__ >= '3.2.0':
adapter_class = wsgiserver.get_ssl_adapter_class('builtin')
botswebserver.ssl_adapter = adapter_class(ssl_certificate,ssl_private_key)
else:
#but: pyOpenssl should be there!
botswebserver.ssl_certificate = ssl_certificate
botswebserver.ssl_private_key = ssl_private_key
botsglobal.logger.log(25,_(u'Bots %(process_name)s uses ssl (https).'),{'process_name':process_name})
else:
botsglobal.logger.log(25,_(u'Bots %(process_name)s uses plain http (no ssl).'),{'process_name':process_name})
#***start the cherrypy webserver.************************************************
try:
botswebserver.start()
except KeyboardInterrupt:
botswebserver.stop()
