def start(configdir):
botsinit.generalinit(
configdir) #find locating of bots, configfiles, init paths etc.
process_name = 'apache_webserver_' + configdir
botsglobal.logger = botsinit.initserverlogging(
process_name
) #initialise file-logging for web-server. This logging only contains the logging from bots-webserver.
def start():
#NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
#***command line arguments**************************
usage = '''
This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
Server program that ensures only a single bots-engine runs at any time, and no engine run requests are
lost/discarded. Each request goes to a queue and is run in sequence when the previous run completes.
Use of the job queue is optional and must be configured in bots.ini (jobqueue section, enabled = True).
Usage:
%(name)s -c<directory>
Options:
-c<directory> directory for configuration files (default: config).
'''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version}
configdir = 'config'
for arg in sys.argv[1:]:
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Error: configuration directory indicated, but no directory name.'
sys.exit(1)
else:
print usage
sys.exit(0)
#***end handling command line arguments**************************
botsinit.generalinit(configdir) #find locating of bots, configfiles, init paths etc.
if not botsglobal.ini.getboolean('jobqueue','enabled',False):
print 'Error: bots jobqueue cannot start; not enabled in %s/bots.ini' % configdir
sys.exit(1)
process_name = 'jobqueue'
logger = botsinit.initserverlogging(process_name)
logger.log(25,u'Bots %(process_name)s started.',process_name=process_name)
logger.log(25,u'Bots %(process_name)s configdir: "%(configdir)s".',process_name=process_name,configdir=botsglobal.ini.get('directories','config'))
port = botsglobal.ini.getint('jobqueue','port',28082)
logger.log(25,u'Bots %(process_name)s listens for xmlrpc at port: "%(port)s".',process_name=process_name,port=port)
#start launcher thread
lauchfrequency = botsglobal.ini.getint('jobqueue','lauchfrequency',5)
maxruntime = botsglobal.ini.getint('settings','maxruntime',60)
launcher_thread = threading.Thread(name='launcher', target=launcher, args=(logger,port,lauchfrequency,maxruntime))
launcher_thread.daemon = True
launcher_thread.start()
logger.info(u'Jobqueue launcher started.')
# this main thread is the jobqserver (the xmlrpc server for handling jobqueue)
logger.info(u'Jobqueue server started.')
server = SimpleXMLRPCServer(('localhost', port),logRequests=False)
server.register_instance(Jobqueue(logger))
try:
server.serve_forever()
except KeyboardInterrupt:
pass
sys.exit(0)
def start():
#NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
#***command line arguments**************************
usage = '''
This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
Server program that ensures only a single bots-engine runs at any time, and no engine run requests are
lost/discarded. Each request goes to a queue and is run in sequence when the previous run completes.
Use of the job queue is optional and must be configured in bots.ini (jobqueue section, enabled = True).
Usage:
%(name)s -c<directory>
Options:
-c<directory> directory for configuration files (default: config).
'''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version}
configdir = 'config'
for arg in sys.argv[1:]:
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Error: configuration directory indicated, but no directory name.'
sys.exit(1)
else:
print usage
sys.exit(0)
#***end handling command line arguments**************************
botsinit.generalinit(configdir) #find locating of bots, configfiles, init paths etc.
if not botsglobal.ini.getboolean('jobqueue','enabled',False):
print 'Error: bots jobqueue cannot start; not enabled in %s/bots.ini' % configdir
sys.exit(1)
process_name = 'jobqueue'
logger = botsinit.initserverlogging(process_name)
logger.log(25,u'Bots %(process_name)s started.',{'process_name':process_name})
logger.log(25,u'Bots %(process_name)s configdir: "%(configdir)s".',{'process_name':process_name,'configdir':botsglobal.ini.get('directories','config')})
port = botsglobal.ini.getint('jobqueue','port',28082)
logger.log(25,u'Bots %(process_name)s listens for xmlrpc at port: "%(port)s".',{'process_name':process_name,'port':port})
#start launcher thread
lauchfrequency = botsglobal.ini.getint('jobqueue','lauchfrequency',5)
maxruntime = botsglobal.ini.getint('settings','maxruntime',60)
launcher_thread = threading.Thread(name='launcher', target=launcher, args=(logger,port,lauchfrequency,maxruntime))
launcher_thread.daemon = True
launcher_thread.start()
logger.info(u'Jobqueue launcher started.')
#the main thread is the xmlrpc server: all adding, getting etc for jobqueue is done via xmlrpc.
logger.info(u'Jobqueue server started.')
server = SimpleXMLRPCServer(('localhost', port),logRequests=False)
server.register_instance(Jobqueue(logger))
try:
server.serve_forever()
except KeyboardInterrupt:
pass
sys.exit(0)
def start():
# NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
# ***command line arguments**************************
usage = '''
This is "%(name)s", developed by Jagged Peak Inc.
A utility to read a plugin. Enter a comma separated list of plugin files.
Usage:
%(name)s -c<config_directory> -d<plugin_directory> <pluginfile(s)>
Options:
-c<directory> directory for configuration files (default: config).
-d<directory> directory for plugin file(s) default: D:/BOTS/plugins
''' % {
'name': os.path.basename(sys.argv[0]),
'version': botsglobal.version
}
configdir = 'config'
plugindir = 'D:/BOTS/plugins'
pluginfiles = ''
# Handle cmd arguments
for arg in sys.argv[1:]:
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Error: configuration directory indicated, but no directory name.'
sys.exit(1)
if arg.startswith('-d'):
plugindir = arg[2:]
if not configdir:
print 'Error: plugin directory indicated, but no directory name.'
sys.exit(1)
else:
pluginfiles = arg
if not pluginfiles:
print 'Plugin file was not entered. See usage'
print usage
sys.exit(1)
# Init required vars and libs
botsinit.generalinit(
configdir) # find locating of bots, configfiles, init paths etc.
process_name = 'plug_read'
botsglobal.logger = botsinit.initserverlogging(process_name)
import pluglib # import here, import at start of file gives error; first initialize.
# Call read plugin for each file
pluginfiles_list = pluginfiles.split(',')
for pluginfile in pluginfiles_list:
plugin_file_path = os.path.join(plugindir, pluginfile)
pluglib.read_plugin(plugin_file_path)
def start():
#NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
#***command line arguments**************************
usage = '''
This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
A utility to generate the index file of a plugin; this can be seen as a database dump of the configuration.
This is eg useful for version control.
Usage:
%(name)s -c<directory>
Options:
-c<directory> directory for configuration files (default: config).
''' % {
'name': os.path.basename(sys.argv[0]),
'version': botsglobal.version
}
configdir = 'config'
for arg in sys.argv[1:]:
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Error: configuration directory indicated, but no directory name.'
sys.exit(1)
else:
print usage
sys.exit(0)
#***end handling command line arguments**************************
botsinit.generalinit(
configdir) #find locating of bots, configfiles, init paths etc.
if not botsglobal.ini.getboolean('jobqueue', 'enabled', False):
print 'Error: bots jobqueue cannot start; not enabled in %s/bots.ini' % configdir
sys.exit(1)
process_name = 'dirmonitor'
logger = botsinit.initserverlogging(process_name)
logger.log(25, u'Bots %(process_name)s started.',
{'process_name': process_name})
logger.log(
25, u'Bots %(process_name)s configdir: "%(configdir)s".', {
'process_name': process_name,
'configdir': botsglobal.ini.get('directories', 'config')
})
botsenginepath = os.path.join(os.path.dirname(os.path.abspath(
sys.argv[0])), 'bots-engine.py') #get path to bots-engine
cond = threading.Condition()
tasks = set()
dir_watch_data = []
for section in botsglobal.ini.sections():
if section.startswith('dirmonitor') and section[len('dirmonitor'):]:
dir_watch_data.append({})
dir_watch_data[-1]['path'] = botsglobal.ini.get(section, 'path')
dir_watch_data[-1]['rec'] = botsglobal.ini.getboolean(
section, 'recursive', False)
dir_watch_data[-1]['filemask'] = botsglobal.ini.get(
section, 'filemask', '*')
dir_watch_data[-1]['route'] = botsglobal.ini.get(
section, 'route', '')
if not dir_watch_data:
logger.error(u'Nothing to watch!')
sys.exit(0)
if os.name == 'nt':
#for windows: start a thread per directory watcher
for dir_watch in dir_watch_data:
dir_watch_thread = threading.Thread(target=windows_event_handler,
args=(logger, dir_watch, cond,
tasks))
dir_watch_thread.daemon = True #do not wait for thread when exiting
dir_watch_thread.start()
else:
#for linux: one watch-thread, but multiple watches.
dir_watch_thread = threading.Thread(target=linux_event_handler,
args=(logger, dir_watch_data, cond,
tasks))
dir_watch_thread.daemon = True #do not wait for thread when exiting
dir_watch_thread.start()
# this main thread get the results from the watch-thread(s).
logger.info(u'Bots %(process_name)s started.',
{'process_name': process_name})
active_receiving = False
timeout = 2.0
cond.acquire()
while True:
#this functions as a buffer: all events go into set tasks.
#the tasks are fired to jobqueue after TIMOUT sec.
#this is to avoid firing to many tasks to jobqueue; events typically come in bursts.
#is value of timeout is larger, reaction times are slower...but less tasks are fired to jobqueue.
#in itself this is not a problem, as jobqueue will alos discard duplicate jobs.
#2 sec seems to e a good value: reasonable quick, not to nervous.
cond.wait(
timeout=timeout) #get back when results, or after timeout sec
if tasks:
if not active_receiving: #first request (after tasks have been fired, or startup of dirmonitor)
active_receiving = True
last_time = time.time()
else: #active receiving events
current_time = time.time()
if current_time - last_time >= timeout: #cond.wait returned probably because of a timeout
try:
for task in tasks:
logger.info(
u'Send to queue "%(path)s %(config)s %(task)s".',
{
'path': botsenginepath,
'config': '-c' + configdir,
'task': task
})
job2queue.send_job_to_jobqueue([
sys.executable, botsenginepath,
'-c' + configdir, task
])
except Exception as msg:
logger.info(u'Error in running task: "%(msg)s".',
{'msg': msg})
tasks.clear()
active_receiving = False
else: #cond.wait returned probably because of a timeout
logger.debug(u'time difference to small.')
last_time = current_time
cond.release()
sys.exit(0)
def start():
#NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
#***command line arguments**************************
usage = '''
This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
The %(name)s is the web server for bots; the interface (bots-monitor) can be accessed in a
browser, eg 'http://localhost:8080'.
Usage:
%(name)s -c<directory>
Options:
-c<directory> directory for configuration files (default: config).
''' % {
'name': os.path.basename(sys.argv[0]),
'version': botsglobal.version
}
configdir = 'config'
for arg in sys.argv[1:]:
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Error: configuration directory indicated, but no directory name.'
sys.exit(1)
else:
print usage
sys.exit(0)
#***end handling command line arguments**************************
botsinit.generalinit(
configdir) #find locating of bots, configfiles, init paths etc.
process_name = 'webserver'
botsglobal.logger = botsinit.initserverlogging(
process_name
) #initialise file-logging for web-server. This logging only contains the logging from bots-webserver, not from cherrypy.
#***init cherrypy as webserver*********************************************
#global configuration for cherrypy
cherrypy.config.update({
'global': {
'log.screen':
False,
'server.environment':
botsglobal.ini.get('webserver', 'environment', 'production')
}
})
#cherrypy handling of static files
conf = {
'/': {
'tools.staticdir.on': True,
'tools.staticdir.dir': 'media',
'tools.staticdir.root': botsglobal.ini.get('directories',
'botspath')
}
}
servestaticfiles = cherrypy.tree.mount(
None, '/media', conf
) #None: no cherrypy application (as this only serves static files)
#cherrypy handling of django
servedjango = WSGIHandler(
) #was: servedjango = AdminMediaHandler(WSGIHandler()) but django does not need the AdminMediaHandler in this setup. is much faster.
#cherrypy uses a dispatcher in order to handle the serving of static files and django.
dispatcher = wsgiserver.WSGIPathInfoDispatcher({
'/': servedjango,
'/media': servestaticfiles
})
botswebserver = wsgiserver.CherryPyWSGIServer(
bind_addr=('0.0.0.0', botsglobal.ini.getint('webserver', 'port',
8080)),
wsgi_app=dispatcher,
server_name=botsglobal.ini.get('webserver', 'name', 'bots-webserver'))
botsglobal.logger.log(25, _(u'Bots %(process_name)s started.'),
{'process_name': process_name})
botsglobal.logger.log(
25, _(u'Bots %(process_name)s configdir: "%(configdir)s".'), {
'process_name': process_name,
'configdir': botsglobal.ini.get('directories', 'config')
})
botsglobal.logger.log(
25, _(u'Bots %(process_name)s serving at port: "%(port)s".'), {
'process_name': process_name,
'port': botsglobal.ini.getint('webserver', 'port', 8080)
})
#handle ssl: cherrypy < 3.2 always uses pyOpenssl. cherrypy >= 3.2 uses python buildin ssl (python >= 2.6 has buildin support for ssl).
ssl_certificate = botsglobal.ini.get('webserver', 'ssl_certificate', None)
ssl_private_key = botsglobal.ini.get('webserver', 'ssl_private_key', None)
if ssl_certificate and ssl_private_key:
if cherrypy.__version__ >= '3.2.0':
adapter_class = wsgiserver.get_ssl_adapter_class('builtin')
botswebserver.ssl_adapter = adapter_class(ssl_certificate,
ssl_private_key)
else:
#but: pyOpenssl should be there!
botswebserver.ssl_certificate = ssl_certificate
botswebserver.ssl_private_key = ssl_private_key
botsglobal.logger.log(25,
_(u'Bots %(process_name)s uses ssl (https).'),
{'process_name': process_name})
else:
botsglobal.logger.log(
25, _(u'Bots %(process_name)s uses plain http (no ssl).'),
{'process_name': process_name})
#***start the cherrypy webserver.************************************************
try:
botswebserver.start()
except KeyboardInterrupt:
botswebserver.stop()
def start(configdir):
botsinit.generalinit(configdir) #find locating of bots, configfiles, init paths etc.
process_name = 'apache_webserver_' + configdir
botsglobal.logger = botsinit.initserverlogging(process_name) #initialise file-logging for web-server. This logging only contains the logging from bots-webserver.
def start():
# NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
# ***command line arguments**************************
usage = """
This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
A utility to generate the index file of a plugin; this can be seen as a database dump of the configuration.
This is eg useful for version control.
Usage:
%(name)s -c<directory>
Options:
-c<directory> directory for configuration files (default: config).
""" % {
"name": os.path.basename(sys.argv[0]),
"version": botsglobal.version,
}
configdir = "config"
for arg in sys.argv[1:]:
if arg.startswith("-c"):
configdir = arg[2:]
if not configdir:
print "Error: configuration directory indicated, but no directory name."
sys.exit(1)
else:
print usage
sys.exit(0)
# ***end handling command line arguments**************************
botsinit.generalinit(configdir) # find locating of bots, configfiles, init paths etc.
if not botsglobal.ini.getboolean("jobqueue", "enabled", False):
print "Error: bots jobqueue cannot start; not enabled in %s/bots.ini" % configdir
sys.exit(1)
process_name = "dirmonitor"
logger = botsinit.initserverlogging(process_name)
logger.log(25, u"Bots %(process_name)s started.", {"process_name": process_name})
logger.log(
25,
u'Bots %(process_name)s configdir: "%(configdir)s".',
{"process_name": process_name, "configdir": botsglobal.ini.get("directories", "config")},
)
botsenginepath = os.path.join(
os.path.dirname(os.path.abspath(sys.argv[0])), "bots-engine.py"
) # get path to bots-engine
cond = threading.Condition()
tasks = set()
dir_watch_data = []
for section in botsglobal.ini.sections():
if section.startswith("dirmonitor") and section[len("dirmonitor") :]:
dir_watch_data.append({})
dir_watch_data[-1]["path"] = botsglobal.ini.get(section, "path")
dir_watch_data[-1]["rec"] = botsglobal.ini.getboolean(section, "recursive", False)
dir_watch_data[-1]["filemask"] = botsglobal.ini.get(section, "filemask", "*")
dir_watch_data[-1]["route"] = botsglobal.ini.get(section, "route", "")
if not dir_watch_data:
logger.error(u"Nothing to watch!")
sys.exit(0)
if os.name == "nt":
# for windows: start a thread per directory watcher
for dir_watch in dir_watch_data:
dir_watch_thread = threading.Thread(target=windows_event_handler, args=(logger, dir_watch, cond, tasks))
dir_watch_thread.daemon = True # do not wait for thread when exiting
dir_watch_thread.start()
else:
# for linux: one watch-thread, but multiple watches.
dir_watch_thread = threading.Thread(target=linux_event_handler, args=(logger, dir_watch_data, cond, tasks))
dir_watch_thread.daemon = True # do not wait for thread when exiting
dir_watch_thread.start()
# this main thread get the results from the watch-thread(s).
logger.info(u"Bots %(process_name)s started.", {"process_name": process_name})
active_receiving = False
timeout = 2.0
cond.acquire()
while True:
# this functions as a buffer: all events go into set tasks.
# the tasks are fired to jobqueue after TIMOUT sec.
# this is to avoid firing to many tasks to jobqueue; events typically come in bursts.
# is value of timeout is larger, reaction times are slower...but less tasks are fired to jobqueue.
# in itself this is not a problem, as jobqueue will alos discard duplicate jobs.
# 2 sec seems to e a good value: reasonable quick, not to nervous.
cond.wait(timeout=timeout) # get back when results, or after timeout sec
if tasks:
if not active_receiving: # first request (after tasks have been fired, or startup of dirmonitor)
active_receiving = True
last_time = time.time()
else: # active receiving events
current_time = time.time()
if current_time - last_time >= timeout: # cond.wait returned probably because of a timeout
try:
for task in tasks:
logger.info(
u'Send to queue "%(path)s %(config)s %(task)s".',
{"path": botsenginepath, "config": "-c" + configdir, "task": task},
)
job2queue.send_job_to_jobqueue([sys.executable, botsenginepath, "-c" + configdir, task])
except Exception as msg:
logger.info(u'Error in running task: "%(msg)s".', {"msg": msg})
tasks.clear()
active_receiving = False
else: # cond.wait returned probably because of a timeout
logger.debug(u"time difference to small.")
last_time = current_time
cond.release()
sys.exit(0)
def start():
#NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
#***command line arguments**************************
usage = '''
This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
The %(name)s is the web server for bots; the interface (bots-monitor) can be accessed in a
browser, eg 'http://localhost:8080'.
Usage:
%(name)s -c<directory>
Options:
-c<directory> directory for configuration files (default: config).
'''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version}
configdir = 'config'
for arg in sys.argv[1:]:
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Error: configuration directory indicated, but no directory name.'
sys.exit(1)
else:
print usage
sys.exit(0)
#***end handling command line arguments**************************
botsinit.generalinit(configdir) #find locating of bots, configfiles, init paths etc.
process_name = 'webserver'
botsglobal.logger = botsinit.initserverlogging(process_name) #initialise file-logging for web-server. This logging only contains the logging from bots-webserver, not from cherrypy.
#***init cherrypy as webserver*********************************************
#global configuration for cherrypy
cherrypy.config.update({'global': {'log.screen': False, 'server.environment': botsglobal.ini.get('webserver','environment','production')}})
#cherrypy handling of static files
conf = {'/': {'tools.staticdir.on' : True,'tools.staticdir.dir' : 'media' ,'tools.staticdir.root': botsglobal.ini.get('directories','botspath')}}
servestaticfiles = cherrypy.tree.mount(None, '/media', conf) #None: no cherrypy application (as this only serves static files)
#cherrypy handling of django
servedjango = WSGIHandler() #was: servedjango = AdminMediaHandler(WSGIHandler()) but django does not need the AdminMediaHandler in this setup. is much faster.
#cherrypy uses a dispatcher in order to handle the serving of static files and django.
dispatcher = wsgiserver.WSGIPathInfoDispatcher({'/': servedjango, '/media': servestaticfiles})
botswebserver = wsgiserver.CherryPyWSGIServer(bind_addr=('0.0.0.0', botsglobal.ini.getint('webserver','port',8080)), wsgi_app=dispatcher, server_name=botsglobal.ini.get('webserver','name','bots-webserver'))
botsglobal.logger.log(25,_(u'Bots %(process_name)s started.'),
{'process_name':process_name})
botsglobal.logger.log(25,_(u'Bots %(process_name)s configdir: "%(configdir)s".'),
{'process_name':process_name, 'configdir':botsglobal.ini.get('directories','config')})
botsglobal.logger.log(25,_(u'Bots %(process_name)s serving at port: "%(port)s".'),
{'process_name':process_name,'port':botsglobal.ini.getint('webserver','port',8080)})
#handle ssl: cherrypy < 3.2 always uses pyOpenssl. cherrypy >= 3.2 uses python buildin ssl (python >= 2.6 has buildin support for ssl).
ssl_certificate = botsglobal.ini.get('webserver','ssl_certificate',None)
ssl_private_key = botsglobal.ini.get('webserver','ssl_private_key',None)
if ssl_certificate and ssl_private_key:
if cherrypy.__version__ >= '3.2.0':
adapter_class = wsgiserver.get_ssl_adapter_class('builtin')
botswebserver.ssl_adapter = adapter_class(ssl_certificate,ssl_private_key)
else:
#but: pyOpenssl should be there!
botswebserver.ssl_certificate = ssl_certificate
botswebserver.ssl_private_key = ssl_private_key
botsglobal.logger.log(25,_(u'Bots %(process_name)s uses ssl (https).'),{'process_name':process_name})
else:
botsglobal.logger.log(25,_(u'Bots %(process_name)s uses plain http (no ssl).'),{'process_name':process_name})
#***start the cherrypy webserver.************************************************
try:
botswebserver.start()
except KeyboardInterrupt:
botswebserver.stop()
