def _cleandatafile():
''' delete all data files older than xx days.'''
vanaf = time.time() - (botsglobal.ini.getint('settings','maxdays',30) * 3600 * 24)
frompath = botslib.join(botsglobal.ini.get('directories','data','botssys/data'),'*')
for filename in glob.iglob(frompath):
statinfo = os.stat(filename)
if not stat.S_ISDIR(statinfo.st_mode):
try:
os.remove(filename) #remove files - should be no files in root of data dir
except:
botsglobal.logger.exception(_(u'Cleanup could not remove file'))
elif statinfo.st_mtime > vanaf :
continue #directory is newer than maxdays, which is also true for the data files in it. Skip it.
else: #check files in dir and remove all older than maxdays
frompath2 = botslib.join(filename,'*')
emptydir = True #track check if directory is empty after loop (should directory itself be deleted/)
for filename2 in glob.iglob(frompath2):
statinfo2 = os.stat(filename2)
if statinfo2.st_mtime > vanaf or stat.S_ISDIR(statinfo2.st_mode): #check files in dir and remove all older than maxdays
emptydir = False
else:
try:
os.remove(filename2)
except:
botsglobal.logger.exception(_(u'Cleanup could not remove file'))
if emptydir:
try:
os.rmdir(filename)
except:
botsglobal.logger.exception(_(u'Cleanup could not remove directory'))
def _cleandatafile():
''' delete all data files older than xx days.'''
vanaf = time.time() - (botsglobal.ini.getint('settings','maxdays',30) * 3600 * 24)
frompath = botslib.join(botsglobal.ini.get('directories','data','botssys/data'),'*')
for filename in glob.iglob(frompath):
statinfo = os.stat(filename)
if not stat.S_ISDIR(statinfo.st_mode):
try:
os.remove(filename) #remove files - should be no files in root of data dir
except:
botsglobal.logger.exception(_(u'Cleanup could not remove file'))
elif statinfo.st_mtime > vanaf :
continue #directory is newer than maxdays, which is also true for the data files in it. Skip it.
else: #check files in dir and remove all older than maxdays
frompath2 = botslib.join(filename,'*')
emptydir = True #track check if directory is empty after loop (should directory itself be deleted/)
for filename2 in glob.iglob(frompath2):
statinfo2 = os.stat(filename2)
if statinfo2.st_mtime > vanaf or stat.S_ISDIR(statinfo2.st_mode): #check files in dir and remove all older than maxdays
emptydir = False
else:
try:
os.remove(filename2)
except:
botsglobal.logger.exception(_(u'Cleanup could not remove file'))
if emptydir:
try:
os.rmdir(filename)
except:
botsglobal.logger.exception(_(u'Cleanup could not remove directory'))
def abspathdata(filename):
''' abspathdata if filename incl dir: return absolute path; else (only filename): return absolute path (datadir).
for engine2 the current abspathdata is overwritten, as this uses subdirectories.
'''
if '/' in filename: #if filename already contains path
return botslib.join(filename)
else:
return botslib.join(data_storage, filename)
def abspathdata(filename):
''' abspathdata if filename incl dir: return absolute path; else (only filename): return absolute path (datadir).
for engine2 the current abspathdata is overwritten, as this uses subdirectories.
'''
if '/' in filename: #if filename already contains path
return botslib.join(filename)
else:
return botslib.join(data_storage,filename)
def _cleanarchive():
''' delete all archive directories older than maxdaysarchive days.'''
vanaf = (datetime.date.today()-datetime.timedelta(days=botsglobal.ini.getint('settings','maxdaysarchive',180))).strftime('%Y%m%d')
for row in botslib.query('''SELECT archivepath FROM channel '''):
if row['archivepath']:
vanafdir = botslib.join(row['archivepath'],vanaf)
for dir in glob.glob(botslib.join(row['archivepath'],'*')):
if dir < vanafdir:
shutil.rmtree(dir,ignore_errors=True)
def _cleanarchive():
''' delete all archive directories older than maxdaysarchive days.'''
vanaf = (datetime.date.today()-datetime.timedelta(days=botsglobal.ini.getint('settings','maxdaysarchive',180))).strftime('%Y%m%d')
for row in botslib.query('''SELECT archivepath FROM channel WHERE archivepath != '' '''):
vanafdir = botslib.join(row['archivepath'],vanaf)
for entry in glob.iglob(botslib.join(row['archivepath'],'*')):
if entry < vanafdir:
if entry.endswith('.zip'):
os.remove(entry)
else:
shutil.rmtree(entry,ignore_errors=True)
def delete(request, *kw, **kwargs):
if request.method == 'GET':
if 'transactions' in request.GET:
#while testing with very big loads, deleting transaction when wrong. Using raw SQL solved this.
from django.db import connection, transaction
cursor = connection.cursor()
cursor.execute("DELETE FROM ta")
cursor.execute("DELETE FROM filereport")
cursor.execute("DELETE FROM report")
#~ models.ta.objects.all().delete()
#~ models.filereport.objects.all().delete()
#~ models.report.objects.all().delete()
transaction.commit_unless_managed()
request.user.message_set.create(
message=_(u'All transactions are deleted.'))
try:
frompath = botslib.join(
botsglobal.ini.get('directories', 'data', 'botssys/data'),
'*')
for filename in glob.glob(frompath):
if os.path.isdir(filename):
frompath2 = botslib.join(filename, '*')
emptydir = True
for filename2 in glob.glob(frompath2):
if os.path.isdir(filename2):
emptydir = False
else:
os.remove(filename2)
if emptydir:
os.rmdir(filename)
else:
os.remove(filename2)
except:
pass
elif 'configuration' in request.GET:
models.confirmrule.objects.all().delete()
models.channel.objects.all().delete()
models.chanpar.objects.all().delete()
models.partner.objects.all().delete()
models.translate.objects.all().delete()
models.routes.objects.all().delete()
request.user.message_set.create(
message=_(u'All configuration is deleted.'))
elif 'codelists' in request.GET:
models.ccode.objects.all().delete()
models.ccodetrigger.objects.all().delete()
request.user.message_set.create(
message=_(u'All user code lists are deleted.'))
return django.shortcuts.redirect('/')
def write_outgoing(run):
outputdir = botslib.join(run.outpath)
botslib.dirshouldbethere(outputdir)
for outgoing in run.outgoing:
if not outgoing['error']:
try:
unique_filename = filename_formatter(run.outfilename, outgoing)
tofilepath = botslib.join(outputdir, unique_filename)
fromfilepath = botslib.abspathdata(outgoing['filename'])
shutil.move(fromfilepath, tofilepath)
except:
txt = botslib.txtexc()
outgoing.update({'error': txt})
else:
outgoing.update({'outfilename': tofilepath})
def write_outgoing(run):
outputdir = botslib.join(run.outpath)
botslib.dirshouldbethere(outputdir)
for outgoing in run.outgoing:
if not outgoing['error']:
try:
unique_filename = filename_formatter(run.outfilename,outgoing)
tofilepath = botslib.join(outputdir,unique_filename)
fromfilepath = botslib.abspathdata(outgoing['filename'])
shutil.move(fromfilepath,tofilepath)
except:
txt = botslib.txtexc()
outgoing.update({'error':txt})
else:
outgoing.update({'outfilename':tofilepath})
def initenginelogging():
convertini2logger={'DEBUG':logging.DEBUG,'INFO':logging.INFO,'WARNING':logging.WARNING,'ERROR':logging.ERROR,'CRITICAL':logging.CRITICAL}
# create main logger 'bots'
botsglobal.logger = logging.getLogger('bots')
botsglobal.logger.setLevel(logging.DEBUG)
# create rotating file handler
log_file = botslib.join(botsglobal.ini.get('directories','logging'),'engine.log')
rotatingfile = logging.handlers.RotatingFileHandler(log_file,backupCount=botsglobal.ini.getint('settings','log_file_number',10))
rotatingfile.setLevel(convertini2logger[botsglobal.ini.get('settings','log_file_level','ERROR')])
fileformat = logging.Formatter("%(asctime)s %(levelname)-8s %(name)s : %(message)s",'%Y%m%d %H:%M:%S')
rotatingfile.setFormatter(fileformat)
rotatingfile.doRollover() #each run a new log file is used; old one is rotated
# add rotating file handler to main logger
botsglobal.logger.addHandler(rotatingfile)
#logger for trace of mapping; tried to use filters but got this not to work.....
botsglobal.logmap = logging.getLogger('bots.map')
if not botsglobal.ini.getboolean('settings','mappingdebug',False):
botsglobal.logmap.setLevel(logging.CRITICAL)
#logger for reading edifile. is now used only very limited (1 place); is done with 'if'
#~ botsglobal.ini.getboolean('settings','readrecorddebug',False)
# create console handler
if botsglobal.ini.getboolean('settings','log_console',True):
console = logging.StreamHandler()
console.setLevel(logging.INFO)
consuleformat = logging.Formatter("%(levelname)-8s %(message)s")
console.setFormatter(consuleformat) # add formatter to console
botsglobal.logger.addHandler(console) # add console to logger
def plugout(request, *kw, **kwargs):
if request.method == 'GET':
form = forms.PlugoutForm()
return django.shortcuts.render(request, 'bots/plugout.html',
{'form': form})
else:
if 'submit' in request.POST:
form = forms.PlugoutForm(request.POST)
if form.is_valid():
filename = botslib.join(
botsglobal.ini.get('directories', 'botssys'),
'plugin_temp.zip')
botsglobal.logger.info(_(u'Start writing plugin "%(file)s".'),
{'file': filename})
try:
pluglib.make_plugin(form.cleaned_data, filename)
except botslib.PluginError as msg:
botsglobal.logger.error(unicode(msg))
messages.add_message(request, messages.INFO, unicode(msg))
else:
botsglobal.logger.info(
_(u'Plugin "%(file)s" created successful.'),
{'file': filename})
response = django.http.HttpResponse(
open(filename, 'rb').read(),
content_type='application/zip')
# response['Content-Length'] = os.path.getsize(filename)
response[
'Content-Disposition'] = 'attachment; filename=' + 'plugin' + time.strftime(
'_%Y%m%d') + '.zip'
return response
return django.shortcuts.redirect('/home')
def plugout_index(request, *kw, **kwargs):
if request.method == 'GET':
filename = botslib.join(
botsglobal.ini.get('directories', 'usersysabs'), 'index.py')
botsglobal.logger.info(
_(u'Start writing configuration index file "%(file)s".'),
{'file': filename})
try:
dummy_for_cleaned_data = {
'databaseconfiguration':
True,
'umlists':
botsglobal.ini.getboolean('settings', 'codelists_in_plugin',
True),
'databasetransactions':
False
}
pluglib.make_index(dummy_for_cleaned_data, filename)
except Exception as msg:
notification = _(u'Error writing configuration index file: "%s".'
) % unicode(msg)
botsglobal.logger.error(notification)
messages.add_message(request, messages.INFO, notification)
else:
notification = _(
u'Configuration index file "%s" is written successful.'
) % filename
botsglobal.logger.info(notification)
messages.add_message(request, messages.INFO, notification)
return django.shortcuts.redirect('/home')
def start():
#NOTE bots is always on PYTHONPATH!!! - otherwise it will not start.
#***command line arguments**************************
configdir = 'config'
for arg in sys.argv[1:]:
if not arg:
continue
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Configuration directory indicated, but no directory name.'
sys.exit(1)
elif arg in ["?", "/?"] or arg.startswith('-'):
showusage()
else:
showusage()
#***init general: find locating of bots, configfiles, init paths etc.***********************
botsinit.generalinit(configdir)
#***initialise logging. This logging only contains the logging from bots-webserver, not from cherrypy.
botsglobal.logger = logging.getLogger('bots-webserver')
botsglobal.logger.setLevel(logging.DEBUG)
h = TimedRotatingFileHandler(botslib.join(botsglobal.ini.get('directories','logging'),'webserver.log'), backupCount=10)
fileformat = logging.Formatter("%(asctime)s %(levelname)-8s: %(message)s",'%Y%m%d %H:%M:%S')
h.setFormatter(fileformat)
botsglobal.logger.addHandler(h)
#***init cherrypy as webserver*********************************************
#global configuration for cherrypy
cherrypy.config.update({'global': {'log.screen': False, 'server.environment': botsglobal.ini.get('webserver','environment','production')}})
#cherrypy handling of static files
conf = {'/': {'tools.staticdir.on' : True,'tools.staticdir.dir' : 'media' ,'tools.staticdir.root': botsglobal.ini.get('directories','botspath')}}
servestaticfiles = cherrypy.tree.mount(None, '/media', conf) #None: no cherrypy application (as this only serves static files)
#cherrypy handling of django
servedjango = WSGIHandler() #was: servedjango = AdminMediaHandler(WSGIHandler()) but django does not need the AdminMediaHandler in this setup. is much faster.
#cherrypy uses a dispatcher in order to handle the serving of static files and django.
dispatcher = wsgiserver.WSGIPathInfoDispatcher({'/': servedjango, '/media': servestaticfiles})
botswebserver = wsgiserver.CherryPyWSGIServer(bind_addr=('0.0.0.0', botsglobal.ini.getint('webserver','port',8080)), wsgi_app=dispatcher, server_name=botsglobal.ini.get('webserver','name','bots-webserver'))
botsglobal.logger.info(_(u'Bots web-server started.'))
#handle ssl: cherrypy < 3.2 always uses pyOpenssl. cherrypy >= 3.2 uses python buildin ssl (python >= 2.6 has buildin support for ssl).
ssl_certificate = botsglobal.ini.get('webserver','ssl_certificate',None)
ssl_private_key = botsglobal.ini.get('webserver','ssl_private_key',None)
if ssl_certificate and ssl_private_key:
if cherrypy.__version__ >= '3.2.0':
adapter_class = wsgiserver.get_ssl_adapter_class('builtin')
botswebserver.ssl_adapter = adapter_class(ssl_certificate,ssl_private_key)
else:
#but: pyOpenssl should be there!
botswebserver.ssl_certificate = ssl_certificate
botswebserver.ssl_private_key = ssl_private_key
botsglobal.logger.info(_(u'Bots web-server uses ssl (https).'))
else:
botsglobal.logger.info(_(u'Bots web-server uses plain http (no ssl).'))
#***start the cherrypy webserver.
try:
botswebserver.start()
except KeyboardInterrupt:
botswebserver.stop()
def initenginelogging(logname):
#initialise file logging: create main logger 'bots'
logger = logging.getLogger(logname)
logger.setLevel(convertini2logger[botsglobal.ini.get(
'settings', 'log_file_level', 'INFO')])
handler = logging.handlers.RotatingFileHandler(
botslib.join(botsglobal.ini.get('directories', 'logging'),
logname + '.log'),
backupCount=botsglobal.ini.getint('settings', 'log_file_number', 10))
fileformat = logging.Formatter(
"%(asctime)s %(levelname)-8s %(name)s : %(message)s",
'%Y%m%d %H:%M:%S')
handler.setFormatter(fileformat)
handler.doRollover() #each run a new log file is used; old one is rotated
logger.addHandler(handler)
#initialise file logging: logger for trace of mapping; tried to use filters but got this not to work.....
botsglobal.logmap = logging.getLogger('engine.map')
if not botsglobal.ini.getboolean('settings', 'mappingdebug', False):
botsglobal.logmap.setLevel(logging.CRITICAL)
#logger for reading edifile. is now used only very limited (1 place); is done with 'if'
#~ botsglobal.ini.getboolean('settings','readrecorddebug',False)
# initialise console/screen logging
if botsglobal.ini.getboolean('settings', 'log_console', True):
console = logging.StreamHandler()
console.setLevel(logging.INFO)
consuleformat = logging.Formatter("%(levelname)-8s %(message)s")
console.setFormatter(consuleformat) # add formatter to console
logger.addHandler(console) # add console to logger
return logger
def delete(request, *kw, **kwargs):
if request.method == "GET":
if "transactions" in request.GET:
# while testing with very big loads, deleting transaction when wrong. Using raw SQL solved this.
from django.db import connection, transaction
cursor = connection.cursor()
cursor.execute("DELETE FROM ta")
cursor.execute("DELETE FROM filereport")
cursor.execute("DELETE FROM report")
# ~ models.ta.objects.all().delete()
# ~ models.filereport.objects.all().delete()
# ~ models.report.objects.all().delete()
transaction.commit_unless_managed()
request.user.message_set.create(message=_(u"All transactions are deleted."))
try:
frompath = botslib.join(botsglobal.ini.get("directories", "data", "botssys/data"), "*")
for filename in glob.glob(frompath):
if os.path.isdir(filename):
frompath2 = botslib.join(filename, "*")
emptydir = True
for filename2 in glob.glob(frompath2):
if os.path.isdir(filename2):
emptydir = False
else:
os.remove(filename2)
if emptydir:
os.rmdir(filename)
else:
os.remove(filename2)
except:
pass
elif "configuration" in request.GET:
models.confirmrule.objects.all().delete()
models.channel.objects.all().delete()
models.chanpar.objects.all().delete()
models.partner.objects.all().delete()
models.translate.objects.all().delete()
models.routes.objects.all().delete()
request.user.message_set.create(message=_(u"All configuration is deleted."))
elif "codelists" in request.GET:
models.ccode.objects.all().delete()
models.ccodetrigger.objects.all().delete()
request.user.message_set.create(message=_(u"All user code lists are deleted."))
return django.shortcuts.redirect("/")
def _cleanarchive():
''' delete all archive directories older than maxdaysarchive days. Errors are ignored.'''
vanaf_default = (datetime.date.today()-datetime.timedelta(days=botsglobal.ini.getint('settings','maxdaysarchive',180))).strftime('%Y%m%d')
for row in botslib.query('''SELECT archivepath,rsrv3 FROM channel WHERE archivepath != '' '''):
if row['rsrv3']:
vanaf = (datetime.date.today()-datetime.timedelta(days=row['rsrv3'])).strftime('%Y%m%d')
else:
vanaf = vanaf_default
vanafdir = botslib.join(row['archivepath'],vanaf)
for entry in glob.iglob(botslib.join(row['archivepath'],'*')):
if entry < vanafdir:
if entry.endswith('.zip'):
try:
os.remove(entry)
except:
pass
else:
shutil.rmtree(entry,ignore_errors=True)
def plugout(request, *kw, **kwargs):
if request.method == "GET":
filename = botslib.join(botsglobal.ini.get("directories", "botssys"), request.GET["function"])
# ~ filename = os.path.abspath(request.GET['function'])
botsglobal.logger.info(_(u'Start writing plugin "%s".'), filename)
try:
pluglib.dump(filename, request.GET["function"])
except botslib.PluginError, txt:
botsglobal.logger.info(u"%s", str(txt))
request.user.message_set.create(message="%s" % txt)
else:
botsglobal.logger.info(_(u'Plugin "%s" created succesful.'), filename)
request.user.message_set.create(message=_(u"Plugin %s created succesful.") % filename)
def read_incoming(run):
outputdir = botslib.join(run.inpath,run.infilename)
filelist = [filename for filename in glob.iglob(outputdir) if os.path.isfile(filename)]
filelist.sort()
for infilename in filelist:
try:
filename = transform.unique('bots_file_name')
abs_filename = botslib.abspathdata(filename)
shutil.copy(infilename,abs_filename) #move if to be delted
except:
txt = botslib.txtexc()
else:
txt = '' #no errors
finally:
run.incoming.append({'infilename':infilename,'filename':filename,'error':txt,'editype':run.translation['editype'],'messagetype':run.translation['messagetype']})
def plugout_index(request,*kw,**kwargs):
if request.method == 'GET':
filename = botslib.join(botsglobal.ini.get('directories','usersysabs'),'index.py')
botsglobal.logger.info(_(u'Start writing configuration index file "%s".'),filename)
try:
dummy_for_cleaned_data = {'databaseconfiguration':True,'umlists':True,'databasetransactions':False}
pluglib.make_index(dummy_for_cleaned_data,filename)
except Exception,msg:
notification = _(u'Error writing configuration index file: "%s".')%str(msg)
botsglobal.logger.error(notification)
messages.add_message(request, messages.INFO, notification)
else:
notification = _(u'Configuration index file "%s" is written successful.')%filename
botsglobal.logger.info(notification)
messages.add_message(request, messages.INFO, notification)
return django.shortcuts.redirect('/home')
def plugout(request, *kw, **kwargs):
if request.method == 'GET':
filename = botslib.join(botsglobal.ini.get('directories', 'botssys'),
request.GET['function'])
#~ filename = os.path.abspath(request.GET['function'])
botsglobal.logger.info(_(u'Start writing plugin "%s".'), filename)
try:
pluglib.dump(filename, request.GET['function'])
except botslib.PluginError, txt:
botsglobal.logger.info(u'%s', str(txt))
request.user.message_set.create(message='%s' % txt)
else:
botsglobal.logger.info(_(u'Plugin "%s" created succesful.'),
filename)
request.user.message_set.create(
message=_(u'Plugin %s created succesful.') % filename)
def initenginelogging():
convertini2logger = {
'DEBUG': logging.DEBUG,
'INFO': logging.INFO,
'WARNING': logging.WARNING,
'ERROR': logging.ERROR,
'CRITICAL': logging.CRITICAL
}
# create main logger 'bots'
botsglobal.logger = logging.getLogger('bots')
botsglobal.logger.setLevel(logging.DEBUG)
# create rotating file handler
log_file = botslib.join(botsglobal.ini.get('directories', 'logging'),
'engine.log')
rotatingfile = logging.handlers.RotatingFileHandler(
log_file,
backupCount=botsglobal.ini.getint('settings', 'log_file_number', 10))
rotatingfile.setLevel(convertini2logger[botsglobal.ini.get(
'settings', 'log_file_level', 'ERROR')])
fileformat = logging.Formatter(
"%(asctime)s %(levelname)-8s %(name)s : %(message)s",
'%Y%m%d %H:%M:%S')
rotatingfile.setFormatter(fileformat)
rotatingfile.doRollover(
) #each run a new log file is used; old one is rotated
# add rotating file handler to main logger
botsglobal.logger.addHandler(rotatingfile)
#logger for trace of mapping; tried to use filters but got this not to work.....
botsglobal.logmap = logging.getLogger('bots.map')
if not botsglobal.ini.getboolean('settings', 'mappingdebug', False):
botsglobal.logmap.setLevel(logging.CRITICAL)
#logger for reading edifile. is now used only very limited (1 place); is done with 'if'
#~ botsglobal.ini.getboolean('settings','readrecorddebug',False)
# create console handler
if botsglobal.ini.getboolean('settings', 'log_console', True):
console = logging.StreamHandler()
console.setLevel(logging.INFO)
consuleformat = logging.Formatter("%(levelname)-8s %(message)s")
console.setFormatter(consuleformat) # add formatter to console
botsglobal.logger.addHandler(console) # add console to logger
def plugout_backup_core(request,*kw,**kwargs):
filename = botslib.join(botsglobal.ini.get('directories','botssys'),'backup_plugin_%s.zip'%time.strftime('%Y%m%d%H%M%S'))
botsglobal.logger.info(_(u'Start writing backup plugin "%s".'),filename)
try:
dummy_for_cleaned_data = {'databaseconfiguration':True,
'umlists':True,
'fileconfiguration':True,
'infiles':False,
'charset':True,
'databasetransactions':False,
'data':False,
'logfiles':False,
'config':False,
'database':False,
}
pluglib.make_plugin(dummy_for_cleaned_data,filename)
except Exception,msg:
notification = u'Error writing backup plugin: "%s".' % str(msg)
botsglobal.logger.error(notification)
messages.add_message(request, messages.INFO, notification)
def plugout(request,*kw,**kwargs):
if request.method == 'GET':
form = forms.PlugoutForm()
return django.shortcuts.render_to_response('bots/plugout.html', {'form': form},context_instance=django.template.RequestContext(request))
else:
if 'submit' in request.POST:
form = forms.PlugoutForm(request.POST)
if form.is_valid():
filename = botslib.join(botsglobal.ini.get('directories','botssys'),'plugin_temp.zip')
botsglobal.logger.info(_(u'Start writing plugin "%s".'),filename)
try:
pluglib.make_plugin(form.cleaned_data,filename)
except botslib.PluginError, msg:
botsglobal.logger.error(str(msg))
messages.add_message(request,messages.INFO,str(msg))
else:
botsglobal.logger.info(_(u'Plugin "%s" created successful.'),filename)
response = django.http.HttpResponse(open(filename, 'rb').read(), content_type='application/zip')
# response['Content-Length'] = os.path.getsize(filename)
response['Content-Disposition'] = 'attachment; filename=' + 'plugin' + time.strftime('_%Y%m%d') + '.zip'
return response
def plugout_backup_core(request, *kw, **kwargs):
filename = botslib.join(
botsglobal.ini.get('directories', 'botssys'),
'backup_plugin_%s.zip' % time.strftime('%Y%m%d%H%M%S'))
botsglobal.logger.info(_(u'Start writing backup plugin "%(file)s".'),
{'file': filename})
try:
dummy_for_cleaned_data = {
'databaseconfiguration':
True,
'umlists':
botsglobal.ini.getboolean('settings', 'codelists_in_plugin', True),
'fileconfiguration':
True,
'infiles':
False,
'charset':
True,
'databasetransactions':
False,
'data':
False,
'logfiles':
False,
'config':
False,
'database':
False,
}
pluglib.make_plugin(dummy_for_cleaned_data, filename)
except Exception as msg:
notification = u'Error writing backup plugin: "%s".' % unicode(msg)
botsglobal.logger.error(notification)
messages.add_message(request, messages.INFO, notification)
else:
notification = _(
u'Backup plugin "%s" is written successful.') % filename
botsglobal.logger.info(notification)
messages.add_message(request, messages.INFO, notification)
def initenginelogging(logname):
#initialise file logging: create main logger 'bots'
logger = logging.getLogger(logname)
logger.setLevel(convertini2logger[botsglobal.ini.get('settings','log_file_level','INFO')])
handler = logging.handlers.RotatingFileHandler(botslib.join(botsglobal.ini.get('directories','logging'),logname+'.log'),backupCount=botsglobal.ini.getint('settings','log_file_number',10))
fileformat = logging.Formatter("%(asctime)s %(levelname)-8s %(name)s : %(message)s",'%Y%m%d %H:%M:%S')
handler.setFormatter(fileformat)
handler.doRollover() #each run a new log file is used; old one is rotated
logger.addHandler(handler)
#initialise file logging: logger for trace of mapping; tried to use filters but got this not to work.....
botsglobal.logmap = logging.getLogger('engine.map')
if not botsglobal.ini.getboolean('settings','mappingdebug',False):
botsglobal.logmap.setLevel(logging.CRITICAL)
#logger for reading edifile. is now used only very limited (1 place); is done with 'if'
#~ botsglobal.ini.getboolean('settings','readrecorddebug',False)
# initialise console/screen logging
if botsglobal.ini.getboolean('settings','log_console',True):
console = logging.StreamHandler()
console.setLevel(logging.INFO)
consuleformat = logging.Formatter("%(levelname)-8s %(message)s")
console.setFormatter(consuleformat) # add formatter to console
logger.addHandler(console) # add console to logger
return logger
def read_incoming(run):
outputdir = botslib.join(run.inpath, run.infilename)
filelist = [
filename for filename in glob.iglob(outputdir)
if os.path.isfile(filename)
]
filelist.sort()
for infilename in filelist:
try:
filename = transform.unique('bots_file_name')
abs_filename = botslib.abspathdata(filename)
shutil.copy(infilename, abs_filename) #move if to be delted
except:
txt = botslib.txtexc()
else:
txt = '' #no errors
finally:
run.incoming.append({
'infilename': infilename,
'filename': filename,
'error': txt,
'editype': run.translation['editype'],
'messagetype': run.translation['messagetype']
})
def plugout_backup_core(request,*kw,**kwargs):
filename = botslib.join(botsglobal.ini.get('directories','botssys'),'backup_plugin_%s.zip'%time.strftime('%Y%m%d%H%M%S'))
botsglobal.logger.info(_(u'Start writing backup plugin "%(file)s".'),{'file':filename})
try:
dummy_for_cleaned_data = {'databaseconfiguration':True,
'umlists':botsglobal.ini.getboolean('settings','codelists_in_plugin',True),
'fileconfiguration':True,
'infiles':False,
'charset':True,
'databasetransactions':False,
'data':False,
'logfiles':False,
'config':False,
'database':False,
}
pluglib.make_plugin(dummy_for_cleaned_data,filename)
except Exception as msg:
notification = u'Error writing backup plugin: "%s".'%unicode(msg)
botsglobal.logger.error(notification)
messages.add_message(request, messages.INFO, notification)
else:
notification = _(u'Backup plugin "%s" is written successful.')%filename
botsglobal.logger.info(notification)
messages.add_message(request, messages.INFO, notification)
def delete(request, *kw, **kwargs):
if request.method == 'GET':
form = forms.DeleteForm()
return django.shortcuts.render(request, 'bots/delete.html',
{'form': form})
else:
if 'submit' in request.POST:
form = forms.DeleteForm(request.POST)
if form.is_valid():
from django.db import connection, transaction
if form.cleaned_data['delconfiguration'] or form.cleaned_data[
'delcodelists'] or form.cleaned_data['deluserscripts']:
#write backup plugin first
plugout_backup_core(request, *kw, **kwargs)
botsglobal.logger.info(_(u'Start deleting in configuration.'))
if form.cleaned_data['deltransactions']:
#while testing with very big loads, deleting gave error. Using raw SQL solved this.
cursor = connection.cursor()
cursor.execute("DELETE FROM ta")
cursor.execute("DELETE FROM filereport")
cursor.execute("DELETE FROM report")
transaction.commit_unless_managed()
messages.add_message(request, messages.INFO,
_(u'Transactions are deleted.'))
botsglobal.logger.info(_(u'Transactions are deleted.'))
#clean data files
deletefrompath = botsglobal.ini.get(
'directories', 'data', 'botssys/data')
shutil.rmtree(deletefrompath, ignore_errors=True)
botslib.dirshouldbethere(deletefrompath)
notification = _(u'Data files are deleted.')
messages.add_message(request, messages.INFO, notification)
botsglobal.logger.info(notification)
elif form.cleaned_data['delacceptance']:
from django.db.models import Min
list_file = [
] #list of files for deletion in data-directory
report_idta_lowest = 0
for acc_report in models.report.objects.filter(
acceptance=1
): #for each acceptance report. is not very efficient.
if not report_idta_lowest:
report_idta_lowest = acc_report.idta
max_report_idta = models.report.objects.filter(
idta__gt=acc_report.idta).aggregate(Min('idta'))[
'idta__min'] #select 'next' report...
if not max_report_idta: #if report is report of last run, there is no next report
max_report_idta = sys.maxint
#we have a idta-range now: from (and including) acc_report.idta till (and excluding) max_report_idta
list_file += models.ta.objects.filter(
idta__gte=acc_report.idta,
idta__lt=max_report_idta).exclude(
status=1).values_list('filename',
flat=True).distinct()
models.ta.objects.filter(
idta__gte=acc_report.idta,
idta__lt=max_report_idta).delete(
) #delete ta in range
models.filereport.objects.filter(
idta__gte=acc_report.idta,
idta__lt=max_report_idta).delete(
) #delete filereports in range
if report_idta_lowest:
models.report.objects.filter(
idta__gte=report_idta_lowest, acceptance=1).delete(
) #delete all acceptance reports
for filename in list_file: #delete all files in data directory geenrated during acceptance testing
if filename.isdigit():
botslib.deldata(filename)
notification = _(
u'Transactions from acceptance-testing deleted.')
messages.add_message(request, messages.INFO, notification)
botsglobal.logger.info(notification)
if form.cleaned_data['delconfiguration']:
models.confirmrule.objects.all().delete()
models.routes.objects.all().delete()
models.channel.objects.all().delete()
models.chanpar.objects.all().delete()
models.translate.objects.all().delete()
models.partner.objects.all().delete()
notification = _(u'Database configuration is deleted.')
messages.add_message(request, messages.INFO, notification)
botsglobal.logger.info(notification)
if form.cleaned_data['delcodelists']:
#while testing with very big loads, deleting gave error. Using raw SQL solved this.
cursor = connection.cursor()
cursor.execute("DELETE FROM ccode")
cursor.execute("DELETE FROM ccodetrigger")
transaction.commit_unless_managed()
notification = _(u'User code lists are deleted.')
messages.add_message(request, messages.INFO, notification)
botsglobal.logger.info(notification)
if form.cleaned_data['delpersist']:
cursor = connection.cursor()
cursor.execute("DELETE FROM persist")
transaction.commit_unless_managed()
notification = _(u'Persist data is deleted.')
messages.add_message(request, messages.INFO, notification)
botsglobal.logger.info(notification)
if form.cleaned_data['delinfile']:
deletefrompath = botslib.join(
botsglobal.ini.get('directories', 'botssys',
'botssys'), 'infile')
shutil.rmtree(deletefrompath, ignore_errors=True)
notification = _(u'Files in botssys/infile are deleted.')
messages.add_message(request, messages.INFO, notification)
botsglobal.logger.info(notification)
if form.cleaned_data['deloutfile']:
deletefrompath = botslib.join(
botsglobal.ini.get('directories', 'botssys',
'botssys'), 'outfile')
shutil.rmtree(deletefrompath, ignore_errors=True)
notification = _(u'Files in botssys/outfile are deleted.')
messages.add_message(request, messages.INFO, notification)
botsglobal.logger.info(notification)
if form.cleaned_data['deluserscripts']:
deletefrompath = botsglobal.ini.get(
'directories', 'usersysabs')
for root, dirs, files in os.walk(deletefrompath):
head, tail = os.path.split(root)
if tail == 'charsets':
del dirs[:]
continue
for bestand in files:
if bestand != '__init__.py':
os.remove(os.path.join(root, bestand))
notification = _(u'User scripts are deleted (in usersys).')
messages.add_message(request, messages.INFO, notification)
botsglobal.logger.info(notification)
botsglobal.logger.info(
_(u'Finished deleting in configuration.'))
return django.shortcuts.redirect('/home')
def start():
#NOTE bots is always on PYTHONPATH!!! - otherwise it will not start.
#********command line arguments**************************
configdir = 'config'
for arg in sys.argv[1:]:
if not arg:
continue
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Configuration directory indicated, but no directory name.'
sys.exit(1)
elif arg in ["?", "/?"] or arg.startswith('-'):
showusage()
else:
showusage()
#init general: find locating of bots, configfiles, init paths etc.***********************
botsinit.generalinit(configdir)
#init cherrypy; only needed for webserver. *********************************************
cherrypy.config.update({'global': { 'tools.staticdir.root': botsglobal.ini.get('directories','botspath'),
'server.socket_host' : "0.0.0.0", #to what IP addresses should be server. 0.0.0.0: all. See cherrypy docs
'server.socket_port': botsglobal.ini.getint('webserver','port',8080),
'server.environment': botsglobal.ini.get('webserver','environment','development'), # development production
'log.screen': False,
#~ 'log.error_file': '', #set later to rotating log file
#~ 'log.access_file': '', #set later to rotating log file
}})
conf = {'/': {'tools.staticdir.on' : True,'tools.staticdir.dir' : 'media' }}
#~ '/favicon.ico': {'tools.staticfile.on': True,'tools.staticfile.filename': '/home/hje/botsup-django/bots/media/images/favicon.ico'}}
cherrypy.tree.graft(AdminMediaHandler(WSGIHandler()), '/')
myroot = Root()
myappl = cherrypy.tree.mount(myroot, '/media', conf) #myappl is needed to set logging
botsglobal.logger = logging.getLogger('webserver')
botsglobal.logger.setLevel(logging.DEBUG)
h = logging.handlers.TimedRotatingFileHandler(botslib.join(botsglobal.ini.get('directories','logging'),'webserver.log'),when='midnight', backupCount=10)
fileformat = logging.Formatter("%(asctime)s %(levelname)-8s %(name)s : %(message)s",'%Y%m%d %H:%M:%S')
h.setFormatter(fileformat)
# add rotating file handler to main logger
botsglobal.logger.addHandler(h)
# Make RotatingFileHandler for the error log.
#~ h = logging.handlers.TimedRotatingFileHandler(botslib.join(botsglobal.ini.get('directories','logging'),'webserver.log'),when='midnight', backupCount=10)
#~ fileformat = logging.Formatter("%(asctime)s %(levelname)-8s %(name)s : %(message)s",'%Y%m%d %H:%M:%S')
#~ h.setLevel(logging.INFO)
#~ h.setFormatter(fileformat)
#~ myappl.log.error_log.addHandler(h)
# MakeRotatingFileHandler for the access log.
#~ h = logging.handlers.TimedRotatingFileHandler(os.path.normpath(os.path.join(botsglobal.ini.get('directories','botspath'), 'botssys/logging/webserver.log')),when='midnight', backupCount=10)
#~ h.setLevel(logging.DEBUG)
#~ myappl.log.access_log.addHandler(h)
#~ botsglobal.logger = myappl.log.access_log
#write start info to cherrypy log********************************************
botsglobal.logger.info(_(u'Bots web server started.'))
botsglobal.logger.info(_(u'Python version: "%s".'),sys.version)
botsglobal.logger.info(_(u'Django version: "%s".'),django.VERSION)
#start cherrypy *********************************************************************
cherrypy.engine.start()
cherrypy.engine.block()
def generalinit(configdir):
##########################################################################
#Configdir: settings.py & bots.ini#########################################
#Configdir MUST be importable. So configdir is relative to PYTHONPATH. Try several options for this import.
try: #first check if is configdir outside bots-directory: import configdir.settings.py
importnameforsettings = os.path.normpath(
os.path.join(configdir, 'settings')).replace(os.sep, '.')
settings = botslib.botsbaseimport(importnameforsettings)
except ImportError: #normal: configdir is in bots directory: import bots.configdir.settings.py
try:
importnameforsettings = os.path.normpath(
os.path.join('bots', configdir,
'settings')).replace(os.sep, '.')
settings = botslib.botsbaseimport(importnameforsettings)
except ImportError: #set pythonpath to config directory first
if not os.path.exists(configdir): #check if configdir exists.
raise botslib.PanicError(
u'In initilisation: path to configuration does not exists: "%(configdir)s".',
{'configdir': configdir})
addtopythonpath = os.path.abspath(os.path.dirname(configdir))
moduletoimport = os.path.basename(configdir)
sys.path.append(addtopythonpath)
importnameforsettings = os.path.normpath(
os.path.join(moduletoimport, 'settings')).replace(os.sep, '.')
settings = botslib.botsbaseimport(importnameforsettings)
#settings is imported, so now we know where to find settings.py: importnameforsettings
#note: the imported settings.py itself is NOT used, this is doen via django.conf.settings
configdirectory = os.path.abspath(os.path.dirname(settings.__file__))
#Read configuration-file bots.ini.
botsglobal.ini = BotsConfig()
botsglobal.ini.read(os.path.join(configdirectory, 'bots.ini'))
# 'directories','botspath': absolute path for bots directory
botsglobal.ini.set('directories', 'botspath',
os.path.abspath(os.path.dirname(__file__)))
# 'directories','config': absolute path for config directory
botsglobal.ini.set('directories', 'config', configdirectory)
#set config as originally received; used in starting engine via bots-monitor
botsglobal.ini.set('directories', 'config_org', configdir)
############################################################################
#Usersys####################################################################
#usersys MUST be importable. So usersys is relative to PYTHONPATH. Try several options for this import.
usersys = botsglobal.ini.get('directories', 'usersys', 'usersys')
try: #usersys outside bots-directory: import usersys
importnameforusersys = os.path.normpath(usersys).replace(os.sep, '.')
importedusersys = botslib.botsbaseimport(importnameforusersys)
except ImportError: #usersys is in bots directory: import bots.usersys
try:
importnameforusersys = os.path.normpath(
os.path.join('bots', usersys)).replace(os.sep, '.')
importedusersys = botslib.botsbaseimport(importnameforusersys)
except ImportError: #set pythonpath to usersys directory first
if not os.path.exists(usersys): #check if configdir exists.
raise botslib.PanicError(
u'In initilisation: path to configuration does not exists: "%(usersys)s".',
{'usersys': usersys})
addtopythonpath = os.path.abspath(os.path.dirname(usersys)) #????
moduletoimport = os.path.basename(usersys)
sys.path.append(addtopythonpath)
importnameforusersys = os.path.normpath(usersys).replace(
os.sep, '.')
importedusersys = botslib.botsbaseimport(importnameforusersys)
# 'directories','usersysabs': absolute path for config usersysabs
botsglobal.ini.set('directories', 'usersysabs',
os.path.abspath(
os.path.dirname(importedusersys.__file__))
) #???Find pathname usersys using imported usersys
# botsglobal.usersysimportpath: used for imports from usersys
botsglobal.usersysimportpath = importnameforusersys
botsglobal.ini.set(
'directories', 'templatehtml',
botslib.join(botsglobal.ini.get('directories', 'usersysabs'),
'grammars/templatehtml/templates'))
############################################################################
#Botssys####################################################################
# 'directories','botssys': absolute path for config botssys
botssys = botsglobal.ini.get('directories', 'botssys', 'botssys')
botsglobal.ini.set('directories', 'botssys_org',
botssys) #store original botssys setting
botsglobal.ini.set('directories', 'botssys',
botslib.join(botssys)) #use absolute path
botsglobal.ini.set('directories', 'data', botslib.join(botssys, 'data'))
botsglobal.ini.set('directories', 'logging',
botslib.join(botssys, 'logging'))
############################################################################
#other inits##############################################################
if botsglobal.ini.get(
'webserver', 'environment', 'development'
) != 'development': #values in bots.ini are also used in setting up cherrypy
logging.raiseExceptions = 0 # during production: if errors occurs in writing to log: ignore error. (leads to a missing log line, better than error;-).
botslib.dirshouldbethere(botsglobal.ini.get('directories', 'data'))
botslib.dirshouldbethere(botsglobal.ini.get('directories', 'logging'))
initbotscharsets() #initialise bots charsets
node.Node.checklevel = botsglobal.ini.getint('settings', 'get_checklevel',
1)
botslib.settimeout(botsglobal.ini.getint('settings', 'globaltimeout', 10))
############################################################################
#Init django#################################################################################
os.environ['DJANGO_SETTINGS_MODULE'] = importnameforsettings
import django
if hasattr(django, 'setup'):
django.setup()
from django.conf import settings
botsglobal.settings = settings #settings are accessed using botsglobal
def read_plugin(pathzipfile):
""" process uploaded plugin. """
# test if valid zipfile
if not zipfile.is_zipfile(pathzipfile):
raise botslib.PluginError(_(u"Plugin is not a valid file."))
# read index file
try:
myzipimport = zipimport.zipimporter(pathzipfile)
importedbotsindex = myzipimport.load_module("botsindex")
pluglist = importedbotsindex.plugins[:]
if "botsindex" in sys.modules:
del sys.modules["botsindex"]
except:
txt = botslib.txtexc()
raise botslib.PluginError(_(u"Error in plugin. Nothing is written. Error:\n%(txt)s"), {"txt": txt})
else:
botsglobal.logger.info(_(u"Plugin is OK."))
botsglobal.logger.info(_(u"Start writing to database."))
# write content of index file to the bots database
try:
read_index2database(pluglist)
except:
txt = botslib.txtexc()
raise botslib.PluginError(
_(u"Error writing plugin to database. Nothing is written. Error:\n%(txt)s"), {"txt": txt}
)
else:
botsglobal.logger.info(_(u"Writing to database is OK."))
# write files to the file system.
botsglobal.logger.info(_(u"Start writing to files"))
try:
warnrenamed = False # to report in GUI files have been overwritten.
myzip = zipfile.ZipFile(pathzipfile, mode="r")
orgtargetpath = botsglobal.ini.get("directories", "botspath")
if orgtargetpath[-1:] in (os.path.sep, os.path.altsep) and len(os.path.splitdrive(orgtargetpath)[1]) > 1:
orgtargetpath = orgtargetpath[:-1]
for zipfileobject in myzip.infolist():
if zipfileobject.filename not in [
"botsindex.py",
"README",
"botssys/sqlitedb/botsdb",
"config/bots.ini",
] and os.path.splitext(zipfileobject.filename)[1] not in [".pyo", ".pyc"]:
# ~ botsglobal.logger.info(u'Filename in zip "%s".',zipfileobject.filename)
if zipfileobject.filename[0] == "/":
targetpath = zipfileobject.filename[1:]
else:
targetpath = zipfileobject.filename
# convert for correct environment: repacle botssys, config, usersys in filenames
if targetpath.startswith("usersys"):
targetpath = targetpath.replace("usersys", botsglobal.ini.get("directories", "usersysabs"), 1)
elif targetpath.startswith("botssys"):
targetpath = targetpath.replace("botssys", botsglobal.ini.get("directories", "botssys"), 1)
elif targetpath.startswith("config"):
targetpath = targetpath.replace("config", botsglobal.ini.get("directories", "config"), 1)
targetpath = botslib.join(orgtargetpath, targetpath)
# targetpath is OK now.
botsglobal.logger.info(_(u' Start writing file: "%(targetpath)s".'), {"targetpath": targetpath})
if botslib.dirshouldbethere(os.path.dirname(targetpath)):
botsglobal.logger.info(
_(u' Create directory "%(directory)s".'), {"directory": os.path.dirname(targetpath)}
)
if zipfileobject.filename[-1] == "/": # check if this is a dir; if so continue
continue
if os.path.isfile(targetpath): # check if file already exists
try: # this ***sometimes*** fails. (python25, for static/help/home.html...only there...)
warnrenamed = True
except:
pass
source = myzip.read(zipfileobject.filename)
target = open(targetpath, "wb")
target.write(source)
target.close()
botsglobal.logger.info(_(u' File written: "%(targetpath)s".'), {"targetpath": targetpath})
except:
txt = botslib.txtexc()
myzip.close()
raise botslib.PluginError(
_(u"Error writing files to system. Nothing is written to database. Error:\n%(txt)s"), {"txt": txt}
)
else:
myzip.close()
botsglobal.logger.info(_(u"Writing files to filesystem is OK."))
return warnrenamed
def load(pathzipfile):
''' process uploaded plugin. '''
#test is valid zipfile
if not zipfile.is_zipfile(pathzipfile):
raise botslib.PluginError(_(u'Plugin is not a valid file.'))
#read index file
try:
Zipimporter = zipimport.zipimporter(pathzipfile)
importedbotsindex = Zipimporter.load_module('botsindex')
pluglist = importedbotsindex.plugins[:]
if 'botsindex' in sys.modules:
del sys.modules['botsindex']
except:
txt = botslib.txtexc()
raise botslib.PluginError(
_(u'Error in plugin. Nothing is written. Error: "%s"') % (txt))
else:
botsglobal.logger.info(_(u'Plugin is OK.\nStart writing to database.'))
#write content of index file to the bots database
try:
writetodatabase(pluglist)
except:
txt = botslib.txtexc()
raise botslib.PluginError(
'Error writing plugin to database. Nothing is written. Error: "%s"'
% (txt))
else:
botsglobal.logger.info(
u'Writing to database is OK.\nStart writing to files')
#write files to the file system.
try:
warnrenamed = False #to report in GUI files have been overwritten.
z = zipfile.ZipFile(pathzipfile, mode="r")
orgtargetpath = botsglobal.ini.get('directories', 'botspath')
if (orgtargetpath[-1:] in (os.path.sep, os.path.altsep)
and len(os.path.splitdrive(orgtargetpath)[1]) > 1):
orgtargetpath = orgtargetpath[:-1]
for f in z.infolist():
if f.filename not in [
'botsindex.py', 'README', 'botssys/sqlitedb/botsdb',
'config/bots.ini'
] and os.path.splitext(f.filename)[1] not in ['.pyo', '.pyc']:
#~ botsglobal.logger.info(u'filename in zip "%s".',f.filename)
if f.filename[0] == '/':
targetpath = f.filename[1:]
else:
targetpath = f.filename
targetpath = targetpath.replace(
'usersys', botsglobal.ini.get('directories', 'usersysabs'),
1)
targetpath = targetpath.replace(
'botssys', botsglobal.ini.get('directories', 'botssys'), 1)
targetpath = botslib.join(orgtargetpath, targetpath)
#targetpath is OK now.
botsglobal.logger.info(_(u' Start writing file: "%s".'),
targetpath)
if botslib.dirshouldbethere(os.path.dirname(targetpath)):
botsglobal.logger.info(
_(u' Create directory "%s".'),
os.path.dirname(targetpath))
if f.filename[
-1] == '/': #check if this is a dir; if so continue
continue
if os.path.isfile(targetpath): #check if file already exists
try: #this ***sometimes*** fails. (python25, for static/help/home.html...only there...)
os.rename(
targetpath,
targetpath + '.' + time.strftime('%Y%m%d%H%M%S'))
warnrenamed = True
botsglobal.logger.info(
_(u' Renamed existing file "%(from)s" to "%(to)s".'
), {
'from': targetpath,
'to':
targetpath + time.strftime('%Y%m%d%H%M%S')
})
except:
pass
source = z.read(f.filename)
target = open(targetpath, "wb")
target.write(source)
target.close()
botsglobal.logger.info(_(u' File written: "%s".'),
targetpath)
except:
txt = botslib.txtexc()
z.close()
raise botslib.PluginError(
_(u'Error writing files to system. Nothing is written to database. Error: "%s"'
) % (txt))
else:
z.close()
botsglobal.logger.info(_(u'Writing files to filesystem is OK.'))
return warnrenamed
def delete(request,*kw,**kwargs):
if request.method == 'GET':
form = forms.DeleteForm()
return django.shortcuts.render_to_response('bots/delete.html', {'form': form},context_instance=django.template.RequestContext(request))
else:
if 'submit' in request.POST:
form = forms.DeleteForm(request.POST)
if form.is_valid():
botsglobal.logger.info(_(u'Start deleting in configuration.'))
if form.cleaned_data['deltransactions']:
from django.db import connection, transaction
#while testing with very big loads, deleting transaction when wrong. Using raw SQL solved this.
cursor = connection.cursor()
cursor.execute("DELETE FROM ta")
cursor.execute("DELETE FROM filereport")
cursor.execute("DELETE FROM report")
transaction.commit_unless_managed()
messages.add_message(request, messages.INFO, _(u'Transactions are deleted.'))
botsglobal.logger.info(_(u'Transactions are deleted.'))
#clean data files
deletefrompath = botsglobal.ini.get('directories','data','botssys/data')
shutil.rmtree(deletefrompath,ignore_errors=True)
botslib.dirshouldbethere(deletefrompath)
notification = _(u'Data files are deleted.')
messages.add_message(request, messages.INFO, notification)
botsglobal.logger.info(notification)
elif form.cleaned_data['delacceptance']:
from django.db.models import Min
list_file = [] #list of files for deletion in data-directory
report_idta_lowest = 0
for acc_report in models.report.objects.filter(acceptance=1): #for each acceptance report. is not very efficient.
if not report_idta_lowest:
report_idta_lowest = acc_report.idta
max_report_idta = models.report.objects.filter(idta__gt=acc_report.idta).aggregate(Min('idta'))['idta__min'] #select 'next' report...
if not max_report_idta: #if report is report of last run, there is no next report
max_report_idta = sys.maxint
#we have a idta-range now: from (and including) acc_report.idta till (and excluding) max_report_idta
list_file += models.ta.objects.filter(idta__gte=acc_report.idta,idta__lt=max_report_idta).exclude(status=1).values_list('filename', flat=True).distinct()
models.ta.objects.filter(idta__gte=acc_report.idta,idta__lt=max_report_idta).delete() #delete ta in range
models.filereport.objects.filter(idta__gte=acc_report.idta,idta__lt=max_report_idta).delete() #delete filereports in range
if report_idta_lowest:
models.report.objects.filter(idta__gte=report_idta_lowest,acceptance=1).delete() #delete all acceptance reports
for filename in list_file: #delete all files in data directory geenrated during acceptance testing
if filename.isdigit():
botslib.deldata(filename)
notification = _(u'Transactions from acceptance-testing deleted.')
messages.add_message(request, messages.INFO, notification)
botsglobal.logger.info(notification)
if form.cleaned_data['delconfiguration']:
models.confirmrule.objects.all().delete()
models.routes.objects.all().delete()
models.channel.objects.all().delete()
models.chanpar.objects.all().delete()
models.translate.objects.all().delete()
models.partner.objects.all().delete()
notification = _(u'Database configuration is deleted.')
messages.add_message(request, messages.INFO, notification)
botsglobal.logger.info(notification)
if form.cleaned_data['delcodelists']:
models.ccode.objects.all().delete()
models.ccodetrigger.objects.all().delete()
notification = _(u'User code lists are deleted.')
messages.add_message(request, messages.INFO, notification)
botsglobal.logger.info(notification)
if form.cleaned_data['delinfile']:
deletefrompath = botslib.join(botsglobal.ini.get('directories','botssys','botssys'),'infile')
shutil.rmtree(deletefrompath,ignore_errors=True)
notification = _(u'Files in botssys/infile are deleted.')
messages.add_message(request, messages.INFO, notification)
botsglobal.logger.info(notification)
if form.cleaned_data['deloutfile']:
deletefrompath = botslib.join(botsglobal.ini.get('directories','botssys','botssys'),'outfile')
shutil.rmtree(deletefrompath,ignore_errors=True)
notification = _(u'Files in botssys/outfile are deleted.')
messages.add_message(request, messages.INFO, notification)
botsglobal.logger.info(notification)
if form.cleaned_data['deluserscripts']:
deletefrompath = botsglobal.ini.get('directories','usersysabs')
for root, dirs, files in os.walk(deletefrompath):
head, tail = os.path.split(root)
if tail == 'charsets':
del dirs[:]
continue
for bestand in files:
if bestand != '__init__.py':
os.remove(os.path.join(root,bestand))
notification = _(u'User scripts are deleted (in usersys).')
messages.add_message(request, messages.INFO, notification)
botsglobal.logger.info(notification)
elif form.cleaned_data['delbackup']:
deletefrompath = botsglobal.ini.get('directories','usersysabs')
for root, dirs, files in os.walk(deletefrompath):
head, tail = os.path.split(root)
if tail == 'charsets':
del dirs[:]
continue
for bestand in files:
name,ext = os.path.splitext(bestand)
if ext and len(ext) == 15 and ext[1:].isdigit() :
os.remove(os.path.join(root,bestand))
notification = _(u'Backupped user scripts are deleted/purged (in usersys).')
messages.add_message(request, messages.INFO, notification)
botsglobal.logger.info(notification)
botsglobal.logger.info(_(u'Finished deleting in configuration.'))
return django.shortcuts.redirect('/home')
def read_plugin(pathzipfile):
''' process uploaded plugin. '''
#test if valid zipfile
if not zipfile.is_zipfile(pathzipfile):
raise botslib.PluginError(_(u'Plugin is not a valid file.'))
#read index file
try:
myzipimport = zipimport.zipimporter(pathzipfile)
importedbotsindex = myzipimport.load_module('botsindex')
pluglist = importedbotsindex.plugins[:]
if 'botsindex' in sys.modules:
del sys.modules['botsindex']
except:
txt = botslib.txtexc()
raise botslib.PluginError(_(u'Error in plugin. Nothing is written. Error:\n%(txt)s'),{'txt':txt})
else:
botsglobal.logger.info(_(u'Plugin is OK.'))
botsglobal.logger.info(_(u'Start writing to database.'))
#write content of index file to the bots database
try:
read_index2database(pluglist)
except:
txt = botslib.txtexc()
raise botslib.PluginError(_(u'Error writing plugin to database. Nothing is written. Error:\n%(txt)s'),{'txt':txt})
else:
botsglobal.logger.info(_(u'Writing to database is OK.'))
#write files to the file system.
botsglobal.logger.info(_(u'Start writing to files'))
try:
warnrenamed = False #to report in GUI files have been overwritten.
myzip = zipfile.ZipFile(pathzipfile, mode="r")
orgtargetpath = botsglobal.ini.get('directories','botspath')
if (orgtargetpath[-1:] in (os.path.sep, os.path.altsep) and len(os.path.splitdrive(orgtargetpath)[1]) > 1):
orgtargetpath = orgtargetpath[:-1]
for zipfileobject in myzip.infolist():
if zipfileobject.filename not in ['botsindex.py','README','botssys/sqlitedb/botsdb','config/bots.ini'] and os.path.splitext(zipfileobject.filename)[1] not in ['.pyo','.pyc']:
#~ botsglobal.logger.info(u'Filename in zip "%s".',zipfileobject.filename)
if zipfileobject.filename[0] == '/':
targetpath = zipfileobject.filename[1:]
else:
targetpath = zipfileobject.filename
#convert for correct environment: repacle botssys, config, usersys in filenames
if targetpath.startswith('usersys'):
targetpath = targetpath.replace('usersys',botsglobal.ini.get('directories','usersysabs'),1)
elif targetpath.startswith('botssys'):
targetpath = targetpath.replace('botssys',botsglobal.ini.get('directories','botssys'),1)
elif targetpath.startswith('config'):
targetpath = targetpath.replace('config',botsglobal.ini.get('directories','config'),1)
targetpath = botslib.join(orgtargetpath, targetpath)
#targetpath is OK now.
botsglobal.logger.info(_(u' Start writing file: "%(targetpath)s".'),{'targetpath':targetpath})
if botslib.dirshouldbethere(os.path.dirname(targetpath)):
botsglobal.logger.info(_(u' Create directory "%(directory)s".'),{'directory':os.path.dirname(targetpath)})
if zipfileobject.filename[-1] == '/': #check if this is a dir; if so continue
continue
if os.path.isfile(targetpath): #check if file already exists
try: #this ***sometimes*** fails. (python25, for static/help/home.html...only there...)
warnrenamed = True
except:
pass
source = myzip.read(zipfileobject.filename)
target = open(targetpath, "wb")
target.write(source)
target.close()
botsglobal.logger.info(_(u' File written: "%(targetpath)s".'),{'targetpath':targetpath})
except:
txt = botslib.txtexc()
myzip.close()
raise botslib.PluginError(_(u'Error writing files to system. Nothing is written to database. Error:\n%(txt)s'),{'txt':txt})
else:
myzip.close()
botsglobal.logger.info(_(u'Writing files to filesystem is OK.'))
return warnrenamed
def __init__(self, *args, **kwargs):
super(PlugoutForm, self).__init__(*args, **kwargs)
self.fields['filename'].initial = botslib.join(botsglobal.ini.get('directories','botssys'),'myplugin' + time.strftime('_%Y%m%d') + '.zip')
def generalinit(configdir):
##########################################################################
#Configdir: settings.py & bots.ini#########################################
#Configdir MUST be importable. So configdir is relative to PYTHONPATH. Try several options for this import.
try: #first check if is configdir outside bots-directory: import configdir.settings.py
importnameforsettings = os.path.normpath(os.path.join(configdir,'settings')).replace(os.sep,'.')
settings = botslib.botsbaseimport(importnameforsettings)
except ImportError: #normal: configdir is in bots directory: import bots.configdir.settings.py
try:
importnameforsettings = os.path.normpath(os.path.join('bots',configdir,'settings')).replace(os.sep,'.')
settings = botslib.botsbaseimport(importnameforsettings)
except ImportError: #set pythonpath to config directory first
if not os.path.exists(configdir): #check if configdir exists.
raise botslib.PanicError(u'In initilisation: path to configuration does not exists: "%(configdir)s".',{'configdir':configdir})
addtopythonpath = os.path.abspath(os.path.dirname(configdir))
moduletoimport = os.path.basename(configdir)
sys.path.append(addtopythonpath)
importnameforsettings = os.path.normpath(os.path.join(moduletoimport,'settings')).replace(os.sep,'.')
settings = botslib.botsbaseimport(importnameforsettings)
#settings is imported, so now we know where to find settings.py: importnameforsettings
#note: the imported settings.py itself is NOT used, this is doen via django.conf.settings
configdirectory = os.path.abspath(os.path.dirname(settings.__file__))
#Read configuration-file bots.ini.
botsglobal.ini = BotsConfig()
botsglobal.ini.read(os.path.join(configdirectory,'bots.ini'))
# 'directories','botspath': absolute path for bots directory
botsglobal.ini.set('directories','botspath',os.path.abspath(os.path.dirname(__file__)))
# 'directories','config': absolute path for config directory
botsglobal.ini.set('directories','config',configdirectory)
#set config as originally received; used in starting engine via bots-monitor
botsglobal.ini.set('directories','config_org',configdir)
############################################################################
#Usersys####################################################################
#usersys MUST be importable. So usersys is relative to PYTHONPATH. Try several options for this import.
usersys = botsglobal.ini.get('directories','usersys','usersys')
try: #usersys outside bots-directory: import usersys
importnameforusersys = os.path.normpath(usersys).replace(os.sep,'.')
importedusersys = botslib.botsbaseimport(importnameforusersys)
except ImportError: #usersys is in bots directory: import bots.usersys
try:
importnameforusersys = os.path.normpath(os.path.join('bots',usersys)).replace(os.sep,'.')
importedusersys = botslib.botsbaseimport(importnameforusersys)
except ImportError: #set pythonpath to usersys directory first
if not os.path.exists(usersys): #check if configdir exists.
raise botslib.PanicError(u'In initilisation: path to configuration does not exists: "%(usersys)s".',{'usersys':usersys})
addtopythonpath = os.path.abspath(os.path.dirname(usersys)) #????
moduletoimport = os.path.basename(usersys)
sys.path.append(addtopythonpath)
importnameforusersys = os.path.normpath(usersys).replace(os.sep,'.')
importedusersys = botslib.botsbaseimport(importnameforusersys)
# 'directories','usersysabs': absolute path for config usersysabs
botsglobal.ini.set('directories','usersysabs',os.path.abspath(os.path.dirname(importedusersys.__file__))) #???Find pathname usersys using imported usersys
# botsglobal.usersysimportpath: used for imports from usersys
botsglobal.usersysimportpath = importnameforusersys
botsglobal.ini.set('directories','templatehtml',botslib.join(botsglobal.ini.get('directories','usersysabs'),'grammars/templatehtml/templates'))
############################################################################
#Botssys####################################################################
# 'directories','botssys': absolute path for config botssys
botssys = botsglobal.ini.get('directories','botssys','botssys')
botsglobal.ini.set('directories','botssys_org',botssys) #store original botssys setting
botsglobal.ini.set('directories','botssys',botslib.join(botssys)) #use absolute path
botsglobal.ini.set('directories','data',botslib.join(botssys,'data'))
botsglobal.ini.set('directories','logging',botslib.join(botssys,'logging'))
############################################################################
#other inits##############################################################
if botsglobal.ini.get('webserver','environment','development') != 'development': #values in bots.ini are also used in setting up cherrypy
logging.raiseExceptions = 0 # during production: if errors occurs in writing to log: ignore error. (leads to a missing log line, better than error;-).
botslib.dirshouldbethere(botsglobal.ini.get('directories','data'))
botslib.dirshouldbethere(botsglobal.ini.get('directories','logging'))
initbotscharsets() #initialise bots charsets
node.Node.checklevel = botsglobal.ini.getint('settings','get_checklevel',1)
botslib.settimeout(botsglobal.ini.getint('settings','globaltimeout',10))
############################################################################
#Init django#################################################################################
os.environ['DJANGO_SETTINGS_MODULE'] = importnameforsettings
import django
if hasattr(django,'setup'):
django.setup()
from django.conf import settings
botsglobal.settings = settings #settings are accessed using botsglobal
def load(pathzipfile):
''' process uploaded plugin. '''
#test is valid zipfile
if not zipfile.is_zipfile(pathzipfile):
raise botslib.PluginError(_(u'Plugin is not a valid file.'))
#read index file
try:
myzipimport = zipimport.zipimporter(pathzipfile)
importedbotsindex = myzipimport.load_module('botsindex')
pluglist = importedbotsindex.plugins[:]
if 'botsindex' in sys.modules:
del sys.modules['botsindex']
except:
txt = botslib.txtexc()
raise botslib.PluginError(_(u'Error in plugin. Nothing is written. Error:\n%s')%(txt))
else:
botsglobal.logger.info(_(u'Plugin is OK.'))
botsglobal.logger.info(_(u'Start writing to database.'))
#write content of index file to the bots database
try:
writetodatabase(pluglist)
except:
txt = botslib.txtexc()
raise botslib.PluginError(_(u'Error writing plugin to database. Nothing is written. Error:\n%s'%(txt)))
else:
botsglobal.logger.info(u'Writing to database is OK.')
botsglobal.logger.info(u'Start writing to files')
#write files to the file system.
try:
warnrenamed = False #to report in GUI files have been overwritten.
myzip = zipfile.ZipFile(pathzipfile, mode="r")
orgtargetpath = botsglobal.ini.get('directories','botspath')
if (orgtargetpath[-1:] in (os.path.sep, os.path.altsep) and len(os.path.splitdrive(orgtargetpath)[1]) > 1):
orgtargetpath = orgtargetpath[:-1]
for zipfileobject in myzip.infolist():
if zipfileobject.filename not in ['botsindex.py','README','botssys/sqlitedb/botsdb','config/bots.ini'] and os.path.splitext(zipfileobject.filename)[1] not in ['.pyo','.pyc']:
#~ botsglobal.logger.info(u'filename in zip "%s".',zipfileobject.filename)
if zipfileobject.filename[0] == '/':
targetpath = zipfileobject.filename[1:]
else:
targetpath = zipfileobject.filename
targetpath = targetpath.replace('usersys',botsglobal.ini.get('directories','usersysabs'),1)
targetpath = targetpath.replace('botssys',botsglobal.ini.get('directories','botssys'),1)
targetpath = botslib.join(orgtargetpath, targetpath)
#targetpath is OK now.
botsglobal.logger.info(_(u' Start writing file: "%s".'),targetpath)
if botslib.dirshouldbethere(os.path.dirname(targetpath)):
botsglobal.logger.info(_(u' Create directory "%s".'),os.path.dirname(targetpath))
if zipfileobject.filename[-1] == '/': #check if this is a dir; if so continue
continue
if os.path.isfile(targetpath): #check if file already exists
try: #this ***sometimes*** fails. (python25, for static/help/home.html...only there...)
os.rename(targetpath,targetpath+'.'+time.strftime('%Y%m%d%H%M%S'))
warnrenamed = True
botsglobal.logger.info(_(u' Renamed existing file "%(from)s" to "%(to)s".'),{'from':targetpath,'to':targetpath+time.strftime('%Y%m%d%H%M%S')})
except:
pass
source = myzip.read(zipfileobject.filename)
target = open(targetpath, "wb")
target.write(source)
target.close()
botsglobal.logger.info(_(u' File written: "%s".'),targetpath)
except:
txt = botslib.txtexc()
myzip.close()
raise botslib.PluginError(_(u'Error writing files to system. Nothing is written to database. Error:\n%s')%(txt))
else:
myzip.close()
botsglobal.logger.info(_(u'Writing files to filesystem is OK.'))
return warnrenamed
def generalinit(configdir):
#Set Configdir
#Configdir MUST be importable. So configdir is relative to PYTHONPATH. Try several options for this import.
try: #configdir outside bots-directory: import configdir.settings.py
importnameforsettings = os.path.normpath(os.path.join(configdir,'settings')).replace(os.sep,'.')
settings = botslib.botsbaseimport(importnameforsettings)
except ImportError: #configdir is in bots directory: import bots.configdir.settings.py
try:
importnameforsettings = os.path.normpath(os.path.join('bots',configdir,'settings')).replace(os.sep,'.')
settings = botslib.botsbaseimport(importnameforsettings)
except ImportError: #set pythonpath to config directory first
if not os.path.exists(configdir): #check if configdir exists.
raise botslib.BotsError(_(u'In initilisation: path to configuration does not exists: "$path".'),path=configdir)
addtopythonpath = os.path.abspath(os.path.dirname(configdir))
#~ print 'add pythonpath for usersys',addtopythonpath
moduletoimport = os.path.basename(configdir)
sys.path.append(addtopythonpath)
importnameforsettings = os.path.normpath(os.path.join(moduletoimport,'settings')).replace(os.sep,'.')
settings = botslib.botsbaseimport(importnameforsettings)
#settings are accessed using botsglobal
botsglobal.settings = settings
#Find pathname configdir using imported settings.py.
configdirectory = os.path.abspath(os.path.dirname(settings.__file__))
#Read configuration-file bots.ini.
botsglobal.ini = BotsConfig()
cfgfile = open(os.path.join(configdirectory,'bots.ini'), 'r')
botsglobal.ini.readfp(cfgfile)
cfgfile.close()
#Set usersys.
#usersys MUST be importable. So usersys is relative to PYTHONPATH. Try several options for this import.
usersys = botsglobal.ini.get('directories','usersys','usersys')
try: #usersys outside bots-directory: import usersys
importnameforusersys = os.path.normpath(usersys).replace(os.sep,'.')
importedusersys = botslib.botsbaseimport(importnameforusersys)
except ImportError: #usersys is in bots directory: import bots.usersys
try:
importnameforusersys = os.path.normpath(os.path.join('bots',usersys)).replace(os.sep,'.')
importedusersys = botslib.botsbaseimport(importnameforusersys)
except ImportError: #set pythonpath to usersys directory first
if not os.path.exists(usersys): #check if configdir exists.
raise botslib.BotsError(_(u'In initilisation: path to configuration does not exists: "$path".'),path=usersys)
addtopythonpath = os.path.abspath(os.path.dirname(usersys)) #????
moduletoimport = os.path.basename(usersys)
#~ print 'add pythonpath for usersys',addtopythonpath
sys.path.append(addtopythonpath)
importnameforusersys = os.path.normpath(usersys).replace(os.sep,'.')
importedusersys = botslib.botsbaseimport(importnameforusersys)
#set directory settings in bots.ini************************************************************
botsglobal.ini.set('directories','botspath',botsglobal.settings.PROJECT_PATH)
botsglobal.ini.set('directories','config',configdirectory)
botsglobal.ini.set('directories','usersysabs',os.path.abspath(os.path.dirname(importedusersys.__file__))) #???Find pathname usersys using imported usersys
botsglobal.usersysimportpath = importnameforusersys
botssys = botsglobal.ini.get('directories','botssys','botssys')
botsglobal.ini.set('directories','botssys',botslib.join(botssys))
botsglobal.ini.set('directories','data',botslib.join(botssys,'data'))
botslib.dirshouldbethere(botsglobal.ini.get('directories','data'))
botsglobal.ini.set('directories','logging',botslib.join(botssys,'logging'))
botslib.dirshouldbethere(botsglobal.ini.get('directories','logging'))
botsglobal.ini.set('directories','templates',botslib.join(botsglobal.ini.get('directories','usersysabs'),'grammars/template/templates'))
botsglobal.ini.set('directories','templateshtml',botslib.join(botsglobal.ini.get('directories','usersysabs'),'grammars/templatehtml/templates'))
#set values in setting.py**********************************************************************
if botsglobal.ini.get('webserver','environment','development') == 'development': #values in bots.ini are also used in setting up cherrypy
settings.DEBUG = True
else:
settings.DEBUG = False
settings.TEMPLATE_DEBUG = settings.DEBUG
#set paths in settings.py:
#~ settings.FILE_UPLOAD_TEMP_DIR = os.path.join(settings.PROJECT_PATH, 'botssys/pluginsuploaded')
#start initializing bots charsets
initbotscharsets()
#set environment for django to start***************************************************************************************************
os.environ['DJANGO_SETTINGS_MODULE'] = importnameforsettings
initbotscharsets()
botslib.settimeout(botsglobal.ini.getint('settings','globaltimeout',10)) #
def start():
#********command line arguments**************************
edifile =''
grammarfile = ''
configdir = 'config'
for arg in sys.argv[1:]:
if not arg:
continue
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print ' !!Indicated Bots should use specific .ini file but no file name was given.'
showusage()
elif arg in ["?", "/?"] or arg.startswith('-'):
showusage()
else:
if not edifile:
edifile = arg
else:
grammarfile = arg
if not (edifile and grammarfile):
print ' !!Both edifile and grammarfile are required.'
showusage()
#********end handling command line arguments**************************
editype='xmlnocheck'
messagetype='xmlnocheckxxxtemporaryforxml2grammar'
mpath = []
botsinit.generalinit(configdir)
os.chdir(botsglobal.ini.get('directories','botspath'))
botsinit.initenginelogging()
#the xml file is parsed as an xmlnocheck message....so a (temp) xmlnocheck grammar is needed....without content... this file is not removed....
tmpgrammarfile = botslib.join(botsglobal.ini.get('directories','usersysabs'),'grammars',editype,messagetype+'.py')
f = open(tmpgrammarfile,'w')
f.close()
inn = inmessage.edifromfile(editype=editype,messagetype=messagetype,filename=edifile)
#~ inn.root.display()
out = outmessage.outmessage_init(editype=editype,messagetype=messagetype,filename='botssys/infile/unitnode/output/inisout03.edi',divtext='',topartner='') #make outmessage object
#handle root
rootmpath = [{'BOTSID':inn.root.record['BOTSID']}]
out.put(*rootmpath)
writefields(out,inn.root,rootmpath)
#walk tree; write results to out-tree
for node,mpath in treewalker(inn.root,mpath):
mpath.append({'BOTSID':node.record['BOTSID']})
if out.get(*mpath) is None:
out.put(*mpath)
writefields(out,node,mpath)
#~ out.root.display()
#out-tree is finished; represents ' normalised' tree suited for writing as a grammar
structure = []
recorddefs = {}
tree2grammar(out.root,structure,recorddefs)
#~ for key,value in recorddefs.items():
#~ print key,value
#~ print '\n'
sortedstructurelist = structure2list(structure)
recorddefsstring = recorddefs2string(recorddefs,sortedstructurelist)
structurestring = structure2string(structure)
#write grammar file
grammar = open(grammarfile,'wb')
grammar.write('#grammar automatically generated by bots open source edi software.')
grammar.write('\n')
grammar.write('from bots.botsconfig import *')
grammar.write('\n\n')
grammar.write('syntax = {}')
grammar.write('\n\n')
grammar.write('structure = [\n%s]\n'%(structurestring))
grammar.write('\n\n')
grammar.write('recorddefs = %s'%(recorddefsstring))
grammar.write('\n\n')
grammar.close()
print 'grammar file is written',grammarfile
def start():
#********command line arguments**************************
usage = '''
This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
Creates a grammar from an xml file.'
Usage:'
%(name)s -c<directory> <xml_file> <xml_grammar_file>
Options:
-c<directory> directory for configuration files (default: config).
<xml_file> name of the xml file to read
<xml_grammar_file> name of the grammar file to write
'''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version}
configdir = 'config'
edifile =''
grammarfile = ''
for arg in sys.argv[1:]:
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Error: configuration directory indicated, but no directory name.'
sys.exit(1)
elif arg in ["?", "/?",'-h', '--help'] or arg.startswith('-'):
print usage
sys.exit(0)
else:
if not edifile:
edifile = arg
else:
grammarfile = arg
if not edifile or not grammarfile:
print 'Error: both edifile and grammarfile are required.'
sys.exit(0)
#***end handling command line arguments**************************
botsinit.generalinit(configdir) #find locating of bots, configfiles, init paths etc.
process_name = 'xml2botsgrammar'
botsglobal.logger = botsinit.initenginelogging(process_name)
atexit.register(logging.shutdown)
#the xml file is parsed as an xmlnocheck message
editype = 'xmlnocheck'
messagetype = 'xmlnocheckxxxtemporaryforxml2grammar'
mpath = []
#a (temp) xmlnocheck grammar is needed (but needs not actual content. This file is not removed.
tmpgrammarfile = botslib.join(botsglobal.ini.get('directories','usersysabs'),'grammars',editype,messagetype+'.py')
filehandler = open(tmpgrammarfile,'w')
filehandler.close()
#make inmessage object: read the xml file
inn = inmessage.parse_edi_file(editype=editype,messagetype=messagetype,filename=edifile,remove_empties_from_xml=False)
#make outmessage object; nothing is 'filled' yet.
out = outmessage.outmessage_init(editype=editype,messagetype=messagetype,filename='botssys/infile/unitnode/output/inisout03.edi',divtext='',topartner='')
#***do the mapping***************************************************
#handle root
rootmpath = [{'BOTSID':inn.root.record['BOTSID'],'BOTSIDnr':'1'}]
out.put(*rootmpath)
map_writefields(out,inn.root,rootmpath)
#walk tree; write results to out-tree
for node_instance,mpath in map_treewalker(inn.root,mpath):
mpath.append({'BOTSID':node_instance.record['BOTSID']})
if out.get(*mpath) is None:
out.put(*mpath)
map_writefields(out,node_instance,mpath)
#***mapping is done; out-tree is finished; represents 'normalised' tree suited for writing as a grammar
structure = []
recorddefs = {}
tree2grammar(out.root,structure,recorddefs)
#~ for key,value in recorddefs.items():
#~ print key,value
#~ print '\n'
sortedstructurelist = structure2list(structure)
recorddefsstring = recorddefs2string(recorddefs,sortedstructurelist)
structurestring = structure2string(structure)
#write grammar file
grammar = open(grammarfile,'wb')
grammar.write('#grammar automatically generated by bots open source edi translator.')
grammar.write('\n')
grammar.write('from bots.botsconfig import *')
grammar.write('\n\n')
grammar.write('syntax = {}')
grammar.write('\n\n')
grammar.write('structure = [\n%s]\n'%(structurestring))
grammar.write('\n\n')
grammar.write('recorddefs = %s'%(recorddefsstring))
grammar.write('\n\n')
grammar.close()
print 'grammar file is written:',grammarfile
def generalinit(configdir):
#Set Configdir
#Configdir MUST be importable. So configdir is relative to PYTHONPATH. Try several options for this import.
try: #configdir outside bots-directory: import configdir.settings.py
importnameforsettings = os.path.normpath(
os.path.join(configdir, 'settings')).replace(os.sep, '.')
settings = botslib.botsbaseimport(importnameforsettings)
except ImportError: #configdir is in bots directory: import bots.configdir.settings.py
try:
importnameforsettings = os.path.normpath(
os.path.join('bots', configdir,
'settings')).replace(os.sep, '.')
settings = botslib.botsbaseimport(importnameforsettings)
except ImportError: #set pythonpath to config directory first
if not os.path.exists(configdir): #check if configdir exists.
raise botslib.BotsError(_(
u'In initilisation: path to configuration does not exists: "$path".'
),
path=configdir)
addtopythonpath = os.path.abspath(os.path.dirname(configdir))
#~ print 'add pythonpath for usersys',addtopythonpath
moduletoimport = os.path.basename(configdir)
sys.path.append(addtopythonpath)
importnameforsettings = os.path.normpath(
os.path.join(moduletoimport, 'settings')).replace(os.sep, '.')
settings = botslib.botsbaseimport(importnameforsettings)
#settings are accessed using botsglobal
botsglobal.settings = settings
#Find pathname configdir using imported settings.py.
configdirectory = os.path.abspath(os.path.dirname(settings.__file__))
#Read configuration-file bots.ini.
botsglobal.ini = BotsConfig()
cfgfile = open(os.path.join(configdirectory, 'bots.ini'), 'r')
botsglobal.ini.readfp(cfgfile)
cfgfile.close()
#Set usersys.
#usersys MUST be importable. So usersys is relative to PYTHONPATH. Try several options for this import.
usersys = botsglobal.ini.get('directories', 'usersys', 'usersys')
try: #usersys outside bots-directory: import usersys
importnameforusersys = os.path.normpath(usersys).replace(os.sep, '.')
importedusersys = botslib.botsbaseimport(importnameforusersys)
except ImportError: #usersys is in bots directory: import bots.usersys
try:
importnameforusersys = os.path.normpath(
os.path.join('bots', usersys)).replace(os.sep, '.')
importedusersys = botslib.botsbaseimport(importnameforusersys)
except ImportError: #set pythonpath to usersys directory first
if not os.path.exists(usersys): #check if configdir exists.
raise botslib.BotsError(_(
u'In initilisation: path to configuration does not exists: "$path".'
),
path=usersys)
addtopythonpath = os.path.abspath(os.path.dirname(usersys)) #????
moduletoimport = os.path.basename(usersys)
#~ print 'add pythonpath for usersys',addtopythonpath
sys.path.append(addtopythonpath)
importnameforusersys = os.path.normpath(usersys).replace(
os.sep, '.')
importedusersys = botslib.botsbaseimport(importnameforusersys)
#set directory settings in bots.ini************************************************************
botsglobal.ini.set('directories', 'botspath',
botsglobal.settings.PROJECT_PATH)
botsglobal.ini.set('directories', 'config', configdirectory)
botsglobal.ini.set('directories', 'usersysabs',
os.path.abspath(
os.path.dirname(importedusersys.__file__))
) #???Find pathname usersys using imported usersys
botsglobal.usersysimportpath = importnameforusersys
botssys = botsglobal.ini.get('directories', 'botssys', 'botssys')
botsglobal.ini.set('directories', 'botssys', botslib.join(botssys))
botsglobal.ini.set('directories', 'data', botslib.join(botssys, 'data'))
botslib.dirshouldbethere(botsglobal.ini.get('directories', 'data'))
botsglobal.ini.set('directories', 'logging',
botslib.join(botssys, 'logging'))
botslib.dirshouldbethere(botsglobal.ini.get('directories', 'logging'))
botsglobal.ini.set(
'directories', 'templates',
botslib.join(botsglobal.ini.get('directories', 'usersysabs'),
'grammars/template/templates'))
botsglobal.ini.set(
'directories', 'templateshtml',
botslib.join(botsglobal.ini.get('directories', 'usersysabs'),
'grammars/templatehtml/templates'))
#set values in setting.py**********************************************************************
if botsglobal.ini.get(
'webserver', 'environment', 'development'
) == 'development': #values in bots.ini are also used in setting up cherrypy
settings.DEBUG = True
else:
settings.DEBUG = False
settings.TEMPLATE_DEBUG = settings.DEBUG
#set paths in settings.py:
#~ settings.FILE_UPLOAD_TEMP_DIR = os.path.join(settings.PROJECT_PATH, 'botssys/pluginsuploaded')
#start initializing bots charsets
initbotscharsets()
#set environment for django to start***************************************************************************************************
os.environ['DJANGO_SETTINGS_MODULE'] = importnameforsettings
initbotscharsets()
botslib.settimeout(botsglobal.ini.getint('settings', 'globaltimeout',
10)) #
def delete(request,*kw,**kwargs):
if request.method == 'GET':
form = forms.DeleteForm()
return django.shortcuts.render_to_response('bots/delete.html', {'form': form},context_instance=django.template.RequestContext(request))
else:
if 'submit' in request.POST:
form = forms.DeleteForm(request.POST)
if form.is_valid():
botsglobal.logger.info(_(u'Start deleting in configuration.'))
if form.cleaned_data['deltransactions']:
#while testing with very big loads, deleting transaction when wrong. Using raw SQL solved this.
from django.db import connection, transaction
cursor = connection.cursor()
cursor.execute("DELETE FROM ta")
cursor.execute("DELETE FROM filereport")
cursor.execute("DELETE FROM report")
transaction.commit_unless_managed()
request.user.message_set.create(message=_(u'Transactions are deleted.'))
botsglobal.logger.info(_(u' Transactions are deleted.'))
#clean data files
deletefrompath = botsglobal.ini.get('directories','data','botssys/data')
shutil.rmtree(deletefrompath,ignore_errors=True)
botslib.dirshouldbethere(deletefrompath)
request.user.message_set.create(message=_(u'Data files are deleted.'))
botsglobal.logger.info(_(u' Data files are deleted.'))
if form.cleaned_data['delconfiguration']:
models.confirmrule.objects.all().delete()
models.channel.objects.all().delete()
models.chanpar.objects.all().delete()
models.partner.objects.all().delete()
models.translate.objects.all().delete()
models.routes.objects.all().delete()
request.user.message_set.create(message=_(u'Database configuration is deleted.'))
botsglobal.logger.info(_(u' Database configuration is deleted.'))
if form.cleaned_data['delcodelists']:
models.ccode.objects.all().delete()
models.ccodetrigger.objects.all().delete()
request.user.message_set.create(message=_(u'User code lists are deleted.'))
botsglobal.logger.info(_(u' User code lists are deleted.'))
if form.cleaned_data['delinfile']:
deletefrompath = botslib.join(botsglobal.ini.get('directories','botssys','botssys'),'infile')
shutil.rmtree('bots/botssys/infile',ignore_errors=True)
request.user.message_set.create(message=_(u'Files in botssys/infile are deleted.'))
botsglobal.logger.info(_(u' Files in botssys/infile are deleted.'))
if form.cleaned_data['deloutfile']:
deletefrompath = botslib.join(botsglobal.ini.get('directories','botssys','botssys'),'outfile')
shutil.rmtree('bots/botssys/outfile',ignore_errors=True)
request.user.message_set.create(message=_(u'Files in botssys/outfile are deleted.'))
botsglobal.logger.info(_(u' Files in botssys/outfile are deleted.'))
if form.cleaned_data['deluserscripts']:
deletefrompath = botsglobal.ini.get('directories','usersysabs')
for root, dirs, files in os.walk(deletefrompath):
head, tail = os.path.split(root)
if tail == 'charsets':
del dirs[:]
continue
for bestand in files:
if bestand != '__init__.py':
os.remove(os.path.join(root,bestand))
request.user.message_set.create(message=_(u'User scripts are deleted (in usersys).'))
botsglobal.logger.info(_(u' User scripts are deleted (in usersys).'))
elif form.cleaned_data['delbackup']:
deletefrompath = botsglobal.ini.get('directories','usersysabs')
for root, dirs, files in os.walk(deletefrompath):
head, tail = os.path.split(root)
if tail == 'charsets':
del dirs[:]
continue
for bestand in files:
name,ext = os.path.splitext(bestand)
if ext and len(ext) == 15 and ext[1:].isdigit() :
os.remove(os.path.join(root,bestand))
request.user.message_set.create(message=_(u'Backupped user scripts are deleted/purged (in usersys).'))
botsglobal.logger.info(_(u' Backupped user scripts are deleted/purged (in usersys).'))
botsglobal.logger.info(_(u'Finished deleting in configuration.'))
return django.shortcuts.redirect('/home')
def generalinit(configdir):
#Set Configdir
#Configdir MUST be importable. So configdir is relative to PYTHONPATH. Try several options for this import.
try: #configdir outside bots-directory: import configdir.settings.py
importnameforsettings = os.path.normpath(os.path.join(configdir,'settings')).replace(os.sep,'.')
settings = botslib.botsbaseimport(importnameforsettings)
except ImportError: #configdir is in bots directory: import bots.configdir.settings.py
try:
importnameforsettings = os.path.normpath(os.path.join('bots',configdir,'settings')).replace(os.sep,'.')
settings = botslib.botsbaseimport(importnameforsettings)
except ImportError: #set pythonpath to config directory first
if not os.path.exists(configdir): #check if configdir exists.
raise botslib.BotsError(u'In initilisation: path to configuration does not exists: "%(configdir)s".',{'configdir':configdir})
addtopythonpath = os.path.abspath(os.path.dirname(configdir))
#~ print 'add pythonpath for usersys',addtopythonpath
moduletoimport = os.path.basename(configdir)
sys.path.append(addtopythonpath)
importnameforsettings = os.path.normpath(os.path.join(moduletoimport,'settings')).replace(os.sep,'.')
settings = botslib.botsbaseimport(importnameforsettings)
#settings are accessed using botsglobal
botsglobal.settings = settings
if hasattr(settings,'DATABASE_ENGINE'): #check for old django settings.py
print u'You use an old settings.py. Please change settings.py first. See migration instructions in wiki: http://code.google.com/p/bots/wiki/Migrate'
sys.exit(0)
#Find pathname configdir using imported settings.py.
configdirectory = os.path.abspath(os.path.dirname(settings.__file__))
#Read configuration-file bots.ini.
botsglobal.ini = BotsConfig()
botsglobal.ini.read(os.path.join(configdirectory,'bots.ini'))
#Set usersys.
#usersys MUST be importable. So usersys is relative to PYTHONPATH. Try several options for this import.
usersys = botsglobal.ini.get('directories','usersys','usersys')
try: #usersys outside bots-directory: import usersys
importnameforusersys = os.path.normpath(usersys).replace(os.sep,'.')
importedusersys = botslib.botsbaseimport(importnameforusersys)
except ImportError: #usersys is in bots directory: import bots.usersys
try:
importnameforusersys = os.path.normpath(os.path.join('bots',usersys)).replace(os.sep,'.')
importedusersys = botslib.botsbaseimport(importnameforusersys)
except ImportError: #set pythonpath to usersys directory first
if not os.path.exists(usersys): #check if configdir exists.
raise botslib.BotsError(u'In initilisation: path to configuration does not exists: "%(usersys)s".',{'usersys':usersys})
addtopythonpath = os.path.abspath(os.path.dirname(usersys)) #????
moduletoimport = os.path.basename(usersys)
#~ print 'add pythonpath for usersys',addtopythonpath
sys.path.append(addtopythonpath)
importnameforusersys = os.path.normpath(usersys).replace(os.sep,'.')
importedusersys = botslib.botsbaseimport(importnameforusersys)
#set directory settings in bots.ini************************************************************
# 'directories','botspath': absolute path for bots directory
botsglobal.ini.set('directories','botspath',settings.PROJECT_PATH)
# 'directories','config': absolute path for config directory
botsglobal.ini.set('directories','config',configdirectory)
botsglobal.ini.set('directories','config_org',configdir) #set config as originally received.
# 'directories','usersysabs': absolute path for config usersysabs
botsglobal.ini.set('directories','usersysabs',os.path.abspath(os.path.dirname(importedusersys.__file__))) #???Find pathname usersys using imported usersys
# botsglobal.usersysimportpath: used for imports from usersys
botsglobal.usersysimportpath = importnameforusersys
# 'directories','botssys': absolute path for config botssys
botssys = botsglobal.ini.get('directories','botssys','botssys')
botsglobal.ini.set('directories','botssys_org',botssys) #store original botssys setting
botsglobal.ini.set('directories','botssys',botslib.join(botssys))
botsglobal.ini.set('directories','data',botslib.join(botssys,'data'))
botslib.dirshouldbethere(botsglobal.ini.get('directories','data'))
botsglobal.ini.set('directories','logging',botslib.join(botssys,'logging'))
botslib.dirshouldbethere(botsglobal.ini.get('directories','logging'))
botsglobal.ini.set('directories','templates',botslib.join(botsglobal.ini.get('directories','usersysabs'),'grammars/template/templates'))
botsglobal.ini.set('directories','templateshtml',botslib.join(botsglobal.ini.get('directories','usersysabs'),'grammars/templatehtml/templates'))
if botsglobal.ini.get('webserver','environment','development') != 'development': #values in bots.ini are also used in setting up cherrypy
logging.raiseExceptions = 0 # during production: if errors occurs in writing to log: ignore error. (leads to a missing log line, better than error;-).
#initialise bots charsets
initbotscharsets()
#set environment for django to start***************************************************************************************************
os.environ['DJANGO_SETTINGS_MODULE'] = importnameforsettings
botslib.settimeout(botsglobal.ini.getint('settings','globaltimeout',10))
