def startmulti(grammardir,editype): ''' specialized tool for bulk checking of grammars while developing botsgrammars grammardir: directory with gramars (eg bots/usersys/grammars/edifact) editype: eg edifact ''' configdir = 'config' botsinit.generalinit(configdir) #find locating of bots, configfiles, init paths etc. process_name = 'grammarcheck' botsglobal.logger = botsinit.initenginelogging(process_name) atexit.register(logging.shutdown) for filename in glob.iglob(grammardir): filename_basename = os.path.basename(filename) if filename_basename in ['__init__.py','envelope.py']: continue if filename_basename.startswith('edifact') or filename_basename.startswith('records') or filename_basename.endswith('records.py'): continue if filename_basename.endswith('pyc'): continue filename_noextension = os.path.splitext(filename_basename)[0] try: grammar.grammarread(editype,filename_noextension) except: print botslib.txtexc() print '\n' else: print 'OK - no error found in grammar',filename,'\n'
def writeall(self): ''' Very different writeall: there is no tree of nodes; there is no grammar.structure/recorddefs; kid opens file by itself. ''' try: import kid except: txt=botslib.txtexc() raise ImportError(_(u'Dependency failure: editype "template" requires python library "kid". Error:\n%s'%txt)) #for template-grammar: only syntax is used. Section 'syntax' has to have 'template' self.outmessagegrammarread(self.ta_info['editype'],self.ta_info['messagetype']) templatefile = botslib.abspath(u'templates',self.ta_info['template']) try: botsglobal.logger.debug(u'Start writing to file "%s".',self.ta_info['filename']) ediprint = kid.Template(file=templatefile, data=self.data) except: txt=botslib.txtexc() raise botslib.OutMessageError(_(u'While templating "$editype.$messagetype", error:\n$txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt) try: f = botslib.opendata(self.ta_info['filename'],'wb') ediprint.write(f, #~ ediprint.write(botslib.abspathdata(self.ta_info['filename']), encoding=self.ta_info['charset'], output=self.ta_info['output'], #output is specific parameter for class; init from grammar.syntax fragment=self.ta_info['merge']) except: txt=botslib.txtexc() raise botslib.OutMessageError(_(u'While templating "$editype.$messagetype", error:\n$txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt) botsglobal.logger.debug(_(u'End writing to file "%s".'),self.ta_info['filename'])
def run(self): ''' class for (test) orderprint; delevers a valid html-file. Uses a kid-template for the enveloping/merging. use kid to write; no envelope grammar is used ''' try: from genshi.template import TemplateLoader except: txt=botslib.txtexc() raise ImportError(_(u'Dependency failure: editype "template" requires python library "genshi". Error:\n%s'%txt)) defmessage = grammar.grammarread(self.ta_info['editype'],self.ta_info['messagetype']) #needed because we do not know envelope; read syntax for editype/messagetype self.ta_info.update(defmessage.syntax) botslib.tryrunscript(self.userscript,self.scriptname,'ta_infocontent',ta_info=self.ta_info) if not self.ta_info['envelope-template']: raise botslib.OutMessageError(_(u'While enveloping in "$editype.$messagetype": syntax option "envelope-template" not filled; is required.'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype']) templatefile = botslib.abspath('templateshtml',self.ta_info['envelope-template']) ta_list = self.filelist2absolutepaths() try: botsglobal.logger.debug(u'Start writing envelope to file "%s".',self.ta_info['filename']) loader = TemplateLoader(auto_reload=False) tmpl = loader.load(templatefile) except: txt=botslib.txtexc() raise botslib.OutMessageError(_(u'While enveloping in "$editype.$messagetype", error:\n$txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt) try: f = botslib.opendata(self.ta_info['filename'],'wb') stream = tmpl.generate(data=ta_list) stream.render(method='xhtml',encoding=self.ta_info['charset'],out=f) except: txt=botslib.txtexc() raise botslib.OutMessageError(_(u'While enveloping in "$editype.$messagetype", error:\n$txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt)
def read_index(filename): ''' process index file in default location. ''' try: importedbotsindex, scriptname = botslib.botsimport('index') pluglist = importedbotsindex.plugins[:] if importedbotsindex.__name__ in sys.modules: del sys.modules[importedbotsindex.__name__] except: txt = botslib.txtexc() raise botslib.PluginError( _(u'Error in configuration index file. Nothing is written. Error:\n%(txt)s' ), {'txt': txt}) else: botsglobal.logger.info(_(u'Configuration index file is OK.')) botsglobal.logger.info(_(u'Start writing to database.')) #write content of index file to the bots database try: read_index2database(pluglist) except: txt = botslib.txtexc() raise botslib.PluginError( _(u'Error writing configuration index to database. Nothing is written. Error:\n%(txt)s' ), {'txt': txt}) else: botsglobal.logger.info(_(u'Writing to database is OK.'))
def startmulti(grammardir, editype): ''' specialized tool for bulk checking of grammars while developing botsgrammars grammardir: directory with gramars (eg bots/usersys/grammars/edifact) editype: eg edifact ''' configdir = 'config' botsinit.generalinit( configdir) #find locating of bots, configfiles, init paths etc. process_name = 'grammarcheck' botsglobal.logger = botsinit.initenginelogging(process_name) atexit.register(logging.shutdown) for filename in glob.iglob(grammardir): filename_basename = os.path.basename(filename) if filename_basename in ['__init__.py', 'envelope.py']: continue if filename_basename.startswith( 'edifact') or filename_basename.startswith( 'records') or filename_basename.endswith('records.py'): continue if filename_basename.endswith('pyc'): continue filename_noextension = os.path.splitext(filename_basename)[0] try: grammar.grammarread(editype, filename_noextension) except: print botslib.txtexc() print '\n' else: print 'OK - no error found in grammar', filename, '\n'
def start(): #********command line arguments************************** editype = '' messagetype = '' configdir = 'config' for arg in sys.argv[1:]: if not arg: continue if arg.startswith('-c'): configdir = arg[2:] if not configdir: print ' !!Indicated Bots should use specific .ini file but no file name was given.' showusage() elif arg in ["?", "/?"] or arg.startswith('-'): showusage() else: if not editype: editype = arg else: messagetype = arg if not (editype and messagetype): print ' !!Both editype and messagetype are required.' showusage() #********end handling command line arguments************************** try: botsinit.generalinit(configdir) botsinit.initenginelogging() grammar.grammarread(editype, messagetype) except: print 'Found error in grammar:' print botslib.txtexc() else: print 'OK - no error found in grammar'
def read_index(filename): """ process index file in default location. """ try: importedbotsindex, scriptname = botslib.botsimport("index") pluglist = importedbotsindex.plugins[:] if importedbotsindex.__name__ in sys.modules: del sys.modules[importedbotsindex.__name__] except: txt = botslib.txtexc() raise botslib.PluginError( _(u"Error in configuration index file. Nothing is written. Error:\n%(txt)s"), {"txt": txt} ) else: botsglobal.logger.info(_(u"Configuration index file is OK.")) botsglobal.logger.info(_(u"Start writing to database.")) # write content of index file to the bots database try: read_index2database(pluglist) except: txt = botslib.txtexc() raise botslib.PluginError( _(u"Error writing configuration index to database. Nothing is written. Error:\n%(txt)s"), {"txt": txt} ) else: botsglobal.logger.info(_(u"Writing to database is OK."))
def writeall(self): ''' Very different writeall: there is no tree of nodes; there is no grammar.structure/recorddefs; kid opens file by itself. ''' try: import kid except: txt = botslib.txtexc() raise ImportError(_(u'Dependency failure: editype "template" requires python library "kid". Error:\n%s'%txt)) #for template-grammar: only syntax is used. Section 'syntax' has to have 'template' self.outmessagegrammarread(self.ta_info['editype'],self.ta_info['messagetype']) templatefile = botslib.abspath(u'templates',self.ta_info['template']) try: botsglobal.logger.debug(u'Start writing to file "%s".',self.ta_info['filename']) ediprint = kid.Template(file=templatefile, data=self.data) except: txt = botslib.txtexc() raise botslib.OutMessageError(_(u'While templating "$editype.$messagetype", error:\n$txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt) try: filehandler = botslib.opendata(self.ta_info['filename'],'wb') ediprint.write(filehandler, encoding=self.ta_info['charset'], output=self.ta_info['output'], #output is specific parameter for class; init from grammar.syntax fragment=self.ta_info['merge']) except: txt = botslib.txtexc() raise botslib.OutMessageError(_(u'While templating "$editype.$messagetype", error:\n$txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt) botsglobal.logger.debug(_(u'End writing to file "%s".'),self.ta_info['filename'])
def writeall(self): ''' Very different writeall: there is no tree of nodes; there is no grammar.structure/recorddefs; kid opens file by itself. ''' try: from genshi.template import TemplateLoader except: txt=botslib.txtexc() raise ImportError(_(u'Dependency failure: editype "template" requires python library "genshi". Error:\n%s'%txt)) #for template-grammar: only syntax is used. Section 'syntax' has to have 'template' self.outmessagegrammarread(self.ta_info['editype'],self.ta_info['messagetype']) templatefile = botslib.abspath(u'templateshtml',self.ta_info['template']) try: botsglobal.logger.debug(u'Start writing to file "%s".',self.ta_info['filename']) loader = TemplateLoader(auto_reload=False) tmpl = loader.load(templatefile) except: txt=botslib.txtexc() raise botslib.OutMessageError(_(u'While templating "$editype.$messagetype", error:\n$txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt) try: f = botslib.opendata(self.ta_info['filename'],'wb') stream = tmpl.generate(data=self.data) stream.render(method='xhtml',encoding=self.ta_info['charset'],out=f) except: txt=botslib.txtexc() raise botslib.OutMessageError(_(u'While templating "$editype.$messagetype", error:\n$txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt) botsglobal.logger.debug(_(u'End writing to file "%s".'),self.ta_info['filename'])
def run(self): ''' class for (test) orderprint; delevers a valid html-file. Uses a kid-template for the enveloping/merging. use kid to write; no envelope grammar is used ''' import kid defmessage = grammar.grammarread(self.ta_info['editype'],self.ta_info['messagetype']) #needed beause we do not know envelope; read syntax for editype/messagetype self.ta_info.update(defmessage.syntax) botslib.tryrunscript(self.userscript,self.scriptname,'ta_infocontent',ta_info=self.ta_info) if not self.ta_info['envelope-template']: raise botslib.OutMessageError(_(u'While enveloping in "$editype.$messagetype": syntax option "envelope-template" not filled; is required.'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype']) templatefile = botslib.abspath('templates',self.ta_info['envelope-template']) ta_list = self.filelist2absolutepaths() try: botsglobal.logger.debug(u'Start writing envelope to file "%s".',self.ta_info['filename']) ediprint = kid.Template(file=templatefile, data=ta_list) #init template; pass list with filenames except: txt=botslib.txtexc() raise botslib.OutMessageError(_(u'While enveloping in "$editype.$messagetype": $txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt) try: f = botslib.opendata(self.ta_info['filename'],'wb') ediprint.write(f, encoding=self.ta_info['charset'], output=self.ta_info['output']) except: txt=botslib.txtexc() raise botslib.OutMessageError(_(u'While enveloping error in "$editype.$messagetype"; probably in html file(s) to be merged: $txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt)
def start(): #********command line arguments************************** editype ='' messagetype = '' configdir = 'config' for arg in sys.argv[1:]: if not arg: continue if arg.startswith('-c'): configdir = arg[2:] if not configdir: print ' !!Indicated Bots should use specific .ini file but no file name was given.' showusage() elif arg in ["?", "/?"] or arg.startswith('-'): showusage() else: if not editype: editype = arg else: messagetype = arg if not (editype and messagetype): print ' !!Both editype and messagetype are required.' showusage() #********end handling command line arguments************************** try: botsinit.generalinit(configdir) botsinit.initenginelogging() grammar.grammarread(editype,messagetype) except: print 'Found error in grammar:' print botslib.txtexc() else: print 'OK - no error found in grammar'
def runengine(request, *kw, **kwargs): if request.method == "GET": # ~ logger = logging.getLogger('bots') if os.name == "nt": lijst = [sys.executable, os.path.normpath(os.path.join(sys.prefix, "Scripts", "bots-engine.py"))] elif os.path.exists(os.path.join(sys.prefix, "bin", "bots-engine.py")): lijst = [os.path.normpath(os.path.join(sys.prefix, "bin", "bots-engine.py"))] elif os.path.exists(os.path.join(sys.prefix, "local/bin", "bots-engine.py")): lijst = [os.path.normpath(os.path.join(sys.prefix, "local/bin", "bots-engine.py"))] else: request.user.message_set.create(message=_(u"Bots can not find executable for bots-engine.")) # ~ logger.info('Bots can not find executable for bots-engine.') return django.shortcuts.redirect("/") try: if "clparameter" in request.GET: lijst.append(request.GET["clparameter"]) # ~ logger.info('Run bots-engine with parameters: "%s"',str(lijst)) terug = subprocess.Popen(lijst).pid request.user.message_set.create(message=_(u"Bots-engine is started.")) # ~ logger.info('Bots-engine is started.') except: print botslib.txtexc() request.user.message_set.create(message=_(u"Errors while trying to run bots-engine.")) # ~ logger.info('Errors while trying to run bots-engine.') return django.shortcuts.redirect("/")
def run(self): ''' class for template enveloping; delevers a valid html-file. Uses a kid-template for the enveloping/merging. use kid to write; no envelope grammar is used #20120101 depreciated. use class templatehtml ''' try: import kid except: raise ImportError(_(u'Dependency failure: editype "template" requires python library "kid".')) self._openoutenvelope() self.ta_info.update(self.out.ta_info) botslib.tryrunscript(self.userscript,self.scriptname,'ta_infocontent',ta_info=self.ta_info) if not self.ta_info['envelope-template']: raise botslib.OutMessageError(_(u'While enveloping in "%(editype)s.%(messagetype)s": syntax option "envelope-template" not filled; is required.'), self.ta_info) templatefile = botslib.abspath('templates',self.ta_info['envelope-template']) ta_list = self.filelist2absolutepaths() try: botsglobal.logger.debug(u'Start writing envelope to file "%(filename)s".',self.ta_info) ediprint = kid.Template(file=templatefile, data=ta_list) #init template; pass list with filenames except: txt = botslib.txtexc() raise botslib.OutMessageError(_(u'While enveloping in "%(editype)s.%(messagetype)s", error:\n%(txt)s'), {'editype':self.ta_info['editype'],'messagetype':self.ta_info['messagetype'],'txt':txt}) try: filehandler = botslib.opendata(self.ta_info['filename'],'wb') ediprint.write(filehandler, encoding=self.ta_info['charset'], output=self.ta_info['output']) except: txt = botslib.txtexc() raise botslib.OutMessageError(_(u'While enveloping in "%(editype)s.%(messagetype)s", error:\n%(txt)s'), {'editype':self.ta_info['editype'],'messagetype':self.ta_info['messagetype'],'txt':txt})
def writeall(self): ''' Very different writeall: there is no tree of nodes; there is no grammar.structure/recorddefs; kid opens file by itself. ''' try: from genshi.template import TemplateLoader except: txt = botslib.txtexc() raise ImportError(_(u'Dependency failure: editype "template" requires python library "genshi". Error:\n%s'%txt)) #for template-grammar: only syntax is used. Section 'syntax' has to have 'template' self.outmessagegrammarread(self.ta_info['editype'],self.ta_info['messagetype']) templatefile = botslib.abspath(u'templateshtml',self.ta_info['template']) try: botsglobal.logger.debug(u'Start writing to file "%s".',self.ta_info['filename']) loader = TemplateLoader(auto_reload=False) tmpl = loader.load(templatefile) except: txt = botslib.txtexc() raise botslib.OutMessageError(_(u'While templating "$editype.$messagetype", error:\n$txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt) try: filehandler = botslib.opendata(self.ta_info['filename'],'wb') stream = tmpl.generate(data=self.data) stream.render(method='xhtml',encoding=self.ta_info['charset'],out=filehandler) except: txt = botslib.txtexc() raise botslib.OutMessageError(_(u'While templating "$editype.$messagetype", error:\n$txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt) botsglobal.logger.debug(_(u'End writing to file "%s".'),self.ta_info['filename'])
def run(self): try: from genshi.template import TemplateLoader except: raise ImportError(u'Dependency failure: editype "templatehtml" requires python library "genshi".') self._openoutenvelope() self.ta_info.update(self.out.ta_info) botslib.tryrunscript(self.userscript,self.scriptname,'ta_infocontent',ta_info=self.ta_info) if not self.ta_info['envelope-template']: raise botslib.OutMessageError(_(u'While enveloping in "%(editype)s.%(messagetype)s": syntax option "envelope-template" not filled; is required.'), self.ta_info) templatefile = botslib.abspath(self.__class__.__name__,self.ta_info['envelope-template']) ta_list = self.filelist2absolutepaths() try: botsglobal.logger.debug(u'Start writing envelope to file "%(filename)s".',self.ta_info) loader = TemplateLoader(auto_reload=False) tmpl = loader.load(templatefile) except: txt = botslib.txtexc() raise botslib.OutMessageError(_(u'While enveloping in "%(editype)s.%(messagetype)s", error:\n%(txt)s'), {'editype':self.ta_info['editype'],'messagetype':self.ta_info['messagetype'],'txt':txt}) try: filehandler = botslib.opendata(self.ta_info['filename'],'wb') stream = tmpl.generate(data=ta_list) stream.render(method='xhtml',encoding=self.ta_info['charset'],out=filehandler) except: txt = botslib.txtexc() raise botslib.OutMessageError(_(u'While enveloping in "%(editype)s.%(messagetype)s", error:\n%(txt)s'), {'editype':self.ta_info['editype'],'messagetype':self.ta_info['messagetype'],'txt':txt})
def start(): # ********command line arguments************************** editype = "" messagetype = "" botsinifile = "config" for arg in sys.argv[1:]: if not arg: continue if arg.startswith("-c"): botsinifile = arg[2:] if not botsinifile: print " !!Indicated Bots should use specific .ini file but no file name was given." showusage() elif arg in ["?", "/?"] or arg.startswith("-"): showusage() else: if not editype: editype = arg else: messagetype = arg if not (editype and messagetype): print " !!Both editype and messagetype are required." showusage() # ********end handling command line arguments************************** try: botslib.initconfigurationfile(botsinifile) botslib.initlogging() grammar.grammarread(editype, messagetype) except: print "Found error in grammar:" print botslib.txtexc() else: print "OK - no error found in grammar"
def run(self): ''' class for (test) orderprint; delevers a valid html-file. Uses a kid-template for the enveloping/merging. use kid to write; no envelope grammar is used ''' try: import kid except: txt = botslib.txtexc() raise ImportError( _(u'Dependency failure: editype "template" requires python library "kid". Error:\n%s' % txt)) defmessage = grammar.grammarread( self.ta_info['editype'], self.ta_info['messagetype'] ) #needed because we do not know envelope; read syntax for editype/messagetype self.ta_info.update(defmessage.syntax) botslib.tryrunscript(self.userscript, self.scriptname, 'ta_infocontent', ta_info=self.ta_info) if not self.ta_info['envelope-template']: raise botslib.OutMessageError(_( u'While enveloping in "$editype.$messagetype": syntax option "envelope-template" not filled; is required.' ), editype=self.ta_info['editype'], messagetype=self. ta_info['messagetype']) templatefile = botslib.abspath('templates', self.ta_info['envelope-template']) ta_list = self.filelist2absolutepaths() try: botsglobal.logger.debug(u'Start writing envelope to file "%s".', self.ta_info['filename']) ediprint = kid.Template( file=templatefile, data=ta_list) #init template; pass list with filenames except: txt = botslib.txtexc() raise botslib.OutMessageError( _(u'While enveloping in "$editype.$messagetype", error:\n$txt' ), editype=self.ta_info['editype'], messagetype=self.ta_info['messagetype'], txt=txt) try: filehandler = botslib.opendata(self.ta_info['filename'], 'wb') ediprint.write(filehandler, encoding=self.ta_info['charset'], output=self.ta_info['output']) except: txt = botslib.txtexc() raise botslib.OutMessageError( _(u'While enveloping in "$editype.$messagetype", error:\n$txt' ), editype=self.ta_info['editype'], messagetype=self.ta_info['messagetype'], txt=txt)
def run(self): ''' class for (test) orderprint; delevers a valid html-file. Uses a kid-template for the enveloping/merging. use kid to write; no envelope grammar is used ''' try: from genshi.template import TemplateLoader except: txt = botslib.txtexc() raise ImportError( _(u'Dependency failure: editype "template" requires python library "genshi". Error:\n%s' % txt)) defmessage = grammar.grammarread( self.ta_info['editype'], self.ta_info['messagetype'] ) #needed because we do not know envelope; read syntax for editype/messagetype self.ta_info.update(defmessage.syntax) botslib.tryrunscript(self.userscript, self.scriptname, 'ta_infocontent', ta_info=self.ta_info) if not self.ta_info['envelope-template']: raise botslib.OutMessageError(_( u'While enveloping in "$editype.$messagetype": syntax option "envelope-template" not filled; is required.' ), editype=self.ta_info['editype'], messagetype=self. ta_info['messagetype']) templatefile = botslib.abspath('templateshtml', self.ta_info['envelope-template']) ta_list = self.filelist2absolutepaths() try: botsglobal.logger.debug(u'Start writing envelope to file "%s".', self.ta_info['filename']) loader = TemplateLoader(auto_reload=False) tmpl = loader.load(templatefile) except: txt = botslib.txtexc() raise botslib.OutMessageError( _(u'While enveloping in "$editype.$messagetype", error:\n$txt' ), editype=self.ta_info['editype'], messagetype=self.ta_info['messagetype'], txt=txt) try: filehandler = botslib.opendata(self.ta_info['filename'], 'wb') stream = tmpl.generate(data=ta_list) stream.render(method='xhtml', encoding=self.ta_info['charset'], out=filehandler) except: txt = botslib.txtexc() raise botslib.OutMessageError( _(u'While enveloping in "$editype.$messagetype", error:\n$txt' ), editype=self.ta_info['editype'], messagetype=self.ta_info['messagetype'], txt=txt)
def mergemessages(run): names_envelope_criteria = ('editype', 'messagetype', 'frompartner', 'topartner', 'testindicator', 'charset', 'contenttype', 'envelope', 'rsrv3') merge_yes = {} merge_no = [] #can be non-unique: as a list #walk over run.translated; sort by envelope-criteria in 2 dicts for translated in run.translated: filename = translated.get('filename') infilename = translated.get('infilename') nrmessages = translated.get('nrmessages') if translated.get('merge'): env_criteria = tuple( translated.get(field) for field in names_envelope_criteria) if env_criteria in merge_yes: merge_yes[env_criteria][0].append(filename) merge_yes[env_criteria][1].append(infilename) merge_yes[env_criteria][2] += nrmessages else: merge_yes[env_criteria] = [[filename], [infilename], nrmessages] else: ta_info = dict((field, translated.get(field)) for field in names_envelope_criteria) ta_info['nrmessages'] = nrmessages merge_no.append((ta_info, [filename], [infilename])) #envelope for env_criteria, rest_of_info in merge_yes.iteritems(): ta_info = dict(zip(names_envelope_criteria, env_criteria)) ta_info['filename'] = transform.unique( 'bots_file_name') #create filename for enveloped message ta_info['nrmessages'] = rest_of_info[2] ta_info['infilename'] = rest_of_info[ 1] #for reference: list of infilenames ta_info['error'] = '' try: envelope.envelope(ta_info, rest_of_info[0]) except: txt = botslib.txtexc() ta_info['error'] = txt.strip() finally: run.outgoing.append(ta_info) for ta_info, filenames, infilenames in merge_no: ta_info['filename'] = transform.unique( 'bots_file_name') #create filename for enveloped message ta_info[ 'infilename'] = infilenames #for reference: list of infilenames ta_info['error'] = '' try: envelope.envelope(ta_info, filenames) except: txt = botslib.txtexc() ta_info['error'] = txt.strip() finally: run.outgoing.append(ta_info)
def run(self): try: from genshi.template import TemplateLoader except: raise ImportError( u'Dependency failure: editype "templatehtml" requires python library "genshi".' ) self._openoutenvelope() self.ta_info.update(self.out.ta_info) botslib.tryrunscript(self.userscript, self.scriptname, 'ta_infocontent', ta_info=self.ta_info) if not self.ta_info['envelope-template']: raise botslib.OutMessageError( _(u'While enveloping in "%(editype)s.%(messagetype)s": syntax option "envelope-template" not filled; is required.' ), self.ta_info) templatefile = botslib.abspath(self.__class__.__name__, self.ta_info['envelope-template']) ta_list = self.filelist2absolutepaths() try: botsglobal.logger.debug( u'Start writing envelope to file "%(filename)s".', self.ta_info) loader = TemplateLoader(auto_reload=False) tmpl = loader.load(templatefile) except: txt = botslib.txtexc() raise botslib.OutMessageError( _(u'While enveloping in "%(editype)s.%(messagetype)s", error:\n%(txt)s' ), { 'editype': self.ta_info['editype'], 'messagetype': self.ta_info['messagetype'], 'txt': txt }) try: filehandler = botslib.opendata(self.ta_info['filename'], 'wb') stream = tmpl.generate(data=ta_list) stream.render(method='xhtml', encoding=self.ta_info['charset'], out=filehandler) except: txt = botslib.txtexc() raise botslib.OutMessageError( _(u'While enveloping in "%(editype)s.%(messagetype)s", error:\n%(txt)s' ), { 'editype': self.ta_info['editype'], 'messagetype': self.ta_info['messagetype'], 'txt': txt })
def run(self): """ class for (test) orderprint; delevers a valid html-file. Uses a kid-template for the enveloping/merging. use kid to write; no envelope grammar is used """ try: import kid except: txt = botslib.txtexc() raise ImportError( _(u'Dependency failure: editype "template" requires python library "kid". Error:\n%s' % txt) ) defmessage = grammar.grammarread( self.ta_info["editype"], self.ta_info["messagetype"] ) # needed because we do not know envelope; read syntax for editype/messagetype self.ta_info.update(defmessage.syntax) botslib.tryrunscript(self.userscript, self.scriptname, "ta_infocontent", ta_info=self.ta_info) if not self.ta_info["envelope-template"]: raise botslib.OutMessageError( _( u'While enveloping in "$editype.$messagetype": syntax option "envelope-template" not filled; is required.' ), editype=self.ta_info["editype"], messagetype=self.ta_info["messagetype"], ) templatefile = botslib.abspath("templates", self.ta_info["envelope-template"]) ta_list = self.filelist2absolutepaths() try: botsglobal.logger.debug(u'Start writing envelope to file "%s".', self.ta_info["filename"]) ediprint = kid.Template(file=templatefile, data=ta_list) # init template; pass list with filenames except: txt = botslib.txtexc() raise botslib.OutMessageError( _(u'While enveloping in "$editype.$messagetype", error:\n$txt'), editype=self.ta_info["editype"], messagetype=self.ta_info["messagetype"], txt=txt, ) try: filehandler = botslib.opendata(self.ta_info["filename"], "wb") ediprint.write(filehandler, encoding=self.ta_info["charset"], output=self.ta_info["output"]) except: txt = botslib.txtexc() raise botslib.OutMessageError( _(u'While enveloping in "$editype.$messagetype", error:\n$txt'), editype=self.ta_info["editype"], messagetype=self.ta_info["messagetype"], txt=txt, )
def sendtestmailmanagers(request, *kw, **kwargs): try: sendornot = botsglobal.ini.getboolean('settings', 'sendreportiferror', False) except botslib.BotsError: sendornot = False if not sendornot: notification = _( u'Trying to send test mail, but in bots.ini, section [settings], "sendreportiferror" is not "True".' ) botsglobal.logger.info(notification) messages.add_message(request, messages.INFO, notification) return django.shortcuts.redirect('/home') from django.core.mail import mail_managers try: mail_managers(_(u'testsubject'), _(u'test content of report')) except: txt = botslib.txtexc() messages.add_message(request, messages.INFO, _(u'Sending test mail failed.')) botsglobal.logger.info(_(u'Sending test mail failed, error:\n%(txt)s'), {'txt': txt}) return django.shortcuts.redirect('/home') notification = _(u'Sending test mail succeeded.') messages.add_message(request, messages.INFO, notification) botsglobal.logger.info(notification) return django.shortcuts.redirect('/home')
def postprocess(routedict,function,status=FILEOUT,rootidta=None,**argv): ''' for postprocessing of files. these are NOT translations; translation involve grammars, mapping scripts etc. think of eg: - zip files. If errors occur during processing, no ta are left with status FILEOUT ! postprocess is called right before the out-communicatiation ''' if rootidta is None: rootidta = botsglobal.currentrun.get_minta4query_routepart() nr_files = 0 for row in botslib.query(u'''SELECT idta,filename FROM ta WHERE idta>%(rootidta)s AND status=%(status)s AND statust=%(statust)s AND idroute=%(idroute)s AND tochannel=%(tochannel)s ''', {'status':status,'statust':OK,'idroute':routedict['idroute'],'tochannel':routedict['tochannel'],'rootidta':rootidta}): try: botsglobal.logger.debug(u'Start postprocessing "%(name)s" for file "%(filename)s".', {'name':function.__name__,'filename':row['filename']}) ta_from = botslib.OldTransaction(row['idta']) ta_from.filename = row['filename'] function(ta_from=ta_from,endstatus=status,routedict=routedict,**argv) except: txt = botslib.txtexc() ta_from.update(statust=ERROR,errortext=txt) ta_from.deletechildren() else: botsglobal.logger.debug(u'OK postprocessing "%(name)s" for file "%(filename)s".', {'name':function.__name__,'filename':row['filename']}) ta_from.update(statust=DONE) nr_files += 1 return nr_files
def mergemessages(run): names_envelope_criteria = ('editype','messagetype','frompartner','topartner','testindicator','charset','contenttype','envelope','rsrv3') merge_yes = {} merge_no = [] #can be non-unique: as a list #walk over run.translated; sort by envelope-criteria in 2 dicts for translated in run.translated: filename = translated.get('filename') infilename = translated.get('infilename') nrmessages = translated.get('nrmessages') if translated.get('merge'): env_criteria = tuple(translated.get(field) for field in names_envelope_criteria) if env_criteria in merge_yes: merge_yes[env_criteria][0].append(filename) merge_yes[env_criteria][1].append(infilename) merge_yes[env_criteria][2] += nrmessages else: merge_yes[env_criteria] = [[filename],[infilename],nrmessages] else: ta_info = dict((field,translated.get(field)) for field in names_envelope_criteria) ta_info['nrmessages'] = nrmessages merge_no.append((ta_info, [filename],[infilename])) #envelope for env_criteria,rest_of_info in merge_yes.iteritems(): ta_info = dict(zip(names_envelope_criteria,env_criteria)) ta_info['filename'] = transform.unique('bots_file_name') #create filename for enveloped message ta_info['nrmessages'] = rest_of_info[2] ta_info['infilename'] = rest_of_info[1] #for reference: list of infilenames ta_info['error'] = '' try: envelope.envelope(ta_info,rest_of_info[0]) except: txt = botslib.txtexc() ta_info['error'] = txt.strip() finally: run.outgoing.append(ta_info) for ta_info,filenames,infilenames in merge_no: ta_info['filename'] = transform.unique('bots_file_name') #create filename for enveloped message ta_info['infilename'] = infilenames #for reference: list of infilenames ta_info['error'] = '' try: envelope.envelope(ta_info,filenames) except: txt = botslib.txtexc() ta_info['error'] = txt.strip() finally: run.outgoing.append(ta_info)
def preprocess(routedict, function, status=FILEIN, **argv): ''' for pre- and postprocessing of files. these are NOT translations; translation involve grammars, mapping scripts etc. think of eg: - unzipping zipped files. - convert excel to csv - password protected files. Select files from INFILE -> SET_FOR_PROCESSING using criteria Than the actual processing function is called. The processing function does: SET_FOR_PROCESSING -> PROCESSING -> FILEIN If errors occur during processing, no ta are left with status FILEIN ! preprocess is called right after the in-communicatiation ''' nr_files = 0 preprocessnumber = botslib.getpreprocessnumber() if not botslib.addinfo(change={'status': preprocessnumber}, where={ 'status': status, 'idroute': routedict['idroute'], 'fromchannel': routedict['fromchannel'] }): #check if there is something to do return 0 for row in botslib.query( u'''SELECT idta,filename,charset FROM ta WHERE idta>%(rootidta)s AND status=%(status)s AND statust=%(statust)s AND idroute=%(idroute)s AND fromchannel=%(fromchannel)s ''', { 'status': preprocessnumber, 'statust': OK, 'idroute': routedict['idroute'], 'fromchannel': routedict['fromchannel'], 'rootidta': botslib.get_minta4query() }): try: botsglobal.logmap.debug(u'Start preprocessing "%s" for file "%s".', function.__name__, row['filename']) ta_set_for_processing = botslib.OldTransaction(row['idta']) ta_processing = ta_set_for_processing.copyta( status=preprocessnumber + 1) ta_processing.filename = row['filename'] function(ta_from=ta_processing, endstatus=status, routedict=routedict, **argv) except: txt = botslib.txtexc() ta_processing.failure() ta_processing.update(statust=ERROR, errortext=txt) else: botsglobal.logmap.debug(u'OK preprocessing "%s" for file "%s".', function.__name__, row['filename']) ta_set_for_processing.update(statust=DONE) ta_processing.update(statust=DONE) nr_files += 1 return nr_files
def start(): #NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start. #********command line arguments************************** usage = ''' This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net). Checks a Bots grammar. Same checks are used as in translations with bots-engine. Searches for grammar in regular place: bots/usersys/grammars/<editype>/<messagetype>.py (even if a path is passed). Usage: %(name)s -c<directory> <editype> <messagetype> or %(name)s -c<directory> <path to grammar> Options: -c<directory> directory for configuration files (default: config). Examples: %(name)s -cconfig edifact ORDERSD96AUNEAN008 %(name)s -cconfig C:/python27/lib/site-packages/bots/usersys/grammars/edifact/ORDERSD96AUNEAN008.py '''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version} configdir = 'config' editype ='' messagetype = '' for arg in sys.argv[1:]: if arg.startswith('-c'): configdir = arg[2:] if not configdir: print 'Error: configuration directory indicated, but no directory name.' sys.exit(1) elif arg in ["?", "/?",'-h', '--help'] or arg.startswith('-'): print usage sys.exit(0) else: if os.path.isfile(arg): p1,p2 = os.path.split(arg) editype = os.path.basename(p1) messagetype,ext = os.path.splitext(p2) messagetype = str(messagetype) print 'grammarcheck',editype,messagetype elif not editype: editype = arg else: messagetype = arg if not (editype and messagetype): print 'Error: both editype and messagetype, or a file path, are required.' sys.exit(1) #***end handling command line arguments************************** botsinit.generalinit(configdir) #find locating of bots, configfiles, init paths etc. process_name = 'grammarcheck' botsglobal.logger = botsinit.initenginelogging(process_name) atexit.register(logging.shutdown) try: grammar.grammarread(editype,messagetype) except: print 'Found error in grammar: ',botslib.txtexc() sys.exit(1) else: print 'OK - no error found in grammar' sys.exit(0)
def startmulti(grammardir,editype): import glob botslib.initconfigurationfile('config') botslib.initlogging() for g in glob.glob(grammardir): g1 = os.path.basename(g) g2 = os.path.splitext(g1)[0] if g1 in ['__init__.py']: continue if g1.startswith('edifact'): continue if g1.startswith('records') or g1.endswith('records.py'): continue try: grammar.grammarread(editype,g2) except: #~ print 'Found error in grammar:',g print botslib.txtexc() print '\n' else: print 'OK - no error found in grammar',g,'\n'
def runengine(request, *kw, **kwargs): if request.method == 'GET': #~ logger = logging.getLogger('bots') if os.name == 'nt': lijst = [ sys.executable, os.path.normpath( os.path.join(sys.prefix, 'Scripts', 'bots-engine.py')) ] elif os.path.exists(os.path.join(sys.prefix, 'bin', 'bots-engine.py')): lijst = [ os.path.normpath( os.path.join(sys.prefix, 'bin', 'bots-engine.py')) ] elif os.path.exists( os.path.join(sys.prefix, 'local/bin', 'bots-engine.py')): lijst = [ os.path.normpath( os.path.join(sys.prefix, 'local/bin', 'bots-engine.py')) ] else: request.user.message_set.create( message=_(u'Bots can not find executable for bots-engine.')) #~ logger.info('Bots can not find executable for bots-engine.') return django.shortcuts.redirect('/') try: if 'clparameter' in request.GET: lijst.append(request.GET['clparameter']) #~ logger.info('Run bots-engine with parameters: "%s"',str(lijst)) terug = subprocess.Popen(lijst).pid request.user.message_set.create( message=_(u'Bots-engine is started.')) #~ logger.info('Bots-engine is started.') except: print botslib.txtexc() request.user.message_set.create( message=_(u'Errors while trying to run bots-engine.')) #~ logger.info('Errors while trying to run bots-engine.') return django.shortcuts.redirect('/')
def startmulti(grammardir,editype): ''' used in seperate tool for bulk checking of gramamrs while developing edifact->botsgramamrs ''' import glob botslib.generalinit('config') botslib.initenginelogging() for g in glob.glob(grammardir): g1 = os.path.basename(g) g2 = os.path.splitext(g1)[0] if g1 in ['__init__.py']: continue if g1.startswith('edifact'): continue if g1.startswith('records') or g1.endswith('records.py'): continue try: grammar.grammarread(editype,g2) except: #~ print 'Found error in grammar:',g print botslib.txtexc() print '\n' else: print 'OK - no error found in grammar',g,'\n'
def startmulti(grammardir, editype): import glob botslib.initconfigurationfile("config") botslib.initlogging() for g in glob.glob(grammardir): g1 = os.path.basename(g) g2 = os.path.splitext(g1)[0] if g1 in ["__init__.py"]: continue if g1.startswith("edifact"): continue if g1.startswith("records") or g1.endswith("records.py"): continue try: grammar.grammarread(editype, g2) except: # ~ print 'Found error in grammar:',g print botslib.txtexc() print "\n" else: print "OK - no error found in grammar", g, "\n"
def startmulti(grammardir, editype): ''' used in seperate tool for bulk checking of gramamrs while developing edifact->botsgramamrs ''' import glob botslib.generalinit('config') botslib.initenginelogging() for g in glob.glob(grammardir): g1 = os.path.basename(g) g2 = os.path.splitext(g1)[0] if g1 in ['__init__.py']: continue if g1.startswith('edifact'): continue if g1.startswith('records') or g1.endswith('records.py'): continue try: grammar.grammarread(editype, g2) except: #~ print 'Found error in grammar:',g print botslib.txtexc() print '\n' else: print 'OK - no error found in grammar', g, '\n'
def read_index(filename): ''' process index file in default location. ''' try: importedbotsindex,scriptname = botslib.botsimport('index') pluglist = importedbotsindex.plugins[:] if 'botsindex' in sys.modules: del sys.modules['botsindex'] except: txt = botslib.txtexc() raise botslib.PluginError(_(u'Error in configuration index file. Nothing is written. Error:\n%(txt)s'),{'txt':txt}) else: botsglobal.logger.info(_(u'Configuration index file is OK.')) botsglobal.logger.info(_(u'Start writing to database.')) #write content of index file to the bots database try: read_index2database(pluglist) except: txt = botslib.txtexc() raise botslib.PluginError(_(u'Error writing configuration index to database. Nothing is written. Error:\n%(txt)s'),{'txt':txt}) else: botsglobal.logger.info(_(u'Writing to database is OK.'))
def _write(self,node_instance): templatefile = botslib.abspath(self.__class__.__name__,self.ta_info['template']) try: botsglobal.logger.debug(u'Start writing to file "%(filename)s".',self.ta_info) loader = self.template.TemplateLoader(auto_reload=False) tmpl = loader.load(templatefile) except: txt = botslib.txtexc() raise botslib.OutMessageError(_(u'While templating "%(editype)s.%(messagetype)s", error:\n%(txt)s'), {'editype':self.ta_info['editype'],'messagetype':self.ta_info['messagetype'],'txt':txt}) try: filehandler = botslib.opendata(self.ta_info['filename'],'wb') if self.ta_info['has_structure']: #new way of working if self.ta_info['print_as_row']: node_instance.collectlines(self.ta_info['print_as_row']) stream = tmpl.generate(node=node_instance) else: stream = tmpl.generate(data=self.data) stream.render(method='xhtml',encoding=self.ta_info['charset'],out=filehandler) except: txt = botslib.txtexc() raise botslib.OutMessageError(_(u'While templating "%(editype)s.%(messagetype)s", error:\n%(txt)s'), {'editype':self.ta_info['editype'],'messagetype':self.ta_info['messagetype'],'txt':txt}) botsglobal.logger.debug(_(u'End writing to file "%(filename)s".'),self.ta_info)
def read_incoming(run): outputdir = botslib.join(run.inpath,run.infilename) filelist = [filename for filename in glob.iglob(outputdir) if os.path.isfile(filename)] filelist.sort() for infilename in filelist: try: filename = transform.unique('bots_file_name') abs_filename = botslib.abspathdata(filename) shutil.copy(infilename,abs_filename) #move if to be delted except: txt = botslib.txtexc() else: txt = '' #no errors finally: run.incoming.append({'infilename':infilename,'filename':filename,'error':txt,'editype':run.translation['editype'],'messagetype':run.translation['messagetype']})
def write_outgoing(run): outputdir = botslib.join(run.outpath) botslib.dirshouldbethere(outputdir) for outgoing in run.outgoing: if not outgoing['error']: try: unique_filename = filename_formatter(run.outfilename,outgoing) tofilepath = botslib.join(outputdir,unique_filename) fromfilepath = botslib.abspathdata(outgoing['filename']) shutil.move(fromfilepath,tofilepath) except: txt = botslib.txtexc() outgoing.update({'error':txt}) else: outgoing.update({'outfilename':tofilepath})
def write_outgoing(run): outputdir = botslib.join(run.outpath) botslib.dirshouldbethere(outputdir) for outgoing in run.outgoing: if not outgoing['error']: try: unique_filename = filename_formatter(run.outfilename, outgoing) tofilepath = botslib.join(outputdir, unique_filename) fromfilepath = botslib.abspathdata(outgoing['filename']) shutil.move(fromfilepath, tofilepath) except: txt = botslib.txtexc() outgoing.update({'error': txt}) else: outgoing.update({'outfilename': tofilepath})
def preprocess(routedict,function, status=FILEIN,**argv): ''' for pre- and postprocessing of files. these are NOT translations; translation involve grammars, mapping scripts etc. think of eg: - unzipping zipped files. - convert excel to csv - password protected files. Select files from INFILE -> SET_FOR_PROCESSING using criteria Than the actual processing function is called. The processing function does: SET_FOR_PROCESSING -> PROCESSING -> FILEIN If errors occur during processing, no ta are left with status FILEIN ! preprocess is called right after the in-communicatiation ''' nr_files = 0 preprocessnumber = botslib.getpreprocessnumber() if not botslib.addinfo(change={'status':preprocessnumber},where={'status':status,'idroute':routedict['idroute'],'fromchannel':routedict['fromchannel']}): #check if there is something to do return 0 for row in botslib.query(u'''SELECT idta,filename,charset FROM ta WHERE idta>%(rootidta)s AND status=%(status)s AND statust=%(statust)s AND idroute=%(idroute)s AND fromchannel=%(fromchannel)s ''', {'status':preprocessnumber,'statust':OK,'idroute':routedict['idroute'],'fromchannel':routedict['fromchannel'],'rootidta':botslib.get_minta4query()}): try: botsglobal.logmap.debug(u'Start preprocessing "%s" for file "%s".',function.__name__,row['filename']) ta_set_for_processing = botslib.OldTransaction(row['idta']) ta_processing = ta_set_for_processing.copyta(status=preprocessnumber+1) ta_processing.filename = row['filename'] function(ta_from=ta_processing,endstatus=status,routedict=routedict,**argv) except: txt = botslib.txtexc() ta_processing.failure() ta_processing.update(statust=ERROR,errortext=txt) else: botsglobal.logmap.debug(u'OK preprocessing "%s" for file "%s".',function.__name__,row['filename']) ta_set_for_processing.update(statust=DONE) ta_processing.update(statust=DONE) nr_files += 1 return nr_files
def sendtestmailmanagers(request,*kw,**kwargs): try: sendornot = botsglobal.ini.getboolean('settings','sendreportiferror',False) except botslib.BotsError: sendornot = False if not sendornot: botsglobal.logger.info(_(u'Trying to send test mail, but in bots.ini, section [settings], "sendreportiferror" is not "True".')) request.user.message_set.create(message=_(u'Trying to send test mail, but in bots.ini, section [settings], "sendreportiferror" is not "True".')) return django.shortcuts.redirect('/home') from django.core.mail import mail_managers try: mail_managers(_(u'testsubject'), _(u'test content of report')) except: txt = botslib.txtexc() request.user.message_set.create(message=_(u'Sending test mail failed.')) botsglobal.logger.info(_(u'Sending test mail failed, error:\n%s'), txt) return django.shortcuts.redirect('/home') request.user.message_set.create(message=_(u'Sending test mail succeeded.')) botsglobal.logger.info(_(u'Sending test mail succeeded.')) return django.shortcuts.redirect('/home')
def sendtestmailmanagers(request, *kw, **kwargs): try: sendornot = botsglobal.ini.getboolean('settings', 'sendreportiferror', False) except botslib.BotsError: sendornot = False if not sendornot: request.user.message_set.create(message=_( u'In bots.ini, section [settings], "sendreportiferror" is not set (to "True").' )) return django.shortcuts.redirect('/') from django.core.mail import mail_managers try: mail_managers(_(u'testsubject'), _(u'test content of report')) except: request.user.message_set.create( message=_(u'Sending test report failed: "%s".') % botslib.txtexc()) return django.shortcuts.redirect('/') request.user.message_set.create( message=_(u'Sending test report succeeded.')) return django.shortcuts.redirect('/')
def runengine(request,*kw,**kwargs): if request.method == 'GET': #check if bots-engine is not already running if list(models.mutex.objects.filter(mutexk=1).all()): request.user.message_set.create(message=_(u'Trying to run "bots-engine", but database is locked by another run in progress. Please try again later.')) botsglobal.logger.info(_(u'Trying to run "bots-engine", but database is locked by another run in progress.')) return django.shortcuts.redirect('/home') #find out the right arguments to use botsenginepath = os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])),'bots-engine.py') #find the bots-engine lijst = [sys.executable,botsenginepath] + sys.argv[1:] if 'clparameter' in request.GET: lijst.append(request.GET['clparameter']) try: botsglobal.logger.info(_(u'Run bots-engine with parameters: "%s"'),str(lijst)) terug = subprocess.Popen(lijst).pid request.user.message_set.create(message=_(u'Bots-engine is started.')) except: txt = botslib.txtexc() request.user.message_set.create(message=_(u'Errors while trying to run bots-engine.')) botsglobal.logger.info(_(u'Errors while trying to run bots-engine:\n%s.'),txt) return django.shortcuts.redirect('/home')
def sendtestmailmanagers(request,*kw,**kwargs): try: sendornot = botsglobal.ini.getboolean('settings','sendreportiferror',False) except botslib.BotsError: sendornot = False if not sendornot: notification = _(u'Trying to send test mail, but in bots.ini, section [settings], "sendreportiferror" is not "True".') botsglobal.logger.info(notification) messages.add_message(request, messages.INFO, notification) return django.shortcuts.redirect('/home') from django.core.mail import mail_managers try: mail_managers(_(u'testsubject'), _(u'test content of report')) except: txt = botslib.txtexc() messages.add_message(request, messages.INFO, _(u'Sending test mail failed.')) botsglobal.logger.info(_(u'Sending test mail failed, error:\n%s(txt)'), {'txt':txt}) return django.shortcuts.redirect('/home') notification = _(u'Sending test mail succeeded.') messages.add_message(request, messages.INFO, notification) botsglobal.logger.info(notification) return django.shortcuts.redirect('/home')
def read_incoming(run): outputdir = botslib.join(run.inpath, run.infilename) filelist = [ filename for filename in glob.iglob(outputdir) if os.path.isfile(filename) ] filelist.sort() for infilename in filelist: try: filename = transform.unique('bots_file_name') abs_filename = botslib.abspathdata(filename) shutil.copy(infilename, abs_filename) #move if to be delted except: txt = botslib.txtexc() else: txt = '' #no errors finally: run.incoming.append({ 'infilename': infilename, 'filename': filename, 'error': txt, 'editype': run.translation['editype'], 'messagetype': run.translation['messagetype'] })
def mergemessages(startstatus=TRANSLATED, endstatus=MERGED, idroute=""): """ Merges en envelopes several messages to one file; In db-ta: attribute 'merge' indicates message should be merged with similar messages; 'merge' is generated in translation from messagetype-grammar If merge==False: 1 message per envelope - no merging, else append all similar messages to one file Implementation as separate loops: one for merge&envelope, another for enveloping only db-ta status TRANSLATED---->MERGED """ outerqueryparameters = { "status": startstatus, "statust": OK, "idroute": idroute, "rootidta": botslib.get_minta4query(), "merge": False, } # **********for messages only to envelope (no merging) for row in botslib.query( u"""SELECT editype,messagetype,frompartner,topartner,testindicator,charset,contenttype,tochannel,envelope,nrmessages,idta,filename,idroute,merge FROM ta WHERE idta>%(rootidta)s AND status=%(status)s AND statust=%(statust)s AND idroute=%(idroute)s AND merge=%(merge)s """, outerqueryparameters, ): try: ta_info = dict([(key, row[key]) for key in row.keys()]) # ~ ta_info={'merge':False,'idroute':idroute} # ~ for key in row.keys(): # ~ ta_info[key] = row[key] ta_fromfile = botslib.OldTransaction(row["idta"]) # edi message to envelope ta_tofile = ta_fromfile.copyta( status=endstatus ) # edifile for enveloped message; attributes of not-enveloped message are copied... # ~ ta_fromfile.update(child=ta_tofile.idta) #??there is already a parent-child relation (1-1)... ta_info["filename"] = str(ta_tofile.idta) # create filename for enveloped message botsglobal.logger.debug( u"Envelope 1 message editype: %s, messagetype: %s.", ta_info["editype"], ta_info["messagetype"] ) envelope(ta_info, [row["filename"]]) except: txt = botslib.txtexc() ta_tofile.update(statust=ERROR, errortext=txt) else: ta_fromfile.update(statust=DONE) ta_tofile.update(statust=OK, **ta_info) # selection is used to update enveloped message; # **********for messages to merge & envelope # all GROUP BY fields must be used in SELECT! # as files get merged: can not copy idta; must extract relevant attributes. outerqueryparameters["merge"] = True for row in botslib.query( u"""SELECT editype,messagetype,frompartner,topartner,tochannel,testindicator,charset,contenttype,envelope,sum(nrmessages) as nrmessages FROM ta WHERE idta>%(rootidta)s AND status=%(status)s AND statust=%(statust)s AND idroute=%(idroute)s AND merge=%(merge)s GROUP BY editype,messagetype,frompartner,topartner,tochannel,testindicator,charset,contenttype,envelope """, outerqueryparameters, ): try: ta_info = dict([(key, row[key]) for key in row.keys()]) ta_info.update({"merge": False, "idroute": idroute}) # ~ for key in row.keys(): # ~ ta_info[key] = row[key] ta_tofile = botslib.NewTransaction(status=endstatus, idroute=idroute) # edifile for enveloped messages ta_info["filename"] = str(ta_tofile.idta) # create filename for enveloped message innerqueryparameters = ta_info.copy() innerqueryparameters.update(outerqueryparameters) ta_list = [] # gather individual idta and filenames # explicitly allow formpartner/topartner to be None/NULL for row2 in botslib.query( u"""SELECT idta, filename FROM ta WHERE idta>%(rootidta)s AND status=%(status)s AND statust=%(statust)s AND merge=%(merge)s AND editype=%(editype)s AND messagetype=%(messagetype)s AND (frompartner=%(frompartner)s OR frompartner IS NULL) AND (topartner=%(topartner)s OR topartner IS NULL) AND tochannel=%(tochannel)s AND testindicator=%(testindicator)s AND charset=%(charset)s AND idroute=%(idroute)s """, innerqueryparameters, ): ta_fromfile = botslib.OldTransaction(row2["idta"]) # edi message to envelope ta_fromfile.update(statust=DONE, child=ta_tofile.idta) # st child because of n->1 relation ta_list.append(row2["filename"]) botsglobal.logger.debug( u"Merge and envelope: editype: %s, messagetype: %s, %s messages", ta_info["editype"], ta_info["messagetype"], ta_info["nrmessages"], ) envelope(ta_info, ta_list) except: txt = botslib.txtexc() ta_tofile.mergefailure() ta_tofile.update(statust=ERROR, errortext=txt) else: ta_tofile.update(statust=OK, **ta_info)
def mergemessages(startstatus, endstatus, idroute, rootidta=None): ''' Merges and/or envelopes one or more messages to one file; In db-ta: attribute 'merge' indicates message should be merged with similar messages; 'merge' is generated in translation from messagetype-grammar If merge is False: 1 message per envelope - no merging, else append all similar messages to one file Implementation as separate loops: one for merge&envelope, another for enveloping only db-ta status TRANSLATED---->FILEOUT ''' if rootidta is None: rootidta = botsglobal.currentrun.get_minta4query() #**********for messages only to envelope (no merging) for row in botslib.query( u'''SELECT editype,messagetype,frompartner,topartner,testindicator,charset,contenttype,envelope,nrmessages,idroute,merge,idta,filename,rsrv3 FROM ta WHERE idta>%(rootidta)s AND status=%(status)s AND statust=%(statust)s AND merge=%(merge)s AND idroute=%(idroute)s ORDER BY idta ''', { 'rootidta': rootidta, 'status': startstatus, 'statust': OK, 'merge': False, 'idroute': idroute }): try: ta_info = dict(row) ta_fromfile = botslib.OldTransaction( ta_info['idta']) #edi message to envelope ta_tofile = ta_fromfile.copyta( status=endstatus ) #edifile for enveloped message; attributes of not-enveloped message are copied... ta_info['filename'] = unicode( ta_tofile.idta) #create filename for enveloped message botsglobal.logger.debug( u'Envelope 1 message editype: %(editype)s, messagetype: %(messagetype)s.', ta_info) envelope(ta_info, [row['filename']]) ta_info['filesize'] = os.path.getsize( botslib.abspathdata(ta_info['filename'])) #get filesize except: txt = botslib.txtexc() ta_tofile.update(statust=ERROR, errortext=txt) else: ta_tofile.update( statust=OK, **ta_info) #selection is used to update enveloped message; finally: ta_fromfile.update(statust=DONE) #**********for messages to merge & envelope for row in botslib.query( u'''SELECT editype,messagetype,frompartner,topartner,testindicator,charset,contenttype,envelope,rsrv3,sum(nrmessages) as nrmessages FROM ta WHERE idta>%(rootidta)s AND status=%(status)s AND statust=%(statust)s AND merge=%(merge)s AND idroute=%(idroute)s GROUP BY editype,messagetype,frompartner,topartner,testindicator,charset,contenttype,envelope,rsrv3 ORDER BY editype,messagetype,frompartner,topartner,testindicator,charset,contenttype,envelope,rsrv3 ''', { 'rootidta': rootidta, 'status': startstatus, 'statust': OK, 'merge': True, 'idroute': idroute }): try: ta_info = dict(row) ta_tofile = botslib.NewTransaction( status=endstatus, idroute=idroute) #edifile for enveloped messages ta_info.update({ 'idroute': idroute, 'merge': False, 'filename': unicode(ta_tofile.idta) }) #SELECT/GROUP BY gives only values that are the grouped filename_list = [] #gather individual idta and filenames #explicitly allow formpartner/topartner to be None/NULL for row2 in botslib.query( u'''SELECT idta, filename FROM ta WHERE idta>%(rootidta)s AND status=%(status)s AND statust=%(statust)s AND merge=%(merge)s AND editype=%(editype)s AND messagetype=%(messagetype)s AND (frompartner=%(frompartner)s OR frompartner IS NULL) AND (topartner=%(topartner)s OR topartner IS NULL) AND testindicator=%(testindicator)s AND charset=%(charset)s ORDER BY idta ''', { 'rootidta': rootidta, 'status': startstatus, 'statust': OK, 'merge': True, 'editype': ta_info['editype'], 'messagetype': ta_info['messagetype'], 'frompartner': ta_info['frompartner'], 'topartner': ta_info['topartner'], 'testindicator': ta_info['testindicator'], 'charset': ta_info['charset'], 'rsrv3': ta_info['rsrv3'] }): ta_fromfile = botslib.OldTransaction( row2['idta']) #edi message to be merged/envelope ta_fromfile.update( child=ta_tofile.idta, statust=DONE) #st child because of n->1 relation filename_list.append(row2['filename']) botsglobal.logger.debug( u'Merge and envelope: editype: %(editype)s, messagetype: %(messagetype)s, %(nrmessages)s messages', ta_info) envelope(ta_info, filename_list) ta_info['filesize'] = os.path.getsize( botslib.abspathdata(ta_info['filename'])) #get filesize except: txt = botslib.txtexc() ta_tofile.update(statust=ERROR, errortext=txt) else: ta_tofile.update(statust=OK, **ta_info)
def extractpdf(ta_from,endstatus,**argv): ''' Try to extract text content of a PDF file to a csv. You know this is not a great idea, right? But we'll do the best we can anyway! Page and line numbers are added to each row. Columns and rows are based on the x and y coordinates of each text element within tolerance allowed. Multiple text elements may combine to make one field, some PDFs have every character separated! You may need to experiment with x_group and y_group values, but defaults seem ok for most files. Output csv is UTF-8 encoded - The csv module doesn't directly support reading and writing Unicode If the PDF is just an image, all bets are off. Maybe try OCR, good luck with that! Mike Griffin 14/12/2011 ''' from pdfminer.pdfinterp import PDFResourceManager, process_pdf from pdfminer.converter import TextConverter from pdfminer.layout import LAParams, LTContainer, LTText, LTTextBox import csv class CsvConverter(TextConverter): def __init__(self, *args, **kwargs): TextConverter.__init__(self, *args, **kwargs) def receive_layout(self, ltpage): # recursively get every text element and it's coordinates def render(item): if isinstance(item, LTContainer): for child in item: render(child) elif isinstance(item, LTText): (_,_,x,y) = item.bbox # group the y values (rows) within group tolerance for v in yv: if y > v-y_group and y < v+y_group: y = v yv.append(y) line = lines[int(-y)] line[x] = item.get_text().encode('utf-8') from collections import defaultdict lines = defaultdict(lambda : {}) yv = [] render(ltpage) lineid = 0 for y in sorted(lines.keys()): line = lines[y] lineid += 1 csvdata = [ltpage.pageid,lineid] # first 2 columns are page and line numbers # group the x values (fields) within group tolerance p = 0 field_txt = '' for x in sorted(line.keys()): gap = x - p if p > 0 and gap > x_group: csvdata.append(field_txt) field_txt = '' field_txt += line[x] p = x csvdata.append(field_txt) csvout.writerow(csvdata) if lineid == 0: raise botslib.InMessageError(_(u'PDF text extraction failed, it may contain just image(s)?')) #get some optional parameters x_group = argv.get('x_group',10) # group text closer than this as one field y_group = argv.get('y_group',5) # group lines closer than this as one line password = argv.get('password','') quotechar = argv.get('quotechar','"') field_sep = argv.get('field_sep',',') escape = argv.get('escape','\\') charset = argv.get('charset','utf-8') if not escape: doublequote = True else: doublequote = False try: pdf_stream = botslib.opendata(ta_from.filename, 'rb') ta_to = ta_from.copyta(status=endstatus) tofilename = str(ta_to.idta) csv_stream = botslib.opendata(tofilename,'wb') csvout = csv.writer(csv_stream, quotechar=quotechar, delimiter=field_sep, doublequote=doublequote, escapechar=escape) # Process PDF rsrcmgr = PDFResourceManager(caching=True) device = CsvConverter(rsrcmgr, csv_stream, codec=charset) process_pdf(rsrcmgr, device, pdf_stream, pagenos=set(), password=password, caching=True, check_extractable=True) device.close() pdf_stream.close() csv_stream.close() ta_to.update(statust=OK,filename=tofilename) #update outmessage transaction with ta_info; botsglobal.logger.debug(_(u' File written: "%s".'),tofilename) except: txt = botslib.txtexc() botsglobal.logger.error(_(u'PDF extraction failed, may not be a PDF file? Error:\n%s'),txt) raise botslib.InMessageError(_(u'PDF extraction failed, may not be a PDF file? Error:\n$error'),error=txt)
def extractexcel(ta_from,endstatus,**argv): ''' extract excel file. editype & messagetype are unchanged. ''' #***functions used by extractexcel #------------------------------------------------------------------------------- def read_xls(infilename): # Read excel first sheet into a 2-d array book = xlrd.open_workbook(infilename) sheet = book.sheet_by_index(0) formatter = lambda(t,v): format_excelval(book,t,v,False) xlsdata = [] for row in range(sheet.nrows): (types, values) = (sheet.row_types(row), sheet.row_values(row)) xlsdata.append(map(formatter, zip(types, values))) return xlsdata #------------------------------------------------------------------------------- def dump_csv(xlsdata, tofilename): stream = botslib.opendata(tofilename, 'wb') csvout = csv.writer(stream, quotechar=quotechar, delimiter=field_sep, doublequote=doublequote, escapechar=escape) csvout.writerows( map(utf8ize, xlsdata) ) stream.close() #------------------------------------------------------------------------------- def format_excelval(book, type, value, wanttupledate): # Clean up the incoming excel data for some data types returnrow = [] if type == 2: if value == int(value): value = int(value) elif type == 3: datetuple = xlrd.xldate_as_tuple(value, book.datemode) value = datetuple if wanttupledate else tupledate_to_isodate(datetuple) elif type == 5: value = xlrd.error_text_from_code[value] return value #------------------------------------------------------------------------------- def tupledate_to_isodate(tupledate): # Turns a gregorian (year, month, day, hour, minute, nearest_second) into a # standard YYYY-MM-DDTHH:MM:SS ISO date. (y,m,d, hh,mm,ss) = tupledate nonzero = lambda n: n != 0 date = "%04d-%02d-%02d" % (y,m,d) if filter(nonzero, (y,m,d)) else '' time = "T%02d:%02d:%02d" % (hh,mm,ss) if filter(nonzero, (hh,mm,ss)) or not date else '' return date+time #------------------------------------------------------------------------------- def utf8ize(l): # Make string-like things into utf-8, leave other things alone return [unicode(s).encode(charset) if hasattr(s,'encode') else s for s in l] #***end functions used by extractexcel import xlrd import csv #get parameters for csv-format; defaults are as the csv defaults (in grammar.py) charset = argv.get('charset',"utf-8") quotechar = argv.get('quotechar',"'") field_sep = argv.get('field_sep',':') escape = argv.get('escape','') if escape: doublequote = False else: doublequote = True try: infilename = botslib.abspathdata(ta_from.filename) xlsdata = read_xls(infilename) ta_to = ta_from.copyta(status=endstatus) tofilename = str(ta_to.idta) dump_csv(xlsdata,tofilename) ta_to.update(statust=OK,filename=tofilename) #update outmessage transaction with ta_info; botsglobal.logger.debug(_(u' File written: "%s".'),tofilename) except: txt = botslib.txtexc() botsglobal.logger.error(_(u'Excel extraction failed, may not be an Excel file? Error:\n%s'),txt) raise botslib.InMessageError(_(u'Excel extraction failed, may not be an Excel file? Error:\n$error'),error=txt)
def _translate_one_file(row,routedict,endstatus,userscript,scriptname): ''' - read, lex, parse, make tree of nodes. - split up files into messages (using 'nextmessage' of grammar) - get mappingscript, start mappingscript. - write the results of translation (no enveloping yet) ''' try: ta_fromfile = botslib.OldTransaction(row['idta']) ta_parsed = ta_fromfile.copyta(status=PARSED) if row['filesize'] > botsglobal.ini.getint('settings','maxfilesizeincoming',5000000): ta_parsed.update(filesize=row['filesize']) raise botslib.FileTooLarge(_(u'File size of %(filesize)s is too big; option "maxfilesizeincoming" in bots.ini is %(maxfilesizeincoming)s.'), {'filesize':row['filesize'],'maxfilesizeincoming':botsglobal.ini.getint('settings','maxfilesizeincoming',5000000)}) botsglobal.logger.debug(_(u'Start translating file "%(filename)s" editype "%(editype)s" messagetype "%(messagetype)s".'),row) #read whole edi-file: read, parse and made into a inmessage-object. Message is represented as a tree (inmessage.root is the root of the tree). edifile = inmessage.parse_edi_file(frompartner=row['frompartner'], topartner=row['topartner'], filename=row['filename'], messagetype=row['messagetype'], testindicator=row['testindicator'], editype=row['editype'], charset=row['charset'], alt=row['alt'], fromchannel=row['fromchannel'], idroute=routedict['idroute'], command=routedict['command']) edifile.checkforerrorlist() #no exception if infile has been lexed and parsed OK else raises an error if int(routedict['translateind']) == 3: #parse & passthrough; file is parsed, partners are known, no mapping, does confirm. raise botslib.GotoException('dummy') #edifile.ta_info contains info: QUERIES, charset etc for inn_splitup in edifile.nextmessage(): #splitup messages in parsed edifile try: ta_splitup = ta_parsed.copyta(status=SPLITUP,**inn_splitup.ta_info) #copy PARSED to SPLITUP ta #inn_splitup.ta_info: parameters from inmessage.parse_edi_file(), syntax-information and parse-information inn_splitup.ta_info['idta_fromfile'] = ta_fromfile.idta #for confirmations in userscript; used to give idta of 'confirming message' inn_splitup.ta_info['idta'] = ta_splitup.idta #for confirmations in userscript; used to give idta of 'confirming message' number_of_loops_with_same_alt = 0 while 1: #continue as long as there are (alt-)translations #lookup the translation************************ tscript,toeditype,tomessagetype = botslib.lookup_translation(fromeditype=inn_splitup.ta_info['editype'], frommessagetype=inn_splitup.ta_info['messagetype'], frompartner=inn_splitup.ta_info['frompartner'], topartner=inn_splitup.ta_info['topartner'], alt=inn_splitup.ta_info['alt']) if not tscript: #no translation found in translate table; check if can find translation via user script if userscript and hasattr(userscript,'gettranslation'): tscript,toeditype,tomessagetype = botslib.runscript(userscript,scriptname,'gettranslation',idroute=routedict['idroute'],message=inn_splitup) if not tscript: raise botslib.TranslationNotFoundError(_(u'Translation not found for editype "%(editype)s", messagetype "%(messagetype)s", frompartner "%(frompartner)s", topartner "%(topartner)s", alt "%(alt)s".'), inn_splitup.ta_info) inn_splitup.ta_info['divtext'] = tscript #ifor reporting used mapping script to database (for display in GUI). #initialize new out-object************************* ta_translated = ta_splitup.copyta(status=endstatus) #make ta for translated message (new out-ta) filename_translated = str(ta_translated.idta) out_translated = outmessage.outmessage_init(editype=toeditype,messagetype=tomessagetype,filename=filename_translated,reference=unique('messagecounter'),statust=OK,divtext=tscript) #make outmessage object #run mapping script************************ botsglobal.logger.debug(_(u'Mappingscript "%(tscript)s" translates messagetype "%(messagetype)s" to messagetype "%(tomessagetype)s".'), {'tscript':tscript,'messagetype':inn_splitup.ta_info['messagetype'],'tomessagetype':out_translated.ta_info['messagetype']}) translationscript,scriptfilename = botslib.botsimport('mappings',inn_splitup.ta_info['editype'],tscript) #get the mappingscript alt_from_previous_run = inn_splitup.ta_info['alt'] #needed to check for infinite loop doalttranslation = botslib.runscript(translationscript,scriptfilename,'main',inn=inn_splitup,out=out_translated) botsglobal.logger.debug(_(u'Mappingscript "%(tscript)s" finished.'),{'tscript':tscript}) #manipulate for some attributes after mapping script if 'topartner' not in out_translated.ta_info: #out_translated does not contain values from ta...... out_translated.ta_info['topartner'] = inn_splitup.ta_info['topartner'] if 'botskey' in inn_splitup.ta_info: inn_splitup.ta_info['reference'] = inn_splitup.ta_info['botskey'] if 'botskey' in out_translated.ta_info: #out_translated does not contain values from ta...... out_translated.ta_info['reference'] = out_translated.ta_info['botskey'] #check the value received from the mappingscript to determine what to do in this while-loop. Handling of chained trasnlations. if doalttranslation is None: #translation(s) are done; handle out-message handle_out_message(out_translated,ta_translated) break #break out of while loop elif isinstance(doalttranslation,dict): #some extended cases; a dict is returned that contains 'instructions' for some type of chained translations if 'type' not in doalttranslation or 'alt' not in doalttranslation: raise botslib.BotsError(_(u"Mappingscript returned '%(alt)s'. This dict should not have 'type' and 'alt'."),{'alt':doalttranslation}) if alt_from_previous_run == doalttranslation['alt']: number_of_loops_with_same_alt += 1 else: number_of_loops_with_same_alt = 0 if doalttranslation['type'] == u'out_as_inn': #do chained translation: use the out-object as inn-object, new out-object #use case: detected error in incoming file; use out-object to generate warning email handle_out_message(out_translated,ta_translated) inn_splitup = out_translated #out-object is now inn-object if isinstance(inn_splitup,outmessage.fixed): #for fixed: strip all values in node inn_splitup.root.stripnode() inn_splitup.ta_info['alt'] = doalttranslation['alt'] #get the alt-value for the next chained translation if not 'frompartner' in inn_splitup.ta_info: inn_splitup.ta_info['frompartner'] = '' if not 'topartner' in inn_splitup.ta_info: inn_splitup.ta_info['topartner'] = '' inn_splitup.ta_info.pop('statust') elif doalttranslation['type'] == u'no_check_on_infinite_loop': #do chained translation: allow many loops wit hsame alt-value. #mapping script will have to handle this correctly. number_of_loops_with_same_alt = 0 handle_out_message(out_translated,ta_translated) inn_splitup.ta_info['alt'] = doalttranslation['alt'] #get the alt-value for the next chained translation else: #there is nothing else raise botslib.BotsError(_(u'Mappingscript returned dict with an unknown "type": "%(doalttranslation)s".'),{'doalttranslation':doalttranslation}) else: #note: this includes alt '' (empty string) if alt_from_previous_run == doalttranslation: number_of_loops_with_same_alt += 1 else: number_of_loops_with_same_alt = 0 #do normal chained translation: same inn-object, new out-object handle_out_message(out_translated,ta_translated) inn_splitup.ta_info['alt'] = doalttranslation #get the alt-value for the next chained translation if number_of_loops_with_same_alt > 10: raise botslib.BotsError(_(u'Mappingscript returns same alt value over and over again (infinite loop?). Alt: "%(doalttranslation)s".'),{'doalttranslation':doalttranslation}) #end of while-loop (trans********************************************************************************** #exceptions file_out-level: exception in mappingscript or writing of out-file except: #2 modes: either every error leads to skipping of whole infile (old mode) or errors in mappingscript/outfile only affect that branche if botsglobal.ini.getboolean('settings','oldmessageerrors',False): raise txt = botslib.txtexc() ta_splitup.update(statust=ERROR,errortext=txt,**inn_splitup.ta_info) #update db. inn_splitup.ta_info could be changed by mappingscript. Is this useful? ta_splitup.deletechildren() else: ta_splitup.update(statust=DONE, **inn_splitup.ta_info) #update db. inn_splitup.ta_info could be changed by mappingscript. Is this useful? #exceptions file_in-level except botslib.GotoException: #edi-file is OK, file is passed-through after parsing. ta_parsed.update(statust=DONE,filesize=row['filesize'],**edifile.ta_info) #update with info from eg queries ta_parsed.copyta(status=MERGED,statust=OK) #original file goes straight to MERGED edifile.handleconfirm(ta_fromfile,error=False) botsglobal.logger.debug(_(u'Parse & passthrough for input file "%(filename)s".'),row) except botslib.FileTooLarge as msg: ta_parsed.update(statust=ERROR,errortext=str(msg)) ta_parsed.deletechildren() botsglobal.logger.debug(u'Error in translating input file "%(filename)s":\n%(msg)s',{'filename':row['filename'],'msg':msg}) except: txt = botslib.txtexc() ta_parsed.update(statust=ERROR,errortext=txt,**edifile.ta_info) ta_parsed.deletechildren() edifile.handleconfirm(ta_fromfile,error=True) botsglobal.logger.debug(u'Error in translating input file "%(filename)s":\n%(msg)s',{'filename':row['filename'],'msg':txt}) else: edifile.handleconfirm(ta_fromfile,error=False) ta_parsed.update(statust=DONE,filesize=row['filesize'],**edifile.ta_info) botsglobal.logger.debug(_(u'Translated input file "%(filename)s".'),row) finally: ta_fromfile.update(statust=DONE)
def load(pathzipfile): ''' process uploaded plugin. ''' #test is valid zipfile if not zipfile.is_zipfile(pathzipfile): raise botslib.PluginError(_(u'Plugin is not a valid file.')) #read index file try: Zipimporter = zipimport.zipimporter(pathzipfile) importedbotsindex = Zipimporter.load_module('botsindex') pluglist = importedbotsindex.plugins[:] if 'botsindex' in sys.modules: del sys.modules['botsindex'] except: txt = botslib.txtexc() raise botslib.PluginError( _(u'Error in plugin. Nothing is written. Error: "%s"') % (txt)) else: botsglobal.logger.info(_(u'Plugin is OK.\nStart writing to database.')) #write content of index file to the bots database try: writetodatabase(pluglist) except: txt = botslib.txtexc() raise botslib.PluginError( 'Error writing plugin to database. Nothing is written. Error: "%s"' % (txt)) else: botsglobal.logger.info( u'Writing to database is OK.\nStart writing to files') #write files to the file system. try: warnrenamed = False #to report in GUI files have been overwritten. z = zipfile.ZipFile(pathzipfile, mode="r") orgtargetpath = botsglobal.ini.get('directories', 'botspath') if (orgtargetpath[-1:] in (os.path.sep, os.path.altsep) and len(os.path.splitdrive(orgtargetpath)[1]) > 1): orgtargetpath = orgtargetpath[:-1] for f in z.infolist(): if f.filename not in [ 'botsindex.py', 'README', 'botssys/sqlitedb/botsdb', 'config/bots.ini' ] and os.path.splitext(f.filename)[1] not in ['.pyo', '.pyc']: #~ botsglobal.logger.info(u'filename in zip "%s".',f.filename) if f.filename[0] == '/': targetpath = f.filename[1:] else: targetpath = f.filename targetpath = targetpath.replace( 'usersys', botsglobal.ini.get('directories', 'usersysabs'), 1) targetpath = targetpath.replace( 'botssys', botsglobal.ini.get('directories', 'botssys'), 1) targetpath = botslib.join(orgtargetpath, targetpath) #targetpath is OK now. botsglobal.logger.info(_(u' Start writing file: "%s".'), targetpath) if botslib.dirshouldbethere(os.path.dirname(targetpath)): botsglobal.logger.info( _(u' Create directory "%s".'), os.path.dirname(targetpath)) if f.filename[ -1] == '/': #check if this is a dir; if so continue continue if os.path.isfile(targetpath): #check if file already exists try: #this ***sometimes*** fails. (python25, for static/help/home.html...only there...) os.rename( targetpath, targetpath + '.' + time.strftime('%Y%m%d%H%M%S')) warnrenamed = True botsglobal.logger.info( _(u' Renamed existing file "%(from)s" to "%(to)s".' ), { 'from': targetpath, 'to': targetpath + time.strftime('%Y%m%d%H%M%S') }) except: pass source = z.read(f.filename) target = open(targetpath, "wb") target.write(source) target.close() botsglobal.logger.info(_(u' File written: "%s".'), targetpath) except: txt = botslib.txtexc() z.close() raise botslib.PluginError( _(u'Error writing files to system. Nothing is written to database. Error: "%s"' ) % (txt)) else: z.close() botsglobal.logger.info(_(u'Writing files to filesystem is OK.')) return warnrenamed
def mysql(): print 'Start changing mysql database to bots version 3.' cursor = botsglobal.db.cursor() try: #channel **************************************** cursor.execute( '''ALTER TABLE `channel` MODIFY `filename` VARCHAR(256) NOT NULL''' ) cursor.execute( '''ALTER TABLE `channel` ADD COLUMN `rsrv3` INTEGER DEFAULT 0''') cursor.execute( '''ALTER TABLE `channel` ADD COLUMN `keyfile` VARCHAR(256) ''') cursor.execute( '''ALTER TABLE `channel` ADD COLUMN `certfile` VARCHAR(256) ''') cursor.execute( '''ALTER TABLE `channel` ADD COLUMN `testpath` VARCHAR(256) ''') #ccode **************************************** cursor.execute( '''ALTER TABLE `ccode` MODIFY `rightcode` VARCHAR(70)''') cursor.execute('''ALTER TABLE `ccode` MODIFY `attr1` VARCHAR(70)''') #filereport **************************************** cursor.execute('''ALTER TABLE `filereport` CHANGE `id` `id` INTEGER ''' ) #drop autoincrement cursor.execute('''ALTER TABLE `filereport` DROP PRIMARY KEY ''' ) #drop index on id cursor.execute( '''ALTER TABLE `filereport` DROP COLUMN `id` ''') #drop id veld cursor.execute('''ALTER TABLE `filereport` DROP KEY `idta` ''' ) #remove UNIQUE constraint #~ cursor.execute('''ALTER TABLE `filereport` DROP INDEX `reportidta` ''') #not possible as index name is not known cursor.execute('''ALTER TABLE `filereport` MODIFY `errortext` TEXT''') cursor.execute( '''ALTER TABLE `filereport` ADD COLUMN `filesize` INTEGER DEFAULT 0''' ) cursor.execute( '''ALTER TABLE `filereport` ADD COLUMN `acceptance` INTEGER DEFAULT 0''' ) #partner ************************************* cursor.execute( '''ALTER TABLE `partner` ADD COLUMN `attr1` VARCHAR(35) ''') cursor.execute( '''ALTER TABLE `partner` ADD COLUMN `attr2` VARCHAR(35) ''') cursor.execute( '''ALTER TABLE `partner` ADD COLUMN `attr3` VARCHAR(35) ''') cursor.execute( '''ALTER TABLE `partner` ADD COLUMN `attr4` VARCHAR(35) ''') cursor.execute( '''ALTER TABLE `partner` ADD COLUMN `attr5` VARCHAR(70) ''') cursor.execute( '''ALTER TABLE `partner` ADD COLUMN `name1` VARCHAR(70) ''') cursor.execute( '''ALTER TABLE `partner` ADD COLUMN `name2` VARCHAR(70) ''') cursor.execute( '''ALTER TABLE `partner` ADD COLUMN `name3` VARCHAR(70) ''') cursor.execute( '''ALTER TABLE `partner` ADD COLUMN `address1` VARCHAR(70) ''') cursor.execute( '''ALTER TABLE `partner` ADD COLUMN `address2` VARCHAR(70) ''') cursor.execute( '''ALTER TABLE `partner` ADD COLUMN `address3` VARCHAR(70) ''') cursor.execute( '''ALTER TABLE `partner` ADD COLUMN `city` VARCHAR(35) ''') cursor.execute( '''ALTER TABLE `partner` ADD COLUMN `postalcode` VARCHAR(17) ''') cursor.execute( '''ALTER TABLE `partner` ADD COLUMN `countrysubdivision` VARCHAR(9) ''' ) cursor.execute( '''ALTER TABLE `partner` ADD COLUMN `countrycode` VARCHAR(3) ''') cursor.execute( '''ALTER TABLE `partner` ADD COLUMN `phone1` VARCHAR(17) ''') cursor.execute( '''ALTER TABLE `partner` ADD COLUMN `phone2` VARCHAR(17) ''') cursor.execute('''ALTER TABLE `partner` ADD COLUMN `desc` TEXT ''') cursor.execute( '''ALTER TABLE `partner` ADD COLUMN `startdate` DATE ''') cursor.execute('''ALTER TABLE `partner` ADD COLUMN `enddate` DATE ''') #persist **************************************** cursor.execute('''ALTER TABLE `persist` MODIFY `content` TEXT''') #report **************************************** cursor.execute('''CREATE INDEX `report_ts` ON `report` (`ts`)''') cursor.execute( '''ALTER TABLE `report` ADD COLUMN `filesize` INTEGER DEFAULT 0''') #routes **************************************** cursor.execute( '''ALTER TABLE `routes` ADD COLUMN `zip_incoming` INTEGER DEFAULT 0''' ) cursor.execute( '''ALTER TABLE `routes` ADD COLUMN `zip_outgoing` INTEGER DEFAULT 0''' ) cursor.execute( '''ALTER TABLE `routes` MODIFY `translateind` integer NOT NULL''') #ta **************************************** #~ cursor.execute('''ALTER TABLE `ta` DROP INDEX `script` ''') #not possible as index name is not known cursor.execute('''CREATE INDEX `ta_reference` ON `ta` (`reference`)''') cursor.execute( '''ALTER TABLE `ta` ADD COLUMN `filesize` INTEGER DEFAULT 0''') cursor.execute( '''ALTER TABLE `ta` ADD COLUMN `numberofresends` INTEGER DEFAULT 0''' ) cursor.execute( '''ALTER TABLE `ta` ADD COLUMN `rsrv5` VARCHAR(35) DEFAULT '' ''') cursor.execute('''ALTER TABLE `ta` MODIFY `errortext` TEXT ''') except: txt = botslib.txtexc() botsglobal.db.rollback() cursor.close() print 'Error in changing mysql database: "%s".' % (txt) return 1 else: botsglobal.db.commit() cursor.close() print 'Succesful changed mysql database to bots version 3.' return 0
def postgresql_psycopg2(): print 'Start changing postgresql database to bots version 3.' cursor = botsglobal.db.cursor() try: #channel **************************************** cursor.execute( '''ALTER TABLE "channel" ALTER COLUMN "filename" TYPE VARCHAR(256)''' ) cursor.execute( '''ALTER TABLE "channel" ADD COLUMN "rsrv3" INTEGER DEFAULT 0''') cursor.execute( '''ALTER TABLE "channel" ADD COLUMN "keyfile" VARCHAR(256) ''') cursor.execute( '''ALTER TABLE "channel" ADD COLUMN "certfile" VARCHAR(256) ''') cursor.execute( '''ALTER TABLE "channel" ADD COLUMN "testpath" VARCHAR(256) ''') #ccode **************************************** cursor.execute( '''ALTER TABLE "ccode" ALTER COLUMN "rightcode" TYPE VARCHAR(70)''' ) cursor.execute( '''ALTER TABLE "ccode" ALTER COLUMN "attr1" TYPE VARCHAR(70)''') #filereport **************************************** cursor.execute( '''ALTER TABLE "filereport" DROP CONSTRAINT "filereport_pkey" ''' ) #remove primary key cursor.execute( '''ALTER TABLE "filereport" DROP CONSTRAINT "filereport_idta_key" ''' ) #drop contraint UNIQUE(idta, reportidta) cursor.execute('''DROP INDEX "filereport_idta" ''' ) #drop index on idta (will be primary key) cursor.execute('''DROP INDEX "filereport_reportidta" ''') cursor.execute('''ALTER TABLE "filereport" DROP COLUMN "id" ''') cursor.execute( '''ALTER TABLE "filereport" ADD CONSTRAINT "filereport_pkey" PRIMARY KEY("idta")''' ) #idta is primary key cursor.execute( '''ALTER TABLE "filereport" ALTER COLUMN "errortext" TYPE TEXT''') cursor.execute( '''ALTER TABLE "filereport" ADD COLUMN "filesize" INTEGER DEFAULT 0''' ) cursor.execute( '''ALTER TABLE "filereport" ADD COLUMN "acceptance" INTEGER DEFAULT 0''' ) #partner ************************************* cursor.execute( '''ALTER TABLE "partner" ADD COLUMN "attr1" VARCHAR(35) ''') cursor.execute( '''ALTER TABLE "partner" ADD COLUMN "attr2" VARCHAR(35) ''') cursor.execute( '''ALTER TABLE "partner" ADD COLUMN "attr3" VARCHAR(35) ''') cursor.execute( '''ALTER TABLE "partner" ADD COLUMN "attr4" VARCHAR(35) ''') cursor.execute( '''ALTER TABLE "partner" ADD COLUMN "attr5" VARCHAR(70) ''') cursor.execute( '''ALTER TABLE "partner" ADD COLUMN "name1" VARCHAR(70) ''') cursor.execute( '''ALTER TABLE "partner" ADD COLUMN "name2" VARCHAR(70) ''') cursor.execute( '''ALTER TABLE "partner" ADD COLUMN "name3" VARCHAR(70) ''') cursor.execute( '''ALTER TABLE "partner" ADD COLUMN "address1" VARCHAR(70) ''') cursor.execute( '''ALTER TABLE "partner" ADD COLUMN "address2" VARCHAR(70) ''') cursor.execute( '''ALTER TABLE "partner" ADD COLUMN "address3" VARCHAR(70) ''') cursor.execute( '''ALTER TABLE "partner" ADD COLUMN "city" VARCHAR(35) ''') cursor.execute( '''ALTER TABLE "partner" ADD COLUMN "postalcode" VARCHAR(17) ''') cursor.execute( '''ALTER TABLE "partner" ADD COLUMN "countrysubdivision" VARCHAR(9) ''' ) cursor.execute( '''ALTER TABLE "partner" ADD COLUMN "countrycode" VARCHAR(3) ''') cursor.execute( '''ALTER TABLE "partner" ADD COLUMN "phone1" VARCHAR(17) ''') cursor.execute( '''ALTER TABLE "partner" ADD COLUMN "phone2" VARCHAR(17) ''') cursor.execute('''ALTER TABLE "partner" ADD COLUMN "desc" TEXT ''') cursor.execute( '''ALTER TABLE "partner" ADD COLUMN "startdate" DATE ''') cursor.execute('''ALTER TABLE "partner" ADD COLUMN "enddate" DATE ''') #persist **************************************** cursor.execute( '''ALTER TABLE "persist" ALTER COLUMN "content" TYPE TEXT''') #report **************************************** cursor.execute('''CREATE INDEX "report_ts" ON "report" ("ts")''') cursor.execute( '''ALTER TABLE "report" ADD COLUMN "filesize" INTEGER DEFAULT 0''') #routes **************************************** cursor.execute( '''ALTER TABLE "routes" ALTER COLUMN "translateind" TYPE integer USING CASE WHEN "translateind"=FALSE THEN 0 ELSE 1 END''' ) cursor.execute( '''ALTER TABLE "routes" ADD COLUMN "zip_incoming" INTEGER DEFAULT 0''' ) cursor.execute( '''ALTER TABLE "routes" ADD COLUMN "zip_outgoing" INTEGER DEFAULT 0''' ) #ta **************************************** cursor.execute('''DROP INDEX "ta_script" ''') cursor.execute('''CREATE INDEX "ta_reference" ON "ta" ("reference")''') cursor.execute( '''ALTER TABLE "ta" ALTER COLUMN "errortext" TYPE TEXT ''') cursor.execute( '''ALTER TABLE "ta" ADD COLUMN "filesize" INTEGER DEFAULT 0''') cursor.execute( '''ALTER TABLE "ta" ADD COLUMN "numberofresends" INTEGER DEFAULT 0''' ) cursor.execute( '''ALTER TABLE "ta" ADD COLUMN "rsrv5" VARCHAR(35) DEFAULT '' ''') except: txt = botslib.txtexc() botsglobal.db.rollback() cursor.close() print 'Error in changing postgresql database: "%s".' % (txt) return 1 else: botsglobal.db.commit() cursor.close() print 'Succesful changed postgresql database to bots version 3.' return 0
def filer(request, *kw, **kwargs): ''' handles bots file viewer. Only files in data dir of Bots are displayed.''' nosuchfile = _(u'No such file.') if request.method == 'GET': try: idta = request.GET['idta'] except: return django.shortcuts.render_to_response( 'bots/filer.html', {'error_content': nosuchfile}, context_instance=django.template.RequestContext(request)) if idta == 0: #for the 'starred' file names (eg multiple output) return django.shortcuts.render_to_response( 'bots/filer.html', {'error_content': nosuchfile}, context_instance=django.template.RequestContext(request)) try: currentta = list(models.ta.objects.filter(idta=idta))[0] if request.GET['action'] == 'downl': response = django.http.HttpResponse( mimetype=currentta.contenttype) response[ 'Content-Disposition'] = 'attachment; filename=' + currentta.filename + '.txt' #~ absfilename = botslib.abspathdata(currentta.filename) #~ response['Content-Length'] = os.path.getsize(absfilename) response.write( botslib.readdata(currentta.filename, charset=currentta.charset, errors='ignore')) return response elif request.GET['action'] == 'previous': if currentta.parent: #has a explicit parent talijst = list( models.ta.objects.filter(idta=currentta.parent)) else: #get list of ta's referring to this idta as child talijst = list( models.ta.objects.filter(child=currentta.idta)) elif request.GET['action'] == 'this': if currentta.status == EXTERNIN: #jump strait to next file, as EXTERNIN can not be displayed. talijst = list( models.ta.objects.filter(parent=currentta.idta)) elif currentta.status == EXTERNOUT: talijst = list( models.ta.objects.filter(idta=currentta.parent)) else: talijst = [currentta] elif request.GET['action'] == 'next': if currentta.child: #has a explicit child talijst = list( models.ta.objects.filter(idta=currentta.child)) else: talijst = list( models.ta.objects.filter(parent=currentta.idta)) for ta in talijst: ta.has_next = True if ta.status == EXTERNIN: ta.content = _( u'External file. Can not be displayed. Use "next".') elif ta.status == EXTERNOUT: ta.content = _( u'External file. Can not be displayed. Use "previous".' ) ta.has_next = False elif ta.statust in [OPEN, ERROR]: ta.content = _( u'File has error status and does not exist. Use "previous".' ) ta.has_next = False elif not ta.filename: ta.content = _(u'File can not be displayed.') else: if ta.charset: #guess charset; uft-8 is reasonable ta.content = botslib.readdata(ta.filename, charset=ta.charset, errors='ignore') else: ta.content = botslib.readdata(ta.filename, charset='utf-8', errors='ignore') return django.shortcuts.render_to_response( 'bots/filer.html', {'idtas': talijst}, context_instance=django.template.RequestContext(request)) except: print botslib.txtexc() return django.shortcuts.render_to_response( 'bots/filer.html', {'error_content': nosuchfile}, context_instance=django.template.RequestContext(request))
def read_plugin(pathzipfile): ''' process uploaded plugin. ''' #test if valid zipfile if not zipfile.is_zipfile(pathzipfile): raise botslib.PluginError(_(u'Plugin is not a valid file.')) #read index file try: myzipimport = zipimport.zipimporter(pathzipfile) importedbotsindex = myzipimport.load_module('botsindex') pluglist = importedbotsindex.plugins[:] if 'botsindex' in sys.modules: del sys.modules['botsindex'] except: txt = botslib.txtexc() raise botslib.PluginError(_(u'Error in plugin. Nothing is written. Error:\n%(txt)s'),{'txt':txt}) else: botsglobal.logger.info(_(u'Plugin is OK.')) botsglobal.logger.info(_(u'Start writing to database.')) #write content of index file to the bots database try: read_index2database(pluglist) except: txt = botslib.txtexc() raise botslib.PluginError(_(u'Error writing plugin to database. Nothing is written. Error:\n%(txt)s'),{'txt':txt}) else: botsglobal.logger.info(_(u'Writing to database is OK.')) #write files to the file system. botsglobal.logger.info(_(u'Start writing to files')) try: warnrenamed = False #to report in GUI files have been overwritten. myzip = zipfile.ZipFile(pathzipfile, mode="r") orgtargetpath = botsglobal.ini.get('directories','botspath') if (orgtargetpath[-1:] in (os.path.sep, os.path.altsep) and len(os.path.splitdrive(orgtargetpath)[1]) > 1): orgtargetpath = orgtargetpath[:-1] for zipfileobject in myzip.infolist(): if zipfileobject.filename not in ['botsindex.py','README','botssys/sqlitedb/botsdb','config/bots.ini'] and os.path.splitext(zipfileobject.filename)[1] not in ['.pyo','.pyc']: #~ botsglobal.logger.info(u'Filename in zip "%s".',zipfileobject.filename) if zipfileobject.filename[0] == '/': targetpath = zipfileobject.filename[1:] else: targetpath = zipfileobject.filename #convert for correct environment: repacle botssys, config, usersys in filenames if targetpath.startswith('usersys'): targetpath = targetpath.replace('usersys',botsglobal.ini.get('directories','usersysabs'),1) elif targetpath.startswith('botssys'): targetpath = targetpath.replace('botssys',botsglobal.ini.get('directories','botssys'),1) elif targetpath.startswith('config'): targetpath = targetpath.replace('config',botsglobal.ini.get('directories','config'),1) targetpath = botslib.join(orgtargetpath, targetpath) #targetpath is OK now. botsglobal.logger.info(_(u' Start writing file: "%(targetpath)s".'),{'targetpath':targetpath}) if botslib.dirshouldbethere(os.path.dirname(targetpath)): botsglobal.logger.info(_(u' Create directory "%(directory)s".'),{'directory':os.path.dirname(targetpath)}) if zipfileobject.filename[-1] == '/': #check if this is a dir; if so continue continue if os.path.isfile(targetpath): #check if file already exists try: #this ***sometimes*** fails. (python25, for static/help/home.html...only there...) warnrenamed = True except: pass source = myzip.read(zipfileobject.filename) target = open(targetpath, "wb") target.write(source) target.close() botsglobal.logger.info(_(u' File written: "%(targetpath)s".'),{'targetpath':targetpath}) except: txt = botslib.txtexc() myzip.close() raise botslib.PluginError(_(u'Error writing files to system. Nothing is written to database. Error:\n%(txt)s'),{'txt':txt}) else: myzip.close() botsglobal.logger.info(_(u'Writing files to filesystem is OK.')) return warnrenamed
def sqlite3(): if sqlite_database_is_version3(): print 'Database sqlite3 is already bots version 3. No action is taken.' return 2 print 'Start changing sqlite3 database to bots version 3.' cursor = botsglobal.db.cursor() try: #channel **************************************** cursor.execute( '''ALTER TABLE "channel" ADD COLUMN "rsrv3" INTEGER DEFAULT 0''') cursor.execute( '''ALTER TABLE "channel" ADD COLUMN "keyfile" VARCHAR(256) ''') cursor.execute( '''ALTER TABLE "channel" ADD COLUMN "certfile" VARCHAR(256) ''') cursor.execute( '''ALTER TABLE "channel" ADD COLUMN "testpath" VARCHAR(256) ''') #filereport **************************************** cursor.execute('''DROP INDEX "filereport_reportidta" ''') cursor.execute( '''ALTER TABLE "filereport" ADD COLUMN "filesize" INTEGER DEFAULT 0''' ) #partner ************************************* cursor.execute( '''ALTER TABLE "partner" ADD COLUMN "attr1" VARCHAR(35) ''') cursor.execute( '''ALTER TABLE "partner" ADD COLUMN "attr2" VARCHAR(35) ''') cursor.execute( '''ALTER TABLE "partner" ADD COLUMN "attr3" VARCHAR(35) ''') cursor.execute( '''ALTER TABLE "partner" ADD COLUMN "attr4" VARCHAR(35) ''') cursor.execute( '''ALTER TABLE "partner" ADD COLUMN "attr5" VARCHAR(70) ''') cursor.execute( '''ALTER TABLE "partner" ADD COLUMN "name1" VARCHAR(70) ''') cursor.execute( '''ALTER TABLE "partner" ADD COLUMN "name2" VARCHAR(70) ''') cursor.execute( '''ALTER TABLE "partner" ADD COLUMN "name3" VARCHAR(70) ''') cursor.execute( '''ALTER TABLE "partner" ADD COLUMN "address1" VARCHAR(70) ''') cursor.execute( '''ALTER TABLE "partner" ADD COLUMN "address2" VARCHAR(70) ''') cursor.execute( '''ALTER TABLE "partner" ADD COLUMN "address3" VARCHAR(70) ''') cursor.execute( '''ALTER TABLE "partner" ADD COLUMN "city" VARCHAR(35) ''') cursor.execute( '''ALTER TABLE "partner" ADD COLUMN "postalcode" VARCHAR(17) ''') cursor.execute( '''ALTER TABLE "partner" ADD COLUMN "countrysubdivision" VARCHAR(9) ''' ) cursor.execute( '''ALTER TABLE "partner" ADD COLUMN "countrycode" VARCHAR(3) ''') cursor.execute( '''ALTER TABLE "partner" ADD COLUMN "phone1" VARCHAR(17) ''') cursor.execute( '''ALTER TABLE "partner" ADD COLUMN "phone2" VARCHAR(17) ''') cursor.execute('''ALTER TABLE "partner" ADD COLUMN "desc" TEXT ''') cursor.execute( '''ALTER TABLE "partner" ADD COLUMN "startdate" DATE ''') cursor.execute('''ALTER TABLE "partner" ADD COLUMN "enddate" DATE ''') #report **************************************** cursor.execute('''CREATE INDEX "report_ts" ON "report" ("ts")''') cursor.execute( '''ALTER TABLE "report" ADD COLUMN "filesize" INTEGER DEFAULT 0''') cursor.execute( '''ALTER TABLE "report" ADD COLUMN "acceptance" INTEGER DEFAULT 0''' ) #routes **************************************** cursor.execute( '''ALTER TABLE "routes" ADD COLUMN "zip_incoming" INTEGER DEFAULT 0''' ) cursor.execute( '''ALTER TABLE "routes" ADD COLUMN "zip_outgoing" INTEGER DEFAULT 0''' ) #ta **************************************** cursor.execute('''DROP INDEX "ta_script" ''') cursor.execute('''CREATE INDEX "ta_reference" ON "ta" ("reference")''') cursor.execute( '''ALTER TABLE "ta" ADD COLUMN "filesize" INTEGER DEFAULT 0''') cursor.execute( '''ALTER TABLE "ta" ADD COLUMN "numberofresends" INTEGER DEFAULT 0''' ) cursor.execute( '''ALTER TABLE "ta" ADD COLUMN "rsrv5" VARCHAR(35) DEFAULT '' ''') except: txt = botslib.txtexc() botsglobal.db.rollback() cursor.close() print 'Error in adding fields to sqlite3 database: "%s".' % (txt) return 1 else: botsglobal.db.commit() cursor.close() cursor = botsglobal.db.cursor() try: cursor.executescript(QUERYSTRING) except: txt = botslib.txtexc() botsglobal.db.rollback() cursor.close() print 'Error in changing sqlite3 database-schema "routes": "%s".' % ( txt) return 1 else: botsglobal.db.commit() cursor.close() print 'Succesful changed sqlite3 database to bots version 3.' return 0
def load(pathzipfile): ''' process uploaded plugin. ''' #test is valid zipfile if not zipfile.is_zipfile(pathzipfile): raise botslib.PluginError(_(u'Plugin is not a valid file.')) #read index file try: myzipimport = zipimport.zipimporter(pathzipfile) importedbotsindex = myzipimport.load_module('botsindex') pluglist = importedbotsindex.plugins[:] if 'botsindex' in sys.modules: del sys.modules['botsindex'] except: txt = botslib.txtexc() raise botslib.PluginError(_(u'Error in plugin. Nothing is written. Error:\n%s')%(txt)) else: botsglobal.logger.info(_(u'Plugin is OK.')) botsglobal.logger.info(_(u'Start writing to database.')) #write content of index file to the bots database try: writetodatabase(pluglist) except: txt = botslib.txtexc() raise botslib.PluginError(_(u'Error writing plugin to database. Nothing is written. Error:\n%s'%(txt))) else: botsglobal.logger.info(u'Writing to database is OK.') botsglobal.logger.info(u'Start writing to files') #write files to the file system. try: warnrenamed = False #to report in GUI files have been overwritten. myzip = zipfile.ZipFile(pathzipfile, mode="r") orgtargetpath = botsglobal.ini.get('directories','botspath') if (orgtargetpath[-1:] in (os.path.sep, os.path.altsep) and len(os.path.splitdrive(orgtargetpath)[1]) > 1): orgtargetpath = orgtargetpath[:-1] for zipfileobject in myzip.infolist(): if zipfileobject.filename not in ['botsindex.py','README','botssys/sqlitedb/botsdb','config/bots.ini'] and os.path.splitext(zipfileobject.filename)[1] not in ['.pyo','.pyc']: #~ botsglobal.logger.info(u'filename in zip "%s".',zipfileobject.filename) if zipfileobject.filename[0] == '/': targetpath = zipfileobject.filename[1:] else: targetpath = zipfileobject.filename targetpath = targetpath.replace('usersys',botsglobal.ini.get('directories','usersysabs'),1) targetpath = targetpath.replace('botssys',botsglobal.ini.get('directories','botssys'),1) targetpath = botslib.join(orgtargetpath, targetpath) #targetpath is OK now. botsglobal.logger.info(_(u' Start writing file: "%s".'),targetpath) if botslib.dirshouldbethere(os.path.dirname(targetpath)): botsglobal.logger.info(_(u' Create directory "%s".'),os.path.dirname(targetpath)) if zipfileobject.filename[-1] == '/': #check if this is a dir; if so continue continue if os.path.isfile(targetpath): #check if file already exists try: #this ***sometimes*** fails. (python25, for static/help/home.html...only there...) os.rename(targetpath,targetpath+'.'+time.strftime('%Y%m%d%H%M%S')) warnrenamed = True botsglobal.logger.info(_(u' Renamed existing file "%(from)s" to "%(to)s".'),{'from':targetpath,'to':targetpath+time.strftime('%Y%m%d%H%M%S')}) except: pass source = myzip.read(zipfileobject.filename) target = open(targetpath, "wb") target.write(source) target.close() botsglobal.logger.info(_(u' File written: "%s".'),targetpath) except: txt = botslib.txtexc() myzip.close() raise botslib.PluginError(_(u'Error writing files to system. Nothing is written to database. Error:\n%s')%(txt)) else: myzip.close() botsglobal.logger.info(_(u'Writing files to filesystem is OK.')) return warnrenamed
def extractexcel(ta_from, endstatus, **argv): ''' extract excel file. editype & messagetype are unchanged. ''' #***functions used by extractexcel #------------------------------------------------------------------------------- def read_xls(infilename): # Read excel first sheet into a 2-d array book = xlrd.open_workbook(infilename) sheet = book.sheet_by_index(0) formatter = lambda (t, v): format_excelval(book, t, v, False) xlsdata = [] for row in range(sheet.nrows): (types, values) = (sheet.row_types(row), sheet.row_values(row)) xlsdata.append(map(formatter, zip(types, values))) return xlsdata #------------------------------------------------------------------------------- def dump_csv(xlsdata, tofilename): stream = botslib.opendata(tofilename, 'wb') csvout = csv.writer(stream, quotechar=quotechar, delimiter=field_sep, doublequote=doublequote, escapechar=escape) csvout.writerows(map(utf8ize, xlsdata)) stream.close() #------------------------------------------------------------------------------- def format_excelval(book, type, value, wanttupledate): # Clean up the incoming excel data for some data types returnrow = [] if type == 2: if value == int(value): value = int(value) elif type == 3: datetuple = xlrd.xldate_as_tuple(value, book.datemode) value = datetuple if wanttupledate else tupledate_to_isodate( datetuple) elif type == 5: value = xlrd.error_text_from_code[value] return value #------------------------------------------------------------------------------- def tupledate_to_isodate(tupledate): # Turns a gregorian (year, month, day, hour, minute, nearest_second) into a # standard YYYY-MM-DDTHH:MM:SS ISO date. (y, m, d, hh, mm, ss) = tupledate nonzero = lambda n: n != 0 date = "%04d-%02d-%02d" % (y, m, d) if filter(nonzero, (y, m, d)) else '' time = "T%02d:%02d:%02d" % (hh, mm, ss) if filter( nonzero, (hh, mm, ss)) or not date else '' return date + time #------------------------------------------------------------------------------- def utf8ize(l): # Make string-like things into utf-8, leave other things alone return [ unicode(s).encode(charset) if hasattr(s, 'encode') else s for s in l ] #***end functions used by extractexcel import xlrd import csv #get parameters for csv-format; defaults are as the csv defaults (in grammar.py) charset = argv.get('charset', "utf-8") quotechar = argv.get('quotechar', "'") field_sep = argv.get('field_sep', ':') escape = argv.get('escape', '') if escape: doublequote = False else: doublequote = True try: infilename = botslib.abspathdata(ta_from.filename) xlsdata = read_xls(infilename) ta_to = ta_from.copyta(status=endstatus) tofilename = str(ta_to.idta) dump_csv(xlsdata, tofilename) ta_to.update( statust=OK, filename=tofilename) #update outmessage transaction with ta_info; botsglobal.logger.debug(_(u' File written: "%s".'), tofilename) except: txt = botslib.txtexc() botsglobal.logger.error( _(u'Excel extraction failed, may not be an Excel file? Error:\n%s' ), txt) raise botslib.InMessageError(_( u'Excel extraction failed, may not be an Excel file? Error:\n$error' ), error=txt)
def extractpdf(ta_from, endstatus, **argv): ''' Try to extract text content of a PDF file to a csv. You know this is not a great idea, right? But we'll do the best we can anyway! Page and line numbers are added to each row. Columns and rows are based on the x and y coordinates of each text element within tolerance allowed. Multiple text elements may combine to make one field, some PDFs have every character separated! You may need to experiment with x_group and y_group values, but defaults seem ok for most files. Output csv is UTF-8 encoded - The csv module doesn't directly support reading and writing Unicode If the PDF is just an image, all bets are off. Maybe try OCR, good luck with that! Mike Griffin 14/12/2011 ''' from pdfminer.pdfinterp import PDFResourceManager, process_pdf from pdfminer.converter import TextConverter from pdfminer.layout import LAParams, LTContainer, LTText, LTTextBox import csv class CsvConverter(TextConverter): def __init__(self, *args, **kwargs): TextConverter.__init__(self, *args, **kwargs) def receive_layout(self, ltpage): # recursively get every text element and it's coordinates def render(item): if isinstance(item, LTContainer): for child in item: render(child) elif isinstance(item, LTText): (_, _, x, y) = item.bbox # group the y values (rows) within group tolerance for v in yv: if y > v - y_group and y < v + y_group: y = v yv.append(y) line = lines[int(-y)] line[x] = item.get_text().encode('utf-8') from collections import defaultdict lines = defaultdict(lambda: {}) yv = [] render(ltpage) lineid = 0 for y in sorted(lines.keys()): line = lines[y] lineid += 1 csvdata = [ltpage.pageid, lineid] # first 2 columns are page and line numbers # group the x values (fields) within group tolerance p = 0 field_txt = '' for x in sorted(line.keys()): gap = x - p if p > 0 and gap > x_group: csvdata.append(field_txt) field_txt = '' field_txt += line[x] p = x csvdata.append(field_txt) csvout.writerow(csvdata) if lineid == 0: raise botslib.InMessageError( _(u'PDF text extraction failed, it may contain just image(s)?' )) #get some optional parameters x_group = argv.get('x_group', 10) # group text closer than this as one field y_group = argv.get('y_group', 5) # group lines closer than this as one line password = argv.get('password', '') quotechar = argv.get('quotechar', '"') field_sep = argv.get('field_sep', ',') escape = argv.get('escape', '\\') charset = argv.get('charset', 'utf-8') if not escape: doublequote = True else: doublequote = False try: pdf_stream = botslib.opendata(ta_from.filename, 'rb') ta_to = ta_from.copyta(status=endstatus) tofilename = str(ta_to.idta) csv_stream = botslib.opendata(tofilename, 'wb') csvout = csv.writer(csv_stream, quotechar=quotechar, delimiter=field_sep, doublequote=doublequote, escapechar=escape) # Process PDF rsrcmgr = PDFResourceManager(caching=True) device = CsvConverter(rsrcmgr, csv_stream, codec=charset) process_pdf(rsrcmgr, device, pdf_stream, pagenos=set(), password=password, caching=True, check_extractable=True) device.close() pdf_stream.close() csv_stream.close() ta_to.update( statust=OK, filename=tofilename) #update outmessage transaction with ta_info; botsglobal.logger.debug(_(u' File written: "%s".'), tofilename) except: txt = botslib.txtexc() botsglobal.logger.error( _(u'PDF extraction failed, may not be a PDF file? Error:\n%s'), txt) raise botslib.InMessageError( _(u'PDF extraction failed, may not be a PDF file? Error:\n$error'), error=txt)
def translate(startstatus=TRANSLATE, endstatus=TRANSLATED, idroute=''): ''' translates edifiles in one or more edimessages. reads and parses edifiles that have to be translated. tries to split files into messages (using 'nextmessage' of grammar); if no splitting: edifile is one message. searches the right translation in translate-table; runs the mapping-script for the translation; Function takes db-ta with status=TRANSLATE->PARSED->SPLITUP->TRANSLATED ''' #select edifiles to translate; fill ta-object for row in botslib.query( u'''SELECT idta,frompartner,topartner,filename,messagetype,testindicator,editype,charset,alt,fromchannel FROM ta WHERE idta>%(rootidta)s AND status=%(status)s AND statust=%(statust)s AND idroute=%(idroute)s ''', { 'status': startstatus, 'statust': OK, 'idroute': idroute, 'rootidta': botslib.get_minta4query() }): try: ta_fromfile = botslib.OldTransaction(row['idta']) #TRANSLATE ta ta_parsedfile = ta_fromfile.copyta( status=PARSED) #copy TRANSLATE to PARSED ta #read whole edi-file: read, parse and made into a inmessage-object. Message is represented a a tree. edifile = inmessage.edifromfile(frompartner=row['frompartner'], topartner=row['topartner'], filename=row['filename'], messagetype=row['messagetype'], testindicator=row['testindicator'], editype=row['editype'], charset=row['charset'], alt=row['alt'], fromchannel=row['fromchannel'], idroute=idroute) botsglobal.logger.debug( u'start read and parse input file "%s" editype "%s" messagetype "%s".', row['filename'], row['editype'], row['messagetype']) for inn in edifile.nextmessage( ): #for each message in the edifile: #inn.ta_info: parameters from inmessage.edifromfile(), syntax-information and parse-information ta_frommes = ta_parsedfile.copyta( status=SPLITUP) #copy PARSED to SPLITUP ta inn.ta_info[ 'idta_fromfile'] = ta_fromfile.idta #for confirmations in user script; used to give idta of 'confirming message' ta_frommes.update( **inn.ta_info ) #update ta-record SLIPTUP with info from message content and/or grammar while 1: #whileloop continues as long as there are alt-translations #************select parameters for translation(script): for row2 in botslib.query( u'''SELECT tscript,tomessagetype,toeditype FROM translate WHERE frommessagetype = %(frommessagetype)s AND fromeditype = %(fromeditype)s AND active=%(booll)s AND alt=%(alt)s AND (frompartner_id IS NULL OR frompartner_id=%(frompartner)s OR frompartner_id in (SELECT to_partner_id FROM partnergroup WHERE from_partner_id=%(frompartner)s )) AND (topartner_id IS NULL OR topartner_id=%(topartner)s OR topartner_id in (SELECT to_partner_id FROM partnergroup WHERE from_partner_id=%(topartner)s )) ORDER BY alt DESC, CASE WHEN frompartner_id IS NULL THEN 1 ELSE 0 END, frompartner_id , CASE WHEN topartner_id IS NULL THEN 1 ELSE 0 END, topartner_id ''', { 'frommessagetype': inn.ta_info['messagetype'], 'fromeditype': inn.ta_info['editype'], 'alt': inn.ta_info['alt'], 'frompartner': inn.ta_info['frompartner'], 'topartner': inn.ta_info['topartner'], 'booll': True }): break #escape if found; we need only the first - ORDER BY in the query else: #no translation record is found raise botslib.TranslationNotFoundError( _(u'Editype "$editype", messagetype "$messagetype", frompartner "$frompartner", topartner "$topartner", alt "$alt"' ), editype=inn.ta_info['editype'], messagetype=inn.ta_info['messagetype'], frompartner=inn.ta_info['frompartner'], topartner=inn.ta_info['topartner'], alt=inn.ta_info['alt']) ta_tomes = ta_frommes.copyta( status=endstatus) #copy SPLITUP to TRANSLATED ta tofilename = str(ta_tomes.idta) tscript = row2['tscript'] tomessage = outmessage.outmessage_init( messagetype=row2['tomessagetype'], editype=row2['toeditype'], filename=tofilename, reference=unique('messagecounter'), statust=OK, divtext=tscript) #make outmessage object #copy ta_info botsglobal.logger.debug( u'script "%s" translates messagetype "%s" to messagetype "%s".', tscript, inn.ta_info['messagetype'], tomessage.ta_info['messagetype']) translationscript, scriptfilename = botslib.botsimport( 'mappings', inn.ta_info['editype'] + '.' + tscript) #get the mapping-script doalttranslation = botslib.runscript(translationscript, scriptfilename, 'main', inn=inn, out=tomessage) botsglobal.logger.debug(u'script "%s" finished.', tscript) if 'topartner' not in tomessage.ta_info: #tomessage does not contain values from ta...... tomessage.ta_info['topartner'] = inn.ta_info[ 'topartner'] if tomessage.ta_info[ 'statust'] == DONE: #if indicated in user script the message should be discarded botsglobal.logger.debug( u'No output file because mapping script explicitly indicated this.' ) tomessage.ta_info['filename'] = '' tomessage.ta_info['status'] = DISCARD else: botsglobal.logger.debug( u'Start writing output file editype "%s" messagetype "%s".', tomessage.ta_info['editype'], tomessage.ta_info['messagetype']) tomessage.writeall( ) #write tomessage (result of translation). #problem is that not all values ta_tomes are know to to_message.... #~ print 'tomessage.ta_info',tomessage.ta_info ta_tomes.update( **tomessage.ta_info ) #update outmessage transaction with ta_info; #check the value received from the mappingscript to see if another traanslation needs to be done (chained translation) if doalttranslation is None: del tomessage break #break out of while loop; do no other translation elif isinstance(doalttranslation, dict): #some extended cases; a dict is returned that contains 'instructions' if 'type' not in doalttranslation: raise botslib.BotsError( "Mapping script returned dict. This dict does not have a 'type', like in eg: {'type:'out_as_inn', 'alt':'alt-value'}." ) if doalttranslation['type'] == u'out_as_inn': if 'alt' not in doalttranslation: raise botslib.BotsError( "Mapping script returned dict, type 'out_as_inn'. This dict does not have a 'alt'-value, like in eg: {'type:'out_as_inn', 'alt':'alt-value'}." ) inn = tomessage if isinstance(inn, outmessage.fixed): inn.root.stripnode() inn.ta_info['alt'] = doalttranslation[ 'alt'] #get the alt-value for the next chainded translation inn.ta_info.pop('statust') else: del tomessage inn.ta_info[ 'alt'] = doalttranslation #get the alt-value for the next chainded translation #end of while-loop # #~ print inn.ta_info ta_frommes.update( statust=DONE, **inn.ta_info ) #update db. inn.ta_info could be changed by script. Is this useful? del inn #exceptions file_in-level except: #~ edifile.handleconfirm(ta_fromfile,error=True) #only useful if errors are reported in acknowledgement (eg x12 997). Not used now. txt = botslib.txtexc() ta_parsedfile.failure() ta_parsedfile.update(statust=ERROR, errortext=txt) botsglobal.logger.debug( u'error in translating input file "%s":\n%s', row['filename'], txt) else: edifile.handleconfirm(ta_fromfile, error=False) ta_fromfile.update(statust=DONE) ta_parsedfile.update(statust=DONE, **edifile.confirminfo) botsglobal.logger.debug(u'translated input file "%s".', row['filename']) del edifile