def writeall(self): ''' Very different writeall: there is no tree of nodes; there is no grammar.structure/recorddefs; kid opens file by itself. ''' try: import kid except: txt = botslib.txtexc() raise ImportError(_(u'Dependency failure: editype "template" requires python library "kid". Error:\n%s'%txt)) #for template-grammar: only syntax is used. Section 'syntax' has to have 'template' self.outmessagegrammarread(self.ta_info['editype'],self.ta_info['messagetype']) templatefile = botslib.abspath(u'templates',self.ta_info['template']) try: botsglobal.logger.debug(u'Start writing to file "%s".',self.ta_info['filename']) ediprint = kid.Template(file=templatefile, data=self.data) except: txt = botslib.txtexc() raise botslib.OutMessageError(_(u'While templating "$editype.$messagetype", error:\n$txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt) try: filehandler = botslib.opendata(self.ta_info['filename'],'wb') ediprint.write(filehandler, encoding=self.ta_info['charset'], output=self.ta_info['output'], #output is specific parameter for class; init from grammar.syntax fragment=self.ta_info['merge']) except: txt = botslib.txtexc() raise botslib.OutMessageError(_(u'While templating "$editype.$messagetype", error:\n$txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt) botsglobal.logger.debug(_(u'End writing to file "%s".'),self.ta_info['filename'])
def writeall(self): ''' Very different writeall: there is no tree of nodes; there is no grammar.structure/recorddefs; kid opens file by itself. ''' try: from genshi.template import TemplateLoader except: txt = botslib.txtexc() raise ImportError(_(u'Dependency failure: editype "template" requires python library "genshi". Error:\n%s'%txt)) #for template-grammar: only syntax is used. Section 'syntax' has to have 'template' self.outmessagegrammarread(self.ta_info['editype'],self.ta_info['messagetype']) templatefile = botslib.abspath(u'templateshtml',self.ta_info['template']) try: botsglobal.logger.debug(u'Start writing to file "%s".',self.ta_info['filename']) loader = TemplateLoader(auto_reload=False) tmpl = loader.load(templatefile) except: txt = botslib.txtexc() raise botslib.OutMessageError(_(u'While templating "$editype.$messagetype", error:\n$txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt) try: filehandler = botslib.opendata(self.ta_info['filename'],'wb') stream = tmpl.generate(data=self.data) stream.render(method='xhtml',encoding=self.ta_info['charset'],out=filehandler) except: txt = botslib.txtexc() raise botslib.OutMessageError(_(u'While templating "$editype.$messagetype", error:\n$txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt) botsglobal.logger.debug(_(u'End writing to file "%s".'),self.ta_info['filename'])
def run(self): ''' class for template enveloping; delevers a valid html-file. Uses a kid-template for the enveloping/merging. use kid to write; no envelope grammar is used #20120101 depreciated. use class templatehtml ''' try: import kid except: raise ImportError(_(u'Dependency failure: editype "template" requires python library "kid".')) self._openoutenvelope() self.ta_info.update(self.out.ta_info) botslib.tryrunscript(self.userscript,self.scriptname,'ta_infocontent',ta_info=self.ta_info) if not self.ta_info['envelope-template']: raise botslib.OutMessageError(_(u'While enveloping in "%(editype)s.%(messagetype)s": syntax option "envelope-template" not filled; is required.'), self.ta_info) templatefile = botslib.abspath('templates',self.ta_info['envelope-template']) ta_list = self.filelist2absolutepaths() try: botsglobal.logger.debug(u'Start writing envelope to file "%(filename)s".',self.ta_info) ediprint = kid.Template(file=templatefile, data=ta_list) #init template; pass list with filenames except: txt = botslib.txtexc() raise botslib.OutMessageError(_(u'While enveloping in "%(editype)s.%(messagetype)s", error:\n%(txt)s'), {'editype':self.ta_info['editype'],'messagetype':self.ta_info['messagetype'],'txt':txt}) try: filehandler = botslib.opendata(self.ta_info['filename'],'wb') ediprint.write(filehandler, encoding=self.ta_info['charset'], output=self.ta_info['output']) except: txt = botslib.txtexc() raise botslib.OutMessageError(_(u'While enveloping in "%(editype)s.%(messagetype)s", error:\n%(txt)s'), {'editype':self.ta_info['editype'],'messagetype':self.ta_info['messagetype'],'txt':txt})
def botsunzip(ta_from,endstatus,password=None,pass_non_zip=False,**argv): ''' unzip file; editype & messagetype are unchanged. ''' try: z = zipfile.ZipFile(botslib.abspathdata(filename=ta_from.filename),mode='r') except zipfile.BadZipfile: botsglobal.logger.debug(_(u'File is not a zip-file.')) if pass_non_zip: #just pass the file botsglobal.logger.debug(_(u'"pass_non_zip" is True, just pass the file.')) ta_to = ta_from.copyta(status=endstatus,statust=OK) return raise botslib.InMessageError(_(u'File is not a zip-file.')) if password: z.setpassword(password) for f in z.infolist(): if f.filename[-1] == '/': #check if this is a dir; if so continue continue ta_to = ta_from.copyta(status=endstatus) tofilename = str(ta_to.idta) tofile = botslib.opendata(tofilename,'wb') tofile.write(z.read(f.filename)) tofile.close() ta_to.update(statust=OK,filename=tofilename) #update outmessage transaction with ta_info; botsglobal.logger.debug(_(u' File written: "%s".'),tofilename)
def run(self): ''' class for (test) orderprint; delevers a valid html-file. Uses a kid-template for the enveloping/merging. use kid to write; no envelope grammar is used ''' try: from genshi.template import TemplateLoader except: txt=botslib.txtexc() raise ImportError(_(u'Dependency failure: editype "template" requires python library "genshi". Error:\n%s'%txt)) defmessage = grammar.grammarread(self.ta_info['editype'],self.ta_info['messagetype']) #needed because we do not know envelope; read syntax for editype/messagetype self.ta_info.update(defmessage.syntax) botslib.tryrunscript(self.userscript,self.scriptname,'ta_infocontent',ta_info=self.ta_info) if not self.ta_info['envelope-template']: raise botslib.OutMessageError(_(u'While enveloping in "$editype.$messagetype": syntax option "envelope-template" not filled; is required.'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype']) templatefile = botslib.abspath('templateshtml',self.ta_info['envelope-template']) ta_list = self.filelist2absolutepaths() try: botsglobal.logger.debug(u'Start writing envelope to file "%s".',self.ta_info['filename']) loader = TemplateLoader(auto_reload=False) tmpl = loader.load(templatefile) except: txt=botslib.txtexc() raise botslib.OutMessageError(_(u'While enveloping in "$editype.$messagetype", error:\n$txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt) try: f = botslib.opendata(self.ta_info['filename'],'wb') stream = tmpl.generate(data=ta_list) stream.render(method='xhtml',encoding=self.ta_info['charset'],out=f) except: txt=botslib.txtexc() raise botslib.OutMessageError(_(u'While enveloping in "$editype.$messagetype", error:\n$txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt)
def run(self): try: from genshi.template import TemplateLoader except: raise ImportError(u'Dependency failure: editype "templatehtml" requires python library "genshi".') self._openoutenvelope() self.ta_info.update(self.out.ta_info) botslib.tryrunscript(self.userscript,self.scriptname,'ta_infocontent',ta_info=self.ta_info) if not self.ta_info['envelope-template']: raise botslib.OutMessageError(_(u'While enveloping in "%(editype)s.%(messagetype)s": syntax option "envelope-template" not filled; is required.'), self.ta_info) templatefile = botslib.abspath(self.__class__.__name__,self.ta_info['envelope-template']) ta_list = self.filelist2absolutepaths() try: botsglobal.logger.debug(u'Start writing envelope to file "%(filename)s".',self.ta_info) loader = TemplateLoader(auto_reload=False) tmpl = loader.load(templatefile) except: txt = botslib.txtexc() raise botslib.OutMessageError(_(u'While enveloping in "%(editype)s.%(messagetype)s", error:\n%(txt)s'), {'editype':self.ta_info['editype'],'messagetype':self.ta_info['messagetype'],'txt':txt}) try: filehandler = botslib.opendata(self.ta_info['filename'],'wb') stream = tmpl.generate(data=ta_list) stream.render(method='xhtml',encoding=self.ta_info['charset'],out=filehandler) except: txt = botslib.txtexc() raise botslib.OutMessageError(_(u'While enveloping in "%(editype)s.%(messagetype)s", error:\n%(txt)s'), {'editype':self.ta_info['editype'],'messagetype':self.ta_info['messagetype'],'txt':txt})
def writeall(self): ''' Very different writeall: there is no tree of nodes; there is no grammar.structure/recorddefs; kid opens file by itself. ''' try: from genshi.template import TemplateLoader except: txt=botslib.txtexc() raise ImportError(_(u'Dependency failure: editype "template" requires python library "genshi". Error:\n%s'%txt)) #for template-grammar: only syntax is used. Section 'syntax' has to have 'template' self.outmessagegrammarread(self.ta_info['editype'],self.ta_info['messagetype']) templatefile = botslib.abspath(u'templateshtml',self.ta_info['template']) try: botsglobal.logger.debug(u'Start writing to file "%s".',self.ta_info['filename']) loader = TemplateLoader(auto_reload=False) tmpl = loader.load(templatefile) except: txt=botslib.txtexc() raise botslib.OutMessageError(_(u'While templating "$editype.$messagetype", error:\n$txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt) try: f = botslib.opendata(self.ta_info['filename'],'wb') stream = tmpl.generate(data=self.data) stream.render(method='xhtml',encoding=self.ta_info['charset'],out=f) except: txt=botslib.txtexc() raise botslib.OutMessageError(_(u'While templating "$editype.$messagetype", error:\n$txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt) botsglobal.logger.debug(_(u'End writing to file "%s".'),self.ta_info['filename'])
def writeall(self): ''' Very different writeall: there is no tree of nodes; there is no grammar.structure/recorddefs; kid opens file by itself. ''' try: import kid except: txt=botslib.txtexc() raise ImportError(_(u'Dependency failure: editype "template" requires python library "kid". Error:\n%s'%txt)) #for template-grammar: only syntax is used. Section 'syntax' has to have 'template' self.outmessagegrammarread(self.ta_info['editype'],self.ta_info['messagetype']) templatefile = botslib.abspath(u'templates',self.ta_info['template']) try: botsglobal.logger.debug(u'Start writing to file "%s".',self.ta_info['filename']) ediprint = kid.Template(file=templatefile, data=self.data) except: txt=botslib.txtexc() raise botslib.OutMessageError(_(u'While templating "$editype.$messagetype", error:\n$txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt) try: f = botslib.opendata(self.ta_info['filename'],'wb') ediprint.write(f, #~ ediprint.write(botslib.abspathdata(self.ta_info['filename']), encoding=self.ta_info['charset'], output=self.ta_info['output'], #output is specific parameter for class; init from grammar.syntax fragment=self.ta_info['merge']) except: txt=botslib.txtexc() raise botslib.OutMessageError(_(u'While templating "$editype.$messagetype", error:\n$txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt) botsglobal.logger.debug(_(u'End writing to file "%s".'),self.ta_info['filename'])
def botsunzip(ta_from, endstatus, password=None, pass_non_zip=False, **argv): ''' unzip file; editype & messagetype are unchanged. ''' try: z = zipfile.ZipFile(botslib.abspathdata(filename=ta_from.filename), mode='r') except zipfile.BadZipfile: botsglobal.logger.debug(_(u'File is not a zip-file.')) if pass_non_zip: #just pass the file botsglobal.logger.debug( _(u'"pass_non_zip" is True, just pass the file.')) ta_to = ta_from.copyta(status=endstatus, statust=OK) return raise botslib.InMessageError(_(u'File is not a zip-file.')) if password: z.setpassword(password) for f in z.infolist(): if f.filename[-1] == '/': #check if this is a dir; if so continue continue ta_to = ta_from.copyta(status=endstatus) tofilename = str(ta_to.idta) tofile = botslib.opendata(tofilename, 'wb') tofile.write(z.read(f.filename)) tofile.close() ta_to.update( statust=OK, filename=tofilename) #update outmessage transaction with ta_info; botsglobal.logger.debug(_(u' File written: "%s".'), tofilename)
def run(self): ''' class for (test) orderprint; delevers a valid html-file. Uses a kid-template for the enveloping/merging. use kid to write; no envelope grammar is used ''' import kid defmessage = grammar.grammarread(self.ta_info['editype'],self.ta_info['messagetype']) #needed beause we do not know envelope; read syntax for editype/messagetype self.ta_info.update(defmessage.syntax) botslib.tryrunscript(self.userscript,self.scriptname,'ta_infocontent',ta_info=self.ta_info) if not self.ta_info['envelope-template']: raise botslib.OutMessageError(_(u'While enveloping in "$editype.$messagetype": syntax option "envelope-template" not filled; is required.'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype']) templatefile = botslib.abspath('templates',self.ta_info['envelope-template']) ta_list = self.filelist2absolutepaths() try: botsglobal.logger.debug(u'Start writing envelope to file "%s".',self.ta_info['filename']) ediprint = kid.Template(file=templatefile, data=ta_list) #init template; pass list with filenames except: txt=botslib.txtexc() raise botslib.OutMessageError(_(u'While enveloping in "$editype.$messagetype": $txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt) try: f = botslib.opendata(self.ta_info['filename'],'wb') ediprint.write(f, encoding=self.ta_info['charset'], output=self.ta_info['output']) except: txt=botslib.txtexc() raise botslib.OutMessageError(_(u'While enveloping error in "$editype.$messagetype"; probably in html file(s) to be merged: $txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt)
def _initwrite(self): botsglobal.logger.debug(u'Start writing to file "%s".', self.ta_info['filename']) self._outstream = botslib.opendata( self.ta_info['filename'], 'wb', charset=self.ta_info['charset'], errors=self.ta_info['checkcharsetout'])
def run(self): botslib.tryrunscript(self.userscript,self.scriptname,'ta_infocontent',ta_info=self.ta_info) if len(self.ta_list) > 1: tofile = botslib.opendata(self.ta_info['filename'],'wb',self.ta_info['charset']) self.writefilelist(tofile) tofile.close() else: self.ta_info['filename'] = self.ta_list[0]
def run(self): ''' class for (test) orderprint; delevers a valid html-file. Uses a kid-template for the enveloping/merging. use kid to write; no envelope grammar is used ''' try: import kid except: txt = botslib.txtexc() raise ImportError( _(u'Dependency failure: editype "template" requires python library "kid". Error:\n%s' % txt)) defmessage = grammar.grammarread( self.ta_info['editype'], self.ta_info['messagetype'] ) #needed because we do not know envelope; read syntax for editype/messagetype self.ta_info.update(defmessage.syntax) botslib.tryrunscript(self.userscript, self.scriptname, 'ta_infocontent', ta_info=self.ta_info) if not self.ta_info['envelope-template']: raise botslib.OutMessageError(_( u'While enveloping in "$editype.$messagetype": syntax option "envelope-template" not filled; is required.' ), editype=self.ta_info['editype'], messagetype=self. ta_info['messagetype']) templatefile = botslib.abspath('templates', self.ta_info['envelope-template']) ta_list = self.filelist2absolutepaths() try: botsglobal.logger.debug(u'Start writing envelope to file "%s".', self.ta_info['filename']) ediprint = kid.Template( file=templatefile, data=ta_list) #init template; pass list with filenames except: txt = botslib.txtexc() raise botslib.OutMessageError( _(u'While enveloping in "$editype.$messagetype", error:\n$txt' ), editype=self.ta_info['editype'], messagetype=self.ta_info['messagetype'], txt=txt) try: filehandler = botslib.opendata(self.ta_info['filename'], 'wb') ediprint.write(filehandler, encoding=self.ta_info['charset'], output=self.ta_info['output']) except: txt = botslib.txtexc() raise botslib.OutMessageError( _(u'While enveloping in "$editype.$messagetype", error:\n$txt' ), editype=self.ta_info['editype'], messagetype=self.ta_info['messagetype'], txt=txt)
def dump_csv(xlsdata, tofilename): stream = botslib.opendata(tofilename, 'wb') csvout = csv.writer(stream, quotechar=quotechar, delimiter=field_sep, doublequote=doublequote, escapechar=escape) csvout.writerows(map(utf8ize, xlsdata)) stream.close()
def writeall(self): if self.root is None: raise botslib.OutMessageError(_(u'No outgoing message')) #then there is nothing to write... botsglobal.logger.debug(u'Start writing to file "%s".',self.ta_info['filename']) self._outstream = botslib.opendata(self.ta_info['filename'],'wb') self._outstream.write(self.root) self._outstream.close() botsglobal.logger.debug(u'End writing to file "%s".',self.ta_info['filename']) self.ta_info['envelope'] = 'raw' #use right enveloping for raw: no coping etc, use same file.
def run(self): botslib.tryrunscript(self.userscript, self.scriptname, 'ta_infocontent', ta_info=self.ta_info) tofile = botslib.opendata(self.ta_info['filename'], 'wb', self.ta_info['charset']) self.writefilelist(tofile) tofile.close()
def run(self): ''' class for (test) orderprint; delevers a valid html-file. Uses a kid-template for the enveloping/merging. use kid to write; no envelope grammar is used ''' try: from genshi.template import TemplateLoader except: txt = botslib.txtexc() raise ImportError( _(u'Dependency failure: editype "template" requires python library "genshi". Error:\n%s' % txt)) defmessage = grammar.grammarread( self.ta_info['editype'], self.ta_info['messagetype'] ) #needed because we do not know envelope; read syntax for editype/messagetype self.ta_info.update(defmessage.syntax) botslib.tryrunscript(self.userscript, self.scriptname, 'ta_infocontent', ta_info=self.ta_info) if not self.ta_info['envelope-template']: raise botslib.OutMessageError(_( u'While enveloping in "$editype.$messagetype": syntax option "envelope-template" not filled; is required.' ), editype=self.ta_info['editype'], messagetype=self. ta_info['messagetype']) templatefile = botslib.abspath('templateshtml', self.ta_info['envelope-template']) ta_list = self.filelist2absolutepaths() try: botsglobal.logger.debug(u'Start writing envelope to file "%s".', self.ta_info['filename']) loader = TemplateLoader(auto_reload=False) tmpl = loader.load(templatefile) except: txt = botslib.txtexc() raise botslib.OutMessageError( _(u'While enveloping in "$editype.$messagetype", error:\n$txt' ), editype=self.ta_info['editype'], messagetype=self.ta_info['messagetype'], txt=txt) try: filehandler = botslib.opendata(self.ta_info['filename'], 'wb') stream = tmpl.generate(data=ta_list) stream.render(method='xhtml', encoding=self.ta_info['charset'], out=filehandler) except: txt = botslib.txtexc() raise botslib.OutMessageError( _(u'While enveloping in "$editype.$messagetype", error:\n$txt' ), editype=self.ta_info['editype'], messagetype=self.ta_info['messagetype'], txt=txt)
def writeall(self): if self.root is None: raise botslib.OutMessageError(_(u'No outgoing message')) #then there is nothing to write... botsglobal.logger.debug(u'Start writing to file "%(filename)s".',self.ta_info) self._outstream = botslib.opendata(self.ta_info['filename'],'wb') self._outstream.write(self.root) self._outstream.close() botsglobal.logger.debug(u'End writing to file "%(filename)s".',self.ta_info) self.ta_info['envelope'] = 'raw' self.ta_info['merge'] = False
def run(self): self._openoutenvelope(self.ta_info["editype"], self.ta_info["messagetype"]) botslib.tryrunscript(self.userscript, self.scriptname, "ta_infocontent", ta_info=self.ta_info) # self.ta_info is not overwritten tofile = botslib.opendata(self.ta_info["filename"], "wb", self.ta_info["charset"]) headers = dict([(field[ID], field[ID]) for field in self.out.defmessage.structure[0][FIELDS]]) self.out.put(headers) self.out.tree2records(self.out.root) tofile.write(self.out.record2string(self.out.records[0])) self.writefilelist(tofile) tofile.close()
def run(self): self._openoutenvelope(self.ta_info['editype'],self.ta_info['messagetype']) botslib.tryrunscript(self.userscript,self.scriptname,'ta_infocontent',ta_info=self.ta_info) #self.ta_info is not overwritten tofile = botslib.opendata(self.ta_info['filename'],'wb',self.ta_info['charset']) if len(self.out.defmessage.recorddefs) == 1 : #if only only type of record headers = dict([(field[ID],field[ID]) for field in self.out.defmessage.structure[0][FIELDS]]) self.out.put(headers) self.out.tree2records(self.out.root) tofile.write(self.out._record2string(self.out.records[0])) self.writefilelist(tofile) tofile.close()
def _write(self, node): ''' write normal XML messages (no envelope)''' xmltree = ET.ElementTree(self._node2xml(node)) f = botslib.opendata(self.ta_info['filename'], "wb") if self.ta_info['indented']: indentstring = '\n' else: indentstring = '' #xml prolog: always use.********************************* #syntax parameter controls if stand-alone is used within prolog. #in ET 1.3.0: if standalone is to be used: should surpress ET-generated prolog - explicit parameter #in ET 1.2.6: always generates prolog if encoding != utf-8/ascii. SO: can not use stadnalone for encoding !=utf-8,ascii if ET.VERSION not in ['1.2.6', '1.0.6'] or self.ta_info['charset'] in [ 'us-ascii', 'utf-8' ]: if self.ta_info['standalone']: standalonestring = 'standalone="%s" ' % ( self.ta_info['standalone']) else: standalonestring = '' PI = ET.ProcessingInstruction( 'xml', 'version="%s" encoding="%s" %s' % (self.ta_info['version'], self.ta_info['charset'], standalonestring)) f.write( ET.tostring(PI) + indentstring ) #do not use encoding here. gives double xml prolog; possibly because ET.ElementTree.write i used again by write() #doctype /DTD ************************************** if self.ta_info['DOCTYPE']: f.write('<!DOCTYPE %s>' % (self.ta_info['DOCTYPE']) + indentstring) #processing instructions (other than prolog) ************ if self.ta_info['processing_instructions']: for pi in self.ta_info['processing_instructions']: PI = ET.ProcessingInstruction(pi[0], pi[1]) f.write( ET.tostring(PI) + indentstring ) #do not use encoding here. gives double xml prolog; possibly because ET.ElementTree.write i used again by write() #indent the xml elements if self.ta_info['indented']: root = xmltree.getroot() self.botsindent(root) if ET.VERSION <= '1.2.6': xmltree.write(f, encoding=self.ta_info['charset']) else: xmltree.write(f, encoding=self.ta_info['charset'], xml_declaration=False)
def run(self): try: from genshi.template import TemplateLoader except: raise ImportError( u'Dependency failure: editype "templatehtml" requires python library "genshi".' ) self._openoutenvelope() self.ta_info.update(self.out.ta_info) botslib.tryrunscript(self.userscript, self.scriptname, 'ta_infocontent', ta_info=self.ta_info) if not self.ta_info['envelope-template']: raise botslib.OutMessageError( _(u'While enveloping in "%(editype)s.%(messagetype)s": syntax option "envelope-template" not filled; is required.' ), self.ta_info) templatefile = botslib.abspath(self.__class__.__name__, self.ta_info['envelope-template']) ta_list = self.filelist2absolutepaths() try: botsglobal.logger.debug( u'Start writing envelope to file "%(filename)s".', self.ta_info) loader = TemplateLoader(auto_reload=False) tmpl = loader.load(templatefile) except: txt = botslib.txtexc() raise botslib.OutMessageError( _(u'While enveloping in "%(editype)s.%(messagetype)s", error:\n%(txt)s' ), { 'editype': self.ta_info['editype'], 'messagetype': self.ta_info['messagetype'], 'txt': txt }) try: filehandler = botslib.opendata(self.ta_info['filename'], 'wb') stream = tmpl.generate(data=ta_list) stream.render(method='xhtml', encoding=self.ta_info['charset'], out=filehandler) except: txt = botslib.txtexc() raise botslib.OutMessageError( _(u'While enveloping in "%(editype)s.%(messagetype)s", error:\n%(txt)s' ), { 'editype': self.ta_info['editype'], 'messagetype': self.ta_info['messagetype'], 'txt': txt })
def run(self): if not self.ta_info["topartner"] or not self.ta_info["frompartner"]: raise botslib.OutMessageError( _(u'In enveloping "frompartner" or "topartner" unknown: "$ta_info".'), ta_info=self.ta_info ) self._openoutenvelope(self.ta_info["editype"], self.ta_info["envelope"]) self.ta_info.update(self.out.ta_info) botslib.tryrunscript(self.userscript, self.scriptname, "ta_infocontent", ta_info=self.ta_info) # prepare data for envelope if botsglobal.ini.getboolean("settings", "interchangecontrolperpartner", False): self.ta_info["reference"] = str(botslib.unique("stxcounter_" + self.ta_info["topartner"])) else: self.ta_info["reference"] = str(botslib.unique("stxcounter_" + self.ta_info["frompartner"])) # build the envelope segments (that is, the tree from which the segments will be generated) self.out.put( { "BOTSID": "STX", "STDS1": self.ta_info["STX.STDS1"], "STDS2": self.ta_info["STX.STDS2"], "FROM.01": self.ta_info["frompartner"], "UNTO.01": self.ta_info["topartner"], "TRDT.01": time.strftime("%y%m%d"), "TRDT.02": time.strftime("%H%M%S"), "SNRF": self.ta_info["reference"], } ) if self.ta_info["STX.FROM.02"]: self.out.put({"BOTSID": "STX", "FROM.02": self.ta_info["STX.FROM.02"]}) if self.ta_info["STX.UNTO.02"]: self.out.put({"BOTSID": "STX", "UNTO.02": self.ta_info["STX.UNTO.02"]}) if self.ta_info["STX.APRF"]: self.out.put({"BOTSID": "STX", "APRF": self.ta_info["STX.APRF"]}) if self.ta_info["STX.PRCD"]: self.out.put({"BOTSID": "STX", "PRCD": self.ta_info["STX.PRCD"]}) self.out.put( {"BOTSID": "STX"}, {"BOTSID": "END", "NMST": self.ta_info["nrmessages"]} ) # dummy segment; is not used # user exit botslib.tryrunscript(self.userscript, self.scriptname, "envelopecontent", ta_info=self.ta_info, out=self.out) # convert the tree into segments; here only the STX is written (first segment) self.out.checkmessage(self.out.root, self.out.defmessage) self.out.tree2records(self.out.root) # start doing the actual writing: tofile = botslib.opendata(self.ta_info["filename"], "wb", self.ta_info["charset"]) tofile.write(self.out.record2string(self.out.records[0])) self.writefilelist(tofile) tofile.write(self.out.record2string(self.out.records[-1])) tofile.close()
def run(self): """ class for (test) orderprint; delevers a valid html-file. Uses a kid-template for the enveloping/merging. use kid to write; no envelope grammar is used """ try: import kid except: txt = botslib.txtexc() raise ImportError( _(u'Dependency failure: editype "template" requires python library "kid". Error:\n%s' % txt) ) defmessage = grammar.grammarread( self.ta_info["editype"], self.ta_info["messagetype"] ) # needed because we do not know envelope; read syntax for editype/messagetype self.ta_info.update(defmessage.syntax) botslib.tryrunscript(self.userscript, self.scriptname, "ta_infocontent", ta_info=self.ta_info) if not self.ta_info["envelope-template"]: raise botslib.OutMessageError( _( u'While enveloping in "$editype.$messagetype": syntax option "envelope-template" not filled; is required.' ), editype=self.ta_info["editype"], messagetype=self.ta_info["messagetype"], ) templatefile = botslib.abspath("templates", self.ta_info["envelope-template"]) ta_list = self.filelist2absolutepaths() try: botsglobal.logger.debug(u'Start writing envelope to file "%s".', self.ta_info["filename"]) ediprint = kid.Template(file=templatefile, data=ta_list) # init template; pass list with filenames except: txt = botslib.txtexc() raise botslib.OutMessageError( _(u'While enveloping in "$editype.$messagetype", error:\n$txt'), editype=self.ta_info["editype"], messagetype=self.ta_info["messagetype"], txt=txt, ) try: filehandler = botslib.opendata(self.ta_info["filename"], "wb") ediprint.write(filehandler, encoding=self.ta_info["charset"], output=self.ta_info["output"]) except: txt = botslib.txtexc() raise botslib.OutMessageError( _(u'While enveloping in "$editype.$messagetype", error:\n$txt'), editype=self.ta_info["editype"], messagetype=self.ta_info["messagetype"], txt=txt, )
def run(self): if self.ta_info['envelope'] == 'csvheader': #~ Adds first line to csv files with fieldnames; than write files. self._openoutenvelope() botslib.tryrunscript(self.userscript,self.scriptname,'ta_infocontent',ta_info=self.ta_info) tofile = botslib.opendata(self.ta_info['filename'],'wb',self.ta_info['charset']) headers = dict((field[ID],field[ID]) for field in self.out.defmessage.structure[0][FIELDS]) self.out.put(headers) self.out.tree2records(self.out.root) tofile.write(self.out.record2string(self.out.lex_records[0:1])) self.writefilelist(tofile) tofile.close() else: super(csv,self).run()
def run(self): self._openoutenvelope(self.ta_info['editype'], self.ta_info['messagetype']) botslib.tryrunscript(self.userscript, self.scriptname, 'ta_infocontent', ta_info=self.ta_info) #self.ta_info is not overwritten tofile = botslib.opendata(self.ta_info['filename'], 'wb', self.ta_info['charset']) headers = dict([(field[ID], field[ID]) for field in self.out.defmessage.structure[0][FIELDS]]) self.out.put(headers) self.out.tree2records(self.out.root) tofile.write(self.out.record2string(self.out.records[0])) self.writefilelist(tofile) tofile.close()
def run(self): if not self.ta_info['topartner'] or not self.ta_info['frompartner']: raise botslib.OutMessageError(_(u'In enveloping "frompartner" or "topartner" unknown: "%(ta_info)s".'), {'ta_info':self.ta_info}) self._openoutenvelope() self.ta_info.update(self.out.ta_info) botslib.tryrunscript(self.userscript,self.scriptname,'ta_infocontent',ta_info=self.ta_info) #prepare data for envelope if botsglobal.ini.getboolean('settings','interchangecontrolperpartner',False): self.ta_info['reference'] = unicode(botslib.unique('stxcounter_' + self.ta_info['topartner'])) else: self.ta_info['reference'] = unicode(botslib.unique('stxcounter_' + self.ta_info['frompartner'])) #build the envelope segments (that is, the tree from which the segments will be generated) self.out.put({'BOTSID':'STX', 'STDS1':self.ta_info['STX.STDS1'], 'STDS2':self.ta_info['STX.STDS2'], 'FROM.01':self.ta_info['frompartner'], 'UNTO.01':self.ta_info['topartner'], 'TRDT.01':botslib.strftime('%y%m%d'), 'TRDT.02':botslib.strftime('%H%M%S'), 'SNRF':self.ta_info['reference']}) if self.ta_info['STX.FROM.02']: self.out.put({'BOTSID':'STX','FROM.02':self.ta_info['STX.FROM.02']}) if self.ta_info['STX.UNTO.02']: self.out.put({'BOTSID':'STX','UNTO.02':self.ta_info['STX.UNTO.02']}) if self.ta_info['STX.APRF']: self.out.put({'BOTSID':'STX','APRF':self.ta_info['STX.APRF']}) if self.ta_info['STX.PRCD']: self.out.put({'BOTSID':'STX','PRCD':self.ta_info['STX.PRCD']}) self.out.put({'BOTSID':'STX'},{'BOTSID':'END','NMST':self.ta_info['nrmessages']}) #dummy segment; is not used #user exit botslib.tryrunscript(self.userscript,self.scriptname,'envelopecontent',ta_info=self.ta_info,out=self.out) #convert the tree into segments; here only the STX is written (first segment) self.out.checkmessage(self.out.root,self.out.defmessage) self.out.checkforerrorlist() self.out.tree2records(self.out.root) #start doing the actual writing: tofile = botslib.opendata(self.ta_info['filename'],'wb',self.ta_info['charset']) tofile.write(self.out.record2string(self.out.lex_records[0:1])) self.writefilelist(tofile) tofile.write(self.out.record2string(self.out.lex_records[1:2])) tofile.close()
def run(self): if self.ta_info['envelope'] == 'csvheader': #~ Adds first line to csv files with fieldnames; than write files. self._openoutenvelope() botslib.tryrunscript(self.userscript, self.scriptname, 'ta_infocontent', ta_info=self.ta_info) tofile = botslib.opendata(self.ta_info['filename'], 'wb', self.ta_info['charset']) headers = dict( (field[ID], field[ID]) for field in self.out.defmessage.structure[0][FIELDS]) self.out.put(headers) self.out.tree2records(self.out.root) tofile.write(self.out.record2string(self.out.lex_records[0:1])) self.writefilelist(tofile) tofile.close() else: super(csv, self).run()
def _write(self,node): ''' write normal XML messages (no envelope)''' xmltree = ET.ElementTree(self._node2xml(node)) f = botslib.opendata(self.ta_info['filename'],"wb") if self.ta_info['indented']: indentstring = '\n' else: indentstring = '' #xml prolog: always use.********************************* #syntax parameter controls if stand-alone is used within prolog. #in ET 1.3.0: if standalone is to be used: should surpress ET-generated prolog - explicit parameter #in ET 1.2.6: always generates prolog if encoding != utf-8/ascii. SO: can not use stadnalone for encoding !=utf-8,ascii if ET.VERSION not in ['1.2.6','1.0.6'] or self.ta_info['charset'] in ['us-ascii','utf-8']: if self.ta_info['standalone']: standalonestring = 'standalone="%s" '%(self.ta_info['standalone']) else: standalonestring = '' PI = ET.ProcessingInstruction('xml', 'version="%s" encoding="%s" %s'%(self.ta_info['version'],self.ta_info['charset'], standalonestring)) f.write(ET.tostring(PI) + indentstring) #do not use encoding here. gives double xml prolog; possibly because ET.ElementTree.write i used again by write() #doctype /DTD ************************************** if self.ta_info['DOCTYPE']: f.write('<!DOCTYPE %s>'%(self.ta_info['DOCTYPE']) + indentstring) #processing instructions (other than prolog) ************ if self.ta_info['processing_instructions']: for pi in self.ta_info['processing_instructions']: PI = ET.ProcessingInstruction(pi[0], pi[1]) f.write(ET.tostring(PI) + indentstring) #do not use encoding here. gives double xml prolog; possibly because ET.ElementTree.write i used again by write() #indent the xml elements if self.ta_info['indented']: root = xmltree.getroot() self.botsindent(root) if ET.VERSION <= '1.2.6': xmltree.write(f,encoding=self.ta_info['charset']) else: xmltree.write(f,encoding=self.ta_info['charset'],xml_declaration=False)
def _write(self,node_instance): templatefile = botslib.abspath(self.__class__.__name__,self.ta_info['template']) try: botsglobal.logger.debug(u'Start writing to file "%(filename)s".',self.ta_info) loader = self.template.TemplateLoader(auto_reload=False) tmpl = loader.load(templatefile) except: txt = botslib.txtexc() raise botslib.OutMessageError(_(u'While templating "%(editype)s.%(messagetype)s", error:\n%(txt)s'), {'editype':self.ta_info['editype'],'messagetype':self.ta_info['messagetype'],'txt':txt}) try: filehandler = botslib.opendata(self.ta_info['filename'],'wb') if self.ta_info['has_structure']: #new way of working if self.ta_info['print_as_row']: node_instance.collectlines(self.ta_info['print_as_row']) stream = tmpl.generate(node=node_instance) else: stream = tmpl.generate(data=self.data) stream.render(method='xhtml',encoding=self.ta_info['charset'],out=filehandler) except: txt = botslib.txtexc() raise botslib.OutMessageError(_(u'While templating "%(editype)s.%(messagetype)s", error:\n%(txt)s'), {'editype':self.ta_info['editype'],'messagetype':self.ta_info['messagetype'],'txt':txt}) botsglobal.logger.debug(_(u'End writing to file "%(filename)s".'),self.ta_info)
def run(self): self._openoutenvelope(self.ta_info['editype'], self.ta_info['envelope']) self.ta_info.update(self.out.ta_info) #need to know the functionalgroup code: defmessage = grammar.grammarread(self.ta_info['editype'], self.ta_info['messagetype']) self.ta_info['functionalgroup'] = defmessage.syntax['functionalgroup'] botslib.tryrunscript(self.userscript, self.scriptname, 'ta_infocontent', ta_info=self.ta_info) #prepare data for envelope ISA09date = time.strftime('%y%m%d') #test indicator can either be from configuration (self.ta_info['ISA15']) or by mapping (self.ta_info['testindicator']) #mapping overrules. if self.ta_info['testindicator'] and self.ta_info[ 'testindicator'] != '0': #'0' is default value (in db) testindicator = self.ta_info['testindicator'] else: testindicator = self.ta_info['ISA15'] #~ print self.ta_info['messagetype'], 'grammar:',self.ta_info['ISA15'],'ta:',self.ta_info['testindicator'],'out:',testindicator if botsglobal.ini.getboolean('settings', 'interchangecontrolperpartner', False): self.ta_info['reference'] = str( botslib.unique('isacounter_' + self.ta_info['topartner'])) else: self.ta_info['reference'] = str( botslib.unique('isacounter_' + self.ta_info['frompartner'])) #ISA06 and GS02 can be different; eg ISA06 is a service provider. #ISA06 and GS02 can be in the syntax.... ISA06 = self.ta_info.get('ISA06', self.ta_info['frompartner']) GS02 = self.ta_info.get('GS02', self.ta_info['frompartner']) #also for ISA08 and GS03 ISA08 = self.ta_info.get('ISA08', self.ta_info['topartner']) GS03 = self.ta_info.get('GS03', self.ta_info['topartner']) #build the envelope segments (that is, the tree from which the segments will be generated) self.out.put({ 'BOTSID': 'ISA', 'ISA01': self.ta_info['ISA01'], 'ISA02': self.ta_info['ISA02'], 'ISA03': self.ta_info['ISA03'], 'ISA04': self.ta_info['ISA04'], 'ISA05': self.ta_info['ISA05'], 'ISA06': ISA06, 'ISA07': self.ta_info['ISA07'], 'ISA08': ISA08, 'ISA09': ISA09date, 'ISA10': time.strftime('%H%M'), 'ISA11': self.ta_info[ 'ISA11'], #if ISA version > 00403, replaced by reprtion separator 'ISA12': self.ta_info['version'], 'ISA13': self.ta_info['reference'], 'ISA14': self.ta_info['ISA14'], 'ISA15': testindicator }) self.out.put({'BOTSID': 'ISA'}, { 'BOTSID': 'IEA', 'IEA01': '1', 'IEA02': self.ta_info['reference'] }) GS08 = self.ta_info['messagetype'][3:] if GS08[:6] < '004010': GS04date = time.strftime('%y%m%d') else: GS04date = time.strftime('%Y%m%d') self.out.put({'BOTSID': 'ISA'}, { 'BOTSID': 'GS', 'GS01': self.ta_info['functionalgroup'], 'GS02': GS02, 'GS03': GS03, 'GS04': GS04date, 'GS05': time.strftime('%H%M'), 'GS06': self.ta_info['reference'], 'GS07': self.ta_info['GS07'], 'GS08': GS08 }) self.out.put({'BOTSID': 'ISA'}, {'BOTSID': 'GS'}, { 'BOTSID': 'GE', 'GE01': self.ta_info['nrmessages'], 'GE02': self.ta_info['reference'] }) #dummy segment; is not used #user exit botslib.tryrunscript(self.userscript, self.scriptname, 'envelopecontent', ta_info=self.ta_info, out=self.out) #convert the tree into segments; here only the UNB is written (first segment) self.out.normalisetree(self.out.root) self.out.tree2records(self.out.root) #start doing the actual writing: tofile = botslib.opendata(self.ta_info['filename'], 'wb', self.ta_info['charset']) ISAstring = self.out._record2string(self.out.records[0]) if self.ta_info['version'] < '00403': ISAstring = ISAstring[:103] + self.ta_info[ 'field_sep'] + self.ta_info['sfield_sep'] + ISAstring[ 103:] #hack for strange characters at end of ISA; hardcoded else: ISAstring = ISAstring[:82] + self.ta_info['reserve'] + ISAstring[ 83:103] + self.ta_info['field_sep'] + self.ta_info[ 'sfield_sep'] + ISAstring[ 103:] #hack for strange characters at end of ISA; hardcoded tofile.write(ISAstring) #write ISA tofile.write(self.out._record2string(self.out.records[1])) #write GS self.writefilelist(tofile) tofile.write(self.out._record2string(self.out.records[-2])) #write GE tofile.write(self.out._record2string(self.out.records[-1])) #write IEA tofile.close() if self.ta_info[ 'functionalgroup'] != 'FA' and botslib.checkconfirmrules( 'ask-x12-997', idroute=self.ta_info['idroute'], idchannel=self.ta_info['tochannel'], topartner=self.ta_info['topartner'], frompartner=self.ta_info['frompartner'], editype=self.ta_info['editype'], messagetype=self.ta_info['messagetype']): self.ta_info['confirmtype'] = u'ask-x12-997' self.ta_info['confirmasked'] = True
def mailbag(ta_from, endstatus, **argv): ''' 2 main functions: - recognizes and distuinguishes several edi types: x12 edifact tradacoms xml ('mailbag' in, correct editype out) - split up interchanges (edifact, x12, tradacoms) details: - edifact, x12 and tradacoms can be can be mixed, - recognizes xml files; but messagetype 'xml' has a special handling when reading xml-files: xpath identifiers are used. this is needed for using xml in mailbag! - when more interchanges in one file: strongly recommended to mailbag/split these. - handle multiple UNA in one file, including different charsets. - handle multiple x12 seperators in one file. ''' edifile = botslib.readdata(filename=ta_from.filename) #read as binary... startpos = 0 while (1): found = HEADER.search(edifile[startpos:]) if found is None: if startpos: #ISA/UNB have been found in file; no new ISA/UNB is found. So all processing is done. break #guess if this is an xml file..... sniffxml = edifile[:25] sniffxml = sniffxml.lstrip( ' \t\n\r\f\v\xFF\xFE\xEF\xBB\xBF\x00' ) #to find first ' real' data; some char are because of BOM, UTF-16 etc if sniffxml and sniffxml[0] == '<': ta_to = ta_from.copyta( status=endstatus, statust=OK, filename=ta_from.filename, editype='xml', messagetype='mailbag' ) #make transaction for translated message; gets ta_info of ta_frommes #~ ta_tomes.update(status=STATUSTMP,statust=OK,filename=ta_set_for_processing.filename,editype='xml') #update outmessage transaction with ta_info; break else: raise botslib.InMessageError( _(u'Found no content in mailbag.')) elif found.group(1): editype = 'x12' headpos = startpos + found.start(2) count = 0 for char in edifile[ headpos:headpos + 120]: #search first 120 characters to find separators if char in '\r\n' and count != 105: continue count += 1 if count == 4: field_sep = char elif count == 106: record_sep = char break #~ foundtrailer = re.search(re.escape(record_sep)+'\s*IEA'+re.escape(field_sep)+'.+?'+re.escape(record_sep),edifile[headpos:],re.DOTALL) foundtrailer = re.search( re.escape(record_sep) + '\s*I\s*E\s*A\s*' + re.escape(field_sep) + '.+?' + re.escape(record_sep), edifile[headpos:], re.DOTALL) elif found.group(3): editype = 'edifact' if found.group(4): field_sep = edifile[startpos + found.start(4) + 4] record_sep = edifile[startpos + found.start(4) + 8] headpos = startpos + found.start(4) else: field_sep = '+' record_sep = "'" headpos = startpos + found.start(5) foundtrailer = re.search( re.escape(record_sep) + '\s*U\s*N\s*Z\s*' + re.escape(field_sep) + '.+?' + re.escape(record_sep), edifile[headpos:], re.DOTALL) elif found.group(8): editype = 'tradacoms' headpos = startpos + found.start(9) field_sep = '=' #the tradacoms 'after-segment-tag-seperator' record_sep = "'" foundtrailer = re.search( re.escape(record_sep) + '\s*E\s*N\s*D\s*' + re.escape(field_sep) + '.+?' + re.escape(record_sep), edifile[headpos:], re.DOTALL) if not foundtrailer: raise botslib.InMessageError( _(u'Found no valid envelope trailer in mailbag.')) endpos = headpos + foundtrailer.end() #so: interchange is from headerpos untill endpos #~ if HEADER.search(edifile[headpos+25:endpos]): #check if there is another header in the interchange #~ raise botslib.InMessageError(u'Error in mailbag format: found no valid envelope trailer.') ta_to = ta_from.copyta( status=endstatus ) #make transaction for translated message; gets ta_info of ta_frommes tofilename = str(ta_to.idta) tofile = botslib.opendata(tofilename, 'wb') tofile.write(edifile[headpos:endpos]) tofile.close() ta_to.update( statust=OK, filename=tofilename, editype=editype, messagetype=editype) #update outmessage transaction with ta_info; startpos = endpos botsglobal.logger.debug(_(u' File written: "%s".'), tofilename)
def _initwrite(self): botsglobal.logger.debug(u'Start writing to file "%s".',self.ta_info['filename']) self._outstream = botslib.opendata(self.ta_info['filename'],"wb")
def extractpdf(ta_from, endstatus, **argv): ''' Try to extract text content of a PDF file to a csv. You know this is not a great idea, right? But we'll do the best we can anyway! Page and line numbers are added to each row. Columns and rows are based on the x and y coordinates of each text element within tolerance allowed. Multiple text elements may combine to make one field, some PDFs have every character separated! You may need to experiment with x_group and y_group values, but defaults seem ok for most files. Output csv is UTF-8 encoded - The csv module doesn't directly support reading and writing Unicode If the PDF is just an image, all bets are off. Maybe try OCR, good luck with that! Mike Griffin 14/12/2011 ''' from pdfminer.pdfinterp import PDFResourceManager, process_pdf from pdfminer.converter import TextConverter from pdfminer.layout import LAParams, LTContainer, LTText, LTTextBox import csv class CsvConverter(TextConverter): def __init__(self, *args, **kwargs): TextConverter.__init__(self, *args, **kwargs) def receive_layout(self, ltpage): # recursively get every text element and it's coordinates def render(item): if isinstance(item, LTContainer): for child in item: render(child) elif isinstance(item, LTText): (_, _, x, y) = item.bbox # group the y values (rows) within group tolerance for v in yv: if y > v - y_group and y < v + y_group: y = v yv.append(y) line = lines[int(-y)] line[x] = item.get_text().encode('utf-8') from collections import defaultdict lines = defaultdict(lambda: {}) yv = [] render(ltpage) lineid = 0 for y in sorted(lines.keys()): line = lines[y] lineid += 1 csvdata = [ltpage.pageid, lineid] # first 2 columns are page and line numbers # group the x values (fields) within group tolerance p = 0 field_txt = '' for x in sorted(line.keys()): gap = x - p if p > 0 and gap > x_group: csvdata.append(field_txt) field_txt = '' field_txt += line[x] p = x csvdata.append(field_txt) csvout.writerow(csvdata) if lineid == 0: raise botslib.InMessageError( _(u'PDF text extraction failed, it may contain just image(s)?' )) #get some optional parameters x_group = argv.get('x_group', 10) # group text closer than this as one field y_group = argv.get('y_group', 5) # group lines closer than this as one line password = argv.get('password', '') quotechar = argv.get('quotechar', '"') field_sep = argv.get('field_sep', ',') escape = argv.get('escape', '\\') charset = argv.get('charset', 'utf-8') if not escape: doublequote = True else: doublequote = False try: pdf_stream = botslib.opendata(ta_from.filename, 'rb') ta_to = ta_from.copyta(status=endstatus) tofilename = str(ta_to.idta) csv_stream = botslib.opendata(tofilename, 'wb') csvout = csv.writer(csv_stream, quotechar=quotechar, delimiter=field_sep, doublequote=doublequote, escapechar=escape) # Process PDF rsrcmgr = PDFResourceManager(caching=True) device = CsvConverter(rsrcmgr, csv_stream, codec=charset) process_pdf(rsrcmgr, device, pdf_stream, pagenos=set(), password=password, caching=True, check_extractable=True) device.close() pdf_stream.close() csv_stream.close() ta_to.update( statust=OK, filename=tofilename) #update outmessage transaction with ta_info; botsglobal.logger.debug(_(u' File written: "%s".'), tofilename) except: txt = botslib.txtexc() botsglobal.logger.error( _(u'PDF extraction failed, may not be a PDF file? Error:\n%s'), txt) raise botslib.InMessageError( _(u'PDF extraction failed, may not be a PDF file? Error:\n$error'), error=txt)
def run(self): self._openoutenvelope(self.ta_info['editype'],self.ta_info['envelope']) self.ta_info.update(self.out.ta_info) botslib.tryrunscript(self.userscript,self.scriptname,'ta_infocontent',ta_info=self.ta_info) #version dependent enveloping writeUNA = False if self.ta_info['version']<'4': date = time.strftime('%y%m%d') reserve = ' ' if self.ta_info['charset'] != 'UNOA': writeUNA = True else: date = time.strftime('%Y%m%d') reserve = self.ta_info['reserve'] if self.ta_info['charset'] not in ['UNOA','UNOB']: writeUNA = True #UNB counter is per sender or receiver if botsglobal.ini.getboolean('settings','interchangecontrolperpartner',False): self.ta_info['reference'] = str(botslib.unique('unbcounter_' + self.ta_info['topartner'])) else: self.ta_info['reference'] = str(botslib.unique('unbcounter_' + self.ta_info['frompartner'])) #testindicator is more complex: if self.ta_info['testindicator'] and self.ta_info['testindicator']!='0': #first check value from ta; do not use default testindicator = '1' elif self.ta_info['UNB.0035'] != '0': #than check values from grammar testindicator = '1' else: testindicator = '' #build the envelope segments (that is, the tree from which the segments will be generated) self.out.put({'BOTSID':'UNB', 'S001.0001':self.ta_info['charset'], 'S001.0002':self.ta_info['version'], 'S002.0004':self.ta_info['frompartner'], 'S003.0010':self.ta_info['topartner'], 'S004.0017':date, 'S004.0019':time.strftime('%H%M'), '0020':self.ta_info['reference']}) #the following fields are conditional; do not write these when empty string (separator compression does take empty strings into account) if self.ta_info['UNB.S002.0007']: self.out.put({'BOTSID':'UNB','S002.0007': self.ta_info['UNB.S002.0007']}) if self.ta_info['UNB.S003.0007']: self.out.put({'BOTSID':'UNB','S003.0007': self.ta_info['UNB.S003.0007']}) if self.ta_info['UNB.0026']: self.out.put({'BOTSID':'UNB','0026': self.ta_info['UNB.0026']}) if testindicator: self.out.put({'BOTSID':'UNB','0035': testindicator}) self.out.put({'BOTSID':'UNB'},{'BOTSID':'UNZ','0036':self.ta_info['nrmessages'],'0020':self.ta_info['reference']}) #dummy segment; is not used #user exit botslib.tryrunscript(self.userscript,self.scriptname,'envelopecontent',ta_info=self.ta_info,out=self.out) #convert the tree into segments; here only the UNB is written (first segment) self.out.normalisetree(self.out.root) self.out.tree2records(self.out.root) #start doing the actual writing: tofile = botslib.opendata(self.ta_info['filename'],'wb',self.ta_info['charset']) if writeUNA or self.ta_info['forceUNA']: tofile.write('UNA'+self.ta_info['sfield_sep']+self.ta_info['field_sep']+self.ta_info['decimaal']+self.ta_info['escape']+ reserve +self.ta_info['record_sep']+self.ta_info['add_crlfafterrecord_sep']) tofile.write(self.out._record2string(self.out.records[0])) self.writefilelist(tofile) tofile.write(self.out._record2string(self.out.records[-1])) tofile.close()
def mailbag(ta_from,endstatus,**argv): ''' 2 main functions: - recognizes and distuinguishes several edi types: x12 edifact tradacoms xml ('mailbag' in, correct editype out) - split up interchanges (edifact, x12, tradacoms) details: - edifact, x12 and tradacoms can be can be mixed, - recognizes xml files; but messagetype 'xml' has a special handling when reading xml-files: xpath identifiers are used. this is needed for using xml in mailbag! - when more interchanges in one file: strongly recommended to mailbag/split these. - handle multiple UNA in one file, including different charsets. - handle multiple x12 seperators in one file. ''' edifile = botslib.readdata(filename=ta_from.filename) #read as binary... startpos = 0 while (1): found = HEADER.search(edifile[startpos:]) if found is None: if startpos: #ISA/UNB have been found in file; no new ISA/UNB is found. So all processing is done. break #guess if this is an xml file..... sniffxml = edifile[:25] sniffxml = sniffxml.lstrip(' \t\n\r\f\v\xFF\xFE\xEF\xBB\xBF\x00') #to find first ' real' data; some char are because of BOM, UTF-16 etc if sniffxml and sniffxml[0] == '<': ta_to = ta_from.copyta(status=endstatus,statust=OK,filename=ta_from.filename,editype='xml',messagetype='mailbag') #make transaction for translated message; gets ta_info of ta_frommes #~ ta_tomes.update(status=STATUSTMP,statust=OK,filename=ta_set_for_processing.filename,editype='xml') #update outmessage transaction with ta_info; break else: raise botslib.InMessageError(_(u'Found no content in mailbag.')) elif found.group(1): editype = 'x12' headpos = startpos + found.start(2) count = 0 for char in edifile[headpos:headpos+120]: #search first 120 characters to find separators if char in '\r\n' and count != 105: continue count += 1 if count == 4: field_sep = char elif count == 106: record_sep = char break #~ foundtrailer = re.search(re.escape(record_sep)+'\s*IEA'+re.escape(field_sep)+'.+?'+re.escape(record_sep),edifile[headpos:],re.DOTALL) foundtrailer = re.search(re.escape(record_sep)+'\s*I\s*E\s*A\s*'+re.escape(field_sep)+'.+?'+re.escape(record_sep),edifile[headpos:],re.DOTALL) elif found.group(3): editype = 'edifact' if found.group(4): field_sep = edifile[startpos + found.start(4) + 4] record_sep = edifile[startpos + found.start(4) + 8] headpos = startpos + found.start(4) else: field_sep = '+' record_sep = "'" headpos = startpos + found.start(5) foundtrailer = re.search(re.escape(record_sep)+'\s*U\s*N\s*Z\s*'+re.escape(field_sep)+'.+?'+re.escape(record_sep),edifile[headpos:],re.DOTALL) elif found.group(8): editype = 'tradacoms' headpos = startpos + found.start(9) field_sep = '=' #the tradacoms 'after-segment-tag-seperator' record_sep = "'" foundtrailer = re.search(re.escape(record_sep)+'\s*E\s*N\s*D\s*'+re.escape(field_sep)+'.+?'+re.escape(record_sep),edifile[headpos:],re.DOTALL) if not foundtrailer: raise botslib.InMessageError(_(u'Found no valid envelope trailer in mailbag.')) endpos = headpos + foundtrailer.end() #so: interchange is from headerpos untill endpos #~ if HEADER.search(edifile[headpos+25:endpos]): #check if there is another header in the interchange #~ raise botslib.InMessageError(u'Error in mailbag format: found no valid envelope trailer.') ta_to = ta_from.copyta(status=endstatus) #make transaction for translated message; gets ta_info of ta_frommes tofilename = str(ta_to.idta) tofile = botslib.opendata(tofilename,'wb') tofile.write(edifile[headpos:endpos]) tofile.close() ta_to.update(statust=OK,filename=tofilename,editype=editype,messagetype=editype) #update outmessage transaction with ta_info; startpos = endpos botsglobal.logger.debug(_(u' File written: "%s".'),tofilename)
def run(self): botslib.tryrunscript(self.userscript, self.scriptname, "ta_infocontent", ta_info=self.ta_info) tofile = botslib.opendata(self.ta_info["filename"], "wb", self.ta_info["charset"]) self.writefilelist(tofile) tofile.close()
def _initwrite(self): botsglobal.logger.debug(u'Start writing to file "%s".',self.ta_info['filename']) self._outstream = botslib.opendata(self.ta_info['filename'],'wb',charset=self.ta_info['charset'],errors=self.ta_info['checkcharsetout'])
def run(self): if not self.ta_info['topartner'] or not self.ta_info['frompartner']: raise botslib.OutMessageError(_(u'In enveloping "frompartner" or "topartner" unknown: "%(ta_info)s".'), {'ta_info':self.ta_info}) self._openoutenvelope() self.ta_info.update(self.out.ta_info) botslib.tryrunscript(self.userscript,self.scriptname,'ta_infocontent',ta_info=self.ta_info) #prepare data for envelope isa09date = botslib.strftime('%y%m%d') #test indicator can either be from configuration (self.ta_info['ISA15']) or by mapping (self.ta_info['testindicator']) #mapping overrules. if self.ta_info['testindicator'] and self.ta_info['testindicator'] != '0': #'0' is default value (in db) testindicator = self.ta_info['testindicator'] else: testindicator = self.ta_info['ISA15'] #~ print self.ta_info['messagetype'], 'grammar:',self.ta_info['ISA15'],'ta:',self.ta_info['testindicator'],'out:',testindicator if botsglobal.ini.getboolean('settings','interchangecontrolperpartner',False): self.ta_info['reference'] = unicode(botslib.unique('isacounter_' + self.ta_info['topartner'])) else: self.ta_info['reference'] = unicode(botslib.unique('isacounter_' + self.ta_info['frompartner'])) #ISA06 and GS02 can be different; eg ISA06 is a service provider. #ISA06 and GS02 can be in the syntax.... isa06sender = self.ta_info.get('ISA06',self.ta_info['frompartner']) isa06sender = isa06sender.ljust(15) #add spaces; is fixed length gs02sender = self.ta_info.get('GS02',self.ta_info['frompartner']) #also for ISA08 and GS03 isa08receiver = self.ta_info.get('ISA08',self.ta_info['topartner']) isa08receiver = isa08receiver.ljust(15) #add spaces; is fixed length gs03receiver = self.ta_info.get('GS03',self.ta_info['topartner']) #build the envelope segments (that is, the tree from which the segments will be generated) self.out.put({'BOTSID':'ISA', 'ISA01':self.ta_info['ISA01'], 'ISA02':self.ta_info['ISA02'], 'ISA03':self.ta_info['ISA03'], 'ISA04':self.ta_info['ISA04'], 'ISA05':self.ta_info['ISA05'], 'ISA06':isa06sender, 'ISA07':self.ta_info['ISA07'], 'ISA08':isa08receiver, 'ISA09':isa09date, 'ISA10':botslib.strftime('%H%M'), 'ISA11':self.ta_info['ISA11'], #if ISA version > 00403, replaced by reprtion separator 'ISA12':self.ta_info['version'], 'ISA13':self.ta_info['reference'], 'ISA14':self.ta_info['ISA14'], 'ISA15':testindicator},strip=False) #MIND: strip=False: ISA fields shoudl not be stripped as it is soemwhat like fixed-length self.out.put({'BOTSID':'ISA'},{'BOTSID':'IEA','IEA01':'1','IEA02':self.ta_info['reference']}) gs08messagetype = self.ta_info['messagetype'][3:] if gs08messagetype[:6] < '004010': gs04date = botslib.strftime('%y%m%d') else: gs04date = botslib.strftime('%Y%m%d') self.out.put({'BOTSID':'ISA'},{'BOTSID':'GS', 'GS01':self.ta_info['functionalgroup'], 'GS02':gs02sender, 'GS03':gs03receiver, 'GS04':gs04date, 'GS05':botslib.strftime('%H%M'), 'GS06':self.ta_info['reference'], 'GS07':self.ta_info['GS07'], 'GS08':gs08messagetype}) self.out.put({'BOTSID':'ISA'},{'BOTSID':'GS'},{'BOTSID':'GE','GE01':self.ta_info['nrmessages'],'GE02':self.ta_info['reference']}) #dummy segment; is not used #user exit botslib.tryrunscript(self.userscript,self.scriptname,'envelopecontent',ta_info=self.ta_info,out=self.out) #convert the tree into segments; here only the UNB is written (first segment) self.out.checkmessage(self.out.root,self.out.defmessage) self.out.checkforerrorlist() self.out.tree2records(self.out.root) #start doing the actual writing: tofile = botslib.opendata(self.ta_info['filename'],'wb',self.ta_info['charset']) isa_string = self.out.record2string(self.out.lex_records[0:1]) #ISA has the used separators at certain positions. Normally bots would give errors for this (can not use sep as data) or compress these aways. So this is hardcoded. if self.ta_info['version'] < '00403': isa_string = isa_string[:103] + self.ta_info['field_sep']+ self.ta_info['sfield_sep'] + isa_string[103:] else: isa_string = isa_string[:82] +self.ta_info['reserve'] + isa_string[83:103] + self.ta_info['field_sep']+ self.ta_info['sfield_sep'] + isa_string[103:] tofile.write(isa_string) #write ISA tofile.write(self.out.record2string(self.out.lex_records[1:2])) #write GS self.writefilelist(tofile) tofile.write(self.out.record2string(self.out.lex_records[2:])) #write GE and IEA tofile.close()
def run(self): self._openoutenvelope(self.ta_info['editype'],self.ta_info['envelope']) self.ta_info.update(self.out.ta_info) #need to know the functionalgroup code: defmessage = grammar.grammarread(self.ta_info['editype'],self.ta_info['messagetype']) self.ta_info['functionalgroup'] = defmessage.syntax['functionalgroup'] botslib.tryrunscript(self.userscript,self.scriptname,'ta_infocontent',ta_info=self.ta_info) #prepare data for envelope ISA09date = time.strftime('%y%m%d') #test indicator can either be from configuration (self.ta_info['ISA15']) or by mapping (self.ta_info['testindicator']) #mapping overrules. if self.ta_info['testindicator'] and self.ta_info['testindicator']!='0': #'0' is default value (in db) testindicator = self.ta_info['testindicator'] else: testindicator = self.ta_info['ISA15'] #~ print self.ta_info['messagetype'], 'grammar:',self.ta_info['ISA15'],'ta:',self.ta_info['testindicator'],'out:',testindicator if botsglobal.ini.getboolean('settings','interchangecontrolperpartner',False): self.ta_info['reference'] = str(botslib.unique('isacounter_' + self.ta_info['topartner'])) else: self.ta_info['reference'] = str(botslib.unique('isacounter_' + self.ta_info['frompartner'])) #ISA06 and GS02 can be different; eg ISA06 is a service provider. #ISA06 and GS02 can be in the syntax.... ISA06 = self.ta_info.get('ISA06',self.ta_info['frompartner']) GS02 = self.ta_info.get('GS02',self.ta_info['frompartner']) #also for ISA08 and GS03 ISA08 = self.ta_info.get('ISA08',self.ta_info['topartner']) GS03 = self.ta_info.get('GS03',self.ta_info['topartner']) #build the envelope segments (that is, the tree from which the segments will be generated) self.out.put({'BOTSID':'ISA', 'ISA01':self.ta_info['ISA01'], 'ISA02':self.ta_info['ISA02'], 'ISA03':self.ta_info['ISA03'], 'ISA04':self.ta_info['ISA04'], 'ISA05':self.ta_info['ISA05'], 'ISA06':ISA06, 'ISA07':self.ta_info['ISA07'], 'ISA08':ISA08, 'ISA09':ISA09date, 'ISA10':time.strftime('%H%M'), 'ISA11':self.ta_info['ISA11'], #if ISA version > 00403, replaced by reprtion separator 'ISA12':self.ta_info['version'], 'ISA13':self.ta_info['reference'], 'ISA14':self.ta_info['ISA14'], 'ISA15':testindicator}) self.out.put({'BOTSID':'ISA'},{'BOTSID':'IEA','IEA01':'1','IEA02':self.ta_info['reference']}) GS08 = self.ta_info['messagetype'][3:] if GS08[:6]<'004010': GS04date = time.strftime('%y%m%d') else: GS04date = time.strftime('%Y%m%d') self.out.put({'BOTSID':'ISA'},{'BOTSID':'GS', 'GS01':self.ta_info['functionalgroup'], 'GS02':GS02, 'GS03':GS03, 'GS04':GS04date, 'GS05':time.strftime('%H%M'), 'GS06':self.ta_info['reference'], 'GS07':self.ta_info['GS07'], 'GS08':GS08}) self.out.put({'BOTSID':'ISA'},{'BOTSID':'GS'},{'BOTSID':'GE','GE01':self.ta_info['nrmessages'],'GE02':self.ta_info['reference']}) #dummy segment; is not used #user exit botslib.tryrunscript(self.userscript,self.scriptname,'envelopecontent',ta_info=self.ta_info,out=self.out) #convert the tree into segments; here only the UNB is written (first segment) self.out.normalisetree(self.out.root) self.out.tree2records(self.out.root) #start doing the actual writing: tofile = botslib.opendata(self.ta_info['filename'],'wb',self.ta_info['charset']) ISAstring = self.out._record2string(self.out.records[0]) if self.ta_info['version']<'00403': ISAstring = ISAstring[:103] + self.ta_info['field_sep']+ self.ta_info['sfield_sep'] + ISAstring[103:] #hack for strange characters at end of ISA; hardcoded else: ISAstring = ISAstring[:82] +self.ta_info['reserve'] + ISAstring[83:103] + self.ta_info['field_sep']+ self.ta_info['sfield_sep'] + ISAstring[103:] #hack for strange characters at end of ISA; hardcoded tofile.write(ISAstring) #write ISA tofile.write(self.out._record2string(self.out.records[1])) #write GS self.writefilelist(tofile) tofile.write(self.out._record2string(self.out.records[-2])) #write GE tofile.write(self.out._record2string(self.out.records[-1])) #write IEA tofile.close() if self.ta_info['functionalgroup']!='FA' and botslib.checkconfirmrules('ask-x12-997',idroute=self.ta_info['idroute'],idchannel=self.ta_info['tochannel'], topartner=self.ta_info['topartner'],frompartner=self.ta_info['frompartner'], editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype']): self.ta_info['confirmtype'] = u'ask-x12-997' self.ta_info['confirmasked'] = True
def writefilelist(self, tofile): for filename in self.ta_list: fromfile = botslib.opendata(filename, 'rb', self.ta_info['charset']) shutil.copyfileobj(fromfile, tofile, 1048576) fromfile.close()
def extractpdf(ta_from,endstatus,**argv): ''' Try to extract text content of a PDF file to a csv. You know this is not a great idea, right? But we'll do the best we can anyway! Page and line numbers are added to each row. Columns and rows are based on the x and y coordinates of each text element within tolerance allowed. Multiple text elements may combine to make one field, some PDFs have every character separated! You may need to experiment with x_group and y_group values, but defaults seem ok for most files. Output csv is UTF-8 encoded - The csv module doesn't directly support reading and writing Unicode If the PDF is just an image, all bets are off. Maybe try OCR, good luck with that! Mike Griffin 14/12/2011 ''' from pdfminer.pdfinterp import PDFResourceManager, process_pdf from pdfminer.converter import TextConverter from pdfminer.layout import LAParams, LTContainer, LTText, LTTextBox import csv class CsvConverter(TextConverter): def __init__(self, *args, **kwargs): TextConverter.__init__(self, *args, **kwargs) def receive_layout(self, ltpage): # recursively get every text element and it's coordinates def render(item): if isinstance(item, LTContainer): for child in item: render(child) elif isinstance(item, LTText): (_,_,x,y) = item.bbox # group the y values (rows) within group tolerance for v in yv: if y > v-y_group and y < v+y_group: y = v yv.append(y) line = lines[int(-y)] line[x] = item.get_text().encode('utf-8') from collections import defaultdict lines = defaultdict(lambda : {}) yv = [] render(ltpage) lineid = 0 for y in sorted(lines.keys()): line = lines[y] lineid += 1 csvdata = [ltpage.pageid,lineid] # first 2 columns are page and line numbers # group the x values (fields) within group tolerance p = 0 field_txt = '' for x in sorted(line.keys()): gap = x - p if p > 0 and gap > x_group: csvdata.append(field_txt) field_txt = '' field_txt += line[x] p = x csvdata.append(field_txt) csvout.writerow(csvdata) if lineid == 0: raise botslib.InMessageError(_(u'PDF text extraction failed, it may contain just image(s)?')) #get some optional parameters x_group = argv.get('x_group',10) # group text closer than this as one field y_group = argv.get('y_group',5) # group lines closer than this as one line password = argv.get('password','') quotechar = argv.get('quotechar','"') field_sep = argv.get('field_sep',',') escape = argv.get('escape','\\') charset = argv.get('charset','utf-8') if not escape: doublequote = True else: doublequote = False try: pdf_stream = botslib.opendata(ta_from.filename, 'rb') ta_to = ta_from.copyta(status=endstatus) tofilename = str(ta_to.idta) csv_stream = botslib.opendata(tofilename,'wb') csvout = csv.writer(csv_stream, quotechar=quotechar, delimiter=field_sep, doublequote=doublequote, escapechar=escape) # Process PDF rsrcmgr = PDFResourceManager(caching=True) device = CsvConverter(rsrcmgr, csv_stream, codec=charset) process_pdf(rsrcmgr, device, pdf_stream, pagenos=set(), password=password, caching=True, check_extractable=True) device.close() pdf_stream.close() csv_stream.close() ta_to.update(statust=OK,filename=tofilename) #update outmessage transaction with ta_info; botsglobal.logger.debug(_(u' File written: "%s".'),tofilename) except: txt = botslib.txtexc() botsglobal.logger.error(_(u'PDF extraction failed, may not be a PDF file? Error:\n%s'),txt) raise botslib.InMessageError(_(u'PDF extraction failed, may not be a PDF file? Error:\n$error'),error=txt)
def dump_csv(xlsdata, tofilename): stream = botslib.opendata(tofilename, 'wb') csvout = csv.writer(stream, quotechar=quotechar, delimiter=field_sep, doublequote=doublequote, escapechar=escape) csvout.writerows( map(utf8ize, xlsdata) ) stream.close()
def run(self): if not self.ta_info["topartner"] or not self.ta_info["frompartner"]: raise botslib.OutMessageError( _(u'In enveloping "frompartner" or "topartner" unknown: "$ta_info".'), ta_info=self.ta_info ) self._openoutenvelope(self.ta_info["editype"], self.ta_info["envelope"]) self.ta_info.update(self.out.ta_info) botslib.tryrunscript(self.userscript, self.scriptname, "ta_infocontent", ta_info=self.ta_info) # version dependent enveloping write_una = False if self.ta_info["version"] < "4": date = time.strftime("%y%m%d") reserve = " " if self.ta_info["charset"] != "UNOA": write_una = True else: date = time.strftime("%Y%m%d") reserve = self.ta_info["reserve"] if self.ta_info["charset"] not in ["UNOA", "UNOB"]: write_una = True # UNB counter is per sender or receiver if botsglobal.ini.getboolean("settings", "interchangecontrolperpartner", False): self.ta_info["reference"] = str(botslib.unique("unbcounter_" + self.ta_info["topartner"])) else: self.ta_info["reference"] = str(botslib.unique("unbcounter_" + self.ta_info["frompartner"])) # testindicator is more complex: if ( self.ta_info["testindicator"] and self.ta_info["testindicator"] != "0" ): # first check value from ta; do not use default testindicator = "1" elif self.ta_info["UNB.0035"] != "0": # than check values from grammar testindicator = "1" else: testindicator = "" # build the envelope segments (that is, the tree from which the segments will be generated) self.out.put( { "BOTSID": "UNB", "S001.0001": self.ta_info["charset"], "S001.0002": self.ta_info["version"], "S002.0004": self.ta_info["frompartner"], "S003.0010": self.ta_info["topartner"], "S004.0017": date, "S004.0019": time.strftime("%H%M"), "0020": self.ta_info["reference"], } ) # the following fields are conditional; do not write these when empty string (separator compression does take empty strings into account) for field in ( "S001.0080", "S001.0133", "S002.0007", "S002.0008", "S002.0042", "S003.0007", "S003.0014", "S003.0046", "S005.0022", "S005.0025", "0026", "0029", "0031", "0032", ): if self.ta_info["UNB." + field]: self.out.put({"BOTSID": "UNB", field: self.ta_info["UNB." + field]}) if testindicator: self.out.put({"BOTSID": "UNB", "0035": testindicator}) self.out.put( {"BOTSID": "UNB"}, {"BOTSID": "UNZ", "0036": self.ta_info["nrmessages"], "0020": self.ta_info["reference"]} ) # dummy segment; is not used # user exit botslib.tryrunscript(self.userscript, self.scriptname, "envelopecontent", ta_info=self.ta_info, out=self.out) # convert the tree into segments; here only the UNB is written (first segment) self.out.checkmessage(self.out.root, self.out.defmessage) self.out.tree2records(self.out.root) # start doing the actual writing: tofile = botslib.opendata(self.ta_info["filename"], "wb", self.ta_info["charset"]) if write_una or self.ta_info["forceUNA"]: tofile.write( "UNA" + self.ta_info["sfield_sep"] + self.ta_info["field_sep"] + self.ta_info["decimaal"] + self.ta_info["escape"] + reserve + self.ta_info["record_sep"] + self.ta_info["add_crlfafterrecord_sep"] ) tofile.write(self.out.record2string(self.out.records[0])) self.writefilelist(tofile) tofile.write(self.out.record2string(self.out.records[-1])) tofile.close() if self.ta_info["messagetype"][:6] != "CONTRL" and botslib.checkconfirmrules( "ask-edifact-CONTRL", idroute=self.ta_info["idroute"], idchannel=self.ta_info["tochannel"], topartner=self.ta_info["topartner"], frompartner=self.ta_info["frompartner"], editype=self.ta_info["editype"], messagetype=self.ta_info["messagetype"], ): self.ta_info["confirmtype"] = u"ask-edifact-CONTRL" self.ta_info["confirmasked"] = True
def writefilelist(self, tofile): for filename in self.ta_list: fromfile = botslib.opendata(filename, "rb", self.ta_info["charset"]) shutil.copyfileobj(fromfile, tofile) fromfile.close()
def run(self): if not self.ta_info['topartner'] or not self.ta_info['frompartner']: raise botslib.OutMessageError( _(u'In enveloping "frompartner" or "topartner" unknown: "%(ta_info)s".' ), {'ta_info': self.ta_info}) self._openoutenvelope() self.ta_info.update(self.out.ta_info) botslib.tryrunscript(self.userscript, self.scriptname, 'ta_infocontent', ta_info=self.ta_info) #prepare data for envelope if botsglobal.ini.getboolean('settings', 'interchangecontrolperpartner', False): self.ta_info['reference'] = unicode( botslib.unique('stxcounter_' + self.ta_info['topartner'])) else: self.ta_info['reference'] = unicode( botslib.unique('stxcounter_' + self.ta_info['frompartner'])) #build the envelope segments (that is, the tree from which the segments will be generated) self.out.put({ 'BOTSID': 'STX', 'STDS1': self.ta_info['STX.STDS1'], 'STDS2': self.ta_info['STX.STDS2'], 'FROM.01': self.ta_info['frompartner'], 'UNTO.01': self.ta_info['topartner'], 'TRDT.01': botslib.strftime('%y%m%d'), 'TRDT.02': botslib.strftime('%H%M%S'), 'SNRF': self.ta_info['reference'] }) if self.ta_info['STX.FROM.02']: self.out.put({ 'BOTSID': 'STX', 'FROM.02': self.ta_info['STX.FROM.02'] }) if self.ta_info['STX.UNTO.02']: self.out.put({ 'BOTSID': 'STX', 'UNTO.02': self.ta_info['STX.UNTO.02'] }) if self.ta_info['STX.APRF']: self.out.put({'BOTSID': 'STX', 'APRF': self.ta_info['STX.APRF']}) if self.ta_info['STX.PRCD']: self.out.put({'BOTSID': 'STX', 'PRCD': self.ta_info['STX.PRCD']}) self.out.put({'BOTSID': 'STX'}, { 'BOTSID': 'END', 'NMST': self.ta_info['nrmessages'] }) #dummy segment; is not used #user exit botslib.tryrunscript(self.userscript, self.scriptname, 'envelopecontent', ta_info=self.ta_info, out=self.out) #convert the tree into segments; here only the STX is written (first segment) self.out.checkmessage(self.out.root, self.out.defmessage) self.out.checkforerrorlist() self.out.tree2records(self.out.root) #start doing the actual writing: tofile = botslib.opendata(self.ta_info['filename'], 'wb', self.ta_info['charset']) tofile.write(self.out.record2string(self.out.lex_records[0:1])) self.writefilelist(tofile) tofile.write(self.out.record2string(self.out.lex_records[1:2])) tofile.close()
def run(self): self._openoutenvelope(self.ta_info['editype'], self.ta_info['envelope']) self.ta_info.update(self.out.ta_info) botslib.tryrunscript(self.userscript, self.scriptname, 'ta_infocontent', ta_info=self.ta_info) #version dependent enveloping writeUNA = False if self.ta_info['version'] < '4': date = time.strftime('%y%m%d') reserve = ' ' if self.ta_info['charset'] != 'UNOA': writeUNA = True else: date = time.strftime('%Y%m%d') reserve = self.ta_info['reserve'] if self.ta_info['charset'] not in ['UNOA', 'UNOB']: writeUNA = True #UNB counter is per sender or receiver if botsglobal.ini.getboolean('settings', 'interchangecontrolperpartner', False): self.ta_info['reference'] = str( botslib.unique('unbcounter_' + self.ta_info['topartner'])) else: self.ta_info['reference'] = str( botslib.unique('unbcounter_' + self.ta_info['frompartner'])) #testindicator is more complex: if self.ta_info['testindicator'] and self.ta_info[ 'testindicator'] != '0': #first check value from ta; do not use default testindicator = '1' elif self.ta_info['UNB.0035'] != '0': #than check values from grammar testindicator = '1' else: testindicator = '' #build the envelope segments (that is, the tree from which the segments will be generated) self.out.put({ 'BOTSID': 'UNB', 'S001.0001': self.ta_info['charset'], 'S001.0002': self.ta_info['version'], 'S002.0004': self.ta_info['frompartner'], 'S003.0010': self.ta_info['topartner'], 'S004.0017': date, 'S004.0019': time.strftime('%H%M'), '0020': self.ta_info['reference'] }) #the following fields are conditional; do not write these when empty string (separator compression does take empty strings into account) if self.ta_info['UNB.S002.0007']: self.out.put({ 'BOTSID': 'UNB', 'S002.0007': self.ta_info['UNB.S002.0007'] }) if self.ta_info['UNB.S003.0007']: self.out.put({ 'BOTSID': 'UNB', 'S003.0007': self.ta_info['UNB.S003.0007'] }) if self.ta_info['UNB.0026']: self.out.put({'BOTSID': 'UNB', '0026': self.ta_info['UNB.0026']}) if testindicator: self.out.put({'BOTSID': 'UNB', '0035': testindicator}) self.out.put({'BOTSID': 'UNB'}, { 'BOTSID': 'UNZ', '0036': self.ta_info['nrmessages'], '0020': self.ta_info['reference'] }) #dummy segment; is not used #user exit botslib.tryrunscript(self.userscript, self.scriptname, 'envelopecontent', ta_info=self.ta_info, out=self.out) #convert the tree into segments; here only the UNB is written (first segment) self.out.normalisetree(self.out.root) self.out.tree2records(self.out.root) #start doing the actual writing: tofile = botslib.opendata(self.ta_info['filename'], 'wb', self.ta_info['charset']) if writeUNA or self.ta_info['forceUNA']: tofile.write('UNA' + self.ta_info['sfield_sep'] + self.ta_info['field_sep'] + self.ta_info['decimaal'] + self.ta_info['escape'] + reserve + self.ta_info['record_sep'] + self.ta_info['add_crlfafterrecord_sep']) tofile.write(self.out._record2string(self.out.records[0])) self.writefilelist(tofile) tofile.write(self.out._record2string(self.out.records[-1])) tofile.close()
def mailbag(ta_from,endstatus,frommessagetype,**argv): ''' 2 main functions: - recognizes and distuinguishes several edi types: x12 edifact tradacoms ('mailbag' in, correct editype out) - split up interchanges (edifact, x12, tradacoms) details: - edifact, x12 and tradacoms can be can be mixed - handle multiple UNA in one file, including different charsets. - handle multiple ISA's with different separators in one file in bots > 3.0.0 all mailbag, edifact, x12 and tradacoms go via mailbag. ''' edifile = botslib.readdata(filename=ta_from.filename) startpos = 0 nr_interchanges = 0 while (1): found = HEADER.match(edifile[startpos:]) if found is None: if edifile[startpos:].strip(string.whitespace+'\x1A\x00'): #there is content...but not valid if nr_interchanges: #found interchanges, but remainder is not valid raise botslib.InMessageError(_(u'[M50]: Found data not in a valid interchange at position %(pos)s.'),{'pos':startpos}) else: #no interchanges found, content is not a valid edifact/x12/tradacoms interchange if frommessagetype == 'mailbag': #if indicated 'mailbag': guess if this is an xml file..... sniffxml = edifile[:25] sniffxml = sniffxml.lstrip(' \t\n\r\f\v\xFF\xFE\xEF\xBB\xBF\x00') #to find first ' real' data; some char are because of BOM, UTF-16 etc if sniffxml and sniffxml[0] == '<': #is a xml file; inmessage.py can determine the right xml messagetype via xpath. filesize = len(edifile) ta_to = ta_from.copyta(status=endstatus,statust=OK,filename=ta_from.filename,editype='xml',messagetype='mailbag',filesize=filesize) return raise botslib.InMessageError(_(u'[M51]: Edi file does not start with a valid interchange.')) else: #no parseble content if nr_interchanges: #OK: there are interchanges, but no new interchange is found. return else: #no edifact/x12/tradacoms envelope at all raise botslib.InMessageError(_(u'[M52]: Edi file contains only whitespace.')) elif found.group('x12'): editype = 'x12' headpos = startpos + found.start('x12') #determine field_sep and record_sep count = 0 for char in edifile[headpos:headpos+120]: #search first 120 characters to determine separators if char in '\r\n' and count != 105: continue count += 1 if count == 4: field_sep = char elif count in [7,18,21,32,35,51,54,70]: #extra checks for fixed ISA. if char != field_sep: raise botslib.InMessageError(_(u'[M53]: Non-valid ISA header at position %(pos)s; position %(pos_element)s of ISA is "%(foundchar)s", expect here element separator "%(field_sep)s".'), {'pos':headpos,'pos_element':unicode(count),'foundchar':char,'field_sep':field_sep}) elif count == 106: record_sep = char break foundtrailer = re.search('''%(record_sep)s \s* I[\n\r]*E[\n\r]*A .+? %(record_sep)s '''%{'record_sep':re.escape(record_sep)}, edifile[headpos:],re.DOTALL|re.VERBOSE) if not foundtrailer: foundtrailer2 = re.search('''%(record_sep)s \s* I[\n\r]*E[\n\r]*A '''%{'record_sep':re.escape(record_sep)}, edifile[headpos:],re.DOTALL|re.VERBOSE) if foundtrailer2: raise botslib.InMessageError(_(u'[M60]: Found no segment terminator for IEA trailer at position %(pos)s.'),{'pos':foundtrailer2.start()}) else: raise botslib.InMessageError(_(u'[M54]: Found no valid IEA trailer for the ISA header at position %(pos)s.'),{'pos':headpos}) elif found.group('edifact'): editype = 'edifact' headpos = startpos + found.start('edifact') #parse UNA. valid UNA: UNA:+.? ' if found.group('UNA'): count = 0 for char in found.group('UNAstring'): if char in '\r\n': continue count += 1 if count == 2: field_sep = char elif count == 4: escape = char elif count == 6: record_sep = char if count != 6 and len(found.group('UNAstring').rstrip()) != 6: raise botslib.InMessageError(_(u'[M55]: Non-valid UNA-segment at position %(pos)s. UNA-segment should be 6 positions.'),{'pos':headpos}) if found.group('field_sep') != field_sep: raise botslib.InMessageError(_(u'[M56]: Data element separator used in edifact file differs from value indicated in UNA-segment.')) else: #no UNA, interpret UNB if found.group('field_sep') == '+': record_sep = "'" escape = '?' elif found.group('field_sep') == '\x1D': #according to std this was preffered way...probably quite theoretic...but does no harm record_sep = '\x1C' escape = '' else: raise botslib.InMessageError(_(u'[M57]: Edifact file with non-standard separators. UNA segment should be used.')) #search trailer foundtrailer = re.search('''[^%(escape)s\n\r] #char that is not escape or cr/lf [\n\r]*? #maybe some cr/lf's %(record_sep)s #segment separator \s* #whitespace between segments U[\n\r]*N[\n\r]*Z #UNZ .+? #any chars [^%(escape)s\n\r] #char that is not escape or cr/lf [\n\r]*? #maybe some cr/lf's %(record_sep)s #segment separator '''%{'escape':escape,'record_sep':re.escape(record_sep)}, edifile[headpos:],re.DOTALL|re.VERBOSE) if not foundtrailer: raise botslib.InMessageError(_(u'[M58]: Found no valid UNZ trailer for the UNB header at position %(pos)s.'),{'pos':headpos}) elif found.group('tradacoms'): editype = 'tradacoms' #~ field_sep = '=' #the tradacoms 'after-segment-tag-separator' record_sep = "'" escape = '?' headpos = startpos + found.start('STX') foundtrailer = re.search('''[^%(escape)s\n\r] #char that is not escape or cr/lf [\n\r]*? #maybe some cr/lf's %(record_sep)s #segment separator \s* #whitespace between segments E[\n\r]*N[\n\r]*D .+? [^%(escape)s\n\r] #char that is not escape or cr/lf [\n\r]*? #maybe some cr/lf's %(record_sep)s #segment separator '''%{'escape':escape,'record_sep':re.escape(record_sep)}, edifile[headpos:],re.DOTALL|re.VERBOSE) if not foundtrailer: raise botslib.InMessageError(_(u'[M59]: Found no valid END trailer for the STX header at position %(pos)s.'),{'pos':headpos}) #so: found an interchange (from headerpos until endpos) endpos = headpos + foundtrailer.end() ta_to = ta_from.copyta(status=endstatus) #make transaction for translated message; gets ta_info of ta_frommes tofilename = unicode(ta_to.idta) filesize = len(edifile[headpos:endpos]) tofile = botslib.opendata(tofilename,'wb') tofile.write(edifile[headpos:endpos]) tofile.close() #editype is now either edifact, x12 or tradacoms #frommessagetype is the original frommessagetype (from route). #frommessagetype would normally be edifact, x12, tradacoms or mailbag, but could also be eg ORDERSD96AUNEAN007. #If so, we want to preserve that. if frommessagetype != 'mailbag' and frommessagetype != editype: messagetype = frommessagetype else: messagetype = editype ta_to.update(statust=OK,filename=tofilename,editype=editype,messagetype=messagetype,filesize=filesize) #update outmessage transaction with ta_info; startpos = endpos nr_interchanges += 1 botsglobal.logger.debug(_(u' File written: "%(tofilename)s".'),{'tofilename':tofilename})
def run(self): if not self.ta_info['topartner'] or not self.ta_info['frompartner']: raise botslib.OutMessageError( _(u'In enveloping "frompartner" or "topartner" unknown: "%(ta_info)s".' ), {'ta_info': self.ta_info}) self._openoutenvelope() self.ta_info.update(self.out.ta_info) botslib.tryrunscript(self.userscript, self.scriptname, 'ta_infocontent', ta_info=self.ta_info) #prepare data for envelope isa09date = botslib.strftime('%y%m%d') #test indicator can either be from configuration (self.ta_info['ISA15']) or by mapping (self.ta_info['testindicator']) #mapping overrules. if self.ta_info['testindicator'] and self.ta_info[ 'testindicator'] != '0': #'0' is default value (in db) testindicator = self.ta_info['testindicator'] else: testindicator = self.ta_info['ISA15'] #~ print self.ta_info['messagetype'], 'grammar:',self.ta_info['ISA15'],'ta:',self.ta_info['testindicator'],'out:',testindicator if botsglobal.ini.getboolean('settings', 'interchangecontrolperpartner', False): self.ta_info['reference'] = unicode( botslib.unique('isacounter_' + self.ta_info['topartner'])) else: self.ta_info['reference'] = unicode( botslib.unique('isacounter_' + self.ta_info['frompartner'])) #ISA06 and GS02 can be different; eg ISA06 is a service provider. #ISA06 and GS02 can be in the syntax.... isa06sender = self.ta_info.get('ISA06', self.ta_info['frompartner']) isa06sender = isa06sender.ljust(15) #add spaces; is fixed length gs02sender = self.ta_info.get('GS02', self.ta_info['frompartner']) #also for ISA08 and GS03 isa08receiver = self.ta_info.get('ISA08', self.ta_info['topartner']) isa08receiver = isa08receiver.ljust(15) #add spaces; is fixed length gs03receiver = self.ta_info.get('GS03', self.ta_info['topartner']) #build the envelope segments (that is, the tree from which the segments will be generated) self.out.put( { 'BOTSID': 'ISA', 'ISA01': self.ta_info['ISA01'], 'ISA02': self.ta_info['ISA02'], 'ISA03': self.ta_info['ISA03'], 'ISA04': self.ta_info['ISA04'], 'ISA05': self.ta_info['ISA05'], 'ISA06': isa06sender, 'ISA07': self.ta_info['ISA07'], 'ISA08': isa08receiver, 'ISA09': isa09date, 'ISA10': botslib.strftime('%H%M'), 'ISA11': self.ta_info[ 'ISA11'], #if ISA version > 00403, replaced by reprtion separator 'ISA12': self.ta_info['version'], 'ISA13': self.ta_info['reference'], 'ISA14': self.ta_info['ISA14'], 'ISA15': testindicator }, strip=False ) #MIND: strip=False: ISA fields shoudl not be stripped as it is soemwhat like fixed-length self.out.put({'BOTSID': 'ISA'}, { 'BOTSID': 'IEA', 'IEA01': '1', 'IEA02': self.ta_info['reference'] }) gs08messagetype = self.ta_info['messagetype'][3:] if gs08messagetype[:6] < '004010': gs04date = botslib.strftime('%y%m%d') else: gs04date = botslib.strftime('%Y%m%d') self.out.put({'BOTSID': 'ISA'}, { 'BOTSID': 'GS', 'GS01': self.ta_info['functionalgroup'], 'GS02': gs02sender, 'GS03': gs03receiver, 'GS04': gs04date, 'GS05': botslib.strftime('%H%M'), 'GS06': self.ta_info['reference'], 'GS07': self.ta_info['GS07'], 'GS08': gs08messagetype }) self.out.put({'BOTSID': 'ISA'}, {'BOTSID': 'GS'}, { 'BOTSID': 'GE', 'GE01': self.ta_info['nrmessages'], 'GE02': self.ta_info['reference'] }) #dummy segment; is not used #user exit botslib.tryrunscript(self.userscript, self.scriptname, 'envelopecontent', ta_info=self.ta_info, out=self.out) #convert the tree into segments; here only the UNB is written (first segment) self.out.checkmessage(self.out.root, self.out.defmessage) self.out.checkforerrorlist() self.out.tree2records(self.out.root) #start doing the actual writing: tofile = botslib.opendata(self.ta_info['filename'], 'wb', self.ta_info['charset']) isa_string = self.out.record2string(self.out.lex_records[0:1]) #ISA has the used separators at certain positions. Normally bots would give errors for this (can not use sep as data) or compress these aways. So this is hardcoded. if self.ta_info['version'] < '00403': isa_string = isa_string[:103] + self.ta_info[ 'field_sep'] + self.ta_info['sfield_sep'] + isa_string[103:] else: isa_string = isa_string[:82] + self.ta_info[ 'reserve'] + isa_string[83:103] + self.ta_info[ 'field_sep'] + self.ta_info['sfield_sep'] + isa_string[103:] tofile.write(isa_string) #write ISA tofile.write(self.out.record2string( self.out.lex_records[1:2])) #write GS self.writefilelist(tofile) tofile.write(self.out.record2string( self.out.lex_records[2:])) #write GE and IEA tofile.close()
def run(self): if not self.ta_info['topartner'] or not self.ta_info['frompartner']: raise botslib.OutMessageError( _(u'In enveloping "frompartner" or "topartner" unknown: "%(ta_info)s".' ), {'ta_info': self.ta_info}) self._openoutenvelope() self.ta_info.update(self.out.ta_info) botslib.tryrunscript(self.userscript, self.scriptname, 'ta_infocontent', ta_info=self.ta_info) #version dependent enveloping if self.ta_info['version'] < '4': date = botslib.strftime('%y%m%d') reserve = ' ' else: date = botslib.strftime('%Y%m%d') reserve = self.ta_info['reserve'] #UNB reference is counter is per sender or receiver if botsglobal.ini.getboolean('settings', 'interchangecontrolperpartner', False): self.ta_info['reference'] = unicode( botslib.unique('unbcounter_' + self.ta_info['topartner'])) else: self.ta_info['reference'] = unicode( botslib.unique('unbcounter_' + self.ta_info['frompartner'])) #testindicator is more complex: if self.ta_info['testindicator'] and self.ta_info[ 'testindicator'] != '0': #first check value from ta; do not use default testindicator = '1' elif self.ta_info['UNB.0035'] != '0': #than check values from grammar testindicator = '1' else: testindicator = '' #build the envelope segments (that is, the tree from which the segments will be generated) self.out.put({ 'BOTSID': 'UNB', 'S001.0001': self.ta_info['charset'], 'S001.0002': self.ta_info['version'], 'S002.0004': self.ta_info['frompartner'], 'S003.0010': self.ta_info['topartner'], 'S004.0017': date, 'S004.0019': botslib.strftime('%H%M'), '0020': self.ta_info['reference'] }) #the following fields are conditional; do not write these when empty string (separator compression does take empty strings into account) for field in ('S001.0080', 'S001.0133', 'S002.0007', 'S002.0008', 'S002.0042', 'S003.0007', 'S003.0014', 'S003.0046', 'S005.0022', 'S005.0025', '0026', '0029', '0031', '0032'): if self.ta_info['UNB.' + field]: self.out.put({ 'BOTSID': 'UNB', field: self.ta_info['UNB.' + field] }) if testindicator: self.out.put({'BOTSID': 'UNB', '0035': testindicator}) self.out.put({'BOTSID': 'UNB'}, { 'BOTSID': 'UNZ', '0036': self.ta_info['nrmessages'], '0020': self.ta_info['reference'] }) #dummy segment; is not used #user exit botslib.tryrunscript(self.userscript, self.scriptname, 'envelopecontent', ta_info=self.ta_info, out=self.out) #convert the tree into segments; here only the UNB is written (first segment) self.out.checkmessage(self.out.root, self.out.defmessage) self.out.checkforerrorlist() self.out.tree2records(self.out.root) #start doing the actual writing: tofile = botslib.opendata(self.ta_info['filename'], 'wb', self.ta_info['charset']) if self.ta_info['forceUNA'] or self.ta_info['charset'] != 'UNOA': tofile.write('UNA' + self.ta_info['sfield_sep'] + self.ta_info['field_sep'] + self.ta_info['decimaal'] + self.ta_info['escape'] + reserve + self.ta_info['record_sep'] + self.ta_info['add_crlfafterrecord_sep']) tofile.write(self.out.record2string(self.out.lex_records[0:1])) self.writefilelist(tofile) tofile.write(self.out.record2string(self.out.lex_records[1:2])) tofile.close()
def run(self): if not self.ta_info["topartner"] or not self.ta_info["frompartner"]: raise botslib.OutMessageError( _(u'In enveloping "frompartner" or "topartner" unknown: "$ta_info".'), ta_info=self.ta_info ) self._openoutenvelope(self.ta_info["editype"], self.ta_info["envelope"]) self.ta_info.update(self.out.ta_info) # need to know the functionalgroup code: defmessage = grammar.grammarread(self.ta_info["editype"], self.ta_info["messagetype"]) self.ta_info["functionalgroup"] = defmessage.syntax["functionalgroup"] botslib.tryrunscript(self.userscript, self.scriptname, "ta_infocontent", ta_info=self.ta_info) # prepare data for envelope isa09date = time.strftime("%y%m%d") # test indicator can either be from configuration (self.ta_info['ISA15']) or by mapping (self.ta_info['testindicator']) # mapping overrules. if self.ta_info["testindicator"] and self.ta_info["testindicator"] != "0": #'0' is default value (in db) testindicator = self.ta_info["testindicator"] else: testindicator = self.ta_info["ISA15"] # ~ print self.ta_info['messagetype'], 'grammar:',self.ta_info['ISA15'],'ta:',self.ta_info['testindicator'],'out:',testindicator if botsglobal.ini.getboolean("settings", "interchangecontrolperpartner", False): self.ta_info["reference"] = str(botslib.unique("isacounter_" + self.ta_info["topartner"])) else: self.ta_info["reference"] = str(botslib.unique("isacounter_" + self.ta_info["frompartner"])) # ISA06 and GS02 can be different; eg ISA06 is a service provider. # ISA06 and GS02 can be in the syntax.... isa06sender = self.ta_info.get("ISA06", self.ta_info["frompartner"]) isa06sender = isa06sender.ljust(15) # add spaces; is fixed length gs02sender = self.ta_info.get("GS02", self.ta_info["frompartner"]) # also for ISA08 and GS03 isa08receiver = self.ta_info.get("ISA08", self.ta_info["topartner"]) isa08receiver = isa08receiver.ljust(15) # add spaces; is fixed length gs03receiver = self.ta_info.get("GS03", self.ta_info["topartner"]) # build the envelope segments (that is, the tree from which the segments will be generated) self.out.put( { "BOTSID": "ISA", "ISA01": self.ta_info["ISA01"], "ISA02": self.ta_info["ISA02"], "ISA03": self.ta_info["ISA03"], "ISA04": self.ta_info["ISA04"], "ISA05": self.ta_info["ISA05"], "ISA06": isa06sender, "ISA07": self.ta_info["ISA07"], "ISA08": isa08receiver, "ISA09": isa09date, "ISA10": time.strftime("%H%M"), "ISA11": self.ta_info["ISA11"], # if ISA version > 00403, replaced by reprtion separator "ISA12": self.ta_info["version"], "ISA13": self.ta_info["reference"], "ISA14": self.ta_info["ISA14"], "ISA15": testindicator, }, strip=False, ) # MIND: strip=False: ISA fields shoudl not be stripped as it is soemwhat like fixed-length self.out.put({"BOTSID": "ISA"}, {"BOTSID": "IEA", "IEA01": "1", "IEA02": self.ta_info["reference"]}) gs08messagetype = self.ta_info["messagetype"][3:] if gs08messagetype[:6] < "004010": gs04date = time.strftime("%y%m%d") else: gs04date = time.strftime("%Y%m%d") self.out.put( {"BOTSID": "ISA"}, { "BOTSID": "GS", "GS01": self.ta_info["functionalgroup"], "GS02": gs02sender, "GS03": gs03receiver, "GS04": gs04date, "GS05": time.strftime("%H%M"), "GS06": self.ta_info["reference"], "GS07": self.ta_info["GS07"], "GS08": gs08messagetype, }, ) self.out.put( {"BOTSID": "ISA"}, {"BOTSID": "GS"}, {"BOTSID": "GE", "GE01": self.ta_info["nrmessages"], "GE02": self.ta_info["reference"]}, ) # dummy segment; is not used # user exit botslib.tryrunscript(self.userscript, self.scriptname, "envelopecontent", ta_info=self.ta_info, out=self.out) # convert the tree into segments; here only the UNB is written (first segment) self.out.checkmessage(self.out.root, self.out.defmessage) self.out.tree2records(self.out.root) # start doing the actual writing: tofile = botslib.opendata(self.ta_info["filename"], "wb", self.ta_info["charset"]) isa_string = self.out.record2string(self.out.records[0]) if self.ta_info["version"] < "00403": isa_string = ( isa_string[:103] + self.ta_info["field_sep"] + self.ta_info["sfield_sep"] + isa_string[103:] ) # hack for strange characters at end of ISA; hardcoded else: isa_string = ( isa_string[:82] + self.ta_info["reserve"] + isa_string[83:103] + self.ta_info["field_sep"] + self.ta_info["sfield_sep"] + isa_string[103:] ) # hack for strange characters at end of ISA; hardcoded tofile.write(isa_string) # write ISA tofile.write(self.out.record2string(self.out.records[1])) # write GS self.writefilelist(tofile) tofile.write(self.out.record2string(self.out.records[-2])) # write GE tofile.write(self.out.record2string(self.out.records[-1])) # write IEA tofile.close() if self.ta_info["functionalgroup"] != "FA" and botslib.checkconfirmrules( "ask-x12-997", idroute=self.ta_info["idroute"], idchannel=self.ta_info["tochannel"], topartner=self.ta_info["topartner"], frompartner=self.ta_info["frompartner"], editype=self.ta_info["editype"], messagetype=self.ta_info["messagetype"], ): self.ta_info["confirmtype"] = u"ask-x12-997" self.ta_info["confirmasked"] = True
def splitmailbag(startstatus=MAILBAG,endstatus=TRANSLATE,idroute=''): ''' splits 'mailbag'files to seperate files each containging one interchange (ISA-IEA or UNA/UNB-UNZ). handles x12 and edifact; these can be mixed. ''' header = re.compile('(\s*(ISA))|(\s*(UNA.{6})?\s*(U\s*N\s*B)s*.{1}(.{4}).{1}(.{1}))',re.DOTALL) # group: 1 2 3 4 5 6 7 for row in botslib.query(u'''SELECT idta,filename,charset FROM ta WHERE idta>%(rootidta)s AND status=%(status)s AND statust=%(statust)s AND idroute=%(idroute)s ''', {'status':startstatus,'statust':OK,'idroute':idroute,'rootidta':botslib.get_minta4query()}): try: ta_org=botslib.OldTransaction(row['idta']) ta_intermediate = ta_org.copyta(status=MAILBAGPARSED) edifile = botslib.readdata(filename=row['filename']) #read as binary... botsglobal.logmap.debug(u'Start parsing mailbag file "%s".',row['filename']) startpos=0 while (1): found = header.search(edifile[startpos:]) if found is None: if startpos: #ISA/UNB have been found in file; no new ISA/UNB is found. So all processing is done. break #guess if this is an xml file..... sniffxml = edifile[:25] sniffxml = sniffxml.lstrip(' \t\n\r\f\v\xFF\xFE\xEF\xBB\xBF\x00') #to find first ' real' data; some char are because of BOM, UTF-16 etc if sniffxml and sniffxml[0]=='<': ta_tomes=ta_intermediate.copyta(status=endstatus) #make transaction for translated message; gets ta_info of ta_frommes ta_tomes.update(status=STATUSTMP,statust=OK,filename=row['filename'],editype='xml') #update outmessage transaction with ta_info; break; else: raise botslib.InMessageError(_(u'Found no content in mailbag.')) elif found.group(1): editype='x12' headpos=startpos+ found.start(2) count=0 for c in edifile[headpos:headpos+120]: #search first 120 characters to find seperators if c in '\r\n' and count!=105: continue count +=1 if count==4: field_sep = c elif count==106: record_sep = c break foundtrailer = re.search(re.escape(record_sep)+'\s*IEA'+re.escape(field_sep)+'.+?'+re.escape(record_sep),edifile[headpos:],re.DOTALL) elif found.group(3): editype='edifact' if found.group(4): field_sep = edifile[startpos + found.start(4) + 4] record_sep = edifile[startpos + found.start(4) + 8] headpos=startpos+ found.start(4) else: field_sep = '+' record_sep = "'" headpos=startpos+ found.start(5) foundtrailer = re.search(re.escape(record_sep)+'\s*U\s*N\s*Z\s*'+re.escape(field_sep)+'.+?'+re.escape(record_sep),edifile[headpos:],re.DOTALL) if not foundtrailer: raise botslib.InMessageError(_(u'Found no valid envelope trailer in mailbag.')) endpos = headpos+foundtrailer.end() #so: interchange is from headerpos untill endpos #~ if header.search(edifile[headpos+25:endpos]): #check if there is another header in the interchange #~ raise botslib.InMessageError(u'Error in mailbag format: found no valid envelope trailer.') ta_tomes=ta_intermediate.copyta(status=endstatus) #make transaction for translated message; gets ta_info of ta_frommes tofilename = str(ta_tomes.idta) tofile = botslib.opendata(tofilename,'wb') tofile.write(edifile[headpos:endpos]) tofile.close() ta_tomes.update(status=STATUSTMP,statust=OK,filename=tofilename,editype=editype,messagetype=editype) #update outmessage transaction with ta_info; startpos=endpos except: txt=botslib.txtexc() ta_intermediate.failure() ta_intermediate.update(statust=ERROR,errortext=txt) else: botsglobal.logmap.debug(u'OK Parsing mailbag file "%s".',row['filename']) ta_org.update(statust=DONE) ta_intermediate.succes(endstatus) ta_intermediate.update(statust=DONE)