def writeall(self):
''' Very different writeall:
there is no tree of nodes; there is no grammar.structure/recorddefs; kid opens file by itself.
'''
try:
import kid
except:
txt = botslib.txtexc()
raise ImportError(_(u'Dependency failure: editype "template" requires python library "kid". Error:\n%s'%txt))
#for template-grammar: only syntax is used. Section 'syntax' has to have 'template'
self.outmessagegrammarread(self.ta_info['editype'],self.ta_info['messagetype'])
templatefile = botslib.abspath(u'templates',self.ta_info['template'])
try:
botsglobal.logger.debug(u'Start writing to file "%s".',self.ta_info['filename'])
ediprint = kid.Template(file=templatefile, data=self.data)
except:
txt = botslib.txtexc()
raise botslib.OutMessageError(_(u'While templating "$editype.$messagetype", error:\n$txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt)
try:
filehandler = botslib.opendata(self.ta_info['filename'],'wb')
ediprint.write(filehandler,
encoding=self.ta_info['charset'],
output=self.ta_info['output'], #output is specific parameter for class; init from grammar.syntax
fragment=self.ta_info['merge'])
except:
txt = botslib.txtexc()
raise botslib.OutMessageError(_(u'While templating "$editype.$messagetype", error:\n$txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt)
botsglobal.logger.debug(_(u'End writing to file "%s".'),self.ta_info['filename'])
def writeall(self):
''' Very different writeall:
there is no tree of nodes; there is no grammar.structure/recorddefs; kid opens file by itself.
'''
try:
from genshi.template import TemplateLoader
except:
txt = botslib.txtexc()
raise ImportError(_(u'Dependency failure: editype "template" requires python library "genshi". Error:\n%s'%txt))
#for template-grammar: only syntax is used. Section 'syntax' has to have 'template'
self.outmessagegrammarread(self.ta_info['editype'],self.ta_info['messagetype'])
templatefile = botslib.abspath(u'templateshtml',self.ta_info['template'])
try:
botsglobal.logger.debug(u'Start writing to file "%s".',self.ta_info['filename'])
loader = TemplateLoader(auto_reload=False)
tmpl = loader.load(templatefile)
except:
txt = botslib.txtexc()
raise botslib.OutMessageError(_(u'While templating "$editype.$messagetype", error:\n$txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt)
try:
filehandler = botslib.opendata(self.ta_info['filename'],'wb')
stream = tmpl.generate(data=self.data)
stream.render(method='xhtml',encoding=self.ta_info['charset'],out=filehandler)
except:
txt = botslib.txtexc()
raise botslib.OutMessageError(_(u'While templating "$editype.$messagetype", error:\n$txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt)
botsglobal.logger.debug(_(u'End writing to file "%s".'),self.ta_info['filename'])
def run(self):
''' class for template enveloping; delevers a valid html-file.
Uses a kid-template for the enveloping/merging.
use kid to write; no envelope grammar is used
#20120101 depreciated. use class templatehtml
'''
try:
import kid
except:
raise ImportError(_(u'Dependency failure: editype "template" requires python library "kid".'))
self._openoutenvelope()
self.ta_info.update(self.out.ta_info)
botslib.tryrunscript(self.userscript,self.scriptname,'ta_infocontent',ta_info=self.ta_info)
if not self.ta_info['envelope-template']:
raise botslib.OutMessageError(_(u'While enveloping in "%(editype)s.%(messagetype)s": syntax option "envelope-template" not filled; is required.'),
self.ta_info)
templatefile = botslib.abspath('templates',self.ta_info['envelope-template'])
ta_list = self.filelist2absolutepaths()
try:
botsglobal.logger.debug(u'Start writing envelope to file "%(filename)s".',self.ta_info)
ediprint = kid.Template(file=templatefile, data=ta_list) #init template; pass list with filenames
except:
txt = botslib.txtexc()
raise botslib.OutMessageError(_(u'While enveloping in "%(editype)s.%(messagetype)s", error:\n%(txt)s'),
{'editype':self.ta_info['editype'],'messagetype':self.ta_info['messagetype'],'txt':txt})
try:
filehandler = botslib.opendata(self.ta_info['filename'],'wb')
ediprint.write(filehandler,
encoding=self.ta_info['charset'],
output=self.ta_info['output'])
except:
txt = botslib.txtexc()
raise botslib.OutMessageError(_(u'While enveloping in "%(editype)s.%(messagetype)s", error:\n%(txt)s'),
{'editype':self.ta_info['editype'],'messagetype':self.ta_info['messagetype'],'txt':txt})
def botsunzip(ta_from,endstatus,password=None,pass_non_zip=False,**argv):
''' unzip file;
editype & messagetype are unchanged.
'''
try:
z = zipfile.ZipFile(botslib.abspathdata(filename=ta_from.filename),mode='r')
except zipfile.BadZipfile:
botsglobal.logger.debug(_(u'File is not a zip-file.'))
if pass_non_zip: #just pass the file
botsglobal.logger.debug(_(u'"pass_non_zip" is True, just pass the file.'))
ta_to = ta_from.copyta(status=endstatus,statust=OK)
return
raise botslib.InMessageError(_(u'File is not a zip-file.'))
if password:
z.setpassword(password)
for f in z.infolist():
if f.filename[-1] == '/': #check if this is a dir; if so continue
continue
ta_to = ta_from.copyta(status=endstatus)
tofilename = str(ta_to.idta)
tofile = botslib.opendata(tofilename,'wb')
tofile.write(z.read(f.filename))
tofile.close()
ta_to.update(statust=OK,filename=tofilename) #update outmessage transaction with ta_info;
botsglobal.logger.debug(_(u' File written: "%s".'),tofilename)
def run(self):
''' class for (test) orderprint; delevers a valid html-file.
Uses a kid-template for the enveloping/merging.
use kid to write; no envelope grammar is used
'''
try:
from genshi.template import TemplateLoader
except:
txt=botslib.txtexc()
raise ImportError(_(u'Dependency failure: editype "template" requires python library "genshi". Error:\n%s'%txt))
defmessage = grammar.grammarread(self.ta_info['editype'],self.ta_info['messagetype']) #needed because we do not know envelope; read syntax for editype/messagetype
self.ta_info.update(defmessage.syntax)
botslib.tryrunscript(self.userscript,self.scriptname,'ta_infocontent',ta_info=self.ta_info)
if not self.ta_info['envelope-template']:
raise botslib.OutMessageError(_(u'While enveloping in "$editype.$messagetype": syntax option "envelope-template" not filled; is required.'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'])
templatefile = botslib.abspath('templateshtml',self.ta_info['envelope-template'])
ta_list = self.filelist2absolutepaths()
try:
botsglobal.logger.debug(u'Start writing envelope to file "%s".',self.ta_info['filename'])
loader = TemplateLoader(auto_reload=False)
tmpl = loader.load(templatefile)
except:
txt=botslib.txtexc()
raise botslib.OutMessageError(_(u'While enveloping in "$editype.$messagetype", error:\n$txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt)
try:
f = botslib.opendata(self.ta_info['filename'],'wb')
stream = tmpl.generate(data=ta_list)
stream.render(method='xhtml',encoding=self.ta_info['charset'],out=f)
except:
txt=botslib.txtexc()
raise botslib.OutMessageError(_(u'While enveloping in "$editype.$messagetype", error:\n$txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt)
def run(self):
try:
from genshi.template import TemplateLoader
except:
raise ImportError(u'Dependency failure: editype "templatehtml" requires python library "genshi".')
self._openoutenvelope()
self.ta_info.update(self.out.ta_info)
botslib.tryrunscript(self.userscript,self.scriptname,'ta_infocontent',ta_info=self.ta_info)
if not self.ta_info['envelope-template']:
raise botslib.OutMessageError(_(u'While enveloping in "%(editype)s.%(messagetype)s": syntax option "envelope-template" not filled; is required.'),
self.ta_info)
templatefile = botslib.abspath(self.__class__.__name__,self.ta_info['envelope-template'])
ta_list = self.filelist2absolutepaths()
try:
botsglobal.logger.debug(u'Start writing envelope to file "%(filename)s".',self.ta_info)
loader = TemplateLoader(auto_reload=False)
tmpl = loader.load(templatefile)
except:
txt = botslib.txtexc()
raise botslib.OutMessageError(_(u'While enveloping in "%(editype)s.%(messagetype)s", error:\n%(txt)s'),
{'editype':self.ta_info['editype'],'messagetype':self.ta_info['messagetype'],'txt':txt})
try:
filehandler = botslib.opendata(self.ta_info['filename'],'wb')
stream = tmpl.generate(data=ta_list)
stream.render(method='xhtml',encoding=self.ta_info['charset'],out=filehandler)
except:
txt = botslib.txtexc()
raise botslib.OutMessageError(_(u'While enveloping in "%(editype)s.%(messagetype)s", error:\n%(txt)s'),
{'editype':self.ta_info['editype'],'messagetype':self.ta_info['messagetype'],'txt':txt})
def writeall(self):
''' Very different writeall:
there is no tree of nodes; there is no grammar.structure/recorddefs; kid opens file by itself.
'''
try:
from genshi.template import TemplateLoader
except:
txt=botslib.txtexc()
raise ImportError(_(u'Dependency failure: editype "template" requires python library "genshi". Error:\n%s'%txt))
#for template-grammar: only syntax is used. Section 'syntax' has to have 'template'
self.outmessagegrammarread(self.ta_info['editype'],self.ta_info['messagetype'])
templatefile = botslib.abspath(u'templateshtml',self.ta_info['template'])
try:
botsglobal.logger.debug(u'Start writing to file "%s".',self.ta_info['filename'])
loader = TemplateLoader(auto_reload=False)
tmpl = loader.load(templatefile)
except:
txt=botslib.txtexc()
raise botslib.OutMessageError(_(u'While templating "$editype.$messagetype", error:\n$txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt)
try:
f = botslib.opendata(self.ta_info['filename'],'wb')
stream = tmpl.generate(data=self.data)
stream.render(method='xhtml',encoding=self.ta_info['charset'],out=f)
except:
txt=botslib.txtexc()
raise botslib.OutMessageError(_(u'While templating "$editype.$messagetype", error:\n$txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt)
botsglobal.logger.debug(_(u'End writing to file "%s".'),self.ta_info['filename'])
def writeall(self):
''' Very different writeall:
there is no tree of nodes; there is no grammar.structure/recorddefs; kid opens file by itself.
'''
try:
import kid
except:
txt=botslib.txtexc()
raise ImportError(_(u'Dependency failure: editype "template" requires python library "kid". Error:\n%s'%txt))
#for template-grammar: only syntax is used. Section 'syntax' has to have 'template'
self.outmessagegrammarread(self.ta_info['editype'],self.ta_info['messagetype'])
templatefile = botslib.abspath(u'templates',self.ta_info['template'])
try:
botsglobal.logger.debug(u'Start writing to file "%s".',self.ta_info['filename'])
ediprint = kid.Template(file=templatefile, data=self.data)
except:
txt=botslib.txtexc()
raise botslib.OutMessageError(_(u'While templating "$editype.$messagetype", error:\n$txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt)
try:
f = botslib.opendata(self.ta_info['filename'],'wb')
ediprint.write(f,
#~ ediprint.write(botslib.abspathdata(self.ta_info['filename']),
encoding=self.ta_info['charset'],
output=self.ta_info['output'], #output is specific parameter for class; init from grammar.syntax
fragment=self.ta_info['merge'])
except:
txt=botslib.txtexc()
raise botslib.OutMessageError(_(u'While templating "$editype.$messagetype", error:\n$txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt)
botsglobal.logger.debug(_(u'End writing to file "%s".'),self.ta_info['filename'])
def botsunzip(ta_from, endstatus, password=None, pass_non_zip=False, **argv):
''' unzip file;
editype & messagetype are unchanged.
'''
try:
z = zipfile.ZipFile(botslib.abspathdata(filename=ta_from.filename),
mode='r')
except zipfile.BadZipfile:
botsglobal.logger.debug(_(u'File is not a zip-file.'))
if pass_non_zip: #just pass the file
botsglobal.logger.debug(
_(u'"pass_non_zip" is True, just pass the file.'))
ta_to = ta_from.copyta(status=endstatus, statust=OK)
return
raise botslib.InMessageError(_(u'File is not a zip-file.'))
if password:
z.setpassword(password)
for f in z.infolist():
if f.filename[-1] == '/': #check if this is a dir; if so continue
continue
ta_to = ta_from.copyta(status=endstatus)
tofilename = str(ta_to.idta)
tofile = botslib.opendata(tofilename, 'wb')
tofile.write(z.read(f.filename))
tofile.close()
ta_to.update(
statust=OK,
filename=tofilename) #update outmessage transaction with ta_info;
botsglobal.logger.debug(_(u' File written: "%s".'), tofilename)
def run(self):
''' class for (test) orderprint; delevers a valid html-file.
Uses a kid-template for the enveloping/merging.
use kid to write; no envelope grammar is used
'''
import kid
defmessage = grammar.grammarread(self.ta_info['editype'],self.ta_info['messagetype']) #needed beause we do not know envelope; read syntax for editype/messagetype
self.ta_info.update(defmessage.syntax)
botslib.tryrunscript(self.userscript,self.scriptname,'ta_infocontent',ta_info=self.ta_info)
if not self.ta_info['envelope-template']:
raise botslib.OutMessageError(_(u'While enveloping in "$editype.$messagetype": syntax option "envelope-template" not filled; is required.'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'])
templatefile = botslib.abspath('templates',self.ta_info['envelope-template'])
ta_list = self.filelist2absolutepaths()
try:
botsglobal.logger.debug(u'Start writing envelope to file "%s".',self.ta_info['filename'])
ediprint = kid.Template(file=templatefile, data=ta_list) #init template; pass list with filenames
except:
txt=botslib.txtexc()
raise botslib.OutMessageError(_(u'While enveloping in "$editype.$messagetype": $txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt)
try:
f = botslib.opendata(self.ta_info['filename'],'wb')
ediprint.write(f,
encoding=self.ta_info['charset'],
output=self.ta_info['output'])
except:
txt=botslib.txtexc()
raise botslib.OutMessageError(_(u'While enveloping error in "$editype.$messagetype"; probably in html file(s) to be merged: $txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt)
def _initwrite(self):
botsglobal.logger.debug(u'Start writing to file "%s".',
self.ta_info['filename'])
self._outstream = botslib.opendata(
self.ta_info['filename'],
'wb',
charset=self.ta_info['charset'],
errors=self.ta_info['checkcharsetout'])
def run(self):
botslib.tryrunscript(self.userscript,self.scriptname,'ta_infocontent',ta_info=self.ta_info)
if len(self.ta_list) > 1:
tofile = botslib.opendata(self.ta_info['filename'],'wb',self.ta_info['charset'])
self.writefilelist(tofile)
tofile.close()
else:
self.ta_info['filename'] = self.ta_list[0]
def run(self):
''' class for (test) orderprint; delevers a valid html-file.
Uses a kid-template for the enveloping/merging.
use kid to write; no envelope grammar is used
'''
try:
import kid
except:
txt = botslib.txtexc()
raise ImportError(
_(u'Dependency failure: editype "template" requires python library "kid". Error:\n%s'
% txt))
defmessage = grammar.grammarread(
self.ta_info['editype'], self.ta_info['messagetype']
) #needed because we do not know envelope; read syntax for editype/messagetype
self.ta_info.update(defmessage.syntax)
botslib.tryrunscript(self.userscript,
self.scriptname,
'ta_infocontent',
ta_info=self.ta_info)
if not self.ta_info['envelope-template']:
raise botslib.OutMessageError(_(
u'While enveloping in "$editype.$messagetype": syntax option "envelope-template" not filled; is required.'
),
editype=self.ta_info['editype'],
messagetype=self.
ta_info['messagetype'])
templatefile = botslib.abspath('templates',
self.ta_info['envelope-template'])
ta_list = self.filelist2absolutepaths()
try:
botsglobal.logger.debug(u'Start writing envelope to file "%s".',
self.ta_info['filename'])
ediprint = kid.Template(
file=templatefile,
data=ta_list) #init template; pass list with filenames
except:
txt = botslib.txtexc()
raise botslib.OutMessageError(
_(u'While enveloping in "$editype.$messagetype", error:\n$txt'
),
editype=self.ta_info['editype'],
messagetype=self.ta_info['messagetype'],
txt=txt)
try:
filehandler = botslib.opendata(self.ta_info['filename'], 'wb')
ediprint.write(filehandler,
encoding=self.ta_info['charset'],
output=self.ta_info['output'])
except:
txt = botslib.txtexc()
raise botslib.OutMessageError(
_(u'While enveloping in "$editype.$messagetype", error:\n$txt'
),
editype=self.ta_info['editype'],
messagetype=self.ta_info['messagetype'],
txt=txt)
def dump_csv(xlsdata, tofilename):
stream = botslib.opendata(tofilename, 'wb')
csvout = csv.writer(stream,
quotechar=quotechar,
delimiter=field_sep,
doublequote=doublequote,
escapechar=escape)
csvout.writerows(map(utf8ize, xlsdata))
stream.close()
def writeall(self):
if self.root is None:
raise botslib.OutMessageError(_(u'No outgoing message')) #then there is nothing to write...
botsglobal.logger.debug(u'Start writing to file "%s".',self.ta_info['filename'])
self._outstream = botslib.opendata(self.ta_info['filename'],'wb')
self._outstream.write(self.root)
self._outstream.close()
botsglobal.logger.debug(u'End writing to file "%s".',self.ta_info['filename'])
self.ta_info['envelope'] = 'raw' #use right enveloping for raw: no coping etc, use same file.
def writeall(self):
if self.root is None:
raise botslib.OutMessageError(_(u'No outgoing message')) #then there is nothing to write...
botsglobal.logger.debug(u'Start writing to file "%s".',self.ta_info['filename'])
self._outstream = botslib.opendata(self.ta_info['filename'],'wb')
self._outstream.write(self.root)
self._outstream.close()
botsglobal.logger.debug(u'End writing to file "%s".',self.ta_info['filename'])
self.ta_info['envelope'] = 'raw' #use right enveloping for raw: no coping etc, use same file.
def run(self):
botslib.tryrunscript(self.userscript,
self.scriptname,
'ta_infocontent',
ta_info=self.ta_info)
tofile = botslib.opendata(self.ta_info['filename'], 'wb',
self.ta_info['charset'])
self.writefilelist(tofile)
tofile.close()
def run(self):
''' class for (test) orderprint; delevers a valid html-file.
Uses a kid-template for the enveloping/merging.
use kid to write; no envelope grammar is used
'''
try:
from genshi.template import TemplateLoader
except:
txt = botslib.txtexc()
raise ImportError(
_(u'Dependency failure: editype "template" requires python library "genshi". Error:\n%s'
% txt))
defmessage = grammar.grammarread(
self.ta_info['editype'], self.ta_info['messagetype']
) #needed because we do not know envelope; read syntax for editype/messagetype
self.ta_info.update(defmessage.syntax)
botslib.tryrunscript(self.userscript,
self.scriptname,
'ta_infocontent',
ta_info=self.ta_info)
if not self.ta_info['envelope-template']:
raise botslib.OutMessageError(_(
u'While enveloping in "$editype.$messagetype": syntax option "envelope-template" not filled; is required.'
),
editype=self.ta_info['editype'],
messagetype=self.
ta_info['messagetype'])
templatefile = botslib.abspath('templateshtml',
self.ta_info['envelope-template'])
ta_list = self.filelist2absolutepaths()
try:
botsglobal.logger.debug(u'Start writing envelope to file "%s".',
self.ta_info['filename'])
loader = TemplateLoader(auto_reload=False)
tmpl = loader.load(templatefile)
except:
txt = botslib.txtexc()
raise botslib.OutMessageError(
_(u'While enveloping in "$editype.$messagetype", error:\n$txt'
),
editype=self.ta_info['editype'],
messagetype=self.ta_info['messagetype'],
txt=txt)
try:
filehandler = botslib.opendata(self.ta_info['filename'], 'wb')
stream = tmpl.generate(data=ta_list)
stream.render(method='xhtml',
encoding=self.ta_info['charset'],
out=filehandler)
except:
txt = botslib.txtexc()
raise botslib.OutMessageError(
_(u'While enveloping in "$editype.$messagetype", error:\n$txt'
),
editype=self.ta_info['editype'],
messagetype=self.ta_info['messagetype'],
txt=txt)
def writeall(self):
if self.root is None:
raise botslib.OutMessageError(_(u'No outgoing message')) #then there is nothing to write...
botsglobal.logger.debug(u'Start writing to file "%(filename)s".',self.ta_info)
self._outstream = botslib.opendata(self.ta_info['filename'],'wb')
self._outstream.write(self.root)
self._outstream.close()
botsglobal.logger.debug(u'End writing to file "%(filename)s".',self.ta_info)
self.ta_info['envelope'] = 'raw'
self.ta_info['merge'] = False
def run(self):
self._openoutenvelope(self.ta_info["editype"], self.ta_info["messagetype"])
botslib.tryrunscript(self.userscript, self.scriptname, "ta_infocontent", ta_info=self.ta_info)
# self.ta_info is not overwritten
tofile = botslib.opendata(self.ta_info["filename"], "wb", self.ta_info["charset"])
headers = dict([(field[ID], field[ID]) for field in self.out.defmessage.structure[0][FIELDS]])
self.out.put(headers)
self.out.tree2records(self.out.root)
tofile.write(self.out.record2string(self.out.records[0]))
self.writefilelist(tofile)
tofile.close()
def run(self):
self._openoutenvelope(self.ta_info['editype'],self.ta_info['messagetype'])
botslib.tryrunscript(self.userscript,self.scriptname,'ta_infocontent',ta_info=self.ta_info)
#self.ta_info is not overwritten
tofile = botslib.opendata(self.ta_info['filename'],'wb',self.ta_info['charset'])
if len(self.out.defmessage.recorddefs) == 1 : #if only only type of record
headers = dict([(field[ID],field[ID]) for field in self.out.defmessage.structure[0][FIELDS]])
self.out.put(headers)
self.out.tree2records(self.out.root)
tofile.write(self.out._record2string(self.out.records[0]))
self.writefilelist(tofile)
tofile.close()
def _write(self, node):
''' write normal XML messages (no envelope)'''
xmltree = ET.ElementTree(self._node2xml(node))
f = botslib.opendata(self.ta_info['filename'], "wb")
if self.ta_info['indented']:
indentstring = '\n'
else:
indentstring = ''
#xml prolog: always use.*********************************
#syntax parameter controls if stand-alone is used within prolog.
#in ET 1.3.0: if standalone is to be used: should surpress ET-generated prolog - explicit parameter
#in ET 1.2.6: always generates prolog if encoding != utf-8/ascii. SO: can not use stadnalone for encoding !=utf-8,ascii
if ET.VERSION not in ['1.2.6', '1.0.6'] or self.ta_info['charset'] in [
'us-ascii', 'utf-8'
]:
if self.ta_info['standalone']:
standalonestring = 'standalone="%s" ' % (
self.ta_info['standalone'])
else:
standalonestring = ''
PI = ET.ProcessingInstruction(
'xml', 'version="%s" encoding="%s" %s' %
(self.ta_info['version'], self.ta_info['charset'],
standalonestring))
f.write(
ET.tostring(PI) + indentstring
) #do not use encoding here. gives double xml prolog; possibly because ET.ElementTree.write i used again by write()
#doctype /DTD **************************************
if self.ta_info['DOCTYPE']:
f.write('<!DOCTYPE %s>' % (self.ta_info['DOCTYPE']) + indentstring)
#processing instructions (other than prolog) ************
if self.ta_info['processing_instructions']:
for pi in self.ta_info['processing_instructions']:
PI = ET.ProcessingInstruction(pi[0], pi[1])
f.write(
ET.tostring(PI) + indentstring
) #do not use encoding here. gives double xml prolog; possibly because ET.ElementTree.write i used again by write()
#indent the xml elements
if self.ta_info['indented']:
root = xmltree.getroot()
self.botsindent(root)
if ET.VERSION <= '1.2.6':
xmltree.write(f, encoding=self.ta_info['charset'])
else:
xmltree.write(f,
encoding=self.ta_info['charset'],
xml_declaration=False)
def run(self):
try:
from genshi.template import TemplateLoader
except:
raise ImportError(
u'Dependency failure: editype "templatehtml" requires python library "genshi".'
)
self._openoutenvelope()
self.ta_info.update(self.out.ta_info)
botslib.tryrunscript(self.userscript,
self.scriptname,
'ta_infocontent',
ta_info=self.ta_info)
if not self.ta_info['envelope-template']:
raise botslib.OutMessageError(
_(u'While enveloping in "%(editype)s.%(messagetype)s": syntax option "envelope-template" not filled; is required.'
), self.ta_info)
templatefile = botslib.abspath(self.__class__.__name__,
self.ta_info['envelope-template'])
ta_list = self.filelist2absolutepaths()
try:
botsglobal.logger.debug(
u'Start writing envelope to file "%(filename)s".',
self.ta_info)
loader = TemplateLoader(auto_reload=False)
tmpl = loader.load(templatefile)
except:
txt = botslib.txtexc()
raise botslib.OutMessageError(
_(u'While enveloping in "%(editype)s.%(messagetype)s", error:\n%(txt)s'
), {
'editype': self.ta_info['editype'],
'messagetype': self.ta_info['messagetype'],
'txt': txt
})
try:
filehandler = botslib.opendata(self.ta_info['filename'], 'wb')
stream = tmpl.generate(data=ta_list)
stream.render(method='xhtml',
encoding=self.ta_info['charset'],
out=filehandler)
except:
txt = botslib.txtexc()
raise botslib.OutMessageError(
_(u'While enveloping in "%(editype)s.%(messagetype)s", error:\n%(txt)s'
), {
'editype': self.ta_info['editype'],
'messagetype': self.ta_info['messagetype'],
'txt': txt
})
def run(self):
if not self.ta_info["topartner"] or not self.ta_info["frompartner"]:
raise botslib.OutMessageError(
_(u'In enveloping "frompartner" or "topartner" unknown: "$ta_info".'), ta_info=self.ta_info
)
self._openoutenvelope(self.ta_info["editype"], self.ta_info["envelope"])
self.ta_info.update(self.out.ta_info)
botslib.tryrunscript(self.userscript, self.scriptname, "ta_infocontent", ta_info=self.ta_info)
# prepare data for envelope
if botsglobal.ini.getboolean("settings", "interchangecontrolperpartner", False):
self.ta_info["reference"] = str(botslib.unique("stxcounter_" + self.ta_info["topartner"]))
else:
self.ta_info["reference"] = str(botslib.unique("stxcounter_" + self.ta_info["frompartner"]))
# build the envelope segments (that is, the tree from which the segments will be generated)
self.out.put(
{
"BOTSID": "STX",
"STDS1": self.ta_info["STX.STDS1"],
"STDS2": self.ta_info["STX.STDS2"],
"FROM.01": self.ta_info["frompartner"],
"UNTO.01": self.ta_info["topartner"],
"TRDT.01": time.strftime("%y%m%d"),
"TRDT.02": time.strftime("%H%M%S"),
"SNRF": self.ta_info["reference"],
}
)
if self.ta_info["STX.FROM.02"]:
self.out.put({"BOTSID": "STX", "FROM.02": self.ta_info["STX.FROM.02"]})
if self.ta_info["STX.UNTO.02"]:
self.out.put({"BOTSID": "STX", "UNTO.02": self.ta_info["STX.UNTO.02"]})
if self.ta_info["STX.APRF"]:
self.out.put({"BOTSID": "STX", "APRF": self.ta_info["STX.APRF"]})
if self.ta_info["STX.PRCD"]:
self.out.put({"BOTSID": "STX", "PRCD": self.ta_info["STX.PRCD"]})
self.out.put(
{"BOTSID": "STX"}, {"BOTSID": "END", "NMST": self.ta_info["nrmessages"]}
) # dummy segment; is not used
# user exit
botslib.tryrunscript(self.userscript, self.scriptname, "envelopecontent", ta_info=self.ta_info, out=self.out)
# convert the tree into segments; here only the STX is written (first segment)
self.out.checkmessage(self.out.root, self.out.defmessage)
self.out.tree2records(self.out.root)
# start doing the actual writing:
tofile = botslib.opendata(self.ta_info["filename"], "wb", self.ta_info["charset"])
tofile.write(self.out.record2string(self.out.records[0]))
self.writefilelist(tofile)
tofile.write(self.out.record2string(self.out.records[-1]))
tofile.close()
def run(self):
""" class for (test) orderprint; delevers a valid html-file.
Uses a kid-template for the enveloping/merging.
use kid to write; no envelope grammar is used
"""
try:
import kid
except:
txt = botslib.txtexc()
raise ImportError(
_(u'Dependency failure: editype "template" requires python library "kid". Error:\n%s' % txt)
)
defmessage = grammar.grammarread(
self.ta_info["editype"], self.ta_info["messagetype"]
) # needed because we do not know envelope; read syntax for editype/messagetype
self.ta_info.update(defmessage.syntax)
botslib.tryrunscript(self.userscript, self.scriptname, "ta_infocontent", ta_info=self.ta_info)
if not self.ta_info["envelope-template"]:
raise botslib.OutMessageError(
_(
u'While enveloping in "$editype.$messagetype": syntax option "envelope-template" not filled; is required.'
),
editype=self.ta_info["editype"],
messagetype=self.ta_info["messagetype"],
)
templatefile = botslib.abspath("templates", self.ta_info["envelope-template"])
ta_list = self.filelist2absolutepaths()
try:
botsglobal.logger.debug(u'Start writing envelope to file "%s".', self.ta_info["filename"])
ediprint = kid.Template(file=templatefile, data=ta_list) # init template; pass list with filenames
except:
txt = botslib.txtexc()
raise botslib.OutMessageError(
_(u'While enveloping in "$editype.$messagetype", error:\n$txt'),
editype=self.ta_info["editype"],
messagetype=self.ta_info["messagetype"],
txt=txt,
)
try:
filehandler = botslib.opendata(self.ta_info["filename"], "wb")
ediprint.write(filehandler, encoding=self.ta_info["charset"], output=self.ta_info["output"])
except:
txt = botslib.txtexc()
raise botslib.OutMessageError(
_(u'While enveloping in "$editype.$messagetype", error:\n$txt'),
editype=self.ta_info["editype"],
messagetype=self.ta_info["messagetype"],
txt=txt,
)
def run(self):
if self.ta_info['envelope'] == 'csvheader':
#~ Adds first line to csv files with fieldnames; than write files.
self._openoutenvelope()
botslib.tryrunscript(self.userscript,self.scriptname,'ta_infocontent',ta_info=self.ta_info)
tofile = botslib.opendata(self.ta_info['filename'],'wb',self.ta_info['charset'])
headers = dict((field[ID],field[ID]) for field in self.out.defmessage.structure[0][FIELDS])
self.out.put(headers)
self.out.tree2records(self.out.root)
tofile.write(self.out.record2string(self.out.lex_records[0:1]))
self.writefilelist(tofile)
tofile.close()
else:
super(csv,self).run()
def run(self):
self._openoutenvelope(self.ta_info['editype'],
self.ta_info['messagetype'])
botslib.tryrunscript(self.userscript,
self.scriptname,
'ta_infocontent',
ta_info=self.ta_info)
#self.ta_info is not overwritten
tofile = botslib.opendata(self.ta_info['filename'], 'wb',
self.ta_info['charset'])
headers = dict([(field[ID], field[ID])
for field in self.out.defmessage.structure[0][FIELDS]])
self.out.put(headers)
self.out.tree2records(self.out.root)
tofile.write(self.out.record2string(self.out.records[0]))
self.writefilelist(tofile)
tofile.close()
def run(self):
if not self.ta_info['topartner'] or not self.ta_info['frompartner']:
raise botslib.OutMessageError(_(u'In enveloping "frompartner" or "topartner" unknown: "%(ta_info)s".'),
{'ta_info':self.ta_info})
self._openoutenvelope()
self.ta_info.update(self.out.ta_info)
botslib.tryrunscript(self.userscript,self.scriptname,'ta_infocontent',ta_info=self.ta_info)
#prepare data for envelope
if botsglobal.ini.getboolean('settings','interchangecontrolperpartner',False):
self.ta_info['reference'] = unicode(botslib.unique('stxcounter_' + self.ta_info['topartner']))
else:
self.ta_info['reference'] = unicode(botslib.unique('stxcounter_' + self.ta_info['frompartner']))
#build the envelope segments (that is, the tree from which the segments will be generated)
self.out.put({'BOTSID':'STX',
'STDS1':self.ta_info['STX.STDS1'],
'STDS2':self.ta_info['STX.STDS2'],
'FROM.01':self.ta_info['frompartner'],
'UNTO.01':self.ta_info['topartner'],
'TRDT.01':botslib.strftime('%y%m%d'),
'TRDT.02':botslib.strftime('%H%M%S'),
'SNRF':self.ta_info['reference']})
if self.ta_info['STX.FROM.02']:
self.out.put({'BOTSID':'STX','FROM.02':self.ta_info['STX.FROM.02']})
if self.ta_info['STX.UNTO.02']:
self.out.put({'BOTSID':'STX','UNTO.02':self.ta_info['STX.UNTO.02']})
if self.ta_info['STX.APRF']:
self.out.put({'BOTSID':'STX','APRF':self.ta_info['STX.APRF']})
if self.ta_info['STX.PRCD']:
self.out.put({'BOTSID':'STX','PRCD':self.ta_info['STX.PRCD']})
self.out.put({'BOTSID':'STX'},{'BOTSID':'END','NMST':self.ta_info['nrmessages']}) #dummy segment; is not used
#user exit
botslib.tryrunscript(self.userscript,self.scriptname,'envelopecontent',ta_info=self.ta_info,out=self.out)
#convert the tree into segments; here only the STX is written (first segment)
self.out.checkmessage(self.out.root,self.out.defmessage)
self.out.checkforerrorlist()
self.out.tree2records(self.out.root)
#start doing the actual writing:
tofile = botslib.opendata(self.ta_info['filename'],'wb',self.ta_info['charset'])
tofile.write(self.out.record2string(self.out.lex_records[0:1]))
self.writefilelist(tofile)
tofile.write(self.out.record2string(self.out.lex_records[1:2]))
tofile.close()
def run(self):
if self.ta_info['envelope'] == 'csvheader':
#~ Adds first line to csv files with fieldnames; than write files.
self._openoutenvelope()
botslib.tryrunscript(self.userscript,
self.scriptname,
'ta_infocontent',
ta_info=self.ta_info)
tofile = botslib.opendata(self.ta_info['filename'], 'wb',
self.ta_info['charset'])
headers = dict(
(field[ID], field[ID])
for field in self.out.defmessage.structure[0][FIELDS])
self.out.put(headers)
self.out.tree2records(self.out.root)
tofile.write(self.out.record2string(self.out.lex_records[0:1]))
self.writefilelist(tofile)
tofile.close()
else:
super(csv, self).run()
def _write(self,node):
''' write normal XML messages (no envelope)'''
xmltree = ET.ElementTree(self._node2xml(node))
f = botslib.opendata(self.ta_info['filename'],"wb")
if self.ta_info['indented']:
indentstring = '\n'
else:
indentstring = ''
#xml prolog: always use.*********************************
#syntax parameter controls if stand-alone is used within prolog.
#in ET 1.3.0: if standalone is to be used: should surpress ET-generated prolog - explicit parameter
#in ET 1.2.6: always generates prolog if encoding != utf-8/ascii. SO: can not use stadnalone for encoding !=utf-8,ascii
if ET.VERSION not in ['1.2.6','1.0.6'] or self.ta_info['charset'] in ['us-ascii','utf-8']:
if self.ta_info['standalone']:
standalonestring = 'standalone="%s" '%(self.ta_info['standalone'])
else:
standalonestring = ''
PI = ET.ProcessingInstruction('xml', 'version="%s" encoding="%s" %s'%(self.ta_info['version'],self.ta_info['charset'], standalonestring))
f.write(ET.tostring(PI) + indentstring) #do not use encoding here. gives double xml prolog; possibly because ET.ElementTree.write i used again by write()
#doctype /DTD **************************************
if self.ta_info['DOCTYPE']:
f.write('<!DOCTYPE %s>'%(self.ta_info['DOCTYPE']) + indentstring)
#processing instructions (other than prolog) ************
if self.ta_info['processing_instructions']:
for pi in self.ta_info['processing_instructions']:
PI = ET.ProcessingInstruction(pi[0], pi[1])
f.write(ET.tostring(PI) + indentstring) #do not use encoding here. gives double xml prolog; possibly because ET.ElementTree.write i used again by write()
#indent the xml elements
if self.ta_info['indented']:
root = xmltree.getroot()
self.botsindent(root)
if ET.VERSION <= '1.2.6':
xmltree.write(f,encoding=self.ta_info['charset'])
else:
xmltree.write(f,encoding=self.ta_info['charset'],xml_declaration=False)
def _write(self,node_instance):
templatefile = botslib.abspath(self.__class__.__name__,self.ta_info['template'])
try:
botsglobal.logger.debug(u'Start writing to file "%(filename)s".',self.ta_info)
loader = self.template.TemplateLoader(auto_reload=False)
tmpl = loader.load(templatefile)
except:
txt = botslib.txtexc()
raise botslib.OutMessageError(_(u'While templating "%(editype)s.%(messagetype)s", error:\n%(txt)s'),
{'editype':self.ta_info['editype'],'messagetype':self.ta_info['messagetype'],'txt':txt})
try:
filehandler = botslib.opendata(self.ta_info['filename'],'wb')
if self.ta_info['has_structure']: #new way of working
if self.ta_info['print_as_row']:
node_instance.collectlines(self.ta_info['print_as_row'])
stream = tmpl.generate(node=node_instance)
else:
stream = tmpl.generate(data=self.data)
stream.render(method='xhtml',encoding=self.ta_info['charset'],out=filehandler)
except:
txt = botslib.txtexc()
raise botslib.OutMessageError(_(u'While templating "%(editype)s.%(messagetype)s", error:\n%(txt)s'),
{'editype':self.ta_info['editype'],'messagetype':self.ta_info['messagetype'],'txt':txt})
botsglobal.logger.debug(_(u'End writing to file "%(filename)s".'),self.ta_info)
def run(self):
self._openoutenvelope(self.ta_info['editype'],
self.ta_info['envelope'])
self.ta_info.update(self.out.ta_info)
#need to know the functionalgroup code:
defmessage = grammar.grammarread(self.ta_info['editype'],
self.ta_info['messagetype'])
self.ta_info['functionalgroup'] = defmessage.syntax['functionalgroup']
botslib.tryrunscript(self.userscript,
self.scriptname,
'ta_infocontent',
ta_info=self.ta_info)
#prepare data for envelope
ISA09date = time.strftime('%y%m%d')
#test indicator can either be from configuration (self.ta_info['ISA15']) or by mapping (self.ta_info['testindicator'])
#mapping overrules.
if self.ta_info['testindicator'] and self.ta_info[
'testindicator'] != '0': #'0' is default value (in db)
testindicator = self.ta_info['testindicator']
else:
testindicator = self.ta_info['ISA15']
#~ print self.ta_info['messagetype'], 'grammar:',self.ta_info['ISA15'],'ta:',self.ta_info['testindicator'],'out:',testindicator
if botsglobal.ini.getboolean('settings',
'interchangecontrolperpartner', False):
self.ta_info['reference'] = str(
botslib.unique('isacounter_' + self.ta_info['topartner']))
else:
self.ta_info['reference'] = str(
botslib.unique('isacounter_' + self.ta_info['frompartner']))
#ISA06 and GS02 can be different; eg ISA06 is a service provider.
#ISA06 and GS02 can be in the syntax....
ISA06 = self.ta_info.get('ISA06', self.ta_info['frompartner'])
GS02 = self.ta_info.get('GS02', self.ta_info['frompartner'])
#also for ISA08 and GS03
ISA08 = self.ta_info.get('ISA08', self.ta_info['topartner'])
GS03 = self.ta_info.get('GS03', self.ta_info['topartner'])
#build the envelope segments (that is, the tree from which the segments will be generated)
self.out.put({
'BOTSID': 'ISA',
'ISA01': self.ta_info['ISA01'],
'ISA02': self.ta_info['ISA02'],
'ISA03': self.ta_info['ISA03'],
'ISA04': self.ta_info['ISA04'],
'ISA05': self.ta_info['ISA05'],
'ISA06': ISA06,
'ISA07': self.ta_info['ISA07'],
'ISA08': ISA08,
'ISA09': ISA09date,
'ISA10': time.strftime('%H%M'),
'ISA11': self.ta_info[
'ISA11'], #if ISA version > 00403, replaced by reprtion separator
'ISA12': self.ta_info['version'],
'ISA13': self.ta_info['reference'],
'ISA14': self.ta_info['ISA14'],
'ISA15': testindicator
})
self.out.put({'BOTSID': 'ISA'}, {
'BOTSID': 'IEA',
'IEA01': '1',
'IEA02': self.ta_info['reference']
})
GS08 = self.ta_info['messagetype'][3:]
if GS08[:6] < '004010':
GS04date = time.strftime('%y%m%d')
else:
GS04date = time.strftime('%Y%m%d')
self.out.put({'BOTSID': 'ISA'}, {
'BOTSID': 'GS',
'GS01': self.ta_info['functionalgroup'],
'GS02': GS02,
'GS03': GS03,
'GS04': GS04date,
'GS05': time.strftime('%H%M'),
'GS06': self.ta_info['reference'],
'GS07': self.ta_info['GS07'],
'GS08': GS08
})
self.out.put({'BOTSID': 'ISA'}, {'BOTSID': 'GS'}, {
'BOTSID': 'GE',
'GE01': self.ta_info['nrmessages'],
'GE02': self.ta_info['reference']
}) #dummy segment; is not used
#user exit
botslib.tryrunscript(self.userscript,
self.scriptname,
'envelopecontent',
ta_info=self.ta_info,
out=self.out)
#convert the tree into segments; here only the UNB is written (first segment)
self.out.normalisetree(self.out.root)
self.out.tree2records(self.out.root)
#start doing the actual writing:
tofile = botslib.opendata(self.ta_info['filename'], 'wb',
self.ta_info['charset'])
ISAstring = self.out._record2string(self.out.records[0])
if self.ta_info['version'] < '00403':
ISAstring = ISAstring[:103] + self.ta_info[
'field_sep'] + self.ta_info['sfield_sep'] + ISAstring[
103:] #hack for strange characters at end of ISA; hardcoded
else:
ISAstring = ISAstring[:82] + self.ta_info['reserve'] + ISAstring[
83:103] + self.ta_info['field_sep'] + self.ta_info[
'sfield_sep'] + ISAstring[
103:] #hack for strange characters at end of ISA; hardcoded
tofile.write(ISAstring) #write ISA
tofile.write(self.out._record2string(self.out.records[1])) #write GS
self.writefilelist(tofile)
tofile.write(self.out._record2string(self.out.records[-2])) #write GE
tofile.write(self.out._record2string(self.out.records[-1])) #write IEA
tofile.close()
if self.ta_info[
'functionalgroup'] != 'FA' and botslib.checkconfirmrules(
'ask-x12-997',
idroute=self.ta_info['idroute'],
idchannel=self.ta_info['tochannel'],
topartner=self.ta_info['topartner'],
frompartner=self.ta_info['frompartner'],
editype=self.ta_info['editype'],
messagetype=self.ta_info['messagetype']):
self.ta_info['confirmtype'] = u'ask-x12-997'
self.ta_info['confirmasked'] = True
def mailbag(ta_from, endstatus, **argv):
''' 2 main functions:
- recognizes and distuinguishes several edi types: x12 edifact tradacoms xml
('mailbag' in, correct editype out)
- split up interchanges (edifact, x12, tradacoms)
details:
- edifact, x12 and tradacoms can be can be mixed,
- recognizes xml files; but messagetype 'xml' has a special handling when reading xml-files: xpath identifiers are used.
this is needed for using xml in mailbag!
- when more interchanges in one file: strongly recommended to mailbag/split these.
- handle multiple UNA in one file, including different charsets.
- handle multiple x12 seperators in one file.
'''
edifile = botslib.readdata(filename=ta_from.filename) #read as binary...
startpos = 0
while (1):
found = HEADER.search(edifile[startpos:])
if found is None:
if startpos: #ISA/UNB have been found in file; no new ISA/UNB is found. So all processing is done.
break
#guess if this is an xml file.....
sniffxml = edifile[:25]
sniffxml = sniffxml.lstrip(
' \t\n\r\f\v\xFF\xFE\xEF\xBB\xBF\x00'
) #to find first ' real' data; some char are because of BOM, UTF-16 etc
if sniffxml and sniffxml[0] == '<':
ta_to = ta_from.copyta(
status=endstatus,
statust=OK,
filename=ta_from.filename,
editype='xml',
messagetype='mailbag'
) #make transaction for translated message; gets ta_info of ta_frommes
#~ ta_tomes.update(status=STATUSTMP,statust=OK,filename=ta_set_for_processing.filename,editype='xml') #update outmessage transaction with ta_info;
break
else:
raise botslib.InMessageError(
_(u'Found no content in mailbag.'))
elif found.group(1):
editype = 'x12'
headpos = startpos + found.start(2)
count = 0
for char in edifile[
headpos:headpos +
120]: #search first 120 characters to find separators
if char in '\r\n' and count != 105:
continue
count += 1
if count == 4:
field_sep = char
elif count == 106:
record_sep = char
break
#~ foundtrailer = re.search(re.escape(record_sep)+'\s*IEA'+re.escape(field_sep)+'.+?'+re.escape(record_sep),edifile[headpos:],re.DOTALL)
foundtrailer = re.search(
re.escape(record_sep) + '\s*I\s*E\s*A\s*' +
re.escape(field_sep) + '.+?' + re.escape(record_sep),
edifile[headpos:], re.DOTALL)
elif found.group(3):
editype = 'edifact'
if found.group(4):
field_sep = edifile[startpos + found.start(4) + 4]
record_sep = edifile[startpos + found.start(4) + 8]
headpos = startpos + found.start(4)
else:
field_sep = '+'
record_sep = "'"
headpos = startpos + found.start(5)
foundtrailer = re.search(
re.escape(record_sep) + '\s*U\s*N\s*Z\s*' +
re.escape(field_sep) + '.+?' + re.escape(record_sep),
edifile[headpos:], re.DOTALL)
elif found.group(8):
editype = 'tradacoms'
headpos = startpos + found.start(9)
field_sep = '=' #the tradacoms 'after-segment-tag-seperator'
record_sep = "'"
foundtrailer = re.search(
re.escape(record_sep) + '\s*E\s*N\s*D\s*' +
re.escape(field_sep) + '.+?' + re.escape(record_sep),
edifile[headpos:], re.DOTALL)
if not foundtrailer:
raise botslib.InMessageError(
_(u'Found no valid envelope trailer in mailbag.'))
endpos = headpos + foundtrailer.end()
#so: interchange is from headerpos untill endpos
#~ if HEADER.search(edifile[headpos+25:endpos]): #check if there is another header in the interchange
#~ raise botslib.InMessageError(u'Error in mailbag format: found no valid envelope trailer.')
ta_to = ta_from.copyta(
status=endstatus
) #make transaction for translated message; gets ta_info of ta_frommes
tofilename = str(ta_to.idta)
tofile = botslib.opendata(tofilename, 'wb')
tofile.write(edifile[headpos:endpos])
tofile.close()
ta_to.update(
statust=OK,
filename=tofilename,
editype=editype,
messagetype=editype) #update outmessage transaction with ta_info;
startpos = endpos
botsglobal.logger.debug(_(u' File written: "%s".'), tofilename)
def _initwrite(self):
botsglobal.logger.debug(u'Start writing to file "%s".',self.ta_info['filename'])
self._outstream = botslib.opendata(self.ta_info['filename'],"wb")
def extractpdf(ta_from, endstatus, **argv):
''' Try to extract text content of a PDF file to a csv.
You know this is not a great idea, right? But we'll do the best we can anyway!
Page and line numbers are added to each row.
Columns and rows are based on the x and y coordinates of each text element within tolerance allowed.
Multiple text elements may combine to make one field, some PDFs have every character separated!
You may need to experiment with x_group and y_group values, but defaults seem ok for most files.
Output csv is UTF-8 encoded - The csv module doesn't directly support reading and writing Unicode
If the PDF is just an image, all bets are off. Maybe try OCR, good luck with that!
Mike Griffin 14/12/2011
'''
from pdfminer.pdfinterp import PDFResourceManager, process_pdf
from pdfminer.converter import TextConverter
from pdfminer.layout import LAParams, LTContainer, LTText, LTTextBox
import csv
class CsvConverter(TextConverter):
def __init__(self, *args, **kwargs):
TextConverter.__init__(self, *args, **kwargs)
def receive_layout(self, ltpage):
# recursively get every text element and it's coordinates
def render(item):
if isinstance(item, LTContainer):
for child in item:
render(child)
elif isinstance(item, LTText):
(_, _, x, y) = item.bbox
# group the y values (rows) within group tolerance
for v in yv:
if y > v - y_group and y < v + y_group:
y = v
yv.append(y)
line = lines[int(-y)]
line[x] = item.get_text().encode('utf-8')
from collections import defaultdict
lines = defaultdict(lambda: {})
yv = []
render(ltpage)
lineid = 0
for y in sorted(lines.keys()):
line = lines[y]
lineid += 1
csvdata = [ltpage.pageid,
lineid] # first 2 columns are page and line numbers
# group the x values (fields) within group tolerance
p = 0
field_txt = ''
for x in sorted(line.keys()):
gap = x - p
if p > 0 and gap > x_group:
csvdata.append(field_txt)
field_txt = ''
field_txt += line[x]
p = x
csvdata.append(field_txt)
csvout.writerow(csvdata)
if lineid == 0:
raise botslib.InMessageError(
_(u'PDF text extraction failed, it may contain just image(s)?'
))
#get some optional parameters
x_group = argv.get('x_group',
10) # group text closer than this as one field
y_group = argv.get('y_group',
5) # group lines closer than this as one line
password = argv.get('password', '')
quotechar = argv.get('quotechar', '"')
field_sep = argv.get('field_sep', ',')
escape = argv.get('escape', '\\')
charset = argv.get('charset', 'utf-8')
if not escape:
doublequote = True
else:
doublequote = False
try:
pdf_stream = botslib.opendata(ta_from.filename, 'rb')
ta_to = ta_from.copyta(status=endstatus)
tofilename = str(ta_to.idta)
csv_stream = botslib.opendata(tofilename, 'wb')
csvout = csv.writer(csv_stream,
quotechar=quotechar,
delimiter=field_sep,
doublequote=doublequote,
escapechar=escape)
# Process PDF
rsrcmgr = PDFResourceManager(caching=True)
device = CsvConverter(rsrcmgr, csv_stream, codec=charset)
process_pdf(rsrcmgr,
device,
pdf_stream,
pagenos=set(),
password=password,
caching=True,
check_extractable=True)
device.close()
pdf_stream.close()
csv_stream.close()
ta_to.update(
statust=OK,
filename=tofilename) #update outmessage transaction with ta_info;
botsglobal.logger.debug(_(u' File written: "%s".'), tofilename)
except:
txt = botslib.txtexc()
botsglobal.logger.error(
_(u'PDF extraction failed, may not be a PDF file? Error:\n%s'),
txt)
raise botslib.InMessageError(
_(u'PDF extraction failed, may not be a PDF file? Error:\n$error'),
error=txt)
def run(self):
self._openoutenvelope(self.ta_info['editype'],self.ta_info['envelope'])
self.ta_info.update(self.out.ta_info)
botslib.tryrunscript(self.userscript,self.scriptname,'ta_infocontent',ta_info=self.ta_info)
#version dependent enveloping
writeUNA = False
if self.ta_info['version']<'4':
date = time.strftime('%y%m%d')
reserve = ' '
if self.ta_info['charset'] != 'UNOA':
writeUNA = True
else:
date = time.strftime('%Y%m%d')
reserve = self.ta_info['reserve']
if self.ta_info['charset'] not in ['UNOA','UNOB']:
writeUNA = True
#UNB counter is per sender or receiver
if botsglobal.ini.getboolean('settings','interchangecontrolperpartner',False):
self.ta_info['reference'] = str(botslib.unique('unbcounter_' + self.ta_info['topartner']))
else:
self.ta_info['reference'] = str(botslib.unique('unbcounter_' + self.ta_info['frompartner']))
#testindicator is more complex:
if self.ta_info['testindicator'] and self.ta_info['testindicator']!='0': #first check value from ta; do not use default
testindicator = '1'
elif self.ta_info['UNB.0035'] != '0': #than check values from grammar
testindicator = '1'
else:
testindicator = ''
#build the envelope segments (that is, the tree from which the segments will be generated)
self.out.put({'BOTSID':'UNB',
'S001.0001':self.ta_info['charset'],
'S001.0002':self.ta_info['version'],
'S002.0004':self.ta_info['frompartner'],
'S003.0010':self.ta_info['topartner'],
'S004.0017':date,
'S004.0019':time.strftime('%H%M'),
'0020':self.ta_info['reference']})
#the following fields are conditional; do not write these when empty string (separator compression does take empty strings into account)
if self.ta_info['UNB.S002.0007']:
self.out.put({'BOTSID':'UNB','S002.0007': self.ta_info['UNB.S002.0007']})
if self.ta_info['UNB.S003.0007']:
self.out.put({'BOTSID':'UNB','S003.0007': self.ta_info['UNB.S003.0007']})
if self.ta_info['UNB.0026']:
self.out.put({'BOTSID':'UNB','0026': self.ta_info['UNB.0026']})
if testindicator:
self.out.put({'BOTSID':'UNB','0035': testindicator})
self.out.put({'BOTSID':'UNB'},{'BOTSID':'UNZ','0036':self.ta_info['nrmessages'],'0020':self.ta_info['reference']}) #dummy segment; is not used
#user exit
botslib.tryrunscript(self.userscript,self.scriptname,'envelopecontent',ta_info=self.ta_info,out=self.out)
#convert the tree into segments; here only the UNB is written (first segment)
self.out.normalisetree(self.out.root)
self.out.tree2records(self.out.root)
#start doing the actual writing:
tofile = botslib.opendata(self.ta_info['filename'],'wb',self.ta_info['charset'])
if writeUNA or self.ta_info['forceUNA']:
tofile.write('UNA'+self.ta_info['sfield_sep']+self.ta_info['field_sep']+self.ta_info['decimaal']+self.ta_info['escape']+ reserve +self.ta_info['record_sep']+self.ta_info['add_crlfafterrecord_sep'])
tofile.write(self.out._record2string(self.out.records[0]))
self.writefilelist(tofile)
tofile.write(self.out._record2string(self.out.records[-1]))
tofile.close()
def mailbag(ta_from,endstatus,**argv):
''' 2 main functions:
- recognizes and distuinguishes several edi types: x12 edifact tradacoms xml
('mailbag' in, correct editype out)
- split up interchanges (edifact, x12, tradacoms)
details:
- edifact, x12 and tradacoms can be can be mixed,
- recognizes xml files; but messagetype 'xml' has a special handling when reading xml-files: xpath identifiers are used.
this is needed for using xml in mailbag!
- when more interchanges in one file: strongly recommended to mailbag/split these.
- handle multiple UNA in one file, including different charsets.
- handle multiple x12 seperators in one file.
'''
edifile = botslib.readdata(filename=ta_from.filename) #read as binary...
startpos = 0
while (1):
found = HEADER.search(edifile[startpos:])
if found is None:
if startpos: #ISA/UNB have been found in file; no new ISA/UNB is found. So all processing is done.
break
#guess if this is an xml file.....
sniffxml = edifile[:25]
sniffxml = sniffxml.lstrip(' \t\n\r\f\v\xFF\xFE\xEF\xBB\xBF\x00') #to find first ' real' data; some char are because of BOM, UTF-16 etc
if sniffxml and sniffxml[0] == '<':
ta_to = ta_from.copyta(status=endstatus,statust=OK,filename=ta_from.filename,editype='xml',messagetype='mailbag') #make transaction for translated message; gets ta_info of ta_frommes
#~ ta_tomes.update(status=STATUSTMP,statust=OK,filename=ta_set_for_processing.filename,editype='xml') #update outmessage transaction with ta_info;
break
else:
raise botslib.InMessageError(_(u'Found no content in mailbag.'))
elif found.group(1):
editype = 'x12'
headpos = startpos + found.start(2)
count = 0
for char in edifile[headpos:headpos+120]: #search first 120 characters to find separators
if char in '\r\n' and count != 105:
continue
count += 1
if count == 4:
field_sep = char
elif count == 106:
record_sep = char
break
#~ foundtrailer = re.search(re.escape(record_sep)+'\s*IEA'+re.escape(field_sep)+'.+?'+re.escape(record_sep),edifile[headpos:],re.DOTALL)
foundtrailer = re.search(re.escape(record_sep)+'\s*I\s*E\s*A\s*'+re.escape(field_sep)+'.+?'+re.escape(record_sep),edifile[headpos:],re.DOTALL)
elif found.group(3):
editype = 'edifact'
if found.group(4):
field_sep = edifile[startpos + found.start(4) + 4]
record_sep = edifile[startpos + found.start(4) + 8]
headpos = startpos + found.start(4)
else:
field_sep = '+'
record_sep = "'"
headpos = startpos + found.start(5)
foundtrailer = re.search(re.escape(record_sep)+'\s*U\s*N\s*Z\s*'+re.escape(field_sep)+'.+?'+re.escape(record_sep),edifile[headpos:],re.DOTALL)
elif found.group(8):
editype = 'tradacoms'
headpos = startpos + found.start(9)
field_sep = '=' #the tradacoms 'after-segment-tag-seperator'
record_sep = "'"
foundtrailer = re.search(re.escape(record_sep)+'\s*E\s*N\s*D\s*'+re.escape(field_sep)+'.+?'+re.escape(record_sep),edifile[headpos:],re.DOTALL)
if not foundtrailer:
raise botslib.InMessageError(_(u'Found no valid envelope trailer in mailbag.'))
endpos = headpos + foundtrailer.end()
#so: interchange is from headerpos untill endpos
#~ if HEADER.search(edifile[headpos+25:endpos]): #check if there is another header in the interchange
#~ raise botslib.InMessageError(u'Error in mailbag format: found no valid envelope trailer.')
ta_to = ta_from.copyta(status=endstatus) #make transaction for translated message; gets ta_info of ta_frommes
tofilename = str(ta_to.idta)
tofile = botslib.opendata(tofilename,'wb')
tofile.write(edifile[headpos:endpos])
tofile.close()
ta_to.update(statust=OK,filename=tofilename,editype=editype,messagetype=editype) #update outmessage transaction with ta_info;
startpos = endpos
botsglobal.logger.debug(_(u' File written: "%s".'),tofilename)
def run(self):
botslib.tryrunscript(self.userscript, self.scriptname, "ta_infocontent", ta_info=self.ta_info)
tofile = botslib.opendata(self.ta_info["filename"], "wb", self.ta_info["charset"])
self.writefilelist(tofile)
tofile.close()
def _initwrite(self):
botsglobal.logger.debug(u'Start writing to file "%s".',self.ta_info['filename'])
self._outstream = botslib.opendata(self.ta_info['filename'],"wb")
def _initwrite(self):
botsglobal.logger.debug(u'Start writing to file "%s".',self.ta_info['filename'])
self._outstream = botslib.opendata(self.ta_info['filename'],'wb',charset=self.ta_info['charset'],errors=self.ta_info['checkcharsetout'])
def run(self):
if not self.ta_info['topartner'] or not self.ta_info['frompartner']:
raise botslib.OutMessageError(_(u'In enveloping "frompartner" or "topartner" unknown: "%(ta_info)s".'),
{'ta_info':self.ta_info})
self._openoutenvelope()
self.ta_info.update(self.out.ta_info)
botslib.tryrunscript(self.userscript,self.scriptname,'ta_infocontent',ta_info=self.ta_info)
#prepare data for envelope
isa09date = botslib.strftime('%y%m%d')
#test indicator can either be from configuration (self.ta_info['ISA15']) or by mapping (self.ta_info['testindicator'])
#mapping overrules.
if self.ta_info['testindicator'] and self.ta_info['testindicator'] != '0': #'0' is default value (in db)
testindicator = self.ta_info['testindicator']
else:
testindicator = self.ta_info['ISA15']
#~ print self.ta_info['messagetype'], 'grammar:',self.ta_info['ISA15'],'ta:',self.ta_info['testindicator'],'out:',testindicator
if botsglobal.ini.getboolean('settings','interchangecontrolperpartner',False):
self.ta_info['reference'] = unicode(botslib.unique('isacounter_' + self.ta_info['topartner']))
else:
self.ta_info['reference'] = unicode(botslib.unique('isacounter_' + self.ta_info['frompartner']))
#ISA06 and GS02 can be different; eg ISA06 is a service provider.
#ISA06 and GS02 can be in the syntax....
isa06sender = self.ta_info.get('ISA06',self.ta_info['frompartner'])
isa06sender = isa06sender.ljust(15) #add spaces; is fixed length
gs02sender = self.ta_info.get('GS02',self.ta_info['frompartner'])
#also for ISA08 and GS03
isa08receiver = self.ta_info.get('ISA08',self.ta_info['topartner'])
isa08receiver = isa08receiver.ljust(15) #add spaces; is fixed length
gs03receiver = self.ta_info.get('GS03',self.ta_info['topartner'])
#build the envelope segments (that is, the tree from which the segments will be generated)
self.out.put({'BOTSID':'ISA',
'ISA01':self.ta_info['ISA01'],
'ISA02':self.ta_info['ISA02'],
'ISA03':self.ta_info['ISA03'],
'ISA04':self.ta_info['ISA04'],
'ISA05':self.ta_info['ISA05'],
'ISA06':isa06sender,
'ISA07':self.ta_info['ISA07'],
'ISA08':isa08receiver,
'ISA09':isa09date,
'ISA10':botslib.strftime('%H%M'),
'ISA11':self.ta_info['ISA11'], #if ISA version > 00403, replaced by reprtion separator
'ISA12':self.ta_info['version'],
'ISA13':self.ta_info['reference'],
'ISA14':self.ta_info['ISA14'],
'ISA15':testindicator},strip=False) #MIND: strip=False: ISA fields shoudl not be stripped as it is soemwhat like fixed-length
self.out.put({'BOTSID':'ISA'},{'BOTSID':'IEA','IEA01':'1','IEA02':self.ta_info['reference']})
gs08messagetype = self.ta_info['messagetype'][3:]
if gs08messagetype[:6] < '004010':
gs04date = botslib.strftime('%y%m%d')
else:
gs04date = botslib.strftime('%Y%m%d')
self.out.put({'BOTSID':'ISA'},{'BOTSID':'GS',
'GS01':self.ta_info['functionalgroup'],
'GS02':gs02sender,
'GS03':gs03receiver,
'GS04':gs04date,
'GS05':botslib.strftime('%H%M'),
'GS06':self.ta_info['reference'],
'GS07':self.ta_info['GS07'],
'GS08':gs08messagetype})
self.out.put({'BOTSID':'ISA'},{'BOTSID':'GS'},{'BOTSID':'GE','GE01':self.ta_info['nrmessages'],'GE02':self.ta_info['reference']}) #dummy segment; is not used
#user exit
botslib.tryrunscript(self.userscript,self.scriptname,'envelopecontent',ta_info=self.ta_info,out=self.out)
#convert the tree into segments; here only the UNB is written (first segment)
self.out.checkmessage(self.out.root,self.out.defmessage)
self.out.checkforerrorlist()
self.out.tree2records(self.out.root)
#start doing the actual writing:
tofile = botslib.opendata(self.ta_info['filename'],'wb',self.ta_info['charset'])
isa_string = self.out.record2string(self.out.lex_records[0:1])
#ISA has the used separators at certain positions. Normally bots would give errors for this (can not use sep as data) or compress these aways. So this is hardcoded.
if self.ta_info['version'] < '00403':
isa_string = isa_string[:103] + self.ta_info['field_sep']+ self.ta_info['sfield_sep'] + isa_string[103:]
else:
isa_string = isa_string[:82] +self.ta_info['reserve'] + isa_string[83:103] + self.ta_info['field_sep']+ self.ta_info['sfield_sep'] + isa_string[103:]
tofile.write(isa_string) #write ISA
tofile.write(self.out.record2string(self.out.lex_records[1:2])) #write GS
self.writefilelist(tofile)
tofile.write(self.out.record2string(self.out.lex_records[2:])) #write GE and IEA
tofile.close()
def run(self):
self._openoutenvelope(self.ta_info['editype'],self.ta_info['envelope'])
self.ta_info.update(self.out.ta_info)
#need to know the functionalgroup code:
defmessage = grammar.grammarread(self.ta_info['editype'],self.ta_info['messagetype'])
self.ta_info['functionalgroup'] = defmessage.syntax['functionalgroup']
botslib.tryrunscript(self.userscript,self.scriptname,'ta_infocontent',ta_info=self.ta_info)
#prepare data for envelope
ISA09date = time.strftime('%y%m%d')
#test indicator can either be from configuration (self.ta_info['ISA15']) or by mapping (self.ta_info['testindicator'])
#mapping overrules.
if self.ta_info['testindicator'] and self.ta_info['testindicator']!='0': #'0' is default value (in db)
testindicator = self.ta_info['testindicator']
else:
testindicator = self.ta_info['ISA15']
#~ print self.ta_info['messagetype'], 'grammar:',self.ta_info['ISA15'],'ta:',self.ta_info['testindicator'],'out:',testindicator
if botsglobal.ini.getboolean('settings','interchangecontrolperpartner',False):
self.ta_info['reference'] = str(botslib.unique('isacounter_' + self.ta_info['topartner']))
else:
self.ta_info['reference'] = str(botslib.unique('isacounter_' + self.ta_info['frompartner']))
#ISA06 and GS02 can be different; eg ISA06 is a service provider.
#ISA06 and GS02 can be in the syntax....
ISA06 = self.ta_info.get('ISA06',self.ta_info['frompartner'])
GS02 = self.ta_info.get('GS02',self.ta_info['frompartner'])
#also for ISA08 and GS03
ISA08 = self.ta_info.get('ISA08',self.ta_info['topartner'])
GS03 = self.ta_info.get('GS03',self.ta_info['topartner'])
#build the envelope segments (that is, the tree from which the segments will be generated)
self.out.put({'BOTSID':'ISA',
'ISA01':self.ta_info['ISA01'],
'ISA02':self.ta_info['ISA02'],
'ISA03':self.ta_info['ISA03'],
'ISA04':self.ta_info['ISA04'],
'ISA05':self.ta_info['ISA05'],
'ISA06':ISA06,
'ISA07':self.ta_info['ISA07'],
'ISA08':ISA08,
'ISA09':ISA09date,
'ISA10':time.strftime('%H%M'),
'ISA11':self.ta_info['ISA11'], #if ISA version > 00403, replaced by reprtion separator
'ISA12':self.ta_info['version'],
'ISA13':self.ta_info['reference'],
'ISA14':self.ta_info['ISA14'],
'ISA15':testindicator})
self.out.put({'BOTSID':'ISA'},{'BOTSID':'IEA','IEA01':'1','IEA02':self.ta_info['reference']})
GS08 = self.ta_info['messagetype'][3:]
if GS08[:6]<'004010':
GS04date = time.strftime('%y%m%d')
else:
GS04date = time.strftime('%Y%m%d')
self.out.put({'BOTSID':'ISA'},{'BOTSID':'GS',
'GS01':self.ta_info['functionalgroup'],
'GS02':GS02,
'GS03':GS03,
'GS04':GS04date,
'GS05':time.strftime('%H%M'),
'GS06':self.ta_info['reference'],
'GS07':self.ta_info['GS07'],
'GS08':GS08})
self.out.put({'BOTSID':'ISA'},{'BOTSID':'GS'},{'BOTSID':'GE','GE01':self.ta_info['nrmessages'],'GE02':self.ta_info['reference']}) #dummy segment; is not used
#user exit
botslib.tryrunscript(self.userscript,self.scriptname,'envelopecontent',ta_info=self.ta_info,out=self.out)
#convert the tree into segments; here only the UNB is written (first segment)
self.out.normalisetree(self.out.root)
self.out.tree2records(self.out.root)
#start doing the actual writing:
tofile = botslib.opendata(self.ta_info['filename'],'wb',self.ta_info['charset'])
ISAstring = self.out._record2string(self.out.records[0])
if self.ta_info['version']<'00403':
ISAstring = ISAstring[:103] + self.ta_info['field_sep']+ self.ta_info['sfield_sep'] + ISAstring[103:] #hack for strange characters at end of ISA; hardcoded
else:
ISAstring = ISAstring[:82] +self.ta_info['reserve'] + ISAstring[83:103] + self.ta_info['field_sep']+ self.ta_info['sfield_sep'] + ISAstring[103:] #hack for strange characters at end of ISA; hardcoded
tofile.write(ISAstring) #write ISA
tofile.write(self.out._record2string(self.out.records[1])) #write GS
self.writefilelist(tofile)
tofile.write(self.out._record2string(self.out.records[-2])) #write GE
tofile.write(self.out._record2string(self.out.records[-1])) #write IEA
tofile.close()
if self.ta_info['functionalgroup']!='FA' and botslib.checkconfirmrules('ask-x12-997',idroute=self.ta_info['idroute'],idchannel=self.ta_info['tochannel'],
topartner=self.ta_info['topartner'],frompartner=self.ta_info['frompartner'],
editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype']):
self.ta_info['confirmtype'] = u'ask-x12-997'
self.ta_info['confirmasked'] = True
def writefilelist(self, tofile):
for filename in self.ta_list:
fromfile = botslib.opendata(filename, 'rb',
self.ta_info['charset'])
shutil.copyfileobj(fromfile, tofile, 1048576)
fromfile.close()
def extractpdf(ta_from,endstatus,**argv):
''' Try to extract text content of a PDF file to a csv.
You know this is not a great idea, right? But we'll do the best we can anyway!
Page and line numbers are added to each row.
Columns and rows are based on the x and y coordinates of each text element within tolerance allowed.
Multiple text elements may combine to make one field, some PDFs have every character separated!
You may need to experiment with x_group and y_group values, but defaults seem ok for most files.
Output csv is UTF-8 encoded - The csv module doesn't directly support reading and writing Unicode
If the PDF is just an image, all bets are off. Maybe try OCR, good luck with that!
Mike Griffin 14/12/2011
'''
from pdfminer.pdfinterp import PDFResourceManager, process_pdf
from pdfminer.converter import TextConverter
from pdfminer.layout import LAParams, LTContainer, LTText, LTTextBox
import csv
class CsvConverter(TextConverter):
def __init__(self, *args, **kwargs):
TextConverter.__init__(self, *args, **kwargs)
def receive_layout(self, ltpage):
# recursively get every text element and it's coordinates
def render(item):
if isinstance(item, LTContainer):
for child in item:
render(child)
elif isinstance(item, LTText):
(_,_,x,y) = item.bbox
# group the y values (rows) within group tolerance
for v in yv:
if y > v-y_group and y < v+y_group:
y = v
yv.append(y)
line = lines[int(-y)]
line[x] = item.get_text().encode('utf-8')
from collections import defaultdict
lines = defaultdict(lambda : {})
yv = []
render(ltpage)
lineid = 0
for y in sorted(lines.keys()):
line = lines[y]
lineid += 1
csvdata = [ltpage.pageid,lineid] # first 2 columns are page and line numbers
# group the x values (fields) within group tolerance
p = 0
field_txt = ''
for x in sorted(line.keys()):
gap = x - p
if p > 0 and gap > x_group:
csvdata.append(field_txt)
field_txt = ''
field_txt += line[x]
p = x
csvdata.append(field_txt)
csvout.writerow(csvdata)
if lineid == 0:
raise botslib.InMessageError(_(u'PDF text extraction failed, it may contain just image(s)?'))
#get some optional parameters
x_group = argv.get('x_group',10) # group text closer than this as one field
y_group = argv.get('y_group',5) # group lines closer than this as one line
password = argv.get('password','')
quotechar = argv.get('quotechar','"')
field_sep = argv.get('field_sep',',')
escape = argv.get('escape','\\')
charset = argv.get('charset','utf-8')
if not escape:
doublequote = True
else:
doublequote = False
try:
pdf_stream = botslib.opendata(ta_from.filename, 'rb')
ta_to = ta_from.copyta(status=endstatus)
tofilename = str(ta_to.idta)
csv_stream = botslib.opendata(tofilename,'wb')
csvout = csv.writer(csv_stream, quotechar=quotechar, delimiter=field_sep, doublequote=doublequote, escapechar=escape)
# Process PDF
rsrcmgr = PDFResourceManager(caching=True)
device = CsvConverter(rsrcmgr, csv_stream, codec=charset)
process_pdf(rsrcmgr, device, pdf_stream, pagenos=set(), password=password, caching=True, check_extractable=True)
device.close()
pdf_stream.close()
csv_stream.close()
ta_to.update(statust=OK,filename=tofilename) #update outmessage transaction with ta_info;
botsglobal.logger.debug(_(u' File written: "%s".'),tofilename)
except:
txt = botslib.txtexc()
botsglobal.logger.error(_(u'PDF extraction failed, may not be a PDF file? Error:\n%s'),txt)
raise botslib.InMessageError(_(u'PDF extraction failed, may not be a PDF file? Error:\n$error'),error=txt)
def dump_csv(xlsdata, tofilename):
stream = botslib.opendata(tofilename, 'wb')
csvout = csv.writer(stream, quotechar=quotechar, delimiter=field_sep, doublequote=doublequote, escapechar=escape)
csvout.writerows( map(utf8ize, xlsdata) )
stream.close()
def run(self):
if not self.ta_info["topartner"] or not self.ta_info["frompartner"]:
raise botslib.OutMessageError(
_(u'In enveloping "frompartner" or "topartner" unknown: "$ta_info".'), ta_info=self.ta_info
)
self._openoutenvelope(self.ta_info["editype"], self.ta_info["envelope"])
self.ta_info.update(self.out.ta_info)
botslib.tryrunscript(self.userscript, self.scriptname, "ta_infocontent", ta_info=self.ta_info)
# version dependent enveloping
write_una = False
if self.ta_info["version"] < "4":
date = time.strftime("%y%m%d")
reserve = " "
if self.ta_info["charset"] != "UNOA":
write_una = True
else:
date = time.strftime("%Y%m%d")
reserve = self.ta_info["reserve"]
if self.ta_info["charset"] not in ["UNOA", "UNOB"]:
write_una = True
# UNB counter is per sender or receiver
if botsglobal.ini.getboolean("settings", "interchangecontrolperpartner", False):
self.ta_info["reference"] = str(botslib.unique("unbcounter_" + self.ta_info["topartner"]))
else:
self.ta_info["reference"] = str(botslib.unique("unbcounter_" + self.ta_info["frompartner"]))
# testindicator is more complex:
if (
self.ta_info["testindicator"] and self.ta_info["testindicator"] != "0"
): # first check value from ta; do not use default
testindicator = "1"
elif self.ta_info["UNB.0035"] != "0": # than check values from grammar
testindicator = "1"
else:
testindicator = ""
# build the envelope segments (that is, the tree from which the segments will be generated)
self.out.put(
{
"BOTSID": "UNB",
"S001.0001": self.ta_info["charset"],
"S001.0002": self.ta_info["version"],
"S002.0004": self.ta_info["frompartner"],
"S003.0010": self.ta_info["topartner"],
"S004.0017": date,
"S004.0019": time.strftime("%H%M"),
"0020": self.ta_info["reference"],
}
)
# the following fields are conditional; do not write these when empty string (separator compression does take empty strings into account)
for field in (
"S001.0080",
"S001.0133",
"S002.0007",
"S002.0008",
"S002.0042",
"S003.0007",
"S003.0014",
"S003.0046",
"S005.0022",
"S005.0025",
"0026",
"0029",
"0031",
"0032",
):
if self.ta_info["UNB." + field]:
self.out.put({"BOTSID": "UNB", field: self.ta_info["UNB." + field]})
if testindicator:
self.out.put({"BOTSID": "UNB", "0035": testindicator})
self.out.put(
{"BOTSID": "UNB"}, {"BOTSID": "UNZ", "0036": self.ta_info["nrmessages"], "0020": self.ta_info["reference"]}
) # dummy segment; is not used
# user exit
botslib.tryrunscript(self.userscript, self.scriptname, "envelopecontent", ta_info=self.ta_info, out=self.out)
# convert the tree into segments; here only the UNB is written (first segment)
self.out.checkmessage(self.out.root, self.out.defmessage)
self.out.tree2records(self.out.root)
# start doing the actual writing:
tofile = botslib.opendata(self.ta_info["filename"], "wb", self.ta_info["charset"])
if write_una or self.ta_info["forceUNA"]:
tofile.write(
"UNA"
+ self.ta_info["sfield_sep"]
+ self.ta_info["field_sep"]
+ self.ta_info["decimaal"]
+ self.ta_info["escape"]
+ reserve
+ self.ta_info["record_sep"]
+ self.ta_info["add_crlfafterrecord_sep"]
)
tofile.write(self.out.record2string(self.out.records[0]))
self.writefilelist(tofile)
tofile.write(self.out.record2string(self.out.records[-1]))
tofile.close()
if self.ta_info["messagetype"][:6] != "CONTRL" and botslib.checkconfirmrules(
"ask-edifact-CONTRL",
idroute=self.ta_info["idroute"],
idchannel=self.ta_info["tochannel"],
topartner=self.ta_info["topartner"],
frompartner=self.ta_info["frompartner"],
editype=self.ta_info["editype"],
messagetype=self.ta_info["messagetype"],
):
self.ta_info["confirmtype"] = u"ask-edifact-CONTRL"
self.ta_info["confirmasked"] = True
def writefilelist(self, tofile):
for filename in self.ta_list:
fromfile = botslib.opendata(filename, "rb", self.ta_info["charset"])
shutil.copyfileobj(fromfile, tofile)
fromfile.close()
def run(self):
if not self.ta_info['topartner'] or not self.ta_info['frompartner']:
raise botslib.OutMessageError(
_(u'In enveloping "frompartner" or "topartner" unknown: "%(ta_info)s".'
), {'ta_info': self.ta_info})
self._openoutenvelope()
self.ta_info.update(self.out.ta_info)
botslib.tryrunscript(self.userscript,
self.scriptname,
'ta_infocontent',
ta_info=self.ta_info)
#prepare data for envelope
if botsglobal.ini.getboolean('settings',
'interchangecontrolperpartner', False):
self.ta_info['reference'] = unicode(
botslib.unique('stxcounter_' + self.ta_info['topartner']))
else:
self.ta_info['reference'] = unicode(
botslib.unique('stxcounter_' + self.ta_info['frompartner']))
#build the envelope segments (that is, the tree from which the segments will be generated)
self.out.put({
'BOTSID': 'STX',
'STDS1': self.ta_info['STX.STDS1'],
'STDS2': self.ta_info['STX.STDS2'],
'FROM.01': self.ta_info['frompartner'],
'UNTO.01': self.ta_info['topartner'],
'TRDT.01': botslib.strftime('%y%m%d'),
'TRDT.02': botslib.strftime('%H%M%S'),
'SNRF': self.ta_info['reference']
})
if self.ta_info['STX.FROM.02']:
self.out.put({
'BOTSID': 'STX',
'FROM.02': self.ta_info['STX.FROM.02']
})
if self.ta_info['STX.UNTO.02']:
self.out.put({
'BOTSID': 'STX',
'UNTO.02': self.ta_info['STX.UNTO.02']
})
if self.ta_info['STX.APRF']:
self.out.put({'BOTSID': 'STX', 'APRF': self.ta_info['STX.APRF']})
if self.ta_info['STX.PRCD']:
self.out.put({'BOTSID': 'STX', 'PRCD': self.ta_info['STX.PRCD']})
self.out.put({'BOTSID': 'STX'}, {
'BOTSID': 'END',
'NMST': self.ta_info['nrmessages']
}) #dummy segment; is not used
#user exit
botslib.tryrunscript(self.userscript,
self.scriptname,
'envelopecontent',
ta_info=self.ta_info,
out=self.out)
#convert the tree into segments; here only the STX is written (first segment)
self.out.checkmessage(self.out.root, self.out.defmessage)
self.out.checkforerrorlist()
self.out.tree2records(self.out.root)
#start doing the actual writing:
tofile = botslib.opendata(self.ta_info['filename'], 'wb',
self.ta_info['charset'])
tofile.write(self.out.record2string(self.out.lex_records[0:1]))
self.writefilelist(tofile)
tofile.write(self.out.record2string(self.out.lex_records[1:2]))
tofile.close()
def run(self):
self._openoutenvelope(self.ta_info['editype'],
self.ta_info['envelope'])
self.ta_info.update(self.out.ta_info)
botslib.tryrunscript(self.userscript,
self.scriptname,
'ta_infocontent',
ta_info=self.ta_info)
#version dependent enveloping
writeUNA = False
if self.ta_info['version'] < '4':
date = time.strftime('%y%m%d')
reserve = ' '
if self.ta_info['charset'] != 'UNOA':
writeUNA = True
else:
date = time.strftime('%Y%m%d')
reserve = self.ta_info['reserve']
if self.ta_info['charset'] not in ['UNOA', 'UNOB']:
writeUNA = True
#UNB counter is per sender or receiver
if botsglobal.ini.getboolean('settings',
'interchangecontrolperpartner', False):
self.ta_info['reference'] = str(
botslib.unique('unbcounter_' + self.ta_info['topartner']))
else:
self.ta_info['reference'] = str(
botslib.unique('unbcounter_' + self.ta_info['frompartner']))
#testindicator is more complex:
if self.ta_info['testindicator'] and self.ta_info[
'testindicator'] != '0': #first check value from ta; do not use default
testindicator = '1'
elif self.ta_info['UNB.0035'] != '0': #than check values from grammar
testindicator = '1'
else:
testindicator = ''
#build the envelope segments (that is, the tree from which the segments will be generated)
self.out.put({
'BOTSID': 'UNB',
'S001.0001': self.ta_info['charset'],
'S001.0002': self.ta_info['version'],
'S002.0004': self.ta_info['frompartner'],
'S003.0010': self.ta_info['topartner'],
'S004.0017': date,
'S004.0019': time.strftime('%H%M'),
'0020': self.ta_info['reference']
})
#the following fields are conditional; do not write these when empty string (separator compression does take empty strings into account)
if self.ta_info['UNB.S002.0007']:
self.out.put({
'BOTSID': 'UNB',
'S002.0007': self.ta_info['UNB.S002.0007']
})
if self.ta_info['UNB.S003.0007']:
self.out.put({
'BOTSID': 'UNB',
'S003.0007': self.ta_info['UNB.S003.0007']
})
if self.ta_info['UNB.0026']:
self.out.put({'BOTSID': 'UNB', '0026': self.ta_info['UNB.0026']})
if testindicator:
self.out.put({'BOTSID': 'UNB', '0035': testindicator})
self.out.put({'BOTSID': 'UNB'}, {
'BOTSID': 'UNZ',
'0036': self.ta_info['nrmessages'],
'0020': self.ta_info['reference']
}) #dummy segment; is not used
#user exit
botslib.tryrunscript(self.userscript,
self.scriptname,
'envelopecontent',
ta_info=self.ta_info,
out=self.out)
#convert the tree into segments; here only the UNB is written (first segment)
self.out.normalisetree(self.out.root)
self.out.tree2records(self.out.root)
#start doing the actual writing:
tofile = botslib.opendata(self.ta_info['filename'], 'wb',
self.ta_info['charset'])
if writeUNA or self.ta_info['forceUNA']:
tofile.write('UNA' + self.ta_info['sfield_sep'] +
self.ta_info['field_sep'] + self.ta_info['decimaal'] +
self.ta_info['escape'] + reserve +
self.ta_info['record_sep'] +
self.ta_info['add_crlfafterrecord_sep'])
tofile.write(self.out._record2string(self.out.records[0]))
self.writefilelist(tofile)
tofile.write(self.out._record2string(self.out.records[-1]))
tofile.close()
def mailbag(ta_from,endstatus,frommessagetype,**argv):
''' 2 main functions:
- recognizes and distuinguishes several edi types: x12 edifact tradacoms ('mailbag' in, correct editype out)
- split up interchanges (edifact, x12, tradacoms)
details:
- edifact, x12 and tradacoms can be can be mixed
- handle multiple UNA in one file, including different charsets.
- handle multiple ISA's with different separators in one file
in bots > 3.0.0 all mailbag, edifact, x12 and tradacoms go via mailbag.
'''
edifile = botslib.readdata(filename=ta_from.filename)
startpos = 0
nr_interchanges = 0
while (1):
found = HEADER.match(edifile[startpos:])
if found is None:
if edifile[startpos:].strip(string.whitespace+'\x1A\x00'): #there is content...but not valid
if nr_interchanges: #found interchanges, but remainder is not valid
raise botslib.InMessageError(_(u'[M50]: Found data not in a valid interchange at position %(pos)s.'),{'pos':startpos})
else: #no interchanges found, content is not a valid edifact/x12/tradacoms interchange
if frommessagetype == 'mailbag': #if indicated 'mailbag': guess if this is an xml file.....
sniffxml = edifile[:25]
sniffxml = sniffxml.lstrip(' \t\n\r\f\v\xFF\xFE\xEF\xBB\xBF\x00') #to find first ' real' data; some char are because of BOM, UTF-16 etc
if sniffxml and sniffxml[0] == '<':
#is a xml file; inmessage.py can determine the right xml messagetype via xpath.
filesize = len(edifile)
ta_to = ta_from.copyta(status=endstatus,statust=OK,filename=ta_from.filename,editype='xml',messagetype='mailbag',filesize=filesize)
return
raise botslib.InMessageError(_(u'[M51]: Edi file does not start with a valid interchange.'))
else: #no parseble content
if nr_interchanges: #OK: there are interchanges, but no new interchange is found.
return
else: #no edifact/x12/tradacoms envelope at all
raise botslib.InMessageError(_(u'[M52]: Edi file contains only whitespace.'))
elif found.group('x12'):
editype = 'x12'
headpos = startpos + found.start('x12')
#determine field_sep and record_sep
count = 0
for char in edifile[headpos:headpos+120]: #search first 120 characters to determine separators
if char in '\r\n' and count != 105:
continue
count += 1
if count == 4:
field_sep = char
elif count in [7,18,21,32,35,51,54,70]: #extra checks for fixed ISA.
if char != field_sep:
raise botslib.InMessageError(_(u'[M53]: Non-valid ISA header at position %(pos)s; position %(pos_element)s of ISA is "%(foundchar)s", expect here element separator "%(field_sep)s".'),
{'pos':headpos,'pos_element':unicode(count),'foundchar':char,'field_sep':field_sep})
elif count == 106:
record_sep = char
break
foundtrailer = re.search('''%(record_sep)s
\s*
I[\n\r]*E[\n\r]*A
.+?
%(record_sep)s
'''%{'record_sep':re.escape(record_sep)},
edifile[headpos:],re.DOTALL|re.VERBOSE)
if not foundtrailer:
foundtrailer2 = re.search('''%(record_sep)s
\s*
I[\n\r]*E[\n\r]*A
'''%{'record_sep':re.escape(record_sep)},
edifile[headpos:],re.DOTALL|re.VERBOSE)
if foundtrailer2:
raise botslib.InMessageError(_(u'[M60]: Found no segment terminator for IEA trailer at position %(pos)s.'),{'pos':foundtrailer2.start()})
else:
raise botslib.InMessageError(_(u'[M54]: Found no valid IEA trailer for the ISA header at position %(pos)s.'),{'pos':headpos})
elif found.group('edifact'):
editype = 'edifact'
headpos = startpos + found.start('edifact')
#parse UNA. valid UNA: UNA:+.? '
if found.group('UNA'):
count = 0
for char in found.group('UNAstring'):
if char in '\r\n':
continue
count += 1
if count == 2:
field_sep = char
elif count == 4:
escape = char
elif count == 6:
record_sep = char
if count != 6 and len(found.group('UNAstring').rstrip()) != 6:
raise botslib.InMessageError(_(u'[M55]: Non-valid UNA-segment at position %(pos)s. UNA-segment should be 6 positions.'),{'pos':headpos})
if found.group('field_sep') != field_sep:
raise botslib.InMessageError(_(u'[M56]: Data element separator used in edifact file differs from value indicated in UNA-segment.'))
else: #no UNA, interpret UNB
if found.group('field_sep') == '+':
record_sep = "'"
escape = '?'
elif found.group('field_sep') == '\x1D': #according to std this was preffered way...probably quite theoretic...but does no harm
record_sep = '\x1C'
escape = ''
else:
raise botslib.InMessageError(_(u'[M57]: Edifact file with non-standard separators. UNA segment should be used.'))
#search trailer
foundtrailer = re.search('''[^%(escape)s\n\r] #char that is not escape or cr/lf
[\n\r]*? #maybe some cr/lf's
%(record_sep)s #segment separator
\s* #whitespace between segments
U[\n\r]*N[\n\r]*Z #UNZ
.+? #any chars
[^%(escape)s\n\r] #char that is not escape or cr/lf
[\n\r]*? #maybe some cr/lf's
%(record_sep)s #segment separator
'''%{'escape':escape,'record_sep':re.escape(record_sep)},
edifile[headpos:],re.DOTALL|re.VERBOSE)
if not foundtrailer:
raise botslib.InMessageError(_(u'[M58]: Found no valid UNZ trailer for the UNB header at position %(pos)s.'),{'pos':headpos})
elif found.group('tradacoms'):
editype = 'tradacoms'
#~ field_sep = '=' #the tradacoms 'after-segment-tag-separator'
record_sep = "'"
escape = '?'
headpos = startpos + found.start('STX')
foundtrailer = re.search('''[^%(escape)s\n\r] #char that is not escape or cr/lf
[\n\r]*? #maybe some cr/lf's
%(record_sep)s #segment separator
\s* #whitespace between segments
E[\n\r]*N[\n\r]*D
.+?
[^%(escape)s\n\r] #char that is not escape or cr/lf
[\n\r]*? #maybe some cr/lf's
%(record_sep)s #segment separator
'''%{'escape':escape,'record_sep':re.escape(record_sep)},
edifile[headpos:],re.DOTALL|re.VERBOSE)
if not foundtrailer:
raise botslib.InMessageError(_(u'[M59]: Found no valid END trailer for the STX header at position %(pos)s.'),{'pos':headpos})
#so: found an interchange (from headerpos until endpos)
endpos = headpos + foundtrailer.end()
ta_to = ta_from.copyta(status=endstatus) #make transaction for translated message; gets ta_info of ta_frommes
tofilename = unicode(ta_to.idta)
filesize = len(edifile[headpos:endpos])
tofile = botslib.opendata(tofilename,'wb')
tofile.write(edifile[headpos:endpos])
tofile.close()
#editype is now either edifact, x12 or tradacoms
#frommessagetype is the original frommessagetype (from route).
#frommessagetype would normally be edifact, x12, tradacoms or mailbag, but could also be eg ORDERSD96AUNEAN007.
#If so, we want to preserve that.
if frommessagetype != 'mailbag' and frommessagetype != editype:
messagetype = frommessagetype
else:
messagetype = editype
ta_to.update(statust=OK,filename=tofilename,editype=editype,messagetype=messagetype,filesize=filesize) #update outmessage transaction with ta_info;
startpos = endpos
nr_interchanges += 1
botsglobal.logger.debug(_(u' File written: "%(tofilename)s".'),{'tofilename':tofilename})
def run(self):
if not self.ta_info['topartner'] or not self.ta_info['frompartner']:
raise botslib.OutMessageError(
_(u'In enveloping "frompartner" or "topartner" unknown: "%(ta_info)s".'
), {'ta_info': self.ta_info})
self._openoutenvelope()
self.ta_info.update(self.out.ta_info)
botslib.tryrunscript(self.userscript,
self.scriptname,
'ta_infocontent',
ta_info=self.ta_info)
#prepare data for envelope
isa09date = botslib.strftime('%y%m%d')
#test indicator can either be from configuration (self.ta_info['ISA15']) or by mapping (self.ta_info['testindicator'])
#mapping overrules.
if self.ta_info['testindicator'] and self.ta_info[
'testindicator'] != '0': #'0' is default value (in db)
testindicator = self.ta_info['testindicator']
else:
testindicator = self.ta_info['ISA15']
#~ print self.ta_info['messagetype'], 'grammar:',self.ta_info['ISA15'],'ta:',self.ta_info['testindicator'],'out:',testindicator
if botsglobal.ini.getboolean('settings',
'interchangecontrolperpartner', False):
self.ta_info['reference'] = unicode(
botslib.unique('isacounter_' + self.ta_info['topartner']))
else:
self.ta_info['reference'] = unicode(
botslib.unique('isacounter_' + self.ta_info['frompartner']))
#ISA06 and GS02 can be different; eg ISA06 is a service provider.
#ISA06 and GS02 can be in the syntax....
isa06sender = self.ta_info.get('ISA06', self.ta_info['frompartner'])
isa06sender = isa06sender.ljust(15) #add spaces; is fixed length
gs02sender = self.ta_info.get('GS02', self.ta_info['frompartner'])
#also for ISA08 and GS03
isa08receiver = self.ta_info.get('ISA08', self.ta_info['topartner'])
isa08receiver = isa08receiver.ljust(15) #add spaces; is fixed length
gs03receiver = self.ta_info.get('GS03', self.ta_info['topartner'])
#build the envelope segments (that is, the tree from which the segments will be generated)
self.out.put(
{
'BOTSID': 'ISA',
'ISA01': self.ta_info['ISA01'],
'ISA02': self.ta_info['ISA02'],
'ISA03': self.ta_info['ISA03'],
'ISA04': self.ta_info['ISA04'],
'ISA05': self.ta_info['ISA05'],
'ISA06': isa06sender,
'ISA07': self.ta_info['ISA07'],
'ISA08': isa08receiver,
'ISA09': isa09date,
'ISA10': botslib.strftime('%H%M'),
'ISA11': self.ta_info[
'ISA11'], #if ISA version > 00403, replaced by reprtion separator
'ISA12': self.ta_info['version'],
'ISA13': self.ta_info['reference'],
'ISA14': self.ta_info['ISA14'],
'ISA15': testindicator
},
strip=False
) #MIND: strip=False: ISA fields shoudl not be stripped as it is soemwhat like fixed-length
self.out.put({'BOTSID': 'ISA'}, {
'BOTSID': 'IEA',
'IEA01': '1',
'IEA02': self.ta_info['reference']
})
gs08messagetype = self.ta_info['messagetype'][3:]
if gs08messagetype[:6] < '004010':
gs04date = botslib.strftime('%y%m%d')
else:
gs04date = botslib.strftime('%Y%m%d')
self.out.put({'BOTSID': 'ISA'}, {
'BOTSID': 'GS',
'GS01': self.ta_info['functionalgroup'],
'GS02': gs02sender,
'GS03': gs03receiver,
'GS04': gs04date,
'GS05': botslib.strftime('%H%M'),
'GS06': self.ta_info['reference'],
'GS07': self.ta_info['GS07'],
'GS08': gs08messagetype
})
self.out.put({'BOTSID': 'ISA'}, {'BOTSID': 'GS'}, {
'BOTSID': 'GE',
'GE01': self.ta_info['nrmessages'],
'GE02': self.ta_info['reference']
}) #dummy segment; is not used
#user exit
botslib.tryrunscript(self.userscript,
self.scriptname,
'envelopecontent',
ta_info=self.ta_info,
out=self.out)
#convert the tree into segments; here only the UNB is written (first segment)
self.out.checkmessage(self.out.root, self.out.defmessage)
self.out.checkforerrorlist()
self.out.tree2records(self.out.root)
#start doing the actual writing:
tofile = botslib.opendata(self.ta_info['filename'], 'wb',
self.ta_info['charset'])
isa_string = self.out.record2string(self.out.lex_records[0:1])
#ISA has the used separators at certain positions. Normally bots would give errors for this (can not use sep as data) or compress these aways. So this is hardcoded.
if self.ta_info['version'] < '00403':
isa_string = isa_string[:103] + self.ta_info[
'field_sep'] + self.ta_info['sfield_sep'] + isa_string[103:]
else:
isa_string = isa_string[:82] + self.ta_info[
'reserve'] + isa_string[83:103] + self.ta_info[
'field_sep'] + self.ta_info['sfield_sep'] + isa_string[103:]
tofile.write(isa_string) #write ISA
tofile.write(self.out.record2string(
self.out.lex_records[1:2])) #write GS
self.writefilelist(tofile)
tofile.write(self.out.record2string(
self.out.lex_records[2:])) #write GE and IEA
tofile.close()
def run(self):
if not self.ta_info['topartner'] or not self.ta_info['frompartner']:
raise botslib.OutMessageError(
_(u'In enveloping "frompartner" or "topartner" unknown: "%(ta_info)s".'
), {'ta_info': self.ta_info})
self._openoutenvelope()
self.ta_info.update(self.out.ta_info)
botslib.tryrunscript(self.userscript,
self.scriptname,
'ta_infocontent',
ta_info=self.ta_info)
#version dependent enveloping
if self.ta_info['version'] < '4':
date = botslib.strftime('%y%m%d')
reserve = ' '
else:
date = botslib.strftime('%Y%m%d')
reserve = self.ta_info['reserve']
#UNB reference is counter is per sender or receiver
if botsglobal.ini.getboolean('settings',
'interchangecontrolperpartner', False):
self.ta_info['reference'] = unicode(
botslib.unique('unbcounter_' + self.ta_info['topartner']))
else:
self.ta_info['reference'] = unicode(
botslib.unique('unbcounter_' + self.ta_info['frompartner']))
#testindicator is more complex:
if self.ta_info['testindicator'] and self.ta_info[
'testindicator'] != '0': #first check value from ta; do not use default
testindicator = '1'
elif self.ta_info['UNB.0035'] != '0': #than check values from grammar
testindicator = '1'
else:
testindicator = ''
#build the envelope segments (that is, the tree from which the segments will be generated)
self.out.put({
'BOTSID': 'UNB',
'S001.0001': self.ta_info['charset'],
'S001.0002': self.ta_info['version'],
'S002.0004': self.ta_info['frompartner'],
'S003.0010': self.ta_info['topartner'],
'S004.0017': date,
'S004.0019': botslib.strftime('%H%M'),
'0020': self.ta_info['reference']
})
#the following fields are conditional; do not write these when empty string (separator compression does take empty strings into account)
for field in ('S001.0080', 'S001.0133', 'S002.0007', 'S002.0008',
'S002.0042', 'S003.0007', 'S003.0014', 'S003.0046',
'S005.0022', 'S005.0025', '0026', '0029', '0031',
'0032'):
if self.ta_info['UNB.' + field]:
self.out.put({
'BOTSID': 'UNB',
field: self.ta_info['UNB.' + field]
})
if testindicator:
self.out.put({'BOTSID': 'UNB', '0035': testindicator})
self.out.put({'BOTSID': 'UNB'}, {
'BOTSID': 'UNZ',
'0036': self.ta_info['nrmessages'],
'0020': self.ta_info['reference']
}) #dummy segment; is not used
#user exit
botslib.tryrunscript(self.userscript,
self.scriptname,
'envelopecontent',
ta_info=self.ta_info,
out=self.out)
#convert the tree into segments; here only the UNB is written (first segment)
self.out.checkmessage(self.out.root, self.out.defmessage)
self.out.checkforerrorlist()
self.out.tree2records(self.out.root)
#start doing the actual writing:
tofile = botslib.opendata(self.ta_info['filename'], 'wb',
self.ta_info['charset'])
if self.ta_info['forceUNA'] or self.ta_info['charset'] != 'UNOA':
tofile.write('UNA' + self.ta_info['sfield_sep'] +
self.ta_info['field_sep'] + self.ta_info['decimaal'] +
self.ta_info['escape'] + reserve +
self.ta_info['record_sep'] +
self.ta_info['add_crlfafterrecord_sep'])
tofile.write(self.out.record2string(self.out.lex_records[0:1]))
self.writefilelist(tofile)
tofile.write(self.out.record2string(self.out.lex_records[1:2]))
tofile.close()
def run(self):
if not self.ta_info["topartner"] or not self.ta_info["frompartner"]:
raise botslib.OutMessageError(
_(u'In enveloping "frompartner" or "topartner" unknown: "$ta_info".'), ta_info=self.ta_info
)
self._openoutenvelope(self.ta_info["editype"], self.ta_info["envelope"])
self.ta_info.update(self.out.ta_info)
# need to know the functionalgroup code:
defmessage = grammar.grammarread(self.ta_info["editype"], self.ta_info["messagetype"])
self.ta_info["functionalgroup"] = defmessage.syntax["functionalgroup"]
botslib.tryrunscript(self.userscript, self.scriptname, "ta_infocontent", ta_info=self.ta_info)
# prepare data for envelope
isa09date = time.strftime("%y%m%d")
# test indicator can either be from configuration (self.ta_info['ISA15']) or by mapping (self.ta_info['testindicator'])
# mapping overrules.
if self.ta_info["testindicator"] and self.ta_info["testindicator"] != "0": #'0' is default value (in db)
testindicator = self.ta_info["testindicator"]
else:
testindicator = self.ta_info["ISA15"]
# ~ print self.ta_info['messagetype'], 'grammar:',self.ta_info['ISA15'],'ta:',self.ta_info['testindicator'],'out:',testindicator
if botsglobal.ini.getboolean("settings", "interchangecontrolperpartner", False):
self.ta_info["reference"] = str(botslib.unique("isacounter_" + self.ta_info["topartner"]))
else:
self.ta_info["reference"] = str(botslib.unique("isacounter_" + self.ta_info["frompartner"]))
# ISA06 and GS02 can be different; eg ISA06 is a service provider.
# ISA06 and GS02 can be in the syntax....
isa06sender = self.ta_info.get("ISA06", self.ta_info["frompartner"])
isa06sender = isa06sender.ljust(15) # add spaces; is fixed length
gs02sender = self.ta_info.get("GS02", self.ta_info["frompartner"])
# also for ISA08 and GS03
isa08receiver = self.ta_info.get("ISA08", self.ta_info["topartner"])
isa08receiver = isa08receiver.ljust(15) # add spaces; is fixed length
gs03receiver = self.ta_info.get("GS03", self.ta_info["topartner"])
# build the envelope segments (that is, the tree from which the segments will be generated)
self.out.put(
{
"BOTSID": "ISA",
"ISA01": self.ta_info["ISA01"],
"ISA02": self.ta_info["ISA02"],
"ISA03": self.ta_info["ISA03"],
"ISA04": self.ta_info["ISA04"],
"ISA05": self.ta_info["ISA05"],
"ISA06": isa06sender,
"ISA07": self.ta_info["ISA07"],
"ISA08": isa08receiver,
"ISA09": isa09date,
"ISA10": time.strftime("%H%M"),
"ISA11": self.ta_info["ISA11"], # if ISA version > 00403, replaced by reprtion separator
"ISA12": self.ta_info["version"],
"ISA13": self.ta_info["reference"],
"ISA14": self.ta_info["ISA14"],
"ISA15": testindicator,
},
strip=False,
) # MIND: strip=False: ISA fields shoudl not be stripped as it is soemwhat like fixed-length
self.out.put({"BOTSID": "ISA"}, {"BOTSID": "IEA", "IEA01": "1", "IEA02": self.ta_info["reference"]})
gs08messagetype = self.ta_info["messagetype"][3:]
if gs08messagetype[:6] < "004010":
gs04date = time.strftime("%y%m%d")
else:
gs04date = time.strftime("%Y%m%d")
self.out.put(
{"BOTSID": "ISA"},
{
"BOTSID": "GS",
"GS01": self.ta_info["functionalgroup"],
"GS02": gs02sender,
"GS03": gs03receiver,
"GS04": gs04date,
"GS05": time.strftime("%H%M"),
"GS06": self.ta_info["reference"],
"GS07": self.ta_info["GS07"],
"GS08": gs08messagetype,
},
)
self.out.put(
{"BOTSID": "ISA"},
{"BOTSID": "GS"},
{"BOTSID": "GE", "GE01": self.ta_info["nrmessages"], "GE02": self.ta_info["reference"]},
) # dummy segment; is not used
# user exit
botslib.tryrunscript(self.userscript, self.scriptname, "envelopecontent", ta_info=self.ta_info, out=self.out)
# convert the tree into segments; here only the UNB is written (first segment)
self.out.checkmessage(self.out.root, self.out.defmessage)
self.out.tree2records(self.out.root)
# start doing the actual writing:
tofile = botslib.opendata(self.ta_info["filename"], "wb", self.ta_info["charset"])
isa_string = self.out.record2string(self.out.records[0])
if self.ta_info["version"] < "00403":
isa_string = (
isa_string[:103] + self.ta_info["field_sep"] + self.ta_info["sfield_sep"] + isa_string[103:]
) # hack for strange characters at end of ISA; hardcoded
else:
isa_string = (
isa_string[:82]
+ self.ta_info["reserve"]
+ isa_string[83:103]
+ self.ta_info["field_sep"]
+ self.ta_info["sfield_sep"]
+ isa_string[103:]
) # hack for strange characters at end of ISA; hardcoded
tofile.write(isa_string) # write ISA
tofile.write(self.out.record2string(self.out.records[1])) # write GS
self.writefilelist(tofile)
tofile.write(self.out.record2string(self.out.records[-2])) # write GE
tofile.write(self.out.record2string(self.out.records[-1])) # write IEA
tofile.close()
if self.ta_info["functionalgroup"] != "FA" and botslib.checkconfirmrules(
"ask-x12-997",
idroute=self.ta_info["idroute"],
idchannel=self.ta_info["tochannel"],
topartner=self.ta_info["topartner"],
frompartner=self.ta_info["frompartner"],
editype=self.ta_info["editype"],
messagetype=self.ta_info["messagetype"],
):
self.ta_info["confirmtype"] = u"ask-x12-997"
self.ta_info["confirmasked"] = True
def splitmailbag(startstatus=MAILBAG,endstatus=TRANSLATE,idroute=''):
''' splits 'mailbag'files to seperate files each containging one interchange (ISA-IEA or UNA/UNB-UNZ).
handles x12 and edifact; these can be mixed.
'''
header = re.compile('(\s*(ISA))|(\s*(UNA.{6})?\s*(U\s*N\s*B)s*.{1}(.{4}).{1}(.{1}))',re.DOTALL)
# group: 1 2 3 4 5 6 7
for row in botslib.query(u'''SELECT idta,filename,charset
FROM ta
WHERE idta>%(rootidta)s
AND status=%(status)s
AND statust=%(statust)s
AND idroute=%(idroute)s
''',
{'status':startstatus,'statust':OK,'idroute':idroute,'rootidta':botslib.get_minta4query()}):
try:
ta_org=botslib.OldTransaction(row['idta'])
ta_intermediate = ta_org.copyta(status=MAILBAGPARSED)
edifile = botslib.readdata(filename=row['filename']) #read as binary...
botsglobal.logmap.debug(u'Start parsing mailbag file "%s".',row['filename'])
startpos=0
while (1):
found = header.search(edifile[startpos:])
if found is None:
if startpos: #ISA/UNB have been found in file; no new ISA/UNB is found. So all processing is done.
break
#guess if this is an xml file.....
sniffxml = edifile[:25]
sniffxml = sniffxml.lstrip(' \t\n\r\f\v\xFF\xFE\xEF\xBB\xBF\x00') #to find first ' real' data; some char are because of BOM, UTF-16 etc
if sniffxml and sniffxml[0]=='<':
ta_tomes=ta_intermediate.copyta(status=endstatus) #make transaction for translated message; gets ta_info of ta_frommes
ta_tomes.update(status=STATUSTMP,statust=OK,filename=row['filename'],editype='xml') #update outmessage transaction with ta_info;
break;
else:
raise botslib.InMessageError(_(u'Found no content in mailbag.'))
elif found.group(1):
editype='x12'
headpos=startpos+ found.start(2)
count=0
for c in edifile[headpos:headpos+120]: #search first 120 characters to find seperators
if c in '\r\n' and count!=105:
continue
count +=1
if count==4:
field_sep = c
elif count==106:
record_sep = c
break
foundtrailer = re.search(re.escape(record_sep)+'\s*IEA'+re.escape(field_sep)+'.+?'+re.escape(record_sep),edifile[headpos:],re.DOTALL)
elif found.group(3):
editype='edifact'
if found.group(4):
field_sep = edifile[startpos + found.start(4) + 4]
record_sep = edifile[startpos + found.start(4) + 8]
headpos=startpos+ found.start(4)
else:
field_sep = '+'
record_sep = "'"
headpos=startpos+ found.start(5)
foundtrailer = re.search(re.escape(record_sep)+'\s*U\s*N\s*Z\s*'+re.escape(field_sep)+'.+?'+re.escape(record_sep),edifile[headpos:],re.DOTALL)
if not foundtrailer:
raise botslib.InMessageError(_(u'Found no valid envelope trailer in mailbag.'))
endpos = headpos+foundtrailer.end()
#so: interchange is from headerpos untill endpos
#~ if header.search(edifile[headpos+25:endpos]): #check if there is another header in the interchange
#~ raise botslib.InMessageError(u'Error in mailbag format: found no valid envelope trailer.')
ta_tomes=ta_intermediate.copyta(status=endstatus) #make transaction for translated message; gets ta_info of ta_frommes
tofilename = str(ta_tomes.idta)
tofile = botslib.opendata(tofilename,'wb')
tofile.write(edifile[headpos:endpos])
tofile.close()
ta_tomes.update(status=STATUSTMP,statust=OK,filename=tofilename,editype=editype,messagetype=editype) #update outmessage transaction with ta_info;
startpos=endpos
except:
txt=botslib.txtexc()
ta_intermediate.failure()
ta_intermediate.update(statust=ERROR,errortext=txt)
else:
botsglobal.logmap.debug(u'OK Parsing mailbag file "%s".',row['filename'])
ta_org.update(statust=DONE)
ta_intermediate.succes(endstatus)
ta_intermediate.update(statust=DONE)
