def run(self):
''' class for template enveloping; delevers a valid html-file.
Uses a kid-template for the enveloping/merging.
use kid to write; no envelope grammar is used
#20120101 depreciated. use class templatehtml
'''
try:
import kid
except:
raise ImportError(_(u'Dependency failure: editype "template" requires python library "kid".'))
self._openoutenvelope()
self.ta_info.update(self.out.ta_info)
botslib.tryrunscript(self.userscript,self.scriptname,'ta_infocontent',ta_info=self.ta_info)
if not self.ta_info['envelope-template']:
raise botslib.OutMessageError(_(u'While enveloping in "%(editype)s.%(messagetype)s": syntax option "envelope-template" not filled; is required.'),
self.ta_info)
templatefile = botslib.abspath('templates',self.ta_info['envelope-template'])
ta_list = self.filelist2absolutepaths()
try:
botsglobal.logger.debug(u'Start writing envelope to file "%(filename)s".',self.ta_info)
ediprint = kid.Template(file=templatefile, data=ta_list) #init template; pass list with filenames
except:
txt = botslib.txtexc()
raise botslib.OutMessageError(_(u'While enveloping in "%(editype)s.%(messagetype)s", error:\n%(txt)s'),
{'editype':self.ta_info['editype'],'messagetype':self.ta_info['messagetype'],'txt':txt})
try:
filehandler = botslib.opendata(self.ta_info['filename'],'wb')
ediprint.write(filehandler,
encoding=self.ta_info['charset'],
output=self.ta_info['output'])
except:
txt = botslib.txtexc()
raise botslib.OutMessageError(_(u'While enveloping in "%(editype)s.%(messagetype)s", error:\n%(txt)s'),
{'editype':self.ta_info['editype'],'messagetype':self.ta_info['messagetype'],'txt':txt})
def cleanup(do_cleanup_parameter,userscript,scriptname):
''' public function, does all cleanup of the database and file system.
most cleanup functions are by default done only once a day.
'''
if botsglobal.ini.getboolean('acceptance','runacceptancetest',False): # no cleanup during acceptance testing
return
whencleanup = botsglobal.ini.get('settings','whencleanup','daily')
if do_cleanup_parameter: #if explicit indicated via commandline parameter
do_full_cleanup = True
elif whencleanup in ['always','daily']:
#perform full cleanup only first run of the day.
cur_day = int(time.strftime('%Y%m%d')) #get current date, convert to int
if cur_day != botslib.unique('bots_cleanup_day',updatewith=cur_day):
do_full_cleanup = True
else:
do_full_cleanup = False
else:
do_full_cleanup = False
try:
if do_full_cleanup:
botsglobal.logger.info(u'Cleanup files')
_cleandatafile()
_cleanarchive()
botsglobal.logger.info(u'Cleanup database')
_cleanupsession()
_cleanpersist()
_cleantransactions()
botsglobal.logger.info(u'Vacuum database')
_vacuum()
# postcleanup user exit in botsengine script
botslib.tryrunscript(userscript,scriptname,'postcleanup',whencleanup=whencleanup)
botsglobal.logger.info(u'Done full cleanup.')
_cleanrunsnothingreceived() #do this every run, but not logged
except:
botsglobal.logger.exception(u'Cleanup error.')
def run(self):
try:
from genshi.template import TemplateLoader
except:
raise ImportError(u'Dependency failure: editype "templatehtml" requires python library "genshi".')
self._openoutenvelope()
self.ta_info.update(self.out.ta_info)
botslib.tryrunscript(self.userscript,self.scriptname,'ta_infocontent',ta_info=self.ta_info)
if not self.ta_info['envelope-template']:
raise botslib.OutMessageError(_(u'While enveloping in "%(editype)s.%(messagetype)s": syntax option "envelope-template" not filled; is required.'),
self.ta_info)
templatefile = botslib.abspath(self.__class__.__name__,self.ta_info['envelope-template'])
ta_list = self.filelist2absolutepaths()
try:
botsglobal.logger.debug(u'Start writing envelope to file "%(filename)s".',self.ta_info)
loader = TemplateLoader(auto_reload=False)
tmpl = loader.load(templatefile)
except:
txt = botslib.txtexc()
raise botslib.OutMessageError(_(u'While enveloping in "%(editype)s.%(messagetype)s", error:\n%(txt)s'),
{'editype':self.ta_info['editype'],'messagetype':self.ta_info['messagetype'],'txt':txt})
try:
filehandler = botslib.opendata(self.ta_info['filename'],'wb')
stream = tmpl.generate(data=ta_list)
stream.render(method='xhtml',encoding=self.ta_info['charset'],out=filehandler)
except:
txt = botslib.txtexc()
raise botslib.OutMessageError(_(u'While enveloping in "%(editype)s.%(messagetype)s", error:\n%(txt)s'),
{'editype':self.ta_info['editype'],'messagetype':self.ta_info['messagetype'],'txt':txt})
def run(self):
''' class for (test) orderprint; delevers a valid html-file.
Uses a kid-template for the enveloping/merging.
use kid to write; no envelope grammar is used
'''
import kid
defmessage = grammar.grammarread(self.ta_info['editype'],self.ta_info['messagetype']) #needed beause we do not know envelope; read syntax for editype/messagetype
self.ta_info.update(defmessage.syntax)
botslib.tryrunscript(self.userscript,self.scriptname,'ta_infocontent',ta_info=self.ta_info)
if not self.ta_info['envelope-template']:
raise botslib.OutMessageError(_(u'While enveloping in "$editype.$messagetype": syntax option "envelope-template" not filled; is required.'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'])
templatefile = botslib.abspath('templates',self.ta_info['envelope-template'])
ta_list = self.filelist2absolutepaths()
try:
botsglobal.logger.debug(u'Start writing envelope to file "%s".',self.ta_info['filename'])
ediprint = kid.Template(file=templatefile, data=ta_list) #init template; pass list with filenames
except:
txt=botslib.txtexc()
raise botslib.OutMessageError(_(u'While enveloping in "$editype.$messagetype": $txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt)
try:
f = botslib.opendata(self.ta_info['filename'],'wb')
ediprint.write(f,
encoding=self.ta_info['charset'],
output=self.ta_info['output'])
except:
txt=botslib.txtexc()
raise botslib.OutMessageError(_(u'While enveloping error in "$editype.$messagetype"; probably in html file(s) to be merged: $txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt)
def run(self):
''' class for (test) orderprint; delevers a valid html-file.
Uses a kid-template for the enveloping/merging.
use kid to write; no envelope grammar is used
'''
try:
from genshi.template import TemplateLoader
except:
txt=botslib.txtexc()
raise ImportError(_(u'Dependency failure: editype "template" requires python library "genshi". Error:\n%s'%txt))
defmessage = grammar.grammarread(self.ta_info['editype'],self.ta_info['messagetype']) #needed because we do not know envelope; read syntax for editype/messagetype
self.ta_info.update(defmessage.syntax)
botslib.tryrunscript(self.userscript,self.scriptname,'ta_infocontent',ta_info=self.ta_info)
if not self.ta_info['envelope-template']:
raise botslib.OutMessageError(_(u'While enveloping in "$editype.$messagetype": syntax option "envelope-template" not filled; is required.'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'])
templatefile = botslib.abspath('templateshtml',self.ta_info['envelope-template'])
ta_list = self.filelist2absolutepaths()
try:
botsglobal.logger.debug(u'Start writing envelope to file "%s".',self.ta_info['filename'])
loader = TemplateLoader(auto_reload=False)
tmpl = loader.load(templatefile)
except:
txt=botslib.txtexc()
raise botslib.OutMessageError(_(u'While enveloping in "$editype.$messagetype", error:\n$txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt)
try:
f = botslib.opendata(self.ta_info['filename'],'wb')
stream = tmpl.generate(data=ta_list)
stream.render(method='xhtml',encoding=self.ta_info['charset'],out=f)
except:
txt=botslib.txtexc()
raise botslib.OutMessageError(_(u'While enveloping in "$editype.$messagetype", error:\n$txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt)
def run(self):
botslib.tryrunscript(self.userscript,self.scriptname,'ta_infocontent',ta_info=self.ta_info)
if len(self.ta_list) > 1:
tofile = botslib.opendata(self.ta_info['filename'],'wb',self.ta_info['charset'])
self.writefilelist(tofile)
tofile.close()
else:
self.ta_info['filename'] = self.ta_list[0]
def run(self):
''' class for (test) orderprint; delevers a valid html-file.
Uses a kid-template for the enveloping/merging.
use kid to write; no envelope grammar is used
'''
try:
import kid
except:
txt = botslib.txtexc()
raise ImportError(
_(u'Dependency failure: editype "template" requires python library "kid". Error:\n%s'
% txt))
defmessage = grammar.grammarread(
self.ta_info['editype'], self.ta_info['messagetype']
) #needed because we do not know envelope; read syntax for editype/messagetype
self.ta_info.update(defmessage.syntax)
botslib.tryrunscript(self.userscript,
self.scriptname,
'ta_infocontent',
ta_info=self.ta_info)
if not self.ta_info['envelope-template']:
raise botslib.OutMessageError(_(
u'While enveloping in "$editype.$messagetype": syntax option "envelope-template" not filled; is required.'
),
editype=self.ta_info['editype'],
messagetype=self.
ta_info['messagetype'])
templatefile = botslib.abspath('templates',
self.ta_info['envelope-template'])
ta_list = self.filelist2absolutepaths()
try:
botsglobal.logger.debug(u'Start writing envelope to file "%s".',
self.ta_info['filename'])
ediprint = kid.Template(
file=templatefile,
data=ta_list) #init template; pass list with filenames
except:
txt = botslib.txtexc()
raise botslib.OutMessageError(
_(u'While enveloping in "$editype.$messagetype", error:\n$txt'
),
editype=self.ta_info['editype'],
messagetype=self.ta_info['messagetype'],
txt=txt)
try:
filehandler = botslib.opendata(self.ta_info['filename'], 'wb')
ediprint.write(filehandler,
encoding=self.ta_info['charset'],
output=self.ta_info['output'])
except:
txt = botslib.txtexc()
raise botslib.OutMessageError(
_(u'While enveloping in "$editype.$messagetype", error:\n$txt'
),
editype=self.ta_info['editype'],
messagetype=self.ta_info['messagetype'],
txt=txt)
def run(self):
botslib.tryrunscript(self.userscript,
self.scriptname,
'ta_infocontent',
ta_info=self.ta_info)
tofile = botslib.opendata(self.ta_info['filename'], 'wb',
self.ta_info['charset'])
self.writefilelist(tofile)
tofile.close()
def run(self):
''' class for (test) orderprint; delevers a valid html-file.
Uses a kid-template for the enveloping/merging.
use kid to write; no envelope grammar is used
'''
try:
from genshi.template import TemplateLoader
except:
txt = botslib.txtexc()
raise ImportError(
_(u'Dependency failure: editype "template" requires python library "genshi". Error:\n%s'
% txt))
defmessage = grammar.grammarread(
self.ta_info['editype'], self.ta_info['messagetype']
) #needed because we do not know envelope; read syntax for editype/messagetype
self.ta_info.update(defmessage.syntax)
botslib.tryrunscript(self.userscript,
self.scriptname,
'ta_infocontent',
ta_info=self.ta_info)
if not self.ta_info['envelope-template']:
raise botslib.OutMessageError(_(
u'While enveloping in "$editype.$messagetype": syntax option "envelope-template" not filled; is required.'
),
editype=self.ta_info['editype'],
messagetype=self.
ta_info['messagetype'])
templatefile = botslib.abspath('templateshtml',
self.ta_info['envelope-template'])
ta_list = self.filelist2absolutepaths()
try:
botsglobal.logger.debug(u'Start writing envelope to file "%s".',
self.ta_info['filename'])
loader = TemplateLoader(auto_reload=False)
tmpl = loader.load(templatefile)
except:
txt = botslib.txtexc()
raise botslib.OutMessageError(
_(u'While enveloping in "$editype.$messagetype", error:\n$txt'
),
editype=self.ta_info['editype'],
messagetype=self.ta_info['messagetype'],
txt=txt)
try:
filehandler = botslib.opendata(self.ta_info['filename'], 'wb')
stream = tmpl.generate(data=ta_list)
stream.render(method='xhtml',
encoding=self.ta_info['charset'],
out=filehandler)
except:
txt = botslib.txtexc()
raise botslib.OutMessageError(
_(u'While enveloping in "$editype.$messagetype", error:\n$txt'
),
editype=self.ta_info['editype'],
messagetype=self.ta_info['messagetype'],
txt=txt)
def run(self):
self._openoutenvelope(self.ta_info["editype"], self.ta_info["messagetype"])
botslib.tryrunscript(self.userscript, self.scriptname, "ta_infocontent", ta_info=self.ta_info)
# self.ta_info is not overwritten
tofile = botslib.opendata(self.ta_info["filename"], "wb", self.ta_info["charset"])
headers = dict([(field[ID], field[ID]) for field in self.out.defmessage.structure[0][FIELDS]])
self.out.put(headers)
self.out.tree2records(self.out.root)
tofile.write(self.out.record2string(self.out.records[0]))
self.writefilelist(tofile)
tofile.close()
def run(self):
self._openoutenvelope(self.ta_info['editype'],self.ta_info['messagetype'])
botslib.tryrunscript(self.userscript,self.scriptname,'ta_infocontent',ta_info=self.ta_info)
#self.ta_info is not overwritten
tofile = botslib.opendata(self.ta_info['filename'],'wb',self.ta_info['charset'])
if len(self.out.defmessage.recorddefs) == 1 : #if only only type of record
headers = dict([(field[ID],field[ID]) for field in self.out.defmessage.structure[0][FIELDS]])
self.out.put(headers)
self.out.tree2records(self.out.root)
tofile.write(self.out._record2string(self.out.records[0]))
self.writefilelist(tofile)
tofile.close()
def run(self):
''' class for (test) xml envelope. There is no standardised XML-envelope!
writes a new XML-tree; uses places-holders for XML-files to include; real enveloping is done by ElementTree's include'''
include = '{http://www.w3.org/2001/XInclude}include'
self._openoutenvelope(self.ta_info['editype'],self.ta_info['envelope'])
botslib.tryrunscript(self.userscript,self.scriptname,'ta_infocontent',ta_info=self.ta_info)
#~ self.out.put({'BOTSID':'root','xmlns:xi':"http://www.w3.org/2001/XInclude"}) #works, but attribute is not removed bij ETI.include
self.out.put({'BOTSID':'root'}) #start filling out-tree
ta_list = self.filelist2absolutepaths()
for filename in ta_list:
self.out.put({'BOTSID':'root'},{'BOTSID':include,include + '__parse':'xml',include + '__href':filename})
self.out.envelopewrite(self.out.root) #'resolves' the included xml files
def run(self):
try:
from genshi.template import TemplateLoader
except:
raise ImportError(
u'Dependency failure: editype "templatehtml" requires python library "genshi".'
)
self._openoutenvelope()
self.ta_info.update(self.out.ta_info)
botslib.tryrunscript(self.userscript,
self.scriptname,
'ta_infocontent',
ta_info=self.ta_info)
if not self.ta_info['envelope-template']:
raise botslib.OutMessageError(
_(u'While enveloping in "%(editype)s.%(messagetype)s": syntax option "envelope-template" not filled; is required.'
), self.ta_info)
templatefile = botslib.abspath(self.__class__.__name__,
self.ta_info['envelope-template'])
ta_list = self.filelist2absolutepaths()
try:
botsglobal.logger.debug(
u'Start writing envelope to file "%(filename)s".',
self.ta_info)
loader = TemplateLoader(auto_reload=False)
tmpl = loader.load(templatefile)
except:
txt = botslib.txtexc()
raise botslib.OutMessageError(
_(u'While enveloping in "%(editype)s.%(messagetype)s", error:\n%(txt)s'
), {
'editype': self.ta_info['editype'],
'messagetype': self.ta_info['messagetype'],
'txt': txt
})
try:
filehandler = botslib.opendata(self.ta_info['filename'], 'wb')
stream = tmpl.generate(data=ta_list)
stream.render(method='xhtml',
encoding=self.ta_info['charset'],
out=filehandler)
except:
txt = botslib.txtexc()
raise botslib.OutMessageError(
_(u'While enveloping in "%(editype)s.%(messagetype)s", error:\n%(txt)s'
), {
'editype': self.ta_info['editype'],
'messagetype': self.ta_info['messagetype'],
'txt': txt
})
def run(self):
if not self.ta_info["topartner"] or not self.ta_info["frompartner"]:
raise botslib.OutMessageError(
_(u'In enveloping "frompartner" or "topartner" unknown: "$ta_info".'), ta_info=self.ta_info
)
self._openoutenvelope(self.ta_info["editype"], self.ta_info["envelope"])
self.ta_info.update(self.out.ta_info)
botslib.tryrunscript(self.userscript, self.scriptname, "ta_infocontent", ta_info=self.ta_info)
# prepare data for envelope
if botsglobal.ini.getboolean("settings", "interchangecontrolperpartner", False):
self.ta_info["reference"] = str(botslib.unique("stxcounter_" + self.ta_info["topartner"]))
else:
self.ta_info["reference"] = str(botslib.unique("stxcounter_" + self.ta_info["frompartner"]))
# build the envelope segments (that is, the tree from which the segments will be generated)
self.out.put(
{
"BOTSID": "STX",
"STDS1": self.ta_info["STX.STDS1"],
"STDS2": self.ta_info["STX.STDS2"],
"FROM.01": self.ta_info["frompartner"],
"UNTO.01": self.ta_info["topartner"],
"TRDT.01": time.strftime("%y%m%d"),
"TRDT.02": time.strftime("%H%M%S"),
"SNRF": self.ta_info["reference"],
}
)
if self.ta_info["STX.FROM.02"]:
self.out.put({"BOTSID": "STX", "FROM.02": self.ta_info["STX.FROM.02"]})
if self.ta_info["STX.UNTO.02"]:
self.out.put({"BOTSID": "STX", "UNTO.02": self.ta_info["STX.UNTO.02"]})
if self.ta_info["STX.APRF"]:
self.out.put({"BOTSID": "STX", "APRF": self.ta_info["STX.APRF"]})
if self.ta_info["STX.PRCD"]:
self.out.put({"BOTSID": "STX", "PRCD": self.ta_info["STX.PRCD"]})
self.out.put(
{"BOTSID": "STX"}, {"BOTSID": "END", "NMST": self.ta_info["nrmessages"]}
) # dummy segment; is not used
# user exit
botslib.tryrunscript(self.userscript, self.scriptname, "envelopecontent", ta_info=self.ta_info, out=self.out)
# convert the tree into segments; here only the STX is written (first segment)
self.out.checkmessage(self.out.root, self.out.defmessage)
self.out.tree2records(self.out.root)
# start doing the actual writing:
tofile = botslib.opendata(self.ta_info["filename"], "wb", self.ta_info["charset"])
tofile.write(self.out.record2string(self.out.records[0]))
self.writefilelist(tofile)
tofile.write(self.out.record2string(self.out.records[-1]))
tofile.close()
def run(self):
""" class for (test) orderprint; delevers a valid html-file.
Uses a kid-template for the enveloping/merging.
use kid to write; no envelope grammar is used
"""
try:
import kid
except:
txt = botslib.txtexc()
raise ImportError(
_(u'Dependency failure: editype "template" requires python library "kid". Error:\n%s' % txt)
)
defmessage = grammar.grammarread(
self.ta_info["editype"], self.ta_info["messagetype"]
) # needed because we do not know envelope; read syntax for editype/messagetype
self.ta_info.update(defmessage.syntax)
botslib.tryrunscript(self.userscript, self.scriptname, "ta_infocontent", ta_info=self.ta_info)
if not self.ta_info["envelope-template"]:
raise botslib.OutMessageError(
_(
u'While enveloping in "$editype.$messagetype": syntax option "envelope-template" not filled; is required.'
),
editype=self.ta_info["editype"],
messagetype=self.ta_info["messagetype"],
)
templatefile = botslib.abspath("templates", self.ta_info["envelope-template"])
ta_list = self.filelist2absolutepaths()
try:
botsglobal.logger.debug(u'Start writing envelope to file "%s".', self.ta_info["filename"])
ediprint = kid.Template(file=templatefile, data=ta_list) # init template; pass list with filenames
except:
txt = botslib.txtexc()
raise botslib.OutMessageError(
_(u'While enveloping in "$editype.$messagetype", error:\n$txt'),
editype=self.ta_info["editype"],
messagetype=self.ta_info["messagetype"],
txt=txt,
)
try:
filehandler = botslib.opendata(self.ta_info["filename"], "wb")
ediprint.write(filehandler, encoding=self.ta_info["charset"], output=self.ta_info["output"])
except:
txt = botslib.txtexc()
raise botslib.OutMessageError(
_(u'While enveloping in "$editype.$messagetype", error:\n$txt'),
editype=self.ta_info["editype"],
messagetype=self.ta_info["messagetype"],
txt=txt,
)
def run(self):
if self.ta_info['envelope'] == 'csvheader':
#~ Adds first line to csv files with fieldnames; than write files.
self._openoutenvelope()
botslib.tryrunscript(self.userscript,self.scriptname,'ta_infocontent',ta_info=self.ta_info)
tofile = botslib.opendata(self.ta_info['filename'],'wb',self.ta_info['charset'])
headers = dict((field[ID],field[ID]) for field in self.out.defmessage.structure[0][FIELDS])
self.out.put(headers)
self.out.tree2records(self.out.root)
tofile.write(self.out.record2string(self.out.lex_records[0:1]))
self.writefilelist(tofile)
tofile.close()
else:
super(csv,self).run()
def run(self):
self._openoutenvelope(self.ta_info['editype'],
self.ta_info['messagetype'])
botslib.tryrunscript(self.userscript,
self.scriptname,
'ta_infocontent',
ta_info=self.ta_info)
#self.ta_info is not overwritten
tofile = botslib.opendata(self.ta_info['filename'], 'wb',
self.ta_info['charset'])
headers = dict([(field[ID], field[ID])
for field in self.out.defmessage.structure[0][FIELDS]])
self.out.put(headers)
self.out.tree2records(self.out.root)
tofile.write(self.out.record2string(self.out.records[0]))
self.writefilelist(tofile)
tofile.close()
def run(self):
if not self.ta_info['topartner'] or not self.ta_info['frompartner']:
raise botslib.OutMessageError(_(u'In enveloping "frompartner" or "topartner" unknown: "%(ta_info)s".'),
{'ta_info':self.ta_info})
self._openoutenvelope()
self.ta_info.update(self.out.ta_info)
botslib.tryrunscript(self.userscript,self.scriptname,'ta_infocontent',ta_info=self.ta_info)
#prepare data for envelope
if botsglobal.ini.getboolean('settings','interchangecontrolperpartner',False):
self.ta_info['reference'] = unicode(botslib.unique('stxcounter_' + self.ta_info['topartner']))
else:
self.ta_info['reference'] = unicode(botslib.unique('stxcounter_' + self.ta_info['frompartner']))
#build the envelope segments (that is, the tree from which the segments will be generated)
self.out.put({'BOTSID':'STX',
'STDS1':self.ta_info['STX.STDS1'],
'STDS2':self.ta_info['STX.STDS2'],
'FROM.01':self.ta_info['frompartner'],
'UNTO.01':self.ta_info['topartner'],
'TRDT.01':botslib.strftime('%y%m%d'),
'TRDT.02':botslib.strftime('%H%M%S'),
'SNRF':self.ta_info['reference']})
if self.ta_info['STX.FROM.02']:
self.out.put({'BOTSID':'STX','FROM.02':self.ta_info['STX.FROM.02']})
if self.ta_info['STX.UNTO.02']:
self.out.put({'BOTSID':'STX','UNTO.02':self.ta_info['STX.UNTO.02']})
if self.ta_info['STX.APRF']:
self.out.put({'BOTSID':'STX','APRF':self.ta_info['STX.APRF']})
if self.ta_info['STX.PRCD']:
self.out.put({'BOTSID':'STX','PRCD':self.ta_info['STX.PRCD']})
self.out.put({'BOTSID':'STX'},{'BOTSID':'END','NMST':self.ta_info['nrmessages']}) #dummy segment; is not used
#user exit
botslib.tryrunscript(self.userscript,self.scriptname,'envelopecontent',ta_info=self.ta_info,out=self.out)
#convert the tree into segments; here only the STX is written (first segment)
self.out.checkmessage(self.out.root,self.out.defmessage)
self.out.checkforerrorlist()
self.out.tree2records(self.out.root)
#start doing the actual writing:
tofile = botslib.opendata(self.ta_info['filename'],'wb',self.ta_info['charset'])
tofile.write(self.out.record2string(self.out.lex_records[0:1]))
self.writefilelist(tofile)
tofile.write(self.out.record2string(self.out.lex_records[1:2]))
tofile.close()
def run(self):
''' class for (test) xml envelope. There is no standardised XML-envelope!
writes a new XML-tree; uses places-holders for XML-files to include; real enveloping is done by ElementTree's include'''
include = '{http://www.w3.org/2001/XInclude}include'
self._openoutenvelope(self.ta_info['editype'],
self.ta_info['envelope'])
botslib.tryrunscript(self.userscript,
self.scriptname,
'ta_infocontent',
ta_info=self.ta_info)
#~ self.out.put({'BOTSID':'root','xmlns:xi':"http://www.w3.org/2001/XInclude"}) #works, but attribute is not removed bij ETI.include
self.out.put({'BOTSID': 'root'}) #start filling out-tree
ta_list = self.filelist2absolutepaths()
for filename in ta_list:
self.out.put({'BOTSID': 'root'}, {
'BOTSID': include,
include + '__parse': 'xml',
include + '__href': filename
})
self.out.envelopewrite(
self.out.root) #'resolves' the included xml files
def run(self):
if self.ta_info['envelope'] == 'csvheader':
#~ Adds first line to csv files with fieldnames; than write files.
self._openoutenvelope()
botslib.tryrunscript(self.userscript,
self.scriptname,
'ta_infocontent',
ta_info=self.ta_info)
tofile = botslib.opendata(self.ta_info['filename'], 'wb',
self.ta_info['charset'])
headers = dict(
(field[ID], field[ID])
for field in self.out.defmessage.structure[0][FIELDS])
self.out.put(headers)
self.out.tree2records(self.out.root)
tofile.write(self.out.record2string(self.out.lex_records[0:1]))
self.writefilelist(tofile)
tofile.close()
else:
super(csv, self).run()
def run(self):
botslib.tryrunscript(self.userscript, self.scriptname, "ta_infocontent", ta_info=self.ta_info)
tofile = botslib.opendata(self.ta_info["filename"], "wb", self.ta_info["charset"])
self.writefilelist(tofile)
tofile.close()
def run(self):
self._openoutenvelope(self.ta_info['editype'],
self.ta_info['envelope'])
self.ta_info.update(self.out.ta_info)
#need to know the functionalgroup code:
defmessage = grammar.grammarread(self.ta_info['editype'],
self.ta_info['messagetype'])
self.ta_info['functionalgroup'] = defmessage.syntax['functionalgroup']
botslib.tryrunscript(self.userscript,
self.scriptname,
'ta_infocontent',
ta_info=self.ta_info)
#prepare data for envelope
ISA09date = time.strftime('%y%m%d')
#test indicator can either be from configuration (self.ta_info['ISA15']) or by mapping (self.ta_info['testindicator'])
#mapping overrules.
if self.ta_info['testindicator'] and self.ta_info[
'testindicator'] != '0': #'0' is default value (in db)
testindicator = self.ta_info['testindicator']
else:
testindicator = self.ta_info['ISA15']
#~ print self.ta_info['messagetype'], 'grammar:',self.ta_info['ISA15'],'ta:',self.ta_info['testindicator'],'out:',testindicator
if botsglobal.ini.getboolean('settings',
'interchangecontrolperpartner', False):
self.ta_info['reference'] = str(
botslib.unique('isacounter_' + self.ta_info['topartner']))
else:
self.ta_info['reference'] = str(
botslib.unique('isacounter_' + self.ta_info['frompartner']))
#ISA06 and GS02 can be different; eg ISA06 is a service provider.
#ISA06 and GS02 can be in the syntax....
ISA06 = self.ta_info.get('ISA06', self.ta_info['frompartner'])
GS02 = self.ta_info.get('GS02', self.ta_info['frompartner'])
#also for ISA08 and GS03
ISA08 = self.ta_info.get('ISA08', self.ta_info['topartner'])
GS03 = self.ta_info.get('GS03', self.ta_info['topartner'])
#build the envelope segments (that is, the tree from which the segments will be generated)
self.out.put({
'BOTSID': 'ISA',
'ISA01': self.ta_info['ISA01'],
'ISA02': self.ta_info['ISA02'],
'ISA03': self.ta_info['ISA03'],
'ISA04': self.ta_info['ISA04'],
'ISA05': self.ta_info['ISA05'],
'ISA06': ISA06,
'ISA07': self.ta_info['ISA07'],
'ISA08': ISA08,
'ISA09': ISA09date,
'ISA10': time.strftime('%H%M'),
'ISA11': self.ta_info[
'ISA11'], #if ISA version > 00403, replaced by reprtion separator
'ISA12': self.ta_info['version'],
'ISA13': self.ta_info['reference'],
'ISA14': self.ta_info['ISA14'],
'ISA15': testindicator
})
self.out.put({'BOTSID': 'ISA'}, {
'BOTSID': 'IEA',
'IEA01': '1',
'IEA02': self.ta_info['reference']
})
GS08 = self.ta_info['messagetype'][3:]
if GS08[:6] < '004010':
GS04date = time.strftime('%y%m%d')
else:
GS04date = time.strftime('%Y%m%d')
self.out.put({'BOTSID': 'ISA'}, {
'BOTSID': 'GS',
'GS01': self.ta_info['functionalgroup'],
'GS02': GS02,
'GS03': GS03,
'GS04': GS04date,
'GS05': time.strftime('%H%M'),
'GS06': self.ta_info['reference'],
'GS07': self.ta_info['GS07'],
'GS08': GS08
})
self.out.put({'BOTSID': 'ISA'}, {'BOTSID': 'GS'}, {
'BOTSID': 'GE',
'GE01': self.ta_info['nrmessages'],
'GE02': self.ta_info['reference']
}) #dummy segment; is not used
#user exit
botslib.tryrunscript(self.userscript,
self.scriptname,
'envelopecontent',
ta_info=self.ta_info,
out=self.out)
#convert the tree into segments; here only the UNB is written (first segment)
self.out.normalisetree(self.out.root)
self.out.tree2records(self.out.root)
#start doing the actual writing:
tofile = botslib.opendata(self.ta_info['filename'], 'wb',
self.ta_info['charset'])
ISAstring = self.out._record2string(self.out.records[0])
if self.ta_info['version'] < '00403':
ISAstring = ISAstring[:103] + self.ta_info[
'field_sep'] + self.ta_info['sfield_sep'] + ISAstring[
103:] #hack for strange characters at end of ISA; hardcoded
else:
ISAstring = ISAstring[:82] + self.ta_info['reserve'] + ISAstring[
83:103] + self.ta_info['field_sep'] + self.ta_info[
'sfield_sep'] + ISAstring[
103:] #hack for strange characters at end of ISA; hardcoded
tofile.write(ISAstring) #write ISA
tofile.write(self.out._record2string(self.out.records[1])) #write GS
self.writefilelist(tofile)
tofile.write(self.out._record2string(self.out.records[-2])) #write GE
tofile.write(self.out._record2string(self.out.records[-1])) #write IEA
tofile.close()
if self.ta_info[
'functionalgroup'] != 'FA' and botslib.checkconfirmrules(
'ask-x12-997',
idroute=self.ta_info['idroute'],
idchannel=self.ta_info['tochannel'],
topartner=self.ta_info['topartner'],
frompartner=self.ta_info['frompartner'],
editype=self.ta_info['editype'],
messagetype=self.ta_info['messagetype']):
self.ta_info['confirmtype'] = u'ask-x12-997'
self.ta_info['confirmasked'] = True
def router(routedict):
''' communication.run one route. variants:
- a route can be just script;
- a route can do only incoming
- a route can do only outgoing
- a route can do both incoming and outgoing
- at several points functions from a route script are called - if function is in route script
'''
#is there a user route script?
try:
userscript,scriptname = botslib.botsimport('routescripts',routedict['idroute'])
except ImportError: #other errors, eg syntax errors are just passed
userscript = scriptname = None
#if user route script has function 'main': communication.run 'main' (and do nothing else)
if botslib.tryrunscript(userscript,scriptname,'main',routedict=routedict):
return #so: if function ' main' : communication.run only the routescript, nothing else.
if not (userscript or routedict['fromchannel'] or routedict['tochannel'] or routedict['translateind']):
raise botslib.ScriptError(_(u'Route "$route" is empty: no script, not enough parameters.'),route=routedict['idroute'])
botslib.tryrunscript(userscript,scriptname,'start',routedict=routedict)
#communication.run incoming channel
if routedict['fromchannel']: #do incoming part of route: in-communication; set ready for translation; translate
botslib.tryrunscript(userscript,scriptname,'preincommunication',routedict=routedict)
communication.run(idchannel=routedict['fromchannel'],idroute=routedict['idroute']) #communication.run incommunication
#add attributes from route to the received files
where={'status':FILEIN,'fromchannel':routedict['fromchannel'],'idroute':routedict['idroute']}
change={'editype':routedict['fromeditype'],'messagetype':routedict['frommessagetype'],'frompartner':routedict['frompartner'],'topartner':routedict['topartner'],'alt':routedict['alt']}
botslib.updateinfo(change=change,where=where)
#all received files have status FILEIN
botslib.tryrunscript(userscript,scriptname,'postincommunication',routedict=routedict)
if routedict['fromeditype'] == 'mailbag': #mailbag for the route.
preprocess.preprocess(routedict,preprocess.mailbag)
#communication.run translation
if routedict['translateind']:
botslib.tryrunscript(userscript,scriptname,'pretranslation',routedict=routedict)
botslib.addinfo(change={'status':TRANSLATE},where={'status':FILEIN,'idroute':routedict['idroute']})
transform.translate(idroute=routedict['idroute'])
botslib.tryrunscript(userscript,scriptname,'posttranslation',routedict=routedict)
#merge messages & communication.run outgoing channel
if routedict['tochannel']: #do outgoing part of route
botslib.tryrunscript(userscript,scriptname,'premerge',routedict=routedict)
envelope.mergemessages(idroute=routedict['idroute'])
botslib.tryrunscript(userscript,scriptname,'postmerge',routedict=routedict)
#communication.run outgoing channel
#build for query: towhere (dict) and wherestring
towhere=dict(status=MERGED,
idroute=routedict['idroute'],
editype=routedict['toeditype'],
messagetype=routedict['tomessagetype'],
testindicator=routedict['testindicator'])
towhere=dict([(key, value) for (key, value) in towhere.iteritems() if value]) #remove nul-values from dict
wherestring = ' AND '.join([key+'=%('+key+')s' for key in towhere])
if routedict['frompartner_tochannel_id']: #use frompartner_tochannel in where-clause of query (partner/group dependent outchannel
towhere['frompartner_tochannel_id']=routedict['frompartner_tochannel_id']
wherestring += ''' AND (frompartner=%(frompartner_tochannel_id)s
OR frompartner in (SELECT from_partner_id
FROM partnergroup
WHERE to_partner_id =%(frompartner_tochannel_id)s ))'''
if routedict['topartner_tochannel_id']: #use topartner_tochannel in where-clause of query (partner/group dependent outchannel
towhere['topartner_tochannel_id']=routedict['topartner_tochannel_id']
wherestring += ''' AND (topartner=%(topartner_tochannel_id)s
OR topartner in (SELECT from_partner_id
FROM partnergroup
WHERE to_partner_id=%(topartner_tochannel_id)s ))'''
toset={'tochannel':routedict['tochannel'],'status':FILEOUT}
botslib.addinfocore(change=toset,where=towhere,wherestring=wherestring)
if not routedict['defer']: #do outgoing part of route
botslib.tryrunscript(userscript,scriptname,'preoutcommunication',routedict=routedict)
communication.run(idchannel=routedict['tochannel'],idroute=routedict['idroute']) #communication.run outcommunication
botslib.tryrunscript(userscript,scriptname,'postoutcommunication',routedict=routedict)
botslib.tryrunscript(userscript,scriptname,'end',routedict=routedict)
def run(self):
botslib.tryrunscript(self.userscript,
self.scriptname,
'ta_infocontent',
ta_info=self.ta_info)
self.ta_info['filename'] = self.ta_list[0]
def run(self):
self._openoutenvelope(self.ta_info['editype'],self.ta_info['envelope'])
self.ta_info.update(self.out.ta_info)
#need to know the functionalgroup code:
defmessage = grammar.grammarread(self.ta_info['editype'],self.ta_info['messagetype'])
self.ta_info['functionalgroup'] = defmessage.syntax['functionalgroup']
botslib.tryrunscript(self.userscript,self.scriptname,'ta_infocontent',ta_info=self.ta_info)
#prepare data for envelope
ISA09date = time.strftime('%y%m%d')
#test indicator can either be from configuration (self.ta_info['ISA15']) or by mapping (self.ta_info['testindicator'])
#mapping overrules.
if self.ta_info['testindicator'] and self.ta_info['testindicator']!='0': #'0' is default value (in db)
testindicator = self.ta_info['testindicator']
else:
testindicator = self.ta_info['ISA15']
#~ print self.ta_info['messagetype'], 'grammar:',self.ta_info['ISA15'],'ta:',self.ta_info['testindicator'],'out:',testindicator
if botsglobal.ini.getboolean('settings','interchangecontrolperpartner',False):
self.ta_info['reference'] = str(botslib.unique('isacounter_' + self.ta_info['topartner']))
else:
self.ta_info['reference'] = str(botslib.unique('isacounter_' + self.ta_info['frompartner']))
#ISA06 and GS02 can be different; eg ISA06 is a service provider.
#ISA06 and GS02 can be in the syntax....
ISA06 = self.ta_info.get('ISA06',self.ta_info['frompartner'])
GS02 = self.ta_info.get('GS02',self.ta_info['frompartner'])
#also for ISA08 and GS03
ISA08 = self.ta_info.get('ISA08',self.ta_info['topartner'])
GS03 = self.ta_info.get('GS03',self.ta_info['topartner'])
#build the envelope segments (that is, the tree from which the segments will be generated)
self.out.put({'BOTSID':'ISA',
'ISA01':self.ta_info['ISA01'],
'ISA02':self.ta_info['ISA02'],
'ISA03':self.ta_info['ISA03'],
'ISA04':self.ta_info['ISA04'],
'ISA05':self.ta_info['ISA05'],
'ISA06':ISA06,
'ISA07':self.ta_info['ISA07'],
'ISA08':ISA08,
'ISA09':ISA09date,
'ISA10':time.strftime('%H%M'),
'ISA11':self.ta_info['ISA11'], #if ISA version > 00403, replaced by reprtion separator
'ISA12':self.ta_info['version'],
'ISA13':self.ta_info['reference'],
'ISA14':self.ta_info['ISA14'],
'ISA15':testindicator})
self.out.put({'BOTSID':'ISA'},{'BOTSID':'IEA','IEA01':'1','IEA02':self.ta_info['reference']})
GS08 = self.ta_info['messagetype'][3:]
if GS08[:6]<'004010':
GS04date = time.strftime('%y%m%d')
else:
GS04date = time.strftime('%Y%m%d')
self.out.put({'BOTSID':'ISA'},{'BOTSID':'GS',
'GS01':self.ta_info['functionalgroup'],
'GS02':GS02,
'GS03':GS03,
'GS04':GS04date,
'GS05':time.strftime('%H%M'),
'GS06':self.ta_info['reference'],
'GS07':self.ta_info['GS07'],
'GS08':GS08})
self.out.put({'BOTSID':'ISA'},{'BOTSID':'GS'},{'BOTSID':'GE','GE01':self.ta_info['nrmessages'],'GE02':self.ta_info['reference']}) #dummy segment; is not used
#user exit
botslib.tryrunscript(self.userscript,self.scriptname,'envelopecontent',ta_info=self.ta_info,out=self.out)
#convert the tree into segments; here only the UNB is written (first segment)
self.out.normalisetree(self.out.root)
self.out.tree2records(self.out.root)
#start doing the actual writing:
tofile = botslib.opendata(self.ta_info['filename'],'wb',self.ta_info['charset'])
ISAstring = self.out._record2string(self.out.records[0])
if self.ta_info['version']<'00403':
ISAstring = ISAstring[:103] + self.ta_info['field_sep']+ self.ta_info['sfield_sep'] + ISAstring[103:] #hack for strange characters at end of ISA; hardcoded
else:
ISAstring = ISAstring[:82] +self.ta_info['reserve'] + ISAstring[83:103] + self.ta_info['field_sep']+ self.ta_info['sfield_sep'] + ISAstring[103:] #hack for strange characters at end of ISA; hardcoded
tofile.write(ISAstring) #write ISA
tofile.write(self.out._record2string(self.out.records[1])) #write GS
self.writefilelist(tofile)
tofile.write(self.out._record2string(self.out.records[-2])) #write GE
tofile.write(self.out._record2string(self.out.records[-1])) #write IEA
tofile.close()
if self.ta_info['functionalgroup']!='FA' and botslib.checkconfirmrules('ask-x12-997',idroute=self.ta_info['idroute'],idchannel=self.ta_info['tochannel'],
topartner=self.ta_info['topartner'],frompartner=self.ta_info['frompartner'],
editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype']):
self.ta_info['confirmtype'] = u'ask-x12-997'
self.ta_info['confirmasked'] = True
def run(self):
botslib.tryrunscript(self.userscript,self.scriptname,'ta_infocontent',ta_info=self.ta_info)
self.ta_info['filename'] = self.ta_list[0]
def run(self):
if not self.ta_info['topartner'] or not self.ta_info['frompartner']:
raise botslib.OutMessageError(_(u'In enveloping "frompartner" or "topartner" unknown: "%(ta_info)s".'),
{'ta_info':self.ta_info})
self._openoutenvelope()
self.ta_info.update(self.out.ta_info)
botslib.tryrunscript(self.userscript,self.scriptname,'ta_infocontent',ta_info=self.ta_info)
#version dependent enveloping
if self.ta_info['version'] < '4':
date = botslib.strftime('%y%m%d')
reserve = ' '
else:
date = botslib.strftime('%Y%m%d')
reserve = self.ta_info['reserve']
#UNB reference is counter is per sender or receiver
if botsglobal.ini.getboolean('settings','interchangecontrolperpartner',False):
self.ta_info['reference'] = unicode(botslib.unique('unbcounter_' + self.ta_info['topartner']))
else:
self.ta_info['reference'] = unicode(botslib.unique('unbcounter_' + self.ta_info['frompartner']))
#testindicator is more complex:
if self.ta_info['testindicator'] and self.ta_info['testindicator'] != '0': #first check value from ta; do not use default
testindicator = '1'
elif self.ta_info['UNB.0035'] != '0': #than check values from grammar
testindicator = '1'
else:
testindicator = ''
#build the envelope segments (that is, the tree from which the segments will be generated)
self.out.put({'BOTSID':'UNB',
'S001.0001':self.ta_info['charset'],
'S001.0002':self.ta_info['version'],
'S002.0004':self.ta_info['frompartner'],
'S003.0010':self.ta_info['topartner'],
'S004.0017':date,
'S004.0019':botslib.strftime('%H%M'),
'0020':self.ta_info['reference']})
#the following fields are conditional; do not write these when empty string (separator compression does take empty strings into account)
for field in ('S001.0080','S001.0133','S002.0007','S002.0008','S002.0042',
'S003.0007','S003.0014','S003.0046','S005.0022','S005.0025',
'0026','0029','0031','0032'):
if self.ta_info['UNB.'+field]:
self.out.put({'BOTSID':'UNB',field:self.ta_info['UNB.'+field]})
if testindicator:
self.out.put({'BOTSID':'UNB','0035': testindicator})
self.out.put({'BOTSID':'UNB'},{'BOTSID':'UNZ','0036':self.ta_info['nrmessages'],'0020':self.ta_info['reference']}) #dummy segment; is not used
#user exit
botslib.tryrunscript(self.userscript,self.scriptname,'envelopecontent',ta_info=self.ta_info,out=self.out)
#convert the tree into segments; here only the UNB is written (first segment)
self.out.checkmessage(self.out.root,self.out.defmessage)
self.out.checkforerrorlist()
self.out.tree2records(self.out.root)
#start doing the actual writing:
tofile = botslib.opendata(self.ta_info['filename'],'wb',self.ta_info['charset'])
if self.ta_info['forceUNA'] or self.ta_info['charset'] != 'UNOA':
tofile.write('UNA'+self.ta_info['sfield_sep']+self.ta_info['field_sep']+self.ta_info['decimaal']+self.ta_info['escape']+ reserve +self.ta_info['record_sep']+self.ta_info['add_crlfafterrecord_sep'])
tofile.write(self.out.record2string(self.out.lex_records[0:1]))
self.writefilelist(tofile)
tofile.write(self.out.record2string(self.out.lex_records[1:2]))
tofile.close()
def start():
''' sysexit codes:
0: OK, no errors
1: (system) errors incl parsing of command line arguments
2: bots ran OK, but there are errors/process errors in the run
3: Database is locked, but "maxruntime" has not been exceeded.
'''
#NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
#********command line arguments**************************
usage = '''
This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
Does the actual translations and communications; it's the workhorse. It does not have a fancy interface.
Usage:
%(name)s [run-options] [config-option] [routes]
Run-options (can be combined):
--new receive new edi files (default: if no run-option given: run as new).
--resend resend as indicated by user.
--rereceive rereceive as indicated by user.
--automaticretrycommunication - automatically retry outgoing communication.
--cleanup remove older data from database.
Config-option:
-c<directory> directory for configuration files (default: config).
Routes: list of routes to run. Default: all active routes (in the database)
'''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version}
configdir = 'config'
commandspossible = ['--automaticretrycommunication','--resend','--rereceive','--new']
commandstorun = []
routestorun = [] #list with routes to run
do_cleanup_parameter = False
for arg in sys.argv[1:]:
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Error: configuration directory indicated, but no directory name.'
sys.exit(1)
elif arg in commandspossible:
commandstorun.append(arg)
elif arg == '--cleanup':
do_cleanup_parameter = True
elif arg in ["?", "/?",'-h', '--help'] or arg.startswith('-'):
print usage
sys.exit(0)
else: #pick up names of routes to run
routestorun.append(arg)
if not commandstorun and not do_cleanup_parameter: #if no command on command line, use new (default)
commandstorun = ['--new']
commandstorun = [command[2:] for command in commandspossible if command in commandstorun] #sort commands
#***********end handling command line arguments**************************
botsinit.generalinit(configdir) #find locating of bots, configfiles, init paths etc.
#set working directory to bots installation. advantage: when using relative paths it is clear that this point paths within bots installation.
os.chdir(botsglobal.ini.get('directories','botspath'))
#**************check if another instance of bots-engine is running/if port is free******************************
try:
engine_socket = botslib.check_if_other_engine_is_running()
except socket.error:
sys.exit(3)
else:
atexit.register(engine_socket.close)
#**************initialise logging******************************
process_name = 'engine'
botsglobal.logger = botsinit.initenginelogging(process_name)
atexit.register(logging.shutdown)
for key,value in botslib.botsinfo(): #log info about environement, versions, etc
botsglobal.logger.info(u'%(key)s: "%(value)s".',{'key':key,'value':value})
#**************connect to database**********************************
try:
botsinit.connect()
except Exception as msg:
botsglobal.logger.exception(_(u'Could not connect to database. Database settings are in bots/config/settings.py. Error: "%(msg)s".'),{'msg':msg})
sys.exit(1)
else:
botsglobal.logger.info(_(u'Connected to database.'))
atexit.register(botsglobal.db.close)
#************initialise user exits for the whole bots-engine*************************
try:
userscript,scriptname = botslib.botsimport('routescripts','botsengine')
except ImportError: #userscript is not there; other errors like syntax errors are not catched
userscript = scriptname = None
#***acceptance tests: initialiase acceptance user script******************************
acceptance_userscript = acceptance_scriptname = None
if botsglobal.ini.getboolean('acceptance','runacceptancetest',False):
botsglobal.logger.info(_(u'This run is an acceptance test - as indicated in option "runacceptancetest" in bots.ini.'))
try:
acceptance_userscript,acceptance_scriptname = botslib.botsimport('routescripts','bots_acceptancetest')
except ImportError:
botsglobal.logger.info(_(u'In acceptance test there is no script file "bots_acceptancetest.py" to check the results of the acceptance test.'))
#**************handle database lock****************************************
#set a lock on the database; if not possible, the database is locked: an earlier instance of bots-engine was terminated unexpectedly.
if not botslib.set_database_lock():
#for SQLite: do a integrity check on the database
if botsglobal.settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3':
cursor = botsglobal.db.execute('''PRAGMA integrity_check''')
result = cursor.fetchone()
if result[0] != u'ok':
warn = _(u'!Bots database is locked!\n'\
'Bots did an integrity check on the database, but database was not OK.\n'\
'Manual action is needed!\n'\
'Bots has stopped processing EDI files.')
botsglobal.logger.critical(warn)
botslib.sendbotserrorreport(_(u'[Bots severe error]Database is damaged'),warn)
sys.exit(1)
warn = _(u'!Bots database is locked!\n'\
'Bots-engine has ended in an unexpected way during the last run.\n'\
'Most likely causes: sudden power-down, system crash, problems with disk I/O, bots-engine terminated by user, etc.\n'
'Bots will do an automatic crash recovery now.')
botsglobal.logger.critical(warn)
botslib.sendbotserrorreport(_(u'[Bots severe error]Database is locked'),warn)
commandstorun.insert(0,'crashrecovery') #there is a database lock. Add a crashrecovery as first command to run.
atexit.register(botslib.remove_database_lock)
#**************run the routes**********************************************
#commandstorun determines the type(s) of run. eg: ['automaticretrycommunication','new']
try:
botslib.prepare_confirmrules()
#in acceptance tests: run a user script before running eg to clean output directories******************************
botslib.tryrunscript(acceptance_userscript,acceptance_scriptname,'pretest',routestorun=routestorun)
botslib.tryrunscript(userscript,scriptname,'pre',commandstorun=commandstorun,routestorun=routestorun)
errorinrun = 0 #detect if there has been some error. Only used for correct exit() code
first_command_2_run = True
for command in commandstorun:
#if multiple commands in run: reports etc are based on timestamp; so there needs to be at least one second between these runs.
if first_command_2_run:
first_command_2_run = False
else:
time.sleep(1)
botsglobal.logger.info(_(u'Run "%(command)s".'),{'command':command})
#************get list of routes to run*******************************
if routestorun:
use_routestorun = routestorun[:]
botsglobal.logger.info(_(u'Run routes from command line: "%(routes)s".'),{'routes':str(use_routestorun)})
elif command == 'new': #fetch all active routes from database unless 'not in default run' or not active.
use_routestorun = []
for row in botslib.query('''SELECT DISTINCT idroute
FROM routes
WHERE active=%(active)s
AND (notindefaultrun=%(notindefaultrun)s OR notindefaultrun IS NULL)
ORDER BY idroute ''',
{'active':True,'notindefaultrun':False}):
use_routestorun.append(row['idroute'])
botsglobal.logger.info(_(u'Run active routes from database that are in default run: "%(routes)s".'),{'routes':str(use_routestorun)})
else: #for command other than 'new': use all active routes.
use_routestorun = []
for row in botslib.query('''SELECT DISTINCT idroute
FROM routes
WHERE active=%(active)s
ORDER BY idroute ''',
{'active':True}):
use_routestorun.append(row['idroute'])
botsglobal.logger.info(_(u'Run all active routes from database: "%(routes)s".'),{'routes':str(use_routestorun)})
#************run routes for this command******************************
botslib.tryrunscript(userscript,scriptname,'pre' + command,routestorun=use_routestorun)
errorinrun += router.rundispatcher(command,use_routestorun)
botslib.tryrunscript(userscript,scriptname,'post' + command,routestorun=use_routestorun)
#*********finished running routes for this command****************************
#*********finished all commands****************************************
botslib.tryrunscript(userscript,scriptname,'post',commandstorun=commandstorun,routestorun=routestorun)
try: #in acceptance tests: run a user script. no good reporting of errors/results in post-test script. Reason: this is after automaticmaintence.
botslib.tryrunscript(acceptance_userscript,acceptance_scriptname,'posttest',routestorun=use_routestorun)
except Exception as msg:
print str(msg)
cleanup.cleanup(do_cleanup_parameter,userscript,scriptname)
except Exception as msg:
botsglobal.logger.exception(_(u'Severe error in bots system:\n%(msg)s'),{'msg':str(msg)}) #of course this 'should' not happen.
sys.exit(1)
else:
if errorinrun:
sys.exit(2) #indicate: error(s) in run(s)
else:
sys.exit(0) #OK
def routepart(self,routedict):
''' communication.run one route part. variants:
- a route can be a routescript
- a route can do only incoming
- a route can do only outgoing
- a route can do both incoming and outgoing
- at several points functions from a routescript are called - if function is in routescript
'''
self.minta4query_routepart = botslib._Transaction.processlist[-1] #the idta of routepart
#if routescript has function 'main': communication.run 'main' (and do nothing else)
if botslib.tryrunscript(self.userscript,self.scriptname,'main',routedict=routedict):
return #so: if function ' main' : communication.run only the routescript, nothing else.
if not (self.userscript or routedict['fromchannel'] or routedict['tochannel'] or routedict['translateind']):
raise botslib.ScriptError(_(u'Route "%(idroute)s" is empty: no routescript, not enough parameters.'),routedict)
botslib.tryrunscript(self.userscript,self.scriptname,'start',routedict=routedict)
#incoming part of route:
#- incommunication
#- assign attributes from route to incoming files
#- preprocessing
if routedict['fromchannel']:
#only done for edi files from this route-part, this inchannel
if routedict['command'] == 'rereceive':
rootidta = self.get_minta4query()
else:
rootidta = self.get_minta4query_routepart()
botslib.tryrunscript(self.userscript,self.scriptname,'preincommunication',routedict=routedict)
communication.run(idchannel=routedict['fromchannel'],command=routedict['command'],idroute=routedict['idroute'],rootidta=rootidta) #communication.run incommunication
#add attributes from route to the received files;
where = {'statust':OK,'status':FILEIN,'fromchannel':routedict['fromchannel'],'idroute':routedict['idroute'],'rootidta':rootidta}
change = {'editype':routedict['fromeditype'],'messagetype':routedict['frommessagetype'],'frompartner':routedict['frompartner'],'topartner':routedict['topartner'],'alt':routedict['alt']}
nr_of_incoming_files_for_channel = botslib.updateinfo(change=change,where=where)
botslib.tryrunscript(self.userscript,self.scriptname,'postincommunication',routedict=routedict)
if nr_of_incoming_files_for_channel:
#unzip incoming files (if indicated)
if routedict['zip_incoming'] == 1: #unzip incoming (non-zipped gives error).
preprocess.preprocess(routedict=routedict,function=preprocess.botsunzip,rootidta=self.get_minta4query_routepart(),pass_non_zip=False)
elif routedict['zip_incoming'] == 2: #unzip incoming if zipped.
preprocess.preprocess(routedict=routedict,function=preprocess.botsunzip,rootidta=self.get_minta4query_routepart(),pass_non_zip=True)
#run mailbag-module.
if botsglobal.ini.getboolean('settings','compatibility_mailbag',False):
editypes_via_mailbag = ['mailbag']
else:
editypes_via_mailbag = ['mailbag','edifact','x12','tradacoms']
if routedict['fromeditype'] in editypes_via_mailbag: #mailbag for the route.
preprocess.preprocess(routedict=routedict,function=preprocess.mailbag,rootidta=self.get_minta4query_routepart(),frommessagetype=routedict['frommessagetype'])
#translate, merge, pass through: INFILE->MERGED
if int(routedict['translateind']) in [1,3]:
#translate: for files in route
botslib.tryrunscript(self.userscript,self.scriptname,'pretranslation',routedict=routedict)
if routedict['command'] in ['rereceive',]:
rootidta = self.get_minta4query()
else:
rootidta = self.get_minta4query_route()
transform.translate(startstatus=FILEIN,endstatus=TRANSLATED,routedict=routedict,rootidta=rootidta)
botslib.tryrunscript(self.userscript,self.scriptname,'posttranslation',routedict=routedict)
#**merge: for files in this route-part (the translated files)
botslib.tryrunscript(self.userscript,self.scriptname,'premerge',routedict=routedict)
envelope.mergemessages(startstatus=TRANSLATED,endstatus=MERGED,idroute=routedict['idroute'],rootidta=self.get_minta4query_routepart())
botslib.tryrunscript(self.userscript,self.scriptname,'postmerge',routedict=routedict)
elif routedict['translateind'] == 2: #pass-through: pickup the incoming files and mark these as MERGED (==translation is finished)
botslib.addinfo(change={'status':MERGED,'statust':OK},where={'status':FILEIN,'statust':OK,'idroute':routedict['idroute'],'rootidta':self.get_minta4query_route()})
#NOTE: routedict['translateind'] == 0 than nothing will happen with the files in this route.
#ommunication outgoing channel: MERGED->RAWOUT
if routedict['tochannel']:
#**build query to add outchannel as attribute to outgoing files***
#filter files in route for outchannel
towhere = { 'status':MERGED,
'statust':OK,
'idroute':routedict['idroute'],
'editype':routedict['toeditype'],
'messagetype':routedict['tomessagetype'],
'testindicator':routedict['testindicator'],
}
towhere = dict((key, value) for key,value in towhere.iteritems() if value) #remove nul-values from dict
wherestring = ' AND '.join([key+'=%('+key+')s ' for key in towhere])
if routedict['frompartner_tochannel_id']: #use frompartner_tochannel in where-clause of query (partner/group dependent outchannel
towhere['frompartner_tochannel_id'] = routedict['frompartner_tochannel_id']
wherestring += ''' AND (frompartner=%(frompartner_tochannel_id)s
OR frompartner in (SELECT from_partner_id
FROM partnergroup
WHERE to_partner_id=%(frompartner_tochannel_id)s )) '''
if routedict['topartner_tochannel_id']: #use topartner_tochannel in where-clause of query (partner/group dependent outchannel
towhere['topartner_tochannel_id'] = routedict['topartner_tochannel_id']
wherestring += ''' AND (topartner=%(topartner_tochannel_id)s
OR topartner in (SELECT from_partner_id
FROM partnergroup
WHERE to_partner_id=%(topartner_tochannel_id)s )) '''
toset = {'status':FILEOUT,'statust':OK,'tochannel':routedict['tochannel']}
towhere['rootidta'] = self.get_minta4query_route()
nr_of_outgoing_files_for_channel = botslib.addinfocore(change=toset,where=towhere,wherestring=wherestring)
if nr_of_outgoing_files_for_channel:
#**set asked confirmation/acknowledgements
botslib.set_asked_confirmrules(routedict,rootidta=self.get_minta4query_routepart())
#**zip outgoing
#for files in this route-part for this out-channel
if routedict['zip_outgoing'] == 1:
preprocess.postprocess(routedict=routedict,function=preprocess.botszip,rootidta=self.get_minta4query_routepart())
#actual communication: run outgoing channel (if not deferred)
#for all files in run that are for this channel (including the deferred ones from other routes)
if not routedict['defer']:
#determine range of idta to query: if channel was not deferred earlier in run: query only for route part else query for whole run
if self.keep_track_if_outchannel_deferred.get(routedict['tochannel'],False) or routedict['command'] in ['resend','automaticretrycommunication']:
rootidta = self.get_minta4query()
else:
rootidta = self.get_minta4query_routepart()
if botslib.countoutfiles(idchannel=routedict['tochannel'],rootidta=rootidta):
botslib.tryrunscript(self.userscript,self.scriptname,'preoutcommunication',routedict=routedict)
communication.run(idchannel=routedict['tochannel'],command=routedict['command'],idroute=routedict['idroute'],rootidta=rootidta)
#in communication several things can go wrong.
#all outgoing files should have same status; that way all recomnnunication can be handled the same:
#- status EXTERNOUT statust DONE (if communication goes OK)
#- status EXTERNOUT status ERROR (if file is not communicatied)
#to have the same status for all outgoing files some manipulation is needed, eg in case no connection could be made.
botslib.addinfo(change={'status':EXTERNOUT,'statust':ERROR},where={'status':FILEOUT,'statust':OK,'tochannel':routedict['tochannel'],'rootidta':rootidta})
botslib.tryrunscript(self.userscript,self.scriptname,'postoutcommunication',routedict=routedict)
botslib.tryrunscript(self.userscript,self.scriptname,'end',routedict=routedict)
def run(self):
if not self.ta_info['topartner'] or not self.ta_info['frompartner']:
raise botslib.OutMessageError(
_(u'In enveloping "frompartner" or "topartner" unknown: "%(ta_info)s".'
), {'ta_info': self.ta_info})
self._openoutenvelope()
self.ta_info.update(self.out.ta_info)
botslib.tryrunscript(self.userscript,
self.scriptname,
'ta_infocontent',
ta_info=self.ta_info)
#prepare data for envelope
isa09date = botslib.strftime('%y%m%d')
#test indicator can either be from configuration (self.ta_info['ISA15']) or by mapping (self.ta_info['testindicator'])
#mapping overrules.
if self.ta_info['testindicator'] and self.ta_info[
'testindicator'] != '0': #'0' is default value (in db)
testindicator = self.ta_info['testindicator']
else:
testindicator = self.ta_info['ISA15']
#~ print self.ta_info['messagetype'], 'grammar:',self.ta_info['ISA15'],'ta:',self.ta_info['testindicator'],'out:',testindicator
if botsglobal.ini.getboolean('settings',
'interchangecontrolperpartner', False):
self.ta_info['reference'] = unicode(
botslib.unique('isacounter_' + self.ta_info['topartner']))
else:
self.ta_info['reference'] = unicode(
botslib.unique('isacounter_' + self.ta_info['frompartner']))
#ISA06 and GS02 can be different; eg ISA06 is a service provider.
#ISA06 and GS02 can be in the syntax....
isa06sender = self.ta_info.get('ISA06', self.ta_info['frompartner'])
isa06sender = isa06sender.ljust(15) #add spaces; is fixed length
gs02sender = self.ta_info.get('GS02', self.ta_info['frompartner'])
#also for ISA08 and GS03
isa08receiver = self.ta_info.get('ISA08', self.ta_info['topartner'])
isa08receiver = isa08receiver.ljust(15) #add spaces; is fixed length
gs03receiver = self.ta_info.get('GS03', self.ta_info['topartner'])
#build the envelope segments (that is, the tree from which the segments will be generated)
self.out.put(
{
'BOTSID': 'ISA',
'ISA01': self.ta_info['ISA01'],
'ISA02': self.ta_info['ISA02'],
'ISA03': self.ta_info['ISA03'],
'ISA04': self.ta_info['ISA04'],
'ISA05': self.ta_info['ISA05'],
'ISA06': isa06sender,
'ISA07': self.ta_info['ISA07'],
'ISA08': isa08receiver,
'ISA09': isa09date,
'ISA10': botslib.strftime('%H%M'),
'ISA11': self.ta_info[
'ISA11'], #if ISA version > 00403, replaced by reprtion separator
'ISA12': self.ta_info['version'],
'ISA13': self.ta_info['reference'],
'ISA14': self.ta_info['ISA14'],
'ISA15': testindicator
},
strip=False
) #MIND: strip=False: ISA fields shoudl not be stripped as it is soemwhat like fixed-length
self.out.put({'BOTSID': 'ISA'}, {
'BOTSID': 'IEA',
'IEA01': '1',
'IEA02': self.ta_info['reference']
})
gs08messagetype = self.ta_info['messagetype'][3:]
if gs08messagetype[:6] < '004010':
gs04date = botslib.strftime('%y%m%d')
else:
gs04date = botslib.strftime('%Y%m%d')
self.out.put({'BOTSID': 'ISA'}, {
'BOTSID': 'GS',
'GS01': self.ta_info['functionalgroup'],
'GS02': gs02sender,
'GS03': gs03receiver,
'GS04': gs04date,
'GS05': botslib.strftime('%H%M'),
'GS06': self.ta_info['reference'],
'GS07': self.ta_info['GS07'],
'GS08': gs08messagetype
})
self.out.put({'BOTSID': 'ISA'}, {'BOTSID': 'GS'}, {
'BOTSID': 'GE',
'GE01': self.ta_info['nrmessages'],
'GE02': self.ta_info['reference']
}) #dummy segment; is not used
#user exit
botslib.tryrunscript(self.userscript,
self.scriptname,
'envelopecontent',
ta_info=self.ta_info,
out=self.out)
#convert the tree into segments; here only the UNB is written (first segment)
self.out.checkmessage(self.out.root, self.out.defmessage)
self.out.checkforerrorlist()
self.out.tree2records(self.out.root)
#start doing the actual writing:
tofile = botslib.opendata(self.ta_info['filename'], 'wb',
self.ta_info['charset'])
isa_string = self.out.record2string(self.out.lex_records[0:1])
#ISA has the used separators at certain positions. Normally bots would give errors for this (can not use sep as data) or compress these aways. So this is hardcoded.
if self.ta_info['version'] < '00403':
isa_string = isa_string[:103] + self.ta_info[
'field_sep'] + self.ta_info['sfield_sep'] + isa_string[103:]
else:
isa_string = isa_string[:82] + self.ta_info[
'reserve'] + isa_string[83:103] + self.ta_info[
'field_sep'] + self.ta_info['sfield_sep'] + isa_string[103:]
tofile.write(isa_string) #write ISA
tofile.write(self.out.record2string(
self.out.lex_records[1:2])) #write GS
self.writefilelist(tofile)
tofile.write(self.out.record2string(
self.out.lex_records[2:])) #write GE and IEA
tofile.close()
def start():
''' sysexit codes:
0: OK, no errors
1: (system) errors incl parsing of command line arguments
2: bots ran OK, but there are errors/process errors in the run
3: Database is locked, but "maxruntime" has not been exceeded.
'''
#NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
#********command line arguments**************************
usage = '''
This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
Does the actual translations and communications; it's the workhorse. It does not have a fancy interface.
Usage:
%(name)s [config-option]
Config-option:
-c<directory> directory for configuration files (default: config).
'''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version}
configdir = 'config'
for arg in sys.argv[1:]:
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Error: configuration directory indicated, but no directory name.'
sys.exit(1)
elif arg in ["?", "/?",'-h', '--help'] or arg.startswith('-'):
print usage
sys.exit(0)
#***********end handling command line arguments**************************
botsinit.generalinit(configdir) #find locating of bots, configfiles, init paths etc.
#set working directory to bots installation. advantage: when using relative paths it is clear that this point paths within bots installation.
os.chdir(botsglobal.ini.get('directories','botspath'))
#**************initialise logging******************************
process_name = 'engine2'
botsglobal.logger = botsinit.initenginelogging(process_name)
atexit.register(logging.shutdown)
for key,value in botslib.botsinfo(): #log info about environement, versions, etc
botsglobal.logger.info(u'%(key)s: "%(value)s".',{'key':key,'value':value})
#**************connect to database**********************************
try:
botsinit.connect()
except Exception as msg:
botsglobal.logger.exception(_(u'Could not connect to database. Database settings are in bots/config/settings.py. Error: "%(msg)s".'),{'msg':msg})
sys.exit(1)
else:
botsglobal.logger.info(_(u'Connected to database.'))
atexit.register(botsglobal.db.close)
warnings.simplefilter('error', UnicodeWarning)
#import global scripts for bots-engine
try:
userscript,scriptname = botslib.botsimport('routescripts','botsengine')
except botslib.BotsImportError: #userscript is not there; other errors like syntax errors are not catched
userscript = scriptname = None
#***acceptance tests: initialiase acceptance user script******************************
acceptance_userscript = acceptance_scriptname = None
if botsglobal.ini.getboolean('acceptance','runacceptancetest',False):
botsglobal.logger.info(_(u'This run is an acceptance test - as indicated in option "runacceptancetest" in bots.ini.'))
try:
acceptance_userscript,acceptance_scriptname = botslib.botsimport('routescripts','bots_acceptancetest')
except botslib.BotsImportError:
botsglobal.logger.info(_(u'In acceptance test there is no script file "bots_acceptancetest.py" to check the results of the acceptance test.'))
try:
#~ botslib.prepare_confirmrules()
#in acceptance tests: run a user script before running eg to clean output directories******************************
botslib.tryrunscript(acceptance_userscript,acceptance_scriptname,'pretest')
botslib.tryrunscript(userscript,scriptname,'pre')
errorinrun = engine2_run()
except Exception as msg:
botsglobal.logger.exception(_(u'Severe error in bots system:\n%(msg)s'),{'msg':unicode(msg)}) #of course this 'should' not happen.
sys.exit(1)
else:
if errorinrun:
sys.exit(2) #indicate: error(s) in run(s)
else:
sys.exit(0) #OK
def start():
''' sysexit codes:
0: OK, no errors
1: (system) errors incl parsing of command line arguments
2: bots ran OK, but there are errors/process errors in the run
3: Database is locked, but "maxruntime" has not been exceeded.
'''
#NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
#********command line arguments**************************
usage = '''
This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
Does the actual translations and communications; it's the workhorse. It does not have a fancy interface.
Usage:
%(name)s [run-options] [config-option] [routes]
Run-options (can be combined):
--new receive new edi files (default: if no run-option given: run as new).
--resend resend as indicated by user.
--rereceive rereceive as indicated by user.
--automaticretrycommunication - automatically retry outgoing communication.
--cleanup remove older data from database.
Config-option:
-c<directory> directory for configuration files (default: config).
Routes: list of routes to run. Default: all active routes (in the database)
''' % {
'name': os.path.basename(sys.argv[0]),
'version': botsglobal.version
}
configdir = 'config'
commandspossible = [
'--automaticretrycommunication', '--resend', '--rereceive', '--new'
]
commandstorun = []
routestorun = [] #list with routes to run
do_cleanup_parameter = False
for arg in sys.argv[1:]:
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Error: configuration directory indicated, but no directory name.'
sys.exit(1)
elif arg in commandspossible:
commandstorun.append(arg)
elif arg == '--cleanup':
do_cleanup_parameter = True
elif arg in ["?", "/?", '-h', '--help'] or arg.startswith('-'):
print usage
sys.exit(0)
else: #pick up names of routes to run
routestorun.append(arg)
if not commandstorun and not do_cleanup_parameter: #if no command on command line, use new (default)
commandstorun = ['--new']
commandstorun = [
command[2:] for command in commandspossible if command in commandstorun
] #sort commands
#***********end handling command line arguments**************************
botsinit.generalinit(
configdir) #find locating of bots, configfiles, init paths etc.
#set working directory to bots installation. advantage: when using relative paths it is clear that this point paths within bots installation.
os.chdir(botsglobal.ini.get('directories', 'botspath'))
#**************check if another instance of bots-engine is running/if port is free******************************
try:
engine_socket = botslib.check_if_other_engine_is_running()
except socket.error:
sys.exit(3)
else:
atexit.register(engine_socket.close)
#**************initialise logging******************************
process_name = 'engine'
botsglobal.logger = botsinit.initenginelogging(process_name)
atexit.register(logging.shutdown)
for key, value in botslib.botsinfo(
): #log info about environement, versions, etc
botsglobal.logger.info(u'%(key)s: "%(value)s".', {
'key': key,
'value': value
})
#**************connect to database**********************************
try:
botsinit.connect()
except Exception as msg:
botsglobal.logger.exception(
_(u'Could not connect to database. Database settings are in bots/config/settings.py. Error: "%(msg)s".'
), {'msg': msg})
sys.exit(1)
else:
botsglobal.logger.info(_(u'Connected to database.'))
atexit.register(botsglobal.db.close)
#************initialise user exits for the whole bots-engine*************************
try:
userscript, scriptname = botslib.botsimport('routescripts',
'botsengine')
except botslib.BotsImportError: #userscript is not there; other errors like syntax errors are not catched
userscript = scriptname = None
#***acceptance tests: initialiase acceptance user script******************************
acceptance_userscript = acceptance_scriptname = None
if botsglobal.ini.getboolean('acceptance', 'runacceptancetest', False):
botsglobal.logger.info(
_(u'This run is an acceptance test - as indicated in option "runacceptancetest" in bots.ini.'
))
try:
acceptance_userscript, acceptance_scriptname = botslib.botsimport(
'routescripts', 'bots_acceptancetest')
except botslib.BotsImportError:
botsglobal.logger.info(
_(u'In acceptance test there is no script file "bots_acceptancetest.py" to check the results of the acceptance test.'
))
#**************handle database lock****************************************
#set a lock on the database; if not possible, the database is locked: an earlier instance of bots-engine was terminated unexpectedly.
if not botslib.set_database_lock():
#for SQLite: do a integrity check on the database
if botsglobal.settings.DATABASES['default'][
'ENGINE'] == 'django.db.backends.sqlite3':
cursor = botsglobal.db.execute('''PRAGMA integrity_check''')
result = cursor.fetchone()
if result[0] != u'ok':
warn = _(u'!Bots database is locked!\n'\
'Bots did an integrity check on the database, but database was not OK.\n'\
'Manual action is needed!\n'\
'Bots has stopped processing EDI files.')
botsglobal.logger.critical(warn)
botslib.sendbotserrorreport(
_(u'[Bots severe error]Database is damaged'), warn)
sys.exit(1)
warn = _(u'!Bots database is locked!\n'\
'Bots-engine has ended in an unexpected way during the last run.\n'\
'Most likely causes: sudden power-down, system crash, problems with disk I/O, bots-engine terminated by user, etc.\n'
'Bots will do an automatic crash recovery now.')
botsglobal.logger.critical(warn)
botslib.sendbotserrorreport(
_(u'[Bots severe error]Database is locked'), warn)
commandstorun.insert(
0, 'crashrecovery'
) #there is a database lock. Add a crashrecovery as first command to run.
atexit.register(botslib.remove_database_lock)
warnings.simplefilter('error', UnicodeWarning)
#**************run the routes**********************************************
#commandstorun determines the type(s) of run. eg: ['automaticretrycommunication','new']
try:
botslib.prepare_confirmrules()
#in acceptance tests: run a user script before running eg to clean output directories******************************
botslib.tryrunscript(acceptance_userscript,
acceptance_scriptname,
'pretest',
routestorun=routestorun)
botslib.tryrunscript(userscript,
scriptname,
'pre',
commandstorun=commandstorun,
routestorun=routestorun)
errorinrun = 0 #detect if there has been some error. Only used for correct exit() code
first_command_2_run = True
for command in commandstorun:
#if multiple commands in run: reports etc are based on timestamp; so there needs to be at least one second between these runs.
if first_command_2_run:
first_command_2_run = False
else:
time.sleep(1)
botsglobal.logger.info(_(u'Run "%(command)s".'),
{'command': command})
#************get list of routes to run*******************************
if routestorun:
use_routestorun = routestorun[:]
botsglobal.logger.info(
_(u'Run routes from command line: "%(routes)s".'),
{'routes': unicode(use_routestorun)})
elif command == 'new': #fetch all active routes from database unless 'not in default run' or not active.
use_routestorun = []
for row in botslib.query(
'''SELECT DISTINCT idroute
FROM routes
WHERE active=%(active)s
AND (notindefaultrun=%(notindefaultrun)s OR notindefaultrun IS NULL)
ORDER BY idroute ''', {
'active': True,
'notindefaultrun': False
}):
use_routestorun.append(row['idroute'])
botsglobal.logger.info(
_(u'Run active routes from database that are in default run: "%(routes)s".'
), {'routes': unicode(use_routestorun)})
else: #for command other than 'new': use all active routes.
use_routestorun = []
for row in botslib.query(
'''SELECT DISTINCT idroute
FROM routes
WHERE active=%(active)s
ORDER BY idroute ''',
{'active': True}):
use_routestorun.append(row['idroute'])
botsglobal.logger.info(
_(u'Run all active routes from database: "%(routes)s".'),
{'routes': unicode(use_routestorun)})
#************run routes for this command******************************
botslib.tryrunscript(userscript,
scriptname,
'pre' + command,
routestorun=use_routestorun)
errorinrun += router.rundispatcher(command, use_routestorun)
botslib.tryrunscript(userscript,
scriptname,
'post' + command,
routestorun=use_routestorun)
#*********finished running routes for this command****************************
#*********finished all commands****************************************
botslib.tryrunscript(userscript,
scriptname,
'post',
commandstorun=commandstorun,
routestorun=routestorun)
try: #in acceptance tests: run a user script. no good reporting of errors/results in post-test script. Reason: this is after automaticmaintence.
botslib.tryrunscript(acceptance_userscript,
acceptance_scriptname,
'posttest',
routestorun=use_routestorun)
except Exception as msg:
print unicode(msg)
cleanup.cleanup(do_cleanup_parameter, userscript, scriptname)
except Exception as msg:
botsglobal.logger.exception(
_(u'Severe error in bots system:\n%(msg)s'),
{'msg': unicode(msg)}) #of course this 'should' not happen.
sys.exit(1)
else:
if errorinrun:
sys.exit(2) #indicate: error(s) in run(s)
else:
sys.exit(0) #OK
def run(self):
if not self.ta_info['topartner'] or not self.ta_info['frompartner']:
raise botslib.OutMessageError(
_(u'In enveloping "frompartner" or "topartner" unknown: "%(ta_info)s".'
), {'ta_info': self.ta_info})
self._openoutenvelope()
self.ta_info.update(self.out.ta_info)
botslib.tryrunscript(self.userscript,
self.scriptname,
'ta_infocontent',
ta_info=self.ta_info)
#prepare data for envelope
if botsglobal.ini.getboolean('settings',
'interchangecontrolperpartner', False):
self.ta_info['reference'] = unicode(
botslib.unique('stxcounter_' + self.ta_info['topartner']))
else:
self.ta_info['reference'] = unicode(
botslib.unique('stxcounter_' + self.ta_info['frompartner']))
#build the envelope segments (that is, the tree from which the segments will be generated)
self.out.put({
'BOTSID': 'STX',
'STDS1': self.ta_info['STX.STDS1'],
'STDS2': self.ta_info['STX.STDS2'],
'FROM.01': self.ta_info['frompartner'],
'UNTO.01': self.ta_info['topartner'],
'TRDT.01': botslib.strftime('%y%m%d'),
'TRDT.02': botslib.strftime('%H%M%S'),
'SNRF': self.ta_info['reference']
})
if self.ta_info['STX.FROM.02']:
self.out.put({
'BOTSID': 'STX',
'FROM.02': self.ta_info['STX.FROM.02']
})
if self.ta_info['STX.UNTO.02']:
self.out.put({
'BOTSID': 'STX',
'UNTO.02': self.ta_info['STX.UNTO.02']
})
if self.ta_info['STX.APRF']:
self.out.put({'BOTSID': 'STX', 'APRF': self.ta_info['STX.APRF']})
if self.ta_info['STX.PRCD']:
self.out.put({'BOTSID': 'STX', 'PRCD': self.ta_info['STX.PRCD']})
self.out.put({'BOTSID': 'STX'}, {
'BOTSID': 'END',
'NMST': self.ta_info['nrmessages']
}) #dummy segment; is not used
#user exit
botslib.tryrunscript(self.userscript,
self.scriptname,
'envelopecontent',
ta_info=self.ta_info,
out=self.out)
#convert the tree into segments; here only the STX is written (first segment)
self.out.checkmessage(self.out.root, self.out.defmessage)
self.out.checkforerrorlist()
self.out.tree2records(self.out.root)
#start doing the actual writing:
tofile = botslib.opendata(self.ta_info['filename'], 'wb',
self.ta_info['charset'])
tofile.write(self.out.record2string(self.out.lex_records[0:1]))
self.writefilelist(tofile)
tofile.write(self.out.record2string(self.out.lex_records[1:2]))
tofile.close()
def run(self):
if not self.ta_info['topartner'] or not self.ta_info['frompartner']:
raise botslib.OutMessageError(
_(u'In enveloping "frompartner" or "topartner" unknown: "%(ta_info)s".'
), {'ta_info': self.ta_info})
self._openoutenvelope()
self.ta_info.update(self.out.ta_info)
botslib.tryrunscript(self.userscript,
self.scriptname,
'ta_infocontent',
ta_info=self.ta_info)
#version dependent enveloping
if self.ta_info['version'] < '4':
date = botslib.strftime('%y%m%d')
reserve = ' '
else:
date = botslib.strftime('%Y%m%d')
reserve = self.ta_info['reserve']
#UNB reference is counter is per sender or receiver
if botsglobal.ini.getboolean('settings',
'interchangecontrolperpartner', False):
self.ta_info['reference'] = unicode(
botslib.unique('unbcounter_' + self.ta_info['topartner']))
else:
self.ta_info['reference'] = unicode(
botslib.unique('unbcounter_' + self.ta_info['frompartner']))
#testindicator is more complex:
if self.ta_info['testindicator'] and self.ta_info[
'testindicator'] != '0': #first check value from ta; do not use default
testindicator = '1'
elif self.ta_info['UNB.0035'] != '0': #than check values from grammar
testindicator = '1'
else:
testindicator = ''
#build the envelope segments (that is, the tree from which the segments will be generated)
self.out.put({
'BOTSID': 'UNB',
'S001.0001': self.ta_info['charset'],
'S001.0002': self.ta_info['version'],
'S002.0004': self.ta_info['frompartner'],
'S003.0010': self.ta_info['topartner'],
'S004.0017': date,
'S004.0019': botslib.strftime('%H%M'),
'0020': self.ta_info['reference']
})
#the following fields are conditional; do not write these when empty string (separator compression does take empty strings into account)
for field in ('S001.0080', 'S001.0133', 'S002.0007', 'S002.0008',
'S002.0042', 'S003.0007', 'S003.0014', 'S003.0046',
'S005.0022', 'S005.0025', '0026', '0029', '0031',
'0032'):
if self.ta_info['UNB.' + field]:
self.out.put({
'BOTSID': 'UNB',
field: self.ta_info['UNB.' + field]
})
if testindicator:
self.out.put({'BOTSID': 'UNB', '0035': testindicator})
self.out.put({'BOTSID': 'UNB'}, {
'BOTSID': 'UNZ',
'0036': self.ta_info['nrmessages'],
'0020': self.ta_info['reference']
}) #dummy segment; is not used
#user exit
botslib.tryrunscript(self.userscript,
self.scriptname,
'envelopecontent',
ta_info=self.ta_info,
out=self.out)
#convert the tree into segments; here only the UNB is written (first segment)
self.out.checkmessage(self.out.root, self.out.defmessage)
self.out.checkforerrorlist()
self.out.tree2records(self.out.root)
#start doing the actual writing:
tofile = botslib.opendata(self.ta_info['filename'], 'wb',
self.ta_info['charset'])
if self.ta_info['forceUNA'] or self.ta_info['charset'] != 'UNOA':
tofile.write('UNA' + self.ta_info['sfield_sep'] +
self.ta_info['field_sep'] + self.ta_info['decimaal'] +
self.ta_info['escape'] + reserve +
self.ta_info['record_sep'] +
self.ta_info['add_crlfafterrecord_sep'])
tofile.write(self.out.record2string(self.out.lex_records[0:1]))
self.writefilelist(tofile)
tofile.write(self.out.record2string(self.out.lex_records[1:2]))
tofile.close()
def run(self):
self._openoutenvelope(self.ta_info['editype'],
self.ta_info['envelope'])
self.ta_info.update(self.out.ta_info)
botslib.tryrunscript(self.userscript,
self.scriptname,
'ta_infocontent',
ta_info=self.ta_info)
#version dependent enveloping
writeUNA = False
if self.ta_info['version'] < '4':
date = time.strftime('%y%m%d')
reserve = ' '
if self.ta_info['charset'] != 'UNOA':
writeUNA = True
else:
date = time.strftime('%Y%m%d')
reserve = self.ta_info['reserve']
if self.ta_info['charset'] not in ['UNOA', 'UNOB']:
writeUNA = True
#UNB counter is per sender or receiver
if botsglobal.ini.getboolean('settings',
'interchangecontrolperpartner', False):
self.ta_info['reference'] = str(
botslib.unique('unbcounter_' + self.ta_info['topartner']))
else:
self.ta_info['reference'] = str(
botslib.unique('unbcounter_' + self.ta_info['frompartner']))
#testindicator is more complex:
if self.ta_info['testindicator'] and self.ta_info[
'testindicator'] != '0': #first check value from ta; do not use default
testindicator = '1'
elif self.ta_info['UNB.0035'] != '0': #than check values from grammar
testindicator = '1'
else:
testindicator = ''
#build the envelope segments (that is, the tree from which the segments will be generated)
self.out.put({
'BOTSID': 'UNB',
'S001.0001': self.ta_info['charset'],
'S001.0002': self.ta_info['version'],
'S002.0004': self.ta_info['frompartner'],
'S003.0010': self.ta_info['topartner'],
'S004.0017': date,
'S004.0019': time.strftime('%H%M'),
'0020': self.ta_info['reference']
})
#the following fields are conditional; do not write these when empty string (separator compression does take empty strings into account)
if self.ta_info['UNB.S002.0007']:
self.out.put({
'BOTSID': 'UNB',
'S002.0007': self.ta_info['UNB.S002.0007']
})
if self.ta_info['UNB.S003.0007']:
self.out.put({
'BOTSID': 'UNB',
'S003.0007': self.ta_info['UNB.S003.0007']
})
if self.ta_info['UNB.0026']:
self.out.put({'BOTSID': 'UNB', '0026': self.ta_info['UNB.0026']})
if testindicator:
self.out.put({'BOTSID': 'UNB', '0035': testindicator})
self.out.put({'BOTSID': 'UNB'}, {
'BOTSID': 'UNZ',
'0036': self.ta_info['nrmessages'],
'0020': self.ta_info['reference']
}) #dummy segment; is not used
#user exit
botslib.tryrunscript(self.userscript,
self.scriptname,
'envelopecontent',
ta_info=self.ta_info,
out=self.out)
#convert the tree into segments; here only the UNB is written (first segment)
self.out.normalisetree(self.out.root)
self.out.tree2records(self.out.root)
#start doing the actual writing:
tofile = botslib.opendata(self.ta_info['filename'], 'wb',
self.ta_info['charset'])
if writeUNA or self.ta_info['forceUNA']:
tofile.write('UNA' + self.ta_info['sfield_sep'] +
self.ta_info['field_sep'] + self.ta_info['decimaal'] +
self.ta_info['escape'] + reserve +
self.ta_info['record_sep'] +
self.ta_info['add_crlfafterrecord_sep'])
tofile.write(self.out._record2string(self.out.records[0]))
self.writefilelist(tofile)
tofile.write(self.out._record2string(self.out.records[-1]))
tofile.close()
def router(routedict):
''' communication.run one route. variants:
- a route can be just script;
- a route can do only incoming
- a route can do only outgoing
- a route can do both incoming and outgoing
- at several points functions from a route script are called - if function is in route script
'''
#is there a user route script?
try:
userscript,scriptname = botslib.botsimport('routescripts',routedict['idroute'])
except ImportError: #other errors, eg syntax errors are just passed
userscript = scriptname = None
#if user route script has function 'main': communication.run 'main' (and do nothing else)
if botslib.tryrunscript(userscript,scriptname,'main',routedict=routedict):
return #so: if function ' main' : communication.run only the routescript, nothing else.
if not (userscript or routedict['fromchannel'] or routedict['tochannel'] or routedict['translateind']):
raise botslib.ScriptError(_(u'Route "$route" is empty: no script, not enough parameters.'),route=routedict['idroute'])
botslib.tryrunscript(userscript,scriptname,'start',routedict=routedict)
#communication.run incoming channel
if routedict['fromchannel']: #do incoming part of route: in-communication; set ready for translation; translate
botslib.tryrunscript(userscript,scriptname,'preincommunication',routedict=routedict)
communication.run(idchannel=routedict['fromchannel'],idroute=routedict['idroute']) #communication.run incommunication
#add attributes from route to the received files
where = {'status':FILEIN,'fromchannel':routedict['fromchannel'],'idroute':routedict['idroute']}
change = {'editype':routedict['fromeditype'],'messagetype':routedict['frommessagetype'],'frompartner':routedict['frompartner'],'topartner':routedict['topartner'],'alt':routedict['alt']}
botslib.updateinfo(change=change,where=where)
#all received files have status FILEIN
botslib.tryrunscript(userscript,scriptname,'postincommunication',routedict=routedict)
if routedict['fromeditype'] == 'mailbag': #mailbag for the route.
preprocess.preprocess(routedict,preprocess.mailbag)
#communication.run translation
if routedict['translateind']:
botslib.tryrunscript(userscript,scriptname,'pretranslation',routedict=routedict)
botslib.addinfo(change={'status':TRANSLATE},where={'status':FILEIN,'idroute':routedict['idroute']})
transform.translate(idroute=routedict['idroute'])
botslib.tryrunscript(userscript,scriptname,'posttranslation',routedict=routedict)
#merge messages & communication.run outgoing channel
if routedict['tochannel']: #do outgoing part of route
botslib.tryrunscript(userscript,scriptname,'premerge',routedict=routedict)
envelope.mergemessages(idroute=routedict['idroute'])
botslib.tryrunscript(userscript,scriptname,'postmerge',routedict=routedict)
#communication.run outgoing channel
#build for query: towhere (dict) and wherestring
towhere = dict(status=MERGED,
idroute=routedict['idroute'],
editype=routedict['toeditype'],
messagetype=routedict['tomessagetype'],
testindicator=routedict['testindicator'])
towhere = dict([(key, value) for (key, value) in towhere.iteritems() if value]) #remove nul-values from dict
wherestring = ' AND '.join([key+'=%('+key+')s' for key in towhere])
if routedict['frompartner_tochannel_id']: #use frompartner_tochannel in where-clause of query (partner/group dependent outchannel
towhere['frompartner_tochannel_id'] = routedict['frompartner_tochannel_id']
wherestring += ''' AND (frompartner=%(frompartner_tochannel_id)s
OR frompartner in (SELECT from_partner_id
FROM partnergroup
WHERE to_partner_id =%(frompartner_tochannel_id)s ))'''
if routedict['topartner_tochannel_id']: #use topartner_tochannel in where-clause of query (partner/group dependent outchannel
towhere['topartner_tochannel_id'] = routedict['topartner_tochannel_id']
wherestring += ''' AND (topartner=%(topartner_tochannel_id)s
OR topartner in (SELECT from_partner_id
FROM partnergroup
WHERE to_partner_id=%(topartner_tochannel_id)s ))'''
toset = {'tochannel':routedict['tochannel'],'status':FILEOUT}
botslib.addinfocore(change=toset,where=towhere,wherestring=wherestring)
if not routedict['defer']: #do outgoing part of route
botslib.tryrunscript(userscript,scriptname,'preoutcommunication',routedict=routedict)
communication.run(idchannel=routedict['tochannel'],idroute=routedict['idroute']) #communication.run outcommunication
botslib.tryrunscript(userscript,scriptname,'postoutcommunication',routedict=routedict)
botslib.tryrunscript(userscript,scriptname,'end',routedict=routedict)
def run(self):
if not self.ta_info["topartner"] or not self.ta_info["frompartner"]:
raise botslib.OutMessageError(
_(u'In enveloping "frompartner" or "topartner" unknown: "$ta_info".'), ta_info=self.ta_info
)
self._openoutenvelope(self.ta_info["editype"], self.ta_info["envelope"])
self.ta_info.update(self.out.ta_info)
botslib.tryrunscript(self.userscript, self.scriptname, "ta_infocontent", ta_info=self.ta_info)
# version dependent enveloping
write_una = False
if self.ta_info["version"] < "4":
date = time.strftime("%y%m%d")
reserve = " "
if self.ta_info["charset"] != "UNOA":
write_una = True
else:
date = time.strftime("%Y%m%d")
reserve = self.ta_info["reserve"]
if self.ta_info["charset"] not in ["UNOA", "UNOB"]:
write_una = True
# UNB counter is per sender or receiver
if botsglobal.ini.getboolean("settings", "interchangecontrolperpartner", False):
self.ta_info["reference"] = str(botslib.unique("unbcounter_" + self.ta_info["topartner"]))
else:
self.ta_info["reference"] = str(botslib.unique("unbcounter_" + self.ta_info["frompartner"]))
# testindicator is more complex:
if (
self.ta_info["testindicator"] and self.ta_info["testindicator"] != "0"
): # first check value from ta; do not use default
testindicator = "1"
elif self.ta_info["UNB.0035"] != "0": # than check values from grammar
testindicator = "1"
else:
testindicator = ""
# build the envelope segments (that is, the tree from which the segments will be generated)
self.out.put(
{
"BOTSID": "UNB",
"S001.0001": self.ta_info["charset"],
"S001.0002": self.ta_info["version"],
"S002.0004": self.ta_info["frompartner"],
"S003.0010": self.ta_info["topartner"],
"S004.0017": date,
"S004.0019": time.strftime("%H%M"),
"0020": self.ta_info["reference"],
}
)
# the following fields are conditional; do not write these when empty string (separator compression does take empty strings into account)
for field in (
"S001.0080",
"S001.0133",
"S002.0007",
"S002.0008",
"S002.0042",
"S003.0007",
"S003.0014",
"S003.0046",
"S005.0022",
"S005.0025",
"0026",
"0029",
"0031",
"0032",
):
if self.ta_info["UNB." + field]:
self.out.put({"BOTSID": "UNB", field: self.ta_info["UNB." + field]})
if testindicator:
self.out.put({"BOTSID": "UNB", "0035": testindicator})
self.out.put(
{"BOTSID": "UNB"}, {"BOTSID": "UNZ", "0036": self.ta_info["nrmessages"], "0020": self.ta_info["reference"]}
) # dummy segment; is not used
# user exit
botslib.tryrunscript(self.userscript, self.scriptname, "envelopecontent", ta_info=self.ta_info, out=self.out)
# convert the tree into segments; here only the UNB is written (first segment)
self.out.checkmessage(self.out.root, self.out.defmessage)
self.out.tree2records(self.out.root)
# start doing the actual writing:
tofile = botslib.opendata(self.ta_info["filename"], "wb", self.ta_info["charset"])
if write_una or self.ta_info["forceUNA"]:
tofile.write(
"UNA"
+ self.ta_info["sfield_sep"]
+ self.ta_info["field_sep"]
+ self.ta_info["decimaal"]
+ self.ta_info["escape"]
+ reserve
+ self.ta_info["record_sep"]
+ self.ta_info["add_crlfafterrecord_sep"]
)
tofile.write(self.out.record2string(self.out.records[0]))
self.writefilelist(tofile)
tofile.write(self.out.record2string(self.out.records[-1]))
tofile.close()
if self.ta_info["messagetype"][:6] != "CONTRL" and botslib.checkconfirmrules(
"ask-edifact-CONTRL",
idroute=self.ta_info["idroute"],
idchannel=self.ta_info["tochannel"],
topartner=self.ta_info["topartner"],
frompartner=self.ta_info["frompartner"],
editype=self.ta_info["editype"],
messagetype=self.ta_info["messagetype"],
):
self.ta_info["confirmtype"] = u"ask-edifact-CONTRL"
self.ta_info["confirmasked"] = True
def run(self):
self._openoutenvelope(self.ta_info['editype'],self.ta_info['envelope'])
self.ta_info.update(self.out.ta_info)
botslib.tryrunscript(self.userscript,self.scriptname,'ta_infocontent',ta_info=self.ta_info)
#version dependent enveloping
writeUNA = False
if self.ta_info['version']<'4':
date = time.strftime('%y%m%d')
reserve = ' '
if self.ta_info['charset'] != 'UNOA':
writeUNA = True
else:
date = time.strftime('%Y%m%d')
reserve = self.ta_info['reserve']
if self.ta_info['charset'] not in ['UNOA','UNOB']:
writeUNA = True
#UNB counter is per sender or receiver
if botsglobal.ini.getboolean('settings','interchangecontrolperpartner',False):
self.ta_info['reference'] = str(botslib.unique('unbcounter_' + self.ta_info['topartner']))
else:
self.ta_info['reference'] = str(botslib.unique('unbcounter_' + self.ta_info['frompartner']))
#testindicator is more complex:
if self.ta_info['testindicator'] and self.ta_info['testindicator']!='0': #first check value from ta; do not use default
testindicator = '1'
elif self.ta_info['UNB.0035'] != '0': #than check values from grammar
testindicator = '1'
else:
testindicator = ''
#build the envelope segments (that is, the tree from which the segments will be generated)
self.out.put({'BOTSID':'UNB',
'S001.0001':self.ta_info['charset'],
'S001.0002':self.ta_info['version'],
'S002.0004':self.ta_info['frompartner'],
'S003.0010':self.ta_info['topartner'],
'S004.0017':date,
'S004.0019':time.strftime('%H%M'),
'0020':self.ta_info['reference']})
#the following fields are conditional; do not write these when empty string (separator compression does take empty strings into account)
if self.ta_info['UNB.S002.0007']:
self.out.put({'BOTSID':'UNB','S002.0007': self.ta_info['UNB.S002.0007']})
if self.ta_info['UNB.S003.0007']:
self.out.put({'BOTSID':'UNB','S003.0007': self.ta_info['UNB.S003.0007']})
if self.ta_info['UNB.0026']:
self.out.put({'BOTSID':'UNB','0026': self.ta_info['UNB.0026']})
if testindicator:
self.out.put({'BOTSID':'UNB','0035': testindicator})
self.out.put({'BOTSID':'UNB'},{'BOTSID':'UNZ','0036':self.ta_info['nrmessages'],'0020':self.ta_info['reference']}) #dummy segment; is not used
#user exit
botslib.tryrunscript(self.userscript,self.scriptname,'envelopecontent',ta_info=self.ta_info,out=self.out)
#convert the tree into segments; here only the UNB is written (first segment)
self.out.normalisetree(self.out.root)
self.out.tree2records(self.out.root)
#start doing the actual writing:
tofile = botslib.opendata(self.ta_info['filename'],'wb',self.ta_info['charset'])
if writeUNA or self.ta_info['forceUNA']:
tofile.write('UNA'+self.ta_info['sfield_sep']+self.ta_info['field_sep']+self.ta_info['decimaal']+self.ta_info['escape']+ reserve +self.ta_info['record_sep']+self.ta_info['add_crlfafterrecord_sep'])
tofile.write(self.out._record2string(self.out.records[0]))
self.writefilelist(tofile)
tofile.write(self.out._record2string(self.out.records[-1]))
tofile.close()
def run(self):
if not self.ta_info['topartner'] or not self.ta_info['frompartner']:
raise botslib.OutMessageError(_(u'In enveloping "frompartner" or "topartner" unknown: "%(ta_info)s".'),
{'ta_info':self.ta_info})
self._openoutenvelope()
self.ta_info.update(self.out.ta_info)
botslib.tryrunscript(self.userscript,self.scriptname,'ta_infocontent',ta_info=self.ta_info)
#prepare data for envelope
isa09date = botslib.strftime('%y%m%d')
#test indicator can either be from configuration (self.ta_info['ISA15']) or by mapping (self.ta_info['testindicator'])
#mapping overrules.
if self.ta_info['testindicator'] and self.ta_info['testindicator'] != '0': #'0' is default value (in db)
testindicator = self.ta_info['testindicator']
else:
testindicator = self.ta_info['ISA15']
#~ print self.ta_info['messagetype'], 'grammar:',self.ta_info['ISA15'],'ta:',self.ta_info['testindicator'],'out:',testindicator
if botsglobal.ini.getboolean('settings','interchangecontrolperpartner',False):
self.ta_info['reference'] = unicode(botslib.unique('isacounter_' + self.ta_info['topartner']))
else:
self.ta_info['reference'] = unicode(botslib.unique('isacounter_' + self.ta_info['frompartner']))
#ISA06 and GS02 can be different; eg ISA06 is a service provider.
#ISA06 and GS02 can be in the syntax....
isa06sender = self.ta_info.get('ISA06',self.ta_info['frompartner'])
isa06sender = isa06sender.ljust(15) #add spaces; is fixed length
gs02sender = self.ta_info.get('GS02',self.ta_info['frompartner'])
#also for ISA08 and GS03
isa08receiver = self.ta_info.get('ISA08',self.ta_info['topartner'])
isa08receiver = isa08receiver.ljust(15) #add spaces; is fixed length
gs03receiver = self.ta_info.get('GS03',self.ta_info['topartner'])
#build the envelope segments (that is, the tree from which the segments will be generated)
self.out.put({'BOTSID':'ISA',
'ISA01':self.ta_info['ISA01'],
'ISA02':self.ta_info['ISA02'],
'ISA03':self.ta_info['ISA03'],
'ISA04':self.ta_info['ISA04'],
'ISA05':self.ta_info['ISA05'],
'ISA06':isa06sender,
'ISA07':self.ta_info['ISA07'],
'ISA08':isa08receiver,
'ISA09':isa09date,
'ISA10':botslib.strftime('%H%M'),
'ISA11':self.ta_info['ISA11'], #if ISA version > 00403, replaced by reprtion separator
'ISA12':self.ta_info['version'],
'ISA13':self.ta_info['reference'],
'ISA14':self.ta_info['ISA14'],
'ISA15':testindicator},strip=False) #MIND: strip=False: ISA fields shoudl not be stripped as it is soemwhat like fixed-length
self.out.put({'BOTSID':'ISA'},{'BOTSID':'IEA','IEA01':'1','IEA02':self.ta_info['reference']})
gs08messagetype = self.ta_info['messagetype'][3:]
if gs08messagetype[:6] < '004010':
gs04date = botslib.strftime('%y%m%d')
else:
gs04date = botslib.strftime('%Y%m%d')
self.out.put({'BOTSID':'ISA'},{'BOTSID':'GS',
'GS01':self.ta_info['functionalgroup'],
'GS02':gs02sender,
'GS03':gs03receiver,
'GS04':gs04date,
'GS05':botslib.strftime('%H%M'),
'GS06':self.ta_info['reference'],
'GS07':self.ta_info['GS07'],
'GS08':gs08messagetype})
self.out.put({'BOTSID':'ISA'},{'BOTSID':'GS'},{'BOTSID':'GE','GE01':self.ta_info['nrmessages'],'GE02':self.ta_info['reference']}) #dummy segment; is not used
#user exit
botslib.tryrunscript(self.userscript,self.scriptname,'envelopecontent',ta_info=self.ta_info,out=self.out)
#convert the tree into segments; here only the UNB is written (first segment)
self.out.checkmessage(self.out.root,self.out.defmessage)
self.out.checkforerrorlist()
self.out.tree2records(self.out.root)
#start doing the actual writing:
tofile = botslib.opendata(self.ta_info['filename'],'wb',self.ta_info['charset'])
isa_string = self.out.record2string(self.out.lex_records[0:1])
#ISA has the used separators at certain positions. Normally bots would give errors for this (can not use sep as data) or compress these aways. So this is hardcoded.
if self.ta_info['version'] < '00403':
isa_string = isa_string[:103] + self.ta_info['field_sep']+ self.ta_info['sfield_sep'] + isa_string[103:]
else:
isa_string = isa_string[:82] +self.ta_info['reserve'] + isa_string[83:103] + self.ta_info['field_sep']+ self.ta_info['sfield_sep'] + isa_string[103:]
tofile.write(isa_string) #write ISA
tofile.write(self.out.record2string(self.out.lex_records[1:2])) #write GS
self.writefilelist(tofile)
tofile.write(self.out.record2string(self.out.lex_records[2:])) #write GE and IEA
tofile.close()
def run(self):
if not self.ta_info["topartner"] or not self.ta_info["frompartner"]:
raise botslib.OutMessageError(
_(u'In enveloping "frompartner" or "topartner" unknown: "$ta_info".'), ta_info=self.ta_info
)
self._openoutenvelope(self.ta_info["editype"], self.ta_info["envelope"])
self.ta_info.update(self.out.ta_info)
# need to know the functionalgroup code:
defmessage = grammar.grammarread(self.ta_info["editype"], self.ta_info["messagetype"])
self.ta_info["functionalgroup"] = defmessage.syntax["functionalgroup"]
botslib.tryrunscript(self.userscript, self.scriptname, "ta_infocontent", ta_info=self.ta_info)
# prepare data for envelope
isa09date = time.strftime("%y%m%d")
# test indicator can either be from configuration (self.ta_info['ISA15']) or by mapping (self.ta_info['testindicator'])
# mapping overrules.
if self.ta_info["testindicator"] and self.ta_info["testindicator"] != "0": #'0' is default value (in db)
testindicator = self.ta_info["testindicator"]
else:
testindicator = self.ta_info["ISA15"]
# ~ print self.ta_info['messagetype'], 'grammar:',self.ta_info['ISA15'],'ta:',self.ta_info['testindicator'],'out:',testindicator
if botsglobal.ini.getboolean("settings", "interchangecontrolperpartner", False):
self.ta_info["reference"] = str(botslib.unique("isacounter_" + self.ta_info["topartner"]))
else:
self.ta_info["reference"] = str(botslib.unique("isacounter_" + self.ta_info["frompartner"]))
# ISA06 and GS02 can be different; eg ISA06 is a service provider.
# ISA06 and GS02 can be in the syntax....
isa06sender = self.ta_info.get("ISA06", self.ta_info["frompartner"])
isa06sender = isa06sender.ljust(15) # add spaces; is fixed length
gs02sender = self.ta_info.get("GS02", self.ta_info["frompartner"])
# also for ISA08 and GS03
isa08receiver = self.ta_info.get("ISA08", self.ta_info["topartner"])
isa08receiver = isa08receiver.ljust(15) # add spaces; is fixed length
gs03receiver = self.ta_info.get("GS03", self.ta_info["topartner"])
# build the envelope segments (that is, the tree from which the segments will be generated)
self.out.put(
{
"BOTSID": "ISA",
"ISA01": self.ta_info["ISA01"],
"ISA02": self.ta_info["ISA02"],
"ISA03": self.ta_info["ISA03"],
"ISA04": self.ta_info["ISA04"],
"ISA05": self.ta_info["ISA05"],
"ISA06": isa06sender,
"ISA07": self.ta_info["ISA07"],
"ISA08": isa08receiver,
"ISA09": isa09date,
"ISA10": time.strftime("%H%M"),
"ISA11": self.ta_info["ISA11"], # if ISA version > 00403, replaced by reprtion separator
"ISA12": self.ta_info["version"],
"ISA13": self.ta_info["reference"],
"ISA14": self.ta_info["ISA14"],
"ISA15": testindicator,
},
strip=False,
) # MIND: strip=False: ISA fields shoudl not be stripped as it is soemwhat like fixed-length
self.out.put({"BOTSID": "ISA"}, {"BOTSID": "IEA", "IEA01": "1", "IEA02": self.ta_info["reference"]})
gs08messagetype = self.ta_info["messagetype"][3:]
if gs08messagetype[:6] < "004010":
gs04date = time.strftime("%y%m%d")
else:
gs04date = time.strftime("%Y%m%d")
self.out.put(
{"BOTSID": "ISA"},
{
"BOTSID": "GS",
"GS01": self.ta_info["functionalgroup"],
"GS02": gs02sender,
"GS03": gs03receiver,
"GS04": gs04date,
"GS05": time.strftime("%H%M"),
"GS06": self.ta_info["reference"],
"GS07": self.ta_info["GS07"],
"GS08": gs08messagetype,
},
)
self.out.put(
{"BOTSID": "ISA"},
{"BOTSID": "GS"},
{"BOTSID": "GE", "GE01": self.ta_info["nrmessages"], "GE02": self.ta_info["reference"]},
) # dummy segment; is not used
# user exit
botslib.tryrunscript(self.userscript, self.scriptname, "envelopecontent", ta_info=self.ta_info, out=self.out)
# convert the tree into segments; here only the UNB is written (first segment)
self.out.checkmessage(self.out.root, self.out.defmessage)
self.out.tree2records(self.out.root)
# start doing the actual writing:
tofile = botslib.opendata(self.ta_info["filename"], "wb", self.ta_info["charset"])
isa_string = self.out.record2string(self.out.records[0])
if self.ta_info["version"] < "00403":
isa_string = (
isa_string[:103] + self.ta_info["field_sep"] + self.ta_info["sfield_sep"] + isa_string[103:]
) # hack for strange characters at end of ISA; hardcoded
else:
isa_string = (
isa_string[:82]
+ self.ta_info["reserve"]
+ isa_string[83:103]
+ self.ta_info["field_sep"]
+ self.ta_info["sfield_sep"]
+ isa_string[103:]
) # hack for strange characters at end of ISA; hardcoded
tofile.write(isa_string) # write ISA
tofile.write(self.out.record2string(self.out.records[1])) # write GS
self.writefilelist(tofile)
tofile.write(self.out.record2string(self.out.records[-2])) # write GE
tofile.write(self.out.record2string(self.out.records[-1])) # write IEA
tofile.close()
if self.ta_info["functionalgroup"] != "FA" and botslib.checkconfirmrules(
"ask-x12-997",
idroute=self.ta_info["idroute"],
idchannel=self.ta_info["tochannel"],
topartner=self.ta_info["topartner"],
frompartner=self.ta_info["frompartner"],
editype=self.ta_info["editype"],
messagetype=self.ta_info["messagetype"],
):
self.ta_info["confirmtype"] = u"ask-x12-997"
self.ta_info["confirmasked"] = True
def run(self):
botslib.tryrunscript(self.userscript, self.scriptname, "ta_infocontent", ta_info=self.ta_info)
self.ta_info["filename"] = self.ta_list[0]
def start():
''' sysexit codes:
0: OK, no errors
1: (system) errors incl parsing of command line arguments
2: bots ran OK, but there are errors/process errors in the run
3: Database is locked, but "maxruntime" has not been exceeded.
'''
#NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
#********command line arguments**************************
usage = '''
This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
Does the actual translations and communications; it's the workhorse. It does not have a fancy interface.
Usage:
%(name)s [config-option]
Config-option:
-c<directory> directory for configuration files (default: config).
''' % {
'name': os.path.basename(sys.argv[0]),
'version': botsglobal.version
}
configdir = 'config'
for arg in sys.argv[1:]:
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Error: configuration directory indicated, but no directory name.'
sys.exit(1)
elif arg in ["?", "/?", '-h', '--help'] or arg.startswith('-'):
print usage
sys.exit(0)
#***********end handling command line arguments**************************
botsinit.generalinit(
configdir) #find locating of bots, configfiles, init paths etc.
#set working directory to bots installation. advantage: when using relative paths it is clear that this point paths within bots installation.
os.chdir(botsglobal.ini.get('directories', 'botspath'))
#**************initialise logging******************************
process_name = 'engine2'
botsglobal.logger = botsinit.initenginelogging(process_name)
atexit.register(logging.shutdown)
for key, value in botslib.botsinfo(
): #log info about environement, versions, etc
botsglobal.logger.info(u'%(key)s: "%(value)s".', {
'key': key,
'value': value
})
#**************connect to database**********************************
try:
botsinit.connect()
except Exception as msg:
botsglobal.logger.exception(
_(u'Could not connect to database. Database settings are in bots/config/settings.py. Error: "%(msg)s".'
), {'msg': msg})
sys.exit(1)
else:
botsglobal.logger.info(_(u'Connected to database.'))
atexit.register(botsglobal.db.close)
warnings.simplefilter('error', UnicodeWarning)
#import global scripts for bots-engine
try:
userscript, scriptname = botslib.botsimport('routescripts',
'botsengine')
except botslib.BotsImportError: #userscript is not there; other errors like syntax errors are not catched
userscript = scriptname = None
#***acceptance tests: initialiase acceptance user script******************************
acceptance_userscript = acceptance_scriptname = None
if botsglobal.ini.getboolean('acceptance', 'runacceptancetest', False):
botsglobal.logger.info(
_(u'This run is an acceptance test - as indicated in option "runacceptancetest" in bots.ini.'
))
try:
acceptance_userscript, acceptance_scriptname = botslib.botsimport(
'routescripts', 'bots_acceptancetest')
except botslib.BotsImportError:
botsglobal.logger.info(
_(u'In acceptance test there is no script file "bots_acceptancetest.py" to check the results of the acceptance test.'
))
try:
#~ botslib.prepare_confirmrules()
#in acceptance tests: run a user script before running eg to clean output directories******************************
botslib.tryrunscript(acceptance_userscript, acceptance_scriptname,
'pretest')
botslib.tryrunscript(userscript, scriptname, 'pre')
errorinrun = engine2_run()
except Exception as msg:
botsglobal.logger.exception(
_(u'Severe error in bots system:\n%(msg)s'),
{'msg': unicode(msg)}) #of course this 'should' not happen.
sys.exit(1)
else:
if errorinrun:
sys.exit(2) #indicate: error(s) in run(s)
else:
sys.exit(0) #OK