def preprocess(routedict, function, status=FILEIN, **argv):
''' for pre- and postprocessing of files.
these are NOT translations; translation involve grammars, mapping scripts etc. think of eg:
- unzipping zipped files.
- convert excel to csv
- password protected files.
Select files from INFILE -> SET_FOR_PROCESSING using criteria
Than the actual processing function is called.
The processing function does: SET_FOR_PROCESSING -> PROCESSING -> FILEIN
If errors occur during processing, no ta are left with status FILEIN !
preprocess is called right after the in-communicatiation
'''
nr_files = 0
preprocessnumber = botslib.getpreprocessnumber()
if not botslib.addinfo(change={'status': preprocessnumber},
where={
'status': status,
'idroute': routedict['idroute'],
'fromchannel': routedict['fromchannel']
}): #check if there is something to do
return 0
for row in botslib.query(
u'''SELECT idta,filename,charset
FROM ta
WHERE idta>%(rootidta)s
AND status=%(status)s
AND statust=%(statust)s
AND idroute=%(idroute)s
AND fromchannel=%(fromchannel)s
''', {
'status': preprocessnumber,
'statust': OK,
'idroute': routedict['idroute'],
'fromchannel': routedict['fromchannel'],
'rootidta': botslib.get_minta4query()
}):
try:
botsglobal.logmap.debug(u'Start preprocessing "%s" for file "%s".',
function.__name__, row['filename'])
ta_set_for_processing = botslib.OldTransaction(row['idta'])
ta_processing = ta_set_for_processing.copyta(
status=preprocessnumber + 1)
ta_processing.filename = row['filename']
function(ta_from=ta_processing,
endstatus=status,
routedict=routedict,
**argv)
except:
txt = botslib.txtexc()
ta_processing.failure()
ta_processing.update(statust=ERROR, errortext=txt)
else:
botsglobal.logmap.debug(u'OK preprocessing "%s" for file "%s".',
function.__name__, row['filename'])
ta_set_for_processing.update(statust=DONE)
ta_processing.update(statust=DONE)
nr_files += 1
return nr_files
def mergemessages(startstatus=TRANSLATED, endstatus=MERGED, idroute=""):
""" Merges en envelopes several messages to one file;
In db-ta: attribute 'merge' indicates message should be merged with similar messages; 'merge' is generated in translation from messagetype-grammar
If merge==False: 1 message per envelope - no merging, else append all similar messages to one file
Implementation as separate loops: one for merge&envelope, another for enveloping only
db-ta status TRANSLATED---->MERGED
"""
outerqueryparameters = {
"status": startstatus,
"statust": OK,
"idroute": idroute,
"rootidta": botslib.get_minta4query(),
"merge": False,
}
# **********for messages only to envelope (no merging)
for row in botslib.query(
u"""SELECT editype,messagetype,frompartner,topartner,testindicator,charset,contenttype,tochannel,envelope,nrmessages,idta,filename,idroute,merge
FROM ta
WHERE idta>%(rootidta)s
AND status=%(status)s
AND statust=%(statust)s
AND idroute=%(idroute)s
AND merge=%(merge)s
""",
outerqueryparameters,
):
try:
ta_info = dict([(key, row[key]) for key in row.keys()])
# ~ ta_info={'merge':False,'idroute':idroute}
# ~ for key in row.keys():
# ~ ta_info[key] = row[key]
ta_fromfile = botslib.OldTransaction(row["idta"]) # edi message to envelope
ta_tofile = ta_fromfile.copyta(
status=endstatus
) # edifile for enveloped message; attributes of not-enveloped message are copied...
# ~ ta_fromfile.update(child=ta_tofile.idta) #??there is already a parent-child relation (1-1)...
ta_info["filename"] = str(ta_tofile.idta) # create filename for enveloped message
botsglobal.logger.debug(
u"Envelope 1 message editype: %s, messagetype: %s.", ta_info["editype"], ta_info["messagetype"]
)
envelope(ta_info, [row["filename"]])
except:
txt = botslib.txtexc()
ta_tofile.update(statust=ERROR, errortext=txt)
else:
ta_fromfile.update(statust=DONE)
ta_tofile.update(statust=OK, **ta_info) # selection is used to update enveloped message;
# **********for messages to merge & envelope
# all GROUP BY fields must be used in SELECT!
# as files get merged: can not copy idta; must extract relevant attributes.
outerqueryparameters["merge"] = True
for row in botslib.query(
u"""SELECT editype,messagetype,frompartner,topartner,tochannel,testindicator,charset,contenttype,envelope,sum(nrmessages) as nrmessages
FROM ta
WHERE idta>%(rootidta)s
AND status=%(status)s
AND statust=%(statust)s
AND idroute=%(idroute)s
AND merge=%(merge)s
GROUP BY editype,messagetype,frompartner,topartner,tochannel,testindicator,charset,contenttype,envelope
""",
outerqueryparameters,
):
try:
ta_info = dict([(key, row[key]) for key in row.keys()])
ta_info.update({"merge": False, "idroute": idroute})
# ~ for key in row.keys():
# ~ ta_info[key] = row[key]
ta_tofile = botslib.NewTransaction(status=endstatus, idroute=idroute) # edifile for enveloped messages
ta_info["filename"] = str(ta_tofile.idta) # create filename for enveloped message
innerqueryparameters = ta_info.copy()
innerqueryparameters.update(outerqueryparameters)
ta_list = []
# gather individual idta and filenames
# explicitly allow formpartner/topartner to be None/NULL
for row2 in botslib.query(
u"""SELECT idta, filename
FROM ta
WHERE idta>%(rootidta)s
AND status=%(status)s
AND statust=%(statust)s
AND merge=%(merge)s
AND editype=%(editype)s
AND messagetype=%(messagetype)s
AND (frompartner=%(frompartner)s OR frompartner IS NULL)
AND (topartner=%(topartner)s OR topartner IS NULL)
AND tochannel=%(tochannel)s
AND testindicator=%(testindicator)s
AND charset=%(charset)s
AND idroute=%(idroute)s
""",
innerqueryparameters,
):
ta_fromfile = botslib.OldTransaction(row2["idta"]) # edi message to envelope
ta_fromfile.update(statust=DONE, child=ta_tofile.idta) # st child because of n->1 relation
ta_list.append(row2["filename"])
botsglobal.logger.debug(
u"Merge and envelope: editype: %s, messagetype: %s, %s messages",
ta_info["editype"],
ta_info["messagetype"],
ta_info["nrmessages"],
)
envelope(ta_info, ta_list)
except:
txt = botslib.txtexc()
ta_tofile.mergefailure()
ta_tofile.update(statust=ERROR, errortext=txt)
else:
ta_tofile.update(statust=OK, **ta_info)
def preprocess(routedict,function, status=FILEIN,**argv):
''' for pre- and postprocessing of files.
these are NOT translations; translation involve grammars, mapping scripts etc. think of eg:
- unzipping zipped files.
- convert excel to csv
- password protected files.
Select files from INFILE -> SET_FOR_PROCESSING using criteria
Than the actual processing function is called.
The processing function does: SET_FOR_PROCESSING -> PROCESSING -> FILEIN
If errors occur during processing, no ta are left with status FILEIN !
preprocess is called right after the in-communicatiation
'''
nr_files = 0
preprocessnumber = botslib.getpreprocessnumber()
if not botslib.addinfo(change={'status':preprocessnumber},where={'status':status,'idroute':routedict['idroute'],'fromchannel':routedict['fromchannel']}): #check if there is something to do
return 0
for row in botslib.query(u'''SELECT idta,filename,charset
FROM ta
WHERE idta>%(rootidta)s
AND status=%(status)s
AND statust=%(statust)s
AND idroute=%(idroute)s
AND fromchannel=%(fromchannel)s
''',
{'status':preprocessnumber,'statust':OK,'idroute':routedict['idroute'],'fromchannel':routedict['fromchannel'],'rootidta':botslib.get_minta4query()}):
try:
botsglobal.logmap.debug(u'Start preprocessing "%s" for file "%s".',function.__name__,row['filename'])
ta_set_for_processing = botslib.OldTransaction(row['idta'])
ta_processing = ta_set_for_processing.copyta(status=preprocessnumber+1)
ta_processing.filename = row['filename']
function(ta_from=ta_processing,endstatus=status,routedict=routedict,**argv)
except:
txt = botslib.txtexc()
ta_processing.failure()
ta_processing.update(statust=ERROR,errortext=txt)
else:
botsglobal.logmap.debug(u'OK preprocessing "%s" for file "%s".',function.__name__,row['filename'])
ta_set_for_processing.update(statust=DONE)
ta_processing.update(statust=DONE)
nr_files += 1
return nr_files
def translate(startstatus=TRANSLATE, endstatus=TRANSLATED, idroute=""):
""" translates edifiles in one or more edimessages.
reads and parses edifiles that have to be translated.
tries to split files into messages (using 'nextmessage' of grammar); if no splitting: edifile is one message.
searches the right translation in translate-table;
runs the mapping-script for the translation;
Function takes db-ta with status=TRANSLATE->PARSED->SPLITUP->TRANSLATED
"""
# select edifiles to translate; fill ta-object
# ~ import gc
# ~ gc.disable()
for row in botslib.query(
u"""SELECT idta,frompartner,topartner,filename,messagetype,testindicator,editype,charset,alt,fromchannel
FROM ta
WHERE idta>%(rootidta)s
AND status=%(status)s
AND statust=%(statust)s
AND idroute=%(idroute)s
""",
{"status": startstatus, "statust": OK, "idroute": idroute, "rootidta": botslib.get_minta4query()},
):
try:
ta_fromfile = botslib.OldTransaction(row["idta"]) # TRANSLATE ta
ta_parsedfile = ta_fromfile.copyta(status=PARSED) # copy TRANSLATE to PARSED ta
# whole edi-file is read, parsed and made into a inmessage-object:
edifile = inmessage.edifromfile(
frompartner=row["frompartner"],
topartner=row["topartner"],
filename=row["filename"],
messagetype=row["messagetype"],
testindicator=row["testindicator"],
editype=row["editype"],
charset=row["charset"],
alt=row["alt"],
fromchannel=row["fromchannel"],
idroute=idroute,
)
botsglobal.logger.debug(
u'start read and parse input file "%s" editype "%s" messagetype "%s".',
row["filename"],
row["editype"],
row["messagetype"],
)
for inn in edifile.nextmessage(): # for each message in the edifile:
# inn.ta_info: parameters from inmessage.edifromfile(), syntax-information and parse-information
ta_frommes = ta_parsedfile.copyta(status=SPLITUP) # copy PARSED to SPLITUP ta
inn.ta_info[
"idta_fromfile"
] = ta_fromfile.idta # for confirmations in user script; used to give idta of 'confirming message'
ta_frommes.update(
**inn.ta_info
) # update ta-record SLIPTUP with info from message content and/or grammar
while 1: # whileloop continues as long as there are alt-translations
# ************select parameters for translation(script):
for row2 in botslib.query(
u"""SELECT tscript,tomessagetype,toeditype
FROM translate
WHERE frommessagetype = %(frommessagetype)s
AND fromeditype = %(fromeditype)s
AND active=%(booll)s
AND alt=%(alt)s
AND (frompartner_id IS NULL OR frompartner_id=%(frompartner)s OR frompartner_id in (SELECT to_partner_id
FROM partnergroup
WHERE from_partner_id=%(frompartner)s ))
AND (topartner_id IS NULL OR topartner_id=%(topartner)s OR topartner_id in (SELECT to_partner_id
FROM partnergroup
WHERE from_partner_id=%(topartner)s ))
ORDER BY alt DESC,
CASE WHEN frompartner_id IS NULL THEN 1 ELSE 0 END, frompartner_id ,
CASE WHEN topartner_id IS NULL THEN 1 ELSE 0 END, topartner_id """,
{
"frommessagetype": inn.ta_info["messagetype"],
"fromeditype": inn.ta_info["editype"],
"alt": inn.ta_info["alt"],
"frompartner": inn.ta_info["frompartner"],
"topartner": inn.ta_info["topartner"],
"booll": True,
},
):
break # escape if found; we need only the first - ORDER BY in the query
else: # no translation record is found
raise botslib.TranslationNotFoundError(
_(
u'Editype "$editype", messagetype "$messagetype", frompartner "$frompartner", topartner "$topartner", alt "$alt"'
),
editype=inn.ta_info["editype"],
messagetype=inn.ta_info["messagetype"],
frompartner=inn.ta_info["frompartner"],
topartner=inn.ta_info["topartner"],
alt=inn.ta_info["alt"],
)
ta_tomes = ta_frommes.copyta(status=endstatus) # copy SPLITUP to TRANSLATED ta
tofilename = str(ta_tomes.idta)
tscript = row2["tscript"]
tomessage = outmessage.outmessage_init(
messagetype=row2["tomessagetype"],
editype=row2["toeditype"],
filename=tofilename,
reference=unique("messagecounter"),
statust=OK,
divtext=tscript,
) # make outmessage object
# copy ta_info
botsglobal.logger.debug(
u'script "%s" translates messagetype "%s" to messagetype "%s".',
tscript,
inn.ta_info["messagetype"],
tomessage.ta_info["messagetype"],
)
translationscript, scriptfilename = botslib.botsimport(
"mappings", inn.ta_info["editype"] + "." + tscript
) # get the mapping-script
doalttranslation = botslib.runscript(
translationscript, scriptfilename, "main", inn=inn, out=tomessage
)
botsglobal.logger.debug(u'script "%s" finished.', tscript)
if "topartner" not in tomessage.ta_info: # tomessage does not contain values from ta......
tomessage.ta_info["topartner"] = inn.ta_info["topartner"]
if (
tomessage.ta_info["statust"] == DONE
): # if indicated in user script the message should be discarded
botsglobal.logger.debug(u"No output file because mapping script explicitly indicated this.")
tomessage.ta_info["filename"] = ""
tomessage.ta_info["status"] = DISCARD
else:
botsglobal.logger.debug(
u'Start writing output file editype "%s" messagetype "%s".',
tomessage.ta_info["editype"],
tomessage.ta_info["messagetype"],
)
tomessage.writeall() # write tomessage (result of translation).
# problem is that not all values ta_tomes are know to to_message....
# ~ print 'tomessage.ta_info',tomessage.ta_info
ta_tomes.update(**tomessage.ta_info) # update outmessage transaction with ta_info;
del tomessage
# ~ gc.collect()
if not doalttranslation:
break # out of while loop
else:
inn.ta_info["alt"] = doalttranslation
# end of while-loop
# ~ print inn.ta_info
ta_frommes.update(
statust=DONE, **inn.ta_info
) # update db. inn.ta_info could be changed by script. Is this useful?
del inn
# ~ gc.collect()
# exceptions file_in-level
except:
# ~ edifile.handleconfirm(ta_fromfile,error=True) #only useful if errors are reported in acknowledgement (eg x12 997). Not used now.
txt = botslib.txtexc()
ta_parsedfile.failure()
ta_parsedfile.update(statust=ERROR, errortext=txt)
botsglobal.logger.debug(u'error in translating input file "%s":\n%s', row["filename"], txt)
else:
edifile.handleconfirm(ta_fromfile, error=False)
ta_fromfile.update(statust=DONE)
ta_parsedfile.update(statust=DONE, **edifile.confirminfo)
botsglobal.logger.debug(u'translated input file "%s".', row["filename"])
del edifile
def translate(startstatus=TRANSLATE, endstatus=TRANSLATED, idroute=''):
''' translates edifiles in one or more edimessages.
reads and parses edifiles that have to be translated.
tries to split files into messages (using 'nextmessage' of grammar); if no splitting: edifile is one message.
searches the right translation in translate-table;
runs the mapping-script for the translation;
Function takes db-ta with status=TRANSLATE->PARSED->SPLITUP->TRANSLATED
'''
#select edifiles to translate; fill ta-object
for row in botslib.query(
u'''SELECT idta,frompartner,topartner,filename,messagetype,testindicator,editype,charset,alt,fromchannel
FROM ta
WHERE idta>%(rootidta)s
AND status=%(status)s
AND statust=%(statust)s
AND idroute=%(idroute)s
''', {
'status': startstatus,
'statust': OK,
'idroute': idroute,
'rootidta': botslib.get_minta4query()
}):
try:
ta_fromfile = botslib.OldTransaction(row['idta']) #TRANSLATE ta
ta_parsedfile = ta_fromfile.copyta(
status=PARSED) #copy TRANSLATE to PARSED ta
#read whole edi-file: read, parse and made into a inmessage-object. Message is represented a a tree.
edifile = inmessage.edifromfile(frompartner=row['frompartner'],
topartner=row['topartner'],
filename=row['filename'],
messagetype=row['messagetype'],
testindicator=row['testindicator'],
editype=row['editype'],
charset=row['charset'],
alt=row['alt'],
fromchannel=row['fromchannel'],
idroute=idroute)
botsglobal.logger.debug(
u'start read and parse input file "%s" editype "%s" messagetype "%s".',
row['filename'], row['editype'], row['messagetype'])
for inn in edifile.nextmessage(
): #for each message in the edifile:
#inn.ta_info: parameters from inmessage.edifromfile(), syntax-information and parse-information
ta_frommes = ta_parsedfile.copyta(
status=SPLITUP) #copy PARSED to SPLITUP ta
inn.ta_info[
'idta_fromfile'] = ta_fromfile.idta #for confirmations in user script; used to give idta of 'confirming message'
ta_frommes.update(
**inn.ta_info
) #update ta-record SLIPTUP with info from message content and/or grammar
while 1: #whileloop continues as long as there are alt-translations
#************select parameters for translation(script):
for row2 in botslib.query(
u'''SELECT tscript,tomessagetype,toeditype
FROM translate
WHERE frommessagetype = %(frommessagetype)s
AND fromeditype = %(fromeditype)s
AND active=%(booll)s
AND alt=%(alt)s
AND (frompartner_id IS NULL OR frompartner_id=%(frompartner)s OR frompartner_id in (SELECT to_partner_id
FROM partnergroup
WHERE from_partner_id=%(frompartner)s ))
AND (topartner_id IS NULL OR topartner_id=%(topartner)s OR topartner_id in (SELECT to_partner_id
FROM partnergroup
WHERE from_partner_id=%(topartner)s ))
ORDER BY alt DESC,
CASE WHEN frompartner_id IS NULL THEN 1 ELSE 0 END, frompartner_id ,
CASE WHEN topartner_id IS NULL THEN 1 ELSE 0 END, topartner_id ''',
{
'frommessagetype': inn.ta_info['messagetype'],
'fromeditype': inn.ta_info['editype'],
'alt': inn.ta_info['alt'],
'frompartner': inn.ta_info['frompartner'],
'topartner': inn.ta_info['topartner'],
'booll': True
}):
break #escape if found; we need only the first - ORDER BY in the query
else: #no translation record is found
raise botslib.TranslationNotFoundError(
_(u'Editype "$editype", messagetype "$messagetype", frompartner "$frompartner", topartner "$topartner", alt "$alt"'
),
editype=inn.ta_info['editype'],
messagetype=inn.ta_info['messagetype'],
frompartner=inn.ta_info['frompartner'],
topartner=inn.ta_info['topartner'],
alt=inn.ta_info['alt'])
ta_tomes = ta_frommes.copyta(
status=endstatus) #copy SPLITUP to TRANSLATED ta
tofilename = str(ta_tomes.idta)
tscript = row2['tscript']
tomessage = outmessage.outmessage_init(
messagetype=row2['tomessagetype'],
editype=row2['toeditype'],
filename=tofilename,
reference=unique('messagecounter'),
statust=OK,
divtext=tscript) #make outmessage object
#copy ta_info
botsglobal.logger.debug(
u'script "%s" translates messagetype "%s" to messagetype "%s".',
tscript, inn.ta_info['messagetype'],
tomessage.ta_info['messagetype'])
translationscript, scriptfilename = botslib.botsimport(
'mappings', inn.ta_info['editype'] + '.' +
tscript) #get the mapping-script
doalttranslation = botslib.runscript(translationscript,
scriptfilename,
'main',
inn=inn,
out=tomessage)
botsglobal.logger.debug(u'script "%s" finished.', tscript)
if 'topartner' not in tomessage.ta_info: #tomessage does not contain values from ta......
tomessage.ta_info['topartner'] = inn.ta_info[
'topartner']
if tomessage.ta_info[
'statust'] == DONE: #if indicated in user script the message should be discarded
botsglobal.logger.debug(
u'No output file because mapping script explicitly indicated this.'
)
tomessage.ta_info['filename'] = ''
tomessage.ta_info['status'] = DISCARD
else:
botsglobal.logger.debug(
u'Start writing output file editype "%s" messagetype "%s".',
tomessage.ta_info['editype'],
tomessage.ta_info['messagetype'])
tomessage.writeall(
) #write tomessage (result of translation).
#problem is that not all values ta_tomes are know to to_message....
#~ print 'tomessage.ta_info',tomessage.ta_info
ta_tomes.update(
**tomessage.ta_info
) #update outmessage transaction with ta_info;
#check the value received from the mappingscript to see if another traanslation needs to be done (chained translation)
if doalttranslation is None:
del tomessage
break #break out of while loop; do no other translation
elif isinstance(doalttranslation, dict):
#some extended cases; a dict is returned that contains 'instructions'
if 'type' not in doalttranslation:
raise botslib.BotsError(
"Mapping script returned dict. This dict does not have a 'type', like in eg: {'type:'out_as_inn', 'alt':'alt-value'}."
)
if doalttranslation['type'] == u'out_as_inn':
if 'alt' not in doalttranslation:
raise botslib.BotsError(
"Mapping script returned dict, type 'out_as_inn'. This dict does not have a 'alt'-value, like in eg: {'type:'out_as_inn', 'alt':'alt-value'}."
)
inn = tomessage
if isinstance(inn, outmessage.fixed):
inn.root.stripnode()
inn.ta_info['alt'] = doalttranslation[
'alt'] #get the alt-value for the next chainded translation
inn.ta_info.pop('statust')
else:
del tomessage
inn.ta_info[
'alt'] = doalttranslation #get the alt-value for the next chainded translation
#end of while-loop
#
#~ print inn.ta_info
ta_frommes.update(
statust=DONE, **inn.ta_info
) #update db. inn.ta_info could be changed by script. Is this useful?
del inn
#exceptions file_in-level
except:
#~ edifile.handleconfirm(ta_fromfile,error=True) #only useful if errors are reported in acknowledgement (eg x12 997). Not used now.
txt = botslib.txtexc()
ta_parsedfile.failure()
ta_parsedfile.update(statust=ERROR, errortext=txt)
botsglobal.logger.debug(
u'error in translating input file "%s":\n%s', row['filename'],
txt)
else:
edifile.handleconfirm(ta_fromfile, error=False)
ta_fromfile.update(statust=DONE)
ta_parsedfile.update(statust=DONE, **edifile.confirminfo)
botsglobal.logger.debug(u'translated input file "%s".',
row['filename'])
del edifile
def mergemessages(startstatus=TRANSLATED, endstatus=MERGED, idroute=''):
''' Merges en envelopes several messages to one file;
In db-ta: attribute 'merge' indicates message should be merged with similar messages; 'merge' is generated in translation from messagetype-grammar
If merge==False: 1 message per envelope - no merging, else append all similar messages to one file
Implementation as separate loops: one for merge&envelope, another for enveloping only
db-ta status TRANSLATED---->MERGED
'''
outerqueryparameters = {
'status': startstatus,
'statust': OK,
'idroute': idroute,
'rootidta': botslib.get_minta4query(),
'merge': False
}
#**********for messages only to envelope (no merging)
for row in botslib.query(
u'''SELECT editype,messagetype,frompartner,topartner,testindicator,charset,contenttype,tochannel,envelope,nrmessages,idta,filename,idroute,merge
FROM ta
WHERE idta>%(rootidta)s
AND status=%(status)s
AND statust=%(statust)s
AND idroute=%(idroute)s
AND merge=%(merge)s
''', outerqueryparameters):
try:
ta_info = dict([(key, row[key]) for key in row.keys()])
#~ ta_info={'merge':False,'idroute':idroute}
#~ for key in row.keys():
#~ ta_info[key] = row[key]
ta_fromfile = botslib.OldTransaction(
row['idta']) #edi message to envelope
ta_tofile = ta_fromfile.copyta(
status=endstatus
) #edifile for enveloped message; attributes of not-enveloped message are copied...
#~ ta_fromfile.update(child=ta_tofile.idta) #??there is already a parent-child relation (1-1)...
ta_info['filename'] = str(
ta_tofile.idta) #create filename for enveloped message
botsglobal.logger.debug(
u'Envelope 1 message editype: %s, messagetype: %s.',
ta_info['editype'], ta_info['messagetype'])
envelope(ta_info, [row['filename']])
except:
txt = botslib.txtexc()
ta_tofile.update(statust=ERROR, errortext=txt)
else:
ta_fromfile.update(statust=DONE)
ta_tofile.update(
statust=OK,
**ta_info) #selection is used to update enveloped message;
#**********for messages to merge & envelope
#all GROUP BY fields must be used in SELECT!
#as files get merged: can not copy idta; must extract relevant attributes.
outerqueryparameters['merge'] = True
for row in botslib.query(
u'''SELECT editype,messagetype,frompartner,topartner,tochannel,testindicator,charset,contenttype,envelope,sum(nrmessages) as nrmessages
FROM ta
WHERE idta>%(rootidta)s
AND status=%(status)s
AND statust=%(statust)s
AND idroute=%(idroute)s
AND merge=%(merge)s
GROUP BY editype,messagetype,frompartner,topartner,tochannel,testindicator,charset,contenttype,envelope
''', outerqueryparameters):
try:
ta_info = dict([(key, row[key]) for key in row.keys()])
ta_info.update({'merge': False, 'idroute': idroute})
#~ for key in row.keys():
#~ ta_info[key] = row[key]
ta_tofile = botslib.NewTransaction(
status=endstatus,
idroute=idroute) #edifile for enveloped messages
ta_info['filename'] = str(
ta_tofile.idta) #create filename for enveloped message
innerqueryparameters = ta_info.copy()
innerqueryparameters.update(outerqueryparameters)
ta_list = []
#gather individual idta and filenames
#explicitly allow formpartner/topartner to be None/NULL
for row2 in botslib.query(
u'''SELECT idta, filename
FROM ta
WHERE idta>%(rootidta)s
AND status=%(status)s
AND statust=%(statust)s
AND merge=%(merge)s
AND editype=%(editype)s
AND messagetype=%(messagetype)s
AND (frompartner=%(frompartner)s OR frompartner IS NULL)
AND (topartner=%(topartner)s OR topartner IS NULL)
AND tochannel=%(tochannel)s
AND testindicator=%(testindicator)s
AND charset=%(charset)s
AND idroute=%(idroute)s
''', innerqueryparameters):
ta_fromfile = botslib.OldTransaction(
row2['idta']) #edi message to envelope
ta_fromfile.update(
statust=DONE,
child=ta_tofile.idta) #st child because of n->1 relation
ta_list.append(row2['filename'])
botsglobal.logger.debug(
u'Merge and envelope: editype: %s, messagetype: %s, %s messages',
ta_info['editype'], ta_info['messagetype'],
ta_info['nrmessages'])
envelope(ta_info, ta_list)
except:
txt = botslib.txtexc()
ta_tofile.mergefailure()
ta_tofile.update(statust=ERROR, errortext=txt)
else:
ta_tofile.update(statust=OK, **ta_info)
def translate(startstatus=TRANSLATE,endstatus=TRANSLATED,idroute=''):
''' translates edifiles in one or more edimessages.
reads and parses edifiles that have to be translated.
tries to split files into messages (using 'nextmessage' of grammar); if no splitting: edifile is one message.
searches the right translation in translate-table;
runs the mapping-script for the translation;
Function takes db-ta with status=TRANSLATE->PARSED->SPLITUP->TRANSLATED
'''
#select edifiles to translate; fill ta-object
for row in botslib.query(u'''SELECT idta,frompartner,topartner,filename,messagetype,testindicator,editype,charset,alt,fromchannel
FROM ta
WHERE idta>%(rootidta)s
AND status=%(status)s
AND statust=%(statust)s
AND idroute=%(idroute)s
''',
{'status':startstatus,'statust':OK,'idroute':idroute,'rootidta':botslib.get_minta4query()}):
try:
ta_fromfile = botslib.OldTransaction(row['idta']) #TRANSLATE ta
ta_parsedfile = ta_fromfile.copyta(status=PARSED) #copy TRANSLATE to PARSED ta
#read whole edi-file: read, parse and made into a inmessage-object. Message is represented a a tree.
edifile = inmessage.edifromfile(frompartner=row['frompartner'],
topartner=row['topartner'],
filename=row['filename'],
messagetype=row['messagetype'],
testindicator=row['testindicator'],
editype=row['editype'],
charset=row['charset'],
alt=row['alt'],
fromchannel=row['fromchannel'],
idroute=idroute)
botsglobal.logger.debug(u'start read and parse input file "%s" editype "%s" messagetype "%s".',row['filename'],row['editype'],row['messagetype'])
for inn in edifile.nextmessage(): #for each message in the edifile:
#inn.ta_info: parameters from inmessage.edifromfile(), syntax-information and parse-information
ta_frommes = ta_parsedfile.copyta(status=SPLITUP) #copy PARSED to SPLITUP ta
inn.ta_info['idta_fromfile'] = ta_fromfile.idta #for confirmations in user script; used to give idta of 'confirming message'
ta_frommes.update(**inn.ta_info) #update ta-record SLIPTUP with info from message content and/or grammar
while 1: #whileloop continues as long as there are alt-translations
#************select parameters for translation(script):
for row2 in botslib.query(u'''SELECT tscript,tomessagetype,toeditype
FROM translate
WHERE frommessagetype = %(frommessagetype)s
AND fromeditype = %(fromeditype)s
AND active=%(booll)s
AND alt=%(alt)s
AND (frompartner_id IS NULL OR frompartner_id=%(frompartner)s OR frompartner_id in (SELECT to_partner_id
FROM partnergroup
WHERE from_partner_id=%(frompartner)s ))
AND (topartner_id IS NULL OR topartner_id=%(topartner)s OR topartner_id in (SELECT to_partner_id
FROM partnergroup
WHERE from_partner_id=%(topartner)s ))
ORDER BY alt DESC,
CASE WHEN frompartner_id IS NULL THEN 1 ELSE 0 END, frompartner_id ,
CASE WHEN topartner_id IS NULL THEN 1 ELSE 0 END, topartner_id ''',
{'frommessagetype':inn.ta_info['messagetype'],
'fromeditype':inn.ta_info['editype'],
'alt':inn.ta_info['alt'],
'frompartner':inn.ta_info['frompartner'],
'topartner':inn.ta_info['topartner'],
'booll':True}):
break #escape if found; we need only the first - ORDER BY in the query
else: #no translation record is found
raise botslib.TranslationNotFoundError(_(u'Editype "$editype", messagetype "$messagetype", frompartner "$frompartner", topartner "$topartner", alt "$alt"'),
editype=inn.ta_info['editype'],
messagetype=inn.ta_info['messagetype'],
frompartner=inn.ta_info['frompartner'],
topartner=inn.ta_info['topartner'],
alt=inn.ta_info['alt'])
ta_tomes = ta_frommes.copyta(status=endstatus) #copy SPLITUP to TRANSLATED ta
tofilename = str(ta_tomes.idta)
tscript = row2['tscript']
tomessage = outmessage.outmessage_init(messagetype=row2['tomessagetype'],editype=row2['toeditype'],filename=tofilename,reference=unique('messagecounter'),statust=OK,divtext=tscript) #make outmessage object
#copy ta_info
botsglobal.logger.debug(u'script "%s" translates messagetype "%s" to messagetype "%s".',tscript,inn.ta_info['messagetype'],tomessage.ta_info['messagetype'])
translationscript,scriptfilename = botslib.botsimport('mappings',inn.ta_info['editype'] + '.' + tscript) #get the mapping-script
doalttranslation = botslib.runscript(translationscript,scriptfilename,'main',inn=inn,out=tomessage)
botsglobal.logger.debug(u'script "%s" finished.',tscript)
if 'topartner' not in tomessage.ta_info: #tomessage does not contain values from ta......
tomessage.ta_info['topartner'] = inn.ta_info['topartner']
if tomessage.ta_info['statust'] == DONE: #if indicated in user script the message should be discarded
botsglobal.logger.debug(u'No output file because mapping script explicitly indicated this.')
tomessage.ta_info['filename'] = ''
tomessage.ta_info['status'] = DISCARD
else:
botsglobal.logger.debug(u'Start writing output file editype "%s" messagetype "%s".',tomessage.ta_info['editype'],tomessage.ta_info['messagetype'])
tomessage.writeall() #write tomessage (result of translation).
#problem is that not all values ta_tomes are know to to_message....
#~ print 'tomessage.ta_info',tomessage.ta_info
ta_tomes.update(**tomessage.ta_info) #update outmessage transaction with ta_info;
#check the value received from the mappingscript to see if another traanslation needs to be done (chained translation)
if doalttranslation is None:
del tomessage
break #break out of while loop; do no other translation
elif isinstance(doalttranslation,dict):
#some extended cases; a dict is returned that contains 'instructions'
if 'type' not in doalttranslation:
raise botslib.BotsError("Mapping script returned dict. This dict does not have a 'type', like in eg: {'type:'out_as_inn', 'alt':'alt-value'}.")
if doalttranslation['type'] == u'out_as_inn':
if 'alt' not in doalttranslation:
raise botslib.BotsError("Mapping script returned dict, type 'out_as_inn'. This dict does not have a 'alt'-value, like in eg: {'type:'out_as_inn', 'alt':'alt-value'}.")
inn = tomessage
if isinstance(inn,outmessage.fixed):
inn.root.stripnode()
inn.ta_info['alt'] = doalttranslation['alt'] #get the alt-value for the next chainded translation
inn.ta_info.pop('statust')
else:
del tomessage
inn.ta_info['alt'] = doalttranslation #get the alt-value for the next chainded translation
#end of while-loop
#
#~ print inn.ta_info
ta_frommes.update(statust=DONE,**inn.ta_info) #update db. inn.ta_info could be changed by script. Is this useful?
del inn
#exceptions file_in-level
except:
#~ edifile.handleconfirm(ta_fromfile,error=True) #only useful if errors are reported in acknowledgement (eg x12 997). Not used now.
txt = botslib.txtexc()
ta_parsedfile.failure()
ta_parsedfile.update(statust=ERROR,errortext=txt)
botsglobal.logger.debug(u'error in translating input file "%s":\n%s',row['filename'],txt)
else:
edifile.handleconfirm(ta_fromfile,error=False)
ta_fromfile.update(statust=DONE)
ta_parsedfile.update(statust=DONE,**edifile.confirminfo)
botsglobal.logger.debug(u'translated input file "%s".',row['filename'])
del edifile
def splitmailbag(startstatus=MAILBAG,endstatus=TRANSLATE,idroute=''):
''' splits 'mailbag'files to seperate files each containging one interchange (ISA-IEA or UNA/UNB-UNZ).
handles x12 and edifact; these can be mixed.
'''
header = re.compile('(\s*(ISA))|(\s*(UNA.{6})?\s*(U\s*N\s*B)s*.{1}(.{4}).{1}(.{1}))',re.DOTALL)
# group: 1 2 3 4 5 6 7
for row in botslib.query(u'''SELECT idta,filename,charset
FROM ta
WHERE idta>%(rootidta)s
AND status=%(status)s
AND statust=%(statust)s
AND idroute=%(idroute)s
''',
{'status':startstatus,'statust':OK,'idroute':idroute,'rootidta':botslib.get_minta4query()}):
try:
ta_org=botslib.OldTransaction(row['idta'])
ta_intermediate = ta_org.copyta(status=MAILBAGPARSED)
edifile = botslib.readdata(filename=row['filename']) #read as binary...
botsglobal.logmap.debug(u'Start parsing mailbag file "%s".',row['filename'])
startpos=0
while (1):
found = header.search(edifile[startpos:])
if found is None:
if startpos: #ISA/UNB have been found in file; no new ISA/UNB is found. So all processing is done.
break
#guess if this is an xml file.....
sniffxml = edifile[:25]
sniffxml = sniffxml.lstrip(' \t\n\r\f\v\xFF\xFE\xEF\xBB\xBF\x00') #to find first ' real' data; some char are because of BOM, UTF-16 etc
if sniffxml and sniffxml[0]=='<':
ta_tomes=ta_intermediate.copyta(status=endstatus) #make transaction for translated message; gets ta_info of ta_frommes
ta_tomes.update(status=STATUSTMP,statust=OK,filename=row['filename'],editype='xml') #update outmessage transaction with ta_info;
break;
else:
raise botslib.InMessageError(_(u'Found no content in mailbag.'))
elif found.group(1):
editype='x12'
headpos=startpos+ found.start(2)
count=0
for c in edifile[headpos:headpos+120]: #search first 120 characters to find seperators
if c in '\r\n' and count!=105:
continue
count +=1
if count==4:
field_sep = c
elif count==106:
record_sep = c
break
foundtrailer = re.search(re.escape(record_sep)+'\s*IEA'+re.escape(field_sep)+'.+?'+re.escape(record_sep),edifile[headpos:],re.DOTALL)
elif found.group(3):
editype='edifact'
if found.group(4):
field_sep = edifile[startpos + found.start(4) + 4]
record_sep = edifile[startpos + found.start(4) + 8]
headpos=startpos+ found.start(4)
else:
field_sep = '+'
record_sep = "'"
headpos=startpos+ found.start(5)
foundtrailer = re.search(re.escape(record_sep)+'\s*U\s*N\s*Z\s*'+re.escape(field_sep)+'.+?'+re.escape(record_sep),edifile[headpos:],re.DOTALL)
if not foundtrailer:
raise botslib.InMessageError(_(u'Found no valid envelope trailer in mailbag.'))
endpos = headpos+foundtrailer.end()
#so: interchange is from headerpos untill endpos
#~ if header.search(edifile[headpos+25:endpos]): #check if there is another header in the interchange
#~ raise botslib.InMessageError(u'Error in mailbag format: found no valid envelope trailer.')
ta_tomes=ta_intermediate.copyta(status=endstatus) #make transaction for translated message; gets ta_info of ta_frommes
tofilename = str(ta_tomes.idta)
tofile = botslib.opendata(tofilename,'wb')
tofile.write(edifile[headpos:endpos])
tofile.close()
ta_tomes.update(status=STATUSTMP,statust=OK,filename=tofilename,editype=editype,messagetype=editype) #update outmessage transaction with ta_info;
startpos=endpos
except:
txt=botslib.txtexc()
ta_intermediate.failure()
ta_intermediate.update(statust=ERROR,errortext=txt)
else:
botsglobal.logmap.debug(u'OK Parsing mailbag file "%s".',row['filename'])
ta_org.update(statust=DONE)
ta_intermediate.succes(endstatus)
ta_intermediate.update(statust=DONE)
