def preprocess(routedict, function, status=FILEIN, **argv): ''' for pre- and postprocessing of files. these are NOT translations; translation involve grammars, mapping scripts etc. think of eg: - unzipping zipped files. - convert excel to csv - password protected files. Select files from INFILE -> SET_FOR_PROCESSING using criteria Than the actual processing function is called. The processing function does: SET_FOR_PROCESSING -> PROCESSING -> FILEIN If errors occur during processing, no ta are left with status FILEIN ! preprocess is called right after the in-communicatiation ''' nr_files = 0 preprocessnumber = botslib.getpreprocessnumber() if not botslib.addinfo(change={'status': preprocessnumber}, where={ 'status': status, 'idroute': routedict['idroute'], 'fromchannel': routedict['fromchannel'] }): #check if there is something to do return 0 for row in botslib.query( u'''SELECT idta,filename,charset FROM ta WHERE idta>%(rootidta)s AND status=%(status)s AND statust=%(statust)s AND idroute=%(idroute)s AND fromchannel=%(fromchannel)s ''', { 'status': preprocessnumber, 'statust': OK, 'idroute': routedict['idroute'], 'fromchannel': routedict['fromchannel'], 'rootidta': botslib.get_minta4query() }): try: botsglobal.logmap.debug(u'Start preprocessing "%s" for file "%s".', function.__name__, row['filename']) ta_set_for_processing = botslib.OldTransaction(row['idta']) ta_processing = ta_set_for_processing.copyta( status=preprocessnumber + 1) ta_processing.filename = row['filename'] function(ta_from=ta_processing, endstatus=status, routedict=routedict, **argv) except: txt = botslib.txtexc() ta_processing.failure() ta_processing.update(statust=ERROR, errortext=txt) else: botsglobal.logmap.debug(u'OK preprocessing "%s" for file "%s".', function.__name__, row['filename']) ta_set_for_processing.update(statust=DONE) ta_processing.update(statust=DONE) nr_files += 1 return nr_files
def mergemessages(startstatus=TRANSLATED, endstatus=MERGED, idroute=""): """ Merges en envelopes several messages to one file; In db-ta: attribute 'merge' indicates message should be merged with similar messages; 'merge' is generated in translation from messagetype-grammar If merge==False: 1 message per envelope - no merging, else append all similar messages to one file Implementation as separate loops: one for merge&envelope, another for enveloping only db-ta status TRANSLATED---->MERGED """ outerqueryparameters = { "status": startstatus, "statust": OK, "idroute": idroute, "rootidta": botslib.get_minta4query(), "merge": False, } # **********for messages only to envelope (no merging) for row in botslib.query( u"""SELECT editype,messagetype,frompartner,topartner,testindicator,charset,contenttype,tochannel,envelope,nrmessages,idta,filename,idroute,merge FROM ta WHERE idta>%(rootidta)s AND status=%(status)s AND statust=%(statust)s AND idroute=%(idroute)s AND merge=%(merge)s """, outerqueryparameters, ): try: ta_info = dict([(key, row[key]) for key in row.keys()]) # ~ ta_info={'merge':False,'idroute':idroute} # ~ for key in row.keys(): # ~ ta_info[key] = row[key] ta_fromfile = botslib.OldTransaction(row["idta"]) # edi message to envelope ta_tofile = ta_fromfile.copyta( status=endstatus ) # edifile for enveloped message; attributes of not-enveloped message are copied... # ~ ta_fromfile.update(child=ta_tofile.idta) #??there is already a parent-child relation (1-1)... ta_info["filename"] = str(ta_tofile.idta) # create filename for enveloped message botsglobal.logger.debug( u"Envelope 1 message editype: %s, messagetype: %s.", ta_info["editype"], ta_info["messagetype"] ) envelope(ta_info, [row["filename"]]) except: txt = botslib.txtexc() ta_tofile.update(statust=ERROR, errortext=txt) else: ta_fromfile.update(statust=DONE) ta_tofile.update(statust=OK, **ta_info) # selection is used to update enveloped message; # **********for messages to merge & envelope # all GROUP BY fields must be used in SELECT! # as files get merged: can not copy idta; must extract relevant attributes. outerqueryparameters["merge"] = True for row in botslib.query( u"""SELECT editype,messagetype,frompartner,topartner,tochannel,testindicator,charset,contenttype,envelope,sum(nrmessages) as nrmessages FROM ta WHERE idta>%(rootidta)s AND status=%(status)s AND statust=%(statust)s AND idroute=%(idroute)s AND merge=%(merge)s GROUP BY editype,messagetype,frompartner,topartner,tochannel,testindicator,charset,contenttype,envelope """, outerqueryparameters, ): try: ta_info = dict([(key, row[key]) for key in row.keys()]) ta_info.update({"merge": False, "idroute": idroute}) # ~ for key in row.keys(): # ~ ta_info[key] = row[key] ta_tofile = botslib.NewTransaction(status=endstatus, idroute=idroute) # edifile for enveloped messages ta_info["filename"] = str(ta_tofile.idta) # create filename for enveloped message innerqueryparameters = ta_info.copy() innerqueryparameters.update(outerqueryparameters) ta_list = [] # gather individual idta and filenames # explicitly allow formpartner/topartner to be None/NULL for row2 in botslib.query( u"""SELECT idta, filename FROM ta WHERE idta>%(rootidta)s AND status=%(status)s AND statust=%(statust)s AND merge=%(merge)s AND editype=%(editype)s AND messagetype=%(messagetype)s AND (frompartner=%(frompartner)s OR frompartner IS NULL) AND (topartner=%(topartner)s OR topartner IS NULL) AND tochannel=%(tochannel)s AND testindicator=%(testindicator)s AND charset=%(charset)s AND idroute=%(idroute)s """, innerqueryparameters, ): ta_fromfile = botslib.OldTransaction(row2["idta"]) # edi message to envelope ta_fromfile.update(statust=DONE, child=ta_tofile.idta) # st child because of n->1 relation ta_list.append(row2["filename"]) botsglobal.logger.debug( u"Merge and envelope: editype: %s, messagetype: %s, %s messages", ta_info["editype"], ta_info["messagetype"], ta_info["nrmessages"], ) envelope(ta_info, ta_list) except: txt = botslib.txtexc() ta_tofile.mergefailure() ta_tofile.update(statust=ERROR, errortext=txt) else: ta_tofile.update(statust=OK, **ta_info)
def preprocess(routedict,function, status=FILEIN,**argv): ''' for pre- and postprocessing of files. these are NOT translations; translation involve grammars, mapping scripts etc. think of eg: - unzipping zipped files. - convert excel to csv - password protected files. Select files from INFILE -> SET_FOR_PROCESSING using criteria Than the actual processing function is called. The processing function does: SET_FOR_PROCESSING -> PROCESSING -> FILEIN If errors occur during processing, no ta are left with status FILEIN ! preprocess is called right after the in-communicatiation ''' nr_files = 0 preprocessnumber = botslib.getpreprocessnumber() if not botslib.addinfo(change={'status':preprocessnumber},where={'status':status,'idroute':routedict['idroute'],'fromchannel':routedict['fromchannel']}): #check if there is something to do return 0 for row in botslib.query(u'''SELECT idta,filename,charset FROM ta WHERE idta>%(rootidta)s AND status=%(status)s AND statust=%(statust)s AND idroute=%(idroute)s AND fromchannel=%(fromchannel)s ''', {'status':preprocessnumber,'statust':OK,'idroute':routedict['idroute'],'fromchannel':routedict['fromchannel'],'rootidta':botslib.get_minta4query()}): try: botsglobal.logmap.debug(u'Start preprocessing "%s" for file "%s".',function.__name__,row['filename']) ta_set_for_processing = botslib.OldTransaction(row['idta']) ta_processing = ta_set_for_processing.copyta(status=preprocessnumber+1) ta_processing.filename = row['filename'] function(ta_from=ta_processing,endstatus=status,routedict=routedict,**argv) except: txt = botslib.txtexc() ta_processing.failure() ta_processing.update(statust=ERROR,errortext=txt) else: botsglobal.logmap.debug(u'OK preprocessing "%s" for file "%s".',function.__name__,row['filename']) ta_set_for_processing.update(statust=DONE) ta_processing.update(statust=DONE) nr_files += 1 return nr_files
def translate(startstatus=TRANSLATE, endstatus=TRANSLATED, idroute=""): """ translates edifiles in one or more edimessages. reads and parses edifiles that have to be translated. tries to split files into messages (using 'nextmessage' of grammar); if no splitting: edifile is one message. searches the right translation in translate-table; runs the mapping-script for the translation; Function takes db-ta with status=TRANSLATE->PARSED->SPLITUP->TRANSLATED """ # select edifiles to translate; fill ta-object # ~ import gc # ~ gc.disable() for row in botslib.query( u"""SELECT idta,frompartner,topartner,filename,messagetype,testindicator,editype,charset,alt,fromchannel FROM ta WHERE idta>%(rootidta)s AND status=%(status)s AND statust=%(statust)s AND idroute=%(idroute)s """, {"status": startstatus, "statust": OK, "idroute": idroute, "rootidta": botslib.get_minta4query()}, ): try: ta_fromfile = botslib.OldTransaction(row["idta"]) # TRANSLATE ta ta_parsedfile = ta_fromfile.copyta(status=PARSED) # copy TRANSLATE to PARSED ta # whole edi-file is read, parsed and made into a inmessage-object: edifile = inmessage.edifromfile( frompartner=row["frompartner"], topartner=row["topartner"], filename=row["filename"], messagetype=row["messagetype"], testindicator=row["testindicator"], editype=row["editype"], charset=row["charset"], alt=row["alt"], fromchannel=row["fromchannel"], idroute=idroute, ) botsglobal.logger.debug( u'start read and parse input file "%s" editype "%s" messagetype "%s".', row["filename"], row["editype"], row["messagetype"], ) for inn in edifile.nextmessage(): # for each message in the edifile: # inn.ta_info: parameters from inmessage.edifromfile(), syntax-information and parse-information ta_frommes = ta_parsedfile.copyta(status=SPLITUP) # copy PARSED to SPLITUP ta inn.ta_info[ "idta_fromfile" ] = ta_fromfile.idta # for confirmations in user script; used to give idta of 'confirming message' ta_frommes.update( **inn.ta_info ) # update ta-record SLIPTUP with info from message content and/or grammar while 1: # whileloop continues as long as there are alt-translations # ************select parameters for translation(script): for row2 in botslib.query( u"""SELECT tscript,tomessagetype,toeditype FROM translate WHERE frommessagetype = %(frommessagetype)s AND fromeditype = %(fromeditype)s AND active=%(booll)s AND alt=%(alt)s AND (frompartner_id IS NULL OR frompartner_id=%(frompartner)s OR frompartner_id in (SELECT to_partner_id FROM partnergroup WHERE from_partner_id=%(frompartner)s )) AND (topartner_id IS NULL OR topartner_id=%(topartner)s OR topartner_id in (SELECT to_partner_id FROM partnergroup WHERE from_partner_id=%(topartner)s )) ORDER BY alt DESC, CASE WHEN frompartner_id IS NULL THEN 1 ELSE 0 END, frompartner_id , CASE WHEN topartner_id IS NULL THEN 1 ELSE 0 END, topartner_id """, { "frommessagetype": inn.ta_info["messagetype"], "fromeditype": inn.ta_info["editype"], "alt": inn.ta_info["alt"], "frompartner": inn.ta_info["frompartner"], "topartner": inn.ta_info["topartner"], "booll": True, }, ): break # escape if found; we need only the first - ORDER BY in the query else: # no translation record is found raise botslib.TranslationNotFoundError( _( u'Editype "$editype", messagetype "$messagetype", frompartner "$frompartner", topartner "$topartner", alt "$alt"' ), editype=inn.ta_info["editype"], messagetype=inn.ta_info["messagetype"], frompartner=inn.ta_info["frompartner"], topartner=inn.ta_info["topartner"], alt=inn.ta_info["alt"], ) ta_tomes = ta_frommes.copyta(status=endstatus) # copy SPLITUP to TRANSLATED ta tofilename = str(ta_tomes.idta) tscript = row2["tscript"] tomessage = outmessage.outmessage_init( messagetype=row2["tomessagetype"], editype=row2["toeditype"], filename=tofilename, reference=unique("messagecounter"), statust=OK, divtext=tscript, ) # make outmessage object # copy ta_info botsglobal.logger.debug( u'script "%s" translates messagetype "%s" to messagetype "%s".', tscript, inn.ta_info["messagetype"], tomessage.ta_info["messagetype"], ) translationscript, scriptfilename = botslib.botsimport( "mappings", inn.ta_info["editype"] + "." + tscript ) # get the mapping-script doalttranslation = botslib.runscript( translationscript, scriptfilename, "main", inn=inn, out=tomessage ) botsglobal.logger.debug(u'script "%s" finished.', tscript) if "topartner" not in tomessage.ta_info: # tomessage does not contain values from ta...... tomessage.ta_info["topartner"] = inn.ta_info["topartner"] if ( tomessage.ta_info["statust"] == DONE ): # if indicated in user script the message should be discarded botsglobal.logger.debug(u"No output file because mapping script explicitly indicated this.") tomessage.ta_info["filename"] = "" tomessage.ta_info["status"] = DISCARD else: botsglobal.logger.debug( u'Start writing output file editype "%s" messagetype "%s".', tomessage.ta_info["editype"], tomessage.ta_info["messagetype"], ) tomessage.writeall() # write tomessage (result of translation). # problem is that not all values ta_tomes are know to to_message.... # ~ print 'tomessage.ta_info',tomessage.ta_info ta_tomes.update(**tomessage.ta_info) # update outmessage transaction with ta_info; del tomessage # ~ gc.collect() if not doalttranslation: break # out of while loop else: inn.ta_info["alt"] = doalttranslation # end of while-loop # ~ print inn.ta_info ta_frommes.update( statust=DONE, **inn.ta_info ) # update db. inn.ta_info could be changed by script. Is this useful? del inn # ~ gc.collect() # exceptions file_in-level except: # ~ edifile.handleconfirm(ta_fromfile,error=True) #only useful if errors are reported in acknowledgement (eg x12 997). Not used now. txt = botslib.txtexc() ta_parsedfile.failure() ta_parsedfile.update(statust=ERROR, errortext=txt) botsglobal.logger.debug(u'error in translating input file "%s":\n%s', row["filename"], txt) else: edifile.handleconfirm(ta_fromfile, error=False) ta_fromfile.update(statust=DONE) ta_parsedfile.update(statust=DONE, **edifile.confirminfo) botsglobal.logger.debug(u'translated input file "%s".', row["filename"]) del edifile
def translate(startstatus=TRANSLATE, endstatus=TRANSLATED, idroute=''): ''' translates edifiles in one or more edimessages. reads and parses edifiles that have to be translated. tries to split files into messages (using 'nextmessage' of grammar); if no splitting: edifile is one message. searches the right translation in translate-table; runs the mapping-script for the translation; Function takes db-ta with status=TRANSLATE->PARSED->SPLITUP->TRANSLATED ''' #select edifiles to translate; fill ta-object for row in botslib.query( u'''SELECT idta,frompartner,topartner,filename,messagetype,testindicator,editype,charset,alt,fromchannel FROM ta WHERE idta>%(rootidta)s AND status=%(status)s AND statust=%(statust)s AND idroute=%(idroute)s ''', { 'status': startstatus, 'statust': OK, 'idroute': idroute, 'rootidta': botslib.get_minta4query() }): try: ta_fromfile = botslib.OldTransaction(row['idta']) #TRANSLATE ta ta_parsedfile = ta_fromfile.copyta( status=PARSED) #copy TRANSLATE to PARSED ta #read whole edi-file: read, parse and made into a inmessage-object. Message is represented a a tree. edifile = inmessage.edifromfile(frompartner=row['frompartner'], topartner=row['topartner'], filename=row['filename'], messagetype=row['messagetype'], testindicator=row['testindicator'], editype=row['editype'], charset=row['charset'], alt=row['alt'], fromchannel=row['fromchannel'], idroute=idroute) botsglobal.logger.debug( u'start read and parse input file "%s" editype "%s" messagetype "%s".', row['filename'], row['editype'], row['messagetype']) for inn in edifile.nextmessage( ): #for each message in the edifile: #inn.ta_info: parameters from inmessage.edifromfile(), syntax-information and parse-information ta_frommes = ta_parsedfile.copyta( status=SPLITUP) #copy PARSED to SPLITUP ta inn.ta_info[ 'idta_fromfile'] = ta_fromfile.idta #for confirmations in user script; used to give idta of 'confirming message' ta_frommes.update( **inn.ta_info ) #update ta-record SLIPTUP with info from message content and/or grammar while 1: #whileloop continues as long as there are alt-translations #************select parameters for translation(script): for row2 in botslib.query( u'''SELECT tscript,tomessagetype,toeditype FROM translate WHERE frommessagetype = %(frommessagetype)s AND fromeditype = %(fromeditype)s AND active=%(booll)s AND alt=%(alt)s AND (frompartner_id IS NULL OR frompartner_id=%(frompartner)s OR frompartner_id in (SELECT to_partner_id FROM partnergroup WHERE from_partner_id=%(frompartner)s )) AND (topartner_id IS NULL OR topartner_id=%(topartner)s OR topartner_id in (SELECT to_partner_id FROM partnergroup WHERE from_partner_id=%(topartner)s )) ORDER BY alt DESC, CASE WHEN frompartner_id IS NULL THEN 1 ELSE 0 END, frompartner_id , CASE WHEN topartner_id IS NULL THEN 1 ELSE 0 END, topartner_id ''', { 'frommessagetype': inn.ta_info['messagetype'], 'fromeditype': inn.ta_info['editype'], 'alt': inn.ta_info['alt'], 'frompartner': inn.ta_info['frompartner'], 'topartner': inn.ta_info['topartner'], 'booll': True }): break #escape if found; we need only the first - ORDER BY in the query else: #no translation record is found raise botslib.TranslationNotFoundError( _(u'Editype "$editype", messagetype "$messagetype", frompartner "$frompartner", topartner "$topartner", alt "$alt"' ), editype=inn.ta_info['editype'], messagetype=inn.ta_info['messagetype'], frompartner=inn.ta_info['frompartner'], topartner=inn.ta_info['topartner'], alt=inn.ta_info['alt']) ta_tomes = ta_frommes.copyta( status=endstatus) #copy SPLITUP to TRANSLATED ta tofilename = str(ta_tomes.idta) tscript = row2['tscript'] tomessage = outmessage.outmessage_init( messagetype=row2['tomessagetype'], editype=row2['toeditype'], filename=tofilename, reference=unique('messagecounter'), statust=OK, divtext=tscript) #make outmessage object #copy ta_info botsglobal.logger.debug( u'script "%s" translates messagetype "%s" to messagetype "%s".', tscript, inn.ta_info['messagetype'], tomessage.ta_info['messagetype']) translationscript, scriptfilename = botslib.botsimport( 'mappings', inn.ta_info['editype'] + '.' + tscript) #get the mapping-script doalttranslation = botslib.runscript(translationscript, scriptfilename, 'main', inn=inn, out=tomessage) botsglobal.logger.debug(u'script "%s" finished.', tscript) if 'topartner' not in tomessage.ta_info: #tomessage does not contain values from ta...... tomessage.ta_info['topartner'] = inn.ta_info[ 'topartner'] if tomessage.ta_info[ 'statust'] == DONE: #if indicated in user script the message should be discarded botsglobal.logger.debug( u'No output file because mapping script explicitly indicated this.' ) tomessage.ta_info['filename'] = '' tomessage.ta_info['status'] = DISCARD else: botsglobal.logger.debug( u'Start writing output file editype "%s" messagetype "%s".', tomessage.ta_info['editype'], tomessage.ta_info['messagetype']) tomessage.writeall( ) #write tomessage (result of translation). #problem is that not all values ta_tomes are know to to_message.... #~ print 'tomessage.ta_info',tomessage.ta_info ta_tomes.update( **tomessage.ta_info ) #update outmessage transaction with ta_info; #check the value received from the mappingscript to see if another traanslation needs to be done (chained translation) if doalttranslation is None: del tomessage break #break out of while loop; do no other translation elif isinstance(doalttranslation, dict): #some extended cases; a dict is returned that contains 'instructions' if 'type' not in doalttranslation: raise botslib.BotsError( "Mapping script returned dict. This dict does not have a 'type', like in eg: {'type:'out_as_inn', 'alt':'alt-value'}." ) if doalttranslation['type'] == u'out_as_inn': if 'alt' not in doalttranslation: raise botslib.BotsError( "Mapping script returned dict, type 'out_as_inn'. This dict does not have a 'alt'-value, like in eg: {'type:'out_as_inn', 'alt':'alt-value'}." ) inn = tomessage if isinstance(inn, outmessage.fixed): inn.root.stripnode() inn.ta_info['alt'] = doalttranslation[ 'alt'] #get the alt-value for the next chainded translation inn.ta_info.pop('statust') else: del tomessage inn.ta_info[ 'alt'] = doalttranslation #get the alt-value for the next chainded translation #end of while-loop # #~ print inn.ta_info ta_frommes.update( statust=DONE, **inn.ta_info ) #update db. inn.ta_info could be changed by script. Is this useful? del inn #exceptions file_in-level except: #~ edifile.handleconfirm(ta_fromfile,error=True) #only useful if errors are reported in acknowledgement (eg x12 997). Not used now. txt = botslib.txtexc() ta_parsedfile.failure() ta_parsedfile.update(statust=ERROR, errortext=txt) botsglobal.logger.debug( u'error in translating input file "%s":\n%s', row['filename'], txt) else: edifile.handleconfirm(ta_fromfile, error=False) ta_fromfile.update(statust=DONE) ta_parsedfile.update(statust=DONE, **edifile.confirminfo) botsglobal.logger.debug(u'translated input file "%s".', row['filename']) del edifile
def mergemessages(startstatus=TRANSLATED, endstatus=MERGED, idroute=''): ''' Merges en envelopes several messages to one file; In db-ta: attribute 'merge' indicates message should be merged with similar messages; 'merge' is generated in translation from messagetype-grammar If merge==False: 1 message per envelope - no merging, else append all similar messages to one file Implementation as separate loops: one for merge&envelope, another for enveloping only db-ta status TRANSLATED---->MERGED ''' outerqueryparameters = { 'status': startstatus, 'statust': OK, 'idroute': idroute, 'rootidta': botslib.get_minta4query(), 'merge': False } #**********for messages only to envelope (no merging) for row in botslib.query( u'''SELECT editype,messagetype,frompartner,topartner,testindicator,charset,contenttype,tochannel,envelope,nrmessages,idta,filename,idroute,merge FROM ta WHERE idta>%(rootidta)s AND status=%(status)s AND statust=%(statust)s AND idroute=%(idroute)s AND merge=%(merge)s ''', outerqueryparameters): try: ta_info = dict([(key, row[key]) for key in row.keys()]) #~ ta_info={'merge':False,'idroute':idroute} #~ for key in row.keys(): #~ ta_info[key] = row[key] ta_fromfile = botslib.OldTransaction( row['idta']) #edi message to envelope ta_tofile = ta_fromfile.copyta( status=endstatus ) #edifile for enveloped message; attributes of not-enveloped message are copied... #~ ta_fromfile.update(child=ta_tofile.idta) #??there is already a parent-child relation (1-1)... ta_info['filename'] = str( ta_tofile.idta) #create filename for enveloped message botsglobal.logger.debug( u'Envelope 1 message editype: %s, messagetype: %s.', ta_info['editype'], ta_info['messagetype']) envelope(ta_info, [row['filename']]) except: txt = botslib.txtexc() ta_tofile.update(statust=ERROR, errortext=txt) else: ta_fromfile.update(statust=DONE) ta_tofile.update( statust=OK, **ta_info) #selection is used to update enveloped message; #**********for messages to merge & envelope #all GROUP BY fields must be used in SELECT! #as files get merged: can not copy idta; must extract relevant attributes. outerqueryparameters['merge'] = True for row in botslib.query( u'''SELECT editype,messagetype,frompartner,topartner,tochannel,testindicator,charset,contenttype,envelope,sum(nrmessages) as nrmessages FROM ta WHERE idta>%(rootidta)s AND status=%(status)s AND statust=%(statust)s AND idroute=%(idroute)s AND merge=%(merge)s GROUP BY editype,messagetype,frompartner,topartner,tochannel,testindicator,charset,contenttype,envelope ''', outerqueryparameters): try: ta_info = dict([(key, row[key]) for key in row.keys()]) ta_info.update({'merge': False, 'idroute': idroute}) #~ for key in row.keys(): #~ ta_info[key] = row[key] ta_tofile = botslib.NewTransaction( status=endstatus, idroute=idroute) #edifile for enveloped messages ta_info['filename'] = str( ta_tofile.idta) #create filename for enveloped message innerqueryparameters = ta_info.copy() innerqueryparameters.update(outerqueryparameters) ta_list = [] #gather individual idta and filenames #explicitly allow formpartner/topartner to be None/NULL for row2 in botslib.query( u'''SELECT idta, filename FROM ta WHERE idta>%(rootidta)s AND status=%(status)s AND statust=%(statust)s AND merge=%(merge)s AND editype=%(editype)s AND messagetype=%(messagetype)s AND (frompartner=%(frompartner)s OR frompartner IS NULL) AND (topartner=%(topartner)s OR topartner IS NULL) AND tochannel=%(tochannel)s AND testindicator=%(testindicator)s AND charset=%(charset)s AND idroute=%(idroute)s ''', innerqueryparameters): ta_fromfile = botslib.OldTransaction( row2['idta']) #edi message to envelope ta_fromfile.update( statust=DONE, child=ta_tofile.idta) #st child because of n->1 relation ta_list.append(row2['filename']) botsglobal.logger.debug( u'Merge and envelope: editype: %s, messagetype: %s, %s messages', ta_info['editype'], ta_info['messagetype'], ta_info['nrmessages']) envelope(ta_info, ta_list) except: txt = botslib.txtexc() ta_tofile.mergefailure() ta_tofile.update(statust=ERROR, errortext=txt) else: ta_tofile.update(statust=OK, **ta_info)
def translate(startstatus=TRANSLATE,endstatus=TRANSLATED,idroute=''): ''' translates edifiles in one or more edimessages. reads and parses edifiles that have to be translated. tries to split files into messages (using 'nextmessage' of grammar); if no splitting: edifile is one message. searches the right translation in translate-table; runs the mapping-script for the translation; Function takes db-ta with status=TRANSLATE->PARSED->SPLITUP->TRANSLATED ''' #select edifiles to translate; fill ta-object for row in botslib.query(u'''SELECT idta,frompartner,topartner,filename,messagetype,testindicator,editype,charset,alt,fromchannel FROM ta WHERE idta>%(rootidta)s AND status=%(status)s AND statust=%(statust)s AND idroute=%(idroute)s ''', {'status':startstatus,'statust':OK,'idroute':idroute,'rootidta':botslib.get_minta4query()}): try: ta_fromfile = botslib.OldTransaction(row['idta']) #TRANSLATE ta ta_parsedfile = ta_fromfile.copyta(status=PARSED) #copy TRANSLATE to PARSED ta #read whole edi-file: read, parse and made into a inmessage-object. Message is represented a a tree. edifile = inmessage.edifromfile(frompartner=row['frompartner'], topartner=row['topartner'], filename=row['filename'], messagetype=row['messagetype'], testindicator=row['testindicator'], editype=row['editype'], charset=row['charset'], alt=row['alt'], fromchannel=row['fromchannel'], idroute=idroute) botsglobal.logger.debug(u'start read and parse input file "%s" editype "%s" messagetype "%s".',row['filename'],row['editype'],row['messagetype']) for inn in edifile.nextmessage(): #for each message in the edifile: #inn.ta_info: parameters from inmessage.edifromfile(), syntax-information and parse-information ta_frommes = ta_parsedfile.copyta(status=SPLITUP) #copy PARSED to SPLITUP ta inn.ta_info['idta_fromfile'] = ta_fromfile.idta #for confirmations in user script; used to give idta of 'confirming message' ta_frommes.update(**inn.ta_info) #update ta-record SLIPTUP with info from message content and/or grammar while 1: #whileloop continues as long as there are alt-translations #************select parameters for translation(script): for row2 in botslib.query(u'''SELECT tscript,tomessagetype,toeditype FROM translate WHERE frommessagetype = %(frommessagetype)s AND fromeditype = %(fromeditype)s AND active=%(booll)s AND alt=%(alt)s AND (frompartner_id IS NULL OR frompartner_id=%(frompartner)s OR frompartner_id in (SELECT to_partner_id FROM partnergroup WHERE from_partner_id=%(frompartner)s )) AND (topartner_id IS NULL OR topartner_id=%(topartner)s OR topartner_id in (SELECT to_partner_id FROM partnergroup WHERE from_partner_id=%(topartner)s )) ORDER BY alt DESC, CASE WHEN frompartner_id IS NULL THEN 1 ELSE 0 END, frompartner_id , CASE WHEN topartner_id IS NULL THEN 1 ELSE 0 END, topartner_id ''', {'frommessagetype':inn.ta_info['messagetype'], 'fromeditype':inn.ta_info['editype'], 'alt':inn.ta_info['alt'], 'frompartner':inn.ta_info['frompartner'], 'topartner':inn.ta_info['topartner'], 'booll':True}): break #escape if found; we need only the first - ORDER BY in the query else: #no translation record is found raise botslib.TranslationNotFoundError(_(u'Editype "$editype", messagetype "$messagetype", frompartner "$frompartner", topartner "$topartner", alt "$alt"'), editype=inn.ta_info['editype'], messagetype=inn.ta_info['messagetype'], frompartner=inn.ta_info['frompartner'], topartner=inn.ta_info['topartner'], alt=inn.ta_info['alt']) ta_tomes = ta_frommes.copyta(status=endstatus) #copy SPLITUP to TRANSLATED ta tofilename = str(ta_tomes.idta) tscript = row2['tscript'] tomessage = outmessage.outmessage_init(messagetype=row2['tomessagetype'],editype=row2['toeditype'],filename=tofilename,reference=unique('messagecounter'),statust=OK,divtext=tscript) #make outmessage object #copy ta_info botsglobal.logger.debug(u'script "%s" translates messagetype "%s" to messagetype "%s".',tscript,inn.ta_info['messagetype'],tomessage.ta_info['messagetype']) translationscript,scriptfilename = botslib.botsimport('mappings',inn.ta_info['editype'] + '.' + tscript) #get the mapping-script doalttranslation = botslib.runscript(translationscript,scriptfilename,'main',inn=inn,out=tomessage) botsglobal.logger.debug(u'script "%s" finished.',tscript) if 'topartner' not in tomessage.ta_info: #tomessage does not contain values from ta...... tomessage.ta_info['topartner'] = inn.ta_info['topartner'] if tomessage.ta_info['statust'] == DONE: #if indicated in user script the message should be discarded botsglobal.logger.debug(u'No output file because mapping script explicitly indicated this.') tomessage.ta_info['filename'] = '' tomessage.ta_info['status'] = DISCARD else: botsglobal.logger.debug(u'Start writing output file editype "%s" messagetype "%s".',tomessage.ta_info['editype'],tomessage.ta_info['messagetype']) tomessage.writeall() #write tomessage (result of translation). #problem is that not all values ta_tomes are know to to_message.... #~ print 'tomessage.ta_info',tomessage.ta_info ta_tomes.update(**tomessage.ta_info) #update outmessage transaction with ta_info; #check the value received from the mappingscript to see if another traanslation needs to be done (chained translation) if doalttranslation is None: del tomessage break #break out of while loop; do no other translation elif isinstance(doalttranslation,dict): #some extended cases; a dict is returned that contains 'instructions' if 'type' not in doalttranslation: raise botslib.BotsError("Mapping script returned dict. This dict does not have a 'type', like in eg: {'type:'out_as_inn', 'alt':'alt-value'}.") if doalttranslation['type'] == u'out_as_inn': if 'alt' not in doalttranslation: raise botslib.BotsError("Mapping script returned dict, type 'out_as_inn'. This dict does not have a 'alt'-value, like in eg: {'type:'out_as_inn', 'alt':'alt-value'}.") inn = tomessage if isinstance(inn,outmessage.fixed): inn.root.stripnode() inn.ta_info['alt'] = doalttranslation['alt'] #get the alt-value for the next chainded translation inn.ta_info.pop('statust') else: del tomessage inn.ta_info['alt'] = doalttranslation #get the alt-value for the next chainded translation #end of while-loop # #~ print inn.ta_info ta_frommes.update(statust=DONE,**inn.ta_info) #update db. inn.ta_info could be changed by script. Is this useful? del inn #exceptions file_in-level except: #~ edifile.handleconfirm(ta_fromfile,error=True) #only useful if errors are reported in acknowledgement (eg x12 997). Not used now. txt = botslib.txtexc() ta_parsedfile.failure() ta_parsedfile.update(statust=ERROR,errortext=txt) botsglobal.logger.debug(u'error in translating input file "%s":\n%s',row['filename'],txt) else: edifile.handleconfirm(ta_fromfile,error=False) ta_fromfile.update(statust=DONE) ta_parsedfile.update(statust=DONE,**edifile.confirminfo) botsglobal.logger.debug(u'translated input file "%s".',row['filename']) del edifile
def splitmailbag(startstatus=MAILBAG,endstatus=TRANSLATE,idroute=''): ''' splits 'mailbag'files to seperate files each containging one interchange (ISA-IEA or UNA/UNB-UNZ). handles x12 and edifact; these can be mixed. ''' header = re.compile('(\s*(ISA))|(\s*(UNA.{6})?\s*(U\s*N\s*B)s*.{1}(.{4}).{1}(.{1}))',re.DOTALL) # group: 1 2 3 4 5 6 7 for row in botslib.query(u'''SELECT idta,filename,charset FROM ta WHERE idta>%(rootidta)s AND status=%(status)s AND statust=%(statust)s AND idroute=%(idroute)s ''', {'status':startstatus,'statust':OK,'idroute':idroute,'rootidta':botslib.get_minta4query()}): try: ta_org=botslib.OldTransaction(row['idta']) ta_intermediate = ta_org.copyta(status=MAILBAGPARSED) edifile = botslib.readdata(filename=row['filename']) #read as binary... botsglobal.logmap.debug(u'Start parsing mailbag file "%s".',row['filename']) startpos=0 while (1): found = header.search(edifile[startpos:]) if found is None: if startpos: #ISA/UNB have been found in file; no new ISA/UNB is found. So all processing is done. break #guess if this is an xml file..... sniffxml = edifile[:25] sniffxml = sniffxml.lstrip(' \t\n\r\f\v\xFF\xFE\xEF\xBB\xBF\x00') #to find first ' real' data; some char are because of BOM, UTF-16 etc if sniffxml and sniffxml[0]=='<': ta_tomes=ta_intermediate.copyta(status=endstatus) #make transaction for translated message; gets ta_info of ta_frommes ta_tomes.update(status=STATUSTMP,statust=OK,filename=row['filename'],editype='xml') #update outmessage transaction with ta_info; break; else: raise botslib.InMessageError(_(u'Found no content in mailbag.')) elif found.group(1): editype='x12' headpos=startpos+ found.start(2) count=0 for c in edifile[headpos:headpos+120]: #search first 120 characters to find seperators if c in '\r\n' and count!=105: continue count +=1 if count==4: field_sep = c elif count==106: record_sep = c break foundtrailer = re.search(re.escape(record_sep)+'\s*IEA'+re.escape(field_sep)+'.+?'+re.escape(record_sep),edifile[headpos:],re.DOTALL) elif found.group(3): editype='edifact' if found.group(4): field_sep = edifile[startpos + found.start(4) + 4] record_sep = edifile[startpos + found.start(4) + 8] headpos=startpos+ found.start(4) else: field_sep = '+' record_sep = "'" headpos=startpos+ found.start(5) foundtrailer = re.search(re.escape(record_sep)+'\s*U\s*N\s*Z\s*'+re.escape(field_sep)+'.+?'+re.escape(record_sep),edifile[headpos:],re.DOTALL) if not foundtrailer: raise botslib.InMessageError(_(u'Found no valid envelope trailer in mailbag.')) endpos = headpos+foundtrailer.end() #so: interchange is from headerpos untill endpos #~ if header.search(edifile[headpos+25:endpos]): #check if there is another header in the interchange #~ raise botslib.InMessageError(u'Error in mailbag format: found no valid envelope trailer.') ta_tomes=ta_intermediate.copyta(status=endstatus) #make transaction for translated message; gets ta_info of ta_frommes tofilename = str(ta_tomes.idta) tofile = botslib.opendata(tofilename,'wb') tofile.write(edifile[headpos:endpos]) tofile.close() ta_tomes.update(status=STATUSTMP,statust=OK,filename=tofilename,editype=editype,messagetype=editype) #update outmessage transaction with ta_info; startpos=endpos except: txt=botslib.txtexc() ta_intermediate.failure() ta_intermediate.update(statust=ERROR,errortext=txt) else: botsglobal.logmap.debug(u'OK Parsing mailbag file "%s".',row['filename']) ta_org.update(statust=DONE) ta_intermediate.succes(endstatus) ta_intermediate.update(statust=DONE)