def translate(startstatus,endstatus,idroute,rootidta): ''' main translation loop. get edifiles to be translated, than: - read, lex, parse, make tree of nodes. - split up files into messages (using 'nextmessage' of grammar) - get mappingscript, start mappingscript. - write the results of translation (no enveloping yet) status: FILEIN--PARSED-<SPLITUP--TRANSLATED ''' try: #see if there is a userscript that can determine the translation userscript,scriptname = botslib.botsimport('mappings','translation') except ImportError: #userscript is not there; other errors like syntax errors are not catched userscript = scriptname = None #select edifiles to translate for rawrow in botslib.query(u'''SELECT idta,frompartner,topartner,filename,messagetype,testindicator,editype,charset,alt,fromchannel,filesize FROM ta WHERE idta>%(rootidta)s AND status=%(status)s AND statust=%(statust)s AND idroute=%(idroute)s ''', {'status':startstatus,'statust':OK,'idroute':idroute,'rootidta':rootidta}): try: row = dict(rawrow) #convert to real dictionary () ta_fromfile = botslib.OldTransaction(row['idta']) ta_parsed = ta_fromfile.copyta(status=PARSED) #make PARSED ta if row['filesize'] > botsglobal.ini.getint('settings','maxfilesizeincoming',5000000): ta_parsed.update(filesize=row['filesize']) raise botslib.InMessageError(_(u'File size of %(filesize)s is too big; option "maxfilesizeincoming" in bots.ini is %(maxfilesizeincoming)s.'), {'filesize':row['filesize'],'maxfilesizeincoming':botsglobal.ini.getint('settings','maxfilesizeincoming',5000000)}) botsglobal.logger.debug(_(u'Start translating file "%(filename)s" editype "%(editype)s" messagetype "%(messagetype)s".'),row) #read whole edi-file: read, parse and made into a inmessage-object. Message is represented as a tree (inmessage.root is the root of the tree). edifile = inmessage.parse_edi_file(frompartner=row['frompartner'], topartner=row['topartner'], filename=row['filename'], messagetype=row['messagetype'], testindicator=row['testindicator'], editype=row['editype'], charset=row['charset'], alt=row['alt'], fromchannel=row['fromchannel'], idroute=idroute) #if no exception: infile has been lexed and parsed OK. #edifile.ta_info contains info: QUERIES, charset etc for inn_splitup in edifile.nextmessage(): #splitup messages in parsed edifile try: ta_splitup = ta_parsed.copyta(status=SPLITUP,**inn_splitup.ta_info) #copy PARSED to SPLITUP ta #inn_splitup.ta_info: parameters from inmessage.parse_edi_file(), syntax-information and parse-information inn_splitup.ta_info['idta_fromfile'] = ta_fromfile.idta #for confirmations in userscript; used to give idta of 'confirming message' post_mapping_mode = False #if post_mapping: reuse out-object, if no translation is found no error number_of_loops_with_same_alt = 0 while 1: #continue as long as there are (alt-)translations #lookup the translation************************ tscript,toeditype,tomessagetype = botslib.lookup_translation(fromeditype=inn_splitup.ta_info['editype'], frommessagetype=inn_splitup.ta_info['messagetype'], frompartner=inn_splitup.ta_info['frompartner'], topartner=inn_splitup.ta_info['topartner'], alt=inn_splitup.ta_info['alt']) if not tscript: #no translation found in translate table; check if can find translation via user script if userscript and hasattr(userscript,'gettranslation'): tscript,toeditype,tomessagetype = botslib.runscript(userscript,scriptname,'gettranslation',idroute=idroute,message=inn_splitup) if not tscript: if post_mapping_mode: #in post-mapping mode, not finding the (partern specific!) script is no problem: translation is done #there is no post-mapping script found (in other words, default mapping script is enough). #all OK, handle the generated out-file and continue with next message botsglobal.logger.debug(_(u'Found no "post-mapping" translation for editype "%(editype)s", messagetype "%(messagetype)s", frompartner "%(frompartner)s", topartner "%(topartner)s", alt "%(alt)s".'), inn_splitup.ta_info) handle_out_message(out_translated,ta_translated) del out_translated break #break out of while loop else: raise botslib.TranslationNotFoundError(_(u'Translation not found for editype "%(editype)s", messagetype "%(messagetype)s", frompartner "%(frompartner)s", topartner "%(topartner)s", alt "%(alt)s".'), inn_splitup.ta_info) inn_splitup.ta_info['divtext'] = tscript #ifor reporting used mapping script to database (for display in GUI). if not post_mapping_mode: #initialize new out-object************************* ta_translated = ta_splitup.copyta(status=endstatus) #make ta for translated message (new out-ta) filename_translated = str(ta_translated.idta) out_translated = outmessage.outmessage_init(editype=toeditype,messagetype=tomessagetype,filename=filename_translated,reference=unique('messagecounter'),statust=OK,divtext=tscript) #make outmessage object #run mapping script************************ botsglobal.logger.debug(_(u'Mappingscript "%(tscript)s" translates messagetype "%(messagetype)s" to messagetype "%(tomessagetype)s".'), {'tscript':tscript,'messagetype':inn_splitup.ta_info['messagetype'],'tomessagetype':out_translated.ta_info['messagetype']}) translationscript,scriptfilename = botslib.botsimport('mappings',inn_splitup.ta_info['editype'],tscript) #get the mappingscript alt_from_previous_run = inn_splitup.ta_info['alt'] #needed to check for infinite loop doalttranslation = botslib.runscript(translationscript,scriptfilename,'main',inn=inn_splitup,out=out_translated) botsglobal.logger.debug(_(u'Mappingscript "%(tscript)s" finished.'),{'tscript':tscript}) #manipulate for some attributes after mapping script if 'topartner' not in out_translated.ta_info: #out_translated does not contain values from ta...... out_translated.ta_info['topartner'] = inn_splitup.ta_info['topartner'] if 'botskey' in inn_splitup.ta_info: inn_splitup.ta_info['reference'] = inn_splitup.ta_info['botskey'] if 'botskey' in out_translated.ta_info: #out_translated does not contain values from ta...... out_translated.ta_info['reference'] = out_translated.ta_info['botskey'] #check the value received from the mappingscript to determine what to do in this while-loop. Handling of chained trasnlations. if doalttranslation is None: #translation(s) are done; handle out-message handle_out_message(out_translated,ta_translated) del out_translated break #break out of while loop elif isinstance(doalttranslation,dict): #some extended cases; a dict is returned that contains 'instructions' for some type of chained translations if 'type' not in doalttranslation or 'alt' not in doalttranslation: raise botslib.BotsError(_(u"Mappingscript returned '%(alt)s'. This dict should not have 'type' and 'alt'."),{'alt':doalttranslation}) if alt_from_previous_run == doalttranslation['alt']: number_of_loops_with_same_alt += 1 else: number_of_loops_with_same_alt = 0 if doalttranslation['type'] == u'out_as_inn': #do chained translation: use the out-object as inn-object, new out-object #use case: detected error in incoming file; use out-object to generate warning email handle_out_message(out_translated,ta_translated) inn_splitup = out_translated #out-object is now inn-object if isinstance(inn_splitup,outmessage.fixed): #for fixed: strip all values in node inn_splitup.root.stripnode() inn_splitup.ta_info['alt'] = doalttranslation['alt'] #get the alt-value for the next chained translation if not 'frompartner' in inn_splitup.ta_info: inn_splitup.ta_info['frompartner'] = '' if not 'topartner' in inn_splitup.ta_info: inn_splitup.ta_info['topartner'] = '' inn_splitup.ta_info.pop('statust') elif doalttranslation['type'] == u'no_check_on_infinite_loop': #do chained translation: allow many loops wit hsame alt-value. #mapping script will have to handle this correctly. number_of_loops_with_same_alt = 0 handle_out_message(out_translated,ta_translated) del out_translated inn_splitup.ta_info['alt'] = doalttranslation['alt'] #get the alt-value for the next chained translation elif doalttranslation['type'] == u'post_mapping': #do chained (post)translation: same inn and out-objects. #in other words, the post-translation continue where is 'default' mapping ended. #use case: default mapping (for all partners), partner-specific post-translation #note: this is easier via import defaultmapping in a partner specific mapping. Please use that method! post_mapping_mode = True #translation is done; reset post_mapping_mode inn_splitup.ta_info['alt'] = doalttranslation['alt'] #get the alt-value for the next chained translation #~ inn_splitup.ta_info.pop('statust') else: #there is nothing else raise botslib.BotsError(_(u'Mappingscript returned dict with an unknown "type": "%(doalttranslation)s".'),{'doalttranslation':doalttranslation}) else: #note: this includes alt '' (empty string) if alt_from_previous_run == doalttranslation: number_of_loops_with_same_alt += 1 else: number_of_loops_with_same_alt = 0 #do normal chained translation: same inn-object, new out-object handle_out_message(out_translated,ta_translated) del out_translated inn_splitup.ta_info['alt'] = doalttranslation #get the alt-value for the next chained translation if number_of_loops_with_same_alt > 10: raise botslib.BotsError(_(u'Mappingscript returns same alt value over and over again (infinite loop?). Alt: "%(doalttranslation)s".'),{'doalttranslation':doalttranslation}) #end of while-loop (trans********************************************************************************** #exceptions file_out-level: exception in mappingscript or writing of out-file except: #2 modes: either every error leads to skipping of whole infile (old mode) or errors in mappingscript/outfile only affect that branche if botsglobal.ini.getboolean('settings','oldmessageerrors',False): raise txt = botslib.txtexc() ta_splitup.update(statust=ERROR,errortext=txt,**inn_splitup.ta_info) #update db. inn_splitup.ta_info could be changed by mappingscript. Is this useful? ta_splitup.deletechildren() else: ta_splitup.update(statust=DONE, **inn_splitup.ta_info) #update db. inn_splitup.ta_info could be changed by mappingscript. Is this useful? #exceptions file_in-level (file not OK according to grammar) except: txt = botslib.txtexc() ta_parsed.update(statust=ERROR,errortext=txt) ta_parsed.deletechildren() botsglobal.logger.debug(u'Error in translating input file "%(filename)s":\n%(msg)s',{'filename':row['filename'],'msg':txt}) else: edifile.handleconfirm(ta_fromfile,error=False) ta_parsed.update(statust=DONE,filesize=row['filesize'],**edifile.ta_info) botsglobal.logger.debug(_(u'Translated input file "%(filename)s".'),row) finally: ta_fromfile.update(statust=DONE)
def _translate_one_file(row,routedict,endstatus,userscript,scriptname): ''' - read, lex, parse, make tree of nodes. - split up files into messages (using 'nextmessage' of grammar) - get mappingscript, start mappingscript. - write the results of translation (no enveloping yet) ''' try: ta_fromfile = botslib.OldTransaction(row['idta']) ta_parsed = ta_fromfile.copyta(status=PARSED) if row['filesize'] > botsglobal.ini.getint('settings','maxfilesizeincoming',5000000): ta_parsed.update(filesize=row['filesize']) raise botslib.FileTooLarge(_(u'File size of %(filesize)s is too big; option "maxfilesizeincoming" in bots.ini is %(maxfilesizeincoming)s.'), {'filesize':row['filesize'],'maxfilesizeincoming':botsglobal.ini.getint('settings','maxfilesizeincoming',5000000)}) botsglobal.logger.debug(_(u'Start translating file "%(filename)s" editype "%(editype)s" messagetype "%(messagetype)s".'),row) #read whole edi-file: read, parse and made into a inmessage-object. Message is represented as a tree (inmessage.root is the root of the tree). edifile = inmessage.parse_edi_file(frompartner=row['frompartner'], topartner=row['topartner'], filename=row['filename'], messagetype=row['messagetype'], testindicator=row['testindicator'], editype=row['editype'], charset=row['charset'], alt=row['alt'], fromchannel=row['fromchannel'], idroute=routedict['idroute'], command=routedict['command']) edifile.checkforerrorlist() #no exception if infile has been lexed and parsed OK else raises an error if int(routedict['translateind']) == 3: #parse & passthrough; file is parsed, partners are known, no mapping, does confirm. raise botslib.GotoException('dummy') #edifile.ta_info contains info: QUERIES, charset etc for inn_splitup in edifile.nextmessage(): #splitup messages in parsed edifile try: ta_splitup = ta_parsed.copyta(status=SPLITUP,**inn_splitup.ta_info) #copy PARSED to SPLITUP ta #inn_splitup.ta_info: parameters from inmessage.parse_edi_file(), syntax-information and parse-information inn_splitup.ta_info['idta_fromfile'] = ta_fromfile.idta #for confirmations in userscript; used to give idta of 'confirming message' inn_splitup.ta_info['idta'] = ta_splitup.idta #for confirmations in userscript; used to give idta of 'confirming message' number_of_loops_with_same_alt = 0 while 1: #continue as long as there are (alt-)translations #lookup the translation************************ tscript,toeditype,tomessagetype = botslib.lookup_translation(fromeditype=inn_splitup.ta_info['editype'], frommessagetype=inn_splitup.ta_info['messagetype'], frompartner=inn_splitup.ta_info['frompartner'], topartner=inn_splitup.ta_info['topartner'], alt=inn_splitup.ta_info['alt']) if not tscript: #no translation found in translate table; check if can find translation via user script if userscript and hasattr(userscript,'gettranslation'): tscript,toeditype,tomessagetype = botslib.runscript(userscript,scriptname,'gettranslation',idroute=routedict['idroute'],message=inn_splitup) if not tscript: raise botslib.TranslationNotFoundError(_(u'Translation not found for editype "%(editype)s", messagetype "%(messagetype)s", frompartner "%(frompartner)s", topartner "%(topartner)s", alt "%(alt)s".'), inn_splitup.ta_info) inn_splitup.ta_info['divtext'] = tscript #ifor reporting used mapping script to database (for display in GUI). #initialize new out-object************************* ta_translated = ta_splitup.copyta(status=endstatus) #make ta for translated message (new out-ta) filename_translated = str(ta_translated.idta) out_translated = outmessage.outmessage_init(editype=toeditype,messagetype=tomessagetype,filename=filename_translated,reference=unique('messagecounter'),statust=OK,divtext=tscript) #make outmessage object #run mapping script************************ botsglobal.logger.debug(_(u'Mappingscript "%(tscript)s" translates messagetype "%(messagetype)s" to messagetype "%(tomessagetype)s".'), {'tscript':tscript,'messagetype':inn_splitup.ta_info['messagetype'],'tomessagetype':out_translated.ta_info['messagetype']}) translationscript,scriptfilename = botslib.botsimport('mappings',inn_splitup.ta_info['editype'],tscript) #get the mappingscript alt_from_previous_run = inn_splitup.ta_info['alt'] #needed to check for infinite loop doalttranslation = botslib.runscript(translationscript,scriptfilename,'main',inn=inn_splitup,out=out_translated) botsglobal.logger.debug(_(u'Mappingscript "%(tscript)s" finished.'),{'tscript':tscript}) #manipulate for some attributes after mapping script if 'topartner' not in out_translated.ta_info: #out_translated does not contain values from ta...... out_translated.ta_info['topartner'] = inn_splitup.ta_info['topartner'] if 'botskey' in inn_splitup.ta_info: inn_splitup.ta_info['reference'] = inn_splitup.ta_info['botskey'] if 'botskey' in out_translated.ta_info: #out_translated does not contain values from ta...... out_translated.ta_info['reference'] = out_translated.ta_info['botskey'] #check the value received from the mappingscript to determine what to do in this while-loop. Handling of chained trasnlations. if doalttranslation is None: #translation(s) are done; handle out-message handle_out_message(out_translated,ta_translated) break #break out of while loop elif isinstance(doalttranslation,dict): #some extended cases; a dict is returned that contains 'instructions' for some type of chained translations if 'type' not in doalttranslation or 'alt' not in doalttranslation: raise botslib.BotsError(_(u"Mappingscript returned '%(alt)s'. This dict should not have 'type' and 'alt'."),{'alt':doalttranslation}) if alt_from_previous_run == doalttranslation['alt']: number_of_loops_with_same_alt += 1 else: number_of_loops_with_same_alt = 0 if doalttranslation['type'] == u'out_as_inn': #do chained translation: use the out-object as inn-object, new out-object #use case: detected error in incoming file; use out-object to generate warning email handle_out_message(out_translated,ta_translated) inn_splitup = out_translated #out-object is now inn-object if isinstance(inn_splitup,outmessage.fixed): #for fixed: strip all values in node inn_splitup.root.stripnode() inn_splitup.ta_info['alt'] = doalttranslation['alt'] #get the alt-value for the next chained translation if not 'frompartner' in inn_splitup.ta_info: inn_splitup.ta_info['frompartner'] = '' if not 'topartner' in inn_splitup.ta_info: inn_splitup.ta_info['topartner'] = '' inn_splitup.ta_info.pop('statust') elif doalttranslation['type'] == u'no_check_on_infinite_loop': #do chained translation: allow many loops wit hsame alt-value. #mapping script will have to handle this correctly. number_of_loops_with_same_alt = 0 handle_out_message(out_translated,ta_translated) inn_splitup.ta_info['alt'] = doalttranslation['alt'] #get the alt-value for the next chained translation else: #there is nothing else raise botslib.BotsError(_(u'Mappingscript returned dict with an unknown "type": "%(doalttranslation)s".'),{'doalttranslation':doalttranslation}) else: #note: this includes alt '' (empty string) if alt_from_previous_run == doalttranslation: number_of_loops_with_same_alt += 1 else: number_of_loops_with_same_alt = 0 #do normal chained translation: same inn-object, new out-object handle_out_message(out_translated,ta_translated) inn_splitup.ta_info['alt'] = doalttranslation #get the alt-value for the next chained translation if number_of_loops_with_same_alt > 10: raise botslib.BotsError(_(u'Mappingscript returns same alt value over and over again (infinite loop?). Alt: "%(doalttranslation)s".'),{'doalttranslation':doalttranslation}) #end of while-loop (trans********************************************************************************** #exceptions file_out-level: exception in mappingscript or writing of out-file except: #2 modes: either every error leads to skipping of whole infile (old mode) or errors in mappingscript/outfile only affect that branche if botsglobal.ini.getboolean('settings','oldmessageerrors',False): raise txt = botslib.txtexc() ta_splitup.update(statust=ERROR,errortext=txt,**inn_splitup.ta_info) #update db. inn_splitup.ta_info could be changed by mappingscript. Is this useful? ta_splitup.deletechildren() else: ta_splitup.update(statust=DONE, **inn_splitup.ta_info) #update db. inn_splitup.ta_info could be changed by mappingscript. Is this useful? #exceptions file_in-level except botslib.GotoException: #edi-file is OK, file is passed-through after parsing. ta_parsed.update(statust=DONE,filesize=row['filesize'],**edifile.ta_info) #update with info from eg queries ta_parsed.copyta(status=MERGED,statust=OK) #original file goes straight to MERGED edifile.handleconfirm(ta_fromfile,error=False) botsglobal.logger.debug(_(u'Parse & passthrough for input file "%(filename)s".'),row) except botslib.FileTooLarge as msg: ta_parsed.update(statust=ERROR,errortext=str(msg)) ta_parsed.deletechildren() botsglobal.logger.debug(u'Error in translating input file "%(filename)s":\n%(msg)s',{'filename':row['filename'],'msg':msg}) except: txt = botslib.txtexc() ta_parsed.update(statust=ERROR,errortext=txt,**edifile.ta_info) ta_parsed.deletechildren() edifile.handleconfirm(ta_fromfile,error=True) botsglobal.logger.debug(u'Error in translating input file "%(filename)s":\n%(msg)s',{'filename':row['filename'],'msg':txt}) else: edifile.handleconfirm(ta_fromfile,error=False) ta_parsed.update(statust=DONE,filesize=row['filesize'],**edifile.ta_info) botsglobal.logger.debug(_(u'Translated input file "%(filename)s".'),row) finally: ta_fromfile.update(statust=DONE)
def translate(run): for messagedict in run.incoming: try: #read whole edi-file: read, parse and made into a inmessage-object. Message is represented as a tree (inmessage.root is the root of the tree). edifile = inmessage.parse_edi_file(frompartner='', topartner='', filename=messagedict['filename'], messagetype=run.translation['messagetype'], testindicator='', editype=run.translation['editype'], charset='', alt='', fromchannel='', idroute='', command='') edifile.checkforerrorlist() #no exception if infile has been lexed and parsed OK else raises an error #~ if int(routedict['translateind']) == 3: #parse & passthrough; file is parsed, partners are known, no mapping, does confirm. #~ raise botslib.GotoException('dummy') #~ continue #file is parsed; no errors. no translation #edifile.ta_info contains info about incoming file: QUERIES, charset etc for inn_splitup in edifile.nextmessage(): #splitup messages in parsed edifile try: #inn_splitup.ta_info: parameters from inmessage.parse_edi_file(), syntax-information and parse-information number_of_loops_with_same_alt = 0 while 1: #continue as long as there are (alt-)translations #lookup the translation************************ tscript,toeditype,tomessagetype = 'orders_edifact2xml' ,'xml','orders' if 'tscript' in run.translation: tscript = run.translation['tscript'] toeditype = run.translation['toeditype'] tomessagetype = run.translation['tomessagetype'] else: tscript,toeditype,tomessagetype = botslib.lookup_translation(fromeditype=inn_splitup.ta_info['editype'], frommessagetype=inn_splitup.ta_info['messagetype'], frompartner=inn_splitup.ta_info['frompartner'], topartner=inn_splitup.ta_info['topartner'], alt=inn_splitup.ta_info['alt']) #run mapping script************************ filename_translated = transform.unique('bots_file_name') out_translated = outmessage.outmessage_init(editype=toeditype, messagetype=tomessagetype, filename=filename_translated, reference=transform.unique('messagecounter'), statust=OK, divtext=tscript) #make outmessage object #~ botsglobal.logger.debug(_(u'Mappingscript "%(tscript)s" translates messagetype "%(messagetype)s" to messagetype "%(tomessagetype)s".'), #~ {'tscript':tscript,'messagetype':inn_splitup.ta_info['messagetype'],'tomessagetype':out_translated.ta_info['messagetype']}) translationscript,scriptfilename = botslib.botsimport('mappings',inn_splitup.ta_info['editype'],tscript) #import mappingscript alt_from_previous_run = inn_splitup.ta_info['alt'] #needed to check for infinite loop doalttranslation = botslib.runscript(translationscript,scriptfilename,'main',inn=inn_splitup,out=out_translated) botsglobal.logger.debug(_(u'Mappingscript "%(tscript)s" finished.'),{'tscript':tscript}) #manipulate for some attributes after mapping script if 'topartner' not in out_translated.ta_info: #out_translated does not contain values from run...... out_translated.ta_info['topartner'] = inn_splitup.ta_info['topartner'] if 'botskey' in inn_splitup.ta_info: inn_splitup.ta_info['reference'] = inn_splitup.ta_info['botskey'] if 'botskey' in out_translated.ta_info: #out_translated does not contain values from run...... out_translated.ta_info['reference'] = out_translated.ta_info['botskey'] #check the value received from the mappingscript to determine what to do in this while-loop. Handling of chained trasnlations. if doalttranslation is None: #translation(s) are done; handle out-message out_translated.writeall() #write result of translation. #make translated record (if all is OK) translated_dict = inn_splitup.ta_info.copy() translated_dict.update(messagedict) translated_dict.update(out_translated.ta_info) run.translated.append(translated_dict) del out_translated break #break out of while loop elif isinstance(doalttranslation,dict): #some extended cases; a dict is returned that contains 'instructions' for some type of chained translations if 'type' not in doalttranslation or 'alt' not in doalttranslation: raise botslib.BotsError(_(u"Mappingscript returned '%(alt)s'. This dict should not have 'type' and 'alt'."),{'alt':doalttranslation}) if alt_from_previous_run == doalttranslation['alt']: number_of_loops_with_same_alt += 1 else: number_of_loops_with_same_alt = 0 if doalttranslation['type'] == u'out_as_inn': #do chained translation: use the out-object as inn-object, new out-object #use case: detected error in incoming file; use out-object to generate warning email handle_out_message(out_translated,ta_translated) inn_splitup = out_translated #out-object is now inn-object if isinstance(inn_splitup,outmessage.fixed): #for fixed: strip all values in node inn_splitup.root.stripnode() inn_splitup.ta_info['alt'] = doalttranslation['alt'] #get the alt-value for the next chained translation if not 'frompartner' in inn_splitup.ta_info: inn_splitup.ta_info['frompartner'] = '' if not 'topartner' in inn_splitup.ta_info: inn_splitup.ta_info['topartner'] = '' inn_splitup.ta_info.pop('statust') elif doalttranslation['type'] == u'no_check_on_infinite_loop': #do chained translation: allow many loops wit hsame alt-value. #mapping script will have to handle this correctly. number_of_loops_with_same_alt = 0 handle_out_message(out_translated,ta_translated) del out_translated inn_splitup.ta_info['alt'] = doalttranslation['alt'] #get the alt-value for the next chained translation else: #there is nothing else raise botslib.BotsError(_(u'Mappingscript returned dict with an unknown "type": "%(doalttranslation)s".'),{'doalttranslation':doalttranslation}) else: #note: this includes alt '' (empty string) if alt_from_previous_run == doalttranslation: number_of_loops_with_same_alt += 1 else: number_of_loops_with_same_alt = 0 #do normal chained translation: same inn-object, new out-object out_translated.writeall() del out_translated inn_splitup.ta_info['alt'] = doalttranslation #get the alt-value for the next chained translation if number_of_loops_with_same_alt > 10: raise botslib.BotsError(_(u'Mappingscript returns same alt value over and over again (infinite loop?). Alt: "%(doalttranslation)s".'),{'doalttranslation':doalttranslation}) #end of while-loop (trans********************************************************************************** #exceptions file_out-level: exception in mappingscript or writing of out-file except: #2 modes: either every error leads to skipping of whole infile (old mode) or errors in mappingscript/outfile only affect that branche txt = botslib.txtexc() print txt messagedict['error'] += txt.strip() else: pass #~ print 'succes' #exceptions file_in-level except botslib.GotoException: #edi-file is OK, file is passed-through after parsing. #~ edifile.handleconfirm(ta_fromfile,error=False) #~ botsglobal.logger.debug(_(u'Parse & passthrough for input file "%(filename)s".'),row) txt = botslib.txtexc() print txt except: txt = botslib.txtexc() messagedict['error'] += txt.strip() #~ edifile.handleconfirm(ta_fromfile,error=True) #~ botsglobal.logger.debug(u'Error in translating input file "%(filename)s":\n%(msg)s',{'filename':row['filename'],'msg':txt}) else: pass
def translate(run): for messagedict in run.incoming: try: #read whole edi-file: read, parse and made into a inmessage-object. Message is represented as a tree (inmessage.root is the root of the tree). edifile = inmessage.parse_edi_file(frompartner='', topartner='', filename=messagedict['filename'], messagetype=run.translation['messagetype'], testindicator='', editype=run.translation['editype'], charset='', alt='', fromchannel='', idroute='', command='') edifile.checkforerrorlist() #no exception if infile has been lexed and parsed OK else raises an error #~ if int(routedict['translateind']) == 3: #parse & passthrough; file is parsed, partners are known, no mapping, does confirm. #~ raise botslib.GotoException('dummy') #~ continue #file is parsed; no errors. no translation #edifile.ta_info contains info about incoming file: QUERIES, charset etc for inn_splitup in edifile.nextmessage(): #splitup messages in parsed edifile try: #inn_splitup.ta_info: parameters from inmessage.parse_edi_file(), syntax-information and parse-information number_of_loops_with_same_alt = 0 while True: #continue as long as there are (alt-)translations #lookup the translation************************ tscript,toeditype,tomessagetype = 'orders_edifact2xml' ,'xml','orders' if 'tscript' in run.translation: tscript = run.translation['tscript'] toeditype = run.translation['toeditype'] tomessagetype = run.translation['tomessagetype'] else: tscript,toeditype,tomessagetype = botslib.lookup_translation(fromeditype=inn_splitup.ta_info['editype'], frommessagetype=inn_splitup.ta_info['messagetype'], frompartner=inn_splitup.ta_info['frompartner'], topartner=inn_splitup.ta_info['topartner'], alt=inn_splitup.ta_info['alt']) #run mapping script************************ filename_translated = transform.unique('bots_file_name') out_translated = outmessage.outmessage_init(editype=toeditype, messagetype=tomessagetype, filename=filename_translated, reference=transform.unique('messagecounter'), statust=OK, divtext=tscript) #make outmessage object #~ botsglobal.logger.debug(_('Mappingscript "%(tscript)s" translates messagetype "%(messagetype)s" to messagetype "%(tomessagetype)s".'), #~ {'tscript':tscript,'messagetype':inn_splitup.ta_info['messagetype'],'tomessagetype':out_translated.ta_info['messagetype']}) translationscript,scriptfilename = botslib.botsimport('mappings',inn_splitup.ta_info['editype'],tscript) #import mappingscript alt_from_previous_run = inn_splitup.ta_info['alt'] #needed to check for infinite loop doalttranslation = botslib.runscript(translationscript,scriptfilename,'main',inn=inn_splitup,out=out_translated) botsglobal.logger.debug(_('Mappingscript "%(tscript)s" finished.'),{'tscript':tscript}) #manipulate for some attributes after mapping script if 'topartner' not in out_translated.ta_info: #out_translated does not contain values from run...... out_translated.ta_info['topartner'] = inn_splitup.ta_info['topartner'] if 'botskey' in inn_splitup.ta_info: inn_splitup.ta_info['reference'] = inn_splitup.ta_info['botskey'] if 'botskey' in out_translated.ta_info: #out_translated does not contain values from run...... out_translated.ta_info['reference'] = out_translated.ta_info['botskey'] #check the value received from the mappingscript to determine what to do in this while-loop. Handling of chained trasnlations. if doalttranslation is None: #translation(s) are done; handle out-message out_translated.writeall() #write result of translation. #make translated record (if all is OK) translated_dict = inn_splitup.ta_info.copy() translated_dict.update(messagedict) translated_dict.update(out_translated.ta_info) run.translated.append(translated_dict) del out_translated break #break out of while loop elif isinstance(doalttranslation,dict): #some extended cases; a dict is returned that contains 'instructions' for some type of chained translations if 'type' not in doalttranslation or 'alt' not in doalttranslation: raise botslib.BotsError(_('Mappingscript returned "%(alt)s". This dict should not have "type" and "alt".'),{'alt':doalttranslation}) if alt_from_previous_run == doalttranslation['alt']: number_of_loops_with_same_alt += 1 else: number_of_loops_with_same_alt = 0 if doalttranslation['type'] == 'out_as_inn': #do chained translation: use the out-object as inn-object, new out-object #use case: detected error in incoming file; use out-object to generate warning email handle_out_message(out_translated,ta_translated) inn_splitup = out_translated #out-object is now inn-object if isinstance(inn_splitup,outmessage.fixed): #for fixed: strip all values in node inn_splitup.root.stripnode() inn_splitup.ta_info['alt'] = doalttranslation['alt'] #get the alt-value for the next chained translation if not 'frompartner' in inn_splitup.ta_info: inn_splitup.ta_info['frompartner'] = '' if not 'topartner' in inn_splitup.ta_info: inn_splitup.ta_info['topartner'] = '' inn_splitup.ta_info.pop('statust') elif doalttranslation['type'] == 'no_check_on_infinite_loop': #do chained translation: allow many loops wit hsame alt-value. #mapping script will have to handle this correctly. number_of_loops_with_same_alt = 0 handle_out_message(out_translated,ta_translated) del out_translated inn_splitup.ta_info['alt'] = doalttranslation['alt'] #get the alt-value for the next chained translation else: #there is nothing else raise botslib.BotsError(_('Mappingscript returned dict with an unknown "type": "%(doalttranslation)s".'),{'doalttranslation':doalttranslation}) else: #note: this includes alt '' (empty string) if alt_from_previous_run == doalttranslation: number_of_loops_with_same_alt += 1 else: number_of_loops_with_same_alt = 0 #do normal chained translation: same inn-object, new out-object out_translated.writeall() del out_translated inn_splitup.ta_info['alt'] = doalttranslation #get the alt-value for the next chained translation if number_of_loops_with_same_alt > 10: raise botslib.BotsError(_('Mappingscript returns same alt value over and over again (infinite loop?). Alt: "%(doalttranslation)s".'),{'doalttranslation':doalttranslation}) #end of while-loop (trans********************************************************************************** #exceptions file_out-level: exception in mappingscript or writing of out-file except: #2 modes: either every error leads to skipping of whole infile (old mode) or errors in mappingscript/outfile only affect that branche txt = botslib.txtexc() print(txt) messagedict['error'] += txt.strip() else: pass #exceptions file_in-level except botslib.GotoException: #edi-file is OK, file is passed-through after parsing. #~ edifile.handleconfirm(ta_fromfile,error=False) #~ botsglobal.logger.debug(_('Parse & passthrough for input file "%(filename)s".'),row) txt = botslib.txtexc() print(txt) except: txt = botslib.txtexc() messagedict['error'] += txt.strip() #~ edifile.handleconfirm(ta_fromfile,error=True) #~ botsglobal.logger.debug('Error in translating input file "%(filename)s":\n%(msg)s',{'filename':row['filename'],'msg':txt}) else: pass
def start(): #********command line arguments************************** usage = ''' This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net). Creates a grammar from an xml file.' Usage:' %(name)s -c<directory> <xml_file> <xml_grammar_file> Options: -c<directory> directory for configuration files (default: config). <xml_file> name of the xml file to read <xml_grammar_file> name of the grammar file to write '''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version} configdir = 'config' edifile ='' grammarfile = '' for arg in sys.argv[1:]: if arg.startswith('-c'): configdir = arg[2:] if not configdir: print 'Error: configuration directory indicated, but no directory name.' sys.exit(1) elif arg in ["?", "/?",'-h', '--help'] or arg.startswith('-'): print usage sys.exit(0) else: if not edifile: edifile = arg else: grammarfile = arg if not edifile or not grammarfile: print 'Error: both edifile and grammarfile are required.' sys.exit(0) #***end handling command line arguments************************** botsinit.generalinit(configdir) #find locating of bots, configfiles, init paths etc. process_name = 'xml2botsgrammar' botsglobal.logger = botsinit.initenginelogging(process_name) atexit.register(logging.shutdown) #the xml file is parsed as an xmlnocheck message editype = 'xmlnocheck' messagetype = 'xmlnocheckxxxtemporaryforxml2grammar' mpath = [] #a (temp) xmlnocheck grammar is needed (but needs not actual content. This file is not removed. tmpgrammarfile = botslib.join(botsglobal.ini.get('directories','usersysabs'),'grammars',editype,messagetype+'.py') filehandler = open(tmpgrammarfile,'w') filehandler.close() #make inmessage object: read the xml file inn = inmessage.parse_edi_file(editype=editype,messagetype=messagetype,filename=edifile,remove_empties_from_xml=False) #make outmessage object; nothing is 'filled' yet. out = outmessage.outmessage_init(editype=editype,messagetype=messagetype,filename='botssys/infile/unitnode/output/inisout03.edi',divtext='',topartner='') #***do the mapping*************************************************** #handle root rootmpath = [{'BOTSID':inn.root.record['BOTSID'],'BOTSIDnr':'1'}] out.put(*rootmpath) map_writefields(out,inn.root,rootmpath) #walk tree; write results to out-tree for node_instance,mpath in map_treewalker(inn.root,mpath): mpath.append({'BOTSID':node_instance.record['BOTSID']}) if out.get(*mpath) is None: out.put(*mpath) map_writefields(out,node_instance,mpath) #***mapping is done; out-tree is finished; represents 'normalised' tree suited for writing as a grammar structure = [] recorddefs = {} tree2grammar(out.root,structure,recorddefs) #~ for key,value in recorddefs.items(): #~ print key,value #~ print '\n' sortedstructurelist = structure2list(structure) recorddefsstring = recorddefs2string(recorddefs,sortedstructurelist) structurestring = structure2string(structure) #write grammar file grammar = open(grammarfile,'wb') grammar.write('#grammar automatically generated by bots open source edi translator.') grammar.write('\n') grammar.write('from bots.botsconfig import *') grammar.write('\n\n') grammar.write('syntax = {}') grammar.write('\n\n') grammar.write('structure = [\n%s]\n'%(structurestring)) grammar.write('\n\n') grammar.write('recorddefs = %s'%(recorddefsstring)) grammar.write('\n\n') grammar.close() print 'grammar file is written:',grammarfile
def _translate_one_file(row, routedict, endstatus, userscript, scriptname): ''' - read, lex, parse, make tree of nodes. - split up files into messages (using 'nextmessage' of grammar) - get mappingscript, start mappingscript. - write the results of translation (no enveloping yet) ''' try: ta_fromfile = botslib.OldTransaction(row['idta']) ta_parsed = ta_fromfile.copyta(status=PARSED) if row['filesize'] > botsglobal.ini.getint( 'settings', 'maxfilesizeincoming', 5000000): ta_parsed.update(filesize=row['filesize']) raise botslib.FileTooLargeError( _(u'File size of %(filesize)s is too big; option "maxfilesizeincoming" in bots.ini is %(maxfilesizeincoming)s.' ), { 'filesize': row['filesize'], 'maxfilesizeincoming': botsglobal.ini.getint('settings', 'maxfilesizeincoming', 5000000) }) botsglobal.logger.debug( _(u'Start translating file "%(filename)s" editype "%(editype)s" messagetype "%(messagetype)s".' ), row) #read whole edi-file: read, parse and made into a inmessage-object. Message is represented as a tree (inmessage.root is the root of the tree). edifile = inmessage.parse_edi_file(frompartner=row['frompartner'], topartner=row['topartner'], filename=row['filename'], messagetype=row['messagetype'], testindicator=row['testindicator'], editype=row['editype'], charset=row['charset'], alt=row['alt'], fromchannel=row['fromchannel'], idroute=routedict['idroute'], command=routedict['command']) edifile.checkforerrorlist( ) #no exception if infile has been lexed and parsed OK else raises an error if int( routedict['translateind'] ) == 3: #parse & passthrough; file is parsed, partners are known, no mapping, does confirm. raise botslib.GotoException('dummy') #edifile.ta_info contains info: QUERIES, charset etc for inn_splitup in edifile.nextmessage( ): #splitup messages in parsed edifile try: ta_splitup = ta_parsed.copyta( status=SPLITUP, **inn_splitup.ta_info) #copy PARSED to SPLITUP ta #inn_splitup.ta_info: parameters from inmessage.parse_edi_file(), syntax-information and parse-information inn_splitup.ta_info[ 'idta_fromfile'] = ta_fromfile.idta #for confirmations in userscript; used to give idta of 'confirming message' inn_splitup.ta_info[ 'idta'] = ta_splitup.idta #for confirmations in userscript; used to give idta of 'confirming message' number_of_loops_with_same_alt = 0 while 1: #continue as long as there are (alt-)translations #lookup the translation************************ tscript, toeditype, tomessagetype = botslib.lookup_translation( fromeditype=inn_splitup.ta_info['editype'], frommessagetype=inn_splitup.ta_info['messagetype'], frompartner=inn_splitup.ta_info['frompartner'], topartner=inn_splitup.ta_info['topartner'], alt=inn_splitup.ta_info['alt']) if not tscript: #no translation found in translate table; check if can find translation via user script if userscript and hasattr(userscript, 'gettranslation'): tscript, toeditype, tomessagetype = botslib.runscript( userscript, scriptname, 'gettranslation', idroute=routedict['idroute'], message=inn_splitup) if not tscript: raise botslib.TranslationNotFoundError( _(u'Translation not found for editype "%(editype)s", messagetype "%(messagetype)s", frompartner "%(frompartner)s", topartner "%(topartner)s", alt "%(alt)s".' ), inn_splitup.ta_info) inn_splitup.ta_info[ 'divtext'] = tscript #ifor reporting used mapping script to database (for display in GUI). #initialize new out-object************************* ta_translated = ta_splitup.copyta( status=endstatus ) #make ta for translated message (new out-ta) filename_translated = unicode(ta_translated.idta) out_translated = outmessage.outmessage_init( editype=toeditype, messagetype=tomessagetype, filename=filename_translated, reference=unique('messagecounter'), statust=OK, divtext=tscript) #make outmessage object #run mapping script************************ botsglobal.logger.debug( _(u'Mappingscript "%(tscript)s" translates messagetype "%(messagetype)s" to messagetype "%(tomessagetype)s".' ), { 'tscript': tscript, 'messagetype': inn_splitup.ta_info['messagetype'], 'tomessagetype': out_translated.ta_info['messagetype'] }) translationscript, scriptfilename = botslib.botsimport( 'mappings', inn_splitup.ta_info['editype'], tscript) #get the mappingscript alt_from_previous_run = inn_splitup.ta_info[ 'alt'] #needed to check for infinite loop doalttranslation = botslib.runscript(translationscript, scriptfilename, 'main', inn=inn_splitup, out=out_translated) botsglobal.logger.debug( _(u'Mappingscript "%(tscript)s" finished.'), {'tscript': tscript}) #manipulate for some attributes after mapping script if 'topartner' not in out_translated.ta_info: #out_translated does not contain values from ta...... out_translated.ta_info[ 'topartner'] = inn_splitup.ta_info['topartner'] if 'botskey' in inn_splitup.ta_info: inn_splitup.ta_info['reference'] = inn_splitup.ta_info[ 'botskey'] if 'botskey' in out_translated.ta_info: #out_translated does not contain values from ta...... out_translated.ta_info[ 'reference'] = out_translated.ta_info['botskey'] #check the value received from the mappingscript to determine what to do in this while-loop. Handling of chained trasnlations. if doalttranslation is None: #translation(s) are done; handle out-message handle_out_message(out_translated, ta_translated) break #break out of while loop elif isinstance(doalttranslation, dict): #some extended cases; a dict is returned that contains 'instructions' for some type of chained translations if 'type' not in doalttranslation or 'alt' not in doalttranslation: raise botslib.BotsError( _(u"Mappingscript returned '%(alt)s'. This dict should not have 'type' and 'alt'." ), {'alt': doalttranslation}) if alt_from_previous_run == doalttranslation['alt']: number_of_loops_with_same_alt += 1 else: number_of_loops_with_same_alt = 0 if doalttranslation['type'] == u'out_as_inn': #do chained translation: use the out-object as inn-object, new out-object #use case: detected error in incoming file; use out-object to generate warning email handle_out_message(out_translated, ta_translated) inn_splitup = out_translated #out-object is now inn-object if isinstance( inn_splitup, outmessage.fixed ): #for fixed: strip all values in node inn_splitup.root.stripnode() inn_splitup.ta_info['alt'] = doalttranslation[ 'alt'] #get the alt-value for the next chained translation if not 'frompartner' in inn_splitup.ta_info: inn_splitup.ta_info['frompartner'] = '' if not 'topartner' in inn_splitup.ta_info: inn_splitup.ta_info['topartner'] = '' inn_splitup.ta_info.pop('statust') elif doalttranslation[ 'type'] == u'no_check_on_infinite_loop': #do chained translation: allow many loops wit hsame alt-value. #mapping script will have to handle this correctly. number_of_loops_with_same_alt = 0 handle_out_message(out_translated, ta_translated) inn_splitup.ta_info['alt'] = doalttranslation[ 'alt'] #get the alt-value for the next chained translation else: #there is nothing else raise botslib.BotsError( _(u'Mappingscript returned dict with an unknown "type": "%(doalttranslation)s".' ), {'doalttranslation': doalttranslation}) else: #note: this includes alt '' (empty string) if alt_from_previous_run == doalttranslation: number_of_loops_with_same_alt += 1 else: number_of_loops_with_same_alt = 0 #do normal chained translation: same inn-object, new out-object handle_out_message(out_translated, ta_translated) inn_splitup.ta_info[ 'alt'] = doalttranslation #get the alt-value for the next chained translation if number_of_loops_with_same_alt > 10: raise botslib.BotsError( _(u'Mappingscript returns same alt value over and over again (infinite loop?). Alt: "%(doalttranslation)s".' ), {'doalttranslation': doalttranslation}) #end of while-loop (trans********************************************************************************** #exceptions file_out-level: exception in mappingscript or writing of out-file except: #2 modes: either every error leads to skipping of whole infile (old mode) or errors in mappingscript/outfile only affect that branche if botsglobal.ini.getboolean('settings', 'oldmessageerrors', False): raise txt = botslib.txtexc() ta_splitup.update( statust=ERROR, errortext=txt, **inn_splitup.ta_info ) #update db. inn_splitup.ta_info could be changed by mappingscript. Is this useful? ta_splitup.deletechildren() else: ta_splitup.update( statust=DONE, **inn_splitup.ta_info ) #update db. inn_splitup.ta_info could be changed by mappingscript. Is this useful? #exceptions file_in-level except botslib.GotoException: #edi-file is OK, file is passed-through after parsing. ta_parsed.update(statust=DONE, filesize=row['filesize'], **edifile.ta_info) #update with info from eg queries ta_parsed.copyta(status=MERGED, statust=OK) #original file goes straight to MERGED edifile.handleconfirm(ta_fromfile, error=False) botsglobal.logger.debug( _(u'Parse & passthrough for input file "%(filename)s".'), row) except botslib.FileTooLargeError as msg: ta_parsed.update(statust=ERROR, errortext=unicode(msg)) ta_parsed.deletechildren() botsglobal.logger.debug( u'Error in translating input file "%(filename)s":\n%(msg)s', { 'filename': row['filename'], 'msg': msg }) except: txt = botslib.txtexc() ta_parsed.update(statust=ERROR, errortext=txt, **edifile.ta_info) ta_parsed.deletechildren() edifile.handleconfirm(ta_fromfile, error=True) botsglobal.logger.debug( u'Error in translating input file "%(filename)s":\n%(msg)s', { 'filename': row['filename'], 'msg': txt }) else: edifile.handleconfirm(ta_fromfile, error=False) ta_parsed.update(statust=DONE, filesize=row['filesize'], **edifile.ta_info) botsglobal.logger.debug(_(u'Translated input file "%(filename)s".'), row) finally: ta_fromfile.update(statust=DONE)
def start(): #********command line arguments************************** usage = ''' This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net). Creates a grammar from an xml file.' Usage:' %(name)s -c<directory> <xml_file> <xml_grammar_file> Options: -c<directory> directory for configuration files (default: config). -a all xml elements as records <xml_file> name of the xml file to read <xml_grammar_file> name of the grammar file to write '''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version} configdir = 'config' edifile ='' botsgrammarfilename = '' allrecords = False for arg in sys.argv[1:]: if arg.startswith('-c'): configdir = arg[2:] if not configdir: print 'Error: configuration directory indicated, but no directory name.' sys.exit(1) elif arg.startswith('-a'): allrecords = True elif arg in ["?", "/?",'-h', '--help'] or arg.startswith('-'): print usage sys.exit(0) else: if not edifile: edifile = arg else: botsgrammarfilename = arg if not edifile or not botsgrammarfilename: print 'Error: both edifile and grammarfile are required.' sys.exit(0) #***end handling command line arguments************************** botsinit.generalinit(configdir) #find locating of bots, configfiles, init paths etc. process_name = 'xml2botsgrammar' botsglobal.logger = botsinit.initenginelogging(process_name) atexit.register(logging.shutdown) targetNamespace = '' #******************************************************************* #***add classes for handling editype xml to inmessage #******************************************************************* if allrecords: #~ editype = 'xmlforgrammar_allrecords' inmessage.xmlforgrammar = xmlforgrammar_allrecords else: #~ editype = 'xmlforgrammar' inmessage.xmlforgrammar = xmlforgrammar #make inmessage object: read the xml file inn = inmessage.parse_edi_file(editype='xmlforgrammar',messagetype='',filename=edifile) inn.checkforerrorlist() #no exception if infile has been lexed and parsed OK else raises an error #make outmessage object; nothing is 'filled' yet. In mapping tree is filled; nothing is written to file. out = outmessage.outmessage_init(editype='xmlnocheck',messagetype='',filename='',divtext='',topartner='') #***mapping: make 'normalised' out-tree suited for writing as a grammar*************************************************** mpath_root = [OrderedDict({'BOTSID':inn.root.record['BOTSID'],'BOTSIDnr':'1'})] #handle root out.put(*mpath_root) map_writefields(out,inn.root,mpath_root) #walk tree; write results to out-tree mpath_start = [] for node_instance,mpath in map_treewalker(inn.root,mpath_start): mpath.append(OrderedDict({'BOTSID':node_instance.record['BOTSID']})) if out.get(*mpath) is None: #if node does not exist: write it. out.put(*mpath) map_writefields(out,node_instance,mpath) #***mapping is done #***convert out-tree to grammar structure = [] recorddefs = {} tree2grammar(out.root,structure,recorddefs) #***write grammar to file grammar2file(botsgrammarfilename,structure,recorddefs,targetNamespace)