def cleanup(do_cleanup_parameter,userscript,scriptname): ''' public function, does all cleanup of the database and file system. most cleanup functions are by default done only once a day. ''' if botsglobal.ini.getboolean('acceptance','runacceptancetest',False): # no cleanup during acceptance testing return whencleanup = botsglobal.ini.get('settings','whencleanup','daily') if do_cleanup_parameter: #if explicit indicated via commandline parameter do_full_cleanup = True elif whencleanup in ['always','daily']: #perform full cleanup only first run of the day. cur_day = int(time.strftime('%Y%m%d')) #get current date, convert to int if cur_day != botslib.uniquecore('bots_cleanup_day',updatewith=cur_day): do_full_cleanup = True else: do_full_cleanup = False else: do_full_cleanup = False try: if do_full_cleanup: botsglobal.logger.info(u'Cleanup files') _cleandatafile() _cleanarchive() botsglobal.logger.info(u'Cleanup database') _cleanupsession() _cleanpersist() _cleantransactions() botsglobal.logger.info(u'Vacuum database') _vacuum() # postcleanup user exit in botsengine script if userscript and hasattr(userscript,'postcleanup'): botslib.runscript(userscript,scriptname,'postcleanup',whencleanup=whencleanup) botsglobal.logger.info(u'Done full cleanup.') _cleanrunsnothingreceived() #do this every run, but not logged except: botsglobal.logger.exception(u'Cleanup error.')
def _translate_one_file(row,routedict,endstatus,userscript,scriptname): ''' - read, lex, parse, make tree of nodes. - split up files into messages (using 'nextmessage' of grammar) - get mappingscript, start mappingscript. - write the results of translation (no enveloping yet) ''' try: ta_fromfile = botslib.OldTransaction(row['idta']) ta_parsed = ta_fromfile.copyta(status=PARSED) if row['filesize'] > botsglobal.ini.getint('settings','maxfilesizeincoming',5000000): ta_parsed.update(filesize=row['filesize']) raise botslib.FileTooLarge(_(u'File size of %(filesize)s is too big; option "maxfilesizeincoming" in bots.ini is %(maxfilesizeincoming)s.'), {'filesize':row['filesize'],'maxfilesizeincoming':botsglobal.ini.getint('settings','maxfilesizeincoming',5000000)}) botsglobal.logger.debug(_(u'Start translating file "%(filename)s" editype "%(editype)s" messagetype "%(messagetype)s".'),row) #read whole edi-file: read, parse and made into a inmessage-object. Message is represented as a tree (inmessage.root is the root of the tree). edifile = inmessage.parse_edi_file(frompartner=row['frompartner'], topartner=row['topartner'], filename=row['filename'], messagetype=row['messagetype'], testindicator=row['testindicator'], editype=row['editype'], charset=row['charset'], alt=row['alt'], fromchannel=row['fromchannel'], idroute=routedict['idroute'], command=routedict['command']) edifile.checkforerrorlist() #no exception if infile has been lexed and parsed OK else raises an error if int(routedict['translateind']) == 3: #parse & passthrough; file is parsed, partners are known, no mapping, does confirm. raise botslib.GotoException('dummy') #edifile.ta_info contains info: QUERIES, charset etc for inn_splitup in edifile.nextmessage(): #splitup messages in parsed edifile try: ta_splitup = ta_parsed.copyta(status=SPLITUP,**inn_splitup.ta_info) #copy PARSED to SPLITUP ta #inn_splitup.ta_info: parameters from inmessage.parse_edi_file(), syntax-information and parse-information inn_splitup.ta_info['idta_fromfile'] = ta_fromfile.idta #for confirmations in userscript; used to give idta of 'confirming message' inn_splitup.ta_info['idta'] = ta_splitup.idta #for confirmations in userscript; used to give idta of 'confirming message' number_of_loops_with_same_alt = 0 while 1: #continue as long as there are (alt-)translations #lookup the translation************************ tscript,toeditype,tomessagetype = botslib.lookup_translation(fromeditype=inn_splitup.ta_info['editype'], frommessagetype=inn_splitup.ta_info['messagetype'], frompartner=inn_splitup.ta_info['frompartner'], topartner=inn_splitup.ta_info['topartner'], alt=inn_splitup.ta_info['alt']) if not tscript: #no translation found in translate table; check if can find translation via user script if userscript and hasattr(userscript,'gettranslation'): tscript,toeditype,tomessagetype = botslib.runscript(userscript,scriptname,'gettranslation',idroute=routedict['idroute'],message=inn_splitup) if not tscript: raise botslib.TranslationNotFoundError(_(u'Translation not found for editype "%(editype)s", messagetype "%(messagetype)s", frompartner "%(frompartner)s", topartner "%(topartner)s", alt "%(alt)s".'), inn_splitup.ta_info) inn_splitup.ta_info['divtext'] = tscript #ifor reporting used mapping script to database (for display in GUI). #initialize new out-object************************* ta_translated = ta_splitup.copyta(status=endstatus) #make ta for translated message (new out-ta) filename_translated = str(ta_translated.idta) out_translated = outmessage.outmessage_init(editype=toeditype,messagetype=tomessagetype,filename=filename_translated,reference=unique('messagecounter'),statust=OK,divtext=tscript) #make outmessage object #run mapping script************************ botsglobal.logger.debug(_(u'Mappingscript "%(tscript)s" translates messagetype "%(messagetype)s" to messagetype "%(tomessagetype)s".'), {'tscript':tscript,'messagetype':inn_splitup.ta_info['messagetype'],'tomessagetype':out_translated.ta_info['messagetype']}) translationscript,scriptfilename = botslib.botsimport('mappings',inn_splitup.ta_info['editype'],tscript) #get the mappingscript alt_from_previous_run = inn_splitup.ta_info['alt'] #needed to check for infinite loop doalttranslation = botslib.runscript(translationscript,scriptfilename,'main',inn=inn_splitup,out=out_translated) botsglobal.logger.debug(_(u'Mappingscript "%(tscript)s" finished.'),{'tscript':tscript}) #manipulate for some attributes after mapping script if 'topartner' not in out_translated.ta_info: #out_translated does not contain values from ta...... out_translated.ta_info['topartner'] = inn_splitup.ta_info['topartner'] if 'botskey' in inn_splitup.ta_info: inn_splitup.ta_info['reference'] = inn_splitup.ta_info['botskey'] if 'botskey' in out_translated.ta_info: #out_translated does not contain values from ta...... out_translated.ta_info['reference'] = out_translated.ta_info['botskey'] #check the value received from the mappingscript to determine what to do in this while-loop. Handling of chained trasnlations. if doalttranslation is None: #translation(s) are done; handle out-message handle_out_message(out_translated,ta_translated) break #break out of while loop elif isinstance(doalttranslation,dict): #some extended cases; a dict is returned that contains 'instructions' for some type of chained translations if 'type' not in doalttranslation or 'alt' not in doalttranslation: raise botslib.BotsError(_(u"Mappingscript returned '%(alt)s'. This dict should not have 'type' and 'alt'."),{'alt':doalttranslation}) if alt_from_previous_run == doalttranslation['alt']: number_of_loops_with_same_alt += 1 else: number_of_loops_with_same_alt = 0 if doalttranslation['type'] == u'out_as_inn': #do chained translation: use the out-object as inn-object, new out-object #use case: detected error in incoming file; use out-object to generate warning email handle_out_message(out_translated,ta_translated) inn_splitup = out_translated #out-object is now inn-object if isinstance(inn_splitup,outmessage.fixed): #for fixed: strip all values in node inn_splitup.root.stripnode() inn_splitup.ta_info['alt'] = doalttranslation['alt'] #get the alt-value for the next chained translation if not 'frompartner' in inn_splitup.ta_info: inn_splitup.ta_info['frompartner'] = '' if not 'topartner' in inn_splitup.ta_info: inn_splitup.ta_info['topartner'] = '' inn_splitup.ta_info.pop('statust') elif doalttranslation['type'] == u'no_check_on_infinite_loop': #do chained translation: allow many loops wit hsame alt-value. #mapping script will have to handle this correctly. number_of_loops_with_same_alt = 0 handle_out_message(out_translated,ta_translated) inn_splitup.ta_info['alt'] = doalttranslation['alt'] #get the alt-value for the next chained translation else: #there is nothing else raise botslib.BotsError(_(u'Mappingscript returned dict with an unknown "type": "%(doalttranslation)s".'),{'doalttranslation':doalttranslation}) else: #note: this includes alt '' (empty string) if alt_from_previous_run == doalttranslation: number_of_loops_with_same_alt += 1 else: number_of_loops_with_same_alt = 0 #do normal chained translation: same inn-object, new out-object handle_out_message(out_translated,ta_translated) inn_splitup.ta_info['alt'] = doalttranslation #get the alt-value for the next chained translation if number_of_loops_with_same_alt > 10: raise botslib.BotsError(_(u'Mappingscript returns same alt value over and over again (infinite loop?). Alt: "%(doalttranslation)s".'),{'doalttranslation':doalttranslation}) #end of while-loop (trans********************************************************************************** #exceptions file_out-level: exception in mappingscript or writing of out-file except: #2 modes: either every error leads to skipping of whole infile (old mode) or errors in mappingscript/outfile only affect that branche if botsglobal.ini.getboolean('settings','oldmessageerrors',False): raise txt = botslib.txtexc() ta_splitup.update(statust=ERROR,errortext=txt,**inn_splitup.ta_info) #update db. inn_splitup.ta_info could be changed by mappingscript. Is this useful? ta_splitup.deletechildren() else: ta_splitup.update(statust=DONE, **inn_splitup.ta_info) #update db. inn_splitup.ta_info could be changed by mappingscript. Is this useful? #exceptions file_in-level except botslib.GotoException: #edi-file is OK, file is passed-through after parsing. ta_parsed.update(statust=DONE,filesize=row['filesize'],**edifile.ta_info) #update with info from eg queries ta_parsed.copyta(status=MERGED,statust=OK) #original file goes straight to MERGED edifile.handleconfirm(ta_fromfile,error=False) botsglobal.logger.debug(_(u'Parse & passthrough for input file "%(filename)s".'),row) except botslib.FileTooLarge as msg: ta_parsed.update(statust=ERROR,errortext=str(msg)) ta_parsed.deletechildren() botsglobal.logger.debug(u'Error in translating input file "%(filename)s":\n%(msg)s',{'filename':row['filename'],'msg':msg}) except: txt = botslib.txtexc() ta_parsed.update(statust=ERROR,errortext=txt,**edifile.ta_info) ta_parsed.deletechildren() edifile.handleconfirm(ta_fromfile,error=True) botsglobal.logger.debug(u'Error in translating input file "%(filename)s":\n%(msg)s',{'filename':row['filename'],'msg':txt}) else: edifile.handleconfirm(ta_fromfile,error=False) ta_parsed.update(statust=DONE,filesize=row['filesize'],**edifile.ta_info) botsglobal.logger.debug(_(u'Translated input file "%(filename)s".'),row) finally: ta_fromfile.update(statust=DONE)
def start(): #exit codes: # 0: OK, no errors # 1: (system) errors # 2: bots ran OK, but there are errors/process errors in the run # 3: Database is locked, but "maxruntime" has not been exceeded. #********command line arguments************************** commandspossible = [ '--new', '--retry', '--retransmit', '--cleanup', '--crashrecovery', '--retrycommunication', '--automaticretrycommunication' ] commandstorun = [] routestorun = [] #list with routes to run configdir = 'config' for arg in sys.argv[1:]: if not arg: continue if arg.startswith('-c'): configdir = arg[2:] if not configdir: print 'Configuration directory indicated, but no directory name.' sys.exit(1) elif arg in commandspossible: commandstorun.append(arg) elif arg in ["?", "/?"] or arg.startswith('-'): showusage() sys.exit(0) else: #pick up names of routes to run routestorun.append(arg) if not commandstorun: #if no command on command line, use new (default) commandstorun = ['--new'] #**************init general: find locating of bots, configfiles, init paths etc.**************** botsinit.generalinit(configdir) #set current working directory to botspath #~ old_current_directory = os.getcwdu() os.chdir(botsglobal.ini.get('directories', 'botspath')) #**************initialise logging****************************** try: botsinit.initenginelogging() except: print _('Error in initialising logging system.') traceback.print_exc() sys.exit(1) else: atexit.register(logging.shutdown) for key, value in botslib.botsinfo(): #log start info botsglobal.logger.info(u'%s: "%s".', key, value) #**************connect to database********************************** try: botsinit.connect() except: botsglobal.logger.exception( _(u'Could not connect to database. Database settings are in bots/config/settings.py.' )) sys.exit(1) else: botsglobal.logger.info(_(u'Connected to database.')) atexit.register(botsglobal.db.close) #initialise user exits for the whole bots-engine (this script file) try: userscript, scriptname = botslib.botsimport('routescripts', 'botsengine') except ImportError: userscript = scriptname = None #**************handle database lock**************************************** #try to set a lock on the database; if this is not possible, the database is already locked. Either: #1 another instance bots bots-engine is (still) running #2 or bots-engine had a severe crash. #What to do? #first: check ts of database lock. If below a certain value (set in bots.ini) we assume an other instance is running. Exit quietly - no errors, no logging. # else: Warn user, give advise on what to do. gather data: nr files in, errors. #next: warn with report & logging. advise a crashrecovery. if not botslib.set_database_lock(): if '--crashrecovery' in commandstorun: #user starts recovery operation; the databaselock is ignored; the databaselock is unlocked when routes have run. commandstorun = ['--crashrecovery'] #is an exclusive option! else: #when scheduling bots it is possible that the last run is still running. Check if maxruntime has passed: vanaf = datetime.datetime.today() - datetime.timedelta( minutes=botsglobal.ini.getint('settings', 'maxruntime', 60)) for row in botslib.query( '''SELECT ts FROM mutex WHERE ts < %(vanaf)s ''', {'vanaf': vanaf}): warn = _( u'!Bots database is locked!\nBots-engine has ended in an unexpected way during the last run.\nThis happens, but is very very rare.\nPossible causes: bots-engine terminated by user, system crash, power-down, etc.\nA forced retry of the last run is advised; bots will (try to) repair the last run.' ) botsglobal.logger.critical(warn) botslib.sendbotserrorreport( _(u'[Bots severe error]Database is locked'), warn) #add: count errors etc. sys.exit(1) else: #maxruntime has not passed. Exit silently, nothing reported botsglobal.logger.info( _(u'Database is locked, but "maxruntime" has not been exceeded.' )) sys.exit(3) else: if '--crashrecovery' in commandstorun: #user starts recovery operation but there is no databaselock. warn = _( u'User started a forced retry of the last run.\nOnly use this when the database is locked.\nThe database was not locked (database is OK).\nSo Bots has done nothing now.' ) botsglobal.logger.error(warn) botslib.sendbotserrorreport( _(u'[Bots Error Report] User started a forced retry of last run, but this was not needed' ), warn) botslib.remove_database_lock() sys.exit(1) #*************get list of routes to run**************************************** #~ raise Exception('locked database') #for testing database lock: abort, database will be locked if routestorun: botsglobal.logger.info(u'Run routes from command line: "%s".', str(routestorun)) else: # no routes from command line parameters: fetch all active routes from database for row in botslib.query( '''SELECT DISTINCT idroute FROM routes WHERE active=%(active)s AND (notindefaultrun=%(notindefaultrun)s OR notindefaultrun IS NULL) ORDER BY idroute ''', { 'active': True, 'notindefaultrun': False }): routestorun.append(row['idroute']) botsglobal.logger.info(_(u'Run active routes from database: "%s".'), str(routestorun)) #routestorun is now either a list with routes from commandline, or the list of active routes for the routes table in the db. #**************run the routes for retry, retransmit and new runs************************************* try: #commandstorun determines the type(s) of run #routes to run is a listof the routes that are runs (for each command to run #botsglobal.incommunicate is used to control if there is communication in; only 'new' incommunicates. #botsglobal.minta4query controls which ta's are queried by the routes. #stuff2evaluate controls what is evaluated in automatic maintenance. errorinrun = 0 #detect if there has been some error. Only used for good exit() code botsglobal.incommunicate = False if '--crashrecovery' in commandstorun: botsglobal.logger.info(_(u'Run crash recovery.')) stuff2evaluate = botslib.set_minta4query_crashrecovery() if stuff2evaluate: router.routedispatcher(routestorun) errorinrun += automaticmaintenance.evaluate( '--crashrecovery', stuff2evaluate) else: botsglobal.logger.info( _(u'No retry of the last run - there was no last run.')) if userscript and hasattr(userscript, 'postcrashrecovery'): botslib.runscript(userscript, scriptname, 'postcrashrecovery', routestorun=routestorun) if '--retrycommunication' in commandstorun: botsglobal.logger.info(_(u'Run communication retry.')) stuff2evaluate = router.routedispatcher(routestorun, '--retrycommunication') if stuff2evaluate: errorinrun += automaticmaintenance.evaluate( '--retrycommunication', stuff2evaluate) else: botsglobal.logger.info( _(u'Run recommunicate: nothing to recommunicate.')) if userscript and hasattr(userscript, 'postretrycommunication'): botslib.runscript(userscript, scriptname, 'postretrycommunication', routestorun=routestorun) if '--automaticretrycommunication' in commandstorun: botsglobal.logger.info(_(u'Run automatic communication retry.')) stuff2evaluate = router.routedispatcher( routestorun, '--automaticretrycommunication') if stuff2evaluate: errorinrun += automaticmaintenance.evaluate( '--automaticretrycommunication', stuff2evaluate) else: botsglobal.logger.info( _(u'Run automatic recommunicate: nothing to recommunicate.' )) if userscript and hasattr(userscript, 'postautomaticretrycommunication'): botslib.runscript(userscript, scriptname, 'postautomaticretrycommunication', routestorun=routestorun) if '--retry' in commandstorun: botsglobal.logger.info(u'Run retry.') stuff2evaluate = router.routedispatcher(routestorun, '--retry') if stuff2evaluate: errorinrun += automaticmaintenance.evaluate( '--retry', stuff2evaluate) else: botsglobal.logger.info(_(u'Run retry: nothing to retry.')) if userscript and hasattr(userscript, 'postretry'): botslib.runscript(userscript, scriptname, 'postretry', routestorun=routestorun) if '--retransmit' in commandstorun: botsglobal.logger.info(u'Run retransmit.') stuff2evaluate = router.routedispatcher(routestorun, '--retransmit') if stuff2evaluate: errorinrun += automaticmaintenance.evaluate( '--retransmit', stuff2evaluate) else: botsglobal.logger.info( _(u'Run retransmit: nothing to retransmit.')) if userscript and hasattr(userscript, 'postretransmit'): botslib.runscript(userscript, scriptname, 'postretransmit', routestorun=routestorun) if '--new' in commandstorun: botsglobal.logger.info('Run new.') botsglobal.incommunicate = True botsglobal.minta4query = 0 #meaning: reset. the actual value is set later (in routedispatcher) stuff2evaluate = router.routedispatcher(routestorun) errorinrun += automaticmaintenance.evaluate( '--new', stuff2evaluate) if userscript and hasattr(userscript, 'postnewrun'): botslib.runscript(userscript, scriptname, 'postnewrun', routestorun=routestorun) if '--cleanup' in commandstorun or botsglobal.ini.get( 'settings', 'whencleanup', 'always') == 'always': botsglobal.logger.debug(u'Do cleanup.') cleanup.cleanup() botslib.remove_database_lock() except Exception, e: botsglobal.logger.exception( _(u'Severe error in bots system:\n%s') % (e)) #of course this 'should' not happen. sys.exit(1)
def translate(startstatus,endstatus,idroute,rootidta): ''' main translation loop. get edifiles to be translated, than: - read, lex, parse, make tree of nodes. - split up files into messages (using 'nextmessage' of grammar) - get mappingscript, start mappingscript. - write the results of translation (no enveloping yet) status: FILEIN--PARSED-<SPLITUP--TRANSLATED ''' try: #see if there is a userscript that can determine the translation userscript,scriptname = botslib.botsimport('mappings','translation') except ImportError: #userscript is not there; other errors like syntax errors are not catched userscript = scriptname = None #select edifiles to translate for rawrow in botslib.query(u'''SELECT idta,frompartner,topartner,filename,messagetype,testindicator,editype,charset,alt,fromchannel,filesize FROM ta WHERE idta>%(rootidta)s AND status=%(status)s AND statust=%(statust)s AND idroute=%(idroute)s ''', {'status':startstatus,'statust':OK,'idroute':idroute,'rootidta':rootidta}): try: row = dict(rawrow) #convert to real dictionary () ta_fromfile = botslib.OldTransaction(row['idta']) ta_parsed = ta_fromfile.copyta(status=PARSED) #make PARSED ta if row['filesize'] > botsglobal.ini.getint('settings','maxfilesizeincoming',5000000): ta_parsed.update(filesize=row['filesize']) raise botslib.InMessageError(_(u'File size of %(filesize)s is too big; option "maxfilesizeincoming" in bots.ini is %(maxfilesizeincoming)s.'), {'filesize':row['filesize'],'maxfilesizeincoming':botsglobal.ini.getint('settings','maxfilesizeincoming',5000000)}) botsglobal.logger.debug(_(u'Start translating file "%(filename)s" editype "%(editype)s" messagetype "%(messagetype)s".'),row) #read whole edi-file: read, parse and made into a inmessage-object. Message is represented as a tree (inmessage.root is the root of the tree). edifile = inmessage.parse_edi_file(frompartner=row['frompartner'], topartner=row['topartner'], filename=row['filename'], messagetype=row['messagetype'], testindicator=row['testindicator'], editype=row['editype'], charset=row['charset'], alt=row['alt'], fromchannel=row['fromchannel'], idroute=idroute) #if no exception: infile has been lexed and parsed OK. #edifile.ta_info contains info: QUERIES, charset etc for inn_splitup in edifile.nextmessage(): #splitup messages in parsed edifile try: ta_splitup = ta_parsed.copyta(status=SPLITUP,**inn_splitup.ta_info) #copy PARSED to SPLITUP ta #inn_splitup.ta_info: parameters from inmessage.parse_edi_file(), syntax-information and parse-information inn_splitup.ta_info['idta_fromfile'] = ta_fromfile.idta #for confirmations in userscript; used to give idta of 'confirming message' post_mapping_mode = False #if post_mapping: reuse out-object, if no translation is found no error number_of_loops_with_same_alt = 0 while 1: #continue as long as there are (alt-)translations #lookup the translation************************ tscript,toeditype,tomessagetype = botslib.lookup_translation(fromeditype=inn_splitup.ta_info['editype'], frommessagetype=inn_splitup.ta_info['messagetype'], frompartner=inn_splitup.ta_info['frompartner'], topartner=inn_splitup.ta_info['topartner'], alt=inn_splitup.ta_info['alt']) if not tscript: #no translation found in translate table; check if can find translation via user script if userscript and hasattr(userscript,'gettranslation'): tscript,toeditype,tomessagetype = botslib.runscript(userscript,scriptname,'gettranslation',idroute=idroute,message=inn_splitup) if not tscript: if post_mapping_mode: #in post-mapping mode, not finding the (partern specific!) script is no problem: translation is done #there is no post-mapping script found (in other words, default mapping script is enough). #all OK, handle the generated out-file and continue with next message botsglobal.logger.debug(_(u'Found no "post-mapping" translation for editype "%(editype)s", messagetype "%(messagetype)s", frompartner "%(frompartner)s", topartner "%(topartner)s", alt "%(alt)s".'), inn_splitup.ta_info) handle_out_message(out_translated,ta_translated) del out_translated break #break out of while loop else: raise botslib.TranslationNotFoundError(_(u'Translation not found for editype "%(editype)s", messagetype "%(messagetype)s", frompartner "%(frompartner)s", topartner "%(topartner)s", alt "%(alt)s".'), inn_splitup.ta_info) inn_splitup.ta_info['divtext'] = tscript #ifor reporting used mapping script to database (for display in GUI). if not post_mapping_mode: #initialize new out-object************************* ta_translated = ta_splitup.copyta(status=endstatus) #make ta for translated message (new out-ta) filename_translated = str(ta_translated.idta) out_translated = outmessage.outmessage_init(editype=toeditype,messagetype=tomessagetype,filename=filename_translated,reference=unique('messagecounter'),statust=OK,divtext=tscript) #make outmessage object #run mapping script************************ botsglobal.logger.debug(_(u'Mappingscript "%(tscript)s" translates messagetype "%(messagetype)s" to messagetype "%(tomessagetype)s".'), {'tscript':tscript,'messagetype':inn_splitup.ta_info['messagetype'],'tomessagetype':out_translated.ta_info['messagetype']}) translationscript,scriptfilename = botslib.botsimport('mappings',inn_splitup.ta_info['editype'],tscript) #get the mappingscript alt_from_previous_run = inn_splitup.ta_info['alt'] #needed to check for infinite loop doalttranslation = botslib.runscript(translationscript,scriptfilename,'main',inn=inn_splitup,out=out_translated) botsglobal.logger.debug(_(u'Mappingscript "%(tscript)s" finished.'),{'tscript':tscript}) #manipulate for some attributes after mapping script if 'topartner' not in out_translated.ta_info: #out_translated does not contain values from ta...... out_translated.ta_info['topartner'] = inn_splitup.ta_info['topartner'] if 'botskey' in inn_splitup.ta_info: inn_splitup.ta_info['reference'] = inn_splitup.ta_info['botskey'] if 'botskey' in out_translated.ta_info: #out_translated does not contain values from ta...... out_translated.ta_info['reference'] = out_translated.ta_info['botskey'] #check the value received from the mappingscript to determine what to do in this while-loop. Handling of chained trasnlations. if doalttranslation is None: #translation(s) are done; handle out-message handle_out_message(out_translated,ta_translated) del out_translated break #break out of while loop elif isinstance(doalttranslation,dict): #some extended cases; a dict is returned that contains 'instructions' for some type of chained translations if 'type' not in doalttranslation or 'alt' not in doalttranslation: raise botslib.BotsError(_(u"Mappingscript returned '%(alt)s'. This dict should not have 'type' and 'alt'."),{'alt':doalttranslation}) if alt_from_previous_run == doalttranslation['alt']: number_of_loops_with_same_alt += 1 else: number_of_loops_with_same_alt = 0 if doalttranslation['type'] == u'out_as_inn': #do chained translation: use the out-object as inn-object, new out-object #use case: detected error in incoming file; use out-object to generate warning email handle_out_message(out_translated,ta_translated) inn_splitup = out_translated #out-object is now inn-object if isinstance(inn_splitup,outmessage.fixed): #for fixed: strip all values in node inn_splitup.root.stripnode() inn_splitup.ta_info['alt'] = doalttranslation['alt'] #get the alt-value for the next chained translation if not 'frompartner' in inn_splitup.ta_info: inn_splitup.ta_info['frompartner'] = '' if not 'topartner' in inn_splitup.ta_info: inn_splitup.ta_info['topartner'] = '' inn_splitup.ta_info.pop('statust') elif doalttranslation['type'] == u'no_check_on_infinite_loop': #do chained translation: allow many loops wit hsame alt-value. #mapping script will have to handle this correctly. number_of_loops_with_same_alt = 0 handle_out_message(out_translated,ta_translated) del out_translated inn_splitup.ta_info['alt'] = doalttranslation['alt'] #get the alt-value for the next chained translation elif doalttranslation['type'] == u'post_mapping': #do chained (post)translation: same inn and out-objects. #in other words, the post-translation continue where is 'default' mapping ended. #use case: default mapping (for all partners), partner-specific post-translation #note: this is easier via import defaultmapping in a partner specific mapping. Please use that method! post_mapping_mode = True #translation is done; reset post_mapping_mode inn_splitup.ta_info['alt'] = doalttranslation['alt'] #get the alt-value for the next chained translation #~ inn_splitup.ta_info.pop('statust') else: #there is nothing else raise botslib.BotsError(_(u'Mappingscript returned dict with an unknown "type": "%(doalttranslation)s".'),{'doalttranslation':doalttranslation}) else: #note: this includes alt '' (empty string) if alt_from_previous_run == doalttranslation: number_of_loops_with_same_alt += 1 else: number_of_loops_with_same_alt = 0 #do normal chained translation: same inn-object, new out-object handle_out_message(out_translated,ta_translated) del out_translated inn_splitup.ta_info['alt'] = doalttranslation #get the alt-value for the next chained translation if number_of_loops_with_same_alt > 10: raise botslib.BotsError(_(u'Mappingscript returns same alt value over and over again (infinite loop?). Alt: "%(doalttranslation)s".'),{'doalttranslation':doalttranslation}) #end of while-loop (trans********************************************************************************** #exceptions file_out-level: exception in mappingscript or writing of out-file except: #2 modes: either every error leads to skipping of whole infile (old mode) or errors in mappingscript/outfile only affect that branche if botsglobal.ini.getboolean('settings','oldmessageerrors',False): raise txt = botslib.txtexc() ta_splitup.update(statust=ERROR,errortext=txt,**inn_splitup.ta_info) #update db. inn_splitup.ta_info could be changed by mappingscript. Is this useful? ta_splitup.deletechildren() else: ta_splitup.update(statust=DONE, **inn_splitup.ta_info) #update db. inn_splitup.ta_info could be changed by mappingscript. Is this useful? #exceptions file_in-level (file not OK according to grammar) except: txt = botslib.txtexc() ta_parsed.update(statust=ERROR,errortext=txt) ta_parsed.deletechildren() botsglobal.logger.debug(u'Error in translating input file "%(filename)s":\n%(msg)s',{'filename':row['filename'],'msg':txt}) else: edifile.handleconfirm(ta_fromfile,error=False) ta_parsed.update(statust=DONE,filesize=row['filesize'],**edifile.ta_info) botsglobal.logger.debug(_(u'Translated input file "%(filename)s".'),row) finally: ta_fromfile.update(statust=DONE)
def translate(run): for messagedict in run.incoming: try: #read whole edi-file: read, parse and made into a inmessage-object. Message is represented as a tree (inmessage.root is the root of the tree). edifile = inmessage.parse_edi_file(frompartner='', topartner='', filename=messagedict['filename'], messagetype=run.translation['messagetype'], testindicator='', editype=run.translation['editype'], charset='', alt='', fromchannel='', idroute='', command='') edifile.checkforerrorlist() #no exception if infile has been lexed and parsed OK else raises an error #~ if int(routedict['translateind']) == 3: #parse & passthrough; file is parsed, partners are known, no mapping, does confirm. #~ raise botslib.GotoException('dummy') #~ continue #file is parsed; no errors. no translation #edifile.ta_info contains info about incoming file: QUERIES, charset etc for inn_splitup in edifile.nextmessage(): #splitup messages in parsed edifile try: #inn_splitup.ta_info: parameters from inmessage.parse_edi_file(), syntax-information and parse-information number_of_loops_with_same_alt = 0 while 1: #continue as long as there are (alt-)translations #lookup the translation************************ tscript,toeditype,tomessagetype = 'orders_edifact2xml' ,'xml','orders' if 'tscript' in run.translation: tscript = run.translation['tscript'] toeditype = run.translation['toeditype'] tomessagetype = run.translation['tomessagetype'] else: tscript,toeditype,tomessagetype = botslib.lookup_translation(fromeditype=inn_splitup.ta_info['editype'], frommessagetype=inn_splitup.ta_info['messagetype'], frompartner=inn_splitup.ta_info['frompartner'], topartner=inn_splitup.ta_info['topartner'], alt=inn_splitup.ta_info['alt']) #run mapping script************************ filename_translated = transform.unique('bots_file_name') out_translated = outmessage.outmessage_init(editype=toeditype, messagetype=tomessagetype, filename=filename_translated, reference=transform.unique('messagecounter'), statust=OK, divtext=tscript) #make outmessage object #~ botsglobal.logger.debug(_(u'Mappingscript "%(tscript)s" translates messagetype "%(messagetype)s" to messagetype "%(tomessagetype)s".'), #~ {'tscript':tscript,'messagetype':inn_splitup.ta_info['messagetype'],'tomessagetype':out_translated.ta_info['messagetype']}) translationscript,scriptfilename = botslib.botsimport('mappings',inn_splitup.ta_info['editype'],tscript) #import mappingscript alt_from_previous_run = inn_splitup.ta_info['alt'] #needed to check for infinite loop doalttranslation = botslib.runscript(translationscript,scriptfilename,'main',inn=inn_splitup,out=out_translated) botsglobal.logger.debug(_(u'Mappingscript "%(tscript)s" finished.'),{'tscript':tscript}) #manipulate for some attributes after mapping script if 'topartner' not in out_translated.ta_info: #out_translated does not contain values from run...... out_translated.ta_info['topartner'] = inn_splitup.ta_info['topartner'] if 'botskey' in inn_splitup.ta_info: inn_splitup.ta_info['reference'] = inn_splitup.ta_info['botskey'] if 'botskey' in out_translated.ta_info: #out_translated does not contain values from run...... out_translated.ta_info['reference'] = out_translated.ta_info['botskey'] #check the value received from the mappingscript to determine what to do in this while-loop. Handling of chained trasnlations. if doalttranslation is None: #translation(s) are done; handle out-message out_translated.writeall() #write result of translation. #make translated record (if all is OK) translated_dict = inn_splitup.ta_info.copy() translated_dict.update(messagedict) translated_dict.update(out_translated.ta_info) run.translated.append(translated_dict) del out_translated break #break out of while loop elif isinstance(doalttranslation,dict): #some extended cases; a dict is returned that contains 'instructions' for some type of chained translations if 'type' not in doalttranslation or 'alt' not in doalttranslation: raise botslib.BotsError(_(u"Mappingscript returned '%(alt)s'. This dict should not have 'type' and 'alt'."),{'alt':doalttranslation}) if alt_from_previous_run == doalttranslation['alt']: number_of_loops_with_same_alt += 1 else: number_of_loops_with_same_alt = 0 if doalttranslation['type'] == u'out_as_inn': #do chained translation: use the out-object as inn-object, new out-object #use case: detected error in incoming file; use out-object to generate warning email handle_out_message(out_translated,ta_translated) inn_splitup = out_translated #out-object is now inn-object if isinstance(inn_splitup,outmessage.fixed): #for fixed: strip all values in node inn_splitup.root.stripnode() inn_splitup.ta_info['alt'] = doalttranslation['alt'] #get the alt-value for the next chained translation if not 'frompartner' in inn_splitup.ta_info: inn_splitup.ta_info['frompartner'] = '' if not 'topartner' in inn_splitup.ta_info: inn_splitup.ta_info['topartner'] = '' inn_splitup.ta_info.pop('statust') elif doalttranslation['type'] == u'no_check_on_infinite_loop': #do chained translation: allow many loops wit hsame alt-value. #mapping script will have to handle this correctly. number_of_loops_with_same_alt = 0 handle_out_message(out_translated,ta_translated) del out_translated inn_splitup.ta_info['alt'] = doalttranslation['alt'] #get the alt-value for the next chained translation else: #there is nothing else raise botslib.BotsError(_(u'Mappingscript returned dict with an unknown "type": "%(doalttranslation)s".'),{'doalttranslation':doalttranslation}) else: #note: this includes alt '' (empty string) if alt_from_previous_run == doalttranslation: number_of_loops_with_same_alt += 1 else: number_of_loops_with_same_alt = 0 #do normal chained translation: same inn-object, new out-object out_translated.writeall() del out_translated inn_splitup.ta_info['alt'] = doalttranslation #get the alt-value for the next chained translation if number_of_loops_with_same_alt > 10: raise botslib.BotsError(_(u'Mappingscript returns same alt value over and over again (infinite loop?). Alt: "%(doalttranslation)s".'),{'doalttranslation':doalttranslation}) #end of while-loop (trans********************************************************************************** #exceptions file_out-level: exception in mappingscript or writing of out-file except: #2 modes: either every error leads to skipping of whole infile (old mode) or errors in mappingscript/outfile only affect that branche txt = botslib.txtexc() print txt messagedict['error'] += txt.strip() else: pass #~ print 'succes' #exceptions file_in-level except botslib.GotoException: #edi-file is OK, file is passed-through after parsing. #~ edifile.handleconfirm(ta_fromfile,error=False) #~ botsglobal.logger.debug(_(u'Parse & passthrough for input file "%(filename)s".'),row) txt = botslib.txtexc() print txt except: txt = botslib.txtexc() messagedict['error'] += txt.strip() #~ edifile.handleconfirm(ta_fromfile,error=True) #~ botsglobal.logger.debug(u'Error in translating input file "%(filename)s":\n%(msg)s',{'filename':row['filename'],'msg':txt}) else: pass
def translate(startstatus=TRANSLATE, endstatus=TRANSLATED, idroute=""): """ translates edifiles in one or more edimessages. reads and parses edifiles that have to be translated. tries to split files into messages (using 'nextmessage' of grammar); if no splitting: edifile is one message. searches the right translation in translate-table; runs the mapping-script for the translation; Function takes db-ta with status=TRANSLATE->PARSED->SPLITUP->TRANSLATED """ # select edifiles to translate; fill ta-object # ~ import gc # ~ gc.disable() for row in botslib.query( u"""SELECT idta,frompartner,topartner,filename,messagetype,testindicator,editype,charset,alt,fromchannel FROM ta WHERE idta>%(rootidta)s AND status=%(status)s AND statust=%(statust)s AND idroute=%(idroute)s """, {"status": startstatus, "statust": OK, "idroute": idroute, "rootidta": botslib.get_minta4query()}, ): try: ta_fromfile = botslib.OldTransaction(row["idta"]) # TRANSLATE ta ta_parsedfile = ta_fromfile.copyta(status=PARSED) # copy TRANSLATE to PARSED ta # whole edi-file is read, parsed and made into a inmessage-object: edifile = inmessage.edifromfile( frompartner=row["frompartner"], topartner=row["topartner"], filename=row["filename"], messagetype=row["messagetype"], testindicator=row["testindicator"], editype=row["editype"], charset=row["charset"], alt=row["alt"], fromchannel=row["fromchannel"], idroute=idroute, ) botsglobal.logger.debug( u'start read and parse input file "%s" editype "%s" messagetype "%s".', row["filename"], row["editype"], row["messagetype"], ) for inn in edifile.nextmessage(): # for each message in the edifile: # inn.ta_info: parameters from inmessage.edifromfile(), syntax-information and parse-information ta_frommes = ta_parsedfile.copyta(status=SPLITUP) # copy PARSED to SPLITUP ta inn.ta_info[ "idta_fromfile" ] = ta_fromfile.idta # for confirmations in user script; used to give idta of 'confirming message' ta_frommes.update( **inn.ta_info ) # update ta-record SLIPTUP with info from message content and/or grammar while 1: # whileloop continues as long as there are alt-translations # ************select parameters for translation(script): for row2 in botslib.query( u"""SELECT tscript,tomessagetype,toeditype FROM translate WHERE frommessagetype = %(frommessagetype)s AND fromeditype = %(fromeditype)s AND active=%(booll)s AND alt=%(alt)s AND (frompartner_id IS NULL OR frompartner_id=%(frompartner)s OR frompartner_id in (SELECT to_partner_id FROM partnergroup WHERE from_partner_id=%(frompartner)s )) AND (topartner_id IS NULL OR topartner_id=%(topartner)s OR topartner_id in (SELECT to_partner_id FROM partnergroup WHERE from_partner_id=%(topartner)s )) ORDER BY alt DESC, CASE WHEN frompartner_id IS NULL THEN 1 ELSE 0 END, frompartner_id , CASE WHEN topartner_id IS NULL THEN 1 ELSE 0 END, topartner_id """, { "frommessagetype": inn.ta_info["messagetype"], "fromeditype": inn.ta_info["editype"], "alt": inn.ta_info["alt"], "frompartner": inn.ta_info["frompartner"], "topartner": inn.ta_info["topartner"], "booll": True, }, ): break # escape if found; we need only the first - ORDER BY in the query else: # no translation record is found raise botslib.TranslationNotFoundError( _( u'Editype "$editype", messagetype "$messagetype", frompartner "$frompartner", topartner "$topartner", alt "$alt"' ), editype=inn.ta_info["editype"], messagetype=inn.ta_info["messagetype"], frompartner=inn.ta_info["frompartner"], topartner=inn.ta_info["topartner"], alt=inn.ta_info["alt"], ) ta_tomes = ta_frommes.copyta(status=endstatus) # copy SPLITUP to TRANSLATED ta tofilename = str(ta_tomes.idta) tscript = row2["tscript"] tomessage = outmessage.outmessage_init( messagetype=row2["tomessagetype"], editype=row2["toeditype"], filename=tofilename, reference=unique("messagecounter"), statust=OK, divtext=tscript, ) # make outmessage object # copy ta_info botsglobal.logger.debug( u'script "%s" translates messagetype "%s" to messagetype "%s".', tscript, inn.ta_info["messagetype"], tomessage.ta_info["messagetype"], ) translationscript, scriptfilename = botslib.botsimport( "mappings", inn.ta_info["editype"] + "." + tscript ) # get the mapping-script doalttranslation = botslib.runscript( translationscript, scriptfilename, "main", inn=inn, out=tomessage ) botsglobal.logger.debug(u'script "%s" finished.', tscript) if "topartner" not in tomessage.ta_info: # tomessage does not contain values from ta...... tomessage.ta_info["topartner"] = inn.ta_info["topartner"] if ( tomessage.ta_info["statust"] == DONE ): # if indicated in user script the message should be discarded botsglobal.logger.debug(u"No output file because mapping script explicitly indicated this.") tomessage.ta_info["filename"] = "" tomessage.ta_info["status"] = DISCARD else: botsglobal.logger.debug( u'Start writing output file editype "%s" messagetype "%s".', tomessage.ta_info["editype"], tomessage.ta_info["messagetype"], ) tomessage.writeall() # write tomessage (result of translation). # problem is that not all values ta_tomes are know to to_message.... # ~ print 'tomessage.ta_info',tomessage.ta_info ta_tomes.update(**tomessage.ta_info) # update outmessage transaction with ta_info; del tomessage # ~ gc.collect() if not doalttranslation: break # out of while loop else: inn.ta_info["alt"] = doalttranslation # end of while-loop # ~ print inn.ta_info ta_frommes.update( statust=DONE, **inn.ta_info ) # update db. inn.ta_info could be changed by script. Is this useful? del inn # ~ gc.collect() # exceptions file_in-level except: # ~ edifile.handleconfirm(ta_fromfile,error=True) #only useful if errors are reported in acknowledgement (eg x12 997). Not used now. txt = botslib.txtexc() ta_parsedfile.failure() ta_parsedfile.update(statust=ERROR, errortext=txt) botsglobal.logger.debug(u'error in translating input file "%s":\n%s', row["filename"], txt) else: edifile.handleconfirm(ta_fromfile, error=False) ta_fromfile.update(statust=DONE) ta_parsedfile.update(statust=DONE, **edifile.confirminfo) botsglobal.logger.debug(u'translated input file "%s".', row["filename"]) del edifile
def translate(startstatus=TRANSLATE, endstatus=TRANSLATED, idroute=''): ''' translates edifiles in one or more edimessages. reads and parses edifiles that have to be translated. tries to split files into messages (using 'nextmessage' of grammar); if no splitting: edifile is one message. searches the right translation in translate-table; runs the mapping-script for the translation; Function takes db-ta with status=TRANSLATE->PARSED->SPLITUP->TRANSLATED ''' #select edifiles to translate; fill ta-object for row in botslib.query( u'''SELECT idta,frompartner,topartner,filename,messagetype,testindicator,editype,charset,alt,fromchannel FROM ta WHERE idta>%(rootidta)s AND status=%(status)s AND statust=%(statust)s AND idroute=%(idroute)s ''', { 'status': startstatus, 'statust': OK, 'idroute': idroute, 'rootidta': botslib.get_minta4query() }): try: ta_fromfile = botslib.OldTransaction(row['idta']) #TRANSLATE ta ta_parsedfile = ta_fromfile.copyta( status=PARSED) #copy TRANSLATE to PARSED ta #read whole edi-file: read, parse and made into a inmessage-object. Message is represented a a tree. edifile = inmessage.edifromfile(frompartner=row['frompartner'], topartner=row['topartner'], filename=row['filename'], messagetype=row['messagetype'], testindicator=row['testindicator'], editype=row['editype'], charset=row['charset'], alt=row['alt'], fromchannel=row['fromchannel'], idroute=idroute) botsglobal.logger.debug( u'start read and parse input file "%s" editype "%s" messagetype "%s".', row['filename'], row['editype'], row['messagetype']) for inn in edifile.nextmessage( ): #for each message in the edifile: #inn.ta_info: parameters from inmessage.edifromfile(), syntax-information and parse-information ta_frommes = ta_parsedfile.copyta( status=SPLITUP) #copy PARSED to SPLITUP ta inn.ta_info[ 'idta_fromfile'] = ta_fromfile.idta #for confirmations in user script; used to give idta of 'confirming message' ta_frommes.update( **inn.ta_info ) #update ta-record SLIPTUP with info from message content and/or grammar while 1: #whileloop continues as long as there are alt-translations #************select parameters for translation(script): for row2 in botslib.query( u'''SELECT tscript,tomessagetype,toeditype FROM translate WHERE frommessagetype = %(frommessagetype)s AND fromeditype = %(fromeditype)s AND active=%(booll)s AND alt=%(alt)s AND (frompartner_id IS NULL OR frompartner_id=%(frompartner)s OR frompartner_id in (SELECT to_partner_id FROM partnergroup WHERE from_partner_id=%(frompartner)s )) AND (topartner_id IS NULL OR topartner_id=%(topartner)s OR topartner_id in (SELECT to_partner_id FROM partnergroup WHERE from_partner_id=%(topartner)s )) ORDER BY alt DESC, CASE WHEN frompartner_id IS NULL THEN 1 ELSE 0 END, frompartner_id , CASE WHEN topartner_id IS NULL THEN 1 ELSE 0 END, topartner_id ''', { 'frommessagetype': inn.ta_info['messagetype'], 'fromeditype': inn.ta_info['editype'], 'alt': inn.ta_info['alt'], 'frompartner': inn.ta_info['frompartner'], 'topartner': inn.ta_info['topartner'], 'booll': True }): break #escape if found; we need only the first - ORDER BY in the query else: #no translation record is found raise botslib.TranslationNotFoundError( _(u'Editype "$editype", messagetype "$messagetype", frompartner "$frompartner", topartner "$topartner", alt "$alt"' ), editype=inn.ta_info['editype'], messagetype=inn.ta_info['messagetype'], frompartner=inn.ta_info['frompartner'], topartner=inn.ta_info['topartner'], alt=inn.ta_info['alt']) ta_tomes = ta_frommes.copyta( status=endstatus) #copy SPLITUP to TRANSLATED ta tofilename = str(ta_tomes.idta) tscript = row2['tscript'] tomessage = outmessage.outmessage_init( messagetype=row2['tomessagetype'], editype=row2['toeditype'], filename=tofilename, reference=unique('messagecounter'), statust=OK, divtext=tscript) #make outmessage object #copy ta_info botsglobal.logger.debug( u'script "%s" translates messagetype "%s" to messagetype "%s".', tscript, inn.ta_info['messagetype'], tomessage.ta_info['messagetype']) translationscript, scriptfilename = botslib.botsimport( 'mappings', inn.ta_info['editype'] + '.' + tscript) #get the mapping-script doalttranslation = botslib.runscript(translationscript, scriptfilename, 'main', inn=inn, out=tomessage) botsglobal.logger.debug(u'script "%s" finished.', tscript) if 'topartner' not in tomessage.ta_info: #tomessage does not contain values from ta...... tomessage.ta_info['topartner'] = inn.ta_info[ 'topartner'] if tomessage.ta_info[ 'statust'] == DONE: #if indicated in user script the message should be discarded botsglobal.logger.debug( u'No output file because mapping script explicitly indicated this.' ) tomessage.ta_info['filename'] = '' tomessage.ta_info['status'] = DISCARD else: botsglobal.logger.debug( u'Start writing output file editype "%s" messagetype "%s".', tomessage.ta_info['editype'], tomessage.ta_info['messagetype']) tomessage.writeall( ) #write tomessage (result of translation). #problem is that not all values ta_tomes are know to to_message.... #~ print 'tomessage.ta_info',tomessage.ta_info ta_tomes.update( **tomessage.ta_info ) #update outmessage transaction with ta_info; #check the value received from the mappingscript to see if another traanslation needs to be done (chained translation) if doalttranslation is None: del tomessage break #break out of while loop; do no other translation elif isinstance(doalttranslation, dict): #some extended cases; a dict is returned that contains 'instructions' if 'type' not in doalttranslation: raise botslib.BotsError( "Mapping script returned dict. This dict does not have a 'type', like in eg: {'type:'out_as_inn', 'alt':'alt-value'}." ) if doalttranslation['type'] == u'out_as_inn': if 'alt' not in doalttranslation: raise botslib.BotsError( "Mapping script returned dict, type 'out_as_inn'. This dict does not have a 'alt'-value, like in eg: {'type:'out_as_inn', 'alt':'alt-value'}." ) inn = tomessage if isinstance(inn, outmessage.fixed): inn.root.stripnode() inn.ta_info['alt'] = doalttranslation[ 'alt'] #get the alt-value for the next chainded translation inn.ta_info.pop('statust') else: del tomessage inn.ta_info[ 'alt'] = doalttranslation #get the alt-value for the next chainded translation #end of while-loop # #~ print inn.ta_info ta_frommes.update( statust=DONE, **inn.ta_info ) #update db. inn.ta_info could be changed by script. Is this useful? del inn #exceptions file_in-level except: #~ edifile.handleconfirm(ta_fromfile,error=True) #only useful if errors are reported in acknowledgement (eg x12 997). Not used now. txt = botslib.txtexc() ta_parsedfile.failure() ta_parsedfile.update(statust=ERROR, errortext=txt) botsglobal.logger.debug( u'error in translating input file "%s":\n%s', row['filename'], txt) else: edifile.handleconfirm(ta_fromfile, error=False) ta_fromfile.update(statust=DONE) ta_parsedfile.update(statust=DONE, **edifile.confirminfo) botsglobal.logger.debug(u'translated input file "%s".', row['filename']) del edifile
def start(): #exit codes: # 0: OK, no errors # 1: (system) errors # 2: bots ran OK, but there are errors/process errors in the run # 3: Database is locked, but "maxruntime" has not been exceeded. #********command line arguments************************** commandspossible = ['--new','--retry','--retransmit','--cleanup','--crashrecovery','--retrycommunication','--automaticretrycommunication'] commandstorun = [] routestorun = [] #list with routes to run configdir = 'config' for arg in sys.argv[1:]: if not arg: continue if arg.startswith('-c'): configdir = arg[2:] if not configdir: print 'Configuration directory indicated, but no directory name.' sys.exit(1) elif arg in commandspossible: commandstorun.append(arg) elif arg in ["?", "/?"] or arg.startswith('-'): showusage() sys.exit(0) else: #pick up names of routes to run routestorun.append(arg) if not commandstorun: #if no command on command line, use new (default) commandstorun = ['--new'] #**************init general: find locating of bots, configfiles, init paths etc.**************** botsinit.generalinit(configdir) #set current working directory to botspath #~ old_current_directory = os.getcwdu() os.chdir(botsglobal.ini.get('directories','botspath')) #**************initialise logging****************************** try: botsinit.initenginelogging() except: print _('Error in initialising logging system.') traceback.print_exc() sys.exit(1) else: atexit.register(logging.shutdown) for key,value in botslib.botsinfo(): #log start info botsglobal.logger.info(u'%s: "%s".',key,value) #**************connect to database********************************** try: botsinit.connect() except: botsglobal.logger.exception(_(u'Could not connect to database. Database settings are in bots/config/settings.py.')) sys.exit(1) else: botsglobal.logger.info(_(u'Connected to database.')) atexit.register(botsglobal.db.close) #initialise user exits for the whole bots-engine (this script file) try: userscript,scriptname = botslib.botsimport('routescripts','botsengine') except ImportError: userscript = scriptname = None #**************handle database lock**************************************** #try to set a lock on the database; if this is not possible, the database is already locked. Either: #1 another instance bots bots-engine is (still) running #2 or bots-engine had a severe crash. #What to do? #first: check ts of database lock. If below a certain value (set in bots.ini) we assume an other instance is running. Exit quietly - no errors, no logging. # else: Warn user, give advise on what to do. gather data: nr files in, errors. #next: warn with report & logging. advise a crashrecovery. if not botslib.set_database_lock(): if '--crashrecovery' in commandstorun: #user starts recovery operation; the databaselock is ignored; the databaselock is unlocked when routes have run. commandstorun = ['--crashrecovery'] #is an exclusive option! else: #when scheduling bots it is possible that the last run is still running. Check if maxruntime has passed: vanaf = datetime.datetime.today() - datetime.timedelta(minutes=botsglobal.ini.getint('settings','maxruntime',60)) for row in botslib.query('''SELECT ts FROM mutex WHERE ts < %(vanaf)s ''',{'vanaf':vanaf}): warn = _(u'!Bots database is locked!\nBots-engine has ended in an unexpected way during the last run.\nThis happens, but is very very rare.\nPossible causes: bots-engine terminated by user, system crash, power-down, etc.\nA forced retry of the last run is advised; bots will (try to) repair the last run.') botsglobal.logger.critical(warn) botslib.sendbotserrorreport(_(u'[Bots severe error]Database is locked'),warn) #add: count errors etc. sys.exit(1) else: #maxruntime has not passed. Exit silently, nothing reported botsglobal.logger.info(_(u'Database is locked, but "maxruntime" has not been exceeded.')) sys.exit(3) else: if '--crashrecovery' in commandstorun: #user starts recovery operation but there is no databaselock. warn = _(u'User started a forced retry of the last run.\nOnly use this when the database is locked.\nThe database was not locked (database is OK).\nSo Bots has done nothing now.') botsglobal.logger.error(warn) botslib.sendbotserrorreport(_(u'[Bots Error Report] User started a forced retry of last run, but this was not needed'),warn) botslib.remove_database_lock() sys.exit(1) #*************get list of routes to run**************************************** #~ raise Exception('locked database') #for testing database lock: abort, database will be locked if routestorun: botsglobal.logger.info(u'Run routes from command line: "%s".',str(routestorun)) else: # no routes from command line parameters: fetch all active routes from database for row in botslib.query('''SELECT DISTINCT idroute FROM routes WHERE active=%(active)s AND (notindefaultrun=%(notindefaultrun)s OR notindefaultrun IS NULL) ORDER BY idroute ''', {'active':True,'notindefaultrun':False}): routestorun.append(row['idroute']) botsglobal.logger.info(_(u'Run active routes from database: "%s".'),str(routestorun)) #routestorun is now either a list with routes from commandline, or the list of active routes for the routes table in the db. #**************run the routes for retry, retransmit and new runs************************************* try: #commandstorun determines the type(s) of run #routes to run is a listof the routes that are runs (for each command to run #botsglobal.incommunicate is used to control if there is communication in; only 'new' incommunicates. #botsglobal.minta4query controls which ta's are queried by the routes. #stuff2evaluate controls what is evaluated in automatic maintenance. errorinrun = 0 #detect if there has been some error. Only used for good exit() code botsglobal.incommunicate = False if '--crashrecovery' in commandstorun: botsglobal.logger.info(_(u'Run crash recovery.')) stuff2evaluate = botslib.set_minta4query_crashrecovery() if stuff2evaluate: router.routedispatcher(routestorun) errorinrun += automaticmaintenance.evaluate('--crashrecovery',stuff2evaluate) else: botsglobal.logger.info(_(u'No retry of the last run - there was no last run.')) if userscript and hasattr(userscript,'postcrashrecovery'): botslib.runscript(userscript,scriptname,'postcrashrecovery',routestorun=routestorun) if '--retrycommunication' in commandstorun: botsglobal.logger.info(_(u'Run communication retry.')) stuff2evaluate = router.routedispatcher(routestorun,'--retrycommunication') if stuff2evaluate: errorinrun += automaticmaintenance.evaluate('--retrycommunication',stuff2evaluate) else: botsglobal.logger.info(_(u'Run recommunicate: nothing to recommunicate.')) if userscript and hasattr(userscript,'postretrycommunication'): botslib.runscript(userscript,scriptname,'postretrycommunication',routestorun=routestorun) if '--automaticretrycommunication' in commandstorun: botsglobal.logger.info(_(u'Run automatic communication retry.')) stuff2evaluate = router.routedispatcher(routestorun,'--automaticretrycommunication') if stuff2evaluate: errorinrun += automaticmaintenance.evaluate('--automaticretrycommunication',stuff2evaluate) else: botsglobal.logger.info(_(u'Run automatic recommunicate: nothing to recommunicate.')) if userscript and hasattr(userscript,'postautomaticretrycommunication'): botslib.runscript(userscript,scriptname,'postautomaticretrycommunication',routestorun=routestorun) if '--retry' in commandstorun: botsglobal.logger.info(u'Run retry.') stuff2evaluate = router.routedispatcher(routestorun,'--retry') if stuff2evaluate: errorinrun += automaticmaintenance.evaluate('--retry',stuff2evaluate) else: botsglobal.logger.info(_(u'Run retry: nothing to retry.')) if userscript and hasattr(userscript,'postretry'): botslib.runscript(userscript,scriptname,'postretry',routestorun=routestorun) if '--retransmit' in commandstorun: botsglobal.logger.info(u'Run retransmit.') stuff2evaluate = router.routedispatcher(routestorun,'--retransmit') if stuff2evaluate: errorinrun += automaticmaintenance.evaluate('--retransmit',stuff2evaluate) else: botsglobal.logger.info(_(u'Run retransmit: nothing to retransmit.')) if userscript and hasattr(userscript,'postretransmit'): botslib.runscript(userscript,scriptname,'postretransmit',routestorun=routestorun) if '--new' in commandstorun: botsglobal.logger.info('Run new.') botsglobal.incommunicate = True botsglobal.minta4query = 0 #meaning: reset. the actual value is set later (in routedispatcher) stuff2evaluate = router.routedispatcher(routestorun) errorinrun += automaticmaintenance.evaluate('--new',stuff2evaluate) if userscript and hasattr(userscript,'postnewrun'): botslib.runscript(userscript,scriptname,'postnewrun',routestorun=routestorun) if '--cleanup' in commandstorun or botsglobal.ini.get('settings','whencleanup','always')=='always': botsglobal.logger.debug(u'Do cleanup.') cleanup.cleanup() botslib.remove_database_lock() except Exception,e: botsglobal.logger.exception(_(u'Severe error in bots system:\n%s')%(e)) #of course this 'should' not happen. sys.exit(1)
'Bots-engine has ended in an unexpected way during the last run.\n'\ 'Most likely causes: sudden power-down, system crash, problems with disk I/O, bots-engine terminated by user, etc.\n' 'Bots will do an automatic crash recovery now.') botsglobal.logger.critical(warn) botslib.sendbotserrorreport(_(u'[Bots severe error]Database is locked'),warn) commandstorun.insert(0,'crashrecovery') #there is a database lock. Add a crashrecovery as first command to run. atexit.register(botslib.remove_database_lock) #**************run the routes********************************************** #commandstorun determines the type(s) of run. eg: ['automaticretrycommunication','new'] #for each command: run all routes # for each route: run all seq try: #in acceptance tests: run a user script before running eg to clean output directories****************************** if acceptance_userscript: if hasattr(acceptance_userscript,'pretest'): botslib.runscript(acceptance_userscript,acceptance_scriptname,'pretest',routestorun=routestorun) botslib.prepare_confirmrules() errorinrun = 0 #detect if there has been some error. Only used for correct exit() code for command in commandstorun: botsglobal.logger.info(_(u'Run "%(command)s".'),{'command':command}) #get list of routes to run if routestorun: use_routestorun = routestorun[:] botsglobal.logger.info(_(u'Run routes from command line: "%(routes)s".'),{'routes':str(use_routestorun)}) elif command == 'new': #fetch all active routes from database unless 'not in default run' or not active. use_routestorun = [] for row in botslib.query('''SELECT DISTINCT idroute FROM routes WHERE active=%(active)s AND (notindefaultrun=%(notindefaultrun)s OR notindefaultrun IS NULL)
def _translate_one_file(row, routedict, endstatus, userscript, scriptname): ''' - read, lex, parse, make tree of nodes. - split up files into messages (using 'nextmessage' of grammar) - get mappingscript, start mappingscript. - write the results of translation (no enveloping yet) ''' try: ta_fromfile = botslib.OldTransaction(row['idta']) ta_parsed = ta_fromfile.copyta(status=PARSED) if row['filesize'] > botsglobal.ini.getint( 'settings', 'maxfilesizeincoming', 5000000): ta_parsed.update(filesize=row['filesize']) raise botslib.FileTooLargeError( _(u'File size of %(filesize)s is too big; option "maxfilesizeincoming" in bots.ini is %(maxfilesizeincoming)s.' ), { 'filesize': row['filesize'], 'maxfilesizeincoming': botsglobal.ini.getint('settings', 'maxfilesizeincoming', 5000000) }) botsglobal.logger.debug( _(u'Start translating file "%(filename)s" editype "%(editype)s" messagetype "%(messagetype)s".' ), row) #read whole edi-file: read, parse and made into a inmessage-object. Message is represented as a tree (inmessage.root is the root of the tree). edifile = inmessage.parse_edi_file(frompartner=row['frompartner'], topartner=row['topartner'], filename=row['filename'], messagetype=row['messagetype'], testindicator=row['testindicator'], editype=row['editype'], charset=row['charset'], alt=row['alt'], fromchannel=row['fromchannel'], idroute=routedict['idroute'], command=routedict['command']) edifile.checkforerrorlist( ) #no exception if infile has been lexed and parsed OK else raises an error if int( routedict['translateind'] ) == 3: #parse & passthrough; file is parsed, partners are known, no mapping, does confirm. raise botslib.GotoException('dummy') #edifile.ta_info contains info: QUERIES, charset etc for inn_splitup in edifile.nextmessage( ): #splitup messages in parsed edifile try: ta_splitup = ta_parsed.copyta( status=SPLITUP, **inn_splitup.ta_info) #copy PARSED to SPLITUP ta #inn_splitup.ta_info: parameters from inmessage.parse_edi_file(), syntax-information and parse-information inn_splitup.ta_info[ 'idta_fromfile'] = ta_fromfile.idta #for confirmations in userscript; used to give idta of 'confirming message' inn_splitup.ta_info[ 'idta'] = ta_splitup.idta #for confirmations in userscript; used to give idta of 'confirming message' number_of_loops_with_same_alt = 0 while 1: #continue as long as there are (alt-)translations #lookup the translation************************ tscript, toeditype, tomessagetype = botslib.lookup_translation( fromeditype=inn_splitup.ta_info['editype'], frommessagetype=inn_splitup.ta_info['messagetype'], frompartner=inn_splitup.ta_info['frompartner'], topartner=inn_splitup.ta_info['topartner'], alt=inn_splitup.ta_info['alt']) if not tscript: #no translation found in translate table; check if can find translation via user script if userscript and hasattr(userscript, 'gettranslation'): tscript, toeditype, tomessagetype = botslib.runscript( userscript, scriptname, 'gettranslation', idroute=routedict['idroute'], message=inn_splitup) if not tscript: raise botslib.TranslationNotFoundError( _(u'Translation not found for editype "%(editype)s", messagetype "%(messagetype)s", frompartner "%(frompartner)s", topartner "%(topartner)s", alt "%(alt)s".' ), inn_splitup.ta_info) inn_splitup.ta_info[ 'divtext'] = tscript #ifor reporting used mapping script to database (for display in GUI). #initialize new out-object************************* ta_translated = ta_splitup.copyta( status=endstatus ) #make ta for translated message (new out-ta) filename_translated = unicode(ta_translated.idta) out_translated = outmessage.outmessage_init( editype=toeditype, messagetype=tomessagetype, filename=filename_translated, reference=unique('messagecounter'), statust=OK, divtext=tscript) #make outmessage object #run mapping script************************ botsglobal.logger.debug( _(u'Mappingscript "%(tscript)s" translates messagetype "%(messagetype)s" to messagetype "%(tomessagetype)s".' ), { 'tscript': tscript, 'messagetype': inn_splitup.ta_info['messagetype'], 'tomessagetype': out_translated.ta_info['messagetype'] }) translationscript, scriptfilename = botslib.botsimport( 'mappings', inn_splitup.ta_info['editype'], tscript) #get the mappingscript alt_from_previous_run = inn_splitup.ta_info[ 'alt'] #needed to check for infinite loop doalttranslation = botslib.runscript(translationscript, scriptfilename, 'main', inn=inn_splitup, out=out_translated) botsglobal.logger.debug( _(u'Mappingscript "%(tscript)s" finished.'), {'tscript': tscript}) #manipulate for some attributes after mapping script if 'topartner' not in out_translated.ta_info: #out_translated does not contain values from ta...... out_translated.ta_info[ 'topartner'] = inn_splitup.ta_info['topartner'] if 'botskey' in inn_splitup.ta_info: inn_splitup.ta_info['reference'] = inn_splitup.ta_info[ 'botskey'] if 'botskey' in out_translated.ta_info: #out_translated does not contain values from ta...... out_translated.ta_info[ 'reference'] = out_translated.ta_info['botskey'] #check the value received from the mappingscript to determine what to do in this while-loop. Handling of chained trasnlations. if doalttranslation is None: #translation(s) are done; handle out-message handle_out_message(out_translated, ta_translated) break #break out of while loop elif isinstance(doalttranslation, dict): #some extended cases; a dict is returned that contains 'instructions' for some type of chained translations if 'type' not in doalttranslation or 'alt' not in doalttranslation: raise botslib.BotsError( _(u"Mappingscript returned '%(alt)s'. This dict should not have 'type' and 'alt'." ), {'alt': doalttranslation}) if alt_from_previous_run == doalttranslation['alt']: number_of_loops_with_same_alt += 1 else: number_of_loops_with_same_alt = 0 if doalttranslation['type'] == u'out_as_inn': #do chained translation: use the out-object as inn-object, new out-object #use case: detected error in incoming file; use out-object to generate warning email handle_out_message(out_translated, ta_translated) inn_splitup = out_translated #out-object is now inn-object if isinstance( inn_splitup, outmessage.fixed ): #for fixed: strip all values in node inn_splitup.root.stripnode() inn_splitup.ta_info['alt'] = doalttranslation[ 'alt'] #get the alt-value for the next chained translation if not 'frompartner' in inn_splitup.ta_info: inn_splitup.ta_info['frompartner'] = '' if not 'topartner' in inn_splitup.ta_info: inn_splitup.ta_info['topartner'] = '' inn_splitup.ta_info.pop('statust') elif doalttranslation[ 'type'] == u'no_check_on_infinite_loop': #do chained translation: allow many loops wit hsame alt-value. #mapping script will have to handle this correctly. number_of_loops_with_same_alt = 0 handle_out_message(out_translated, ta_translated) inn_splitup.ta_info['alt'] = doalttranslation[ 'alt'] #get the alt-value for the next chained translation else: #there is nothing else raise botslib.BotsError( _(u'Mappingscript returned dict with an unknown "type": "%(doalttranslation)s".' ), {'doalttranslation': doalttranslation}) else: #note: this includes alt '' (empty string) if alt_from_previous_run == doalttranslation: number_of_loops_with_same_alt += 1 else: number_of_loops_with_same_alt = 0 #do normal chained translation: same inn-object, new out-object handle_out_message(out_translated, ta_translated) inn_splitup.ta_info[ 'alt'] = doalttranslation #get the alt-value for the next chained translation if number_of_loops_with_same_alt > 10: raise botslib.BotsError( _(u'Mappingscript returns same alt value over and over again (infinite loop?). Alt: "%(doalttranslation)s".' ), {'doalttranslation': doalttranslation}) #end of while-loop (trans********************************************************************************** #exceptions file_out-level: exception in mappingscript or writing of out-file except: #2 modes: either every error leads to skipping of whole infile (old mode) or errors in mappingscript/outfile only affect that branche if botsglobal.ini.getboolean('settings', 'oldmessageerrors', False): raise txt = botslib.txtexc() ta_splitup.update( statust=ERROR, errortext=txt, **inn_splitup.ta_info ) #update db. inn_splitup.ta_info could be changed by mappingscript. Is this useful? ta_splitup.deletechildren() else: ta_splitup.update( statust=DONE, **inn_splitup.ta_info ) #update db. inn_splitup.ta_info could be changed by mappingscript. Is this useful? #exceptions file_in-level except botslib.GotoException: #edi-file is OK, file is passed-through after parsing. ta_parsed.update(statust=DONE, filesize=row['filesize'], **edifile.ta_info) #update with info from eg queries ta_parsed.copyta(status=MERGED, statust=OK) #original file goes straight to MERGED edifile.handleconfirm(ta_fromfile, error=False) botsglobal.logger.debug( _(u'Parse & passthrough for input file "%(filename)s".'), row) except botslib.FileTooLargeError as msg: ta_parsed.update(statust=ERROR, errortext=unicode(msg)) ta_parsed.deletechildren() botsglobal.logger.debug( u'Error in translating input file "%(filename)s":\n%(msg)s', { 'filename': row['filename'], 'msg': msg }) except: txt = botslib.txtexc() ta_parsed.update(statust=ERROR, errortext=txt, **edifile.ta_info) ta_parsed.deletechildren() edifile.handleconfirm(ta_fromfile, error=True) botsglobal.logger.debug( u'Error in translating input file "%(filename)s":\n%(msg)s', { 'filename': row['filename'], 'msg': txt }) else: edifile.handleconfirm(ta_fromfile, error=False) ta_parsed.update(statust=DONE, filesize=row['filesize'], **edifile.ta_info) botsglobal.logger.debug(_(u'Translated input file "%(filename)s".'), row) finally: ta_fromfile.update(statust=DONE)
def translate(startstatus=TRANSLATE,endstatus=TRANSLATED,idroute=''): ''' translates edifiles in one or more edimessages. reads and parses edifiles that have to be translated. tries to split files into messages (using 'nextmessage' of grammar); if no splitting: edifile is one message. searches the right translation in translate-table; runs the mapping-script for the translation; Function takes db-ta with status=TRANSLATE->PARSED->SPLITUP->TRANSLATED ''' #select edifiles to translate; fill ta-object for row in botslib.query(u'''SELECT idta,frompartner,topartner,filename,messagetype,testindicator,editype,charset,alt,fromchannel FROM ta WHERE idta>%(rootidta)s AND status=%(status)s AND statust=%(statust)s AND idroute=%(idroute)s ''', {'status':startstatus,'statust':OK,'idroute':idroute,'rootidta':botslib.get_minta4query()}): try: ta_fromfile = botslib.OldTransaction(row['idta']) #TRANSLATE ta ta_parsedfile = ta_fromfile.copyta(status=PARSED) #copy TRANSLATE to PARSED ta #read whole edi-file: read, parse and made into a inmessage-object. Message is represented a a tree. edifile = inmessage.edifromfile(frompartner=row['frompartner'], topartner=row['topartner'], filename=row['filename'], messagetype=row['messagetype'], testindicator=row['testindicator'], editype=row['editype'], charset=row['charset'], alt=row['alt'], fromchannel=row['fromchannel'], idroute=idroute) botsglobal.logger.debug(u'start read and parse input file "%s" editype "%s" messagetype "%s".',row['filename'],row['editype'],row['messagetype']) for inn in edifile.nextmessage(): #for each message in the edifile: #inn.ta_info: parameters from inmessage.edifromfile(), syntax-information and parse-information ta_frommes = ta_parsedfile.copyta(status=SPLITUP) #copy PARSED to SPLITUP ta inn.ta_info['idta_fromfile'] = ta_fromfile.idta #for confirmations in user script; used to give idta of 'confirming message' ta_frommes.update(**inn.ta_info) #update ta-record SLIPTUP with info from message content and/or grammar while 1: #whileloop continues as long as there are alt-translations #************select parameters for translation(script): for row2 in botslib.query(u'''SELECT tscript,tomessagetype,toeditype FROM translate WHERE frommessagetype = %(frommessagetype)s AND fromeditype = %(fromeditype)s AND active=%(booll)s AND alt=%(alt)s AND (frompartner_id IS NULL OR frompartner_id=%(frompartner)s OR frompartner_id in (SELECT to_partner_id FROM partnergroup WHERE from_partner_id=%(frompartner)s )) AND (topartner_id IS NULL OR topartner_id=%(topartner)s OR topartner_id in (SELECT to_partner_id FROM partnergroup WHERE from_partner_id=%(topartner)s )) ORDER BY alt DESC, CASE WHEN frompartner_id IS NULL THEN 1 ELSE 0 END, frompartner_id , CASE WHEN topartner_id IS NULL THEN 1 ELSE 0 END, topartner_id ''', {'frommessagetype':inn.ta_info['messagetype'], 'fromeditype':inn.ta_info['editype'], 'alt':inn.ta_info['alt'], 'frompartner':inn.ta_info['frompartner'], 'topartner':inn.ta_info['topartner'], 'booll':True}): break #escape if found; we need only the first - ORDER BY in the query else: #no translation record is found raise botslib.TranslationNotFoundError(_(u'Editype "$editype", messagetype "$messagetype", frompartner "$frompartner", topartner "$topartner", alt "$alt"'), editype=inn.ta_info['editype'], messagetype=inn.ta_info['messagetype'], frompartner=inn.ta_info['frompartner'], topartner=inn.ta_info['topartner'], alt=inn.ta_info['alt']) ta_tomes = ta_frommes.copyta(status=endstatus) #copy SPLITUP to TRANSLATED ta tofilename = str(ta_tomes.idta) tscript = row2['tscript'] tomessage = outmessage.outmessage_init(messagetype=row2['tomessagetype'],editype=row2['toeditype'],filename=tofilename,reference=unique('messagecounter'),statust=OK,divtext=tscript) #make outmessage object #copy ta_info botsglobal.logger.debug(u'script "%s" translates messagetype "%s" to messagetype "%s".',tscript,inn.ta_info['messagetype'],tomessage.ta_info['messagetype']) translationscript,scriptfilename = botslib.botsimport('mappings',inn.ta_info['editype'] + '.' + tscript) #get the mapping-script doalttranslation = botslib.runscript(translationscript,scriptfilename,'main',inn=inn,out=tomessage) botsglobal.logger.debug(u'script "%s" finished.',tscript) if 'topartner' not in tomessage.ta_info: #tomessage does not contain values from ta...... tomessage.ta_info['topartner'] = inn.ta_info['topartner'] if tomessage.ta_info['statust'] == DONE: #if indicated in user script the message should be discarded botsglobal.logger.debug(u'No output file because mapping script explicitly indicated this.') tomessage.ta_info['filename'] = '' tomessage.ta_info['status'] = DISCARD else: botsglobal.logger.debug(u'Start writing output file editype "%s" messagetype "%s".',tomessage.ta_info['editype'],tomessage.ta_info['messagetype']) tomessage.writeall() #write tomessage (result of translation). #problem is that not all values ta_tomes are know to to_message.... #~ print 'tomessage.ta_info',tomessage.ta_info ta_tomes.update(**tomessage.ta_info) #update outmessage transaction with ta_info; #check the value received from the mappingscript to see if another traanslation needs to be done (chained translation) if doalttranslation is None: del tomessage break #break out of while loop; do no other translation elif isinstance(doalttranslation,dict): #some extended cases; a dict is returned that contains 'instructions' if 'type' not in doalttranslation: raise botslib.BotsError("Mapping script returned dict. This dict does not have a 'type', like in eg: {'type:'out_as_inn', 'alt':'alt-value'}.") if doalttranslation['type'] == u'out_as_inn': if 'alt' not in doalttranslation: raise botslib.BotsError("Mapping script returned dict, type 'out_as_inn'. This dict does not have a 'alt'-value, like in eg: {'type:'out_as_inn', 'alt':'alt-value'}.") inn = tomessage if isinstance(inn,outmessage.fixed): inn.root.stripnode() inn.ta_info['alt'] = doalttranslation['alt'] #get the alt-value for the next chainded translation inn.ta_info.pop('statust') else: del tomessage inn.ta_info['alt'] = doalttranslation #get the alt-value for the next chainded translation #end of while-loop # #~ print inn.ta_info ta_frommes.update(statust=DONE,**inn.ta_info) #update db. inn.ta_info could be changed by script. Is this useful? del inn #exceptions file_in-level except: #~ edifile.handleconfirm(ta_fromfile,error=True) #only useful if errors are reported in acknowledgement (eg x12 997). Not used now. txt = botslib.txtexc() ta_parsedfile.failure() ta_parsedfile.update(statust=ERROR,errortext=txt) botsglobal.logger.debug(u'error in translating input file "%s":\n%s',row['filename'],txt) else: edifile.handleconfirm(ta_fromfile,error=False) ta_fromfile.update(statust=DONE) ta_parsedfile.update(statust=DONE,**edifile.confirminfo) botsglobal.logger.debug(u'translated input file "%s".',row['filename']) del edifile
def translate(run): for messagedict in run.incoming: try: #read whole edi-file: read, parse and made into a inmessage-object. Message is represented as a tree (inmessage.root is the root of the tree). edifile = inmessage.parse_edi_file( frompartner='', topartner='', filename=messagedict['filename'], messagetype=run.translation['messagetype'], testindicator='', editype=run.translation['editype'], charset='', alt='', fromchannel='', idroute='', command='') edifile.checkforerrorlist( ) #no exception if infile has been lexed and parsed OK else raises an error #~ if int(routedict['translateind']) == 3: #parse & passthrough; file is parsed, partners are known, no mapping, does confirm. #~ raise botslib.GotoException('dummy') #~ continue #file is parsed; no errors. no translation #edifile.ta_info contains info about incoming file: QUERIES, charset etc for inn_splitup in edifile.nextmessage( ): #splitup messages in parsed edifile try: #inn_splitup.ta_info: parameters from inmessage.parse_edi_file(), syntax-information and parse-information number_of_loops_with_same_alt = 0 while 1: #continue as long as there are (alt-)translations #lookup the translation************************ tscript, toeditype, tomessagetype = 'orders_edifact2xml', 'xml', 'orders' if 'tscript' in run.translation: tscript = run.translation['tscript'] toeditype = run.translation['toeditype'] tomessagetype = run.translation['tomessagetype'] else: tscript, toeditype, tomessagetype = botslib.lookup_translation( fromeditype=inn_splitup.ta_info['editype'], frommessagetype=inn_splitup. ta_info['messagetype'], frompartner=inn_splitup.ta_info['frompartner'], topartner=inn_splitup.ta_info['topartner'], alt=inn_splitup.ta_info['alt']) #run mapping script************************ filename_translated = transform.unique( 'bots_file_name') out_translated = outmessage.outmessage_init( editype=toeditype, messagetype=tomessagetype, filename=filename_translated, reference=transform.unique('messagecounter'), statust=OK, divtext=tscript) #make outmessage object #~ botsglobal.logger.debug(_(u'Mappingscript "%(tscript)s" translates messagetype "%(messagetype)s" to messagetype "%(tomessagetype)s".'), #~ {'tscript':tscript,'messagetype':inn_splitup.ta_info['messagetype'],'tomessagetype':out_translated.ta_info['messagetype']}) translationscript, scriptfilename = botslib.botsimport( 'mappings', inn_splitup.ta_info['editype'], tscript) #import mappingscript alt_from_previous_run = inn_splitup.ta_info[ 'alt'] #needed to check for infinite loop doalttranslation = botslib.runscript( translationscript, scriptfilename, 'main', inn=inn_splitup, out=out_translated) botsglobal.logger.debug( _(u'Mappingscript "%(tscript)s" finished.'), {'tscript': tscript}) #manipulate for some attributes after mapping script if 'topartner' not in out_translated.ta_info: #out_translated does not contain values from run...... out_translated.ta_info[ 'topartner'] = inn_splitup.ta_info['topartner'] if 'botskey' in inn_splitup.ta_info: inn_splitup.ta_info[ 'reference'] = inn_splitup.ta_info['botskey'] if 'botskey' in out_translated.ta_info: #out_translated does not contain values from run...... out_translated.ta_info[ 'reference'] = out_translated.ta_info[ 'botskey'] #check the value received from the mappingscript to determine what to do in this while-loop. Handling of chained trasnlations. if doalttranslation is None: #translation(s) are done; handle out-message out_translated.writeall( ) #write result of translation. #make translated record (if all is OK) translated_dict = inn_splitup.ta_info.copy() translated_dict.update(messagedict) translated_dict.update(out_translated.ta_info) run.translated.append(translated_dict) del out_translated break #break out of while loop elif isinstance(doalttranslation, dict): #some extended cases; a dict is returned that contains 'instructions' for some type of chained translations if 'type' not in doalttranslation or 'alt' not in doalttranslation: raise botslib.BotsError( _(u"Mappingscript returned '%(alt)s'. This dict should not have 'type' and 'alt'." ), {'alt': doalttranslation}) if alt_from_previous_run == doalttranslation[ 'alt']: number_of_loops_with_same_alt += 1 else: number_of_loops_with_same_alt = 0 if doalttranslation['type'] == u'out_as_inn': #do chained translation: use the out-object as inn-object, new out-object #use case: detected error in incoming file; use out-object to generate warning email handle_out_message(out_translated, ta_translated) inn_splitup = out_translated #out-object is now inn-object if isinstance( inn_splitup, outmessage.fixed ): #for fixed: strip all values in node inn_splitup.root.stripnode() inn_splitup.ta_info['alt'] = doalttranslation[ 'alt'] #get the alt-value for the next chained translation if not 'frompartner' in inn_splitup.ta_info: inn_splitup.ta_info['frompartner'] = '' if not 'topartner' in inn_splitup.ta_info: inn_splitup.ta_info['topartner'] = '' inn_splitup.ta_info.pop('statust') elif doalttranslation[ 'type'] == u'no_check_on_infinite_loop': #do chained translation: allow many loops wit hsame alt-value. #mapping script will have to handle this correctly. number_of_loops_with_same_alt = 0 handle_out_message(out_translated, ta_translated) del out_translated inn_splitup.ta_info['alt'] = doalttranslation[ 'alt'] #get the alt-value for the next chained translation else: #there is nothing else raise botslib.BotsError( _(u'Mappingscript returned dict with an unknown "type": "%(doalttranslation)s".' ), {'doalttranslation': doalttranslation}) else: #note: this includes alt '' (empty string) if alt_from_previous_run == doalttranslation: number_of_loops_with_same_alt += 1 else: number_of_loops_with_same_alt = 0 #do normal chained translation: same inn-object, new out-object out_translated.writeall() del out_translated inn_splitup.ta_info[ 'alt'] = doalttranslation #get the alt-value for the next chained translation if number_of_loops_with_same_alt > 10: raise botslib.BotsError( _(u'Mappingscript returns same alt value over and over again (infinite loop?). Alt: "%(doalttranslation)s".' ), {'doalttranslation': doalttranslation}) #end of while-loop (trans********************************************************************************** #exceptions file_out-level: exception in mappingscript or writing of out-file except: #2 modes: either every error leads to skipping of whole infile (old mode) or errors in mappingscript/outfile only affect that branche txt = botslib.txtexc() print txt messagedict['error'] += txt.strip() else: pass #~ print 'succes' #exceptions file_in-level except botslib.GotoException: #edi-file is OK, file is passed-through after parsing. #~ edifile.handleconfirm(ta_fromfile,error=False) #~ botsglobal.logger.debug(_(u'Parse & passthrough for input file "%(filename)s".'),row) txt = botslib.txtexc() print txt except: txt = botslib.txtexc() messagedict['error'] += txt.strip() #~ edifile.handleconfirm(ta_fromfile,error=True) #~ botsglobal.logger.debug(u'Error in translating input file "%(filename)s":\n%(msg)s',{'filename':row['filename'],'msg':txt}) else: pass