def envelope(ta_info,ta_list): ''' despatch function for class Envelope and subclasses. editype, edimessage and envelope essential for enveloping. determine the class for enveloping: 1. empty string: no enveloping (class noenvelope); file(s) is/are just copied. No user scripting for envelope. 2. if envelope is a class in this module, use it 3. if editype is a class in this module, use it 4. if user defined enveloping in usersys/envelope/<editype>/<envelope>.<envelope>, use it (user defined scripting overrides) Always check if user envelope script. user exits extends/replaces default enveloping. ''' #determine which class to use for enveloping userscript = scriptname = None if not ta_info['envelope']: classtocall = noenvelope else: try: #see if the is user scripted enveloping botsglobal.logger.debug(u'(try) to read user envelopescript editype "%s", envelope "%s".',ta_info['editype'],ta_info['envelope']) userscript,scriptname = botslib.botsimport('envelopescripts',ta_info['editype'] + '.' + ta_info['envelope']) except ImportError: #other errors, eg syntax errors are just passed pass try: classtocall = globals()[ta_info['envelope']] except KeyError: try: classtocall = globals()[ta_info['editype']] except KeyError: raise botslib.OutMessageError(_(u'Not found envelope "$envelope".'),envelope=ta_info['editype']) if userscript and hasattr(userscript,ta_info['envelope']): classtocall = getattr(userscript,ta_info['envelope']) env = classtocall(ta_info,ta_list,userscript,scriptname) env.run()
def read_index(filename): """ process index file in default location. """ try: importedbotsindex, scriptname = botslib.botsimport("index") pluglist = importedbotsindex.plugins[:] if importedbotsindex.__name__ in sys.modules: del sys.modules[importedbotsindex.__name__] except: txt = botslib.txtexc() raise botslib.PluginError( _(u"Error in configuration index file. Nothing is written. Error:\n%(txt)s"), {"txt": txt} ) else: botsglobal.logger.info(_(u"Configuration index file is OK.")) botsglobal.logger.info(_(u"Start writing to database.")) # write content of index file to the bots database try: read_index2database(pluglist) except: txt = botslib.txtexc() raise botslib.PluginError( _(u"Error writing configuration index to database. Nothing is written. Error:\n%(txt)s"), {"txt": txt} ) else: botsglobal.logger.info(_(u"Writing to database is OK."))
def safecodeconversion(modulename, value): """ converts code using a codelist. converted value is returned. codelist is first imported from file in codeconversions (lookup right place/mudule in bots.ini) """ module, filename = botslib.botsimport("codeconversions", modulename) try: return module.codeconversions[value] except KeyError: return value
def safercodeconversion(modulename, value): """ as codeconversion but reverses the dictionary first""" module, filename = botslib.botsimport("codeconversions", modulename) if not hasattr(module, "botsreversed" + "codeconversions"): reversedict = dict((value, key) for key, value in module.codeconversions.items()) setattr(module, "botsreversed" + "codeconversions", reversedict) try: return module.botsreversedcodeconversions[value] except KeyError: return value
def codec_search_function(encoding): try: module,filename = botslib.botsimport('charsets',encoding) except botslib.BotsImportError: #charsetscript not there; other errors like syntax errors are not catched return None else: if hasattr(module,'getregentry'): return module.getregentry() else: return None
def rcodeconversion(modulename,value): ''' as codeconversion but reverses the dictionary first''' module,filename = botslib.botsimport('codeconversions',modulename) if not hasattr(module,'botsreversed'+'codeconversions'): reversedict = dict((value,key) for key,value in module.codeconversions.items()) setattr(module,'botsreversed'+'codeconversions',reversedict) try: return module.botsreversedcodeconversions[value] except KeyError: raise botslib.CodeConversionError(_(u'Value "$value" not in file for reversed codeconversion "$filename".'),value=value,filename=filename)
def safercodeconversion(modulename,value): ''' as codeconversion but reverses the dictionary first''' module,filename = botslib.botsimport('codeconversions',modulename) if not hasattr(module,'botsreversed'+'codeconversions'): reversedict = dict((value,key) for key,value in module.codeconversions.items()) setattr(module,'botsreversed'+'codeconversions',reversedict) try: return module.botsreversedcodeconversions[value] except KeyError: return value
def codeconversion(modulename,value): ''' converts code using a codelist. converted value is returned. codelist is first imported from file in codeconversions (lookup right place/mudule in bots.ini) ''' module,filename = botslib.botsimport('codeconversions',modulename) try: return module.codeconversions[value] except KeyError: raise botslib.CodeConversionError(_(u'Value "$value" not in file for codeconversion "$filename".'),value=value,filename=filename)
def codec_search_function(encoding): try: module,filename = botslib.botsimport('charsets',encoding) except: return None else: if hasattr(module,'getregentry'): return module.getregentry() else: return None
def translate(startstatus,endstatus,routedict,rootidta): ''' query edifiles to be translated. status: FILEIN--PARSED-<SPLITUP--TRANSLATED ''' try: #see if there is a userscript that can determine the translation userscript,scriptname = botslib.botsimport('mappings','translation') except ImportError: #userscript is not there; other errors like syntax errors are not catched userscript = scriptname = None #select edifiles to translate for rawrow in botslib.query(u'''SELECT idta,frompartner,topartner,filename,messagetype,testindicator,editype,charset,alt,fromchannel,filesize FROM ta WHERE idta>%(rootidta)s AND status=%(status)s AND statust=%(statust)s AND idroute=%(idroute)s ''', {'status':startstatus,'statust':OK,'idroute':routedict['idroute'],'rootidta':rootidta}): row = dict(rawrow) #convert to real dictionary _translate_one_file(row,routedict,endstatus,userscript,scriptname)
def router(self,route): ''' for each route (as in self.routestorun). a route can have multiple parts (seq) ''' #is there a user routescript? try: self.userscript,self.scriptname = botslib.botsimport('routescripts',route) except botslib.BotsImportError: #routescript is not there; other errors like syntax errors are not catched self.userscript = self.scriptname = None self.minta4query_route = botslib._Transaction.processlist[-1] #the idta of route foundroute = False for row in botslib.query('''SELECT idroute , fromchannel_id as fromchannel, tochannel_id as tochannel, fromeditype, frommessagetype, alt, frompartner_id as frompartner, topartner_id as topartner, toeditype, tomessagetype, seq, frompartner_tochannel_id, topartner_tochannel_id, testindicator, translateind, defer, zip_incoming, zip_outgoing FROM routes WHERE idroute=%(idroute)s AND active=%(active)s ORDER BY seq''', {'idroute':route,'active':True}): routedict = dict(row) #convert to real dictionary (as self.command is added to routedict) routedict['command'] = self.command #this way command is passed to ohter functions. foundroute = True botsglobal.logger.info(_(u'Running route %(idroute)s %(seq)s'),routedict) self.routepart(routedict) #handle deferred-logic: mark if channel is deffered, umark if run self.keep_track_if_outchannel_deferred[routedict['tochannel']] = routedict['defer'] botsglobal.logger.debug(u'Finished route %(idroute)s %(seq)s',routedict) if not foundroute: botsglobal.logger.warning(_(u'There is no (active) route "%(route)s".'),{'route':route})
def read_index(filename): ''' process index file in default location. ''' try: importedbotsindex,scriptname = botslib.botsimport('index') pluglist = importedbotsindex.plugins[:] if 'botsindex' in sys.modules: del sys.modules['botsindex'] except: txt = botslib.txtexc() raise botslib.PluginError(_(u'Error in configuration index file. Nothing is written. Error:\n%(txt)s'),{'txt':txt}) else: botsglobal.logger.info(_(u'Configuration index file is OK.')) botsglobal.logger.info(_(u'Start writing to database.')) #write content of index file to the bots database try: read_index2database(pluglist) except: txt = botslib.txtexc() raise botslib.PluginError(_(u'Error writing configuration index to database. Nothing is written. Error:\n%(txt)s'),{'txt':txt}) else: botsglobal.logger.info(_(u'Writing to database is OK.'))
def envelope(ta_info, ta_list): """ dispatch function for class Envelope and subclasses. editype, edimessage and envelope essential for enveloping. determine the class for enveloping: 1. empty string: no enveloping (class noenvelope); file(s) is/are just copied. No user scripting for envelope. 2. if envelope is a class in this module, use it 3. if editype is a class in this module, use it 4. if user defined enveloping in usersys/envelope/<editype>/<envelope>.<envelope>, use it (user defined scripting overrides) Always check if user envelope script. user exits extends/replaces default enveloping. """ # determine which class to use for enveloping userscript = scriptname = None if not ta_info["envelope"]: # used when enveloping is just appending files. classtocall = noenvelope else: try: # see if the is user scripted enveloping userscript, scriptname = botslib.botsimport( "envelopescripts", ta_info["editype"] + "." + ta_info["envelope"] ) except ImportError: # other errors, eg syntax errors are just passed pass # first: check if there is a class with name ta_info['envelope'] in the user scripting # this allows complete enveloping in user scripting if userscript and hasattr(userscript, ta_info["envelope"]): classtocall = getattr(userscript, ta_info["envelope"]) else: try: # check if there is a envelope class with name ta_info['envelope'] in this file (envelope.py) classtocall = globals()[ta_info["envelope"]] except KeyError: try: # check if there is a envelope class with name ta_info['editype'] in this file (envelope.py). # 20110919: this should disappear in the long run....use this now for orders2printenvelope and myxmlenvelop # reason to disappear: confusing when setting up. classtocall = globals()[ta_info["editype"]] except KeyError: raise botslib.OutMessageError(_(u'Not found envelope "$envelope".'), envelope=ta_info["editype"]) env = classtocall(ta_info, ta_list, userscript, scriptname) env.run()
def envelope(ta_info,ta_list): ''' dispatch function for class Envelope and subclasses. editype, edimessage and envelope essential for enveloping. determine the class for enveloping: 1. empty string: no enveloping (class noenvelope); file(s) is/are just copied. No user scripting for envelope. 2. if user defined enveloping in usersys/envelope/<editype>/<envelope>.<envelope>, use it (user defined scripting overrides) user exits extends/replaces default enveloping. 3. if editype is a class in this module, use it. Complex is how enveloping and grammar/syntax work together for enveloping: 1. no envelope script: noenvelope syntax: - 2. user scripted script: envelopescripts.editype.envelope syntax: grammar.editype.envelope (alt could be envelopescripts.editype.envelope; but this is inline with incoming) grammar.editype.messagetype 3. class editype script: envelope.editype syntax: grammar.editype.envelope grammar.editype.messagetype ''' classtocall = userscript = scriptname = None if not ta_info['envelope']: #used when enveloping is just appending files. classtocall = noenvelope else: try: #check for user scripted enveloping userscript,scriptname = botslib.botsimport('envelopescripts',ta_info['editype'], ta_info['envelope']) #check if there is a user scripted class with name ta_info['envelope']. classtocall = getattr(userscript,ta_info['envelope'],None) except botslib.BotsImportError: #no user enveloping. pass if classtocall is None: try: classtocall = globals()[ta_info['editype']] except KeyError: raise botslib.OutMessageError(_(u'Not found envelope "%(envelope)s" for editype "%(editype)s".'),ta_info) env = classtocall(ta_info,ta_list,userscript,scriptname) env.run()
def start(): ''' sysexit codes: 0: OK, no errors 1: (system) errors incl parsing of command line arguments 2: bots ran OK, but there are errors/process errors in the run 3: Database is locked, but "maxruntime" has not been exceeded. ''' #NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start. #********command line arguments************************** usage = ''' This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net). Does the actual translations and communications; it's the workhorse. It does not have a fancy interface. Usage: %(name)s [config-option] Config-option: -c<directory> directory for configuration files (default: config). ''' % { 'name': os.path.basename(sys.argv[0]), 'version': botsglobal.version } configdir = 'config' for arg in sys.argv[1:]: if arg.startswith('-c'): configdir = arg[2:] if not configdir: print 'Error: configuration directory indicated, but no directory name.' sys.exit(1) elif arg in ["?", "/?", '-h', '--help'] or arg.startswith('-'): print usage sys.exit(0) #***********end handling command line arguments************************** botsinit.generalinit( configdir) #find locating of bots, configfiles, init paths etc. #set working directory to bots installation. advantage: when using relative paths it is clear that this point paths within bots installation. os.chdir(botsglobal.ini.get('directories', 'botspath')) #**************initialise logging****************************** process_name = 'engine2' botsglobal.logger = botsinit.initenginelogging(process_name) atexit.register(logging.shutdown) for key, value in botslib.botsinfo( ): #log info about environement, versions, etc botsglobal.logger.info(u'%(key)s: "%(value)s".', { 'key': key, 'value': value }) #**************connect to database********************************** try: botsinit.connect() except Exception as msg: botsglobal.logger.exception( _(u'Could not connect to database. Database settings are in bots/config/settings.py. Error: "%(msg)s".' ), {'msg': msg}) sys.exit(1) else: botsglobal.logger.info(_(u'Connected to database.')) atexit.register(botsglobal.db.close) warnings.simplefilter('error', UnicodeWarning) #import global scripts for bots-engine try: userscript, scriptname = botslib.botsimport('routescripts', 'botsengine') except botslib.BotsImportError: #userscript is not there; other errors like syntax errors are not catched userscript = scriptname = None #***acceptance tests: initialiase acceptance user script****************************** acceptance_userscript = acceptance_scriptname = None if botsglobal.ini.getboolean('acceptance', 'runacceptancetest', False): botsglobal.logger.info( _(u'This run is an acceptance test - as indicated in option "runacceptancetest" in bots.ini.' )) try: acceptance_userscript, acceptance_scriptname = botslib.botsimport( 'routescripts', 'bots_acceptancetest') except botslib.BotsImportError: botsglobal.logger.info( _(u'In acceptance test there is no script file "bots_acceptancetest.py" to check the results of the acceptance test.' )) try: #~ botslib.prepare_confirmrules() #in acceptance tests: run a user script before running eg to clean output directories****************************** botslib.tryrunscript(acceptance_userscript, acceptance_scriptname, 'pretest') botslib.tryrunscript(userscript, scriptname, 'pre') errorinrun = engine2_run() except Exception as msg: botsglobal.logger.exception( _(u'Severe error in bots system:\n%(msg)s'), {'msg': unicode(msg)}) #of course this 'should' not happen. sys.exit(1) else: if errorinrun: sys.exit(2) #indicate: error(s) in run(s) else: sys.exit(0) #OK
def translate(run): for messagedict in run.incoming: try: #read whole edi-file: read, parse and made into a inmessage-object. Message is represented as a tree (inmessage.root is the root of the tree). edifile = inmessage.parse_edi_file( frompartner='', topartner='', filename=messagedict['filename'], messagetype=run.translation['messagetype'], testindicator='', editype=run.translation['editype'], charset='', alt='', fromchannel='', idroute='', command='') edifile.checkforerrorlist( ) #no exception if infile has been lexed and parsed OK else raises an error #~ if int(routedict['translateind']) == 3: #parse & passthrough; file is parsed, partners are known, no mapping, does confirm. #~ raise botslib.GotoException('dummy') #~ continue #file is parsed; no errors. no translation #edifile.ta_info contains info about incoming file: QUERIES, charset etc for inn_splitup in edifile.nextmessage( ): #splitup messages in parsed edifile try: #inn_splitup.ta_info: parameters from inmessage.parse_edi_file(), syntax-information and parse-information number_of_loops_with_same_alt = 0 while 1: #continue as long as there are (alt-)translations #lookup the translation************************ tscript, toeditype, tomessagetype = 'orders_edifact2xml', 'xml', 'orders' if 'tscript' in run.translation: tscript = run.translation['tscript'] toeditype = run.translation['toeditype'] tomessagetype = run.translation['tomessagetype'] else: tscript, toeditype, tomessagetype = botslib.lookup_translation( fromeditype=inn_splitup.ta_info['editype'], frommessagetype=inn_splitup. ta_info['messagetype'], frompartner=inn_splitup.ta_info['frompartner'], topartner=inn_splitup.ta_info['topartner'], alt=inn_splitup.ta_info['alt']) #run mapping script************************ filename_translated = transform.unique( 'bots_file_name') out_translated = outmessage.outmessage_init( editype=toeditype, messagetype=tomessagetype, filename=filename_translated, reference=transform.unique('messagecounter'), statust=OK, divtext=tscript) #make outmessage object #~ botsglobal.logger.debug(_(u'Mappingscript "%(tscript)s" translates messagetype "%(messagetype)s" to messagetype "%(tomessagetype)s".'), #~ {'tscript':tscript,'messagetype':inn_splitup.ta_info['messagetype'],'tomessagetype':out_translated.ta_info['messagetype']}) translationscript, scriptfilename = botslib.botsimport( 'mappings', inn_splitup.ta_info['editype'], tscript) #import mappingscript alt_from_previous_run = inn_splitup.ta_info[ 'alt'] #needed to check for infinite loop doalttranslation = botslib.runscript( translationscript, scriptfilename, 'main', inn=inn_splitup, out=out_translated) botsglobal.logger.debug( _(u'Mappingscript "%(tscript)s" finished.'), {'tscript': tscript}) #manipulate for some attributes after mapping script if 'topartner' not in out_translated.ta_info: #out_translated does not contain values from run...... out_translated.ta_info[ 'topartner'] = inn_splitup.ta_info['topartner'] if 'botskey' in inn_splitup.ta_info: inn_splitup.ta_info[ 'reference'] = inn_splitup.ta_info['botskey'] if 'botskey' in out_translated.ta_info: #out_translated does not contain values from run...... out_translated.ta_info[ 'reference'] = out_translated.ta_info[ 'botskey'] #check the value received from the mappingscript to determine what to do in this while-loop. Handling of chained trasnlations. if doalttranslation is None: #translation(s) are done; handle out-message out_translated.writeall( ) #write result of translation. #make translated record (if all is OK) translated_dict = inn_splitup.ta_info.copy() translated_dict.update(messagedict) translated_dict.update(out_translated.ta_info) run.translated.append(translated_dict) del out_translated break #break out of while loop elif isinstance(doalttranslation, dict): #some extended cases; a dict is returned that contains 'instructions' for some type of chained translations if 'type' not in doalttranslation or 'alt' not in doalttranslation: raise botslib.BotsError( _(u"Mappingscript returned '%(alt)s'. This dict should not have 'type' and 'alt'." ), {'alt': doalttranslation}) if alt_from_previous_run == doalttranslation[ 'alt']: number_of_loops_with_same_alt += 1 else: number_of_loops_with_same_alt = 0 if doalttranslation['type'] == u'out_as_inn': #do chained translation: use the out-object as inn-object, new out-object #use case: detected error in incoming file; use out-object to generate warning email handle_out_message(out_translated, ta_translated) inn_splitup = out_translated #out-object is now inn-object if isinstance( inn_splitup, outmessage.fixed ): #for fixed: strip all values in node inn_splitup.root.stripnode() inn_splitup.ta_info['alt'] = doalttranslation[ 'alt'] #get the alt-value for the next chained translation if not 'frompartner' in inn_splitup.ta_info: inn_splitup.ta_info['frompartner'] = '' if not 'topartner' in inn_splitup.ta_info: inn_splitup.ta_info['topartner'] = '' inn_splitup.ta_info.pop('statust') elif doalttranslation[ 'type'] == u'no_check_on_infinite_loop': #do chained translation: allow many loops wit hsame alt-value. #mapping script will have to handle this correctly. number_of_loops_with_same_alt = 0 handle_out_message(out_translated, ta_translated) del out_translated inn_splitup.ta_info['alt'] = doalttranslation[ 'alt'] #get the alt-value for the next chained translation else: #there is nothing else raise botslib.BotsError( _(u'Mappingscript returned dict with an unknown "type": "%(doalttranslation)s".' ), {'doalttranslation': doalttranslation}) else: #note: this includes alt '' (empty string) if alt_from_previous_run == doalttranslation: number_of_loops_with_same_alt += 1 else: number_of_loops_with_same_alt = 0 #do normal chained translation: same inn-object, new out-object out_translated.writeall() del out_translated inn_splitup.ta_info[ 'alt'] = doalttranslation #get the alt-value for the next chained translation if number_of_loops_with_same_alt > 10: raise botslib.BotsError( _(u'Mappingscript returns same alt value over and over again (infinite loop?). Alt: "%(doalttranslation)s".' ), {'doalttranslation': doalttranslation}) #end of while-loop (trans********************************************************************************** #exceptions file_out-level: exception in mappingscript or writing of out-file except: #2 modes: either every error leads to skipping of whole infile (old mode) or errors in mappingscript/outfile only affect that branche txt = botslib.txtexc() print txt messagedict['error'] += txt.strip() else: pass #~ print 'succes' #exceptions file_in-level except botslib.GotoException: #edi-file is OK, file is passed-through after parsing. #~ edifile.handleconfirm(ta_fromfile,error=False) #~ botsglobal.logger.debug(_(u'Parse & passthrough for input file "%(filename)s".'),row) txt = botslib.txtexc() print txt except: txt = botslib.txtexc() messagedict['error'] += txt.strip() #~ edifile.handleconfirm(ta_fromfile,error=True) #~ botsglobal.logger.debug(u'Error in translating input file "%(filename)s":\n%(msg)s',{'filename':row['filename'],'msg':txt}) else: pass
def translate(startstatus=TRANSLATE,endstatus=TRANSLATED,idroute=''): ''' translates edifiles in one or more edimessages. reads and parses edifiles that have to be translated. tries to split files into messages (using 'nextmessage' of grammar); if no splitting: edifile is one message. searches the right translation in translate-table; runs the mapping-script for the translation; Function takes db-ta with status=TRANSLATE->PARSED->SPLITUP->TRANSLATED ''' #select edifiles to translate; fill ta-object for row in botslib.query(u'''SELECT idta,frompartner,topartner,filename,messagetype,testindicator,editype,charset,alt,fromchannel FROM ta WHERE idta>%(rootidta)s AND status=%(status)s AND statust=%(statust)s AND idroute=%(idroute)s ''', {'status':startstatus,'statust':OK,'idroute':idroute,'rootidta':botslib.get_minta4query()}): try: ta_fromfile = botslib.OldTransaction(row['idta']) #TRANSLATE ta ta_parsedfile = ta_fromfile.copyta(status=PARSED) #copy TRANSLATE to PARSED ta #read whole edi-file: read, parse and made into a inmessage-object. Message is represented a a tree. edifile = inmessage.edifromfile(frompartner=row['frompartner'], topartner=row['topartner'], filename=row['filename'], messagetype=row['messagetype'], testindicator=row['testindicator'], editype=row['editype'], charset=row['charset'], alt=row['alt'], fromchannel=row['fromchannel'], idroute=idroute) botsglobal.logger.debug(u'start read and parse input file "%s" editype "%s" messagetype "%s".',row['filename'],row['editype'],row['messagetype']) for inn in edifile.nextmessage(): #for each message in the edifile: #inn.ta_info: parameters from inmessage.edifromfile(), syntax-information and parse-information ta_frommes = ta_parsedfile.copyta(status=SPLITUP) #copy PARSED to SPLITUP ta inn.ta_info['idta_fromfile'] = ta_fromfile.idta #for confirmations in user script; used to give idta of 'confirming message' ta_frommes.update(**inn.ta_info) #update ta-record SLIPTUP with info from message content and/or grammar while 1: #whileloop continues as long as there are alt-translations #************select parameters for translation(script): for row2 in botslib.query(u'''SELECT tscript,tomessagetype,toeditype FROM translate WHERE frommessagetype = %(frommessagetype)s AND fromeditype = %(fromeditype)s AND active=%(booll)s AND alt=%(alt)s AND (frompartner_id IS NULL OR frompartner_id=%(frompartner)s OR frompartner_id in (SELECT to_partner_id FROM partnergroup WHERE from_partner_id=%(frompartner)s )) AND (topartner_id IS NULL OR topartner_id=%(topartner)s OR topartner_id in (SELECT to_partner_id FROM partnergroup WHERE from_partner_id=%(topartner)s )) ORDER BY alt DESC, CASE WHEN frompartner_id IS NULL THEN 1 ELSE 0 END, frompartner_id , CASE WHEN topartner_id IS NULL THEN 1 ELSE 0 END, topartner_id ''', {'frommessagetype':inn.ta_info['messagetype'], 'fromeditype':inn.ta_info['editype'], 'alt':inn.ta_info['alt'], 'frompartner':inn.ta_info['frompartner'], 'topartner':inn.ta_info['topartner'], 'booll':True}): break #escape if found; we need only the first - ORDER BY in the query else: #no translation record is found raise botslib.TranslationNotFoundError(_(u'Editype "$editype", messagetype "$messagetype", frompartner "$frompartner", topartner "$topartner", alt "$alt"'), editype=inn.ta_info['editype'], messagetype=inn.ta_info['messagetype'], frompartner=inn.ta_info['frompartner'], topartner=inn.ta_info['topartner'], alt=inn.ta_info['alt']) ta_tomes = ta_frommes.copyta(status=endstatus) #copy SPLITUP to TRANSLATED ta tofilename = str(ta_tomes.idta) tscript = row2['tscript'] tomessage = outmessage.outmessage_init(messagetype=row2['tomessagetype'],editype=row2['toeditype'],filename=tofilename,reference=unique('messagecounter'),statust=OK,divtext=tscript) #make outmessage object #copy ta_info botsglobal.logger.debug(u'script "%s" translates messagetype "%s" to messagetype "%s".',tscript,inn.ta_info['messagetype'],tomessage.ta_info['messagetype']) translationscript,scriptfilename = botslib.botsimport('mappings',inn.ta_info['editype'] + '.' + tscript) #get the mapping-script doalttranslation = botslib.runscript(translationscript,scriptfilename,'main',inn=inn,out=tomessage) botsglobal.logger.debug(u'script "%s" finished.',tscript) if 'topartner' not in tomessage.ta_info: #tomessage does not contain values from ta...... tomessage.ta_info['topartner'] = inn.ta_info['topartner'] if tomessage.ta_info['statust'] == DONE: #if indicated in user script the message should be discarded botsglobal.logger.debug(u'No output file because mapping script explicitly indicated this.') tomessage.ta_info['filename'] = '' tomessage.ta_info['status'] = DISCARD else: botsglobal.logger.debug(u'Start writing output file editype "%s" messagetype "%s".',tomessage.ta_info['editype'],tomessage.ta_info['messagetype']) tomessage.writeall() #write tomessage (result of translation). #problem is that not all values ta_tomes are know to to_message.... #~ print 'tomessage.ta_info',tomessage.ta_info ta_tomes.update(**tomessage.ta_info) #update outmessage transaction with ta_info; #check the value received from the mappingscript to see if another traanslation needs to be done (chained translation) if doalttranslation is None: del tomessage break #break out of while loop; do no other translation elif isinstance(doalttranslation,dict): #some extended cases; a dict is returned that contains 'instructions' if 'type' not in doalttranslation: raise botslib.BotsError("Mapping script returned dict. This dict does not have a 'type', like in eg: {'type:'out_as_inn', 'alt':'alt-value'}.") if doalttranslation['type'] == u'out_as_inn': if 'alt' not in doalttranslation: raise botslib.BotsError("Mapping script returned dict, type 'out_as_inn'. This dict does not have a 'alt'-value, like in eg: {'type:'out_as_inn', 'alt':'alt-value'}.") inn = tomessage if isinstance(inn,outmessage.fixed): inn.root.stripnode() inn.ta_info['alt'] = doalttranslation['alt'] #get the alt-value for the next chainded translation inn.ta_info.pop('statust') else: del tomessage inn.ta_info['alt'] = doalttranslation #get the alt-value for the next chainded translation #end of while-loop # #~ print inn.ta_info ta_frommes.update(statust=DONE,**inn.ta_info) #update db. inn.ta_info could be changed by script. Is this useful? del inn #exceptions file_in-level except: #~ edifile.handleconfirm(ta_fromfile,error=True) #only useful if errors are reported in acknowledgement (eg x12 997). Not used now. txt = botslib.txtexc() ta_parsedfile.failure() ta_parsedfile.update(statust=ERROR,errortext=txt) botsglobal.logger.debug(u'error in translating input file "%s":\n%s',row['filename'],txt) else: edifile.handleconfirm(ta_fromfile,error=False) ta_fromfile.update(statust=DONE) ta_parsedfile.update(statust=DONE,**edifile.confirminfo) botsglobal.logger.debug(u'translated input file "%s".',row['filename']) del edifile
def router(routedict): ''' communication.run one route. variants: - a route can be just script; - a route can do only incoming - a route can do only outgoing - a route can do both incoming and outgoing - at several points functions from a route script are called - if function is in route script ''' #is there a user route script? try: userscript,scriptname = botslib.botsimport('routescripts',routedict['idroute']) except ImportError: #other errors, eg syntax errors are just passed userscript = scriptname = None #if user route script has function 'main': communication.run 'main' (and do nothing else) if botslib.tryrunscript(userscript,scriptname,'main',routedict=routedict): return #so: if function ' main' : communication.run only the routescript, nothing else. if not (userscript or routedict['fromchannel'] or routedict['tochannel'] or routedict['translateind']): raise botslib.ScriptError(_(u'Route "$route" is empty: no script, not enough parameters.'),route=routedict['idroute']) botslib.tryrunscript(userscript,scriptname,'start',routedict=routedict) #communication.run incoming channel if routedict['fromchannel']: #do incoming part of route: in-communication; set ready for translation; translate botslib.tryrunscript(userscript,scriptname,'preincommunication',routedict=routedict) communication.run(idchannel=routedict['fromchannel'],idroute=routedict['idroute']) #communication.run incommunication #add attributes from route to the received files where={'status':FILEIN,'fromchannel':routedict['fromchannel'],'idroute':routedict['idroute']} change={'editype':routedict['fromeditype'],'messagetype':routedict['frommessagetype'],'frompartner':routedict['frompartner'],'topartner':routedict['topartner'],'alt':routedict['alt']} botslib.updateinfo(change=change,where=where) #all received files have status FILEIN botslib.tryrunscript(userscript,scriptname,'postincommunication',routedict=routedict) if routedict['fromeditype'] == 'mailbag': #mailbag for the route. preprocess.preprocess(routedict,preprocess.mailbag) #communication.run translation if routedict['translateind']: botslib.tryrunscript(userscript,scriptname,'pretranslation',routedict=routedict) botslib.addinfo(change={'status':TRANSLATE},where={'status':FILEIN,'idroute':routedict['idroute']}) transform.translate(idroute=routedict['idroute']) botslib.tryrunscript(userscript,scriptname,'posttranslation',routedict=routedict) #merge messages & communication.run outgoing channel if routedict['tochannel']: #do outgoing part of route botslib.tryrunscript(userscript,scriptname,'premerge',routedict=routedict) envelope.mergemessages(idroute=routedict['idroute']) botslib.tryrunscript(userscript,scriptname,'postmerge',routedict=routedict) #communication.run outgoing channel #build for query: towhere (dict) and wherestring towhere=dict(status=MERGED, idroute=routedict['idroute'], editype=routedict['toeditype'], messagetype=routedict['tomessagetype'], testindicator=routedict['testindicator']) towhere=dict([(key, value) for (key, value) in towhere.iteritems() if value]) #remove nul-values from dict wherestring = ' AND '.join([key+'=%('+key+')s' for key in towhere]) if routedict['frompartner_tochannel_id']: #use frompartner_tochannel in where-clause of query (partner/group dependent outchannel towhere['frompartner_tochannel_id']=routedict['frompartner_tochannel_id'] wherestring += ''' AND (frompartner=%(frompartner_tochannel_id)s OR frompartner in (SELECT from_partner_id FROM partnergroup WHERE to_partner_id =%(frompartner_tochannel_id)s ))''' if routedict['topartner_tochannel_id']: #use topartner_tochannel in where-clause of query (partner/group dependent outchannel towhere['topartner_tochannel_id']=routedict['topartner_tochannel_id'] wherestring += ''' AND (topartner=%(topartner_tochannel_id)s OR topartner in (SELECT from_partner_id FROM partnergroup WHERE to_partner_id=%(topartner_tochannel_id)s ))''' toset={'tochannel':routedict['tochannel'],'status':FILEOUT} botslib.addinfocore(change=toset,where=towhere,wherestring=wherestring) if not routedict['defer']: #do outgoing part of route botslib.tryrunscript(userscript,scriptname,'preoutcommunication',routedict=routedict) communication.run(idchannel=routedict['tochannel'],idroute=routedict['idroute']) #communication.run outcommunication botslib.tryrunscript(userscript,scriptname,'postoutcommunication',routedict=routedict) botslib.tryrunscript(userscript,scriptname,'end',routedict=routedict)
def translate(startstatus,endstatus,idroute,rootidta): ''' main translation loop. get edifiles to be translated, than: - read, lex, parse, make tree of nodes. - split up files into messages (using 'nextmessage' of grammar) - get mappingscript, start mappingscript. - write the results of translation (no enveloping yet) status: FILEIN--PARSED-<SPLITUP--TRANSLATED ''' try: #see if there is a userscript that can determine the translation userscript,scriptname = botslib.botsimport('mappings','translation') except ImportError: #userscript is not there; other errors like syntax errors are not catched userscript = scriptname = None #select edifiles to translate for rawrow in botslib.query(u'''SELECT idta,frompartner,topartner,filename,messagetype,testindicator,editype,charset,alt,fromchannel,filesize FROM ta WHERE idta>%(rootidta)s AND status=%(status)s AND statust=%(statust)s AND idroute=%(idroute)s ''', {'status':startstatus,'statust':OK,'idroute':idroute,'rootidta':rootidta}): try: row = dict(rawrow) #convert to real dictionary () ta_fromfile = botslib.OldTransaction(row['idta']) ta_parsed = ta_fromfile.copyta(status=PARSED) #make PARSED ta if row['filesize'] > botsglobal.ini.getint('settings','maxfilesizeincoming',5000000): ta_parsed.update(filesize=row['filesize']) raise botslib.InMessageError(_(u'File size of %(filesize)s is too big; option "maxfilesizeincoming" in bots.ini is %(maxfilesizeincoming)s.'), {'filesize':row['filesize'],'maxfilesizeincoming':botsglobal.ini.getint('settings','maxfilesizeincoming',5000000)}) botsglobal.logger.debug(_(u'Start translating file "%(filename)s" editype "%(editype)s" messagetype "%(messagetype)s".'),row) #read whole edi-file: read, parse and made into a inmessage-object. Message is represented as a tree (inmessage.root is the root of the tree). edifile = inmessage.parse_edi_file(frompartner=row['frompartner'], topartner=row['topartner'], filename=row['filename'], messagetype=row['messagetype'], testindicator=row['testindicator'], editype=row['editype'], charset=row['charset'], alt=row['alt'], fromchannel=row['fromchannel'], idroute=idroute) #if no exception: infile has been lexed and parsed OK. #edifile.ta_info contains info: QUERIES, charset etc for inn_splitup in edifile.nextmessage(): #splitup messages in parsed edifile try: ta_splitup = ta_parsed.copyta(status=SPLITUP,**inn_splitup.ta_info) #copy PARSED to SPLITUP ta #inn_splitup.ta_info: parameters from inmessage.parse_edi_file(), syntax-information and parse-information inn_splitup.ta_info['idta_fromfile'] = ta_fromfile.idta #for confirmations in userscript; used to give idta of 'confirming message' post_mapping_mode = False #if post_mapping: reuse out-object, if no translation is found no error number_of_loops_with_same_alt = 0 while 1: #continue as long as there are (alt-)translations #lookup the translation************************ tscript,toeditype,tomessagetype = botslib.lookup_translation(fromeditype=inn_splitup.ta_info['editype'], frommessagetype=inn_splitup.ta_info['messagetype'], frompartner=inn_splitup.ta_info['frompartner'], topartner=inn_splitup.ta_info['topartner'], alt=inn_splitup.ta_info['alt']) if not tscript: #no translation found in translate table; check if can find translation via user script if userscript and hasattr(userscript,'gettranslation'): tscript,toeditype,tomessagetype = botslib.runscript(userscript,scriptname,'gettranslation',idroute=idroute,message=inn_splitup) if not tscript: if post_mapping_mode: #in post-mapping mode, not finding the (partern specific!) script is no problem: translation is done #there is no post-mapping script found (in other words, default mapping script is enough). #all OK, handle the generated out-file and continue with next message botsglobal.logger.debug(_(u'Found no "post-mapping" translation for editype "%(editype)s", messagetype "%(messagetype)s", frompartner "%(frompartner)s", topartner "%(topartner)s", alt "%(alt)s".'), inn_splitup.ta_info) handle_out_message(out_translated,ta_translated) del out_translated break #break out of while loop else: raise botslib.TranslationNotFoundError(_(u'Translation not found for editype "%(editype)s", messagetype "%(messagetype)s", frompartner "%(frompartner)s", topartner "%(topartner)s", alt "%(alt)s".'), inn_splitup.ta_info) inn_splitup.ta_info['divtext'] = tscript #ifor reporting used mapping script to database (for display in GUI). if not post_mapping_mode: #initialize new out-object************************* ta_translated = ta_splitup.copyta(status=endstatus) #make ta for translated message (new out-ta) filename_translated = str(ta_translated.idta) out_translated = outmessage.outmessage_init(editype=toeditype,messagetype=tomessagetype,filename=filename_translated,reference=unique('messagecounter'),statust=OK,divtext=tscript) #make outmessage object #run mapping script************************ botsglobal.logger.debug(_(u'Mappingscript "%(tscript)s" translates messagetype "%(messagetype)s" to messagetype "%(tomessagetype)s".'), {'tscript':tscript,'messagetype':inn_splitup.ta_info['messagetype'],'tomessagetype':out_translated.ta_info['messagetype']}) translationscript,scriptfilename = botslib.botsimport('mappings',inn_splitup.ta_info['editype'],tscript) #get the mappingscript alt_from_previous_run = inn_splitup.ta_info['alt'] #needed to check for infinite loop doalttranslation = botslib.runscript(translationscript,scriptfilename,'main',inn=inn_splitup,out=out_translated) botsglobal.logger.debug(_(u'Mappingscript "%(tscript)s" finished.'),{'tscript':tscript}) #manipulate for some attributes after mapping script if 'topartner' not in out_translated.ta_info: #out_translated does not contain values from ta...... out_translated.ta_info['topartner'] = inn_splitup.ta_info['topartner'] if 'botskey' in inn_splitup.ta_info: inn_splitup.ta_info['reference'] = inn_splitup.ta_info['botskey'] if 'botskey' in out_translated.ta_info: #out_translated does not contain values from ta...... out_translated.ta_info['reference'] = out_translated.ta_info['botskey'] #check the value received from the mappingscript to determine what to do in this while-loop. Handling of chained trasnlations. if doalttranslation is None: #translation(s) are done; handle out-message handle_out_message(out_translated,ta_translated) del out_translated break #break out of while loop elif isinstance(doalttranslation,dict): #some extended cases; a dict is returned that contains 'instructions' for some type of chained translations if 'type' not in doalttranslation or 'alt' not in doalttranslation: raise botslib.BotsError(_(u"Mappingscript returned '%(alt)s'. This dict should not have 'type' and 'alt'."),{'alt':doalttranslation}) if alt_from_previous_run == doalttranslation['alt']: number_of_loops_with_same_alt += 1 else: number_of_loops_with_same_alt = 0 if doalttranslation['type'] == u'out_as_inn': #do chained translation: use the out-object as inn-object, new out-object #use case: detected error in incoming file; use out-object to generate warning email handle_out_message(out_translated,ta_translated) inn_splitup = out_translated #out-object is now inn-object if isinstance(inn_splitup,outmessage.fixed): #for fixed: strip all values in node inn_splitup.root.stripnode() inn_splitup.ta_info['alt'] = doalttranslation['alt'] #get the alt-value for the next chained translation if not 'frompartner' in inn_splitup.ta_info: inn_splitup.ta_info['frompartner'] = '' if not 'topartner' in inn_splitup.ta_info: inn_splitup.ta_info['topartner'] = '' inn_splitup.ta_info.pop('statust') elif doalttranslation['type'] == u'no_check_on_infinite_loop': #do chained translation: allow many loops wit hsame alt-value. #mapping script will have to handle this correctly. number_of_loops_with_same_alt = 0 handle_out_message(out_translated,ta_translated) del out_translated inn_splitup.ta_info['alt'] = doalttranslation['alt'] #get the alt-value for the next chained translation elif doalttranslation['type'] == u'post_mapping': #do chained (post)translation: same inn and out-objects. #in other words, the post-translation continue where is 'default' mapping ended. #use case: default mapping (for all partners), partner-specific post-translation #note: this is easier via import defaultmapping in a partner specific mapping. Please use that method! post_mapping_mode = True #translation is done; reset post_mapping_mode inn_splitup.ta_info['alt'] = doalttranslation['alt'] #get the alt-value for the next chained translation #~ inn_splitup.ta_info.pop('statust') else: #there is nothing else raise botslib.BotsError(_(u'Mappingscript returned dict with an unknown "type": "%(doalttranslation)s".'),{'doalttranslation':doalttranslation}) else: #note: this includes alt '' (empty string) if alt_from_previous_run == doalttranslation: number_of_loops_with_same_alt += 1 else: number_of_loops_with_same_alt = 0 #do normal chained translation: same inn-object, new out-object handle_out_message(out_translated,ta_translated) del out_translated inn_splitup.ta_info['alt'] = doalttranslation #get the alt-value for the next chained translation if number_of_loops_with_same_alt > 10: raise botslib.BotsError(_(u'Mappingscript returns same alt value over and over again (infinite loop?). Alt: "%(doalttranslation)s".'),{'doalttranslation':doalttranslation}) #end of while-loop (trans********************************************************************************** #exceptions file_out-level: exception in mappingscript or writing of out-file except: #2 modes: either every error leads to skipping of whole infile (old mode) or errors in mappingscript/outfile only affect that branche if botsglobal.ini.getboolean('settings','oldmessageerrors',False): raise txt = botslib.txtexc() ta_splitup.update(statust=ERROR,errortext=txt,**inn_splitup.ta_info) #update db. inn_splitup.ta_info could be changed by mappingscript. Is this useful? ta_splitup.deletechildren() else: ta_splitup.update(statust=DONE, **inn_splitup.ta_info) #update db. inn_splitup.ta_info could be changed by mappingscript. Is this useful? #exceptions file_in-level (file not OK according to grammar) except: txt = botslib.txtexc() ta_parsed.update(statust=ERROR,errortext=txt) ta_parsed.deletechildren() botsglobal.logger.debug(u'Error in translating input file "%(filename)s":\n%(msg)s',{'filename':row['filename'],'msg':txt}) else: edifile.handleconfirm(ta_fromfile,error=False) ta_parsed.update(statust=DONE,filesize=row['filesize'],**edifile.ta_info) botsglobal.logger.debug(_(u'Translated input file "%(filename)s".'),row) finally: ta_fromfile.update(statust=DONE)
def routepart(self,routedict): ''' communication.run one route part. variants: - a route can be a routescript - a route can do only incoming - a route can do only outgoing - a route can do both incoming and outgoing - at several points functions from a routescript are called - if function is in routescript ''' self.minta4query_routepart = botslib._Transaction.processlist[-1] #the idta of routepart #is there a user routescript? try: userscript,scriptname = botslib.botsimport('routescripts',routedict['idroute']) except ImportError: #routescript is not there; other errors like syntax errors are not catched userscript = scriptname = None #if routescript has function 'main': communication.run 'main' (and do nothing else) if botslib.tryrunscript(userscript,scriptname,'main',routedict=routedict): return #so: if function ' main' : communication.run only the routescript, nothing else. if not (userscript or routedict['fromchannel'] or routedict['tochannel'] or routedict['translateind']): raise botslib.ScriptError(_(u'Route "%(route)s" is empty: no routescript, not enough parameters.'),routedict) botslib.tryrunscript(userscript,scriptname,'start',routedict=routedict) #incoming part of route: #- incommunication #- assign attributes from route to incoming files #- preprocessing if routedict['fromchannel']: #only done for edi files from this route-part, this inchannel if routedict['command'] == 'rereceive': rootidta = self.get_minta4query() else: rootidta = self.get_minta4query_routepart() botslib.tryrunscript(userscript,scriptname,'preincommunication',routedict=routedict) communication.run(idchannel=routedict['fromchannel'],command=routedict['command'],idroute=routedict['idroute'],rootidta=rootidta) #communication.run incommunication #add attributes from route to the received files; where = {'statust':OK,'status':FILEIN,'fromchannel':routedict['fromchannel'],'idroute':routedict['idroute'],'rootidta':rootidta} change = {'editype':routedict['fromeditype'],'messagetype':routedict['frommessagetype'],'frompartner':routedict['frompartner'],'topartner':routedict['topartner'],'alt':routedict['alt']} nr_of_incoming_files_for_channel = botslib.updateinfo(change=change,where=where) botslib.tryrunscript(userscript,scriptname,'postincommunication',routedict=routedict) if nr_of_incoming_files_for_channel: #unzip incoming files (if indicated) if routedict['zip_incoming'] == 1: #unzip incoming (non-zipped gives error). preprocess.preprocess(routedict=routedict,function=preprocess.botsunzip,rootidta=self.get_minta4query_routepart(),pass_non_zip=False) elif routedict['zip_incoming'] == 2: #unzip incoming if zipped. preprocess.preprocess(routedict=routedict,function=preprocess.botsunzip,rootidta=self.get_minta4query_routepart(),pass_non_zip=True) #run mailbag-module. if botsglobal.ini.getboolean('settings','compatibility_mailbag',False): editypes_via_mailbag = ['mailbag'] else: editypes_via_mailbag = ['mailbag','edifact','x12','tradacoms'] if routedict['fromeditype'] in editypes_via_mailbag: #mailbag for the route. preprocess.preprocess(routedict=routedict,function=preprocess.mailbag,rootidta=self.get_minta4query_routepart()) #translate, merge, pass through: INFILE->MERGED if int(routedict['translateind']) == 1: #translate: for files in route botslib.tryrunscript(userscript,scriptname,'pretranslation',routedict=routedict) if routedict['command'] in ['rereceive',]: rootidta = self.get_minta4query() else: rootidta = self.get_minta4query_route() transform.translate(startstatus=FILEIN,endstatus=TRANSLATED,idroute=routedict['idroute'],rootidta=rootidta) botslib.tryrunscript(userscript,scriptname,'posttranslation',routedict=routedict) #**merge: for files in this route-part (the translated files) botslib.tryrunscript(userscript,scriptname,'premerge',routedict=routedict) envelope.mergemessages(startstatus=TRANSLATED,endstatus=MERGED,idroute=routedict['idroute'],rootidta=self.get_minta4query_routepart()) botslib.tryrunscript(userscript,scriptname,'postmerge',routedict=routedict) elif routedict['translateind'] == 2: #pass-through: pickup the incoming files and mark these as MERGED (==translation is finished) botslib.addinfo(change={'status':MERGED,'statust':OK},where={'status':FILEIN,'statust':OK,'idroute':routedict['idroute'],'rootidta':self.get_minta4query_route()}) #NOTE: routedict['translateind'] == 0 than nothing will happen with the files in this route. #ommunication outgoing channel: MERGED->RAWOUT if routedict['tochannel']: #**build query to add outchannel as attribute to outgoing files*** #filter files in route for outchannel towhere = { 'status':MERGED, 'statust':OK, 'idroute':routedict['idroute'], 'editype':routedict['toeditype'], 'messagetype':routedict['tomessagetype'], 'testindicator':routedict['testindicator'], } towhere = dict([(key, value) for key,value in towhere.iteritems() if value]) #remove nul-values from dict wherestring = ' AND '.join([key+'=%('+key+')s ' for key in towhere]) if routedict['frompartner_tochannel_id']: #use frompartner_tochannel in where-clause of query (partner/group dependent outchannel towhere['frompartner_tochannel_id'] = routedict['frompartner_tochannel_id'] wherestring += ''' AND (frompartner=%(frompartner_tochannel_id)s OR frompartner in (SELECT from_partner_id FROM partnergroup WHERE to_partner_id=%(frompartner_tochannel_id)s )) ''' if routedict['topartner_tochannel_id']: #use topartner_tochannel in where-clause of query (partner/group dependent outchannel towhere['topartner_tochannel_id'] = routedict['topartner_tochannel_id'] wherestring += ''' AND (topartner=%(topartner_tochannel_id)s OR topartner in (SELECT from_partner_id FROM partnergroup WHERE to_partner_id=%(topartner_tochannel_id)s )) ''' toset = {'status':FILEOUT,'statust':OK,'tochannel':routedict['tochannel']} towhere['rootidta'] = self.get_minta4query_route() nr_of_outgoing_files_for_channel = botslib.addinfocore(change=toset,where=towhere,wherestring=wherestring) if nr_of_outgoing_files_for_channel: #**set asked confirmation/acknowledgements botslib.set_asked_confirmrules(routedict,rootidta=self.get_minta4query_routepart()) #**zip outgoing #for files in this route-part for this out-channel if routedict['zip_outgoing'] == 1: preprocess.postprocess(routedict=routedict,function=preprocess.botszip,rootidta=self.get_minta4query_routepart()) #actual communication: run outgoing channel (if not deferred) #for all files in run that are for this channel (including the deferred ones from other routes) if not routedict['defer']: #determine range of idta to query: if channel was not deferred earlier in run: query only for route part else query for whole run if self.keep_track_if_outchannel_deferred.get(routedict['tochannel'],False) or routedict['command'] in ['resend','automaticretrycommunication']: rootidta = self.get_minta4query() else: rootidta = self.get_minta4query_routepart() if botslib.countoutfiles(idchannel=routedict['tochannel'],rootidta=rootidta): botslib.tryrunscript(userscript,scriptname,'preoutcommunication',routedict=routedict) communication.run(idchannel=routedict['tochannel'],command=routedict['command'],idroute=routedict['idroute'],rootidta=rootidta) #in communication several things can go wrong. #all outgoing files should have same status; that way all recomnnunication can be handled the same: #- status EXTERNOUT statust DONE (if communication goes OK) #- status EXTERNOUT status ERROR (if file is not communicatied) #to have the same status for all outgoing files some manipulation is needed, eg in case no connection could be made. botslib.addinfo(change={'status':EXTERNOUT,'statust':ERROR},where={'status':FILEOUT,'statust':OK,'tochannel':routedict['tochannel'],'rootidta':rootidta}) botslib.tryrunscript(userscript,scriptname,'postoutcommunication',routedict=routedict) botslib.tryrunscript(userscript,scriptname,'end',routedict=routedict)
def __init__(self,soortpythonfile,editype,grammarname): #***********import grammar-file. Grammar file has several grammar parts , eg 'structure'and 'recorddefs'. #***********every grammar part is in a module is either the grammar part itself or a import from another module. #***********every module is read once, using python import-machinery. #***********if a structure of recorddef has been read, Bots remembers this and skip most of the checks self.module,self.grammarname = botslib.botsimport(soortpythonfile,editype + '.' + grammarname)
def translate(startstatus=TRANSLATE,endstatus=TRANSLATED,idroute=''): ''' translates edifiles in one or more edimessages. reads and parses edifiles that have to be translated. tries to split files into messages (using 'nextmessage' of grammar); if no splitting: edifile is one message. seaches the right translation in translate-table; runs the mapping-script for the translation; Function takes db-ta with status=TRANSLATE->PARSED->SPLITUP->TRANSLATED ''' #select edifiles to translate; fill ta-object #~ import gc #~ gc.disable() for row in botslib.query(u'''SELECT idta,frompartner,topartner,filename,messagetype,testindicator,editype,charset,alt,fromchannel FROM ta WHERE idta>%(rootidta)s AND status=%(status)s AND statust=%(statust)s AND idroute=%(idroute)s ''', {'status':startstatus,'statust':OK,'idroute':idroute,'rootidta':botslib.get_minta4query()}): try: ta_fromfile=botslib.OldTransaction(row['idta']) #TRANSLATE ta ta_parsedfile = ta_fromfile.copyta(status=PARSED) #copy TRANSLATE to PARSED ta #whole edi-file is read, parsed and made into a inmessage-object: edifile = inmessage.edifromfile(frompartner=row['frompartner'], topartner=row['topartner'], filename=row['filename'], messagetype=row['messagetype'], testindicator=row['testindicator'], editype=row['editype'], charset=row['charset'], alt=row['alt'], fromchannel=row['fromchannel'], idroute=idroute) botsglobal.logger.debug(u'Start read and parse input file editype "%s" messagetype "%s".',row['editype'],row['messagetype']) for inn in edifile.nextmessage(): #for each message in the edifile: #inn.ta_info: parameters from inmessage.edifromfile(), syntax-information and parse-information ta_frommes=ta_parsedfile.copyta(status=SPLITUP) #copy PARSED to SPLITUP ta inn.ta_info['idta_fromfile'] = ta_fromfile.idta #for confirmations in user script; used to give idta of 'confirming message' ta_frommes.update(**inn.ta_info) #update ta-record SLIPTUP with info from message content and/or grammar while 1: #whileloop continues as long as there are alt-translations #************select parameters for translation(script): for row2 in botslib.query(u'''SELECT tscript,tomessagetype,toeditype FROM translate WHERE frommessagetype = %(frommessagetype)s AND fromeditype = %(fromeditype)s AND active=%(booll)s AND (alt='' OR alt=%(alt)s) AND (frompartner_id IS NULL OR frompartner_id=%(frompartner)s OR frompartner_id in (SELECT to_partner_id FROM partnergroup WHERE from_partner_id=%(frompartner)s )) AND (topartner_id IS NULL OR topartner_id=%(topartner)s OR topartner_id in (SELECT to_partner_id FROM partnergroup WHERE from_partner_id=%(topartner)s )) ORDER BY alt DESC, CASE WHEN frompartner_id IS NULL THEN 1 ELSE 0 END, frompartner_id , CASE WHEN topartner_id IS NULL THEN 1 ELSE 0 END, topartner_id ''', {'frommessagetype':inn.ta_info['messagetype'], 'fromeditype':inn.ta_info['editype'], 'alt':inn.ta_info['alt'], 'frompartner':inn.ta_info['frompartner'], 'topartner':inn.ta_info['topartner'], 'booll':True}): break #escape if found; we need only the first - ORDER BY in the query else: #no translation record is found raise botslib.TranslationNotFoundError(_(u'Editype "$editype", messagetype "$messagetype", frompartner "$frompartner", topartner "$topartner", alt "$alt"'), editype=inn.ta_info['editype'], messagetype=inn.ta_info['messagetype'], frompartner=inn.ta_info['frompartner'], topartner=inn.ta_info['topartner'], alt=inn.ta_info['alt']) ta_tomes=ta_frommes.copyta(status=endstatus) #copy SPLITUP to TRANSLATED ta tofilename = str(ta_tomes.idta) tscript=row2['tscript'] tomessage = outmessage.outmessage_init(messagetype=row2['tomessagetype'],editype=row2['toeditype'],filename=tofilename,reference=unique('messagecounter'),statust=OK,divtext=tscript) #make outmessage object #copy ta_info botsglobal.logger.debug(u'Script "%s" translates messagetype "%s" to messagetype "%s".',tscript,inn.ta_info['messagetype'],tomessage.ta_info['messagetype']) translationscript,scriptfilename = botslib.botsimport('mappings',inn.ta_info['editype'] + '.' + tscript) #get the mapping-script doalttranslation = botslib.runscript(translationscript,scriptfilename,'main',inn=inn,out=tomessage) botsglobal.logger.debug(u'Script "%s" finished.',tscript) if 'topartner' not in tomessage.ta_info: #tomessage does not contain values from ta...... tomessage.ta_info['topartner']=inn.ta_info['topartner'] if tomessage.ta_info['statust'] == DONE: #if indicated in user script the message should be discarded botsglobal.logger.debug(u'No output file because mapping script explicitly indicated this.') tomessage.ta_info['filename'] = '' tomessage.ta_info['status'] = DISCARD else: botsglobal.logger.debug(u'Start writing output file editype "%s" messagetype "%s".',tomessage.ta_info['editype'],tomessage.ta_info['messagetype']) tomessage.writeall() #write tomessage (result of translation). #problem is that not all values ta_tomes are know to to_message.... #~ print 'tomessage.ta_info',tomessage.ta_info ta_tomes.update(**tomessage.ta_info) #update outmessage transaction with ta_info; del tomessage #~ gc.collect() if not doalttranslation: break #out of while loop else: inn.ta_info['alt'] = doalttranslation #end of while-loop #~ print inn.ta_info ta_frommes.update(statust=DONE,**inn.ta_info) #update db. inn.ta_info could be changed by script. Is this useful? del inn #~ gc.collect() #exceptions file_in-level except: #~ edifile.close(ta_fromfile,error=True) #only useful if errors are reported in acknowledgement (eg x12 997). Not used now. txt=botslib.txtexc() ta_parsedfile.failure() ta_parsedfile.update(statust=ERROR,errortext=txt) botsglobal.logger.debug(u'Error reading and parsing input file: %s',txt) else: edifile.close(ta_fromfile,error=False) ta_fromfile.update(statust=DONE) ta_parsedfile.update(statust=DONE,**edifile.confirminfo) botsglobal.logger.debug(u'Succesfull read and parse of input file') del edifile
def translate(run): for messagedict in run.incoming: try: #read whole edi-file: read, parse and made into a inmessage-object. Message is represented as a tree (inmessage.root is the root of the tree). edifile = inmessage.parse_edi_file(frompartner='', topartner='', filename=messagedict['filename'], messagetype=run.translation['messagetype'], testindicator='', editype=run.translation['editype'], charset='', alt='', fromchannel='', idroute='', command='') edifile.checkforerrorlist() #no exception if infile has been lexed and parsed OK else raises an error #~ if int(routedict['translateind']) == 3: #parse & passthrough; file is parsed, partners are known, no mapping, does confirm. #~ raise botslib.GotoException('dummy') #~ continue #file is parsed; no errors. no translation #edifile.ta_info contains info about incoming file: QUERIES, charset etc for inn_splitup in edifile.nextmessage(): #splitup messages in parsed edifile try: #inn_splitup.ta_info: parameters from inmessage.parse_edi_file(), syntax-information and parse-information number_of_loops_with_same_alt = 0 while 1: #continue as long as there are (alt-)translations #lookup the translation************************ tscript,toeditype,tomessagetype = 'orders_edifact2xml' ,'xml','orders' if 'tscript' in run.translation: tscript = run.translation['tscript'] toeditype = run.translation['toeditype'] tomessagetype = run.translation['tomessagetype'] else: tscript,toeditype,tomessagetype = botslib.lookup_translation(fromeditype=inn_splitup.ta_info['editype'], frommessagetype=inn_splitup.ta_info['messagetype'], frompartner=inn_splitup.ta_info['frompartner'], topartner=inn_splitup.ta_info['topartner'], alt=inn_splitup.ta_info['alt']) #run mapping script************************ filename_translated = transform.unique('bots_file_name') out_translated = outmessage.outmessage_init(editype=toeditype, messagetype=tomessagetype, filename=filename_translated, reference=transform.unique('messagecounter'), statust=OK, divtext=tscript) #make outmessage object #~ botsglobal.logger.debug(_(u'Mappingscript "%(tscript)s" translates messagetype "%(messagetype)s" to messagetype "%(tomessagetype)s".'), #~ {'tscript':tscript,'messagetype':inn_splitup.ta_info['messagetype'],'tomessagetype':out_translated.ta_info['messagetype']}) translationscript,scriptfilename = botslib.botsimport('mappings',inn_splitup.ta_info['editype'],tscript) #import mappingscript alt_from_previous_run = inn_splitup.ta_info['alt'] #needed to check for infinite loop doalttranslation = botslib.runscript(translationscript,scriptfilename,'main',inn=inn_splitup,out=out_translated) botsglobal.logger.debug(_(u'Mappingscript "%(tscript)s" finished.'),{'tscript':tscript}) #manipulate for some attributes after mapping script if 'topartner' not in out_translated.ta_info: #out_translated does not contain values from run...... out_translated.ta_info['topartner'] = inn_splitup.ta_info['topartner'] if 'botskey' in inn_splitup.ta_info: inn_splitup.ta_info['reference'] = inn_splitup.ta_info['botskey'] if 'botskey' in out_translated.ta_info: #out_translated does not contain values from run...... out_translated.ta_info['reference'] = out_translated.ta_info['botskey'] #check the value received from the mappingscript to determine what to do in this while-loop. Handling of chained trasnlations. if doalttranslation is None: #translation(s) are done; handle out-message out_translated.writeall() #write result of translation. #make translated record (if all is OK) translated_dict = inn_splitup.ta_info.copy() translated_dict.update(messagedict) translated_dict.update(out_translated.ta_info) run.translated.append(translated_dict) del out_translated break #break out of while loop elif isinstance(doalttranslation,dict): #some extended cases; a dict is returned that contains 'instructions' for some type of chained translations if 'type' not in doalttranslation or 'alt' not in doalttranslation: raise botslib.BotsError(_(u"Mappingscript returned '%(alt)s'. This dict should not have 'type' and 'alt'."),{'alt':doalttranslation}) if alt_from_previous_run == doalttranslation['alt']: number_of_loops_with_same_alt += 1 else: number_of_loops_with_same_alt = 0 if doalttranslation['type'] == u'out_as_inn': #do chained translation: use the out-object as inn-object, new out-object #use case: detected error in incoming file; use out-object to generate warning email handle_out_message(out_translated,ta_translated) inn_splitup = out_translated #out-object is now inn-object if isinstance(inn_splitup,outmessage.fixed): #for fixed: strip all values in node inn_splitup.root.stripnode() inn_splitup.ta_info['alt'] = doalttranslation['alt'] #get the alt-value for the next chained translation if not 'frompartner' in inn_splitup.ta_info: inn_splitup.ta_info['frompartner'] = '' if not 'topartner' in inn_splitup.ta_info: inn_splitup.ta_info['topartner'] = '' inn_splitup.ta_info.pop('statust') elif doalttranslation['type'] == u'no_check_on_infinite_loop': #do chained translation: allow many loops wit hsame alt-value. #mapping script will have to handle this correctly. number_of_loops_with_same_alt = 0 handle_out_message(out_translated,ta_translated) del out_translated inn_splitup.ta_info['alt'] = doalttranslation['alt'] #get the alt-value for the next chained translation else: #there is nothing else raise botslib.BotsError(_(u'Mappingscript returned dict with an unknown "type": "%(doalttranslation)s".'),{'doalttranslation':doalttranslation}) else: #note: this includes alt '' (empty string) if alt_from_previous_run == doalttranslation: number_of_loops_with_same_alt += 1 else: number_of_loops_with_same_alt = 0 #do normal chained translation: same inn-object, new out-object out_translated.writeall() del out_translated inn_splitup.ta_info['alt'] = doalttranslation #get the alt-value for the next chained translation if number_of_loops_with_same_alt > 10: raise botslib.BotsError(_(u'Mappingscript returns same alt value over and over again (infinite loop?). Alt: "%(doalttranslation)s".'),{'doalttranslation':doalttranslation}) #end of while-loop (trans********************************************************************************** #exceptions file_out-level: exception in mappingscript or writing of out-file except: #2 modes: either every error leads to skipping of whole infile (old mode) or errors in mappingscript/outfile only affect that branche txt = botslib.txtexc() print txt messagedict['error'] += txt.strip() else: pass #~ print 'succes' #exceptions file_in-level except botslib.GotoException: #edi-file is OK, file is passed-through after parsing. #~ edifile.handleconfirm(ta_fromfile,error=False) #~ botsglobal.logger.debug(_(u'Parse & passthrough for input file "%(filename)s".'),row) txt = botslib.txtexc() print txt except: txt = botslib.txtexc() messagedict['error'] += txt.strip() #~ edifile.handleconfirm(ta_fromfile,error=True) #~ botsglobal.logger.debug(u'Error in translating input file "%(filename)s":\n%(msg)s',{'filename':row['filename'],'msg':txt}) else: pass
def start(): ''' sysexit codes: 0: OK, no errors 1: (system) errors incl parsing of command line arguments 2: bots ran OK, but there are errors/process errors in the run 3: Database is locked, but "maxruntime" has not been exceeded. ''' #NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start. #********command line arguments************************** usage = ''' This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net). Does the actual translations and communications; it's the workhorse. It does not have a fancy interface. Usage: %(name)s [config-option] Config-option: -c<directory> directory for configuration files (default: config). '''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version} configdir = 'config' for arg in sys.argv[1:]: if arg.startswith('-c'): configdir = arg[2:] if not configdir: print 'Error: configuration directory indicated, but no directory name.' sys.exit(1) elif arg in ["?", "/?",'-h', '--help'] or arg.startswith('-'): print usage sys.exit(0) #***********end handling command line arguments************************** botsinit.generalinit(configdir) #find locating of bots, configfiles, init paths etc. #set working directory to bots installation. advantage: when using relative paths it is clear that this point paths within bots installation. os.chdir(botsglobal.ini.get('directories','botspath')) #**************initialise logging****************************** process_name = 'engine2' botsglobal.logger = botsinit.initenginelogging(process_name) atexit.register(logging.shutdown) for key,value in botslib.botsinfo(): #log info about environement, versions, etc botsglobal.logger.info(u'%(key)s: "%(value)s".',{'key':key,'value':value}) #**************connect to database********************************** try: botsinit.connect() except Exception as msg: botsglobal.logger.exception(_(u'Could not connect to database. Database settings are in bots/config/settings.py. Error: "%(msg)s".'),{'msg':msg}) sys.exit(1) else: botsglobal.logger.info(_(u'Connected to database.')) atexit.register(botsglobal.db.close) warnings.simplefilter('error', UnicodeWarning) #import global scripts for bots-engine try: userscript,scriptname = botslib.botsimport('routescripts','botsengine') except botslib.BotsImportError: #userscript is not there; other errors like syntax errors are not catched userscript = scriptname = None #***acceptance tests: initialiase acceptance user script****************************** acceptance_userscript = acceptance_scriptname = None if botsglobal.ini.getboolean('acceptance','runacceptancetest',False): botsglobal.logger.info(_(u'This run is an acceptance test - as indicated in option "runacceptancetest" in bots.ini.')) try: acceptance_userscript,acceptance_scriptname = botslib.botsimport('routescripts','bots_acceptancetest') except botslib.BotsImportError: botsglobal.logger.info(_(u'In acceptance test there is no script file "bots_acceptancetest.py" to check the results of the acceptance test.')) try: #~ botslib.prepare_confirmrules() #in acceptance tests: run a user script before running eg to clean output directories****************************** botslib.tryrunscript(acceptance_userscript,acceptance_scriptname,'pretest') botslib.tryrunscript(userscript,scriptname,'pre') errorinrun = engine2_run() except Exception as msg: botsglobal.logger.exception(_(u'Severe error in bots system:\n%(msg)s'),{'msg':unicode(msg)}) #of course this 'should' not happen. sys.exit(1) else: if errorinrun: sys.exit(2) #indicate: error(s) in run(s) else: sys.exit(0) #OK
atexit.register(logging.shutdown) for key,value in botslib.botsinfo(): #log info about environement, versions, etc botsglobal.logger.info(u'%(key)s: "%(value)s".',{'key':key,'value':value}) #**************connect to database********************************** try: botsinit.connect() except Exception,msg: botsglobal.logger.exception(_(u'Could not connect to database. Database settings are in bots/config/settings.py. Error: "%(msg)s".'),{'msg':msg}) sys.exit(1) else: botsglobal.logger.info(_(u'Connected to database.')) atexit.register(botsglobal.db.close) #************initialise user exits for the whole bots-engine************************* try: userscript,scriptname = botslib.botsimport('routescripts','botsengine') except ImportError: #userscript is not there; other errors like syntax errors are not catched userscript = scriptname = None #***acceptance tests: initialiase acceptance user script****************************** acceptance_userscript = acceptance_scriptname = None if botsglobal.ini.getboolean('acceptance','runacceptancetest',False): botsglobal.logger.info(_(u'This run is an acceptance test - as indicated in option "runacceptancetest" in bots.ini.')) try: acceptance_userscript,acceptance_scriptname = botslib.botsimport('routescripts','bots_acceptancetest') except ImportError: botsglobal.logger.info(_(u'In acceptance test there is no script file "bots_acceptancetest.py" to check the results of the acceptance test.')) #**************handle database lock**************************************** #set a lock on the database; if not possible, the database is locked: an earlier instance of bots-engine was terminated unexpectedly. if not botslib.set_database_lock(): #for SQLite: do a integrity check on the database
def start(): #exit codes: # 0: OK, no errors # 1: (system) errors # 2: bots ran OK, but there are errors/process errors in the run # 3: Database is locked, but "maxruntime" has not been exceeded. #********command line arguments************************** commandspossible = ['--new','--retry','--retransmit','--cleanup','--crashrecovery','--retrycommunication','--automaticretrycommunication'] commandstorun = [] routestorun = [] #list with routes to run configdir = 'config' for arg in sys.argv[1:]: if not arg: continue if arg.startswith('-c'): configdir = arg[2:] if not configdir: print 'Configuration directory indicated, but no directory name.' sys.exit(1) elif arg in commandspossible: commandstorun.append(arg) elif arg in ["?", "/?"] or arg.startswith('-'): showusage() sys.exit(0) else: #pick up names of routes to run routestorun.append(arg) if not commandstorun: #if no command on command line, use new (default) commandstorun = ['--new'] #**************init general: find locating of bots, configfiles, init paths etc.**************** botsinit.generalinit(configdir) #set current working directory to botspath #~ old_current_directory = os.getcwdu() os.chdir(botsglobal.ini.get('directories','botspath')) #**************initialise logging****************************** try: botsinit.initenginelogging() except: print _('Error in initialising logging system.') traceback.print_exc() sys.exit(1) else: atexit.register(logging.shutdown) for key,value in botslib.botsinfo(): #log start info botsglobal.logger.info(u'%s: "%s".',key,value) #**************connect to database********************************** try: botsinit.connect() except: botsglobal.logger.exception(_(u'Could not connect to database. Database settings are in bots/config/settings.py.')) sys.exit(1) else: botsglobal.logger.info(_(u'Connected to database.')) atexit.register(botsglobal.db.close) #initialise user exits for the whole bots-engine (this script file) try: userscript,scriptname = botslib.botsimport('routescripts','botsengine') except ImportError: userscript = scriptname = None #**************handle database lock**************************************** #try to set a lock on the database; if this is not possible, the database is already locked. Either: #1 another instance bots bots-engine is (still) running #2 or bots-engine had a severe crash. #What to do? #first: check ts of database lock. If below a certain value (set in bots.ini) we assume an other instance is running. Exit quietly - no errors, no logging. # else: Warn user, give advise on what to do. gather data: nr files in, errors. #next: warn with report & logging. advise a crashrecovery. if not botslib.set_database_lock(): if '--crashrecovery' in commandstorun: #user starts recovery operation; the databaselock is ignored; the databaselock is unlocked when routes have run. commandstorun = ['--crashrecovery'] #is an exclusive option! else: #when scheduling bots it is possible that the last run is still running. Check if maxruntime has passed: vanaf = datetime.datetime.today() - datetime.timedelta(minutes=botsglobal.ini.getint('settings','maxruntime',60)) for row in botslib.query('''SELECT ts FROM mutex WHERE ts < %(vanaf)s ''',{'vanaf':vanaf}): warn = _(u'!Bots database is locked!\nBots-engine has ended in an unexpected way during the last run.\nThis happens, but is very very rare.\nPossible causes: bots-engine terminated by user, system crash, power-down, etc.\nA forced retry of the last run is advised; bots will (try to) repair the last run.') botsglobal.logger.critical(warn) botslib.sendbotserrorreport(_(u'[Bots severe error]Database is locked'),warn) #add: count errors etc. sys.exit(1) else: #maxruntime has not passed. Exit silently, nothing reported botsglobal.logger.info(_(u'Database is locked, but "maxruntime" has not been exceeded.')) sys.exit(3) else: if '--crashrecovery' in commandstorun: #user starts recovery operation but there is no databaselock. warn = _(u'User started a forced retry of the last run.\nOnly use this when the database is locked.\nThe database was not locked (database is OK).\nSo Bots has done nothing now.') botsglobal.logger.error(warn) botslib.sendbotserrorreport(_(u'[Bots Error Report] User started a forced retry of last run, but this was not needed'),warn) botslib.remove_database_lock() sys.exit(1) #*************get list of routes to run**************************************** #~ raise Exception('locked database') #for testing database lock: abort, database will be locked if routestorun: botsglobal.logger.info(u'Run routes from command line: "%s".',str(routestorun)) else: # no routes from command line parameters: fetch all active routes from database for row in botslib.query('''SELECT DISTINCT idroute FROM routes WHERE active=%(active)s AND (notindefaultrun=%(notindefaultrun)s OR notindefaultrun IS NULL) ORDER BY idroute ''', {'active':True,'notindefaultrun':False}): routestorun.append(row['idroute']) botsglobal.logger.info(_(u'Run active routes from database: "%s".'),str(routestorun)) #routestorun is now either a list with routes from commandline, or the list of active routes for the routes table in the db. #**************run the routes for retry, retransmit and new runs************************************* try: #commandstorun determines the type(s) of run #routes to run is a listof the routes that are runs (for each command to run #botsglobal.incommunicate is used to control if there is communication in; only 'new' incommunicates. #botsglobal.minta4query controls which ta's are queried by the routes. #stuff2evaluate controls what is evaluated in automatic maintenance. errorinrun = 0 #detect if there has been some error. Only used for good exit() code botsglobal.incommunicate = False if '--crashrecovery' in commandstorun: botsglobal.logger.info(_(u'Run crash recovery.')) stuff2evaluate = botslib.set_minta4query_crashrecovery() if stuff2evaluate: router.routedispatcher(routestorun) errorinrun += automaticmaintenance.evaluate('--crashrecovery',stuff2evaluate) else: botsglobal.logger.info(_(u'No retry of the last run - there was no last run.')) if userscript and hasattr(userscript,'postcrashrecovery'): botslib.runscript(userscript,scriptname,'postcrashrecovery',routestorun=routestorun) if '--retrycommunication' in commandstorun: botsglobal.logger.info(_(u'Run communication retry.')) stuff2evaluate = router.routedispatcher(routestorun,'--retrycommunication') if stuff2evaluate: errorinrun += automaticmaintenance.evaluate('--retrycommunication',stuff2evaluate) else: botsglobal.logger.info(_(u'Run recommunicate: nothing to recommunicate.')) if userscript and hasattr(userscript,'postretrycommunication'): botslib.runscript(userscript,scriptname,'postretrycommunication',routestorun=routestorun) if '--automaticretrycommunication' in commandstorun: botsglobal.logger.info(_(u'Run automatic communication retry.')) stuff2evaluate = router.routedispatcher(routestorun,'--automaticretrycommunication') if stuff2evaluate: errorinrun += automaticmaintenance.evaluate('--automaticretrycommunication',stuff2evaluate) else: botsglobal.logger.info(_(u'Run automatic recommunicate: nothing to recommunicate.')) if userscript and hasattr(userscript,'postautomaticretrycommunication'): botslib.runscript(userscript,scriptname,'postautomaticretrycommunication',routestorun=routestorun) if '--retry' in commandstorun: botsglobal.logger.info(u'Run retry.') stuff2evaluate = router.routedispatcher(routestorun,'--retry') if stuff2evaluate: errorinrun += automaticmaintenance.evaluate('--retry',stuff2evaluate) else: botsglobal.logger.info(_(u'Run retry: nothing to retry.')) if userscript and hasattr(userscript,'postretry'): botslib.runscript(userscript,scriptname,'postretry',routestorun=routestorun) if '--retransmit' in commandstorun: botsglobal.logger.info(u'Run retransmit.') stuff2evaluate = router.routedispatcher(routestorun,'--retransmit') if stuff2evaluate: errorinrun += automaticmaintenance.evaluate('--retransmit',stuff2evaluate) else: botsglobal.logger.info(_(u'Run retransmit: nothing to retransmit.')) if userscript and hasattr(userscript,'postretransmit'): botslib.runscript(userscript,scriptname,'postretransmit',routestorun=routestorun) if '--new' in commandstorun: botsglobal.logger.info('Run new.') botsglobal.incommunicate = True botsglobal.minta4query = 0 #meaning: reset. the actual value is set later (in routedispatcher) stuff2evaluate = router.routedispatcher(routestorun) errorinrun += automaticmaintenance.evaluate('--new',stuff2evaluate) if userscript and hasattr(userscript,'postnewrun'): botslib.runscript(userscript,scriptname,'postnewrun',routestorun=routestorun) if '--cleanup' in commandstorun or botsglobal.ini.get('settings','whencleanup','always')=='always': botsglobal.logger.debug(u'Do cleanup.') cleanup.cleanup() botslib.remove_database_lock() except Exception,e: botsglobal.logger.exception(_(u'Severe error in bots system:\n%s')%(e)) #of course this 'should' not happen. sys.exit(1)
def translate(startstatus=TRANSLATE, endstatus=TRANSLATED, idroute=""): """ translates edifiles in one or more edimessages. reads and parses edifiles that have to be translated. tries to split files into messages (using 'nextmessage' of grammar); if no splitting: edifile is one message. searches the right translation in translate-table; runs the mapping-script for the translation; Function takes db-ta with status=TRANSLATE->PARSED->SPLITUP->TRANSLATED """ # select edifiles to translate; fill ta-object # ~ import gc # ~ gc.disable() for row in botslib.query( u"""SELECT idta,frompartner,topartner,filename,messagetype,testindicator,editype,charset,alt,fromchannel FROM ta WHERE idta>%(rootidta)s AND status=%(status)s AND statust=%(statust)s AND idroute=%(idroute)s """, {"status": startstatus, "statust": OK, "idroute": idroute, "rootidta": botslib.get_minta4query()}, ): try: ta_fromfile = botslib.OldTransaction(row["idta"]) # TRANSLATE ta ta_parsedfile = ta_fromfile.copyta(status=PARSED) # copy TRANSLATE to PARSED ta # whole edi-file is read, parsed and made into a inmessage-object: edifile = inmessage.edifromfile( frompartner=row["frompartner"], topartner=row["topartner"], filename=row["filename"], messagetype=row["messagetype"], testindicator=row["testindicator"], editype=row["editype"], charset=row["charset"], alt=row["alt"], fromchannel=row["fromchannel"], idroute=idroute, ) botsglobal.logger.debug( u'start read and parse input file "%s" editype "%s" messagetype "%s".', row["filename"], row["editype"], row["messagetype"], ) for inn in edifile.nextmessage(): # for each message in the edifile: # inn.ta_info: parameters from inmessage.edifromfile(), syntax-information and parse-information ta_frommes = ta_parsedfile.copyta(status=SPLITUP) # copy PARSED to SPLITUP ta inn.ta_info[ "idta_fromfile" ] = ta_fromfile.idta # for confirmations in user script; used to give idta of 'confirming message' ta_frommes.update( **inn.ta_info ) # update ta-record SLIPTUP with info from message content and/or grammar while 1: # whileloop continues as long as there are alt-translations # ************select parameters for translation(script): for row2 in botslib.query( u"""SELECT tscript,tomessagetype,toeditype FROM translate WHERE frommessagetype = %(frommessagetype)s AND fromeditype = %(fromeditype)s AND active=%(booll)s AND alt=%(alt)s AND (frompartner_id IS NULL OR frompartner_id=%(frompartner)s OR frompartner_id in (SELECT to_partner_id FROM partnergroup WHERE from_partner_id=%(frompartner)s )) AND (topartner_id IS NULL OR topartner_id=%(topartner)s OR topartner_id in (SELECT to_partner_id FROM partnergroup WHERE from_partner_id=%(topartner)s )) ORDER BY alt DESC, CASE WHEN frompartner_id IS NULL THEN 1 ELSE 0 END, frompartner_id , CASE WHEN topartner_id IS NULL THEN 1 ELSE 0 END, topartner_id """, { "frommessagetype": inn.ta_info["messagetype"], "fromeditype": inn.ta_info["editype"], "alt": inn.ta_info["alt"], "frompartner": inn.ta_info["frompartner"], "topartner": inn.ta_info["topartner"], "booll": True, }, ): break # escape if found; we need only the first - ORDER BY in the query else: # no translation record is found raise botslib.TranslationNotFoundError( _( u'Editype "$editype", messagetype "$messagetype", frompartner "$frompartner", topartner "$topartner", alt "$alt"' ), editype=inn.ta_info["editype"], messagetype=inn.ta_info["messagetype"], frompartner=inn.ta_info["frompartner"], topartner=inn.ta_info["topartner"], alt=inn.ta_info["alt"], ) ta_tomes = ta_frommes.copyta(status=endstatus) # copy SPLITUP to TRANSLATED ta tofilename = str(ta_tomes.idta) tscript = row2["tscript"] tomessage = outmessage.outmessage_init( messagetype=row2["tomessagetype"], editype=row2["toeditype"], filename=tofilename, reference=unique("messagecounter"), statust=OK, divtext=tscript, ) # make outmessage object # copy ta_info botsglobal.logger.debug( u'script "%s" translates messagetype "%s" to messagetype "%s".', tscript, inn.ta_info["messagetype"], tomessage.ta_info["messagetype"], ) translationscript, scriptfilename = botslib.botsimport( "mappings", inn.ta_info["editype"] + "." + tscript ) # get the mapping-script doalttranslation = botslib.runscript( translationscript, scriptfilename, "main", inn=inn, out=tomessage ) botsglobal.logger.debug(u'script "%s" finished.', tscript) if "topartner" not in tomessage.ta_info: # tomessage does not contain values from ta...... tomessage.ta_info["topartner"] = inn.ta_info["topartner"] if ( tomessage.ta_info["statust"] == DONE ): # if indicated in user script the message should be discarded botsglobal.logger.debug(u"No output file because mapping script explicitly indicated this.") tomessage.ta_info["filename"] = "" tomessage.ta_info["status"] = DISCARD else: botsglobal.logger.debug( u'Start writing output file editype "%s" messagetype "%s".', tomessage.ta_info["editype"], tomessage.ta_info["messagetype"], ) tomessage.writeall() # write tomessage (result of translation). # problem is that not all values ta_tomes are know to to_message.... # ~ print 'tomessage.ta_info',tomessage.ta_info ta_tomes.update(**tomessage.ta_info) # update outmessage transaction with ta_info; del tomessage # ~ gc.collect() if not doalttranslation: break # out of while loop else: inn.ta_info["alt"] = doalttranslation # end of while-loop # ~ print inn.ta_info ta_frommes.update( statust=DONE, **inn.ta_info ) # update db. inn.ta_info could be changed by script. Is this useful? del inn # ~ gc.collect() # exceptions file_in-level except: # ~ edifile.handleconfirm(ta_fromfile,error=True) #only useful if errors are reported in acknowledgement (eg x12 997). Not used now. txt = botslib.txtexc() ta_parsedfile.failure() ta_parsedfile.update(statust=ERROR, errortext=txt) botsglobal.logger.debug(u'error in translating input file "%s":\n%s', row["filename"], txt) else: edifile.handleconfirm(ta_fromfile, error=False) ta_fromfile.update(statust=DONE) ta_parsedfile.update(statust=DONE, **edifile.confirminfo) botsglobal.logger.debug(u'translated input file "%s".', row["filename"]) del edifile
def _translate_one_file(row,routedict,endstatus,userscript,scriptname): ''' - read, lex, parse, make tree of nodes. - split up files into messages (using 'nextmessage' of grammar) - get mappingscript, start mappingscript. - write the results of translation (no enveloping yet) ''' try: ta_fromfile = botslib.OldTransaction(row['idta']) ta_parsed = ta_fromfile.copyta(status=PARSED) if row['filesize'] > botsglobal.ini.getint('settings','maxfilesizeincoming',5000000): ta_parsed.update(filesize=row['filesize']) raise botslib.FileTooLarge(_(u'File size of %(filesize)s is too big; option "maxfilesizeincoming" in bots.ini is %(maxfilesizeincoming)s.'), {'filesize':row['filesize'],'maxfilesizeincoming':botsglobal.ini.getint('settings','maxfilesizeincoming',5000000)}) botsglobal.logger.debug(_(u'Start translating file "%(filename)s" editype "%(editype)s" messagetype "%(messagetype)s".'),row) #read whole edi-file: read, parse and made into a inmessage-object. Message is represented as a tree (inmessage.root is the root of the tree). edifile = inmessage.parse_edi_file(frompartner=row['frompartner'], topartner=row['topartner'], filename=row['filename'], messagetype=row['messagetype'], testindicator=row['testindicator'], editype=row['editype'], charset=row['charset'], alt=row['alt'], fromchannel=row['fromchannel'], idroute=routedict['idroute'], command=routedict['command']) edifile.checkforerrorlist() #no exception if infile has been lexed and parsed OK else raises an error if int(routedict['translateind']) == 3: #parse & passthrough; file is parsed, partners are known, no mapping, does confirm. raise botslib.GotoException('dummy') #edifile.ta_info contains info: QUERIES, charset etc for inn_splitup in edifile.nextmessage(): #splitup messages in parsed edifile try: ta_splitup = ta_parsed.copyta(status=SPLITUP,**inn_splitup.ta_info) #copy PARSED to SPLITUP ta #inn_splitup.ta_info: parameters from inmessage.parse_edi_file(), syntax-information and parse-information inn_splitup.ta_info['idta_fromfile'] = ta_fromfile.idta #for confirmations in userscript; used to give idta of 'confirming message' inn_splitup.ta_info['idta'] = ta_splitup.idta #for confirmations in userscript; used to give idta of 'confirming message' number_of_loops_with_same_alt = 0 while 1: #continue as long as there are (alt-)translations #lookup the translation************************ tscript,toeditype,tomessagetype = botslib.lookup_translation(fromeditype=inn_splitup.ta_info['editype'], frommessagetype=inn_splitup.ta_info['messagetype'], frompartner=inn_splitup.ta_info['frompartner'], topartner=inn_splitup.ta_info['topartner'], alt=inn_splitup.ta_info['alt']) if not tscript: #no translation found in translate table; check if can find translation via user script if userscript and hasattr(userscript,'gettranslation'): tscript,toeditype,tomessagetype = botslib.runscript(userscript,scriptname,'gettranslation',idroute=routedict['idroute'],message=inn_splitup) if not tscript: raise botslib.TranslationNotFoundError(_(u'Translation not found for editype "%(editype)s", messagetype "%(messagetype)s", frompartner "%(frompartner)s", topartner "%(topartner)s", alt "%(alt)s".'), inn_splitup.ta_info) inn_splitup.ta_info['divtext'] = tscript #ifor reporting used mapping script to database (for display in GUI). #initialize new out-object************************* ta_translated = ta_splitup.copyta(status=endstatus) #make ta for translated message (new out-ta) filename_translated = str(ta_translated.idta) out_translated = outmessage.outmessage_init(editype=toeditype,messagetype=tomessagetype,filename=filename_translated,reference=unique('messagecounter'),statust=OK,divtext=tscript) #make outmessage object #run mapping script************************ botsglobal.logger.debug(_(u'Mappingscript "%(tscript)s" translates messagetype "%(messagetype)s" to messagetype "%(tomessagetype)s".'), {'tscript':tscript,'messagetype':inn_splitup.ta_info['messagetype'],'tomessagetype':out_translated.ta_info['messagetype']}) translationscript,scriptfilename = botslib.botsimport('mappings',inn_splitup.ta_info['editype'],tscript) #get the mappingscript alt_from_previous_run = inn_splitup.ta_info['alt'] #needed to check for infinite loop doalttranslation = botslib.runscript(translationscript,scriptfilename,'main',inn=inn_splitup,out=out_translated) botsglobal.logger.debug(_(u'Mappingscript "%(tscript)s" finished.'),{'tscript':tscript}) #manipulate for some attributes after mapping script if 'topartner' not in out_translated.ta_info: #out_translated does not contain values from ta...... out_translated.ta_info['topartner'] = inn_splitup.ta_info['topartner'] if 'botskey' in inn_splitup.ta_info: inn_splitup.ta_info['reference'] = inn_splitup.ta_info['botskey'] if 'botskey' in out_translated.ta_info: #out_translated does not contain values from ta...... out_translated.ta_info['reference'] = out_translated.ta_info['botskey'] #check the value received from the mappingscript to determine what to do in this while-loop. Handling of chained trasnlations. if doalttranslation is None: #translation(s) are done; handle out-message handle_out_message(out_translated,ta_translated) break #break out of while loop elif isinstance(doalttranslation,dict): #some extended cases; a dict is returned that contains 'instructions' for some type of chained translations if 'type' not in doalttranslation or 'alt' not in doalttranslation: raise botslib.BotsError(_(u"Mappingscript returned '%(alt)s'. This dict should not have 'type' and 'alt'."),{'alt':doalttranslation}) if alt_from_previous_run == doalttranslation['alt']: number_of_loops_with_same_alt += 1 else: number_of_loops_with_same_alt = 0 if doalttranslation['type'] == u'out_as_inn': #do chained translation: use the out-object as inn-object, new out-object #use case: detected error in incoming file; use out-object to generate warning email handle_out_message(out_translated,ta_translated) inn_splitup = out_translated #out-object is now inn-object if isinstance(inn_splitup,outmessage.fixed): #for fixed: strip all values in node inn_splitup.root.stripnode() inn_splitup.ta_info['alt'] = doalttranslation['alt'] #get the alt-value for the next chained translation if not 'frompartner' in inn_splitup.ta_info: inn_splitup.ta_info['frompartner'] = '' if not 'topartner' in inn_splitup.ta_info: inn_splitup.ta_info['topartner'] = '' inn_splitup.ta_info.pop('statust') elif doalttranslation['type'] == u'no_check_on_infinite_loop': #do chained translation: allow many loops wit hsame alt-value. #mapping script will have to handle this correctly. number_of_loops_with_same_alt = 0 handle_out_message(out_translated,ta_translated) inn_splitup.ta_info['alt'] = doalttranslation['alt'] #get the alt-value for the next chained translation else: #there is nothing else raise botslib.BotsError(_(u'Mappingscript returned dict with an unknown "type": "%(doalttranslation)s".'),{'doalttranslation':doalttranslation}) else: #note: this includes alt '' (empty string) if alt_from_previous_run == doalttranslation: number_of_loops_with_same_alt += 1 else: number_of_loops_with_same_alt = 0 #do normal chained translation: same inn-object, new out-object handle_out_message(out_translated,ta_translated) inn_splitup.ta_info['alt'] = doalttranslation #get the alt-value for the next chained translation if number_of_loops_with_same_alt > 10: raise botslib.BotsError(_(u'Mappingscript returns same alt value over and over again (infinite loop?). Alt: "%(doalttranslation)s".'),{'doalttranslation':doalttranslation}) #end of while-loop (trans********************************************************************************** #exceptions file_out-level: exception in mappingscript or writing of out-file except: #2 modes: either every error leads to skipping of whole infile (old mode) or errors in mappingscript/outfile only affect that branche if botsglobal.ini.getboolean('settings','oldmessageerrors',False): raise txt = botslib.txtexc() ta_splitup.update(statust=ERROR,errortext=txt,**inn_splitup.ta_info) #update db. inn_splitup.ta_info could be changed by mappingscript. Is this useful? ta_splitup.deletechildren() else: ta_splitup.update(statust=DONE, **inn_splitup.ta_info) #update db. inn_splitup.ta_info could be changed by mappingscript. Is this useful? #exceptions file_in-level except botslib.GotoException: #edi-file is OK, file is passed-through after parsing. ta_parsed.update(statust=DONE,filesize=row['filesize'],**edifile.ta_info) #update with info from eg queries ta_parsed.copyta(status=MERGED,statust=OK) #original file goes straight to MERGED edifile.handleconfirm(ta_fromfile,error=False) botsglobal.logger.debug(_(u'Parse & passthrough for input file "%(filename)s".'),row) except botslib.FileTooLarge as msg: ta_parsed.update(statust=ERROR,errortext=str(msg)) ta_parsed.deletechildren() botsglobal.logger.debug(u'Error in translating input file "%(filename)s":\n%(msg)s',{'filename':row['filename'],'msg':msg}) except: txt = botslib.txtexc() ta_parsed.update(statust=ERROR,errortext=txt,**edifile.ta_info) ta_parsed.deletechildren() edifile.handleconfirm(ta_fromfile,error=True) botsglobal.logger.debug(u'Error in translating input file "%(filename)s":\n%(msg)s',{'filename':row['filename'],'msg':txt}) else: edifile.handleconfirm(ta_fromfile,error=False) ta_parsed.update(statust=DONE,filesize=row['filesize'],**edifile.ta_info) botsglobal.logger.debug(_(u'Translated input file "%(filename)s".'),row) finally: ta_fromfile.update(statust=DONE)
def start(): ''' sysexit codes: 0: OK, no errors 1: (system) errors incl parsing of command line arguments 2: bots ran OK, but there are errors/process errors in the run 3: Database is locked, but "maxruntime" has not been exceeded. ''' #NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start. #********command line arguments************************** usage = ''' This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net). Does the actual translations and communications; it's the workhorse. It does not have a fancy interface. Usage: %(name)s [run-options] [config-option] [routes] Run-options (can be combined): --new receive new edi files (default: if no run-option given: run as new). --resend resend as indicated by user. --rereceive rereceive as indicated by user. --automaticretrycommunication - automatically retry outgoing communication. --cleanup remove older data from database. Config-option: -c<directory> directory for configuration files (default: config). Routes: list of routes to run. Default: all active routes (in the database) '''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version} configdir = 'config' commandspossible = ['--automaticretrycommunication','--resend','--rereceive','--new'] commandstorun = [] routestorun = [] #list with routes to run do_cleanup_parameter = False for arg in sys.argv[1:]: if arg.startswith('-c'): configdir = arg[2:] if not configdir: print 'Error: configuration directory indicated, but no directory name.' sys.exit(1) elif arg in commandspossible: commandstorun.append(arg) elif arg == '--cleanup': do_cleanup_parameter = True elif arg in ["?", "/?",'-h', '--help'] or arg.startswith('-'): print usage sys.exit(0) else: #pick up names of routes to run routestorun.append(arg) if not commandstorun and not do_cleanup_parameter: #if no command on command line, use new (default) commandstorun = ['--new'] commandstorun = [command[2:] for command in commandspossible if command in commandstorun] #sort commands #***********end handling command line arguments************************** botsinit.generalinit(configdir) #find locating of bots, configfiles, init paths etc. #set working directory to bots installation. advantage: when using relative paths it is clear that this point paths within bots installation. os.chdir(botsglobal.ini.get('directories','botspath')) #**************check if another instance of bots-engine is running/if port is free****************************** try: engine_socket = botslib.check_if_other_engine_is_running() except socket.error: sys.exit(3) else: atexit.register(engine_socket.close) #**************initialise logging****************************** process_name = 'engine' botsglobal.logger = botsinit.initenginelogging(process_name) atexit.register(logging.shutdown) for key,value in botslib.botsinfo(): #log info about environement, versions, etc botsglobal.logger.info(u'%(key)s: "%(value)s".',{'key':key,'value':value}) #**************connect to database********************************** try: botsinit.connect() except Exception as msg: botsglobal.logger.exception(_(u'Could not connect to database. Database settings are in bots/config/settings.py. Error: "%(msg)s".'),{'msg':msg}) sys.exit(1) else: botsglobal.logger.info(_(u'Connected to database.')) atexit.register(botsglobal.db.close) #************initialise user exits for the whole bots-engine************************* try: userscript,scriptname = botslib.botsimport('routescripts','botsengine') except ImportError: #userscript is not there; other errors like syntax errors are not catched userscript = scriptname = None #***acceptance tests: initialiase acceptance user script****************************** acceptance_userscript = acceptance_scriptname = None if botsglobal.ini.getboolean('acceptance','runacceptancetest',False): botsglobal.logger.info(_(u'This run is an acceptance test - as indicated in option "runacceptancetest" in bots.ini.')) try: acceptance_userscript,acceptance_scriptname = botslib.botsimport('routescripts','bots_acceptancetest') except ImportError: botsglobal.logger.info(_(u'In acceptance test there is no script file "bots_acceptancetest.py" to check the results of the acceptance test.')) #**************handle database lock**************************************** #set a lock on the database; if not possible, the database is locked: an earlier instance of bots-engine was terminated unexpectedly. if not botslib.set_database_lock(): #for SQLite: do a integrity check on the database if botsglobal.settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3': cursor = botsglobal.db.execute('''PRAGMA integrity_check''') result = cursor.fetchone() if result[0] != u'ok': warn = _(u'!Bots database is locked!\n'\ 'Bots did an integrity check on the database, but database was not OK.\n'\ 'Manual action is needed!\n'\ 'Bots has stopped processing EDI files.') botsglobal.logger.critical(warn) botslib.sendbotserrorreport(_(u'[Bots severe error]Database is damaged'),warn) sys.exit(1) warn = _(u'!Bots database is locked!\n'\ 'Bots-engine has ended in an unexpected way during the last run.\n'\ 'Most likely causes: sudden power-down, system crash, problems with disk I/O, bots-engine terminated by user, etc.\n' 'Bots will do an automatic crash recovery now.') botsglobal.logger.critical(warn) botslib.sendbotserrorreport(_(u'[Bots severe error]Database is locked'),warn) commandstorun.insert(0,'crashrecovery') #there is a database lock. Add a crashrecovery as first command to run. atexit.register(botslib.remove_database_lock) #**************run the routes********************************************** #commandstorun determines the type(s) of run. eg: ['automaticretrycommunication','new'] try: botslib.prepare_confirmrules() #in acceptance tests: run a user script before running eg to clean output directories****************************** botslib.tryrunscript(acceptance_userscript,acceptance_scriptname,'pretest',routestorun=routestorun) botslib.tryrunscript(userscript,scriptname,'pre',commandstorun=commandstorun,routestorun=routestorun) errorinrun = 0 #detect if there has been some error. Only used for correct exit() code first_command_2_run = True for command in commandstorun: #if multiple commands in run: reports etc are based on timestamp; so there needs to be at least one second between these runs. if first_command_2_run: first_command_2_run = False else: time.sleep(1) botsglobal.logger.info(_(u'Run "%(command)s".'),{'command':command}) #************get list of routes to run******************************* if routestorun: use_routestorun = routestorun[:] botsglobal.logger.info(_(u'Run routes from command line: "%(routes)s".'),{'routes':str(use_routestorun)}) elif command == 'new': #fetch all active routes from database unless 'not in default run' or not active. use_routestorun = [] for row in botslib.query('''SELECT DISTINCT idroute FROM routes WHERE active=%(active)s AND (notindefaultrun=%(notindefaultrun)s OR notindefaultrun IS NULL) ORDER BY idroute ''', {'active':True,'notindefaultrun':False}): use_routestorun.append(row['idroute']) botsglobal.logger.info(_(u'Run active routes from database that are in default run: "%(routes)s".'),{'routes':str(use_routestorun)}) else: #for command other than 'new': use all active routes. use_routestorun = [] for row in botslib.query('''SELECT DISTINCT idroute FROM routes WHERE active=%(active)s ORDER BY idroute ''', {'active':True}): use_routestorun.append(row['idroute']) botsglobal.logger.info(_(u'Run all active routes from database: "%(routes)s".'),{'routes':str(use_routestorun)}) #************run routes for this command****************************** botslib.tryrunscript(userscript,scriptname,'pre' + command,routestorun=use_routestorun) errorinrun += router.rundispatcher(command,use_routestorun) botslib.tryrunscript(userscript,scriptname,'post' + command,routestorun=use_routestorun) #*********finished running routes for this command**************************** #*********finished all commands**************************************** botslib.tryrunscript(userscript,scriptname,'post',commandstorun=commandstorun,routestorun=routestorun) try: #in acceptance tests: run a user script. no good reporting of errors/results in post-test script. Reason: this is after automaticmaintence. botslib.tryrunscript(acceptance_userscript,acceptance_scriptname,'posttest',routestorun=use_routestorun) except Exception as msg: print str(msg) cleanup.cleanup(do_cleanup_parameter,userscript,scriptname) except Exception as msg: botsglobal.logger.exception(_(u'Severe error in bots system:\n%(msg)s'),{'msg':str(msg)}) #of course this 'should' not happen. sys.exit(1) else: if errorinrun: sys.exit(2) #indicate: error(s) in run(s) else: sys.exit(0) #OK
def start(): #exit codes: # 0: OK, no errors # 1: (system) errors # 2: bots ran OK, but there are errors/process errors in the run # 3: Database is locked, but "maxruntime" has not been exceeded. #********command line arguments************************** commandspossible = [ '--new', '--retry', '--retransmit', '--cleanup', '--crashrecovery', '--retrycommunication', '--automaticretrycommunication' ] commandstorun = [] routestorun = [] #list with routes to run configdir = 'config' for arg in sys.argv[1:]: if not arg: continue if arg.startswith('-c'): configdir = arg[2:] if not configdir: print 'Configuration directory indicated, but no directory name.' sys.exit(1) elif arg in commandspossible: commandstorun.append(arg) elif arg in ["?", "/?"] or arg.startswith('-'): showusage() sys.exit(0) else: #pick up names of routes to run routestorun.append(arg) if not commandstorun: #if no command on command line, use new (default) commandstorun = ['--new'] #**************init general: find locating of bots, configfiles, init paths etc.**************** botsinit.generalinit(configdir) #set current working directory to botspath #~ old_current_directory = os.getcwdu() os.chdir(botsglobal.ini.get('directories', 'botspath')) #**************initialise logging****************************** try: botsinit.initenginelogging() except: print _('Error in initialising logging system.') traceback.print_exc() sys.exit(1) else: atexit.register(logging.shutdown) for key, value in botslib.botsinfo(): #log start info botsglobal.logger.info(u'%s: "%s".', key, value) #**************connect to database********************************** try: botsinit.connect() except: botsglobal.logger.exception( _(u'Could not connect to database. Database settings are in bots/config/settings.py.' )) sys.exit(1) else: botsglobal.logger.info(_(u'Connected to database.')) atexit.register(botsglobal.db.close) #initialise user exits for the whole bots-engine (this script file) try: userscript, scriptname = botslib.botsimport('routescripts', 'botsengine') except ImportError: userscript = scriptname = None #**************handle database lock**************************************** #try to set a lock on the database; if this is not possible, the database is already locked. Either: #1 another instance bots bots-engine is (still) running #2 or bots-engine had a severe crash. #What to do? #first: check ts of database lock. If below a certain value (set in bots.ini) we assume an other instance is running. Exit quietly - no errors, no logging. # else: Warn user, give advise on what to do. gather data: nr files in, errors. #next: warn with report & logging. advise a crashrecovery. if not botslib.set_database_lock(): if '--crashrecovery' in commandstorun: #user starts recovery operation; the databaselock is ignored; the databaselock is unlocked when routes have run. commandstorun = ['--crashrecovery'] #is an exclusive option! else: #when scheduling bots it is possible that the last run is still running. Check if maxruntime has passed: vanaf = datetime.datetime.today() - datetime.timedelta( minutes=botsglobal.ini.getint('settings', 'maxruntime', 60)) for row in botslib.query( '''SELECT ts FROM mutex WHERE ts < %(vanaf)s ''', {'vanaf': vanaf}): warn = _( u'!Bots database is locked!\nBots-engine has ended in an unexpected way during the last run.\nThis happens, but is very very rare.\nPossible causes: bots-engine terminated by user, system crash, power-down, etc.\nA forced retry of the last run is advised; bots will (try to) repair the last run.' ) botsglobal.logger.critical(warn) botslib.sendbotserrorreport( _(u'[Bots severe error]Database is locked'), warn) #add: count errors etc. sys.exit(1) else: #maxruntime has not passed. Exit silently, nothing reported botsglobal.logger.info( _(u'Database is locked, but "maxruntime" has not been exceeded.' )) sys.exit(3) else: if '--crashrecovery' in commandstorun: #user starts recovery operation but there is no databaselock. warn = _( u'User started a forced retry of the last run.\nOnly use this when the database is locked.\nThe database was not locked (database is OK).\nSo Bots has done nothing now.' ) botsglobal.logger.error(warn) botslib.sendbotserrorreport( _(u'[Bots Error Report] User started a forced retry of last run, but this was not needed' ), warn) botslib.remove_database_lock() sys.exit(1) #*************get list of routes to run**************************************** #~ raise Exception('locked database') #for testing database lock: abort, database will be locked if routestorun: botsglobal.logger.info(u'Run routes from command line: "%s".', str(routestorun)) else: # no routes from command line parameters: fetch all active routes from database for row in botslib.query( '''SELECT DISTINCT idroute FROM routes WHERE active=%(active)s AND (notindefaultrun=%(notindefaultrun)s OR notindefaultrun IS NULL) ORDER BY idroute ''', { 'active': True, 'notindefaultrun': False }): routestorun.append(row['idroute']) botsglobal.logger.info(_(u'Run active routes from database: "%s".'), str(routestorun)) #routestorun is now either a list with routes from commandline, or the list of active routes for the routes table in the db. #**************run the routes for retry, retransmit and new runs************************************* try: #commandstorun determines the type(s) of run #routes to run is a listof the routes that are runs (for each command to run #botsglobal.incommunicate is used to control if there is communication in; only 'new' incommunicates. #botsglobal.minta4query controls which ta's are queried by the routes. #stuff2evaluate controls what is evaluated in automatic maintenance. errorinrun = 0 #detect if there has been some error. Only used for good exit() code botsglobal.incommunicate = False if '--crashrecovery' in commandstorun: botsglobal.logger.info(_(u'Run crash recovery.')) stuff2evaluate = botslib.set_minta4query_crashrecovery() if stuff2evaluate: router.routedispatcher(routestorun) errorinrun += automaticmaintenance.evaluate( '--crashrecovery', stuff2evaluate) else: botsglobal.logger.info( _(u'No retry of the last run - there was no last run.')) if userscript and hasattr(userscript, 'postcrashrecovery'): botslib.runscript(userscript, scriptname, 'postcrashrecovery', routestorun=routestorun) if '--retrycommunication' in commandstorun: botsglobal.logger.info(_(u'Run communication retry.')) stuff2evaluate = router.routedispatcher(routestorun, '--retrycommunication') if stuff2evaluate: errorinrun += automaticmaintenance.evaluate( '--retrycommunication', stuff2evaluate) else: botsglobal.logger.info( _(u'Run recommunicate: nothing to recommunicate.')) if userscript and hasattr(userscript, 'postretrycommunication'): botslib.runscript(userscript, scriptname, 'postretrycommunication', routestorun=routestorun) if '--automaticretrycommunication' in commandstorun: botsglobal.logger.info(_(u'Run automatic communication retry.')) stuff2evaluate = router.routedispatcher( routestorun, '--automaticretrycommunication') if stuff2evaluate: errorinrun += automaticmaintenance.evaluate( '--automaticretrycommunication', stuff2evaluate) else: botsglobal.logger.info( _(u'Run automatic recommunicate: nothing to recommunicate.' )) if userscript and hasattr(userscript, 'postautomaticretrycommunication'): botslib.runscript(userscript, scriptname, 'postautomaticretrycommunication', routestorun=routestorun) if '--retry' in commandstorun: botsglobal.logger.info(u'Run retry.') stuff2evaluate = router.routedispatcher(routestorun, '--retry') if stuff2evaluate: errorinrun += automaticmaintenance.evaluate( '--retry', stuff2evaluate) else: botsglobal.logger.info(_(u'Run retry: nothing to retry.')) if userscript and hasattr(userscript, 'postretry'): botslib.runscript(userscript, scriptname, 'postretry', routestorun=routestorun) if '--retransmit' in commandstorun: botsglobal.logger.info(u'Run retransmit.') stuff2evaluate = router.routedispatcher(routestorun, '--retransmit') if stuff2evaluate: errorinrun += automaticmaintenance.evaluate( '--retransmit', stuff2evaluate) else: botsglobal.logger.info( _(u'Run retransmit: nothing to retransmit.')) if userscript and hasattr(userscript, 'postretransmit'): botslib.runscript(userscript, scriptname, 'postretransmit', routestorun=routestorun) if '--new' in commandstorun: botsglobal.logger.info('Run new.') botsglobal.incommunicate = True botsglobal.minta4query = 0 #meaning: reset. the actual value is set later (in routedispatcher) stuff2evaluate = router.routedispatcher(routestorun) errorinrun += automaticmaintenance.evaluate( '--new', stuff2evaluate) if userscript and hasattr(userscript, 'postnewrun'): botslib.runscript(userscript, scriptname, 'postnewrun', routestorun=routestorun) if '--cleanup' in commandstorun or botsglobal.ini.get( 'settings', 'whencleanup', 'always') == 'always': botsglobal.logger.debug(u'Do cleanup.') cleanup.cleanup() botslib.remove_database_lock() except Exception, msg: botsglobal.logger.exception( _(u'Severe error in bots system:\n%s') % (msg)) #of course this 'should' not happen. sys.exit(1)
def __init__(self,soortpythonfile,editype,grammarname): self.module,self.grammarname = botslib.botsimport(soortpythonfile,editype + '.' + grammarname)