def cleanup(do_cleanup_parameter,userscript,scriptname):
''' public function, does all cleanup of the database and file system.
most cleanup functions are by default done only once a day.
'''
if botsglobal.ini.getboolean('acceptance','runacceptancetest',False): # no cleanup during acceptance testing
return
whencleanup = botsglobal.ini.get('settings','whencleanup','daily')
if do_cleanup_parameter: #if explicit indicated via commandline parameter
do_full_cleanup = True
elif whencleanup in ['always','daily']:
#perform full cleanup only first run of the day.
cur_day = int(time.strftime('%Y%m%d')) #get current date, convert to int
if cur_day != botslib.uniquecore('bots_cleanup_day',updatewith=cur_day):
do_full_cleanup = True
else:
do_full_cleanup = False
else:
do_full_cleanup = False
try:
if do_full_cleanup:
botsglobal.logger.info(u'Cleanup files')
_cleandatafile()
_cleanarchive()
botsglobal.logger.info(u'Cleanup database')
_cleanupsession()
_cleanpersist()
_cleantransactions()
botsglobal.logger.info(u'Vacuum database')
_vacuum()
# postcleanup user exit in botsengine script
if userscript and hasattr(userscript,'postcleanup'):
botslib.runscript(userscript,scriptname,'postcleanup',whencleanup=whencleanup)
botsglobal.logger.info(u'Done full cleanup.')
_cleanrunsnothingreceived() #do this every run, but not logged
except:
botsglobal.logger.exception(u'Cleanup error.')
def _translate_one_file(row,routedict,endstatus,userscript,scriptname):
''' - read, lex, parse, make tree of nodes.
- split up files into messages (using 'nextmessage' of grammar)
- get mappingscript, start mappingscript.
- write the results of translation (no enveloping yet)
'''
try:
ta_fromfile = botslib.OldTransaction(row['idta'])
ta_parsed = ta_fromfile.copyta(status=PARSED)
if row['filesize'] > botsglobal.ini.getint('settings','maxfilesizeincoming',5000000):
ta_parsed.update(filesize=row['filesize'])
raise botslib.FileTooLarge(_(u'File size of %(filesize)s is too big; option "maxfilesizeincoming" in bots.ini is %(maxfilesizeincoming)s.'),
{'filesize':row['filesize'],'maxfilesizeincoming':botsglobal.ini.getint('settings','maxfilesizeincoming',5000000)})
botsglobal.logger.debug(_(u'Start translating file "%(filename)s" editype "%(editype)s" messagetype "%(messagetype)s".'),row)
#read whole edi-file: read, parse and made into a inmessage-object. Message is represented as a tree (inmessage.root is the root of the tree).
edifile = inmessage.parse_edi_file(frompartner=row['frompartner'],
topartner=row['topartner'],
filename=row['filename'],
messagetype=row['messagetype'],
testindicator=row['testindicator'],
editype=row['editype'],
charset=row['charset'],
alt=row['alt'],
fromchannel=row['fromchannel'],
idroute=routedict['idroute'],
command=routedict['command'])
edifile.checkforerrorlist() #no exception if infile has been lexed and parsed OK else raises an error
if int(routedict['translateind']) == 3: #parse & passthrough; file is parsed, partners are known, no mapping, does confirm.
raise botslib.GotoException('dummy')
#edifile.ta_info contains info: QUERIES, charset etc
for inn_splitup in edifile.nextmessage(): #splitup messages in parsed edifile
try:
ta_splitup = ta_parsed.copyta(status=SPLITUP,**inn_splitup.ta_info) #copy PARSED to SPLITUP ta
#inn_splitup.ta_info: parameters from inmessage.parse_edi_file(), syntax-information and parse-information
inn_splitup.ta_info['idta_fromfile'] = ta_fromfile.idta #for confirmations in userscript; used to give idta of 'confirming message'
inn_splitup.ta_info['idta'] = ta_splitup.idta #for confirmations in userscript; used to give idta of 'confirming message'
number_of_loops_with_same_alt = 0
while 1: #continue as long as there are (alt-)translations
#lookup the translation************************
tscript,toeditype,tomessagetype = botslib.lookup_translation(fromeditype=inn_splitup.ta_info['editype'],
frommessagetype=inn_splitup.ta_info['messagetype'],
frompartner=inn_splitup.ta_info['frompartner'],
topartner=inn_splitup.ta_info['topartner'],
alt=inn_splitup.ta_info['alt'])
if not tscript: #no translation found in translate table; check if can find translation via user script
if userscript and hasattr(userscript,'gettranslation'):
tscript,toeditype,tomessagetype = botslib.runscript(userscript,scriptname,'gettranslation',idroute=routedict['idroute'],message=inn_splitup)
if not tscript:
raise botslib.TranslationNotFoundError(_(u'Translation not found for editype "%(editype)s", messagetype "%(messagetype)s", frompartner "%(frompartner)s", topartner "%(topartner)s", alt "%(alt)s".'),
inn_splitup.ta_info)
inn_splitup.ta_info['divtext'] = tscript #ifor reporting used mapping script to database (for display in GUI).
#initialize new out-object*************************
ta_translated = ta_splitup.copyta(status=endstatus) #make ta for translated message (new out-ta)
filename_translated = str(ta_translated.idta)
out_translated = outmessage.outmessage_init(editype=toeditype,messagetype=tomessagetype,filename=filename_translated,reference=unique('messagecounter'),statust=OK,divtext=tscript) #make outmessage object
#run mapping script************************
botsglobal.logger.debug(_(u'Mappingscript "%(tscript)s" translates messagetype "%(messagetype)s" to messagetype "%(tomessagetype)s".'),
{'tscript':tscript,'messagetype':inn_splitup.ta_info['messagetype'],'tomessagetype':out_translated.ta_info['messagetype']})
translationscript,scriptfilename = botslib.botsimport('mappings',inn_splitup.ta_info['editype'],tscript) #get the mappingscript
alt_from_previous_run = inn_splitup.ta_info['alt'] #needed to check for infinite loop
doalttranslation = botslib.runscript(translationscript,scriptfilename,'main',inn=inn_splitup,out=out_translated)
botsglobal.logger.debug(_(u'Mappingscript "%(tscript)s" finished.'),{'tscript':tscript})
#manipulate for some attributes after mapping script
if 'topartner' not in out_translated.ta_info: #out_translated does not contain values from ta......
out_translated.ta_info['topartner'] = inn_splitup.ta_info['topartner']
if 'botskey' in inn_splitup.ta_info:
inn_splitup.ta_info['reference'] = inn_splitup.ta_info['botskey']
if 'botskey' in out_translated.ta_info: #out_translated does not contain values from ta......
out_translated.ta_info['reference'] = out_translated.ta_info['botskey']
#check the value received from the mappingscript to determine what to do in this while-loop. Handling of chained trasnlations.
if doalttranslation is None:
#translation(s) are done; handle out-message
handle_out_message(out_translated,ta_translated)
break #break out of while loop
elif isinstance(doalttranslation,dict):
#some extended cases; a dict is returned that contains 'instructions' for some type of chained translations
if 'type' not in doalttranslation or 'alt' not in doalttranslation:
raise botslib.BotsError(_(u"Mappingscript returned '%(alt)s'. This dict should not have 'type' and 'alt'."),{'alt':doalttranslation})
if alt_from_previous_run == doalttranslation['alt']:
number_of_loops_with_same_alt += 1
else:
number_of_loops_with_same_alt = 0
if doalttranslation['type'] == u'out_as_inn':
#do chained translation: use the out-object as inn-object, new out-object
#use case: detected error in incoming file; use out-object to generate warning email
handle_out_message(out_translated,ta_translated)
inn_splitup = out_translated #out-object is now inn-object
if isinstance(inn_splitup,outmessage.fixed): #for fixed: strip all values in node
inn_splitup.root.stripnode()
inn_splitup.ta_info['alt'] = doalttranslation['alt'] #get the alt-value for the next chained translation
if not 'frompartner' in inn_splitup.ta_info:
inn_splitup.ta_info['frompartner'] = ''
if not 'topartner' in inn_splitup.ta_info:
inn_splitup.ta_info['topartner'] = ''
inn_splitup.ta_info.pop('statust')
elif doalttranslation['type'] == u'no_check_on_infinite_loop':
#do chained translation: allow many loops wit hsame alt-value.
#mapping script will have to handle this correctly.
number_of_loops_with_same_alt = 0
handle_out_message(out_translated,ta_translated)
inn_splitup.ta_info['alt'] = doalttranslation['alt'] #get the alt-value for the next chained translation
else: #there is nothing else
raise botslib.BotsError(_(u'Mappingscript returned dict with an unknown "type": "%(doalttranslation)s".'),{'doalttranslation':doalttranslation})
else: #note: this includes alt '' (empty string)
if alt_from_previous_run == doalttranslation:
number_of_loops_with_same_alt += 1
else:
number_of_loops_with_same_alt = 0
#do normal chained translation: same inn-object, new out-object
handle_out_message(out_translated,ta_translated)
inn_splitup.ta_info['alt'] = doalttranslation #get the alt-value for the next chained translation
if number_of_loops_with_same_alt > 10:
raise botslib.BotsError(_(u'Mappingscript returns same alt value over and over again (infinite loop?). Alt: "%(doalttranslation)s".'),{'doalttranslation':doalttranslation})
#end of while-loop (trans**********************************************************************************
#exceptions file_out-level: exception in mappingscript or writing of out-file
except:
#2 modes: either every error leads to skipping of whole infile (old mode) or errors in mappingscript/outfile only affect that branche
if botsglobal.ini.getboolean('settings','oldmessageerrors',False):
raise
txt = botslib.txtexc()
ta_splitup.update(statust=ERROR,errortext=txt,**inn_splitup.ta_info) #update db. inn_splitup.ta_info could be changed by mappingscript. Is this useful?
ta_splitup.deletechildren()
else:
ta_splitup.update(statust=DONE, **inn_splitup.ta_info) #update db. inn_splitup.ta_info could be changed by mappingscript. Is this useful?
#exceptions file_in-level
except botslib.GotoException: #edi-file is OK, file is passed-through after parsing.
ta_parsed.update(statust=DONE,filesize=row['filesize'],**edifile.ta_info) #update with info from eg queries
ta_parsed.copyta(status=MERGED,statust=OK) #original file goes straight to MERGED
edifile.handleconfirm(ta_fromfile,error=False)
botsglobal.logger.debug(_(u'Parse & passthrough for input file "%(filename)s".'),row)
except botslib.FileTooLarge as msg:
ta_parsed.update(statust=ERROR,errortext=str(msg))
ta_parsed.deletechildren()
botsglobal.logger.debug(u'Error in translating input file "%(filename)s":\n%(msg)s',{'filename':row['filename'],'msg':msg})
except:
txt = botslib.txtexc()
ta_parsed.update(statust=ERROR,errortext=txt,**edifile.ta_info)
ta_parsed.deletechildren()
edifile.handleconfirm(ta_fromfile,error=True)
botsglobal.logger.debug(u'Error in translating input file "%(filename)s":\n%(msg)s',{'filename':row['filename'],'msg':txt})
else:
edifile.handleconfirm(ta_fromfile,error=False)
ta_parsed.update(statust=DONE,filesize=row['filesize'],**edifile.ta_info)
botsglobal.logger.debug(_(u'Translated input file "%(filename)s".'),row)
finally:
ta_fromfile.update(statust=DONE)
def start():
#exit codes:
# 0: OK, no errors
# 1: (system) errors
# 2: bots ran OK, but there are errors/process errors in the run
# 3: Database is locked, but "maxruntime" has not been exceeded.
#********command line arguments**************************
commandspossible = [
'--new', '--retry', '--retransmit', '--cleanup', '--crashrecovery',
'--retrycommunication', '--automaticretrycommunication'
]
commandstorun = []
routestorun = [] #list with routes to run
configdir = 'config'
for arg in sys.argv[1:]:
if not arg:
continue
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Configuration directory indicated, but no directory name.'
sys.exit(1)
elif arg in commandspossible:
commandstorun.append(arg)
elif arg in ["?", "/?"] or arg.startswith('-'):
showusage()
sys.exit(0)
else: #pick up names of routes to run
routestorun.append(arg)
if not commandstorun: #if no command on command line, use new (default)
commandstorun = ['--new']
#**************init general: find locating of bots, configfiles, init paths etc.****************
botsinit.generalinit(configdir)
#set current working directory to botspath
#~ old_current_directory = os.getcwdu()
os.chdir(botsglobal.ini.get('directories', 'botspath'))
#**************initialise logging******************************
try:
botsinit.initenginelogging()
except:
print _('Error in initialising logging system.')
traceback.print_exc()
sys.exit(1)
else:
atexit.register(logging.shutdown)
for key, value in botslib.botsinfo(): #log start info
botsglobal.logger.info(u'%s: "%s".', key, value)
#**************connect to database**********************************
try:
botsinit.connect()
except:
botsglobal.logger.exception(
_(u'Could not connect to database. Database settings are in bots/config/settings.py.'
))
sys.exit(1)
else:
botsglobal.logger.info(_(u'Connected to database.'))
atexit.register(botsglobal.db.close)
#initialise user exits for the whole bots-engine (this script file)
try:
userscript, scriptname = botslib.botsimport('routescripts',
'botsengine')
except ImportError:
userscript = scriptname = None
#**************handle database lock****************************************
#try to set a lock on the database; if this is not possible, the database is already locked. Either:
#1 another instance bots bots-engine is (still) running
#2 or bots-engine had a severe crash.
#What to do?
#first: check ts of database lock. If below a certain value (set in bots.ini) we assume an other instance is running. Exit quietly - no errors, no logging.
# else: Warn user, give advise on what to do. gather data: nr files in, errors.
#next: warn with report & logging. advise a crashrecovery.
if not botslib.set_database_lock():
if '--crashrecovery' in commandstorun: #user starts recovery operation; the databaselock is ignored; the databaselock is unlocked when routes have run.
commandstorun = ['--crashrecovery'] #is an exclusive option!
else:
#when scheduling bots it is possible that the last run is still running. Check if maxruntime has passed:
vanaf = datetime.datetime.today() - datetime.timedelta(
minutes=botsglobal.ini.getint('settings', 'maxruntime', 60))
for row in botslib.query(
'''SELECT ts FROM mutex WHERE ts < %(vanaf)s ''',
{'vanaf': vanaf}):
warn = _(
u'!Bots database is locked!\nBots-engine has ended in an unexpected way during the last run.\nThis happens, but is very very rare.\nPossible causes: bots-engine terminated by user, system crash, power-down, etc.\nA forced retry of the last run is advised; bots will (try to) repair the last run.'
)
botsglobal.logger.critical(warn)
botslib.sendbotserrorreport(
_(u'[Bots severe error]Database is locked'), warn)
#add: count errors etc.
sys.exit(1)
else: #maxruntime has not passed. Exit silently, nothing reported
botsglobal.logger.info(
_(u'Database is locked, but "maxruntime" has not been exceeded.'
))
sys.exit(3)
else:
if '--crashrecovery' in commandstorun: #user starts recovery operation but there is no databaselock.
warn = _(
u'User started a forced retry of the last run.\nOnly use this when the database is locked.\nThe database was not locked (database is OK).\nSo Bots has done nothing now.'
)
botsglobal.logger.error(warn)
botslib.sendbotserrorreport(
_(u'[Bots Error Report] User started a forced retry of last run, but this was not needed'
), warn)
botslib.remove_database_lock()
sys.exit(1)
#*************get list of routes to run****************************************
#~ raise Exception('locked database') #for testing database lock: abort, database will be locked
if routestorun:
botsglobal.logger.info(u'Run routes from command line: "%s".',
str(routestorun))
else: # no routes from command line parameters: fetch all active routes from database
for row in botslib.query(
'''SELECT DISTINCT idroute
FROM routes
WHERE active=%(active)s
AND (notindefaultrun=%(notindefaultrun)s OR notindefaultrun IS NULL)
ORDER BY idroute ''', {
'active': True,
'notindefaultrun': False
}):
routestorun.append(row['idroute'])
botsglobal.logger.info(_(u'Run active routes from database: "%s".'),
str(routestorun))
#routestorun is now either a list with routes from commandline, or the list of active routes for the routes table in the db.
#**************run the routes for retry, retransmit and new runs*************************************
try:
#commandstorun determines the type(s) of run
#routes to run is a listof the routes that are runs (for each command to run
#botsglobal.incommunicate is used to control if there is communication in; only 'new' incommunicates.
#botsglobal.minta4query controls which ta's are queried by the routes.
#stuff2evaluate controls what is evaluated in automatic maintenance.
errorinrun = 0 #detect if there has been some error. Only used for good exit() code
botsglobal.incommunicate = False
if '--crashrecovery' in commandstorun:
botsglobal.logger.info(_(u'Run crash recovery.'))
stuff2evaluate = botslib.set_minta4query_crashrecovery()
if stuff2evaluate:
router.routedispatcher(routestorun)
errorinrun += automaticmaintenance.evaluate(
'--crashrecovery', stuff2evaluate)
else:
botsglobal.logger.info(
_(u'No retry of the last run - there was no last run.'))
if userscript and hasattr(userscript, 'postcrashrecovery'):
botslib.runscript(userscript,
scriptname,
'postcrashrecovery',
routestorun=routestorun)
if '--retrycommunication' in commandstorun:
botsglobal.logger.info(_(u'Run communication retry.'))
stuff2evaluate = router.routedispatcher(routestorun,
'--retrycommunication')
if stuff2evaluate:
errorinrun += automaticmaintenance.evaluate(
'--retrycommunication', stuff2evaluate)
else:
botsglobal.logger.info(
_(u'Run recommunicate: nothing to recommunicate.'))
if userscript and hasattr(userscript, 'postretrycommunication'):
botslib.runscript(userscript,
scriptname,
'postretrycommunication',
routestorun=routestorun)
if '--automaticretrycommunication' in commandstorun:
botsglobal.logger.info(_(u'Run automatic communication retry.'))
stuff2evaluate = router.routedispatcher(
routestorun, '--automaticretrycommunication')
if stuff2evaluate:
errorinrun += automaticmaintenance.evaluate(
'--automaticretrycommunication', stuff2evaluate)
else:
botsglobal.logger.info(
_(u'Run automatic recommunicate: nothing to recommunicate.'
))
if userscript and hasattr(userscript,
'postautomaticretrycommunication'):
botslib.runscript(userscript,
scriptname,
'postautomaticretrycommunication',
routestorun=routestorun)
if '--retry' in commandstorun:
botsglobal.logger.info(u'Run retry.')
stuff2evaluate = router.routedispatcher(routestorun, '--retry')
if stuff2evaluate:
errorinrun += automaticmaintenance.evaluate(
'--retry', stuff2evaluate)
else:
botsglobal.logger.info(_(u'Run retry: nothing to retry.'))
if userscript and hasattr(userscript, 'postretry'):
botslib.runscript(userscript,
scriptname,
'postretry',
routestorun=routestorun)
if '--retransmit' in commandstorun:
botsglobal.logger.info(u'Run retransmit.')
stuff2evaluate = router.routedispatcher(routestorun,
'--retransmit')
if stuff2evaluate:
errorinrun += automaticmaintenance.evaluate(
'--retransmit', stuff2evaluate)
else:
botsglobal.logger.info(
_(u'Run retransmit: nothing to retransmit.'))
if userscript and hasattr(userscript, 'postretransmit'):
botslib.runscript(userscript,
scriptname,
'postretransmit',
routestorun=routestorun)
if '--new' in commandstorun:
botsglobal.logger.info('Run new.')
botsglobal.incommunicate = True
botsglobal.minta4query = 0 #meaning: reset. the actual value is set later (in routedispatcher)
stuff2evaluate = router.routedispatcher(routestorun)
errorinrun += automaticmaintenance.evaluate(
'--new', stuff2evaluate)
if userscript and hasattr(userscript, 'postnewrun'):
botslib.runscript(userscript,
scriptname,
'postnewrun',
routestorun=routestorun)
if '--cleanup' in commandstorun or botsglobal.ini.get(
'settings', 'whencleanup', 'always') == 'always':
botsglobal.logger.debug(u'Do cleanup.')
cleanup.cleanup()
botslib.remove_database_lock()
except Exception, e:
botsglobal.logger.exception(
_(u'Severe error in bots system:\n%s') %
(e)) #of course this 'should' not happen.
sys.exit(1)
def translate(startstatus,endstatus,idroute,rootidta):
''' main translation loop.
get edifiles to be translated, than:
- read, lex, parse, make tree of nodes.
- split up files into messages (using 'nextmessage' of grammar)
- get mappingscript, start mappingscript.
- write the results of translation (no enveloping yet)
status: FILEIN--PARSED-<SPLITUP--TRANSLATED
'''
try: #see if there is a userscript that can determine the translation
userscript,scriptname = botslib.botsimport('mappings','translation')
except ImportError: #userscript is not there; other errors like syntax errors are not catched
userscript = scriptname = None
#select edifiles to translate
for rawrow in botslib.query(u'''SELECT idta,frompartner,topartner,filename,messagetype,testindicator,editype,charset,alt,fromchannel,filesize
FROM ta
WHERE idta>%(rootidta)s
AND status=%(status)s
AND statust=%(statust)s
AND idroute=%(idroute)s ''',
{'status':startstatus,'statust':OK,'idroute':idroute,'rootidta':rootidta}):
try:
row = dict(rawrow) #convert to real dictionary ()
ta_fromfile = botslib.OldTransaction(row['idta'])
ta_parsed = ta_fromfile.copyta(status=PARSED) #make PARSED ta
if row['filesize'] > botsglobal.ini.getint('settings','maxfilesizeincoming',5000000):
ta_parsed.update(filesize=row['filesize'])
raise botslib.InMessageError(_(u'File size of %(filesize)s is too big; option "maxfilesizeincoming" in bots.ini is %(maxfilesizeincoming)s.'),
{'filesize':row['filesize'],'maxfilesizeincoming':botsglobal.ini.getint('settings','maxfilesizeincoming',5000000)})
botsglobal.logger.debug(_(u'Start translating file "%(filename)s" editype "%(editype)s" messagetype "%(messagetype)s".'),row)
#read whole edi-file: read, parse and made into a inmessage-object. Message is represented as a tree (inmessage.root is the root of the tree).
edifile = inmessage.parse_edi_file(frompartner=row['frompartner'],
topartner=row['topartner'],
filename=row['filename'],
messagetype=row['messagetype'],
testindicator=row['testindicator'],
editype=row['editype'],
charset=row['charset'],
alt=row['alt'],
fromchannel=row['fromchannel'],
idroute=idroute)
#if no exception: infile has been lexed and parsed OK.
#edifile.ta_info contains info: QUERIES, charset etc
for inn_splitup in edifile.nextmessage(): #splitup messages in parsed edifile
try:
ta_splitup = ta_parsed.copyta(status=SPLITUP,**inn_splitup.ta_info) #copy PARSED to SPLITUP ta
#inn_splitup.ta_info: parameters from inmessage.parse_edi_file(), syntax-information and parse-information
inn_splitup.ta_info['idta_fromfile'] = ta_fromfile.idta #for confirmations in userscript; used to give idta of 'confirming message'
post_mapping_mode = False #if post_mapping: reuse out-object, if no translation is found no error
number_of_loops_with_same_alt = 0
while 1: #continue as long as there are (alt-)translations
#lookup the translation************************
tscript,toeditype,tomessagetype = botslib.lookup_translation(fromeditype=inn_splitup.ta_info['editype'],
frommessagetype=inn_splitup.ta_info['messagetype'],
frompartner=inn_splitup.ta_info['frompartner'],
topartner=inn_splitup.ta_info['topartner'],
alt=inn_splitup.ta_info['alt'])
if not tscript: #no translation found in translate table; check if can find translation via user script
if userscript and hasattr(userscript,'gettranslation'):
tscript,toeditype,tomessagetype = botslib.runscript(userscript,scriptname,'gettranslation',idroute=idroute,message=inn_splitup)
if not tscript:
if post_mapping_mode: #in post-mapping mode, not finding the (partern specific!) script is no problem: translation is done
#there is no post-mapping script found (in other words, default mapping script is enough).
#all OK, handle the generated out-file and continue with next message
botsglobal.logger.debug(_(u'Found no "post-mapping" translation for editype "%(editype)s", messagetype "%(messagetype)s", frompartner "%(frompartner)s", topartner "%(topartner)s", alt "%(alt)s".'),
inn_splitup.ta_info)
handle_out_message(out_translated,ta_translated)
del out_translated
break #break out of while loop
else:
raise botslib.TranslationNotFoundError(_(u'Translation not found for editype "%(editype)s", messagetype "%(messagetype)s", frompartner "%(frompartner)s", topartner "%(topartner)s", alt "%(alt)s".'),
inn_splitup.ta_info)
inn_splitup.ta_info['divtext'] = tscript #ifor reporting used mapping script to database (for display in GUI).
if not post_mapping_mode:
#initialize new out-object*************************
ta_translated = ta_splitup.copyta(status=endstatus) #make ta for translated message (new out-ta)
filename_translated = str(ta_translated.idta)
out_translated = outmessage.outmessage_init(editype=toeditype,messagetype=tomessagetype,filename=filename_translated,reference=unique('messagecounter'),statust=OK,divtext=tscript) #make outmessage object
#run mapping script************************
botsglobal.logger.debug(_(u'Mappingscript "%(tscript)s" translates messagetype "%(messagetype)s" to messagetype "%(tomessagetype)s".'),
{'tscript':tscript,'messagetype':inn_splitup.ta_info['messagetype'],'tomessagetype':out_translated.ta_info['messagetype']})
translationscript,scriptfilename = botslib.botsimport('mappings',inn_splitup.ta_info['editype'],tscript) #get the mappingscript
alt_from_previous_run = inn_splitup.ta_info['alt'] #needed to check for infinite loop
doalttranslation = botslib.runscript(translationscript,scriptfilename,'main',inn=inn_splitup,out=out_translated)
botsglobal.logger.debug(_(u'Mappingscript "%(tscript)s" finished.'),{'tscript':tscript})
#manipulate for some attributes after mapping script
if 'topartner' not in out_translated.ta_info: #out_translated does not contain values from ta......
out_translated.ta_info['topartner'] = inn_splitup.ta_info['topartner']
if 'botskey' in inn_splitup.ta_info:
inn_splitup.ta_info['reference'] = inn_splitup.ta_info['botskey']
if 'botskey' in out_translated.ta_info: #out_translated does not contain values from ta......
out_translated.ta_info['reference'] = out_translated.ta_info['botskey']
#check the value received from the mappingscript to determine what to do in this while-loop. Handling of chained trasnlations.
if doalttranslation is None:
#translation(s) are done; handle out-message
handle_out_message(out_translated,ta_translated)
del out_translated
break #break out of while loop
elif isinstance(doalttranslation,dict):
#some extended cases; a dict is returned that contains 'instructions' for some type of chained translations
if 'type' not in doalttranslation or 'alt' not in doalttranslation:
raise botslib.BotsError(_(u"Mappingscript returned '%(alt)s'. This dict should not have 'type' and 'alt'."),{'alt':doalttranslation})
if alt_from_previous_run == doalttranslation['alt']:
number_of_loops_with_same_alt += 1
else:
number_of_loops_with_same_alt = 0
if doalttranslation['type'] == u'out_as_inn':
#do chained translation: use the out-object as inn-object, new out-object
#use case: detected error in incoming file; use out-object to generate warning email
handle_out_message(out_translated,ta_translated)
inn_splitup = out_translated #out-object is now inn-object
if isinstance(inn_splitup,outmessage.fixed): #for fixed: strip all values in node
inn_splitup.root.stripnode()
inn_splitup.ta_info['alt'] = doalttranslation['alt'] #get the alt-value for the next chained translation
if not 'frompartner' in inn_splitup.ta_info:
inn_splitup.ta_info['frompartner'] = ''
if not 'topartner' in inn_splitup.ta_info:
inn_splitup.ta_info['topartner'] = ''
inn_splitup.ta_info.pop('statust')
elif doalttranslation['type'] == u'no_check_on_infinite_loop':
#do chained translation: allow many loops wit hsame alt-value.
#mapping script will have to handle this correctly.
number_of_loops_with_same_alt = 0
handle_out_message(out_translated,ta_translated)
del out_translated
inn_splitup.ta_info['alt'] = doalttranslation['alt'] #get the alt-value for the next chained translation
elif doalttranslation['type'] == u'post_mapping':
#do chained (post)translation: same inn and out-objects.
#in other words, the post-translation continue where is 'default' mapping ended.
#use case: default mapping (for all partners), partner-specific post-translation
#note: this is easier via import defaultmapping in a partner specific mapping. Please use that method!
post_mapping_mode = True #translation is done; reset post_mapping_mode
inn_splitup.ta_info['alt'] = doalttranslation['alt'] #get the alt-value for the next chained translation
#~ inn_splitup.ta_info.pop('statust')
else: #there is nothing else
raise botslib.BotsError(_(u'Mappingscript returned dict with an unknown "type": "%(doalttranslation)s".'),{'doalttranslation':doalttranslation})
else: #note: this includes alt '' (empty string)
if alt_from_previous_run == doalttranslation:
number_of_loops_with_same_alt += 1
else:
number_of_loops_with_same_alt = 0
#do normal chained translation: same inn-object, new out-object
handle_out_message(out_translated,ta_translated)
del out_translated
inn_splitup.ta_info['alt'] = doalttranslation #get the alt-value for the next chained translation
if number_of_loops_with_same_alt > 10:
raise botslib.BotsError(_(u'Mappingscript returns same alt value over and over again (infinite loop?). Alt: "%(doalttranslation)s".'),{'doalttranslation':doalttranslation})
#end of while-loop (trans**********************************************************************************
#exceptions file_out-level: exception in mappingscript or writing of out-file
except:
#2 modes: either every error leads to skipping of whole infile (old mode) or errors in mappingscript/outfile only affect that branche
if botsglobal.ini.getboolean('settings','oldmessageerrors',False):
raise
txt = botslib.txtexc()
ta_splitup.update(statust=ERROR,errortext=txt,**inn_splitup.ta_info) #update db. inn_splitup.ta_info could be changed by mappingscript. Is this useful?
ta_splitup.deletechildren()
else:
ta_splitup.update(statust=DONE, **inn_splitup.ta_info) #update db. inn_splitup.ta_info could be changed by mappingscript. Is this useful?
#exceptions file_in-level (file not OK according to grammar)
except:
txt = botslib.txtexc()
ta_parsed.update(statust=ERROR,errortext=txt)
ta_parsed.deletechildren()
botsglobal.logger.debug(u'Error in translating input file "%(filename)s":\n%(msg)s',{'filename':row['filename'],'msg':txt})
else:
edifile.handleconfirm(ta_fromfile,error=False)
ta_parsed.update(statust=DONE,filesize=row['filesize'],**edifile.ta_info)
botsglobal.logger.debug(_(u'Translated input file "%(filename)s".'),row)
finally:
ta_fromfile.update(statust=DONE)
def translate(run):
for messagedict in run.incoming:
try:
#read whole edi-file: read, parse and made into a inmessage-object. Message is represented as a tree (inmessage.root is the root of the tree).
edifile = inmessage.parse_edi_file(frompartner='',
topartner='',
filename=messagedict['filename'],
messagetype=run.translation['messagetype'],
testindicator='',
editype=run.translation['editype'],
charset='',
alt='',
fromchannel='',
idroute='',
command='')
edifile.checkforerrorlist() #no exception if infile has been lexed and parsed OK else raises an error
#~ if int(routedict['translateind']) == 3: #parse & passthrough; file is parsed, partners are known, no mapping, does confirm.
#~ raise botslib.GotoException('dummy')
#~ continue #file is parsed; no errors. no translation
#edifile.ta_info contains info about incoming file: QUERIES, charset etc
for inn_splitup in edifile.nextmessage(): #splitup messages in parsed edifile
try:
#inn_splitup.ta_info: parameters from inmessage.parse_edi_file(), syntax-information and parse-information
number_of_loops_with_same_alt = 0
while 1: #continue as long as there are (alt-)translations
#lookup the translation************************
tscript,toeditype,tomessagetype = 'orders_edifact2xml' ,'xml','orders'
if 'tscript' in run.translation:
tscript = run.translation['tscript']
toeditype = run.translation['toeditype']
tomessagetype = run.translation['tomessagetype']
else:
tscript,toeditype,tomessagetype = botslib.lookup_translation(fromeditype=inn_splitup.ta_info['editype'],
frommessagetype=inn_splitup.ta_info['messagetype'],
frompartner=inn_splitup.ta_info['frompartner'],
topartner=inn_splitup.ta_info['topartner'],
alt=inn_splitup.ta_info['alt'])
#run mapping script************************
filename_translated = transform.unique('bots_file_name')
out_translated = outmessage.outmessage_init(editype=toeditype,
messagetype=tomessagetype,
filename=filename_translated,
reference=transform.unique('messagecounter'),
statust=OK,
divtext=tscript) #make outmessage object
#~ botsglobal.logger.debug(_(u'Mappingscript "%(tscript)s" translates messagetype "%(messagetype)s" to messagetype "%(tomessagetype)s".'),
#~ {'tscript':tscript,'messagetype':inn_splitup.ta_info['messagetype'],'tomessagetype':out_translated.ta_info['messagetype']})
translationscript,scriptfilename = botslib.botsimport('mappings',inn_splitup.ta_info['editype'],tscript) #import mappingscript
alt_from_previous_run = inn_splitup.ta_info['alt'] #needed to check for infinite loop
doalttranslation = botslib.runscript(translationscript,scriptfilename,'main',inn=inn_splitup,out=out_translated)
botsglobal.logger.debug(_(u'Mappingscript "%(tscript)s" finished.'),{'tscript':tscript})
#manipulate for some attributes after mapping script
if 'topartner' not in out_translated.ta_info: #out_translated does not contain values from run......
out_translated.ta_info['topartner'] = inn_splitup.ta_info['topartner']
if 'botskey' in inn_splitup.ta_info:
inn_splitup.ta_info['reference'] = inn_splitup.ta_info['botskey']
if 'botskey' in out_translated.ta_info: #out_translated does not contain values from run......
out_translated.ta_info['reference'] = out_translated.ta_info['botskey']
#check the value received from the mappingscript to determine what to do in this while-loop. Handling of chained trasnlations.
if doalttranslation is None:
#translation(s) are done; handle out-message
out_translated.writeall() #write result of translation.
#make translated record (if all is OK)
translated_dict = inn_splitup.ta_info.copy()
translated_dict.update(messagedict)
translated_dict.update(out_translated.ta_info)
run.translated.append(translated_dict)
del out_translated
break #break out of while loop
elif isinstance(doalttranslation,dict):
#some extended cases; a dict is returned that contains 'instructions' for some type of chained translations
if 'type' not in doalttranslation or 'alt' not in doalttranslation:
raise botslib.BotsError(_(u"Mappingscript returned '%(alt)s'. This dict should not have 'type' and 'alt'."),{'alt':doalttranslation})
if alt_from_previous_run == doalttranslation['alt']:
number_of_loops_with_same_alt += 1
else:
number_of_loops_with_same_alt = 0
if doalttranslation['type'] == u'out_as_inn':
#do chained translation: use the out-object as inn-object, new out-object
#use case: detected error in incoming file; use out-object to generate warning email
handle_out_message(out_translated,ta_translated)
inn_splitup = out_translated #out-object is now inn-object
if isinstance(inn_splitup,outmessage.fixed): #for fixed: strip all values in node
inn_splitup.root.stripnode()
inn_splitup.ta_info['alt'] = doalttranslation['alt'] #get the alt-value for the next chained translation
if not 'frompartner' in inn_splitup.ta_info:
inn_splitup.ta_info['frompartner'] = ''
if not 'topartner' in inn_splitup.ta_info:
inn_splitup.ta_info['topartner'] = ''
inn_splitup.ta_info.pop('statust')
elif doalttranslation['type'] == u'no_check_on_infinite_loop':
#do chained translation: allow many loops wit hsame alt-value.
#mapping script will have to handle this correctly.
number_of_loops_with_same_alt = 0
handle_out_message(out_translated,ta_translated)
del out_translated
inn_splitup.ta_info['alt'] = doalttranslation['alt'] #get the alt-value for the next chained translation
else: #there is nothing else
raise botslib.BotsError(_(u'Mappingscript returned dict with an unknown "type": "%(doalttranslation)s".'),{'doalttranslation':doalttranslation})
else: #note: this includes alt '' (empty string)
if alt_from_previous_run == doalttranslation:
number_of_loops_with_same_alt += 1
else:
number_of_loops_with_same_alt = 0
#do normal chained translation: same inn-object, new out-object
out_translated.writeall()
del out_translated
inn_splitup.ta_info['alt'] = doalttranslation #get the alt-value for the next chained translation
if number_of_loops_with_same_alt > 10:
raise botslib.BotsError(_(u'Mappingscript returns same alt value over and over again (infinite loop?). Alt: "%(doalttranslation)s".'),{'doalttranslation':doalttranslation})
#end of while-loop (trans**********************************************************************************
#exceptions file_out-level: exception in mappingscript or writing of out-file
except:
#2 modes: either every error leads to skipping of whole infile (old mode) or errors in mappingscript/outfile only affect that branche
txt = botslib.txtexc()
print txt
messagedict['error'] += txt.strip()
else:
pass
#~ print 'succes'
#exceptions file_in-level
except botslib.GotoException: #edi-file is OK, file is passed-through after parsing.
#~ edifile.handleconfirm(ta_fromfile,error=False)
#~ botsglobal.logger.debug(_(u'Parse & passthrough for input file "%(filename)s".'),row)
txt = botslib.txtexc()
print txt
except:
txt = botslib.txtexc()
messagedict['error'] += txt.strip()
#~ edifile.handleconfirm(ta_fromfile,error=True)
#~ botsglobal.logger.debug(u'Error in translating input file "%(filename)s":\n%(msg)s',{'filename':row['filename'],'msg':txt})
else:
pass
def translate(startstatus=TRANSLATE, endstatus=TRANSLATED, idroute=""):
""" translates edifiles in one or more edimessages.
reads and parses edifiles that have to be translated.
tries to split files into messages (using 'nextmessage' of grammar); if no splitting: edifile is one message.
searches the right translation in translate-table;
runs the mapping-script for the translation;
Function takes db-ta with status=TRANSLATE->PARSED->SPLITUP->TRANSLATED
"""
# select edifiles to translate; fill ta-object
# ~ import gc
# ~ gc.disable()
for row in botslib.query(
u"""SELECT idta,frompartner,topartner,filename,messagetype,testindicator,editype,charset,alt,fromchannel
FROM ta
WHERE idta>%(rootidta)s
AND status=%(status)s
AND statust=%(statust)s
AND idroute=%(idroute)s
""",
{"status": startstatus, "statust": OK, "idroute": idroute, "rootidta": botslib.get_minta4query()},
):
try:
ta_fromfile = botslib.OldTransaction(row["idta"]) # TRANSLATE ta
ta_parsedfile = ta_fromfile.copyta(status=PARSED) # copy TRANSLATE to PARSED ta
# whole edi-file is read, parsed and made into a inmessage-object:
edifile = inmessage.edifromfile(
frompartner=row["frompartner"],
topartner=row["topartner"],
filename=row["filename"],
messagetype=row["messagetype"],
testindicator=row["testindicator"],
editype=row["editype"],
charset=row["charset"],
alt=row["alt"],
fromchannel=row["fromchannel"],
idroute=idroute,
)
botsglobal.logger.debug(
u'start read and parse input file "%s" editype "%s" messagetype "%s".',
row["filename"],
row["editype"],
row["messagetype"],
)
for inn in edifile.nextmessage(): # for each message in the edifile:
# inn.ta_info: parameters from inmessage.edifromfile(), syntax-information and parse-information
ta_frommes = ta_parsedfile.copyta(status=SPLITUP) # copy PARSED to SPLITUP ta
inn.ta_info[
"idta_fromfile"
] = ta_fromfile.idta # for confirmations in user script; used to give idta of 'confirming message'
ta_frommes.update(
**inn.ta_info
) # update ta-record SLIPTUP with info from message content and/or grammar
while 1: # whileloop continues as long as there are alt-translations
# ************select parameters for translation(script):
for row2 in botslib.query(
u"""SELECT tscript,tomessagetype,toeditype
FROM translate
WHERE frommessagetype = %(frommessagetype)s
AND fromeditype = %(fromeditype)s
AND active=%(booll)s
AND alt=%(alt)s
AND (frompartner_id IS NULL OR frompartner_id=%(frompartner)s OR frompartner_id in (SELECT to_partner_id
FROM partnergroup
WHERE from_partner_id=%(frompartner)s ))
AND (topartner_id IS NULL OR topartner_id=%(topartner)s OR topartner_id in (SELECT to_partner_id
FROM partnergroup
WHERE from_partner_id=%(topartner)s ))
ORDER BY alt DESC,
CASE WHEN frompartner_id IS NULL THEN 1 ELSE 0 END, frompartner_id ,
CASE WHEN topartner_id IS NULL THEN 1 ELSE 0 END, topartner_id """,
{
"frommessagetype": inn.ta_info["messagetype"],
"fromeditype": inn.ta_info["editype"],
"alt": inn.ta_info["alt"],
"frompartner": inn.ta_info["frompartner"],
"topartner": inn.ta_info["topartner"],
"booll": True,
},
):
break # escape if found; we need only the first - ORDER BY in the query
else: # no translation record is found
raise botslib.TranslationNotFoundError(
_(
u'Editype "$editype", messagetype "$messagetype", frompartner "$frompartner", topartner "$topartner", alt "$alt"'
),
editype=inn.ta_info["editype"],
messagetype=inn.ta_info["messagetype"],
frompartner=inn.ta_info["frompartner"],
topartner=inn.ta_info["topartner"],
alt=inn.ta_info["alt"],
)
ta_tomes = ta_frommes.copyta(status=endstatus) # copy SPLITUP to TRANSLATED ta
tofilename = str(ta_tomes.idta)
tscript = row2["tscript"]
tomessage = outmessage.outmessage_init(
messagetype=row2["tomessagetype"],
editype=row2["toeditype"],
filename=tofilename,
reference=unique("messagecounter"),
statust=OK,
divtext=tscript,
) # make outmessage object
# copy ta_info
botsglobal.logger.debug(
u'script "%s" translates messagetype "%s" to messagetype "%s".',
tscript,
inn.ta_info["messagetype"],
tomessage.ta_info["messagetype"],
)
translationscript, scriptfilename = botslib.botsimport(
"mappings", inn.ta_info["editype"] + "." + tscript
) # get the mapping-script
doalttranslation = botslib.runscript(
translationscript, scriptfilename, "main", inn=inn, out=tomessage
)
botsglobal.logger.debug(u'script "%s" finished.', tscript)
if "topartner" not in tomessage.ta_info: # tomessage does not contain values from ta......
tomessage.ta_info["topartner"] = inn.ta_info["topartner"]
if (
tomessage.ta_info["statust"] == DONE
): # if indicated in user script the message should be discarded
botsglobal.logger.debug(u"No output file because mapping script explicitly indicated this.")
tomessage.ta_info["filename"] = ""
tomessage.ta_info["status"] = DISCARD
else:
botsglobal.logger.debug(
u'Start writing output file editype "%s" messagetype "%s".',
tomessage.ta_info["editype"],
tomessage.ta_info["messagetype"],
)
tomessage.writeall() # write tomessage (result of translation).
# problem is that not all values ta_tomes are know to to_message....
# ~ print 'tomessage.ta_info',tomessage.ta_info
ta_tomes.update(**tomessage.ta_info) # update outmessage transaction with ta_info;
del tomessage
# ~ gc.collect()
if not doalttranslation:
break # out of while loop
else:
inn.ta_info["alt"] = doalttranslation
# end of while-loop
# ~ print inn.ta_info
ta_frommes.update(
statust=DONE, **inn.ta_info
) # update db. inn.ta_info could be changed by script. Is this useful?
del inn
# ~ gc.collect()
# exceptions file_in-level
except:
# ~ edifile.handleconfirm(ta_fromfile,error=True) #only useful if errors are reported in acknowledgement (eg x12 997). Not used now.
txt = botslib.txtexc()
ta_parsedfile.failure()
ta_parsedfile.update(statust=ERROR, errortext=txt)
botsglobal.logger.debug(u'error in translating input file "%s":\n%s', row["filename"], txt)
else:
edifile.handleconfirm(ta_fromfile, error=False)
ta_fromfile.update(statust=DONE)
ta_parsedfile.update(statust=DONE, **edifile.confirminfo)
botsglobal.logger.debug(u'translated input file "%s".', row["filename"])
del edifile
def translate(startstatus=TRANSLATE, endstatus=TRANSLATED, idroute=''):
''' translates edifiles in one or more edimessages.
reads and parses edifiles that have to be translated.
tries to split files into messages (using 'nextmessage' of grammar); if no splitting: edifile is one message.
searches the right translation in translate-table;
runs the mapping-script for the translation;
Function takes db-ta with status=TRANSLATE->PARSED->SPLITUP->TRANSLATED
'''
#select edifiles to translate; fill ta-object
for row in botslib.query(
u'''SELECT idta,frompartner,topartner,filename,messagetype,testindicator,editype,charset,alt,fromchannel
FROM ta
WHERE idta>%(rootidta)s
AND status=%(status)s
AND statust=%(statust)s
AND idroute=%(idroute)s
''', {
'status': startstatus,
'statust': OK,
'idroute': idroute,
'rootidta': botslib.get_minta4query()
}):
try:
ta_fromfile = botslib.OldTransaction(row['idta']) #TRANSLATE ta
ta_parsedfile = ta_fromfile.copyta(
status=PARSED) #copy TRANSLATE to PARSED ta
#read whole edi-file: read, parse and made into a inmessage-object. Message is represented a a tree.
edifile = inmessage.edifromfile(frompartner=row['frompartner'],
topartner=row['topartner'],
filename=row['filename'],
messagetype=row['messagetype'],
testindicator=row['testindicator'],
editype=row['editype'],
charset=row['charset'],
alt=row['alt'],
fromchannel=row['fromchannel'],
idroute=idroute)
botsglobal.logger.debug(
u'start read and parse input file "%s" editype "%s" messagetype "%s".',
row['filename'], row['editype'], row['messagetype'])
for inn in edifile.nextmessage(
): #for each message in the edifile:
#inn.ta_info: parameters from inmessage.edifromfile(), syntax-information and parse-information
ta_frommes = ta_parsedfile.copyta(
status=SPLITUP) #copy PARSED to SPLITUP ta
inn.ta_info[
'idta_fromfile'] = ta_fromfile.idta #for confirmations in user script; used to give idta of 'confirming message'
ta_frommes.update(
**inn.ta_info
) #update ta-record SLIPTUP with info from message content and/or grammar
while 1: #whileloop continues as long as there are alt-translations
#************select parameters for translation(script):
for row2 in botslib.query(
u'''SELECT tscript,tomessagetype,toeditype
FROM translate
WHERE frommessagetype = %(frommessagetype)s
AND fromeditype = %(fromeditype)s
AND active=%(booll)s
AND alt=%(alt)s
AND (frompartner_id IS NULL OR frompartner_id=%(frompartner)s OR frompartner_id in (SELECT to_partner_id
FROM partnergroup
WHERE from_partner_id=%(frompartner)s ))
AND (topartner_id IS NULL OR topartner_id=%(topartner)s OR topartner_id in (SELECT to_partner_id
FROM partnergroup
WHERE from_partner_id=%(topartner)s ))
ORDER BY alt DESC,
CASE WHEN frompartner_id IS NULL THEN 1 ELSE 0 END, frompartner_id ,
CASE WHEN topartner_id IS NULL THEN 1 ELSE 0 END, topartner_id ''',
{
'frommessagetype': inn.ta_info['messagetype'],
'fromeditype': inn.ta_info['editype'],
'alt': inn.ta_info['alt'],
'frompartner': inn.ta_info['frompartner'],
'topartner': inn.ta_info['topartner'],
'booll': True
}):
break #escape if found; we need only the first - ORDER BY in the query
else: #no translation record is found
raise botslib.TranslationNotFoundError(
_(u'Editype "$editype", messagetype "$messagetype", frompartner "$frompartner", topartner "$topartner", alt "$alt"'
),
editype=inn.ta_info['editype'],
messagetype=inn.ta_info['messagetype'],
frompartner=inn.ta_info['frompartner'],
topartner=inn.ta_info['topartner'],
alt=inn.ta_info['alt'])
ta_tomes = ta_frommes.copyta(
status=endstatus) #copy SPLITUP to TRANSLATED ta
tofilename = str(ta_tomes.idta)
tscript = row2['tscript']
tomessage = outmessage.outmessage_init(
messagetype=row2['tomessagetype'],
editype=row2['toeditype'],
filename=tofilename,
reference=unique('messagecounter'),
statust=OK,
divtext=tscript) #make outmessage object
#copy ta_info
botsglobal.logger.debug(
u'script "%s" translates messagetype "%s" to messagetype "%s".',
tscript, inn.ta_info['messagetype'],
tomessage.ta_info['messagetype'])
translationscript, scriptfilename = botslib.botsimport(
'mappings', inn.ta_info['editype'] + '.' +
tscript) #get the mapping-script
doalttranslation = botslib.runscript(translationscript,
scriptfilename,
'main',
inn=inn,
out=tomessage)
botsglobal.logger.debug(u'script "%s" finished.', tscript)
if 'topartner' not in tomessage.ta_info: #tomessage does not contain values from ta......
tomessage.ta_info['topartner'] = inn.ta_info[
'topartner']
if tomessage.ta_info[
'statust'] == DONE: #if indicated in user script the message should be discarded
botsglobal.logger.debug(
u'No output file because mapping script explicitly indicated this.'
)
tomessage.ta_info['filename'] = ''
tomessage.ta_info['status'] = DISCARD
else:
botsglobal.logger.debug(
u'Start writing output file editype "%s" messagetype "%s".',
tomessage.ta_info['editype'],
tomessage.ta_info['messagetype'])
tomessage.writeall(
) #write tomessage (result of translation).
#problem is that not all values ta_tomes are know to to_message....
#~ print 'tomessage.ta_info',tomessage.ta_info
ta_tomes.update(
**tomessage.ta_info
) #update outmessage transaction with ta_info;
#check the value received from the mappingscript to see if another traanslation needs to be done (chained translation)
if doalttranslation is None:
del tomessage
break #break out of while loop; do no other translation
elif isinstance(doalttranslation, dict):
#some extended cases; a dict is returned that contains 'instructions'
if 'type' not in doalttranslation:
raise botslib.BotsError(
"Mapping script returned dict. This dict does not have a 'type', like in eg: {'type:'out_as_inn', 'alt':'alt-value'}."
)
if doalttranslation['type'] == u'out_as_inn':
if 'alt' not in doalttranslation:
raise botslib.BotsError(
"Mapping script returned dict, type 'out_as_inn'. This dict does not have a 'alt'-value, like in eg: {'type:'out_as_inn', 'alt':'alt-value'}."
)
inn = tomessage
if isinstance(inn, outmessage.fixed):
inn.root.stripnode()
inn.ta_info['alt'] = doalttranslation[
'alt'] #get the alt-value for the next chainded translation
inn.ta_info.pop('statust')
else:
del tomessage
inn.ta_info[
'alt'] = doalttranslation #get the alt-value for the next chainded translation
#end of while-loop
#
#~ print inn.ta_info
ta_frommes.update(
statust=DONE, **inn.ta_info
) #update db. inn.ta_info could be changed by script. Is this useful?
del inn
#exceptions file_in-level
except:
#~ edifile.handleconfirm(ta_fromfile,error=True) #only useful if errors are reported in acknowledgement (eg x12 997). Not used now.
txt = botslib.txtexc()
ta_parsedfile.failure()
ta_parsedfile.update(statust=ERROR, errortext=txt)
botsglobal.logger.debug(
u'error in translating input file "%s":\n%s', row['filename'],
txt)
else:
edifile.handleconfirm(ta_fromfile, error=False)
ta_fromfile.update(statust=DONE)
ta_parsedfile.update(statust=DONE, **edifile.confirminfo)
botsglobal.logger.debug(u'translated input file "%s".',
row['filename'])
del edifile
def start():
#exit codes:
# 0: OK, no errors
# 1: (system) errors
# 2: bots ran OK, but there are errors/process errors in the run
# 3: Database is locked, but "maxruntime" has not been exceeded.
#********command line arguments**************************
commandspossible = ['--new','--retry','--retransmit','--cleanup','--crashrecovery','--retrycommunication','--automaticretrycommunication']
commandstorun = []
routestorun = [] #list with routes to run
configdir = 'config'
for arg in sys.argv[1:]:
if not arg:
continue
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Configuration directory indicated, but no directory name.'
sys.exit(1)
elif arg in commandspossible:
commandstorun.append(arg)
elif arg in ["?", "/?"] or arg.startswith('-'):
showusage()
sys.exit(0)
else: #pick up names of routes to run
routestorun.append(arg)
if not commandstorun: #if no command on command line, use new (default)
commandstorun = ['--new']
#**************init general: find locating of bots, configfiles, init paths etc.****************
botsinit.generalinit(configdir)
#set current working directory to botspath
#~ old_current_directory = os.getcwdu()
os.chdir(botsglobal.ini.get('directories','botspath'))
#**************initialise logging******************************
try:
botsinit.initenginelogging()
except:
print _('Error in initialising logging system.')
traceback.print_exc()
sys.exit(1)
else:
atexit.register(logging.shutdown)
for key,value in botslib.botsinfo(): #log start info
botsglobal.logger.info(u'%s: "%s".',key,value)
#**************connect to database**********************************
try:
botsinit.connect()
except:
botsglobal.logger.exception(_(u'Could not connect to database. Database settings are in bots/config/settings.py.'))
sys.exit(1)
else:
botsglobal.logger.info(_(u'Connected to database.'))
atexit.register(botsglobal.db.close)
#initialise user exits for the whole bots-engine (this script file)
try:
userscript,scriptname = botslib.botsimport('routescripts','botsengine')
except ImportError:
userscript = scriptname = None
#**************handle database lock****************************************
#try to set a lock on the database; if this is not possible, the database is already locked. Either:
#1 another instance bots bots-engine is (still) running
#2 or bots-engine had a severe crash.
#What to do?
#first: check ts of database lock. If below a certain value (set in bots.ini) we assume an other instance is running. Exit quietly - no errors, no logging.
# else: Warn user, give advise on what to do. gather data: nr files in, errors.
#next: warn with report & logging. advise a crashrecovery.
if not botslib.set_database_lock():
if '--crashrecovery' in commandstorun: #user starts recovery operation; the databaselock is ignored; the databaselock is unlocked when routes have run.
commandstorun = ['--crashrecovery'] #is an exclusive option!
else:
#when scheduling bots it is possible that the last run is still running. Check if maxruntime has passed:
vanaf = datetime.datetime.today() - datetime.timedelta(minutes=botsglobal.ini.getint('settings','maxruntime',60))
for row in botslib.query('''SELECT ts FROM mutex WHERE ts < %(vanaf)s ''',{'vanaf':vanaf}):
warn = _(u'!Bots database is locked!\nBots-engine has ended in an unexpected way during the last run.\nThis happens, but is very very rare.\nPossible causes: bots-engine terminated by user, system crash, power-down, etc.\nA forced retry of the last run is advised; bots will (try to) repair the last run.')
botsglobal.logger.critical(warn)
botslib.sendbotserrorreport(_(u'[Bots severe error]Database is locked'),warn)
#add: count errors etc.
sys.exit(1)
else: #maxruntime has not passed. Exit silently, nothing reported
botsglobal.logger.info(_(u'Database is locked, but "maxruntime" has not been exceeded.'))
sys.exit(3)
else:
if '--crashrecovery' in commandstorun: #user starts recovery operation but there is no databaselock.
warn = _(u'User started a forced retry of the last run.\nOnly use this when the database is locked.\nThe database was not locked (database is OK).\nSo Bots has done nothing now.')
botsglobal.logger.error(warn)
botslib.sendbotserrorreport(_(u'[Bots Error Report] User started a forced retry of last run, but this was not needed'),warn)
botslib.remove_database_lock()
sys.exit(1)
#*************get list of routes to run****************************************
#~ raise Exception('locked database') #for testing database lock: abort, database will be locked
if routestorun:
botsglobal.logger.info(u'Run routes from command line: "%s".',str(routestorun))
else: # no routes from command line parameters: fetch all active routes from database
for row in botslib.query('''SELECT DISTINCT idroute
FROM routes
WHERE active=%(active)s
AND (notindefaultrun=%(notindefaultrun)s OR notindefaultrun IS NULL)
ORDER BY idroute ''',
{'active':True,'notindefaultrun':False}):
routestorun.append(row['idroute'])
botsglobal.logger.info(_(u'Run active routes from database: "%s".'),str(routestorun))
#routestorun is now either a list with routes from commandline, or the list of active routes for the routes table in the db.
#**************run the routes for retry, retransmit and new runs*************************************
try:
#commandstorun determines the type(s) of run
#routes to run is a listof the routes that are runs (for each command to run
#botsglobal.incommunicate is used to control if there is communication in; only 'new' incommunicates.
#botsglobal.minta4query controls which ta's are queried by the routes.
#stuff2evaluate controls what is evaluated in automatic maintenance.
errorinrun = 0 #detect if there has been some error. Only used for good exit() code
botsglobal.incommunicate = False
if '--crashrecovery' in commandstorun:
botsglobal.logger.info(_(u'Run crash recovery.'))
stuff2evaluate = botslib.set_minta4query_crashrecovery()
if stuff2evaluate:
router.routedispatcher(routestorun)
errorinrun += automaticmaintenance.evaluate('--crashrecovery',stuff2evaluate)
else:
botsglobal.logger.info(_(u'No retry of the last run - there was no last run.'))
if userscript and hasattr(userscript,'postcrashrecovery'):
botslib.runscript(userscript,scriptname,'postcrashrecovery',routestorun=routestorun)
if '--retrycommunication' in commandstorun:
botsglobal.logger.info(_(u'Run communication retry.'))
stuff2evaluate = router.routedispatcher(routestorun,'--retrycommunication')
if stuff2evaluate:
errorinrun += automaticmaintenance.evaluate('--retrycommunication',stuff2evaluate)
else:
botsglobal.logger.info(_(u'Run recommunicate: nothing to recommunicate.'))
if userscript and hasattr(userscript,'postretrycommunication'):
botslib.runscript(userscript,scriptname,'postretrycommunication',routestorun=routestorun)
if '--automaticretrycommunication' in commandstorun:
botsglobal.logger.info(_(u'Run automatic communication retry.'))
stuff2evaluate = router.routedispatcher(routestorun,'--automaticretrycommunication')
if stuff2evaluate:
errorinrun += automaticmaintenance.evaluate('--automaticretrycommunication',stuff2evaluate)
else:
botsglobal.logger.info(_(u'Run automatic recommunicate: nothing to recommunicate.'))
if userscript and hasattr(userscript,'postautomaticretrycommunication'):
botslib.runscript(userscript,scriptname,'postautomaticretrycommunication',routestorun=routestorun)
if '--retry' in commandstorun:
botsglobal.logger.info(u'Run retry.')
stuff2evaluate = router.routedispatcher(routestorun,'--retry')
if stuff2evaluate:
errorinrun += automaticmaintenance.evaluate('--retry',stuff2evaluate)
else:
botsglobal.logger.info(_(u'Run retry: nothing to retry.'))
if userscript and hasattr(userscript,'postretry'):
botslib.runscript(userscript,scriptname,'postretry',routestorun=routestorun)
if '--retransmit' in commandstorun:
botsglobal.logger.info(u'Run retransmit.')
stuff2evaluate = router.routedispatcher(routestorun,'--retransmit')
if stuff2evaluate:
errorinrun += automaticmaintenance.evaluate('--retransmit',stuff2evaluate)
else:
botsglobal.logger.info(_(u'Run retransmit: nothing to retransmit.'))
if userscript and hasattr(userscript,'postretransmit'):
botslib.runscript(userscript,scriptname,'postretransmit',routestorun=routestorun)
if '--new' in commandstorun:
botsglobal.logger.info('Run new.')
botsglobal.incommunicate = True
botsglobal.minta4query = 0 #meaning: reset. the actual value is set later (in routedispatcher)
stuff2evaluate = router.routedispatcher(routestorun)
errorinrun += automaticmaintenance.evaluate('--new',stuff2evaluate)
if userscript and hasattr(userscript,'postnewrun'):
botslib.runscript(userscript,scriptname,'postnewrun',routestorun=routestorun)
if '--cleanup' in commandstorun or botsglobal.ini.get('settings','whencleanup','always')=='always':
botsglobal.logger.debug(u'Do cleanup.')
cleanup.cleanup()
botslib.remove_database_lock()
except Exception,e:
botsglobal.logger.exception(_(u'Severe error in bots system:\n%s')%(e)) #of course this 'should' not happen.
sys.exit(1)
'Bots-engine has ended in an unexpected way during the last run.\n'\
'Most likely causes: sudden power-down, system crash, problems with disk I/O, bots-engine terminated by user, etc.\n'
'Bots will do an automatic crash recovery now.')
botsglobal.logger.critical(warn)
botslib.sendbotserrorreport(_(u'[Bots severe error]Database is locked'),warn)
commandstorun.insert(0,'crashrecovery') #there is a database lock. Add a crashrecovery as first command to run.
atexit.register(botslib.remove_database_lock)
#**************run the routes**********************************************
#commandstorun determines the type(s) of run. eg: ['automaticretrycommunication','new']
#for each command: run all routes
# for each route: run all seq
try:
#in acceptance tests: run a user script before running eg to clean output directories******************************
if acceptance_userscript:
if hasattr(acceptance_userscript,'pretest'):
botslib.runscript(acceptance_userscript,acceptance_scriptname,'pretest',routestorun=routestorun)
botslib.prepare_confirmrules()
errorinrun = 0 #detect if there has been some error. Only used for correct exit() code
for command in commandstorun:
botsglobal.logger.info(_(u'Run "%(command)s".'),{'command':command})
#get list of routes to run
if routestorun:
use_routestorun = routestorun[:]
botsglobal.logger.info(_(u'Run routes from command line: "%(routes)s".'),{'routes':str(use_routestorun)})
elif command == 'new': #fetch all active routes from database unless 'not in default run' or not active.
use_routestorun = []
for row in botslib.query('''SELECT DISTINCT idroute
FROM routes
WHERE active=%(active)s
AND (notindefaultrun=%(notindefaultrun)s OR notindefaultrun IS NULL)
def _translate_one_file(row, routedict, endstatus, userscript, scriptname):
''' - read, lex, parse, make tree of nodes.
- split up files into messages (using 'nextmessage' of grammar)
- get mappingscript, start mappingscript.
- write the results of translation (no enveloping yet)
'''
try:
ta_fromfile = botslib.OldTransaction(row['idta'])
ta_parsed = ta_fromfile.copyta(status=PARSED)
if row['filesize'] > botsglobal.ini.getint(
'settings', 'maxfilesizeincoming', 5000000):
ta_parsed.update(filesize=row['filesize'])
raise botslib.FileTooLargeError(
_(u'File size of %(filesize)s is too big; option "maxfilesizeincoming" in bots.ini is %(maxfilesizeincoming)s.'
), {
'filesize':
row['filesize'],
'maxfilesizeincoming':
botsglobal.ini.getint('settings', 'maxfilesizeincoming',
5000000)
})
botsglobal.logger.debug(
_(u'Start translating file "%(filename)s" editype "%(editype)s" messagetype "%(messagetype)s".'
), row)
#read whole edi-file: read, parse and made into a inmessage-object. Message is represented as a tree (inmessage.root is the root of the tree).
edifile = inmessage.parse_edi_file(frompartner=row['frompartner'],
topartner=row['topartner'],
filename=row['filename'],
messagetype=row['messagetype'],
testindicator=row['testindicator'],
editype=row['editype'],
charset=row['charset'],
alt=row['alt'],
fromchannel=row['fromchannel'],
idroute=routedict['idroute'],
command=routedict['command'])
edifile.checkforerrorlist(
) #no exception if infile has been lexed and parsed OK else raises an error
if int(
routedict['translateind']
) == 3: #parse & passthrough; file is parsed, partners are known, no mapping, does confirm.
raise botslib.GotoException('dummy')
#edifile.ta_info contains info: QUERIES, charset etc
for inn_splitup in edifile.nextmessage(
): #splitup messages in parsed edifile
try:
ta_splitup = ta_parsed.copyta(
status=SPLITUP,
**inn_splitup.ta_info) #copy PARSED to SPLITUP ta
#inn_splitup.ta_info: parameters from inmessage.parse_edi_file(), syntax-information and parse-information
inn_splitup.ta_info[
'idta_fromfile'] = ta_fromfile.idta #for confirmations in userscript; used to give idta of 'confirming message'
inn_splitup.ta_info[
'idta'] = ta_splitup.idta #for confirmations in userscript; used to give idta of 'confirming message'
number_of_loops_with_same_alt = 0
while 1: #continue as long as there are (alt-)translations
#lookup the translation************************
tscript, toeditype, tomessagetype = botslib.lookup_translation(
fromeditype=inn_splitup.ta_info['editype'],
frommessagetype=inn_splitup.ta_info['messagetype'],
frompartner=inn_splitup.ta_info['frompartner'],
topartner=inn_splitup.ta_info['topartner'],
alt=inn_splitup.ta_info['alt'])
if not tscript: #no translation found in translate table; check if can find translation via user script
if userscript and hasattr(userscript,
'gettranslation'):
tscript, toeditype, tomessagetype = botslib.runscript(
userscript,
scriptname,
'gettranslation',
idroute=routedict['idroute'],
message=inn_splitup)
if not tscript:
raise botslib.TranslationNotFoundError(
_(u'Translation not found for editype "%(editype)s", messagetype "%(messagetype)s", frompartner "%(frompartner)s", topartner "%(topartner)s", alt "%(alt)s".'
), inn_splitup.ta_info)
inn_splitup.ta_info[
'divtext'] = tscript #ifor reporting used mapping script to database (for display in GUI).
#initialize new out-object*************************
ta_translated = ta_splitup.copyta(
status=endstatus
) #make ta for translated message (new out-ta)
filename_translated = unicode(ta_translated.idta)
out_translated = outmessage.outmessage_init(
editype=toeditype,
messagetype=tomessagetype,
filename=filename_translated,
reference=unique('messagecounter'),
statust=OK,
divtext=tscript) #make outmessage object
#run mapping script************************
botsglobal.logger.debug(
_(u'Mappingscript "%(tscript)s" translates messagetype "%(messagetype)s" to messagetype "%(tomessagetype)s".'
),
{
'tscript': tscript,
'messagetype': inn_splitup.ta_info['messagetype'],
'tomessagetype':
out_translated.ta_info['messagetype']
})
translationscript, scriptfilename = botslib.botsimport(
'mappings', inn_splitup.ta_info['editype'],
tscript) #get the mappingscript
alt_from_previous_run = inn_splitup.ta_info[
'alt'] #needed to check for infinite loop
doalttranslation = botslib.runscript(translationscript,
scriptfilename,
'main',
inn=inn_splitup,
out=out_translated)
botsglobal.logger.debug(
_(u'Mappingscript "%(tscript)s" finished.'),
{'tscript': tscript})
#manipulate for some attributes after mapping script
if 'topartner' not in out_translated.ta_info: #out_translated does not contain values from ta......
out_translated.ta_info[
'topartner'] = inn_splitup.ta_info['topartner']
if 'botskey' in inn_splitup.ta_info:
inn_splitup.ta_info['reference'] = inn_splitup.ta_info[
'botskey']
if 'botskey' in out_translated.ta_info: #out_translated does not contain values from ta......
out_translated.ta_info[
'reference'] = out_translated.ta_info['botskey']
#check the value received from the mappingscript to determine what to do in this while-loop. Handling of chained trasnlations.
if doalttranslation is None:
#translation(s) are done; handle out-message
handle_out_message(out_translated, ta_translated)
break #break out of while loop
elif isinstance(doalttranslation, dict):
#some extended cases; a dict is returned that contains 'instructions' for some type of chained translations
if 'type' not in doalttranslation or 'alt' not in doalttranslation:
raise botslib.BotsError(
_(u"Mappingscript returned '%(alt)s'. This dict should not have 'type' and 'alt'."
), {'alt': doalttranslation})
if alt_from_previous_run == doalttranslation['alt']:
number_of_loops_with_same_alt += 1
else:
number_of_loops_with_same_alt = 0
if doalttranslation['type'] == u'out_as_inn':
#do chained translation: use the out-object as inn-object, new out-object
#use case: detected error in incoming file; use out-object to generate warning email
handle_out_message(out_translated, ta_translated)
inn_splitup = out_translated #out-object is now inn-object
if isinstance(
inn_splitup, outmessage.fixed
): #for fixed: strip all values in node
inn_splitup.root.stripnode()
inn_splitup.ta_info['alt'] = doalttranslation[
'alt'] #get the alt-value for the next chained translation
if not 'frompartner' in inn_splitup.ta_info:
inn_splitup.ta_info['frompartner'] = ''
if not 'topartner' in inn_splitup.ta_info:
inn_splitup.ta_info['topartner'] = ''
inn_splitup.ta_info.pop('statust')
elif doalttranslation[
'type'] == u'no_check_on_infinite_loop':
#do chained translation: allow many loops wit hsame alt-value.
#mapping script will have to handle this correctly.
number_of_loops_with_same_alt = 0
handle_out_message(out_translated, ta_translated)
inn_splitup.ta_info['alt'] = doalttranslation[
'alt'] #get the alt-value for the next chained translation
else: #there is nothing else
raise botslib.BotsError(
_(u'Mappingscript returned dict with an unknown "type": "%(doalttranslation)s".'
), {'doalttranslation': doalttranslation})
else: #note: this includes alt '' (empty string)
if alt_from_previous_run == doalttranslation:
number_of_loops_with_same_alt += 1
else:
number_of_loops_with_same_alt = 0
#do normal chained translation: same inn-object, new out-object
handle_out_message(out_translated, ta_translated)
inn_splitup.ta_info[
'alt'] = doalttranslation #get the alt-value for the next chained translation
if number_of_loops_with_same_alt > 10:
raise botslib.BotsError(
_(u'Mappingscript returns same alt value over and over again (infinite loop?). Alt: "%(doalttranslation)s".'
), {'doalttranslation': doalttranslation})
#end of while-loop (trans**********************************************************************************
#exceptions file_out-level: exception in mappingscript or writing of out-file
except:
#2 modes: either every error leads to skipping of whole infile (old mode) or errors in mappingscript/outfile only affect that branche
if botsglobal.ini.getboolean('settings', 'oldmessageerrors',
False):
raise
txt = botslib.txtexc()
ta_splitup.update(
statust=ERROR, errortext=txt, **inn_splitup.ta_info
) #update db. inn_splitup.ta_info could be changed by mappingscript. Is this useful?
ta_splitup.deletechildren()
else:
ta_splitup.update(
statust=DONE, **inn_splitup.ta_info
) #update db. inn_splitup.ta_info could be changed by mappingscript. Is this useful?
#exceptions file_in-level
except botslib.GotoException: #edi-file is OK, file is passed-through after parsing.
ta_parsed.update(statust=DONE,
filesize=row['filesize'],
**edifile.ta_info) #update with info from eg queries
ta_parsed.copyta(status=MERGED,
statust=OK) #original file goes straight to MERGED
edifile.handleconfirm(ta_fromfile, error=False)
botsglobal.logger.debug(
_(u'Parse & passthrough for input file "%(filename)s".'), row)
except botslib.FileTooLargeError as msg:
ta_parsed.update(statust=ERROR, errortext=unicode(msg))
ta_parsed.deletechildren()
botsglobal.logger.debug(
u'Error in translating input file "%(filename)s":\n%(msg)s', {
'filename': row['filename'],
'msg': msg
})
except:
txt = botslib.txtexc()
ta_parsed.update(statust=ERROR, errortext=txt, **edifile.ta_info)
ta_parsed.deletechildren()
edifile.handleconfirm(ta_fromfile, error=True)
botsglobal.logger.debug(
u'Error in translating input file "%(filename)s":\n%(msg)s', {
'filename': row['filename'],
'msg': txt
})
else:
edifile.handleconfirm(ta_fromfile, error=False)
ta_parsed.update(statust=DONE,
filesize=row['filesize'],
**edifile.ta_info)
botsglobal.logger.debug(_(u'Translated input file "%(filename)s".'),
row)
finally:
ta_fromfile.update(statust=DONE)
def translate(startstatus=TRANSLATE,endstatus=TRANSLATED,idroute=''):
''' translates edifiles in one or more edimessages.
reads and parses edifiles that have to be translated.
tries to split files into messages (using 'nextmessage' of grammar); if no splitting: edifile is one message.
searches the right translation in translate-table;
runs the mapping-script for the translation;
Function takes db-ta with status=TRANSLATE->PARSED->SPLITUP->TRANSLATED
'''
#select edifiles to translate; fill ta-object
for row in botslib.query(u'''SELECT idta,frompartner,topartner,filename,messagetype,testindicator,editype,charset,alt,fromchannel
FROM ta
WHERE idta>%(rootidta)s
AND status=%(status)s
AND statust=%(statust)s
AND idroute=%(idroute)s
''',
{'status':startstatus,'statust':OK,'idroute':idroute,'rootidta':botslib.get_minta4query()}):
try:
ta_fromfile = botslib.OldTransaction(row['idta']) #TRANSLATE ta
ta_parsedfile = ta_fromfile.copyta(status=PARSED) #copy TRANSLATE to PARSED ta
#read whole edi-file: read, parse and made into a inmessage-object. Message is represented a a tree.
edifile = inmessage.edifromfile(frompartner=row['frompartner'],
topartner=row['topartner'],
filename=row['filename'],
messagetype=row['messagetype'],
testindicator=row['testindicator'],
editype=row['editype'],
charset=row['charset'],
alt=row['alt'],
fromchannel=row['fromchannel'],
idroute=idroute)
botsglobal.logger.debug(u'start read and parse input file "%s" editype "%s" messagetype "%s".',row['filename'],row['editype'],row['messagetype'])
for inn in edifile.nextmessage(): #for each message in the edifile:
#inn.ta_info: parameters from inmessage.edifromfile(), syntax-information and parse-information
ta_frommes = ta_parsedfile.copyta(status=SPLITUP) #copy PARSED to SPLITUP ta
inn.ta_info['idta_fromfile'] = ta_fromfile.idta #for confirmations in user script; used to give idta of 'confirming message'
ta_frommes.update(**inn.ta_info) #update ta-record SLIPTUP with info from message content and/or grammar
while 1: #whileloop continues as long as there are alt-translations
#************select parameters for translation(script):
for row2 in botslib.query(u'''SELECT tscript,tomessagetype,toeditype
FROM translate
WHERE frommessagetype = %(frommessagetype)s
AND fromeditype = %(fromeditype)s
AND active=%(booll)s
AND alt=%(alt)s
AND (frompartner_id IS NULL OR frompartner_id=%(frompartner)s OR frompartner_id in (SELECT to_partner_id
FROM partnergroup
WHERE from_partner_id=%(frompartner)s ))
AND (topartner_id IS NULL OR topartner_id=%(topartner)s OR topartner_id in (SELECT to_partner_id
FROM partnergroup
WHERE from_partner_id=%(topartner)s ))
ORDER BY alt DESC,
CASE WHEN frompartner_id IS NULL THEN 1 ELSE 0 END, frompartner_id ,
CASE WHEN topartner_id IS NULL THEN 1 ELSE 0 END, topartner_id ''',
{'frommessagetype':inn.ta_info['messagetype'],
'fromeditype':inn.ta_info['editype'],
'alt':inn.ta_info['alt'],
'frompartner':inn.ta_info['frompartner'],
'topartner':inn.ta_info['topartner'],
'booll':True}):
break #escape if found; we need only the first - ORDER BY in the query
else: #no translation record is found
raise botslib.TranslationNotFoundError(_(u'Editype "$editype", messagetype "$messagetype", frompartner "$frompartner", topartner "$topartner", alt "$alt"'),
editype=inn.ta_info['editype'],
messagetype=inn.ta_info['messagetype'],
frompartner=inn.ta_info['frompartner'],
topartner=inn.ta_info['topartner'],
alt=inn.ta_info['alt'])
ta_tomes = ta_frommes.copyta(status=endstatus) #copy SPLITUP to TRANSLATED ta
tofilename = str(ta_tomes.idta)
tscript = row2['tscript']
tomessage = outmessage.outmessage_init(messagetype=row2['tomessagetype'],editype=row2['toeditype'],filename=tofilename,reference=unique('messagecounter'),statust=OK,divtext=tscript) #make outmessage object
#copy ta_info
botsglobal.logger.debug(u'script "%s" translates messagetype "%s" to messagetype "%s".',tscript,inn.ta_info['messagetype'],tomessage.ta_info['messagetype'])
translationscript,scriptfilename = botslib.botsimport('mappings',inn.ta_info['editype'] + '.' + tscript) #get the mapping-script
doalttranslation = botslib.runscript(translationscript,scriptfilename,'main',inn=inn,out=tomessage)
botsglobal.logger.debug(u'script "%s" finished.',tscript)
if 'topartner' not in tomessage.ta_info: #tomessage does not contain values from ta......
tomessage.ta_info['topartner'] = inn.ta_info['topartner']
if tomessage.ta_info['statust'] == DONE: #if indicated in user script the message should be discarded
botsglobal.logger.debug(u'No output file because mapping script explicitly indicated this.')
tomessage.ta_info['filename'] = ''
tomessage.ta_info['status'] = DISCARD
else:
botsglobal.logger.debug(u'Start writing output file editype "%s" messagetype "%s".',tomessage.ta_info['editype'],tomessage.ta_info['messagetype'])
tomessage.writeall() #write tomessage (result of translation).
#problem is that not all values ta_tomes are know to to_message....
#~ print 'tomessage.ta_info',tomessage.ta_info
ta_tomes.update(**tomessage.ta_info) #update outmessage transaction with ta_info;
#check the value received from the mappingscript to see if another traanslation needs to be done (chained translation)
if doalttranslation is None:
del tomessage
break #break out of while loop; do no other translation
elif isinstance(doalttranslation,dict):
#some extended cases; a dict is returned that contains 'instructions'
if 'type' not in doalttranslation:
raise botslib.BotsError("Mapping script returned dict. This dict does not have a 'type', like in eg: {'type:'out_as_inn', 'alt':'alt-value'}.")
if doalttranslation['type'] == u'out_as_inn':
if 'alt' not in doalttranslation:
raise botslib.BotsError("Mapping script returned dict, type 'out_as_inn'. This dict does not have a 'alt'-value, like in eg: {'type:'out_as_inn', 'alt':'alt-value'}.")
inn = tomessage
if isinstance(inn,outmessage.fixed):
inn.root.stripnode()
inn.ta_info['alt'] = doalttranslation['alt'] #get the alt-value for the next chainded translation
inn.ta_info.pop('statust')
else:
del tomessage
inn.ta_info['alt'] = doalttranslation #get the alt-value for the next chainded translation
#end of while-loop
#
#~ print inn.ta_info
ta_frommes.update(statust=DONE,**inn.ta_info) #update db. inn.ta_info could be changed by script. Is this useful?
del inn
#exceptions file_in-level
except:
#~ edifile.handleconfirm(ta_fromfile,error=True) #only useful if errors are reported in acknowledgement (eg x12 997). Not used now.
txt = botslib.txtexc()
ta_parsedfile.failure()
ta_parsedfile.update(statust=ERROR,errortext=txt)
botsglobal.logger.debug(u'error in translating input file "%s":\n%s',row['filename'],txt)
else:
edifile.handleconfirm(ta_fromfile,error=False)
ta_fromfile.update(statust=DONE)
ta_parsedfile.update(statust=DONE,**edifile.confirminfo)
botsglobal.logger.debug(u'translated input file "%s".',row['filename'])
del edifile
def translate(run):
for messagedict in run.incoming:
try:
#read whole edi-file: read, parse and made into a inmessage-object. Message is represented as a tree (inmessage.root is the root of the tree).
edifile = inmessage.parse_edi_file(
frompartner='',
topartner='',
filename=messagedict['filename'],
messagetype=run.translation['messagetype'],
testindicator='',
editype=run.translation['editype'],
charset='',
alt='',
fromchannel='',
idroute='',
command='')
edifile.checkforerrorlist(
) #no exception if infile has been lexed and parsed OK else raises an error
#~ if int(routedict['translateind']) == 3: #parse & passthrough; file is parsed, partners are known, no mapping, does confirm.
#~ raise botslib.GotoException('dummy')
#~ continue #file is parsed; no errors. no translation
#edifile.ta_info contains info about incoming file: QUERIES, charset etc
for inn_splitup in edifile.nextmessage(
): #splitup messages in parsed edifile
try:
#inn_splitup.ta_info: parameters from inmessage.parse_edi_file(), syntax-information and parse-information
number_of_loops_with_same_alt = 0
while 1: #continue as long as there are (alt-)translations
#lookup the translation************************
tscript, toeditype, tomessagetype = 'orders_edifact2xml', 'xml', 'orders'
if 'tscript' in run.translation:
tscript = run.translation['tscript']
toeditype = run.translation['toeditype']
tomessagetype = run.translation['tomessagetype']
else:
tscript, toeditype, tomessagetype = botslib.lookup_translation(
fromeditype=inn_splitup.ta_info['editype'],
frommessagetype=inn_splitup.
ta_info['messagetype'],
frompartner=inn_splitup.ta_info['frompartner'],
topartner=inn_splitup.ta_info['topartner'],
alt=inn_splitup.ta_info['alt'])
#run mapping script************************
filename_translated = transform.unique(
'bots_file_name')
out_translated = outmessage.outmessage_init(
editype=toeditype,
messagetype=tomessagetype,
filename=filename_translated,
reference=transform.unique('messagecounter'),
statust=OK,
divtext=tscript) #make outmessage object
#~ botsglobal.logger.debug(_(u'Mappingscript "%(tscript)s" translates messagetype "%(messagetype)s" to messagetype "%(tomessagetype)s".'),
#~ {'tscript':tscript,'messagetype':inn_splitup.ta_info['messagetype'],'tomessagetype':out_translated.ta_info['messagetype']})
translationscript, scriptfilename = botslib.botsimport(
'mappings', inn_splitup.ta_info['editype'],
tscript) #import mappingscript
alt_from_previous_run = inn_splitup.ta_info[
'alt'] #needed to check for infinite loop
doalttranslation = botslib.runscript(
translationscript,
scriptfilename,
'main',
inn=inn_splitup,
out=out_translated)
botsglobal.logger.debug(
_(u'Mappingscript "%(tscript)s" finished.'),
{'tscript': tscript})
#manipulate for some attributes after mapping script
if 'topartner' not in out_translated.ta_info: #out_translated does not contain values from run......
out_translated.ta_info[
'topartner'] = inn_splitup.ta_info['topartner']
if 'botskey' in inn_splitup.ta_info:
inn_splitup.ta_info[
'reference'] = inn_splitup.ta_info['botskey']
if 'botskey' in out_translated.ta_info: #out_translated does not contain values from run......
out_translated.ta_info[
'reference'] = out_translated.ta_info[
'botskey']
#check the value received from the mappingscript to determine what to do in this while-loop. Handling of chained trasnlations.
if doalttranslation is None:
#translation(s) are done; handle out-message
out_translated.writeall(
) #write result of translation.
#make translated record (if all is OK)
translated_dict = inn_splitup.ta_info.copy()
translated_dict.update(messagedict)
translated_dict.update(out_translated.ta_info)
run.translated.append(translated_dict)
del out_translated
break #break out of while loop
elif isinstance(doalttranslation, dict):
#some extended cases; a dict is returned that contains 'instructions' for some type of chained translations
if 'type' not in doalttranslation or 'alt' not in doalttranslation:
raise botslib.BotsError(
_(u"Mappingscript returned '%(alt)s'. This dict should not have 'type' and 'alt'."
), {'alt': doalttranslation})
if alt_from_previous_run == doalttranslation[
'alt']:
number_of_loops_with_same_alt += 1
else:
number_of_loops_with_same_alt = 0
if doalttranslation['type'] == u'out_as_inn':
#do chained translation: use the out-object as inn-object, new out-object
#use case: detected error in incoming file; use out-object to generate warning email
handle_out_message(out_translated,
ta_translated)
inn_splitup = out_translated #out-object is now inn-object
if isinstance(
inn_splitup, outmessage.fixed
): #for fixed: strip all values in node
inn_splitup.root.stripnode()
inn_splitup.ta_info['alt'] = doalttranslation[
'alt'] #get the alt-value for the next chained translation
if not 'frompartner' in inn_splitup.ta_info:
inn_splitup.ta_info['frompartner'] = ''
if not 'topartner' in inn_splitup.ta_info:
inn_splitup.ta_info['topartner'] = ''
inn_splitup.ta_info.pop('statust')
elif doalttranslation[
'type'] == u'no_check_on_infinite_loop':
#do chained translation: allow many loops wit hsame alt-value.
#mapping script will have to handle this correctly.
number_of_loops_with_same_alt = 0
handle_out_message(out_translated,
ta_translated)
del out_translated
inn_splitup.ta_info['alt'] = doalttranslation[
'alt'] #get the alt-value for the next chained translation
else: #there is nothing else
raise botslib.BotsError(
_(u'Mappingscript returned dict with an unknown "type": "%(doalttranslation)s".'
),
{'doalttranslation': doalttranslation})
else: #note: this includes alt '' (empty string)
if alt_from_previous_run == doalttranslation:
number_of_loops_with_same_alt += 1
else:
number_of_loops_with_same_alt = 0
#do normal chained translation: same inn-object, new out-object
out_translated.writeall()
del out_translated
inn_splitup.ta_info[
'alt'] = doalttranslation #get the alt-value for the next chained translation
if number_of_loops_with_same_alt > 10:
raise botslib.BotsError(
_(u'Mappingscript returns same alt value over and over again (infinite loop?). Alt: "%(doalttranslation)s".'
), {'doalttranslation': doalttranslation})
#end of while-loop (trans**********************************************************************************
#exceptions file_out-level: exception in mappingscript or writing of out-file
except:
#2 modes: either every error leads to skipping of whole infile (old mode) or errors in mappingscript/outfile only affect that branche
txt = botslib.txtexc()
print txt
messagedict['error'] += txt.strip()
else:
pass
#~ print 'succes'
#exceptions file_in-level
except botslib.GotoException: #edi-file is OK, file is passed-through after parsing.
#~ edifile.handleconfirm(ta_fromfile,error=False)
#~ botsglobal.logger.debug(_(u'Parse & passthrough for input file "%(filename)s".'),row)
txt = botslib.txtexc()
print txt
except:
txt = botslib.txtexc()
messagedict['error'] += txt.strip()
#~ edifile.handleconfirm(ta_fromfile,error=True)
#~ botsglobal.logger.debug(u'Error in translating input file "%(filename)s":\n%(msg)s',{'filename':row['filename'],'msg':txt})
else:
pass
