def preprocess(routedict, function, status=FILEIN, **argv):
''' for pre- and postprocessing of files.
these are NOT translations; translation involve grammars, mapping scripts etc. think of eg:
- unzipping zipped files.
- convert excel to csv
- password protected files.
Select files from INFILE -> SET_FOR_PROCESSING using criteria
Than the actual processing function is called.
The processing function does: SET_FOR_PROCESSING -> PROCESSING -> FILEIN
If errors occur during processing, no ta are left with status FILEIN !
preprocess is called right after the in-communicatiation
'''
nr_files = 0
preprocessnumber = botslib.getpreprocessnumber()
if not botslib.addinfo(change={'status': preprocessnumber},
where={
'status': status,
'idroute': routedict['idroute'],
'fromchannel': routedict['fromchannel']
}): #check if there is something to do
return 0
for row in botslib.query(
u'''SELECT idta,filename,charset
FROM ta
WHERE idta>%(rootidta)s
AND status=%(status)s
AND statust=%(statust)s
AND idroute=%(idroute)s
AND fromchannel=%(fromchannel)s
''', {
'status': preprocessnumber,
'statust': OK,
'idroute': routedict['idroute'],
'fromchannel': routedict['fromchannel'],
'rootidta': botslib.get_minta4query()
}):
try:
botsglobal.logmap.debug(u'Start preprocessing "%s" for file "%s".',
function.__name__, row['filename'])
ta_set_for_processing = botslib.OldTransaction(row['idta'])
ta_processing = ta_set_for_processing.copyta(
status=preprocessnumber + 1)
ta_processing.filename = row['filename']
function(ta_from=ta_processing,
endstatus=status,
routedict=routedict,
**argv)
except:
txt = botslib.txtexc()
ta_processing.failure()
ta_processing.update(statust=ERROR, errortext=txt)
else:
botsglobal.logmap.debug(u'OK preprocessing "%s" for file "%s".',
function.__name__, row['filename'])
ta_set_for_processing.update(statust=DONE)
ta_processing.update(statust=DONE)
nr_files += 1
return nr_files
def preprocess(routedict,function, status=FILEIN,**argv):
''' for pre- and postprocessing of files.
these are NOT translations; translation involve grammars, mapping scripts etc. think of eg:
- unzipping zipped files.
- convert excel to csv
- password protected files.
Select files from INFILE -> SET_FOR_PROCESSING using criteria
Than the actual processing function is called.
The processing function does: SET_FOR_PROCESSING -> PROCESSING -> FILEIN
If errors occur during processing, no ta are left with status FILEIN !
preprocess is called right after the in-communicatiation
'''
nr_files = 0
preprocessnumber = botslib.getpreprocessnumber()
if not botslib.addinfo(change={'status':preprocessnumber},where={'status':status,'idroute':routedict['idroute'],'fromchannel':routedict['fromchannel']}): #check if there is something to do
return 0
for row in botslib.query(u'''SELECT idta,filename,charset
FROM ta
WHERE idta>%(rootidta)s
AND status=%(status)s
AND statust=%(statust)s
AND idroute=%(idroute)s
AND fromchannel=%(fromchannel)s
''',
{'status':preprocessnumber,'statust':OK,'idroute':routedict['idroute'],'fromchannel':routedict['fromchannel'],'rootidta':botslib.get_minta4query()}):
try:
botsglobal.logmap.debug(u'Start preprocessing "%s" for file "%s".',function.__name__,row['filename'])
ta_set_for_processing = botslib.OldTransaction(row['idta'])
ta_processing = ta_set_for_processing.copyta(status=preprocessnumber+1)
ta_processing.filename = row['filename']
function(ta_from=ta_processing,endstatus=status,routedict=routedict,**argv)
except:
txt = botslib.txtexc()
ta_processing.failure()
ta_processing.update(statust=ERROR,errortext=txt)
else:
botsglobal.logmap.debug(u'OK preprocessing "%s" for file "%s".',function.__name__,row['filename'])
ta_set_for_processing.update(statust=DONE)
ta_processing.update(statust=DONE)
nr_files += 1
return nr_files
def routepart(self,routedict):
''' communication.run one route part. variants:
- a route can be a routescript
- a route can do only incoming
- a route can do only outgoing
- a route can do both incoming and outgoing
- at several points functions from a routescript are called - if function is in routescript
'''
self.minta4query_routepart = botslib._Transaction.processlist[-1] #the idta of routepart
#if routescript has function 'main': communication.run 'main' (and do nothing else)
if botslib.tryrunscript(self.userscript,self.scriptname,'main',routedict=routedict):
return #so: if function ' main' : communication.run only the routescript, nothing else.
if not (self.userscript or routedict['fromchannel'] or routedict['tochannel'] or routedict['translateind']):
raise botslib.ScriptError(_(u'Route "%(idroute)s" is empty: no routescript, not enough parameters.'),routedict)
botslib.tryrunscript(self.userscript,self.scriptname,'start',routedict=routedict)
#incoming part of route:
#- incommunication
#- assign attributes from route to incoming files
#- preprocessing
if routedict['fromchannel']:
#only done for edi files from this route-part, this inchannel
if routedict['command'] == 'rereceive':
rootidta = self.get_minta4query()
else:
rootidta = self.get_minta4query_routepart()
botslib.tryrunscript(self.userscript,self.scriptname,'preincommunication',routedict=routedict)
communication.run(idchannel=routedict['fromchannel'],command=routedict['command'],idroute=routedict['idroute'],rootidta=rootidta) #communication.run incommunication
#add attributes from route to the received files;
where = {'statust':OK,'status':FILEIN,'fromchannel':routedict['fromchannel'],'idroute':routedict['idroute'],'rootidta':rootidta}
change = {'editype':routedict['fromeditype'],'messagetype':routedict['frommessagetype'],'frompartner':routedict['frompartner'],'topartner':routedict['topartner'],'alt':routedict['alt']}
nr_of_incoming_files_for_channel = botslib.updateinfo(change=change,where=where)
botslib.tryrunscript(self.userscript,self.scriptname,'postincommunication',routedict=routedict)
if nr_of_incoming_files_for_channel:
#unzip incoming files (if indicated)
if routedict['zip_incoming'] == 1: #unzip incoming (non-zipped gives error).
preprocess.preprocess(routedict=routedict,function=preprocess.botsunzip,rootidta=self.get_minta4query_routepart(),pass_non_zip=False)
elif routedict['zip_incoming'] == 2: #unzip incoming if zipped.
preprocess.preprocess(routedict=routedict,function=preprocess.botsunzip,rootidta=self.get_minta4query_routepart(),pass_non_zip=True)
#run mailbag-module.
if botsglobal.ini.getboolean('settings','compatibility_mailbag',False):
editypes_via_mailbag = ['mailbag']
else:
editypes_via_mailbag = ['mailbag','edifact','x12','tradacoms']
if routedict['fromeditype'] in editypes_via_mailbag: #mailbag for the route.
preprocess.preprocess(routedict=routedict,function=preprocess.mailbag,rootidta=self.get_minta4query_routepart(),frommessagetype=routedict['frommessagetype'])
#translate, merge, pass through: INFILE->MERGED
if int(routedict['translateind']) in [1,3]:
#translate: for files in route
botslib.tryrunscript(self.userscript,self.scriptname,'pretranslation',routedict=routedict)
if routedict['command'] in ['rereceive',]:
rootidta = self.get_minta4query()
else:
rootidta = self.get_minta4query_route()
transform.translate(startstatus=FILEIN,endstatus=TRANSLATED,routedict=routedict,rootidta=rootidta)
botslib.tryrunscript(self.userscript,self.scriptname,'posttranslation',routedict=routedict)
#**merge: for files in this route-part (the translated files)
botslib.tryrunscript(self.userscript,self.scriptname,'premerge',routedict=routedict)
envelope.mergemessages(startstatus=TRANSLATED,endstatus=MERGED,idroute=routedict['idroute'],rootidta=self.get_minta4query_routepart())
botslib.tryrunscript(self.userscript,self.scriptname,'postmerge',routedict=routedict)
elif routedict['translateind'] == 2: #pass-through: pickup the incoming files and mark these as MERGED (==translation is finished)
botslib.addinfo(change={'status':MERGED,'statust':OK},where={'status':FILEIN,'statust':OK,'idroute':routedict['idroute'],'rootidta':self.get_minta4query_route()})
#NOTE: routedict['translateind'] == 0 than nothing will happen with the files in this route.
#ommunication outgoing channel: MERGED->RAWOUT
if routedict['tochannel']:
#**build query to add outchannel as attribute to outgoing files***
#filter files in route for outchannel
towhere = { 'status':MERGED,
'statust':OK,
'idroute':routedict['idroute'],
'editype':routedict['toeditype'],
'messagetype':routedict['tomessagetype'],
'testindicator':routedict['testindicator'],
}
towhere = dict((key, value) for key,value in towhere.iteritems() if value) #remove nul-values from dict
wherestring = ' AND '.join([key+'=%('+key+')s ' for key in towhere])
if routedict['frompartner_tochannel_id']: #use frompartner_tochannel in where-clause of query (partner/group dependent outchannel
towhere['frompartner_tochannel_id'] = routedict['frompartner_tochannel_id']
wherestring += ''' AND (frompartner=%(frompartner_tochannel_id)s
OR frompartner in (SELECT from_partner_id
FROM partnergroup
WHERE to_partner_id=%(frompartner_tochannel_id)s )) '''
if routedict['topartner_tochannel_id']: #use topartner_tochannel in where-clause of query (partner/group dependent outchannel
towhere['topartner_tochannel_id'] = routedict['topartner_tochannel_id']
wherestring += ''' AND (topartner=%(topartner_tochannel_id)s
OR topartner in (SELECT from_partner_id
FROM partnergroup
WHERE to_partner_id=%(topartner_tochannel_id)s )) '''
toset = {'status':FILEOUT,'statust':OK,'tochannel':routedict['tochannel']}
towhere['rootidta'] = self.get_minta4query_route()
nr_of_outgoing_files_for_channel = botslib.addinfocore(change=toset,where=towhere,wherestring=wherestring)
if nr_of_outgoing_files_for_channel:
#**set asked confirmation/acknowledgements
botslib.set_asked_confirmrules(routedict,rootidta=self.get_minta4query_routepart())
#**zip outgoing
#for files in this route-part for this out-channel
if routedict['zip_outgoing'] == 1:
preprocess.postprocess(routedict=routedict,function=preprocess.botszip,rootidta=self.get_minta4query_routepart())
#actual communication: run outgoing channel (if not deferred)
#for all files in run that are for this channel (including the deferred ones from other routes)
if not routedict['defer']:
#determine range of idta to query: if channel was not deferred earlier in run: query only for route part else query for whole run
if self.keep_track_if_outchannel_deferred.get(routedict['tochannel'],False) or routedict['command'] in ['resend','automaticretrycommunication']:
rootidta = self.get_minta4query()
else:
rootidta = self.get_minta4query_routepart()
if botslib.countoutfiles(idchannel=routedict['tochannel'],rootidta=rootidta):
botslib.tryrunscript(self.userscript,self.scriptname,'preoutcommunication',routedict=routedict)
communication.run(idchannel=routedict['tochannel'],command=routedict['command'],idroute=routedict['idroute'],rootidta=rootidta)
#in communication several things can go wrong.
#all outgoing files should have same status; that way all recomnnunication can be handled the same:
#- status EXTERNOUT statust DONE (if communication goes OK)
#- status EXTERNOUT status ERROR (if file is not communicatied)
#to have the same status for all outgoing files some manipulation is needed, eg in case no connection could be made.
botslib.addinfo(change={'status':EXTERNOUT,'statust':ERROR},where={'status':FILEOUT,'statust':OK,'tochannel':routedict['tochannel'],'rootidta':rootidta})
botslib.tryrunscript(self.userscript,self.scriptname,'postoutcommunication',routedict=routedict)
botslib.tryrunscript(self.userscript,self.scriptname,'end',routedict=routedict)
def router(routedict):
''' communication.run one route. variants:
- a route can be just script;
- a route can do only incoming
- a route can do only outgoing
- a route can do both incoming and outgoing
- at several points functions from a route script are called - if function is in route script
'''
#is there a user route script?
try:
userscript,scriptname = botslib.botsimport('routescripts',routedict['idroute'])
except ImportError: #other errors, eg syntax errors are just passed
userscript = scriptname = None
#if user route script has function 'main': communication.run 'main' (and do nothing else)
if botslib.tryrunscript(userscript,scriptname,'main',routedict=routedict):
return #so: if function ' main' : communication.run only the routescript, nothing else.
if not (userscript or routedict['fromchannel'] or routedict['tochannel'] or routedict['translateind']):
raise botslib.ScriptError(_(u'Route "$route" is empty: no script, not enough parameters.'),route=routedict['idroute'])
botslib.tryrunscript(userscript,scriptname,'start',routedict=routedict)
#communication.run incoming channel
if routedict['fromchannel']: #do incoming part of route: in-communication; set ready for translation; translate
botslib.tryrunscript(userscript,scriptname,'preincommunication',routedict=routedict)
communication.run(idchannel=routedict['fromchannel'],idroute=routedict['idroute']) #communication.run incommunication
#add attributes from route to the received files
where={'status':FILEIN,'fromchannel':routedict['fromchannel'],'idroute':routedict['idroute']}
change={'editype':routedict['fromeditype'],'messagetype':routedict['frommessagetype'],'frompartner':routedict['frompartner'],'topartner':routedict['topartner'],'alt':routedict['alt']}
botslib.updateinfo(change=change,where=where)
#all received files have status FILEIN
botslib.tryrunscript(userscript,scriptname,'postincommunication',routedict=routedict)
if routedict['fromeditype'] == 'mailbag': #mailbag for the route.
preprocess.preprocess(routedict,preprocess.mailbag)
#communication.run translation
if routedict['translateind']:
botslib.tryrunscript(userscript,scriptname,'pretranslation',routedict=routedict)
botslib.addinfo(change={'status':TRANSLATE},where={'status':FILEIN,'idroute':routedict['idroute']})
transform.translate(idroute=routedict['idroute'])
botslib.tryrunscript(userscript,scriptname,'posttranslation',routedict=routedict)
#merge messages & communication.run outgoing channel
if routedict['tochannel']: #do outgoing part of route
botslib.tryrunscript(userscript,scriptname,'premerge',routedict=routedict)
envelope.mergemessages(idroute=routedict['idroute'])
botslib.tryrunscript(userscript,scriptname,'postmerge',routedict=routedict)
#communication.run outgoing channel
#build for query: towhere (dict) and wherestring
towhere=dict(status=MERGED,
idroute=routedict['idroute'],
editype=routedict['toeditype'],
messagetype=routedict['tomessagetype'],
testindicator=routedict['testindicator'])
towhere=dict([(key, value) for (key, value) in towhere.iteritems() if value]) #remove nul-values from dict
wherestring = ' AND '.join([key+'=%('+key+')s' for key in towhere])
if routedict['frompartner_tochannel_id']: #use frompartner_tochannel in where-clause of query (partner/group dependent outchannel
towhere['frompartner_tochannel_id']=routedict['frompartner_tochannel_id']
wherestring += ''' AND (frompartner=%(frompartner_tochannel_id)s
OR frompartner in (SELECT from_partner_id
FROM partnergroup
WHERE to_partner_id =%(frompartner_tochannel_id)s ))'''
if routedict['topartner_tochannel_id']: #use topartner_tochannel in where-clause of query (partner/group dependent outchannel
towhere['topartner_tochannel_id']=routedict['topartner_tochannel_id']
wherestring += ''' AND (topartner=%(topartner_tochannel_id)s
OR topartner in (SELECT from_partner_id
FROM partnergroup
WHERE to_partner_id=%(topartner_tochannel_id)s ))'''
toset={'tochannel':routedict['tochannel'],'status':FILEOUT}
botslib.addinfocore(change=toset,where=towhere,wherestring=wherestring)
if not routedict['defer']: #do outgoing part of route
botslib.tryrunscript(userscript,scriptname,'preoutcommunication',routedict=routedict)
communication.run(idchannel=routedict['tochannel'],idroute=routedict['idroute']) #communication.run outcommunication
botslib.tryrunscript(userscript,scriptname,'postoutcommunication',routedict=routedict)
botslib.tryrunscript(userscript,scriptname,'end',routedict=routedict)
def router(routedict):
''' communication.run one route. variants:
- a route can be just script;
- a route can do only incoming
- a route can do only outgoing
- a route can do both incoming and outgoing
- at several points functions from a route script are called - if function is in route script
'''
#is there a user route script?
try:
userscript,scriptname = botslib.botsimport('routescripts',routedict['idroute'])
except ImportError: #other errors, eg syntax errors are just passed
userscript = scriptname = None
#if user route script has function 'main': communication.run 'main' (and do nothing else)
if botslib.tryrunscript(userscript,scriptname,'main',routedict=routedict):
return #so: if function ' main' : communication.run only the routescript, nothing else.
if not (userscript or routedict['fromchannel'] or routedict['tochannel'] or routedict['translateind']):
raise botslib.ScriptError(_(u'Route "$route" is empty: no script, not enough parameters.'),route=routedict['idroute'])
botslib.tryrunscript(userscript,scriptname,'start',routedict=routedict)
#communication.run incoming channel
if routedict['fromchannel']: #do incoming part of route: in-communication; set ready for translation; translate
botslib.tryrunscript(userscript,scriptname,'preincommunication',routedict=routedict)
communication.run(idchannel=routedict['fromchannel'],idroute=routedict['idroute']) #communication.run incommunication
#add attributes from route to the received files
where = {'status':FILEIN,'fromchannel':routedict['fromchannel'],'idroute':routedict['idroute']}
change = {'editype':routedict['fromeditype'],'messagetype':routedict['frommessagetype'],'frompartner':routedict['frompartner'],'topartner':routedict['topartner'],'alt':routedict['alt']}
botslib.updateinfo(change=change,where=where)
#all received files have status FILEIN
botslib.tryrunscript(userscript,scriptname,'postincommunication',routedict=routedict)
if routedict['fromeditype'] == 'mailbag': #mailbag for the route.
preprocess.preprocess(routedict,preprocess.mailbag)
#communication.run translation
if routedict['translateind']:
botslib.tryrunscript(userscript,scriptname,'pretranslation',routedict=routedict)
botslib.addinfo(change={'status':TRANSLATE},where={'status':FILEIN,'idroute':routedict['idroute']})
transform.translate(idroute=routedict['idroute'])
botslib.tryrunscript(userscript,scriptname,'posttranslation',routedict=routedict)
#merge messages & communication.run outgoing channel
if routedict['tochannel']: #do outgoing part of route
botslib.tryrunscript(userscript,scriptname,'premerge',routedict=routedict)
envelope.mergemessages(idroute=routedict['idroute'])
botslib.tryrunscript(userscript,scriptname,'postmerge',routedict=routedict)
#communication.run outgoing channel
#build for query: towhere (dict) and wherestring
towhere = dict(status=MERGED,
idroute=routedict['idroute'],
editype=routedict['toeditype'],
messagetype=routedict['tomessagetype'],
testindicator=routedict['testindicator'])
towhere = dict([(key, value) for (key, value) in towhere.iteritems() if value]) #remove nul-values from dict
wherestring = ' AND '.join([key+'=%('+key+')s' for key in towhere])
if routedict['frompartner_tochannel_id']: #use frompartner_tochannel in where-clause of query (partner/group dependent outchannel
towhere['frompartner_tochannel_id'] = routedict['frompartner_tochannel_id']
wherestring += ''' AND (frompartner=%(frompartner_tochannel_id)s
OR frompartner in (SELECT from_partner_id
FROM partnergroup
WHERE to_partner_id =%(frompartner_tochannel_id)s ))'''
if routedict['topartner_tochannel_id']: #use topartner_tochannel in where-clause of query (partner/group dependent outchannel
towhere['topartner_tochannel_id'] = routedict['topartner_tochannel_id']
wherestring += ''' AND (topartner=%(topartner_tochannel_id)s
OR topartner in (SELECT from_partner_id
FROM partnergroup
WHERE to_partner_id=%(topartner_tochannel_id)s ))'''
toset = {'tochannel':routedict['tochannel'],'status':FILEOUT}
botslib.addinfocore(change=toset,where=towhere,wherestring=wherestring)
if not routedict['defer']: #do outgoing part of route
botslib.tryrunscript(userscript,scriptname,'preoutcommunication',routedict=routedict)
communication.run(idchannel=routedict['tochannel'],idroute=routedict['idroute']) #communication.run outcommunication
botslib.tryrunscript(userscript,scriptname,'postoutcommunication',routedict=routedict)
botslib.tryrunscript(userscript,scriptname,'end',routedict=routedict)