Ejemplo n.º 1
0
def preprocess(routedict, function, status=FILEIN, **argv):
    ''' for pre- and postprocessing of files.
        these are NOT translations; translation involve grammars, mapping scripts etc. think of eg:
        - unzipping zipped files.
        - convert excel to csv
        - password protected files.
        Select files from INFILE -> SET_FOR_PROCESSING using criteria
        Than the actual processing function is called.
        The processing function does: SET_FOR_PROCESSING -> PROCESSING -> FILEIN
        If errors occur during processing, no ta are left with status FILEIN !
        preprocess is called right after the in-communicatiation
    '''
    nr_files = 0
    preprocessnumber = botslib.getpreprocessnumber()
    if not botslib.addinfo(change={'status': preprocessnumber},
                           where={
                               'status': status,
                               'idroute': routedict['idroute'],
                               'fromchannel': routedict['fromchannel']
                           }):  #check if there is something to do
        return 0
    for row in botslib.query(
            u'''SELECT idta,filename,charset
                                FROM  ta
                                WHERE   idta>%(rootidta)s
                                AND     status=%(status)s
                                AND     statust=%(statust)s
                                AND     idroute=%(idroute)s
                                AND     fromchannel=%(fromchannel)s
                                ''', {
                'status': preprocessnumber,
                'statust': OK,
                'idroute': routedict['idroute'],
                'fromchannel': routedict['fromchannel'],
                'rootidta': botslib.get_minta4query()
            }):
        try:
            botsglobal.logmap.debug(u'Start preprocessing "%s" for file "%s".',
                                    function.__name__, row['filename'])
            ta_set_for_processing = botslib.OldTransaction(row['idta'])
            ta_processing = ta_set_for_processing.copyta(
                status=preprocessnumber + 1)
            ta_processing.filename = row['filename']
            function(ta_from=ta_processing,
                     endstatus=status,
                     routedict=routedict,
                     **argv)
        except:
            txt = botslib.txtexc()
            ta_processing.failure()
            ta_processing.update(statust=ERROR, errortext=txt)
        else:
            botsglobal.logmap.debug(u'OK preprocessing  "%s" for file "%s".',
                                    function.__name__, row['filename'])
            ta_set_for_processing.update(statust=DONE)
            ta_processing.update(statust=DONE)
            nr_files += 1
    return nr_files
Ejemplo n.º 2
0
def preprocess(routedict,function, status=FILEIN,**argv):
    ''' for pre- and postprocessing of files.
        these are NOT translations; translation involve grammars, mapping scripts etc. think of eg:
        - unzipping zipped files.
        - convert excel to csv
        - password protected files.
        Select files from INFILE -> SET_FOR_PROCESSING using criteria
        Than the actual processing function is called.
        The processing function does: SET_FOR_PROCESSING -> PROCESSING -> FILEIN
        If errors occur during processing, no ta are left with status FILEIN !
        preprocess is called right after the in-communicatiation
    '''
    nr_files = 0
    preprocessnumber = botslib.getpreprocessnumber()
    if not botslib.addinfo(change={'status':preprocessnumber},where={'status':status,'idroute':routedict['idroute'],'fromchannel':routedict['fromchannel']}):    #check if there is something to do
        return 0
    for row in botslib.query(u'''SELECT idta,filename,charset
                                FROM  ta
                                WHERE   idta>%(rootidta)s
                                AND     status=%(status)s
                                AND     statust=%(statust)s
                                AND     idroute=%(idroute)s
                                AND     fromchannel=%(fromchannel)s
                                ''',
                                {'status':preprocessnumber,'statust':OK,'idroute':routedict['idroute'],'fromchannel':routedict['fromchannel'],'rootidta':botslib.get_minta4query()}):
        try:
            botsglobal.logmap.debug(u'Start preprocessing "%s" for file "%s".',function.__name__,row['filename'])
            ta_set_for_processing = botslib.OldTransaction(row['idta'])
            ta_processing = ta_set_for_processing.copyta(status=preprocessnumber+1)
            ta_processing.filename = row['filename']
            function(ta_from=ta_processing,endstatus=status,routedict=routedict,**argv)
        except:
            txt = botslib.txtexc()
            ta_processing.failure()
            ta_processing.update(statust=ERROR,errortext=txt)
        else:
            botsglobal.logmap.debug(u'OK preprocessing  "%s" for file "%s".',function.__name__,row['filename'])
            ta_set_for_processing.update(statust=DONE)
            ta_processing.update(statust=DONE)
            nr_files += 1
    return nr_files
Ejemplo n.º 3
0
    def routepart(self,routedict):
        ''' communication.run one route part. variants:
            -   a route can be a routescript
            -   a route can do only incoming
            -   a route can do only outgoing
            -   a route can do both incoming and outgoing
            -   at several points functions from a routescript are called - if function is in routescript
        '''
        self.minta4query_routepart = botslib._Transaction.processlist[-1]     #the idta of routepart

        #if routescript has function 'main': communication.run 'main' (and do nothing else)
        if botslib.tryrunscript(self.userscript,self.scriptname,'main',routedict=routedict):
            return  #so: if function ' main' : communication.run only the routescript, nothing else.
        if not (self.userscript or routedict['fromchannel'] or routedict['tochannel'] or routedict['translateind']):
            raise botslib.ScriptError(_(u'Route "%(idroute)s" is empty: no routescript, not enough parameters.'),routedict)

        botslib.tryrunscript(self.userscript,self.scriptname,'start',routedict=routedict)

        #incoming part of route:
        #- incommunication
        #- assign attributes from route to incoming files
        #- preprocessing
        if routedict['fromchannel']:
            #only done for edi files from this route-part, this inchannel
            if routedict['command'] == 'rereceive':
                rootidta = self.get_minta4query()
            else:
                rootidta = self.get_minta4query_routepart()
            botslib.tryrunscript(self.userscript,self.scriptname,'preincommunication',routedict=routedict)
            communication.run(idchannel=routedict['fromchannel'],command=routedict['command'],idroute=routedict['idroute'],rootidta=rootidta)  #communication.run incommunication
            #add attributes from route to the received files;
            where = {'statust':OK,'status':FILEIN,'fromchannel':routedict['fromchannel'],'idroute':routedict['idroute'],'rootidta':rootidta}
            change = {'editype':routedict['fromeditype'],'messagetype':routedict['frommessagetype'],'frompartner':routedict['frompartner'],'topartner':routedict['topartner'],'alt':routedict['alt']}
            nr_of_incoming_files_for_channel = botslib.updateinfo(change=change,where=where)
            botslib.tryrunscript(self.userscript,self.scriptname,'postincommunication',routedict=routedict)
            if nr_of_incoming_files_for_channel:
                #unzip incoming files (if indicated)
                if routedict['zip_incoming'] == 1:               #unzip incoming (non-zipped gives error).
                    preprocess.preprocess(routedict=routedict,function=preprocess.botsunzip,rootidta=self.get_minta4query_routepart(),pass_non_zip=False)
                elif routedict['zip_incoming'] == 2:               #unzip incoming if zipped.
                    preprocess.preprocess(routedict=routedict,function=preprocess.botsunzip,rootidta=self.get_minta4query_routepart(),pass_non_zip=True)
                #run mailbag-module.
                if botsglobal.ini.getboolean('settings','compatibility_mailbag',False):
                    editypes_via_mailbag = ['mailbag']
                else:
                    editypes_via_mailbag = ['mailbag','edifact','x12','tradacoms']
                if routedict['fromeditype'] in editypes_via_mailbag:               #mailbag for the route.
                    preprocess.preprocess(routedict=routedict,function=preprocess.mailbag,rootidta=self.get_minta4query_routepart(),frommessagetype=routedict['frommessagetype'])

        #translate, merge, pass through: INFILE->MERGED
        if int(routedict['translateind']) in [1,3]:
            #translate: for files in route
            botslib.tryrunscript(self.userscript,self.scriptname,'pretranslation',routedict=routedict)
            if routedict['command'] in ['rereceive',]:
                rootidta = self.get_minta4query()
            else:
                rootidta = self.get_minta4query_route()
            transform.translate(startstatus=FILEIN,endstatus=TRANSLATED,routedict=routedict,rootidta=rootidta)
            botslib.tryrunscript(self.userscript,self.scriptname,'posttranslation',routedict=routedict)
            #**merge: for files in this route-part (the translated files)
            botslib.tryrunscript(self.userscript,self.scriptname,'premerge',routedict=routedict)
            envelope.mergemessages(startstatus=TRANSLATED,endstatus=MERGED,idroute=routedict['idroute'],rootidta=self.get_minta4query_routepart())
            botslib.tryrunscript(self.userscript,self.scriptname,'postmerge',routedict=routedict)
        elif routedict['translateind'] == 2:        #pass-through: pickup the incoming files and mark these as MERGED (==translation is finished)
            botslib.addinfo(change={'status':MERGED,'statust':OK},where={'status':FILEIN,'statust':OK,'idroute':routedict['idroute'],'rootidta':self.get_minta4query_route()})
        #NOTE: routedict['translateind'] == 0 than nothing will happen with the files in this route. 

        #ommunication outgoing channel: MERGED->RAWOUT
        if routedict['tochannel']:
            #**build query to add outchannel as attribute to outgoing files***
            #filter files in route for outchannel
            towhere = { 'status':MERGED,
                        'statust':OK,
                        'idroute':routedict['idroute'],
                        'editype':routedict['toeditype'],
                        'messagetype':routedict['tomessagetype'],
                        'testindicator':routedict['testindicator'],
                        }
            towhere = dict((key, value) for key,value in towhere.iteritems() if value)   #remove nul-values from dict
            wherestring = ' AND '.join([key+'=%('+key+')s ' for key in towhere])
            if routedict['frompartner_tochannel_id']:   #use frompartner_tochannel in where-clause of query (partner/group dependent outchannel
                towhere['frompartner_tochannel_id'] = routedict['frompartner_tochannel_id']
                wherestring += ''' AND (frompartner=%(frompartner_tochannel_id)s
                                    OR frompartner in (SELECT from_partner_id
                                    FROM partnergroup
                                    WHERE to_partner_id=%(frompartner_tochannel_id)s )) '''
            if routedict['topartner_tochannel_id']:   #use topartner_tochannel in where-clause of query (partner/group dependent outchannel
                towhere['topartner_tochannel_id'] = routedict['topartner_tochannel_id']
                wherestring += ''' AND (topartner=%(topartner_tochannel_id)s
                                    OR topartner in (SELECT from_partner_id
                                    FROM partnergroup
                                    WHERE to_partner_id=%(topartner_tochannel_id)s )) '''
            toset = {'status':FILEOUT,'statust':OK,'tochannel':routedict['tochannel']}
            towhere['rootidta'] = self.get_minta4query_route()
            nr_of_outgoing_files_for_channel = botslib.addinfocore(change=toset,where=towhere,wherestring=wherestring)
            
            if nr_of_outgoing_files_for_channel:
                #**set asked confirmation/acknowledgements
                botslib.set_asked_confirmrules(routedict,rootidta=self.get_minta4query_routepart())
                #**zip outgoing
                #for files in this route-part for this out-channel
                if routedict['zip_outgoing'] == 1:               
                    preprocess.postprocess(routedict=routedict,function=preprocess.botszip,rootidta=self.get_minta4query_routepart())
                    
            #actual communication: run outgoing channel (if not deferred)
            #for all files in run that are for this channel (including the deferred ones from other routes)
            if not routedict['defer']:
                #determine range of idta to query: if channel was not deferred earlier in run: query only for route part else query for whole run
                if self.keep_track_if_outchannel_deferred.get(routedict['tochannel'],False) or routedict['command'] in ['resend','automaticretrycommunication']:
                    rootidta = self.get_minta4query()
                else:
                    rootidta = self.get_minta4query_routepart()
                if botslib.countoutfiles(idchannel=routedict['tochannel'],rootidta=rootidta):
                    botslib.tryrunscript(self.userscript,self.scriptname,'preoutcommunication',routedict=routedict)
                    communication.run(idchannel=routedict['tochannel'],command=routedict['command'],idroute=routedict['idroute'],rootidta=rootidta)
                    #in communication several things can go wrong.
                    #all outgoing files should have same status; that way all recomnnunication can be handled the same:
                    #- status EXTERNOUT statust DONE (if communication goes OK)
                    #- status EXTERNOUT status ERROR (if file is not communicatied)
                    #to have the same status for all outgoing files some manipulation is needed, eg in case no connection could be made.
                    botslib.addinfo(change={'status':EXTERNOUT,'statust':ERROR},where={'status':FILEOUT,'statust':OK,'tochannel':routedict['tochannel'],'rootidta':rootidta})
                    botslib.tryrunscript(self.userscript,self.scriptname,'postoutcommunication',routedict=routedict)
                
        botslib.tryrunscript(self.userscript,self.scriptname,'end',routedict=routedict)
Ejemplo n.º 4
0
def router(routedict):
    ''' communication.run one route. variants:
        -   a route can be just script; 
        -   a route can do only incoming
        -   a route can do only outgoing
        -   a route can do both incoming and outgoing
        -   at several points functions from a route script are called - if function is in route script
    '''
    #is there a user route script?
    try:
        userscript,scriptname = botslib.botsimport('routescripts',routedict['idroute'])
    except ImportError: #other errors, eg syntax errors are just passed
        userscript = scriptname = None
        
    #if user route script has function 'main': communication.run 'main' (and do nothing else)
    if botslib.tryrunscript(userscript,scriptname,'main',routedict=routedict):
        return  #so: if function ' main' : communication.run only the routescript, nothing else.
    if not (userscript or routedict['fromchannel'] or routedict['tochannel'] or routedict['translateind']): 
        raise botslib.ScriptError(_(u'Route "$route" is empty: no script, not enough parameters.'),route=routedict['idroute'])
    
    botslib.tryrunscript(userscript,scriptname,'start',routedict=routedict)
    
    #communication.run incoming channel
    if routedict['fromchannel']:     #do incoming part of route: in-communication; set ready for translation; translate
        botslib.tryrunscript(userscript,scriptname,'preincommunication',routedict=routedict)
        communication.run(idchannel=routedict['fromchannel'],idroute=routedict['idroute'])  #communication.run incommunication
        #add attributes from route to the received files
        where={'status':FILEIN,'fromchannel':routedict['fromchannel'],'idroute':routedict['idroute']}
        change={'editype':routedict['fromeditype'],'messagetype':routedict['frommessagetype'],'frompartner':routedict['frompartner'],'topartner':routedict['topartner'],'alt':routedict['alt']}
        botslib.updateinfo(change=change,where=where)
            
        #all received files have status FILEIN
        botslib.tryrunscript(userscript,scriptname,'postincommunication',routedict=routedict)
        if routedict['fromeditype'] == 'mailbag':               #mailbag for the route.
            preprocess.preprocess(routedict,preprocess.mailbag)
    
    #communication.run translation
    if routedict['translateind']:
        botslib.tryrunscript(userscript,scriptname,'pretranslation',routedict=routedict)
        botslib.addinfo(change={'status':TRANSLATE},where={'status':FILEIN,'idroute':routedict['idroute']})
        transform.translate(idroute=routedict['idroute'])
        botslib.tryrunscript(userscript,scriptname,'posttranslation',routedict=routedict)
        
    #merge messages & communication.run outgoing channel
    if routedict['tochannel']:   #do outgoing part of route
        botslib.tryrunscript(userscript,scriptname,'premerge',routedict=routedict)
        envelope.mergemessages(idroute=routedict['idroute'])
        botslib.tryrunscript(userscript,scriptname,'postmerge',routedict=routedict)
            
        #communication.run outgoing channel
        #build for query: towhere (dict) and wherestring  
        towhere=dict(status=MERGED,
                    idroute=routedict['idroute'],
                    editype=routedict['toeditype'],
                    messagetype=routedict['tomessagetype'],
                    testindicator=routedict['testindicator'])
        towhere=dict([(key, value) for (key, value) in towhere.iteritems() if value])   #remove nul-values from dict
        wherestring = ' AND '.join([key+'=%('+key+')s' for key in towhere])
        if routedict['frompartner_tochannel_id']:   #use frompartner_tochannel in where-clause of query (partner/group dependent outchannel
            towhere['frompartner_tochannel_id']=routedict['frompartner_tochannel_id']
            wherestring += ''' AND (frompartner=%(frompartner_tochannel_id)s 
                                    OR frompartner in (SELECT from_partner_id 
                                                        FROM partnergroup
                                                        WHERE to_partner_id =%(frompartner_tochannel_id)s ))'''
        if routedict['topartner_tochannel_id']:   #use topartner_tochannel in where-clause of query (partner/group dependent outchannel
            towhere['topartner_tochannel_id']=routedict['topartner_tochannel_id']
            wherestring += ''' AND (topartner=%(topartner_tochannel_id)s 
                                    OR topartner in (SELECT from_partner_id 
                                                        FROM partnergroup
                                                        WHERE to_partner_id=%(topartner_tochannel_id)s ))'''
        toset={'tochannel':routedict['tochannel'],'status':FILEOUT}
        botslib.addinfocore(change=toset,where=towhere,wherestring=wherestring)
        
        if not routedict['defer']:   #do outgoing part of route
            botslib.tryrunscript(userscript,scriptname,'preoutcommunication',routedict=routedict)
            communication.run(idchannel=routedict['tochannel'],idroute=routedict['idroute'])    #communication.run outcommunication
            botslib.tryrunscript(userscript,scriptname,'postoutcommunication',routedict=routedict)
    
    botslib.tryrunscript(userscript,scriptname,'end',routedict=routedict)
Ejemplo n.º 5
0
def router(routedict):
    ''' communication.run one route. variants:
        -   a route can be just script;
        -   a route can do only incoming
        -   a route can do only outgoing
        -   a route can do both incoming and outgoing
        -   at several points functions from a route script are called - if function is in route script
    '''
    #is there a user route script?
    try:
        userscript,scriptname = botslib.botsimport('routescripts',routedict['idroute'])
    except ImportError: #other errors, eg syntax errors are just passed
        userscript = scriptname = None

    #if user route script has function 'main': communication.run 'main' (and do nothing else)
    if botslib.tryrunscript(userscript,scriptname,'main',routedict=routedict):
        return  #so: if function ' main' : communication.run only the routescript, nothing else.
    if not (userscript or routedict['fromchannel'] or routedict['tochannel'] or routedict['translateind']):
        raise botslib.ScriptError(_(u'Route "$route" is empty: no script, not enough parameters.'),route=routedict['idroute'])

    botslib.tryrunscript(userscript,scriptname,'start',routedict=routedict)

    #communication.run incoming channel
    if routedict['fromchannel']:     #do incoming part of route: in-communication; set ready for translation; translate
        botslib.tryrunscript(userscript,scriptname,'preincommunication',routedict=routedict)
        communication.run(idchannel=routedict['fromchannel'],idroute=routedict['idroute'])  #communication.run incommunication
        #add attributes from route to the received files
        where = {'status':FILEIN,'fromchannel':routedict['fromchannel'],'idroute':routedict['idroute']}
        change = {'editype':routedict['fromeditype'],'messagetype':routedict['frommessagetype'],'frompartner':routedict['frompartner'],'topartner':routedict['topartner'],'alt':routedict['alt']}
        botslib.updateinfo(change=change,where=where)

        #all received files have status FILEIN
        botslib.tryrunscript(userscript,scriptname,'postincommunication',routedict=routedict)
        if routedict['fromeditype'] == 'mailbag':               #mailbag for the route.
            preprocess.preprocess(routedict,preprocess.mailbag)

    #communication.run translation
    if routedict['translateind']:
        botslib.tryrunscript(userscript,scriptname,'pretranslation',routedict=routedict)
        botslib.addinfo(change={'status':TRANSLATE},where={'status':FILEIN,'idroute':routedict['idroute']})
        transform.translate(idroute=routedict['idroute'])
        botslib.tryrunscript(userscript,scriptname,'posttranslation',routedict=routedict)

    #merge messages & communication.run outgoing channel
    if routedict['tochannel']:   #do outgoing part of route
        botslib.tryrunscript(userscript,scriptname,'premerge',routedict=routedict)
        envelope.mergemessages(idroute=routedict['idroute'])
        botslib.tryrunscript(userscript,scriptname,'postmerge',routedict=routedict)

        #communication.run outgoing channel
        #build for query: towhere (dict) and wherestring
        towhere = dict(status=MERGED,
                    idroute=routedict['idroute'],
                    editype=routedict['toeditype'],
                    messagetype=routedict['tomessagetype'],
                    testindicator=routedict['testindicator'])
        towhere = dict([(key, value) for (key, value) in towhere.iteritems() if value])   #remove nul-values from dict
        wherestring = ' AND '.join([key+'=%('+key+')s' for key in towhere])
        if routedict['frompartner_tochannel_id']:   #use frompartner_tochannel in where-clause of query (partner/group dependent outchannel
            towhere['frompartner_tochannel_id'] = routedict['frompartner_tochannel_id']
            wherestring += ''' AND (frompartner=%(frompartner_tochannel_id)s
                                    OR frompartner in (SELECT from_partner_id
                                                        FROM partnergroup
                                                        WHERE to_partner_id =%(frompartner_tochannel_id)s ))'''
        if routedict['topartner_tochannel_id']:   #use topartner_tochannel in where-clause of query (partner/group dependent outchannel
            towhere['topartner_tochannel_id'] = routedict['topartner_tochannel_id']
            wherestring += ''' AND (topartner=%(topartner_tochannel_id)s
                                    OR topartner in (SELECT from_partner_id
                                                        FROM partnergroup
                                                        WHERE to_partner_id=%(topartner_tochannel_id)s ))'''
        toset = {'tochannel':routedict['tochannel'],'status':FILEOUT}
        botslib.addinfocore(change=toset,where=towhere,wherestring=wherestring)

        if not routedict['defer']:   #do outgoing part of route
            botslib.tryrunscript(userscript,scriptname,'preoutcommunication',routedict=routedict)
            communication.run(idchannel=routedict['tochannel'],idroute=routedict['idroute'])    #communication.run outcommunication
            botslib.tryrunscript(userscript,scriptname,'postoutcommunication',routedict=routedict)

    botslib.tryrunscript(userscript,scriptname,'end',routedict=routedict)