Example #1
0
def _translate_one_file(row,routedict,endstatus,userscript,scriptname):
    ''' -   read, lex, parse, make tree of nodes.
        -   split up files into messages (using 'nextmessage' of grammar)
        -   get mappingscript, start mappingscript.
        -   write the results of translation (no enveloping yet)
    '''
    try:
        ta_fromfile = botslib.OldTransaction(row['idta'])
        ta_parsed = ta_fromfile.copyta(status=PARSED)
        if row['filesize'] > botsglobal.ini.getint('settings','maxfilesizeincoming',5000000):
            ta_parsed.update(filesize=row['filesize'])
            raise botslib.FileTooLarge(_(u'File size of %(filesize)s is too big; option "maxfilesizeincoming" in bots.ini is %(maxfilesizeincoming)s.'),
                                            {'filesize':row['filesize'],'maxfilesizeincoming':botsglobal.ini.getint('settings','maxfilesizeincoming',5000000)})
        botsglobal.logger.debug(_(u'Start translating file "%(filename)s" editype "%(editype)s" messagetype "%(messagetype)s".'),row)
        #read whole edi-file: read, parse and made into a inmessage-object. Message is represented as a tree (inmessage.root is the root of the tree).
        edifile = inmessage.parse_edi_file(frompartner=row['frompartner'],
                                            topartner=row['topartner'],
                                            filename=row['filename'],
                                            messagetype=row['messagetype'],
                                            testindicator=row['testindicator'],
                                            editype=row['editype'],
                                            charset=row['charset'],
                                            alt=row['alt'],
                                            fromchannel=row['fromchannel'],
                                            idroute=routedict['idroute'],
                                            command=routedict['command'])
        edifile.checkforerrorlist() #no exception if infile has been lexed and parsed OK else raises an error

        if int(routedict['translateind']) == 3: #parse & passthrough; file is parsed, partners are known, no mapping, does confirm.
            raise botslib.GotoException('dummy')    
        
        #edifile.ta_info contains info: QUERIES, charset etc
        for inn_splitup in edifile.nextmessage():   #splitup messages in parsed edifile
            try:
                ta_splitup = ta_parsed.copyta(status=SPLITUP,**inn_splitup.ta_info)    #copy PARSED to SPLITUP ta
                #inn_splitup.ta_info: parameters from inmessage.parse_edi_file(), syntax-information and parse-information
                inn_splitup.ta_info['idta_fromfile'] = ta_fromfile.idta     #for confirmations in userscript; used to give idta of 'confirming message'
                inn_splitup.ta_info['idta'] = ta_splitup.idta     #for confirmations in userscript; used to give idta of 'confirming message'
                number_of_loops_with_same_alt = 0
                while 1:    #continue as long as there are (alt-)translations
                    #lookup the translation************************
                    tscript,toeditype,tomessagetype = botslib.lookup_translation(fromeditype=inn_splitup.ta_info['editype'],
                                                                        frommessagetype=inn_splitup.ta_info['messagetype'],
                                                                        frompartner=inn_splitup.ta_info['frompartner'],
                                                                        topartner=inn_splitup.ta_info['topartner'],
                                                                        alt=inn_splitup.ta_info['alt'])
                    if not tscript:       #no translation found in translate table; check if can find translation via user script
                        if userscript and hasattr(userscript,'gettranslation'):
                            tscript,toeditype,tomessagetype = botslib.runscript(userscript,scriptname,'gettranslation',idroute=routedict['idroute'],message=inn_splitup)
                        if not tscript:
                            raise botslib.TranslationNotFoundError(_(u'Translation not found for editype "%(editype)s", messagetype "%(messagetype)s", frompartner "%(frompartner)s", topartner "%(topartner)s", alt "%(alt)s".'),
                                                                        inn_splitup.ta_info)

                    inn_splitup.ta_info['divtext'] = tscript     #ifor reporting used mapping script to database (for display in GUI).
                    #initialize new out-object*************************
                    ta_translated = ta_splitup.copyta(status=endstatus)     #make ta for translated message (new out-ta)
                    filename_translated = str(ta_translated.idta)
                    out_translated = outmessage.outmessage_init(editype=toeditype,messagetype=tomessagetype,filename=filename_translated,reference=unique('messagecounter'),statust=OK,divtext=tscript)    #make outmessage object
                        
                    #run mapping script************************
                    botsglobal.logger.debug(_(u'Mappingscript "%(tscript)s" translates messagetype "%(messagetype)s" to messagetype "%(tomessagetype)s".'),
                                            {'tscript':tscript,'messagetype':inn_splitup.ta_info['messagetype'],'tomessagetype':out_translated.ta_info['messagetype']})
                    translationscript,scriptfilename = botslib.botsimport('mappings',inn_splitup.ta_info['editype'],tscript) #get the mappingscript
                    alt_from_previous_run = inn_splitup.ta_info['alt']      #needed to check for infinite loop
                    doalttranslation = botslib.runscript(translationscript,scriptfilename,'main',inn=inn_splitup,out=out_translated)
                    botsglobal.logger.debug(_(u'Mappingscript "%(tscript)s" finished.'),{'tscript':tscript})
                    
                    #manipulate for some attributes after mapping script
                    if 'topartner' not in out_translated.ta_info:    #out_translated does not contain values from ta......
                        out_translated.ta_info['topartner'] = inn_splitup.ta_info['topartner']
                    if 'botskey' in inn_splitup.ta_info:
                        inn_splitup.ta_info['reference'] = inn_splitup.ta_info['botskey']
                    if 'botskey' in out_translated.ta_info:    #out_translated does not contain values from ta......
                        out_translated.ta_info['reference'] = out_translated.ta_info['botskey']
                        
                    #check the value received from the mappingscript to determine what to do in this while-loop. Handling of chained trasnlations.
                    if doalttranslation is None:    
                        #translation(s) are done; handle out-message 
                        handle_out_message(out_translated,ta_translated)
                        break   #break out of while loop
                    elif isinstance(doalttranslation,dict):
                        #some extended cases; a dict is returned that contains 'instructions' for some type of chained translations
                        if 'type' not in doalttranslation or 'alt' not in doalttranslation:
                            raise botslib.BotsError(_(u"Mappingscript returned '%(alt)s'. This dict should not have 'type' and 'alt'."),{'alt':doalttranslation})
                        if alt_from_previous_run == doalttranslation['alt']:
                            number_of_loops_with_same_alt += 1
                        else:
                            number_of_loops_with_same_alt = 0
                        if doalttranslation['type'] == u'out_as_inn':
                            #do chained translation: use the out-object as inn-object, new out-object
                            #use case: detected error in incoming file; use out-object to generate warning email
                            handle_out_message(out_translated,ta_translated)
                            inn_splitup = out_translated    #out-object is now inn-object
                            if isinstance(inn_splitup,outmessage.fixed):    #for fixed: strip all values in node
                                inn_splitup.root.stripnode()
                            inn_splitup.ta_info['alt'] = doalttranslation['alt']   #get the alt-value for the next chained translation
                            if not 'frompartner' in inn_splitup.ta_info:
                                inn_splitup.ta_info['frompartner'] = ''
                            if not 'topartner' in inn_splitup.ta_info:
                                inn_splitup.ta_info['topartner'] = ''
                            inn_splitup.ta_info.pop('statust')
                        elif doalttranslation['type'] == u'no_check_on_infinite_loop':
                            #do chained translation: allow many loops wit hsame alt-value.
                            #mapping script will have to handle this correctly.
                            number_of_loops_with_same_alt = 0
                            handle_out_message(out_translated,ta_translated)
                            inn_splitup.ta_info['alt'] = doalttranslation['alt']   #get the alt-value for the next chained translation
                        else:   #there is nothing else
                            raise botslib.BotsError(_(u'Mappingscript returned dict with an unknown "type": "%(doalttranslation)s".'),{'doalttranslation':doalttranslation})
                    else:  #note: this includes alt '' (empty string)
                        if alt_from_previous_run == doalttranslation:
                            number_of_loops_with_same_alt += 1
                        else:
                            number_of_loops_with_same_alt = 0
                        #do normal chained translation: same inn-object, new out-object
                        handle_out_message(out_translated,ta_translated)
                        inn_splitup.ta_info['alt'] = doalttranslation   #get the alt-value for the next chained translation
                    if number_of_loops_with_same_alt > 10:
                        raise botslib.BotsError(_(u'Mappingscript returns same alt value over and over again (infinite loop?). Alt: "%(doalttranslation)s".'),{'doalttranslation':doalttranslation})
                #end of while-loop (trans**********************************************************************************
            #exceptions file_out-level: exception in mappingscript or writing of out-file
            except:
                #2 modes: either every error leads to skipping of  whole infile (old  mode) or errors in mappingscript/outfile only affect that branche
                if botsglobal.ini.getboolean('settings','oldmessageerrors',False):
                    raise
                txt = botslib.txtexc()
                ta_splitup.update(statust=ERROR,errortext=txt,**inn_splitup.ta_info)   #update db. inn_splitup.ta_info could be changed by mappingscript. Is this useful?
                ta_splitup.deletechildren()
            else:
                ta_splitup.update(statust=DONE, **inn_splitup.ta_info)   #update db. inn_splitup.ta_info could be changed by mappingscript. Is this useful?

    #exceptions file_in-level
    except botslib.GotoException:   #edi-file is OK, file is passed-through after parsing.
        ta_parsed.update(statust=DONE,filesize=row['filesize'],**edifile.ta_info)   #update with info from eg queries
        ta_parsed.copyta(status=MERGED,statust=OK)          #original file goes straight to MERGED
        edifile.handleconfirm(ta_fromfile,error=False)
        botsglobal.logger.debug(_(u'Parse & passthrough for input file "%(filename)s".'),row)
    except botslib.FileTooLarge as msg:
        ta_parsed.update(statust=ERROR,errortext=str(msg))
        ta_parsed.deletechildren()
        botsglobal.logger.debug(u'Error in translating input file "%(filename)s":\n%(msg)s',{'filename':row['filename'],'msg':msg})
    except:
        txt = botslib.txtexc()
        ta_parsed.update(statust=ERROR,errortext=txt,**edifile.ta_info)
        ta_parsed.deletechildren()
        edifile.handleconfirm(ta_fromfile,error=True)
        botsglobal.logger.debug(u'Error in translating input file "%(filename)s":\n%(msg)s',{'filename':row['filename'],'msg':txt})
    else:
        edifile.handleconfirm(ta_fromfile,error=False)
        ta_parsed.update(statust=DONE,filesize=row['filesize'],**edifile.ta_info)
        botsglobal.logger.debug(_(u'Translated input file "%(filename)s".'),row)
    finally:
        ta_fromfile.update(statust=DONE)
Example #2
0
def translate(startstatus,endstatus,idroute,rootidta):
    ''' main translation loop.
        get edifiles to be translated, than:
        -   read, lex, parse, make tree of nodes.
        -   split up files into messages (using 'nextmessage' of grammar)
        -   get mappingscript, start mappingscript.
        -   write the results of translation (no enveloping yet)
        status: FILEIN--PARSED-<SPLITUP--TRANSLATED
    '''
    try:    #see if there is a userscript that can determine the translation
        userscript,scriptname = botslib.botsimport('mappings','translation')
    except ImportError:       #userscript is not there; other errors like syntax errors are not catched
        userscript = scriptname = None
    #select edifiles to translate
    for rawrow in botslib.query(u'''SELECT idta,frompartner,topartner,filename,messagetype,testindicator,editype,charset,alt,fromchannel,filesize
                                FROM ta
                                WHERE idta>%(rootidta)s
                                AND status=%(status)s
                                AND statust=%(statust)s
                                AND idroute=%(idroute)s ''',
                                {'status':startstatus,'statust':OK,'idroute':idroute,'rootidta':rootidta}):
        try:
            row = dict(rawrow)   #convert to real dictionary ()
            ta_fromfile = botslib.OldTransaction(row['idta'])
            ta_parsed = ta_fromfile.copyta(status=PARSED)       #make PARSED ta
            if row['filesize'] > botsglobal.ini.getint('settings','maxfilesizeincoming',5000000):
                ta_parsed.update(filesize=row['filesize'])
                raise botslib.InMessageError(_(u'File size of %(filesize)s is too big; option "maxfilesizeincoming" in bots.ini is %(maxfilesizeincoming)s.'),
                                                {'filesize':row['filesize'],'maxfilesizeincoming':botsglobal.ini.getint('settings','maxfilesizeincoming',5000000)})
            botsglobal.logger.debug(_(u'Start translating file "%(filename)s" editype "%(editype)s" messagetype "%(messagetype)s".'),row)
            #read whole edi-file: read, parse and made into a inmessage-object. Message is represented as a tree (inmessage.root is the root of the tree).
            edifile = inmessage.parse_edi_file(frompartner=row['frompartner'],
                                                topartner=row['topartner'],
                                                filename=row['filename'],
                                                messagetype=row['messagetype'],
                                                testindicator=row['testindicator'],
                                                editype=row['editype'],
                                                charset=row['charset'],
                                                alt=row['alt'],
                                                fromchannel=row['fromchannel'],
                                                idroute=idroute)
            #if no exception: infile has been lexed and parsed OK.
            #edifile.ta_info contains info: QUERIES, charset etc
            for inn_splitup in edifile.nextmessage():   #splitup messages in parsed edifile
                try:
                    ta_splitup = ta_parsed.copyta(status=SPLITUP,**inn_splitup.ta_info)    #copy PARSED to SPLITUP ta
                    #inn_splitup.ta_info: parameters from inmessage.parse_edi_file(), syntax-information and parse-information
                    inn_splitup.ta_info['idta_fromfile'] = ta_fromfile.idta     #for confirmations in userscript; used to give idta of 'confirming message'
                    post_mapping_mode = False       #if post_mapping: reuse out-object, if no translation is found no error
                    number_of_loops_with_same_alt = 0
                    while 1:    #continue as long as there are (alt-)translations
                        #lookup the translation************************
                        tscript,toeditype,tomessagetype = botslib.lookup_translation(fromeditype=inn_splitup.ta_info['editype'],
                                                                            frommessagetype=inn_splitup.ta_info['messagetype'],
                                                                            frompartner=inn_splitup.ta_info['frompartner'],
                                                                            topartner=inn_splitup.ta_info['topartner'],
                                                                            alt=inn_splitup.ta_info['alt'])
                        if not tscript:       #no translation found in translate table; check if can find translation via user script
                            if userscript and hasattr(userscript,'gettranslation'):
                                tscript,toeditype,tomessagetype = botslib.runscript(userscript,scriptname,'gettranslation',idroute=idroute,message=inn_splitup)
                            if not tscript:
                                if post_mapping_mode:   #in post-mapping mode, not finding the (partern specific!) script is no problem: translation is done
                                    #there is no post-mapping script found (in other words, default mapping script is enough).
                                    #all OK, handle the generated out-file and continue with next message
                                    botsglobal.logger.debug(_(u'Found no "post-mapping" translation for editype "%(editype)s", messagetype "%(messagetype)s", frompartner "%(frompartner)s", topartner "%(topartner)s", alt "%(alt)s".'),
                                                                inn_splitup.ta_info)
                                    handle_out_message(out_translated,ta_translated)
                                    del out_translated
                                    break   #break out of while loop
                                else:
                                    raise botslib.TranslationNotFoundError(_(u'Translation not found for editype "%(editype)s", messagetype "%(messagetype)s", frompartner "%(frompartner)s", topartner "%(topartner)s", alt "%(alt)s".'),
                                                                                inn_splitup.ta_info)

                        inn_splitup.ta_info['divtext'] = tscript     #ifor reporting used mapping script to database (for display in GUI).
                        if not post_mapping_mode:
                            #initialize new out-object*************************
                            ta_translated = ta_splitup.copyta(status=endstatus)     #make ta for translated message (new out-ta)
                            filename_translated = str(ta_translated.idta)
                            out_translated = outmessage.outmessage_init(editype=toeditype,messagetype=tomessagetype,filename=filename_translated,reference=unique('messagecounter'),statust=OK,divtext=tscript)    #make outmessage object
                            
                        #run mapping script************************
                        botsglobal.logger.debug(_(u'Mappingscript "%(tscript)s" translates messagetype "%(messagetype)s" to messagetype "%(tomessagetype)s".'),
                                                {'tscript':tscript,'messagetype':inn_splitup.ta_info['messagetype'],'tomessagetype':out_translated.ta_info['messagetype']})
                        translationscript,scriptfilename = botslib.botsimport('mappings',inn_splitup.ta_info['editype'],tscript) #get the mappingscript
                        alt_from_previous_run = inn_splitup.ta_info['alt']      #needed to check for infinite loop
                        doalttranslation = botslib.runscript(translationscript,scriptfilename,'main',inn=inn_splitup,out=out_translated)
                        botsglobal.logger.debug(_(u'Mappingscript "%(tscript)s" finished.'),{'tscript':tscript})
                        
                        #manipulate for some attributes after mapping script
                        if 'topartner' not in out_translated.ta_info:    #out_translated does not contain values from ta......
                            out_translated.ta_info['topartner'] = inn_splitup.ta_info['topartner']
                        if 'botskey' in inn_splitup.ta_info:
                            inn_splitup.ta_info['reference'] = inn_splitup.ta_info['botskey']
                        if 'botskey' in out_translated.ta_info:    #out_translated does not contain values from ta......
                            out_translated.ta_info['reference'] = out_translated.ta_info['botskey']
                            
                        #check the value received from the mappingscript to determine what to do in this while-loop. Handling of chained trasnlations.
                        if doalttranslation is None:    
                            #translation(s) are done; handle out-message 
                            handle_out_message(out_translated,ta_translated)
                            del out_translated
                            break   #break out of while loop
                        elif isinstance(doalttranslation,dict):
                            #some extended cases; a dict is returned that contains 'instructions' for some type of chained translations
                            if 'type' not in doalttranslation or 'alt' not in doalttranslation:
                                raise botslib.BotsError(_(u"Mappingscript returned '%(alt)s'. This dict should not have 'type' and 'alt'."),{'alt':doalttranslation})
                            if alt_from_previous_run == doalttranslation['alt']:
                                number_of_loops_with_same_alt += 1
                            else:
                                number_of_loops_with_same_alt = 0
                            if doalttranslation['type'] == u'out_as_inn':
                                #do chained translation: use the out-object as inn-object, new out-object
                                #use case: detected error in incoming file; use out-object to generate warning email
                                handle_out_message(out_translated,ta_translated)
                                inn_splitup = out_translated    #out-object is now inn-object
                                if isinstance(inn_splitup,outmessage.fixed):    #for fixed: strip all values in node
                                    inn_splitup.root.stripnode()
                                inn_splitup.ta_info['alt'] = doalttranslation['alt']   #get the alt-value for the next chained translation
                                if not 'frompartner' in inn_splitup.ta_info:
                                    inn_splitup.ta_info['frompartner'] = ''
                                if not 'topartner' in inn_splitup.ta_info:
                                    inn_splitup.ta_info['topartner'] = ''
                                inn_splitup.ta_info.pop('statust')
                            elif doalttranslation['type'] == u'no_check_on_infinite_loop':
                                #do chained translation: allow many loops wit hsame alt-value.
                                #mapping script will have to handle this correctly.
                                number_of_loops_with_same_alt = 0
                                handle_out_message(out_translated,ta_translated)
                                del out_translated
                                inn_splitup.ta_info['alt'] = doalttranslation['alt']   #get the alt-value for the next chained translation
                            elif doalttranslation['type'] == u'post_mapping':
                                #do chained  (post)translation: same inn and out-objects.
                                #in other words, the post-translation continue where is 'default' mapping ended.
                                #use case: default mapping (for all partners), partner-specific post-translation
                                #note: this is easier via import defaultmapping in a partner specific mapping. Please use that method!
                                post_mapping_mode = True       #translation is done; reset post_mapping_mode
                                inn_splitup.ta_info['alt'] = doalttranslation['alt']   #get the alt-value for the next chained translation
                                #~ inn_splitup.ta_info.pop('statust')
                            else:   #there is nothing else
                                raise botslib.BotsError(_(u'Mappingscript returned dict with an unknown "type": "%(doalttranslation)s".'),{'doalttranslation':doalttranslation})
                        else:  #note: this includes alt '' (empty string)
                            if alt_from_previous_run == doalttranslation:
                                number_of_loops_with_same_alt += 1
                            else:
                                number_of_loops_with_same_alt = 0
                            #do normal chained translation: same inn-object, new out-object
                            handle_out_message(out_translated,ta_translated)
                            del out_translated
                            inn_splitup.ta_info['alt'] = doalttranslation   #get the alt-value for the next chained translation
                        if number_of_loops_with_same_alt > 10:
                            raise botslib.BotsError(_(u'Mappingscript returns same alt value over and over again (infinite loop?). Alt: "%(doalttranslation)s".'),{'doalttranslation':doalttranslation})
                    #end of while-loop (trans**********************************************************************************
                #exceptions file_out-level: exception in mappingscript or writing of out-file
                except:
                    #2 modes: either every error leads to skipping of  whole infile (old  mode) or errors in mappingscript/outfile only affect that branche
                    if botsglobal.ini.getboolean('settings','oldmessageerrors',False):
                        raise
                    txt = botslib.txtexc()
                    ta_splitup.update(statust=ERROR,errortext=txt,**inn_splitup.ta_info)   #update db. inn_splitup.ta_info could be changed by mappingscript. Is this useful?
                    ta_splitup.deletechildren()
                else:
                    ta_splitup.update(statust=DONE, **inn_splitup.ta_info)   #update db. inn_splitup.ta_info could be changed by mappingscript. Is this useful?


        #exceptions file_in-level (file not OK according to grammar)
        except:
            txt = botslib.txtexc()
            ta_parsed.update(statust=ERROR,errortext=txt)
            ta_parsed.deletechildren()
            botsglobal.logger.debug(u'Error in translating input file "%(filename)s":\n%(msg)s',{'filename':row['filename'],'msg':txt})
        else:
            edifile.handleconfirm(ta_fromfile,error=False)
            ta_parsed.update(statust=DONE,filesize=row['filesize'],**edifile.ta_info)
            botsglobal.logger.debug(_(u'Translated input file "%(filename)s".'),row)
        finally:
            ta_fromfile.update(statust=DONE)
Example #3
0
def start():
    #********command line arguments**************************
    usage = '''
    This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
    Creates a grammar from an xml file.'
    Usage:'
        %(name)s  -c<directory>  <xml_file>  <xml_grammar_file>
    Options:
        -c<directory>      directory for configuration files (default: config).
        <xml_file>         name of the xml file to read
        <xml_grammar_file> name of the grammar file to write
    
    '''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version}
    configdir = 'config'
    edifile =''
    grammarfile = ''
    for arg in sys.argv[1:]:
        if arg.startswith('-c'):
            configdir = arg[2:]
            if not configdir:
                print 'Error: configuration directory indicated, but no directory name.'
                sys.exit(1)
        elif arg in ["?", "/?",'-h', '--help'] or arg.startswith('-'):
            print usage
            sys.exit(0)
        else:
            if not edifile:
                edifile = arg
            else:
                grammarfile = arg
    if not edifile or not grammarfile:
        print 'Error: both edifile and grammarfile are required.'
        sys.exit(0)
    #***end handling command line arguments**************************
    botsinit.generalinit(configdir)     #find locating of bots, configfiles, init paths etc.
    process_name = 'xml2botsgrammar'
    botsglobal.logger = botsinit.initenginelogging(process_name)
    atexit.register(logging.shutdown)
    
    #the xml file is parsed as an xmlnocheck message
    editype = 'xmlnocheck'
    messagetype = 'xmlnocheckxxxtemporaryforxml2grammar'
    mpath = []
    #a (temp) xmlnocheck grammar is needed (but needs not actual content. This file is not removed.
    tmpgrammarfile = botslib.join(botsglobal.ini.get('directories','usersysabs'),'grammars',editype,messagetype+'.py')
    filehandler = open(tmpgrammarfile,'w')
    filehandler.close()
    
    #make inmessage object: read the xml file
    inn = inmessage.parse_edi_file(editype=editype,messagetype=messagetype,filename=edifile,remove_empties_from_xml=False)
    #make outmessage object; nothing is 'filled' yet.
    out = outmessage.outmessage_init(editype=editype,messagetype=messagetype,filename='botssys/infile/unitnode/output/inisout03.edi',divtext='',topartner='')    
    
    #***do the mapping***************************************************
    #handle root
    rootmpath = [{'BOTSID':inn.root.record['BOTSID'],'BOTSIDnr':'1'}]
    out.put(*rootmpath)
    map_writefields(out,inn.root,rootmpath)
    #walk tree; write results to out-tree
    for node_instance,mpath in map_treewalker(inn.root,mpath):
        mpath.append({'BOTSID':node_instance.record['BOTSID']})
        if out.get(*mpath) is None:
            out.put(*mpath)
        map_writefields(out,node_instance,mpath)

    #***mapping is done; out-tree is finished; represents 'normalised' tree suited for writing as a grammar
    structure = []
    recorddefs = {}
    tree2grammar(out.root,structure,recorddefs)
    #~ for key,value in recorddefs.items():
        #~ print key,value
        #~ print '\n'
    sortedstructurelist = structure2list(structure)
    recorddefsstring = recorddefs2string(recorddefs,sortedstructurelist)
    structurestring = structure2string(structure)

    #write grammar file
    grammar = open(grammarfile,'wb')
    grammar.write('#grammar automatically generated by bots open source edi translator.')
    grammar.write('\n')
    grammar.write('from bots.botsconfig import *')
    grammar.write('\n\n')
    grammar.write('syntax = {}')
    grammar.write('\n\n')
    grammar.write('structure = [\n%s]\n'%(structurestring))
    grammar.write('\n\n')
    grammar.write('recorddefs = %s'%(recorddefsstring))
    grammar.write('\n\n')
    grammar.close()
    print 'grammar file is written:',grammarfile
Example #4
0
def translate(run):
    for messagedict in run.incoming:
        try:
            #read whole edi-file: read, parse and made into a inmessage-object. Message is represented as a tree (inmessage.root is the root of the tree).
            edifile = inmessage.parse_edi_file(frompartner='',
                                                topartner='',
                                                filename=messagedict['filename'],
                                                messagetype=run.translation['messagetype'],
                                                testindicator='',
                                                editype=run.translation['editype'],
                                                charset='',
                                                alt='',
                                                fromchannel='',
                                                idroute='',
                                                command='')
            edifile.checkforerrorlist() #no exception if infile has been lexed and parsed OK else raises an error

            #~ if int(routedict['translateind']) == 3: #parse & passthrough; file is parsed, partners are known, no mapping, does confirm. 
                #~ raise botslib.GotoException('dummy')
            #~ continue    #file is parsed; no errors. no translation
            
            #edifile.ta_info contains info about incoming file: QUERIES, charset etc
            for inn_splitup in edifile.nextmessage():   #splitup messages in parsed edifile
                try:
                    #inn_splitup.ta_info: parameters from inmessage.parse_edi_file(), syntax-information and parse-information
                    number_of_loops_with_same_alt = 0
                    while 1:    #continue as long as there are (alt-)translations
                        #lookup the translation************************
                        tscript,toeditype,tomessagetype = 'orders_edifact2xml' ,'xml','orders'
                        if 'tscript' in run.translation:
                            tscript = run.translation['tscript']
                            toeditype = run.translation['toeditype']
                            tomessagetype = run.translation['tomessagetype']
                        else:
                            tscript,toeditype,tomessagetype = botslib.lookup_translation(fromeditype=inn_splitup.ta_info['editype'],
                                                                                frommessagetype=inn_splitup.ta_info['messagetype'],
                                                                                frompartner=inn_splitup.ta_info['frompartner'],
                                                                                topartner=inn_splitup.ta_info['topartner'],
                                                                                alt=inn_splitup.ta_info['alt'])
                            
                        #run mapping script************************
                        filename_translated = transform.unique('bots_file_name')
                        out_translated = outmessage.outmessage_init(editype=toeditype,
                                                                    messagetype=tomessagetype,
                                                                    filename=filename_translated,
                                                                    reference=transform.unique('messagecounter'),
                                                                    statust=OK,
                                                                    divtext=tscript)    #make outmessage object
                        
                        #~ botsglobal.logger.debug(_(u'Mappingscript "%(tscript)s" translates messagetype "%(messagetype)s" to messagetype "%(tomessagetype)s".'),
                                                #~ {'tscript':tscript,'messagetype':inn_splitup.ta_info['messagetype'],'tomessagetype':out_translated.ta_info['messagetype']})
                        translationscript,scriptfilename = botslib.botsimport('mappings',inn_splitup.ta_info['editype'],tscript)    #import mappingscript
                        alt_from_previous_run = inn_splitup.ta_info['alt']      #needed to check for infinite loop
                        doalttranslation = botslib.runscript(translationscript,scriptfilename,'main',inn=inn_splitup,out=out_translated)
                        botsglobal.logger.debug(_(u'Mappingscript "%(tscript)s" finished.'),{'tscript':tscript})
                        
                        #manipulate for some attributes after mapping script
                        if 'topartner' not in out_translated.ta_info:    #out_translated does not contain values from run......
                            out_translated.ta_info['topartner'] = inn_splitup.ta_info['topartner']
                        if 'botskey' in inn_splitup.ta_info:
                            inn_splitup.ta_info['reference'] = inn_splitup.ta_info['botskey']
                        if 'botskey' in out_translated.ta_info:    #out_translated does not contain values from run......
                            out_translated.ta_info['reference'] = out_translated.ta_info['botskey']
                            
                        #check the value received from the mappingscript to determine what to do in this while-loop. Handling of chained trasnlations.
                        if doalttranslation is None:    
                            #translation(s) are done; handle out-message 
                            out_translated.writeall()   #write result of translation.
                            #make translated record (if all is OK)
                            translated_dict = inn_splitup.ta_info.copy()
                            translated_dict.update(messagedict)
                            translated_dict.update(out_translated.ta_info)
                            run.translated.append(translated_dict)
                            del out_translated
                            break   #break out of while loop
                        elif isinstance(doalttranslation,dict):
                            #some extended cases; a dict is returned that contains 'instructions' for some type of chained translations
                            if 'type' not in doalttranslation or 'alt' not in doalttranslation:
                                raise botslib.BotsError(_(u"Mappingscript returned '%(alt)s'. This dict should not have 'type' and 'alt'."),{'alt':doalttranslation})
                            if alt_from_previous_run == doalttranslation['alt']:
                                number_of_loops_with_same_alt += 1
                            else:
                                number_of_loops_with_same_alt = 0
                            if doalttranslation['type'] == u'out_as_inn':
                                #do chained translation: use the out-object as inn-object, new out-object
                                #use case: detected error in incoming file; use out-object to generate warning email
                                handle_out_message(out_translated,ta_translated)
                                inn_splitup = out_translated    #out-object is now inn-object
                                if isinstance(inn_splitup,outmessage.fixed):    #for fixed: strip all values in node
                                    inn_splitup.root.stripnode()
                                inn_splitup.ta_info['alt'] = doalttranslation['alt']   #get the alt-value for the next chained translation
                                if not 'frompartner' in inn_splitup.ta_info:
                                    inn_splitup.ta_info['frompartner'] = ''
                                if not 'topartner' in inn_splitup.ta_info:
                                    inn_splitup.ta_info['topartner'] = ''
                                inn_splitup.ta_info.pop('statust')
                            elif doalttranslation['type'] == u'no_check_on_infinite_loop':
                                #do chained translation: allow many loops wit hsame alt-value.
                                #mapping script will have to handle this correctly.
                                number_of_loops_with_same_alt = 0
                                handle_out_message(out_translated,ta_translated)
                                del out_translated
                                inn_splitup.ta_info['alt'] = doalttranslation['alt']   #get the alt-value for the next chained translation
                            else:   #there is nothing else
                                raise botslib.BotsError(_(u'Mappingscript returned dict with an unknown "type": "%(doalttranslation)s".'),{'doalttranslation':doalttranslation})
                        else:  #note: this includes alt '' (empty string)
                            if alt_from_previous_run == doalttranslation:
                                number_of_loops_with_same_alt += 1
                            else:
                                number_of_loops_with_same_alt = 0
                            #do normal chained translation: same inn-object, new out-object
                            out_translated.writeall()
                            del out_translated
                            inn_splitup.ta_info['alt'] = doalttranslation   #get the alt-value for the next chained translation
                        if number_of_loops_with_same_alt > 10:
                            raise botslib.BotsError(_(u'Mappingscript returns same alt value over and over again (infinite loop?). Alt: "%(doalttranslation)s".'),{'doalttranslation':doalttranslation})
                    #end of while-loop (trans**********************************************************************************
                #exceptions file_out-level: exception in mappingscript or writing of out-file
                except:
                    #2 modes: either every error leads to skipping of  whole infile (old  mode) or errors in mappingscript/outfile only affect that branche
                    txt = botslib.txtexc()
                    print txt
                    messagedict['error'] += txt.strip()
                else:
                    pass
                    #~ print 'succes'
        #exceptions file_in-level
        except botslib.GotoException:   #edi-file is OK, file is passed-through after parsing.
            #~ edifile.handleconfirm(ta_fromfile,error=False)
            #~ botsglobal.logger.debug(_(u'Parse & passthrough for input file "%(filename)s".'),row)
            txt = botslib.txtexc()
            print txt
        except:
            txt = botslib.txtexc()
            messagedict['error'] += txt.strip()
            #~ edifile.handleconfirm(ta_fromfile,error=True)
            #~ botsglobal.logger.debug(u'Error in translating input file "%(filename)s":\n%(msg)s',{'filename':row['filename'],'msg':txt})
        else:
            pass
Example #5
0
def start():
    #********command line arguments**************************
    usage = '''
    This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
    Creates a grammar from an xml file.'
    Usage:'
        %(name)s  -c<directory>  <xml_file>  <xml_grammar_file>
    Options:
        -c<directory>      directory for configuration files (default: config).
        -a                 all xml elements as records
        <xml_file>         name of the xml file to read
        <xml_grammar_file> name of the grammar file to write
    
    '''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version}
    configdir = 'config'
    edifile =''
    botsgrammarfilename = ''
    allrecords = False
    for arg in sys.argv[1:]:
        if arg.startswith('-c'):
            configdir = arg[2:]
            if not configdir:
                print 'Error: configuration directory indicated, but no directory name.'
                sys.exit(1)
        elif arg.startswith('-a'):
            allrecords = True
        elif arg in ["?", "/?",'-h', '--help'] or arg.startswith('-'):
            print usage
            sys.exit(0)
        else:
            if not edifile:
                edifile = arg
            else:
                botsgrammarfilename = arg
    if not edifile or not botsgrammarfilename:
        print 'Error: both edifile and grammarfile are required.'
        sys.exit(0)
    #***end handling command line arguments**************************
    botsinit.generalinit(configdir)     #find locating of bots, configfiles, init paths etc.
    process_name = 'xml2botsgrammar'
    botsglobal.logger = botsinit.initenginelogging(process_name)
    atexit.register(logging.shutdown)

    targetNamespace = ''
    #*******************************************************************
    #***add classes for handling editype xml to inmessage
    #*******************************************************************
    if allrecords:
        #~ editype = 'xmlforgrammar_allrecords'
        inmessage.xmlforgrammar = xmlforgrammar_allrecords
    else:
        #~ editype = 'xmlforgrammar' 
        inmessage.xmlforgrammar = xmlforgrammar
    #make inmessage object: read the xml file
    inn = inmessage.parse_edi_file(editype='xmlforgrammar',messagetype='',filename=edifile)
    inn.checkforerrorlist() #no exception if infile has been lexed and parsed OK else raises an error
    #make outmessage object; nothing is 'filled' yet. In mapping tree is filled; nothing is written to file.
    out = outmessage.outmessage_init(editype='xmlnocheck',messagetype='',filename='',divtext='',topartner='')    
    
    #***mapping: make 'normalised' out-tree suited for writing as a grammar***************************************************
    mpath_root = [OrderedDict({'BOTSID':inn.root.record['BOTSID'],'BOTSIDnr':'1'})] #handle root
    out.put(*mpath_root)
    map_writefields(out,inn.root,mpath_root)
    
    #walk tree; write results to out-tree
    mpath_start = []
    for node_instance,mpath in map_treewalker(inn.root,mpath_start):
        mpath.append(OrderedDict({'BOTSID':node_instance.record['BOTSID']}))
        if out.get(*mpath) is None:     #if node does not exist: write it.
            out.put(*mpath)
        map_writefields(out,node_instance,mpath)
    #***mapping is done 

    #***convert out-tree to grammar
    structure = []
    recorddefs = {}
    tree2grammar(out.root,structure,recorddefs)

    #***write grammar to file
    grammar2file(botsgrammarfilename,structure,recorddefs,targetNamespace)