示例#1
0
def read_index(filename):
    """ process index file in default location. """
    try:
        importedbotsindex, scriptname = botslib.botsimport("index")
        pluglist = importedbotsindex.plugins[:]
        if importedbotsindex.__name__ in sys.modules:
            del sys.modules[importedbotsindex.__name__]
    except:
        txt = botslib.txtexc()
        raise botslib.PluginError(
            _(u"Error in configuration index file. Nothing is written. Error:\n%(txt)s"), {"txt": txt}
        )
    else:
        botsglobal.logger.info(_(u"Configuration index file is OK."))
        botsglobal.logger.info(_(u"Start writing to database."))

    # write content of index file to the bots database
    try:
        read_index2database(pluglist)
    except:
        txt = botslib.txtexc()
        raise botslib.PluginError(
            _(u"Error writing configuration index to database. Nothing is written. Error:\n%(txt)s"), {"txt": txt}
        )
    else:
        botsglobal.logger.info(_(u"Writing to database is OK."))
示例#2
0
 def run(self):
     try:
         from genshi.template import TemplateLoader
     except:
         raise ImportError(u'Dependency failure: editype "templatehtml" requires python library "genshi".')
     self._openoutenvelope()
     self.ta_info.update(self.out.ta_info)
     botslib.tryrunscript(self.userscript,self.scriptname,'ta_infocontent',ta_info=self.ta_info)
     if not self.ta_info['envelope-template']:
         raise botslib.OutMessageError(_(u'While enveloping in "%(editype)s.%(messagetype)s": syntax option "envelope-template" not filled; is required.'),
                                         self.ta_info)
     templatefile = botslib.abspath(self.__class__.__name__,self.ta_info['envelope-template'])
     ta_list = self.filelist2absolutepaths()
     try:
         botsglobal.logger.debug(u'Start writing envelope to file "%(filename)s".',self.ta_info)
         loader = TemplateLoader(auto_reload=False)
         tmpl = loader.load(templatefile)
     except:
         txt = botslib.txtexc()
         raise botslib.OutMessageError(_(u'While enveloping in "%(editype)s.%(messagetype)s", error:\n%(txt)s'),
                                     {'editype':self.ta_info['editype'],'messagetype':self.ta_info['messagetype'],'txt':txt})
     try:
         filehandler = botslib.opendata(self.ta_info['filename'],'wb')
         stream = tmpl.generate(data=ta_list)
         stream.render(method='xhtml',encoding=self.ta_info['charset'],out=filehandler)
     except:
         txt = botslib.txtexc()
         raise botslib.OutMessageError(_(u'While enveloping in "%(editype)s.%(messagetype)s", error:\n%(txt)s'),
                                     {'editype':self.ta_info['editype'],'messagetype':self.ta_info['messagetype'],'txt':txt})
示例#3
0
def runengine(request, *kw, **kwargs):
    if request.method == "GET":
        # ~ logger = logging.getLogger('bots')
        if os.name == "nt":
            lijst = [sys.executable, os.path.normpath(os.path.join(sys.prefix, "Scripts", "bots-engine.py"))]
        elif os.path.exists(os.path.join(sys.prefix, "bin", "bots-engine.py")):
            lijst = [os.path.normpath(os.path.join(sys.prefix, "bin", "bots-engine.py"))]
        elif os.path.exists(os.path.join(sys.prefix, "local/bin", "bots-engine.py")):
            lijst = [os.path.normpath(os.path.join(sys.prefix, "local/bin", "bots-engine.py"))]
        else:
            request.user.message_set.create(message=_(u"Bots can not find executable for bots-engine."))
            # ~ logger.info('Bots can not find executable for bots-engine.')
            return django.shortcuts.redirect("/")

        try:
            if "clparameter" in request.GET:
                lijst.append(request.GET["clparameter"])
            # ~ logger.info('Run bots-engine with parameters: "%s"',str(lijst))
            terug = subprocess.Popen(lijst).pid
            request.user.message_set.create(message=_(u"Bots-engine is started."))
            # ~ logger.info('Bots-engine is started.')
        except:
            print botslib.txtexc()
            request.user.message_set.create(message=_(u"Errors while trying to run bots-engine."))
            # ~ logger.info('Errors while trying to run bots-engine.')
    return django.shortcuts.redirect("/")
示例#4
0
 def run(self):
     ''' class for (test) orderprint; delevers a valid html-file.
         Uses a kid-template for the enveloping/merging.
         use kid to write; no envelope grammar is used
     '''
     import kid
     defmessage = grammar.grammarread(self.ta_info['editype'],self.ta_info['messagetype'])   #needed beause we do not know envelope; read syntax for editype/messagetype
     self.ta_info.update(defmessage.syntax)
     botslib.tryrunscript(self.userscript,self.scriptname,'ta_infocontent',ta_info=self.ta_info)
     if not self.ta_info['envelope-template']:
         raise botslib.OutMessageError(_(u'While enveloping in "$editype.$messagetype": syntax option "envelope-template" not filled; is required.'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'])
     templatefile = botslib.abspath('templates',self.ta_info['envelope-template'])
     ta_list = self.filelist2absolutepaths()
     try:
         botsglobal.logger.debug(u'Start writing envelope to file "%s".',self.ta_info['filename'])
         ediprint = kid.Template(file=templatefile, data=ta_list) #init template; pass list with filenames
     except:
         txt=botslib.txtexc()
         raise botslib.OutMessageError(_(u'While enveloping in "$editype.$messagetype": $txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt)
     try:
         f = botslib.opendata(self.ta_info['filename'],'wb')
         ediprint.write(f,
                         encoding=self.ta_info['charset'],
                         output=self.ta_info['output'])
     except:
         txt=botslib.txtexc()
         raise botslib.OutMessageError(_(u'While enveloping error in "$editype.$messagetype"; probably in html file(s) to be merged: $txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt)
示例#5
0
 def writeall(self):
     ''' Very different writeall:
         there is no tree of nodes; there is no grammar.structure/recorddefs; kid opens file by itself.
     '''
     try:
         import kid
     except:
         txt = botslib.txtexc()
         raise ImportError(_(u'Dependency failure: editype "template" requires python library "kid". Error:\n%s'%txt))
     #for template-grammar: only syntax is used. Section 'syntax' has to have 'template'
     self.outmessagegrammarread(self.ta_info['editype'],self.ta_info['messagetype'])
     templatefile = botslib.abspath(u'templates',self.ta_info['template'])
     try:
         botsglobal.logger.debug(u'Start writing to file "%s".',self.ta_info['filename'])
         ediprint = kid.Template(file=templatefile, data=self.data)
     except:
         txt = botslib.txtexc()
         raise botslib.OutMessageError(_(u'While templating "$editype.$messagetype", error:\n$txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt)
     try:
         filehandler = botslib.opendata(self.ta_info['filename'],'wb')
         ediprint.write(filehandler,
                         encoding=self.ta_info['charset'],
                         output=self.ta_info['output'],    #output is specific parameter for class; init from grammar.syntax
                         fragment=self.ta_info['merge'])
     except:
         txt = botslib.txtexc()
         raise botslib.OutMessageError(_(u'While templating "$editype.$messagetype", error:\n$txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt)
     botsglobal.logger.debug(_(u'End writing to file "%s".'),self.ta_info['filename'])
示例#6
0
def start():
    #********command line arguments**************************
    editype =''
    messagetype = ''
    configdir = 'config'
    for arg in sys.argv[1:]:
        if not arg:
            continue
        if arg.startswith('-c'):
            configdir = arg[2:]
            if not configdir:
                print '    !!Indicated Bots should use specific .ini file but no file name was given.'
                showusage()
        elif arg in ["?", "/?"] or arg.startswith('-'):
            showusage()
        else:
            if not editype:
                editype = arg
            else:
                messagetype = arg
    if not (editype and messagetype):
        print '    !!Both editype and messagetype are required.'
        showusage()
    #********end handling command line arguments**************************

    try:
        botsinit.generalinit(configdir)
        botsinit.initenginelogging()
        grammar.grammarread(editype,messagetype)
    except:
        print 'Found error in grammar:'
        print botslib.txtexc()
    else:
        print 'OK - no error found in grammar'
示例#7
0
文件: envelope.py 项目: divadrei/bots
 def run(self):
     ''' class for template enveloping; delevers a valid html-file.
         Uses a kid-template for the enveloping/merging.
         use kid to write; no envelope grammar is used
         #20120101 depreciated. use class templatehtml
     '''
     try:
         import kid
     except:
         raise ImportError(_(u'Dependency failure: editype "template" requires python library "kid".'))
     self._openoutenvelope()
     self.ta_info.update(self.out.ta_info)
     botslib.tryrunscript(self.userscript,self.scriptname,'ta_infocontent',ta_info=self.ta_info)
     if not self.ta_info['envelope-template']:
         raise botslib.OutMessageError(_(u'While enveloping in "%(editype)s.%(messagetype)s": syntax option "envelope-template" not filled; is required.'),
                                         self.ta_info)
     templatefile = botslib.abspath('templates',self.ta_info['envelope-template'])
     ta_list = self.filelist2absolutepaths()
     try:
         botsglobal.logger.debug(u'Start writing envelope to file "%(filename)s".',self.ta_info)
         ediprint = kid.Template(file=templatefile, data=ta_list) #init template; pass list with filenames
     except:
         txt = botslib.txtexc()
         raise botslib.OutMessageError(_(u'While enveloping in "%(editype)s.%(messagetype)s", error:\n%(txt)s'),
                                         {'editype':self.ta_info['editype'],'messagetype':self.ta_info['messagetype'],'txt':txt})
     try:
         filehandler = botslib.opendata(self.ta_info['filename'],'wb')
         ediprint.write(filehandler,
                         encoding=self.ta_info['charset'],
                         output=self.ta_info['output'])
     except:
         txt = botslib.txtexc()
         raise botslib.OutMessageError(_(u'While enveloping in "%(editype)s.%(messagetype)s", error:\n%(txt)s'),
                                         {'editype':self.ta_info['editype'],'messagetype':self.ta_info['messagetype'],'txt':txt})
示例#8
0
def start():
    # ********command line arguments**************************
    editype = ""
    messagetype = ""
    botsinifile = "config"
    for arg in sys.argv[1:]:
        if not arg:
            continue
        if arg.startswith("-c"):
            botsinifile = arg[2:]
            if not botsinifile:
                print "    !!Indicated Bots should use specific .ini file but no file name was given."
                showusage()
        elif arg in ["?", "/?"] or arg.startswith("-"):
            showusage()
        else:
            if not editype:
                editype = arg
            else:
                messagetype = arg
    if not (editype and messagetype):
        print "    !!Both editype and messagetype are required."
        showusage()
    # ********end handling command line arguments**************************

    try:
        botslib.initconfigurationfile(botsinifile)
        botslib.initlogging()
        grammar.grammarread(editype, messagetype)
    except:
        print "Found error in grammar:"
        print botslib.txtexc()
    else:
        print "OK - no error found in grammar"
示例#9
0
 def writeall(self):
     ''' Very different writeall:
         there is no tree of nodes; there is no grammar.structure/recorddefs; kid opens file by itself.
     '''
     try:
         from genshi.template import TemplateLoader
     except:
         txt = botslib.txtexc()
         raise ImportError(_(u'Dependency failure: editype "template" requires python library "genshi". Error:\n%s'%txt))
     #for template-grammar: only syntax is used. Section 'syntax' has to have 'template'
     self.outmessagegrammarread(self.ta_info['editype'],self.ta_info['messagetype'])
     templatefile = botslib.abspath(u'templateshtml',self.ta_info['template'])
     try:
         botsglobal.logger.debug(u'Start writing to file "%s".',self.ta_info['filename'])
         loader = TemplateLoader(auto_reload=False)
         tmpl = loader.load(templatefile)
     except:
         txt = botslib.txtexc()
         raise botslib.OutMessageError(_(u'While templating "$editype.$messagetype", error:\n$txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt)
     try:
         filehandler = botslib.opendata(self.ta_info['filename'],'wb')
         stream = tmpl.generate(data=self.data)
         stream.render(method='xhtml',encoding=self.ta_info['charset'],out=filehandler)
     except:
         txt = botslib.txtexc()
         raise botslib.OutMessageError(_(u'While templating "$editype.$messagetype", error:\n$txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt)
     botsglobal.logger.debug(_(u'End writing to file "%s".'),self.ta_info['filename'])
示例#10
0
def startmulti(grammardir,editype):
    ''' specialized tool for bulk checking of grammars while developing botsgrammars 
        grammardir: directory with gramars (eg bots/usersys/grammars/edifact)
        editype: eg edifact
    '''
    configdir = 'config'
    botsinit.generalinit(configdir)     #find locating of bots, configfiles, init paths etc.
    process_name = 'grammarcheck'
    botsglobal.logger = botsinit.initenginelogging(process_name)
    atexit.register(logging.shutdown)
    
    for filename in glob.iglob(grammardir):
        filename_basename = os.path.basename(filename)
        if filename_basename in ['__init__.py','envelope.py']:
            continue
        if filename_basename.startswith('edifact') or filename_basename.startswith('records') or filename_basename.endswith('records.py'):
            continue
        if filename_basename.endswith('pyc'):
            continue
        filename_noextension = os.path.splitext(filename_basename)[0]
        try:
            grammar.grammarread(editype,filename_noextension)
        except:
            print botslib.txtexc()
            print '\n'
        else:
            print 'OK - no error found in grammar',filename,'\n'
示例#11
0
 def run(self):
     ''' class for (test) orderprint; delevers a valid html-file.
         Uses a kid-template for the enveloping/merging.
         use kid to write; no envelope grammar is used
     '''
     try:
         from genshi.template import TemplateLoader
     except:
         txt=botslib.txtexc()
         raise ImportError(_(u'Dependency failure: editype "template" requires python library "genshi". Error:\n%s'%txt))
     defmessage = grammar.grammarread(self.ta_info['editype'],self.ta_info['messagetype'])   #needed because we do not know envelope; read syntax for editype/messagetype
     self.ta_info.update(defmessage.syntax)
     botslib.tryrunscript(self.userscript,self.scriptname,'ta_infocontent',ta_info=self.ta_info)
     if not self.ta_info['envelope-template']:
         raise botslib.OutMessageError(_(u'While enveloping in "$editype.$messagetype": syntax option "envelope-template" not filled; is required.'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'])
     templatefile = botslib.abspath('templateshtml',self.ta_info['envelope-template'])
     ta_list = self.filelist2absolutepaths()
     try:
         botsglobal.logger.debug(u'Start writing envelope to file "%s".',self.ta_info['filename'])
         loader = TemplateLoader(auto_reload=False)
         tmpl = loader.load(templatefile)
     except:
         txt=botslib.txtexc()
         raise botslib.OutMessageError(_(u'While enveloping in "$editype.$messagetype", error:\n$txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt)
     try:
         f = botslib.opendata(self.ta_info['filename'],'wb')
         stream = tmpl.generate(data=ta_list)
         stream.render(method='xhtml',encoding=self.ta_info['charset'],out=f)
     except:
         txt=botslib.txtexc()
         raise botslib.OutMessageError(_(u'While enveloping in "$editype.$messagetype", error:\n$txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt)
示例#12
0
 def run(self):
     """ class for (test) orderprint; delevers a valid html-file.
         Uses a kid-template for the enveloping/merging.
         use kid to write; no envelope grammar is used
     """
     try:
         import kid
     except:
         txt = botslib.txtexc()
         raise ImportError(
             _(u'Dependency failure: editype "template" requires python library "kid". Error:\n%s' % txt)
         )
     defmessage = grammar.grammarread(
         self.ta_info["editype"], self.ta_info["messagetype"]
     )  # needed because we do not know envelope; read syntax for editype/messagetype
     self.ta_info.update(defmessage.syntax)
     botslib.tryrunscript(self.userscript, self.scriptname, "ta_infocontent", ta_info=self.ta_info)
     if not self.ta_info["envelope-template"]:
         raise botslib.OutMessageError(
             _(
                 u'While enveloping in "$editype.$messagetype": syntax option "envelope-template" not filled; is required.'
             ),
             editype=self.ta_info["editype"],
             messagetype=self.ta_info["messagetype"],
         )
     templatefile = botslib.abspath("templates", self.ta_info["envelope-template"])
     ta_list = self.filelist2absolutepaths()
     try:
         botsglobal.logger.debug(u'Start writing envelope to file "%s".', self.ta_info["filename"])
         ediprint = kid.Template(file=templatefile, data=ta_list)  # init template; pass list with filenames
     except:
         txt = botslib.txtexc()
         raise botslib.OutMessageError(
             _(u'While enveloping in "$editype.$messagetype", error:\n$txt'),
             editype=self.ta_info["editype"],
             messagetype=self.ta_info["messagetype"],
             txt=txt,
         )
     try:
         filehandler = botslib.opendata(self.ta_info["filename"], "wb")
         ediprint.write(filehandler, encoding=self.ta_info["charset"], output=self.ta_info["output"])
     except:
         txt = botslib.txtexc()
         raise botslib.OutMessageError(
             _(u'While enveloping in "$editype.$messagetype", error:\n$txt'),
             editype=self.ta_info["editype"],
             messagetype=self.ta_info["messagetype"],
             txt=txt,
         )
示例#13
0
def postprocess(routedict,function,status=FILEOUT,rootidta=None,**argv):
    ''' for postprocessing of files.
        these are NOT translations; translation involve grammars, mapping scripts etc. think of eg:
        - zip files.
        If errors occur during processing, no ta are left with status FILEOUT !
        postprocess is called right before the out-communicatiation
    '''
    if rootidta is None:
        rootidta = botsglobal.currentrun.get_minta4query_routepart()
    nr_files = 0
    for row in botslib.query(u'''SELECT idta,filename
                                FROM ta
                                WHERE idta>%(rootidta)s
                                AND status=%(status)s
                                AND statust=%(statust)s
                                AND idroute=%(idroute)s
                                AND tochannel=%(tochannel)s
                                ''',
                                {'status':status,'statust':OK,'idroute':routedict['idroute'],'tochannel':routedict['tochannel'],'rootidta':rootidta}):
        try:
            botsglobal.logger.debug(u'Start postprocessing "%(name)s" for file "%(filename)s".',
                                    {'name':function.__name__,'filename':row['filename']})
            ta_from = botslib.OldTransaction(row['idta'])
            ta_from.filename = row['filename']
            function(ta_from=ta_from,endstatus=status,routedict=routedict,**argv)
        except:
            txt = botslib.txtexc()
            ta_from.update(statust=ERROR,errortext=txt)
            ta_from.deletechildren()
        else:
            botsglobal.logger.debug(u'OK postprocessing "%(name)s" for file "%(filename)s".',
                                    {'name':function.__name__,'filename':row['filename']})
            ta_from.update(statust=DONE)
            nr_files += 1
    return nr_files
示例#14
0
def mergemessages(run):
    names_envelope_criteria = ('editype','messagetype','frompartner','topartner','testindicator','charset','contenttype','envelope','rsrv3')
    merge_yes = {}
    merge_no = []       #can be non-unique: as a list
    #walk over run.translated; sort by envelope-criteria in 2 dicts
    for translated in run.translated:
        filename = translated.get('filename')
        infilename = translated.get('infilename')
        nrmessages = translated.get('nrmessages')
        if translated.get('merge'):
            env_criteria = tuple(translated.get(field) for field in names_envelope_criteria)
            if env_criteria in merge_yes:
                merge_yes[env_criteria][0].append(filename)
                merge_yes[env_criteria][1].append(infilename)
                merge_yes[env_criteria][2] += nrmessages
            else:
                merge_yes[env_criteria] = [[filename],[infilename],nrmessages]
        else:
            ta_info = dict((field,translated.get(field)) for field in names_envelope_criteria)
            ta_info['nrmessages'] = nrmessages
            merge_no.append((ta_info, [filename],[infilename]))
    #envelope
    for env_criteria,rest_of_info in merge_yes.iteritems():
        ta_info = dict(zip(names_envelope_criteria,env_criteria))
        ta_info['filename'] = transform.unique('bots_file_name')   #create filename for enveloped message
        ta_info['nrmessages'] = rest_of_info[2]
        ta_info['infilename'] = rest_of_info[1]      #for reference: list of infilenames
        ta_info['error'] = ''
        try:
            envelope.envelope(ta_info,rest_of_info[0])
        except:
            txt = botslib.txtexc()
            ta_info['error'] = txt.strip()
        finally:
            run.outgoing.append(ta_info)
    for ta_info,filenames,infilenames in merge_no:
        ta_info['filename'] = transform.unique('bots_file_name')   #create filename for enveloped message
        ta_info['infilename'] = infilenames      #for reference: list of infilenames
        ta_info['error'] = ''
        try:
            envelope.envelope(ta_info,filenames)
        except:
            txt = botslib.txtexc()
            ta_info['error'] = txt.strip()
        finally:
            run.outgoing.append(ta_info)
示例#15
0
def start():
    #NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
    #********command line arguments**************************
    usage = '''
    This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
    Checks a Bots grammar. Same checks are used as in translations with bots-engine. Searches for grammar in 
    regular place: bots/usersys/grammars/<editype>/<messagetype>.py  (even if a path is passed).
    
    Usage:  %(name)s  -c<directory> <editype> <messagetype>
       or   %(name)s  -c<directory> <path to grammar>
    Options:
        -c<directory>   directory for configuration files (default: config).
    Examples:
        %(name)s -cconfig  edifact  ORDERSD96AUNEAN008
        %(name)s -cconfig  C:/python27/lib/site-packages/bots/usersys/grammars/edifact/ORDERSD96AUNEAN008.py
        
    '''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version}
    configdir = 'config'
    editype =''
    messagetype = ''
    for arg in sys.argv[1:]:
        if arg.startswith('-c'):
            configdir = arg[2:]
            if not configdir:
                print 'Error: configuration directory indicated, but no directory name.'
                sys.exit(1)
        elif arg in ["?", "/?",'-h', '--help'] or arg.startswith('-'):
            print usage
            sys.exit(0)
        else:
            if os.path.isfile(arg):
                p1,p2 = os.path.split(arg)
                editype = os.path.basename(p1)
                messagetype,ext = os.path.splitext(p2)
                messagetype = str(messagetype)
                print 'grammarcheck',editype,messagetype
            elif not editype:
                editype = arg
            else:
                messagetype = arg
    if not (editype and messagetype):
        print 'Error: both editype and messagetype, or a file path, are required.'
        sys.exit(1)
    #***end handling command line arguments**************************
    botsinit.generalinit(configdir)     #find locating of bots, configfiles, init paths etc.
    process_name = 'grammarcheck'
    botsglobal.logger = botsinit.initenginelogging(process_name)
    atexit.register(logging.shutdown)

    try:
        grammar.grammarread(editype,messagetype)
    except:
        print 'Found error in grammar: ',botslib.txtexc()
        sys.exit(1)
    else:
        print 'OK - no error found in grammar'
        sys.exit(0)
示例#16
0
def startmulti(grammardir,editype):
    ''' used in seperate tool for bulk checking of gramamrs while developing edifact->botsgramamrs '''
    import glob
    botslib.generalinit('config')
    botslib.initenginelogging()
    for g in glob.glob(grammardir):
        g1 = os.path.basename(g)
        g2 = os.path.splitext(g1)[0]
        if g1 in ['__init__.py']:
            continue
        if g1.startswith('edifact'):
            continue
        if g1.startswith('records') or g1.endswith('records.py'):
            continue
        try:
            grammar.grammarread(editype,g2)
        except:
            #~ print 'Found error in grammar:',g
            print botslib.txtexc()
            print '\n'
        else:
            print 'OK - no error found in grammar',g,'\n'
示例#17
0
def startmulti(grammardir, editype):
    import glob

    botslib.initconfigurationfile("config")
    botslib.initlogging()
    for g in glob.glob(grammardir):
        g1 = os.path.basename(g)
        g2 = os.path.splitext(g1)[0]
        if g1 in ["__init__.py"]:
            continue
        if g1.startswith("edifact"):
            continue
        if g1.startswith("records") or g1.endswith("records.py"):
            continue
        try:
            grammar.grammarread(editype, g2)
        except:
            # ~ print 'Found error in grammar:',g
            print botslib.txtexc()
            print "\n"
        else:
            print "OK - no error found in grammar", g, "\n"
示例#18
0
文件: pluglib.py 项目: alexproca/bots
def read_index(filename):
    ''' process index file in default location. '''
    try:
        importedbotsindex,scriptname = botslib.botsimport('index')
        pluglist = importedbotsindex.plugins[:]
        if 'botsindex' in sys.modules:
            del sys.modules['botsindex']
    except:
        txt = botslib.txtexc()
        raise botslib.PluginError(_(u'Error in configuration index file. Nothing is written. Error:\n%(txt)s'),{'txt':txt})
    else:
        botsglobal.logger.info(_(u'Configuration index file is OK.'))
        botsglobal.logger.info(_(u'Start writing to database.'))

    #write content of index file to the bots database
    try:
        read_index2database(pluglist)
    except:
        txt = botslib.txtexc()
        raise botslib.PluginError(_(u'Error writing configuration index to database. Nothing is written. Error:\n%(txt)s'),{'txt':txt})
    else:
        botsglobal.logger.info(_(u'Writing to database is OK.'))
示例#19
0
def write_outgoing(run):
    outputdir = botslib.join(run.outpath)
    botslib.dirshouldbethere(outputdir)
    for outgoing in run.outgoing:
        if not outgoing['error']:
            try:
                unique_filename = filename_formatter(run.outfilename,outgoing)
                tofilepath = botslib.join(outputdir,unique_filename)
                fromfilepath = botslib.abspathdata(outgoing['filename'])
                shutil.move(fromfilepath,tofilepath)
            except:
                txt = botslib.txtexc()
                outgoing.update({'error':txt})
            else:
                outgoing.update({'outfilename':tofilepath})
示例#20
0
 def _write(self,node_instance):
     templatefile = botslib.abspath(self.__class__.__name__,self.ta_info['template'])
     try:
         botsglobal.logger.debug(u'Start writing to file "%(filename)s".',self.ta_info)
         loader = self.template.TemplateLoader(auto_reload=False)
         tmpl = loader.load(templatefile)
     except:
         txt = botslib.txtexc()
         raise botslib.OutMessageError(_(u'While templating "%(editype)s.%(messagetype)s", error:\n%(txt)s'),
                                         {'editype':self.ta_info['editype'],'messagetype':self.ta_info['messagetype'],'txt':txt})
     try:
         filehandler = botslib.opendata(self.ta_info['filename'],'wb')
         if self.ta_info['has_structure']:   #new way of working
             if self.ta_info['print_as_row']:
                 node_instance.collectlines(self.ta_info['print_as_row'])
             stream = tmpl.generate(node=node_instance)
         else:
             stream = tmpl.generate(data=self.data)
         stream.render(method='xhtml',encoding=self.ta_info['charset'],out=filehandler)
     except:
         txt = botslib.txtexc()
         raise botslib.OutMessageError(_(u'While templating "%(editype)s.%(messagetype)s", error:\n%(txt)s'),
                                         {'editype':self.ta_info['editype'],'messagetype':self.ta_info['messagetype'],'txt':txt})
     botsglobal.logger.debug(_(u'End writing to file "%(filename)s".'),self.ta_info)
示例#21
0
def read_incoming(run):
    outputdir = botslib.join(run.inpath,run.infilename)
    filelist = [filename for filename in glob.iglob(outputdir) if os.path.isfile(filename)]
    filelist.sort()
    for infilename in filelist:
        try:
            filename = transform.unique('bots_file_name')
            abs_filename = botslib.abspathdata(filename)
            shutil.copy(infilename,abs_filename)          #move if to be delted
        except:
            txt = botslib.txtexc()
        else:
            txt = ''    #no errors
        finally:
            run.incoming.append({'infilename':infilename,'filename':filename,'error':txt,'editype':run.translation['editype'],'messagetype':run.translation['messagetype']})
示例#22
0
def preprocess(routedict,function, status=FILEIN,**argv):
    ''' for pre- and postprocessing of files.
        these are NOT translations; translation involve grammars, mapping scripts etc. think of eg:
        - unzipping zipped files.
        - convert excel to csv
        - password protected files.
        Select files from INFILE -> SET_FOR_PROCESSING using criteria
        Than the actual processing function is called.
        The processing function does: SET_FOR_PROCESSING -> PROCESSING -> FILEIN
        If errors occur during processing, no ta are left with status FILEIN !
        preprocess is called right after the in-communicatiation
    '''
    nr_files = 0
    preprocessnumber = botslib.getpreprocessnumber()
    if not botslib.addinfo(change={'status':preprocessnumber},where={'status':status,'idroute':routedict['idroute'],'fromchannel':routedict['fromchannel']}):    #check if there is something to do
        return 0
    for row in botslib.query(u'''SELECT idta,filename,charset
                                FROM  ta
                                WHERE   idta>%(rootidta)s
                                AND     status=%(status)s
                                AND     statust=%(statust)s
                                AND     idroute=%(idroute)s
                                AND     fromchannel=%(fromchannel)s
                                ''',
                                {'status':preprocessnumber,'statust':OK,'idroute':routedict['idroute'],'fromchannel':routedict['fromchannel'],'rootidta':botslib.get_minta4query()}):
        try:
            botsglobal.logmap.debug(u'Start preprocessing "%s" for file "%s".',function.__name__,row['filename'])
            ta_set_for_processing = botslib.OldTransaction(row['idta'])
            ta_processing = ta_set_for_processing.copyta(status=preprocessnumber+1)
            ta_processing.filename = row['filename']
            function(ta_from=ta_processing,endstatus=status,routedict=routedict,**argv)
        except:
            txt = botslib.txtexc()
            ta_processing.failure()
            ta_processing.update(statust=ERROR,errortext=txt)
        else:
            botsglobal.logmap.debug(u'OK preprocessing  "%s" for file "%s".',function.__name__,row['filename'])
            ta_set_for_processing.update(statust=DONE)
            ta_processing.update(statust=DONE)
            nr_files += 1
    return nr_files
示例#23
0
def sendtestmailmanagers(request,*kw,**kwargs):
    try:
        sendornot = botsglobal.ini.getboolean('settings','sendreportiferror',False)
    except botslib.BotsError:
        sendornot = False
    if not sendornot:
        botsglobal.logger.info(_(u'Trying to send test mail, but in bots.ini, section [settings], "sendreportiferror" is not "True".'))
        request.user.message_set.create(message=_(u'Trying to send test mail, but in bots.ini, section [settings], "sendreportiferror" is not "True".'))
        return django.shortcuts.redirect('/home')

    from django.core.mail import mail_managers
    try:
        mail_managers(_(u'testsubject'), _(u'test content of report'))
    except:
        txt = botslib.txtexc()
        request.user.message_set.create(message=_(u'Sending test mail failed.'))
        botsglobal.logger.info(_(u'Sending test mail failed, error:\n%s'), txt)
        return django.shortcuts.redirect('/home')
    request.user.message_set.create(message=_(u'Sending test mail succeeded.'))
    botsglobal.logger.info(_(u'Sending test mail succeeded.'))
    return django.shortcuts.redirect('/home')
示例#24
0
def runengine(request,*kw,**kwargs):
    if request.method == 'GET':
        #check if bots-engine is not already running
        if list(models.mutex.objects.filter(mutexk=1).all()):
            request.user.message_set.create(message=_(u'Trying to run "bots-engine", but database is locked by another run in progress. Please try again later.'))
            botsglobal.logger.info(_(u'Trying to run "bots-engine", but database is locked by another run in progress.'))
            return django.shortcuts.redirect('/home')
        #find out the right arguments to use
        botsenginepath = os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])),'bots-engine.py')        #find the bots-engine
        lijst = [sys.executable,botsenginepath] + sys.argv[1:]
        if 'clparameter' in request.GET:
            lijst.append(request.GET['clparameter'])

        try:
            botsglobal.logger.info(_(u'Run bots-engine with parameters: "%s"'),str(lijst))
            terug = subprocess.Popen(lijst).pid
            request.user.message_set.create(message=_(u'Bots-engine is started.'))
        except:
            txt =  botslib.txtexc()
            request.user.message_set.create(message=_(u'Errors while trying to run bots-engine.'))
            botsglobal.logger.info(_(u'Errors while trying to run bots-engine:\n%s.'),txt)
    return django.shortcuts.redirect('/home')
示例#25
0
文件: views.py 项目: divadrei/bots
def sendtestmailmanagers(request,*kw,**kwargs):
    try:
        sendornot = botsglobal.ini.getboolean('settings','sendreportiferror',False)
    except botslib.BotsError:
        sendornot = False
    if not sendornot:
        notification = _(u'Trying to send test mail, but in bots.ini, section [settings], "sendreportiferror" is not "True".')
        botsglobal.logger.info(notification)
        messages.add_message(request, messages.INFO, notification)
        return django.shortcuts.redirect('/home')

    from django.core.mail import mail_managers
    try:
        mail_managers(_(u'testsubject'), _(u'test content of report'))
    except:
        txt = botslib.txtexc()
        messages.add_message(request, messages.INFO, _(u'Sending test mail failed.'))
        botsglobal.logger.info(_(u'Sending test mail failed, error:\n%s(txt)'), {'txt':txt})
        return django.shortcuts.redirect('/home')
    notification = _(u'Sending test mail succeeded.')
    messages.add_message(request, messages.INFO, notification)
    botsglobal.logger.info(notification)
    return django.shortcuts.redirect('/home')
示例#26
0
def mergemessages(startstatus=TRANSLATED, endstatus=MERGED, idroute=""):
    """ Merges en envelopes several messages to one file;
        In db-ta: attribute 'merge' indicates message should be merged with similar messages; 'merge' is generated in translation from messagetype-grammar
        If merge==False: 1 message per envelope - no merging, else append all similar messages to one file
        Implementation as separate loops: one for merge&envelope, another for enveloping only
        db-ta status TRANSLATED---->MERGED
    """
    outerqueryparameters = {
        "status": startstatus,
        "statust": OK,
        "idroute": idroute,
        "rootidta": botslib.get_minta4query(),
        "merge": False,
    }
    # **********for messages only to envelope (no merging)
    for row in botslib.query(
        u"""SELECT editype,messagetype,frompartner,topartner,testindicator,charset,contenttype,tochannel,envelope,nrmessages,idta,filename,idroute,merge
                                FROM  ta
                                WHERE   idta>%(rootidta)s
                                AND     status=%(status)s
                                AND     statust=%(statust)s
                                AND     idroute=%(idroute)s
                                AND     merge=%(merge)s
                                """,
        outerqueryparameters,
    ):
        try:
            ta_info = dict([(key, row[key]) for key in row.keys()])
            # ~ ta_info={'merge':False,'idroute':idroute}
            # ~ for key in row.keys():
            # ~ ta_info[key] = row[key]
            ta_fromfile = botslib.OldTransaction(row["idta"])  # edi message to envelope
            ta_tofile = ta_fromfile.copyta(
                status=endstatus
            )  # edifile for enveloped message; attributes of not-enveloped message are copied...
            # ~ ta_fromfile.update(child=ta_tofile.idta)        #??there is already a parent-child relation (1-1)...
            ta_info["filename"] = str(ta_tofile.idta)  # create filename for enveloped message
            botsglobal.logger.debug(
                u"Envelope 1 message editype: %s, messagetype: %s.", ta_info["editype"], ta_info["messagetype"]
            )
            envelope(ta_info, [row["filename"]])
        except:
            txt = botslib.txtexc()
            ta_tofile.update(statust=ERROR, errortext=txt)
        else:
            ta_fromfile.update(statust=DONE)
            ta_tofile.update(statust=OK, **ta_info)  # selection is used to update enveloped message;

    # **********for messages to merge & envelope
    # all GROUP BY fields must be used in SELECT!
    # as files get merged: can not copy idta; must extract relevant attributes.
    outerqueryparameters["merge"] = True
    for row in botslib.query(
        u"""SELECT editype,messagetype,frompartner,topartner,tochannel,testindicator,charset,contenttype,envelope,sum(nrmessages) as nrmessages
                                FROM  ta
                                WHERE   idta>%(rootidta)s
                                AND     status=%(status)s
                                AND     statust=%(statust)s
                                AND     idroute=%(idroute)s
                                AND     merge=%(merge)s
                                GROUP BY editype,messagetype,frompartner,topartner,tochannel,testindicator,charset,contenttype,envelope
                                """,
        outerqueryparameters,
    ):
        try:
            ta_info = dict([(key, row[key]) for key in row.keys()])
            ta_info.update({"merge": False, "idroute": idroute})
            # ~ for key in row.keys():
            # ~ ta_info[key] = row[key]
            ta_tofile = botslib.NewTransaction(status=endstatus, idroute=idroute)  # edifile for enveloped messages
            ta_info["filename"] = str(ta_tofile.idta)  # create filename for enveloped message
            innerqueryparameters = ta_info.copy()
            innerqueryparameters.update(outerqueryparameters)
            ta_list = []
            # gather individual idta and filenames
            # explicitly allow formpartner/topartner to be None/NULL
            for row2 in botslib.query(
                u"""SELECT idta, filename
                                                    FROM ta
                                                    WHERE idta>%(rootidta)s
                                                    AND status=%(status)s
                                                    AND statust=%(statust)s
                                                    AND merge=%(merge)s
                                                    AND editype=%(editype)s
                                                    AND messagetype=%(messagetype)s
                                                    AND (frompartner=%(frompartner)s OR frompartner IS NULL)
                                                    AND (topartner=%(topartner)s OR topartner IS NULL)
                                                    AND tochannel=%(tochannel)s
                                                    AND testindicator=%(testindicator)s
                                                    AND charset=%(charset)s
                                                    AND idroute=%(idroute)s
                                                    """,
                innerqueryparameters,
            ):
                ta_fromfile = botslib.OldTransaction(row2["idta"])  # edi message to envelope
                ta_fromfile.update(statust=DONE, child=ta_tofile.idta)  # st child because of n->1 relation
                ta_list.append(row2["filename"])
            botsglobal.logger.debug(
                u"Merge and envelope: editype: %s, messagetype: %s, %s messages",
                ta_info["editype"],
                ta_info["messagetype"],
                ta_info["nrmessages"],
            )
            envelope(ta_info, ta_list)
        except:
            txt = botslib.txtexc()
            ta_tofile.mergefailure()
            ta_tofile.update(statust=ERROR, errortext=txt)
        else:
            ta_tofile.update(statust=OK, **ta_info)
示例#27
0
def extractexcel(ta_from,endstatus,**argv):
    ''' extract excel file.
        editype & messagetype are unchanged.
    '''
    #***functions used by extractexcel
    #-------------------------------------------------------------------------------
    def read_xls(infilename):
        # Read excel first sheet into a 2-d array
        book       = xlrd.open_workbook(infilename)
        sheet      = book.sheet_by_index(0)
        formatter  = lambda(t,v): format_excelval(book,t,v,False)
        xlsdata = []
        for row in range(sheet.nrows):
            (types, values) = (sheet.row_types(row), sheet.row_values(row))
            xlsdata.append(map(formatter, zip(types, values)))
        return xlsdata
    #-------------------------------------------------------------------------------
    def dump_csv(xlsdata, tofilename):
        stream = botslib.opendata(tofilename, 'wb')
        csvout = csv.writer(stream, quotechar=quotechar, delimiter=field_sep, doublequote=doublequote, escapechar=escape)
        csvout.writerows( map(utf8ize, xlsdata) )
        stream.close()
    #-------------------------------------------------------------------------------
    def format_excelval(book, type, value, wanttupledate):
        #  Clean up the incoming excel data for some data types
        returnrow = []
        if   type == 2:
            if value == int(value):
                value = int(value)
        elif type == 3:
            datetuple = xlrd.xldate_as_tuple(value, book.datemode)
            value = datetuple if wanttupledate else tupledate_to_isodate(datetuple)
        elif type == 5:
            value = xlrd.error_text_from_code[value]
        return value
    #-------------------------------------------------------------------------------
    def tupledate_to_isodate(tupledate):
        # Turns a gregorian (year, month, day, hour, minute, nearest_second) into a
        # standard YYYY-MM-DDTHH:MM:SS ISO date.
        (y,m,d, hh,mm,ss) = tupledate
        nonzero = lambda n: n != 0
        date = "%04d-%02d-%02d"  % (y,m,d)    if filter(nonzero, (y,m,d))                else ''
        time = "T%02d:%02d:%02d" % (hh,mm,ss) if filter(nonzero, (hh,mm,ss)) or not date else ''
        return date+time
    #-------------------------------------------------------------------------------
    def utf8ize(l):
        # Make string-like things into utf-8, leave other things alone
        return [unicode(s).encode(charset) if hasattr(s,'encode') else s for s in l]
    #***end functions used by extractexcel
    import xlrd
    import csv
    #get parameters for csv-format; defaults are as the csv defaults (in grammar.py)
    charset = argv.get('charset',"utf-8")
    quotechar = argv.get('quotechar',"'")
    field_sep = argv.get('field_sep',':')
    escape = argv.get('escape','')
    if escape:
        doublequote = False
    else:
        doublequote = True
    try:
        infilename = botslib.abspathdata(ta_from.filename)
        xlsdata = read_xls(infilename)
        ta_to = ta_from.copyta(status=endstatus)
        tofilename = str(ta_to.idta)
        dump_csv(xlsdata,tofilename)
        ta_to.update(statust=OK,filename=tofilename) #update outmessage transaction with ta_info;
        botsglobal.logger.debug(_(u'        File written: "%s".'),tofilename)
    except:
        txt = botslib.txtexc()
        botsglobal.logger.error(_(u'Excel extraction failed, may not be an Excel file? Error:\n%s'),txt)
        raise botslib.InMessageError(_(u'Excel extraction failed, may not be an Excel file? Error:\n$error'),error=txt)
示例#28
0
def extractpdf(ta_from,endstatus,**argv):
    ''' Try to extract text content of a PDF file to a csv.
        You know this is not a great idea, right? But we'll do the best we can anyway!
        Page and line numbers are added to each row.
        Columns and rows are based on the x and y coordinates of each text element within tolerance allowed.
        Multiple text elements may combine to make one field, some PDFs have every character separated!
        You may need to experiment with x_group and y_group values, but defaults seem ok for most files.
        Output csv is UTF-8 encoded - The csv module doesn't directly support reading and writing Unicode
        If the PDF is just an image, all bets are off. Maybe try OCR, good luck with that!
        Mike Griffin 14/12/2011
    '''
    from pdfminer.pdfinterp import PDFResourceManager, process_pdf
    from pdfminer.converter import TextConverter
    from pdfminer.layout import LAParams, LTContainer, LTText, LTTextBox
    import csv

    class CsvConverter(TextConverter):
        def __init__(self, *args, **kwargs):
            TextConverter.__init__(self, *args, **kwargs)

        def receive_layout(self, ltpage):

            # recursively get every text element and it's coordinates
            def render(item):
                if isinstance(item, LTContainer):
                    for child in item:
                        render(child)
                elif isinstance(item, LTText):
                    (_,_,x,y) = item.bbox

                    # group the y values (rows) within group tolerance
                    for v in yv:
                        if y > v-y_group and y < v+y_group:
                            y = v
                    yv.append(y)

                    line = lines[int(-y)]
                    line[x] = item.get_text().encode('utf-8')

            from collections import defaultdict
            lines = defaultdict(lambda : {})

            yv = []
            render(ltpage)

            lineid = 0
            for y in sorted(lines.keys()):
                line = lines[y]
                lineid += 1
                csvdata = [ltpage.pageid,lineid] # first 2 columns are page and line numbers

                # group the x values (fields) within group tolerance
                p = 0
                field_txt = ''
                for x in sorted(line.keys()):
                    gap = x - p
                    if p > 0 and gap > x_group:
                        csvdata.append(field_txt)
                        field_txt = ''
                    field_txt += line[x]
                    p = x
                csvdata.append(field_txt)
                csvout.writerow(csvdata)
            if lineid == 0:
                raise botslib.InMessageError(_(u'PDF text extraction failed, it may contain just image(s)?'))


    #get some optional parameters
    x_group = argv.get('x_group',10) # group text closer than this as one field
    y_group = argv.get('y_group',5)  # group lines closer than this as one line
    password = argv.get('password','')
    quotechar = argv.get('quotechar','"')
    field_sep = argv.get('field_sep',',')
    escape = argv.get('escape','\\')
    charset = argv.get('charset','utf-8')
    if not escape:
        doublequote = True
    else:
        doublequote = False

    try:
        pdf_stream = botslib.opendata(ta_from.filename, 'rb')
        ta_to = ta_from.copyta(status=endstatus)
        tofilename = str(ta_to.idta)
        csv_stream = botslib.opendata(tofilename,'wb')
        csvout = csv.writer(csv_stream, quotechar=quotechar, delimiter=field_sep, doublequote=doublequote, escapechar=escape)

        # Process PDF
        rsrcmgr = PDFResourceManager(caching=True)
        device = CsvConverter(rsrcmgr, csv_stream, codec=charset)
        process_pdf(rsrcmgr, device, pdf_stream, pagenos=set(), password=password, caching=True, check_extractable=True)

        device.close()
        pdf_stream.close()
        csv_stream.close()
        ta_to.update(statust=OK,filename=tofilename) #update outmessage transaction with ta_info;
        botsglobal.logger.debug(_(u'        File written: "%s".'),tofilename)
    except:
        txt = botslib.txtexc()
        botsglobal.logger.error(_(u'PDF extraction failed, may not be a PDF file? Error:\n%s'),txt)
        raise botslib.InMessageError(_(u'PDF extraction failed, may not be a PDF file? Error:\n$error'),error=txt)
示例#29
0
def _translate_one_file(row,routedict,endstatus,userscript,scriptname):
    ''' -   read, lex, parse, make tree of nodes.
        -   split up files into messages (using 'nextmessage' of grammar)
        -   get mappingscript, start mappingscript.
        -   write the results of translation (no enveloping yet)
    '''
    try:
        ta_fromfile = botslib.OldTransaction(row['idta'])
        ta_parsed = ta_fromfile.copyta(status=PARSED)
        if row['filesize'] > botsglobal.ini.getint('settings','maxfilesizeincoming',5000000):
            ta_parsed.update(filesize=row['filesize'])
            raise botslib.FileTooLarge(_(u'File size of %(filesize)s is too big; option "maxfilesizeincoming" in bots.ini is %(maxfilesizeincoming)s.'),
                                            {'filesize':row['filesize'],'maxfilesizeincoming':botsglobal.ini.getint('settings','maxfilesizeincoming',5000000)})
        botsglobal.logger.debug(_(u'Start translating file "%(filename)s" editype "%(editype)s" messagetype "%(messagetype)s".'),row)
        #read whole edi-file: read, parse and made into a inmessage-object. Message is represented as a tree (inmessage.root is the root of the tree).
        edifile = inmessage.parse_edi_file(frompartner=row['frompartner'],
                                            topartner=row['topartner'],
                                            filename=row['filename'],
                                            messagetype=row['messagetype'],
                                            testindicator=row['testindicator'],
                                            editype=row['editype'],
                                            charset=row['charset'],
                                            alt=row['alt'],
                                            fromchannel=row['fromchannel'],
                                            idroute=routedict['idroute'],
                                            command=routedict['command'])
        edifile.checkforerrorlist() #no exception if infile has been lexed and parsed OK else raises an error

        if int(routedict['translateind']) == 3: #parse & passthrough; file is parsed, partners are known, no mapping, does confirm.
            raise botslib.GotoException('dummy')    
        
        #edifile.ta_info contains info: QUERIES, charset etc
        for inn_splitup in edifile.nextmessage():   #splitup messages in parsed edifile
            try:
                ta_splitup = ta_parsed.copyta(status=SPLITUP,**inn_splitup.ta_info)    #copy PARSED to SPLITUP ta
                #inn_splitup.ta_info: parameters from inmessage.parse_edi_file(), syntax-information and parse-information
                inn_splitup.ta_info['idta_fromfile'] = ta_fromfile.idta     #for confirmations in userscript; used to give idta of 'confirming message'
                inn_splitup.ta_info['idta'] = ta_splitup.idta     #for confirmations in userscript; used to give idta of 'confirming message'
                number_of_loops_with_same_alt = 0
                while 1:    #continue as long as there are (alt-)translations
                    #lookup the translation************************
                    tscript,toeditype,tomessagetype = botslib.lookup_translation(fromeditype=inn_splitup.ta_info['editype'],
                                                                        frommessagetype=inn_splitup.ta_info['messagetype'],
                                                                        frompartner=inn_splitup.ta_info['frompartner'],
                                                                        topartner=inn_splitup.ta_info['topartner'],
                                                                        alt=inn_splitup.ta_info['alt'])
                    if not tscript:       #no translation found in translate table; check if can find translation via user script
                        if userscript and hasattr(userscript,'gettranslation'):
                            tscript,toeditype,tomessagetype = botslib.runscript(userscript,scriptname,'gettranslation',idroute=routedict['idroute'],message=inn_splitup)
                        if not tscript:
                            raise botslib.TranslationNotFoundError(_(u'Translation not found for editype "%(editype)s", messagetype "%(messagetype)s", frompartner "%(frompartner)s", topartner "%(topartner)s", alt "%(alt)s".'),
                                                                        inn_splitup.ta_info)

                    inn_splitup.ta_info['divtext'] = tscript     #ifor reporting used mapping script to database (for display in GUI).
                    #initialize new out-object*************************
                    ta_translated = ta_splitup.copyta(status=endstatus)     #make ta for translated message (new out-ta)
                    filename_translated = str(ta_translated.idta)
                    out_translated = outmessage.outmessage_init(editype=toeditype,messagetype=tomessagetype,filename=filename_translated,reference=unique('messagecounter'),statust=OK,divtext=tscript)    #make outmessage object
                        
                    #run mapping script************************
                    botsglobal.logger.debug(_(u'Mappingscript "%(tscript)s" translates messagetype "%(messagetype)s" to messagetype "%(tomessagetype)s".'),
                                            {'tscript':tscript,'messagetype':inn_splitup.ta_info['messagetype'],'tomessagetype':out_translated.ta_info['messagetype']})
                    translationscript,scriptfilename = botslib.botsimport('mappings',inn_splitup.ta_info['editype'],tscript) #get the mappingscript
                    alt_from_previous_run = inn_splitup.ta_info['alt']      #needed to check for infinite loop
                    doalttranslation = botslib.runscript(translationscript,scriptfilename,'main',inn=inn_splitup,out=out_translated)
                    botsglobal.logger.debug(_(u'Mappingscript "%(tscript)s" finished.'),{'tscript':tscript})
                    
                    #manipulate for some attributes after mapping script
                    if 'topartner' not in out_translated.ta_info:    #out_translated does not contain values from ta......
                        out_translated.ta_info['topartner'] = inn_splitup.ta_info['topartner']
                    if 'botskey' in inn_splitup.ta_info:
                        inn_splitup.ta_info['reference'] = inn_splitup.ta_info['botskey']
                    if 'botskey' in out_translated.ta_info:    #out_translated does not contain values from ta......
                        out_translated.ta_info['reference'] = out_translated.ta_info['botskey']
                        
                    #check the value received from the mappingscript to determine what to do in this while-loop. Handling of chained trasnlations.
                    if doalttranslation is None:    
                        #translation(s) are done; handle out-message 
                        handle_out_message(out_translated,ta_translated)
                        break   #break out of while loop
                    elif isinstance(doalttranslation,dict):
                        #some extended cases; a dict is returned that contains 'instructions' for some type of chained translations
                        if 'type' not in doalttranslation or 'alt' not in doalttranslation:
                            raise botslib.BotsError(_(u"Mappingscript returned '%(alt)s'. This dict should not have 'type' and 'alt'."),{'alt':doalttranslation})
                        if alt_from_previous_run == doalttranslation['alt']:
                            number_of_loops_with_same_alt += 1
                        else:
                            number_of_loops_with_same_alt = 0
                        if doalttranslation['type'] == u'out_as_inn':
                            #do chained translation: use the out-object as inn-object, new out-object
                            #use case: detected error in incoming file; use out-object to generate warning email
                            handle_out_message(out_translated,ta_translated)
                            inn_splitup = out_translated    #out-object is now inn-object
                            if isinstance(inn_splitup,outmessage.fixed):    #for fixed: strip all values in node
                                inn_splitup.root.stripnode()
                            inn_splitup.ta_info['alt'] = doalttranslation['alt']   #get the alt-value for the next chained translation
                            if not 'frompartner' in inn_splitup.ta_info:
                                inn_splitup.ta_info['frompartner'] = ''
                            if not 'topartner' in inn_splitup.ta_info:
                                inn_splitup.ta_info['topartner'] = ''
                            inn_splitup.ta_info.pop('statust')
                        elif doalttranslation['type'] == u'no_check_on_infinite_loop':
                            #do chained translation: allow many loops wit hsame alt-value.
                            #mapping script will have to handle this correctly.
                            number_of_loops_with_same_alt = 0
                            handle_out_message(out_translated,ta_translated)
                            inn_splitup.ta_info['alt'] = doalttranslation['alt']   #get the alt-value for the next chained translation
                        else:   #there is nothing else
                            raise botslib.BotsError(_(u'Mappingscript returned dict with an unknown "type": "%(doalttranslation)s".'),{'doalttranslation':doalttranslation})
                    else:  #note: this includes alt '' (empty string)
                        if alt_from_previous_run == doalttranslation:
                            number_of_loops_with_same_alt += 1
                        else:
                            number_of_loops_with_same_alt = 0
                        #do normal chained translation: same inn-object, new out-object
                        handle_out_message(out_translated,ta_translated)
                        inn_splitup.ta_info['alt'] = doalttranslation   #get the alt-value for the next chained translation
                    if number_of_loops_with_same_alt > 10:
                        raise botslib.BotsError(_(u'Mappingscript returns same alt value over and over again (infinite loop?). Alt: "%(doalttranslation)s".'),{'doalttranslation':doalttranslation})
                #end of while-loop (trans**********************************************************************************
            #exceptions file_out-level: exception in mappingscript or writing of out-file
            except:
                #2 modes: either every error leads to skipping of  whole infile (old  mode) or errors in mappingscript/outfile only affect that branche
                if botsglobal.ini.getboolean('settings','oldmessageerrors',False):
                    raise
                txt = botslib.txtexc()
                ta_splitup.update(statust=ERROR,errortext=txt,**inn_splitup.ta_info)   #update db. inn_splitup.ta_info could be changed by mappingscript. Is this useful?
                ta_splitup.deletechildren()
            else:
                ta_splitup.update(statust=DONE, **inn_splitup.ta_info)   #update db. inn_splitup.ta_info could be changed by mappingscript. Is this useful?

    #exceptions file_in-level
    except botslib.GotoException:   #edi-file is OK, file is passed-through after parsing.
        ta_parsed.update(statust=DONE,filesize=row['filesize'],**edifile.ta_info)   #update with info from eg queries
        ta_parsed.copyta(status=MERGED,statust=OK)          #original file goes straight to MERGED
        edifile.handleconfirm(ta_fromfile,error=False)
        botsglobal.logger.debug(_(u'Parse & passthrough for input file "%(filename)s".'),row)
    except botslib.FileTooLarge as msg:
        ta_parsed.update(statust=ERROR,errortext=str(msg))
        ta_parsed.deletechildren()
        botsglobal.logger.debug(u'Error in translating input file "%(filename)s":\n%(msg)s',{'filename':row['filename'],'msg':msg})
    except:
        txt = botslib.txtexc()
        ta_parsed.update(statust=ERROR,errortext=txt,**edifile.ta_info)
        ta_parsed.deletechildren()
        edifile.handleconfirm(ta_fromfile,error=True)
        botsglobal.logger.debug(u'Error in translating input file "%(filename)s":\n%(msg)s',{'filename':row['filename'],'msg':txt})
    else:
        edifile.handleconfirm(ta_fromfile,error=False)
        ta_parsed.update(statust=DONE,filesize=row['filesize'],**edifile.ta_info)
        botsglobal.logger.debug(_(u'Translated input file "%(filename)s".'),row)
    finally:
        ta_fromfile.update(statust=DONE)
示例#30
0
def load(pathzipfile):
    ''' process uploaded plugin. '''
    #test is valid zipfile
    if not zipfile.is_zipfile(pathzipfile):
        raise botslib.PluginError(_(u'Plugin is not a valid file.'))

    #read index file
    try:
        myzipimport = zipimport.zipimporter(pathzipfile)
        importedbotsindex = myzipimport.load_module('botsindex')
        pluglist = importedbotsindex.plugins[:]
        if 'botsindex' in sys.modules:
            del sys.modules['botsindex']
    except:
        txt = botslib.txtexc()
        raise botslib.PluginError(_(u'Error in plugin. Nothing is written. Error:\n%s')%(txt))
    else:
        botsglobal.logger.info(_(u'Plugin is OK.'))
        botsglobal.logger.info(_(u'Start writing to database.'))

    #write content of index file to the bots database
    try:
        writetodatabase(pluglist)
    except:
        txt = botslib.txtexc()
        raise botslib.PluginError(_(u'Error writing plugin to database. Nothing is written. Error:\n%s'%(txt)))
    else:
        botsglobal.logger.info(u'Writing to database is OK.')
        botsglobal.logger.info(u'Start writing to files')

    #write files to the file system.
    try:
        warnrenamed = False     #to report in GUI files have been overwritten.
        myzip = zipfile.ZipFile(pathzipfile, mode="r")
        orgtargetpath = botsglobal.ini.get('directories','botspath')
        if (orgtargetpath[-1:] in (os.path.sep, os.path.altsep) and len(os.path.splitdrive(orgtargetpath)[1]) > 1):
            orgtargetpath = orgtargetpath[:-1]
        for zipfileobject in myzip.infolist():
            if zipfileobject.filename not in ['botsindex.py','README','botssys/sqlitedb/botsdb','config/bots.ini'] and os.path.splitext(zipfileobject.filename)[1] not in ['.pyo','.pyc']:
                #~ botsglobal.logger.info(u'filename in zip "%s".',zipfileobject.filename)
                if zipfileobject.filename[0] == '/':
                    targetpath = zipfileobject.filename[1:]
                else:
                    targetpath = zipfileobject.filename
                targetpath = targetpath.replace('usersys',botsglobal.ini.get('directories','usersysabs'),1)
                targetpath = targetpath.replace('botssys',botsglobal.ini.get('directories','botssys'),1)
                targetpath = botslib.join(orgtargetpath, targetpath)
                #targetpath is OK now.
                botsglobal.logger.info(_(u'    Start writing file: "%s".'),targetpath)

                if botslib.dirshouldbethere(os.path.dirname(targetpath)):
                    botsglobal.logger.info(_(u'        Create directory "%s".'),os.path.dirname(targetpath))
                if zipfileobject.filename[-1] == '/':    #check if this is a dir; if so continue
                    continue
                if os.path.isfile(targetpath):  #check if file already exists
                    try:    #this ***sometimes*** fails. (python25, for static/help/home.html...only there...)
                        os.rename(targetpath,targetpath+'.'+time.strftime('%Y%m%d%H%M%S'))
                        warnrenamed = True
                        botsglobal.logger.info(_(u'        Renamed existing file "%(from)s" to "%(to)s".'),{'from':targetpath,'to':targetpath+time.strftime('%Y%m%d%H%M%S')})
                    except:
                        pass
                source = myzip.read(zipfileobject.filename)
                target = open(targetpath, "wb")
                target.write(source)
                target.close()
                botsglobal.logger.info(_(u'        File written: "%s".'),targetpath)
    except:
        txt = botslib.txtexc()
        myzip.close()
        raise botslib.PluginError(_(u'Error writing files to system. Nothing is written to database. Error:\n%s')%(txt))
    else:
        myzip.close()
        botsglobal.logger.info(_(u'Writing files to filesystem is OK.'))
        return warnrenamed