Пример #1
0
def startmulti(grammardir,editype):
    ''' specialized tool for bulk checking of grammars while developing botsgrammars 
        grammardir: directory with gramars (eg bots/usersys/grammars/edifact)
        editype: eg edifact
    '''
    configdir = 'config'
    botsinit.generalinit(configdir)     #find locating of bots, configfiles, init paths etc.
    process_name = 'grammarcheck'
    botsglobal.logger = botsinit.initenginelogging(process_name)
    atexit.register(logging.shutdown)
    
    for filename in glob.iglob(grammardir):
        filename_basename = os.path.basename(filename)
        if filename_basename in ['__init__.py','envelope.py']:
            continue
        if filename_basename.startswith('edifact') or filename_basename.startswith('records') or filename_basename.endswith('records.py'):
            continue
        if filename_basename.endswith('pyc'):
            continue
        filename_noextension = os.path.splitext(filename_basename)[0]
        try:
            grammar.grammarread(editype,filename_noextension)
        except:
            print botslib.txtexc()
            print '\n'
        else:
            print 'OK - no error found in grammar',filename,'\n'
Пример #2
0
 def writeall(self):
     ''' Very different writeall:
         there is no tree of nodes; there is no grammar.structure/recorddefs; kid opens file by itself.
     '''
     try:
         import kid
     except:
         txt=botslib.txtexc()
         raise ImportError(_(u'Dependency failure: editype "template" requires python library "kid". Error:\n%s'%txt))
     #for template-grammar: only syntax is used. Section 'syntax' has to have 'template'
     self.outmessagegrammarread(self.ta_info['editype'],self.ta_info['messagetype'])
     templatefile = botslib.abspath(u'templates',self.ta_info['template'])
     try:
         botsglobal.logger.debug(u'Start writing to file "%s".',self.ta_info['filename'])
         ediprint = kid.Template(file=templatefile, data=self.data)
     except:
         txt=botslib.txtexc()
         raise botslib.OutMessageError(_(u'While templating "$editype.$messagetype", error:\n$txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt)
     try:
         f = botslib.opendata(self.ta_info['filename'],'wb')
         ediprint.write(f,
         #~ ediprint.write(botslib.abspathdata(self.ta_info['filename']),
                         encoding=self.ta_info['charset'],
                         output=self.ta_info['output'],    #output is specific parameter for class; init from grammar.syntax
                         fragment=self.ta_info['merge'])
     except:
         txt=botslib.txtexc()
         raise botslib.OutMessageError(_(u'While templating "$editype.$messagetype", error:\n$txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt)
     botsglobal.logger.debug(_(u'End writing to file "%s".'),self.ta_info['filename'])
Пример #3
0
 def run(self):
     ''' class for (test) orderprint; delevers a valid html-file.
         Uses a kid-template for the enveloping/merging.
         use kid to write; no envelope grammar is used
     '''
     try:
         from genshi.template import TemplateLoader
     except:
         txt=botslib.txtexc()
         raise ImportError(_(u'Dependency failure: editype "template" requires python library "genshi". Error:\n%s'%txt))
     defmessage = grammar.grammarread(self.ta_info['editype'],self.ta_info['messagetype'])   #needed because we do not know envelope; read syntax for editype/messagetype
     self.ta_info.update(defmessage.syntax)
     botslib.tryrunscript(self.userscript,self.scriptname,'ta_infocontent',ta_info=self.ta_info)
     if not self.ta_info['envelope-template']:
         raise botslib.OutMessageError(_(u'While enveloping in "$editype.$messagetype": syntax option "envelope-template" not filled; is required.'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'])
     templatefile = botslib.abspath('templateshtml',self.ta_info['envelope-template'])
     ta_list = self.filelist2absolutepaths()
     try:
         botsglobal.logger.debug(u'Start writing envelope to file "%s".',self.ta_info['filename'])
         loader = TemplateLoader(auto_reload=False)
         tmpl = loader.load(templatefile)
     except:
         txt=botslib.txtexc()
         raise botslib.OutMessageError(_(u'While enveloping in "$editype.$messagetype", error:\n$txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt)
     try:
         f = botslib.opendata(self.ta_info['filename'],'wb')
         stream = tmpl.generate(data=ta_list)
         stream.render(method='xhtml',encoding=self.ta_info['charset'],out=f)
     except:
         txt=botslib.txtexc()
         raise botslib.OutMessageError(_(u'While enveloping in "$editype.$messagetype", error:\n$txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt)
Пример #4
0
def read_index(filename):
    ''' process index file in default location. '''
    try:
        importedbotsindex, scriptname = botslib.botsimport('index')
        pluglist = importedbotsindex.plugins[:]
        if importedbotsindex.__name__ in sys.modules:
            del sys.modules[importedbotsindex.__name__]
    except:
        txt = botslib.txtexc()
        raise botslib.PluginError(
            _(u'Error in configuration index file. Nothing is written. Error:\n%(txt)s'
              ), {'txt': txt})
    else:
        botsglobal.logger.info(_(u'Configuration index file is OK.'))
        botsglobal.logger.info(_(u'Start writing to database.'))

    #write content of index file to the bots database
    try:
        read_index2database(pluglist)
    except:
        txt = botslib.txtexc()
        raise botslib.PluginError(
            _(u'Error writing configuration index to database. Nothing is written. Error:\n%(txt)s'
              ), {'txt': txt})
    else:
        botsglobal.logger.info(_(u'Writing to database is OK.'))
Пример #5
0
def startmulti(grammardir, editype):
    ''' specialized tool for bulk checking of grammars while developing botsgrammars 
        grammardir: directory with gramars (eg bots/usersys/grammars/edifact)
        editype: eg edifact
    '''
    configdir = 'config'
    botsinit.generalinit(
        configdir)  #find locating of bots, configfiles, init paths etc.
    process_name = 'grammarcheck'
    botsglobal.logger = botsinit.initenginelogging(process_name)
    atexit.register(logging.shutdown)

    for filename in glob.iglob(grammardir):
        filename_basename = os.path.basename(filename)
        if filename_basename in ['__init__.py', 'envelope.py']:
            continue
        if filename_basename.startswith(
                'edifact') or filename_basename.startswith(
                    'records') or filename_basename.endswith('records.py'):
            continue
        if filename_basename.endswith('pyc'):
            continue
        filename_noextension = os.path.splitext(filename_basename)[0]
        try:
            grammar.grammarread(editype, filename_noextension)
        except:
            print botslib.txtexc()
            print '\n'
        else:
            print 'OK - no error found in grammar', filename, '\n'
Пример #6
0
def start():
    #********command line arguments**************************
    editype = ''
    messagetype = ''
    configdir = 'config'
    for arg in sys.argv[1:]:
        if not arg:
            continue
        if arg.startswith('-c'):
            configdir = arg[2:]
            if not configdir:
                print '    !!Indicated Bots should use specific .ini file but no file name was given.'
                showusage()
        elif arg in ["?", "/?"] or arg.startswith('-'):
            showusage()
        else:
            if not editype:
                editype = arg
            else:
                messagetype = arg
    if not (editype and messagetype):
        print '    !!Both editype and messagetype are required.'
        showusage()
    #********end handling command line arguments**************************

    try:
        botsinit.generalinit(configdir)
        botsinit.initenginelogging()
        grammar.grammarread(editype, messagetype)
    except:
        print 'Found error in grammar:'
        print botslib.txtexc()
    else:
        print 'OK - no error found in grammar'
Пример #7
0
def read_index(filename):
    """ process index file in default location. """
    try:
        importedbotsindex, scriptname = botslib.botsimport("index")
        pluglist = importedbotsindex.plugins[:]
        if importedbotsindex.__name__ in sys.modules:
            del sys.modules[importedbotsindex.__name__]
    except:
        txt = botslib.txtexc()
        raise botslib.PluginError(
            _(u"Error in configuration index file. Nothing is written. Error:\n%(txt)s"), {"txt": txt}
        )
    else:
        botsglobal.logger.info(_(u"Configuration index file is OK."))
        botsglobal.logger.info(_(u"Start writing to database."))

    # write content of index file to the bots database
    try:
        read_index2database(pluglist)
    except:
        txt = botslib.txtexc()
        raise botslib.PluginError(
            _(u"Error writing configuration index to database. Nothing is written. Error:\n%(txt)s"), {"txt": txt}
        )
    else:
        botsglobal.logger.info(_(u"Writing to database is OK."))
Пример #8
0
 def writeall(self):
     ''' Very different writeall:
         there is no tree of nodes; there is no grammar.structure/recorddefs; kid opens file by itself.
     '''
     try:
         import kid
     except:
         txt = botslib.txtexc()
         raise ImportError(_(u'Dependency failure: editype "template" requires python library "kid". Error:\n%s'%txt))
     #for template-grammar: only syntax is used. Section 'syntax' has to have 'template'
     self.outmessagegrammarread(self.ta_info['editype'],self.ta_info['messagetype'])
     templatefile = botslib.abspath(u'templates',self.ta_info['template'])
     try:
         botsglobal.logger.debug(u'Start writing to file "%s".',self.ta_info['filename'])
         ediprint = kid.Template(file=templatefile, data=self.data)
     except:
         txt = botslib.txtexc()
         raise botslib.OutMessageError(_(u'While templating "$editype.$messagetype", error:\n$txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt)
     try:
         filehandler = botslib.opendata(self.ta_info['filename'],'wb')
         ediprint.write(filehandler,
                         encoding=self.ta_info['charset'],
                         output=self.ta_info['output'],    #output is specific parameter for class; init from grammar.syntax
                         fragment=self.ta_info['merge'])
     except:
         txt = botslib.txtexc()
         raise botslib.OutMessageError(_(u'While templating "$editype.$messagetype", error:\n$txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt)
     botsglobal.logger.debug(_(u'End writing to file "%s".'),self.ta_info['filename'])
Пример #9
0
 def writeall(self):
     ''' Very different writeall:
         there is no tree of nodes; there is no grammar.structure/recorddefs; kid opens file by itself.
     '''
     try:
         from genshi.template import TemplateLoader
     except:
         txt=botslib.txtexc()
         raise ImportError(_(u'Dependency failure: editype "template" requires python library "genshi". Error:\n%s'%txt))
     #for template-grammar: only syntax is used. Section 'syntax' has to have 'template'
     self.outmessagegrammarread(self.ta_info['editype'],self.ta_info['messagetype'])
     templatefile = botslib.abspath(u'templateshtml',self.ta_info['template'])
     try:
         botsglobal.logger.debug(u'Start writing to file "%s".',self.ta_info['filename'])
         loader = TemplateLoader(auto_reload=False)
         tmpl = loader.load(templatefile)
     except:
         txt=botslib.txtexc()
         raise botslib.OutMessageError(_(u'While templating "$editype.$messagetype", error:\n$txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt)
     try:
         f = botslib.opendata(self.ta_info['filename'],'wb')
         stream = tmpl.generate(data=self.data)
         stream.render(method='xhtml',encoding=self.ta_info['charset'],out=f)
     except:
         txt=botslib.txtexc()
         raise botslib.OutMessageError(_(u'While templating "$editype.$messagetype", error:\n$txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt)
     botsglobal.logger.debug(_(u'End writing to file "%s".'),self.ta_info['filename'])
Пример #10
0
 def run(self):
     ''' class for (test) orderprint; delevers a valid html-file.
         Uses a kid-template for the enveloping/merging.
         use kid to write; no envelope grammar is used
     '''
     import kid
     defmessage = grammar.grammarread(self.ta_info['editype'],self.ta_info['messagetype'])   #needed beause we do not know envelope; read syntax for editype/messagetype
     self.ta_info.update(defmessage.syntax)
     botslib.tryrunscript(self.userscript,self.scriptname,'ta_infocontent',ta_info=self.ta_info)
     if not self.ta_info['envelope-template']:
         raise botslib.OutMessageError(_(u'While enveloping in "$editype.$messagetype": syntax option "envelope-template" not filled; is required.'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'])
     templatefile = botslib.abspath('templates',self.ta_info['envelope-template'])
     ta_list = self.filelist2absolutepaths()
     try:
         botsglobal.logger.debug(u'Start writing envelope to file "%s".',self.ta_info['filename'])
         ediprint = kid.Template(file=templatefile, data=ta_list) #init template; pass list with filenames
     except:
         txt=botslib.txtexc()
         raise botslib.OutMessageError(_(u'While enveloping in "$editype.$messagetype": $txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt)
     try:
         f = botslib.opendata(self.ta_info['filename'],'wb')
         ediprint.write(f,
                         encoding=self.ta_info['charset'],
                         output=self.ta_info['output'])
     except:
         txt=botslib.txtexc()
         raise botslib.OutMessageError(_(u'While enveloping error in "$editype.$messagetype"; probably in html file(s) to be merged: $txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt)
Пример #11
0
def start():
    #********command line arguments**************************
    editype =''
    messagetype = ''
    configdir = 'config'
    for arg in sys.argv[1:]:
        if not arg:
            continue
        if arg.startswith('-c'):
            configdir = arg[2:]
            if not configdir:
                print '    !!Indicated Bots should use specific .ini file but no file name was given.'
                showusage()
        elif arg in ["?", "/?"] or arg.startswith('-'):
            showusage()
        else:
            if not editype:
                editype = arg
            else:
                messagetype = arg
    if not (editype and messagetype):
        print '    !!Both editype and messagetype are required.'
        showusage()
    #********end handling command line arguments**************************

    try:
        botsinit.generalinit(configdir)
        botsinit.initenginelogging()
        grammar.grammarread(editype,messagetype)
    except:
        print 'Found error in grammar:'
        print botslib.txtexc()
    else:
        print 'OK - no error found in grammar'
Пример #12
0
def runengine(request, *kw, **kwargs):
    if request.method == "GET":
        # ~ logger = logging.getLogger('bots')
        if os.name == "nt":
            lijst = [sys.executable, os.path.normpath(os.path.join(sys.prefix, "Scripts", "bots-engine.py"))]
        elif os.path.exists(os.path.join(sys.prefix, "bin", "bots-engine.py")):
            lijst = [os.path.normpath(os.path.join(sys.prefix, "bin", "bots-engine.py"))]
        elif os.path.exists(os.path.join(sys.prefix, "local/bin", "bots-engine.py")):
            lijst = [os.path.normpath(os.path.join(sys.prefix, "local/bin", "bots-engine.py"))]
        else:
            request.user.message_set.create(message=_(u"Bots can not find executable for bots-engine."))
            # ~ logger.info('Bots can not find executable for bots-engine.')
            return django.shortcuts.redirect("/")

        try:
            if "clparameter" in request.GET:
                lijst.append(request.GET["clparameter"])
            # ~ logger.info('Run bots-engine with parameters: "%s"',str(lijst))
            terug = subprocess.Popen(lijst).pid
            request.user.message_set.create(message=_(u"Bots-engine is started."))
            # ~ logger.info('Bots-engine is started.')
        except:
            print botslib.txtexc()
            request.user.message_set.create(message=_(u"Errors while trying to run bots-engine."))
            # ~ logger.info('Errors while trying to run bots-engine.')
    return django.shortcuts.redirect("/")
Пример #13
0
 def run(self):
     ''' class for template enveloping; delevers a valid html-file.
         Uses a kid-template for the enveloping/merging.
         use kid to write; no envelope grammar is used
         #20120101 depreciated. use class templatehtml
     '''
     try:
         import kid
     except:
         raise ImportError(_(u'Dependency failure: editype "template" requires python library "kid".'))
     self._openoutenvelope()
     self.ta_info.update(self.out.ta_info)
     botslib.tryrunscript(self.userscript,self.scriptname,'ta_infocontent',ta_info=self.ta_info)
     if not self.ta_info['envelope-template']:
         raise botslib.OutMessageError(_(u'While enveloping in "%(editype)s.%(messagetype)s": syntax option "envelope-template" not filled; is required.'),
                                         self.ta_info)
     templatefile = botslib.abspath('templates',self.ta_info['envelope-template'])
     ta_list = self.filelist2absolutepaths()
     try:
         botsglobal.logger.debug(u'Start writing envelope to file "%(filename)s".',self.ta_info)
         ediprint = kid.Template(file=templatefile, data=ta_list) #init template; pass list with filenames
     except:
         txt = botslib.txtexc()
         raise botslib.OutMessageError(_(u'While enveloping in "%(editype)s.%(messagetype)s", error:\n%(txt)s'),
                                         {'editype':self.ta_info['editype'],'messagetype':self.ta_info['messagetype'],'txt':txt})
     try:
         filehandler = botslib.opendata(self.ta_info['filename'],'wb')
         ediprint.write(filehandler,
                         encoding=self.ta_info['charset'],
                         output=self.ta_info['output'])
     except:
         txt = botslib.txtexc()
         raise botslib.OutMessageError(_(u'While enveloping in "%(editype)s.%(messagetype)s", error:\n%(txt)s'),
                                         {'editype':self.ta_info['editype'],'messagetype':self.ta_info['messagetype'],'txt':txt})
Пример #14
0
 def writeall(self):
     ''' Very different writeall:
         there is no tree of nodes; there is no grammar.structure/recorddefs; kid opens file by itself.
     '''
     try:
         from genshi.template import TemplateLoader
     except:
         txt = botslib.txtexc()
         raise ImportError(_(u'Dependency failure: editype "template" requires python library "genshi". Error:\n%s'%txt))
     #for template-grammar: only syntax is used. Section 'syntax' has to have 'template'
     self.outmessagegrammarread(self.ta_info['editype'],self.ta_info['messagetype'])
     templatefile = botslib.abspath(u'templateshtml',self.ta_info['template'])
     try:
         botsglobal.logger.debug(u'Start writing to file "%s".',self.ta_info['filename'])
         loader = TemplateLoader(auto_reload=False)
         tmpl = loader.load(templatefile)
     except:
         txt = botslib.txtexc()
         raise botslib.OutMessageError(_(u'While templating "$editype.$messagetype", error:\n$txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt)
     try:
         filehandler = botslib.opendata(self.ta_info['filename'],'wb')
         stream = tmpl.generate(data=self.data)
         stream.render(method='xhtml',encoding=self.ta_info['charset'],out=filehandler)
     except:
         txt = botslib.txtexc()
         raise botslib.OutMessageError(_(u'While templating "$editype.$messagetype", error:\n$txt'),editype=self.ta_info['editype'],messagetype=self.ta_info['messagetype'],txt=txt)
     botsglobal.logger.debug(_(u'End writing to file "%s".'),self.ta_info['filename'])
Пример #15
0
 def run(self):
     try:
         from genshi.template import TemplateLoader
     except:
         raise ImportError(u'Dependency failure: editype "templatehtml" requires python library "genshi".')
     self._openoutenvelope()
     self.ta_info.update(self.out.ta_info)
     botslib.tryrunscript(self.userscript,self.scriptname,'ta_infocontent',ta_info=self.ta_info)
     if not self.ta_info['envelope-template']:
         raise botslib.OutMessageError(_(u'While enveloping in "%(editype)s.%(messagetype)s": syntax option "envelope-template" not filled; is required.'),
                                         self.ta_info)
     templatefile = botslib.abspath(self.__class__.__name__,self.ta_info['envelope-template'])
     ta_list = self.filelist2absolutepaths()
     try:
         botsglobal.logger.debug(u'Start writing envelope to file "%(filename)s".',self.ta_info)
         loader = TemplateLoader(auto_reload=False)
         tmpl = loader.load(templatefile)
     except:
         txt = botslib.txtexc()
         raise botslib.OutMessageError(_(u'While enveloping in "%(editype)s.%(messagetype)s", error:\n%(txt)s'),
                                     {'editype':self.ta_info['editype'],'messagetype':self.ta_info['messagetype'],'txt':txt})
     try:
         filehandler = botslib.opendata(self.ta_info['filename'],'wb')
         stream = tmpl.generate(data=ta_list)
         stream.render(method='xhtml',encoding=self.ta_info['charset'],out=filehandler)
     except:
         txt = botslib.txtexc()
         raise botslib.OutMessageError(_(u'While enveloping in "%(editype)s.%(messagetype)s", error:\n%(txt)s'),
                                     {'editype':self.ta_info['editype'],'messagetype':self.ta_info['messagetype'],'txt':txt})
Пример #16
0
def start():
    # ********command line arguments**************************
    editype = ""
    messagetype = ""
    botsinifile = "config"
    for arg in sys.argv[1:]:
        if not arg:
            continue
        if arg.startswith("-c"):
            botsinifile = arg[2:]
            if not botsinifile:
                print "    !!Indicated Bots should use specific .ini file but no file name was given."
                showusage()
        elif arg in ["?", "/?"] or arg.startswith("-"):
            showusage()
        else:
            if not editype:
                editype = arg
            else:
                messagetype = arg
    if not (editype and messagetype):
        print "    !!Both editype and messagetype are required."
        showusage()
    # ********end handling command line arguments**************************

    try:
        botslib.initconfigurationfile(botsinifile)
        botslib.initlogging()
        grammar.grammarread(editype, messagetype)
    except:
        print "Found error in grammar:"
        print botslib.txtexc()
    else:
        print "OK - no error found in grammar"
Пример #17
0
 def run(self):
     ''' class for (test) orderprint; delevers a valid html-file.
         Uses a kid-template for the enveloping/merging.
         use kid to write; no envelope grammar is used
     '''
     try:
         import kid
     except:
         txt = botslib.txtexc()
         raise ImportError(
             _(u'Dependency failure: editype "template" requires python library "kid". Error:\n%s'
               % txt))
     defmessage = grammar.grammarread(
         self.ta_info['editype'], self.ta_info['messagetype']
     )  #needed because we do not know envelope; read syntax for editype/messagetype
     self.ta_info.update(defmessage.syntax)
     botslib.tryrunscript(self.userscript,
                          self.scriptname,
                          'ta_infocontent',
                          ta_info=self.ta_info)
     if not self.ta_info['envelope-template']:
         raise botslib.OutMessageError(_(
             u'While enveloping in "$editype.$messagetype": syntax option "envelope-template" not filled; is required.'
         ),
                                       editype=self.ta_info['editype'],
                                       messagetype=self.
                                       ta_info['messagetype'])
     templatefile = botslib.abspath('templates',
                                    self.ta_info['envelope-template'])
     ta_list = self.filelist2absolutepaths()
     try:
         botsglobal.logger.debug(u'Start writing envelope to file "%s".',
                                 self.ta_info['filename'])
         ediprint = kid.Template(
             file=templatefile,
             data=ta_list)  #init template; pass list with filenames
     except:
         txt = botslib.txtexc()
         raise botslib.OutMessageError(
             _(u'While enveloping in "$editype.$messagetype", error:\n$txt'
               ),
             editype=self.ta_info['editype'],
             messagetype=self.ta_info['messagetype'],
             txt=txt)
     try:
         filehandler = botslib.opendata(self.ta_info['filename'], 'wb')
         ediprint.write(filehandler,
                        encoding=self.ta_info['charset'],
                        output=self.ta_info['output'])
     except:
         txt = botslib.txtexc()
         raise botslib.OutMessageError(
             _(u'While enveloping in "$editype.$messagetype", error:\n$txt'
               ),
             editype=self.ta_info['editype'],
             messagetype=self.ta_info['messagetype'],
             txt=txt)
Пример #18
0
 def run(self):
     ''' class for (test) orderprint; delevers a valid html-file.
         Uses a kid-template for the enveloping/merging.
         use kid to write; no envelope grammar is used
     '''
     try:
         from genshi.template import TemplateLoader
     except:
         txt = botslib.txtexc()
         raise ImportError(
             _(u'Dependency failure: editype "template" requires python library "genshi". Error:\n%s'
               % txt))
     defmessage = grammar.grammarread(
         self.ta_info['editype'], self.ta_info['messagetype']
     )  #needed because we do not know envelope; read syntax for editype/messagetype
     self.ta_info.update(defmessage.syntax)
     botslib.tryrunscript(self.userscript,
                          self.scriptname,
                          'ta_infocontent',
                          ta_info=self.ta_info)
     if not self.ta_info['envelope-template']:
         raise botslib.OutMessageError(_(
             u'While enveloping in "$editype.$messagetype": syntax option "envelope-template" not filled; is required.'
         ),
                                       editype=self.ta_info['editype'],
                                       messagetype=self.
                                       ta_info['messagetype'])
     templatefile = botslib.abspath('templateshtml',
                                    self.ta_info['envelope-template'])
     ta_list = self.filelist2absolutepaths()
     try:
         botsglobal.logger.debug(u'Start writing envelope to file "%s".',
                                 self.ta_info['filename'])
         loader = TemplateLoader(auto_reload=False)
         tmpl = loader.load(templatefile)
     except:
         txt = botslib.txtexc()
         raise botslib.OutMessageError(
             _(u'While enveloping in "$editype.$messagetype", error:\n$txt'
               ),
             editype=self.ta_info['editype'],
             messagetype=self.ta_info['messagetype'],
             txt=txt)
     try:
         filehandler = botslib.opendata(self.ta_info['filename'], 'wb')
         stream = tmpl.generate(data=ta_list)
         stream.render(method='xhtml',
                       encoding=self.ta_info['charset'],
                       out=filehandler)
     except:
         txt = botslib.txtexc()
         raise botslib.OutMessageError(
             _(u'While enveloping in "$editype.$messagetype", error:\n$txt'
               ),
             editype=self.ta_info['editype'],
             messagetype=self.ta_info['messagetype'],
             txt=txt)
Пример #19
0
def mergemessages(run):
    names_envelope_criteria = ('editype', 'messagetype', 'frompartner',
                               'topartner', 'testindicator', 'charset',
                               'contenttype', 'envelope', 'rsrv3')
    merge_yes = {}
    merge_no = []  #can be non-unique: as a list
    #walk over run.translated; sort by envelope-criteria in 2 dicts
    for translated in run.translated:
        filename = translated.get('filename')
        infilename = translated.get('infilename')
        nrmessages = translated.get('nrmessages')
        if translated.get('merge'):
            env_criteria = tuple(
                translated.get(field) for field in names_envelope_criteria)
            if env_criteria in merge_yes:
                merge_yes[env_criteria][0].append(filename)
                merge_yes[env_criteria][1].append(infilename)
                merge_yes[env_criteria][2] += nrmessages
            else:
                merge_yes[env_criteria] = [[filename], [infilename],
                                           nrmessages]
        else:
            ta_info = dict((field, translated.get(field))
                           for field in names_envelope_criteria)
            ta_info['nrmessages'] = nrmessages
            merge_no.append((ta_info, [filename], [infilename]))
    #envelope
    for env_criteria, rest_of_info in merge_yes.iteritems():
        ta_info = dict(zip(names_envelope_criteria, env_criteria))
        ta_info['filename'] = transform.unique(
            'bots_file_name')  #create filename for enveloped message
        ta_info['nrmessages'] = rest_of_info[2]
        ta_info['infilename'] = rest_of_info[
            1]  #for reference: list of infilenames
        ta_info['error'] = ''
        try:
            envelope.envelope(ta_info, rest_of_info[0])
        except:
            txt = botslib.txtexc()
            ta_info['error'] = txt.strip()
        finally:
            run.outgoing.append(ta_info)
    for ta_info, filenames, infilenames in merge_no:
        ta_info['filename'] = transform.unique(
            'bots_file_name')  #create filename for enveloped message
        ta_info[
            'infilename'] = infilenames  #for reference: list of infilenames
        ta_info['error'] = ''
        try:
            envelope.envelope(ta_info, filenames)
        except:
            txt = botslib.txtexc()
            ta_info['error'] = txt.strip()
        finally:
            run.outgoing.append(ta_info)
Пример #20
0
 def run(self):
     try:
         from genshi.template import TemplateLoader
     except:
         raise ImportError(
             u'Dependency failure: editype "templatehtml" requires python library "genshi".'
         )
     self._openoutenvelope()
     self.ta_info.update(self.out.ta_info)
     botslib.tryrunscript(self.userscript,
                          self.scriptname,
                          'ta_infocontent',
                          ta_info=self.ta_info)
     if not self.ta_info['envelope-template']:
         raise botslib.OutMessageError(
             _(u'While enveloping in "%(editype)s.%(messagetype)s": syntax option "envelope-template" not filled; is required.'
               ), self.ta_info)
     templatefile = botslib.abspath(self.__class__.__name__,
                                    self.ta_info['envelope-template'])
     ta_list = self.filelist2absolutepaths()
     try:
         botsglobal.logger.debug(
             u'Start writing envelope to file "%(filename)s".',
             self.ta_info)
         loader = TemplateLoader(auto_reload=False)
         tmpl = loader.load(templatefile)
     except:
         txt = botslib.txtexc()
         raise botslib.OutMessageError(
             _(u'While enveloping in "%(editype)s.%(messagetype)s", error:\n%(txt)s'
               ), {
                   'editype': self.ta_info['editype'],
                   'messagetype': self.ta_info['messagetype'],
                   'txt': txt
               })
     try:
         filehandler = botslib.opendata(self.ta_info['filename'], 'wb')
         stream = tmpl.generate(data=ta_list)
         stream.render(method='xhtml',
                       encoding=self.ta_info['charset'],
                       out=filehandler)
     except:
         txt = botslib.txtexc()
         raise botslib.OutMessageError(
             _(u'While enveloping in "%(editype)s.%(messagetype)s", error:\n%(txt)s'
               ), {
                   'editype': self.ta_info['editype'],
                   'messagetype': self.ta_info['messagetype'],
                   'txt': txt
               })
Пример #21
0
 def run(self):
     """ class for (test) orderprint; delevers a valid html-file.
         Uses a kid-template for the enveloping/merging.
         use kid to write; no envelope grammar is used
     """
     try:
         import kid
     except:
         txt = botslib.txtexc()
         raise ImportError(
             _(u'Dependency failure: editype "template" requires python library "kid". Error:\n%s' % txt)
         )
     defmessage = grammar.grammarread(
         self.ta_info["editype"], self.ta_info["messagetype"]
     )  # needed because we do not know envelope; read syntax for editype/messagetype
     self.ta_info.update(defmessage.syntax)
     botslib.tryrunscript(self.userscript, self.scriptname, "ta_infocontent", ta_info=self.ta_info)
     if not self.ta_info["envelope-template"]:
         raise botslib.OutMessageError(
             _(
                 u'While enveloping in "$editype.$messagetype": syntax option "envelope-template" not filled; is required.'
             ),
             editype=self.ta_info["editype"],
             messagetype=self.ta_info["messagetype"],
         )
     templatefile = botslib.abspath("templates", self.ta_info["envelope-template"])
     ta_list = self.filelist2absolutepaths()
     try:
         botsglobal.logger.debug(u'Start writing envelope to file "%s".', self.ta_info["filename"])
         ediprint = kid.Template(file=templatefile, data=ta_list)  # init template; pass list with filenames
     except:
         txt = botslib.txtexc()
         raise botslib.OutMessageError(
             _(u'While enveloping in "$editype.$messagetype", error:\n$txt'),
             editype=self.ta_info["editype"],
             messagetype=self.ta_info["messagetype"],
             txt=txt,
         )
     try:
         filehandler = botslib.opendata(self.ta_info["filename"], "wb")
         ediprint.write(filehandler, encoding=self.ta_info["charset"], output=self.ta_info["output"])
     except:
         txt = botslib.txtexc()
         raise botslib.OutMessageError(
             _(u'While enveloping in "$editype.$messagetype", error:\n$txt'),
             editype=self.ta_info["editype"],
             messagetype=self.ta_info["messagetype"],
             txt=txt,
         )
Пример #22
0
def sendtestmailmanagers(request, *kw, **kwargs):
    try:
        sendornot = botsglobal.ini.getboolean('settings', 'sendreportiferror',
                                              False)
    except botslib.BotsError:
        sendornot = False
    if not sendornot:
        notification = _(
            u'Trying to send test mail, but in bots.ini, section [settings], "sendreportiferror" is not "True".'
        )
        botsglobal.logger.info(notification)
        messages.add_message(request, messages.INFO, notification)
        return django.shortcuts.redirect('/home')

    from django.core.mail import mail_managers
    try:
        mail_managers(_(u'testsubject'), _(u'test content of report'))
    except:
        txt = botslib.txtexc()
        messages.add_message(request, messages.INFO,
                             _(u'Sending test mail failed.'))
        botsglobal.logger.info(_(u'Sending test mail failed, error:\n%(txt)s'),
                               {'txt': txt})
        return django.shortcuts.redirect('/home')
    notification = _(u'Sending test mail succeeded.')
    messages.add_message(request, messages.INFO, notification)
    botsglobal.logger.info(notification)
    return django.shortcuts.redirect('/home')
Пример #23
0
def postprocess(routedict,function,status=FILEOUT,rootidta=None,**argv):
    ''' for postprocessing of files.
        these are NOT translations; translation involve grammars, mapping scripts etc. think of eg:
        - zip files.
        If errors occur during processing, no ta are left with status FILEOUT !
        postprocess is called right before the out-communicatiation
    '''
    if rootidta is None:
        rootidta = botsglobal.currentrun.get_minta4query_routepart()
    nr_files = 0
    for row in botslib.query(u'''SELECT idta,filename
                                FROM ta
                                WHERE idta>%(rootidta)s
                                AND status=%(status)s
                                AND statust=%(statust)s
                                AND idroute=%(idroute)s
                                AND tochannel=%(tochannel)s
                                ''',
                                {'status':status,'statust':OK,'idroute':routedict['idroute'],'tochannel':routedict['tochannel'],'rootidta':rootidta}):
        try:
            botsglobal.logger.debug(u'Start postprocessing "%(name)s" for file "%(filename)s".',
                                    {'name':function.__name__,'filename':row['filename']})
            ta_from = botslib.OldTransaction(row['idta'])
            ta_from.filename = row['filename']
            function(ta_from=ta_from,endstatus=status,routedict=routedict,**argv)
        except:
            txt = botslib.txtexc()
            ta_from.update(statust=ERROR,errortext=txt)
            ta_from.deletechildren()
        else:
            botsglobal.logger.debug(u'OK postprocessing "%(name)s" for file "%(filename)s".',
                                    {'name':function.__name__,'filename':row['filename']})
            ta_from.update(statust=DONE)
            nr_files += 1
    return nr_files
Пример #24
0
def mergemessages(run):
    names_envelope_criteria = ('editype','messagetype','frompartner','topartner','testindicator','charset','contenttype','envelope','rsrv3')
    merge_yes = {}
    merge_no = []       #can be non-unique: as a list
    #walk over run.translated; sort by envelope-criteria in 2 dicts
    for translated in run.translated:
        filename = translated.get('filename')
        infilename = translated.get('infilename')
        nrmessages = translated.get('nrmessages')
        if translated.get('merge'):
            env_criteria = tuple(translated.get(field) for field in names_envelope_criteria)
            if env_criteria in merge_yes:
                merge_yes[env_criteria][0].append(filename)
                merge_yes[env_criteria][1].append(infilename)
                merge_yes[env_criteria][2] += nrmessages
            else:
                merge_yes[env_criteria] = [[filename],[infilename],nrmessages]
        else:
            ta_info = dict((field,translated.get(field)) for field in names_envelope_criteria)
            ta_info['nrmessages'] = nrmessages
            merge_no.append((ta_info, [filename],[infilename]))
    #envelope
    for env_criteria,rest_of_info in merge_yes.iteritems():
        ta_info = dict(zip(names_envelope_criteria,env_criteria))
        ta_info['filename'] = transform.unique('bots_file_name')   #create filename for enveloped message
        ta_info['nrmessages'] = rest_of_info[2]
        ta_info['infilename'] = rest_of_info[1]      #for reference: list of infilenames
        ta_info['error'] = ''
        try:
            envelope.envelope(ta_info,rest_of_info[0])
        except:
            txt = botslib.txtexc()
            ta_info['error'] = txt.strip()
        finally:
            run.outgoing.append(ta_info)
    for ta_info,filenames,infilenames in merge_no:
        ta_info['filename'] = transform.unique('bots_file_name')   #create filename for enveloped message
        ta_info['infilename'] = infilenames      #for reference: list of infilenames
        ta_info['error'] = ''
        try:
            envelope.envelope(ta_info,filenames)
        except:
            txt = botslib.txtexc()
            ta_info['error'] = txt.strip()
        finally:
            run.outgoing.append(ta_info)
Пример #25
0
def preprocess(routedict, function, status=FILEIN, **argv):
    ''' for pre- and postprocessing of files.
        these are NOT translations; translation involve grammars, mapping scripts etc. think of eg:
        - unzipping zipped files.
        - convert excel to csv
        - password protected files.
        Select files from INFILE -> SET_FOR_PROCESSING using criteria
        Than the actual processing function is called.
        The processing function does: SET_FOR_PROCESSING -> PROCESSING -> FILEIN
        If errors occur during processing, no ta are left with status FILEIN !
        preprocess is called right after the in-communicatiation
    '''
    nr_files = 0
    preprocessnumber = botslib.getpreprocessnumber()
    if not botslib.addinfo(change={'status': preprocessnumber},
                           where={
                               'status': status,
                               'idroute': routedict['idroute'],
                               'fromchannel': routedict['fromchannel']
                           }):  #check if there is something to do
        return 0
    for row in botslib.query(
            u'''SELECT idta,filename,charset
                                FROM  ta
                                WHERE   idta>%(rootidta)s
                                AND     status=%(status)s
                                AND     statust=%(statust)s
                                AND     idroute=%(idroute)s
                                AND     fromchannel=%(fromchannel)s
                                ''', {
                'status': preprocessnumber,
                'statust': OK,
                'idroute': routedict['idroute'],
                'fromchannel': routedict['fromchannel'],
                'rootidta': botslib.get_minta4query()
            }):
        try:
            botsglobal.logmap.debug(u'Start preprocessing "%s" for file "%s".',
                                    function.__name__, row['filename'])
            ta_set_for_processing = botslib.OldTransaction(row['idta'])
            ta_processing = ta_set_for_processing.copyta(
                status=preprocessnumber + 1)
            ta_processing.filename = row['filename']
            function(ta_from=ta_processing,
                     endstatus=status,
                     routedict=routedict,
                     **argv)
        except:
            txt = botslib.txtexc()
            ta_processing.failure()
            ta_processing.update(statust=ERROR, errortext=txt)
        else:
            botsglobal.logmap.debug(u'OK preprocessing  "%s" for file "%s".',
                                    function.__name__, row['filename'])
            ta_set_for_processing.update(statust=DONE)
            ta_processing.update(statust=DONE)
            nr_files += 1
    return nr_files
Пример #26
0
def start():
    #NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
    #********command line arguments**************************
    usage = '''
    This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
    Checks a Bots grammar. Same checks are used as in translations with bots-engine. Searches for grammar in 
    regular place: bots/usersys/grammars/<editype>/<messagetype>.py  (even if a path is passed).
    
    Usage:  %(name)s  -c<directory> <editype> <messagetype>
       or   %(name)s  -c<directory> <path to grammar>
    Options:
        -c<directory>   directory for configuration files (default: config).
    Examples:
        %(name)s -cconfig  edifact  ORDERSD96AUNEAN008
        %(name)s -cconfig  C:/python27/lib/site-packages/bots/usersys/grammars/edifact/ORDERSD96AUNEAN008.py
        
    '''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version}
    configdir = 'config'
    editype =''
    messagetype = ''
    for arg in sys.argv[1:]:
        if arg.startswith('-c'):
            configdir = arg[2:]
            if not configdir:
                print 'Error: configuration directory indicated, but no directory name.'
                sys.exit(1)
        elif arg in ["?", "/?",'-h', '--help'] or arg.startswith('-'):
            print usage
            sys.exit(0)
        else:
            if os.path.isfile(arg):
                p1,p2 = os.path.split(arg)
                editype = os.path.basename(p1)
                messagetype,ext = os.path.splitext(p2)
                messagetype = str(messagetype)
                print 'grammarcheck',editype,messagetype
            elif not editype:
                editype = arg
            else:
                messagetype = arg
    if not (editype and messagetype):
        print 'Error: both editype and messagetype, or a file path, are required.'
        sys.exit(1)
    #***end handling command line arguments**************************
    botsinit.generalinit(configdir)     #find locating of bots, configfiles, init paths etc.
    process_name = 'grammarcheck'
    botsglobal.logger = botsinit.initenginelogging(process_name)
    atexit.register(logging.shutdown)

    try:
        grammar.grammarread(editype,messagetype)
    except:
        print 'Found error in grammar: ',botslib.txtexc()
        sys.exit(1)
    else:
        print 'OK - no error found in grammar'
        sys.exit(0)
Пример #27
0
def startmulti(grammardir,editype):
    import glob
    botslib.initconfigurationfile('config')
    botslib.initlogging()
    for g in glob.glob(grammardir):
        g1 = os.path.basename(g)
        g2 = os.path.splitext(g1)[0]
        if g1 in ['__init__.py']:
            continue
        if g1.startswith('edifact'):
            continue
        if g1.startswith('records') or g1.endswith('records.py'):
            continue
        try:
            grammar.grammarread(editype,g2)
        except:
            #~ print 'Found error in grammar:',g
            print botslib.txtexc()
            print '\n'
        else:
            print 'OK - no error found in grammar',g,'\n'
Пример #28
0
def runengine(request, *kw, **kwargs):
    if request.method == 'GET':
        #~ logger = logging.getLogger('bots')
        if os.name == 'nt':
            lijst = [
                sys.executable,
                os.path.normpath(
                    os.path.join(sys.prefix, 'Scripts', 'bots-engine.py'))
            ]
        elif os.path.exists(os.path.join(sys.prefix, 'bin', 'bots-engine.py')):
            lijst = [
                os.path.normpath(
                    os.path.join(sys.prefix, 'bin', 'bots-engine.py'))
            ]
        elif os.path.exists(
                os.path.join(sys.prefix, 'local/bin', 'bots-engine.py')):
            lijst = [
                os.path.normpath(
                    os.path.join(sys.prefix, 'local/bin', 'bots-engine.py'))
            ]
        else:
            request.user.message_set.create(
                message=_(u'Bots can not find executable for bots-engine.'))
            #~ logger.info('Bots can not find executable for bots-engine.')
            return django.shortcuts.redirect('/')

        try:
            if 'clparameter' in request.GET:
                lijst.append(request.GET['clparameter'])
            #~ logger.info('Run bots-engine with parameters: "%s"',str(lijst))
            terug = subprocess.Popen(lijst).pid
            request.user.message_set.create(
                message=_(u'Bots-engine is started.'))
            #~ logger.info('Bots-engine is started.')
        except:
            print botslib.txtexc()
            request.user.message_set.create(
                message=_(u'Errors while trying to run bots-engine.'))
            #~ logger.info('Errors while trying to run bots-engine.')
    return django.shortcuts.redirect('/')
Пример #29
0
def startmulti(grammardir,editype):
    ''' used in seperate tool for bulk checking of gramamrs while developing edifact->botsgramamrs '''
    import glob
    botslib.generalinit('config')
    botslib.initenginelogging()
    for g in glob.glob(grammardir):
        g1 = os.path.basename(g)
        g2 = os.path.splitext(g1)[0]
        if g1 in ['__init__.py']:
            continue
        if g1.startswith('edifact'):
            continue
        if g1.startswith('records') or g1.endswith('records.py'):
            continue
        try:
            grammar.grammarread(editype,g2)
        except:
            #~ print 'Found error in grammar:',g
            print botslib.txtexc()
            print '\n'
        else:
            print 'OK - no error found in grammar',g,'\n'
Пример #30
0
def startmulti(grammardir, editype):
    import glob

    botslib.initconfigurationfile("config")
    botslib.initlogging()
    for g in glob.glob(grammardir):
        g1 = os.path.basename(g)
        g2 = os.path.splitext(g1)[0]
        if g1 in ["__init__.py"]:
            continue
        if g1.startswith("edifact"):
            continue
        if g1.startswith("records") or g1.endswith("records.py"):
            continue
        try:
            grammar.grammarread(editype, g2)
        except:
            # ~ print 'Found error in grammar:',g
            print botslib.txtexc()
            print "\n"
        else:
            print "OK - no error found in grammar", g, "\n"
Пример #31
0
def startmulti(grammardir, editype):
    ''' used in seperate tool for bulk checking of gramamrs while developing edifact->botsgramamrs '''
    import glob
    botslib.generalinit('config')
    botslib.initenginelogging()
    for g in glob.glob(grammardir):
        g1 = os.path.basename(g)
        g2 = os.path.splitext(g1)[0]
        if g1 in ['__init__.py']:
            continue
        if g1.startswith('edifact'):
            continue
        if g1.startswith('records') or g1.endswith('records.py'):
            continue
        try:
            grammar.grammarread(editype, g2)
        except:
            #~ print 'Found error in grammar:',g
            print botslib.txtexc()
            print '\n'
        else:
            print 'OK - no error found in grammar', g, '\n'
Пример #32
0
def read_index(filename):
    ''' process index file in default location. '''
    try:
        importedbotsindex,scriptname = botslib.botsimport('index')
        pluglist = importedbotsindex.plugins[:]
        if 'botsindex' in sys.modules:
            del sys.modules['botsindex']
    except:
        txt = botslib.txtexc()
        raise botslib.PluginError(_(u'Error in configuration index file. Nothing is written. Error:\n%(txt)s'),{'txt':txt})
    else:
        botsglobal.logger.info(_(u'Configuration index file is OK.'))
        botsglobal.logger.info(_(u'Start writing to database.'))

    #write content of index file to the bots database
    try:
        read_index2database(pluglist)
    except:
        txt = botslib.txtexc()
        raise botslib.PluginError(_(u'Error writing configuration index to database. Nothing is written. Error:\n%(txt)s'),{'txt':txt})
    else:
        botsglobal.logger.info(_(u'Writing to database is OK.'))
Пример #33
0
 def _write(self,node_instance):
     templatefile = botslib.abspath(self.__class__.__name__,self.ta_info['template'])
     try:
         botsglobal.logger.debug(u'Start writing to file "%(filename)s".',self.ta_info)
         loader = self.template.TemplateLoader(auto_reload=False)
         tmpl = loader.load(templatefile)
     except:
         txt = botslib.txtexc()
         raise botslib.OutMessageError(_(u'While templating "%(editype)s.%(messagetype)s", error:\n%(txt)s'),
                                         {'editype':self.ta_info['editype'],'messagetype':self.ta_info['messagetype'],'txt':txt})
     try:
         filehandler = botslib.opendata(self.ta_info['filename'],'wb')
         if self.ta_info['has_structure']:   #new way of working
             if self.ta_info['print_as_row']:
                 node_instance.collectlines(self.ta_info['print_as_row'])
             stream = tmpl.generate(node=node_instance)
         else:
             stream = tmpl.generate(data=self.data)
         stream.render(method='xhtml',encoding=self.ta_info['charset'],out=filehandler)
     except:
         txt = botslib.txtexc()
         raise botslib.OutMessageError(_(u'While templating "%(editype)s.%(messagetype)s", error:\n%(txt)s'),
                                         {'editype':self.ta_info['editype'],'messagetype':self.ta_info['messagetype'],'txt':txt})
     botsglobal.logger.debug(_(u'End writing to file "%(filename)s".'),self.ta_info)
Пример #34
0
def read_incoming(run):
    outputdir = botslib.join(run.inpath,run.infilename)
    filelist = [filename for filename in glob.iglob(outputdir) if os.path.isfile(filename)]
    filelist.sort()
    for infilename in filelist:
        try:
            filename = transform.unique('bots_file_name')
            abs_filename = botslib.abspathdata(filename)
            shutil.copy(infilename,abs_filename)          #move if to be delted
        except:
            txt = botslib.txtexc()
        else:
            txt = ''    #no errors
        finally:
            run.incoming.append({'infilename':infilename,'filename':filename,'error':txt,'editype':run.translation['editype'],'messagetype':run.translation['messagetype']})
Пример #35
0
def write_outgoing(run):
    outputdir = botslib.join(run.outpath)
    botslib.dirshouldbethere(outputdir)
    for outgoing in run.outgoing:
        if not outgoing['error']:
            try:
                unique_filename = filename_formatter(run.outfilename,outgoing)
                tofilepath = botslib.join(outputdir,unique_filename)
                fromfilepath = botslib.abspathdata(outgoing['filename'])
                shutil.move(fromfilepath,tofilepath)
            except:
                txt = botslib.txtexc()
                outgoing.update({'error':txt})
            else:
                outgoing.update({'outfilename':tofilepath})
Пример #36
0
def write_outgoing(run):
    outputdir = botslib.join(run.outpath)
    botslib.dirshouldbethere(outputdir)
    for outgoing in run.outgoing:
        if not outgoing['error']:
            try:
                unique_filename = filename_formatter(run.outfilename, outgoing)
                tofilepath = botslib.join(outputdir, unique_filename)
                fromfilepath = botslib.abspathdata(outgoing['filename'])
                shutil.move(fromfilepath, tofilepath)
            except:
                txt = botslib.txtexc()
                outgoing.update({'error': txt})
            else:
                outgoing.update({'outfilename': tofilepath})
Пример #37
0
def preprocess(routedict,function, status=FILEIN,**argv):
    ''' for pre- and postprocessing of files.
        these are NOT translations; translation involve grammars, mapping scripts etc. think of eg:
        - unzipping zipped files.
        - convert excel to csv
        - password protected files.
        Select files from INFILE -> SET_FOR_PROCESSING using criteria
        Than the actual processing function is called.
        The processing function does: SET_FOR_PROCESSING -> PROCESSING -> FILEIN
        If errors occur during processing, no ta are left with status FILEIN !
        preprocess is called right after the in-communicatiation
    '''
    nr_files = 0
    preprocessnumber = botslib.getpreprocessnumber()
    if not botslib.addinfo(change={'status':preprocessnumber},where={'status':status,'idroute':routedict['idroute'],'fromchannel':routedict['fromchannel']}):    #check if there is something to do
        return 0
    for row in botslib.query(u'''SELECT idta,filename,charset
                                FROM  ta
                                WHERE   idta>%(rootidta)s
                                AND     status=%(status)s
                                AND     statust=%(statust)s
                                AND     idroute=%(idroute)s
                                AND     fromchannel=%(fromchannel)s
                                ''',
                                {'status':preprocessnumber,'statust':OK,'idroute':routedict['idroute'],'fromchannel':routedict['fromchannel'],'rootidta':botslib.get_minta4query()}):
        try:
            botsglobal.logmap.debug(u'Start preprocessing "%s" for file "%s".',function.__name__,row['filename'])
            ta_set_for_processing = botslib.OldTransaction(row['idta'])
            ta_processing = ta_set_for_processing.copyta(status=preprocessnumber+1)
            ta_processing.filename = row['filename']
            function(ta_from=ta_processing,endstatus=status,routedict=routedict,**argv)
        except:
            txt = botslib.txtexc()
            ta_processing.failure()
            ta_processing.update(statust=ERROR,errortext=txt)
        else:
            botsglobal.logmap.debug(u'OK preprocessing  "%s" for file "%s".',function.__name__,row['filename'])
            ta_set_for_processing.update(statust=DONE)
            ta_processing.update(statust=DONE)
            nr_files += 1
    return nr_files
Пример #38
0
def sendtestmailmanagers(request,*kw,**kwargs):
    try:
        sendornot = botsglobal.ini.getboolean('settings','sendreportiferror',False)
    except botslib.BotsError:
        sendornot = False
    if not sendornot:
        botsglobal.logger.info(_(u'Trying to send test mail, but in bots.ini, section [settings], "sendreportiferror" is not "True".'))
        request.user.message_set.create(message=_(u'Trying to send test mail, but in bots.ini, section [settings], "sendreportiferror" is not "True".'))
        return django.shortcuts.redirect('/home')

    from django.core.mail import mail_managers
    try:
        mail_managers(_(u'testsubject'), _(u'test content of report'))
    except:
        txt = botslib.txtexc()
        request.user.message_set.create(message=_(u'Sending test mail failed.'))
        botsglobal.logger.info(_(u'Sending test mail failed, error:\n%s'), txt)
        return django.shortcuts.redirect('/home')
    request.user.message_set.create(message=_(u'Sending test mail succeeded.'))
    botsglobal.logger.info(_(u'Sending test mail succeeded.'))
    return django.shortcuts.redirect('/home')
Пример #39
0
def sendtestmailmanagers(request, *kw, **kwargs):
    try:
        sendornot = botsglobal.ini.getboolean('settings', 'sendreportiferror',
                                              False)
    except botslib.BotsError:
        sendornot = False
    if not sendornot:
        request.user.message_set.create(message=_(
            u'In bots.ini, section [settings], "sendreportiferror" is not set (to "True").'
        ))
        return django.shortcuts.redirect('/')

    from django.core.mail import mail_managers
    try:
        mail_managers(_(u'testsubject'), _(u'test content of report'))
    except:
        request.user.message_set.create(
            message=_(u'Sending test report failed: "%s".') % botslib.txtexc())
        return django.shortcuts.redirect('/')
    request.user.message_set.create(
        message=_(u'Sending test report succeeded.'))
    return django.shortcuts.redirect('/')
Пример #40
0
def runengine(request,*kw,**kwargs):
    if request.method == 'GET':
        #check if bots-engine is not already running
        if list(models.mutex.objects.filter(mutexk=1).all()):
            request.user.message_set.create(message=_(u'Trying to run "bots-engine", but database is locked by another run in progress. Please try again later.'))
            botsglobal.logger.info(_(u'Trying to run "bots-engine", but database is locked by another run in progress.'))
            return django.shortcuts.redirect('/home')
        #find out the right arguments to use
        botsenginepath = os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])),'bots-engine.py')        #find the bots-engine
        lijst = [sys.executable,botsenginepath] + sys.argv[1:]
        if 'clparameter' in request.GET:
            lijst.append(request.GET['clparameter'])

        try:
            botsglobal.logger.info(_(u'Run bots-engine with parameters: "%s"'),str(lijst))
            terug = subprocess.Popen(lijst).pid
            request.user.message_set.create(message=_(u'Bots-engine is started.'))
        except:
            txt =  botslib.txtexc()
            request.user.message_set.create(message=_(u'Errors while trying to run bots-engine.'))
            botsglobal.logger.info(_(u'Errors while trying to run bots-engine:\n%s.'),txt)
    return django.shortcuts.redirect('/home')
Пример #41
0
def sendtestmailmanagers(request,*kw,**kwargs):
    try:
        sendornot = botsglobal.ini.getboolean('settings','sendreportiferror',False)
    except botslib.BotsError:
        sendornot = False
    if not sendornot:
        notification = _(u'Trying to send test mail, but in bots.ini, section [settings], "sendreportiferror" is not "True".')
        botsglobal.logger.info(notification)
        messages.add_message(request, messages.INFO, notification)
        return django.shortcuts.redirect('/home')

    from django.core.mail import mail_managers
    try:
        mail_managers(_(u'testsubject'), _(u'test content of report'))
    except:
        txt = botslib.txtexc()
        messages.add_message(request, messages.INFO, _(u'Sending test mail failed.'))
        botsglobal.logger.info(_(u'Sending test mail failed, error:\n%s(txt)'), {'txt':txt})
        return django.shortcuts.redirect('/home')
    notification = _(u'Sending test mail succeeded.')
    messages.add_message(request, messages.INFO, notification)
    botsglobal.logger.info(notification)
    return django.shortcuts.redirect('/home')
Пример #42
0
def read_incoming(run):
    outputdir = botslib.join(run.inpath, run.infilename)
    filelist = [
        filename for filename in glob.iglob(outputdir)
        if os.path.isfile(filename)
    ]
    filelist.sort()
    for infilename in filelist:
        try:
            filename = transform.unique('bots_file_name')
            abs_filename = botslib.abspathdata(filename)
            shutil.copy(infilename, abs_filename)  #move if to be delted
        except:
            txt = botslib.txtexc()
        else:
            txt = ''  #no errors
        finally:
            run.incoming.append({
                'infilename': infilename,
                'filename': filename,
                'error': txt,
                'editype': run.translation['editype'],
                'messagetype': run.translation['messagetype']
            })
Пример #43
0
def mergemessages(startstatus=TRANSLATED, endstatus=MERGED, idroute=""):
    """ Merges en envelopes several messages to one file;
        In db-ta: attribute 'merge' indicates message should be merged with similar messages; 'merge' is generated in translation from messagetype-grammar
        If merge==False: 1 message per envelope - no merging, else append all similar messages to one file
        Implementation as separate loops: one for merge&envelope, another for enveloping only
        db-ta status TRANSLATED---->MERGED
    """
    outerqueryparameters = {
        "status": startstatus,
        "statust": OK,
        "idroute": idroute,
        "rootidta": botslib.get_minta4query(),
        "merge": False,
    }
    # **********for messages only to envelope (no merging)
    for row in botslib.query(
        u"""SELECT editype,messagetype,frompartner,topartner,testindicator,charset,contenttype,tochannel,envelope,nrmessages,idta,filename,idroute,merge
                                FROM  ta
                                WHERE   idta>%(rootidta)s
                                AND     status=%(status)s
                                AND     statust=%(statust)s
                                AND     idroute=%(idroute)s
                                AND     merge=%(merge)s
                                """,
        outerqueryparameters,
    ):
        try:
            ta_info = dict([(key, row[key]) for key in row.keys()])
            # ~ ta_info={'merge':False,'idroute':idroute}
            # ~ for key in row.keys():
            # ~ ta_info[key] = row[key]
            ta_fromfile = botslib.OldTransaction(row["idta"])  # edi message to envelope
            ta_tofile = ta_fromfile.copyta(
                status=endstatus
            )  # edifile for enveloped message; attributes of not-enveloped message are copied...
            # ~ ta_fromfile.update(child=ta_tofile.idta)        #??there is already a parent-child relation (1-1)...
            ta_info["filename"] = str(ta_tofile.idta)  # create filename for enveloped message
            botsglobal.logger.debug(
                u"Envelope 1 message editype: %s, messagetype: %s.", ta_info["editype"], ta_info["messagetype"]
            )
            envelope(ta_info, [row["filename"]])
        except:
            txt = botslib.txtexc()
            ta_tofile.update(statust=ERROR, errortext=txt)
        else:
            ta_fromfile.update(statust=DONE)
            ta_tofile.update(statust=OK, **ta_info)  # selection is used to update enveloped message;

    # **********for messages to merge & envelope
    # all GROUP BY fields must be used in SELECT!
    # as files get merged: can not copy idta; must extract relevant attributes.
    outerqueryparameters["merge"] = True
    for row in botslib.query(
        u"""SELECT editype,messagetype,frompartner,topartner,tochannel,testindicator,charset,contenttype,envelope,sum(nrmessages) as nrmessages
                                FROM  ta
                                WHERE   idta>%(rootidta)s
                                AND     status=%(status)s
                                AND     statust=%(statust)s
                                AND     idroute=%(idroute)s
                                AND     merge=%(merge)s
                                GROUP BY editype,messagetype,frompartner,topartner,tochannel,testindicator,charset,contenttype,envelope
                                """,
        outerqueryparameters,
    ):
        try:
            ta_info = dict([(key, row[key]) for key in row.keys()])
            ta_info.update({"merge": False, "idroute": idroute})
            # ~ for key in row.keys():
            # ~ ta_info[key] = row[key]
            ta_tofile = botslib.NewTransaction(status=endstatus, idroute=idroute)  # edifile for enveloped messages
            ta_info["filename"] = str(ta_tofile.idta)  # create filename for enveloped message
            innerqueryparameters = ta_info.copy()
            innerqueryparameters.update(outerqueryparameters)
            ta_list = []
            # gather individual idta and filenames
            # explicitly allow formpartner/topartner to be None/NULL
            for row2 in botslib.query(
                u"""SELECT idta, filename
                                                    FROM ta
                                                    WHERE idta>%(rootidta)s
                                                    AND status=%(status)s
                                                    AND statust=%(statust)s
                                                    AND merge=%(merge)s
                                                    AND editype=%(editype)s
                                                    AND messagetype=%(messagetype)s
                                                    AND (frompartner=%(frompartner)s OR frompartner IS NULL)
                                                    AND (topartner=%(topartner)s OR topartner IS NULL)
                                                    AND tochannel=%(tochannel)s
                                                    AND testindicator=%(testindicator)s
                                                    AND charset=%(charset)s
                                                    AND idroute=%(idroute)s
                                                    """,
                innerqueryparameters,
            ):
                ta_fromfile = botslib.OldTransaction(row2["idta"])  # edi message to envelope
                ta_fromfile.update(statust=DONE, child=ta_tofile.idta)  # st child because of n->1 relation
                ta_list.append(row2["filename"])
            botsglobal.logger.debug(
                u"Merge and envelope: editype: %s, messagetype: %s, %s messages",
                ta_info["editype"],
                ta_info["messagetype"],
                ta_info["nrmessages"],
            )
            envelope(ta_info, ta_list)
        except:
            txt = botslib.txtexc()
            ta_tofile.mergefailure()
            ta_tofile.update(statust=ERROR, errortext=txt)
        else:
            ta_tofile.update(statust=OK, **ta_info)
Пример #44
0
def mergemessages(startstatus, endstatus, idroute, rootidta=None):
    ''' Merges and/or envelopes one or more messages to one file;
        In db-ta: attribute 'merge' indicates message should be merged with similar messages; 'merge' is generated in translation from messagetype-grammar
        If merge is False: 1 message per envelope - no merging, else append all similar messages to one file
        Implementation as separate loops: one for merge&envelope, another for enveloping only
        db-ta status TRANSLATED---->FILEOUT
    '''
    if rootidta is None:
        rootidta = botsglobal.currentrun.get_minta4query()
    #**********for messages only to envelope (no merging)
    for row in botslib.query(
            u'''SELECT editype,messagetype,frompartner,topartner,testindicator,charset,contenttype,envelope,nrmessages,idroute,merge,idta,filename,rsrv3
                                FROM ta
                                WHERE idta>%(rootidta)s
                                AND status=%(status)s
                                AND statust=%(statust)s
                                AND merge=%(merge)s
                                AND idroute=%(idroute)s
                                ORDER BY idta
                                ''', {
                'rootidta': rootidta,
                'status': startstatus,
                'statust': OK,
                'merge': False,
                'idroute': idroute
            }):
        try:
            ta_info = dict(row)
            ta_fromfile = botslib.OldTransaction(
                ta_info['idta'])  #edi message to envelope
            ta_tofile = ta_fromfile.copyta(
                status=endstatus
            )  #edifile for enveloped message; attributes of not-enveloped message are copied...
            ta_info['filename'] = unicode(
                ta_tofile.idta)  #create filename for enveloped message
            botsglobal.logger.debug(
                u'Envelope 1 message editype: %(editype)s, messagetype: %(messagetype)s.',
                ta_info)
            envelope(ta_info, [row['filename']])
            ta_info['filesize'] = os.path.getsize(
                botslib.abspathdata(ta_info['filename']))  #get filesize
        except:
            txt = botslib.txtexc()
            ta_tofile.update(statust=ERROR, errortext=txt)
        else:
            ta_tofile.update(
                statust=OK,
                **ta_info)  #selection is used to update enveloped message;
        finally:
            ta_fromfile.update(statust=DONE)

    #**********for messages to merge & envelope
    for row in botslib.query(
            u'''SELECT editype,messagetype,frompartner,topartner,testindicator,charset,contenttype,envelope,rsrv3,sum(nrmessages) as nrmessages
                                FROM ta
                                WHERE idta>%(rootidta)s
                                AND status=%(status)s
                                AND statust=%(statust)s
                                AND merge=%(merge)s
                                AND idroute=%(idroute)s
                                GROUP BY editype,messagetype,frompartner,topartner,testindicator,charset,contenttype,envelope,rsrv3
                                ORDER BY editype,messagetype,frompartner,topartner,testindicator,charset,contenttype,envelope,rsrv3
                                ''', {
                'rootidta': rootidta,
                'status': startstatus,
                'statust': OK,
                'merge': True,
                'idroute': idroute
            }):
        try:
            ta_info = dict(row)
            ta_tofile = botslib.NewTransaction(
                status=endstatus,
                idroute=idroute)  #edifile for enveloped messages
            ta_info.update({
                'idroute': idroute,
                'merge': False,
                'filename': unicode(ta_tofile.idta)
            })  #SELECT/GROUP BY gives only values that are the grouped
            filename_list = []
            #gather individual idta and filenames
            #explicitly allow formpartner/topartner to be None/NULL
            for row2 in botslib.query(
                    u'''SELECT idta, filename
                                            FROM ta
                                            WHERE idta>%(rootidta)s
                                            AND status=%(status)s
                                            AND statust=%(statust)s
                                            AND merge=%(merge)s
                                            AND editype=%(editype)s
                                            AND messagetype=%(messagetype)s
                                            AND (frompartner=%(frompartner)s OR frompartner IS NULL)
                                            AND (topartner=%(topartner)s OR topartner IS NULL)
                                            AND testindicator=%(testindicator)s
                                            AND charset=%(charset)s
                                            ORDER BY idta
                                            ''', {
                        'rootidta': rootidta,
                        'status': startstatus,
                        'statust': OK,
                        'merge': True,
                        'editype': ta_info['editype'],
                        'messagetype': ta_info['messagetype'],
                        'frompartner': ta_info['frompartner'],
                        'topartner': ta_info['topartner'],
                        'testindicator': ta_info['testindicator'],
                        'charset': ta_info['charset'],
                        'rsrv3': ta_info['rsrv3']
                    }):
                ta_fromfile = botslib.OldTransaction(
                    row2['idta'])  #edi message to be merged/envelope
                ta_fromfile.update(
                    child=ta_tofile.idta,
                    statust=DONE)  #st child because of n->1 relation
                filename_list.append(row2['filename'])
            botsglobal.logger.debug(
                u'Merge and envelope: editype: %(editype)s, messagetype: %(messagetype)s, %(nrmessages)s messages',
                ta_info)
            envelope(ta_info, filename_list)
            ta_info['filesize'] = os.path.getsize(
                botslib.abspathdata(ta_info['filename']))  #get filesize
        except:
            txt = botslib.txtexc()
            ta_tofile.update(statust=ERROR, errortext=txt)
        else:
            ta_tofile.update(statust=OK, **ta_info)
Пример #45
0
def extractpdf(ta_from,endstatus,**argv):
    ''' Try to extract text content of a PDF file to a csv.
        You know this is not a great idea, right? But we'll do the best we can anyway!
        Page and line numbers are added to each row.
        Columns and rows are based on the x and y coordinates of each text element within tolerance allowed.
        Multiple text elements may combine to make one field, some PDFs have every character separated!
        You may need to experiment with x_group and y_group values, but defaults seem ok for most files.
        Output csv is UTF-8 encoded - The csv module doesn't directly support reading and writing Unicode
        If the PDF is just an image, all bets are off. Maybe try OCR, good luck with that!
        Mike Griffin 14/12/2011
    '''
    from pdfminer.pdfinterp import PDFResourceManager, process_pdf
    from pdfminer.converter import TextConverter
    from pdfminer.layout import LAParams, LTContainer, LTText, LTTextBox
    import csv

    class CsvConverter(TextConverter):
        def __init__(self, *args, **kwargs):
            TextConverter.__init__(self, *args, **kwargs)

        def receive_layout(self, ltpage):

            # recursively get every text element and it's coordinates
            def render(item):
                if isinstance(item, LTContainer):
                    for child in item:
                        render(child)
                elif isinstance(item, LTText):
                    (_,_,x,y) = item.bbox

                    # group the y values (rows) within group tolerance
                    for v in yv:
                        if y > v-y_group and y < v+y_group:
                            y = v
                    yv.append(y)

                    line = lines[int(-y)]
                    line[x] = item.get_text().encode('utf-8')

            from collections import defaultdict
            lines = defaultdict(lambda : {})

            yv = []
            render(ltpage)

            lineid = 0
            for y in sorted(lines.keys()):
                line = lines[y]
                lineid += 1
                csvdata = [ltpage.pageid,lineid] # first 2 columns are page and line numbers

                # group the x values (fields) within group tolerance
                p = 0
                field_txt = ''
                for x in sorted(line.keys()):
                    gap = x - p
                    if p > 0 and gap > x_group:
                        csvdata.append(field_txt)
                        field_txt = ''
                    field_txt += line[x]
                    p = x
                csvdata.append(field_txt)
                csvout.writerow(csvdata)
            if lineid == 0:
                raise botslib.InMessageError(_(u'PDF text extraction failed, it may contain just image(s)?'))


    #get some optional parameters
    x_group = argv.get('x_group',10) # group text closer than this as one field
    y_group = argv.get('y_group',5)  # group lines closer than this as one line
    password = argv.get('password','')
    quotechar = argv.get('quotechar','"')
    field_sep = argv.get('field_sep',',')
    escape = argv.get('escape','\\')
    charset = argv.get('charset','utf-8')
    if not escape:
        doublequote = True
    else:
        doublequote = False

    try:
        pdf_stream = botslib.opendata(ta_from.filename, 'rb')
        ta_to = ta_from.copyta(status=endstatus)
        tofilename = str(ta_to.idta)
        csv_stream = botslib.opendata(tofilename,'wb')
        csvout = csv.writer(csv_stream, quotechar=quotechar, delimiter=field_sep, doublequote=doublequote, escapechar=escape)

        # Process PDF
        rsrcmgr = PDFResourceManager(caching=True)
        device = CsvConverter(rsrcmgr, csv_stream, codec=charset)
        process_pdf(rsrcmgr, device, pdf_stream, pagenos=set(), password=password, caching=True, check_extractable=True)

        device.close()
        pdf_stream.close()
        csv_stream.close()
        ta_to.update(statust=OK,filename=tofilename) #update outmessage transaction with ta_info;
        botsglobal.logger.debug(_(u'        File written: "%s".'),tofilename)
    except:
        txt = botslib.txtexc()
        botsglobal.logger.error(_(u'PDF extraction failed, may not be a PDF file? Error:\n%s'),txt)
        raise botslib.InMessageError(_(u'PDF extraction failed, may not be a PDF file? Error:\n$error'),error=txt)
Пример #46
0
def extractexcel(ta_from,endstatus,**argv):
    ''' extract excel file.
        editype & messagetype are unchanged.
    '''
    #***functions used by extractexcel
    #-------------------------------------------------------------------------------
    def read_xls(infilename):
        # Read excel first sheet into a 2-d array
        book       = xlrd.open_workbook(infilename)
        sheet      = book.sheet_by_index(0)
        formatter  = lambda(t,v): format_excelval(book,t,v,False)
        xlsdata = []
        for row in range(sheet.nrows):
            (types, values) = (sheet.row_types(row), sheet.row_values(row))
            xlsdata.append(map(formatter, zip(types, values)))
        return xlsdata
    #-------------------------------------------------------------------------------
    def dump_csv(xlsdata, tofilename):
        stream = botslib.opendata(tofilename, 'wb')
        csvout = csv.writer(stream, quotechar=quotechar, delimiter=field_sep, doublequote=doublequote, escapechar=escape)
        csvout.writerows( map(utf8ize, xlsdata) )
        stream.close()
    #-------------------------------------------------------------------------------
    def format_excelval(book, type, value, wanttupledate):
        #  Clean up the incoming excel data for some data types
        returnrow = []
        if   type == 2:
            if value == int(value):
                value = int(value)
        elif type == 3:
            datetuple = xlrd.xldate_as_tuple(value, book.datemode)
            value = datetuple if wanttupledate else tupledate_to_isodate(datetuple)
        elif type == 5:
            value = xlrd.error_text_from_code[value]
        return value
    #-------------------------------------------------------------------------------
    def tupledate_to_isodate(tupledate):
        # Turns a gregorian (year, month, day, hour, minute, nearest_second) into a
        # standard YYYY-MM-DDTHH:MM:SS ISO date.
        (y,m,d, hh,mm,ss) = tupledate
        nonzero = lambda n: n != 0
        date = "%04d-%02d-%02d"  % (y,m,d)    if filter(nonzero, (y,m,d))                else ''
        time = "T%02d:%02d:%02d" % (hh,mm,ss) if filter(nonzero, (hh,mm,ss)) or not date else ''
        return date+time
    #-------------------------------------------------------------------------------
    def utf8ize(l):
        # Make string-like things into utf-8, leave other things alone
        return [unicode(s).encode(charset) if hasattr(s,'encode') else s for s in l]
    #***end functions used by extractexcel
    import xlrd
    import csv
    #get parameters for csv-format; defaults are as the csv defaults (in grammar.py)
    charset = argv.get('charset',"utf-8")
    quotechar = argv.get('quotechar',"'")
    field_sep = argv.get('field_sep',':')
    escape = argv.get('escape','')
    if escape:
        doublequote = False
    else:
        doublequote = True
    try:
        infilename = botslib.abspathdata(ta_from.filename)
        xlsdata = read_xls(infilename)
        ta_to = ta_from.copyta(status=endstatus)
        tofilename = str(ta_to.idta)
        dump_csv(xlsdata,tofilename)
        ta_to.update(statust=OK,filename=tofilename) #update outmessage transaction with ta_info;
        botsglobal.logger.debug(_(u'        File written: "%s".'),tofilename)
    except:
        txt = botslib.txtexc()
        botsglobal.logger.error(_(u'Excel extraction failed, may not be an Excel file? Error:\n%s'),txt)
        raise botslib.InMessageError(_(u'Excel extraction failed, may not be an Excel file? Error:\n$error'),error=txt)
Пример #47
0
def _translate_one_file(row,routedict,endstatus,userscript,scriptname):
    ''' -   read, lex, parse, make tree of nodes.
        -   split up files into messages (using 'nextmessage' of grammar)
        -   get mappingscript, start mappingscript.
        -   write the results of translation (no enveloping yet)
    '''
    try:
        ta_fromfile = botslib.OldTransaction(row['idta'])
        ta_parsed = ta_fromfile.copyta(status=PARSED)
        if row['filesize'] > botsglobal.ini.getint('settings','maxfilesizeincoming',5000000):
            ta_parsed.update(filesize=row['filesize'])
            raise botslib.FileTooLarge(_(u'File size of %(filesize)s is too big; option "maxfilesizeincoming" in bots.ini is %(maxfilesizeincoming)s.'),
                                            {'filesize':row['filesize'],'maxfilesizeincoming':botsglobal.ini.getint('settings','maxfilesizeincoming',5000000)})
        botsglobal.logger.debug(_(u'Start translating file "%(filename)s" editype "%(editype)s" messagetype "%(messagetype)s".'),row)
        #read whole edi-file: read, parse and made into a inmessage-object. Message is represented as a tree (inmessage.root is the root of the tree).
        edifile = inmessage.parse_edi_file(frompartner=row['frompartner'],
                                            topartner=row['topartner'],
                                            filename=row['filename'],
                                            messagetype=row['messagetype'],
                                            testindicator=row['testindicator'],
                                            editype=row['editype'],
                                            charset=row['charset'],
                                            alt=row['alt'],
                                            fromchannel=row['fromchannel'],
                                            idroute=routedict['idroute'],
                                            command=routedict['command'])
        edifile.checkforerrorlist() #no exception if infile has been lexed and parsed OK else raises an error

        if int(routedict['translateind']) == 3: #parse & passthrough; file is parsed, partners are known, no mapping, does confirm.
            raise botslib.GotoException('dummy')    
        
        #edifile.ta_info contains info: QUERIES, charset etc
        for inn_splitup in edifile.nextmessage():   #splitup messages in parsed edifile
            try:
                ta_splitup = ta_parsed.copyta(status=SPLITUP,**inn_splitup.ta_info)    #copy PARSED to SPLITUP ta
                #inn_splitup.ta_info: parameters from inmessage.parse_edi_file(), syntax-information and parse-information
                inn_splitup.ta_info['idta_fromfile'] = ta_fromfile.idta     #for confirmations in userscript; used to give idta of 'confirming message'
                inn_splitup.ta_info['idta'] = ta_splitup.idta     #for confirmations in userscript; used to give idta of 'confirming message'
                number_of_loops_with_same_alt = 0
                while 1:    #continue as long as there are (alt-)translations
                    #lookup the translation************************
                    tscript,toeditype,tomessagetype = botslib.lookup_translation(fromeditype=inn_splitup.ta_info['editype'],
                                                                        frommessagetype=inn_splitup.ta_info['messagetype'],
                                                                        frompartner=inn_splitup.ta_info['frompartner'],
                                                                        topartner=inn_splitup.ta_info['topartner'],
                                                                        alt=inn_splitup.ta_info['alt'])
                    if not tscript:       #no translation found in translate table; check if can find translation via user script
                        if userscript and hasattr(userscript,'gettranslation'):
                            tscript,toeditype,tomessagetype = botslib.runscript(userscript,scriptname,'gettranslation',idroute=routedict['idroute'],message=inn_splitup)
                        if not tscript:
                            raise botslib.TranslationNotFoundError(_(u'Translation not found for editype "%(editype)s", messagetype "%(messagetype)s", frompartner "%(frompartner)s", topartner "%(topartner)s", alt "%(alt)s".'),
                                                                        inn_splitup.ta_info)

                    inn_splitup.ta_info['divtext'] = tscript     #ifor reporting used mapping script to database (for display in GUI).
                    #initialize new out-object*************************
                    ta_translated = ta_splitup.copyta(status=endstatus)     #make ta for translated message (new out-ta)
                    filename_translated = str(ta_translated.idta)
                    out_translated = outmessage.outmessage_init(editype=toeditype,messagetype=tomessagetype,filename=filename_translated,reference=unique('messagecounter'),statust=OK,divtext=tscript)    #make outmessage object
                        
                    #run mapping script************************
                    botsglobal.logger.debug(_(u'Mappingscript "%(tscript)s" translates messagetype "%(messagetype)s" to messagetype "%(tomessagetype)s".'),
                                            {'tscript':tscript,'messagetype':inn_splitup.ta_info['messagetype'],'tomessagetype':out_translated.ta_info['messagetype']})
                    translationscript,scriptfilename = botslib.botsimport('mappings',inn_splitup.ta_info['editype'],tscript) #get the mappingscript
                    alt_from_previous_run = inn_splitup.ta_info['alt']      #needed to check for infinite loop
                    doalttranslation = botslib.runscript(translationscript,scriptfilename,'main',inn=inn_splitup,out=out_translated)
                    botsglobal.logger.debug(_(u'Mappingscript "%(tscript)s" finished.'),{'tscript':tscript})
                    
                    #manipulate for some attributes after mapping script
                    if 'topartner' not in out_translated.ta_info:    #out_translated does not contain values from ta......
                        out_translated.ta_info['topartner'] = inn_splitup.ta_info['topartner']
                    if 'botskey' in inn_splitup.ta_info:
                        inn_splitup.ta_info['reference'] = inn_splitup.ta_info['botskey']
                    if 'botskey' in out_translated.ta_info:    #out_translated does not contain values from ta......
                        out_translated.ta_info['reference'] = out_translated.ta_info['botskey']
                        
                    #check the value received from the mappingscript to determine what to do in this while-loop. Handling of chained trasnlations.
                    if doalttranslation is None:    
                        #translation(s) are done; handle out-message 
                        handle_out_message(out_translated,ta_translated)
                        break   #break out of while loop
                    elif isinstance(doalttranslation,dict):
                        #some extended cases; a dict is returned that contains 'instructions' for some type of chained translations
                        if 'type' not in doalttranslation or 'alt' not in doalttranslation:
                            raise botslib.BotsError(_(u"Mappingscript returned '%(alt)s'. This dict should not have 'type' and 'alt'."),{'alt':doalttranslation})
                        if alt_from_previous_run == doalttranslation['alt']:
                            number_of_loops_with_same_alt += 1
                        else:
                            number_of_loops_with_same_alt = 0
                        if doalttranslation['type'] == u'out_as_inn':
                            #do chained translation: use the out-object as inn-object, new out-object
                            #use case: detected error in incoming file; use out-object to generate warning email
                            handle_out_message(out_translated,ta_translated)
                            inn_splitup = out_translated    #out-object is now inn-object
                            if isinstance(inn_splitup,outmessage.fixed):    #for fixed: strip all values in node
                                inn_splitup.root.stripnode()
                            inn_splitup.ta_info['alt'] = doalttranslation['alt']   #get the alt-value for the next chained translation
                            if not 'frompartner' in inn_splitup.ta_info:
                                inn_splitup.ta_info['frompartner'] = ''
                            if not 'topartner' in inn_splitup.ta_info:
                                inn_splitup.ta_info['topartner'] = ''
                            inn_splitup.ta_info.pop('statust')
                        elif doalttranslation['type'] == u'no_check_on_infinite_loop':
                            #do chained translation: allow many loops wit hsame alt-value.
                            #mapping script will have to handle this correctly.
                            number_of_loops_with_same_alt = 0
                            handle_out_message(out_translated,ta_translated)
                            inn_splitup.ta_info['alt'] = doalttranslation['alt']   #get the alt-value for the next chained translation
                        else:   #there is nothing else
                            raise botslib.BotsError(_(u'Mappingscript returned dict with an unknown "type": "%(doalttranslation)s".'),{'doalttranslation':doalttranslation})
                    else:  #note: this includes alt '' (empty string)
                        if alt_from_previous_run == doalttranslation:
                            number_of_loops_with_same_alt += 1
                        else:
                            number_of_loops_with_same_alt = 0
                        #do normal chained translation: same inn-object, new out-object
                        handle_out_message(out_translated,ta_translated)
                        inn_splitup.ta_info['alt'] = doalttranslation   #get the alt-value for the next chained translation
                    if number_of_loops_with_same_alt > 10:
                        raise botslib.BotsError(_(u'Mappingscript returns same alt value over and over again (infinite loop?). Alt: "%(doalttranslation)s".'),{'doalttranslation':doalttranslation})
                #end of while-loop (trans**********************************************************************************
            #exceptions file_out-level: exception in mappingscript or writing of out-file
            except:
                #2 modes: either every error leads to skipping of  whole infile (old  mode) or errors in mappingscript/outfile only affect that branche
                if botsglobal.ini.getboolean('settings','oldmessageerrors',False):
                    raise
                txt = botslib.txtexc()
                ta_splitup.update(statust=ERROR,errortext=txt,**inn_splitup.ta_info)   #update db. inn_splitup.ta_info could be changed by mappingscript. Is this useful?
                ta_splitup.deletechildren()
            else:
                ta_splitup.update(statust=DONE, **inn_splitup.ta_info)   #update db. inn_splitup.ta_info could be changed by mappingscript. Is this useful?

    #exceptions file_in-level
    except botslib.GotoException:   #edi-file is OK, file is passed-through after parsing.
        ta_parsed.update(statust=DONE,filesize=row['filesize'],**edifile.ta_info)   #update with info from eg queries
        ta_parsed.copyta(status=MERGED,statust=OK)          #original file goes straight to MERGED
        edifile.handleconfirm(ta_fromfile,error=False)
        botsglobal.logger.debug(_(u'Parse & passthrough for input file "%(filename)s".'),row)
    except botslib.FileTooLarge as msg:
        ta_parsed.update(statust=ERROR,errortext=str(msg))
        ta_parsed.deletechildren()
        botsglobal.logger.debug(u'Error in translating input file "%(filename)s":\n%(msg)s',{'filename':row['filename'],'msg':msg})
    except:
        txt = botslib.txtexc()
        ta_parsed.update(statust=ERROR,errortext=txt,**edifile.ta_info)
        ta_parsed.deletechildren()
        edifile.handleconfirm(ta_fromfile,error=True)
        botsglobal.logger.debug(u'Error in translating input file "%(filename)s":\n%(msg)s',{'filename':row['filename'],'msg':txt})
    else:
        edifile.handleconfirm(ta_fromfile,error=False)
        ta_parsed.update(statust=DONE,filesize=row['filesize'],**edifile.ta_info)
        botsglobal.logger.debug(_(u'Translated input file "%(filename)s".'),row)
    finally:
        ta_fromfile.update(statust=DONE)
Пример #48
0
def load(pathzipfile):
    ''' process uploaded plugin. '''
    #test is valid zipfile
    if not zipfile.is_zipfile(pathzipfile):
        raise botslib.PluginError(_(u'Plugin is not a valid file.'))

    #read index file
    try:
        Zipimporter = zipimport.zipimporter(pathzipfile)
        importedbotsindex = Zipimporter.load_module('botsindex')
        pluglist = importedbotsindex.plugins[:]
        if 'botsindex' in sys.modules:
            del sys.modules['botsindex']
    except:
        txt = botslib.txtexc()
        raise botslib.PluginError(
            _(u'Error in plugin. Nothing is written. Error: "%s"') % (txt))
    else:
        botsglobal.logger.info(_(u'Plugin is OK.\nStart writing to database.'))

    #write content of index file to the bots database
    try:
        writetodatabase(pluglist)
    except:
        txt = botslib.txtexc()
        raise botslib.PluginError(
            'Error writing plugin to database. Nothing is written. Error: "%s"'
            % (txt))
    else:
        botsglobal.logger.info(
            u'Writing to database is OK.\nStart writing to files')

    #write files to the file system.
    try:
        warnrenamed = False  #to report in GUI files have been overwritten.
        z = zipfile.ZipFile(pathzipfile, mode="r")
        orgtargetpath = botsglobal.ini.get('directories', 'botspath')
        if (orgtargetpath[-1:] in (os.path.sep, os.path.altsep)
                and len(os.path.splitdrive(orgtargetpath)[1]) > 1):
            orgtargetpath = orgtargetpath[:-1]
        for f in z.infolist():
            if f.filename not in [
                    'botsindex.py', 'README', 'botssys/sqlitedb/botsdb',
                    'config/bots.ini'
            ] and os.path.splitext(f.filename)[1] not in ['.pyo', '.pyc']:
                #~ botsglobal.logger.info(u'filename in zip "%s".',f.filename)
                if f.filename[0] == '/':
                    targetpath = f.filename[1:]
                else:
                    targetpath = f.filename
                targetpath = targetpath.replace(
                    'usersys', botsglobal.ini.get('directories', 'usersysabs'),
                    1)
                targetpath = targetpath.replace(
                    'botssys', botsglobal.ini.get('directories', 'botssys'), 1)
                targetpath = botslib.join(orgtargetpath, targetpath)
                #targetpath is OK now.
                botsglobal.logger.info(_(u'    Start writing file: "%s".'),
                                       targetpath)

                if botslib.dirshouldbethere(os.path.dirname(targetpath)):
                    botsglobal.logger.info(
                        _(u'        Create directory "%s".'),
                        os.path.dirname(targetpath))
                if f.filename[
                        -1] == '/':  #check if this is a dir; if so continue
                    continue
                if os.path.isfile(targetpath):  #check if file already exists
                    try:  #this ***sometimes*** fails. (python25, for static/help/home.html...only there...)
                        os.rename(
                            targetpath,
                            targetpath + '.' + time.strftime('%Y%m%d%H%M%S'))
                        warnrenamed = True
                        botsglobal.logger.info(
                            _(u'        Renamed existing file "%(from)s" to "%(to)s".'
                              ), {
                                  'from': targetpath,
                                  'to':
                                  targetpath + time.strftime('%Y%m%d%H%M%S')
                              })
                    except:
                        pass
                source = z.read(f.filename)
                target = open(targetpath, "wb")
                target.write(source)
                target.close()
                botsglobal.logger.info(_(u'        File written: "%s".'),
                                       targetpath)
    except:
        txt = botslib.txtexc()
        z.close()
        raise botslib.PluginError(
            _(u'Error writing files to system. Nothing is written to database. Error: "%s"'
              ) % (txt))
    else:
        z.close()
        botsglobal.logger.info(_(u'Writing files to filesystem is OK.'))
        return warnrenamed
Пример #49
0
def mysql():
    print 'Start changing mysql database to bots version 3.'
    cursor = botsglobal.db.cursor()
    try:
        #channel ****************************************
        cursor.execute(
            '''ALTER TABLE `channel` MODIFY `filename` VARCHAR(256) NOT NULL'''
        )
        cursor.execute(
            '''ALTER TABLE `channel` ADD COLUMN `rsrv3` INTEGER DEFAULT 0''')
        cursor.execute(
            '''ALTER TABLE `channel` ADD COLUMN `keyfile` VARCHAR(256) ''')
        cursor.execute(
            '''ALTER TABLE `channel` ADD COLUMN `certfile` VARCHAR(256) ''')
        cursor.execute(
            '''ALTER TABLE `channel` ADD COLUMN `testpath` VARCHAR(256) ''')
        #ccode ****************************************
        cursor.execute(
            '''ALTER TABLE `ccode` MODIFY `rightcode` VARCHAR(70)''')
        cursor.execute('''ALTER TABLE `ccode` MODIFY `attr1` VARCHAR(70)''')
        #filereport ****************************************
        cursor.execute('''ALTER TABLE `filereport` CHANGE `id` `id` INTEGER '''
                       )  #drop autoincrement
        cursor.execute('''ALTER TABLE `filereport` DROP PRIMARY KEY '''
                       )  #drop index on id
        cursor.execute(
            '''ALTER TABLE `filereport` DROP COLUMN `id` ''')  #drop id veld
        cursor.execute('''ALTER TABLE `filereport` DROP KEY `idta` '''
                       )  #remove UNIQUE constraint
        #~ cursor.execute('''ALTER TABLE `filereport` DROP INDEX `reportidta` ''')  #not possible as index name is not known
        cursor.execute('''ALTER TABLE `filereport` MODIFY `errortext` TEXT''')
        cursor.execute(
            '''ALTER TABLE `filereport` ADD COLUMN `filesize` INTEGER DEFAULT 0'''
        )
        cursor.execute(
            '''ALTER TABLE `filereport` ADD COLUMN `acceptance` INTEGER DEFAULT 0'''
        )
        #partner *************************************
        cursor.execute(
            '''ALTER TABLE `partner` ADD COLUMN `attr1` VARCHAR(35) ''')
        cursor.execute(
            '''ALTER TABLE `partner` ADD COLUMN `attr2` VARCHAR(35) ''')
        cursor.execute(
            '''ALTER TABLE `partner` ADD COLUMN `attr3` VARCHAR(35) ''')
        cursor.execute(
            '''ALTER TABLE `partner` ADD COLUMN `attr4` VARCHAR(35) ''')
        cursor.execute(
            '''ALTER TABLE `partner` ADD COLUMN `attr5` VARCHAR(70) ''')
        cursor.execute(
            '''ALTER TABLE `partner` ADD COLUMN `name1` VARCHAR(70) ''')
        cursor.execute(
            '''ALTER TABLE `partner` ADD COLUMN `name2` VARCHAR(70) ''')
        cursor.execute(
            '''ALTER TABLE `partner` ADD COLUMN `name3` VARCHAR(70) ''')
        cursor.execute(
            '''ALTER TABLE `partner` ADD COLUMN `address1` VARCHAR(70) ''')
        cursor.execute(
            '''ALTER TABLE `partner` ADD COLUMN `address2` VARCHAR(70) ''')
        cursor.execute(
            '''ALTER TABLE `partner` ADD COLUMN `address3` VARCHAR(70) ''')
        cursor.execute(
            '''ALTER TABLE `partner` ADD COLUMN `city` VARCHAR(35) ''')
        cursor.execute(
            '''ALTER TABLE `partner` ADD COLUMN `postalcode` VARCHAR(17) ''')
        cursor.execute(
            '''ALTER TABLE `partner` ADD COLUMN `countrysubdivision` VARCHAR(9) '''
        )
        cursor.execute(
            '''ALTER TABLE `partner` ADD COLUMN `countrycode` VARCHAR(3) ''')
        cursor.execute(
            '''ALTER TABLE `partner` ADD COLUMN `phone1` VARCHAR(17) ''')
        cursor.execute(
            '''ALTER TABLE `partner` ADD COLUMN `phone2` VARCHAR(17) ''')
        cursor.execute('''ALTER TABLE `partner` ADD COLUMN `desc` TEXT ''')
        cursor.execute(
            '''ALTER TABLE `partner` ADD COLUMN `startdate` DATE ''')
        cursor.execute('''ALTER TABLE `partner` ADD COLUMN `enddate` DATE ''')
        #persist ****************************************
        cursor.execute('''ALTER TABLE `persist` MODIFY `content` TEXT''')
        #report ****************************************
        cursor.execute('''CREATE INDEX `report_ts` ON `report` (`ts`)''')
        cursor.execute(
            '''ALTER TABLE `report` ADD COLUMN `filesize` INTEGER DEFAULT 0''')
        #routes ****************************************
        cursor.execute(
            '''ALTER TABLE `routes` ADD COLUMN `zip_incoming` INTEGER DEFAULT 0'''
        )
        cursor.execute(
            '''ALTER TABLE `routes` ADD COLUMN `zip_outgoing` INTEGER DEFAULT 0'''
        )
        cursor.execute(
            '''ALTER TABLE `routes` MODIFY `translateind` integer NOT NULL''')
        #ta ****************************************
        #~ cursor.execute('''ALTER TABLE `ta` DROP INDEX `script` ''')   #not possible as index name is not known
        cursor.execute('''CREATE INDEX `ta_reference` ON `ta` (`reference`)''')
        cursor.execute(
            '''ALTER TABLE `ta` ADD COLUMN `filesize` INTEGER DEFAULT 0''')
        cursor.execute(
            '''ALTER TABLE `ta` ADD COLUMN `numberofresends` INTEGER DEFAULT 0'''
        )
        cursor.execute(
            '''ALTER TABLE `ta` ADD COLUMN `rsrv5` VARCHAR(35) DEFAULT '' ''')
        cursor.execute('''ALTER TABLE `ta` MODIFY `errortext` TEXT ''')
    except:
        txt = botslib.txtexc()
        botsglobal.db.rollback()
        cursor.close()
        print 'Error in changing mysql database: "%s".' % (txt)
        return 1
    else:
        botsglobal.db.commit()
        cursor.close()

    print 'Succesful changed mysql database to bots version 3.'
    return 0
Пример #50
0
def postgresql_psycopg2():
    print 'Start changing postgresql database to bots version 3.'
    cursor = botsglobal.db.cursor()
    try:
        #channel ****************************************
        cursor.execute(
            '''ALTER TABLE "channel" ALTER COLUMN "filename" TYPE VARCHAR(256)'''
        )
        cursor.execute(
            '''ALTER TABLE "channel" ADD COLUMN "rsrv3" INTEGER DEFAULT 0''')
        cursor.execute(
            '''ALTER TABLE "channel" ADD COLUMN "keyfile" VARCHAR(256) ''')
        cursor.execute(
            '''ALTER TABLE "channel" ADD COLUMN "certfile" VARCHAR(256) ''')
        cursor.execute(
            '''ALTER TABLE "channel" ADD COLUMN "testpath" VARCHAR(256) ''')
        #ccode ****************************************
        cursor.execute(
            '''ALTER TABLE "ccode" ALTER COLUMN "rightcode" TYPE VARCHAR(70)'''
        )
        cursor.execute(
            '''ALTER TABLE "ccode" ALTER COLUMN "attr1" TYPE VARCHAR(70)''')
        #filereport ****************************************
        cursor.execute(
            '''ALTER TABLE "filereport" DROP CONSTRAINT "filereport_pkey" '''
        )  #remove primary key
        cursor.execute(
            '''ALTER TABLE "filereport" DROP CONSTRAINT "filereport_idta_key" '''
        )  #drop contraint UNIQUE(idta, reportidta)
        cursor.execute('''DROP INDEX "filereport_idta" '''
                       )  #drop index on idta (will be primary key)
        cursor.execute('''DROP INDEX "filereport_reportidta" ''')
        cursor.execute('''ALTER TABLE "filereport" DROP COLUMN "id" ''')
        cursor.execute(
            '''ALTER TABLE "filereport" ADD CONSTRAINT "filereport_pkey" PRIMARY KEY("idta")'''
        )  #idta is primary key
        cursor.execute(
            '''ALTER TABLE "filereport" ALTER COLUMN "errortext" TYPE TEXT''')
        cursor.execute(
            '''ALTER TABLE "filereport" ADD COLUMN "filesize" INTEGER DEFAULT 0'''
        )
        cursor.execute(
            '''ALTER TABLE "filereport" ADD COLUMN "acceptance" INTEGER DEFAULT 0'''
        )
        #partner *************************************
        cursor.execute(
            '''ALTER TABLE "partner" ADD COLUMN "attr1" VARCHAR(35) ''')
        cursor.execute(
            '''ALTER TABLE "partner" ADD COLUMN "attr2" VARCHAR(35) ''')
        cursor.execute(
            '''ALTER TABLE "partner" ADD COLUMN "attr3" VARCHAR(35) ''')
        cursor.execute(
            '''ALTER TABLE "partner" ADD COLUMN "attr4" VARCHAR(35) ''')
        cursor.execute(
            '''ALTER TABLE "partner" ADD COLUMN "attr5" VARCHAR(70) ''')
        cursor.execute(
            '''ALTER TABLE "partner" ADD COLUMN "name1" VARCHAR(70) ''')
        cursor.execute(
            '''ALTER TABLE "partner" ADD COLUMN "name2" VARCHAR(70) ''')
        cursor.execute(
            '''ALTER TABLE "partner" ADD COLUMN "name3" VARCHAR(70) ''')
        cursor.execute(
            '''ALTER TABLE "partner" ADD COLUMN "address1" VARCHAR(70) ''')
        cursor.execute(
            '''ALTER TABLE "partner" ADD COLUMN "address2" VARCHAR(70) ''')
        cursor.execute(
            '''ALTER TABLE "partner" ADD COLUMN "address3" VARCHAR(70) ''')
        cursor.execute(
            '''ALTER TABLE "partner" ADD COLUMN "city" VARCHAR(35) ''')
        cursor.execute(
            '''ALTER TABLE "partner" ADD COLUMN "postalcode" VARCHAR(17) ''')
        cursor.execute(
            '''ALTER TABLE "partner" ADD COLUMN "countrysubdivision" VARCHAR(9) '''
        )
        cursor.execute(
            '''ALTER TABLE "partner" ADD COLUMN "countrycode" VARCHAR(3) ''')
        cursor.execute(
            '''ALTER TABLE "partner" ADD COLUMN "phone1" VARCHAR(17) ''')
        cursor.execute(
            '''ALTER TABLE "partner" ADD COLUMN "phone2" VARCHAR(17) ''')
        cursor.execute('''ALTER TABLE "partner" ADD COLUMN "desc" TEXT ''')
        cursor.execute(
            '''ALTER TABLE "partner" ADD COLUMN "startdate" DATE ''')
        cursor.execute('''ALTER TABLE "partner" ADD COLUMN "enddate" DATE ''')
        #persist ****************************************
        cursor.execute(
            '''ALTER TABLE "persist" ALTER COLUMN "content" TYPE TEXT''')
        #report ****************************************
        cursor.execute('''CREATE INDEX "report_ts" ON "report" ("ts")''')
        cursor.execute(
            '''ALTER TABLE "report" ADD COLUMN "filesize" INTEGER DEFAULT 0''')
        #routes ****************************************
        cursor.execute(
            '''ALTER TABLE "routes" ALTER COLUMN "translateind" TYPE integer USING CASE WHEN "translateind"=FALSE THEN 0 ELSE 1 END'''
        )
        cursor.execute(
            '''ALTER TABLE "routes" ADD COLUMN "zip_incoming" INTEGER DEFAULT 0'''
        )
        cursor.execute(
            '''ALTER TABLE "routes" ADD COLUMN "zip_outgoing" INTEGER DEFAULT 0'''
        )
        #ta ****************************************
        cursor.execute('''DROP INDEX "ta_script" ''')
        cursor.execute('''CREATE INDEX "ta_reference" ON "ta" ("reference")''')
        cursor.execute(
            '''ALTER TABLE "ta" ALTER COLUMN "errortext" TYPE TEXT ''')
        cursor.execute(
            '''ALTER TABLE "ta" ADD COLUMN "filesize" INTEGER DEFAULT 0''')
        cursor.execute(
            '''ALTER TABLE "ta" ADD COLUMN "numberofresends" INTEGER DEFAULT 0'''
        )
        cursor.execute(
            '''ALTER TABLE "ta" ADD COLUMN "rsrv5" VARCHAR(35) DEFAULT '' ''')
    except:
        txt = botslib.txtexc()
        botsglobal.db.rollback()
        cursor.close()
        print 'Error in changing postgresql database: "%s".' % (txt)
        return 1
    else:
        botsglobal.db.commit()
        cursor.close()

    print 'Succesful changed postgresql database to bots version 3.'
    return 0
Пример #51
0
def filer(request, *kw, **kwargs):
    ''' handles bots file viewer. Only files in data dir of Bots are displayed.'''
    nosuchfile = _(u'No such file.')
    if request.method == 'GET':
        try:
            idta = request.GET['idta']
        except:
            return django.shortcuts.render_to_response(
                'bots/filer.html', {'error_content': nosuchfile},
                context_instance=django.template.RequestContext(request))
        if idta == 0:  #for the 'starred' file names (eg multiple output)
            return django.shortcuts.render_to_response(
                'bots/filer.html', {'error_content': nosuchfile},
                context_instance=django.template.RequestContext(request))
        try:
            currentta = list(models.ta.objects.filter(idta=idta))[0]
            if request.GET['action'] == 'downl':
                response = django.http.HttpResponse(
                    mimetype=currentta.contenttype)
                response[
                    'Content-Disposition'] = 'attachment; filename=' + currentta.filename + '.txt'
                #~ absfilename = botslib.abspathdata(currentta.filename)
                #~ response['Content-Length'] = os.path.getsize(absfilename)
                response.write(
                    botslib.readdata(currentta.filename,
                                     charset=currentta.charset,
                                     errors='ignore'))
                return response
            elif request.GET['action'] == 'previous':
                if currentta.parent:  #has a explicit parent
                    talijst = list(
                        models.ta.objects.filter(idta=currentta.parent))
                else:  #get list of ta's referring to this idta as child
                    talijst = list(
                        models.ta.objects.filter(child=currentta.idta))
            elif request.GET['action'] == 'this':
                if currentta.status == EXTERNIN:  #jump strait to next file, as EXTERNIN can not be displayed.
                    talijst = list(
                        models.ta.objects.filter(parent=currentta.idta))
                elif currentta.status == EXTERNOUT:
                    talijst = list(
                        models.ta.objects.filter(idta=currentta.parent))
                else:
                    talijst = [currentta]
            elif request.GET['action'] == 'next':
                if currentta.child:  #has a explicit child
                    talijst = list(
                        models.ta.objects.filter(idta=currentta.child))
                else:
                    talijst = list(
                        models.ta.objects.filter(parent=currentta.idta))
            for ta in talijst:
                ta.has_next = True
                if ta.status == EXTERNIN:
                    ta.content = _(
                        u'External file. Can not be displayed. Use "next".')
                elif ta.status == EXTERNOUT:
                    ta.content = _(
                        u'External file. Can not be displayed. Use "previous".'
                    )
                    ta.has_next = False
                elif ta.statust in [OPEN, ERROR]:
                    ta.content = _(
                        u'File has error status and does not exist. Use "previous".'
                    )
                    ta.has_next = False
                elif not ta.filename:
                    ta.content = _(u'File can not be displayed.')
                else:
                    if ta.charset:  #guess charset; uft-8 is reasonable
                        ta.content = botslib.readdata(ta.filename,
                                                      charset=ta.charset,
                                                      errors='ignore')
                    else:
                        ta.content = botslib.readdata(ta.filename,
                                                      charset='utf-8',
                                                      errors='ignore')
            return django.shortcuts.render_to_response(
                'bots/filer.html', {'idtas': talijst},
                context_instance=django.template.RequestContext(request))
        except:
            print botslib.txtexc()
            return django.shortcuts.render_to_response(
                'bots/filer.html', {'error_content': nosuchfile},
                context_instance=django.template.RequestContext(request))
Пример #52
0
def read_plugin(pathzipfile):
    ''' process uploaded plugin. '''
    #test if valid zipfile
    if not zipfile.is_zipfile(pathzipfile):
        raise botslib.PluginError(_(u'Plugin is not a valid file.'))

    #read index file
    try:
        myzipimport = zipimport.zipimporter(pathzipfile)
        importedbotsindex = myzipimport.load_module('botsindex')
        pluglist = importedbotsindex.plugins[:]
        if 'botsindex' in sys.modules:
            del sys.modules['botsindex']
    except:
        txt = botslib.txtexc()
        raise botslib.PluginError(_(u'Error in plugin. Nothing is written. Error:\n%(txt)s'),{'txt':txt})
    else:
        botsglobal.logger.info(_(u'Plugin is OK.'))
        botsglobal.logger.info(_(u'Start writing to database.'))

    #write content of index file to the bots database
    try:
        read_index2database(pluglist)
    except:
        txt = botslib.txtexc()
        raise botslib.PluginError(_(u'Error writing plugin to database. Nothing is written. Error:\n%(txt)s'),{'txt':txt})
    else:
        botsglobal.logger.info(_(u'Writing to database is OK.'))

    #write files to the file system.
    botsglobal.logger.info(_(u'Start writing to files'))
    try:
        warnrenamed = False     #to report in GUI files have been overwritten.
        myzip = zipfile.ZipFile(pathzipfile, mode="r")
        orgtargetpath = botsglobal.ini.get('directories','botspath')
        if (orgtargetpath[-1:] in (os.path.sep, os.path.altsep) and len(os.path.splitdrive(orgtargetpath)[1]) > 1):
            orgtargetpath = orgtargetpath[:-1]
        for zipfileobject in myzip.infolist():
            if zipfileobject.filename not in ['botsindex.py','README','botssys/sqlitedb/botsdb','config/bots.ini'] and os.path.splitext(zipfileobject.filename)[1] not in ['.pyo','.pyc']:
                #~ botsglobal.logger.info(u'Filename in zip "%s".',zipfileobject.filename)
                if zipfileobject.filename[0] == '/':
                    targetpath = zipfileobject.filename[1:]
                else:
                    targetpath = zipfileobject.filename
                #convert for correct environment: repacle botssys, config, usersys in filenames
                if targetpath.startswith('usersys'):
                    targetpath = targetpath.replace('usersys',botsglobal.ini.get('directories','usersysabs'),1)
                elif targetpath.startswith('botssys'):
                    targetpath = targetpath.replace('botssys',botsglobal.ini.get('directories','botssys'),1)
                elif targetpath.startswith('config'):
                    targetpath = targetpath.replace('config',botsglobal.ini.get('directories','config'),1)
                targetpath = botslib.join(orgtargetpath, targetpath)
                #targetpath is OK now.
                botsglobal.logger.info(_(u'    Start writing file: "%(targetpath)s".'),{'targetpath':targetpath})

                if botslib.dirshouldbethere(os.path.dirname(targetpath)):
                    botsglobal.logger.info(_(u'        Create directory "%(directory)s".'),{'directory':os.path.dirname(targetpath)})
                if zipfileobject.filename[-1] == '/':    #check if this is a dir; if so continue
                    continue
                if os.path.isfile(targetpath):  #check if file already exists
                    try:    #this ***sometimes*** fails. (python25, for static/help/home.html...only there...)
                        warnrenamed = True
                    except:
                        pass
                source = myzip.read(zipfileobject.filename)
                target = open(targetpath, "wb")
                target.write(source)
                target.close()
                botsglobal.logger.info(_(u'        File written: "%(targetpath)s".'),{'targetpath':targetpath})
    except:
        txt = botslib.txtexc()
        myzip.close()
        raise botslib.PluginError(_(u'Error writing files to system. Nothing is written to database. Error:\n%(txt)s'),{'txt':txt})
    else:
        myzip.close()
        botsglobal.logger.info(_(u'Writing files to filesystem is OK.'))
        return warnrenamed
Пример #53
0
def sqlite3():
    if sqlite_database_is_version3():
        print 'Database sqlite3 is already bots version 3. No action is taken.'
        return 2

    print 'Start changing sqlite3 database to bots version 3.'
    cursor = botsglobal.db.cursor()
    try:
        #channel ****************************************
        cursor.execute(
            '''ALTER TABLE "channel" ADD COLUMN "rsrv3" INTEGER DEFAULT 0''')
        cursor.execute(
            '''ALTER TABLE "channel" ADD COLUMN "keyfile" VARCHAR(256) ''')
        cursor.execute(
            '''ALTER TABLE "channel" ADD COLUMN "certfile" VARCHAR(256) ''')
        cursor.execute(
            '''ALTER TABLE "channel" ADD COLUMN "testpath" VARCHAR(256) ''')
        #filereport ****************************************
        cursor.execute('''DROP INDEX "filereport_reportidta" ''')
        cursor.execute(
            '''ALTER TABLE "filereport" ADD COLUMN "filesize" INTEGER DEFAULT 0'''
        )
        #partner *************************************
        cursor.execute(
            '''ALTER TABLE "partner" ADD COLUMN "attr1" VARCHAR(35) ''')
        cursor.execute(
            '''ALTER TABLE "partner" ADD COLUMN "attr2" VARCHAR(35) ''')
        cursor.execute(
            '''ALTER TABLE "partner" ADD COLUMN "attr3" VARCHAR(35) ''')
        cursor.execute(
            '''ALTER TABLE "partner" ADD COLUMN "attr4" VARCHAR(35) ''')
        cursor.execute(
            '''ALTER TABLE "partner" ADD COLUMN "attr5" VARCHAR(70) ''')
        cursor.execute(
            '''ALTER TABLE "partner" ADD COLUMN "name1" VARCHAR(70) ''')
        cursor.execute(
            '''ALTER TABLE "partner" ADD COLUMN "name2" VARCHAR(70) ''')
        cursor.execute(
            '''ALTER TABLE "partner" ADD COLUMN "name3" VARCHAR(70) ''')
        cursor.execute(
            '''ALTER TABLE "partner" ADD COLUMN "address1" VARCHAR(70) ''')
        cursor.execute(
            '''ALTER TABLE "partner" ADD COLUMN "address2" VARCHAR(70) ''')
        cursor.execute(
            '''ALTER TABLE "partner" ADD COLUMN "address3" VARCHAR(70) ''')
        cursor.execute(
            '''ALTER TABLE "partner" ADD COLUMN "city" VARCHAR(35) ''')
        cursor.execute(
            '''ALTER TABLE "partner" ADD COLUMN "postalcode" VARCHAR(17) ''')
        cursor.execute(
            '''ALTER TABLE "partner" ADD COLUMN "countrysubdivision" VARCHAR(9) '''
        )
        cursor.execute(
            '''ALTER TABLE "partner" ADD COLUMN "countrycode" VARCHAR(3) ''')
        cursor.execute(
            '''ALTER TABLE "partner" ADD COLUMN "phone1" VARCHAR(17) ''')
        cursor.execute(
            '''ALTER TABLE "partner" ADD COLUMN "phone2" VARCHAR(17) ''')
        cursor.execute('''ALTER TABLE "partner" ADD COLUMN "desc" TEXT ''')
        cursor.execute(
            '''ALTER TABLE "partner" ADD COLUMN "startdate" DATE ''')
        cursor.execute('''ALTER TABLE "partner" ADD COLUMN "enddate" DATE ''')
        #report ****************************************
        cursor.execute('''CREATE INDEX "report_ts" ON "report" ("ts")''')
        cursor.execute(
            '''ALTER TABLE "report" ADD COLUMN "filesize" INTEGER DEFAULT 0''')
        cursor.execute(
            '''ALTER TABLE "report" ADD COLUMN "acceptance" INTEGER DEFAULT 0'''
        )
        #routes ****************************************
        cursor.execute(
            '''ALTER TABLE "routes" ADD COLUMN "zip_incoming" INTEGER DEFAULT 0'''
        )
        cursor.execute(
            '''ALTER TABLE "routes" ADD COLUMN "zip_outgoing" INTEGER DEFAULT 0'''
        )
        #ta ****************************************
        cursor.execute('''DROP INDEX "ta_script" ''')
        cursor.execute('''CREATE INDEX "ta_reference" ON "ta" ("reference")''')
        cursor.execute(
            '''ALTER TABLE "ta" ADD COLUMN "filesize" INTEGER DEFAULT 0''')
        cursor.execute(
            '''ALTER TABLE "ta" ADD COLUMN "numberofresends" INTEGER DEFAULT 0'''
        )
        cursor.execute(
            '''ALTER TABLE "ta" ADD COLUMN "rsrv5" VARCHAR(35) DEFAULT '' ''')
    except:
        txt = botslib.txtexc()
        botsglobal.db.rollback()
        cursor.close()
        print 'Error in adding fields to sqlite3 database: "%s".' % (txt)
        return 1
    else:
        botsglobal.db.commit()
        cursor.close()

    cursor = botsglobal.db.cursor()
    try:
        cursor.executescript(QUERYSTRING)
    except:
        txt = botslib.txtexc()
        botsglobal.db.rollback()
        cursor.close()
        print 'Error in changing sqlite3 database-schema "routes": "%s".' % (
            txt)
        return 1
    else:
        botsglobal.db.commit()
        cursor.close()

    print 'Succesful changed sqlite3 database to bots version 3.'
    return 0
Пример #54
0
def load(pathzipfile):
    ''' process uploaded plugin. '''
    #test is valid zipfile
    if not zipfile.is_zipfile(pathzipfile):
        raise botslib.PluginError(_(u'Plugin is not a valid file.'))

    #read index file
    try:
        myzipimport = zipimport.zipimporter(pathzipfile)
        importedbotsindex = myzipimport.load_module('botsindex')
        pluglist = importedbotsindex.plugins[:]
        if 'botsindex' in sys.modules:
            del sys.modules['botsindex']
    except:
        txt = botslib.txtexc()
        raise botslib.PluginError(_(u'Error in plugin. Nothing is written. Error:\n%s')%(txt))
    else:
        botsglobal.logger.info(_(u'Plugin is OK.'))
        botsglobal.logger.info(_(u'Start writing to database.'))

    #write content of index file to the bots database
    try:
        writetodatabase(pluglist)
    except:
        txt = botslib.txtexc()
        raise botslib.PluginError(_(u'Error writing plugin to database. Nothing is written. Error:\n%s'%(txt)))
    else:
        botsglobal.logger.info(u'Writing to database is OK.')
        botsglobal.logger.info(u'Start writing to files')

    #write files to the file system.
    try:
        warnrenamed = False     #to report in GUI files have been overwritten.
        myzip = zipfile.ZipFile(pathzipfile, mode="r")
        orgtargetpath = botsglobal.ini.get('directories','botspath')
        if (orgtargetpath[-1:] in (os.path.sep, os.path.altsep) and len(os.path.splitdrive(orgtargetpath)[1]) > 1):
            orgtargetpath = orgtargetpath[:-1]
        for zipfileobject in myzip.infolist():
            if zipfileobject.filename not in ['botsindex.py','README','botssys/sqlitedb/botsdb','config/bots.ini'] and os.path.splitext(zipfileobject.filename)[1] not in ['.pyo','.pyc']:
                #~ botsglobal.logger.info(u'filename in zip "%s".',zipfileobject.filename)
                if zipfileobject.filename[0] == '/':
                    targetpath = zipfileobject.filename[1:]
                else:
                    targetpath = zipfileobject.filename
                targetpath = targetpath.replace('usersys',botsglobal.ini.get('directories','usersysabs'),1)
                targetpath = targetpath.replace('botssys',botsglobal.ini.get('directories','botssys'),1)
                targetpath = botslib.join(orgtargetpath, targetpath)
                #targetpath is OK now.
                botsglobal.logger.info(_(u'    Start writing file: "%s".'),targetpath)

                if botslib.dirshouldbethere(os.path.dirname(targetpath)):
                    botsglobal.logger.info(_(u'        Create directory "%s".'),os.path.dirname(targetpath))
                if zipfileobject.filename[-1] == '/':    #check if this is a dir; if so continue
                    continue
                if os.path.isfile(targetpath):  #check if file already exists
                    try:    #this ***sometimes*** fails. (python25, for static/help/home.html...only there...)
                        os.rename(targetpath,targetpath+'.'+time.strftime('%Y%m%d%H%M%S'))
                        warnrenamed = True
                        botsglobal.logger.info(_(u'        Renamed existing file "%(from)s" to "%(to)s".'),{'from':targetpath,'to':targetpath+time.strftime('%Y%m%d%H%M%S')})
                    except:
                        pass
                source = myzip.read(zipfileobject.filename)
                target = open(targetpath, "wb")
                target.write(source)
                target.close()
                botsglobal.logger.info(_(u'        File written: "%s".'),targetpath)
    except:
        txt = botslib.txtexc()
        myzip.close()
        raise botslib.PluginError(_(u'Error writing files to system. Nothing is written to database. Error:\n%s')%(txt))
    else:
        myzip.close()
        botsglobal.logger.info(_(u'Writing files to filesystem is OK.'))
        return warnrenamed
Пример #55
0
def extractexcel(ta_from, endstatus, **argv):
    ''' extract excel file.
        editype & messagetype are unchanged.
    '''

    #***functions used by extractexcel
    #-------------------------------------------------------------------------------
    def read_xls(infilename):
        # Read excel first sheet into a 2-d array
        book = xlrd.open_workbook(infilename)
        sheet = book.sheet_by_index(0)
        formatter = lambda (t, v): format_excelval(book, t, v, False)
        xlsdata = []
        for row in range(sheet.nrows):
            (types, values) = (sheet.row_types(row), sheet.row_values(row))
            xlsdata.append(map(formatter, zip(types, values)))
        return xlsdata

    #-------------------------------------------------------------------------------
    def dump_csv(xlsdata, tofilename):
        stream = botslib.opendata(tofilename, 'wb')
        csvout = csv.writer(stream,
                            quotechar=quotechar,
                            delimiter=field_sep,
                            doublequote=doublequote,
                            escapechar=escape)
        csvout.writerows(map(utf8ize, xlsdata))
        stream.close()

    #-------------------------------------------------------------------------------
    def format_excelval(book, type, value, wanttupledate):
        #  Clean up the incoming excel data for some data types
        returnrow = []
        if type == 2:
            if value == int(value):
                value = int(value)
        elif type == 3:
            datetuple = xlrd.xldate_as_tuple(value, book.datemode)
            value = datetuple if wanttupledate else tupledate_to_isodate(
                datetuple)
        elif type == 5:
            value = xlrd.error_text_from_code[value]
        return value

    #-------------------------------------------------------------------------------
    def tupledate_to_isodate(tupledate):
        # Turns a gregorian (year, month, day, hour, minute, nearest_second) into a
        # standard YYYY-MM-DDTHH:MM:SS ISO date.
        (y, m, d, hh, mm, ss) = tupledate
        nonzero = lambda n: n != 0
        date = "%04d-%02d-%02d" % (y, m, d) if filter(nonzero,
                                                      (y, m, d)) else ''
        time = "T%02d:%02d:%02d" % (hh, mm, ss) if filter(
            nonzero, (hh, mm, ss)) or not date else ''
        return date + time

    #-------------------------------------------------------------------------------
    def utf8ize(l):
        # Make string-like things into utf-8, leave other things alone
        return [
            unicode(s).encode(charset) if hasattr(s, 'encode') else s
            for s in l
        ]

    #***end functions used by extractexcel
    import xlrd
    import csv
    #get parameters for csv-format; defaults are as the csv defaults (in grammar.py)
    charset = argv.get('charset', "utf-8")
    quotechar = argv.get('quotechar', "'")
    field_sep = argv.get('field_sep', ':')
    escape = argv.get('escape', '')
    if escape:
        doublequote = False
    else:
        doublequote = True
    try:
        infilename = botslib.abspathdata(ta_from.filename)
        xlsdata = read_xls(infilename)
        ta_to = ta_from.copyta(status=endstatus)
        tofilename = str(ta_to.idta)
        dump_csv(xlsdata, tofilename)
        ta_to.update(
            statust=OK,
            filename=tofilename)  #update outmessage transaction with ta_info;
        botsglobal.logger.debug(_(u'        File written: "%s".'), tofilename)
    except:
        txt = botslib.txtexc()
        botsglobal.logger.error(
            _(u'Excel extraction failed, may not be an Excel file? Error:\n%s'
              ), txt)
        raise botslib.InMessageError(_(
            u'Excel extraction failed, may not be an Excel file? Error:\n$error'
        ),
                                     error=txt)
Пример #56
0
def extractpdf(ta_from, endstatus, **argv):
    ''' Try to extract text content of a PDF file to a csv.
        You know this is not a great idea, right? But we'll do the best we can anyway!
        Page and line numbers are added to each row.
        Columns and rows are based on the x and y coordinates of each text element within tolerance allowed.
        Multiple text elements may combine to make one field, some PDFs have every character separated!
        You may need to experiment with x_group and y_group values, but defaults seem ok for most files.
        Output csv is UTF-8 encoded - The csv module doesn't directly support reading and writing Unicode
        If the PDF is just an image, all bets are off. Maybe try OCR, good luck with that!
        Mike Griffin 14/12/2011
    '''
    from pdfminer.pdfinterp import PDFResourceManager, process_pdf
    from pdfminer.converter import TextConverter
    from pdfminer.layout import LAParams, LTContainer, LTText, LTTextBox
    import csv

    class CsvConverter(TextConverter):
        def __init__(self, *args, **kwargs):
            TextConverter.__init__(self, *args, **kwargs)

        def receive_layout(self, ltpage):

            # recursively get every text element and it's coordinates
            def render(item):
                if isinstance(item, LTContainer):
                    for child in item:
                        render(child)
                elif isinstance(item, LTText):
                    (_, _, x, y) = item.bbox

                    # group the y values (rows) within group tolerance
                    for v in yv:
                        if y > v - y_group and y < v + y_group:
                            y = v
                    yv.append(y)

                    line = lines[int(-y)]
                    line[x] = item.get_text().encode('utf-8')

            from collections import defaultdict
            lines = defaultdict(lambda: {})

            yv = []
            render(ltpage)

            lineid = 0
            for y in sorted(lines.keys()):
                line = lines[y]
                lineid += 1
                csvdata = [ltpage.pageid,
                           lineid]  # first 2 columns are page and line numbers

                # group the x values (fields) within group tolerance
                p = 0
                field_txt = ''
                for x in sorted(line.keys()):
                    gap = x - p
                    if p > 0 and gap > x_group:
                        csvdata.append(field_txt)
                        field_txt = ''
                    field_txt += line[x]
                    p = x
                csvdata.append(field_txt)
                csvout.writerow(csvdata)
            if lineid == 0:
                raise botslib.InMessageError(
                    _(u'PDF text extraction failed, it may contain just image(s)?'
                      ))

    #get some optional parameters
    x_group = argv.get('x_group',
                       10)  # group text closer than this as one field
    y_group = argv.get('y_group',
                       5)  # group lines closer than this as one line
    password = argv.get('password', '')
    quotechar = argv.get('quotechar', '"')
    field_sep = argv.get('field_sep', ',')
    escape = argv.get('escape', '\\')
    charset = argv.get('charset', 'utf-8')
    if not escape:
        doublequote = True
    else:
        doublequote = False

    try:
        pdf_stream = botslib.opendata(ta_from.filename, 'rb')
        ta_to = ta_from.copyta(status=endstatus)
        tofilename = str(ta_to.idta)
        csv_stream = botslib.opendata(tofilename, 'wb')
        csvout = csv.writer(csv_stream,
                            quotechar=quotechar,
                            delimiter=field_sep,
                            doublequote=doublequote,
                            escapechar=escape)

        # Process PDF
        rsrcmgr = PDFResourceManager(caching=True)
        device = CsvConverter(rsrcmgr, csv_stream, codec=charset)
        process_pdf(rsrcmgr,
                    device,
                    pdf_stream,
                    pagenos=set(),
                    password=password,
                    caching=True,
                    check_extractable=True)

        device.close()
        pdf_stream.close()
        csv_stream.close()
        ta_to.update(
            statust=OK,
            filename=tofilename)  #update outmessage transaction with ta_info;
        botsglobal.logger.debug(_(u'        File written: "%s".'), tofilename)
    except:
        txt = botslib.txtexc()
        botsglobal.logger.error(
            _(u'PDF extraction failed, may not be a PDF file? Error:\n%s'),
            txt)
        raise botslib.InMessageError(
            _(u'PDF extraction failed, may not be a PDF file? Error:\n$error'),
            error=txt)
Пример #57
0
def translate(startstatus=TRANSLATE, endstatus=TRANSLATED, idroute=''):
    ''' translates edifiles in one or more edimessages.
        reads and parses edifiles that have to be translated.
        tries to split files into messages (using 'nextmessage' of grammar); if no splitting: edifile is one message.
        searches the right translation in translate-table;
        runs the mapping-script for the translation;
        Function takes db-ta with status=TRANSLATE->PARSED->SPLITUP->TRANSLATED
    '''
    #select edifiles to translate; fill ta-object
    for row in botslib.query(
            u'''SELECT idta,frompartner,topartner,filename,messagetype,testindicator,editype,charset,alt,fromchannel
                                FROM  ta
                                WHERE   idta>%(rootidta)s
                                AND     status=%(status)s
                                AND     statust=%(statust)s
                                AND     idroute=%(idroute)s
                                ''', {
                'status': startstatus,
                'statust': OK,
                'idroute': idroute,
                'rootidta': botslib.get_minta4query()
            }):
        try:
            ta_fromfile = botslib.OldTransaction(row['idta'])  #TRANSLATE ta
            ta_parsedfile = ta_fromfile.copyta(
                status=PARSED)  #copy TRANSLATE to PARSED ta
            #read whole edi-file: read, parse and made into a inmessage-object. Message is represented a a tree.
            edifile = inmessage.edifromfile(frompartner=row['frompartner'],
                                            topartner=row['topartner'],
                                            filename=row['filename'],
                                            messagetype=row['messagetype'],
                                            testindicator=row['testindicator'],
                                            editype=row['editype'],
                                            charset=row['charset'],
                                            alt=row['alt'],
                                            fromchannel=row['fromchannel'],
                                            idroute=idroute)

            botsglobal.logger.debug(
                u'start read and parse input file "%s" editype "%s" messagetype "%s".',
                row['filename'], row['editype'], row['messagetype'])
            for inn in edifile.nextmessage(
            ):  #for each message in the edifile:
                #inn.ta_info: parameters from inmessage.edifromfile(), syntax-information and parse-information
                ta_frommes = ta_parsedfile.copyta(
                    status=SPLITUP)  #copy PARSED to SPLITUP ta
                inn.ta_info[
                    'idta_fromfile'] = ta_fromfile.idta  #for confirmations in user script; used to give idta of 'confirming message'
                ta_frommes.update(
                    **inn.ta_info
                )  #update ta-record SLIPTUP with info from message content and/or grammar
                while 1:  #whileloop continues as long as there are alt-translations
                    #************select parameters for translation(script):
                    for row2 in botslib.query(
                            u'''SELECT tscript,tomessagetype,toeditype
                                                FROM    translate
                                                WHERE   frommessagetype = %(frommessagetype)s
                                                AND     fromeditype = %(fromeditype)s
                                                AND     active=%(booll)s
                                                AND     alt=%(alt)s
                                                AND     (frompartner_id IS NULL OR frompartner_id=%(frompartner)s OR frompartner_id in (SELECT to_partner_id
                                                                                                                            FROM partnergroup
                                                                                                                            WHERE from_partner_id=%(frompartner)s ))
                                                AND     (topartner_id IS NULL OR topartner_id=%(topartner)s OR topartner_id in (SELECT to_partner_id
                                                                                                                            FROM partnergroup
                                                                                                                            WHERE from_partner_id=%(topartner)s ))
                                                ORDER BY alt DESC,
                                                         CASE WHEN frompartner_id IS NULL THEN 1 ELSE 0 END, frompartner_id ,
                                                         CASE WHEN topartner_id IS NULL THEN 1 ELSE 0 END, topartner_id ''',
                        {
                            'frommessagetype': inn.ta_info['messagetype'],
                            'fromeditype': inn.ta_info['editype'],
                            'alt': inn.ta_info['alt'],
                            'frompartner': inn.ta_info['frompartner'],
                            'topartner': inn.ta_info['topartner'],
                            'booll': True
                        }):
                        break  #escape if found; we need only the first - ORDER BY in the query
                    else:  #no translation record is found
                        raise botslib.TranslationNotFoundError(
                            _(u'Editype "$editype", messagetype "$messagetype", frompartner "$frompartner", topartner "$topartner", alt "$alt"'
                              ),
                            editype=inn.ta_info['editype'],
                            messagetype=inn.ta_info['messagetype'],
                            frompartner=inn.ta_info['frompartner'],
                            topartner=inn.ta_info['topartner'],
                            alt=inn.ta_info['alt'])
                    ta_tomes = ta_frommes.copyta(
                        status=endstatus)  #copy SPLITUP to TRANSLATED ta
                    tofilename = str(ta_tomes.idta)
                    tscript = row2['tscript']
                    tomessage = outmessage.outmessage_init(
                        messagetype=row2['tomessagetype'],
                        editype=row2['toeditype'],
                        filename=tofilename,
                        reference=unique('messagecounter'),
                        statust=OK,
                        divtext=tscript)  #make outmessage object
                    #copy ta_info
                    botsglobal.logger.debug(
                        u'script "%s" translates messagetype "%s" to messagetype "%s".',
                        tscript, inn.ta_info['messagetype'],
                        tomessage.ta_info['messagetype'])
                    translationscript, scriptfilename = botslib.botsimport(
                        'mappings', inn.ta_info['editype'] + '.' +
                        tscript)  #get the mapping-script
                    doalttranslation = botslib.runscript(translationscript,
                                                         scriptfilename,
                                                         'main',
                                                         inn=inn,
                                                         out=tomessage)
                    botsglobal.logger.debug(u'script "%s" finished.', tscript)
                    if 'topartner' not in tomessage.ta_info:  #tomessage does not contain values from ta......
                        tomessage.ta_info['topartner'] = inn.ta_info[
                            'topartner']
                    if tomessage.ta_info[
                            'statust'] == DONE:  #if indicated in user script the message should be discarded
                        botsglobal.logger.debug(
                            u'No output file because mapping script explicitly indicated this.'
                        )
                        tomessage.ta_info['filename'] = ''
                        tomessage.ta_info['status'] = DISCARD
                    else:
                        botsglobal.logger.debug(
                            u'Start writing output file editype "%s" messagetype "%s".',
                            tomessage.ta_info['editype'],
                            tomessage.ta_info['messagetype'])
                        tomessage.writeall(
                        )  #write tomessage (result of translation).
                    #problem is that not all values ta_tomes are know to to_message....
                    #~ print 'tomessage.ta_info',tomessage.ta_info
                    ta_tomes.update(
                        **tomessage.ta_info
                    )  #update outmessage transaction with ta_info;
                    #check the value received from the mappingscript to see if another traanslation needs to be done (chained translation)
                    if doalttranslation is None:
                        del tomessage
                        break  #break out of while loop; do no other translation
                    elif isinstance(doalttranslation, dict):
                        #some extended cases; a dict is returned that contains 'instructions'
                        if 'type' not in doalttranslation:
                            raise botslib.BotsError(
                                "Mapping script returned dict. This dict does not have a 'type', like in eg: {'type:'out_as_inn', 'alt':'alt-value'}."
                            )
                        if doalttranslation['type'] == u'out_as_inn':
                            if 'alt' not in doalttranslation:
                                raise botslib.BotsError(
                                    "Mapping script returned dict, type 'out_as_inn'. This dict does not have a 'alt'-value, like in eg: {'type:'out_as_inn', 'alt':'alt-value'}."
                                )
                            inn = tomessage
                            if isinstance(inn, outmessage.fixed):
                                inn.root.stripnode()
                            inn.ta_info['alt'] = doalttranslation[
                                'alt']  #get the alt-value for the next chainded translation
                            inn.ta_info.pop('statust')
                    else:
                        del tomessage
                        inn.ta_info[
                            'alt'] = doalttranslation  #get the alt-value for the next chainded translation
                #end of while-loop

                #
                #~ print inn.ta_info
                ta_frommes.update(
                    statust=DONE, **inn.ta_info
                )  #update db. inn.ta_info could be changed by script. Is this useful?
                del inn

        #exceptions file_in-level
        except:
            #~ edifile.handleconfirm(ta_fromfile,error=True)    #only useful if errors are reported in acknowledgement (eg x12 997). Not used now.
            txt = botslib.txtexc()
            ta_parsedfile.failure()
            ta_parsedfile.update(statust=ERROR, errortext=txt)
            botsglobal.logger.debug(
                u'error in translating input file "%s":\n%s', row['filename'],
                txt)
        else:
            edifile.handleconfirm(ta_fromfile, error=False)
            ta_fromfile.update(statust=DONE)
            ta_parsedfile.update(statust=DONE, **edifile.confirminfo)
            botsglobal.logger.debug(u'translated input file "%s".',
                                    row['filename'])
            del edifile