Esempio n. 1
0
    def execute(commandList, logPath, outFileName = None, errFileName = None):
        """Wrapper to execute a sub process.
        """
        #Make sure the directory exists
        MyFile.checkDirExists(logPath)

        stdout, stderr, retCode = None, None, 0

        try:
            #Default to one log
            p = subprocess.Popen(commandList, stdout=subprocess.PIPE, 
                                 stderr=subprocess.STDOUT)

            if errFileName is not None:
                p = subprocess.Popen(commandList, stdout=subprocess.PIPE, 
                                 stderr=subprocess.PIPE)
                
            #Run the subprocess
            stdout, stderr = p.communicate()
            retCode = p.poll()
        except Exception, e:
            AsrtSubprocess.logger.critical("Subprocess error: %s" % str(e))
            errorMessage = str(commandList) + "\n" + \
                           "------------ Begin stack ------------\n" + \
                           traceback.format_exc().rstrip() + "\n" + \
                           "------------ End stack --------------"
            print errorMessage
            
            #Make sure the trace is logged
            if stderr is None: 
                stderr = errorMessage
            else:
                stderr += errorMessage
            
            retCode = 1
Esempio n. 2
0
def setupLogging(logLevel, fileName=None, logToStd=True):
    """Logging for the server module. Default
       level is INFO.
    """
    #Message queue
    taskLogger = logging.getLogger("Asrt")

    taskLogger.setLevel(logLevel)

    #Rendering engines
    mediaparlFormatter = MultiLineFormatter(
        "%(lineno)-4d : %(levelname)-10s %(name)-30s %(asctime)-25s %(message)s"
    )

    if fileName != None:
        #Check and make directory
        MyFile.checkDirExists(MyFile(fileName).getFileDir())

        fileHandler = logging.handlers.RotatingFileHandler(filename=fileName,
                                                           maxBytes=1024000,
                                                           backupCount=5)
        fileHandler.setLevel(logLevel)
        fileHandler.setFormatter(mediaparlFormatter)

        taskLogger.addHandler(fileHandler)

    if logToStd:
        streamHandler = logging.StreamHandler(sys.stdout)
        streamHandler.setLevel(logLevel)
        streamHandler.setFormatter(mediaparlFormatter)
        taskLogger.addHandler(streamHandler)

    return taskLogger
Esempio n. 3
0
def setupLogging(logLevel, fileName = None, logToStd = True):
    """Logging for the server module. Default
       level is INFO.
    """
    #Message queue
    taskLogger = logging.getLogger("Asrt")

    taskLogger.setLevel(logLevel)
       
    #Rendering engines
    mediaparlFormatter = MultiLineFormatter("%(lineno)-4d : %(levelname)-10s %(name)-30s %(asctime)-25s %(message)s")    
    
    if fileName != None:
        #Check and make directory
        MyFile.checkDirExists(MyFile(fileName).getFileDir())
            
        fileHandler = logging.handlers.RotatingFileHandler(filename=fileName,maxBytes=1024000, backupCount=5)
        fileHandler.setLevel(logLevel)
        fileHandler.setFormatter(mediaparlFormatter)

        taskLogger.addHandler(fileHandler)
        
    if logToStd:        
        streamHandler = logging.StreamHandler(sys.stdout)
        streamHandler.setLevel(logLevel)
        streamHandler.setFormatter(mediaparlFormatter)
        taskLogger.addHandler(streamHandler)
        
    return taskLogger
Esempio n. 4
0
    def execute(commandList, logPath, outFileName=None, errFileName=None):
        """Wrapper to execute a sub process.
        """
        #Make sure the directory exists
        MyFile.checkDirExists(logPath)

        stdout, stderr, retCode = None, None, 0

        try:
            #Default to one log
            p = subprocess.Popen(commandList,
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.STDOUT)

            if errFileName is not None:
                p = subprocess.Popen(commandList,
                                     stdout=subprocess.PIPE,
                                     stderr=subprocess.PIPE)

            #Run the subprocess
            stdout, stderr = p.communicate()
            retCode = p.poll()
        except Exception as e:
            AsrtSubprocess.logger.critical("Subprocess error: %s" % str(e))
            errorMessage = str(commandList) + "\n" + \
                           "------------ Begin stack ------------\n" + \
                           traceback.format_exc().rstrip() + "\n" + \
                           "------------ End stack --------------"
            print(errorMessage)

            #Make sure the trace is logged
            if stderr is None:
                stderr = errorMessage
            else:
                stderr += errorMessage

            retCode = 1

        #Now log results
        #It is important to be ouside exception management as we
        #still want to log what happened
        io = Ioread()

        if stdout != None and len(stdout) > 0 and outFileName != None:
            io.writeFileContent("%s/%s" % (logPath, outFileName),
                                str(stdout, 'utf-8'))

        if stderr != None and len(stderr) > 0 and errFileName != None:
            io.writeFileContent("%s/%s" % (logPath, errFileName),
                                str(stderr, 'utf-8'))

        return retCode, stdout, stderr
Esempio n. 5
0
    def execute(commandList, logPath, outFileName=None, errFileName=None):
        """Wrapper to execute a sub process.
        """
        #Make sure the directory exists
        MyFile.checkDirExists(logPath)

        stdout, stderr, retCode = None, None, 0

        try:
            #Default to one log
            p = subprocess.Popen(commandList,
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.STDOUT)

            if errFileName is not None:
                p = subprocess.Popen(commandList,
                                     stdout=subprocess.PIPE,
                                     stderr=subprocess.PIPE)

            #Run the subprocess
            stdout, stderr = p.communicate()
            retCode = p.poll()
        except Exception, e:
            AsrtSubprocess.logger.critical("Subprocess error: %s" % str(e))
            errorMessage = str(commandList) + "\n" + \
                           "------------ Begin stack ------------\n" + \
                           traceback.format_exc().rstrip() + "\n" + \
                           "------------ End stack --------------"
            print errorMessage

            #Make sure the trace is logged
            if stderr is None:
                stderr = errorMessage
            else:
                stderr += errorMessage

            retCode = 1
    setupLogging(logging.INFO, outputDir + "/data_preparation_log.txt")

    #Api setup
    api = DataPreparationAPI(None, outputDir)
    api.setRegexFile(regexFile)
    api.setFilterSentences(filterSentences)
    api.setFilterSentences2ndStage(filterSentences2ndStage)
    api.setLMModeling(lmModeling)
    api.setRemovePunctuation(removePunctuation)
    api.setVerbalizePunctuation(verbalizePunctuation)
    api.setSegmentWithNLTK(not rawSeg)
    api.setKeepNewWords(keepNewWords)

    if language == 0:
        api.trainClassifier()

    #Main processing
    MyFile.checkDirExists(outputDir)

    io = Ioread()
    inputList = io.readFileContentList(inputList)

    for i, f in enumerate(inputList):
        api.setInputFile(f)
        api.prepareDocument(language)
        strUnformatted = api.getCleanedText()

        outputFile = "%s/%s.lab" % (outputDir, os.path.splitext(os.path.basename(f))[0])
        io.writeFileContent(outputFile, strUnformatted + u"\n")
Esempio n. 7
0
    allTestSuite = []
    if commonTestSuite is not None:
        allTestSuite.extend(commonTestSuite)
    if formulaTestSuite is not None:
        allTestSuite.extend(formulaTestSuite)
    if frenchTestSuite is not None:
        allTestSuite.extend(frenchTestSuite)
    if germanTestSuite is not None:
        allTestSuite.extend(germanTestSuite)
    if englishTestSuite is not None:
        allTestSuite.extend(englishTestSuite)

    allTests = unittest.TestSuite(allTestSuite)

    return allTests


if __name__ == "__main__":
    if len(sys.argv) < 2:
        print((getUsage()))
        print(("    usage: %s 'unit test name 1 or all' 'unit test name 2' " %
               sys.argv[0]))
        print("")
        sys.exit(0)

    MyFile.checkDirExists(TEMPDIRUNITTEST)

    runner = unittest.TextTestRunner(verbosity=2)
    runner.run(asrtTestSuite(sys.argv[1:]))
Esempio n. 8
0
    commonTestSuite = CommonTestSuite.getCommonTestSuite(unitTestList)
    frenchTestSuite = FrenchTestSuite.getFrenchTestSuite(unitTestList)
    germanTestSuite = GermanTestSuite.getGermanTestSuite(unitTestList)
    formulaTestSuite = FormulaTestSuite.getFormulaTestSuite(unitTestList)

    allTestSuite = []
    if commonTestSuite is not None:
        allTestSuite.extend(commonTestSuite)
    if formulaTestSuite is not None:
        allTestSuite.extend(formulaTestSuite)
    if frenchTestSuite is not None:
        allTestSuite.extend(frenchTestSuite)
    if germanTestSuite is not None:
        allTestSuite.extend(germanTestSuite)

    allTests = unittest.TestSuite(allTestSuite)

    return allTests

if __name__ == "__main__":
    if len(sys.argv) < 2:
        print getUsage()
        print "    usage: %s 'unit test name 1 or all' 'unit test name 2' " % sys.argv[0]
        print ""
        sys.exit(0)

    MyFile.checkDirExists(TEMPDIRUNITTEST)

    runner = unittest.TextTestRunner(verbosity = 2)
    runner.run(asrtTestSuite(sys.argv[1:]))
    setupLogging(logging.INFO, outputDir + "/data_preparation_log.txt")

    # Api setup
    api = DataPreparationAPI(None, outputDir)
    api.setRegexFile(regexFile)
    api.setFilterSentences(filterSentences)
    api.setFilterSentences2ndStage(filterSentences2ndStage)
    api.setLMModeling(lmModeling)
    api.setRemovePunctuation(removePunctuation)
    api.setVerbalizePunctuation(verbalizePunctuation)
    api.setSegmentWithNLTK(not rawSeg)
    api.setExpandNumberInWords(expandNumberInWords)

    if language == 0:
        api.trainClassifier()

    # Main processing
    MyFile.checkDirExists(outputDir)

    io = Ioread()
    inputList = io.readFileContentList(inputList)

    for i, f in enumerate(inputList):
        api.setInputFile(f)
        api.prepareDocument(language)
        strUnformatted = api.getCleanedText()

        outputFile = "%s/%s.lab" % (outputDir,
                                    os.path.splitext(os.path.basename(f))[0])
        io.writeFileContent(outputFile, strUnformatted + "\n")