def execute(commandList, logPath, outFileName = None, errFileName = None):
"""Wrapper to execute a sub process.
"""
#Make sure the directory exists
MyFile.checkDirExists(logPath)
stdout, stderr, retCode = None, None, 0
try:
#Default to one log
p = subprocess.Popen(commandList, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
if errFileName is not None:
p = subprocess.Popen(commandList, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
#Run the subprocess
stdout, stderr = p.communicate()
retCode = p.poll()
except Exception, e:
AsrtSubprocess.logger.critical("Subprocess error: %s" % str(e))
errorMessage = str(commandList) + "\n" + \
"------------ Begin stack ------------\n" + \
traceback.format_exc().rstrip() + "\n" + \
"------------ End stack --------------"
print errorMessage
#Make sure the trace is logged
if stderr is None:
stderr = errorMessage
else:
stderr += errorMessage
retCode = 1
def setupLogging(logLevel, fileName=None, logToStd=True):
"""Logging for the server module. Default
level is INFO.
"""
#Message queue
taskLogger = logging.getLogger("Asrt")
taskLogger.setLevel(logLevel)
#Rendering engines
mediaparlFormatter = MultiLineFormatter(
"%(lineno)-4d : %(levelname)-10s %(name)-30s %(asctime)-25s %(message)s"
)
if fileName != None:
#Check and make directory
MyFile.checkDirExists(MyFile(fileName).getFileDir())
fileHandler = logging.handlers.RotatingFileHandler(filename=fileName,
maxBytes=1024000,
backupCount=5)
fileHandler.setLevel(logLevel)
fileHandler.setFormatter(mediaparlFormatter)
taskLogger.addHandler(fileHandler)
if logToStd:
streamHandler = logging.StreamHandler(sys.stdout)
streamHandler.setLevel(logLevel)
streamHandler.setFormatter(mediaparlFormatter)
taskLogger.addHandler(streamHandler)
return taskLogger
def setupLogging(logLevel, fileName = None, logToStd = True):
"""Logging for the server module. Default
level is INFO.
"""
#Message queue
taskLogger = logging.getLogger("Asrt")
taskLogger.setLevel(logLevel)
#Rendering engines
mediaparlFormatter = MultiLineFormatter("%(lineno)-4d : %(levelname)-10s %(name)-30s %(asctime)-25s %(message)s")
if fileName != None:
#Check and make directory
MyFile.checkDirExists(MyFile(fileName).getFileDir())
fileHandler = logging.handlers.RotatingFileHandler(filename=fileName,maxBytes=1024000, backupCount=5)
fileHandler.setLevel(logLevel)
fileHandler.setFormatter(mediaparlFormatter)
taskLogger.addHandler(fileHandler)
if logToStd:
streamHandler = logging.StreamHandler(sys.stdout)
streamHandler.setLevel(logLevel)
streamHandler.setFormatter(mediaparlFormatter)
taskLogger.addHandler(streamHandler)
return taskLogger
def execute(commandList, logPath, outFileName=None, errFileName=None):
"""Wrapper to execute a sub process.
"""
#Make sure the directory exists
MyFile.checkDirExists(logPath)
stdout, stderr, retCode = None, None, 0
try:
#Default to one log
p = subprocess.Popen(commandList,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
if errFileName is not None:
p = subprocess.Popen(commandList,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
#Run the subprocess
stdout, stderr = p.communicate()
retCode = p.poll()
except Exception as e:
AsrtSubprocess.logger.critical("Subprocess error: %s" % str(e))
errorMessage = str(commandList) + "\n" + \
"------------ Begin stack ------------\n" + \
traceback.format_exc().rstrip() + "\n" + \
"------------ End stack --------------"
print(errorMessage)
#Make sure the trace is logged
if stderr is None:
stderr = errorMessage
else:
stderr += errorMessage
retCode = 1
#Now log results
#It is important to be ouside exception management as we
#still want to log what happened
io = Ioread()
if stdout != None and len(stdout) > 0 and outFileName != None:
io.writeFileContent("%s/%s" % (logPath, outFileName),
str(stdout, 'utf-8'))
if stderr != None and len(stderr) > 0 and errFileName != None:
io.writeFileContent("%s/%s" % (logPath, errFileName),
str(stderr, 'utf-8'))
return retCode, stdout, stderr
def execute(commandList, logPath, outFileName=None, errFileName=None):
"""Wrapper to execute a sub process.
"""
#Make sure the directory exists
MyFile.checkDirExists(logPath)
stdout, stderr, retCode = None, None, 0
try:
#Default to one log
p = subprocess.Popen(commandList,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
if errFileName is not None:
p = subprocess.Popen(commandList,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
#Run the subprocess
stdout, stderr = p.communicate()
retCode = p.poll()
except Exception, e:
AsrtSubprocess.logger.critical("Subprocess error: %s" % str(e))
errorMessage = str(commandList) + "\n" + \
"------------ Begin stack ------------\n" + \
traceback.format_exc().rstrip() + "\n" + \
"------------ End stack --------------"
print errorMessage
#Make sure the trace is logged
if stderr is None:
stderr = errorMessage
else:
stderr += errorMessage
retCode = 1
setupLogging(logging.INFO, outputDir + "/data_preparation_log.txt")
#Api setup
api = DataPreparationAPI(None, outputDir)
api.setRegexFile(regexFile)
api.setFilterSentences(filterSentences)
api.setFilterSentences2ndStage(filterSentences2ndStage)
api.setLMModeling(lmModeling)
api.setRemovePunctuation(removePunctuation)
api.setVerbalizePunctuation(verbalizePunctuation)
api.setSegmentWithNLTK(not rawSeg)
api.setKeepNewWords(keepNewWords)
if language == 0:
api.trainClassifier()
#Main processing
MyFile.checkDirExists(outputDir)
io = Ioread()
inputList = io.readFileContentList(inputList)
for i, f in enumerate(inputList):
api.setInputFile(f)
api.prepareDocument(language)
strUnformatted = api.getCleanedText()
outputFile = "%s/%s.lab" % (outputDir, os.path.splitext(os.path.basename(f))[0])
io.writeFileContent(outputFile, strUnformatted + u"\n")
allTestSuite = []
if commonTestSuite is not None:
allTestSuite.extend(commonTestSuite)
if formulaTestSuite is not None:
allTestSuite.extend(formulaTestSuite)
if frenchTestSuite is not None:
allTestSuite.extend(frenchTestSuite)
if germanTestSuite is not None:
allTestSuite.extend(germanTestSuite)
if englishTestSuite is not None:
allTestSuite.extend(englishTestSuite)
allTests = unittest.TestSuite(allTestSuite)
return allTests
if __name__ == "__main__":
if len(sys.argv) < 2:
print((getUsage()))
print((" usage: %s 'unit test name 1 or all' 'unit test name 2' " %
sys.argv[0]))
print("")
sys.exit(0)
MyFile.checkDirExists(TEMPDIRUNITTEST)
runner = unittest.TextTestRunner(verbosity=2)
runner.run(asrtTestSuite(sys.argv[1:]))
commonTestSuite = CommonTestSuite.getCommonTestSuite(unitTestList)
frenchTestSuite = FrenchTestSuite.getFrenchTestSuite(unitTestList)
germanTestSuite = GermanTestSuite.getGermanTestSuite(unitTestList)
formulaTestSuite = FormulaTestSuite.getFormulaTestSuite(unitTestList)
allTestSuite = []
if commonTestSuite is not None:
allTestSuite.extend(commonTestSuite)
if formulaTestSuite is not None:
allTestSuite.extend(formulaTestSuite)
if frenchTestSuite is not None:
allTestSuite.extend(frenchTestSuite)
if germanTestSuite is not None:
allTestSuite.extend(germanTestSuite)
allTests = unittest.TestSuite(allTestSuite)
return allTests
if __name__ == "__main__":
if len(sys.argv) < 2:
print getUsage()
print " usage: %s 'unit test name 1 or all' 'unit test name 2' " % sys.argv[0]
print ""
sys.exit(0)
MyFile.checkDirExists(TEMPDIRUNITTEST)
runner = unittest.TextTestRunner(verbosity = 2)
runner.run(asrtTestSuite(sys.argv[1:]))
setupLogging(logging.INFO, outputDir + "/data_preparation_log.txt")
# Api setup
api = DataPreparationAPI(None, outputDir)
api.setRegexFile(regexFile)
api.setFilterSentences(filterSentences)
api.setFilterSentences2ndStage(filterSentences2ndStage)
api.setLMModeling(lmModeling)
api.setRemovePunctuation(removePunctuation)
api.setVerbalizePunctuation(verbalizePunctuation)
api.setSegmentWithNLTK(not rawSeg)
api.setExpandNumberInWords(expandNumberInWords)
if language == 0:
api.trainClassifier()
# Main processing
MyFile.checkDirExists(outputDir)
io = Ioread()
inputList = io.readFileContentList(inputList)
for i, f in enumerate(inputList):
api.setInputFile(f)
api.prepareDocument(language)
strUnformatted = api.getCleanedText()
outputFile = "%s/%s.lab" % (outputDir,
os.path.splitext(os.path.basename(f))[0])
io.writeFileContent(outputFile, strUnformatted + "\n")
