def testGetParametersDict(self):
taskInfo = TaskInfo("param1=1;param2=2", "", "")
paramDict = taskInfo.getParametersDict()
self.assertTrue('param1' in paramDict)
self.assertTrue('param2' in paramDict)
taskInfo = TaskInfo("", "", "")
paramDict = taskInfo.getParametersDict()
self.assertEqual(0, len(paramDict))
taskInfo = TaskInfo("param1", "", "")
with self.assertRaises(Exception):
taskInfo.getParametersDict()
def testGatherInputData(self):
#No data olist
task1 = Task(
TaskInfo("", TestTask.workingDirectory, TestTask.targetFolderErr1))
with self.assertRaises(Exception):
task1.gatherInputData()
#No map .omap
task1 = Task(
TaskInfo("", TestTask.workingDirectory, TestTask.targetFolderErr2))
with self.assertRaises(Exception):
task1.gatherInputData()
task1 = Task(
TaskInfo("", TestTask.workingDirectory, TestTask.targetFolder1))
task1.gatherInputData()
self.assertTrue(MyFile.checkFileExists(task1.getTaskDirectory()))
self.assertTrue(MyFile.checkFileExists(task1.getInputDirectory()))
self.assertTrue(MyFile.checkFileExists(task1.getTempDirectory()))
self.assertTrue(MyFile.checkFileExists(task1.getOutputDirectory()))
dataListPath = "%s%s%s" % (task1.getInputDirectory(), os.sep,
'data.ilist')
dataMap1Path = "%s%s%s" % (task1.getInputDirectory(), os.sep,
'audio.imap')
dataMap2Path = "%s%s%s" % (task1.getInputDirectory(), os.sep,
'model.imap')
self.assertTrue(MyFile.checkFileExists(dataListPath))
self.assertTrue(MyFile.checkFileExists(dataMap1Path))
self.assertTrue(MyFile.checkFileExists(dataMap2Path))
self.assertEqual(15, task1.inputList.getCount())
self.assertEqual(2, len(task1.mapLists))
for dataMap in task1.mapLists:
self.assertTrue(dataMap.getCount() in [2, 1])
task1 = Task(
TaskInfo("", TestTask.workingDirectory, TestTask.targetFolderErr))
#Two input lists
with self.assertRaises(Exception):
task1.gatherInputData()
def testSetResults(self):
task1 = Task(
TaskInfo("", TestTask.workingDirectory, TestTask.targetFolder1))
task1.setResult("error", "message")
self.assertEqual("error", task1.resultErrorFlag)
self.assertEqual("message", task1.resultMessage)
def testExecute(self):
task1 = Task(
TaskInfo("", TestTask.workingDirectory, TestTask.targetFolder1))
task1.validateParameters = lambda: Task.validateParameters(task1, [])
task1.execute()
def testGetParametersDict(self):
task1 = Task(
TaskInfo("", TestTask.workingDirectory, TestTask.targetFolder1))
paramDict = task1.getTaskInfo().getParametersDict()
self.assertEqual(0, len(paramDict))
def testBuildParametersDict(self):
task1 = Task(
TaskInfo("param1=v1;param2=v2", TestTask.workingDirectory,
TestTask.targetFolder1))
task1._buildParametersDictionary()
task1 = Task(
TaskInfo("param1=;param2=v2", TestTask.workingDirectory,
TestTask.targetFolder1))
with self.assertRaises(Exception):
task1._buildParametersDictionary()
task1 = Task(
TaskInfo("=v1;param2=v2", TestTask.workingDirectory,
TestTask.targetFolder1))
with self.assertRaises(Exception):
task1._buildParametersDictionary()
def testValidateParameters(self):
task1 = Task(
TaskInfo("", TestTask.workingDirectory, TestTask.targetFolder1))
with self.assertRaises(Exception):
task1.validateParameters(["param1"])
Task.COMMON_PARAMETERS = ["param1"]
with self.assertRaises(Exception):
task1.validateParameters([])
Task.COMMON_PARAMETERS = []
def testGetParametersDict(self):
taskInfo = TaskInfo("param1=1;param2=2", "","")
paramDict = taskInfo.getParametersDict()
self.assertTrue('param1' in paramDict)
self.assertTrue('param2' in paramDict)
taskInfo = TaskInfo("", "","")
paramDict = taskInfo.getParametersDict()
self.assertEqual(0,len(paramDict))
taskInfo = TaskInfo("param1", "","")
with self.assertRaises(Exception):
taskInfo.getParametersDict()
#
if __name__ == "__main__":
#Setup parser
parser = argparse.ArgumentParser(description=usage)
parser.add_argument("-t", "--target", help="target directory containing the data.olist and data.omap",
nargs=1, dest="targetDir", required=True)
parser.add_argument("-o", "--output", help="output directory", nargs=1, dest="outputDir", required=True)
parser.add_argument("-r", "--regex", help="regex file", nargs=1, dest="regexFile", required=True)
parser.add_argument("-f", "--filter", help="filter sentences", dest="filter",action="store_true")
parser.add_argument("-d", "--debug", help="enable debug output", action="store_true")
parser.add_argument("-n", "--rmpunctuation", help="remove punctuation", action="store_true")
parser.add_argument("-p", "--vbpunctuation", help="verbalize punctuation", action="store_true")
parser.add_argument("-s", "--rawseg", help="do not segment sentences with NLTK", dest="rawseg",action="store_true")
parser.add_argument("-m", "--lm", help="prepare for lm modeling", dest="lm",action="store_true")
#Parse arguments
args = parser.parse_args()
targetDir = args.targetDir[0]
outputDir = args.outputDir[0]
regexFile = args.regexFile[0]
segmentWithNLTK = "True" if not args.rawseg else "False"
setupLogging(logging.INFO, outputDir + "/task_log.txt")
task = ImportDocumentTask(TaskInfo(STRPARAMETERS % (regexFile, str(args.debug),
args.rmpunctuation, args.vbpunctuation,
segmentWithNLTK, args.filter, args.lm),
outputDir, targetDir))
task.execute()
def testBuildTaskPath(self):
task1 = Task(
TaskInfo("", TestTask.workingDirectory, TestTask.targetFolder1))
with self.assertRaises(Exception):
task1._buildTaskPath("path")
def testBuildTask(self):
task1 = Task(
TaskInfo("", TestTask.workingDirectory, TestTask.targetFolder1))
self.assertTrue(task1.taskUniqueId > 0)
def testTaskInfo(self):
taskInfo = TaskInfo("param1=1;param2=2", "", "")
self.assertEqual("param1=1;param2=2", taskInfo.parametersString)
self.assertTrue("" != taskInfo.workingDirectory)
self.assertTrue("" != taskInfo.targetDirectory)
help="enable debug output",
action="store_true")
parser.add_argument("-n",
"--rmpunctuation",
help="remove punctuation",
action="store_true")
parser.add_argument("-p",
"--vbpunctuation",
help="verbalize punctuation",
action="store_true")
parser.add_argument("-m",
"--lm",
help="prepare for lm modeling",
dest="lm",
action="store_true")
#Parse arguments
args = parser.parse_args()
targetDir = args.targetDir[0]
outputDir = args.outputDir[0]
regexFile = args.regexFile[0]
setupLogging(logging.INFO, outputDir + "/task_log.txt")
task = ImportDocumentTask(
TaskInfo(
STRPARAMETERS % (regexFile, str(args.debug), args.rmpunctuation,
args.vbpunctuation, args.filter, args.lm),
outputDir, targetDir))
task.execute()
