def test_output_results_option_when_false(self):
runner = ExtractionRunner()
runner.add_runnable(SelfExtractor, output_results=False)
runner.run('test', output_dir=self.results_dir)
result_file_path = os.path.join(self.results_dir, 'SelfExtractor.xml')
self.assertFalse(os.path.isfile(result_file_path))
def test_output_results_option_defaults_to_true(self):
runner = ExtractionRunner()
runner.add_runnable(SelfExtractor)
runner.run('test', output_dir = self.results_dir)
result_file_path = os.path.join(self.results_dir, 'SelfExtractor.xml')
self.assertTrue(os.path.isfile(result_file_path))
def test_output_results_option_defaults_to_true(self):
runner = ExtractionRunner()
runner.add_runnable(SelfExtractor)
runner.run('test', output_dir=self.results_dir)
result_file_path = os.path.join(self.results_dir, 'SelfExtractor.xml')
self.assertTrue(os.path.isfile(result_file_path))
def test_output_results_option_when_false(self):
runner = ExtractionRunner()
runner.add_runnable(SelfExtractor, output_results=False)
runner.run('test', output_dir = self.results_dir)
result_file_path = os.path.join(self.results_dir, 'SelfExtractor.xml')
self.assertFalse(os.path.isfile(result_file_path))
def test_file_prefix_option_works(self):
runner = ExtractionRunner()
runner.add_runnable(ImplTestFileExtractor)
runner.run('whatever', output_dir=self.results_dir, file_prefix = 'prefix.')
result_file_path = os.path.join(self.results_dir, 'prefix.test.txt')
self.assertTrue(os.path.isfile(result_file_path))
self.assertEqual(open(result_file_path, 'r').read(), 'test test')
def test_file_name_result_works(self):
runner = ExtractionRunner()
runner.add_runnable(SelfChangeNameExtractor)
runner.run('pizza', output_dir=self.results_dir)
result_file_path = os.path.join(self.results_dir, SelfChangeNameExtractor.result_file_name)
self.assertTrue(os.path.isfile(result_file_path))
self.assertEqual(ET.parse(result_file_path).getroot().text, 'pizza')
def test_files_get_written(self):
runner = ExtractionRunner()
runner.add_runnable(ImplTestFileExtractor)
runner.run('whatever', output_dir=self.results_dir)
result_file_path = os.path.join(self.results_dir, 'test.txt')
self.assertTrue(os.path.isfile(result_file_path))
self.assertEqual(open(result_file_path, 'r').read(), 'test test')
def test_file_name_result_works(self):
runner = ExtractionRunner()
runner.add_runnable(SelfChangeNameExtractor)
runner.run('pizza', output_dir=self.results_dir)
result_file_path = os.path.join(
self.results_dir, SelfChangeNameExtractor.result_file_name)
self.assertTrue(os.path.isfile(result_file_path))
self.assertEqual(ET.parse(result_file_path).getroot().text, 'pizza')
def test_dependency_results_work(self):
runner = ExtractionRunner()
runner.add_runnable(ImplTestFileExtractor)
runner.add_runnable(DepsOnTestFileExtractor)
# an error will be thrown if dependency doesn't work
# so no need to assert anything in this test
runner.run('whatever', output_dir=self.results_dir)
runner = ExtractionRunner()
runner.add_runnable(DepsOnTestFileExtractor)
self.assertRaises(LookupError, runner.run, 'whatever', output_dir=self.results_dir)
def test_filter_results_cascade(self):
runner = ExtractionRunner()
runner.add_runnable(FailFilter)
runner.add_runnable(FailingDepsExtractor)
runner.run('Test', output_dir = self.results_dir)
fde_path = os.path.join(self.results_dir, 'FailingDepsExtractor.xml')
self.assertFalse(os.path.isfile(fde_path))
runner.run('Test', output_dir = self.results_dir, write_dep_errors=True)
self.assertTrue(os.path.isfile(fde_path))
self.assertEqual(ET.parse(fde_path).getroot().tag, 'error')
runner = ExtractionRunner()
runner.add_runnable(PassFilter)
runner.add_runnable(PassingDepsExtractor)
runner.run('Test', output_dir = self.results_dir)
pde_path = os.path.join(self.results_dir, 'PassingDepsExtractor.xml')
self.assertTrue(os.path.isfile(pde_path))
self.assertEqual(ET.parse(pde_path).getroot().text, 'Test')
os.remove(pde_path)
runner.run('Test', output_dir = self.results_dir, write_dep_errors=True)
self.assertTrue(os.path.isfile(pde_path))
self.assertEqual(ET.parse(pde_path).getroot().text, 'Test')
def test_filter_results_cascade(self):
runner = ExtractionRunner()
runner.add_runnable(FailFilter)
runner.add_runnable(FailingDepsExtractor)
runner.run('Test', output_dir=self.results_dir)
fde_path = os.path.join(self.results_dir, 'FailingDepsExtractor.xml')
self.assertFalse(os.path.isfile(fde_path))
runner.run('Test', output_dir=self.results_dir, write_dep_errors=True)
self.assertTrue(os.path.isfile(fde_path))
self.assertEqual(ET.parse(fde_path).getroot().tag, 'error')
runner = ExtractionRunner()
runner.add_runnable(PassFilter)
runner.add_runnable(PassingDepsExtractor)
runner.run('Test', output_dir=self.results_dir)
pde_path = os.path.join(self.results_dir, 'PassingDepsExtractor.xml')
self.assertTrue(os.path.isfile(pde_path))
self.assertEqual(ET.parse(pde_path).getroot().text, 'Test')
os.remove(pde_path)
runner.run('Test', output_dir=self.results_dir, write_dep_errors=True)
self.assertTrue(os.path.isfile(pde_path))
self.assertEqual(ET.parse(pde_path).getroot().text, 'Test')
def test_disable_logs_works(self):
runner = ExtractionRunner()
results_log_path = os.path.join(self.results_dir, 'results')
runnables_log_path = os.path.join(self.results_dir, 'runnables')
runner.enable_logging(results_log_path, runnables_log_path)
runner.disable_logging()
runner.add_runnable(SelfLogExtractor)
runner.run('abc', output_dir=self.results_dir, run_name='RUN!')
log_list = glob.glob(results_log_path + "*.log")
self.assertFalse(log_list)
log_list = glob.glob(runnables_log_path + "*.log")
self.assertFalse(log_list)
def test_disable_logs_works(self):
runner = ExtractionRunner()
results_log_path = os.path.join(self.results_dir, 'results')
runnables_log_path = os.path.join(self.results_dir, 'runnables')
runner.enable_logging(results_log_path, runnables_log_path)
runner.disable_logging()
runner.add_runnable(SelfLogExtractor)
runner.run('abc', output_dir = self.results_dir, run_name = 'RUN!')
log_list = glob.glob(results_log_path + "*.log")
self.assertFalse(log_list)
log_list = glob.glob(runnables_log_path + "*.log")
self.assertFalse(log_list)
def test_dependency_results_work(self):
runner = ExtractionRunner()
runner.add_runnable(ImplTestFileExtractor)
runner.add_runnable(DepsOnTestFileExtractor)
# an error will be thrown if dependency doesn't work
# so no need to assert anything in this test
runner.run('whatever', output_dir=self.results_dir)
runner = ExtractionRunner()
runner.add_runnable(DepsOnTestFileExtractor)
self.assertRaises(LookupError,
runner.run,
'whatever',
output_dir=self.results_dir)
def test_extractor_errors_cascade_no_write_dep_errors(self):
runner = ExtractionRunner()
runner.add_runnable(ErrorExtractor)
runner.add_runnable(DepsOnErrorExtractor)
runner.add_runnable(DepsOnErrorExtractor2)
runner.run('Test', output_dir = self.results_dir)
ee_path = os.path.join(self.results_dir, 'ErrorExtractor.xml')
self.assertTrue(os.path.isfile(ee_path))
self.assertEqual(ET.parse(ee_path).getroot().tag, 'error')
doee_path = os.path.join(self.results_dir, 'DepsOnErrorExtractor.xml')
self.assertFalse(os.path.isfile(doee_path))
doee2_path = os.path.join(self.results_dir, 'DepsOnErrorExtractor2.xml')
def test_extractor_errors_cascade_no_write_dep_errors(self):
runner = ExtractionRunner()
runner.add_runnable(ErrorExtractor)
runner.add_runnable(DepsOnErrorExtractor)
runner.add_runnable(DepsOnErrorExtractor2)
runner.run('Test', output_dir=self.results_dir)
ee_path = os.path.join(self.results_dir, 'ErrorExtractor.xml')
self.assertTrue(os.path.isfile(ee_path))
self.assertEqual(ET.parse(ee_path).getroot().tag, 'error')
doee_path = os.path.join(self.results_dir, 'DepsOnErrorExtractor.xml')
self.assertFalse(os.path.isfile(doee_path))
doee2_path = os.path.join(self.results_dir,
'DepsOnErrorExtractor2.xml')
def test_logs_work(self):
runner = ExtractionRunner()
results_log_path = os.path.join(self.results_dir, 'results')
runnables_log_path = os.path.join(self.results_dir, 'runnables')
runner.enable_logging(results_log_path, runnables_log_path)
runner.add_runnable(SelfLogExtractor)
runner.run('abc', output_dir = self.results_dir, run_name = 'RUN!')
results_log = glob.glob(results_log_path + "*.log")[0]
log_data = open(results_log, 'r').read()
self.assertTrue('[SUCCESS]' in log_data)
self.assertTrue('RUN!' in log_data)
runnables_log = glob.glob(runnables_log_path + "*.log")[0]
log_data = open(runnables_log, 'r').read()
self.assertTrue('abc' in log_data)
self.assertTrue('SelfLogExtractor' in log_data)
self.assertTrue('RUN!' in log_data)
def test_logs_work(self):
runner = ExtractionRunner()
results_log_path = os.path.join(self.results_dir, 'results')
runnables_log_path = os.path.join(self.results_dir, 'runnables')
runner.enable_logging(results_log_path, runnables_log_path)
runner.add_runnable(SelfLogExtractor)
runner.run('abc', output_dir=self.results_dir, run_name='RUN!')
results_log = glob.glob(results_log_path + "*.log")[0]
log_data = open(results_log, 'r').read()
self.assertTrue('[SUCCESS]' in log_data)
self.assertTrue('RUN!' in log_data)
runnables_log = glob.glob(runnables_log_path + "*.log")[0]
log_data = open(runnables_log, 'r').read()
self.assertTrue('abc' in log_data)
self.assertTrue('SelfLogExtractor' in log_data)
self.assertTrue('RUN!' in log_data)
except subprocess.TimeoutExpired:
raise RunnableError('awk timed out')
lines = [line for line in stdout.split("\n") if line]
root = ET.Element('extraction')
for line in lines:
ele = ET.SubElement(root, 'line')
ele.text = line
return ExtractorResult(xml_result=root)
# Set up and run extraction
extraction_runner = ExtractionRunner()
extraction_runner.add_runnable(HasNumbersFilter)
extraction_runner.add_runnable(EmailExtractor)
extraction_runner.add_runnable(LinesStartWithNumberExtractor)
extraction_runner.run(u'''Random data that contains some emails [email protected]
Test lines with some @ signs now and then. Meet you@[email protected].
Line with another email embedded [email protected] in the line.
[email protected] [email protected]
123 Some lines even start with numbers
Some lines don't start with numbers
004 The final line in the test data''', 'extraction/test/sample_output', run_name = 'Sample Data')
def test_nothing_run(self):
runner = ExtractionRunner()
runner.run(u'data!', output_dir=self.results_dir)
# should be no files in output_dir
self.assertFalse(os.listdir(self.results_dir))
def test_no_extraction_result_works(self):
runner = ExtractionRunner()
runner.add_runnable(NothingExtractor)
runner.run('pizza', output_dir=self.results_dir)
self.assertFalse(os.listdir(self.results_dir))
def test_no_extraction_result_works(self):
runner = ExtractionRunner()
runner.add_runnable(NothingExtractor)
runner.run('pizza', output_dir=self.results_dir)
self.assertFalse(os.listdir(self.results_dir))
def test_nothing_run(self):
runner = ExtractionRunner()
runner.run(u'data!', output_dir=self.results_dir)
# should be no files in output_dir
self.assertFalse(os.listdir(self.results_dir))
(status, stdout, stderr) = utils.external_process(
['awk', '/^[0-9]/ {print;}', '-'], input_data=data, timeout=5)
except subprocess.TimeoutExpired:
raise RunnableError('awk timed out')
lines = [line for line in stdout.split("\n") if line]
root = ET.Element('extraction')
for line in lines:
ele = ET.SubElement(root, 'line')
ele.text = line
return ExtractorResult(xml_result=root)
# Set up and run extraction
extraction_runner = ExtractionRunner()
extraction_runner.add_runnable(HasNumbersFilter)
extraction_runner.add_runnable(EmailExtractor)
extraction_runner.add_runnable(LinesStartWithNumberExtractor)
extraction_runner.run(u'''Random data that contains some emails [email protected]
Test lines with some @ signs now and then. Meet you@[email protected].
Line with another email embedded [email protected] in the line.
[email protected] [email protected]
123 Some lines even start with numbers
Some lines don't start with numbers
004 The final line in the test data''',
'extraction/test/sample_output',
run_name='Sample Data')
