def test_linear_graph(self):
pc = PipelineConfiguration(ExecutorFactory=TestExecutorFactory)
exA = testExecutorA()
exB = testExecutorB(exA)
exC = testExecutorC(exB)
exD = testExecutorD(exC)
pc.load_graph(exD)
roots, sink = pc.generate_graph()
self.assertEqual(len(roots), 1)
res_exA = roots[0]
self.assertEqual(res_exA.get_name(), exA.get_name())
res_exB = res_exA.next
self.assertEqual(res_exB.get_name(), exB.get_name())
res_exC = res_exB.next
self.assertEqual(res_exC.get_name(), exC.get_name())
res_exD = res_exC.next
self.assertEqual(res_exD.get_name(), exD.get_name())
self.assertEqual(sink.get_name(), exD.get_name())
def test_multiple_valid_input_nodes(self):
pc = PipelineConfiguration(ExecutorFactory=TestExecutorFactory)
exA = testExecutorA()
exB = testExecutorB()
exC = testExecutorC([exA, exB])
exD = testExecutorD(exC)
pc.load_graph(exD)
roots, sink = pc.generate_graph()
self.assertEqual(len(roots), 2)
res_exA, res_exB = roots
self.assertEqual(res_exA.get_name(), exA.get_name())
self.assertEqual(res_exB.get_name(), exB.get_name())
res_exC_A = res_exA.next
res_exC_B = res_exB.next
self.assertEqual(res_exC_A.get_name(), res_exC_B.get_name())
self.assertEqual(exC.get_name(), res_exC_A.get_name())
res_exD_A = res_exC_A.next
res_exD_B = res_exC_B.next
self.assertEqual(res_exD_A.get_name(), res_exD_B.get_name())
self.assertEqual(exD.get_name(), res_exD_A.get_name())
def test_single_node_graph(self):
pc = PipelineConfiguration(ExecutorFactory=TestExecutorFactory)
ex = testExecutorA()
pc.load_graph(ex)
roots, sink = pc.generate_graph()
self.assertEqual(len(roots), 1)
self.assertEqual(roots[0].get_name(), ex.get_name())
self.assertEqual(sink.get_name(), ex.get_name())
def test_read_write_invalid_format(self):
pc = PipelineConfiguration(ExecutorFactory=TestExecutorFactory)
exA = testExecutorA()
pc.load_graph(exA)
self.assertRaises(ValueError, PipelineConfiguration.write, pc,
'out.mov')
self.assertRaises(ValueError, PipelineConfiguration.read, pc,
'out.mov')
def test_main_multi_process(self):
num_items = 10
pc = PipelineConfiguration()
ptr = TestExecutor(0)
for x in range(1, num_items):
ptr = TestExecutor(x, ptr)
pc.load_graph(ptr)
administrator(pc, max_iterations=3, storage='local')
for i in range(num_items * 3):
events.get(timeout=2)
self.assertRaises(EmptyException, events.get, True, 0.25)
def test_main_multiple_returns(self):
num_items = 10
pc = PipelineConfiguration()
graph = MultipleReturnExecutor(num_items)
TestExecutor(1, graph)
pc.load_graph(graph)
administrator(pc, n_workers=1, max_iterations=1, storage='local')
for i in range(num_items):
self.assertEqual(events.get(timeout=2), 1, "not enough items")
self.assertRaises(EmptyException, events.get, True, 0.25)
def test_main_single_process(self):
num_items = 10
pc = PipelineConfiguration()
ptr = TestExecutor(0)
for x in range(1, num_items):
ptr = TestExecutor(x, ptr)
pc.load_graph(ptr)
administrator(pc, n_workers=1, max_iterations=1, storage='local')
for i in range(num_items):
self.assertEqual(events.get(timeout=2), i, "items in wrong order")
self.assertRaises(EmptyException, events.get, True, 0.25)
def test_multiple_invalid_input_nodes(self):
pc = PipelineConfiguration(ExecutorFactory=TestExecutorFactory)
exA = testExecutorA()
exB = testExecutorB(exA)
exD = testExecutorD()
exC = testExecutorC([exB, exD])
self.assertRaises(RuntimeError, pc.load_graph, exC)
def test_youtube_crawl_and_download_and_modify(self):
search_terms = ["car accident"]
pc = PipelineConfiguration()
x = YoutubeCrawler(search_terms) \
.set_database(MockDataAccessor())
x = UniversalDownloader(x) \
.register_downloader(".*youtube.*", YoutubeDownloader()) \
.set_pathname(dest_dir)
x = TestExecutorGenericVideoModification(x)
pc.load_graph(x)
Administrator(pc, max_iterations=2, n_workers=2)
downloaded_videos = os.listdir(dest_dir)
self.assertEqual(len(downloaded_videos), 2)
for vid_name in downloaded_videos:
self.assertTrue('.mp4' in vid_name)
def test_generate_without_load(self):
pc = PipelineConfiguration()
self.assertRaises(RuntimeError, PipelineConfiguration.generate_graph,
pc)
def test_load_null_graph(self):
pc = PipelineConfiguration()
self.assertRaises(ValueError, PipelineConfiguration.load_graph, pc,
None)
def test_config_no_list(self):
pc = PipelineConfiguration()
self.assertRaises(InvalidPipelineException, pc.read,
filepath("no_list_config.yaml"))
def test_pipelineConfig_with_args(self):
pc = PipelineConfiguration()
pc.read(filepath("pipelineConfigWithArgs.yaml"))
source_executors, _ = pc.generate_graph()
self.assertEqual(len(source_executors), 1)
self.assertEqual(source_executors[0].msg, "TEST MESSAGE")