def test_init_invalid_result_manager(self): Data = ListDataProxy([i for i in range(100)]) with self.assertRaises(AssertionError) as e: QueuedJobExecutor(worker=test_worker, data_partitioner=SimpleListPartitioner( data_proxy=Data, processors=4), result_manager=Data)
def test_init(self): data_proxy = ListDataProxy(data=range(100)) partitioner = SimpleIndexPartitioner(processors=4, data_proxy=data_proxy) self.assertEqual(partitioner.start_index, 0) self.assertEqual(partitioner.end_index, 0) self.assertEqual(partitioner.data_size, 100) self.assertEqual(partitioner.batch_size, 25) self.assertEqual(partitioner.processors, 4)
def test_update_indexes(self): data_proxy = ListDataProxy(data=range(100)) partitioner = SimpleIndexPartitioner(processors=4, data_proxy=data_proxy) for i in range(partitioner.processors): pt = partitioner.update_indexes() self.assertEqual(pt, i*partitioner.batch_size) self.assertEqual(partitioner.start_index, (i+1)*partitioner.batch_size) self.assertEqual(partitioner.end_index, (i+1)*partitioner.batch_size) self.assertEqual(partitioner.partitions, i+1)
def test_get_partition(self): data_proxy = ListDataProxy(data=range(100)) partitioner = SimpleIndexPartitioner(processors=4, data_proxy=data_proxy) for i in range(partitioner.processors): part_id, pt_start, pt_end = partitioner.get_partition() self.assertEqual(pt_start, i * partitioner.batch_size) self.assertEqual(pt_end, (i + 1) * partitioner.batch_size) part_id, pt_start, pt_end = partitioner.get_partition() self.assertEqual(pt_start, -1) self.assertEqual(pt_end, -1)
def test_init(self): Data = ListDataProxy([i for i in range(100)]) executor = PipedJobExecutor(worker=test_worker, data_partitioner=SimpleListPartitioner(data_proxy=Data, processors=4), result_manager=ListResultManager()) self.assertEqual(executor.processors, 4) aa = executor.response_holder() self.assertTrue(type(aa).__name__, tuple.__name__) self.assertTrue(type(aa[0]).__name__, Pipe.__name__) self.assertTrue(type(aa[1]).__name__, Pipe.__name__)
def test_execute(self): Data = ListDataProxy([i for i in range(100)]) result = PipedJobExecutor(worker=test_worker, data_partitioner=SimpleListPartitioner(data_proxy=Data, processors=4), result_manager=ListResultManager()).execute() result.show_results() self.assertEqual(len(result.show_results()), 4) self.assertEqual(len(result.show_results()[0].result), 25) self.assertEqual(len(result.show_results()[1].result), 25) self.assertEqual(len(result.show_results()[2].result), 25) self.assertEqual(len(result.show_results()[3].result), 25)
def test_update_indexes_non_zero_start(self): data_proxy = ListDataProxy(data=range(100)) partitioner = SimpleIndexPartitioner(processors=4, data_proxy=data_proxy, start_index=10) for i in range(partitioner.processors-1): pt = partitioner.update_indexes() self.assertEqual(pt, (i*partitioner.batch_size)+10) self.assertEqual(partitioner.start_index, (i+1)*partitioner.batch_size+10) self.assertEqual(partitioner.end_index, (i+1)*partitioner.batch_size+10) self.assertEqual(partitioner.partitions, i+1) pt = partitioner.update_indexes() self.assertEqual(pt, 85) self.assertEqual(partitioner.start_index, 100) self.assertEqual(partitioner.end_index, 100) self.assertEqual(partitioner.partitions, 4)
def test_get_partition(self): data_proxy = ListDataProxy(data=range(100)) partitioner = SimpleListPartitioner(processors=4, data_proxy=data_proxy) part_id, part = partitioner.get_partition() self.assertEqual(part, range(0, 25)) part_id, part = partitioner.get_partition() self.assertEqual(part, range(25, 50)) part_id, part = partitioner.get_partition() self.assertEqual(part, range(50, 75)) part_id, part = partitioner.get_partition() self.assertEqual(part, range(75, 100)) part_id, part = partitioner.get_partition() self.assertEqual(part, [])
def test_init_invalid_partitioner(self): Data = ListDataProxy([i for i in range(100)]) with self.assertRaises(AssertionError) as e: PipedJobExecutor(worker=test_worker, data_partitioner=Data, result_manager=ListResultManager())