def test_init_invalid_result_manager(self):
Data = ListDataProxy([i for i in range(100)])
with self.assertRaises(AssertionError) as e:
QueuedJobExecutor(worker=test_worker,
data_partitioner=SimpleListPartitioner(
data_proxy=Data, processors=4),
result_manager=Data)
def test_init(self):
data_proxy = ListDataProxy(data=range(100))
partitioner = SimpleIndexPartitioner(processors=4, data_proxy=data_proxy)
self.assertEqual(partitioner.start_index, 0)
self.assertEqual(partitioner.end_index, 0)
self.assertEqual(partitioner.data_size, 100)
self.assertEqual(partitioner.batch_size, 25)
self.assertEqual(partitioner.processors, 4)
def test_update_indexes(self):
data_proxy = ListDataProxy(data=range(100))
partitioner = SimpleIndexPartitioner(processors=4, data_proxy=data_proxy)
for i in range(partitioner.processors):
pt = partitioner.update_indexes()
self.assertEqual(pt, i*partitioner.batch_size)
self.assertEqual(partitioner.start_index, (i+1)*partitioner.batch_size)
self.assertEqual(partitioner.end_index, (i+1)*partitioner.batch_size)
self.assertEqual(partitioner.partitions, i+1)
def test_get_partition(self):
data_proxy = ListDataProxy(data=range(100))
partitioner = SimpleIndexPartitioner(processors=4, data_proxy=data_proxy)
for i in range(partitioner.processors):
part_id, pt_start, pt_end = partitioner.get_partition()
self.assertEqual(pt_start, i * partitioner.batch_size)
self.assertEqual(pt_end, (i + 1) * partitioner.batch_size)
part_id, pt_start, pt_end = partitioner.get_partition()
self.assertEqual(pt_start, -1)
self.assertEqual(pt_end, -1)
def test_init(self):
Data = ListDataProxy([i for i in range(100)])
executor = PipedJobExecutor(worker=test_worker,
data_partitioner=SimpleListPartitioner(data_proxy=Data, processors=4),
result_manager=ListResultManager())
self.assertEqual(executor.processors, 4)
aa = executor.response_holder()
self.assertTrue(type(aa).__name__, tuple.__name__)
self.assertTrue(type(aa[0]).__name__, Pipe.__name__)
self.assertTrue(type(aa[1]).__name__, Pipe.__name__)
def test_execute(self):
Data = ListDataProxy([i for i in range(100)])
result = PipedJobExecutor(worker=test_worker,
data_partitioner=SimpleListPartitioner(data_proxy=Data, processors=4),
result_manager=ListResultManager()).execute()
result.show_results()
self.assertEqual(len(result.show_results()), 4)
self.assertEqual(len(result.show_results()[0].result), 25)
self.assertEqual(len(result.show_results()[1].result), 25)
self.assertEqual(len(result.show_results()[2].result), 25)
self.assertEqual(len(result.show_results()[3].result), 25)
def test_update_indexes_non_zero_start(self):
data_proxy = ListDataProxy(data=range(100))
partitioner = SimpleIndexPartitioner(processors=4, data_proxy=data_proxy, start_index=10)
for i in range(partitioner.processors-1):
pt = partitioner.update_indexes()
self.assertEqual(pt, (i*partitioner.batch_size)+10)
self.assertEqual(partitioner.start_index, (i+1)*partitioner.batch_size+10)
self.assertEqual(partitioner.end_index, (i+1)*partitioner.batch_size+10)
self.assertEqual(partitioner.partitions, i+1)
pt = partitioner.update_indexes()
self.assertEqual(pt, 85)
self.assertEqual(partitioner.start_index, 100)
self.assertEqual(partitioner.end_index, 100)
self.assertEqual(partitioner.partitions, 4)
def test_get_partition(self):
data_proxy = ListDataProxy(data=range(100))
partitioner = SimpleListPartitioner(processors=4,
data_proxy=data_proxy)
part_id, part = partitioner.get_partition()
self.assertEqual(part, range(0, 25))
part_id, part = partitioner.get_partition()
self.assertEqual(part, range(25, 50))
part_id, part = partitioner.get_partition()
self.assertEqual(part, range(50, 75))
part_id, part = partitioner.get_partition()
self.assertEqual(part, range(75, 100))
part_id, part = partitioner.get_partition()
self.assertEqual(part, [])
def test_init_invalid_partitioner(self):
Data = ListDataProxy([i for i in range(100)])
with self.assertRaises(AssertionError) as e:
PipedJobExecutor(worker=test_worker,
data_partitioner=Data,
result_manager=ListResultManager())