def test_computation(self):
p2p_node = P2PNode()
ctd = ComputeTaskDef()
ctd['task_id'] = "xyz"
ctd['subtask_id'] = "xxyyzz"
ctd['src_code'] = \
"cnt=0\n" \
"for i in range(10000):\n" \
"\tcnt += 1\n" \
"output={'data': cnt, 'result_type': 0}"
ctd['extra_data'] = {}
ctd['short_description'] = "add cnt"
ctd['deadline'] = timeout_to_deadline(10)
task_server = self.task_server
task_server.task_keeper.task_headers = {
ctd['subtask_id']:
mock.Mock(subtask_timeout=5, deadline=timeout_to_deadline(5)),
ctd['task_id']:
mock.Mock(subtask_timeout=5, deadline=timeout_to_deadline(20))
}
mock_finished = mock.Mock()
tc = TaskComputer(task_server,
use_docker_manager=False,
finished_cb=mock_finished)
self.assertEqual(tc.assigned_subtask, None)
tc.task_given(ctd)
self.assertEqual(tc.assigned_subtask, ctd)
self.assertLessEqual(tc.assigned_subtask['deadline'],
timeout_to_deadline(10))
tc.task_server.request_resource.assert_called_with("xyz", "xxyyzz")
assert tc.task_resource_collected("xyz")
tc.task_server.unpack_delta.assert_called_with(
tc.dir_manager.get_task_resource_dir("xyz"), None, "xyz")
assert tc.counting_thread is None
assert tc.assigned_subtask is None
task_server.send_task_failed.assert_called_with(
"xxyyzz", "xyz", "Host direct task not supported")
tc.support_direct_computation = True
tc.task_given(ctd)
assert tc.task_resource_collected("xyz")
assert not tc.waiting_for_task
assert tc.counting_thread is not None
self.assertGreater(tc.counting_thread.time_to_compute, 9)
self.assertLessEqual(tc.counting_thread.time_to_compute, 10)
mock_finished.assert_called_once_with()
mock_finished.reset_mock()
self.__wait_for_tasks(tc)
prev_task_failed_count = task_server.send_task_failed.call_count
self.assertIsNone(tc.counting_task)
self.assertIsNone(tc.counting_thread)
self.assertIsNone(tc.assigned_subtask)
assert task_server.send_task_failed.call_count == prev_task_failed_count
self.assertTrue(task_server.send_results.called)
args = task_server.send_results.call_args[0]
self.assertEqual(args[0], "xxyyzz")
self.assertEqual(args[1], "xyz")
self.assertEqual(args[2]["data"], 10000)
mock_finished.assert_called_once_with()
mock_finished.reset_mock()
ctd['subtask_id'] = "aabbcc"
ctd['src_code'] = "raise Exception('some exception')"
ctd['deadline'] = timeout_to_deadline(5)
tc.task_given(ctd)
self.assertEqual(tc.assigned_subtask, ctd)
self.assertLessEqual(tc.assigned_subtask['deadline'],
timeout_to_deadline(5))
tc.task_server.request_resource.assert_called_with("xyz", "aabbcc")
self.assertTrue(tc.task_resource_collected("xyz"))
self.__wait_for_tasks(tc)
self.assertIsNone(tc.counting_task)
self.assertIsNone(tc.counting_thread)
self.assertIsNone(tc.assigned_subtask)
task_server.send_task_failed.assert_called_with(
"aabbcc", "xyz", 'some exception')
mock_finished.assert_called_once_with()
mock_finished.reset_mock()
ctd['subtask_id'] = "aabbcc2"
ctd['src_code'] = "print('Hello world')"
ctd['deadline'] = timeout_to_deadline(5)
tc.task_given(ctd)
self.assertTrue(tc.task_resource_collected("xyz"))
self.__wait_for_tasks(tc)
task_server.send_task_failed.assert_called_with(
"aabbcc2", "xyz", "Wrong result format")
mock_finished.assert_called_once_with()
mock_finished.reset_mock()
task_server.task_keeper.task_headers["xyz"].deadline = \
timeout_to_deadline(20)
ctd['subtask_id'] = "aabbcc3"
ctd['src_code'] = "output={'data': 0, 'result_type': 0}"
ctd['deadline'] = timeout_to_deadline(40)
tc.task_given(ctd)
self.assertTrue(tc.task_resource_collected("xyz"))
self.assertIsNotNone(tc.counting_thread)
self.assertGreater(tc.counting_thread.time_to_compute, 10)
self.assertLessEqual(tc.counting_thread.time_to_compute, 20)
self.__wait_for_tasks(tc)
ctd['subtask_id'] = "xxyyzz2"
ctd['deadline'] = timeout_to_deadline(1)
tc.task_given(ctd)
self.assertTrue(tc.task_resource_collected("xyz"))
mock_finished.assert_called_once_with()
mock_finished.reset_mock()
tt = tc.counting_thread
tc.task_computed(tc.counting_thread)
self.assertIsNone(tc.counting_thread)
mock_finished.assert_called_once_with()
mock_finished.reset_mock()
task_server.send_task_failed.assert_called_with(
"xxyyzz2", "xyz", "Wrong result format")
tt.end_comp()
time.sleep(0.5)
if tt.is_alive():
tt.join(timeout=5)
def test_computation(self):
task_server = mock.MagicMock()
task_server.get_task_computer_root.return_value = self.path
task_server.config_desc = config_desc()
tc = TaskComputer("ABC", task_server, use_docker_machine_manager=False)
ctd = ComputeTaskDef()
ctd.task_id = "xyz"
ctd.subtask_id = "xxyyzz"
ctd.return_address = "10.10.10.10"
ctd.return_port = 10203
ctd.key_id = "key"
ctd.task_owner = "owner"
ctd.src_code = "cnt=0\nfor i in range(10000):\n\tcnt += 1\noutput={'data': cnt, 'result_type': 0}"
ctd.extra_data = {}
ctd.short_description = "add cnt"
ctd.deadline = timeout_to_deadline(10)
self.assertEqual(len(tc.assigned_subtasks), 0)
tc.task_given(ctd)
self.assertEqual(tc.assigned_subtasks["xxyyzz"], ctd)
self.assertLessEqual(tc.assigned_subtasks["xxyyzz"].deadline,
timeout_to_deadline(10))
self.assertEqual(tc.task_to_subtask_mapping["xyz"], "xxyyzz")
tc.task_server.request_resource.assert_called_with(
"xyz", tc.resource_manager.get_resource_header("xyz"),
"10.10.10.10", 10203, "key", "owner")
assert tc.task_resource_collected("xyz")
tc.task_server.unpack_delta.assert_called_with(
tc.dir_manager.get_task_resource_dir("xyz"), None, "xyz")
assert len(tc.current_computations) == 0
assert tc.assigned_subtasks.get("xxyyzz") is None
task_server.send_task_failed.assert_called_with(
"xxyyzz", "xyz", "Host direct task not supported", "10.10.10.10",
10203, "key", "owner", "ABC")
tc.support_direct_computation = True
tc.task_given(ctd)
assert tc.task_resource_collected("xyz")
assert not tc.waiting_for_task
assert len(tc.current_computations) == 1
self.__wait_for_tasks(tc)
prev_task_failed_count = task_server.send_task_failed.call_count
self.assertFalse(tc.counting_task)
self.assertEqual(len(tc.current_computations), 0)
self.assertIsNone(tc.assigned_subtasks.get("xxyyzz"))
assert task_server.send_task_failed.call_count == prev_task_failed_count
self.assertTrue(task_server.send_results.called)
args = task_server.send_results.call_args[0]
self.assertEqual(args[0], "xxyyzz")
self.assertEqual(args[1], "xyz")
self.assertEqual(args[2]["data"], 10000)
self.assertGreater(args[3], 0)
self.assertLess(args[3], 10)
self.assertEqual(args[4], "10.10.10.10")
self.assertEqual(args[5], 10203)
self.assertEqual(args[6], "key")
self.assertEqual(args[7], "owner")
self.assertEqual(args[8], "ABC")
ctd.subtask_id = "aabbcc"
ctd.src_code = "raise Exception('some exception')"
ctd.deadline = timeout_to_deadline(5)
tc.task_given(ctd)
self.assertEqual(tc.assigned_subtasks["aabbcc"], ctd)
self.assertLessEqual(tc.assigned_subtasks["aabbcc"].deadline,
timeout_to_deadline(5))
self.assertEqual(tc.task_to_subtask_mapping["xyz"], "aabbcc")
tc.task_server.request_resource.assert_called_with(
"xyz", tc.resource_manager.get_resource_header("xyz"),
"10.10.10.10", 10203, "key", "owner")
self.assertTrue(tc.task_resource_collected("xyz"))
self.__wait_for_tasks(tc)
self.assertFalse(tc.counting_task)
self.assertEqual(len(tc.current_computations), 0)
self.assertIsNone(tc.assigned_subtasks.get("aabbcc"))
task_server.send_task_failed.assert_called_with(
"aabbcc", "xyz", 'some exception', "10.10.10.10", 10203, "key",
"owner", "ABC")
ctd.subtask_id = "aabbcc2"
ctd.src_code = "print 'Hello world'"
ctd.timeout = timeout_to_deadline(5)
tc.task_given(ctd)
self.assertTrue(tc.task_resource_collected("xyz"))
self.__wait_for_tasks(tc)
task_server.send_task_failed.assert_called_with(
"aabbcc2", "xyz", "Wrong result format", "10.10.10.10", 10203,
"key", "owner", "ABC")
ctd.subtask_id = "xxyyzz2"
ctd.timeout = timeout_to_deadline(1)
tc.task_given(ctd)
self.assertTrue(tc.task_resource_collected("xyz"))
tt = tc.current_computations[0]
tc.task_computed(tc.current_computations[0])
self.assertEqual(len(tc.current_computations), 0)
task_server.send_task_failed.assert_called_with(
"xxyyzz2", "xyz", "Wrong result format", "10.10.10.10", 10203,
"key", "owner", "ABC")
tt.end_comp()
time.sleep(0.5)
if tt.is_alive():
tt.join(timeout=5)