def query_extra_data_for_test_task(self):
ctd = ComputeTaskDef()
ctd.subtask_id = "xxyyzz"
ctd.task_id = "xyz"
ctd.working_directory = self.header.root_path
ctd.src_code = self.src_code
ctd.extra_data = self.extra_data
ctd.short_description = ""
return ctd
def _new_compute_task_def(self, hash, extra_data, working_directory,
perf_index):
ctd = ComputeTaskDef()
ctd.task_id = self.header.task_id
ctd.subtask_id = hash
ctd.extra_data = extra_data
ctd.short_description = self._short_extra_data_repr(
perf_index, extra_data)
ctd.src_code = self.src_code
ctd.performance = perf_index
ctd.working_directory = working_directory
ctd.docker_images = self.header.docker_images
ctd.deadline = timeout_to_deadline(self.header.subtask_timeout)
return ctd
def query_extra_data(self, perf_index, num_cores=1, node_id=None, node_name=None):
"""Returns data for the next subtask.
:param int perf_index:
:param int num_cores:
:param str | None node_id:
:param str | None node_name:
:rtype: ComputeTaskDef"""
# create new subtask_id
subtask_id = uuid.uuid4().get_hex()
with self._lock:
# check if a task has been assigned to this node
if node_id in self.assigned_nodes:
return self.ExtraData(should_wait=True)
# assign a task
self.assigned_nodes[node_id] = subtask_id
self.assigned_subtasks[subtask_id] = node_id
# create subtask-specific data, 4 bits go for one char (hex digit)
data = random.getrandbits(self.task_params.subtask_data_size * 4)
self.subtask_ids.append(subtask_id)
self.subtask_data[subtask_id] = '%x' % data
subtask_def = ComputeTaskDef()
subtask_def.task_id = self.task_id
subtask_def.subtask_id = subtask_id
subtask_def.src_code = self.src_code
subtask_def.task_owner = self.header.task_owner
subtask_def.environment = self.header.environment
subtask_def.return_address = self.header.task_owner_address
subtask_def.return_port = self.header.task_owner_port
subtask_def.deadline = timeout_to_deadline(5 * 60)
subtask_def.extra_data = {
'data_file': self.shared_data_file,
'subtask_data': self.subtask_data[subtask_id],
'difficulty': self.task_params.difficulty,
'result_size': self.task_params.result_size,
'result_file': 'result.' + subtask_id[0:6]
}
return self.ExtraData(ctd=subtask_def)
def __get_merge_ctd(self, files):
script_file = dirmanager.find_task_script(APP_DIR,
"docker_luxmerge.py")
if script_file is None:
logger.error("Cannot find merger script")
return
with open(script_file) as f:
src_code = f.read()
ctd = ComputeTaskDef()
ctd.task_id = self.header.task_id
ctd.subtask_id = self.header.task_id
ctd.extra_data = {'output_flm': self.output_file, 'flm_files': files}
ctd.src_code = src_code
ctd.working_directory = "."
ctd.docker_images = self.header.docker_images
ctd.deadline = timeout_to_deadline(self.merge_timeout)
return ctd
def query_extra_data_for_advanced_verification(self, extra_data):
ctd = ComputeTaskDef()
ctd.extra_data = extra_data
return ctd
def test_computation(self):
task_server = mock.MagicMock()
task_server.get_task_computer_root.return_value = self.path
task_server.config_desc = config_desc()
tc = TaskComputer("ABC", task_server, use_docker_machine_manager=False)
ctd = ComputeTaskDef()
ctd.task_id = "xyz"
ctd.subtask_id = "xxyyzz"
ctd.return_address = "10.10.10.10"
ctd.return_port = 10203
ctd.key_id = "key"
ctd.task_owner = "owner"
ctd.src_code = "cnt=0\nfor i in range(10000):\n\tcnt += 1\noutput={'data': cnt, 'result_type': 0}"
ctd.extra_data = {}
ctd.short_description = "add cnt"
ctd.deadline = timeout_to_deadline(10)
self.assertEqual(len(tc.assigned_subtasks), 0)
tc.task_given(ctd)
self.assertEqual(tc.assigned_subtasks["xxyyzz"], ctd)
self.assertLessEqual(tc.assigned_subtasks["xxyyzz"].deadline,
timeout_to_deadline(10))
self.assertEqual(tc.task_to_subtask_mapping["xyz"], "xxyyzz")
tc.task_server.request_resource.assert_called_with(
"xyz", tc.resource_manager.get_resource_header("xyz"),
"10.10.10.10", 10203, "key", "owner")
assert tc.task_resource_collected("xyz")
tc.task_server.unpack_delta.assert_called_with(
tc.dir_manager.get_task_resource_dir("xyz"), None, "xyz")
assert len(tc.current_computations) == 0
assert tc.assigned_subtasks.get("xxyyzz") is None
task_server.send_task_failed.assert_called_with(
"xxyyzz", "xyz", "Host direct task not supported", "10.10.10.10",
10203, "key", "owner", "ABC")
tc.support_direct_computation = True
tc.task_given(ctd)
assert tc.task_resource_collected("xyz")
assert not tc.waiting_for_task
assert len(tc.current_computations) == 1
self.__wait_for_tasks(tc)
prev_task_failed_count = task_server.send_task_failed.call_count
self.assertFalse(tc.counting_task)
self.assertEqual(len(tc.current_computations), 0)
self.assertIsNone(tc.assigned_subtasks.get("xxyyzz"))
assert task_server.send_task_failed.call_count == prev_task_failed_count
self.assertTrue(task_server.send_results.called)
args = task_server.send_results.call_args[0]
self.assertEqual(args[0], "xxyyzz")
self.assertEqual(args[1], "xyz")
self.assertEqual(args[2]["data"], 10000)
self.assertGreater(args[3], 0)
self.assertLess(args[3], 10)
self.assertEqual(args[4], "10.10.10.10")
self.assertEqual(args[5], 10203)
self.assertEqual(args[6], "key")
self.assertEqual(args[7], "owner")
self.assertEqual(args[8], "ABC")
ctd.subtask_id = "aabbcc"
ctd.src_code = "raise Exception('some exception')"
ctd.deadline = timeout_to_deadline(5)
tc.task_given(ctd)
self.assertEqual(tc.assigned_subtasks["aabbcc"], ctd)
self.assertLessEqual(tc.assigned_subtasks["aabbcc"].deadline,
timeout_to_deadline(5))
self.assertEqual(tc.task_to_subtask_mapping["xyz"], "aabbcc")
tc.task_server.request_resource.assert_called_with(
"xyz", tc.resource_manager.get_resource_header("xyz"),
"10.10.10.10", 10203, "key", "owner")
self.assertTrue(tc.task_resource_collected("xyz"))
self.__wait_for_tasks(tc)
self.assertFalse(tc.counting_task)
self.assertEqual(len(tc.current_computations), 0)
self.assertIsNone(tc.assigned_subtasks.get("aabbcc"))
task_server.send_task_failed.assert_called_with(
"aabbcc", "xyz", 'some exception', "10.10.10.10", 10203, "key",
"owner", "ABC")
ctd.subtask_id = "aabbcc2"
ctd.src_code = "print 'Hello world'"
ctd.timeout = timeout_to_deadline(5)
tc.task_given(ctd)
self.assertTrue(tc.task_resource_collected("xyz"))
self.__wait_for_tasks(tc)
task_server.send_task_failed.assert_called_with(
"aabbcc2", "xyz", "Wrong result format", "10.10.10.10", 10203,
"key", "owner", "ABC")
ctd.subtask_id = "xxyyzz2"
ctd.timeout = timeout_to_deadline(1)
tc.task_given(ctd)
self.assertTrue(tc.task_resource_collected("xyz"))
tt = tc.current_computations[0]
tc.task_computed(tc.current_computations[0])
self.assertEqual(len(tc.current_computations), 0)
task_server.send_task_failed.assert_called_with(
"xxyyzz2", "xyz", "Wrong result format", "10.10.10.10", 10203,
"key", "owner", "ABC")
tt.end_comp()
time.sleep(0.5)
if tt.is_alive():
tt.join(timeout=5)