def test_send_results(self, trust):
ccd = self._get_config_desc()
ccd.min_price = 11
n = Node()
ka = EllipticalKeysAuth(self.path)
ts = TaskServer(n, ccd, ka, self.client,
use_docker_machine_manager=False)
ts.verify_header_sig = lambda x: True
self.ts = ts
ts.client.get_suggested_addr.return_value = "10.10.10.10"
results = {"data": "", "result_type": 0}
task_header = get_example_task_header()
task_header["task_id"] = "xyz"
ts.add_task_header(task_header)
ts.request_task()
self.assertTrue(ts.send_results("xxyyzz", "xyz", results, 40, "10.10.10.10", 10101, "key", n, "node_name"))
self.assertTrue(ts.send_results("xyzxyz", "xyz", results, 40, "10.10.10.10", 10101, "key", n, "node_name"))
self.assertEqual(ts.get_subtask_ttl("xyz"), 120)
wtr = ts.results_to_send["xxyyzz"]
self.assertIsInstance(wtr, WaitingTaskResult)
self.assertEqual(wtr.subtask_id, "xxyyzz")
self.assertEqual(wtr.result, "")
self.assertEqual(wtr.result_type, 0)
self.assertEqual(wtr.computing_time, 40)
self.assertEqual(wtr.last_sending_trial, 0)
self.assertEqual(wtr.delay_time, 0)
self.assertEqual(wtr.owner_address, "10.10.10.10")
self.assertEqual(wtr.owner_port, 10101)
self.assertEqual(wtr.owner_key_id, "key")
self.assertEqual(wtr.owner, n)
self.assertEqual(wtr.already_sending, False)
ts.client.transaction_system.add_to_waiting_payments.assert_called_with(
"xyz", "key", 1)
with self.assertLogs(logger, level='WARNING'):
ts.subtask_rejected("aabbcc")
self.assertIsNotNone(ts.task_keeper.task_headers.get("xyz"))
prev_call_count = trust.PAYMENT.increase.call_count
with self.assertLogs(logger, level="WARNING"):
ts.reward_for_subtask_paid("aa2bb2cc")
self.assertEqual(trust.PAYMENT.increase.call_count, prev_call_count)
ctd = ComputeTaskDef()
ctd.task_id = "xyz"
ctd.subtask_id = "xxyyzz"
ts.task_manager.comp_task_keeper.receive_subtask(ctd)
ts.reward_for_subtask_paid("xxyyzz")
self.assertGreater(trust.PAYMENT.increase.call_count, prev_call_count)
prev_call_count = trust.PAYMENT.increase.call_count
ts.increase_trust_payment("xyz")
self.assertGreater(trust.PAYMENT.increase.call_count, prev_call_count)
prev_call_count = trust.PAYMENT.decrease.call_count
ts.decrease_trust_payment("xyz")
self.assertGreater(trust.PAYMENT.decrease.call_count, prev_call_count)
def test_results_no_payment_addr(self, trust, mock_addr, dump_mock):
mock_addr.return_value = ("10.10.10.10", 1111, "Full NAT")
# FIXME: This test is too heavy, it starts up whole Golem Client.
ccd = self._get_config_desc()
ts = TaskServer(Node(), ccd, EllipticalKeysAuth(self.path), self.client,
use_docker_machine_manager=False)
ts.task_manager.listen_address = "10.10.10.10"
ts.task_manager.listen_port = 1111
ts.receive_subtask_computation_time("xxyyzz", 1031)
self.ts = ts
extra_data = Mock()
extra_data.ctd = ComputeTaskDef()
extra_data.ctd.task_id = "xyz"
extra_data.ctd.subtask_id = "xxyyzz"
extra_data.ctd.environment = "DEFAULT"
extra_data.should_wait = False
task_mock = get_mock_task("xyz", "xxyyzz")
task_mock.query_extra_data.return_value = extra_data
sync_wait(ts.task_manager.add_new_task(task_mock))
ts.task_manager.tasks_states["xyz"].status = ts.task_manager.activeStatus[0]
subtask, wrong_task, wait = ts.task_manager.get_next_subtask(
"DEF", "DEF", "xyz", 1000, 10, 5, 10, 2, "10.10.10.10")
ts.receive_subtask_computation_time("xxyyzz", 1031)
account_info = Mock()
account_info.key_id = "key"
account_info.eth_account = Mock()
account_info.eth_account.address = None
ts.accept_result("xxyyzz", account_info)
self.assertEqual(ts.client.transaction_system.add_payment_info.call_count, 0)
def test_results(self, trust, mock_addr, dump_mock):
mock_addr.return_value = ("10.10.10.10", 1111, "Full NAT")
ccd = self._get_config_desc()
ts = TaskServer(Node(), ccd, EllipticalKeysAuth(self.path), self.client,
use_docker_machine_manager=False)
self.ts = ts
ts.task_manager.listen_port = 1111
ts.task_manager.listen_address = "10.10.10.10"
ts.receive_subtask_computation_time("xxyyzz", 1031)
extra_data = Mock()
extra_data.ctd = ComputeTaskDef()
extra_data.ctd.task_id = "xyz"
extra_data.ctd.subtask_id = "xxyyzz"
extra_data.ctd.environment = "DEFAULT"
extra_data.should_wait = False
task_mock = get_mock_task("xyz", "xxyyzz")
task_mock.query_extra_data.return_value = extra_data
sync_wait(ts.task_manager.add_new_task(task_mock))
ts.task_manager.tasks_states["xyz"].status = ts.task_manager.activeStatus[0]
subtask, wrong_task, wait = ts.task_manager.get_next_subtask("DEF", "DEF", "xyz",
1000, 10, 5, 10, 2,
"10.10.10.10")
ts.receive_subtask_computation_time("xxyyzz", 1031)
self.assertEqual(ts.task_manager.tasks_states["xyz"].subtask_states["xxyyzz"].computation_time, 1031)
expected_value = ceil(1031 * 10 / 3600)
self.assertEqual(ts.task_manager.tasks_states["xyz"].subtask_states["xxyyzz"].value, expected_value)
account_info = Mock()
account_info.key_id = "key"
prev_calls = trust.COMPUTED.increase.call_count
ts.accept_result("xxyyzz", account_info)
ts.client.transaction_system.add_payment_info.assert_called_with("xyz", "xxyyzz", expected_value, account_info)
self.assertGreater(trust.COMPUTED.increase.call_count, prev_calls)
def query_extra_data_for_test_task(self):
ctd = ComputeTaskDef()
ctd.subtask_id = "xxyyzz"
ctd.task_id = "xyz"
ctd.working_directory = self.header.root_path
ctd.src_code = self.src_code
ctd.extra_data = self.extra_data
ctd.short_description = ""
return ctd
def test_comp_keeper(self, dump_mock):
ctk = CompTaskKeeper(Path('ignored'))
header = get_task_header()
header.task_id = "xyz"
with self.assertRaises(TypeError):
ctk.add_request(header, "not a number")
with self.assertRaises(ValueError):
ctk.add_request(header, -2)
ctk.add_request(header, 7200)
self.assertEqual(ctk.active_tasks["xyz"].requests, 1)
self.assertEqual(ctk.active_tasks["xyz"].price, 7200)
self.assertEqual(ctk.active_tasks["xyz"].header, header)
ctk.add_request(header, 23)
self.assertEqual(ctk.active_tasks["xyz"].requests, 2)
self.assertEqual(ctk.active_tasks["xyz"].price, 7200)
self.assertEqual(ctk.active_tasks["xyz"].header, header)
self.assertEqual(ctk.get_value("xyz", 1), 2)
header.task_id = "xyz2"
ctk.add_request(header, 25000)
self.assertEqual(ctk.active_tasks["xyz2"].price, 25000)
self.assertEqual(ctk.get_value("xyz2", 4.5), 32)
header.task_id = "xyz"
thread = get_task_header()
thread.task_id = "qaz123WSX"
with self.assertRaises(ValueError):
ctk.add_request(thread, -1)
with self.assertRaises(TypeError):
ctk.add_request(thread, '1')
ctk.add_request(thread, 12)
header = get_task_header()
header.task_id = "qwerty"
ctk.active_tasks["qwerty"] = CompTaskInfo(header, 12)
ctk.active_tasks["qwerty"].price = "abc"
with self.assertRaises(TypeError):
ctk.get_value('qwerty', 12)
self.assertEqual(ctk.get_value(thread.task_id, 600), 2)
self.assertIsNone(ctk.get_subtask_ttl("abc"))
ctd = ComputeTaskDef()
with self.assertLogs(logger, level="WARNING"):
self.assertFalse(ctk.receive_subtask(ctd))
with self.assertLogs(logger, level="WARNING"):
self.assertIsNone(ctk.get_node_for_task_id("abc"))
with self.assertLogs(logger, level="WARNING"):
self.assertIsNone(ctk.get_value("abc", 10))
with self.assertLogs(logger, level="WARNING"):
ctk.request_failure("abc")
ctk.request_failure("xyz")
self.assertEqual(ctk.active_tasks["xyz"].requests, 1)
def __get_merge_ctd(self, files):
script_file = dirmanager.find_task_script(APP_DIR,
"docker_luxmerge.py")
if script_file is None:
logger.error("Cannot find merger script")
return
with open(script_file) as f:
src_code = f.read()
ctd = ComputeTaskDef()
ctd.task_id = self.header.task_id
ctd.subtask_id = self.header.task_id
ctd.extra_data = {'output_flm': self.output_file, 'flm_files': files}
ctd.src_code = src_code
ctd.working_directory = "."
ctd.docker_images = self.header.docker_images
ctd.deadline = timeout_to_deadline(self.merge_timeout)
return ctd
def test_receive_subtask_problems(self):
ctk = CompTaskKeeper(Path(self.path), False)
th = get_task_header()
ctk.add_request(th, 5)
ctd = ComputeTaskDef()
ctd.task_id = "xyz"
ctd.subtask_id = "abc"
ctk.receive_subtask(ctd)
assert ctk.active_tasks["xyz"].requests == 0
assert ctk.subtask_to_task["abc"] == "xyz"
ctd2 = ComputeTaskDef()
ctd2.task_id = "xyz"
ctd2.subtask_id = "def"
ctk.receive_subtask(ctd2)
assert ctk.active_tasks["xyz"].requests == 0
assert ctk.subtask_to_task.get("def") is None
assert ctk.subtask_to_task["abc"] == "xyz"
ctk.active_tasks["xyz"].requests = 1
ctk.receive_subtask(ctd)
assert ctk.active_tasks["xyz"].requests == 1
def query_extra_data(self,
perf_index,
num_cores=1,
node_id=None,
node_name=None):
ctd = ComputeTaskDef()
ctd.task_id = self.header.task_id
ctd.subtask_id = self.subtasks_id[0]
ctd.environment = "DEFAULT"
ctd.should_wait = False
self.subtasks_id = self.subtasks_id[1:]
e = self.ExtraData(False, ctd)
return e
def _get_task_mock(self,
task_id="xyz",
subtask_id="xxyyzz",
timeout=120.0,
subtask_timeout=120.0):
header = self._get_task_header(task_id, timeout, subtask_timeout)
task_mock = TaskMock(header, src_code='')
ctd = ComputeTaskDef()
ctd.task_id = task_id
ctd.subtask_id = subtask_id
ctd.environment = "DEFAULT"
ctd.deadline = timeout_to_deadline(subtask_timeout)
task_mock.query_extra_data_return_value = Task.ExtraData(
should_wait=False, ctd=ctd)
Task.get_progress = Mock()
task_mock.get_progress.return_value = 0.3
return task_mock
def _new_compute_task_def(self, hash, extra_data, working_directory,
perf_index):
ctd = ComputeTaskDef()
ctd.task_id = self.header.task_id
ctd.subtask_id = hash
ctd.extra_data = extra_data
ctd.short_description = self._short_extra_data_repr(
perf_index, extra_data)
ctd.src_code = self.src_code
ctd.performance = perf_index
ctd.working_directory = working_directory
ctd.docker_images = self.header.docker_images
ctd.deadline = timeout_to_deadline(self.header.subtask_timeout)
return ctd
def query_extra_data(self, perf_index, num_cores=1, node_id=None, node_name=None):
"""Returns data for the next subtask.
:param int perf_index:
:param int num_cores:
:param str | None node_id:
:param str | None node_name:
:rtype: ComputeTaskDef"""
# create new subtask_id
subtask_id = uuid.uuid4().get_hex()
with self._lock:
# check if a task has been assigned to this node
if node_id in self.assigned_nodes:
return self.ExtraData(should_wait=True)
# assign a task
self.assigned_nodes[node_id] = subtask_id
self.assigned_subtasks[subtask_id] = node_id
# create subtask-specific data, 4 bits go for one char (hex digit)
data = random.getrandbits(self.task_params.subtask_data_size * 4)
self.subtask_ids.append(subtask_id)
self.subtask_data[subtask_id] = '%x' % data
subtask_def = ComputeTaskDef()
subtask_def.task_id = self.task_id
subtask_def.subtask_id = subtask_id
subtask_def.src_code = self.src_code
subtask_def.task_owner = self.header.task_owner
subtask_def.environment = self.header.environment
subtask_def.return_address = self.header.task_owner_address
subtask_def.return_port = self.header.task_owner_port
subtask_def.deadline = timeout_to_deadline(5 * 60)
subtask_def.extra_data = {
'data_file': self.shared_data_file,
'subtask_data': self.subtask_data[subtask_id],
'difficulty': self.task_params.difficulty,
'result_size': self.task_params.result_size,
'result_file': 'result.' + subtask_id[0:6]
}
return self.ExtraData(ctd=subtask_def)
def query_extra_data_for_advanced_verification(self, extra_data):
ctd = ComputeTaskDef()
ctd.extra_data = extra_data
return ctd
def test_react_to_task_to_compute(self):
conn = Mock()
ts = TaskSession(conn)
ts.key_id = "KEY_ID"
ts.task_manager = Mock()
ts.task_computer = Mock()
ts.task_server = Mock()
ts.task_server.get_subtask_ttl.return_value = 31313
env = Mock()
env.docker_images = [DockerImage("dockerix/xii", tag="323")]
env.allow_custom_main_program_file = False
env.get_source_code.return_value = None
ts.task_server.get_environment_by_id.return_value = env
def __reset_mocks():
ts.task_manager.reset_mock()
ts.task_computer.reset_mock()
conn.reset_mock()
# msg.ctd is None -> failure
msg = MessageTaskToCompute()
with self.assertLogs(logger, level="WARNING"):
ts._react_to_task_to_compute(msg)
ts.task_manager.comp_task_keeper.receive_subtask.assert_not_called()
ts.task_computer.session_closed.assert_called_with()
assert conn.close.called
# No source code in the local environment -> failure
__reset_mocks()
ctd = ComputeTaskDef()
ctd.key_id = "KEY_ID"
ctd.subtask_id = "SUBTASKID"
ctd.task_owner = Node()
ctd.task_owner.key = "KEY_ID"
ctd.return_address = "10.10.10.10"
ctd.return_port = 1112
ctd.docker_images = [DockerImage("dockerix/xiii", tag="323")]
msg = MessageTaskToCompute(ctd)
ts._react_to_task_to_compute(msg)
ts.task_manager.comp_task_keeper.receive_subtask.assert_not_called()
ts.task_computer.session_closed.assert_called_with()
assert conn.close.called
# Source code from local environment -> proper execution
__reset_mocks()
env.get_source_code.return_value = "print 'Hello world'"
ts._react_to_task_to_compute(msg)
ts.task_manager.comp_task_keeper.receive_subtask.assert_called_with(
ctd)
ts.task_computer.session_closed.assert_not_called()
ts.task_server.add_task_session.assert_called_with("SUBTASKID", ts)
ts.task_computer.task_given.assert_called_with(ctd)
conn.close.assert_not_called()
# Wrong key id -> failure
__reset_mocks()
ctd.key_id = "KEY_ID2"
ts._react_to_task_to_compute(MessageTaskToCompute(ctd))
ts.task_manager.comp_task_keeper.receive_subtask.assert_not_called()
ts.task_computer.session_closed.assert_called_with()
assert conn.close.called
# Wrong task owner key id -> failure
__reset_mocks()
ctd.key_id = "KEY_ID"
ctd.task_owner.key = "KEY_ID2"
ts._react_to_task_to_compute(MessageTaskToCompute(ctd))
ts.task_manager.comp_task_keeper.receive_subtask.assert_not_called()
ts.task_computer.session_closed.assert_called_with()
assert conn.close.called
# Wrong return port -> failure
__reset_mocks()
ctd.task_owner.key = "KEY_ID"
ctd.return_port = 0
ts._react_to_task_to_compute(MessageTaskToCompute(ctd))
ts.task_manager.comp_task_keeper.receive_subtask.assert_not_called()
ts.task_computer.session_closed.assert_called_with()
assert conn.close.called
# Proper port and key -> proper execution
__reset_mocks()
ctd.task_owner.key = "KEY_ID"
ctd.return_port = 1319
ts._react_to_task_to_compute(MessageTaskToCompute(ctd))
conn.close.assert_not_called()
# Allow custom code / no code in ComputeTaskDef -> failure
__reset_mocks()
env.allow_custom_main_program_file = True
ctd.src_code = ""
ts._react_to_task_to_compute(MessageTaskToCompute(ctd))
ts.task_manager.comp_task_keeper.receive_subtask.assert_not_called()
ts.task_computer.session_closed.assert_called_with()
assert conn.close.called
# Allow custom code / code in ComputerTaskDef -> proper execution
__reset_mocks()
ctd.src_code = "print 'Hello world!'"
ts._react_to_task_to_compute(MessageTaskToCompute(ctd))
ts.task_computer.session_closed.assert_not_called()
ts.task_server.add_task_session.assert_called_with("SUBTASKID", ts)
ts.task_computer.task_given.assert_called_with(ctd)
conn.close.assert_not_called()
# No environment available -> failure
__reset_mocks()
ts.task_server.get_environment_by_id.return_value = None
ts._react_to_task_to_compute(MessageTaskToCompute(ctd))
assert ts.err_msg.startswith("Wrong environment")
ts.task_manager.comp_task_keeper.receive_subtask.assert_not_called()
ts.task_computer.session_closed.assert_called_with()
assert conn.close.called
# Envrionment is Docker environment but with different images -> failure
__reset_mocks()
ts.task_server.get_environment_by_id.return_value = \
DockerEnvironment([DockerImage("dockerix/xii", tag="323"),
DockerImage("dockerix/xiii", tag="325"),
DockerImage("dockerix/xiii")])
ts._react_to_task_to_compute(MessageTaskToCompute(ctd))
assert ts.err_msg.startswith("Wrong docker images")
ts.task_manager.comp_task_keeper.receive_subtask.assert_not_called()
ts.task_computer.session_closed.assert_called_with()
assert conn.close.called
# Envrionment is Docker environment with proper images, but no srouce code -> failure
__reset_mocks()
de = DockerEnvironment([
DockerImage("dockerix/xii", tag="323"),
DockerImage("dockerix/xiii", tag="325"),
DockerImage("dockerix/xiii", tag="323")
])
ts.task_server.get_environment_by_id.return_value = de
ts._react_to_task_to_compute(MessageTaskToCompute(ctd))
assert ts.err_msg.startswith("No source code")
ts.task_manager.comp_task_keeper.receive_subtask.assert_not_called()
ts.task_computer.session_closed.assert_called_with()
assert conn.close.called
# Proper Docker environment with source code
__reset_mocks()
file_name = os.path.join(self.path, "main_program_file")
with open(file_name, 'w') as f:
f.write("Hello world!")
de.main_program_file = file_name
ts._react_to_task_to_compute(MessageTaskToCompute(ctd))
ts.task_server.add_task_session.assert_called_with("SUBTASKID", ts)
ts.task_computer.task_given.assert_called_with(ctd)
conn.close.assert_not_called()
def _get_better_task_def(self):
ctd = ComputeTaskDef()
ctd.docker_images = BlenderEnvironment().docker_images
return ctd
def _get_bad_task_def(self):
ctd = ComputeTaskDef()
return ctd
def test_computation(self):
task_server = mock.MagicMock()
task_server.get_task_computer_root.return_value = self.path
task_server.config_desc = config_desc()
tc = TaskComputer("ABC", task_server, use_docker_machine_manager=False)
ctd = ComputeTaskDef()
ctd.task_id = "xyz"
ctd.subtask_id = "xxyyzz"
ctd.return_address = "10.10.10.10"
ctd.return_port = 10203
ctd.key_id = "key"
ctd.task_owner = "owner"
ctd.src_code = "cnt=0\nfor i in range(10000):\n\tcnt += 1\noutput={'data': cnt, 'result_type': 0}"
ctd.extra_data = {}
ctd.short_description = "add cnt"
ctd.deadline = timeout_to_deadline(10)
self.assertEqual(len(tc.assigned_subtasks), 0)
tc.task_given(ctd)
self.assertEqual(tc.assigned_subtasks["xxyyzz"], ctd)
self.assertLessEqual(tc.assigned_subtasks["xxyyzz"].deadline,
timeout_to_deadline(10))
self.assertEqual(tc.task_to_subtask_mapping["xyz"], "xxyyzz")
tc.task_server.request_resource.assert_called_with(
"xyz", tc.resource_manager.get_resource_header("xyz"),
"10.10.10.10", 10203, "key", "owner")
assert tc.task_resource_collected("xyz")
tc.task_server.unpack_delta.assert_called_with(
tc.dir_manager.get_task_resource_dir("xyz"), None, "xyz")
assert len(tc.current_computations) == 0
assert tc.assigned_subtasks.get("xxyyzz") is None
task_server.send_task_failed.assert_called_with(
"xxyyzz", "xyz", "Host direct task not supported", "10.10.10.10",
10203, "key", "owner", "ABC")
tc.support_direct_computation = True
tc.task_given(ctd)
assert tc.task_resource_collected("xyz")
assert not tc.waiting_for_task
assert len(tc.current_computations) == 1
self.__wait_for_tasks(tc)
prev_task_failed_count = task_server.send_task_failed.call_count
self.assertFalse(tc.counting_task)
self.assertEqual(len(tc.current_computations), 0)
self.assertIsNone(tc.assigned_subtasks.get("xxyyzz"))
assert task_server.send_task_failed.call_count == prev_task_failed_count
self.assertTrue(task_server.send_results.called)
args = task_server.send_results.call_args[0]
self.assertEqual(args[0], "xxyyzz")
self.assertEqual(args[1], "xyz")
self.assertEqual(args[2]["data"], 10000)
self.assertGreater(args[3], 0)
self.assertLess(args[3], 10)
self.assertEqual(args[4], "10.10.10.10")
self.assertEqual(args[5], 10203)
self.assertEqual(args[6], "key")
self.assertEqual(args[7], "owner")
self.assertEqual(args[8], "ABC")
ctd.subtask_id = "aabbcc"
ctd.src_code = "raise Exception('some exception')"
ctd.deadline = timeout_to_deadline(5)
tc.task_given(ctd)
self.assertEqual(tc.assigned_subtasks["aabbcc"], ctd)
self.assertLessEqual(tc.assigned_subtasks["aabbcc"].deadline,
timeout_to_deadline(5))
self.assertEqual(tc.task_to_subtask_mapping["xyz"], "aabbcc")
tc.task_server.request_resource.assert_called_with(
"xyz", tc.resource_manager.get_resource_header("xyz"),
"10.10.10.10", 10203, "key", "owner")
self.assertTrue(tc.task_resource_collected("xyz"))
self.__wait_for_tasks(tc)
self.assertFalse(tc.counting_task)
self.assertEqual(len(tc.current_computations), 0)
self.assertIsNone(tc.assigned_subtasks.get("aabbcc"))
task_server.send_task_failed.assert_called_with(
"aabbcc", "xyz", 'some exception', "10.10.10.10", 10203, "key",
"owner", "ABC")
ctd.subtask_id = "aabbcc2"
ctd.src_code = "print 'Hello world'"
ctd.timeout = timeout_to_deadline(5)
tc.task_given(ctd)
self.assertTrue(tc.task_resource_collected("xyz"))
self.__wait_for_tasks(tc)
task_server.send_task_failed.assert_called_with(
"aabbcc2", "xyz", "Wrong result format", "10.10.10.10", 10203,
"key", "owner", "ABC")
ctd.subtask_id = "xxyyzz2"
ctd.timeout = timeout_to_deadline(1)
tc.task_given(ctd)
self.assertTrue(tc.task_resource_collected("xyz"))
tt = tc.current_computations[0]
tc.task_computed(tc.current_computations[0])
self.assertEqual(len(tc.current_computations), 0)
task_server.send_task_failed.assert_called_with(
"xxyyzz2", "xyz", "Wrong result format", "10.10.10.10", 10203,
"key", "owner", "ABC")
tt.end_comp()
time.sleep(0.5)
if tt.is_alive():
tt.join(timeout=5)