def test_validate_no_fixed_header(self): del self.th_dict_repr['fixed_header'] with self.assertRaisesRegex( exceptions.TaskHeaderError, "Fixed header is missing" ): TaskHeader.validate(self.th_dict_repr)
def test_validate_no_task_owner_node_name(self): del self.th_dict_repr['fixed_header']['task_owner']['node_name'] with self.assertRaisesRegex( exceptions.TaskHeaderError, "node name missing", ): TaskHeader.validate(self.th_dict_repr)
def test_validate_no_task_id(self): del self.th_dict_repr['fixed_header']['task_id'] with self.assertRaisesRegex( exceptions.TaskHeaderError, "Task ID missing", ): TaskHeader.validate(self.th_dict_repr)
def test_validate_no_subtasks_count(self): del self.th_dict_repr['fixed_header']['subtasks_count'] with self.assertRaisesRegex( exceptions.TaskHeaderError, r"^Subtasks count is missing" ): TaskHeader.validate(self.th_dict_repr)
def test_validate_subtasks_count_too_low(self): self.th_dict_repr['fixed_header']['subtasks_count'] = -1 with self.assertRaisesRegex( exceptions.TaskHeaderError, r"^Subtasks count is less than 1 {.*'subtasks_count': -1.*}$", ): TaskHeader.validate(self.th_dict_repr)
def test_validate_subtasks_count_invalid_type(self): self.th_dict_repr['fixed_header']['subtasks_count'] = None with self.assertRaisesRegex( exceptions.TaskHeaderError, r"^Subtasks count is missing", ): TaskHeader.validate(self.th_dict_repr)
def test_header_serialization(self): node = Node(node_name="test node", pub_addr="10.10.10.10", pub_port=1023) task_header = TaskHeader( task_id="xyz", environment="DEFAULT", task_owner=node) # ignore dynamic properties task_header.fixed_header.last_checking = 0 task_header_dict = task_header.to_dict() serialized = CBORSerializer.dumps(task_header_dict) deserialized = CBORSerializer.loads(serialized) task_header_from_dict = TaskHeader.from_dict(deserialized) # ignore dynamic properties task_header_from_dict.fixed_header.last_checking = 0 assert task_header_from_dict.to_dict() == task_header_dict assert isinstance(task_header_from_dict.task_owner, Node) task_header_bin = task_header.to_binary() bin_serialized = CBORSerializer.dumps(task_header_bin) bin_deserialized = CBORSerializer.loads(bin_serialized) assert bin_deserialized == task_header_bin
def test_validate_illegal_deadline(self): self.th_dict_repr['fixed_header']['deadline'] = datetime.now() with self.assertRaisesRegex( exceptions.TaskHeaderError, "Deadline is not a timestamp" ): TaskHeader.validate(self.th_dict_repr)
def test_validate_deadline_passed(self): self.th_dict_repr['fixed_header']['deadline'] = get_timestamp_utc() - 10 with self.assertRaisesRegex( exceptions.TaskHeaderError, "Deadline already passed" ): TaskHeader.validate(self.th_dict_repr)
def test_validate_negative_timeout(self): self.th_dict_repr['fixed_header']['subtask_timeout'] = -131 with self.assertRaisesRegex( exceptions.TaskHeaderError, "Subtask timeout is less than 0", ): TaskHeader.validate(self.th_dict_repr)
def test_validate_illegal_timeout(self): self.th_dict_repr['fixed_header']['subtask_timeout'] = "abc" with self.assertRaisesRegex( exceptions.TaskHeaderError, "Subtask timeout is not a number" ): TaskHeader.validate(self.th_dict_repr)
def get_task_header(): header = get_dict_task_header() return TaskHeader(header["node_name"], header["task_id"], header["task_owner_address"], header["task_owner_port"], header["task_owner_key_id"], header["environment"], header["task_owner"], header["deadline"], header["subtask_timeout"], 1024, 1.0, 1000)
def __init__(self, src_code, task_definition, node_name, environment, resource_size=0, owner_address="", owner_port=0, owner_key_id="", max_pending_client_results=MAX_PENDING_CLIENT_RESULTS): """Create more specific task implementation """ self.task_definition = task_definition task_timeout = task_definition.full_task_timeout deadline = timeout_to_deadline(task_timeout) th = TaskHeader( node_name=node_name, task_id=task_definition.task_id, task_owner_address=owner_address, task_owner_port=owner_port, task_owner_key_id=owner_key_id, environment=environment, task_owner=Node(), deadline=deadline, subtask_timeout=task_definition.subtask_timeout, resource_size=resource_size, estimated_memory=task_definition.estimated_memory, max_price=task_definition.max_price, docker_images=task_definition.docker_images, ) Task.__init__(self, th, src_code) self.task_resources = list() self.total_tasks = 0 self.last_task = 0 self.num_tasks_received = 0 self.subtasks_given = {} self.num_failed_subtasks = 0 self.full_task_timeout = task_timeout self.counting_nodes = {} self.root_path = None self.stdout = { } # for each subtask keep info about stdout received from computing node self.stderr = { } # for each subtask keep info about stderr received from computing node self.results = { } # for each subtask keep info about files containing results self.res_files = {} self.tmp_dir = None self.verificator = self.VERIFICATOR_CLASS() self.max_pending_client_results = max_pending_client_results
def header(self, max_price, last_checking=None, deadline=None, min_version="4"): if not last_checking: last_checking = time.time() if not deadline: deadline = timeout_to_deadline(36000) ret = TaskHeader( task_id=str(uuid4()), environment="DEFAULT", task_owner=Node(), max_price=max_price, deadline=deadline, min_version=min_version) if last_checking: ret.last_checking = last_checking return ret
def get_mock_task(task_id, subtask_id): task_mock = Mock() task_mock.header = TaskHeader.from_dict(get_example_task_header()) task_mock.header.task_id = task_id task_mock.header.max_price = 10000 task_mock.query_extra_data.return_value.ctd.task_id = task_id task_mock.query_extra_data.return_value.ctd.subtask_id = subtask_id return task_mock
def add_task_header(self, th_dict_repr: dict) -> bool: try: TaskHeader.validate(th_dict_repr) header = TaskHeader.from_dict(th_dict_repr) if not self.verify_header_sig(header): raise ValueError("Invalid signature") if self.task_manager.is_this_my_task(header): return True # Own tasks are not added to task keeper return self.task_keeper.add_task_header(header) except exceptions.TaskHeaderError as e: logger.warning("Wrong task header received: %s", e) return False except Exception: # pylint: disable=broad-except logger.exception("Task header validation failed") return False
def test_add_task_header(self, *_): keys_auth_2 = KeysAuth( os.path.join(self.path, "2"), 'priv_key', 'password', ) ts = self.ts task_header = get_example_task_header(keys_auth_2.public_key) with self.assertRaises(Exception) as raised: ts.add_task_header(task_header) self.assertEqual(raised.exception.message, "Invalid signature") self.assertEqual(len(ts.get_others_tasks_headers()), 0) task_header["signature"] = keys_auth_2.sign( TaskHeader.dict_to_binary(task_header)) self.assertTrue(ts.add_task_header(task_header)) self.assertEqual(len(ts.get_others_tasks_headers()), 1) task_header = get_example_task_header(keys_auth_2.public_key) task_id2 = task_header["fixed_header"]["task_id"] task_header["signature"] = keys_auth_2.sign( TaskHeader.dict_to_binary(task_header)) self.assertTrue(ts.add_task_header(task_header)) self.assertEqual(len(ts.get_others_tasks_headers()), 2) self.assertTrue(ts.add_task_header(task_header)) self.assertEqual(len(ts.get_others_tasks_headers()), 2) new_header = dict(task_header) new_header["fixed_header"]["task_owner"]["pub_port"] = 9999 new_header["signature"] = keys_auth_2.sign( TaskHeader.dict_to_binary(new_header)) # An attempt to update fixed header should *not* succeed self.assertFalse(ts.add_task_header(new_header)) self.assertEqual(len(ts.get_others_tasks_headers()), 2) saved_task = next(th for th in ts.get_others_tasks_headers() if th["fixed_header"]["task_id"] == task_id2) self.assertEqual(saved_task["signature"], task_header["signature"])
def get_mock_task(key_gen="whatsoever", subtask_id="whatever"): task_mock = Mock() key_id = str.encode(key_gen) task_mock.header = TaskHeader.from_dict(get_example_task_header(key_id)) task_id = task_mock.header.task_id task_mock.header.max_price = 1010 task_mock.query_extra_data.return_value.ctd = ComputeTaskDef( task_id=task_id, subtask_id=subtask_id, ) return task_mock
def test_create_task(self, *_): c = self.client c.enqueue_new_task = Mock() # create a task t = Task(TaskHeader("node_name", "task_id", "10.10.10.10", 123, "owner_id", "DEFAULT"), src_code="print('hello')") c.create_task(DictSerializer.dump(t)) self.assertTrue(c.enqueue_new_task.called)
def test_add_task_header(self): config = self._get_config_desc() keys_auth = EllipticalKeysAuth(self.path) keys_auth_2 = EllipticalKeysAuth(os.path.join(self.path, "2")) self.ts = ts = TaskServer(Node(), config, keys_auth, self.client, use_docker_machine_manager=False) task_header = get_example_task_header() task_header["task_id"] = "xyz" with self.assertRaises(Exception) as raised: ts.add_task_header(task_header) self.assertEqual(raised.exception.message, "Invalid signature") self.assertEqual(len(ts.get_tasks_headers()), 0) task_header["task_owner_key_id"] = keys_auth_2.key_id task_header["signature"] = keys_auth_2.sign(TaskHeader.dict_to_binary(task_header)) self.assertIsNotNone(ts.add_task_header(task_header)) self.assertEqual(len(ts.get_tasks_headers()), 1) task_header = get_example_task_header() task_header["task_id"] = "xyz_2" task_header["task_owner_key_id"] = keys_auth_2.key_id task_header["signature"] = keys_auth_2.sign(TaskHeader.dict_to_binary(task_header)) self.assertIsNotNone(ts.add_task_header(task_header)) self.assertEqual(len(ts.get_tasks_headers()), 2) self.assertIsNotNone(ts.add_task_header(task_header)) self.assertEqual(len(ts.get_tasks_headers()), 2) new_header = dict(task_header) new_header["task_owner"]["pub_port"] = 9999 new_header["signature"] = keys_auth_2.sign(TaskHeader.dict_to_binary(new_header)) self.assertIsNotNone(ts.add_task_header(new_header)) self.assertEqual(len(ts.get_tasks_headers()), 2) saved_task = next(th for th in ts.get_tasks_headers() if th["task_id"] == "xyz_2") self.assertEqual(saved_task["signature"], new_header["signature"])
def __init__(self, client_id, params, num_subtasks, public_key): """Creates a new dummy task :param string client_id: client id :param DummyTaskParameters params: task parameters 1024 hashes on average """ task_id = idgenerator.generate_id(public_key) owner_address = '' owner_port = 0 owner_key_id = encode_hex(public_key)[2:] environment = self.ENVIRONMENT_NAME header = TaskHeader(task_id=task_id, environment=environment, task_owner=Node(node_name=client_id, pub_addr=owner_address, pub_port=owner_port, key=owner_key_id), deadline=timeout_to_deadline(14400), subtask_timeout=1200, subtasks_count=num_subtasks, resource_size=params.shared_data_size + params.subtask_data_size, estimated_memory=0, max_price=MIN_PRICE) # load the script to be run remotely from the file in the current dir script_path = path.join(path.dirname(__file__), 'computation.py') with open(script_path, 'r') as f: src_code = f.read() src_code += '\noutput = run_dummy_task(' \ 'data_file, subtask_data, difficulty, result_size, tmp_path)' from apps.dummy.task.dummytaskstate import DummyTaskDefinition from apps.dummy.task.dummytaskstate import DummyTaskDefaults task_definition = DummyTaskDefinition(DummyTaskDefaults()) Task.__init__(self, header, src_code, task_definition) self.task_id = task_id self.task_params = params self.task_resources = [] self.resource_parts = {} self.shared_data_file = None self.subtasks_count = num_subtasks self.total_tasks = self.subtasks_count self.subtask_ids = [] self.subtask_data = {} self.subtask_results = {} self.assigned_nodes = {} self.assigned_subtasks = {} self.total_tasks = 1 self._lock = Lock()
def build(self): t = self.task_class() t.header = TaskHeader(node_name="node1", task_id="xyz", task_owner_address="127.0.0.1", task_owner_port=45000, task_owner_key_id="key2", environment="test", max_price=30 * denoms.ether) t.header.root_path = self.path t.src_code = self.src_code t.extra_data = self.extra_data return t
def _get_task_header(self, task_id, timeout, subtask_timeout): return TaskHeader( node_name="test_node_%s" % (self.test_nonce, ), task_id=task_id, task_owner_address="task_owner_address_%s" % (self.test_nonce, ), task_owner_port="task_owner_port_%s" % (self.test_nonce, ), task_owner_key_id="task_owner_key_id_%s" % (self.test_nonce, ), environment="test_environ_%s" % (self.test_nonce, ), resource_size=2 * 1024, estimated_memory=3 * 1024, max_price=10000, deadline=timeout_to_deadline(timeout), subtask_timeout=subtask_timeout, )
def __init__(self, client_id, params, num_subtasks): """Creates a new dummy task :param string client_id: client id :param DummyTaskParameters params: task parameters 1024 hashes on average """ task_id = SimpleAuth.generate_uuid().get_hex() owner_address = '' owner_port = 0 owner_key_id = '' environment = self.ENVIRONMENT_NAME header = TaskHeader( client_id, task_id, owner_address, owner_port, owner_key_id, environment, task_owner=Node(), deadline=timeout_to_deadline(14400), subtask_timeout=1200, resource_size=params.shared_data_size + params.subtask_data_size, estimated_memory=0, max_price=MIN_PRICE) # load the script to be run remotely from the file in the current dir script_path = path.join(path.dirname(__file__), 'computation.py') with open(script_path, 'r') as f: src_code = f.read() src_code += '\noutput = run_dummy_task(' \ 'data_file, subtask_data, difficulty, result_size)' Task.__init__(self, header, src_code) self.task_id = task_id self.task_params = params self.task_resources = [] self.resource_parts = {} self.shared_data_file = None self.total_subtasks = num_subtasks self.subtask_ids = [] self.subtask_data = {} self.subtask_results = {} self.assigned_nodes = {} self.assigned_subtasks = {} self._lock = Lock()
def test_resource_send(self, mock_addr): from pydispatch import dispatcher mock_addr.return_value = self.addr_return self.tm.task_persistence = True t = Task( TaskHeader("ABC", "xyz", "10.10.10.10", 1023, "abcde", "DEFAULT"), "print 'hello world'") listener_mock = Mock() def listener(sender, signal, event, task_id): self.assertEquals(event, 'task_status_updated') self.assertEquals(task_id, t.header.task_id) listener_mock() dispatcher.connect(listener, signal='golem.taskmanager') try: sync_wait(self.tm.add_new_task(t)) self.tm.resources_send("xyz") self.assertEquals(listener_mock.call_count, 2) finally: dispatcher.disconnect(listener, signal='golem.taskmanager')
def test_update_signatures(self, _): node = Node("node", "key_id", "10.0.0.10", 40103, "1.2.3.4", 40103, None, 40102, 40102) task = Task( TaskHeader("node", "task_id", "1.2.3.4", 1234, "key_id", "environment", task_owner=node), '') self.tm.keys_auth = EllipticalKeysAuth(self.path) sync_wait(self.tm.add_new_task(task)) sig = task.header.signature self.tm.update_task_signatures() assert task.header.signature == sig task.header.task_owner.pub_port = 40104 self.tm.update_task_signatures() assert task.header.signature != sig
def test_task_simple_serializer(self): with self.assertRaises(TypeError): Task.build_task("Not Task Builder") with self.assertRaises(TypeError): Task.register_listener("Not Listener") t = Task(Mock(), "") self.assertIsInstance(t, Task) self.assertEqual(t.get_stdout("abc"), "") self.assertEqual(t.get_stderr("abc"), "") self.assertEqual(t.get_results("abc"), []) t = Task( TaskHeader("ABC", "xyz", "10.10.10.10", 1023, "key", "DEFAULT", Node()), "print 'Hello world'") tl1 = TaskEventListener() tl2 = TaskEventListener() t.register_listener(tl1) t.register_listener(tl2) assert len(t.listeners) == 2 p = SimpleSerializer.dumps(t) u = SimpleSerializer.loads(p) assert t.src_code == u.src_code assert t.header.task_id == u.header.task_id assert t.header.task_owner.node_name == u.header.task_owner.node_name assert u.get_results("abc") == [] assert len(t.listeners) == 2 assert len(u.listeners) == 0 t.unregister_listener(tl2) assert len(t.listeners) == 1 assert t.listeners[0] == tl1 t.listeners[0].notify_update_task("abc") t.unregister_listener(tl1) assert len(t.listeners) == 0 with self.assertLogs(logger, level="WARNING"): t.unregister_listener(tl1)
def test_header_serialization(self): node = dict(node_name="test node") docker_images = [ dict(repository="repo_{}".format(i), id="id_{}".format(i), tag="tag_{}".format(i)) for i in xrange(4) ] task_header = TaskHeader("ABC", "xyz", "10.10.10.10", 1023, "key", "DEFAULT", task_owner=node, docker_images=docker_images) # ignore dynamic properties task_header.last_checking = 0 task_header_dict = task_header.to_dict() serialized = CBORSerializer.dumps(task_header_dict) deserialized = CBORSerializer.loads(serialized) task_header_from_dict = TaskHeader.from_dict(deserialized) # ignore dynamic properties task_header_from_dict.last_checking = 0 assert task_header_from_dict.to_dict() == task_header_dict assert isinstance(task_header_from_dict.task_owner, Node) assert all([ isinstance(di, DockerImage) for di in task_header_from_dict.docker_images ]) task_header_bin = task_header.to_binary() bin_serialized = CBORSerializer.dumps(task_header_bin) bin_deserialized = CBORSerializer.loads(bin_serialized) assert bin_deserialized == task_header_bin
def test_validate_ok(self): TaskHeader.validate(self.th_dict_repr)
def test_from_dict_no_subtasks_count(self): del self.th_dict_repr['fixed_header']['subtasks_count'] with mock.patch("golem.task.taskbase.logger.debug") as log_mock: TaskHeader.from_dict(self.th_dict_repr) log_mock.assert_called_once()