def _dump_some_tasks(self, tasks_dir):
ctk = CompTaskKeeper(tasks_dir)
test_headers = []
test_subtasks_ids = []
for x in range(10):
header = get_task_header()
header.deadline = timeout_to_deadline(1)
header.subtask_timeout = 1.5
header.resource_size = 1
test_headers.append(header)
price_bid = int(random.random() * 100)
ctk.add_request(header, price_bid)
ctd = ComputeTaskDef()
ctd['task_id'] = header.task_id
ctd['subtask_id'] = idgenerator.generate_new_id_from_id(
header.task_id, )
ctd['deadline'] = timeout_to_deadline(header.subtask_timeout - 0.5)
price = taskkeeper.compute_subtask_value(
price_bid,
header.subtask_timeout,
)
ttc = msg_factories.tasks.TaskToComputeFactory(price=price)
ttc.compute_task_def = ctd
self.assertTrue(ctk.receive_subtask(ttc))
test_subtasks_ids.append(ctd['subtask_id'])
del ctk
another_ctk = CompTaskKeeper(tasks_dir)
for (subtask_id, header) in zip(test_subtasks_ids, test_headers):
self.assertIn(subtask_id, another_ctk.subtask_to_task)
self.assertIn(header.task_id, another_ctk.active_tasks)
def test_old_tasks(frozen_time, _): # pylint: disable=no-self-argument
tk = TaskHeaderKeeper(environments_manager=EnvironmentsManager(),
node=p2p.Node(),
min_price=10)
e = Environment()
e.accept_tasks = True
tk.environments_manager.add_environment(e)
task_header = get_task_header()
task_header.fixed_header.deadline = timeout_to_deadline(10)
assert tk.add_task_header(task_header)
task_id = task_header.task_id
task_header2 = get_task_header("abc")
task_header2.fixed_header.deadline = timeout_to_deadline(1)
task_id2 = task_header2.task_id
assert tk.add_task_header(task_header2)
assert tk.task_headers.get(task_id2) is not None
assert tk.task_headers.get(task_id) is not None
assert tk.removed_tasks.get(task_id2) is None
assert tk.removed_tasks.get(task_id) is None
assert len(tk.supported_tasks) == 2
frozen_time.tick(timedelta(seconds=1.1)) # pylint: disable=no-member
tk.remove_old_tasks()
assert tk.task_headers.get(task_id2) is None
assert tk.task_headers.get(task_id) is not None
assert tk.removed_tasks.get(task_id2) is not None
assert tk.removed_tasks.get(task_id) is None
assert len(tk.supported_tasks) == 1
assert tk.supported_tasks[0] == task_id
def test_task_header_update(self):
e = Environment()
e.accept_tasks = True
tk = TaskHeaderKeeper(environments_manager=EnvironmentsManager(),
node=p2p.Node(),
min_price=10)
tk.environments_manager.add_environment(e)
task_header = get_task_header()
task_id = task_header.task_id
task_header.fixed_header.deadline = timeout_to_deadline(10)
task_header.fixed_header.update_checksum()
assert tk.add_task_header(task_header)
assert task_id in tk.supported_tasks
assert tk.add_task_header(task_header)
assert task_id in tk.supported_tasks
task_header = copy.deepcopy(task_header)
task_header.fixed_header.max_price = 1
task_header.fixed_header.update_checksum()
# An attempt to update fixed header should *not* succeed
assert not tk.add_task_header(task_header)
assert task_id in tk.supported_tasks
tk.task_headers = {}
tk.supported_tasks = []
assert tk.add_task_header(task_header)
assert task_id not in tk.supported_tasks
def get_dict_task_header(key_id_seed="kkk"):
key_id = str.encode(key_id_seed)
return {
'fixed_header': {
"task_id": idgenerator.generate_id(key_id),
"task_owner": {
"node_name": "Bob's node",
"key": encode_hex(key_id)[2:],
"pub_addr": "10.10.10.10",
"pub_port": 10101
},
"environment": "DEFAULT",
"last_checking": time.time(),
"deadline": timeout_to_deadline(1201),
"subtask_timeout": 120,
"max_price": 10,
"min_version": golem.__version__,
"resource_size": 0,
"estimated_memory": 0,
},
'mask': {
'byte_repr': Mask().to_bytes()
},
'timestamp': 0,
'signature': None
}
def query_extra_data(self,
perf_index: float,
num_cores: int = 1,
node_id: Optional[str] = None,
node_name: Optional[str] = None) -> Task.ExtraData:
""" Returns data for the next subtask. """
# create new subtask_id
subtask_id = idgenerator.generate_new_id_from_id(self.header.task_id)
with self._lock:
# assign a task
self.assigned_nodes[node_id] = subtask_id
self.assigned_subtasks[subtask_id] = node_id
# create subtask-specific data, 4 bits go for one char (hex digit)
data = random.getrandbits(self.task_params.subtask_data_size * 4)
self.subtask_ids.append(subtask_id)
self.subtask_data[subtask_id] = '%x' % data
subtask_def = ComputeTaskDef()
subtask_def['task_id'] = self.task_id
subtask_def['subtask_id'] = subtask_id
subtask_def['src_code'] = self.src_code
subtask_def['deadline'] = timeout_to_deadline(5 * 60)
subtask_def['extra_data'] = {
'data_file': self.shared_data_file,
'subtask_data': self.subtask_data[subtask_id],
'difficulty': self.task_params.difficulty,
'result_size': self.task_params.result_size,
'result_file': 'result.' + subtask_id[0:6]
}
return self.ExtraData(ctd=subtask_def)
def test_task_header_update(self):
e = Environment()
e.accept_tasks = True
tk = TaskHeaderKeeper(EnvironmentsManager(), 10)
tk.environments_manager.add_environment(e)
assert not tk.add_task_header(dict())
task_header = get_dict_task_header()
task_id = task_header["task_id"]
task_header["deadline"] = timeout_to_deadline(10)
assert tk.add_task_header(task_header)
assert task_id in tk.supported_tasks
assert tk.add_task_header(task_header)
assert task_id in tk.supported_tasks
task_header["max_price"] = 1
assert tk.add_task_header(task_header)
assert task_id not in tk.supported_tasks
tk.task_headers = {}
tk.supported_tasks = []
task_header["max_price"] = 1
assert tk.add_task_header(task_header)
assert task_id not in tk.supported_tasks
task_header['task_id'] = "newtaskID"
task_header['deadline'] = "WRONG DEADLINE"
assert not tk.add_task_header(task_header)
def __init__(self,
src_code,
task_definition,
node_name,
environment,
resource_size=0,
owner_address="",
owner_port=0,
owner_key_id="",
max_pending_client_results=MAX_PENDING_CLIENT_RESULTS):
"""Create more specific task implementation
"""
self.task_definition = task_definition
task_timeout = task_definition.full_task_timeout
deadline = timeout_to_deadline(task_timeout)
th = TaskHeader(
node_name=node_name,
task_id=task_definition.task_id,
task_owner_address=owner_address,
task_owner_port=owner_port,
task_owner_key_id=owner_key_id,
environment=environment,
task_owner=Node(),
deadline=deadline,
subtask_timeout=task_definition.subtask_timeout,
resource_size=resource_size,
estimated_memory=task_definition.estimated_memory,
max_price=task_definition.max_price,
docker_images=task_definition.docker_images,
)
Task.__init__(self, th, src_code)
self.task_resources = list()
self.total_tasks = 0
self.last_task = 0
self.num_tasks_received = 0
self.subtasks_given = {}
self.num_failed_subtasks = 0
self.full_task_timeout = task_timeout
self.counting_nodes = {}
self.root_path = None
self.stdout = {
} # for each subtask keep info about stdout received from computing node
self.stderr = {
} # for each subtask keep info about stderr received from computing node
self.results = {
} # for each subtask keep info about files containing results
self.res_files = {}
self.tmp_dir = None
self.verificator = self.VERIFICATOR_CLASS()
self.max_pending_client_results = max_pending_client_results
def _run_docker_task(self, render_task, timeout=60 * 5):
task_id = render_task.header.task_id
extra_data = render_task.query_extra_data(1.0)
ctd = extra_data.ctd
ctd.deadline = timeout_to_deadline(timeout)
# Create the computing node
self.node = OptNode(datadir=self.path)
self.node.client.start = Mock()
self.node.initialize()
ccd = ClientConfigDescriptor()
ccd.estimated_blender_performance = 2000.0
ccd.estimated_lux_performance = 2000.0
task_server = TaskServer(Mock(),
ccd,
Mock(),
self.node.client,
use_docker_machine_manager=False)
task_computer = task_server.task_computer
resource_dir = task_computer.resource_manager.get_resource_dir(task_id)
temp_dir = task_computer.resource_manager.get_temporary_dir(task_id)
self.dirs_to_remove.append(resource_dir)
self.dirs_to_remove.append(temp_dir)
# Copy the task resources
common_prefix = path.commonprefix(render_task.task_resources)
common_prefix = path.dirname(common_prefix)
for res_file in render_task.task_resources:
dest_file = path.join(resource_dir,
res_file[len(common_prefix) + 1:])
dest_dirname = path.dirname(dest_file)
if not path.exists(dest_dirname):
makedirs(dest_dirname)
shutil.copyfile(res_file, dest_file)
def send_task_failed(self_, subtask_id, task_id, error_msg, *args):
self.error_msg = error_msg
TaskServer.send_task_failed = send_task_failed
# Start task computation
task_computer.task_given(ctd)
result = task_computer.resource_given(ctd.task_id)
self.assertTrue(result)
# Thread for task computation should be created by now
task_thread = None
with task_computer.lock:
if task_computer.current_computations:
task_thread = task_computer.current_computations[0]
if task_thread:
task_thread.join(60.0)
return task_thread, self.error_msg, temp_dir
def _run_docker_local_comp_task(self, render_task, timeout=60 * 5):
render_task.deadline = timeout_to_deadline(timeout)
local_computer = LocalComputer(
render_task, self.tempdir, Mock(), Mock(),
render_task.query_extra_data_for_test_task)
local_computer.run()
local_computer.tt.join(60)
return local_computer.tt
def query_extra_data_for_advanced_verification(self, extra_data):
ctd = super(
BlenderVerificator,
self).query_extra_data_for_advanced_verification(extra_data)
ctd.subtask_id = str(random.getrandbits(128))
ctd.src_code = self.src_code
ctd.docker_images = self.docker_images
ctd.deadline = timeout_to_deadline(self.verification_timeout)
return ctd
def comp_task_info_keeping_timeout(
subtask_timeout: int,
resource_size: int,
num_of_res_transfers_needed: int = NUM_OF_RES_TRANSFERS_NEEDED_FOR_VER
):
verification_timeout = subtask_timeout
resource_timeout = helpers.maximum_download_time(resource_size).seconds
resource_timeout *= num_of_res_transfers_needed
return common.timeout_to_deadline(subtask_timeout + verification_timeout +
resource_timeout)
def test_task_limit(frozen_time, self): # pylint: disable=no-self-argument
tk = TaskHeaderKeeper(environments_manager=EnvironmentsManager(),
node=p2p.Node(),
min_price=10)
limit = tk.max_tasks_per_requestor
thd = get_task_header("ta")
thd.fixed_header.deadline = timeout_to_deadline(0.1)
tk.add_task_header(thd)
ids = [thd.task_id]
for i in range(1, limit):
thd = get_task_header("ta")
ids.append(thd.task_id)
tk.add_task_header(thd)
last_add_time = time.time()
for id_ in ids:
self.assertIn(id_, tk.task_headers)
thd = get_task_header("tb0")
tb_id = thd.task_id
tk.add_task_header(thd)
for id_ in ids:
self.assertIn(id_, tk.task_headers)
self.assertIn(tb_id, tk.task_headers)
while time.time() == last_add_time:
frozen_time.tick( # pylint: disable=no-member
delta=timedelta(milliseconds=100))
thd = get_task_header("ta")
new_task_id = thd.task_id
tk.add_task_header(thd)
self.assertNotIn(new_task_id, tk.task_headers)
for id_ in ids:
self.assertIn(id_, tk.task_headers)
self.assertIn(tb_id, tk.task_headers)
frozen_time.tick( # pylint: disable=no-member
delta=timedelta(milliseconds=100))
tk.remove_old_tasks()
thd = get_task_header("ta")
new_task_id = thd.task_id
tk.add_task_header(thd)
self.assertIn(new_task_id, tk.task_headers)
self.assertNotIn(ids[0], tk.task_headers)
for i in range(1, limit):
self.assertIn(ids[i], tk.task_headers)
self.assertIn(tb_id, tk.task_headers)
def _run_docker_local_comp_task(self, render_task, timeout=60 * 5):
render_task.deadline = timeout_to_deadline(timeout)
local_computer = LocalComputer(
root_path=self.tempdir,
success_callback=Mock(),
error_callback=Mock(),
get_compute_task_def=render_task.query_extra_data_for_test_task,
resources=render_task.task_resources)
local_computer.run()
local_computer.tt.join(60)
return local_computer.tt
def __init__(self, client_id, params, num_subtasks, public_key):
"""Creates a new dummy task
:param string client_id: client id
:param DummyTaskParameters params: task parameters
1024 hashes on average
"""
task_id = idgenerator.generate_id(public_key)
owner_address = ''
owner_port = 0
owner_key_id = encode_hex(public_key)[2:]
environment = self.ENVIRONMENT_NAME
header = TaskHeader(task_id=task_id,
environment=environment,
task_owner=Node(node_name=client_id,
pub_addr=owner_address,
pub_port=owner_port,
key=owner_key_id),
deadline=timeout_to_deadline(14400),
subtask_timeout=1200,
subtasks_count=num_subtasks,
resource_size=params.shared_data_size +
params.subtask_data_size,
estimated_memory=0,
max_price=MIN_PRICE)
# load the script to be run remotely from the file in the current dir
script_path = path.join(path.dirname(__file__), 'computation.py')
with open(script_path, 'r') as f:
src_code = f.read()
src_code += '\noutput = run_dummy_task(' \
'data_file, subtask_data, difficulty, result_size, tmp_path)'
from apps.dummy.task.dummytaskstate import DummyTaskDefinition
from apps.dummy.task.dummytaskstate import DummyTaskDefaults
task_definition = DummyTaskDefinition(DummyTaskDefaults())
Task.__init__(self, header, src_code, task_definition)
self.task_id = task_id
self.task_params = params
self.task_resources = []
self.resource_parts = {}
self.shared_data_file = None
self.subtasks_count = num_subtasks
self.total_tasks = self.subtasks_count
self.subtask_ids = []
self.subtask_data = {}
self.subtask_results = {}
self.assigned_nodes = {}
self.assigned_subtasks = {}
self.total_tasks = 1
self._lock = Lock()
def _new_compute_task_def(self, hash, extra_data, working_directory,
perf_index):
ctd = ComputeTaskDef()
ctd.task_id = self.header.task_id
ctd.subtask_id = hash
ctd.extra_data = extra_data
ctd.short_description = self._short_extra_data_repr(
perf_index, extra_data)
ctd.src_code = self.src_code
ctd.performance = perf_index
ctd.working_directory = working_directory
ctd.docker_images = self.header.docker_images
ctd.deadline = timeout_to_deadline(self.header.subtask_timeout)
return ctd
def _get_task_header(self, task_id, timeout, subtask_timeout):
return TaskHeader(
node_name="test_node_%s" % (self.test_nonce, ),
task_id=task_id,
task_owner_address="task_owner_address_%s" % (self.test_nonce, ),
task_owner_port="task_owner_port_%s" % (self.test_nonce, ),
task_owner_key_id="task_owner_key_id_%s" % (self.test_nonce, ),
environment="test_environ_%s" % (self.test_nonce, ),
resource_size=2 * 1024,
estimated_memory=3 * 1024,
max_price=10000,
deadline=timeout_to_deadline(timeout),
subtask_timeout=subtask_timeout,
)
def test_old_tasks(self):
tk = TaskHeaderKeeper(EnvironmentsManager(), 10)
e = Environment()
e.accept_tasks = True
tk.environments_manager.add_environment(e)
task_header = get_dict_task_header()
task_header["deadline"] = timeout_to_deadline(10)
assert tk.add_task_header(task_header)
task_header["deadline"] = timeout_to_deadline(1)
task_header["task_id"] = "abc"
assert tk.add_task_header(task_header)
assert tk.task_headers.get("abc") is not None
assert tk.task_headers.get("xyz") is not None
assert tk.removed_tasks.get("abc") is None
assert tk.removed_tasks.get("xyz") is None
assert len(tk.supported_tasks) == 2
time.sleep(1.1)
tk.remove_old_tasks()
assert tk.task_headers.get("abc") is None
assert tk.task_headers.get("xyz") is not None
assert tk.removed_tasks.get("abc") is not None
assert tk.removed_tasks.get("xyz") is None
assert len(tk.supported_tasks) == 1
assert tk.supported_tasks[0] == "xyz"
def generate_computational_task_definition(subtask_info: Dict[str, Any],
script_src: str) \
-> Dict[str, Any]:
task_definition = deepcopy(subtask_info['ctd'])
task_definition['extra_data']['outfilebasename'] = \
"ref_" + subtask_info['outfilebasename']
task_definition['extra_data']['script_src'] = script_src
task_definition['deadline'] = timeout_to_deadline(
subtask_info['subtask_timeout'])
return task_definition
def get_dict_task_header():
return {
"task_id": "xyz",
"node_name": "ABC",
"task_owner": dict(),
"task_owner_address": "10.10.10.10",
"task_owner_port": 10101,
"task_owner_key_id": "kkkk",
"environment": "DEFAULT",
"last_checking": time.time(),
"deadline": timeout_to_deadline(1201),
"subtask_timeout": 120,
"max_price": 10,
"min_version": APP_VERSION
}
def get_example_task_header():
return {
"task_id": "uvw",
"node_name": "ABC",
"environment": "DEFAULT",
"task_owner": dict(),
"task_owner_port": 10101,
"task_owner_key_id": "key",
"task_owner_address": "10.10.10.10",
"deadline": timeout_to_deadline(1201),
"subtask_timeout": 120,
"max_price": 20,
"resource_size": 2 * 1024,
"estimated_memory": 3 * 1024,
"signature": None,
"min_version": APP_VERSION
}
def test_timeout():
subtask_info = {}
subtask_info['subtask_id'] = 'deadbeef'
self.queue.submit(
functools.partial(BlenderVerifier,
cropper_cls=BlenderReferenceGenerator,
docker_task_cls=DockerTaskThread),
"deadbeef",
timeout_to_deadline(10),
lambda subtask_id, verdict, result: subtask_id,
subtask_info=subtask_info,
results=[],
resources=[],
reference_data=[]
)
def header(self, max_price, last_checking=None,
deadline=None, min_version="4"):
if not last_checking:
last_checking = time.time()
if not deadline:
deadline = timeout_to_deadline(36000)
ret = TaskHeader(
task_id=str(uuid4()),
environment="DEFAULT",
task_owner=Node(),
max_price=max_price,
deadline=deadline,
min_version=min_version)
if last_checking:
ret.last_checking = last_checking
return ret
def __init__(self, client_id, params, num_subtasks):
"""Creates a new dummy task
:param string client_id: client id
:param DummyTaskParameters params: task parameters
1024 hashes on average
"""
task_id = SimpleAuth.generate_uuid().get_hex()
owner_address = ''
owner_port = 0
owner_key_id = ''
environment = self.ENVIRONMENT_NAME
header = TaskHeader(
client_id, task_id,
owner_address, owner_port, owner_key_id, environment,
task_owner=Node(),
deadline=timeout_to_deadline(14400),
subtask_timeout=1200,
resource_size=params.shared_data_size + params.subtask_data_size,
estimated_memory=0,
max_price=MIN_PRICE)
# load the script to be run remotely from the file in the current dir
script_path = path.join(path.dirname(__file__), 'computation.py')
with open(script_path, 'r') as f:
src_code = f.read()
src_code += '\noutput = run_dummy_task(' \
'data_file, subtask_data, difficulty, result_size)'
Task.__init__(self, header, src_code)
self.task_id = task_id
self.task_params = params
self.task_resources = []
self.resource_parts = {}
self.shared_data_file = None
self.total_subtasks = num_subtasks
self.subtask_ids = []
self.subtask_data = {}
self.subtask_results = {}
self.assigned_nodes = {}
self.assigned_subtasks = {}
self._lock = Lock()
def query_extra_data(self, perf_index, num_cores=1, node_id=None, node_name=None):
"""Returns data for the next subtask.
:param int perf_index:
:param int num_cores:
:param str | None node_id:
:param str | None node_name:
:rtype: ComputeTaskDef"""
# create new subtask_id
subtask_id = uuid.uuid4().get_hex()
with self._lock:
# check if a task has been assigned to this node
if node_id in self.assigned_nodes:
return self.ExtraData(should_wait=True)
# assign a task
self.assigned_nodes[node_id] = subtask_id
self.assigned_subtasks[subtask_id] = node_id
# create subtask-specific data, 4 bits go for one char (hex digit)
data = random.getrandbits(self.task_params.subtask_data_size * 4)
self.subtask_ids.append(subtask_id)
self.subtask_data[subtask_id] = '%x' % data
subtask_def = ComputeTaskDef()
subtask_def.task_id = self.task_id
subtask_def.subtask_id = subtask_id
subtask_def.src_code = self.src_code
subtask_def.task_owner = self.header.task_owner
subtask_def.environment = self.header.environment
subtask_def.return_address = self.header.task_owner_address
subtask_def.return_port = self.header.task_owner_port
subtask_def.deadline = timeout_to_deadline(5 * 60)
subtask_def.extra_data = {
'data_file': self.shared_data_file,
'subtask_data': self.subtask_data[subtask_id],
'difficulty': self.task_params.difficulty,
'result_size': self.task_params.result_size,
'result_file': 'result.' + subtask_id[0:6]
}
return self.ExtraData(ctd=subtask_def)
def __get_merge_ctd(self, files):
script_file = dirmanager.find_task_script(APP_DIR,
"docker_luxmerge.py")
if script_file is None:
logger.error("Cannot find merger script")
return
with open(script_file) as f:
src_code = f.read()
ctd = ComputeTaskDef()
ctd.task_id = self.header.task_id
ctd.subtask_id = self.header.task_id
ctd.extra_data = {'output_flm': self.output_file, 'flm_files': files}
ctd.src_code = src_code
ctd.working_directory = "."
ctd.docker_images = self.header.docker_images
ctd.deadline = timeout_to_deadline(self.merge_timeout)
return ctd
def _get_task_mock(self,
task_id="xyz",
subtask_id="xxyyzz",
timeout=120.0,
subtask_timeout=120.0):
header = self._get_task_header(task_id, timeout, subtask_timeout)
task_mock = TaskMock(header, src_code='')
ctd = ComputeTaskDef()
ctd.task_id = task_id
ctd.subtask_id = subtask_id
ctd.environment = "DEFAULT"
ctd.deadline = timeout_to_deadline(subtask_timeout)
task_mock.query_extra_data_return_value = Task.ExtraData(
should_wait=False, ctd=ctd)
Task.get_progress = Mock()
task_mock.get_progress.return_value = 0.3
return task_mock
def get_example_task_header(key_id):
return {
"fixed_header": {
"task_id":
idgenerator.generate_id(key_id),
"environment":
"DEFAULT",
"task_owner":
dict(key=encode_hex(key_id)[2:],
node_name="ABC",
prv_port=40103,
prv_addr='10.0.0.10',
pub_port=40103,
pub_addr='1.2.3.4'),
"deadline":
timeout_to_deadline(1201),
"subtask_timeout":
120,
"max_price":
20,
"resource_size":
2 * 1024,
"estimated_memory":
3 * 1024,
"signature":
None,
"min_version":
golem.__version__,
"subtasks_count":
21,
"concent_enabled":
False,
},
"mask": {
"byte_repr": Mask().to_bytes()
},
"timestamp": 0,
}
def test_to_dictionary():
ss = SubtaskState()
ss.subtask_definition = "My long task definition"
ss.subtask_id = "ABCDEF"
ss.subtask_progress = 0.92
ss.time_started = get_timestamp_utc()
ss.deadline = timeout_to_deadline(ss.time_started + 5)
ss.extra_data = {"param1": 1323, "param2": "myparam"}
ss.subtask_rem_time = deadline_to_timeout(
ss.deadline) - ss.time_started
ss.subtask_status = SubtaskStatus.starting
ss.value = 138
ss.stdout = "path/to/file"
ss.stderr = "path/to/file2"
ss.results = ["path/to/file3", "path/to/file4"]
ss.computation_time = 130
ss.node_id = "NODE1"
ss_dict = ss.to_dictionary()
assert ss_dict['description'] == "My long task definition"
assert ss_dict['subtask_id'] == "ABCDEF"
assert ss_dict['progress'] == 0.92
assert ss_dict['time_started'] == get_timestamp_utc()
assert ss_dict.get('deadline') is None
assert ss_dict.get('extra_data') is None
assert ss_dict['time_remaining'] == 5
assert ss_dict['status'] == SubtaskStatus.starting.value
assert ss_dict.get('value') is None
assert ss_dict['stdout'] == "path/to/file"
assert ss_dict['stderr'] == "path/to/file2"
assert ss_dict['results'] == ["path/to/file3", "path/to/file4"]
assert ss_dict.get('computation_time') is None
assert ss_dict['node_id'] == "NODE1"
def setUp(self):
self.key_id = b'key_id'
self.th_dict_repr = {
'mask': None,
'fixed_header': {
"task_id": idgenerator.generate_id(self.key_id),
"task_owner": {
"node_name": "Bob's node",
"key": encode_hex(self.key_id),
"pub_addr": "10.10.10.10",
"pub_port": 10101
},
"environment": "DEFAULT",
"last_checking": time.time(),
"deadline": timeout_to_deadline(1201),
"subtask_timeout": 120,
"subtasks_count": 21,
"max_price": 10,
"min_version": golem.__version__,
"resource_size": 0,
"estimated_memory": 0,
}
}
def test_receive_subtask_problems(self):
ctk = CompTaskKeeper(Path(self.path), False)
th = get_task_header()
task_id = th.task_id
price_bid = 5
ctk.add_request(th, price_bid)
subtask_id = idgenerator.generate_new_id_from_id(task_id)
ctd = ComputeTaskDef()
ctd['task_id'] = task_id
ctd['subtask_id'] = subtask_id
ctd['deadline'] = timeout_to_deadline(th.subtask_timeout - 1)
price = taskkeeper.compute_subtask_value(
price_bid,
th.subtask_timeout,
)
ttc = msg_factories.tasks.TaskToComputeFactory(price=price)
ttc.compute_task_def = ctd
self.assertTrue(ctk.receive_subtask(ttc))
assert ctk.active_tasks[task_id].requests == 0
assert ctk.subtask_to_task[subtask_id] == task_id
assert ctk.check_task_owner_by_subtask(th.task_owner.key, subtask_id)
assert not ctk.check_task_owner_by_subtask(th.task_owner.key, "!!!")
assert not ctk.check_task_owner_by_subtask('???', subtask_id)
subtask_id2 = idgenerator.generate_new_id_from_id(task_id)
ctd2 = ComputeTaskDef()
ctd2['task_id'] = task_id
ctd2['subtask_id'] = subtask_id2
ttc.compute_task_def = ctd2
self.assertFalse(ctk.receive_subtask(ttc))
assert ctk.active_tasks[task_id].requests == 0
assert ctk.subtask_to_task.get(subtask_id2) is None
assert ctk.subtask_to_task[subtask_id] == task_id
ctk.active_tasks[task_id].requests = 1
ttc.compute_task_def = ctd
self.assertFalse(ctk.receive_subtask(ttc))
assert ctk.active_tasks[task_id].requests == 1
