def computation_failed(self, subtask_id):
CoreTask.computation_failed(self, subtask_id)
if self.use_frames:
self._update_frame_task_preview()
self._update_subtask_frame_status(subtask_id)
else:
self._update_task_preview()
def _get_mask_for_task(client, task: coretask.CoreTask) -> masking.Mask:
desired_num_workers = max(
task.get_total_tasks() * client.config_desc.initial_mask_size_factor,
client.config_desc.min_num_workers_for_mask,
)
if client.p2pservice is None:
raise RuntimeError('P2PService not ready')
if client.task_server is None:
raise RuntimeError('TaskServer not ready')
network_size = client.p2pservice.get_estimated_network_size()
min_perf = client.task_server.get_min_performance_for_task(task)
perf_rank = client.p2pservice.get_performance_percentile_rank(
min_perf, task.header.environment)
potential_num_workers = int(network_size * (1 - perf_rank))
mask = masking.Mask.get_mask_for_task(
desired_num_workers=desired_num_workers,
potential_num_workers=potential_num_workers)
logger.info(f'Task {task.header.task_id} '
f'initial mask size: {mask.num_bits} '
f'expected number of providers: {desired_num_workers} '
f'potential number of providers: {potential_num_workers}')
return mask
def test_instantiation(self):
task_def = self._get_core_task_definition()
node = Node()
# abstract class cannot be instantiated
# pylint: disable=abstract-class-instantiated
with self.assertRaises(TypeError):
CoreTask(task_def, owner=Node(node_name="ABC"))
class CoreTaskDeabstacted(CoreTask):
def query_extra_data(self, *args, **kwargs):
pass
def short_extra_data_repr(self, extra_data):
pass
# ENVIRONMENT has to be set
with self.assertRaises(TypeError):
CoreTaskDeabstacted(task_def, node)
class CoreTaskDeabstractedEnv(CoreTask):
ENVIRONMENT_CLASS = MagicMock()
def query_extra_data(self, *args, **kwargs):
pass
def short_extra_data_repr(self, extra_data):
pass
def query_extra_data_for_test_task(self):
pass
task = CoreTaskDeabstractedEnv(task_def, node)
self.assertIsInstance(task, CoreTask)
def test_create_subtask_id(self):
# given
t = self._get_core_task()
t.header.task_id = CoreTask.create_task_id(b'\xbe\xef\xde\xad\xbe\xef')
# when
subtask_id = t.create_subtask_id()
# then
self.assertRegex(subtask_id, "^[-0-9a-f]{23}-beefdeadbeef$")
def _get_core_task(self):
task_definition = TaskDefinition()
task_definition.max_price = 100
task_definition.task_id = "xyz"
task_definition.estimated_memory = 1024
task_definition.full_task_timeout = 3000
task_definition.subtask_timeout = 30
task = CoreTask(
src_code="src code",
task_definition=task_definition,
node_name="ABC",
owner_address="10.10.10.10",
owner_port=123,
owner_key_id="key",
environment="environment",
resource_size=1024,
)
dm = DirManager(self.path)
task.initialize(dm)
return task
class CoreTaskDeabstracted(CoreTask):
ENVIRONMENT_CLASS = MagicMock()
EXTRA_DATA = CoreTask.ExtraData(sth="sth")
def query_extra_data(self, *args, **kwargs):
return self.EXTRA_DATA
def short_extra_data_repr(self, extra_data):
pass
def query_extra_data_for_test_task(self):
pass
def __init__(self, task_definition, total_tasks, root_path, owner):
CoreTask.__init__(self,
task_definition=task_definition,
owner=owner,
root_path=root_path,
total_tasks=total_tasks)
if task_definition.docker_images is None:
task_definition.docker_images = self.environment.docker_images
self.main_scene_file = task_definition.main_scene_file
self.main_scene_dir = str(Path(task_definition.main_scene_file).parent)
self.outfilebasename = Path(task_definition.output_file).stem
self.output_file = task_definition.output_file
self.output_format = task_definition.output_format
self.res_x, self.res_y = task_definition.resolution
self.preview_file_path = None
self.preview_task_file_path = None
self.collected_file_names = {}
preview_x = PREVIEW_X
preview_y = PREVIEW_Y
if self.res_x != 0 and self.res_y != 0:
if self.res_x / self.res_y > preview_x / preview_y:
self.scale_factor = preview_x / self.res_x
else:
self.scale_factor = preview_y / self.res_y
self.scale_factor = min(1.0, self.scale_factor)
else:
self.scale_factor = 1.0
self.test_task_res_path = None
def create_task(self, dictionary, minimal=False):
purpose = TaskPurpose.TESTING if minimal else TaskPurpose.REQUESTING
type_name = dictionary['type'].lower()
compute_on = dictionary.get('compute_on', 'cpu').lower()
is_requesting = purpose == TaskPurpose.REQUESTING
if type_name == "blender" and is_requesting and compute_on == "gpu":
type_name = type_name + "_nvgpu"
task_type = self.task_types[type_name].for_purpose(purpose)
builder_type = task_type.task_builder_type
definition = builder_type.build_definition(task_type, dictionary,
minimal)
definition.task_id = CoreTask.create_task_id(self.keys_auth.public_key)
definition.concent_enabled = dictionary.get('concent_enabled', False)
builder = builder_type(self.node, definition, self.dir_manager)
return builder.build()
def __build_tasks(self, tm, n):
tm.tasks = dict()
tm.tasks_states = dict()
tm.subtask_states = dict()
task_id = None
subtask_id = None
previews = [None, 'result', ['result_1', 'result_2']]
for i in xrange(0, n):
task_id = str(uuid.uuid4())
definition = Mock()
definition.task_id = task_id
definition.task_type = "blender"
definition.subtask_timeout = 3671
definition.full_task_timeout = 3671 * 10
definition.max_price = 1 * 10**18
definition.resolution = [1920, 1080]
definition.resources = [str(uuid.uuid4()) for _ in range(5)]
definition.output_file = os.path.join(self.tempdir, 'somefile')
definition.options.frames = [0]
subtask_states, subtask_id = self.__build_subtasks(n)
state = TaskState()
state.status = 'waiting'
state.remaining_time = 100 - i
state.extra_data = dict(result_preview=previews[i % 3])
state.subtask_states = subtask_states
task = CoreTask('source.code', definition, 'node', 'blender')
task.get_total_tasks = Mock()
task.get_progress = Mock()
task.get_preview = Mock()
task.get_total_tasks.return_value = i + 2
task.get_progress.return_value = i * 10
task.subtask_states = subtask_states
tm.tasks[task_id] = task
tm.tasks_states[task_id] = state
tm.subtask_states.update(subtask_states)
tm.subtask2task_mapping = self.__build_subtask2task(tm.tasks)
return task_id, subtask_id
def __init__(self,
node_name,
task_definition,
total_tasks,
root_path,
owner_address="",
owner_port=0,
owner_key_id=""):
environment = self.ENVIRONMENT_CLASS()
if task_definition.docker_images is None:
task_definition.docker_images = environment.docker_images
main_program_file = environment.main_program_file
try:
with open(main_program_file, "r") as src_file:
src_code = src_file.read()
except IOError as err:
logger.warning("Wrong main program file: {}".format(err))
src_code = ""
self.main_program_file = main_program_file
resource_size = 0
task_resources = set(filter(os.path.isfile, task_definition.resources))
for resource in task_resources:
resource_size += os.stat(resource).st_size
CoreTask.__init__(self,
src_code=src_code,
task_definition=task_definition,
node_name=node_name,
owner_address=owner_address,
owner_port=owner_port,
owner_key_id=owner_key_id,
environment=environment.get_id(),
resource_size=resource_size)
self.main_scene_file = task_definition.main_scene_file
self.main_scene_dir = str(Path(task_definition.main_scene_file).parent)
if isinstance(task_definition.output_file, unicode):
task_definition.output_file = task_definition.output_file.encode(
'utf-8', 'replace')
self.outfilebasename = Path(task_definition.output_file).stem
self.output_file = task_definition.output_file
self.output_format = task_definition.output_format
self.total_tasks = total_tasks
self.res_x, self.res_y = task_definition.resolution
self.root_path = root_path
self.preview_file_path = None
self.preview_task_file_path = None
self.task_resources = deepcopy(list(task_resources))
self.collected_file_names = {}
preview_x = 300
preview_y = 200
if self.res_x != 0 and self.res_y != 0:
if self.res_x / self.res_y > preview_x / preview_y:
self.scale_factor = preview_x / self.res_x
else:
self.scale_factor = preview_y / self.res_y
self.scale_factor = min(1.0, self.scale_factor)
else:
self.scale_factor = 1.0
self.test_task_res_path = None
self.verificator.res_x = self.res_x
self.verificator.res_y = self.res_y
self.verificator.total_tasks = self.total_tasks
self.verificator.root_path = self.root_path
def restart_subtask(self, subtask_id):
if self.subtasks_given[subtask_id]['status'] == SubtaskStatus.finished:
self._remove_from_preview(subtask_id)
CoreTask.restart_subtask(self, subtask_id)
def computation_failed(self, subtask_id):
CoreTask.computation_failed(self, subtask_id)
self._update_task_preview()
def test_create_task_id(self):
# when
task_id = CoreTask.create_task_id(b'\xbe\xef\xde\xad\xbe\xef')
# then
self.assertRegex(task_id, "^[-0-9a-f]{23}-beefdeadbeef$")