def calculate_subtask_verification_time(
report_computed_task: message.ReportComputedTask) -> int:
"""
This function calls `subtask_verification_time` helper function from golem messages or Concent custom implementation
of it, depending on CUSTOM_PROTOCOL_TIMES setting value.
The reason for using custom implementation is because it has hard-coded values and we cannot make it use values
from our settings.
"""
assert isinstance(report_computed_task, message.ReportComputedTask)
if settings.CUSTOM_PROTOCOL_TIMES:
mdt = maximum_download_time(size=report_computed_task.size, )
ttc_dt = datetime.datetime.utcfromtimestamp(
report_computed_task.task_to_compute.timestamp, )
subtask_dt = datetime.datetime.utcfromtimestamp(
report_computed_task.task_to_compute.compute_task_def['deadline'],
)
subtask_timeout = subtask_dt - ttc_dt
return int((4 * settings.CONCENT_MESSAGING_TIME) +
(3 * mdt.total_seconds()) +
(0.5 * subtask_timeout.total_seconds()))
else:
return int(
subtask_verification_time(report_computed_task).total_seconds())
def comp_task_info_keeping_timeout(
subtask_timeout: int,
resource_size: int,
num_of_res_transfers_needed: int = NUM_OF_RES_TRANSFERS_NEEDED_FOR_VER
):
verification_timeout = subtask_timeout
resource_timeout = helpers.maximum_download_time(resource_size).seconds
resource_timeout *= num_of_res_transfers_needed
return common.timeout_to_deadline(subtask_timeout + verification_timeout +
resource_timeout)
def _download_params(cls, content_hash, client_options, **kwargs):
path = kwargs['filepath']
peers, size, timeout = None, None, None
if client_options:
size = client_options.get(cls.CLIENT_ID, cls.VERSION, 'size')
if size:
timeout = msg_helpers.maximum_download_time(size).seconds
else:
timeout = None
peers = client_options.peers
return dict(command='download',
hash=content_hash,
dest=path,
peers=peers or [],
size=size,
timeout=timeout)
def test_that_both_maximum_download_time_implementation_should_return_same_result_when_golem_messages_constants_match_concent_settings(self):
for size, rate in [
(7, 19),
(10, 10),
(19, 7),
(100, 100),
(331, 331),
(1000, 1000),
(9999, 9999),
]:
self.assertEqual(
calculate_maximum_download_time(size, rate),
int(helpers.maximum_download_time(size, rate).total_seconds())
)
self.assertEqual(
calculate_maximum_download_time(size, settings.MINIMUM_UPLOAD_RATE),
calculate_maximum_download_time(size, constants.DEFAULT_UPLOAD_RATE),
)
def calculate_maximum_download_time(size: int, rate: int) -> int:
"""
This function calls `maximum_download_time` helper function from golem messages or Concent custom implementation
of it, depending on CUSTOM_PROTOCOL_TIMES setting value.
The reason for using custom implementation is because it has hard-coded values and we cannot make it use values
from our settings.
"""
assert isinstance(size, int)
assert isinstance(rate, int)
assert size > 0
assert rate > 0
assert settings.DOWNLOAD_LEADIN_TIME > 0
if settings.CUSTOM_PROTOCOL_TIMES:
bytes_per_sec = rate << 10
download_time = int(
datetime.timedelta(
seconds=int(math.ceil(size / bytes_per_sec))).total_seconds())
return settings.DOWNLOAD_LEADIN_TIME + download_time
else:
return int(maximum_download_time(size, rate).total_seconds())
def test_maximum_download_time_w_rate(self):
size = 10240 << 10 # 10M
rate = 1024 # KB/s
expected = datetime.timedelta(seconds=10) + DOWNLOAD_LEADIN_TIME
self.assertEqual(expected, helpers.maximum_download_time(size, rate))
def test_maximum_download_time(self):
secs = 100
size = secs * (DEFAULT_UPLOAD_RATE << 10)
expected = datetime.timedelta(seconds=secs) + DOWNLOAD_LEADIN_TIME
self.assertEqual(expected, helpers.maximum_download_time(size))
def computed_task_reported(task_server,
report_computed_task,
after_success=lambda: None,
after_error=lambda: None):
task_manager = task_server.task_manager
concent_service = task_server.client.concent_service
task = task_manager.tasks.get(report_computed_task.task_id, None)
output_dir = task.tmp_dir if hasattr(task, 'tmp_dir') else None
client_options = task_server.get_download_options(
report_computed_task.options,
report_computed_task.task_id,
)
fgtr = message.concents.ForceGetTaskResult(
report_computed_task=report_computed_task)
# submit a delayed `ForceGetTaskResult` to the Concent
# in case the download exceeds the maximum allowable download time.
# however, if it succeeds, the message will get cancelled
# in the success handler
concent_service.submit_task_message(
report_computed_task.subtask_id,
fgtr,
msg_helpers.maximum_download_time(report_computed_task.size, ),
)
# Pepare callbacks for received resources
def on_success(extracted_pkg, *_args, **_kwargs):
logger.debug("Task result extracted %r", extracted_pkg.__dict__)
concent_service.cancel_task_message(
report_computed_task.subtask_id,
'ForceGetTaskResult',
)
task_server.verify_results(
report_computed_task=report_computed_task,
extracted_package=extracted_pkg,
)
after_success()
def on_error(exc, *_args, **_kwargs):
logger.warning(
"Task result error: %s (%s)",
report_computed_task.subtask_id,
exc or "unspecified",
)
if report_computed_task.task_to_compute.concent_enabled:
# we're resorting to mediation through the Concent
# to obtain the task results
logger.debug('[CONCENT] sending ForceGetTaskResult: %s', fgtr)
concent_service.submit_task_message(
report_computed_task.subtask_id,
fgtr,
)
after_error()
# Actually request results
task_manager.task_result_incoming(report_computed_task.subtask_id)
task_manager.task_result_manager.pull_package(
report_computed_task.multihash,
report_computed_task.task_id,
report_computed_task.subtask_id,
report_computed_task.secret,
success=on_success,
error=on_error,
client_options=client_options,
output_dir=output_dir)
def _precalculate_subtask_verification_time(concent_messaging_time):
return ((4 * concent_messaging_time) + (3 * int(
maximum_download_time(REPORT_COMPUTED_TASK_SIZE).total_seconds())))