def calculate_subtask_verification_time( report_computed_task: message.ReportComputedTask) -> int: """ This function calls `subtask_verification_time` helper function from golem messages or Concent custom implementation of it, depending on CUSTOM_PROTOCOL_TIMES setting value. The reason for using custom implementation is because it has hard-coded values and we cannot make it use values from our settings. """ assert isinstance(report_computed_task, message.ReportComputedTask) if settings.CUSTOM_PROTOCOL_TIMES: mdt = maximum_download_time(size=report_computed_task.size, ) ttc_dt = datetime.datetime.utcfromtimestamp( report_computed_task.task_to_compute.timestamp, ) subtask_dt = datetime.datetime.utcfromtimestamp( report_computed_task.task_to_compute.compute_task_def['deadline'], ) subtask_timeout = subtask_dt - ttc_dt return int((4 * settings.CONCENT_MESSAGING_TIME) + (3 * mdt.total_seconds()) + (0.5 * subtask_timeout.total_seconds())) else: return int( subtask_verification_time(report_computed_task).total_seconds())
def comp_task_info_keeping_timeout( subtask_timeout: int, resource_size: int, num_of_res_transfers_needed: int = NUM_OF_RES_TRANSFERS_NEEDED_FOR_VER ): verification_timeout = subtask_timeout resource_timeout = helpers.maximum_download_time(resource_size).seconds resource_timeout *= num_of_res_transfers_needed return common.timeout_to_deadline(subtask_timeout + verification_timeout + resource_timeout)
def _download_params(cls, content_hash, client_options, **kwargs): path = kwargs['filepath'] peers, size, timeout = None, None, None if client_options: size = client_options.get(cls.CLIENT_ID, cls.VERSION, 'size') if size: timeout = msg_helpers.maximum_download_time(size).seconds else: timeout = None peers = client_options.peers return dict(command='download', hash=content_hash, dest=path, peers=peers or [], size=size, timeout=timeout)
def test_that_both_maximum_download_time_implementation_should_return_same_result_when_golem_messages_constants_match_concent_settings(self): for size, rate in [ (7, 19), (10, 10), (19, 7), (100, 100), (331, 331), (1000, 1000), (9999, 9999), ]: self.assertEqual( calculate_maximum_download_time(size, rate), int(helpers.maximum_download_time(size, rate).total_seconds()) ) self.assertEqual( calculate_maximum_download_time(size, settings.MINIMUM_UPLOAD_RATE), calculate_maximum_download_time(size, constants.DEFAULT_UPLOAD_RATE), )
def calculate_maximum_download_time(size: int, rate: int) -> int: """ This function calls `maximum_download_time` helper function from golem messages or Concent custom implementation of it, depending on CUSTOM_PROTOCOL_TIMES setting value. The reason for using custom implementation is because it has hard-coded values and we cannot make it use values from our settings. """ assert isinstance(size, int) assert isinstance(rate, int) assert size > 0 assert rate > 0 assert settings.DOWNLOAD_LEADIN_TIME > 0 if settings.CUSTOM_PROTOCOL_TIMES: bytes_per_sec = rate << 10 download_time = int( datetime.timedelta( seconds=int(math.ceil(size / bytes_per_sec))).total_seconds()) return settings.DOWNLOAD_LEADIN_TIME + download_time else: return int(maximum_download_time(size, rate).total_seconds())
def test_maximum_download_time_w_rate(self): size = 10240 << 10 # 10M rate = 1024 # KB/s expected = datetime.timedelta(seconds=10) + DOWNLOAD_LEADIN_TIME self.assertEqual(expected, helpers.maximum_download_time(size, rate))
def test_maximum_download_time(self): secs = 100 size = secs * (DEFAULT_UPLOAD_RATE << 10) expected = datetime.timedelta(seconds=secs) + DOWNLOAD_LEADIN_TIME self.assertEqual(expected, helpers.maximum_download_time(size))
def computed_task_reported(task_server, report_computed_task, after_success=lambda: None, after_error=lambda: None): task_manager = task_server.task_manager concent_service = task_server.client.concent_service task = task_manager.tasks.get(report_computed_task.task_id, None) output_dir = task.tmp_dir if hasattr(task, 'tmp_dir') else None client_options = task_server.get_download_options( report_computed_task.options, report_computed_task.task_id, ) fgtr = message.concents.ForceGetTaskResult( report_computed_task=report_computed_task) # submit a delayed `ForceGetTaskResult` to the Concent # in case the download exceeds the maximum allowable download time. # however, if it succeeds, the message will get cancelled # in the success handler concent_service.submit_task_message( report_computed_task.subtask_id, fgtr, msg_helpers.maximum_download_time(report_computed_task.size, ), ) # Pepare callbacks for received resources def on_success(extracted_pkg, *_args, **_kwargs): logger.debug("Task result extracted %r", extracted_pkg.__dict__) concent_service.cancel_task_message( report_computed_task.subtask_id, 'ForceGetTaskResult', ) task_server.verify_results( report_computed_task=report_computed_task, extracted_package=extracted_pkg, ) after_success() def on_error(exc, *_args, **_kwargs): logger.warning( "Task result error: %s (%s)", report_computed_task.subtask_id, exc or "unspecified", ) if report_computed_task.task_to_compute.concent_enabled: # we're resorting to mediation through the Concent # to obtain the task results logger.debug('[CONCENT] sending ForceGetTaskResult: %s', fgtr) concent_service.submit_task_message( report_computed_task.subtask_id, fgtr, ) after_error() # Actually request results task_manager.task_result_incoming(report_computed_task.subtask_id) task_manager.task_result_manager.pull_package( report_computed_task.multihash, report_computed_task.task_id, report_computed_task.subtask_id, report_computed_task.secret, success=on_success, error=on_error, client_options=client_options, output_dir=output_dir)
def _precalculate_subtask_verification_time(concent_messaging_time): return ((4 * concent_messaging_time) + (3 * int( maximum_download_time(REPORT_COMPUTED_TASK_SIZE).total_seconds())))