def setUp(self):
super().setUp()
deadline_offset = 10
message_timestamp = get_current_utc_timestamp() + deadline_offset
compute_task_def = message.ComputeTaskDef()
compute_task_def['task_id'] = '8'
compute_task_def['subtask_id'] = '8'
compute_task_def['deadline'] = message_timestamp
task_to_compute = tasks.TaskToComputeFactory(
compute_task_def=compute_task_def,
requestor_public_key=encode_hex(REQUESTOR_PUBLIC_KEY),
provider_public_key=encode_hex(PROVIDER_PUBLIC_KEY),
price=0,
)
task_to_compute = load(
dump(
task_to_compute,
REQUESTOR_PRIVATE_KEY,
settings.CONCENT_PUBLIC_KEY,
),
settings.CONCENT_PRIVATE_KEY,
REQUESTOR_PUBLIC_KEY,
check_time=False,
)
report_computed_task = tasks.ReportComputedTaskFactory(
task_to_compute=task_to_compute)
self.force_report_computed_task = message.ForceReportComputedTask(
report_computed_task=report_computed_task)
def test_that_scheduling_task_for_subtask_after_deadline_should_process_timeout(
self):
datetime = parse_timestamp_to_utc_datetime(
get_current_utc_timestamp() + settings.CONCENT_MESSAGING_TIME + 1)
with freeze_time(datetime):
with mock.patch('core.tasks.base.make_force_payment_to_provider',
autospec=True) as payment_function_mock:
upload_finished(self.subtask.subtask_id) # pylint: disable=no-value-for-parameter
self.subtask.refresh_from_db()
self.assertEqual(self.subtask.state_enum, Subtask.SubtaskState.FAILED)
self.assertEqual(self.subtask.next_deadline, None)
self.assertEqual(PendingResponse.objects.count(), 2)
self.assertTrue(
PendingResponse.objects.filter(
client=self.subtask.provider).exists())
self.assertTrue(
PendingResponse.objects.filter(
client=self.subtask.requestor).exists())
payment_function_mock.assert_called_once_with(
requestor_eth_address=self.task_to_compute.
requestor_ethereum_address,
provider_eth_address=self.task_to_compute.
provider_ethereum_address,
value=self.task_to_compute.price,
payment_ts=parse_datetime_to_timestamp(datetime),
)
def test_case_2_test_for_non_existing_file(cluster_consts, cluster_url,
test_id):
current_time = get_current_utc_timestamp()
(subtask_id, task_id) = get_task_id_and_subtask_id(test_id,
'non_existing_file')
api_request(
cluster_url,
'send',
REQUESTOR_PRIVATE_KEY,
CONCENT_PUBLIC_KEY,
get_force_get_task_result(
task_id,
subtask_id,
current_time,
cluster_consts,
size=1024,
package_hash='sha1:b3ff7013c4644cdcbb6c7e4f1e5fdb10b9ceda5d'),
headers={
'Content-Type': 'application/octet-stream',
},
expected_status=200,
expected_message_type=message.concents.AckForceGetTaskResult.TYPE,
expected_content_type='application/octet-stream',
)
api_request(
cluster_url,
'receive',
PROVIDER_PRIVATE_KEY,
CONCENT_PUBLIC_KEY,
create_client_auth_message(PROVIDER_PRIVATE_KEY, PROVIDER_PUBLIC_KEY,
CONCENT_PUBLIC_KEY),
headers={
'Content-Type': 'application/octet-stream',
},
expected_status=200,
expected_message_type=message.concents.ForceGetTaskResultUpload.TYPE,
expected_content_type='application/octet-stream',
)
api_request(
cluster_url,
'receive',
REQUESTOR_PRIVATE_KEY,
CONCENT_PUBLIC_KEY,
create_client_auth_message(REQUESTOR_PRIVATE_KEY, REQUESTOR_PUBLIC_KEY,
CONCENT_PUBLIC_KEY),
headers={
'Content-Type': 'application/octet-stream',
},
expected_status=204,
)
def setUp(self):
self.request_factory = RequestFactory()
self.want_to_compute = message.WantToComputeTask(
node_name=1,
task_id=2,
perf_index=3,
price=4,
max_resource_size=5,
max_memory_size=6,
num_cores=7,
)
self.message_to_view = {
"node_name": self.want_to_compute.node_name, # pylint: disable=no-member
"task_id": self.want_to_compute.task_id, # pylint: disable=no-member
"perf_index": self.want_to_compute.perf_index, # pylint: disable=no-member
"price": self.want_to_compute.price, # pylint: disable=no-member
"max_resource_size": self.want_to_compute.max_resource_size, # pylint: disable=no-member
"max_memory_size": self.want_to_compute.max_memory_size, # pylint: disable=no-member
"num_cores": self.want_to_compute.num_cores, # pylint: disable=no-member
}
deadline_offset = 10
message_timestamp = get_current_utc_timestamp() + deadline_offset
compute_task_def = tasks.ComputeTaskDefFactory(
task_id='8',
subtask_id='8',
deadline=message_timestamp,
)
task_to_compute = tasks.TaskToComputeFactory(
compute_task_def=compute_task_def,
requestor_public_key=encode_hex(REQUESTOR_PUBLIC_KEY),
provider_public_key=encode_hex(PROVIDER_PUBLIC_KEY),
price=0,
)
task_to_compute = load(
dump(
task_to_compute,
REQUESTOR_PRIVATE_KEY,
settings.CONCENT_PUBLIC_KEY,
),
settings.CONCENT_PRIVATE_KEY,
REQUESTOR_PUBLIC_KEY,
check_time=False,
)
self.force_report_computed_task = message.ForceReportComputedTask()
self.force_report_computed_task.report_computed_task = message.tasks.ReportComputedTask(
)
self.force_report_computed_task.report_computed_task.task_to_compute = task_to_compute
def setUp(self):
with freeze_time("2017-11-17 10:00:00"):
super().setUp()
self.compute_task_def = message.ComputeTaskDef()
self.compute_task_def['task_id'] = '1'
self.compute_task_def['subtask_id'] = '1'
self.compute_task_def['deadline'] = get_current_utc_timestamp() + (60 * 37)
self.task_to_compute = tasks.TaskToComputeFactory(
compute_task_def = self.compute_task_def,
provider_public_key = PROVIDER_PUBLIC_KEY,
requestor_public_key = REQUESTOR_PUBLIC_KEY,
price=0,
)
self.force_golem_data = message.ForceReportComputedTask(
report_computed_task = message.tasks.ReportComputedTask(
task_to_compute = self.task_to_compute
)
)
def setUp(self):
self.message_timestamp = get_current_utc_timestamp()
self.public_key = '85cZzVjahnRpUBwm0zlNnqTdYom1LF1P1WNShLg17cmhN2UssnPrCjHKTi5susO3wrr/q07eswumbL82b4HgOw=='
self.download_token = FileTransferTokenFactory(
token_expiration_deadline=self.message_timestamp + 3600,
storage_cluster_address='http://devel.concent.golem.network/',
authorized_client_public_key=settings.CONCENT_PUBLIC_KEY,
operation=FileTransferToken.Operation.download,
)
self.download_token.files[0][
'path'] = 'blender/benchmark/test_task/scene-Helicopter-27-cycles.blend'
self.download_token.files[0][
'checksum'] = 'sha1:95a0f391c7ad86686ab1366bcd519ba5ab3cce89'
self.header_concent_auth = self._create_client_auth_message_as_header(
settings.CONCENT_PRIVATE_KEY,
settings.CONCENT_PUBLIC_KEY,
)
def test_receive_should_return_ack_if_the_receive_queue_contains_only_force_report_and_its_past_deadline(self):
with freeze_time("2017-11-17 10:00:00"):
self.compute_task_def = message.ComputeTaskDef()
self.compute_task_def['task_id'] = '2'
self.compute_task_def['subtask_id'] = '2'
self.compute_task_def['deadline'] = get_current_utc_timestamp()
self.task_to_compute = message.TaskToCompute(
compute_task_def = self.compute_task_def,
provider_public_key = PROVIDER_PUBLIC_KEY,
requestor_public_key = REQUESTOR_PUBLIC_KEY,
price=0,
)
self.force_golem_data = message.ForceReportComputedTask(
report_computed_task = message.tasks.ReportComputedTask(
task_to_compute = self.task_to_compute
)
)
def setUp(self):
super().setUp()
self.task_to_compute = self._get_deserialized_task_to_compute(
task_id='1',
subtask_id='8',
)
self.report_computed_task = self._get_deserialized_report_computed_task(
task_to_compute=self.task_to_compute, )
self.subtask = store_subtask(
task_id='1',
subtask_id='8',
provider_public_key=self.PROVIDER_PUBLIC_KEY,
requestor_public_key=self.REQUESTOR_PUBLIC_KEY,
state=Subtask.SubtaskState.VERIFICATION_FILE_TRANSFER,
next_deadline=get_current_utc_timestamp() +
settings.CONCENT_MESSAGING_TIME,
task_to_compute=self.task_to_compute,
report_computed_task=self.report_computed_task)
def setUp(self):
super().setUp()
self.stored_message_counter = 0
self.message_timestamp = get_current_utc_timestamp() # 1510912800
self.compute_task_def = self._get_deserialized_compute_task_def(
task_id='8',
subtask_id='8',
deadline=self.message_timestamp
)
self.task_to_compute = self._get_deserialized_task_to_compute(
compute_task_def = self.compute_task_def,
)
self.report_computed_task = self._get_deserialized_report_computed_task(
task_to_compute = self.task_to_compute
)
self.correct_golem_data = self._get_deserialized_force_report_computed_task(
report_computed_task=self.report_computed_task
)
self.want_to_compute = message.WantToComputeTask(
node_name = 1,
task_id = 2,
perf_index = 3,
price = 4,
max_resource_size = 5,
max_memory_size = 6,
num_cores = 7,
)
compute_task_def = message.ComputeTaskDef()
compute_task_def['task_id'] = '8'
compute_task_def['subtask_id'] = '8'
compute_task_def['deadline'] = self.message_timestamp + 600
task_to_compute = self._get_deserialized_task_to_compute(
compute_task_def = compute_task_def
)
self.cannot_compute_task = message.CannotComputeTask(
task_to_compute = task_to_compute
)
self.reject_report_computed_task = self._get_deserialized_reject_report_computed_task(
task_to_compute=task_to_compute,
reason=message.RejectReportComputedTask.REASON.SubtaskTimeLimitExceeded,
)
def create_file_transfer_token_for_concent(
subtask_id: str,
result_package_path: str,
result_size: int,
result_package_hash: str,
operation: FileTransferToken.Operation,
source_package_path: Optional[str] = None,
source_size: Optional[int] = None,
source_package_hash: Optional[str] = None,
) -> FileTransferToken:
return _create_file_transfer_token(
subtask_id=subtask_id,
source_package_path=source_package_path,
source_size=source_size,
source_package_hash=source_package_hash,
result_package_path=result_package_path,
result_size=result_size,
result_package_hash=result_package_hash,
authorized_client_public_key=settings.CONCENT_PUBLIC_KEY,
operation=operation,
token_expiration_deadline=get_current_utc_timestamp() + calculate_maximum_download_time(result_size, settings.MINIMUM_UPLOAD_RATE)
)
def test_case_2b_not_enough_funds(cluster_consts, cluster_url, test_id):
# Test CASE 2B - Send ForceSubtaskResults with not enough amount of funds on account
current_time = get_current_utc_timestamp()
(task_id, subtask_id) = get_task_id_and_subtask_id(test_id, '2B')
api_request(
cluster_url,
'send',
PROVIDER_PRIVATE_KEY,
CONCENT_PUBLIC_KEY,
force_subtask_results(
timestamp=timestamp_to_isoformat(current_time),
ack_report_computed_task=ack_report_computed_task(
timestamp=timestamp_to_isoformat(current_time),
report_computed_task=report_computed_task(
task_to_compute=create_signed_task_to_compute(
timestamp=calculate_timestamp(
current_time,
cluster_consts.concent_messaging_time),
task_id=task_id,
subtask_id=subtask_id,
deadline=calculate_deadline(
current_time,
cluster_consts.concent_messaging_time),
requestor_ethereum_public_key=encode_hex(
b'0' * GOLEM_PUBLIC_KEY_LENGTH),
provider_ethereum_public_key=encode_hex(
b'1' * GOLEM_PUBLIC_KEY_LENGTH),
price=0,
)))),
headers={
'Content-Type': 'application/octet-stream',
},
expected_status=200,
expected_message_type=message.concents.ServiceRefused.TYPE,
expected_content_type='application/octet-stream',
)
def test_in_soft_shutdown_mode_concent_should_accept_messages_that_would_cause_transition_to_passive_state(
self):
"""
Tests if in soft shutdown mode
Concent will accept new messages which update subtasks in passive state.
It also checks if email to admins is sent when all subtasks are turned into passive states.
"""
compute_task_def = self._get_deserialized_compute_task_def(
task_id='1', subtask_id='8', deadline="2017-12-01 11:00:00")
task_to_compute = self._get_deserialized_task_to_compute(
timestamp="2017-12-01 10:00:00",
compute_task_def=compute_task_def,
)
report_computed_task = self._get_deserialized_report_computed_task(
timestamp="2017-12-01 10:59:00",
task_to_compute=task_to_compute,
)
with freeze_time("2017-12-01 11:00:00"):
config.SOFT_SHUTDOWN_MODE = False
store_subtask(
task_id='1',
subtask_id='8',
provider_public_key=self.PROVIDER_PUBLIC_KEY,
requestor_public_key=self.REQUESTOR_PUBLIC_KEY,
state=Subtask.SubtaskState.FORCING_REPORT,
next_deadline=get_current_utc_timestamp() +
settings.CONCENT_MESSAGING_TIME,
task_to_compute=task_to_compute,
report_computed_task=report_computed_task,
)
config.SOFT_SHUTDOWN_MODE = True
self.stored_message_counter = 2
serialized_ack_report_computed_task = self._get_serialized_ack_report_computed_task(
timestamp="2017-12-01 11:00:05",
ack_report_computed_task=self.
_get_deserialized_ack_report_computed_task(
timestamp="2017-12-01 11:00:05",
subtask_id='8',
report_computed_task=report_computed_task),
requestor_private_key=self.REQUESTOR_PRIVATE_KEY)
with freeze_time("2017-12-01 11:00:05"):
response = self.client.post(
reverse('core:send'),
data=serialized_ack_report_computed_task,
content_type='application/octet-stream',
HTTP_CONCENT_CLIENT_PUBLIC_KEY=self.
_get_encoded_requestor_public_key(),
)
self.assertEqual(response.status_code, 202)
self.assertEqual(len(response.content), 0)
self._assert_stored_message_counter_increased(increased_by=1)
self._test_subtask_state(
task_id='1',
subtask_id='8',
subtask_state=Subtask.SubtaskState.REPORTED,
provider_key=self._get_encoded_provider_public_key(),
requestor_key=self._get_encoded_requestor_public_key(),
expected_nested_messages={
'task_to_compute', 'report_computed_task',
'ack_report_computed_task'
},
)
self._test_last_stored_messages(expected_messages=[
message.AckReportComputedTask,
],
task_id='1',
subtask_id='8',
timestamp="2017-12-01 11:00:05")
self._test_undelivered_pending_responses(
subtask_id='8',
client_public_key=self._get_encoded_provider_public_key(),
expected_pending_responses_receive=[
PendingResponse.ResponseType.ForceReportComputedTaskResponse,
])
self.assertEqual(len(mail.outbox), len(settings.ADMINS))
def test_case_2a_send_duplicated_force_subtask_results(cluster_consts,
cluster_url, test_id):
# Test CASE 2A + 2D + 3 - Send ForceSubtaskResults with same task_id as stored by Concent before
# Step 1. Send ForceSubtaskResults first time
current_time = get_current_utc_timestamp()
(task_id, subtask_id) = get_task_id_and_subtask_id(test_id, '2A')
signed_task_to_compute = create_signed_task_to_compute(
timestamp=calculate_timestamp(current_time,
cluster_consts.concent_messaging_time),
task_id=task_id,
subtask_id=subtask_id,
deadline=calculate_deadline(current_time,
cluster_consts.concent_messaging_time),
price=10000,
)
api_request(
cluster_url,
'send',
PROVIDER_PRIVATE_KEY,
CONCENT_PUBLIC_KEY,
force_subtask_results(
timestamp=timestamp_to_isoformat(current_time),
ack_report_computed_task=ack_report_computed_task(
timestamp=timestamp_to_isoformat(current_time),
report_computed_task=report_computed_task(
task_to_compute=signed_task_to_compute))),
headers={
'Content-Type': 'application/octet-stream',
},
expected_status=202,
)
time.sleep(1)
# Step 2. Send ForceSubtaskResults second time with same task_id
api_request(
cluster_url,
'send',
PROVIDER_PRIVATE_KEY,
CONCENT_PUBLIC_KEY,
force_subtask_results(
timestamp=timestamp_to_isoformat(current_time),
ack_report_computed_task=ack_report_computed_task(
timestamp=timestamp_to_isoformat(current_time),
report_computed_task=report_computed_task(
task_to_compute=signed_task_to_compute))),
headers={
'Content-Type': 'application/octet-stream',
},
expected_status=200,
expected_message_type=message.concents.ServiceRefused.TYPE,
expected_content_type='application/octet-stream',
)
# Step 3. Requestor wants to receive ForceSubtaskResults from Concent
api_request(
cluster_url,
'receive',
REQUESTOR_PRIVATE_KEY,
CONCENT_PUBLIC_KEY,
create_client_auth_message(REQUESTOR_PRIVATE_KEY, REQUESTOR_PUBLIC_KEY,
CONCENT_PUBLIC_KEY),
headers={
'Content-Type': 'application/octet-stream',
},
expected_status=200,
expected_message_type=message.concents.ForceSubtaskResults.TYPE,
expected_content_type='application/octet-stream',
)
def test_case_4b_requestor_accepts_subtaks_results(cluster_consts, cluster_url,
test_id):
# Test CASE 4B + 5. Requestor sends ForceSubtaskResultsResponse with SubtaskResultsAccepted
# Step 1. Provider sends ForceSubtaskResults
current_time = get_current_utc_timestamp()
(task_id, subtask_id) = get_task_id_and_subtask_id(test_id, '4B')
signed_task_to_compute = create_signed_task_to_compute(
timestamp=calculate_timestamp(current_time,
cluster_consts.concent_messaging_time),
task_id=task_id,
subtask_id=subtask_id,
deadline=calculate_deadline(current_time,
cluster_consts.concent_messaging_time),
price=10000,
)
api_request(
cluster_url,
'send',
PROVIDER_PRIVATE_KEY,
CONCENT_PUBLIC_KEY,
force_subtask_results(
timestamp=timestamp_to_isoformat(current_time),
ack_report_computed_task=ack_report_computed_task(
timestamp=timestamp_to_isoformat(current_time),
report_computed_task=report_computed_task(
task_to_compute=signed_task_to_compute))),
headers={
'Content-Type': 'application/octet-stream',
},
expected_status=202,
)
time.sleep(1)
# Step 2. Requestor receives ForceSubtaskResults
api_request(
cluster_url,
'receive',
REQUESTOR_PRIVATE_KEY,
CONCENT_PUBLIC_KEY,
create_client_auth_message(REQUESTOR_PRIVATE_KEY, REQUESTOR_PUBLIC_KEY,
CONCENT_PUBLIC_KEY),
headers={
'Content-Type': 'application/octet-stream',
},
expected_status=200,
expected_message_type=message.concents.ForceSubtaskResults.TYPE,
expected_content_type='application/octet-stream',
)
# Step 3. Requestor sends ForceSubtaskResultsResponse
api_request(
cluster_url,
'send',
REQUESTOR_PRIVATE_KEY,
CONCENT_PUBLIC_KEY,
force_subtask_results_response(
timestamp=timestamp_to_isoformat(current_time),
subtask_results_accepted=subtask_results_accepted(
timestamp=timestamp_to_isoformat(current_time),
payment_ts=timestamp_to_isoformat(current_time + 1),
task_to_compute=signed_task_to_compute)),
headers={
'Content-Type': 'application/octet-stream',
},
expected_status=202,
)
# Step 4. Provider receives ForceSubtaskResultsResponse
api_request(
cluster_url,
'receive',
PROVIDER_PRIVATE_KEY,
CONCENT_PUBLIC_KEY,
create_client_auth_message(PROVIDER_PRIVATE_KEY, PROVIDER_PUBLIC_KEY,
CONCENT_PUBLIC_KEY),
headers={
'Content-Type': 'application/octet-stream',
},
expected_status=200,
expected_message_type=message.concents.ForceSubtaskResultsResponse.
TYPE,
expected_content_type='application/octet-stream',
)
def update_timed_out_subtasks(client_public_key: bytes, ):
verify_file_status(client_public_key)
clients_subtask_list = Subtask.objects.filter(
Q(requestor__public_key=b64encode(client_public_key))
| Q(provider__public_key=b64encode(client_public_key)),
state__in=[state.name for state in Subtask.ACTIVE_STATES],
next_deadline__lte=timezone.now())
for subtask in clients_subtask_list:
if subtask.state == Subtask.SubtaskState.FORCING_REPORT.name: # pylint: disable=no-member
update_subtask_state(
subtask=subtask,
state=Subtask.SubtaskState.REPORTED.name, # pylint: disable=no-member
)
store_pending_message(
response_type=PendingResponse.ResponseType.
ForceReportComputedTaskResponse,
client_public_key=subtask.provider.public_key_bytes,
queue=PendingResponse.Queue.Receive,
subtask=subtask,
)
store_pending_message(
response_type=PendingResponse.ResponseType.
VerdictReportComputedTask,
client_public_key=subtask.requestor.public_key_bytes,
queue=PendingResponse.Queue.ReceiveOutOfBand,
subtask=subtask,
)
elif subtask.state == Subtask.SubtaskState.FORCING_RESULT_TRANSFER.name: # pylint: disable=no-member
update_subtask_state(
subtask=subtask,
state=Subtask.SubtaskState.FAILED.name, # pylint: disable=no-member
)
store_pending_message(
response_type=PendingResponse.ResponseType.
ForceGetTaskResultFailed,
client_public_key=subtask.requestor.public_key_bytes,
queue=PendingResponse.Queue.Receive,
subtask=subtask,
)
elif subtask.state == Subtask.SubtaskState.FORCING_ACCEPTANCE.name: # pylint: disable=no-member
update_subtask_state(
subtask=subtask,
state=Subtask.SubtaskState.ACCEPTED.name, # pylint: disable=no-member
)
store_pending_message(
response_type=PendingResponse.ResponseType.
SubtaskResultsSettled,
client_public_key=subtask.provider.public_key_bytes,
queue=PendingResponse.Queue.Receive,
subtask=subtask,
)
store_pending_message(
response_type=PendingResponse.ResponseType.
SubtaskResultsSettled,
client_public_key=subtask.requestor.public_key_bytes,
queue=PendingResponse.Queue.ReceiveOutOfBand,
subtask=subtask,
)
elif subtask.state == Subtask.SubtaskState.ADDITIONAL_VERIFICATION.name: # pylint: disable=no-member
locked_subtask = Subtask.objects.select_for_update().get(
pk=subtask.pk)
task_to_compute = deserialize_message(
locked_subtask.task_to_compute.data.tobytes())
# Worker makes a payment from requestor's deposit just like in the forced acceptance use case.
core.payments.base.make_force_payment_to_provider( # pylint: disable=no-value-for-parameter
requestor_eth_address=task_to_compute.
requestor_ethereum_address,
provider_eth_address=task_to_compute.provider_ethereum_address,
value=task_to_compute.price,
payment_ts=get_current_utc_timestamp(),
)
update_subtask_state(
subtask=locked_subtask,
state=Subtask.SubtaskState.ACCEPTED.name, # pylint: disable=no-member
)
store_pending_message(
response_type=PendingResponse.ResponseType.
SubtaskResultsSettled,
client_public_key=locked_subtask.provider.public_key_bytes,
queue=PendingResponse.Queue.ReceiveOutOfBand,
subtask=locked_subtask,
)
store_pending_message(
response_type=PendingResponse.ResponseType.
SubtaskResultsSettled,
client_public_key=locked_subtask.requestor.public_key_bytes,
queue=PendingResponse.Queue.ReceiveOutOfBand,
subtask=locked_subtask,
)
logging.log_changes_in_subtask_states(
logger,
client_public_key,
clients_subtask_list.count(),
)
def upload_finished(subtask_id: str):
try:
subtask = Subtask.objects.get(subtask_id=subtask_id)
except Subtask.DoesNotExist:
logging.error(
f'Task `upload_finished` tried to get Subtask object with ID {subtask_id} but it does not exist.'
)
return
report_computed_task = deserialize_message(
subtask.report_computed_task.data.tobytes())
# Check subtask state, if it's VERIFICATION FILE TRANSFER, proceed with the task.
if subtask.state_enum == Subtask.SubtaskState.VERIFICATION_FILE_TRANSFER:
# If subtask is past the deadline, processes the timeout.
if subtask.next_deadline.timestamp() < get_current_utc_timestamp():
# Worker makes a payment from requestor's deposit just like in the forced acceptance use case.
base.make_force_payment_to_provider( # pylint: disable=no-value-for-parameter
requestor_eth_address=report_computed_task.task_to_compute.
requestor_ethereum_address,
provider_eth_address=report_computed_task.task_to_compute.
provider_ethereum_address,
value=report_computed_task.task_to_compute.price,
payment_ts=get_current_utc_timestamp(),
)
update_subtask_state(
subtask=subtask,
state=Subtask.SubtaskState.FAILED.name, # pylint: disable=no-member
)
# Worker adds SubtaskResultsSettled to provider's and requestor's receive queues (both out-of-band)
for public_key in [
subtask.provider.public_key_bytes,
subtask.requestor.public_key_bytes
]:
store_pending_message(
response_type=PendingResponse.ResponseType.
SubtaskResultsSettled,
client_public_key=public_key,
queue=PendingResponse.Queue.ReceiveOutOfBand,
subtask=subtask,
)
return
# Change subtask state to ADDITIONAL VERIFICATION.
update_subtask_state(
subtask=subtask,
state=Subtask.SubtaskState.ADDITIONAL_VERIFICATION.name, # pylint: disable=no-member
next_deadline=int(subtask.next_deadline.timestamp()) +
settings.SUBTASK_VERIFICATION_TIME)
# Add upload_acknowledged task to the work queue.
tasks.upload_acknowledged.delay(
subtask_id=subtask_id,
source_file_size=report_computed_task.task_to_compute.size,
source_package_hash=report_computed_task.task_to_compute.
package_hash,
result_file_size=report_computed_task.size,
result_package_hash=report_computed_task.package_hash,
)
# If it's ADDITIONAL VERIFICATION, ACCEPTED or FAILED, log a warning and ignore the notification.
# Processing ends here. This means that it's a duplicate notification.
elif subtask.state_enum in [
Subtask.SubtaskState.ADDITIONAL_VERIFICATION,
Subtask.SubtaskState.ACCEPTED, Subtask.SubtaskState.FAILED
]:
logging.warning(
f'Subtask with ID {subtask_id} is expected to be in `VERIFICATION_FILE_TRANSFER` state, but was in {subtask.state}.'
)
# If it's one of the states that can precede verification, report an error. Processing ends here.
else:
logging.error(
f'Subtask with ID {subtask_id} is expected to be in `VERIFICATION_FILE_TRANSFER` state, but was in {subtask.state}.'
)
def verification_result(
self,
subtask_id: str,
result: str,
error_message: Optional[str] = None,
error_code: Optional[str] = None,
):
logger.info(
f'verification_result_task starts with: SUBTASK_ID {subtask_id} -- RESULT {result}'
)
assert isinstance(subtask_id, str)
assert isinstance(result, str)
assert result in VerificationResult.__members__.keys()
assert isinstance(error_message, (str, type(None)))
assert isinstance(error_code, (str, type(None)))
result_enum = VerificationResult[result]
assert result_enum != VerificationResult.ERROR or all(
[error_message, error_code])
# Worker locks database row corresponding to the subtask in the subtask table.
try:
subtask = Subtask.objects.select_for_update(nowait=True).get(
subtask_id=subtask_id)
except DatabaseError:
logging.warning(
f'Subtask object with ID {subtask_id} database row is locked, '
f'retrying task {self.request.retries}/{self.max_retries}')
# If the row is already locked, task fails so that Celery can retry later.
self.retry(
countdown=CELERY_LOCKED_SUBTASK_DELAY,
max_retries=MAXIMUM_VERIFICATION_RESULT_TASK_RETRIES,
throw=False,
)
return
if subtask.state_enum == Subtask.SubtaskState.ACCEPTED:
logger.warning(
VERIFICATION_RESULT_SUBTASK_STATE_ACCEPTED_LOG_MESSAGE.format(
subtask_id))
return
elif subtask.state_enum == Subtask.SubtaskState.FAILED:
logger.warning(
VERIFICATION_RESULT_SUBTASK_STATE_FAILED_LOG_MESSAGE.format(
subtask_id))
return
elif subtask.state_enum != Subtask.SubtaskState.ADDITIONAL_VERIFICATION:
logger.error(
VERIFICATION_RESULT_SUBTASK_STATE_UNEXPECTED_LOG_MESSAGE.format(
subtask_id, subtask.state))
return
# If the time is already past next_deadline for the subtask (SubtaskResultsRejected.timestamp + AVCT)
# worker ignores worker's message and processes the timeout.
if subtask.next_deadline < parse_timestamp_to_utc_datetime(
get_current_utc_timestamp()):
task_to_compute = deserialize_message(
subtask.task_to_compute.data.tobytes())
# Worker makes a payment from requestor's deposit just like in the forced acceptance use case.
base.make_force_payment_to_provider( # pylint: disable=no-value-for-parameter
requestor_eth_address=task_to_compute.requestor_ethereum_address,
provider_eth_address=task_to_compute.provider_ethereum_address,
value=task_to_compute.price,
payment_ts=get_current_utc_timestamp(),
)
update_subtask_state(
subtask=subtask,
state=Subtask.SubtaskState.ACCEPTED.name, # pylint: disable=no-member
)
for public_key in [
subtask.provider.public_key_bytes,
subtask.requestor.public_key_bytes
]:
store_pending_message(
response_type=PendingResponse.ResponseType.
SubtaskResultsSettled,
client_public_key=public_key,
queue=PendingResponse.Queue.ReceiveOutOfBand,
subtask=subtask,
)
return
if result_enum == VerificationResult.MISMATCH:
# Worker adds SubtaskResultsRejected to provider's and requestor's receive queues (both out-of-band)
for public_key in [
subtask.provider.public_key_bytes,
subtask.requestor.public_key_bytes
]:
store_pending_message(
response_type=PendingResponse.ResponseType.
SubtaskResultsRejected,
client_public_key=public_key,
queue=PendingResponse.Queue.ReceiveOutOfBand,
subtask=subtask,
)
# Worker changes subtask state to FAILED
subtask.state = Subtask.SubtaskState.FAILED.name # pylint: disable=no-member
subtask.next_deadline = None
subtask.full_clean()
subtask.save()
elif result_enum in (VerificationResult.MATCH, VerificationResult.ERROR):
# Worker logs the error code and message
if result_enum == VerificationResult.ERROR:
logger.info(
f'verification_result_task processing error result with: '
f'SUBTASK_ID {subtask_id} -- RESULT {result_enum.name} -- ERROR MESSAGE {error_message} -- ERROR CODE {error_code}'
)
task_to_compute = deserialize_message(
subtask.task_to_compute.data.tobytes())
# Worker makes a payment from requestor's deposit just like in the forced acceptance use case.
base.make_force_payment_to_provider( # pylint: disable=no-value-for-parameter
requestor_eth_address=task_to_compute.requestor_ethereum_address,
provider_eth_address=task_to_compute.provider_ethereum_address,
value=task_to_compute.price,
payment_ts=get_current_utc_timestamp(),
)
# Worker adds SubtaskResultsSettled to provider's and requestor's receive queues (both out-of-band)
for public_key in [
subtask.provider.public_key_bytes,
subtask.requestor.public_key_bytes
]:
store_pending_message(
response_type=PendingResponse.ResponseType.
SubtaskResultsSettled,
client_public_key=public_key,
queue=PendingResponse.Queue.ReceiveOutOfBand,
subtask=subtask,
)
# Worker changes subtask state to ACCEPTED
update_subtask_state(
subtask=subtask,
state=Subtask.SubtaskState.ACCEPTED.name, # pylint: disable=no-member
)
logger.info(
f'verification_result_task ends with: SUBTASK_ID {subtask_id} -- RESULT {result_enum.name}'
)
def test_case_1_test_for_existing_file(cluster_consts, cluster_url, test_id):
current_time = get_current_utc_timestamp()
(subtask_id, task_id) = get_task_id_and_subtask_id(test_id,
'existing_file')
file_content = task_id
file_size = len(file_content)
file_check_sum = 'sha1:' + hashlib.sha1(file_content.encode()).hexdigest()
file_path = get_storage_result_file_path(
task_id=task_id,
subtask_id=subtask_id,
)
# Case 1 - test for existing file
api_request(
cluster_url,
'send',
REQUESTOR_PRIVATE_KEY,
CONCENT_PUBLIC_KEY,
get_force_get_task_result(
task_id,
subtask_id,
current_time,
cluster_consts,
size=file_size,
package_hash=file_check_sum,
),
headers={
'Content-Type': 'application/octet-stream',
},
expected_status=200,
expected_message_type=message.concents.AckForceGetTaskResult.TYPE,
expected_content_type='application/octet-stream',
)
force_get_task_result_upload = api_request(
cluster_url,
'receive',
PROVIDER_PRIVATE_KEY,
CONCENT_PUBLIC_KEY,
create_client_auth_message(PROVIDER_PRIVATE_KEY, PROVIDER_PUBLIC_KEY,
CONCENT_PUBLIC_KEY),
headers={
'Content-Type': 'application/octet-stream',
},
expected_status=200,
expected_message_type=message.concents.ForceGetTaskResultUpload.TYPE,
expected_content_type='application/octet-stream',
)
response = upload_file_to_storage_cluster(
file_content, file_path,
force_get_task_result_upload.file_transfer_token)
assert_condition(response.status_code, 200,
'File has not been stored on cluster')
print(
'\nUploaded file with task_id {}. Checksum of this file is {}, and size of this file is {}.\n'
.format(task_id, file_check_sum, file_size))
api_request(
cluster_url,
'receive',
REQUESTOR_PRIVATE_KEY,
CONCENT_PUBLIC_KEY,
create_client_auth_message(REQUESTOR_PRIVATE_KEY, REQUESTOR_PUBLIC_KEY,
CONCENT_PUBLIC_KEY),
headers={
'Content-Type': 'application/octet-stream',
},
expected_status=200,
expected_message_type=message.concents.ForceGetTaskResultDownload.TYPE,
expected_content_type='application/octet-stream',
)
def setUp(self):
super().setUp()
self.compute_task_def = message.ComputeTaskDef()
self.compute_task_def['task_id'] = '1'
self.compute_task_def['subtask_id'] = '1'
self.compute_task_def['deadline'] = get_current_utc_timestamp() - 60
self.task_to_compute = tasks.TaskToComputeFactory(
compute_task_def = self.compute_task_def,
provider_public_key = PROVIDER_PUBLIC_KEY,
requestor_public_key = REQUESTOR_PUBLIC_KEY,
price=0,
)
self.force_golem_data = message.ForceReportComputedTask(
report_computed_task = message.tasks.ReportComputedTask(
task_to_compute = self.task_to_compute
)
)
with freeze_time("2017-11-17 10:00:00"):
self.force_golem_data = message.ForceReportComputedTask(
report_computed_task = message.tasks.ReportComputedTask(
task_to_compute = self.task_to_compute
)
)
message_timestamp = datetime.datetime.now(timezone.utc)
new_message = StoredMessage(
type = self.force_golem_data.report_computed_task.TYPE,
timestamp = message_timestamp,
data = self.force_golem_data.report_computed_task.serialize(),
task_id = self.compute_task_def['task_id'], # pylint: disable=no-member
subtask_id = self.compute_task_def['subtask_id'], # pylint: disable=no-member
)
new_message.full_clean()
new_message.save()
task_to_compute_message = StoredMessage(
type = self.task_to_compute.TYPE,
timestamp = message_timestamp,
data = self.task_to_compute.serialize(),
task_id = self.compute_task_def['task_id'], # pylint: disable=no-member
subtask_id = self.compute_task_def['subtask_id'], # pylint: disable=no-member
)
task_to_compute_message.full_clean()
task_to_compute_message.save()
ack_report_computed_task = message.AckReportComputedTask(
report_computed_task=message.tasks.ReportComputedTask(
task_to_compute=self.task_to_compute,
)
)
stored_ack_report_computed_task = StoredMessage(
type = ack_report_computed_task.TYPE,
timestamp = message_timestamp,
data = ack_report_computed_task.serialize(),
task_id = self.compute_task_def['task_id'], # pylint: disable=no-member
subtask_id = self.compute_task_def['subtask_id'], # pylint: disable=no-member
)
stored_ack_report_computed_task.full_clean()
stored_ack_report_computed_task.save()
client_provider = Client(
public_key_bytes = PROVIDER_PUBLIC_KEY
)
client_provider.full_clean()
client_provider.save()
client_requestor = Client(
public_key_bytes = REQUESTOR_PUBLIC_KEY
)
client_requestor.full_clean()
client_requestor.save()
subtask = Subtask(
task_id = self.compute_task_def['task_id'],
subtask_id = self.compute_task_def['subtask_id'],
report_computed_task = new_message,
task_to_compute = task_to_compute_message,
ack_report_computed_task = stored_ack_report_computed_task,
state = Subtask.SubtaskState.REPORTED.name, # pylint: disable=no-member
provider = client_provider,
requestor = client_requestor,
)
subtask.full_clean()
subtask.save()
new_message_inbox = PendingResponse(
response_type = PendingResponse.ResponseType.ForceReportComputedTask.name, # pylint: disable=no-member
client = client_requestor,
queue = PendingResponse.Queue.ReceiveOutOfBand.name, # pylint: disable=no-member
subtask = subtask,
)
new_message_inbox.full_clean()
new_message_inbox.save()
def _get_list_of_force_transactions(self, requestor_eth_address = None, provider_eth_address = None, payment_ts = None, current_time = None, transaction_type = None): # pylint: disable=unused-argument
current_time = get_current_utc_timestamp()
item1 = self._create_payment_object(amount = 1000, closure_time = current_time - 2000)
item2 = self._create_payment_object(amount = 2000, closure_time = current_time - 1000)
return [item1, item2]
def test_that_concent_should_change_subtask_state_if_verification_is_after_deadline(
self):
"""
Tests that Concent should change subtask state if verification is after deadline.
To achieve changing state by working queue mechanism, a duplicated SubtaskResultsVerify is being sent.
Provider -> Concent: SubtaskResultsVerify
"""
with freeze_time("2018-04-01 10:30:00"):
subtask = store_or_update_subtask(
task_id=self.task_id,
subtask_id=self.subtask_id,
provider_public_key=self.PROVIDER_PUBLIC_KEY,
requestor_public_key=self.REQUESTOR_PUBLIC_KEY,
state=Subtask.SubtaskState.ADDITIONAL_VERIFICATION,
next_deadline=get_current_utc_timestamp() +
settings.ADDITIONAL_VERIFICATION_CALL_TIME,
task_to_compute=self.report_computed_task.task_to_compute,
report_computed_task=self.report_computed_task,
)
self._assert_stored_message_counter_increased(2)
with freeze_time(
parse_timestamp_to_utc_datetime(
subtask.next_deadline.timestamp() + 1)):
serialized_subtask_results_verify = self._get_serialized_subtask_results_verify(
subtask_results_verify=self.
_get_deserialized_subtask_results_verify(
subtask_results_rejected=self.
_get_deserialized_subtask_results_rejected(
reason=message.tasks.SubtaskResultsRejected.REASON.
VerificationNegative,
report_computed_task=self.report_computed_task,
)))
response = self.client.post(
reverse('core:send'),
data=serialized_subtask_results_verify,
content_type='application/octet-stream',
)
assert response.status_code == 200
subtask.refresh_from_db()
self.assertEqual(subtask.state_enum, Subtask.SubtaskState.ACCEPTED)
self.assertEqual(subtask.next_deadline, None)
self._test_undelivered_pending_responses(
subtask_id=subtask.subtask_id,
client_public_key=self._get_encoded_provider_public_key(),
client_public_key_out_of_band=self.
_get_encoded_provider_public_key(),
expected_pending_responses_receive_out_of_band=[
PendingResponse.ResponseType.SubtaskResultsSettled,
])
self._test_undelivered_pending_responses(
subtask_id=subtask.subtask_id,
client_public_key=self._get_encoded_requestor_public_key(),
client_public_key_out_of_band=self.
_get_encoded_requestor_public_key(),
expected_pending_responses_receive_out_of_band=[
PendingResponse.ResponseType.SubtaskResultsSettled,
])
response_2 = self.client.post(
reverse('core:receive_out_of_band'),
data=self._create_requestor_auth_message(),
content_type='application/octet-stream',
)
self._test_response(
response_2,
status=200,
key=self.REQUESTOR_PRIVATE_KEY,
message_type=message.concents.SubtaskResultsSettled,
fields={
'task_to_compute': self.report_computed_task.task_to_compute,
})
response_3 = self.client.post(
reverse('core:receive_out_of_band'),
data=self._create_provider_auth_message(),
content_type='application/octet-stream',
)
self._test_response(
response_3,
status=200,
key=self.PROVIDER_PRIVATE_KEY,
message_type=message.concents.SubtaskResultsSettled,
fields={
'task_to_compute': self.report_computed_task.task_to_compute,
})
def parse_headers(request: WSGIRequest, path_to_file: str, operation: FileTransferToken.Operation) -> Union[FileTransferToken.FileInfo, JsonResponse]:
# Decode and check if request header contains a golem message:
if 'HTTP_AUTHORIZATION' not in request.META:
return gatekeeper_access_denied_response(
"Missing 'Authorization' header.",
operation,
ErrorCode.HEADER_AUTHORIZATION_MISSING,
path_to_file,
)
authorization_scheme_and_token = request.META['HTTP_AUTHORIZATION'].split(" ", 1)
assert len(authorization_scheme_and_token) in [1, 2]
if len(authorization_scheme_and_token) == 1:
return gatekeeper_access_denied_response(
"Missing token in the 'Authorization' header.",
operation,
ErrorCode.HEADER_AUTHORIZATION_MISSING_TOKEN,
path_to_file,
)
(scheme, token) = authorization_scheme_and_token
if scheme != 'Golem':
return gatekeeper_access_denied_response(
"Unrecognized scheme in the 'Authorization' header.",
operation,
ErrorCode.HEADER_AUTHORIZATION_UNRECOGNIZED_SCHEME,
path_to_file,
)
try:
decoded_auth_header_content = b64decode(token, validate = True)
except binascii.Error:
return gatekeeper_access_denied_response(
"Unable to decode token in the 'Authorization' header.",
operation,
ErrorCode.HEADER_AUTHORIZATION_NOT_BASE64_ENCODED_VALUE,
path_to_file,
)
try:
loaded_golem_message = load(decoded_auth_header_content, settings.CONCENT_PRIVATE_KEY, settings.CONCENT_PUBLIC_KEY, check_time = False)
except MessageError:
return gatekeeper_access_denied_response(
"Token in the 'Authorization' header is not a valid Golem message.",
operation,
ErrorCode.HEADER_AUTHORIZATION_TOKEN_INVALID_MESSAGE,
path_to_file,
)
assert isinstance(loaded_golem_message, Message)
# Check if request header contains Concent-Auth:
if 'HTTP_CONCENT_AUTH' not in request.META:
return gatekeeper_access_denied_response(
'Missing Concent-Auth header.',
operation,
ErrorCode.AUTH_CLIENT_AUTH_MESSAGE_MISSING,
path_to_file,
loaded_golem_message.subtask_id
)
# Try to load in ClientAuthorization message from Concent-Auth header
try:
client_authorization = load(
b64decode(request.META['HTTP_CONCENT_AUTH'], validate=True),
settings.CONCENT_PRIVATE_KEY,
loaded_golem_message.authorized_client_public_key,
)
concent_client_public_key = b64encode(client_authorization.client_public_key).decode('ascii')
except (MessageError, binascii.Error):
return gatekeeper_access_denied_response(
'Cannot load ClientAuthorization message from Concent-Auth header.',
operation,
ErrorCode.AUTH_CLIENT_AUTH_MESSAGE_INVALID,
path_to_file,
loaded_golem_message.subtask_id
)
logging.log_message_under_validation(
logger,
loaded_golem_message.operation,
loaded_golem_message.__class__.__name__,
path_to_file,
loaded_golem_message.subtask_id,
concent_client_public_key
)
current_time = get_current_utc_timestamp()
if current_time > loaded_golem_message.token_expiration_deadline:
return gatekeeper_access_denied_response(
'token_expiration_deadline has passed.',
operation,
ErrorCode.MESSAGE_TOKEN_EXPIRATION_DEADLINE_PASSED,
path_to_file,
loaded_golem_message.subtask_id,
concent_client_public_key
)
try:
validate_file_transfer_token(loaded_golem_message)
except FileTransferTokenError as exception:
return gatekeeper_access_denied_response(
exception.error_message,
operation,
exception.error_code,
path_to_file,
loaded_golem_message.subtask_id,
concent_client_public_key
)
authorized_client_public_key_base64 = (b64encode(loaded_golem_message.authorized_client_public_key)).decode('ascii')
if concent_client_public_key != authorized_client_public_key_base64:
return gatekeeper_access_denied_response(
'You are not authorized to use this token.',
operation,
ErrorCode.MESSAGE_AUTHORIZED_CLIENT_PUBLIC_KEY_UNAUTHORIZED_CLIENT,
path_to_file,
loaded_golem_message.subtask_id,
concent_client_public_key
)
# -OPERATION
if request.method == 'POST' and loaded_golem_message.operation != FileTransferToken.Operation.upload:
return gatekeeper_access_denied_response(
'Upload requests must use POST method.',
operation,
ErrorCode.MESSAGE_OPERATION_INVALID,
path_to_file,
loaded_golem_message.subtask_id,
concent_client_public_key
)
if request.method in ['GET', 'HEAD'] and loaded_golem_message.operation != FileTransferToken.Operation.download:
return gatekeeper_access_denied_response(
'Download requests must use GET or HEAD method.',
operation,
ErrorCode.MESSAGE_OPERATION_INVALID,
path_to_file,
loaded_golem_message.subtask_id,
concent_client_public_key
)
matching_files = [file for file in loaded_golem_message.files if path_to_file == file['path']]
if len(matching_files) == 1:
logging.log_message_successfully_validated(
logger,
loaded_golem_message.operation,
loaded_golem_message.__class__.__name__,
path_to_file,
loaded_golem_message.subtask_id,
concent_client_public_key
)
return matching_files[0]
else:
assert len(matching_files) == 0
return gatekeeper_access_denied_response(
'Your token does not authorize you to transfer the requested file.',
operation,
ErrorCode.MESSAGE_FILES_PATH_NOT_LISTED_IN_FILES,
path_to_file,
loaded_golem_message.subtask_id,
concent_client_public_key
)
