def _deserialize_database_message(
self, serialized_message: StoredMessage
) -> message.Message: # pylint: disable=no-self-use
if isinstance(serialized_message.data, bytes):
deserialized_message = deserialize_message(serialized_message.data)
else:
deserialized_message = deserialize_message(
serialized_message.data.tobytes()) # pylint: disable=no-member
return deserialized_message
def verify_file_status(client_public_key: bytes, ):
"""
Function to verify existence of a file on cluster storage
"""
encoded_client_public_key = b64encode(client_public_key)
force_get_task_result_list = Subtask.objects.filter(
requestor__public_key=encoded_client_public_key,
state=Subtask.SubtaskState.FORCING_RESULT_TRANSFER.name, # pylint: disable=no-member
)
for get_task_result in force_get_task_result_list:
report_computed_task = deserialize_message(
get_task_result.report_computed_task.data.tobytes())
if request_upload_status(report_computed_task):
subtask = get_task_result
subtask.state = Subtask.SubtaskState.RESULT_UPLOADED.name # pylint: disable=no-member
subtask.next_deadline = None
subtask.full_clean()
subtask.save()
store_pending_message(
response_type=PendingResponse.ResponseType.
ForceGetTaskResultDownload,
client_public_key=subtask.requestor.public_key_bytes,
queue=PendingResponse.Queue.Receive,
subtask=subtask,
)
logging.log_file_status(
logger,
subtask.task_id,
subtask.subtask_id,
subtask.requestor.public_key_bytes,
subtask.provider.public_key_bytes,
)
def populate_subtask_size_field(apps, _schema_editor):
Subtask = apps.get_model('core', 'Subtask')
for subtask in Subtask.objects.all():
subtask.result_package_size = deserialize_message(
subtask.report_computed_task.data).size
subtask.full_clean()
subtask.save()
def populate_subtask_deadline_field(apps, _schema_editor):
Subtask = apps.get_model('core', 'Subtask')
for subtask in Subtask.objects.all():
subtask.computation_deadline = parse_timestamp_to_utc_datetime(
deserialize_message(
subtask.task_to_compute.data).compute_task_def['deadline'])
subtask.full_clean()
subtask.save()
def switch_stored_message_timestamp_to_message_creation_time(apps, _schema_editor):
StoredMessage = apps.get_model('core', 'StoredMessage')
for stored_message in StoredMessage.objects.all():
stored_message.timestamp = parse_timestamp_to_utc_datetime(
deserialize_message(stored_message.data.tobytes()).timestamp
)
stored_message.full_clean()
stored_message.save()
def insert_force_get_task_result_if_empty(apps, _schema_editor):
Subtask = apps.get_model('core', 'Subtask')
StoredMessage = apps.get_model('core', 'StoredMessage')
for subtask in Subtask.objects.filter(force_get_task_result__isnull=True):
report_computed_task_serialized = subtask.report_computed_task
force_get_task_result = ForceGetTaskResult(
report_computed_task=deserialize_message(
report_computed_task_serialized.data.tobytes()))
stored_message = StoredMessage(
type=force_get_task_result.TYPE,
timestamp=now(),
data=force_get_task_result.serialize(),
task_id=subtask.task_id,
subtask_id=subtask.subtask_id,
)
stored_message.full_clean()
stored_message.save()
subtask.force_get_task_result = stored_message
subtask.full_clean()
subtask.save()
def finalize_payment(deposit_claim: DepositClaim) -> Optional[str]:
"""
This operation tells Bankster to pay out funds from deposit.
For each claim, Bankster uses SCI to submit an Ethereum transaction to the Ethereum client which then propagates it
to the rest of the network.
Hopefully the transaction is included in one of the upcoming blocks on the blockchain.
IMPORTANT!: This function must never be called in parallel for the same DepositClaim - otherwise provider might get
paid twice for the same thing. It's caller responsibility to ensure that.
"""
assert isinstance(deposit_claim, DepositClaim)
assert deposit_claim.tx_hash is None
# Bankster asks SCI about the amount of funds available on the deposit account listed in the DepositClaim.
available_funds = service.get_deposit_value( # pylint: disable=no-value-for-parameter
client_eth_address=deposit_claim.payer_deposit_account.ethereum_address
)
# Bankster begins a database transaction and puts a database lock on the DepositAccount object.
with non_nesting_atomic(using='control'):
DepositAccount.objects.select_for_update().get(
pk=deposit_claim.payer_deposit_account_id)
# Bankster sums the amounts of all existing DepositClaims that have the same payer as the one being processed.
aggregated_client_claims = DepositClaim.objects.filter(
payer_deposit_account=deposit_claim.payer_deposit_account).exclude(
pk=deposit_claim.pk).aggregate(
sum_of_existing_claims=Coalesce(Sum('amount'), 0))
# Bankster subtracts that value from the amount of funds available in the deposit.
available_funds_without_claims = available_funds - aggregated_client_claims[
'sum_of_existing_claims']
# If the result is negative or zero, Bankster removes the DepositClaim object being processed.
if available_funds_without_claims <= 0:
deposit_claim.delete()
return None
# Otherwise if the result is lower than DepositAccount.amount,
# Bankster sets this field to the amount that's actually available.
elif available_funds_without_claims < deposit_claim.amount:
deposit_claim.amount = available_funds_without_claims
deposit_claim.save()
# If the DepositClaim still exists at this point, Bankster uses SCI to create an Ethereum transaction.
subtask = Subtask.objects.filter(
subtask_id=deposit_claim.subtask_id).first() # pylint: disable=no-member
task_to_compute: TaskToCompute = deserialize_message(
subtask.task_to_compute.data.tobytes())
v, r, s = task_to_compute.promissory_note_sig
if deposit_claim.concent_use_case == ConcentUseCase.FORCED_ACCEPTANCE:
ethereum_transaction_hash = service.force_subtask_payment( # pylint: disable=no-value-for-parameter
requestor_eth_address=deposit_claim.payer_deposit_account.
ethereum_address,
provider_eth_address=deposit_claim.payee_ethereum_address,
value=task_to_compute.price,
subtask_id=deposit_claim.subtask_id,
v=v,
r=r,
s=s,
reimburse_amount=deposit_claim.amount_as_int,
)
elif deposit_claim.concent_use_case == ConcentUseCase.ADDITIONAL_VERIFICATION:
if subtask is not None:
if task_to_compute.requestor_ethereum_address == deposit_claim.payer_deposit_account.ethereum_address:
ethereum_transaction_hash = service.force_subtask_payment( # pylint: disable=no-value-for-parameter
requestor_eth_address=deposit_claim.payer_deposit_account.
ethereum_address,
provider_eth_address=deposit_claim.payee_ethereum_address,
value=task_to_compute.price,
subtask_id=deposit_claim.subtask_id,
v=v,
r=r,
s=s,
reimburse_amount=deposit_claim.amount_as_int,
)
elif task_to_compute.provider_ethereum_address == deposit_claim.payer_deposit_account.ethereum_address:
subtask_results_verify: SubtaskResultsVerify = deserialize_message(
subtask.subtask_results_verify.data.tobytes())
(v, r, s) = subtask_results_verify.concent_promissory_note_sig
ethereum_transaction_hash = service.cover_additional_verification_cost( # pylint: disable=no-value-for-parameter
provider_eth_address=deposit_claim.payer_deposit_account.
ethereum_address,
value=subtask_results_verify.task_to_compute.price,
subtask_id=deposit_claim.subtask_id,
v=v,
r=r,
s=s,
reimburse_amount=deposit_claim.amount_as_int,
)
else:
assert False
else:
assert False
with non_nesting_atomic(using='control'):
# The code below is executed in another transaction, so - in theory - deposit_claim object could be modified in
# the meantime. Here we are working under assumption that it's not the case and it is coder's responsibility to
# ensure that.
ethereum_transaction_hash = adjust_transaction_hash(
ethereum_transaction_hash)
deposit_claim.tx_hash = ethereum_transaction_hash
deposit_claim.full_clean()
deposit_claim.save()
service.register_confirmed_transaction_handler( # pylint: disable=no-value-for-parameter
tx_hash=deposit_claim.tx_hash,
callback=lambda _: discard_claim(deposit_claim),
)
return deposit_claim.tx_hash
def upload_finished(subtask_id: str) -> None:
try:
subtask = Subtask.objects.select_for_update().get(subtask_id=subtask_id)
except Subtask.DoesNotExist:
logging.log(
logger,
f'Task `upload_finished` tried to get Subtask object, but it does not exist.',
subtask_id=subtask_id,
logging_level=logging.LoggingLevel.ERROR,
)
return
report_computed_task = deserialize_message(subtask.report_computed_task.data.tobytes())
# Check subtask state, if it's VERIFICATION FILE TRANSFER, proceed with the task.
if subtask.state_enum == Subtask.SubtaskState.VERIFICATION_FILE_TRANSFER:
# If subtask is past the deadline, processes the timeout.
if parse_datetime_to_timestamp(subtask.next_deadline) < get_current_utc_timestamp():
# Worker makes a payment from requestor's deposit just like in the forced acceptance use case.
def finalize_claims() -> None:
finalize_deposit_claim(
subtask_id=subtask_id,
concent_use_case=ConcentUseCase.ADDITIONAL_VERIFICATION,
ethereum_address=report_computed_task.task_to_compute.requestor_ethereum_address,
)
finalize_deposit_claim(
subtask_id=subtask_id,
concent_use_case=ConcentUseCase.ADDITIONAL_VERIFICATION,
ethereum_address=report_computed_task.task_to_compute.provider_ethereum_address,
)
transaction.on_commit(
finalize_claims,
using='control',
)
update_subtask_state(
subtask=subtask,
state=Subtask.SubtaskState.FAILED.name, # pylint: disable=no-member
)
# Worker adds SubtaskResultsSettled to provider's and requestor's receive queues (both out-of-band)
for public_key in [subtask.provider.public_key_bytes, subtask.requestor.public_key_bytes]:
store_pending_message(
response_type=PendingResponse.ResponseType.SubtaskResultsSettled,
client_public_key=public_key,
queue=PendingResponse.Queue.ReceiveOutOfBand,
subtask=subtask,
)
return
# Change subtask state to ADDITIONAL VERIFICATION.
update_subtask_state(
subtask=subtask,
state=Subtask.SubtaskState.ADDITIONAL_VERIFICATION.name, # pylint: disable=no-member
next_deadline=(
parse_datetime_to_timestamp(subtask.next_deadline) +
calculate_concent_verification_time(report_computed_task.task_to_compute)
)
)
# Add upload_acknowledged task to the work queue.
tasks.upload_acknowledged.delay(
subtask_id=subtask_id,
source_file_size=report_computed_task.task_to_compute.size,
source_package_hash=report_computed_task.task_to_compute.package_hash,
result_file_size=report_computed_task.size,
result_package_hash=report_computed_task.package_hash,
)
# If it's ADDITIONAL VERIFICATION, ACCEPTED or FAILED, log a warning and ignore the notification.
# Processing ends here. This means that it's a duplicate notification.
elif subtask.state_enum in [
Subtask.SubtaskState.ADDITIONAL_VERIFICATION,
Subtask.SubtaskState.ACCEPTED,
Subtask.SubtaskState.FAILED
]:
logging.log(
logger,
f'Subtask is expected to be in `VERIFICATION_FILE_TRANSFER` state, but was in {subtask.state}.',
subtask_id=subtask_id,
logging_level=logging.LoggingLevel.WARNING,
)
# If it's one of the states that can precede verification, report an error. Processing ends here.
else:
logging.log(
logger,
f'Subtask is expected to be in `VERIFICATION_FILE_TRANSFER` state, but was in {subtask.state}.',
subtask_id=subtask_id,
logging_level=logging.LoggingLevel.ERROR,
)
def verification_result(
self: Task,
subtask_id: str,
result: str,
error_message: Optional[str] = None,
error_code: Optional[str] = None,
) -> None:
logging.log(
logger,
f'Verification_result_task starts. Result: {result}',
subtask_id=subtask_id
)
assert isinstance(subtask_id, str)
assert isinstance(result, str)
assert result in VerificationResult.__members__.keys()
assert isinstance(error_message, (str, type(None)))
assert isinstance(error_code, (str, type(None)))
result_enum = VerificationResult[result]
assert result_enum != VerificationResult.ERROR or all([isinstance(error_message, str), isinstance(error_code, str)])
# Worker locks database row corresponding to the subtask in the subtask table.
try:
subtask = Subtask.objects.select_for_update(nowait=True).get(subtask_id=subtask_id)
except DatabaseError:
logging.log(
logger,
f'Row in database corresponding with Subtask object is already locked.'
f'retrying task {self.request.retries}/{self.max_retries}',
subtask_id=subtask_id,
logging_level=logging.LoggingLevel.WARNING,
)
# If the row is already locked, task fails so that Celery can retry later.
self.retry(
countdown=CELERY_LOCKED_SUBTASK_DELAY,
max_retries=MAXIMUM_VERIFICATION_RESULT_TASK_RETRIES,
throw=False,
)
return
if subtask.state_enum == Subtask.SubtaskState.ACCEPTED:
logging.log(
logger,
VERIFICATION_RESULT_SUBTASK_STATE_ACCEPTED_LOG_MESSAGE.format(subtask_id),
subtask_id=subtask_id,
logging_level=logging.LoggingLevel.WARNING
)
return
elif subtask.state_enum == Subtask.SubtaskState.FAILED:
logging.log(
logger,
VERIFICATION_RESULT_SUBTASK_STATE_FAILED_LOG_MESSAGE.format(subtask_id),
subtask_id=subtask_id,
logging_level=logging.LoggingLevel.WARNING
)
return
elif subtask.state_enum != Subtask.SubtaskState.ADDITIONAL_VERIFICATION:
logging.log(
logger,
VERIFICATION_RESULT_SUBTASK_STATE_UNEXPECTED_LOG_MESSAGE.format(subtask.state),
subtask_id=subtask_id,
logging_level=logging.LoggingLevel.ERROR,
)
return
# If the time is already past next_deadline for the subtask (SubtaskResultsRejected.timestamp + AVCT)
# worker ignores worker's message and processes the timeout.
if subtask.next_deadline < parse_timestamp_to_utc_datetime(get_current_utc_timestamp()):
task_to_compute = deserialize_message(subtask.task_to_compute.data.tobytes())
# Worker makes a payment from requestor's deposit just like in the forced acceptance use case.
def finalize_claims() -> None:
finalize_deposit_claim(
subtask_id=subtask_id,
concent_use_case=ConcentUseCase.ADDITIONAL_VERIFICATION,
ethereum_address=task_to_compute.requestor_ethereum_address,
)
finalize_deposit_claim(
subtask_id=subtask_id,
concent_use_case=ConcentUseCase.ADDITIONAL_VERIFICATION,
ethereum_address=task_to_compute.provider_ethereum_address,
)
transaction.on_commit(
finalize_claims,
using='control',
)
update_subtask_state(
subtask=subtask,
state=Subtask.SubtaskState.ACCEPTED.name, # pylint: disable=no-member
)
for public_key in [subtask.provider.public_key_bytes, subtask.requestor.public_key_bytes]:
store_pending_message(
response_type=PendingResponse.ResponseType.SubtaskResultsSettled,
client_public_key=public_key,
queue=PendingResponse.Queue.ReceiveOutOfBand,
subtask=subtask,
)
return
if result_enum == VerificationResult.MISMATCH:
# Worker adds SubtaskResultsRejected to provider's and requestor's receive queues (both out-of-band)
for public_key in [subtask.provider.public_key_bytes, subtask.requestor.public_key_bytes]:
store_pending_message(
response_type=PendingResponse.ResponseType.SubtaskResultsRejected,
client_public_key=public_key,
queue=PendingResponse.Queue.ReceiveOutOfBand,
subtask=subtask,
)
task_to_compute = deserialize_message(subtask.task_to_compute.data.tobytes())
def finalize_claims() -> None: # pylint: disable=function-redefined
delete_deposit_claim(
subtask_id=subtask_id,
concent_use_case=ConcentUseCase.ADDITIONAL_VERIFICATION,
ethereum_address=task_to_compute.requestor_ethereum_address,
)
finalize_deposit_claim(
subtask_id=subtask_id,
concent_use_case=ConcentUseCase.ADDITIONAL_VERIFICATION,
ethereum_address=task_to_compute.provider_ethereum_address,
)
transaction.on_commit(
finalize_claims,
using='control',
)
# Worker changes subtask state to FAILED
subtask.state = Subtask.SubtaskState.FAILED.name # pylint: disable=no-member
subtask.next_deadline = None
subtask.full_clean()
subtask.save()
elif result_enum in (VerificationResult.MATCH, VerificationResult.ERROR):
# Worker logs the error code and message
if result_enum == VerificationResult.ERROR:
logging.log(
logger,
f'Verification_result_task processing error result with: RESULT {result_enum.name}. ERROR MESSAGE {error_message}. ERROR CODE {error_code}',
subtask_id=subtask_id,
)
task_to_compute = deserialize_message(subtask.task_to_compute.data.tobytes())
# Worker makes a payment from requestor's deposit just like in the forced acceptance use case.
def finalize_claims() -> None: # pylint: disable=function-redefined
finalize_deposit_claim(
subtask_id=subtask_id,
concent_use_case=ConcentUseCase.ADDITIONAL_VERIFICATION,
ethereum_address=task_to_compute.requestor_ethereum_address,
)
finalize_deposit_claim(
subtask_id=subtask_id,
concent_use_case=ConcentUseCase.ADDITIONAL_VERIFICATION,
ethereum_address=task_to_compute.provider_ethereum_address,
)
transaction.on_commit(
finalize_claims,
using='control',
)
# Worker adds SubtaskResultsSettled to provider's and requestor's receive queues (both out-of-band)
for public_key in [subtask.provider.public_key_bytes, subtask.requestor.public_key_bytes]:
store_pending_message(
response_type=PendingResponse.ResponseType.SubtaskResultsSettled,
client_public_key=public_key,
queue=PendingResponse.Queue.ReceiveOutOfBand,
subtask=subtask,
)
# Worker changes subtask state to ACCEPTED
update_subtask_state(
subtask=subtask,
state=Subtask.SubtaskState.ACCEPTED.name, # pylint: disable=no-member
)
logging.log(
logger,
f'Verification_result_task ends. Result: {result_enum.name}',
subtask_id=subtask_id
)
def clean(self):
super().clean()
# Concent should not accept anything that cause a transition to an active state in soft shutdown mode.
if config.SOFT_SHUTDOWN_MODE is True and self.state_enum in self.ACTIVE_STATES:
raise ConcentInSoftShutdownMode
# next_deadline must be datetime only for active states
if (
not self._state.adding and
not isinstance(self.next_deadline, datetime.datetime) and
self.state_enum in self.ACTIVE_STATES
):
raise ValidationError({
'next_deadline': 'next_deadline must be datetime for active state.'
})
# next_deadline must be None in passive states
if not self._state.adding and self.next_deadline is not None and self.state_enum in self.PASSIVE_STATES:
raise ValidationError({
'next_deadline': 'next_deadline must be None for passive state.'
})
# State transition can happen only by defined rule
# but not when we create new object
# and not when state is not being changed
if (
not self._state.adding and
self._current_state_enum != self.state_enum and
self._current_state_enum not in self.POSSIBLE_TRANSITIONS_TO[self.state_enum]
):
raise ValidationError({
'state': 'Subtask cannot change its state from {} to {}.'.format(
self._current_state_name,
self.state,
)
})
else:
self._current_state_name = self.state
# Both ack_report_computed_task and reject_report_computed_task cannot set at the same time.
if self.ack_report_computed_task is not None and self.reject_report_computed_task is not None:
raise ValidationError(
'Both ack_report_computed_task and reject_report_computed_task cannot be set at the same time.'
)
# Requestor and provider cannot be the same clients
if self.requestor_id == self.provider_id:
raise ValidationError('Requestor and provided are the same client.')
# Check if all required related messages are not None in current state.
for stored_message_name, states in Subtask.REQUIRED_RELATED_MESSAGES_IN_STATES.items():
if self.state_enum in states and getattr(self, stored_message_name) is None:
raise ValidationError({
stored_message_name: '{} cannot be None in state {}.'.format(
stored_message_name,
self.state,
)
})
# Check if all related messages which must be None are None in current state.
for stored_message_name, states in Subtask.UNSET_RELATED_MESSAGES_IN_STATES.items():
if self.state_enum in states and getattr(self, stored_message_name) is not None:
raise ValidationError({
stored_message_name: '{} must be None in state {}.'.format(
stored_message_name,
self.state,
)
})
if isinstance(self.report_computed_task.data, bytes):
deserialized_report_computed_task = deserialize_message(self.report_computed_task.data)
else:
deserialized_report_computed_task = deserialize_message(self.report_computed_task.data.tobytes()) # pylint: disable=no-member
# If available, the report_computed_task nested in force_get_task_result must match report_computed_task.
if (
self.force_get_task_result is not None and
deserialize_message(self.force_get_task_result.data).report_computed_task != deserialized_report_computed_task
):
raise ValidationError({
'force_get_task_result': "ReportComputedTask nested in ForceGetTaskResult must match Subtask's ReportComputedTask."
})
if not self.result_package_size == deserialized_report_computed_task.size:
raise ValidationError({
'result_package_size': "ReportComputedTask size mismatch"
})
if isinstance(self.task_to_compute.data, bytes):
deserialized_task_to_compute = deserialize_message(self.task_to_compute.data)
else:
deserialized_task_to_compute = deserialize_message(self.task_to_compute.data.tobytes()) # pylint: disable=no-member
if not self.computation_deadline.timestamp() == deserialized_task_to_compute.compute_task_def['deadline']:
raise ValidationError({
'computation_deadline': "TaskToCompute deadline mismatch"
})
def _update_timed_out_subtask(subtask: Subtask) -> None:
"""
Function called for timed out subtasks - checks state and changes it from one of actives to one of passives
"""
subtasks_initial_state = subtask.state
if subtask.state == Subtask.SubtaskState.FORCING_REPORT.name: # pylint: disable=no-member
update_subtask_state(
subtask=subtask,
state=Subtask.SubtaskState.REPORTED.name, # pylint: disable=no-member
)
store_pending_message(
response_type=PendingResponse.ResponseType.
ForceReportComputedTaskResponse,
client_public_key=subtask.provider.public_key_bytes,
queue=PendingResponse.Queue.Receive,
subtask=subtask,
)
store_pending_message(
response_type=PendingResponse.ResponseType.
VerdictReportComputedTask,
client_public_key=subtask.requestor.public_key_bytes,
queue=PendingResponse.Queue.ReceiveOutOfBand,
subtask=subtask,
)
elif subtask.state == Subtask.SubtaskState.FORCING_RESULT_TRANSFER.name: # pylint: disable=no-member
update_subtask_state(
subtask=subtask,
state=Subtask.SubtaskState.FAILED.name, # pylint: disable=no-member
)
store_pending_message(
response_type=PendingResponse.ResponseType.
ForceGetTaskResultFailed,
client_public_key=subtask.requestor.public_key_bytes,
queue=PendingResponse.Queue.Receive,
subtask=subtask,
)
elif subtask.state == Subtask.SubtaskState.FORCING_ACCEPTANCE.name: # pylint: disable=no-member
task_to_compute = deserialize_message(
subtask.task_to_compute.data.tobytes())
def finalize_claim_for_acceptance_case() -> None:
finalize_deposit_claim(
subtask_id=subtask.subtask_id,
concent_use_case=ConcentUseCase.FORCED_ACCEPTANCE,
ethereum_address=task_to_compute.requestor_ethereum_address,
)
transaction.on_commit(
finalize_claim_for_acceptance_case,
using='control',
)
update_subtask_state(
subtask=subtask,
state=Subtask.SubtaskState.ACCEPTED.name, # pylint: disable=no-member
)
store_pending_message(
response_type=PendingResponse.ResponseType.SubtaskResultsSettled,
client_public_key=subtask.provider.public_key_bytes,
queue=PendingResponse.Queue.Receive,
subtask=subtask,
)
store_pending_message(
response_type=PendingResponse.ResponseType.SubtaskResultsSettled,
client_public_key=subtask.requestor.public_key_bytes,
queue=PendingResponse.Queue.ReceiveOutOfBand,
subtask=subtask,
)
elif subtask.state == Subtask.SubtaskState.VERIFICATION_FILE_TRANSFER.name: # pylint: disable=no-member
update_subtask_state(
subtask=subtask,
state=Subtask.SubtaskState.FAILED.name, # pylint: disable=no-member
)
store_pending_message(
response_type=PendingResponse.ResponseType.SubtaskResultsRejected,
client_public_key=subtask.provider.public_key_bytes,
queue=PendingResponse.Queue.ReceiveOutOfBand,
subtask=subtask,
)
store_pending_message(
response_type=PendingResponse.ResponseType.SubtaskResultsRejected,
client_public_key=subtask.requestor.public_key_bytes,
queue=PendingResponse.Queue.ReceiveOutOfBand,
subtask=subtask,
)
elif subtask.state == Subtask.SubtaskState.ADDITIONAL_VERIFICATION.name: # pylint: disable=no-member
task_to_compute = deserialize_message(
subtask.task_to_compute.data.tobytes())
def finalize_claim_for_additional_verification_case() -> None:
finalize_deposit_claim(
subtask_id=subtask.subtask_id,
concent_use_case=ConcentUseCase.ADDITIONAL_VERIFICATION,
ethereum_address=task_to_compute.requestor_ethereum_address,
)
finalize_deposit_claim(
subtask_id=subtask.subtask_id,
concent_use_case=ConcentUseCase.ADDITIONAL_VERIFICATION,
ethereum_address=task_to_compute.provider_ethereum_address,
)
transaction.on_commit(
finalize_claim_for_additional_verification_case,
using='control',
)
update_subtask_state(
subtask=subtask,
state=Subtask.SubtaskState.ACCEPTED.name, # pylint: disable=no-member
)
store_pending_message(
response_type=PendingResponse.ResponseType.SubtaskResultsSettled,
client_public_key=subtask.provider.public_key_bytes,
queue=PendingResponse.Queue.ReceiveOutOfBand,
subtask=subtask,
)
store_pending_message(
response_type=PendingResponse.ResponseType.SubtaskResultsSettled,
client_public_key=subtask.requestor.public_key_bytes,
queue=PendingResponse.Queue.ReceiveOutOfBand,
subtask=subtask,
)
log(logger,
f"Subtask changed it's state from: {subtasks_initial_state} to: {subtask.state}. "
f"Provider id: {subtask.provider_id}. Requestor id: {subtask.requestor_id}.",
subtask_id=subtask.subtask_id)
def clean(self) -> None:
super().clean()
# Concent should not accept anything that cause a transition to an active state in soft shutdown mode.
if config.SOFT_SHUTDOWN_MODE is True and self.state_enum in self.ACTIVE_STATES:
raise ConcentInSoftShutdownMode
# next_deadline must be datetime only for active states
if (not self._state.adding
and not isinstance(self.next_deadline, datetime.datetime)
and self.state_enum in self.ACTIVE_STATES):
raise ValidationError({
'next_deadline':
'next_deadline must be datetime for active state.'
})
# next_deadline must be None in passive states
if not self._state.adding and self.next_deadline is not None and self.state_enum in self.PASSIVE_STATES:
raise ValidationError({
'next_deadline':
'next_deadline must be None for passive state.'
})
# State transition can happen only by defined rule
# but not when we create new object
# and not when state is not being changed
if (not self._state.adding
and self._current_state_enum != self.state_enum
and self._current_state_enum not in
self.POSSIBLE_TRANSITIONS_TO[self.state_enum] # type: ignore
):
raise ValidationError({
'state':
'Subtask cannot change its state from {} to {}.'.format(
self._current_state_name,
self.state,
)
})
else:
self._current_state_name = self.state
# Both ack_report_computed_task and reject_report_computed_task cannot set at the same time.
if self.ack_report_computed_task is not None and self.reject_report_computed_task is not None:
raise ValidationError(
'Both ack_report_computed_task and reject_report_computed_task cannot be set at the same time.'
)
# Requestor and provider cannot be the same clients
if self.requestor_id == self.provider_id:
raise ValidationError(
'Requestor and provided are the same client.')
# Check if all required related messages are not None in current state.
for stored_message_name, states in Subtask.REQUIRED_RELATED_MESSAGES_IN_STATES.items(
):
if self.state_enum in states and getattr(
self, stored_message_name) is None:
raise ValidationError({
stored_message_name:
'{} cannot be None in state {}.'.format(
stored_message_name,
self.state,
)
})
# Check if all related messages which must be None are None in current state.
for stored_message_name, states in Subtask.UNSET_RELATED_MESSAGES_IN_STATES.items(
):
if self.state_enum in states and getattr(
self, stored_message_name) is not None:
raise ValidationError({
stored_message_name:
'{} must be None in state {}.'.format(
stored_message_name,
self.state,
)
})
if isinstance(self.report_computed_task.data, bytes):
deserialized_report_computed_task = deserialize_message(
self.report_computed_task.data)
else:
deserialized_report_computed_task = deserialize_message(
self.report_computed_task.data.tobytes()) # pylint: disable=no-member
# If available, the report_computed_task nested in force_get_task_result must match report_computed_task.
if (self.force_get_task_result is not None and deserialize_message(
self.force_get_task_result.data).report_computed_task !=
deserialized_report_computed_task):
raise ValidationError({
'force_get_task_result':
"ReportComputedTask nested in ForceGetTaskResult must match Subtask's ReportComputedTask."
})
if not self.result_package_size == deserialized_report_computed_task.size:
raise ValidationError(
{'result_package_size': "ReportComputedTask size mismatch"})
if isinstance(self.task_to_compute.data, bytes):
deserialized_task_to_compute = deserialize_message(
self.task_to_compute.data)
else:
deserialized_task_to_compute = deserialize_message(
self.task_to_compute.data.tobytes()) # pylint: disable=no-member
if not parse_datetime_to_timestamp(
self.computation_deadline
) == deserialized_task_to_compute.compute_task_def['deadline']:
raise ValidationError(
{'computation_deadline': "TaskToCompute deadline mismatch"})
# Validation for every nested message which is stored in Control database
# Every nested message must be the same as message stored separately.
MESSAGES_TO_VALIDATE_TASK_TO_COMPUTE = [
self.report_computed_task,
self.subtask_results_accepted,
self.reject_report_computed_task,
]
MESSAGES_TO_VALIDATE_REPORT_COMPUTED_TASK = [
self.ack_report_computed_task,
self.subtask_results_rejected,
self.force_get_task_result,
]
for task_to_compute_to_validate in MESSAGES_TO_VALIDATE_TASK_TO_COMPUTE:
if task_to_compute_to_validate is not None:
validate_database_task_to_compute(
task_to_compute=deserialized_task_to_compute,
message_to_compare=deserialize_database_message(
task_to_compute_to_validate),
)
for report_computed_task_to_validate in MESSAGES_TO_VALIDATE_REPORT_COMPUTED_TASK:
if report_computed_task_to_validate is not None:
validate_database_report_computed_task(
report_computed_task=deserialized_report_computed_task,
message_to_compare=deserialize_database_message(
report_computed_task_to_validate),
)
for related_message_name in Subtask.MESSAGE_FOR_FIELD:
related_message = getattr(self, related_message_name)
assert isinstance(related_message,
StoredMessage) or related_message is None
if related_message is not None and related_message.protocol_version != settings.GOLEM_MESSAGES_VERSION:
raise ValidationError(
f'Unsupported Golem Message version. Version in: `{related_message_name}` is {related_message.protocol_version}, '
f'Version in Concent is {settings.GOLEM_MESSAGES_VERSION}')
def upload_finished(subtask_id: str):
try:
subtask = Subtask.objects.get(subtask_id=subtask_id)
except Subtask.DoesNotExist:
logging.error(
f'Task `upload_finished` tried to get Subtask object with ID {subtask_id} but it does not exist.'
)
return
report_computed_task = deserialize_message(
subtask.report_computed_task.data.tobytes())
# Check subtask state, if it's VERIFICATION FILE TRANSFER, proceed with the task.
if subtask.state_enum == Subtask.SubtaskState.VERIFICATION_FILE_TRANSFER:
# If subtask is past the deadline, processes the timeout.
if subtask.next_deadline.timestamp() < get_current_utc_timestamp():
# Worker makes a payment from requestor's deposit just like in the forced acceptance use case.
payments_service.make_force_payment_to_provider( # pylint: disable=no-value-for-parameter
requestor_eth_address=report_computed_task.task_to_compute.
requestor_ethereum_address,
provider_eth_address=report_computed_task.task_to_compute.
provider_ethereum_address,
value=report_computed_task.task_to_compute.price,
payment_ts=get_current_utc_timestamp(),
)
update_subtask_state(
subtask=subtask,
state=Subtask.SubtaskState.FAILED.name, # pylint: disable=no-member
)
# Worker adds SubtaskResultsSettled to provider's and requestor's receive queues (both out-of-band)
for public_key in [
subtask.provider.public_key_bytes,
subtask.requestor.public_key_bytes
]:
store_pending_message(
response_type=PendingResponse.ResponseType.
SubtaskResultsSettled,
client_public_key=public_key,
queue=PendingResponse.Queue.ReceiveOutOfBand,
subtask=subtask,
)
return
# Change subtask state to ADDITIONAL VERIFICATION.
update_subtask_state(
subtask=subtask,
state=Subtask.SubtaskState.ADDITIONAL_VERIFICATION.name, # pylint: disable=no-member
next_deadline=int(subtask.next_deadline.timestamp()) +
calculate_subtask_verification_time(report_computed_task))
# Add upload_acknowledged task to the work queue.
tasks.upload_acknowledged.delay(
subtask_id=subtask_id,
source_file_size=report_computed_task.task_to_compute.size,
source_package_hash=report_computed_task.task_to_compute.
package_hash,
result_file_size=report_computed_task.size,
result_package_hash=report_computed_task.package_hash,
)
# If it's ADDITIONAL VERIFICATION, ACCEPTED or FAILED, log a warning and ignore the notification.
# Processing ends here. This means that it's a duplicate notification.
elif subtask.state_enum in [
Subtask.SubtaskState.ADDITIONAL_VERIFICATION,
Subtask.SubtaskState.ACCEPTED, Subtask.SubtaskState.FAILED
]:
logging.warning(
f'Subtask with ID {subtask_id} is expected to be in `VERIFICATION_FILE_TRANSFER` state, but was in {subtask.state}.'
)
# If it's one of the states that can precede verification, report an error. Processing ends here.
else:
logging.error(
f'Subtask with ID {subtask_id} is expected to be in `VERIFICATION_FILE_TRANSFER` state, but was in {subtask.state}.'
)
def verification_result(
self,
subtask_id: str,
result: str,
error_message: Optional[str] = None,
error_code: Optional[str] = None,
):
logger.info(
f'verification_result_task starts with: SUBTASK_ID {subtask_id} -- RESULT {result}'
)
assert isinstance(subtask_id, str)
assert isinstance(result, str)
assert result in VerificationResult.__members__.keys()
assert isinstance(error_message, (str, type(None)))
assert isinstance(error_code, (str, type(None)))
result_enum = VerificationResult[result]
assert result_enum != VerificationResult.ERROR or all(
[error_message, error_code])
# Worker locks database row corresponding to the subtask in the subtask table.
try:
subtask = Subtask.objects.select_for_update(nowait=True).get(
subtask_id=subtask_id)
except DatabaseError:
logging.warning(
f'Subtask object with ID {subtask_id} database row is locked, '
f'retrying task {self.request.retries}/{self.max_retries}')
# If the row is already locked, task fails so that Celery can retry later.
self.retry(
countdown=CELERY_LOCKED_SUBTASK_DELAY,
max_retries=MAXIMUM_VERIFICATION_RESULT_TASK_RETRIES,
throw=False,
)
return
if subtask.state_enum == Subtask.SubtaskState.ACCEPTED:
logger.warning(
VERIFICATION_RESULT_SUBTASK_STATE_ACCEPTED_LOG_MESSAGE.format(
subtask_id))
return
elif subtask.state_enum == Subtask.SubtaskState.FAILED:
logger.warning(
VERIFICATION_RESULT_SUBTASK_STATE_FAILED_LOG_MESSAGE.format(
subtask_id))
return
elif subtask.state_enum != Subtask.SubtaskState.ADDITIONAL_VERIFICATION:
logger.error(
VERIFICATION_RESULT_SUBTASK_STATE_UNEXPECTED_LOG_MESSAGE.format(
subtask_id, subtask.state))
return
# If the time is already past next_deadline for the subtask (SubtaskResultsRejected.timestamp + AVCT)
# worker ignores worker's message and processes the timeout.
if subtask.next_deadline < parse_timestamp_to_utc_datetime(
get_current_utc_timestamp()):
task_to_compute = deserialize_message(
subtask.task_to_compute.data.tobytes())
# Worker makes a payment from requestor's deposit just like in the forced acceptance use case.
payments_service.make_force_payment_to_provider( # pylint: disable=no-value-for-parameter
requestor_eth_address=task_to_compute.requestor_ethereum_address,
provider_eth_address=task_to_compute.provider_ethereum_address,
value=task_to_compute.price,
payment_ts=get_current_utc_timestamp(),
)
update_subtask_state(
subtask=subtask,
state=Subtask.SubtaskState.ACCEPTED.name, # pylint: disable=no-member
)
for public_key in [
subtask.provider.public_key_bytes,
subtask.requestor.public_key_bytes
]:
store_pending_message(
response_type=PendingResponse.ResponseType.
SubtaskResultsSettled,
client_public_key=public_key,
queue=PendingResponse.Queue.ReceiveOutOfBand,
subtask=subtask,
)
return
if result_enum == VerificationResult.MISMATCH:
# Worker adds SubtaskResultsRejected to provider's and requestor's receive queues (both out-of-band)
for public_key in [
subtask.provider.public_key_bytes,
subtask.requestor.public_key_bytes
]:
store_pending_message(
response_type=PendingResponse.ResponseType.
SubtaskResultsRejected,
client_public_key=public_key,
queue=PendingResponse.Queue.ReceiveOutOfBand,
subtask=subtask,
)
# Worker changes subtask state to FAILED
subtask.state = Subtask.SubtaskState.FAILED.name # pylint: disable=no-member
subtask.next_deadline = None
subtask.full_clean()
subtask.save()
elif result_enum in (VerificationResult.MATCH, VerificationResult.ERROR):
# Worker logs the error code and message
if result_enum == VerificationResult.ERROR:
logger.info(
f'verification_result_task processing error result with: '
f'SUBTASK_ID {subtask_id} -- RESULT {result_enum.name} -- ERROR MESSAGE {error_message} -- ERROR CODE {error_code}'
)
task_to_compute = deserialize_message(
subtask.task_to_compute.data.tobytes())
# Worker makes a payment from requestor's deposit just like in the forced acceptance use case.
payments_service.make_force_payment_to_provider( # pylint: disable=no-value-for-parameter
requestor_eth_address=task_to_compute.requestor_ethereum_address,
provider_eth_address=task_to_compute.provider_ethereum_address,
value=task_to_compute.price,
payment_ts=get_current_utc_timestamp(),
)
# Worker adds SubtaskResultsSettled to provider's and requestor's receive queues (both out-of-band)
for public_key in [
subtask.provider.public_key_bytes,
subtask.requestor.public_key_bytes
]:
store_pending_message(
response_type=PendingResponse.ResponseType.
SubtaskResultsSettled,
client_public_key=public_key,
queue=PendingResponse.Queue.ReceiveOutOfBand,
subtask=subtask,
)
# Worker changes subtask state to ACCEPTED
update_subtask_state(
subtask=subtask,
state=Subtask.SubtaskState.ACCEPTED.name, # pylint: disable=no-member
)
logger.info(
f'verification_result_task ends with: SUBTASK_ID {subtask_id} -- RESULT {result_enum.name}'
)
def finalize_payment(deposit_claim: DepositClaim) -> Optional[str]:
"""
This operation tells Bankster to pay out funds from deposit.
For each claim, Bankster uses SCI to submit an Ethereum transaction to the Ethereum client which then propagates it
to the rest of the network.
Hopefully the transaction is included in one of the upcoming blocks on the blockchain.
"""
assert isinstance(deposit_claim, DepositClaim)
# Bankster asks SCI about the amount of funds available on the deposit account listed in the DepositClaim.
available_funds = service.get_deposit_value( # pylint: disable=no-value-for-parameter
client_eth_address=deposit_claim.payer_deposit_account.ethereum_address
)
# Bankster begins a database transaction and puts a database lock on the DepositAccount object.
with transaction.atomic(using='control'):
DepositAccount.objects.select_for_update().get(
pk=deposit_claim.payer_deposit_account_id)
# Bankster sums the amounts of all existing DepositClaims that have the same payer as the one being processed.
aggregated_client_claims = DepositClaim.objects.filter(
payer_deposit_account=deposit_claim.payer_deposit_account).exclude(
pk=deposit_claim.pk).aggregate(
sum_of_existing_claims=Coalesce(Sum('amount'), 0))
# Bankster subtracts that value from the amount of funds available in the deposit.
available_funds_without_claims = available_funds - aggregated_client_claims[
'sum_of_existing_claims']
# If the result is negative or zero, Bankster removes the DepositClaim object being processed.
if available_funds_without_claims <= 0:
deposit_claim.delete()
return None
# Otherwise if the result is lower than DepositAccount.amount,
# Bankster sets this field to the amount that's actually available.
elif available_funds_without_claims < deposit_claim.amount:
deposit_claim.amount = available_funds_without_claims
# If the DepositClaim still exists at this point, Bankster uses SCI to create an Ethereum transaction.
if deposit_claim.concent_use_case == ConcentUseCase.FORCED_ACCEPTANCE:
ethereum_transaction_hash = service.force_subtask_payment( # pylint: disable=no-value-for-parameter
requestor_eth_address=deposit_claim.payer_deposit_account.
ethereum_address,
provider_eth_address=deposit_claim.payee_ethereum_address,
value=deposit_claim.amount,
subtask_id=deposit_claim.subtask_id,
)
elif deposit_claim.concent_use_case == ConcentUseCase.ADDITIONAL_VERIFICATION:
subtask = Subtask.objects.filter(
subtask_id=deposit_claim.subtask_id).first() # pylint: disable=no-member
if subtask is not None:
task_to_compute = deserialize_message(
subtask.task_to_compute.data.tobytes())
if task_to_compute.requestor_ethereum_address == deposit_claim.payer_deposit_account.ethereum_address:
ethereum_transaction_hash = service.force_subtask_payment( # pylint: disable=no-value-for-parameter
requestor_eth_address=deposit_claim.
payer_deposit_account.ethereum_address,
provider_eth_address=deposit_claim.
payee_ethereum_address,
value=deposit_claim.amount,
subtask_id=deposit_claim.subtask_id,
)
elif task_to_compute.provider_ethereum_address == deposit_claim.payer_deposit_account.ethereum_address:
ethereum_transaction_hash = service.cover_additional_verification_cost( # pylint: disable=no-value-for-parameter
provider_eth_address=deposit_claim.
payer_deposit_account.ethereum_address,
value=deposit_claim.amount,
subtask_id=deposit_claim.subtask_id,
)
else:
assert False
else:
assert False
# Bankster puts transaction ID in DepositClaim.tx_hash.
ethereum_transaction_hash = adjust_transaction_hash(
ethereum_transaction_hash)
deposit_claim.tx_hash = ethereum_transaction_hash
deposit_claim.full_clean()
deposit_claim.save()
service.call_on_confirmed_transaction( # pylint: disable=no-value-for-parameter
tx_hash=deposit_claim.tx_hash,
callback=lambda _: discard_claim(deposit_claim),
)
return deposit_claim.tx_hash