def upload_finished(subtask_id: str) -> None:
try:
subtask = Subtask.objects.select_for_update().get(subtask_id=subtask_id)
except Subtask.DoesNotExist:
logging.log(
logger,
f'Task `upload_finished` tried to get Subtask object, but it does not exist.',
subtask_id=subtask_id,
logging_level=logging.LoggingLevel.ERROR,
)
return
report_computed_task = deserialize_message(subtask.report_computed_task.data.tobytes())
# Check subtask state, if it's VERIFICATION FILE TRANSFER, proceed with the task.
if subtask.state_enum == Subtask.SubtaskState.VERIFICATION_FILE_TRANSFER:
# If subtask is past the deadline, processes the timeout.
if parse_datetime_to_timestamp(subtask.next_deadline) < get_current_utc_timestamp():
# Worker makes a payment from requestor's deposit just like in the forced acceptance use case.
def finalize_claims() -> None:
finalize_deposit_claim(
subtask_id=subtask_id,
concent_use_case=ConcentUseCase.ADDITIONAL_VERIFICATION,
ethereum_address=report_computed_task.task_to_compute.requestor_ethereum_address,
)
finalize_deposit_claim(
subtask_id=subtask_id,
concent_use_case=ConcentUseCase.ADDITIONAL_VERIFICATION,
ethereum_address=report_computed_task.task_to_compute.provider_ethereum_address,
)
transaction.on_commit(
finalize_claims,
using='control',
)
update_subtask_state(
subtask=subtask,
state=Subtask.SubtaskState.FAILED.name, # pylint: disable=no-member
)
# Worker adds SubtaskResultsSettled to provider's and requestor's receive queues (both out-of-band)
for public_key in [subtask.provider.public_key_bytes, subtask.requestor.public_key_bytes]:
store_pending_message(
response_type=PendingResponse.ResponseType.SubtaskResultsSettled,
client_public_key=public_key,
queue=PendingResponse.Queue.ReceiveOutOfBand,
subtask=subtask,
)
return
# Change subtask state to ADDITIONAL VERIFICATION.
update_subtask_state(
subtask=subtask,
state=Subtask.SubtaskState.ADDITIONAL_VERIFICATION.name, # pylint: disable=no-member
next_deadline=(
parse_datetime_to_timestamp(subtask.next_deadline) +
calculate_concent_verification_time(report_computed_task.task_to_compute)
)
)
# Add upload_acknowledged task to the work queue.
tasks.upload_acknowledged.delay(
subtask_id=subtask_id,
source_file_size=report_computed_task.task_to_compute.size,
source_package_hash=report_computed_task.task_to_compute.package_hash,
result_file_size=report_computed_task.size,
result_package_hash=report_computed_task.package_hash,
)
# If it's ADDITIONAL VERIFICATION, ACCEPTED or FAILED, log a warning and ignore the notification.
# Processing ends here. This means that it's a duplicate notification.
elif subtask.state_enum in [
Subtask.SubtaskState.ADDITIONAL_VERIFICATION,
Subtask.SubtaskState.ACCEPTED,
Subtask.SubtaskState.FAILED
]:
logging.log(
logger,
f'Subtask is expected to be in `VERIFICATION_FILE_TRANSFER` state, but was in {subtask.state}.',
subtask_id=subtask_id,
logging_level=logging.LoggingLevel.WARNING,
)
# If it's one of the states that can precede verification, report an error. Processing ends here.
else:
logging.log(
logger,
f'Subtask is expected to be in `VERIFICATION_FILE_TRANSFER` state, but was in {subtask.state}.',
subtask_id=subtask_id,
logging_level=logging.LoggingLevel.ERROR,
)
def upload_finished(subtask_id: str):
try:
subtask = Subtask.objects.get(subtask_id=subtask_id)
except Subtask.DoesNotExist:
logging.error(
f'Task `upload_finished` tried to get Subtask object with ID {subtask_id} but it does not exist.'
)
return
report_computed_task = deserialize_message(
subtask.report_computed_task.data.tobytes())
# Check subtask state, if it's VERIFICATION FILE TRANSFER, proceed with the task.
if subtask.state_enum == Subtask.SubtaskState.VERIFICATION_FILE_TRANSFER:
# If subtask is past the deadline, processes the timeout.
if subtask.next_deadline.timestamp() < get_current_utc_timestamp():
# Worker makes a payment from requestor's deposit just like in the forced acceptance use case.
payments_service.make_force_payment_to_provider( # pylint: disable=no-value-for-parameter
requestor_eth_address=report_computed_task.task_to_compute.
requestor_ethereum_address,
provider_eth_address=report_computed_task.task_to_compute.
provider_ethereum_address,
value=report_computed_task.task_to_compute.price,
payment_ts=get_current_utc_timestamp(),
)
update_subtask_state(
subtask=subtask,
state=Subtask.SubtaskState.FAILED.name, # pylint: disable=no-member
)
# Worker adds SubtaskResultsSettled to provider's and requestor's receive queues (both out-of-band)
for public_key in [
subtask.provider.public_key_bytes,
subtask.requestor.public_key_bytes
]:
store_pending_message(
response_type=PendingResponse.ResponseType.
SubtaskResultsSettled,
client_public_key=public_key,
queue=PendingResponse.Queue.ReceiveOutOfBand,
subtask=subtask,
)
return
# Change subtask state to ADDITIONAL VERIFICATION.
update_subtask_state(
subtask=subtask,
state=Subtask.SubtaskState.ADDITIONAL_VERIFICATION.name, # pylint: disable=no-member
next_deadline=int(subtask.next_deadline.timestamp()) +
calculate_subtask_verification_time(report_computed_task))
# Add upload_acknowledged task to the work queue.
tasks.upload_acknowledged.delay(
subtask_id=subtask_id,
source_file_size=report_computed_task.task_to_compute.size,
source_package_hash=report_computed_task.task_to_compute.
package_hash,
result_file_size=report_computed_task.size,
result_package_hash=report_computed_task.package_hash,
)
# If it's ADDITIONAL VERIFICATION, ACCEPTED or FAILED, log a warning and ignore the notification.
# Processing ends here. This means that it's a duplicate notification.
elif subtask.state_enum in [
Subtask.SubtaskState.ADDITIONAL_VERIFICATION,
Subtask.SubtaskState.ACCEPTED, Subtask.SubtaskState.FAILED
]:
logging.warning(
f'Subtask with ID {subtask_id} is expected to be in `VERIFICATION_FILE_TRANSFER` state, but was in {subtask.state}.'
)
# If it's one of the states that can precede verification, report an error. Processing ends here.
else:
logging.error(
f'Subtask with ID {subtask_id} is expected to be in `VERIFICATION_FILE_TRANSFER` state, but was in {subtask.state}.'
)
def verification_result(
self: Task,
subtask_id: str,
result: str,
error_message: Optional[str] = None,
error_code: Optional[str] = None,
) -> None:
logging.log(
logger,
f'Verification_result_task starts. Result: {result}',
subtask_id=subtask_id
)
assert isinstance(subtask_id, str)
assert isinstance(result, str)
assert result in VerificationResult.__members__.keys()
assert isinstance(error_message, (str, type(None)))
assert isinstance(error_code, (str, type(None)))
result_enum = VerificationResult[result]
assert result_enum != VerificationResult.ERROR or all([isinstance(error_message, str), isinstance(error_code, str)])
# Worker locks database row corresponding to the subtask in the subtask table.
try:
subtask = Subtask.objects.select_for_update(nowait=True).get(subtask_id=subtask_id)
except DatabaseError:
logging.log(
logger,
f'Row in database corresponding with Subtask object is already locked.'
f'retrying task {self.request.retries}/{self.max_retries}',
subtask_id=subtask_id,
logging_level=logging.LoggingLevel.WARNING,
)
# If the row is already locked, task fails so that Celery can retry later.
self.retry(
countdown=CELERY_LOCKED_SUBTASK_DELAY,
max_retries=MAXIMUM_VERIFICATION_RESULT_TASK_RETRIES,
throw=False,
)
return
if subtask.state_enum == Subtask.SubtaskState.ACCEPTED:
logging.log(
logger,
VERIFICATION_RESULT_SUBTASK_STATE_ACCEPTED_LOG_MESSAGE.format(subtask_id),
subtask_id=subtask_id,
logging_level=logging.LoggingLevel.WARNING
)
return
elif subtask.state_enum == Subtask.SubtaskState.FAILED:
logging.log(
logger,
VERIFICATION_RESULT_SUBTASK_STATE_FAILED_LOG_MESSAGE.format(subtask_id),
subtask_id=subtask_id,
logging_level=logging.LoggingLevel.WARNING
)
return
elif subtask.state_enum != Subtask.SubtaskState.ADDITIONAL_VERIFICATION:
logging.log(
logger,
VERIFICATION_RESULT_SUBTASK_STATE_UNEXPECTED_LOG_MESSAGE.format(subtask.state),
subtask_id=subtask_id,
logging_level=logging.LoggingLevel.ERROR,
)
return
# If the time is already past next_deadline for the subtask (SubtaskResultsRejected.timestamp + AVCT)
# worker ignores worker's message and processes the timeout.
if subtask.next_deadline < parse_timestamp_to_utc_datetime(get_current_utc_timestamp()):
task_to_compute = deserialize_message(subtask.task_to_compute.data.tobytes())
# Worker makes a payment from requestor's deposit just like in the forced acceptance use case.
def finalize_claims() -> None:
finalize_deposit_claim(
subtask_id=subtask_id,
concent_use_case=ConcentUseCase.ADDITIONAL_VERIFICATION,
ethereum_address=task_to_compute.requestor_ethereum_address,
)
finalize_deposit_claim(
subtask_id=subtask_id,
concent_use_case=ConcentUseCase.ADDITIONAL_VERIFICATION,
ethereum_address=task_to_compute.provider_ethereum_address,
)
transaction.on_commit(
finalize_claims,
using='control',
)
update_subtask_state(
subtask=subtask,
state=Subtask.SubtaskState.ACCEPTED.name, # pylint: disable=no-member
)
for public_key in [subtask.provider.public_key_bytes, subtask.requestor.public_key_bytes]:
store_pending_message(
response_type=PendingResponse.ResponseType.SubtaskResultsSettled,
client_public_key=public_key,
queue=PendingResponse.Queue.ReceiveOutOfBand,
subtask=subtask,
)
return
if result_enum == VerificationResult.MISMATCH:
# Worker adds SubtaskResultsRejected to provider's and requestor's receive queues (both out-of-band)
for public_key in [subtask.provider.public_key_bytes, subtask.requestor.public_key_bytes]:
store_pending_message(
response_type=PendingResponse.ResponseType.SubtaskResultsRejected,
client_public_key=public_key,
queue=PendingResponse.Queue.ReceiveOutOfBand,
subtask=subtask,
)
task_to_compute = deserialize_message(subtask.task_to_compute.data.tobytes())
def finalize_claims() -> None: # pylint: disable=function-redefined
delete_deposit_claim(
subtask_id=subtask_id,
concent_use_case=ConcentUseCase.ADDITIONAL_VERIFICATION,
ethereum_address=task_to_compute.requestor_ethereum_address,
)
finalize_deposit_claim(
subtask_id=subtask_id,
concent_use_case=ConcentUseCase.ADDITIONAL_VERIFICATION,
ethereum_address=task_to_compute.provider_ethereum_address,
)
transaction.on_commit(
finalize_claims,
using='control',
)
# Worker changes subtask state to FAILED
subtask.state = Subtask.SubtaskState.FAILED.name # pylint: disable=no-member
subtask.next_deadline = None
subtask.full_clean()
subtask.save()
elif result_enum in (VerificationResult.MATCH, VerificationResult.ERROR):
# Worker logs the error code and message
if result_enum == VerificationResult.ERROR:
logging.log(
logger,
f'Verification_result_task processing error result with: RESULT {result_enum.name}. ERROR MESSAGE {error_message}. ERROR CODE {error_code}',
subtask_id=subtask_id,
)
task_to_compute = deserialize_message(subtask.task_to_compute.data.tobytes())
# Worker makes a payment from requestor's deposit just like in the forced acceptance use case.
def finalize_claims() -> None: # pylint: disable=function-redefined
finalize_deposit_claim(
subtask_id=subtask_id,
concent_use_case=ConcentUseCase.ADDITIONAL_VERIFICATION,
ethereum_address=task_to_compute.requestor_ethereum_address,
)
finalize_deposit_claim(
subtask_id=subtask_id,
concent_use_case=ConcentUseCase.ADDITIONAL_VERIFICATION,
ethereum_address=task_to_compute.provider_ethereum_address,
)
transaction.on_commit(
finalize_claims,
using='control',
)
# Worker adds SubtaskResultsSettled to provider's and requestor's receive queues (both out-of-band)
for public_key in [subtask.provider.public_key_bytes, subtask.requestor.public_key_bytes]:
store_pending_message(
response_type=PendingResponse.ResponseType.SubtaskResultsSettled,
client_public_key=public_key,
queue=PendingResponse.Queue.ReceiveOutOfBand,
subtask=subtask,
)
# Worker changes subtask state to ACCEPTED
update_subtask_state(
subtask=subtask,
state=Subtask.SubtaskState.ACCEPTED.name, # pylint: disable=no-member
)
logging.log(
logger,
f'Verification_result_task ends. Result: {result_enum.name}',
subtask_id=subtask_id
)
def verification_result(
self,
subtask_id: str,
result: str,
error_message: Optional[str] = None,
error_code: Optional[str] = None,
):
logger.info(
f'verification_result_task starts with: SUBTASK_ID {subtask_id} -- RESULT {result}'
)
assert isinstance(subtask_id, str)
assert isinstance(result, str)
assert result in VerificationResult.__members__.keys()
assert isinstance(error_message, (str, type(None)))
assert isinstance(error_code, (str, type(None)))
result_enum = VerificationResult[result]
assert result_enum != VerificationResult.ERROR or all(
[error_message, error_code])
# Worker locks database row corresponding to the subtask in the subtask table.
try:
subtask = Subtask.objects.select_for_update(nowait=True).get(
subtask_id=subtask_id)
except DatabaseError:
logging.warning(
f'Subtask object with ID {subtask_id} database row is locked, '
f'retrying task {self.request.retries}/{self.max_retries}')
# If the row is already locked, task fails so that Celery can retry later.
self.retry(
countdown=CELERY_LOCKED_SUBTASK_DELAY,
max_retries=MAXIMUM_VERIFICATION_RESULT_TASK_RETRIES,
throw=False,
)
return
if subtask.state_enum == Subtask.SubtaskState.ACCEPTED:
logger.warning(
VERIFICATION_RESULT_SUBTASK_STATE_ACCEPTED_LOG_MESSAGE.format(
subtask_id))
return
elif subtask.state_enum == Subtask.SubtaskState.FAILED:
logger.warning(
VERIFICATION_RESULT_SUBTASK_STATE_FAILED_LOG_MESSAGE.format(
subtask_id))
return
elif subtask.state_enum != Subtask.SubtaskState.ADDITIONAL_VERIFICATION:
logger.error(
VERIFICATION_RESULT_SUBTASK_STATE_UNEXPECTED_LOG_MESSAGE.format(
subtask_id, subtask.state))
return
# If the time is already past next_deadline for the subtask (SubtaskResultsRejected.timestamp + AVCT)
# worker ignores worker's message and processes the timeout.
if subtask.next_deadline < parse_timestamp_to_utc_datetime(
get_current_utc_timestamp()):
task_to_compute = deserialize_message(
subtask.task_to_compute.data.tobytes())
# Worker makes a payment from requestor's deposit just like in the forced acceptance use case.
payments_service.make_force_payment_to_provider( # pylint: disable=no-value-for-parameter
requestor_eth_address=task_to_compute.requestor_ethereum_address,
provider_eth_address=task_to_compute.provider_ethereum_address,
value=task_to_compute.price,
payment_ts=get_current_utc_timestamp(),
)
update_subtask_state(
subtask=subtask,
state=Subtask.SubtaskState.ACCEPTED.name, # pylint: disable=no-member
)
for public_key in [
subtask.provider.public_key_bytes,
subtask.requestor.public_key_bytes
]:
store_pending_message(
response_type=PendingResponse.ResponseType.
SubtaskResultsSettled,
client_public_key=public_key,
queue=PendingResponse.Queue.ReceiveOutOfBand,
subtask=subtask,
)
return
if result_enum == VerificationResult.MISMATCH:
# Worker adds SubtaskResultsRejected to provider's and requestor's receive queues (both out-of-band)
for public_key in [
subtask.provider.public_key_bytes,
subtask.requestor.public_key_bytes
]:
store_pending_message(
response_type=PendingResponse.ResponseType.
SubtaskResultsRejected,
client_public_key=public_key,
queue=PendingResponse.Queue.ReceiveOutOfBand,
subtask=subtask,
)
# Worker changes subtask state to FAILED
subtask.state = Subtask.SubtaskState.FAILED.name # pylint: disable=no-member
subtask.next_deadline = None
subtask.full_clean()
subtask.save()
elif result_enum in (VerificationResult.MATCH, VerificationResult.ERROR):
# Worker logs the error code and message
if result_enum == VerificationResult.ERROR:
logger.info(
f'verification_result_task processing error result with: '
f'SUBTASK_ID {subtask_id} -- RESULT {result_enum.name} -- ERROR MESSAGE {error_message} -- ERROR CODE {error_code}'
)
task_to_compute = deserialize_message(
subtask.task_to_compute.data.tobytes())
# Worker makes a payment from requestor's deposit just like in the forced acceptance use case.
payments_service.make_force_payment_to_provider( # pylint: disable=no-value-for-parameter
requestor_eth_address=task_to_compute.requestor_ethereum_address,
provider_eth_address=task_to_compute.provider_ethereum_address,
value=task_to_compute.price,
payment_ts=get_current_utc_timestamp(),
)
# Worker adds SubtaskResultsSettled to provider's and requestor's receive queues (both out-of-band)
for public_key in [
subtask.provider.public_key_bytes,
subtask.requestor.public_key_bytes
]:
store_pending_message(
response_type=PendingResponse.ResponseType.
SubtaskResultsSettled,
client_public_key=public_key,
queue=PendingResponse.Queue.ReceiveOutOfBand,
subtask=subtask,
)
# Worker changes subtask state to ACCEPTED
update_subtask_state(
subtask=subtask,
state=Subtask.SubtaskState.ACCEPTED.name, # pylint: disable=no-member
)
logger.info(
f'verification_result_task ends with: SUBTASK_ID {subtask_id} -- RESULT {result_enum.name}'
)