def download_archives_from_storage(
file_transfer_token: message.concents.FileTransferToken,
subtask_id: str,
package_paths_to_downloaded_file_names: Dict[str, str],
) -> None:
# Remove any files from VERIFIER_STORAGE_PATH.
clean_directory(settings.VERIFIER_STORAGE_PATH)
# Download all the files listed in the message from the storage server to local storage.
for file_path, download_file_name in package_paths_to_downloaded_file_names.items():
try:
file_transfer_token.sig = None
cluster_response = send_request_to_storage_cluster(
prepare_storage_request_headers(file_transfer_token),
settings.STORAGE_SERVER_INTERNAL_ADDRESS + CLUSTER_DOWNLOAD_PATH + file_path,
method='get',
)
path_to_store = os.path.join(settings.VERIFIER_STORAGE_PATH, download_file_name)
store_file_from_response_in_chunks(
cluster_response,
path_to_store
)
except Exception as exception:
log_string_message(
logger,
f'blender_verification_order for SUBTASK_ID {subtask_id} failed with error {exception}.'
f'ErrorCode: {ErrorCode.VERIFIER_FILE_DOWNLOAD_FAILED.name}'
)
raise VerificationError(
str(exception),
ErrorCode.VERIFIER_FILE_DOWNLOAD_FAILED,
subtask_id=subtask_id,
)
def compare_all_rendered_images_with_user_results_files(parsed_files_to_compare: FramesToParsedFilePaths, subtask_id: str) -> List[float]:
ssim_list = []
for (result_file, blender_output_file_name) in parsed_files_to_compare.values():
image_1, image_2 = load_images(
blender_output_file_name,
result_file,
subtask_id
)
if not are_image_sizes_and_color_channels_equal(image_1, image_2):
log_string_message(
logger,
f'Blender verification failed. Sizes in pixels of images are not equal. SUBTASK_ID: {subtask_id}.'
f'VerificationResult: {VerificationResult.MISMATCH.name}'
)
raise VerificationMismatch(subtask_id=subtask_id)
ssim_list.append(compare_images(image_1, image_2, subtask_id))
return ssim_list
def report_upload(_request, file_path):
log_request_received(logger, file_path, FileTransferToken.Operation.upload)
# If there's a corresponding VerificationRequest, the load it and link it to UploadReport.
try:
verification_request = VerificationRequest.objects.get(
Q(source_package_path=file_path) | Q(result_package_path=file_path)
)
except VerificationRequest.DoesNotExist:
verification_request = None
# The app creates a new instance of UploadReport in the database.
upload_report_obj = UploadReport(
path = file_path,
verification_request = verification_request,
)
upload_report_obj.full_clean()
upload_report_obj.save()
# The app gets the VerificationRequest and checks if both source and result packages have reports.
if (
verification_request is not None and
verification_request.blender_subtask_definition is not None and
verification_request.upload_reports.filter(path=verification_request.source_package_path).exists() and
verification_request.upload_reports.filter(path=verification_request.result_package_path).exists() and
verification_request.upload_reports.filter(path=file_path).count() == 1
):
assert file_path in [verification_request.source_package_path, verification_request.result_package_path]
# If all expected files have been uploaded, the app sends upload_finished task to the work queue.
upload_finished.delay(verification_request.subtask_id)
verification_request.upload_finished = True
verification_request.full_clean()
verification_request.save()
log_string_message(
logger, 'All expected files have been uploaded',
f'Subtask ID: {verification_request.subtask_id}.'
f'Result package path: {verification_request.result_package_path}.'
f'Source package path: {verification_request.source_package_path}.'
)
return HttpResponse()
def unpack_archives(file_paths: Iterable[str], subtask_id: str) -> None:
# Verifier unpacks the archive with project source.
for archive_file_path in file_paths:
try:
unpack_archive(
os.path.basename(archive_file_path)
)
except zipfile.BadZipFile as exception:
log_string_message(
logger,
f'Verifier failed to unpack the archive with project source with error {exception} '
f'SUBTASK_ID {subtask_id}. '
f'ErrorCode: {ErrorCode.VERIFIER_UNPACKING_ARCHIVE_FAILED.name}'
)
raise VerificationError(
str(exception),
ErrorCode.VERIFIER_UNPACKING_ARCHIVE_FAILED,
subtask_id,
)
def render_image(frame_number: int, output_format: str, scene_file: str, subtask_id: str, verification_deadline: int, blender_crop_script: Optional[str]=None) -> None:
# Verifier stores Blender crop script to a file.
if blender_crop_script is not None:
blender_script_file_name = store_blender_script_file(subtask_id, blender_crop_script) # type: Optional[str]
else:
blender_script_file_name = None
# Verifier runs blender process.
try:
completed_process = run_blender(
scene_file,
output_format,
frame_number,
verification_deadline,
blender_script_file_name,
)
# If Blender finishes with errors, verification ends here
# Verification_result informing about the error is sent to the work queue.
if completed_process.returncode != 0:
log_string_message(
logger,
'Blender finished with errors',
f'SUBTASK_ID: {subtask_id}.'
f'Returncode: {str(completed_process.returncode)}.'
f'stderr: {str(completed_process.stderr)}.'
f'stdout: {str(completed_process.stdout)}.'
)
raise VerificationError(
str(completed_process.stderr),
ErrorCode.VERIFIER_RUNNING_BLENDER_FAILED,
subtask_id,
)
except subprocess.SubprocessError as exception:
log_string_message(logger, f'Blender finished with errors. Error: {exception} SUBTASK_ID {subtask_id}')
raise VerificationError(
str(exception),
ErrorCode.VERIFIER_RUNNING_BLENDER_FAILED,
subtask_id,
)
def wrapper(request, *args, **kwargs):
if request.content_type == '':
log_string_message(logger, 'error: Content-Type is missing')
return JsonResponse({'error': 'Content-Type is missing.'},
status=400)
elif request.content_type == 'application/octet-stream':
try:
auth_message = load_without_public_key(request.body)
if isinstance(auth_message,
message.concents.ClientAuthorization):
if is_golem_message_signed_with_key(
auth_message.client_public_key,
auth_message,
):
log_message_received_in_endpoint(
logger, request.resolver_match.view_name
if request.resolver_match is not None else None,
auth_message.__class__.__name__,
auth_message.client_public_key)
else:
log_string_message(
logger,
f'ClientAuthorization message is not signed with public key {auth_message.client_public_key}.',
)
return JsonResponse(
{
'error':
f'ClientAuthorization message is not signed with public key {auth_message.client_public_key}.',
'error_code':
ErrorCode.MESSAGE_SIGNATURE_WRONG.value,
},
status=400)
else:
log_string_message(
logger,
'error: Client Authentication message not included')
return JsonResponse(
{
'error':
'Client Authentication message not included'
},
status=400)
except FieldError as exception:
log_string_message(logger,
'Golem Message contains wrong fields.',
exception.__class__.__name__)
return JsonResponse(
{
'error':
join_messages('Golem Message contains wrong fields.',
str(exception))
},
status=400)
except MessageFromFutureError as exception:
log_string_message(logger,
'Message timestamp too far in the future.',
exception.__class__.__name__)
return JsonResponse(
{
'error':
join_messages(
'Message timestamp too far in the future.',
str(exception))
},
status=400)
except MessageTooOldError as exception:
log_string_message(logger, 'Message is too old.',
exception.__class__.__name__)
return JsonResponse(
{
'error':
join_messages('Message is too old.', str(exception))
},
status=400)
except TimestampError as exception:
log_string_message(logger, 'Error:',
exception.__class__.__name__)
return JsonResponse({'error': f'{exception}'}, status=400)
except MessageError as exception:
log_string_message(logger, 'Error in Golem Message.',
exception.__class__.__name__)
return JsonResponse(
{
'error':
join_messages('Error in Golem Message.',
str(exception))
},
status=400)
else:
log_string_message(
logger,
'error: Concent supports only application/octet-stream.')
return JsonResponse(
{'error': "Concent supports only application/octet-stream."},
status=415)
return view(request, auth_message, auth_message.client_public_key,
*args, *kwargs)
def wrapper(request, client_message, client_public_key, *args,
**kwargs):
assert database_name in settings.DATABASES or database_name is None
try:
if database_name is not None:
sid = transaction.savepoint(using=database_name)
response_from_view = view(request, client_message,
client_public_key, *args, **kwargs)
if database_name is not None:
transaction.savepoint_commit(sid, using=database_name)
except ConcentBaseException as exception:
log_400_error(logger, view.__name__, client_public_key,
client_message, exception.error_code,
exception.error_message)
if database_name is not None:
transaction.savepoint_rollback(sid, using=database_name)
return JsonResponse(
{
'error': exception.error_message,
'error_code': exception.error_code.value,
},
status=400)
except ConcentInSoftShutdownMode:
transaction.savepoint_rollback(sid, using=database_name)
json_response = JsonResponse(
{'error': 'Concent is in soft shutdown mode.'}, status=503)
log_json_message(logger, json_response)
return json_response
if isinstance(response_from_view, message.Message):
assert response_from_view.sig is None
logging.log_message_returned(
logger,
response_from_view,
client_public_key,
request.resolver_match._func_path
if request.resolver_match is not None else None,
)
serialized_message = dump(
response_from_view,
settings.CONCENT_PRIVATE_KEY,
client_public_key,
)
return HttpResponse(serialized_message,
content_type='application/octet-stream')
elif isinstance(response_from_view, dict):
json_response = JsonResponse(response_from_view, safe=False)
log_json_message(logger, json_response)
return json_response
elif isinstance(response_from_view, HttpResponseNotAllowed):
logging.log_message_not_allowed(
logger,
view.__name__,
client_public_key,
request.method,
)
return response_from_view
elif isinstance(response_from_view, HttpResponse):
logging.log_message_accepted(
logger,
client_message,
client_public_key,
)
return response_from_view
elif response_from_view is None:
logging.log_empty_queue(
logger,
view.__name__,
client_public_key,
)
return HttpResponse("", status=204)
elif isinstance(response_from_view, bytes):
logging.log_string_message(
logger,
'Response from core.views - Response is bytes instance')
return HttpResponse(response_from_view)
logging.log_string_message(
logger, 'Invalid response from core.views type')
assert False, "Invalid response type"
raise Exception("Invalid response type")
def wrapper(request, *args, **kwargs):
if request.content_type == '':
log_string_message(logger, 'error: Content-Type is missing')
return JsonResponse({'error': 'Content-Type is missing.'},
status=400)
elif request.content_type == 'application/octet-stream':
try:
golem_message = load_without_public_key(request.body)
assert golem_message is not None
client_public_key = get_validated_client_public_key_from_client_message(
golem_message)
log_message_received_in_endpoint(
logger, request.resolver_match.view_name
if request.resolver_match is not None else None,
golem_message.__class__.__name__, client_public_key,
request.META['CONTENT_TYPE'] if 'CONTENT_TYPE'
in request.META.keys() else None, golem_message.task_id
if 'task_id' in dir(golem_message) else None,
golem_message.subtask_id
if 'subtask_id' in dir(golem_message) else None)
except ConcentValidationError as exception:
log_string_message(
logger,
f"error_code: {exception.error_code.value} error: {exception.error_message} "
)
return JsonResponse(
{
'error': f'{exception.error_message}',
'error_code': exception.error_code.value,
},
status=400)
except FieldError as exception:
log_string_message(logger,
'Golem Message contains wrong fields.',
exception.__class__.__name__)
return JsonResponse(
{
'error':
join_messages('Golem Message contains wrong fields.',
str(exception))
},
status=400)
except MessageFromFutureError as exception:
log_string_message(logger,
'Message timestamp too far in the future.',
exception.__class__.__name__)
return JsonResponse(
{
'error':
join_messages(
'Message timestamp too far in the future.',
str(exception))
},
status=400)
except MessageTooOldError as exception:
log_string_message(logger, 'Message is too old.',
exception.__class__.__name__)
return JsonResponse(
{
'error':
join_messages('Message is too old.', str(exception))
},
status=400)
except TimestampError as exception:
log_string_message(logger, 'Error:',
exception.__class__.__name__)
return JsonResponse({'error': f'{exception}'}, status=400)
except MessageError as exception:
log_string_message(logger, 'Error in Golem Message.',
exception.__class__.__name__)
return JsonResponse(
{
'error':
join_messages('Error in Golem Message.',
str(exception))
},
status=400)
else:
log_string_message(
logger,
'error: Concent supports only application/octet-stream.')
return JsonResponse(
{'error': "Concent supports only application/octet-stream."},
status=415)
return view(request, golem_message, client_public_key, *args, *kwargs)
def blender_verification_order(
subtask_id: str,
source_package_path: str,
source_size: int,
source_package_hash: str,
result_package_path: str,
result_size: int,
result_package_hash: str,
output_format: str,
scene_file: str,
verification_deadline: int,
frames: List[int],
blender_crop_script: Optional[str],
):
log_string_message(
logger,
f'Blender_verification_order_starts. SUBTASK_ID: {subtask_id}.',
f'Source_package_path: {source_package_path}.',
f'Source_size: {source_size}.',
f'Source_package_hash: {source_package_hash}.',
f'Result_package_path: {result_package_path}.',
f'Result_size: {result_size}.',
f'Result_package_hash: {result_package_hash}.',
f'Output_format: {output_format}.', f'Scene_file: {scene_file}.',
f'Frames: {frames}.')
assert output_format in BlenderSubtaskDefinition.OutputFormat.__members__.keys(
)
assert source_package_path != result_package_path
assert source_package_hash != result_package_hash
assert (source_size and source_package_hash
and source_package_path) and (result_size and result_package_hash
and result_package_path)
assert isinstance(subtask_id, str)
assert isinstance(verification_deadline, int)
# this is a temporary hack - dummy verification which's result depends on subtask_id only
if settings.MOCK_VERIFICATION_ENABLED:
result = VerificationResult.MATCH.name if subtask_id[
-1] == 'm' else VerificationResult.MISMATCH.name
if subtask_id[-1] == 'm':
verification_result.delay(
subtask_id,
result,
)
log_string_message(
logger,
f'Temporary hack, verification result depends on subtask_id only - SUBTASK_ID: {subtask_id}. Result: {result}'
)
return
# Generate a FileTransferToken valid for a download of any file listed in the order.
file_transfer_token = create_file_transfer_token_for_concent(
subtask_id=subtask_id,
source_package_path=source_package_path,
source_size=source_size,
source_package_hash=source_package_hash,
result_package_path=result_package_path,
result_size=result_size,
result_package_hash=result_package_hash,
operation=message.FileTransferToken.Operation.download,
)
package_paths_to_downloaded_archive_names = {
source_package_path: f'source_{os.path.basename(source_package_path)}',
result_package_path: f'result_{os.path.basename(result_package_path)}',
}
download_archives_from_storage(file_transfer_token, subtask_id,
package_paths_to_downloaded_archive_names)
validate_downloaded_archives(
subtask_id, package_paths_to_downloaded_archive_names.values(),
scene_file)
unpack_archives(package_paths_to_downloaded_archive_names.values(),
subtask_id)
result_files_list = get_files_list_from_archive(
generate_verifier_storage_file_path(
package_paths_to_downloaded_archive_names[result_package_path]))
ensure_enough_result_files_provided(
frames=frames,
result_files_list=result_files_list,
subtask_id=subtask_id,
)
parsed_files_to_compare = parse_result_files_with_frames(
frames=frames,
result_files_list=result_files_list,
output_format=output_format,
)
ensure_frames_have_related_files_to_compare(
frames=frames,
parsed_files_to_compare=parsed_files_to_compare,
subtask_id=subtask_id,
)
(blender_output_file_name_list,
parsed_files_to_compare) = render_images_by_frames(
parsed_files_to_compare=parsed_files_to_compare,
frames=frames,
output_format=output_format,
scene_file=scene_file,
subtask_id=subtask_id,
verification_deadline=verification_deadline,
blender_crop_script=blender_crop_script,
)
delete_source_files(
package_paths_to_downloaded_archive_names[source_package_path])
upload_blender_output_file(
frames=frames,
blender_output_file_name_list=blender_output_file_name_list,
output_format=output_format,
subtask_id=subtask_id,
)
ssim_list = compare_all_rendered_images_with_user_results_files(
parsed_files_to_compare=parsed_files_to_compare,
subtask_id=subtask_id,
)
compare_minimum_ssim_with_results(ssim_list, subtask_id)
def blender_verification_request(
subtask_id: str,
source_package_path: str,
result_package_path: str,
output_format: str,
scene_file: str,
verification_deadline: int,
frames: List[int],
blender_crop_script: Optional[str],
):
log_string_message(
logger,
f'Blender verification request starts. SUBTASK_ID: {subtask_id}',
f'Source_package_path {source_package_path}',
f'Result_package_path: {result_package_path}',
f'Output_format: {output_format}',
f'Scene_file: {scene_file}',
f'Frames: {frames}',
f'Verification_deadline: {verification_deadline}',
f'With blender_crop_script: {bool(blender_crop_script)}',
)
validate_frames(frames)
assert isinstance(output_format, str)
assert isinstance(verification_deadline, int)
output_format = output_format.upper()
assert output_format in BlenderSubtaskDefinition.OutputFormat.__members__.keys(
)
# The app creates a new instance of VerificationRequest in the database
# and a BlenderSubtaskDefinition instance associated with it.
(verification_request, blender_subtask_definition
) = store_verification_request_and_blender_subtask_definition(
subtask_id=subtask_id,
source_package_path=source_package_path,
result_package_path=result_package_path,
verification_deadline=verification_deadline,
output_format=output_format,
scene_file=scene_file,
blender_crop_script=blender_crop_script,
)
store_frames(
blender_subtask_definition=blender_subtask_definition,
frame_list=frames,
)
# If there are already UploadReports corresponding to some files, the app links them with the VerificationRequest
# by setting the value of the foreign key in UploadReport.
for path in [source_package_path, result_package_path]:
UploadReport.objects.filter(
path=path,
verification_request=None,
).update(verification_request=verification_request)
# The app checks if files indicated by source_package_path
# and result_package_path in the VerificationRequest have reports.
if (verification_request.upload_reports.filter(
path=verification_request.source_package_path).exists()
and verification_request.upload_reports.filter(
path=verification_request.result_package_path).exists()):
log_string_message(
logger, 'All expected files have been uploaded',
f'Subtask ID: {verification_request.subtask_id}'
f'Result package path: {verification_request.result_package_path}'
f'Source package path: {verification_request.source_package_path}')
# If all expected files have been uploaded, the app sends upload_finished task to the work queue.
tasks.upload_finished.delay(verification_request.subtask_id)
verification_request.upload_finished = True
verification_request.full_clean()
verification_request.save()
def upload_acknowledged(
subtask_id: str,
source_file_size: str,
source_package_hash: str,
result_file_size: str,
result_package_hash: str,
):
log_string_message(
logger, f'Upload acknowledgment starts. SUBTASK_ID: {subtask_id}',
f'Source_file_size {source_file_size}',
f'Source_package_hash: {source_package_hash}',
f'Result_file_size: {result_file_size}',
f'Result_package_hash: {result_package_hash}')
assert isinstance(subtask_id, str)
try:
verification_request = VerificationRequest.objects.get(
subtask_id=subtask_id)
except VerificationRequest.DoesNotExist:
log_error_message(
logger,
f'Task `upload_acknowledged` tried to get VerificationRequest object with ID {subtask_id} but it does not exist.'
)
return
if verification_request.upload_acknowledged is True:
logging.error(
f'Task `upload_acknowledged` scheduled but VerificationRequest with with ID {subtask_id} is already acknowledged.'
)
raise VerificationRequestAlreadyAcknowledgedError(
f'Task `upload_acknowledged` scheduled but VerificationRequest with with ID {subtask_id} is already acknowledged.',
ErrorCode.CONDUCTOR_VERIFICATION_REQUEST_ALREADY_ACKNOWLEDGED)
else:
verification_request.upload_acknowledged = True
verification_request.full_clean()
verification_request.save()
frames = filter_frames_by_blender_subtask_definition(
verification_request.blender_subtask_definition)
blender_verification_order.delay(
subtask_id=verification_request.subtask_id,
source_package_path=verification_request.source_package_path,
source_size=source_file_size,
source_package_hash=source_package_hash,
result_package_path=verification_request.result_package_path,
result_size=result_file_size,
result_package_hash=result_package_hash,
output_format=verification_request.blender_subtask_definition.
output_format,
scene_file=verification_request.blender_subtask_definition.scene_file,
verification_deadline=int(
verification_request.verification_deadline.timestamp()),
frames=frames,
blender_crop_script=verification_request.blender_subtask_definition.
blender_crop_script,
)
log_string_message(
logger, f'Upload acknowledgment finished. SUBTASK_ID: {subtask_id}',
f'Source_file_size {source_file_size}',
f'Source_package_hash: {source_package_hash}',
f'Result_file_size: {result_file_size}',
f'Result_package_hash: {result_package_hash}')