def load_images(blender_output_file_name: str, result_file: str,
subtask_id: str) -> Tuple[ndarray, ndarray]:
# Read both files with OpenCV.
cv2 = import_cv2() # type: ignore
try:
image_1 = cv2.imread( # pylint: disable=no-member
generate_verifier_storage_file_path(blender_output_file_name))
image_2 = cv2.imread( # pylint: disable=no-member
result_file)
except MemoryError as exception:
log(
logger,
f'Loading result files into memory exceeded available memory and failed with: {exception}',
subtask_id=subtask_id,
)
raise VerificationError(
str(exception),
ErrorCode.VERIFIER_LOADING_FILES_INTO_MEMORY_FAILED,
subtask_id,
)
# If loading fails because of wrong path, cv2.imread does not raise any error but returns None.
if image_1 is None or image_2 is None:
log(
logger,
f'Loading files using OpenCV fails.',
subtask_id=subtask_id,
)
raise VerificationError(
'Loading files using OpenCV fails.',
ErrorCode.VERIFIER_LOADING_FILES_WITH_OPENCV_FAILED,
subtask_id,
)
return (image_1, image_2)
def test_blender_verification_order_should_call_verification_result_with_result_error_if_render_image_raises_exception(
self):
with mock.patch('verifier.tasks.download_archives_from_storage', autospec=True) as mock_download_archives_from_storage, \
mock.patch('verifier.tasks.validate_downloaded_archives', autospec=True) as mock_validate_downloaded_archives, \
mock.patch('verifier.tasks.unpack_archives', autospec=True) as mock_unpack_archives, \
mock.patch('verifier.tasks.get_files_list_from_archive', autospec=True, side_effect=[self.frames]) as mock_get_files_list_from_archive, \
mock.patch('verifier.tasks.parse_result_files_with_frames', autospec=True, return_value=self.mocked_parse_result_files_with_frames()) as mock_parse_result_files_with_frames, \
mock.patch('core.tasks.verification_result.delay', autospec=True) as mock_verification_result, \
mock.patch(
'verifier.tasks.render_images_by_frames',
side_effect=VerificationError(
'error',
ErrorCode.VERIFIER_RUNNING_BLENDER_FAILED,
self.subtask_id
),
autospec=True, return_value=(True, True)): # noqa: E125
self._send_blender_verification_order()
self.assertEqual(mock_download_archives_from_storage.call_count, 1)
self.assertEqual(mock_validate_downloaded_archives.call_count, 1)
self.assertEqual(mock_get_files_list_from_archive.call_count, 1)
self.assertEqual(mock_parse_result_files_with_frames.call_count, 1)
self.assertEqual(mock_unpack_archives.call_count, 1)
mock_verification_result.assert_called_once_with(
self.subtask_id,
VerificationResult.ERROR.name,
'error',
ErrorCode.VERIFIER_RUNNING_BLENDER_FAILED.name,
)
def download_archives_from_storage(
file_transfer_token: message.concents.FileTransferToken,
subtask_id: str,
package_paths_to_downloaded_file_names: Dict[str, str],
) -> None:
# Remove any files from VERIFIER_STORAGE_PATH.
clean_directory(settings.VERIFIER_STORAGE_PATH, subtask_id)
# Download all the files listed in the message from the storage server to local storage.
for file_path, download_file_name in package_paths_to_downloaded_file_names.items(
):
try:
file_transfer_token.sig = None
cluster_response = send_request_to_storage_cluster(
prepare_storage_request_headers(file_transfer_token),
settings.STORAGE_SERVER_INTERNAL_ADDRESS +
CLUSTER_DOWNLOAD_PATH + file_path,
method='get',
)
path_to_store = os.path.join(settings.VERIFIER_STORAGE_PATH,
download_file_name)
store_file_from_response_in_chunks(cluster_response, path_to_store)
except Exception as exception:
log(
logger,
f'blender_verification_order for SUBTASK_ID {subtask_id} failed with error {exception}.'
f'ErrorCode: {ErrorCode.VERIFIER_FILE_DOWNLOAD_FAILED.name}')
raise VerificationError(
str(exception),
ErrorCode.VERIFIER_FILE_DOWNLOAD_FAILED,
subtask_id=subtask_id,
)
def validate_downloaded_archives(subtask_id: str, archives_list: Iterable[str],
scene_file: str) -> None:
# If archive is broken, it means that Provider must have intentionally uploaded damaged zip file.
# In such case verification end with MISMATCH result.
package_files_list = [] # type: List[str]
try:
# If any file which is supposed to be unpacked from archives already exists, finish with error and raise exception.
for package_file_path in archives_list:
package_files_list += get_files_list_from_archive(
generate_verifier_storage_file_path(package_file_path))
except zipfile.BadZipFile:
raise VerificationMismatch(subtask_id)
already_existing_files = set(os.listdir(
settings.VERIFIER_STORAGE_PATH)).intersection(package_files_list)
if already_existing_files:
# This should not happen normally as the directory is cleaned before
raise VerificationError(
f'Files:<{", ".join(already_existing_files)}> already exist.',
ErrorCode.VERIFIER_UNPACKING_ARCHIVE_FAILED,
subtask_id,
)
# Similarly, it is Provider's responsibility to upload scene file.
# If it is missing, verification end with MISMATCH result.
if scene_file not in package_files_list:
raise VerificationMismatch(subtask_id)
def render_image(
frame_number: int,
output_format: str,
scene_file: str,
subtask_id: str,
verification_deadline: Union[int, float],
blender_crop_script_parameters: Dict[str, Union[int, List[float], bool]],
) -> None:
# Verifier runs blender process.
try:
completed_process = run_blender(
scene_file,
output_format,
frame_number,
verification_deadline,
blender_crop_script_parameters,
subtask_id,
)
clean_directory(
os.path.join(settings.VERIFIER_STORAGE_PATH, 'render-scripts'),
subtask_id)
# If Blender finishes with errors, verification ends here
# Verification_result informing about the error is sent to the work queue.
if completed_process.returncode != 0:
log(
logger, 'Blender finished with errors',
f'SUBTASK_ID: {subtask_id}.'
f'Returncode: {str(completed_process.returncode)}.'
f'stderr: {str(completed_process.stderr)}.'
f'stdout: {str(completed_process.stdout)}.')
raise VerificationError(
str(completed_process.stderr),
ErrorCode.VERIFIER_RUNNING_BLENDER_FAILED,
subtask_id,
)
except subprocess.SubprocessError as exception:
log(
logger,
f'Blender finished with errors. Error: {exception} SUBTASK_ID {subtask_id}'
)
raise VerificationError(
str(exception),
ErrorCode.VERIFIER_RUNNING_BLENDER_FAILED,
subtask_id,
)
def compare_images(image_1: ndarray, image_2: ndarray, subtask_id: str) -> float:
# Compute SSIM for the image pair.
try:
ssim = compare_ssim(image_1, image_2, multichannel=True)
except ValueError as exception:
logger.info(f'Computing SSIM fails with: {exception}')
raise VerificationError(
str(exception),
ErrorCode.VERIFIER_COMPUTING_SSIM_FAILED,
subtask_id,
)
assert isinstance(ssim, float)
return ssim
def test_blender_verification_order_should_call_verification_result_with_result_error_if_compare_ssim_raise_value_error(
self):
with mock.patch('verifier.tasks.download_archives_from_storage', autospec=True) as mock_download_archives_from_storage, \
mock.patch('verifier.tasks.validate_downloaded_archives', autospec=True) as mock_validate_downloaded_archives, \
mock.patch('verifier.tasks.unpack_archives', autospec=True) as mock_unpack_archives, \
mock.patch('verifier.tasks.get_files_list_from_archive', autospec=True, side_effect=[self.frames]) as mock_get_files_list_from_archive, \
mock.patch('verifier.tasks.parse_result_files_with_frames', autospec=True, return_value=self.mocked_parse_result_files_with_frames()) as mock_parse_result_files_with_frames, \
mock.patch('verifier.tasks.render_images_by_frames', autospec=True, return_value=(self.blender_output_file_name_list, self.parsed_all_files)) as mock_render_image, \
mock.patch('verifier.tasks.delete_source_files', autospec=True) as mock_delete_source_files, \
mock.patch('verifier.tasks.upload_blender_output_file', autospec=True) as mock_try_to_upload_file, \
mock.patch(
'verifier.tasks.compare_all_rendered_images_with_user_results_files',
autospec=True,
side_effect=VerificationError(
'error',
ErrorCode.VERIFIER_COMPUTING_SSIM_FAILED,
self.subtask_id
),
), \
mock.patch('core.tasks.verification_result.delay', autospec=True) as mock_verification_result: # noqa: E125
current_time = get_current_utc_timestamp()
self._send_blender_verification_order(current_time=current_time)
self.assertEqual(mock_download_archives_from_storage.call_count, 1)
self.assertEqual(mock_parse_result_files_with_frames.call_count, 1)
self.assertEqual(mock_validate_downloaded_archives.call_count, 1)
self.assertEqual(mock_get_files_list_from_archive.call_count, 1)
self.assertEqual(mock_unpack_archives.call_count, 1)
mock_render_image.assert_called_once_with(
frames=self.frames,
parsed_files_to_compare=self.mocked_parse_result_files_with_frames(
),
output_format=self.output_format,
scene_file=self.scene_file,
subtask_id=self.subtask_id,
verification_deadline=self._get_verification_deadline_as_timestamp(
current_time,
self.report_computed_task.size,
),
blender_crop_script=self.compute_task_def['extra_data']
['script_src'],
)
self.assertEqual(mock_delete_source_files.call_count, 1)
self.assertEqual(mock_try_to_upload_file.call_count, 1)
mock_verification_result.assert_called_once_with(
self.subtask_id,
VerificationResult.ERROR.name,
'error',
ErrorCode.VERIFIER_COMPUTING_SSIM_FAILED.name,
)
def render_image(frame_number: int, output_format: str, scene_file: str, subtask_id: str, verification_deadline: int, blender_crop_script: Optional[str]=None) -> None:
# Verifier stores Blender crop script to a file.
if blender_crop_script is not None:
blender_script_file_name = store_blender_script_file(subtask_id, blender_crop_script) # type: Optional[str]
else:
blender_script_file_name = None
# Verifier runs blender process.
try:
completed_process = run_blender(
scene_file,
output_format,
frame_number,
verification_deadline,
blender_script_file_name,
)
# If Blender finishes with errors, verification ends here
# Verification_result informing about the error is sent to the work queue.
if completed_process.returncode != 0:
log_string_message(
logger,
'Blender finished with errors',
f'SUBTASK_ID: {subtask_id}.'
f'Returncode: {str(completed_process.returncode)}.'
f'stderr: {str(completed_process.stderr)}.'
f'stdout: {str(completed_process.stdout)}.'
)
raise VerificationError(
str(completed_process.stderr),
ErrorCode.VERIFIER_RUNNING_BLENDER_FAILED,
subtask_id,
)
except subprocess.SubprocessError as exception:
log_string_message(logger, f'Blender finished with errors. Error: {exception} SUBTASK_ID {subtask_id}')
raise VerificationError(
str(exception),
ErrorCode.VERIFIER_RUNNING_BLENDER_FAILED,
subtask_id,
)
def try_to_upload_blender_output_file(blender_output_file_name: str,
output_format: str, subtask_id: str,
frame_number: int) -> None:
upload_file_path = generate_upload_file_path(subtask_id, output_format,
frame_number)
# Read Blender output file.
try:
with open(
generate_verifier_storage_file_path(blender_output_file_name),
'rb') as upload_file:
upload_file_content = upload_file.read() # type: bytes
upload_file_checksum = 'sha1:' + hashlib.sha1(
upload_file_content).hexdigest()
# Generate a FileTransferToken valid for an upload of the image generated by blender.
upload_file_transfer_token = create_file_transfer_token_for_concent(
subtask_id=subtask_id,
result_package_path=upload_file_path,
result_size=len(upload_file_content),
result_package_hash=upload_file_checksum,
operation=message.concents.FileTransferToken.Operation.upload,
)
# Upload the image.
upload_file_to_storage_cluster(
upload_file_content,
upload_file_path,
upload_file_transfer_token,
settings.CONCENT_PRIVATE_KEY,
settings.CONCENT_PUBLIC_KEY,
settings.CONCENT_PUBLIC_KEY,
settings.STORAGE_SERVER_INTERNAL_ADDRESS,
)
except OSError as exception:
log(crash_logger,
str(exception),
subtask_id=subtask_id,
logging_level=LoggingLevel.ERROR)
except MemoryError as exception:
log(logger,
f'Loading result files into memory failed with: {exception}',
subtask_id=subtask_id,
logging_level=LoggingLevel.ERROR)
raise VerificationError(
str(exception),
ErrorCode.VERIFIER_LOADING_FILES_INTO_MEMORY_FAILED,
subtask_id,
)
def unpack_archives(file_paths: Iterable[str], subtask_id: str) -> None:
# Verifier unpacks the archive with project source.
for archive_file_path in file_paths:
try:
unpack_archive(os.path.basename(archive_file_path))
except zipfile.BadZipFile as exception:
log(
logger,
f'Verifier failed to unpack the archive with project source with error {exception} '
f'SUBTASK_ID {subtask_id}. '
f'ErrorCode: {ErrorCode.VERIFIER_UNPACKING_ARCHIVE_FAILED.name}'
)
raise VerificationError(
str(exception),
ErrorCode.VERIFIER_UNPACKING_ARCHIVE_FAILED,
subtask_id,
)
def test_blender_verification_order_should_call_verification_result_with_result_error_if_unpacking_archive_fails(
self):
with mock.patch('verifier.tasks.download_archives_from_storage', autospec=True) as mock_download_archives_from_storage, \
mock.patch('verifier.tasks.validate_downloaded_archives', autospec=True) as mock_validate_downloaded_archives, \
mock.patch('verifier.tasks.unpack_archives', side_effect=VerificationError(
"error",
ErrorCode.VERIFIER_UNPACKING_ARCHIVE_FAILED,
self.subtask_id
), autospec=True), \
mock.patch('core.tasks.verification_result.delay', autospec=True) as mock_verification_result: # noqa: E125
self._send_blender_verification_order()
self.assertEqual(mock_download_archives_from_storage.call_count, 1)
self.assertEqual(mock_validate_downloaded_archives.call_count, 1)
mock_verification_result.assert_called_once_with(
self.subtask_id,
VerificationResult.ERROR.name,
'error',
ErrorCode.VERIFIER_UNPACKING_ARCHIVE_FAILED.name,
)
def test_that_blender_verification_order_should_call_verification_result_with_result_error_if_download_fails(
self):
with mock.patch(
'verifier.tasks.download_archives_from_storage',
autospec=True,
side_effect=VerificationError(
'error',
ErrorCode.VERIFIER_FILE_DOWNLOAD_FAILED,
self.compute_task_def['subtask_id']
)
) as mock_download_archives_from_storage, \
mock.patch('core.tasks.verification_result.delay', autospec=True) as mock_verification_result: # noqa: E129
self._send_blender_verification_order()
self.assertEqual(mock_download_archives_from_storage.call_count, 1)
mock_verification_result.assert_called_once_with(
self.compute_task_def['subtask_id'],
VerificationResult.ERROR.name,
'error',
ErrorCode.VERIFIER_FILE_DOWNLOAD_FAILED.name,
)