def should_save_non_s3_url_validation_results( get_param_mock: MagicMock, validation_results_factory_mock: MagicMock, ) -> None: validation_results_table_name = any_table_name() get_param_mock.return_value = validation_results_table_name non_s3_url = any_https_url() dataset_id = any_dataset_id() version_id = any_dataset_version_id() with patch("backend.check_stac_metadata.utils.processing_assets_model_with_meta"): lambda_handler( { DATASET_ID_KEY: dataset_id, VERSION_ID_KEY: version_id, METADATA_URL_KEY: non_s3_url, }, any_lambda_context(), ) hash_key = f"{DATASET_ID_PREFIX}{dataset_id}{DB_KEY_SEPARATOR}{VERSION_ID_PREFIX}{version_id}" assert validation_results_factory_mock.mock_calls == [ call(hash_key, validation_results_table_name), call().save( non_s3_url, Check.NON_S3_URL, ValidationResult.FAILED, details={MESSAGE_KEY: f"URL doesn't start with “{S3_URL_PREFIX}”: “{non_s3_url}”"}, ), ]
def should_log_event_payload() -> None: payload = deepcopy(MINIMAL_PAYLOAD) expected_log = dumps({"event": payload}) with patch.object(LOGGER, "debug") as logger_mock, patch( "backend.check_stac_metadata.task.STACDatasetValidator.run"): lambda_handler(payload, any_lambda_context()) logger_mock.assert_any_call(expected_log)
def should_succeed_with_validation_failure(validate_url_mock: MagicMock) -> None: validate_url_mock.side_effect = ValidationError(any_error_message()) with patch("backend.check_stac_metadata.utils.processing_assets_model_with_meta"): lambda_handler( { DATASET_ID_KEY: any_dataset_id(), VERSION_ID_KEY: any_dataset_version_id(), METADATA_URL_KEY: any_s3_url(), }, any_lambda_context(), )
def should_validate_given_url(validate_url_mock: MagicMock) -> None: url = any_s3_url() with patch("backend.check_stac_metadata.utils.processing_assets_model_with_meta"): lambda_handler( { DATASET_ID_KEY: any_dataset_id(), VERSION_ID_KEY: any_dataset_version_id(), METADATA_URL_KEY: url, }, any_lambda_context(), ) validate_url_mock.assert_called_once_with(url)
def should_log_event_payload() -> None: payload = { DATASET_ID_KEY: any_dataset_id(), VERSION_ID_KEY: any_dataset_version_id(), METADATA_URL_KEY: any_s3_url(), } expected_log = dumps({"event": payload}) with patch.object( logging.getLogger("backend.check_stac_metadata.task"), "debug") as logger_mock, patch( "backend.check_stac_metadata.task.STACDatasetValidator.run"): lambda_handler( payload, any_lambda_context(), ) logger_mock.assert_any_call(expected_log)
def should_save_staging_access_validation_results( validation_results_factory_mock: MagicMock, get_object_mock: MagicMock, ) -> None: validation_results_table_name = get_param(ParameterName.STORAGE_VALIDATION_RESULTS_TABLE_NAME) expected_error = ClientError( { AWS_RESPONSE_ERROR_KEY: { AWS_RESPONSE_ERROR_CODE_KEY: "TEST", AWS_RESPONSE_ERROR_MESSAGE_KEY: "TEST", } }, operation_name="get_object", ) get_object_mock.side_effect = expected_error s3_url = any_s3_url() dataset_id = any_dataset_id() version_id = any_dataset_version_id() lambda_handler( { DATASET_ID_KEY: dataset_id, VERSION_ID_KEY: version_id, METADATA_URL_KEY: s3_url, }, any_lambda_context(), ) hash_key = f"{DATASET_ID_PREFIX}{dataset_id}{DB_KEY_SEPARATOR}{VERSION_ID_PREFIX}{version_id}" assert validation_results_factory_mock.mock_calls == [ call(hash_key, validation_results_table_name), call().save( s3_url, Check.STAGING_ACCESS, ValidationResult.FAILED, details={MESSAGE_KEY: str(expected_error)}, ), ]
def should_save_staging_access_validation_results( validation_results_factory_mock: MagicMock, get_object_mock: MagicMock, ) -> None: validation_results_table_name = get_param( ParameterName.STORAGE_VALIDATION_RESULTS_TABLE_NAME) expected_error = ClientError( {"Error": { "Code": "TEST", "Message": "TEST" }}, operation_name="get_object") get_object_mock.side_effect = expected_error s3_url = any_s3_url() dataset_id = any_dataset_id() version_id = any_dataset_version_id() lambda_handler( { DATASET_ID_KEY: dataset_id, VERSION_ID_KEY: version_id, METADATA_URL_KEY: s3_url, }, any_lambda_context(), ) hash_key = f"DATASET#{dataset_id}#VERSION#{version_id}" assert validation_results_factory_mock.mock_calls == [ call(hash_key, validation_results_table_name), call().save( s3_url, Check.STAGING_ACCESS, ValidationResult.FAILED, details={"message": str(expected_error)}, ), ]
def should_return_error_when_schema_validation_fails( validate_schema_mock: MagicMock, subtests: SubTests) -> None: # Given error_message = any_error_message() error = ValidationError(error_message) validate_schema_mock.side_effect = error expected_log = dumps({ERROR_KEY: error}, default=str) with patch.object(LOGGER, "warning") as logger_mock: # When with subtests.test(msg="response"): response = lambda_handler({}, any_lambda_context()) # Then assert response == {ERROR_MESSAGE_KEY: error_message} # Then with subtests.test(msg="log"): logger_mock.assert_any_call(expected_log)
def should_insert_asset_urls_and_checksums_into_database(subtests: SubTests) -> None: # pylint: disable=too-many-locals # Given a metadata file with two assets first_asset_content = any_file_contents() first_asset_multihash = sha256(first_asset_content).hexdigest() second_asset_content = any_file_contents() second_asset_multihash = sha512(second_asset_content).hexdigest() dataset_id = any_dataset_id() version_id = any_dataset_version_id() with S3Object( file_object=BytesIO(initial_bytes=first_asset_content), bucket_name=ResourceName.STORAGE_BUCKET_NAME.value, key=any_safe_filename(), ) as first_asset_s3_object, S3Object( file_object=BytesIO(initial_bytes=second_asset_content), bucket_name=ResourceName.STORAGE_BUCKET_NAME.value, key=any_safe_filename(), ) as second_asset_s3_object: expected_hash_key = ( f"{DATASET_ID_PREFIX}{dataset_id}{DB_KEY_SEPARATOR}{VERSION_ID_PREFIX}{version_id}" ) metadata_stac_object = deepcopy(MINIMAL_VALID_STAC_COLLECTION_OBJECT) metadata_stac_object[STAC_ASSETS_KEY] = { any_asset_name(): { STAC_HREF_KEY: first_asset_s3_object.url, STAC_FILE_CHECKSUM_KEY: first_asset_multihash, }, any_asset_name(): { STAC_HREF_KEY: second_asset_s3_object.url, STAC_FILE_CHECKSUM_KEY: second_asset_multihash, }, } metadata_content = dumps(metadata_stac_object).encode() with S3Object( file_object=BytesIO(initial_bytes=metadata_content), bucket_name=ResourceName.STORAGE_BUCKET_NAME.value, key=any_safe_filename(), ) as metadata_s3_object: # When processing_assets_model = processing_assets_model_with_meta() expected_asset_items = [ processing_assets_model( hash_key=expected_hash_key, range_key=f"{ProcessingAssetType.DATA.value}{DB_KEY_SEPARATOR}0", url=first_asset_s3_object.url, multihash=first_asset_multihash, ), processing_assets_model( hash_key=expected_hash_key, range_key=f"{ProcessingAssetType.DATA.value}{DB_KEY_SEPARATOR}1", url=second_asset_s3_object.url, multihash=second_asset_multihash, ), ] expected_metadata_items = [ processing_assets_model( hash_key=expected_hash_key, range_key=f"{ProcessingAssetType.METADATA.value}{DB_KEY_SEPARATOR}0", url=metadata_s3_object.url, ), ] lambda_handler( { DATASET_ID_KEY: dataset_id, VERSION_ID_KEY: version_id, METADATA_URL_KEY: metadata_s3_object.url, }, any_lambda_context(), ) # Then actual_items = processing_assets_model.query( expected_hash_key, processing_assets_model.sk.startswith( f"{ProcessingAssetType.DATA.value}{DB_KEY_SEPARATOR}" ), ) for actual_item, expected_item in zip(actual_items, expected_asset_items): with subtests.test(): assert actual_item.attribute_values == expected_item.attribute_values actual_items = processing_assets_model.query( expected_hash_key, processing_assets_model.sk.startswith( f"{ProcessingAssetType.METADATA.value}{DB_KEY_SEPARATOR}" ), ) for actual_item, expected_item in zip(actual_items, expected_metadata_items): with subtests.test(): assert actual_item.attribute_values == expected_item.attribute_values
def should_save_json_schema_validation_results_per_file(subtests: SubTests) -> None: base_url = f"{S3_URL_PREFIX}{ResourceName.STAGING_BUCKET_NAME.value}/" valid_child_key = any_safe_filename() invalid_child_key = any_safe_filename() invalid_stac_object = deepcopy(MINIMAL_VALID_STAC_COLLECTION_OBJECT) invalid_stac_object.pop(STAC_ID_KEY) dataset_id = any_dataset_id() version_id = any_dataset_version_id() with S3Object( file_object=json_dict_to_file_object( { **deepcopy(MINIMAL_VALID_STAC_COLLECTION_OBJECT), STAC_LINKS_KEY: [ {STAC_HREF_KEY: f"{base_url}{valid_child_key}", "rel": "child"}, {STAC_HREF_KEY: f"{base_url}{invalid_child_key}", "rel": "child"}, ], } ), bucket_name=ResourceName.STAGING_BUCKET_NAME.value, key=any_safe_filename(), ) as root_s3_object, S3Object( file_object=json_dict_to_file_object(deepcopy(MINIMAL_VALID_STAC_COLLECTION_OBJECT)), bucket_name=ResourceName.STAGING_BUCKET_NAME.value, key=valid_child_key, ) as valid_child_s3_object, S3Object( file_object=json_dict_to_file_object(invalid_stac_object), bucket_name=ResourceName.STAGING_BUCKET_NAME.value, key=invalid_child_key, ) as invalid_child_s3_object: # When lambda_handler( { DATASET_ID_KEY: dataset_id, VERSION_ID_KEY: version_id, METADATA_URL_KEY: root_s3_object.url, }, any_lambda_context(), ) hash_key = f"{DATASET_ID_PREFIX}{dataset_id}{DB_KEY_SEPARATOR}{VERSION_ID_PREFIX}{version_id}" validation_results_model = validation_results_model_with_meta() with subtests.test(msg="Root validation results"): assert ( validation_results_model.get( hash_key=hash_key, range_key=( f"{CHECK_ID_PREFIX}{Check.JSON_SCHEMA.value}" f"{DB_KEY_SEPARATOR}{URL_ID_PREFIX}{root_s3_object.url}" ), consistent_read=True, ).result == ValidationResult.PASSED.value ) with subtests.test(msg="Valid child validation results"): assert ( validation_results_model.get( hash_key=hash_key, range_key=( f"{CHECK_ID_PREFIX}{Check.JSON_SCHEMA.value}" f"{DB_KEY_SEPARATOR}{URL_ID_PREFIX}{valid_child_s3_object.url}" ), consistent_read=True, ).result == ValidationResult.PASSED.value ) with subtests.test(msg="Invalid child validation results"): assert ( validation_results_model.get( hash_key=hash_key, range_key=( f"{CHECK_ID_PREFIX}{Check.JSON_SCHEMA.value}" f"{DB_KEY_SEPARATOR}{URL_ID_PREFIX}{invalid_child_s3_object.url}" ), consistent_read=True, ).result == ValidationResult.FAILED.value )
def should_save_json_schema_validation_results_per_file( subtests: SubTests) -> None: staging_bucket_name = get_param(ParameterName.STAGING_BUCKET_NAME) base_url = f"s3://{staging_bucket_name}/" valid_child_key = any_safe_filename() invalid_child_key = any_safe_filename() invalid_stac_object = deepcopy(MINIMAL_VALID_STAC_COLLECTION_OBJECT) invalid_stac_object.pop("id") dataset_id = any_dataset_id() version_id = any_dataset_version_id() with S3Object( file_object=json_dict_to_file_object({ **deepcopy(MINIMAL_VALID_STAC_COLLECTION_OBJECT), "links": [ { "href": f"{base_url}{valid_child_key}", "rel": "child" }, { "href": f"{base_url}{invalid_child_key}", "rel": "child" }, ], }), bucket_name=staging_bucket_name, key=any_safe_filename(), ) as root_s3_object, S3Object( file_object=json_dict_to_file_object( deepcopy(MINIMAL_VALID_STAC_COLLECTION_OBJECT)), bucket_name=staging_bucket_name, key=valid_child_key, ) as valid_child_s3_object, S3Object( file_object=json_dict_to_file_object(invalid_stac_object), bucket_name=staging_bucket_name, key=invalid_child_key, ) as invalid_child_s3_object: # When lambda_handler( { DATASET_ID_KEY: dataset_id, VERSION_ID_KEY: version_id, METADATA_URL_KEY: root_s3_object.url, }, any_lambda_context(), ) hash_key = f"DATASET#{dataset_id}#VERSION#{version_id}" validation_results_model = validation_results_model_with_meta() with subtests.test(msg="Root validation results"): assert (validation_results_model.get( hash_key=hash_key, range_key= f"CHECK#{Check.JSON_SCHEMA.value}#URL#{root_s3_object.url}", consistent_read=True, ).result == ValidationResult.PASSED.value) with subtests.test(msg="Valid child validation results"): assert (validation_results_model.get( hash_key=hash_key, range_key= f"CHECK#{Check.JSON_SCHEMA.value}#URL#{valid_child_s3_object.url}", consistent_read=True, ).result == ValidationResult.PASSED.value) with subtests.test(msg="Invalid child validation results"): assert (validation_results_model.get( hash_key=hash_key, range_key= f"CHECK#{Check.JSON_SCHEMA.value}#URL#{invalid_child_s3_object.url}", consistent_read=True, ).result == ValidationResult.FAILED.value)