def should_log_event_payload() -> None:
payload = deepcopy(MINIMAL_PAYLOAD)
expected_log = dumps({"event": payload})
with patch.object(LOGGER, "debug") as logger_mock, patch(
"backend.check_stac_metadata.task.STACDatasetValidator.run"):
lambda_handler(payload, any_lambda_context())
logger_mock.assert_any_call(expected_log)
def should_return_required_property_error_when_missing_mandatory_property(
validate_url_mock: MagicMock, ) -> None:
error_message = any_error_message()
validate_url_mock.side_effect = ValidationError(error_message)
with Dataset() as dataset:
response = lambda_handler(
{
DATASET_ID_KEY: dataset.dataset_id,
VERSION_ID_KEY: any_dataset_version_id(),
},
any_lambda_context(),
)
assert response == {ERROR_MESSAGE_KEY: error_message}
def should_fail_if_unknown_sqs_message_type() -> None:
with raises(UnhandledSQSMessageException):
lambda_handler(
{
RECORDS_KEY: [{
BODY_KEY: any_dataset_version_id(),
MESSAGE_ATTRIBUTES_KEY: {
MESSAGE_ATTRIBUTE_TYPE_KEY: {
STRING_VALUE_KEY_LOWER: "test",
DATA_TYPE_KEY: DATA_TYPE_STRING,
}
},
}]
},
any_lambda_context(),
)
def should_log_event_payload() -> None:
payload = {
DATASET_ID_KEY: any_dataset_id(),
VERSION_ID_KEY: any_dataset_version_id(),
METADATA_URL_KEY: any_s3_url(),
}
expected_log = dumps({"event": payload})
with patch.object(
logging.getLogger("backend.check_stac_metadata.task"),
"debug") as logger_mock, patch(
"backend.check_stac_metadata.task.STACDatasetValidator.run"):
lambda_handler(
payload,
any_lambda_context(),
)
logger_mock.assert_any_call(expected_log)
def should_return_error_when_schema_validation_fails(
validate_schema_mock: MagicMock, subtests: SubTests) -> None:
# Given
error_message = any_error_message()
error = ValidationError(error_message)
validate_schema_mock.side_effect = error
expected_log = dumps({ERROR_KEY: error}, default=str)
with patch.object(LOGGER, "warning") as logger_mock:
# When
with subtests.test(msg="response"):
response = lambda_handler({}, any_lambda_context())
# Then
assert response == {ERROR_MESSAGE_KEY: error_message}
# Then
with subtests.test(msg="log"):
logger_mock.assert_any_call(expected_log)
def should_succeed_and_trigger_sqs_update_to_catalog(
subtests: SubTests) -> None:
dataset_version = any_dataset_version_id()
filename = f"{any_safe_filename()}.json"
with Dataset() as dataset, patch(
"backend.update_dataset_catalog.task.SQS_RESOURCE"
) as sqs_mock, S3Object(
file_object=json_dict_to_file_object({
**deepcopy(MINIMAL_VALID_STAC_COLLECTION_OBJECT),
}),
bucket_name=ResourceName.STORAGE_BUCKET_NAME.value,
key=f"{dataset.dataset_prefix}/{dataset_version}/{filename}",
) as dataset_version_metadata:
response = lambda_handler(
{
DATASET_ID_KEY: dataset.dataset_id,
DATASET_PREFIX_KEY: dataset.dataset_prefix,
VERSION_ID_KEY: dataset_version,
METADATA_URL_KEY: f"{any_s3_url()}/{filename}",
},
any_lambda_context(),
)
expected_sqs_call = {
"MessageBody": dataset_version_metadata.key,
"MessageAttributes": {
MESSAGE_ATTRIBUTE_TYPE_KEY: {
STRING_VALUE_KEY: MESSAGE_ATTRIBUTE_TYPE_DATASET,
DATA_TYPE_KEY: DATA_TYPE_STRING,
}
},
}
with subtests.test(msg="success"):
assert response == {}
with subtests.test(msg="sqs called"):
assert sqs_mock.get_queue_by_name.return_value.send_message.called
with subtests.test(msg="correct url passed to sqs"):
assert (sqs_mock.get_queue_by_name.return_value.send_message.
call_args[1] == expected_sqs_call)
def should_create_new_root_catalog_if_doesnt_exist(
subtests: SubTests, s3_client: S3Client) -> None:
with Dataset() as dataset, S3Object(
file_object=json_dict_to_file_object({
**deepcopy(MINIMAL_VALID_STAC_CATALOG_OBJECT),
STAC_ID_KEY:
dataset.dataset_prefix,
STAC_TITLE_KEY:
dataset.title,
}),
bucket_name=ResourceName.STORAGE_BUCKET_NAME.value,
key=f"{dataset.dataset_prefix}/{CATALOG_KEY}",
):
expected_links: JsonList = [
{
STAC_REL_KEY: STAC_REL_ROOT,
STAC_HREF_KEY: f"./{CATALOG_KEY}",
STAC_TYPE_KEY: STAC_MEDIA_TYPE_JSON,
},
{
STAC_REL_KEY: STAC_REL_CHILD,
STAC_HREF_KEY: f"./{dataset.dataset_prefix}/{CATALOG_KEY}",
STAC_TYPE_KEY: STAC_MEDIA_TYPE_JSON,
},
]
try:
lambda_handler(
{
RECORDS_KEY: [{
BODY_KEY: dataset.dataset_prefix,
MESSAGE_ATTRIBUTES_KEY: {
MESSAGE_ATTRIBUTE_TYPE_KEY: {
STRING_VALUE_KEY_LOWER:
MESSAGE_ATTRIBUTE_TYPE_ROOT,
DATA_TYPE_KEY: DATA_TYPE_STRING,
}
},
}]
},
any_lambda_context(),
)
with smart_open(
f"{S3_URL_PREFIX}{ResourceName.STORAGE_BUCKET_NAME.value}/{CATALOG_KEY}"
) as new_root_metadata_file:
catalog_json = load(new_root_metadata_file)
with subtests.test(msg="catalog title"):
assert catalog_json[STAC_TITLE_KEY] == ROOT_CATALOG_TITLE
with subtests.test(msg="catalog description"):
assert catalog_json[
STAC_DESCRIPTION_KEY] == ROOT_CATALOG_DESCRIPTION
with subtests.test(msg="catalog links"):
assert catalog_json[STAC_LINKS_KEY] == expected_links
finally:
delete_s3_key(ResourceName.STORAGE_BUCKET_NAME.value, CATALOG_KEY,
s3_client)
def should_update_dataset_catalog_with_new_version_collection(
subtests: SubTests) -> None:
dataset_version = any_dataset_version_id()
collection_filename = f"{any_safe_filename()}.json"
item_filename = f"{any_safe_filename()}.json"
with Dataset() as dataset, S3Object(
file_object=json_dict_to_file_object({
**deepcopy(MINIMAL_VALID_STAC_ITEM_OBJECT),
STAC_ID_KEY:
any_dataset_version_id(),
STAC_LINKS_KEY: [
{
STAC_REL_KEY: STAC_REL_ROOT,
STAC_HREF_KEY: f"./{collection_filename}",
STAC_TYPE_KEY: STAC_MEDIA_TYPE_JSON,
},
{
STAC_REL_KEY: STAC_REL_PARENT,
STAC_HREF_KEY: f"./{collection_filename}",
STAC_TYPE_KEY: STAC_MEDIA_TYPE_JSON,
},
],
}),
bucket_name=ResourceName.STORAGE_BUCKET_NAME.value,
key=f"{dataset.dataset_prefix}/{dataset_version}/{item_filename}",
) as item_metadata, S3Object(
file_object=json_dict_to_file_object({
**deepcopy(MINIMAL_VALID_STAC_COLLECTION_OBJECT),
STAC_ID_KEY:
dataset_version,
STAC_TITLE_KEY:
dataset.title,
STAC_LINKS_KEY: [
{
STAC_REL_KEY: STAC_REL_ROOT,
STAC_HREF_KEY: f"./{collection_filename}",
STAC_TYPE_KEY: STAC_MEDIA_TYPE_JSON,
},
{
STAC_REL_KEY: STAC_REL_ITEM,
STAC_HREF_KEY: f"./{item_filename}",
STAC_TYPE_KEY: STAC_MEDIA_TYPE_GEOJSON,
},
],
}),
bucket_name=ResourceName.STORAGE_BUCKET_NAME.value,
key=
f"{dataset.dataset_prefix}/{dataset_version}/{collection_filename}",
) as dataset_version_metadata, S3Object(
file_object=json_dict_to_file_object({
**deepcopy(MINIMAL_VALID_STAC_CATALOG_OBJECT),
STAC_ID_KEY:
dataset.dataset_prefix,
STAC_TITLE_KEY:
dataset.title,
STAC_LINKS_KEY: [
{
STAC_REL_KEY: STAC_REL_ROOT,
STAC_HREF_KEY: f"../{CATALOG_KEY}",
STAC_TYPE_KEY: STAC_MEDIA_TYPE_JSON,
},
{
STAC_REL_KEY: STAC_REL_PARENT,
STAC_HREF_KEY: f"../{CATALOG_KEY}",
STAC_TYPE_KEY: STAC_MEDIA_TYPE_JSON,
},
],
}),
bucket_name=ResourceName.STORAGE_BUCKET_NAME.value,
key=f"{dataset.dataset_prefix}/{CATALOG_KEY}",
), S3Object(
file_object=json_dict_to_file_object({
**deepcopy(MINIMAL_VALID_STAC_CATALOG_OBJECT),
STAC_ID_KEY:
ROOT_CATALOG_ID,
STAC_DESCRIPTION_KEY:
ROOT_CATALOG_DESCRIPTION,
STAC_TITLE_KEY:
ROOT_CATALOG_TITLE,
STAC_LINKS_KEY: [
{
STAC_REL_KEY: STAC_REL_ROOT,
STAC_HREF_KEY: f"./{CATALOG_KEY}",
STAC_TYPE_KEY: STAC_MEDIA_TYPE_JSON,
},
{
STAC_REL_KEY: STAC_REL_CHILD,
STAC_HREF_KEY:
f"./{dataset.dataset_prefix}/{CATALOG_KEY}",
STAC_TYPE_KEY: STAC_MEDIA_TYPE_JSON,
},
],
}),
bucket_name=ResourceName.STORAGE_BUCKET_NAME.value,
key=CATALOG_KEY,
):
expected_dataset_catalog_links: JsonList = [
{
STAC_REL_KEY: STAC_REL_ROOT,
STAC_HREF_KEY: f"../{CATALOG_KEY}",
STAC_TYPE_KEY: STAC_MEDIA_TYPE_JSON,
},
{
STAC_REL_KEY: STAC_REL_PARENT,
STAC_HREF_KEY: f"../{CATALOG_KEY}",
STAC_TYPE_KEY: STAC_MEDIA_TYPE_JSON,
},
{
STAC_REL_KEY: STAC_REL_CHILD,
STAC_HREF_KEY: f"./{dataset_version}/{collection_filename}",
STAC_TYPE_KEY: STAC_MEDIA_TYPE_JSON,
},
]
expected_dataset_version_links: JsonList = [
{
STAC_REL_KEY: STAC_REL_ROOT,
STAC_HREF_KEY: f"../../{CATALOG_KEY}",
STAC_TYPE_KEY: STAC_MEDIA_TYPE_JSON,
},
{
STAC_REL_KEY: STAC_REL_ITEM,
STAC_HREF_KEY: f"./{item_filename}",
STAC_TYPE_KEY: STAC_MEDIA_TYPE_GEOJSON,
},
{
STAC_REL_KEY: STAC_REL_PARENT,
STAC_HREF_KEY: f"../{CATALOG_KEY}",
STAC_TYPE_KEY: STAC_MEDIA_TYPE_JSON,
},
]
expected_item_links: JsonList = [
{
STAC_REL_KEY: STAC_REL_ROOT,
STAC_HREF_KEY: f"../../{CATALOG_KEY}",
STAC_TYPE_KEY: STAC_MEDIA_TYPE_JSON,
},
{
STAC_REL_KEY: STAC_REL_PARENT,
STAC_HREF_KEY: f"./{collection_filename}",
STAC_TYPE_KEY: STAC_MEDIA_TYPE_JSON,
},
]
lambda_handler(
{
RECORDS_KEY: [{
BODY_KEY: dataset_version_metadata.key,
MESSAGE_ATTRIBUTES_KEY: {
MESSAGE_ATTRIBUTE_TYPE_KEY: {
STRING_VALUE_KEY_LOWER:
MESSAGE_ATTRIBUTE_TYPE_DATASET,
DATA_TYPE_KEY: DATA_TYPE_STRING,
}
},
}]
},
any_lambda_context(),
)
with subtests.test(msg="dataset catalog links"), smart_open(
f"{S3_URL_PREFIX}{ResourceName.STORAGE_BUCKET_NAME.value}/"
f"{dataset.dataset_prefix}/{CATALOG_KEY}"
) as updated_dataset_metadata_file:
catalog_json = load(updated_dataset_metadata_file)
assert catalog_json[
STAC_LINKS_KEY] == expected_dataset_catalog_links
with subtests.test(msg="dataset version links"), smart_open(
f"{S3_URL_PREFIX}{ResourceName.STORAGE_BUCKET_NAME.value}"
f"/{dataset_version_metadata.key}"
) as updated_dataset_metadata_file:
version_json = load(updated_dataset_metadata_file)
assert version_json[
STAC_LINKS_KEY] == expected_dataset_version_links
with subtests.test(msg="item links"), smart_open(
f"{S3_URL_PREFIX}{ResourceName.STORAGE_BUCKET_NAME.value}"
f"/{item_metadata.key}") as updated_item_metadata_file:
item_json = load(updated_item_metadata_file)
assert item_json[STAC_LINKS_KEY] == expected_item_links
def should_update_existing_root_catalog(subtests: SubTests) -> None:
with Dataset() as existing_dataset, S3Object(
file_object=json_dict_to_file_object({
**deepcopy(MINIMAL_VALID_STAC_CATALOG_OBJECT),
STAC_ID_KEY:
existing_dataset.dataset_prefix,
STAC_TITLE_KEY:
existing_dataset.title,
}),
bucket_name=ResourceName.STORAGE_BUCKET_NAME.value,
key=f"{existing_dataset.dataset_prefix}/{CATALOG_KEY}",
):
original_links: JsonList = [
{
STAC_REL_KEY: STAC_REL_ROOT,
STAC_HREF_KEY: f"./{CATALOG_KEY}",
STAC_TYPE_KEY: STAC_MEDIA_TYPE_JSON,
},
{
STAC_REL_KEY: STAC_REL_CHILD,
STAC_HREF_KEY:
f"./{existing_dataset.dataset_prefix}/{CATALOG_KEY}",
STAC_TYPE_KEY: STAC_MEDIA_TYPE_JSON,
},
]
with Dataset() as dataset, S3Object(
file_object=json_dict_to_file_object({
**deepcopy(MINIMAL_VALID_STAC_CATALOG_OBJECT),
STAC_ID_KEY:
dataset.dataset_prefix,
STAC_TITLE_KEY:
dataset.title,
}),
bucket_name=ResourceName.STORAGE_BUCKET_NAME.value,
key=f"{dataset.dataset_prefix}/{CATALOG_KEY}",
), S3Object(
file_object=json_dict_to_file_object({
**deepcopy(MINIMAL_VALID_STAC_CATALOG_OBJECT),
STAC_ID_KEY:
ROOT_CATALOG_ID,
STAC_DESCRIPTION_KEY:
ROOT_CATALOG_DESCRIPTION,
STAC_TITLE_KEY:
ROOT_CATALOG_TITLE,
STAC_LINKS_KEY:
original_links,
}),
bucket_name=ResourceName.STORAGE_BUCKET_NAME.value,
key=CATALOG_KEY,
):
expected_root_links: JsonList = original_links + [
{
STAC_REL_KEY: STAC_REL_CHILD,
STAC_HREF_KEY: f"./{dataset.dataset_prefix}/{CATALOG_KEY}",
STAC_TYPE_KEY: STAC_MEDIA_TYPE_JSON,
}
]
expected_dataset_links: JsonList = [
{
STAC_REL_KEY: STAC_REL_ROOT,
STAC_HREF_KEY: f"../{CATALOG_KEY}",
STAC_TYPE_KEY: STAC_MEDIA_TYPE_JSON,
},
{
STAC_REL_KEY: STAC_REL_PARENT,
STAC_HREF_KEY: f"../{CATALOG_KEY}",
STAC_TYPE_KEY: STAC_MEDIA_TYPE_JSON,
},
]
lambda_handler(
{
RECORDS_KEY: [{
BODY_KEY: dataset.dataset_prefix,
MESSAGE_ATTRIBUTES_KEY: {
MESSAGE_ATTRIBUTE_TYPE_KEY: {
STRING_VALUE_KEY_LOWER:
MESSAGE_ATTRIBUTE_TYPE_ROOT,
DATA_TYPE_KEY: DATA_TYPE_STRING,
}
},
}]
},
any_lambda_context(),
)
with smart_open(
f"{S3_URL_PREFIX}{ResourceName.STORAGE_BUCKET_NAME.value}/{CATALOG_KEY}"
) as root_metadata_file, subtests.test(msg="root catalog links"):
root_catalog_json = load(root_metadata_file)
assert root_catalog_json[STAC_LINKS_KEY] == expected_root_links
with smart_open(
f"{S3_URL_PREFIX}{ResourceName.STORAGE_BUCKET_NAME.value}"
f"/{dataset.dataset_prefix}/{CATALOG_KEY}"
) as dataset_metadata_file, subtests.test(
msg="dataset catalog links"):
dataset_catalog_json = load(dataset_metadata_file)
assert dataset_catalog_json[
STAC_LINKS_KEY] == expected_dataset_links