def test_getIngestViewExportTaskArgs_happy(self) -> None:
# Arrange
region = self.create_fake_region(ingest_view_exports_enabled=True)
export_manager = self.create_export_manager(region)
export_manager.file_metadata_manager.get_ingest_view_metadata_for_most_recent_valid_job = Mock( # type: ignore
return_value=DirectIngestIngestFileMetadata(
file_id=_ID,
region_code=region.region_code,
file_tag="ingest_view",
normalized_file_name="normalized_file_name",
processed_time=_DATE_1,
is_invalidated=False,
is_file_split=False,
job_creation_time=_DATE_1,
export_time=_DATE_1,
datetimes_contained_lower_bound_exclusive=_DATE_1,
datetimes_contained_upper_bound_inclusive=_DATE_1,
discovery_time=_DATE_1,
))
export_manager.file_metadata_manager.get_metadata_for_raw_files_discovered_after_datetime = Mock( # type: ignore
return_value=[
DirectIngestRawFileMetadata(
file_id=2,
region_code=region.region_code,
file_tag="ingest_view",
discovery_time=_DATE_2,
normalized_file_name=
"unprocessed_2015-01-02T03:03:03:000003_raw_file_tag.csv",
processed_time=None,
datetimes_contained_upper_bound_inclusive=_DATE_2,
)
])
# Act
args = export_manager.get_ingest_view_export_task_args()
# Assert
self.assertListEqual(
args,
[
GcsfsIngestViewExportArgs(
ingest_view_name="ingest_view",
upper_bound_datetime_prev=_DATE_1,
upper_bound_datetime_to_export=_DATE_2,
)
],
)
def test_getIngestViewExportTaskArgs_rawCodeTableOlderThanLastExport(
self) -> None:
# Arrange
CODE_TABLE_TAG = "RECIDIVIZ_REFERENCE_ingest_view"
region = self.create_fake_region(ingest_view_exports_enabled=True)
export_manager = self.create_export_manager(
region, controller_file_tags=[CODE_TABLE_TAG])
export_manager.file_metadata_manager.get_ingest_view_metadata_for_most_recent_valid_job = Mock( # type: ignore
return_value=DirectIngestIngestFileMetadata(
file_id=_ID,
region_code=region.region_code,
file_tag=CODE_TABLE_TAG,
normalized_file_name="normalized_file_name",
processed_time=_DATE_2,
is_invalidated=False,
is_file_split=False,
job_creation_time=_DATE_2,
export_time=_DATE_2,
datetimes_contained_lower_bound_exclusive=_DATE_2 -
datetime.timedelta(days=7),
datetimes_contained_upper_bound_inclusive=_DATE_2,
discovery_time=_DATE_2,
))
export_manager.file_metadata_manager.get_metadata_for_raw_files_discovered_after_datetime = Mock( # type: ignore
return_value=[
DirectIngestRawFileMetadata(
file_id=2,
region_code=region.region_code,
file_tag=CODE_TABLE_TAG,
discovery_time=_DATE_1,
normalized_file_name=
"unprocessed_2015-01-02T03:03:03:000003_raw_file_tag.csv",
processed_time=None,
datetimes_contained_upper_bound_inclusive=_DATE_1,
)
])
# Act
args = export_manager.get_ingest_view_export_task_args()
# Assert
# New code tables are backdated but don't need to be re-ingested, so ignore them.
self.assertListEqual(args, [])
def test_getIngestViewExportTaskArgs_rawFileOlderThanLastExport(
self) -> None:
# Arrange
region = self.create_fake_region(ingest_view_exports_enabled=True)
export_manager = self.create_export_manager(region)
export_manager.file_metadata_manager.get_ingest_view_metadata_for_most_recent_valid_job = Mock( # type: ignore
return_value=DirectIngestIngestFileMetadata(
file_id=_ID,
region_code=region.region_code,
file_tag="ingest_view",
normalized_file_name="normalized_file_name",
processed_time=_DATE_2,
is_invalidated=False,
is_file_split=False,
job_creation_time=_DATE_2,
export_time=_DATE_2,
datetimes_contained_lower_bound_exclusive=_DATE_2,
datetimes_contained_upper_bound_inclusive=_DATE_2,
discovery_time=_DATE_2,
))
export_manager.file_metadata_manager.get_metadata_for_raw_files_discovered_after_datetime = Mock( # type: ignore
return_value=[
DirectIngestRawFileMetadata(
file_id=2,
region_code=region.region_code,
file_tag="ingest_view",
discovery_time=_DATE_1,
normalized_file_name=
"unprocessed_2015-01-02T03:03:03:000003_raw_file_tag.csv",
processed_time=None,
datetimes_contained_upper_bound_inclusive=_DATE_1,
)
])
# Act
with pytest.raises(
ValueError,
match=r"upper bound date.*before the last valid export"):
export_manager.get_ingest_view_export_task_args()