def test_getIngestViewExportTaskArgs_happy(self) -> None: # Arrange region = self.create_fake_region(ingest_view_exports_enabled=True) export_manager = self.create_export_manager(region) export_manager.file_metadata_manager.get_ingest_view_metadata_for_most_recent_valid_job = Mock( # type: ignore return_value=DirectIngestIngestFileMetadata( file_id=_ID, region_code=region.region_code, file_tag="ingest_view", normalized_file_name="normalized_file_name", processed_time=_DATE_1, is_invalidated=False, is_file_split=False, job_creation_time=_DATE_1, export_time=_DATE_1, datetimes_contained_lower_bound_exclusive=_DATE_1, datetimes_contained_upper_bound_inclusive=_DATE_1, discovery_time=_DATE_1, )) export_manager.file_metadata_manager.get_metadata_for_raw_files_discovered_after_datetime = Mock( # type: ignore return_value=[ DirectIngestRawFileMetadata( file_id=2, region_code=region.region_code, file_tag="ingest_view", discovery_time=_DATE_2, normalized_file_name= "unprocessed_2015-01-02T03:03:03:000003_raw_file_tag.csv", processed_time=None, datetimes_contained_upper_bound_inclusive=_DATE_2, ) ]) # Act args = export_manager.get_ingest_view_export_task_args() # Assert self.assertListEqual( args, [ GcsfsIngestViewExportArgs( ingest_view_name="ingest_view", upper_bound_datetime_prev=_DATE_1, upper_bound_datetime_to_export=_DATE_2, ) ], )
def test_getIngestViewExportTaskArgs_rawCodeTableOlderThanLastExport( self) -> None: # Arrange CODE_TABLE_TAG = "RECIDIVIZ_REFERENCE_ingest_view" region = self.create_fake_region(ingest_view_exports_enabled=True) export_manager = self.create_export_manager( region, controller_file_tags=[CODE_TABLE_TAG]) export_manager.file_metadata_manager.get_ingest_view_metadata_for_most_recent_valid_job = Mock( # type: ignore return_value=DirectIngestIngestFileMetadata( file_id=_ID, region_code=region.region_code, file_tag=CODE_TABLE_TAG, normalized_file_name="normalized_file_name", processed_time=_DATE_2, is_invalidated=False, is_file_split=False, job_creation_time=_DATE_2, export_time=_DATE_2, datetimes_contained_lower_bound_exclusive=_DATE_2 - datetime.timedelta(days=7), datetimes_contained_upper_bound_inclusive=_DATE_2, discovery_time=_DATE_2, )) export_manager.file_metadata_manager.get_metadata_for_raw_files_discovered_after_datetime = Mock( # type: ignore return_value=[ DirectIngestRawFileMetadata( file_id=2, region_code=region.region_code, file_tag=CODE_TABLE_TAG, discovery_time=_DATE_1, normalized_file_name= "unprocessed_2015-01-02T03:03:03:000003_raw_file_tag.csv", processed_time=None, datetimes_contained_upper_bound_inclusive=_DATE_1, ) ]) # Act args = export_manager.get_ingest_view_export_task_args() # Assert # New code tables are backdated but don't need to be re-ingested, so ignore them. self.assertListEqual(args, [])
def test_getIngestViewExportTaskArgs_rawFileOlderThanLastExport( self) -> None: # Arrange region = self.create_fake_region(ingest_view_exports_enabled=True) export_manager = self.create_export_manager(region) export_manager.file_metadata_manager.get_ingest_view_metadata_for_most_recent_valid_job = Mock( # type: ignore return_value=DirectIngestIngestFileMetadata( file_id=_ID, region_code=region.region_code, file_tag="ingest_view", normalized_file_name="normalized_file_name", processed_time=_DATE_2, is_invalidated=False, is_file_split=False, job_creation_time=_DATE_2, export_time=_DATE_2, datetimes_contained_lower_bound_exclusive=_DATE_2, datetimes_contained_upper_bound_inclusive=_DATE_2, discovery_time=_DATE_2, )) export_manager.file_metadata_manager.get_metadata_for_raw_files_discovered_after_datetime = Mock( # type: ignore return_value=[ DirectIngestRawFileMetadata( file_id=2, region_code=region.region_code, file_tag="ingest_view", discovery_time=_DATE_1, normalized_file_name= "unprocessed_2015-01-02T03:03:03:000003_raw_file_tag.csv", processed_time=None, datetimes_contained_upper_bound_inclusive=_DATE_1, ) ]) # Act with pytest.raises( ValueError, match=r"upper bound date.*before the last valid export"): export_manager.get_ingest_view_export_task_args()