def test_get_ingest_view_metadata_pending_export_basic(self):
args = GcsfsIngestViewExportArgs(
ingest_view_name='file_tag',
upper_bound_datetime_prev=datetime.datetime(
2015, 1, 2, 2, 2, 2, 2),
upper_bound_datetime_to_export=datetime.datetime(
2015, 1, 2, 3, 3, 3, 3))
with freeze_time('2015-01-02T03:06:06'):
self.metadata_manager.register_ingest_file_export_job(args)
expected_list = [
DirectIngestIngestFileMetadata.new_with_defaults(
region_code='US_XX',
file_tag='file_tag',
is_invalidated=False,
is_file_split=False,
job_creation_time=datetime.datetime(2015, 1, 2, 3, 6, 6),
datetimes_contained_lower_bound_exclusive=datetime.datetime(
2015, 1, 2, 2, 2, 2, 2),
datetimes_contained_upper_bound_inclusive=datetime.datetime(
2015, 1, 2, 3, 3, 3, 3))
]
self.assertEqual(
expected_list,
self.metadata_manager.get_ingest_view_metadata_pending_export())
def run_split_ingest_file_progression_pre_processing(
self,
metadata_manager: PostgresDirectIngestFileMetadataManager,
original_file_metadata: DirectIngestIngestFileMetadata,
split_file_path: GcsfsFilePath,
discovery_before_export_recorded: bool = False):
"""Runs through the full progression of operations we expect to run on a split ingest file, up until processing.
"""
expected_metadata = DirectIngestIngestFileMetadata.new_with_defaults(
region_code=metadata_manager.region_code,
file_tag=original_file_metadata.file_tag,
is_invalidated=False,
is_file_split=True,
job_creation_time=datetime.datetime(2015, 1, 2, 3, 5, 5),
datetimes_contained_lower_bound_exclusive=original_file_metadata.
datetimes_contained_lower_bound_exclusive,
datetimes_contained_upper_bound_inclusive=original_file_metadata.
datetimes_contained_upper_bound_inclusive,
normalized_file_name=split_file_path.file_name,
export_time=None,
discovery_time=None,
processed_time=None,
)
with freeze_time('2015-01-02T03:05:05'):
split_file_metadata = self.metadata_manager.register_ingest_file_split(
original_file_metadata, split_file_path)
self.assertEqual(expected_metadata, split_file_metadata)
metadata = metadata_manager.get_file_metadata(split_file_path)
self.assertEqual(expected_metadata, metadata)
# ... export actually performed in here
if discovery_before_export_recorded:
with freeze_time('2015-01-02T03:06:07'):
metadata_manager.mark_file_as_discovered(split_file_path)
expected_metadata.discovery_time = datetime.datetime(
2015, 1, 2, 3, 6, 7)
expected_metadata.export_time = datetime.datetime(
2015, 1, 2, 3, 6, 7)
metadata = metadata_manager.get_file_metadata(split_file_path)
self.assertEqual(expected_metadata, metadata)
with freeze_time('2015-01-02T03:06:08'):
self.metadata_manager.mark_ingest_view_exported(
split_file_metadata)
expected_metadata.export_time = datetime.datetime(2015, 1, 2, 3, 6, 8)
metadata = metadata_manager.get_file_metadata(split_file_path)
self.assertEqual(expected_metadata, metadata)
if not discovery_before_export_recorded:
with freeze_time('2015-01-02T03:07:07'):
metadata_manager.mark_file_as_discovered(split_file_path)
expected_metadata.discovery_time = datetime.datetime(
2015, 1, 2, 3, 7, 7)
metadata = metadata_manager.get_file_metadata(split_file_path)
self.assertEqual(expected_metadata, metadata)
def run_ingest_view_file_progression(
self,
export_args: GcsfsIngestViewExportArgs,
metadata_manager: PostgresDirectIngestFileMetadataManager,
ingest_view_unprocessed_path: GcsfsFilePath,
discovery_before_export_recorded: bool = False,
split_file: bool = False):
"""Runs through the full progression of operations we expect to run on an individual ingest view file."""
with freeze_time('2015-01-02T03:05:05'):
metadata_manager.register_ingest_file_export_job(export_args)
ingest_file_metadata = metadata_manager.get_ingest_view_metadata_for_export_job(
export_args)
expected_metadata = DirectIngestIngestFileMetadata.new_with_defaults(
region_code=metadata_manager.region_code,
file_tag=export_args.ingest_view_name,
is_invalidated=False,
is_file_split=False,
job_creation_time=datetime.datetime(2015, 1, 2, 3, 5, 5),
datetimes_contained_lower_bound_exclusive=export_args.
upper_bound_datetime_prev,
datetimes_contained_upper_bound_inclusive=export_args.
upper_bound_datetime_to_export,
normalized_file_name=None,
export_time=None,
discovery_time=None,
processed_time=None,
)
self.assertEqual(expected_metadata, ingest_file_metadata)
with freeze_time('2015-01-02T03:06:06'):
self.metadata_manager.register_ingest_view_export_file_name(
ingest_file_metadata, ingest_view_unprocessed_path)
expected_metadata.normalized_file_name = ingest_view_unprocessed_path.file_name
metadata = metadata_manager.get_file_metadata(
ingest_view_unprocessed_path)
self.assertEqual(expected_metadata, metadata)
# ... export actually performed in here
if discovery_before_export_recorded:
with freeze_time('2015-01-02T03:06:07'):
metadata_manager.mark_file_as_discovered(
ingest_view_unprocessed_path)
expected_metadata.discovery_time = datetime.datetime(
2015, 1, 2, 3, 6, 7)
expected_metadata.export_time = datetime.datetime(
2015, 1, 2, 3, 6, 7)
metadata = metadata_manager.get_file_metadata(
ingest_view_unprocessed_path)
self.assertEqual(expected_metadata, metadata)
with freeze_time('2015-01-02T03:06:08'):
self.metadata_manager.mark_ingest_view_exported(
ingest_file_metadata)
expected_metadata.export_time = datetime.datetime(2015, 1, 2, 3, 6, 8)
metadata = metadata_manager.get_file_metadata(
ingest_view_unprocessed_path)
self.assertEqual(expected_metadata, metadata)
if not discovery_before_export_recorded:
with freeze_time('2015-01-02T03:07:07'):
metadata_manager.mark_file_as_discovered(
ingest_view_unprocessed_path)
expected_metadata.discovery_time = datetime.datetime(
2015, 1, 2, 3, 7, 7)
metadata = metadata_manager.get_file_metadata(
ingest_view_unprocessed_path)
self.assertEqual(expected_metadata, metadata)
split_file_paths_and_metadata: List[Tuple[
GcsfsFilePath, DirectIngestIngestFileMetadata]] = []
if split_file:
metadata = metadata_manager.get_file_metadata(
ingest_view_unprocessed_path)
if not isinstance(metadata, DirectIngestIngestFileMetadata):
self.fail(f'Unexpected metadata type {type(metadata)}')
for i in range(2):
split_file_path = self._make_unprocessed_path(
f'bucket/split{i}.csv',
GcsfsDirectIngestFileType.INGEST_VIEW)
self.run_split_ingest_file_progression_pre_processing(
metadata_manager, metadata, split_file_path,
discovery_before_export_recorded)
split_file_metadata = metadata_manager.get_file_metadata(
split_file_path)
if not isinstance(split_file_metadata,
DirectIngestIngestFileMetadata):
self.fail(
f'Unexpected split_file_metadata type [{split_file_metadata}].'
)
split_file_paths_and_metadata.append(
(split_file_path, split_file_metadata))
with freeze_time('2015-01-02T03:08:08'):
metadata_manager.mark_file_as_processed(
ingest_view_unprocessed_path)
expected_metadata.processed_time = datetime.datetime(
2015, 1, 2, 3, 8, 8)
metadata = metadata_manager.get_file_metadata(
ingest_view_unprocessed_path)
self.assertEqual(expected_metadata, metadata)
for split_file_path, split_file_metadata in split_file_paths_and_metadata:
expected_metadata = split_file_metadata
with freeze_time('2015-01-02T03:09:09'):
metadata_manager.mark_file_as_processed(split_file_path)
expected_metadata.processed_time = datetime.datetime(
2015, 1, 2, 3, 9, 9)
metadata = metadata_manager.get_file_metadata(split_file_path)
self.assertEqual(expected_metadata, metadata)