def test_noop_without_staging(self) -> None:
not_pointed_at_staging_file = GcsfsFilePath.from_directory_and_file_name(
self.config_with_path("gnarly").output_directory,
"staging_results.txt")
self.assertEqual(
ExportBigQueryViewConfig.revert_staging_path_to_original(
not_pointed_at_staging_file),
GcsfsFilePath.from_absolute_path(
"gs://gnarly/staging_results.txt"),
)
def test_happy_path(self) -> None:
pointed_at_staging_file = GcsfsFilePath.from_directory_and_file_name(
self.config_with_path(
"gnarly").pointed_to_staging_subdirectory().output_directory,
"foo.txt",
)
self.assertEqual(pointed_at_staging_file.abs_path(),
"gnarly/staging/foo.txt")
self.assertEqual(
ExportBigQueryViewConfig.revert_staging_path_to_original(
pointed_at_staging_file),
GcsfsFilePath.from_absolute_path("gs://gnarly/foo.txt"),
)
def export(
self, export_configs: Sequence[ExportBigQueryViewConfig]
) -> List[GcsfsFilePath]:
logging.info("Starting composite BigQuery view export.")
staging_configs = [
config.pointed_to_staging_subdirectory()
for config in export_configs
]
all_staging_paths: List[GcsfsFilePath] = []
for view_exporter in self.delegate_view_exporters:
logging.info(
"Beginning staged export of results for view exporter delegate [%s]",
view_exporter.__class__)
staging_paths = view_exporter.export_and_validate(staging_configs)
all_staging_paths.extend(staging_paths)
logging.info(
"Completed staged export of results for view exporter delegate [%s]",
view_exporter.__class__)
logging.info("Copying staged export results to final location")
final_paths = []
for staging_path in all_staging_paths:
final_path = ExportBigQueryViewConfig.revert_staging_path_to_original(
staging_path)
self.fs.copy(staging_path, final_path)
final_paths.append(final_path)
logging.info("Deleting staged copies of the final output paths")
for staging_path in all_staging_paths:
self.fs.delete(staging_path)
logging.info("Completed composite BigQuery view export.")
return final_paths
