def test_noop_without_staging(self) -> None: not_pointed_at_staging_file = GcsfsFilePath.from_directory_and_file_name( self.config_with_path("gnarly").output_directory, "staging_results.txt") self.assertEqual( ExportBigQueryViewConfig.revert_staging_path_to_original( not_pointed_at_staging_file), GcsfsFilePath.from_absolute_path( "gs://gnarly/staging_results.txt"), )
def test_happy_path(self) -> None: pointed_at_staging_file = GcsfsFilePath.from_directory_and_file_name( self.config_with_path( "gnarly").pointed_to_staging_subdirectory().output_directory, "foo.txt", ) self.assertEqual(pointed_at_staging_file.abs_path(), "gnarly/staging/foo.txt") self.assertEqual( ExportBigQueryViewConfig.revert_staging_path_to_original( pointed_at_staging_file), GcsfsFilePath.from_absolute_path("gs://gnarly/foo.txt"), )
def export( self, export_configs: Sequence[ExportBigQueryViewConfig] ) -> List[GcsfsFilePath]: logging.info("Starting composite BigQuery view export.") staging_configs = [ config.pointed_to_staging_subdirectory() for config in export_configs ] all_staging_paths: List[GcsfsFilePath] = [] for view_exporter in self.delegate_view_exporters: logging.info( "Beginning staged export of results for view exporter delegate [%s]", view_exporter.__class__) staging_paths = view_exporter.export_and_validate(staging_configs) all_staging_paths.extend(staging_paths) logging.info( "Completed staged export of results for view exporter delegate [%s]", view_exporter.__class__) logging.info("Copying staged export results to final location") final_paths = [] for staging_path in all_staging_paths: final_path = ExportBigQueryViewConfig.revert_staging_path_to_original( staging_path) self.fs.copy(staging_path, final_path) final_paths.append(final_path) logging.info("Deleting staged copies of the final output paths") for staging_path in all_staging_paths: self.fs.delete(staging_path) logging.info("Completed composite BigQuery view export.") return final_paths