def test_to_normalized_unprocessed_file_name(self) -> None: original_file_name = 'test_file_tag.csv' expected_file_name = 'unprocessed_2019-08-12T00:00:00:000000_raw_test_file_tag.csv' self.assertEqual( expected_file_name, to_normalized_unprocessed_file_name( original_file_name, GcsfsDirectIngestFileType.RAW_DATA, datetime.datetime(2019, 8, 12, 0, 0, 0)))
def _upload_raw_data_file_to_bq(self, local_file_path: str): """Attempts to upload the given |local_file_path| to all relevant tables in BQ.""" logging.info( '\n\n\n ============== Beginning processing for file %s ================ \n\n\n', local_file_path) _, file_name = os.path.split(local_file_path) file_tag, _ = os.path.splitext(file_name) normalized_file_name = to_normalized_unprocessed_file_name( file_name=file_name, file_type=GcsfsDirectIngestFileType.RAW_DATA, dt=self.import_time) file_id_processed_tuple = get_file_id_and_processed_status_for_file( metadata_type=MetadataType.RAW, region_code=self.region_code, client=self.client, project_id=self.project_id, normalized_file_name=normalized_file_name) file_in_metadata = file_id_processed_tuple[0] is not None file_id = file_id_processed_tuple[0] if file_id_processed_tuple[0] is not None \ else get_next_available_file_id( metadata_type=MetadataType.RAW, client=self.client, project_id=self.project_id) file_already_processed = file_id_processed_tuple[1] if file_already_processed: logging.warning( 'File %s is already marked as processed. Skipping file processing.', normalized_file_name) return df = self._get_dataframe_from_csv_with_extra_cols( local_file_path=local_file_path, file_id=file_id) raw_data_dataset = f'{self.region_code.lower()}_raw_data' logging.info('\n\nLoaded dataframe has %d rows\n', df.shape[0]) logging.info( '\n\nLoaded dataframe with intent of uploading to %s.%s: \n\n%s', raw_data_dataset, file_tag, str(df.head())) if not self.dry_run: i = input('Continue? [y/n]: ') if i.upper() != 'Y': return self._append_df_to_table(dataset=raw_data_dataset, table_name=file_tag, df=df) self._mark_file_as_processed(file_id=file_id, file_tag=file_tag, normalized_file_name=normalized_file_name, processed_time=datetime.datetime.now(), file_exists_in_metadata=file_in_metadata)
def _generate_output_path(self, ingest_view_export_args: GcsfsIngestViewExportArgs, metadata: DirectIngestIngestFileMetadata) -> GcsfsFilePath: ingest_view = self.ingest_views_by_tag[ingest_view_export_args.ingest_view_name] if not metadata.normalized_file_name: output_file_name = to_normalized_unprocessed_file_name( f'{ingest_view.file_tag}.csv', GcsfsDirectIngestFileType.INGEST_VIEW, dt=ingest_view_export_args.upper_bound_datetime_to_export ) else: output_file_name = metadata.normalized_file_name return GcsfsFilePath.from_directory_and_file_name(self.ingest_directory_path, output_file_name)