def fetch_listening_groups_csvs(google_cloud_credentials_file_path, pipeline_configuration, raw_data_dir): for listening_group_csv_url in pipeline_configuration.listening_group_csv_urls: listening_group = listening_group_csv_url.split("/")[-1] log.info(f"Saving '{listening_group}' to file '{raw_data_dir}'...") with open(f'{raw_data_dir}/{listening_group}', "wb") as listening_group_output_file: google_cloud_utils.download_blob_to_file( google_cloud_credentials_file_path, listening_group_csv_url, listening_group_output_file)
def fetch_from_gcloud_bucket(google_cloud_credentials_file_path, raw_data_dir, gcloud_source): log.info("Fetching data from a gcloud bucket...") for blob_url in gcloud_source.activation_flow_urls + gcloud_source.survey_flow_urls: flow = blob_url.split("/")[-1] traced_runs_output_path = f"{raw_data_dir}/{flow}" if os.path.exists(traced_runs_output_path): log.info(f"File '{traced_runs_output_path}' for flow '{flow}' already exists; skipping download") continue log.info(f"Saving '{flow}' to file '{traced_runs_output_path}'...") with open(traced_runs_output_path, "wb") as traced_runs_output_file: google_cloud_utils.download_blob_to_file( google_cloud_credentials_file_path, blob_url, traced_runs_output_file)
def fetch_beneficiary_file_csvs(google_cloud_credentials_file_path, pipeline_configuration, raw_data_dir): for beneficiary_file_url in pipeline_configuration.beneficiary_file_urls: beneficiary_file = beneficiary_file_url.split("/")[-1] if os.path.exists(f'{raw_data_dir}/{beneficiary_file}'): log.info( f"File '{beneficiary_file}'already exists; skipping download") continue log.info( f"Saving '{beneficiary_file}' to directory '{raw_data_dir}/{beneficiary_file}'..." ) with open(f'{raw_data_dir}/{beneficiary_file}', "wb") as beneficiary_output_file: google_cloud_utils.download_blob_to_file( google_cloud_credentials_file_path, beneficiary_file_url, beneficiary_output_file)