def fetch_listening_groups_csvs(google_cloud_credentials_file_path,
pipeline_configuration, raw_data_dir):
for listening_group_csv_url in pipeline_configuration.listening_group_csv_urls:
listening_group = listening_group_csv_url.split("/")[-1]
log.info(f"Saving '{listening_group}' to file '{raw_data_dir}'...")
with open(f'{raw_data_dir}/{listening_group}',
"wb") as listening_group_output_file:
google_cloud_utils.download_blob_to_file(
google_cloud_credentials_file_path, listening_group_csv_url,
listening_group_output_file)
def fetch_from_gcloud_bucket(google_cloud_credentials_file_path, raw_data_dir, gcloud_source):
log.info("Fetching data from a gcloud bucket...")
for blob_url in gcloud_source.activation_flow_urls + gcloud_source.survey_flow_urls:
flow = blob_url.split("/")[-1]
traced_runs_output_path = f"{raw_data_dir}/{flow}"
if os.path.exists(traced_runs_output_path):
log.info(f"File '{traced_runs_output_path}' for flow '{flow}' already exists; skipping download")
continue
log.info(f"Saving '{flow}' to file '{traced_runs_output_path}'...")
with open(traced_runs_output_path, "wb") as traced_runs_output_file:
google_cloud_utils.download_blob_to_file(
google_cloud_credentials_file_path, blob_url, traced_runs_output_file)
def fetch_beneficiary_file_csvs(google_cloud_credentials_file_path,
pipeline_configuration, raw_data_dir):
for beneficiary_file_url in pipeline_configuration.beneficiary_file_urls:
beneficiary_file = beneficiary_file_url.split("/")[-1]
if os.path.exists(f'{raw_data_dir}/{beneficiary_file}'):
log.info(
f"File '{beneficiary_file}'already exists; skipping download")
continue
log.info(
f"Saving '{beneficiary_file}' to directory '{raw_data_dir}/{beneficiary_file}'..."
)
with open(f'{raw_data_dir}/{beneficiary_file}',
"wb") as beneficiary_output_file:
google_cloud_utils.download_blob_to_file(
google_cloud_credentials_file_path, beneficiary_file_url,
beneficiary_output_file)