"Code 4": None
}
with open(coda_output_path, "w") as f, open(prev_coda_path,
"r") as prev_f:
TracedDataCodaIO.export_traced_data_iterable_to_coda_with_scheme(
show_messages, show_message_key, scheme_keys, f, prev_f=prev_f)
else:
with open(coda_output_path, "w") as f:
TracedDataCodaIO.export_traced_data_iterable_to_coda(
show_messages, show_message_key, f)
# Randomly select some messages to export for ICR
random.seed(0)
random.shuffle(show_messages)
icr_messages = show_messages[:ICR_MESSAGES_COUNT]
# Output ICR data to a CSV file
run_id_key = "{} (Run ID) - {}".format(variable_name, flow_name)
raw_text_key = "{} (Text) - {}".format(variable_name, flow_name)
IOUtils.ensure_dirs_exist_for_file(icr_output_path)
with open(icr_output_path, "w") as f:
TracedDataCSVIO.export_traced_data_iterable_to_csv(
icr_messages, f, headers=[run_id_key, raw_text_key])
# Output to JSON
IOUtils.ensure_dirs_exist_for_file(json_output_path)
with open(json_output_path, "w") as f:
TracedDataJsonIO.export_traced_data_iterable_to_json(show_messages,
f,
pretty_print=True)
# Load data from JSON file
with open(input_path_messages, "r") as f:
messages_data = TracedDataJsonIO.import_json_to_traced_data_iterable(f)
# Load data from JSON file
with open(input_path_adverts, "r") as f:
adverts_data = TracedDataJsonIO.import_json_to_traced_data_iterable(f)
# Map "QUESTION_R" => "Message"
for td in adverts_data:
assert "QUESTION_R" in td.keys()
td.append_data(
{
"Message": td["QUESTION_R"],
"Date": td["start_date"],
"Group": group
}, Metadata(user, Metadata.get_call_location(), time.time()))
merged_data = list(messages_data)
merged_data.extend(adverts_data)
# Write json output
if os.path.dirname(json_output_path) is not "" and not os.path.exists(
os.path.dirname(json_output_path)):
os.makedirs(os.path.dirname(json_output_path))
with open(json_output_path, "w") as f:
TracedDataJsonIO.export_traced_data_iterable_to_json(merged_data,
f,
pretty_print=True)
raw_runs = rapid_pro.update_raw_runs_with_latest_modified(
flow_id, raw_runs, raw_export_log_file=raw_runs_log_file)
except FileNotFoundError:
log.info(f"File '{raw_runs_path}' not found, will fetch all runs from the Rapid Pro server for flow '{flow}'")
raw_runs = rapid_pro.get_raw_runs_for_flow_id(flow_id, raw_export_log_file=raw_runs_log_file)
# Fetch the latest contacts from Rapid Pro.
with open(contacts_log_path, "a") as raw_contacts_log_file:
raw_contacts = rapid_pro.update_raw_contacts_with_latest_modified(raw_contacts,
raw_export_log_file=raw_contacts_log_file)
# Convert the runs to TracedData.
traced_runs = rapid_pro.convert_runs_to_traced_data(
user, raw_runs, raw_contacts, phone_number_uuid_table, pipeline_configuration.rapid_pro_test_contact_uuids)
log.info(f"Saving {len(raw_runs)} raw runs to {raw_runs_path}...")
with open(raw_runs_path, "w") as raw_runs_file:
json.dump([run.serialize() for run in raw_runs], raw_runs_file)
log.info(f"Saved {len(raw_runs)} raw runs")
log.info(f"Saving {len(traced_runs)} traced runs to {traced_runs_output_path}...")
IOUtils.ensure_dirs_exist_for_file(traced_runs_output_path)
with open(traced_runs_output_path, "w") as traced_runs_output_file:
TracedDataJsonIO.export_traced_data_iterable_to_json(traced_runs, traced_runs_output_file, pretty_print=True)
log.info(f"Saved {len(traced_runs)} traced runs")
log.info(f"Saving {len(raw_contacts)} raw contacts to file '{raw_contacts_path}'...")
with open(raw_contacts_path, "w") as raw_contacts_file:
json.dump([contact.serialize() for contact in raw_contacts], raw_contacts_file)
log.info(f"Saved {len(raw_contacts)} contacts")
# Create a set of all the keys appearing in the data
keys = {key for message in messages for key in message.keys()}
keys = list(keys)
keys.remove("avf_phone_id")
# Add group name to each key
group_name = filename.split("_with_id.json")[0]
create_unique_keys(messages, keys, group_name)
# Output updated td-s
message_output_path = os.path.join(
messages_traced_json_output_path,
"{}_updated_keys.json".format(group_name))
with open(message_output_path, "w") as f:
TracedDataJsonIO.export_traced_data_iterable_to_json(
messages, f, pretty_print=True)
# Load surveys
for filename in os.listdir(surveys_input_path):
with open(os.path.join(surveys_input_path, filename)) as f:
surveys = TracedDataJsonIO.import_json_to_traced_data_iterable(f)
# Create a set of all the keys appearing in the data
keys = {key for survey in surveys for key in survey.keys()}
keys = list(keys)
keys.remove("avf_phone_id")
# Add group name to each key
group_name = filename.split("_with_id.json")[0]
create_unique_keys(surveys, keys, group_name)
Metadata(user, Metadata.get_call_location(),
time.time()))
# Mark missing entries in the raw data as true missing
for td in all_survey_data:
for key in cleaning_plan:
if key not in td:
td.append_data({key: Codes.TRUE_MISSING},
Metadata(user, Metadata.get_call_location(),
time.time()))
# Write json output
IOUtils.ensure_dirs_exist_for_file(json_output_path)
with open(json_output_path, "w") as f:
TracedDataJsonIO.export_traced_data_iterable_to_json(all_survey_data,
f,
pretty_print=True)
# Output for manual verification + coding
IOUtils.ensure_dirs_exist(coded_output_path)
# TODO: Tidy up the usage of keys here once the format of the keys has been updated.
for key in cleaning_plan.keys():
coded_output_file_path = path.join(coded_output_path,
"{}.csv".format(key.split(" ")[0]))
prev_coded_output_file_path = path.join(
prev_coded_path, "{}_coded.csv".format(key.split(" ")[0]))
if os.path.exists(prev_coded_output_file_path):
with open(coded_output_file_path,
"w") as f, open(prev_coded_output_file_path,
"r") as prev_f:
