# Output messages to Coda
IOUtils.ensure_dirs_exist_for_file(coda_output_path)
if os.path.exists(prev_coda_path):
# TODO: Modifying this line once the coding frame has been developed to include lots of Nones feels a bit
# TODO: cumbersome. We could instead modify export_traced_data_iterable_to_coda to support a prev_f argument.
# TODO: Modify by adding code scheme keys once they are ready
scheme_keys = {
"Relevance": None,
"Code 1": None,
"Code 2": None,
"Code 3": None,
"Code 4": None
}
with open(coda_output_path, "w") as f, open(prev_coda_path,
"r") as prev_f:
TracedDataCodaIO.export_traced_data_iterable_to_coda_with_scheme(
show_messages, show_message_key, scheme_keys, f, prev_f=prev_f)
else:
with open(coda_output_path, "w") as f:
TracedDataCodaIO.export_traced_data_iterable_to_coda(
show_messages, show_message_key, f)
# Randomly select some messages to export for ICR
random.seed(0)
random.shuffle(show_messages)
icr_messages = show_messages[:ICR_MESSAGES_COUNT]
# Output ICR data to a CSV file
run_id_key = "{} (Run ID) - {}".format(variable_name, flow_name)
raw_text_key = "{} (Text) - {}".format(variable_name, flow_name)
IOUtils.ensure_dirs_exist_for_file(icr_output_path)
with open(icr_output_path, "w") as f:
# Write json output
if os.path.dirname(json_output_path) is not "" and not os.path.exists(
os.path.dirname(json_output_path)):
os.makedirs(os.path.dirname(json_output_path))
with open(json_output_path, "w") as f:
TracedDataJsonIO.export_traced_data_iterable_to_json(data,
f,
pretty_print=True)
if coding_mode == "coda":
# Write Coda output
if not os.path.exists(coded_output_directory):
os.makedirs(coded_output_directory)
with open(path.join(coded_output_directory, "gender.csv"), "w") as f:
TracedDataCodaIO.export_traced_data_iterable_to_coda_with_scheme(
data, "GENDER_R", "GENDER_R_clean", "Gender", f)
with open(path.join(coded_output_directory, "age.csv"), "w") as f:
TracedDataCodaIO.export_traced_data_iterable_to_coda_with_scheme(
data, "AGE_R", "AGE_R_clean", "Age", f)
# ATA uses LOCATION/NATIONALITY whereas BIBLIA uses LOCATION 1/LOCATION 2
if len(data) > 0 and "LOCATION_R" in data[0]:
with open(path.join(coded_output_directory, "location.csv"),
"w") as f:
TracedDataCodaIO.export_traced_data_iterable_to_coda(
data, "LOCATION_R", f)
with open(path.join(coded_output_directory, "nationality.csv"),
"w") as f:
TracedDataCodaIO.export_traced_data_iterable_to_coda(
data, "NATIONALITY_R", f)
os.path.dirname(json_output_path)):
os.makedirs(os.path.dirname(json_output_path))
with open(json_output_path, "w") as f:
TracedDataJsonIO.export_traced_data_iterable_to_json(data,
f,
pretty_print=True)
# Output for manual verification + coding
if coding_mode == "coda":
# Write Coda output
if not os.path.exists(coded_output_path):
os.makedirs(coded_output_path)
# FIXME: Set the <example-arguments> to export a particular column e.g. "age", "age_clean", "Age"
with open(path.join(coded_output_path, "<output-file>.csv"), "w") as f:
TracedDataCodaIO.export_traced_data_iterable_to_coda_with_scheme(
data, "<key-of-raw>", "<key-of-coded>", "<name-in-Coda>", f)
# FIXME: Re-use the above code sample to export other columns which need verifying/coding.
else:
assert coding_mode == "coding-csv", "coding_mode was not one of 'coda' or 'coding-csv'"
# Write Coding CSV output
if not os.path.exists(coded_output_path):
os.makedirs(coded_output_path)
# FIXME: Set the <example-arguments> to export a particular column e.g. "age", "age_clean", "Age"
with open(path.join(coded_output_path, "<output-file>.csv"), "w") as f:
TracedDataCodingCSVIO.export_traced_data_iterable_to_coding_csv_with_scheme(
data, "<key-of-raw>", "<key-of-coded>", f)
# FIXME: Re-use the above code sample to export other columns which need verifying/coding.
time.time()))
# Write json output
IOUtils.ensure_dirs_exist_for_file(json_output_path)
with open(json_output_path, "w") as f:
TracedDataJsonIO.export_traced_data_iterable_to_json(all_survey_data,
f,
pretty_print=True)
# Output for manual verification + coding
IOUtils.ensure_dirs_exist(coded_output_path)
# TODO: Tidy up the usage of keys here once the format of the keys has been updated.
for key in cleaning_plan.keys():
coded_output_file_path = path.join(coded_output_path,
"{}.csv".format(key.split(" ")[0]))
prev_coded_output_file_path = path.join(
prev_coded_path, "{}_coded.csv".format(key.split(" ")[0]))
if os.path.exists(prev_coded_output_file_path):
with open(coded_output_file_path,
"w") as f, open(prev_coded_output_file_path,
"r") as prev_f:
TracedDataCodaIO.export_traced_data_iterable_to_coda_with_scheme(
all_survey_data, key,
{key.split(" ")[0]: "{}_clean".format(key)}, f, prev_f)
else:
with open(coded_output_file_path, "w") as f:
TracedDataCodaIO.export_traced_data_iterable_to_coda_with_scheme(
all_survey_data, key,
{key.split(" ")[0]: "{}_clean".format(key)}, f)