def run_cleaners(cls, user, data):
for plan in PipelineConfiguration.RQA_CODING_PLANS + PipelineConfiguration.SURVEY_CODING_PLANS:
for cc in plan.coding_configurations:
if cc.cleaner is not None:
CleaningUtils.apply_cleaner_to_traced_data_iterable(
user, data, plan.raw_field, cc.coded_field, cc.cleaner,
cc.code_scheme)
def auto_code_surveys(cls, user, data, coda_output_dir):
# Auto-code surveys
for plan in PipelineConfiguration.SURVEY_CODING_PLANS:
for cc in plan.coding_configurations:
if cc.cleaner is not None:
CleaningUtils.apply_cleaner_to_traced_data_iterable(
user, data, plan.raw_field, cc.coded_field, cc.cleaner,
cc.code_scheme)
# Output single-scheme answers to coda for manual verification + coding
IOUtils.ensure_dirs_exist(coda_output_dir)
for plan in PipelineConfiguration.SURVEY_CODING_PLANS:
TracedDataCodaV2IO.compute_message_ids(user, data, plan.raw_field,
plan.id_field)
coda_output_path = path.join(coda_output_dir, plan.coda_filename)
with open(coda_output_path, "w") as f:
TracedDataCodaV2IO.export_traced_data_iterable_to_coda_2(
data, plan.raw_field, plan.time_field, plan.id_field, {
cc.coded_field: cc.code_scheme
for cc in plan.coding_configurations
}, f)
# Note: no need to handle location in any special way on this project because it is not being auto-coded
return data
def auto_code_surveys(cls, user, data, phone_uuid_table, coda_output_dir):
# Label missing data
for td in data:
missing_dict = dict()
for plan in PipelineConfiguration.DEMOGS_CODING_PLANS:
if td.get(plan.raw_field, "") == "":
na_label = CleaningUtils.make_label_from_cleaner_code(
plan.code_scheme, plan.code_scheme.get_code_with_control_code(Codes.TRUE_MISSING),
Metadata.get_call_location()
)
missing_dict[plan.coded_field] = na_label.to_dict()
td.append_data(missing_dict, Metadata(user, Metadata.get_call_location(), time.time()))
# Auto-code remaining data
for plan in PipelineConfiguration.DEMOGS_CODING_PLANS:
if plan.cleaner is not None:
CleaningUtils.apply_cleaner_to_traced_data_iterable(user, data, plan.raw_field, plan.coded_field,
plan.cleaner, plan.code_scheme)
# Output survey answers to coda for manual verification + coding
IOUtils.ensure_dirs_exist(coda_output_dir)
for plan in PipelineConfiguration.DEMOGS_CODING_PLANS:
TracedDataCodaV2IO.compute_message_ids(user, data, plan.raw_field, plan.id_field)
coda_output_path = path.join(coda_output_dir, plan.coda_filename)
with open(coda_output_path, 'w') as f:
TracedDataCodaV2IO.export_traced_data_iterable_to_coda_2(
data, plan.raw_field, plan.time_field, plan.id_field, {plan.coded_field: plan.code_scheme}, f
)
print("Coda demogs files successfully exported")
return data
def auto_code_surveys(cls, user, data, icr_output_dir, coda_output_dir):
# Auto-code surveys
for plan in PipelineConfiguration.SURVEY_CODING_PLANS:
if plan.cleaner is not None:
CleaningUtils.apply_cleaner_to_traced_data_iterable(
user, data, plan.raw_field, plan.coded_field, plan.cleaner,
plan.code_scheme)
# Output single-scheme answers to coda for manual verification + coding
IOUtils.ensure_dirs_exist(coda_output_dir)
for plan in PipelineConfiguration.SURVEY_CODING_PLANS:
TracedDataCodaV2IO.compute_message_ids(user, data, plan.raw_field,
plan.id_field)
coda_output_path = path.join(coda_output_dir, plan.coda_filename)
with open(coda_output_path, "w") as f:
TracedDataCodaV2IO.export_traced_data_iterable_to_coda_2(
data, plan.raw_field, plan.time_field, plan.id_field,
{plan.coded_field: plan.code_scheme}, f)
# Output messages for ICR
IOUtils.ensure_dirs_exist(icr_output_dir)
for plan in PipelineConfiguration.SURVEY_CODING_PLANS:
rqa_messages = []
for td in data:
if plan.raw_field in td:
rqa_messages.append(td)
icr_messages = ICRTools.generate_sample_for_icr(
rqa_messages, cls.ICR_MESSAGES_COUNT,
random.Random(cls.ICR_SEED))
icr_output_path = path.join(icr_output_dir, plan.icr_filename)
with open(icr_output_path, "w") as f:
TracedDataCSVIO.export_traced_data_iterable_to_csv(
icr_messages,
f,
headers=[plan.run_id_field, plan.raw_field])
return data
def auto_code_surveys(cls, user, data, pipeline_configuration,
coda_output_dir):
# Auto-code surveys
for plan in PipelineConfiguration.SURVEY_CODING_PLANS:
for cc in plan.coding_configurations:
if cc.cleaner is not None:
CleaningUtils.apply_cleaner_to_traced_data_iterable(
user, data, plan.raw_field, cc.coded_field, cc.cleaner,
cc.code_scheme)
# Remove survey data sent after the project finished
log.info(
"Hiding survey messages sent after the end of the project. These will not be exported in "
"production/analysis files")
out_of_range_count = 0
for td in data:
for plan in PipelineConfiguration.SURVEY_CODING_PLANS:
# TODO: Come up with a better solution here e.g. separate DEMOG/SURVEY lists
if plan.raw_field in ["have_voice_raw", "suggestions_raw"]:
continue
if plan.time_field in td and isoparse(
td[plan.time_field]
) > pipeline_configuration.project_end_date:
out_of_range_count += 1
td.hide_keys({plan.raw_field, plan.time_field},
Metadata(user, Metadata.get_call_location(),
time.time()))
log.info(
f"Hid {out_of_range_count} survey messages sent after the end of the project"
)
# For any locations where the cleaners assigned a code to a sub district, set the district code to NC
# (this is because only one column should have a value set in Coda)
for td in data:
if "mogadishu_sub_district_coded" in td:
mogadishu_code_id = td["mogadishu_sub_district_coded"][
"CodeID"]
if CodeSchemes.MOGADISHU_SUB_DISTRICT.get_code_with_id(
mogadishu_code_id).code_type == "Normal":
nc_label = CleaningUtils.make_label_from_cleaner_code(
CodeSchemes.MOGADISHU_SUB_DISTRICT,
CodeSchemes.MOGADISHU_SUB_DISTRICT.
get_code_with_control_code(Codes.NOT_CODED),
Metadata.get_call_location(),
)
td.append_data({"district_coded": nc_label.to_dict()},
Metadata(user, Metadata.get_call_location(),
time.time()))
# Output survey responses to coda for manual verification + coding
IOUtils.ensure_dirs_exist(coda_output_dir)
for plan in PipelineConfiguration.SURVEY_CODING_PLANS:
TracedDataCodaV2IO.compute_message_ids(user, data, plan.raw_field,
plan.id_field)
coda_output_path = path.join(coda_output_dir, plan.coda_filename)
with open(coda_output_path, "w") as f:
TracedDataCodaV2IO.export_traced_data_iterable_to_coda_2(
data, plan.raw_field, plan.time_field, plan.id_field, {
cc.coded_field: cc.code_scheme
for cc in plan.coding_configurations
}, f)
return data
def auto_code_surveys(cls, user, data, phone_uuid_table, coda_output_dir):
# Label missing data
for td in data:
missing_dict = dict()
for plan in PipelineConfiguration.SURVEY_CODING_PLANS:
if td.get(plan.raw_field, "") == "":
na_label = CleaningUtils.make_label_from_cleaner_code(
plan.code_scheme,
plan.code_scheme.get_code_with_control_code(
Codes.TRUE_MISSING), Metadata.get_call_location())
missing_dict[plan.coded_field] = na_label.to_dict()
td.append_data(
missing_dict,
Metadata(user, Metadata.get_call_location(), time.time()))
# Auto-code remaining data
for plan in PipelineConfiguration.SURVEY_CODING_PLANS:
if plan.cleaner is not None:
CleaningUtils.apply_cleaner_to_traced_data_iterable(
user, data, plan.raw_field, plan.coded_field, plan.cleaner,
plan.code_scheme)
# For any locations where the cleaners assigned a code to a sub district, set the district code to NC
# (this is because only one column should have a value set in Coda)
for td in data:
if "mogadishu_sub_district_coded" in td:
mogadishu_code_id = td["mogadishu_sub_district_coded"][
"CodeID"]
if CodeSchemes.MOGADISHU_SUB_DISTRICT.get_code_with_id(
mogadishu_code_id).code_type == "Normal":
nc_label = CleaningUtils.make_label_from_cleaner_code(
CodeSchemes.MOGADISHU_SUB_DISTRICT,
CodeSchemes.MOGADISHU_SUB_DISTRICT.
get_code_with_control_code(Codes.NOT_CODED),
Metadata.get_call_location(),
)
td.append_data({"district_coded": nc_label.to_dict()},
Metadata(user, Metadata.get_call_location(),
time.time()))
# Set operator from phone number
for td in data:
operator_clean = PhoneCleaner.clean_operator(
phone_uuid_table.get_phone(td["uid"]))
if operator_clean == Codes.NOT_CODED:
label = CleaningUtils.make_label_from_cleaner_code(
CodeSchemes.OPERATOR,
CodeSchemes.OPERATOR.get_code_with_control_code(
Codes.NOT_CODED), Metadata.get_call_location())
else:
label = CleaningUtils.make_label_from_cleaner_code(
CodeSchemes.OPERATOR,
CodeSchemes.OPERATOR.get_code_with_match_value(
operator_clean), Metadata.get_call_location())
td.append_data({"operator_coded": label.to_dict()},
Metadata(user, Metadata.get_call_location(),
time.time()))
# Output single-scheme answers to coda for manual verification + coding
IOUtils.ensure_dirs_exist(coda_output_dir)
for plan in PipelineConfiguration.SURVEY_CODING_PLANS:
if plan.raw_field == "mogadishu_sub_district_raw":
continue
TracedDataCodaV2IO.compute_message_ids(user, data, plan.raw_field,
plan.id_field)
coda_output_path = path.join(coda_output_dir, plan.coda_filename)
with open(coda_output_path, "w") as f:
TracedDataCodaV2IO.export_traced_data_iterable_to_coda_2(
data, plan.raw_field, plan.time_field, plan.id_field,
{plan.coded_field: plan.code_scheme}, f)
# Output location scheme to coda for manual verification + coding
output_path = path.join(coda_output_dir, "location.json")
TracedDataCodaV2IO.compute_message_ids(
user, data, "mogadishu_sub_district_raw",
"mogadishu_sub_district_raw_id")
with open(output_path, "w") as f:
TracedDataCodaV2IO.export_traced_data_iterable_to_coda_2(
data, "mogadishu_sub_district_raw",
"mogadishu_sub_district_time", "mogadishu_sub_district_raw_id",
{
"mogadishu_sub_district_coded":
CodeSchemes.MOGADISHU_SUB_DISTRICT,
"district_coded": CodeSchemes.DISTRICT,
"region_coded": CodeSchemes.REGION,
"state_coded": CodeSchemes.STATE,
"zone_coded": CodeSchemes.ZONE
}, f)
return data
def apply_manual_codes(cls, user, data, coda_input_dir):
# Merge manually coded data into the cleaned dataset
for plan in PipelineConfiguration.RQA_CODING_PLANS + PipelineConfiguration.SURVEY_CODING_PLANS:
if plan.coda_filename is None:
continue
coda_input_path = path.join(coda_input_dir, plan.coda_filename)
for cc in plan.coding_configurations:
if not cc.requires_manual_verification:
continue
f = None
try:
if path.exists(coda_input_path):
f = open(coda_input_path, "r")
if cc.coding_mode == CodingModes.SINGLE:
TracedDataCodaV2IO.import_coda_2_to_traced_data_iterable(
user, data, plan.id_field,
{cc.coded_field: cc.code_scheme}, f)
else:
TracedDataCodaV2IO.import_coda_2_to_traced_data_iterable_multi_coded(
user, data, plan.id_field,
{cc.coded_field: cc.code_scheme}, f)
finally:
if f is not None:
f.close()
if PipelineConfiguration.WS_CORRECT_DATASET_SCHEME is not None:
f = None
try:
if path.exists(coda_input_path):
f = open(coda_input_path, "r")
TracedDataCodaV2IO.import_coda_2_to_traced_data_iterable(
user, data, plan.id_field, {
f"{plan.raw_field}_correct_dataset":
PipelineConfiguration.WS_CORRECT_DATASET_SCHEME
}, f)
finally:
if f is not None:
f.close()
# Label data for which there is no response as TRUE_MISSING.
# Label data for which the response is the empty string as NOT_CODED.
for td in data:
missing_dict = dict()
for plan in PipelineConfiguration.RQA_CODING_PLANS + PipelineConfiguration.SURVEY_CODING_PLANS:
for cc in plan.coding_configurations:
raw_field = cc.raw_field if cc.raw_field is not None else plan.raw_field
if raw_field not in td:
na_label = CleaningUtils.make_label_from_cleaner_code(
cc.code_scheme,
cc.code_scheme.get_code_with_control_code(
Codes.TRUE_MISSING),
Metadata.get_call_location()).to_dict()
missing_dict[
cc.
coded_field] = na_label if cc.coding_mode == CodingModes.SINGLE else [
na_label
]
for cc in plan.coding_configurations:
raw_field = cc.raw_field if cc.raw_field is not None else plan.raw_field
if td.get(raw_field) == "":
nc_label = CleaningUtils.make_label_from_cleaner_code(
cc.code_scheme,
cc.code_scheme.get_code_with_control_code(
Codes.NOT_CODED),
Metadata.get_call_location()).to_dict()
missing_dict[
cc.
coded_field] = nc_label if cc.coding_mode == CodingModes.SINGLE else [
nc_label
]
td.append_data(
missing_dict,
Metadata(user, Metadata.get_call_location(), time.time()))
# Mark data that is noise as Codes.NOT_CODED
for td in data:
if td.get("noise", False):
nc_dict = dict()
for plan in PipelineConfiguration.RQA_CODING_PLANS:
for cc in plan.coding_configurations:
if cc.coded_field not in td:
nc_label = CleaningUtils.make_label_from_cleaner_code(
cc.code_scheme,
cc.code_scheme.get_code_with_control_code(
Codes.NOT_CODED),
Metadata.get_call_location()).to_dict()
nc_dict[
cc.
coded_field] = nc_label if cc.coding_mode == CodingModes.SINGLE else [
nc_label
]
td.append_data(
nc_dict,
Metadata(user, Metadata.get_call_location(), time.time()))
# Run the cleaners that don't require manual verification again, this time setting "checked" to True
for plan in PipelineConfiguration.RQA_CODING_PLANS + PipelineConfiguration.SURVEY_CODING_PLANS:
for cc in plan.coding_configurations:
if cc.cleaner is not None and not cc.requires_manual_verification:
raw_field = cc.raw_field if cc.raw_field is not None else plan.raw_field
CleaningUtils.apply_cleaner_to_traced_data_iterable(
user,
data,
raw_field,
cc.coded_field,
cc.cleaner,
cc.code_scheme,
set_checked=True)
# Run code imputation functions
for plan in PipelineConfiguration.RQA_CODING_PLANS + PipelineConfiguration.SURVEY_CODING_PLANS:
if plan.code_imputation_function is not None:
plan.code_imputation_function(user, data,
plan.coding_configurations)
cls._impute_coding_error_codes(user, data)
return data
def test_export_import_one_single_coded_scheme(self):
file_path = path.join(self.test_dir, "coda_2_test.json")
# Build raw input data
message_dicts = [
{"gender_raw": "woman", "gender_sent_on": "2018-11-01T07:13:04+03:00"},
{"gender_raw": "", "gender_sent_on": "2018-11-01T07:17:04+03:00"},
{"gender_raw": "hiya", "gender_sent_on": "2018-11-01T07:19:04+05:00"},
{},
{"gender_raw": "boy", "gender_sent_on": "2018-11-02T19:00:29+03:00"},
{"gender_raw": "man", "gender_sent_on": "2018-11-02T19:00:29+03:00"},
]
messages = [TracedData(d, Metadata("test_user", Metadata.get_call_location(), i))
for i, d in enumerate(message_dicts)]
# Add message ids
TracedDataCodaV2IO.compute_message_ids("test_user", messages, "gender_raw", "gender_coda_id")
# Load gender scheme
with open("tests/traced_data/resources/coda_2_gender_scheme.json") as f:
gender_scheme = CodeScheme.from_firebase_map(json.load(f))
# Apply the English gender cleaner
with mock.patch("core_data_modules.util.TimeUtils.utc_now_as_iso_string") as time_mock, \
mock.patch("core_data_modules.traced_data.Metadata.get_function_location") as location_mock:
time_mock.return_value = "2018-11-02T15:00:07+00:00"
location_mock.return_value = "english.DemographicCleaner.clean_gender"
CleaningUtils.apply_cleaner_to_traced_data_iterable(
"test_user", messages, "gender_raw", "gender_coded",
english.DemographicCleaner.clean_gender, gender_scheme
)
# Export to a Coda 2 messages file
with open(file_path, "w") as f:
TracedDataCodaV2IO.export_traced_data_iterable_to_coda_2(
messages, "gender_raw", "gender_sent_on", "gender_coda_id", {"gender_coded": gender_scheme}, f)
self.assertTrue(filecmp.cmp(file_path, "tests/traced_data/resources/coda_2_export_expected_one_scheme.json"))
# Test importing with no file available
imported_messages = []
for td in messages:
imported_messages.append(td.copy())
TracedDataCodaV2IO.import_coda_2_to_traced_data_iterable(
"test_user", imported_messages, "gender_coda_id", {"gender_coded": gender_scheme})
# Deliberately testing the read can be done twice
TracedDataCodaV2IO.import_coda_2_to_traced_data_iterable(
"test_user", imported_messages, "gender_coda_id", {"gender_coded": gender_scheme})
na_id = gender_scheme.get_code_with_control_code(Codes.TRUE_MISSING).code_id
nr_id = gender_scheme.get_code_with_control_code(Codes.NOT_REVIEWED).code_id
# Set TRUE_MISSING codes
for td in imported_messages:
na_label = CleaningUtils.make_label_from_cleaner_code(
gender_scheme,
gender_scheme.get_code_with_control_code(Codes.TRUE_MISSING),
"test_export_traced_data_iterable_to_coda_2",
date_time_utc="2018-11-02T10:00:00+00:00"
)
if td.get("gender_raw", "") == "":
td.append_data({"gender_coded": na_label.to_dict()},
Metadata("test_user", Metadata.get_call_location(), time.time()))
imported_code_ids = [td["gender_coded"]["CodeID"] for td in imported_messages]
self.assertListEqual(imported_code_ids, [nr_id, na_id, nr_id, na_id, nr_id, nr_id])
# Test importing from the test file
imported_messages = []
for td in messages:
imported_messages.append(td.copy())
with open("tests/traced_data/resources/coda_2_import_test_one_scheme.json", "r") as f:
TracedDataCodaV2IO.import_coda_2_to_traced_data_iterable(
"test_user", imported_messages, "gender_coda_id", {"gender_coded": gender_scheme}, f)
# Set TRUE_MISSING codes
for td in imported_messages:
na_label = CleaningUtils.make_label_from_cleaner_code(
gender_scheme,
gender_scheme.get_code_with_control_code(Codes.TRUE_MISSING),
"test_export_traced_data_iterable_to_coda_2",
date_time_utc="2018-11-02T10:00:00+00:00"
)
if td.get("gender_raw", "") == "":
td.append_data({"gender_coded": na_label.to_dict()},
Metadata("test_user", Metadata.get_call_location(), time.time()))
imported_code_ids = [td["gender_coded"]["CodeID"] for td in imported_messages]
expected_code_ids = [
gender_scheme.get_code_with_match_value("female").code_id, # Manually approved auto-code
gender_scheme.get_code_with_control_code(Codes.TRUE_MISSING).code_id, # Empty raw message
gender_scheme.get_code_with_control_code(Codes.NOT_REVIEWED).code_id, # Manually assigned code which isn't checked
gender_scheme.get_code_with_control_code(Codes.TRUE_MISSING).code_id, # No raw message
gender_scheme.get_code_with_control_code(Codes.NOT_CODED).code_id, # Manually Not Coded
gender_scheme.get_code_with_control_code(Codes.NOT_REVIEWED).code_id, # Manually un-coded
]
self.assertListEqual(imported_code_ids, expected_code_ids)
# Add an element with the same raw text but a conflicting
messages.append(TracedData({
"gender_raw": "woman", "gender_sent_on": "2018-11-01T07:13:04+03:00",
"gender_coded": CleaningUtils.make_label_from_cleaner_code(
gender_scheme, gender_scheme.get_code_with_match_value("male"), "make_location_label",
date_time_utc="2018-11-03T13:40:50Z").to_dict()
}, Metadata("test_user", Metadata.get_call_location(), time.time())))
TracedDataCodaV2IO.compute_message_ids("test_user", messages, "gender_raw", "gender_coda_id")
with open(file_path, "w") as f:
try:
TracedDataCodaV2IO.export_traced_data_iterable_to_coda_2(
messages, "gender_raw", "gender_sent_on", "gender_coda_id", {"gender_coded": gender_scheme}, f)
except AssertionError as e:
assert str(e) == "Messages with the same id " \
"(cf2e5bff1ef03dcd20d1a0b18ef7d89fc80a3554434165753672f6f40fde1d25) have different " \
"labels for coded_key 'gender_coded'"
return
self.fail("Exporting data with conflicting labels did not fail")