def test_yes_no_amb(self):
self.assertEqual(FoldStrategies.yes_no_amb(Codes.YES, Codes.YES), Codes.YES)
self.assertEqual(FoldStrategies.yes_no_amb(Codes.NO, Codes.NO), Codes.NO)
self.assertEqual(FoldStrategies.yes_no_amb(Codes.YES, Codes.NO), Codes.AMBIVALENT)
self.assertEqual(FoldStrategies.yes_no_amb(Codes.NOT_CODED, Codes.NOT_CODED), Codes.NOT_CODED)
# TODO: Check that this test case is desired
self.assertEqual(FoldStrategies.yes_no_amb(Codes.NOT_REVIEWED, Codes.YES), Codes.YES)
def test_assert_equal(self):
self.assertEqual(FoldStrategies.assert_equal("5", "5"), "5")
try:
FoldStrategies.assert_equal("6", "7")
self.fail("No AssertionError raised")
except AssertionError as e:
if str(e) == "No AssertionError raised":
raise e
self.assertEqual(str(e),
"Values should be the same but are different "
"(differing values were '6' and '7')")
def get_follow_up_coding_plans(pipeline_name):
return [
CodingPlan(
raw_field="s09_have_voice_raw",
time_field="s09_have_voice_time",
coda_filename="TIS_Plus_s09_have_voice.json",
coding_configurations=[
CodingConfiguration(
coding_mode=CodingModes.SINGLE,
code_scheme=CodeSchemes.S09_HAVE_VOICE,
cleaner=somali.DemographicCleaner.clean_yes_no,
coded_field="s09_have_voice_coded",
analysis_file_key="s09_have_voice",
fold_strategy=FoldStrategies.assert_label_ids_equal)
],
ws_code=CodeSchemes.WS_CORRECT_DATASET.get_code_with_match_value(
"have voice"),
raw_field_fold_strategy=FoldStrategies.assert_equal),
CodingPlan(
raw_field="s09_suggestions_raw",
time_field="s09_suggestions_time",
coda_filename="TIS_Plus_s09_suggestions.json",
coding_configurations=[
CodingConfiguration(
coding_mode=CodingModes.MULTIPLE,
code_scheme=CodeSchemes.S09_SUGGESTIONS,
coded_field="s09_suggestions_coded",
analysis_file_key="s09_suggestions",
fold_strategy=lambda x, y: FoldStrategies.list_of_labels(
CodeSchemes.S09_SUGGESTIONS, x, y))
],
ws_code=CodeSchemes.WS_CORRECT_DATASET.get_code_with_match_value(
"suggestions"),
raw_field_fold_strategy=FoldStrategies.assert_equal)
]
def test_concatenate(self):
self.assertEqual(FoldStrategies.concatenate("abc", "def"), "abc;def")
self.assertEqual(FoldStrategies.concatenate("abc", ""), "abc;")
self.assertEqual(FoldStrategies.concatenate("abc", None), "abc")
self.assertEqual(FoldStrategies.concatenate("", "def"), ";def")
self.assertEqual(FoldStrategies.concatenate(None, "def"), "def")
self.assertEqual(FoldStrategies.concatenate(None, None), None)
def get_follow_up_coding_plans(pipeline_name):
return [
CodingPlan(
raw_field="rqa_s01_programme_evaluation_raw",
time_field="sent_on",
coda_filename="OXFAM_WASH_s01_Programme_Evaluation.json",
icr_filename="oxfam_programme_evaluation.csv",
coding_configurations=[
CodingConfiguration(
coding_mode=CodingModes.MULTIPLE,
code_scheme=CodeSchemes.S01_PROGRAMME_EVALUATION,
coded_field="rqa_s01_programme_evaluation_coded",
analysis_file_key="rqa_s01_programme_evaluation",
fold_strategy=lambda x, y: FoldStrategies.list_of_labels(
CodeSchemes.S01_PROGRAMME_EVALUATION, x, y))
],
ws_code=CodeSchemes.WS_CORRECT_DATASET.get_code_with_match_value(
"OXFAM WASH s01 Programme Evaluation"),
raw_field_fold_strategy=FoldStrategies.concatenate),
CodingPlan(
raw_field="rqa_s01_accountability_raw",
time_field="sent_on",
coda_filename="OXFAM_WASH_s01_Accountability.json",
run_id_field="rqa_s01_accountability_run_id",
icr_filename="oxfam_accountability.csv",
coding_configurations=[
CodingConfiguration(
coding_mode=CodingModes.MULTIPLE,
code_scheme=CodeSchemes.S01_ACCOUNTABILITY,
coded_field="rqa_s01_accountability_coded",
analysis_file_key="rqa_s01_accountability",
fold_strategy=lambda x, y: FoldStrategies.list_of_labels(
CodeSchemes.S01_ACCOUNTABILITY, x, y))
],
ws_code=CodeSchemes.WS_CORRECT_DATASET.get_code_with_match_value(
"OXFAM WASH s01 Accountability"),
raw_field_fold_strategy=FoldStrategies.concatenate),
]
def get_rqa_coding_plans(pipeline_name):
return [
CodingPlan(raw_field="rqa_s01e01_raw",
time_field="sent_on",
run_id_field="rqa_s01e01_run_id",
coda_filename="TEST_PIPELINE_DANIEL_s01e01.json",
icr_filename="s01e01.csv",
coding_configurations=[
CodingConfiguration(
coding_mode=CodingModes.MULTIPLE,
code_scheme=CodeSchemes.S01E01,
coded_field="rqa_s01e01_coded",
analysis_file_key="rqa_s01e01",
fold_strategy=lambda x, y: FoldStrategies.list_of_labels(CodeSchemes.S01E01, x, y)
)
],
ws_code=CodeSchemes.WS_CORRECT_DATASET.get_code_with_match_value("s01e01"),
raw_field_fold_strategy=FoldStrategies.concatenate),
]
def test_assert_label_ids_equal(self):
self.assertEqual(FoldStrategies.assert_label_ids_equal(
Label("scheme-1", "code-2", "2019-10-01T12:20:14Z", Origin("x", "test", "automatic")).to_dict(),
Label("scheme-1", "code-2", "2019-10-01T12:20:14Z", Origin("x", "test", "automatic")).to_dict()
), Label("scheme-1", "code-2", "2019-10-01T12:20:14Z", Origin("x", "test", "automatic")).to_dict())
self.assertEqual(FoldStrategies.assert_label_ids_equal(
Label("scheme-1", "code-2", "2019-10-01T12:20:14Z", Origin("x", "test", "automatic")).to_dict(),
Label("scheme-1", "code-2", "2019-10-14T12:20:14Z", Origin("y", "test-2", "manual")).to_dict()
), Label("scheme-1", "code-2", "2019-10-01T12:20:14Z", Origin("x", "test", "automatic")).to_dict())
try:
FoldStrategies.assert_label_ids_equal(
Label("scheme-1", "code-1", "2019-10-01T12:20:14Z", Origin("x", "test", "automatic")).to_dict(),
Label("scheme-1", "code-2", "2019-10-01T12:20:14Z", Origin("x", "test", "automatic")).to_dict()
),
self.fail("No AssertionError raised")
except AssertionError as e:
if str(e) == "No AssertionError raised":
raise e
self.assertEqual(str(e),
"Labels should have the same SchemeID and CodeID, but at least one of those is different "
"(differing values were {'SchemeID': 'scheme-1', 'CodeID': 'code-1'} "
"and {'SchemeID': 'scheme-1', 'CodeID': 'code-2'})")
try:
FoldStrategies.assert_label_ids_equal(
Label("scheme-1", "code-2", "2019-10-01T12:20:14Z", Origin("x", "test", "automatic")).to_dict(),
Label("scheme-2", "code-2", "2019-10-01T12:20:14Z", Origin("x", "test", "automatic")).to_dict()
),
self.fail("No AssertionError raised")
except AssertionError as e:
if str(e) == "No AssertionError raised":
raise e
self.assertEqual(str(e),
"Labels should have the same SchemeID and CodeID, but at least one of those is different "
"(differing values were {'SchemeID': 'scheme-1', 'CodeID': 'code-2'} "
"and {'SchemeID': 'scheme-2', 'CodeID': 'code-2'})")
def get_rqa_coding_plans(pipeline_name):
return [
CodingPlan(raw_field="rqa_s01e01_raw",
time_field="sent_on",
run_id_field="rqa_s01e01_run_id",
coda_filename="UNICEF_COVID19_SOM_s01e01.json",
icr_filename="rqa_s01e01.csv",
coding_configurations=[
CodingConfiguration(
coding_mode=CodingModes.MULTIPLE,
code_scheme=CodeSchemes.S01E01,
coded_field="rqa_s01e01_coded",
analysis_file_key="rqa_s01e01",
fold_strategy=lambda x, y: FoldStrategies.list_of_labels(CodeSchemes.S01E01, x, y)
)
],
ws_code=CodeSchemes.WS_CORRECT_DATASET_SCHEME.get_code_with_match_value("s01e01"),
raw_field_fold_strategy=FoldStrategies.concatenate),
CodingPlan(raw_field="rqa_s01e02_raw",
time_field="sent_on",
run_id_field="rqa_s01e02_run_id",
coda_filename="UNICEF_COVID19_SOM_s01e02.json",
icr_filename="rqa_s01e02.csv",
coding_configurations=[
CodingConfiguration(
coding_mode=CodingModes.MULTIPLE,
code_scheme=CodeSchemes.S01E02,
coded_field="rqa_s01e02_coded",
analysis_file_key="rqa_s01e02",
fold_strategy=lambda x, y: FoldStrategies.list_of_labels(CodeSchemes.S01E02, x, y)
)
],
ws_code=CodeSchemes.WS_CORRECT_DATASET_SCHEME.get_code_with_match_value("s01e02"),
raw_field_fold_strategy=FoldStrategies.concatenate),
CodingPlan(raw_field="rqa_s01e03_raw",
time_field="sent_on",
run_id_field="rqa_s01e03_run_id",
coda_filename="UNICEF_COVID19_SOM_s01e03.json",
icr_filename="rqa_s01e03.csv",
katikati_survey_time_ranges=[("2020-07-24T14:00:00+03:00","2020-07-26T00:00:00+03:00")],
coding_configurations=[
CodingConfiguration(
coding_mode=CodingModes.MULTIPLE,
code_scheme=CodeSchemes.S01E03,
coded_field="rqa_s01e03_coded",
analysis_file_key="rqa_s01e03",
fold_strategy=lambda x, y: FoldStrategies.list_of_labels(CodeSchemes.S01E03, x, y)
)
],
ws_code=CodeSchemes.WS_CORRECT_DATASET_SCHEME.get_code_with_match_value("s01e03"),
raw_field_fold_strategy=FoldStrategies.concatenate),
CodingPlan(raw_field="rqa_s01e04_raw",
time_field="sent_on",
run_id_field="rqa_s01e04_run_id",
coda_filename="UNICEF_COVID19_SOM_s01e04.json",
icr_filename="rqa_s01e04.csv",
coding_configurations=[
CodingConfiguration(
coding_mode=CodingModes.MULTIPLE,
code_scheme=CodeSchemes.S01E04,
coded_field="rqa_s01e04_coded",
analysis_file_key="rqa_s01e04",
fold_strategy=lambda x, y: FoldStrategies.list_of_labels(CodeSchemes.S01E04, x, y)
)
],
ws_code=CodeSchemes.WS_CORRECT_DATASET_SCHEME.get_code_with_match_value("s01e04"),
raw_field_fold_strategy=FoldStrategies.concatenate),
CodingPlan(raw_field="csap_kalkaal_consent_raw",
time_field="sent_on",
run_id_field="csap_kalkaal_consent_run_id",
coda_filename="UNICEF_COVID19_SOM_csap_kalkaal_consent.json",
icr_filename="csap_kalkaal_consent.csv",
coding_configurations=[
CodingConfiguration(
coding_mode=CodingModes.MULTIPLE,
code_scheme=CodeSchemes.CSAP_KALKAAL_CONSENT,
coded_field="csap_kalkaal_consent_coded",
analysis_file_key="csap_kalkaal_consent",
fold_strategy=lambda x, y: FoldStrategies.list_of_labels(CodeSchemes.CSAP_KALKAAL_CONSENT, x, y)
)
],
ws_code=CodeSchemes.WS_CORRECT_DATASET_SCHEME.get_code_with_match_value("csap kalkaal consent"),
raw_field_fold_strategy=FoldStrategies.concatenate),
CodingPlan(raw_field="s01_closeout_raw",
time_field="sent_on",
run_id_field="s01_closeout_run_id",
coda_filename="UNICEF_COVID19_SOM_s01_closeout.json",
icr_filename="s01_closeout.csv",
coding_configurations=[
CodingConfiguration(
coding_mode=CodingModes.MULTIPLE,
code_scheme=CodeSchemes.S01_CLOSEOUT,
coded_field="s01_closeout_coded",
analysis_file_key="s01_closeout",
fold_strategy=lambda x, y: FoldStrategies.list_of_labels(CodeSchemes.S01_CLOSEOUT, x,
y)
)
],
ws_code=CodeSchemes.WS_CORRECT_DATASET_SCHEME.get_code_with_match_value("s01 closeout"),
raw_field_fold_strategy=FoldStrategies.concatenate)
]
def test_fold_list_of_labels(self):
na_code = Code("code-NA", "Control", "NA", -10, "NA", True, control_code=Codes.TRUE_MISSING)
nr_code = Code("code-NR", "Control", "NR", -20, "NR", True, control_code=Codes.NOT_REVIEWED)
nc_code = Code("code-NC", "Control", "NC", -30, "NC", True, control_code=Codes.NOT_CODED)
normal_1_code = Code("code-normal-1", "Normal", "Normal 1", 1, "normal_1", True)
normal_2_code = Code("code-normal-2", "Normal", "Normal 2", 2, "normal_2", True)
scheme_1 = CodeScheme("scheme-1", "Scheme 1", "1", [na_code, nr_code, nc_code, normal_1_code, normal_2_code])
scheme_2 = CodeScheme("scheme-2", "Scheme 2", "2", [])
na_label = Label("scheme-1", "code-NA", "2019-10-01T12:20:14Z", Origin("x", "test", "automatic")).to_dict()
nr_label = Label("scheme-1", "code-NR", "2019-10-01T12:25:18Z", Origin("x", "test", "automatic")).to_dict()
nc_label = Label("scheme-1", "code-NC", "2019-10-01T12:30:00Z", Origin("x", "test", "automatic")).to_dict()
na_label_2 = Label("scheme-1", "code-NA", "2019-10-01T13:00:00Z", Origin("x", "test", "automatic")).to_dict()
normal_1_label = Label("scheme-1", "code-normal-1", "2019-10-01T12:20:14Z", Origin("x", "test", "automatic")).to_dict()
normal_1_label_2 = Label("scheme-1", "code-normal-1", "2019-10-03T00:00:00Z", Origin("x", "test", "automatic")).to_dict()
normal_2_label = Label("scheme-1", "code-normal-2", "2019-10-01T15:00:00Z", Origin("x", "test", "automatic")).to_dict()
# Test empty lists are rejected
self.assertRaises(AssertionError, lambda: FoldStrategies.list_of_labels(scheme_1, [], []))
self.assertRaises(AssertionError, lambda: FoldStrategies.list_of_labels(scheme_1, [na_label], []))
# Test lists containing only NA labels return a single NA label
self.assertEqual(FoldStrategies.list_of_labels(scheme_1, [na_label], [na_label]), [na_label])
self.assertEqual(FoldStrategies.list_of_labels(scheme_1, [na_label], [na_label_2]), [na_label])
# Test lists containing an NA label and another label (including another NA label) are rejected
self.assertRaises(AssertionError, lambda: FoldStrategies.list_of_labels(scheme_1, [na_label, na_label], [na_label]))
self.assertRaises(AssertionError, lambda: FoldStrategies.list_of_labels(scheme_1, [na_label, normal_1_label], [na_label]))
# Test folding a normal label with an NA label
self.assertEqual(FoldStrategies.list_of_labels(scheme_1, [na_label], [normal_1_label]), [normal_1_label])
# Test folding various combinations of only normal labels
self.assertEqual(FoldStrategies.list_of_labels(scheme_1, [normal_1_label], [normal_1_label]), [normal_1_label])
self.assertEqual(FoldStrategies.list_of_labels(scheme_1, [normal_1_label, normal_2_label], [normal_1_label]),
[normal_1_label, normal_2_label])
self.assertEqual(FoldStrategies.list_of_labels(scheme_1, [normal_1_label, normal_2_label], [normal_1_label_2]),
[normal_1_label, normal_2_label])
# Test folding normal labels with a control code that isn't NA or NC
self.assertEqual(FoldStrategies.list_of_labels(scheme_1, [normal_1_label, normal_2_label], [nr_label]),
[normal_1_label, normal_2_label, nr_label])
# Test folding a label from a different code scheme
self.assertRaises(AssertionError, lambda: FoldStrategies.list_of_labels(scheme_2, [normal_1_label], [na_label]))
# (make sure that test would have been ok with the correct code scheme)
FoldStrategies.list_of_labels(scheme_1, [normal_1_label], [na_label])
# Test folding normal codes with NC codes
self.assertEqual(FoldStrategies.list_of_labels(scheme_1, [nc_label], [nc_label]), [nc_label])
self.assertEqual(FoldStrategies.list_of_labels(scheme_1, [na_label], [nc_label]), [nc_label])
self.assertEqual(FoldStrategies.list_of_labels(scheme_1, [normal_1_label], [nc_label]), [normal_1_label])
self.assertEqual(FoldStrategies.list_of_labels(scheme_1, [normal_1_label], [normal_2_label, nc_label]),
[normal_1_label, normal_2_label])
def test_matrix(self):
self.assertEqual(FoldStrategies.matrix(Codes.MATRIX_1, Codes.MATRIX_1), Codes.MATRIX_1)
self.assertEqual(FoldStrategies.matrix(Codes.MATRIX_0, Codes.MATRIX_1), Codes.MATRIX_1)
self.assertEqual(FoldStrategies.matrix(Codes.MATRIX_1, Codes.MATRIX_0), Codes.MATRIX_1)
self.assertEqual(FoldStrategies.matrix(Codes.MATRIX_0, Codes.MATRIX_0), Codes.MATRIX_0)
def get_rqa_coding_plans(pipeline_name):
return [
CodingPlan(raw_field="rqa_s01e01_raw",
time_field="sent_on",
run_id_field="rqa_s01e01_run_id",
coda_filename="UNDP_Kenya_s01e01.json",
icr_filename="s01e01.csv",
coding_configurations=[
CodingConfiguration(
coding_mode=CodingModes.MULTIPLE,
code_scheme=CodeSchemes.S01E01,
coded_field="rqa_s01e01_coded",
analysis_file_key="rqa_s01e01",
fold_strategy=lambda x, y: FoldStrategies.list_of_labels(CodeSchemes.S01E01, x, y)
)
],
ws_code=CodeSchemes.WS_CORRECT_DATASET.get_code_with_match_value("UNDP-Kenya s01e01"),
raw_field_fold_strategy=FoldStrategies.concatenate),
CodingPlan(raw_field="rqa_s01e02_raw",
time_field="sent_on",
run_id_field="rqa_s01e02_run_id",
coda_filename="UNDP_Kenya_s01e02.json",
icr_filename="s01e02.csv",
coding_configurations=[
CodingConfiguration(
coding_mode=CodingModes.MULTIPLE,
code_scheme=CodeSchemes.S01E02,
coded_field="rqa_s01e02_coded",
analysis_file_key="rqa_s01e02",
fold_strategy=lambda x, y: FoldStrategies.list_of_labels(CodeSchemes.S01E02, x, y)
)
],
ws_code=CodeSchemes.WS_CORRECT_DATASET.get_code_with_match_value("UNDP-Kenya s01e02"),
raw_field_fold_strategy=FoldStrategies.concatenate),
CodingPlan(raw_field="rqa_s01e03_raw",
time_field="sent_on",
run_id_field="rqa_s01e03_run_id",
coda_filename="UNDP_Kenya_s01e03.json",
icr_filename="s01e03.csv",
coding_configurations=[
CodingConfiguration(
coding_mode=CodingModes.MULTIPLE,
code_scheme=CodeSchemes.S01E03,
coded_field="rqa_s01e03_coded",
analysis_file_key="rqa_s01e03",
fold_strategy=lambda x, y: FoldStrategies.list_of_labels(CodeSchemes.S01E03, x, y)
)
],
ws_code=CodeSchemes.WS_CORRECT_DATASET.get_code_with_match_value("UNDP-Kenya s01e03"),
raw_field_fold_strategy=FoldStrategies.concatenate),
CodingPlan(raw_field="rqa_s01e04_raw",
time_field="sent_on",
run_id_field="rqa_s01e04_run_id",
coda_filename="UNDP_Kenya_s01e04.json",
icr_filename="s01e04.csv",
coding_configurations=[
CodingConfiguration(
coding_mode=CodingModes.MULTIPLE,
code_scheme=CodeSchemes.S01E04,
coded_field="rqa_s01e04_coded",
analysis_file_key="rqa_s01e04",
fold_strategy=lambda x, y: FoldStrategies.list_of_labels(CodeSchemes.S01E04, x, y)
)
],
ws_code=CodeSchemes.WS_CORRECT_DATASET.get_code_with_match_value("UNDP-Kenya s01e04"),
raw_field_fold_strategy=FoldStrategies.concatenate),
CodingPlan(raw_field="rqa_s01e05_raw",
time_field="sent_on",
run_id_field="rqa_s01e05_run_id",
coda_filename="UNDP_Kenya_s01e05.json",
icr_filename="s01e05.csv",
coding_configurations=[
CodingConfiguration(
coding_mode=CodingModes.MULTIPLE,
code_scheme=CodeSchemes.S01E05,
coded_field="rqa_s01e05_coded",
analysis_file_key="rqa_s01e05",
fold_strategy=lambda x, y: FoldStrategies.list_of_labels(CodeSchemes.S01E05, x, y)
)
],
ws_code=CodeSchemes.WS_CORRECT_DATASET.get_code_with_match_value("UNDP-Kenya s01e05"),
raw_field_fold_strategy=FoldStrategies.concatenate),
CodingPlan(raw_field="rqa_s01e06_raw",
time_field="sent_on",
run_id_field="rqa_s01e06_run_id",
coda_filename="UNDP_Kenya_s01e06.json",
icr_filename="s01e06.csv",
coding_configurations=[
CodingConfiguration(
coding_mode=CodingModes.MULTIPLE,
code_scheme=CodeSchemes.S01E06,
coded_field="rqa_s01e06_coded",
analysis_file_key="rqa_s01e06",
fold_strategy=lambda x, y: FoldStrategies.list_of_labels(CodeSchemes.S01E06, x, y)
)
],
ws_code=CodeSchemes.WS_CORRECT_DATASET.get_code_with_match_value("UNDP-Kenya s01e06"),
raw_field_fold_strategy=FoldStrategies.concatenate),
CodingPlan(raw_field="rqa_s01e07_raw",
time_field="sent_on",
run_id_field="rqa_s01e07_run_id",
coda_filename="UNDP_Kenya_s01e07.json",
icr_filename="s01e07.csv",
coding_configurations=[
CodingConfiguration(
coding_mode=CodingModes.MULTIPLE,
code_scheme=CodeSchemes.S01E07,
coded_field="rqa_s01e07_coded",
analysis_file_key="rqa_s01e07",
fold_strategy=lambda x, y: FoldStrategies.list_of_labels(CodeSchemes.S01E07, x, y)
)
],
ws_code=CodeSchemes.WS_CORRECT_DATASET.get_code_with_match_value("UNDP-Kenya s01e07"),
raw_field_fold_strategy=FoldStrategies.concatenate),
CodingPlan(raw_field="rqa_s01e08_raw",
time_field="sent_on",
run_id_field="rqa_s01e08_run_id",
coda_filename="UNDP_Kenya_s01e08.json",
icr_filename="s01e08.csv",
coding_configurations=[
CodingConfiguration(
coding_mode=CodingModes.MULTIPLE,
code_scheme=CodeSchemes.S01E08,
coded_field="rqa_s01e08_coded",
analysis_file_key="rqa_s01e08",
fold_strategy=lambda x, y: FoldStrategies.list_of_labels(CodeSchemes.S01E08, x, y)
)
],
ws_code=CodeSchemes.WS_CORRECT_DATASET.get_code_with_match_value("UNDP-Kenya s01e08"),
raw_field_fold_strategy=FoldStrategies.concatenate),
CodingPlan(raw_field="s01_close_out_raw",
time_field="sent_on",
run_id_field="s01_close_out_run_id",
coda_filename="UNDP_Kenya_s01_close_out.json",
icr_filename="s01_close_out.csv",
coding_configurations=[
CodingConfiguration(
coding_mode=CodingModes.MULTIPLE,
code_scheme=CodeSchemes.S01_CLOSE_OUT,
coded_field="s01_close_out_coded",
analysis_file_key="s01_close_out",
fold_strategy=lambda x, y: FoldStrategies.list_of_labels(CodeSchemes.S01_CLOSE_OUT, x, y)
)
],
ws_code=CodeSchemes.WS_CORRECT_DATASET.get_code_with_match_value("UNDP-Kenya s01 close out"),
raw_field_fold_strategy=FoldStrategies.concatenate)
]
return Codes.NOT_CODED
S01_RQA_CODING_PLAN = [
CodingPlan(raw_field="diagnostic_s01e01_raw",
time_field="sent_on",
run_id_field="diagnostic_s01e01_run_id",
coda_filename="COVID19_SOM_s01e01.json",
icr_filename="diagnostic_s01e01.csv",
coding_configurations=[
CodingConfiguration(
coding_mode=CodingModes.MULTIPLE,
code_scheme=CodeSchemes.DIAGNOSTIC_S01E01,
coded_field="diagnostic_s01e01_coded",
analysis_file_key="diagnostic_s01e01",
fold_strategy=lambda x, y: FoldStrategies.list_of_labels(CodeSchemes.DIAGNOSTIC_S01E01, x, y)
)
],
ws_code=CodeSchemes.WS_CORRECT_DATASET_SCHEME.get_code_with_match_value("covid19 som s01e01"),
raw_field_fold_strategy=FoldStrategies.concatenate),
CodingPlan(raw_field="diagnostic_s01e02_raw",
time_field="sent_on",
run_id_field="diagnostic_s01e02_run_id",
coda_filename="COVID19_SOM_s01e02.json",
icr_filename="diagnostic_s01e02.csv",
coding_configurations=[
CodingConfiguration(
coding_mode=CodingModes.MULTIPLE,
code_scheme=CodeSchemes.DIAGNOSTIC_S01E02,
coded_field="diagnostic_s01e02_coded",
def test_boolean_or(self):
self.assertEqual(FoldStrategies.boolean_or(Codes.TRUE, Codes.TRUE), Codes.TRUE)
self.assertEqual(FoldStrategies.boolean_or(Codes.FALSE, Codes.TRUE), Codes.TRUE)
self.assertEqual(FoldStrategies.boolean_or(Codes.FALSE, Codes.FALSE), Codes.FALSE)
def test_control_code(self):
self.assertEqual(FoldStrategies.control_code_by_precedence(Codes.TRUE_MISSING, Codes.NOT_CODED), Codes.NOT_CODED)
self.assertEqual(FoldStrategies.control_code_by_precedence(Codes.STOP, Codes.NOT_CODED), Codes.STOP)
def get_rqa_coding_plans(pipeline_name):
return [
CodingPlan(
raw_field="rqa_s01_pilot_raw",
time_field="sent_on",
run_id_field="rqa_s01_pilot_run_id",
coda_filename="COVID19_s01e01.json",
icr_filename="s01_pilot.csv",
coding_configurations=[
CodingConfiguration(
coding_mode=CodingModes.MULTIPLE,
code_scheme=CodeSchemes.S01_PILOT,
coded_field="rqa_s01_pilot_coded",
analysis_file_key="rqa_s01_pilot_",
fold_strategy=lambda x, y: FoldStrategies.list_of_labels(
CodeSchemes.S01_PILOT, x, y))
],
ws_code=CodeSchemes.WS_CORRECT_DATASET.get_code_with_match_value(
"COVID19 s01e01"),
raw_field_fold_strategy=FoldStrategies.concatenate),
CodingPlan(
raw_field="rqa_s01e01_raw",
time_field="sent_on",
run_id_field="rqa_s01e01_run_id",
coda_filename="COVID19_KE_Urban_s01e01.json",
icr_filename="s01e01.csv",
coding_configurations=[
CodingConfiguration(coding_mode=CodingModes.MULTIPLE,
code_scheme=CodeSchemes.S01E01,
coded_field="rqa_s01e01_coded",
analysis_file_key="rqa_s01e01_",
fold_strategy=lambda x, y: FoldStrategies.
list_of_labels(CodeSchemes.S01E01, x, y))
],
ws_code=CodeSchemes.WS_CORRECT_DATASET.get_code_with_match_value(
"COVID19-KE-Urban s01e01"),
raw_field_fold_strategy=FoldStrategies.concatenate),
CodingPlan(
raw_field="rqa_s01e02_raw",
time_field="sent_on",
run_id_field="rqa_s01e02_run_id",
coda_filename="COVID19_KE_Urban_s01e02.json",
icr_filename="s01e02.csv",
coding_configurations=[
CodingConfiguration(coding_mode=CodingModes.MULTIPLE,
code_scheme=CodeSchemes.S01E02,
coded_field="rqa_s01e02_coded",
analysis_file_key="rqa_s01e02_",
fold_strategy=lambda x, y: FoldStrategies.
list_of_labels(CodeSchemes.S01E02, x, y))
],
ws_code=CodeSchemes.WS_CORRECT_DATASET.get_code_with_match_value(
"COVID19-KE-Urban s01e02"),
raw_field_fold_strategy=FoldStrategies.concatenate),
CodingPlan(
raw_field="rqa_s01e03_raw",
time_field="sent_on",
run_id_field="rqa_s01e03_run_id",
coda_filename="COVID19_KE_Urban_s01e03.json",
icr_filename="s01e03.csv",
coding_configurations=[
CodingConfiguration(coding_mode=CodingModes.MULTIPLE,
code_scheme=CodeSchemes.S01E03,
coded_field="rqa_s01e03_coded",
analysis_file_key="rqa_s01e03_",
fold_strategy=lambda x, y: FoldStrategies.
list_of_labels(CodeSchemes.S01E03, x, y))
],
ws_code=CodeSchemes.WS_CORRECT_DATASET.get_code_with_match_value(
"COVID19-KE-Urban s01e03"),
raw_field_fold_strategy=FoldStrategies.concatenate),
CodingPlan(
raw_field="rqa_s01e04_raw",
time_field="sent_on",
run_id_field="rqa_s01e04_run_id",
coda_filename="COVID19_KE_Urban_s01e04.json",
icr_filename="s01e04.csv",
coding_configurations=[
CodingConfiguration(coding_mode=CodingModes.MULTIPLE,
code_scheme=CodeSchemes.S01E04,
coded_field="rqa_s01e04_coded",
analysis_file_key="rqa_s01e04_",
fold_strategy=lambda x, y: FoldStrategies.
list_of_labels(CodeSchemes.S01E04, x, y))
],
ws_code=CodeSchemes.WS_CORRECT_DATASET.get_code_with_match_value(
"COVID19-KE-Urban s01e04"),
raw_field_fold_strategy=FoldStrategies.concatenate),
CodingPlan(
raw_field="rqa_s01e05_raw",
time_field="sent_on",
run_id_field="rqa_s01e05_run_id",
coda_filename="COVID19_KE_Urban_s01e05.json",
icr_filename="s01e05.csv",
coding_configurations=[
CodingConfiguration(coding_mode=CodingModes.MULTIPLE,
code_scheme=CodeSchemes.S01E05,
coded_field="rqa_s01e05_coded",
analysis_file_key="rqa_s01e05_",
fold_strategy=lambda x, y: FoldStrategies.
list_of_labels(CodeSchemes.S01E05, x, y))
],
ws_code=CodeSchemes.WS_CORRECT_DATASET.get_code_with_match_value(
"COVID19-KE-Urban s01e05"),
raw_field_fold_strategy=FoldStrategies.concatenate),
CodingPlan(
raw_field="rqa_s01e06_raw",
time_field="sent_on",
run_id_field="rqa_s01e06_run_id",
coda_filename="COVID19_KE_Urban_s01e06.json",
icr_filename="s01e06.csv",
coding_configurations=[
CodingConfiguration(coding_mode=CodingModes.MULTIPLE,
code_scheme=CodeSchemes.S01E06,
coded_field="rqa_s01e06_coded",
analysis_file_key="rqa_s01e06_",
fold_strategy=lambda x, y: FoldStrategies.
list_of_labels(CodeSchemes.S01E06, x, y))
],
ws_code=CodeSchemes.WS_CORRECT_DATASET.get_code_with_match_value(
"COVID19-KE-Urban s01e06"),
raw_field_fold_strategy=FoldStrategies.concatenate)
]
def get_rqa_coding_plans(pipeline_name):
return [
CodingPlan(
raw_field="rqa_s01e01_raw",
time_field="sent_on",
run_id_field="rqa_s01e01_run_id",
coda_filename="WorldVision_s01e01.json",
icr_filename="s01e01.csv",
coding_configurations=[
CodingConfiguration(coding_mode=CodingModes.MULTIPLE,
code_scheme=CodeSchemes.S01E01,
coded_field="rqa_s01e01_coded",
analysis_file_key="rqa_s01e01_",
fold_strategy=lambda x, y: FoldStrategies.
list_of_labels(CodeSchemes.S01E01, x, y))
],
ws_code=CodeSchemes.WS_CORRECT_DATASET.get_code_with_match_value(
"s01e01"),
raw_field_fold_strategy=FoldStrategies.concatenate),
CodingPlan(
raw_field="rqa_s01e02_raw",
time_field="sent_on",
run_id_field="rqa_s01e02_run_id",
coda_filename="WorldVision_s01e02.json",
icr_filename="s01e02.csv",
coding_configurations=[
CodingConfiguration(coding_mode=CodingModes.MULTIPLE,
code_scheme=CodeSchemes.S01E02,
coded_field="rqa_s01e02_coded",
analysis_file_key="rqa_s01e02_",
fold_strategy=lambda x, y: FoldStrategies.
list_of_labels(CodeSchemes.S01E02, x, y))
],
ws_code=CodeSchemes.WS_CORRECT_DATASET.get_code_with_match_value(
"s01e02"),
raw_field_fold_strategy=FoldStrategies.concatenate),
CodingPlan(
raw_field="rqa_s01e03_raw",
time_field="sent_on",
run_id_field="rqa_s01e03_run_id",
coda_filename="WorldVision_s01e03.json",
icr_filename="s01e03.csv",
coding_configurations=[
CodingConfiguration(coding_mode=CodingModes.MULTIPLE,
code_scheme=CodeSchemes.S01E03,
coded_field="rqa_s01e03_coded",
analysis_file_key="rqa_s01e03_",
fold_strategy=lambda x, y: FoldStrategies.
list_of_labels(CodeSchemes.S01E03, x, y))
],
ws_code=CodeSchemes.WS_CORRECT_DATASET.get_code_with_match_value(
"s01e03"),
raw_field_fold_strategy=FoldStrategies.concatenate),
CodingPlan(
raw_field="s01_close_out_raw",
time_field="sent_on",
run_id_field="s01_close_out_run_id",
coda_filename="WorldVision_s01_close_out.json",
icr_filename="s01_close_out.csv",
coding_configurations=[
CodingConfiguration(
coding_mode=CodingModes.MULTIPLE,
code_scheme=CodeSchemes.S01_CLOSE_OUT,
coded_field="s01_close_out_coded",
analysis_file_key="s01_close_out_",
fold_strategy=lambda x, y: FoldStrategies.list_of_labels(
CodeSchemes.S01_CLOSE_OUT, x, y))
],
ws_code=CodeSchemes.WS_CORRECT_DATASET.get_code_with_match_value(
"s01 close-out"),
raw_field_fold_strategy=FoldStrategies.concatenate)
]
class PipelineConfiguration(object):
RQA_CODING_PLANS = [
CodingPlan(raw_field="rqa_s06e01_raw",
time_field="sent_on",
run_id_field="rqa_s06e01_run_id",
coda_filename="s06e01.json",
icr_filename="s06e01.csv",
coding_configurations=[
CodingConfiguration(
coding_mode=CodingModes.MULTIPLE,
code_scheme=CodeSchemes.S06E01_REASONS,
coded_field="rqa_s06e01_coded",
analysis_file_key="rqa_s06e01_",
fold_strategy=lambda x, y: FoldStrategies.list_of_labels(CodeSchemes.S06E01_REASONS, x, y)
)
],
ws_code=CodeSchemes.WS_CORRECT_DATASET.get_code_with_match_value("s06e01"),
raw_field_fold_strategy=FoldStrategies.concatenate),
CodingPlan(raw_field="rqa_s06e02_raw",
time_field="sent_on",
run_id_field="rqa_s06e02_run_id",
coda_filename="s06e02.json",
icr_filename="s06e02.csv",
coding_configurations=[
CodingConfiguration(
coding_mode=CodingModes.MULTIPLE,
code_scheme=CodeSchemes.S06E02_REASONS,
coded_field="rqa_s06e02_coded",
analysis_file_key="rqa_s06e02_",
fold_strategy=lambda x, y: FoldStrategies.list_of_labels(CodeSchemes.S06E02_REASONS, x, y)
)
],
ws_code=CodeSchemes.WS_CORRECT_DATASET.get_code_with_match_value("s06e02"),
raw_field_fold_strategy=FoldStrategies.concatenate),
]
@staticmethod
def clean_age_with_range_filter(text):
"""
Cleans age from the given `text`, setting to NC if the cleaned age is not in the range 10 <= age < 100.
"""
age = swahili.DemographicCleaner.clean_age(text)
if type(age) == int and 10 <= age < 100:
return str(age)
# TODO: Once the cleaners are updated to not return Codes.NOT_CODED, this should be updated to still return
# NC in the case where age is an int but is out of range
else:
return Codes.NOT_CODED
@staticmethod
def clean_district_if_no_mogadishu_sub_district(text):
mogadishu_sub_district = somali.DemographicCleaner.clean_mogadishu_sub_district(text)
if mogadishu_sub_district == Codes.NOT_CODED:
return somali.DemographicCleaner.clean_somalia_district(text)
else:
return Codes.NOT_CODED
SURVEY_CODING_PLANS = [
CodingPlan(raw_field="operator_raw",
coding_configurations=[
CodingConfiguration(
coding_mode=CodingModes.SINGLE,
code_scheme=CodeSchemes.SOMALIA_OPERATOR,
coded_field="operator_coded",
analysis_file_key="operator",
fold_strategy=FoldStrategies.assert_label_ids_equal
)
],
raw_field_fold_strategy=FoldStrategies.assert_equal),
CodingPlan(raw_field="location_raw",
time_field="location_time",
coda_filename="location.json",
coding_configurations=[
CodingConfiguration(
coding_mode=CodingModes.SINGLE,
code_scheme=CodeSchemes.MOGADISHU_SUB_DISTRICT,
cleaner=somali.DemographicCleaner.clean_mogadishu_sub_district,
coded_field="mogadishu_sub_district_coded",
# This code exists for compatibility with the previous CSAP demog datasets.
# Not including in the analysis file because this project is not in Mogadishu.
fold_strategy=FoldStrategies.assert_label_ids_equal
),
CodingConfiguration(
coding_mode=CodingModes.SINGLE,
code_scheme=CodeSchemes.SOMALIA_DISTRICT,
cleaner=lambda text: PipelineConfiguration.clean_district_if_no_mogadishu_sub_district(text),
coded_field="district_coded",
analysis_file_key="district",
fold_strategy=FoldStrategies.assert_label_ids_equal
),
CodingConfiguration(
coding_mode=CodingModes.SINGLE,
code_scheme=CodeSchemes.SOMALIA_REGION,
coded_field="region_coded",
analysis_file_key="region",
fold_strategy=FoldStrategies.assert_label_ids_equal
),
CodingConfiguration(
coding_mode=CodingModes.SINGLE,
code_scheme=CodeSchemes.SOMALIA_STATE,
coded_field="state_coded",
analysis_file_key="state",
fold_strategy=FoldStrategies.assert_label_ids_equal
),
CodingConfiguration(
coding_mode=CodingModes.SINGLE,
code_scheme=CodeSchemes.SOMALIA_ZONE,
coded_field="zone_coded",
analysis_file_key="zone",
fold_strategy=FoldStrategies.assert_label_ids_equal
)
],
code_imputation_function=code_imputation_functions.impute_somalia_location_codes,
ws_code=CodeSchemes.WS_CORRECT_DATASET.get_code_with_match_value("location"),
raw_field_fold_strategy=FoldStrategies.assert_equal),
CodingPlan(raw_field="gender_raw",
time_field="gender_time",
coda_filename="gender.json",
coding_configurations=[
CodingConfiguration(
coding_mode=CodingModes.SINGLE,
code_scheme=CodeSchemes.GENDER,
cleaner=somali.DemographicCleaner.clean_gender,
coded_field="gender_coded",
analysis_file_key="gender",
fold_strategy=FoldStrategies.assert_label_ids_equal
)
],
ws_code=CodeSchemes.WS_CORRECT_DATASET.get_code_with_match_value("gender"),
raw_field_fold_strategy=FoldStrategies.assert_equal),
CodingPlan(raw_field="age_raw",
time_field="age_time",
coda_filename="age.json",
coding_configurations=[
CodingConfiguration(
coding_mode=CodingModes.SINGLE,
code_scheme=CodeSchemes.AGE,
cleaner=lambda text: PipelineConfiguration.clean_age_with_range_filter(text),
coded_field="age_coded",
fold_strategy=FoldStrategies.assert_label_ids_equal
),
CodingConfiguration(
coding_mode=CodingModes.SINGLE,
code_scheme=CodeSchemes.AGE_CATEGORY,
coded_field="age_category_coded",
analysis_file_key="age_category",
fold_strategy=FoldStrategies.assert_label_ids_equal
)
],
code_imputation_function=code_imputation_functions.impute_age_category,
ws_code=CodeSchemes.WS_CORRECT_DATASET.get_code_with_match_value("age"),
raw_field_fold_strategy=FoldStrategies.assert_equal),
CodingPlan(raw_field="recently_displaced_raw",
time_field="recently_displaced_time",
coda_filename="recently_displaced.json",
coding_configurations=[
CodingConfiguration(
coding_mode=CodingModes.SINGLE,
code_scheme=CodeSchemes.RECENTLY_DISPLACED,
cleaner=somali.DemographicCleaner.clean_yes_no,
coded_field="recently_displaced_coded",
analysis_file_key="recently_displaced",
fold_strategy=FoldStrategies.assert_label_ids_equal
)
],
ws_code=CodeSchemes.WS_CORRECT_DATASET.get_code_with_match_value("recently displaced"),
raw_field_fold_strategy=FoldStrategies.assert_equal),
CodingPlan(raw_field="in_idp_camp_raw",
time_field="in_idp_camp_time",
coda_filename="in_idp_camp.json",
coding_configurations=[
CodingConfiguration(
coding_mode=CodingModes.SINGLE,
code_scheme=CodeSchemes.IN_IDP_CAMP,
cleaner=somali.DemographicCleaner.clean_yes_no,
coded_field="in_idp_camp_coded",
analysis_file_key="in_idp_camp",
fold_strategy=FoldStrategies.assert_label_ids_equal
)
],
ws_code=CodeSchemes.WS_CORRECT_DATASET.get_code_with_match_value("in idp camp"),
raw_field_fold_strategy=FoldStrategies.assert_equal),
CodingPlan(raw_field="have_voice_raw",
time_field="have_voice_time",
coda_filename="have_voice.json",
coding_configurations=[
CodingConfiguration(
coding_mode=CodingModes.SINGLE,
code_scheme=CodeSchemes.HAVE_VOICE,
cleaner=somali.DemographicCleaner.clean_yes_no,
coded_field="have_voice_coded",
analysis_file_key="have_voice",
fold_strategy=FoldStrategies.assert_label_ids_equal
)
],
ws_code=CodeSchemes.WS_CORRECT_DATASET.get_code_with_match_value("have voice"),
raw_field_fold_strategy=FoldStrategies.assert_equal),
CodingPlan(raw_field="suggestions_raw",
time_field="suggestions_time",
coda_filename="suggestions.json",
coding_configurations=[
CodingConfiguration(
coding_mode=CodingModes.MULTIPLE,
code_scheme=CodeSchemes.SUGGESTIONS,
coded_field="suggestions_coded",
analysis_file_key="suggestions_",
fold_strategy=lambda x, y: FoldStrategies.list_of_labels(CodeSchemes.SUGGESTIONS, x, y)
)
],
ws_code=CodeSchemes.WS_CORRECT_DATASET.get_code_with_match_value("suggestions"),
raw_field_fold_strategy=FoldStrategies.assert_equal)
]
def __init__(self, pipeline_name, raw_data_sources, phone_number_uuid_table, timestamp_remappings,
rapid_pro_key_remappings, project_start_date, project_end_date, filter_test_messages, move_ws_messages,
memory_profile_upload_url_prefix, data_archive_upload_url_prefix, drive_upload=None):
"""
:param pipeline_name: The name of this pipeline.
:type pipeline_name: str
:param raw_data_sources: List of sources to pull the various raw run files from.
:type raw_data_sources: list of RawDataSource
:param phone_number_uuid_table: Configuration for the Firestore phone number <-> uuid table.
:type phone_number_uuid_table: PhoneNumberUuidTable
:param rapid_pro_key_remappings: List of rapid_pro_key -> pipeline_key remappings.
:type rapid_pro_key_remappings: list of RapidProKeyRemapping
:param project_start_date: When data collection started - all activation messages received before this date
time will be dropped.
:type project_start_date: datetime.datetime
:param project_end_date: When data collection stopped - all activation messages received on or after this date
time will be dropped.
:type project_end_date: datetime.datetime
:param filter_test_messages: Whether to filter out messages sent from the rapid_pro_test_contact_uuids
:type filter_test_messages: bool
:param move_ws_messages: Whether to move messages labelled as Wrong Scheme to the correct dataset.
:type move_ws_messages: bool
:param memory_profile_upload_url_prefix: The prefix of the GS URL to upload the memory profile log to.
This prefix will be appended by the id of the pipeline run (provided
as a command line argument), and the ".profile" file extension.
:type memory_profile_upload_url_prefix: str
:param drive_upload: Configuration for uploading to Google Drive, or None.
If None, does not upload to Google Drive.
:type drive_upload: DriveUploadPaths | None
"""
self.pipeline_name = pipeline_name
self.raw_data_sources = raw_data_sources
self.phone_number_uuid_table = phone_number_uuid_table
self.timestamp_remappings = timestamp_remappings
self.rapid_pro_key_remappings = rapid_pro_key_remappings
self.project_start_date = project_start_date
self.project_end_date = project_end_date
self.filter_test_messages = filter_test_messages
self.move_ws_messages = move_ws_messages
self.drive_upload = drive_upload
self.memory_profile_upload_url_prefix = memory_profile_upload_url_prefix
self.data_archive_upload_url_prefix = data_archive_upload_url_prefix
self.validate()
@classmethod
def from_configuration_dict(cls, configuration_dict):
pipeline_name = configuration_dict["PipelineName"]
raw_data_sources = []
for raw_data_source in configuration_dict["RawDataSources"]:
if raw_data_source["SourceType"] == "RapidPro":
raw_data_sources.append(RapidProSource.from_configuration_dict(raw_data_source))
elif raw_data_source["SourceType"] == "GCloudBucket":
raw_data_sources.append(GCloudBucketSource.from_configuration_dict(raw_data_source))
elif raw_data_source["SourceType"] == "RecoveryCSV":
raw_data_sources.append(RecoveryCSVSource.from_configuration_dict(raw_data_source))
else:
assert False, f"Unknown SourceType '{raw_data_source['SourceType']}'. " \
f"Must be 'RapidPro', 'GCloudBucket', or 'RecoveryCSV'."
phone_number_uuid_table = PhoneNumberUuidTable.from_configuration_dict(
configuration_dict["PhoneNumberUuidTable"])
timestamp_remappings = []
for remapping_dict in configuration_dict.get("TimestampRemappings", []):
timestamp_remappings.append(TimestampRemapping.from_configuration_dict(remapping_dict))
rapid_pro_key_remappings = []
for remapping_dict in configuration_dict["RapidProKeyRemappings"]:
rapid_pro_key_remappings.append(RapidProKeyRemapping.from_configuration_dict(remapping_dict))
project_start_date = isoparse(configuration_dict["ProjectStartDate"])
project_end_date = isoparse(configuration_dict["ProjectEndDate"])
filter_test_messages = configuration_dict["FilterTestMessages"]
move_ws_messages = configuration_dict["MoveWSMessages"]
drive_upload_paths = None
if "DriveUpload" in configuration_dict:
drive_upload_paths = DriveUpload.from_configuration_dict(configuration_dict["DriveUpload"])
memory_profile_upload_url_prefix = configuration_dict["MemoryProfileUploadURLPrefix"]
data_archive_upload_url_prefix = configuration_dict["DataArchiveUploadURLPrefix"]
return cls(pipeline_name, raw_data_sources, phone_number_uuid_table, timestamp_remappings,
rapid_pro_key_remappings, project_start_date, project_end_date, filter_test_messages,
move_ws_messages, memory_profile_upload_url_prefix, data_archive_upload_url_prefix,
drive_upload_paths)
@classmethod
def from_configuration_file(cls, f):
return cls.from_configuration_dict(json.load(f))
def validate(self):
validators.validate_string(self.pipeline_name, "pipeline_name")
validators.validate_list(self.raw_data_sources, "raw_data_sources")
for i, raw_data_source in enumerate(self.raw_data_sources):
assert isinstance(raw_data_source, RawDataSource), f"raw_data_sources[{i}] is not of type of RawDataSource"
raw_data_source.validate()
assert isinstance(self.phone_number_uuid_table, PhoneNumberUuidTable)
self.phone_number_uuid_table.validate()
validators.validate_list(self.rapid_pro_key_remappings, "rapid_pro_key_remappings")
for i, remapping in enumerate(self.rapid_pro_key_remappings):
assert isinstance(remapping, RapidProKeyRemapping), \
f"rapid_pro_key_mappings[{i}] is not of type RapidProKeyRemapping"
remapping.validate()
validators.validate_datetime(self.project_start_date, "project_start_date")
validators.validate_datetime(self.project_end_date, "project_end_date")
validators.validate_bool(self.filter_test_messages, "filter_test_messages")
validators.validate_bool(self.move_ws_messages, "move_ws_messages")
if self.drive_upload is not None:
assert isinstance(self.drive_upload, DriveUpload), \
"drive_upload is not of type DriveUpload"
self.drive_upload.validate()
validators.validate_string(self.memory_profile_upload_url_prefix, "memory_profile_upload_url_prefix")
def get_rqa_coding_plans(pipeline_name):
return [
CodingPlan(
raw_field="rqa_s01e01_raw",
time_field="sent_on",
run_id_field="rqa_s01e01_run_id",
coda_filename="OXFAM_WASH_s01e01.json",
icr_filename="s01e01.csv",
coding_configurations=[
CodingConfiguration(coding_mode=CodingModes.MULTIPLE,
code_scheme=CodeSchemes.S01E01,
coded_field="rqa_s01e01_coded",
analysis_file_key="rqa_s01e01",
fold_strategy=lambda x, y: FoldStrategies.
list_of_labels(CodeSchemes.S01E01, x, y))
],
ws_code=CodeSchemes.WS_CORRECT_DATASET.get_code_with_match_value(
"OXFAM-WASH s01e01"),
raw_field_fold_strategy=FoldStrategies.concatenate),
CodingPlan(
raw_field="rqa_s01e02_raw",
time_field="sent_on",
run_id_field="rqa_s01e02_run_id",
coda_filename="OXFAM_WASH_s01e02.json",
icr_filename="s01e02.csv",
coding_configurations=[
CodingConfiguration(coding_mode=CodingModes.MULTIPLE,
code_scheme=CodeSchemes.S01E02,
coded_field="rqa_s01e02_coded",
analysis_file_key="rqa_s01e02",
fold_strategy=lambda x, y: FoldStrategies.
list_of_labels(CodeSchemes.S01E02, x, y))
],
ws_code=CodeSchemes.WS_CORRECT_DATASET.get_code_with_match_value(
"OXFAM-WASH s01e02"),
raw_field_fold_strategy=FoldStrategies.concatenate),
CodingPlan(
raw_field="rqa_s01e03_raw",
time_field="sent_on",
run_id_field="rqa_s01e03_run_id",
coda_filename="OXFAM_WASH_s01e03.json",
icr_filename="s01e03.csv",
coding_configurations=[
CodingConfiguration(coding_mode=CodingModes.MULTIPLE,
code_scheme=CodeSchemes.S01E03,
coded_field="rqa_s01e03_coded",
analysis_file_key="rqa_s01e03",
fold_strategy=lambda x, y: FoldStrategies.
list_of_labels(CodeSchemes.S01E03, x, y))
],
ws_code=CodeSchemes.WS_CORRECT_DATASET.get_code_with_match_value(
"OXFAM-WASH s01e03"),
raw_field_fold_strategy=FoldStrategies.concatenate),
CodingPlan(
raw_field="oxfam_beneficiary_consent_raw",
time_field="sent_on",
run_id_field="oxfam_beneficiary_consent_run_id",
coda_filename="OXFAM_WASH_Beneficiary_Consent.json",
icr_filename="oxfam_beneficiary_consent.csv",
coding_configurations=[
CodingConfiguration(
coding_mode=CodingModes.MULTIPLE,
code_scheme=CodeSchemes.BENEFICIARY_CONSENT,
coded_field="oxfam_beneficiary_consent_coded",
analysis_file_key="oxfam_beneficiary_consent",
fold_strategy=lambda x, y: FoldStrategies.list_of_labels(
CodeSchemes.BENEFICIARY_CONSENT, x, y))
],
ws_code=CodeSchemes.WS_CORRECT_DATASET.get_code_with_match_value(
"OXFAM WASH Beneficiary Consent"),
raw_field_fold_strategy=FoldStrategies.concatenate),
CodingPlan(
raw_field="rqa_s01e03_noise_handler_raw",
time_field="sent_on",
run_id_field="rqa_s01e03_noise_handler_run_id",
coda_filename="OXFAM_WASH_s01e03_Noise_Handler.json",
icr_filename="oxfam_wash_s01e03_noise_handler.csv",
coding_configurations=[
CodingConfiguration(
coding_mode=CodingModes.MULTIPLE,
code_scheme=CodeSchemes.S01E03_NOISE_HANDLER,
coded_field="rqa_s01e03_noise_handler_coded",
analysis_file_key="rqa_s01e03_s01e03_noise_handler",
fold_strategy=lambda x, y: FoldStrategies.list_of_labels(
CodeSchemes.S01E03_NOISE_HANDLER, x, y))
],
ws_code=CodeSchemes.WS_CORRECT_DATASET.get_code_with_match_value(
"OXFAM WASH s01e03 Noise Handler"),
raw_field_fold_strategy=FoldStrategies.concatenate),
CodingPlan(
raw_field="s01_close_out_raw",
time_field="sent_on",
run_id_field="s01_close_out_run_id",
coda_filename="OXFAM_WASH_s01_Close_Out.json",
icr_filename="oxfam_wash_s01_close_out.csv",
coding_configurations=[
CodingConfiguration(
coding_mode=CodingModes.MULTIPLE,
code_scheme=CodeSchemes.S01_CLOSE_OUT,
coded_field="s01_close_out_coded",
analysis_file_key="s01_close_out",
fold_strategy=lambda x, y: FoldStrategies.list_of_labels(
CodeSchemes.S01_CLOSE_OUT, x, y))
],
ws_code=CodeSchemes.WS_CORRECT_DATASET.get_code_with_match_value(
"OXFAM WASH S01 Close Out"),
raw_field_fold_strategy=FoldStrategies.concatenate),
]