"json_output", metavar="json-output", help= "Path to write results of merging to, as a serialised TracedData JSON file", nargs=1) args = parser.parse_args() user = args.user[0] input_path_messages = args.input_messages[0] group = args.group[0] input_path_adverts = args.input_adverts[0] json_output_path = args.json_output[0] # Load data from JSON file with open(input_path_messages, "r") as f: messages_data = TracedDataJsonIO.import_json_to_traced_data_iterable(f) # Load data from JSON file with open(input_path_adverts, "r") as f: adverts_data = TracedDataJsonIO.import_json_to_traced_data_iterable(f) # Map "QUESTION_R" => "Message" for td in adverts_data: assert "QUESTION_R" in td.keys() td.append_data( { "Message": td["QUESTION_R"], "Date": td["start_date"], "Group": group }, Metadata(user, Metadata.get_call_location(), time.time()))
fixup_table = json.load(open(fixup_table_path, 'r')) # print ("Loading code schemes") code_scheme_paths_list = [ os.path.join(code_schemes_in_folder, f) for f in os.listdir(code_schemes_in_folder) if os.path.isfile(os.path.join(code_schemes_in_folder, f)) ] code_schemes = {} for code_scheme_path in code_scheme_paths_list: scheme = json.load(open(code_scheme_path, 'r')) code_schemes[scheme["SchemeID"]] = scheme # print ("Loading demog data") demog_td = TracedDataJsonIO.import_json_to_traced_data_iterable( open(os.path.join(demogs_in_folder, "Demog_survey_with_id.json"), 'r')) # print ("Remapping demogs") for msg in demog_td: for demog_map in demog_maps: id = msg[demog_map["MessageId"]] dataset_in_principle = demog_map["Coda-Dataset"] remap(id, dataset_in_principle, fixup_table, code_schemes) # print ("Remapping messages") for message_map in message_maps: # print ("Loading message_map: {}".format(message_map["FileName"])) messages_td = TracedDataJsonIO.import_json_to_traced_data_iterable( open(os.path.join(messages_in_folder, message_map["FileName"]), 'r'))
parser.add_argument("demogs_traced_json_output_path", help="Path to updted survey TraceData JSON") args = parser.parse_args() user = args.user messages_input_path = args.messages_input_path surveys_input_path = args.surveys_input_path demogs_input_path = args.demogs_input_path messages_traced_json_output_path = args.messages_traced_json_output_path surveys_traced_json_output_path = args.surveys_traced_json_output_path demogs_traced_json_output_path = args.demogs_traced_json_output_path # Load messages for filename in os.listdir(messages_input_path): with open(os.path.join(messages_input_path, filename)) as f: messages = TracedDataJsonIO.import_json_to_traced_data_iterable(f) # Create a set of all the keys appearing in the data keys = {key for message in messages for key in message.keys()} keys = list(keys) keys.remove("avf_phone_id") # Add group name to each key group_name = filename.split("_with_id.json")[0] create_unique_keys(messages, keys, group_name) # Output updated td-s message_output_path = os.path.join( messages_traced_json_output_path, "{}_updated_keys.json".format(group_name)) with open(message_output_path, "w") as f:
if pipeline_configuration.drive_upload is not None: log.info(f"Downloading Google Drive service account credentials...") credentials_info = json.loads( google_cloud_utils.download_blob_to_string( google_cloud_credentials_file_path, pipeline_configuration. drive_upload.drive_credentials_file_url)) drive_client_wrapper.init_client_from_info(credentials_info) # Load messages messages_datasets = [] for i, activation_flow_name in enumerate( pipeline_configuration.activation_flow_names): raw_activation_path = f"{raw_data_dir}/{activation_flow_name}.json" log.info(f"Loading {raw_activation_path}...") with open(raw_activation_path, "r") as f: messages = TracedDataJsonIO.import_json_to_traced_data_iterable(f) log.info(f"Loaded {len(messages)} messages") messages_datasets.append(messages) log.info("Loading surveys datasets:") surveys_datasets = [] for i, survey_flow_name in enumerate( pipeline_configuration.survey_flow_names): raw_survey_path = f"{raw_data_dir}/{survey_flow_name}.json" log.info(f"Loading {raw_survey_path}...") with open(raw_survey_path, "r") as f: contacts = TracedDataJsonIO.import_json_to_traced_data_iterable(f) log.info(f"Loaded {len(contacts)} contacts") surveys_datasets.append(contacts) # Add survey data to the messages
"Path to input file to concatenate, containing a list of TracedData objects as JSON", nargs=1) parser.add_argument("json_output", metavar="json-output", help="Path to write results of cleaning to", nargs=1) args = parser.parse_args() user = args.user[0] json_input_path_1 = args.json_input_1[0] json_input_path_2 = args.json_input_2[0] json_output_path = args.json_output[0] # Load data from JSON file with open(json_input_path_1, "r") as f: input_data_1 = TracedDataJsonIO.import_json_to_traced_data_iterable(f) with open(json_input_path_2, "r") as f: input_data_2 = TracedDataJsonIO.import_json_to_traced_data_iterable(f) # Concatenate files output_data = list(input_data_1) output_data.extend(input_data_2) # Write json output if os.path.dirname(json_output_path) is not "" and not os.path.exists( os.path.dirname(json_output_path)): os.makedirs(os.path.dirname(json_output_path)) with open(json_output_path, "w") as f: TracedDataJsonIO.export_traced_data_iterable_to_json(output_data, f, pretty_print=True)
metavar="json-input-path", help= "Path to the input JSON file, containing a list of serialized TracedData objects" ) parser.add_argument( "json_output_path", metavar="json-output-path", help="Path to a JSON file to write processed TracedData messages to") args = parser.parse_args() user = args.user json_input_path = args.json_input_path json_output_path = args.json_output_path # Load data from JSON file with open(json_input_path, "r") as f: data = TracedDataJsonIO.import_json_to_traced_data_iterable(f) # FIXME: Clean/filter messages # Write json output if os.path.dirname(json_output_path) is not "" and not os.path.exists( os.path.dirname(json_output_path)): os.makedirs(os.path.dirname(json_output_path)) with open(json_output_path, "w") as f: TracedDataJsonIO.export_traced_data_iterable_to_json(data, f, pretty_print=True) # FIXME: Output to other formats for analysis, using the TracedData exporters in core_data_modules.traced_data.io
message_paths = [s02e01_input_path] # Load the pipeline configuration file print("Loading Pipeline Configuration File...") with open(pipeline_configuration_file_path) as f: pipeline_configuration = PipelineConfiguration.from_configuration_file(f) # Load phone number <-> UUID table print("Loading Phone Number <-> UUID Table...") with open(phone_number_uuid_table_path, "r") as f: phone_number_uuid_table = PhoneNumberUuidTable.load(f) # Load demographics print("Loading Demographics 1/1...") with open(demog_input_path, "r") as f: demographics = TracedDataJsonIO.import_json_to_traced_data_iterable(f) print(f"Loaded {len(demographics)} contacts") # Load messages messages_datasets = [] for i, path in enumerate(message_paths): print("Loading Episode {}/{}...".format(i + 1, len(message_paths))) with open(path, "r") as f: messages_datasets.append(TracedDataJsonIO.import_json_to_traced_data_iterable(f)) # Add survey data to the messages print("Combining Datasets...") import argparse import os import random
json_path = args.json_path rapid_pro = TembaClient(server, token) # Load the existing phone number <-> UUID table. if not os.path.exists(phone_uuid_path): raise FileNotFoundError( "No such phone uuid table file '{}'. " "To create a new, empty UUID table, " "run $ echo \"{{}}\" > <target-json-file>".format(phone_uuid_path)) with open(phone_uuid_path, "r") as f: phone_uuids = PhoneNumberUuidTable.load(f) # Load the existing messages for week 4 with open(json_path, "r") as f: week_4_messages = TracedDataJsonIO.import_json_to_traced_data_iterable( f) # Fetch messages in the S06e04 resend group mislocated_messages = rapid_pro.get_messages(label="S06e04_resend").all( retry_on_rate_exceed=True) print("Number of mislocated messages found: {}".format( len(mislocated_messages))) # Sort by ascending order of modification date. mislocated_messages = list(mislocated_messages) mislocated_messages.reverse() # Convert the mislocated messages to de-identified TracedData traced_messages = [] for message in mislocated_messages: traced_messages.append(
json_output_path = args.json_output_path fgd_csv_output_path = args.fgd_csv_output_path cc_csv_output_path = args.cc_csv_output_path prev_exports_path = args.prev_exports_path MINIMUM_AGE = 18 TOTAL_CC_CONTACTS = 160 TOTAL_FGD_CONTACTS = 100 # Load phone uuid table with open(phone_uuid_table_path, "r") as f: phone_uuids = PhoneNumberUuidTable.load(f) # Load FGD/CC survey responses with open(fgd_cc_input_path, "r") as f: fgd_cc_data = TracedDataJsonIO.import_json_to_traced_data_iterable(f) # Load the previous export prev_exports = [] if prev_exports_path is not None: with open(prev_exports_path, "r") as f: prev_exports = list( TracedDataCSVIO.import_csv_to_traced_data_iterable(user, f)) # Load coded demog surveys with open(demog_surveys_input_path, "r") as f: surveys = TracedDataJsonIO.import_json_to_traced_data_iterable(f) # Filter out people who haven't answered the fgd_cc consent question fgd_cc_consent_key = "Response_1 (Category) - wt_fgd_cc" fgd_cc_data = [td for td in fgd_cc_data if fgd_cc_consent_key in td]
metavar="json-output-path", help="Path to a JSON file to write processed messages to") args = parser.parse_args() user = args.user messages_input_path = args.messages_input_path survey_input_path = args.survey_input_path demog_input_path = args.demog_input_path json_output_path = args.json_output_path # Load messages messages_datasets = [] for filename in os.listdir(messages_input_path): with open(os.path.join(messages_input_path, filename)) as f: messages_datasets.append( TracedDataJsonIO.import_json_to_traced_data_iterable(f)) # Load followup surveys survey_datasets = [] for filename in os.listdir(survey_input_path): with open(os.path.join(survey_input_path, filename)) as f: survey_datasets.append( TracedDataJsonIO.import_json_to_traced_data_iterable(f)) # Load demogs print("Loading Demographics...") with open(demog_input_path, "r") as f: demographics = TracedDataJsonIO.import_json_to_traced_data_iterable(f) # Add survey data to the messages print("Combining Datasets...")
"Radio_Station (Text) - wt_demog_2": None, "Age (Text) - wt_demog_2": somali.DemographicCleaner.clean_age, "Education_Level (Text) - wt_demog_2": None, "Idp (Text) - wt_demog_2": somali.DemographicCleaner.clean_yes_no, "Origin_District (Text) - wt_demog_2": somali.DemographicCleaner.clean_somalia_district, "Household_Sickness (Text) - wt_practice": somali.DemographicCleaner.clean_yes_no, "Cholera_Vaccination (Text) - wt_practice": somali.DemographicCleaner.clean_yes_no, "Trustworthy_Advisors (Text) - wt_practice": None } # Load data from JSON file with open(demog_1_input_path, "r") as f: demog_1_data = TracedDataJsonIO.import_json_to_traced_data_iterable(f) with open(demog_2_input_path, "r") as f: demog_2_data = TracedDataJsonIO.import_json_to_traced_data_iterable(f) with open(practice_input_path, "r") as f: practice_data = TracedDataJsonIO.import_json_to_traced_data_iterable(f) # Join the survey data on "avf_phone_id" demog_data = TracedData.join_iterables(user, "avf_phone_id", demog_1_data, demog_2_data, "wt_demog_2") all_survey_data = TracedData.join_iterables(user, "avf_phone_id", demog_data, practice_data, "wt_practice") # Clean the survey responses for td in all_survey_data: for key, cleaner in cleaning_plan.items():