def main(): pd.set_option('display.max_columns', None) import argparse my_parser = argparse.ArgumentParser(prog='create_flat_file', description='Create dummy data flat file in Google Spreadsheets', epilog="python create_flat_file Lt6P15ps7f6 --with_teis_from=GZ5Ty90HtW [email protected]" "\npython create_flat_file Lt6P15ps7f6 --repeat_stage Hj38Uhfo012 5 --repeat_stage 77Ujkfoi9kG 3 [email protected] [email protected]", formatter_class=argparse.RawDescriptionHelpFormatter) my_parser.add_argument('Program_UID', metavar='program_uid', type=str, help='the uid of the program to use') my_parser.add_argument('-wtf', '--with_teis_from', action="store", dest="OrgUnit", type=str, help='Pulls TEIs from specified org unit and adds them to flat file. ' 'Eg: --with_teis_from_ou=Q7RbNZcHrQ9') my_parser.add_argument('-rs', '--repeat_stage', action="append", metavar=('stage_uid', 'number_repeats'), nargs=2, help='provide a stage uid which is REPEATABLE and specify how many times you are planning to enter it. ' 'Eg: --repeat_stage QXtjg5dh34A 3') my_parser.add_argument('-sw', '--share_with', action="append", metavar='email', nargs=1, help='email address to share the generated spreadsheet with as OWNER. ' 'Eg: [email protected]') args = my_parser.parse_args() program_uid = args.Program_UID if not is_valid_uid(program_uid): print('The program uid specified is not valid') sys.exit() if args.OrgUnit is not None and not is_valid_uid(args.OrgUnit): print('The orgunit uid specified is not valid') sys.exit() if args.repeat_stage is not None and len(args.repeat_stage) > 0: for param in args.repeat_stage: if not is_valid_uid(param[0]): print('The program stage uid specified ' + param[0] + ' is not valid') sys.exit() try: int(param[1]) except ValueError: print('The repetition value ' + param[1] + ' is not an integer') sys.exit() if args.share_with is not None and len(args.share_with) > 0: for param in args.share_with: if not (re.search('^[a-z0-9]+[\._]?[a-z0-9]+[@]\w+[.]\w{2,3}$', param[0])): print("The email address " + param[0] + " is not valid") # Print DHIS2 Info logger.warning("Server source running DHIS2 version {} revision {}" .format(api_source.version, api_source.revision)) ############## # df = pd.read_csv('program-Case_Based_Surveillance.csv', sep=None, engine='python') # # # # # stages_counter = { 'K5ac7u3V5bB': 1, 'ang4CLldbIu': 5, 'UvYb6qJpQu0': 1 } # # # # #json_tei = api_source.get('trackedEntityInstances/dRdztYSReOZ', params={'fields':'*'}).json() # # # params = { # 'ou': 'RI95HQRHbKc', # GD7TowwI46c # 'ouMode': 'DESCENDANTS', # 'program': program_uid, # 'skipPaging': 'true', # 'lastUpdatedDuration': '4d', # 'fields': '*', # 'includeAllAttributes': 'true' # } # # list_teis = api_source.get('trackedEntityInstances', params=params).json()['trackedEntityInstances'] # # logger.info("Found " + str(len(list_teis)) + " TEIs") # # user = '******' # stages_counter = dict() # for tei in list_teis: # counter = dict() # if "enrollments" in tei and len(tei["enrollments"][0]) > 0: # and tei["enrollments"][0]["storedBy"] == user: # if len(tei['enrollments']) == 1: # if tei['enrollments'][0]['program'] == program_uid: # if 'events' in tei['enrollments'][0]: # events = tei['enrollments'][0]['events'] # for event in events: # if event["programStage"] in counter: # counter[event["programStage"]] +=1 # else: # counter[event["programStage"]] = 1 # else: # logger.error("TEI enrolled in program " + tei['enrollments'][0]['program'] + " not supported") # else: # logger.error('error, multi-enrollment not supported') # for key in counter: # if key not in stages_counter or stages_counter[key] < counter[key]: # stages_counter[key] = counter[key] # logger.info('Found ' + str(stages_counter[key]) + ' instances of ' + key) # # df = add_repeatable_stages(df, stages_counter) # for tei in list_teis: # # if tei['trackedEntityInstance'] != 'j17HROzXGEn': # # continue # if len(tei["enrollments"][0]) > 0: # and tei["enrollments"][0]["storedBy"] == user: # result = add_json_tei_to_metadata_df(tei, df) # export_csv = df.to_csv(r'./program-Case_Based_Surveillance-Dummy_data.csv', index=None, header=True) ########### df = pd.DataFrame({}, columns=["Stage", "Section", "TEA / DE / eventDate", "UID", "valueType", "optionSet", "mandatory"]) try: program = api_source.get('programs/' + program_uid, params={"paging": "false", "fields": "id,name,enrollmentDateLabel,programTrackedEntityAttributes,programStages,programRuleVariables,organisationUnits,trackedEntityType,version"}).json() except RequestException as e: if e.code == 404: logger.error('Program ' + program_uid + ' specified does not exist') sys.exit() if isinstance(program, dict): # If the program has many org units assigned, this can take a long time to run!!! # orgunits_uid = json_extract_nested_ids(program, 'organisationUnits') # if args.OrgUnit is not None and args.OrgUnit not in orgunits_uid: # logger.error('The organisation unit ' + args.OrgUnit + ' is not assigned to program ' + program_uid) # print('Number of OrgUnits:' + str(len(orgunits_uid))) programStages_uid = json_extract_nested_ids(program, 'programStages') if args.repeat_stage is not None: for param in args.repeat_stage: found = False for uid in programStages_uid: if param[0] == uid: found = True break if not found: logger.error(uid + ' specified is not a valid stage for program ' + program_uid) sys.exit() teas_uid = json_extract_nested_ids(program, 'trackedEntityAttribute') programRuleVariables_uid = json_extract_nested_ids(program, 'programRuleVariables') print('Program:' + program['name']) print('Number of TEAs:' + str(len(teas_uid))) TEAs = api_source.get('trackedEntityAttributes', params={"paging": "false", "fields": "id,name,aggregationType,valueType,optionSet", "filter": "id:in:[" + ','.join(teas_uid) + "]"}).json()[ 'trackedEntityAttributes'] TEAs = reindex(TEAs, 'id') # Add the first row with eventDate and Enrollment label enrollmentDateLabel = "Enrollment date" if 'enrollmentDateLabel' in program: enrollmentDateLabel = program['enrollmentDateLabel'] # Add the program UID as UID for enrollmentDate df = df.append({"Stage": "Enrollment", "Section": "", "TEA / DE / eventDate": enrollmentDateLabel, "UID": program_uid, "valueType": "DATE", "optionSet": "", "mandatory": 'True'}, ignore_index=True) optionSetDict = dict() for TEA in program['programTrackedEntityAttributes']: tea_uid = TEA['trackedEntityAttribute']['id'] optionSet_def = "" if 'optionSet' in TEAs[tea_uid]: optionSet = TEAs[tea_uid]['optionSet']['id'] if optionSet not in optionSetDict: options = api_source.get('options', params={"paging": "false", "order": "sortOrder:asc", "fields": "id,code", "filter": "optionSet.id:eq:" + optionSet}).json()[ 'options'] optionsList = json_extract(options, 'code') optionSetDict[optionSet] = optionsList optionSet_def = '\n'.join(optionSetDict[optionSet]) df = df.append({"Stage": "", "Section": "", "TEA / DE / eventDate": TEA['name'], "UID": tea_uid, "valueType": TEA['valueType'], "optionSet": optionSet_def, "mandatory": TEA['mandatory']}, ignore_index=True) # print("TEA: " + TEA['name'] + " (" + TEA['valueType'] + ")") print('Number of Program Rule Variables:' + str(len(programRuleVariables_uid))) programRuleVariables = api_source.get('programRuleVariables', params={"paging": "false", "filter": "id:in:[" + ','.join(programRuleVariables_uid) + "]", "fields": "id,name,programRuleVariableSourceType,dataElement,trackedEntityAttribute" }).json()['programRuleVariables'] programRules = api_source.get('programRules', params={"paging": "false", "filter": "program.id:eq:" + program_uid, "fields": "id,name,condition"}).json()['programRules'] programRules_uid = json_extract(programRules, 'id') programRules = reindex(programRules, 'id') print('Number of Program Rules:' + str(len(programRules_uid))) # for uid in programRules: # print('Program Rule: ' + programRules[uid]['name']) programRuleActions = api_source.get('programRuleActions', params={"paging": "false", "filter": "programRule.id:in:[" + ','.join(programRules_uid) + "]", "fields": "id,name,programRuleActionType,data,content"}).json()[ 'programRuleActions'] programRuleActions_uid = json_extract(programRuleActions, 'id') print('Number of Program Rule Actions:' + str(len(programRuleActions_uid))) print('Number of Program Stages:' + str(len(programStages_uid))) programStages = api_source.get('programStages', params={"paging": "false", "order": "sortOrder:asc", "filter": "id:in:[" + ','.join(programStages_uid) + "]", "fields": "id,name,executionDateLabel,programStageSections,programStageDataElements"}).json()[ 'programStages'] for programStage in programStages: print('Stage:' + programStage['name'] + " (" + programStage['id'] + ")") # Add header to dataframe event_date_label = 'Event Date' if 'executionDateLabel' in programStage: event_date_label = programStage['executionDateLabel'] df = df.append({"Stage": programStage['name'], "Section": "", "TEA / DE / eventDate": event_date_label, "UID": programStage['id'], "valueType": "DATE", "optionSet": "", "mandatory": 'True'}, ignore_index=True) des_uid = json_extract_nested_ids(programStage, 'dataElement') dataElements = api_source.get('dataElements', params={"paging": "false", "fields": "id,name,categoryCombo,aggregationType,valueType,optionSet", "filter": "id:in:[" + ','.join(des_uid) + "]"}).json()[ 'dataElements'] dataElements = reindex(dataElements, 'id') # dataElements = reindex(dataElements, 'id') print('Number of DEs:' + str(len(des_uid))) if 'programStageSections' in programStage and len(programStage['programStageSections']) > 0: programStageSections_uid = json_extract_nested_ids(programStage, 'programStageSections') programStageSections = api_source.get('programStageSections', params={"paging": "false", "order": "sortOrder:asc", "fields": "id,name,dataElements", "filter": "id:in:[" + ','.join( programStageSections_uid) + "]"}).json()[ 'programStageSections'] dataElements_programStage = dict() for elem in programStage['programStageDataElements']: key_value = elem['dataElement']['id'] dataElements_programStage[key_value] = elem for programStageSection in programStageSections: print("Program Stage Section:" + programStageSection['name']) section_label = programStageSection['name'] for dataElement in programStageSection['dataElements']: dataElement_id = dataElement['id'] # This will fail if the DE is present in the PSSection but not in the PS, so we check first # if the key exists. If not, we warn the user and skip this if dataElement_id not in dataElements: logger.warning("Data Element with UID " + dataElement_id + " is present in program stage section but not assigned to the program stage") logger.warning("SKIPPING") else: dataElement_def = dataElements[dataElement_id] dataElement_PS = dataElements_programStage[dataElement_id] print('DE: ' + dataElement_def['name'] + " (" + dataElement_def['valueType'] + ")") optionSet_def = "" if 'optionSet' in dataElement_def: optionSet = dataElement_def['optionSet']['id'] if optionSet not in optionSetDict: options = api_source.get('options', params={"paging": "false", "order": "sortOrder:asc", "fields": "id,code", "filter": "optionSet.id:eq:" + optionSet}).json()[ 'options'] optionsList = json_extract(options, 'code') optionSetDict[optionSet] = optionsList optionSet_def = '\n'.join(optionSetDict[optionSet]) df = df.append({"Stage": "", "Section": section_label, "TEA / DE / eventDate": dataElement_def['name'], "UID": dataElement_id, "valueType": dataElement_def['valueType'], "optionSet": optionSet_def, "mandatory": dataElement_PS['compulsory']}, ignore_index=True) if section_label != "": section_label = "" else: # Assume BASIC todo: create CUSTOM for dataElement in programStage['programStageDataElements']: dataElement_id = dataElement['dataElement']['id'] dataElement_def = dataElements[dataElement_id] print('DE: ' + dataElement_def['name'] + " (" + dataElement_def['valueType'] + ")") optionSet_def = "" if 'optionSet' in dataElement_def: optionSet = dataElement_def['optionSet']['id'] if optionSet not in optionSetDict: options = api_source.get('options', params={"paging": "false", "order": "sortOrder:asc", "fields": "id,code", "filter": "optionSet.id:eq:" + optionSet}).json()[ 'options'] optionsList = json_extract(options, 'code') optionSetDict[optionSet] = optionsList optionSet_def = '\n'.join(optionSetDict[optionSet]) # print(' with optionSet = ' + dataElement['optionSet']['id']) df = df.append({"Stage": "", "Section": "", "TEA / DE / eventDate": dataElement_def['name'], "UID": dataElement_id, "valueType": dataElement_def['valueType'], "optionSet": optionSet_def, "mandatory": dataElement['compulsory']}, ignore_index=True) # Find out if it is used in programRuleVariable # for PRV in programRuleVariables: # if 'dataElement' in PRV and PRV['dataElement']['id'] == dataElement['id']: # print('Used in PRV:' + PRV['name'] + " (" + PRV['id'] + ")") # # Find out if used in ProgramRuleAction # for PRA in programRuleActions: # if 'dataElement' in PRA and PRA['dataElement']['id'] == dataElement['id']: # print('Used in PRA:' + PRA['name'] + " (" + PRA['id'] + ")") # print('Program Rule:' + programRules[PRA['programRule']['id']]['name']) # stages_counter = { 'ang4CLldbIu':25 } # df = add_repeatable_stages(df, stages_counter) # for tei in list_teis: # if len(tei["enrollments"][0]) > 0: # and tei["enrollments"][0]["storedBy"] == user: # result = add_json_tei_to_metadata_df(tei, df) # # export_csv = df.to_csv(r'./program-Case_Based_Surveillance-Dummy_data.csv', index=None, header=True) # get TEIs from OU if args.OrgUnit is not None: params = { 'ou': args.OrgUnit, 'ouMode': 'DESCENDANTS', 'program': program_uid, 'skipPaging': 'true', # 'lastUpdatedDuration': '4d', 'fields': '*', 'includeAllAttributes': 'true' } list_teis = api_source.get('trackedEntityInstances', params=params).json()['trackedEntityInstances'] logger.info("Found " + str(len(list_teis)) + " TEIs") stages_counter = dict() for tei in list_teis: counter = dict() if "enrollments" in tei and len( tei["enrollments"][0]) > 0: # and tei["enrollments"][0]["storedBy"] == user: if len(tei['enrollments']) == 1: if tei['enrollments'][0]['program'] == program_uid: if 'events' in tei['enrollments'][0]: events = tei['enrollments'][0]['events'] for event in events: if event["programStage"] in counter: counter[event["programStage"]] += 1 else: counter[event["programStage"]] = 1 else: logger.error( "TEI enrolled in program " + tei['enrollments'][0]['program'] + " not supported") else: logger.error('error, multi-enrollment not supported') for key in counter: if key not in stages_counter or stages_counter[key] < counter[key]: stages_counter[key] = counter[key] # logger.info('Found ' + str(stages_counter[key]) + ' instances of ' + key) df = add_repeatable_stages(df, stages_counter) for tei in list_teis: if len(tei["enrollments"][0]) > 0: # and tei["enrollments"][0]["storedBy"] == user: result = add_json_tei_to_metadata_df(tei, df) # Check if there are repeatable stages (only if TEIs were not provided) elif args.repeat_stage is not None and len(args.repeat_stage) > 0: stages_counter = dict() for param in args.repeat_stage: stages_counter[param[0]] = int(param[1]) df = add_repeatable_stages(df, stages_counter) # Create the spreadsheet url = create_google_spreadsheet(program, df, args.share_with) if url != "": logger.info('Spreadsheet created here: ' + url) else: logger.error("Something went wrong")
def add_json_tei_to_metadata_df(json_tei, df): def set_value(df, uid, value, min_pos=0): positions = df.index[(df.UID == uid) & (df.value == '')].tolist() # The idea with min_pos is to avoid filling data elements in the wrong stage # if a DE has a value in the next repeatable stage but it wasn't filled in the current one, # when we start filling the next stage we risk filling it in the previous one (because # it satisfies (df.UID == uid) & (df.value == '')). With min_pos we tell the script # start considering indexes starting from a certain value (the position of eventDate, the first DE which # is always present for the stage # Note: Packages like DRS use the same DE in different Program Stages. We need to verify that this works with # that use case if min_pos > 0: positions = [x for x in positions if x >= min_pos] if len(positions) == 0: # logger.error("Dataframe has not sufficient stages to fill datalement " + uid) return -1 else: df.at[positions[0], 'value'] = value return positions[0] # program UID is in the first row in Stage Enrollment program_id = df.iloc[0]['UID'] list_of_UIDs = df['UID'].tolist() # df_uid = df.set_index('UID') column = 'TEI_' + json_tei['trackedEntityInstance'] df['value'] = "" # np.nan # We are assuming just one enrollment in the program if len(json_tei['enrollments']) == 1 and json_tei['enrollments'][0]['program'] == program_id: json_enrollment = json_tei['enrollments'][0] # json_extract returns a list of values. It should be just one value in the list, so we get element 0 # dates are in the format 2020-11-05T00:00:00.000, so we truncate them set_value(df, program_id, json_extract(json_enrollment, 'enrollmentDate')[0][0:10]) for attribute in json_tei["attributes"]: if attribute["attribute"] not in list_of_UIDs: logger.error('Attribute = ' + attribute["attribute"] + ' in TEI = ' + json_tei[ 'trackedEntityInstance'] + ' not present in df') return False set_value(df, attribute["attribute"], attribute["value"]) json_events = json_enrollment["events"] pos = dict() for event in json_events: # Considering here that program stages appear in order but it might be better to loop through them in order program_stage_uid = event['programStage'] # if the programme allows for the future scheduling of events, # this will mean that even though the event date is mandatory, # scheduled events which have not yet happen, will not yet have an event date if 'eventDate' in event: pos[program_stage_uid] = set_value(df, program_stage_uid, event['eventDate'][0:10]) else: pos[program_stage_uid] = set_value(df, program_stage_uid, '') if pos == -1: # There was a problem return False if 'dataValues' in event: for dataValue in event['dataValues']: if dataValue["dataElement"] not in list_of_UIDs: logger.error('Data Element = ' + dataValue["dataElement"] + ' in TEI = ' + json_tei[ 'trackedEntityInstance'] + ' not present in df') return False result = set_value(df, dataValue["dataElement"], dataValue["value"], pos[program_stage_uid]) if result == -1: # Check that the DE is assigned to the proram stage program_stage_info = api_source.get('programStages/' + program_stage_uid, params={ "fields": "programStageDataElements[dataElement]"}).json() data_elements_in_ps = json_extract_nested_ids(program_stage_info, 'dataElement') if dataValue["dataElement"] not in data_elements_in_ps: logger.error("TEI " + json_tei['trackedEntityInstance'] + " has a dataValue for DE " + dataValue["dataElement"] + " in stage " + program_stage_uid + " but this DE is NOT assigned to this PS") else: logger.error("Dataframe has not sufficient stages to fill datalement " + dataValue["dataElement"]) else: if len(json_tei['enrollments']) > 1: logger.error('Multi-enrollments not supported') return False else: logger.error('TEI does not belong to the dataframe program') return False # Rename column df.rename(columns={"value": 'TEI_' + json_tei['trackedEntityInstance']}, inplace=True) return True
def build_analytics_payload(json_object, verbose=False): def get_group_set_dimensions(json_object, key): # parent_key, child_key): parent_key = key + 'Set' child_key = key + 's' for grp_set_dimension in json_object: grp_set_dimension_uid = grp_set_dimension[parent_key]['id'] if child_key in grp_set_dimension: group_set_list = list() for group in grp_set_dimension[child_key]: group_set_list.append(group['id']) dimensions[grp_set_dimension_uid] = group_set_list else: dimensions[grp_set_dimension_uid] = "" dimensions = dict() dimensions['ou'] = list() dimensions['pe'] = list() dimensions['dx'] = list() if 'organisationUnits' not in json_object or len( json_object['organisationUnits']) == 0: ou_global_selections = { 'userOrganisationUnit': 'USER_ORGUNIT', 'userOrganisationUnitChildren': 'USER_ORGUNIT_CHILDREN', 'userOrganisationUnitGrandChildren': 'USER_ORGUNIT_GRANDCHILDREN' } for ou_selection in ou_global_selections: if ou_selection in json_object and json_object[ ou_selection] == True: dimensions['ou'].append(ou_global_selections[ou_selection]) if 'organisationUnitLevels' in json_object and len( json_object['organisationUnitLevels']) > 0: ou_level_list = list() for org_unit_level in json_object['organisationUnitLevels']: ou_level_list.append('LEVEL-' + str(org_unit_level)) if len(ou_level_list) > 0: dimensions['ou'] += ou_level_list if 'itemOrganisationUnitGroups' in json_object and len( json_object['itemOrganisationUnitGroups']) > 0: ou_group_list = list() for org_unit_group in json_object['itemOrganisationUnitGroups']: ou_group_list.append('OU_GROUP-' + str(org_unit_group['id'])) if len(ou_group_list) > 0: dimensions['ou'] += ou_group_list else: dimensions['ou'] = json_extract(json_object['organisationUnits'], 'id') if 'periods' not in json_object or len(json_object['periods']) == 0: if 'relativePeriods' in json_object: pe_global_selections = { 'thisDay': 'TODAY', 'thisWeek': 'THIS_WEEK', 'thisMonth': 'THIS_MONTH', 'thisQuarter': 'THIS_QUARTER', 'thisYear': 'THIS_YEAR', 'lastDay': 'LAST_DAY', 'lastWeek': 'LAST_WEEK', 'lastMonth': 'LAST_MONTH', 'lastQuarter': 'LAST_QUARTER', 'lastYear': 'LAST_YEAR', 'last30Days': 'LAST_30_DAYS', 'last52Weeks': 'LAST_52_WEEKS', 'last90Days': 'LAST_90_DAYS', 'last60Days': 'LAST_60_DAYS', 'last14Days': 'LAST_14_DAYS', 'last2SixMonths': 'LAST_2_SIXMONTHS', 'last12Months': 'LAST_12_MONTHS', 'last4Weeks': 'LAST_4_WEEKS', 'last3Months': 'LAST_3_MONTHS', 'last5Years': 'LAST_5_YEARS', 'last6Months': 'LAST_6_MONTHS', 'last3Days': 'LAST_3_DAYS', 'last7Days': 'LAST_7_DAYS', 'last180Days': 'LAST_180_DAYS', 'last12Weeks': 'LAST_12_WEEKS', 'last4Quarters': 'LAST_4_QUARTERS', 'weeksThisYear': 'WEEKS_THIS_YEAR', 'yesterday': 'YESTERDAY', 'quartersLastYear': 'QUARTERS_LAST_YEAR', 'monthsThisYear': 'MONTHS_THIS_YEAR', 'biMonthsThisYear': 'BI_MONTHS_THIS_YEAR', 'last5FinancialYears': 'LAST_5_FINANCIAL_YEARS', 'thisSixMonth': 'THIS_SIX_MONTH', 'thisFinancialYear': 'THIS_FINANCIAL_YEAR', 'last6BiMonths': 'LAST_6_BI_MONTHS', 'last4BiWeeks': 'LAST_6_BI_WEEKS', 'lastFinancialYear': 'LAST_FINANCIAL_YEAR', 'lastBiWeek': 'LAST_BI_WEEK', 'quartersThisYear': 'QUARTERS_THIS_YEAR', 'monthsLastYear': 'MONTHS_LAST_YEAR', 'thisBimonth': 'THIS_BI_MONTH', 'lastBimonth': 'LAST_BI_MONTH', 'lastSixMonth': 'LAST_SIX_MONTH', 'thisBiWeek': 'THIS_BI_WEEK', 'last10Years': 'LAST_10_YEARS', 'last10FinancialYears': 'LAST_10_FINANCIAL_YEARS' } for relative_period in json_object['relativePeriods']: if relative_period in pe_global_selections: if json_object['relativePeriods'][relative_period]: dimensions['pe'].append( pe_global_selections[relative_period]) else: logger.error("Unknown relativePeriod " + relative_period) exit(1) if len(dimensions['pe']) == 0 and 'periods' in json_object: dimensions['pe'] = json_extract(json_object['periods'], 'id') else: return {} else: dimensions['pe'] = json_extract(json_object['periods'], 'id') if len(dimensions['pe'] ) == 0 and 'startDate' in json_object and 'endDate' in json_object: del dimensions['pe'] if 'dataDimensionItems' in json_object and len( json_object['dataDimensionItems']) > 0: data_dimension_keys = { 'PROGRAM_INDICATOR': 'programIndicator', 'INDICATOR': 'indicator', 'DATA_ELEMENT': 'dataElement', 'REPORTING_RATE': 'reportingRate', 'PROGRAM_DATA_ELEMENT': 'programDataElement' } for data_dimension in json_object['dataDimensionItems']: # Sometimes there are empty dimensions if data_dimension != {}: if data_dimension[ 'dataDimensionItemType'] in data_dimension_keys: # Special case, it joins to UIDs with a . if data_dimension[ 'dataDimensionItemType'] == 'PROGRAM_DATA_ELEMENT': UID1 = data_dimension[data_dimension_keys[ data_dimension['dataDimensionItemType']]][ 'program']['id'] UID2 = data_dimension[data_dimension_keys[ data_dimension['dataDimensionItemType']]][ 'dataElement']['id'] dimensions['dx'].append(UID1 + '.' + UID2) else: data_dimension_uid = data_dimension[ data_dimension_keys[ data_dimension['dataDimensionItemType']]]['id'] if data_dimension[ 'dataDimensionItemType'] == 'REPORTING_RATE': # For reporting rates, we need to add the keyword to the id dimensions['dx'].append(data_dimension_uid + '.REPORTING_RATE') else: dimensions['dx'].append(data_dimension_uid) else: logger.error('Unrecognized data dimension type ' + data_dimension['dataDimensionItemType']) exit(1) dataElementDimension_filters = dict() if 'dataElementDimensions' in json_object and len( json_object['dataElementDimensions']) > 0: for data_element_dimension in json_object['dataElementDimensions']: data_element_dimension_uid = data_element_dimension['dataElement'][ 'id'] if 'filter' in data_element_dimension: dimensions[ data_element_dimension_uid] = data_element_dimension[ 'filter'] else: dimensions[data_element_dimension_uid] = "" if 'categoryOptionGroupSetDimensions' in json_object and len( json_object['categoryOptionGroupSetDimensions']) > 0: get_group_set_dimensions( json_object['categoryOptionGroupSetDimensions'], 'categoryOptionGroup') if 'organisationUnitGroupSetDimensions' in json_object and len( json_object['organisationUnitGroupSetDimensions']) > 0: get_group_set_dimensions( json_object['organisationUnitGroupSetDimensions'], 'organisationUnitGroup') if 'dataElementGroupSetDimensions' in json_object and len( json_object['dataElementGroupSetDimensions']) > 0: get_group_set_dimensions(json_object['dataElementGroupSetDimensions'], 'dataElementGroup') if 'categoryDimensions' in json_object and len( json_object['categoryDimensions']) > 0: for category_dimension in json_object['categoryDimensions']: category_dimension_uid = category_dimension['category']['id'] category_options = list() for cat_options in category_dimension['categoryOptions']: category_options.append(cat_options['id']) dimensions[category_dimension_uid] = category_options # Build the payload payload = "" params = dict() payload += 'dimension=' params['dimension'] = "" added_column_dimension = False if 'columns' in json_object: first_element = True for column in json_object['columns']: added_column_dimension = True if not first_element: payload += ',' params['dimension'] += ',' else: first_element = False key = column['id'] if key in dimensions: if isinstance(dimensions[key], list): right_expression = ';'.join(dimensions[key]) else: right_expression = dimensions[key] payload += key + ':' + right_expression params['dimension'] += key + ':' + right_expression else: if key == 'pe' and 'startDate' in json_object and 'endDate' in json_object: payload += "&startDate=" + json_object[ 'startDate'] + "&endDate=" + json_object['endDate'] params['startDate'] = json_object['startDate'] params['endDate'] = json_object['endDate'] else: logger.error(json_object['id'] + ': Dimension ' + key + ' is missing') # exit(1) else: logger.error('columns missing') exit(1) # A very specific and strange case for maps # empty columns but styleDataItem is present. In that case, it gets added to the dimension if 'columns' in json_object and len( json_object['columns']) == 0 and 'styleDataItem' in json_object: payload += json_object['styleDataItem']['id'] params['dimension'] += json_object['styleDataItem']['id'] if 'rows' in json_object and len(json_object['rows']) > 0: payload += ',' params['dimension'] += ',' if 'rows' in json_object: if len(json_object['rows']) > 0: # If we have already added some stuff, separate it with a comma if added_column_dimension: payload += ',' params['dimension'] += ',' first_element = True for row in json_object['rows']: if not first_element: payload += ',' params['dimension'] += ',' else: first_element = False key = row['id'] if key in dimensions: payload += key + ':' + ';'.join(dimensions[key]) params['dimension'] += key + ':' + ';'.join( dimensions[key]) else: if key == 'pe' and 'startDate' in json_object and 'endDate' in json_object: payload += "&startDate=" + json_object[ 'startDate'] + "&endDate=" + json_object['endDate'] params['startDate'] = json_object['startDate'] params['endDate'] = json_object['endDate'] else: logger.error(json_object['id'] + ': Dimension ' + key + ' is missing') # exit(1) else: logger.error('rows missing') exit(1) if 'filters' in json_object: if len(json_object['filters']) > 0: payload += '&filter=' params['filter'] = "" first_element = True for filter in json_object['filters']: if not first_element: payload += ',' params['filter'] += ',' else: first_element = False key = filter['id'] if key in dimensions: payload += key + ':' + ';'.join(dimensions[key]) params['filter'] += key + ':' + ';'.join(dimensions[key]) else: if key == 'pe' and 'startDate' in json_object and 'endDate' in json_object: payload += "&startDate=" + json_object[ 'startDate'] + "&endDate=" + json_object['endDate'] params['startDate'] = json_object['startDate'] params['endDate'] = json_object['endDate'] else: logger.error(json_object['id'] + ': Dimension ' + key + ' is missing') # exit(1) else: logger.error('filters missing') exit(1) if 'programStage' in json_object: payload += '&stage' + ':' + json_object['programStage']['id'] params['stage'] = json_object['programStage']['id'] # Important, to get the data payload += "&skipData=false" params['skipData'] = 'false' if verbose: logger.info(payload) return params