def main():
pd.set_option('display.max_columns', None)
import argparse
my_parser = argparse.ArgumentParser(prog='create_flat_file',
description='Create dummy data flat file in Google Spreadsheets',
epilog="python create_flat_file Lt6P15ps7f6 --with_teis_from=GZ5Ty90HtW [email protected]"
"\npython create_flat_file Lt6P15ps7f6 --repeat_stage Hj38Uhfo012 5 --repeat_stage 77Ujkfoi9kG 3 [email protected] [email protected]",
formatter_class=argparse.RawDescriptionHelpFormatter)
my_parser.add_argument('Program_UID', metavar='program_uid', type=str,
help='the uid of the program to use')
my_parser.add_argument('-wtf', '--with_teis_from', action="store", dest="OrgUnit", type=str,
help='Pulls TEIs from specified org unit and adds them to flat file. '
'Eg: --with_teis_from_ou=Q7RbNZcHrQ9')
my_parser.add_argument('-rs', '--repeat_stage', action="append", metavar=('stage_uid', 'number_repeats'), nargs=2,
help='provide a stage uid which is REPEATABLE and specify how many times you are planning to enter it. '
'Eg: --repeat_stage QXtjg5dh34A 3')
my_parser.add_argument('-sw', '--share_with', action="append", metavar='email', nargs=1,
help='email address to share the generated spreadsheet with as OWNER. '
'Eg: [email protected]')
args = my_parser.parse_args()
program_uid = args.Program_UID
if not is_valid_uid(program_uid):
print('The program uid specified is not valid')
sys.exit()
if args.OrgUnit is not None and not is_valid_uid(args.OrgUnit):
print('The orgunit uid specified is not valid')
sys.exit()
if args.repeat_stage is not None and len(args.repeat_stage) > 0:
for param in args.repeat_stage:
if not is_valid_uid(param[0]):
print('The program stage uid specified ' + param[0] + ' is not valid')
sys.exit()
try:
int(param[1])
except ValueError:
print('The repetition value ' + param[1] + ' is not an integer')
sys.exit()
if args.share_with is not None and len(args.share_with) > 0:
for param in args.share_with:
if not (re.search('^[a-z0-9]+[\._]?[a-z0-9]+[@]\w+[.]\w{2,3}$', param[0])):
print("The email address " + param[0] + " is not valid")
# Print DHIS2 Info
logger.warning("Server source running DHIS2 version {} revision {}"
.format(api_source.version, api_source.revision))
##############
# df = pd.read_csv('program-Case_Based_Surveillance.csv', sep=None, engine='python')
# #
# # # stages_counter = { 'K5ac7u3V5bB': 1, 'ang4CLldbIu': 5, 'UvYb6qJpQu0': 1 }
# #
# # #json_tei = api_source.get('trackedEntityInstances/dRdztYSReOZ', params={'fields':'*'}).json()
# #
# params = {
# 'ou': 'RI95HQRHbKc', # GD7TowwI46c
# 'ouMode': 'DESCENDANTS',
# 'program': program_uid,
# 'skipPaging': 'true',
# 'lastUpdatedDuration': '4d',
# 'fields': '*',
# 'includeAllAttributes': 'true'
# }
#
# list_teis = api_source.get('trackedEntityInstances', params=params).json()['trackedEntityInstances']
#
# logger.info("Found " + str(len(list_teis)) + " TEIs")
#
# user = '******'
# stages_counter = dict()
# for tei in list_teis:
# counter = dict()
# if "enrollments" in tei and len(tei["enrollments"][0]) > 0: # and tei["enrollments"][0]["storedBy"] == user:
# if len(tei['enrollments']) == 1:
# if tei['enrollments'][0]['program'] == program_uid:
# if 'events' in tei['enrollments'][0]:
# events = tei['enrollments'][0]['events']
# for event in events:
# if event["programStage"] in counter:
# counter[event["programStage"]] +=1
# else:
# counter[event["programStage"]] = 1
# else:
# logger.error("TEI enrolled in program " + tei['enrollments'][0]['program'] + " not supported")
# else:
# logger.error('error, multi-enrollment not supported')
# for key in counter:
# if key not in stages_counter or stages_counter[key] < counter[key]:
# stages_counter[key] = counter[key]
# logger.info('Found ' + str(stages_counter[key]) + ' instances of ' + key)
#
# df = add_repeatable_stages(df, stages_counter)
# for tei in list_teis:
# # if tei['trackedEntityInstance'] != 'j17HROzXGEn':
# # continue
# if len(tei["enrollments"][0]) > 0: # and tei["enrollments"][0]["storedBy"] == user:
# result = add_json_tei_to_metadata_df(tei, df)
# export_csv = df.to_csv(r'./program-Case_Based_Surveillance-Dummy_data.csv', index=None, header=True)
###########
df = pd.DataFrame({}, columns=["Stage", "Section", "TEA / DE / eventDate", "UID", "valueType", "optionSet",
"mandatory"])
try:
program = api_source.get('programs/' + program_uid,
params={"paging": "false",
"fields": "id,name,enrollmentDateLabel,programTrackedEntityAttributes,programStages,programRuleVariables,organisationUnits,trackedEntityType,version"}).json()
except RequestException as e:
if e.code == 404:
logger.error('Program ' + program_uid + ' specified does not exist')
sys.exit()
if isinstance(program, dict):
# If the program has many org units assigned, this can take a long time to run!!!
# orgunits_uid = json_extract_nested_ids(program, 'organisationUnits')
# if args.OrgUnit is not None and args.OrgUnit not in orgunits_uid:
# logger.error('The organisation unit ' + args.OrgUnit + ' is not assigned to program ' + program_uid)
# print('Number of OrgUnits:' + str(len(orgunits_uid)))
programStages_uid = json_extract_nested_ids(program, 'programStages')
if args.repeat_stage is not None:
for param in args.repeat_stage:
found = False
for uid in programStages_uid:
if param[0] == uid:
found = True
break
if not found:
logger.error(uid + ' specified is not a valid stage for program ' + program_uid)
sys.exit()
teas_uid = json_extract_nested_ids(program, 'trackedEntityAttribute')
programRuleVariables_uid = json_extract_nested_ids(program, 'programRuleVariables')
print('Program:' + program['name'])
print('Number of TEAs:' + str(len(teas_uid)))
TEAs = api_source.get('trackedEntityAttributes',
params={"paging": "false", "fields": "id,name,aggregationType,valueType,optionSet",
"filter": "id:in:[" + ','.join(teas_uid) + "]"}).json()[
'trackedEntityAttributes']
TEAs = reindex(TEAs, 'id')
# Add the first row with eventDate and Enrollment label
enrollmentDateLabel = "Enrollment date"
if 'enrollmentDateLabel' in program:
enrollmentDateLabel = program['enrollmentDateLabel']
# Add the program UID as UID for enrollmentDate
df = df.append({"Stage": "Enrollment", "Section": "", "TEA / DE / eventDate": enrollmentDateLabel,
"UID": program_uid, "valueType": "DATE", "optionSet": "", "mandatory": 'True'},
ignore_index=True)
optionSetDict = dict()
for TEA in program['programTrackedEntityAttributes']:
tea_uid = TEA['trackedEntityAttribute']['id']
optionSet_def = ""
if 'optionSet' in TEAs[tea_uid]:
optionSet = TEAs[tea_uid]['optionSet']['id']
if optionSet not in optionSetDict:
options = api_source.get('options', params={"paging": "false",
"order": "sortOrder:asc",
"fields": "id,code",
"filter": "optionSet.id:eq:" + optionSet}).json()[
'options']
optionsList = json_extract(options, 'code')
optionSetDict[optionSet] = optionsList
optionSet_def = '\n'.join(optionSetDict[optionSet])
df = df.append({"Stage": "", "Section": "", "TEA / DE / eventDate": TEA['name'],
"UID": tea_uid,
"valueType": TEA['valueType'], "optionSet": optionSet_def,
"mandatory": TEA['mandatory']}, ignore_index=True)
# print("TEA: " + TEA['name'] + " (" + TEA['valueType'] + ")")
print('Number of Program Rule Variables:' + str(len(programRuleVariables_uid)))
programRuleVariables = api_source.get('programRuleVariables',
params={"paging": "false",
"filter": "id:in:[" + ','.join(programRuleVariables_uid) + "]",
"fields": "id,name,programRuleVariableSourceType,dataElement,trackedEntityAttribute"
}).json()['programRuleVariables']
programRules = api_source.get('programRules',
params={"paging": "false",
"filter": "program.id:eq:" + program_uid,
"fields": "id,name,condition"}).json()['programRules']
programRules_uid = json_extract(programRules, 'id')
programRules = reindex(programRules, 'id')
print('Number of Program Rules:' + str(len(programRules_uid)))
# for uid in programRules:
# print('Program Rule: ' + programRules[uid]['name'])
programRuleActions = api_source.get('programRuleActions',
params={"paging": "false",
"filter": "programRule.id:in:[" + ','.join(programRules_uid) + "]",
"fields": "id,name,programRuleActionType,data,content"}).json()[
'programRuleActions']
programRuleActions_uid = json_extract(programRuleActions, 'id')
print('Number of Program Rule Actions:' + str(len(programRuleActions_uid)))
print('Number of Program Stages:' + str(len(programStages_uid)))
programStages = api_source.get('programStages',
params={"paging": "false", "order": "sortOrder:asc",
"filter": "id:in:[" + ','.join(programStages_uid) + "]",
"fields": "id,name,executionDateLabel,programStageSections,programStageDataElements"}).json()[
'programStages']
for programStage in programStages:
print('Stage:' + programStage['name'] + " (" + programStage['id'] + ")")
# Add header to dataframe
event_date_label = 'Event Date'
if 'executionDateLabel' in programStage:
event_date_label = programStage['executionDateLabel']
df = df.append({"Stage": programStage['name'], "Section": "",
"TEA / DE / eventDate": event_date_label,
"UID": programStage['id'], "valueType": "DATE", "optionSet": "", "mandatory": 'True'},
ignore_index=True)
des_uid = json_extract_nested_ids(programStage, 'dataElement')
dataElements = api_source.get('dataElements',
params={"paging": "false",
"fields": "id,name,categoryCombo,aggregationType,valueType,optionSet",
"filter": "id:in:[" + ','.join(des_uid) + "]"}).json()[
'dataElements']
dataElements = reindex(dataElements, 'id')
# dataElements = reindex(dataElements, 'id')
print('Number of DEs:' + str(len(des_uid)))
if 'programStageSections' in programStage and len(programStage['programStageSections']) > 0:
programStageSections_uid = json_extract_nested_ids(programStage, 'programStageSections')
programStageSections = api_source.get('programStageSections',
params={"paging": "false", "order": "sortOrder:asc",
"fields": "id,name,dataElements",
"filter": "id:in:[" + ','.join(
programStageSections_uid) + "]"}).json()[
'programStageSections']
dataElements_programStage = dict()
for elem in programStage['programStageDataElements']:
key_value = elem['dataElement']['id']
dataElements_programStage[key_value] = elem
for programStageSection in programStageSections:
print("Program Stage Section:" + programStageSection['name'])
section_label = programStageSection['name']
for dataElement in programStageSection['dataElements']:
dataElement_id = dataElement['id']
# This will fail if the DE is present in the PSSection but not in the PS, so we check first
# if the key exists. If not, we warn the user and skip this
if dataElement_id not in dataElements:
logger.warning("Data Element with UID " + dataElement_id +
" is present in program stage section but not assigned to the program stage")
logger.warning("SKIPPING")
else:
dataElement_def = dataElements[dataElement_id]
dataElement_PS = dataElements_programStage[dataElement_id]
print('DE: ' + dataElement_def['name'] + " (" + dataElement_def['valueType'] + ")")
optionSet_def = ""
if 'optionSet' in dataElement_def:
optionSet = dataElement_def['optionSet']['id']
if optionSet not in optionSetDict:
options = api_source.get('options', params={"paging": "false",
"order": "sortOrder:asc",
"fields": "id,code",
"filter": "optionSet.id:eq:" + optionSet}).json()[
'options']
optionsList = json_extract(options, 'code')
optionSetDict[optionSet] = optionsList
optionSet_def = '\n'.join(optionSetDict[optionSet])
df = df.append({"Stage": "", "Section": section_label,
"TEA / DE / eventDate": dataElement_def['name'],
"UID": dataElement_id, "valueType": dataElement_def['valueType'],
"optionSet": optionSet_def, "mandatory": dataElement_PS['compulsory']},
ignore_index=True)
if section_label != "":
section_label = ""
else: # Assume BASIC todo: create CUSTOM
for dataElement in programStage['programStageDataElements']:
dataElement_id = dataElement['dataElement']['id']
dataElement_def = dataElements[dataElement_id]
print('DE: ' + dataElement_def['name'] + " (" + dataElement_def['valueType'] + ")")
optionSet_def = ""
if 'optionSet' in dataElement_def:
optionSet = dataElement_def['optionSet']['id']
if optionSet not in optionSetDict:
options = api_source.get('options', params={"paging": "false",
"order": "sortOrder:asc",
"fields": "id,code",
"filter": "optionSet.id:eq:" + optionSet}).json()[
'options']
optionsList = json_extract(options, 'code')
optionSetDict[optionSet] = optionsList
optionSet_def = '\n'.join(optionSetDict[optionSet])
# print(' with optionSet = ' + dataElement['optionSet']['id'])
df = df.append({"Stage": "", "Section": "", "TEA / DE / eventDate": dataElement_def['name'],
"UID": dataElement_id, "valueType": dataElement_def['valueType'],
"optionSet": optionSet_def, "mandatory": dataElement['compulsory']},
ignore_index=True)
# Find out if it is used in programRuleVariable
# for PRV in programRuleVariables:
# if 'dataElement' in PRV and PRV['dataElement']['id'] == dataElement['id']:
# print('Used in PRV:' + PRV['name'] + " (" + PRV['id'] + ")")
# # Find out if used in ProgramRuleAction
# for PRA in programRuleActions:
# if 'dataElement' in PRA and PRA['dataElement']['id'] == dataElement['id']:
# print('Used in PRA:' + PRA['name'] + " (" + PRA['id'] + ")")
# print('Program Rule:' + programRules[PRA['programRule']['id']]['name'])
# stages_counter = { 'ang4CLldbIu':25 }
# df = add_repeatable_stages(df, stages_counter)
# for tei in list_teis:
# if len(tei["enrollments"][0]) > 0: # and tei["enrollments"][0]["storedBy"] == user:
# result = add_json_tei_to_metadata_df(tei, df)
#
# export_csv = df.to_csv(r'./program-Case_Based_Surveillance-Dummy_data.csv', index=None, header=True)
# get TEIs from OU
if args.OrgUnit is not None:
params = {
'ou': args.OrgUnit,
'ouMode': 'DESCENDANTS',
'program': program_uid,
'skipPaging': 'true',
# 'lastUpdatedDuration': '4d',
'fields': '*',
'includeAllAttributes': 'true'
}
list_teis = api_source.get('trackedEntityInstances', params=params).json()['trackedEntityInstances']
logger.info("Found " + str(len(list_teis)) + " TEIs")
stages_counter = dict()
for tei in list_teis:
counter = dict()
if "enrollments" in tei and len(
tei["enrollments"][0]) > 0: # and tei["enrollments"][0]["storedBy"] == user:
if len(tei['enrollments']) == 1:
if tei['enrollments'][0]['program'] == program_uid:
if 'events' in tei['enrollments'][0]:
events = tei['enrollments'][0]['events']
for event in events:
if event["programStage"] in counter:
counter[event["programStage"]] += 1
else:
counter[event["programStage"]] = 1
else:
logger.error(
"TEI enrolled in program " + tei['enrollments'][0]['program'] + " not supported")
else:
logger.error('error, multi-enrollment not supported')
for key in counter:
if key not in stages_counter or stages_counter[key] < counter[key]:
stages_counter[key] = counter[key]
# logger.info('Found ' + str(stages_counter[key]) + ' instances of ' + key)
df = add_repeatable_stages(df, stages_counter)
for tei in list_teis:
if len(tei["enrollments"][0]) > 0: # and tei["enrollments"][0]["storedBy"] == user:
result = add_json_tei_to_metadata_df(tei, df)
# Check if there are repeatable stages (only if TEIs were not provided)
elif args.repeat_stage is not None and len(args.repeat_stage) > 0:
stages_counter = dict()
for param in args.repeat_stage:
stages_counter[param[0]] = int(param[1])
df = add_repeatable_stages(df, stages_counter)
# Create the spreadsheet
url = create_google_spreadsheet(program, df, args.share_with)
if url != "":
logger.info('Spreadsheet created here: ' + url)
else:
logger.error("Something went wrong")
def add_json_tei_to_metadata_df(json_tei, df):
def set_value(df, uid, value, min_pos=0):
positions = df.index[(df.UID == uid) & (df.value == '')].tolist()
# The idea with min_pos is to avoid filling data elements in the wrong stage
# if a DE has a value in the next repeatable stage but it wasn't filled in the current one,
# when we start filling the next stage we risk filling it in the previous one (because
# it satisfies (df.UID == uid) & (df.value == '')). With min_pos we tell the script
# start considering indexes starting from a certain value (the position of eventDate, the first DE which
# is always present for the stage
# Note: Packages like DRS use the same DE in different Program Stages. We need to verify that this works with
# that use case
if min_pos > 0:
positions = [x for x in positions if x >= min_pos]
if len(positions) == 0:
# logger.error("Dataframe has not sufficient stages to fill datalement " + uid)
return -1
else:
df.at[positions[0], 'value'] = value
return positions[0]
# program UID is in the first row in Stage Enrollment
program_id = df.iloc[0]['UID']
list_of_UIDs = df['UID'].tolist()
# df_uid = df.set_index('UID')
column = 'TEI_' + json_tei['trackedEntityInstance']
df['value'] = "" # np.nan
# We are assuming just one enrollment in the program
if len(json_tei['enrollments']) == 1 and json_tei['enrollments'][0]['program'] == program_id:
json_enrollment = json_tei['enrollments'][0]
# json_extract returns a list of values. It should be just one value in the list, so we get element 0
# dates are in the format 2020-11-05T00:00:00.000, so we truncate them
set_value(df, program_id, json_extract(json_enrollment, 'enrollmentDate')[0][0:10])
for attribute in json_tei["attributes"]:
if attribute["attribute"] not in list_of_UIDs:
logger.error('Attribute = ' + attribute["attribute"] + ' in TEI = ' + json_tei[
'trackedEntityInstance'] + ' not present in df')
return False
set_value(df, attribute["attribute"], attribute["value"])
json_events = json_enrollment["events"]
pos = dict()
for event in json_events:
# Considering here that program stages appear in order but it might be better to loop through them in order
program_stage_uid = event['programStage']
# if the programme allows for the future scheduling of events,
# this will mean that even though the event date is mandatory,
# scheduled events which have not yet happen, will not yet have an event date
if 'eventDate' in event:
pos[program_stage_uid] = set_value(df, program_stage_uid, event['eventDate'][0:10])
else:
pos[program_stage_uid] = set_value(df, program_stage_uid, '')
if pos == -1: # There was a problem
return False
if 'dataValues' in event:
for dataValue in event['dataValues']:
if dataValue["dataElement"] not in list_of_UIDs:
logger.error('Data Element = ' + dataValue["dataElement"] + ' in TEI = ' + json_tei[
'trackedEntityInstance'] + ' not present in df')
return False
result = set_value(df, dataValue["dataElement"], dataValue["value"], pos[program_stage_uid])
if result == -1:
# Check that the DE is assigned to the proram stage
program_stage_info = api_source.get('programStages/' + program_stage_uid,
params={
"fields": "programStageDataElements[dataElement]"}).json()
data_elements_in_ps = json_extract_nested_ids(program_stage_info, 'dataElement')
if dataValue["dataElement"] not in data_elements_in_ps:
logger.error("TEI " + json_tei['trackedEntityInstance'] +
" has a dataValue for DE " + dataValue["dataElement"] +
" in stage " + program_stage_uid +
" but this DE is NOT assigned to this PS")
else:
logger.error("Dataframe has not sufficient stages to fill datalement "
+ dataValue["dataElement"])
else:
if len(json_tei['enrollments']) > 1:
logger.error('Multi-enrollments not supported')
return False
else:
logger.error('TEI does not belong to the dataframe program')
return False
# Rename column
df.rename(columns={"value": 'TEI_' + json_tei['trackedEntityInstance']}, inplace=True)
return True
def build_analytics_payload(json_object, verbose=False):
def get_group_set_dimensions(json_object, key): # parent_key, child_key):
parent_key = key + 'Set'
child_key = key + 's'
for grp_set_dimension in json_object:
grp_set_dimension_uid = grp_set_dimension[parent_key]['id']
if child_key in grp_set_dimension:
group_set_list = list()
for group in grp_set_dimension[child_key]:
group_set_list.append(group['id'])
dimensions[grp_set_dimension_uid] = group_set_list
else:
dimensions[grp_set_dimension_uid] = ""
dimensions = dict()
dimensions['ou'] = list()
dimensions['pe'] = list()
dimensions['dx'] = list()
if 'organisationUnits' not in json_object or len(
json_object['organisationUnits']) == 0:
ou_global_selections = {
'userOrganisationUnit': 'USER_ORGUNIT',
'userOrganisationUnitChildren': 'USER_ORGUNIT_CHILDREN',
'userOrganisationUnitGrandChildren': 'USER_ORGUNIT_GRANDCHILDREN'
}
for ou_selection in ou_global_selections:
if ou_selection in json_object and json_object[
ou_selection] == True:
dimensions['ou'].append(ou_global_selections[ou_selection])
if 'organisationUnitLevels' in json_object and len(
json_object['organisationUnitLevels']) > 0:
ou_level_list = list()
for org_unit_level in json_object['organisationUnitLevels']:
ou_level_list.append('LEVEL-' + str(org_unit_level))
if len(ou_level_list) > 0:
dimensions['ou'] += ou_level_list
if 'itemOrganisationUnitGroups' in json_object and len(
json_object['itemOrganisationUnitGroups']) > 0:
ou_group_list = list()
for org_unit_group in json_object['itemOrganisationUnitGroups']:
ou_group_list.append('OU_GROUP-' + str(org_unit_group['id']))
if len(ou_group_list) > 0:
dimensions['ou'] += ou_group_list
else:
dimensions['ou'] = json_extract(json_object['organisationUnits'], 'id')
if 'periods' not in json_object or len(json_object['periods']) == 0:
if 'relativePeriods' in json_object:
pe_global_selections = {
'thisDay': 'TODAY',
'thisWeek': 'THIS_WEEK',
'thisMonth': 'THIS_MONTH',
'thisQuarter': 'THIS_QUARTER',
'thisYear': 'THIS_YEAR',
'lastDay': 'LAST_DAY',
'lastWeek': 'LAST_WEEK',
'lastMonth': 'LAST_MONTH',
'lastQuarter': 'LAST_QUARTER',
'lastYear': 'LAST_YEAR',
'last30Days': 'LAST_30_DAYS',
'last52Weeks': 'LAST_52_WEEKS',
'last90Days': 'LAST_90_DAYS',
'last60Days': 'LAST_60_DAYS',
'last14Days': 'LAST_14_DAYS',
'last2SixMonths': 'LAST_2_SIXMONTHS',
'last12Months': 'LAST_12_MONTHS',
'last4Weeks': 'LAST_4_WEEKS',
'last3Months': 'LAST_3_MONTHS',
'last5Years': 'LAST_5_YEARS',
'last6Months': 'LAST_6_MONTHS',
'last3Days': 'LAST_3_DAYS',
'last7Days': 'LAST_7_DAYS',
'last180Days': 'LAST_180_DAYS',
'last12Weeks': 'LAST_12_WEEKS',
'last4Quarters': 'LAST_4_QUARTERS',
'weeksThisYear': 'WEEKS_THIS_YEAR',
'yesterday': 'YESTERDAY',
'quartersLastYear': 'QUARTERS_LAST_YEAR',
'monthsThisYear': 'MONTHS_THIS_YEAR',
'biMonthsThisYear': 'BI_MONTHS_THIS_YEAR',
'last5FinancialYears': 'LAST_5_FINANCIAL_YEARS',
'thisSixMonth': 'THIS_SIX_MONTH',
'thisFinancialYear': 'THIS_FINANCIAL_YEAR',
'last6BiMonths': 'LAST_6_BI_MONTHS',
'last4BiWeeks': 'LAST_6_BI_WEEKS',
'lastFinancialYear': 'LAST_FINANCIAL_YEAR',
'lastBiWeek': 'LAST_BI_WEEK',
'quartersThisYear': 'QUARTERS_THIS_YEAR',
'monthsLastYear': 'MONTHS_LAST_YEAR',
'thisBimonth': 'THIS_BI_MONTH',
'lastBimonth': 'LAST_BI_MONTH',
'lastSixMonth': 'LAST_SIX_MONTH',
'thisBiWeek': 'THIS_BI_WEEK',
'last10Years': 'LAST_10_YEARS',
'last10FinancialYears': 'LAST_10_FINANCIAL_YEARS'
}
for relative_period in json_object['relativePeriods']:
if relative_period in pe_global_selections:
if json_object['relativePeriods'][relative_period]:
dimensions['pe'].append(
pe_global_selections[relative_period])
else:
logger.error("Unknown relativePeriod " + relative_period)
exit(1)
if len(dimensions['pe']) == 0 and 'periods' in json_object:
dimensions['pe'] = json_extract(json_object['periods'], 'id')
else:
return {}
else:
dimensions['pe'] = json_extract(json_object['periods'], 'id')
if len(dimensions['pe']
) == 0 and 'startDate' in json_object and 'endDate' in json_object:
del dimensions['pe']
if 'dataDimensionItems' in json_object and len(
json_object['dataDimensionItems']) > 0:
data_dimension_keys = {
'PROGRAM_INDICATOR': 'programIndicator',
'INDICATOR': 'indicator',
'DATA_ELEMENT': 'dataElement',
'REPORTING_RATE': 'reportingRate',
'PROGRAM_DATA_ELEMENT': 'programDataElement'
}
for data_dimension in json_object['dataDimensionItems']:
# Sometimes there are empty dimensions
if data_dimension != {}:
if data_dimension[
'dataDimensionItemType'] in data_dimension_keys:
# Special case, it joins to UIDs with a .
if data_dimension[
'dataDimensionItemType'] == 'PROGRAM_DATA_ELEMENT':
UID1 = data_dimension[data_dimension_keys[
data_dimension['dataDimensionItemType']]][
'program']['id']
UID2 = data_dimension[data_dimension_keys[
data_dimension['dataDimensionItemType']]][
'dataElement']['id']
dimensions['dx'].append(UID1 + '.' + UID2)
else:
data_dimension_uid = data_dimension[
data_dimension_keys[
data_dimension['dataDimensionItemType']]]['id']
if data_dimension[
'dataDimensionItemType'] == 'REPORTING_RATE':
# For reporting rates, we need to add the keyword to the id
dimensions['dx'].append(data_dimension_uid +
'.REPORTING_RATE')
else:
dimensions['dx'].append(data_dimension_uid)
else:
logger.error('Unrecognized data dimension type ' +
data_dimension['dataDimensionItemType'])
exit(1)
dataElementDimension_filters = dict()
if 'dataElementDimensions' in json_object and len(
json_object['dataElementDimensions']) > 0:
for data_element_dimension in json_object['dataElementDimensions']:
data_element_dimension_uid = data_element_dimension['dataElement'][
'id']
if 'filter' in data_element_dimension:
dimensions[
data_element_dimension_uid] = data_element_dimension[
'filter']
else:
dimensions[data_element_dimension_uid] = ""
if 'categoryOptionGroupSetDimensions' in json_object and len(
json_object['categoryOptionGroupSetDimensions']) > 0:
get_group_set_dimensions(
json_object['categoryOptionGroupSetDimensions'],
'categoryOptionGroup')
if 'organisationUnitGroupSetDimensions' in json_object and len(
json_object['organisationUnitGroupSetDimensions']) > 0:
get_group_set_dimensions(
json_object['organisationUnitGroupSetDimensions'],
'organisationUnitGroup')
if 'dataElementGroupSetDimensions' in json_object and len(
json_object['dataElementGroupSetDimensions']) > 0:
get_group_set_dimensions(json_object['dataElementGroupSetDimensions'],
'dataElementGroup')
if 'categoryDimensions' in json_object and len(
json_object['categoryDimensions']) > 0:
for category_dimension in json_object['categoryDimensions']:
category_dimension_uid = category_dimension['category']['id']
category_options = list()
for cat_options in category_dimension['categoryOptions']:
category_options.append(cat_options['id'])
dimensions[category_dimension_uid] = category_options
# Build the payload
payload = ""
params = dict()
payload += 'dimension='
params['dimension'] = ""
added_column_dimension = False
if 'columns' in json_object:
first_element = True
for column in json_object['columns']:
added_column_dimension = True
if not first_element:
payload += ','
params['dimension'] += ','
else:
first_element = False
key = column['id']
if key in dimensions:
if isinstance(dimensions[key], list):
right_expression = ';'.join(dimensions[key])
else:
right_expression = dimensions[key]
payload += key + ':' + right_expression
params['dimension'] += key + ':' + right_expression
else:
if key == 'pe' and 'startDate' in json_object and 'endDate' in json_object:
payload += "&startDate=" + json_object[
'startDate'] + "&endDate=" + json_object['endDate']
params['startDate'] = json_object['startDate']
params['endDate'] = json_object['endDate']
else:
logger.error(json_object['id'] + ': Dimension ' + key +
' is missing')
# exit(1)
else:
logger.error('columns missing')
exit(1)
# A very specific and strange case for maps
# empty columns but styleDataItem is present. In that case, it gets added to the dimension
if 'columns' in json_object and len(
json_object['columns']) == 0 and 'styleDataItem' in json_object:
payload += json_object['styleDataItem']['id']
params['dimension'] += json_object['styleDataItem']['id']
if 'rows' in json_object and len(json_object['rows']) > 0:
payload += ','
params['dimension'] += ','
if 'rows' in json_object:
if len(json_object['rows']) > 0:
# If we have already added some stuff, separate it with a comma
if added_column_dimension:
payload += ','
params['dimension'] += ','
first_element = True
for row in json_object['rows']:
if not first_element:
payload += ','
params['dimension'] += ','
else:
first_element = False
key = row['id']
if key in dimensions:
payload += key + ':' + ';'.join(dimensions[key])
params['dimension'] += key + ':' + ';'.join(
dimensions[key])
else:
if key == 'pe' and 'startDate' in json_object and 'endDate' in json_object:
payload += "&startDate=" + json_object[
'startDate'] + "&endDate=" + json_object['endDate']
params['startDate'] = json_object['startDate']
params['endDate'] = json_object['endDate']
else:
logger.error(json_object['id'] + ': Dimension ' + key +
' is missing')
# exit(1)
else:
logger.error('rows missing')
exit(1)
if 'filters' in json_object:
if len(json_object['filters']) > 0:
payload += '&filter='
params['filter'] = ""
first_element = True
for filter in json_object['filters']:
if not first_element:
payload += ','
params['filter'] += ','
else:
first_element = False
key = filter['id']
if key in dimensions:
payload += key + ':' + ';'.join(dimensions[key])
params['filter'] += key + ':' + ';'.join(dimensions[key])
else:
if key == 'pe' and 'startDate' in json_object and 'endDate' in json_object:
payload += "&startDate=" + json_object[
'startDate'] + "&endDate=" + json_object['endDate']
params['startDate'] = json_object['startDate']
params['endDate'] = json_object['endDate']
else:
logger.error(json_object['id'] + ': Dimension ' + key +
' is missing')
# exit(1)
else:
logger.error('filters missing')
exit(1)
if 'programStage' in json_object:
payload += '&stage' + ':' + json_object['programStage']['id']
params['stage'] = json_object['programStage']['id']
# Important, to get the data
payload += "&skipData=false"
params['skipData'] = 'false'
if verbose:
logger.info(payload)
return params