def create_api(server=None, username=None, password=None, api_version=None):
"""Return a fully configured dhis2.Dhis instance"""
if not any([server, username, password]):
api = Api.from_auth_file(api_version=api_version,
user_agent='dhis2-pk/{}'.format(__version__))
logger.info("Found a file for server {}".format(api.base_url))
return api
else:
return Api(server, username, password, api_version,
'dhis2-pk/{}'.format(__version__))
def main():
import argparse
global api_source
my_parser = argparse.ArgumentParser(
prog='dummy_data_agg',
description='Create dummy data for aggregated datasets',
epilog="example1"
"\nexample2",
formatter_class=argparse.RawDescriptionHelpFormatter)
my_parser.add_argument(
'Dataset',
metavar='dataset_param',
type=str,
help='the uid of the dataset to use or a string to filter datasets')
my_parser.add_argument(
'-sd',
'--start_date',
action="store",
dest="start_date",
type=str,
help=
'start date for the period to use to generate data (default is today - 1 year)'
)
my_parser.add_argument(
'-ptf',
'--period_type_filter',
action="store",
dest="period_type_filter",
type=str,
help='only applicable when having multiple datasets: d, w, m, y')
my_parser.add_argument(
'-ed',
'--end_date',
action="store",
dest="end_date",
type=str,
help=
'end date for the period to use to generate data (default is today)')
my_parser.add_argument(
'-ous',
'--org_unit_selection',
action="store",
metavar=('type', 'value'),
nargs=2,
help=
'Provide a type of org unit selection from [uid,uid_children,name,code,level] and the value to use'
'Eg: --ous uid QXtjg5dh34A')
# Parameters should be 0 or 1
my_parser.add_argument('-cf',
'--create_flat_file',
action="store",
metavar='file_name',
const='xxx',
nargs='?',
help='Create spreadsheet for min/max values'
'Eg: --create_flat_file=my_file.csv')
my_parser.add_argument('-uf',
'--use_flat_file',
action="store",
metavar='file_name',
nargs=1,
help='Use spreadsheet for min/max values'
'Eg: --use_flat_file=my_file.csv')
my_parser.add_argument(
'-i',
'--instance',
action="store",
dest="instance",
type=str,
help=
'instance to use for dummy data injection (robot account is required!) - default is the URL in auth.json'
)
my_parser.add_argument(
'-ours',
'--ous_random_size',
action="store",
dest="ous_random_size",
type=str,
help=
'From all OUs selected from ous command, takes a random sample of ous_random_size'
)
args = my_parser.parse_args()
credentials_file = 'auth.json'
try:
f = open(credentials_file)
except IOError:
print(
"Please provide file auth.json with credentials for DHIS2 server")
exit(1)
else:
with open(credentials_file, 'r') as json_file:
credentials = json.load(json_file)
if args.instance is not None:
api_source = Api(args.instance, credentials['dhis']['username'],
credentials['dhis']['password'])
else:
api_source = Api.from_auth_file(credentials_file)
logger.warning("Server source running DHIS2 version {} revision {}".format(
api_source.version, api_source.revision))
#WHAT
dsParam = args.Dataset
# WHERE
ouUIDs = list()
#WHEN
start_date = ""
end_date = ""
periods = list()
# Assign values from parameters provided if applicable
if args.create_flat_file is None: # If we are creating a flat file it does not matter if not provided
if args.org_unit_selection is None:
print(
'Please provide a value for org_unit_selection to create the dummy data'
)
else:
if len(args.org_unit_selection) >= 1:
ouUIDs = get_org_units(args.org_unit_selection[0],
args.org_unit_selection[1],
int(args.ous_random_size))
if len(ouUIDs) == 0:
print('The OU selection ' + args.org_unit_selection[0] +
' ' + args.org_unit_selection[1] +
' returned no result')
exit(1)
if args.start_date is None:
start_date = (date.today() -
timedelta(days=365)).strftime("%Y-%m-%d")
else:
start_date = args.start_date
if not isDateFormat(start_date):
print('Start date provided ' + start_date +
' has a wrong format')
exit(1)
if args.end_date is None:
end_date = (date.today()).strftime("%Y-%m-%d")
else:
end_date = args.end_date
if not isDateFormat(end_date):
print('End date provided ' + end_date + ' has a wrong format')
exit(1)
periods = list()
if args.create_flat_file is not None:
df_min_max = pd.DataFrame({},
columns=[
'DE UID', 'COC UID', 'DE Name',
'COC Name', 'valueType', 'min', 'max'
])
else:
df_min_max = None
if args.use_flat_file is not None:
filename = args.use_flat_file
logger.info("Reading " + filename + " for min/max value")
df_min_max = pd.read_csv(filename, sep=None, engine='python')
CC = api_source.get('categoryCombos',
params={
"paging": "false",
"fields": "id,name,categoryOptionCombos"
}).json()['categoryCombos']
CC = reindex(CC, 'id')
defaultCC = ''
for catcomboUID in CC:
if CC[catcomboUID]['name'] == 'default':
defaultCC = catcomboUID
break
if defaultCC == '':
logger.warning('Could not find default Category Combo')
COC = api_source.get('categoryOptionCombos',
params={
"paging": "false",
"fields": "id,name"
}).json()['categoryOptionCombos']
COC = reindex(COC, 'id')
DE = api_source.get(
'dataElements',
params={
"paging": "false",
"fields":
"id,name,categoryCombo,aggregationType,valueType,optionSet"
}).json()['dataElements']
DE = reindex(DE, 'id')
# Check for optionSets in the DE
optionSetUIDs = list()
for de in DE:
if 'optionSet' in de:
optionSetUIDs.append(de['optionSet']['id'])
if len(optionSetUIDs) > 0:
options = api_source.get('options',
params={
"paging":
"false",
"fields":
"id,name,code",
"filter":
"optionSet.id:eq:" +
','.join(optionSetUIDs)
}).json()['options']
de_numeric_types = [
'INTEGER_POSITIVE', 'INTEGER', 'INTEGER_ZERO_OR_POSITIVE', 'NUMBER',
'PERCENTAGE', 'INTEGER_ZERO_OR_NEGATIVE'
]
# Get the datasets"
if is_valid_uid(dsParam):
dataset_filter = "id:eq:" + dsParam
else:
dataset_filter = "name:like:" + dsParam
dataSets = api_source.get(
'dataSets',
params={
"paging": "false",
"fields": "id,name,dataSetElements,periodType,"
"formType,dataEntryForm,sections,organisationUnits",
"filter": dataset_filter
}).json()['dataSets']
# Only one dataSet
if len(dataSets) == 0:
logger.error("Could not find any dataset")
exit(1)
else:
if len(dataSets) > 1 and args.period_type_filter is not None:
periodTypeFilter = args.period_type_filter
if periodTypeFilter.lower() not in [
'daily', 'weekly', 'monthly', 'quarterly', 'yearly'
]:
logger.error('Period type to filter not supported:' +
periodTypeFilter)
else:
filteredDatasets = list()
for ds in dataSets:
if ds['periodType'].lower() == periodTypeFilter.lower():
filteredDatasets.append(ds)
dataSets = filteredDatasets
# Create workbook
if args.create_flat_file is not None:
ouput_file_name = 'datasets_' + dsParam + '.xlsx'
ouput_file_name = args.create_flat_file + '.xlsx'
writer = pd.ExcelWriter(ouput_file_name)
for ds in dataSets:
logger.info("Processing dataset " + ds['name'])
if start_date != "" and end_date != "":
logger.info("Period type is " + ds['periodType'] +
" - Generating periods from " + start_date +
" to " + end_date)
periods = get_periods(ds['periodType'], start_date, end_date)
if len(ouUIDs) > 0:
logger.info("Verifying org unit selection")
for ou_uid in ouUIDs:
if not is_ou_assigned_to_ds(ou_uid, ds):
ouUIDs.remove(ou_uid)
logger.warning("Org unit " + ou_uid +
" is not assigned to dataset " +
ds['id'])
dsDataElements = dict()
greyedFields = list()
# Analyse the sections of the dataSet looking for greyedFields
if 'sections' in ds:
sectionUIDs = ""
for section in ds['sections']:
sectionUIDs += (section['id'] + ",")
logger.info("Found " + str(sectionUIDs.count(',')) +
" sections in dataset")
# Get sections
sections = api_source.get(
'sections',
params={
"paging": "false",
"fields":
"id,name,greyedFields[dataElement,categoryOptionCombo]",
"filter": "id:in:[" + sectionUIDs + "]"
}).json()['sections']
for section in sections:
if len(section['greyedFields']) > 0:
for element in section['greyedFields']:
greyedFields.append(
element['dataElement']['id'] + '.' +
element['categoryOptionCombo']['id'])
# Get dataElements
for DSE in ds['dataSetElements']:
df_min_max = pd.DataFrame({},
columns=[
'DE UID', 'COC UID', 'DE Name',
'COC Name', 'valueType', 'min',
'max'
])
de = ''
if 'dataElement' in DSE:
deUID = DSE['dataElement']['id']
dsDataElements[deUID] = dict()
de = DE[deUID] # Get all dataElement information
dsDataElements[deUID]['valueType'] = de['valueType']
# Add options to the dataelement dict if pertinent
if 'optionSet' in de:
options = api_source.get('options',
params={
"paging":
"false",
"fields":
"id,name,code",
"filter":
"optionSet.id:eq:" +
de['optionSet']['id']
}).json()['options']
dsDataElements[deUID]['options'] = list()
for option in options:
dsDataElements[deUID]['options'].append(
option['code'])
# Check if the Category Combo is specified in the dataElement definition
COCs = list()
if 'categoryCombo' in de and de['categoryCombo'][
'id'] != defaultCC:
COCs = CC[de['categoryCombo']
['id']]['categoryOptionCombos']
# Check if Category Combo is specified for the dataElement in the dataSet
elif 'categoryCombo' in DSE and DSE['categoryCombo'][
'id'] != defaultCC:
COCs = CC[DSE['categoryCombo']
['id']]['categoryOptionCombos']
# Add COCs to the dataElement dictionary
if len(COCs) > 0:
dsDataElements[deUID]['COCs'] = list()
for coc in COCs:
dsDataElements[deUID]['COCs'].append(coc['id'])
logger.info("Found " + str(len(dsDataElements)) +
" dataElements in dataset")
if args.create_flat_file is not None:
for de in dsDataElements:
if 'COCs' in dsDataElements[de]:
for coc in dsDataElements[de]['COCs']:
str_pair = de + "." + coc
if str_pair not in greyedFields:
df_min_max = df_min_max.append(
{
"DE UID":
de,
"COC UID":
coc,
"DE Name":
DE[de]['name'],
"COC Name":
COC[coc]['name'],
"valueType":
dsDataElements[de]['valueType'],
"min":
"",
"max":
""
},
ignore_index=True)
else:
df_min_max = df_min_max.append(
{
"DE UID": de,
"COC UID": "",
"DE Name": DE[de]['name'],
"COC Name": "",
"valueType": dsDataElements[de]['valueType'],
"min": "",
"max": ""
},
ignore_index=True)
# Save csv file
# export_csv = df_min_max.to_csv(r'./ds_' + ds['name'].replace(' ', '_') + '_min_max.csv', index=None,
# header=True)
df_min_max.to_excel(writer, ds['id'], index=False)
else:
dataValueSets = list()
ouCount = 1
for ouUID in ouUIDs:
logger.info("Processing org unit " + ouUID + " - " +
str(ouCount) + "/" + str(len(ouUIDs)))
for period in periods:
#logger.info("Processing period " + period)
for de in dsDataElements:
value_type = dsDataElements[de]['valueType']
min_value = max_value = None
options = None
if 'options' in dsDataElements[de]:
options = dsDataElements[de]['options']
if 'COCs' in dsDataElements[de]:
for coc in dsDataElements[de]['COCs']:
str_pair = de + "." + coc
if str_pair not in greyedFields:
if df_min_max is not None:
min_value, max_value = get_min_max_from_df(
df_min_max, value_type, de,
coc)
# logger.info(
# "Generating value for DE (" + value_type + "): " + DE[de]['name'] + " with COC")
value = generate_dummy_value({
'value_type':
value_type,
'min_value':
min_value,
'max_value':
max_value,
'options':
options
})
if value is not None: # Skip if it is None
dataValueSets.append({
"dataElement":
de,
"categoryOptionCombo":
coc,
"value":
value,
"orgUnit":
ouUID,
"period":
period
})
# else:
# logger.warning('Skipping ' + str_pair + ' because is greyed in section')
else:
if df_min_max is not None:
min_value, max_value = get_min_max_from_df(
df_min_max, value_type, de)
# logger.info("Generating value for DE (" + value_type + "): " + DE[de]['name'])
value = generate_dummy_value({
'value_type': value_type,
'min_value': min_value,
'max_value': max_value,
'options': options
})
if value is not None: # Skip if it is None
dataValueSets.append({
"dataElement": de,
"value": value,
"orgUnit": ouUID,
"period": period
})
post_to_server({'dataValues': dataValueSets},
'dataValueSets')
dataValueSets = list()
ouCount += 1
if args.create_flat_file is not None:
writer.save()
import pandas as pd
from tools.json import reindex, json_extract, json_extract_nested_ids
import gspread
from oauth2client.service_account import ServiceAccountCredentials
from gspread_dataframe import get_as_dataframe, set_with_dataframe
from gspread_formatting import *
import re
try:
f = open("./auth.json")
except IOError:
print("Please provide file auth.json with credentials for DHIS2 server")
exit(1)
else:
api_source = Api.from_auth_file('./auth.json')
# If no file path is specified, it tries to find a file called dish.json in:
#
# the DHIS_HOME environment variable
# your Home folder
# setup the logger
log_file = "./dummyDataTracker.log"
logzero.logfile(log_file)
def add_repeatable_stages(df, stage_counter):
if df['Stage'].isna().sum() > 0:
stage_indexes = df.index[df['Stage'].notnull()].tolist()
import re
from os import path
import Levenshtein as lev
import pandas as pd
from bs4 import BeautifulSoup
from dhis2 import Api, RequestException, setup_logger, logger, generate_uid, \
is_valid_uid # make sure you have dhis2.py installed, otherwise run "pip3 install dhis2.py"
try:
f = open("./auth.json")
except IOError:
print("Please provide file auth.json with credentials for DHIS2 server")
exit(1)
else:
api = Api.from_auth_file('./auth.json')
# "dhis2.util.on( 'dhis2.de.event.formReady', function( event, ds ) {\n" \
# "} );\n" \
js_code = "" \
"console.log('Applying translations');\n" \
" $(function() {\n" \
" $.ajax({\n" \
" type: 'GET',\n" \
" url: '../api/me.json',\n" \
" success: function(data){\n" \
" if('settings' in data) {\n" \
" var locale = data.settings.keyDbLocale;\n" \
" console.log('DB Locale: ' + locale);\n" \
" }\n" \
" else {\n" \
credentials_file = 'auth.json'
try:
f = open(credentials_file)
except IOError:
print("Please provide file auth.json with credentials for DHIS2 server")
exit(1)
else:
with open(credentials_file, 'r') as json_file:
credentials = json.load(json_file)
if instance is not None:
api = Api(instance, credentials['dhis']['username'],
credentials['dhis']['password'])
else:
api = Api.from_auth_file(credentials_file)
de_uid = generate_uid()
dummy_data_de = {
"id": de_uid,
"name": "Dummy data placeholder",
"shortName": "Dummy data placeholder",
"aggregationType": "NONE",
"domainType": "AGGREGATE",
"publicAccess": "--------",
"externalAccess": False,
"valueType": "NUMBER",
"zeroIsSignificant": False,
"favorite": False,
"optionSetValue": False,
