def create_api(server=None, username=None, password=None, api_version=None): """Return a fully configured dhis2.Dhis instance""" if not any([server, username, password]): api = Api.from_auth_file(api_version=api_version, user_agent='dhis2-pk/{}'.format(__version__)) logger.info("Found a file for server {}".format(api.base_url)) return api else: return Api(server, username, password, api_version, 'dhis2-pk/{}'.format(__version__))
def main(): import argparse global api_source my_parser = argparse.ArgumentParser( prog='dummy_data_agg', description='Create dummy data for aggregated datasets', epilog="example1" "\nexample2", formatter_class=argparse.RawDescriptionHelpFormatter) my_parser.add_argument( 'Dataset', metavar='dataset_param', type=str, help='the uid of the dataset to use or a string to filter datasets') my_parser.add_argument( '-sd', '--start_date', action="store", dest="start_date", type=str, help= 'start date for the period to use to generate data (default is today - 1 year)' ) my_parser.add_argument( '-ptf', '--period_type_filter', action="store", dest="period_type_filter", type=str, help='only applicable when having multiple datasets: d, w, m, y') my_parser.add_argument( '-ed', '--end_date', action="store", dest="end_date", type=str, help= 'end date for the period to use to generate data (default is today)') my_parser.add_argument( '-ous', '--org_unit_selection', action="store", metavar=('type', 'value'), nargs=2, help= 'Provide a type of org unit selection from [uid,uid_children,name,code,level] and the value to use' 'Eg: --ous uid QXtjg5dh34A') # Parameters should be 0 or 1 my_parser.add_argument('-cf', '--create_flat_file', action="store", metavar='file_name', const='xxx', nargs='?', help='Create spreadsheet for min/max values' 'Eg: --create_flat_file=my_file.csv') my_parser.add_argument('-uf', '--use_flat_file', action="store", metavar='file_name', nargs=1, help='Use spreadsheet for min/max values' 'Eg: --use_flat_file=my_file.csv') my_parser.add_argument( '-i', '--instance', action="store", dest="instance", type=str, help= 'instance to use for dummy data injection (robot account is required!) - default is the URL in auth.json' ) my_parser.add_argument( '-ours', '--ous_random_size', action="store", dest="ous_random_size", type=str, help= 'From all OUs selected from ous command, takes a random sample of ous_random_size' ) args = my_parser.parse_args() credentials_file = 'auth.json' try: f = open(credentials_file) except IOError: print( "Please provide file auth.json with credentials for DHIS2 server") exit(1) else: with open(credentials_file, 'r') as json_file: credentials = json.load(json_file) if args.instance is not None: api_source = Api(args.instance, credentials['dhis']['username'], credentials['dhis']['password']) else: api_source = Api.from_auth_file(credentials_file) logger.warning("Server source running DHIS2 version {} revision {}".format( api_source.version, api_source.revision)) #WHAT dsParam = args.Dataset # WHERE ouUIDs = list() #WHEN start_date = "" end_date = "" periods = list() # Assign values from parameters provided if applicable if args.create_flat_file is None: # If we are creating a flat file it does not matter if not provided if args.org_unit_selection is None: print( 'Please provide a value for org_unit_selection to create the dummy data' ) else: if len(args.org_unit_selection) >= 1: ouUIDs = get_org_units(args.org_unit_selection[0], args.org_unit_selection[1], int(args.ous_random_size)) if len(ouUIDs) == 0: print('The OU selection ' + args.org_unit_selection[0] + ' ' + args.org_unit_selection[1] + ' returned no result') exit(1) if args.start_date is None: start_date = (date.today() - timedelta(days=365)).strftime("%Y-%m-%d") else: start_date = args.start_date if not isDateFormat(start_date): print('Start date provided ' + start_date + ' has a wrong format') exit(1) if args.end_date is None: end_date = (date.today()).strftime("%Y-%m-%d") else: end_date = args.end_date if not isDateFormat(end_date): print('End date provided ' + end_date + ' has a wrong format') exit(1) periods = list() if args.create_flat_file is not None: df_min_max = pd.DataFrame({}, columns=[ 'DE UID', 'COC UID', 'DE Name', 'COC Name', 'valueType', 'min', 'max' ]) else: df_min_max = None if args.use_flat_file is not None: filename = args.use_flat_file logger.info("Reading " + filename + " for min/max value") df_min_max = pd.read_csv(filename, sep=None, engine='python') CC = api_source.get('categoryCombos', params={ "paging": "false", "fields": "id,name,categoryOptionCombos" }).json()['categoryCombos'] CC = reindex(CC, 'id') defaultCC = '' for catcomboUID in CC: if CC[catcomboUID]['name'] == 'default': defaultCC = catcomboUID break if defaultCC == '': logger.warning('Could not find default Category Combo') COC = api_source.get('categoryOptionCombos', params={ "paging": "false", "fields": "id,name" }).json()['categoryOptionCombos'] COC = reindex(COC, 'id') DE = api_source.get( 'dataElements', params={ "paging": "false", "fields": "id,name,categoryCombo,aggregationType,valueType,optionSet" }).json()['dataElements'] DE = reindex(DE, 'id') # Check for optionSets in the DE optionSetUIDs = list() for de in DE: if 'optionSet' in de: optionSetUIDs.append(de['optionSet']['id']) if len(optionSetUIDs) > 0: options = api_source.get('options', params={ "paging": "false", "fields": "id,name,code", "filter": "optionSet.id:eq:" + ','.join(optionSetUIDs) }).json()['options'] de_numeric_types = [ 'INTEGER_POSITIVE', 'INTEGER', 'INTEGER_ZERO_OR_POSITIVE', 'NUMBER', 'PERCENTAGE', 'INTEGER_ZERO_OR_NEGATIVE' ] # Get the datasets" if is_valid_uid(dsParam): dataset_filter = "id:eq:" + dsParam else: dataset_filter = "name:like:" + dsParam dataSets = api_source.get( 'dataSets', params={ "paging": "false", "fields": "id,name,dataSetElements,periodType," "formType,dataEntryForm,sections,organisationUnits", "filter": dataset_filter }).json()['dataSets'] # Only one dataSet if len(dataSets) == 0: logger.error("Could not find any dataset") exit(1) else: if len(dataSets) > 1 and args.period_type_filter is not None: periodTypeFilter = args.period_type_filter if periodTypeFilter.lower() not in [ 'daily', 'weekly', 'monthly', 'quarterly', 'yearly' ]: logger.error('Period type to filter not supported:' + periodTypeFilter) else: filteredDatasets = list() for ds in dataSets: if ds['periodType'].lower() == periodTypeFilter.lower(): filteredDatasets.append(ds) dataSets = filteredDatasets # Create workbook if args.create_flat_file is not None: ouput_file_name = 'datasets_' + dsParam + '.xlsx' ouput_file_name = args.create_flat_file + '.xlsx' writer = pd.ExcelWriter(ouput_file_name) for ds in dataSets: logger.info("Processing dataset " + ds['name']) if start_date != "" and end_date != "": logger.info("Period type is " + ds['periodType'] + " - Generating periods from " + start_date + " to " + end_date) periods = get_periods(ds['periodType'], start_date, end_date) if len(ouUIDs) > 0: logger.info("Verifying org unit selection") for ou_uid in ouUIDs: if not is_ou_assigned_to_ds(ou_uid, ds): ouUIDs.remove(ou_uid) logger.warning("Org unit " + ou_uid + " is not assigned to dataset " + ds['id']) dsDataElements = dict() greyedFields = list() # Analyse the sections of the dataSet looking for greyedFields if 'sections' in ds: sectionUIDs = "" for section in ds['sections']: sectionUIDs += (section['id'] + ",") logger.info("Found " + str(sectionUIDs.count(',')) + " sections in dataset") # Get sections sections = api_source.get( 'sections', params={ "paging": "false", "fields": "id,name,greyedFields[dataElement,categoryOptionCombo]", "filter": "id:in:[" + sectionUIDs + "]" }).json()['sections'] for section in sections: if len(section['greyedFields']) > 0: for element in section['greyedFields']: greyedFields.append( element['dataElement']['id'] + '.' + element['categoryOptionCombo']['id']) # Get dataElements for DSE in ds['dataSetElements']: df_min_max = pd.DataFrame({}, columns=[ 'DE UID', 'COC UID', 'DE Name', 'COC Name', 'valueType', 'min', 'max' ]) de = '' if 'dataElement' in DSE: deUID = DSE['dataElement']['id'] dsDataElements[deUID] = dict() de = DE[deUID] # Get all dataElement information dsDataElements[deUID]['valueType'] = de['valueType'] # Add options to the dataelement dict if pertinent if 'optionSet' in de: options = api_source.get('options', params={ "paging": "false", "fields": "id,name,code", "filter": "optionSet.id:eq:" + de['optionSet']['id'] }).json()['options'] dsDataElements[deUID]['options'] = list() for option in options: dsDataElements[deUID]['options'].append( option['code']) # Check if the Category Combo is specified in the dataElement definition COCs = list() if 'categoryCombo' in de and de['categoryCombo'][ 'id'] != defaultCC: COCs = CC[de['categoryCombo'] ['id']]['categoryOptionCombos'] # Check if Category Combo is specified for the dataElement in the dataSet elif 'categoryCombo' in DSE and DSE['categoryCombo'][ 'id'] != defaultCC: COCs = CC[DSE['categoryCombo'] ['id']]['categoryOptionCombos'] # Add COCs to the dataElement dictionary if len(COCs) > 0: dsDataElements[deUID]['COCs'] = list() for coc in COCs: dsDataElements[deUID]['COCs'].append(coc['id']) logger.info("Found " + str(len(dsDataElements)) + " dataElements in dataset") if args.create_flat_file is not None: for de in dsDataElements: if 'COCs' in dsDataElements[de]: for coc in dsDataElements[de]['COCs']: str_pair = de + "." + coc if str_pair not in greyedFields: df_min_max = df_min_max.append( { "DE UID": de, "COC UID": coc, "DE Name": DE[de]['name'], "COC Name": COC[coc]['name'], "valueType": dsDataElements[de]['valueType'], "min": "", "max": "" }, ignore_index=True) else: df_min_max = df_min_max.append( { "DE UID": de, "COC UID": "", "DE Name": DE[de]['name'], "COC Name": "", "valueType": dsDataElements[de]['valueType'], "min": "", "max": "" }, ignore_index=True) # Save csv file # export_csv = df_min_max.to_csv(r'./ds_' + ds['name'].replace(' ', '_') + '_min_max.csv', index=None, # header=True) df_min_max.to_excel(writer, ds['id'], index=False) else: dataValueSets = list() ouCount = 1 for ouUID in ouUIDs: logger.info("Processing org unit " + ouUID + " - " + str(ouCount) + "/" + str(len(ouUIDs))) for period in periods: #logger.info("Processing period " + period) for de in dsDataElements: value_type = dsDataElements[de]['valueType'] min_value = max_value = None options = None if 'options' in dsDataElements[de]: options = dsDataElements[de]['options'] if 'COCs' in dsDataElements[de]: for coc in dsDataElements[de]['COCs']: str_pair = de + "." + coc if str_pair not in greyedFields: if df_min_max is not None: min_value, max_value = get_min_max_from_df( df_min_max, value_type, de, coc) # logger.info( # "Generating value for DE (" + value_type + "): " + DE[de]['name'] + " with COC") value = generate_dummy_value({ 'value_type': value_type, 'min_value': min_value, 'max_value': max_value, 'options': options }) if value is not None: # Skip if it is None dataValueSets.append({ "dataElement": de, "categoryOptionCombo": coc, "value": value, "orgUnit": ouUID, "period": period }) # else: # logger.warning('Skipping ' + str_pair + ' because is greyed in section') else: if df_min_max is not None: min_value, max_value = get_min_max_from_df( df_min_max, value_type, de) # logger.info("Generating value for DE (" + value_type + "): " + DE[de]['name']) value = generate_dummy_value({ 'value_type': value_type, 'min_value': min_value, 'max_value': max_value, 'options': options }) if value is not None: # Skip if it is None dataValueSets.append({ "dataElement": de, "value": value, "orgUnit": ouUID, "period": period }) post_to_server({'dataValues': dataValueSets}, 'dataValueSets') dataValueSets = list() ouCount += 1 if args.create_flat_file is not None: writer.save()
import pandas as pd from tools.json import reindex, json_extract, json_extract_nested_ids import gspread from oauth2client.service_account import ServiceAccountCredentials from gspread_dataframe import get_as_dataframe, set_with_dataframe from gspread_formatting import * import re try: f = open("./auth.json") except IOError: print("Please provide file auth.json with credentials for DHIS2 server") exit(1) else: api_source = Api.from_auth_file('./auth.json') # If no file path is specified, it tries to find a file called dish.json in: # # the DHIS_HOME environment variable # your Home folder # setup the logger log_file = "./dummyDataTracker.log" logzero.logfile(log_file) def add_repeatable_stages(df, stage_counter): if df['Stage'].isna().sum() > 0: stage_indexes = df.index[df['Stage'].notnull()].tolist()
import re from os import path import Levenshtein as lev import pandas as pd from bs4 import BeautifulSoup from dhis2 import Api, RequestException, setup_logger, logger, generate_uid, \ is_valid_uid # make sure you have dhis2.py installed, otherwise run "pip3 install dhis2.py" try: f = open("./auth.json") except IOError: print("Please provide file auth.json with credentials for DHIS2 server") exit(1) else: api = Api.from_auth_file('./auth.json') # "dhis2.util.on( 'dhis2.de.event.formReady', function( event, ds ) {\n" \ # "} );\n" \ js_code = "" \ "console.log('Applying translations');\n" \ " $(function() {\n" \ " $.ajax({\n" \ " type: 'GET',\n" \ " url: '../api/me.json',\n" \ " success: function(data){\n" \ " if('settings' in data) {\n" \ " var locale = data.settings.keyDbLocale;\n" \ " console.log('DB Locale: ' + locale);\n" \ " }\n" \ " else {\n" \
credentials_file = 'auth.json' try: f = open(credentials_file) except IOError: print("Please provide file auth.json with credentials for DHIS2 server") exit(1) else: with open(credentials_file, 'r') as json_file: credentials = json.load(json_file) if instance is not None: api = Api(instance, credentials['dhis']['username'], credentials['dhis']['password']) else: api = Api.from_auth_file(credentials_file) de_uid = generate_uid() dummy_data_de = { "id": de_uid, "name": "Dummy data placeholder", "shortName": "Dummy data placeholder", "aggregationType": "NONE", "domainType": "AGGREGATE", "publicAccess": "--------", "externalAccess": False, "valueType": "NUMBER", "zeroIsSignificant": False, "favorite": False, "optionSetValue": False,