short_unit=s_unit, description=global_cat[indicator_code]['description'], code=indicator_code, timespan='', datasetId=newdataset, variableTypeId=VariableType.objects.get(pk=4), sourceId=newsource) newvariable.save() global_cat[indicator_code]['variable_object'] = newvariable vars_to_add.remove(indicator_code) global_cat[indicator_code]['saved'] = True logger.info("Inserting a variable %s." % newvariable.name.encode('utf8')) else: if not global_cat[indicator_code]['saved']: newsource = Source.objects.get(name='World Bank Poverty and Equity database: ' + Variable.objects.get(code=indicator_code, datasetId__in=Dataset.objects.filter(namespace='povstats')).name) newsource.name = 'World Bank Poverty and Equity database: ' + global_cat[indicator_code]['name'] source_description['additionalInfo'] = "Definitions and characteristics of countries and other territories: " + "https://ourworldindata.org" + reverse("servepovstatscountryinfo") + "\n" source_description['additionalInfo'] += "Limitations and exceptions:\n" + global_cat[indicator_code]['limitations'] + "\n" if global_cat[indicator_code]['limitations'] else '' source_description['additionalInfo'] += "Notes from original source:\n" + global_cat[indicator_code]['sourcenotes'] + "\n" if global_cat[indicator_code]['sourcenotes'] else '' source_description['additionalInfo'] += "General comments:\n" + global_cat[indicator_code]['comments'] + "\n" if global_cat[indicator_code]['comments'] else '' source_description['additionalInfo'] += "Statistical concept and methodology:\n" + global_cat[indicator_code]['concept'] + "\n" if global_cat[indicator_code]['concept'] else '' source_description['additionalInfo'] += "Related source links:\n" + global_cat[indicator_code]['sourcelinks'] + "\n" if global_cat[indicator_code]['sourcelinks'] else '' source_description['additionalInfo'] += "Other web links:\n" + global_cat[indicator_code]['weblinks'] + "\n" if global_cat[indicator_code]['weblinks'] else '' source_description['dataPublisherSource'] = global_cat[indicator_code]['source'] newsource.description=json.dumps(source_description) newsource.datasetId=newdataset.pk newsource.save() logger.info("Updating the source %s." % newsource.name.encode('utf8')) s_unit = short_unit_extract(global_cat[indicator_code]['unitofmeasure']) newvariable = Variable.objects.get(code=indicator_code, datasetId__in=Dataset.objects.filter(namespace='povstats')) newvariable.name = global_cat[indicator_code]['name']
global_cat[indicator_code]['unitofmeasure'] else '', short_unit=s_unit, description=global_cat[indicator_code]['description'], code=indicator_code, timespan='1960-' + str(last_available_year), datasetId=newdataset, sourceId=newsource) newvariable.save() global_cat[indicator_code]['variable_object'] = newvariable var_codes_to_add.remove(indicator_code) global_cat[indicator_code]['saved'] = True logger.info("Inserting a variable %s." % newvariable.name.encode('utf8')) else: if not global_cat[indicator_code]['saved']: newsource = Source.objects.get(name='World Bank – WDI: ' + Variable.objects.get(code=indicator_code, datasetId__in=Dataset.objects.filter(namespace=DATASET_NAMESPACE)).name) newsource.name = 'World Bank – WDI: ' + global_cat[indicator_code]['name'] source_description['additionalInfo'] = "Definitions and characteristics of countries and other territories: " + "https://ourworldindata.org" + reverse("servewdicountryinfo") + "\n" source_description['additionalInfo'] += "Limitations and exceptions:\n" + global_cat[indicator_code]['limitations'] + "\n" if global_cat[indicator_code]['limitations'] else '' source_description['additionalInfo'] += "Notes from original source:\n" + global_cat[indicator_code]['sourcenotes'] + "\n" if global_cat[indicator_code]['sourcenotes'] else '' source_description['additionalInfo'] += "General comments:\n" + global_cat[indicator_code]['comments'] + "\n" if global_cat[indicator_code]['comments'] else '' source_description['additionalInfo'] += "Statistical concept and methodology:\n" + global_cat[indicator_code]['concept'] if global_cat[indicator_code]['concept'] else '' source_description['additionalInfo'] += "Related source links:\n" + global_cat[indicator_code]['sourcelinks'] + "\n" if global_cat[indicator_code]['sourcelinks'] else '' source_description['additionalInfo'] += "Other web links:\n" + global_cat[indicator_code]['weblinks'] + "\n" if global_cat[indicator_code]['weblinks'] else '' source_description['dataPublisherSource'] = global_cat[indicator_code]['source'] if 'iea.org' in json.dumps( source_description).lower() or 'iea stat' in json.dumps( source_description).lower() or 'iea 2014' in json.dumps( source_description).lower(): source_description[ 'dataPublishedBy'] = 'International Energy Agency (IEA) via The World Bank' else:
'link'] += ", " + qog_sources[source_name][ 'url'] if qog_sources[source_name]['url'] else "" source_description['dataPublisherSource'] = qog_sources[ source_name]['name'] newsource = Source( name='%s via the Quality of Government dataset' % (qog_sources[source_name]['name']), description=json.dumps(source_description), datasetId=datasets_ref_models[vardata['category']].pk) # in the metadata file, some of the sources have the same name, but are treated as different sources # so if we see a source with the same name in the same category, we switch to using the original dataset name try: with transaction.atomic(): newsource.save() except django.db.utils.IntegrityError: newsource.name = '%s via the Quality of Government dataset' % ( qog_sources[source_name]['original_dataset']) newsource.save() logger.info("Inserting a source %s." % newsource.name.encode('utf8')) saved_sources[source_name].update( {vardata['category']: newsource}) else: source_description['additionalInfo'] = qog_sources[ source_name]['description'] source_description['link'] = "http://qog.pol.gu.se/data" source_description['link'] += ", " + qog_sources[source_name][ 'url'] if qog_sources[source_name]['url'] else "" source_description['dataPublisherSource'] = qog_sources[ source_name]['name'] newsource = Source( name='%s via the Quality of Government dataset' %
short_unit=s_unit, description=global_cat[indicator_code]['description'], code=indicator_code, timespan='1960-' + str(last_available_year), fk_dst_id=newdataset, fk_var_type_id=VariableType.objects.get(pk=4), sourceId=newsource) newvariable.save() global_cat[indicator_code]['variable_object'] = newvariable vars_to_add.remove(indicator_code) global_cat[indicator_code]['saved'] = True logger.info("Inserting a variable %s." % newvariable.name.encode('utf8')) else: if not global_cat[indicator_code]['saved']: newsource = Source.objects.get(name='World Bank Health Nutrition and Population Statistics: ' + Variable.objects.get(code=indicator_code, fk_dst_id__in=Dataset.objects.filter(namespace='hnpstats')).name) newsource.name = 'World Bank Health Nutrition and Population Statistics: ' + global_cat[indicator_code]['name'] source_description['additionalInfo'] = "Definitions and characteristics of countries and other territories: " + "https://ourworldindata.org" + reverse("servehnpstatscountryinfo") + "\n" source_description['additionalInfo'] += "Limitations and exceptions:\n" + global_cat[indicator_code]['limitations'] + "\n" if global_cat[indicator_code]['limitations'] else '' source_description['additionalInfo'] += "Notes from original source:\n" + global_cat[indicator_code]['sourcenotes'] + "\n" if global_cat[indicator_code]['sourcenotes'] else '' source_description['additionalInfo'] += "General comments:\n" + global_cat[indicator_code]['comments'] + "\n" if global_cat[indicator_code]['comments'] else '' source_description['additionalInfo'] += "Statistical concept and methodology:\n" + global_cat[indicator_code]['concept'] + "\n" if global_cat[indicator_code]['concept'] else '' source_description['additionalInfo'] += "Related source links:\n" + global_cat[indicator_code]['sourcelinks'] + "\n" if global_cat[indicator_code]['sourcelinks'] else '' source_description['additionalInfo'] += "Other web links:\n" + global_cat[indicator_code]['weblinks'] + "\n" if global_cat[indicator_code]['weblinks'] else '' source_description['dataPublisherSource'] = global_cat[indicator_code]['source'] newsource.description=json.dumps(source_description) newsource.datasetId=newdataset.pk newsource.save() logger.info("Updating the source %s." % newsource.name.encode('utf8')) s_unit = short_unit_extract(global_cat[indicator_code]['unitofmeasure']) newvariable = Variable.objects.get(code=indicator_code, fk_dst_id__in=Dataset.objects.filter(namespace='hnpstats')) newvariable.name = global_cat[indicator_code]['name']
logger.info( "Inserting a variable %s." % newvariable.name.encode('utf8')) else: if not global_cat[indicator_code][ 'saved']: newsource = Source.objects.get( name= 'World Bank Climate Change Data: ' + Variable.objects.get( code=indicator_code, fk_dst_id__in=Dataset. objects.filter( namespace='climatech' )).name) newsource.name = 'World Bank Climate Change Data: ' + global_cat[ indicator_code]['name'] source_description[ 'additionalInfo'] = None source_description[ 'dataPublisherSource'] = global_cat[ indicator_code]['source'] newsource.description = json.dumps( source_description) newsource.datasetId = newdataset.pk newsource.save() logger.info( "Updating the source %s." % newsource.name.encode('utf8')) if global_cat[indicator_code][ 'unitofmeasure']: if len(global_cat[
short_unit=s_unit, description=global_cat[indicator_code]['description'], code=indicator_code, timespan='', datasetId=newdataset, variableTypeId=VariableType.objects.get(pk=4), sourceId=newsource) newvariable.save() global_cat[indicator_code]['variable_object'] = newvariable vars_to_add.remove(indicator_code) global_cat[indicator_code]['saved'] = True logger.info("Inserting a variable %s." % newvariable.name.encode('utf8')) else: if not global_cat[indicator_code]['saved']: newsource = Source.objects.get(name='World Bank Data on Statistical Capacity: ' + Variable.objects.get(code=indicator_code, datasetId__in=Dataset.objects.filter(namespace='bbsc')).name) newsource.name = 'World Bank Data on Statistical Capacity: ' + global_cat[indicator_code]['name'] source_description['additionalInfo'] = "Definitions and characteristics of countries and other territories: " + "https://ourworldindata.org" + reverse("servebbsccountryinfo") + "\n" source_description['additionalInfo'] += "Limitations and exceptions:\n" + global_cat[indicator_code]['limitations'] + "\n" if global_cat[indicator_code]['limitations'] else '' source_description['additionalInfo'] += "Notes from original source:\n" + global_cat[indicator_code]['sourcenotes'] + "\n" if global_cat[indicator_code]['sourcenotes'] else '' source_description['additionalInfo'] += "General comments:\n" + global_cat[indicator_code]['comments'] + "\n" if global_cat[indicator_code]['comments'] else '' source_description['additionalInfo'] += "Statistical concept and methodology:\n" + global_cat[indicator_code]['concept'] + "\n" if global_cat[indicator_code]['concept'] else '' source_description['additionalInfo'] += "Related source links:\n" + global_cat[indicator_code]['sourcelinks'] + "\n" if global_cat[indicator_code]['sourcelinks'] else '' source_description['additionalInfo'] += "Other web links:\n" + global_cat[indicator_code]['weblinks'] + "\n" if global_cat[indicator_code]['weblinks'] else '' source_description['dataPublisherSource'] = global_cat[indicator_code]['source'] newsource.description=json.dumps(source_description) newsource.datasetId=newdataset.pk newsource.save() logger.info("Updating the source %s." % newsource.name.encode('utf8')) s_unit = short_unit_extract(global_cat[indicator_code]['unitofmeasure']) newvariable = Variable.objects.get(code=indicator_code, datasetId__in=Dataset.objects.filter(namespace='bbsc')) newvariable.name = global_cat[indicator_code]['name']
short_unit=s_unit, description=global_cat[indicator_code]['description'], code=indicator_code, timespan='', fk_dst_id=newdataset, fk_var_type_id=VariableType.objects.get(pk=4), sourceId=newsource) newvariable.save() global_cat[indicator_code]['variable_object'] = newvariable vars_to_add.remove(indicator_code) global_cat[indicator_code]['saved'] = True logger.info("Inserting a variable %s." % newvariable.name.encode('utf8')) else: if not global_cat[indicator_code]['saved']: newsource = Source.objects.get(name='World Bank Global Findex: ' + Variable.objects.get(code=indicator_code, fk_dst_id__in=Dataset.objects.filter(namespace='findex')).name) newsource.name = 'World Bank Global Findex: ' + global_cat[indicator_code]['name'] source_description['additionalInfo'] = "Definitions and characteristics of countries and other territories: " + "https://ourworldindata.org" + reverse("servefindexcountryinfo") + "\n" source_description['additionalInfo'] += "Limitations and exceptions:\n" + global_cat[indicator_code]['limitations'] + "\n" if global_cat[indicator_code]['limitations'] else '' source_description['additionalInfo'] += "Notes from original source:\n" + global_cat[indicator_code]['sourcenotes'] + "\n" if global_cat[indicator_code]['sourcenotes'] else '' source_description['additionalInfo'] += "General comments:\n" + global_cat[indicator_code]['comments'] + "\n" if global_cat[indicator_code]['comments'] else '' source_description['additionalInfo'] += "Statistical concept and methodology:\n" + global_cat[indicator_code]['concept'] + "\n" if global_cat[indicator_code]['concept'] else '' source_description['additionalInfo'] += "Related source links:\n" + global_cat[indicator_code]['sourcelinks'] + "\n" if global_cat[indicator_code]['sourcelinks'] else '' source_description['additionalInfo'] += "Other web links:\n" + global_cat[indicator_code]['weblinks'] + "\n" if global_cat[indicator_code]['weblinks'] else '' source_description['dataPublisherSource'] = global_cat[indicator_code]['source'] newsource.description=json.dumps(source_description) newsource.datasetId=newdataset.pk newsource.save() logger.info("Updating the source %s." % newsource.name.encode('utf8')) s_unit = short_unit_extract(global_cat[indicator_code]['unitofmeasure']) newvariable = Variable.objects.get(code=indicator_code, fk_dst_id__in=Dataset.objects.filter(namespace='findex')) newvariable.name = global_cat[indicator_code]['name']