short_unit=s_unit,
description=global_cat[indicator_code]['description'],
code=indicator_code,
timespan='',
datasetId=newdataset,
variableTypeId=VariableType.objects.get(pk=4),
sourceId=newsource)
newvariable.save()
global_cat[indicator_code]['variable_object'] = newvariable
vars_to_add.remove(indicator_code)
global_cat[indicator_code]['saved'] = True
logger.info("Inserting a variable %s." % newvariable.name.encode('utf8'))
else:
if not global_cat[indicator_code]['saved']:
newsource = Source.objects.get(name='World Bank Poverty and Equity database: ' + Variable.objects.get(code=indicator_code, datasetId__in=Dataset.objects.filter(namespace='povstats')).name)
newsource.name = 'World Bank Poverty and Equity database: ' + global_cat[indicator_code]['name']
source_description['additionalInfo'] = "Definitions and characteristics of countries and other territories: " + "https://ourworldindata.org" + reverse("servepovstatscountryinfo") + "\n"
source_description['additionalInfo'] += "Limitations and exceptions:\n" + global_cat[indicator_code]['limitations'] + "\n" if global_cat[indicator_code]['limitations'] else ''
source_description['additionalInfo'] += "Notes from original source:\n" + global_cat[indicator_code]['sourcenotes'] + "\n" if global_cat[indicator_code]['sourcenotes'] else ''
source_description['additionalInfo'] += "General comments:\n" + global_cat[indicator_code]['comments'] + "\n" if global_cat[indicator_code]['comments'] else ''
source_description['additionalInfo'] += "Statistical concept and methodology:\n" + global_cat[indicator_code]['concept'] + "\n" if global_cat[indicator_code]['concept'] else ''
source_description['additionalInfo'] += "Related source links:\n" + global_cat[indicator_code]['sourcelinks'] + "\n" if global_cat[indicator_code]['sourcelinks'] else ''
source_description['additionalInfo'] += "Other web links:\n" + global_cat[indicator_code]['weblinks'] + "\n" if global_cat[indicator_code]['weblinks'] else ''
source_description['dataPublisherSource'] = global_cat[indicator_code]['source']
newsource.description=json.dumps(source_description)
newsource.datasetId=newdataset.pk
newsource.save()
logger.info("Updating the source %s." % newsource.name.encode('utf8'))
s_unit = short_unit_extract(global_cat[indicator_code]['unitofmeasure'])
newvariable = Variable.objects.get(code=indicator_code, datasetId__in=Dataset.objects.filter(namespace='povstats'))
newvariable.name = global_cat[indicator_code]['name']
global_cat[indicator_code]['unitofmeasure'] else '',
short_unit=s_unit,
description=global_cat[indicator_code]['description'],
code=indicator_code,
timespan='1960-' + str(last_available_year),
datasetId=newdataset,
sourceId=newsource)
newvariable.save()
global_cat[indicator_code]['variable_object'] = newvariable
var_codes_to_add.remove(indicator_code)
global_cat[indicator_code]['saved'] = True
logger.info("Inserting a variable %s." % newvariable.name.encode('utf8'))
else:
if not global_cat[indicator_code]['saved']:
newsource = Source.objects.get(name='World Bank – WDI: ' + Variable.objects.get(code=indicator_code, datasetId__in=Dataset.objects.filter(namespace=DATASET_NAMESPACE)).name)
newsource.name = 'World Bank – WDI: ' + global_cat[indicator_code]['name']
source_description['additionalInfo'] = "Definitions and characteristics of countries and other territories: " + "https://ourworldindata.org" + reverse("servewdicountryinfo") + "\n"
source_description['additionalInfo'] += "Limitations and exceptions:\n" + global_cat[indicator_code]['limitations'] + "\n" if global_cat[indicator_code]['limitations'] else ''
source_description['additionalInfo'] += "Notes from original source:\n" + global_cat[indicator_code]['sourcenotes'] + "\n" if global_cat[indicator_code]['sourcenotes'] else ''
source_description['additionalInfo'] += "General comments:\n" + global_cat[indicator_code]['comments'] + "\n" if global_cat[indicator_code]['comments'] else ''
source_description['additionalInfo'] += "Statistical concept and methodology:\n" + global_cat[indicator_code]['concept'] if global_cat[indicator_code]['concept'] else ''
source_description['additionalInfo'] += "Related source links:\n" + global_cat[indicator_code]['sourcelinks'] + "\n" if global_cat[indicator_code]['sourcelinks'] else ''
source_description['additionalInfo'] += "Other web links:\n" + global_cat[indicator_code]['weblinks'] + "\n" if global_cat[indicator_code]['weblinks'] else ''
source_description['dataPublisherSource'] = global_cat[indicator_code]['source']
if 'iea.org' in json.dumps(
source_description).lower() or 'iea stat' in json.dumps(
source_description).lower() or 'iea 2014' in json.dumps(
source_description).lower():
source_description[
'dataPublishedBy'] = 'International Energy Agency (IEA) via The World Bank'
else:
'link'] += ", " + qog_sources[source_name][
'url'] if qog_sources[source_name]['url'] else ""
source_description['dataPublisherSource'] = qog_sources[
source_name]['name']
newsource = Source(
name='%s via the Quality of Government dataset' %
(qog_sources[source_name]['name']),
description=json.dumps(source_description),
datasetId=datasets_ref_models[vardata['category']].pk)
# in the metadata file, some of the sources have the same name, but are treated as different sources
# so if we see a source with the same name in the same category, we switch to using the original dataset name
try:
with transaction.atomic():
newsource.save()
except django.db.utils.IntegrityError:
newsource.name = '%s via the Quality of Government dataset' % (
qog_sources[source_name]['original_dataset'])
newsource.save()
logger.info("Inserting a source %s." %
newsource.name.encode('utf8'))
saved_sources[source_name].update(
{vardata['category']: newsource})
else:
source_description['additionalInfo'] = qog_sources[
source_name]['description']
source_description['link'] = "http://qog.pol.gu.se/data"
source_description['link'] += ", " + qog_sources[source_name][
'url'] if qog_sources[source_name]['url'] else ""
source_description['dataPublisherSource'] = qog_sources[
source_name]['name']
newsource = Source(
name='%s via the Quality of Government dataset' %
short_unit=s_unit,
description=global_cat[indicator_code]['description'],
code=indicator_code,
timespan='1960-' + str(last_available_year),
fk_dst_id=newdataset,
fk_var_type_id=VariableType.objects.get(pk=4),
sourceId=newsource)
newvariable.save()
global_cat[indicator_code]['variable_object'] = newvariable
vars_to_add.remove(indicator_code)
global_cat[indicator_code]['saved'] = True
logger.info("Inserting a variable %s." % newvariable.name.encode('utf8'))
else:
if not global_cat[indicator_code]['saved']:
newsource = Source.objects.get(name='World Bank Health Nutrition and Population Statistics: ' + Variable.objects.get(code=indicator_code, fk_dst_id__in=Dataset.objects.filter(namespace='hnpstats')).name)
newsource.name = 'World Bank Health Nutrition and Population Statistics: ' + global_cat[indicator_code]['name']
source_description['additionalInfo'] = "Definitions and characteristics of countries and other territories: " + "https://ourworldindata.org" + reverse("servehnpstatscountryinfo") + "\n"
source_description['additionalInfo'] += "Limitations and exceptions:\n" + global_cat[indicator_code]['limitations'] + "\n" if global_cat[indicator_code]['limitations'] else ''
source_description['additionalInfo'] += "Notes from original source:\n" + global_cat[indicator_code]['sourcenotes'] + "\n" if global_cat[indicator_code]['sourcenotes'] else ''
source_description['additionalInfo'] += "General comments:\n" + global_cat[indicator_code]['comments'] + "\n" if global_cat[indicator_code]['comments'] else ''
source_description['additionalInfo'] += "Statistical concept and methodology:\n" + global_cat[indicator_code]['concept'] + "\n" if global_cat[indicator_code]['concept'] else ''
source_description['additionalInfo'] += "Related source links:\n" + global_cat[indicator_code]['sourcelinks'] + "\n" if global_cat[indicator_code]['sourcelinks'] else ''
source_description['additionalInfo'] += "Other web links:\n" + global_cat[indicator_code]['weblinks'] + "\n" if global_cat[indicator_code]['weblinks'] else ''
source_description['dataPublisherSource'] = global_cat[indicator_code]['source']
newsource.description=json.dumps(source_description)
newsource.datasetId=newdataset.pk
newsource.save()
logger.info("Updating the source %s." % newsource.name.encode('utf8'))
s_unit = short_unit_extract(global_cat[indicator_code]['unitofmeasure'])
newvariable = Variable.objects.get(code=indicator_code, fk_dst_id__in=Dataset.objects.filter(namespace='hnpstats'))
newvariable.name = global_cat[indicator_code]['name']
logger.info(
"Inserting a variable %s." %
newvariable.name.encode('utf8'))
else:
if not global_cat[indicator_code][
'saved']:
newsource = Source.objects.get(
name=
'World Bank Climate Change Data: '
+ Variable.objects.get(
code=indicator_code,
fk_dst_id__in=Dataset.
objects.filter(
namespace='climatech'
)).name)
newsource.name = 'World Bank Climate Change Data: ' + global_cat[
indicator_code]['name']
source_description[
'additionalInfo'] = None
source_description[
'dataPublisherSource'] = global_cat[
indicator_code]['source']
newsource.description = json.dumps(
source_description)
newsource.datasetId = newdataset.pk
newsource.save()
logger.info(
"Updating the source %s." %
newsource.name.encode('utf8'))
if global_cat[indicator_code][
'unitofmeasure']:
if len(global_cat[
short_unit=s_unit,
description=global_cat[indicator_code]['description'],
code=indicator_code,
timespan='',
datasetId=newdataset,
variableTypeId=VariableType.objects.get(pk=4),
sourceId=newsource)
newvariable.save()
global_cat[indicator_code]['variable_object'] = newvariable
vars_to_add.remove(indicator_code)
global_cat[indicator_code]['saved'] = True
logger.info("Inserting a variable %s." % newvariable.name.encode('utf8'))
else:
if not global_cat[indicator_code]['saved']:
newsource = Source.objects.get(name='World Bank Data on Statistical Capacity: ' + Variable.objects.get(code=indicator_code, datasetId__in=Dataset.objects.filter(namespace='bbsc')).name)
newsource.name = 'World Bank Data on Statistical Capacity: ' + global_cat[indicator_code]['name']
source_description['additionalInfo'] = "Definitions and characteristics of countries and other territories: " + "https://ourworldindata.org" + reverse("servebbsccountryinfo") + "\n"
source_description['additionalInfo'] += "Limitations and exceptions:\n" + global_cat[indicator_code]['limitations'] + "\n" if global_cat[indicator_code]['limitations'] else ''
source_description['additionalInfo'] += "Notes from original source:\n" + global_cat[indicator_code]['sourcenotes'] + "\n" if global_cat[indicator_code]['sourcenotes'] else ''
source_description['additionalInfo'] += "General comments:\n" + global_cat[indicator_code]['comments'] + "\n" if global_cat[indicator_code]['comments'] else ''
source_description['additionalInfo'] += "Statistical concept and methodology:\n" + global_cat[indicator_code]['concept'] + "\n" if global_cat[indicator_code]['concept'] else ''
source_description['additionalInfo'] += "Related source links:\n" + global_cat[indicator_code]['sourcelinks'] + "\n" if global_cat[indicator_code]['sourcelinks'] else ''
source_description['additionalInfo'] += "Other web links:\n" + global_cat[indicator_code]['weblinks'] + "\n" if global_cat[indicator_code]['weblinks'] else ''
source_description['dataPublisherSource'] = global_cat[indicator_code]['source']
newsource.description=json.dumps(source_description)
newsource.datasetId=newdataset.pk
newsource.save()
logger.info("Updating the source %s." % newsource.name.encode('utf8'))
s_unit = short_unit_extract(global_cat[indicator_code]['unitofmeasure'])
newvariable = Variable.objects.get(code=indicator_code, datasetId__in=Dataset.objects.filter(namespace='bbsc'))
newvariable.name = global_cat[indicator_code]['name']
short_unit=s_unit,
description=global_cat[indicator_code]['description'],
code=indicator_code,
timespan='',
fk_dst_id=newdataset,
fk_var_type_id=VariableType.objects.get(pk=4),
sourceId=newsource)
newvariable.save()
global_cat[indicator_code]['variable_object'] = newvariable
vars_to_add.remove(indicator_code)
global_cat[indicator_code]['saved'] = True
logger.info("Inserting a variable %s." % newvariable.name.encode('utf8'))
else:
if not global_cat[indicator_code]['saved']:
newsource = Source.objects.get(name='World Bank Global Findex: ' + Variable.objects.get(code=indicator_code, fk_dst_id__in=Dataset.objects.filter(namespace='findex')).name)
newsource.name = 'World Bank Global Findex: ' + global_cat[indicator_code]['name']
source_description['additionalInfo'] = "Definitions and characteristics of countries and other territories: " + "https://ourworldindata.org" + reverse("servefindexcountryinfo") + "\n"
source_description['additionalInfo'] += "Limitations and exceptions:\n" + global_cat[indicator_code]['limitations'] + "\n" if global_cat[indicator_code]['limitations'] else ''
source_description['additionalInfo'] += "Notes from original source:\n" + global_cat[indicator_code]['sourcenotes'] + "\n" if global_cat[indicator_code]['sourcenotes'] else ''
source_description['additionalInfo'] += "General comments:\n" + global_cat[indicator_code]['comments'] + "\n" if global_cat[indicator_code]['comments'] else ''
source_description['additionalInfo'] += "Statistical concept and methodology:\n" + global_cat[indicator_code]['concept'] + "\n" if global_cat[indicator_code]['concept'] else ''
source_description['additionalInfo'] += "Related source links:\n" + global_cat[indicator_code]['sourcelinks'] + "\n" if global_cat[indicator_code]['sourcelinks'] else ''
source_description['additionalInfo'] += "Other web links:\n" + global_cat[indicator_code]['weblinks'] + "\n" if global_cat[indicator_code]['weblinks'] else ''
source_description['dataPublisherSource'] = global_cat[indicator_code]['source']
newsource.description=json.dumps(source_description)
newsource.datasetId=newdataset.pk
newsource.save()
logger.info("Updating the source %s." % newsource.name.encode('utf8'))
s_unit = short_unit_extract(global_cat[indicator_code]['unitofmeasure'])
newvariable = Variable.objects.get(code=indicator_code, fk_dst_id__in=Dataset.objects.filter(namespace='findex'))
newvariable.name = global_cat[indicator_code]['name']