source_description).lower() or 'iea 2014' in json.dumps(
source_description).lower():
source_description[
'dataPublishedBy'] = 'International Energy Agency (IEA) via The World Bank'
else:
source_description[
'dataPublishedBy'] = 'World Bank – World Development Indicators'
newsource.description=json.dumps(source_description)
newsource.datasetId=newdataset
newsource.save()
logger.info("Updating the source %s." % newsource.name.encode('utf8'))
s_unit = extract_short_unit(global_cat[indicator_code]['unitofmeasure'])
newvariable = Variable.objects.get(code=indicator_code, datasetId__in=Dataset.objects.filter(namespace=DATASET_NAMESPACE))
newvariable.name = global_cat[indicator_code]['name']
newvariable.unit=global_cat[indicator_code]['unitofmeasure'] if global_cat[indicator_code]['unitofmeasure'] else ''
newvariable.short_unit = s_unit
newvariable.description=global_cat[indicator_code]['description']
newvariable.timespan='1960-' + str(last_available_year)
newvariable.datasetId=newdataset
newvariable.sourceId=newsource
newvariable.save()
global_cat[indicator_code]['variable_object'] = newvariable
logger.info("Updating the variable %s." % newvariable.name.encode('utf8'))
global_cat[indicator_code]['saved'] = True
else:
newvariable = global_cat[indicator_code]['variable_object']
if indicator_code not in newly_added_var_codes:
if not deleted_indicators.get(indicator_code, 0):
while DataValue.objects.filter(variableId__pk=newvariable.pk).first():
with connection.cursor() as c:
c.execute(
datasetId__namespace='clioinfra').sourceId
source_description[
'link'] = filename_to_pagelink[one_file]
newsource.description = json.dumps(
source_description)
newsource.datasetId = newdataset.pk
newsource.save()
newvariable = Variable.objects.get(
code=filename_to_pagelink[one_file]
[filename_to_pagelink[one_file].
rfind('/') + 1:],
datasetId__namespace='clioinfra')
newvariable.name = varname
newvariable.unit = varunit if varunit else ''
newvariable.short_unit = short_unit_extract(
varunit)
newvariable.description = ''
newvariable.timespan = ''
newvariable.datasetId = newdataset
newvariable.variableTypeId = VariableType.objects.get(
pk=4)
newvariable.sourceId = newsource
newvariable.save()
# Deleting old data values
while DataValue.objects.filter(
variableId__pk=newvariable.pk).first():
with connection.cursor(
) as c: # if we don't limit the deleted values, the db might just hang
c.execute(