'link'] = filename_to_pagelink[one_file]
newsource.description = json.dumps(
source_description)
newsource.datasetId = newdataset.pk
newsource.save()
newvariable = Variable.objects.get(
code=filename_to_pagelink[one_file]
[filename_to_pagelink[one_file].
rfind('/') + 1:],
datasetId__namespace='clioinfra')
newvariable.name = varname
newvariable.unit = varunit if varunit else ''
newvariable.short_unit = short_unit_extract(
varunit)
newvariable.description = ''
newvariable.timespan = ''
newvariable.datasetId = newdataset
newvariable.variableTypeId = VariableType.objects.get(
pk=4)
newvariable.sourceId = newsource
newvariable.save()
# Deleting old data values
while DataValue.objects.filter(
variableId__pk=newvariable.pk).first():
with connection.cursor(
) as c: # if we don't limit the deleted values, the db might just hang
c.execute(
'DELETE FROM %s WHERE variableId = %s LIMIT 10000;'
source_description).lower():
source_description[
'dataPublishedBy'] = 'International Energy Agency (IEA) via The World Bank'
else:
source_description[
'dataPublishedBy'] = 'World Bank – World Development Indicators'
newsource.description=json.dumps(source_description)
newsource.datasetId=newdataset
newsource.save()
logger.info("Updating the source %s." % newsource.name.encode('utf8'))
s_unit = extract_short_unit(global_cat[indicator_code]['unitofmeasure'])
newvariable = Variable.objects.get(code=indicator_code, datasetId__in=Dataset.objects.filter(namespace=DATASET_NAMESPACE))
newvariable.name = global_cat[indicator_code]['name']
newvariable.unit=global_cat[indicator_code]['unitofmeasure'] if global_cat[indicator_code]['unitofmeasure'] else ''
newvariable.short_unit = s_unit
newvariable.description=global_cat[indicator_code]['description']
newvariable.timespan='1960-' + str(last_available_year)
newvariable.datasetId=newdataset
newvariable.sourceId=newsource
newvariable.save()
global_cat[indicator_code]['variable_object'] = newvariable
logger.info("Updating the variable %s." % newvariable.name.encode('utf8'))
global_cat[indicator_code]['saved'] = True
else:
newvariable = global_cat[indicator_code]['variable_object']
if indicator_code not in newly_added_var_codes:
if not deleted_indicators.get(indicator_code, 0):
while DataValue.objects.filter(variableId__pk=newvariable.pk).first():
with connection.cursor() as c:
c.execute(
'DELETE FROM %s WHERE variableId = %s LIMIT 10000;' %
if varcode_for_reference not in existing_vars_list:
newvariable = Variable(name=variable_name,
unit=varunit if
varunit else '', short_unit=short_unit_extract(varunit),
description='See concepts and methods provided by ILOSTAT at http://www.ilo.org/ilostat/faces/ilostat-home/metadata',
code=varcode_for_reference,
timespan='',
datasetId=Dataset.objects.get(
name=file_name_to_category[row['indicator']], namespace='ilostat'),
variableTypeId=VariableType.objects.get(pk=4),
sourceId=newsource)
else:
newvariable = Variable.objects.get(code=varcode_for_reference, datasetId__namespace__exact='ilostat')
newvariable.description = 'See concepts and methods provided by ILOSTAT at http://www.ilo.org/ilostat/faces/ilostat-home/metadata'
newvariable.name = variable_name
newvariable.unit = varunit if varunit else ''
newvariable.short_unit = short_unit_extract(varunit)
newvariable.sourceId = newsource
while DataValue.objects.filter(variableId__pk=newvariable.pk).first():
with connection.cursor() as c: # if we don't limit the deleted values, the db might just hang
c.execute('DELETE FROM %s WHERE variableId = %s LIMIT 10000;' %
(DataValue._meta.db_table, newvariable.pk))
varcode_to_object[varcode_for_reference] = newvariable
newvariable.save()
variables = None