else:
source_description[
'dataPublishedBy'] = 'World Bank – World Development Indicators'
newsource.description=json.dumps(source_description)
newsource.datasetId=newdataset
newsource.save()
logger.info("Updating the source %s." % newsource.name.encode('utf8'))
s_unit = extract_short_unit(global_cat[indicator_code]['unitofmeasure'])
newvariable = Variable.objects.get(code=indicator_code, datasetId__in=Dataset.objects.filter(namespace=DATASET_NAMESPACE))
newvariable.name = global_cat[indicator_code]['name']
newvariable.unit=global_cat[indicator_code]['unitofmeasure'] if global_cat[indicator_code]['unitofmeasure'] else ''
newvariable.short_unit = s_unit
newvariable.description=global_cat[indicator_code]['description']
newvariable.timespan='1960-' + str(last_available_year)
newvariable.datasetId=newdataset
newvariable.sourceId=newsource
newvariable.save()
global_cat[indicator_code]['variable_object'] = newvariable
logger.info("Updating the variable %s." % newvariable.name.encode('utf8'))
global_cat[indicator_code]['saved'] = True
else:
newvariable = global_cat[indicator_code]['variable_object']
if indicator_code not in newly_added_var_codes:
if not deleted_indicators.get(indicator_code, 0):
while DataValue.objects.filter(variableId__pk=newvariable.pk).first():
with connection.cursor() as c:
c.execute(
'DELETE FROM %s WHERE variableId = %s LIMIT 10000;' %
(DataValue._meta.db_table, newvariable.pk))
deleted_indicators[indicator_code] = True
logger.info("Deleting data values for the variable %s." % indicator_code.encode('utf8'))
datasetId=dataset_name_to_object[subcategory_name],
variableTypeId=VariableType.objects.get(pk=4),
sourceId=source_name_to_object[
source_name.lower()])
newvariable.save()
variable_name_to_object[
variable_name.lower()] = newvariable
existing_variables_list.add(newvariable.name.lower())
else:
if variable_name.lower(
) not in variable_name_to_object:
newvariable = Variable.objects.get(
name=variable_name,
datasetId=dataset_name_to_object[
subcategory_name])
newvariable.sourceId = source_name_to_object[
source_name.lower()]
newvariable.save()
while DataValue.objects.filter(
variableId__pk=newvariable.pk).first():
with connection.cursor(
) as c: # if we don't limit the deleted values, the db might just hang
c.execute(
'DELETE FROM %s WHERE variableId = %s LIMIT 10000;'
% (DataValue._meta.db_table,
newvariable.pk))
variable_name_to_object[
variable_name.lower()] = newvariable
country_name = row['Country']
if 'LIFE' in file_name:
if row['Country'] == 'Bolivia (Plurinational State of)Bolivia (Plurinational State of)':