def process_uploaded_file(self, uploaded, object, request, **kwargs):
'''
This method will be called for every csv file uploaded.
Parameters:
:uploaded: instance of uploaded file
:object: instance of object if in form_multiupload else None
:kwargs: request.POST received with file
Return:
It MUST return at least a dict with:
{
'url': 'url to download the file',
'thumbnail_url': 'some url for an image_thumbnail or icon',
'id': 'id of instance created in this method',
'name': 'the name of created file',
}
'''
try:
#getting the title of the file
title = kwargs.get('title', [''])[0] or uploaded.name
import csv
try:
dialect = csv.Sniffer().sniff(uploaded.read(4048))
except csv.Error:
dialect = csv.excel
file = csv.DictReader(uploaded, dialect=dialect)
line_counter = 0
indicator_from_db = None
city_found = []
city_not_found = []
country_found = []
country_not_found = []
total_items_saved = 0
cities = get_cities()
countries = get_countries()
for line in file:
#getting data from the csv file
city_csv = line.get('city')
deprivation_type_csv = line.get('deprivation_type')
description_csv = line.get('description')
selection_type_csv = line.get('selection_type')
country_csv = line.get('country')
region_csv = line.get('region_csv')
friendly_label_csv = line.get('friendly_name')
value_csv = line.get('value')
year_range_csv = line.get('year_range')
indicator_id_csv = line.get('indicator_id')
year_csv = line.get('year')
type_data_csv = line.get('type_data')
category_csv = line.get('category')
value_csv = str(value_csv)
value_csv = value_csv.replace(".", ",")
if value_csv == None or value_csv == "NULL":
continue
#here we are checking if this indicator already exists, or if we have to create a new one
if line_counter == 0:
#try to find the indicator that is uploaded or create a new one
if Indicator.objects.filter(id=indicator_id_csv).exists():
indicator_from_db = Indicator.objects.get(
id=indicator_id_csv)
else:
# create new indicator
try:
indicator_from_db = Indicator(
id=indicator_id_csv,
description=description_csv,
friendly_label=friendly_label_csv,
type_data=type_data_csv,
deprivation_type=deprivation_type_csv,
category=category_csv)
indicator_from_db.save()
except Exception as e:
print e
#getting country from our database
country_from_db = find_country(country_name=country_csv,
countries=countries)
#add country to the log array
if country_from_db:
country_found.append(country_csv)
country_id = country_from_db.code
else:
if country_csv:
country_not_found.append(country_csv)
country_id = None
else:
country_id = None
city_from_db = find_city(city_name=city_csv,
cities=cities,
country_id=country_id)
#add city to the log array
if city_from_db:
city_found.append(city_csv)
else:
if city_csv:
city_not_found.append(city_csv)
try:
if city_from_db:
#this block is for storing data related to cities
if save_city_data(
city_from_db=city_from_db,
country_from_db=country_from_db,
selection_type_csv=selection_type_csv,
indicator_from_db=indicator_from_db,
year_csv=year_csv,
value_csv=value_csv):
total_items_saved += 1
elif country_from_db:
#this block is for storing country related indicator data
if save_country_data(
country_from_db=country_from_db,
city_csv=city_csv,
selection_type_csv=selection_type_csv,
year_csv=year_csv,
indicator_from_db=indicator_from_db,
value_csv=value_csv):
total_items_saved += 1
line_counter += 1
except Exception as e:
print e
log = save_log(file=uploaded,
uploaded_by_user=request.user,
cities_not_found=city_not_found,
countries_not_found=country_not_found,
total_cities_found=city_found,
total_countries_found=country_found,
total_cities_not_found=city_not_found,
total_countries_not_found=country_not_found,
total_items_saved=total_items_saved)
return {
'url': '/admin/indicator/csvuploadlog/%s/' % str(log.id),
'thumbnail_url': '',
'id': str(log.id),
'name': title,
'country_not_found': log.countries_not_found,
'total_countries_not_found': country_not_found.__len__(),
'city_not_found': log.cities_not_found,
'total_cities_not_found': city_not_found.__len__(),
'total_items_saved': str(total_items_saved),
}
except Exception as e:
print e
def process_uploaded_file(self, uploaded, object,request, **kwargs):
'''
This method will be called for every csv file uploaded.
Parameters:
:uploaded: instance of uploaded file
:object: instance of object if in form_multiupload else None
:kwargs: request.POST received with file
Return:
It MUST return at least a dict with:
{
'url': 'url to download the file',
'thumbnail_url': 'some url for an image_thumbnail or icon',
'id': 'id of instance created in this method',
'name': 'the name of created file',
}
'''
try:
#getting the title of the file
title = kwargs.get('title', [''])[0] or uploaded.name
import csv
try:
dialect = csv.Sniffer().sniff(uploaded.read(4048))
except csv.Error:
dialect = csv.excel
file = csv.DictReader(uploaded, dialect=dialect)
line_counter = 0
indicator_from_db = None
city_found = []
city_not_found = []
country_found = []
country_not_found = []
total_items_saved = 0
cities = get_cities()
countries = get_countries()
for line in file:
#getting data from the csv file
city_csv = line.get('city')
deprivation_type_csv = line.get('deprivation_type')
description_csv = line.get('description')
selection_type_csv = line.get('selection_type')
country_csv = line.get('country')
region_csv = line.get('region_csv')
friendly_label_csv = line.get('friendly_name')
value_csv = line.get('value')
year_range_csv = line.get('year_range')
indicator_id_csv = line.get('indicator_id')
year_csv = line.get('year')
type_data_csv = line.get('type_data')
category_csv = line.get('category')
value_csv = str(value_csv)
value_csv = value_csv.replace(".", ",")
if value_csv == None or value_csv == "NULL":
continue
#here we are checking if this indicator already exists, or if we have to create a new one
if line_counter == 0:
#try to find the indicator that is uploaded or create a new one
if Indicator.objects.filter(id=indicator_id_csv).exists():
indicator_from_db = Indicator.objects.get(id=indicator_id_csv)
else:
# create new indicator
try:
indicator_from_db = Indicator(id=indicator_id_csv, description=description_csv, friendly_label=friendly_label_csv, type_data=type_data_csv, deprivation_type=deprivation_type_csv, category=category_csv)
indicator_from_db.save()
except Exception as e:
print(e)
#getting country from our database
country_from_db = find_country(country_name=country_csv, countries=countries)
#add country to the log array
if country_from_db:
country_found.append(country_csv)
country_id = country_from_db.code
else:
if country_csv:
country_not_found.append(country_csv)
country_id = None
else:
country_id = None
city_from_db = find_city(city_name=city_csv, cities=cities, country_id=country_id)
#add city to the log array
if city_from_db:
city_found.append(city_csv)
else:
if city_csv:
city_not_found.append(city_csv)
try:
if city_from_db:
#this block is for storing data related to cities
if save_city_data(
city_from_db=city_from_db,
country_from_db=country_from_db,
selection_type_csv=selection_type_csv,
indicator_from_db=indicator_from_db,
year_csv=year_csv,
value_csv=value_csv
): total_items_saved += 1
elif country_from_db:
#this block is for storing country related indicator data
if save_country_data(
country_from_db=country_from_db,
city_csv=city_csv,
selection_type_csv=selection_type_csv,
year_csv=year_csv,
indicator_from_db=indicator_from_db,
value_csv=value_csv
): total_items_saved += 1
line_counter += 1
except Exception as e:
print(e)
log = save_log(file=uploaded,
uploaded_by_user=request.user,
cities_not_found=city_not_found,
countries_not_found=country_not_found,
total_cities_found=city_found,
total_countries_found=country_found,
total_cities_not_found=city_not_found,
total_countries_not_found=country_not_found,
total_items_saved=total_items_saved
)
return {
'url': '/admin/indicator/csvuploadlog/%s/' % str(log.id),
'thumbnail_url': '',
'id': str(log.id),
'name' : title,
'country_not_found' : log.countries_not_found,
'total_countries_not_found' : country_not_found.__len__(),
'city_not_found' : log.cities_not_found,
'total_cities_not_found' : city_not_found.__len__(),
'total_items_saved' : str(total_items_saved),
}
except Exception as e:
print(e)
def process_uploaded_file(self, uploaded, object, request, **kwargs):
'''
This method will be called for every csv file uploaded.
Parameters:
:uploaded: instance of uploaded file
:object: instance of object if in form_multiupload else None
:kwargs: request.POST received with file
Return:
It MUST return at least a dict with:
{
'url': 'url to download the file',
'thumbnail_url': 'some url for an image_thumbnail or icon',
'id': 'id of instance created in this method',
'name': 'the name of created file',
}
'''
line_counter = 0
country_found = []
country_not_found = []
total_items_saved = 0
countries = get_countries()
#getting the title of the file
title = kwargs.get('title', [''])[0] or uploaded.name
xmlDoc = uploaded
xmlDocData = xmlDoc.read()
xmlDocTree = etree.XML(xmlDocData)
for indicator in xmlDocTree.iter('CountryId'):
indicator_name_en = indicator[1].text.rstrip()
indicator_name_fr = indicator[2].text.rstrip()
indicator_country = indicator[0].text.rstrip()
country_iso = indicator.get('countryid').rstrip()
value = indicator[3].text.rstrip()
type_value = None
try:
website_en = indicator[4].text.rstrip()
website_fr = indicator[5].text.rstrip()
except IndexError:
website_en = None
website_fr = None
#try to find the indicator that is uploaded or create a new one
indicator_from_db = UnescoIndicator.objects.get_or_create(
id=indicator_name_en)[0]
#getting country from our database
country_from_db = find_country(country_name=indicator_country,
countries=countries,
iso2=country_iso)
#add country to the log array
if country_from_db:
country_found.append(indicator_country)
else:
if indicator_country:
country_not_found.append(indicator_country)
#saving the unesco indicator data
if country_from_db:
indicator_data_from_db = UnescoIndicatorData.objects.get_or_create(
unesco_indicator=indicator_from_db,
country=country_from_db,
value=value)[0]
#storing the translation of the indicator
TranslationModel.objects.get_or_create(
key=indicator_name_en,
language='en',
translation=indicator_name_en)
TranslationModel.objects.get_or_create(
key=indicator_name_en,
language='fr',
translation=indicator_name_fr)
if website_en:
indicator_data_from_db.website = website_en
indicator_data_from_db.save()
#we need to store the translations as well
TranslationModel.objects.get_or_create(
key=website_en, language='en', translation=website_en)
TranslationModel.objects.get_or_create(
key=website_en, language='fr', translation=website_fr)
total_items_saved += 1
line_counter += 1
log = save_log(file=uploaded,
uploaded_by_user=request.user,
cities_not_found=[],
countries_not_found=country_not_found,
total_cities_found=[],
total_countries_found=country_found,
total_cities_not_found=[],
total_countries_not_found=country_not_found,
total_items_saved=total_items_saved)
return {
'url': '/admin/indicator/csvuploadlog/%s/' % str(log.id),
'thumbnail_url': '',
'id': str(log.id),
'name': title,
'country_not_found': log.countries_not_found,
'total_countries_not_found': country_not_found.__len__(),
'city_not_found': log.cities_not_found,
'total_cities_not_found': 0,
'total_items_saved': str(total_items_saved),
}
def process_uploaded_file(self, uploaded, object,request, **kwargs):
'''
This method will be called for every csv file uploaded.
Parameters:
:uploaded: instance of uploaded file
:object: instance of object if in form_multiupload else None
:kwargs: request.POST received with file
Return:
It MUST return at least a dict with:
{
'url': 'url to download the file',
'thumbnail_url': 'some url for an image_thumbnail or icon',
'id': 'id of instance created in this method',
'name': 'the name of created file',
}
'''
line_counter = 0
country_found = []
country_not_found = []
total_items_saved = 0
countries = get_countries()
#getting the title of the file
title = kwargs.get('title', [''])[0] or uploaded.name
xmlDoc = uploaded
xmlDocData = xmlDoc.read()
xmlDocTree = etree.XML(xmlDocData)
for indicator in xmlDocTree.iter('CountryId'):
indicator_name_en = indicator[1].text.rstrip()
indicator_name_fr = indicator[2].text.rstrip()
indicator_country = indicator[0].text.rstrip()
country_iso = indicator.get('countryid').rstrip()
value = indicator[3].text.rstrip()
type_value = None
try:
website_en = indicator[4].text.rstrip()
website_fr = indicator[5].text.rstrip()
except IndexError:
website_en = None
website_fr = None
#try to find the indicator that is uploaded or create a new one
indicator_from_db = UnescoIndicator.objects.get_or_create(id=indicator_name_en)[0]
#getting country from our database
country_from_db = find_country(country_name=indicator_country, countries=countries, iso2=country_iso)
#add country to the log array
if country_from_db:
country_found.append(indicator_country)
else:
if indicator_country:
country_not_found.append(indicator_country)
#saving the unesco indicator data
if country_from_db:
indicator_data_from_db = UnescoIndicatorData.objects.get_or_create(unesco_indicator=indicator_from_db, country=country_from_db, value=value)[0]
#storing the translation of the indicator
TranslationModel.objects.get_or_create(key=indicator_name_en, language='en', translation=indicator_name_en)
TranslationModel.objects.get_or_create(key=indicator_name_en, language='fr', translation=indicator_name_fr)
if website_en:
indicator_data_from_db.website = website_en
indicator_data_from_db.save()
#we need to store the translations as well
TranslationModel.objects.get_or_create(key=website_en, language='en', translation=website_en)
TranslationModel.objects.get_or_create(key=website_en, language='fr', translation=website_fr)
total_items_saved += 1
line_counter += 1
log = save_log(file=uploaded,
uploaded_by_user=request.user,
cities_not_found=[],
countries_not_found=country_not_found,
total_cities_found=[],
total_countries_found=country_found,
total_cities_not_found=[],
total_countries_not_found=country_not_found,
total_items_saved=total_items_saved
)
return {
'url': '/admin/indicator/csvuploadlog/%s/' % str(log.id),
'thumbnail_url': '',
'id': str(log.id),
'name' : title,
'country_not_found' : log.countries_not_found,
'total_countries_not_found' : country_not_found.__len__(),
'city_not_found' : log.cities_not_found,
'total_cities_not_found' : 0,
'total_items_saved' : str(total_items_saved),
}
