def process_uploaded_file(self, uploaded, object, request, **kwargs): ''' This method will be called for every csv file uploaded. Parameters: :uploaded: instance of uploaded file :object: instance of object if in form_multiupload else None :kwargs: request.POST received with file Return: It MUST return at least a dict with: { 'url': 'url to download the file', 'thumbnail_url': 'some url for an image_thumbnail or icon', 'id': 'id of instance created in this method', 'name': 'the name of created file', } ''' try: #getting the title of the file title = kwargs.get('title', [''])[0] or uploaded.name import csv try: dialect = csv.Sniffer().sniff(uploaded.read(4048)) except csv.Error: dialect = csv.excel file = csv.DictReader(uploaded, dialect=dialect) line_counter = 0 indicator_from_db = None city_found = [] city_not_found = [] country_found = [] country_not_found = [] total_items_saved = 0 cities = get_cities() countries = get_countries() for line in file: #getting data from the csv file city_csv = line.get('city') deprivation_type_csv = line.get('deprivation_type') description_csv = line.get('description') selection_type_csv = line.get('selection_type') country_csv = line.get('country') region_csv = line.get('region_csv') friendly_label_csv = line.get('friendly_name') value_csv = line.get('value') year_range_csv = line.get('year_range') indicator_id_csv = line.get('indicator_id') year_csv = line.get('year') type_data_csv = line.get('type_data') category_csv = line.get('category') value_csv = str(value_csv) value_csv = value_csv.replace(".", ",") if value_csv == None or value_csv == "NULL": continue #here we are checking if this indicator already exists, or if we have to create a new one if line_counter == 0: #try to find the indicator that is uploaded or create a new one if Indicator.objects.filter(id=indicator_id_csv).exists(): indicator_from_db = Indicator.objects.get( id=indicator_id_csv) else: # create new indicator try: indicator_from_db = Indicator( id=indicator_id_csv, description=description_csv, friendly_label=friendly_label_csv, type_data=type_data_csv, deprivation_type=deprivation_type_csv, category=category_csv) indicator_from_db.save() except Exception as e: print e #getting country from our database country_from_db = find_country(country_name=country_csv, countries=countries) #add country to the log array if country_from_db: country_found.append(country_csv) country_id = country_from_db.code else: if country_csv: country_not_found.append(country_csv) country_id = None else: country_id = None city_from_db = find_city(city_name=city_csv, cities=cities, country_id=country_id) #add city to the log array if city_from_db: city_found.append(city_csv) else: if city_csv: city_not_found.append(city_csv) try: if city_from_db: #this block is for storing data related to cities if save_city_data( city_from_db=city_from_db, country_from_db=country_from_db, selection_type_csv=selection_type_csv, indicator_from_db=indicator_from_db, year_csv=year_csv, value_csv=value_csv): total_items_saved += 1 elif country_from_db: #this block is for storing country related indicator data if save_country_data( country_from_db=country_from_db, city_csv=city_csv, selection_type_csv=selection_type_csv, year_csv=year_csv, indicator_from_db=indicator_from_db, value_csv=value_csv): total_items_saved += 1 line_counter += 1 except Exception as e: print e log = save_log(file=uploaded, uploaded_by_user=request.user, cities_not_found=city_not_found, countries_not_found=country_not_found, total_cities_found=city_found, total_countries_found=country_found, total_cities_not_found=city_not_found, total_countries_not_found=country_not_found, total_items_saved=total_items_saved) return { 'url': '/admin/indicator/csvuploadlog/%s/' % str(log.id), 'thumbnail_url': '', 'id': str(log.id), 'name': title, 'country_not_found': log.countries_not_found, 'total_countries_not_found': country_not_found.__len__(), 'city_not_found': log.cities_not_found, 'total_cities_not_found': city_not_found.__len__(), 'total_items_saved': str(total_items_saved), } except Exception as e: print e
def process_uploaded_file(self, uploaded, object,request, **kwargs): ''' This method will be called for every csv file uploaded. Parameters: :uploaded: instance of uploaded file :object: instance of object if in form_multiupload else None :kwargs: request.POST received with file Return: It MUST return at least a dict with: { 'url': 'url to download the file', 'thumbnail_url': 'some url for an image_thumbnail or icon', 'id': 'id of instance created in this method', 'name': 'the name of created file', } ''' try: #getting the title of the file title = kwargs.get('title', [''])[0] or uploaded.name import csv try: dialect = csv.Sniffer().sniff(uploaded.read(4048)) except csv.Error: dialect = csv.excel file = csv.DictReader(uploaded, dialect=dialect) line_counter = 0 indicator_from_db = None city_found = [] city_not_found = [] country_found = [] country_not_found = [] total_items_saved = 0 cities = get_cities() countries = get_countries() for line in file: #getting data from the csv file city_csv = line.get('city') deprivation_type_csv = line.get('deprivation_type') description_csv = line.get('description') selection_type_csv = line.get('selection_type') country_csv = line.get('country') region_csv = line.get('region_csv') friendly_label_csv = line.get('friendly_name') value_csv = line.get('value') year_range_csv = line.get('year_range') indicator_id_csv = line.get('indicator_id') year_csv = line.get('year') type_data_csv = line.get('type_data') category_csv = line.get('category') value_csv = str(value_csv) value_csv = value_csv.replace(".", ",") if value_csv == None or value_csv == "NULL": continue #here we are checking if this indicator already exists, or if we have to create a new one if line_counter == 0: #try to find the indicator that is uploaded or create a new one if Indicator.objects.filter(id=indicator_id_csv).exists(): indicator_from_db = Indicator.objects.get(id=indicator_id_csv) else: # create new indicator try: indicator_from_db = Indicator(id=indicator_id_csv, description=description_csv, friendly_label=friendly_label_csv, type_data=type_data_csv, deprivation_type=deprivation_type_csv, category=category_csv) indicator_from_db.save() except Exception as e: print(e) #getting country from our database country_from_db = find_country(country_name=country_csv, countries=countries) #add country to the log array if country_from_db: country_found.append(country_csv) country_id = country_from_db.code else: if country_csv: country_not_found.append(country_csv) country_id = None else: country_id = None city_from_db = find_city(city_name=city_csv, cities=cities, country_id=country_id) #add city to the log array if city_from_db: city_found.append(city_csv) else: if city_csv: city_not_found.append(city_csv) try: if city_from_db: #this block is for storing data related to cities if save_city_data( city_from_db=city_from_db, country_from_db=country_from_db, selection_type_csv=selection_type_csv, indicator_from_db=indicator_from_db, year_csv=year_csv, value_csv=value_csv ): total_items_saved += 1 elif country_from_db: #this block is for storing country related indicator data if save_country_data( country_from_db=country_from_db, city_csv=city_csv, selection_type_csv=selection_type_csv, year_csv=year_csv, indicator_from_db=indicator_from_db, value_csv=value_csv ): total_items_saved += 1 line_counter += 1 except Exception as e: print(e) log = save_log(file=uploaded, uploaded_by_user=request.user, cities_not_found=city_not_found, countries_not_found=country_not_found, total_cities_found=city_found, total_countries_found=country_found, total_cities_not_found=city_not_found, total_countries_not_found=country_not_found, total_items_saved=total_items_saved ) return { 'url': '/admin/indicator/csvuploadlog/%s/' % str(log.id), 'thumbnail_url': '', 'id': str(log.id), 'name' : title, 'country_not_found' : log.countries_not_found, 'total_countries_not_found' : country_not_found.__len__(), 'city_not_found' : log.cities_not_found, 'total_cities_not_found' : city_not_found.__len__(), 'total_items_saved' : str(total_items_saved), } except Exception as e: print(e)
def process_uploaded_file(self, uploaded, object, request, **kwargs): ''' This method will be called for every csv file uploaded. Parameters: :uploaded: instance of uploaded file :object: instance of object if in form_multiupload else None :kwargs: request.POST received with file Return: It MUST return at least a dict with: { 'url': 'url to download the file', 'thumbnail_url': 'some url for an image_thumbnail or icon', 'id': 'id of instance created in this method', 'name': 'the name of created file', } ''' line_counter = 0 country_found = [] country_not_found = [] total_items_saved = 0 countries = get_countries() #getting the title of the file title = kwargs.get('title', [''])[0] or uploaded.name xmlDoc = uploaded xmlDocData = xmlDoc.read() xmlDocTree = etree.XML(xmlDocData) for indicator in xmlDocTree.iter('CountryId'): indicator_name_en = indicator[1].text.rstrip() indicator_name_fr = indicator[2].text.rstrip() indicator_country = indicator[0].text.rstrip() country_iso = indicator.get('countryid').rstrip() value = indicator[3].text.rstrip() type_value = None try: website_en = indicator[4].text.rstrip() website_fr = indicator[5].text.rstrip() except IndexError: website_en = None website_fr = None #try to find the indicator that is uploaded or create a new one indicator_from_db = UnescoIndicator.objects.get_or_create( id=indicator_name_en)[0] #getting country from our database country_from_db = find_country(country_name=indicator_country, countries=countries, iso2=country_iso) #add country to the log array if country_from_db: country_found.append(indicator_country) else: if indicator_country: country_not_found.append(indicator_country) #saving the unesco indicator data if country_from_db: indicator_data_from_db = UnescoIndicatorData.objects.get_or_create( unesco_indicator=indicator_from_db, country=country_from_db, value=value)[0] #storing the translation of the indicator TranslationModel.objects.get_or_create( key=indicator_name_en, language='en', translation=indicator_name_en) TranslationModel.objects.get_or_create( key=indicator_name_en, language='fr', translation=indicator_name_fr) if website_en: indicator_data_from_db.website = website_en indicator_data_from_db.save() #we need to store the translations as well TranslationModel.objects.get_or_create( key=website_en, language='en', translation=website_en) TranslationModel.objects.get_or_create( key=website_en, language='fr', translation=website_fr) total_items_saved += 1 line_counter += 1 log = save_log(file=uploaded, uploaded_by_user=request.user, cities_not_found=[], countries_not_found=country_not_found, total_cities_found=[], total_countries_found=country_found, total_cities_not_found=[], total_countries_not_found=country_not_found, total_items_saved=total_items_saved) return { 'url': '/admin/indicator/csvuploadlog/%s/' % str(log.id), 'thumbnail_url': '', 'id': str(log.id), 'name': title, 'country_not_found': log.countries_not_found, 'total_countries_not_found': country_not_found.__len__(), 'city_not_found': log.cities_not_found, 'total_cities_not_found': 0, 'total_items_saved': str(total_items_saved), }
def process_uploaded_file(self, uploaded, object,request, **kwargs): ''' This method will be called for every csv file uploaded. Parameters: :uploaded: instance of uploaded file :object: instance of object if in form_multiupload else None :kwargs: request.POST received with file Return: It MUST return at least a dict with: { 'url': 'url to download the file', 'thumbnail_url': 'some url for an image_thumbnail or icon', 'id': 'id of instance created in this method', 'name': 'the name of created file', } ''' line_counter = 0 country_found = [] country_not_found = [] total_items_saved = 0 countries = get_countries() #getting the title of the file title = kwargs.get('title', [''])[0] or uploaded.name xmlDoc = uploaded xmlDocData = xmlDoc.read() xmlDocTree = etree.XML(xmlDocData) for indicator in xmlDocTree.iter('CountryId'): indicator_name_en = indicator[1].text.rstrip() indicator_name_fr = indicator[2].text.rstrip() indicator_country = indicator[0].text.rstrip() country_iso = indicator.get('countryid').rstrip() value = indicator[3].text.rstrip() type_value = None try: website_en = indicator[4].text.rstrip() website_fr = indicator[5].text.rstrip() except IndexError: website_en = None website_fr = None #try to find the indicator that is uploaded or create a new one indicator_from_db = UnescoIndicator.objects.get_or_create(id=indicator_name_en)[0] #getting country from our database country_from_db = find_country(country_name=indicator_country, countries=countries, iso2=country_iso) #add country to the log array if country_from_db: country_found.append(indicator_country) else: if indicator_country: country_not_found.append(indicator_country) #saving the unesco indicator data if country_from_db: indicator_data_from_db = UnescoIndicatorData.objects.get_or_create(unesco_indicator=indicator_from_db, country=country_from_db, value=value)[0] #storing the translation of the indicator TranslationModel.objects.get_or_create(key=indicator_name_en, language='en', translation=indicator_name_en) TranslationModel.objects.get_or_create(key=indicator_name_en, language='fr', translation=indicator_name_fr) if website_en: indicator_data_from_db.website = website_en indicator_data_from_db.save() #we need to store the translations as well TranslationModel.objects.get_or_create(key=website_en, language='en', translation=website_en) TranslationModel.objects.get_or_create(key=website_en, language='fr', translation=website_fr) total_items_saved += 1 line_counter += 1 log = save_log(file=uploaded, uploaded_by_user=request.user, cities_not_found=[], countries_not_found=country_not_found, total_cities_found=[], total_countries_found=country_found, total_cities_not_found=[], total_countries_not_found=country_not_found, total_items_saved=total_items_saved ) return { 'url': '/admin/indicator/csvuploadlog/%s/' % str(log.id), 'thumbnail_url': '', 'id': str(log.id), 'name' : title, 'country_not_found' : log.countries_not_found, 'total_countries_not_found' : country_not_found.__len__(), 'city_not_found' : log.cities_not_found, 'total_cities_not_found' : 0, 'total_items_saved' : str(total_items_saved), }