def get_country_id(self, data):
        """
        Try to get country id by name from data
        """
        country = find_country(data.get('country', None), self.countries)

        try:
            return country.code
        except AttributeError:
            try:
                self.stderr.write("Can not find country '{country}'".format(
                    country=data.get('country', None)))
            except UnicodeDecodeError:
                pass
            return None
Beispiel #2
0
    def process_uploaded_file(self, uploaded, object, request, **kwargs):
        '''
        This method will be called for every csv file uploaded.
        Parameters:
            :uploaded: instance of uploaded file
            :object: instance of object if in form_multiupload else None
            :kwargs: request.POST received with file
        Return:
            It MUST return at least a dict with:
            {
                'url': 'url to download the file',
                'thumbnail_url': 'some url for an image_thumbnail or icon',
                'id': 'id of instance created in this method',
                'name': 'the name of created file',
            }
        '''
        try:
            #getting the title of the file
            title = kwargs.get('title', [''])[0] or uploaded.name

            import csv
            try:
                dialect = csv.Sniffer().sniff(uploaded.read(4048))
            except csv.Error:
                dialect = csv.excel

            file = csv.DictReader(uploaded, dialect=dialect)

            line_counter = 0
            indicator_from_db = None
            city_found = []
            city_not_found = []
            country_found = []
            country_not_found = []
            total_items_saved = 0

            cities = get_cities()
            countries = get_countries()
            for line in file:
                #getting data from the csv file
                city_csv = line.get('city')
                deprivation_type_csv = line.get('deprivation_type')
                description_csv = line.get('description')
                selection_type_csv = line.get('selection_type')
                country_csv = line.get('country')
                region_csv = line.get('region_csv')
                friendly_label_csv = line.get('friendly_name')
                value_csv = line.get('value')
                year_range_csv = line.get('year_range')
                indicator_id_csv = line.get('indicator_id')
                year_csv = line.get('year')
                type_data_csv = line.get('type_data')
                category_csv = line.get('category')

                value_csv = str(value_csv)
                value_csv = value_csv.replace(".", ",")

                if value_csv == None or value_csv == "NULL":
                    continue

                #here we are checking if this indicator already exists, or if we have to create a new one
                if line_counter == 0:
                    #try to find the indicator that is uploaded or create a new one
                    if Indicator.objects.filter(id=indicator_id_csv).exists():
                        indicator_from_db = Indicator.objects.get(
                            id=indicator_id_csv)
                    else:
                        # create new indicator
                        try:
                            indicator_from_db = Indicator(
                                id=indicator_id_csv,
                                description=description_csv,
                                friendly_label=friendly_label_csv,
                                type_data=type_data_csv,
                                deprivation_type=deprivation_type_csv,
                                category=category_csv)
                            indicator_from_db.save()
                        except Exception as e:
                            print e

                #getting country from our database
                country_from_db = find_country(country_name=country_csv,
                                               countries=countries)

                #add country to the log array
                if country_from_db:
                    country_found.append(country_csv)
                    country_id = country_from_db.code
                else:
                    if country_csv:
                        country_not_found.append(country_csv)
                        country_id = None
                    else:
                        country_id = None

                city_from_db = find_city(city_name=city_csv,
                                         cities=cities,
                                         country_id=country_id)

                #add city to the log array
                if city_from_db:
                    city_found.append(city_csv)
                else:
                    if city_csv:
                        city_not_found.append(city_csv)

                try:
                    if city_from_db:
                        #this block is for storing data related to cities
                        if save_city_data(
                                city_from_db=city_from_db,
                                country_from_db=country_from_db,
                                selection_type_csv=selection_type_csv,
                                indicator_from_db=indicator_from_db,
                                year_csv=year_csv,
                                value_csv=value_csv):
                            total_items_saved += 1
                    elif country_from_db:
                        #this block is for storing country related indicator data
                        if save_country_data(
                                country_from_db=country_from_db,
                                city_csv=city_csv,
                                selection_type_csv=selection_type_csv,
                                year_csv=year_csv,
                                indicator_from_db=indicator_from_db,
                                value_csv=value_csv):
                            total_items_saved += 1

                    line_counter += 1
                except Exception as e:
                    print e

            log = save_log(file=uploaded,
                           uploaded_by_user=request.user,
                           cities_not_found=city_not_found,
                           countries_not_found=country_not_found,
                           total_cities_found=city_found,
                           total_countries_found=country_found,
                           total_cities_not_found=city_not_found,
                           total_countries_not_found=country_not_found,
                           total_items_saved=total_items_saved)

            return {
                'url': '/admin/indicator/csvuploadlog/%s/' % str(log.id),
                'thumbnail_url': '',
                'id': str(log.id),
                'name': title,
                'country_not_found': log.countries_not_found,
                'total_countries_not_found': country_not_found.__len__(),
                'city_not_found': log.cities_not_found,
                'total_cities_not_found': city_not_found.__len__(),
                'total_items_saved': str(total_items_saved),
            }

        except Exception as e:
            print e
Beispiel #3
0
    def process_uploaded_file(self, uploaded, object, request, **kwargs):
        '''
        This method will be called for every csv file uploaded.
        Parameters:
            :uploaded: instance of uploaded file
            :object: instance of object if in form_multiupload else None
            :kwargs: request.POST received with file
        Return:
            It MUST return at least a dict with:
            {
                'url': 'url to download the file',
                'thumbnail_url': 'some url for an image_thumbnail or icon',
                'id': 'id of instance created in this method',
                'name': 'the name of created file',
            }
        '''
        line_counter = 0
        country_found = []
        country_not_found = []
        total_items_saved = 0
        countries = get_countries()

        #getting the title of the file
        title = kwargs.get('title', [''])[0] or uploaded.name

        xmlDoc = uploaded
        xmlDocData = xmlDoc.read()
        xmlDocTree = etree.XML(xmlDocData)

        for indicator in xmlDocTree.iter('CountryId'):
            indicator_name_en = indicator[1].text.rstrip()
            indicator_name_fr = indicator[2].text.rstrip()
            indicator_country = indicator[0].text.rstrip()
            country_iso = indicator.get('countryid').rstrip()
            value = indicator[3].text.rstrip()
            type_value = None
            try:
                website_en = indicator[4].text.rstrip()
                website_fr = indicator[5].text.rstrip()
            except IndexError:
                website_en = None
                website_fr = None

            #try to find the indicator that is uploaded or create a new one
            indicator_from_db = UnescoIndicator.objects.get_or_create(
                id=indicator_name_en)[0]

            #getting country from our database
            country_from_db = find_country(country_name=indicator_country,
                                           countries=countries,
                                           iso2=country_iso)

            #add country to the log array
            if country_from_db:
                country_found.append(indicator_country)
            else:
                if indicator_country:
                    country_not_found.append(indicator_country)

            #saving the unesco indicator data
            if country_from_db:
                indicator_data_from_db = UnescoIndicatorData.objects.get_or_create(
                    unesco_indicator=indicator_from_db,
                    country=country_from_db,
                    value=value)[0]

                #storing the translation of the indicator
                TranslationModel.objects.get_or_create(
                    key=indicator_name_en,
                    language='en',
                    translation=indicator_name_en)
                TranslationModel.objects.get_or_create(
                    key=indicator_name_en,
                    language='fr',
                    translation=indicator_name_fr)

                if website_en:
                    indicator_data_from_db.website = website_en
                    indicator_data_from_db.save()

                    #we need to store the translations as well
                    TranslationModel.objects.get_or_create(
                        key=website_en, language='en', translation=website_en)
                    TranslationModel.objects.get_or_create(
                        key=website_en, language='fr', translation=website_fr)

                total_items_saved += 1

            line_counter += 1

        log = save_log(file=uploaded,
                       uploaded_by_user=request.user,
                       cities_not_found=[],
                       countries_not_found=country_not_found,
                       total_cities_found=[],
                       total_countries_found=country_found,
                       total_cities_not_found=[],
                       total_countries_not_found=country_not_found,
                       total_items_saved=total_items_saved)

        return {
            'url': '/admin/indicator/csvuploadlog/%s/' % str(log.id),
            'thumbnail_url': '',
            'id': str(log.id),
            'name': title,
            'country_not_found': log.countries_not_found,
            'total_countries_not_found': country_not_found.__len__(),
            'city_not_found': log.cities_not_found,
            'total_cities_not_found': 0,
            'total_items_saved': str(total_items_saved),
        }
Beispiel #4
0
    def process_uploaded_file(self, uploaded, object,request, **kwargs):
        '''
        This method will be called for every csv file uploaded.
        Parameters:
            :uploaded: instance of uploaded file
            :object: instance of object if in form_multiupload else None
            :kwargs: request.POST received with file
        Return:
            It MUST return at least a dict with:
            {
                'url': 'url to download the file',
                'thumbnail_url': 'some url for an image_thumbnail or icon',
                'id': 'id of instance created in this method',
                'name': 'the name of created file',
            }
        '''
        try:
            #getting the title of the file
            title = kwargs.get('title', [''])[0] or uploaded.name

            import csv
            try:
                dialect = csv.Sniffer().sniff(uploaded.read(4048))
            except csv.Error:
                dialect = csv.excel

            file = csv.DictReader(uploaded, dialect=dialect)

            line_counter = 0
            indicator_from_db = None
            city_found = []
            city_not_found = []
            country_found = []
            country_not_found = []
            total_items_saved = 0

            cities = get_cities()
            countries = get_countries()
            for line in file:
                #getting data from the csv file
                city_csv = line.get('city')
                deprivation_type_csv = line.get('deprivation_type')
                description_csv = line.get('description')
                selection_type_csv = line.get('selection_type')
                country_csv = line.get('country')
                region_csv = line.get('region_csv')
                friendly_label_csv = line.get('friendly_name')
                value_csv = line.get('value')
                year_range_csv = line.get('year_range')
                indicator_id_csv = line.get('indicator_id')
                year_csv = line.get('year')
                type_data_csv = line.get('type_data')
                category_csv = line.get('category')

                value_csv = str(value_csv)
                value_csv = value_csv.replace(".", ",")


                if value_csv == None or value_csv == "NULL":
                    continue


                #here we are checking if this indicator already exists, or if we have to create a new one
                if line_counter == 0:
                    #try to find the indicator that is uploaded or create a new one
                    if Indicator.objects.filter(id=indicator_id_csv).exists():
                        indicator_from_db = Indicator.objects.get(id=indicator_id_csv)
                    else:
                        # create new indicator
                        try:
                            indicator_from_db = Indicator(id=indicator_id_csv, description=description_csv, friendly_label=friendly_label_csv, type_data=type_data_csv, deprivation_type=deprivation_type_csv, category=category_csv)
                            indicator_from_db.save()
                        except Exception as e:
                            print(e)

                #getting country from our database
                country_from_db = find_country(country_name=country_csv, countries=countries)

                #add country to the log array
                if country_from_db:
                    country_found.append(country_csv)
                    country_id = country_from_db.code
                else:
                    if country_csv:
                        country_not_found.append(country_csv)
                        country_id = None
                    else:
                        country_id = None

                city_from_db = find_city(city_name=city_csv, cities=cities, country_id=country_id)

                #add city to the log array
                if city_from_db:
                    city_found.append(city_csv)
                else:
                    if city_csv:
                        city_not_found.append(city_csv)

                try:
                    if city_from_db:
                        #this block is for storing data related to cities
                        if save_city_data(
                            city_from_db=city_from_db,
                            country_from_db=country_from_db,
                            selection_type_csv=selection_type_csv,
                            indicator_from_db=indicator_from_db,
                            year_csv=year_csv,
                            value_csv=value_csv
                        ): total_items_saved += 1
                    elif country_from_db:
                        #this block is for storing country related indicator data
                        if save_country_data(
                                country_from_db=country_from_db,
                                city_csv=city_csv,
                                selection_type_csv=selection_type_csv,
                                year_csv=year_csv,
                                indicator_from_db=indicator_from_db,
                                value_csv=value_csv
                        ): total_items_saved += 1

                    line_counter += 1
                except Exception as e:
                    print(e)

            log = save_log(file=uploaded,
                     uploaded_by_user=request.user,
                     cities_not_found=city_not_found,
                     countries_not_found=country_not_found,
                     total_cities_found=city_found,
                     total_countries_found=country_found,
                     total_cities_not_found=city_not_found,
                     total_countries_not_found=country_not_found,
                     total_items_saved=total_items_saved
            )


            return {
                'url': '/admin/indicator/csvuploadlog/%s/' % str(log.id),
                'thumbnail_url': '',
                'id': str(log.id),
                'name' : title,
                'country_not_found' : log.countries_not_found,
                'total_countries_not_found' : country_not_found.__len__(),
                'city_not_found' : log.cities_not_found,
                'total_cities_not_found' : city_not_found.__len__(),
                'total_items_saved' : str(total_items_saved),

            }

        except Exception as e:
                        print(e)
Beispiel #5
0
    def process_uploaded_file(self, uploaded, object,request, **kwargs):
        '''
        This method will be called for every csv file uploaded.
        Parameters:
            :uploaded: instance of uploaded file
            :object: instance of object if in form_multiupload else None
            :kwargs: request.POST received with file
        Return:
            It MUST return at least a dict with:
            {
                'url': 'url to download the file',
                'thumbnail_url': 'some url for an image_thumbnail or icon',
                'id': 'id of instance created in this method',
                'name': 'the name of created file',
            }
        '''
        line_counter = 0
        country_found = []
        country_not_found = []
        total_items_saved = 0
        countries = get_countries()

        #getting the title of the file
        title = kwargs.get('title', [''])[0] or uploaded.name

        xmlDoc = uploaded
        xmlDocData = xmlDoc.read()
        xmlDocTree = etree.XML(xmlDocData)

        for indicator in xmlDocTree.iter('CountryId'):
            indicator_name_en = indicator[1].text.rstrip()
            indicator_name_fr = indicator[2].text.rstrip()
            indicator_country = indicator[0].text.rstrip()
            country_iso = indicator.get('countryid').rstrip()
            value = indicator[3].text.rstrip()
            type_value = None
            try:
                website_en = indicator[4].text.rstrip()
                website_fr = indicator[5].text.rstrip()
            except IndexError:
                website_en = None
                website_fr = None

            #try to find the indicator that is uploaded or create a new one
            indicator_from_db = UnescoIndicator.objects.get_or_create(id=indicator_name_en)[0]

            #getting country from our database
            country_from_db = find_country(country_name=indicator_country, countries=countries, iso2=country_iso)

            #add country to the log array
            if country_from_db:
                country_found.append(indicator_country)
            else:
                if indicator_country:
                    country_not_found.append(indicator_country)

            #saving the unesco indicator data
            if country_from_db:
                indicator_data_from_db = UnescoIndicatorData.objects.get_or_create(unesco_indicator=indicator_from_db, country=country_from_db, value=value)[0]

                #storing the translation of the indicator
                TranslationModel.objects.get_or_create(key=indicator_name_en, language='en', translation=indicator_name_en)
                TranslationModel.objects.get_or_create(key=indicator_name_en, language='fr', translation=indicator_name_fr)

                if website_en:
                    indicator_data_from_db.website = website_en
                    indicator_data_from_db.save()

                    #we need to store the translations as well
                    TranslationModel.objects.get_or_create(key=website_en, language='en', translation=website_en)
                    TranslationModel.objects.get_or_create(key=website_en, language='fr', translation=website_fr)


                total_items_saved += 1


            line_counter += 1


        log = save_log(file=uploaded,
                 uploaded_by_user=request.user,
                 cities_not_found=[],
                 countries_not_found=country_not_found,
                 total_cities_found=[],
                 total_countries_found=country_found,
                 total_cities_not_found=[],
                 total_countries_not_found=country_not_found,
                 total_items_saved=total_items_saved
        )


        return {
            'url': '/admin/indicator/csvuploadlog/%s/' % str(log.id),
            'thumbnail_url': '',
            'id': str(log.id),
            'name' : title,
            'country_not_found' : log.countries_not_found,
            'total_countries_not_found' : country_not_found.__len__(),
            'city_not_found' : log.cities_not_found,
            'total_cities_not_found' : 0,
            'total_items_saved' : str(total_items_saved),

        }