def update_regions(self):
country_regions = self.get_json_data(
"/../data_backup/country_regions.json")
for cr in country_regions:
the_country = None
country_iso2 = cr['iso2'].lower()
region_dac_code = cr['dac_region_code'].lower()
region_dac_name = cr['dac_region_name'].lower()
# Get country by iso2
if Country.objects.filter(iso2=country_iso2).exists():
the_country = Country.objects.get(iso2=country_iso2,
primary_name=True)
# Update or create region
if Region.objects.filter(code=region_dac_code).exists():
the_region = Region.objects.get(code=region_dac_code)
the_region.name = region_dac_name
the_region.save()
else:
the_region = Region(code=region_dac_code, name=region_dac_name)
the_region.save()
# Create geolocation record by region
if not Geolocation.objects.filter(tag=region_dac_name).exists():
Geolocation(tag=region_dac_name,
content_object=the_region,
type='region').save()
# Update the region of the country
if the_country:
if the_country.region is None:
the_country.region = the_region
the_country.save()
def update_polygon(self):
admin_countries = self.get_json_data(
"/../data_backup/allcountrycodes.json")
for k in admin_countries: # .get('features'):
country_iso2 = k.get('alpha-2').lower()
name = k.get('name').lower()
country_iso3 = k.get('alpha-3').lower()
if not country_iso2:
continue
c, created = Country.objects.get_or_create(iso2=country_iso2,
iso3=country_iso3,
name=name,
primary_name=True)
if created:
c.save()
# Add or update geolocation
try:
geolocation = Geolocation.objects.get(tag=c.name)
except Geolocation.DoesNotExist:
geolocation = Geolocation(content_object=c,
tag=c.name,
type='country')
geolocation.save()
print('Country : {name}:'.format(name=c.name))
poly_countries = self.get_json_data(
"/../data_backup/country_data.json").get('features')
for k in poly_countries: # .get('features'):
if 'iso2' in k.get('properties'):
iso2 = k.get('properties').get('iso2').lower()
filtered_set = Country.objects.filter(iso2=iso2,
primary_name=True)
if len(filtered_set) > 0:
c = filtered_set[0]
else:
name = k.get('properties').get('name').lower()
c = Country(name=name, iso2=iso2, primary_name=True)
c.save()
c.polygons = json.dumps(k.get('geometry'))
c.save()
# Add or update geolocation
try:
geolocation = Geolocation.objects.get(tag=c.name)
except Geolocation.DoesNotExist:
geolocation = Geolocation(content_object=c,
tag=c.name,
type='country')
geolocation.save()
print('Polygon : {name}:'.format(name=c.name))
def update_alt_name(self):
# We don't use an alternative name anymore
admin_countries = self.get_json_data(
"/../data_backup/alternative_names.json")
for k in admin_countries:
country_iso2 = k.get('iso2').lower()
name = k.get('name').lower()
c, created = Country.objects.get_or_create(name=name,
iso2=country_iso2)
if created:
c.save()
Geolocation(content_object=c, tag=name, type='country').save()
else:
try:
Geolocation.objects.get(tag=name)
except Geolocation.DoesNotExist:
Geolocation(content_object=c, tag=name,
type='country').save()
def update_alt_name(self):
admin_countries = self.get_json_data(
"/../data_backup/alternative_names.json")
for k in admin_countries:
country_iso2 = k.get('iso2').lower()
name = k.get('name').lower()
c, created = Country.objects.get_or_create(name=name,
iso2=country_iso2)
if created:
c.save()
Geolocation(content_object=c, tag=name, type='country').save()
def update_region_center(self):
region_poly = self.get_json_data("/../data_backup/continents.json")
for region in region_poly.get('features'):
name = region.get('properties').get('CONTINENT').lower()
polygon = json.dumps(region.get('geometry'))
instance, created = Region.objects.get_or_create(name=name,
polygons=polygon)
if created:
instance.save()
Geolocation(content_object=instance, tag=name,
type='region').save()
region_centers = self.get_json_data(
"/../data_backup/region_center_locations.json")
'''for r in region_centers:
def update(self):
country = Country.objects.get(iso2=self.iso2_code)
items = get_json_data('/../data_backup/township_nl.json').get(
'features')
for item in items:
properties = item.get('properties')
name = properties.get('name').lower()
city, created = City.objects.get_or_create(
name=name,
country=country,
polygons=json.dumps(item.get('geometry')),
language=self.language)
if created:
Geolocation(content_object=city, tag=name, type='city').save()
print(item.get('properties').get('name').lower())
def update_polygon(self):
kenya_counties = self.get_json_data(
"/../data_backup/kenyan-counties.geojson")
for k in kenya_counties.get('features'):
name = k.get('properties').get('COUNTY').lower()
country = Country.objects.get(name='kenya')
polygons = json.dumps(k.get('geometry'))
c, created = SubNational.objects.get_or_create(name=name,
country=country,
polygons=polygons)
if created:
c.save()
Geolocation(content_object=c, tag=name,
type='subnational').save()
def update(self):
items = get_json_data('/../data_backup/cities.json').get('features')
for item in items:
name = item['properties']['NAME'].lower()
the_country = None
latitude = item['properties']['LATITUDE']
longitude = item['properties']['LONGITUDE']
ascii_name = item['properties']['NAMEASCII']
country_iso2 = item['properties']['ISO_A2']
polygons = json.dumps(item.get('geometry'))
point_loc_str = 'POINT(' + str(longitude) + ' ' + str(
latitude) + ')'
longlat = fromstr(point_loc_str, srid=4326)
country = None
if Country.objects.filter(iso2=country_iso2).exists():
country = Country.objects.get(code=country_iso2)
try:
city = City.objects.get(name=name)
except City.DoesNotExist:
city = City(name=name,
country=country,
ascii_name=ascii_name,
polygons=polygons,
center_longlat=longlat,
language=self.language)
city.save()
try:
Geolocation.objects.get(tag=name)
except Geolocation.DoesNotExist:
Geolocation(content_object=city,
tag=name,
type='city',
polygons=polygons,
center_longlat=longlat).save()
if item['properties']['FEATURECLA'] == "Admin-0 capital":
if the_country:
the_country.capital_city = city
the_country.save()
print(city.name)
def update_pc2(self):
country = Country.objects.get(iso2=self.iso2_code)
items = get_json_data('/../data_backup/PC2.geo.json').get('features')
for item in items:
properties = item.get('properties')
code = properties.get('postalcode').lower()
polygons = json.dumps(item.get('geometry'))
post_code, created = PostCode.objects.get_or_create(
code=code,
country=country,
polygons=polygons,
language=self.language)
if created:
Geolocation(content_object=post_code,
tag=code,
type='postcode',
polygons=polygons).save()
print(code)
def update_country_center(self):
country_centers = self.get_json_data(
"/../data_backup/country_center.json")
for c in country_centers:
if Country.objects.filter(iso2=c.lower()).exists():
current_country = Country.objects.get(iso2=c.lower(),
primary_name=True)
point_loc_str = ''.join([
'POINT(',
str(country_centers[c]["longitude"]), ' ',
str(country_centers[c]["latitude"]), ')'
])
longlat = fromstr(point_loc_str, srid=4326)
current_country.center_longlat = longlat
current_country.save()
Geolocation(tag=current_country.name,
content_object=current_country,
type='country').save()
def get_save_unique_datapoints(df_data,
final_file_headings,
file,
date_format,
filter_headings_dict,
point_based=False):
"""
Gets foreign keys for IndicatorDatapoint and saves new entries if needed.
"""
count = 0
ind_dict = {}
filters_dict = {}
headings_dict = {}
geolocation_dict = {}
value_format_dict = {}
# Important returns a list
unique_indicator = df_data[final_file_headings['indicator']].unique()
unique_geolocation = df_data[final_file_headings['geolocation']].unique()
unique_value_format = df_data[final_file_headings['value_format']].unique()
unique_lists = [
unique_indicator,
unique_geolocation,
unique_value_format,
]
print('Saving unique datapoints %s' % datetime.datetime.now())
for unique_list in unique_lists:
for i in range(len(unique_list)):
if count == 0: # indicator
instance, created = Indicator.objects.get_or_create(
name=unique_list[i], file_source=file.source, file=file)
if created:
instance.save()
ind_dict[unique_list[i]] = instance
for file_heading in filter_headings_dict:
heading_instance, created = \
FilterHeadings.objects.get_or_create(
# df_groupby[heading used in file][row of entry]
# Quite confusing!!
name=filter_headings_dict[file_heading],
indicator=instance)
if created:
heading_instance.save()
heading_index = unique_list[i] + \
filter_headings_dict[file_heading]
headings_dict[heading_index] = heading_instance
elif count == 1: # Location#
if point_based:
lon = float(unique_list[i].split(',')[0])
lat = float(unique_list[i].split(',')[1])
point = Point(lon, lat)
p, created = PointBased.objects.get_or_create(
name=unique_list[i], center_longlat=point)
if created:
p.save()
instance = Geolocation(content_object=p,
tag=unique_list[i],
type='pointbased')
instance.save()
else:
instance = Geolocation.objects.get(tag=unique_list[i])
else:
if isinstance(unique_list[i], str):
if len(unique_list[i]) == 2:
instance = Geolocation.objects.get(
iso2=unique_list[i])
elif len(unique_list[i]) == 3:
instance = Geolocation.objects.get(
iso3=unique_list[i])
else:
instance = Geolocation.objects.get(
tag=unique_list[i])
else:
instance = Geolocation.objects.get(tag=unique_list[i])
geolocation_dict[unique_list[i]] = instance # shold use get?
else:
instance, created = ValueFormat.objects.get_or_create(
type=unique_list[i])
if created:
instance.save()
value_format_dict[unique_list[i]] = instance
count += 1
# Save filters
print('Saving filters')
for j in range(len(final_file_headings['filters'])):
unique_filters = df_data.groupby([
final_file_headings['indicator'],
# Important returns a dataframe
final_file_headings['filters'][j]
]).size().reset_index()
unique_filter_value_formats = df_data.groupby([
final_file_headings['indicator'],
final_file_headings['filters'][j],
final_file_headings['value_format'],
]).size().reset_index()
unique_filter_date_formats = df_data.groupby([
final_file_headings['indicator'],
final_file_headings['filters'][j],
final_file_headings['date_format'],
]).size().reset_index()
unique_filter_geolocation_formats = df_data.groupby([
final_file_headings['indicator'],
final_file_headings['filters'][j],
final_file_headings['geolocation'],
]).size().reset_index()
for i in range(len(unique_filters)):
filter_name = unique_filters[final_file_headings['filters'][j]][i]
indicator = unique_filters[final_file_headings['indicator']][i]
filter_heading_name = \
filter_headings_dict[final_file_headings['filters'][j]]
instance, created = Filters.objects.get_or_create(
name=filter_name,
heading=headings_dict[indicator + filter_heading_name],
indicator=ind_dict[indicator])
if created:
instance.save()
filters_dict[indicator + filter_heading_name +
filter_name] = instance
# Loop here
instance.value_format = value_format_dict[
unique_filter_value_formats[
final_file_headings['value_format']][i]]
# [df_data[final_file_headings['date_format']][i]] +
# instance.date_formats
instance.date_format = date_format
instance.metadata = df_data['metadata'][0]
geo_filter = unique_filter_geolocation_formats[
final_file_headings['filters'][j]] == unique_filters[
final_file_headings['filters'][j]][i]
geolocation_list = list(
unique_filter_geolocation_formats[geo_filter][
final_file_headings['geolocation']].map(
geolocation_dict)) # TODO Vectorise
instance.geolocations.add(*geolocation_list)
instance.save()
# unique list is not always the same datatype, consists of lists and
# dataframes
print('Finsihed with unique datapoints %s' % datetime.datetime.now())
return ind_dict, headings_dict, geolocation_dict, value_format_dict, \
filters_dict