def save_region_data(self, regions_data_list):
for dictionary in regions_data_list:
code = str(dictionary['Region_Id'])
number = dictionary['Region_Num']
name = clean_name(dictionary['Region_Name'])
short_name = dictionary['Region_Short']
capital = dictionary['Region_Center']
if capital:
capital = clean_name(capital)
region = self.regions_dict.get(name)
if region:
update_fields = []
if region.code != code:
region.code = code
update_fields.append('code')
if region.number != number:
region.number = number
update_fields.append('number')
if region.short_name != short_name:
region.short_name = short_name
update_fields.append('short_name')
if update_fields:
update_fields.append('updated_at')
region.save(update_fields=update_fields)
else:
region = DrvRegion.objects.create(
code=code,
number=number,
name=name,
short_name=short_name,
capital=capital
)
self.regions_dict[name] = region
atos_data_list = self.parse_atos_data(region)
self.save_ato_data(atos_data_list, region)
def save_region_data(self, regions_data_list):
for dictionary in regions_data_list:
code = dictionary['Region_Id']
number = dictionary['Region_Num']
name = dictionary['Region_Name']
name = clean_name(name)
short_name = dictionary['Region_Short']
capital = dictionary['Region_Center']
if capital:
capital = clean_name(capital)
if name in self.all_regions_dict:
region = self.all_regions_dict[name]
region.code = code
region.number = number
region.short_name = short_name
region.capital = capital
else:
region = DrvRegion(code=code,
number=number,
name=name,
short_name=short_name,
capital=capital)
self.all_regions_dict[name] = region
region.save()
print(f'Дані регіону {region.name} збережено')
atos_data_list = self.parse_atos_data(region)
self.save_ato_data(atos_data_list, region)
def save_ato_data(self, atos_data_list, region):
for dictionary in atos_data_list:
district_name = dictionary['ATO_Raj']
district_name = change_to_full_name(district_name)
district_name = clean_name(district_name)
council_name = dictionary['ATO_Rad']
ato_name = dictionary['ATO_Name']
ato_name = clean_name(ato_name)
# converting an integer value to string for comparing with codes from DB that are strings
ato_code = str(dictionary['ATO_Id'])
district = DrvDistrict.objects.filter(region=region, name=district_name).first()
if not district:
district = DrvDistrict(region=region, name=district_name)
district.save()
council = DrvCouncil.objects.filter(region=region, name=council_name).first()
if not council:
council = DrvCouncil(region=region, name=council_name)
council.save()
if ato_code in self.all_atos_dict:
ato = self.all_atos_dict[ato_code]
ato.region=region
ato.district=district
ato.council=council
ato.name = ato_name
else:
ato = DrvAto(region=region, district=district, council=council, name=ato_name, code=ato_code)
self.all_atos_dict[ato_code] = ato
ato.save()
print(f'Дані населеного пункту {ato.name} збережено')
streets_data_list = self.parse_streets_data(ato)
self.save_street_data(streets_data_list, region, district, council, ato)
def save_ato_data(self, atos_data_list, region):
for dictionary in atos_data_list:
district_name = dictionary['ATO_Raj']
district_name = change_to_full_name(district_name)
district_name = clean_name(district_name)
district_code = region.name + district_name
council_name = dictionary['ATO_Rad'].lower()
council_code = region.name + district_name + council_name
ato_name = dictionary['ATO_Name']
ato_name = clean_name(ato_name)
ato_code = str(dictionary['ATO_Id'])
district = self.districts_dict.get(district_code)
if district:
if self.outdated_districts_dict.get(district_code):
del self.outdated_districts_dict[district_code]
else:
district = DrvDistrict.objects.create(region=region,
name=district_name,
code=district_code)
self.districts_dict[district_code] = district
council = self.councils_dict.get(council_code)
if council:
if self.outdated_councils_dict.get(council_code):
del self.outdated_councils_dict[council_code]
else:
council = DrvCouncil.objects.create(region=region,
name=council_name,
code=council_code)
self.councils_dict[council_code] = council
ato = self.atos_dict.get(ato_code)
if ato:
update_fields = []
if ato.region_id != region.id:
ato.region = region
update_fields.append('region')
if ato.district_id != district.id:
ato.district = district
update_fields.append('district')
if ato.council_id != council.id:
ato.council = council
update_fields.append('council')
if ato.name != ato_name:
ato.name = ato_name
update_fields.append('name')
if update_fields:
update_fields.append('updated_at')
ato.save(update_fields=update_fields)
if self.outdated_atos_dict.get(ato_code):
del self.outdated_atos_dict[ato_code]
else:
ato = DrvAto.objects.create(region=region,
district=district,
council=council,
name=ato_name,
code=ato_code)
self.atos_dict[ato_code] = ato
streets_data_list = self.parse_streets_data(ato)
self.save_street_data(streets_data_list, region, district, council,
ato)
def save_to_db(self, json_file):
data = self.load_json(json_file)
for object_koatuu in data:
first_level_code = object_koatuu['Перший рівень']
# checking that all codes are strings, not integers that happens
if not isinstance(first_level_code, str):
first_level_code = str(first_level_code)
second_level_code = object_koatuu['Другий рівень']
if not isinstance(second_level_code, str):
second_level_code = str(second_level_code)
third_level_code = object_koatuu['Третій рівень']
if not isinstance(third_level_code, str):
third_level_code = str(third_level_code)
fourth_level_code = object_koatuu['Четвертий рівень']
if not isinstance(fourth_level_code, str):
fourth_level_code = str(fourth_level_code)
category_code = object_koatuu['Категорія']
category = None
if category_code:
category = self.save_or_get_category(category_code)
name = object_koatuu["Назва об'єкта українською мовою"]
if first_level_code and not second_level_code:
name = clean_name(
get_lowercase_substring_before_slash(name)
)
self.save_or_update_first_level(name, first_level_code)
continue
if first_level_code and second_level_code and not third_level_code:
name = change_to_full_name(
clean_name(
get_lowercase_substring_before_slash(name))
)
self.save_or_update_second_level(first_level_code, category, name,
second_level_code)
continue
if (first_level_code and second_level_code and third_level_code and
not fourth_level_code):
name = change_to_full_name(
clean_name(
get_lowercase_substring_before_slash(name))
)
self.save_or_update_third_level(first_level_code, second_level_code, category,
name, third_level_code)
continue
if (first_level_code and second_level_code and third_level_code and
fourth_level_code):
# omitting the incorrect record from data.gov.ua
# ToDo: get explanation from the government
if third_level_code == '2320381000':
logger.warning(f'Код третього рівня не існує в json: {object_koatuu}')
continue
name = name.lower()
self.save_or_update_fourth_level(first_level_code, second_level_code,
third_level_code, category, name,
fourth_level_code)
def save_or_get_region(self, name):
region_name = clean_name(name)
region_name = change_to_full_name(region_name)
if region_name not in self.all_regions_dict:
region = RatuRegion.objects.create(name=region_name)
self.all_regions_dict[region_name] = region
return region
return self.all_regions_dict[region_name]
def save_to_district_table(self, record, region):
if record['REGION_NAME']:
district_name = clean_name(record['REGION_NAME'])
district_name = change_to_full_name(district_name)
else:
district_name = District.EMPTY_FIELD
if [region.id, district_name] not in self.district_list:
district = District(region=region, name=district_name)
district.save()
self.district_list.insert(0, [region.id, district_name])
district = District.objects.get(name=district_name, region=region.id)
return district
def save_to_city_table(self, record, region, district):
if record['CITY_NAME']:
city_name = clean_name(record['CITY_NAME'])
else:
city_name = City.EMPTY_FIELD
if [region.id, district.id, city_name] not in self.city_list:
city = City(region=region, district=district, name=city_name)
city.save()
self.city_list.insert(0, [region.id, district.id, city_name])
city = City.objects.get(name=city_name,
region=region.id,
district=district.id)
return city
def save_or_get_district(self, name, region):
district_name = clean_name(name)
district_name = change_to_full_name(district_name)
district_code = region.name + district_name
if district_code not in self.all_districts_dict:
district = RatuDistrict.objects.create(region=region,
name=district_name,
code=district_code)
self.all_districts_dict[district_code] = district
return district
else:
if self.outdated_districts_dict.get(district_code):
del self.outdated_districts_dict[district_code]
return self.all_districts_dict[district_code]
def save_or_get_city(self, name, region, district):
city_name = clean_name(name)
district_name = 'EMPTY' if not district else district.name
city_code = region.name + district_name + city_name
if city_code not in self.all_cities_dict:
city = RatuCity.objects.create(region=region,
district=district,
name=city_name,
code=city_code)
self.all_cities_dict[city_code] = city
return city
else:
if self.outdated_cities_dict.get(city_code):
del self.outdated_cities_dict[city_code]
return self.all_cities_dict[city_code]
def save_or_get_citydistrict(self, name, region, district, city):
citydistrict_name = clean_name(name)
citydistrict_code = region.name + city.name + citydistrict_name
if citydistrict_code not in self.all_citydistricts_dict:
citydistrict = RatuCityDistrict.objects.create(
region=region,
district=district,
city=city,
name=citydistrict_name,
code=citydistrict_code)
self.all_citydistricts_dict[citydistrict_code] = citydistrict
return citydistrict
else:
if self.outdated_citydistricts_dict.get(citydistrict_code):
del self.outdated_citydistricts_dict[citydistrict_code]
return self.all_citydistricts_dict[citydistrict_code]
