def save_street_data(self, streets_data_list, region, district, council,
ato):
for dictionary in streets_data_list:
# converting an integer value to string for comparing with codes from DB that are strings
code = str(dictionary['Geon_Id'])
name = dictionary['Geon_Name']
name = change_to_full_name(name)
previous_name = dictionary['Geon_OldNames']
if previous_name:
previous_name = change_to_full_name(previous_name)
buildings_info = dictionary['BUILDS']
number_of_buildings = None
buildings_data_list = None
if buildings_info:
number_of_buildings = dictionary['BUILDS']['BUILDS_COUNT']
buildings_data_list = dictionary['BUILDS']['BUILD']
street = DrvStreet.objects.filter(code=code).first()
if not street:
street = DrvStreet(region=region,
district=district,
council=council,
ato=ato,
code=code,
name=name,
previous_name=previous_name,
number_of_buildings=number_of_buildings)
else:
street.name = name
street.previous_name = previous_name
street.number_of_buildings = number_of_buildings
street.save()
if buildings_data_list:
self.save_building_data(buildings_data_list, region, district,
council, ato, street)
def save_street_data(self, streets_data_list, region, district, council,
ato):
for dictionary in streets_data_list:
code = str(dictionary['Geon_Id'])
name = dictionary['Geon_Name']
name = change_to_full_name(name)
previous_name = dictionary['Geon_OldNames']
if previous_name:
previous_name = change_to_full_name(previous_name)
buildings_info = dictionary['BUILDS']
number_of_buildings = None
buildings_data_list = None
if buildings_info:
number_of_buildings = dictionary['BUILDS']['BUILDS_COUNT']
buildings_data_list = dictionary['BUILDS']['BUILD']
street = self.streets_dict.get(code)
if street:
update_fields = []
if street.region_id != region.id:
street.region = region
update_fields.append('region')
if street.district_id != district.id:
street.district = district
update_fields.append('district')
if street.council_id != council.id:
street.council = council
update_fields.append('council')
if street.ato_id != ato.id:
street.ato = ato
update_fields.append('ato')
if street.name != name:
street.name = name
update_fields.append('name')
if street.previous_name != previous_name:
street.previous_name = previous_name
update_fields.append('previous_name')
if street.number_of_buildings != number_of_buildings:
street.number_of_buildings = number_of_buildings
update_fields.append('number_of_buildings')
if update_fields:
update_fields.append('updated_at')
street.save(update_fields=update_fields)
if self.outdated_streets_dict.get(code):
del self.outdated_streets_dict[code]
else:
street = DrvStreet.objects.create(
region=region,
district=district,
council=council,
ato=ato,
code=code,
name=name,
previous_name=previous_name,
number_of_buildings=number_of_buildings)
self.streets_dict[code] = street
if buildings_data_list:
self.save_building_data(buildings_data_list, region, district,
council, ato, street)
def save_to_db(self, json_file):
data = self.load_json(json_file)
for object_koatuu in data:
first_level_code = object_koatuu['Перший рівень']
# checking that all codes are strings, not integers that happens
if not isinstance(first_level_code, str):
first_level_code = str(first_level_code)
second_level_code = object_koatuu['Другий рівень']
if not isinstance(second_level_code, str):
second_level_code = str(second_level_code)
third_level_code = object_koatuu['Третій рівень']
if not isinstance(third_level_code, str):
third_level_code = str(third_level_code)
fourth_level_code = object_koatuu['Четвертий рівень']
if not isinstance(fourth_level_code, str):
fourth_level_code = str(fourth_level_code)
category_code = object_koatuu['Категорія']
category = None
if category_code:
category = self.save_or_get_category(category_code)
name = object_koatuu["Назва об'єкта українською мовою"]
if first_level_code and not second_level_code:
name = clean_name(
get_lowercase_substring_before_slash(name)
)
self.save_or_update_first_level(name, first_level_code)
continue
if first_level_code and second_level_code and not third_level_code:
name = change_to_full_name(
clean_name(
get_lowercase_substring_before_slash(name))
)
self.save_or_update_second_level(first_level_code, category, name,
second_level_code)
continue
if (first_level_code and second_level_code and third_level_code and
not fourth_level_code):
name = change_to_full_name(
clean_name(
get_lowercase_substring_before_slash(name))
)
self.save_or_update_third_level(first_level_code, second_level_code, category,
name, third_level_code)
continue
if (first_level_code and second_level_code and third_level_code and
fourth_level_code):
# omitting the incorrect record from data.gov.ua
# ToDo: get explanation from the government
if third_level_code == '2320381000':
logger.warning(f'Код третього рівня не існує в json: {object_koatuu}')
continue
name = name.lower()
self.save_or_update_fourth_level(first_level_code, second_level_code,
third_level_code, category, name,
fourth_level_code)
def save_ato_data(self, atos_data_list, region):
for dictionary in atos_data_list:
district_name = dictionary['ATO_Raj']
district_name = change_to_full_name(district_name)
district_name = clean_name(district_name)
council_name = dictionary['ATO_Rad']
ato_name = dictionary['ATO_Name']
ato_name = clean_name(ato_name)
# converting an integer value to string for comparing with codes from DB that are strings
ato_code = str(dictionary['ATO_Id'])
district = DrvDistrict.objects.filter(region=region, name=district_name).first()
if not district:
district = DrvDistrict(region=region, name=district_name)
district.save()
council = DrvCouncil.objects.filter(region=region, name=council_name).first()
if not council:
council = DrvCouncil(region=region, name=council_name)
council.save()
if ato_code in self.all_atos_dict:
ato = self.all_atos_dict[ato_code]
ato.region=region
ato.district=district
ato.council=council
ato.name = ato_name
else:
ato = DrvAto(region=region, district=district, council=council, name=ato_name, code=ato_code)
self.all_atos_dict[ato_code] = ato
ato.save()
print(f'Дані населеного пункту {ato.name} збережено')
streets_data_list = self.parse_streets_data(ato)
self.save_street_data(streets_data_list, region, district, council, ato)
def save_ato_data(self, atos_data_list, region):
for dictionary in atos_data_list:
district_name = dictionary['ATO_Raj']
district_name = change_to_full_name(district_name)
district_name = clean_name(district_name)
district_code = region.name + district_name
council_name = dictionary['ATO_Rad'].lower()
council_code = region.name + district_name + council_name
ato_name = dictionary['ATO_Name']
ato_name = clean_name(ato_name)
ato_code = str(dictionary['ATO_Id'])
district = self.districts_dict.get(district_code)
if district:
if self.outdated_districts_dict.get(district_code):
del self.outdated_districts_dict[district_code]
else:
district = DrvDistrict.objects.create(region=region,
name=district_name,
code=district_code)
self.districts_dict[district_code] = district
council = self.councils_dict.get(council_code)
if council:
if self.outdated_councils_dict.get(council_code):
del self.outdated_councils_dict[council_code]
else:
council = DrvCouncil.objects.create(region=region,
name=council_name,
code=council_code)
self.councils_dict[council_code] = council
ato = self.atos_dict.get(ato_code)
if ato:
update_fields = []
if ato.region_id != region.id:
ato.region = region
update_fields.append('region')
if ato.district_id != district.id:
ato.district = district
update_fields.append('district')
if ato.council_id != council.id:
ato.council = council
update_fields.append('council')
if ato.name != ato_name:
ato.name = ato_name
update_fields.append('name')
if update_fields:
update_fields.append('updated_at')
ato.save(update_fields=update_fields)
if self.outdated_atos_dict.get(ato_code):
del self.outdated_atos_dict[ato_code]
else:
ato = DrvAto.objects.create(region=region,
district=district,
council=council,
name=ato_name,
code=ato_code)
self.atos_dict[ato_code] = ato
streets_data_list = self.parse_streets_data(ato)
self.save_street_data(streets_data_list, region, district, council,
ato)
def save_or_get_region(self, name):
region_name = clean_name(name)
region_name = change_to_full_name(region_name)
if region_name not in self.all_regions_dict:
region = RatuRegion.objects.create(name=region_name)
self.all_regions_dict[region_name] = region
return region
return self.all_regions_dict[region_name]
def save_to_region_table(self, record):
record['OBL_NAME'] = change_to_full_name(record['OBL_NAME'])
if record['OBL_NAME'] not in self.region_dict:
region = Region(name=record['OBL_NAME'])
region.save()
self.region_dict[record['OBL_NAME']] = region
return region
region = self.region_dict[record['OBL_NAME']]
return region
def save_to_district_table(self, record, region):
if record['REGION_NAME']:
district_name = clean_name(record['REGION_NAME'])
district_name = change_to_full_name(district_name)
else:
district_name = District.EMPTY_FIELD
if [region.id, district_name] not in self.district_list:
district = District(region=region, name=district_name)
district.save()
self.district_list.insert(0, [region.id, district_name])
district = District.objects.get(name=district_name, region=region.id)
return district
def save_or_get_district(self, name, region):
district_name = clean_name(name)
district_name = change_to_full_name(district_name)
district_code = region.name + district_name
if district_code not in self.all_districts_dict:
district = RatuDistrict.objects.create(region=region,
name=district_name,
code=district_code)
self.all_districts_dict[district_code] = district
return district
else:
if self.outdated_districts_dict.get(district_code):
del self.outdated_districts_dict[district_code]
return self.all_districts_dict[district_code]
def save_street(self, name, region, district, city, citydistrict):
street_name = change_to_full_name(name)
# Saving streets that are located in Kyiv and Sevastopol that are regions
if not city:
city = self.save_or_get_city(region.name, region, district)
district_name = 'EMPTY' if not district else district.name
citydistrict_name = 'EMPTY' if not citydistrict else citydistrict.name
street_code = region.name + district_name + city.name + citydistrict_name + street_name
if street_code not in self.all_streets_dict:
street = RatuStreet.objects.create(region=region,
district=district,
city=city,
citydistrict=citydistrict,
name=street_name,
code=street_code)
else:
if self.outdated_streets_dict.get(street_code):
del self.outdated_streets_dict[street_code]