def save_region_data(self, regions_data_list): for dictionary in regions_data_list: code = str(dictionary['Region_Id']) number = dictionary['Region_Num'] name = clean_name(dictionary['Region_Name']) short_name = dictionary['Region_Short'] capital = dictionary['Region_Center'] if capital: capital = clean_name(capital) region = self.regions_dict.get(name) if region: update_fields = [] if region.code != code: region.code = code update_fields.append('code') if region.number != number: region.number = number update_fields.append('number') if region.short_name != short_name: region.short_name = short_name update_fields.append('short_name') if update_fields: update_fields.append('updated_at') region.save(update_fields=update_fields) else: region = DrvRegion.objects.create( code=code, number=number, name=name, short_name=short_name, capital=capital ) self.regions_dict[name] = region atos_data_list = self.parse_atos_data(region) self.save_ato_data(atos_data_list, region)
def save_region_data(self, regions_data_list): for dictionary in regions_data_list: code = dictionary['Region_Id'] number = dictionary['Region_Num'] name = dictionary['Region_Name'] name = clean_name(name) short_name = dictionary['Region_Short'] capital = dictionary['Region_Center'] if capital: capital = clean_name(capital) if name in self.all_regions_dict: region = self.all_regions_dict[name] region.code = code region.number = number region.short_name = short_name region.capital = capital else: region = DrvRegion(code=code, number=number, name=name, short_name=short_name, capital=capital) self.all_regions_dict[name] = region region.save() print(f'Дані регіону {region.name} збережено') atos_data_list = self.parse_atos_data(region) self.save_ato_data(atos_data_list, region)
def save_ato_data(self, atos_data_list, region): for dictionary in atos_data_list: district_name = dictionary['ATO_Raj'] district_name = change_to_full_name(district_name) district_name = clean_name(district_name) council_name = dictionary['ATO_Rad'] ato_name = dictionary['ATO_Name'] ato_name = clean_name(ato_name) # converting an integer value to string for comparing with codes from DB that are strings ato_code = str(dictionary['ATO_Id']) district = DrvDistrict.objects.filter(region=region, name=district_name).first() if not district: district = DrvDistrict(region=region, name=district_name) district.save() council = DrvCouncil.objects.filter(region=region, name=council_name).first() if not council: council = DrvCouncil(region=region, name=council_name) council.save() if ato_code in self.all_atos_dict: ato = self.all_atos_dict[ato_code] ato.region=region ato.district=district ato.council=council ato.name = ato_name else: ato = DrvAto(region=region, district=district, council=council, name=ato_name, code=ato_code) self.all_atos_dict[ato_code] = ato ato.save() print(f'Дані населеного пункту {ato.name} збережено') streets_data_list = self.parse_streets_data(ato) self.save_street_data(streets_data_list, region, district, council, ato)
def save_ato_data(self, atos_data_list, region): for dictionary in atos_data_list: district_name = dictionary['ATO_Raj'] district_name = change_to_full_name(district_name) district_name = clean_name(district_name) district_code = region.name + district_name council_name = dictionary['ATO_Rad'].lower() council_code = region.name + district_name + council_name ato_name = dictionary['ATO_Name'] ato_name = clean_name(ato_name) ato_code = str(dictionary['ATO_Id']) district = self.districts_dict.get(district_code) if district: if self.outdated_districts_dict.get(district_code): del self.outdated_districts_dict[district_code] else: district = DrvDistrict.objects.create(region=region, name=district_name, code=district_code) self.districts_dict[district_code] = district council = self.councils_dict.get(council_code) if council: if self.outdated_councils_dict.get(council_code): del self.outdated_councils_dict[council_code] else: council = DrvCouncil.objects.create(region=region, name=council_name, code=council_code) self.councils_dict[council_code] = council ato = self.atos_dict.get(ato_code) if ato: update_fields = [] if ato.region_id != region.id: ato.region = region update_fields.append('region') if ato.district_id != district.id: ato.district = district update_fields.append('district') if ato.council_id != council.id: ato.council = council update_fields.append('council') if ato.name != ato_name: ato.name = ato_name update_fields.append('name') if update_fields: update_fields.append('updated_at') ato.save(update_fields=update_fields) if self.outdated_atos_dict.get(ato_code): del self.outdated_atos_dict[ato_code] else: ato = DrvAto.objects.create(region=region, district=district, council=council, name=ato_name, code=ato_code) self.atos_dict[ato_code] = ato streets_data_list = self.parse_streets_data(ato) self.save_street_data(streets_data_list, region, district, council, ato)
def save_to_db(self, json_file): data = self.load_json(json_file) for object_koatuu in data: first_level_code = object_koatuu['Перший рівень'] # checking that all codes are strings, not integers that happens if not isinstance(first_level_code, str): first_level_code = str(first_level_code) second_level_code = object_koatuu['Другий рівень'] if not isinstance(second_level_code, str): second_level_code = str(second_level_code) third_level_code = object_koatuu['Третій рівень'] if not isinstance(third_level_code, str): third_level_code = str(third_level_code) fourth_level_code = object_koatuu['Четвертий рівень'] if not isinstance(fourth_level_code, str): fourth_level_code = str(fourth_level_code) category_code = object_koatuu['Категорія'] category = None if category_code: category = self.save_or_get_category(category_code) name = object_koatuu["Назва об'єкта українською мовою"] if first_level_code and not second_level_code: name = clean_name( get_lowercase_substring_before_slash(name) ) self.save_or_update_first_level(name, first_level_code) continue if first_level_code and second_level_code and not third_level_code: name = change_to_full_name( clean_name( get_lowercase_substring_before_slash(name)) ) self.save_or_update_second_level(first_level_code, category, name, second_level_code) continue if (first_level_code and second_level_code and third_level_code and not fourth_level_code): name = change_to_full_name( clean_name( get_lowercase_substring_before_slash(name)) ) self.save_or_update_third_level(first_level_code, second_level_code, category, name, third_level_code) continue if (first_level_code and second_level_code and third_level_code and fourth_level_code): # omitting the incorrect record from data.gov.ua # ToDo: get explanation from the government if third_level_code == '2320381000': logger.warning(f'Код третього рівня не існує в json: {object_koatuu}') continue name = name.lower() self.save_or_update_fourth_level(first_level_code, second_level_code, third_level_code, category, name, fourth_level_code)
def save_or_get_region(self, name): region_name = clean_name(name) region_name = change_to_full_name(region_name) if region_name not in self.all_regions_dict: region = RatuRegion.objects.create(name=region_name) self.all_regions_dict[region_name] = region return region return self.all_regions_dict[region_name]
def save_to_district_table(self, record, region): if record['REGION_NAME']: district_name = clean_name(record['REGION_NAME']) district_name = change_to_full_name(district_name) else: district_name = District.EMPTY_FIELD if [region.id, district_name] not in self.district_list: district = District(region=region, name=district_name) district.save() self.district_list.insert(0, [region.id, district_name]) district = District.objects.get(name=district_name, region=region.id) return district
def save_to_city_table(self, record, region, district): if record['CITY_NAME']: city_name = clean_name(record['CITY_NAME']) else: city_name = City.EMPTY_FIELD if [region.id, district.id, city_name] not in self.city_list: city = City(region=region, district=district, name=city_name) city.save() self.city_list.insert(0, [region.id, district.id, city_name]) city = City.objects.get(name=city_name, region=region.id, district=district.id) return city
def save_or_get_district(self, name, region): district_name = clean_name(name) district_name = change_to_full_name(district_name) district_code = region.name + district_name if district_code not in self.all_districts_dict: district = RatuDistrict.objects.create(region=region, name=district_name, code=district_code) self.all_districts_dict[district_code] = district return district else: if self.outdated_districts_dict.get(district_code): del self.outdated_districts_dict[district_code] return self.all_districts_dict[district_code]
def save_or_get_city(self, name, region, district): city_name = clean_name(name) district_name = 'EMPTY' if not district else district.name city_code = region.name + district_name + city_name if city_code not in self.all_cities_dict: city = RatuCity.objects.create(region=region, district=district, name=city_name, code=city_code) self.all_cities_dict[city_code] = city return city else: if self.outdated_cities_dict.get(city_code): del self.outdated_cities_dict[city_code] return self.all_cities_dict[city_code]
def save_or_get_citydistrict(self, name, region, district, city): citydistrict_name = clean_name(name) citydistrict_code = region.name + city.name + citydistrict_name if citydistrict_code not in self.all_citydistricts_dict: citydistrict = RatuCityDistrict.objects.create( region=region, district=district, city=city, name=citydistrict_name, code=citydistrict_code) self.all_citydistricts_dict[citydistrict_code] = citydistrict return citydistrict else: if self.outdated_citydistricts_dict.get(citydistrict_code): del self.outdated_citydistricts_dict[citydistrict_code] return self.all_citydistricts_dict[citydistrict_code]