def save_street_data(self, streets_data_list, region, district, council, ato): for dictionary in streets_data_list: # converting an integer value to string for comparing with codes from DB that are strings code = str(dictionary['Geon_Id']) name = dictionary['Geon_Name'] name = change_to_full_name(name) previous_name = dictionary['Geon_OldNames'] if previous_name: previous_name = change_to_full_name(previous_name) buildings_info = dictionary['BUILDS'] number_of_buildings = None buildings_data_list = None if buildings_info: number_of_buildings = dictionary['BUILDS']['BUILDS_COUNT'] buildings_data_list = dictionary['BUILDS']['BUILD'] street = DrvStreet.objects.filter(code=code).first() if not street: street = DrvStreet(region=region, district=district, council=council, ato=ato, code=code, name=name, previous_name=previous_name, number_of_buildings=number_of_buildings) else: street.name = name street.previous_name = previous_name street.number_of_buildings = number_of_buildings street.save() if buildings_data_list: self.save_building_data(buildings_data_list, region, district, council, ato, street)
def save_street_data(self, streets_data_list, region, district, council, ato): for dictionary in streets_data_list: code = str(dictionary['Geon_Id']) name = dictionary['Geon_Name'] name = change_to_full_name(name) previous_name = dictionary['Geon_OldNames'] if previous_name: previous_name = change_to_full_name(previous_name) buildings_info = dictionary['BUILDS'] number_of_buildings = None buildings_data_list = None if buildings_info: number_of_buildings = dictionary['BUILDS']['BUILDS_COUNT'] buildings_data_list = dictionary['BUILDS']['BUILD'] street = self.streets_dict.get(code) if street: update_fields = [] if street.region_id != region.id: street.region = region update_fields.append('region') if street.district_id != district.id: street.district = district update_fields.append('district') if street.council_id != council.id: street.council = council update_fields.append('council') if street.ato_id != ato.id: street.ato = ato update_fields.append('ato') if street.name != name: street.name = name update_fields.append('name') if street.previous_name != previous_name: street.previous_name = previous_name update_fields.append('previous_name') if street.number_of_buildings != number_of_buildings: street.number_of_buildings = number_of_buildings update_fields.append('number_of_buildings') if update_fields: update_fields.append('updated_at') street.save(update_fields=update_fields) if self.outdated_streets_dict.get(code): del self.outdated_streets_dict[code] else: street = DrvStreet.objects.create( region=region, district=district, council=council, ato=ato, code=code, name=name, previous_name=previous_name, number_of_buildings=number_of_buildings) self.streets_dict[code] = street if buildings_data_list: self.save_building_data(buildings_data_list, region, district, council, ato, street)
def save_to_db(self, json_file): data = self.load_json(json_file) for object_koatuu in data: first_level_code = object_koatuu['Перший рівень'] # checking that all codes are strings, not integers that happens if not isinstance(first_level_code, str): first_level_code = str(first_level_code) second_level_code = object_koatuu['Другий рівень'] if not isinstance(second_level_code, str): second_level_code = str(second_level_code) third_level_code = object_koatuu['Третій рівень'] if not isinstance(third_level_code, str): third_level_code = str(third_level_code) fourth_level_code = object_koatuu['Четвертий рівень'] if not isinstance(fourth_level_code, str): fourth_level_code = str(fourth_level_code) category_code = object_koatuu['Категорія'] category = None if category_code: category = self.save_or_get_category(category_code) name = object_koatuu["Назва об'єкта українською мовою"] if first_level_code and not second_level_code: name = clean_name( get_lowercase_substring_before_slash(name) ) self.save_or_update_first_level(name, first_level_code) continue if first_level_code and second_level_code and not third_level_code: name = change_to_full_name( clean_name( get_lowercase_substring_before_slash(name)) ) self.save_or_update_second_level(first_level_code, category, name, second_level_code) continue if (first_level_code and second_level_code and third_level_code and not fourth_level_code): name = change_to_full_name( clean_name( get_lowercase_substring_before_slash(name)) ) self.save_or_update_third_level(first_level_code, second_level_code, category, name, third_level_code) continue if (first_level_code and second_level_code and third_level_code and fourth_level_code): # omitting the incorrect record from data.gov.ua # ToDo: get explanation from the government if third_level_code == '2320381000': logger.warning(f'Код третього рівня не існує в json: {object_koatuu}') continue name = name.lower() self.save_or_update_fourth_level(first_level_code, second_level_code, third_level_code, category, name, fourth_level_code)
def save_ato_data(self, atos_data_list, region): for dictionary in atos_data_list: district_name = dictionary['ATO_Raj'] district_name = change_to_full_name(district_name) district_name = clean_name(district_name) council_name = dictionary['ATO_Rad'] ato_name = dictionary['ATO_Name'] ato_name = clean_name(ato_name) # converting an integer value to string for comparing with codes from DB that are strings ato_code = str(dictionary['ATO_Id']) district = DrvDistrict.objects.filter(region=region, name=district_name).first() if not district: district = DrvDistrict(region=region, name=district_name) district.save() council = DrvCouncil.objects.filter(region=region, name=council_name).first() if not council: council = DrvCouncil(region=region, name=council_name) council.save() if ato_code in self.all_atos_dict: ato = self.all_atos_dict[ato_code] ato.region=region ato.district=district ato.council=council ato.name = ato_name else: ato = DrvAto(region=region, district=district, council=council, name=ato_name, code=ato_code) self.all_atos_dict[ato_code] = ato ato.save() print(f'Дані населеного пункту {ato.name} збережено') streets_data_list = self.parse_streets_data(ato) self.save_street_data(streets_data_list, region, district, council, ato)
def save_ato_data(self, atos_data_list, region): for dictionary in atos_data_list: district_name = dictionary['ATO_Raj'] district_name = change_to_full_name(district_name) district_name = clean_name(district_name) district_code = region.name + district_name council_name = dictionary['ATO_Rad'].lower() council_code = region.name + district_name + council_name ato_name = dictionary['ATO_Name'] ato_name = clean_name(ato_name) ato_code = str(dictionary['ATO_Id']) district = self.districts_dict.get(district_code) if district: if self.outdated_districts_dict.get(district_code): del self.outdated_districts_dict[district_code] else: district = DrvDistrict.objects.create(region=region, name=district_name, code=district_code) self.districts_dict[district_code] = district council = self.councils_dict.get(council_code) if council: if self.outdated_councils_dict.get(council_code): del self.outdated_councils_dict[council_code] else: council = DrvCouncil.objects.create(region=region, name=council_name, code=council_code) self.councils_dict[council_code] = council ato = self.atos_dict.get(ato_code) if ato: update_fields = [] if ato.region_id != region.id: ato.region = region update_fields.append('region') if ato.district_id != district.id: ato.district = district update_fields.append('district') if ato.council_id != council.id: ato.council = council update_fields.append('council') if ato.name != ato_name: ato.name = ato_name update_fields.append('name') if update_fields: update_fields.append('updated_at') ato.save(update_fields=update_fields) if self.outdated_atos_dict.get(ato_code): del self.outdated_atos_dict[ato_code] else: ato = DrvAto.objects.create(region=region, district=district, council=council, name=ato_name, code=ato_code) self.atos_dict[ato_code] = ato streets_data_list = self.parse_streets_data(ato) self.save_street_data(streets_data_list, region, district, council, ato)
def save_or_get_region(self, name): region_name = clean_name(name) region_name = change_to_full_name(region_name) if region_name not in self.all_regions_dict: region = RatuRegion.objects.create(name=region_name) self.all_regions_dict[region_name] = region return region return self.all_regions_dict[region_name]
def save_to_region_table(self, record): record['OBL_NAME'] = change_to_full_name(record['OBL_NAME']) if record['OBL_NAME'] not in self.region_dict: region = Region(name=record['OBL_NAME']) region.save() self.region_dict[record['OBL_NAME']] = region return region region = self.region_dict[record['OBL_NAME']] return region
def save_to_district_table(self, record, region): if record['REGION_NAME']: district_name = clean_name(record['REGION_NAME']) district_name = change_to_full_name(district_name) else: district_name = District.EMPTY_FIELD if [region.id, district_name] not in self.district_list: district = District(region=region, name=district_name) district.save() self.district_list.insert(0, [region.id, district_name]) district = District.objects.get(name=district_name, region=region.id) return district
def save_or_get_district(self, name, region): district_name = clean_name(name) district_name = change_to_full_name(district_name) district_code = region.name + district_name if district_code not in self.all_districts_dict: district = RatuDistrict.objects.create(region=region, name=district_name, code=district_code) self.all_districts_dict[district_code] = district return district else: if self.outdated_districts_dict.get(district_code): del self.outdated_districts_dict[district_code] return self.all_districts_dict[district_code]
def save_street(self, name, region, district, city, citydistrict): street_name = change_to_full_name(name) # Saving streets that are located in Kyiv and Sevastopol that are regions if not city: city = self.save_or_get_city(region.name, region, district) district_name = 'EMPTY' if not district else district.name citydistrict_name = 'EMPTY' if not citydistrict else citydistrict.name street_code = region.name + district_name + city.name + citydistrict_name + street_name if street_code not in self.all_streets_dict: street = RatuStreet.objects.create(region=region, district=district, city=city, citydistrict=citydistrict, name=street_name, code=street_code) else: if self.outdated_streets_dict.get(street_code): del self.outdated_streets_dict[street_code]