class StationSetTest(TestCase): def setUp(self): Council.objects.update_or_create(pk="AAA") uprns = ["1", "2", "3", "4"] addressbase = get_addressbase() polling_stations = get_stations() for address in addressbase: Address.objects.update_or_create(**address) for uprn in uprns: UprnToCouncil.objects.update_or_create(pk=uprn, lad="AAA") self.station_set = StationSet() for element in polling_stations: self.station_set.add(element) def test_council_id(self): self.assertEqual(self.station_set.council_id, "AAA") def test_save(self): self.station_set.save() self.assertEqual( set(PollingStation.objects.all().values_list( "internal_council_id", "council", "polling_district_id")), {("PS-2", "AAA", "B"), ("PS-1", "AAA", "A")}, )
class BaseGenericApiImporter(BaseStationsDistrictsImporter): srid = 4326 districts_srid = 4326 districts_name = None districts_url = None stations_name = None stations_url = None local_files = False def import_data(self): # Optional step for pre import tasks try: self.pre_import() except NotImplementedError: pass self.districts = DistrictSet() self.stations = StationSet() # deal with 'stations only' or 'districts only' data if self.districts_url is not None: self.import_polling_districts() if self.stations_url is not None: self.import_polling_stations() self.districts.save() self.stations.save() self.districts.update_uprn_to_council_model( self.districts_have_station_ids) def get_districts(self): with tempfile.NamedTemporaryFile() as tmp: urllib.request.urlretrieve(self.districts_url, tmp.name) return self.get_data(self.districts_filetype, tmp.name) def get_stations(self): with tempfile.NamedTemporaryFile() as tmp: urllib.request.urlretrieve(self.stations_url, tmp.name) return self.get_data(self.stations_filetype, tmp.name)
class BaseStationsAddressesImporter(BaseStationsImporter, BaseAddressesImporter): def pre_import(self): raise NotImplementedError def import_data(self): # Optional step for pre import tasks try: self.pre_import() except NotImplementedError: pass self.stations = StationSet() self.addresses = AddressList(self.logger) self.import_residential_addresses() self.import_polling_stations() self.addresses.check_records() self.addresses.update_uprn_to_council_model() self.stations.save()
class BaseStationsDistrictsImporter(BaseStationsImporter, BaseDistrictsImporter): def pre_import(self): raise NotImplementedError @property def districts_have_station_ids(self): """ Check that we've called self.import_polling_{districts,stations} Don't raise an exception though, because we might still want to import just stations or districts for debugging - i.e. to see them in qgis/the db. However if we don't also have the other half we won't be able to update the UprnToCouncil table. """ if len(self.districts.elements) < 1: self.logger.log_message( logging.WARNING, "No district records added to self.districts") if len(self.stations.elements) < 1: self.logger.log_message( logging.WARNING, "No station records added to self.stations") district_ids = { e.internal_council_id for e in self.districts.elements if e.internal_council_id != "" } station_ids_from_districts = { e.polling_station_id for e in self.districts.elements if e.polling_station_id != "" } station_ids = { e.internal_council_id for e in self.stations.elements if e.internal_council_id != "" } district_ids_from_stations = { e.polling_district_id for e in self.stations.elements if e.polling_district_id != "" } def get_missing(set_a, set_b): return set_a - set_b if station_ids_from_districts: self.write_info("Districts have station ids attached") missing_ids = get_missing(station_ids_from_districts, station_ids) for station_id in missing_ids: self.logger.log_message( logging.WARNING, "Station id: %s found in districts but not in stations", variable=station_id, ) return True elif district_ids_from_stations: self.write_info("Stations have district ids attached") missing_ids = get_missing(district_ids_from_stations, district_ids) for district_id in missing_ids: self.logger.log_message( logging.WARNING, "Station id: %s found in districts but not in stations", variable=district_id, ) return False def import_data(self): # Optional step for pre import tasks try: self.pre_import() except NotImplementedError: pass self.stations = StationSet() self.districts = DistrictSet() self.import_polling_districts() self.import_polling_stations() self.districts.save() self.stations.save() self.districts.update_uprn_to_council_model( self.districts_have_station_ids)
class Command(BaseGitHubImporter): srid = 4326 districts_srid = 4326 council_id = "E07000106" elections = ["parl.2019-12-12"] scraper_name = "wdiv-scrapers/DC-PollingStations-Canterbury" geom_type = "geojson" # Canterbury embed the station addresses in the districts file # The stations endpoint only serves up the geo data # (it doesn't include the station addresses) station_addresses = {} def district_record_to_dict(self, record): poly = self.extract_geometry(record, self.geom_type, self.get_srid("districts")) code = record["ID"].strip() address = record["POLLING_PL"].strip() # Ad-hoc fixs for parl.2019-12-12 # The points got updated in API, but the addresses didn't if code == "CWI2": address = "Thanington Neighbourhood Resource Centre\nThanington Road\nCanterbury\nCT1 3XE" if code == "RCS2": address = ( "Chartham Sports Club\nBeech Avenue\nChartham\nCanterbury\nCT4 7TA" ) if code in self.station_addresses and self.station_addresses[ code] != address: raise ValueError( "District code appears twice with 2 different station addresses" ) self.station_addresses[code] = address return { "internal_council_id": code, "name": record["NAME"].strip() + " - " + code, "area": poly, "polling_station_id": code, } def station_record_to_dict(self, record): code = record["Polling_di"].strip() address = self.station_addresses[code] del self.station_addresses[ code] # remove station addresses as we use them location = self.extract_geometry(record, self.geom_type, self.get_srid("stations")) if isinstance(location, MultiPoint) and len(location) == 1: location = location[0] # point supplied is bang on the building # but causes google directions API to give us a strange route if code == "CWE2" and address.startswith("St Dunstan"): location = Point(1.070064, 51.283614, srid=4326) return { "internal_council_id": code, "postcode": "", "address": address, "location": location, } def post_import(self): # mop up any districts where we have a station address # attached to a district code but no point self.stations = StationSet() for code in self.station_addresses: self.add_polling_station({ "internal_council_id": code, "postcode": "", "address": self.station_addresses[code], "location": None, "council": self.council, }) self.stations.save()