class StationSetTest(TestCase):
def setUp(self):
Council.objects.update_or_create(pk="AAA")
uprns = ["1", "2", "3", "4"]
addressbase = get_addressbase()
polling_stations = get_stations()
for address in addressbase:
Address.objects.update_or_create(**address)
for uprn in uprns:
UprnToCouncil.objects.update_or_create(pk=uprn, lad="AAA")
self.station_set = StationSet()
for element in polling_stations:
self.station_set.add(element)
def test_council_id(self):
self.assertEqual(self.station_set.council_id, "AAA")
def test_save(self):
self.station_set.save()
self.assertEqual(
set(PollingStation.objects.all().values_list(
"internal_council_id", "council", "polling_district_id")),
{("PS-2", "AAA", "B"), ("PS-1", "AAA", "A")},
)
class BaseGenericApiImporter(BaseStationsDistrictsImporter):
srid = 4326
districts_srid = 4326
districts_name = None
districts_url = None
stations_name = None
stations_url = None
local_files = False
def import_data(self):
# Optional step for pre import tasks
try:
self.pre_import()
except NotImplementedError:
pass
self.districts = DistrictSet()
self.stations = StationSet()
# deal with 'stations only' or 'districts only' data
if self.districts_url is not None:
self.import_polling_districts()
if self.stations_url is not None:
self.import_polling_stations()
self.districts.save()
self.stations.save()
self.districts.update_uprn_to_council_model(
self.districts_have_station_ids)
def get_districts(self):
with tempfile.NamedTemporaryFile() as tmp:
urllib.request.urlretrieve(self.districts_url, tmp.name)
return self.get_data(self.districts_filetype, tmp.name)
def get_stations(self):
with tempfile.NamedTemporaryFile() as tmp:
urllib.request.urlretrieve(self.stations_url, tmp.name)
return self.get_data(self.stations_filetype, tmp.name)
class BaseStationsAddressesImporter(BaseStationsImporter, BaseAddressesImporter):
def pre_import(self):
raise NotImplementedError
def import_data(self):
# Optional step for pre import tasks
try:
self.pre_import()
except NotImplementedError:
pass
self.stations = StationSet()
self.addresses = AddressList(self.logger)
self.import_residential_addresses()
self.import_polling_stations()
self.addresses.check_records()
self.addresses.update_uprn_to_council_model()
self.stations.save()
class BaseStationsDistrictsImporter(BaseStationsImporter,
BaseDistrictsImporter):
def pre_import(self):
raise NotImplementedError
@property
def districts_have_station_ids(self):
"""
Check that we've called self.import_polling_{districts,stations}
Don't raise an exception though, because we might still want to
import just stations or districts for debugging - i.e. to see them
in qgis/the db. However if we don't also have the other half we
won't be able to update the UprnToCouncil table.
"""
if len(self.districts.elements) < 1:
self.logger.log_message(
logging.WARNING, "No district records added to self.districts")
if len(self.stations.elements) < 1:
self.logger.log_message(
logging.WARNING, "No station records added to self.stations")
district_ids = {
e.internal_council_id
for e in self.districts.elements if e.internal_council_id != ""
}
station_ids_from_districts = {
e.polling_station_id
for e in self.districts.elements if e.polling_station_id != ""
}
station_ids = {
e.internal_council_id
for e in self.stations.elements if e.internal_council_id != ""
}
district_ids_from_stations = {
e.polling_district_id
for e in self.stations.elements if e.polling_district_id != ""
}
def get_missing(set_a, set_b):
return set_a - set_b
if station_ids_from_districts:
self.write_info("Districts have station ids attached")
missing_ids = get_missing(station_ids_from_districts, station_ids)
for station_id in missing_ids:
self.logger.log_message(
logging.WARNING,
"Station id: %s found in districts but not in stations",
variable=station_id,
)
return True
elif district_ids_from_stations:
self.write_info("Stations have district ids attached")
missing_ids = get_missing(district_ids_from_stations, district_ids)
for district_id in missing_ids:
self.logger.log_message(
logging.WARNING,
"Station id: %s found in districts but not in stations",
variable=district_id,
)
return False
def import_data(self):
# Optional step for pre import tasks
try:
self.pre_import()
except NotImplementedError:
pass
self.stations = StationSet()
self.districts = DistrictSet()
self.import_polling_districts()
self.import_polling_stations()
self.districts.save()
self.stations.save()
self.districts.update_uprn_to_council_model(
self.districts_have_station_ids)
class Command(BaseGitHubImporter):
srid = 4326
districts_srid = 4326
council_id = "E07000106"
elections = ["parl.2019-12-12"]
scraper_name = "wdiv-scrapers/DC-PollingStations-Canterbury"
geom_type = "geojson"
# Canterbury embed the station addresses in the districts file
# The stations endpoint only serves up the geo data
# (it doesn't include the station addresses)
station_addresses = {}
def district_record_to_dict(self, record):
poly = self.extract_geometry(record, self.geom_type,
self.get_srid("districts"))
code = record["ID"].strip()
address = record["POLLING_PL"].strip()
# Ad-hoc fixs for parl.2019-12-12
# The points got updated in API, but the addresses didn't
if code == "CWI2":
address = "Thanington Neighbourhood Resource Centre\nThanington Road\nCanterbury\nCT1 3XE"
if code == "RCS2":
address = (
"Chartham Sports Club\nBeech Avenue\nChartham\nCanterbury\nCT4 7TA"
)
if code in self.station_addresses and self.station_addresses[
code] != address:
raise ValueError(
"District code appears twice with 2 different station addresses"
)
self.station_addresses[code] = address
return {
"internal_council_id": code,
"name": record["NAME"].strip() + " - " + code,
"area": poly,
"polling_station_id": code,
}
def station_record_to_dict(self, record):
code = record["Polling_di"].strip()
address = self.station_addresses[code]
del self.station_addresses[
code] # remove station addresses as we use them
location = self.extract_geometry(record, self.geom_type,
self.get_srid("stations"))
if isinstance(location, MultiPoint) and len(location) == 1:
location = location[0]
# point supplied is bang on the building
# but causes google directions API to give us a strange route
if code == "CWE2" and address.startswith("St Dunstan"):
location = Point(1.070064, 51.283614, srid=4326)
return {
"internal_council_id": code,
"postcode": "",
"address": address,
"location": location,
}
def post_import(self):
# mop up any districts where we have a station address
# attached to a district code but no point
self.stations = StationSet()
for code in self.station_addresses:
self.add_polling_station({
"internal_council_id": code,
"postcode": "",
"address": self.station_addresses[code],
"location": None,
"council": self.council,
})
self.stations.save()
