def post_import(self):
filepath = os.path.join(self.base_folder_path,
"local.2018-05-03/Version 2 - UPRN/uprns.csv")
self.csv_delimiter = ","
uprns = self.get_data("csv", filepath)
print("Updating UPRNs...")
for record in uprns:
stations = PollingStation.objects.filter(
council_id=self.council_id,
internal_council_id=record.polling_place_id)
if len(stations) == 1:
station = stations[0]
try:
uprn = record.uprn.lstrip("0")
g = AddressBaseGeocoder(record.polling_place_postcode)
location = g.get_point(uprn)
except (ObjectDoesNotExist, AddressBaseException):
location = None
station.location = location
self.check_station_point({
"location": station.location,
"council_id": station.council_id
})
station.save()
else:
print("Could not find station id " +
self.get_station_hash(record))
print("...done")
def get_station_point(self, record):
location = None
if (hasattr(record, self.easting_field)
and hasattr(record, self.northing_field)
and getattr(record, self.easting_field) != "0"
and getattr(record, self.easting_field) != ""
and getattr(record, self.northing_field) != "0"
and getattr(record, self.northing_field) != ""):
# if we've got points, use them
location = Point(
float(getattr(record, self.easting_field)),
float(getattr(record, self.northing_field)),
srid=27700,
)
elif (self.station_uprn_field
and getattr(record, self.station_uprn_field).strip()):
# if we have a UPRN, try that
try:
uprn = getattr(record, self.station_uprn_field)
uprn = uprn.lstrip("0")
g = AddressBaseGeocoder(self.get_station_postcode(record))
location = g.get_point(getattr(record,
self.station_uprn_field))
except (ObjectDoesNotExist, AddressBaseException):
# otherwise, don't set a point
location = None
else:
# otherwise, don't set a point
location = None
return location
def test_addresses_property(self):
with self.assertNumQueries(FuzzyInt(0, 4)):
addressbase = AddressBaseGeocoder("AA1 1AA")
addressbase._addresses = addressbase._addresses.order_by(
"-address")
self.assertNotEqual(addressbase._addresses, addressbase.addresses)
sorter = AddressSorter(addressbase._addresses)
self.assertEqual(addressbase.addresses, sorter.natural_sort())
def test_get_code_by_uprn_no_onsud(self):
"""
'00000006' is a valid UPRN in AddressBase but not in ONSUD
"""
with self.assertNumQueries(FuzzyInt(0, 4)):
addressbase = AddressBaseGeocoder("BB1 1BB")
with self.assertRaises(get_onsud_model().DoesNotExist):
result = addressbase.get_code("lad", "00000006")
self.assertIsInstance(addressbase.get_point("00000006"), Point)
def get_station_point(self, record):
if record.uprn and record.uprn != "0":
try:
g = AddressBaseGeocoder(record.postcode)
return g.get_point(record.uprn)
except (ObjectDoesNotExist, AddressBaseException):
return self.geocode_from_postcode(record)
else:
return self.geocode_from_postcode(record)
def test_get_code_by_uprn_invalid_uprn(self):
"""
'foo' is not a valid UPRN in our DB
"""
with self.assertNumQueries(FuzzyInt(0, 4)):
addressbase = AddressBaseGeocoder("CC1 1CC")
with self.assertRaises(get_address_model().DoesNotExist):
result = addressbase.get_code("lad", "foo")
with self.assertRaises(get_address_model().DoesNotExist):
result = addressbase.get_point("foo")
def test_no_codes(self):
"""
We find records for the given postcode in the AddressBase table
but there are no corresponding records in the ONSUD for the UPRNs we found
"""
with self.assertNumQueries(FuzzyInt(0, 5)):
addressbase = AddressBaseGeocoder("AA11AA")
with self.assertRaises(CodesNotFoundException):
result = addressbase.get_code("lad")
self.assertIsInstance(addressbase.centroid, Point)
def get_station_point(self, record):
location = None
if (hasattr(record, self.easting_field)
and hasattr(record, self.northing_field)
and getattr(record, self.easting_field) != "0"
and getattr(record, self.easting_field) != ""
and getattr(record, self.northing_field) != "0"
and getattr(record, self.northing_field) != ""):
# if we've got points, use them
location = Point(
float(getattr(record, self.easting_field)),
float(getattr(record, self.northing_field)),
srid=27700,
)
self.logger.log_message(
logging.INFO,
"using grid reference for station %s",
getattr(record, self.station_id_field),
)
elif (self.station_uprn_field
and getattr(record, self.station_uprn_field).strip()):
# if we have a UPRN, try that
try:
uprn = getattr(record, self.station_uprn_field)
uprn = uprn.lstrip("0")
g = AddressBaseGeocoder(self.get_station_postcode(record))
location = g.get_point(getattr(record,
self.station_uprn_field))
self.logger.log_message(
logging.INFO,
"using UPRN for station %s",
getattr(record, self.station_id_field),
)
except (ObjectDoesNotExist, AddressBaseException):
# if that fails, fall back to postcode
location = self.geocode_from_postcode(record)
self.logger.log_message(
logging.INFO,
"using postcode for station %s",
getattr(record, self.station_id_field),
)
else:
# otherwise, geocode using postcode
location = self.geocode_from_postcode(record)
self.logger.log_message(
logging.INFO,
"using postcode for station %s",
getattr(record, self.station_id_field),
)
return location
def test_get_code_by_uprn_invalid_uprn_for_postcode(self):
"""
'00000001' is a valid UPRN in our DB,
but for a different postcode
than the one we constructed with
"""
with self.assertNumQueries(FuzzyInt(0, 4)):
addressbase = AddressBaseGeocoder("CC1 1CC")
with self.assertRaises(get_address_model().DoesNotExist):
result = addressbase.get_code("lad", "00000001")
with self.assertRaises(get_address_model().DoesNotExist):
result = addressbase.get_point("00000001")
def geocode(self):
geocoder = AddressBaseGeocoder(self.postcode)
geocoder.centroid
try:
geocoder.get_code("lad")
except MultipleCodesException as e:
# re-raise as a more specific MultipleCouncilsException
# because that is what the calling code expects to handle
raise MultipleCouncilsException(str(e))
return geocoder
def test_strict_mode(self):
"""
We find records for the given postcode in the AddressBase table
There are some corresponding records in the ONSUD for the UPRNs we found
Note that in this case, the ONSUD table does not contain corresponding
records for *all* of the UPRNs we found, and we are passing strict=True
so we raise a StrictMatchException
"""
with self.assertNumQueries(FuzzyInt(0, 4)):
addressbase = AddressBaseGeocoder("BB11BB")
with self.assertRaises(StrictMatchException):
addressbase.get_code("lad", strict=True)
def test_valid(self):
"""
We find records for the given postcode in the AddressBase table
There are some corresponding records in the ONSUD for the UPRNs we found
Valid result should be returned
Note that in this case, the ONSUD table does not contain corresponding
records for *all* of the UPRNs we found, but we accept the result anyway
"""
with self.assertNumQueries(FuzzyInt(0, 5)):
addressbase = AddressBaseGeocoder(
"bb 1 1B B") # intentionally spurious whitespace and case
self.assertEqual("B01000001", addressbase.get_code("lad"))
self.assertIsInstance(addressbase.centroid, Point)
def test_no_records(self):
"""
We can't find any records for the given postcode in the AddressBase table
"""
with self.assertNumQueries(FuzzyInt(0, 4)):
with self.assertRaises(get_address_model().DoesNotExist):
addressbase = AddressBaseGeocoder("ZZ1 1ZZ")
def test_multiple_codes(self):
"""
We find records for the given postcode in the AddressBase table
There are corresponding records in the ONSUD for the UPRNs we found
The UPRNs described by this postcode map to more than one 'lad'
but they all map to the same 'cty'
"""
with self.assertNumQueries(FuzzyInt(0, 5)):
addressbase = AddressBaseGeocoder("CC1 1CC")
with self.assertRaises(MultipleCodesException):
result = addressbase.get_code("lad")
self.assertEqual("A01000001", addressbase.get_code("cty"))
self.assertIsInstance(addressbase.centroid, Point)
def test_empty_onsud_table(self):
"""
The ONSUD table has no records in it
"""
get_onsud_model().objects.all().delete()
with self.assertNumQueries(FuzzyInt(0, 4)):
with self.assertRaises(OnsudNotImportedException):
addressbase = AddressBaseGeocoder("AA11AA")
def test_centroid_ignores_type_l(self):
addressbase = AddressBaseGeocoder("BB11BB")
before_centroid = addressbase.centroid
# adding a type L UPRN shouldn't change the postcode centroid
Address.objects.create(
postcode="BB1 1BB",
address="foobar",
location=Point(94.5, 65.7, srid=4326),
addressbase_postal="L",
)
after_centroid = addressbase.centroid
self.assertEqual(before_centroid, after_centroid)
def test_get_code_by_uprn_valid(self):
"""
valid get_code() by UPRN queries
"""
with self.assertNumQueries(FuzzyInt(0, 4)):
addressbase = AddressBaseGeocoder("CC1 1CC")
self.assertEqual("B01000001",
addressbase.get_code("lad", "00000008"))
self.assertIsInstance(addressbase.get_point("00000008"), Point)
self.assertEqual("B01000002",
addressbase.get_code("lad", "00000009"))
self.assertIsInstance(addressbase.get_point("00000009"), Point)
def geocode_point_only(self):
return AddressBaseGeocoder(self.postcode)
def test_invalid_code_type(self):
with self.assertNumQueries(FuzzyInt(0, 4)):
addressbase = AddressBaseGeocoder("CC1 1CC")
with self.assertRaises(FieldDoesNotExist):
result = addressbase.get_code("foo") # not a real code type
def geocode(self):
return AddressBaseGeocoder(self.postcode)
def test_northern_ireland(self):
with self.assertNumQueries(FuzzyInt(0, 4)):
with self.assertRaises(NorthernIrelandException):
addressbase = AddressBaseGeocoder("BT11AA")