def handle(self, **options):
code_version = CodeType.objects.get(code='gss')
# Get the polygons that we want to fix
# The areas with bad polygons are:
# E05009392 - LBW Colville
# E05009400 - LBW Pembridge
# W04000985 - CPC Pen Tranch Community
# W04000980 - CPC Abersychan Community
# W05000992 - UTE Abersychan
# W05000999 - UTE Snatchwood
areas_to_fix = (
'E05009392',
'E05009400',
'W04000985',
'W04000980',
'W05000992',
'W05000999',
)
for ons_code in areas_to_fix:
area = Area.objects.get(codes__code=ons_code, codes__type=code_version)
assert area.polygons.count() == 1
area_polygon = area.polygons.all()[0]
fixed_polygon = fix_invalid_geos_geometry(area_polygon.polygon)
if fixed_polygon:
print("Fixed polygon {0}".format(area_polygon))
area_polygon.polygon = fixed_polygon
if options['commit']:
area_polygon.save()
else:
print("Could not fix polygon {0}".format(area_polygon))
def handle(self, **options):
code_version = CodeType.objects.get(code="gss")
# Get the polygons that we want to fix
# The areas with bad polygons are:
# E05009392 - LBW Colville
# E05009400 - LBW Pembridge
# W04000985 - CPC Pen Tranch Community
# W04000980 - CPC Abersychan Community
# W05000992 - UTE Abersychan
# W05000999 - UTE Snatchwood
areas_to_fix = ("E05009392", "E05009400", "W04000985", "W04000980", "W05000992", "W05000999")
for ons_code in areas_to_fix:
area = Area.objects.get(codes__code=ons_code, codes__type=code_version)
assert area.polygons.count() == 1
area_polygon = area.polygons.all()[0]
fixed_polygon = fix_invalid_geos_geometry(area_polygon.polygon)
if fixed_polygon:
print("Fixed polygon {0}".format(area_polygon))
area_polygon.polygon = fixed_polygon
if options["commit"]:
area_polygon.save()
else:
print("Could not fix polygon {0}".format(area_polygon))
def fix_geometry(self, g):
# Make a GEOS geometry only to check for validity:
geos_g = g.geos
if not geos_g.valid:
geos_g = fix_invalid_geos_geometry(geos_g)
if geos_g is None:
print "The geometry was invalid and couldn't be fixed"
g = None
else:
g = geos_g.ogr
return g
def fix_geometry(self, g):
# Make a GEOS geometry only to check for validity:
geos_g = g.geos
if not geos_g.valid:
geos_g = fix_invalid_geos_geometry(geos_g)
if geos_g is None:
print "The geometry was invalid and couldn't be fixed"
g = None
else:
g = geos_g.ogr
return g
def handle_label(self, filename, **options):
missing_options = []
for k in ['generation_id', 'area_type_code', 'name_type_code', 'country_code']:
if options[k]:
continue
else:
missing_options.append(k)
if missing_options:
message_start = "Missing arguments " if len(missing_options) > 1 else "Missing argument "
message = message_start + " ".join('--{0}'.format(k) for k in missing_options)
raise CommandError(message)
generation_id = options['generation_id']
area_type_code = options['area_type_code']
name_type_code = options['name_type_code']
country_code = options['country_code']
override_name = options['override_name']
name_field = options['name_field']
if not (override_name or name_field):
name_field = 'Name'
override_code = options['override_code']
code_field = options['code_field']
code_type_code = options['code_type']
encoding = options['encoding'] or 'utf-8'
if len(area_type_code) > 3:
raise CommandError("Area type code must be 3 letters or fewer, sorry")
if name_field and override_name:
raise CommandError("You must not specify both --name_field and --override_name")
if code_field and override_code:
raise CommandError("You must not specify both --code_field and --override_code")
using_code = (code_field or override_code)
if (using_code and not code_type_code) or (not using_code and code_type_code):
raise CommandError(
"If you want to save a code, specify --code_type and either --code_field or --override_code")
try:
area_type = Type.objects.get(code=area_type_code)
except:
type_desc = input('Please give a description for area type code %s: ' % area_type_code)
area_type = Type(code=area_type_code, description=type_desc)
if options['commit']:
area_type.save()
try:
name_type = NameType.objects.get(code=name_type_code)
except:
name_desc = input('Please give a description for name type code %s: ' % name_type_code)
name_type = NameType(code=name_type_code, description=name_desc)
if options['commit']:
name_type.save()
try:
country = Country.objects.get(code=country_code)
except:
country_name = input('Please give the name for country code %s: ' % country_code)
country = Country(code=country_code, name=country_name)
if options['commit']:
country.save()
if code_type_code:
try:
code_type = CodeType.objects.get(code=code_type_code)
except:
code_desc = input('Please give a description for code type %s: ' % code_type_code)
code_type = CodeType(code=code_type_code, description=code_desc)
if options['commit']:
code_type.save()
self.stdout.write("Importing from %s" % filename)
if not options['commit']:
self.stdout.write('(will not save to db as --commit not specified)')
current_generation = Generation.objects.current()
new_generation = Generation.objects.get(id=generation_id)
def verbose(*args):
if int(options['verbosity']) > 1:
self.stdout.write(" ".join(str(a) for a in args))
ds = DataSource(filename)
layer = ds[0]
if (override_name or override_code) and len(layer) > 1:
message = (
"Warning: you have specified an override %s and this file contains more than one feature; "
"multiple areas with the same %s will be created")
if override_name:
self.stdout.write(message % ('name', 'name'))
if override_code:
self.stdout.write(message % ('code', 'code'))
for feat in layer:
if override_name:
name = override_name
else:
try:
name = feat[name_field].value
except:
choices = ', '.join(layer.fields)
raise CommandError(
"Could not find name using name field '%s' - should it be something else? "
"It will be one of these: %s. Specify which with --name_field" % (name_field, choices))
try:
if not isinstance(name, six.text_type):
name = name.decode(encoding)
except:
raise CommandError(
"Could not decode name using encoding '%s' - is it in another encoding? "
"Specify one with --encoding" % encoding)
name = re.sub('\s+', ' ', name)
if not name:
raise Exception("Could not find a name to use for area")
code = None
if override_code:
code = override_code
elif code_field:
try:
code = feat[code_field].value
except:
choices = ', '.join(layer.fields)
raise CommandError(
"Could not find code using code field '%s' - should it be something else? "
"It will be one of these: %s. Specify which with --code_field" % (code_field, choices))
self.stdout.write(" looking at '%s'%s" % (name, (' (%s)' % code) if code else ''))
g = None
if hasattr(feat, 'geom'):
g = feat.geom.transform(settings.MAPIT_AREA_SRID, clone=True)
try:
if options['new']: # Always want a new area
raise Area.DoesNotExist
if code:
matching_message = "code %s of code type %s" % (code, code_type)
areas = Area.objects.filter(codes__code=code, codes__type=code_type).order_by('-generation_high')
else:
matching_message = "name %s of area type %s" % (name, area_type)
areas = Area.objects.filter(name=name, type=area_type).order_by('-generation_high')
if len(areas) == 0:
verbose(" the area was not found - creating a new one")
raise Area.DoesNotExist
m = areas[0]
verbose(" found the area")
if options['preserve']:
# Find whether we need to create a new Area:
previous_geos_geometry = m.polygons.collect()
if m.generation_high < current_generation.id:
# Then it was missing in current_generation:
verbose(" area existed previously, but was missing from", current_generation)
raise Area.DoesNotExist
elif g is None:
if previous_geos_geometry is not None:
verbose(" area is now empty")
raise Area.DoesNotExist
else:
verbose(" the area has remained empty")
elif previous_geos_geometry is None:
# It was empty in the previous generation:
verbose(" area was empty in", current_generation)
raise Area.DoesNotExist
else:
# Otherwise, create a new Area unless the
# polygons were the same in current_generation:
previous_geos_geometry = previous_geos_geometry.simplify(tolerance=0)
new_geos_geometry = g.geos.simplify(tolerance=0)
create_new_area = not previous_geos_geometry.equals(new_geos_geometry)
p = previous_geos_geometry.sym_difference(new_geos_geometry).area / previous_geos_geometry.area
verbose(" change in area is:", "%.03f%%" % (100 * p,))
if create_new_area:
verbose(" the area", m, "has changed, creating a new area due to --preserve")
raise Area.DoesNotExist
else:
verbose(" the area remained the same")
else:
# If --preserve is not specified, the code or the name must be unique:
if len(areas) > 1:
raise Area.MultipleObjectsReturned(
"There was more than one area with %s, and --preserve was not specified" % (
matching_message,))
except Area.DoesNotExist:
m = Area(
name=name,
type=area_type,
country=country,
# parent_area=parent_area,
generation_low=new_generation,
generation_high=new_generation,
)
if options['use_code_as_id'] and code:
m.id = int(code)
# check that we are not about to skip a generation
if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
raise Exception("Area %s found, but not in current generation %s" % (m, current_generation))
m.generation_high = new_generation
if options['fix_invalid_polygons'] and g is not None:
# Make a GEOS geometry only to check for validity:
geos_g = g.geos
if not geos_g.valid:
geos_g = fix_invalid_geos_geometry(geos_g)
if geos_g is None:
self.stdout.write("The geometry for area %s was invalid and couldn't be fixed" % name)
g = None
else:
g = geos_g.ogr
poly = [g] if g is not None else []
if options['commit']:
m.save()
m.names.update_or_create(type=name_type, defaults={'name': name})
if code:
m.codes.update_or_create(type=code_type, defaults={'code': code})
save_polygons({m.id: (m, poly)})
def handle_label(self, filename, **options):
if not options['control']:
raise Exception("You must specify a control file")
__import__(options['control'])
control = sys.modules[options['control']]
code_version = CodeType.objects.get(code='gss')
name_type = NameType.objects.get(code='O')
code_type_os = CodeType.objects.get(code='unit_id')
print(filename)
current_generation = Generation.objects.current()
new_generation = Generation.objects.new()
if not new_generation:
raise Exception("No new generation to be used for import!")
ds = DataSource(filename)
layer = ds[0]
for feat in layer:
name = feat['NAME'].value
if not isinstance(name, six.text_type):
name = name.decode('iso-8859-1')
name = re.sub(r'\s*\(DET( NO \d+|)\)\s*(?i)', '', name)
name = re.sub(r'\s+', ' ', name)
ons_code = feat['CODE'].value if feat['CODE'].value not in ('999999', '999999999') else None
unit_id = str(feat['UNIT_ID'].value)
area_code = feat['AREA_CODE'].value
patch = self.patch_boundary_line(name, ons_code, unit_id, area_code)
if 'ons-code' in patch:
ons_code = patch['ons-code']
elif 'unit-id' in patch:
unit_id = patch['unit-id']
if area_code == 'NCP':
continue # Ignore Non Parished Areas
if ons_code in self.ons_code_to_shape:
m, poly = self.ons_code_to_shape[ons_code]
try:
m_name = m.names.get(type=name_type).name
except Name.DoesNotExist:
m_name = m.name # If running without commit for dry run, so nothing being stored in db
if name != m_name:
raise Exception("ONS code %s is used for %s and %s" % (ons_code, name, m_name))
# Otherwise, combine the two shapes for one area
poly.append(feat.geom)
continue
if unit_id in self.unit_id_to_shape:
m, poly = self.unit_id_to_shape[unit_id]
try:
m_name = m.names.get(type=name_type).name
except Name.DoesNotExist:
m_name = m.name # If running without commit for dry run, so nothing being stored in db
if name != m_name:
raise Exception("Unit ID code %s is used for %s and %s" % (unit_id, name, m_name))
# Otherwise, combine the two shapes for one area
poly.append(feat.geom)
continue
if ons_code:
country = ons_code[0] # Hooray!
elif area_code in ('CED', 'CTY', 'DIW', 'DIS', 'MTW', 'MTD', 'LBW', 'LBO', 'LAC', 'GLA'):
country = 'E'
else:
raise Exception(area_code)
try:
check = control.check(name, area_code, country, feat.geom, ons_code=ons_code, commit=options['commit'])
if check is True:
raise Area.DoesNotExist
if isinstance(check, Area):
m = check
try:
ons_code = m.codes.get(type=code_version).code
except Code.DoesNotExist:
ons_code = None
elif ons_code:
m = Area.objects.get(codes__type=code_version, codes__code=ons_code)
elif unit_id:
m = Area.objects.get(
codes__type=code_type_os, codes__code=unit_id, generation_high=current_generation)
m_name = m.names.get(type=name_type).name
if name != m_name:
raise Exception("Unit ID code %s is %s in DB but %s in SHP file" % (unit_id, m_name, name))
else:
raise Exception('Area "%s" (%s) has neither ONS code nor unit ID' % (name, area_code))
if int(options['verbosity']) > 1:
print(" Area matched, %s" % (m, ))
except Area.DoesNotExist:
print(" New area: %s %s %s %s" % (area_code, ons_code, unit_id, name))
m = Area(
name=name, # If committing, this will be overwritten by the m.names.update_or_create
type=Type.objects.get(code=area_code),
country=Country.objects.get(code=country),
generation_low=new_generation,
generation_high=new_generation,
)
if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
raise Exception("Area %s found, but not in current generation %s" % (m, current_generation))
m.generation_high = new_generation
if options['commit']:
m.save()
# Make a GEOS geometry only to check for validity:
g = feat.geom
geos_g = g.geos
if not geos_g.valid:
print(" Geometry of %s %s not valid" % (ons_code, m))
geos_g = fix_invalid_geos_geometry(geos_g)
if geos_g is None:
raise Exception("The geometry for area %s was invalid and couldn't be fixed" % name)
g = None
else:
g = geos_g.ogr
poly = [g]
if options['commit']:
m.names.update_or_create(type=name_type, defaults={'name': name})
if ons_code:
self.ons_code_to_shape[ons_code] = (m, poly)
if options['commit']:
m.codes.update_or_create(type=code_version, defaults={'code': ons_code})
if unit_id:
self.unit_id_to_shape[unit_id] = (m, poly)
if options['commit']:
m.codes.update_or_create(type=code_type_os, defaults={'code': unit_id})
if options['commit']:
save_polygons(self.unit_id_to_shape)
save_polygons(self.ons_code_to_shape)
def handle_label(self, filename, **options):
missing_options = []
for k in [
'generation_id', 'area_type_code', 'name_type_code',
'country_code'
]:
if options[k]:
continue
else:
missing_options.append(k)
if missing_options:
message_start = "Missing arguments " if len(
missing_options) > 1 else "Missing argument "
message = message_start + " ".join('--{0}'.format(k)
for k in missing_options)
raise CommandError(message)
generation_id = options['generation_id']
area_type_code = options['area_type_code']
name_type_code = options['name_type_code']
country_code = options['country_code']
override_name = options['override_name']
name_field = options['name_field']
if not (override_name or name_field):
name_field = 'Name'
override_code = options['override_code']
code_field = options['code_field']
code_type_code = options['code_type']
encoding = options['encoding'] or 'utf-8'
if len(area_type_code) > 3:
raise CommandError(
"Area type code must be 3 letters or fewer, sorry")
if name_field and override_name:
raise CommandError(
"You must not specify both --name_field and --override_name")
if code_field and override_code:
raise CommandError(
"You must not specify both --code_field and --override_code")
using_code = (code_field or override_code)
if (using_code and not code_type_code) or (not using_code
and code_type_code):
raise CommandError(
"If you want to save a code, specify --code_type and either --code_field or --override_code"
)
try:
area_type = Type.objects.get(code=area_type_code)
except:
type_desc = raw_input(
'Please give a description for area type code %s: ' %
area_type_code)
area_type = Type(code=area_type_code, description=type_desc)
if options['commit']: area_type.save()
try:
name_type = NameType.objects.get(code=name_type_code)
except:
name_desc = raw_input(
'Please give a description for name type code %s: ' %
name_type_code)
name_type = NameType(code=name_type_code, description=name_desc)
if options['commit']: name_type.save()
try:
country = Country.objects.get(code=country_code)
except:
country_name = raw_input(
'Please give the name for country code %s: ' % country_code)
country = Country(code=country_code, name=country_name)
if options['commit']: country.save()
if code_type_code:
try:
code_type = CodeType.objects.get(code=code_type_code)
except:
code_desc = raw_input(
'Please give a description for code type %s: ' %
code_type_code)
code_type = CodeType(code=code_type_code,
description=code_desc)
if options['commit']: code_type.save()
self.stdout.write("Importing from %s" % filename)
if not options['commit']:
self.stdout.write(
'(will not save to db as --commit not specified)')
current_generation = Generation.objects.current()
new_generation = Generation.objects.get(id=generation_id)
def verbose(*args):
if int(options['verbosity']) > 1:
self.stdout.write(" ".join(str(a) for a in args))
ds = DataSource(filename)
layer = ds[0]
if (override_name or override_code) and len(layer) > 1:
message = "Warning: you have specified an override %s and this file contains more than one feature; multiple areas with the same %s will be created"
if override_name:
self.stdout.write(message % ('name', 'name'))
if override_code:
self.stdout.write(message % ('code', 'code'))
for feat in layer:
if override_name:
name = override_name
else:
try:
name = feat[name_field].value
except:
choices = ', '.join(layer.fields)
raise CommandError(
"Could not find name using name field '%s' - should it be something else? It will be one of these: %s. Specify which with --name_field"
% (name_field, choices))
try:
if not isinstance(name, unicode):
name = name.decode(encoding)
except:
raise CommandError(
"Could not decode name using encoding '%s' - is it in another encoding? Specify one with --encoding"
% encoding)
name = re.sub('\s+', ' ', name)
if not name:
raise Exception("Could not find a name to use for area")
code = None
if override_code:
code = override_code
elif code_field:
try:
code = feat[code_field].value
except:
choices = ', '.join(layer.fields)
raise CommandError(
"Could not find code using code field '%s' - should it be something else? It will be one of these: %s. Specify which with --code_field"
% (code_field, choices))
self.stdout.write(" looking at '%s'%s" %
(name.encode('utf-8'),
(' (%s)' % code) if code else ''))
g = None
if hasattr(feat, 'geom'):
g = feat.geom.transform(settings.MAPIT_AREA_SRID, clone=True)
try:
if options['new']: # Always want a new area
raise Area.DoesNotExist
if code:
matching_message = "code %s of code type %s" % (code,
code_type)
areas = Area.objects.filter(
codes__code=code,
codes__type=code_type).order_by('-generation_high')
else:
matching_message = "name %s of area type %s" % (name,
area_type)
areas = Area.objects.filter(
name=name, type=area_type).order_by('-generation_high')
if len(areas) == 0:
verbose(" the area was not found - creating a new one")
raise Area.DoesNotExist
m = areas[0]
verbose(" found the area")
if options['preserve']:
# Find whether we need to create a new Area:
previous_geos_geometry = m.polygons.collect()
if m.generation_high < current_generation.id:
# Then it was missing in current_generation:
verbose(
" area existed previously, but was missing from",
current_generation)
raise Area.DoesNotExist
elif g is None:
if previous_geos_geometry is not None:
verbose(" area is now empty")
raise Area.DoesNotExist
else:
verbose(" the area has remained empty")
elif previous_geos_geometry is None:
# It was empty in the previous generation:
verbose(" area was empty in", current_generation)
raise Area.DoesNotExist
else:
# Otherwise, create a new Area unless the
# polygons were the same in current_generation:
previous_geos_geometry = previous_geos_geometry.simplify(
tolerance=0)
new_geos_geometry = g.geos.simplify(tolerance=0)
create_new_area = not previous_geos_geometry.equals(
new_geos_geometry)
p = previous_geos_geometry.sym_difference(
new_geos_geometry
).area / previous_geos_geometry.area
verbose(" change in area is:",
"%.03f%%" % (100 * p, ))
if create_new_area:
verbose(
" the area", m,
"has changed, creating a new area due to --preserve"
)
raise Area.DoesNotExist
else:
verbose(" the area remained the same")
else:
# If --preserve is not specified, the code or the name must be unique:
if len(areas) > 1:
raise Area.MultipleObjectsReturned, "There was more than one area with %s, and --preserve was not specified" % (
matching_message, )
except Area.DoesNotExist:
m = Area(
name=name,
type=area_type,
country=country,
# parent_area = parent_area,
generation_low=new_generation,
generation_high=new_generation,
)
if options['use_code_as_id'] and code:
m.id = int(code)
# check that we are not about to skip a generation
if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
raise Exception, "Area %s found, but not in current generation %s" % (
m, current_generation)
m.generation_high = new_generation
if options['fix_invalid_polygons'] and g is not None:
# Make a GEOS geometry only to check for validity:
geos_g = g.geos
if not geos_g.valid:
geos_g = fix_invalid_geos_geometry(geos_g)
if geos_g is None:
self.stdout.write(
"The geometry for area %s was invalid and couldn't be fixed"
% name)
g = None
else:
g = geos_g.ogr
poly = [g] if g is not None else []
if options['commit']:
m.save()
m.names.update_or_create({'type': name_type}, {'name': name})
if code:
m.codes.update_or_create({'type': code_type},
{'code': code})
save_polygons({m.id: (m, poly)})
def process_file(self, filename, area_code, srid, control, options):
code_version = CodeType.objects.get(code=control.code_version())
name_type = NameType.objects.get(code="N")
code_type_osni = CodeType.objects.get(code="osni_oid")
if not hasattr(self, area_code):
raise Exception("Don't know how to extract features from %s files" % area_code)
area_code_info = getattr(self, area_code)(srid)
print(filename)
current_generation = Generation.objects.current()
new_generation = Generation.objects.new()
if not new_generation:
raise Exception("No new generation to be used for import!")
ds = DataSource(filename)
layer = ds[0]
for feat in layer:
name, ons_code, osni_object_id = area_code_info.extract_fields(feat)
name = self.format_name(name)
if ons_code in self.ons_code_to_shape:
m, poly = self.ons_code_to_shape[ons_code]
try:
m_name = m.names.get(type=name_type).name
except Name.DoesNotExist:
m_name = m.name # If running without commit for dry run, so nothing being stored in db
if name != m_name:
raise Exception("ONS code %s is used for %s and %s" % (ons_code, name, m_name))
# Otherwise, combine the two shapes for one area
poly.append(feat.geom)
continue
if osni_object_id in self.osni_object_id_to_shape:
m, poly = self.osni_object_id_to_shape[osni_object_id]
try:
m_name = m.names.get(type=name_type).name
except Name.DoesNotExist:
m_name = m.name # If running without commit for dry run, so nothing being stored in db
if name != m_name:
raise Exception("OSNI Object ID code %s is used for %s and %s" % (osni_object_id, name, m_name))
# Otherwise, combine the two shapes for one area
poly.append(feat.geom)
continue
country = "N"
try:
check = control.check(name, area_code, country, feat.geom)
if check is True:
raise Area.DoesNotExist
if isinstance(check, Area):
m = check
ons_code = m.codes.get(type=code_version).code
elif ons_code:
m = Area.objects.get(codes__type=code_version, codes__code=ons_code)
elif osni_object_id:
m = Area.objects.get(
codes__type=code_type_osni, codes__code=osni_object_id, generation_high=current_generation
)
m_name = m.names.get(type=name_type).name
if name != m_name:
raise Exception(
"OSNI Object ID code %s is %s in DB but %s in SHP file" % (osni_object_id, m_name, name)
)
else:
raise Exception('Area "%s" (%s) has neither ONS code nor OSNI Object ID' % (name, area_code))
if int(options["verbosity"]) > 1:
print(" Area matched, %s" % (m,))
except Area.DoesNotExist:
area_type = Type.objects.get(code=area_code)
# It's possible we already have NIE entries (without any codes) in the db
try:
if area_code == "NIE":
matching_name = name.title().replace("And", "and")
m = Area.objects.get(name=matching_name, type=area_type, generation_high=current_generation)
if int(options["verbosity"]) > 1:
print(" Area matched (via name), %s" % (m,))
else:
raise Area.DoesNotExist("Still doesn't exist")
except Area.DoesNotExist:
print(" New area: %s %s %s %s" % (area_code, ons_code, osni_object_id, name))
m = Area(
name=name, # Not overwritten by m.names.update_or_create as no "N" use
type=area_type,
country=Country.objects.get(code=country),
generation_low=new_generation,
generation_high=new_generation,
)
if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
raise Exception("Area %s found, but not in current generation %s" % (m, current_generation))
m.generation_high = new_generation
if options["commit"]:
m.save()
# Make a GEOS geometry only to check for validity:
g = feat.geom
geos_g = g.geos
if not geos_g.valid:
print(" Geometry of %s %s not valid" % (ons_code, m))
geos_g = fix_invalid_geos_geometry(geos_g)
if geos_g is None:
raise Exception("The geometry for area %s was invalid and couldn't be fixed" % name)
g = None
else:
g = geos_g.ogr
g = area_code_info.transform_geom(g)
poly = [g]
if options["commit"]:
m.names.update_or_create(type=name_type, defaults={"name": name})
if ons_code:
self.ons_code_to_shape[ons_code] = (m, poly)
if options["commit"]:
m.codes.update_or_create(type=code_version, defaults={"code": ons_code})
if osni_object_id:
self.osni_object_id_to_shape[osni_object_id] = (m, poly)
if options["commit"]:
m.codes.update_or_create(type=code_type_osni, defaults={"code": osni_object_id})
if options["commit"]:
save_polygons(self.osni_object_id_to_shape)
save_polygons(self.ons_code_to_shape)
def handle_label(self, filename, **options):
if not options['control']:
raise Exception("You must specify a control file")
__import__(options['control'])
control = sys.modules[options['control']]
code_version = CodeType.objects.get(code='gss')
name_type = NameType.objects.get(code='O')
code_type_os = CodeType.objects.get(code='unit_id')
print(filename)
current_generation = Generation.objects.current()
new_generation = Generation.objects.new()
if not new_generation:
raise Exception("No new generation to be used for import!")
ds = DataSource(filename)
layer = ds[0]
for feat in layer:
name = feat['NAME'].value
if not isinstance(name, six.text_type):
name = name.decode('iso-8859-1')
name = re.sub(r'\s*\(DET( NO \d+|)\)\s*(?i)', '', name)
name = re.sub(r'\s+', ' ', name)
ons_code = feat['CODE'].value if feat['CODE'].value not in ('999999', '999999999') else None
unit_id = str(feat['UNIT_ID'].value)
area_code = feat['AREA_CODE'].value
patch = self.patch_boundary_line(name, ons_code, unit_id, area_code)
if 'ons-code' in patch:
ons_code = patch['ons-code']
elif 'unit-id' in patch:
unit_id = patch['unit-id']
if area_code == 'NCP':
continue # Ignore Non Parished Areas
if ons_code in self.ons_code_to_shape:
m, poly = self.ons_code_to_shape[ons_code]
try:
m_name = m.names.get(type=name_type).name
except Name.DoesNotExist:
m_name = m.name # If running without commit for dry run, so nothing being stored in db
if name != m_name:
raise Exception("ONS code %s is used for %s and %s" % (ons_code, name, m_name))
# Otherwise, combine the two shapes for one area
poly.append(feat.geom)
continue
if unit_id in self.unit_id_to_shape:
m, poly = self.unit_id_to_shape[unit_id]
try:
m_name = m.names.get(type=name_type).name
except Name.DoesNotExist:
m_name = m.name # If running without commit for dry run, so nothing being stored in db
if name != m_name:
raise Exception("Unit ID code %s is used for %s and %s" % (unit_id, name, m_name))
# Otherwise, combine the two shapes for one area
poly.append(feat.geom)
continue
if ons_code:
country = ons_code[0] # Hooray!
elif area_code in ('CED', 'CTY', 'DIW', 'DIS', 'MTW', 'MTD', 'LBW', 'LBO', 'LAC', 'GLA'):
country = 'E'
else:
raise Exception(area_code)
try:
check = control.check(name, area_code, country, feat.geom, ons_code=ons_code, commit=options['commit'])
if check is True:
raise Area.DoesNotExist
if isinstance(check, Area):
m = check
try:
ons_code = m.codes.get(type=code_version).code
except Code.DoesNotExist:
ons_code = None
elif ons_code:
m = Area.objects.get(codes__type=code_version, codes__code=ons_code)
elif unit_id:
m = Area.objects.get(
codes__type=code_type_os, codes__code=unit_id, generation_high=current_generation)
m_name = m.names.get(type=name_type).name
if name != m_name:
raise Exception("Unit ID code %s is %s in DB but %s in SHP file" % (unit_id, m_name, name))
else:
raise Exception('Area "%s" (%s) has neither ONS code nor unit ID' % (name, area_code))
if int(options['verbosity']) > 1:
print(" Area matched, %s" % (m, ))
except Area.DoesNotExist:
print(" New area: %s %s %s %s" % (area_code, ons_code, unit_id, name))
m = Area(
name=name, # If committing, this will be overwritten by the m.names.update_or_create
type=Type.objects.get(code=area_code),
country=Country.objects.get(code=country),
generation_low=new_generation,
generation_high=new_generation,
)
if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
raise Exception("Area %s found, but not in current generation %s" % (m, current_generation))
m.generation_high = new_generation
if options['commit']:
m.save()
# Make a GEOS geometry only to check for validity:
g = feat.geom
geos_g = g.geos
if not geos_g.valid:
print(" Geometry of %s %s not valid" % (ons_code, m))
geos_g = fix_invalid_geos_geometry(geos_g)
if geos_g is None:
raise Exception("The geometry for area %s was invalid and couldn't be fixed" % name)
g = None
else:
g = geos_g.ogr
poly = [g]
if options['commit']:
m.names.update_or_create(type=name_type, defaults={'name': name})
if ons_code:
self.ons_code_to_shape[ons_code] = (m, poly)
if options['commit']:
m.codes.update_or_create(type=code_version, defaults={'code': ons_code})
if unit_id:
self.unit_id_to_shape[unit_id] = (m, poly)
if options['commit']:
m.codes.update_or_create(type=code_type_os, defaults={'code': unit_id})
if options['commit']:
save_polygons(self.unit_id_to_shape)
save_polygons(self.ons_code_to_shape)
def output_kml(point_index_and_triangle_indices):
point_index, triangle_indices = point_index_and_triangle_indices
centre_x = x[point_index]
centre_y = y[point_index]
position_tuple = centre_x, centre_y
if len(triangle_indices) < 3:
# Skip any point with fewer than 3 triangle_indices
return
if position_tuple in position_to_postcodes:
postcodes = sorted(position_to_postcodes[position_tuple])
file_basename = postcodes[0]
outcode = file_basename.split()[0]
else:
postcodes = []
file_basename = 'point-{0:09d}'.format(point_index)
outcode = 'points-at-infinity'
mkdir_p(join(postcodes_output_directory, outcode))
leafname = file_basename + ".kml"
if len(postcodes) > 1:
json_leafname = file_basename + ".json"
with open(join(postcodes_output_directory, outcode, json_leafname), "w") as fp:
json.dump(postcodes, fp)
kml_filename = join(postcodes_output_directory, outcode, leafname)
if not os.path.exists(kml_filename):
circumcentres = [ccs[i] for i in triangle_indices]
def compare_points(a, b):
ax = a[0] - centre_x
ay = a[1] - centre_y
bx = b[0] - centre_x
by = b[1] - centre_y
angle_a = math.atan2(ay, ax)
angle_b = math.atan2(by, bx)
result = angle_b - angle_a
if result > 0:
return 1
elif result < 0:
return -1
return 0
sccs = np.array(sorted(circumcentres, cmp=compare_points))
xs = [cc[0] for cc in sccs]
ys = [cc[1] for cc in sccs]
border = []
for i in range(0, len(sccs) + 1):
index_to_use = i
if i == len(sccs):
index_to_use = 0
cc = (float(xs[index_to_use]),
float(ys[index_to_use]))
border.append(cc)
polygon = Polygon(border, srid=27700)
wgs_84_polygon = polygon.transform(4326, clone=True)
# If the polygon isn't valid after transformation, try to
# fix it. (There is one such case.)
if not wgs_84_polygon.valid:
tqdm.write("Warning: had to fix polygon {0}".format(kml_filename))
wgs_84_polygon = fix_invalid_geos_geometry(wgs_84_polygon)
requires_clipping = polygon_requires_clipping(wgs_84_polygon)
if requires_clipping:
try:
if wgs_84_polygon.intersects(uk_multipolygon):
clipped_polygon = wgs_84_polygon.intersection(uk_multipolygon)
else:
clipped_polygon = wgs_84_polygon
except Exception, e:
tqdm.write("Got exception when generating:", kml_filename)
tqdm.write("The exception was:", e)
tqdm.write("The polygon's KML was:", wgs_84_polygon.kml)
clipped_polygon = wgs_84_polygon
else:
clipped_polygon = wgs_84_polygon
output_boundary_kml(kml_filename, requires_clipping, postcodes, clipped_polygon)
def process_file(self, filename, area_code, srid, control, options):
code_version = CodeType.objects.get(code='gss')
name_type = NameType.objects.get(code='N')
code_type_osni = CodeType.objects.get(code='osni_oid')
if not hasattr(self, area_code):
raise Exception(
"Don't know how to extract features from %s files" % area_code)
area_code_info = getattr(self, area_code)(srid)
print(filename)
current_generation = Generation.objects.current()
new_generation = Generation.objects.new()
if not new_generation:
raise Exception("No new generation to be used for import!")
ds = DataSource(filename)
layer = ds[0]
for feat in layer:
name, ons_code, osni_object_id = area_code_info.extract_fields(
feat)
name = self.format_name(name)
if ons_code in self.ons_code_to_shape:
m, poly = self.ons_code_to_shape[ons_code]
try:
m_name = m.names.get(type=name_type).name
except Name.DoesNotExist:
m_name = m.name # If running without commit for dry run, so nothing being stored in db
if name != m_name:
raise Exception("ONS code %s is used for %s and %s" %
(ons_code, name, m_name))
# Otherwise, combine the two shapes for one area
poly.append(feat.geom)
continue
if osni_object_id in self.osni_object_id_to_shape:
m, poly = self.osni_object_id_to_shape[osni_object_id]
try:
m_name = m.names.get(type=name_type).name
except Name.DoesNotExist:
m_name = m.name # If running without commit for dry run, so nothing being stored in db
if name != m_name:
raise Exception(
"OSNI Object ID code %s is used for %s and %s" %
(osni_object_id, name, m_name))
# Otherwise, combine the two shapes for one area
poly.append(feat.geom)
continue
country = 'N'
try:
check = control.check(name, area_code, country, feat.geom)
if check is True:
raise Area.DoesNotExist
if isinstance(check, Area):
m = check
ons_code = m.codes.get(type=code_version).code
elif ons_code:
m = Area.objects.get(codes__type=code_version,
codes__code=ons_code)
elif osni_object_id:
m = Area.objects.get(codes__type=code_type_osni,
codes__code=osni_object_id,
generation_high=current_generation)
m_name = m.names.get(type=name_type).name
if name != m_name:
raise Exception(
"OSNI Object ID code %s is %s in DB but %s in SHP file"
% (osni_object_id, m_name, name))
else:
raise Exception(
'Area "%s" (%s) has neither ONS code nor OSNI Object ID'
% (name, area_code))
if int(options['verbosity']) > 1:
print(" Area matched, %s" % (m, ))
except Area.DoesNotExist:
area_type = Type.objects.get(code=area_code)
# It's possible we already have NIE entries (without any codes) in the db
try:
if area_code == 'NIE':
matching_name = name.title().replace('And', 'and')
m = Area.objects.get(
name=matching_name,
type=area_type,
generation_high=current_generation)
if int(options['verbosity']) > 1:
print(" Area matched (via name), %s" % (m, ))
else:
raise Area.DoesNotExist("Still doesn't exist")
except Area.DoesNotExist:
print(" New area: %s %s %s %s" %
(area_code, ons_code, osni_object_id, name))
m = Area(
name=
name, # Not overwritten by m.names.update_or_create as no "N" use
type=area_type,
country=Country.objects.get(code=country),
generation_low=new_generation,
generation_high=new_generation,
)
if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
raise Exception(
"Area %s found, but not in current generation %s" %
(m, current_generation))
m.generation_high = new_generation
if options['commit']:
m.save()
# Make a GEOS geometry only to check for validity:
g = feat.geom
geos_g = g.geos
if not geos_g.valid:
print(" Geometry of %s %s not valid" % (ons_code, m))
geos_g = fix_invalid_geos_geometry(geos_g)
if geos_g is None:
raise Exception(
"The geometry for area %s was invalid and couldn't be fixed"
% name)
g = None
else:
g = geos_g.ogr
g = area_code_info.transform_geom(g)
poly = [g]
if options['commit']:
m.names.update_or_create(type=name_type,
defaults={'name': name})
if ons_code:
self.ons_code_to_shape[ons_code] = (m, poly)
if options['commit']:
m.codes.update_or_create(type=code_version,
defaults={'code': ons_code})
if osni_object_id:
self.osni_object_id_to_shape[osni_object_id] = (m, poly)
if options['commit']:
m.codes.update_or_create(type=code_type_osni,
defaults={'code': osni_object_id})
if options['commit']:
save_polygons(self.osni_object_id_to_shape)
save_polygons(self.ons_code_to_shape)
def handle_label(self, filename, **options):
if not options["control"]:
raise Exception("You must specify a control file")
__import__(options["control"])
control = sys.modules[options["control"]]
code_version = CodeType.objects.get(code=control.code_version())
name_type = NameType.objects.get(code="O")
code_type_os = CodeType.objects.get(code="unit_id")
print(filename)
current_generation = Generation.objects.current()
new_generation = Generation.objects.new()
if not new_generation:
raise Exception("No new generation to be used for import!")
ds = DataSource(filename)
layer = ds[0]
for feat in layer:
name = feat["NAME"].value
if not isinstance(name, six.text_type):
name = name.decode("iso-8859-1")
name = re.sub("\s*\(DET( NO \d+|)\)\s*(?i)", "", name)
name = re.sub("\s+", " ", name)
ons_code = feat["CODE"].value if feat["CODE"].value not in ("999999", "999999999") else None
unit_id = str(feat["UNIT_ID"].value)
area_code = feat["AREA_CODE"].value
patch = self.patch_boundary_line(name, ons_code, unit_id, area_code)
if "ons-code" in patch:
ons_code = patch["ons-code"]
elif "unit-id" in patch:
unit_id = patch["unit-id"]
if area_code == "NCP":
continue # Ignore Non Parished Areas
if ons_code in self.ons_code_to_shape:
m, poly = self.ons_code_to_shape[ons_code]
try:
m_name = m.names.get(type=name_type).name
except Name.DoesNotExist:
m_name = m.name # If running without commit for dry run, so nothing being stored in db
if name != m_name:
raise Exception("ONS code %s is used for %s and %s" % (ons_code, name, m_name))
# Otherwise, combine the two shapes for one area
poly.append(feat.geom)
continue
if unit_id in self.unit_id_to_shape:
m, poly = self.unit_id_to_shape[unit_id]
try:
m_name = m.names.get(type=name_type).name
except Name.DoesNotExist:
m_name = m.name # If running without commit for dry run, so nothing being stored in db
if name != m_name:
raise Exception("Unit ID code %s is used for %s and %s" % (unit_id, name, m_name))
# Otherwise, combine the two shapes for one area
poly.append(feat.geom)
continue
if code_version.code == "gss" and ons_code:
country = ons_code[0] # Hooray!
elif area_code in ("CED", "CTY", "DIW", "DIS", "MTW", "MTD", "LBW", "LBO", "LAC", "GLA"):
country = "E"
elif code_version.code == "gss":
raise Exception(area_code)
elif (
(area_code == "EUR" and "Scotland" in name)
or area_code in ("SPC", "SPE")
or (ons_code and ons_code[0:3] in ("00Q", "00R"))
):
country = "S"
elif (
(area_code == "EUR" and "Wales" in name)
or area_code in ("WAC", "WAE")
or (ons_code and ons_code[0:3] in ("00N", "00P"))
):
country = "W"
elif area_code in ("EUR", "UTA", "UTE", "UTW", "CPC"):
country = "E"
else: # WMC
# Make sure WMC are loaded after all wards...
area_within = Area.objects.filter(
type__code__in=("UTW", "UTE", "MTW", "COP", "LBW", "DIW"),
polygons__polygon__contains=feat.geom.geos.point_on_surface,
)[0]
country = area_within.country.code
# Can't do the above ons_code checks with new GSS codes, will have to do more PinP checks
# Do parents in separate P-in-P code after this is done.
try:
check = control.check(name, area_code, country, feat.geom, ons_code=ons_code, commit=options["commit"])
if check is True:
raise Area.DoesNotExist
if isinstance(check, Area):
m = check
ons_code = m.codes.get(type=code_version).code
elif ons_code:
m = Area.objects.get(codes__type=code_version, codes__code=ons_code)
elif unit_id:
m = Area.objects.get(
codes__type=code_type_os, codes__code=unit_id, generation_high=current_generation
)
m_name = m.names.get(type=name_type).name
if name != m_name:
raise Exception("Unit ID code %s is %s in DB but %s in SHP file" % (unit_id, m_name, name))
else:
raise Exception('Area "%s" (%s) has neither ONS code nor unit ID' % (name, area_code))
if int(options["verbosity"]) > 1:
print(" Area matched, %s" % (m,))
except Area.DoesNotExist:
print(" New area: %s %s %s %s" % (area_code, ons_code, unit_id, name))
m = Area(
name=name, # If committing, this will be overwritten by the m.names.update_or_create
type=Type.objects.get(code=area_code),
country=Country.objects.get(code=country),
generation_low=new_generation,
generation_high=new_generation,
)
if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
raise Exception("Area %s found, but not in current generation %s" % (m, current_generation))
m.generation_high = new_generation
if options["commit"]:
m.save()
# Make a GEOS geometry only to check for validity:
g = feat.geom
geos_g = g.geos
if not geos_g.valid:
print(" Geometry of %s %s not valid" % (ons_code, m))
geos_g = fix_invalid_geos_geometry(geos_g)
if geos_g is None:
raise Exception("The geometry for area %s was invalid and couldn't be fixed" % name)
g = None
else:
g = geos_g.ogr
poly = [g]
if options["commit"]:
m.names.update_or_create(type=name_type, defaults={"name": name})
if ons_code:
self.ons_code_to_shape[ons_code] = (m, poly)
if options["commit"]:
m.codes.update_or_create(type=code_version, defaults={"code": ons_code})
if unit_id:
self.unit_id_to_shape[unit_id] = (m, poly)
if options["commit"]:
m.codes.update_or_create(type=code_type_os, defaults={"code": unit_id})
if options["commit"]:
save_polygons(self.unit_id_to_shape)
save_polygons(self.ons_code_to_shape)