def handle(self, **options): code_version = CodeType.objects.get(code='gss') # Get the polygons that we want to fix # The areas with bad polygons are: # E05009392 - LBW Colville # E05009400 - LBW Pembridge # W04000985 - CPC Pen Tranch Community # W04000980 - CPC Abersychan Community # W05000992 - UTE Abersychan # W05000999 - UTE Snatchwood areas_to_fix = ( 'E05009392', 'E05009400', 'W04000985', 'W04000980', 'W05000992', 'W05000999', ) for ons_code in areas_to_fix: area = Area.objects.get(codes__code=ons_code, codes__type=code_version) assert area.polygons.count() == 1 area_polygon = area.polygons.all()[0] fixed_polygon = fix_invalid_geos_geometry(area_polygon.polygon) if fixed_polygon: print("Fixed polygon {0}".format(area_polygon)) area_polygon.polygon = fixed_polygon if options['commit']: area_polygon.save() else: print("Could not fix polygon {0}".format(area_polygon))
def handle(self, **options): code_version = CodeType.objects.get(code="gss") # Get the polygons that we want to fix # The areas with bad polygons are: # E05009392 - LBW Colville # E05009400 - LBW Pembridge # W04000985 - CPC Pen Tranch Community # W04000980 - CPC Abersychan Community # W05000992 - UTE Abersychan # W05000999 - UTE Snatchwood areas_to_fix = ("E05009392", "E05009400", "W04000985", "W04000980", "W05000992", "W05000999") for ons_code in areas_to_fix: area = Area.objects.get(codes__code=ons_code, codes__type=code_version) assert area.polygons.count() == 1 area_polygon = area.polygons.all()[0] fixed_polygon = fix_invalid_geos_geometry(area_polygon.polygon) if fixed_polygon: print("Fixed polygon {0}".format(area_polygon)) area_polygon.polygon = fixed_polygon if options["commit"]: area_polygon.save() else: print("Could not fix polygon {0}".format(area_polygon))
def fix_geometry(self, g): # Make a GEOS geometry only to check for validity: geos_g = g.geos if not geos_g.valid: geos_g = fix_invalid_geos_geometry(geos_g) if geos_g is None: print "The geometry was invalid and couldn't be fixed" g = None else: g = geos_g.ogr return g
def handle_label(self, filename, **options): missing_options = [] for k in ['generation_id', 'area_type_code', 'name_type_code', 'country_code']: if options[k]: continue else: missing_options.append(k) if missing_options: message_start = "Missing arguments " if len(missing_options) > 1 else "Missing argument " message = message_start + " ".join('--{0}'.format(k) for k in missing_options) raise CommandError(message) generation_id = options['generation_id'] area_type_code = options['area_type_code'] name_type_code = options['name_type_code'] country_code = options['country_code'] override_name = options['override_name'] name_field = options['name_field'] if not (override_name or name_field): name_field = 'Name' override_code = options['override_code'] code_field = options['code_field'] code_type_code = options['code_type'] encoding = options['encoding'] or 'utf-8' if len(area_type_code) > 3: raise CommandError("Area type code must be 3 letters or fewer, sorry") if name_field and override_name: raise CommandError("You must not specify both --name_field and --override_name") if code_field and override_code: raise CommandError("You must not specify both --code_field and --override_code") using_code = (code_field or override_code) if (using_code and not code_type_code) or (not using_code and code_type_code): raise CommandError( "If you want to save a code, specify --code_type and either --code_field or --override_code") try: area_type = Type.objects.get(code=area_type_code) except: type_desc = input('Please give a description for area type code %s: ' % area_type_code) area_type = Type(code=area_type_code, description=type_desc) if options['commit']: area_type.save() try: name_type = NameType.objects.get(code=name_type_code) except: name_desc = input('Please give a description for name type code %s: ' % name_type_code) name_type = NameType(code=name_type_code, description=name_desc) if options['commit']: name_type.save() try: country = Country.objects.get(code=country_code) except: country_name = input('Please give the name for country code %s: ' % country_code) country = Country(code=country_code, name=country_name) if options['commit']: country.save() if code_type_code: try: code_type = CodeType.objects.get(code=code_type_code) except: code_desc = input('Please give a description for code type %s: ' % code_type_code) code_type = CodeType(code=code_type_code, description=code_desc) if options['commit']: code_type.save() self.stdout.write("Importing from %s" % filename) if not options['commit']: self.stdout.write('(will not save to db as --commit not specified)') current_generation = Generation.objects.current() new_generation = Generation.objects.get(id=generation_id) def verbose(*args): if int(options['verbosity']) > 1: self.stdout.write(" ".join(str(a) for a in args)) ds = DataSource(filename) layer = ds[0] if (override_name or override_code) and len(layer) > 1: message = ( "Warning: you have specified an override %s and this file contains more than one feature; " "multiple areas with the same %s will be created") if override_name: self.stdout.write(message % ('name', 'name')) if override_code: self.stdout.write(message % ('code', 'code')) for feat in layer: if override_name: name = override_name else: try: name = feat[name_field].value except: choices = ', '.join(layer.fields) raise CommandError( "Could not find name using name field '%s' - should it be something else? " "It will be one of these: %s. Specify which with --name_field" % (name_field, choices)) try: if not isinstance(name, six.text_type): name = name.decode(encoding) except: raise CommandError( "Could not decode name using encoding '%s' - is it in another encoding? " "Specify one with --encoding" % encoding) name = re.sub('\s+', ' ', name) if not name: raise Exception("Could not find a name to use for area") code = None if override_code: code = override_code elif code_field: try: code = feat[code_field].value except: choices = ', '.join(layer.fields) raise CommandError( "Could not find code using code field '%s' - should it be something else? " "It will be one of these: %s. Specify which with --code_field" % (code_field, choices)) self.stdout.write(" looking at '%s'%s" % (name, (' (%s)' % code) if code else '')) g = None if hasattr(feat, 'geom'): g = feat.geom.transform(settings.MAPIT_AREA_SRID, clone=True) try: if options['new']: # Always want a new area raise Area.DoesNotExist if code: matching_message = "code %s of code type %s" % (code, code_type) areas = Area.objects.filter(codes__code=code, codes__type=code_type).order_by('-generation_high') else: matching_message = "name %s of area type %s" % (name, area_type) areas = Area.objects.filter(name=name, type=area_type).order_by('-generation_high') if len(areas) == 0: verbose(" the area was not found - creating a new one") raise Area.DoesNotExist m = areas[0] verbose(" found the area") if options['preserve']: # Find whether we need to create a new Area: previous_geos_geometry = m.polygons.collect() if m.generation_high < current_generation.id: # Then it was missing in current_generation: verbose(" area existed previously, but was missing from", current_generation) raise Area.DoesNotExist elif g is None: if previous_geos_geometry is not None: verbose(" area is now empty") raise Area.DoesNotExist else: verbose(" the area has remained empty") elif previous_geos_geometry is None: # It was empty in the previous generation: verbose(" area was empty in", current_generation) raise Area.DoesNotExist else: # Otherwise, create a new Area unless the # polygons were the same in current_generation: previous_geos_geometry = previous_geos_geometry.simplify(tolerance=0) new_geos_geometry = g.geos.simplify(tolerance=0) create_new_area = not previous_geos_geometry.equals(new_geos_geometry) p = previous_geos_geometry.sym_difference(new_geos_geometry).area / previous_geos_geometry.area verbose(" change in area is:", "%.03f%%" % (100 * p,)) if create_new_area: verbose(" the area", m, "has changed, creating a new area due to --preserve") raise Area.DoesNotExist else: verbose(" the area remained the same") else: # If --preserve is not specified, the code or the name must be unique: if len(areas) > 1: raise Area.MultipleObjectsReturned( "There was more than one area with %s, and --preserve was not specified" % ( matching_message,)) except Area.DoesNotExist: m = Area( name=name, type=area_type, country=country, # parent_area=parent_area, generation_low=new_generation, generation_high=new_generation, ) if options['use_code_as_id'] and code: m.id = int(code) # check that we are not about to skip a generation if m.generation_high and current_generation and m.generation_high.id < current_generation.id: raise Exception("Area %s found, but not in current generation %s" % (m, current_generation)) m.generation_high = new_generation if options['fix_invalid_polygons'] and g is not None: # Make a GEOS geometry only to check for validity: geos_g = g.geos if not geos_g.valid: geos_g = fix_invalid_geos_geometry(geos_g) if geos_g is None: self.stdout.write("The geometry for area %s was invalid and couldn't be fixed" % name) g = None else: g = geos_g.ogr poly = [g] if g is not None else [] if options['commit']: m.save() m.names.update_or_create(type=name_type, defaults={'name': name}) if code: m.codes.update_or_create(type=code_type, defaults={'code': code}) save_polygons({m.id: (m, poly)})
def handle_label(self, filename, **options): if not options['control']: raise Exception("You must specify a control file") __import__(options['control']) control = sys.modules[options['control']] code_version = CodeType.objects.get(code='gss') name_type = NameType.objects.get(code='O') code_type_os = CodeType.objects.get(code='unit_id') print(filename) current_generation = Generation.objects.current() new_generation = Generation.objects.new() if not new_generation: raise Exception("No new generation to be used for import!") ds = DataSource(filename) layer = ds[0] for feat in layer: name = feat['NAME'].value if not isinstance(name, six.text_type): name = name.decode('iso-8859-1') name = re.sub(r'\s*\(DET( NO \d+|)\)\s*(?i)', '', name) name = re.sub(r'\s+', ' ', name) ons_code = feat['CODE'].value if feat['CODE'].value not in ('999999', '999999999') else None unit_id = str(feat['UNIT_ID'].value) area_code = feat['AREA_CODE'].value patch = self.patch_boundary_line(name, ons_code, unit_id, area_code) if 'ons-code' in patch: ons_code = patch['ons-code'] elif 'unit-id' in patch: unit_id = patch['unit-id'] if area_code == 'NCP': continue # Ignore Non Parished Areas if ons_code in self.ons_code_to_shape: m, poly = self.ons_code_to_shape[ons_code] try: m_name = m.names.get(type=name_type).name except Name.DoesNotExist: m_name = m.name # If running without commit for dry run, so nothing being stored in db if name != m_name: raise Exception("ONS code %s is used for %s and %s" % (ons_code, name, m_name)) # Otherwise, combine the two shapes for one area poly.append(feat.geom) continue if unit_id in self.unit_id_to_shape: m, poly = self.unit_id_to_shape[unit_id] try: m_name = m.names.get(type=name_type).name except Name.DoesNotExist: m_name = m.name # If running without commit for dry run, so nothing being stored in db if name != m_name: raise Exception("Unit ID code %s is used for %s and %s" % (unit_id, name, m_name)) # Otherwise, combine the two shapes for one area poly.append(feat.geom) continue if ons_code: country = ons_code[0] # Hooray! elif area_code in ('CED', 'CTY', 'DIW', 'DIS', 'MTW', 'MTD', 'LBW', 'LBO', 'LAC', 'GLA'): country = 'E' else: raise Exception(area_code) try: check = control.check(name, area_code, country, feat.geom, ons_code=ons_code, commit=options['commit']) if check is True: raise Area.DoesNotExist if isinstance(check, Area): m = check try: ons_code = m.codes.get(type=code_version).code except Code.DoesNotExist: ons_code = None elif ons_code: m = Area.objects.get(codes__type=code_version, codes__code=ons_code) elif unit_id: m = Area.objects.get( codes__type=code_type_os, codes__code=unit_id, generation_high=current_generation) m_name = m.names.get(type=name_type).name if name != m_name: raise Exception("Unit ID code %s is %s in DB but %s in SHP file" % (unit_id, m_name, name)) else: raise Exception('Area "%s" (%s) has neither ONS code nor unit ID' % (name, area_code)) if int(options['verbosity']) > 1: print(" Area matched, %s" % (m, )) except Area.DoesNotExist: print(" New area: %s %s %s %s" % (area_code, ons_code, unit_id, name)) m = Area( name=name, # If committing, this will be overwritten by the m.names.update_or_create type=Type.objects.get(code=area_code), country=Country.objects.get(code=country), generation_low=new_generation, generation_high=new_generation, ) if m.generation_high and current_generation and m.generation_high.id < current_generation.id: raise Exception("Area %s found, but not in current generation %s" % (m, current_generation)) m.generation_high = new_generation if options['commit']: m.save() # Make a GEOS geometry only to check for validity: g = feat.geom geos_g = g.geos if not geos_g.valid: print(" Geometry of %s %s not valid" % (ons_code, m)) geos_g = fix_invalid_geos_geometry(geos_g) if geos_g is None: raise Exception("The geometry for area %s was invalid and couldn't be fixed" % name) g = None else: g = geos_g.ogr poly = [g] if options['commit']: m.names.update_or_create(type=name_type, defaults={'name': name}) if ons_code: self.ons_code_to_shape[ons_code] = (m, poly) if options['commit']: m.codes.update_or_create(type=code_version, defaults={'code': ons_code}) if unit_id: self.unit_id_to_shape[unit_id] = (m, poly) if options['commit']: m.codes.update_or_create(type=code_type_os, defaults={'code': unit_id}) if options['commit']: save_polygons(self.unit_id_to_shape) save_polygons(self.ons_code_to_shape)
def handle_label(self, filename, **options): missing_options = [] for k in [ 'generation_id', 'area_type_code', 'name_type_code', 'country_code' ]: if options[k]: continue else: missing_options.append(k) if missing_options: message_start = "Missing arguments " if len( missing_options) > 1 else "Missing argument " message = message_start + " ".join('--{0}'.format(k) for k in missing_options) raise CommandError(message) generation_id = options['generation_id'] area_type_code = options['area_type_code'] name_type_code = options['name_type_code'] country_code = options['country_code'] override_name = options['override_name'] name_field = options['name_field'] if not (override_name or name_field): name_field = 'Name' override_code = options['override_code'] code_field = options['code_field'] code_type_code = options['code_type'] encoding = options['encoding'] or 'utf-8' if len(area_type_code) > 3: raise CommandError( "Area type code must be 3 letters or fewer, sorry") if name_field and override_name: raise CommandError( "You must not specify both --name_field and --override_name") if code_field and override_code: raise CommandError( "You must not specify both --code_field and --override_code") using_code = (code_field or override_code) if (using_code and not code_type_code) or (not using_code and code_type_code): raise CommandError( "If you want to save a code, specify --code_type and either --code_field or --override_code" ) try: area_type = Type.objects.get(code=area_type_code) except: type_desc = raw_input( 'Please give a description for area type code %s: ' % area_type_code) area_type = Type(code=area_type_code, description=type_desc) if options['commit']: area_type.save() try: name_type = NameType.objects.get(code=name_type_code) except: name_desc = raw_input( 'Please give a description for name type code %s: ' % name_type_code) name_type = NameType(code=name_type_code, description=name_desc) if options['commit']: name_type.save() try: country = Country.objects.get(code=country_code) except: country_name = raw_input( 'Please give the name for country code %s: ' % country_code) country = Country(code=country_code, name=country_name) if options['commit']: country.save() if code_type_code: try: code_type = CodeType.objects.get(code=code_type_code) except: code_desc = raw_input( 'Please give a description for code type %s: ' % code_type_code) code_type = CodeType(code=code_type_code, description=code_desc) if options['commit']: code_type.save() self.stdout.write("Importing from %s" % filename) if not options['commit']: self.stdout.write( '(will not save to db as --commit not specified)') current_generation = Generation.objects.current() new_generation = Generation.objects.get(id=generation_id) def verbose(*args): if int(options['verbosity']) > 1: self.stdout.write(" ".join(str(a) for a in args)) ds = DataSource(filename) layer = ds[0] if (override_name or override_code) and len(layer) > 1: message = "Warning: you have specified an override %s and this file contains more than one feature; multiple areas with the same %s will be created" if override_name: self.stdout.write(message % ('name', 'name')) if override_code: self.stdout.write(message % ('code', 'code')) for feat in layer: if override_name: name = override_name else: try: name = feat[name_field].value except: choices = ', '.join(layer.fields) raise CommandError( "Could not find name using name field '%s' - should it be something else? It will be one of these: %s. Specify which with --name_field" % (name_field, choices)) try: if not isinstance(name, unicode): name = name.decode(encoding) except: raise CommandError( "Could not decode name using encoding '%s' - is it in another encoding? Specify one with --encoding" % encoding) name = re.sub('\s+', ' ', name) if not name: raise Exception("Could not find a name to use for area") code = None if override_code: code = override_code elif code_field: try: code = feat[code_field].value except: choices = ', '.join(layer.fields) raise CommandError( "Could not find code using code field '%s' - should it be something else? It will be one of these: %s. Specify which with --code_field" % (code_field, choices)) self.stdout.write(" looking at '%s'%s" % (name.encode('utf-8'), (' (%s)' % code) if code else '')) g = None if hasattr(feat, 'geom'): g = feat.geom.transform(settings.MAPIT_AREA_SRID, clone=True) try: if options['new']: # Always want a new area raise Area.DoesNotExist if code: matching_message = "code %s of code type %s" % (code, code_type) areas = Area.objects.filter( codes__code=code, codes__type=code_type).order_by('-generation_high') else: matching_message = "name %s of area type %s" % (name, area_type) areas = Area.objects.filter( name=name, type=area_type).order_by('-generation_high') if len(areas) == 0: verbose(" the area was not found - creating a new one") raise Area.DoesNotExist m = areas[0] verbose(" found the area") if options['preserve']: # Find whether we need to create a new Area: previous_geos_geometry = m.polygons.collect() if m.generation_high < current_generation.id: # Then it was missing in current_generation: verbose( " area existed previously, but was missing from", current_generation) raise Area.DoesNotExist elif g is None: if previous_geos_geometry is not None: verbose(" area is now empty") raise Area.DoesNotExist else: verbose(" the area has remained empty") elif previous_geos_geometry is None: # It was empty in the previous generation: verbose(" area was empty in", current_generation) raise Area.DoesNotExist else: # Otherwise, create a new Area unless the # polygons were the same in current_generation: previous_geos_geometry = previous_geos_geometry.simplify( tolerance=0) new_geos_geometry = g.geos.simplify(tolerance=0) create_new_area = not previous_geos_geometry.equals( new_geos_geometry) p = previous_geos_geometry.sym_difference( new_geos_geometry ).area / previous_geos_geometry.area verbose(" change in area is:", "%.03f%%" % (100 * p, )) if create_new_area: verbose( " the area", m, "has changed, creating a new area due to --preserve" ) raise Area.DoesNotExist else: verbose(" the area remained the same") else: # If --preserve is not specified, the code or the name must be unique: if len(areas) > 1: raise Area.MultipleObjectsReturned, "There was more than one area with %s, and --preserve was not specified" % ( matching_message, ) except Area.DoesNotExist: m = Area( name=name, type=area_type, country=country, # parent_area = parent_area, generation_low=new_generation, generation_high=new_generation, ) if options['use_code_as_id'] and code: m.id = int(code) # check that we are not about to skip a generation if m.generation_high and current_generation and m.generation_high.id < current_generation.id: raise Exception, "Area %s found, but not in current generation %s" % ( m, current_generation) m.generation_high = new_generation if options['fix_invalid_polygons'] and g is not None: # Make a GEOS geometry only to check for validity: geos_g = g.geos if not geos_g.valid: geos_g = fix_invalid_geos_geometry(geos_g) if geos_g is None: self.stdout.write( "The geometry for area %s was invalid and couldn't be fixed" % name) g = None else: g = geos_g.ogr poly = [g] if g is not None else [] if options['commit']: m.save() m.names.update_or_create({'type': name_type}, {'name': name}) if code: m.codes.update_or_create({'type': code_type}, {'code': code}) save_polygons({m.id: (m, poly)})
def process_file(self, filename, area_code, srid, control, options): code_version = CodeType.objects.get(code=control.code_version()) name_type = NameType.objects.get(code="N") code_type_osni = CodeType.objects.get(code="osni_oid") if not hasattr(self, area_code): raise Exception("Don't know how to extract features from %s files" % area_code) area_code_info = getattr(self, area_code)(srid) print(filename) current_generation = Generation.objects.current() new_generation = Generation.objects.new() if not new_generation: raise Exception("No new generation to be used for import!") ds = DataSource(filename) layer = ds[0] for feat in layer: name, ons_code, osni_object_id = area_code_info.extract_fields(feat) name = self.format_name(name) if ons_code in self.ons_code_to_shape: m, poly = self.ons_code_to_shape[ons_code] try: m_name = m.names.get(type=name_type).name except Name.DoesNotExist: m_name = m.name # If running without commit for dry run, so nothing being stored in db if name != m_name: raise Exception("ONS code %s is used for %s and %s" % (ons_code, name, m_name)) # Otherwise, combine the two shapes for one area poly.append(feat.geom) continue if osni_object_id in self.osni_object_id_to_shape: m, poly = self.osni_object_id_to_shape[osni_object_id] try: m_name = m.names.get(type=name_type).name except Name.DoesNotExist: m_name = m.name # If running without commit for dry run, so nothing being stored in db if name != m_name: raise Exception("OSNI Object ID code %s is used for %s and %s" % (osni_object_id, name, m_name)) # Otherwise, combine the two shapes for one area poly.append(feat.geom) continue country = "N" try: check = control.check(name, area_code, country, feat.geom) if check is True: raise Area.DoesNotExist if isinstance(check, Area): m = check ons_code = m.codes.get(type=code_version).code elif ons_code: m = Area.objects.get(codes__type=code_version, codes__code=ons_code) elif osni_object_id: m = Area.objects.get( codes__type=code_type_osni, codes__code=osni_object_id, generation_high=current_generation ) m_name = m.names.get(type=name_type).name if name != m_name: raise Exception( "OSNI Object ID code %s is %s in DB but %s in SHP file" % (osni_object_id, m_name, name) ) else: raise Exception('Area "%s" (%s) has neither ONS code nor OSNI Object ID' % (name, area_code)) if int(options["verbosity"]) > 1: print(" Area matched, %s" % (m,)) except Area.DoesNotExist: area_type = Type.objects.get(code=area_code) # It's possible we already have NIE entries (without any codes) in the db try: if area_code == "NIE": matching_name = name.title().replace("And", "and") m = Area.objects.get(name=matching_name, type=area_type, generation_high=current_generation) if int(options["verbosity"]) > 1: print(" Area matched (via name), %s" % (m,)) else: raise Area.DoesNotExist("Still doesn't exist") except Area.DoesNotExist: print(" New area: %s %s %s %s" % (area_code, ons_code, osni_object_id, name)) m = Area( name=name, # Not overwritten by m.names.update_or_create as no "N" use type=area_type, country=Country.objects.get(code=country), generation_low=new_generation, generation_high=new_generation, ) if m.generation_high and current_generation and m.generation_high.id < current_generation.id: raise Exception("Area %s found, but not in current generation %s" % (m, current_generation)) m.generation_high = new_generation if options["commit"]: m.save() # Make a GEOS geometry only to check for validity: g = feat.geom geos_g = g.geos if not geos_g.valid: print(" Geometry of %s %s not valid" % (ons_code, m)) geos_g = fix_invalid_geos_geometry(geos_g) if geos_g is None: raise Exception("The geometry for area %s was invalid and couldn't be fixed" % name) g = None else: g = geos_g.ogr g = area_code_info.transform_geom(g) poly = [g] if options["commit"]: m.names.update_or_create(type=name_type, defaults={"name": name}) if ons_code: self.ons_code_to_shape[ons_code] = (m, poly) if options["commit"]: m.codes.update_or_create(type=code_version, defaults={"code": ons_code}) if osni_object_id: self.osni_object_id_to_shape[osni_object_id] = (m, poly) if options["commit"]: m.codes.update_or_create(type=code_type_osni, defaults={"code": osni_object_id}) if options["commit"]: save_polygons(self.osni_object_id_to_shape) save_polygons(self.ons_code_to_shape)
def output_kml(point_index_and_triangle_indices): point_index, triangle_indices = point_index_and_triangle_indices centre_x = x[point_index] centre_y = y[point_index] position_tuple = centre_x, centre_y if len(triangle_indices) < 3: # Skip any point with fewer than 3 triangle_indices return if position_tuple in position_to_postcodes: postcodes = sorted(position_to_postcodes[position_tuple]) file_basename = postcodes[0] outcode = file_basename.split()[0] else: postcodes = [] file_basename = 'point-{0:09d}'.format(point_index) outcode = 'points-at-infinity' mkdir_p(join(postcodes_output_directory, outcode)) leafname = file_basename + ".kml" if len(postcodes) > 1: json_leafname = file_basename + ".json" with open(join(postcodes_output_directory, outcode, json_leafname), "w") as fp: json.dump(postcodes, fp) kml_filename = join(postcodes_output_directory, outcode, leafname) if not os.path.exists(kml_filename): circumcentres = [ccs[i] for i in triangle_indices] def compare_points(a, b): ax = a[0] - centre_x ay = a[1] - centre_y bx = b[0] - centre_x by = b[1] - centre_y angle_a = math.atan2(ay, ax) angle_b = math.atan2(by, bx) result = angle_b - angle_a if result > 0: return 1 elif result < 0: return -1 return 0 sccs = np.array(sorted(circumcentres, cmp=compare_points)) xs = [cc[0] for cc in sccs] ys = [cc[1] for cc in sccs] border = [] for i in range(0, len(sccs) + 1): index_to_use = i if i == len(sccs): index_to_use = 0 cc = (float(xs[index_to_use]), float(ys[index_to_use])) border.append(cc) polygon = Polygon(border, srid=27700) wgs_84_polygon = polygon.transform(4326, clone=True) # If the polygon isn't valid after transformation, try to # fix it. (There is one such case.) if not wgs_84_polygon.valid: tqdm.write("Warning: had to fix polygon {0}".format(kml_filename)) wgs_84_polygon = fix_invalid_geos_geometry(wgs_84_polygon) requires_clipping = polygon_requires_clipping(wgs_84_polygon) if requires_clipping: try: if wgs_84_polygon.intersects(uk_multipolygon): clipped_polygon = wgs_84_polygon.intersection(uk_multipolygon) else: clipped_polygon = wgs_84_polygon except Exception, e: tqdm.write("Got exception when generating:", kml_filename) tqdm.write("The exception was:", e) tqdm.write("The polygon's KML was:", wgs_84_polygon.kml) clipped_polygon = wgs_84_polygon else: clipped_polygon = wgs_84_polygon output_boundary_kml(kml_filename, requires_clipping, postcodes, clipped_polygon)
def process_file(self, filename, area_code, srid, control, options): code_version = CodeType.objects.get(code='gss') name_type = NameType.objects.get(code='N') code_type_osni = CodeType.objects.get(code='osni_oid') if not hasattr(self, area_code): raise Exception( "Don't know how to extract features from %s files" % area_code) area_code_info = getattr(self, area_code)(srid) print(filename) current_generation = Generation.objects.current() new_generation = Generation.objects.new() if not new_generation: raise Exception("No new generation to be used for import!") ds = DataSource(filename) layer = ds[0] for feat in layer: name, ons_code, osni_object_id = area_code_info.extract_fields( feat) name = self.format_name(name) if ons_code in self.ons_code_to_shape: m, poly = self.ons_code_to_shape[ons_code] try: m_name = m.names.get(type=name_type).name except Name.DoesNotExist: m_name = m.name # If running without commit for dry run, so nothing being stored in db if name != m_name: raise Exception("ONS code %s is used for %s and %s" % (ons_code, name, m_name)) # Otherwise, combine the two shapes for one area poly.append(feat.geom) continue if osni_object_id in self.osni_object_id_to_shape: m, poly = self.osni_object_id_to_shape[osni_object_id] try: m_name = m.names.get(type=name_type).name except Name.DoesNotExist: m_name = m.name # If running without commit for dry run, so nothing being stored in db if name != m_name: raise Exception( "OSNI Object ID code %s is used for %s and %s" % (osni_object_id, name, m_name)) # Otherwise, combine the two shapes for one area poly.append(feat.geom) continue country = 'N' try: check = control.check(name, area_code, country, feat.geom) if check is True: raise Area.DoesNotExist if isinstance(check, Area): m = check ons_code = m.codes.get(type=code_version).code elif ons_code: m = Area.objects.get(codes__type=code_version, codes__code=ons_code) elif osni_object_id: m = Area.objects.get(codes__type=code_type_osni, codes__code=osni_object_id, generation_high=current_generation) m_name = m.names.get(type=name_type).name if name != m_name: raise Exception( "OSNI Object ID code %s is %s in DB but %s in SHP file" % (osni_object_id, m_name, name)) else: raise Exception( 'Area "%s" (%s) has neither ONS code nor OSNI Object ID' % (name, area_code)) if int(options['verbosity']) > 1: print(" Area matched, %s" % (m, )) except Area.DoesNotExist: area_type = Type.objects.get(code=area_code) # It's possible we already have NIE entries (without any codes) in the db try: if area_code == 'NIE': matching_name = name.title().replace('And', 'and') m = Area.objects.get( name=matching_name, type=area_type, generation_high=current_generation) if int(options['verbosity']) > 1: print(" Area matched (via name), %s" % (m, )) else: raise Area.DoesNotExist("Still doesn't exist") except Area.DoesNotExist: print(" New area: %s %s %s %s" % (area_code, ons_code, osni_object_id, name)) m = Area( name= name, # Not overwritten by m.names.update_or_create as no "N" use type=area_type, country=Country.objects.get(code=country), generation_low=new_generation, generation_high=new_generation, ) if m.generation_high and current_generation and m.generation_high.id < current_generation.id: raise Exception( "Area %s found, but not in current generation %s" % (m, current_generation)) m.generation_high = new_generation if options['commit']: m.save() # Make a GEOS geometry only to check for validity: g = feat.geom geos_g = g.geos if not geos_g.valid: print(" Geometry of %s %s not valid" % (ons_code, m)) geos_g = fix_invalid_geos_geometry(geos_g) if geos_g is None: raise Exception( "The geometry for area %s was invalid and couldn't be fixed" % name) g = None else: g = geos_g.ogr g = area_code_info.transform_geom(g) poly = [g] if options['commit']: m.names.update_or_create(type=name_type, defaults={'name': name}) if ons_code: self.ons_code_to_shape[ons_code] = (m, poly) if options['commit']: m.codes.update_or_create(type=code_version, defaults={'code': ons_code}) if osni_object_id: self.osni_object_id_to_shape[osni_object_id] = (m, poly) if options['commit']: m.codes.update_or_create(type=code_type_osni, defaults={'code': osni_object_id}) if options['commit']: save_polygons(self.osni_object_id_to_shape) save_polygons(self.ons_code_to_shape)
def handle_label(self, filename, **options): if not options["control"]: raise Exception("You must specify a control file") __import__(options["control"]) control = sys.modules[options["control"]] code_version = CodeType.objects.get(code=control.code_version()) name_type = NameType.objects.get(code="O") code_type_os = CodeType.objects.get(code="unit_id") print(filename) current_generation = Generation.objects.current() new_generation = Generation.objects.new() if not new_generation: raise Exception("No new generation to be used for import!") ds = DataSource(filename) layer = ds[0] for feat in layer: name = feat["NAME"].value if not isinstance(name, six.text_type): name = name.decode("iso-8859-1") name = re.sub("\s*\(DET( NO \d+|)\)\s*(?i)", "", name) name = re.sub("\s+", " ", name) ons_code = feat["CODE"].value if feat["CODE"].value not in ("999999", "999999999") else None unit_id = str(feat["UNIT_ID"].value) area_code = feat["AREA_CODE"].value patch = self.patch_boundary_line(name, ons_code, unit_id, area_code) if "ons-code" in patch: ons_code = patch["ons-code"] elif "unit-id" in patch: unit_id = patch["unit-id"] if area_code == "NCP": continue # Ignore Non Parished Areas if ons_code in self.ons_code_to_shape: m, poly = self.ons_code_to_shape[ons_code] try: m_name = m.names.get(type=name_type).name except Name.DoesNotExist: m_name = m.name # If running without commit for dry run, so nothing being stored in db if name != m_name: raise Exception("ONS code %s is used for %s and %s" % (ons_code, name, m_name)) # Otherwise, combine the two shapes for one area poly.append(feat.geom) continue if unit_id in self.unit_id_to_shape: m, poly = self.unit_id_to_shape[unit_id] try: m_name = m.names.get(type=name_type).name except Name.DoesNotExist: m_name = m.name # If running without commit for dry run, so nothing being stored in db if name != m_name: raise Exception("Unit ID code %s is used for %s and %s" % (unit_id, name, m_name)) # Otherwise, combine the two shapes for one area poly.append(feat.geom) continue if code_version.code == "gss" and ons_code: country = ons_code[0] # Hooray! elif area_code in ("CED", "CTY", "DIW", "DIS", "MTW", "MTD", "LBW", "LBO", "LAC", "GLA"): country = "E" elif code_version.code == "gss": raise Exception(area_code) elif ( (area_code == "EUR" and "Scotland" in name) or area_code in ("SPC", "SPE") or (ons_code and ons_code[0:3] in ("00Q", "00R")) ): country = "S" elif ( (area_code == "EUR" and "Wales" in name) or area_code in ("WAC", "WAE") or (ons_code and ons_code[0:3] in ("00N", "00P")) ): country = "W" elif area_code in ("EUR", "UTA", "UTE", "UTW", "CPC"): country = "E" else: # WMC # Make sure WMC are loaded after all wards... area_within = Area.objects.filter( type__code__in=("UTW", "UTE", "MTW", "COP", "LBW", "DIW"), polygons__polygon__contains=feat.geom.geos.point_on_surface, )[0] country = area_within.country.code # Can't do the above ons_code checks with new GSS codes, will have to do more PinP checks # Do parents in separate P-in-P code after this is done. try: check = control.check(name, area_code, country, feat.geom, ons_code=ons_code, commit=options["commit"]) if check is True: raise Area.DoesNotExist if isinstance(check, Area): m = check ons_code = m.codes.get(type=code_version).code elif ons_code: m = Area.objects.get(codes__type=code_version, codes__code=ons_code) elif unit_id: m = Area.objects.get( codes__type=code_type_os, codes__code=unit_id, generation_high=current_generation ) m_name = m.names.get(type=name_type).name if name != m_name: raise Exception("Unit ID code %s is %s in DB but %s in SHP file" % (unit_id, m_name, name)) else: raise Exception('Area "%s" (%s) has neither ONS code nor unit ID' % (name, area_code)) if int(options["verbosity"]) > 1: print(" Area matched, %s" % (m,)) except Area.DoesNotExist: print(" New area: %s %s %s %s" % (area_code, ons_code, unit_id, name)) m = Area( name=name, # If committing, this will be overwritten by the m.names.update_or_create type=Type.objects.get(code=area_code), country=Country.objects.get(code=country), generation_low=new_generation, generation_high=new_generation, ) if m.generation_high and current_generation and m.generation_high.id < current_generation.id: raise Exception("Area %s found, but not in current generation %s" % (m, current_generation)) m.generation_high = new_generation if options["commit"]: m.save() # Make a GEOS geometry only to check for validity: g = feat.geom geos_g = g.geos if not geos_g.valid: print(" Geometry of %s %s not valid" % (ons_code, m)) geos_g = fix_invalid_geos_geometry(geos_g) if geos_g is None: raise Exception("The geometry for area %s was invalid and couldn't be fixed" % name) g = None else: g = geos_g.ogr poly = [g] if options["commit"]: m.names.update_or_create(type=name_type, defaults={"name": name}) if ons_code: self.ons_code_to_shape[ons_code] = (m, poly) if options["commit"]: m.codes.update_or_create(type=code_version, defaults={"code": ons_code}) if unit_id: self.unit_id_to_shape[unit_id] = (m, poly) if options["commit"]: m.codes.update_or_create(type=code_type_os, defaults={"code": unit_id}) if options["commit"]: save_polygons(self.unit_id_to_shape) save_polygons(self.ons_code_to_shape)