def update_or_create(): try: m = Area.objects.get(id=code) except Area.DoesNotExist: m = Area( id=code, name=name, type=Type.objects.get(code=area_code), country=Country.objects.get(code='O'), parent_area=parent_area, generation_low=new_generation, generation_high=new_generation, ) if m.generation_high and current_generation and m.generation_high.id < current_generation.id: raise Exception("Area %s found, but not in current generation %s" % (m, current_generation)) m.generation_high = new_generation g = feat.geom.transform(4326, clone=True) poly = [g] if options['commit']: m.save() for k, v in kml_data.data[name].items(): if k in ('name:smi', 'name:fi'): lang = 'N' + k[5:] m.names.update_or_create(type=NameType.objects.get(code=lang), defaults={'name': v}) m.codes.update_or_create(type=code_type_n5000, defaults={'code': code_str}) m.codes.update_or_create(type=code_type_osm, defaults={'code': int(kml_data.data[name]['osm'])}) save_polygons({code: (m, poly)})
def update_or_create(): try: m = Area.objects.get(id=code) except Area.DoesNotExist: m = Area( id=code, name=name, type=Type.objects.get(code=area_code), country=Country.objects.get(code='O'), parent_area=parent_area, generation_low=new_generation, generation_high=new_generation, ) if m.generation_high and current_generation and m.generation_high.id < current_generation.id: raise Exception("Area %s found, but not in current generation %s" % (m, current_generation)) m.generation_high = new_generation g = feat.geom.transform(4326, clone=True) poly = [g] if options['commit']: m.save() for k, v in kml_data.data[name].items(): if k in ('name:smi', 'name:fi'): lang = 'N' + k[5:] m.names.update_or_create(type=NameType.objects.get(code=lang), defaults={'name': v}) m.codes.update_or_create(type=code_type_n5000, defaults={'code': code_str}) m.codes.update_or_create(type=code_type_osm, defaults={'code': int(kml_data.data[name]['osm'])}) save_polygons({code: (m, poly)})
def update_or_create(): try: m = Area.objects.get(id=int(regionid)) print "Updating area %s with id %d" % (regionname, int(regionid)) except Area.DoesNotExist: print "Creating new area %s with id %d" % ( regionname, int(regionid)) m = Area( id=int(regionid), name=regionname, type=Type.objects.get(code=area_type), country=Country.objects.get(code='O'), generation_low=new_generation, generation_high=new_generation, ) if m.generation_high and current_generation \ and m.generation_high.id < current_generation.id: raise Exception, "Area %s found, but not in current generation %s" % ( m, current_generation) m.generation_high = new_generation poly = [GEOSGeometry(unionoutline).ogr] if options['commit']: m.save() save_polygons({regionid: (m, poly)})
def update_or_create(): if osm_code: m = osm_code.area else: m = Area( name = name, type = Type.objects.get(code=area_code), country = Country.objects.get(code='G'), parent_area = parent_area, generation_low = new_generation, generation_high = new_generation, ) if m.generation_high and current_generation and m.generation_high.id < current_generation.id: raise Exception, "Area %s found, but not in current generation %s" % (m, current_generation) m.generation_high = new_generation g = feat.geom.transform(4326, clone=True) # In generating the data we should have # excluded any "polygons" with less than four # points (the final one being the same as the # first), but just in case: for polygon in g: if g.num_points < 4: return poly = [ g ] if options['commit']: m.save() if name not in kml_data.data: print json.dumps(kml_data.data, sort_keys=True, indent=4) raise Exception, u"Will fail to find '%s' in the dictionary" % (name,) for k, v in kml_data.data[name].items(): language_name = None if k == 'name': lang = 'default' language_name = "OSM Default" else: name_match = re.search(r'^name:(.+)$', k) if name_match: lang = name_match.group(1) if lang in language_code_to_name: language_name = language_code_to_name[lang] if not language_name: continue # Otherwise, make sure that a NameType for this language exists: NameType.objects.update_or_create({'code': lang}, {'code': lang, 'description': language_name}) name_type = NameType.objects.get(code=lang) m.names.update_or_create({ 'type': name_type }, { 'name': v }) m.codes.update_or_create({ 'type': code_type_osm }, { 'code': osm_id }) save_polygons({ code : (m, poly) })
def handle_label(self, filename, **options): current_generation = Generation.objects.current() new_generation = Generation.objects.new() if not new_generation: raise Exception, "No new generation to be used for import!" code_type = CodeType.objects.get(code='n5000') name_type = NameType.objects.get(code='M') ds = DataSource(filename) layer = ds[0] for feat in layer: name = unicode(feat['NAVN'].value, 'iso-8859-1') name = re.sub('\s+', ' ', name) print " ", name code = feat['KOMM'].value code_str = '%04d' % code area_code = 'NKO' try: m = Area.objects.get(codes__type=code_type, codes__code=code_str) except Area.DoesNotExist: m = Area( id=code, type=Type.objects.get(code=area_code), country=Country.objects.get(code='O'), generation_low=new_generation, generation_high=new_generation, ) if m.generation_high and current_generation and m.generation_high.id < current_generation.id: raise Exception, "Area %s found, but not in current generation %s" % ( m, current_generation) m.generation_high = new_generation g = feat.geom.transform(4326, clone=True) poly = [g] if options['commit']: m.save() m.names.update_or_create({'type': name_type}, {'name': name}) m.codes.update_or_create({'type': code_type}, {'code': code_str}) save_polygons({code: (m, poly)})
def handle_label(self, filename, **options): current_generation = Generation.objects.current() new_generation = Generation.objects.new() if not new_generation: raise Exception, "No new generation to be used for import!" code_type = CodeType.objects.get(code='n5000') name_type = NameType.objects.get(code='M') ds = DataSource(filename) layer = ds[0] for feat in layer: name = unicode(feat['NAVN'].value, 'iso-8859-1') name = re.sub('\s+', ' ', name) print " ", name code = feat['KOMM'].value code_str = '%04d' % code area_code = 'NKO' try: m = Area.objects.get(codes__type=code_type, codes__code=code_str) except Area.DoesNotExist: m = Area( id = code, type = Type.objects.get(code=area_code), country = Country.objects.get(code='O'), generation_low = new_generation, generation_high = new_generation, ) if m.generation_high and current_generation and m.generation_high.id < current_generation.id: raise Exception, "Area %s found, but not in current generation %s" % (m, current_generation) m.generation_high = new_generation g = feat.geom.transform(4326, clone=True) poly = [ g ] if options['commit']: m.save() m.names.update_or_create({ 'type': name_type }, { 'name': name }) m.codes.update_or_create({ 'type': code_type }, { 'code': code_str }) save_polygons({ code : (m, poly) })
def update_or_create(): try: m = Area.objects.get(id=int(regionid)) print "Updating area %s with id %d" % (regionname, int(regionid)) except Area.DoesNotExist: print "Creating new area %s with id %d" % (regionname, int(regionid)) m = Area( id = int(regionid), name = regionname, type = Type.objects.get(code=area_type), country = Country.objects.get(code='O'), generation_low = new_generation, generation_high = new_generation, ) if m.generation_high and current_generation \ and m.generation_high.id < current_generation.id: raise Exception, "Area %s found, but not in current generation %s" % (m, current_generation) m.generation_high = new_generation poly = [ GEOSGeometry(unionoutline).ogr ] if options['commit']: m.save() save_polygons({ regionid : (m, poly) })
def handle_label(self, filename, **options): missing_options = [] for k in ['generation_id', 'area_type_code', 'name_type_code', 'country_code']: if options[k]: continue else: missing_options.append(k) if missing_options: message_start = "Missing arguments " if len(missing_options) > 1 else "Missing argument " message = message_start + " ".join('--{0}'.format(k) for k in missing_options) raise CommandError(message) generation_id = options['generation_id'] area_type_code = options['area_type_code'] name_type_code = options['name_type_code'] country_code = options['country_code'] override_name = options['override_name'] name_field = options['name_field'] if not (override_name or name_field): name_field = 'Name' override_code = options['override_code'] code_field = options['code_field'] code_type_code = options['code_type'] encoding = options['encoding'] or 'utf-8' if len(area_type_code) > 3: raise CommandError("Area type code must be 3 letters or fewer, sorry") if name_field and override_name: raise CommandError("You must not specify both --name_field and --override_name") if code_field and override_code: raise CommandError("You must not specify both --code_field and --override_code") using_code = (code_field or override_code) if (using_code and not code_type_code) or (not using_code and code_type_code): raise CommandError( "If you want to save a code, specify --code_type and either --code_field or --override_code") try: area_type = Type.objects.get(code=area_type_code) except: type_desc = input('Please give a description for area type code %s: ' % area_type_code) area_type = Type(code=area_type_code, description=type_desc) if options['commit']: area_type.save() try: name_type = NameType.objects.get(code=name_type_code) except: name_desc = input('Please give a description for name type code %s: ' % name_type_code) name_type = NameType(code=name_type_code, description=name_desc) if options['commit']: name_type.save() try: country = Country.objects.get(code=country_code) except: country_name = input('Please give the name for country code %s: ' % country_code) country = Country(code=country_code, name=country_name) if options['commit']: country.save() if code_type_code: try: code_type = CodeType.objects.get(code=code_type_code) except: code_desc = input('Please give a description for code type %s: ' % code_type_code) code_type = CodeType(code=code_type_code, description=code_desc) if options['commit']: code_type.save() self.stdout.write("Importing from %s" % filename) if not options['commit']: self.stdout.write('(will not save to db as --commit not specified)') current_generation = Generation.objects.current() new_generation = Generation.objects.get(id=generation_id) def verbose(*args): if int(options['verbosity']) > 1: self.stdout.write(" ".join(str(a) for a in args)) ds = DataSource(filename) layer = ds[0] if (override_name or override_code) and len(layer) > 1: message = ( "Warning: you have specified an override %s and this file contains more than one feature; " "multiple areas with the same %s will be created") if override_name: self.stdout.write(message % ('name', 'name')) if override_code: self.stdout.write(message % ('code', 'code')) for feat in layer: if override_name: name = override_name else: try: name = feat[name_field].value except: choices = ', '.join(layer.fields) raise CommandError( "Could not find name using name field '%s' - should it be something else? " "It will be one of these: %s. Specify which with --name_field" % (name_field, choices)) try: if not isinstance(name, six.text_type): name = name.decode(encoding) except: raise CommandError( "Could not decode name using encoding '%s' - is it in another encoding? " "Specify one with --encoding" % encoding) name = re.sub('\s+', ' ', name) if not name: raise Exception("Could not find a name to use for area") code = None if override_code: code = override_code elif code_field: try: code = feat[code_field].value except: choices = ', '.join(layer.fields) raise CommandError( "Could not find code using code field '%s' - should it be something else? " "It will be one of these: %s. Specify which with --code_field" % (code_field, choices)) self.stdout.write(" looking at '%s'%s" % (name, (' (%s)' % code) if code else '')) g = None if hasattr(feat, 'geom'): g = feat.geom.transform(settings.MAPIT_AREA_SRID, clone=True) try: if options['new']: # Always want a new area raise Area.DoesNotExist if code: matching_message = "code %s of code type %s" % (code, code_type) areas = Area.objects.filter(codes__code=code, codes__type=code_type).order_by('-generation_high') else: matching_message = "name %s of area type %s" % (name, area_type) areas = Area.objects.filter(name=name, type=area_type).order_by('-generation_high') if len(areas) == 0: verbose(" the area was not found - creating a new one") raise Area.DoesNotExist m = areas[0] verbose(" found the area") if options['preserve']: # Find whether we need to create a new Area: previous_geos_geometry = m.polygons.collect() if m.generation_high < current_generation.id: # Then it was missing in current_generation: verbose(" area existed previously, but was missing from", current_generation) raise Area.DoesNotExist elif g is None: if previous_geos_geometry is not None: verbose(" area is now empty") raise Area.DoesNotExist else: verbose(" the area has remained empty") elif previous_geos_geometry is None: # It was empty in the previous generation: verbose(" area was empty in", current_generation) raise Area.DoesNotExist else: # Otherwise, create a new Area unless the # polygons were the same in current_generation: previous_geos_geometry = previous_geos_geometry.simplify(tolerance=0) new_geos_geometry = g.geos.simplify(tolerance=0) create_new_area = not previous_geos_geometry.equals(new_geos_geometry) p = previous_geos_geometry.sym_difference(new_geos_geometry).area / previous_geos_geometry.area verbose(" change in area is:", "%.03f%%" % (100 * p,)) if create_new_area: verbose(" the area", m, "has changed, creating a new area due to --preserve") raise Area.DoesNotExist else: verbose(" the area remained the same") else: # If --preserve is not specified, the code or the name must be unique: if len(areas) > 1: raise Area.MultipleObjectsReturned( "There was more than one area with %s, and --preserve was not specified" % ( matching_message,)) except Area.DoesNotExist: m = Area( name=name, type=area_type, country=country, # parent_area=parent_area, generation_low=new_generation, generation_high=new_generation, ) if options['use_code_as_id'] and code: m.id = int(code) # check that we are not about to skip a generation if m.generation_high and current_generation and m.generation_high.id < current_generation.id: raise Exception("Area %s found, but not in current generation %s" % (m, current_generation)) m.generation_high = new_generation if options['fix_invalid_polygons'] and g is not None: # Make a GEOS geometry only to check for validity: geos_g = g.geos if not geos_g.valid: geos_g = fix_invalid_geos_geometry(geos_g) if geos_g is None: self.stdout.write("The geometry for area %s was invalid and couldn't be fixed" % name) g = None else: g = geos_g.ogr poly = [g] if g is not None else [] if options['commit']: m.save() m.names.update_or_create(type=name_type, defaults={'name': name}) if code: m.codes.update_or_create(type=code_type, defaults={'code': code}) save_polygons({m.id: (m, poly)})
def handle_label(self, filename, **options): if not options['control']: raise Exception("You must specify a control file") __import__(options['control']) control = sys.modules[options['control']] code_version = CodeType.objects.get(code='gss') name_type = NameType.objects.get(code='O') code_type_os = CodeType.objects.get(code='unit_id') print(filename) current_generation = Generation.objects.current() new_generation = Generation.objects.new() if not new_generation: raise Exception("No new generation to be used for import!") ds = DataSource(filename) layer = ds[0] for feat in layer: name = feat['NAME'].value if not isinstance(name, six.text_type): name = name.decode('iso-8859-1') name = re.sub(r'\s*\(DET( NO \d+|)\)\s*(?i)', '', name) name = re.sub(r'\s+', ' ', name) ons_code = feat['CODE'].value if feat['CODE'].value not in ('999999', '999999999') else None unit_id = str(feat['UNIT_ID'].value) area_code = feat['AREA_CODE'].value patch = self.patch_boundary_line(name, ons_code, unit_id, area_code) if 'ons-code' in patch: ons_code = patch['ons-code'] elif 'unit-id' in patch: unit_id = patch['unit-id'] if area_code == 'NCP': continue # Ignore Non Parished Areas if ons_code in self.ons_code_to_shape: m, poly = self.ons_code_to_shape[ons_code] try: m_name = m.names.get(type=name_type).name except Name.DoesNotExist: m_name = m.name # If running without commit for dry run, so nothing being stored in db if name != m_name: raise Exception("ONS code %s is used for %s and %s" % (ons_code, name, m_name)) # Otherwise, combine the two shapes for one area poly.append(feat.geom) continue if unit_id in self.unit_id_to_shape: m, poly = self.unit_id_to_shape[unit_id] try: m_name = m.names.get(type=name_type).name except Name.DoesNotExist: m_name = m.name # If running without commit for dry run, so nothing being stored in db if name != m_name: raise Exception("Unit ID code %s is used for %s and %s" % (unit_id, name, m_name)) # Otherwise, combine the two shapes for one area poly.append(feat.geom) continue if ons_code: country = ons_code[0] # Hooray! elif area_code in ('CED', 'CTY', 'DIW', 'DIS', 'MTW', 'MTD', 'LBW', 'LBO', 'LAC', 'GLA'): country = 'E' else: raise Exception(area_code) try: check = control.check(name, area_code, country, feat.geom, ons_code=ons_code, commit=options['commit']) if check is True: raise Area.DoesNotExist if isinstance(check, Area): m = check try: ons_code = m.codes.get(type=code_version).code except Code.DoesNotExist: ons_code = None elif ons_code: m = Area.objects.get(codes__type=code_version, codes__code=ons_code) elif unit_id: m = Area.objects.get( codes__type=code_type_os, codes__code=unit_id, generation_high=current_generation) m_name = m.names.get(type=name_type).name if name != m_name: raise Exception("Unit ID code %s is %s in DB but %s in SHP file" % (unit_id, m_name, name)) else: raise Exception('Area "%s" (%s) has neither ONS code nor unit ID' % (name, area_code)) if int(options['verbosity']) > 1: print(" Area matched, %s" % (m, )) except Area.DoesNotExist: print(" New area: %s %s %s %s" % (area_code, ons_code, unit_id, name)) m = Area( name=name, # If committing, this will be overwritten by the m.names.update_or_create type=Type.objects.get(code=area_code), country=Country.objects.get(code=country), generation_low=new_generation, generation_high=new_generation, ) if m.generation_high and current_generation and m.generation_high.id < current_generation.id: raise Exception("Area %s found, but not in current generation %s" % (m, current_generation)) m.generation_high = new_generation if options['commit']: m.save() # Make a GEOS geometry only to check for validity: g = feat.geom geos_g = g.geos if not geos_g.valid: print(" Geometry of %s %s not valid" % (ons_code, m)) geos_g = fix_invalid_geos_geometry(geos_g) if geos_g is None: raise Exception("The geometry for area %s was invalid and couldn't be fixed" % name) g = None else: g = geos_g.ogr poly = [g] if options['commit']: m.names.update_or_create(type=name_type, defaults={'name': name}) if ons_code: self.ons_code_to_shape[ons_code] = (m, poly) if options['commit']: m.codes.update_or_create(type=code_version, defaults={'code': ons_code}) if unit_id: self.unit_id_to_shape[unit_id] = (m, poly) if options['commit']: m.codes.update_or_create(type=code_type_os, defaults={'code': unit_id}) if options['commit']: save_polygons(self.unit_id_to_shape) save_polygons(self.ons_code_to_shape)
def update_or_create(): if osm_code: m = osm_code.area else: m = Area( name=name, type=Type.objects.get(code=area_code), country=Country.objects.get(code='G'), parent_area=parent_area, generation_low=new_generation, generation_high=new_generation, ) if m.generation_high and current_generation and m.generation_high.id < current_generation.id: raise Exception, "Area %s found, but not in current generation %s" % ( m, current_generation) m.generation_high = new_generation g = feat.geom.transform(4326, clone=True) # In generating the data we should have # excluded any "polygons" with less than four # points (the final one being the same as the # first), but just in case: for polygon in g: if g.num_points < 4: return poly = [g] if options['commit']: m.save() if name not in kml_data.data: print json.dumps(kml_data.data, sort_keys=True, indent=4) raise Exception, u"Will fail to find '%s' in the dictionary" % ( name, ) for k, v in kml_data.data[name].items(): language_name = None if k == 'name': lang = 'default' language_name = "OSM Default" else: name_match = re.search(r'^name:(.+)$', k) if name_match: lang = name_match.group(1) if lang in language_code_to_name: language_name = language_code_to_name[ lang] if not language_name: continue # Otherwise, make sure that a NameType for this language exists: NameType.objects.update_or_create( {'code': lang}, { 'code': lang, 'description': language_name }) name_type = NameType.objects.get(code=lang) m.names.update_or_create({'type': name_type}, {'name': v}) m.codes.update_or_create({'type': code_type_osm}, {'code': osm_id}) save_polygons({code: (m, poly)})
def handle_label(self, filename, **options): missing_options = [] for k in [ 'generation_id', 'area_type_code', 'name_type_code', 'country_code' ]: if options[k]: continue else: missing_options.append(k) if missing_options: message_start = "Missing arguments " if len( missing_options) > 1 else "Missing argument " message = message_start + " ".join('--{0}'.format(k) for k in missing_options) raise CommandError(message) generation_id = options['generation_id'] area_type_code = options['area_type_code'] name_type_code = options['name_type_code'] country_code = options['country_code'] override_name = options['override_name'] name_field = options['name_field'] if not (override_name or name_field): name_field = 'Name' override_code = options['override_code'] code_field = options['code_field'] code_type_code = options['code_type'] encoding = options['encoding'] or 'utf-8' if len(area_type_code) > 3: raise CommandError( "Area type code must be 3 letters or fewer, sorry") if name_field and override_name: raise CommandError( "You must not specify both --name_field and --override_name") if code_field and override_code: raise CommandError( "You must not specify both --code_field and --override_code") using_code = (code_field or override_code) if (using_code and not code_type_code) or (not using_code and code_type_code): raise CommandError( "If you want to save a code, specify --code_type and either --code_field or --override_code" ) try: area_type = Type.objects.get(code=area_type_code) except: type_desc = raw_input( 'Please give a description for area type code %s: ' % area_type_code) area_type = Type(code=area_type_code, description=type_desc) if options['commit']: area_type.save() try: name_type = NameType.objects.get(code=name_type_code) except: name_desc = raw_input( 'Please give a description for name type code %s: ' % name_type_code) name_type = NameType(code=name_type_code, description=name_desc) if options['commit']: name_type.save() try: country = Country.objects.get(code=country_code) except: country_name = raw_input( 'Please give the name for country code %s: ' % country_code) country = Country(code=country_code, name=country_name) if options['commit']: country.save() if code_type_code: try: code_type = CodeType.objects.get(code=code_type_code) except: code_desc = raw_input( 'Please give a description for code type %s: ' % code_type_code) code_type = CodeType(code=code_type_code, description=code_desc) if options['commit']: code_type.save() self.stdout.write("Importing from %s" % filename) if not options['commit']: self.stdout.write( '(will not save to db as --commit not specified)') current_generation = Generation.objects.current() new_generation = Generation.objects.get(id=generation_id) def verbose(*args): if int(options['verbosity']) > 1: self.stdout.write(" ".join(str(a) for a in args)) ds = DataSource(filename) layer = ds[0] if (override_name or override_code) and len(layer) > 1: message = "Warning: you have specified an override %s and this file contains more than one feature; multiple areas with the same %s will be created" if override_name: self.stdout.write(message % ('name', 'name')) if override_code: self.stdout.write(message % ('code', 'code')) for feat in layer: if override_name: name = override_name else: try: name = feat[name_field].value except: choices = ', '.join(layer.fields) raise CommandError( "Could not find name using name field '%s' - should it be something else? It will be one of these: %s. Specify which with --name_field" % (name_field, choices)) try: if not isinstance(name, unicode): name = name.decode(encoding) except: raise CommandError( "Could not decode name using encoding '%s' - is it in another encoding? Specify one with --encoding" % encoding) name = re.sub('\s+', ' ', name) if not name: raise Exception("Could not find a name to use for area") code = None if override_code: code = override_code elif code_field: try: code = feat[code_field].value except: choices = ', '.join(layer.fields) raise CommandError( "Could not find code using code field '%s' - should it be something else? It will be one of these: %s. Specify which with --code_field" % (code_field, choices)) self.stdout.write(" looking at '%s'%s" % (name.encode('utf-8'), (' (%s)' % code) if code else '')) g = None if hasattr(feat, 'geom'): g = feat.geom.transform(settings.MAPIT_AREA_SRID, clone=True) try: if options['new']: # Always want a new area raise Area.DoesNotExist if code: matching_message = "code %s of code type %s" % (code, code_type) areas = Area.objects.filter( codes__code=code, codes__type=code_type).order_by('-generation_high') else: matching_message = "name %s of area type %s" % (name, area_type) areas = Area.objects.filter( name=name, type=area_type).order_by('-generation_high') if len(areas) == 0: verbose(" the area was not found - creating a new one") raise Area.DoesNotExist m = areas[0] verbose(" found the area") if options['preserve']: # Find whether we need to create a new Area: previous_geos_geometry = m.polygons.collect() if m.generation_high < current_generation.id: # Then it was missing in current_generation: verbose( " area existed previously, but was missing from", current_generation) raise Area.DoesNotExist elif g is None: if previous_geos_geometry is not None: verbose(" area is now empty") raise Area.DoesNotExist else: verbose(" the area has remained empty") elif previous_geos_geometry is None: # It was empty in the previous generation: verbose(" area was empty in", current_generation) raise Area.DoesNotExist else: # Otherwise, create a new Area unless the # polygons were the same in current_generation: previous_geos_geometry = previous_geos_geometry.simplify( tolerance=0) new_geos_geometry = g.geos.simplify(tolerance=0) create_new_area = not previous_geos_geometry.equals( new_geos_geometry) p = previous_geos_geometry.sym_difference( new_geos_geometry ).area / previous_geos_geometry.area verbose(" change in area is:", "%.03f%%" % (100 * p, )) if create_new_area: verbose( " the area", m, "has changed, creating a new area due to --preserve" ) raise Area.DoesNotExist else: verbose(" the area remained the same") else: # If --preserve is not specified, the code or the name must be unique: if len(areas) > 1: raise Area.MultipleObjectsReturned, "There was more than one area with %s, and --preserve was not specified" % ( matching_message, ) except Area.DoesNotExist: m = Area( name=name, type=area_type, country=country, # parent_area = parent_area, generation_low=new_generation, generation_high=new_generation, ) if options['use_code_as_id'] and code: m.id = int(code) # check that we are not about to skip a generation if m.generation_high and current_generation and m.generation_high.id < current_generation.id: raise Exception, "Area %s found, but not in current generation %s" % ( m, current_generation) m.generation_high = new_generation if options['fix_invalid_polygons'] and g is not None: # Make a GEOS geometry only to check for validity: geos_g = g.geos if not geos_g.valid: geos_g = fix_invalid_geos_geometry(geos_g) if geos_g is None: self.stdout.write( "The geometry for area %s was invalid and couldn't be fixed" % name) g = None else: g = geos_g.ogr poly = [g] if g is not None else [] if options['commit']: m.save() m.names.update_or_create({'type': name_type}, {'name': name}) if code: m.codes.update_or_create({'type': code_type}, {'code': code}) save_polygons({m.id: (m, poly)})
def handle_label(self, filename, **options): country = Country.objects.get(code='N') oa_type = Type.objects.get(code='OUA') soa_type = Type.objects.get(code='OLF') name_type = NameType.objects.get(code='S') code_type = CodeType.objects.get(code='ons') current_generation = Generation.objects.current() new_generation = Generation.objects.new() if not new_generation: raise Exception("No new generation to be used for import!") # Compile an alphabetical list of NI councils and their wards, OA codes # are assigned alphabetically. if not self.councils: self.councils = Area.objects.filter(type=Type.objects.get( code='LGD')).order_by('name').values() for lgd in self.councils: lges = Area.objects.filter(parent_area=lgd['id']) areas = [] for lge in lges: lgws = Area.objects.filter(parent_area=lge).values() areas += lgws lgd['wards'] = sorted(areas, key=lambda x: x['name']) ds = DataSource(filename) layer = ds[0] layer_name = str(layer) for feat in layer: if layer_name == 'soa': area_type = soa_type ons_code = feat['SOA_CODE'].value name = feat['SOA_LABEL'].value.replace('_', ' ') elif layer_name == 'OA_ni': area_type = oa_type ons_code = feat['OA_CODE'].value name = 'Output Area %s' % ons_code else: raise Exception('Bad data passed in') council = ord(ons_code[2:3]) - 65 ward = int(ons_code[4:6]) - 1 if ward == 98: # SOA covers two wards, set parent to council, best we can do parent = self.councils[council]['id'] else: parent = self.councils[council]['wards'][ward]['id'] try: m = Area.objects.get(codes__type=code_type, codes__code=ons_code) if int(options['verbosity']) > 1: print(" Area matched, %s" % (m, )) except Area.DoesNotExist: print(" New area: %s" % (ons_code)) m = Area( name= name, # If committing, this will be overwritten by the m.names.update_or_create type=area_type, country=country, parent_area_id=parent, generation_low=new_generation, generation_high=new_generation, ) if m.generation_high and current_generation and m.generation_high.id < current_generation.id: raise Exception( "Area %s found, but not in current generation %s" % (m, current_generation)) m.generation_high = new_generation m.parent_area_id = parent if options['commit']: m.save() f = feat.geom f.srid = 29902 poly = [f] if options['commit']: m.names.update_or_create(type=name_type, defaults={'name': name}) if ons_code: self.ons_code_to_shape[ons_code] = (m, poly) if options['commit']: m.codes.update_or_create(type=code_type, defaults={'code': ons_code}) if options['commit']: save_polygons(self.ons_code_to_shape)
def handle_label(self, filename, **options): for k in ['generation_id','area_type_code','name_type_code','country_code']: if options[k]: continue raise Exception("Missing argument '--%s'" % k) generation_id = options['generation_id'] area_type_code = options['area_type_code'] name_type_code = options['name_type_code'] country_code = options['country_code'] area_type = Type.objects.get(code=area_type_code) name_type = NameType.objects.get(code=name_type_code) country = Country.objects.get(code=country_code) print "Importing from %s" % filename if not options['commit']: print '(will not save to db as --commit not specified)' current_generation = Generation.objects.current() new_generation = Generation.objects.get( id=generation_id ) # Need to parse the KML manually to get the ExtendedData kml_data = KML() xml.sax.parse(filename, kml_data) ds = DataSource(filename) layer = ds[0] for feat in layer: name = feat['Name'].value.decode('utf-8') name = re.sub('\s+', ' ', name) if not name: raise Exception( "Could not find a name to use for area" ) print " looking at '%s'" % name.encode('utf-8') try: m = Area.objects.get(name=name, type=area_type) except Area.DoesNotExist: m = Area( name = name, type = area_type, country = country, # parent_area = parent_area, generation_low = new_generation, generation_high = new_generation, ) # check that we are not about to skip a generation if m.generation_high and current_generation and m.generation_high.id < current_generation.id: raise Exception, "Area %s found, but not in current generation %s" % (m, current_generation) m.generation_high = new_generation g = feat.geom.transform(4326, clone=True) poly = [ g ] if options['commit']: m.save() m.names.update_or_create({ 'type': name_type }, { 'name': name }) save_polygons({ m.id : (m, poly) })
def handle_label(self, filename, **options): if not options['control']: raise Exception, "You must specify a control file" __import__(options['control']) control = sys.modules[options['control']] code_version = CodeType.objects.get(code=control.code_version()) name_type = NameType.objects.get(code='O') code_type_os = CodeType.objects.get(code='unit_id') print filename current_generation = Generation.objects.current() new_generation = Generation.objects.new() if not new_generation: raise Exception, "No new generation to be used for import!" ds = DataSource(filename) layer = ds[0] for feat in layer: name = feat['NAME'].value if not isinstance(name, unicode): name = name.decode('iso-8859-1') name = re.sub('\s*\(DET( NO \d+|)\)\s*(?i)', '', name) name = re.sub('\s+', ' ', name) ons_code = feat['CODE'].value if feat['CODE'].value not in ( '999999', '999999999') else None unit_id = str(feat['UNIT_ID'].value) area_code = feat['AREA_CODE'].value patch = self.patch_boundary_line(ons_code, area_code) if patch == True: ons_code = None elif patch: ons_code = patch if area_code == 'NCP': continue # Ignore Non Parished Areas if ons_code in self.ons_code_to_shape: m, poly = self.ons_code_to_shape[ons_code] try: m_name = m.names.get(type=name_type).name except Name.DoesNotExist: m_name = m.name # If running without commit for dry run, so nothing being stored in db if name != m_name: raise Exception, "ONS code %s is used for %s and %s" % ( ons_code, name, m_name) # Otherwise, combine the two shapes for one area poly.append(feat.geom) continue if unit_id in self.unit_id_to_shape: m, poly = self.unit_id_to_shape[unit_id] try: m_name = m.names.get(type=name_type).name except Name.DoesNotExist: m_name = m.name # If running without commit for dry run, so nothing being stored in db if name != m_name: raise Exception, "Unit ID code %s is used for %s and %s" % ( unit_id, name, m_name) # Otherwise, combine the two shapes for one area poly.append(feat.geom) continue if code_version.code == 'gss' and ons_code: country = ons_code[0] # Hooray! elif area_code in ('CED', 'CTY', 'DIW', 'DIS', 'MTW', 'MTD', 'LBW', 'LBO', 'LAC', 'GLA'): country = 'E' elif code_version.code == 'gss': raise Exception, area_code elif (area_code == 'EUR' and 'Scotland' in name) or area_code in ( 'SPC', 'SPE') or (ons_code and ons_code[0:3] in ('00Q', '00R')): country = 'S' elif (area_code == 'EUR' and 'Wales' in name) or area_code in ( 'WAC', 'WAE') or (ons_code and ons_code[0:3] in ('00N', '00P')): country = 'W' elif area_code in ('EUR', 'UTA', 'UTE', 'UTW', 'CPC'): country = 'E' else: # WMC # Make sure WMC are loaded after all wards... area_within = Area.objects.filter( type__code__in=('UTW', 'UTE', 'MTW', 'COP', 'LBW', 'DIW'), polygons__polygon__contains=feat.geom.geos.point_on_surface )[0] country = area_within.country.code # Can't do the above ons_code checks with new GSS codes, will have to do more PinP checks # Do parents in separate P-in-P code after this is done. try: check = control.check(name, area_code, country, feat.geom) if check == True: raise Area.DoesNotExist if isinstance(check, Area): m = check ons_code = m.codes.get(type=code_version).code elif ons_code: m = Area.objects.get(codes__type=code_version, codes__code=ons_code) elif unit_id: m = Area.objects.get(codes__type=code_type_os, codes__code=unit_id, generation_high=current_generation) m_name = m.names.get(type=name_type).name if name != m_name: raise Exception, "Unit ID code %s is %s in DB but %s in SHP file" % ( unit_id, m_name, name) else: raise Exception, 'Area "%s" (%s) has neither ONS code nor unit ID' % ( name, area_code) if int(options['verbosity']) > 1: print " Area matched, %s" % (m, ) except Area.DoesNotExist: print " New area: %s %s %s %s" % (area_code, ons_code, unit_id, name) m = Area( name= name, # If committing, this will be overwritten by the m.names.update_or_create type=Type.objects.get(code=area_code), country=Country.objects.get(code=country), generation_low=new_generation, generation_high=new_generation, ) if m.generation_high and current_generation and m.generation_high.id < current_generation.id: raise Exception, "Area %s found, but not in current generation %s" % ( m, current_generation) m.generation_high = new_generation if options['commit']: m.save() poly = [feat.geom] if options['commit']: m.names.update_or_create({'type': name_type}, {'name': name}) if ons_code: self.ons_code_to_shape[ons_code] = (m, poly) if options['commit']: m.codes.update_or_create({'type': code_version}, {'code': ons_code}) if unit_id: self.unit_id_to_shape[unit_id] = (m, poly) if options['commit']: m.codes.update_or_create({'type': code_type_os}, {'code': unit_id}) if options['commit']: save_polygons(self.unit_id_to_shape) save_polygons(self.ons_code_to_shape)
def handle_label(self, filename, **options): country = Country.objects.get(code='N') oa_type = Type.objects.get(code='OUA') soa_type = Type.objects.get(code='OLF') name_type = NameType.objects.get(code='S') code_type = CodeType.objects.get(code='ons') current_generation = Generation.objects.current() new_generation = Generation.objects.new() if not new_generation: raise Exception, "No new generation to be used for import!" # Compile an alphabetical list of NI councils and their wards, OA codes # are assigned alphabetically. if not self.councils: self.councils = Area.objects.filter(type=Type.objects.get(code='LGD')).order_by('name').values() for lgd in self.councils: lges = Area.objects.filter(parent_area=lgd['id']) areas = [] for lge in lges: lgws = Area.objects.filter(parent_area=lge).values() areas += lgws lgd['wards'] = sorted(areas, key=lambda x: x['name']) ds = DataSource(filename) layer = ds[0] layer_name = str(layer) for feat in layer: if layer_name == 'soa': area_type = soa_type ons_code = feat['SOA_CODE'].value name = feat['SOA_LABEL'].value.replace('_', ' ') elif layer_name == 'OA_ni': area_type = oa_type ons_code = feat['OA_CODE'].value name = 'Output Area %s' % ons_code else: raise Exception, 'Bad data passed in' council = ord(ons_code[2:3]) - 65 ward = int(ons_code[4:6]) - 1 if ward == 98: # SOA covers two wards, set parent to council, best we can do parent = self.councils[council]['id'] else: parent = self.councils[council]['wards'][ward]['id'] try: m = Area.objects.get(codes__type=code_type, codes__code=ons_code) if int(options['verbosity']) > 1: print " Area matched, %s" % (m, ) except Area.DoesNotExist: print " New area: %s" % (ons_code) m = Area( name = name, # If committing, this will be overwritten by the m.names.update_or_create type = area_type, country = country, parent_area_id = parent, generation_low = new_generation, generation_high = new_generation, ) if m.generation_high and current_generation and m.generation_high.id < current_generation.id: raise Exception, "Area %s found, but not in current generation %s" % (m, current_generation) m.generation_high = new_generation m.parent_area_id = parent if options['commit']: m.save() f = feat.geom f.srid = 29902 poly = [ f ] if options['commit']: m.names.update_or_create({ 'type': name_type }, { 'name': name }) if ons_code: self.ons_code_to_shape[ons_code] = (m, poly) if options['commit']: m.codes.update_or_create({ 'type': code_type }, { 'code': ons_code }) if options['commit']: save_polygons(self.ons_code_to_shape)
def handle_label(self, filename, **options): err = False for k in ['generation_id','area_type_code','name_type_code','country_code']: if options[k]: continue print "Missing argument '--%s'" % k err = True if err: sys.exit(1) generation_id = options['generation_id'] area_type_code = options['area_type_code'] name_type_code = options['name_type_code'] country_code = options['country_code'] name_field = options['name_field'] or 'Name' code_field = options['code_field'] code_type_code = options['code_type'] encoding = options['encoding'] or 'utf-8' if len(area_type_code)>3: print "Area type code must be 3 letters or fewer, sorry" sys.exit(1) if (code_field and not code_type_code) or (not code_field and code_type_code): print "You must specify both code_field and code_type, or neither." sys.exit(1) try: area_type = Type.objects.get(code=area_type_code) except: type_desc = raw_input('Please give a description for area type code %s: ' % area_type_code) area_type = Type(code=area_type_code, description=type_desc) if options['commit']: area_type.save() try: name_type = NameType.objects.get(code=name_type_code) except: name_desc = raw_input('Please give a description for name type code %s: ' % name_type_code) name_type = NameType(code=name_type_code, description=name_desc) if options['commit']: name_type.save() try: country = Country.objects.get(code=country_code) except: country_name = raw_input('Please give the name for country code %s: ' % country_code) country = Country(code=country_code, name=country_name) if options['commit']: country.save() if code_type_code: try: code_type = CodeType.objects.get(code=code_type_code) except: code_desc = raw_input('Please give a description for code type %s: ' % code_type_code) code_type = CodeType(code=code_type_code, description=code_desc) if options['commit']: code_type.save() print "Importing from %s" % filename if not options['commit']: print '(will not save to db as --commit not specified)' current_generation = Generation.objects.current() new_generation = Generation.objects.get( id=generation_id ) ds = DataSource(filename) layer = ds[0] for feat in layer: try: name = feat[name_field].value except: choices = ', '.join(layer.fields) print "Could not find name using name field '%s' - should it be something else? It will be one of these: %s. Specify which with --name_field" % (name_field, choices) sys.exit(1) try: name = name.decode(encoding) except: print "Could not decode name using encoding '%s' - is it in another encoding? Specify one with --encoding" % encoding sys.exit(1) name = re.sub('\s+', ' ', name) if not name: raise Exception( "Could not find a name to use for area" ) code = None if code_field: try: code = feat[code_field].value except: choices = ', '.join(layer.fields) print "Could not find code using code field '%s' - should it be something else? It will be one of these: %s. Specify which with --code_field" % (code_field, choices) sys.exit(1) print " looking at '%s'%s" % ( name.encode('utf-8'), (' (%s)' % code) if code else '' ) try: if code: m = Area.objects.get(codes__code=code, codes__type=code_type) else: # Assumes unique names if no code column used m = Area.objects.get(name=name, type=area_type) except Area.DoesNotExist: m = Area( name = name, type = area_type, country = country, # parent_area = parent_area, generation_low = new_generation, generation_high = new_generation, ) if options['use_code_as_id'] and code: m.id = int(code) # check that we are not about to skip a generation if m.generation_high and current_generation and m.generation_high.id < current_generation.id: raise Exception, "Area %s found, but not in current generation %s" % (m, current_generation) m.generation_high = new_generation g = feat.geom.transform(4326, clone=True) poly = [ g ] if options['commit']: m.save() m.names.update_or_create({ 'type': name_type }, { 'name': name }) if code: m.codes.update_or_create({ 'type': code_type }, { 'code': code }) save_polygons({ m.id : (m, poly) })
def handle_label(self, filename, **options): err = False for k in [ 'generation_id', 'area_type_code', 'name_type_code', 'country_code' ]: if options[k]: continue print "Missing argument '--%s'" % k err = True if err: sys.exit(1) generation_id = options['generation_id'] area_type_code = options['area_type_code'] name_type_code = options['name_type_code'] country_code = options['country_code'] name_field = options['name_field'] or 'Name' code_field = options['code_field'] code_type_code = options['code_type'] encoding = options['encoding'] or 'utf-8' if len(area_type_code) > 3: print "Area type code must be 3 letters or fewer, sorry" sys.exit(1) if (code_field and not code_type_code) or (not code_field and code_type_code): print "You must specify both code_field and code_type, or neither." sys.exit(1) try: area_type = Type.objects.get(code=area_type_code) except: type_desc = raw_input( 'Please give a description for area type code %s: ' % area_type_code) area_type = Type(code=area_type_code, description=type_desc) if options['commit']: area_type.save() try: name_type = NameType.objects.get(code=name_type_code) except: name_desc = raw_input( 'Please give a description for name type code %s: ' % name_type_code) name_type = NameType(code=name_type_code, description=name_desc) if options['commit']: name_type.save() try: country = Country.objects.get(code=country_code) except: country_name = raw_input( 'Please give the name for country code %s: ' % country_code) country = Country(code=country_code, name=country_name) if options['commit']: country.save() if code_type_code: try: code_type = CodeType.objects.get(code=code_type_code) except: code_desc = raw_input( 'Please give a description for code type %s: ' % code_type_code) code_type = CodeType(code=code_type_code, description=code_desc) if options['commit']: code_type.save() print "Importing from %s" % filename if not options['commit']: print '(will not save to db as --commit not specified)' current_generation = Generation.objects.current() new_generation = Generation.objects.get(id=generation_id) ds = DataSource(filename) layer = ds[0] for feat in layer: try: name = feat[name_field].value except: choices = ', '.join(layer.fields) print "Could not find name using name field '%s' - should it be something else? It will be one of these: %s. Specify which with --name_field" % ( name_field, choices) sys.exit(1) try: name = name.decode(encoding) except: print "Could not decode name using encoding '%s' - is it in another encoding? Specify one with --encoding" % encoding sys.exit(1) name = re.sub('\s+', ' ', name) if not name: raise Exception("Could not find a name to use for area") code = None if code_field: try: code = feat[code_field].value except: choices = ', '.join(layer.fields) print "Could not find code using code field '%s' - should it be something else? It will be one of these: %s. Specify which with --code_field" % ( code_field, choices) sys.exit(1) print " looking at '%s'%s" % (name.encode('utf-8'), (' (%s)' % code) if code else '') try: if code: m = Area.objects.get(codes__code=code, codes__type=code_type) else: # Assumes unique names if no code column used m = Area.objects.get(name=name, type=area_type) except Area.DoesNotExist: m = Area( name=name, type=area_type, country=country, # parent_area = parent_area, generation_low=new_generation, generation_high=new_generation, ) if options['use_code_as_id'] and code: m.id = int(code) # check that we are not about to skip a generation if m.generation_high and current_generation and m.generation_high.id < current_generation.id: raise Exception, "Area %s found, but not in current generation %s" % ( m, current_generation) m.generation_high = new_generation g = feat.geom.transform(4326, clone=True) poly = [g] if options['commit']: m.save() m.names.update_or_create({'type': name_type}, {'name': name}) if code: m.codes.update_or_create({'type': code_type}, {'code': code}) save_polygons({m.id: (m, poly)})
def process_file(self, filename, area_code, srid, control, options): code_version = CodeType.objects.get(code=control.code_version()) name_type = NameType.objects.get(code="N") code_type_osni = CodeType.objects.get(code="osni_oid") if not hasattr(self, area_code): raise Exception("Don't know how to extract features from %s files" % area_code) area_code_info = getattr(self, area_code)(srid) print(filename) current_generation = Generation.objects.current() new_generation = Generation.objects.new() if not new_generation: raise Exception("No new generation to be used for import!") ds = DataSource(filename) layer = ds[0] for feat in layer: name, ons_code, osni_object_id = area_code_info.extract_fields(feat) name = self.format_name(name) if ons_code in self.ons_code_to_shape: m, poly = self.ons_code_to_shape[ons_code] try: m_name = m.names.get(type=name_type).name except Name.DoesNotExist: m_name = m.name # If running without commit for dry run, so nothing being stored in db if name != m_name: raise Exception("ONS code %s is used for %s and %s" % (ons_code, name, m_name)) # Otherwise, combine the two shapes for one area poly.append(feat.geom) continue if osni_object_id in self.osni_object_id_to_shape: m, poly = self.osni_object_id_to_shape[osni_object_id] try: m_name = m.names.get(type=name_type).name except Name.DoesNotExist: m_name = m.name # If running without commit for dry run, so nothing being stored in db if name != m_name: raise Exception("OSNI Object ID code %s is used for %s and %s" % (osni_object_id, name, m_name)) # Otherwise, combine the two shapes for one area poly.append(feat.geom) continue country = "N" try: check = control.check(name, area_code, country, feat.geom) if check is True: raise Area.DoesNotExist if isinstance(check, Area): m = check ons_code = m.codes.get(type=code_version).code elif ons_code: m = Area.objects.get(codes__type=code_version, codes__code=ons_code) elif osni_object_id: m = Area.objects.get( codes__type=code_type_osni, codes__code=osni_object_id, generation_high=current_generation ) m_name = m.names.get(type=name_type).name if name != m_name: raise Exception( "OSNI Object ID code %s is %s in DB but %s in SHP file" % (osni_object_id, m_name, name) ) else: raise Exception('Area "%s" (%s) has neither ONS code nor OSNI Object ID' % (name, area_code)) if int(options["verbosity"]) > 1: print(" Area matched, %s" % (m,)) except Area.DoesNotExist: area_type = Type.objects.get(code=area_code) # It's possible we already have NIE entries (without any codes) in the db try: if area_code == "NIE": matching_name = name.title().replace("And", "and") m = Area.objects.get(name=matching_name, type=area_type, generation_high=current_generation) if int(options["verbosity"]) > 1: print(" Area matched (via name), %s" % (m,)) else: raise Area.DoesNotExist("Still doesn't exist") except Area.DoesNotExist: print(" New area: %s %s %s %s" % (area_code, ons_code, osni_object_id, name)) m = Area( name=name, # Not overwritten by m.names.update_or_create as no "N" use type=area_type, country=Country.objects.get(code=country), generation_low=new_generation, generation_high=new_generation, ) if m.generation_high and current_generation and m.generation_high.id < current_generation.id: raise Exception("Area %s found, but not in current generation %s" % (m, current_generation)) m.generation_high = new_generation if options["commit"]: m.save() # Make a GEOS geometry only to check for validity: g = feat.geom geos_g = g.geos if not geos_g.valid: print(" Geometry of %s %s not valid" % (ons_code, m)) geos_g = fix_invalid_geos_geometry(geos_g) if geos_g is None: raise Exception("The geometry for area %s was invalid and couldn't be fixed" % name) g = None else: g = geos_g.ogr g = area_code_info.transform_geom(g) poly = [g] if options["commit"]: m.names.update_or_create(type=name_type, defaults={"name": name}) if ons_code: self.ons_code_to_shape[ons_code] = (m, poly) if options["commit"]: m.codes.update_or_create(type=code_version, defaults={"code": ons_code}) if osni_object_id: self.osni_object_id_to_shape[osni_object_id] = (m, poly) if options["commit"]: m.codes.update_or_create(type=code_type_osni, defaults={"code": osni_object_id}) if options["commit"]: save_polygons(self.osni_object_id_to_shape) save_polygons(self.ons_code_to_shape)
def handle_label(self, filename, **options): if not options['control']: raise Exception("You must specify a control file") __import__(options['control']) control = sys.modules[options['control']] code_version = CodeType.objects.get(code='gss') name_type = NameType.objects.get(code='O') code_type_os = CodeType.objects.get(code='unit_id') print(filename) current_generation = Generation.objects.current() new_generation = Generation.objects.new() if not new_generation: raise Exception("No new generation to be used for import!") ds = DataSource(filename) layer = ds[0] for feat in layer: name = feat['NAME'].value if not isinstance(name, six.text_type): name = name.decode('iso-8859-1') name = re.sub(r'\s*\(DET( NO \d+|)\)\s*(?i)', '', name) name = re.sub(r'\s+', ' ', name) ons_code = feat['CODE'].value if feat['CODE'].value not in ('999999', '999999999') else None unit_id = str(feat['UNIT_ID'].value) area_code = feat['AREA_CODE'].value patch = self.patch_boundary_line(name, ons_code, unit_id, area_code) if 'ons-code' in patch: ons_code = patch['ons-code'] elif 'unit-id' in patch: unit_id = patch['unit-id'] if area_code == 'NCP': continue # Ignore Non Parished Areas if ons_code in self.ons_code_to_shape: m, poly = self.ons_code_to_shape[ons_code] try: m_name = m.names.get(type=name_type).name except Name.DoesNotExist: m_name = m.name # If running without commit for dry run, so nothing being stored in db if name != m_name: raise Exception("ONS code %s is used for %s and %s" % (ons_code, name, m_name)) # Otherwise, combine the two shapes for one area poly.append(feat.geom) continue if unit_id in self.unit_id_to_shape: m, poly = self.unit_id_to_shape[unit_id] try: m_name = m.names.get(type=name_type).name except Name.DoesNotExist: m_name = m.name # If running without commit for dry run, so nothing being stored in db if name != m_name: raise Exception("Unit ID code %s is used for %s and %s" % (unit_id, name, m_name)) # Otherwise, combine the two shapes for one area poly.append(feat.geom) continue if ons_code: country = ons_code[0] # Hooray! elif area_code in ('CED', 'CTY', 'DIW', 'DIS', 'MTW', 'MTD', 'LBW', 'LBO', 'LAC', 'GLA'): country = 'E' else: raise Exception(area_code) try: check = control.check(name, area_code, country, feat.geom, ons_code=ons_code, commit=options['commit']) if check is True: raise Area.DoesNotExist if isinstance(check, Area): m = check try: ons_code = m.codes.get(type=code_version).code except Code.DoesNotExist: ons_code = None elif ons_code: m = Area.objects.get(codes__type=code_version, codes__code=ons_code) elif unit_id: m = Area.objects.get( codes__type=code_type_os, codes__code=unit_id, generation_high=current_generation) m_name = m.names.get(type=name_type).name if name != m_name: raise Exception("Unit ID code %s is %s in DB but %s in SHP file" % (unit_id, m_name, name)) else: raise Exception('Area "%s" (%s) has neither ONS code nor unit ID' % (name, area_code)) if int(options['verbosity']) > 1: print(" Area matched, %s" % (m, )) except Area.DoesNotExist: print(" New area: %s %s %s %s" % (area_code, ons_code, unit_id, name)) m = Area( name=name, # If committing, this will be overwritten by the m.names.update_or_create type=Type.objects.get(code=area_code), country=Country.objects.get(code=country), generation_low=new_generation, generation_high=new_generation, ) if m.generation_high and current_generation and m.generation_high.id < current_generation.id: raise Exception("Area %s found, but not in current generation %s" % (m, current_generation)) m.generation_high = new_generation if options['commit']: m.save() # Make a GEOS geometry only to check for validity: g = feat.geom geos_g = g.geos if not geos_g.valid: print(" Geometry of %s %s not valid" % (ons_code, m)) geos_g = fix_invalid_geos_geometry(geos_g) if geos_g is None: raise Exception("The geometry for area %s was invalid and couldn't be fixed" % name) g = None else: g = geos_g.ogr poly = [g] if options['commit']: m.names.update_or_create(type=name_type, defaults={'name': name}) if ons_code: self.ons_code_to_shape[ons_code] = (m, poly) if options['commit']: m.codes.update_or_create(type=code_version, defaults={'code': ons_code}) if unit_id: self.unit_id_to_shape[unit_id] = (m, poly) if options['commit']: m.codes.update_or_create(type=code_type_os, defaults={'code': unit_id}) if options['commit']: save_polygons(self.unit_id_to_shape) save_polygons(self.ons_code_to_shape)
def handle_label(self, filename, **options): if not options['control']: raise Exception, "You must specify a control file" __import__(options['control']) control = sys.modules[options['control']] code_version = CodeType.objects.get(code=control.code_version()) name_type = NameType.objects.get(code='O') code_type_os = CodeType.objects.get(code='unit_id') print filename current_generation = Generation.objects.current() new_generation = Generation.objects.new() if not new_generation: raise Exception, "No new generation to be used for import!" ds = DataSource(filename) layer = ds[0] for feat in layer: name = feat['NAME'].value if not isinstance(name, unicode): name = name.decode('iso-8859-1') name = re.sub('\s*\(DET( NO \d+|)\)\s*(?i)', '', name) name = re.sub('\s+', ' ', name) ons_code = feat['CODE'].value if feat['CODE'].value not in ('999999', '999999999') else None unit_id = str(feat['UNIT_ID'].value) area_code = feat['AREA_CODE'].value patch = self.patch_boundary_line(ons_code, area_code) if patch == True: ons_code = None elif patch: ons_code = patch if area_code == 'NCP': continue # Ignore Non Parished Areas if ons_code in self.ons_code_to_shape: m, poly = self.ons_code_to_shape[ons_code] try: m_name = m.names.get(type=name_type).name except Name.DoesNotExist: m_name = m.name # If running without commit for dry run, so nothing being stored in db if name != m_name: raise Exception, "ONS code %s is used for %s and %s" % (ons_code, name, m_name) # Otherwise, combine the two shapes for one area poly.append(feat.geom) continue if unit_id in self.unit_id_to_shape: m, poly = self.unit_id_to_shape[unit_id] try: m_name = m.names.get(type=name_type).name except Name.DoesNotExist: m_name = m.name # If running without commit for dry run, so nothing being stored in db if name != m_name: raise Exception, "Unit ID code %s is used for %s and %s" % (unit_id, name, m_name) # Otherwise, combine the two shapes for one area poly.append(feat.geom) continue if code_version.code == 'gss' and ons_code: country = ons_code[0] # Hooray! elif area_code in ('CED', 'CTY', 'DIW', 'DIS', 'MTW', 'MTD', 'LBW', 'LBO', 'LAC', 'GLA'): country = 'E' elif code_version.code == 'gss': raise Exception, area_code elif (area_code == 'EUR' and 'Scotland' in name) or area_code in ('SPC', 'SPE') or (ons_code and ons_code[0:3] in ('00Q', '00R')): country = 'S' elif (area_code == 'EUR' and 'Wales' in name) or area_code in ('WAC', 'WAE') or (ons_code and ons_code[0:3] in ('00N', '00P')): country = 'W' elif area_code in ('EUR', 'UTA', 'UTE', 'UTW', 'CPC'): country = 'E' else: # WMC # Make sure WMC are loaded after all wards... area_within = Area.objects.filter(type__code__in=('UTW','UTE','MTW','COP','LBW','DIW'), polygons__polygon__contains=feat.geom.geos.point_on_surface)[0] country = area_within.country.code # Can't do the above ons_code checks with new GSS codes, will have to do more PinP checks # Do parents in separate P-in-P code after this is done. try: check = control.check(name, area_code, country, feat.geom) if check == True: raise Area.DoesNotExist if isinstance(check, Area): m = check ons_code = m.codes.get(type=code_version).code elif ons_code: m = Area.objects.get(codes__type=code_version, codes__code=ons_code) elif unit_id: m = Area.objects.get(codes__type=code_type_os, codes__code=unit_id, generation_high=current_generation) m_name = m.names.get(type=name_type).name if name != m_name: raise Exception, "Unit ID code %s is %s in DB but %s in SHP file" % (unit_id, m_name, name) else: raise Exception, 'Area "%s" (%s) has neither ONS code nor unit ID' % (name, area_code) if int(options['verbosity']) > 1: print " Area matched, %s" % (m, ) except Area.DoesNotExist: print " New area: %s %s %s %s" % (area_code, ons_code, unit_id, name) m = Area( name = name, # If committing, this will be overwritten by the m.names.update_or_create type = Type.objects.get(code=area_code), country = Country.objects.get(code=country), generation_low = new_generation, generation_high = new_generation, ) if m.generation_high and current_generation and m.generation_high.id < current_generation.id: raise Exception, "Area %s found, but not in current generation %s" % (m, current_generation) m.generation_high = new_generation if options['commit']: m.save() poly = [ feat.geom ] if options['commit']: m.names.update_or_create({ 'type': name_type }, { 'name': name }) if ons_code: self.ons_code_to_shape[ons_code] = (m, poly) if options['commit']: m.codes.update_or_create({ 'type': code_version }, { 'code': ons_code }) if unit_id: self.unit_id_to_shape[unit_id] = (m, poly) if options['commit']: m.codes.update_or_create({ 'type': code_type_os }, { 'code': unit_id }) if options['commit']: save_polygons(self.unit_id_to_shape) save_polygons(self.ons_code_to_shape)
def handle_label(self, filename, **options): err = False for k in ['generation_id','area_type_code','name_type_code','country_code']: if options[k]: continue print "Missing argument '--%s'" % k err = True if err: sys.exit(1) generation_id = options['generation_id'] area_type_code = options['area_type_code'] name_type_code = options['name_type_code'] country_code = options['country_code'] name_field = options['name_field'] or 'Name' code_field = options['code_field'] code_type_code = options['code_type'] encoding = options['encoding'] or 'utf-8' if len(area_type_code)>3: print "Area type code must be 3 letters or fewer, sorry" sys.exit(1) if (code_field and not code_type_code) or (not code_field and code_type_code): print "You must specify both code_field and code_type, or neither." sys.exit(1) try: area_type = Type.objects.get(code=area_type_code) except: type_desc = raw_input('Please give a description for area type code %s: ' % area_type_code) area_type = Type(code=area_type_code, description=type_desc) if options['commit']: area_type.save() try: name_type = NameType.objects.get(code=name_type_code) except: name_desc = raw_input('Please give a description for name type code %s: ' % name_type_code) name_type = NameType(code=name_type_code, description=name_desc) if options['commit']: name_type.save() try: country = Country.objects.get(code=country_code) except: country_name = raw_input('Please give the name for country code %s: ' % country_code) country = Country(code=country_code, name=country_name) if options['commit']: country.save() if code_type_code: try: code_type = CodeType.objects.get(code=code_type_code) except: code_desc = raw_input('Please give a description for code type %s: ' % code_type_code) code_type = CodeType(code=code_type_code, description=code_desc) if options['commit']: code_type.save() print "Importing from %s" % filename if not options['commit']: print '(will not save to db as --commit not specified)' current_generation = Generation.objects.current() new_generation = Generation.objects.get( id=generation_id ) def verbose(*args): if options['verbose']: print " ".join(str(a) for a in args) ds = DataSource(filename) layer = ds[0] for feat in layer: try: name = feat[name_field].value except: choices = ', '.join(layer.fields) print "Could not find name using name field '%s' - should it be something else? It will be one of these: %s. Specify which with --name_field" % (name_field, choices) sys.exit(1) try: name = name.decode(encoding) except: print "Could not decode name using encoding '%s' - is it in another encoding? Specify one with --encoding" % encoding sys.exit(1) name = re.sub('\s+', ' ', name) if not name: raise Exception( "Could not find a name to use for area" ) code = None if code_field: try: code = feat[code_field].value except: choices = ', '.join(layer.fields) print "Could not find code using code field '%s' - should it be something else? It will be one of these: %s. Specify which with --code_field" % (code_field, choices) sys.exit(1) print " looking at '%s'%s" % ( name.encode('utf-8'), (' (%s)' % code) if code else '' ) g = feat.geom.transform(4326, clone=True) try: if code: matching_message = "code %s of code type %s" % (code, code_type) areas = Area.objects.filter(codes__code=code, codes__type=code_type).order_by('-generation_high') else: matching_message = "name %s of area type %s" % (name, area_type) areas = Area.objects.filter(name=name, type=area_type).order_by('-generation_high') if len(areas) == 0: raise Area.DoesNotExist m = areas[0] verbose(" found the area") if options['preserve']: # Find whether we need to create a new Area: previous_geos_geometry = m.polygons.collect() if m.generation_high < current_generation.id: # Then it was missing in current_generation: verbose(" area existed previously, but was missing from", current_generation) create_new_area = True elif previous_geos_geometry is None: # It was empty in the previous generation: verbose(" area was empty in", current_generation) create_new_area = True else: # Otherwise, create a new Area unless the # polygons were the same in current_generation: previous_geos_geometry = previous_geos_geometry.simplify(tolerance=0) new_geos_geometry = g.geos.simplify(tolerance=0) create_new_area = not previous_geos_geometry.equals(new_geos_geometry) p = previous_geos_geometry.sym_difference(new_geos_geometry).area / previous_geos_geometry.area verbose(" change in area is:", "%.03f%%" % (100 * p,)) if create_new_area: verbose(" the area", m, "has changed, creating a new area due to --preserve") else: verbose(" the area remained the same") if create_new_area: m = Area( name = name, type = area_type, country = country, # parent_area = parent_area, generation_low = new_generation, generation_high = new_generation ) if options['use_code_as_id'] and code: m.id = int(code) else: # If --preserve is not specified, the code or the name must be unique: if len(areas) > 1: raise Area.MultipleObjectsReturned, "There was more than one area with %s, and --preserve was not specified" % (matching_message,) except Area.DoesNotExist: verbose(" the area was not found - creating a new one") m = Area( name = name, type = area_type, country = country, # parent_area = parent_area, generation_low = new_generation, generation_high = new_generation, ) if options['use_code_as_id'] and code: m.id = int(code) # check that we are not about to skip a generation if m.generation_high and current_generation and m.generation_high.id < current_generation.id: raise Exception, "Area %s found, but not in current generation %s" % (m, current_generation) m.generation_high = new_generation poly = [ g ] if options['commit']: m.save() m.names.update_or_create({ 'type': name_type }, { 'name': name }) if code: m.codes.update_or_create({ 'type': code_type }, { 'code': code }) save_polygons({ m.id : (m, poly) })
def process_file(self, filename, area_code, srid, control, options): code_version = CodeType.objects.get(code='gss') name_type = NameType.objects.get(code='N') code_type_osni = CodeType.objects.get(code='osni_oid') if not hasattr(self, area_code): raise Exception( "Don't know how to extract features from %s files" % area_code) area_code_info = getattr(self, area_code)(srid) print(filename) current_generation = Generation.objects.current() new_generation = Generation.objects.new() if not new_generation: raise Exception("No new generation to be used for import!") ds = DataSource(filename) layer = ds[0] for feat in layer: name, ons_code, osni_object_id = area_code_info.extract_fields( feat) name = self.format_name(name) if ons_code in self.ons_code_to_shape: m, poly = self.ons_code_to_shape[ons_code] try: m_name = m.names.get(type=name_type).name except Name.DoesNotExist: m_name = m.name # If running without commit for dry run, so nothing being stored in db if name != m_name: raise Exception("ONS code %s is used for %s and %s" % (ons_code, name, m_name)) # Otherwise, combine the two shapes for one area poly.append(feat.geom) continue if osni_object_id in self.osni_object_id_to_shape: m, poly = self.osni_object_id_to_shape[osni_object_id] try: m_name = m.names.get(type=name_type).name except Name.DoesNotExist: m_name = m.name # If running without commit for dry run, so nothing being stored in db if name != m_name: raise Exception( "OSNI Object ID code %s is used for %s and %s" % (osni_object_id, name, m_name)) # Otherwise, combine the two shapes for one area poly.append(feat.geom) continue country = 'N' try: check = control.check(name, area_code, country, feat.geom) if check is True: raise Area.DoesNotExist if isinstance(check, Area): m = check ons_code = m.codes.get(type=code_version).code elif ons_code: m = Area.objects.get(codes__type=code_version, codes__code=ons_code) elif osni_object_id: m = Area.objects.get(codes__type=code_type_osni, codes__code=osni_object_id, generation_high=current_generation) m_name = m.names.get(type=name_type).name if name != m_name: raise Exception( "OSNI Object ID code %s is %s in DB but %s in SHP file" % (osni_object_id, m_name, name)) else: raise Exception( 'Area "%s" (%s) has neither ONS code nor OSNI Object ID' % (name, area_code)) if int(options['verbosity']) > 1: print(" Area matched, %s" % (m, )) except Area.DoesNotExist: area_type = Type.objects.get(code=area_code) # It's possible we already have NIE entries (without any codes) in the db try: if area_code == 'NIE': matching_name = name.title().replace('And', 'and') m = Area.objects.get( name=matching_name, type=area_type, generation_high=current_generation) if int(options['verbosity']) > 1: print(" Area matched (via name), %s" % (m, )) else: raise Area.DoesNotExist("Still doesn't exist") except Area.DoesNotExist: print(" New area: %s %s %s %s" % (area_code, ons_code, osni_object_id, name)) m = Area( name= name, # Not overwritten by m.names.update_or_create as no "N" use type=area_type, country=Country.objects.get(code=country), generation_low=new_generation, generation_high=new_generation, ) if m.generation_high and current_generation and m.generation_high.id < current_generation.id: raise Exception( "Area %s found, but not in current generation %s" % (m, current_generation)) m.generation_high = new_generation if options['commit']: m.save() # Make a GEOS geometry only to check for validity: g = feat.geom geos_g = g.geos if not geos_g.valid: print(" Geometry of %s %s not valid" % (ons_code, m)) geos_g = fix_invalid_geos_geometry(geos_g) if geos_g is None: raise Exception( "The geometry for area %s was invalid and couldn't be fixed" % name) g = None else: g = geos_g.ogr g = area_code_info.transform_geom(g) poly = [g] if options['commit']: m.names.update_or_create(type=name_type, defaults={'name': name}) if ons_code: self.ons_code_to_shape[ons_code] = (m, poly) if options['commit']: m.codes.update_or_create(type=code_version, defaults={'code': ons_code}) if osni_object_id: self.osni_object_id_to_shape[osni_object_id] = (m, poly) if options['commit']: m.codes.update_or_create(type=code_type_osni, defaults={'code': osni_object_id}) if options['commit']: save_polygons(self.osni_object_id_to_shape) save_polygons(self.ons_code_to_shape)
def handle_label(self, filename, **options): if not options["control"]: raise Exception, "You must specify a control file" __import__(options["control"]) control = sys.modules[options["control"]] code_version = CodeType.objects.get(code=control.code_version()) name_type = NameType.objects.get(code="O") code_type_os = CodeType.objects.get(code="unit_id") print filename current_generation = Generation.objects.current() new_generation = Generation.objects.new() if not new_generation: raise Exception, "No new generation to be used for import!" ds = DataSource(filename) layer = ds[0] for feat in layer: name = feat["NAME"].value if not isinstance(name, unicode): name = name.decode("iso-8859-1") name = re.sub("\s*\(DET( NO \d+|)\)\s*(?i)", "", name) name = re.sub("\s+", " ", name) ons_code = feat["CODE"].value if feat["CODE"].value not in ("999999", "999999999") else None unit_id = str(feat["UNIT_ID"].value) area_code = feat["AREA_CODE"].value patch = self.patch_boundary_line(ons_code, area_code) if patch == True: ons_code = None elif patch: ons_code = patch if area_code == "NCP": continue # Ignore Non Parished Areas if ons_code in self.ons_code_to_shape: m, poly = self.ons_code_to_shape[ons_code] try: m_name = m.names.get(type=name_type).name except Name.DoesNotExist: m_name = m.name # If running without commit for dry run, so nothing being stored in db if name != m_name: raise Exception, "ONS code %s is used for %s and %s" % (ons_code, name, m_name) # Otherwise, combine the two shapes for one area poly.append(feat.geom) continue if unit_id in self.unit_id_to_shape: m, poly = self.unit_id_to_shape[unit_id] try: m_name = m.names.get(type=name_type).name except Name.DoesNotExist: m_name = m.name # If running without commit for dry run, so nothing being stored in db if name != m_name: raise Exception, "Unit ID code %s is used for %s and %s" % (unit_id, name, m_name) # Otherwise, combine the two shapes for one area poly.append(feat.geom) continue if code_version.code == "gss" and ons_code: country = ons_code[0] # Hooray! elif area_code in ("CED", "CTY", "DIW", "DIS", "MTW", "MTD", "LBW", "LBO", "LAC", "GLA"): country = "E" elif code_version.code == "gss": raise Exception, area_code elif ( (area_code == "EUR" and "Scotland" in name) or area_code in ("SPC", "SPE") or (ons_code and ons_code[0:3] in ("00Q", "00R")) ): country = "S" elif ( (area_code == "EUR" and "Wales" in name) or area_code in ("WAC", "WAE") or (ons_code and ons_code[0:3] in ("00N", "00P")) ): country = "W" elif area_code in ("EUR", "UTA", "UTE", "UTW", "CPC"): country = "E" else: # WMC # Make sure WMC are loaded after all wards... area_within = Area.objects.filter( type__code__in=("UTW", "UTE", "MTW", "COP", "LBW", "DIW"), polygons__polygon__contains=feat.geom.geos.point_on_surface, )[0] country = area_within.country.code # Can't do the above ons_code checks with new GSS codes, will have to do more PinP checks # Do parents in separate P-in-P code after this is done. try: check = control.check(name, area_code, country, feat.geom) if check == True: raise Area.DoesNotExist if isinstance(check, Area): m = check ons_code = m.codes.get(type=code_version).code elif ons_code: m = Area.objects.get(codes__type=code_version, codes__code=ons_code) elif unit_id: m = Area.objects.get(codes__type=code_type_os, codes__code=unit_id) m_name = m.names.get(type=name_type).name if name != m_name: raise Exception, "Unit ID code %s is %s in DB but %s in SHP file" % (unit_id, m_name, name) else: raise Exception, 'Area "%s" (%s) has neither ONS code nor unit ID' % (name, area_code) if int(options["verbosity"]) > 1: print " Area matched, %s" % (m,) except Area.DoesNotExist: print " New area: %s %s %s %s" % (area_code, ons_code, unit_id, name) m = Area( name=name, # If committing, this will be overwritten by the m.names.update_or_create type=Type.objects.get(code=area_code), country=Country.objects.get(code=country), generation_low=new_generation, generation_high=new_generation, ) if m.generation_high and current_generation and m.generation_high.id < current_generation.id: raise Exception, "Area %s found, but not in current generation %s" % (m, current_generation) m.generation_high = new_generation if options["commit"]: m.save() poly = [feat.geom] if options["commit"]: m.names.update_or_create({"type": name_type}, {"name": name}) if ons_code: self.ons_code_to_shape[ons_code] = (m, poly) if options["commit"]: m.codes.update_or_create({"type": code_version}, {"code": ons_code}) if unit_id: self.unit_id_to_shape[unit_id] = (m, poly) if options["commit"]: m.codes.update_or_create({"type": code_type_os}, {"code": unit_id}) if options["commit"]: save_polygons(self.unit_id_to_shape) save_polygons(self.ons_code_to_shape)