def update_or_create():
try:
m = Area.objects.get(id=code)
except Area.DoesNotExist:
m = Area(
id=code,
name=name,
type=Type.objects.get(code=area_code),
country=Country.objects.get(code='O'),
parent_area=parent_area,
generation_low=new_generation,
generation_high=new_generation,
)
if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
raise Exception("Area %s found, but not in current generation %s" % (m, current_generation))
m.generation_high = new_generation
g = feat.geom.transform(4326, clone=True)
poly = [g]
if options['commit']:
m.save()
for k, v in kml_data.data[name].items():
if k in ('name:smi', 'name:fi'):
lang = 'N' + k[5:]
m.names.update_or_create(type=NameType.objects.get(code=lang), defaults={'name': v})
m.codes.update_or_create(type=code_type_n5000, defaults={'code': code_str})
m.codes.update_or_create(type=code_type_osm, defaults={'code': int(kml_data.data[name]['osm'])})
save_polygons({code: (m, poly)})
def update_or_create():
try:
m = Area.objects.get(id=code)
except Area.DoesNotExist:
m = Area(
id=code,
name=name,
type=Type.objects.get(code=area_code),
country=Country.objects.get(code='O'),
parent_area=parent_area,
generation_low=new_generation,
generation_high=new_generation,
)
if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
raise Exception("Area %s found, but not in current generation %s" % (m, current_generation))
m.generation_high = new_generation
g = feat.geom.transform(4326, clone=True)
poly = [g]
if options['commit']:
m.save()
for k, v in kml_data.data[name].items():
if k in ('name:smi', 'name:fi'):
lang = 'N' + k[5:]
m.names.update_or_create(type=NameType.objects.get(code=lang), defaults={'name': v})
m.codes.update_or_create(type=code_type_n5000, defaults={'code': code_str})
m.codes.update_or_create(type=code_type_osm, defaults={'code': int(kml_data.data[name]['osm'])})
save_polygons({code: (m, poly)})
def update_or_create():
try:
m = Area.objects.get(id=int(regionid))
print "Updating area %s with id %d" % (regionname,
int(regionid))
except Area.DoesNotExist:
print "Creating new area %s with id %d" % (
regionname, int(regionid))
m = Area(
id=int(regionid),
name=regionname,
type=Type.objects.get(code=area_type),
country=Country.objects.get(code='O'),
generation_low=new_generation,
generation_high=new_generation,
)
if m.generation_high and current_generation \
and m.generation_high.id < current_generation.id:
raise Exception, "Area %s found, but not in current generation %s" % (
m, current_generation)
m.generation_high = new_generation
poly = [GEOSGeometry(unionoutline).ogr]
if options['commit']:
m.save()
save_polygons({regionid: (m, poly)})
def handle_label(self, filename, **options):
code_version = CodeType.objects.get(code='gss')
name_type = NameType.objects.get(code='O')
for feat in DataSource(filename)[0]:
name = unicode(feat['NAME'].value, 'iso-8859-1')
name = re.sub('\s*\(DET( NO \d+|)\)\s*(?i)', '', name)
name = re.sub('\s+', ' ', name)
ons_code = feat['CODE'].value
area_code = feat['AREA_CODE'].value
country = ons_code[0]
if ons_code in ('E07000100', 'E07000104', 'S12000009',
'S12000043'):
assert Area.objects.filter(codes__type=code_version,
codes__code=ons_code).count() == 0
print(ons_code, area_code, country, name)
m = Area(
type=Type.objects.get(code=area_code),
country=Country.objects.get(code=country),
generation_low=Generation.objects.get(id=1),
generation_high=Generation.objects.get(id=14),
)
if options['commit']:
m.save()
m.names.update_or_create(type=name_type,
defaults={'name': name})
m.codes.update_or_create(type=code_version,
defaults={'code': ons_code})
save_polygons({ons_code: (m, [feat.geom])})
def handle_label(self, filename, **options):
code_version = CodeType.objects.get(code='gss')
name_type = NameType.objects.get(code='O')
for feat in DataSource(filename)[0]:
name = feat['NAME'].value
if not isinstance(name, six.text_type):
name = name.decode('iso-8859-1')
name = re.sub(r'\s*\(DET( NO \d+|)\)\s*(?i)', '', name)
name = re.sub(r'\s+', ' ', name)
ons_code = feat['CODE'].value
area_code = feat['AREA_CODE'].value
country = ons_code[0]
if ons_code in ('E07000100', 'E07000104', 'S12000009', 'S12000043'):
assert Area.objects.filter(codes__type=code_version, codes__code=ons_code).count() == 0
print(ons_code, area_code, country, name)
m = Area(
type=Type.objects.get(code=area_code),
country=Country.objects.get(code=country),
generation_low=Generation.objects.get(id=1),
generation_high=Generation.objects.get(id=14),
)
if options['commit']:
m.save()
m.names.update_or_create(type=name_type, defaults={'name': name})
m.codes.update_or_create(type=code_version, defaults={'code': ons_code})
save_polygons({ons_code: (m, [feat.geom])})
def handle_label(self, filename, **options):
print filename
generation = Generation.objects.current()
short_filename = filename.split("/")[-1]
filename_prefix = short_filename[:4]
filename_suffix = short_filename.split(".")[0][-3:]
# check shapefile type - we handle both LSOA and MSOA
if filename_prefix == "LSOA":
feat_name = 'LSOA04NM'
feat_code = 'LSOA04CD'
if filename_suffix == 'BGC':
area_type = 'OLG'
else:
area_type = 'OLF'
elif filename_prefix == "MSOA":
feat_name = 'MSOA04NM'
feat_code = 'MSOA04CD'
if filename_suffix == 'BGC':
area_type = 'OMG'
else:
area_type = 'OMF'
else:
raise Exception, "Sorry, this script only handles LSOA/MSOA shapefiles!"
ds = DataSource(filename)
layer = ds[0]
for feat in layer:
# retrieve name and code, and set country
name = feat[feat_name].value
lsoa_code = feat[feat_code].value
country = lsoa_code[0]
# skip if the SOA already exists in db (SOAs don't change)
if Area.objects.filter(type__code=area_type,
codes__code=lsoa_code).count():
continue
print "Adding %s (%s) %s" % (name, lsoa_code, feat.geom.geom_name)
m = Area(
type=Type.objects.get(code=area_type),
country=Country.objects.get(code=country),
generation_low=generation,
generation_high=generation,
)
m.save()
m.names.update_or_create({'type': NameType.objects.get(code='S')},
{'name': name})
m.codes.update_or_create(
{'type': CodeType.objects.get(code='ons')},
{'code': lsoa_code})
p = feat.geom
if p.geom_name == 'POLYGON':
shapes = [p]
else:
shapes = p
for g in shapes:
m.polygons.create(polygon=g.wkt)
def handle_label(self, filename, **options):
print filename
new_generation = Generation.objects.new()
if not new_generation:
raise Exception, "No new generation to be used for import!"
name_type = NameType.objects.get(code='O')
code_type = CodeType.objects.get(code='gss')
ds = DataSource(filename)
layer = ds[0]
for feat in layer:
name = unicode(feat['NAME'].value, 'iso-8859-1')
print " ", name
name = re.sub('\s*\(DET( NO \d+|)\)\s*(?i)', '', name)
name = re.sub('\s+', ' ', name)
if "P Const" in name: area_code = 'SPC'
elif "PER" in name: area_code = 'SPE'
else: raise Exception, "Unknown type of area %s" % name
ons_code = name_to_code[name]
if ons_code in self.ons_code_to_shape:
m, poly = self.ons_code_to_shape[ons_code]
if options['commit']:
m_name = m.names.get(type=name_type).name
if name != m_name:
raise Exception, "ONS code %s is used for %s and %s" % (ons_code, name, m_name)
# Otherwise, combine the two shapes for one area
print " Adding subsequent shape to ONS code %s" % ons_code
poly.append(feat.geom)
continue
try:
m = Area.objects.get(codes__type=code_type, codes__code=ons_code)
except Area.DoesNotExist:
m = Area(
type = Type.objects.get(code=area_code),
country = Country.objects.get(name='Scotland'),
generation_low = new_generation,
generation_high = new_generation,
)
if options['commit']:
m.save()
poly = [ feat.geom ]
if options['commit']:
m.names.update_or_create({ 'type': name_type }, { 'name': name })
if ons_code:
self.ons_code_to_shape[ons_code] = (m, poly)
if options['commit']:
m.codes.update_or_create({ 'type': code_type }, { 'code': ons_code })
if options['commit']:
save_polygons(self.ons_code_to_shape)
def update_or_create():
if osm_code:
m = osm_code.area
else:
m = Area(
name = name,
type = Type.objects.get(code=area_code),
country = Country.objects.get(code='G'),
parent_area = parent_area,
generation_low = new_generation,
generation_high = new_generation,
)
if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
raise Exception, "Area %s found, but not in current generation %s" % (m, current_generation)
m.generation_high = new_generation
g = feat.geom.transform(4326, clone=True)
# In generating the data we should have
# excluded any "polygons" with less than four
# points (the final one being the same as the
# first), but just in case:
for polygon in g:
if g.num_points < 4:
return
poly = [ g ]
if options['commit']:
m.save()
if name not in kml_data.data:
print json.dumps(kml_data.data, sort_keys=True, indent=4)
raise Exception, u"Will fail to find '%s' in the dictionary" % (name,)
for k, v in kml_data.data[name].items():
language_name = None
if k == 'name':
lang = 'default'
language_name = "OSM Default"
else:
name_match = re.search(r'^name:(.+)$', k)
if name_match:
lang = name_match.group(1)
if lang in language_code_to_name:
language_name = language_code_to_name[lang]
if not language_name:
continue
# Otherwise, make sure that a NameType for this language exists:
NameType.objects.update_or_create({'code': lang},
{'code': lang,
'description': language_name})
name_type = NameType.objects.get(code=lang)
m.names.update_or_create({ 'type': name_type }, { 'name': v })
m.codes.update_or_create({ 'type': code_type_osm }, { 'code': osm_id })
save_polygons({ code : (m, poly) })
def handle_label(self, filename, **options):
print(filename)
generation = Generation.objects.current()
short_filename = filename.split("/")[-1]
filename_prefix = short_filename[:4]
filename_suffix = short_filename.split(".")[0][-3:]
# check shapefile type - we handle both LSOA and MSOA
if filename_prefix == "LSOA":
feat_name = 'LSOA04NM'
feat_code = 'LSOA04CD'
if filename_suffix == 'BGC':
area_type = 'OLG'
else:
area_type = 'OLF'
elif filename_prefix == "MSOA":
feat_name = 'MSOA04NM'
feat_code = 'MSOA04CD'
if filename_suffix == 'BGC':
area_type = 'OMG'
else:
area_type = 'OMF'
else:
raise Exception("Sorry, this script only handles LSOA/MSOA shapefiles!")
ds = DataSource(filename)
layer = ds[0]
for feat in layer:
# retrieve name and code, and set country
name = feat[feat_name].value
lsoa_code = feat[feat_code].value
country = lsoa_code[0]
# skip if the SOA already exists in db (SOAs don't change)
if Area.objects.filter(type__code=area_type, codes__code=lsoa_code).count():
continue
print("Adding %s (%s) %s" % (name, lsoa_code, feat.geom.geom_name))
m = Area(
type=Type.objects.get(code=area_type),
country=Country.objects.get(code=country),
generation_low=generation,
generation_high=generation,
)
m.save()
m.names.update_or_create(type=NameType.objects.get(code='S'), defaults={'name': name})
m.codes.update_or_create(type=CodeType.objects.get(code='ons'), defaults={'code': lsoa_code})
p = feat.geom
if p.geom_name == 'POLYGON':
shapes = [p]
else:
shapes = p
for g in shapes:
m.polygons.create(polygon=g.wkt)
def _create(self, name, typ, geom, gss=None):
m = Area(
name=name, type=Type.objects.get(code=typ), country=self.country,
generation_low=self.gn, generation_high=self.gn,
)
if self.commit:
m.save()
m.names.update_or_create(type=self.name_type, defaults={'name': name})
if gss:
m.codes.update_or_create(type=self.code_type, defaults={'code': gss})
m.polygons.create(polygon=geom)
else:
print('Would create', name, typ)
def handle_label(self, filename, **options):
code_version = CodeType.objects.get(code='gss')
name_type = NameType.objects.get(code='O')
# First, the easy ones
for old, new in FIX_DIRECTLY.items():
assert Area.objects.filter(codes__type=code_version, codes__code=new).count() == 0
code = Code.objects.get(type=code_version, code=old)
code.code = new
print('Updating area %s to %s' % (old, new))
if options['commit']:
code.save()
# Now, the harder ones
previous_generation_low = {}
previous_name = {}
for old, new in FIX_WITH_OLD_BOUNDARY.items():
if new != 'E05012199' or options['commit']: # That one will exist unless it was updated above
assert Area.objects.filter(codes__type=code_version, codes__code=new).count() == 0
code = Code.objects.get(type=code_version, code=old)
code.code = new
previous_generation_low[old] = code.area.generation_low
previous_name[old] = code.area.name
code.area.generation_low = Generation.objects.current()
print('Updating area %s to %s' % (old, new))
if options['commit']:
code.area.save()
code.save()
# Now we need to reinsert the October 2018 boundaries
generation_high = Generation.objects.get(id=Generation.objects.current().id - 1)
for feat in boundary_line_shapes(filename):
ons_code = feat['CODE'].value
if ons_code in FIX_WITH_OLD_BOUNDARY.keys():
area_code = feat['AREA_CODE'].value
country = Country.objects.get(code=ons_code[0])
if options['commit']: # They will be present if non-commit, they weren't changed above
assert Area.objects.filter(codes__type=code_version, codes__code=ons_code).count() == 0
print('Creating new %s (%s), %s' % (ons_code, area_code, feat.name))
m = Area(
type=Type.objects.get(code=area_code),
country=country,
generation_low=previous_generation_low[ons_code],
generation_high=generation_high,
)
if options['commit']:
m.save()
m.names.update_or_create(type=name_type, defaults={'name': feat.name})
m.codes.update_or_create(type=code_version, defaults={'code': ons_code})
save_polygons({ons_code: (m, [feat.geom])})
def handle_label(self, filename, **options):
current_generation = Generation.objects.current()
new_generation = Generation.objects.new()
if not new_generation:
raise Exception, "No new generation to be used for import!"
code_type = CodeType.objects.get(code='n5000')
name_type = NameType.objects.get(code='M')
ds = DataSource(filename)
layer = ds[0]
for feat in layer:
name = unicode(feat['NAVN'].value, 'iso-8859-1')
name = re.sub('\s+', ' ', name)
print " ", name
code = feat['KOMM'].value
code_str = '%04d' % code
area_code = 'NKO'
try:
m = Area.objects.get(codes__type=code_type,
codes__code=code_str)
except Area.DoesNotExist:
m = Area(
id=code,
type=Type.objects.get(code=area_code),
country=Country.objects.get(code='O'),
generation_low=new_generation,
generation_high=new_generation,
)
if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
raise Exception, "Area %s found, but not in current generation %s" % (
m, current_generation)
m.generation_high = new_generation
g = feat.geom.transform(4326, clone=True)
poly = [g]
if options['commit']:
m.save()
m.names.update_or_create({'type': name_type}, {'name': name})
m.codes.update_or_create({'type': code_type},
{'code': code_str})
save_polygons({code: (m, poly)})
def _create(self, name, typ, geom, gss=None):
m = Area(
name=name,
type=Type.objects.get(code=typ),
country=self.country,
generation_low=self.gn,
generation_high=self.gn,
)
if self.commit:
m.save()
m.names.update_or_create(type=self.name_type,
defaults={'name': name})
if gss:
m.codes.update_or_create(type=self.code_type,
defaults={'code': gss})
m.polygons.create(polygon=geom)
else:
print('Would create', name, typ)
def make_new_area(self, name, ons_code, area_code, code_version, generation_low, generation_high, country):
assert Area.objects.filter(codes__type=code_version, codes__code=ons_code).count() == 0
print(ons_code, area_code, country, name)
return Area(
type=Type.objects.get(code=area_code),
country=Country.objects.get(code=country),
generation_low=Generation.objects.get(id=generation_low),
generation_high=Generation.objects.get(id=generation_high),
)
def handle_label(self, filename, **options):
current_generation = Generation.objects.current()
new_generation = Generation.objects.new()
if not new_generation:
raise Exception, "No new generation to be used for import!"
code_type = CodeType.objects.get(code='n5000')
name_type = NameType.objects.get(code='M')
ds = DataSource(filename)
layer = ds[0]
for feat in layer:
name = unicode(feat['NAVN'].value, 'iso-8859-1')
name = re.sub('\s+', ' ', name)
print " ", name
code = feat['KOMM'].value
code_str = '%04d' % code
area_code = 'NKO'
try:
m = Area.objects.get(codes__type=code_type, codes__code=code_str)
except Area.DoesNotExist:
m = Area(
id = code,
type = Type.objects.get(code=area_code),
country = Country.objects.get(code='O'),
generation_low = new_generation,
generation_high = new_generation,
)
if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
raise Exception, "Area %s found, but not in current generation %s" % (m, current_generation)
m.generation_high = new_generation
g = feat.geom.transform(4326, clone=True)
poly = [ g ]
if options['commit']:
m.save()
m.names.update_or_create({ 'type': name_type }, { 'name': name })
m.codes.update_or_create({ 'type': code_type }, { 'code': code_str })
save_polygons({ code : (m, poly) })
def update_or_create():
try:
m = Area.objects.get(id=int(regionid))
print "Updating area %s with id %d" % (regionname, int(regionid))
except Area.DoesNotExist:
print "Creating new area %s with id %d" % (regionname, int(regionid))
m = Area(
id = int(regionid),
name = regionname,
type = Type.objects.get(code=area_type),
country = Country.objects.get(code='O'),
generation_low = new_generation,
generation_high = new_generation,
)
if m.generation_high and current_generation \
and m.generation_high.id < current_generation.id:
raise Exception, "Area %s found, but not in current generation %s" % (m, current_generation)
m.generation_high = new_generation
poly = [ GEOSGeometry(unionoutline).ogr ]
if options['commit']:
m.save()
save_polygons({ regionid : (m, poly) })
def handle_label(self, filename, **options):
print(filename)
new_generation = Generation.objects.new()
if not new_generation:
raise Exception("No new generation to be used for import!")
name_type = NameType.objects.get(code='O')
code_type = CodeType.objects.get(code='gss')
ds = DataSource(filename)
layer = ds[0]
for feat in layer:
name = feat['NAME'].value
if not isinstance(name, six.text_type):
name = name.decode('iso-8859-1')
print(" %s" % name)
name = re.sub(r'\s*\(DET( NO \d+|)\)\s*(?i)', '', name)
name = re.sub(r'\s+', ' ', name)
if "P Const" in name:
area_code = 'SPC'
elif "PER" in name:
area_code = 'SPE'
else:
raise Exception("Unknown type of area %s" % name)
ons_code = name_to_code[name]
if ons_code in self.ons_code_to_shape:
m, poly = self.ons_code_to_shape[ons_code]
if options['commit']:
m_name = m.names.get(type=name_type).name
if name != m_name:
raise Exception("ONS code %s is used for %s and %s" %
(ons_code, name, m_name))
# Otherwise, combine the two shapes for one area
print(" Adding subsequent shape to ONS code %s" % ons_code)
poly.append(feat.geom)
continue
try:
m = Area.objects.get(codes__type=code_type,
codes__code=ons_code)
except Area.DoesNotExist:
m = Area(
type=Type.objects.get(code=area_code),
country=Country.objects.get(name='Scotland'),
generation_low=new_generation,
generation_high=new_generation,
)
if options['commit']:
m.save()
poly = [feat.geom]
if options['commit']:
m.names.update_or_create(type=name_type,
defaults={'name': name})
if ons_code:
self.ons_code_to_shape[ons_code] = (m, poly)
if options['commit']:
m.codes.update_or_create(type=code_type,
defaults={'code': ons_code})
if options['commit']:
save_polygons(self.ons_code_to_shape)
def handle_label(self, directory_name, **options):
current_generation = Generation.objects.current()
new_generation = Generation.objects.new()
if not new_generation:
raise Exception("No new generation to be used for import!")
if not os.path.isdir(directory_name):
raise Exception("'%s' is not a directory" % (directory_name,))
os.chdir(directory_name)
mapit_type_glob = smart_text("[A-Z0-9][A-Z0-9][A-Z0-9]")
if not glob(mapit_type_glob):
raise Exception(
"'%s' did not contain any directories that look like MapIt types (e.g. O11, OWA, etc.)" % (
directory_name,))
def verbose(s):
if int(options['verbosity']) > 1:
print(smart_str(s))
verbose("Loading any admin boundaries from " + directory_name)
verbose("Finding language codes...")
language_code_to_name = {}
code_keys = ('two_letter', 'three_letter')
for row in get_iso639_2_table():
english_name = getattr(row, 'english_name')
for k in code_keys:
code = getattr(row, k)
if not code:
continue
language_code_to_name[code] = english_name
global_country = Country.objects.get(code='G')
# print json.dumps(language_code_to_name, sort_keys=True, indent=4)
skip_up_to = None
# skip_up_to = 'relation-80370'
skipping = bool(skip_up_to)
for type_directory in sorted(glob(mapit_type_glob)):
verbose("Loading type " + type_directory)
if not os.path.exists(type_directory):
verbose("Skipping the non-existent " + type_directory)
continue
verbose("Loading all KML in " + type_directory)
files = sorted(os.listdir(type_directory))
total_files = len(files)
for i, e in enumerate(files):
progress = "[%d%% complete] " % ((i * 100) / total_files,)
if skipping:
if skip_up_to in e:
skipping = False
else:
continue
if not e.endswith('.kml'):
verbose("Ignoring non-KML file: " + e)
continue
m = re.search(r'^(way|relation)-(\d+)-', e)
if not m:
raise Exception("Couldn't extract OSM element type and ID from: " + e)
osm_type, osm_id = m.groups()
kml_filename = os.path.join(type_directory, e)
verbose(progress + "Loading " + os.path.realpath(kml_filename))
# Need to parse the KML manually to get the ExtendedData
kml_data = KML()
xml.sax.parse(smart_str(kml_filename), kml_data)
useful_names = [n for n in kml_data.data.keys() if not n.startswith('Boundaries for')]
if len(useful_names) == 0:
raise Exception("No useful names found in KML data")
elif len(useful_names) > 1:
raise Exception("Multiple useful names found in KML data")
name = useful_names[0]
print(smart_str(" %s" % name))
if osm_type == 'relation':
code_type_osm = CodeType.objects.get(code='osm_rel')
elif osm_type == 'way':
code_type_osm = CodeType.objects.get(code='osm_way')
else:
raise Exception("Unknown OSM element type: " + osm_type)
ds = DataSource(kml_filename)
layer = ds[0]
if len(layer) != 1:
raise Exception("We only expect one feature in each layer")
feat = layer[1]
g = feat.geom.transform(4326, clone=True)
if g.geom_count == 0:
# Just ignore any KML files that have no polygons in them:
verbose(' Ignoring that file - it contained no polygons')
continue
# Nowadays, in generating the data we should have
# excluded any "polygons" with less than four points
# (the final one being the same as the first), but
# just in case:
polygons_too_small = 0
for polygon in g:
if polygon.num_points < 4:
polygons_too_small += 1
if polygons_too_small:
message = "%d out of %d polygon(s) were too small" % (polygons_too_small, g.geom_count)
verbose(' Skipping, since ' + message)
continue
g_geos = g.geos
if not g_geos.valid:
verbose(" Invalid KML:" + kml_filename)
fixed_multipolygon = fix_invalid_geos_multipolygon(g_geos)
if len(fixed_multipolygon) == 0:
verbose(" Invalid polygons couldn't be fixed")
continue
g = fixed_multipolygon.ogr
area_type = Type.objects.get(code=type_directory)
try:
osm_code = Code.objects.get(type=code_type_osm,
code=osm_id,
area__generation_high__lte=current_generation,
area__generation_high__gte=current_generation)
except Code.DoesNotExist:
verbose(' No area existed in the current generation with that OSM element type and ID')
osm_code = None
was_the_same_in_current = False
if osm_code:
m = osm_code.area
# First, we need to check if the polygons are
# still the same as in the previous generation:
previous_geos_geometry = m.polygons.aggregate(Collect('polygon'))['polygon__collect']
if previous_geos_geometry is None:
verbose(' In the current generation, that area was empty - skipping')
else:
# Simplify it to make sure the polygons are valid:
previous_geos_geometry = shapely.wkb.loads(
str(previous_geos_geometry.simplify(tolerance=0).ewkb))
new_geos_geometry = shapely.wkb.loads(str(g.geos.simplify(tolerance=0).ewkb))
if previous_geos_geometry.almost_equals(new_geos_geometry, decimal=7):
was_the_same_in_current = True
else:
verbose(' In the current generation, the boundary was different')
if was_the_same_in_current:
# Extend the high generation to the new one:
verbose(' The boundary was identical in the previous generation; raising generation_high')
m.generation_high = new_generation
else:
# Otherwise, create a completely new area:
m = Area(
name=name,
type=area_type,
country=global_country,
parent_area=None,
generation_low=new_generation,
generation_high=new_generation,
)
poly = [g]
if options['commit']:
m.save()
verbose(' Area ID: ' + str(m.id))
if name not in kml_data.data:
print(json.dumps(kml_data.data, sort_keys=True, indent=4))
raise Exception("Will fail to find '%s' in the dictionary" % (name,))
old_lang_codes = set(n.type.code for n in m.names.all())
for k, translated_name in kml_data.data[name].items():
language_name = None
if k == 'name':
lang = 'default'
language_name = "OSM Default"
else:
name_match = re.search(r'^name:(.+)$', k)
if name_match:
lang = name_match.group(1)
if lang in language_code_to_name:
language_name = language_code_to_name[lang]
if not language_name:
continue
old_lang_codes.discard(lang)
# Otherwise, make sure that a NameType for this language exists:
NameType.objects.update_or_create(code=lang, defaults={'description': language_name})
name_type = NameType.objects.get(code=lang)
m.names.update_or_create(type=name_type, defaults={'name': translated_name})
if old_lang_codes:
verbose('Removing deleted languages codes: ' + ' '.join(old_lang_codes))
m.names.filter(type__code__in=old_lang_codes).delete()
# If the boundary was the same, the old Code
# object will still be pointing to the same Area,
# which just had its generation_high incremented.
# In every other case, there's a new area object,
# so create a new Code and save it:
if not was_the_same_in_current:
new_code = Code(area=m, type=code_type_osm, code=osm_id)
new_code.save()
save_polygons({'dummy': (m, poly)})
def handle_label(self, filename, **options):
for k in ['generation_id','area_type_code','name_type_code','country_code']:
if options[k]: continue
raise Exception("Missing argument '--%s'" % k)
generation_id = options['generation_id']
area_type_code = options['area_type_code']
name_type_code = options['name_type_code']
country_code = options['country_code']
area_type = Type.objects.get(code=area_type_code)
name_type = NameType.objects.get(code=name_type_code)
country = Country.objects.get(code=country_code)
print "Importing from %s" % filename
if not options['commit']:
print '(will not save to db as --commit not specified)'
current_generation = Generation.objects.current()
new_generation = Generation.objects.get( id=generation_id )
# Need to parse the KML manually to get the ExtendedData
kml_data = KML()
xml.sax.parse(filename, kml_data)
ds = DataSource(filename)
layer = ds[0]
for feat in layer:
name = feat['Name'].value.decode('utf-8')
name = re.sub('\s+', ' ', name)
if not name:
raise Exception( "Could not find a name to use for area" )
print " looking at '%s'" % name.encode('utf-8')
try:
m = Area.objects.get(name=name, type=area_type)
except Area.DoesNotExist:
m = Area(
name = name,
type = area_type,
country = country,
# parent_area = parent_area,
generation_low = new_generation,
generation_high = new_generation,
)
# check that we are not about to skip a generation
if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
raise Exception, "Area %s found, but not in current generation %s" % (m, current_generation)
m.generation_high = new_generation
g = feat.geom.transform(4326, clone=True)
poly = [ g ]
if options['commit']:
m.save()
m.names.update_or_create({ 'type': name_type }, { 'name': name })
save_polygons({ m.id : (m, poly) })
def handle_label(self, filename, **options):
if not options['control']:
raise Exception("You must specify a control file")
__import__(options['control'])
control = sys.modules[options['control']]
code_version = CodeType.objects.get(code='gss')
name_type = NameType.objects.get(code='O')
code_type_os = CodeType.objects.get(code='unit_id')
print(filename)
current_generation = Generation.objects.current()
new_generation = Generation.objects.new()
if not new_generation:
raise Exception("No new generation to be used for import!")
ds = DataSource(filename)
layer = ds[0]
for feat in layer:
name = feat['NAME'].value
if not isinstance(name, six.text_type):
name = name.decode('iso-8859-1')
name = re.sub(r'\s*\(DET( NO \d+|)\)\s*(?i)', '', name)
name = re.sub(r'\s+', ' ', name)
ons_code = feat['CODE'].value if feat['CODE'].value not in ('999999', '999999999') else None
unit_id = str(feat['UNIT_ID'].value)
area_code = feat['AREA_CODE'].value
patch = self.patch_boundary_line(name, ons_code, unit_id, area_code)
if 'ons-code' in patch:
ons_code = patch['ons-code']
elif 'unit-id' in patch:
unit_id = patch['unit-id']
if area_code == 'NCP':
continue # Ignore Non Parished Areas
if ons_code in self.ons_code_to_shape:
m, poly = self.ons_code_to_shape[ons_code]
try:
m_name = m.names.get(type=name_type).name
except Name.DoesNotExist:
m_name = m.name # If running without commit for dry run, so nothing being stored in db
if name != m_name:
raise Exception("ONS code %s is used for %s and %s" % (ons_code, name, m_name))
# Otherwise, combine the two shapes for one area
poly.append(feat.geom)
continue
if unit_id in self.unit_id_to_shape:
m, poly = self.unit_id_to_shape[unit_id]
try:
m_name = m.names.get(type=name_type).name
except Name.DoesNotExist:
m_name = m.name # If running without commit for dry run, so nothing being stored in db
if name != m_name:
raise Exception("Unit ID code %s is used for %s and %s" % (unit_id, name, m_name))
# Otherwise, combine the two shapes for one area
poly.append(feat.geom)
continue
if ons_code:
country = ons_code[0] # Hooray!
elif area_code in ('CED', 'CTY', 'DIW', 'DIS', 'MTW', 'MTD', 'LBW', 'LBO', 'LAC', 'GLA'):
country = 'E'
else:
raise Exception(area_code)
try:
check = control.check(name, area_code, country, feat.geom, ons_code=ons_code, commit=options['commit'])
if check is True:
raise Area.DoesNotExist
if isinstance(check, Area):
m = check
try:
ons_code = m.codes.get(type=code_version).code
except Code.DoesNotExist:
ons_code = None
elif ons_code:
m = Area.objects.get(codes__type=code_version, codes__code=ons_code)
elif unit_id:
m = Area.objects.get(
codes__type=code_type_os, codes__code=unit_id, generation_high=current_generation)
m_name = m.names.get(type=name_type).name
if name != m_name:
raise Exception("Unit ID code %s is %s in DB but %s in SHP file" % (unit_id, m_name, name))
else:
raise Exception('Area "%s" (%s) has neither ONS code nor unit ID' % (name, area_code))
if int(options['verbosity']) > 1:
print(" Area matched, %s" % (m, ))
except Area.DoesNotExist:
print(" New area: %s %s %s %s" % (area_code, ons_code, unit_id, name))
m = Area(
name=name, # If committing, this will be overwritten by the m.names.update_or_create
type=Type.objects.get(code=area_code),
country=Country.objects.get(code=country),
generation_low=new_generation,
generation_high=new_generation,
)
if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
raise Exception("Area %s found, but not in current generation %s" % (m, current_generation))
m.generation_high = new_generation
if options['commit']:
m.save()
# Make a GEOS geometry only to check for validity:
g = feat.geom
geos_g = g.geos
if not geos_g.valid:
print(" Geometry of %s %s not valid" % (ons_code, m))
geos_g = fix_invalid_geos_geometry(geos_g)
if geos_g is None:
raise Exception("The geometry for area %s was invalid and couldn't be fixed" % name)
g = None
else:
g = geos_g.ogr
poly = [g]
if options['commit']:
m.names.update_or_create(type=name_type, defaults={'name': name})
if ons_code:
self.ons_code_to_shape[ons_code] = (m, poly)
if options['commit']:
m.codes.update_or_create(type=code_version, defaults={'code': ons_code})
if unit_id:
self.unit_id_to_shape[unit_id] = (m, poly)
if options['commit']:
m.codes.update_or_create(type=code_type_os, defaults={'code': unit_id})
if options['commit']:
save_polygons(self.unit_id_to_shape)
save_polygons(self.ons_code_to_shape)
def handle_label(self, filename, **options):
err = False
for k in [
'generation_id', 'area_type_code', 'name_type_code',
'country_code'
]:
if options[k]: continue
print "Missing argument '--%s'" % k
err = True
if err:
sys.exit(1)
generation_id = options['generation_id']
area_type_code = options['area_type_code']
name_type_code = options['name_type_code']
country_code = options['country_code']
name_field = options['name_field'] or 'Name'
code_field = options['code_field']
code_type_code = options['code_type']
encoding = options['encoding'] or 'utf-8'
if len(area_type_code) > 3:
print "Area type code must be 3 letters or fewer, sorry"
sys.exit(1)
if (code_field and not code_type_code) or (not code_field
and code_type_code):
print "You must specify both code_field and code_type, or neither."
sys.exit(1)
try:
area_type = Type.objects.get(code=area_type_code)
except:
type_desc = raw_input(
'Please give a description for area type code %s: ' %
area_type_code)
area_type = Type(code=area_type_code, description=type_desc)
if options['commit']: area_type.save()
try:
name_type = NameType.objects.get(code=name_type_code)
except:
name_desc = raw_input(
'Please give a description for name type code %s: ' %
name_type_code)
name_type = NameType(code=name_type_code, description=name_desc)
if options['commit']: name_type.save()
try:
country = Country.objects.get(code=country_code)
except:
country_name = raw_input(
'Please give the name for country code %s: ' % country_code)
country = Country(code=country_code, name=country_name)
if options['commit']: country.save()
if code_type_code:
try:
code_type = CodeType.objects.get(code=code_type_code)
except:
code_desc = raw_input(
'Please give a description for code type %s: ' %
code_type_code)
code_type = CodeType(code=code_type_code,
description=code_desc)
if options['commit']: code_type.save()
print "Importing from %s" % filename
if not options['commit']:
print '(will not save to db as --commit not specified)'
current_generation = Generation.objects.current()
new_generation = Generation.objects.get(id=generation_id)
ds = DataSource(filename)
layer = ds[0]
for feat in layer:
try:
name = feat[name_field].value
except:
choices = ', '.join(layer.fields)
print "Could not find name using name field '%s' - should it be something else? It will be one of these: %s. Specify which with --name_field" % (
name_field, choices)
sys.exit(1)
try:
name = name.decode(encoding)
except:
print "Could not decode name using encoding '%s' - is it in another encoding? Specify one with --encoding" % encoding
sys.exit(1)
name = re.sub('\s+', ' ', name)
if not name:
raise Exception("Could not find a name to use for area")
code = None
if code_field:
try:
code = feat[code_field].value
except:
choices = ', '.join(layer.fields)
print "Could not find code using code field '%s' - should it be something else? It will be one of these: %s. Specify which with --code_field" % (
code_field, choices)
sys.exit(1)
print " looking at '%s'%s" % (name.encode('utf-8'),
(' (%s)' % code) if code else '')
try:
if code:
m = Area.objects.get(codes__code=code,
codes__type=code_type)
else:
# Assumes unique names if no code column used
m = Area.objects.get(name=name, type=area_type)
except Area.DoesNotExist:
m = Area(
name=name,
type=area_type,
country=country,
# parent_area = parent_area,
generation_low=new_generation,
generation_high=new_generation,
)
if options['use_code_as_id'] and code:
m.id = int(code)
# check that we are not about to skip a generation
if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
raise Exception, "Area %s found, but not in current generation %s" % (
m, current_generation)
m.generation_high = new_generation
g = feat.geom.transform(4326, clone=True)
poly = [g]
if options['commit']:
m.save()
m.names.update_or_create({'type': name_type}, {'name': name})
if code:
m.codes.update_or_create({'type': code_type},
{'code': code})
save_polygons({m.id: (m, poly)})
def handle_label(self, filename, **options):
missing_options = []
for k in [
'generation_id', 'area_type_code', 'name_type_code',
'country_code'
]:
if options[k]:
continue
else:
missing_options.append(k)
if missing_options:
message_start = "Missing arguments " if len(
missing_options) > 1 else "Missing argument "
message = message_start + " ".join('--{0}'.format(k)
for k in missing_options)
raise CommandError(message)
generation_id = options['generation_id']
area_type_code = options['area_type_code']
name_type_code = options['name_type_code']
country_code = options['country_code']
override_name = options['override_name']
name_field = options['name_field']
if not (override_name or name_field):
name_field = 'Name'
override_code = options['override_code']
code_field = options['code_field']
code_type_code = options['code_type']
encoding = options['encoding'] or 'utf-8'
if len(area_type_code) > 3:
raise CommandError(
"Area type code must be 3 letters or fewer, sorry")
if name_field and override_name:
raise CommandError(
"You must not specify both --name_field and --override_name")
if code_field and override_code:
raise CommandError(
"You must not specify both --code_field and --override_code")
using_code = (code_field or override_code)
if (using_code and not code_type_code) or (not using_code
and code_type_code):
raise CommandError(
"If you want to save a code, specify --code_type and either --code_field or --override_code"
)
try:
area_type = Type.objects.get(code=area_type_code)
except:
type_desc = raw_input(
'Please give a description for area type code %s: ' %
area_type_code)
area_type = Type(code=area_type_code, description=type_desc)
if options['commit']: area_type.save()
try:
name_type = NameType.objects.get(code=name_type_code)
except:
name_desc = raw_input(
'Please give a description for name type code %s: ' %
name_type_code)
name_type = NameType(code=name_type_code, description=name_desc)
if options['commit']: name_type.save()
try:
country = Country.objects.get(code=country_code)
except:
country_name = raw_input(
'Please give the name for country code %s: ' % country_code)
country = Country(code=country_code, name=country_name)
if options['commit']: country.save()
if code_type_code:
try:
code_type = CodeType.objects.get(code=code_type_code)
except:
code_desc = raw_input(
'Please give a description for code type %s: ' %
code_type_code)
code_type = CodeType(code=code_type_code,
description=code_desc)
if options['commit']: code_type.save()
self.stdout.write("Importing from %s" % filename)
if not options['commit']:
self.stdout.write(
'(will not save to db as --commit not specified)')
current_generation = Generation.objects.current()
new_generation = Generation.objects.get(id=generation_id)
def verbose(*args):
if int(options['verbosity']) > 1:
self.stdout.write(" ".join(str(a) for a in args))
ds = DataSource(filename)
layer = ds[0]
if (override_name or override_code) and len(layer) > 1:
message = "Warning: you have specified an override %s and this file contains more than one feature; multiple areas with the same %s will be created"
if override_name:
self.stdout.write(message % ('name', 'name'))
if override_code:
self.stdout.write(message % ('code', 'code'))
for feat in layer:
if override_name:
name = override_name
else:
try:
name = feat[name_field].value
except:
choices = ', '.join(layer.fields)
raise CommandError(
"Could not find name using name field '%s' - should it be something else? It will be one of these: %s. Specify which with --name_field"
% (name_field, choices))
try:
if not isinstance(name, unicode):
name = name.decode(encoding)
except:
raise CommandError(
"Could not decode name using encoding '%s' - is it in another encoding? Specify one with --encoding"
% encoding)
name = re.sub('\s+', ' ', name)
if not name:
raise Exception("Could not find a name to use for area")
code = None
if override_code:
code = override_code
elif code_field:
try:
code = feat[code_field].value
except:
choices = ', '.join(layer.fields)
raise CommandError(
"Could not find code using code field '%s' - should it be something else? It will be one of these: %s. Specify which with --code_field"
% (code_field, choices))
self.stdout.write(" looking at '%s'%s" %
(name.encode('utf-8'),
(' (%s)' % code) if code else ''))
g = None
if hasattr(feat, 'geom'):
g = feat.geom.transform(settings.MAPIT_AREA_SRID, clone=True)
try:
if options['new']: # Always want a new area
raise Area.DoesNotExist
if code:
matching_message = "code %s of code type %s" % (code,
code_type)
areas = Area.objects.filter(
codes__code=code,
codes__type=code_type).order_by('-generation_high')
else:
matching_message = "name %s of area type %s" % (name,
area_type)
areas = Area.objects.filter(
name=name, type=area_type).order_by('-generation_high')
if len(areas) == 0:
verbose(" the area was not found - creating a new one")
raise Area.DoesNotExist
m = areas[0]
verbose(" found the area")
if options['preserve']:
# Find whether we need to create a new Area:
previous_geos_geometry = m.polygons.collect()
if m.generation_high < current_generation.id:
# Then it was missing in current_generation:
verbose(
" area existed previously, but was missing from",
current_generation)
raise Area.DoesNotExist
elif g is None:
if previous_geos_geometry is not None:
verbose(" area is now empty")
raise Area.DoesNotExist
else:
verbose(" the area has remained empty")
elif previous_geos_geometry is None:
# It was empty in the previous generation:
verbose(" area was empty in", current_generation)
raise Area.DoesNotExist
else:
# Otherwise, create a new Area unless the
# polygons were the same in current_generation:
previous_geos_geometry = previous_geos_geometry.simplify(
tolerance=0)
new_geos_geometry = g.geos.simplify(tolerance=0)
create_new_area = not previous_geos_geometry.equals(
new_geos_geometry)
p = previous_geos_geometry.sym_difference(
new_geos_geometry
).area / previous_geos_geometry.area
verbose(" change in area is:",
"%.03f%%" % (100 * p, ))
if create_new_area:
verbose(
" the area", m,
"has changed, creating a new area due to --preserve"
)
raise Area.DoesNotExist
else:
verbose(" the area remained the same")
else:
# If --preserve is not specified, the code or the name must be unique:
if len(areas) > 1:
raise Area.MultipleObjectsReturned, "There was more than one area with %s, and --preserve was not specified" % (
matching_message, )
except Area.DoesNotExist:
m = Area(
name=name,
type=area_type,
country=country,
# parent_area = parent_area,
generation_low=new_generation,
generation_high=new_generation,
)
if options['use_code_as_id'] and code:
m.id = int(code)
# check that we are not about to skip a generation
if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
raise Exception, "Area %s found, but not in current generation %s" % (
m, current_generation)
m.generation_high = new_generation
if options['fix_invalid_polygons'] and g is not None:
# Make a GEOS geometry only to check for validity:
geos_g = g.geos
if not geos_g.valid:
geos_g = fix_invalid_geos_geometry(geos_g)
if geos_g is None:
self.stdout.write(
"The geometry for area %s was invalid and couldn't be fixed"
% name)
g = None
else:
g = geos_g.ogr
poly = [g] if g is not None else []
if options['commit']:
m.save()
m.names.update_or_create({'type': name_type}, {'name': name})
if code:
m.codes.update_or_create({'type': code_type},
{'code': code})
save_polygons({m.id: (m, poly)})
def process_file(self, filename, area_code, srid, control, options):
code_version = CodeType.objects.get(code='gss')
name_type = NameType.objects.get(code='N')
code_type_osni = CodeType.objects.get(code='osni_oid')
if not hasattr(self, area_code):
raise Exception(
"Don't know how to extract features from %s files" % area_code)
area_code_info = getattr(self, area_code)(srid)
print(filename)
current_generation = Generation.objects.current()
new_generation = Generation.objects.new()
if not new_generation:
raise Exception("No new generation to be used for import!")
ds = DataSource(filename)
layer = ds[0]
for feat in layer:
name, ons_code, osni_object_id = area_code_info.extract_fields(
feat)
name = self.format_name(name)
if ons_code in self.ons_code_to_shape:
m, poly = self.ons_code_to_shape[ons_code]
try:
m_name = m.names.get(type=name_type).name
except Name.DoesNotExist:
m_name = m.name # If running without commit for dry run, so nothing being stored in db
if name != m_name:
raise Exception("ONS code %s is used for %s and %s" %
(ons_code, name, m_name))
# Otherwise, combine the two shapes for one area
poly.append(feat.geom)
continue
if osni_object_id in self.osni_object_id_to_shape:
m, poly = self.osni_object_id_to_shape[osni_object_id]
try:
m_name = m.names.get(type=name_type).name
except Name.DoesNotExist:
m_name = m.name # If running without commit for dry run, so nothing being stored in db
if name != m_name:
raise Exception(
"OSNI Object ID code %s is used for %s and %s" %
(osni_object_id, name, m_name))
# Otherwise, combine the two shapes for one area
poly.append(feat.geom)
continue
country = 'N'
try:
check = control.check(name, area_code, country, feat.geom)
if check is True:
raise Area.DoesNotExist
if isinstance(check, Area):
m = check
ons_code = m.codes.get(type=code_version).code
elif ons_code:
m = Area.objects.get(codes__type=code_version,
codes__code=ons_code)
elif osni_object_id:
m = Area.objects.get(codes__type=code_type_osni,
codes__code=osni_object_id,
generation_high=current_generation)
m_name = m.names.get(type=name_type).name
if name != m_name:
raise Exception(
"OSNI Object ID code %s is %s in DB but %s in SHP file"
% (osni_object_id, m_name, name))
else:
raise Exception(
'Area "%s" (%s) has neither ONS code nor OSNI Object ID'
% (name, area_code))
if int(options['verbosity']) > 1:
print(" Area matched, %s" % (m, ))
except Area.DoesNotExist:
area_type = Type.objects.get(code=area_code)
# It's possible we already have NIE entries (without any codes) in the db
try:
if area_code == 'NIE':
matching_name = name.title().replace('And', 'and')
m = Area.objects.get(
name=matching_name,
type=area_type,
generation_high=current_generation)
if int(options['verbosity']) > 1:
print(" Area matched (via name), %s" % (m, ))
else:
raise Area.DoesNotExist("Still doesn't exist")
except Area.DoesNotExist:
print(" New area: %s %s %s %s" %
(area_code, ons_code, osni_object_id, name))
m = Area(
name=
name, # Not overwritten by m.names.update_or_create as no "N" use
type=area_type,
country=Country.objects.get(code=country),
generation_low=new_generation,
generation_high=new_generation,
)
if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
raise Exception(
"Area %s found, but not in current generation %s" %
(m, current_generation))
m.generation_high = new_generation
if options['commit']:
m.save()
# Make a GEOS geometry only to check for validity:
g = feat.geom
geos_g = g.geos
if not geos_g.valid:
print(" Geometry of %s %s not valid" % (ons_code, m))
geos_g = fix_invalid_geos_geometry(geos_g)
if geos_g is None:
raise Exception(
"The geometry for area %s was invalid and couldn't be fixed"
% name)
g = None
else:
g = geos_g.ogr
g = area_code_info.transform_geom(g)
poly = [g]
if options['commit']:
m.names.update_or_create(type=name_type,
defaults={'name': name})
if ons_code:
self.ons_code_to_shape[ons_code] = (m, poly)
if options['commit']:
m.codes.update_or_create(type=code_version,
defaults={'code': ons_code})
if osni_object_id:
self.osni_object_id_to_shape[osni_object_id] = (m, poly)
if options['commit']:
m.codes.update_or_create(type=code_type_osni,
defaults={'code': osni_object_id})
if options['commit']:
save_polygons(self.osni_object_id_to_shape)
save_polygons(self.ons_code_to_shape)
def handle_label(self, filename, **options):
err = False
for k in ['generation_id','area_type_code','name_type_code','country_code']:
if options[k]: continue
print "Missing argument '--%s'" % k
err = True
if err:
sys.exit(1)
generation_id = options['generation_id']
area_type_code = options['area_type_code']
name_type_code = options['name_type_code']
country_code = options['country_code']
name_field = options['name_field'] or 'Name'
code_field = options['code_field']
code_type_code = options['code_type']
encoding = options['encoding'] or 'utf-8'
if len(area_type_code)>3:
print "Area type code must be 3 letters or fewer, sorry"
sys.exit(1)
if (code_field and not code_type_code) or (not code_field and code_type_code):
print "You must specify both code_field and code_type, or neither."
sys.exit(1)
try:
area_type = Type.objects.get(code=area_type_code)
except:
type_desc = raw_input('Please give a description for area type code %s: ' % area_type_code)
area_type = Type(code=area_type_code, description=type_desc)
if options['commit']: area_type.save()
try:
name_type = NameType.objects.get(code=name_type_code)
except:
name_desc = raw_input('Please give a description for name type code %s: ' % name_type_code)
name_type = NameType(code=name_type_code, description=name_desc)
if options['commit']: name_type.save()
try:
country = Country.objects.get(code=country_code)
except:
country_name = raw_input('Please give the name for country code %s: ' % country_code)
country = Country(code=country_code, name=country_name)
if options['commit']: country.save()
if code_type_code:
try:
code_type = CodeType.objects.get(code=code_type_code)
except:
code_desc = raw_input('Please give a description for code type %s: ' % code_type_code)
code_type = CodeType(code=code_type_code, description=code_desc)
if options['commit']: code_type.save()
print "Importing from %s" % filename
if not options['commit']:
print '(will not save to db as --commit not specified)'
current_generation = Generation.objects.current()
new_generation = Generation.objects.get( id=generation_id )
def verbose(*args):
if options['verbose']:
print " ".join(str(a) for a in args)
ds = DataSource(filename)
layer = ds[0]
for feat in layer:
try:
name = feat[name_field].value
except:
choices = ', '.join(layer.fields)
print "Could not find name using name field '%s' - should it be something else? It will be one of these: %s. Specify which with --name_field" % (name_field, choices)
sys.exit(1)
try:
name = name.decode(encoding)
except:
print "Could not decode name using encoding '%s' - is it in another encoding? Specify one with --encoding" % encoding
sys.exit(1)
name = re.sub('\s+', ' ', name)
if not name:
raise Exception( "Could not find a name to use for area" )
code = None
if code_field:
try:
code = feat[code_field].value
except:
choices = ', '.join(layer.fields)
print "Could not find code using code field '%s' - should it be something else? It will be one of these: %s. Specify which with --code_field" % (code_field, choices)
sys.exit(1)
print " looking at '%s'%s" % ( name.encode('utf-8'), (' (%s)' % code) if code else '' )
g = feat.geom.transform(4326, clone=True)
try:
if code:
matching_message = "code %s of code type %s" % (code, code_type)
areas = Area.objects.filter(codes__code=code, codes__type=code_type).order_by('-generation_high')
else:
matching_message = "name %s of area type %s" % (name, area_type)
areas = Area.objects.filter(name=name, type=area_type).order_by('-generation_high')
if len(areas) == 0:
raise Area.DoesNotExist
m = areas[0]
verbose(" found the area")
if options['preserve']:
# Find whether we need to create a new Area:
previous_geos_geometry = m.polygons.collect()
if m.generation_high < current_generation.id:
# Then it was missing in current_generation:
verbose(" area existed previously, but was missing from", current_generation)
create_new_area = True
elif previous_geos_geometry is None:
# It was empty in the previous generation:
verbose(" area was empty in", current_generation)
create_new_area = True
else:
# Otherwise, create a new Area unless the
# polygons were the same in current_generation:
previous_geos_geometry = previous_geos_geometry.simplify(tolerance=0)
new_geos_geometry = g.geos.simplify(tolerance=0)
create_new_area = not previous_geos_geometry.equals(new_geos_geometry)
p = previous_geos_geometry.sym_difference(new_geos_geometry).area / previous_geos_geometry.area
verbose(" change in area is:", "%.03f%%" % (100 * p,))
if create_new_area:
verbose(" the area", m, "has changed, creating a new area due to --preserve")
else:
verbose(" the area remained the same")
if create_new_area:
m = Area(
name = name,
type = area_type,
country = country,
# parent_area = parent_area,
generation_low = new_generation,
generation_high = new_generation
)
if options['use_code_as_id'] and code:
m.id = int(code)
else:
# If --preserve is not specified, the code or the name must be unique:
if len(areas) > 1:
raise Area.MultipleObjectsReturned, "There was more than one area with %s, and --preserve was not specified" % (matching_message,)
except Area.DoesNotExist:
verbose(" the area was not found - creating a new one")
m = Area(
name = name,
type = area_type,
country = country,
# parent_area = parent_area,
generation_low = new_generation,
generation_high = new_generation,
)
if options['use_code_as_id'] and code:
m.id = int(code)
# check that we are not about to skip a generation
if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
raise Exception, "Area %s found, but not in current generation %s" % (m, current_generation)
m.generation_high = new_generation
poly = [ g ]
if options['commit']:
m.save()
m.names.update_or_create({ 'type': name_type }, { 'name': name })
if code:
m.codes.update_or_create({ 'type': code_type }, { 'code': code })
save_polygons({ m.id : (m, poly) })
def handle_label(self, filename, **options):
country = Country.objects.get(code='N')
oa_type = Type.objects.get(code='OUA')
soa_type = Type.objects.get(code='OLF')
name_type = NameType.objects.get(code='S')
code_type = CodeType.objects.get(code='ons')
current_generation = Generation.objects.current()
new_generation = Generation.objects.new()
if not new_generation:
raise Exception, "No new generation to be used for import!"
# Compile an alphabetical list of NI councils and their wards, OA codes
# are assigned alphabetically.
if not self.councils:
self.councils = Area.objects.filter(type=Type.objects.get(code='LGD')).order_by('name').values()
for lgd in self.councils:
lges = Area.objects.filter(parent_area=lgd['id'])
areas = []
for lge in lges:
lgws = Area.objects.filter(parent_area=lge).values()
areas += lgws
lgd['wards'] = sorted(areas, key=lambda x: x['name'])
ds = DataSource(filename)
layer = ds[0]
layer_name = str(layer)
for feat in layer:
if layer_name == 'soa':
area_type = soa_type
ons_code = feat['SOA_CODE'].value
name = feat['SOA_LABEL'].value.replace('_', ' ')
elif layer_name == 'OA_ni':
area_type = oa_type
ons_code = feat['OA_CODE'].value
name = 'Output Area %s' % ons_code
else:
raise Exception, 'Bad data passed in'
council = ord(ons_code[2:3]) - 65
ward = int(ons_code[4:6]) - 1
if ward == 98: # SOA covers two wards, set parent to council, best we can do
parent = self.councils[council]['id']
else:
parent = self.councils[council]['wards'][ward]['id']
try:
m = Area.objects.get(codes__type=code_type, codes__code=ons_code)
if int(options['verbosity']) > 1:
print " Area matched, %s" % (m, )
except Area.DoesNotExist:
print " New area: %s" % (ons_code)
m = Area(
name = name, # If committing, this will be overwritten by the m.names.update_or_create
type = area_type,
country = country,
parent_area_id = parent,
generation_low = new_generation,
generation_high = new_generation,
)
if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
raise Exception, "Area %s found, but not in current generation %s" % (m, current_generation)
m.generation_high = new_generation
m.parent_area_id = parent
if options['commit']:
m.save()
f = feat.geom
f.srid = 29902
poly = [ f ]
if options['commit']:
m.names.update_or_create({ 'type': name_type }, { 'name': name })
if ons_code:
self.ons_code_to_shape[ons_code] = (m, poly)
if options['commit']:
m.codes.update_or_create({ 'type': code_type }, { 'code': ons_code })
if options['commit']:
save_polygons(self.ons_code_to_shape)
def handle_label(self, filename, **options):
if not options["control"]:
raise Exception, "You must specify a control file"
__import__(options["control"])
control = sys.modules[options["control"]]
code_version = CodeType.objects.get(code=control.code_version())
name_type = NameType.objects.get(code="O")
code_type_os = CodeType.objects.get(code="unit_id")
print filename
current_generation = Generation.objects.current()
new_generation = Generation.objects.new()
if not new_generation:
raise Exception, "No new generation to be used for import!"
ds = DataSource(filename)
layer = ds[0]
for feat in layer:
name = feat["NAME"].value
if not isinstance(name, unicode):
name = name.decode("iso-8859-1")
name = re.sub("\s*\(DET( NO \d+|)\)\s*(?i)", "", name)
name = re.sub("\s+", " ", name)
ons_code = feat["CODE"].value if feat["CODE"].value not in ("999999", "999999999") else None
unit_id = str(feat["UNIT_ID"].value)
area_code = feat["AREA_CODE"].value
patch = self.patch_boundary_line(ons_code, area_code)
if patch == True:
ons_code = None
elif patch:
ons_code = patch
if area_code == "NCP":
continue # Ignore Non Parished Areas
if ons_code in self.ons_code_to_shape:
m, poly = self.ons_code_to_shape[ons_code]
try:
m_name = m.names.get(type=name_type).name
except Name.DoesNotExist:
m_name = m.name # If running without commit for dry run, so nothing being stored in db
if name != m_name:
raise Exception, "ONS code %s is used for %s and %s" % (ons_code, name, m_name)
# Otherwise, combine the two shapes for one area
poly.append(feat.geom)
continue
if unit_id in self.unit_id_to_shape:
m, poly = self.unit_id_to_shape[unit_id]
try:
m_name = m.names.get(type=name_type).name
except Name.DoesNotExist:
m_name = m.name # If running without commit for dry run, so nothing being stored in db
if name != m_name:
raise Exception, "Unit ID code %s is used for %s and %s" % (unit_id, name, m_name)
# Otherwise, combine the two shapes for one area
poly.append(feat.geom)
continue
if code_version.code == "gss" and ons_code:
country = ons_code[0] # Hooray!
elif area_code in ("CED", "CTY", "DIW", "DIS", "MTW", "MTD", "LBW", "LBO", "LAC", "GLA"):
country = "E"
elif code_version.code == "gss":
raise Exception, area_code
elif (
(area_code == "EUR" and "Scotland" in name)
or area_code in ("SPC", "SPE")
or (ons_code and ons_code[0:3] in ("00Q", "00R"))
):
country = "S"
elif (
(area_code == "EUR" and "Wales" in name)
or area_code in ("WAC", "WAE")
or (ons_code and ons_code[0:3] in ("00N", "00P"))
):
country = "W"
elif area_code in ("EUR", "UTA", "UTE", "UTW", "CPC"):
country = "E"
else: # WMC
# Make sure WMC are loaded after all wards...
area_within = Area.objects.filter(
type__code__in=("UTW", "UTE", "MTW", "COP", "LBW", "DIW"),
polygons__polygon__contains=feat.geom.geos.point_on_surface,
)[0]
country = area_within.country.code
# Can't do the above ons_code checks with new GSS codes, will have to do more PinP checks
# Do parents in separate P-in-P code after this is done.
try:
check = control.check(name, area_code, country, feat.geom)
if check == True:
raise Area.DoesNotExist
if isinstance(check, Area):
m = check
ons_code = m.codes.get(type=code_version).code
elif ons_code:
m = Area.objects.get(codes__type=code_version, codes__code=ons_code)
elif unit_id:
m = Area.objects.get(codes__type=code_type_os, codes__code=unit_id)
m_name = m.names.get(type=name_type).name
if name != m_name:
raise Exception, "Unit ID code %s is %s in DB but %s in SHP file" % (unit_id, m_name, name)
else:
raise Exception, 'Area "%s" (%s) has neither ONS code nor unit ID' % (name, area_code)
if int(options["verbosity"]) > 1:
print " Area matched, %s" % (m,)
except Area.DoesNotExist:
print " New area: %s %s %s %s" % (area_code, ons_code, unit_id, name)
m = Area(
name=name, # If committing, this will be overwritten by the m.names.update_or_create
type=Type.objects.get(code=area_code),
country=Country.objects.get(code=country),
generation_low=new_generation,
generation_high=new_generation,
)
if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
raise Exception, "Area %s found, but not in current generation %s" % (m, current_generation)
m.generation_high = new_generation
if options["commit"]:
m.save()
poly = [feat.geom]
if options["commit"]:
m.names.update_or_create({"type": name_type}, {"name": name})
if ons_code:
self.ons_code_to_shape[ons_code] = (m, poly)
if options["commit"]:
m.codes.update_or_create({"type": code_version}, {"code": ons_code})
if unit_id:
self.unit_id_to_shape[unit_id] = (m, poly)
if options["commit"]:
m.codes.update_or_create({"type": code_type_os}, {"code": unit_id})
if options["commit"]:
save_polygons(self.unit_id_to_shape)
save_polygons(self.ons_code_to_shape)
def handle_label(self, filename, **options):
err = False
for k in ['generation_id','area_type_code','name_type_code','country_code']:
if options[k]: continue
print "Missing argument '--%s'" % k
err = True
if err:
sys.exit(1)
generation_id = options['generation_id']
area_type_code = options['area_type_code']
name_type_code = options['name_type_code']
country_code = options['country_code']
name_field = options['name_field'] or 'Name'
code_field = options['code_field']
code_type_code = options['code_type']
encoding = options['encoding'] or 'utf-8'
if len(area_type_code)>3:
print "Area type code must be 3 letters or fewer, sorry"
sys.exit(1)
if (code_field and not code_type_code) or (not code_field and code_type_code):
print "You must specify both code_field and code_type, or neither."
sys.exit(1)
try:
area_type = Type.objects.get(code=area_type_code)
except:
type_desc = raw_input('Please give a description for area type code %s: ' % area_type_code)
area_type = Type(code=area_type_code, description=type_desc)
if options['commit']: area_type.save()
try:
name_type = NameType.objects.get(code=name_type_code)
except:
name_desc = raw_input('Please give a description for name type code %s: ' % name_type_code)
name_type = NameType(code=name_type_code, description=name_desc)
if options['commit']: name_type.save()
try:
country = Country.objects.get(code=country_code)
except:
country_name = raw_input('Please give the name for country code %s: ' % country_code)
country = Country(code=country_code, name=country_name)
if options['commit']: country.save()
if code_type_code:
try:
code_type = CodeType.objects.get(code=code_type_code)
except:
code_desc = raw_input('Please give a description for code type %s: ' % code_type_code)
code_type = CodeType(code=code_type_code, description=code_desc)
if options['commit']: code_type.save()
print "Importing from %s" % filename
if not options['commit']:
print '(will not save to db as --commit not specified)'
current_generation = Generation.objects.current()
new_generation = Generation.objects.get( id=generation_id )
ds = DataSource(filename)
layer = ds[0]
for feat in layer:
try:
name = feat[name_field].value
except:
choices = ', '.join(layer.fields)
print "Could not find name using name field '%s' - should it be something else? It will be one of these: %s. Specify which with --name_field" % (name_field, choices)
sys.exit(1)
try:
name = name.decode(encoding)
except:
print "Could not decode name using encoding '%s' - is it in another encoding? Specify one with --encoding" % encoding
sys.exit(1)
name = re.sub('\s+', ' ', name)
if not name:
raise Exception( "Could not find a name to use for area" )
code = None
if code_field:
try:
code = feat[code_field].value
except:
choices = ', '.join(layer.fields)
print "Could not find code using code field '%s' - should it be something else? It will be one of these: %s. Specify which with --code_field" % (code_field, choices)
sys.exit(1)
print " looking at '%s'%s" % ( name.encode('utf-8'), (' (%s)' % code) if code else '' )
try:
if code:
m = Area.objects.get(codes__code=code, codes__type=code_type)
else:
# Assumes unique names if no code column used
m = Area.objects.get(name=name, type=area_type)
except Area.DoesNotExist:
m = Area(
name = name,
type = area_type,
country = country,
# parent_area = parent_area,
generation_low = new_generation,
generation_high = new_generation,
)
if options['use_code_as_id'] and code:
m.id = int(code)
# check that we are not about to skip a generation
if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
raise Exception, "Area %s found, but not in current generation %s" % (m, current_generation)
m.generation_high = new_generation
g = feat.geom.transform(4326, clone=True)
poly = [ g ]
if options['commit']:
m.save()
m.names.update_or_create({ 'type': name_type }, { 'name': name })
if code:
m.codes.update_or_create({ 'type': code_type }, { 'code': code })
save_polygons({ m.id : (m, poly) })
def handle_label(self, filename, **options):
if not options['control']:
raise Exception, "You must specify a control file"
__import__(options['control'])
control = sys.modules[options['control']]
code_version = CodeType.objects.get(code=control.code_version())
name_type = NameType.objects.get(code='O')
code_type_os = CodeType.objects.get(code='unit_id')
print filename
current_generation = Generation.objects.current()
new_generation = Generation.objects.new()
if not new_generation:
raise Exception, "No new generation to be used for import!"
ds = DataSource(filename)
layer = ds[0]
for feat in layer:
name = feat['NAME'].value
if not isinstance(name, unicode):
name = name.decode('iso-8859-1')
name = re.sub('\s*\(DET( NO \d+|)\)\s*(?i)', '', name)
name = re.sub('\s+', ' ', name)
ons_code = feat['CODE'].value if feat['CODE'].value not in (
'999999', '999999999') else None
unit_id = str(feat['UNIT_ID'].value)
area_code = feat['AREA_CODE'].value
patch = self.patch_boundary_line(ons_code, area_code)
if patch == True: ons_code = None
elif patch: ons_code = patch
if area_code == 'NCP': continue # Ignore Non Parished Areas
if ons_code in self.ons_code_to_shape:
m, poly = self.ons_code_to_shape[ons_code]
try:
m_name = m.names.get(type=name_type).name
except Name.DoesNotExist:
m_name = m.name # If running without commit for dry run, so nothing being stored in db
if name != m_name:
raise Exception, "ONS code %s is used for %s and %s" % (
ons_code, name, m_name)
# Otherwise, combine the two shapes for one area
poly.append(feat.geom)
continue
if unit_id in self.unit_id_to_shape:
m, poly = self.unit_id_to_shape[unit_id]
try:
m_name = m.names.get(type=name_type).name
except Name.DoesNotExist:
m_name = m.name # If running without commit for dry run, so nothing being stored in db
if name != m_name:
raise Exception, "Unit ID code %s is used for %s and %s" % (
unit_id, name, m_name)
# Otherwise, combine the two shapes for one area
poly.append(feat.geom)
continue
if code_version.code == 'gss' and ons_code:
country = ons_code[0] # Hooray!
elif area_code in ('CED', 'CTY', 'DIW', 'DIS', 'MTW', 'MTD', 'LBW',
'LBO', 'LAC', 'GLA'):
country = 'E'
elif code_version.code == 'gss':
raise Exception, area_code
elif (area_code == 'EUR' and 'Scotland' in name) or area_code in (
'SPC', 'SPE') or (ons_code
and ons_code[0:3] in ('00Q', '00R')):
country = 'S'
elif (area_code == 'EUR' and 'Wales' in name) or area_code in (
'WAC', 'WAE') or (ons_code
and ons_code[0:3] in ('00N', '00P')):
country = 'W'
elif area_code in ('EUR', 'UTA', 'UTE', 'UTW', 'CPC'):
country = 'E'
else: # WMC
# Make sure WMC are loaded after all wards...
area_within = Area.objects.filter(
type__code__in=('UTW', 'UTE', 'MTW', 'COP', 'LBW', 'DIW'),
polygons__polygon__contains=feat.geom.geos.point_on_surface
)[0]
country = area_within.country.code
# Can't do the above ons_code checks with new GSS codes, will have to do more PinP checks
# Do parents in separate P-in-P code after this is done.
try:
check = control.check(name, area_code, country, feat.geom)
if check == True:
raise Area.DoesNotExist
if isinstance(check, Area):
m = check
ons_code = m.codes.get(type=code_version).code
elif ons_code:
m = Area.objects.get(codes__type=code_version,
codes__code=ons_code)
elif unit_id:
m = Area.objects.get(codes__type=code_type_os,
codes__code=unit_id,
generation_high=current_generation)
m_name = m.names.get(type=name_type).name
if name != m_name:
raise Exception, "Unit ID code %s is %s in DB but %s in SHP file" % (
unit_id, m_name, name)
else:
raise Exception, 'Area "%s" (%s) has neither ONS code nor unit ID' % (
name, area_code)
if int(options['verbosity']) > 1:
print " Area matched, %s" % (m, )
except Area.DoesNotExist:
print " New area: %s %s %s %s" % (area_code, ons_code,
unit_id, name)
m = Area(
name=
name, # If committing, this will be overwritten by the m.names.update_or_create
type=Type.objects.get(code=area_code),
country=Country.objects.get(code=country),
generation_low=new_generation,
generation_high=new_generation,
)
if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
raise Exception, "Area %s found, but not in current generation %s" % (
m, current_generation)
m.generation_high = new_generation
if options['commit']:
m.save()
poly = [feat.geom]
if options['commit']:
m.names.update_or_create({'type': name_type}, {'name': name})
if ons_code:
self.ons_code_to_shape[ons_code] = (m, poly)
if options['commit']:
m.codes.update_or_create({'type': code_version},
{'code': ons_code})
if unit_id:
self.unit_id_to_shape[unit_id] = (m, poly)
if options['commit']:
m.codes.update_or_create({'type': code_type_os},
{'code': unit_id})
if options['commit']:
save_polygons(self.unit_id_to_shape)
save_polygons(self.ons_code_to_shape)
def process_file(self, filename, area_code, srid, control, options):
code_version = CodeType.objects.get(code=control.code_version())
name_type = NameType.objects.get(code="N")
code_type_osni = CodeType.objects.get(code="osni_oid")
if not hasattr(self, area_code):
raise Exception("Don't know how to extract features from %s files" % area_code)
area_code_info = getattr(self, area_code)(srid)
print(filename)
current_generation = Generation.objects.current()
new_generation = Generation.objects.new()
if not new_generation:
raise Exception("No new generation to be used for import!")
ds = DataSource(filename)
layer = ds[0]
for feat in layer:
name, ons_code, osni_object_id = area_code_info.extract_fields(feat)
name = self.format_name(name)
if ons_code in self.ons_code_to_shape:
m, poly = self.ons_code_to_shape[ons_code]
try:
m_name = m.names.get(type=name_type).name
except Name.DoesNotExist:
m_name = m.name # If running without commit for dry run, so nothing being stored in db
if name != m_name:
raise Exception("ONS code %s is used for %s and %s" % (ons_code, name, m_name))
# Otherwise, combine the two shapes for one area
poly.append(feat.geom)
continue
if osni_object_id in self.osni_object_id_to_shape:
m, poly = self.osni_object_id_to_shape[osni_object_id]
try:
m_name = m.names.get(type=name_type).name
except Name.DoesNotExist:
m_name = m.name # If running without commit for dry run, so nothing being stored in db
if name != m_name:
raise Exception("OSNI Object ID code %s is used for %s and %s" % (osni_object_id, name, m_name))
# Otherwise, combine the two shapes for one area
poly.append(feat.geom)
continue
country = "N"
try:
check = control.check(name, area_code, country, feat.geom)
if check is True:
raise Area.DoesNotExist
if isinstance(check, Area):
m = check
ons_code = m.codes.get(type=code_version).code
elif ons_code:
m = Area.objects.get(codes__type=code_version, codes__code=ons_code)
elif osni_object_id:
m = Area.objects.get(
codes__type=code_type_osni, codes__code=osni_object_id, generation_high=current_generation
)
m_name = m.names.get(type=name_type).name
if name != m_name:
raise Exception(
"OSNI Object ID code %s is %s in DB but %s in SHP file" % (osni_object_id, m_name, name)
)
else:
raise Exception('Area "%s" (%s) has neither ONS code nor OSNI Object ID' % (name, area_code))
if int(options["verbosity"]) > 1:
print(" Area matched, %s" % (m,))
except Area.DoesNotExist:
area_type = Type.objects.get(code=area_code)
# It's possible we already have NIE entries (without any codes) in the db
try:
if area_code == "NIE":
matching_name = name.title().replace("And", "and")
m = Area.objects.get(name=matching_name, type=area_type, generation_high=current_generation)
if int(options["verbosity"]) > 1:
print(" Area matched (via name), %s" % (m,))
else:
raise Area.DoesNotExist("Still doesn't exist")
except Area.DoesNotExist:
print(" New area: %s %s %s %s" % (area_code, ons_code, osni_object_id, name))
m = Area(
name=name, # Not overwritten by m.names.update_or_create as no "N" use
type=area_type,
country=Country.objects.get(code=country),
generation_low=new_generation,
generation_high=new_generation,
)
if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
raise Exception("Area %s found, but not in current generation %s" % (m, current_generation))
m.generation_high = new_generation
if options["commit"]:
m.save()
# Make a GEOS geometry only to check for validity:
g = feat.geom
geos_g = g.geos
if not geos_g.valid:
print(" Geometry of %s %s not valid" % (ons_code, m))
geos_g = fix_invalid_geos_geometry(geos_g)
if geos_g is None:
raise Exception("The geometry for area %s was invalid and couldn't be fixed" % name)
g = None
else:
g = geos_g.ogr
g = area_code_info.transform_geom(g)
poly = [g]
if options["commit"]:
m.names.update_or_create(type=name_type, defaults={"name": name})
if ons_code:
self.ons_code_to_shape[ons_code] = (m, poly)
if options["commit"]:
m.codes.update_or_create(type=code_version, defaults={"code": ons_code})
if osni_object_id:
self.osni_object_id_to_shape[osni_object_id] = (m, poly)
if options["commit"]:
m.codes.update_or_create(type=code_type_osni, defaults={"code": osni_object_id})
if options["commit"]:
save_polygons(self.osni_object_id_to_shape)
save_polygons(self.ons_code_to_shape)
def update_or_create():
if osm_code:
m = osm_code.area
else:
m = Area(
name=name,
type=Type.objects.get(code=area_code),
country=Country.objects.get(code='G'),
parent_area=parent_area,
generation_low=new_generation,
generation_high=new_generation,
)
if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
raise Exception, "Area %s found, but not in current generation %s" % (
m, current_generation)
m.generation_high = new_generation
g = feat.geom.transform(4326, clone=True)
# In generating the data we should have
# excluded any "polygons" with less than four
# points (the final one being the same as the
# first), but just in case:
for polygon in g:
if g.num_points < 4:
return
poly = [g]
if options['commit']:
m.save()
if name not in kml_data.data:
print json.dumps(kml_data.data,
sort_keys=True,
indent=4)
raise Exception, u"Will fail to find '%s' in the dictionary" % (
name, )
for k, v in kml_data.data[name].items():
language_name = None
if k == 'name':
lang = 'default'
language_name = "OSM Default"
else:
name_match = re.search(r'^name:(.+)$', k)
if name_match:
lang = name_match.group(1)
if lang in language_code_to_name:
language_name = language_code_to_name[
lang]
if not language_name:
continue
# Otherwise, make sure that a NameType for this language exists:
NameType.objects.update_or_create(
{'code': lang}, {
'code': lang,
'description': language_name
})
name_type = NameType.objects.get(code=lang)
m.names.update_or_create({'type': name_type},
{'name': v})
m.codes.update_or_create({'type': code_type_osm},
{'code': osm_id})
save_polygons({code: (m, poly)})
def handle_label(self, filename, **options):
self.stdout.write("Importing from %s" % filename)
current_generation = Generation.objects.current()
new_generation = Generation.objects.new()
file_type = options["file_type"].value.lower()
generalized = options["generalized"]
if file_type == "lsoa": # England and Wales
feat_name = 'LSOA11NM'
feat_code = 'LSOA11CD'
elif file_type == "msoa": # England and Wales
feat_name = 'MSOA11NM'
feat_code = 'MSOA11CD'
elif file_type == "soa": # super output areas - NI
feat_name = 'SOA_LABEL'
feat_code = 'SOA_CODE'
elif file_type == "dz": # datazone - scotland
feat_name = 'Name'
feat_code = 'DataZone'
elif file_type == "iz": # intermediate zone - scotland
feat_name = 'Name'
feat_code = 'InterZone'
else:
raise Exception(
"Need to specify a small area type (lsoa, dz, soa, iz, msoa)")
if file_type in ["lsoa", "dz", "soa"]:
area_type = 'OLF'
elif file_type in ["msoa", "iz"]:
area_type = 'OMF'
if generalized:
area_type = area_type[:2] + "G"
ds = DataSource(filename)
layer = ds[0]
for feat in layer:
# retrieve name and code, and set country
name = feat[feat_name].value
code = feat[feat_code].value
if code[:2] == "95":
country = "N"
else:
country = code[0]
self.stdout.write(" looking at '%s' (%s)" % (name, code))
g = feat.geom.transform(settings.MAPIT_AREA_SRID, clone=True)
# skip if the SOA already exists in db (SOAs don't change)
areas = Area.objects.filter(type__code=area_type, codes__code=code)
if len(areas):
m = areas[0]
# check that we are not about to skip a generation
if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
raise Exception(
"Area %s found, but not in current generation %s" %
(m, current_generation))
if m.generation_high != new_generation and options['commit']:
m.generation_high = new_generation
m.save()
continue
print("Adding %s (%s) %s" % (name, code, feat.geom.geom_name))
m = Area(
name=name,
type=Type.objects.get(code=area_type),
country=Country.objects.get(code=country),
generation_low=new_generation,
generation_high=new_generation,
)
poly = [g] if g is not None else []
if options['commit']:
m.save()
m.names.update_or_create(type=NameType.objects.get(code='S'),
defaults={'name': name})
m.codes.update_or_create(type=CodeType.objects.get(code='ons'),
defaults={'code': code})
save_polygons({m.id: (m, poly)})
def handle_label(self, filename, **options):
missing_options = []
for k in ['generation_id', 'area_type_code', 'name_type_code', 'country_code']:
if options[k]:
continue
else:
missing_options.append(k)
if missing_options:
message_start = "Missing arguments " if len(missing_options) > 1 else "Missing argument "
message = message_start + " ".join('--{0}'.format(k) for k in missing_options)
raise CommandError(message)
generation_id = options['generation_id']
area_type_code = options['area_type_code']
name_type_code = options['name_type_code']
country_code = options['country_code']
override_name = options['override_name']
name_field = options['name_field']
if not (override_name or name_field):
name_field = 'Name'
override_code = options['override_code']
code_field = options['code_field']
code_type_code = options['code_type']
encoding = options['encoding'] or 'utf-8'
if len(area_type_code) > 3:
raise CommandError("Area type code must be 3 letters or fewer, sorry")
if name_field and override_name:
raise CommandError("You must not specify both --name_field and --override_name")
if code_field and override_code:
raise CommandError("You must not specify both --code_field and --override_code")
using_code = (code_field or override_code)
if (using_code and not code_type_code) or (not using_code and code_type_code):
raise CommandError(
"If you want to save a code, specify --code_type and either --code_field or --override_code")
try:
area_type = Type.objects.get(code=area_type_code)
except:
type_desc = input('Please give a description for area type code %s: ' % area_type_code)
area_type = Type(code=area_type_code, description=type_desc)
if options['commit']:
area_type.save()
try:
name_type = NameType.objects.get(code=name_type_code)
except:
name_desc = input('Please give a description for name type code %s: ' % name_type_code)
name_type = NameType(code=name_type_code, description=name_desc)
if options['commit']:
name_type.save()
try:
country = Country.objects.get(code=country_code)
except:
country_name = input('Please give the name for country code %s: ' % country_code)
country = Country(code=country_code, name=country_name)
if options['commit']:
country.save()
if code_type_code:
try:
code_type = CodeType.objects.get(code=code_type_code)
except:
code_desc = input('Please give a description for code type %s: ' % code_type_code)
code_type = CodeType(code=code_type_code, description=code_desc)
if options['commit']:
code_type.save()
self.stdout.write("Importing from %s" % filename)
if not options['commit']:
self.stdout.write('(will not save to db as --commit not specified)')
current_generation = Generation.objects.current()
new_generation = Generation.objects.get(id=generation_id)
def verbose(*args):
if int(options['verbosity']) > 1:
self.stdout.write(" ".join(str(a) for a in args))
ds = DataSource(filename)
layer = ds[0]
if (override_name or override_code) and len(layer) > 1:
message = (
"Warning: you have specified an override %s and this file contains more than one feature; "
"multiple areas with the same %s will be created")
if override_name:
self.stdout.write(message % ('name', 'name'))
if override_code:
self.stdout.write(message % ('code', 'code'))
for feat in layer:
if override_name:
name = override_name
else:
try:
name = feat[name_field].value
except:
choices = ', '.join(layer.fields)
raise CommandError(
"Could not find name using name field '%s' - should it be something else? "
"It will be one of these: %s. Specify which with --name_field" % (name_field, choices))
try:
if not isinstance(name, six.text_type):
name = name.decode(encoding)
except:
raise CommandError(
"Could not decode name using encoding '%s' - is it in another encoding? "
"Specify one with --encoding" % encoding)
name = re.sub('\s+', ' ', name)
if not name:
raise Exception("Could not find a name to use for area")
code = None
if override_code:
code = override_code
elif code_field:
try:
code = feat[code_field].value
except:
choices = ', '.join(layer.fields)
raise CommandError(
"Could not find code using code field '%s' - should it be something else? "
"It will be one of these: %s. Specify which with --code_field" % (code_field, choices))
self.stdout.write(" looking at '%s'%s" % (name, (' (%s)' % code) if code else ''))
g = None
if hasattr(feat, 'geom'):
g = feat.geom.transform(settings.MAPIT_AREA_SRID, clone=True)
try:
if options['new']: # Always want a new area
raise Area.DoesNotExist
if code:
matching_message = "code %s of code type %s" % (code, code_type)
areas = Area.objects.filter(codes__code=code, codes__type=code_type).order_by('-generation_high')
else:
matching_message = "name %s of area type %s" % (name, area_type)
areas = Area.objects.filter(name=name, type=area_type).order_by('-generation_high')
if len(areas) == 0:
verbose(" the area was not found - creating a new one")
raise Area.DoesNotExist
m = areas[0]
verbose(" found the area")
if options['preserve']:
# Find whether we need to create a new Area:
previous_geos_geometry = m.polygons.collect()
if m.generation_high < current_generation.id:
# Then it was missing in current_generation:
verbose(" area existed previously, but was missing from", current_generation)
raise Area.DoesNotExist
elif g is None:
if previous_geos_geometry is not None:
verbose(" area is now empty")
raise Area.DoesNotExist
else:
verbose(" the area has remained empty")
elif previous_geos_geometry is None:
# It was empty in the previous generation:
verbose(" area was empty in", current_generation)
raise Area.DoesNotExist
else:
# Otherwise, create a new Area unless the
# polygons were the same in current_generation:
previous_geos_geometry = previous_geos_geometry.simplify(tolerance=0)
new_geos_geometry = g.geos.simplify(tolerance=0)
create_new_area = not previous_geos_geometry.equals(new_geos_geometry)
p = previous_geos_geometry.sym_difference(new_geos_geometry).area / previous_geos_geometry.area
verbose(" change in area is:", "%.03f%%" % (100 * p,))
if create_new_area:
verbose(" the area", m, "has changed, creating a new area due to --preserve")
raise Area.DoesNotExist
else:
verbose(" the area remained the same")
else:
# If --preserve is not specified, the code or the name must be unique:
if len(areas) > 1:
raise Area.MultipleObjectsReturned(
"There was more than one area with %s, and --preserve was not specified" % (
matching_message,))
except Area.DoesNotExist:
m = Area(
name=name,
type=area_type,
country=country,
# parent_area=parent_area,
generation_low=new_generation,
generation_high=new_generation,
)
if options['use_code_as_id'] and code:
m.id = int(code)
# check that we are not about to skip a generation
if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
raise Exception("Area %s found, but not in current generation %s" % (m, current_generation))
m.generation_high = new_generation
if options['fix_invalid_polygons'] and g is not None:
# Make a GEOS geometry only to check for validity:
geos_g = g.geos
if not geos_g.valid:
geos_g = fix_invalid_geos_geometry(geos_g)
if geos_g is None:
self.stdout.write("The geometry for area %s was invalid and couldn't be fixed" % name)
g = None
else:
g = geos_g.ogr
poly = [g] if g is not None else []
if options['commit']:
m.save()
m.names.update_or_create(type=name_type, defaults={'name': name})
if code:
m.codes.update_or_create(type=code_type, defaults={'code': code})
save_polygons({m.id: (m, poly)})
def handle_label(self, filename, **options):
country = Country.objects.get(code='N')
oa_type = Type.objects.get(code='OUA')
soa_type = Type.objects.get(code='OLF')
name_type = NameType.objects.get(code='S')
code_type = CodeType.objects.get(code='ons')
current_generation = Generation.objects.current()
new_generation = Generation.objects.new()
if not new_generation:
raise Exception("No new generation to be used for import!")
# Compile an alphabetical list of NI councils and their wards, OA codes
# are assigned alphabetically.
if not self.councils:
self.councils = Area.objects.filter(type=Type.objects.get(
code='LGD')).order_by('name').values()
for lgd in self.councils:
lges = Area.objects.filter(parent_area=lgd['id'])
areas = []
for lge in lges:
lgws = Area.objects.filter(parent_area=lge).values()
areas += lgws
lgd['wards'] = sorted(areas, key=lambda x: x['name'])
ds = DataSource(filename)
layer = ds[0]
layer_name = str(layer)
for feat in layer:
if layer_name == 'soa':
area_type = soa_type
ons_code = feat['SOA_CODE'].value
name = feat['SOA_LABEL'].value.replace('_', ' ')
elif layer_name == 'OA_ni':
area_type = oa_type
ons_code = feat['OA_CODE'].value
name = 'Output Area %s' % ons_code
else:
raise Exception('Bad data passed in')
council = ord(ons_code[2:3]) - 65
ward = int(ons_code[4:6]) - 1
if ward == 98: # SOA covers two wards, set parent to council, best we can do
parent = self.councils[council]['id']
else:
parent = self.councils[council]['wards'][ward]['id']
try:
m = Area.objects.get(codes__type=code_type,
codes__code=ons_code)
if int(options['verbosity']) > 1:
print(" Area matched, %s" % (m, ))
except Area.DoesNotExist:
print(" New area: %s" % (ons_code))
m = Area(
name=
name, # If committing, this will be overwritten by the m.names.update_or_create
type=area_type,
country=country,
parent_area_id=parent,
generation_low=new_generation,
generation_high=new_generation,
)
if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
raise Exception(
"Area %s found, but not in current generation %s" %
(m, current_generation))
m.generation_high = new_generation
m.parent_area_id = parent
if options['commit']:
m.save()
f = feat.geom
f.srid = 29902
poly = [f]
if options['commit']:
m.names.update_or_create(type=name_type,
defaults={'name': name})
if ons_code:
self.ons_code_to_shape[ons_code] = (m, poly)
if options['commit']:
m.codes.update_or_create(type=code_type,
defaults={'code': ons_code})
if options['commit']:
save_polygons(self.ons_code_to_shape)
def handle_label(self, filename, **options):
if not options['control']:
raise Exception("You must specify a control file")
__import__(options['control'])
control = sys.modules[options['control']]
code_version = CodeType.objects.get(code='gss')
name_type = NameType.objects.get(code='O')
code_type_os = CodeType.objects.get(code='unit_id')
print(filename)
current_generation = Generation.objects.current()
new_generation = Generation.objects.new()
if not new_generation:
raise Exception("No new generation to be used for import!")
ds = DataSource(filename)
layer = ds[0]
for feat in layer:
name = feat['NAME'].value
if not isinstance(name, six.text_type):
name = name.decode('iso-8859-1')
name = re.sub(r'\s*\(DET( NO \d+|)\)\s*(?i)', '', name)
name = re.sub(r'\s+', ' ', name)
ons_code = feat['CODE'].value if feat['CODE'].value not in ('999999', '999999999') else None
unit_id = str(feat['UNIT_ID'].value)
area_code = feat['AREA_CODE'].value
patch = self.patch_boundary_line(name, ons_code, unit_id, area_code)
if 'ons-code' in patch:
ons_code = patch['ons-code']
elif 'unit-id' in patch:
unit_id = patch['unit-id']
if area_code == 'NCP':
continue # Ignore Non Parished Areas
if ons_code in self.ons_code_to_shape:
m, poly = self.ons_code_to_shape[ons_code]
try:
m_name = m.names.get(type=name_type).name
except Name.DoesNotExist:
m_name = m.name # If running without commit for dry run, so nothing being stored in db
if name != m_name:
raise Exception("ONS code %s is used for %s and %s" % (ons_code, name, m_name))
# Otherwise, combine the two shapes for one area
poly.append(feat.geom)
continue
if unit_id in self.unit_id_to_shape:
m, poly = self.unit_id_to_shape[unit_id]
try:
m_name = m.names.get(type=name_type).name
except Name.DoesNotExist:
m_name = m.name # If running without commit for dry run, so nothing being stored in db
if name != m_name:
raise Exception("Unit ID code %s is used for %s and %s" % (unit_id, name, m_name))
# Otherwise, combine the two shapes for one area
poly.append(feat.geom)
continue
if ons_code:
country = ons_code[0] # Hooray!
elif area_code in ('CED', 'CTY', 'DIW', 'DIS', 'MTW', 'MTD', 'LBW', 'LBO', 'LAC', 'GLA'):
country = 'E'
else:
raise Exception(area_code)
try:
check = control.check(name, area_code, country, feat.geom, ons_code=ons_code, commit=options['commit'])
if check is True:
raise Area.DoesNotExist
if isinstance(check, Area):
m = check
try:
ons_code = m.codes.get(type=code_version).code
except Code.DoesNotExist:
ons_code = None
elif ons_code:
m = Area.objects.get(codes__type=code_version, codes__code=ons_code)
elif unit_id:
m = Area.objects.get(
codes__type=code_type_os, codes__code=unit_id, generation_high=current_generation)
m_name = m.names.get(type=name_type).name
if name != m_name:
raise Exception("Unit ID code %s is %s in DB but %s in SHP file" % (unit_id, m_name, name))
else:
raise Exception('Area "%s" (%s) has neither ONS code nor unit ID' % (name, area_code))
if int(options['verbosity']) > 1:
print(" Area matched, %s" % (m, ))
except Area.DoesNotExist:
print(" New area: %s %s %s %s" % (area_code, ons_code, unit_id, name))
m = Area(
name=name, # If committing, this will be overwritten by the m.names.update_or_create
type=Type.objects.get(code=area_code),
country=Country.objects.get(code=country),
generation_low=new_generation,
generation_high=new_generation,
)
if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
raise Exception("Area %s found, but not in current generation %s" % (m, current_generation))
m.generation_high = new_generation
if options['commit']:
m.save()
# Make a GEOS geometry only to check for validity:
g = feat.geom
geos_g = g.geos
if not geos_g.valid:
print(" Geometry of %s %s not valid" % (ons_code, m))
geos_g = fix_invalid_geos_geometry(geos_g)
if geos_g is None:
raise Exception("The geometry for area %s was invalid and couldn't be fixed" % name)
g = None
else:
g = geos_g.ogr
poly = [g]
if options['commit']:
m.names.update_or_create(type=name_type, defaults={'name': name})
if ons_code:
self.ons_code_to_shape[ons_code] = (m, poly)
if options['commit']:
m.codes.update_or_create(type=code_version, defaults={'code': ons_code})
if unit_id:
self.unit_id_to_shape[unit_id] = (m, poly)
if options['commit']:
m.codes.update_or_create(type=code_type_os, defaults={'code': unit_id})
if options['commit']:
save_polygons(self.unit_id_to_shape)
save_polygons(self.ons_code_to_shape)
def handle_label(self, filename, **options):
if not options['control']:
raise Exception, "You must specify a control file"
__import__(options['control'])
control = sys.modules[options['control']]
code_version = CodeType.objects.get(code=control.code_version())
name_type = NameType.objects.get(code='O')
code_type_os = CodeType.objects.get(code='unit_id')
print filename
current_generation = Generation.objects.current()
new_generation = Generation.objects.new()
if not new_generation:
raise Exception, "No new generation to be used for import!"
ds = DataSource(filename)
layer = ds[0]
for feat in layer:
name = feat['NAME'].value
if not isinstance(name, unicode):
name = name.decode('iso-8859-1')
name = re.sub('\s*\(DET( NO \d+|)\)\s*(?i)', '', name)
name = re.sub('\s+', ' ', name)
ons_code = feat['CODE'].value if feat['CODE'].value not in ('999999', '999999999') else None
unit_id = str(feat['UNIT_ID'].value)
area_code = feat['AREA_CODE'].value
patch = self.patch_boundary_line(ons_code, area_code)
if patch == True: ons_code = None
elif patch: ons_code = patch
if area_code == 'NCP': continue # Ignore Non Parished Areas
if ons_code in self.ons_code_to_shape:
m, poly = self.ons_code_to_shape[ons_code]
try:
m_name = m.names.get(type=name_type).name
except Name.DoesNotExist:
m_name = m.name # If running without commit for dry run, so nothing being stored in db
if name != m_name:
raise Exception, "ONS code %s is used for %s and %s" % (ons_code, name, m_name)
# Otherwise, combine the two shapes for one area
poly.append(feat.geom)
continue
if unit_id in self.unit_id_to_shape:
m, poly = self.unit_id_to_shape[unit_id]
try:
m_name = m.names.get(type=name_type).name
except Name.DoesNotExist:
m_name = m.name # If running without commit for dry run, so nothing being stored in db
if name != m_name:
raise Exception, "Unit ID code %s is used for %s and %s" % (unit_id, name, m_name)
# Otherwise, combine the two shapes for one area
poly.append(feat.geom)
continue
if code_version.code == 'gss' and ons_code:
country = ons_code[0] # Hooray!
elif area_code in ('CED', 'CTY', 'DIW', 'DIS', 'MTW', 'MTD', 'LBW', 'LBO', 'LAC', 'GLA'):
country = 'E'
elif code_version.code == 'gss':
raise Exception, area_code
elif (area_code == 'EUR' and 'Scotland' in name) or area_code in ('SPC', 'SPE') or (ons_code and ons_code[0:3] in ('00Q', '00R')):
country = 'S'
elif (area_code == 'EUR' and 'Wales' in name) or area_code in ('WAC', 'WAE') or (ons_code and ons_code[0:3] in ('00N', '00P')):
country = 'W'
elif area_code in ('EUR', 'UTA', 'UTE', 'UTW', 'CPC'):
country = 'E'
else: # WMC
# Make sure WMC are loaded after all wards...
area_within = Area.objects.filter(type__code__in=('UTW','UTE','MTW','COP','LBW','DIW'), polygons__polygon__contains=feat.geom.geos.point_on_surface)[0]
country = area_within.country.code
# Can't do the above ons_code checks with new GSS codes, will have to do more PinP checks
# Do parents in separate P-in-P code after this is done.
try:
check = control.check(name, area_code, country, feat.geom)
if check == True:
raise Area.DoesNotExist
if isinstance(check, Area):
m = check
ons_code = m.codes.get(type=code_version).code
elif ons_code:
m = Area.objects.get(codes__type=code_version, codes__code=ons_code)
elif unit_id:
m = Area.objects.get(codes__type=code_type_os, codes__code=unit_id, generation_high=current_generation)
m_name = m.names.get(type=name_type).name
if name != m_name:
raise Exception, "Unit ID code %s is %s in DB but %s in SHP file" % (unit_id, m_name, name)
else:
raise Exception, 'Area "%s" (%s) has neither ONS code nor unit ID' % (name, area_code)
if int(options['verbosity']) > 1:
print " Area matched, %s" % (m, )
except Area.DoesNotExist:
print " New area: %s %s %s %s" % (area_code, ons_code, unit_id, name)
m = Area(
name = name, # If committing, this will be overwritten by the m.names.update_or_create
type = Type.objects.get(code=area_code),
country = Country.objects.get(code=country),
generation_low = new_generation,
generation_high = new_generation,
)
if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
raise Exception, "Area %s found, but not in current generation %s" % (m, current_generation)
m.generation_high = new_generation
if options['commit']:
m.save()
poly = [ feat.geom ]
if options['commit']:
m.names.update_or_create({ 'type': name_type }, { 'name': name })
if ons_code:
self.ons_code_to_shape[ons_code] = (m, poly)
if options['commit']:
m.codes.update_or_create({ 'type': code_version }, { 'code': ons_code })
if unit_id:
self.unit_id_to_shape[unit_id] = (m, poly)
if options['commit']:
m.codes.update_or_create({ 'type': code_type_os }, { 'code': unit_id })
if options['commit']:
save_polygons(self.unit_id_to_shape)
save_polygons(self.ons_code_to_shape)