コード例 #1
0
            def update_or_create():
                try:
                    m = Area.objects.get(id=code)
                except Area.DoesNotExist:
                    m = Area(
                        id=code,
                        name=name,
                        type=Type.objects.get(code=area_code),
                        country=Country.objects.get(code='O'),
                        parent_area=parent_area,
                        generation_low=new_generation,
                        generation_high=new_generation,
                    )

                if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
                    raise Exception("Area %s found, but not in current generation %s" % (m, current_generation))
                m.generation_high = new_generation

                g = feat.geom.transform(4326, clone=True)
                poly = [g]

                if options['commit']:
                    m.save()
                    for k, v in kml_data.data[name].items():
                        if k in ('name:smi', 'name:fi'):
                            lang = 'N' + k[5:]
                            m.names.update_or_create(type=NameType.objects.get(code=lang), defaults={'name': v})
                    m.codes.update_or_create(type=code_type_n5000, defaults={'code': code_str})
                    m.codes.update_or_create(type=code_type_osm, defaults={'code': int(kml_data.data[name]['osm'])})
                    save_polygons({code: (m, poly)})
コード例 #2
0
            def update_or_create():
                try:
                    m = Area.objects.get(id=code)
                except Area.DoesNotExist:
                    m = Area(
                        id=code,
                        name=name,
                        type=Type.objects.get(code=area_code),
                        country=Country.objects.get(code='O'),
                        parent_area=parent_area,
                        generation_low=new_generation,
                        generation_high=new_generation,
                    )

                if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
                    raise Exception("Area %s found, but not in current generation %s" % (m, current_generation))
                m.generation_high = new_generation

                g = feat.geom.transform(4326, clone=True)
                poly = [g]

                if options['commit']:
                    m.save()
                    for k, v in kml_data.data[name].items():
                        if k in ('name:smi', 'name:fi'):
                            lang = 'N' + k[5:]
                            m.names.update_or_create(type=NameType.objects.get(code=lang), defaults={'name': v})
                    m.codes.update_or_create(type=code_type_n5000, defaults={'code': code_str})
                    m.codes.update_or_create(type=code_type_osm, defaults={'code': int(kml_data.data[name]['osm'])})
                    save_polygons({code: (m, poly)})
コード例 #3
0
                def update_or_create():
                    try:
                        m = Area.objects.get(id=int(regionid))
                        print "Updating area %s with id %d" % (regionname,
                                                               int(regionid))
                    except Area.DoesNotExist:
                        print "Creating new area %s with id %d" % (
                            regionname, int(regionid))
                        m = Area(
                            id=int(regionid),
                            name=regionname,
                            type=Type.objects.get(code=area_type),
                            country=Country.objects.get(code='O'),
                            generation_low=new_generation,
                            generation_high=new_generation,
                        )

                    if m.generation_high and current_generation \
                            and m.generation_high.id < current_generation.id:
                        raise Exception, "Area %s found, but not in current generation %s" % (
                            m, current_generation)
                    m.generation_high = new_generation

                    poly = [GEOSGeometry(unionoutline).ogr]
                    if options['commit']:
                        m.save()
                        save_polygons({regionid: (m, poly)})
コード例 #4
0
    def handle_label(self, filename, **options):
        code_version = CodeType.objects.get(code='gss')
        name_type = NameType.objects.get(code='O')
        for feat in DataSource(filename)[0]:
            name = unicode(feat['NAME'].value, 'iso-8859-1')
            name = re.sub('\s*\(DET( NO \d+|)\)\s*(?i)', '', name)
            name = re.sub('\s+', ' ', name)
            ons_code = feat['CODE'].value
            area_code = feat['AREA_CODE'].value
            country = ons_code[0]
            if ons_code in ('E07000100', 'E07000104', 'S12000009',
                            'S12000043'):
                assert Area.objects.filter(codes__type=code_version,
                                           codes__code=ons_code).count() == 0
                print(ons_code, area_code, country, name)

                m = Area(
                    type=Type.objects.get(code=area_code),
                    country=Country.objects.get(code=country),
                    generation_low=Generation.objects.get(id=1),
                    generation_high=Generation.objects.get(id=14),
                )
                if options['commit']:
                    m.save()
                    m.names.update_or_create(type=name_type,
                                             defaults={'name': name})
                    m.codes.update_or_create(type=code_version,
                                             defaults={'code': ons_code})
                    save_polygons({ons_code: (m, [feat.geom])})
コード例 #5
0
    def handle_label(self, filename, **options):
        code_version = CodeType.objects.get(code='gss')
        name_type = NameType.objects.get(code='O')
        for feat in DataSource(filename)[0]:
            name = feat['NAME'].value
            if not isinstance(name, six.text_type):
                name = name.decode('iso-8859-1')
            name = re.sub(r'\s*\(DET( NO \d+|)\)\s*(?i)', '', name)
            name = re.sub(r'\s+', ' ', name)
            ons_code = feat['CODE'].value
            area_code = feat['AREA_CODE'].value
            country = ons_code[0]
            if ons_code in ('E07000100', 'E07000104', 'S12000009', 'S12000043'):
                assert Area.objects.filter(codes__type=code_version, codes__code=ons_code).count() == 0
                print(ons_code, area_code, country, name)

                m = Area(
                    type=Type.objects.get(code=area_code),
                    country=Country.objects.get(code=country),
                    generation_low=Generation.objects.get(id=1),
                    generation_high=Generation.objects.get(id=14),
                )
                if options['commit']:
                    m.save()
                    m.names.update_or_create(type=name_type, defaults={'name': name})
                    m.codes.update_or_create(type=code_version, defaults={'code': ons_code})
                    save_polygons({ons_code: (m, [feat.geom])})
コード例 #6
0
    def handle_label(self, filename, **options):
        print filename
        generation = Generation.objects.current()

        short_filename = filename.split("/")[-1]
        filename_prefix = short_filename[:4]
        filename_suffix = short_filename.split(".")[0][-3:]

        # check shapefile type - we handle both LSOA and MSOA
        if filename_prefix == "LSOA":
            feat_name = 'LSOA04NM'
            feat_code = 'LSOA04CD'
            if filename_suffix == 'BGC':
                area_type = 'OLG'
            else:
                area_type = 'OLF'
        elif filename_prefix == "MSOA":
            feat_name = 'MSOA04NM'
            feat_code = 'MSOA04CD'
            if filename_suffix == 'BGC':
                area_type = 'OMG'
            else:
                area_type = 'OMF'
        else:
            raise Exception, "Sorry, this script only handles LSOA/MSOA shapefiles!"

        ds = DataSource(filename)
        layer = ds[0]
        for feat in layer:
            # retrieve name and code, and set country
            name = feat[feat_name].value
            lsoa_code = feat[feat_code].value
            country = lsoa_code[0]
            # skip if the SOA already exists in db (SOAs don't change)
            if Area.objects.filter(type__code=area_type,
                                   codes__code=lsoa_code).count():
                continue
            print "Adding %s (%s) %s" % (name, lsoa_code, feat.geom.geom_name)
            m = Area(
                type=Type.objects.get(code=area_type),
                country=Country.objects.get(code=country),
                generation_low=generation,
                generation_high=generation,
            )
            m.save()
            m.names.update_or_create({'type': NameType.objects.get(code='S')},
                                     {'name': name})
            m.codes.update_or_create(
                {'type': CodeType.objects.get(code='ons')},
                {'code': lsoa_code})

            p = feat.geom
            if p.geom_name == 'POLYGON':
                shapes = [p]
            else:
                shapes = p
            for g in shapes:
                m.polygons.create(polygon=g.wkt)
コード例 #7
0
    def handle_label(self,  filename, **options):
        print filename
        new_generation = Generation.objects.new()
        if not new_generation:
            raise Exception, "No new generation to be used for import!"

        name_type = NameType.objects.get(code='O')
        code_type = CodeType.objects.get(code='gss')

        ds = DataSource(filename)
        layer = ds[0]
        for feat in layer:
            name = unicode(feat['NAME'].value, 'iso-8859-1')
            print " ", name
            name = re.sub('\s*\(DET( NO \d+|)\)\s*(?i)', '', name)
            name = re.sub('\s+', ' ', name)

            if "P Const" in name: area_code = 'SPC'
            elif "PER" in name: area_code = 'SPE'
            else: raise Exception, "Unknown type of area %s" % name

            ons_code = name_to_code[name]

            if ons_code in self.ons_code_to_shape:
                m, poly = self.ons_code_to_shape[ons_code]
                if options['commit']:
                    m_name = m.names.get(type=name_type).name
                    if name != m_name:
                        raise Exception, "ONS code %s is used for %s and %s" % (ons_code, name, m_name)
                # Otherwise, combine the two shapes for one area
                print "    Adding subsequent shape to ONS code %s" % ons_code
                poly.append(feat.geom)
                continue

            try:
                m = Area.objects.get(codes__type=code_type, codes__code=ons_code)
            except Area.DoesNotExist:
                m = Area(
                    type = Type.objects.get(code=area_code),
                    country = Country.objects.get(name='Scotland'),
                    generation_low = new_generation,
                    generation_high = new_generation,
                )

            if options['commit']:
                m.save()

            poly = [ feat.geom ]

            if options['commit']:
                m.names.update_or_create({ 'type': name_type }, { 'name': name })
            if ons_code:
                self.ons_code_to_shape[ons_code] = (m, poly)
                if options['commit']:
                    m.codes.update_or_create({ 'type': code_type }, { 'code': ons_code })

        if options['commit']:
            save_polygons(self.ons_code_to_shape)
コード例 #8
0
ファイル: mapit_import_osm.py プロジェクト: MorusAB/mapit
                    def update_or_create():
                        if osm_code:
                            m = osm_code.area
                        else:
                            m = Area(
                                name = name,
                                type = Type.objects.get(code=area_code),
                                country = Country.objects.get(code='G'),
                                parent_area = parent_area,
                                generation_low = new_generation,
                                generation_high = new_generation,
                            )

                        if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
                            raise Exception, "Area %s found, but not in current generation %s" % (m, current_generation)
                        m.generation_high = new_generation

                        g = feat.geom.transform(4326, clone=True)

                        # In generating the data we should have
                        # excluded any "polygons" with less than four
                        # points (the final one being the same as the
                        # first), but just in case:
                        for polygon in g:
                            if g.num_points < 4:
                                return

                        poly = [ g ]

                        if options['commit']:
                            m.save()

                            if name not in kml_data.data:
                                print json.dumps(kml_data.data, sort_keys=True, indent=4)
                                raise Exception, u"Will fail to find '%s' in the dictionary" % (name,)

                            for k, v in kml_data.data[name].items():
                                language_name = None
                                if k == 'name':
                                    lang = 'default'
                                    language_name = "OSM Default"
                                else:
                                    name_match = re.search(r'^name:(.+)$', k)
                                    if name_match:
                                        lang = name_match.group(1)
                                        if lang in language_code_to_name:
                                            language_name = language_code_to_name[lang]
                                if not language_name:
                                    continue
                                # Otherwise, make sure that a NameType for this language exists:
                                NameType.objects.update_or_create({'code': lang},
                                                                  {'code': lang,
                                                                   'description': language_name})
                                name_type = NameType.objects.get(code=lang)
                                m.names.update_or_create({ 'type': name_type }, { 'name': v })
                            m.codes.update_or_create({ 'type': code_type_osm }, { 'code': osm_id })
                            save_polygons({ code : (m, poly) })
コード例 #9
0
ファイル: mapit_UK_import_soa.py プロジェクト: Code4SA/mapit
    def handle_label(self, filename, **options):
        print(filename)
        generation = Generation.objects.current()

        short_filename = filename.split("/")[-1]
        filename_prefix = short_filename[:4]
        filename_suffix = short_filename.split(".")[0][-3:]

        # check shapefile type - we handle both LSOA and MSOA
        if filename_prefix == "LSOA":
            feat_name = 'LSOA04NM'
            feat_code = 'LSOA04CD'
            if filename_suffix == 'BGC':
                area_type = 'OLG'
            else:
                area_type = 'OLF'
        elif filename_prefix == "MSOA":
            feat_name = 'MSOA04NM'
            feat_code = 'MSOA04CD'
            if filename_suffix == 'BGC':
                area_type = 'OMG'
            else:
                area_type = 'OMF'
        else:
            raise Exception("Sorry, this script only handles LSOA/MSOA shapefiles!")

        ds = DataSource(filename)
        layer = ds[0]
        for feat in layer:
            # retrieve name and code, and set country
            name = feat[feat_name].value
            lsoa_code = feat[feat_code].value
            country = lsoa_code[0]
            # skip if the SOA already exists in db (SOAs don't change)
            if Area.objects.filter(type__code=area_type, codes__code=lsoa_code).count():
                continue
            print("Adding %s (%s) %s" % (name, lsoa_code, feat.geom.geom_name))
            m = Area(
                type=Type.objects.get(code=area_type),
                country=Country.objects.get(code=country),
                generation_low=generation,
                generation_high=generation,
            )
            m.save()
            m.names.update_or_create(type=NameType.objects.get(code='S'), defaults={'name': name})
            m.codes.update_or_create(type=CodeType.objects.get(code='ons'), defaults={'code': lsoa_code})

            p = feat.geom
            if p.geom_name == 'POLYGON':
                shapes = [p]
            else:
                shapes = p
            for g in shapes:
                m.polygons.create(polygon=g.wkt)
コード例 #10
0
 def _create(self, name, typ, geom, gss=None):
     m = Area(
         name=name, type=Type.objects.get(code=typ), country=self.country,
         generation_low=self.gn, generation_high=self.gn,
     )
     if self.commit:
         m.save()
         m.names.update_or_create(type=self.name_type, defaults={'name': name})
         if gss:
             m.codes.update_or_create(type=self.code_type, defaults={'code': gss})
         m.polygons.create(polygon=geom)
     else:
         print('Would create', name, typ)
コード例 #11
0
    def handle_label(self, filename, **options):
        code_version = CodeType.objects.get(code='gss')
        name_type = NameType.objects.get(code='O')

        # First, the easy ones
        for old, new in FIX_DIRECTLY.items():
            assert Area.objects.filter(codes__type=code_version, codes__code=new).count() == 0
            code = Code.objects.get(type=code_version, code=old)
            code.code = new
            print('Updating area %s to %s' % (old, new))
            if options['commit']:
                code.save()

        # Now, the harder ones
        previous_generation_low = {}
        previous_name = {}
        for old, new in FIX_WITH_OLD_BOUNDARY.items():
            if new != 'E05012199' or options['commit']:  # That one will exist unless it was updated above
                assert Area.objects.filter(codes__type=code_version, codes__code=new).count() == 0
            code = Code.objects.get(type=code_version, code=old)
            code.code = new
            previous_generation_low[old] = code.area.generation_low
            previous_name[old] = code.area.name
            code.area.generation_low = Generation.objects.current()
            print('Updating area %s to %s' % (old, new))
            if options['commit']:
                code.area.save()
                code.save()

        # Now we need to reinsert the October 2018 boundaries
        generation_high = Generation.objects.get(id=Generation.objects.current().id - 1)
        for feat in boundary_line_shapes(filename):
            ons_code = feat['CODE'].value
            if ons_code in FIX_WITH_OLD_BOUNDARY.keys():
                area_code = feat['AREA_CODE'].value
                country = Country.objects.get(code=ons_code[0])
                if options['commit']:  # They will be present if non-commit, they weren't changed above
                    assert Area.objects.filter(codes__type=code_version, codes__code=ons_code).count() == 0
                print('Creating new %s (%s), %s' % (ons_code, area_code, feat.name))

                m = Area(
                    type=Type.objects.get(code=area_code),
                    country=country,
                    generation_low=previous_generation_low[ons_code],
                    generation_high=generation_high,
                )
                if options['commit']:
                    m.save()
                    m.names.update_or_create(type=name_type, defaults={'name': feat.name})
                    m.codes.update_or_create(type=code_version, defaults={'code': ons_code})
                    save_polygons({ons_code: (m, [feat.geom])})
コード例 #12
0
    def handle_label(self, filename, **options):
        current_generation = Generation.objects.current()
        new_generation = Generation.objects.new()
        if not new_generation:
            raise Exception, "No new generation to be used for import!"

        code_type = CodeType.objects.get(code='n5000')
        name_type = NameType.objects.get(code='M')

        ds = DataSource(filename)
        layer = ds[0]
        for feat in layer:
            name = unicode(feat['NAVN'].value, 'iso-8859-1')
            name = re.sub('\s+', ' ', name)
            print " ", name

            code = feat['KOMM'].value
            code_str = '%04d' % code
            area_code = 'NKO'

            try:
                m = Area.objects.get(codes__type=code_type,
                                     codes__code=code_str)
            except Area.DoesNotExist:
                m = Area(
                    id=code,
                    type=Type.objects.get(code=area_code),
                    country=Country.objects.get(code='O'),
                    generation_low=new_generation,
                    generation_high=new_generation,
                )

            if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
                raise Exception, "Area %s found, but not in current generation %s" % (
                    m, current_generation)
            m.generation_high = new_generation

            g = feat.geom.transform(4326, clone=True)
            poly = [g]

            if options['commit']:
                m.save()
                m.names.update_or_create({'type': name_type}, {'name': name})
                m.codes.update_or_create({'type': code_type},
                                         {'code': code_str})
                save_polygons({code: (m, poly)})
コード例 #13
0
 def _create(self, name, typ, geom, gss=None):
     m = Area(
         name=name,
         type=Type.objects.get(code=typ),
         country=self.country,
         generation_low=self.gn,
         generation_high=self.gn,
     )
     if self.commit:
         m.save()
         m.names.update_or_create(type=self.name_type,
                                  defaults={'name': name})
         if gss:
             m.codes.update_or_create(type=self.code_type,
                                      defaults={'code': gss})
         m.polygons.create(polygon=geom)
     else:
         print('Would create', name, typ)
コード例 #14
0
    def make_new_area(self, name, ons_code, area_code, code_version, generation_low, generation_high, country):
        assert Area.objects.filter(codes__type=code_version, codes__code=ons_code).count() == 0
        print(ons_code, area_code, country, name)

        return Area(
            type=Type.objects.get(code=area_code),
            country=Country.objects.get(code=country),
            generation_low=Generation.objects.get(id=generation_low),
            generation_high=Generation.objects.get(id=generation_high),
        )
コード例 #15
0
    def handle_label(self,  filename, **options):
        current_generation = Generation.objects.current()
        new_generation = Generation.objects.new()
        if not new_generation:
            raise Exception, "No new generation to be used for import!"

        code_type = CodeType.objects.get(code='n5000')
        name_type = NameType.objects.get(code='M')

        ds = DataSource(filename)
        layer = ds[0]
        for feat in layer:
            name = unicode(feat['NAVN'].value, 'iso-8859-1')
            name = re.sub('\s+', ' ', name)
            print " ", name

            code = feat['KOMM'].value
            code_str = '%04d' % code
            area_code = 'NKO'
            
            try:
                m = Area.objects.get(codes__type=code_type, codes__code=code_str)
            except Area.DoesNotExist:
                m = Area(
                    id = code,
                    type = Type.objects.get(code=area_code),
                    country = Country.objects.get(code='O'),
                    generation_low = new_generation,
                    generation_high = new_generation,
                )

            if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
                raise Exception, "Area %s found, but not in current generation %s" % (m, current_generation)
            m.generation_high = new_generation

            g = feat.geom.transform(4326, clone=True)
            poly = [ g ]

            if options['commit']:
                m.save()
                m.names.update_or_create({ 'type': name_type }, { 'name': name })
                m.codes.update_or_create({ 'type': code_type }, { 'code': code_str })
                save_polygons({ code : (m, poly) })
コード例 #16
0
ファイル: import_area_unions.py プロジェクト: grischard/mapit
                def update_or_create():
                    try:
                        m = Area.objects.get(id=int(regionid))
                        print "Updating area %s with id %d" % (regionname, int(regionid))
                    except Area.DoesNotExist:
                        print "Creating new area %s with id %d" % (regionname, int(regionid))
                        m = Area(
                            id = int(regionid),
                            name = regionname,
                            type = Type.objects.get(code=area_type),
                            country = Country.objects.get(code='O'),
                            generation_low = new_generation,
                            generation_high = new_generation,
                            )

                    if m.generation_high and current_generation \
                            and m.generation_high.id < current_generation.id:
                        raise Exception, "Area %s found, but not in current generation %s" % (m, current_generation)
                    m.generation_high = new_generation

                    poly = [ GEOSGeometry(unionoutline).ogr ]
                    if options['commit']:
                        m.save()
                        save_polygons({ regionid : (m, poly) })
コード例 #17
0
    def handle_label(self, filename, **options):
        print(filename)
        new_generation = Generation.objects.new()
        if not new_generation:
            raise Exception("No new generation to be used for import!")

        name_type = NameType.objects.get(code='O')
        code_type = CodeType.objects.get(code='gss')

        ds = DataSource(filename)
        layer = ds[0]
        for feat in layer:
            name = feat['NAME'].value
            if not isinstance(name, six.text_type):
                name = name.decode('iso-8859-1')
            print("  %s" % name)
            name = re.sub(r'\s*\(DET( NO \d+|)\)\s*(?i)', '', name)
            name = re.sub(r'\s+', ' ', name)

            if "P Const" in name:
                area_code = 'SPC'
            elif "PER" in name:
                area_code = 'SPE'
            else:
                raise Exception("Unknown type of area %s" % name)

            ons_code = name_to_code[name]

            if ons_code in self.ons_code_to_shape:
                m, poly = self.ons_code_to_shape[ons_code]
                if options['commit']:
                    m_name = m.names.get(type=name_type).name
                    if name != m_name:
                        raise Exception("ONS code %s is used for %s and %s" %
                                        (ons_code, name, m_name))
                # Otherwise, combine the two shapes for one area
                print("    Adding subsequent shape to ONS code %s" % ons_code)
                poly.append(feat.geom)
                continue

            try:
                m = Area.objects.get(codes__type=code_type,
                                     codes__code=ons_code)
            except Area.DoesNotExist:
                m = Area(
                    type=Type.objects.get(code=area_code),
                    country=Country.objects.get(name='Scotland'),
                    generation_low=new_generation,
                    generation_high=new_generation,
                )

            if options['commit']:
                m.save()

            poly = [feat.geom]

            if options['commit']:
                m.names.update_or_create(type=name_type,
                                         defaults={'name': name})
            if ons_code:
                self.ons_code_to_shape[ons_code] = (m, poly)
                if options['commit']:
                    m.codes.update_or_create(type=code_type,
                                             defaults={'code': ons_code})

        if options['commit']:
            save_polygons(self.ons_code_to_shape)
コード例 #18
0
    def handle_label(self, directory_name, **options):
        current_generation = Generation.objects.current()
        new_generation = Generation.objects.new()
        if not new_generation:
            raise Exception("No new generation to be used for import!")

        if not os.path.isdir(directory_name):
            raise Exception("'%s' is not a directory" % (directory_name,))

        os.chdir(directory_name)

        mapit_type_glob = smart_text("[A-Z0-9][A-Z0-9][A-Z0-9]")

        if not glob(mapit_type_glob):
            raise Exception(
                "'%s' did not contain any directories that look like MapIt types (e.g. O11, OWA, etc.)" % (
                    directory_name,))

        def verbose(s):
            if int(options['verbosity']) > 1:
                print(smart_str(s))

        verbose("Loading any admin boundaries from " + directory_name)

        verbose("Finding language codes...")

        language_code_to_name = {}
        code_keys = ('two_letter', 'three_letter')
        for row in get_iso639_2_table():
            english_name = getattr(row, 'english_name')
            for k in code_keys:
                code = getattr(row, k)
                if not code:
                    continue
                language_code_to_name[code] = english_name

        global_country = Country.objects.get(code='G')

        # print json.dumps(language_code_to_name, sort_keys=True, indent=4)

        skip_up_to = None
        # skip_up_to = 'relation-80370'

        skipping = bool(skip_up_to)

        for type_directory in sorted(glob(mapit_type_glob)):

            verbose("Loading type " + type_directory)

            if not os.path.exists(type_directory):
                verbose("Skipping the non-existent " + type_directory)
                continue

            verbose("Loading all KML in " + type_directory)

            files = sorted(os.listdir(type_directory))
            total_files = len(files)

            for i, e in enumerate(files):

                progress = "[%d%% complete] " % ((i * 100) / total_files,)

                if skipping:
                    if skip_up_to in e:
                        skipping = False
                    else:
                        continue

                if not e.endswith('.kml'):
                    verbose("Ignoring non-KML file: " + e)
                    continue

                m = re.search(r'^(way|relation)-(\d+)-', e)
                if not m:
                    raise Exception("Couldn't extract OSM element type and ID from: " + e)

                osm_type, osm_id = m.groups()

                kml_filename = os.path.join(type_directory, e)

                verbose(progress + "Loading " + os.path.realpath(kml_filename))

                # Need to parse the KML manually to get the ExtendedData
                kml_data = KML()
                xml.sax.parse(smart_str(kml_filename), kml_data)

                useful_names = [n for n in kml_data.data.keys() if not n.startswith('Boundaries for')]
                if len(useful_names) == 0:
                    raise Exception("No useful names found in KML data")
                elif len(useful_names) > 1:
                    raise Exception("Multiple useful names found in KML data")
                name = useful_names[0]
                print(smart_str("  %s" % name))

                if osm_type == 'relation':
                    code_type_osm = CodeType.objects.get(code='osm_rel')
                elif osm_type == 'way':
                    code_type_osm = CodeType.objects.get(code='osm_way')
                else:
                    raise Exception("Unknown OSM element type: " + osm_type)

                ds = DataSource(kml_filename)
                layer = ds[0]
                if len(layer) != 1:
                    raise Exception("We only expect one feature in each layer")

                feat = layer[1]

                g = feat.geom.transform(4326, clone=True)

                if g.geom_count == 0:
                    # Just ignore any KML files that have no polygons in them:
                    verbose('    Ignoring that file - it contained no polygons')
                    continue

                # Nowadays, in generating the data we should have
                # excluded any "polygons" with less than four points
                # (the final one being the same as the first), but
                # just in case:
                polygons_too_small = 0
                for polygon in g:
                    if polygon.num_points < 4:
                        polygons_too_small += 1
                if polygons_too_small:
                    message = "%d out of %d polygon(s) were too small" % (polygons_too_small, g.geom_count)
                    verbose('    Skipping, since ' + message)
                    continue

                g_geos = g.geos

                if not g_geos.valid:
                    verbose("    Invalid KML:" + kml_filename)
                    fixed_multipolygon = fix_invalid_geos_multipolygon(g_geos)
                    if len(fixed_multipolygon) == 0:
                        verbose("    Invalid polygons couldn't be fixed")
                        continue
                    g = fixed_multipolygon.ogr

                area_type = Type.objects.get(code=type_directory)

                try:
                    osm_code = Code.objects.get(type=code_type_osm,
                                                code=osm_id,
                                                area__generation_high__lte=current_generation,
                                                area__generation_high__gte=current_generation)
                except Code.DoesNotExist:
                    verbose('    No area existed in the current generation with that OSM element type and ID')
                    osm_code = None

                was_the_same_in_current = False

                if osm_code:
                    m = osm_code.area

                    # First, we need to check if the polygons are
                    # still the same as in the previous generation:
                    previous_geos_geometry = m.polygons.aggregate(Collect('polygon'))['polygon__collect']
                    if previous_geos_geometry is None:
                        verbose('    In the current generation, that area was empty - skipping')
                    else:
                        # Simplify it to make sure the polygons are valid:
                        previous_geos_geometry = shapely.wkb.loads(
                            str(previous_geos_geometry.simplify(tolerance=0).ewkb))
                        new_geos_geometry = shapely.wkb.loads(str(g.geos.simplify(tolerance=0).ewkb))
                        if previous_geos_geometry.almost_equals(new_geos_geometry, decimal=7):
                            was_the_same_in_current = True
                        else:
                            verbose('    In the current generation, the boundary was different')

                if was_the_same_in_current:
                    # Extend the high generation to the new one:
                    verbose('    The boundary was identical in the previous generation; raising generation_high')
                    m.generation_high = new_generation

                else:
                    # Otherwise, create a completely new area:
                    m = Area(
                        name=name,
                        type=area_type,
                        country=global_country,
                        parent_area=None,
                        generation_low=new_generation,
                        generation_high=new_generation,
                    )

                poly = [g]

                if options['commit']:
                    m.save()
                    verbose('    Area ID: ' + str(m.id))

                    if name not in kml_data.data:
                        print(json.dumps(kml_data.data, sort_keys=True, indent=4))
                        raise Exception("Will fail to find '%s' in the dictionary" % (name,))

                    old_lang_codes = set(n.type.code for n in m.names.all())

                    for k, translated_name in kml_data.data[name].items():
                        language_name = None
                        if k == 'name':
                            lang = 'default'
                            language_name = "OSM Default"
                        else:
                            name_match = re.search(r'^name:(.+)$', k)
                            if name_match:
                                lang = name_match.group(1)
                                if lang in language_code_to_name:
                                    language_name = language_code_to_name[lang]
                        if not language_name:
                            continue
                        old_lang_codes.discard(lang)

                        # Otherwise, make sure that a NameType for this language exists:
                        NameType.objects.update_or_create(code=lang, defaults={'description': language_name})
                        name_type = NameType.objects.get(code=lang)

                        m.names.update_or_create(type=name_type, defaults={'name': translated_name})

                    if old_lang_codes:
                        verbose('Removing deleted languages codes: ' + ' '.join(old_lang_codes))
                    m.names.filter(type__code__in=old_lang_codes).delete()
                    # If the boundary was the same, the old Code
                    # object will still be pointing to the same Area,
                    # which just had its generation_high incremented.
                    # In every other case, there's a new area object,
                    # so create a new Code and save it:
                    if not was_the_same_in_current:
                        new_code = Code(area=m, type=code_type_osm, code=osm_id)
                        new_code.save()
                    save_polygons({'dummy': (m, poly)})
コード例 #19
0
ファイル: import_kml.py プロジェクト: grischard/mapit
    def handle_label(self, filename, **options):

        for k in ['generation_id','area_type_code','name_type_code','country_code']:
            if options[k]: continue
            raise Exception("Missing argument '--%s'" % k)

        generation_id  = options['generation_id']
        area_type_code = options['area_type_code']
        name_type_code = options['name_type_code']
        country_code   = options['country_code']

        area_type = Type.objects.get(code=area_type_code)
        name_type = NameType.objects.get(code=name_type_code)
        country   = Country.objects.get(code=country_code)

        print "Importing from %s" % filename
        
        if not options['commit']:
            print '(will not save to db as --commit not specified)'            

        current_generation = Generation.objects.current()
        new_generation     = Generation.objects.get( id=generation_id )

        # Need to parse the KML manually to get the ExtendedData
        kml_data = KML()
        xml.sax.parse(filename, kml_data)
        
        

        ds = DataSource(filename)
        layer = ds[0]
        for feat in layer:

            name = feat['Name'].value.decode('utf-8')
            name = re.sub('\s+', ' ', name)
            
            if not name:
                raise Exception( "Could not find a name to use for area" )
                
            print "  looking at '%s'" % name.encode('utf-8')

            try:
                m = Area.objects.get(name=name, type=area_type)
            except Area.DoesNotExist:
                m = Area(
                    name            = name,
                    type            = area_type,
                    country         = country,
                    # parent_area     = parent_area,
                    generation_low  = new_generation,
                    generation_high = new_generation,
                )

            # check that we are not about to skip a generation
            if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
                raise Exception, "Area %s found, but not in current generation %s" % (m, current_generation)
            m.generation_high = new_generation
            
            g = feat.geom.transform(4326, clone=True)
            
            poly = [ g ]
            
            if options['commit']:
                m.save()
                m.names.update_or_create({ 'type': name_type }, { 'name': name })
                save_polygons({ m.id : (m, poly) })
コード例 #20
0
    def handle_label(self, filename, **options):
        if not options['control']:
            raise Exception("You must specify a control file")
        __import__(options['control'])
        control = sys.modules[options['control']]

        code_version = CodeType.objects.get(code='gss')
        name_type = NameType.objects.get(code='O')
        code_type_os = CodeType.objects.get(code='unit_id')

        print(filename)
        current_generation = Generation.objects.current()
        new_generation = Generation.objects.new()
        if not new_generation:
            raise Exception("No new generation to be used for import!")

        ds = DataSource(filename)
        layer = ds[0]
        for feat in layer:
            name = feat['NAME'].value
            if not isinstance(name, six.text_type):
                name = name.decode('iso-8859-1')

            name = re.sub(r'\s*\(DET( NO \d+|)\)\s*(?i)', '', name)
            name = re.sub(r'\s+', ' ', name)

            ons_code = feat['CODE'].value if feat['CODE'].value not in ('999999', '999999999') else None
            unit_id = str(feat['UNIT_ID'].value)
            area_code = feat['AREA_CODE'].value
            patch = self.patch_boundary_line(name, ons_code, unit_id, area_code)
            if 'ons-code' in patch:
                ons_code = patch['ons-code']
            elif 'unit-id' in patch:
                unit_id = patch['unit-id']

            if area_code == 'NCP':
                continue  # Ignore Non Parished Areas

            if ons_code in self.ons_code_to_shape:
                m, poly = self.ons_code_to_shape[ons_code]
                try:
                    m_name = m.names.get(type=name_type).name
                except Name.DoesNotExist:
                    m_name = m.name  # If running without commit for dry run, so nothing being stored in db
                if name != m_name:
                    raise Exception("ONS code %s is used for %s and %s" % (ons_code, name, m_name))
                # Otherwise, combine the two shapes for one area
                poly.append(feat.geom)
                continue

            if unit_id in self.unit_id_to_shape:
                m, poly = self.unit_id_to_shape[unit_id]
                try:
                    m_name = m.names.get(type=name_type).name
                except Name.DoesNotExist:
                    m_name = m.name  # If running without commit for dry run, so nothing being stored in db
                if name != m_name:
                    raise Exception("Unit ID code %s is used for %s and %s" % (unit_id, name, m_name))
                # Otherwise, combine the two shapes for one area
                poly.append(feat.geom)
                continue

            if ons_code:
                country = ons_code[0]  # Hooray!
            elif area_code in ('CED', 'CTY', 'DIW', 'DIS', 'MTW', 'MTD', 'LBW', 'LBO', 'LAC', 'GLA'):
                country = 'E'
            else:
                raise Exception(area_code)

            try:
                check = control.check(name, area_code, country, feat.geom, ons_code=ons_code, commit=options['commit'])
                if check is True:
                    raise Area.DoesNotExist
                if isinstance(check, Area):
                    m = check
                    try:
                        ons_code = m.codes.get(type=code_version).code
                    except Code.DoesNotExist:
                        ons_code = None
                elif ons_code:
                    m = Area.objects.get(codes__type=code_version, codes__code=ons_code)
                elif unit_id:
                    m = Area.objects.get(
                        codes__type=code_type_os, codes__code=unit_id, generation_high=current_generation)
                    m_name = m.names.get(type=name_type).name
                    if name != m_name:
                        raise Exception("Unit ID code %s is %s in DB but %s in SHP file" % (unit_id, m_name, name))
                else:
                    raise Exception('Area "%s" (%s) has neither ONS code nor unit ID' % (name, area_code))
                if int(options['verbosity']) > 1:
                    print("  Area matched, %s" % (m, ))
            except Area.DoesNotExist:
                print("  New area: %s %s %s %s" % (area_code, ons_code, unit_id, name))
                m = Area(
                    name=name,  # If committing, this will be overwritten by the m.names.update_or_create
                    type=Type.objects.get(code=area_code),
                    country=Country.objects.get(code=country),
                    generation_low=new_generation,
                    generation_high=new_generation,
                )

            if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
                raise Exception("Area %s found, but not in current generation %s" % (m, current_generation))
            m.generation_high = new_generation
            if options['commit']:
                m.save()

            # Make a GEOS geometry only to check for validity:
            g = feat.geom
            geos_g = g.geos
            if not geos_g.valid:
                print("  Geometry of %s %s not valid" % (ons_code, m))
                geos_g = fix_invalid_geos_geometry(geos_g)
                if geos_g is None:
                    raise Exception("The geometry for area %s was invalid and couldn't be fixed" % name)
                    g = None
                else:
                    g = geos_g.ogr

            poly = [g]

            if options['commit']:
                m.names.update_or_create(type=name_type, defaults={'name': name})
            if ons_code:
                self.ons_code_to_shape[ons_code] = (m, poly)
                if options['commit']:
                    m.codes.update_or_create(type=code_version, defaults={'code': ons_code})
            if unit_id:
                self.unit_id_to_shape[unit_id] = (m, poly)
                if options['commit']:
                    m.codes.update_or_create(type=code_type_os, defaults={'code': unit_id})

        if options['commit']:
            save_polygons(self.unit_id_to_shape)
            save_polygons(self.ons_code_to_shape)
コード例 #21
0
    def handle_label(self, filename, **options):

        err = False
        for k in [
                'generation_id', 'area_type_code', 'name_type_code',
                'country_code'
        ]:
            if options[k]: continue
            print "Missing argument '--%s'" % k
            err = True
        if err:
            sys.exit(1)

        generation_id = options['generation_id']
        area_type_code = options['area_type_code']
        name_type_code = options['name_type_code']
        country_code = options['country_code']
        name_field = options['name_field'] or 'Name'
        code_field = options['code_field']
        code_type_code = options['code_type']
        encoding = options['encoding'] or 'utf-8'

        if len(area_type_code) > 3:
            print "Area type code must be 3 letters or fewer, sorry"
            sys.exit(1)

        if (code_field and not code_type_code) or (not code_field
                                                   and code_type_code):
            print "You must specify both code_field and code_type, or neither."
            sys.exit(1)
        try:
            area_type = Type.objects.get(code=area_type_code)
        except:
            type_desc = raw_input(
                'Please give a description for area type code %s: ' %
                area_type_code)
            area_type = Type(code=area_type_code, description=type_desc)
            if options['commit']: area_type.save()

        try:
            name_type = NameType.objects.get(code=name_type_code)
        except:
            name_desc = raw_input(
                'Please give a description for name type code %s: ' %
                name_type_code)
            name_type = NameType(code=name_type_code, description=name_desc)
            if options['commit']: name_type.save()

        try:
            country = Country.objects.get(code=country_code)
        except:
            country_name = raw_input(
                'Please give the name for country code %s: ' % country_code)
            country = Country(code=country_code, name=country_name)
            if options['commit']: country.save()

        if code_type_code:
            try:
                code_type = CodeType.objects.get(code=code_type_code)
            except:
                code_desc = raw_input(
                    'Please give a description for code type %s: ' %
                    code_type_code)
                code_type = CodeType(code=code_type_code,
                                     description=code_desc)
                if options['commit']: code_type.save()

        print "Importing from %s" % filename

        if not options['commit']:
            print '(will not save to db as --commit not specified)'

        current_generation = Generation.objects.current()
        new_generation = Generation.objects.get(id=generation_id)

        ds = DataSource(filename)
        layer = ds[0]
        for feat in layer:

            try:
                name = feat[name_field].value
            except:
                choices = ', '.join(layer.fields)
                print "Could not find name using name field '%s' - should it be something else? It will be one of these: %s. Specify which with --name_field" % (
                    name_field, choices)
                sys.exit(1)
            try:
                name = name.decode(encoding)
            except:
                print "Could not decode name using encoding '%s' - is it in another encoding? Specify one with --encoding" % encoding
                sys.exit(1)

            name = re.sub('\s+', ' ', name)
            if not name:
                raise Exception("Could not find a name to use for area")

            code = None
            if code_field:
                try:
                    code = feat[code_field].value
                except:
                    choices = ', '.join(layer.fields)
                    print "Could not find code using code field '%s' - should it be something else? It will be one of these: %s. Specify which with --code_field" % (
                        code_field, choices)
                    sys.exit(1)

            print "  looking at '%s'%s" % (name.encode('utf-8'),
                                           (' (%s)' % code) if code else '')

            try:
                if code:
                    m = Area.objects.get(codes__code=code,
                                         codes__type=code_type)
                else:
                    # Assumes unique names if no code column used
                    m = Area.objects.get(name=name, type=area_type)
            except Area.DoesNotExist:
                m = Area(
                    name=name,
                    type=area_type,
                    country=country,
                    # parent_area     = parent_area,
                    generation_low=new_generation,
                    generation_high=new_generation,
                )
                if options['use_code_as_id'] and code:
                    m.id = int(code)

            # check that we are not about to skip a generation
            if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
                raise Exception, "Area %s found, but not in current generation %s" % (
                    m, current_generation)
            m.generation_high = new_generation

            g = feat.geom.transform(4326, clone=True)

            poly = [g]

            if options['commit']:
                m.save()
                m.names.update_or_create({'type': name_type}, {'name': name})
                if code:
                    m.codes.update_or_create({'type': code_type},
                                             {'code': code})
                save_polygons({m.id: (m, poly)})
コード例 #22
0
ファイル: mapit_import.py プロジェクト: mabeltron/mapit
    def handle_label(self, filename, **options):

        missing_options = []
        for k in [
                'generation_id', 'area_type_code', 'name_type_code',
                'country_code'
        ]:
            if options[k]:
                continue
            else:
                missing_options.append(k)
        if missing_options:
            message_start = "Missing arguments " if len(
                missing_options) > 1 else "Missing argument "
            message = message_start + " ".join('--{0}'.format(k)
                                               for k in missing_options)
            raise CommandError(message)

        generation_id = options['generation_id']
        area_type_code = options['area_type_code']
        name_type_code = options['name_type_code']
        country_code = options['country_code']
        override_name = options['override_name']
        name_field = options['name_field']
        if not (override_name or name_field):
            name_field = 'Name'
        override_code = options['override_code']
        code_field = options['code_field']
        code_type_code = options['code_type']
        encoding = options['encoding'] or 'utf-8'

        if len(area_type_code) > 3:
            raise CommandError(
                "Area type code must be 3 letters or fewer, sorry")

        if name_field and override_name:
            raise CommandError(
                "You must not specify both --name_field and --override_name")
        if code_field and override_code:
            raise CommandError(
                "You must not specify both --code_field and --override_code")

        using_code = (code_field or override_code)
        if (using_code and not code_type_code) or (not using_code
                                                   and code_type_code):
            raise CommandError(
                "If you want to save a code, specify --code_type and either --code_field or --override_code"
            )
        try:
            area_type = Type.objects.get(code=area_type_code)
        except:
            type_desc = raw_input(
                'Please give a description for area type code %s: ' %
                area_type_code)
            area_type = Type(code=area_type_code, description=type_desc)
            if options['commit']: area_type.save()

        try:
            name_type = NameType.objects.get(code=name_type_code)
        except:
            name_desc = raw_input(
                'Please give a description for name type code %s: ' %
                name_type_code)
            name_type = NameType(code=name_type_code, description=name_desc)
            if options['commit']: name_type.save()

        try:
            country = Country.objects.get(code=country_code)
        except:
            country_name = raw_input(
                'Please give the name for country code %s: ' % country_code)
            country = Country(code=country_code, name=country_name)
            if options['commit']: country.save()

        if code_type_code:
            try:
                code_type = CodeType.objects.get(code=code_type_code)
            except:
                code_desc = raw_input(
                    'Please give a description for code type %s: ' %
                    code_type_code)
                code_type = CodeType(code=code_type_code,
                                     description=code_desc)
                if options['commit']: code_type.save()

        self.stdout.write("Importing from %s" % filename)

        if not options['commit']:
            self.stdout.write(
                '(will not save to db as --commit not specified)')

        current_generation = Generation.objects.current()
        new_generation = Generation.objects.get(id=generation_id)

        def verbose(*args):
            if int(options['verbosity']) > 1:
                self.stdout.write(" ".join(str(a) for a in args))

        ds = DataSource(filename)
        layer = ds[0]
        if (override_name or override_code) and len(layer) > 1:
            message = "Warning: you have specified an override %s and this file contains more than one feature; multiple areas with the same %s will be created"
            if override_name:
                self.stdout.write(message % ('name', 'name'))
            if override_code:
                self.stdout.write(message % ('code', 'code'))

        for feat in layer:

            if override_name:
                name = override_name
            else:
                try:
                    name = feat[name_field].value
                except:
                    choices = ', '.join(layer.fields)
                    raise CommandError(
                        "Could not find name using name field '%s' - should it be something else? It will be one of these: %s. Specify which with --name_field"
                        % (name_field, choices))
                try:
                    if not isinstance(name, unicode):
                        name = name.decode(encoding)
                except:
                    raise CommandError(
                        "Could not decode name using encoding '%s' - is it in another encoding? Specify one with --encoding"
                        % encoding)

            name = re.sub('\s+', ' ', name)
            if not name:
                raise Exception("Could not find a name to use for area")

            code = None
            if override_code:
                code = override_code
            elif code_field:
                try:
                    code = feat[code_field].value
                except:
                    choices = ', '.join(layer.fields)
                    raise CommandError(
                        "Could not find code using code field '%s' - should it be something else? It will be one of these: %s. Specify which with --code_field"
                        % (code_field, choices))

            self.stdout.write("  looking at '%s'%s" %
                              (name.encode('utf-8'),
                               (' (%s)' % code) if code else ''))

            g = None
            if hasattr(feat, 'geom'):
                g = feat.geom.transform(settings.MAPIT_AREA_SRID, clone=True)

            try:
                if options['new']:  # Always want a new area
                    raise Area.DoesNotExist
                if code:
                    matching_message = "code %s of code type %s" % (code,
                                                                    code_type)
                    areas = Area.objects.filter(
                        codes__code=code,
                        codes__type=code_type).order_by('-generation_high')
                else:
                    matching_message = "name %s of area type %s" % (name,
                                                                    area_type)
                    areas = Area.objects.filter(
                        name=name, type=area_type).order_by('-generation_high')
                if len(areas) == 0:
                    verbose("    the area was not found - creating a new one")
                    raise Area.DoesNotExist
                m = areas[0]
                verbose("    found the area")
                if options['preserve']:
                    # Find whether we need to create a new Area:
                    previous_geos_geometry = m.polygons.collect()
                    if m.generation_high < current_generation.id:
                        # Then it was missing in current_generation:
                        verbose(
                            "    area existed previously, but was missing from",
                            current_generation)
                        raise Area.DoesNotExist
                    elif g is None:
                        if previous_geos_geometry is not None:
                            verbose("    area is now empty")
                            raise Area.DoesNotExist
                        else:
                            verbose("    the area has remained empty")
                    elif previous_geos_geometry is None:
                        # It was empty in the previous generation:
                        verbose("    area was empty in", current_generation)
                        raise Area.DoesNotExist
                    else:
                        # Otherwise, create a new Area unless the
                        # polygons were the same in current_generation:
                        previous_geos_geometry = previous_geos_geometry.simplify(
                            tolerance=0)
                        new_geos_geometry = g.geos.simplify(tolerance=0)
                        create_new_area = not previous_geos_geometry.equals(
                            new_geos_geometry)
                        p = previous_geos_geometry.sym_difference(
                            new_geos_geometry
                        ).area / previous_geos_geometry.area
                        verbose("    change in area is:",
                                "%.03f%%" % (100 * p, ))
                        if create_new_area:
                            verbose(
                                "    the area", m,
                                "has changed, creating a new area due to --preserve"
                            )
                            raise Area.DoesNotExist
                        else:
                            verbose("    the area remained the same")
                else:
                    # If --preserve is not specified, the code or the name must be unique:
                    if len(areas) > 1:
                        raise Area.MultipleObjectsReturned, "There was more than one area with %s, and --preserve was not specified" % (
                            matching_message, )

            except Area.DoesNotExist:
                m = Area(
                    name=name,
                    type=area_type,
                    country=country,
                    # parent_area     = parent_area,
                    generation_low=new_generation,
                    generation_high=new_generation,
                )
                if options['use_code_as_id'] and code:
                    m.id = int(code)

            # check that we are not about to skip a generation
            if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
                raise Exception, "Area %s found, but not in current generation %s" % (
                    m, current_generation)
            m.generation_high = new_generation

            if options['fix_invalid_polygons'] and g is not None:
                # Make a GEOS geometry only to check for validity:
                geos_g = g.geos
                if not geos_g.valid:
                    geos_g = fix_invalid_geos_geometry(geos_g)
                    if geos_g is None:
                        self.stdout.write(
                            "The geometry for area %s was invalid and couldn't be fixed"
                            % name)
                        g = None
                    else:
                        g = geos_g.ogr

            poly = [g] if g is not None else []

            if options['commit']:
                m.save()
                m.names.update_or_create({'type': name_type}, {'name': name})
                if code:
                    m.codes.update_or_create({'type': code_type},
                                             {'code': code})
                save_polygons({m.id: (m, poly)})
コード例 #23
0
    def process_file(self, filename, area_code, srid, control, options):
        code_version = CodeType.objects.get(code='gss')
        name_type = NameType.objects.get(code='N')
        code_type_osni = CodeType.objects.get(code='osni_oid')
        if not hasattr(self, area_code):
            raise Exception(
                "Don't know how to extract features from %s files" % area_code)

        area_code_info = getattr(self, area_code)(srid)

        print(filename)
        current_generation = Generation.objects.current()
        new_generation = Generation.objects.new()
        if not new_generation:
            raise Exception("No new generation to be used for import!")

        ds = DataSource(filename)
        layer = ds[0]

        for feat in layer:
            name, ons_code, osni_object_id = area_code_info.extract_fields(
                feat)
            name = self.format_name(name)
            if ons_code in self.ons_code_to_shape:
                m, poly = self.ons_code_to_shape[ons_code]
                try:
                    m_name = m.names.get(type=name_type).name
                except Name.DoesNotExist:
                    m_name = m.name  # If running without commit for dry run, so nothing being stored in db
                if name != m_name:
                    raise Exception("ONS code %s is used for %s and %s" %
                                    (ons_code, name, m_name))
                # Otherwise, combine the two shapes for one area
                poly.append(feat.geom)
                continue

            if osni_object_id in self.osni_object_id_to_shape:
                m, poly = self.osni_object_id_to_shape[osni_object_id]
                try:
                    m_name = m.names.get(type=name_type).name
                except Name.DoesNotExist:
                    m_name = m.name  # If running without commit for dry run, so nothing being stored in db
                if name != m_name:
                    raise Exception(
                        "OSNI Object ID code %s is used for %s and %s" %
                        (osni_object_id, name, m_name))
                # Otherwise, combine the two shapes for one area
                poly.append(feat.geom)
                continue

            country = 'N'

            try:
                check = control.check(name, area_code, country, feat.geom)
                if check is True:
                    raise Area.DoesNotExist
                if isinstance(check, Area):
                    m = check
                    ons_code = m.codes.get(type=code_version).code
                elif ons_code:
                    m = Area.objects.get(codes__type=code_version,
                                         codes__code=ons_code)
                elif osni_object_id:
                    m = Area.objects.get(codes__type=code_type_osni,
                                         codes__code=osni_object_id,
                                         generation_high=current_generation)
                    m_name = m.names.get(type=name_type).name
                    if name != m_name:
                        raise Exception(
                            "OSNI Object ID code %s is %s in DB but %s in SHP file"
                            % (osni_object_id, m_name, name))
                else:
                    raise Exception(
                        'Area "%s" (%s) has neither ONS code nor OSNI Object ID'
                        % (name, area_code))
                if int(options['verbosity']) > 1:
                    print("  Area matched, %s" % (m, ))
            except Area.DoesNotExist:
                area_type = Type.objects.get(code=area_code)
                # It's possible we already have NIE entries (without any codes) in the db
                try:
                    if area_code == 'NIE':
                        matching_name = name.title().replace('And', 'and')
                        m = Area.objects.get(
                            name=matching_name,
                            type=area_type,
                            generation_high=current_generation)
                        if int(options['verbosity']) > 1:
                            print("  Area matched (via name), %s" % (m, ))
                    else:
                        raise Area.DoesNotExist("Still doesn't exist")
                except Area.DoesNotExist:
                    print("  New area: %s %s %s %s" %
                          (area_code, ons_code, osni_object_id, name))
                    m = Area(
                        name=
                        name,  # Not overwritten by m.names.update_or_create as no "N" use
                        type=area_type,
                        country=Country.objects.get(code=country),
                        generation_low=new_generation,
                        generation_high=new_generation,
                    )

            if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
                raise Exception(
                    "Area %s found, but not in current generation %s" %
                    (m, current_generation))
            m.generation_high = new_generation
            if options['commit']:
                m.save()

            # Make a GEOS geometry only to check for validity:
            g = feat.geom
            geos_g = g.geos
            if not geos_g.valid:
                print("  Geometry of %s %s not valid" % (ons_code, m))
                geos_g = fix_invalid_geos_geometry(geos_g)
                if geos_g is None:
                    raise Exception(
                        "The geometry for area %s was invalid and couldn't be fixed"
                        % name)
                    g = None
                else:
                    g = geos_g.ogr

            g = area_code_info.transform_geom(g)
            poly = [g]

            if options['commit']:
                m.names.update_or_create(type=name_type,
                                         defaults={'name': name})
            if ons_code:
                self.ons_code_to_shape[ons_code] = (m, poly)
                if options['commit']:
                    m.codes.update_or_create(type=code_version,
                                             defaults={'code': ons_code})
            if osni_object_id:
                self.osni_object_id_to_shape[osni_object_id] = (m, poly)
                if options['commit']:
                    m.codes.update_or_create(type=code_type_osni,
                                             defaults={'code': osni_object_id})

        if options['commit']:
            save_polygons(self.osni_object_id_to_shape)
            save_polygons(self.ons_code_to_shape)
コード例 #24
0
ファイル: mapit_import.py プロジェクト: AltisCorp/mapit
    def handle_label(self, filename, **options):

        err = False
        for k in ['generation_id','area_type_code','name_type_code','country_code']:
            if options[k]: continue
            print "Missing argument '--%s'" % k
            err = True
        if err:
            sys.exit(1)

        generation_id = options['generation_id']
        area_type_code = options['area_type_code']
        name_type_code = options['name_type_code']
        country_code = options['country_code']
        name_field = options['name_field'] or 'Name'
        code_field = options['code_field']
        code_type_code = options['code_type']
        encoding = options['encoding'] or 'utf-8'

        if len(area_type_code)>3:
            print "Area type code must be 3 letters or fewer, sorry"
            sys.exit(1)

        if (code_field and not code_type_code) or (not code_field and code_type_code):
            print "You must specify both code_field and code_type, or neither."
            sys.exit(1)
        try:
            area_type = Type.objects.get(code=area_type_code)
        except:
            type_desc = raw_input('Please give a description for area type code %s: ' % area_type_code)
            area_type = Type(code=area_type_code, description=type_desc)
            if options['commit']: area_type.save()

        try:
            name_type = NameType.objects.get(code=name_type_code)
        except:
            name_desc = raw_input('Please give a description for name type code %s: ' % name_type_code)
            name_type = NameType(code=name_type_code, description=name_desc)
            if options['commit']: name_type.save()

        try:
            country = Country.objects.get(code=country_code)
        except:
            country_name = raw_input('Please give the name for country code %s: ' % country_code)
            country = Country(code=country_code, name=country_name)
            if options['commit']: country.save()

        if code_type_code:
            try:
                code_type = CodeType.objects.get(code=code_type_code)
            except:
                code_desc = raw_input('Please give a description for code type %s: ' % code_type_code)
                code_type = CodeType(code=code_type_code, description=code_desc)
                if options['commit']: code_type.save()

        print "Importing from %s" % filename

        if not options['commit']:
            print '(will not save to db as --commit not specified)'

        current_generation = Generation.objects.current()
        new_generation     = Generation.objects.get( id=generation_id )

        def verbose(*args):
            if options['verbose']:
                print " ".join(str(a) for a in args)

        ds = DataSource(filename)
        layer = ds[0]
        for feat in layer:

            try:
                name = feat[name_field].value
            except:
                choices = ', '.join(layer.fields)
                print "Could not find name using name field '%s' - should it be something else? It will be one of these: %s. Specify which with --name_field" % (name_field, choices)
                sys.exit(1)
            try:
                name = name.decode(encoding)
            except:
                print "Could not decode name using encoding '%s' - is it in another encoding? Specify one with --encoding" % encoding
                sys.exit(1)

            name = re.sub('\s+', ' ', name)
            if not name:
                raise Exception( "Could not find a name to use for area" )

            code = None
            if code_field:
                try:
                    code = feat[code_field].value
                except:
                    choices = ', '.join(layer.fields)
                    print "Could not find code using code field '%s' - should it be something else? It will be one of these: %s. Specify which with --code_field" % (code_field, choices)
                    sys.exit(1)

            print "  looking at '%s'%s" % ( name.encode('utf-8'), (' (%s)' % code) if code else '' )

            g = feat.geom.transform(4326, clone=True)

            try:
                if code:
                    matching_message = "code %s of code type %s" % (code, code_type)
                    areas = Area.objects.filter(codes__code=code, codes__type=code_type).order_by('-generation_high')
                else:
                    matching_message = "name %s of area type %s" % (name, area_type)
                    areas = Area.objects.filter(name=name, type=area_type).order_by('-generation_high')
                if len(areas) == 0:
                    raise Area.DoesNotExist
                m = areas[0]
                verbose("    found the area")
                if options['preserve']:
                    # Find whether we need to create a new Area:
                    previous_geos_geometry = m.polygons.collect()
                    if m.generation_high < current_generation.id:
                        # Then it was missing in current_generation:
                        verbose("    area existed previously, but was missing from", current_generation)
                        create_new_area = True
                    elif previous_geos_geometry is None:
                        # It was empty in the previous generation:
                        verbose("    area was empty in", current_generation)
                        create_new_area = True
                    else:
                        # Otherwise, create a new Area unless the
                        # polygons were the same in current_generation:
                        previous_geos_geometry = previous_geos_geometry.simplify(tolerance=0)
                        new_geos_geometry = g.geos.simplify(tolerance=0)
                        create_new_area = not previous_geos_geometry.equals(new_geos_geometry)
                        p = previous_geos_geometry.sym_difference(new_geos_geometry).area / previous_geos_geometry.area
                        verbose("    change in area is:", "%.03f%%" % (100 * p,))
                        if create_new_area:
                            verbose("    the area", m, "has changed, creating a new area due to --preserve")
                        else:
                            verbose("    the area remained the same")
                    if create_new_area:
                        m = Area(
                            name            = name,
                            type            = area_type,
                            country         = country,
                            # parent_area     = parent_area,
                            generation_low  = new_generation,
                            generation_high = new_generation
                        )
                        if options['use_code_as_id'] and code:
                            m.id = int(code)
                else:
                    # If --preserve is not specified, the code or the name must be unique:
                    if len(areas) > 1:
                        raise Area.MultipleObjectsReturned, "There was more than one area with %s, and --preserve was not specified" % (matching_message,)

            except Area.DoesNotExist:
                verbose("    the area was not found - creating a new one")
                m = Area(
                    name            = name,
                    type            = area_type,
                    country         = country,
                    # parent_area     = parent_area,
                    generation_low  = new_generation,
                    generation_high = new_generation,
                )
                if options['use_code_as_id'] and code:
                    m.id = int(code)

            # check that we are not about to skip a generation
            if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
                raise Exception, "Area %s found, but not in current generation %s" % (m, current_generation)
            m.generation_high = new_generation

            poly = [ g ]

            if options['commit']:
                m.save()
                m.names.update_or_create({ 'type': name_type }, { 'name': name })
                if code:
                    m.codes.update_or_create({ 'type': code_type }, { 'code': code })
                save_polygons({ m.id : (m, poly) })
コード例 #25
0
    def handle_label(self,  filename, **options):
        country = Country.objects.get(code='N')
        oa_type = Type.objects.get(code='OUA')
        soa_type = Type.objects.get(code='OLF')
        name_type = NameType.objects.get(code='S')
        code_type = CodeType.objects.get(code='ons')

        current_generation = Generation.objects.current()
        new_generation = Generation.objects.new()
        if not new_generation:
            raise Exception, "No new generation to be used for import!"

        # Compile an alphabetical list of NI councils and their wards, OA codes
        # are assigned alphabetically.
        if not self.councils:
            self.councils = Area.objects.filter(type=Type.objects.get(code='LGD')).order_by('name').values()
            for lgd in self.councils:
                lges = Area.objects.filter(parent_area=lgd['id'])
                areas = []
                for lge in lges:
                    lgws = Area.objects.filter(parent_area=lge).values()
                    areas += lgws
                lgd['wards'] = sorted(areas, key=lambda x: x['name'])

        ds = DataSource(filename)
        layer = ds[0]
        layer_name = str(layer)
        for feat in layer:
            if layer_name == 'soa':
                area_type = soa_type
                ons_code = feat['SOA_CODE'].value
                name = feat['SOA_LABEL'].value.replace('_', ' ')
            elif layer_name == 'OA_ni':
                area_type = oa_type
                ons_code = feat['OA_CODE'].value
                name = 'Output Area %s' % ons_code
            else:
                raise Exception, 'Bad data passed in'

            council = ord(ons_code[2:3]) - 65
            ward = int(ons_code[4:6]) - 1
            if ward == 98: # SOA covers two wards, set parent to council, best we can do
                parent = self.councils[council]['id']
            else:
                parent = self.councils[council]['wards'][ward]['id']

            try:
                m = Area.objects.get(codes__type=code_type, codes__code=ons_code)
                if int(options['verbosity']) > 1:
                    print "  Area matched, %s" % (m, )
            except Area.DoesNotExist:
                print "  New area: %s" % (ons_code)
                m = Area(
                    name = name, # If committing, this will be overwritten by the m.names.update_or_create
                    type = area_type,
                    country = country,
                    parent_area_id = parent,
                    generation_low = new_generation,
                    generation_high = new_generation,
                )

            if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
                raise Exception, "Area %s found, but not in current generation %s" % (m, current_generation)
            m.generation_high = new_generation
            m.parent_area_id = parent
            if options['commit']:
                m.save()

            f = feat.geom
            f.srid = 29902
            poly = [ f ]

            if options['commit']:
                m.names.update_or_create({ 'type': name_type }, { 'name': name })
            if ons_code:
                self.ons_code_to_shape[ons_code] = (m, poly)
                if options['commit']:
                    m.codes.update_or_create({ 'type': code_type }, { 'code': ons_code })

        if options['commit']:
            save_polygons(self.ons_code_to_shape)
コード例 #26
0
    def handle_label(self, filename, **options):
        if not options["control"]:
            raise Exception, "You must specify a control file"
        __import__(options["control"])
        control = sys.modules[options["control"]]

        code_version = CodeType.objects.get(code=control.code_version())
        name_type = NameType.objects.get(code="O")
        code_type_os = CodeType.objects.get(code="unit_id")

        print filename
        current_generation = Generation.objects.current()
        new_generation = Generation.objects.new()
        if not new_generation:
            raise Exception, "No new generation to be used for import!"

        ds = DataSource(filename)
        layer = ds[0]
        for feat in layer:
            name = feat["NAME"].value
            if not isinstance(name, unicode):
                name = name.decode("iso-8859-1")

            name = re.sub("\s*\(DET( NO \d+|)\)\s*(?i)", "", name)
            name = re.sub("\s+", " ", name)

            ons_code = feat["CODE"].value if feat["CODE"].value not in ("999999", "999999999") else None
            unit_id = str(feat["UNIT_ID"].value)
            area_code = feat["AREA_CODE"].value
            patch = self.patch_boundary_line(ons_code, area_code)
            if patch == True:
                ons_code = None
            elif patch:
                ons_code = patch

            if area_code == "NCP":
                continue  # Ignore Non Parished Areas

            if ons_code in self.ons_code_to_shape:
                m, poly = self.ons_code_to_shape[ons_code]
                try:
                    m_name = m.names.get(type=name_type).name
                except Name.DoesNotExist:
                    m_name = m.name  # If running without commit for dry run, so nothing being stored in db
                if name != m_name:
                    raise Exception, "ONS code %s is used for %s and %s" % (ons_code, name, m_name)
                # Otherwise, combine the two shapes for one area
                poly.append(feat.geom)
                continue

            if unit_id in self.unit_id_to_shape:
                m, poly = self.unit_id_to_shape[unit_id]
                try:
                    m_name = m.names.get(type=name_type).name
                except Name.DoesNotExist:
                    m_name = m.name  # If running without commit for dry run, so nothing being stored in db
                if name != m_name:
                    raise Exception, "Unit ID code %s is used for %s and %s" % (unit_id, name, m_name)
                # Otherwise, combine the two shapes for one area
                poly.append(feat.geom)
                continue

            if code_version.code == "gss" and ons_code:
                country = ons_code[0]  # Hooray!
            elif area_code in ("CED", "CTY", "DIW", "DIS", "MTW", "MTD", "LBW", "LBO", "LAC", "GLA"):
                country = "E"
            elif code_version.code == "gss":
                raise Exception, area_code
            elif (
                (area_code == "EUR" and "Scotland" in name)
                or area_code in ("SPC", "SPE")
                or (ons_code and ons_code[0:3] in ("00Q", "00R"))
            ):
                country = "S"
            elif (
                (area_code == "EUR" and "Wales" in name)
                or area_code in ("WAC", "WAE")
                or (ons_code and ons_code[0:3] in ("00N", "00P"))
            ):
                country = "W"
            elif area_code in ("EUR", "UTA", "UTE", "UTW", "CPC"):
                country = "E"
            else:  # WMC
                # Make sure WMC are loaded after all wards...
                area_within = Area.objects.filter(
                    type__code__in=("UTW", "UTE", "MTW", "COP", "LBW", "DIW"),
                    polygons__polygon__contains=feat.geom.geos.point_on_surface,
                )[0]
                country = area_within.country.code
            # Can't do the above ons_code checks with new GSS codes, will have to do more PinP checks
            # Do parents in separate P-in-P code after this is done.

            try:
                check = control.check(name, area_code, country, feat.geom)
                if check == True:
                    raise Area.DoesNotExist
                if isinstance(check, Area):
                    m = check
                    ons_code = m.codes.get(type=code_version).code
                elif ons_code:
                    m = Area.objects.get(codes__type=code_version, codes__code=ons_code)
                elif unit_id:
                    m = Area.objects.get(codes__type=code_type_os, codes__code=unit_id)
                    m_name = m.names.get(type=name_type).name
                    if name != m_name:
                        raise Exception, "Unit ID code %s is %s in DB but %s in SHP file" % (unit_id, m_name, name)
                else:
                    raise Exception, 'Area "%s" (%s) has neither ONS code nor unit ID' % (name, area_code)
                if int(options["verbosity"]) > 1:
                    print "  Area matched, %s" % (m,)
            except Area.DoesNotExist:
                print "  New area: %s %s %s %s" % (area_code, ons_code, unit_id, name)
                m = Area(
                    name=name,  # If committing, this will be overwritten by the m.names.update_or_create
                    type=Type.objects.get(code=area_code),
                    country=Country.objects.get(code=country),
                    generation_low=new_generation,
                    generation_high=new_generation,
                )

            if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
                raise Exception, "Area %s found, but not in current generation %s" % (m, current_generation)
            m.generation_high = new_generation
            if options["commit"]:
                m.save()

            poly = [feat.geom]

            if options["commit"]:
                m.names.update_or_create({"type": name_type}, {"name": name})
            if ons_code:
                self.ons_code_to_shape[ons_code] = (m, poly)
                if options["commit"]:
                    m.codes.update_or_create({"type": code_version}, {"code": ons_code})
            if unit_id:
                self.unit_id_to_shape[unit_id] = (m, poly)
                if options["commit"]:
                    m.codes.update_or_create({"type": code_type_os}, {"code": unit_id})

        if options["commit"]:
            save_polygons(self.unit_id_to_shape)
            save_polygons(self.ons_code_to_shape)
コード例 #27
0
ファイル: mapit_import.py プロジェクト: Hutspace/mapit
    def handle_label(self, filename, **options):

        err = False
        for k in ['generation_id','area_type_code','name_type_code','country_code']:
            if options[k]: continue
            print "Missing argument '--%s'" % k
            err = True
        if err:
            sys.exit(1)

        generation_id = options['generation_id']
        area_type_code = options['area_type_code']
        name_type_code = options['name_type_code']
        country_code = options['country_code']
        name_field = options['name_field'] or 'Name'
        code_field = options['code_field']
        code_type_code = options['code_type']
        encoding = options['encoding'] or 'utf-8'

        if len(area_type_code)>3:
            print "Area type code must be 3 letters or fewer, sorry"
            sys.exit(1)

        if (code_field and not code_type_code) or (not code_field and code_type_code):
            print "You must specify both code_field and code_type, or neither."
            sys.exit(1)
        try:
            area_type = Type.objects.get(code=area_type_code)
        except:
            type_desc = raw_input('Please give a description for area type code %s: ' % area_type_code)
            area_type = Type(code=area_type_code, description=type_desc)
            if options['commit']: area_type.save()

        try:
            name_type = NameType.objects.get(code=name_type_code)
        except:
            name_desc = raw_input('Please give a description for name type code %s: ' % name_type_code)
            name_type = NameType(code=name_type_code, description=name_desc)
            if options['commit']: name_type.save()

        try:
            country = Country.objects.get(code=country_code)
        except:
            country_name = raw_input('Please give the name for country code %s: ' % country_code)
            country = Country(code=country_code, name=country_name)
            if options['commit']: country.save()

        if code_type_code:
            try:
                code_type = CodeType.objects.get(code=code_type_code)
            except:
                code_desc = raw_input('Please give a description for code type %s: ' % code_type_code)
                code_type = CodeType(code=code_type_code, description=code_desc)
                if options['commit']: code_type.save()

        print "Importing from %s" % filename

        if not options['commit']:
            print '(will not save to db as --commit not specified)'

        current_generation = Generation.objects.current()
        new_generation     = Generation.objects.get( id=generation_id )

        ds = DataSource(filename)
        layer = ds[0]
        for feat in layer:

            try:
                name = feat[name_field].value
            except:
                choices = ', '.join(layer.fields)
                print "Could not find name using name field '%s' - should it be something else? It will be one of these: %s. Specify which with --name_field" % (name_field, choices)
                sys.exit(1)
            try:
                name = name.decode(encoding)
            except:
                print "Could not decode name using encoding '%s' - is it in another encoding? Specify one with --encoding" % encoding
                sys.exit(1)

            name = re.sub('\s+', ' ', name)
            if not name:
                raise Exception( "Could not find a name to use for area" )

            code = None
            if code_field:
                try:
                    code = feat[code_field].value
                except:
                    choices = ', '.join(layer.fields)
                    print "Could not find code using code field '%s' - should it be something else? It will be one of these: %s. Specify which with --code_field" % (code_field, choices)
                    sys.exit(1)

            print "  looking at '%s'%s" % ( name.encode('utf-8'), (' (%s)' % code) if code else '' )

            try:
                if code:
                    m = Area.objects.get(codes__code=code, codes__type=code_type)
                else:
                    # Assumes unique names if no code column used
                    m = Area.objects.get(name=name, type=area_type)
            except Area.DoesNotExist:
                m = Area(
                    name            = name,
                    type            = area_type,
                    country         = country,
                    # parent_area     = parent_area,
                    generation_low  = new_generation,
                    generation_high = new_generation,
                )
                if options['use_code_as_id'] and code:
                    m.id = int(code)

            # check that we are not about to skip a generation
            if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
                raise Exception, "Area %s found, but not in current generation %s" % (m, current_generation)
            m.generation_high = new_generation

            g = feat.geom.transform(4326, clone=True)

            poly = [ g ]

            if options['commit']:
                m.save()
                m.names.update_or_create({ 'type': name_type }, { 'name': name })
                if code:
                    m.codes.update_or_create({ 'type': code_type }, { 'code': code })
                save_polygons({ m.id : (m, poly) })
コード例 #28
0
    def handle_label(self, filename, **options):
        if not options['control']:
            raise Exception, "You must specify a control file"
        __import__(options['control'])
        control = sys.modules[options['control']]

        code_version = CodeType.objects.get(code=control.code_version())
        name_type = NameType.objects.get(code='O')
        code_type_os = CodeType.objects.get(code='unit_id')

        print filename
        current_generation = Generation.objects.current()
        new_generation = Generation.objects.new()
        if not new_generation:
            raise Exception, "No new generation to be used for import!"

        ds = DataSource(filename)
        layer = ds[0]
        for feat in layer:
            name = feat['NAME'].value
            if not isinstance(name, unicode):
                name = name.decode('iso-8859-1')

            name = re.sub('\s*\(DET( NO \d+|)\)\s*(?i)', '', name)
            name = re.sub('\s+', ' ', name)

            ons_code = feat['CODE'].value if feat['CODE'].value not in (
                '999999', '999999999') else None
            unit_id = str(feat['UNIT_ID'].value)
            area_code = feat['AREA_CODE'].value
            patch = self.patch_boundary_line(ons_code, area_code)
            if patch == True: ons_code = None
            elif patch: ons_code = patch

            if area_code == 'NCP': continue  # Ignore Non Parished Areas

            if ons_code in self.ons_code_to_shape:
                m, poly = self.ons_code_to_shape[ons_code]
                try:
                    m_name = m.names.get(type=name_type).name
                except Name.DoesNotExist:
                    m_name = m.name  # If running without commit for dry run, so nothing being stored in db
                if name != m_name:
                    raise Exception, "ONS code %s is used for %s and %s" % (
                        ons_code, name, m_name)
                # Otherwise, combine the two shapes for one area
                poly.append(feat.geom)
                continue

            if unit_id in self.unit_id_to_shape:
                m, poly = self.unit_id_to_shape[unit_id]
                try:
                    m_name = m.names.get(type=name_type).name
                except Name.DoesNotExist:
                    m_name = m.name  # If running without commit for dry run, so nothing being stored in db
                if name != m_name:
                    raise Exception, "Unit ID code %s is used for %s and %s" % (
                        unit_id, name, m_name)
                # Otherwise, combine the two shapes for one area
                poly.append(feat.geom)
                continue

            if code_version.code == 'gss' and ons_code:
                country = ons_code[0]  # Hooray!
            elif area_code in ('CED', 'CTY', 'DIW', 'DIS', 'MTW', 'MTD', 'LBW',
                               'LBO', 'LAC', 'GLA'):
                country = 'E'
            elif code_version.code == 'gss':
                raise Exception, area_code
            elif (area_code == 'EUR' and 'Scotland' in name) or area_code in (
                    'SPC', 'SPE') or (ons_code
                                      and ons_code[0:3] in ('00Q', '00R')):
                country = 'S'
            elif (area_code == 'EUR' and 'Wales' in name) or area_code in (
                    'WAC', 'WAE') or (ons_code
                                      and ons_code[0:3] in ('00N', '00P')):
                country = 'W'
            elif area_code in ('EUR', 'UTA', 'UTE', 'UTW', 'CPC'):
                country = 'E'
            else:  # WMC
                # Make sure WMC are loaded after all wards...
                area_within = Area.objects.filter(
                    type__code__in=('UTW', 'UTE', 'MTW', 'COP', 'LBW', 'DIW'),
                    polygons__polygon__contains=feat.geom.geos.point_on_surface
                )[0]
                country = area_within.country.code
            # Can't do the above ons_code checks with new GSS codes, will have to do more PinP checks
            # Do parents in separate P-in-P code after this is done.

            try:
                check = control.check(name, area_code, country, feat.geom)
                if check == True:
                    raise Area.DoesNotExist
                if isinstance(check, Area):
                    m = check
                    ons_code = m.codes.get(type=code_version).code
                elif ons_code:
                    m = Area.objects.get(codes__type=code_version,
                                         codes__code=ons_code)
                elif unit_id:
                    m = Area.objects.get(codes__type=code_type_os,
                                         codes__code=unit_id,
                                         generation_high=current_generation)
                    m_name = m.names.get(type=name_type).name
                    if name != m_name:
                        raise Exception, "Unit ID code %s is %s in DB but %s in SHP file" % (
                            unit_id, m_name, name)
                else:
                    raise Exception, 'Area "%s" (%s) has neither ONS code nor unit ID' % (
                        name, area_code)
                if int(options['verbosity']) > 1:
                    print "  Area matched, %s" % (m, )
            except Area.DoesNotExist:
                print "  New area: %s %s %s %s" % (area_code, ons_code,
                                                   unit_id, name)
                m = Area(
                    name=
                    name,  # If committing, this will be overwritten by the m.names.update_or_create
                    type=Type.objects.get(code=area_code),
                    country=Country.objects.get(code=country),
                    generation_low=new_generation,
                    generation_high=new_generation,
                )

            if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
                raise Exception, "Area %s found, but not in current generation %s" % (
                    m, current_generation)
            m.generation_high = new_generation
            if options['commit']:
                m.save()

            poly = [feat.geom]

            if options['commit']:
                m.names.update_or_create({'type': name_type}, {'name': name})
            if ons_code:
                self.ons_code_to_shape[ons_code] = (m, poly)
                if options['commit']:
                    m.codes.update_or_create({'type': code_version},
                                             {'code': ons_code})
            if unit_id:
                self.unit_id_to_shape[unit_id] = (m, poly)
                if options['commit']:
                    m.codes.update_or_create({'type': code_type_os},
                                             {'code': unit_id})

        if options['commit']:
            save_polygons(self.unit_id_to_shape)
            save_polygons(self.ons_code_to_shape)
コード例 #29
0
    def process_file(self, filename, area_code, srid, control, options):
        code_version = CodeType.objects.get(code=control.code_version())
        name_type = NameType.objects.get(code="N")
        code_type_osni = CodeType.objects.get(code="osni_oid")
        if not hasattr(self, area_code):
            raise Exception("Don't know how to extract features from %s files" % area_code)

        area_code_info = getattr(self, area_code)(srid)

        print(filename)
        current_generation = Generation.objects.current()
        new_generation = Generation.objects.new()
        if not new_generation:
            raise Exception("No new generation to be used for import!")

        ds = DataSource(filename)
        layer = ds[0]

        for feat in layer:
            name, ons_code, osni_object_id = area_code_info.extract_fields(feat)
            name = self.format_name(name)
            if ons_code in self.ons_code_to_shape:
                m, poly = self.ons_code_to_shape[ons_code]
                try:
                    m_name = m.names.get(type=name_type).name
                except Name.DoesNotExist:
                    m_name = m.name  # If running without commit for dry run, so nothing being stored in db
                if name != m_name:
                    raise Exception("ONS code %s is used for %s and %s" % (ons_code, name, m_name))
                # Otherwise, combine the two shapes for one area
                poly.append(feat.geom)
                continue

            if osni_object_id in self.osni_object_id_to_shape:
                m, poly = self.osni_object_id_to_shape[osni_object_id]
                try:
                    m_name = m.names.get(type=name_type).name
                except Name.DoesNotExist:
                    m_name = m.name  # If running without commit for dry run, so nothing being stored in db
                if name != m_name:
                    raise Exception("OSNI Object ID code %s is used for %s and %s" % (osni_object_id, name, m_name))
                # Otherwise, combine the two shapes for one area
                poly.append(feat.geom)
                continue

            country = "N"

            try:
                check = control.check(name, area_code, country, feat.geom)
                if check is True:
                    raise Area.DoesNotExist
                if isinstance(check, Area):
                    m = check
                    ons_code = m.codes.get(type=code_version).code
                elif ons_code:
                    m = Area.objects.get(codes__type=code_version, codes__code=ons_code)
                elif osni_object_id:
                    m = Area.objects.get(
                        codes__type=code_type_osni, codes__code=osni_object_id, generation_high=current_generation
                    )
                    m_name = m.names.get(type=name_type).name
                    if name != m_name:
                        raise Exception(
                            "OSNI Object ID code %s is %s in DB but %s in SHP file" % (osni_object_id, m_name, name)
                        )
                else:
                    raise Exception('Area "%s" (%s) has neither ONS code nor OSNI Object ID' % (name, area_code))
                if int(options["verbosity"]) > 1:
                    print("  Area matched, %s" % (m,))
            except Area.DoesNotExist:
                area_type = Type.objects.get(code=area_code)
                # It's possible we already have NIE entries (without any codes) in the db
                try:
                    if area_code == "NIE":
                        matching_name = name.title().replace("And", "and")
                        m = Area.objects.get(name=matching_name, type=area_type, generation_high=current_generation)
                        if int(options["verbosity"]) > 1:
                            print("  Area matched (via name), %s" % (m,))
                    else:
                        raise Area.DoesNotExist("Still doesn't exist")
                except Area.DoesNotExist:
                    print("  New area: %s %s %s %s" % (area_code, ons_code, osni_object_id, name))
                    m = Area(
                        name=name,  # Not overwritten by m.names.update_or_create as no "N" use
                        type=area_type,
                        country=Country.objects.get(code=country),
                        generation_low=new_generation,
                        generation_high=new_generation,
                    )

            if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
                raise Exception("Area %s found, but not in current generation %s" % (m, current_generation))
            m.generation_high = new_generation
            if options["commit"]:
                m.save()

            # Make a GEOS geometry only to check for validity:
            g = feat.geom
            geos_g = g.geos
            if not geos_g.valid:
                print("  Geometry of %s %s not valid" % (ons_code, m))
                geos_g = fix_invalid_geos_geometry(geos_g)
                if geos_g is None:
                    raise Exception("The geometry for area %s was invalid and couldn't be fixed" % name)
                    g = None
                else:
                    g = geos_g.ogr

            g = area_code_info.transform_geom(g)
            poly = [g]

            if options["commit"]:
                m.names.update_or_create(type=name_type, defaults={"name": name})
            if ons_code:
                self.ons_code_to_shape[ons_code] = (m, poly)
                if options["commit"]:
                    m.codes.update_or_create(type=code_version, defaults={"code": ons_code})
            if osni_object_id:
                self.osni_object_id_to_shape[osni_object_id] = (m, poly)
                if options["commit"]:
                    m.codes.update_or_create(type=code_type_osni, defaults={"code": osni_object_id})

        if options["commit"]:
            save_polygons(self.osni_object_id_to_shape)
            save_polygons(self.ons_code_to_shape)
コード例 #30
0
                    def update_or_create():
                        if osm_code:
                            m = osm_code.area
                        else:
                            m = Area(
                                name=name,
                                type=Type.objects.get(code=area_code),
                                country=Country.objects.get(code='G'),
                                parent_area=parent_area,
                                generation_low=new_generation,
                                generation_high=new_generation,
                            )

                        if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
                            raise Exception, "Area %s found, but not in current generation %s" % (
                                m, current_generation)
                        m.generation_high = new_generation

                        g = feat.geom.transform(4326, clone=True)

                        # In generating the data we should have
                        # excluded any "polygons" with less than four
                        # points (the final one being the same as the
                        # first), but just in case:
                        for polygon in g:
                            if g.num_points < 4:
                                return

                        poly = [g]

                        if options['commit']:
                            m.save()

                            if name not in kml_data.data:
                                print json.dumps(kml_data.data,
                                                 sort_keys=True,
                                                 indent=4)
                                raise Exception, u"Will fail to find '%s' in the dictionary" % (
                                    name, )

                            for k, v in kml_data.data[name].items():
                                language_name = None
                                if k == 'name':
                                    lang = 'default'
                                    language_name = "OSM Default"
                                else:
                                    name_match = re.search(r'^name:(.+)$', k)
                                    if name_match:
                                        lang = name_match.group(1)
                                        if lang in language_code_to_name:
                                            language_name = language_code_to_name[
                                                lang]
                                if not language_name:
                                    continue
                                # Otherwise, make sure that a NameType for this language exists:
                                NameType.objects.update_or_create(
                                    {'code': lang}, {
                                        'code': lang,
                                        'description': language_name
                                    })
                                name_type = NameType.objects.get(code=lang)
                                m.names.update_or_create({'type': name_type},
                                                         {'name': v})
                            m.codes.update_or_create({'type': code_type_osm},
                                                     {'code': osm_id})
                            save_polygons({code: (m, poly)})
コード例 #31
0
    def handle_label(self, filename, **options):
        self.stdout.write("Importing from %s" % filename)

        current_generation = Generation.objects.current()
        new_generation = Generation.objects.new()

        file_type = options["file_type"].value.lower()
        generalized = options["generalized"]

        if file_type == "lsoa":  # England and Wales
            feat_name = 'LSOA11NM'
            feat_code = 'LSOA11CD'
        elif file_type == "msoa":  # England and Wales
            feat_name = 'MSOA11NM'
            feat_code = 'MSOA11CD'
        elif file_type == "soa":  # super output areas - NI
            feat_name = 'SOA_LABEL'
            feat_code = 'SOA_CODE'
        elif file_type == "dz":  # datazone - scotland
            feat_name = 'Name'
            feat_code = 'DataZone'
        elif file_type == "iz":  # intermediate zone - scotland
            feat_name = 'Name'
            feat_code = 'InterZone'
        else:
            raise Exception(
                "Need to specify a small area type (lsoa, dz, soa, iz, msoa)")

        if file_type in ["lsoa", "dz", "soa"]:
            area_type = 'OLF'
        elif file_type in ["msoa", "iz"]:
            area_type = 'OMF'

        if generalized:
            area_type = area_type[:2] + "G"

        ds = DataSource(filename)
        layer = ds[0]
        for feat in layer:
            # retrieve name and code, and set country
            name = feat[feat_name].value
            code = feat[feat_code].value
            if code[:2] == "95":
                country = "N"
            else:
                country = code[0]

            self.stdout.write("  looking at '%s' (%s)" % (name, code))

            g = feat.geom.transform(settings.MAPIT_AREA_SRID, clone=True)

            # skip if the SOA already exists in db (SOAs don't change)
            areas = Area.objects.filter(type__code=area_type, codes__code=code)
            if len(areas):
                m = areas[0]
                # check that we are not about to skip a generation
                if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
                    raise Exception(
                        "Area %s found, but not in current generation %s" %
                        (m, current_generation))
                if m.generation_high != new_generation and options['commit']:
                    m.generation_high = new_generation
                    m.save()
                continue

            print("Adding %s (%s) %s" % (name, code, feat.geom.geom_name))
            m = Area(
                name=name,
                type=Type.objects.get(code=area_type),
                country=Country.objects.get(code=country),
                generation_low=new_generation,
                generation_high=new_generation,
            )

            poly = [g] if g is not None else []

            if options['commit']:
                m.save()
                m.names.update_or_create(type=NameType.objects.get(code='S'),
                                         defaults={'name': name})
                m.codes.update_or_create(type=CodeType.objects.get(code='ons'),
                                         defaults={'code': code})
                save_polygons({m.id: (m, poly)})
コード例 #32
0
ファイル: mapit_import.py プロジェクト: Code4SA/mapit
    def handle_label(self, filename, **options):

        missing_options = []
        for k in ['generation_id', 'area_type_code', 'name_type_code', 'country_code']:
            if options[k]:
                continue
            else:
                missing_options.append(k)
        if missing_options:
            message_start = "Missing arguments " if len(missing_options) > 1 else "Missing argument "
            message = message_start + " ".join('--{0}'.format(k) for k in missing_options)
            raise CommandError(message)

        generation_id = options['generation_id']
        area_type_code = options['area_type_code']
        name_type_code = options['name_type_code']
        country_code = options['country_code']
        override_name = options['override_name']
        name_field = options['name_field']
        if not (override_name or name_field):
            name_field = 'Name'
        override_code = options['override_code']
        code_field = options['code_field']
        code_type_code = options['code_type']
        encoding = options['encoding'] or 'utf-8'

        if len(area_type_code) > 3:
            raise CommandError("Area type code must be 3 letters or fewer, sorry")

        if name_field and override_name:
            raise CommandError("You must not specify both --name_field and --override_name")
        if code_field and override_code:
            raise CommandError("You must not specify both --code_field and --override_code")

        using_code = (code_field or override_code)
        if (using_code and not code_type_code) or (not using_code and code_type_code):
            raise CommandError(
                "If you want to save a code, specify --code_type and either --code_field or --override_code")
        try:
            area_type = Type.objects.get(code=area_type_code)
        except:
            type_desc = input('Please give a description for area type code %s: ' % area_type_code)
            area_type = Type(code=area_type_code, description=type_desc)
            if options['commit']:
                area_type.save()

        try:
            name_type = NameType.objects.get(code=name_type_code)
        except:
            name_desc = input('Please give a description for name type code %s: ' % name_type_code)
            name_type = NameType(code=name_type_code, description=name_desc)
            if options['commit']:
                name_type.save()

        try:
            country = Country.objects.get(code=country_code)
        except:
            country_name = input('Please give the name for country code %s: ' % country_code)
            country = Country(code=country_code, name=country_name)
            if options['commit']:
                country.save()

        if code_type_code:
            try:
                code_type = CodeType.objects.get(code=code_type_code)
            except:
                code_desc = input('Please give a description for code type %s: ' % code_type_code)
                code_type = CodeType(code=code_type_code, description=code_desc)
                if options['commit']:
                    code_type.save()

        self.stdout.write("Importing from %s" % filename)

        if not options['commit']:
            self.stdout.write('(will not save to db as --commit not specified)')

        current_generation = Generation.objects.current()
        new_generation = Generation.objects.get(id=generation_id)

        def verbose(*args):
            if int(options['verbosity']) > 1:
                self.stdout.write(" ".join(str(a) for a in args))

        ds = DataSource(filename)
        layer = ds[0]
        if (override_name or override_code) and len(layer) > 1:
            message = (
                "Warning: you have specified an override %s and this file contains more than one feature; "
                "multiple areas with the same %s will be created")
            if override_name:
                self.stdout.write(message % ('name', 'name'))
            if override_code:
                self.stdout.write(message % ('code', 'code'))

        for feat in layer:

            if override_name:
                name = override_name
            else:
                try:
                    name = feat[name_field].value
                except:
                    choices = ', '.join(layer.fields)
                    raise CommandError(
                        "Could not find name using name field '%s' - should it be something else? "
                        "It will be one of these: %s. Specify which with --name_field" % (name_field, choices))
                try:
                    if not isinstance(name, six.text_type):
                        name = name.decode(encoding)
                except:
                    raise CommandError(
                        "Could not decode name using encoding '%s' - is it in another encoding? "
                        "Specify one with --encoding" % encoding)

            name = re.sub('\s+', ' ', name)
            if not name:
                raise Exception("Could not find a name to use for area")

            code = None
            if override_code:
                code = override_code
            elif code_field:
                try:
                    code = feat[code_field].value
                except:
                    choices = ', '.join(layer.fields)
                    raise CommandError(
                        "Could not find code using code field '%s' - should it be something else? "
                        "It will be one of these: %s. Specify which with --code_field" % (code_field, choices))

            self.stdout.write("  looking at '%s'%s" % (name, (' (%s)' % code) if code else ''))

            g = None
            if hasattr(feat, 'geom'):
                g = feat.geom.transform(settings.MAPIT_AREA_SRID, clone=True)

            try:
                if options['new']:  # Always want a new area
                    raise Area.DoesNotExist
                if code:
                    matching_message = "code %s of code type %s" % (code, code_type)
                    areas = Area.objects.filter(codes__code=code, codes__type=code_type).order_by('-generation_high')
                else:
                    matching_message = "name %s of area type %s" % (name, area_type)
                    areas = Area.objects.filter(name=name, type=area_type).order_by('-generation_high')
                if len(areas) == 0:
                    verbose("    the area was not found - creating a new one")
                    raise Area.DoesNotExist
                m = areas[0]
                verbose("    found the area")
                if options['preserve']:
                    # Find whether we need to create a new Area:
                    previous_geos_geometry = m.polygons.collect()
                    if m.generation_high < current_generation.id:
                        # Then it was missing in current_generation:
                        verbose("    area existed previously, but was missing from", current_generation)
                        raise Area.DoesNotExist
                    elif g is None:
                        if previous_geos_geometry is not None:
                            verbose("    area is now empty")
                            raise Area.DoesNotExist
                        else:
                            verbose("    the area has remained empty")
                    elif previous_geos_geometry is None:
                        # It was empty in the previous generation:
                        verbose("    area was empty in", current_generation)
                        raise Area.DoesNotExist
                    else:
                        # Otherwise, create a new Area unless the
                        # polygons were the same in current_generation:
                        previous_geos_geometry = previous_geos_geometry.simplify(tolerance=0)
                        new_geos_geometry = g.geos.simplify(tolerance=0)
                        create_new_area = not previous_geos_geometry.equals(new_geos_geometry)
                        p = previous_geos_geometry.sym_difference(new_geos_geometry).area / previous_geos_geometry.area
                        verbose("    change in area is:", "%.03f%%" % (100 * p,))
                        if create_new_area:
                            verbose("    the area", m, "has changed, creating a new area due to --preserve")
                            raise Area.DoesNotExist
                        else:
                            verbose("    the area remained the same")
                else:
                    # If --preserve is not specified, the code or the name must be unique:
                    if len(areas) > 1:
                        raise Area.MultipleObjectsReturned(
                            "There was more than one area with %s, and --preserve was not specified" % (
                                matching_message,))

            except Area.DoesNotExist:
                m = Area(
                    name=name,
                    type=area_type,
                    country=country,
                    # parent_area=parent_area,
                    generation_low=new_generation,
                    generation_high=new_generation,
                )
                if options['use_code_as_id'] and code:
                    m.id = int(code)

            # check that we are not about to skip a generation
            if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
                raise Exception("Area %s found, but not in current generation %s" % (m, current_generation))
            m.generation_high = new_generation

            if options['fix_invalid_polygons'] and g is not None:
                # Make a GEOS geometry only to check for validity:
                geos_g = g.geos
                if not geos_g.valid:
                    geos_g = fix_invalid_geos_geometry(geos_g)
                    if geos_g is None:
                        self.stdout.write("The geometry for area %s was invalid and couldn't be fixed" % name)
                        g = None
                    else:
                        g = geos_g.ogr

            poly = [g] if g is not None else []

            if options['commit']:
                m.save()
                m.names.update_or_create(type=name_type, defaults={'name': name})
                if code:
                    m.codes.update_or_create(type=code_type, defaults={'code': code})
                save_polygons({m.id: (m, poly)})
コード例 #33
0
    def handle_label(self, filename, **options):
        country = Country.objects.get(code='N')
        oa_type = Type.objects.get(code='OUA')
        soa_type = Type.objects.get(code='OLF')
        name_type = NameType.objects.get(code='S')
        code_type = CodeType.objects.get(code='ons')

        current_generation = Generation.objects.current()
        new_generation = Generation.objects.new()
        if not new_generation:
            raise Exception("No new generation to be used for import!")

        # Compile an alphabetical list of NI councils and their wards, OA codes
        # are assigned alphabetically.
        if not self.councils:
            self.councils = Area.objects.filter(type=Type.objects.get(
                code='LGD')).order_by('name').values()
            for lgd in self.councils:
                lges = Area.objects.filter(parent_area=lgd['id'])
                areas = []
                for lge in lges:
                    lgws = Area.objects.filter(parent_area=lge).values()
                    areas += lgws
                lgd['wards'] = sorted(areas, key=lambda x: x['name'])

        ds = DataSource(filename)
        layer = ds[0]
        layer_name = str(layer)
        for feat in layer:
            if layer_name == 'soa':
                area_type = soa_type
                ons_code = feat['SOA_CODE'].value
                name = feat['SOA_LABEL'].value.replace('_', ' ')
            elif layer_name == 'OA_ni':
                area_type = oa_type
                ons_code = feat['OA_CODE'].value
                name = 'Output Area %s' % ons_code
            else:
                raise Exception('Bad data passed in')

            council = ord(ons_code[2:3]) - 65
            ward = int(ons_code[4:6]) - 1
            if ward == 98:  # SOA covers two wards, set parent to council, best we can do
                parent = self.councils[council]['id']
            else:
                parent = self.councils[council]['wards'][ward]['id']

            try:
                m = Area.objects.get(codes__type=code_type,
                                     codes__code=ons_code)
                if int(options['verbosity']) > 1:
                    print("  Area matched, %s" % (m, ))
            except Area.DoesNotExist:
                print("  New area: %s" % (ons_code))
                m = Area(
                    name=
                    name,  # If committing, this will be overwritten by the m.names.update_or_create
                    type=area_type,
                    country=country,
                    parent_area_id=parent,
                    generation_low=new_generation,
                    generation_high=new_generation,
                )

            if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
                raise Exception(
                    "Area %s found, but not in current generation %s" %
                    (m, current_generation))
            m.generation_high = new_generation
            m.parent_area_id = parent
            if options['commit']:
                m.save()

            f = feat.geom
            f.srid = 29902
            poly = [f]

            if options['commit']:
                m.names.update_or_create(type=name_type,
                                         defaults={'name': name})
            if ons_code:
                self.ons_code_to_shape[ons_code] = (m, poly)
                if options['commit']:
                    m.codes.update_or_create(type=code_type,
                                             defaults={'code': ons_code})

        if options['commit']:
            save_polygons(self.ons_code_to_shape)
コード例 #34
0
    def handle_label(self, filename, **options):
        if not options['control']:
            raise Exception("You must specify a control file")
        __import__(options['control'])
        control = sys.modules[options['control']]

        code_version = CodeType.objects.get(code='gss')
        name_type = NameType.objects.get(code='O')
        code_type_os = CodeType.objects.get(code='unit_id')

        print(filename)
        current_generation = Generation.objects.current()
        new_generation = Generation.objects.new()
        if not new_generation:
            raise Exception("No new generation to be used for import!")

        ds = DataSource(filename)
        layer = ds[0]
        for feat in layer:
            name = feat['NAME'].value
            if not isinstance(name, six.text_type):
                name = name.decode('iso-8859-1')

            name = re.sub(r'\s*\(DET( NO \d+|)\)\s*(?i)', '', name)
            name = re.sub(r'\s+', ' ', name)

            ons_code = feat['CODE'].value if feat['CODE'].value not in ('999999', '999999999') else None
            unit_id = str(feat['UNIT_ID'].value)
            area_code = feat['AREA_CODE'].value
            patch = self.patch_boundary_line(name, ons_code, unit_id, area_code)
            if 'ons-code' in patch:
                ons_code = patch['ons-code']
            elif 'unit-id' in patch:
                unit_id = patch['unit-id']

            if area_code == 'NCP':
                continue  # Ignore Non Parished Areas

            if ons_code in self.ons_code_to_shape:
                m, poly = self.ons_code_to_shape[ons_code]
                try:
                    m_name = m.names.get(type=name_type).name
                except Name.DoesNotExist:
                    m_name = m.name  # If running without commit for dry run, so nothing being stored in db
                if name != m_name:
                    raise Exception("ONS code %s is used for %s and %s" % (ons_code, name, m_name))
                # Otherwise, combine the two shapes for one area
                poly.append(feat.geom)
                continue

            if unit_id in self.unit_id_to_shape:
                m, poly = self.unit_id_to_shape[unit_id]
                try:
                    m_name = m.names.get(type=name_type).name
                except Name.DoesNotExist:
                    m_name = m.name  # If running without commit for dry run, so nothing being stored in db
                if name != m_name:
                    raise Exception("Unit ID code %s is used for %s and %s" % (unit_id, name, m_name))
                # Otherwise, combine the two shapes for one area
                poly.append(feat.geom)
                continue

            if ons_code:
                country = ons_code[0]  # Hooray!
            elif area_code in ('CED', 'CTY', 'DIW', 'DIS', 'MTW', 'MTD', 'LBW', 'LBO', 'LAC', 'GLA'):
                country = 'E'
            else:
                raise Exception(area_code)

            try:
                check = control.check(name, area_code, country, feat.geom, ons_code=ons_code, commit=options['commit'])
                if check is True:
                    raise Area.DoesNotExist
                if isinstance(check, Area):
                    m = check
                    try:
                        ons_code = m.codes.get(type=code_version).code
                    except Code.DoesNotExist:
                        ons_code = None
                elif ons_code:
                    m = Area.objects.get(codes__type=code_version, codes__code=ons_code)
                elif unit_id:
                    m = Area.objects.get(
                        codes__type=code_type_os, codes__code=unit_id, generation_high=current_generation)
                    m_name = m.names.get(type=name_type).name
                    if name != m_name:
                        raise Exception("Unit ID code %s is %s in DB but %s in SHP file" % (unit_id, m_name, name))
                else:
                    raise Exception('Area "%s" (%s) has neither ONS code nor unit ID' % (name, area_code))
                if int(options['verbosity']) > 1:
                    print("  Area matched, %s" % (m, ))
            except Area.DoesNotExist:
                print("  New area: %s %s %s %s" % (area_code, ons_code, unit_id, name))
                m = Area(
                    name=name,  # If committing, this will be overwritten by the m.names.update_or_create
                    type=Type.objects.get(code=area_code),
                    country=Country.objects.get(code=country),
                    generation_low=new_generation,
                    generation_high=new_generation,
                )

            if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
                raise Exception("Area %s found, but not in current generation %s" % (m, current_generation))
            m.generation_high = new_generation
            if options['commit']:
                m.save()

            # Make a GEOS geometry only to check for validity:
            g = feat.geom
            geos_g = g.geos
            if not geos_g.valid:
                print("  Geometry of %s %s not valid" % (ons_code, m))
                geos_g = fix_invalid_geos_geometry(geos_g)
                if geos_g is None:
                    raise Exception("The geometry for area %s was invalid and couldn't be fixed" % name)
                    g = None
                else:
                    g = geos_g.ogr

            poly = [g]

            if options['commit']:
                m.names.update_or_create(type=name_type, defaults={'name': name})
            if ons_code:
                self.ons_code_to_shape[ons_code] = (m, poly)
                if options['commit']:
                    m.codes.update_or_create(type=code_version, defaults={'code': ons_code})
            if unit_id:
                self.unit_id_to_shape[unit_id] = (m, poly)
                if options['commit']:
                    m.codes.update_or_create(type=code_type_os, defaults={'code': unit_id})

        if options['commit']:
            save_polygons(self.unit_id_to_shape)
            save_polygons(self.ons_code_to_shape)
コード例 #35
0
    def handle_label(self,  filename, **options):
        if not options['control']:
            raise Exception, "You must specify a control file"
        __import__(options['control'])
        control = sys.modules[options['control']]

        code_version = CodeType.objects.get(code=control.code_version())
        name_type = NameType.objects.get(code='O')
        code_type_os = CodeType.objects.get(code='unit_id')

        print filename
        current_generation = Generation.objects.current()
        new_generation = Generation.objects.new()
        if not new_generation:
            raise Exception, "No new generation to be used for import!"

        ds = DataSource(filename)
        layer = ds[0]
        for feat in layer:
            name = feat['NAME'].value
            if not isinstance(name, unicode):
                name = name.decode('iso-8859-1')

            name = re.sub('\s*\(DET( NO \d+|)\)\s*(?i)', '', name)
            name = re.sub('\s+', ' ', name)

            ons_code = feat['CODE'].value if feat['CODE'].value not in ('999999', '999999999') else None
            unit_id = str(feat['UNIT_ID'].value)
            area_code = feat['AREA_CODE'].value
            patch = self.patch_boundary_line(ons_code, area_code)
            if patch == True: ons_code = None
            elif patch: ons_code = patch

            if area_code == 'NCP': continue # Ignore Non Parished Areas

            if ons_code in self.ons_code_to_shape:
                m, poly = self.ons_code_to_shape[ons_code]
                try:
                    m_name = m.names.get(type=name_type).name
                except Name.DoesNotExist:
                    m_name = m.name # If running without commit for dry run, so nothing being stored in db
                if name != m_name:
                    raise Exception, "ONS code %s is used for %s and %s" % (ons_code, name, m_name)
                # Otherwise, combine the two shapes for one area
                poly.append(feat.geom)
                continue

            if unit_id in self.unit_id_to_shape:
                m, poly = self.unit_id_to_shape[unit_id]
                try:
                    m_name = m.names.get(type=name_type).name
                except Name.DoesNotExist:
                    m_name = m.name # If running without commit for dry run, so nothing being stored in db
                if name != m_name:
                    raise Exception, "Unit ID code %s is used for %s and %s" % (unit_id, name, m_name)
                # Otherwise, combine the two shapes for one area
                poly.append(feat.geom)
                continue

            if code_version.code == 'gss' and ons_code:
                country = ons_code[0] # Hooray!
            elif area_code in ('CED', 'CTY', 'DIW', 'DIS', 'MTW', 'MTD', 'LBW', 'LBO', 'LAC', 'GLA'):
                country = 'E'
            elif code_version.code == 'gss':
                raise Exception, area_code
            elif (area_code == 'EUR' and 'Scotland' in name) or area_code in ('SPC', 'SPE') or (ons_code and ons_code[0:3] in ('00Q', '00R')):
                country = 'S'
            elif (area_code == 'EUR' and 'Wales' in name) or area_code in ('WAC', 'WAE') or (ons_code and ons_code[0:3] in ('00N', '00P')):
                country = 'W'
            elif area_code in ('EUR', 'UTA', 'UTE', 'UTW', 'CPC'):
                country = 'E'
            else: # WMC
                # Make sure WMC are loaded after all wards...
                area_within = Area.objects.filter(type__code__in=('UTW','UTE','MTW','COP','LBW','DIW'), polygons__polygon__contains=feat.geom.geos.point_on_surface)[0]
                country = area_within.country.code
            # Can't do the above ons_code checks with new GSS codes, will have to do more PinP checks
            # Do parents in separate P-in-P code after this is done.

            try:
                check = control.check(name, area_code, country, feat.geom)
                if check == True:
                    raise Area.DoesNotExist
                if isinstance(check, Area):
                    m = check
                    ons_code = m.codes.get(type=code_version).code
                elif ons_code:
                    m = Area.objects.get(codes__type=code_version, codes__code=ons_code)
                elif unit_id:
                    m = Area.objects.get(codes__type=code_type_os, codes__code=unit_id, generation_high=current_generation)
                    m_name = m.names.get(type=name_type).name
                    if name != m_name:
                        raise Exception, "Unit ID code %s is %s in DB but %s in SHP file" % (unit_id, m_name, name)
                else:
                    raise Exception, 'Area "%s" (%s) has neither ONS code nor unit ID' % (name, area_code)
                if int(options['verbosity']) > 1:
                    print "  Area matched, %s" % (m, )
            except Area.DoesNotExist:
                print "  New area: %s %s %s %s" % (area_code, ons_code, unit_id, name)
                m = Area(
                    name = name, # If committing, this will be overwritten by the m.names.update_or_create
                    type = Type.objects.get(code=area_code),
                    country = Country.objects.get(code=country),
                    generation_low = new_generation,
                    generation_high = new_generation,
                )

            if m.generation_high and current_generation and m.generation_high.id < current_generation.id:
                raise Exception, "Area %s found, but not in current generation %s" % (m, current_generation)
            m.generation_high = new_generation
            if options['commit']:
                m.save()

            poly = [ feat.geom ]

            if options['commit']:
                m.names.update_or_create({ 'type': name_type }, { 'name': name })
            if ons_code:
                self.ons_code_to_shape[ons_code] = (m, poly)
                if options['commit']:
                    m.codes.update_or_create({ 'type': code_version }, { 'code': ons_code })
            if unit_id:
                self.unit_id_to_shape[unit_id] = (m, poly)
                if options['commit']:
                    m.codes.update_or_create({ 'type': code_type_os }, { 'code': unit_id })

        if options['commit']:
            save_polygons(self.unit_id_to_shape)
            save_polygons(self.ons_code_to_shape)