def locate(self, tw, conf=0.95): poly = tw.geom.buffer(5000) # FIXME: change to meters? multiregion = geos.MultiPolygon((poly), srid=poly.srid) return self.LE(point=tw.geom, cregion=multiregion, confidence=conf)
def import_events(data_from_feed, tolerance=0.0001): """ For every shapefile in _merged folder, creates or updates event in Django DB and Geoserver DB. The geometry inserted in Geoserver DB is the union of all features found in layer. """ db = DbUtils() conn = db.get_db_conn() curs = conn.cursor() allowed_extensions = [".shp"] try: for root, dirs, files in os.walk(SHAPEFILES_BASE_DIR): print('shapefiles base dir: {}'.format(SHAPEFILES_BASE_DIR)) for d in dirs: if d.endswith('_merged'): print('found merged folder') for f in os.listdir(os.path.join(SHAPEFILES_BASE_DIR, d)): if f.endswith(tuple(allowed_extensions)): print('found shapefiles: {}'.format(f)) '''try: filepath = os.path.join(SHAPEFILES_BASE_DIR, d, f) ds = DataSource(filepath) except GDALException, e: traceback.print_exc() break for layer in ds: count = 0 polygon_union = None for feat in layer: print('processing feature {}'.format(count)) # Simplify the Geometry geom = geos.fromstr(feat.geom.wkt, srid=4326) if tolerance > 0: geom = geom.simplify(tolerance, preserve_topology=True) # Generalize to 'Multiploygon' if isinstance(geom, geos.Polygon): geom = geos.MultiPolygon(geom) if count == 0: polygon_union = geom.buffer(0) else: polygon_union = polygon_union.union(geom.buffer(0)) count += 1 try: print('re-extracting geom from union polygon') polygon_union = geos.fromstr(polygon_union.wkt, srid=4326) except: break''' ems = d.split('_')[0] select_template = "SELECT ST_AsText(ST_Union(ARRAY(SELECT ST_Buffer(the_geom, 1e-5) FROM {})))".format( ems) row = None try: curs.execute(select_template) row = curs.fetchone() except: pass #set default multipolygon ext_coords = ((0, 0), (0, 1), (1, 1), (1, 0), (0, 0)) int_coords = ((0.4, 0.4), (0.4, 0.6), (0.6, 0.6), (0.6, 0.4), (0.4, 0.4)) polygon_union = geos.MultiPolygon(geos.Polygon( ext_coords, int_coords), srid=4326) if row: polygon_union = geos.fromstr(row[0], srid=4326) # Update event in Django event_from_feed = get_event_from_feed( data_from_feed, ems) new_event = generate_event_from_feed( event_from_feed, polygon_union) if new_event: params = { 'geom': polygon_union, 'event_id': new_event.event_id, 'begin_date': new_event.begin_date, 'end_date': new_event.end_date } update_template = """INSERT INTO events (the_geom, event_id, begin_date, end_date) SELECT '{geom}', '{event_id}', '{begin_date}', '{end_date}' ON CONFLICT (event_id) DO UPDATE SET the_geom = excluded.the_geom;""" print update_template.format(**params) curs.execute(update_template.format(**params)) archive_path = os.path.join(SHAPEFILES_BASE_DIR, d, 'archive') if not os.path.exists(archive_path): os.makedirs(archive_path) for f2 in os.listdir(os.path.join(SHAPEFILES_BASE_DIR, d)): filepath = os.path.join(SHAPEFILES_BASE_DIR, d, f2) if os.path.isfile(filepath): print('moving file {} to {}'.format( f2, archive_path)) os.rename(filepath, os.path.join(archive_path, f2)) conn.commit() except Exception: try: conn.rollback() except: pass traceback.print_exc() finally: conn.close()
def clean_poly(self, poly): if isinstance(poly, geos.Polygon): poly = geos.MultiPolygon(poly, srid=self.get_srid('districts')) return poly return poly
def handle(self, **options): adm_level = options.get('adm_level') region = options.get('region') region_level = options.get('region_level') shape_file = options.get('shape_file') tolerance = options.get('tolerance') if adm_level is None: raise CommandError("Input Administrative Unit Level '--adm-level' \ is mandatory") if region is None: raise CommandError("Input Destination Region '--region' \ is mandatory") if region_level is None: raise CommandError("Input Region Level '--region-level' \ is mandatory") if not shape_file or len(shape_file) == 0: raise CommandError("Input Administrative Unit Shapefile \ '--shape-file' is mandatory") ds = DataSource(shape_file) print('Opening Data Source "%s"' % ds.name) #print('rebuilding tree') #AdministrativeDivision.objects.rebuild() #print('rebuilding complete!') for layer in ds: print('Layer "%s": %i %ss' % (layer.name, len(layer), layer.geom_type.name)) (region_obj, is_new_region) = Region.objects.get_or_create( name=region, defaults=dict( level=region_level ) ) adm_rows = [] for feat in layer: # Simplify the Geometry geom = geos.fromstr(feat.geom.wkt, srid=4326) if tolerance > 0: geom = geom.simplify(tolerance, preserve_topology=True) # Generalize to 'Multiploygon' if isinstance(geom, geos.Polygon): geom = geos.MultiPolygon(geom) if adm_level == 0: (adm_division, is_new_amdiv) = \ AdministrativeDivision.objects.get_or_create( code=feat.get('HRPcode'), defaults=dict( name=feat.get('HRname'), geom=geom.wkt ) ) if not is_new_amdiv: adm_division.name = feat.get('HRname') adm_division.geom = geom.wkt adm_division.save() else: #region_obj.administrative_divisions.add(adm_division) RegionAdministrativeDivisionAssociation.objects.create(region=region_obj, administrativedivision=adm_division) if adm_level == 1: print 'parent {}'.format(feat.get('HRparent')[:2]) adm_division_0 = \ AdministrativeDivision.objects.get( code=feat.get('HRparent')[:2]) (adm_division, is_new_amdiv) = \ AdministrativeDivision.objects.get_or_create( code=feat.get('HRpcode'), defaults=dict( name=feat.get('HRname'), geom=geom.wkt, parent=adm_division_0 ) ) if not is_new_amdiv: adm_division.name = feat.get('HRname') adm_division.geom = geom.wkt adm_division.parent = adm_division_0 adm_division.save() else: #region_obj.administrative_divisions.add(adm_division) RegionAdministrativeDivisionAssociation.objects.create(region=region_obj, administrativedivision=adm_division) if adm_level == 2: print('region = {}'.format(feat.get('HRpcode'))) adm_division_1 = AdministrativeDivision.objects.get(code=feat.get('HRparent')) (adm_division, is_new_amdiv) = AdministrativeDivision.objects.get_or_create( code=feat.get('HRpcode'), defaults=dict( name=feat.get('HRname'), geom=geom.wkt, parent=adm_division_1 ) ) if not is_new_amdiv: adm_division.name = feat.get('HRname') adm_division.geom = geom.wkt adm_division.parent = adm_division_1 adm_division.save() else: #region_obj.administrative_divisions.add(adm_division) RegionAdministrativeDivisionAssociation.objects.create(region=region_obj, administrativedivision=adm_division) if adm_level == 3: print('adm = {}'.format(feat.get('HRpcode'))) print('parent = {}'.format(feat.get('HRparent'))) adm_division_2 = AdministrativeDivision.objects.get(code=feat.get('HRparent')) '''(adm_division, is_new_amdiv) = AdministrativeDivision.objects.get_or_create( code=feat.get('HRpcode'), defaults=dict( name=feat.get('HRname'), geom=geom.wkt, region=region_obj, parent=adm_division_2 ) ) if not is_new_amdiv: adm_division.name = feat.get('HRname') adm_division.geom = geom.wkt adm_division.region = region_obj adm_division.parent = adm_division_2 adm_division.save() else: region_obj.administrative_divisions.add(adm_division)''' lookup_obj = AdministrativeDivision.objects.filter(code=feat.get('HRpcode')) if lookup_obj.exists(): adm_division = AdministrativeDivision.objects.get(code=feat.get('HRpcode')) adm_division.name = feat.get('HRname') adm_division.geom = geom.wkt adm_division.parent = adm_division_2 adm_division.save() association, created = RegionAdministrativeDivisionAssociation.objects.get_or_create(region=region_obj, administrativedivision=adm_division) print('updated object {}'.format(feat.get('HRpcode'))) else: adm_division = AdministrativeDivision( code=feat.get('HRpcode'), name=feat.get('HRname'), geom=geom.wkt, level=adm_level, parent=adm_division_2, lft=1, rght=1, tree_id=1, ) adm_rows.append(adm_division) if(len(adm_rows) > 9999): print('bulk insert starting') AdministrativeDivision.objects.bulk_create(adm_rows) for adm_code in adm_rows: adm_div = AdministrativeDivision.objects.get(code=adm_code) RegionAdministrativeDivisionAssociation.objects.create(region=region_obj, administrativedivision=adm_div) print('bulk insert complete') adm_rows[:] = [] #print(adm_rows.count()) if(len(adm_rows) > 0): print('bulk insert starting') AdministrativeDivision.objects.bulk_create(adm_rows) for adm_code in adm_rows: adm_div = AdministrativeDivision.objects.get(code=adm_code) RegionAdministrativeDivisionAssociation.objects.create(region=region_obj, administrativedivision=adm_div) print('bulk insert complete')
def handle(self, **options): adm_level = options.get('adm_level') shape_file = options.get('shape_file') tolerance = options.get('tolerance') if adm_level is None: raise CommandError("Input Administrative Unit Level '--adm-level' \ is mandatory") if not shape_file or len(shape_file) == 0: raise CommandError("Input Administrative Unit Shapefile \ '--shape-file' is mandatory") ds = DataSource(shape_file) print ('Opening Data Source "%s"' % ds.name) for layer in ds: print('Layer "%s": %i %ss' % (layer.name, len(layer), layer.geom_type.name)) counter = 1 for feat in layer: # Simplify the Geometry geom = geos.fromstr(feat.geom.wkt, srid=4326) if tolerance > 0: geom = geom.simplify(tolerance, preserve_topology=True) if geom: # Generalize to 'Multiploygon' if isinstance(geom, geos.Polygon): geom = geos.MultiPolygon(geom) region_name = feat.get("NAME_{adm_level}".format(adm_level=adm_level - 1)) if adm_level > 0 else \ (feat.get("UNREGION2") if feat.get("UNREGION2") else feat.get("UNREGION1")) (region_obj, is_new_region) = Region.objects.get_or_create( name=region_name, defaults=dict( level=(adm_level - 1 if adm_level > 0 else adm_level) ) ) level = adm_level objectid = feat.get("OBJECTID") iso_id = feat.get("ID_{adm_level}".format(adm_level=adm_level)) name_iso = feat.get("NAME_ISO") if adm_level == 0 else feat.get("NAME_{adm_level}".format(adm_level=adm_level)) iso_parent = feat.get("ISO") iso_parent_id = iso_id if adm_level == 0 else feat.get("ID_{adm_level}".format(adm_level=adm_level - 1)) shape_leng = Decimal(feat.get("Shape_Leng")) shape_area = Decimal(feat.get("Shape_Area")) parent = None if name_iso: if level > 0: type = feat.get("TYPE_{adm_level}".format(adm_level=adm_level)) engtype = feat.get("ENGTYPE_{adm_level}".format(adm_level=adm_level)) try: parent = AdministrativeDivision.objects.get(Q(iso=iso_parent) & Q(iso_id=iso_parent_id) & Q(level=(adm_level - 1))) (adm_division, is_new_amdiv) = \ AdministrativeDivision.objects.get_or_create( name=name_iso, defaults=dict( objectid=objectid, level=level, iso=iso_parent, iso_id=iso_id, type=type, engtype=engtype, shape_leng=shape_leng, shape_area=shape_area, geom=geom, region=region_obj, parent=parent ) ) if adm_division and is_new_amdiv: region_obj.administrative_divisions.add(adm_division) except: pass else: parent = region_obj (adm_division, is_new_amdiv) = \ AdministrativeDivision.objects.get_or_create( name=name_iso, defaults=dict( objectid=objectid, level=level, iso=iso_parent, iso_id=iso_id, name_eng=feat.get("NAME_ENGLI"), name_fao=feat.get("NAME_FAO"), name_local=feat.get("NAME_LOCAL"), contains=feat.get("CONTAINS"), sovereign=feat.get("SOVEREIGN"), fips=feat.get("FIPS"), unregion=feat.get("UNREGION1"), ison=feat.get("ISON"), valid_from=feat.get("VALIDFR"), valid_to=feat.get("VALIDTO"), population=Decimal(feat.get("POP2000")), sqkm=Decimal(feat.get("SQKM")), pop_sqkm=Decimal(feat.get("POPSQKM")), shape_leng=shape_leng, shape_area=shape_area, geom=geom, region=region_obj ) ) if adm_division and is_new_amdiv: region_obj.administrative_divisions.add(adm_division) if parent: print(" %s ************* Region : %s - Parent: %s(%s) - Adm Unit : %s(%s) " % (counter, region_name, parent, iso_parent, name_iso, iso_id)) counter = counter + 1
def handle(self, **options): adm_level = options.get('adm_level') region = options.get('region') region_level = options.get('region_level') shape_file = options.get('shape_file') tolerance = options.get('tolerance') if adm_level is None: raise CommandError("Input Administrative Unit Level '--adm-level' \ is mandatory") if region is None: raise CommandError("Input Destination Region '--region' \ is mandatory") if region_level is None: raise CommandError("Input Region Level '--region-level' \ is mandatory") if not shape_file or len(shape_file) == 0: raise CommandError("Input Administrative Unit Shapefile \ '--shape-file' is mandatory") ds = DataSource(shape_file) print('Opening Data Source "%s"' % ds.name) for layer in ds: print('Layer "%s": %i %ss' % (layer.name, len(layer), layer.geom_type.name)) (region_obj, is_new_region) = Region.objects.get_or_create( name=region, defaults=dict(level=region_level)) for feat in layer: # Simplify the Geometry geom = geos.fromstr(feat.geom.wkt, srid=4326) if tolerance > 0: geom = geom.simplify(tolerance, preserve_topology=True) # Generalize to 'Multiploygon' geom = geos.MultiPolygon(geom) if adm_level == 0: (adm_division, is_new_amdiv) = \ AdministrativeDivision.objects.get_or_create( code=feat.get('HRPcode'), defaults=dict( name=feat.get('HRname'), geom=geom.wkt, region=region_obj ) ) if not is_new_amdiv: adm_division.name = feat.get('HRname') adm_division.geom = geom.wkt adm_division.region = region_obj adm_division.save() else: region_obj.administrative_divisions.add(adm_division) if adm_level == 1: adm_division_0 = \ AdministrativeDivision.objects.get( code=feat.get('HRparent')[:-2]) (adm_division, is_new_amdiv) = \ AdministrativeDivision.objects.get_or_create( code=feat.get('HRpcode'), defaults=dict( name=feat.get('HRname'), geom=geom.wkt, region=region_obj, parent=adm_division_0 ) ) if not is_new_amdiv: adm_division.name = feat.get('HRname') adm_division.geom = geom.wkt adm_division.region = region_obj adm_division.parent = adm_division_0 adm_division.save() else: region_obj.administrative_divisions.add(adm_division) if adm_level == 2: adm_division_1 = AdministrativeDivision.objects.get( code=feat.get('HRparent')) (adm_division, is_new_amdiv ) = AdministrativeDivision.objects.get_or_create( code=feat.get('HRpcode'), defaults=dict(name=feat.get('HRname'), geom=geom.wkt, region=region_obj, parent=adm_division_1)) if not is_new_amdiv: adm_division.name = feat.get('HRname') adm_division.geom = geom.wkt adm_division.region = region_obj adm_division.parent = adm_division_1 adm_division.save() else: region_obj.administrative_divisions.add(adm_division)
def execute(self): # NOQA C901 func_start = perf_counter() if not self.area_types: self.area_types = AREA_IMPORT_TYPES.keys() errors = [] for area_import_type in self.area_types: type_start = perf_counter() self.stdout.write( 'Starting to import the area type "{}"...\n'.format( area_import_type, )) area_import = AREA_IMPORT_TYPES[area_import_type] try: conn = psycopg2.connect( getattr(settings, area_import["source_dsn_setting_name"]), cursor_factory=psycopg2.extras.NamedTupleCursor, ) except (psycopg2.ProgrammingError, psycopg2.OperationalError) as e: self.stderr.write(str(e)) self.stderr.write( 'Could not connect to the database when importing area type "{}". DSN setting name "{}"' .format(area_import_type, area_import["source_dsn_setting_name"])) continue cursor = conn.cursor() self.stdout.write(area_import["source_name"]) (source, source_created) = AreaSource.objects.get_or_create( identifier=area_import["source_identifier"], defaults={"name": area_import["source_name"]}, ) cursor.execute(area_import["query"]) imported_identifiers = [] count = 0 sum_row_time, avg_row_time, min_row_time, max_row_time = (0, ) * 4 self.stdout.write("Starting to update areas...\n") for row in cursor: row_start = perf_counter() try: metadata = { METADATA_COLUMN_NAME_MAP[column_name]: getattr(row, column_name) for column_name in area_import["metadata_columns"] } except AttributeError as e: errors.append( "id #{}, metadata field missing. Error: {}\n".format( row.id, str(e))) count += 1 self.stdout.write("E", ending="") if count % 1000 == 0: self.stdout.write(" {}".format(count)) self.stdout.flush() continue match_data = { "type": area_import["area_type"], "external_id": row.id, "identifier": getattr(row, area_import["identifier_field_name"]), "source": source, } try: geom = geos.GEOSGeometry(row.geom_text) except geos.error.GEOSException as e: errors.append("id #{} error: {}\n".format(row.id, str(e))) count += 1 self.stdout.write("E", ending="") if count % 1000 == 0: self.stdout.write(" {}".format(count)) self.stdout.flush() continue if geom and isinstance(geom, geos.Polygon): geom = geos.MultiPolygon(geom) if geom and not isinstance(geom, geos.MultiPolygon): errors.append( 'id #{} Error! Geometry is not a Multipolygon but "{}"\n' .format(row.id, geom)) count += 1 self.stdout.write("E", ending="") if count % 1000 == 0: self.stdout.write(" {}".format(count)) self.stdout.flush() continue other_data = { "geometry": geom, "metadata": metadata, } try: Area.objects.update_or_create(defaults=other_data, **match_data) except MultipleObjectsReturned: # There should only be one object per identifier... ext_id = other_data.pop("external_id") # ...so we delete them all but spare the one with the correct external_id (if it happens to exist) Area.objects.filter(**match_data).exclude( external_id=ext_id).delete() match_data["external_id"] = ext_id Area.objects.update_or_create(defaults=other_data, **match_data) imported_identifiers.append(match_data["identifier"]) count += 1 if count % 100 == 0: self.stdout.write(".", ending="") if count % 1000 == 0: self.stdout.write(" {}".format(count)) self.stdout.flush() row_end = perf_counter() row_time = row_end - row_start sum_row_time += row_time min_row_time = (row_time if min_row_time == 0 or row_time < min_row_time else min_row_time) max_row_time = row_time if row_time > max_row_time else max_row_time if count > 0: avg_row_time = sum_row_time / count self.stdout.write( "Updated area count {}. Execution time: {:.2f}s " "(Row time avg: {:.2f}s, min: {:.2f}s, max: {:.2f}s)\n".format( count, sum_row_time, avg_row_time, min_row_time, max_row_time)) self.stdout.write("Starting to remove stales...\n") stale_time_start = perf_counter() stale = Area.objects.filter( type=area_import["area_type"], source=source).exclude(identifier__in=imported_identifiers) stale_count = stale.count() stale.delete() stale_time_end = perf_counter() self.stdout.write( "Removed stale count {}. Execution time: {:.2f}s\n".format( stale_count, stale_time_end - stale_time_start)) if errors: self.stdout.write(" {} errors:\n".format(len(errors))) for error in errors: self.stdout.write(error) type_end = perf_counter() self.stdout.write( 'The area import of type "{}" is completed. Execution time: {:.2f}s\n' .format(area_import_type, (type_end - type_start))) func_end = perf_counter() self.stdout.write( "The area import is completed. Execution time: {0:.2f}s\n".format( func_end - func_start))
def import_shapes(shapefiles, source): adm_scheme, _ = SkosConceptScheme.objects.get_or_create( dc_title="administrative_unit") temp_spatial_ids = [] for x in shapefiles: df = gp.read_file(x).to_crs({ 'proj': 'longlat', 'ellps': 'WGS84', 'datum': 'WGS84' }) for i, row in df.iterrows(): add_data = {} if row['geometry'].geom_type == 'MultiPolygon': mp = row['geometry'].wkt else: mp = geos.MultiPolygon(fromstr(row['geometry'].wkt)) spat, _ = TempSpatial.objects.get_or_create( start_date=row['start_date'], end_date=row['end_date'], date_accuracy=row['date_acc'], geom=mp) try: try: adm, _ = SkosConcept.objects.get_or_create( pref_label=row['adm_type']) except IntegrityError: adm, _ = SkosConcept.objects.get_or_create( pref_label="unknown adm") except KeyError: adm, _ = SkosConcept.objects.get_or_create( pref_label="unknown adm") if adm is not None: adm.scheme.add(adm_scheme) spat.administrative_unit = adm try: if row['name']: spat.name = row['name'] except KeyError: pass try: if row['name_alt']: spat.alt_name = row['name_alt'] except KeyError: pass try: if row['id']: spat.orig_id = row['id'] except KeyError: pass try: if row['wiki_id']: spat.wikidata_id = row['wiki_id'] except KeyError: pass for x in list(df.keys()): if x not in mandatory_keys: add_data[x] = row[x] spat.additional_data = json.dumps(add_data) spat.source = source try: spat.save() temp_spatial_ids.append(spat.id) except Exception as e: print(e) return temp_spatial_ids
def execute(self): # NOQA C901 cursor = self.cursor django_cursor = django_connection.cursor() errors = [] area_import = AREA_IMPORT_TYPES["lease_area"] self.stdout.write(area_import["source_name"]) (source, source_created) = AreaSource.objects.get_or_create( identifier=area_import["source_identifier"], defaults={"name": area_import["source_name"]}, ) cursor.execute(area_import["query"]) lease_area_rows = rows_to_dict_list(cursor) count = 0 for row in lease_area_rows: metadata = { METADATA_COLUMN_NAME_MAP[column_name]: row.get(column_name.upper()) for column_name in area_import["metadata_columns"] } match_data = { "type": area_import["area_type"], "identifier": row.get(area_import["identifier_field_name"]), "external_id": row.get("ID"), "source": source, } # There is no SRID info in the data. Assume it's 3879. wkt_geom_with_srid = "SRID=3879;{}".format( row.get("WKT_GEOM").read()) # Convert possible curves to lines in the geometry and transform the geometry to SRID 4326 django_cursor.execute( "SELECT ST_AsText(ST_Transform(ST_CurveToLine(ST_GeomFromEWKT(%s)), 4326));", [wkt_geom_with_srid], ) geom_text = django_cursor.fetchone()[0] try: geom = geos.GEOSGeometry(geom_text) except geos.error.GEOSException as e: errors.append("id #{} error: " + str(e)) count += 1 self.stdout.write("E", ending="") if count % 100 == 0: self.stdout.write(" {}".format(count)) self.stdout.flush() # self.stdout.write(str(e)) continue if geom and isinstance(geom, geos.Polygon): geom = geos.MultiPolygon(geom) if geom and not isinstance(geom, geos.MultiPolygon): errors.append( "id #{} error: " + ' Error! Geometry is not a Multipolygon but "{}"\n'.format( geom)) count += 1 self.stdout.write("E", ending="") if count % 100 == 0: self.stdout.write(" {}".format(count)) self.stdout.flush() # self.stdout.write(' Error! Geometry is not a Multipolygon but "{}"\n'.format(geom)) continue other_data = {"geometry": geom, "metadata": metadata} Area.objects.update_or_create(defaults=other_data, **match_data) count += 1 self.stdout.write(".", ending="") if count % 100 == 0: self.stdout.write(" {}".format(count)) self.stdout.flush() self.stdout.write(" Count {}\n".format(count)) if errors: self.stdout.write(" {} errors:\n".format(len(errors))) for error in errors: self.stdout.write(error)
def save(self, **kwargs): from regionfestival.snippets import unique_slugify unique_slugify(self, self.name) if self.boundaries and isinstance(self.boundaries, geos.Polygon): self.boundaries = geos.MultiPolygon(self.boundaries) super(Region, self).save(**kwargs)
def execute(self): # NOQA C901 if not self.area_types: self.area_types = AREA_IMPORT_TYPES.keys() errors = [] for area_import_type in self.area_types: area_import = AREA_IMPORT_TYPES[area_import_type] try: conn = psycopg2.connect( getattr(settings, area_import['source_dsn_setting_name']), cursor_factory=psycopg2.extras.NamedTupleCursor) except (psycopg2.ProgrammingError, psycopg2.OperationalError) as e: self.stderr.write(str(e)) self.stderr.write( 'Could not connect to the database when importing area type "{}". DSN setting name "{}"' .format(area_import_type, area_import['source_dsn_setting_name'])) continue cursor = conn.cursor() self.stdout.write(area_import['source_name']) (source, source_created) = AreaSource.objects.get_or_create( identifier=area_import['source_identifier'], defaults={'name': area_import['source_name']}) cursor.execute(area_import['query']) count = 0 for row in cursor: metadata = { METADATA_COLUMN_NAME_MAP[column_name]: getattr(row, column_name) for column_name in area_import['metadata_columns'] } match_data = { 'type': area_import['area_type'], 'identifier': getattr(row, area_import['identifier_field_name']), 'external_id': row.id, 'source': source, } try: geom = geos.GEOSGeometry(row.geom_text) except geos.error.GEOSException as e: errors.append('id #{} error: ' + str(e)) count += 1 self.stdout.write('E', ending='') if count % 100 == 0: self.stdout.write(' {}'.format(count)) self.stdout.flush() # self.stdout.write(str(e)) continue if geom and isinstance(geom, geos.Polygon): geom = geos.MultiPolygon(geom) if geom and not isinstance(geom, geos.MultiPolygon): errors.append( 'id #{} error: ' + ' Error! Geometry is not a Multipolygon but "{}"\n'. format(geom)) count += 1 self.stdout.write('E', ending='') if count % 100 == 0: self.stdout.write(' {}'.format(count)) self.stdout.flush() # self.stdout.write(' Error! Geometry is not a Multipolygon but "{}"\n'.format(geom)) continue other_data = { 'geometry': geom, 'metadata': metadata, } Area.objects.update_or_create(defaults=other_data, **match_data) count += 1 self.stdout.write('.', ending='') if count % 100 == 0: self.stdout.write(' {}'.format(count)) self.stdout.flush() self.stdout.write(' Count {}\n'.format(count)) if errors: self.stdout.write(' {} errors:\n'.format(len(errors))) for error in errors: self.stdout.write(error)
def handle(self, *args, **options): feature_index = 'q' features = [] locations = [] while feature_index.lower() == 'q': original_url = 'https://s3.amazonaws.com/osm-polygons.mapzen.com/' countries = requests.get(original_url).text countries = xmltodict.parse(countries) for i, c in enumerate( countries['ListBucketResult']['Contents'][1:], start=1): print("{}: {}".format(i, c['Key'])) print("Press <return> to exit\n") country_index = input('Enter selected country: ') if not country_index: return selected = countries['ListBucketResult']['Contents'][int( country_index)] response = requests.get(original_url + selected['Key']) tar_file = tarfile.open(mode='r:gz', fileobj=BytesIO(response.content)) tmpdir = tempfile.mkdtemp() tar_file.extractall(tmpdir) json_file_path = os.path.join(tmpdir, os.listdir(tmpdir)[0]) all_files = sorted(os.listdir(json_file_path)) for i, f in enumerate(all_files, start=1): file_size = os.path.getsize(os.path.join(json_file_path, f)) print("{}: {} ({})".format(i, f, file_size)) print("Press Q to restart and <return> to exit\n") level_index = input('Enter selected file: ') if level_index.lower() == 'q': continue if not level_index: return selected_file = all_files[int(level_index) - 1] with open(os.path.join(json_file_path, selected_file)) as f: current_json = f.read() content = json.loads(current_json) features = content['features'] shutil.rmtree(tmpdir) locations = sorted( [a for a in features if 'name' in a['properties']], key=lambda x: x['properties']['name']) for i, l in enumerate(locations): print("{}: {} ({})".format( i + 1, l['properties']['name:en'] if 'name:en' in l['properties'] else '', l['properties']['name'])) print("Press Q to restart and <return> to exit\n") feature_index = input('Enter selected feature: ') if not feature_index: return if not features or not locations: return if feature_index.lower() == 'all': to_be_merged = [ geos.GEOSGeometry(json.dumps(a['geometry'])) for a in features ] polys = [] for p in to_be_merged: if type(p) == geos.Polygon: polys.append(p) elif type(p) == geos.MultiPolygon: polys = polys + list(p) feature = geos.MultiPolygon(polys) elif ',' in feature_index: indices = [int(a) for a in feature_index.split(',')] to_be_merged = [ geos.GEOSGeometry(json.dumps(locations[a - 1]['geometry'])) for a in indices ] polys = [] for p in to_be_merged: if type(p) == geos.Polygon: polys.append(p) elif type(p) == geos.MultiPolygon: polys = polys + list(p) feature = geos.MultiPolygon(polys) else: feature_index = int(feature_index) print(locations[feature_index - 1]['properties']['name']) feature_json = json.dumps(locations[feature_index - 1]['geometry']) feature = geos.GEOSGeometry(feature_json) is_new_location = input('Is this a new location? (y/N) ') is_new_location = True if 'Y' in is_new_location.upper() else False # merged = feature.convex_hull if len(feature) > 3 else feature[0] # merged = merged.simplify(tolerance=0.01, preserve_topology=True) if is_new_location: location_name = input('Name of location: ') area_type = input( 'Area Type (1: Country; 2: Region): [1/2] ') or '1' area_type = int(area_type) location_slug = slugify(location_name) parent_id = input('Parent Id: ') location = models.GeographicRegion.objects.create( name=location_name, parent_id=int(parent_id) if parent_id else None, geom=feature, level=area_type, slug=location_slug, ) else: location_id = input('Location Id: ') location = models.GeographicRegion.objects.get(id=int(location_id)) location.geom = feature location.save()
def convert2multi(validated_data, geo_field): geom = validated_data.get(geo_field, None) if geom and isinstance(geom, geos.Polygon): geom = geos.MultiPolygon(geom) validated_data[geo_field] = geom
def test_point_within_mpolygon_double(self): create_points() poly1, poly2 = create_polygons() mpoly = geos.MultiPolygon(poly1, poly2) features = Feature.objects.filter(geom_point__within=mpoly) self.assertEqual(len(features), 2)