def build_aggregate(level, code, name, territories): print 'Building aggregate "{0}" (level={1}, code={2})'.format(name, level, code) polygons = [] for tlevel, tcode in territories: try: territory = Territory.objects.get(level=tlevel, code=tcode) except Territory.DoesNotExist: print 'Territory {0}/{1} not found'.format(tlevel, tcode) continue if not shape(territory.geom).is_valid: print 'Skipping invalid polygon for {0}'.format(territory.name) continue if shape(territory.geom).is_empty: print 'Skipping empty polygon for {0}'.format(territory.name) continue polygons.append(territory.geom) geom = cascaded_union([shape(p) for p in polygons]) if geom.geom_type == 'Polygon': geom = MultiPolygon([geom]) try: territory = Territory.objects.get(level=level, code=code) except Territory.DoesNotExist: territory = Territory(level=level, code=code) territory.name = name territory.geom = geom.__geo_interface__ territory.save()
def apply_shapely(self, method, args=None, call=True, out_geomtype=None, **kwargs): coll = self.collection() out_schema = coll.schema.copy() if not args: args = [] if out_geomtype: out_schema['geometry'] = out_geomtype tempds = self.tempds(method) with fiona.collection(tempds, "w", "ESRI Shapefile", out_schema, crs=self.crs) as out_collection: for in_feature in coll: out_feature = in_feature.copy() if call: geom = mapping( getattr(shape(in_feature['geometry']), method)(*args, **kwargs) ) else: # it's not a method, it's a property geom = mapping( getattr(shape(in_feature['geometry']), method) ) out_feature['geometry'] = geom out_collection.write(out_feature) return Layer(tempds)
def makeShape(code): #takes a 2-letter code geom = findCountry(code) if geom['type'] == 'Polygon': p=Polygon(shape(geom)) elif geom['type'] == 'MultiPolygon': p=MultiPolygon(shape(geom)) return p
def checkPos(lat, lon): with open('5_mile_airport.json', 'r') as f: js = json.load(f) point = Point(lon, lat) for feature in js['features']: polygon = shape(feature['geometry']) if polygon.contains(point): return json.JSONEncoder().encode({ "safe" : "false"}) with open('unitedstates.json', 'r') as f: js = json.load(f) inUS = 0 for feature in js['features']: polygon = shape(feature['geometry']) if polygon.contains(point): inUS = 1 break; if inUS == 1: with open('us_military.json', 'r') as f: js = json.load(f) for feature in js['features']: polygon = shape(feature['geometry']) if polygon.contains(point): return json.JSONEncoder().encode({ "safe" : "false"}) with open('us_national_park.json', 'r') as f: js = json.load(f) for feature in js['features']: polygon = shape(feature['geometry']) if polygon.contains(point): return json.JSONEncoder().encode({ "safe" : "false"}) return json.JSONEncoder().encode({ "safe" : "true"}) else: return json.JSONEncoder().encode({ "safe" : "caution"})
def location_listener(self, name, location): global telem_data, last_point, vehicle, last_location telem_data['params']['location'] = {"lat": location.global_frame.lat, "lon": location.global_frame.lon, "alt": location.global_frame.alt} # print vehicle.location.global_frame current_point = Point(location.global_frame.lon, location.global_frame.lat) telem_data['params']['nofly_zone_status']['inside'] = False telem_data['params']['nofly_zone_status']['indexes'] = [] for i, nofly_zone in enumerate(nofly_polygons): poly = shape(nofly_zone['data']['geometry']) if poly.contains(current_point): telem_data['params']['nofly_zone_status']['inside'] = True telem_data['params']['nofly_zone_status']['indexes'].append(nofly_zone['id']) further_check = False try: if global_multipoly.contains(current_point): further_check = True except: further_check = True if further_check: for i, nofly_zone in enumerate(global_nofly_polygons): poly = shape(nofly_zone['data']['geometry']) if poly.contains(current_point): telem_data['params']['nofly_zone_status']['inside'] = True telem_data['params']['nofly_zone_status']['indexes'].append(nofly_zone['id']) if telem_data['params']['nofly_zone_status']['inside'] is False: last_point = current_point last_location = location.global_frame
def read_geofabric_data(netGDB,bbox=None): catch = {} with fiona.open(netGDB, layer='AHGFCatchment') as c: for feat in c.items(bbox=bbox): geom = shape(feat[1]['geometry']) cid = feat[1]['properties']['HydroID'] assert cid not in catch #shouldnt be duplicates catch[cid] = geom DG=nx.MultiDiGraph() with fiona.open(netGDB, layer='AHGFNetworkStream') as c: for feat in c.items(bbox=bbox): streamLink = shape(feat[1]['geometry']) #for some reason these are coming in as multipart features with only one part - no need for this assert streamLink.type == 'MultiLineString' assert len(streamLink.geoms) == 1 streamLink = streamLink.geoms[0] ##remove this - just here for testing #if streamLink.representative_point().y > -40.5: #tasmania for testing # continue sid = feat[1]['properties']['HydroID'] cid = feat[1]['properties']['DrainID'] fid = feat[1]['properties']['From_Node'] tid = feat[1]['properties']['To_Node'] subCatch = catch.get(cid,Polygon()) DG.add_edge(fid, tid, id=sid,cid=cid,subCatch=subCatch,stream=streamLink) return DG
def get_light_rail_sections(): """""" # units derived from srs of input data buffer_dist = 500 sect_dict = dict() sections = fiona.open(SECTIONS) sect_ix = generate_spatial_index(sections) with fiona.open(RAIL_LINES) as rail_lines: for rail_id, rail_feat in rail_lines.items(): rail_type = rail_feat['properties']['TYPE'] if 'MAX' in rail_type: rail_geom = shape(rail_feat['geometry']) rail_buffer = rail_geom.buffer(buffer_dist) for sect_id in sect_ix.intersection(rail_buffer.bounds): # for some reason fiona sees integers and long # integers of the same value as not equal sect_id = int(sect_id) if sect_id not in sect_dict: sect_feat = sections[sect_id] sect_geom = shape(sect_feat['geometry']) if rail_buffer.intersects(sect_geom): sect_name = sect_feat['properties']['SECTION'] sect_dict[sect_id] = sect_name return sect_dict.values()
def test_with_initial(self): Fake, FakeForm = self.factory() territories = [TerritoryFactory() for _ in range(3)] fake = Fake(spatial=SpatialCoverage( territories=[t.reference() for t in territories], granularity=SPATIAL_GRANULARITIES.keys()[1] )) territory = TerritoryFactory() data = MultiDict({ 'spatial-territories': str(territory.id), 'spatial-granularity': VALID_GRANULARITY }) form = FakeForm(data, fake) form.validate() self.assertEqual(form.errors, {}) form.populate_obj(fake) self.assertEqual(len(fake.spatial.territories), 1) self.assertEqual(fake.spatial.territories[0], territory.reference()) self.assertTrue(shape(fake.spatial.geom).equals(shape(territory.geom)))
def zips_in_DMA(one_dma_id, to_txt = False): #for both zip codes and DMAs, extract properties and geometry from each geojson feature, add each one as a dictionary entry with the DMA ID or zip # as the key and a shapely object as the value zips_ = {} for i in zips.features: zips_[str(i['properties']['ZCTA5CE10'])] = shape(geojson.MultiPolygon(i['geometry']['coordinates'])) dmas_errors = [] dmas_ = {} for i in nielsen.features: try: dmas_[str(i['properties']['id'])] = shape(geojson.Polygon(i['geometry']['coordinates'])) except: try: dmas_[str(i['properties']['id'])] = shape(geojson.Polygon(i['geometry']['coordinates'][0])) except: print "error" dmaziplist = [] for i in zips_.iteritems(): if dmas_[one_dma_id].intersects(i[1]): dmaziplist.append(i[0]) else: pass dma_filename = dmametadata.ix[one_dma_id]['dma_name'].replace(',', '').replace(' ', '') + one_dma_id if to_txt: with open(dma_filename + '.txt', 'wb') as zipfile: for i in dmaziplist: zipfile.write(i) zipfile.write('\n') return dmaziplist
def import_shoreline_file(self, lake_name, shoreline_file): """ Load the shoreline from GIS file. NB: Currently has side effects, loading crs and properties traits. """ with fiona.open(shoreline_file) as f: crs = f.crs geometries = [] for rec in f: geometries.append(rec['geometry']) # XXX: assuming that the properties aren't varying by geometry properties = rec['properties'] if len(geometries) == 1: geom = shape(geometries[0]) else: # XXX: this assumes we'll always get lines, not polygons or other geom = MultiLineString([ shape(geometry) for geometry in geometries]) with self._open_file('a') as f: shoreline_group = self._get_shoreline_group(f) shoreline_group._v_attrs.crs = self._safe_serialize(crs) shoreline_group._v_attrs.lake_name = self._safe_serialize(lake_name) shoreline_group._v_attrs.original_shapefile = self._safe_serialize(shoreline_file) shoreline_group._v_attrs.properties = self._safe_serialize(properties) geometry_str = self._safe_serialize(mapping(geom)) self._write_array(f, shoreline_group, 'geometry', np.array(geometry_str)) f.flush()
def main(infile, outfile, driver): with fio.open(infile) as src: meta = src.meta meta['driver'] = driver with fio.open(infile) as src, fio.open(outfile, 'w', **meta) as dst: with click.progressbar(src) as features: for feat in features: east = deepcopy(feat) west = deepcopy(feat) east_geom = shape(east['geometry']) west_geom = shape(west['geometry']) # if 'Point' not in asShape(feat['geometry']).type: # east_geom = east_geom.simplify(0.0001).buffer(0) # west_geom = west_geom.simplify(0.0001).buffer(0) east_geom = translate(east_geom, xoff=180) west_geom = translate(west_geom, xoff=-180) if not east_geom.is_empty: east['geometry'] = mapping(east_geom) dst.write(east) if not west_geom.is_empty: west['geometry'] = mapping(west_geom) dst.write(west)
def process_web_inputs(task, inputs): for param in task.inputs: if param.name in inputs: if isinstance(param, (params.RasterParameter, params.NdArrayParameter)): inputs[param.name] = params.RegisteredDatasetParameter(param.name).clean(inputs[param.name]) elif isinstance(param, params.FeatureParameter): try: inputs[param.name] = shape(inputs[param.name]) except (ValueError, AttributeError, KeyError): raise ParameterNotValidError elif isinstance(param, params.ListParameter): if isinstance(param.param_type, (params.RasterParameter, params.NdArrayParameter)): inputs[param.name] = [ params.RegisteredDatasetParameter(param.name).clean(x) for x in inputs[param.name] ] elif isinstance(param.param_type, params.FeatureParameter): try: inputs[param.name] = [shape(x) for x in inputs[param.name]] except (ValueError, AttributeError, KeyError): raise ParameterNotValidError return task.validate_inputs(inputs)
def segQuality(inVector, inImage): # open the vector lyr = fiona.open(inVector) features = [x for x in lyr] values = np.zeros([len(features), 5], dtype=float) # loop over features for i in range(len(features)): geometry1 = shape(features[i]['geometry']) restFeatures = features[:i] + features[(i+ 1):] value = zonal_stats(geometry1, inImage, stats=['count'], add_stats={'mymean':mymean, "myvarianza":varianza } ) df = pd.DataFrame.from_dict(value, orient='columns', dtype=None) for j in range(len(restFeatures)): geometry2 = shape(features[j]['geometry']) if geometry2.intersects(geometry1) == True: #print("They touch") value = zonal_stats(geometry2, inImage, stats=['count'], add_stats={'mymean':mymean, "myvarianza":varianza } ) df = df.append(pd.DataFrame.from_dict(value, orient='columns', dtype=None)) values[i,0] = df.iloc[0,0] # count values[i,1] = df.iloc[0,1] # mean values[i,2] = df.iloc[0,2] # myvarianza values[i,3] = np.var(df.iloc[:,1]) # varianza between values[i,4] = len(df.iloc[1:]) # neighbours # get overal values intraVarWeighted = np.nansum( values[:,0]*values[:,2] ) / np.nansum(values[:,0]) interVarWeighted = np.nansum( values[:,4]*values[:,3] ) / np.nansum(values[:,4]) normVariance = (intraVarWeighted - interVarWeighted) / (intraVarWeighted + interVarWeighted) numberSegments = len(values[:,4]) return( intraVarWeighted, interVarWeighted, normVariance, numberSegments )
def test_PointInPolygon(self): """ Test that a feature's point geometry is within its polygon geometry. NOTE: A feature that does not contain a polygon geometry will be skipped. """ features_point_outside_polygon = {} for record in self.test_data: geoms = record['geometry']['geometries'] pt_geom = None poly_geom = None # Check for presence of point and polygon geoms. for geom in geoms: if geom['type'] == 'Point': pt_geom = shape(geom) if geom['type'] in ('MultiPolygon', 'Polygon'): poly_geom = shape(geom) if pt_geom and poly_geom: # Check if point is within polygon. If not, flag it. try: if not pt_geom.within(poly_geom): composite_key = self._get_comp_primary_key(record) features_point_outside_polygon[composite_key] = geoms except Exception as e: continue json_data = json.dumps(features_point_outside_polygon) return json_data
def surroundingNeighborhoods(neighborhood): neighborhood_shape = shape(neighborhood['geometry']) surrounding_neighborhoods = [] for other in SF['features']: if neighborhood_shape.touches(shape(other['geometry'])): surrounding_neighborhoods.append(other['properties']['neighborho']) return surrounding_neighborhoods
def test_PointInPolygonGeom(self): """ Test that a feature's point geometry is within its polygon geometry. NOTE: A feature that does not contain a polygon geometry will be skipped. """ features_point_outside_polygon = [] for record in self.test_data: geoms = record['geometry']['geometries'] pt_geom = None poly_geom = None # Check for presence of point and polygon geoms. for geom in geoms: if geom['type'] == 'Point': pt_geom = shape(geom) if geom['type'] in ('MultiPolygon', 'Polygon'): poly_geom = shape(geom) if pt_geom and poly_geom: # Check if point is within polygon. If not, flag it. try: if not pt_geom.within(poly_geom): composite_key = self._get_comp_primary_key(record) features_point_outside_polygon.append(composite_key) except Exception as e: continue # Raise error if we don't have an empty error list self.assertEqual(len(features_point_outside_polygon), 0, msg="The following features have point geoms outside of polygon geoms %s" % features_point_outside_polygon)
def test_overlappingPolygons(self): """ Test to ensure there are No two polygons of same Class and Mapcode overlay """ #Build a spatial index based on the bounding boxes of the polygons from rtree import index idx = index.Index() count = -1 #iterating through input data to index polygon_shapes = [] self.logger.info("Building the spatial index based on bounding boxes...") for record in self.test_data: if 'geometry' in record.keys(): geoms = record['geometry']['geometries'] # Check for presence of polygon geoms. for geom in geoms: if geom['type'] in ('MultiPolygon', 'Polygon'): class_mapcode_polygons = {'id': record['properties']['id'], 'class': record['properties']['class'], 'map_code': record['properties']['map_code'], 'polygon': shape(geom)} polygon_shapes.append(class_mapcode_polygons) count +=1 idx.insert(count, shape(geom).bounds) #check for overapping polygons features_with_overlapping_polygons = [] ids_with_topology_error = [] self.logger.info("Verifying overlapping polygons...") chumma_count = 0 for data in polygon_shapes: for key in idx.intersection(data['polygon'].bounds): if(data['map_code'] == polygon_shapes[key]['map_code'] and data['class'] == polygon_shapes[key]['class'] and polygon_shapes[key]['polygon'] != data['polygon']): #verifying overlap try: feature_1 = str(polygon_shapes[key]['id']) + '_' + str(polygon_shapes[key]['map_code']) feature_2 = str(data['id']) + '_' + str(data['map_code']) if polygon_shapes[key]['polygon'].overlaps(data['polygon']): if feature_2 + ";" + feature_1 not in features_with_overlapping_polygons: features_with_overlapping_polygons.append(feature_1 + ";" + feature_2) except Exception as e: error_dict = { 'composite_keys': feature_1 + ";" + feature_2, 'error_string': str(e) } ids_with_topology_error.append(error_dict) continue # Print ids that caused topology errors if len(ids_with_topology_error) is not 0: self._print_ids_with_topology_error("These are a list of IDs that have TopologicalErrors that COULD NOT be checked for overlaps", ids_with_topology_error) # Raise error if we don't have an empty error list self.assertEqual(len(features_with_overlapping_polygons), 0, msg="The following features have overlapping polygons %s" % features_with_overlapping_polygons)
def test_prime_meridian_footprint(): bbox = [(-20, 0), (20, 0), (20, 5), (-20, 5)] good_poly = MultiPolygon( [Polygon(bbox + [bbox[0]])] ) footprints = Footprints(bbox) assert shape(footprints.data_polygon).area == good_poly.area # Since the shape is rectangular, these should be the same assert shape(footprints.tile_polygon).area == shape(footprints.data_polygon).area
def __init__(self, osmid, name, geometry, **kwargs): self.osmid = osmid self.name = name self.geometries = [geometry['coordinates']] self.oneway_length = 0 self.total_length = shape(geometry).length self.properties = kwargs if self.properties['oneway']: self.oneway_length = shape(geometry).length
def find_containing_poly(point, polyfile): sample_poly = fiona.open(polyfile,"r") poly = sample_poly.next() spoly = shape(poly["geometry"]) while (not point.within(spoly)): poly = sample_poly.next() spoly = shape(poly["geometry"]) sample_poly.close return spoly
def build_aggregate(self, code, name, zones, properties, db): geoms = [] populations = [] areas = [] if callable(zones): zones = [zone['_id'] for zone in zones(db)] for zoneid in zones: # Resolve wildcard if zoneid.endswith(':*'): level = zoneid.replace(':*', '') ids = db.distinct('_id', {'level': level}) resolved = self.build_aggregate( code, name, ids, properties, db) geoms.append(shape(resolved['geom'])) if resolved.get('population'): populations.append(resolved['population']) if resolved.get('area'): areas.append(resolved['area']) else: zone = db.find_one({'_id': zoneid}) if not zone: warning('Zone {0} not found'.format(zoneid)) continue if 'geom' not in zone: warning('Zone {0} without geometry'.format(zone['name'])) continue shp = shape(zone['geom']) if not shp.is_valid: warning(('Skipping invalid polygon for {0}' '').format(zone['name'])) continue if shp.is_empty: warning('Skipping empty polygon for {0}', zone['name']) continue geoms.append(shp) if zone.get('population'): populations.append(zone['population']) if zone.get('area'): areas.append(zone['area']) if geoms: geom = aggregate_multipolygons(geoms).__geo_interface__ else: geom = None warning('No geometry for {0}', zones) data = { '_id': ':'.join((self.id, code)), 'code': code, 'level': self.id, 'name': name, 'population': sum(populations), 'area': sum(areas), 'geom': geom } data.update(properties) return data
def get_place_region(place_geojson, regions_geojson): place_geom = shape(place_geojson['geometry']) for feature in regions_geojson['features']: region_poly = shape(feature['geometry']) if region_poly.contains(place_geom): return feature return None
def shp2shapely(filepath): """ Read local shapefile and convert to shapely object for easy processing :param filepath: Path to Shapefile :return: Shapely object """ # Todo # -> Should have same structure as view...? with fiona.open(filepath) as shp: shape(shp[0]['geometry'])
def __init__(self, regions_file=REGIONS_FILE, buffer_file=REGIONS_BUFFER_FILE): self._buffered_shapes = {} self._prepared_shapes = {} self._shapes = {} self._tree_ids = {} self._radii = {} with util.gzip_open(regions_file, 'r') as fd: regions_data = simplejson.load(fd) genc_regions = frozenset([rec.alpha2 for rec in genc.REGIONS]) for feature in regions_data['features']: code = feature['properties']['alpha2'] if code in genc_regions: shape = geometry.shape(feature['geometry']) self._shapes[code] = shape self._prepared_shapes[code] = prepared.prep(shape) self._radii[code] = feature['properties']['radius'] with util.gzip_open(buffer_file, 'r') as fd: buffer_data = simplejson.load(fd) i = 0 envelopes = [] for feature in buffer_data['features']: code = feature['properties']['alpha2'] if code in genc_regions: shape = geometry.shape(feature['geometry']) self._buffered_shapes[code] = prepared.prep(shape) # Collect rtree index entries, and maintain a separate id to # code mapping. We don't use index object support as it # requires un/pickling the object entries on each lookup. if isinstance(shape, geometry.base.BaseMultipartGeometry): # Index bounding box of individual polygons instead of # the multipolygon, to avoid issues with regions crossing # the -180.0/+180.0 longitude boundary. for geom in shape.geoms: envelopes.append((i, geom.envelope.bounds, None)) self._tree_ids[i] = code i += 1 else: envelopes.append((i, shape.envelope.bounds, None)) self._tree_ids[i] = code i += 1 props = index.Property() props.fill_factor = 0.9 props.leaf_capacity = 20 self._tree = index.Index(envelopes, interleaved=True, properties=props) for envelope in envelopes: self._tree.insert(*envelope) self._valid_regions = frozenset(self._shapes.keys())
def get_adm_geom(self, point, adm_level, iso3=None): """ """ tmp_int = int(adm_level) tmp_pnt = Point(point) query = {} if iso3 is None: query['tags'] = 'adm{0}_{1}'.format( tmp_int, self.adm_suffix) else: query['datasets'] = '{0}_adm{1}_{2}'.format( iso3.lower(), tmp_int, self.adm_suffix) if iso3 is None or tmp_int != 0: query['geometry'] = { '$geoIntersects': { '$geometry': { 'type': "Point", 'coordinates': [tmp_pnt.x, tmp_pnt.y] } } } # force use master prod db even for dev (temporary) f_client = pymongo.MongoClient("128.239.20.76") results = list(f_client.asdf.features.find(query)) if len(results) == 1: tmp_results = results[0] tmp_adm_geom = shape(tmp_results['geometry']) tmp_iso3 = tmp_results['datasets'][0][:3] elif len(results) == 0: warn('no adm (adm level {0}) geom found for ' 'point ({1})'.format(tmp_int, tmp_pnt)) tmp_adm_geom = "None" tmp_iso3 = None else: warn('multiple adm (adm level {0}) geoms found for ' 'point ({1})'.format(tmp_int, tmp_pnt)) # tmp_adm_geom = "None" tmp_results = results[0] tmp_adm_geom = shape(tmp_results['geometry']) tmp_iso3 = tmp_results['datasets'][0][:3] return tmp_adm_geom, tmp_iso3
def __createShapes__(self): shapes = [] if 'features' in self.__jsonData__: for feature in self.__jsonData__['features']: if feature['geometry']: shapes.append(shape(feature['geometry'])) elif 'geometries' in self.__jsonData__: for geometry in self.__jsonData__['geometries']: shapes.append(shape(geometry)) return shapes
def test_footprint_shift(): bbox = [(179, 1), (-179, 1), (-179, 2), (179, 2)] bad_poly = MultiPolygon( [Polygon(bbox + [bbox[0]])] ) footprints = Footprints(bbox) data_area = shape(footprints.data_polygon).area assert data_area < bad_poly.area assert data_area == 2 # Since the shape is rectangular, these should be the same assert data_area == shape(footprints.data_polygon).area
def setup(self): po1 = [(1,1), (2,2), (1,3), (1,4), (3,3), (3,1), (1,1)] po2 = [(1,4), (1,5), (3,4), (3,3), (1,4)] po3 = [(3,1), (3,3), (5,5), (7,3), (5,1), (3,1)] ZIPS = { '1': [shape({'type': 'polygon', 'coordinates': [po1]})], '2': [shape({'type': 'polygon', 'coordinates': [po2]})], '3': [shape({'type': 'polygon', 'coordinates': [po3]})] } self.ZIPS = ZIPS
def test_image_window_at(self): wv2 = CatalogImage('1030010076B8F500') c1 = shape(wv2).centroid window = wv2.window_at(c1, (256, 256)) c2 = shape(window).centroid bands, x, y = window.shape # check the window is the correct shape self.assertEquals(x, 256) self.assertEquals(y, 256) # make sure the center of the window is within 1 pixel # of where it should be self.assertTrue(c1.distance(c2) < wv2.metadata['georef']['scaleX'])
def _generate_points(self, polygon: Polygon, n) -> list: """Generates sample points within a given geometry :param shapely.geometry.Polygon polygon: the polygon to create points in :param int n: Number of points to generate in polygon :return: A list with point in the polygon :rtype: list[Point] """ if n <= 0: return [] if polygon.area <= 0: return [] bbox = polygon.envelope """(minx, miny, maxx, maxy) bbox""" if (polygon.area * self.t) < bbox.area: if (bbox.bounds[2] - bbox.bounds[0]) > (bbox.bounds[3] - bbox.bounds[1]): bbox_1 = box(*self._bbox_left(bbox.bounds)) bbox_2 = box(*self._bbox_right(bbox.bounds)) else: bbox_1 = box(*self._bbox_bottom(bbox.bounds)) bbox_2 = box(*self._bbox_top(bbox.bounds)) p1 = shape(polygon) p1 = p1.difference(bbox_1) p2 = shape(polygon) p2 = p2.difference(bbox_2) del bbox_1, bbox_2 # k = bisect.bisect_left(u, p1.area / polygon.area) k = int(round(n * (p1.area / polygon.area))) v = self._generate_points(p1, k) + self._generate_points(p2, n - k) del polygon, p1, p2 else: v = [] max_iterations = self.t * n + 5 * math.sqrt(self.t * n) v_length = len(v) while v_length < n and max_iterations > 0: max_iterations -= 1 v.append(self._random_point_in_polygon(polygon)) v_length = len(v) if len(v) < n: raise Exception('Too many iterations') self.logging.debug('Generated %s points', n) del bbox return v
def _force_polygon_ccw(geometry): polygon = shape(geometry) return mapping(orient(polygon))
def calculate_hydro(self, interval=1000, head_distance=500, get_range=False): """ Start at the top of each river, get source GSCDxFlowAccxsquare size. Calculates flow rate in m3/s, head in m and power in Watts. Parameters ---------- interval : int, optional (default 1000.) Interval in metres between testing points. head_distance : int, optional (default 500.) How far upstream to look to calculate head. get_range : boolean, optional (default False.) Whether to attempt to calculate mean, median, max etc. """ # TODO: Make the contribution % dependent on slope (elevation change/distance). # Choose the point_interval in metres # This loops through each stream and adds points down it at the specified interval # And creates a dict with these new point geometries # At 20 deg South: # 1 degree = 110704 metres -> 1 minute = 1845 metres -> 1 second = 30.75 metres # River network is 15 second resolution = 461 metres # Therefore each up_cell size is cell_area = (110704/60/60*15)**2 proj = pyproj.Proj(init='epsg:3395') # Mercator rivers = self.rivers.to_crs(epsg=3395) rho = 998.57 # density of water, kg/m3 g = 9.81 # acceleration due to gravity, m/2 n = 0.5 # overall efficiency of system hydro_points_dicts = [] count = 0 for _, row in rivers.iterrows(): geom = shape(row['geometry']) length = geom.length for _, distance in enumerate(range(0, int(length), interval)): arcid = row['arcid'] up_cells = row['up_cells'] point = Point(proj(*list(geom.interpolate(distance).coords)[0], inverse=True)) upstream_point = Point(proj(*list(geom.interpolate(distance - head_distance).coords)[0], inverse=True)) elevation = next(self.dem.sample(list(point.coords))).tolist()[0] upstream_elevation = next(self.dem.sample(list(upstream_point.coords))).tolist()[0] head = upstream_elevation - elevation runoff = next(self.flow.sample(list(point.coords))).tolist()[0] flowrate = runoff * up_cells * cell_area * (1/1000) * (1/(8760*3600)) # convert mm/year to m3/s power = rho*g*n*flowrate*head if head > 0 and flowrate > 0: hydro_points_dicts.append({'arcid': arcid, 'elevation': elevation, 'head': head, 'flowrate': flowrate, 'power': power, 'geometry': point}) count += 1 if count % 10000 == 0: print(count) self.hydro_points = gpd.GeoDataFrame(hydro_points_dicts) self.hydro_points.crs = {'init' :'epsg:4326'} print('Calculated hydro potential') if get_range: eta_t=0.88 eta_g=0.96 conv=0.6 # transfer discharge values from the river to the points self.hydro_points['discharge_accum'] = -99 self.hydro_points['discharge_max'] = -99 self.hydro_points['discharge_mean'] = -99 self.hydro_points['discharge_min'] = -99 for index, row in self.nodes_out.iterrows(): self.hydro_points.loc[index, 'discharge_accum'] = \ rivers.loc[self.nodes_out.loc[index, 'arcid'], 'discharge_accum'] self.hydro_points.loc[index, 'discharge_max'] = \ rivers.loc[self.nodes_out.loc[index, 'arcid'], 'discharge_max'] self.hydro_points.loc[index, 'discharge_mean'] = \ rivers.loc[self.nodes_out.loc[index, 'arcid'], 'discharge_mean'] self.hydro_points.loc[index, 'discharge_min'] = \ rivers.loc[self.nodes_out.loc[index, 'arcid'], 'discharge_min'] # calculate the power in watts based on Alex's formula self.hydro_points['power_accum'] = rho * g * eta_t * eta_g * conv * self.nodes_out['discharge_accum'] * self.nodes_out['head'] self.hydro_points['power_max'] = rho * g * eta_t * eta_g * conv * self.nodes_out['discharge_max'] * self.nodes_out['head'] self.hydro_points['power_mean'] = rho * g * eta_t * eta_g * conv * self.nodes_out['discharge_mean'] * self.nodes_out['head'] self.hydro_points['power_min'] = rho * g * eta_t * eta_g * conv * self.nodes_out['discharge_min'] * self.nodes_out['head']
def plot_multipolygon(ax, geom, color='red'): """ Can safely call with either Polygon or Multipolygon geometry """ if geom.type == 'Polygon': plot_polygon(ax, geom, color) elif geom.type == 'MultiPolygon': for poly in geom.geoms: plot_polygon(ax, poly, color) cwd = os.path.dirname(os.path.abspath(__file__)) datadir = os.path.join(os.path.split(cwd)[0], 'data') jsonfile = os.path.join(datadir, 'manhattan_island_proj.json') citibikefile = os.path.join(datadir, 'citibike.json') manhattan = shape(geojson.load(open(jsonfile))) man_arr = np.asarray(manhattan.exterior) # load citbike station locations, transform to map coordinates, # and filter for only those in Manhattan c = json.load(open(citibikefile)) stations = [(x['longitude'], x['latitude']) for x in c['stationBeanList']] lon, lat = zip(*stations) nyp = Proj( '+datum=NAD83 +lat_0=40.1666666667 +lat_1=40.6666666667 ' '+lat_2=41.0333333333 +lon_0=-74 +no_defs +proj=lcc +units=us-ft ' '+x_0=300000 +y_0', preserve_units=True) wgs84 = Proj(init='epsg:4326') x, y = transform(wgs84, nyp, lon, lat) points = MultiPoint(list(zip(x, y)))
def read_fiona(source, prop_map, filterer=None): """Process a fiona collection """ collection = { 'type': 'FeatureCollection', 'features': [], 'bbox': [float('inf'), float('inf'), float('-inf'), float('-inf')] } skipped_count = 0 failed_count = 0 transformer = partial(transform_geom, source.crs, 'EPSG:4326', antimeridian_cutting=True, precision=6) for feature in source: if filterer is not None and not filterer.keep(feature): skipped_count += 1 continue if feature['geometry'] is None: logging.error("empty geometry") failed_count += 1 continue try: transformed_geometry = transformer( _force_geometry_2d(feature['geometry'])) fixed_geometry = _fix_geometry(transformed_geometry) feature['geometry'] = _force_geometry_ccw(fixed_geometry) feature['properties'] = get_transformed_properties( feature['properties'], prop_map) shapely_geometry = shape(feature['geometry']) geom_aea = ops.transform( partial( pyproj.transform, pyproj.Proj(init='EPSG:4326'), pyproj.Proj(proj='aea', lat1=shapely_geometry.bounds[1], lat2=shapely_geometry.bounds[3])), shapely_geometry) feature['properties']['acres'] = round(geom_aea.area / 4046.8564224) collection['bbox'] = [ comparator(values) for comparator, values in zip( [min, min, max, max], list(zip(collection['bbox'], _bbox(feature)))) ] collection['features'].append(feature) except PropertyMappingFailedException as e: logging.error(str(e) + ": " + str(feature['properties'])) failed_count += 1 except Exception as e: logging.error(str(e), "error processing feature: " + str(feature)) failed_count += 1 #avoid math error if there are no features if len(collection['features']) == 0: del collection['bbox'] logging.info("skipped %i features, kept %i features, errored %i features" % (skipped_count, len(collection['features']), failed_count)) return collection
def create_index(features): """Create an R-Tree index from a set of features""" index = rtree.index.Index() for id, feature in enumerate(features): index.insert(id, shape(feature['geometry']).bounds) return index
car_shape = Polygon([[-120.84, 40], [-120, 5], [-104.5, 10], [-92, 1.416666031], [-92.0, -0.5], [-89.8, -0.5], [-58.8, -0.5], [-58.8, 40], [-120.84, 40]]) nam_polygons = [] afi_polygons = [] russia_polygons = [] asia_polygons = [] mid_polygons =[] sam_polygons = [] kzwy_polygons = [] uaaa_polygons = [] utaa_polygons = [] for _feature in faa_airspace_data['features']: if _feature['properties']['NAME'] == 'ANCHORAGE ARCTIC FIR': (minx, miny, maxx, maxy) = shape(_feature['geometry']).bounds north_of_alaska = Polygon([[maxx+1, miny], [minx, miny], [minx, maxy], [maxx+1, maxy]]) nam_polygons.append(north_of_alaska) if (_feature['properties']['COUNTRY'] == 'United States' and _feature['properties']['TYPE_CODE'] == 'ARTCC' and _feature['properties']['LOCAL_TYPE'] == 'ARTCC_L' or _feature['properties']['TYPE_CODE'] == 'FIR' and _feature['properties']['IDENT'] in ['PAZA', 'CZVR', 'CZWG', 'CZYZ', 'CZQM', 'CZUL']): nam_polygons.append(shape(_feature['geometry'])) if (_feature['properties']['TYPE_CODE'] == 'FIR' and _feature['properties']['IDENT'] in ['UHMM',]): russia_polygons.append(shape(_feature['geometry'])) if (_feature['properties']['TYPE_CODE'] == 'FIR' and _feature['properties']['IDENT'] in ['KZAK', 'NZZO', 'RJJJ',
from shapely.geometry import shape, mapping url = 'https://opendata.arcgis.com/datasets/ef4b445a53c1406892257fe63129a8ea_0.geojson' link_to_page = 'https://npgeo-corona-npgeo-de.hub.arcgis.com/datasets/' \ 'ef4b445a53c1406892257fe63129a8ea_0?geometry=-67.085%2C46.270%2C83.340%2C55.886' r = requests.get(url, allow_redirects=True) last_fetch = datetime.now() open('static/bundeslaender_simplify200.geojson', 'wb').write(r.content) data = r.json() lat_bounds = [47.27012360470944, 55.099168977100774] long_bounds = [5.866755374775609, 15.04181565646822] for feature in data["features"]: name = feature["properties"]["LAN_ew_GEN"] if "Bodensee" in name: continue try: polygon = max(shape(feature["geometry"]), key=lambda a: a.area) except: polygon = shape(feature["geometry"]) feature["geometry"] = mapping(polygon.simplify(0.01)) print(name) ncases = feature["properties"]["Fallzahl"] feature["properties"]["cases"] = ncases with open('static/bundeslaender_simplify200.geojson', 'w') as json_file: json.dump(data, json_file)
def download(self, source_urls, workdir, conform=None): output_files = [] download_path = os.path.join(workdir, 'esri') mkdirsp(download_path) query_fields = self.field_names_to_request(conform) for source_url in source_urls: size = 0 file_path = self.get_file_path(source_url, download_path) if os.path.exists(file_path): output_files.append(file_path) _L.debug("File exists %s", file_path) continue downloader = EsriDumper(source_url, parent_logger=_L) metadata = downloader.get_metadata() if query_fields is None: field_names = [f['name'] for f in metadata['fields']] else: field_names = query_fields[:] if X_FIELDNAME not in field_names: field_names.append(X_FIELDNAME) if Y_FIELDNAME not in field_names: field_names.append(Y_FIELDNAME) if GEOM_FIELDNAME not in field_names: field_names.append(GEOM_FIELDNAME) # Get the count of rows in the layer try: row_count = downloader.get_feature_count() _L.info("Source has {} rows".format(row_count)) except EsriDownloadError: _L.info("Source doesn't support count") with csvopen(file_path, 'w', encoding='utf-8') as f: writer = csvDictWriter(f, fieldnames=field_names, encoding='utf-8') writer.writeheader() for feature in downloader: try: geom = feature.get('geometry') or {} row = feature.get('properties') or {} if not geom: raise TypeError("No geometry parsed") if any((isinstance(g, float) and math.isnan(g)) for g in traverse(geom)): raise TypeError("Geometry has NaN coordinates") shp = shape(feature['geometry']) row[GEOM_FIELDNAME] = shp.wkt try: centroid = shp.centroid except RuntimeError as e: if 'Invalid number of points in LinearRing found' not in str( e): raise xmin, xmax, ymin, ymax = shp.bounds row[X_FIELDNAME] = round(xmin / 2 + xmax / 2, 7) row[Y_FIELDNAME] = round(ymin / 2 + ymax / 2, 7) else: row[X_FIELDNAME] = round(centroid.x, 7) row[Y_FIELDNAME] = round(centroid.y, 7) writer.writerow( {fn: row.get(fn) for fn in field_names}) size += 1 except TypeError: _L.debug("Skipping a geometry", exc_info=True) _L.info("Downloaded %s ESRI features for file %s", size, file_path) output_files.append(file_path) return output_files
features = ways_to_polygons(features) return features if __name__ == '__main__': progress = tqdm(total=len(CITIES) * len(OSM_FEATURES)) for city_name in CITIES: city = City(city_name) dst_dir = os.path.join(city.intermediary_dir, 'osm') os.makedirs(dst_dir, exist_ok=True) osm = Downloader(aoi=shape(city.aoi), epsg=city.epsg, dst_dir=dst_dir) for key, filename in OSM_FEATURES.items(): features = osm.get_feature(key) features = features[features.is_valid] features = features.to_crs(city.crs) features = features[features.intersects(shape(city.aoi))] geoms = features.intersection(shape(city.aoi)) features = features.assign(geometry=geoms) output_f = os.path.join(dst_dir, filename) features.to_file(output_f) progress.update(1) progress.close()
#!/usr/bin/python from shapely.geometry import Point, mapping, shape from fiona import collection with collection("31471050500_face.shp", "r") as input: for geo in input: print shape(geo['geometry']) print geo['properties']
def execute(trial=False): '''Retrieve some data sets (not using the API here for the sake of simplicity).''' startTime = datetime.datetime.now() # Set up the database connection. client = dml.pymongo.MongoClient() repo = client.repo repo.authenticate('soohyeok_soojee', 'soohyeok_soojee') neighborhoodData = repo['soohyeok_soojee.get_neighborhoods'].find() # AllCoordOfLandmark = repo['soohyeok_soojee.transform_landmark'].find()[0]['coordinates'] LandmarkAndTown = repo['soohyeok_soojee.transform_landmark'].find( )[0]['LandmarkAndTown'] CrimeAndTown = repo['soohyeok_soojee.transform_crime'].find( )[0]['CrimeAndTown'] if (trial == True): neighborhoodData = neighborhoodData.sample(frac=.05) # AllCoordOfLandmark = AllCoordOfLandmark.sample(frac=.05) LandmarkAndTown = LandmarkAndTown.sample(frac=.05) CrimeAndTown = CrimeAndTown.sample(frac=.05) neighborhoods = {} for n in neighborhoodData: key = n['properties']['Name'] neighborhoods[key] = n['geometry'] for name in CrimeAndTown: for point in CrimeAndTown[name]: dist = [[distance.euclidean(point, x)] for x in LandmarkAndTown[name]] if dist != []: index = np.argmin(dist) p = LandmarkAndTown[name][index] LandmarkAndTown[name].remove(p) # AllCoordOfLandmark.remove(p) # data = AllCoordOfLandmark data = [ point for name in LandmarkAndTown for point in LandmarkAndTown[name] ] # kmean plot kmeans = KMeans(n_clusters=10).fit(data) data = np.array(data) pyplot.scatter(data[:, 0], data[:, 1], c=kmeans.labels_) pyplot.scatter(kmeans.cluster_centers_[:, 0], kmeans.cluster_centers_[:, 1], marker='x', c='red') pyplot.show() centroid = [[x[0], x[1]] for x in kmeans.cluster_centers_] towns = [ name for name in neighborhoods for point in centroid if Point(point).within(shape(neighborhoods[name])) ] result = { 'centroid': centroid, 'towns': towns, 'Coordinates': LandmarkAndTown } repo.dropCollection("kmeans_landmark_crime") repo.createCollection("kmeans_landmark_crime") repo['soohyeok_soojee.kmeans_landmark_crime'].insert_many([result]) repo['soohyeok_soojee.kmeans_landmark_crime'].metadata( {'complete': True}) print(repo['soohyeok_soojee.kmeans_landmark_crime'].metadata()) repo.logout() endTime = datetime.datetime.now() return {"start": startTime, "end": endTime}
for barrio in js['features']: # mean mean_proximo = 10000 mean_sintetico = 0 # percentile_5 per5_proximo = 10000 per5_sintetico = 0 # median median_proximo = 10000 median_sintetico = 0 # percentile_95 per95_proximo = 10000 per95_sintentico = 0 polygon = shape(barrio['geometry']) centroide = polygon.centroid.wkt # print(centroide) for index, medida in data_estacion.iterrows(): point = Point(medida['lon'], medida['lat']) distance = polygon.exterior.distance(point) radio = .068 #mean if mean_proximo > distance: mean_proximo = distance mean_sintetico = medida['mean'] * math.exp( -math.pow(distance, 2) / math.pow((2 * radio), 2)) #per5 if per5_proximo > distance:
def simplifyGeom(geom): # Simplify complex polygons shp = geometry.shape(geom) simp = shp.simplify(SIMPLIFICATION_TOLERANCE, PRESERVE_TOPOLOGY) return geometry.mapping(simp)
# print(chko.bounds) # print(chko.schema) # print(chko.schema["geometry"]) # print(json.dumps(chko.meta, sort_keys=True, indent=4, separators=(',', ': '))) # print(len(chko)) # Vrati pocet prvku # for prvek in chko: # print(prvek["properties"]["NAZEV"]) # projde vsechny prvky # print(chko[10]) # Vrati kontkretni prvek # pocet = 0 # for prvek in chko: # if prvek["properties"]["NAZEV"] == "České středohoří": # pocet += 1 # print(pocet) # Knihovna shapely cr = chko[54] geom = shape(cr["geometry"]) # print(geom) # textova reprezentace objektu # print(geom.type) # vypise typ objektu # print(dir(geom)) # vypise vsechny funkce a atributy geometrie # print(geom.area) # print("centroid: ", mapping(geom.centroid)) # vrati hodnotu centroidu jako bod, mapping prevadi do json # print("obvod: ", geom.exterior.length) with open(os.path.join(cesta_data, "bod_centroid.geojson", "w")) as out: # uloyi geojson data = { "type":"FeatureCollection", "features": [ { "type":"feature", "geometry": mapping(geom.exterior)
def create_instituitons_countries_dataset() -> None: """Creates a csv mapping latitude and longitude of institutions in MAG to their country names. Based on https://stackoverflow.com/a/46589405/530399 :return: None :rtype: None """ logger = logging.getLogger(__name__) logger.info(f'Running {create_instituitons_countries_dataset.__name__}') # setup logger.info('Loading config') app_cfg = ConfigLoader.load_config() logger.info('Mapping Latitude/longitude to country names') countries_geojson_filepath = join(app_cfg['paths']['data_dir'], 'external/countries.geojson') with open(countries_geojson_filepath) as json_file: data = json.load(json_file) countries = {} for feature in data["features"]: geom = feature["geometry"] country = feature["properties"]["ADMIN"] countries[country] = prep(shape(geom)) def get_country(lon, lat): try: lon=float(lon) lat=float(lat) point = Point(lon, lat) for country, geom in countries.items(): if geom.contains(point): return country.lower() return None except: return None # print("\n\n\nThis is Austria "+str(get_country(10.0, 47.0))) # bprint(get_country(None,None)) # bprint(get_country("","")) # bprint(get_country("text","nonsense")) # bprint(get_country("10.0", "47.0")) # bprint(get_country(10.0, 47.0)) affiliations_lat_long_filepath = join(app_cfg['paths']['data_dir'], 'raw/dataset_affiliations_lat_long/affiliation_id_lat_long.csv') # bprint(affiliations_lat_long_filepath) affiliations_lat_long_df = pd.read_csv(affiliations_lat_long_filepath, header=0, sep=",") affiliations_lat_long_df = affiliations_lat_long_df.dropna() # Delete all records where any lat/long value is NaN def get_country_for_df_row(row): # Based on http://jonathansoma.com/lede/foundations/classes/pandas%20columns%20and%20functions/apply-a-function-to-every-row-in-a-pandas-dataframe/ return get_country(row['longitude'], row['latitude']) affiliations_lat_long_df['country'] = affiliations_lat_long_df.apply(get_country_for_df_row, axis=1) # print(affiliations_lat_long_df.head()) affiliations_lat_long_df = affiliations_lat_long_df.dropna() # Delete all records where any the country name could not be determined == NaN # Write the output as csv affiliations_countryname_filepath = join(app_cfg['paths']['data_dir'], 'processed/affiliations_country.csv') affiliations_lat_long_df.to_csv(affiliations_countryname_filepath, sep=',', index=False, header=True) logger.info('Wrote affiliations_countryname dataset to '+affiliations_countryname_filepath) logger.info(f'Done running {create_instituitons_countries_dataset.__name__}')
def shp2df(shplist, index=None, index_dtype=None, clipto=[], filter=None, true_values=None, false_values=None, layer=None, skip_empty_geom=True): """Read shapefile/DBF, list of shapefiles/DBFs, or File geodatabase (GDB) into pandas DataFrame. Parameters ---------- shplist : string or list of shapefile/DBF name(s) or FileGDB index : string Column to use as index for dataframe index_dtype : dtype Enforces a datatype for the index column (for example, if the index field is supposed to be integer but pandas reads it as strings, converts to integer) clipto : list limit what is brought in to items in index of clipto (requires index) filter : tuple (xmin, ymin, xmax, ymax) bounding box to filter which records are read from the shapefile. true_values : list same as argument for pandas read_csv false_values : list same as argument for pandas read_csv layer : str Layer name to read (if opening FileGDB) skip_empty_geom : True/False, default True Drops shapefile entries with null geometries. DBF files (which specify null geometries in their schema) will still be read. Returns ------- df : DataFrame with attribute fields as columns; feature geometries are stored as shapely geometry objects in the 'geometry' column. """ if isinstance(shplist, str): shplist = [shplist] if not isinstance(true_values, list) and true_values is not None: true_values = [true_values] if not isinstance(false_values, list) and false_values is not None: false_values = [false_values] if len(clipto) > 0 and index: clip = True else: clip = False df = pd.DataFrame() for shp in shplist: print("\nreading {}...".format(shp)) shp_obj = fiona.open(shp, 'r', layer=layer) if index is not None: # handle capitolization issues with index field name fields = list(shp_obj.schema['properties'].keys()) index = [f for f in fields if index.lower() == f.lower()][0] attributes = [] # for reading in shapefiles meta = shp_obj.meta if meta['schema']['geometry'] != 'None': if filter is not None: print('filtering on bounding box {}, {}, {}, {}...'.format( *filter)) if clip: # limit what is brought in to items in index of clipto for line in shp_obj.filter(bbox=filter): props = line['properties'] if not props[index] in clipto: continue props['geometry'] = line.get('geometry', None) attributes.append(props) else: for line in shp_obj.filter(bbox=filter): props = line['properties'] props['geometry'] = line.get('geometry', None) attributes.append(props) print( '--> building dataframe... (may take a while for large shapefiles)' ) shp_df = pd.DataFrame(attributes) # reorder fields in the DataFrame to match the input shapefile if len(attributes) > 0: shp_df = shp_df[list(attributes[0].keys())] # handle null geometries if len(shp_df) == 0: print('Empty dataframe! No features were read.') if filter is not None: print('Check filter {} for consistency \ with shapefile coordinate system'.format(filter)) geoms = shp_df.geometry.tolist() if geoms.count(None) == 0: shp_df['geometry'] = [shape(g) for g in geoms] elif skip_empty_geom: null_geoms = [i for i, g in enumerate(geoms) if g is None] shp_df.drop(null_geoms, axis=0, inplace=True) shp_df['geometry'] = [ shape(g) for g in shp_df.geometry.tolist() ] else: shp_df['geometry'] = [ shape(g) if g is not None else None for g in geoms ] # for reading in DBF files (just like shps, but without geometry) else: if clip: # limit what is brought in to items in index of clipto for line in shp_obj: props = line['properties'] if not props[index] in clipto: continue attributes.append(props) else: for line in shp_obj: attributes.append(line['properties']) print( '--> building dataframe... (may take a while for large shapefiles)' ) shp_df = pd.DataFrame(attributes) # reorder fields in the DataFrame to match the input shapefile if len(attributes) > 0: shp_df = shp_df[list(attributes[0].keys())] shp_obj.close() if len(shp_df) == 0: continue # set the dataframe index from the index column if index is not None: if index_dtype is not None: shp_df[index] = shp_df[index].astype(index_dtype) shp_df.index = shp_df[index].values df = df.append(shp_df) # convert any t/f columns to numpy boolean data if true_values is not None or false_values is not None: replace_boolean = {} for t in true_values: replace_boolean[t] = True for f in false_values: replace_boolean[f] = False # only remap columns that have values to be replaced cols = [c for c in df.columns if c != 'geometry'] for c in cols: if len(set(replace_boolean.keys()).intersection(set( df[c]))) > 0: df[c] = df[c].map(replace_boolean) return df
def test_tiles_from_bbox(): """Get tiles intersecting with bounding box.""" test_bbox = shape({ "type": "Polygon", "coordinates": [[ (5.625, 61.875), (56.25, 61.875), (56.25, 28.125), (5.625, 28.125), (5.625, 28.125), (5.625, 61.875), ]], }) test_tiles = { (5, 5, 33), (5, 6, 33), (5, 7, 33), (5, 8, 33), (5, 9, 33), (5, 10, 33), (5, 5, 34), (5, 6, 34), (5, 7, 34), (5, 8, 34), (5, 9, 34), (5, 10, 34), (5, 5, 35), (5, 6, 35), (5, 7, 35), (5, 8, 35), (5, 9, 35), (5, 10, 35), (5, 5, 36), (5, 6, 36), (5, 7, 36), (5, 8, 36), (5, 9, 36), (5, 10, 36), (5, 5, 37), (5, 6, 37), (5, 7, 37), (5, 8, 37), (5, 9, 37), (5, 10, 37), (5, 5, 38), (5, 6, 38), (5, 7, 38), (5, 8, 38), (5, 9, 38), (5, 10, 38), (5, 5, 39), (5, 6, 39), (5, 7, 39), (5, 8, 39), (5, 9, 39), (5, 10, 39), (5, 5, 40), (5, 6, 40), (5, 7, 40), (5, 8, 40), (5, 9, 40), (5, 10, 40), (5, 5, 41), (5, 6, 41), (5, 7, 41), (5, 8, 41), (5, 9, 41), (5, 10, 41), } tp = TilePyramid("geodetic") bbox_tiles = {tile.id for tile in tp.tiles_from_bbox(test_bbox, 5)} assert test_tiles == bbox_tiles
args = parser.parse_args() # Import shapefile specified at commandline shp = args.shp # Can override the hardcoded maps directory if args.dir: MAP_DATA_FP = os.path.join(args.dir, 'processed/maps') if args.newmap: MAP_DATA_FP = os.path.join(MAP_DATA_FP, args.newmap) if not os.path.exists(MAP_DATA_FP): os.mkdir(MAP_DATA_FP) # Get all lines, dummy id lines = [(line[0], shape(line[1]['geometry'])) for line in enumerate(fiona.open(shp))] print 'Extracting intersections and writing into ' + MAP_DATA_FP inters = [] pkl_file = os.path.join(MAP_DATA_FP, 'inters.pkl') if not os.path.exists(pkl_file) or args.forceupdate: print 'Generating intersections...' inters = generate_intersections(lines) # Save to pickle in case script breaks with open(pkl_file, 'w') as f: cPickle.dump(inters, f) else: print 'Reading intersections from ' + pkl_file with open(pkl_file, 'r') as f: inters = cPickle.load(f)
def model_fit(request, model_id): model = PredictionModel.objects.get(id=model_id) areas = read_frame(model.area.all()) areavars = [] for index1, row1 in areas.iterrows(): df = read_frame( Site.objects.filter(feature_type=FeatureType.objects.get( name=row1["feature_type"]))) polygon = shape(row1["geom"]) select = [] for index2, row2 in df.iterrows(): select.append(polygon.contains(shape(row2['geom']))) data = read_frame( FeatureData.objects.filter(site__in=df[select]['id']), index_col="date") data["area"] = row1["name"] data["feature_type"] = row1["feature_type"] areavars.append(data) lagvars = [] for i in range(len(areavars)): ft = areavars[i].area.unique() + " " + areavars[i].feature_type.unique( ) d = areavars[i].pivot(columns='site', values='value') if len(d.columns) > 1: d = pd.DataFrame(d.mean(axis=1, skipna=True)) for j in [1, 2, 3, 4, 5]: df = pd.DataFrame() df[ft + '_shift_' + str(j)] = d.rolling(window=j).mean().shift(1) lagvars.append(df) res = pd.concat(lagvars, axis=1) res = res[res.index.month.isin([5, 6, 7, 8, 9])].reset_index() rain_cols = [col for col in res.columns if 'Rainfall' in col] res[rain_cols] = res[rain_cols].add(1).apply(np.log) FIB = read_frame(FeatureData.objects.filter(site=model.site.all()[0])) FIB["date"] = FIB.date.round("D") d = FIB.merge(res, on="date").drop("variable", axis=1) D = d.dropna() y = np.log10(D["value"]) X = D.drop(["date", "value", "id", "site"], axis=1) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=4) if model.fit == None: grid = { 'n_estimators': [1000], 'max_depth': np.linspace(1, 6, 6), 'max_features': [4, 6, 8], 'min_samples_leaf': [1] } # Instantiate the Random regressor: elastic_net rf = RandomForestRegressor() # Setup the GridSearchCV object: gm_cv gm_cv = GridSearchCV(rf, param_grid=grid, cv=5) # Fit it to the training data gm_cv.fit(X_train, y_train) rf = RandomForestRegressor( n_estimators=gm_cv.best_params_["n_estimators"], max_depth=gm_cv.best_params_["max_depth"], max_features=gm_cv.best_params_["max_features"], min_samples_leaf=1) rf.fit(X_train, y_train) model.fit = pickle.dumps(rf) model.save() rf = pickle.loads(model.fit) # Predict on the test set and compute metrics #y_pred1 = gm_cv.predict(X_test) y_pred = rf.predict(X_test) #r2 = rf.score(X_test, y_test) #r2_in = rf.score(X_train, y_train) #mse = mean_squared_error(y_test, y_pred) #mse_in = mean_squared_error(y_train, rf.predict(X_train)) df_test = pd.DataFrame({ 'meas': y_test, 'pred': rf.predict(X_test), 'split': 'out of sample' }) df_train = pd.DataFrame({ 'meas': y_train, 'pred': rf.predict(X_train), 'split': 'in sample' }) df = pd.concat([df_test, df_train]) fig = px.scatter(df, x="meas", y="pred", color="split", color_discrete_sequence=['#212c52', '#75c3ff']) fig.update_layout( font_family="Helvetica Neue, Helvetica, Arial, sans-serif", font_color="black", title={'text': 'Model fit of Random Forest model'}, xaxis_title="measured data (sample)", yaxis_title="fitted values (in sample fit)", #markersize= 12, ) fig.update_layout( legend=dict(yanchor="top", title=None, y=0.99, xanchor="left", x=0.01)) fig.update_traces( marker_size=8) #['#75c3ff', "red"],#, marker_line_color='#212c52', # marker_line_width=1.5, opacity=1) model_fit = plot(fig, output_type="div") importances = pd.Series(data=rf.feature_importances_, index=X.columns) # Sort importances importances_sorted = importances.sort_values() importances_df = importances_sorted.reset_index() importances_df.columns = ["feature", "importance"] fig = px.bar(importances_df, y="feature", x="importance", orientation='h') fig.update_layout( font_family="Helvetica Neue, Helvetica, Arial, sans-serif", font_color="black", title={'text': 'Feature importance of Random Forest model'} #markercolor = "#212c52" ) fig.update_traces(marker_color='#75c3ff', marker_line_color='#75c3ff', marker_line_width=1.5, opacity=1) feature_importance = plot(fig, output_type="div") bathingspot = model.site.all()[0] return render( request, 'ews/model_fit.html', { 'bathingspot': bathingspot, "entries": Site.objects.all(), "model": model, 'areas': model.area.all(), 'model_fit': model_fit, 'feature_importance': feature_importance })
def getBoundary(self, size_x=512, size_y=512, force=False): # Return the img if already loaded if (self.img != "N/A"): return self.img # Find the "shape" of the boundary, if we haven't already if (self.boundary == "N/A"): self.boundary = self.getFeature() # If the "shape" of the boundary cannot be found, that's all we can do if (self.boundary == "N/A"): return "N/A" # Check if the "img" has already been cached, and just open it if it is there os.makedirs("cached_property_images", exist_ok=True) cache_file = os.path.join( "cached_property_images", self.boundary['properties']['LOT'] + "_" + self.boundary['properties']['PLAN'] + ".png") if (os.path.exists(cache_file) and (force == False)): self.img = Image.open(cache_file) return self.img print("Isolating farm", end='') with open( os.path.join( "geometries", "geo-x" + str(self.tile_x) + "-y" + str(self.tile_y) + ".geojson")) as f1: geo_json_features = json.load(f1)["features"] tile = GeometryCollection([ shape(feature["geometry"]).buffer(0) for feature in geo_json_features ]) self.img = Image.new('RGBA', (size_x, size_y)) count_hit = 0 result = GeometryCollection() if shape(self.boundary["geometry"]).intersects(tile): result = result.union( shape(self.boundary["geometry"]).intersection(tile)) if tile.intersects(result): for y in range(size_y): if (y % 15 == 0): #print("y="+ str(y) + ", hits=" + str(count_hit)) print(".", end="") for x in range(size_x): lat_long = self.GetLatLongForCoords(y, x) if result.intersects(Point(lat_long)): self.img.putpixel((y, x), (0, 0, 255, 255)) # Blue count_hit += 1 #print("Hits: " + str(count_hit)) # Find the border for y in range(size_y): for x in range(size_x): # It is a border if it's not blue... if (self.img.getpixel((y, x)) != (0, 0, 255, 255)): # And it has a neighbour that is blue for j in range(y - 1, y + 2): for i in range(x - 1, x + 2): # Check for neighbour out of bounds, don't compare pixel to itself if ((j >= 0) and (j < size_y) and (i >= 0) and (i < size_x) and ((j != y) or (i != x))): if (self.img.getpixel( (j, i)) == (0, 0, 255, 255)): # This is part of the border! self.img.putpixel((y, x), (255, 0, 0, 255)) # Save the image to the cache self.img.save(cache_file) print(" done") return self.img
def feature_from_row(Community, Address, Mission, MissionType, City, CommunityType, Website): feature = { 'type': 'Feature', 'properties': { 'CommunityPartner': '', 'Address': '', 'Projects': '', 'College Name': '', 'Mission Type': '', 'Project Name': '', 'Legislative District Number': '', 'Number of projects': '', 'Income': '', 'City': '', 'County': '', 'Mission Area': '', 'CommunityType': '', 'Campus Partner': '', 'Academic Year': '', 'Website': '' }, 'geometry': { 'type': 'Point', 'coordinates': [] } } geocode_result = gmaps.geocode(Address) # get the coordinates if (geocode_result[0]): latitude = geocode_result[0]['geometry']['location']['lat'] longitude = geocode_result[0]['geometry']['location']['lng'] feature['geometry']['coordinates'] = [longitude, latitude] coord = Point([longitude, latitude]) for i in range( len(district)): # iterate through a list of district polygons property = district[i] polygon = shape(property['geometry']) # get the polygons if polygon.contains(coord): # check if a partner is in a polygon feature['properties'][ 'Legislative District Number'] = property["properties"][ "id"] # assign the district number to a partner for m in range(len(county)): # iterate through the County Geojson properties2 = county[m] polygon = shape(properties2['geometry']) # get the polygon if polygon.contains( coord ): # check if the partner in question belongs to a polygon feature['properties']['County'] = properties2['properties'][ 'NAME'] feature['properties']['Income'] = properties2['properties'][ 'Income'] projectlist = 0 yearlist = [] campuslist = [] projectList = [] collegeList = [] partners = dfProjects['community_partner'] years = dfProjects['academic_year'] campuses = dfProjects['campus_partner'] projects = dfProjects['project_name'] colleges = dfProjects['college_name'] count = 0 for n in range(len(partners)): if (partners[n] == Community): if (years[n] not in yearlist): yearlist.append(years[n]) if (campuses[n] not in campuslist): campuslist.append(campuses[n]) if (projects[n] not in projectList): projectList.append(projects[n]) count += 1 if (colleges[n] not in collegeList): collegeList.append(colleges[n]) feature['properties']['Number of projects'] = count feature['properties']['Campus Partner'] = campuslist feature['properties']['Academic Year'] = yearlist feature['properties']['Projects'] = projectList feature['properties']['College Name'] = collegeList feature['properties']['CommunityPartner'] = Community feature['properties']['CommunityType'] = CommunityType feature['properties']['Website'] = Website feature['properties']['Mission Area'] = Mission feature['properties']['Mission Type'] = MissionType feature['properties']['City'] = City collection['features'].append(feature) return feature
def calc_matrix(): """ Fill the matrix with the passengers numbers from the raw data """ # Reading the data file data = open(DATA_FILE, 'r') reader = csv.reader(data, delimiter=',') # Two unnecessary lines reader.next() reader.next() # Opening the output file output = open(MATRIX_FILE, 'wb') for row, i in enumerate(reader): # Parsing the two points points = [ point.Point(float(i[3]), float(i[2])), point.Point(float(i[6]), float(i[5])) ] points_id = [] # Find the polygon where each point is located for j in points: polygon_id = -1 for k in idx.intersection(j.coords[0]): if j.within(shape(polygons[k]['geometry'])): polygon_id = polygons[k]['properties']['ID'] break points_id.append(polygon_id) if -1 not in points_id: # Ignoring travels with errors try: time = datetime.strptime( i[1][:-3], '%Y-%m-%d %H:%M:%S.%f') # Time format with milliseconds except ValueError: time = datetime.strptime(i[1], '%Y-%m-%d %H:%M:%S') # Sorting the travels by day times (by arrival time) if time.hour <= 6: # Night mat[points_id[0] - 1][points_id[1] - 1][6] += 1 elif time.hour <= 7: # 6 AM mat[points_id[0] - 1][points_id[1] - 1][0] += 1 elif time.hour <= 8: # 7 AM mat[points_id[0] - 1][points_id[1] - 1][1] += 1 elif time.hour <= 9: # 8 AM mat[points_id[0] - 1][points_id[1] - 1][2] += 1 elif time.hour <= 15: # Noon mat[points_id[0] - 1][points_id[1] - 1][3] += 1 elif time.hour <= 19: # Afternoon mat[points_id[0] - 1][points_id[1] - 1][4] += 1 elif time.hour <= 22: # Evening mat[points_id[0] - 1][points_id[1] - 1][5] += 1 else: # Night mat[points_id[0] - 1][points_id[1] - 1][6] += 1 if row % 100000 == 0: print row # Writing to output file print 'Writing to file' output.write('[' + ',\n'.join([str(i) for i in mat]) + ']') data.close() output.close()
import fiona import sys from shapely.geometry import shape, mapping from shapely.ops import cascaded_union with fiona.drivers(): with fiona.open(sys.argv[1]) as source: with fiona.open(sys.argv[2]) as clip: dest = fiona.open(sys.argv[3], 'w', driver=source.driver, crs=source.crs, schema=source.schema) # convert clip into a list of shapely shapes clippers = [shape(s['geometry']) for s in clip] for feature in source: clipped = [] geometry = shape(feature['geometry']) for clipper in clippers: if clipper.intersects(geometry): clipped.append(clipper.intersection(geometry)) if clipped: unioned = cascaded_union(clipped) # Sometimes an intersection can return a point # or a linestring if unioned.type == geometry.type: dest.write({ 'geometry': mapping(unioned), 'properties': feature['properties'] })
msas.head() rw = msas.ix[5] #start_time = timeit.default_timer() msa = rw.namelsad msatag = rw['name'] print('processing {}'.format(msatag)) #####Get the MSA shape q = """select shape from world where datasetid='metro/tiger/2014/1/tl_2014_us' and data['t_namelsad'] like '{}%'; """.format(msa) z = pd.read_sql(q, engine) BB = dumps(shape(z['shape'][0])) ############# GEOHASH POLYGONS LEVEL 7REPLACE TIGER geohash_polygons HERE ################## #### Get geohash polygons level 7 for a given MSA q = """select shape, geohash_label ... """ geohash_polygons = pd.read_sql(q, engine) ## Make a geodataframe wth the geohash_polygons geo = [shape(i) for i in geohash_polygons['shape']] del geohash_polygons['shape'] geohash_polygons.gpd = gpd.GeoDataFrame(geometry=geo, data=geohash_polygons, crs={'init': 'epsg:4326'})
def query(): print('opening file...') curdir = os.path.dirname(__file__) df = pd.read_csv( open(os.path.join(curdir, '../input/colleges/MERGED2015_16_PP.csv'))) print('file opened') schools = df[(df['CONTROL'] == 1) | (df['CONTROL'] == 2)] # only public/private non profits schools = schools[schools['MAIN'] == 1] # only main campuses schools = schools[ schools['PREDDEG'] == 3] # only look at predominately bachelors serving institutions schools = schools[schools['DISTANCEONLY'] == 0] # Only in-person colleges schools = schools[schools['CURROPER'] == 1] # Only currently operating # Dropping NaN values for certain columns schools = schools[(~np.isnan(schools['LATITUDE']))] schools = schools[(~np.isnan(schools['LONGITUDE']))] #Personal Preference schools = schools[schools['UGDS'] > 1800] # reduce # of schools cols = [ 'UNITID', # id 'INSTNM', # name 'CITY', # city 'STABBR', # state 'ZIP', # zip 'INSTURL', # website 'UGDS', # size 'LATITUDE', 'LONGITUDE', 'ADM_RATE', # admit rate 'HIGHDEG', #highest degree offered 'CONTROL', # public/private 'CCBASIC', # Carnegie Classification -- basic 'CCUGPROF', # Carnegie Classification -- undergraduate profile 'CCSIZSET', # Carnegie Classification -- size and setting 'TRIBAL', # Flag for tribal college and university 'NANTI', #Flag for Native American non-tribal institution 'UGDS_AIAN', # Total share of enrollment of undergraduate degree-seeking students who are American Indian/Alaska Native 'UG_AIANOLD', #Total share of enrollment of undergraduate students who are Asian/Pacific Islander ] schools = schools[cols] schools = schools.sort_values(['UGDS']) curdir = os.path.dirname(__file__) gj = json.load( open( os.path.join( curdir, '../input/nativelands/indigenousTerritories.geojson'))) territory_series = pd.Series(index=schools.index) territories = gj.get('features') territories_polygons = [] for i, t in enumerate(territories): polygon = shape(t.get('geometry')) territories_polygons.append({ 'polygon': polygon, 'properties': t.get('properties'), 'id': t.get('id'), 'index': i, }) t['properties']['colleges'] = [] with tqdm(total=len(schools)) as pbar: for i, s in schools.iterrows(): p = Point(s['LONGITUDE'], s['LATITUDE']) territories_match = [] for t_i, t in enumerate(territories_polygons): if t['polygon'].contains(p): territories_match.append({ 'id': t.get('id'), 'name': t.get('properties').get('Name') }) gj.get('features')[t_i]['properties']['colleges'].append( s['UNITID']) territory_series[s.name] = json.dumps(territories_match) pbar.update(1) schools['territories'] = territory_series #schools.to_csv('schools.csv') schools.to_csv('../display/public/schools.csv') json.dump(gj, open('../display/public/territories.geojson', 'w+'))