def build_aggregate(level, code, name, territories):
print 'Building aggregate "{0}" (level={1}, code={2})'.format(name, level, code)
polygons = []
for tlevel, tcode in territories:
try:
territory = Territory.objects.get(level=tlevel, code=tcode)
except Territory.DoesNotExist:
print 'Territory {0}/{1} not found'.format(tlevel, tcode)
continue
if not shape(territory.geom).is_valid:
print 'Skipping invalid polygon for {0}'.format(territory.name)
continue
if shape(territory.geom).is_empty:
print 'Skipping empty polygon for {0}'.format(territory.name)
continue
polygons.append(territory.geom)
geom = cascaded_union([shape(p) for p in polygons])
if geom.geom_type == 'Polygon':
geom = MultiPolygon([geom])
try:
territory = Territory.objects.get(level=level, code=code)
except Territory.DoesNotExist:
territory = Territory(level=level, code=code)
territory.name = name
territory.geom = geom.__geo_interface__
territory.save()
def apply_shapely(self, method, args=None, call=True, out_geomtype=None,
**kwargs):
coll = self.collection()
out_schema = coll.schema.copy()
if not args:
args = []
if out_geomtype:
out_schema['geometry'] = out_geomtype
tempds = self.tempds(method)
with fiona.collection(tempds, "w", "ESRI Shapefile",
out_schema, crs=self.crs) as out_collection:
for in_feature in coll:
out_feature = in_feature.copy()
if call:
geom = mapping(
getattr(shape(in_feature['geometry']),
method)(*args, **kwargs)
)
else:
# it's not a method, it's a property
geom = mapping(
getattr(shape(in_feature['geometry']), method)
)
out_feature['geometry'] = geom
out_collection.write(out_feature)
return Layer(tempds)
def makeShape(code): #takes a 2-letter code
geom = findCountry(code)
if geom['type'] == 'Polygon':
p=Polygon(shape(geom))
elif geom['type'] == 'MultiPolygon':
p=MultiPolygon(shape(geom))
return p
def checkPos(lat, lon):
with open('5_mile_airport.json', 'r') as f:
js = json.load(f)
point = Point(lon, lat)
for feature in js['features']:
polygon = shape(feature['geometry'])
if polygon.contains(point):
return json.JSONEncoder().encode({ "safe" : "false"})
with open('unitedstates.json', 'r') as f:
js = json.load(f)
inUS = 0
for feature in js['features']:
polygon = shape(feature['geometry'])
if polygon.contains(point):
inUS = 1
break;
if inUS == 1:
with open('us_military.json', 'r') as f:
js = json.load(f)
for feature in js['features']:
polygon = shape(feature['geometry'])
if polygon.contains(point):
return json.JSONEncoder().encode({ "safe" : "false"})
with open('us_national_park.json', 'r') as f:
js = json.load(f)
for feature in js['features']:
polygon = shape(feature['geometry'])
if polygon.contains(point):
return json.JSONEncoder().encode({ "safe" : "false"})
return json.JSONEncoder().encode({ "safe" : "true"})
else:
return json.JSONEncoder().encode({ "safe" : "caution"})
def location_listener(self, name, location):
global telem_data, last_point, vehicle, last_location
telem_data['params']['location'] = {"lat": location.global_frame.lat,
"lon": location.global_frame.lon,
"alt": location.global_frame.alt}
# print vehicle.location.global_frame
current_point = Point(location.global_frame.lon, location.global_frame.lat)
telem_data['params']['nofly_zone_status']['inside'] = False
telem_data['params']['nofly_zone_status']['indexes'] = []
for i, nofly_zone in enumerate(nofly_polygons):
poly = shape(nofly_zone['data']['geometry'])
if poly.contains(current_point):
telem_data['params']['nofly_zone_status']['inside'] = True
telem_data['params']['nofly_zone_status']['indexes'].append(nofly_zone['id'])
further_check = False
try:
if global_multipoly.contains(current_point):
further_check = True
except:
further_check = True
if further_check:
for i, nofly_zone in enumerate(global_nofly_polygons):
poly = shape(nofly_zone['data']['geometry'])
if poly.contains(current_point):
telem_data['params']['nofly_zone_status']['inside'] = True
telem_data['params']['nofly_zone_status']['indexes'].append(nofly_zone['id'])
if telem_data['params']['nofly_zone_status']['inside'] is False:
last_point = current_point
last_location = location.global_frame
def read_geofabric_data(netGDB,bbox=None):
catch = {}
with fiona.open(netGDB, layer='AHGFCatchment') as c:
for feat in c.items(bbox=bbox):
geom = shape(feat[1]['geometry'])
cid = feat[1]['properties']['HydroID']
assert cid not in catch #shouldnt be duplicates
catch[cid] = geom
DG=nx.MultiDiGraph()
with fiona.open(netGDB, layer='AHGFNetworkStream') as c:
for feat in c.items(bbox=bbox):
streamLink = shape(feat[1]['geometry'])
#for some reason these are coming in as multipart features with only one part - no need for this
assert streamLink.type == 'MultiLineString'
assert len(streamLink.geoms) == 1
streamLink = streamLink.geoms[0]
##remove this - just here for testing
#if streamLink.representative_point().y > -40.5: #tasmania for testing
# continue
sid = feat[1]['properties']['HydroID']
cid = feat[1]['properties']['DrainID']
fid = feat[1]['properties']['From_Node']
tid = feat[1]['properties']['To_Node']
subCatch = catch.get(cid,Polygon())
DG.add_edge(fid, tid, id=sid,cid=cid,subCatch=subCatch,stream=streamLink)
return DG
def get_light_rail_sections():
""""""
# units derived from srs of input data
buffer_dist = 500
sect_dict = dict()
sections = fiona.open(SECTIONS)
sect_ix = generate_spatial_index(sections)
with fiona.open(RAIL_LINES) as rail_lines:
for rail_id, rail_feat in rail_lines.items():
rail_type = rail_feat['properties']['TYPE']
if 'MAX' in rail_type:
rail_geom = shape(rail_feat['geometry'])
rail_buffer = rail_geom.buffer(buffer_dist)
for sect_id in sect_ix.intersection(rail_buffer.bounds):
# for some reason fiona sees integers and long
# integers of the same value as not equal
sect_id = int(sect_id)
if sect_id not in sect_dict:
sect_feat = sections[sect_id]
sect_geom = shape(sect_feat['geometry'])
if rail_buffer.intersects(sect_geom):
sect_name = sect_feat['properties']['SECTION']
sect_dict[sect_id] = sect_name
return sect_dict.values()
def test_with_initial(self):
Fake, FakeForm = self.factory()
territories = [TerritoryFactory() for _ in range(3)]
fake = Fake(spatial=SpatialCoverage(
territories=[t.reference() for t in territories],
granularity=SPATIAL_GRANULARITIES.keys()[1]
))
territory = TerritoryFactory()
data = MultiDict({
'spatial-territories': str(territory.id),
'spatial-granularity': VALID_GRANULARITY
})
form = FakeForm(data, fake)
form.validate()
self.assertEqual(form.errors, {})
form.populate_obj(fake)
self.assertEqual(len(fake.spatial.territories), 1)
self.assertEqual(fake.spatial.territories[0], territory.reference())
self.assertTrue(shape(fake.spatial.geom).equals(shape(territory.geom)))
def zips_in_DMA(one_dma_id, to_txt = False):
#for both zip codes and DMAs, extract properties and geometry from each geojson feature, add each one as a dictionary entry with the DMA ID or zip # as the key and a shapely object as the value
zips_ = {}
for i in zips.features:
zips_[str(i['properties']['ZCTA5CE10'])] = shape(geojson.MultiPolygon(i['geometry']['coordinates']))
dmas_errors = []
dmas_ = {}
for i in nielsen.features:
try:
dmas_[str(i['properties']['id'])] = shape(geojson.Polygon(i['geometry']['coordinates']))
except:
try:
dmas_[str(i['properties']['id'])] = shape(geojson.Polygon(i['geometry']['coordinates'][0]))
except:
print "error"
dmaziplist = []
for i in zips_.iteritems():
if dmas_[one_dma_id].intersects(i[1]):
dmaziplist.append(i[0])
else:
pass
dma_filename = dmametadata.ix[one_dma_id]['dma_name'].replace(',', '').replace(' ', '') + one_dma_id
if to_txt:
with open(dma_filename + '.txt', 'wb') as zipfile:
for i in dmaziplist:
zipfile.write(i)
zipfile.write('\n')
return dmaziplist
def import_shoreline_file(self, lake_name, shoreline_file):
""" Load the shoreline from GIS file.
NB: Currently has side effects, loading crs and properties traits.
"""
with fiona.open(shoreline_file) as f:
crs = f.crs
geometries = []
for rec in f:
geometries.append(rec['geometry'])
# XXX: assuming that the properties aren't varying by geometry
properties = rec['properties']
if len(geometries) == 1:
geom = shape(geometries[0])
else:
# XXX: this assumes we'll always get lines, not polygons or other
geom = MultiLineString([
shape(geometry) for geometry in geometries])
with self._open_file('a') as f:
shoreline_group = self._get_shoreline_group(f)
shoreline_group._v_attrs.crs = self._safe_serialize(crs)
shoreline_group._v_attrs.lake_name = self._safe_serialize(lake_name)
shoreline_group._v_attrs.original_shapefile = self._safe_serialize(shoreline_file)
shoreline_group._v_attrs.properties = self._safe_serialize(properties)
geometry_str = self._safe_serialize(mapping(geom))
self._write_array(f, shoreline_group, 'geometry', np.array(geometry_str))
f.flush()
def main(infile, outfile, driver):
with fio.open(infile) as src:
meta = src.meta
meta['driver'] = driver
with fio.open(infile) as src, fio.open(outfile, 'w', **meta) as dst:
with click.progressbar(src) as features:
for feat in features:
east = deepcopy(feat)
west = deepcopy(feat)
east_geom = shape(east['geometry'])
west_geom = shape(west['geometry'])
# if 'Point' not in asShape(feat['geometry']).type:
# east_geom = east_geom.simplify(0.0001).buffer(0)
# west_geom = west_geom.simplify(0.0001).buffer(0)
east_geom = translate(east_geom, xoff=180)
west_geom = translate(west_geom, xoff=-180)
if not east_geom.is_empty:
east['geometry'] = mapping(east_geom)
dst.write(east)
if not west_geom.is_empty:
west['geometry'] = mapping(west_geom)
dst.write(west)
def process_web_inputs(task, inputs):
for param in task.inputs:
if param.name in inputs:
if isinstance(param, (params.RasterParameter, params.NdArrayParameter)):
inputs[param.name] = params.RegisteredDatasetParameter(param.name).clean(inputs[param.name])
elif isinstance(param, params.FeatureParameter):
try:
inputs[param.name] = shape(inputs[param.name])
except (ValueError, AttributeError, KeyError):
raise ParameterNotValidError
elif isinstance(param, params.ListParameter):
if isinstance(param.param_type, (params.RasterParameter, params.NdArrayParameter)):
inputs[param.name] = [
params.RegisteredDatasetParameter(param.name).clean(x) for x in inputs[param.name]
]
elif isinstance(param.param_type, params.FeatureParameter):
try:
inputs[param.name] = [shape(x) for x in inputs[param.name]]
except (ValueError, AttributeError, KeyError):
raise ParameterNotValidError
return task.validate_inputs(inputs)
def segQuality(inVector, inImage):
# open the vector
lyr = fiona.open(inVector)
features = [x for x in lyr]
values = np.zeros([len(features), 5], dtype=float)
# loop over features
for i in range(len(features)):
geometry1 = shape(features[i]['geometry'])
restFeatures = features[:i] + features[(i+ 1):]
value = zonal_stats(geometry1, inImage, stats=['count'], add_stats={'mymean':mymean, "myvarianza":varianza } )
df = pd.DataFrame.from_dict(value, orient='columns', dtype=None)
for j in range(len(restFeatures)):
geometry2 = shape(features[j]['geometry'])
if geometry2.intersects(geometry1) == True:
#print("They touch")
value = zonal_stats(geometry2, inImage, stats=['count'], add_stats={'mymean':mymean, "myvarianza":varianza } )
df = df.append(pd.DataFrame.from_dict(value, orient='columns', dtype=None))
values[i,0] = df.iloc[0,0] # count
values[i,1] = df.iloc[0,1] # mean
values[i,2] = df.iloc[0,2] # myvarianza
values[i,3] = np.var(df.iloc[:,1]) # varianza between
values[i,4] = len(df.iloc[1:]) # neighbours
# get overal values
intraVarWeighted = np.nansum( values[:,0]*values[:,2] ) / np.nansum(values[:,0])
interVarWeighted = np.nansum( values[:,4]*values[:,3] ) / np.nansum(values[:,4])
normVariance = (intraVarWeighted - interVarWeighted) / (intraVarWeighted + interVarWeighted)
numberSegments = len(values[:,4])
return( intraVarWeighted, interVarWeighted, normVariance, numberSegments )
def test_PointInPolygon(self):
"""
Test that a feature's point geometry is within its polygon geometry.
NOTE: A feature that does not contain a polygon geometry will be skipped.
"""
features_point_outside_polygon = {}
for record in self.test_data:
geoms = record['geometry']['geometries']
pt_geom = None
poly_geom = None
# Check for presence of point and polygon geoms.
for geom in geoms:
if geom['type'] == 'Point':
pt_geom = shape(geom)
if geom['type'] in ('MultiPolygon', 'Polygon'):
poly_geom = shape(geom)
if pt_geom and poly_geom:
# Check if point is within polygon. If not, flag it.
try:
if not pt_geom.within(poly_geom):
composite_key = self._get_comp_primary_key(record)
features_point_outside_polygon[composite_key] = geoms
except Exception as e:
continue
json_data = json.dumps(features_point_outside_polygon)
return json_data
def surroundingNeighborhoods(neighborhood):
neighborhood_shape = shape(neighborhood['geometry'])
surrounding_neighborhoods = []
for other in SF['features']:
if neighborhood_shape.touches(shape(other['geometry'])):
surrounding_neighborhoods.append(other['properties']['neighborho'])
return surrounding_neighborhoods
def test_PointInPolygonGeom(self):
"""
Test that a feature's point geometry is within its polygon geometry.
NOTE: A feature that does not contain a polygon geometry will be skipped.
"""
features_point_outside_polygon = []
for record in self.test_data:
geoms = record['geometry']['geometries']
pt_geom = None
poly_geom = None
# Check for presence of point and polygon geoms.
for geom in geoms:
if geom['type'] == 'Point':
pt_geom = shape(geom)
if geom['type'] in ('MultiPolygon', 'Polygon'):
poly_geom = shape(geom)
if pt_geom and poly_geom:
# Check if point is within polygon. If not, flag it.
try:
if not pt_geom.within(poly_geom):
composite_key = self._get_comp_primary_key(record)
features_point_outside_polygon.append(composite_key)
except Exception as e:
continue
# Raise error if we don't have an empty error list
self.assertEqual(len(features_point_outside_polygon), 0,
msg="The following features have point geoms outside of polygon geoms %s" % features_point_outside_polygon)
def test_overlappingPolygons(self):
"""
Test to ensure there are No two polygons of same Class and Mapcode overlay
"""
#Build a spatial index based on the bounding boxes of the polygons
from rtree import index
idx = index.Index()
count = -1
#iterating through input data to index
polygon_shapes = []
self.logger.info("Building the spatial index based on bounding boxes...")
for record in self.test_data:
if 'geometry' in record.keys():
geoms = record['geometry']['geometries']
# Check for presence of polygon geoms.
for geom in geoms:
if geom['type'] in ('MultiPolygon', 'Polygon'):
class_mapcode_polygons = {'id': record['properties']['id'],
'class': record['properties']['class'],
'map_code': record['properties']['map_code'],
'polygon': shape(geom)}
polygon_shapes.append(class_mapcode_polygons)
count +=1
idx.insert(count, shape(geom).bounds)
#check for overapping polygons
features_with_overlapping_polygons = []
ids_with_topology_error = []
self.logger.info("Verifying overlapping polygons...")
chumma_count = 0
for data in polygon_shapes:
for key in idx.intersection(data['polygon'].bounds):
if(data['map_code'] == polygon_shapes[key]['map_code'] and
data['class'] == polygon_shapes[key]['class'] and
polygon_shapes[key]['polygon'] != data['polygon']):
#verifying overlap
try:
feature_1 = str(polygon_shapes[key]['id']) + '_' + str(polygon_shapes[key]['map_code'])
feature_2 = str(data['id']) + '_' + str(data['map_code'])
if polygon_shapes[key]['polygon'].overlaps(data['polygon']):
if feature_2 + ";" + feature_1 not in features_with_overlapping_polygons:
features_with_overlapping_polygons.append(feature_1 + ";" + feature_2)
except Exception as e:
error_dict = {
'composite_keys': feature_1 + ";" + feature_2,
'error_string': str(e)
}
ids_with_topology_error.append(error_dict)
continue
# Print ids that caused topology errors
if len(ids_with_topology_error) is not 0:
self._print_ids_with_topology_error("These are a list of IDs that have TopologicalErrors that COULD NOT be checked for overlaps",
ids_with_topology_error)
# Raise error if we don't have an empty error list
self.assertEqual(len(features_with_overlapping_polygons), 0,
msg="The following features have overlapping polygons %s" % features_with_overlapping_polygons)
def test_prime_meridian_footprint():
bbox = [(-20, 0), (20, 0), (20, 5), (-20, 5)]
good_poly = MultiPolygon(
[Polygon(bbox + [bbox[0]])]
)
footprints = Footprints(bbox)
assert shape(footprints.data_polygon).area == good_poly.area
# Since the shape is rectangular, these should be the same
assert shape(footprints.tile_polygon).area == shape(footprints.data_polygon).area
def __init__(self, osmid, name, geometry, **kwargs):
self.osmid = osmid
self.name = name
self.geometries = [geometry['coordinates']]
self.oneway_length = 0
self.total_length = shape(geometry).length
self.properties = kwargs
if self.properties['oneway']:
self.oneway_length = shape(geometry).length
def find_containing_poly(point, polyfile):
sample_poly = fiona.open(polyfile,"r")
poly = sample_poly.next()
spoly = shape(poly["geometry"])
while (not point.within(spoly)):
poly = sample_poly.next()
spoly = shape(poly["geometry"])
sample_poly.close
return spoly
def build_aggregate(self, code, name, zones, properties, db):
geoms = []
populations = []
areas = []
if callable(zones):
zones = [zone['_id'] for zone in zones(db)]
for zoneid in zones:
# Resolve wildcard
if zoneid.endswith(':*'):
level = zoneid.replace(':*', '')
ids = db.distinct('_id', {'level': level})
resolved = self.build_aggregate(
code, name, ids, properties, db)
geoms.append(shape(resolved['geom']))
if resolved.get('population'):
populations.append(resolved['population'])
if resolved.get('area'):
areas.append(resolved['area'])
else:
zone = db.find_one({'_id': zoneid})
if not zone:
warning('Zone {0} not found'.format(zoneid))
continue
if 'geom' not in zone:
warning('Zone {0} without geometry'.format(zone['name']))
continue
shp = shape(zone['geom'])
if not shp.is_valid:
warning(('Skipping invalid polygon for {0}'
'').format(zone['name']))
continue
if shp.is_empty:
warning('Skipping empty polygon for {0}', zone['name'])
continue
geoms.append(shp)
if zone.get('population'):
populations.append(zone['population'])
if zone.get('area'):
areas.append(zone['area'])
if geoms:
geom = aggregate_multipolygons(geoms).__geo_interface__
else:
geom = None
warning('No geometry for {0}', zones)
data = {
'_id': ':'.join((self.id, code)),
'code': code,
'level': self.id,
'name': name,
'population': sum(populations),
'area': sum(areas),
'geom': geom
}
data.update(properties)
return data
def get_place_region(place_geojson, regions_geojson):
place_geom = shape(place_geojson['geometry'])
for feature in regions_geojson['features']:
region_poly = shape(feature['geometry'])
if region_poly.contains(place_geom):
return feature
return None
def shp2shapely(filepath):
"""
Read local shapefile and convert to shapely object for easy processing
:param filepath: Path to Shapefile
:return: Shapely object
"""
# Todo
# -> Should have same structure as view...?
with fiona.open(filepath) as shp:
shape(shp[0]['geometry'])
def __init__(self,
regions_file=REGIONS_FILE,
buffer_file=REGIONS_BUFFER_FILE):
self._buffered_shapes = {}
self._prepared_shapes = {}
self._shapes = {}
self._tree_ids = {}
self._radii = {}
with util.gzip_open(regions_file, 'r') as fd:
regions_data = simplejson.load(fd)
genc_regions = frozenset([rec.alpha2 for rec in genc.REGIONS])
for feature in regions_data['features']:
code = feature['properties']['alpha2']
if code in genc_regions:
shape = geometry.shape(feature['geometry'])
self._shapes[code] = shape
self._prepared_shapes[code] = prepared.prep(shape)
self._radii[code] = feature['properties']['radius']
with util.gzip_open(buffer_file, 'r') as fd:
buffer_data = simplejson.load(fd)
i = 0
envelopes = []
for feature in buffer_data['features']:
code = feature['properties']['alpha2']
if code in genc_regions:
shape = geometry.shape(feature['geometry'])
self._buffered_shapes[code] = prepared.prep(shape)
# Collect rtree index entries, and maintain a separate id to
# code mapping. We don't use index object support as it
# requires un/pickling the object entries on each lookup.
if isinstance(shape, geometry.base.BaseMultipartGeometry):
# Index bounding box of individual polygons instead of
# the multipolygon, to avoid issues with regions crossing
# the -180.0/+180.0 longitude boundary.
for geom in shape.geoms:
envelopes.append((i, geom.envelope.bounds, None))
self._tree_ids[i] = code
i += 1
else:
envelopes.append((i, shape.envelope.bounds, None))
self._tree_ids[i] = code
i += 1
props = index.Property()
props.fill_factor = 0.9
props.leaf_capacity = 20
self._tree = index.Index(envelopes,
interleaved=True, properties=props)
for envelope in envelopes:
self._tree.insert(*envelope)
self._valid_regions = frozenset(self._shapes.keys())
def get_adm_geom(self, point, adm_level, iso3=None):
"""
"""
tmp_int = int(adm_level)
tmp_pnt = Point(point)
query = {}
if iso3 is None:
query['tags'] = 'adm{0}_{1}'.format(
tmp_int, self.adm_suffix)
else:
query['datasets'] = '{0}_adm{1}_{2}'.format(
iso3.lower(), tmp_int, self.adm_suffix)
if iso3 is None or tmp_int != 0:
query['geometry'] = {
'$geoIntersects': {
'$geometry': {
'type': "Point",
'coordinates': [tmp_pnt.x, tmp_pnt.y]
}
}
}
# force use master prod db even for dev (temporary)
f_client = pymongo.MongoClient("128.239.20.76")
results = list(f_client.asdf.features.find(query))
if len(results) == 1:
tmp_results = results[0]
tmp_adm_geom = shape(tmp_results['geometry'])
tmp_iso3 = tmp_results['datasets'][0][:3]
elif len(results) == 0:
warn('no adm (adm level {0}) geom found for '
'point ({1})'.format(tmp_int, tmp_pnt))
tmp_adm_geom = "None"
tmp_iso3 = None
else:
warn('multiple adm (adm level {0}) geoms found for '
'point ({1})'.format(tmp_int, tmp_pnt))
# tmp_adm_geom = "None"
tmp_results = results[0]
tmp_adm_geom = shape(tmp_results['geometry'])
tmp_iso3 = tmp_results['datasets'][0][:3]
return tmp_adm_geom, tmp_iso3
def __createShapes__(self):
shapes = []
if 'features' in self.__jsonData__:
for feature in self.__jsonData__['features']:
if feature['geometry']:
shapes.append(shape(feature['geometry']))
elif 'geometries' in self.__jsonData__:
for geometry in self.__jsonData__['geometries']:
shapes.append(shape(geometry))
return shapes
def test_footprint_shift():
bbox = [(179, 1), (-179, 1), (-179, 2), (179, 2)]
bad_poly = MultiPolygon(
[Polygon(bbox + [bbox[0]])]
)
footprints = Footprints(bbox)
data_area = shape(footprints.data_polygon).area
assert data_area < bad_poly.area
assert data_area == 2
# Since the shape is rectangular, these should be the same
assert data_area == shape(footprints.data_polygon).area
def setup(self):
po1 = [(1,1), (2,2), (1,3), (1,4), (3,3), (3,1), (1,1)]
po2 = [(1,4), (1,5), (3,4), (3,3), (1,4)]
po3 = [(3,1), (3,3), (5,5), (7,3), (5,1), (3,1)]
ZIPS = {
'1': [shape({'type': 'polygon', 'coordinates': [po1]})],
'2': [shape({'type': 'polygon', 'coordinates': [po2]})],
'3': [shape({'type': 'polygon', 'coordinates': [po3]})]
}
self.ZIPS = ZIPS
def test_image_window_at(self):
wv2 = CatalogImage('1030010076B8F500')
c1 = shape(wv2).centroid
window = wv2.window_at(c1, (256, 256))
c2 = shape(window).centroid
bands, x, y = window.shape
# check the window is the correct shape
self.assertEquals(x, 256)
self.assertEquals(y, 256)
# make sure the center of the window is within 1 pixel
# of where it should be
self.assertTrue(c1.distance(c2) < wv2.metadata['georef']['scaleX'])
def _generate_points(self, polygon: Polygon, n) -> list:
"""Generates sample points within a given geometry
:param shapely.geometry.Polygon polygon: the polygon to create points in
:param int n: Number of points to generate in polygon
:return: A list with point in the polygon
:rtype: list[Point]
"""
if n <= 0:
return []
if polygon.area <= 0:
return []
bbox = polygon.envelope
"""(minx, miny, maxx, maxy) bbox"""
if (polygon.area * self.t) < bbox.area:
if (bbox.bounds[2] - bbox.bounds[0]) > (bbox.bounds[3] - bbox.bounds[1]):
bbox_1 = box(*self._bbox_left(bbox.bounds))
bbox_2 = box(*self._bbox_right(bbox.bounds))
else:
bbox_1 = box(*self._bbox_bottom(bbox.bounds))
bbox_2 = box(*self._bbox_top(bbox.bounds))
p1 = shape(polygon)
p1 = p1.difference(bbox_1)
p2 = shape(polygon)
p2 = p2.difference(bbox_2)
del bbox_1, bbox_2
# k = bisect.bisect_left(u, p1.area / polygon.area)
k = int(round(n * (p1.area / polygon.area)))
v = self._generate_points(p1, k) + self._generate_points(p2, n - k)
del polygon, p1, p2
else:
v = []
max_iterations = self.t * n + 5 * math.sqrt(self.t * n)
v_length = len(v)
while v_length < n and max_iterations > 0:
max_iterations -= 1
v.append(self._random_point_in_polygon(polygon))
v_length = len(v)
if len(v) < n:
raise Exception('Too many iterations')
self.logging.debug('Generated %s points', n)
del bbox
return v
def _force_polygon_ccw(geometry):
polygon = shape(geometry)
return mapping(orient(polygon))
def calculate_hydro(self, interval=1000, head_distance=500, get_range=False):
"""
Start at the top of each river, get source GSCDxFlowAccxsquare size.
Calculates flow rate in m3/s, head in m and power in Watts.
Parameters
----------
interval : int, optional (default 1000.)
Interval in metres between testing points.
head_distance : int, optional (default 500.)
How far upstream to look to calculate head.
get_range : boolean, optional (default False.)
Whether to attempt to calculate mean, median, max etc.
"""
# TODO: Make the contribution % dependent on slope (elevation change/distance).
# Choose the point_interval in metres
# This loops through each stream and adds points down it at the specified interval
# And creates a dict with these new point geometries
# At 20 deg South:
# 1 degree = 110704 metres -> 1 minute = 1845 metres -> 1 second = 30.75 metres
# River network is 15 second resolution = 461 metres
# Therefore each up_cell size is
cell_area = (110704/60/60*15)**2
proj = pyproj.Proj(init='epsg:3395') # Mercator
rivers = self.rivers.to_crs(epsg=3395)
rho = 998.57 # density of water, kg/m3
g = 9.81 # acceleration due to gravity, m/2
n = 0.5 # overall efficiency of system
hydro_points_dicts = []
count = 0
for _, row in rivers.iterrows():
geom = shape(row['geometry'])
length = geom.length
for _, distance in enumerate(range(0, int(length), interval)):
arcid = row['arcid']
up_cells = row['up_cells']
point = Point(proj(*list(geom.interpolate(distance).coords)[0], inverse=True))
upstream_point = Point(proj(*list(geom.interpolate(distance - head_distance).coords)[0], inverse=True))
elevation = next(self.dem.sample(list(point.coords))).tolist()[0]
upstream_elevation = next(self.dem.sample(list(upstream_point.coords))).tolist()[0]
head = upstream_elevation - elevation
runoff = next(self.flow.sample(list(point.coords))).tolist()[0]
flowrate = runoff * up_cells * cell_area * (1/1000) * (1/(8760*3600)) # convert mm/year to m3/s
power = rho*g*n*flowrate*head
if head > 0 and flowrate > 0:
hydro_points_dicts.append({'arcid': arcid, 'elevation': elevation, 'head': head,
'flowrate': flowrate, 'power': power, 'geometry': point})
count += 1
if count % 10000 == 0:
print(count)
self.hydro_points = gpd.GeoDataFrame(hydro_points_dicts)
self.hydro_points.crs = {'init' :'epsg:4326'}
print('Calculated hydro potential')
if get_range:
eta_t=0.88
eta_g=0.96
conv=0.6
# transfer discharge values from the river to the points
self.hydro_points['discharge_accum'] = -99
self.hydro_points['discharge_max'] = -99
self.hydro_points['discharge_mean'] = -99
self.hydro_points['discharge_min'] = -99
for index, row in self.nodes_out.iterrows():
self.hydro_points.loc[index, 'discharge_accum'] = \
rivers.loc[self.nodes_out.loc[index, 'arcid'], 'discharge_accum']
self.hydro_points.loc[index, 'discharge_max'] = \
rivers.loc[self.nodes_out.loc[index, 'arcid'], 'discharge_max']
self.hydro_points.loc[index, 'discharge_mean'] = \
rivers.loc[self.nodes_out.loc[index, 'arcid'], 'discharge_mean']
self.hydro_points.loc[index, 'discharge_min'] = \
rivers.loc[self.nodes_out.loc[index, 'arcid'], 'discharge_min']
# calculate the power in watts based on Alex's formula
self.hydro_points['power_accum'] = rho * g * eta_t * eta_g * conv * self.nodes_out['discharge_accum'] * self.nodes_out['head']
self.hydro_points['power_max'] = rho * g * eta_t * eta_g * conv * self.nodes_out['discharge_max'] * self.nodes_out['head']
self.hydro_points['power_mean'] = rho * g * eta_t * eta_g * conv * self.nodes_out['discharge_mean'] * self.nodes_out['head']
self.hydro_points['power_min'] = rho * g * eta_t * eta_g * conv * self.nodes_out['discharge_min'] * self.nodes_out['head']
def plot_multipolygon(ax, geom, color='red'):
""" Can safely call with either Polygon or Multipolygon geometry
"""
if geom.type == 'Polygon':
plot_polygon(ax, geom, color)
elif geom.type == 'MultiPolygon':
for poly in geom.geoms:
plot_polygon(ax, poly, color)
cwd = os.path.dirname(os.path.abspath(__file__))
datadir = os.path.join(os.path.split(cwd)[0], 'data')
jsonfile = os.path.join(datadir, 'manhattan_island_proj.json')
citibikefile = os.path.join(datadir, 'citibike.json')
manhattan = shape(geojson.load(open(jsonfile)))
man_arr = np.asarray(manhattan.exterior)
# load citbike station locations, transform to map coordinates,
# and filter for only those in Manhattan
c = json.load(open(citibikefile))
stations = [(x['longitude'], x['latitude']) for x in c['stationBeanList']]
lon, lat = zip(*stations)
nyp = Proj(
'+datum=NAD83 +lat_0=40.1666666667 +lat_1=40.6666666667 '
'+lat_2=41.0333333333 +lon_0=-74 +no_defs +proj=lcc +units=us-ft '
'+x_0=300000 +y_0',
preserve_units=True)
wgs84 = Proj(init='epsg:4326')
x, y = transform(wgs84, nyp, lon, lat)
points = MultiPoint(list(zip(x, y)))
def read_fiona(source, prop_map, filterer=None):
"""Process a fiona collection
"""
collection = {
'type': 'FeatureCollection',
'features': [],
'bbox': [float('inf'),
float('inf'),
float('-inf'),
float('-inf')]
}
skipped_count = 0
failed_count = 0
transformer = partial(transform_geom,
source.crs,
'EPSG:4326',
antimeridian_cutting=True,
precision=6)
for feature in source:
if filterer is not None and not filterer.keep(feature):
skipped_count += 1
continue
if feature['geometry'] is None:
logging.error("empty geometry")
failed_count += 1
continue
try:
transformed_geometry = transformer(
_force_geometry_2d(feature['geometry']))
fixed_geometry = _fix_geometry(transformed_geometry)
feature['geometry'] = _force_geometry_ccw(fixed_geometry)
feature['properties'] = get_transformed_properties(
feature['properties'], prop_map)
shapely_geometry = shape(feature['geometry'])
geom_aea = ops.transform(
partial(
pyproj.transform, pyproj.Proj(init='EPSG:4326'),
pyproj.Proj(proj='aea',
lat1=shapely_geometry.bounds[1],
lat2=shapely_geometry.bounds[3])),
shapely_geometry)
feature['properties']['acres'] = round(geom_aea.area /
4046.8564224)
collection['bbox'] = [
comparator(values) for comparator, values in zip(
[min, min, max, max],
list(zip(collection['bbox'], _bbox(feature))))
]
collection['features'].append(feature)
except PropertyMappingFailedException as e:
logging.error(str(e) + ": " + str(feature['properties']))
failed_count += 1
except Exception as e:
logging.error(str(e), "error processing feature: " + str(feature))
failed_count += 1
#avoid math error if there are no features
if len(collection['features']) == 0:
del collection['bbox']
logging.info("skipped %i features, kept %i features, errored %i features" %
(skipped_count, len(collection['features']), failed_count))
return collection
def create_index(features):
"""Create an R-Tree index from a set of features"""
index = rtree.index.Index()
for id, feature in enumerate(features):
index.insert(id, shape(feature['geometry']).bounds)
return index
car_shape = Polygon([[-120.84, 40], [-120, 5], [-104.5, 10], [-92, 1.416666031],
[-92.0, -0.5], [-89.8, -0.5], [-58.8, -0.5], [-58.8, 40], [-120.84, 40]])
nam_polygons = []
afi_polygons = []
russia_polygons = []
asia_polygons = []
mid_polygons =[]
sam_polygons = []
kzwy_polygons = []
uaaa_polygons = []
utaa_polygons = []
for _feature in faa_airspace_data['features']:
if _feature['properties']['NAME'] == 'ANCHORAGE ARCTIC FIR':
(minx, miny, maxx, maxy) = shape(_feature['geometry']).bounds
north_of_alaska = Polygon([[maxx+1, miny], [minx, miny], [minx, maxy],
[maxx+1, maxy]])
nam_polygons.append(north_of_alaska)
if (_feature['properties']['COUNTRY'] == 'United States' and
_feature['properties']['TYPE_CODE'] == 'ARTCC' and
_feature['properties']['LOCAL_TYPE'] == 'ARTCC_L' or
_feature['properties']['TYPE_CODE'] == 'FIR' and
_feature['properties']['IDENT'] in ['PAZA', 'CZVR', 'CZWG', 'CZYZ',
'CZQM', 'CZUL']):
nam_polygons.append(shape(_feature['geometry']))
if (_feature['properties']['TYPE_CODE'] == 'FIR' and
_feature['properties']['IDENT'] in ['UHMM',]):
russia_polygons.append(shape(_feature['geometry']))
if (_feature['properties']['TYPE_CODE'] == 'FIR' and
_feature['properties']['IDENT'] in ['KZAK', 'NZZO', 'RJJJ',
from shapely.geometry import shape, mapping
url = 'https://opendata.arcgis.com/datasets/ef4b445a53c1406892257fe63129a8ea_0.geojson'
link_to_page = 'https://npgeo-corona-npgeo-de.hub.arcgis.com/datasets/' \
'ef4b445a53c1406892257fe63129a8ea_0?geometry=-67.085%2C46.270%2C83.340%2C55.886'
r = requests.get(url, allow_redirects=True)
last_fetch = datetime.now()
open('static/bundeslaender_simplify200.geojson', 'wb').write(r.content)
data = r.json()
lat_bounds = [47.27012360470944, 55.099168977100774]
long_bounds = [5.866755374775609, 15.04181565646822]
for feature in data["features"]:
name = feature["properties"]["LAN_ew_GEN"]
if "Bodensee" in name:
continue
try:
polygon = max(shape(feature["geometry"]), key=lambda a: a.area)
except:
polygon = shape(feature["geometry"])
feature["geometry"] = mapping(polygon.simplify(0.01))
print(name)
ncases = feature["properties"]["Fallzahl"]
feature["properties"]["cases"] = ncases
with open('static/bundeslaender_simplify200.geojson', 'w') as json_file:
json.dump(data, json_file)
def download(self, source_urls, workdir, conform=None):
output_files = []
download_path = os.path.join(workdir, 'esri')
mkdirsp(download_path)
query_fields = self.field_names_to_request(conform)
for source_url in source_urls:
size = 0
file_path = self.get_file_path(source_url, download_path)
if os.path.exists(file_path):
output_files.append(file_path)
_L.debug("File exists %s", file_path)
continue
downloader = EsriDumper(source_url, parent_logger=_L)
metadata = downloader.get_metadata()
if query_fields is None:
field_names = [f['name'] for f in metadata['fields']]
else:
field_names = query_fields[:]
if X_FIELDNAME not in field_names:
field_names.append(X_FIELDNAME)
if Y_FIELDNAME not in field_names:
field_names.append(Y_FIELDNAME)
if GEOM_FIELDNAME not in field_names:
field_names.append(GEOM_FIELDNAME)
# Get the count of rows in the layer
try:
row_count = downloader.get_feature_count()
_L.info("Source has {} rows".format(row_count))
except EsriDownloadError:
_L.info("Source doesn't support count")
with csvopen(file_path, 'w', encoding='utf-8') as f:
writer = csvDictWriter(f,
fieldnames=field_names,
encoding='utf-8')
writer.writeheader()
for feature in downloader:
try:
geom = feature.get('geometry') or {}
row = feature.get('properties') or {}
if not geom:
raise TypeError("No geometry parsed")
if any((isinstance(g, float) and math.isnan(g))
for g in traverse(geom)):
raise TypeError("Geometry has NaN coordinates")
shp = shape(feature['geometry'])
row[GEOM_FIELDNAME] = shp.wkt
try:
centroid = shp.centroid
except RuntimeError as e:
if 'Invalid number of points in LinearRing found' not in str(
e):
raise
xmin, xmax, ymin, ymax = shp.bounds
row[X_FIELDNAME] = round(xmin / 2 + xmax / 2, 7)
row[Y_FIELDNAME] = round(ymin / 2 + ymax / 2, 7)
else:
row[X_FIELDNAME] = round(centroid.x, 7)
row[Y_FIELDNAME] = round(centroid.y, 7)
writer.writerow(
{fn: row.get(fn)
for fn in field_names})
size += 1
except TypeError:
_L.debug("Skipping a geometry", exc_info=True)
_L.info("Downloaded %s ESRI features for file %s", size, file_path)
output_files.append(file_path)
return output_files
features = ways_to_polygons(features)
return features
if __name__ == '__main__':
progress = tqdm(total=len(CITIES) * len(OSM_FEATURES))
for city_name in CITIES:
city = City(city_name)
dst_dir = os.path.join(city.intermediary_dir, 'osm')
os.makedirs(dst_dir, exist_ok=True)
osm = Downloader(aoi=shape(city.aoi), epsg=city.epsg, dst_dir=dst_dir)
for key, filename in OSM_FEATURES.items():
features = osm.get_feature(key)
features = features[features.is_valid]
features = features.to_crs(city.crs)
features = features[features.intersects(shape(city.aoi))]
geoms = features.intersection(shape(city.aoi))
features = features.assign(geometry=geoms)
output_f = os.path.join(dst_dir, filename)
features.to_file(output_f)
progress.update(1)
progress.close()
#!/usr/bin/python
from shapely.geometry import Point, mapping, shape
from fiona import collection
with collection("31471050500_face.shp", "r") as input:
for geo in input:
print shape(geo['geometry'])
print geo['properties']
def execute(trial=False):
'''Retrieve some data sets (not using the API here for the sake of simplicity).'''
startTime = datetime.datetime.now()
# Set up the database connection.
client = dml.pymongo.MongoClient()
repo = client.repo
repo.authenticate('soohyeok_soojee', 'soohyeok_soojee')
neighborhoodData = repo['soohyeok_soojee.get_neighborhoods'].find()
# AllCoordOfLandmark = repo['soohyeok_soojee.transform_landmark'].find()[0]['coordinates']
LandmarkAndTown = repo['soohyeok_soojee.transform_landmark'].find(
)[0]['LandmarkAndTown']
CrimeAndTown = repo['soohyeok_soojee.transform_crime'].find(
)[0]['CrimeAndTown']
if (trial == True):
neighborhoodData = neighborhoodData.sample(frac=.05)
# AllCoordOfLandmark = AllCoordOfLandmark.sample(frac=.05)
LandmarkAndTown = LandmarkAndTown.sample(frac=.05)
CrimeAndTown = CrimeAndTown.sample(frac=.05)
neighborhoods = {}
for n in neighborhoodData:
key = n['properties']['Name']
neighborhoods[key] = n['geometry']
for name in CrimeAndTown:
for point in CrimeAndTown[name]:
dist = [[distance.euclidean(point, x)]
for x in LandmarkAndTown[name]]
if dist != []:
index = np.argmin(dist)
p = LandmarkAndTown[name][index]
LandmarkAndTown[name].remove(p)
# AllCoordOfLandmark.remove(p)
# data = AllCoordOfLandmark
data = [
point for name in LandmarkAndTown
for point in LandmarkAndTown[name]
]
# kmean plot
kmeans = KMeans(n_clusters=10).fit(data)
data = np.array(data)
pyplot.scatter(data[:, 0], data[:, 1], c=kmeans.labels_)
pyplot.scatter(kmeans.cluster_centers_[:, 0],
kmeans.cluster_centers_[:, 1],
marker='x',
c='red')
pyplot.show()
centroid = [[x[0], x[1]] for x in kmeans.cluster_centers_]
towns = [
name for name in neighborhoods for point in centroid
if Point(point).within(shape(neighborhoods[name]))
]
result = {
'centroid': centroid,
'towns': towns,
'Coordinates': LandmarkAndTown
}
repo.dropCollection("kmeans_landmark_crime")
repo.createCollection("kmeans_landmark_crime")
repo['soohyeok_soojee.kmeans_landmark_crime'].insert_many([result])
repo['soohyeok_soojee.kmeans_landmark_crime'].metadata(
{'complete': True})
print(repo['soohyeok_soojee.kmeans_landmark_crime'].metadata())
repo.logout()
endTime = datetime.datetime.now()
return {"start": startTime, "end": endTime}
for barrio in js['features']:
# mean
mean_proximo = 10000
mean_sintetico = 0
# percentile_5
per5_proximo = 10000
per5_sintetico = 0
# median
median_proximo = 10000
median_sintetico = 0
# percentile_95
per95_proximo = 10000
per95_sintentico = 0
polygon = shape(barrio['geometry'])
centroide = polygon.centroid.wkt
# print(centroide)
for index, medida in data_estacion.iterrows():
point = Point(medida['lon'], medida['lat'])
distance = polygon.exterior.distance(point)
radio = .068
#mean
if mean_proximo > distance:
mean_proximo = distance
mean_sintetico = medida['mean'] * math.exp(
-math.pow(distance, 2) / math.pow((2 * radio), 2))
#per5
if per5_proximo > distance:
def simplifyGeom(geom):
# Simplify complex polygons
shp = geometry.shape(geom)
simp = shp.simplify(SIMPLIFICATION_TOLERANCE, PRESERVE_TOPOLOGY)
return geometry.mapping(simp)
# print(chko.bounds)
# print(chko.schema)
# print(chko.schema["geometry"])
# print(json.dumps(chko.meta, sort_keys=True, indent=4, separators=(',', ': ')))
# print(len(chko)) # Vrati pocet prvku
# for prvek in chko:
# print(prvek["properties"]["NAZEV"]) # projde vsechny prvky
# print(chko[10]) # Vrati kontkretni prvek
# pocet = 0
# for prvek in chko:
# if prvek["properties"]["NAZEV"] == "České středohoří":
# pocet += 1
# print(pocet)
# Knihovna shapely
cr = chko[54]
geom = shape(cr["geometry"])
# print(geom) # textova reprezentace objektu
# print(geom.type) # vypise typ objektu
# print(dir(geom)) # vypise vsechny funkce a atributy geometrie
# print(geom.area)
# print("centroid: ", mapping(geom.centroid)) # vrati hodnotu centroidu jako bod, mapping prevadi do json
# print("obvod: ", geom.exterior.length)
with open(os.path.join(cesta_data, "bod_centroid.geojson", "w")) as out: # uloyi geojson
data = {
"type":"FeatureCollection",
"features": [
{
"type":"feature",
"geometry": mapping(geom.exterior)
def create_instituitons_countries_dataset() -> None:
"""Creates a csv mapping latitude and longitude of institutions in MAG to their country names.
Based on https://stackoverflow.com/a/46589405/530399
:return: None
:rtype: None
"""
logger = logging.getLogger(__name__)
logger.info(f'Running {create_instituitons_countries_dataset.__name__}')
# setup
logger.info('Loading config')
app_cfg = ConfigLoader.load_config()
logger.info('Mapping Latitude/longitude to country names')
countries_geojson_filepath = join(app_cfg['paths']['data_dir'], 'external/countries.geojson')
with open(countries_geojson_filepath) as json_file:
data = json.load(json_file)
countries = {}
for feature in data["features"]:
geom = feature["geometry"]
country = feature["properties"]["ADMIN"]
countries[country] = prep(shape(geom))
def get_country(lon, lat):
try:
lon=float(lon)
lat=float(lat)
point = Point(lon, lat)
for country, geom in countries.items():
if geom.contains(point):
return country.lower()
return None
except:
return None
# print("\n\n\nThis is Austria "+str(get_country(10.0, 47.0)))
# bprint(get_country(None,None))
# bprint(get_country("",""))
# bprint(get_country("text","nonsense"))
# bprint(get_country("10.0", "47.0"))
# bprint(get_country(10.0, 47.0))
affiliations_lat_long_filepath = join(app_cfg['paths']['data_dir'], 'raw/dataset_affiliations_lat_long/affiliation_id_lat_long.csv')
# bprint(affiliations_lat_long_filepath)
affiliations_lat_long_df = pd.read_csv(affiliations_lat_long_filepath, header=0, sep=",")
affiliations_lat_long_df = affiliations_lat_long_df.dropna() # Delete all records where any lat/long value is NaN
def get_country_for_df_row(row):
# Based on http://jonathansoma.com/lede/foundations/classes/pandas%20columns%20and%20functions/apply-a-function-to-every-row-in-a-pandas-dataframe/
return get_country(row['longitude'], row['latitude'])
affiliations_lat_long_df['country'] = affiliations_lat_long_df.apply(get_country_for_df_row, axis=1)
# print(affiliations_lat_long_df.head())
affiliations_lat_long_df = affiliations_lat_long_df.dropna() # Delete all records where any the country name could not be determined == NaN
# Write the output as csv
affiliations_countryname_filepath = join(app_cfg['paths']['data_dir'], 'processed/affiliations_country.csv')
affiliations_lat_long_df.to_csv(affiliations_countryname_filepath, sep=',', index=False, header=True)
logger.info('Wrote affiliations_countryname dataset to '+affiliations_countryname_filepath)
logger.info(f'Done running {create_instituitons_countries_dataset.__name__}')
def shp2df(shplist,
index=None,
index_dtype=None,
clipto=[],
filter=None,
true_values=None,
false_values=None,
layer=None,
skip_empty_geom=True):
"""Read shapefile/DBF, list of shapefiles/DBFs, or File geodatabase (GDB)
into pandas DataFrame.
Parameters
----------
shplist : string or list
of shapefile/DBF name(s) or FileGDB
index : string
Column to use as index for dataframe
index_dtype : dtype
Enforces a datatype for the index column (for example, if the index field is supposed to be integer
but pandas reads it as strings, converts to integer)
clipto : list
limit what is brought in to items in index of clipto (requires index)
filter : tuple (xmin, ymin, xmax, ymax)
bounding box to filter which records are read from the shapefile.
true_values : list
same as argument for pandas read_csv
false_values : list
same as argument for pandas read_csv
layer : str
Layer name to read (if opening FileGDB)
skip_empty_geom : True/False, default True
Drops shapefile entries with null geometries.
DBF files (which specify null geometries in their schema) will still be read.
Returns
-------
df : DataFrame
with attribute fields as columns; feature geometries are stored as
shapely geometry objects in the 'geometry' column.
"""
if isinstance(shplist, str):
shplist = [shplist]
if not isinstance(true_values, list) and true_values is not None:
true_values = [true_values]
if not isinstance(false_values, list) and false_values is not None:
false_values = [false_values]
if len(clipto) > 0 and index:
clip = True
else:
clip = False
df = pd.DataFrame()
for shp in shplist:
print("\nreading {}...".format(shp))
shp_obj = fiona.open(shp, 'r', layer=layer)
if index is not None:
# handle capitolization issues with index field name
fields = list(shp_obj.schema['properties'].keys())
index = [f for f in fields if index.lower() == f.lower()][0]
attributes = []
# for reading in shapefiles
meta = shp_obj.meta
if meta['schema']['geometry'] != 'None':
if filter is not None:
print('filtering on bounding box {}, {}, {}, {}...'.format(
*filter))
if clip: # limit what is brought in to items in index of clipto
for line in shp_obj.filter(bbox=filter):
props = line['properties']
if not props[index] in clipto:
continue
props['geometry'] = line.get('geometry', None)
attributes.append(props)
else:
for line in shp_obj.filter(bbox=filter):
props = line['properties']
props['geometry'] = line.get('geometry', None)
attributes.append(props)
print(
'--> building dataframe... (may take a while for large shapefiles)'
)
shp_df = pd.DataFrame(attributes)
# reorder fields in the DataFrame to match the input shapefile
if len(attributes) > 0:
shp_df = shp_df[list(attributes[0].keys())]
# handle null geometries
if len(shp_df) == 0:
print('Empty dataframe! No features were read.')
if filter is not None:
print('Check filter {} for consistency \
with shapefile coordinate system'.format(filter))
geoms = shp_df.geometry.tolist()
if geoms.count(None) == 0:
shp_df['geometry'] = [shape(g) for g in geoms]
elif skip_empty_geom:
null_geoms = [i for i, g in enumerate(geoms) if g is None]
shp_df.drop(null_geoms, axis=0, inplace=True)
shp_df['geometry'] = [
shape(g) for g in shp_df.geometry.tolist()
]
else:
shp_df['geometry'] = [
shape(g) if g is not None else None for g in geoms
]
# for reading in DBF files (just like shps, but without geometry)
else:
if clip: # limit what is brought in to items in index of clipto
for line in shp_obj:
props = line['properties']
if not props[index] in clipto:
continue
attributes.append(props)
else:
for line in shp_obj:
attributes.append(line['properties'])
print(
'--> building dataframe... (may take a while for large shapefiles)'
)
shp_df = pd.DataFrame(attributes)
# reorder fields in the DataFrame to match the input shapefile
if len(attributes) > 0:
shp_df = shp_df[list(attributes[0].keys())]
shp_obj.close()
if len(shp_df) == 0:
continue
# set the dataframe index from the index column
if index is not None:
if index_dtype is not None:
shp_df[index] = shp_df[index].astype(index_dtype)
shp_df.index = shp_df[index].values
df = df.append(shp_df)
# convert any t/f columns to numpy boolean data
if true_values is not None or false_values is not None:
replace_boolean = {}
for t in true_values:
replace_boolean[t] = True
for f in false_values:
replace_boolean[f] = False
# only remap columns that have values to be replaced
cols = [c for c in df.columns if c != 'geometry']
for c in cols:
if len(set(replace_boolean.keys()).intersection(set(
df[c]))) > 0:
df[c] = df[c].map(replace_boolean)
return df
def test_tiles_from_bbox():
"""Get tiles intersecting with bounding box."""
test_bbox = shape({
"type":
"Polygon",
"coordinates": [[
(5.625, 61.875),
(56.25, 61.875),
(56.25, 28.125),
(5.625, 28.125),
(5.625, 28.125),
(5.625, 61.875),
]],
})
test_tiles = {
(5, 5, 33),
(5, 6, 33),
(5, 7, 33),
(5, 8, 33),
(5, 9, 33),
(5, 10, 33),
(5, 5, 34),
(5, 6, 34),
(5, 7, 34),
(5, 8, 34),
(5, 9, 34),
(5, 10, 34),
(5, 5, 35),
(5, 6, 35),
(5, 7, 35),
(5, 8, 35),
(5, 9, 35),
(5, 10, 35),
(5, 5, 36),
(5, 6, 36),
(5, 7, 36),
(5, 8, 36),
(5, 9, 36),
(5, 10, 36),
(5, 5, 37),
(5, 6, 37),
(5, 7, 37),
(5, 8, 37),
(5, 9, 37),
(5, 10, 37),
(5, 5, 38),
(5, 6, 38),
(5, 7, 38),
(5, 8, 38),
(5, 9, 38),
(5, 10, 38),
(5, 5, 39),
(5, 6, 39),
(5, 7, 39),
(5, 8, 39),
(5, 9, 39),
(5, 10, 39),
(5, 5, 40),
(5, 6, 40),
(5, 7, 40),
(5, 8, 40),
(5, 9, 40),
(5, 10, 40),
(5, 5, 41),
(5, 6, 41),
(5, 7, 41),
(5, 8, 41),
(5, 9, 41),
(5, 10, 41),
}
tp = TilePyramid("geodetic")
bbox_tiles = {tile.id for tile in tp.tiles_from_bbox(test_bbox, 5)}
assert test_tiles == bbox_tiles
args = parser.parse_args()
# Import shapefile specified at commandline
shp = args.shp
# Can override the hardcoded maps directory
if args.dir:
MAP_DATA_FP = os.path.join(args.dir, 'processed/maps')
if args.newmap:
MAP_DATA_FP = os.path.join(MAP_DATA_FP, args.newmap)
if not os.path.exists(MAP_DATA_FP):
os.mkdir(MAP_DATA_FP)
# Get all lines, dummy id
lines = [(line[0], shape(line[1]['geometry']))
for line in enumerate(fiona.open(shp))]
print 'Extracting intersections and writing into ' + MAP_DATA_FP
inters = []
pkl_file = os.path.join(MAP_DATA_FP, 'inters.pkl')
if not os.path.exists(pkl_file) or args.forceupdate:
print 'Generating intersections...'
inters = generate_intersections(lines)
# Save to pickle in case script breaks
with open(pkl_file, 'w') as f:
cPickle.dump(inters, f)
else:
print 'Reading intersections from ' + pkl_file
with open(pkl_file, 'r') as f:
inters = cPickle.load(f)
def model_fit(request, model_id):
model = PredictionModel.objects.get(id=model_id)
areas = read_frame(model.area.all())
areavars = []
for index1, row1 in areas.iterrows():
df = read_frame(
Site.objects.filter(feature_type=FeatureType.objects.get(
name=row1["feature_type"])))
polygon = shape(row1["geom"])
select = []
for index2, row2 in df.iterrows():
select.append(polygon.contains(shape(row2['geom'])))
data = read_frame(
FeatureData.objects.filter(site__in=df[select]['id']),
index_col="date")
data["area"] = row1["name"]
data["feature_type"] = row1["feature_type"]
areavars.append(data)
lagvars = []
for i in range(len(areavars)):
ft = areavars[i].area.unique() + " " + areavars[i].feature_type.unique(
)
d = areavars[i].pivot(columns='site', values='value')
if len(d.columns) > 1:
d = pd.DataFrame(d.mean(axis=1, skipna=True))
for j in [1, 2, 3, 4, 5]:
df = pd.DataFrame()
df[ft + '_shift_' + str(j)] = d.rolling(window=j).mean().shift(1)
lagvars.append(df)
res = pd.concat(lagvars, axis=1)
res = res[res.index.month.isin([5, 6, 7, 8, 9])].reset_index()
rain_cols = [col for col in res.columns if 'Rainfall' in col]
res[rain_cols] = res[rain_cols].add(1).apply(np.log)
FIB = read_frame(FeatureData.objects.filter(site=model.site.all()[0]))
FIB["date"] = FIB.date.round("D")
d = FIB.merge(res, on="date").drop("variable", axis=1)
D = d.dropna()
y = np.log10(D["value"])
X = D.drop(["date", "value", "id", "site"], axis=1)
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.3,
random_state=4)
if model.fit == None:
grid = {
'n_estimators': [1000],
'max_depth': np.linspace(1, 6, 6),
'max_features': [4, 6, 8],
'min_samples_leaf': [1]
}
# Instantiate the Random regressor: elastic_net
rf = RandomForestRegressor()
# Setup the GridSearchCV object: gm_cv
gm_cv = GridSearchCV(rf, param_grid=grid, cv=5)
# Fit it to the training data
gm_cv.fit(X_train, y_train)
rf = RandomForestRegressor(
n_estimators=gm_cv.best_params_["n_estimators"],
max_depth=gm_cv.best_params_["max_depth"],
max_features=gm_cv.best_params_["max_features"],
min_samples_leaf=1)
rf.fit(X_train, y_train)
model.fit = pickle.dumps(rf)
model.save()
rf = pickle.loads(model.fit)
# Predict on the test set and compute metrics
#y_pred1 = gm_cv.predict(X_test)
y_pred = rf.predict(X_test)
#r2 = rf.score(X_test, y_test)
#r2_in = rf.score(X_train, y_train)
#mse = mean_squared_error(y_test, y_pred)
#mse_in = mean_squared_error(y_train, rf.predict(X_train))
df_test = pd.DataFrame({
'meas': y_test,
'pred': rf.predict(X_test),
'split': 'out of sample'
})
df_train = pd.DataFrame({
'meas': y_train,
'pred': rf.predict(X_train),
'split': 'in sample'
})
df = pd.concat([df_test, df_train])
fig = px.scatter(df,
x="meas",
y="pred",
color="split",
color_discrete_sequence=['#212c52', '#75c3ff'])
fig.update_layout(
font_family="Helvetica Neue, Helvetica, Arial, sans-serif",
font_color="black",
title={'text': 'Model fit of Random Forest model'},
xaxis_title="measured data (sample)",
yaxis_title="fitted values (in sample fit)",
#markersize= 12,
)
fig.update_layout(
legend=dict(yanchor="top", title=None, y=0.99, xanchor="left", x=0.01))
fig.update_traces(
marker_size=8) #['#75c3ff', "red"],#, marker_line_color='#212c52',
# marker_line_width=1.5, opacity=1)
model_fit = plot(fig, output_type="div")
importances = pd.Series(data=rf.feature_importances_, index=X.columns)
# Sort importances
importances_sorted = importances.sort_values()
importances_df = importances_sorted.reset_index()
importances_df.columns = ["feature", "importance"]
fig = px.bar(importances_df, y="feature", x="importance", orientation='h')
fig.update_layout(
font_family="Helvetica Neue, Helvetica, Arial, sans-serif",
font_color="black",
title={'text': 'Feature importance of Random Forest model'}
#markercolor = "#212c52"
)
fig.update_traces(marker_color='#75c3ff',
marker_line_color='#75c3ff',
marker_line_width=1.5,
opacity=1)
feature_importance = plot(fig, output_type="div")
bathingspot = model.site.all()[0]
return render(
request, 'ews/model_fit.html', {
'bathingspot': bathingspot,
"entries": Site.objects.all(),
"model": model,
'areas': model.area.all(),
'model_fit': model_fit,
'feature_importance': feature_importance
})
def getBoundary(self, size_x=512, size_y=512, force=False):
# Return the img if already loaded
if (self.img != "N/A"):
return self.img
# Find the "shape" of the boundary, if we haven't already
if (self.boundary == "N/A"):
self.boundary = self.getFeature()
# If the "shape" of the boundary cannot be found, that's all we can do
if (self.boundary == "N/A"):
return "N/A"
# Check if the "img" has already been cached, and just open it if it is there
os.makedirs("cached_property_images", exist_ok=True)
cache_file = os.path.join(
"cached_property_images", self.boundary['properties']['LOT'] +
"_" + self.boundary['properties']['PLAN'] + ".png")
if (os.path.exists(cache_file) and (force == False)):
self.img = Image.open(cache_file)
return self.img
print("Isolating farm", end='')
with open(
os.path.join(
"geometries", "geo-x" + str(self.tile_x) + "-y" +
str(self.tile_y) + ".geojson")) as f1:
geo_json_features = json.load(f1)["features"]
tile = GeometryCollection([
shape(feature["geometry"]).buffer(0)
for feature in geo_json_features
])
self.img = Image.new('RGBA', (size_x, size_y))
count_hit = 0
result = GeometryCollection()
if shape(self.boundary["geometry"]).intersects(tile):
result = result.union(
shape(self.boundary["geometry"]).intersection(tile))
if tile.intersects(result):
for y in range(size_y):
if (y % 15 == 0):
#print("y="+ str(y) + ", hits=" + str(count_hit))
print(".", end="")
for x in range(size_x):
lat_long = self.GetLatLongForCoords(y, x)
if result.intersects(Point(lat_long)):
self.img.putpixel((y, x), (0, 0, 255, 255)) # Blue
count_hit += 1
#print("Hits: " + str(count_hit))
# Find the border
for y in range(size_y):
for x in range(size_x):
# It is a border if it's not blue...
if (self.img.getpixel((y, x)) != (0, 0, 255, 255)):
# And it has a neighbour that is blue
for j in range(y - 1, y + 2):
for i in range(x - 1, x + 2):
# Check for neighbour out of bounds, don't compare pixel to itself
if ((j >= 0) and (j < size_y) and (i >= 0)
and (i < size_x)
and ((j != y) or (i != x))):
if (self.img.getpixel(
(j, i)) == (0, 0, 255, 255)):
# This is part of the border!
self.img.putpixel((y, x), (255, 0, 0, 255))
# Save the image to the cache
self.img.save(cache_file)
print(" done")
return self.img
def feature_from_row(Community, Address, Mission, MissionType, City,
CommunityType, Website):
feature = {
'type': 'Feature',
'properties': {
'CommunityPartner': '',
'Address': '',
'Projects': '',
'College Name': '',
'Mission Type': '',
'Project Name': '',
'Legislative District Number': '',
'Number of projects': '',
'Income': '',
'City': '',
'County': '',
'Mission Area': '',
'CommunityType': '',
'Campus Partner': '',
'Academic Year': '',
'Website': ''
},
'geometry': {
'type': 'Point',
'coordinates': []
}
}
geocode_result = gmaps.geocode(Address) # get the coordinates
if (geocode_result[0]):
latitude = geocode_result[0]['geometry']['location']['lat']
longitude = geocode_result[0]['geometry']['location']['lng']
feature['geometry']['coordinates'] = [longitude, latitude]
coord = Point([longitude, latitude])
for i in range(
len(district)): # iterate through a list of district polygons
property = district[i]
polygon = shape(property['geometry']) # get the polygons
if polygon.contains(coord): # check if a partner is in a polygon
feature['properties'][
'Legislative District Number'] = property["properties"][
"id"] # assign the district number to a partner
for m in range(len(county)): # iterate through the County Geojson
properties2 = county[m]
polygon = shape(properties2['geometry']) # get the polygon
if polygon.contains(
coord
): # check if the partner in question belongs to a polygon
feature['properties']['County'] = properties2['properties'][
'NAME']
feature['properties']['Income'] = properties2['properties'][
'Income']
projectlist = 0
yearlist = []
campuslist = []
projectList = []
collegeList = []
partners = dfProjects['community_partner']
years = dfProjects['academic_year']
campuses = dfProjects['campus_partner']
projects = dfProjects['project_name']
colleges = dfProjects['college_name']
count = 0
for n in range(len(partners)):
if (partners[n] == Community):
if (years[n] not in yearlist):
yearlist.append(years[n])
if (campuses[n] not in campuslist):
campuslist.append(campuses[n])
if (projects[n] not in projectList):
projectList.append(projects[n])
count += 1
if (colleges[n] not in collegeList):
collegeList.append(colleges[n])
feature['properties']['Number of projects'] = count
feature['properties']['Campus Partner'] = campuslist
feature['properties']['Academic Year'] = yearlist
feature['properties']['Projects'] = projectList
feature['properties']['College Name'] = collegeList
feature['properties']['CommunityPartner'] = Community
feature['properties']['CommunityType'] = CommunityType
feature['properties']['Website'] = Website
feature['properties']['Mission Area'] = Mission
feature['properties']['Mission Type'] = MissionType
feature['properties']['City'] = City
collection['features'].append(feature)
return feature
def calc_matrix():
"""
Fill the matrix with the passengers numbers from the raw data
"""
# Reading the data file
data = open(DATA_FILE, 'r')
reader = csv.reader(data, delimiter=',')
# Two unnecessary lines
reader.next()
reader.next()
# Opening the output file
output = open(MATRIX_FILE, 'wb')
for row, i in enumerate(reader):
# Parsing the two points
points = [
point.Point(float(i[3]), float(i[2])),
point.Point(float(i[6]), float(i[5]))
]
points_id = []
# Find the polygon where each point is located
for j in points:
polygon_id = -1
for k in idx.intersection(j.coords[0]):
if j.within(shape(polygons[k]['geometry'])):
polygon_id = polygons[k]['properties']['ID']
break
points_id.append(polygon_id)
if -1 not in points_id: # Ignoring travels with errors
try:
time = datetime.strptime(
i[1][:-3],
'%Y-%m-%d %H:%M:%S.%f') # Time format with milliseconds
except ValueError:
time = datetime.strptime(i[1], '%Y-%m-%d %H:%M:%S')
# Sorting the travels by day times (by arrival time)
if time.hour <= 6: # Night
mat[points_id[0] - 1][points_id[1] - 1][6] += 1
elif time.hour <= 7: # 6 AM
mat[points_id[0] - 1][points_id[1] - 1][0] += 1
elif time.hour <= 8: # 7 AM
mat[points_id[0] - 1][points_id[1] - 1][1] += 1
elif time.hour <= 9: # 8 AM
mat[points_id[0] - 1][points_id[1] - 1][2] += 1
elif time.hour <= 15: # Noon
mat[points_id[0] - 1][points_id[1] - 1][3] += 1
elif time.hour <= 19: # Afternoon
mat[points_id[0] - 1][points_id[1] - 1][4] += 1
elif time.hour <= 22: # Evening
mat[points_id[0] - 1][points_id[1] - 1][5] += 1
else: # Night
mat[points_id[0] - 1][points_id[1] - 1][6] += 1
if row % 100000 == 0:
print row
# Writing to output file
print 'Writing to file'
output.write('[' + ',\n'.join([str(i) for i in mat]) + ']')
data.close()
output.close()
import fiona
import sys
from shapely.geometry import shape, mapping
from shapely.ops import cascaded_union
with fiona.drivers():
with fiona.open(sys.argv[1]) as source:
with fiona.open(sys.argv[2]) as clip:
dest = fiona.open(sys.argv[3],
'w',
driver=source.driver,
crs=source.crs,
schema=source.schema)
# convert clip into a list of shapely shapes
clippers = [shape(s['geometry']) for s in clip]
for feature in source:
clipped = []
geometry = shape(feature['geometry'])
for clipper in clippers:
if clipper.intersects(geometry):
clipped.append(clipper.intersection(geometry))
if clipped:
unioned = cascaded_union(clipped)
# Sometimes an intersection can return a point
# or a linestring
if unioned.type == geometry.type:
dest.write({
'geometry': mapping(unioned),
'properties': feature['properties']
})
msas.head()
rw = msas.ix[5]
#start_time = timeit.default_timer()
msa = rw.namelsad
msatag = rw['name']
print('processing {}'.format(msatag))
#####Get the MSA shape
q = """select shape from world
where datasetid='metro/tiger/2014/1/tl_2014_us'
and data['t_namelsad'] like '{}%';
""".format(msa)
z = pd.read_sql(q, engine)
BB = dumps(shape(z['shape'][0]))
############# GEOHASH POLYGONS LEVEL 7REPLACE TIGER geohash_polygons HERE ##################
#### Get geohash polygons level 7 for a given MSA
q = """select shape, geohash_label
... """
geohash_polygons = pd.read_sql(q, engine)
## Make a geodataframe wth the geohash_polygons
geo = [shape(i) for i in geohash_polygons['shape']]
del geohash_polygons['shape']
geohash_polygons.gpd = gpd.GeoDataFrame(geometry=geo,
data=geohash_polygons,
crs={'init': 'epsg:4326'})
def query():
print('opening file...')
curdir = os.path.dirname(__file__)
df = pd.read_csv(
open(os.path.join(curdir, '../input/colleges/MERGED2015_16_PP.csv')))
print('file opened')
schools = df[(df['CONTROL'] == 1) |
(df['CONTROL'] == 2)] # only public/private non profits
schools = schools[schools['MAIN'] == 1] # only main campuses
schools = schools[
schools['PREDDEG'] ==
3] # only look at predominately bachelors serving institutions
schools = schools[schools['DISTANCEONLY'] == 0] # Only in-person colleges
schools = schools[schools['CURROPER'] == 1] # Only currently operating
# Dropping NaN values for certain columns
schools = schools[(~np.isnan(schools['LATITUDE']))]
schools = schools[(~np.isnan(schools['LONGITUDE']))]
#Personal Preference
schools = schools[schools['UGDS'] > 1800] # reduce # of schools
cols = [
'UNITID', # id
'INSTNM', # name
'CITY', # city
'STABBR', # state
'ZIP', # zip
'INSTURL', # website
'UGDS', # size
'LATITUDE',
'LONGITUDE',
'ADM_RATE', # admit rate
'HIGHDEG', #highest degree offered
'CONTROL', # public/private
'CCBASIC', # Carnegie Classification -- basic
'CCUGPROF', # Carnegie Classification -- undergraduate profile
'CCSIZSET', # Carnegie Classification -- size and setting
'TRIBAL', # Flag for tribal college and university
'NANTI', #Flag for Native American non-tribal institution
'UGDS_AIAN', # Total share of enrollment of undergraduate degree-seeking students who are American Indian/Alaska Native
'UG_AIANOLD', #Total share of enrollment of undergraduate students who are Asian/Pacific Islander
]
schools = schools[cols]
schools = schools.sort_values(['UGDS'])
curdir = os.path.dirname(__file__)
gj = json.load(
open(
os.path.join(
curdir, '../input/nativelands/indigenousTerritories.geojson')))
territory_series = pd.Series(index=schools.index)
territories = gj.get('features')
territories_polygons = []
for i, t in enumerate(territories):
polygon = shape(t.get('geometry'))
territories_polygons.append({
'polygon': polygon,
'properties': t.get('properties'),
'id': t.get('id'),
'index': i,
})
t['properties']['colleges'] = []
with tqdm(total=len(schools)) as pbar:
for i, s in schools.iterrows():
p = Point(s['LONGITUDE'], s['LATITUDE'])
territories_match = []
for t_i, t in enumerate(territories_polygons):
if t['polygon'].contains(p):
territories_match.append({
'id':
t.get('id'),
'name':
t.get('properties').get('Name')
})
gj.get('features')[t_i]['properties']['colleges'].append(
s['UNITID'])
territory_series[s.name] = json.dumps(territories_match)
pbar.update(1)
schools['territories'] = territory_series
#schools.to_csv('schools.csv')
schools.to_csv('../display/public/schools.csv')
json.dump(gj, open('../display/public/territories.geojson', 'w+'))
