def geoframe(self, simplify=None, predicate=None, crs=None, epsg=None):
"""
Return geopandas dataframe
:param simplify: Integer or None. Simplify the geometry to a tolerance, in the units of the geometry.
:param predicate: A single-argument function to select which records to include in the output.
:param crs: Coordinate reference system information
:param epsg: Specifiy the CRS as an EPGS number.
:return: A Geopandas GeoDataFrame
"""
import geopandas
from shapely.wkt import loads
from fiona.crs import from_epsg
if crs is None and epsg is None and self.epsg is not None:
epsg = self.epsg
if crs is None:
try:
crs = from_epsg(epsg)
except TypeError:
raise TypeError('Must set either crs or epsg for output.')
df = self.dataframe(predicate=predicate)
geometry = df['geometry']
if simplify:
s = geometry.apply(lambda x: loads(x).simplify(simplify))
else:
s = geometry.apply(lambda x: loads(x))
df['geometry'] = geopandas.GeoSeries(s)
return geopandas.GeoDataFrame(df, crs=crs, geometry='geometry')
def test_linestring_multilinestring_result(self):
geom = 'POLYGON((0 0, 10 0, 10 10, 0 10, 0 0))'
limit_to = LimitPolygonGeometry(loads(geom))
geom = limit_to.intersection(loads('LINESTRING(-10 -20, 5 10, 20 -20)'))
assert isinstance(geom, list)
assert geom[0].almost_equals(loads('LINESTRING(0 0, 5 10)'))
assert geom[1].almost_equals(loads('LINESTRING(5 10, 10 0)'))
def geoframe(self, sql, simplify=None, crs=None, epsg=4326):
"""
Return geopandas dataframe
:param simplify: Integer or None. Simplify the geometry to a tolerance, in the units of the geometry.
:param crs: Coordinate reference system information
:param epsg: Specifiy the CRS as an EPGS number.
:return: A Geopandas GeoDataFrame
"""
import geopandas
from shapely.wkt import loads
from fiona.crs import from_epsg
if crs is None:
try:
crs = from_epsg(epsg)
except TypeError:
raise TypeError('Must set either crs or epsg for output.')
df = self.dataframe(sql)
geometry = df['geometry']
if simplify:
s = geometry.apply(lambda x: loads(x).simplify(simplify))
else:
s = geometry.apply(lambda x: loads(x))
df['geometry'] = geopandas.GeoSeries(s)
return geopandas.GeoDataFrame(df, crs=crs, geometry='geometry')
def clean(self,value):
## check the geometry is valid
try:
## try to load the form input from WKT
geom = shapely_wkt.loads(value)
## convert to the url format
ret = reverse_wkt(value)
## try to find the AOI code
except ReadingError:
try:
## return the meta code
user_meta = models.UserGeometryMetadata.objects.filter(code=value)
## confirm it exists in the database
if len(user_meta) == 0: raise(ValidationError)
## convert to shapely geometries. first, return the actual
## geometries
geom = models.UserGeometryData.objects.filter(user_meta=user_meta)
geom = [shapely_wkt.loads(geom.geom.wkt) for geom in geom]
## convert to format acceptable for the url
ret = value
except ValidationError:
raise(ValidationError('Unable to parse WKT or locate unique geometry code.'))
## check that spatial operations will return data
ogeom = shapely_wkt.loads(dataset.spatial_extent.wkt)
igeom = reduce_to_multipolygon(geom)
if not keep(igeom=ogeom,target=igeom):
raise(ValidationError('Input geometry will return an empty intersection.'))
return(ret)
def test_linestring_contained(self):
geom = 'POLYGON((0 0, 10 0, 10 10, 0 10, 0 0))'
limit_to = LimitPolygonGeometry(loads(geom))
test_geom = loads('LINESTRING(1 1, 9 9)')
geom = limit_to.intersection(test_geom)
# should return unmodified input geometry
assert geom is test_geom
def get_extent_by_resource_id(resource_id):
session = DBSession()
resource = session.query(Resource).filter(Resource.id == resource_id).first()
extent = None
for res in resource.children:
if res.identity != VectorLayer.identity or (res.keyname and res.keyname.startswith('real_')):
continue
table_info = TableInfo.from_layer(res)
table_info.setup_metadata(tablename=res._tablename)
columns = [db.func.st_astext(db.func.st_extent(db.text('geom')).label('box'))]
query = sql.select(columns=columns, from_obj=table_info.table)
extent_str = session.connection().scalar(query)
if extent_str:
if not extent:
extent = loads(extent_str).bounds
else:
new_extent = loads(extent_str).bounds
extent = extent_union(extent, new_extent)
session.close()
return extent_buff(extent, 2000)
def testStillSimple (self):
# 147639834
way= wkt.loads ('''
POLYGON ((662012.67 5329425.33,
662013.5600000001 5329427.77,
662020.5699999999 5329417.4,
662024.14 5329413.74,
662032.37 5329406.88,
662034.6 5329405.05,
662032.71 5329403.53,
662027.59 5329407.64,
662019.6800000001 5329414.2,
662016.23 5329419.54,
662012.67 5329425.33))''')
skel= wkt.loads ('''
MULTILINESTRING ((662012.67 5329425.33, 662013.8453861615 5329425.448829351),
(662016.23 5329419.54, 662017.2442048641 5329420.179304471),
(662019.6800000001 5329414.2, 662020.7605526489 5329415.159279451),
(662027.59 5329407.64, 662028.3729275202 5329408.599534022),
(662032.71 5329403.53, 662032.7093037122 5329405.059336644),
(662034.6 5329405.05, 662032.7093037122 5329405.059336644),
(662032.37 5329406.88, 662031.6052956476 5329405.955395874),
(662024.14 5329413.74, 662023.2924827472 5329412.823571924),
(662020.5699999999 5329417.4, 662019.6022560901 5329416.611499771),
(662013.5600000001 5329427.77, 662013.8453861615 5329425.448829351),
(662013.8453861615 5329425.448829351, 662017.2442048641 5329420.179304471),
(662032.7093037122 5329405.059336644, 662031.6052956476 5329405.955395874),
(662017.2442048641 5329420.179304471, 662019.6022560901 5329416.611499771),
(662031.6052956476 5329405.955395874, 662028.3729275202 5329408.599534022),
(662028.3729275202 5329408.599534022, 662023.2924827472 5329412.823571924),
(662019.6022560901 5329416.611499771, 662020.7605526489 5329415.159279451),
(662023.2924827472 5329412.823571924, 662020.7605526489 5329415.159279451))''')
medials= wkt.loads ('''
MULTILINESTRING ((662032.7093037122 5329405.059336644,
662031.6052956476 5329405.955395874,
662028.3729275202 5329408.599534022,
662023.2924827472 5329412.823571924,
662020.7605526489 5329415.159279451,
662019.6022560901 5329416.611499771,
662017.2442048641 5329420.179304471,
662013.8453861615 5329425.448829351))''')
result= centerlines.extend_medials (way, skel, medials)
# honestly, painstakingly calculated from the shape's coords!
expected= wkt.loads ('''
MULTILINESTRING ((662033.655 5329404.29,
662032.7093037122 5329405.059336644,
662031.6052956476 5329405.955395874,
662028.3729275202 5329408.599534022,
662023.2924827472 5329412.823571924,
662020.7605526489 5329415.159279451,
662019.6022560901 5329416.611499771,
662017.2442048641 5329420.179304471,
662013.8453861615 5329425.448829351,
662013.115 5329426.55))''')
self.assertEqual (result, expected, '\n%s\n%s' % (result, expected))
def vector2tiles(cls, vector, pcov=0.0, ptile=0.0, tilelist=None):
""" Return matching tiles and coverage % for provided vector """
import osr
# set spatial filter on tiles vector to speedup
ogrgeom = ogr.CreateGeometryFromWkt(vector.WKT())
tvector = cls.vector()
tlayer = tvector.layer
vsrs = osr.SpatialReference(vector.Projection())
trans = osr.CoordinateTransformation(vsrs, tlayer.GetSpatialRef())
ogrgeom.Transform(trans)
tlayer.SetSpatialFilter(ogrgeom)
# geometry of desired site (transformed)
geom = loads(ogrgeom.ExportToWkt())
tiles = {}
# step through tiles vector
tlayer.ResetReading()
feat = tlayer.GetNextFeature()
while feat is not None:
tgeom = loads(feat.GetGeometryRef().ExportToWkt())
area = geom.intersection(tgeom).area
if area != 0:
tile = cls.feature2tile(feat)
tiles[tile] = (area / geom.area, area / tgeom.area)
feat = tlayer.GetNextFeature()
remove_tiles = []
if tilelist is None:
tilelist = tiles.keys()
for t in tiles:
if (tiles[t][0] < (pcov / 100.0)) or (tiles[t][1] < (ptile / 100.0)) or t not in tilelist:
remove_tiles.append(t)
for t in remove_tiles:
tiles.pop(t, None)
return tiles
def write_extent(bbox, nsmap):
''' Generate BBOX extent '''
from shapely.wkt import loads
if bbox is not None:
extent = etree.Element(util.nspath_eval('gmd:extent', nsmap))
ex_extent = etree.SubElement(extent, util.nspath_eval('gmd:EX_Extent', nsmap))
ge = etree.SubElement(ex_extent, util.nspath_eval('gmd:geographicElement', nsmap))
gbb = etree.SubElement(ge, util.nspath_eval('gmd:EX_GeographicBoundingBox', nsmap))
west = etree.SubElement(gbb, util.nspath_eval('gmd:westBoundLongitude', nsmap))
east = etree.SubElement(gbb, util.nspath_eval('gmd:eastBoundLongitude', nsmap))
south = etree.SubElement(gbb, util.nspath_eval('gmd:southBoundLatitude', nsmap))
north = etree.SubElement(gbb, util.nspath_eval('gmd:northBoundLatitude', nsmap))
if bbox.find('SRID') != -1: # it's EWKT; chop off 'SRID=\d+;'
bbox2 = loads(bbox.split(';')[-1]).envelope.bounds
else:
bbox2 = loads(bbox).envelope.bounds
etree.SubElement(west, util.nspath_eval('gco:Decimal', nsmap)).text = str(bbox2[0])
etree.SubElement(south, util.nspath_eval('gco:Decimal', nsmap)).text = str(bbox2[1])
etree.SubElement(east, util.nspath_eval('gco:Decimal', nsmap)).text = str(bbox2[2])
etree.SubElement(north, util.nspath_eval('gco:Decimal', nsmap)).text = str(bbox2[3])
return extent
return None
def testRectangle (self):
# 147639890
way= wkt.loads ('''
POLYGON ((661994.3 5329434.18, 661995.1899999999 5329436.47,
662006.21 5329433.42, 662005.66 5329431.28,
661994.3 5329434.18))''')
skel= wkt.loads ('''
MULTILINESTRING ((661994.3 5329434.18, 661995.9207244834 5329435.013669997),
(662005.66 5329431.28, 662004.8565612 5329432.636764912),
(662006.21 5329433.42, 662004.8565612 5329432.636764912),
(661995.1899999999 5329436.47, 661995.9207244834 5329435.013669997),
(662004.8565612 5329432.636764912, 661995.9207244834 5329435.013669997))''')
medials= wkt.loads ('''
MULTILINESTRING ((662004.8565612 5329432.636764912,
661995.9207244834 5329435.013669997))''')
result= centerlines.extend_medials (way, skel, medials)
# honestly, painstakingly calculated from the shape's coords!
expected= wkt.loads ('''
MULTILINESTRING ((662005.935 5329432.35,
662004.8565612 5329432.636764912,
661995.9207244834 5329435.013669997,
661994.745 5329435.324999999))''')
self.assertEqual (result, expected, '\n%s\n%s' % (result, expected))
def main(opts):
pattern = loads(open(opts.input, "r").read())
extent = loads(open(opts.extent, "r").read())
if not contains.matches(extent.relate(pattern)):
print "ERROR: pattern must be contained within the extent"
return
c = pattern.centroid
(xs, ys) = extent.boundary.xy
(minx, maxx, miny, maxy) = (min(xs) - c.x, max(xs) - c.x, min(ys) - c.y, max(ys) - c.y)
outputFile = open(opts.output, "w")
geoms = []
while len(geoms) < opts.number:
dx = random.uniform(minx, maxx)
dy = random.uniform(miny, maxy)
geom = translate(pattern, xoff=dx, yoff=dy)
if contains.matches(extent.relate(geom)):
# Check that it is within the extent
overlap = False
for g in geoms:
if intersects.matches(g.relate(geom)):
overlap = True
if overlap == False:
geoms.append(geom)
for geom in geoms:
outputFile.write(dumps(geom) + "\n")
outputFile.close()
def test_add_datasets_as_list(self):
item = self.item
geoms = []
# create dataset 1 & 2 together
vals1 = [(datetime.datetime(2014,1,1,12,0,0) + datetime.timedelta(days=i), i) for i in range(0,100)]
dv1 = DataValues(vals1)
geometry1 = 'POLYGON ((30 10, 40 40, 20 40, 10 20, 30 10))'
geom = Geometry()
geom.set_geom_from_wkt(geometry1)
geom.type(ElementType.Polygon)
geom.srs(utils.get_srs_from_epsg(self.srscode))
geom.datavalues(dv1)
geoms.append(geom)
vals2 = [(datetime.datetime(2014,1,1,12,0,0) + datetime.timedelta(days=i), i) for i in range(0,100)]
dv2 = DataValues(vals2)
geometry2 = 'POLYGON ((40 20, 50 50, 30 50, 20 30, 40 20))'
geom = Geometry()
geom.set_geom_from_wkt(geometry2)
geom.type(ElementType.Polygon)
geom.srs(utils.get_srs_from_epsg(self.srscode))
geom.datavalues(dv2)
geoms.append(geom)
# add both datasets
item.add_geometry(geoms)
datasets = item.get_all_datasets()
self.assertTrue(len(datasets.keys()) == 2)
for g,ts in datasets.iteritems():
if g.geom().almost_equals(loads(geometry1),5):
self.assertTrue(g.datavalues() == dv1)
elif g.geom().almost_equals(loads(geometry2),5):
self.assertTrue(g.datavalues() == dv2)
def get_spatial_overlay_rank(target_geometry, query_geometry):
"""Derive spatial overlay rank for geospatial search as per Lanfear (2006)
http://pubs.usgs.gov/of/2006/1279/2006-1279.pdf"""
from shapely.geometry.base import BaseGeometry
#TODO: Add those parameters to config file
kt = 1.0
kq = 1.0
if target_geometry is not None and query_geometry is not None:
try:
q_geom = loads(query_geometry)
t_geom = loads(target_geometry)
Q = q_geom.area
T = t_geom.area
if any(item == 0.0 for item in [Q, T]):
LOGGER.warn('Geometry has no area')
return '0'
X = t_geom.intersection(q_geom).area
if kt == 1.0 and kq == 1.0:
LOGGER.debug('Spatial Rank: %s', str((X/Q)*(X/T)))
return str((X/Q)*(X/T))
else:
LOGGER.debug('Spatial Rank: %s', str(((X/Q)**kq)*((X/T)**kt)))
return str(((X/Q)**kq)*((X/T)**kt))
except Exception as err:
LOGGER.warn('Cannot derive spatial overlay ranking %s', err)
return '0'
return '0'
def test_spatial_geometry():
poly = Polygon(((1, 2), (1, 3), (2, 3), (2, 2), (1, 2)))
# with epsg undefined
filter = spatial.Spatial(
spatial.Spatial.GEOMETRY,
MappedClass.geometry_column(),
geometry=dumps(poly),
tolerance=1
)
filter = filter.to_sql_expr()
params = filter.compile().params
assert str(filter) == '(expand(geomfromtext(:geomfromtext_1, :geomfromtext_2), :expand_1) && "table".geom) AND distance("table".geom, geomfromtext(:geomfromtext_1, :geomfromtext_2)) <= :distance_1'
assert wkt.loads(params["geomfromtext_1"]).equals(poly)
assert params["geomfromtext_2"] == 4326
assert params["expand_1"] == 1
assert params["distance_1"] == 1
# with epsg defined
filter = spatial.Spatial(
spatial.Spatial.GEOMETRY,
MappedClass.geometry_column(),
geometry=dumps(poly),
tolerance=1,
epsg=900913
)
filter = filter.to_sql_expr()
params = filter.compile().params
assert str(filter) == '(expand(geomfromtext(:geomfromtext_1, :geomfromtext_2), :expand_1) && transform("table".geom, :transform_1)) AND distance(transform("table".geom, :transform_1), geomfromtext(:geomfromtext_1, :geomfromtext_2)) <= :distance_1'
assert wkt.loads(params["geomfromtext_1"]).equals(poly)
assert params["geomfromtext_2"] == 900913
assert params["expand_1"] == 1
assert params["transform_1"] == 900913
assert params["distance_1"] == 1
def assign_ref_shapes(self):
# dictionary that containes pairs rank: Polygon
_shapes = {}
# extract API region
where_clause = "FIPS_API = '%s'" % (self.get_5d_api())
table = 'GISCoreData.dbo.API_Regions_WM'
polygon_WKT = queryWKT(table, where_clause)
if polygon_WKT:
poly = loads(polygon_WKT)
_shapes['Api_region'] = poly
#exctract section
where_clause = "StateCode LIKE '%s' AND TWN LIKE '%s' AND TWNDIR LIKE '%s' AND RNG LIKE '%s' AND RNGDIR LIKE '%s'AND SECTION LIKE '%s'" % (self.state_code, self.twnshp, self.twnshp_dir, self.range_, self.range_dir, self.section)
table = 'GISCoreData.dbo.PLSS_SEC_%i' % self.mcode
polygon_WKT = queryWKT(table, where_clause)
if polygon_WKT:
poly = loads(polygon_WKT)
_shapes['Section'] = poly
# qqsection if exists
if self.qqsection:
where_clause = "TWN LIKE '%s' AND TWNDIR LIKE '%s' AND RNG LIKE '%s' AND RNGDIR LIKE '%s'AND SECTION LIKE '%s' AND qqsection like '%s%s'" % (self.twnshp, self.twnshp_dir, self.range_, self.range_dir, self.section, self.qsection, self.qqsection)
table = 'GISCoreData.dbo.PLSS_QQ_%i' % self.mcode
polygon_WKT = queryWKT(table, where_clause)
if polygon_WKT:
poly = loads(polygon_WKT)
_shapes['qqSection'] = poly
#TODO: add more conditions
self.reference_shapes = _shapes
def test_box_filter(self):
from mapfish.protocol import create_geom_filter
request = FakeRequest(
{"bbox": "-180,-90,180,90", "tolerance": "1"}
)
filter = create_geom_filter(request, MappedClass)
compiled_filter = filter.compile(engine)
params = compiled_filter.params
filter_str = _compiled_to_string(compiled_filter)
eq_(filter_str, '(ST_Expand(GeomFromWKB(%(GeomFromWKB_1)s, %(GeomFromWKB_2)s), %(ST_Expand_1)s) && "table".geom) AND (ST_Expand("table".geom, %(ST_Expand_2)s) && GeomFromWKB(%(GeomFromWKB_3)s, %(GeomFromWKB_4)s)) AND ST_Distance("table".geom, GeomFromWKB(%(GeomFromWKB_5)s, %(GeomFromWKB_6)s)) <= %(ST_Distance_1)s')
assert wkb.loads(str(params["GeomFromWKB_1"])).equals(wkt.loads('POLYGON ((-180 -90, -180 90, 180 90, 180 -90, -180 -90))'))
assert params["GeomFromWKB_2"] == 4326
assert params["ST_Expand_1"] == 1
assert params["ST_Distance_1"] == 1
request = FakeRequest(
{"bbox": "-180,-90,180,90", "tolerance": "1", "epsg": "900913"}
)
filter = create_geom_filter(request, MappedClass)
compiled_filter = filter.compile(engine)
params = compiled_filter.params
filter_str = _compiled_to_string(compiled_filter)
eq_(filter_str, '(ST_Expand(GeomFromWKB(%(GeomFromWKB_1)s, %(GeomFromWKB_2)s), %(ST_Expand_1)s) && ST_Transform("table".geom, %(param_1)s)) AND (ST_Expand(ST_Transform("table".geom, %(param_2)s), %(ST_Expand_2)s) && GeomFromWKB(%(GeomFromWKB_3)s, %(GeomFromWKB_4)s)) AND ST_Distance(ST_Transform("table".geom, %(param_3)s), GeomFromWKB(%(GeomFromWKB_5)s, %(GeomFromWKB_6)s)) <= %(ST_Distance_1)s')
assert wkb.loads(str(params["GeomFromWKB_1"])).equals(wkt.loads('POLYGON ((-180 -90, -180 90, 180 90, 180 -90, -180 -90))'))
assert params["GeomFromWKB_2"] == 900913
assert params["ST_Expand_1"] == 1
assert params["param_1"] == 900913
assert params["ST_Distance_1"] == 1
def plot_view(result):
# Determine bounding box if no clipping boundary was supplied
if not result['bbox']:
result['bbox'] = bbox_of_view(result)
ax = plt.subplot(111)
# plt.box(on=None)
m = Basemap(resolution='i',
projection='merc',
llcrnrlat=result['bbox']['ymin'],
urcrnrlat=result['bbox']['ymax'],
llcrnrlon=result['bbox']['xmin'],
urcrnrlon=result['bbox']['xmax'],
lat_ts=(result['bbox']['xmin'] +
result['bbox']['xmax']) / 2)
m.drawcoastlines()
try:
for el in result['results']:
vectors = get_vectors_from_postgis_map(m, loads(el['geom']))
lines = LineCollection(vectors, antialiaseds=(1, ))
lines.set_facecolors('black')
lines.set_edgecolors('white')
lines.set_linewidth(1)
ax.add_collection(lines)
m.fillcontinents(color='coral', lake_color='aqua')
# If AttributeError assume geom_type 'Point', simply collect all
# points and perform scatterplot
except AttributeError:
xy = m([loads(point['geom']).x for point in result['results']],
[loads(point['geom']).y for point in result['results']])
plt.scatter(xy[0], xy[1])
plt.show()
def test_spatial_within():
# with epsg undefined
filter = spatial.Spatial(
spatial.Spatial.WITHIN,
MappedClass.geometry_column(),
lon=40, lat=5, tolerance=1
)
filter = filter.to_sql_expr()
params = filter.compile().params
assert str(filter) == '(expand(geomfromtext(:geomfromtext_1, :geomfromtext_2), :expand_1) && "table".geom) AND distance("table".geom, geomfromtext(:geomfromtext_1, :geomfromtext_2)) <= :distance_1'
assert wkt.loads(params["geomfromtext_1"]).equals(wkt.loads('POINT (40 5)'))
assert params["geomfromtext_2"] == 4326
assert params["expand_1"] == 1
assert params["distance_1"] == 1
# with epsg defined
filter = spatial.Spatial(
spatial.Spatial.WITHIN,
MappedClass.geometry_column(),
lon=40, lat=5, tolerance=1, epsg=900913
)
filter = filter.to_sql_expr()
params = filter.compile().params
assert str(filter) == '(expand(geomfromtext(:geomfromtext_1, :geomfromtext_2), :expand_1) && transform("table".geom, :transform_1)) AND distance(transform("table".geom, :transform_1), geomfromtext(:geomfromtext_1, :geomfromtext_2)) <= :distance_1'
assert wkt.loads(params["geomfromtext_1"]).equals(wkt.loads('POINT (40 5)'))
assert params["geomfromtext_2"] == 900913
assert params["expand_1"] == 1
assert params["transform_1"] == 900913
assert params["distance_1"] == 1
def test_spatial_box():
# with epsg undefined
filter = spatial.Spatial(
spatial.Spatial.BOX,
MappedClass.geometry_column(),
box=[-180, -90, 180, 90],
tolerance=1
)
filter = filter.to_sql_expr()
params = filter.compile().params
assert str(filter) == '(expand(geomfromtext(:geomfromtext_1, :geomfromtext_2), :expand_1) && "table".geom) AND distance("table".geom, geomfromtext(:geomfromtext_1, :geomfromtext_2)) <= :distance_1'
assert wkt.loads(params["geomfromtext_1"]).equals(wkt.loads('POLYGON ((-180 -90, -180 90, 180 90, 180 -90, -180 -90))'))
assert params["geomfromtext_2"] == 4326
assert params["expand_1"] == 1
assert params["distance_1"] == 1
# with epsg defined
filter = spatial.Spatial(
spatial.Spatial.BOX,
MappedClass.geometry_column(),
box=[-180, -90, 180, 90],
tolerance=1,
epsg=900913
)
filter = filter.to_sql_expr()
params = filter.compile().params
assert str(filter) == '(expand(geomfromtext(:geomfromtext_1, :geomfromtext_2), :expand_1) && transform("table".geom, :transform_1)) AND distance(transform("table".geom, :transform_1), geomfromtext(:geomfromtext_1, :geomfromtext_2)) <= :distance_1'
assert wkt.loads(params["geomfromtext_1"]).equals(wkt.loads('POLYGON ((-180 -90, -180 90, 180 90, 180 -90, -180 -90))'))
assert params["geomfromtext_2"] == 900913
assert params["expand_1"] == 1
assert params["transform_1"] == 900913
assert params["distance_1"] == 1
def value_to_shape(value):
"""Transforms input into a Shapely object"""
if not value:
return wkt.loads('GEOMETRYCOLLECTION EMPTY')
if isinstance(value, basestring):
# We try to do this before parsing json exception
# exception are ressource costly
if '{' in value:
geo_dict = geojson.loads(value)
shape_to_return = asShape(geo_dict)
elif value:
# if value is empty sting we return False to be orm coherent,
#may be we should return an empty shapely
shape_to_return = wkt.loads(value)
else:
return False
elif hasattr(value, 'wkt'):
#Nasty but did not find equivalent of base string for shapely
if 'shapely.geometry' in str(type(value)):
shape_to_return = value
else:
shape_to_return = wkt.loads(value.wkt)
else:
raise TypeError('Write/create/search geo type must be wkt/geojson '
'string or must respond to wkt')
return shape_to_return
def _filter_overlap(self, scenes, wkt_geometry, min_overlap=0):
"""Filter scenes based on the minimum overlap to the area of interest
Args:
scenes: List of scenes to filter
wkt_geometry: Wkt Geometry representation of the area of interest
min_overlap: Minimum overlap (0-1) in decimal format between scene geometry and area of interest
Returns:
Filtered list of scenes
"""
site = loads(wkt_geometry)
filtered = []
for scene in scenes:
footprint = loads(scene['footprint'])
intersect = site.intersection(footprint)
overlap = intersect.area / site.area
# print str(overlap)
if overlap > min_overlap or (
site.area / footprint.area > 1 and intersect.area / footprint.area > min_overlap):
scene['_script_overlap'] = overlap * 100
filtered.append(scene)
return filtered
def test_serialize(self):
parser = nrml_parsers.SourceModelParser(MIXED_SRC_MODEL)
source_model = parser.parse()
inp = models.Input(
owner=helpers.default_user(),
digest='fake',
path='fake',
input_type='source',
size=0
)
inp.save()
db_writer = source_input.SourceDBWriter(
inp, source_model, MESH_SPACING, BIN_WIDTH, AREA_SRC_DISC
)
db_writer.serialize()
# Check that everything was saved properly.
# First, check the Input:
# refresh the record
[inp] = models.Input.objects.filter(id=inp.id)
self.assertEquals(source_model.name, inp.name)
# re-reparse the test file for comparisons:
nrml_sources = list(
nrml_parsers.SourceModelParser(MIXED_SRC_MODEL).parse()
)
parsed_sources = list(models.ParsedSource.objects.filter(input=inp.id))
# compare pristine nrml sources to those stored in pickled form in the
# database (by unpickling them first, of course):
for i, ns in enumerate(nrml_sources):
self.assertTrue(*helpers.deep_eq(ns, parsed_sources[i].nrml))
# now check that the ParsedSource geometry is correct
# it should be the same as the 'rupture-enclosing' geometry for the
# nhlib representation of each source
for i, (ns, ps) in enumerate(zip(nrml_sources, parsed_sources)):
nhlib_src = source_input.nrml_to_nhlib(
ns, MESH_SPACING, BIN_WIDTH, AREA_SRC_DISC
)
nhlib_poly = nhlib_src.get_rupture_enclosing_polygon()
# nhlib tests the generation of wkt from a polygon, so we can trust
# that it is well-formed.
# Since we save the rupture enclosing polygon as geometry (not wkt)
# in the database, the WKT we get back from the DB might have
# slightly different coordinate values (a difference in precision).
# shapely can help us compare two polygons (generated from wkt)
# at a specific level of precision (default=6 digits after the
# decimal point).
expected_poly = wkt.loads(ps.polygon.wkt)
actual_poly = wkt.loads(nhlib_poly.wkt)
self.assertTrue(expected_poly.almost_equals(actual_poly))
def test_operations(self):
point = Point(0.0, 0.0)
# General geometry
self.assertEqual(point.area, 0.0)
self.assertEqual(point.length, 0.0)
self.assertAlmostEqual(point.distance(Point(-1.0, -1.0)),
1.4142135623730951)
# Topology operations
# Envelope
self.assertIsInstance(point.envelope, Point)
# Intersection
self.assertIsInstance(point.intersection(Point(-1, -1)),
GeometryCollection)
# Buffer
self.assertIsInstance(point.buffer(10.0), Polygon)
self.assertIsInstance(point.buffer(10.0, 32), Polygon)
# Simplify
p = loads('POLYGON ((120 120, 121 121, 122 122, 220 120, 180 199, '
'160 200, 140 199, 120 120))')
expected = loads('POLYGON ((120 120, 140 199, 160 200, 180 199, '
'220 120, 120 120))')
s = p.simplify(10.0, preserve_topology=False)
self.assertTrue(s.equals_exact(expected, 0.001))
p = loads('POLYGON ((80 200, 240 200, 240 60, 80 60, 80 200),'
'(120 120, 220 120, 180 199, 160 200, 140 199, 120 120))')
expected = loads(
'POLYGON ((80 200, 240 200, 240 60, 80 60, 80 200),'
'(120 120, 220 120, 180 199, 160 200, 140 199, 120 120))')
s = p.simplify(10.0, preserve_topology=True)
self.assertTrue(s.equals_exact(expected, 0.001))
# Convex Hull
self.assertIsInstance(point.convex_hull, Point)
# Differences
self.assertIsInstance(point.difference(Point(-1, 1)), Point)
self.assertIsInstance(point.symmetric_difference(Point(-1, 1)),
MultiPoint)
# Boundary
self.assertIsInstance(point.boundary, GeometryCollection)
# Union
self.assertIsInstance(point.union(Point(-1, 1)), MultiPoint)
self.assertIsInstance(point.representative_point(), Point)
self.assertIsInstance(point.centroid, Point)
# Relate
self.assertEqual(point.relate(Point(-1, -1)), 'FF0FFF0F2')
def query_spatial(bbox_data_wkt, bbox_input_wkt, predicate, distance):
"""Perform spatial query
Parameters
----------
bbox_data_wkt: str
Well-Known Text representation of the data being queried
bbox_input_wkt: str
Well-Known Text representation of the input being queried
predicate: str
Spatial predicate to use in query
distance: int or float or str
Distance parameter for when using either of ``beyond`` or ``dwithin``
predicates.
Returns
-------
str
Either ``true`` or ``false`` depending on the result of the spatial
query
Raises
------
RuntimeError
If an invalid predicate is used
"""
try:
bbox1 = loads(bbox_data_wkt.split(';')[-1])
bbox2 = loads(bbox_input_wkt)
if predicate == 'bbox':
result = bbox1.intersects(bbox2)
elif predicate == 'beyond':
result = bbox1.distance(bbox2) > float(distance)
elif predicate == 'contains':
result = bbox1.contains(bbox2)
elif predicate == 'crosses':
result = bbox1.crosses(bbox2)
elif predicate == 'disjoint':
result = bbox1.disjoint(bbox2)
elif predicate == 'dwithin':
result = bbox1.distance(bbox2) <= float(distance)
elif predicate == 'equals':
result = bbox1.equals(bbox2)
elif predicate == 'intersects':
result = bbox1.intersects(bbox2)
elif predicate == 'overlaps':
result = bbox1.intersects(bbox2) and not bbox1.touches(bbox2)
elif predicate == 'touches':
result = bbox1.touches(bbox2)
elif predicate == 'within':
result = bbox1.within(bbox2)
else:
raise RuntimeError(
'Invalid spatial query predicate: %s' % predicate)
except (AttributeError, ValueError, ReadingError):
result = False
return "true" if result else "false"
def test_linestring_mixed_result(self):
geom = 'POLYGON((0 0, 10 0, 10 10, 0 10, 0 0))'
limit_to = LimitPolygonGeometry(loads(geom))
geom = limit_to.intersection(loads('LINESTRING(0 0, 5 -10, 5 10)'))
# point and linestring, point not returned
assert isinstance(geom, list)
assert len(geom) == 1
assert geom[0].almost_equals(loads('LINESTRING(5 0, 5 10)'))
def get_route_data(self, start_coords, end_coords):
"""Executes function for getting pgrouting ways vertices from provided
coordinates, executes function for getting route with pgrouting.
Converts route to shapely and ogr geometry, creates buffer around the
route and executes function for calculating numerical attribute data
like driving time and length.
:arg start_coords: dictionary with route starting location coordinates,
e.g. {"x": 15.5, "y": 45.5}
:type start_coords: dictionary
:arg end_coords: dictionary with route ending location coordinates,
e.g. {"x": 15.5, "y": 45.5}
:type end_coords: dictionary
:returns: dictionary with pgrouting route data
:rtype: dictionary
"""
start_vertex_id, end_vertex_id = self.get_way_vertices_from_coords(
start_coords=start_coords,
end_coords=end_coords
)
raw_route, colnames = self.get_route_from_pgrouting(
start_vertex_id=start_vertex_id,
end_vertex_id=end_vertex_id
)
route_ogr = self.create_multiline_from_linesegments(
raw_route=raw_route,
colnames=colnames
)
route_shapely = shapely_wkt.loads(route_ogr.ExportToWkt())
route_buffer_ogr = UTILITY.create_route_buffer(route=route_ogr)
route_buffer_shapely = shapely_wkt.loads(
route_buffer_ogr.ExportToWkt()
)
driving_time = self.sum_cost(
raw_route=raw_route,
colnames=colnames
)
route_length = self.sum_route_length(
raw_route=raw_route,
colnames=colnames
)
return {
'route_ogr': route_ogr,
'route_shapely': route_shapely,
'route_buffer_ogr': route_buffer_ogr,
'route_buffer_shapely': route_buffer_shapely,
'driving_time': driving_time,
'len': route_length,
}
def test_polygon_multipolygon_result(self):
geom = 'POLYGON((0 0, 10 0, 10 10, 0 10, 0 0))'
limit_to = LimitPolygonGeometry(loads(geom))
test_geom = loads('POLYGON((0 -10, 0 5, 2.5 -5, 5 -1, 7.5 -5, 10 5, 10 -10, 0 -10))')
geom = limit_to.intersection(test_geom)
# similar to above, but point does not touch the box, so we should get
# a single multipolygon
assert geom.almost_equals(loads(
'MULTIPOLYGON(((1.25 0, 0 0, 0 5, 1.25 0)),'
'((10 0, 8.75 0, 10 5, 10 0)))'))
def test_polygon_mixed_result(self):
geom = 'POLYGON((0 0, 10 0, 10 10, 0 10, 0 0))'
limit_to = LimitPolygonGeometry(loads(geom))
test_geom = loads('POLYGON((0 -10, 0 5, 2.5 -5, 5 0, 7.5 -5, 10 5, 10 -10, 0 -10))')
geom = limit_to.intersection(test_geom)
# point and two polygons, point not returned
assert isinstance(geom, list)
assert len(geom) == 2
assert geom[0].almost_equals(loads('POLYGON((1.25 0, 0 0, 0 5, 1.25 0))'))
assert geom[1].almost_equals(loads('POLYGON((10 0, 8.75 0, 10 5, 10 0))'))
def get_st_closest_point(self, a, b):
result = self.engine.execute('''SELECT
ST_AsText(ST_ClosestPoint(foo.a, foo.b)) AS a_b,
ST_AsText(ST_ClosestPoint(foo.b, foo.a)) As b_a
FROM (
SELECT '%s'::geometry As a, '%s'::geometry As b
) AS foo;''' % (
a.wkt, b.wkt
)).fetchall()
return wkt.loads(result[0][0]), wkt.loads(result[0][1])
def test_get_field_write_target(self):
p1 = 'Polygon ((-116.94238466549290933 52.12861711455555991, -82.00526805089285176 61.59075286434307372, ' \
'-59.92695130138864101 31.0207758265680269, -107.72286778108455962 22.0438778075388484, ' \
'-122.76523743459291893 37.08624746104720771, -116.94238466549290933 52.12861711455555991))'
p2 = 'Polygon ((-63.08099655131782413 21.31602121140134898, -42.70101185946779765 9.42769680782217279, ' \
'-65.99242293586783603 9.912934538580501, -63.08099655131782413 21.31602121140134898))'
p1 = wkt.loads(p1)
p2 = wkt.loads(p2)
mp1 = MultiPolygon([p1, p2])
mp2 = mp1.buffer(0.1)
geoms = [mp1, mp2]
gvar = GeometryVariable(name='gc', value=geoms, dimensions='elementCount')
gc = gvar.convert_to(node_dim_name='n_node')
field = gc.parent
self.assertEqual(field.grid.node_dim.name, 'n_node')
actual = DriverESMFUnstruct._get_field_write_target_(field)
self.assertEqual(field.grid.node_dim.name, 'n_node')
self.assertNotEqual(id(field), id(actual))
self.assertEqual(actual['numElementConn'].dtype, np.int32)
self.assertEqual(actual['elementConn'].dtype, np.int32)
self.assertNotIn(field.grid.cindex.name, actual)
self.assertEqual(actual['nodeCoords'].dimensions[0].name, 'nodeCount')
path = self.get_temporary_file_path('foo.nc')
actual.write(path)
try:
import ESMF
except ImportError:
pass
else:
_ = ESMF.Mesh(filename=path, filetype=ESMF.FileFormat.ESMFMESH)
path2 = self.get_temporary_file_path('foo2.nc')
driver = DriverKey.NETCDF_ESMF_UNSTRUCT
field.write(path2, driver=driver)
# Test the polygons are equivalent when read from the ESMF unstructured file.
rd = ocgis.RequestDataset(path2, driver=driver)
self.assertEqual(rd.driver.key, driver)
efield = rd.get()
self.assertEqual(efield.driver.key, driver)
grid_actual = efield.grid
self.assertEqual(efield.driver.key, driver)
self.assertEqual(grid_actual.parent.driver.key, driver)
self.assertEqual(grid_actual.x.ndim, 1)
for g in grid_actual.archetype.iter_geometries():
self.assertPolygonSimilar(g[1], geoms[g[0]])
ngv = grid_actual.archetype.convert_to()
self.assertIsInstance(ngv, GeometryVariable)
def test_error_handler_wrong_type():
with pytest.raises(TypeError):
loads(1)
def geom_from_wkt(data, srid=None):
g = wkt.loads(data)
g.__class__ = globals()[g.__class__.__name__]
g._srid = srid
return g
def __init__(self, maz: int, polygon: str):
self.maz = maz
self.polygon = Polygon(loads(polygon))
simpledec = re.compile(r"\d*\.\d+")
def mround(match):
return "{:.0f}".format(float(match.group()))
fids = ['cyano_daymap_20190717.shp']
for fid in fids:
fid = fid.replace('union_bloom', 'daymap')
print(fid)
shapes = gp.read_file(fid)
shapes.geometry = shapes.geometry.apply(
lambda x: loads(re.sub(simpledec, mround, x.wkt)))
shapes.to_file(fid)
# def change_coord(tup):
# return QgsPoint(int(tup[0]), int(tup[1]))
#
#
# def convert_polygon_type(geom):
# new_points = []
# if geom.wkbType() == QgsWKBTypes.Polygon:
# polys = geom.asPolygon()
# for p in polys:
# if isinstance(p, QgsPoint):
# point = change_coord(p)
# new_points.append(point)
# elif isinstance(p, list):
def load_graphml(filename, folder=None):
"""
Load a GraphML file from disk and convert the node/edge attributes to correct data types.
Parameters
----------
filename : string
the name of the graphml file (including file extension)
folder : string
the folder containing the file, if None, use default data folder
Returns
-------
networkx multidigraph
"""
start_time = time.time()
# read the graph from disk
if folder is None:
folder = globals.data_folder
path = '{}/{}'.format(folder, filename)
G = nx.MultiDiGraph(nx.read_graphml(path, node_type=int))
# convert graph crs attribute from saved string to correct dict data type
G.graph['crs'] = ast.literal_eval(G.graph['crs'])
if 'streets_per_node' in G.graph:
G.graph['streets_per_node'] = ast.literal_eval(G.graph['streets_per_node'])
# convert numeric node tags from string to numeric data types
log('Converting node and edge attribute data types')
for node, data in G.nodes(data=True):
data['osmid'] = int(data['osmid'])
data['x'] = float(data['x'])
data['y'] = float(data['y'])
# convert numeric, bool, and list node tags from string to correct data types
for u, v, key, data in G.edges(keys=True, data=True):
# first parse oneway to bool and length to float - they should always have only 1 value each
data['oneway'] = ast.literal_eval(data['oneway'])
data['length'] = float(data['length'])
# these attributes might have a single value, or a list if edge's topology was simplified
for attr in ['highway', 'name', 'bridge', 'tunnel', 'lanes', 'ref', 'maxspeed', 'service', 'access', 'area', 'landuse', 'width', 'est_width']:
# if this edge has this attribute, and it starts with '[' and ends with ']', then it's a list to be parsed
if attr in data and data[attr][0] == '[' and data[attr][-1] == ']':
# convert the string list to a list type, else leave as single-value string
data[attr] = ast.literal_eval(data[attr])
# osmid might have a single value or a list, but if single value, then parse int
if 'osmid' in data:
if data['osmid'][0] == '[' and data['osmid'][-1] == ']':
data['osmid'] = ast.literal_eval(data['osmid'])
else:
data['osmid'] = int(data['osmid'])
# if geometry attribute exists, load the string as well-known text to shapely LineString
if 'geometry' in data:
data['geometry'] = wkt.loads(data['geometry'])
# remove node_default and edge_default metadata keys if they exist
if 'node_default' in G.graph:
del G.graph['node_default']
if 'edge_default' in G.graph:
del G.graph['edge_default']
log('Loaded graph with {:,} nodes and {:,} edges in {:,.2f} seconds from "{}"'.format(len(list(G.nodes())),
len(list(G.edges())),
time.time()-start_time,
path))
return G
def _convert_edge_attr_types(G, node_type):
"""
Convert graph edges' attributes' types from string to numeric.
Parameters
----------
G : networkx.MultiDiGraph
input graph
node_type : type
convert osmid to this type
Returns
-------
G : networkx.MultiDiGraph
"""
# convert numeric, bool, and list edge attributes from string to correct data types
for _, _, data in G.edges(data=True, keys=False):
# parse length to float: should always have only 1 value
data["length"] = float(data["length"])
try:
data["oneway"] = ast.literal_eval(data["oneway"])
except KeyError:
# may lack oneway if settings.all_oneway=True when you created graph
pass
except ValueError:
# may have values it can't eval if settings.all_oneway=True
pass
# parse grade attrs to float: should always have only 1 value each
if "grade" in data:
data["grade"] = float(data["grade"])
if "grade_abs" in data:
data["grade_abs"] = float(data["grade_abs"])
# these attributes might have a single value, or a list if edge's
# topology was simplified
for attr in [
"highway",
"name",
"bridge",
"tunnel",
"lanes",
"ref",
"maxspeed",
"service",
"access",
"area",
"landuse",
"width",
"est_width",
]:
# if this edge has this attribute, and it starts with '[' and ends
# with ']', then it's a list to be parsed
if attr in data and data[attr].startswith("[") and data[attr].endswith("]"):
# try to convert the string list to a list type, else leave as
# single-value string (and leave as string if error)
try:
data[attr] = ast.literal_eval(data[attr])
except Exception:
pass
# osmid might have a single value or a list
if "osmid" in data:
if data["osmid"].startswith("[") and data["osmid"].endswith("]"):
# if it's a list, eval list then convert each element to node_type
data["osmid"] = [node_type(i) for i in ast.literal_eval(data["osmid"])]
else:
# if it's not a list, convert it to the node_type
data["osmid"] = node_type(data["osmid"])
# if geometry attribute exists, load the string as well-known text to
# shapely LineString
if "geometry" in data:
data["geometry"] = wkt.loads(data["geometry"])
return G
def download_quicklook(self, product_uuid, target_dir):
"""Downloads a quicklook of the satellite image to a target directory for a specific product_id.
It performs a very rough geocoding of the quicklooks by shifting the image to the location of the footprint.
:param product_uuid: UUID of the satellite image product (String).
:param target_dir: Target directory that holds the downloaded images (String, Path)
"""
if isinstance(target_dir, str):
target_dir = Path(target_dir)
if self.src == Datahub.STAC_local or self.src == Datahub.STAC_API:
raise NotImplementedError(
f"download_quicklook not supported for {self.src}. It is much easier to get the asset yourself now, "
f"when it is a COG you can read in an overview.")
elif self.src == Datahub.EarthExplorer:
# query EarthExplorer for url, srcid and bounds of product
meta_src = self.api.request(
"metadata",
**{
"datasetName": self.src.value,
"entityIds": [product_uuid],
},
)
url = meta_src[0]["browseUrl"]
bounds = geometry.shape(meta_src[0]["spatialFootprint"]).bounds
product_srcid = meta_src[0]["displayId"]
else:
# query Scihub for url, srcid and bounds of product
meta_src = self.api.get_product_odata(product_uuid)
url = "https://scihub.copernicus.eu/apihub/odata/v1/Products('{}')/Products('Quicklook')/$value".format(
product_uuid)
bounds = wkt.loads(meta_src["footprint"]).bounds
product_srcid = meta_src["title"]
# download quicklook and crop no-data borders
response = requests.get(url, auth=(self.user, self.pw))
quicklook = np.asarray(Image.open(BytesIO(response.content)))
# use threshold of 50 to overcome noise in JPEG compression
xs, ys, zs = np.where(quicklook >= 50)
quicklook = quicklook[min(xs):max(xs) + 1,
min(ys):max(ys) + 1,
min(zs):max(zs) + 1]
Image.fromarray(quicklook).save(
target_dir.joinpath(product_srcid + ".jpg"))
# geocode quicklook
quicklook_size = (quicklook.shape[1], quicklook.shape[0])
dist_x = geometry.Point(bounds[0], bounds[1]).distance(
geometry.Point(bounds[2], bounds[1])) / quicklook_size[0]
dist_y = geometry.Point(bounds[0], bounds[1]).distance(
geometry.Point(bounds[0], bounds[3])) / quicklook_size[1]
ul_x, ul_y = bounds[0], bounds[3]
with open(target_dir.joinpath(product_srcid + ".jpgw"),
"w") as out_file:
out_file.write(str(dist_x) + "\n")
out_file.write(str(0.0) + "\n")
out_file.write(str(0.0) + "\n")
out_file.write(str(-dist_y) + "\n")
out_file.write(str(ul_x) + "\n")
out_file.write(str(ul_y) + "\n")
def from_wkt(cls, value):
# This looks-like-wkt business keeps Shapely from printing a
# warning for non-WKT strings, which it does even though it
# raises an exception.
looks_like_wkt = value.startswith(cls.__name__.upper())
return cls(wkt.loads(value)) if looks_like_wkt else None
def brutusBKoverlapp(mittfilter=None, offisiell=False, kunEnTypeBK=None):
"""
Finner de bruksklasse-objektene som overlapper med bruer.
finner overlapp mellom bruenes vegsegmenter og vegsegmenter for BK-typene "Normaltransport", "Spesialtransport" og "12/100 vegnett"
Bruk evt nøkkelord kunEnTypeBK for å plukke ut en av disse.
Bruk nøkkelord offisiell=True for å hente uoffisielle BK-verdier (krever innlogging)
Brusøket kan snevres inn med nøkkelord mittfilter={}, ref dokumentasjon for spørring etter vegobjekter
https://nvdbapiles-v3.atlas.vegvesen.no/dokumentasjon/openapi/#/Vegobjekter/get_vegobjekter__vegobjekttypeid_
ARGUMENTS:
None
KEYWORDS
mittfilter=None (default) Valgfritt dictionary med eventuelle filtre for bruenes egenskaper, veg- eller områdefilter m.m.
Se nvdbapiv3.nvdbFagdata eller API dokumentasjon
https://nvdbapiles-v3.atlas.vegvesen.no/dokumentasjon/openapi/#/Vegobjekter/get_vegobjekter__vegobjekttypeid_
offisiell=False (default) | True. Angir om vi skal bruke offisielle eller uoffisielle bruksklassedata (krever innlogging)
kunEnTypeBK = None (default) eller en tekststreng som angir hvilke type bruksklasse vi skal hente. Lovlige verdier:
None : Henter alle BK-variantene
'normal' : Kun Bruksklasse, normaltransport
'spesial' : Kun Bruksklasse, Spesialtransport
'tolv65' : Kun Bruksklasse, 12/65 mobilkran m.m.
'tolv100' : Kun Bruksklasse, 12/100 vegnett
RETURNS
geopandas geodataframe
"""
filteret = {}
# Kopierer mittfilter for å unngå sideefekter
if mittfilter:
filteret = deepcopy(mittfilter)
# Kun Brukategori = vegbru
if 'egenskap' in filteret:
filteret['egenskap'] = '1263=7304 and ' + filteret['egenskap']
else:
filteret['egenskap'] = '1263=7304'
brusok = nvdbapiv3.nvdbFagdata(60)
# brusok.filter( filteret )
bruer = pd.DataFrame(brusok.to_records(relasjoner=False))
bruer = bruer[bruer['trafikantgruppe'] == 'K']
# BRUKSLAST/TOTALVEKT i utdraget. I UTDRAG. Antar vi bruker BK normaltransport. Skal sammenligne BK og totalvekt.
# Veggruppe - fra spesialtransport I UTDRAG Spesialtransport
# SV12/65 godkjent J/N I UTDRAG BK 12/65 (finnes eller finnes ikke)
# SV12/00 godjkent J/N I UTDRAG BK 12/100 (finnes / finnes ikke)
# SV 12/100 restriksjoner (sakte/sentrisk etc) - nikx
# SV 12/100 avstand - niks
if offisiell:
normalprefiks = 'bk904_'
spesialprefix = 'bk902_'
tolv65prefix = 'bk891_'
tolv100prefix = 'bk893_'
normalsok = nvdbapiv3.nvdbFagdata(904) #Normal
spesialsok = nvdbapiv3.nvdbFagdata(902) #Spesial
tolv65sok = nvdbapiv3.nvdbFagdata(891) # 12/65
tolv100sok = nvdbapiv3.nvdbFagdata(893) # 12/100 vegnett
else:
normalprefiks = 'bk905_'
spesialprefix = 'bk903_'
tolv65prefix = 'bk892_'
tolv100prefix = 'bk894_'
normalsok = nvdbapiv3.nvdbFagdata(905)
spesialsok = nvdbapiv3.nvdbFagdata(903)
tolv65sok = nvdbapiv3.nvdbFagdata(892)
tolv100sok = nvdbapiv3.nvdbFagdata(894)
normalsok.forbindelse.login(miljo='prodles', username='******')
spesialsok.forbindelse = normalsok.forbindelse
tolv65sok.forbindelse = normalsok.forbindelse
tolv100sok.forbindelse = normalsok.forbindelse
normalsok.filter({'overlapp': '60'})
spesialsok.filter({'overlapp': '60'})
tolv65sok.filter({'overlapp': '60'})
tolv100sok.filter({'overlapp': '60'})
normal = pd.DataFrame(normalsok.to_records(relasjoner=False))
spesial = pd.DataFrame(spesialsok.to_records(relasjoner=False))
tolv65 = pd.DataFrame(tolv65sok.to_records(relasjoner=False))
tolv100 = pd.DataFrame(tolv100sok.to_records(relasjoner=False))
normal['bktall'] = normal['Bruksklasse'].apply(
lambda x: splitBruksklasse_vekt(x)[0])
normal['bkvekt'] = normal['Bruksklasse'].apply(
lambda x: splitBruksklasse_vekt(x)[1])
# normal['Maks vogntoglengde'] = normal['Maks vogntoglengde'].apply( lambda x : float( x.replace( ',', '.') ) if '.' in x )
# 'Vegliste gjelder alltid',
sletteliste = [
'objekttype', 'nvdbId', 'versjon', 'startdato', 'detaljnivå',
'typeVeg', 'kommune', 'fylke', 'veglenkeType', 'segmentlengde',
'geometri', 'vegkategori', 'fase', 'vegnummer', 'adskilte_lop',
'trafikantgruppe', 'Strekningsbeskrivelse'
]
slettelliste_normal = ['Bruksklasse vinter']
normal.drop(columns=sletteliste + slettelliste_normal, inplace=True)
spesial.drop(columns=sletteliste + slettelliste_normal, inplace=True)
tolv65.drop(columns=sletteliste, inplace=True)
tolv100.drop(columns=sletteliste, inplace=True)
bruprefix = 'bru_'
bruer = bruer.add_prefix(bruprefix)
brucol_nvdbId = bruprefix + 'nvdbId'
sluttresultat = None
# Overlapp bruer - normaltransport
if kunEnTypeBK == None or kunEnTypeBK == 'normal':
mellomresultat2 = nvdbgeotricks.finnoverlapp(bruer,
normal,
prefixA=bruprefix,
prefixB=normalprefiks,
join='left')
mellomresultat2.drop(columns=[
normalprefiks + 'veglenkesekvensid',
normalprefiks + 'startposisjon', normalprefiks + 'sluttposisjon'
],
inplace=True)
sluttresultat = mellomresultat2.copy()
else:
mellomresultat2 = bruer
# Overlapp bruer - spesial
if kunEnTypeBK == None or kunEnTypeBK == 'spesial':
mellomresultat4 = nvdbgeotricks.finnoverlapp(mellomresultat2,
spesial,
prefixA=bruprefix,
prefixB=spesialprefix,
join='left')
mellomresultat4.drop(columns=[
spesialprefix + 'veglenkesekvensid',
spesialprefix + 'startposisjon', spesialprefix + 'sluttposisjon'
],
inplace=True)
sluttresultat = mellomresultat4.copy()
else:
mellomresultat4 = bruer
# Overlapp bruer - 12/65
if kunEnTypeBK == None or kunEnTypeBK == 'tolv65':
mellomresultat6 = nvdbgeotricks.finnoverlapp(mellomresultat4,
tolv65,
prefixA=bruprefix,
prefixB=tolv65prefix,
join='left')
mellomresultat6.drop(columns=[
tolv65prefix + 'veglenkesekvensid', tolv65prefix + 'startposisjon',
tolv65prefix + 'sluttposisjon'
],
inplace=True)
sluttresultat = mellomresultat6.copy()
else:
mellomresultat6 = bruer
# Overlapp bruer - 12/100
if kunEnTypeBK == None or kunEnTypeBK == 'tolv100':
mellomresultat8 = nvdbgeotricks.finnoverlapp(mellomresultat6,
tolv100,
prefixA=bruprefix,
prefixB=tolv100prefix,
join='left')
mellomresultat8.drop(columns=[
tolv100prefix + 'veglenkesekvensid',
tolv100prefix + 'startposisjon', tolv100prefix + 'sluttposisjon'
],
inplace=True)
sluttresultat = mellomresultat8.copy()
# Lager geodataframe
bruer['geometry'] = bruer['bru_geometri'].apply(lambda x: wkt.loads(x))
bruer = gpd.GeoDataFrame(bruer, geometry='geometry', crs=5973)
sluttresultat['geometry'] = sluttresultat['bru_geometri'].apply(
lambda x: wkt.loads(x))
minGdf = gpd.GeoDataFrame(sluttresultat, geometry='geometry', crs=5973)
return minGdf
def warp(self, dem=None, proj="EPSG:4326", **kwargs):
"""Delayed warp across an entire AOI or Image
Creates a new dask image by deferring calls to the warp_geometry on chunks
Args:
dem (ndarray): optional. A DEM for warping to specific elevation planes
proj (str): optional. An EPSG proj string to project the image data into ("EPSG:32612")
Returns:
daskarray: a warped image as deferred image array
"""
try:
img_md = self.rda.metadata["image"]
x_size = img_md["tileXSize"]
y_size = img_md["tileYSize"]
except (AttributeError, KeyError):
x_size = kwargs.get("chunk_size", 256)
y_size = kwargs.get("chunk_size", 256)
# Create an affine transform to convert between real-world and pixels
if self.proj is None:
from_proj = "EPSG:4326"
else:
from_proj = self.proj
try:
# NOTE: this only works on images that have rda rpcs metadata
center = wkt.loads(self.rda.metadata["image"]["imageBoundsWGS84"]).centroid
g = box(*center.buffer(self.rda.metadata["rpcs"]["gsd"] / 2).bounds)
tfm = partial(pyproj.transform, pyproj.Proj(init="EPSG:4326"), pyproj.Proj(init=proj))
gsd = kwargs.get("gsd", ops.transform(tfm, g).area ** 0.5)
current_bounds = wkt.loads(self.rda.metadata["image"]["imageBoundsWGS84"]).bounds
except (AttributeError, KeyError, TypeError):
tfm = partial(pyproj.transform, pyproj.Proj(init=self.proj), pyproj.Proj(init=proj))
gsd = kwargs.get("gsd", (ops.transform(tfm, shape(self)).area / (self.shape[1] * self.shape[2])) ** 0.5)
current_bounds = self.bounds
tfm = partial(pyproj.transform, pyproj.Proj(init=from_proj), pyproj.Proj(init=proj))
itfm = partial(pyproj.transform, pyproj.Proj(init=proj), pyproj.Proj(init=from_proj))
output_bounds = ops.transform(tfm, box(*current_bounds)).bounds
gtf = Affine.from_gdal(output_bounds[0], gsd, 0.0, output_bounds[3], 0.0, -1 * gsd)
ll = ~gtf * (output_bounds[:2])
ur = ~gtf * (output_bounds[2:])
x_chunks = int((ur[0] - ll[0]) / x_size) + 1
y_chunks = int((ll[1] - ur[1]) / y_size) + 1
num_bands = self.shape[0]
try:
dtype = img_md["dataType"]
except:
dtype = 'uint8'
daskmeta = {
"dask": {},
"chunks": (num_bands, y_size, x_size),
"dtype": dtype,
"name": "warp-{}".format(self.name),
"shape": (num_bands, y_chunks * y_size, x_chunks * x_size)
}
def px_to_geom(xmin, ymin):
xmax = int(xmin + x_size)
ymax = int(ymin + y_size)
bounds = list((gtf * (xmin, ymax)) + (gtf * (xmax, ymin)))
return box(*bounds)
full_bounds = box(*output_bounds)
dasks = []
if isinstance(dem, GeoDaskImage):
if dem.proj != proj:
dem = dem.warp(proj=proj, dem=dem)
dasks.append(dem.dask)
for y in range(y_chunks):
for x in range(x_chunks):
xmin = x * x_size
ymin = y * y_size
geometry = px_to_geom(xmin, ymin)
daskmeta["dask"][(daskmeta["name"], 0, y, x)] = (self._warp, geometry, gsd, dem, proj, dtype, 5)
daskmeta["dask"], _ = optimization.cull(HighLevelGraph.merge(daskmeta["dask"], *dasks),
list(daskmeta["dask"].keys()))
gi = mapping(full_bounds)
gt = AffineTransform(gtf, proj)
image = GeoDaskImage(daskmeta, __geo_interface__=gi, __geo_transform__=gt)
return image[box(*output_bounds)]
def get_exposure(oqparam):
"""
Read the full exposure in memory and build a list of
:class:`openquake.risklib.asset.Asset` instances.
:param oqparam:
an :class:`openquake.commonlib.oqvalidation.OqParam` instance
:returns:
an :class:`Exposure` instance
"""
out_of_region = 0
if oqparam.region_constraint:
region = wkt.loads(oqparam.region_constraint)
else:
region = None
all_cost_types = set(oqparam.all_cost_types)
fname = oqparam.inputs['exposure']
exposure, assets_node = _get_exposure(fname, all_cost_types)
relevant_cost_types = all_cost_types - set(['occupants'])
asset_refs = set()
ignore_missing_costs = set(oqparam.ignore_missing_costs)
for idx, asset_node in enumerate(assets_node):
values = {}
deductibles = {}
insurance_limits = {}
retrofitteds = {}
with context(fname, asset_node):
asset_id = asset_node['id'].encode('utf8')
if asset_id in asset_refs:
raise read_nrml.DuplicatedID(asset_id)
asset_refs.add(asset_id)
exposure.asset_refs.append(asset_id)
taxonomy = asset_node['taxonomy']
if 'damage' in oqparam.calculation_mode:
# calculators of 'damage' kind require the 'number'
# if it is missing a KeyError is raised
number = asset_node.attrib['number']
else:
# some calculators ignore the 'number' attribute;
# if it is missing it is considered 1, since we are going
# to multiply by it
try:
number = asset_node['number']
except KeyError:
number = 1
else:
if 'occupants' in all_cost_types:
values['occupants_None'] = number
location = asset_node.location['lon'], asset_node.location['lat']
if region and not geometry.Point(*location).within(region):
out_of_region += 1
continue
tagnode = getattr(asset_node, 'tags', None)
assets_by_tag = exposure.assets_by_tag
if tagnode is not None:
# fill missing tagvalues with "?" and raise an error for
# unknown tagnames
with context(fname, tagnode):
dic = tagnode.attrib.copy()
for tagname in assets_by_tag.tagnames:
try:
tagvalue = dic.pop(tagname)
except KeyError:
tagvalue = '?'
else:
if tagvalue in '?*':
raise ValueError(
'Invalid tagvalue="%s"' % tagvalue)
tag = '%s=%s' % (tagname, tagvalue)
assets_by_tag[tag].append(idx)
if dic:
raise ValueError(
'Unknown tagname %s or <tagNames> not '
'specified in the exposure' % ', '.join(dic))
exposure.assets_by_tag['taxonomy=' + taxonomy].append(idx)
try:
costs = asset_node.costs
except AttributeError:
costs = Node('costs', [])
try:
occupancies = asset_node.occupancies
except AttributeError:
occupancies = Node('occupancies', [])
for cost in costs:
with context(fname, cost):
cost_type = cost['type']
if cost_type in relevant_cost_types:
values[cost_type] = cost['value']
retrovalue = cost.attrib.get('retrofitted')
if retrovalue is not None:
retrofitteds[cost_type] = retrovalue
if oqparam.insured_losses:
deductibles[cost_type] = cost['deductible']
insurance_limits[cost_type] = cost['insuranceLimit']
# check we are not missing a cost type
missing = relevant_cost_types - set(values)
if missing and missing <= ignore_missing_costs:
logging.warn(
'Ignoring asset %s, missing cost type(s): %s',
asset_id, ', '.join(missing))
for cost_type in missing:
values[cost_type] = None
elif missing and 'damage' not in oqparam.calculation_mode:
# missing the costs is okay for damage calculators
with context(fname, asset_node):
raise ValueError("Invalid Exposure. "
"Missing cost %s for asset %s" % (
missing, asset_id))
tot_occupants = 0
for occupancy in occupancies:
with context(fname, occupancy):
exposure.time_events.add(occupancy['period'])
occupants = 'occupants_%s' % occupancy['period']
values[occupants] = occupancy['occupants']
tot_occupants += values[occupants]
if occupancies: # store average occupants
values['occupants_None'] = tot_occupants / len(occupancies)
area = float(asset_node.attrib.get('area', 1))
ass = asset.Asset(idx, taxonomy, number, location, values, area,
deductibles, insurance_limits, retrofitteds,
exposure.cost_calculator)
exposure.assets.append(ass)
if region:
logging.info('Read %d assets within the region_constraint '
'and discarded %d assets outside the region',
len(exposure.assets), out_of_region)
if len(exposure.assets) == 0:
raise RuntimeError('Could not find any asset within the region!')
# sanity checks
values = any(len(ass.values) + ass.number for ass in exposure.assets)
assert values, 'Could not find any value??'
return exposure
def from_wkt(cls, value):
return cls(wkt.loads(value))
def search_by_shapefile(shapefile, filters, startDate, endDate):
with collection(shapefile, 'r') as in_shp:
s = shape(in_shp[0]['geometry'])
wkt_string = s.to_wkt()
results = gbdx.catalog.search(searchAreaWkt=wkt_string,
startDate=startDate,
endDate=endDate,
types=["DigitalGlobeAcquisition"],
filters=filters)
#return results
#Make a shapefile of the results
schema3 = {
'geometry': 'Polygon',
'properties': {
'available': 'str',
'catalogID': 'str',
'browseURL': 'str',
'cloudCover': 'float',
'imageBands': 'str',
'multiResolution': 'float',
'offNadirAngle': 'float',
'ordered': 'str',
'panResolution': 'float',
'sensorPlatformName': 'str',
'sunAzimuth': 'float',
'sunElevation': 'float',
'targetAzimuth': 'float',
'timestamp': 'str',
'timestampWkt': 'str',
'vendorName': 'str'
}
}
with collection("rawSearchResults.shp", "w", "ESRI Shapefile",
schema3) as output:
for poly in results:
output.write({
'geometry':
mapping(loads(poly['properties']['footprintWkt'])),
'properties': {
'available':
poly['properties']['available'],
'catalogID':
poly['properties']['catalogID'],
'browseURL':
poly['properties']['browseURL'],
'cloudCover':
float(poly['properties']['cloudCover']),
'imageBands':
poly['properties']['imageBands'],
'multiResolution':
float(poly['properties']['multiResolution']),
'offNadirAngle':
float(poly['properties']['offNadirAngle']),
'ordered':
poly['properties']['ordered'],
'panResolution':
float(poly['properties']['panResolution']),
'sensorPlatformName':
poly['properties']['sensorPlatformName'],
'sunAzimuth':
float(poly['properties']['sunAzimuth']),
'sunElevation':
float(poly['properties']['sunElevation']),
'targetAzimuth':
float(poly['properties']['targetAzimuth']),
'timestamp':
poly['properties']['timestamp'],
'timestampWkt':
poly['properties']['timestampWkt'],
'vendorName':
poly['properties']['vendorName']
}
})
print results
return results
def test_error_handler_for_bytes():
with pytest.raises(TypeError):
loads(b'POINT (10 10)')
def job_geom(request):
id = request.matchdict['job']
session = DBSession()
job = session.query(Job).get(id)
return FeatureCollection([Feature(id=id, geometry=loads(job.geometry))])
if isinstance(value, str):
value = value.strip()
point = {
'layer_id':
layer_id,
'source_object_id':
source_row[object_id_field]
if method != 'seg_id' else None,
'seg_id':
source_row[seg_id_field] if seg_id_field else None,
'value':
value or '',
'geom':
source_row[source_geom_field]
if method != 'seg_id' else loads('POINT(0 0)').wkt,
}
points.append(point)
if WRITE_OUT:
print('Writing service area polygons...')
poly_table.write(polys)
print('Writing service area single-value lines...')
line_single_table.write(line_singles)
print('Writing service area line dual-value lines...')
line_dual_table.write(line_duals)
print('Writing service area points...')
point_table.write(points)
from shapely.wkt import loads
from geojson import dumps
import pprint
msas = {}
for line in open(
'/Users/jlenaghan/Data/WalmartBlackFriday/OutputV1/PolysToMSA.csv',
'r'):
uid, msa = line.strip().split(',')
msas[uid] = msa
for line in open(
'/Users/jlenaghan/Data/WalmartBlackFriday/AnalysisDataSets/prod_walmarts_wkts.psv',
'r'):
sid, name, wkt, msa = line.strip().split('|')
msa = msas[sid]
if msa == 'losangeles' or msa == 'riverside':
wkt = wkt.replace('"', '')
x = loads(wkt)
gdata = dumps(x)
print gdata
def getS1Cat2Gdf(apihub, opener, query):
"""
Get the data from the scihub catalogue
and write it to a GeoPandas GeoDataFrame
"""
# create empty GDF
colNames = ['identifier', 'polarisationmode', 'orbitdirection',
'acquisitiondate', 'relativeorbitnumber', 'orbitnumber',
'producttype', 'slicenumber', 'size', 'beginposition',
'endposition', 'lastrelativeorbitnumber', 'lastorbitnumber',
'uuid', 'platformidentifier', 'missiondatatakeid',
'swathidentifier', 'ingestiondate', 'sensoroperationalmode',
'footprint']
crs = {'init': 'epsg:4326'}
gdfFull = gpd.GeoDataFrame(columns=colNames, crs=crs,
geometry='footprint')
# we need this for the paging
index = 0
rows = 99
next_page = 1
while next_page:
# construct the final url
url = apihub + query + "&rows={}&start={}".format(rows, index)
try:
# get the request
req = opener.open(url)
except URLError as e:
if hasattr(e, 'reason'):
print(' We failed to connect to the server.')
print(' Reason: ', e.reason)
sys.exit()
elif hasattr(e, 'code'):
print(' The server couldn\'t fulfill the request.')
print(' Error code: ', e.code)
sys.exit()
else:
# write the request to to the response variable
# (i.e. the xml coming back from scihub)
response = req.read().decode('utf-8')
# parse the xml page from the response
dom = xml.dom.minidom.parseString(response)
acqList = []
# loop thorugh each entry (with all metadata)
for node in dom.getElementsByTagName('entry'):
# we get all the date entries
dict_date = {s.getAttribute('name'):dateutil.parser.parse(s.firstChild.data).astimezone(dateutil.tz.tzutc()) for s in node.getElementsByTagName('date')}
# we get all the int entries
dict_int = {s.getAttribute('name'):s.firstChild.data for s in node.getElementsByTagName('int')}
# we create a filter for the str entries (we do not want all) and get them
dict_str = {s.getAttribute('name'):s.firstChild.data for s in node.getElementsByTagName('str')}
# merge the dicts and append to the catalogue list
acq = dict(dict_date,**dict_int,**dict_str)
# fill in emtpy fields in dict by using identifier
if not 'swathidentifier' in acq.keys():
acq['swathidentifier'] = acq['identifier'].split("_")[1]
if not 'producttype' in acq.keys():
acq['producttype'] = acq['identifier'].split("_")[2]
if not 'slicenumber' in acq.keys():
acq['slicenumber'] = 0
# append all scenes from this page to a list
acqList.append([acq['identifier'], acq['polarisationmode'],
acq['orbitdirection'], acq['beginposition'].strftime('%Y%m%d'),
acq['relativeorbitnumber'], acq['orbitnumber'],
acq['producttype'],acq['slicenumber'], acq['size'],
acq['beginposition'].isoformat(), acq['endposition'].isoformat(),
acq['lastrelativeorbitnumber'], acq['lastorbitnumber'],
acq['uuid'], acq['platformidentifier'],
acq['missiondatatakeid'], acq['swathidentifier'],
acq['ingestiondate'].isoformat(), acq['sensoroperationalmode'],
loads(acq['footprint'])])
# transofmr all results from that page to a gdf
gdf = gpd.GeoDataFrame(acqList, columns=colNames, crs=crs, geometry='footprint')
# append the gdf to the full gdf
gdfFull = gdfFull.append(gdf)
# retrieve next page and set index up by 99 entries
next_page = scihub.nextPage(dom)
index += rows
return gdfFull
print(file_gpd)
file_gpd.to_file(outpath,driver='GeoJSON')
file_gpd.to_file(outpath2)
# gpol2 = gpd.read_file('/home/ubuntu/sanit3Dsdi/tests/ground_pol_31984.geojson')
###################### part2
small_bbox_wkt = 'POLYGON((-40.3397827083 -20.3176282943, -40.3353356058 -20.3176282943, -40.3353356058 -20.3214263926, -40.3397827083 -20.3214263926, -40.3397827083 -20.3176282943))'
bbox_polygon = wkt.loads(small_bbox_wkt)
crs_proj = 'EPSG:31984'
data2 = {'name':['bbox'],'geometry':[bbox_polygon]}
bbox_gdf = gpd.GeoDataFrame(data2,crs="EPSG:4326")
bbox_proj = bbox_gdf.to_crs(crs_proj)
bbox_proj['id'] = ['0']
ground_pol = bbox_proj.symmetric_difference(file_gpd.unary_union)
# ground_pol['id'] = ['0']
"textual_float": pd.Series([1.1, 2.0], dtype=np.float64),
"textual_float_nan": pd.Series([1.1, 2.0, np.nan], dtype=np.float64),
"mixed": pd.Series([True, False, None], dtype=hasnan_bool_name),
"uuid_series_str": pd.Series(
[
uuid.UUID("0b8a22ca-80ad-4df5-85ac-fa49c44b7ede"),
uuid.UUID("aaa381d6-8442-4f63-88c8-7c900e9a23c6"),
uuid.UUID("00000000-0000-0000-0000-000000000000"),
],
),
"ip_str": pd.Series(
[IPv4Address("127.0.0.1"), IPv4Address("127.0.0.1")],
),
"geometry_string_series": pd.Series(
[
wkt.loads("POINT (-92 42)"),
wkt.loads("POINT (-92 42.1)"),
wkt.loads("POINT (-92 42.2)"),
],
),
"email_address_str": pd.Series(
[FQDA("test", "example.com"), FQDA("info", "example.eu")],
),
"str_url": pd.Series(
[
urlparse("http://www.cwi.nl:80/%7Eguido/Python.html"),
urlparse("https://github.com/dylan-profiling/hurricane"),
],
),
"path_series_windows_str": pd.Series(
[
def _read_risk_data(self):
# read the exposure (if any), the risk model (if any) and then the
# site collection, possibly extracted from the exposure.
oq = self.oqparam
self.load_crmodel() # must be called first
if oq.hazard_calculation_id:
with util.read(oq.hazard_calculation_id) as dstore:
haz_sitecol = dstore['sitecol'].complete
else:
haz_sitecol = readinput.get_site_collection(oq)
if hasattr(self, 'rup'):
# for scenario we reduce the site collection to the sites
# within the maximum distance from the rupture
haz_sitecol, _dctx = self.cmaker.filter(
haz_sitecol, self.rup)
haz_sitecol.make_complete()
if 'site_model' in oq.inputs:
self.datastore['site_model'] = readinput.get_site_model(oq)
oq_hazard = (self.datastore.parent['oqparam']
if self.datastore.parent else None)
if 'exposure' in oq.inputs:
exposure = self.read_exposure(haz_sitecol)
self.datastore['assetcol'] = self.assetcol
self.datastore['cost_calculator'] = exposure.cost_calculator
if hasattr(readinput.exposure, 'exposures'):
self.datastore['assetcol/exposures'] = (
numpy.array(exposure.exposures, hdf5.vstr))
elif 'assetcol' in self.datastore.parent:
assetcol = self.datastore.parent['assetcol']
if oq.region:
region = wkt.loads(oq.region)
self.sitecol = haz_sitecol.within(region)
if oq.shakemap_id or 'shakemap' in oq.inputs:
self.sitecol, self.assetcol = self.read_shakemap(
haz_sitecol, assetcol)
self.datastore['assetcol'] = self.assetcol
logging.info('Extracted %d/%d assets',
len(self.assetcol), len(assetcol))
nsites = len(self.sitecol)
if (oq.spatial_correlation != 'no' and
nsites > MAXSITES): # hard-coded, heuristic
raise ValueError(CORRELATION_MATRIX_TOO_LARGE % nsites)
elif hasattr(self, 'sitecol') and general.not_equal(
self.sitecol.sids, haz_sitecol.sids):
self.assetcol = assetcol.reduce(self.sitecol)
self.datastore['assetcol'] = self.assetcol
logging.info('Extracted %d/%d assets',
len(self.assetcol), len(assetcol))
else:
self.assetcol = assetcol
else: # no exposure
self.sitecol = haz_sitecol
if self.sitecol:
logging.info('Read N=%d hazard sites and L=%d hazard levels',
len(self.sitecol), len(oq.imtls.array))
if oq_hazard:
parent = self.datastore.parent
if 'assetcol' in parent:
check_time_event(oq, parent['assetcol'].occupancy_periods)
elif oq.job_type == 'risk' and 'exposure' not in oq.inputs:
raise ValueError('Missing exposure both in hazard and risk!')
if oq_hazard.time_event and oq_hazard.time_event != oq.time_event:
raise ValueError(
'The risk configuration file has time_event=%s but the '
'hazard was computed with time_event=%s' % (
oq.time_event, oq_hazard.time_event))
if oq.job_type == 'risk':
tmap_arr, tmap_lst = logictree.taxonomy_mapping(
self.oqparam.inputs.get('taxonomy_mapping'),
self.assetcol.tagcol.taxonomy)
self.crmodel.tmap = tmap_lst
if len(tmap_arr):
self.datastore['taxonomy_mapping'] = tmap_arr
taxonomies = set(taxo for items in self.crmodel.tmap
for taxo, weight in items if taxo != '?')
# check that we are covering all the taxonomies in the exposure
missing = taxonomies - set(self.crmodel.taxonomies)
if self.crmodel and missing:
raise RuntimeError('The exposure contains the taxonomies %s '
'which are not in the risk model' % missing)
if len(self.crmodel.taxonomies) > len(taxonomies):
logging.info('Reducing risk model from %d to %d taxonomies',
len(self.crmodel.taxonomies), len(taxonomies))
self.crmodel = self.crmodel.reduce(taxonomies)
self.crmodel.tmap = tmap_lst
self.crmodel.vectorize_cons_model(self.assetcol.tagcol)
if hasattr(self, 'sitecol') and self.sitecol:
if 'site_model' in oq.inputs:
assoc_dist = (oq.region_grid_spacing * 1.414
if oq.region_grid_spacing else 5) # Graeme's 5km
sm = readinput.get_site_model(oq)
self.sitecol.complete.assoc(sm, assoc_dist)
self.datastore['sitecol'] = self.sitecol.complete
# used in the risk calculators
self.param = dict(individual_curves=oq.individual_curves,
avg_losses=oq.avg_losses)
# compute exposure stats
if hasattr(self, 'assetcol'):
arr = self.assetcol.array
num_assets = list(general.countby(arr, 'site_id').values())
self.datastore['assets_by_site'] = get_stats(num_assets)
num_taxos = self.assetcol.num_taxonomies_by_site()
self.datastore['taxonomies_by_site'] = get_stats(num_taxos)
save_exposed_values(
self.datastore, self.assetcol, oq.loss_names, oq.aggregate_by)
def create_mask(poly, params, poly_type="polygon"):
'''
takes a Well Known Text polygon or line
and produces a masking array for use with numpy
@param poly - WKT polygon or line
@param variable - WCS variable to mask off
@param type - one from [polygon, line]
'''
current_app.logger.debug('##########')
current_app.logger.debug(poly)
loaded_poly = wkt.loads(poly)
wcs_envelope = loaded_poly.envelope
bounds = wcs_envelope.bounds
bb = ','.join(map(str,bounds))
current_app.logger.debug('testing polygon for strangeness')
current_app.logger.debug('bound = %s' % bb)
params['bbox']._value = bb
params['url'] = createURL(params)
variable = params['coverage'].value
#wcs_url = wcs_base_url % (bounds[0],bounds[1],bounds[2],bounds[3])
wcs_url = params['url'].value
current_app.logger.debug(wcs_url)
#testfile=urllib.URLopener()
#testfile.retrieve(wcs_url,"%s.nc" % variable)
try:
resp = contactWCSServer(wcs_url)
except urllib2.HTTPError:
params["vertical"]._value = params["vertical"].value[1:]
params['url'] = createURL(params)
wcs_url = params['url'].value
resp = contactWCSServer(wcs_url)
tfile = saveOutTempFile(resp)
to_be_masked = netCDF.Dataset(tfile, 'r+')
chl = to_be_masked.variables[variable][:]
latvals = to_be_masked.variables[str(getCoordinateVariable(to_be_masked, 'Lat').dimensions[0])][:]
lonvals = to_be_masked.variables[str(getCoordinateVariable(to_be_masked, 'Lon').dimensions[0])][:]
from shapely.geometry import Polygon
minlat = min(latvals)
maxlat = max(latvals)
minlon = min(lonvals)
maxlon = max(lonvals)
lonlat_poly = Polygon([[minlon,maxlat],[maxlon,maxlat],[maxlon,minlat],[minlon,minlat],[minlon,maxlat]])
#print '#'*50
#print lonlat_poly
overlap_poly = loaded_poly.intersection(lonlat_poly)
poly = poly[trim_sizes[poly_type]]
poly = poly.split(',')
poly = [x.split() for x in poly]
#found_lats = [find_closest(latvals, float(x[1])) for x in poly]
#found_lons = [find_closest(lonvals, float(x[0])) for x in poly]
if overlap_poly.type == "MultiPolygon":
found = []
for poly in overlap_poly:
found_lats = [find_closest(latvals, float(x)) for x in poly.exterior.xy[1]]
found_lons = [find_closest(lonvals, float(x)) for x in poly.exterior.xy[0]]
found.append(zip(found_lons,found_lats))
elif overlap_poly.type == "MultiLineString":
found = []
for poly in overlap_poly:
found_lats = [find_closest(latvals, float(x)) for x in poly.xy[1]]
found_lons = [find_closest(lonvals, float(x)) for x in poly.xy[0]]
found.append(zip(found_lons,found_lats))
else:
if poly_type is 'line':
found_lats = [find_closest(latvals, float(x)) for x in overlap_poly.xy[1]]
found_lons = [find_closest(lonvals, float(x)) for x in overlap_poly.xy[0]]
else:
found_lats = [find_closest(latvals, float(x)) for x in overlap_poly.exterior.xy[1]]
found_lons = [find_closest(lonvals, float(x)) for x in overlap_poly.exterior.xy[0]]
#found = zip(overlap_poly.exterior.xy[0],overlap_poly.exterior.xy[1])
found = zip(found_lons,found_lats)
current_app.logger.debug('#'*40)
current_app.logger.debug(found)
# img = Image.new('L', (chl.shape[2],chl.shape[1]), 0)
img = Image.new('L', (chl.shape[to_be_masked.variables[variable].dimensions.index(str(getCoordinateVariable(to_be_masked, 'Lon').dimensions[0]))],chl.shape[to_be_masked.variables[variable].dimensions.index(str(getCoordinateVariable(to_be_masked, 'Lat').dimensions[0]))]), 0)
if overlap_poly.type == "MultiPolygon":
for f in found:
ImageDraw.Draw(img).polygon(f, outline=2, fill=2)
elif overlap_poly.type == "MultiLineString":
for f in found:
ImageDraw.Draw(img).polygon(f, outline=2, fill=2)
else:
if poly_type == 'polygon':
ImageDraw.Draw(img).polygon(found, outline=2, fill=2)
if poly_type == 'line':
ImageDraw.Draw(img).line(found, fill=2)
masker = np.array(img)
#fig = plt.figure()
masked_variable = []
for i in range(chl.shape[0]):
#print i
masked_variable.append(np.ma.masked_array(chl[i,:], mask=[x != 2 for x in masker]))
masked_variable[i].filled(-999)
# a = fig.add_subplot(1,5,i+1)
# imgplot = plt.imshow(masked_variable)
# plt.show()
return masked_variable, to_be_masked, masker, tfile, variable
def add_airspace(self, country_code, airspace_class, name, base, top, geom_str):
try:
geom = loads(geom_str)
except ReadingError:
print name + "(" + airspace_class + ") is not a polygon (maybe not enough points?)"
return False
# orient polygon clockwise
geom = polygon.orient(geom, sign=-1)
if not airspace_class:
print name + " has no airspace class"
return False
base = self.normalise_height(base, name)
top = self.normalise_height(top, name)
flightlevel_re = re.compile(r'^FL (\d+)$')
match = flightlevel_re.match(base)
if match and int(match.group(1)) >= 200:
print name + " has it's base above FL 200 and is therefore disregarded"
return False
airspace = Airspace()
airspace.country_code = country_code
airspace.airspace_class = airspace_class
airspace.name = name
airspace.base = base
airspace.top = top
# Check geometry type, disregard everything except POLYGON
if geom.geom_type != 'Polygon':
print name + " is not a polygon (it's a " + geom.geom_type + ")"
return False
wkb = from_shape(geom, srid=4326)
# Try to fix invalid (self-intersecting) geometries
valid_dump = (func.ST_Dump(func.ST_MakeValid(wkb))).geom
valid_query = db.session.query(func.ST_SetSRID(valid_dump, 4326)).order_by(func.ST_Area(valid_dump).desc()).first()
if not valid_query:
print 'Error importing ' + name
print 'Could not validate geometry'
return False
else:
wkb = valid_query[0]
geom_type = db.session.query(func.ST_GeometryType(wkb)).first()[0]
if geom_type != 'ST_Polygon':
print name + " got some errors makeing it valid..."
return False
tolerance = 0.0000001
simplify = lambda x: func.ST_SimplifyPreserveTopology(x, tolerance)
airspace.the_geom = case(
[
(func.ST_IsValid(wkb), wkb),
(func.ST_IsValid(simplify(wkb)), simplify(wkb)),
],
else_=None)
db.session.add(airspace)
return True
def get_coordinates(geometry):
if geometry:
geo_as_string = wkt.loads(geometry)
lat = geo_as_string.centroid.y
lon = geo_as_string.centroid.x
return lat, lon
def test_read_mapchete_input(mapchete_input):
"""Read Mapchete files as input files."""
config = MapcheteConfig(mapchete_input.path)
area = config.area_at_zoom(5)
testpolygon = "POLYGON ((4 1, 3 1, 2 1, 2 4, 3 4, 3 2, 4 2, 4 1))"
assert area.equals(loads(testpolygon))
log_file=True,
use_cache=True,
data_folder=".temp/data",
logs_folder=".temp/logs",
imgs_folder=".temp/imgs",
cache_folder=".temp/cache",
)
# define queries to use throughout tests
location_point = (37.791427, -122.410018)
address = "600 Montgomery St, San Francisco, California, USA"
place1 = {"city": "Piedmont", "state": "California", "country": "USA"}
place2 = "Bunker Hill, Los Angeles, California"
p = ("POLYGON ((-122.262 37.869, -122.255 37.869, -122.255 37.874,"
"-122.262 37.874, -122.262 37.869))")
polygon = wkt.loads(p)
def test_logging():
# test OSMnx's logger
ox.log("test a fake default message")
ox.log("test a fake debug", level=lg.DEBUG)
ox.log("test a fake info", level=lg.INFO)
ox.log("test a fake warning", level=lg.WARNING)
ox.log("test a fake error", level=lg.ERROR)
ox.citation()
ox.ts(style="date")
ox.ts(style="time")
def get_series():
return [
# Int Series
pd.Series([1, 2, 3], name="int_series"),
pd.Series(range(10), name="int_range"),
pd.Series([1, 2, 3], name="Int64_int_series", dtype="Int64"),
pd.Series([1, 2, 3, np.nan],
name="Int64_int_nan_series",
dtype="Int64"),
pd.Series([1, 0, 0, 1, 0, 1, 1, 1, 1, 0, 0, 0, 0],
name="int_series_boolean"),
# Count
pd.Series(np.array([1, 2, 3, 4], dtype=np.uint32), name="np_uint32"),
# Categorical
pd.Series([1, 2, 3], name="categorical_int_series", dtype="category"),
pd.Series(
pd.Categorical(
["A", "B", "C", "C", "B", "A"],
categories=["A", "B", "C"],
ordered=False,
),
name="categorical_char",
),
pd.Series([1.0, 2.0, 3.1],
dtype="category",
name="categorical_float_series"),
pd.Series(["Georgia", "Sam"],
dtype="category",
name="categorical_string_series"),
pd.Series(
[np.complex(0, 0),
np.complex(1, 2),
np.complex(3, -1)],
name="categorical_complex_series",
dtype="category",
),
# Ordinal
pd.Series(
pd.Categorical(["A", "B", "C", "C", "B", "A"],
categories=["A", "B", "C"],
ordered=True),
name="ordinal",
),
# Float Series
pd.Series([1.0, 2.1, 3.0], name="float_series"),
pd.Series([1.0, 2.5, np.nan], name="float_nan_series"),
pd.Series([1.0, 2.0, 3.0, 4.0], name="float_series2"),
pd.Series(np.array([1.2, 2, 3, 4], dtype=np.float),
name="float_series3"),
pd.Series([1, 2, 3.05, 4], dtype=float, name="float_series4"),
pd.Series([np.nan, 1.2], name="float_series5"),
pd.Series([np.nan, 1.1], dtype=np.single, name="float_series6"),
pd.Series([np.inf, np.NINF, np.PINF, 1000000.0, 5.5],
name="float_with_inf"),
pd.Series([np.inf, np.NINF, np.Infinity, np.PINF], name="inf_series"),
pd.Series([1, 2, np.nan], name="int_nan_series"),
# Nan Series
pd.Series([np.nan], name="nan_series"),
pd.Series([np.nan, np.nan, np.nan, np.nan], name="nan_series_2"),
# String Series
pd.Series(["Patty", "Valentine"], name="string_series"),
pd.Series(["1941-05-24", "13/10/2016"],
name="timestamp_string_series"),
pd.Series(["mack", "the", "finger"], name="string_unicode_series"),
pd.Series(
np.array(["upper", "hall"], dtype=np.unicode_),
name="string_np_unicode_series",
),
pd.Series(["1.0", "2.0", np.nan], name="string_num_nan"),
pd.Series(["1.0", "2.0", "3.0"], name="string_num"),
pd.Series(["1.0", "45.67", np.nan], name="string_flt_nan"),
pd.Series(["1.0", "45.67", "3.5"], name="string_flt"),
pd.Series(
["POINT (-92 42)", "POINT (-92 42.1)", "POINT (-92 42.2)"],
name="geometry_string_series",
),
pd.Series(
[
"I was only robbing the register,",
"I hope you understand",
"One of us had better call up the cops",
"In the hot New Jersey night",
np.nan,
],
name="string_str_nan",
),
pd.Series(["True", "False", None], name="string_bool_nan"),
pd.Series(range(20), name="int_str_range").astype("str"),
pd.Series(["1937-05-06", "20/4/2014"], name="string_date"),
pd.Series(
[
"http://www.cwi.nl:80/%7Eguido/Python.html",
"https://github.com/pandas-profiling/pandas-profiling",
],
name="str_url",
),
pd.Series(
[r"C:\\home\\user\\file.txt", r"C:\\home\\user\\test2.txt"],
name="path_series_windows_str",
),
pd.Series(
[r"/home/user/file.txt", r"/home/user/test2.txt"],
name="path_series_linux_str",
),
# Bool Series
pd.Series([True, False], name="bool_series"),
pd.Series([True, False, None], name="bool_nan_series"),
pd.Series([True, False, None],
name="nullable_bool_series",
dtype="Bool"),
pd.Series([True, False, False, True], dtype=bool, name="bool_series2"),
pd.Series(np.array([1, 0, 0, 1], dtype=np.bool), name="bool_series3"),
# Complex Series
pd.Series(
[np.complex(0, 0),
np.complex(1, 2),
np.complex(3, -1)],
name="complex_series",
),
pd.Series(
[
np.complex(0, 0),
np.complex(1, 2),
np.complex(3, -1),
np.complex(np.nan, np.nan),
],
name="complex_series_nan",
),
pd.Series(["(1+1j)", "(2+2j)", "(10+100j)"], name="str_complex"),
pd.Series(
[np.complex(0, 0),
np.complex(1, 2),
np.complex(3, -1), np.nan],
name="complex_series_nan_2",
),
pd.Series(
[complex(0, 0),
complex(1, 2),
complex(3, -1), np.nan],
name="complex_series_py_nan",
),
pd.Series([complex(0, 0), complex(1, 2),
complex(3, -1)],
name="complex_series_py"),
pd.Series(
[
np.complex(0, 0),
np.complex(1, 0),
np.complex(3, 0),
np.complex(-1, 0)
],
name="complex_series_float",
),
# Datetime Series
pd.Series(
[pd.datetime(2017, 3, 5, 12, 2),
pd.datetime(2019, 12, 4)],
name="timestamp_series",
),
pd.Series(
[
pd.datetime(2017, 3, 5),
pd.datetime(2019, 12, 4, 3, 2, 0), pd.NaT
],
name="timestamp_series_nat",
),
pd.Series(
[pd.datetime(2017, 3, 5),
pd.datetime(2019, 12, 4), pd.NaT],
name="date_series_nat",
),
pd.Series(
pd.date_range(
start="2013-05-18 12:00:00",
periods=2,
freq="H",
tz="Europe/Brussels",
name="timestamp_aware_series",
)),
pd.to_datetime(
pd.Series(
[
datetime.date(2011, 1, 1),
datetime.date(2012, 1, 2),
datetime.date(2013, 1, 1),
],
name="datetime",
)),
# Timedelta Series
pd.Series([pd.Timedelta(days=i) for i in range(3)],
name="timedelta_series"),
pd.Series(
[pd.Timedelta(days=i) for i in range(3)] + [pd.NaT],
name="timedelta_series_nat",
),
# Geometry Series
pd.Series(
[
wkt.loads("POINT (-92 42)"),
wkt.loads("POINT (-92 42.1)"),
wkt.loads("POINT (-92 42.2)"),
],
name="geometry_series",
),
# Path Series
pd.Series(
[
PurePosixPath("/home/user/file.txt"),
PurePosixPath("/home/user/test2.txt"),
],
name="path_series_linux",
),
pd.Series(
[
PureWindowsPath("C:\\home\\user\\file.txt"),
PureWindowsPath("C:\\home\\user\\test2.txt"),
],
name="path_series_windows",
),
# Url Series
pd.Series(
[
urlparse("http://www.cwi.nl:80/%7Eguido/Python.html"),
urlparse("https://github.com/dylan-profiling/hurricane"),
],
name="url_series",
),
# UUID Series
pd.Series(
[
uuid.UUID("0b8a22ca-80ad-4df5-85ac-fa49c44b7ede"),
uuid.UUID("aaa381d6-8442-4f63-88c8-7c900e9a23c6"),
uuid.UUID("00000000-0000-0000-0000-000000000000"),
],
name="uuid_series",
),
pd.Series(
[
"0b8a22ca-80ad-4df5-85ac-fa49c44b7ede",
"aaa381d6-8442-4f63-88c8-7c900e9a23c6",
"00000000-0000-0000-0000-000000000000",
],
name="uuid_series_str",
),
# Object Series
pd.Series([[1, ""], [2, "Rubin"], [3, "Carter"]],
name="mixed_list[str,int]"),
pd.Series(
[{
"why": "did you"
}, {
"bring him": "in for he"
}, {
"aint": "the guy"
}],
name="mixed_dict",
),
pd.Series(
[pd.to_datetime, pd.to_timedelta, pd.read_json, pd.to_pickle],
name="callable",
),
pd.Series([pd, wkt, np], name="module"),
pd.Series(["1.1", "2"], name="textual_float"),
pd.Series(["1.1", "2", "NAN"], name="textual_float_nan"),
# Empty
pd.Series([], name="empty"),
pd.Series([], name="empty_float", dtype=float),
pd.Series([], name="empty_int64", dtype="Int64"),
pd.Series([], name="empty_object", dtype="object"),
pd.Series([], name="empty_bool", dtype=bool),
# IP
pd.Series([IPv4Address("127.0.0.1"),
IPv4Address("127.0.0.1")],
name="ip"),
pd.Series(["127.0.0.1", "127.0.0.1"], name="ip_str"),
]
version = task.version
filter = and_(TileHistory.x == task.x, TileHistory.y == task.y,
TileHistory.job_id == job.id)
task = session.query(TileHistory)\
.filter(filter)\
.order_by(TileHistory.version.desc())\
.first()
if task is not None and version == task.version:
prev_task = session.query(Tile).get(
(task.x, task.y, task.job_id, task.zoom))
admin = user.is_admin() if user else False
stats = get_stats(job)
return dict(job=job,
user=user,
bbox=loads(job.geometry).bounds,
tile=current_task,
prev_task=prev_task,
admin=admin,
stats=stats)
@view_config(route_name='job_geom', renderer='geojson', permission='edit')
def job_geom(request):
id = request.matchdict['job']
session = DBSession()
job = session.query(Job).get(id)
return FeatureCollection([Feature(id=id, geometry=loads(job.geometry))])
@view_config(route_name='job_tiles', renderer='geojson', permission='edit')
def filter_query_all_filters(model, q, filters, user):
"""
Return a query filtered with the cruved and all
the filters available in the synthese form
parameters:
- q (SQLAchemyQuery): an SQLAchemy query
- filters (dict): a dict of filter
- user (User): a user object from User
- allowed datasets (List<int>): an array of ID dataset where the users have autorization
"""
q = filter_query_with_cruved(model, q, user)
if "observers" in filters:
q = q.filter(
model.observers.ilike("%" + filters.pop("observers")[0] + "%"))
if "id_organism" in filters:
id_datasets = (DB.session.query(CorDatasetActor.id_dataset).filter(
CorDatasetActor.id_organism.in_(filters.pop("id_organism"))).all())
formated_datasets = [d[0] for d in id_datasets]
q = q.filter(model.id_dataset.in_(formated_datasets))
if "date_min" in filters:
q = q.filter(model.date_min >= filters.pop("date_min")[0])
if "date_max" in filters:
# set the date_max at 23h59 because a hour can be set in timestamp
date_max = datetime.datetime.strptime(
filters.pop("date_max")[0], '%Y-%m-%d')
date_max = date_max.replace(hour=23, minute=59, second=59)
q = q.filter(model.date_max <= date_max)
if "id_acquisition_framework" in filters:
q = q.join(
TAcquisitionFramework,
model.id_acquisition_framework ==
TAcquisitionFramework.id_acquisition_framework,
)
q = q.filter(
TAcquisitionFramework.id_acquisition_framework.in_(
filters.pop("id_acquisition_framework")))
if "geoIntersection" in filters:
# Intersect with the geom send from the map
ors = []
for str_wkt in filters["geoIntersection"]:
# if the geom is a circle
if "radius" in filters:
radius = filters.pop("radius")[0]
wkt = loads(str_wkt)
wkt = circle_from_point(wkt, float(radius))
else:
wkt = loads(str_wkt)
geom_wkb = from_shape(wkt, srid=4326)
ors.append(model.the_geom_4326.ST_Intersects(geom_wkb))
q = q.filter(or_(*ors))
filters.pop("geoIntersection")
if "period_start" in filters and "period_end" in filters:
period_start = filters.pop("period_start")[0]
period_end = filters.pop("period_end")[0]
q = q.filter(
or_(
func.gn_commons.is_in_period(
func.date(model.date_min),
func.to_date(period_start, "DD-MM"),
func.to_date(period_end, "DD-MM"),
),
func.gn_commons.is_in_period(
func.date(model.date_max),
func.to_date(period_start, "DD-MM"),
func.to_date(period_end, "DD-MM"),
),
))
q, filters = filter_taxonomy(model, q, filters)
# generic filters
join_on_cor_area = False
for colname, value in filters.items():
if colname.startswith("area"):
if not join_on_cor_area:
q = q.join(CorAreaSynthese,
CorAreaSynthese.id_synthese == model.id_synthese)
q = q.filter(CorAreaSynthese.id_area.in_(value))
join_on_cor_area = True
else:
col = getattr(model.__table__.columns, colname)
q = q.filter(col.in_(value))
return q
enddate = '20200430'
frequency = 1 # every nth day in date range will be downloaded
# Download list of Sentinel S5-P NO2 products in region of interest
products = api.query(AOI,
date=(startdate,enddate),
platformname='Sentinel-5',
producttype='L2__NO2___', # useful data types 'L2__SO2___' and 'L2__NO2___'
processingmode='Offline', # 'Near real time' or 'Offline'
)
# Convert to pandas dataframe for ease of use
products_df = api.to_dataframe(products)
# Convert AOI to shapely file
AOIshape = wkt.loads(AOI)
# Create empty list of overlaping geometries
differences = []
# Check which images don't have complete overlap with AOI
for image in range(len(products_df)):
# Convert image footprint to shapely file
footprint = products_df.iloc[image,:]['footprint']
footprintshape = wkt.loads(footprint)
# Calculate difference between AOI and image footprint
difference = AOIshape.difference(footprintshape)
# Append tolist
