def computational_func(mode):
if mode == 'crossroads':
return [
wkb_loads(geom) for geom, count in counter.most_common()
if count >= 3
]
elif mode == 'dead_ends':
return [
wkb_loads(geom) for geom, count in counter.most_common()
if count == 1
]
elif mode == 'sections':
return [
wkb_loads(geom) for geom, count in counter.most_common()
if count == 2
]
def build_boundaries(params):
start_time = time.time()
init_logging(app)
site_name = params[0]
output_file = params[1]
current_app.logger.debug("build_boundaries started. Site: %s Outfile: %s" % (site_name, output_file))
try:
boundaries = db.session.query(Boundary) \
.join(Project_Area, Project_Area.id == Boundary.project_site_id) \
.filter(Project_Area.area_name == site_name).all()
except Exception as e:
current_app.logger.exception(e)
else:
try:
with open(output_file, "w") as boundary_file:
#Write header
row = 'WKT,Name\n'
boundary_file.write(row)
for boundary in boundaries:
#Construct the boundary geometry object from the well known binary.
boundary_geom = wkb_loads(boundary.wkb_boundary)
row = '\"%s\",\"%s\"\n' % (boundary_geom.wkt,
boundary.boundary_name)
boundary_file.write(row)
except (IOError, Exception) as e:
current_app.logger.exception(e)
current_app.logger.debug("build_boundaries finished in %f seconds" % (time.time()-start_time))
def timings():
print 'loading'
data = pg.VectorData(
r"C:\Users\kimok\Desktop\gazetteer data\raw\global_urban_extent_polygons_v1.01.shp",
encoding='latin')
#data = list(pg.VectorData(r"C:\Users\kimok\Desktop\gazetteer data\raw\ne_10m_admin_0_countries.shp", encoding='latin')) * 3
print len(data)
print 'making shapely (no copy)'
t = time()
shapelys = [asShape(f.geometry) for f in data]
print time() - t
print 'making shapely (copy)'
t = time()
shapelys = [shape(f.geometry) for f in data]
print time() - t
print 'dump geoj (interface)'
t = time()
geojs = [s.__geo_interface__ for s in shapelys]
print time() - t
##print 'dump geoj (mapping)'
##t = time()
##geojs = [mapping(s) for s in shapelys]
##print time()-t
print 'load geoj asShape (no copy)'
t = time()
shapelys = [asShape(geoj) for geoj in geojs]
print time() - t
print 'load geoj shape (copy)'
t = time()
shapelys = [shape(geoj) for geoj in geojs]
print time() - t
print 'dump wkt'
t = time()
wkts = [s.wkt for s in shapelys]
print time() - t
print 'load wkt'
t = time()
shapelys = [wkt_loads(wkt) for wkt in wkts]
print time() - t
print 'dump wkb'
t = time()
wkbs = [s.wkb for s in shapelys]
print time() - t
print 'load wkb'
t = time()
shapelys = [wkb_loads(wkb) for wkb in wkbs]
print time() - t
def project_shapely_geometry(geom,from_sr,to_sr):
if from_sr.IsSame(to_sr) == 1:
ret = geom
else:
ogr_geom = CreateGeometryFromWkb(geom.wkb)
ogr_geom.AssignSpatialReference(from_sr)
ogr_geom.TransformTo(to_sr)
ret = wkb_loads(ogr_geom.ExportToWkb())
return(ret)
def drop_duplicates_geometry(self, geometry):
self.__assert_multipart_geometry(geometry)
unique_points = list(set([new.wkb for new in geometry.geoms]))
new_geometry = type(geometry)(
[wkb_loads(coords) for coords in unique_points])
return new_geometry
def project_shapely_geometry(geom, from_sr, to_sr):
from ocgis.environment import ogr
CreateGeometryFromWkb = ogr.CreateGeometryFromWkb
if from_sr.IsSame(to_sr) == 1:
ret = geom
else:
ogr_geom = CreateGeometryFromWkb(geom.wkb)
ogr_geom.AssignSpatialReference(from_sr)
ogr_geom.TransformTo(to_sr)
ret = wkb_loads(ogr_geom.ExportToWkb())
return ret
def project_shapely_geometry(geom, from_sr, to_sr):
from ocgis.environment import ogr
CreateGeometryFromWkb = ogr.CreateGeometryFromWkb
if from_sr.IsSame(to_sr) == 1:
ret = geom
else:
ogr_geom = CreateGeometryFromWkb(geom.wkb)
ogr_geom.AssignSpatialReference(from_sr)
ogr_geom.TransformTo(to_sr)
ret = wkb_loads(ogr_geom.ExportToWkb())
return ret
def test_wkt_wkb(wkt, ngw_txn):
ogr_geom = ogr.CreateGeometryFromWkt(wkt)
assert ogr_geom is not None, gdal.GetLastErrorMsg()
wkt_iso = ogr_geom.ExportToIsoWkt()
shp_geom = wkt_loads(wkt)
wkb_shp = shp_geom.wkb
assert shp_geom.wkt == wkt_iso, "Shapely WKT didn't return ISO WKT"
wkb_iso = _pg_wkt_to_wkb_iso(wkt)
wkb_ext = _pg_wkt_to_wkb_ext(wkt)
assert wkb_shp == wkb_ext, "PostGIS EWKT and Shapely WKB didn't match"
assert wkb_loads(
wkb_iso).wkt == wkt_iso, "Shapely didn't understand ISO WKB"
assert wkb_loads(wkb_ext).wkt == wkt_iso, "Shapely didn't understand EWKB"
assert _pg_wkb(wkb_iso) == wkb_ext, "PostGIS didn't understand ISO WKB"
assert _pg_wkb(wkb_ext) == wkb_ext, "PostGIS didn't understand EWKB"
assert Geometry.from_wkb(
wkb_iso).wkt == wkt_iso, "ISO WKB parsing has failed"
assert Geometry.from_wkb(wkb_ext).wkt == wkt_iso, "EWKB parsing has failed"
assert Geometry.from_wkt(wkt_iso).wkb == wkb_ext, "WKT parsing has failed"
def convert_df_wkb_to_shapely(df, cols=[]):
"""Convert geometry columns of DataFrame from WKB to shapely object columns
Parameters
----------
df : :pandas:`pandas.DataFrame`
cols : :obj:`list` of :obj:`str`
Column names
Returns
-------
:pandas:`pandas.DataFrame`
DataFrame with converted columns
"""
for col in cols:
df[col] = df[col].apply(lambda x: wkb_loads(x, hex=True))
return df
def _load_shapely(self):
'''wkb buffer to shapely'''
# TODO: its possible that each reference to geom column in query creates a new geom instance from the db for each
# in that case, storing ._shp doesnt help and shapely creation becomes very duplicative
# explore accessing a dict of cached geometry instances based on the wkb bytes?
## wkb_bytes = self._wkb.tobytes()
## cached = CACHE.get(wkb_bytes, None)
## if cached:
## #print 'getting cached'
## shp = cached
## else:
## #print 'loading new'
## shp = wkb_loads(wkb_bytes)
## CACHE[wkb_bytes] = shp
## self._shp = shp
# non-cache approach
self._shp = wkb_loads(self._wkb.tobytes())
def sqlite_geoms():
print 'load shapefile'
t = time()
#data = pg.VectorData(r"C:\Users\kimok\Desktop\gazetteer data\raw\global_urban_extent_polygons_v1.01.shp", encoding='latin')
#data = pg.VectorData(r"C:\Users\kimok\Desktop\gazetteer data\raw\atlas_urban.geojson", encoding='latin')
data = pg.VectorData(
r"C:\Users\kimok\Desktop\gazetteer data\raw\global_settlement_points_v1.01.shp",
encoding='latin')
#data = pg.VectorData(r"C:\Users\kimok\Desktop\gazetteer data\raw\ne_10m_admin_0_countries.shp", encoding='latin')
print time() - t
print 'making shapely'
t = time()
shapelys = [
shape(f.geometry) for f in data
] # CRUCIAL SPEEDUP, SHAPELY SHOULD BE FROM SHAPE, NOT ASSHAPE WHICH IS INDIRECT REFERENCING
print time() - t
print 'dump wkb'
t = time()
wkbs = [s.wkb for s in shapelys]
print time() - t
print 'convert to binary'
from sqlite3 import Binary
t = time()
blobs = [Binary(wkb) for wkb in wkbs]
print time() - t
print 'insert wkb into db'
fields = ['ID', 'geom']
typs = ['int', 'BLOB']
w = Writer('testgeodb::data', fields=zip(fields, typs), replace=True)
t = time()
for i, blb in enumerate(blobs):
w.add([i, blb])
print time() - t
print 'load wkb from db'
t = time()
shapelys = [wkb_loads(bytes(blb)) for ID, blb in w.select('*')]
print time() - t
def __getattr__(self, name):
if self.obj is None:
self.obj = wkb_loads(self.wkb)
return getattr(self.obj, name)
def fulltextsearch(self):
try:
lang = self.request.registry.settings["default_locale_name"]
except KeyError:
return HTTPInternalServerError(detail="default_locale_name not defined in settings")
try:
lang = self.languages[lang]
except KeyError:
return HTTPInternalServerError(detail="%s not defined in languages" % lang)
if "query" not in self.request.params:
return HTTPBadRequest(detail="no query")
query = self.request.params.get("query")
maxlimit = self.settings.get("maxlimit", 200)
try:
limit = int(self.request.params.get("limit", self.settings.get("defaultlimit", 30)))
except ValueError:
return HTTPBadRequest(detail="limit value is incorrect")
if limit > maxlimit:
limit = maxlimit
try:
partitionlimit = int(self.request.params.get("partitionlimit", 0))
except ValueError:
return HTTPBadRequest(detail="partitionlimit value is incorrect")
if partitionlimit > maxlimit:
partitionlimit = maxlimit
terms = "&".join(w + ":*" for w in query.split(" ") if w != "")
_filter = "%(tsvector)s @@ to_tsquery('%(lang)s', '%(terms)s')" % {
"tsvector": "ts",
"lang": lang,
"terms": terms,
}
# flake8 does not like `== True`
if self.request.user is None:
_filter = and_(_filter, FullTextSearch.public == True) # NOQA
else:
_filter = and_(
_filter,
or_(
FullTextSearch.public == True, # NOQA
FullTextSearch.role_id == None,
FullTextSearch.role_id == self.request.user.role.id,
),
)
# The numbers used in ts_rank_cd() below indicate a normalization method.
# Several normalization methods can be combined using |.
# 2 divides the rank by the document length
# 8 divides the rank by the number of unique words in document
# By combining them, shorter results seem to be preferred over longer ones
# with the same ratio of matching words. But this relies only on testing it
# and on some assumptions about how it might be calculated
# (the normalization is applied two times with the combination of 2 and 8,
# so the effect on at least the one-word-results is therefore stronger).
rank = "ts_rank_cd(%(tsvector)s, " "to_tsquery('%(lang)s', '%(terms)s'), 2|8)" % {
"tsvector": "ts",
"lang": lang,
"terms": terms,
}
if partitionlimit:
# Here we want to partition the search results based on
# layer_name and limit each partition.
row_number = (
func.row_number()
.over(partition_by=FullTextSearch.layer_name, order_by=(desc(rank), FullTextSearch.label))
.label("row_number")
)
subq = DBSession.query(FullTextSearch).add_columns(row_number).filter(_filter).subquery()
query = DBSession.query(subq.c.id, subq.c.label, subq.c.params, subq.c.layer_name, subq.c.the_geom)
query = query.filter(subq.c.row_number <= partitionlimit)
else:
query = DBSession.query(FullTextSearch).filter(_filter)
query = query.order_by(desc(rank))
query = query.order_by(FullTextSearch.label)
query = query.limit(limit)
objs = query.all()
features = []
for o in objs:
if o.the_geom is not None:
properties = {"label": o.label, "layer_name": o.layer_name, "params": o.params}
geom = wkb_loads(str(o.the_geom.geom_wkb))
feature = Feature(id=o.id, geometry=geom, properties=properties, bbox=geom.bounds)
features.append(feature)
# TODO: add callback function if provided in self.request, else return geojson
return FeatureCollection(features)
def geom_to_coords(geom):
coordinates_shp = wkb_loads(geom, hex=True)
coordinates = [coordinates_shp.x, coordinates_shp.y]
return coordinates
def __getattr__(self, name):
if self.obj is None:
self.obj = wkb_loads(self.wkb)
return getattr(self.obj, name)
def bounds(self):
if self.obj is None:
self.obj = wkb_loads(self.wkb)
if self._bounds is None:
self._bounds = self.obj.bounds
return self._bounds
def from_wkb(wkb_buf):
# wkb buffer to shapely
shp = wkb_loads(bytes(wkb_buf))
return shp
def create_geom_in_row(rowdict):
if rowdict:
rowdict['geometry'] = wkb_loads(binascii.unhexlify(rowdict['geometry']))
return rowdict
for layer, key in zip(layers, layer_map.keys()):
print " Features: %i" % layer.featureCount()
provider = layer.dataProvider()
allAttrs = provider.attributeIndexes()
# start data retreival: fetch geometry and all attributes for each feature
provider.select(allAttrs)
feat = QgsFeature()
while provider.nextFeature(feat):
# fetch geometry
geom = feat.geometry()
print " Feature ID: %s" % feat.id()
features.append(
dict(
wkt=wkt_dumps(wkb_loads(geom.asWkb())),
key=key,
desc=layer_map[key]
)
)
print "Total features: %i" % len(features)
flush_and_transmit(features)
dstor_qgis()
remove_temp(dir_out)
def __geo_interface__(self):
from shapely.wkb import loads as wkb_loads
shp = wkb_loads(self.wkb)
geoj = shp.__geo_interface__
return geoj
def fulltextsearch(self):
try:
lang = self.request.registry.settings['default_locale_name']
except KeyError:
return HTTPInternalServerError(
detail='default_locale_name not defined in settings')
try:
lang = self.languages[lang]
except KeyError:
return HTTPInternalServerError(
detail='%s not defined in languages' % lang)
if 'query' not in self.request.params:
return HTTPBadRequest(detail='no query')
query = self.request.params.get('query')
maxlimit = self.settings.get('maxlimit', 200)
try:
limit = int(
self.request.params.get('limit',
self.settings.get('defaultlimit', 30)))
except ValueError:
return HTTPBadRequest(detail='limit value is incorrect')
if limit > maxlimit:
limit = maxlimit
try:
partitionlimit = int(self.request.params.get('partitionlimit', 0))
except ValueError:
return HTTPBadRequest(detail='partitionlimit value is incorrect')
if partitionlimit > maxlimit:
partitionlimit = maxlimit
terms = '&'.join(w + ':*' for w in query.split(' ') if w != '')
_filter = "%(tsvector)s @@ to_tsquery('%(lang)s', '%(terms)s')" % \
{'tsvector': 'ts', 'lang': lang, 'terms': terms}
# flake8 does not like `== True`
if self.request.user is None:
_filter = and_(_filter, FullTextSearch.public == True) # NOQA
else:
_filter = and_(
_filter,
or_(
FullTextSearch.public == True, # NOQA
FullTextSearch.role_id == None,
FullTextSearch.role_id == self.request.user.role.id))
# The numbers used in ts_rank_cd() below indicate a normalization method.
# Several normalization methods can be combined using |.
# 2 divides the rank by the document length
# 8 divides the rank by the number of unique words in document
# By combining them, shorter results seem to be preferred over longer ones
# with the same ratio of matching words. But this relies only on testing it
# and on some assumptions about how it might be calculated
# (the normalization is applied two times with the combination of 2 and 8,
# so the effect on at least the one-word-results is therefore stronger).
rank = "ts_rank_cd(%(tsvector)s, " \
"to_tsquery('%(lang)s', '%(terms)s'), 2|8)" % {
'tsvector': 'ts',
'lang': lang,
'terms': terms
}
if partitionlimit:
# Here we want to partition the search results based on
# layer_name and limit each partition.
row_number = func.row_number() \
.over(
partition_by=FullTextSearch.layer_name,
order_by=(desc(rank), FullTextSearch.label)) \
.label('row_number')
subq = DBSession.query(FullTextSearch) \
.add_columns(row_number).filter(_filter).subquery()
query = DBSession.query(subq.c.id, subq.c.label, subq.c.params,
subq.c.layer_name, subq.c.the_geom)
query = query.filter(subq.c.row_number <= partitionlimit)
else:
query = DBSession.query(FullTextSearch).filter(_filter)
query = query.order_by(desc(rank))
query = query.order_by(FullTextSearch.label)
query = query.limit(limit)
objs = query.all()
features = []
for o in objs:
if o.the_geom is not None:
properties = {
"label": o.label,
"layer_name": o.layer_name,
"params": o.params,
}
geom = wkb_loads(str(o.the_geom.geom_wkb))
feature = Feature(id=o.id,
geometry=geom,
properties=properties,
bbox=geom.bounds)
features.append(feature)
# TODO: add callback function if provided in self.request, else return geojson
return FeatureCollection(features)
def fulltextsearch(request):
try:
lang = request.registry.settings['default_locale_name']
except KeyError:
return HTTPInternalServerError(
detail='default_locale_name not defined in settings')
try:
lang = _language_dict[lang]
except KeyError:
return HTTPInternalServerError(
detail='%s not defined in _language_dict' % lang)
if 'query' not in request.params:
return HTTPBadRequest(detail='no query')
query = request.params.get('query')
maxlimit = request.registry.settings.get('fulltextsearch_maxlimit', 200)
try:
limit = int(request.params.get(
'limit',
request.registry.settings.get('fulltextsearch_defaultlimit', 30)))
except ValueError:
return HTTPBadRequest(detail='limit value is incorrect')
if limit > maxlimit:
limit = maxlimit
try:
partitionlimit = int(request.params.get('partitionlimit', 0))
except ValueError:
return HTTPBadRequest(detail='partitionlimit value is incorrect')
if partitionlimit > maxlimit:
partitionlimit = maxlimit
terms = '&'.join(w + ':*' for w in query.split(' ') if w != '')
_filter = "%(tsvector)s @@ to_tsquery('%(lang)s', '%(terms)s')" % \
{'tsvector': 'ts', 'lang': lang, 'terms': terms}
# flake8 does not like `== True`
if request.user is None:
_filter = and_(_filter, FullTextSearch.public == True) # NOQA
else:
_filter = and_(
_filter,
or_(FullTextSearch.public == True,
FullTextSearch.role_id == None,
FullTextSearch.role_id == request.user.role.id)) # NOQA
# The numbers used in ts_rank_cd() below indicate a normalization method.
# Several normalization methods can be combined using |.
# 2 divides the rank by the document length
# 8 divides the rank by the number of unique words in document
# By combining them, shorter results seem to be preferred over longer ones
# with the same ratio of matching words. But this relies only on testing it
# and on some assumptions about how it might be calculated
# (the normalization is applied two times with the combination of 2 and 8,
# so the effect on at least the one-word-results is therefore stronger).
rank = "ts_rank_cd(%(tsvector)s, " \
"to_tsquery('%(lang)s', '%(terms)s'), 2|8)" % {
'tsvector': 'ts',
'lang': lang,
'terms': terms
}
if partitionlimit:
# Here we want to partition the search results based on
# layer_name and limit each partition.
row_number = func.row_number() \
.over(
partition_by=FullTextSearch.layer_name,
order_by=(desc(rank), FullTextSearch.label)) \
.label('row_number')
subq = DBSession.query(FullTextSearch) \
.add_columns(row_number).filter(_filter).subquery()
query = DBSession.query(subq.c.id, subq.c.label,
subq.c.layer_name, subq.c.the_geom)
query = query.filter(subq.c.row_number <= partitionlimit)
else:
query = DBSession.query(FullTextSearch).filter(_filter)
query = query.order_by(desc(rank))
query = query.order_by(FullTextSearch.label)
query = query.limit(limit)
objs = query.all()
features = []
for o in objs:
if o.the_geom is not None:
properties = {"label": o.label, "layer_name": o.layer_name}
geom = wkb_loads(str(o.the_geom.geom_wkb))
feature = Feature(id=o.id, geometry=geom,
properties=properties, bbox=geom.bounds)
features.append(feature)
# TODO: add callback function if provided in request, else return geojson
return FeatureCollection(features)
def create_geom_in_row(rowdict):
if rowdict:
rowdict['geometry'] = wkb_loads(binascii.unhexlify(rowdict['geometry']))
return rowdict
layers.append(import_shapefile(dir_out, "%s.shp" % k))
features = []
for layer, key in zip(layers, layer_map.keys()):
print " Features: %i" % layer.featureCount()
provider = layer.dataProvider()
allAttrs = provider.attributeIndexes()
# start data retreival: fetch geometry and all attributes for each feature
provider.select(allAttrs)
feat = QgsFeature()
while provider.nextFeature(feat):
# fetch geometry
geom = feat.geometry()
print " Feature ID: %s" % feat.id()
features.append(
dict(wkt=wkt_dumps(wkb_loads(geom.asWkb())),
key=key,
desc=layer_map[key]))
print "Total features: %i" % len(features)
flush_and_transmit(features)
dstor_qgis()
remove_temp(dir_out)
def bounds(self):
if self.obj is None:
self.obj = wkb_loads(self.wkb)
if self._bounds is None:
self._bounds = self.obj.bounds
return self._bounds