def test_stream():
from simplegeom.geometry import Point, Envelope
from StringIO import StringIO
gid = Field("edge_id", "numeric")
left = Field("left_face_id", "numeric")
right = Field("right_face_id", "numeric")
start = Field("start_node_id", "numeric")
end = Field("end_node_id", "numeric")
point = Field("geometry", "point")
centroid = Field("centroid", "point")
envelope = Field("envelope", "box2d")
schema = Schema([], [])
schema.add_field(gid)
schema.add_field(left)
schema.add_field(right)
schema.add_field(start)
schema.add_field(end)
schema.add_field(point)
schema.add_field(centroid)
schema.add_field(envelope)
schema.add_index(Index(fields=[gid], primary_key=True))
schema.add_index(Index(fields=[left]))
schema.add_index(Index(fields=[right]))
schema.add_index(Index(fields=[start]))
schema.add_index(Index(fields=[end]))
schema.add_index(Index(fields=[centroid]))
schema.add_index(Index(fields=[point]))
def example():
from sink import Field, Schema, Index, Layer
from simplegeom.geometry import Point
from random import randint
import timeit
# -- let's define the structure for a little table to hold points
SRID = 28992
gid = Field("gid", "numeric")
geo_point = Field("geo_point", "point")
i_gid_pkey = Index([gid], primary_key=True)
i_geo_point = Index([geo_point], cluster=True)
schema = Schema([gid, geo_point], [i_gid_pkey, i_geo_point])
layer = Layer(schema, "test_table", SRID)
# -- make the schema
with AsyncLoader(workers=1) as loader:
loader.load_schema(layer)
# -- load the data in chunks
with AsyncLoader() as loader:
prev = timeit.default_timer()
for i in range(1, int(5e5)):
layer.append(i, Point(randint(0, 1e8), randint(0, 1e6), SRID))
# check every ten points, whether the layer has more than 50000 features
# if so, dump the data and clear the layer
if (i % 10000 == 0) and len(layer) >= 50000:
now = timeit.default_timer()
# print now - prev, len(layer)
prev = now
loader.load_data(layer)
layer.clear()
# in case of remaining data, also load this data
if len(layer) > 0:
loader.load_data(layer)
layer.clear()
with AsyncLoader(workers=1) as loader:
# -- indexes
loader.load_indexes(layer)
# -- statistics
loader.load_statistics(layer)
def test():
from brep.geometry import Point, Envelope
from StringIO import StringIO
gid = Field("edge_id", "numeric")
left = Field("left_face_id", "numeric")
right = Field("right_face_id", "numeric")
start = Field("start_node_id", "numeric")
end = Field("end_node_id", "numeric")
point = Field("geometry", "point")
centroid = Field("centroid", "point")
envelope = Field("envelope", "box2d")
schema = Schema()
schema.add_field(gid)
schema.add_field(left)
schema.add_field(right)
schema.add_field(start)
schema.add_field(end)
schema.add_field(point)
schema.add_field(centroid)
schema.add_field(envelope)
schema.add_index(Index(fields=[gid], primary_key=True))
schema.add_index(Index(fields=[left]))
schema.add_index(Index(fields=[right]))
schema.add_index(Index(fields=[start]))
schema.add_index(Index(fields=[end]))
schema.add_index(Index(fields=[centroid]))
schema.add_index(Index(fields=[point]))
layer = Layer(schema, "boekie")
layer.append(1, 5, 6, 15, 16, Point(10, 10), Point(10, 10),
Envelope(0, 0, 10, 10))
layer.append(2, 5, 6, 15, 16, Point(10, 15), Point(10, 10),
Envelope(0, 0, 10, 10))
dumps(layer)
fh = StringIO()
dump(layer, fh)
val = fh.getvalue()
print val
fh.close()
def output_layers(name, srid):
"""tables/layers that can be saved in the database as output"""
# tgap = namedtuple('tGAP',
# ('face', 'face_geometry', 'face_hierarchy', 'edge', 'node'))
class tGAP(object):
def __init__(self):
self.face = None
# self.face_geometry = None
self.face_hierarchy = None
self.edge = None
self.node = None
self.edge_statistics = None
def __iter__(self):
return iter([
self.face,
#self.face_geometry,
self.face_hierarchy,
self.edge,
self.node,
self.edge_stats
])
tgap = tGAP()
# imp values
#imp_low = Field("imp_low", "float8")
#imp_high = Field("imp_high", "float8")
#imp_own = Field("imp_own", "float8")
# step
step_low = Field("step_low", "integer") # max: 2147483647
step_high = Field("step_high", "integer")
gstep_low = Field("gstep_low", "integer") # max: 2147483647
gstep_high = Field("gstep_high", "integer")
# face
face_id = Field("face_id", "integer")
area = Field("area", "numeric")
klass = Field("feature_class", "integer")
###### DEVELOPMENT GROUPS ########
# groups1 = Field("groups1", "boolean")
# groups2 = Field("groups2", "boolean")
# groups3 = Field("groups3", "boolean")
# groups4 = Field("groups4", "boolean")
###########
mbr = Field("mbr_geometry", "box2d")
pip = Field("pip_geometry", "point")
parent_id = Field("parent_face_id", "integer")
polygon = Field("geometry", "polygon")
# edge
edge_id = Field("edge_id", "integer")
left_low = Field("left_face_id_low", "integer")
right_low = Field("right_face_id_low", "integer")
left_high = Field("left_face_id_high", "integer")
right_high = Field("right_face_id_high", "integer")
start = Field("start_node_id", "integer")
end = Field("end_node_id", "integer")
edge_class = Field("edge_class",
"integer") # why not also name this feature_class ???
pickled_blg = Field("pickled_blg", "varchar")
smoothline = Field("smooth", "json")
path = Field("geometry", "linestring",
dimension=1) # FIXME: dimension -> needed?
# node
node_id = Field("node_id", "integer")
coord = Field("geometry", "point",
dimension=0) # FIXME: dimension -> needed?
#
face_schema = Schema(
[
face_id,
#imp_low, imp_high,
step_low,
step_high,
#gstep_low, gstep_high,
#imp_own,
area,
klass,
###### DEVELOPMENT GROUPS ########
# groups1,
# groups2,
# groups3,
# groups4,
###### DEVELOPMENT GROUPS ########
mbr,
pip
],
[
Index(fields=[face_id, step_low], primary_key=True),
#Index(fields = [imp_low]),
#Index(fields = [imp_high]),
Index(fields=[mbr])
])
# face_geometry_schema = Schema(
# [face_id,
# imp_low, imp_high,
# step_low, step_high,
# polygon
# ],
# [Index(fields = [face_id, imp_low], primary_key = True),
# Index(fields = [polygon])
# ]
# )
#
face_hier_schema = Schema(
[
face_id,
# imp_low, imp_high,
step_low,
step_high,
parent_id,
],
[
Index(fields=[face_id, step_low], primary_key=True),
#Index(fields = [imp_low]),
#Index(fields = [imp_high]),
Index(fields=[parent_id]),
])
#
edge_schema = Schema(
[
edge_id,
step_low,
step_high,
start,
end,
left_low,
right_low,
left_high,
right_high,
#imp_low, imp_high,
#gstep_low, gstep_high,
edge_class,
#pickled_blg,
#smoothline,
path
],
[
Index(fields=[edge_id, step_low], primary_key=True),
#Index(fields = [imp_low]),
#Index(fields = [imp_high]),
Index(fields=[step_low]),
Index(fields=[step_high]),
Index(fields=[path])
])
#
node_schema = Schema(
[node_id, coord],
[
Index(fields=[node_id]), #, primary_key = True),
Index(fields=[coord])
])
face_step = Field("face_step", "integer")
edge_ct_simplified = Field("edges_simplified",
"integer") # max: 2147483647
edge_ct_total = Field("edges_total", "integer")
edge_stats_schema = Schema([face_step, edge_ct_simplified, edge_ct_total],
[])
#
#folder = '/tmp/' + name + '/'
#if not os.path.exists(folder):
# os.mkdir(folder)
# prefix = 'tgap_'
# face_file = tempfile.NamedTemporaryFile(suffix='_face',
# prefix=prefix,
# #dir=folder,
# delete = False
# )
## face_geometry_file = tempfile.NamedTemporaryFile(suffix='_face_geometry',
## prefix=prefix,
## #dir=folder,
## delete = False
## )
# face_hier_file = tempfile.NamedTemporaryFile(suffix='_face_hier',
# prefix=prefix,
# #dir=folder,
# delete = False
# )
# edge_file = tempfile.NamedTemporaryFile(suffix='_edge',
# prefix=prefix,
# #dir=folder,
# delete = False
# )
# node_file = tempfile.NamedTemporaryFile(suffix='_node',
# prefix=prefix,
# #dir=folder,
# delete = False
# )
face_layer = Layer(face_schema, "{0}_tgap_face".format(name), srid=srid)
# face_geometry_layer = StreamingLayer(face_geometry_schema,
# "{0}_tgap_face_geometry".format(name),
# srid = srid,
# stream=face_geometry_file)
face_hier_layer = Layer(face_hier_schema,
"{0}_tgap_face_hierarchy".format(name),
srid=srid)
edge_layer = Layer(edge_schema, "{0}_tgap_edge".format(name), srid=srid)
node_layer = Layer(node_schema, "{0}_tgap_node".format(name), srid=srid)
edge_stats_layer = Layer(edge_stats_schema,
"{0}_tgap_edge_stats".format(name),
srid=srid)
tgap.face = face_layer
# tgap.face_geometry = face_geometry_layer
tgap.face_hierarchy = face_hier_layer
tgap.edge = edge_layer
tgap.edge_stats = edge_stats_layer
tgap.node = node_layer
return tgap
import warnings
warnings.warn("deprecated", DeprecationWarning)
from sink import Field, Schema, Index, Layer, Phase, loads
# generic attributes
gid = Field("gid", "numeric")
object_begin = Field("object_begin_tijd", "timestamp")
versie_begin = Field("versie_begin_tijd", "timestamp")
status = Field("status", "varchar")
brontype = Field("brontype", "varchar")
bronbeschrijving = Field("bronbeschrijving", "varchar")
bronactualiteit = Field("bronactualiteit", "date")
bronnauwkeurigheid = Field("bronnauwkeurigheid", "numeric")
dimensie = Field("dimensie", "varchar")
# waterdelen
type_water = Field("type_water", "varchar")
breedteklasse = Field("breedteklasse", "varchar")
hoofdafwatering = Field("hoofdafwatering", "varchar")
functie = Field("functie", "varchar")
voorkomen_water = Field("voorkomen_water", "varchar")
stroomrichting = Field("stroomrichting", "varchar")
sluisnaam = Field("sluisnaam", "varchar")
# terrein
type_landgebruik = Field("type_landgebruik", "varchar")
voorkomen = Field("voorkomen", "varchar")
naam = Field("naam", "varchar")
# wegdelen
type_weg = Field("type_weg", "varchar")
def export(cls, topo_map, name, stream, face_geometry=True, wings=False):
if wings:
warnings.warn("Wings have not been fully tested")
# Make schema
# NODES
node_id = Field("node_id", "numeric")
point = Field("geometry", "point")
# TODO:
# - edge_id with node table -> start node = +, end_node = - of edge
# - face_id with node table -> contained in face <face_id>
#
schema = Schema()
schema.add_field(node_id)
schema.add_field(point)
schema.add_index(Index(fields=[node_id], primary_key=True))
schema.add_index(Index(fields=[point], cluster=True))
#
nodes = Layer(schema, '{0}_node'.format(name), srid=topo_map.srid)
# FACES
face_id = Field("face_id", "numeric")
area = Field("area", "float")
class_ = Field("feature_class", "integer")
pip = Field("pip_geometry", "point")
mbr = Field("mbr_geometry", "box2d")
if face_geometry:
poly = Field("geometry", "polygon")
#
schema = Schema()
schema.add_field(face_id)
schema.add_field(area)
schema.add_field(class_)
schema.add_field(pip)
schema.add_field(mbr)
if face_geometry:
schema.add_field(poly)
schema.add_index(Index(fields=[face_id], primary_key=True))
schema.add_index(Index(fields=[pip]))
schema.add_index(Index(fields=[mbr], cluster=True))
if face_geometry:
schema.add_index(Index(fields=[poly], cluster=True))
#
faces = Layer(schema, '{0}_face'.format(name), srid=topo_map.srid)
# TODO:
# - boundary_edge_id: signed edge on outer loop, which starts loop in
# correct direction
# - island_edge_id list
# - island_node_id list
# EDGES
edge_id = Field("edge_id", "numeric")
left = Field("left_face_id", "numeric")
right = Field("right_face_id", "numeric")
start = Field("start_node_id", "numeric")
end = Field("end_node_id", "numeric")
if wings:
# wings
#
# \ /
# \ /
# \e/
# lccw ^ rcw / prvr
# nxtl |
#(LF) | (RF)
# |
# lcw o rccw / nxtr
# prvl/s\
# / \
# / \
# at start:
lcw = Field("lcw", "numeric")
rccw = Field("rccw", "numeric")
# at end:
lccw = Field("lccw", "numeric")
rcw = Field("rcw", "numeric")
linestring = Field("geometry", "linestring")
#
schema = Schema()
schema.add_field(edge_id)
schema.add_field(start)
schema.add_field(end)
schema.add_field(left)
schema.add_field(right)
if wings:
schema.add_field(lcw)
schema.add_field(rccw)
schema.add_field(lccw)
schema.add_field(rcw)
schema.add_field(linestring)
schema.add_index(Index(fields=[edge_id], primary_key=True))
schema.add_index(Index(fields=[left]))
schema.add_index(Index(fields=[right]))
schema.add_index(Index(fields=[start]))
schema.add_index(Index(fields=[end]))
schema.add_index(Index(fields=[linestring], cluster=True))
#
edges = Layer(schema, '{0}_edge'.format(name), srid=topo_map.srid)
# faces
for face in topo_map.faces.values():
if not face.unbounded:
geoms = face.multigeometry(srid=topo_map.srid)
assert len(geoms) == 1
poly = geoms[0]
if face_geometry:
record = (face.id, round(face.area,
3), face.attrs['feature_class'],
poly.representative_point(), poly.envelope, poly)
else:
record = (face.id, round(face.area,
3), face.attrs['feature_class'],
poly.representative_point(), poly.envelope)
faces.append(*record)
# nodes
for node in topo_map.nodes.values():
nodes.append(node.id, node.geometry)
# edges
for edge in topo_map.half_edges.values():
assert edge.anchor is not None
lft = edge
rgt = edge.twin
if wings:
# TODO: signs of wings: + or -
# - lcw / prev left
# - rccw / next right
# - lccw / next left
# - rcw / prev right
lcw, rcw, rccw, lccw = None, None, None, None
for edge in (edge, edge.twin):
if edge.left_face is edge.face:
# lccw / next left
lccw = edge.__next__
if lccw.left_face is edge.face:
lccw_sign = +1
else:
lccw_sign = -1
# lcw / prev left
lcw = edge.prev
if lcw.left_face is edge.face:
lcw_sign = +1
else:
lcw_sign = -1
if edge.right_face is edge.face:
# rccw / next right
rccw = edge.__next__
if rccw.right_face is edge.face:
rccw_sign = -1
else:
rccw_sign = +1
# rcw / prev right
rcw = edge.prev
if rcw.right_face is edge.face:
rcw_sign = -1
else:
rcw_sign = +1
assert lcw is not None
assert rcw is not None
assert lccw is not None
assert rccw is not None
edge = (
lft.id,
lft.origin.id,
rgt.origin.id,
lft.face.id,
rgt.face.id,
lcw_sign * lcw.id,
rccw_sign * rccw.id, # wings at start
lccw_sign * lccw.id,
rcw_sign * rcw.id, # wings at end
lft.geometry)
else:
edge = (lft.id, lft.origin.id, rgt.origin.id, lft.face.id,
rgt.face.id, lft.geometry)
edges.append(*edge)
# TODO: make use of StringIO / file handler
stream.write(dumps(nodes))
stream.write(dumps(edges))
stream.write(dumps(faces))
logging.basicConfig(
format="%(asctime)s %(levelname)s (%(process)d) %(message)s",
stream=sys.stdout,
level=logging.INFO)
log = logging.getLogger(__name__)
from connection.stateful import irecordset, execute
from topomap.io import PolygonFactory
from sink import Schema, Field, dump, dumps, Layer, Index
from sink.bulkload import FileBasedBulkloader
name = "tp_top10nl_small"
universe_id = 0
srid = 28992
edge_id = Field("edge_id", "numeric")
next = Field("next", "numeric")
prev = Field("prev", "numeric")
# rccw = Field("rccw", "numeric")
# rcw = Field("rcw", "numeric")
schema = Schema([edge_id, next, prev])
schema.add_index(Index(fields=[edge_id], primary_key=True))
man0 = FileBasedBulkloader()
stream0 = man0.stream()
layer = Layer(schema, '{0}_edge_left'.format(name), srid=srid)
sql = dumps(layer)
stream0.write(sql)
from sink import use
# use('oracle')
from sink import Field, Schema, Index, Layer, Phase
import sys
##############################################################################
# Here the layer definitions are given for the model
# to which we load the parsed features
##############################################################################
srid = 28992
gid = Field("gid", "numeric")
# GeoObject
identificatie = Field("identificatie", "varchar")
brontype = Field("brontype", "varchar")
bronbeschrijving = Field("bronbeschrijving", "varchar")
bronactualiteit = Field("bronactualiteit", "date")
bronnauwkeurigheid = Field("bronnauwkeurigheid", "float")
dimensie = Field("dimensie", "varchar")
object_begin_tijd = Field("object_begin_tijd", "timestamp")
versie_begin_tijd = Field("versie_begin_tijd", "timestamp")
object_eind_tijd = Field("object_eind_tijd", "timestamp")
versie_eind_tijd = Field("versie_eind_tijd", "timestamp")
visualisatie_code = Field("visualisatie_code", "numeric")
tdn_code = Field("tdn_code", "numeric")
geo_object = [
gid, identificatie, brontype, bronbeschrijving, bronactualiteit,
bronnauwkeurigheid, dimensie, object_begin_tijd, versie_begin_tijd,
object_eind_tijd, versie_eind_tijd, visualisatie_code, tdn_code
]