def _read_geos(self):
'''
Reads the geometries from the shapefile and dumps them into a string
:returns:the id and geometries of the links in a string
'''
irecord = self.shapefile_reader.iterRecords()
ishapes = self.shapefile_reader.iterShapes()
# read entries into a dictionary which has ids mapped to geometries
id_to_geometry = list()
for i in range(self.shapefile_reader.numRecords):
record = irecord.next()
shape = ishapes.next()
points = [tuple(x) for x in shape.points]
id = int(record[0])
orig_id = int(record[7])
line = LineString(points)
line.set_srid(config.EPSG32611)
defaultprojection = line.clone()
defaultprojection.transform(config.canonical_projection)
googleprojection = line.clone()
googleprojection.transform(config.google_projection)
id_to_geometry.append('\t'.join([str(i), str(id), defaultprojection.ewkt, googleprojection.ewkt, line.ewkt, str(orig_id)]))
return '\n'.join(id_to_geometry)
def _read_geos(self):
'''
Reads the geometries from the shapefile and dumps them into a string
:returns:the id and geometries of the links in a string
'''
irecord = self.shapefile_reader.iterRecords()
ishapes = self.shapefile_reader.iterShapes()
# read entries into a dictionary which has ids mapped to geometries
id_to_geometry = list()
for i in range(self.shapefile_reader.numRecords):
record = irecord.next()
shape = ishapes.next()
points = [tuple(x) for x in shape.points]
id = int(record[0])
orig_id = int(record[7])
line = LineString(points)
line.set_srid(config.EPSG32611)
defaultprojection = line.clone()
defaultprojection.transform(config.canonical_projection)
googleprojection = line.clone()
googleprojection.transform(config.google_projection)
id_to_geometry.append('\t'.join([
str(i),
str(id), defaultprojection.ewkt, googleprojection.ewkt,
line.ewkt,
str(orig_id)
]))
return '\n'.join(id_to_geometry)
def save_into_database(rib_path, rmb_path, putdict, sewerdict, rmberrors):
# Get sewerage name, try to create sewerage
# If it exists, return with an error
sewerage_name = os.path.basename(rmb_path)[:-4] # Minus ".RMB"
if models.Sewerage.objects.filter(name=sewerage_name).exists():
rmberrors.append(
Error(line_number=0,
message=("Er bestaat al een stelsel met de naam {name}. "
"Verwijder het op de archiefpagina, of gebruik "
"een andere naam.").format(name=sewerage_name)))
return
# Files are copied only at the end
sewerage = models.Sewerage.objects.create(
name=sewerage_name,
rib=None, # Filled in later
rmb=None,
active=True)
# Save the puts, keep a dictionary
saved_puts = dict()
for put_id, putinfo in putdict.items():
saved_puts[put_id] = models.Manhole.objects.create(
sewerage=sewerage,
code=put_id,
sink=int(putinfo['is_sink']),
ground_level=putinfo['surface_level'],
the_geom=Point(*putinfo['coordinate']))
# Save the sewers, use the dictionary
saved_sewers = dict()
for sewer_id, sewerinfo in sewerdict.items():
manhole1 = saved_puts[sewerinfo['manhole_code_1']]
manhole2 = saved_puts[sewerinfo['manhole_code_2']]
sewer_line_rd = LineString(manhole1.the_geom, manhole2.the_geom)
sewer_line_rd.set_srid(4326)
sewer_line_rd.transform(RD)
saved_sewers[sewer_id] = models.Sewer.objects.create(
sewerage=sewerage,
code=sewer_id,
quality=models.Sewer.QUALITY_UNKNOWN,
diameter=sewerinfo['diameter'],
manhole1=manhole1,
manhole2=manhole2,
bob1=sewerinfo['bob_1'],
bob2=sewerinfo['bob_2'],
the_geom=LineString(manhole1.the_geom, manhole2.the_geom),
the_geom_length=sewer_line_rd.length)
# Save the measurements
sewer_measurements_dict = dict()
for sewer_id, sewerinfo in sewerdict.items():
measurements = sewerinfo['measurements']
sewer = saved_sewers[sewer_id]
if measurements:
sewer_measurements = [
# Create the SewerMeasurement objects, but don't save
# them yet!
models.SewerMeasurement(sewer=sewer,
dist=m['dist'],
virtual=False,
water_level=None,
flooded_pct=None,
bob=m['bob'],
obb=m['bob'] + sewerinfo['diameter'],
the_geom=Point(*m['coordinate']))
for m in measurements
]
# Quality
sewer.judge_quality(sewer_measurements)
sewer.save()
# BOB correction ("sawtooth" phenomenon)
correct_bob_values(sewer, sewer_measurements)
# Create two virtual sewer measurements for the start and
# end of the sewer
virtual_start = models.SewerMeasurement(
sewer=sewer,
dist=0,
virtual=True,
water_level=None,
flooded_pct=None,
bob=sewer.bob1,
obb=sewer.bob1 + sewerinfo['diameter'],
the_geom=sewer.manhole1.the_geom)
virtual_end = models.SewerMeasurement(
sewer=sewer,
dist=sewer.the_geom_length,
virtual=True,
water_level=None,
flooded_pct=None,
bob=sewer.bob2,
obb=sewer.bob2 + sewerinfo['diameter'],
the_geom=sewer.manhole2.the_geom)
# Note: we MUST add those two virtual points only after
# doing the sawtooth correction, otherwise the sawtooth
# correction will think that everything is fine already
# since the first and end points would be equal to the
# bobs of the sewer...
sewer_measurements = ([virtual_start] + sewer_measurements +
[virtual_end])
sewer_measurements_dict[sewer_id] = sewer_measurements
else:
# Create "virtual measurements"
sewer_measurements_dict[sewer_id] = list(
virtual_measurements(sewer))
sewer.quality = models.Sewer.QUALITY_UNKNOWN
sewer.save()
# Actually compute the lost capacity, the point of this app
lost_capacity.compute_lost_capacity(saved_puts, saved_sewers,
sewer_measurements_dict)
# Save all the SewerMeasurement objects to the database. Since
# there are thousands of them, it is essential to use bulk_create.
models.SewerMeasurement.objects.bulk_create(
list(chain(*sewer_measurements_dict.values())))
# Success -- copy files
sewerage.move_files(rib_path, rmb_path)
# The clap on the fireworks
sewerage.generate_rib()
def find_shared_segments(R, S):
"""Return a list of LineStrings where two routes overlap
Keyword arguments:
route_a -- LineString route
route_b -- LineString route
This method should be commutative.
Algorithmic idea:
For a given set of routes that are not identical:
- Go through each route (R and S), and make a list, P(*) for each that contains points not on the other
route.
- Starting with P(R), iterate through elements p_i = P(R)_i:
- Find p_i in original polyline, R.
- Find the vertex of the item before p_i in R, v_s. v_s should be on the other route S.
- Starting from this element in R, move forward through route until you find the first vertex
that is on S, call this v_f. Remove elements not on S from P(R), including p_i
- Starting from v_s, find closest vertex on S, call it g. This could be before or after where v_s
intersects the route.
- Move along S in both directions from g, taking ordered pairs where order is index in S, and identify
the first pair where the first element is on R, but the second element is not.
- If the first element of this tuple is after g, then that element is end of a shared leg.
- Otherwise, v_s is the end of that shared leg.
- Starting from v_f, find closest vertex on S, call it g, this could be before or after where v_f
intersects the route.
- Move along S in both directions from g, taking ordered pairs where order is index in S, and identify
the first pair where the first element is not on R, but the second element is.
- If the second element of this tuple is before g, then that element is the start of a shared leg.
- Otherwise, v_f is the start of that shared leg.
- At this point, we have a list of starting points and ending points of shared legs, combining them
in to a polyline from the two routes seems tractable. TODO: figure this out when brain is less fried.
"""
R_points_not_on_S = []
S_points_not_on_R = []
S_trans = S.clone()
S_trans.set_srid(4326)
S_trans.transform(900913)
for pt in R:
place = Point(pt)
place.set_srid(4326)
place.transform(900913)
if place.distance(S_trans) > FUZZY_DIST:
R_points_not_on_S.append(pt)
#TODO: refactor these into single function call
S_trans = S.clone()
S_trans.set_srid(4326)
S_trans.transform(900913)
R_trans = R.clone()
R_trans.set_srid(4326)
R_trans.transform(900913)
for pt in S:
place = Point(pt, srid=4326)
place.transform(900913)
if place.distance(R_trans) > FUZZY_DIST:
S_points_not_on_R.append(pt)
# we know they start at the same point
shared_leg_list = []
shared_leg = []
shared_leg_start_index = 0
n = len(R_points_not_on_S)
while n > 0:
p_i = R_points_not_on_S[0]
j = R.index(p_i)
v_s = R[j - 1]
f = j
v_f = p_i
while v_f in R_points_not_on_S:
idx = R_points_not_on_S.index(v_f)
del R_points_not_on_S[idx]
n = n - 1
f = f + 1
v_f = R[f]
# We know v_f is fuzzy-on S, so we can iterate through pairs of S and find the segment it is fuzzy-on
before_index = 0
for i, start_vertex in enumerate(S):
if i == len(S):
break
end_vertex = S[i + 1]
line = LineString([start_vertex, end_vertex])
line.set_srid(4326)
line.transform(900913)
pt = Point(v_s)
pt.set_srid(4326)
pt.transform(900913)
if pt.distance(line) < FUZZY_DIST:
before_index = i
break
# At this point, we know shared_leg_start_index..before_index is certainly on the path.
shared_leg = S[shared_leg_start_index:(before_index + 1)]
shared_leg.append(v_s)
after_index = before_index + 1
# Either v_s is the end of the previous shared segment, or after_index or something following that,
# so go until you find a point on S not on R, starting at after_index
pt = Point(S[after_index], srid=4326)
pt.transform(900913)
while pt.distance(R_trans) < FUZZY_DIST:
shared_leg.append(S[after_index])
after_index = after_index + 1
# should check that shared_leg is not just first element. In fact, TODO: go back an check what happens
# if the first element is the only shared element.
shared_leg_list.append(LineString(shared_leg))
return shared_leg_list
shared_leg = []
def find_shared_segments(R, S):
"""Return a list of LineStrings where two routes overlap
Keyword arguments:
route_a -- LineString route
route_b -- LineString route
This method should be commutative.
Algorithmic idea:
For a given set of routes that are not identical:
- Go through each route (R and S), and make a list, P(*) for each that contains points not on the other
route.
- Starting with P(R), iterate through elements p_i = P(R)_i:
- Find p_i in original polyline, R.
- Find the vertex of the item before p_i in R, v_s. v_s should be on the other route S.
- Starting from this element in R, move forward through route until you find the first vertex
that is on S, call this v_f. Remove elements not on S from P(R), including p_i
- Starting from v_s, find closest vertex on S, call it g. This could be before or after where v_s
intersects the route.
- Move along S in both directions from g, taking ordered pairs where order is index in S, and identify
the first pair where the first element is on R, but the second element is not.
- If the first element of this tuple is after g, then that element is end of a shared leg.
- Otherwise, v_s is the end of that shared leg.
- Starting from v_f, find closest vertex on S, call it g, this could be before or after where v_f
intersects the route.
- Move along S in both directions from g, taking ordered pairs where order is index in S, and identify
the first pair where the first element is not on R, but the second element is.
- If the second element of this tuple is before g, then that element is the start of a shared leg.
- Otherwise, v_f is the start of that shared leg.
- At this point, we have a list of starting points and ending points of shared legs, combining them
in to a polyline from the two routes seems tractable. TODO: figure this out when brain is less fried.
"""
R_points_not_on_S = []
S_points_not_on_R = []
S_trans = S.clone()
S_trans.set_srid(4326)
S_trans.transform(900913)
for pt in R:
place = Point(pt)
place.set_srid(4326)
place.transform(900913)
if place.distance(S_trans) > FUZZY_DIST:
R_points_not_on_S.append(pt)
#TODO: refactor these into single function call
S_trans = S.clone()
S_trans.set_srid(4326)
S_trans.transform(900913)
R_trans = R.clone()
R_trans.set_srid(4326)
R_trans.transform(900913)
for pt in S:
place = Point(pt, srid=4326)
place.transform(900913)
if place.distance(R_trans) > FUZZY_DIST:
S_points_not_on_R.append(pt)
# we know they start at the same point
shared_leg_list = []
shared_leg = []
shared_leg_start_index = 0
n = len(R_points_not_on_S)
while n > 0:
p_i = R_points_not_on_S[0]
j = R.index(p_i)
v_s = R[j - 1]
f = j
v_f = p_i
while v_f in R_points_not_on_S:
idx = R_points_not_on_S.index(v_f)
del R_points_not_on_S[idx]
n = n - 1
f = f + 1
v_f = R[f]
# We know v_f is fuzzy-on S, so we can iterate through pairs of S and find the segment it is fuzzy-on
before_index = 0
for i, start_vertex in enumerate(S):
if i == len(S):
break
end_vertex = S[i + 1]
line = LineString([start_vertex, end_vertex])
line.set_srid(4326)
line.transform(900913)
pt = Point(v_s)
pt.set_srid(4326)
pt.transform(900913)
if pt.distance(line) < FUZZY_DIST:
before_index = i
break
# At this point, we know shared_leg_start_index..before_index is certainly on the path.
shared_leg = S[shared_leg_start_index:(before_index+1)]
shared_leg.append(v_s)
after_index = before_index + 1
# Either v_s is the end of the previous shared segment, or after_index or something following that,
# so go until you find a point on S not on R, starting at after_index
pt = Point(S[after_index], srid=4326)
pt.transform(900913)
while pt.distance(R_trans) < FUZZY_DIST:
shared_leg.append(S[after_index])
after_index = after_index + 1
# should check that shared_leg is not just first element. In fact, TODO: go back an check what happens
# if the first element is the only shared element.
shared_leg_list.append(LineString(shared_leg))
return shared_leg_list
shared_leg = []
def save_into_database(rib_path, rmb_path, putdict, sewerdict, rmberrors):
# Get sewerage name, try to create sewerage
# If it exists, return with an error
sewerage_name = os.path.basename(rmb_path)[:-4] # Minus ".RMB"
if models.Sewerage.objects.filter(name=sewerage_name).exists():
rmberrors.append(Error(
line_number=0,
message=("Er bestaat al een stelsel met de naam {name}. "
"Verwijder het op de archiefpagina, of gebruik "
"een andere naam.").format(name=sewerage_name)))
return
# Files are copied only at the end
sewerage = models.Sewerage.objects.create(
name=sewerage_name,
rib=None, # Filled in later
rmb=None,
active=True)
# Save the puts, keep a dictionary
saved_puts = dict()
for put_id, putinfo in putdict.items():
saved_puts[put_id] = models.Manhole.objects.create(
sewerage=sewerage,
code=put_id,
sink=int(putinfo['is_sink']),
ground_level=putinfo['surface_level'],
the_geom=Point(*putinfo['coordinate']))
# Save the sewers, use the dictionary
saved_sewers = dict()
for sewer_id, sewerinfo in sewerdict.items():
manhole1 = saved_puts[sewerinfo['manhole_code_1']]
manhole2 = saved_puts[sewerinfo['manhole_code_2']]
sewer_line_rd = LineString(manhole1.the_geom, manhole2.the_geom)
sewer_line_rd.set_srid(4326)
sewer_line_rd.transform(RD)
saved_sewers[sewer_id] = models.Sewer.objects.create(
sewerage=sewerage,
code=sewer_id,
quality=models.Sewer.QUALITY_UNKNOWN,
diameter=sewerinfo['diameter'],
manhole1=manhole1,
manhole2=manhole2,
bob1=sewerinfo['bob_1'],
bob2=sewerinfo['bob_2'],
the_geom=LineString(manhole1.the_geom, manhole2.the_geom),
the_geom_length=sewer_line_rd.length)
# Save the measurements
sewer_measurements_dict = dict()
for sewer_id, sewerinfo in sewerdict.items():
measurements = sewerinfo['measurements']
sewer = saved_sewers[sewer_id]
if measurements:
sewer_measurements = [
# Create the SewerMeasurement objects, but don't save
# them yet!
models.SewerMeasurement(
sewer=sewer,
dist=m['dist'],
virtual=False,
water_level=None,
flooded_pct=None,
bob=m['bob'],
obb=m['bob'] + sewerinfo['diameter'],
the_geom=Point(*m['coordinate']))
for m in measurements]
# Quality
sewer.judge_quality(sewer_measurements)
sewer.save()
# BOB correction ("sawtooth" phenomenon)
correct_bob_values(sewer, sewer_measurements)
# Create two virtual sewer measurements for the start and
# end of the sewer
virtual_start = models.SewerMeasurement(
sewer=sewer, dist=0, virtual=True, water_level=None,
flooded_pct=None, bob=sewer.bob1,
obb=sewer.bob1 + sewerinfo['diameter'],
the_geom=sewer.manhole1.the_geom)
virtual_end = models.SewerMeasurement(
sewer=sewer, dist=sewer.the_geom_length,
virtual=True, water_level=None,
flooded_pct=None, bob=sewer.bob2,
obb=sewer.bob2 + sewerinfo['diameter'],
the_geom=sewer.manhole2.the_geom)
# Note: we MUST add those two virtual points only after
# doing the sawtooth correction, otherwise the sawtooth
# correction will think that everything is fine already
# since the first and end points would be equal to the
# bobs of the sewer...
sewer_measurements = (
[virtual_start] + sewer_measurements + [virtual_end])
sewer_measurements_dict[sewer_id] = sewer_measurements
else:
# Create "virtual measurements"
sewer_measurements_dict[sewer_id] = list(
virtual_measurements(sewer))
sewer.quality = models.Sewer.QUALITY_UNKNOWN
sewer.save()
# Actually compute the lost capacity, the point of this app
lost_capacity.compute_lost_capacity(
saved_puts, saved_sewers, sewer_measurements_dict)
# Save all the SewerMeasurement objects to the database. Since
# there are thousands of them, it is essential to use bulk_create.
models.SewerMeasurement.objects.bulk_create(list(chain(
*sewer_measurements_dict.values())))
# Success -- copy files
sewerage.move_files(rib_path, rmb_path)
# The clap on the fireworks
sewerage.generate_rib()