def _read_geos(self): ''' Reads the geometries from the shapefile and dumps them into a string :returns:the id and geometries of the links in a string ''' irecord = self.shapefile_reader.iterRecords() ishapes = self.shapefile_reader.iterShapes() # read entries into a dictionary which has ids mapped to geometries id_to_geometry = list() for i in range(self.shapefile_reader.numRecords): record = irecord.next() shape = ishapes.next() points = [tuple(x) for x in shape.points] id = int(record[0]) orig_id = int(record[7]) line = LineString(points) line.set_srid(config.EPSG32611) defaultprojection = line.clone() defaultprojection.transform(config.canonical_projection) googleprojection = line.clone() googleprojection.transform(config.google_projection) id_to_geometry.append('\t'.join([str(i), str(id), defaultprojection.ewkt, googleprojection.ewkt, line.ewkt, str(orig_id)])) return '\n'.join(id_to_geometry)
def _read_geos(self): ''' Reads the geometries from the shapefile and dumps them into a string :returns:the id and geometries of the links in a string ''' irecord = self.shapefile_reader.iterRecords() ishapes = self.shapefile_reader.iterShapes() # read entries into a dictionary which has ids mapped to geometries id_to_geometry = list() for i in range(self.shapefile_reader.numRecords): record = irecord.next() shape = ishapes.next() points = [tuple(x) for x in shape.points] id = int(record[0]) orig_id = int(record[7]) line = LineString(points) line.set_srid(config.EPSG32611) defaultprojection = line.clone() defaultprojection.transform(config.canonical_projection) googleprojection = line.clone() googleprojection.transform(config.google_projection) id_to_geometry.append('\t'.join([ str(i), str(id), defaultprojection.ewkt, googleprojection.ewkt, line.ewkt, str(orig_id) ])) return '\n'.join(id_to_geometry)
def save_into_database(rib_path, rmb_path, putdict, sewerdict, rmberrors): # Get sewerage name, try to create sewerage # If it exists, return with an error sewerage_name = os.path.basename(rmb_path)[:-4] # Minus ".RMB" if models.Sewerage.objects.filter(name=sewerage_name).exists(): rmberrors.append( Error(line_number=0, message=("Er bestaat al een stelsel met de naam {name}. " "Verwijder het op de archiefpagina, of gebruik " "een andere naam.").format(name=sewerage_name))) return # Files are copied only at the end sewerage = models.Sewerage.objects.create( name=sewerage_name, rib=None, # Filled in later rmb=None, active=True) # Save the puts, keep a dictionary saved_puts = dict() for put_id, putinfo in putdict.items(): saved_puts[put_id] = models.Manhole.objects.create( sewerage=sewerage, code=put_id, sink=int(putinfo['is_sink']), ground_level=putinfo['surface_level'], the_geom=Point(*putinfo['coordinate'])) # Save the sewers, use the dictionary saved_sewers = dict() for sewer_id, sewerinfo in sewerdict.items(): manhole1 = saved_puts[sewerinfo['manhole_code_1']] manhole2 = saved_puts[sewerinfo['manhole_code_2']] sewer_line_rd = LineString(manhole1.the_geom, manhole2.the_geom) sewer_line_rd.set_srid(4326) sewer_line_rd.transform(RD) saved_sewers[sewer_id] = models.Sewer.objects.create( sewerage=sewerage, code=sewer_id, quality=models.Sewer.QUALITY_UNKNOWN, diameter=sewerinfo['diameter'], manhole1=manhole1, manhole2=manhole2, bob1=sewerinfo['bob_1'], bob2=sewerinfo['bob_2'], the_geom=LineString(manhole1.the_geom, manhole2.the_geom), the_geom_length=sewer_line_rd.length) # Save the measurements sewer_measurements_dict = dict() for sewer_id, sewerinfo in sewerdict.items(): measurements = sewerinfo['measurements'] sewer = saved_sewers[sewer_id] if measurements: sewer_measurements = [ # Create the SewerMeasurement objects, but don't save # them yet! models.SewerMeasurement(sewer=sewer, dist=m['dist'], virtual=False, water_level=None, flooded_pct=None, bob=m['bob'], obb=m['bob'] + sewerinfo['diameter'], the_geom=Point(*m['coordinate'])) for m in measurements ] # Quality sewer.judge_quality(sewer_measurements) sewer.save() # BOB correction ("sawtooth" phenomenon) correct_bob_values(sewer, sewer_measurements) # Create two virtual sewer measurements for the start and # end of the sewer virtual_start = models.SewerMeasurement( sewer=sewer, dist=0, virtual=True, water_level=None, flooded_pct=None, bob=sewer.bob1, obb=sewer.bob1 + sewerinfo['diameter'], the_geom=sewer.manhole1.the_geom) virtual_end = models.SewerMeasurement( sewer=sewer, dist=sewer.the_geom_length, virtual=True, water_level=None, flooded_pct=None, bob=sewer.bob2, obb=sewer.bob2 + sewerinfo['diameter'], the_geom=sewer.manhole2.the_geom) # Note: we MUST add those two virtual points only after # doing the sawtooth correction, otherwise the sawtooth # correction will think that everything is fine already # since the first and end points would be equal to the # bobs of the sewer... sewer_measurements = ([virtual_start] + sewer_measurements + [virtual_end]) sewer_measurements_dict[sewer_id] = sewer_measurements else: # Create "virtual measurements" sewer_measurements_dict[sewer_id] = list( virtual_measurements(sewer)) sewer.quality = models.Sewer.QUALITY_UNKNOWN sewer.save() # Actually compute the lost capacity, the point of this app lost_capacity.compute_lost_capacity(saved_puts, saved_sewers, sewer_measurements_dict) # Save all the SewerMeasurement objects to the database. Since # there are thousands of them, it is essential to use bulk_create. models.SewerMeasurement.objects.bulk_create( list(chain(*sewer_measurements_dict.values()))) # Success -- copy files sewerage.move_files(rib_path, rmb_path) # The clap on the fireworks sewerage.generate_rib()
def find_shared_segments(R, S): """Return a list of LineStrings where two routes overlap Keyword arguments: route_a -- LineString route route_b -- LineString route This method should be commutative. Algorithmic idea: For a given set of routes that are not identical: - Go through each route (R and S), and make a list, P(*) for each that contains points not on the other route. - Starting with P(R), iterate through elements p_i = P(R)_i: - Find p_i in original polyline, R. - Find the vertex of the item before p_i in R, v_s. v_s should be on the other route S. - Starting from this element in R, move forward through route until you find the first vertex that is on S, call this v_f. Remove elements not on S from P(R), including p_i - Starting from v_s, find closest vertex on S, call it g. This could be before or after where v_s intersects the route. - Move along S in both directions from g, taking ordered pairs where order is index in S, and identify the first pair where the first element is on R, but the second element is not. - If the first element of this tuple is after g, then that element is end of a shared leg. - Otherwise, v_s is the end of that shared leg. - Starting from v_f, find closest vertex on S, call it g, this could be before or after where v_f intersects the route. - Move along S in both directions from g, taking ordered pairs where order is index in S, and identify the first pair where the first element is not on R, but the second element is. - If the second element of this tuple is before g, then that element is the start of a shared leg. - Otherwise, v_f is the start of that shared leg. - At this point, we have a list of starting points and ending points of shared legs, combining them in to a polyline from the two routes seems tractable. TODO: figure this out when brain is less fried. """ R_points_not_on_S = [] S_points_not_on_R = [] S_trans = S.clone() S_trans.set_srid(4326) S_trans.transform(900913) for pt in R: place = Point(pt) place.set_srid(4326) place.transform(900913) if place.distance(S_trans) > FUZZY_DIST: R_points_not_on_S.append(pt) #TODO: refactor these into single function call S_trans = S.clone() S_trans.set_srid(4326) S_trans.transform(900913) R_trans = R.clone() R_trans.set_srid(4326) R_trans.transform(900913) for pt in S: place = Point(pt, srid=4326) place.transform(900913) if place.distance(R_trans) > FUZZY_DIST: S_points_not_on_R.append(pt) # we know they start at the same point shared_leg_list = [] shared_leg = [] shared_leg_start_index = 0 n = len(R_points_not_on_S) while n > 0: p_i = R_points_not_on_S[0] j = R.index(p_i) v_s = R[j - 1] f = j v_f = p_i while v_f in R_points_not_on_S: idx = R_points_not_on_S.index(v_f) del R_points_not_on_S[idx] n = n - 1 f = f + 1 v_f = R[f] # We know v_f is fuzzy-on S, so we can iterate through pairs of S and find the segment it is fuzzy-on before_index = 0 for i, start_vertex in enumerate(S): if i == len(S): break end_vertex = S[i + 1] line = LineString([start_vertex, end_vertex]) line.set_srid(4326) line.transform(900913) pt = Point(v_s) pt.set_srid(4326) pt.transform(900913) if pt.distance(line) < FUZZY_DIST: before_index = i break # At this point, we know shared_leg_start_index..before_index is certainly on the path. shared_leg = S[shared_leg_start_index:(before_index + 1)] shared_leg.append(v_s) after_index = before_index + 1 # Either v_s is the end of the previous shared segment, or after_index or something following that, # so go until you find a point on S not on R, starting at after_index pt = Point(S[after_index], srid=4326) pt.transform(900913) while pt.distance(R_trans) < FUZZY_DIST: shared_leg.append(S[after_index]) after_index = after_index + 1 # should check that shared_leg is not just first element. In fact, TODO: go back an check what happens # if the first element is the only shared element. shared_leg_list.append(LineString(shared_leg)) return shared_leg_list shared_leg = []
def find_shared_segments(R, S): """Return a list of LineStrings where two routes overlap Keyword arguments: route_a -- LineString route route_b -- LineString route This method should be commutative. Algorithmic idea: For a given set of routes that are not identical: - Go through each route (R and S), and make a list, P(*) for each that contains points not on the other route. - Starting with P(R), iterate through elements p_i = P(R)_i: - Find p_i in original polyline, R. - Find the vertex of the item before p_i in R, v_s. v_s should be on the other route S. - Starting from this element in R, move forward through route until you find the first vertex that is on S, call this v_f. Remove elements not on S from P(R), including p_i - Starting from v_s, find closest vertex on S, call it g. This could be before or after where v_s intersects the route. - Move along S in both directions from g, taking ordered pairs where order is index in S, and identify the first pair where the first element is on R, but the second element is not. - If the first element of this tuple is after g, then that element is end of a shared leg. - Otherwise, v_s is the end of that shared leg. - Starting from v_f, find closest vertex on S, call it g, this could be before or after where v_f intersects the route. - Move along S in both directions from g, taking ordered pairs where order is index in S, and identify the first pair where the first element is not on R, but the second element is. - If the second element of this tuple is before g, then that element is the start of a shared leg. - Otherwise, v_f is the start of that shared leg. - At this point, we have a list of starting points and ending points of shared legs, combining them in to a polyline from the two routes seems tractable. TODO: figure this out when brain is less fried. """ R_points_not_on_S = [] S_points_not_on_R = [] S_trans = S.clone() S_trans.set_srid(4326) S_trans.transform(900913) for pt in R: place = Point(pt) place.set_srid(4326) place.transform(900913) if place.distance(S_trans) > FUZZY_DIST: R_points_not_on_S.append(pt) #TODO: refactor these into single function call S_trans = S.clone() S_trans.set_srid(4326) S_trans.transform(900913) R_trans = R.clone() R_trans.set_srid(4326) R_trans.transform(900913) for pt in S: place = Point(pt, srid=4326) place.transform(900913) if place.distance(R_trans) > FUZZY_DIST: S_points_not_on_R.append(pt) # we know they start at the same point shared_leg_list = [] shared_leg = [] shared_leg_start_index = 0 n = len(R_points_not_on_S) while n > 0: p_i = R_points_not_on_S[0] j = R.index(p_i) v_s = R[j - 1] f = j v_f = p_i while v_f in R_points_not_on_S: idx = R_points_not_on_S.index(v_f) del R_points_not_on_S[idx] n = n - 1 f = f + 1 v_f = R[f] # We know v_f is fuzzy-on S, so we can iterate through pairs of S and find the segment it is fuzzy-on before_index = 0 for i, start_vertex in enumerate(S): if i == len(S): break end_vertex = S[i + 1] line = LineString([start_vertex, end_vertex]) line.set_srid(4326) line.transform(900913) pt = Point(v_s) pt.set_srid(4326) pt.transform(900913) if pt.distance(line) < FUZZY_DIST: before_index = i break # At this point, we know shared_leg_start_index..before_index is certainly on the path. shared_leg = S[shared_leg_start_index:(before_index+1)] shared_leg.append(v_s) after_index = before_index + 1 # Either v_s is the end of the previous shared segment, or after_index or something following that, # so go until you find a point on S not on R, starting at after_index pt = Point(S[after_index], srid=4326) pt.transform(900913) while pt.distance(R_trans) < FUZZY_DIST: shared_leg.append(S[after_index]) after_index = after_index + 1 # should check that shared_leg is not just first element. In fact, TODO: go back an check what happens # if the first element is the only shared element. shared_leg_list.append(LineString(shared_leg)) return shared_leg_list shared_leg = []
def save_into_database(rib_path, rmb_path, putdict, sewerdict, rmberrors): # Get sewerage name, try to create sewerage # If it exists, return with an error sewerage_name = os.path.basename(rmb_path)[:-4] # Minus ".RMB" if models.Sewerage.objects.filter(name=sewerage_name).exists(): rmberrors.append(Error( line_number=0, message=("Er bestaat al een stelsel met de naam {name}. " "Verwijder het op de archiefpagina, of gebruik " "een andere naam.").format(name=sewerage_name))) return # Files are copied only at the end sewerage = models.Sewerage.objects.create( name=sewerage_name, rib=None, # Filled in later rmb=None, active=True) # Save the puts, keep a dictionary saved_puts = dict() for put_id, putinfo in putdict.items(): saved_puts[put_id] = models.Manhole.objects.create( sewerage=sewerage, code=put_id, sink=int(putinfo['is_sink']), ground_level=putinfo['surface_level'], the_geom=Point(*putinfo['coordinate'])) # Save the sewers, use the dictionary saved_sewers = dict() for sewer_id, sewerinfo in sewerdict.items(): manhole1 = saved_puts[sewerinfo['manhole_code_1']] manhole2 = saved_puts[sewerinfo['manhole_code_2']] sewer_line_rd = LineString(manhole1.the_geom, manhole2.the_geom) sewer_line_rd.set_srid(4326) sewer_line_rd.transform(RD) saved_sewers[sewer_id] = models.Sewer.objects.create( sewerage=sewerage, code=sewer_id, quality=models.Sewer.QUALITY_UNKNOWN, diameter=sewerinfo['diameter'], manhole1=manhole1, manhole2=manhole2, bob1=sewerinfo['bob_1'], bob2=sewerinfo['bob_2'], the_geom=LineString(manhole1.the_geom, manhole2.the_geom), the_geom_length=sewer_line_rd.length) # Save the measurements sewer_measurements_dict = dict() for sewer_id, sewerinfo in sewerdict.items(): measurements = sewerinfo['measurements'] sewer = saved_sewers[sewer_id] if measurements: sewer_measurements = [ # Create the SewerMeasurement objects, but don't save # them yet! models.SewerMeasurement( sewer=sewer, dist=m['dist'], virtual=False, water_level=None, flooded_pct=None, bob=m['bob'], obb=m['bob'] + sewerinfo['diameter'], the_geom=Point(*m['coordinate'])) for m in measurements] # Quality sewer.judge_quality(sewer_measurements) sewer.save() # BOB correction ("sawtooth" phenomenon) correct_bob_values(sewer, sewer_measurements) # Create two virtual sewer measurements for the start and # end of the sewer virtual_start = models.SewerMeasurement( sewer=sewer, dist=0, virtual=True, water_level=None, flooded_pct=None, bob=sewer.bob1, obb=sewer.bob1 + sewerinfo['diameter'], the_geom=sewer.manhole1.the_geom) virtual_end = models.SewerMeasurement( sewer=sewer, dist=sewer.the_geom_length, virtual=True, water_level=None, flooded_pct=None, bob=sewer.bob2, obb=sewer.bob2 + sewerinfo['diameter'], the_geom=sewer.manhole2.the_geom) # Note: we MUST add those two virtual points only after # doing the sawtooth correction, otherwise the sawtooth # correction will think that everything is fine already # since the first and end points would be equal to the # bobs of the sewer... sewer_measurements = ( [virtual_start] + sewer_measurements + [virtual_end]) sewer_measurements_dict[sewer_id] = sewer_measurements else: # Create "virtual measurements" sewer_measurements_dict[sewer_id] = list( virtual_measurements(sewer)) sewer.quality = models.Sewer.QUALITY_UNKNOWN sewer.save() # Actually compute the lost capacity, the point of this app lost_capacity.compute_lost_capacity( saved_puts, saved_sewers, sewer_measurements_dict) # Save all the SewerMeasurement objects to the database. Since # there are thousands of them, it is essential to use bulk_create. models.SewerMeasurement.objects.bulk_create(list(chain( *sewer_measurements_dict.values()))) # Success -- copy files sewerage.move_files(rib_path, rmb_path) # The clap on the fireworks sewerage.generate_rib()