class LinearReferencingTestCase(unittest.TestCase):
def setUp(self):
self.point = Point(1, 1)
self.line1 = LineString(([0, 0], [2, 0]))
self.line2 = LineString(([3, 0], [3, 6]))
self.multiline = MultiLineString(
[list(self.line1.coords),
list(self.line2.coords)])
def test_line1_project(self):
self.assertEqual(self.line1.project(self.point), 1.0)
self.assertEqual(self.line1.project(self.point, normalized=True), 0.5)
def test_line2_project(self):
self.assertEqual(self.line2.project(self.point), 1.0)
self.assertAlmostEqual(self.line2.project(self.point, normalized=True),
0.16666666666, 8)
def test_multiline_project(self):
self.assertEqual(self.multiline.project(self.point), 1.0)
self.assertEqual(self.multiline.project(self.point, normalized=True),
0.125)
def test_not_supported_project(self):
with pytest.raises(shapely.GEOSException,
match="IllegalArgumentException"):
self.point.buffer(1.0).project(self.point)
def test_not_on_line_project(self):
# Points that aren't on the line project to 0.
self.assertEqual(self.line1.project(Point(-10, -10)), 0.0)
def test_line1_interpolate(self):
self.assertTrue(self.line1.interpolate(0.5).equals(Point(0.5, 0.0)))
self.assertTrue(self.line1.interpolate(-0.5).equals(Point(1.5, 0.0)))
self.assertTrue(
self.line1.interpolate(0.5, normalized=True).equals(Point(1, 0)))
self.assertTrue(
self.line1.interpolate(-0.5, normalized=True).equals(Point(1, 0)))
def test_line2_interpolate(self):
self.assertTrue(self.line2.interpolate(0.5).equals(Point(3.0, 0.5)))
self.assertTrue(
self.line2.interpolate(0.5, normalized=True).equals(Point(3, 3)))
def test_multiline_interpolate(self):
self.assertTrue(self.multiline.interpolate(0.5).equals(Point(0.5, 0)))
self.assertTrue(
self.multiline.interpolate(0.5,
normalized=True).equals(Point(3.0,
2.0)))
def test_line_ends_interpolate(self):
# Distances greater than length of the line or less than
# zero yield the line's ends.
self.assertTrue(self.line1.interpolate(-1000).equals(Point(0.0, 0.0)))
self.assertTrue(self.line1.interpolate(1000).equals(Point(2.0, 0.0)))
class LinearReferencingTestCase(unittest.TestCase):
def setUp(self):
self.point = Point(1, 1)
self.line1 = LineString(([0, 0], [2, 0]))
self.line2 = LineString(([3, 0], [3, 6]))
self.multiline = MultiLineString([
list(self.line1.coords), list(self.line2.coords)
])
def test_line1_project(self):
self.assertEqual(self.line1.project(self.point), 1.0)
self.assertEqual(self.line1.project(self.point, normalized=True), 0.5)
def test_line2_project(self):
self.assertEqual(self.line2.project(self.point), 1.0)
self.assertAlmostEqual(self.line2.project(self.point, normalized=True),
0.16666666666, 8)
def test_multiline_project(self):
self.assertEqual(self.multiline.project(self.point), 1.0)
self.assertEqual(self.multiline.project(self.point, normalized=True),
0.125)
def test_not_supported_project(self):
self.assertRaises(TypeError, self.point.buffer(1.0).project,
self.point)
def test_not_on_line_project(self):
# Points that aren't on the line project to 0.
self.assertEqual(self.line1.project(Point(-10,-10)), 0.0)
def test_line1_interpolate(self):
self.failUnless(self.line1.interpolate(0.5).equals(Point(0.5, 0.0)))
self.failUnless(
self.line1.interpolate(0.5, normalized=True).equals(
Point(1.0, 0.0)))
def test_line2_interpolate(self):
self.failUnless(self.line2.interpolate(0.5).equals(Point(3.0, 0.5)))
self.failUnless(
self.line2.interpolate(0.5, normalized=True).equals(
Point(3.0, 3.0)))
def test_multiline_interpolate(self):
self.failUnless(self.multiline.interpolate(0.5).equals(
Point(0.5, 0.0)))
self.failUnless(
self.multiline.interpolate(0.5, normalized=True).equals(
Point(3.0, 2.0)))
def test_line_ends_interpolate(self):
# Distances greater than length of the line or less than
# zero yield the line's ends.
self.failUnless(self.line1.interpolate(-1000).equals(Point(0.0, 0.0)))
self.failUnless(self.line1.interpolate(1000).equals(Point(2.0, 0.0)))
def get_center_point(segment):
"""
Get the centerpoint for a linestring or multiline string
Args:
segment - Geojson LineString or MultiLineString
Returns:
Geojson point
"""
if segment['geometry']['type'] == 'LineString':
point = LineString(segment['geometry']['coordinates']).interpolate(
.5, normalized=True)
return point.x, point.y
elif segment['geometry']['type'] == 'MultiLineString':
# Make a rectangle around the multiline
coords = [
item for coords in segment['geometry']['coordinates']
for item in coords
]
minx = min([x[0] for x in coords])
maxx = max([x[0] for x in coords])
miny = min([x[1] for x in coords])
maxy = max([x[1] for x in coords])
point = LineString([[minx, miny], [maxx,
maxy]]).interpolate(.5,
normalized=True)
mlstring = MultiLineString(segment['geometry']['coordinates'])
point = mlstring.interpolate(mlstring.project(point))
return point.x, point.y
return None, None
class LinearReferencingTestCase(unittest.TestCase):
def setUp(self):
self.point = Point(1, 1)
self.line1 = LineString(([0, 0], [2, 0]))
self.line2 = LineString(([3, 0], [3, 6]))
self.multiline = MultiLineString([
list(self.line1.coords), list(self.line2.coords)
])
@unittest.skipIf(geos_version < (3, 2, 0), 'GEOS 3.2.0 required')
def test_line1_project(self):
self.assertEqual(self.line1.project(self.point), 1.0)
self.assertEqual(self.line1.project(self.point, normalized=True), 0.5)
@unittest.skipIf(geos_version < (3, 2, 0), 'GEOS 3.2.0 required')
def test_line2_project(self):
self.assertEqual(self.line2.project(self.point), 1.0)
self.assertAlmostEqual(
self.line2.project(self.point, normalized=True), 0.16666666666, 8)
@unittest.skipIf(geos_version < (3, 2, 0), 'GEOS 3.2.0 required')
def test_multiline_project(self):
self.assertEqual(self.multiline.project(self.point), 1.0)
self.assertEqual(
self.multiline.project(self.point, normalized=True), 0.125)
@unittest.skipIf(geos_version < (3, 2, 0), 'GEOS 3.2.0 required')
def test_not_supported_project(self):
with self.assertRaises(TypeError):
self.point.buffer(1.0).project(self.point)
@unittest.skipIf(geos_version < (3, 2, 0), 'GEOS 3.2.0 required')
def test_not_on_line_project(self):
# Points that aren't on the line project to 0.
self.assertEqual(self.line1.project(Point(-10, -10)), 0.0)
@unittest.skipIf(geos_version < (3, 2, 0), 'GEOS 3.2.0 required')
def test_line1_interpolate(self):
self.assertTrue(self.line1.interpolate(0.5).equals(Point(0.5, 0.0)))
self.assertTrue(
self.line1.interpolate(0.5, normalized=True).equals(Point(1, 0)))
@unittest.skipIf(geos_version < (3, 2, 0), 'GEOS 3.2.0 required')
def test_line2_interpolate(self):
self.assertTrue(self.line2.interpolate(0.5).equals(Point(3.0, 0.5)))
self.assertTrue(
self.line2.interpolate(0.5, normalized=True).equals(Point(3, 3)))
@unittest.skipIf(geos_version < (3, 2, 0), 'GEOS 3.2.0 required')
def test_multiline_interpolate(self):
self.assertTrue(self.multiline.interpolate(0.5).equals(Point(0.5, 0)))
self.assertTrue(
self.multiline.interpolate(0.5, normalized=True).equals(
Point(3.0, 2.0)))
@unittest.skipIf(geos_version < (3, 2, 0), 'GEOS 3.2.0 required')
def test_line_ends_interpolate(self):
# Distances greater than length of the line or less than
# zero yield the line's ends.
self.assertTrue(self.line1.interpolate(-1000).equals(Point(0.0, 0.0)))
self.assertTrue(self.line1.interpolate(1000).equals(Point(2.0, 0.0)))
class WaySet:
ways_cache = ObjectCache()
def __init__(self, lines):
self.multiline = MultiLineString(lines)
self.headings = [
Route.get_heading(l.coords[0][1], l.coords[0][0], l.coords[1][1],
l.coords[1][0]) for l in lines
]
@staticmethod
def download_all_ways(sector, tram_only=False, timestamp=None):
bbox = "%f,%f,%f,%f" % sector
ext = 0.01
ext_bbox = "%f,%f,%f,%f" % (sector[0] - ext, sector[1] - ext,
sector[2] + ext, sector[3] + ext)
if tram_only:
query = '[out:json]{{date}};(node({{ext_bbox}});way["railway"="tram"]({{bbox}}););out;'
else:
query = '[out:json]{{date}};(way["highway"]({{bbox}});node({{ext_bbox}}););out;'
query = query.replace("{{bbox}}", bbox)
query = query.replace("{{ext_bbox}}", ext_bbox)
timestamp = "[date:\"%s\"]" % timestamp if timestamp is not None else ""
query = query.replace("{{date}}", timestamp)
ways = WaySet.ways_cache.get_from_cache(query)
if ways is None:
api = overpy.Overpass()
try:
result = api.query(query)
except overpy.OverpassTooManyRequests:
time.sleep(20)
result = api.query(query)
ways = []
for w in result.ways:
try:
nodes = w.get_nodes(resolve_missing=False)
except overpy.exception.DataIncomplete:
try:
nodes = w.get_nodes(resolve_missing=True)
except overpy.exception.DataIncomplete:
print("Overpass can't resolve nodes. Skipping way.")
continue
nodes = [[float(n.lon), float(n.lat)] for n in nodes]
ways += [nodes]
WaySet.ways_cache.store_in_cache(query, ways)
lines = [LineString(c) for c in ways]
return WaySet(lines)
def snap_point_to_lines(self, point, next_point, priority_line_index):
distances = [line.distance(point) for line in self.multiline]
if priority_line_index > -1:
distances[priority_line_index] /= 2
min_distance = min(distances)
i_dist = distances.index(min_distance)
projection = self.multiline[i_dist].project(point)
next_proj = 1 if next_point is None else self.multiline[
i_dist].project(next_point)
new_point = self.multiline[i_dist].interpolate(projection)
new_next_point = self.multiline[i_dist].interpolate(next_proj)
heading = Route.get_heading(new_point.y, new_point.x, new_next_point.y,
new_next_point.x)
return new_point, i_dist, heading
def snap_point_to_multilines(self, point):
return self.multiline.interpolate(self.multiline.project(point))
def plot(self):
for l in self.multiline:
lon, lat = l.xy
plt.plot(lon, lat)
def get_closest_way(self, point):
distances = [line.distance(point) for line in self.multiline]
min_distance = min(distances)
i_min_dist = distances.index(min_distance)
return self.multiline[i_min_dist]
def get_closest_way_with_heading(self,
point,
heading,
heading_tolerance=45):
heading = heading % 360
lines = self.multiline
projections = [l.project(point, normalized=True) for l in lines]
lines = [l for l, p in zip(lines, projections) if 0 < p < 1]
headings = np.array([
WaySet.get_linestring_heading_at_projection(l, p)
for l, p in zip(lines, projections)
])
headings_diff = np.abs(headings % 360 - heading)
# print(headings_diff)
if (headings_diff < heading_tolerance).any():
lines = [
l for l, hd in zip(lines, headings_diff)
if hd < heading_tolerance
]
distances = [line.distance(point) for line in lines]
min_distance = min(distances)
i_min_dist = distances.index(min_distance)
return lines[i_min_dist]
@staticmethod
def get_linestring_heading_at_projection(linestring, projection):
ps = [
linestring.project(Point(c), normalized=True)
for c in linestring.coords
]
for prev_c, c in zip(linestring.coords[:-1], linestring.coords[1:]):
p = linestring.project(Point(c))
if p > projection:
return Route.get_heading(prev_c[1], prev_c[0], c[1], c[0])
prev_c = linestring.coords[-2]
c = linestring.coords[-1]
return Route.get_heading(prev_c[1], prev_c[0], c[1], c[0])
def snap_points(self, points):
last_way = None
snapped_points = []
last_distance = None
# headings before snapping
headings = []
for i, p in enumerate(points):
if i < len(points) - 1:
next_point = points[i + 1]
headings.append(
Route.get_heading(p.y, p.x, next_point.y, next_point.x))
headings.append(headings[-1])
h_diff = []
for point_index in range(len(points)):
point = points[point_index]
heading = headings[point_index]
# if last_way is not None:
# projection = last_way.project(point)
# distance = last_way.distance(point)
# if 0 <= projection <= 1 and distance < 1.1 * last_distance:
# snapped_point = last_way.interpolate(projection)
# last_distance = distance
# snapped_points += [snapped_point]
# continue
last_way = self.get_closest_way_with_heading(point, heading)
# last_way = self.get_closest_way(point)
projection = last_way.project(point)
# last_distance = last_way.distance(point)
snapped_point = last_way.interpolate(projection)
h = WaySet.get_linestring_heading_at_projection(
last_way, projection)
h_diff += [headings[point_index] - h]
snapped_points += [snapped_point]
# plt.plot(h_diff)
# plt.show()
# headings after snapping
headings = []
for i, p in enumerate(snapped_points):
if i < len(snapped_points) - 1:
next_point = snapped_points[i + 1]
headings.append(
Route.get_heading(p.y, p.x, next_point.y, next_point.x))
headings.append(headings[-1])
return Route.from_points(snapped_points, headings)
class match(object):
"""This object is responsible for coming up with a more spatially accurate
version of the trip. We do this by first trying to map match the GPS
track to the street/rail network using OSRM. If that doesn't work well
for any reason, we try altering some parameters to improve the match.
If it's still not great, we can see if there is a default route_geometry
provided. Ultimately, we judge whether the match is sufficent to proceed.
If we do use this match, this object also provides methods for associating
points (vehicles, stops) with points along the route gemetry; these will
be used for time interpolation inside the trip object."""
def __init__(self, trip_object):
# initialize some variables
self.trip = trip_object # trip object that this is a match for
self.geometry = MultiLineString() # MultiLineString shapely geom
self.OSRM_response = {} # python-parsed OSRM response object
self.confidence = 0 #
# error radius to use for map matching, same for all points
self.error_radius = conf['error_radius']
self.default_route_used = False
# fire off a query to OSRM with the default parameters
self.query_OSRM()
if not self.OSRM_match_is_sufficient:
# try again with a larger error radius
self.error_radius *= 1.5
self.query_OSRM()
# still no good?
if not self.OSRM_match_is_sufficient:
# Try a default geometry
if self.get_default_route():
self.locate_vehicles_on_default_route()
else:
return # bad match, no default
else: # have a workable OSRM match geometry
self.parse_OSRM_geometry()
self.locate_vehicles_on_OSRM_route()
if len(self.trip.vehicles) > 2:
self.locate_stops_on_route()
# report on what happened
self.print_outcome()
@property
def OSRM_match_is_sufficient(self):
"""Is this match good enough actually to be used?"""
return self.confidence >= conf['min_OSRM_match_quality']
@property
def is_useable(self):
"""Do we have everything we need to proceed with the match?"""
if not (self.OSRM_match_is_sufficient or self.default_route_used):
return False
if not len(self.trip.vehicles) > 3:
return False
if self.trip.vehicles[0].measure == self.trip.vehicles[-1].measure:
return False
if not len(self.trip.timepoints) > 1:
return False
# and only if we make it past all those conditions:
return True
def query_OSRM(self):
"""Construct the request and send it to OSRM, retrying if necessary."""
# structure it as API requires, rounding coords to 6 decimals
coords = ';'.join([
format(v.lon, '.7g') + ',' + format(v.lat, '.7g')
for v in self.trip.vehicles
])
radii = ';'.join([str(self.error_radius)] * len(self.trip.vehicles))
# construct and send the request
options = {
'radiuses': radii,
'steps': 'false',
'geometries': 'geojson',
'annotations': 'false',
'overview': 'full',
'gaps':
'ignore', # don't split based on time gaps - shouldn't be any
'tidy': 'true',
'generate_hints': 'false'
}
# open a connection, configured to retry in case of errors
with requests.Session() as session:
retries = Retry(total=5, backoff_factor=1)
session.mount('http://', HTTPAdapter(max_retries=retries))
# make the request
try:
raw_response = session.get(
conf['OSRMserver']['url'] + '/match/v1/transit/' + coords,
params=options,
timeout=conf['OSRMserver']['timeout'])
except:
return db.ignore_trip(self.trip.trip_id, 'connection issue')
# parse the result to a python object
self.OSRM_response = json.loads(raw_response.text)
# how confident should we be in this response?
if self.OSRM_response['code'] != 'Ok':
self.confidence = 0
return
else:
# Get an average confidence value from the match result.
confidence_values = [
m['confidence'] for m in self.OSRM_response['matchings']
]
self.confidence = mean(confidence_values)
def parse_OSRM_geometry(self):
"""Parse the OSRM match geometry into a more useable format.
Specifically a simplified and projected MultiLineString."""
# get a list of lists of lat-lon coords which need to be reprojected
lines = [
asShape(matching['geometry'])
for matching in self.OSRM_response['matchings']
]
multilines = MultiLineString(lines)
# reproject to local
local_multilines = reproject(conf['projection'], multilines)
# simplify slightly for speed (2 meter simplification)
simple_local_multilines = local_multilines.simplify(2)
# if the multi actually just had one line, this simplifies to a
# linestring, which can cause problems down the road
if simple_local_multilines.geom_type == 'LineString':
simple_local_multilines = MultiLineString(
[simple_local_multilines])
self.geometry = simple_local_multilines
def get_default_route(self):
"""Check if a default route geometry is available; if so, we'll need to
parse things into the same format, just as though this had come from
OSRM."""
# get the default if there is one
route_geom = db.get_route_geom(self.trip.direction_id,
self.trip.last_seen)
if route_geom: # default available
self.default_route_used = True
self.confidence = 1
self.geometry = MultiLineString([route_geom])
return True
else: # no default
return False
def print_outcome(self):
"""Print the outcome of this match to stdout."""
if self.default_route_used and self.confidence == 1:
print('\tdefault route used for direction', self.trip.direction_id)
elif self.default_route_used and self.confidence == 0:
print('\tdefault route not found for', self.trip.direction_id)
elif not self.default_route_used and self.confidence > conf[
'min_OSRM_match_quality']:
print('\tOSRM match found with', round(self.confidence, 3),
'confidence')
else:
print('\tmatching failed for trip', self.trip.trip_id)
# Below are functions associated with finding the measure of points along
# the route geometry, either as given by OSRM or provided as the default.
# These are called from inside the trip if the match is useable.
def locate_vehicles_on_OSRM_route(self):
"""Find the measure of vehicles along the OSRM-supplied route. This is
easy because OSRM provides the distance of an input coordinate along the
match geometry."""
assert not self.default_route_used
# these are the matched points of the input cordinates
# null (None) entries indicate an omitted (outlier) point
# true where not none
drop_list = [
point is None for point in self.OSRM_response['tracepoints']
]
# drop vehicles that did not contribute to the match,
# backwards to maintain order
for i in reversed(range(0, len(drop_list))):
if drop_list[i]: self.trip.ignore_vehicle(i)
# get cumulative distances of each vehicle along the match geom
# This is based on the leg distances provided by OSRM. Each leg is just
# the trip between matched points. Each match has one more vehicle record
# associated with it than legs
cummulative_distance = 0
v_i = 0
for matching in self.OSRM_response['matchings']:
# the first point is at 0 per match
self.trip.vehicles[v_i].set_measure(cummulative_distance)
v_i += 1
for leg in matching['legs']:
cummulative_distance += leg['distance']
self.trip.vehicles[v_i].set_measure(cummulative_distance)
v_i += 1
# Because the line has been simplified, the distances will be
# slightly off and need correcting
adjust_factor = self.geometry.length / self.trip.vehicles[-1].measure
for v in self.trip.vehicles:
v.measure = v.measure * adjust_factor
def locate_vehicles_on_default_route(self):
"""Find the measure of vehicles along the default route. First discard
observations too far from the route geometry. Next, find the measure of
the remaining vehicles in the order they were observed. If the vehicles
progress monotonically down the line then all is good. Otherwise, we
start dropping observations that are most severely out of order until we
are left with an ordered list moving along the route in the correct
direction. Wrong direction travel will generally result in a minimal
ordered set: 1 remaining observation."""
assert self.default_route_used
# match stops within a distance of the route geometry
vehicles_to_ignore = []
for vehicle in self.trip.vehicles:
# if the vehicle is close enough
distance_from_route = self.geometry.distance(vehicle.geom)
if distance_from_route <= conf['stop_dist']:
m = self.geometry.project(vehicle.geom)
vehicle.set_measure(m)
else:
vehicles_to_ignore.append(vehicle)
for vehicle in vehicles_to_ignore:
self.trip.ignore_vehicle(vehicle)
# while the list is not fully sorted
while self.trip.vehicles != sorted(self.trip.vehicles,
key=lambda v: v.measure):
correct_order = sorted(self.trip.vehicles, key=lambda v: v.measure)
current_order = self.trip.vehicles
transpositions = {}
# compare all vehicles in both lists
for i, v1 in enumerate(correct_order):
for j, v2 in enumerate(current_order):
if v1 == v2:
if abs(i - j) > 0: # not in the same position
# add these vehicles to the list with their distances as keys
if abs(i - j) not in transpositions:
transpositions[abs(i - j)] = [v1]
else:
transpositions[abs(i - j)].append(v1)
else: # are in the same position
continue
max_dist = max(transpositions.keys())
# ignore vehicles associated with the max of the transposition distances
for vehicle in transpositions[max_dist]:
self.trip.ignore_vehicle(vehicle)
# now we either have a sorted list or an essentially empty list if the
# match happened to be bad
def locate_stops_on_route(self):
"""Find the measure of stops along the route geometry for any arbitrary
route. Stops must be within a given distance of the path, but can
repeat if the route passes a stop two or more times. To check for this,
the geometry is sliced up into segments and we check just a portion
of the route at a time."""
assert len(self.trip.stops) > 0
assert self.geometry.length > 0
# list of timepoints
potential_timepoints = []
# copy the geometry so we can slice it up it
path = copy(self.geometry)
traversed = 0
# while there is more than 750m of path remaining
while path.length > 0:
subpath, path = cut(path, 750)
# check for nearby stops
for stop in self.trip.stops:
# if the stop is close enough
stop_dist = subpath.distance(stop.geom)
if stop_dist <= conf['stop_dist']:
# measure how far it is along the trip
m = traversed + subpath.project(stop.geom)
# add it to the list of measures
potential_timepoints.append(TimePoint(stop, m, stop_dist))
# note what we have already traversed
traversed += 750
# Now some of these will be duplicates that are close to the cutpoint
# and thus are added twice with similar measures
# such points need to be removed
final_timepoints = []
for pt in potential_timepoints:
skip_this_timepoint = False
for ft in final_timepoints:
# if same stop and very close
if pt.stop_id == ft.stop_id and abs(
pt.measure - ft.measure) < 2 * conf['stop_dist']:
#choose the closer of the two to use
if ft.dist <= pt.dist:
skip_this_timepoint = True
break
else:
ft = pt
skip_this_timepoint = True
break
if not skip_this_timepoint:
# we didn't have anything like that in the final set yet
final_timepoints.append(pt)
# add terminal stops if they are anywhere near the GPS data
# but not used yet
if not self.default_route_used:
# for first and last stops
for terminal_stop in [self.trip.stops[0], self.trip.stops[-1]]:
if not terminal_stop.id in [
t.stop.id for t in potential_timepoints
]:
# if the terminal stop is less than 500m away from the route
dist = self.geometry.distance(terminal_stop.geom)
if dist < 500:
m = self.geometry.project(terminal_stop.geom)
final_timepoints.append(
TimePoint(
terminal_stop, m -
dist if m < self.geometry.length / 2 else m +
dist, dist))
# for default geometries on the other hand, remove stops that are nowhere
# near the actual GPS data
else:
final_timepoints = [
t for t in final_timepoints
if t.measure > self.trip.vehicles[0].measure -
500 and t.measure < self.trip.vehicles[-1].measure + 500
]
# sort by measure ascending
final_timepoints = sorted(final_timepoints,
key=lambda timepoint: timepoint.measure)
self.trip.timepoints = final_timepoints
class match(object):
"""This object is responsible for coming up with a more spatially accurate
version of the trip. We do this by first trying to map match the GPS
track to the street/rail network using OSRM. If that doesn't work well
for any reason, we try altering some parameters to improve the match.
If it's still not great, we can see if there is a default route_geometry
provided. Ultimately, we judge whether the match is sufficent to proceed.
If we do use this match, this object also provides methods for associating
points (vehicles, stops) with points along the route gemetry; these will
be used for time interpolation inside the trip object."""
def __init__(self,trip_object):
# initialize some variables
self.trip = trip_object # trip object that this is a match for
self.geometry = MultiLineString() # MultiLineString shapely geom
self.OSRM_response = {} # python-parsed OSRM response object
self.confidence = 0 #
# error radius to use for map matching, same for all points
self.error_radius = conf['error_radius']
self.default_route_used = False;
# fire off a query to OSRM with the default parameters
self.query_OSRM()
if not self.OSRM_match_is_sufficient:
# try again with a larger error radius
self.error_radius *= 1.5
self.query_OSRM()
# still no good?
if not self.OSRM_match_is_sufficient:
# Try a default geometry
if self.get_default_route():
self.locate_vehicles_on_default_route()
else:
return # bad match, no default
else: # have a workable OSRM match geometry
self.parse_OSRM_geometry()
self.locate_vehicles_on_OSRM_route()
if len(self.trip.vehicles) > 2:
self.locate_stops_on_route()
# report on what happened
self.print_outcome()
@property
def OSRM_match_is_sufficient(self):
"""Is this match good enough actually to be used?"""
return self.confidence >= conf['min_OSRM_match_quality']
@property
def is_useable(self):
"""Do we have everything we need to proceed with the match?"""
if not (self.OSRM_match_is_sufficient or self.default_route_used):
return False
if not len(self.trip.vehicles) > 3:
return False
if self.trip.vehicles[0].measure == self.trip.vehicles[-1].measure:
return False
if not len(self.trip.timepoints) > 1:
return False
# and only if we make it past all those conditions:
return True
def query_OSRM(self):
"""Construct the request and send it to OSRM, retrying if necessary."""
# structure it as API requires, rounding coords to 6 decimals
coords = ';'.join( [
format(v.lon,'.7g')+','+format(v.lat,'.7g') for v in self.trip.vehicles
] )
radii = ';'.join( [ str(self.error_radius) ] * len(self.trip.vehicles) )
# construct and send the request
options = {
'radiuses':radii,
'steps':'false',
'geometries':'geojson',
'annotations':'false',
'overview':'full',
'gaps':'ignore', # don't split based on time gaps - shouldn't be any
'tidy':'true',
'generate_hints':'false'
}
# open a connection, configured to retry in case of errors
with requests.Session() as session:
retries = Retry( total=5, backoff_factor=1 )
session.mount( 'http://', HTTPAdapter(max_retries=retries) )
# make the request
try:
raw_response = session.get(
conf['OSRMserver']['url']+'/match/v1/transit/'+coords,
params=options,
timeout=conf['OSRMserver']['timeout']
)
except:
return db.ignore_trip(self.trip.trip_id,'connection issue')
# parse the result to a python object
self.OSRM_response = json.loads(raw_response.text)
# how confident should we be in this response?
if self.OSRM_response['code'] != 'Ok':
self.confidence = 0
return
else:
# Get an average confidence value from the match result.
confidence_values = [ m['confidence'] for m in self.OSRM_response['matchings'] ]
self.confidence = mean( confidence_values )
def parse_OSRM_geometry(self):
"""Parse the OSRM match geometry into a more useable format.
Specifically a simplified and projected MultiLineString."""
# get a list of lists of lat-lon coords which need to be reprojected
lines = [asShape(matching['geometry']) for matching in self.OSRM_response['matchings']]
multilines = MultiLineString(lines)
# reproject to local
local_multilines = reproject( conf['projection'], multilines )
# simplify slightly for speed (2 meter simplification)
simple_local_multilines = local_multilines.simplify(2)
# if the multi actually just had one line, this simplifies to a
# linestring, which can cause problems down the road
if simple_local_multilines.geom_type == 'LineString':
simple_local_multilines = MultiLineString([simple_local_multilines])
self.geometry = simple_local_multilines
def get_default_route(self):
"""Check if a default route geometry is available; if so, we'll need to
parse things into the same format, just as though this had come from
OSRM."""
# get the default if there is one
route_geom = db.get_route_geom( self.trip.direction_id, self.trip.last_seen )
if route_geom: # default available
self.default_route_used = True
self.confidence = 1
self.geometry = MultiLineString([route_geom])
return True
else: # no default
return False
def print_outcome(self):
"""Print the outcome of this match to stdout."""
if self.default_route_used and self.confidence == 1:
print( '\tdefault route used for direction',self.trip.direction_id )
elif self.default_route_used and self.confidence == 0:
print( '\tdefault route not found for',self.trip.direction_id )
elif not self.default_route_used and self.confidence > conf['min_OSRM_match_quality']:
print( '\tOSRM match found with',round(self.confidence,3),'confidence' )
else:
print( '\tmatching failed for trip',self.trip.trip_id )
# Below are functions associated with finding the measure of points along
# the route geometry, either as given by OSRM or provided as the default.
# These are called from inside the trip if the match is useable.
def locate_vehicles_on_OSRM_route(self):
"""Find the measure of vehicles along the OSRM-supplied route. This is
easy because OSRM provides the distance of an input coordinate along the
match geometry."""
assert not self.default_route_used
# these are the matched points of the input cordinates
# null (None) entries indicate an omitted (outlier) point
# true where not none
drop_list = [ point is None for point in self.OSRM_response['tracepoints'] ]
# drop vehicles that did not contribute to the match,
# backwards to maintain order
for i in reversed( range( 0, len(drop_list) ) ):
if drop_list[i]: self.trip.ignore_vehicle( i )
# get cumulative distances of each vehicle along the match geom
# This is based on the leg distances provided by OSRM. Each leg is just
# the trip between matched points. Each match has one more vehicle record
# associated with it than legs
cummulative_distance = 0
v_i = 0
for matching in self.OSRM_response['matchings']:
# the first point is at 0 per match
self.trip.vehicles[v_i].set_measure( cummulative_distance )
v_i += 1
for leg in matching['legs']:
cummulative_distance += leg['distance']
self.trip.vehicles[v_i].set_measure( cummulative_distance )
v_i += 1
# Because the line has been simplified, the distances will be
# slightly off and need correcting
adjust_factor = self.geometry.length / self.trip.vehicles[-1].measure
for v in self.trip.vehicles:
v.measure = v.measure * adjust_factor
def locate_vehicles_on_default_route(self):
"""Find the measure of vehicles along the default route. First discard
observations too far from the route geometry. Next, find the measure of
the remaining vehicles in the order they were observed. If the vehicles
progress monotonically down the line then all is good. Otherwise, we
start dropping observations that are most severely out of order until we
are left with an ordered list moving along the route in the correct
direction. Wrong direction travel will generally result in a minimal
ordered set: 1 remaining observation."""
assert self.default_route_used
# match stops within a distance of the route geometry
vehicles_to_ignore = []
for vehicle in self.trip.vehicles:
# if the vehicle is close enough
distance_from_route = self.geometry.distance( vehicle.geom )
if distance_from_route <= conf['stop_dist']:
m = self.geometry.project(vehicle.geom)
vehicle.set_measure(m)
else:
vehicles_to_ignore.append(vehicle)
for vehicle in vehicles_to_ignore:
self.trip.ignore_vehicle( vehicle )
# while the list is not fully sorted
while self.trip.vehicles != sorted(self.trip.vehicles,key=lambda v: v.measure):
correct_order = sorted(self.trip.vehicles,key=lambda v: v.measure)
current_order = self.trip.vehicles
transpositions = {}
# compare all vehicles in both lists
for i,v1 in enumerate(correct_order):
for j,v2 in enumerate(current_order):
if v1 == v2:
if abs(i-j) > 0: # not in the same position
# add these vehicles to the list with their distances as keys
if abs(i-j) not in transpositions: transpositions[abs(i-j)] = [v1]
else: transpositions[abs(i-j)].append(v1)
else: # are in the same position
continue
max_dist = max(transpositions.keys())
# ignore vehicles associated with the max of the transposition distances
for vehicle in transpositions[max_dist]:
self.trip.ignore_vehicle(vehicle)
# now we either have a sorted list or an essentially empty list if the
# match happened to be bad
def locate_stops_on_route(self):
"""Find the measure of stops along the route geometry for any arbitrary
route. Stops must be within a given distance of the path, but can
repeat if the route passes a stop two or more times. To check for this,
the geometry is sliced up into segments and we check just a portion
of the route at a time."""
assert len(self.trip.stops) > 0
assert self.geometry.length > 0
# list of timepoints
potential_timepoints = []
# copy the geometry so we can slice it up it
path = copy(self.geometry)
traversed = 0
# while there is more than 750m of path remaining
while path.length > 0:
subpath, path = cut(path,750)
# check for nearby stops
for stop in self.trip.stops:
# if the stop is close enough
stop_dist = subpath.distance(stop.geom)
if stop_dist <= conf['stop_dist']:
# measure how far it is along the trip
m = traversed + subpath.project(stop.geom)
# add it to the list of measures
potential_timepoints.append( TimePoint(stop,m,stop_dist) )
# note what we have already traversed
traversed += 750
# Now some of these will be duplicates that are close to the cutpoint
# and thus are added twice with similar measures
# such points need to be removed
final_timepoints = []
for pt in potential_timepoints:
skip_this_timepoint = False
for ft in final_timepoints:
# if same stop and very close
if pt.stop_id == ft.stop_id and abs(pt.measure-ft.measure) < 2*conf['stop_dist']:
#choose the closer of the two to use
if ft.dist <= pt.dist:
skip_this_timepoint = True
break
else:
ft = pt
skip_this_timepoint = True
break
if not skip_this_timepoint:
# we didn't have anything like that in the final set yet
final_timepoints.append( pt )
# add terminal stops if they are anywhere near the GPS data
# but not used yet
if not self.default_route_used:
# for first and last stops
for terminal_stop in [self.trip.stops[0],self.trip.stops[-1]]:
if not terminal_stop.id in [ t.stop.id for t in potential_timepoints ]:
# if the terminal stop is less than 500m away from the route
dist = self.geometry.distance(terminal_stop.geom)
if dist < 500:
m = self.geometry.project(terminal_stop.geom)
final_timepoints.append( TimePoint(
terminal_stop,
m-dist if m < self.geometry.length/2 else m+dist,
dist
) )
# for default geometries on the other hand, remove stops that are nowhere
# near the actual GPS data
else:
final_timepoints = [
t for t in final_timepoints if
t.measure > self.trip.vehicles[0].measure - 500 and
t.measure < self.trip.vehicles[-1].measure + 500
]
# sort by measure ascending
final_timepoints = sorted(final_timepoints,key=lambda timepoint: timepoint.measure)
self.trip.timepoints = final_timepoints
