def setGraph(self, graph):
if isinstance(graph, str):
self._graph = Graph(graph)
else:
self._graph = graph
class BatchProcessor:
"""
Handles all of the specifics of marshalling data through the batch processor.
"""
def __init__(
self, graph=None, origins=None, routingRequest=None, profileRequest=None, cutoffMinutes=120, threads=4
):
if graph is not None:
self.setGraph(graph)
if origins is not None:
self.setOrigins(origins)
if routingRequest is not None:
self.setRoutingRequest(routingRequest)
if profileRequest is not None:
self.setProfileRequest(profileRequest)
self.cutoffMinutes = cutoffMinutes
self.threads = threads
def setOrigins(self, origins):
"Set the origins for this batch analysis request"
if isinstance(origins, str):
self._origins = PointSet(origins)
else:
self._origins = origins
def getOrigins(self):
return self._origins
def setDestinations(self, destinations):
"Set the destinations for this batch analysis request"
def getDestinations(self):
return self._destinations
def setGraph(self, graph):
if isinstance(graph, str):
self._graph = Graph(graph)
else:
self._graph = graph
def setRoutingRequest(self, routingRequest):
self._routingRequest = routingRequest
def setProfileRequest(self, profileRequest):
self._profileRequest = profileRequest
def run(self):
"""
Run this batch request. It's perfectly safe to re-run this after tweaking some parameters; indeed, this is the
recommended way to run many batch requests.
This does not return anything. To get results, see eval. The reason for making separate functions is that you
can analyze different sets of destinations without repeating the graph search.
"""
# Create a matrix to hold results
# Silly python, no rep() function
self.results = [None for i in xrange(len(self._origins))]
# TODO: threading
# we use xrange here instead of range because xrange is a generator; there's no reason to create a list of
# 10 million ints in RAM just to keep track of where we are
oLen = len(self._origins)
nextOrigin = xrange(oLen).__iter__()
startLock = Lock()
outputLock = Lock()
def processOrigins(nextOrigin, results, routingRequest, graph, origins, oLen, startLock, outputLock):
# create an spt service
sptService = EarliestArrivalSearch()
sptService.maxDuration = self.cutoffMinutes * 60
while True:
startLock.acquire()
try:
origin = nextOrigin.next()
except StopIteration:
startLock.release()
return
print "Processing origin %s of %s" % (origin + 1, oLen)
# build an SPT
options = routingRequest.clone()
# this is important
options.batch = True
orig = origins[origin]
options.setFrom(orig.getLat(), orig.getLon())
# CH: added to skip unlocated points
try:
options.setGraph(graph)
# now we should not need a lock anymore
startLock.release()
except:
print "passing exception in BatchProcessor"
startLock.release()
continue
spt = sptService.getShortestPathTree(options._routingRequest)
tsurf = TimeSurface(spt)
outputLock.acquire()
results[origin] = tsurf
outputLock.release()
# start as many threads as we want
threads = [
Thread(
target=processOrigins,
args=(
nextOrigin,
self.results,
self._routingRequest,
self._graph,
self._origins,
oLen,
startLock,
outputLock,
),
)
for i in range(self.threads)
]
for thread in threads:
thread.run()
# poll
# TODO: don't poll
while True:
if all([not thread.isAlive() for thread in threads]):
break
sleep(10)
def eval(self, destinations):
"""Produce a travel time matrix from the origins to all destinations"""
if isinstance(destinations, str):
destinations = PointSet(destinations)
mat = Matrix(len(self._origins), len(destinations))
graphId = self._graph.getId()
if not destinations._samples.has_key(graphId):
destinations.link(self._graph)
destSamples = destinations._samples[graphId]
for i in xrange(len(self._origins)):
mat.setRow(i, destSamples.eval(self.results[i]))
return mat
def setGraph(self, graph):
if isinstance(graph, str):
self._graph = Graph(graph)
else:
self._graph = graph
from opentripplanner import RoutingRequest, Graph
from opentripplanner.batch import BatchProcessor, PointSet
import time
init_time = time.time()
# Set the parameters for our search
r = RoutingRequest()
r.dateTime = 1449496800 # Monday, Dec. 7th, 2014, 7:00 am EST (doesn't matter for a walking search)
r.setModes('WALK,TRANSIT')
print 'routing request set in %g seconds' % (time.time() - init_time)
# read the graph
g = Graph('/home/clayton/otp/graphs/nyc/', 'nyc')
print 'graph read at %g seconds (cumulative time)' % (time.time() - init_time)
# read the destinations
# do this before running the batch processor, so that if there is an error reading the destinations, we don't have to wait for
# the batch processor to finish
destinations = PointSet('zcta_wac.shp')
print 'destinations read at %g seconds (cumulative time)' % (time.time() -
init_time)
# Link the destinations to the graph
# In theory this is done automatically, but the automatic linking happens after the searches have happened
# This shouldn't be a problem, but due to OTP bug #1577, it needs to happen before searches are run
destinations.link(g)
print 'destinations linked at %g seconds (cumulative time)' % (time.time() -
class BatchProcessor:
"""
Handles all of the specifics of marshalling data through the batch processor.
"""
def __init__(self,
graph=None,
origins=None,
routingRequest=None,
profileRequest=None,
cutoffMinutes=120,
threads=4):
if graph is not None:
self.setGraph(graph)
if origins is not None:
self.setOrigins(origins)
if routingRequest is not None:
self.setRoutingRequest(routingRequest)
if profileRequest is not None:
self.setProfileRequest(profileRequest)
self.cutoffMinutes = cutoffMinutes
self.threads = threads
def setOrigins(self, origins):
'Set the origins for this batch analysis request'
if isinstance(origins, str):
self._origins = PointSet(origins)
else:
self._origins = origins
def getOrigins(self):
return self._origins
def setDestinations(self, destinations):
'Set the destinations for this batch analysis request'
def getDestinations(self):
return self._destinations
def setGraph(self, graph):
if isinstance(graph, str):
self._graph = Graph(graph)
else:
self._graph = graph
def setRoutingRequest(self, routingRequest):
self._routingRequest = routingRequest
def setProfileRequest(self, profileRequest):
self._profileRequest = profileRequest
def run(self):
"""
Run this batch request. It's perfectly safe to re-run this after tweaking some parameters; indeed, this is the
recommended way to run many batch requests.
This does not return anything. To get results, see eval. The reason for making separate functions is that you
can analyze different sets of destinations without repeating the graph search.
"""
# Create a matrix to hold results
# Silly python, no rep() function
self.results = [None for i in xrange(len(self._origins))]
# TODO: threading
# we use xrange here instead of range because xrange is a generator; there's no reason to create a list of
# 10 million ints in RAM just to keep track of where we are
oLen = len(self._origins)
nextOrigin = xrange(oLen).__iter__()
startLock = Lock()
outputLock = Lock()
def processOrigins(nextOrigin, results, routingRequest, graph, origins,
oLen, startLock, outputLock):
# create an spt service
sptService = EarliestArrivalSPTService()
sptService.maxDuration = self.cutoffMinutes * 60
while True:
startLock.acquire()
try:
origin = nextOrigin.next()
except StopIteration:
startLock.release()
return
print 'Processing origin %s of %s' % (origin + 1, oLen)
# build an SPT
options = routingRequest.clone()
# this is important
options.batch = True
orig = origins[origin]
options.setFrom(orig.getLat(), orig.getLon())
options.setGraph(graph)
# now we should not need a lock anymore
startLock.release()
spt = sptService.getShortestPathTree(options._routingRequest)
tsurf = TimeSurface(spt)
outputLock.acquire()
results[origin] = tsurf
outputLock.release()
# start as many threads as we want
threads = [
Thread(target=processOrigins,
args=(nextOrigin, self.results, self._routingRequest,
self._graph, self._origins, oLen, startLock,
outputLock)) for i in range(self.threads)
]
for thread in threads:
thread.run()
# poll
# TODO: don't poll
while True:
if all([not thread.isAlive() for thread in threads]):
break
sleep(10)
def eval(self, destinations):
'''Produce a travel time matrix from the origins to all destinations'''
if isinstance(destinations, str):
destinations = PointSet(destinations)
mat = Matrix(len(self._origins), len(destinations))
graphId = self._graph.getId()
if not destinations._samples.has_key(graphId):
destinations.link(self._graph)
destSamples = destinations._samples[graphId]
for i in xrange(len(self._origins)):
mat.setRow(i, destSamples.eval(self.results[i]))
return mat
