def phi_generation_sql(self):
origins = defaultdict(dict)
with server_side_cursors(connection):
cursor = connection.cursor()
count = PhiGenerator.route_count()
gen_tt = ConsoleProgress(count, message="Computing Phi")
sql_query = """
SELECT r.orig_taz, r.dest_taz, r.od_route_index,
array(
SELECT (SELECT vector_index FROM orm_experimentsensor es WHERE es.sensor_id = s.id LIMIT 1)
FROM orm_sensor s
WHERE ST_Distance(r.geom_dist, s.location_dist) < 10 AND s.road_type ='Freeway'
) AS sensors
FROM filtered_routes r
WHERE r.od_route_index < %(num_routes)s
"""
cursor.execute(sql_query, {'num_routes':self.num_routes})
for row in cursor:
gen_tt.increment_progress()
o, d, rt, rs = row
origins[(o, d)][rt] = rs
gen_tt.finish()
return origins
def data(self):
if self.info:
return self.info
else:
load_phi_progress = ConsoleProgress(1, message="Loading phi")
if generate_phi:
# TODO: replace with experiment id
self.info = generate_phi.phi_generation_sql(1)
else:
self.info = pickle.load(open(self.__class__.data_prefix+'/phi.pickle'))
load_phi_progress.finish()
return self.info
def data(self):
if self.info:
return self.info
else:
load_phi_progress = ConsoleProgress(1, message="Loading phi")
if generate_phi:
# TODO: replace with experiment id
self.info = generate_phi.phi_generation_sql(1)
else:
self.info = pickle.load(
open(self.__class__.data_prefix + '/phi.pickle'))
load_phi_progress.finish()
return self.info
def __init__(self, compute_trip_counts=False):
self.flow = None
if compute_trip_counts:
raise NotImplementedError
else:
self.rad, TAZ = np.zeros((self.N_TAZ,self.N_TAZ)), np.zeros(self.N_TAZ)
load_radiation_progress = ConsoleProgress(self.N_TAZ*self.N_TAZ, message="Loading radiation model heuristic")
with open(self.data_prefix+'/trips.csv') as file:
reader = csv.reader(file,delimiter=',')
firstline = file.readline() # skip the first line
for prog, row in enumerate(reader):
self.rad[int(row[2]),int(row[3])] = int(float(row[6]))
load_radiation_progress.update_progress(prog)
load_radiation_progress.finish()
def import_routes():
taz_lookup = pickle.load(open(data_prefix+'/lookup.pickle'))
def compute_route_time(route):
travel_time = 0
for leg in route['legs']:
travel_time += leg['duration']['value']
return travel_time
def getRoutesAndSave(o, d):
data = json.load(open(data_prefix+'/data/%s_%s.json' % (o, d)))
for route_index, route in enumerate(data['routes']):
gpolyline = route['overview_polyline']['points']
linestring = google_lines.decode_line(gpolyline)
linestring.set_srid(canonical_projection)
linestring_dist = linestring.clone()
linestring_dist.transform(google_projection)
route_object = Route(geom=linestring, geom_dist=linestring_dist, \
summary=route['summary'], origin_taz=taz_lookup[o], \
destination_taz=taz_lookup[d], \
travel_time=compute_route_time(route), \
od_route_index=route_index, \
json_contents=json.dumps(route))
route_object.save()
# Get list of origins
files = os.listdir(data_prefix+'/data')
origins = {}
for file in files:
file = file.replace('.json', '')
o, d = map(int, file.split('_'))
if not o in origins:
origins[o] = {}
if not d in origins[o]:
origins[o][d] = []
# Load all routes from origins
gen_tt = ConsoleProgress(N_TAZ*(N_TAZ-1), message="Saving to database.")
for index_o, o in enumerate(origins):
for index_d, d in enumerate(origins[o]):
getRoutesAndSave(o, d)
gen_tt.increment_progress()
gen_tt.finish()
def phi_generation_sql(self):
origins = defaultdict(dict)
with server_side_cursors(connection):
cursor = connection.cursor()
count = PhiGenerator.route_count()
gen_tt = ConsoleProgress(count, message="Computing Phi")
sql_query = """
SELECT r.orig_taz, r.dest_taz, r.od_route_index,
array(
SELECT (SELECT vector_index FROM orm_experimentsensor es WHERE es.sensor_id = s.id LIMIT 1)
FROM orm_sensor s
WHERE ST_Distance(r.geom_dist, s.location_dist) < 10 AND s.road_type ='Freeway'
) AS sensors
FROM filtered_routes r
WHERE r.od_route_index < %(num_routes)s
"""
cursor.execute(sql_query, {'num_routes': self.num_routes})
for row in cursor:
gen_tt.increment_progress()
o, d, rt, rs = row
origins[(o, d)][rt] = rs
gen_tt.finish()
return origins
def main():
parser = argparse.ArgumentParser(description='Solve Tomography problem with radiation model.')
parser.add_argument('--verbose', dest='verbose',
const=True, default=False, action='store_const',
help='Show verbose output (default: silent)')
parser.add_argument('--data-prefix', dest='prefix', nargs='?', const='data',
default='.', help='Set prefix for data files (default: .)')
parser.add_argument('--log', dest='log', nargs='?', const='INFO',
default='WARN', help='Set log level (default: WARN)')
args = parser.parse_args()
if args.verbose:
logging.basicConfig(level=logging.DEBUG)
if args.log in ACCEPTED_LOG_LEVELS:
logging.basicConfig(level=eval('logging.'+args.log))
if args.prefix[-1] == '/':
data_prefix = args.prefix[:-1]
else:
data_prefix = args.prefix
script_progress = ConsoleProgress(N_TAZ, args.verbose, message='Processing travel times for routes')
with open(data_prefix+'/travel_times.csv', 'wb') as csvfile:
ttwriter = csv.writer(csvfile, delimiter=',')
ttwriter.writerow(['origin_index','destination_index','route_index','travel_time'])
for o in xrange(N_TAZ):
for d in xrange(N_TAZ):
if o == d:
continue
routes_for_od_pair = od_travel_time(data_prefix, o, d)
for route_index, travel_time in enumerate(routes_for_od_pair):
ttwriter.writerow([o, d, route_index, travel_time])
script_progress.update_progress(o+1)
script_progress.finish()
def phi_generation_sql(experiment_id):
origins = defaultdict(dict)
with server_side_cursors(connection):
cursor = connection.cursor()
gen_tt = ConsoleProgress(N_ROUTES, message="Computing Phi")
sql_query = """
SELECT om.matrix_id, dm.matrix_id, r.od_route_index,
array(
SELECT (SELECT vector_index FROM orm_experimentsensor es WHERE es.sensor_id = s.id AND es.experiment_id = %s LIMIT 1)
FROM orm_sensor s
WHERE ST_Distance(r.geom_dist, s.location_dist) < 10
) AS sensors
FROM orm_route r, orm_matrixtaz om, orm_matrixtaz dm
WHERE r.origin_taz = om.taz_id AND r.destination_taz = dm.taz_id
"""
cursor.execute(sql_query, (experiment_id,))
for row in cursor:
gen_tt.increment_progress()
o, d, rt, rs = row
origins[(o, d)][rt] = rs
gen_tt.finish()
return origins
def generate_od_travel_time_pairs(self):
gen_tt = ConsoleProgress(self.N_TAZ, message="Loading travel times")
od_pair_matrix = [[{} for x in range(self.N_TAZ)] for y in range(self.N_TAZ)]
with open(self.__class__.data_prefix+'/travel_times.csv') as fopen:
reader = csv.reader(fopen,delimiter=',')
# skip reading first line
firstline = fopen.readline()
for row in reader:
od_pair_matrix[int(row[0])][int(row[1])][int(row[2])] = float(int(row[3]))
gen_tt.update_progress(int(row[0]))
gen_tt.finish()
return od_pair_matrix
def __init__(self, compute=True, phi=None, condensed_map=None, generate_phi=None, use_travel_times=False):
if compute:
# Load the data.
data_progress = ConsoleProgress(1, message="Loading phi")
self.condensed_map = condensed_map
# TODO: replace with experiment id
data = generate_phi.phi_generation_sql(1)
phi.set_data(data)
data_progress.finish()
self.generate_routing_matrix(data, use_travel_times)
sio.savemat(self.__class__.data_prefix+'/X_matrix.mat', {
'X' : self.X,
'x' : self.x,
'U' : self.U
})
else:
x_load_progress = ConsoleProgress(1, message="Loading X matrix from file")
loaded_data = sio.loadmat(self.__class__.data_prefix+'/X_matrix.mat')
self.X = loaded_data['X']
self.U = loaded_data['U']
self.x = loaded_data['x']
x_load_progress.finish()
self.phi = phi
def __init__(self, compute_trip_counts=False):
self.flow = None
if compute_trip_counts:
raise NotImplementedError
else:
self.rad, TAZ = np.zeros(
(self.N_TAZ, self.N_TAZ)), np.zeros(self.N_TAZ)
load_radiation_progress = ConsoleProgress(
self.N_TAZ * self.N_TAZ,
message="Loading radiation model heuristic")
with open(self.data_prefix + '/trips.csv') as file:
reader = csv.reader(file, delimiter=',')
firstline = file.readline() # skip the first line
for prog, row in enumerate(reader):
self.rad[int(row[2]), int(row[3])] = int(float(row[6]))
load_radiation_progress.update_progress(prog)
load_radiation_progress.finish()
def import_routes():
taz_lookup = pickle.load(open(data_prefix+'/lookup.pickle'))
def compute_route_time(route):
travel_time = 0
for leg in route['legs']:
travel_time += leg['duration']['value']
return travel_time
def getRoutesAndSave(o, d):
data = json.load(open(data_prefix+'/data/%s_%s.json' % (o, d)))
for route_index, route in enumerate(data['routes']):
gpolyline = route['overview_polyline']['points']
linestring = google_lines.decode_line(gpolyline)
linestring.set_srid(canonical_projection)
linestring_dist = linestring.clone()
linestring_dist.transform(google_projection)
route_object = Route(geom=linestring, geom_dist=linestring_dist, \
summary=route['summary'], origin_taz=taz_lookup[o], \
destination_taz=taz_lookup[d], \
travel_time=compute_route_time(route), \
od_route_index=route_index, \
json_contents=json.dumps(route))
route_object.save()
# Get list of origins
files = os.listdir(data_prefix+'/data')
origins = {}
for file in files:
file = file.replace('.json', '')
o, d = map(int, file.split('_'))
if not o in origins:
origins[o] = {}
if not d in origins[o]:
origins[o][d] = []
# Load all routes from origins
gen_tt = ConsoleProgress(N_TAZ*(N_TAZ-1), message="Saving to database.")
for index_o, o in enumerate(origins):
for index_d, d in enumerate(origins[o]):
getRoutesAndSave(o, d)
gen_tt.increment_progress()
gen_tt.finish()
def phi_generation_sql(experiment_id):
origins = defaultdict(dict)
with server_side_cursors(connection):
cursor = connection.cursor()
gen_tt = ConsoleProgress(N_ROUTES, message="Computing Phi")
sql_query = """
SELECT om.matrix_id, dm.matrix_id, r.od_route_index,
array(
SELECT (SELECT vector_index FROM orm_experimentsensor es WHERE es.sensor_id = s.id AND es.experiment_id = %s LIMIT 1)
FROM orm_sensor s
WHERE ST_Distance(r.geom_dist, s.location_dist) < 10
) AS sensors
FROM orm_route r, orm_matrixtaz om, orm_matrixtaz dm
WHERE r.origin_taz = om.taz_id AND r.destination_taz = dm.taz_id
"""
cursor.execute(sql_query, (experiment_id, ))
for row in cursor:
gen_tt.increment_progress()
o, d, rt, rs = row
origins[(o, d)][rt] = rs
gen_tt.finish()
return origins
def generate_routing_matrix(self, data, use_travel_times):
"""
Given the route index associated with each OD pair, generate a routing matrix.
"""
self.X = lil_matrix((self.N_SENSORS, self.N_TAZ*self.N_TAZ))
self.x = np.zeros(self.N_ROUTES_CONDENSED)
x_ind = 0
if use_travel_times:
od_pair_travel_times = self.generate_od_travel_time_pairs()
x_gen_progress = ConsoleProgress(self.N_ROUTES, message="Generating X and U matrices")
self.U = lil_matrix((self.N_TAZ_CONDENSED*(self.N_TAZ_CONDENSED-1), self.N_ROUTES_CONDENSED))
# For efficiency, the if statement is surrounding these loops so it doesn't check every iteration
if use_travel_times:
for i in np.arange(self.N_TAZ):
for j in np.arange(self.N_TAZ):
if data[i].get(j):
if data[i][j]:
travel_times = od_pair_travel_times[i][j]
mean_tt = np.mean(travel_times.values())
std_tt = np.std(travel_times.values())
if std_tt == 0:
std_tt = 1
travel_times = {rt : (float(tt-mean_tt) / std_tt) for rt, tt in travel_times.items()}
travel_times = {rt : sigmoid(-tt) for rt, tt in travel_times.items()}
normalizer = float(sum(travel_times.values()))
travel_times = {rt : float(tt)/normalizer for rt, tt in travel_times.items()}
for route, sensors in enumerate(data[i][j]):
tt = travel_times[route]
for s in sensors:
self.X[s,i*self.N_TAZ+j] += tt
if i in self.condensed_map and j in self.condensed_map:
od_back_map[x_ind] = (i, j)
i_ind = self.condensed_map[i]
j_ind = self.condensed_map[j]
self.x[x_ind] = tt
row_index = i_ind*(self.N_TAZ_CONDENSED-1)+j_ind
if j_ind > i_ind:
row_index -= 1
self.U[row_index, x_ind] = 1
x_ind = x_ind + 1
x_gen_progress.increment_progress()
else:
for i in np.arange(self.N_TAZ):
for j in np.arange(self.N_TAZ):
if data[i].get(j):
if data[i][j]:
for route, sensors in enumerate(data[i][j]):
if route == self.FIRST_ROUTE:
for s in sensors:
self.X[s,i*self.N_TAZ+j] = 1
self.x[x_ind] = 1
if i in self.condensed_map and j in self.condensed_map:
od_back_map[x_ind] = (i, j)
i_ind = self.condensed_map[i]
j_ind = self.condensed_map[j]
row_index = i_ind*(self.N_TAZ_CONDENSED-1)+j_ind
if j_ind > i_ind:
row_index -= 1
self.U[row_index, x_ind] = 1
x_ind += 1
x_gen_progress.increment_progress()
pickle.dump(od_back_map, open(self.__class__.data_prefix+'/od_back_map.pickle', 'wb'))
x_gen_progress.finish()
import math
import logging
logging.basicConfig(level=logging.DEBUG)
data_prefix = ""
N_TAZ = 321
N_TAZ_CONDENSED = 150
N_ROUTES = 280691
N_ROUTES_CONDENSED = 60394
N_SENSORS = 1033
FIRST_ROUTE = 0
data = pickle.load(open('data/phi.pickle'))
phi_db = pickle.load(open('data/phi_condensed1402867630.8_db.pickle'))['phi']
phi_errors = 0
x_gen_progress = ConsoleProgress(N_TAZ * N_TAZ, message="Comparing phi")
for i in np.arange(N_TAZ):
for j in np.arange(N_TAZ):
if data[i].get(j):
if data[i][j]:
if data[i][j] != map(lambda x: sorted(x[1]), phi_db[(i, j)].iteritems()):
phi_errors += 1
print data[i][j]
print phi_db[(i, j)]
if phi_errors >= 10:
exit()
x_gen_progress.increment_progress()
x_gen_progress.finish()
}
origins = {}
for file in files:
file = file.replace('.json', '')
o, d = map(int, file.split('_'))
if o not in selected_origins or d not in selected_origins:
continue
if not o in origins:
origins[o] = {}
if not d in origins[o]:
origins[o][d] = []
num_routes = 0
gen_tt = ConsoleProgress(N_TAZ_TARGET, message="Computing Phi")
count = 0
for index_o, o in enumerate(origins):
for index_d, d in enumerate(origins[o]):
routes = getRoutes(o, d)
num_routes += len(routes)
for i, route in enumerate(routes):
rs = route_sensors(route)
origins[o][d].append(rs)
gen_tt.update_progress(index_o)
out.close()
gen_tt.finish()
metadata['N_ROUTES_CONDENSED'] = num_routes
metadata['N_SENSORS_USED'] = len(first_leg_sensors)
print 'Sensors used:', metadata['N_SENSORS_USED']
}
origins = {}
for file in files:
file = file.replace('.json', '')
o, d = map(int, file.split('_'))
if o not in selected_origins or d not in selected_origins:
continue
if not o in origins:
origins[o] = {}
if not d in origins[o]:
origins[o][d] = []
num_routes = 0
gen_tt = ConsoleProgress(N_TAZ_TARGET, message="Computing Phi")
count = 0
for index_o, o in enumerate(origins):
for index_d, d in enumerate(origins[o]):
routes = getRoutes(o, d)
num_routes += len(routes)
for i, route in enumerate(routes):
rs = route_sensors(route)
origins[o][d].append(rs)
gen_tt.update_progress(index_o)
out.close()
gen_tt.finish()
metadata['N_ROUTES_CONDENSED'] = num_routes
metadata['N_SENSORS_USED'] = len(first_leg_sensors)
print 'Sensors used:', metadata['N_SENSORS_USED']
# Read pre-computed trip counts for all OD pairs (simulated with radiation model) rad_model = radiation_model.RadiationModel() radflow = rad_model.as_flow() Use_Real_Sensors = args.real_sensors sensor_data = sensors.Sensors(Use_Real_Sensors, x_matrix, radflow) radflow = sensor_data.get_flow() sensors = sensor_data.sensors yescounts = sensor_data.yescounts # # wlse_tomogravity solved with sparse least squares # lsqr_progress = ConsoleProgress(5, args.verbose, message='Solving LSQR') bw = sensors - x_matrix.X*radflow Xcsr = csr_matrix(x_matrix.X) lsqr_progress.update_progress(1) tw = lsqr(Xcsr[yescounts,:], bw[yescounts], damp = 100)[0] plt.hold(True) # transform tw back to t t = radflow[:,0] + tw lsqr_progress.finish() c_lsqr = x_matrix.X*t c_rad = x_matrix.X*radflow[:,0] indexes = [i*N_TAZ_CONDENSED+j for (i, j) in itertools.combinations(condensed_map.keys(), 2)]