def get_active_sim_keys(server_options, overrides):
sys.stdout.flush()
conn = db_manipulator.get_conn(server_options)
# get any open combination of sim_key and iteration
cursor_open = conn.cursor()
command = """SELECT sim_key, CAST(param_value AS INT) AS iteration FROM public.simulation_parameters
WHERE param_key = '%s' ORDER BY sim_key, iteration""" % SP.max_iteration
cursor_open.execute(command)
max_iterations = dict(cursor_open.fetchall())
command = """SELECT sim_key, CAST(param_value AS INT) AS iteration FROM public.simulation_parameters
WHERE param_key = '%s' ORDER BY sim_key, iteration""" % SP.iteration
cursor_open.execute(command)
for sim_key, iteration in cursor_open.fetchall():
sim_params = get_sim_params(conn, sim_key, overrides)
if iteration >= max_iterations[sim_key] and sim_params.get(
SP.status) is not None:
continue
# check whether the iteration is or was already running
if sim_params.get(SP.status):
assert table_exists(
conn, sim_params[SP.status]), "Status table does not exist"
command_status = "SELECT msg_type FROM public.%s WHERE sim_key = '%s' AND iteration = %s ORDER BY status_time DESC LIMIT 1" % (
sim_params[SP.status], sim_key, iteration)
cursor_open.execute(command_status)
status = cursor_open.fetchall()
if status and status[0][0] == "pending":
yield sim_key, iteration, sim_params
else:
yield sim_key, iteration, sim_params
conn.close()
def main():
options = parse_args()
conn = db_manipulator.get_conn(options)
if options.all_pairs:
params = {
SP.representatives: options.representatives,
SP.taz_table: options.taztable,
SP.trip_table_prefix: options.triptable,
SP.modes: options.modes
}
write_all_pairs(conn, 'passenger', options.departure,
options.limit_sql,
tripfile_name(ALL_PAIRS,
options.limit), params, options.seed)
else:
if options.simkey is not None:
sim_keys = dict([
(k, p)
for k, _, p in get_active_sim_keys(options, {SP.status: None})
])
if options.simkey in sim_keys:
write_trips(conn, options.simkey, options.limit_sql,
tripfile_name(options.simkey, options.limit),
sim_keys[options.simkey])
else:
print("Error: simkey '%s' not found. Available:\n%s" %
(options.simkey, list(sim_keys.keys())))
else:
# get all
for sim_key, _, params in get_active_sim_keys(options, {}):
write_trips(conn, sim_key, options.limit_sql,
tripfile_name(sim_key, options.limit), params)
conn.close()
def main():
argParser = ArgumentParser()
db_manipulator.add_db_arguments(argParser)
argParser.add_argument("-n", "--net-file",
help="specifying the net file of the scenario to use")
argParser.add_argument("-k", "--simkey", default="test",
help="simulation key to use")
argParser.add_argument("-l", "--limit", type=int,
help="maximum number of trips to retrieve")
argParser.add_argument("--representatives", default="",
help="set the route alternatives file to read representative travel times from")
argParser.add_argument("--real-trips", default="",
help="set the route file to read travel times for real trips from")
argParser.add_argument("-a", "--all-pairs",
default=False, action="store_true",
help="Only write the all pairs table")
options, args = argParser.parse_known_args()
if len(args) == 2:
aggregate_weights(args[0], [float(x) for x in args[1].split(",")])
return
conn = db_manipulator.get_conn(options)
if os.path.isfile(options.real_trips) and not options.all_pairs:
upload_trip_results(conn, options.simkey, SP.OPTIONAL, options.real_trips, options.limit)
if os.path.isfile(options.representatives):
tables = create_all_pairs(conn, options.simkey, SP.OPTIONAL)
upload_all_pairs(conn, tables, 0, 86400, "passenger", options.real_trips,
options.representatives, readNet(options.net_file), [])
conn.close()
def get_polys(server_options, table='quesadillas.zonierung_d_modell', net=None):
conn = db_manipulator.get_conn(server_options)
if conn is None:
print("Warning! No database, cannot retrieve suburbian TAZ.")
return
command = "SELECT vbz_6561, ST_ASTEXT(ST_TRANSFORM(the_geom, 4326)) FROM %s" % table
cursor = conn.cursor()
cursor.execute(command)
for row in cursor:
tazid = int(row[0])
if row[1].startswith("MULTIPOLYGON"):
multi = ast.literal_eval(row[1][12:].replace(" ", ","))
if type(multi[0]) is float:
multi = (multi,)
shapes = []
for s in multi:
if type(s[0]) is tuple:
s = s[0]
shape = []
x = None
for idx, coord in enumerate(s):
if x is None:
x = coord
else:
if net is not None:
x, coord = net.convertLonLat2XY(x, coord)
shape.append((x, coord))
x = None
shapes.append(shape)
yield tazid, shapes
else:
print("unknown shape", row)
conn.close()
def get_locations(server, table):
conn = db_manipulator.get_conn(server)
for suffix in ("start", "end"):
command = """SELECT DISTINCT taz_id_%s, lon_%s, lat_%s FROM core.%s
WHERE taz_id_%s < -1000000""" % (suffix, suffix, suffix, table, suffix)
cursor = conn.cursor()
cursor.execute(command)
for row in cursor:
yield (row[0], suffix[0]) + row[1:]
conn.close()
def main(args):
# get the options
argParser = ArgumentParser()
options = getOptions(args, argParser)
if options.clean:
shutil.rmtree(options.workdir_folder, True)
conn = db_manipulator.get_conn(options)
if conn is None:
print(
"Warning! No database connection given, deleting files only.")
else:
cursor = conn.cursor()
cursor.execute("DELETE FROM public.%s;" % (SP.OPTIONAL[SP.status]))
conn.commit()
conn.close()
return
processes = {}
now = datetime.datetime.now()
if options.daemon_run_time > 0:
daemon_end_time = now + datetime.timedelta(
seconds=options.daemon_run_time)
else:
daemon_end_time = datetime.datetime.max
while now < daemon_end_time:
# clean up finished processes
for key in list(processes.keys()):
if not processes[key].is_alive():
del processes[key]
# check for a new simulation request
for request in get_simulation_requests(options):
if request[0] in processes: # this should happen in tests only
processes[request[0]].join()
del processes[request[0]]
if len(processes) >= options.parallel:
break
processes[request[0]] = multiprocessing.Process(
target=simulation_request, args=(options, request))
processes[request[0]].start()
if not options.daemon:
break
time.sleep(2)
prev = now
now = datetime.datetime.now()
if prev.hour != now.hour:
print("still listening", now)
for p in processes.values():
p.join()
def get_simulation_requests(options):
overrides = dict(
[item.split(":") for item in options.sim_param.split(",") if item])
if options.fake_tripfile or options.sim_key:
if options.sim_key is None:
options.sim_key = DEFAULT_SIMKEY
initial_sim_params = dict(SP.OPTIONAL)
else:
conn = db_manipulator.get_conn(options)
initial_sim_params = get_trips.get_sim_params(
conn, options.sim_key, overrides)
conn.close()
if initial_sim_params is None:
return []
if options.iteration is None:
destination_path = os.path.join(
options.workdir_folder,
overrides.get(SP.destination, SP.OPTIONAL[SP.destination]))
if os.path.isdir(destination_path):
files_folders = common.listdir_skip_hidden(destination_path)
existing_iterations = [
int(ff[9:]) for ff in files_folders
if ff[:9] == 'iteration'
]
iterations = [max([-1] + existing_iterations) + 1]
else:
iterations = [0]
else:
if ":" in options.iteration:
its = options.iteration.split(":")
iterations = range(int(its[0]), int(its[1]))
else:
iterations = [int(options.iteration)]
simulation_request_list = []
for i in iterations:
sim_params = dict(initial_sim_params)
sim_params.update({
SP.net_param: options.net_param,
SP.iteration: str(i),
SP.max_iteration: str(max(iterations) + 1)
})
sim_params.update(overrides)
simulation_request_list.append((options.sim_key, i, sim_params))
return simulation_request_list
return get_trips.get_active_sim_keys(options, overrides)
def simulation_request(options, request):
conn = None
sim_key, iteration, params = request
try:
# connect to the database (prod-system)
conn = db_manipulator.get_conn(options)
if conn is None:
print(
"Warning! No database connection given, operating on files only."
)
write_status("> Begin", sim_key, params, conn,
constants.MSG_TYPE.started)
print(sorted(params.items()))
# create the destination dir if it's not already existing (not the first iteration)
scenario_basedir = create_new_destination_folder(
options, sim_key, iteration, params)
options.net_file = os.path.abspath(
os.path.join(scenario_basedir, 'net.net.xml'))
if not os.path.exists(options.net_file):
options.net_file += ".gz"
if options.taz_file is None and params.get(
'DB_TABLE_TAZ') == 'berlin_taz_1223':
# just a hack to have a good taz file for the new scenarios
options.taz_file = os.path.abspath(
os.path.join(scenario_basedir, 'Berlin_1223.taz.xml'))
options.bidi_taz_file = os.path.abspath(
os.path.join(scenario_basedir, 'bidi.taz.xml'))
options.tapas_trips = os.path.join(scenario_basedir,
"background_traffic.csv")
options.modes = params[SP.modes].replace(";", ",")
if iteration == 0 and params[
SP.add_traffic_table] and conn is not None:
get_trips.write_background_trips(conn,
params[SP.add_traffic_table],
options.limit,
options.tapas_trips, params)
options.location_priority_file = os.path.abspath(
os.path.join(scenario_basedir, 'location_priorities.xml'))
get_motorway_access.save_locations(options.location_priority_file,
options,
params[SP.add_traffic_table])
options.trips_dir = scenario_basedir
write_status(
'>> starting trip generation for background traffic using tripfile %s'
% options.tapas_trips, sim_key, params, conn)
options.background_trips, _ = t2s.main(options)
options.location_priority_file = ""
else:
options.background_trips = t2s.getSumoTripfileName(
scenario_basedir, options.tapas_trips)
# create a new iteration folder
options.iteration_dir = create_new_iteration_folder(
options, iteration, scenario_basedir)
write_status(">> created dir: %s " % options.iteration_dir, sim_key,
params, conn)
options.trips_dir = os.path.join(options.iteration_dir, 'trips')
if not os.path.exists(options.trips_dir):
os.makedirs(options.trips_dir)
# get the trip files in place - either by faking or processing db data
if options.fake_tripfile is not None:
options.tapas_trips = options.fake_tripfile
else:
assert sim_key is not None, 'no sim_key for db given'
options.tapas_trips = get_trips.tripfile_name(
sim_key, target_dir=options.trips_dir)
get_trips.write_trips(conn, sim_key, options.limit,
options.tapas_trips, params)
print()
write_status('>> starting t2s using tripfile %s' % options.tapas_trips,
sim_key, params, conn)
# run t2s
options.scale /= float(params[SP.sample])
options.script_module = get_script_module(options, params[SP.template])
final_routes, final_weights = t2s.main(options)
write_status('<< finished t2s, routes in %s' % final_routes, sim_key,
params, conn)
assert os.path.exists(
final_routes), "route file %s could not be found" % final_routes
assert os.path.exists(
final_weights
), "weight dump file %s could not be found" % final_weights
# run post processing
if iteration == int(params[
SP.max_iteration]) - 1 and options.script_module is not None:
print()
write_status('>> starting postprocessing', sim_key, params, conn)
options.script_module.post(options, params, conn, final_routes)
write_status('<< finished postprocessing', sim_key, params, conn)
if conn is not None:
print()
# upload trip results to db
write_status('>> starting trip result database upload', sim_key,
params, conn)
s2t_miv.upload_trip_results(conn, sim_key, params, final_routes)
write_status('<< finished trip result database upload', sim_key,
params, conn)
print()
print(sorted(params.items()))
run_all_pairs(options, conn, sim_key, params, final_routes,
final_weights)
cleanup([final_routes, final_weights], options.iteration_dir, sim_key,
iteration, params, conn)
write_status("< End", sim_key, params, conn,
constants.MSG_TYPE.finished)
except (AssertionError, IOError, subprocess.CalledProcessError,
t2s.MappingError) as message:
write_status(message, sim_key, params, conn, constants.MSG_TYPE.error)
except ProgrammingError as message:
conn.rollback()
write_status(message, sim_key, params, conn, constants.MSG_TYPE.error)
# exit gracefully
if conn:
conn.close()
f.write(vehicle.toXML(" "))
f.write('</routes>\n')
sumocfg = abspath_in_dir(options.iteration_dir, 'emissions.sumocfg')
with open(sumocfg, 'w') as f:
f.write("""<configuration>
<net-file value="%s"/>
<route-files value="%s"/>
<additional-files value="%s"/>
<no-step-log value="true"/>
<log-file value="emissions.sumo.log"/>
<mesosim value="%s"/>
<meso-recheck value="10"/>
<meso-multi-queue value="true"/>
<meso-junction-control.limited value="true"/>
</configuration>""" % (options.net_file, mod_routes_file, vtypes_file, meso))
return call([sumolib.checkBinary("sumo"), "-c", sumocfg])
if __name__ == "__main__":
argParser = sumolib.options.ArgumentParser()
db_manipulator.add_db_arguments(argParser)
argParser.add_argument("routes")
options = argParser.parse_args()
options.iteration_dir = "."
conn = db_manipulator.get_conn(options)
run_emission_sumo(options, SP.OPTIONAL, conn, options.routes)
conn.close()