def restart_simulation(self, sim_params, render=None):
"""Restart an already initialized simulation instance.
This is used when visualizing a rollout, in order to update the
rendering with potentially a gui and export emission data from sumo.
This is also used to handle cases when the runtime of an experiment is
too long, causing the sumo instance
Parameters
----------
sim_params : flow.core.params.SimParams
sumo-specific parameters
render: bool, optional
specifies whether to use the gui
"""
self.k.close()
# killed the sumo process if using sumo/TraCI
if self.simulator == 'traci':
self.k.simulation.sumo_proc.kill()
if render is not None:
self.sim_params.render = render
if sim_params.emission_path is not None:
ensure_dir(sim_params.emission_path)
self.sim_params.emission_path = sim_params.emission_path
self.k.scenario.generate_network(self.scenario)
self.traci_connection = self.k.simulation.start_simulation(
scenario=self.k.scenario, sim_params=self.sim_params)
self.k.pass_api(self.traci_connection)
self.setup_initial_state()
def __init__(self, net_params, base):
"""Base class for generating transportation networks.
Uses network specific features to generate the necessary xml files
needed to initialize a sumo instance. The methods of this class are
called by the base scenario class.
Attributes
----------
net_params: NetParams type
see flow/core/params.py
base: str
base name for the transportation network. If not specified in the
child class, this is also the complete name for the network.
"""
# Invoke serializable if using rllab
if Serializable is not object:
Serializable.quick_init(self, locals())
self.net_params = net_params
self.net_path = os.path.dirname(os.path.abspath(__file__)) \
+ "/debug/net/"
self.cfg_path = os.path.dirname(os.path.abspath(__file__)) \
+ "/debug/cfg/"
self.base = base
self.netfn = ""
self.vehicle_ids = []
ensure_dir("%s" % self.net_path)
ensure_dir("%s" % self.cfg_path)
# if a name was not specified by the sub-class's initialization,
# use the base as the name
if not hasattr(self, "name"):
self.name = "%s" % self.base
def train(model_params=None):
args = parse_args(sys.argv[1:])
# import relevant information from the exp_config script
module = __import__("config", fromlist=[args.exp_config])
if hasattr(module, args.exp_config):
submodule = getattr(module, args.exp_config)
else:
assert False, "Unable to find experiment config!"
flow_params = submodule.flow_params
# Path to the saved files
result_name = '{}/{}'.format(flow_params['exp_tag'],
strftime("%Y-%m-%d-%H:%M:%S"))
# Perform training.
print('Beginning training.')
model = run_model_stablebaseline(flow_params=flow_params,
args=args,
model_params=model_params)
# Save the model to a desired folder and then delete it to demonstrate loading.
print('Saving the trained model!')
path = os.path.realpath(os.path.expanduser('~/baseline_results'))
ensure_dir(path)
save_path = os.path.join(path, result_name)
model.save(save_path)
# dump the flow params
with open(os.path.join(path, result_name) + '.json', 'w') as outfile:
json.dump(flow_params,
outfile,
cls=FlowParamsEncoder,
sort_keys=True,
indent=4)
def restart_sumo(self, sumo_params, sumo_binary=None):
"""Restarts an already initialized sumo instance.
This is used when visualizing a rollout, in order to update the
sumo_binary with potentially a gui and export emission data from sumo.
This is also used to handle cases when the runtime of an experiment is
too long, causing the sumo instance
Parameters
----------
sumo_params: SumoParams type
sumo-specific parameters
sumo_binary: str, optional
specifies whether to use sumo's gui
"""
self.traci_connection.close(False)
self.sumo_proc.kill()
if sumo_binary is not None:
self.sumo_params.sumo_binary = sumo_binary
if sumo_params.emission_path is not None:
ensure_dir(sumo_params.emission_path)
self.sumo_params.emission_path = sumo_params.emission_path
self.start_sumo()
self.setup_initial_state()
def __init__(self, master_kernel, sim_params):
"""Instantiate a otm network kernel.
Parameters
----------
master_kernel : flow.core.kernel.Kernel
the higher level kernel (used to call methods from other
sub-kernels)
sim_params : flow.core.params.SimParams
simulation-specific parameters
"""
super(OTMKernelNetwork, self).__init__(master_kernel, sim_params)
# # directories for the network-specific files that will be generated by
# # the `generate_network` method
self.net_path = os.path.dirname(os.path.abspath(__file__)) \
+ '/debug/otmxml/'
ensure_dir('%s' % self.net_path)
# variables to be defined during network generation
self.network = None
self.nodfn = None
# self.guifn = None
self._edges = None
self._connections = None
self._edge_list = None
self._junction_list = None
self.__max_speed = None
self.__length = None # total length
self.__non_internal_length = None # total length of non-internal edges
self.rts = None
self.cfg = None
def train_stable_baselines(submodule, flags):
"""Train policies using the PPO algorithm in stable-baselines."""
from stable_baselines3.common.vec_env import DummyVecEnv
flow_params = submodule.flow_params
# Path to the saved files
exp_tag = flow_params['exp_tag']
result_name = '{}/{}'.format(exp_tag, strftime("%Y-%m-%d-%H:%M:%S"))
# Perform training.
start_time = timeit.default_timer()
# print experiment.json information
print("=========================================")
print('Beginning training.')
print('Algorithm :', flags.algorithm)
model = run_model_stablebaseline(flow_params, flags.num_cpus,
flags.rollout_size, flags.num_steps,
flags.algorithm, flags.exp_config)
stop_time = timeit.default_timer()
run_time = stop_time - start_time
print("Training is Finished")
print("total runtime: ", run_time)
# Save the model to a desired folder and then delete it to demonstrate
# loading.
print('Saving the trained model!')
path = os.path.realpath(os.path.expanduser('~/baseline_results'))
ensure_dir(path)
save_path = os.path.join(path, result_name)
model.save(save_path)
# dump the flow params
with open(os.path.join(path, result_name) + '.json', 'w') as outfile:
json.dump(flow_params,
outfile,
cls=FlowParamsEncoder,
sort_keys=True,
indent=4)
# Replay the result by loading the model
print('Loading the trained model and testing it out!')
if flags.exp_config.lower() == "ppo":
from stable_baselines3 import PPO
model = PPO.load(save_path)
elif flags.exp_config.lower() == "ddpg":
from stable_baselines3 import DDPG
model = DDPG.load(save_path)
flow_params = get_flow_params(os.path.join(path, result_name) + '.json')
flow_params['sim'].render = True
env = env_constructor(params=flow_params, version=0)()
# The algorithms require a vectorized environment to run
eval_env = DummyVecEnv([lambda: env])
obs = eval_env.reset()
reward = 0
for _ in range(flow_params['env'].horizon):
action, _states = model.predict(obs)
obs, rewards, dones, info = eval_env.step(action)
reward += rewards
print('the final reward is {}'.format(reward))
def train_stable_baselines3(submodule, flags):
"""Train policies using the PPO algorithm in stable-baselines3."""
from stable_baselines3.common.vec_env import DummyVecEnv
from stable_baselines3 import PPO
import torch
start_time = timeit.default_timer()
flow_params = submodule.flow_params
# Path to the saved files
exp_tag = flow_params['exp_tag']
result_name = '{}/{}'.format(exp_tag, strftime("%Y-%m-%d-%H:%M:%S"))
# Perform training.
print("cuda is available: ", torch.cuda.is_available())
print('Beginning training.')
print("==========================================")
model = run_model_stablebaseline3(flow_params, flags.num_cpus,
flags.rollout_size, flags.num_steps)
# Save the model to a desired folder and then delete it to demonstrate
# loading.
print('Saving the trained model!')
path = os.path.realpath(os.path.expanduser('~/baseline_results'))
ensure_dir(path)
save_path = os.path.join(path, result_name)
model.save(save_path)
# dump the flow params
# check time for choose GPU and CPU
stop_time = timeit.default_timer()
run_time = stop_time - start_time
with open(os.path.join(path, result_name) + '.json', 'w') as outfile:
json.dump(flow_params,
outfile,
cls=FlowParamsEncoder,
sort_keys=True,
indent=4)
# Replay the result by loading the model
print('Loading the trained model and testing it out!')
model.load(save_path)
flow_params = get_flow_params(os.path.join(path, result_name) + '.json')
flow_params['sim'].render = False
flow_params['env'].horizon = 1500 # 150seconds operation
env = env_constructor(params=flow_params, version=0)()
# The algorithms require a vectorized environment to run
eval_env = DummyVecEnv([lambda: env])
obs = eval_env.reset()
reward = 0
for _ in range(flow_params['env'].horizon):
action, _states = model.predict(obs)
obs, rewards, dones, info = eval_env.step(action)
reward += rewards
print("--------------------------------------------------------")
flow_params['sim'].render = True
simulation = Experiment(flow_params)
simulation.run(num_runs=1)
print('the final reward is {}'.format(reward))
print("total run_time:", run_time, "s")
def run(self, num_runs, num_steps, rl_actions=None):
"""Run the given scenario for a set number of runs and steps per run.
Parameters
----------
num_runs : int
number of runs the experiment should perform
num_steps : int
number of steps to be performs in each run of the experiment
rl_actions : method, optional
maps states to actions to be performed by the RL agents (if
there are any)
Returns
-------
info_dict : dict
contains returns, average speed per step
"""
if rl_actions is None:
def rl_actions(*_):
return None
dir_path = './flow_density/{}'.format(self.env.scenario.orig_name)
ensure_dir(dir_path)
res = {'vels': [], 'outflows': [], 'densities': []}
for i in range(num_runs):
vel = []
outflow = []
density = []
state = self.env.reset()
for j in range(num_steps):
state, reward, done, _ = self.env.step(rl_actions(state))
vel.append(
np.mean(
self.env.k.vehicle.get_speed(
self.env.k.vehicle.get_ids())))
outflow.append(
self.env.k.vehicle.get_outflow_rate(time_span=30))
density.append(self.env.k.vehicle.num_vehicles /
self.env.k.scenario.length())
if done:
break
res['vels'] = vel
res['outflows'] = outflow
res['densities'] = density
print("Round {0}, done".format(i))
with open("{}/{}.csv".format(dir_path, i), "w") \
as outfile:
writer = csv.writer(outfile)
writer.writerow(res.keys())
writer.writerows(zip(*res.values()))
self.env.terminate()
return None
def __init__(self):
# specify path for tensorboard logging avg travel times, avg reward, no. of vehicles in network
# create it if it directory doesn't exist
root_dir = os.path.expanduser('~')
tensorboard_log_dir = root_dir + '/ray_results/real_time_metrics'
self.summaries_dir = ensure_dir(tensorboard_log_dir)
# file location to store simulation numbers csv while training
# ie. if the training is at simulation 50, the last value add the created csv file, will be 50
# this helps us keep track of the average travel times, rewards and other values at the end
# of each simulation
simulation_log_location = root_dir + '/ray_results/num_of_simulations'
ensure_dir(simulation_log_location)
# saving flag, file location and name of histogram of results
# plot and save histogram of travel time distribution at end of simulation
self.save_plots = False
self.experiment_title = "1x1_Centralized_Thesis"
hist_dir = root_dir + '/ray_results/histograms'
self.full_histogram_path = ensure_dir(hist_dir) + "/" + self.experiment_title
# logging evolution of reward during a simulation
self.log_rewards_during_iteration = False
# activate sumo actuated baseline during training for first 6 iterations
self.sumo_actuated_baseline = False
self.sumo_actuated_simulations = 6
# choose grid type for naming the tensorboard directory logging
# will be the name and title of histogram (from save_plots above) to be saved
self.grid_demand = self.experiment_title
# choose exp running, can either be "rl" or "non_rl"
self.exp = "rl"
# choose look-ahead distance can be either 43, 80, 160 or 240
self.look_ahead = 43
# log title for tensorboard and name + file path for simulation log
self.log_title = '/simulation_analysis_{}_{}_{}'.format(self.exp, self.look_ahead, self.grid_demand)
self.filename = "/iterations_{}_{}.csv".format(self.look_ahead, self.grid_demand)
self.full_path = simulation_log_location + self.filename
# RGB colors for vehicles if coloring a vehicle. We color the observed incoming
# vehicles BLUE and the observed outgoing vehicles as RED at each intersection
self.RED = (255, 0, 0)
self.BLUE = (0, 0, 255)
self.CYAN = (0, 255, 255)
# set up more tensorboard logging info
self.exp_name = self.log_title
self.writer = SummaryWriter(self.summaries_dir + self.exp_name)
self.rew_list = []
def test_trpo_runner(self):
# test run_model on figure eight 0
model = run_model(figureeight0.flow_params, 5, 5)
# test save model
ensure_dir("./baseline_results")
save_model(model, figureeight0.flow_params, "./baseline_results")
files = sorted([f for f in listdir("./baseline_results")
if isfile(join("./baseline_results", f))])
self.assertListEqual(files, ['flow_params.json', 'model.pkl'])
# delete the generated files
shutil.rmtree('./baseline_results')
def start_simulation(self, network, sim_params):
"""See parent class.
This method calls the aimsun generator to generate the network, starts
a simulation, and creates a class to communicate with the simulation
via an TCP connection.
"""
# save the simulation step size (for later use)
self.sim_step = sim_params.sim_step
self.emission_path = sim_params.emission_path
if self.emission_path is not None:
ensure_dir(self.emission_path)
return FlowAimsunAPI(port=sim_params.port)
def train_stable_baselines(submodule, flags):
"""Train policies using the PPO algorithm in stable-baselines."""
from stable_baselines.common.vec_env import DummyVecEnv
from stable_baselines import DDPG
flow_params = submodule.flow_params
# Path to the saved files
exp_tag = flow_params['exp_tag']
result_name = '{}/{}'.format(exp_tag, strftime("%Y-%m-%d-%H:%M:%S"))
# Perform training.
print('Beginning training.')
model = run_model_stablebaseline(
flow_params, flags.num_cpus, flags.rollout_size, flags.num_steps)
# Save the model to a desired folder and then delete it to demonstrate
# loading.
print('Saving the trained model!')
path = os.path.realpath(os.path.expanduser('~/baseline_results'))
ensure_dir(path)
save_path = os.path.join(path, result_name)
model.save(save_path)
# dump the flow params
with open(os.path.join(path, result_name) + '.json', 'w') as outfile:
json.dump(flow_params, outfile,
cls=FlowParamsEncoder, sort_keys=True, indent=4)
# Replay the result by loading the model
print('Loading the trained model and testing it out!')
model = DDPG.load(save_path)
flow_params = get_flow_params(os.path.join(path, result_name) + '.json')
flow_params['sim'].render = True
env = env_constructor(params=flow_params, version=0)()
n_actions = env.action_space.shape[-1]
param_noise = None
action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions), sigma=float(0.5) * np.ones(n_actions))
# The algorithms require a vectorized environment to run
eval_env = DummyVecEnv([lambda: env])
obs = eval_env.reset()
reward = 0
for _ in range(flow_params['env'].horizon):
action, _states = model.predict(obs)
obs, rewards, dones, info = eval_env.step(action)
reward += rewards
print('the final reward is {}'.format(reward))
def __init__(self, net_params, base):
Serializable.quick_init(self, locals())
self.net_params = net_params
self.net_path = net_params.net_path
self.cfg_path = net_params.cfg_path
self.base = base
self.netfn = ""
self.vehicle_ids = []
ensure_dir("%s" % self.net_path)
ensure_dir("%s" % self.cfg_path)
# if a name was not specified by the sub-class's initialization,
# use the base as the name
if not hasattr(self, "name"):
self.name = "%s" % self.base
def __init__(self, env_params, sim_params, network, simulator='traci'):
for p in ADDITIONAL_ENV_PARAMS.keys():
if p not in env_params.additional_params:
raise KeyError(
'Environment parameter \'{}\' not supplied'.format(p))
self.continuous_low_speed = 0
self.prev_pos = dict()
self.absolute_position = dict()
if sim_params.emission_path is not None:
self.path = sim_params.emission_path
if self.path[-1] != '/':
self.path += '/'
self.path += 'pics'
ensure_dir(self.path)
super().__init__(env_params, sim_params, network, simulator)
def restart_sumo(self, sumo_params, sumo_binary=None):
"""
Restarts an already initialized environment. Used when visualizing a
rollout.
"""
self.traci_connection.close(False)
if sumo_binary:
self.sumo_binary = sumo_binary
self.port = sumolib.miscutils.getFreeSocketPort()
data_folder = self.emission_path
ensure_dir(data_folder)
self.emission_out = \
data_folder + "{0}-emission.xml".format(self.scenario.name)
self.start_sumo()
self.setup_initial_state()
def __init__(self, master_kernel, sim_params):
"""Instantiate a sumo scenario kernel.
Parameters
----------
master_kernel : flow.core.kernel.Kernel
the higher level kernel (used to call methods from other
sub-kernels)
sim_params : flow.core.params.SimParams
simulation-specific parameters
"""
super(TraCIScenario, self).__init__(master_kernel, sim_params)
# directories for the network-specific files that will be generated by
# the `generate_network` method
self.net_path = os.path.dirname(os.path.abspath(__file__)) \
+ '/debug/net/'
self.cfg_path = os.path.dirname(os.path.abspath(__file__)) \
+ '/debug/cfg/'
ensure_dir('%s' % self.net_path)
ensure_dir('%s' % self.cfg_path)
# variables to be defined during network generation
self.network = None
self.nodfn = None
self.edgfn = None
self.typfn = None
self.cfgfn = None
self.netfn = None
self.confn = None
self.roufn = None
self.addfn = None
self.sumfn = None
self.guifn = None
self._edges = None
self._connections = None
self._edge_list = None
self._junction_list = None
self.__max_speed = None
self.__length = None
self.rts = None
self.cfg = None
def __init__(self, master_kernel, sim_params):
"""Instantiate a sumo network kernel.
Parameters
----------
master_kernel : flow.core.kernel.Kernel
the higher level kernel (used to call methods from other
sub-kernels)
sim_params : flow.core.params.SimParams
simulation-specific parameters
"""
super(TraCIKernelNetwork, self).__init__(master_kernel, sim_params)
# directories for the network-specific files that will be generated by
# the `generate_network` method
self.net_path = os.path.join(tempfile.gettempdir(), 'flow/debug/net/')
self.cfg_path = os.path.join(tempfile.gettempdir(), 'flow/debug/cfg/')
ensure_dir('%s' % self.net_path)
ensure_dir('%s' % self.cfg_path)
# variables to be defined during network generation
self.network = None
self.nodfn = None
self.edgfn = None
self.typfn = None
self.cfgfn = None
self.netfn = None
self.confn = None
self.roufn = None
self.addfn = None
self.sumfn = None
self.guifn = None
self._edges = None
self._connections = None
self._edge_list = None
self._junction_list = None
self.__max_speed = None
self.__length = None # total length
self.__non_internal_length = None # total length of non-internal edges
self.rts = None
self.cfg = None
def __init__(self, net_params, base):
"""Instantiate the base generator class.
Attributes
----------
net_params : NetParams type
see flow/core/params.py
base : str
base name for the transportation network. If not specified in the
child class, this is also the complete name for the network.
"""
# Invoke serializable if using rllab
if Serializable is not object:
Serializable.quick_init(self, locals())
self.net_params = net_params
self.net_path = os.path.dirname(os.path.abspath(__file__)) \
+ "/debug/net/"
self.cfg_path = os.path.dirname(os.path.abspath(__file__)) \
+ "/debug/cfg/"
self.base = base
self.vehicle_ids = []
ensure_dir("%s" % self.net_path)
ensure_dir("%s" % self.cfg_path)
# if a name was not specified by the sub-class's initialization,
# use the base as the name
if not hasattr(self, "name"):
self.name = "%s" % self.base
self.nodfn = "%s.nod.xml" % self.name
self.edgfn = "%s.edg.xml" % self.name
self.typfn = "%s.typ.xml" % self.name
self.cfgfn = "%s.netccfg" % self.name
self.netfn = "%s.net.xml" % self.name
self.confn = "%s.con.xml" % self.name
self.roufn = "%s.rou.xml" % self.name
self.addfn = "%s.add.xml" % self.name
self.sumfn = "%s.sumo.cfg" % self.name
self.guifn = "%s.gui.cfg" % self.name
# exp.run(10, 1000)
#
# # intersection environment for inflows from 400-3000
# for length in np.arange(300, 510, 10):
# env = ring_env(length, param, i)
# exp = FlowDensityExperiment(env)
# exp.run(10, 1000)
# ----------------------------------------------------------------------- #
# ---------------------- Used for data collection ----------------------- #
# ----------------------------------------------------------------------- #
#
# for cf params
for i, param in enumerate(CF_PARAMS):
# create necessary directories
ensure_dir('./data/'.format(i))
# # merge environment
# data = defaultdict(list)
# while len(data.get('speed', [])) < TOTAL_STEPS:
# inflow = random.uniform(MERGE_RANGE[i][0], MERGE_RANGE[i][1])
# env = merge_env(inflow, param, i, False)
# exp = DataGenerationExperiment(env, 'merge')
#
# # collect new samples
# new_data = exp.run(1, 30000)
#
# # extend the new data
# for key in new_data.keys():
# data[key].extend(new_data[key])
#
def generate_cfg(self, net_params, traffic_lights, routes):
"""Generate .sumo.cfg files using net files and netconvert.
This method is responsible for creating the following config files:
- *.add.xml: This file contains the sumo-specific properties of
vehicles with similar types, and properties of the traffic lights.
- *.rou.xml: This file contains the routes vehicles can traverse,
either from a specific starting edge, or by vehicle name, and well as
the inflows of vehicles.
- *.gui.cfg: This file contains the view settings of the gui (whether
the gui is used or not). The background of the gui is set here to be
grey, with RGB values: (100, 100, 100).
- *.sumo.cfg: This is the file that is used by the simulator to
identify the location of the various network, vehicle, and traffic
light properties that are used when instantiating the simulation.
Parameters
----------
net_params : flow.core.params.NetParams
see flow/core/params.py
traffic_lights : flow.core.params.TrafficLightParams
traffic light information, used to determine which nodes are
treated as traffic lights
routes : dict
Key = name of the starting edge
Element = list of edges a vehicle starting from this edge must
traverse.
"""
# this is the data that we will pass to the *.add.xml file
add = makexml('additional',
'http://sumo.dlr.de/xsd/additional_file.xsd')
# add the types of vehicles to the xml file
for params in self.network.vehicles.types:
type_params_str = {
key: str(params['type_params'][key])
for key in params['type_params']
}
add.append(E('vType', id=params['veh_id'], **type_params_str))
# add (optionally) the traffic light properties to the .add.xml file
num_traffic_lights = len(list(traffic_lights.get_properties().keys()))
if num_traffic_lights > 0:
if traffic_lights.baseline:
tl_params = traffic_lights.actuated_default()
tl_type = str(tl_params['tl_type'])
program_id = str(tl_params['program_id'])
phases = tl_params['phases']
max_gap = str(tl_params['max_gap'])
detector_gap = str(tl_params['detector_gap'])
show_detector = tl_params['show_detectors']
detectors = {'key': 'detector-gap', 'value': detector_gap}
gap = {'key': 'max-gap', 'value': max_gap}
if show_detector:
show_detector = {'key': 'show-detectors', 'value': 'true'}
else:
show_detector = {'key': 'show-detectors', 'value': 'false'}
nodes = self._inner_nodes # nodes where there's traffic lights
tll = []
for node in nodes:
tll.append({
'id': node['id'],
'type': tl_type,
'programID': program_id
})
for elem in tll:
e = E('tlLogic', **elem)
e.append(E('param', **show_detector))
e.append(E('param', **gap))
e.append(E('param', **detectors))
for phase in phases:
e.append(E('phase', **phase))
add.append(e)
else:
tl_properties = traffic_lights.get_properties()
for node in tl_properties.values():
# At this point, we assume that traffic lights are properly
# formed. If there are no phases for a static traffic
# light, ignore and use default
if node['type'] == 'static' and not node.get('phases'):
continue
elem = {
'id': str(node['id']),
'type': str(node['type']),
'programID': str(node['programID'])
}
if node.get('offset'):
elem['offset'] = str(node.get('offset'))
e = E('tlLogic', **elem)
for key, value in node.items():
if key == 'phases':
for phase in node.get('phases'):
e.append(E('phase', **phase))
else:
e.append(
E('param', **{
'key': key,
'value': str(value)
}))
add.append(e)
printxml(add, self.cfg_path + self.addfn)
# this is the data that we will pass to the *.gui.cfg file
gui = E('viewsettings')
gui.append(E('scheme', name='real world'))
gui.append(
E('background',
backgroundColor='100,100,100',
showGrid='0',
gridXSize='100.00',
gridYSize='100.00'))
printxml(gui, self.cfg_path + self.guifn)
# this is the data that we will pass to the *.rou.xml file
routes_data = makexml('routes',
'http://sumo.dlr.de/xsd/routes_file.xsd')
# add the routes to the .add.xml file
for route_id in routes.keys():
# in this case, we only have one route, convert into into a
# list of routes with one element
if isinstance(routes[route_id][0], str):
routes[route_id] = [(routes[route_id], 1)]
# add each route incrementally, and add a second term to denote
# the route number of the given route at the given edge
for i in range(len(routes[route_id])):
r, _ = routes[route_id][i]
routes_data.append(
E('route',
id='route{}_{}'.format(route_id, i),
edges=' '.join(r)))
# add the inflows from various edges to the xml file
if self.network.net_params.inflows is not None:
total_inflows = self.network.net_params.inflows.get()
for next_inflow in total_inflows:
# do not want to affect the original values
inflow = deepcopy(next_inflow)
# convert any non-string element in the inflow dict to a string
for key in inflow:
if not isinstance(inflow[key], str):
inflow[key] = repr(inflow[key])
# get the name of the edge the inflows correspond to, and the
# total inflow rate of the specific inflow
edge = deepcopy(inflow['edge'])
if 'vehsPerHour' in inflow:
flag, rate = 0, float(inflow['vehsPerHour'])
else:
flag, rate = 1, float(inflow['probability'])
del inflow['edge']
# distribute the inflow rates across all routes from a given
# edge on the basis of the provided fractions for each route
for i in range(len(routes[edge])):
_, frac = routes[edge][i]
inflow['route'] = 'route{}_{}'.format(edge, i)
if flag:
inflow['probability'] = str(rate * frac)
else:
inflow['vehsPerHour'] = str(rate * frac)
routes_data.append(_flow(**inflow))
printxml(routes_data, self.cfg_path + self.roufn)
# this is the data that we will pass to the *.sumo.cfg file
cfg = makexml('configuration',
'http://sumo.dlr.de/xsd/sumoConfiguration.xsd')
cfg.append(
_inputs(net=self.netfn,
add=self.addfn,
rou=self.roufn,
gui=self.guifn))
# initial time of the simulation
cfg.append(E('begin', value=repr(0)))
# simulation step length
cfg.append(E('step-length', value=repr(self.sim_params.sim_step)))
# add the lateral resolution of the sublanes (if requested)
if self.sim_params.lateral_resolution is not None:
cfg.append(
E("lateral-resolution",
value=str(self.sim_params.lateral_resolution)))
# add the emission path to the sumo command (if requested)
if self.sim_params.emission_path is not None:
ensure_dir(self.sim_params.emission_path)
emission_out = os.path.join(
self.sim_params.emission_path,
"{0}-emission.xml".format(self.network.name))
cfg.append(E("emission-output", value=emission_out))
# add step logs (if requested)
no_step_log = "true" if self.sim_params.no_step_log else "false"
cfg.append(E("no-step-log", value=no_step_log))
if self.sim_params.overtake_right:
cfg.append(E("lanechange.overtake-right", value="true"))
# specify a simulation seed (if requested)
if self.sim_params.seed is not None:
cfg.append(E("seed", value=str(self.sim_params.seed)))
if not self.sim_params.print_warnings:
cfg.append(E("no-warnings", value="true"))
# set the time it takes for a gridlock teleport to occur
cfg.append(
E("time-to-teleport",
value=str(int(self.sim_params.teleport_time))))
# check collisions at intersections
cfg.append(E("collision.check-junctions", value="true"))
printxml(cfg, self.cfg_path + self.sumfn)
return self.sumfn
def start_simulation(self, network, sim_params):
"""Start a sumo simulation instance.
This method uses the configuration files created by the network class
to initialize a sumo instance. Also initializes a traci connection to
interface with sumo from Python.
"""
error = None
for _ in range(RETRIES_ON_ERROR):
try:
# port number the sumo instance will be run on
port = sim_params.port
sumo_binary = "sumo-gui" if sim_params.render is True \
else "sumo"
# command used to start sumo
sumo_call = [
sumo_binary, "-c", network.cfg, "--remote-port",
str(sim_params.port), "--num-clients",
str(sim_params.num_clients), "--step-length",
str(sim_params.sim_step)
]
# add step logs (if requested)
if sim_params.no_step_log:
sumo_call.append("--no-step-log")
# add the lateral resolution of the sublanes (if requested)
if sim_params.lateral_resolution is not None:
sumo_call.append("--lateral-resolution")
sumo_call.append(str(sim_params.lateral_resolution))
# add the emission path to the sumo command (if requested)
if sim_params.emission_path is not None:
ensure_dir(sim_params.emission_path)
emission_out = os.path.join(
sim_params.emission_path,
"{0}-emission.xml".format(network.name))
sumo_call.append("--emission-output")
sumo_call.append(emission_out)
else:
emission_out = None
if sim_params.overtake_right:
sumo_call.append("--lanechange.overtake-right")
sumo_call.append("true")
# specify a simulation seed (if requested)
if sim_params.seed is not None:
sumo_call.append("--seed")
sumo_call.append(str(sim_params.seed))
if not sim_params.print_warnings:
sumo_call.append("--no-warnings")
sumo_call.append("true")
# set the time it takes for a gridlock teleport to occur
sumo_call.append("--time-to-teleport")
sumo_call.append(str(int(sim_params.teleport_time)))
# check collisions at intersections
sumo_call.append("--collision.check-junctions")
sumo_call.append("true")
logging.info(" Starting SUMO on port " + str(port))
logging.debug(" Cfg file: " + str(network.cfg))
if sim_params.num_clients > 1:
logging.info(" Num clients are" +
str(sim_params.num_clients))
logging.debug(" Emission file: " + str(emission_out))
logging.debug(" Step length: " + str(sim_params.sim_step))
# Opening the I/O thread to SUMO
self.sumo_proc = subprocess.Popen(sumo_call,
preexec_fn=os.setsid)
# wait a small period of time for the subprocess to activate
# before trying to connect with traci
if os.environ.get("TEST_FLAG", 0):
time.sleep(0.1)
else:
time.sleep(config.SUMO_SLEEP)
traci_connection = traci.connect(port, numRetries=100)
traci_connection.setOrder(0)
traci_connection.simulationStep()
return traci_connection
except Exception as e:
print("Error during start: {}".format(traceback.format_exc()))
error = e
self.teardown_sumo()
raise error
def train_h_baselines(env_name, args, multiagent):
"""Train policies using SAC and TD3 with h-baselines."""
from hbaselines.algorithms import OffPolicyRLAlgorithm
from hbaselines.utils.train import parse_options, get_hyperparameters
# Get the command-line arguments that are relevant here
args = parse_options(description="", example_usage="", args=args)
# the base directory that the logged data will be stored in
base_dir = "training_data"
for i in range(args.n_training):
# value of the next seed
seed = args.seed + i
# The time when the current experiment started.
now = strftime("%Y-%m-%d-%H:%M:%S")
# Create a save directory folder (if it doesn't exist).
dir_name = os.path.join(base_dir, '{}/{}'.format(args.env_name, now))
ensure_dir(dir_name)
# Get the policy class.
if args.alg == "TD3":
if multiagent:
from hbaselines.multi_fcnet.td3 import MultiFeedForwardPolicy
policy = MultiFeedForwardPolicy
else:
from hbaselines.fcnet.td3 import FeedForwardPolicy
policy = FeedForwardPolicy
elif args.alg == "SAC":
if multiagent:
from hbaselines.multi_fcnet.sac import MultiFeedForwardPolicy
policy = MultiFeedForwardPolicy
else:
from hbaselines.fcnet.sac import FeedForwardPolicy
policy = FeedForwardPolicy
else:
raise ValueError("Unknown algorithm: {}".format(args.alg))
# Get the hyperparameters.
hp = get_hyperparameters(args, policy)
# Add the seed for logging purposes.
params_with_extra = hp.copy()
params_with_extra['seed'] = seed
params_with_extra['env_name'] = args.env_name
params_with_extra['policy_name'] = policy.__name__
params_with_extra['algorithm'] = args.alg
params_with_extra['date/time'] = now
# Add the hyperparameters to the folder.
with open(os.path.join(dir_name, 'hyperparameters.json'), 'w') as f:
json.dump(params_with_extra, f, sort_keys=True, indent=4)
# Create the algorithm object.
alg = OffPolicyRLAlgorithm(
policy=policy,
env="flow:{}".format(env_name),
eval_env="flow:{}".format(env_name) if args.evaluate else None,
**hp)
# Perform training.
alg.learn(
total_steps=args.total_steps,
log_dir=dir_name,
log_interval=args.log_interval,
eval_interval=args.eval_interval,
save_interval=args.save_interval,
initial_exploration_steps=args.initial_exploration_steps,
seed=seed,
)
# dump the flow params
with open(os.path.join(save_path, 'flow_params.json'), 'w') as outfile:
json.dump(
params, outfile, cls=FlowParamsEncoder, sort_keys=True, indent=4)
if __name__ == "__main__":
flags = parse_args(sys.argv[1:])
# Import the benchmark and fetch its flow_params
module = __import__("flow.benchmarks.{}".format(flags.benchmark_name),
fromlist=["flow_params"])
flow_params = module.flow_params
# Path to the saved files
exp_tag = flow_params['exp_tag']
result_name = '{}/{}'.format(exp_tag, strftime("%Y-%m-%d-%H:%M:%S"))
# Define the save path and ensure that the required directories exist.
dir_path = os.path.realpath(os.path.expanduser('~/baseline_results'))
ensure_dir(dir_path)
path = os.path.join(dir_path, result_name)
# Perform the training operation.
train_model = run_model(flow_params, flags.rollout_size, flags.num_steps)
# Save the model to a desired folder and then delete it to demonstrate
# loading.
save_model(train_model, flow_params, path)
elif flags.rl_trainer == "Stable-Baselines":
flow_params = submodule.flow_params
# Path to the saved files
exp_tag = flow_params['exp_tag']
result_name = '{}/{}'.format(exp_tag, strftime("%Y-%m-%d-%H:%M:%S"))
# Perform training.
print('Beginning training.')
model = run_model_stablebaseline(flow_params, flags.num_cpus,
flags.rollout_size, flags.num_steps)
# Save the model to a desired folder and then delete it to demonstrate
# loading.
print('Saving the trained model!')
path = os.path.realpath(os.path.expanduser('~/baseline_results'))
ensure_dir(path)
save_path = os.path.join(path, result_name)
model.save(save_path)
# dump the flow params
with open(os.path.join(path, result_name) + '.json', 'w') as outfile:
json.dump(flow_params,
outfile,
cls=FlowParamsEncoder,
sort_keys=True,
indent=4)
# Replay the result by loading the model
print('Loading the trained model and testing it out!')
model = PPO2.load(save_path)
flow_params = get_flow_params(
def __init__(self, env_params, sumo_params, scenario):
"""
Base environment class. Provides the interface for controlling a SUMO
simulation. Using this class, you can start sumo, provide a scenario to
specify a configuration and controllers, perform simulation steps, and
reset the simulation to an initial configuration.
SumoEnvironment is Serializable to allow for pickling of the policy.
This class cannot be used as is: you must extend it to implement an
action applicator method, and properties to define the MDP if you choose
to use it with RLLab. This can be done by overloading the following
functions in a child class:
- action_space
- observation_space
- apply_rl_action
- get_state
- compute_reward
Attributes
----------
env_params: EnvParams type:
see flow/core/params.py
sumo_params: SumoParams type
see flow/core/params.py
scenario: Scenario type
see flow/scenarios/base_scenario.py
"""
Serializable.quick_init(self, locals())
self.env_params = env_params
self.scenario = scenario
self.sumo_params = sumo_params
self.sumo_binary = self.sumo_params.sumo_binary
self.vehicles = scenario.vehicles
# timer: Represents number of steps taken since the start of a rollout
self.timer = 0
# initial_state:
# Key = Vehicle ID,
# Entry = (type_id, route_id, lane_index, lane_pos, speed, pos)
self.initial_state = {}
# vehicle identifiers for all vehicles
self.ids = []
# vehicle identifiers for specific types of vehicles
self.controlled_ids, self.sumo_ids, self.rl_ids = [], [], []
self.state = None
self.obs_var_labels = []
# SUMO Params
if sumo_params.port:
self.port = sumo_params.port
else:
self.port = sumolib.miscutils.getFreeSocketPort()
self.time_step = sumo_params.time_step
self.vehicle_arrangement_shuffle = \
sumo_params.vehicle_arrangement_shuffle
self.starting_position_shuffle = sumo_params.starting_position_shuffle
self.emission_path = sumo_params.emission_path
# path to the output (emission) file provided by sumo
if self.emission_path:
ensure_dir(self.emission_path)
self.emission_out = \
self.emission_path + "{0}-emission.xml".format(
self.scenario.name)
else:
self.emission_out = None
self.fail_safe = env_params.fail_safe
self.max_speed = env_params.max_speed
self.lane_change_duration = \
env_params.get_lane_change_duration(self.time_step)
self.shared_reward = env_params.shared_reward
self.shared_policy = env_params.shared_policy
# the available_routes variable contains a dictionary of routes vehicles
# can traverse; to be used when routes need to be chosen dynamically
self.available_routes = self.scenario.generator.rts
# Check if we are in a multi-agent scenario
if isinstance(self.action_space, list):
self.multi_agent = True
else:
self.multi_agent = False
self.start_sumo()
self.setup_initial_state()
def start_sumo(self):
"""Starts a sumo instance.
Uses the configuration files created by the generator class to
initialize a sumo instance. Also initializes a traci connection to
interface with sumo from Python.
"""
error = None
for _ in range(RETRIES_ON_ERROR):
try:
# port number the sumo instance will be run on
if self.sumo_params.port is not None:
port = self.sumo_params.port
else:
# Don't do backoff when testing
if os.environ.get("TEST_FLAG", 0):
# backoff to decrease likelihood of race condition
time_stamp = ''.join(str(time.time()).split('.'))
time.sleep(1.0 * int(time_stamp[-6:]) / 1e6) # 1.0 for consistency w/ above
port = sumolib.miscutils.getFreeSocketPort()
# command used to start sumo
sumo_call = [self.sumo_params.sumo_binary,
"-c", self.scenario.cfg,
"--remote-port", str(port),
"--step-length", str(self.sim_step)]
# add step logs (if requested)
if self.sumo_params.no_step_log:
sumo_call.append("--no-step-log")
# add the lateral resolution of the sublanes (if requested)
if self.sumo_params.lateral_resolution is not None:
sumo_call.append("--lateral-resolution")
sumo_call.append(str(self.sumo_params.lateral_resolution))
# add the emission path to the sumo command (if requested)
if self.sumo_params.emission_path is not None:
ensure_dir(self.sumo_params.emission_path)
emission_out = \
self.sumo_params.emission_path + \
"{0}-emission.xml".format(self.scenario.name)
sumo_call.append("--emission-output")
sumo_call.append(emission_out)
else:
emission_out = None
if self.sumo_params.overtake_right:
sumo_call.append("--lanechange.overtake-right")
sumo_call.append("true")
if self.sumo_params.ballistic:
sumo_call.append("--step-method.ballistic")
sumo_call.append("true")
# specify a simulation seed (if requested)
if self.sumo_params.seed is not None:
sumo_call.append("--seed")
sumo_call.append(str(self.sumo_params.seed))
if not self.sumo_params.print_warnings:
sumo_call.append("--no-warnings")
sumo_call.append("true")
# set the time it takes for a gridlock teleport to occur
sumo_call.append("--time-to-teleport")
sumo_call.append(str(int(self.sumo_params.teleport_time)))
logging.info(" Starting SUMO on port " + str(port))
logging.debug(" Cfg file: " + str(self.scenario.cfg))
logging.debug(" Emission file: " + str(emission_out))
logging.debug(" Step length: " + str(self.sim_step))
# Opening the I/O thread to SUMO
self.sumo_proc = subprocess.Popen(sumo_call,
preexec_fn=os.setsid)
# wait a small period of time for the subprocess to activate
# before trying to connect with traci
if os.environ.get("TEST_FLAG", 0):
time.sleep(0.1)
else:
time.sleep(config.SUMO_SLEEP)
self.traci_connection = traci.connect(port, numRetries=100)
self.traci_connection.simulationStep()
return
except Exception as e:
print("Error during start: {}".format(traceback.format_exc()))
error = e
self.teardown_sumo()
raise error
def __init__(self,
name,
vehicles,
net_params,
initial_config=InitialConfig(),
traffic_lights=TrafficLights()):
"""Instantiate the base scenario class.
Attributes
----------
name : str
A tag associated with the scenario
vehicles : Vehicles type
see flow/core/vehicles.py
net_params : NetParams type
see flow/core/params.py
initial_config : InitialConfig type
see flow/core/params.py
traffic_lights : flow.core.traffic_lights.TrafficLights type
see flow/core/traffic_lights.py
"""
# Invoke serializable if using rllab
if Serializable is not object:
Serializable.quick_init(self, locals())
self.orig_name = name # To avoid repeated concatenation upon reset
self.name = name + time.strftime("_%Y%m%d-%H%M%S") + str(time.time())
self.vehicles = vehicles
self.net_params = net_params
self.initial_config = initial_config
self.traffic_lights = traffic_lights
self.net_params = net_params
self.net_path = os.path.dirname(os.path.abspath(__file__)) \
+ "/debug/net/"
self.cfg_path = os.path.dirname(os.path.abspath(__file__)) \
+ "/debug/cfg/"
self.vehicle_ids = []
ensure_dir("%s" % self.net_path)
ensure_dir("%s" % self.cfg_path)
self.nodfn = "%s.nod.xml" % self.name
self.edgfn = "%s.edg.xml" % self.name
self.typfn = "%s.typ.xml" % self.name
self.cfgfn = "%s.netccfg" % self.name
self.netfn = "%s.net.xml" % self.name
self.confn = "%s.con.xml" % self.name
self.roufn = "%s.rou.xml" % self.name
self.addfn = "%s.add.xml" % self.name
self.sumfn = "%s.sumo.cfg" % self.name
self.guifn = "%s.gui.cfg" % self.name
# create the network configuration files
self._edges, self._connections = self.generate_net(
self.net_params, self.traffic_lights)
# list of edges and internal links (junctions)
self._edge_list = [
edge_id for edge_id in self._edges.keys() if edge_id[0] != ":"
]
self._junction_list = list(
set(self._edges.keys()) - set(self._edge_list))
# maximum achievable speed on any edge in the network
self.max_speed = max(
self.speed_limit(edge) for edge in self.get_edge_list())
# parameters to be specified under each unique subclass's
# __init__() function
self.edgestarts = self.specify_edge_starts()
# these optional parameters need only be used if "no-internal-links"
# is set to "false" while calling sumo's netconvert function
self.internal_edgestarts = self.specify_internal_edge_starts()
self.intersection_edgestarts = self.specify_intersection_edge_starts()
# in case the user did not write the intersection edge-starts in
# internal edge-starts as well (because of redundancy), merge the two
# together
self.internal_edgestarts += self.intersection_edgestarts
seen = set()
self.internal_edgestarts = \
[item for item in self.internal_edgestarts
if item[1] not in seen and not seen.add(item[1])]
self.internal_edgestarts_dict = dict(self.internal_edgestarts)
# total_edgestarts and total_edgestarts_dict contain all of the above
# edges, with the former being ordered by position
if self.net_params.no_internal_links:
self.total_edgestarts = self.edgestarts
else:
self.total_edgestarts = self.edgestarts + self.internal_edgestarts
self.total_edgestarts.sort(key=lambda tup: tup[1])
self.total_edgestarts_dict = dict(self.total_edgestarts)
# length of the network, or the portion of the network in
# which cars are meant to be distributed
# (may be overridden by subclass __init__())
if not hasattr(self, "length"):
self.length = sum([
self.edge_length(edge_id) for edge_id in self.get_edge_list()
])
# generate starting position for vehicles in the network
kwargs = initial_config.additional_params
positions, lanes = self.generate_starting_positions(
num_vehicles=vehicles.num_vehicles, **kwargs)
# create the sumo configuration files
cfg_name = self.generate_cfg(self.net_params, self.traffic_lights)
shuffle = initial_config.shuffle
self.make_routes(self, positions, lanes, shuffle)
# specify the location of the sumo configuration file
self.cfg = self.cfg_path + cfg_name
elif flags.rl_trainer == "Stable-Baselines":
flow_params = submodule.flow_params
# Path to the saved files
exp_tag = flow_params['exp_tag']
result_name = '{}/{}'.format(exp_tag, strftime("%Y-%m-%d-%H:%M:%S"))
# Perform training.
print('Beginning training.')
model = run_model_stablebaseline(flow_params, flags.num_cpus, flags.rollout_size, flags.num_steps)
# Save the model to a desired folder and then delete it to demonstrate
# loading.
print('Saving the trained model!111')
path = os.path.realpath(os.path.expanduser('~/baseline_results'))
print('pot 1')
ensure_dir(os.path.join(path,'stabilizing_the_ring'))
print('pot 2')
save_path = os.path.join(path, result_name)
print('pot 3')
model.save(save_path)
print('I am here to test, begin!')
# dump the flow params
with open(os.path.join(path, result_name) + '.json', 'w') as outfile:
json.dump(flow_params, outfile,
cls=FlowParamsEncoder, sort_keys=True, indent=4)
print('I am here to test, over!')
# Replay the result by loading the model
print('Loading the trained model and testing it out!')
model = PPO2.load(save_path)
