def train_stable_baselines(submodule, flags):
"""Train policies using the PPO algorithm in stable-baselines."""
from stable_baselines3.common.vec_env import DummyVecEnv
flow_params = submodule.flow_params
# Path to the saved files
exp_tag = flow_params['exp_tag']
result_name = '{}/{}'.format(exp_tag, strftime("%Y-%m-%d-%H:%M:%S"))
# Perform training.
start_time = timeit.default_timer()
# print experiment.json information
print("=========================================")
print('Beginning training.')
print('Algorithm :', flags.algorithm)
model = run_model_stablebaseline(flow_params, flags.num_cpus,
flags.rollout_size, flags.num_steps,
flags.algorithm, flags.exp_config)
stop_time = timeit.default_timer()
run_time = stop_time - start_time
print("Training is Finished")
print("total runtime: ", run_time)
# Save the model to a desired folder and then delete it to demonstrate
# loading.
print('Saving the trained model!')
path = os.path.realpath(os.path.expanduser('~/baseline_results'))
ensure_dir(path)
save_path = os.path.join(path, result_name)
model.save(save_path)
# dump the flow params
with open(os.path.join(path, result_name) + '.json', 'w') as outfile:
json.dump(flow_params,
outfile,
cls=FlowParamsEncoder,
sort_keys=True,
indent=4)
# Replay the result by loading the model
print('Loading the trained model and testing it out!')
if flags.exp_config.lower() == "ppo":
from stable_baselines3 import PPO
model = PPO.load(save_path)
elif flags.exp_config.lower() == "ddpg":
from stable_baselines3 import DDPG
model = DDPG.load(save_path)
flow_params = get_flow_params(os.path.join(path, result_name) + '.json')
flow_params['sim'].render = True
env = env_constructor(params=flow_params, version=0)()
# The algorithms require a vectorized environment to run
eval_env = DummyVecEnv([lambda: env])
obs = eval_env.reset()
reward = 0
for _ in range(flow_params['env'].horizon):
action, _states = model.predict(obs)
obs, rewards, dones, info = eval_env.step(action)
reward += rewards
print('the final reward is {}'.format(reward))
def train_stable_baselines3(submodule, flags):
"""Train policies using the PPO algorithm in stable-baselines3."""
from stable_baselines3.common.vec_env import DummyVecEnv
from stable_baselines3 import PPO
import torch
start_time = timeit.default_timer()
flow_params = submodule.flow_params
# Path to the saved files
exp_tag = flow_params['exp_tag']
result_name = '{}/{}'.format(exp_tag, strftime("%Y-%m-%d-%H:%M:%S"))
# Perform training.
print("cuda is available: ", torch.cuda.is_available())
print('Beginning training.')
print("==========================================")
model = run_model_stablebaseline3(flow_params, flags.num_cpus,
flags.rollout_size, flags.num_steps)
# Save the model to a desired folder and then delete it to demonstrate
# loading.
print('Saving the trained model!')
path = os.path.realpath(os.path.expanduser('~/baseline_results'))
ensure_dir(path)
save_path = os.path.join(path, result_name)
model.save(save_path)
# dump the flow params
# check time for choose GPU and CPU
stop_time = timeit.default_timer()
run_time = stop_time - start_time
with open(os.path.join(path, result_name) + '.json', 'w') as outfile:
json.dump(flow_params,
outfile,
cls=FlowParamsEncoder,
sort_keys=True,
indent=4)
# Replay the result by loading the model
print('Loading the trained model and testing it out!')
model.load(save_path)
flow_params = get_flow_params(os.path.join(path, result_name) + '.json')
flow_params['sim'].render = False
flow_params['env'].horizon = 1500 # 150seconds operation
env = env_constructor(params=flow_params, version=0)()
# The algorithms require a vectorized environment to run
eval_env = DummyVecEnv([lambda: env])
obs = eval_env.reset()
reward = 0
for _ in range(flow_params['env'].horizon):
action, _states = model.predict(obs)
obs, rewards, dones, info = eval_env.step(action)
reward += rewards
print("--------------------------------------------------------")
flow_params['sim'].render = True
simulation = Experiment(flow_params)
simulation.run(num_runs=1)
print('the final reward is {}'.format(reward))
print("total run_time:", run_time, "s")
def train_stable_baselines(submodule, flags):
"""Train policies using the PPO algorithm in stable-baselines."""
from stable_baselines.common.vec_env import DummyVecEnv
from stable_baselines import PPO2
flow_params = submodule.flow_params
# Path to the saved files
exp_tag = flow_params['exp_tag']
exp_folder_name = os.path.join(os.getcwd(), exp_tag)
if not os.path.exists(exp_folder_name):
os.makedirs(exp_folder_name)
result_name = '{}/{}'.format(exp_tag, strftime("%Y-%m-%d-%H:%M:%S"))
# Perform training.
print_box('Beginning training.')
model = run_model_stablebaseline(flow_params, flags.num_cpus,
flags.rollout_size, flags.num_steps)
# Save the model to a desired folder and then delete it to demonstrate
# loading.
# NOTE changed file saving HERE
print_box('Saving the trained model!')
path = os.getcwd()
save_path = os.path.join(path, result_name)
model.save(save_path)
# dump the flow params
with open(os.path.join(path, result_name) + '.json', 'w') as outfile:
json.dump(flow_params,
outfile,
cls=FlowParamsEncoder,
sort_keys=True,
indent=4)
# Replay the result by loading the model
print_box('Loading the trained model and testing it out!')
model = PPO2.load(save_path)
flow_params = get_flow_params(os.path.join(path, result_name) + '.json')
flow_params['sim'].render = False
env = env_constructor(params=flow_params, version=0)()
# The algorithms require a vectorized environment to run
eval_env = DummyVecEnv([lambda: env])
obs = eval_env.reset()
reward = 0
for _ in range(flow_params['env'].horizon):
action, _states = model.predict(obs)
obs, rewards, dones, info = eval_env.step(action)
reward += rewards
print('The final reward is {}'.format(reward))
def play_results(path, result_name):
print('Loading the trained model and testing it out!')
save_path = os.path.join(path, result_name)
model = DQN.load(save_path)
flow_params = get_flow_params(os.path.join(path, result_name) + '.json')
flow_params['sim'].render = True
env_con = env_constructor(params=flow_params, version=0)()
# The algorithms require a vectorized environment to run
eval_env = DummyVecEnv([lambda: env_con])
obs = eval_env.reset()
reward = 0
for _ in range(flow_params['env'].horizon):
action, _states = model.predict(obs)
obs, rewards, dones, info = eval_env.step(action)
reward += rewards
print('the final reward is {}'.format(reward))
def train_stable_baselines(submodule, flags):
"""Train policies using the PPO algorithm in stable-baselines."""
from stable_baselines.common.vec_env import DummyVecEnv
from stable_baselines import DDPG
flow_params = submodule.flow_params
# Path to the saved files
exp_tag = flow_params['exp_tag']
result_name = '{}/{}'.format(exp_tag, strftime("%Y-%m-%d-%H:%M:%S"))
# Perform training.
print('Beginning training.')
model = run_model_stablebaseline(
flow_params, flags.num_cpus, flags.rollout_size, flags.num_steps)
# Save the model to a desired folder and then delete it to demonstrate
# loading.
print('Saving the trained model!')
path = os.path.realpath(os.path.expanduser('~/baseline_results'))
ensure_dir(path)
save_path = os.path.join(path, result_name)
model.save(save_path)
# dump the flow params
with open(os.path.join(path, result_name) + '.json', 'w') as outfile:
json.dump(flow_params, outfile,
cls=FlowParamsEncoder, sort_keys=True, indent=4)
# Replay the result by loading the model
print('Loading the trained model and testing it out!')
model = DDPG.load(save_path)
flow_params = get_flow_params(os.path.join(path, result_name) + '.json')
flow_params['sim'].render = True
env = env_constructor(params=flow_params, version=0)()
n_actions = env.action_space.shape[-1]
param_noise = None
action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions), sigma=float(0.5) * np.ones(n_actions))
# The algorithms require a vectorized environment to run
eval_env = DummyVecEnv([lambda: env])
obs = eval_env.reset()
reward = 0
for _ in range(flow_params['env'].horizon):
action, _states = model.predict(obs)
obs, rewards, dones, info = eval_env.step(action)
reward += rewards
print('the final reward is {}'.format(reward))
def get_compute_action_rllib(path_to_dir, checkpoint_num, alg):
"""Collect the compute_action method from RLlib's serialized files.
Parameters
----------
path_to_dir : str
RLlib directory containing training results
checkpoint_num : int
checkpoint number / training iteration of the learned policy
alg : str
name of the RLlib algorithm that was used during the training
procedure
Returns
-------
method
the compute_action method from the algorithm along with the trained
parameters
"""
# collect the configuration information from the RLlib checkpoint
result_dir = path_to_dir if path_to_dir[-1] != '/' else path_to_dir[:-1]
config = get_rllib_config(result_dir)
# run on only one cpu for rendering purposes
ray.init(num_cpus=1)
config["num_workers"] = 1
# create and register a gym+rllib env
flow_params = get_flow_params(config)
create_env, env_name = make_create_env(params=flow_params,
version=9999,
render=False)
register_env(env_name, create_env)
# recreate the agent
agent_cls = get_agent_class(alg)
agent = agent_cls(env=env_name, registry=get_registry(), config=config)
# restore the trained parameters into the policy
checkpoint = result_dir + '/checkpoint-{}'.format(checkpoint_num)
agent._restore(checkpoint)
return agent.compute_action
print('Saving the trained model!')
model.save(save_path)
# dump the flow params
with open(os.path.join(path, args.result_name) + '.json', 'w') as outfile:
json.dump(flow_params,
outfile,
cls=FlowParamsEncoder,
sort_keys=True,
indent=4)
del model
del flow_params
# Replay the result by loading the model
print('Loading the trained model and testing it out!')
model = PPO2.load(save_path)
flow_params = get_flow_params(
os.path.join(path, args.result_name) + '.json')
flow_params['sim'].render = True
env_constructor = env_constructor(params=flow_params, version=0)()
env = DummyVecEnv([
lambda: env_constructor
]) # The algorithms require a vectorized environment to run
obs = env.reset()
reward = 0
for i in range(flow_params['env'].horizon):
action, _states = model.predict(obs)
obs, rewards, dones, info = env.step(action)
reward += rewards
print('the final reward is {}'.format(reward))
def visualizer_rllib(args):
result_dir = args.result_dir if args.result_dir[-1] != '/' \
else args.result_dir[:-1]
config = get_rllib_config(result_dir)
# TODO(ev) backwards compatibility hack
try:
pkl = get_rllib_pkl(result_dir)
except Exception:
pass
# check if we have a multiagent scenario but in a
# backwards compatible way
if config.get('multiagent', {}).get('policy_graphs', {}):
multiagent = True
config['multiagent'] = pkl['multiagent']
else:
multiagent = False
# Run on only one cpu for rendering purposes
config['num_workers'] = 0
flow_params = get_flow_params(config)
# hack for old pkl files
# TODO(ev) remove eventually
sim_params = flow_params['sim']
setattr(sim_params, 'num_clients', 1)
# Determine agent and checkpoint
config_run = config['env_config']['run'] if 'run' in config['env_config'] \
else None
if args.run and config_run:
if args.run != config_run:
print('visualizer_rllib.py: error: run argument ' +
'\'{}\' passed in '.format(args.run) +
'differs from the one stored in params.json ' +
'\'{}\''.format(config_run))
sys.exit(1)
if args.run:
agent_cls = get_agent_class(args.run)
elif config_run:
agent_cls = get_agent_class(config_run)
else:
print('visualizer_rllib.py: error: could not find flow parameter '
'\'run\' in params.json, '
'add argument --run to provide the algorithm or model used '
'to train the results\n e.g. '
'python ./visualizer_rllib.py /tmp/ray/result_dir 1 --run PPO')
sys.exit(1)
sim_params.restart_instance = False
dir_path = os.path.dirname(os.path.realpath(__file__))
emission_path = '{0}/test_time_rollout/'.format(dir_path)
sim_params.emission_path = emission_path if args.gen_emission else None
# pick your rendering mode
if args.render_mode == 'sumo_web3d':
sim_params.num_clients = 2
sim_params.render = False
elif args.render_mode == 'drgb':
sim_params.render = 'drgb'
sim_params.pxpm = 4
elif args.render_mode == 'sumo_gui':
sim_params.render = True
elif args.render_mode == 'no_render':
sim_params.render = False
if args.save_render:
sim_params.render = 'drgb'
sim_params.pxpm = 4
sim_params.save_render = True
# Create and register a gym+rllib env
create_env, env_name = make_create_env(params=flow_params, version=0)
register_env(env_name, create_env)
# check if the environment is a single or multiagent environment, and
# get the right address accordingly
# single_agent_envs = [env for env in dir(flow.envs)
# if not env.startswith('__')]
# if flow_params['env_name'] in single_agent_envs:
# env_loc = 'flow.envs'
# else:
# env_loc = 'flow.multiagent_envs'
# Start the environment with the gui turned on and a path for the
# emission file
env_params = flow_params['env']
env_params.restart_instance = False
if args.evaluate:
env_params.evaluate = True
# lower the horizon if testing
if args.horizon:
config['horizon'] = args.horizon
env_params.horizon = args.horizon
# create the agent that will be used to compute the actions
agent = agent_cls(env=env_name, config=config)
checkpoint = result_dir + '/checkpoint_' + args.checkpoint_num
checkpoint = checkpoint + '/checkpoint-' + args.checkpoint_num
agent.restore(checkpoint)
if hasattr(agent, "local_evaluator") and \
os.environ.get("TEST_FLAG") != 'True':
env = agent.local_evaluator.env
else:
env = gym.make(env_name)
if multiagent:
rets = {}
# map the agent id to its policy
policy_map_fn = config['multiagent']['policy_mapping_fn'].func
for key in config['multiagent']['policy_graphs'].keys():
rets[key] = []
else:
rets = []
if config['model']['use_lstm']:
use_lstm = True
if multiagent:
state_init = {}
# map the agent id to its policy
policy_map_fn = config['multiagent']['policy_mapping_fn'].func
size = config['model']['lstm_cell_size']
for key in config['multiagent']['policy_graphs'].keys():
state_init[key] = [
np.zeros(size, np.float32),
np.zeros(size, np.float32)
]
else:
state_init = [
np.zeros(config['model']['lstm_cell_size'], np.float32),
np.zeros(config['model']['lstm_cell_size'], np.float32)
]
else:
use_lstm = False
env.restart_simulation(sim_params=sim_params, render=sim_params.render)
final_outflows = []
mean_speed = []
for i in range(args.num_rollouts):
vel = []
state = env.reset()
if multiagent:
ret = {key: [0] for key in rets.keys()}
else:
ret = 0
for _ in range(env_params.horizon):
vehicles = env.unwrapped.k.vehicle
vel.append(np.mean(vehicles.get_speed(vehicles.get_ids())))
if multiagent:
action = {}
for agent_id in state.keys():
if use_lstm:
action[agent_id], state_init[agent_id], logits = \
agent.compute_action(
state[agent_id], state=state_init[agent_id],
policy_id=policy_map_fn(agent_id))
else:
action[agent_id] = agent.compute_action(
state[agent_id], policy_id=policy_map_fn(agent_id))
else:
action = agent.compute_action(state)
state, reward, done, _ = env.step(action)
if multiagent:
for actor, rew in reward.items():
ret[policy_map_fn(actor)][0] += rew
else:
ret += reward
if multiagent and done['__all__']:
break
if not multiagent and done:
break
if multiagent:
for key in rets.keys():
rets[key].append(ret[key])
else:
rets.append(ret)
outflow = vehicles.get_outflow_rate(500)
final_outflows.append(outflow)
mean_speed.append(np.mean(vel))
if multiagent:
for agent_id, rew in rets.items():
print('Round {}, Return: {} for agent {}'.format(
i, ret, agent_id))
else:
print('Round {}, Return: {}'.format(i, ret))
if multiagent:
for agent_id, rew in rets.items():
print('Average, std return: {}, {} for agent {}'.format(
np.mean(rew), np.std(rew), agent_id))
else:
print('Average, std return: {}, {}'.format(np.mean(rets),
np.std(rets)))
print('Average, std speed: {}, {}'.format(np.mean(mean_speed),
np.std(mean_speed)))
print('Average, std outflow: {}, {}'.format(np.mean(final_outflows),
np.std(final_outflows)))
# terminate the environment
env.unwrapped.terminate()
# if prompted, convert the emission file into a csv file
if args.gen_emission:
time.sleep(0.1)
dir_path = os.path.dirname(os.path.realpath(__file__))
emission_filename = '{0}-emission.xml'.format(env.scenario.name)
emission_path = \
'{0}/test_time_rollout/{1}'.format(dir_path, emission_filename)
emission_to_csv(emission_path)
# if we wanted to save the render, here we create the movie
if args.save_render:
dirs = os.listdir(os.path.expanduser('~') + '/flow_rendering')
dirs.sort(key=lambda date: datetime.strptime(date, "%Y-%m-%d-%H%M%S"))
recent_dir = dirs[-1]
# create the movie
movie_dir = os.path.expanduser('~') + '/flow_rendering/' + recent_dir
save_dir = os.path.expanduser('~') + '/flow_movies'
if not os.path.exists(save_dir):
os.mkdir(save_dir)
os_cmd = "cd " + movie_dir + " && ffmpeg -i frame_%06d.png"
os_cmd += " -pix_fmt yuv420p " + dirs[-1] + ".mp4"
os_cmd += "&& cp " + dirs[-1] + ".mp4 " + save_dir + "/"
os.system(os_cmd)
def visualizer_rllib(args):
"""Visualizer for RLlib experiments.
This function takes args (see function create_parser below for
more detailed information on what information can be fed to this
visualizer), and renders the experiment associated with it.
"""
result_dir = args.result_dir if args.result_dir[-1] != '/' \
else args.result_dir[:-1]
config = get_rllib_config(result_dir)
# check if we have a multiagent environment but in a
# backwards compatible way
if config.get('multiagent', {}).get('policies', None):
multiagent = True
pkl = get_rllib_pkl(result_dir)
config['multiagent'] = pkl['multiagent']
else:
multiagent = False
# Run on only one cpu for rendering purposes
config['num_workers'] = 0
flow_params = get_flow_params(config)
# hack for old pkl files
# TODO(ev) remove eventually
sim_params = flow_params['sim']
setattr(sim_params, 'num_clients', 1)
# Determine agent and checkpoint
config_run = config['env_config']['run'] if 'run' in config['env_config'] \
else None
if args.run and config_run:
if args.run != config_run:
print('visualizer_rllib.py: error: run argument '
+ '\'{}\' passed in '.format(args.run)
+ 'differs from the one stored in params.json '
+ '\'{}\''.format(config_run))
sys.exit(1)
if args.run:
agent_cls = get_agent_class(args.run)
elif config_run:
agent_cls = get_agent_class(config_run)
else:
print('visualizer_rllib.py: error: could not find flow parameter '
'\'run\' in params.json, '
'add argument --run to provide the algorithm or model used '
'to train the results\n e.g. '
'python ./visualizer_rllib.py /tmp/ray/result_dir 1 --run PPO')
sys.exit(1)
sim_params.restart_instance = True
dir_path = os.path.dirname(os.path.realpath(__file__))
emission_path = '{0}/test_time_rollout/'.format(dir_path)
sim_params.emission_path = emission_path if args.gen_emission else None
# pick your rendering mode
if args.render_mode == 'sumo_web3d':
sim_params.num_clients = 2
sim_params.render = False
elif args.render_mode == 'drgb':
sim_params.render = 'drgb'
sim_params.pxpm = 4
elif args.render_mode == 'sumo_gui':
sim_params.render = True
print('NOTE: With render mode {}, an extra instance of the SUMO GUI '
'will display before the GUI for visualizing the result. Click '
'the green Play arrow to continue.'.format(args.render_mode))
elif args.render_mode == 'no_render':
sim_params.render = False
if args.save_render:
sim_params.render = 'drgb'
sim_params.pxpm = 4
sim_params.save_render = True
# Create and register a gym+rllib env
create_env, env_name = make_create_env(params=flow_params, version=0)
register_env(env_name, create_env)
# check if the environment is a single or multiagent environment, and
# get the right address accordingly
# single_agent_envs = [env for env in dir(flow.envs)
# if not env.startswith('__')]
# if flow_params['env_name'] in single_agent_envs:
# env_loc = 'flow.envs'
# else:
# env_loc = 'flow.envs.multiagent'
# Start the environment with the gui turned on and a path for the
# emission file
env_params = flow_params['env']
env_params.restart_instance = False
if args.evaluate:
env_params.evaluate = True
# lower the horizon if testing
if args.horizon:
config['horizon'] = args.horizon
env_params.horizon = args.horizon
# create the agent that will be used to compute the actions
agent = agent_cls(env=env_name, config=config)
checkpoint = result_dir + '/checkpoint_' + args.checkpoint_num
checkpoint = checkpoint + '/checkpoint-' + args.checkpoint_num
agent.restore(checkpoint)
if hasattr(agent, "local_evaluator") and \
os.environ.get("TEST_FLAG") != 'True':
env = agent.local_evaluator.env
else:
env = gym.make(env_name)
if multiagent:
rets = {}
# map the agent id to its policy
policy_map_fn = config['multiagent']['policy_mapping_fn'].func
for key in config['multiagent']['policies'].keys():
rets[key] = []
else:
rets = []
if config['model']['use_lstm']:
use_lstm = True
if multiagent:
state_init = {}
# map the agent id to its policy
policy_map_fn = config['multiagent']['policy_mapping_fn'].func
size = config['model']['lstm_cell_size']
for key in config['multiagent']['policies'].keys():
state_init[key] = [np.zeros(size, np.float32),
np.zeros(size, np.float32)]
else:
state_init = [
np.zeros(config['model']['lstm_cell_size'], np.float32),
np.zeros(config['model']['lstm_cell_size'], np.float32)
]
else:
use_lstm = False
env.restart_simulation(
sim_params=sim_params, render=sim_params.render)
# Simulate and collect metrics
final_outflows = []
final_inflows = []
mean_speed = []
std_speed = []
for i in range(args.num_rollouts):
vel = []
state = env.reset()
if multiagent:
ret = {key: [0] for key in rets.keys()}
else:
ret = 0
for _ in range(env_params.horizon):
vehicles = env.unwrapped.k.vehicle
vel.append(np.mean(vehicles.get_speed(vehicles.get_ids())))
if multiagent:
action = {}
for agent_id in state.keys():
if use_lstm:
action[agent_id], state_init[agent_id], logits = \
agent.compute_action(
state[agent_id], state=state_init[agent_id],
policy_id=policy_map_fn(agent_id))
else:
action[agent_id] = agent.compute_action(
state[agent_id], policy_id=policy_map_fn(agent_id))
else:
action = agent.compute_action(state)
state, reward, done, _ = env.step(action)
if multiagent:
for actor, rew in reward.items():
ret[policy_map_fn(actor)][0] += rew
else:
ret += reward
if multiagent and done['__all__']:
break
if not multiagent and done:
break
if multiagent:
for key in rets.keys():
rets[key].append(ret[key])
else:
rets.append(ret)
outflow = vehicles.get_outflow_rate(500)
final_outflows.append(outflow)
inflow = vehicles.get_inflow_rate(500)
final_inflows.append(inflow)
if np.all(np.array(final_inflows) > 1e-5):
throughput_efficiency = [x / y for x, y in
zip(final_outflows, final_inflows)]
else:
throughput_efficiency = [0] * len(final_inflows)
mean_speed.append(np.mean(vel))
std_speed.append(np.std(vel))
if multiagent:
for agent_id, rew in rets.items():
print('Round {}, Return: {} for agent {}'.format(
i, ret, agent_id))
else:
print('Round {}, Return: {}'.format(i, ret))
print('==== Summary of results ====')
print("Return:")
print(mean_speed)
if multiagent:
for agent_id, rew in rets.items():
print('For agent', agent_id)
print(rew)
print('Average, std return: {}, {} for agent {}'.format(
np.mean(rew), np.std(rew), agent_id))
else:
print(rets)
print('Average, std: {}, {}'.format(
np.mean(rets), np.std(rets)))
print("\nSpeed, mean (m/s):")
print(mean_speed)
print('Average, std: {}, {}'.format(np.mean(mean_speed), np.std(
mean_speed)))
print("\nSpeed, std (m/s):")
print(std_speed)
print('Average, std: {}, {}'.format(np.mean(std_speed), np.std(
std_speed)))
# Compute arrival rate of vehicles in the last 500 sec of the run
print("\nOutflows (veh/hr):")
print(final_outflows)
print('Average, std: {}, {}'.format(np.mean(final_outflows),
np.std(final_outflows)))
# Compute departure rate of vehicles in the last 500 sec of the run
print("Inflows (veh/hr):")
print(final_inflows)
print('Average, std: {}, {}'.format(np.mean(final_inflows),
np.std(final_inflows)))
# Compute throughput efficiency in the last 500 sec of the
print("Throughput efficiency (veh/hr):")
print(throughput_efficiency)
print('Average, std: {}, {}'.format(np.mean(throughput_efficiency),
np.std(throughput_efficiency)))
# terminate the environment
env.unwrapped.terminate()
# if prompted, convert the emission file into a csv file
if args.gen_emission:
time.sleep(0.1)
dir_path = os.path.dirname(os.path.realpath(__file__))
emission_filename = '{0}-emission.xml'.format(env.network.name)
emission_path = \
'{0}/test_time_rollout/{1}'.format(dir_path, emission_filename)
# convert the emission file into a csv file
emission_to_csv(emission_path)
# print the location of the emission csv file
emission_path_csv = emission_path[:-4] + ".csv"
print("\nGenerated emission file at " + emission_path_csv)
# delete the .xml version of the emission file
os.remove(emission_path)
# if we wanted to save the render, here we create the movie
if args.save_render:
dirs = os.listdir(os.path.expanduser('~')+'/flow_rendering')
# Ignore hidden files
dirs = [d for d in dirs if d[0] != '.']
dirs.sort(key=lambda date: datetime.strptime(date, "%Y-%m-%d-%H%M%S"))
recent_dir = dirs[-1]
# create the movie
movie_dir = os.path.expanduser('~') + '/flow_rendering/' + recent_dir
save_dir = os.path.expanduser('~') + '/flow_movies'
if not os.path.exists(save_dir):
os.mkdir(save_dir)
os_cmd = "cd " + movie_dir + " && ffmpeg -i frame_%06d.png"
os_cmd += " -pix_fmt yuv420p " + dirs[-1] + ".mp4"
os_cmd += "&& cp " + dirs[-1] + ".mp4 " + save_dir + "/"
os.system(os_cmd)
def visualizer_rllib(args):
"""Visualizer for RLlib experiments.
This function takes args (see function create_parser below for
more detailed information on what information can be fed to this
visualizer), and renders the experiment associated with it.
"""
result_dir = args.result_dir if args.result_dir[-1] != '/' \
else args.result_dir[:-1]
config = get_rllib_config(result_dir)
# check if we have a multiagent environment but in a
# backwards compatible way
if config.get('multiagent', {}).get('policies', None):
multiagent = True
pkl = get_rllib_pkl(result_dir)
config['multiagent'] = pkl['multiagent']
else:
multiagent = False
# Run on only one cpu for rendering purposes
config['num_workers'] = 0
flow_params = get_flow_params(config)
# hack for old pkl files
# TODO(ev) remove eventually
sim_params = flow_params['sim']
setattr(sim_params, 'num_clients', 1)
# for hacks for old pkl files TODO: remove eventually
if not hasattr(sim_params, 'use_ballistic'):
sim_params.use_ballistic = False
# Determine agent and checkpoint
config_run = config['env_config']['run'] if 'run' in config['env_config'] \
else None
if args.run and config_run:
if args.run != config_run:
print('visualizer_rllib.py: error: run argument ' +
'\'{}\' passed in '.format(args.run) +
'differs from the one stored in params.json ' +
'\'{}\''.format(config_run))
sys.exit(1)
if args.run:
agent_cls = get_agent_class(args.run)
elif config_run:
agent_cls = get_agent_class(config_run)
else:
print('visualizer_rllib.py: error: could not find flow parameter '
'\'run\' in params.json, '
'add argument --run to provide the algorithm or model used '
'to train the results\n e.g. '
'python ./visualizer_rllib.py /tmp/ray/result_dir 1 --run PPO')
sys.exit(1)
sim_params.restart_instance = True
dir_path = os.path.dirname(os.path.realpath(__file__))
emission_path = '{0}/test_time_rollout/'.format(dir_path)
sim_params.emission_path = emission_path if args.gen_emission else None
# pick your rendering mode
if args.render_mode == 'sumo_web3d':
sim_params.num_clients = 2
sim_params.render = False
elif args.render_mode == 'drgb':
sim_params.render = 'drgb'
sim_params.pxpm = 4
elif args.render_mode == 'sumo_gui':
sim_params.render = False # will be set to True below
elif args.render_mode == 'no_render':
sim_params.render = False
if args.save_render:
if args.render_mode != 'sumo_gui':
sim_params.render = 'drgb'
sim_params.pxpm = 4
sim_params.save_render = True
# Create and register a gym+rllib env
create_env, env_name = make_create_env(params=flow_params, version=0)
register_env(env_name, create_env)
# check if the environment is a single or multiagent environment, and
# get the right address accordingly
# single_agent_envs = [env for env in dir(flow.envs)
# if not env.startswith('__')]
# if flow_params['env_name'] in single_agent_envs:
# env_loc = 'flow.envs'
# else:
# env_loc = 'flow.envs.multiagent'
# Start the environment with the gui turned on and a path for the
# emission file
env_params = flow_params['env']
env_params.restart_instance = False
if args.evaluate:
env_params.evaluate = True
# lower the horizon if testing
if args.horizon:
config['horizon'] = args.horizon
env_params.horizon = args.horizon
# create the agent that will be used to compute the actions
agent = agent_cls(env=env_name, config=config)
checkpoint = result_dir + '/checkpoint_' + args.checkpoint_num
checkpoint = checkpoint + '/checkpoint-' + args.checkpoint_num
agent.restore(checkpoint)
if hasattr(agent, "local_evaluator") and \
os.environ.get("TEST_FLAG") != 'True':
env = agent.local_evaluator.env
else:
env = gym.make(env_name)
if args.render_mode == 'sumo_gui':
env.sim_params.render = True # set to True after initializing agent and env
if multiagent:
rets = {}
# map the agent id to its policy
policy_map_fn = config['multiagent']['policy_mapping_fn'].func
for key in config['multiagent']['policies'].keys():
rets[key] = []
else:
rets = []
if config['model']['use_lstm']:
use_lstm = True
if multiagent:
state_init = {}
# map the agent id to its policy
policy_map_fn = config['multiagent']['policy_mapping_fn'].func
size = config['model']['lstm_cell_size']
for key in config['multiagent']['policies'].keys():
state_init[key] = [
np.zeros(size, np.float32),
np.zeros(size, np.float32)
]
else:
state_init = [
np.zeros(config['model']['lstm_cell_size'], np.float32),
np.zeros(config['model']['lstm_cell_size'], np.float32)
]
else:
use_lstm = False
# if restart_instance, don't restart here because env.reset will restart later
if not sim_params.restart_instance:
env.restart_simulation(sim_params=sim_params, render=sim_params.render)
# Simulate and collect metrics
final_outflows = []
final_inflows = []
mean_speed = []
std_speed = []
#bmil edit
acc_rollout = []
power = 0
for i in range(args.num_rollouts):
vel = []
# bmil list for collecting data
timerange = []
vel_dict = defaultdict(list)
rl_acc = []
state = env.reset()
if multiagent:
ret = {key: [0] for key in rets.keys()}
else:
ret = 0
for _ in range(env_params.horizon):
vehicles = env.unwrapped.k.vehicle
# speeds = vehicles.get_speed(vehicles.get_ids())
ids = vehicles.get_ids()
speeds = vehicles.get_speed(ids)
# BMIL EDIT FOR COLLECTING DATA OF ACCELERATION AND VELOCITY
rl = vehicles.get_rl_ids()[0]
act = vehicles.get_realized_accel(rl)
rl_acc.append(act or 0)
timerange.append(vehicles.get_timestep(ids[-1]) / 100000)
# only include non-empty speeds
if speeds:
vel.append(np.mean(speeds))
# bmil edit
for veh_id, speed in zip(ids, speeds):
vel_dict[veh_id].append(speed)
if vehicles.get_timestep(ids[0]) >= 100000:
M = 1200 # mass of average sized vehicle (kg)
g = 9.81 # gravitational acceleration (m/s^2)
Cr = 0.005 # rolling resistance coefficient
Ca = 0.3 # aerodynamic drag coefficient
rho = 1.225 # air density (kg/m^3)
A = 2.6 # vehicle cross sectional area (m^2)
speed = vehicles.get_speed(veh_id)
prev_speed = vehicles.get_previous_speed(veh_id)
accel = abs(speed - prev_speed) / env.sim_step
power += M * speed * accel + M * g * Cr * speed + 0.5 * rho * A * Ca * speed**3
if multiagent:
action = {}
for agent_id in state.keys():
if use_lstm:
action[agent_id], state_init[agent_id], logits = \
agent.compute_action(
state[agent_id], state=state_init[agent_id],
policy_id=policy_map_fn(agent_id))
else:
action[agent_id] = agent.compute_action(
state[agent_id], policy_id=policy_map_fn(agent_id))
else:
action = agent.compute_action(state)
# BMIL EDIT FOR COLLECTING DATA FROM 100000 TO 375000
# Because 0 - 75000 steps are warm up and 75000 - 100000 steps are process of stabilizing
if vehicles.get_timestep(rl) >= 100000:
acc_rollout.append(act)
state, reward, done, _ = env.step(action)
if multiagent:
for actor, rew in reward.items():
ret[policy_map_fn(actor)][0] += rew
else:
ret += reward
if multiagent and done['__all__']:
break
if not multiagent and done:
break
if multiagent:
for key in rets.keys():
rets[key].append(ret[key])
else:
rets.append(ret)
outflow = vehicles.get_outflow_rate(500)
final_outflows.append(outflow)
inflow = vehicles.get_inflow_rate(500)
final_inflows.append(inflow)
if np.all(np.array(final_inflows) > 1e-5):
throughput_efficiency = [
x / y for x, y in zip(final_outflows, final_inflows)
]
else:
throughput_efficiency = [0] * len(final_inflows)
mean_speed.append(np.mean(vel))
std_speed.append(np.std(vel))
if multiagent:
for agent_id, rew in rets.items():
print('Round {}, Return: {} for agent {}'.format(
i, ret, agent_id))
else:
print('Round {}, Return: {}'.format(i, ret))
# BMIL EDIT FOR PLOT DATA
veh = list(vel_dict.keys())
plt.subplot(3, 1, 1)
plt.title('Results')
for v in veh[:-1]:
plt.plot(timerange, vel_dict[v])
plt.xlabel('timestep(s)')
plt.ylabel('speed(m/s)')
plt.legend(veh[:-1], fontsize=9)
plt.grid(True)
# plt.show()
plt.subplot(3, 1, 2)
plt.plot(timerange, vel_dict[veh[-1]], color='r')
plt.xlabel('timestep(s)')
plt.ylabel('speed(m/s)')
plt.legend(['lc'] + veh[-1:])
plt.grid(True)
plt.subplot(3, 1, 3)
plt.plot(timerange, rl_acc, color='b')
plt.xlabel('timestep(s)')
plt.ylabel('acceleration(m/s^2)')
plt.grid(True)
# BASE_DIR = '/home/bmil/BMIL_FLOW_CODE/Graph/'
# plt.savefig(f'{BASE_DIR}{"__".join(name)}', dpi=400)
plt.show()
# BMIL EDIT FOR COMPUTING ACCELERATION's MEAN AND VAR
acc_rollout1 = [accarr for accarr in acc_rollout]
mean_acc_rollout = [np.mean(acc_rollout1)]
variance_acc_rollout = [np.var(acc_rollout1)]
print('==== Summary of results ====')
print("Return:")
print(mean_speed)
if multiagent:
for agent_id, rew in rets.items():
print('For agent', agent_id)
print(rew)
print('Average, std return: {}, {} for agent {}'.format(
np.mean(rew), np.std(rew), agent_id))
else:
print(rets)
print('Average, std: {}, {}'.format(np.mean(rets), np.std(rets)))
print("\nSpeed, mean (m/s):")
print(mean_speed)
print('Average, std: {}, {}'.format(np.mean(mean_speed),
np.std(mean_speed)))
print("\nSpeed, std (m/s):")
print(std_speed)
print('Average, std: {}, {}'.format(np.mean(std_speed), np.std(std_speed)))
# BMIL Edit FOR PRINT ACCEL's MEAN AND VAR
print("\nAccel, mean (m/s^2):")
print(mean_acc_rollout)
print("\nAccel, var (m/s^2):")
print(variance_acc_rollout)
print("\nTotal Power Consumption (kgᐧm^2/s^3):")
print(power)
# Compute arrival rate of vehicles in the last 500 sec of the run
print("\nOutflows (veh/hr):")
print(final_outflows)
print('Average, std: {}, {}'.format(np.mean(final_outflows),
np.std(final_outflows)))
# Compute departure rate of vehicles in the last 500 sec of the run
print("Inflows (veh/hr):")
print(final_inflows)
print('Average, std: {}, {}'.format(np.mean(final_inflows),
np.std(final_inflows)))
# Compute throughput efficiency in the last 500 sec of the
print("Throughput efficiency (veh/hr):")
print(throughput_efficiency)
print('Average, std: {}, {}'.format(np.mean(throughput_efficiency),
np.std(throughput_efficiency)))
# terminate the environment
env.unwrapped.terminate()
# if prompted, convert the emission file into a csv file
if args.gen_emission:
time.sleep(0.1)
dir_path = os.path.dirname(os.path.realpath(__file__))
emission_filename = '{0}-emission.xml'.format(env.network.name)
emission_path = \
'{0}/test_time_rollout/{1}'.format(dir_path, emission_filename)
# convert the emission file into a csv file
emission_to_csv(emission_path)
# print the location of the emission csv file
emission_path_csv = emission_path[:-4] + ".csv"
print("\nGenerated emission file at " + emission_path_csv)
# delete the .xml version of the emission file
os.remove(emission_path)
action='store_true',
help='Specifies whether to use the "evaluate" reward for the environment.')
if __name__ == "__main__":
args = parser.parse_args()
result_dir = args.result_dir if args.result_dir[-1] != '/' \
else args.result_dir[:-1]
config = get_rllib_config(result_dir)
# Run on only one cpu for rendering purposes
ray.init(num_cpus=1)
config["num_workers"] = 1
flow_params = get_flow_params(config)
# Create and register a gym+rllib env
create_env, env_name = make_create_env(params=flow_params,
version=0,
render=False)
register_env(env_name, create_env)
agent_cls = get_agent_class(args.run)
agent = agent_cls(env=env_name, config=config)
checkpoint = result_dir + '/checkpoint-' + args.checkpoint_num
agent._restore(checkpoint)
# Recreate the scenario from the pickled parameters
exp_tag = flow_params["exp_tag"]
net_params = flow_params['net']
def visualizer_rllib(args):
result_dir = args.result_dir if args.result_dir[-1] != '/' \
else args.result_dir[:-1]
config = get_rllib_config(result_dir)
# Run on only one cpu for rendering purposes
config['num_workers'] = 1
flow_params = get_flow_params(config)
# Create and register a gym+rllib env
create_env, env_name = make_create_env(params=flow_params,
version=0,
render=False)
register_env(env_name, create_env)
# Determine agent and checkpoint
config_run = config['env_config']['run'] if 'run' in config['env_config'] \
else None
if (args.run and config_run):
if (args.run != config_run):
print('visualizer_rllib.py: error: run argument ' +
'\'{}\' passed in '.format(args.run) +
'differs from the one stored in params.json ' +
'\'{}\''.format(config_run))
sys.exit(1)
if (args.run):
agent_cls = get_agent_class(args.run)
elif (config_run):
agent_cls = get_agent_class(config_run)
else:
print('visualizer_rllib.py: error: could not find flow parameter '
'\'run\' in params.json, '
'add argument --run to provide the algorithm or model used '
'to train the results\n e.g. '
'python ./visualizer_rllib.py /tmp/ray/result_dir 1 --run PPO')
sys.exit(1)
agent = agent_cls(env=env_name, config=config)
checkpoint = result_dir + '/checkpoint_' + args.checkpoint_num
checkpoint = checkpoint + '/checkpoint-' + args.checkpoint_num
agent.restore(checkpoint)
# Recreate the scenario from the pickled parameters
exp_tag = flow_params['exp_tag']
net_params = flow_params['net']
vehicles = flow_params['veh']
initial_config = flow_params['initial']
module = __import__('flow.scenarios', fromlist=[flow_params['scenario']])
scenario_class = getattr(module, flow_params['scenario'])
scenario = scenario_class(name=exp_tag,
vehicles=vehicles,
net_params=net_params,
initial_config=initial_config)
# Start the environment with the gui turned on and a path for the
# emission file
module = __import__('flow.envs', fromlist=[flow_params['env_name']])
env_class = getattr(module, flow_params['env_name'])
env_params = flow_params['env']
if args.evaluate:
env_params.evaluate = True
sumo_params = flow_params['sumo']
if args.no_render:
sumo_params.render = False
else:
sumo_params.render = True
sumo_params.emission_path = './test_time_rollout/'
env = ModelCatalog.get_preprocessor_as_wrapper(
env_class(env_params=env_params,
sumo_params=sumo_params,
scenario=scenario))
# Run the environment in the presence of the pre-trained RL agent for the
# requested number of time steps / rollouts
rets = []
final_outflows = []
mean_speed = []
for i in range(args.num_rollouts):
vel = []
state = env.reset()
ret = 0
for _ in range(env_params.horizon):
vehicles = env.unwrapped.vehicles
vel.append(np.mean(vehicles.get_speed(vehicles.get_ids())))
action = agent.compute_action(state)
state, reward, done, _ = env.step(action)
ret += reward
if done:
break
rets.append(ret)
outflow = vehicles.get_outflow_rate(500)
final_outflows.append(outflow)
mean_speed.append(np.mean(vel))
print('Round {}, Return: {}'.format(i, ret))
print('Average, std return: {}, {}'.format(np.mean(rets), np.std(rets)))
print('Average, std speed: {}, {}'.format(np.mean(mean_speed),
np.std(mean_speed)))
print('Average, std outflow: {}, {}'.format(np.mean(final_outflows),
np.std(final_outflows)))
# terminate the environment
env.unwrapped.terminate()
# if prompted, convert the emission file into a csv file
if args.emission_to_csv:
dir_path = os.path.dirname(os.path.realpath(__file__))
emission_filename = '{0}-emission.xml'.format(scenario.name)
emission_path = \
'{0}/test_time_rollout/{1}'.format(dir_path, emission_filename)
emission_to_csv(emission_path)
def visualizer_rllib(args):
result_dir = args.result_dir if args.result_dir[-1] != '/' \
else args.result_dir[:-1]
# config = get_rllib_config(result_dir + '/..')
# pkl = get_rllib_pkl(result_dir + '/..')
config = get_rllib_config(result_dir)
# TODO(ev) backwards compatibility hack
try:
pkl = get_rllib_pkl(result_dir)
except Exception:
pass
# check if we have a multiagent scenario but in a
# backwards compatible way
if config.get('multiagent', {}).get('policy_graphs', {}):
multiagent = True
config['multiagent'] = pkl['multiagent']
else:
multiagent = False
# Run on only one cpu for rendering purposes
config['num_workers'] = 0
flow_params = get_flow_params(config)
# hack for old pkl files
# TODO(ev) remove eventually
sumo_params = flow_params['sumo']
setattr(sumo_params, 'num_clients', 1)
# Create and register a gym+rllib env
create_env, env_name = make_create_env(params=flow_params,
version=0,
render=False)
register_env(env_name, create_env)
# Determine agent and checkpoint
config_run = config['env_config']['run'] if 'run' in config['env_config'] \
else None
if (args.run and config_run):
if (args.run != config_run):
print('visualizer_rllib.py: error: run argument ' +
'\'{}\' passed in '.format(args.run) +
'differs from the one stored in params.json ' +
'\'{}\''.format(config_run))
sys.exit(1)
if (args.run):
agent_cls = get_agent_class(args.run)
elif (config_run):
agent_cls = get_agent_class(config_run)
else:
print('visualizer_rllib.py: error: could not find flow parameter '
'\'run\' in params.json, '
'add argument --run to provide the algorithm or model used '
'to train the results\n e.g. '
'python ./visualizer_rllib.py /tmp/ray/result_dir 1 --run PPO')
sys.exit(1)
sumo_params.restart_instance = False
sumo_params.emission_path = './test_time_rollout/'
# pick your rendering mode
if args.render_mode == 'sumo-web3d':
sumo_params.num_clients = 2
sumo_params.render = False
elif args.render_mode == 'drgb':
sumo_params.render = 'drgb'
sumo_params.pxpm = 4
elif args.render_mode == 'sumo-gui':
sumo_params.render = False
elif args.render_mode == 'no-render':
sumo_params.render = False
if args.save_render:
sumo_params.render = 'drgb'
sumo_params.pxpm = 4
sumo_params.save_render = True
# Recreate the scenario from the pickled parameters
exp_tag = flow_params['exp_tag']
net_params = flow_params['net']
vehicles = flow_params['veh']
initial_config = flow_params['initial']
module = __import__('flow.scenarios', fromlist=[flow_params['scenario']])
scenario_class = getattr(module, flow_params['scenario'])
scenario = scenario_class(name=exp_tag,
vehicles=vehicles,
net_params=net_params,
initial_config=initial_config)
# Start the environment with the gui turned on and a path for the
# emission file
module = __import__('flow.envs', fromlist=[flow_params['env_name']])
env_class = getattr(module, flow_params['env_name'])
env_params = flow_params['env']
env_params.restart_instance = False
if args.evaluate:
env_params.evaluate = True
# create the agent that will be used to compute the actions
agent = agent_cls(env=env_name, config=config)
checkpoint = result_dir + '/checkpoint_' + args.checkpoint_num
checkpoint = checkpoint + '/checkpoint-' + args.checkpoint_num
agent.restore(checkpoint)
env = ModelCatalog.get_preprocessor_as_wrapper(
env_class(env_params=env_params,
sumo_params=sumo_params,
scenario=scenario))
import matplotlib.pyplot as plt
from matplotlib import cm
from matplotlib.ticker import LinearLocator, FormatStrFormatter
fig = plt.figure()
h = np.linspace(0, 60, 100)
Deltav = np.linspace(-6, 12, 100)
Headway, DELTAV = np.meshgrid(h, Deltav)
# fix v=20m/s
xn, yn = Headway.shape
geta = np.array(Headway)
for xk in range(xn):
for yk in range(yn):
#输入状态
#Headway[xk,yk]
#DELTAV[xk,yk]
geta[xk, yk] = agent.compute_action(
np.array(
[3.8 / 30, DELTAV[xk, yk] / 30, Headway[xk, yk] / 260]))
surf = plt.contourf(DELTAV, Headway, geta, 20, cmap=cm.coolwarm)
plt.colorbar()
#C = plt.contour(DELTAV, Headway, geta, 20, colors='black')
# plt.clabel(C, inline = True, fontsize = 10)
plt.show()
# terminate the environment
env.unwrapped.terminate()
# if prompted, convert the emission file into a csv file
if args.emission_to_csv:
dir_path = os.path.dirname(os.path.realpath(__file__))
emission_filename = '{0}-emission.xml'.format(scenario.name)
emission_path = \
'{0}/test_time_rollout/{1}'.format(dir_path, emission_filename)
emission_to_csv(emission_path)
# if we wanted to save the render, here we create the movie
'''
def visualizer_rllib(args):
"""Visualizer for RLlib experiments.
This function takes args (see function create_parser below for
more detailed information on what information can be fed to this
visualizer), and renders the experiment associated with it.
"""
result_dir = args.result_dir if args.result_dir[-1] != '/' \
else args.result_dir[:-1]
config = get_rllib_config(result_dir)
# check if we have a multiagent environment but in a
# backwards compatible way
if config.get('multiagent', {}).get('policies', None):
multiagent = True
pkl = get_rllib_pkl(result_dir)
config['multiagent'] = pkl['multiagent']
else:
multiagent = False
# Run on only one cpu for rendering purposes
config['num_workers'] = 0
flow_params = get_flow_params(config)
# hack for old pkl files
# TODO(ev) remove eventually
sim_params = flow_params['sim']
setattr(sim_params, 'num_clients', 1)
# Determine agent and checkpoint
config_run = config['env_config']['run'] if 'run' in config['env_config'] \
else None
if args.run and config_run:
if args.run != config_run:
print('visualizer_rllib.py: error: run argument ' +
'\'{}\' passed in '.format(args.run) +
'differs from the one stored in params.json ' +
'\'{}\''.format(config_run))
sys.exit(1)
if args.run:
agent_cls = get_agent_class(args.run)
elif config_run:
agent_cls = get_agent_class(config_run)
else:
print('visualizer_rllib.py: error: could not find flow parameter '
'\'run\' in params.json, '
'add argument --run to provide the algorithm or model used '
'to train the results\n e.g. '
'python ./visualizer_rllib.py /tmp/ray/result_dir 1 --run PPO')
sys.exit(1)
sim_params.restart_instance = True
# specify emission file path
dir_path = os.path.dirname(os.path.realpath(__file__))
emission_path = '{0}/test_time_rollout/'.format(dir_path)
sim_params.emission_path = emission_path if args.gen_emission else None
# pick your rendering mode
if args.render_mode == 'sumo_web3d':
sim_params.num_clients = 2
sim_params.render = False
elif args.render_mode == 'drgb':
sim_params.render = 'drgb'
sim_params.pxpm = 4
elif args.render_mode == 'sumo_gui':
sim_params.render = False # this will be set to true after creating agent and gym
print('NOTE: With render mode {}, an extra instance of the SUMO GUI '
'will display before the GUI for visualizing the result. Click '
'the green Play arrow to continue.'.format(args.render_mode))
elif args.render_mode == 'no_render':
sim_params.render = False
if args.save_render:
sim_params.render = 'drgb'
sim_params.pxpm = 4
sim_params.save_render = True
# Create and register a gym+rllib env
create_env, env_name = make_create_env(params=flow_params, version=0)
register_env(env_name, create_env)
# Start the environment with the gui turned on and a path for the
# emission file
env_params = flow_params['env']
env_params.restart_instance = False
if args.evaluate:
env_params.evaluate = True
# lower the horizon if testing
if args.horizon:
config['horizon'] = args.horizon
env_params.horizon = args.horizon
# create the agent that will be used to compute the actions
agent = agent_cls(env=env_name, config=config)
checkpoint = result_dir + '/checkpoint_' + args.checkpoint_num
checkpoint = checkpoint + '/checkpoint-' + args.checkpoint_num
agent.restore(checkpoint)
if hasattr(agent, "local_evaluator") and \
os.environ.get("TEST_FLAG") != 'True':
env = agent.local_evaluator.env
else:
env = gym.make(env_name)
if args.render_mode == 'sumo_gui':
env.sim_params.render = True # set to true after initializing agent and env
# if restart_instance, don't restart here because env.reset will restart later
if not sim_params.restart_instance:
env.restart_simulation(sim_params=sim_params)
use_lstm = config['model'].get('use_lstm', False)
if use_lstm:
state_size = config['model']['lstm_cell_size']
lstm_state = [np.zeros(state_size), np.zeros(state_size)]
if multiagent:
lstm_state = {
key: deepcopy(lstm_state)
for key in config['multiagent']['policies'].keys()
}
rewards = []
if multiagent:
rewards = defaultdict(list)
policy_map_fn = config['multiagent']['policy_mapping_fn'].func
# Simulate and collect metrics
final_outflows = []
final_inflows = []
mean_speed = []
std_speed = []
for i in range(args.num_rollouts):
obs = env.reset()
kv = env.k.vehicle
rollout_speeds = []
rollout_reward = 0
if multiagent:
rollout_reward = defaultdict(int)
for _ in range(env_params.horizon):
rollout_speeds.append(np.mean(kv.get_speed(kv.get_ids())))
if multiagent:
action = {}
for agent_id in obs.keys():
if use_lstm:
action[agent_id], obs[
agent_id], logits = agent.compute_action(
obs[agent_id],
obs=lstm_state[agent_id],
policy_id=policy_map_fn(agent_id))
else:
action[agent_id] = agent.compute_action(
obs[agent_id], policy_id=policy_map_fn(agent_id))
else:
action = agent.compute_action(obs)
obs, reward, done, _ = env.step(action)
if multiagent:
done = done['__all__']
for agent_id, agent_reward in reward.items():
rollout_reward[policy_map_fn(agent_id)] += agent_reward
else:
rollout_reward += reward
if done:
break
if multiagent:
for agent_id, reward in rollout_reward.items():
rewards[agent_id].append(reward)
print('rollout %s, agent %s reward: %.5g' %
(i, agent_id, reward))
else:
rewards.append(rollout_reward)
print('rollout %s, reward: %.5g' % (i, rollout_reward))
mean_speed.append(np.nanmean(rollout_speeds))
std_speed.append(np.nanstd(rollout_speeds))
# Compute rate of inflow / outflow in the last 500 steps
final_outflows.append(kv.get_outflow_rate(500))
final_inflows.append(kv.get_inflow_rate(500))
print(
'\n==== Summary of results: mean (std) [rollout1, rollout2, ...] ====')
mean, std = np.mean, np.std
if multiagent:
for agent_id, agent_rewards in rewards.items():
print('agent %s rewards: %.4g (%.4g) %s' %
(agent_id, mean(agent_rewards), std(agent_rewards),
agent_rewards))
else:
print('rewards: %.4g (%.4g) %s' %
(mean(rewards), std(rewards), rewards))
print('mean speeds (m/s): %.4g (%.4g) %s' %
(mean(mean_speed), std(mean_speed), mean_speed))
print('std speeds: %.4g (%.4g) %s' %
(mean(std_speed), std(std_speed), std_speed))
print('inflows (veh/hr): %.4g (%.4g) %s' %
(mean(final_inflows), std(final_inflows), final_inflows))
print('outflows (veh/hr): %.4g (%.4g) %s' %
(mean(final_outflows), std(final_outflows), final_outflows))
# Compute throughput efficiency in the last 500 sec of the
throughput = [o / i for o, i in zip(final_outflows, final_inflows)]
print('throughput efficiency: %.4g (%.4g) %s' %
(mean(throughput), std(throughput), throughput))
# terminate the environment
env.terminate()
# if prompted, convert the emission file into a csv file
if args.gen_emission:
time.sleep(0.1)
dir_path = os.path.dirname(os.path.realpath(__file__))
emission_filename = '{0}-emission.xml'.format(env.network.name)
emission_path = \
'{0}/test_time_rollout/{1}'.format(dir_path, emission_filename)
# convert the emission file into a csv file
emission_to_csv(emission_path)
# print the location of the emission csv file
emission_path_csv = emission_path[:-4] + ".csv"
print("\nGenerated emission file at " + emission_path_csv)
# delete the .xml version of the emission file
os.remove(emission_path)
# if we wanted to save the render, here we create the movie
if args.save_render:
dirs = os.listdir(os.path.expanduser('~') + '/flow_rendering')
# Ignore hidden files
dirs = [d for d in dirs if d[0] != '.']
dirs.sort(key=lambda date: datetime.strptime(date, "%Y-%m-%d-%H%M%S"))
recent_dir = dirs[-1]
# create the movie
movie_dir = os.path.expanduser('~') + '/flow_rendering/' + recent_dir
save_dir = os.path.expanduser('~') + '/flow_movies'
if not os.path.exists(save_dir):
os.mkdir(save_dir)
os_cmd = "cd " + movie_dir + " && ffmpeg -i frame_%06d.png"
os_cmd += " -pix_fmt yuv420p " + dirs[-1] + ".mp4"
os_cmd += "&& cp " + dirs[-1] + ".mp4 " + save_dir + "/"
os.system(os_cmd)
def visualizer_rllib(args):
result_dir = args.result_dir if args.result_dir[-1] != '/' \
else args.result_dir[:-1]
# config = get_rllib_config(result_dir + '/..')
# pkl = get_rllib_pkl(result_dir + '/..')
config = get_rllib_config(result_dir)
# TODO(ev) backwards compatibility hack
try:
pkl = get_rllib_pkl(result_dir)
except Exception:
pass
# check if we have a multiagent scenario but in a
# backwards compatible way
if config.get('multiagent', {}).get('policy_graphs', {}):
multiagent = True
config['multiagent'] = pkl['multiagent']
else:
multiagent = False
# Run on only one cpu for rendering purposes
config['num_workers'] = 0
flow_params = get_flow_params(config)
# hack for old pkl files
# TODO(ev) remove eventually
sumo_params = flow_params['sumo']
setattr(sumo_params, 'num_clients', 1)
# Create and register a gym+rllib env
create_env, env_name = make_create_env(
params=flow_params, version=0, render=False)
register_env(env_name, create_env)
# Determine agent and checkpoint
config_run = config['env_config']['run'] if 'run' in config['env_config'] \
else None
if (args.run and config_run):
if (args.run != config_run):
print('visualizer_rllib.py: error: run argument '
+ '\'{}\' passed in '.format(args.run)
+ 'differs from the one stored in params.json '
+ '\'{}\''.format(config_run))
sys.exit(1)
if (args.run):
agent_cls = get_agent_class(args.run)
elif (config_run):
agent_cls = get_agent_class(config_run)
else:
print('visualizer_rllib.py: error: could not find flow parameter '
'\'run\' in params.json, '
'add argument --run to provide the algorithm or model used '
'to train the results\n e.g. '
'python ./visualizer_rllib.py /tmp/ray/result_dir 1 --run PPO')
sys.exit(1)
sumo_params.restart_instance = False
sumo_params.emission_path = './test_time_rollout/'
# pick your rendering mode
if args.render_mode == 'sumo-web3d':
sumo_params.num_clients = 2
sumo_params.render = False
elif args.render_mode == 'drgb':
sumo_params.render = 'drgb'
sumo_params.pxpm = 4
elif args.render_mode == 'sumo-gui':
sumo_params.render = False
elif args.render_mode == 'no-render':
sumo_params.render = False
if args.save_render:
sumo_params.render = 'drgb'
sumo_params.pxpm = 4
sumo_params.save_render = True
# Recreate the scenario from the pickled parameters
exp_tag = flow_params['exp_tag']
net_params = flow_params['net']
vehicles = flow_params['veh']
initial_config = flow_params['initial']
module = __import__('flow.scenarios', fromlist=[flow_params['scenario']])
scenario_class = getattr(module, flow_params['scenario'])
scenario = scenario_class(
name=exp_tag,
vehicles=vehicles,
net_params=net_params,
initial_config=initial_config)
# Start the environment with the gui turned on and a path for the
# emission file
module = __import__('flow.envs', fromlist=[flow_params['env_name']])
env_class = getattr(module, flow_params['env_name'])
env_params = flow_params['env']
env_params.restart_instance = False
if args.evaluate:
env_params.evaluate = True
# lower the horizon if testing
if args.horizon:
config['horizon'] = 6000 #可以考虑改成6000
env_params.horizon = 6000
# create the agent that will be used to compute the actions
agent = agent_cls(env=env_name, config=config)
checkpoint = result_dir + '/checkpoint_' + args.checkpoint_num
checkpoint = checkpoint + '/checkpoint-' + args.checkpoint_num
agent.restore(checkpoint)
env = ModelCatalog.get_preprocessor_as_wrapper(env_class(
env_params=env_params, sumo_params=sumo_params, scenario=scenario))
if multiagent:
rets = {}
# map the agent id to its policy
policy_map_fn = config['multiagent']['policy_mapping_fn'].func
for key in config['multiagent']['policy_graphs'].keys():
rets[key] = []
else:
rets = []
final_outflows = []
mean_speed = []
for i in range(1):#args.num_rollouts):
vel = []
state = env.reset()
done = False
if multiagent:
ret = {key: [0] for key in rets.keys()}
else:
ret = 0
for _ in range(env_params.horizon):
vehicles = env.unwrapped.vehicles
vel.append(vehicles.get_speed(vehicles.get_ids())[0])#这里是整体平均速度
if multiagent:
action = {}
for agent_id in state.keys():
action[agent_id] = agent.compute_action(
state[agent_id], policy_id=policy_map_fn(agent_id))
else:
print(type(state),state)
action = agent.compute_action(state)
print(type(action),action)
state, reward, done, _ = env.step(action)
if multiagent:
for actor, rew in reward.items():
ret[policy_map_fn(actor)][0] += rew
else:
ret += reward
if multiagent and done['__all__']:
break
if not multiagent and done:
break
if multiagent:
for key in rets.keys():
rets[key].append(ret[key])
else:
rets.append(ret)
outflow = vehicles.get_outflow_rate(500)
final_outflows.append(outflow)
#mean_speed.append(np.mean(vel))#注意这里
print('Round {}, Return: {}'.format(i, ret))
if multiagent:
for agent_id, rew in rets.items():
print('Average, std return: {}, {} for agent {}'.format(
np.mean(rew), np.std(rew), agent_id))
else:
print('Average, std return: {}, {}'.format(
np.mean(rets), np.std(rets)))
print('Average, std speed: {}, {}'.format(
np.mean(mean_speed), np.std(mean_speed)))
print('Average, std outflow: {}, {}'.format(
np.mean(final_outflows), np.std(final_outflows)))
import matplotlib.pyplot as plt
plt.figure()
plt.plot(vel)
plt.show()
# terminate the environment
env.unwrapped.terminate()
# if prompted, convert the emission file into a csv file
if args.emission_to_csv:
dir_path = os.path.dirname(os.path.realpath(__file__))
emission_filename = '{0}-emission.xml'.format(scenario.name)
emission_path = \
'{0}/test_time_rollout/{1}'.format(dir_path, emission_filename)
emission_to_csv(emission_path)
# if we wanted to save the render, here we create the movie
'''
default=0,
help='The minimum speed in the color range.')
parser.add_argument('--start',
type=float,
default=0,
help='initial time (in sec) in the plot.')
parser.add_argument('--stop',
type=float,
default=float('inf'),
help='final time (in sec) in the plot.')
args = parser.parse_args()
# flow_params is imported as a dictionary
if '.json' in args.flow_params:
flow_params = get_flow_params(args.flow_params)
else:
module = __import__("examples.exp_configs.non_rl",
fromlist=[args.flow_params])
flow_params = getattr(module, args.flow_params).flow_params
# import data from the emission.csv file
emission_data = import_data_from_emission(args.emission_path)
# compute the position and speed for all vehicles at all times
pos, speed, time = get_time_space_data(emission_data, flow_params)
# some plotting parameters
cdict = {
'red': ((0, 0, 0), (0.2, 1, 1), (0.6, 1, 1), (1, 0, 0)),
'green': ((0, 0, 0), (0.2, 0, 0), (0.6, 1, 1), (1, 1, 1)),
def visualizer_rllib(args):
"""Visualizer for RLlib experiments.
This function takes args (see function create_parser below for
more detailed information on what information can be fed to this
visualizer), and renders the experiment associated with it.
"""
result_dir = args.result_dir if args.result_dir[-1] != '/' \
else args.result_dir[:-1]
config = get_rllib_pkl(result_dir)
# Run on only one cpu for rendering purposes
config['num_workers'] = 0
flow_params = get_flow_params(config)
# Determine agent and checkpoint
config_run = config['env_config']['run'] if 'run' in config['env_config'] \
else None
if args.run and config_run:
if args.run != config_run:
print('visualizer_rllib.py: error: run argument ' +
'\'{}\' passed in '.format(args.run) +
'differs from the one stored in params.json ' +
'\'{}\''.format(config_run))
sys.exit(1)
if args.run:
agent_cls = get_agent_class(args.run)
elif config_run:
agent_cls = get_agent_class(config_run)
else:
print('visualizer_rllib.py: error: could not find flow parameter '
'\'run\' in params.json, '
'add argument --run to provide the algorithm or model used '
'to train the results\n e.g. '
'python ./visualizer_rllib.py /tmp/ray/result_dir 1 --run PPO')
sys.exit(1)
sim_params = flow_params['sim']
sim_params.restart_instance = True
dir_path = os.path.dirname(os.path.realpath(__file__))
emission_path = '{0}/test_time_rollout/'.format(dir_path)
sim_params.emission_path = emission_path if args.gen_emission else None
# pick your rendering mode
if args.render_mode == 'sumo_web3d':
sim_params.num_clients = 2
sim_params.render = False
elif args.render_mode == 'drgb':
sim_params.render = 'drgb'
sim_params.pxpm = 4
elif args.render_mode == 'sumo_gui':
sim_params.render = True
print('NOTE: With render mode {}, an extra instance of the SUMO GUI '
'will display before the GUI for visualizing the result. Click '
'the green Play arrow to continue.'.format(args.render_mode))
elif args.render_mode == 'no_render':
sim_params.render = False
if args.save_render:
sim_params.render = 'drgb'
sim_params.pxpm = 4
sim_params.save_render = True
# Start the environment with the gui turned on and a path for the
# emission file
env_params = flow_params['env']
sim_params.restart_instance = False
if args.evaluate:
env_params.evaluate = True
# lower the horizon if testing
if args.horizon:
config['horizon'] = args.horizon
env_params.horizon = args.horizon
# Create and register a gym+rllib env
create_env, env_name = make_create_env(params=flow_params, version=0)
register_env(env_name, create_env)
# create the agent that will be used to compute the actions
agent = agent_cls(env=env_name, config=config)
checkpoint = result_dir + '/checkpoint_' + args.checkpoint_num
checkpoint = checkpoint + '/checkpoint-' + args.checkpoint_num
agent.restore(checkpoint)
# Simulate and collect metrics
final_outflows = []
final_inflows = []
mean_speed = []
std_speed = []
policy_agent_mapping = default_policy_agent_mapping
if hasattr(agent, "workers"):
env = agent.workers.local_worker().env
multiagent = isinstance(env, MultiAgentEnv)
if agent.workers.local_worker().multiagent:
policy_agent_mapping = agent.config["multiagent"][
"policy_mapping_fn"]
policy_map = agent.workers.local_worker().policy_map
state_init = {p: m.get_initial_state() for p, m in policy_map.items()}
use_lstm = {p: len(s) > 0 for p, s in state_init.items()}
action_init = {
p: m.action_space.sample()
for p, m in policy_map.items()
}
else:
env = gym.make(env_name)
multiagent = False
use_lstm = {DEFAULT_POLICY_ID: False}
steps = 0
for i in range(args.num_rollouts):
vel = []
mapping_cache = {} # in case policy_agent_mapping is stochastic
reward_dict = {}
obs = env.reset()
agent_states = DefaultMapping(
lambda agent_id: state_init[mapping_cache[agent_id]])
prev_actions = DefaultMapping(
lambda agent_id: action_init[mapping_cache[agent_id]])
prev_rewards = collections.defaultdict(lambda: 0.)
done = False
reward_total = 0.0
while not done and steps < (env_params.horizon or steps + 1):
vehicles = env.unwrapped.k.vehicle
vel.append(np.mean(vehicles.get_speed(vehicles.get_ids())))
multi_obs = obs if multiagent else {_DUMMY_AGENT_ID: obs}
action_dict = {}
for agent_id, a_obs in multi_obs.items():
if a_obs is not None:
policy_id = mapping_cache.setdefault(
agent_id, policy_agent_mapping(agent_id))
p_use_lstm = use_lstm[policy_id]
if p_use_lstm:
a_action, p_state, _ = agent.compute_action(
a_obs,
state=agent_states[agent_id],
prev_action=prev_actions[agent_id],
prev_reward=prev_rewards[agent_id],
policy_id=policy_id)
agent_states[agent_id] = p_state
else:
a_action = agent.compute_action(
a_obs,
prev_action=prev_actions[agent_id],
prev_reward=prev_rewards[agent_id],
policy_id=policy_id)
a_action = _flatten_action(a_action) # tuple actions
action_dict[agent_id] = a_action
prev_actions[agent_id] = a_action
action = action_dict
action = action if multiagent else action[_DUMMY_AGENT_ID]
next_obs, reward, done, _ = env.step(action)
if multiagent:
for agent_id, r in reward.items():
prev_rewards[agent_id] = r
else:
prev_rewards[_DUMMY_AGENT_ID] = reward
if multiagent:
done = done["__all__"]
reward_total += sum(reward.values())
else:
reward_total += reward
steps += 1
obs = next_obs
outflow = vehicles.get_outflow_rate(500)
final_outflows.append(outflow)
inflow = vehicles.get_inflow_rate(500)
final_inflows.append(inflow)
if np.all(np.array(final_inflows) > 1e-5):
throughput_efficiency = [
x / y for x, y in zip(final_outflows, final_inflows)
]
else:
throughput_efficiency = [0] * len(final_inflows)
mean_speed.append(np.mean(vel))
std_speed.append(np.std(vel))
print("Episode reward", reward_total)
print('==== Summary of results ====')
print(mean_speed)
# if multiagent:
# for agent_id, rew in rets.items():
# print('For agent', agent_id)
# print(rew)
# print('Average, std return: {}, {} for agent {}'.format(
# np.mean(rew), np.std(rew), agent_id))
# else:
# print(rets)
# print('Average, std: {}, {}'.format(
# np.mean(rets), np.std(rets)))
print("\nSpeed, mean (m/s): {}".format(mean_speed))
print('Average, std: {}, {}'.format(np.mean(mean_speed),
np.std(mean_speed)))
print("\nSpeed, std (m/s): {}".format(std_speed))
print('Average, std: {}, {}'.format(np.mean(std_speed), np.std(std_speed)))
# Compute arrival rate of vehicles in the last 500 sec of the run
print("\nOutflows (veh/hr): {}".format(final_outflows))
print('Average, std: {}, {}'.format(np.mean(final_outflows),
np.std(final_outflows)))
# Compute departure rate of vehicles in the last 500 sec of the run
print("Inflows (veh/hr): {}".format(final_inflows))
print('Average, std: {}, {}'.format(np.mean(final_inflows),
np.std(final_inflows)))
# Compute throughput efficiency in the last 500 sec of the
print("Throughput efficiency (veh/hr): {}".format(throughput_efficiency))
print('Average, std: {}, {}'.format(np.mean(throughput_efficiency),
np.std(throughput_efficiency)))
# terminate the environment
env.unwrapped.terminate()
# if prompted, convert the emission file into a csv file
if args.gen_emission:
time.sleep(0.1)
dir_path = os.path.dirname(os.path.realpath(__file__))
emission_filename = '{0}-emission.xml'.format(env.scenario.name)
emission_path = \
'{0}/test_time_rollout/{1}'.format(dir_path, emission_filename)
emission_to_csv(emission_path)
# if we wanted to save the render, here we create the movie
if args.save_render:
dirs = os.listdir(os.path.expanduser('~') + '/flow_rendering')
# Ignore hidden files
dirs = [d for d in dirs if d[0] != '.']
dirs.sort(key=lambda date: datetime.strptime(date, "%Y-%m-%d-%H%M%S"))
recent_dir = dirs[-1]
# create the movie
movie_dir = os.path.expanduser('~') + '/flow_rendering/' + recent_dir
save_dir = os.path.expanduser('~') + '/flow_movies'
if not os.path.exists(save_dir):
os.mkdir(save_dir)
os_cmd = "cd " + movie_dir + " && ffmpeg -i frame_%06d.png"
os_cmd += " -pix_fmt yuv420p " + dirs[-1] + ".mp4"
os_cmd += "&& cp " + dirs[-1] + ".mp4 " + save_dir + "/"
os.system(os_cmd)
def visualizer_rllib(args):
"""Visualizer for RLlib experiments.
This function takes args (see function create_parser below for
more detailed information on what information can be fed to this
visualizer), and renders the experiment associated with it.
"""
result_dir = args.result_dir if args.result_dir[-1] != '/' \
else args.result_dir[:-1]
config = get_rllib_config(result_dir)
# check if we have a multiagent environment but in a
# backwards compatible way
if config.get('multiagent', {}).get('policies', None):
multiagent = True
pkl = get_rllib_pkl(result_dir)
config['multiagent'] = pkl['multiagent']
else:
multiagent = False
# Run on only one cpu for rendering purposes
config['num_workers'] = 0
flow_params = get_flow_params(config)
# hack for old pkl files
# TODO(ev) remove eventually
sim_params = flow_params['sim']
setattr(sim_params, 'num_clients', 1)
# Determine agent and checkpoint
config_run = config['env_config']['run'] if 'run' in config['env_config'] \
else None
if args.run and config_run:
if args.run != config_run:
print('visualizer_rllib.py: error: run argument ' +
'\'{}\' passed in '.format(args.run) +
'differs from the one stored in params.json ' +
'\'{}\''.format(config_run))
sys.exit(1)
if args.run:
agent_cls = get_agent_class(args.run)
elif config_run:
agent_cls = get_agent_class(config_run)
else:
print('visualizer_rllib.py: error: could not find flow parameter '
'\'run\' in params.json, '
'add argument --run to provide the algorithm or model used '
'to train the results\n e.g. '
'python ./visualizer_rllib.py /tmp/ray/result_dir 1 --run PPO')
sys.exit(1)
sim_params.restart_instance = True
dir_path = os.path.dirname(os.path.realpath(__file__))
emission_path = '{0}/emission/'.format(dir_path)
sim_params.emission_path = emission_path if args.gen_emission else None
sim_params.summary_path = emission_path if args.gen_emission else None
sim_params.tripinfo_path = emission_path if args.gen_emission else None
# pick your rendering mode
if args.render_mode == 'sumo_web3d':
sim_params.num_clients = 2
sim_params.render = False
elif args.render_mode == 'drgb':
sim_params.render = 'drgb'
sim_params.pxpm = 4
elif args.render_mode == 'sumo_gui':
sim_params.render = False # will be set to True below
elif args.render_mode == 'no_render':
sim_params.render = False
if args.save_render:
if args.render_mode != 'sumo_gui':
sim_params.render = 'drgb'
sim_params.pxpm = 4
sim_params.save_render = True
# Create and register a gym+rllib env
create_env, env_name = make_create_env(params=flow_params, version=0)
register_env(env_name, create_env)
# check if the environment is a single or multiagent environment, and
# get the right address accordingly
# single_agent_envs = [env for env in dir(flow.envs)
# if not env.startswith('__')]
# if flow_params['env_name'] in single_agent_envs:
# env_loc = 'flow.envs'
# else:
# env_loc = 'flow.envs.multiagent'
# Start the environment with the gui turned on and a path for the
# emission file
env_params = flow_params['env']
env_params.restart_instance = False
if args.evaluate:
env_params.evaluate = True
# lower the horizon if testing
if args.horizon:
config['horizon'] = args.horizon
env_params.horizon = args.horizon
# create the agent that will be used to compute the actions
agent = agent_cls(env=env_name, config=config)
checkpoint = result_dir + '/checkpoint_' + args.checkpoint_num
checkpoint = checkpoint + '/checkpoint-' + args.checkpoint_num
agent.restore(checkpoint)
if hasattr(agent, "local_evaluator") and \
os.environ.get("TEST_FLAG") != 'True':
env = agent.local_evaluator.env
else:
env = gym.make(env_name)
if args.render_mode == 'sumo_gui':
env.sim_params.render = True # set to True after initializing agent and env
if multiagent:
rets = {}
# map the agent id to its policy
policy_map_fn = config['multiagent']['policy_mapping_fn'].func
for key in config['multiagent']['policies'].keys():
rets[key] = []
else:
rets = []
if config['model']['use_lstm']:
use_lstm = True
if multiagent:
state_init = {}
# map the agent id to its policy
policy_map_fn = config['multiagent']['policy_mapping_fn'].func
size = config['model']['lstm_cell_size']
for key in config['multiagent']['policies'].keys():
state_init[key] = [
np.zeros(size, np.float32),
np.zeros(size, np.float32)
]
else:
state_init = [
np.zeros(config['model']['lstm_cell_size'], np.float32),
np.zeros(config['model']['lstm_cell_size'], np.float32)
]
else:
use_lstm = False
# if restart_instance, don't restart here because env.reset will restart later
if not sim_params.restart_instance:
env.restart_simulation(sim_params=sim_params, render=sim_params.render)
# Simulate and collect metrics
final_outflows = []
final_inflows = []
mean_speed = []
std_speed = []
for i in range(args.num_rollouts):
vel = []
state = env.reset()
if multiagent:
ret = {key: [0] for key in rets.keys()}
else:
ret = 0
for _ in range(env_params.horizon):
vehicles = env.unwrapped.k.vehicle
vel.append(np.mean(vehicles.get_speed(vehicles.get_ids())))
if multiagent:
action = {}
for agent_id in state.keys():
if use_lstm:
action[agent_id], state_init[agent_id], logits = \
agent.compute_action(
state[agent_id], state=state_init[agent_id],
policy_id=policy_map_fn(agent_id))
else:
action[agent_id] = agent.compute_action(
state[agent_id], policy_id=policy_map_fn(agent_id))
else:
action = agent.compute_action(state)
state, reward, done, _ = env.step(action)
if multiagent:
for actor, rew in reward.items():
ret[policy_map_fn(actor)][0] += rew
else:
ret += reward
if multiagent and done['__all__']:
break
if not multiagent and done:
break
if multiagent:
for key in rets.keys():
rets[key].append(ret[key])
else:
rets.append(ret)
outflow = vehicles.get_outflow_rate(500)
final_outflows.append(outflow)
inflow = vehicles.get_inflow_rate(500)
final_inflows.append(inflow)
mean_speed.append(np.mean(vel))
std_speed.append(np.std(vel))
# terminate the environment
env.unwrapped.terminate()
emission_location = os.path.join(emission_path, env.network.name)
return emission_location
def visualizer_rllib(args):
"""Visualizer for RLlib experiments.
This function takes args (see function create_parser below for
more detailed information on what information can be fed to this
visualizer), and renders the experiment associated with it.
"""
result_dir = args.result_dir if args.result_dir[-1] != '/' \
else args.result_dir[:-1]
config = get_rllib_config(result_dir)
# check if we have a multiagent environment but in a
# backwards compatible way
if config.get('multiagent', {}).get('policies', None):
multiagent = True
pkl = get_rllib_pkl(result_dir)
config['multiagent'] = pkl['multiagent']
else:
multiagent = False
# Run on only one cpu for rendering purposes
config['num_workers'] = 0
flow_params = get_flow_params(config)
# hack for old pkl files
# TODO(ev) remove eventually
sim_params = flow_params['sim']
setattr(sim_params, 'num_clients', 1)
# for hacks for old pkl files TODO: remove eventually
if not hasattr(sim_params, 'use_ballistic'):
sim_params.use_ballistic = False
# Determine agent and checkpoint
config_run = config['env_config']['run'] if 'run' in config['env_config'] \
else None
if args.run and config_run:
if args.run != config_run:
print('visualizer_rllib.py: error: run argument ' +
'\'{}\' passed in '.format(args.run) +
'differs from the one stored in params.json ' +
'\'{}\''.format(config_run))
sys.exit(1)
if args.run:
agent_cls = get_agent_class(args.run)
elif config_run:
agent_cls = get_agent_class(config_run)
else:
print('visualizer_rllib.py: error: could not find flow parameter '
'\'run\' in params.json, '
'add argument --run to provide the algorithm or model used '
'to train the results\n e.g. '
'python ./visualizer_rllib.py /tmp/ray/result_dir 1 --run PPO')
sys.exit(1)
sim_params.restart_instance = True
dir_path = os.path.dirname(os.path.realpath(__file__))
emission_path = '{0}/test_time_rollout/'.format(dir_path)
sim_params.emission_path = emission_path if args.gen_emission else None
# pick your rendering mode
if args.render_mode == 'sumo_web3d':
sim_params.num_clients = 2
sim_params.render = False
elif args.render_mode == 'drgb':
sim_params.render = 'drgb'
sim_params.pxpm = 4
elif args.render_mode == 'sumo_gui':
sim_params.render = False # will be set to True below
elif args.render_mode == 'no_render':
sim_params.render = False
if args.save_render:
if args.render_mode != 'sumo_gui':
sim_params.render = 'drgb'
sim_params.pxpm = 4
sim_params.save_render = True
# Create and register a gym+rllib env
create_env, env_name = make_create_env(params=flow_params, version=0)
register_env(env_name, create_env)
# check if the environment is a single or multiagent environment, and
# get the right address accordingly
# single_agent_envs = [env for env in dir(flow.envs)
# if not env.startswith('__')]
# if flow_params['env_name'] in single_agent_envs:
# env_loc = 'flow.envs'
# else:
# env_loc = 'flow.envs.multiagent'
# Start the environment with the gui turned on and a path for the
# emission file
env_params = flow_params['env']
env_params.restart_instance = False
if args.evaluate:
env_params.evaluate = True
# lower the horizon if testing
if args.horizon:
config['horizon'] = args.horizon
env_params.horizon = args.horizon
# create the agent that will be used to compute the actions
agent = agent_cls(env=env_name, config=config)
checkpoint = result_dir + '/checkpoint_' + args.checkpoint_num
checkpoint = checkpoint + '/checkpoint-' + args.checkpoint_num
agent.restore(checkpoint)
if hasattr(agent, "local_evaluator") and \
os.environ.get("TEST_FLAG") != 'True':
env = agent.local_evaluator.env
else:
env = gym.make(env_name)
if args.render_mode == 'sumo_gui':
env.sim_params.render = True # set to True after initializing agent and env
if multiagent:
rets = {}
# map the agent id to its policy
policy_map_fn = config['multiagent']['policy_mapping_fn'].func
for key in config['multiagent']['policies'].keys():
rets[key] = []
else:
rets = []
if config['model']['use_lstm']:
use_lstm = True
if multiagent:
state_init = {}
# map the agent id to its policy
policy_map_fn = config['multiagent']['policy_mapping_fn'].func
size = config['model']['lstm_cell_size']
for key in config['multiagent']['policies'].keys():
state_init[key] = [
np.zeros(size, np.float32),
np.zeros(size, np.float32)
]
else:
state_init = [
np.zeros(config['model']['lstm_cell_size'], np.float32),
np.zeros(config['model']['lstm_cell_size'], np.float32)
]
else:
use_lstm = False
# if restart_instance, don't restart here because env.reset will restart later
if not sim_params.restart_instance:
env.restart_simulation(sim_params=sim_params, render=sim_params.render)
# Simulate and collect metrics
final_outflows = []
final_inflows = []
mean_speed = []
std_speed = []
state_translation = {
"inflow": 0,
"slowing": 1,
"parking": 2,
"parked": 3,
"outflow": 4
}
custom_outputs = {
k: np.zeros(env_params.horizon * args.num_rollouts)
for k in ["num_rollout", "t", "speed", "position", "state", "reward"]
}
for i in range(args.num_rollouts):
vel = []
state = env.reset()
if multiagent:
ret = {key: [0] for key in rets.keys()}
else:
ret = 0
for j in range(env_params.horizon):
vehicles = env.unwrapped.k.vehicle
speeds = vehicles.get_speed(vehicles.get_ids())
# only include non-empty speeds
if speeds:
vel.append(np.mean(speeds))
if multiagent:
action = {}
for agent_id in state.keys():
if use_lstm:
action[agent_id], state_init[agent_id], logits = \
agent.compute_action(
state[agent_id], state=state_init[agent_id],
policy_id=policy_map_fn(agent_id))
else:
action[agent_id] = agent.compute_action(
state[agent_id], policy_id=policy_map_fn(agent_id))
else:
action = agent.compute_action(state)
state, reward, done, _ = env.step(action)
if multiagent:
for actor, rew in reward.items():
ret[policy_map_fn(actor)][0] += rew
else:
ret += reward
if multiagent and done['__all__']:
break
if not multiagent and done:
break
#"num_rollout", "t", "speed", "position", "state", "reward"]}
m = i * env_params.horizon + j
custom_outputs["num_rollout"][m] = i
custom_outputs["t"][m] = j
custom_outputs["reward"][m] = reward
veh_id = vehicles.get_ids()[0]
custom_outputs["speed"][m] = vehicles.get_speed(veh_id)
custom_outputs["position"][m] = vehicles.get_global_position(
veh_id, env)
custom_outputs["state"][m] = state_translation.get(
vehicles.get_state(veh_id), -1)
if multiagent:
for key in rets.keys():
rets[key].append(ret[key])
else:
rets.append(ret)
outflow = vehicles.get_outflow_rate(500)
final_outflows.append(outflow)
inflow = vehicles.get_inflow_rate(500)
final_inflows.append(inflow)
if np.all(np.array(final_inflows) > 1e-5):
throughput_efficiency = [
x / y for x, y in zip(final_outflows, final_inflows)
]
else:
throughput_efficiency = [0] * len(final_inflows)
mean_speed.append(np.mean(vel))
std_speed.append(np.std(vel))
if multiagent:
for agent_id, rew in rets.items():
print('Round {}, Return: {} for agent {}'.format(
i, ret, agent_id))
else:
print('Round {}, Return: {}'.format(i, ret))
print('==== Summary of results ====')
print("Return:")
print(mean_speed)
if multiagent:
for agent_id, rew in rets.items():
print('For agent', agent_id)
print(rew)
print('Average, std return: {}, {} for agent {}'.format(
np.mean(rew), np.std(rew), agent_id))
else:
print(rets)
print('Average, std: {}, {}'.format(np.mean(rets), np.std(rets)))
print("\nSpeed, mean (m/s):")
print(mean_speed)
print('Average, std: {}, {}'.format(np.mean(mean_speed),
np.std(mean_speed)))
print("\nSpeed, std (m/s):")
print(std_speed)
print('Average, std: {}, {}'.format(np.mean(std_speed), np.std(std_speed)))
# Compute arrival rate of vehicles in the last 500 sec of the run
print("\nOutflows (veh/hr):")
print(final_outflows)
print('Average, std: {}, {}'.format(np.mean(final_outflows),
np.std(final_outflows)))
# Compute departure rate of vehicles in the last 500 sec of the run
print("Inflows (veh/hr):")
print(final_inflows)
print('Average, std: {}, {}'.format(np.mean(final_inflows),
np.std(final_inflows)))
# Compute throughput efficiency in the last 500 sec of the
print("Throughput efficiency (veh/hr):")
print(throughput_efficiency)
print('Average, std: {}, {}'.format(np.mean(throughput_efficiency),
np.std(throughput_efficiency)))
df = pd.DataFrame(custom_outputs)
df.to_csv("output.csv")
# terminate the environment
env.unwrapped.terminate()
# if prompted, convert the emission file into a csv file
if args.gen_emission:
time.sleep(0.1)
dir_path = os.path.dirname(os.path.realpath(__file__))
emission_filename = '{0}-emission.xml'.format(env.network.name)
emission_path = \
'{0}/test_time_rollout/{1}'.format(dir_path, emission_filename)
# convert the emission file into a csv file
emission_to_csv(emission_path)
# print the location of the emission csv file
emission_path_csv = emission_path[:-4] + ".csv"
print("\nGenerated emission file at " + emission_path_csv)
# delete the .xml version of the emission file
os.remove(emission_path)
def main(args):
global current_scenario, scenarios, SCENARIOS_PATH, agent, env
ray.init(num_cpus=5)
result_dir = None
# parse the flow args
if args.result_dir:
result_dir = args.result_dir if args.result_dir[-1] != '/' \
else args.result_dir[:-1]
config = get_rllib_config(result_dir)
flow_params = get_flow_params(config)
# Create and register a gym+rllib env
create_env, env_name = make_create_env(params=flow_params,
version=0,
sumo_binary="sumo")
register_env(env_name, create_env)
agent_cls = get_agent_class("PPO")
config["num_workers"] = 1
agent = agent_cls(env=env_name, config=config)
checkpoint = result_dir + '/checkpoint-' + args.checkpoint_num
agent.restore(checkpoint)
# Recreate the scenario from the pickled parameters
exp_tag = flow_params["exp_tag"]
net_params = flow_params['net']
vehicles = flow_params['veh']
initial_config = flow_params['initial']
module = __import__("flow.scenarios", fromlist=[flow_params["scenario"]])
scenario_class = getattr(module, flow_params["scenario"])
module = __import__("flow.scenarios", fromlist=[flow_params["generator"]])
generator_class = getattr(module, flow_params["generator"])
scenario = scenario_class(name=exp_tag,
generator_class=generator_class,
vehicles=vehicles,
net_params=net_params,
initial_config=initial_config)
# Start the environment with the gui turned on and a path for the
# emission file
module = __import__("flow.envs", fromlist=[flow_params["env_name"]])
env_class = getattr(module, flow_params["env_name"])
env_params = flow_params['env']
sumo_params = flow_params['sumo']
sumo_params.sumo_binary = "sumo"
sumo_params.emission_path = "./test_time_rollout/"
# FIXME this will actually start sumo, since base_env in flow
# comes with a call to start sumo
# make it start on the same port!
# got to set up sumo with two ports
env = env_class(env_params=env_params,
sumo_params=sumo_params,
scenario=scenario)
env.sumo_params.port = 5500
env.restart_sumo(env.sumo_params)
# now that you have the env, you need to pass it to next_simulation_step
# so you can use it to feed actions
task = None
sumo_start_fn = functools.partial(start_sumo_executable, args.gui,
args.sumo_args)
if args.configuration_file:
# Replace the built-in scenarios with a single, user-specified one.
# We don't merge the lists to avoid clashes with two scenarios having is_default set.
SCENARIOS_PATH = None
name = os.path.basename(args.configuration_file)
scenarios = {
to_kebab_case(name):
Scenario.from_config_json({
'name': name,
'description': 'User-specified scenario',
'config_file': args.configuration_file,
'is_default': True
})
}
else:
scenarios = load_scenarios_file({}, SCENARIOS_PATH)
def setup_websockets_server():
return functools.partial(
websocket_simulation_control,
lambda: sumo_start_fn(getattr(current_scenario, 'config_file')),
task)
loop = asyncio.get_event_loop()
# websockets
ws_handler = setup_websockets_server()
ws_server = websockets.serve(ws_handler, '0.0.0.0', 5678)
# http
app = setup_http_server(task, SCENARIOS_PATH, scenarios)
http_server = loop.create_server(app.make_handler(), '0.0.0.0', 5000)
loop.run_until_complete(http_server)
loop.run_until_complete(ws_server)
print("""Listening on:
127.0.0.1:5000 (HTTP)
127.0.0.1:5678 (WebSockets)
""")
loop.run_forever()
def run_bottleneck(args, inflow_rate, num_trials):
result_dir = args.result_dir if args.result_dir[-1] != '/' \
else args.result_dir[:-1]
config = get_rllib_pkl(result_dir)
# Run on only one cpu for rendering purposes
config['num_workers'] = 0
flow_params = get_flow_params(config)
# Determine agent and checkpoint
config_run = config['env_config']['run'] if 'run' in config['env_config'] \
else None
if args.run and config_run:
if args.run != config_run:
print('visualizer_rllib.py: error: run argument '
+ '\'{}\' passed in '.format(args.run)
+ 'differs from the one stored in params.json '
+ '\'{}\''.format(config_run))
sys.exit(1)
if args.run:
agent_cls = get_agent_class(args.run)
elif config_run:
agent_cls = get_agent_class(config_run)
else:
print('visualizer_rllib.py: error: could not find flow parameter '
'\'run\' in params.json, '
'add argument --run to provide the algorithm or model used '
'to train the results\n e.g. '
'python ./visualizer_rllib.py /tmp/ray/result_dir 1 --run PPO')
sys.exit(1)
# if using a custom model
if config['model']['custom_model'] == "cc_model":
if config['model']['use_lstm']:
ModelCatalog.register_custom_model("cc_model", CentralizedCriticModelRNN)
else:
ModelCatalog.register_custom_model("cc_model", CentralizedCriticModel)
from flow.agents.centralized_PPO import CCTrainer
agent_cls = CCTrainer
elif config['model']['custom_model'] == "GRU":
ModelCatalog.register_custom_model("GRU", GRU)
elif config['model']['custom_model'] == "FeedForward":
ModelCatalog.register_custom_model("FeedForward", FeedForward)
# If we trained by imitating
if "imitation_weight" in config['model']['custom_options'].keys():
from flow.agents.ImitationPPO import ImitationTrainer
agent_cls = ImitationTrainer
sim_params = flow_params['sim']
sim_params.restart_instance = False
dir_path = os.path.dirname(os.path.realpath(__file__))
emission_path = '{0}/test_time_rollout/'.format(dir_path)
sim_params.emission_path = emission_path if args.gen_emission else None
# pick your rendering mode
if args.render_mode == 'sumo_web3d':
sim_params.num_clients = 2
sim_params.render = False
elif args.render_mode == 'drgb':
sim_params.render = 'drgb'
sim_params.pxpm = 4
elif args.render_mode == 'sumo_gui':
sim_params.render = False
elif args.render_mode == 'no_render':
sim_params.render = False
if args.save_render:
sim_params.render = 'drgb'
sim_params.pxpm = 4
sim_params.save_render = True
# Start the environment with the gui turned on and a path for the
# emission file
env_params = flow_params['env']
# TODO(@evinitsky) remove this this is a backwards compatibility hack
if 'life_penalty' not in env_params.additional_params.keys():
env_params.additional_params['life_penalty'] = - 3
if args.evaluate:
env_params.evaluate = True
# lower the horizon if testing
if args.horizon:
config['horizon'] = args.horizon
env_params.horizon = args.horizon
# Create and register a gym+rllib env
create_env, env_name = make_create_env(params=flow_params, version=0)
register_env(env_name, create_env)
# create the agent that will be used to compute the actions
agent = agent_cls(env=env_name, config=config)
checkpoint = result_dir + '/checkpoint_' + args.checkpoint_num
checkpoint = checkpoint + '/checkpoint-' + args.checkpoint_num
agent.restore(checkpoint)
policy_agent_mapping = default_policy_agent_mapping
if hasattr(agent, "workers"):
env = agent.workers.local_worker().env
multiagent = isinstance(env, MultiAgentEnv)
if agent.workers.local_worker().multiagent:
policy_agent_mapping = agent.config["multiagent"][
"policy_mapping_fn"]
policy_map = agent.workers.local_worker().policy_map
state_init = {p: m.get_initial_state() for p, m in policy_map.items()}
use_lstm = {p: len(s) > 0 for p, s in state_init.items()}
action_init = {
p: m.action_space.sample()
for p, m in policy_map.items()
}
else:
env = gym.make(env_name)
multiagent = False
use_lstm = {DEFAULT_POLICY_ID: False}
if args.render_mode == 'sumo_gui':
env.sim_params.render = True # set to True after initializing agent and env
# Simulate and collect metrics
outflow_arr = []
final_outflows = []
final_inflows = []
mean_speed = []
std_speed = []
mean_rewards = []
per_agent_rew = collections.defaultdict(lambda: 0.0)
# keep track of the last 500 points of velocity data for lane 0
# and 1 in edge 4
velocity_arr = []
vel = []
mapping_cache = {} # in case policy_agent_mapping is stochastic
for j in range(num_trials):
agent_states = DefaultMapping(
lambda agent_id: state_init[mapping_cache[agent_id]])
prev_actions = DefaultMapping(
lambda agent_id: action_init[mapping_cache[agent_id]])
prev_rewards = collections.defaultdict(lambda: 0.0)
done = False
reward_total = 0.0
obs = env.reset(inflow_rate)
k = 0
while k < env_params.horizon and not done:
vehicles = env.unwrapped.k.vehicle
vel.append(np.mean(vehicles.get_speed(vehicles.get_ids())))
# don't start recording till we have hit the warmup time
if k >= env_params.horizon - args.end_len:
vehs_on_four = vehicles.get_ids_by_edge('4')
lanes = vehicles.get_lane(vehs_on_four)
lane_dict = {veh_id: lane for veh_id, lane in
zip(vehs_on_four, lanes)}
sort_by_lane = sorted(vehs_on_four,
key=lambda x: lane_dict[x])
num_zeros = lanes.count(0)
if num_zeros > 0:
speed_on_zero = np.mean(vehicles.get_speed(
sort_by_lane[0:num_zeros]))
else:
speed_on_zero = 0.0
if num_zeros < len(vehs_on_four):
speed_on_one = np.mean(vehicles.get_speed(
sort_by_lane[num_zeros:]))
else:
speed_on_one = 0.0
velocity_arr.append(
[inflow_rate,
speed_on_zero,
speed_on_one])
multi_obs = obs if multiagent else {_DUMMY_AGENT_ID: obs}
action_dict = {}
for agent_id, a_obs in multi_obs.items():
if a_obs is not None:
policy_id = mapping_cache.setdefault(
agent_id, policy_agent_mapping(agent_id))
p_use_lstm = use_lstm[policy_id]
if p_use_lstm:
a_action, p_state, _ = agent.compute_action(
a_obs,
state=agent_states[agent_id],
prev_action=prev_actions[agent_id],
prev_reward=prev_rewards[agent_id],
policy_id=policy_id)
agent_states[agent_id] = p_state
print(agent_id, a_action)
else:
a_action = agent.compute_action(
a_obs,
prev_action=prev_actions[agent_id],
prev_reward=prev_rewards[agent_id],
policy_id=policy_id)
a_action = _flatten_action(a_action) # tuple actions
action_dict[agent_id] = a_action
prev_actions[agent_id] = a_action
print(agent_id, a_action)
action = action_dict
action = action if multiagent else action[_DUMMY_AGENT_ID]
# print(action)
next_obs, reward, done, _ = env.step(action)
if multiagent:
for agent_id, r in reward.items():
prev_rewards[agent_id] = r
per_agent_rew[agent_id] += r
else:
prev_rewards[_DUMMY_AGENT_ID] = reward
if multiagent:
done = done["__all__"]
reward_total += sum(reward.values())
else:
reward_total += reward
k += 1
obs = next_obs
vehicles = env.unwrapped.k.vehicle
outflow = vehicles.get_outflow_rate(500)
final_outflows.append(outflow)
inflow = vehicles.get_inflow_rate(500)
final_inflows.append(inflow)
outflow_arr.append([inflow_rate, outflow, outflow/inflow_rate])
mean_speed.append(np.mean(vel))
std_speed.append(np.std(vel))
mean_rewards.append([inflow, np.mean(list(per_agent_rew.values()))])
return [outflow_arr, velocity_arr, mean_speed, std_speed, mean_rewards]
def visualizer_rllib(args):
"""Visualizer for RLlib experiments.
This function takes args (see function create_parser below for
more detailed information on what information can be fed to this
visualizer), and renders the experiment associated with it.
"""
result_dir = args.result_dir if args.result_dir[-1] != '/' \
else args.result_dir[:-1]
config = get_rllib_config(result_dir)
name = result_dir.split("/")[-2:]
# check if we have a multiagent environment but in a
# backwards compatible way
if config.get('multiagent', {}).get('policies', None):
multiagent = True
pkl = get_rllib_pkl(result_dir)
config['multiagent'] = pkl['multiagent']
else:
multiagent = False
# Run on only one cpu for rendering purposes
config['num_workers'] = 0
flow_params = get_flow_params(config)
# hack for old pkl files
# TODO(ev) remove eventually
sim_params = flow_params['sim']
setattr(sim_params, 'num_clients', 1)
# for hacks for old pkl files TODO: remove eventually
if not hasattr(sim_params, 'use_ballistic'):
sim_params.use_ballistic = False
# Determine agent and checkpoint
config_run = config['env_config']['run'] if 'run' in config['env_config'] \
else None
if args.run and config_run:
if args.run != config_run:
print('visualizer_rllib.py: error: run argument '
+ '\'{}\' passed in '.format(args.run)
+ 'differs from the one stored in params.json '
+ '\'{}\''.format(config_run))
sys.exit(1)
if args.run:
agent_cls = get_agent_class(args.run)
elif config_run:
agent_cls = get_agent_class(config_run)
else:
print('visualizer_rllib.py: error: could not find flow parameter '
'\'run\' in params.json, '
'add argument --run to provide the algorithm or model used '
'to train the results\n e.g. '
'python ./visualizer_rllib.py /tmp/ray/result_dir 1 --run PPO')
sys.exit(1)
sim_params.restart_instance = True
dir_path = os.path.dirname(os.path.realpath(__file__))
emission_path = '{0}/test_time_rollout/'.format(dir_path)
sim_params.emission_path = emission_path if args.gen_emission else None
# pick your rendering mode
if args.render_mode == 'sumo_web3d':
sim_params.num_clients = 2
sim_params.render = False
elif args.render_mode == 'drgb':
sim_params.render = 'drgb'
sim_params.pxpm = 4
elif args.render_mode == 'sumo_gui':
sim_params.render = False # will be set to True below
elif args.render_mode == 'no_render':
sim_params.render = False
if args.save_render:
if args.render_mode != 'sumo_gui':
sim_params.render = 'drgb'
sim_params.pxpm = 4
sim_params.save_render = True
# Create and register a gym+rllib env
create_env, env_name = make_create_env(params=flow_params, version=0)
register_env(env_name, create_env)
# check if the environment is a single or multiagent environment, and
# get the right address accordingly
# single_agent_envs = [env for env in dir(flow.envs)
# if not env.startswith('__')]
# if flow_params['env_name'] in single_agent_envs:
# env_loc = 'flow.envs'
# else:
# env_loc = 'flow.envs.multiagent'
# Start the environment with the gui turned on and a path for the
# emission file
env_params = flow_params['env']
env_params.restart_instance = False
if args.evaluate:
env_params.evaluate = True # FIXME: this not works
# lower the horizon if testing
if args.horizon:
config['horizon'] = args.horizon
env_params.horizon = args.horizon
# create the agent that will be used to compute the actions
agent = agent_cls(env=env_name, config=config)
checkpoint = result_dir + '/checkpoint_' + args.checkpoint_num
checkpoint = checkpoint + '/checkpoint-' + args.checkpoint_num
agent.restore(checkpoint)
if hasattr(agent, "local_evaluator") and \
os.environ.get("TEST_FLAG") != 'True':
env = agent.local_evaluator.env
else:
env = gym.make(env_name)
if args.render_mode == 'sumo_gui':
env.sim_params.render = True # set to True after initializing agent and env
if multiagent:
rets = {}
# map the agent id to its policy
policy_map_fn = config['multiagent']['policy_mapping_fn'].func
for key in config['multiagent']['policies'].keys():
rets[key] = []
else:
rets = []
if config['model']['use_lstm']:
use_lstm = True
if multiagent:
state_init = {}
# map the agent id to its policy
policy_map_fn = config['multiagent']['policy_mapping_fn'].func
size = config['model']['lstm_cell_size']
for key in config['multiagent']['policies'].keys():
state_init[key] = [np.zeros(size, np.float32),
np.zeros(size, np.float32)]
else:
state_init = [
np.zeros(config['model']['lstm_cell_size'], np.float32),
np.zeros(config['model']['lstm_cell_size'], np.float32)
]
else:
use_lstm = False
# if restart_instance, don't restart here because env.reset will restart later
if not sim_params.restart_instance:
env.restart_simulation(sim_params=sim_params, render=sim_params.render)
# Simulate and collect metrics
final_outflows = []
final_inflows = []
mean_speed = []
std_speed = []
rl_speed = [] # store rl controlled vehicle's speed
log2_stack = defaultdict(list) # This dict stores log2 data during rollouts
if args.evaluate:
env.unwrapped.env_params.evaluate = True # To cover bug
for i in range(args.num_rollouts):
vel = []
vel_dict = defaultdict(list)
timerange = []
state = env.reset()
if multiagent:
ret = {key: [0] for key in rets.keys()}
else:
ret = 0
for _ in range(env_params.horizon):
vehicles = env.unwrapped.k.vehicle
ids = vehicles.get_ids()
rls = vehicles.get_rl_ids()
speeds = vehicles.get_speed(ids)
timerange.append(vehicles.get_timestep(ids[-1]) / 10000)
# only include non-empty speeds
if speeds:
vel.append(np.mean(speeds))
for veh_id, speed in zip(ids, speeds):
vel_dict[veh_id].append(speed)
if multiagent:
action = {}
for agent_id in state.keys():
if use_lstm:
action[agent_id], state_init[agent_id], logits = \
agent.compute_action(
state[agent_id], state=state_init[agent_id],
policy_id=policy_map_fn(agent_id))
else:
action[agent_id] = agent.compute_action(
state[agent_id], policy_id=policy_map_fn(agent_id))
else:
action = agent.compute_action(state)
state, reward, done, _ = env.step(action)
if multiagent:
for actor, rew in reward.items():
ret[policy_map_fn(actor)][0] += rew
else:
ret += reward
if multiagent and done['__all__']:
break
if not multiagent and done:
break
if multiagent:
for key in rets.keys():
rets[key].append(ret[key])
else:
rets.append(ret)
outflow = vehicles.get_outflow_rate(500)
final_outflows.append(outflow)
inflow = vehicles.get_inflow_rate(500)
final_inflows.append(inflow)
if np.all(np.array(final_inflows) > 1e-5):
throughput_efficiency = [x / y for x, y in
zip(final_outflows, final_inflows)]
else:
throughput_efficiency = [0] * len(final_inflows)
mean_speed.append(np.mean(vel))
std_speed.append(np.std(vel))
if multiagent:
for agent_id, rew in rets.items():
print('Round {}, Return: {} for agent {}'.format(
i, ret, agent_id))
else:
print('Round {}, Return: {}'.format(i, ret))
log2 = env.unwrapped.log2
for k in log2:
log2_stack[k].append(log2[k])
# plot non-rl's speed and rl's speed graph
if i == args.num_rollouts - 1 and args.render_mode != "no_render":
veh = list(vel_dict.keys())
plt.subplot(2, 1, 1)
plt.title('/'.join(name))
for v in veh[:-1]:
plt.plot(timerange, vel_dict[v])
plt.xlabel('timestep(s)')
plt.ylabel('speed(m/s)')
plt.legend(veh[:-1])
plt.grid(True)
# plt.show()
plt.subplot(2, 1, 2)
plt.plot(timerange, vel_dict[veh[-1]])
plt.xlabel('timestep(s)')
plt.ylabel('speed(m/s)')
plt.legend(veh[-1:])
plt.grid(True)
plt.show()
rl_speed = [np.mean(vel_dict[rl]) for rl in vehicles.get_rl_ids()]
for k in log2_stack:
log2_stack[k] = np.mean(log2_stack[k]).round(3)
# export the log2_stack
from time import strftime
time = strftime('%Y-%m-%d')
flow_autonomous_home = os.path.expanduser('~/log/')
with open(flow_autonomous_home + f'/log.csv', 'a') as f:
keys = ['\"' + str(k) + '\"' for k in log2_stack.keys()]
values = ['\"' + str(v) + '\"' for v in log2_stack.values()]
# f.write('time,name,'+','.join(keys)+'\n')
f.write(f'{time},{name[0]},{",".join(values)}\n')
print('==== Summary of results ====')
print("Return:")
print(mean_speed)
if multiagent:
for agent_id, rew in rets.items():
print('For agent', agent_id)
print(rew)
print('Average, std return: {}, {} for agent {}'.format(
np.mean(rew), np.std(rew), agent_id))
else:
print(rets)
print('Average, std: {}, {}'.format(
np.mean(rets), np.std(rets)))
print("\nSpeed, mean (m/s):")
print(mean_speed)
print('')
# bmil edit
rls = vehicles.get_rl_ids()
[print(f'{rls[i]} Speed, mean (m/s): {rl_speed[i]}') for i in range(len(rls))]
print('Average, std: {}, {}'.format(np.mean(mean_speed), np.std(
mean_speed)))
print("\nSpeed, std (m/s):")
print(std_speed)
print('Average, std: {}, {}'.format(np.mean(std_speed), np.std(
std_speed)))
# Compute arrival rate of vehicles in the last 500 sec of the run
print("\nOutflows (veh/hr):")
print(final_outflows)
print('Average, std: {}, {}'.format(np.mean(final_outflows),
np.std(final_outflows)))
# Compute departure rate of vehicles in the last 500 sec of the run
print("Inflows (veh/hr):")
print(final_inflows)
print('Average, std: {}, {}'.format(np.mean(final_inflows),
np.std(final_inflows)))
# Compute throughput efficiency in the last 500 sec of the
print("Throughput efficiency (veh/hr):")
print(throughput_efficiency)
print('Average, std: {}, {}'.format(np.mean(throughput_efficiency),
np.std(throughput_efficiency)))
# terminate the environment
env.unwrapped.terminate()
# if prompted, convert the emission file into a csv file
if args.gen_emission:
time.sleep(0.1)
dir_path = os.path.dirname(os.path.realpath(__file__))
emission_filename = '{0}-emission.xml'.format(env.network.name)
emission_path = \
'{0}/test_time_rollout/{1}'.format(dir_path, emission_filename)
# convert the emission file into a csv file
emission_to_csv(emission_path)
# print the location of the emission csv file
emission_path_csv = emission_path[:-4] + ".csv"
print("\nGenerated emission file at " + emission_path_csv)
# delete the .xml version of the emission file
os.remove(emission_path)
def test_encoder_and_get_flow_params(self):
"""Tests both FlowParamsEncoder and get_flow_params.
FlowParamsEncoder is used to serialize the data from a flow_params dict
for replay by the visualizer later. Then, the get_flow_params method is
used to try and read the parameters from the config file, and is
checked to match expected results.
"""
# use a flow_params dict derived from flow/benchmarks/merge0.py
vehicles = VehicleParams()
vehicles.add(
veh_id="human",
acceleration_controller=(IDMController, {}),
car_following_params=SumoCarFollowingParams(
speed_mode="obey_safe_speed", ),
# for testing coverage purposes, we add a routing controller
routing_controller=(ContinuousRouter, {}),
num_vehicles=5)
vehicles.add(veh_id="rl",
acceleration_controller=(RLController, {}),
car_following_params=SumoCarFollowingParams(
speed_mode="obey_safe_speed", ),
num_vehicles=0)
inflow = InFlows()
inflow.add(veh_type="human",
edge="inflow_highway",
vehs_per_hour=1800,
departLane="free",
departSpeed=10)
inflow.add(veh_type="rl",
edge="inflow_highway",
vehs_per_hour=200,
departLane="free",
departSpeed=10)
inflow.add(veh_type="human",
edge="inflow_merge",
vehs_per_hour=100,
departLane="free",
departSpeed=7.5)
flow_params = dict(
exp_tag="merge_0",
env_name="WaveAttenuationMergePOEnv",
scenario="MergeScenario",
sim=SumoParams(
restart_instance=True,
sim_step=0.5,
render=False,
),
env=EnvParams(
horizon=750,
sims_per_step=2,
warmup_steps=0,
additional_params={
"max_accel": 1.5,
"max_decel": 1.5,
"target_velocity": 20,
"num_rl": 5,
},
),
net=NetParams(
inflows=inflow,
no_internal_links=False,
additional_params={
"merge_length": 100,
"pre_merge_length": 500,
"post_merge_length": 100,
"merge_lanes": 1,
"highway_lanes": 1,
"speed_limit": 30,
},
),
veh=vehicles,
initial=InitialConfig(),
tls=TrafficLightParams(),
)
# create an config dict with space for the flow_params dict
config = {"env_config": {}}
# save the flow params for replay
flow_json = json.dumps(flow_params,
cls=FlowParamsEncoder,
sort_keys=True,
indent=4)
config['env_config']['flow_params'] = flow_json
# dump the config so we can fetch it
json_out_file = 'params.json'
with open(os.path.expanduser(json_out_file), 'w+') as outfile:
json.dump(config,
outfile,
cls=FlowParamsEncoder,
sort_keys=True,
indent=4)
# fetch values using utility function `get_flow_params`
imported_flow_params = get_flow_params(config)
# delete the created file
os.remove(os.path.expanduser('params.json'))
# test that this inflows are correct
self.assertTrue(imported_flow_params["net"].inflows.__dict__ ==
flow_params["net"].inflows.__dict__)
imported_flow_params["net"].inflows = None
flow_params["net"].inflows = None
# make sure the rest of the imported flow_params match the originals
self.assertTrue(imported_flow_params["env"].__dict__ ==
flow_params["env"].__dict__)
self.assertTrue(imported_flow_params["initial"].__dict__ ==
flow_params["initial"].__dict__)
self.assertTrue(imported_flow_params["tls"].__dict__ ==
flow_params["tls"].__dict__)
self.assertTrue(imported_flow_params["sim"].__dict__ ==
flow_params["sim"].__dict__)
self.assertTrue(imported_flow_params["net"].__dict__ ==
flow_params["net"].__dict__)
self.assertTrue(
imported_flow_params["exp_tag"] == flow_params["exp_tag"])
self.assertTrue(
imported_flow_params["env_name"] == flow_params["env_name"])
self.assertTrue(
imported_flow_params["scenario"] == flow_params["scenario"])
def search_dicts(obj1, obj2):
"""Searches through dictionaries as well as lists of dictionaries
recursively to determine if any two components are mismatched."""
for key in obj1.keys():
# if an next element is a list, either compare the two lists,
# or if the lists contain dictionaries themselves, look at each
# dictionary component recursively to check for mismatches
if isinstance(obj1[key], list):
if len(obj1[key]) > 0:
if isinstance(obj1[key][0], dict):
for i in range(len(obj1[key])):
if not search_dicts(obj1[key][i],
obj2[key][i]):
return False
elif obj1[key] != obj2[key]:
return False
# if the next element is a dict, run through it recursively to
# determine if the separate elements of the dict match
if isinstance(obj1[key], (dict, collections.OrderedDict)):
if not search_dicts(obj1[key], obj2[key]):
return False
# if it is neither a list or a dictionary, compare to determine
# if the two elements match
elif obj1[key] != obj2[key]:
# if the two elements that are being compared are objects,
# make sure that they are the same type
if not isinstance(obj1[key], type(obj2[key])):
return False
return True
# make sure that the Vehicles class that was imported matches the
# original one
self.assertTrue(
search_dicts(imported_flow_params["veh"].__dict__,
flow_params["veh"].__dict__))
def visualizer_rllib(args, seed=None):
"""Visualizer for RLlib experiments.
This function takes args (see function create_parser below for
more detailed information on what information can be fed to this
visualizer), and renders the experiment associated with it.
"""
result_dir = args.result_dir if args.result_dir[-1] != '/' \
else args.result_dir[:-1]
config = get_rllib_config(result_dir)
# check if we have a multiagent environment but in a
# backwards compatible way
if config.get('multiagent', {}).get('policies', None):
multiagent = True
pkl = get_rllib_pkl(result_dir)
config['multiagent'] = pkl['multiagent']
else:
multiagent = False
config['callbacks'] = MyCallbacks
# Run on only one cpu for rendering purposes
config['num_workers'] = 0
flow_params = get_flow_params(config)
#flow_params['env'].additional_params["use_seeds"]=args.use_seeds
# print(args.use_seeds)
seed_tmp = None
if seed:
with open(seed, 'rb') as f:
seed_tmp = pickle.load(f)
config['seed'] = int(seed_tmp['rllib_seed'])
elif args.use_seeds:
with open(args.use_seeds, 'rb') as f:
seed_tmp = pickle.load(f)
config['seed'] = int(seed_tmp['rllib_seed'])
# hack for old pkl files
# TODO(ev) remove eventually
sim_params = flow_params['sim']
setattr(sim_params, 'num_clients', 1)
if seed_tmp:
#setattr(sim_params, 'seed', seed_tmp['sumo_seed'])
sim_params.seed = int(int(seed_tmp['sumo_seed']) / 10**6)
print(sim_params.seed)
#import IPython
#IPython.embed()
# Determine agent and checkpoint
config_run = config['env_config']['run'] if 'run' in config['env_config'] \
else None
if args.run and config_run:
if args.run != config_run:
print('visualizer_rllib.py: error: run argument ' +
'\'{}\' passed in '.format(args.run) +
'differs from the one stored in params.json ' +
'\'{}\''.format(config_run))
sys.exit(1)
# Merge with `evaluation_config`.
evaluation_config = copy.deepcopy(config.get("evaluation_config", {}))
config = merge_dicts(config, evaluation_config)
if args.run:
agent_cls = get_trainable_cls(args.run)
elif config_run:
agent_cls = get_trainable_cls(config_run)
else:
print('visualizer_rllib.py: error: could not find flow parameter '
'\'run\' in params.json, '
'add argument --run to provide the algorithm or model used '
'to train the results\n e.g. '
'python ./visualizer_rllib.py /tmp/ray/result_dir 1 --run PPO')
sys.exit(1)
sim_params.restart_instance = True
dir_path = os.path.dirname(os.path.realpath(__file__))
emission_path = '{0}/test_time_rollout/'.format(dir_path)
sim_params.emission_path = emission_path if args.gen_emission else None
# pick your rendering mode
if args.render_mode == 'sumo_web3d':
sim_params.num_clients = 2
sim_params.render = False
elif args.render_mode == 'drgb':
sim_params.render = 'drgb'
sim_params.pxpm = 4
elif args.render_mode == 'sumo_gui':
sim_params.render = True
print('NOTE: With render mode {}, an extra instance of the SUMO GUI '
'will display before the GUI for visualizing the result. Click '
'the green Play arrow to continue.'.format(args.render_mode))
elif args.render_mode == 'no_render':
sim_params.render = False
if args.save_render:
sim_params.render = 'drgb'
sim_params.pxpm = 4
sim_params.save_render = True
#if seed is not None:
# print(seed)
# flow_params["env"].additional_params["use_seeds"] = seed
# input()
#else:
# flow_params["env"].additional_params["use_seeds"] = args.use_seeds
if args.horizon:
config['horizon'] = args.horizon
flow_params['env'].horizon = args.horizon
# Create and register a gym+rllib env
register_time = time.time()
create_env, env_name = make_create_env(params=flow_params,
version=0,
seeds_file=seed)
register_env(env_name, create_env)
register_time = time.time() - register_time
print("Register Time:", register_time)
# check if the environment is a single or multiagent environment, and
# get the right address accordingly
# single_agent_envs = [env for env in dir(flow.envs)
# if not env.startswith('__')]
# if flow_params['env_name'] in single_agent_envs:
# env_loc = 'flow.envs'
# else:
# env_loc = 'flow.envs.multiagent'
# Start the environment with the gui turned on and a path for the
# emission file
env_params = flow_params['env']
env_params.restart_instance = True #False
if args.evaluate:
env_params.evaluate = True
# lower the horizon if testing
if args.horizon:
config['horizon'] = args.horizon
env_params.horizon = args.horizon
# create the agent that will be used to compute the actions
agent = agent_cls(env=env_name, config=config)
checkpoint = result_dir + '/checkpoint_' + args.checkpoint_num
checkpoint = checkpoint + '/checkpoint-' + args.checkpoint_num
agent.restore(checkpoint)
create_time = time.time()
if hasattr(agent, "local_evaluator") and \
os.environ.get("TEST_FLAG") != 'True':
env = agent.local_evaluator.env
else:
env = gym.make(env_name)
create_time = time.time() - create_time
print("Create time:", create_time)
if multiagent:
rets = {}
# map the agent id to its policy
print(config['multiagent']['policy_mapping_fn'])
policy_map_fn = config['multiagent']['policy_mapping_fn'] #.func
for key in config['multiagent']['policies'].keys():
rets[key] = []
else:
rets = []
if config['model']['use_lstm']:
use_lstm = True
if multiagent:
state_init = {}
# map the agent id to its policy
policy_map_fn = config['multiagent']['policy_mapping_fn'].func
size = config['model']['lstm_cell_size']
for key in config['multiagent']['policies'].keys():
state_init[key] = [
np.zeros(size, np.float32),
np.zeros(size, np.float32)
]
else:
state_init = [
np.zeros(config['model']['lstm_cell_size'], np.float32),
np.zeros(config['model']['lstm_cell_size'], np.float32)
]
else:
use_lstm = False
restart_time = time.time()
env.restart_simulation(sim_params=sim_params, render=sim_params.render)
restart_time = time.time() - restart_time
print("Restart Time:", restart_time)
# Simulate and collect metrics
final_outflows = []
final_inflows = []
mean_speed = []
std_speed = []
if PRINT_TO_SCREEN:
pp = pprint.PrettyPrinter(indent=2)
print("config ")
pp.pprint(config)
print("flow_params ")
pp.pprint(flow_params)
if REALTIME_PLOTS:
# prepare plots
# You probably won't need this if you're embedding things in a tkinter plot...
plt.ion()
fig = plt.figure()
axA = fig.add_subplot(331)
axA.set_title("Actions")
axR = fig.add_subplot(332)
axR.set_title("Rewards")
axS = fig.add_subplot(333)
axS.set_title("States")
axS0 = fig.add_subplot(334)
axS0.set_title("S0")
axS1 = fig.add_subplot(335)
axS1.set_title("S1")
axS2 = fig.add_subplot(336)
axS2.set_title("S2")
axA_hist = fig.add_subplot(337)
axA_hist.set_title("Actions")
axR_hist = fig.add_subplot(338)
axR_hist.set_title("Rewards")
axS_hist = fig.add_subplot(339)
axS_hist.set_title("States")
axS.set_ylim((-2, 3))
axA.set_ylim((-5, 5))
axR.set_ylim((-1, 1))
initialized_plot = False
# record for visualization purposes
actions = []
rewards = []
states = []
times = []
WARMUP = args.warmup
run_time = time.time()
for i in range(args.num_rollouts):
vel = []
time_to_exit = 0
state = env.reset()
if multiagent:
ret = {key: [0] for key in rets.keys()}
else:
ret = 0
for _ in range(env_params.horizon):
time_to_exit += 1
vehicles = env.unwrapped.k.vehicle
if np.mean(vehicles.get_speed(vehicles.get_ids())) > 0:
vel.append(np.mean(vehicles.get_speed(vehicles.get_ids())))
#vel.append(np.mean(vehicles.get_speed(vehicles.get_ids())))
if multiagent:
action = {}
for agent_id in state.keys():
if use_lstm:
action[agent_id], state_init[agent_id], logits = \
agent.compute_action(
state[agent_id], state=state_init[agent_id],
policy_id=policy_map_fn(agent_id))
else:
action[agent_id] = agent.compute_action(
state[agent_id], policy_id=policy_map_fn(agent_id))
else:
action = agent.compute_action(state)
state, reward, done, _ = env.step(action)
if SUMMARY_PLOTS:
# record for visualization purposes
actions.append(action)
rewards.append(reward)
states.append(state)
if PRINT_TO_SCREEN:
print("action")
pp.pprint(action)
print("reward")
pp.pprint(reward)
print("state")
pp.pprint(state)
print("after step ")
if REALTIME_PLOTS:
# Update plots.
if not initialized_plot: # initialize
lineA, = axA.plot(
[0] * len(action), 'g^'
) # Returns a tuple of line objects, thus the comma
lineR, = axR.plot(
0, 'bs'
) # Returns a tuple of line objects, thus the comma
lineS, = axS.plot(
[0] * len(state), 'r+'
) # Returns a tuple of line objects, thus the comma
initialized_plot = True
lineA.set_ydata(action)
lineR.set_ydata(reward)
lineS.set_ydata(state)
fig.canvas.draw()
fig.canvas.flush_events()
if multiagent:
for actor, rew in reward.items():
ret[policy_map_fn(actor)][0] += rew
else:
ret += reward
if multiagent and done['__all__']:
break
if not multiagent and done:
break
if args.use_delay > 0:
if vehicles.get_num_arrived() >= args.use_delay:
break
if multiagent:
for key in rets.keys():
rets[key].append(ret[key])
else:
rets.append(ret)
outflow = vehicles.get_outflow_rate(5000)
final_outflows.append(outflow)
inflow = vehicles.get_inflow_rate(5000)
final_inflows.append(inflow)
times.append(time_to_exit)
if np.all(np.array(final_inflows) > 1e-5):
throughput_efficiency = [
x / y for x, y in zip(final_outflows, final_inflows)
]
else:
throughput_efficiency = [0] * len(final_inflows)
mean_speed.append(np.mean(vel))
std_speed.append(np.std(vel))
if multiagent:
for agent_id, rew in rets.items():
print('Round {}, Return: {} for agent {}'.format(
i, ret, agent_id))
else:
print('Round {}, Return: {}'.format(i, ret))
run_time = time.time() - run_time
print('==== Summary of results ====')
print("Run Time: ", run_time)
print("Return:")
env.close()
return_reward = 0
if multiagent:
for agent_id, rew in rets.items():
print('For agent', agent_id)
print(rew)
print('Average, std return: {}, {} for agent {}'.format(
np.mean(rew), np.std(rew), agent_id))
return_reward = np.mean(rew)
else:
print(rets)
print('Average, std: {:.2f}, {:.5f}'.format(np.mean(rets),
np.std(rets)))
return_reward = np.mean(rets)
print("\nSpeed, mean (m/s):")
print(mean_speed)
print('Average, std: {:.2f}, {:.5f}'.format(np.mean(mean_speed),
np.std(mean_speed)))
print("\nSpeed, std (m/s):")
print(std_speed)
print('Average, std: {:.2f}, {:.5f}'.format(np.mean(std_speed),
np.std(std_speed)))
# Compute arrival rate of vehicles in the last 500 sec of the run
print("\nOutflows (veh/hr):")
print(final_outflows)
print('Average, std: {:.2f}, {:.5f}'.format(np.mean(final_outflows),
np.std(final_outflows)))
# Compute departure rate of vehicles in the last 500 sec of the run
print("Inflows (veh/hr):")
print(final_inflows)
print('Average, std: {:.2f}, {:.5f}'.format(np.mean(final_inflows),
np.std(final_inflows)))
# Compute throughput efficiency in the last 500 sec of the
print("Throughput efficiency (veh/hr):")
print(throughput_efficiency)
print('Average, std: {:.2f}, {:.5f}'.format(np.mean(throughput_efficiency),
np.std(throughput_efficiency)))
print("Time Delay")
print(times)
print("Time for certain number of vehicles to exit {:.2f},{:.5f}".format(
(np.mean(times)), np.std(times)))
if args.output:
np.savetxt(args.output, [
return_reward, mean_speed, std_speed, final_inflows,
final_outflows, times
])
if SUMMARY_PLOTS:
generateHtmlplots(actions, rewards, states)
# terminate the environment
env.unwrapped.terminate()
env.terminate()
# Deleting the env in order to remove sumo process
del env
del evaluation_config
# if prompted, convert the emission file into a csv file
if args.gen_emission:
time.sleep(0.1)
dir_path = os.path.dirname(os.path.realpath(__file__))
emission_filename = '{0}-emission.xml'.format(env.network.name)
emission_path = \
'{0}/test_time_rollout/{1}'.format(dir_path, emission_filename)
# convert the emission file into a csv file
emission_to_csv(emission_path)
# print the location of the emission csv file
emission_path_csv = emission_path[:-4] + ".csv"
print("\nGenerated emission file at " + emission_path_csv)
# delete the .xml version of the emission file
os.remove(emission_path)
# if we wanted to save the render, here we create the movie
if args.save_render:
dirs = os.listdir(os.path.expanduser('~') + '/flow_rendering')
# Ignore hidden files
dirs = [d for d in dirs if d[0] != '.']
dirs.sort(key=lambda date: datetime.strptime(date, "%Y-%m-%d-%H%M%S"))
recent_dir = dirs[-1]
# create the movie
movie_dir = os.path.expanduser('~') + '/flow_rendering/' + recent_dir
save_dir = os.path.expanduser('~') + '/flow_movies'
if not os.path.exists(save_dir):
os.mkdir(save_dir)
os_cmd = "cd " + movie_dir + " && ffmpeg -i frame_%06d.png"
os_cmd += " -pix_fmt yuv420p " + dirs[-1] + ".mp4"
os_cmd += "&& cp " + dirs[-1] + ".mp4 " + save_dir + "/"
os.system(os_cmd)
return return_reward, mean_speed, final_inflows, final_outflows
