def restart_simulation(self, sim_params, render=None): """Restart an already initialized simulation instance. This is used when visualizing a rollout, in order to update the rendering with potentially a gui and export emission data from sumo. This is also used to handle cases when the runtime of an experiment is too long, causing the sumo instance Parameters ---------- sim_params : flow.core.params.SimParams sumo-specific parameters render: bool, optional specifies whether to use the gui """ self.k.close() # killed the sumo process if using sumo/TraCI if self.simulator == 'traci': self.k.simulation.sumo_proc.kill() if render is not None: self.sim_params.render = render if sim_params.emission_path is not None: ensure_dir(sim_params.emission_path) self.sim_params.emission_path = sim_params.emission_path self.k.scenario.generate_network(self.scenario) self.traci_connection = self.k.simulation.start_simulation( scenario=self.k.scenario, sim_params=self.sim_params) self.k.pass_api(self.traci_connection) self.setup_initial_state()
def __init__(self, net_params, base): """Base class for generating transportation networks. Uses network specific features to generate the necessary xml files needed to initialize a sumo instance. The methods of this class are called by the base scenario class. Attributes ---------- net_params: NetParams type see flow/core/params.py base: str base name for the transportation network. If not specified in the child class, this is also the complete name for the network. """ # Invoke serializable if using rllab if Serializable is not object: Serializable.quick_init(self, locals()) self.net_params = net_params self.net_path = os.path.dirname(os.path.abspath(__file__)) \ + "/debug/net/" self.cfg_path = os.path.dirname(os.path.abspath(__file__)) \ + "/debug/cfg/" self.base = base self.netfn = "" self.vehicle_ids = [] ensure_dir("%s" % self.net_path) ensure_dir("%s" % self.cfg_path) # if a name was not specified by the sub-class's initialization, # use the base as the name if not hasattr(self, "name"): self.name = "%s" % self.base
def train(model_params=None): args = parse_args(sys.argv[1:]) # import relevant information from the exp_config script module = __import__("config", fromlist=[args.exp_config]) if hasattr(module, args.exp_config): submodule = getattr(module, args.exp_config) else: assert False, "Unable to find experiment config!" flow_params = submodule.flow_params # Path to the saved files result_name = '{}/{}'.format(flow_params['exp_tag'], strftime("%Y-%m-%d-%H:%M:%S")) # Perform training. print('Beginning training.') model = run_model_stablebaseline(flow_params=flow_params, args=args, model_params=model_params) # Save the model to a desired folder and then delete it to demonstrate loading. print('Saving the trained model!') path = os.path.realpath(os.path.expanduser('~/baseline_results')) ensure_dir(path) save_path = os.path.join(path, result_name) model.save(save_path) # dump the flow params with open(os.path.join(path, result_name) + '.json', 'w') as outfile: json.dump(flow_params, outfile, cls=FlowParamsEncoder, sort_keys=True, indent=4)
def restart_sumo(self, sumo_params, sumo_binary=None): """Restarts an already initialized sumo instance. This is used when visualizing a rollout, in order to update the sumo_binary with potentially a gui and export emission data from sumo. This is also used to handle cases when the runtime of an experiment is too long, causing the sumo instance Parameters ---------- sumo_params: SumoParams type sumo-specific parameters sumo_binary: str, optional specifies whether to use sumo's gui """ self.traci_connection.close(False) self.sumo_proc.kill() if sumo_binary is not None: self.sumo_params.sumo_binary = sumo_binary if sumo_params.emission_path is not None: ensure_dir(sumo_params.emission_path) self.sumo_params.emission_path = sumo_params.emission_path self.start_sumo() self.setup_initial_state()
def __init__(self, master_kernel, sim_params): """Instantiate a otm network kernel. Parameters ---------- master_kernel : flow.core.kernel.Kernel the higher level kernel (used to call methods from other sub-kernels) sim_params : flow.core.params.SimParams simulation-specific parameters """ super(OTMKernelNetwork, self).__init__(master_kernel, sim_params) # # directories for the network-specific files that will be generated by # # the `generate_network` method self.net_path = os.path.dirname(os.path.abspath(__file__)) \ + '/debug/otmxml/' ensure_dir('%s' % self.net_path) # variables to be defined during network generation self.network = None self.nodfn = None # self.guifn = None self._edges = None self._connections = None self._edge_list = None self._junction_list = None self.__max_speed = None self.__length = None # total length self.__non_internal_length = None # total length of non-internal edges self.rts = None self.cfg = None
def train_stable_baselines(submodule, flags): """Train policies using the PPO algorithm in stable-baselines.""" from stable_baselines3.common.vec_env import DummyVecEnv flow_params = submodule.flow_params # Path to the saved files exp_tag = flow_params['exp_tag'] result_name = '{}/{}'.format(exp_tag, strftime("%Y-%m-%d-%H:%M:%S")) # Perform training. start_time = timeit.default_timer() # print experiment.json information print("=========================================") print('Beginning training.') print('Algorithm :', flags.algorithm) model = run_model_stablebaseline(flow_params, flags.num_cpus, flags.rollout_size, flags.num_steps, flags.algorithm, flags.exp_config) stop_time = timeit.default_timer() run_time = stop_time - start_time print("Training is Finished") print("total runtime: ", run_time) # Save the model to a desired folder and then delete it to demonstrate # loading. print('Saving the trained model!') path = os.path.realpath(os.path.expanduser('~/baseline_results')) ensure_dir(path) save_path = os.path.join(path, result_name) model.save(save_path) # dump the flow params with open(os.path.join(path, result_name) + '.json', 'w') as outfile: json.dump(flow_params, outfile, cls=FlowParamsEncoder, sort_keys=True, indent=4) # Replay the result by loading the model print('Loading the trained model and testing it out!') if flags.exp_config.lower() == "ppo": from stable_baselines3 import PPO model = PPO.load(save_path) elif flags.exp_config.lower() == "ddpg": from stable_baselines3 import DDPG model = DDPG.load(save_path) flow_params = get_flow_params(os.path.join(path, result_name) + '.json') flow_params['sim'].render = True env = env_constructor(params=flow_params, version=0)() # The algorithms require a vectorized environment to run eval_env = DummyVecEnv([lambda: env]) obs = eval_env.reset() reward = 0 for _ in range(flow_params['env'].horizon): action, _states = model.predict(obs) obs, rewards, dones, info = eval_env.step(action) reward += rewards print('the final reward is {}'.format(reward))
def train_stable_baselines3(submodule, flags): """Train policies using the PPO algorithm in stable-baselines3.""" from stable_baselines3.common.vec_env import DummyVecEnv from stable_baselines3 import PPO import torch start_time = timeit.default_timer() flow_params = submodule.flow_params # Path to the saved files exp_tag = flow_params['exp_tag'] result_name = '{}/{}'.format(exp_tag, strftime("%Y-%m-%d-%H:%M:%S")) # Perform training. print("cuda is available: ", torch.cuda.is_available()) print('Beginning training.') print("==========================================") model = run_model_stablebaseline3(flow_params, flags.num_cpus, flags.rollout_size, flags.num_steps) # Save the model to a desired folder and then delete it to demonstrate # loading. print('Saving the trained model!') path = os.path.realpath(os.path.expanduser('~/baseline_results')) ensure_dir(path) save_path = os.path.join(path, result_name) model.save(save_path) # dump the flow params # check time for choose GPU and CPU stop_time = timeit.default_timer() run_time = stop_time - start_time with open(os.path.join(path, result_name) + '.json', 'w') as outfile: json.dump(flow_params, outfile, cls=FlowParamsEncoder, sort_keys=True, indent=4) # Replay the result by loading the model print('Loading the trained model and testing it out!') model.load(save_path) flow_params = get_flow_params(os.path.join(path, result_name) + '.json') flow_params['sim'].render = False flow_params['env'].horizon = 1500 # 150seconds operation env = env_constructor(params=flow_params, version=0)() # The algorithms require a vectorized environment to run eval_env = DummyVecEnv([lambda: env]) obs = eval_env.reset() reward = 0 for _ in range(flow_params['env'].horizon): action, _states = model.predict(obs) obs, rewards, dones, info = eval_env.step(action) reward += rewards print("--------------------------------------------------------") flow_params['sim'].render = True simulation = Experiment(flow_params) simulation.run(num_runs=1) print('the final reward is {}'.format(reward)) print("total run_time:", run_time, "s")
def run(self, num_runs, num_steps, rl_actions=None): """Run the given scenario for a set number of runs and steps per run. Parameters ---------- num_runs : int number of runs the experiment should perform num_steps : int number of steps to be performs in each run of the experiment rl_actions : method, optional maps states to actions to be performed by the RL agents (if there are any) Returns ------- info_dict : dict contains returns, average speed per step """ if rl_actions is None: def rl_actions(*_): return None dir_path = './flow_density/{}'.format(self.env.scenario.orig_name) ensure_dir(dir_path) res = {'vels': [], 'outflows': [], 'densities': []} for i in range(num_runs): vel = [] outflow = [] density = [] state = self.env.reset() for j in range(num_steps): state, reward, done, _ = self.env.step(rl_actions(state)) vel.append( np.mean( self.env.k.vehicle.get_speed( self.env.k.vehicle.get_ids()))) outflow.append( self.env.k.vehicle.get_outflow_rate(time_span=30)) density.append(self.env.k.vehicle.num_vehicles / self.env.k.scenario.length()) if done: break res['vels'] = vel res['outflows'] = outflow res['densities'] = density print("Round {0}, done".format(i)) with open("{}/{}.csv".format(dir_path, i), "w") \ as outfile: writer = csv.writer(outfile) writer.writerow(res.keys()) writer.writerows(zip(*res.values())) self.env.terminate() return None
def __init__(self): # specify path for tensorboard logging avg travel times, avg reward, no. of vehicles in network # create it if it directory doesn't exist root_dir = os.path.expanduser('~') tensorboard_log_dir = root_dir + '/ray_results/real_time_metrics' self.summaries_dir = ensure_dir(tensorboard_log_dir) # file location to store simulation numbers csv while training # ie. if the training is at simulation 50, the last value add the created csv file, will be 50 # this helps us keep track of the average travel times, rewards and other values at the end # of each simulation simulation_log_location = root_dir + '/ray_results/num_of_simulations' ensure_dir(simulation_log_location) # saving flag, file location and name of histogram of results # plot and save histogram of travel time distribution at end of simulation self.save_plots = False self.experiment_title = "1x1_Centralized_Thesis" hist_dir = root_dir + '/ray_results/histograms' self.full_histogram_path = ensure_dir(hist_dir) + "/" + self.experiment_title # logging evolution of reward during a simulation self.log_rewards_during_iteration = False # activate sumo actuated baseline during training for first 6 iterations self.sumo_actuated_baseline = False self.sumo_actuated_simulations = 6 # choose grid type for naming the tensorboard directory logging # will be the name and title of histogram (from save_plots above) to be saved self.grid_demand = self.experiment_title # choose exp running, can either be "rl" or "non_rl" self.exp = "rl" # choose look-ahead distance can be either 43, 80, 160 or 240 self.look_ahead = 43 # log title for tensorboard and name + file path for simulation log self.log_title = '/simulation_analysis_{}_{}_{}'.format(self.exp, self.look_ahead, self.grid_demand) self.filename = "/iterations_{}_{}.csv".format(self.look_ahead, self.grid_demand) self.full_path = simulation_log_location + self.filename # RGB colors for vehicles if coloring a vehicle. We color the observed incoming # vehicles BLUE and the observed outgoing vehicles as RED at each intersection self.RED = (255, 0, 0) self.BLUE = (0, 0, 255) self.CYAN = (0, 255, 255) # set up more tensorboard logging info self.exp_name = self.log_title self.writer = SummaryWriter(self.summaries_dir + self.exp_name) self.rew_list = []
def test_trpo_runner(self): # test run_model on figure eight 0 model = run_model(figureeight0.flow_params, 5, 5) # test save model ensure_dir("./baseline_results") save_model(model, figureeight0.flow_params, "./baseline_results") files = sorted([f for f in listdir("./baseline_results") if isfile(join("./baseline_results", f))]) self.assertListEqual(files, ['flow_params.json', 'model.pkl']) # delete the generated files shutil.rmtree('./baseline_results')
def start_simulation(self, network, sim_params): """See parent class. This method calls the aimsun generator to generate the network, starts a simulation, and creates a class to communicate with the simulation via an TCP connection. """ # save the simulation step size (for later use) self.sim_step = sim_params.sim_step self.emission_path = sim_params.emission_path if self.emission_path is not None: ensure_dir(self.emission_path) return FlowAimsunAPI(port=sim_params.port)
def train_stable_baselines(submodule, flags): """Train policies using the PPO algorithm in stable-baselines.""" from stable_baselines.common.vec_env import DummyVecEnv from stable_baselines import DDPG flow_params = submodule.flow_params # Path to the saved files exp_tag = flow_params['exp_tag'] result_name = '{}/{}'.format(exp_tag, strftime("%Y-%m-%d-%H:%M:%S")) # Perform training. print('Beginning training.') model = run_model_stablebaseline( flow_params, flags.num_cpus, flags.rollout_size, flags.num_steps) # Save the model to a desired folder and then delete it to demonstrate # loading. print('Saving the trained model!') path = os.path.realpath(os.path.expanduser('~/baseline_results')) ensure_dir(path) save_path = os.path.join(path, result_name) model.save(save_path) # dump the flow params with open(os.path.join(path, result_name) + '.json', 'w') as outfile: json.dump(flow_params, outfile, cls=FlowParamsEncoder, sort_keys=True, indent=4) # Replay the result by loading the model print('Loading the trained model and testing it out!') model = DDPG.load(save_path) flow_params = get_flow_params(os.path.join(path, result_name) + '.json') flow_params['sim'].render = True env = env_constructor(params=flow_params, version=0)() n_actions = env.action_space.shape[-1] param_noise = None action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions), sigma=float(0.5) * np.ones(n_actions)) # The algorithms require a vectorized environment to run eval_env = DummyVecEnv([lambda: env]) obs = eval_env.reset() reward = 0 for _ in range(flow_params['env'].horizon): action, _states = model.predict(obs) obs, rewards, dones, info = eval_env.step(action) reward += rewards print('the final reward is {}'.format(reward))
def __init__(self, net_params, base): Serializable.quick_init(self, locals()) self.net_params = net_params self.net_path = net_params.net_path self.cfg_path = net_params.cfg_path self.base = base self.netfn = "" self.vehicle_ids = [] ensure_dir("%s" % self.net_path) ensure_dir("%s" % self.cfg_path) # if a name was not specified by the sub-class's initialization, # use the base as the name if not hasattr(self, "name"): self.name = "%s" % self.base
def __init__(self, env_params, sim_params, network, simulator='traci'): for p in ADDITIONAL_ENV_PARAMS.keys(): if p not in env_params.additional_params: raise KeyError( 'Environment parameter \'{}\' not supplied'.format(p)) self.continuous_low_speed = 0 self.prev_pos = dict() self.absolute_position = dict() if sim_params.emission_path is not None: self.path = sim_params.emission_path if self.path[-1] != '/': self.path += '/' self.path += 'pics' ensure_dir(self.path) super().__init__(env_params, sim_params, network, simulator)
def restart_sumo(self, sumo_params, sumo_binary=None): """ Restarts an already initialized environment. Used when visualizing a rollout. """ self.traci_connection.close(False) if sumo_binary: self.sumo_binary = sumo_binary self.port = sumolib.miscutils.getFreeSocketPort() data_folder = self.emission_path ensure_dir(data_folder) self.emission_out = \ data_folder + "{0}-emission.xml".format(self.scenario.name) self.start_sumo() self.setup_initial_state()
def __init__(self, master_kernel, sim_params): """Instantiate a sumo scenario kernel. Parameters ---------- master_kernel : flow.core.kernel.Kernel the higher level kernel (used to call methods from other sub-kernels) sim_params : flow.core.params.SimParams simulation-specific parameters """ super(TraCIScenario, self).__init__(master_kernel, sim_params) # directories for the network-specific files that will be generated by # the `generate_network` method self.net_path = os.path.dirname(os.path.abspath(__file__)) \ + '/debug/net/' self.cfg_path = os.path.dirname(os.path.abspath(__file__)) \ + '/debug/cfg/' ensure_dir('%s' % self.net_path) ensure_dir('%s' % self.cfg_path) # variables to be defined during network generation self.network = None self.nodfn = None self.edgfn = None self.typfn = None self.cfgfn = None self.netfn = None self.confn = None self.roufn = None self.addfn = None self.sumfn = None self.guifn = None self._edges = None self._connections = None self._edge_list = None self._junction_list = None self.__max_speed = None self.__length = None self.rts = None self.cfg = None
def __init__(self, master_kernel, sim_params): """Instantiate a sumo network kernel. Parameters ---------- master_kernel : flow.core.kernel.Kernel the higher level kernel (used to call methods from other sub-kernels) sim_params : flow.core.params.SimParams simulation-specific parameters """ super(TraCIKernelNetwork, self).__init__(master_kernel, sim_params) # directories for the network-specific files that will be generated by # the `generate_network` method self.net_path = os.path.join(tempfile.gettempdir(), 'flow/debug/net/') self.cfg_path = os.path.join(tempfile.gettempdir(), 'flow/debug/cfg/') ensure_dir('%s' % self.net_path) ensure_dir('%s' % self.cfg_path) # variables to be defined during network generation self.network = None self.nodfn = None self.edgfn = None self.typfn = None self.cfgfn = None self.netfn = None self.confn = None self.roufn = None self.addfn = None self.sumfn = None self.guifn = None self._edges = None self._connections = None self._edge_list = None self._junction_list = None self.__max_speed = None self.__length = None # total length self.__non_internal_length = None # total length of non-internal edges self.rts = None self.cfg = None
def __init__(self, net_params, base): """Instantiate the base generator class. Attributes ---------- net_params : NetParams type see flow/core/params.py base : str base name for the transportation network. If not specified in the child class, this is also the complete name for the network. """ # Invoke serializable if using rllab if Serializable is not object: Serializable.quick_init(self, locals()) self.net_params = net_params self.net_path = os.path.dirname(os.path.abspath(__file__)) \ + "/debug/net/" self.cfg_path = os.path.dirname(os.path.abspath(__file__)) \ + "/debug/cfg/" self.base = base self.vehicle_ids = [] ensure_dir("%s" % self.net_path) ensure_dir("%s" % self.cfg_path) # if a name was not specified by the sub-class's initialization, # use the base as the name if not hasattr(self, "name"): self.name = "%s" % self.base self.nodfn = "%s.nod.xml" % self.name self.edgfn = "%s.edg.xml" % self.name self.typfn = "%s.typ.xml" % self.name self.cfgfn = "%s.netccfg" % self.name self.netfn = "%s.net.xml" % self.name self.confn = "%s.con.xml" % self.name self.roufn = "%s.rou.xml" % self.name self.addfn = "%s.add.xml" % self.name self.sumfn = "%s.sumo.cfg" % self.name self.guifn = "%s.gui.cfg" % self.name
# exp.run(10, 1000) # # # intersection environment for inflows from 400-3000 # for length in np.arange(300, 510, 10): # env = ring_env(length, param, i) # exp = FlowDensityExperiment(env) # exp.run(10, 1000) # ----------------------------------------------------------------------- # # ---------------------- Used for data collection ----------------------- # # ----------------------------------------------------------------------- # # # for cf params for i, param in enumerate(CF_PARAMS): # create necessary directories ensure_dir('./data/'.format(i)) # # merge environment # data = defaultdict(list) # while len(data.get('speed', [])) < TOTAL_STEPS: # inflow = random.uniform(MERGE_RANGE[i][0], MERGE_RANGE[i][1]) # env = merge_env(inflow, param, i, False) # exp = DataGenerationExperiment(env, 'merge') # # # collect new samples # new_data = exp.run(1, 30000) # # # extend the new data # for key in new_data.keys(): # data[key].extend(new_data[key]) #
def generate_cfg(self, net_params, traffic_lights, routes): """Generate .sumo.cfg files using net files and netconvert. This method is responsible for creating the following config files: - *.add.xml: This file contains the sumo-specific properties of vehicles with similar types, and properties of the traffic lights. - *.rou.xml: This file contains the routes vehicles can traverse, either from a specific starting edge, or by vehicle name, and well as the inflows of vehicles. - *.gui.cfg: This file contains the view settings of the gui (whether the gui is used or not). The background of the gui is set here to be grey, with RGB values: (100, 100, 100). - *.sumo.cfg: This is the file that is used by the simulator to identify the location of the various network, vehicle, and traffic light properties that are used when instantiating the simulation. Parameters ---------- net_params : flow.core.params.NetParams see flow/core/params.py traffic_lights : flow.core.params.TrafficLightParams traffic light information, used to determine which nodes are treated as traffic lights routes : dict Key = name of the starting edge Element = list of edges a vehicle starting from this edge must traverse. """ # this is the data that we will pass to the *.add.xml file add = makexml('additional', 'http://sumo.dlr.de/xsd/additional_file.xsd') # add the types of vehicles to the xml file for params in self.network.vehicles.types: type_params_str = { key: str(params['type_params'][key]) for key in params['type_params'] } add.append(E('vType', id=params['veh_id'], **type_params_str)) # add (optionally) the traffic light properties to the .add.xml file num_traffic_lights = len(list(traffic_lights.get_properties().keys())) if num_traffic_lights > 0: if traffic_lights.baseline: tl_params = traffic_lights.actuated_default() tl_type = str(tl_params['tl_type']) program_id = str(tl_params['program_id']) phases = tl_params['phases'] max_gap = str(tl_params['max_gap']) detector_gap = str(tl_params['detector_gap']) show_detector = tl_params['show_detectors'] detectors = {'key': 'detector-gap', 'value': detector_gap} gap = {'key': 'max-gap', 'value': max_gap} if show_detector: show_detector = {'key': 'show-detectors', 'value': 'true'} else: show_detector = {'key': 'show-detectors', 'value': 'false'} nodes = self._inner_nodes # nodes where there's traffic lights tll = [] for node in nodes: tll.append({ 'id': node['id'], 'type': tl_type, 'programID': program_id }) for elem in tll: e = E('tlLogic', **elem) e.append(E('param', **show_detector)) e.append(E('param', **gap)) e.append(E('param', **detectors)) for phase in phases: e.append(E('phase', **phase)) add.append(e) else: tl_properties = traffic_lights.get_properties() for node in tl_properties.values(): # At this point, we assume that traffic lights are properly # formed. If there are no phases for a static traffic # light, ignore and use default if node['type'] == 'static' and not node.get('phases'): continue elem = { 'id': str(node['id']), 'type': str(node['type']), 'programID': str(node['programID']) } if node.get('offset'): elem['offset'] = str(node.get('offset')) e = E('tlLogic', **elem) for key, value in node.items(): if key == 'phases': for phase in node.get('phases'): e.append(E('phase', **phase)) else: e.append( E('param', **{ 'key': key, 'value': str(value) })) add.append(e) printxml(add, self.cfg_path + self.addfn) # this is the data that we will pass to the *.gui.cfg file gui = E('viewsettings') gui.append(E('scheme', name='real world')) gui.append( E('background', backgroundColor='100,100,100', showGrid='0', gridXSize='100.00', gridYSize='100.00')) printxml(gui, self.cfg_path + self.guifn) # this is the data that we will pass to the *.rou.xml file routes_data = makexml('routes', 'http://sumo.dlr.de/xsd/routes_file.xsd') # add the routes to the .add.xml file for route_id in routes.keys(): # in this case, we only have one route, convert into into a # list of routes with one element if isinstance(routes[route_id][0], str): routes[route_id] = [(routes[route_id], 1)] # add each route incrementally, and add a second term to denote # the route number of the given route at the given edge for i in range(len(routes[route_id])): r, _ = routes[route_id][i] routes_data.append( E('route', id='route{}_{}'.format(route_id, i), edges=' '.join(r))) # add the inflows from various edges to the xml file if self.network.net_params.inflows is not None: total_inflows = self.network.net_params.inflows.get() for next_inflow in total_inflows: # do not want to affect the original values inflow = deepcopy(next_inflow) # convert any non-string element in the inflow dict to a string for key in inflow: if not isinstance(inflow[key], str): inflow[key] = repr(inflow[key]) # get the name of the edge the inflows correspond to, and the # total inflow rate of the specific inflow edge = deepcopy(inflow['edge']) if 'vehsPerHour' in inflow: flag, rate = 0, float(inflow['vehsPerHour']) else: flag, rate = 1, float(inflow['probability']) del inflow['edge'] # distribute the inflow rates across all routes from a given # edge on the basis of the provided fractions for each route for i in range(len(routes[edge])): _, frac = routes[edge][i] inflow['route'] = 'route{}_{}'.format(edge, i) if flag: inflow['probability'] = str(rate * frac) else: inflow['vehsPerHour'] = str(rate * frac) routes_data.append(_flow(**inflow)) printxml(routes_data, self.cfg_path + self.roufn) # this is the data that we will pass to the *.sumo.cfg file cfg = makexml('configuration', 'http://sumo.dlr.de/xsd/sumoConfiguration.xsd') cfg.append( _inputs(net=self.netfn, add=self.addfn, rou=self.roufn, gui=self.guifn)) # initial time of the simulation cfg.append(E('begin', value=repr(0))) # simulation step length cfg.append(E('step-length', value=repr(self.sim_params.sim_step))) # add the lateral resolution of the sublanes (if requested) if self.sim_params.lateral_resolution is not None: cfg.append( E("lateral-resolution", value=str(self.sim_params.lateral_resolution))) # add the emission path to the sumo command (if requested) if self.sim_params.emission_path is not None: ensure_dir(self.sim_params.emission_path) emission_out = os.path.join( self.sim_params.emission_path, "{0}-emission.xml".format(self.network.name)) cfg.append(E("emission-output", value=emission_out)) # add step logs (if requested) no_step_log = "true" if self.sim_params.no_step_log else "false" cfg.append(E("no-step-log", value=no_step_log)) if self.sim_params.overtake_right: cfg.append(E("lanechange.overtake-right", value="true")) # specify a simulation seed (if requested) if self.sim_params.seed is not None: cfg.append(E("seed", value=str(self.sim_params.seed))) if not self.sim_params.print_warnings: cfg.append(E("no-warnings", value="true")) # set the time it takes for a gridlock teleport to occur cfg.append( E("time-to-teleport", value=str(int(self.sim_params.teleport_time)))) # check collisions at intersections cfg.append(E("collision.check-junctions", value="true")) printxml(cfg, self.cfg_path + self.sumfn) return self.sumfn
def start_simulation(self, network, sim_params): """Start a sumo simulation instance. This method uses the configuration files created by the network class to initialize a sumo instance. Also initializes a traci connection to interface with sumo from Python. """ error = None for _ in range(RETRIES_ON_ERROR): try: # port number the sumo instance will be run on port = sim_params.port sumo_binary = "sumo-gui" if sim_params.render is True \ else "sumo" # command used to start sumo sumo_call = [ sumo_binary, "-c", network.cfg, "--remote-port", str(sim_params.port), "--num-clients", str(sim_params.num_clients), "--step-length", str(sim_params.sim_step) ] # add step logs (if requested) if sim_params.no_step_log: sumo_call.append("--no-step-log") # add the lateral resolution of the sublanes (if requested) if sim_params.lateral_resolution is not None: sumo_call.append("--lateral-resolution") sumo_call.append(str(sim_params.lateral_resolution)) # add the emission path to the sumo command (if requested) if sim_params.emission_path is not None: ensure_dir(sim_params.emission_path) emission_out = os.path.join( sim_params.emission_path, "{0}-emission.xml".format(network.name)) sumo_call.append("--emission-output") sumo_call.append(emission_out) else: emission_out = None if sim_params.overtake_right: sumo_call.append("--lanechange.overtake-right") sumo_call.append("true") # specify a simulation seed (if requested) if sim_params.seed is not None: sumo_call.append("--seed") sumo_call.append(str(sim_params.seed)) if not sim_params.print_warnings: sumo_call.append("--no-warnings") sumo_call.append("true") # set the time it takes for a gridlock teleport to occur sumo_call.append("--time-to-teleport") sumo_call.append(str(int(sim_params.teleport_time))) # check collisions at intersections sumo_call.append("--collision.check-junctions") sumo_call.append("true") logging.info(" Starting SUMO on port " + str(port)) logging.debug(" Cfg file: " + str(network.cfg)) if sim_params.num_clients > 1: logging.info(" Num clients are" + str(sim_params.num_clients)) logging.debug(" Emission file: " + str(emission_out)) logging.debug(" Step length: " + str(sim_params.sim_step)) # Opening the I/O thread to SUMO self.sumo_proc = subprocess.Popen(sumo_call, preexec_fn=os.setsid) # wait a small period of time for the subprocess to activate # before trying to connect with traci if os.environ.get("TEST_FLAG", 0): time.sleep(0.1) else: time.sleep(config.SUMO_SLEEP) traci_connection = traci.connect(port, numRetries=100) traci_connection.setOrder(0) traci_connection.simulationStep() return traci_connection except Exception as e: print("Error during start: {}".format(traceback.format_exc())) error = e self.teardown_sumo() raise error
def train_h_baselines(env_name, args, multiagent): """Train policies using SAC and TD3 with h-baselines.""" from hbaselines.algorithms import OffPolicyRLAlgorithm from hbaselines.utils.train import parse_options, get_hyperparameters # Get the command-line arguments that are relevant here args = parse_options(description="", example_usage="", args=args) # the base directory that the logged data will be stored in base_dir = "training_data" for i in range(args.n_training): # value of the next seed seed = args.seed + i # The time when the current experiment started. now = strftime("%Y-%m-%d-%H:%M:%S") # Create a save directory folder (if it doesn't exist). dir_name = os.path.join(base_dir, '{}/{}'.format(args.env_name, now)) ensure_dir(dir_name) # Get the policy class. if args.alg == "TD3": if multiagent: from hbaselines.multi_fcnet.td3 import MultiFeedForwardPolicy policy = MultiFeedForwardPolicy else: from hbaselines.fcnet.td3 import FeedForwardPolicy policy = FeedForwardPolicy elif args.alg == "SAC": if multiagent: from hbaselines.multi_fcnet.sac import MultiFeedForwardPolicy policy = MultiFeedForwardPolicy else: from hbaselines.fcnet.sac import FeedForwardPolicy policy = FeedForwardPolicy else: raise ValueError("Unknown algorithm: {}".format(args.alg)) # Get the hyperparameters. hp = get_hyperparameters(args, policy) # Add the seed for logging purposes. params_with_extra = hp.copy() params_with_extra['seed'] = seed params_with_extra['env_name'] = args.env_name params_with_extra['policy_name'] = policy.__name__ params_with_extra['algorithm'] = args.alg params_with_extra['date/time'] = now # Add the hyperparameters to the folder. with open(os.path.join(dir_name, 'hyperparameters.json'), 'w') as f: json.dump(params_with_extra, f, sort_keys=True, indent=4) # Create the algorithm object. alg = OffPolicyRLAlgorithm( policy=policy, env="flow:{}".format(env_name), eval_env="flow:{}".format(env_name) if args.evaluate else None, **hp) # Perform training. alg.learn( total_steps=args.total_steps, log_dir=dir_name, log_interval=args.log_interval, eval_interval=args.eval_interval, save_interval=args.save_interval, initial_exploration_steps=args.initial_exploration_steps, seed=seed, )
# dump the flow params with open(os.path.join(save_path, 'flow_params.json'), 'w') as outfile: json.dump( params, outfile, cls=FlowParamsEncoder, sort_keys=True, indent=4) if __name__ == "__main__": flags = parse_args(sys.argv[1:]) # Import the benchmark and fetch its flow_params module = __import__("flow.benchmarks.{}".format(flags.benchmark_name), fromlist=["flow_params"]) flow_params = module.flow_params # Path to the saved files exp_tag = flow_params['exp_tag'] result_name = '{}/{}'.format(exp_tag, strftime("%Y-%m-%d-%H:%M:%S")) # Define the save path and ensure that the required directories exist. dir_path = os.path.realpath(os.path.expanduser('~/baseline_results')) ensure_dir(dir_path) path = os.path.join(dir_path, result_name) # Perform the training operation. train_model = run_model(flow_params, flags.rollout_size, flags.num_steps) # Save the model to a desired folder and then delete it to demonstrate # loading. save_model(train_model, flow_params, path)
elif flags.rl_trainer == "Stable-Baselines": flow_params = submodule.flow_params # Path to the saved files exp_tag = flow_params['exp_tag'] result_name = '{}/{}'.format(exp_tag, strftime("%Y-%m-%d-%H:%M:%S")) # Perform training. print('Beginning training.') model = run_model_stablebaseline(flow_params, flags.num_cpus, flags.rollout_size, flags.num_steps) # Save the model to a desired folder and then delete it to demonstrate # loading. print('Saving the trained model!') path = os.path.realpath(os.path.expanduser('~/baseline_results')) ensure_dir(path) save_path = os.path.join(path, result_name) model.save(save_path) # dump the flow params with open(os.path.join(path, result_name) + '.json', 'w') as outfile: json.dump(flow_params, outfile, cls=FlowParamsEncoder, sort_keys=True, indent=4) # Replay the result by loading the model print('Loading the trained model and testing it out!') model = PPO2.load(save_path) flow_params = get_flow_params(
def __init__(self, env_params, sumo_params, scenario): """ Base environment class. Provides the interface for controlling a SUMO simulation. Using this class, you can start sumo, provide a scenario to specify a configuration and controllers, perform simulation steps, and reset the simulation to an initial configuration. SumoEnvironment is Serializable to allow for pickling of the policy. This class cannot be used as is: you must extend it to implement an action applicator method, and properties to define the MDP if you choose to use it with RLLab. This can be done by overloading the following functions in a child class: - action_space - observation_space - apply_rl_action - get_state - compute_reward Attributes ---------- env_params: EnvParams type: see flow/core/params.py sumo_params: SumoParams type see flow/core/params.py scenario: Scenario type see flow/scenarios/base_scenario.py """ Serializable.quick_init(self, locals()) self.env_params = env_params self.scenario = scenario self.sumo_params = sumo_params self.sumo_binary = self.sumo_params.sumo_binary self.vehicles = scenario.vehicles # timer: Represents number of steps taken since the start of a rollout self.timer = 0 # initial_state: # Key = Vehicle ID, # Entry = (type_id, route_id, lane_index, lane_pos, speed, pos) self.initial_state = {} # vehicle identifiers for all vehicles self.ids = [] # vehicle identifiers for specific types of vehicles self.controlled_ids, self.sumo_ids, self.rl_ids = [], [], [] self.state = None self.obs_var_labels = [] # SUMO Params if sumo_params.port: self.port = sumo_params.port else: self.port = sumolib.miscutils.getFreeSocketPort() self.time_step = sumo_params.time_step self.vehicle_arrangement_shuffle = \ sumo_params.vehicle_arrangement_shuffle self.starting_position_shuffle = sumo_params.starting_position_shuffle self.emission_path = sumo_params.emission_path # path to the output (emission) file provided by sumo if self.emission_path: ensure_dir(self.emission_path) self.emission_out = \ self.emission_path + "{0}-emission.xml".format( self.scenario.name) else: self.emission_out = None self.fail_safe = env_params.fail_safe self.max_speed = env_params.max_speed self.lane_change_duration = \ env_params.get_lane_change_duration(self.time_step) self.shared_reward = env_params.shared_reward self.shared_policy = env_params.shared_policy # the available_routes variable contains a dictionary of routes vehicles # can traverse; to be used when routes need to be chosen dynamically self.available_routes = self.scenario.generator.rts # Check if we are in a multi-agent scenario if isinstance(self.action_space, list): self.multi_agent = True else: self.multi_agent = False self.start_sumo() self.setup_initial_state()
def start_sumo(self): """Starts a sumo instance. Uses the configuration files created by the generator class to initialize a sumo instance. Also initializes a traci connection to interface with sumo from Python. """ error = None for _ in range(RETRIES_ON_ERROR): try: # port number the sumo instance will be run on if self.sumo_params.port is not None: port = self.sumo_params.port else: # Don't do backoff when testing if os.environ.get("TEST_FLAG", 0): # backoff to decrease likelihood of race condition time_stamp = ''.join(str(time.time()).split('.')) time.sleep(1.0 * int(time_stamp[-6:]) / 1e6) # 1.0 for consistency w/ above port = sumolib.miscutils.getFreeSocketPort() # command used to start sumo sumo_call = [self.sumo_params.sumo_binary, "-c", self.scenario.cfg, "--remote-port", str(port), "--step-length", str(self.sim_step)] # add step logs (if requested) if self.sumo_params.no_step_log: sumo_call.append("--no-step-log") # add the lateral resolution of the sublanes (if requested) if self.sumo_params.lateral_resolution is not None: sumo_call.append("--lateral-resolution") sumo_call.append(str(self.sumo_params.lateral_resolution)) # add the emission path to the sumo command (if requested) if self.sumo_params.emission_path is not None: ensure_dir(self.sumo_params.emission_path) emission_out = \ self.sumo_params.emission_path + \ "{0}-emission.xml".format(self.scenario.name) sumo_call.append("--emission-output") sumo_call.append(emission_out) else: emission_out = None if self.sumo_params.overtake_right: sumo_call.append("--lanechange.overtake-right") sumo_call.append("true") if self.sumo_params.ballistic: sumo_call.append("--step-method.ballistic") sumo_call.append("true") # specify a simulation seed (if requested) if self.sumo_params.seed is not None: sumo_call.append("--seed") sumo_call.append(str(self.sumo_params.seed)) if not self.sumo_params.print_warnings: sumo_call.append("--no-warnings") sumo_call.append("true") # set the time it takes for a gridlock teleport to occur sumo_call.append("--time-to-teleport") sumo_call.append(str(int(self.sumo_params.teleport_time))) logging.info(" Starting SUMO on port " + str(port)) logging.debug(" Cfg file: " + str(self.scenario.cfg)) logging.debug(" Emission file: " + str(emission_out)) logging.debug(" Step length: " + str(self.sim_step)) # Opening the I/O thread to SUMO self.sumo_proc = subprocess.Popen(sumo_call, preexec_fn=os.setsid) # wait a small period of time for the subprocess to activate # before trying to connect with traci if os.environ.get("TEST_FLAG", 0): time.sleep(0.1) else: time.sleep(config.SUMO_SLEEP) self.traci_connection = traci.connect(port, numRetries=100) self.traci_connection.simulationStep() return except Exception as e: print("Error during start: {}".format(traceback.format_exc())) error = e self.teardown_sumo() raise error
def __init__(self, name, vehicles, net_params, initial_config=InitialConfig(), traffic_lights=TrafficLights()): """Instantiate the base scenario class. Attributes ---------- name : str A tag associated with the scenario vehicles : Vehicles type see flow/core/vehicles.py net_params : NetParams type see flow/core/params.py initial_config : InitialConfig type see flow/core/params.py traffic_lights : flow.core.traffic_lights.TrafficLights type see flow/core/traffic_lights.py """ # Invoke serializable if using rllab if Serializable is not object: Serializable.quick_init(self, locals()) self.orig_name = name # To avoid repeated concatenation upon reset self.name = name + time.strftime("_%Y%m%d-%H%M%S") + str(time.time()) self.vehicles = vehicles self.net_params = net_params self.initial_config = initial_config self.traffic_lights = traffic_lights self.net_params = net_params self.net_path = os.path.dirname(os.path.abspath(__file__)) \ + "/debug/net/" self.cfg_path = os.path.dirname(os.path.abspath(__file__)) \ + "/debug/cfg/" self.vehicle_ids = [] ensure_dir("%s" % self.net_path) ensure_dir("%s" % self.cfg_path) self.nodfn = "%s.nod.xml" % self.name self.edgfn = "%s.edg.xml" % self.name self.typfn = "%s.typ.xml" % self.name self.cfgfn = "%s.netccfg" % self.name self.netfn = "%s.net.xml" % self.name self.confn = "%s.con.xml" % self.name self.roufn = "%s.rou.xml" % self.name self.addfn = "%s.add.xml" % self.name self.sumfn = "%s.sumo.cfg" % self.name self.guifn = "%s.gui.cfg" % self.name # create the network configuration files self._edges, self._connections = self.generate_net( self.net_params, self.traffic_lights) # list of edges and internal links (junctions) self._edge_list = [ edge_id for edge_id in self._edges.keys() if edge_id[0] != ":" ] self._junction_list = list( set(self._edges.keys()) - set(self._edge_list)) # maximum achievable speed on any edge in the network self.max_speed = max( self.speed_limit(edge) for edge in self.get_edge_list()) # parameters to be specified under each unique subclass's # __init__() function self.edgestarts = self.specify_edge_starts() # these optional parameters need only be used if "no-internal-links" # is set to "false" while calling sumo's netconvert function self.internal_edgestarts = self.specify_internal_edge_starts() self.intersection_edgestarts = self.specify_intersection_edge_starts() # in case the user did not write the intersection edge-starts in # internal edge-starts as well (because of redundancy), merge the two # together self.internal_edgestarts += self.intersection_edgestarts seen = set() self.internal_edgestarts = \ [item for item in self.internal_edgestarts if item[1] not in seen and not seen.add(item[1])] self.internal_edgestarts_dict = dict(self.internal_edgestarts) # total_edgestarts and total_edgestarts_dict contain all of the above # edges, with the former being ordered by position if self.net_params.no_internal_links: self.total_edgestarts = self.edgestarts else: self.total_edgestarts = self.edgestarts + self.internal_edgestarts self.total_edgestarts.sort(key=lambda tup: tup[1]) self.total_edgestarts_dict = dict(self.total_edgestarts) # length of the network, or the portion of the network in # which cars are meant to be distributed # (may be overridden by subclass __init__()) if not hasattr(self, "length"): self.length = sum([ self.edge_length(edge_id) for edge_id in self.get_edge_list() ]) # generate starting position for vehicles in the network kwargs = initial_config.additional_params positions, lanes = self.generate_starting_positions( num_vehicles=vehicles.num_vehicles, **kwargs) # create the sumo configuration files cfg_name = self.generate_cfg(self.net_params, self.traffic_lights) shuffle = initial_config.shuffle self.make_routes(self, positions, lanes, shuffle) # specify the location of the sumo configuration file self.cfg = self.cfg_path + cfg_name
elif flags.rl_trainer == "Stable-Baselines": flow_params = submodule.flow_params # Path to the saved files exp_tag = flow_params['exp_tag'] result_name = '{}/{}'.format(exp_tag, strftime("%Y-%m-%d-%H:%M:%S")) # Perform training. print('Beginning training.') model = run_model_stablebaseline(flow_params, flags.num_cpus, flags.rollout_size, flags.num_steps) # Save the model to a desired folder and then delete it to demonstrate # loading. print('Saving the trained model!111') path = os.path.realpath(os.path.expanduser('~/baseline_results')) print('pot 1') ensure_dir(os.path.join(path,'stabilizing_the_ring')) print('pot 2') save_path = os.path.join(path, result_name) print('pot 3') model.save(save_path) print('I am here to test, begin!') # dump the flow params with open(os.path.join(path, result_name) + '.json', 'w') as outfile: json.dump(flow_params, outfile, cls=FlowParamsEncoder, sort_keys=True, indent=4) print('I am here to test, over!') # Replay the result by loading the model print('Loading the trained model and testing it out!') model = PPO2.load(save_path)