def __init__(self, simulator, sim_params): """Instantiate a Flow kernel object. Parameters ---------- simulator : str simulator type, must be one of {"traci"} sim_params : flow.core.params.SimParams simulation-specific parameters Raises ------ flow.utils.exceptions.FatalFlowError if the specified input simulator is not a valid type """ self.kernel_api = None if simulator == "traci": self.simulation = TraCISimulation(self) self.network = TraCIKernelNetwork(self, sim_params) self.vehicle = TraCIVehicle(self, sim_params) self.traffic_light = TraCITrafficLight(self) elif simulator == 'aimsun': self.simulation = AimsunKernelSimulation(self) self.network = AimsunKernelNetwork(self, sim_params) self.vehicle = AimsunKernelVehicle(self, sim_params) self.traffic_light = AimsunKernelTrafficLight(self) else: raise FatalFlowError( 'Simulator type "{}" is not valid.'.format(simulator))
def __init__(self, simulator, sim_params, observation_list=None, monitor_rl=False): """Instantiate a Flow kernel object. Parameters ---------- simulator : str simulator type, must be one of {"traci"} sim_params : flow.core.params.SimParams simulation-specific parameters observation_list : list optional arguments to be specified when certain observations wish to be monitored monitor_rl : bool Enable/Disable subscribing to RL vehicles only Raises ------ flow.utils.exceptions.FatalFlowError if the specified input simulator is not a valid type """ self.kernel_api = None if simulator == "traci": self.simulation = TraCISimulation(self) self.scenario = TraCIScenario(self, sim_params) if observation_list: self.vehicle = TraCIVehicle(self, sim_params=sim_params, observation_list=observation_list, monitor_rl=monitor_rl) else: self.vehicle = TraCIVehicle(self, sim_params=sim_params, monitor_rl=monitor_rl) self.traffic_light = TraCITrafficLight(self) elif simulator == 'aimsun': self.simulation = AimsunKernelSimulation(self) self.scenario = AimsunKernelScenario(self, sim_params) self.vehicle = AimsunKernelVehicle(self, sim_params) self.traffic_light = AimsunKernelTrafficLight(self) else: raise FatalFlowError( 'Simulator type "{}" is not valid.'.format(simulator))
def generate_starting_positions(self, initial_config, num_vehicles=None): """Generate starting positions for vehicles in the network. Calls all other starting position generating classes. Parameters ---------- initial_config : flow.core.params.InitialConfig see flow/core/params.py num_vehicles : int, optional number of vehicles to be placed on the network. If no value is specified, the value is collected from the vehicles class Returns ------- list of tuple (float, float) list of start positions [(edge0, pos0), (edge1, pos1), ...] list of int list of start lanes Raises ------ flow.utils.exceptions.FatalFlowError if the spacing mode is not {'uniform', 'random', 'custom'} """ num_vehicles = num_vehicles or self.network.vehicles.num_vehicles if initial_config.spacing == 'uniform': startpositions, startlanes = self.gen_even_start_pos( initial_config, num_vehicles) elif initial_config.spacing == 'random': startpositions, startlanes = self.gen_random_start_pos( initial_config, num_vehicles) elif initial_config.spacing == 'custom': startpositions, startlanes = self.gen_custom_start_pos( initial_config, num_vehicles) else: raise FatalFlowError('"spacing" argument in initial_config does ' 'not contain a valid option') return startpositions, startlanes
def reset(self): """Reset the environment. This method is performed in between rollouts. It resets the state of the environment, and re-initializes the vehicles in their starting positions. If "shuffle" is set to True in InitialConfig, the initial positions of vehicles is recalculated and the vehicles are shuffled. Returns ------- observation : array_like the initial observation of the space. The initial reward is assumed to be zero. """ # reset the time counter self.time_counter = 0 # warn about not using restart_instance when using inflows if len(self.net_params.inflows.get()) > 0 and \ not self.sim_params.restart_instance: print( "**********************************************************\n" "**********************************************************\n" "**********************************************************\n" "WARNING: Inflows will cause computational performance to\n" "significantly decrease after large number of rollouts. In \n" "order to avoid this, set SumoParams(restart_instance=True).\n" "**********************************************************\n" "**********************************************************\n" "**********************************************************") if self.sim_params.restart_instance or \ (self.step_counter > 2e6 and self.simulator != 'aimsun'): self.step_counter = 0 # issue a random seed to induce randomness into the next rollout self.sim_params.seed = random.randint(0, 1e5) self.k.vehicle = deepcopy(self.initial_vehicles) self.k.vehicle.master_kernel = self.k # restart the sumo instance self.restart_simulation(self.sim_params) # perform shuffling (if requested) elif self.initial_config.shuffle: self.setup_initial_state() # clear all vehicles from the network and the vehicles class if self.simulator == 'traci': for veh_id in self.k.kernel_api.vehicle.getIDList(): # FIXME: hack try: self.k.vehicle.remove(veh_id) except (FatalTraCIError, TraCIException): print(traceback.format_exc()) # clear all vehicles from the network and the vehicles class # FIXME (ev, ak) this is weird and shouldn't be necessary for veh_id in list(self.k.vehicle.get_ids()): # do not try to remove the vehicles from the network in the first # step after initializing the network, as there will be no vehicles if self.step_counter == 0: continue try: self.k.vehicle.remove(veh_id) except (FatalTraCIError, TraCIException): print("Error during start: {}".format(traceback.format_exc())) # reintroduce the initial vehicles to the network for veh_id in self.initial_ids: type_id, edge, lane_index, pos, speed = \ self.initial_state[veh_id] try: self.k.vehicle.add(veh_id=veh_id, type_id=type_id, edge=edge, lane=lane_index, pos=pos, speed=speed) except (FatalTraCIError, TraCIException): # if a vehicle was not removed in the first attempt, remove it # now and then reintroduce it self.k.vehicle.remove(veh_id) if self.simulator == 'traci': self.k.kernel_api.vehicle.remove(veh_id) # FIXME: hack self.k.vehicle.add(veh_id=veh_id, type_id=type_id, edge=edge, lane=lane_index, pos=pos, speed=speed) # advance the simulation in the simulator by one step self.k.simulation.simulation_step() # update the information in each kernel to match the current state self.k.update(reset=True) # update the colors of vehicles if self.sim_params.render: self.k.vehicle.update_vehicle_colors() if self.simulator == 'traci': initial_ids = self.k.kernel_api.vehicle.getIDList() else: initial_ids = self.initial_ids # check to make sure all vehicles have been spawned if len(self.initial_ids) > len(initial_ids): missing_vehicles = list(set(self.initial_ids) - set(initial_ids)) msg = '\nNot enough vehicles have spawned! Bad start?\n' \ 'Missing vehicles / initial state:\n' for veh_id in missing_vehicles: msg += '- {}: {}\n'.format(veh_id, self.initial_state[veh_id]) raise FatalFlowError(msg=msg) states = self.get_state() # collect information of the state of the network based on the # environment class used self.state = np.asarray(states).T # observation associated with the reset (no warm-up steps) observation = np.copy(states) # perform (optional) warm-up steps before training for _ in range(self.env_params.warmup_steps): observation, _, _, _ = self.step(rl_actions=None) # render a frame self.render(reset=True) return observation
def __init__(self, env_params, sim_params, scenario, simulator='traci'): """Initialize the environment class. Parameters ---------- env_params : flow.core.params.EnvParams see flow/core/params.py sim_params : flow.core.params.SimParams see flow/core/params.py scenario : flow.scenarios.Scenario see flow/scenarios/base_scenario.py simulator : str the simulator used, one of {'traci', 'aimsun'}. Defaults to 'traci' Raises ------ flow.utils.exceptions.FatalFlowError if the render mode is not set to a valid value """ # Invoke serializable if using rllab if serializable_flag: Serializable.quick_init(self, locals()) self.env_params = env_params self.scenario = scenario self.net_params = scenario.net_params self.initial_config = scenario.initial_config self.sim_params = sim_params time_stamp = ''.join(str(time.time()).split('.')) if os.environ.get("TEST_FLAG", 0): # 1.0 works with stress_test_start 10k times time.sleep(1.0 * int(time_stamp[-6:]) / 1e6) # FIXME: this is sumo-specific self.sim_params.port = sumolib.miscutils.getFreeSocketPort() # time_counter: number of steps taken since the start of a rollout self.time_counter = 0 # step_counter: number of total steps taken self.step_counter = 0 # initial_state: self.initial_state = {} self.state = None self.obs_var_labels = [] # simulation step size self.sim_step = sim_params.sim_step # the simulator used by this environment self.simulator = simulator # create the Flow kernel self.k = Kernel(simulator=self.simulator, sim_params=sim_params) # use the scenario class's network parameters to generate the necessary # scenario components within the scenario kernel self.k.scenario.generate_network(scenario) # initial the vehicles kernel using the VehicleParams object self.k.vehicle.initialize(deepcopy(scenario.vehicles)) # initialize the simulation using the simulation kernel. This will use # the scenario kernel as an input in order to determine what network # needs to be simulated. kernel_api = self.k.simulation.start_simulation( scenario=self.k.scenario, sim_params=sim_params) # pass the kernel api to the kernel and it's subclasses self.k.pass_api(kernel_api) # the available_routes variable contains a dictionary of routes # vehicles can traverse; to be used when routes need to be chosen # dynamically self.available_routes = self.k.scenario.rts # store the initial vehicle ids self.initial_ids = deepcopy(scenario.vehicles.ids) # store the initial state of the vehicles kernel (needed for restarting # the simulation) self.k.vehicle.kernel_api = None self.k.vehicle.master_kernel = None self.initial_vehicles = deepcopy(self.k.vehicle) self.k.vehicle.kernel_api = self.k.kernel_api self.k.vehicle.master_kernel = self.k self.setup_initial_state() # use pyglet to render the simulation if self.sim_params.render in ['gray', 'dgray', 'rgb', 'drgb']: save_render = self.sim_params.save_render sight_radius = self.sim_params.sight_radius pxpm = self.sim_params.pxpm show_radius = self.sim_params.show_radius # get network polygons network = [] # FIXME: add to scenario kernel instead of hack for lane_id in self.k.kernel_api.lane.getIDList(): _lane_poly = self.k.kernel_api.lane.getShape(lane_id) lane_poly = [i for pt in _lane_poly for i in pt] network.append(lane_poly) # instantiate a pyglet renderer self.renderer = Renderer(network, self.sim_params.render, save_render, sight_radius=sight_radius, pxpm=pxpm, show_radius=show_radius) # render a frame self.render(reset=True) elif self.sim_params.render in [True, False]: pass # default to sumo-gui (if True) or sumo (if False) else: raise FatalFlowError('Mode %s is not supported!' % self.sim_params.render) atexit.register(self.terminate)
def reset(self): """Reset the environment. This method is performed in between rollouts. It resets the state of the environment, and re-initializes the vehicles in their starting positions. If "shuffle" is set to True in InitialConfig, the initial positions of vehicles is recalculated and the vehicles are shuffled. Returns ------- observation : array_like the initial observation of the space. The initial reward is assumed to be zero. """ # FIXED SEEDS: always setting same random seeds for each run and see whether the agent learns quicker # Random seeds recording and optionally loading use_seeds = self.env_params.additional_params["use_seeds"] if use_seeds: if use_seeds == "per_process": # FIXED SEEDS: always same random seeds for each rollout worker run and see whether the agent learns quicker if self.process_seeds_file: # i.e. iteration > 1 -- keep loading the file from iteration 1 for each simulation with open(self.process_seeds_file, 'rb') as handle: print("loading seeds file " + self.process_seeds_file) loaded_seeds = pickle.load(handle) random.setstate(loaded_seeds['old_state_random']) np.random.set_state(loaded_seeds['old_state_np']) else: # first iteration, create file to be used by all runs of this worker logs_path = os.path.expanduser( "~/flow_seeds/") + "flow_" + str( datetime.datetime.now()).replace(' ', '_').replace( '-', '_').replace(':', '_') if not os.path.exists(logs_path): os.makedirs(logs_path) seeds = { 'old_state_random': random.getstate(), 'old_state_np': np.random.get_state() } self.process_seeds_file = logs_path + "/seeds.pkl" with open(self.process_seeds_file, 'wb') as handle: pickle.dump(seeds, handle) else: # FIXED SEEDS: same seed for *all* rollout workers with open(use_seeds, 'rb') as handle: loaded_seeds = pickle.load(handle) random.setstate(loaded_seeds['old_state_random']) np.random.set_state(loaded_seeds['old_state_np']) print("loaded seeds file " + use_seeds) # regardless of the above, always save seeds to file seeds = { 'old_state_random': random.getstate(), 'old_state_np': np.random.get_state() } # creating path to experiments' log files, using current time logs_path = os.path.expanduser("~/flow_seeds/") + "flow_" + str( datetime.datetime.now()).replace(' ', '_').replace( '-', '_').replace(':', '_') if not os.path.exists(logs_path): os.makedirs(logs_path) ## send logs_path to sumo to write logs (TODO: but after constructor, a bit hacky) #sim_params.logs_path=logs_path with open(logs_path + "/seeds.pkl", 'wb') as handle: pickle.dump(seeds, handle) #with open('/home/dzgnkq/flow_2020_02_16_16_15_42.001297/seeds.pkl', 'rb') as handle: #with open('/home/dzgnkq/flow_seeds/flow_2020_02_27_15_14_40.694017/seeds.pkl', 'rb') as handle: #with open(os.environ["SEEDSFILE"], 'rb') as handle: # print() # print("loading seeds file " + os.environ["SEEDSFILE"]) # print() # loaded_seeds = pickle.load(handle) # random.setstate(loaded_seeds['old_state_random']) # np.random.set_state(loaded_seeds['old_state_np']) ###################################################################################################### # reset the time counter self.time_counter = 0 # warn about not using restart_instance when using inflows if len(self.net_params.inflows.get()) > 0 and \ not self.sim_params.restart_instance: print( "**********************************************************\n" "**********************************************************\n" "**********************************************************\n" "WARNING: Inflows will cause computational performance to\n" "significantly decrease after large number of rollouts. In \n" "order to avoid this, set SumoParams(restart_instance=True).\n" "**********************************************************\n" "**********************************************************\n" "**********************************************************") if self.sim_params.restart_instance or \ (self.step_counter > 2e6 and self.simulator != 'aimsun'): self.step_counter = 0 # issue a random seed to induce randomness into the next rollout self.sim_params.seed = random.randint(0, 1e5) self.k.vehicle = deepcopy(self.initial_vehicles) self.k.vehicle.master_kernel = self.k # restart the sumo instance self.restart_simulation(self.sim_params) # perform shuffling (if requested) elif self.initial_config.shuffle: self.setup_initial_state() # clear all vehicles from the network and the vehicles class if self.simulator == 'traci': for veh_id in self.k.kernel_api.vehicle.getIDList(): # FIXME: hack try: self.k.vehicle.remove(veh_id) except (FatalTraCIError, TraCIException): print(traceback.format_exc()) # clear all vehicles from the network and the vehicles class # FIXME (ev, ak) this is weird and shouldn't be necessary for veh_id in list(self.k.vehicle.get_ids()): # do not try to remove the vehicles from the network in the first # step after initializing the network, as there will be no vehicles if self.step_counter == 0: continue try: self.k.vehicle.remove(veh_id) except (FatalTraCIError, TraCIException): print("Error during start: {}".format(traceback.format_exc())) # reintroduce the initial vehicles to the network for veh_id in self.initial_ids: type_id, edge, lane_index, pos, speed = \ self.initial_state[veh_id] try: self.k.vehicle.add(veh_id=veh_id, type_id=type_id, edge=edge, lane=lane_index, pos=pos, speed=speed) except (FatalTraCIError, TraCIException): # if a vehicle was not removed in the first attempt, remove it # now and then reintroduce it self.k.vehicle.remove(veh_id) if self.simulator == 'traci': self.k.kernel_api.vehicle.remove(veh_id) # FIXME: hack self.k.vehicle.add(veh_id=veh_id, type_id=type_id, edge=edge, lane=lane_index, pos=pos, speed=speed) # advance the simulation in the simulator by one step self.k.simulation.simulation_step() # update the information in each kernel to match the current state self.k.update(reset=True) # update the colors of vehicles if self.sim_params.render: self.k.vehicle.update_vehicle_colors() if self.simulator == 'traci': initial_ids = self.k.kernel_api.vehicle.getIDList() else: initial_ids = self.initial_ids # check to make sure all vehicles have been spawned if len(self.initial_ids) > len(initial_ids): missing_vehicles = list(set(self.initial_ids) - set(initial_ids)) msg = '\nNot enough vehicles have spawned! Bad start?\n' \ 'Missing vehicles / initial state:\n' for veh_id in missing_vehicles: msg += '- {}: {}\n'.format(veh_id, self.initial_state[veh_id]) raise FatalFlowError(msg=msg) states = self.get_state() # collect information of the state of the network based on the # environment class used self.state = np.asarray(states).T # observation associated with the reset (no warm-up steps) observation = np.copy(states) # perform (optional) warm-up steps before training for _ in range(self.env_params.warmup_steps): observation, _, _, _ = self.step(rl_actions=None) # render a frame self.render(reset=True) return observation
def _get_start_pos_util(self, initial_config, num_vehicles): """Prepare initial_config data for starting position methods. Performs some pre-processing to the initial_config and **kwargs terms, and returns the necessary values for all starting position generating functions. Parameters ---------- initial_config : InitialConfig type see flow/core/params.py num_vehicles : int number of vehicles to be placed on the network Returns ------- x0 : float starting position of the first vehicle, in meters min_gap : float minimum gap between vehicles bunching : float the amount of space freed up in the network (per lane) lanes_distribution : int number of lanes the vehicles are supposed to be distributed over available_length : float total available free space for vehicle to be placed, over all lanes within the distributable lanes, in meters initial_config : InitialConfig type modified version of the initial_config parameter Raises ------ flow.utils.exceptions.FatalFlowError If there is not enough space to place all vehicles in the allocated space in the network with the specified minimum gap. """ min_gap = max(0, initial_config.min_gap) bunching = initial_config.bunching # check if requested bunching value is not valid (negative) if initial_config.bunching < 0: logging.warning('"bunching" cannot be negative; setting to 0') initial_config.bunching = 0 # compute the lanes distribution (adjust of edge cases) if initial_config.edges_distribution == 'all': max_lane = max( [self.num_lanes(edge_id) for edge_id in self.get_edge_list()]) else: max_lane = max([ self.num_lanes(edge_id) for edge_id in initial_config.edges_distribution ]) if initial_config.lanes_distribution > max_lane: lanes_distribution = max_lane elif initial_config.lanes_distribution < 1: logging.warning('"lanes_distribution" is too small; setting to 1') lanes_distribution = 1 else: lanes_distribution = initial_config.lanes_distribution if initial_config.edges_distribution == 'all': distribution_length = \ sum(self.edge_length(edge_id) * min([self.num_lanes(edge_id), lanes_distribution]) for edge_id in self.get_edge_list() if self.edge_length(edge_id) > min_gap + VEHICLE_LENGTH) else: distribution_length = \ sum(self.edge_length(edge_id) * min(self.num_lanes(edge_id), lanes_distribution) for edge_id in initial_config.edges_distribution if self.edge_length(edge_id) > min_gap + VEHICLE_LENGTH) if initial_config.edges_distribution == 'all': available_edges = [ edge for edge in self.get_edge_list() if self.edge_length(edge) > min_gap + VEHICLE_LENGTH] else: available_edges = [ edge for edge in initial_config.edges_distribution if self.edge_length(edge) > min_gap + VEHICLE_LENGTH] available_length = \ distribution_length - lanes_distribution * bunching - \ num_vehicles * (min_gap + VEHICLE_LENGTH) if available_length < 0: raise FatalFlowError('There is not enough space to place all ' 'vehicles in the network.') return (initial_config.x0, min_gap, bunching, lanes_distribution, available_length, available_edges, initial_config)
def evaluate_policy(benchmark, _get_actions, _get_states=None): """Evaluate the performance of a controller on a predefined benchmark. Parameters ---------- benchmark : str name of the benchmark, must be printed as it is in the benchmarks folder; otherwise a FatalFlowError will be raised _get_actions : method the mapping from states to actions for the RL agent(s) _get_states : method, optional a mapping from the environment object in Flow to some state, which overrides the _get_states method of the environment. Note that the same cannot be done for the actions. Returns ------- float mean of the evaluation return of the benchmark from NUM_RUNS number of simulations float standard deviation of the evaluation return of the benchmark from NUM_RUNS number of simulations Raises ------ flow.utils.exceptions.FatalFlowError If the specified benchmark is not available. """ if benchmark not in AVAILABLE_BENCHMARKS.keys(): raise FatalFlowError( "benchmark {} is not available. Check spelling?".format(benchmark)) # get the flow params from the benchmark flow_params = AVAILABLE_BENCHMARKS[benchmark] exp_tag = flow_params["exp_tag"] sim_params = flow_params["sim"] vehicles = flow_params["veh"] env_params = flow_params["env"] env_params.evaluate = True # Set to true to get evaluation returns net_params = flow_params["net"] initial_config = flow_params.get("initial", InitialConfig()) traffic_lights = flow_params.get("tls", TrafficLightParams()) # import the environment and scenario classes module = __import__("flow.envs", fromlist=[flow_params["env_name"]]) env_class = getattr(module, flow_params["env_name"]) module = __import__("flow.scenarios", fromlist=[flow_params["scenario"]]) scenario_class = getattr(module, flow_params["scenario"]) # recreate the scenario and environment scenario = scenario_class(name=exp_tag, vehicles=vehicles, net_params=net_params, initial_config=initial_config, traffic_lights=traffic_lights) # make sure the _get_states method of the environment is the one # specified by the user if _get_states is not None: class _env_class(env_class): def get_state(self): return _get_states(self) env_class = _env_class env = env_class(env_params=env_params, sim_params=sim_params, scenario=scenario) # create a Experiment object with the "rl_actions" method as # described in the inputs. Note that the state may not be that which is # specified by the environment. exp = Experiment(env=env) # run the experiment and return the reward res = exp.run(num_runs=NUM_RUNS, num_steps=env.env_params.horizon, rl_actions=_get_actions) return np.mean(res["returns"]), np.std(res["returns"])
def evaluate_policy(benchmark, _get_actions, _get_states=None): """Evaluate the performance of a controller on a predefined benchmark. Parameters ---------- benchmark : str name of the benchmark, must be printed as it is in the benchmarks folder; otherwise a FatalFlowError will be raised _get_actions : method the mapping from states to actions for the RL agent(s) _get_states : method, optional a mapping from the environment object in Flow to some state, which overrides the _get_states method of the environment. Note that the same cannot be done for the actions. Returns ------- float mean of the evaluation return of the benchmark from NUM_RUNS number of simulations float standard deviation of the evaluation return of the benchmark from NUM_RUNS number of simulations Raises ------ flow.utils.exceptions.FatalFlowError If the specified benchmark is not available. """ if benchmark not in AVAILABLE_BENCHMARKS.keys(): raise FatalFlowError( "benchmark {} is not available. Check spelling?".format(benchmark)) # get the flow params from the benchmark flow_params = AVAILABLE_BENCHMARKS[benchmark] exp_tag = flow_params["exp_tag"] sim_params = flow_params["sim"] vehicles = flow_params["veh"] env_params = flow_params["env"] env_params.evaluate = True # Set to true to get evaluation returns net_params = flow_params["net"] initial_config = flow_params.get("initial", InitialConfig()) traffic_lights = flow_params.get("tls", TrafficLightParams()) # import the environment and network classes module = __import__("flow.envs", fromlist=[flow_params["env_name"]]) env_class = getattr(module, flow_params["env_name"]) module = __import__("flow.networks", fromlist=[flow_params["network"]]) network_class = getattr(module, flow_params["network"]) # recreate the network and environment network = network_class( name=exp_tag, vehicles=vehicles, net_params=net_params, initial_config=initial_config, traffic_lights=traffic_lights) # make sure the _get_states method of the environment is the one # specified by the user if _get_states is not None: class _env_class(env_class): def get_state(self): return _get_states(self) env_class = _env_class env = env_class( env_params=env_params, sim_params=sim_params, network=network) flow_params = dict( # name of the experiment exp_tag=exp_tag, # name of the flow environment the experiment is running on env_name=env_class, # name of the network class the experiment is running on network=network_class, # simulator that is used by the experiment simulator='traci', # sumo-related parameters (see flow.core.params.SumoParams) sim=sim_params, # environment related parameters (see flow.core.params.EnvParams) env=env_params, # network-related parameters (see flow.core.params.NetParams and the # network's documentation or ADDITIONAL_NET_PARAMS component) net=net_params, # vehicles to be placed in the network at the start of a rollout (see # flow.core.params.VehicleParams) veh=vehicles, # parameters specifying the positioning of vehicles upon initialization/ # reset (see flow.core.params.InitialConfig) initial=initial_config, # traffic lights to be introduced to specific nodes (see # flow.core.params.TrafficLightParams) tls=traffic_lights, ) # number of time steps flow_params['env'].horizon = env.env_params.horizon # create a Experiment object. Note that the state may not be that which is # specified by the environment. exp = Experiment(flow_params) exp.env = env exp = Experiment(flow_params) exp.env = env # run the experiment and return the reward res = exp.run( num_runs=NUM_RUNS, rl_actions=_get_actions) return np.mean(res["returns"]), np.std(res["returns"])
def __init__(self, env_params, sim_params, network=None, simulator='traci', scenario=None): """Initialize the environment class. Parameters ---------- env_params : flow.core.params.EnvParams see flow/core/params.py sim_params : flow.core.params.SimParams see flow/core/params.py network : flow.networks.Network see flow/networks/base.py simulator : str the simulator used, one of {'traci', 'aimsun'}. Defaults to 'traci' Raises ------ flow.utils.exceptions.FatalFlowError if the render mode is not set to a valid valuefor _ in range(self.env_params.sims_per_step): self.time_counter += 1 self.step_counter += 1 # perform acceleration actions for controlled human-driven vehicles if len(self.k.vehicle.get_controlled_ids()) > 0: accel = [] for veh_id in self.k.vehicle.get_controlled_ids(): action = self.k.vehicle.get_acc_controller( veh_id).get_action(self) accel.append(action) if self.k.vehicle.get_edge(veh_id)[0] == ":": if self.k.vehicle.get_speed(veh_id) <= 0.00000001: print(self.time_counter,'veh_id:',veh_id,'its leader:',self.k.vehicle.get_leader(veh_id),'headway to leader:',self.k.vehicle.get_headway(veh_id),'action:',action,'speed:',self.k.vehicle.get_speed(veh_id)) #if self.k.vehicle.get_leader(self.k.vehicle.get_leader(veh_id))==veh_id: # break self.k.vehicle.apply_acceleration( self.k.vehicle.get_controlled_ids(), accel) # perform lane change actions for controlled human-driven vehicles if len(self.k.vehicle.get_controlled_lc_ids()) > 0: direction = [] for veh_id in self.k.vehicle.get_controlled_lc_ids(): target_lane = self.k.vehicle.get_lane_changing_controller( veh_id).get_action(self) direction.append(target_lane) self.k.vehicle.apply_lane_change( self.k.vehicle.get_controlled_lc_ids(), direction=direction) # perform (optionally) routing actions for all vehicles in the # network, including RL and SUMO-controlled vehicles routing_ids = [] routing_actions = [] for veh_id in self.k.vehicle.get_ids(): if self.k.vehicle.get_routing_controller(veh_id) \ is not None: routing_ids.append(veh_id) route_contr = self.k.vehicle.get_routing_controller( veh_id) routing_actions.append(route_contr.choose_route(self)) self.k.vehicle.choose_routes(routing_ids, routing_actions) self.apply_rl_actions(rl_actions) #self.additional_command() # advance the simulation in the simulator by one step """ self.process_seeds_file = None self.env_params = env_params self.time_with_no_vehicles = 0 if scenario is not None: deprecated_attribute(self, "scenario", "network") self.network = scenario if scenario is not None else network self.net_params = self.network.net_params self.initial_config = self.network.initial_config self.sim_params = sim_params time_stamp = ''.join(str(time.time()).split('.')) if os.environ.get("TEST_FLAG", 0): # 1.0 works with stress_test_start 10k times time.sleep(1.0 * int(time_stamp[-6:]) / 1e6) # FIXME: this is sumo-specific self.sim_params.port = sumolib.miscutils.getFreeSocketPort() # time_counter: number of steps taken since the start of a rollout self.time_counter = 0 # step_counter: number of total steps taken self.step_counter = 0 # initial_state: self.initial_state = {} self.state = None self.obs_var_labels = [] # simulation step size self.sim_step = sim_params.sim_step # the simulator used by this environment self.simulator = simulator # create the Flow kernel self.k = Kernel(simulator=self.simulator, sim_params=sim_params) # use the network class's network parameters to generate the necessary # network components within the network kernel self.k.network.generate_network(self.network) # initial the vehicles kernel using the VehicleParams object self.k.vehicle.initialize(deepcopy(self.network.vehicles)) # initialize the simulation using the simulation kernel. This will use # the network kernel as an input in order to determine what network # needs to be simulated. kernel_api = self.k.simulation.start_simulation(network=self.k.network, sim_params=sim_params) # pass the kernel api to the kernel and it's subclasses self.k.pass_api(kernel_api) # the available_routes variable contains a dictionary of routes # vehicles can traverse; to be used when routes need to be chosen # dynamically self.available_routes = self.k.network.rts # store the initial vehicle ids self.initial_ids = deepcopy(self.network.vehicles.ids) # store the initial state of the vehicles kernel (needed for restarting # the simulation) self.k.vehicle.kernel_api = None self.k.vehicle.master_kernel = None self.initial_vehicles = deepcopy(self.k.vehicle) self.k.vehicle.kernel_api = self.k.kernel_api self.k.vehicle.master_kernel = self.k self.setup_initial_state() # use pyglet to render the simulation if self.sim_params.render in ['gray', 'dgray', 'rgb', 'drgb']: save_render = self.sim_params.save_render sight_radius = self.sim_params.sight_radius pxpm = self.sim_params.pxpm show_radius = self.sim_params.show_radius # get network polygons network = [] # FIXME: add to network kernel instead of hack for lane_id in self.k.kernel_api.lane.getIDList(): _lane_poly = self.k.kernel_api.lane.getShape(lane_id) lane_poly = [i for pt in _lane_poly for i in pt] network.append(lane_poly) # instantiate a pyglet renderer self.renderer = Renderer(network, self.sim_params.render, save_render, sight_radius=sight_radius, pxpm=pxpm, show_radius=show_radius) # render a frame self.render(reset=True) elif self.sim_params.render in [True, False]: pass # default to sumo-gui (if True) or sumo (if False) else: raise FatalFlowError('Mode %s is not supported!' % self.sim_params.render) atexit.register(self.terminate)