class LeaderboardEvaluator(object): """ TODO: document me! """ ego_vehicles = [] # Tunable parameters client_timeout = 10.0 # in seconds wait_for_world = 20.0 # in seconds frame_rate = 20.0 # in Hz def __init__(self, args, statistics_manager): """ Setup CARLA client and world Setup ScenarioManager """ self.statistics_manager = statistics_manager self.sensors = None self.sensor_icons = [] self._vehicle_lights = carla.VehicleLightState.Position | carla.VehicleLightState.LowBeam # First of all, we need to create the client that will send the requests # to the simulator. Here we'll assume the simulator is accepting # requests in the localhost at port 2000. self.client = carla.Client(args.host, args.port) if args.timeout: self.client_timeout = args.timeout self.client.set_timeout(self.client_timeout) self.traffic_manager = self.client.get_trafficmanager(args.traffic_manager_port) dist = pkg_resources.get_distribution("carla") if dist.version != 'leaderboard': if LooseVersion(dist.version) < LooseVersion('0.9.10'): raise ImportError("CARLA version 0.9.10.1 or newer required. CARLA version found: {}".format(dist)) # Load agent module_name = os.path.basename(args.agent).split('.')[0] sys.path.insert(0, os.path.dirname(args.agent)) self.module_agent = importlib.import_module(module_name) # Create the ScenarioManager self.manager = ScenarioManager(args.timeout, args.debug > 1) # Time control for summary purposes self._start_time = GameTime.get_time() self._end_time = None # Create the agent timer self._agent_watchdog = Watchdog(args.timeout) signal.signal(signal.SIGINT, self._signal_handler) def _signal_handler(self, signum, frame): """ Terminate scenario ticking when receiving a signal interrupt """ if self._agent_watchdog and not self._agent_watchdog.get_status(): raise RuntimeError("Timeout: Agent took longer than {}s to setup".format(self.client_timeout)) elif self.manager: self.manager.signal_handler(signum, frame) def __del__(self): """ Cleanup and delete actors, ScenarioManager and CARLA world """ self._cleanup() if hasattr(self, 'manager') and self.manager: del self.manager if hasattr(self, 'world') and self.world: del self.world def _cleanup(self): """ Remove and destroy all actors """ # Simulation still running and in synchronous mode? if self.manager and self.manager.get_running_status() \ and hasattr(self, 'world') and self.world: # Reset to asynchronous mode settings = self.world.get_settings() settings.synchronous_mode = False settings.fixed_delta_seconds = None self.world.apply_settings(settings) self.traffic_manager.set_synchronous_mode(False) if self.manager: self.manager.cleanup() CarlaDataProvider.cleanup() for i, _ in enumerate(self.ego_vehicles): if self.ego_vehicles[i]: self.ego_vehicles[i].destroy() self.ego_vehicles[i] = None self.ego_vehicles = [] if self._agent_watchdog: self._agent_watchdog.stop() if hasattr(self, 'agent_instance') and self.agent_instance: self.agent_instance.destroy() self.agent_instance = None if hasattr(self, 'statistics_manager') and self.statistics_manager: self.statistics_manager.scenario = None def _prepare_ego_vehicles(self, ego_vehicles, wait_for_ego_vehicles=False): """ Spawn or update the ego vehicles """ if not wait_for_ego_vehicles: for vehicle in ego_vehicles: self.ego_vehicles.append(CarlaDataProvider.request_new_actor(vehicle.model, vehicle.transform, vehicle.rolename, color=vehicle.color, vehicle_category=vehicle.category)) else: ego_vehicle_missing = True while ego_vehicle_missing: self.ego_vehicles = [] ego_vehicle_missing = False for ego_vehicle in ego_vehicles: ego_vehicle_found = False carla_vehicles = CarlaDataProvider.get_world().get_actors().filter('vehicle.*') for carla_vehicle in carla_vehicles: if carla_vehicle.attributes['role_name'] == ego_vehicle.rolename: ego_vehicle_found = True self.ego_vehicles.append(carla_vehicle) break if not ego_vehicle_found: ego_vehicle_missing = True break for i, _ in enumerate(self.ego_vehicles): self.ego_vehicles[i].set_transform(ego_vehicles[i].transform) # sync state CarlaDataProvider.get_world().tick() def _load_and_wait_for_world(self, args, town, ego_vehicles=None): """ Load a new CARLA world and provide data to CarlaDataProvider """ self.world = self.client.load_world(town) settings = self.world.get_settings() settings.fixed_delta_seconds = 1.0 / self.frame_rate settings.synchronous_mode = True self.world.apply_settings(settings) self.world.reset_all_traffic_lights() CarlaDataProvider.set_client(self.client) CarlaDataProvider.set_world(self.world) CarlaDataProvider.set_traffic_manager_port(args.traffic_manager_port) self.traffic_manager.set_synchronous_mode(True) self.traffic_manager.set_random_device_seed(args.traffic_manager_seed) self.traffic_manager.set_hybrid_physics_mode(True) # Wait for the world to be ready if CarlaDataProvider.is_sync_mode(): self.world.tick() else: self.world.wait_for_tick() if CarlaDataProvider.get_map().name != town: raise Exception("The CARLA server uses the wrong map!" "This scenario requires to use map {}".format(town)) def _register_statistics(self, config, checkpoint, entry_status, crash_message=""): """ Computes and saved the simulation statistics """ # register statistics current_stats_record = self.statistics_manager.compute_route_statistics( config, self.manager.scenario_duration_system, self.manager.scenario_duration_game, crash_message ) print("\033[1m> Registering the route statistics\033[0m") self.statistics_manager.save_record(current_stats_record, config.index, checkpoint) self.statistics_manager.save_entry_status(entry_status, False, checkpoint) def _load_and_run_scenario(self, args, config): """ Load and run the scenario given by config. Depending on what code fails, the simulation will either stop the route and continue from the next one, or report a crash and stop. """ crash_message = "" entry_status = "Started" print("\n\033[1m========= Preparing {} (repetition {}) =========".format(config.name, config.repetition_index)) print("> Setting up the agent\033[0m") # Prepare the statistics of the route self.statistics_manager.set_route(config.name, config.index) # Set up the user's agent, and the timer to avoid freezing the simulation try: self._agent_watchdog.start() agent_class_name = getattr(self.module_agent, 'get_entry_point')() self.agent_instance = getattr(self.module_agent, agent_class_name)(args.agent_config) config.agent = self.agent_instance # Check and store the sensors if not self.sensors: self.sensors = self.agent_instance.sensors() track = self.agent_instance.track AgentWrapper.validate_sensor_configuration(self.sensors, track, args.track) self.sensor_icons = [sensors_to_icons[sensor['type']] for sensor in self.sensors] self.statistics_manager.save_sensors(self.sensor_icons, args.checkpoint) self._agent_watchdog.stop() except SensorConfigurationInvalid as e: # The sensors are invalid -> set the ejecution to rejected and stop print("\n\033[91mThe sensor's configuration used is invalid:") print("> {}\033[0m\n".format(e)) traceback.print_exc() crash_message = "Agent's sensors were invalid" entry_status = "Rejected" self._register_statistics(config, args.checkpoint, entry_status, crash_message) self._cleanup() sys.exit(-1) except Exception as e: # The agent setup has failed -> start the next route print("\n\033[91mCould not set up the required agent:") print("> {}\033[0m\n".format(e)) traceback.print_exc() crash_message = "Agent couldn't be set up" self._register_statistics(config, args.checkpoint, entry_status, crash_message) self._cleanup() return print("\033[1m> Loading the world\033[0m") # Load the world and the scenario try: self._load_and_wait_for_world(args, config.town, config.ego_vehicles) self._prepare_ego_vehicles(config.ego_vehicles, False) scenario = RouteScenario(world=self.world, config=config, debug_mode=args.debug) self.statistics_manager.set_scenario(scenario.scenario) # Night mode if config.weather.sun_altitude_angle < 0.0: for vehicle in scenario.ego_vehicles: vehicle.set_light_state(carla.VehicleLightState(self._vehicle_lights)) # Load scenario and run it if args.record: self.client.start_recorder("{}/{}_rep{}.log".format(args.record, config.name, config.repetition_index)) self.manager.load_scenario(scenario, self.agent_instance, config.repetition_index) except Exception as e: # The scenario is wrong -> set the ejecution to crashed and stop print("\n\033[91mThe scenario could not be loaded:") print("> {}\033[0m\n".format(e)) traceback.print_exc() crash_message = "Simulation crashed" entry_status = "Crashed" self._register_statistics(config, args.checkpoint, entry_status, crash_message) if args.record: self.client.stop_recorder() self._cleanup() sys.exit(-1) print("\033[1m> Running the route\033[0m") # Run the scenario try: self.manager.run_scenario() except AgentError as e: # The agent has failed -> stop the route print("\n\033[91mStopping the route, the agent has crashed:") print("> {}\033[0m\n".format(e)) traceback.print_exc() crash_message = "Agent crashed" except Exception as e: print("\n\033[91mError during the simulation:") print("> {}\033[0m\n".format(e)) traceback.print_exc() crash_message = "Simulation crashed" entry_status = "Crashed" # Stop the scenario try: print("\033[1m> Stopping the route\033[0m") self.manager.stop_scenario() self._register_statistics(config, args.checkpoint, entry_status, crash_message) if args.record: self.client.stop_recorder() # Remove all actors scenario.remove_all_actors() self._cleanup() except Exception as e: print("\n\033[91mFailed to stop the scenario, the statistics might be empty:") print("> {}\033[0m\n".format(e)) traceback.print_exc() crash_message = "Simulation crashed" if crash_message == "Simulation crashed": sys.exit(-1) def run(self, args): """ Run the challenge mode """ route_indexer = RouteIndexer(args.routes, args.scenarios, args.repetitions) if args.resume: route_indexer.resume(args.checkpoint) self.statistics_manager.resume(args.checkpoint) else: self.statistics_manager.clear_record(args.checkpoint) route_indexer.save_state(args.checkpoint) while route_indexer.peek(): # setup config = route_indexer.next() # run self._load_and_run_scenario(args, config) route_indexer.save_state(args.checkpoint) # save global statistics print("\033[1m> Registering the global statistics\033[0m") global_stats_record = self.statistics_manager.compute_global_statistics(route_indexer.total) StatisticsManager.save_global_record(global_stats_record, self.sensor_icons, route_indexer.total, args.checkpoint)
class LeaderboardEvaluator(object): """ TODO: document me! """ ego_vehicles = [] # Tunable parameters client_timeout = 10.0 # in seconds wait_for_world = 20.0 # in seconds frame_rate = 20.0 # in Hz def __init__(self, args, statistics_manager): """ Setup CARLA client and world Setup ScenarioManager """ self.statistics_manager = statistics_manager self.sensors = [] # First of all, we need to create the client that will send the requests # to the simulator. Here we'll assume the simulator is accepting # requests in the localhost at port 2000. self.client = carla.Client(args.host, int(args.port)) if args.timeout: self.client_timeout = float(args.timeout) self.client.set_timeout(self.client_timeout) dist = pkg_resources.get_distribution("carla") if LooseVersion(dist.version) < LooseVersion('0.9.6'): raise ImportError( "CARLA version 0.9.6 or newer required. CARLA version found: {}" .format(dist)) # Load agent module_name = os.path.basename(args.agent).split('.')[0] sys.path.insert(0, os.path.dirname(args.agent)) self.module_agent = importlib.import_module(module_name) # Create the ScenarioManager self.manager = ScenarioManager(args.debug, args.challenge_mode, args.track, self.client_timeout) # Time control for summary purposes self._start_time = GameTime.get_time() self._end_time = None def __del__(self): """ Cleanup and delete actors, ScenarioManager and CARLA world """ self._cleanup(True) if hasattr(self, 'manager') and self.manager: del self.manager if hasattr(self, 'world') and self.world: del self.world def _cleanup(self, ego=False): """ Remove and destroy all actors """ self.client.stop_recorder() CarlaDataProvider.cleanup() CarlaActorPool.cleanup() for i, _ in enumerate(self.ego_vehicles): if self.ego_vehicles[i]: if ego: self.ego_vehicles[i].destroy() self.ego_vehicles[i] = None self.ego_vehicles = [] if hasattr(self, 'agent_instance') and self.agent_instance: self.agent_instance.destroy() self.agent_instance = None def _prepare_ego_vehicles(self, ego_vehicles, wait_for_ego_vehicles=False): """ Spawn or update the ego vehicles """ if not wait_for_ego_vehicles: for vehicle in ego_vehicles: self.ego_vehicles.append( CarlaActorPool.setup_actor( vehicle.model, vehicle.transform, vehicle.rolename, True, color=vehicle.color, vehicle_category=vehicle.category)) else: ego_vehicle_missing = True while ego_vehicle_missing: self.ego_vehicles = [] ego_vehicle_missing = False for ego_vehicle in ego_vehicles: ego_vehicle_found = False carla_vehicles = CarlaDataProvider.get_world().get_actors( ).filter('vehicle.*') for carla_vehicle in carla_vehicles: if carla_vehicle.attributes[ 'role_name'] == ego_vehicle.rolename: ego_vehicle_found = True self.ego_vehicles.append(carla_vehicle) break if not ego_vehicle_found: ego_vehicle_missing = True break for i, _ in enumerate(self.ego_vehicles): self.ego_vehicles[i].set_transform(ego_vehicles[i].transform) # sync state CarlaDataProvider.get_world().tick() def _load_and_wait_for_world(self, args, town, ego_vehicles=None): """ Load a new CARLA world and provide data to CarlaActorPool and CarlaDataProvider """ self.world = self.client.load_world(town) settings = self.world.get_settings() settings.fixed_delta_seconds = 1.0 / self.frame_rate settings.synchronous_mode = True self.world.apply_settings(settings) CarlaActorPool.set_client(self.client) CarlaActorPool.set_world(self.world) CarlaDataProvider.set_world(self.world) spectator = CarlaDataProvider.get_world().get_spectator() spectator.set_transform( carla.Transform(carla.Location(x=0, y=0, z=20), carla.Rotation(pitch=-90))) # Wait for the world to be ready if self.world.get_settings().synchronous_mode: self.world.tick() else: self.world.wait_for_tick() if CarlaDataProvider.get_map().name != town: print("The CARLA server uses the wrong map!") print("This scenario requires to use map {}".format(town)) return False return True def _load_and_run_scenario(self, args, config): """ Load and run the scenario given by config """ if not self._load_and_wait_for_world(args, config.town, config.ego_vehicles): self._cleanup() return agent_class_name = getattr(self.module_agent, 'get_entry_point')() try: self.agent_instance = getattr(self.module_agent, agent_class_name)(args.agent_config) config.agent = self.agent_instance self.sensors = [ sensors_to_icons[sensor['type']] for sensor in self.agent_instance.sensors() ] except Exception as e: print("Could not setup required agent due to {}".format(e)) self._cleanup() return # Prepare scenario print("Preparing scenario: " + config.name) try: self._prepare_ego_vehicles(config.ego_vehicles, False) scenario = RouteScenario(world=self.world, config=config, debug_mode=args.debug) except Exception as exception: print("The scenario cannot be loaded") if args.debug: traceback.print_exc() print(exception) self._cleanup() return # Set the appropriate weather conditions weather = carla.WeatherParameters( cloudiness=config.weather.cloudiness, precipitation=config.weather.precipitation, precipitation_deposits=config.weather.precipitation_deposits, wind_intensity=config.weather.wind_intensity, sun_azimuth_angle=config.weather.sun_azimuth_angle, sun_altitude_angle=config.weather.sun_altitude_angle, fog_density=config.weather.fog_density, fog_distance=config.weather.fog_distance, wetness=config.weather.wetness) self.world.set_weather(weather) # Set the appropriate road friction if config.friction is not None: friction_bp = self.world.get_blueprint_library().find( 'static.trigger.friction') extent = carla.Location(1000000.0, 1000000.0, 1000000.0) friction_bp.set_attribute('friction', str(config.friction)) friction_bp.set_attribute('extent_x', str(extent.x)) friction_bp.set_attribute('extent_y', str(extent.y)) friction_bp.set_attribute('extent_z', str(extent.z)) # Spawn Trigger Friction transform = carla.Transform() transform.location = carla.Location(-10000.0, -10000.0, 0.0) self.world.spawn_actor(friction_bp, transform) try: # Load scenario and run it if args.record: self.client.start_recorder("{}/{}.log".format( args.record, config.name)) self.manager.load_scenario(scenario, self.agent_instance) self.statistics_manager.set_route(config.name, config.index, scenario.scenario) self.manager.run_scenario() # Stop scenario self.manager.stop_scenario() # register statistics current_stats_record = self.statistics_manager.compute_route_statistics( config, self.manager.scenario_duration_system, self.manager.scenario_duration_game) # save self.statistics_manager.save_record(current_stats_record, config.index, args.checkpoint) # Remove all actors scenario.remove_all_actors() settings = self.world.get_settings() settings.synchronous_mode = False settings.fixed_delta_seconds = None self.world.apply_settings(settings) except SensorConfigurationInvalid as e: self._cleanup(True) sys.exit(-1) except Exception as e: if args.debug: traceback.print_exc() print(e) self._cleanup() def run(self, args): """ Run the challenge mode """ route_indexer = RouteIndexer(args.routes, args.scenarios, args.repetitions) if args.resume: route_indexer.resume(args.checkpoint) self.statistics_manager.resume(args.checkpoint) else: self.statistics_manager.clear_record(args.checkpoint) while route_indexer.peek(): # setup config = route_indexer.next() # run self._load_and_run_scenario(args, config) self._cleanup(ego=True) route_indexer.save_state(args.checkpoint) # save global statistics global_stats_record = self.statistics_manager.compute_global_statistics( route_indexer.total) StatisticsManager.save_global_record(global_stats_record, self.sensors, args.checkpoint)
class LeaderboardEvaluator(object): """ TODO: document me! """ ego_vehicles = [] # Tunable parameters client_timeout = 10.0 # in seconds wait_for_world = 20.0 # in seconds # modification: 20.0 -> 10.0 frame_rate = 10.0 # in Hz def __init__(self, args, statistics_manager, launch_server=False, episode_max_time=10000): """ Setup CARLA client and world Setup ScenarioManager """ self.statistics_manager = statistics_manager self.sensors = [] self._vehicle_lights = carla.VehicleLightState.Position | carla.VehicleLightState.LowBeam self.episode_max_time = episode_max_time # First of all, we need to create the client that will send the requests # to the simulator. Here we'll assume the simulator is accepting # requests in the localhost at port 2000. # This is currently set to be consistent with os.environ['HAS_DISPLAY']. # however, it is possible to control them separately. if os.environ['HAS_DISPLAY'] == '0': os.environ["DISPLAY"] = '' gpu = port_to_gpu[args.port] # if os.environ.get('SUDO_USER') is not None: # username = os.environ['SUDO_USER'] # else: # username = os.environ['USER'] # print('\n'*10, 'username', username, '\n'*10) os.environ["CUDA_VISIBLE_DEVICES"] = str(gpu) self.cmd_list = shlex.split( 'sh ../carla_0994_no_rss/CarlaUE4.sh -opengl -carla-rpc-port=' + str(args.port) + ' -carla-streaming-port=0') if launch_server: while is_port_in_use(args.port): try: subprocess.run('kill $(lsof -t -i :' + str(args.port) + ')', shell=True) print('-' * 20, 'kill server at port', port) time.sleep(2) except: continue subprocess.Popen(self.cmd_list) print('-' * 20, 'start server at port', args.port) # 10s is usually enough time.sleep(10) while True: try: self.client = carla.Client(args.host, int(args.port)) break except: logging.exception("__init__ error") traceback.print_exc() if args.timeout: self.client_timeout = float(args.timeout) self.client.set_timeout(self.client_timeout) dist = pkg_resources.get_distribution("carla") if LooseVersion(dist.version) < LooseVersion('0.9.9'): raise ImportError( "CARLA version 0.9.9 or newer required. CARLA version found: {}" .format(dist)) # Load agent module_name = os.path.basename(args.agent).split('.')[0] sys.path.insert(0, os.path.dirname(args.agent)) self.module_agent = importlib.import_module(module_name) # Create the ScenarioManager self.manager = ScenarioManager(args.debug, args.sync, args.challenge_mode, args.track, self.client_timeout, self.episode_max_time) # Time control for summary purposes self._start_time = GameTime.get_time() self._end_time = None # addition parent_folder = args.save_folder if not os.path.exists(parent_folder): os.mkdir(parent_folder) string = pathlib.Path(os.environ['ROUTES']).stem current_record_folder = pathlib.Path(parent_folder) / string if os.path.exists(str(current_record_folder)): import shutil shutil.rmtree(current_record_folder) current_record_folder.mkdir(exist_ok=False) (current_record_folder / 'rgb').mkdir() (current_record_folder / 'rgb_left').mkdir() (current_record_folder / 'rgb_right').mkdir() (current_record_folder / 'topdown').mkdir() (current_record_folder / 'rgb_with_car').mkdir() # if args.agent == 'leaderboard/team_code/auto_pilot.py': # (current_record_folder / 'topdown').mkdir() self.save_path = str(current_record_folder / 'events.txt') def __del__(self): """ Cleanup and delete actors, ScenarioManager and CARLA world """ self._cleanup(True) if hasattr(self, 'manager') and self.manager: del self.manager if hasattr(self, 'world') and self.world: del self.world # addition: manually delete client to avoid RuntimeError: Resource temporarily unavailable del self.client def _cleanup(self, ego=False): """ Remove and destroy all actors """ self.client.stop_recorder() CarlaDataProvider.cleanup() for i, _ in enumerate(self.ego_vehicles): if self.ego_vehicles[i]: if ego: self.ego_vehicles[i].destroy() self.ego_vehicles[i] = None self.ego_vehicles = [] if hasattr(self, 'agent_instance') and self.agent_instance: self.agent_instance.destroy() self.agent_instance = None def _prepare_ego_vehicles(self, ego_vehicles, wait_for_ego_vehicles=False): """ Spawn or update the ego vehicles """ if not wait_for_ego_vehicles: for vehicle in ego_vehicles: self.ego_vehicles.append( CarlaDataProvider.setup_actor( vehicle.model, vehicle.transform, vehicle.rolename, True, color=vehicle.color, vehicle_category=vehicle.category)) else: ego_vehicle_missing = True while ego_vehicle_missing: self.ego_vehicles = [] ego_vehicle_missing = False for ego_vehicle in ego_vehicles: ego_vehicle_found = False carla_vehicles = CarlaDataProvider.get_world().get_actors( ).filter('vehicle.*') for carla_vehicle in carla_vehicles: if carla_vehicle.attributes[ 'role_name'] == ego_vehicle.rolename: ego_vehicle_found = True self.ego_vehicles.append(carla_vehicle) break if not ego_vehicle_found: ego_vehicle_missing = True break for i, _ in enumerate(self.ego_vehicles): self.ego_vehicles[i].set_transform(ego_vehicles[i].transform) # sync state CarlaDataProvider.get_world().tick() def _load_and_wait_for_world(self, args, town, ego_vehicles=None): """ Load a new CARLA world and provide data to CarlaDataProvider and CarlaDataProvider """ while True: try: self.world = self.client.load_world(town) break except: logging.exception("_load_and_wait_for_world error") traceback.print_exc() while is_port_in_use(args.port): for proc in process_iter(): for conns in proc.connections(kind='inet'): if conns.laddr.port == args.port: proc.send_signal(SIGKILL) print('-' * 500, 'kill server at port', args.port) time.sleep(2) subprocess.Popen(self.cmd_list) print('-' * 20, 'start server at port', args.port) time.sleep(10) self.client = carla.Client(args.host, int(args.port)) settings = self.world.get_settings() settings.fixed_delta_seconds = 1.0 / self.frame_rate settings.synchronous_mode = True self.world.apply_settings(settings) CarlaDataProvider.set_client(self.client) CarlaDataProvider.set_world(self.world) CarlaDataProvider.set_world(self.world) spectator = CarlaDataProvider.get_world().get_spectator() spectator.set_transform( carla.Transform(carla.Location(x=0, y=0, z=20), carla.Rotation(pitch=-90))) # Wait for the world to be ready if self.world.get_settings().synchronous_mode: self.world.tick() else: self.world.wait_for_tick() if CarlaDataProvider.get_map().name != town: print("The CARLA server uses the wrong map!") print("This scenario requires to use map {}".format(town)) return False return True def _load_and_run_scenario(self, args, config, customized_data): """ Load and run the scenario given by config """ # hack: config.weather = WEATHERS[args.weather_index] config.friction = customized_data['friction'] config.cur_server_port = customized_data['port'] if not self._load_and_wait_for_world(args, config.town, config.ego_vehicles): self._cleanup() return _, route = interpolate_trajectory(self.world, config.trajectory) customized_data['center_transform'] = route[int(len(route) // 2)][0] ''' customized non-default center transforms for actors ['waypoint_ratio', 'absolute_location'] ''' for k, v in customized_data['customized_center_transforms'].items(): if v[0] == 'waypoint_ratio': ind = np.min([int(len(route) * v[1]), len(route) - 1]) loc = route[ind][0].location customized_data[k] = create_transform(loc.x, loc.y, 0, 0, 0, 0) elif v[0] == 'absolute_location': customized_data[k] = create_transform(v[1], v[2], 0, 0, 0, 0) else: print('unknown key', k) if 'weather_index' in customized_data: print('-' * 100) print('port :', customized_data['port']) print('center_transform :', '(', customized_data['center_transform'].location.x, customized_data['center_transform'].location.y, ')') print('friction :', customized_data['friction']) print('weather_index :', customized_data['weather_index']) print('num_of_static :', customized_data['num_of_static']) print('num_of_pedestrians :', customized_data['num_of_pedestrians']) print('num_of_vehicles :', customized_data['num_of_vehicles']) print('-' * 100) agent_class_name = getattr(self.module_agent, 'get_entry_point')() try: self.agent_instance = getattr(self.module_agent, agent_class_name)(args.agent_config) # addition # self.agent_instance.set_trajectory(config.trajectory) self.agent_instance.set_deviations_path(args.deviations_folder) config.agent = self.agent_instance self.sensors = [ sensors_to_icons[sensor['type']] for sensor in self.agent_instance.sensors() ] except Exception as e: print("Could not setup required agent due to {}".format(e)) traceback.print_exc() self._cleanup() return # Prepare scenario print("Preparing scenario: " + config.name) try: self._prepare_ego_vehicles(config.ego_vehicles, False) # print('\n'*10, 'RouteScenario config.cur_server_port', config.cur_server_port, '\n'*10) scenario = RouteScenario(world=self.world, config=config, debug_mode=args.debug, customized_data=customized_data) except Exception as exception: print("The scenario cannot be loaded") if args.debug: traceback.print_exc() print(exception) self._cleanup() return # Set the appropriate weather conditions weather = carla.WeatherParameters( cloudiness=config.weather.cloudiness, precipitation=config.weather.precipitation, precipitation_deposits=config.weather.precipitation_deposits, wind_intensity=config.weather.wind_intensity, sun_azimuth_angle=config.weather.sun_azimuth_angle, sun_altitude_angle=config.weather.sun_altitude_angle, fog_density=config.weather.fog_density, fog_distance=config.weather.fog_distance, wetness=config.weather.wetness) self.world.set_weather(weather) # Set the appropriate road friction if config.friction is not None: friction_bp = self.world.get_blueprint_library().find( 'static.trigger.friction') extent = carla.Location(1000000.0, 1000000.0, 1000000.0) friction_bp.set_attribute('friction', str(config.friction)) friction_bp.set_attribute('extent_x', str(extent.x)) friction_bp.set_attribute('extent_y', str(extent.y)) friction_bp.set_attribute('extent_z', str(extent.z)) # Spawn Trigger Friction transform = carla.Transform() transform.location = carla.Location(-10000.0, -10000.0, 0.0) self.world.spawn_actor(friction_bp, transform) # night mode if config.weather.sun_altitude_angle < 0.0: for vehicle in scenario.ego_vehicles: vehicle.set_light_state( carla.VehicleLightState(self._vehicle_lights)) # addition: to turn on lights of actor_list = self.world.get_actors() vehicle_list = actor_list.filter('*vehicle*') for vehicle in vehicle_list: vehicle.set_light_state( carla.VehicleLightState(self._vehicle_lights)) try: # Load scenario and run it if args.record: self.client.start_recorder("{}/{}.log".format( args.record, config.name)) self.manager.load_scenario(scenario, self.agent_instance) self.statistics_manager.set_route(config.name, config.index, scenario.scenario) print('start to run scenario') self.manager.run_scenario() print('stop to run scanario') # Stop scenario self.manager.stop_scenario() # register statistics current_stats_record = self.statistics_manager.compute_route_statistics( config, self.manager.scenario_duration_system, self.manager.scenario_duration_game) # save # modification self.statistics_manager.save_record(current_stats_record, config.index, self.save_path) # Remove all actors scenario.remove_all_actors() settings = self.world.get_settings() settings.synchronous_mode = False settings.fixed_delta_seconds = None self.world.apply_settings(settings) except SensorConfigurationInvalid as e: self._cleanup(True) sys.exit(-1) except Exception as e: if args.debug: traceback.print_exc() print(e) self._cleanup() def run(self, args, customized_data): """ Run the challenge mode """ route_indexer = RouteIndexer(args.routes, args.scenarios, args.repetitions) if args.resume: route_indexer.resume(self.save_path) self.statistics_manager.resume(self.save_path) else: self.statistics_manager.clear_record(self.save_path) while route_indexer.peek(): # setup config = route_indexer.next() # run self._load_and_run_scenario(args, config, customized_data) self._cleanup(ego=True) route_indexer.save_state(self.save_path) # save global statistics # modification global_stats_record = self.statistics_manager.compute_global_statistics( route_indexer.total) StatisticsManager.save_global_record(global_stats_record, self.sensors, self.save_path)
class NoCrashEvaluator(object): """ TODO: document me! """ ego_vehicles = [] # Tunable parameters client_timeout = 10.0 # in seconds wait_for_world = 20.0 # in seconds frame_rate = 20.0 # in Hz def __init__(self, args, statistics_manager): """ Setup CARLA client and world Setup ScenarioManager """ self.statistics_manager = statistics_manager self.sensors = None self.sensor_icons = [] # First of all, we need to create the client that will send the requests # to the simulator. Here we'll assume the simulator is accepting # requests in the localhost at port 2000. self.client = carla.Client(args.host, int(args.port)) if args.timeout: self.client_timeout = float(args.timeout) self.client.set_timeout(self.client_timeout) self.traffic_manager = self.client.get_trafficmanager( int(args.trafficManagerPort)) dist = pkg_resources.get_distribution("carla") if dist.version != 'leaderboard': if LooseVersion(dist.version) < LooseVersion('0.9.10'): raise ImportError( "CARLA version 0.9.10.1 or newer required. CARLA version found: {}" .format(dist)) # Load agent module_name = os.path.basename(args.agent).split('.')[0] sys.path.insert(0, os.path.dirname(args.agent)) self.module_agent = importlib.import_module(module_name) # Create the ScenarioManager self.manager = ScenarioManager(args.timeout, args.debug > 1) # Time control for summary purposes self._start_time = GameTime.get_time() self._end_time = None # Create the agent timer self._agent_watchdog = Watchdog(int(float(args.timeout))) signal.signal(signal.SIGINT, self._signal_handler) self.town = args.town def _signal_handler(self, signum, frame): """ Terminate scenario ticking when receiving a signal interrupt """ if self._agent_watchdog and not self._agent_watchdog.get_status(): raise RuntimeError("Timeout: Agent took too long to setup") elif self.manager: self.manager.signal_handler(signum, frame) def __del__(self): """ Cleanup and delete actors, ScenarioManager and CARLA world """ self._cleanup() if hasattr(self, 'manager') and self.manager: del self.manager if hasattr(self, 'world') and self.world: del self.world def _cleanup(self): """ Remove and destroy all actors """ # Simulation still running and in synchronous mode? if self.manager and self.manager.get_running_status() \ and hasattr(self, 'world') and self.world: # Reset to asynchronous mode settings = self.world.get_settings() settings.synchronous_mode = False settings.fixed_delta_seconds = None self.world.apply_settings(settings) self.traffic_manager.set_synchronous_mode(False) if self.manager: self.manager.cleanup() CarlaDataProvider.cleanup() for i, _ in enumerate(self.ego_vehicles): if self.ego_vehicles[i]: self.ego_vehicles[i].destroy() self.ego_vehicles[i] = None self.ego_vehicles = [] if self._agent_watchdog: self._agent_watchdog.stop() if hasattr(self, 'agent_instance') and self.agent_instance: self.agent_instance.destroy() self.agent_instance = None if hasattr(self, 'statistics_manager') and self.statistics_manager: self.statistics_manager.scenario = None def _load_and_wait_for_world(self, args): """ Load a new CARLA world and provide data to CarlaDataProvider """ self.world = self.client.load_world(args.town) settings = self.world.get_settings() settings.fixed_delta_seconds = 1.0 / self.frame_rate settings.synchronous_mode = True self.world.apply_settings(settings) self.world.reset_all_traffic_lights() CarlaDataProvider.set_client(self.client) CarlaDataProvider.set_world(self.world) CarlaDataProvider.set_traffic_manager_port(int( args.trafficManagerPort)) self.traffic_manager.set_synchronous_mode(True) self.traffic_manager.set_random_device_seed( int(args.trafficManagerSeed)) # Wait for the world to be ready if CarlaDataProvider.is_sync_mode(): self.world.tick() else: self.world.wait_for_tick() if CarlaDataProvider.get_map().name != args.town: raise Exception("The CARLA server uses the wrong map!" "This scenario requires to use map {}".format( args.town)) # def _register_statistics(self, config, checkpoint, entry_status, crash_message=""): # """ # Computes and saved the simulation statistics # """ # # register statistics # current_stats_record = self.statistics_manager.compute_route_statistics( # config, # self.manager.scenario_duration_system, # self.manager.scenario_duration_game, # crash_message # ) # print("\033[1m> Registering the route statistics\033[0m") # self.statistics_manager.save_record(current_stats_record, config.index, checkpoint) # self.statistics_manager.save_entry_status(entry_status, False, checkpoint) def _load_and_run_scenario(self, args, route, weather_idx, traffic_idx): """ Load and run the scenario given by config. Depending on what code fails, the simulation will either stop the route and continue from the next one, or report a crash and stop. """ crash_message = "" entry_status = "Started" start_idx, target_idx = route traffic_lvl = ['Empty', 'Regular', 'Dense'][traffic_idx] print( "\n\033[1m========= Preparing {} {}: {} to {}, weather {} =========" .format(args.town, traffic_lvl, start_idx, target_idx, weather_idx)) print("> Setting up the agent\033[0m") # Set up the user's agent, and the timer to avoid freezing the simulation try: self._agent_watchdog.start() agent_class_name = getattr(self.module_agent, 'get_entry_point')() self.agent_instance = getattr(self.module_agent, agent_class_name)(args.agent_config) # Check and store the sensors if not self.sensors: self.sensors = self.agent_instance.sensors() track = self.agent_instance.track AgentWrapper.validate_sensor_configuration( self.sensors, track, args.track) self._agent_watchdog.stop() except SensorConfigurationInvalid as e: # The sensors are invalid -> set the ejecution to rejected and stop print("\n\033[91mThe sensor's configuration used is invalid:") print("> {}\033[0m\n".format(e)) traceback.print_exc() crash_message = "Agent's sensors were invalid" entry_status = "Rejected" self._cleanup() sys.exit(-1) except Exception as e: # The agent setup has failed -> start the next route print("\n\033[91mCould not set up the required agent:") print("> {}\033[0m\n".format(e)) traceback.print_exc() crash_message = "Agent couldn't be set up" self._cleanup() return print("\033[1m> Loading the world\033[0m") # Load the world and the scenario try: self._load_and_wait_for_world(args) scenario = NoCrashEvalScenario(world=self.world, agent=self.agent_instance, start_idx=start_idx, target_idx=target_idx, weather_idx=weather_idx, traffic_idx=traffic_idx, debug_mode=args.debug) self.manager.load_scenario(scenario, self.agent_instance, 0) except Exception as e: # The scenario is wrong -> set the ejecution to crashed and stop print("\n\033[91mThe scenario could not be loaded:") print("> {}\033[0m\n".format(e)) traceback.print_exc() crash_message = "Simulation crashed" entry_status = "Crashed" if args.record: self.client.stop_recorder() self._cleanup() sys.exit(-1) print("\033[1m> Running the route\033[0m") # Run the scenario try: self.manager.run_scenario() except AgentError as e: # The agent has failed -> stop the route print("\n\033[91mStopping the route, the agent has crashed:") print("> {}\033[0m\n".format(e)) traceback.print_exc() crash_message = "Agent crashed" except Exception as e: print("\n\033[91mError during the simulation:") print("> {}\033[0m\n".format(e)) traceback.print_exc() crash_message = "Simulation crashed" entry_status = "Crashed" # Stop the scenario try: print("\033[1m> Stopping the route\033[0m") self.manager.stop_scenario() route_completion, lights_ran, duration = self.manager.get_nocrash_diagnostics( ) self.statistics_manager.log(self.town, traffic_idx, weather_idx, start_idx, target_idx, route_completion, lights_ran, duration) if args.record: self.client.stop_recorder() # Remove all actors scenario.remove_all_actors() self._cleanup() except Exception as e: print( "\n\033[91mFailed to stop the scenario, the statistics might be empty:" ) print("> {}\033[0m\n".format(e)) traceback.print_exc() crash_message = "Simulation crashed" if crash_message == "Simulation crashed": sys.exit(-1) def run(self, args): """ Run the challenge mode """ # if args.resume: # route_indexer.resume(args.checkpoint) # self.statistics_manager.resume(args.checkpoint) # else: # self.statistics_manager.clear_record(args.checkpoint) # route_indexer.save_state(args.checkpoint) # Load routes with open(f'runners/suite/nocrash_{args.town}.txt', 'r') as f: routes = [tuple(map(int, l.split())) for l in f.readlines()] weathers = {'train': [1, 3, 6, 8], 'test': [10, 14]}.get(args.weather) traffics = [0, 1, 2] for traffic, route, weather in itertools.product( traffics, routes, weathers): if self.statistics_manager.is_finished(self.town, route, weather, traffic): continue self._load_and_run_scenario(args, route, weather, traffic) # save global statistics print("\033[1m> Registering the global statistics\033[0m")
class LeaderboardEvaluator(object): """ TODO: document me! """ ego_vehicles = [] # Tunable parameters client_timeout = 20.0 # in seconds wait_for_world = 20.0 # in seconds # modification: 20.0 -> 10.0 frame_rate = 10.0 # in Hz def __init__(self, args, statistics_manager, launch_server=False, episode_max_time=10000): """ Setup CARLA client and world Setup ScenarioManager """ self.statistics_manager = statistics_manager self.sensors = [] self._vehicle_lights = (carla.VehicleLightState.Position | carla.VehicleLightState.LowBeam) self.episode_max_time = episode_max_time # First of all, we need to create the client that will send the requests # to the simulator. # This is currently set to be consistent with os.environ['HAS_DISPLAY']. # however, it is possible to control them separately. if os.environ["HAS_DISPLAY"] == "0": os.environ["DISPLAY"] = "" gpu = port_to_gpu(int(args.port)) os.environ["CUDA_VISIBLE_DEVICES"] = str(gpu) if launch_server: start_server(args.port) start_client(self, args.host, int(args.port)) if args.timeout: self.client_timeout = float(args.timeout) self.client.set_timeout(self.client_timeout) dist = pkg_resources.get_distribution("carla") if LooseVersion(dist.version) < LooseVersion("0.9.9"): raise ImportError( "CARLA version 0.9.9 or newer required. CARLA version found: {}" .format(dist)) # Load agent module_name = os.path.basename(args.agent).split(".")[0] sys.path.insert(0, os.path.dirname(args.agent)) self.module_agent = importlib.import_module(module_name) # Create the ScenarioManager self.manager = ScenarioManager( args.debug, args.sync, args.challenge_mode, args.track, self.client_timeout, self.episode_max_time, ) # Time control for summary purposes self._start_time = GameTime.get_time() self._end_time = None # addition parent_folder = args.save_folder if not os.path.exists(parent_folder): os.mkdir(parent_folder) string = pathlib.Path(os.environ["ROUTES"]).stem current_record_folder = pathlib.Path(parent_folder) / string if os.path.exists(str(current_record_folder)): import shutil shutil.rmtree(current_record_folder) current_record_folder.mkdir(exist_ok=False) (current_record_folder / "rgb").mkdir() (current_record_folder / "rgb_left").mkdir() (current_record_folder / "rgb_right").mkdir() (current_record_folder / "topdown").mkdir() (current_record_folder / "rgb_with_car").mkdir() # if args.agent == 'leaderboard/team_code/auto_pilot.py': # (current_record_folder / 'topdown').mkdir() self.save_path = str(current_record_folder / "events.txt") def __del__(self): """ Cleanup and delete actors, ScenarioManager and CARLA world """ self._cleanup(True) if hasattr(self, "manager") and self.manager: del self.manager if hasattr(self, "world") and self.world: del self.world # addition: manually delete client to avoid RuntimeError: Resource temporarily unavailable del self.client def _cleanup(self, ego=False): """ Remove and destroy all actors """ self.client.stop_recorder() CarlaDataProvider.cleanup() for i, _ in enumerate(self.ego_vehicles): if self.ego_vehicles[i]: if ego: self.ego_vehicles[i].destroy() self.ego_vehicles[i] = None self.ego_vehicles = [] if hasattr(self, "agent_instance") and self.agent_instance: self.agent_instance.destroy() self.agent_instance = None def _prepare_ego_vehicles(self, ego_vehicles, wait_for_ego_vehicles=False): """ Spawn or update the ego vehicles """ if not wait_for_ego_vehicles: for vehicle in ego_vehicles: self.ego_vehicles.append( CarlaDataProvider.setup_actor( vehicle.model, vehicle.transform, vehicle.rolename, True, color=vehicle.color, vehicle_category=vehicle.category, )) else: ego_vehicle_missing = True while ego_vehicle_missing: self.ego_vehicles = [] ego_vehicle_missing = False for ego_vehicle in ego_vehicles: ego_vehicle_found = False carla_vehicles = (CarlaDataProvider.get_world().get_actors( ).filter("vehicle.*")) for carla_vehicle in carla_vehicles: if (carla_vehicle.attributes["role_name"] == ego_vehicle.rolename): ego_vehicle_found = True self.ego_vehicles.append(carla_vehicle) break if not ego_vehicle_found: ego_vehicle_missing = True break for i, _ in enumerate(self.ego_vehicles): self.ego_vehicles[i].set_transform(ego_vehicles[i].transform) # sync state CarlaDataProvider.get_world().tick() def _load_and_wait_for_world(self, args, town, ego_vehicles=None): """ Load a new CARLA world and provide data to CarlaDataProvider and CarlaDataProvider """ try_load_world(self, town, args.host, int(args.port)) settings = self.world.get_settings() settings.fixed_delta_seconds = 1.0 / self.frame_rate settings.synchronous_mode = True self.world.apply_settings(settings) CarlaDataProvider.set_client(self.client) CarlaDataProvider.set_world(self.world) spectator = CarlaDataProvider.get_world().get_spectator() spectator.set_transform( carla.Transform(carla.Location(x=0, y=0, z=20), carla.Rotation(pitch=-90))) # Wait for the world to be ready if self.world.get_settings().synchronous_mode: self.world.tick() else: self.world.wait_for_tick() if CarlaDataProvider.get_map().name != town: print("The CARLA server uses the wrong map!") print("This scenario requires to use map {}".format(town)) return False return True def _load_and_run_scenario(self, args, config, customized_data): """ Load and run the scenario given by config """ # hack: if args.weather_index == -1: weather = customized_data["fine_grained_weather"] else: weather = WEATHERS[args.weather_index] config.weather = weather config.friction = customized_data["friction"] config.cur_server_port = customized_data["port"] if not self._load_and_wait_for_world(args, config.town, config.ego_vehicles): self._cleanup() return _, route = interpolate_trajectory(self.world, config.trajectory) customized_data["center_transform"] = route[int(len(route) // 2)][0] # from customized_utils import visualize_route # visualize_route(route) # transforms = [] # for x in route: # transforms.append(x[0]) # print('len(transforms)', len(transforms)) if customized_data['using_customized_route_and_scenario']: """ customized non-default center transforms for actors ['waypoint_ratio', 'absolute_location'] """ for k, v in customized_data["customized_center_transforms"].items( ): if v[0] == "waypoint_ratio": r = v[1] / 100 ind = np.min([int(len(route) * r), len(route) - 1]) loc = route[ind][0].location customized_data[k] = create_transform( loc.x, loc.y, 0, 0, 0, 0) print("waypoint_ratio", loc.x, loc.y) elif v[0] == "absolute_location": customized_data[k] = create_transform( v[1], v[2], 0, 0, 0, 0) else: print("unknown key", k) print("-" * 100) print("port :", customized_data["port"]) print( "center_transform :", "(", customized_data["center_transform"].location.x, customized_data["center_transform"].location.y, ")", ) print("friction :", customized_data["friction"]) print("weather_index :", customized_data["weather_index"]) print("num_of_static :", customized_data["num_of_static"]) print("num_of_pedestrians :", customized_data["num_of_pedestrians"]) print("num_of_vehicles :", customized_data["num_of_vehicles"]) print("-" * 100) agent_class_name = getattr(self.module_agent, "get_entry_point")() try: self.agent_instance = getattr(self.module_agent, agent_class_name)(args.agent_config) # addition # self.agent_instance.set_trajectory(config.trajectory) self.agent_instance.set_args(args) config.agent = self.agent_instance self.sensors = [ sensors_to_icons[sensor["type"]] for sensor in self.agent_instance.sensors() ] except Exception as e: print("Could not setup required agent due to {}".format(e)) traceback.print_exc() self._cleanup() return # Prepare scenario print("Preparing scenario: " + config.name) try: self._prepare_ego_vehicles(config.ego_vehicles, False) # print('\n'*10, 'RouteScenario config.cur_server_port', config.cur_server_port, '\n'*10) scenario = RouteScenario( world=self.world, config=config, debug_mode=args.debug, customized_data=customized_data, ) except Exception as exception: print("The scenario cannot be loaded") if args.debug: traceback.print_exc() print(exception) self._cleanup() return # Set the appropriate weather conditions weather = carla.WeatherParameters( cloudiness=config.weather.cloudiness, precipitation=config.weather.precipitation, precipitation_deposits=config.weather.precipitation_deposits, wind_intensity=config.weather.wind_intensity, sun_azimuth_angle=config.weather.sun_azimuth_angle, sun_altitude_angle=config.weather.sun_altitude_angle, fog_density=config.weather.fog_density, fog_distance=config.weather.fog_distance, wetness=config.weather.wetness, ) self.world.set_weather(weather) # Set the appropriate road friction if config.friction is not None: friction_bp = self.world.get_blueprint_library().find( "static.trigger.friction") extent = carla.Location(1000000.0, 1000000.0, 1000000.0) friction_bp.set_attribute("friction", str(config.friction)) friction_bp.set_attribute("extent_x", str(extent.x)) friction_bp.set_attribute("extent_y", str(extent.y)) friction_bp.set_attribute("extent_z", str(extent.z)) # Spawn Trigger Friction transform = carla.Transform() transform.location = carla.Location(-10000.0, -10000.0, 0.0) self.world.spawn_actor(friction_bp, transform) # night mode if config.weather.sun_altitude_angle < 0.0: for vehicle in scenario.ego_vehicles: vehicle.set_light_state( carla.VehicleLightState(self._vehicle_lights)) # addition: to turn on lights of actor_list = self.world.get_actors() vehicle_list = actor_list.filter("*vehicle*") for vehicle in vehicle_list: vehicle.set_light_state( carla.VehicleLightState(self._vehicle_lights)) try: # Load scenario and run it if args.record: self.client.start_recorder("{}/{}.log".format( args.record, config.name)) self.manager.load_scenario(scenario, self.agent_instance) self.statistics_manager.set_route(config.name, config.index, scenario.scenario) print("start to run scenario") self.manager.run_scenario() print("stop to run scanario") # Stop scenario self.manager.stop_scenario() # register statistics current_stats_record = self.statistics_manager.compute_route_statistics( config, self.manager.scenario_duration_system, self.manager.scenario_duration_game, ) # save # modification self.statistics_manager.save_record(current_stats_record, config.index, self.save_path) # Remove all actors scenario.remove_all_actors() settings = self.world.get_settings() settings.synchronous_mode = False settings.fixed_delta_seconds = None self.world.apply_settings(settings) except SensorConfigurationInvalid as e: self._cleanup(True) sys.exit(-1) except Exception as e: if args.debug: traceback.print_exc() print(e) self._cleanup() def run(self, args, customized_data): """ Run the challenge mode """ route_indexer = RouteIndexer(args.routes, args.scenarios, args.repetitions, args.background_vehicles) if args.resume: route_indexer.resume(self.save_path) self.statistics_manager.resume(self.save_path) else: self.statistics_manager.clear_record(self.save_path) while route_indexer.peek(): # setup config = route_indexer.next() # run self._load_and_run_scenario(args, config, customized_data) self._cleanup(ego=True) route_indexer.save_state(self.save_path) # save global statistics # modification global_stats_record = self.statistics_manager.compute_global_statistics( route_indexer.total) StatisticsManager.save_global_record(global_stats_record, self.sensors, self.save_path)