Esempio n. 1
0
class LeaderboardEvaluator(object):

    """
    TODO: document me!
    """

    ego_vehicles = []

    # Tunable parameters
    client_timeout = 10.0  # in seconds
    wait_for_world = 20.0  # in seconds
    frame_rate = 20.0      # in Hz

    def __init__(self, args, statistics_manager):
        """
        Setup CARLA client and world
        Setup ScenarioManager
        """
        self.statistics_manager = statistics_manager
        self.sensors = None
        self.sensor_icons = []
        self._vehicle_lights = carla.VehicleLightState.Position | carla.VehicleLightState.LowBeam

        # First of all, we need to create the client that will send the requests
        # to the simulator. Here we'll assume the simulator is accepting
        # requests in the localhost at port 2000.
        self.client = carla.Client(args.host, args.port)
        if args.timeout:
            self.client_timeout = args.timeout
        self.client.set_timeout(self.client_timeout)

        self.traffic_manager = self.client.get_trafficmanager(args.traffic_manager_port)

        dist = pkg_resources.get_distribution("carla")
        if dist.version != 'leaderboard':
            if LooseVersion(dist.version) < LooseVersion('0.9.10'):
                raise ImportError("CARLA version 0.9.10.1 or newer required. CARLA version found: {}".format(dist))

        # Load agent
        module_name = os.path.basename(args.agent).split('.')[0]
        sys.path.insert(0, os.path.dirname(args.agent))
        self.module_agent = importlib.import_module(module_name)

        # Create the ScenarioManager
        self.manager = ScenarioManager(args.timeout, args.debug > 1)

        # Time control for summary purposes
        self._start_time = GameTime.get_time()
        self._end_time = None

        # Create the agent timer
        self._agent_watchdog = Watchdog(args.timeout)
        signal.signal(signal.SIGINT, self._signal_handler)

    def _signal_handler(self, signum, frame):
        """
        Terminate scenario ticking when receiving a signal interrupt
        """
        if self._agent_watchdog and not self._agent_watchdog.get_status():
            raise RuntimeError("Timeout: Agent took longer than {}s to setup".format(self.client_timeout))
        elif self.manager:
            self.manager.signal_handler(signum, frame)

    def __del__(self):
        """
        Cleanup and delete actors, ScenarioManager and CARLA world
        """

        self._cleanup()
        if hasattr(self, 'manager') and self.manager:
            del self.manager
        if hasattr(self, 'world') and self.world:
            del self.world

    def _cleanup(self):
        """
        Remove and destroy all actors
        """

        # Simulation still running and in synchronous mode?
        if self.manager and self.manager.get_running_status() \
                and hasattr(self, 'world') and self.world:
            # Reset to asynchronous mode
            settings = self.world.get_settings()
            settings.synchronous_mode = False
            settings.fixed_delta_seconds = None
            self.world.apply_settings(settings)
            self.traffic_manager.set_synchronous_mode(False)

        if self.manager:
            self.manager.cleanup()

        CarlaDataProvider.cleanup()

        for i, _ in enumerate(self.ego_vehicles):
            if self.ego_vehicles[i]:
                self.ego_vehicles[i].destroy()
                self.ego_vehicles[i] = None
        self.ego_vehicles = []

        if self._agent_watchdog:
            self._agent_watchdog.stop()

        if hasattr(self, 'agent_instance') and self.agent_instance:
            self.agent_instance.destroy()
            self.agent_instance = None

        if hasattr(self, 'statistics_manager') and self.statistics_manager:
            self.statistics_manager.scenario = None

    def _prepare_ego_vehicles(self, ego_vehicles, wait_for_ego_vehicles=False):
        """
        Spawn or update the ego vehicles
        """

        if not wait_for_ego_vehicles:
            for vehicle in ego_vehicles:
                self.ego_vehicles.append(CarlaDataProvider.request_new_actor(vehicle.model,
                                                                             vehicle.transform,
                                                                             vehicle.rolename,
                                                                             color=vehicle.color,
                                                                             vehicle_category=vehicle.category))

        else:
            ego_vehicle_missing = True
            while ego_vehicle_missing:
                self.ego_vehicles = []
                ego_vehicle_missing = False
                for ego_vehicle in ego_vehicles:
                    ego_vehicle_found = False
                    carla_vehicles = CarlaDataProvider.get_world().get_actors().filter('vehicle.*')
                    for carla_vehicle in carla_vehicles:
                        if carla_vehicle.attributes['role_name'] == ego_vehicle.rolename:
                            ego_vehicle_found = True
                            self.ego_vehicles.append(carla_vehicle)
                            break
                    if not ego_vehicle_found:
                        ego_vehicle_missing = True
                        break

            for i, _ in enumerate(self.ego_vehicles):
                self.ego_vehicles[i].set_transform(ego_vehicles[i].transform)

        # sync state
        CarlaDataProvider.get_world().tick()

    def _load_and_wait_for_world(self, args, town, ego_vehicles=None):
        """
        Load a new CARLA world and provide data to CarlaDataProvider
        """

        self.world = self.client.load_world(town)
        settings = self.world.get_settings()
        settings.fixed_delta_seconds = 1.0 / self.frame_rate
        settings.synchronous_mode = True
        self.world.apply_settings(settings)

        self.world.reset_all_traffic_lights()

        CarlaDataProvider.set_client(self.client)
        CarlaDataProvider.set_world(self.world)
        CarlaDataProvider.set_traffic_manager_port(args.traffic_manager_port)

        self.traffic_manager.set_synchronous_mode(True)
        self.traffic_manager.set_random_device_seed(args.traffic_manager_seed)
        self.traffic_manager.set_hybrid_physics_mode(True)

        # Wait for the world to be ready
        if CarlaDataProvider.is_sync_mode():
            self.world.tick()
        else:
            self.world.wait_for_tick()

        if CarlaDataProvider.get_map().name != town:
            raise Exception("The CARLA server uses the wrong map!"
                            "This scenario requires to use map {}".format(town))

    def _register_statistics(self, config, checkpoint, entry_status, crash_message=""):
        """
        Computes and saved the simulation statistics
        """
        # register statistics
        current_stats_record = self.statistics_manager.compute_route_statistics(
            config,
            self.manager.scenario_duration_system,
            self.manager.scenario_duration_game,
            crash_message
        )

        print("\033[1m> Registering the route statistics\033[0m")
        self.statistics_manager.save_record(current_stats_record, config.index, checkpoint)
        self.statistics_manager.save_entry_status(entry_status, False, checkpoint)

    def _load_and_run_scenario(self, args, config):
        """
        Load and run the scenario given by config.

        Depending on what code fails, the simulation will either stop the route and
        continue from the next one, or report a crash and stop.
        """
        crash_message = ""
        entry_status = "Started"

        print("\n\033[1m========= Preparing {} (repetition {}) =========".format(config.name, config.repetition_index))
        print("> Setting up the agent\033[0m")

        # Prepare the statistics of the route
        self.statistics_manager.set_route(config.name, config.index)

        # Set up the user's agent, and the timer to avoid freezing the simulation
        try:
            self._agent_watchdog.start()
            agent_class_name = getattr(self.module_agent, 'get_entry_point')()
            self.agent_instance = getattr(self.module_agent, agent_class_name)(args.agent_config)
            config.agent = self.agent_instance

            # Check and store the sensors
            if not self.sensors:
                self.sensors = self.agent_instance.sensors()
                track = self.agent_instance.track

                AgentWrapper.validate_sensor_configuration(self.sensors, track, args.track)

                self.sensor_icons = [sensors_to_icons[sensor['type']] for sensor in self.sensors]
                self.statistics_manager.save_sensors(self.sensor_icons, args.checkpoint)

            self._agent_watchdog.stop()

        except SensorConfigurationInvalid as e:
            # The sensors are invalid -> set the ejecution to rejected and stop
            print("\n\033[91mThe sensor's configuration used is invalid:")
            print("> {}\033[0m\n".format(e))
            traceback.print_exc()

            crash_message = "Agent's sensors were invalid"
            entry_status = "Rejected"

            self._register_statistics(config, args.checkpoint, entry_status, crash_message)
            self._cleanup()
            sys.exit(-1)

        except Exception as e:
            # The agent setup has failed -> start the next route
            print("\n\033[91mCould not set up the required agent:")
            print("> {}\033[0m\n".format(e))
            traceback.print_exc()

            crash_message = "Agent couldn't be set up"

            self._register_statistics(config, args.checkpoint, entry_status, crash_message)
            self._cleanup()
            return

        print("\033[1m> Loading the world\033[0m")

        # Load the world and the scenario
        try:
            self._load_and_wait_for_world(args, config.town, config.ego_vehicles)
            self._prepare_ego_vehicles(config.ego_vehicles, False)
            scenario = RouteScenario(world=self.world, config=config, debug_mode=args.debug)
            self.statistics_manager.set_scenario(scenario.scenario)

            # Night mode
            if config.weather.sun_altitude_angle < 0.0:
                for vehicle in scenario.ego_vehicles:
                    vehicle.set_light_state(carla.VehicleLightState(self._vehicle_lights))

            # Load scenario and run it
            if args.record:
                self.client.start_recorder("{}/{}_rep{}.log".format(args.record, config.name, config.repetition_index))
            self.manager.load_scenario(scenario, self.agent_instance, config.repetition_index)

        except Exception as e:
            # The scenario is wrong -> set the ejecution to crashed and stop
            print("\n\033[91mThe scenario could not be loaded:")
            print("> {}\033[0m\n".format(e))
            traceback.print_exc()

            crash_message = "Simulation crashed"
            entry_status = "Crashed"

            self._register_statistics(config, args.checkpoint, entry_status, crash_message)

            if args.record:
                self.client.stop_recorder()

            self._cleanup()
            sys.exit(-1)

        print("\033[1m> Running the route\033[0m")

        # Run the scenario
        try:
            self.manager.run_scenario()

        except AgentError as e:
            # The agent has failed -> stop the route
            print("\n\033[91mStopping the route, the agent has crashed:")
            print("> {}\033[0m\n".format(e))
            traceback.print_exc()

            crash_message = "Agent crashed"

        except Exception as e:
            print("\n\033[91mError during the simulation:")
            print("> {}\033[0m\n".format(e))
            traceback.print_exc()

            crash_message = "Simulation crashed"
            entry_status = "Crashed"

        # Stop the scenario
        try:
            print("\033[1m> Stopping the route\033[0m")
            self.manager.stop_scenario()
            self._register_statistics(config, args.checkpoint, entry_status, crash_message)

            if args.record:
                self.client.stop_recorder()

            # Remove all actors
            scenario.remove_all_actors()

            self._cleanup()

        except Exception as e:
            print("\n\033[91mFailed to stop the scenario, the statistics might be empty:")
            print("> {}\033[0m\n".format(e))
            traceback.print_exc()

            crash_message = "Simulation crashed"

        if crash_message == "Simulation crashed":
            sys.exit(-1)

    def run(self, args):
        """
        Run the challenge mode
        """
        route_indexer = RouteIndexer(args.routes, args.scenarios, args.repetitions)

        if args.resume:
            route_indexer.resume(args.checkpoint)
            self.statistics_manager.resume(args.checkpoint)
        else:
            self.statistics_manager.clear_record(args.checkpoint)
            route_indexer.save_state(args.checkpoint)

        while route_indexer.peek():
            # setup
            config = route_indexer.next()

            # run
            self._load_and_run_scenario(args, config)

            route_indexer.save_state(args.checkpoint)

        # save global statistics
        print("\033[1m> Registering the global statistics\033[0m")
        global_stats_record = self.statistics_manager.compute_global_statistics(route_indexer.total)
        StatisticsManager.save_global_record(global_stats_record, self.sensor_icons, route_indexer.total, args.checkpoint)
class LeaderboardEvaluator(object):
    """
    TODO: document me!
    """

    ego_vehicles = []

    # Tunable parameters
    client_timeout = 10.0  # in seconds
    wait_for_world = 20.0  # in seconds
    frame_rate = 20.0  # in Hz

    def __init__(self, args, statistics_manager):
        """
        Setup CARLA client and world
        Setup ScenarioManager
        """
        self.statistics_manager = statistics_manager
        self.sensors = []

        # First of all, we need to create the client that will send the requests
        # to the simulator. Here we'll assume the simulator is accepting
        # requests in the localhost at port 2000.
        self.client = carla.Client(args.host, int(args.port))
        if args.timeout:
            self.client_timeout = float(args.timeout)
        self.client.set_timeout(self.client_timeout)

        dist = pkg_resources.get_distribution("carla")
        if LooseVersion(dist.version) < LooseVersion('0.9.6'):
            raise ImportError(
                "CARLA version 0.9.6 or newer required. CARLA version found: {}"
                .format(dist))

        # Load agent
        module_name = os.path.basename(args.agent).split('.')[0]
        sys.path.insert(0, os.path.dirname(args.agent))
        self.module_agent = importlib.import_module(module_name)

        # Create the ScenarioManager
        self.manager = ScenarioManager(args.debug, args.challenge_mode,
                                       args.track, self.client_timeout)

        # Time control for summary purposes
        self._start_time = GameTime.get_time()
        self._end_time = None

    def __del__(self):
        """
        Cleanup and delete actors, ScenarioManager and CARLA world
        """

        self._cleanup(True)
        if hasattr(self, 'manager') and self.manager:
            del self.manager
        if hasattr(self, 'world') and self.world:
            del self.world

    def _cleanup(self, ego=False):
        """
        Remove and destroy all actors
        """

        self.client.stop_recorder()

        CarlaDataProvider.cleanup()
        CarlaActorPool.cleanup()

        for i, _ in enumerate(self.ego_vehicles):
            if self.ego_vehicles[i]:
                if ego:
                    self.ego_vehicles[i].destroy()
                self.ego_vehicles[i] = None
        self.ego_vehicles = []

        if hasattr(self, 'agent_instance') and self.agent_instance:
            self.agent_instance.destroy()
            self.agent_instance = None

    def _prepare_ego_vehicles(self, ego_vehicles, wait_for_ego_vehicles=False):
        """
        Spawn or update the ego vehicles
        """

        if not wait_for_ego_vehicles:
            for vehicle in ego_vehicles:
                self.ego_vehicles.append(
                    CarlaActorPool.setup_actor(
                        vehicle.model,
                        vehicle.transform,
                        vehicle.rolename,
                        True,
                        color=vehicle.color,
                        vehicle_category=vehicle.category))
        else:
            ego_vehicle_missing = True
            while ego_vehicle_missing:
                self.ego_vehicles = []
                ego_vehicle_missing = False
                for ego_vehicle in ego_vehicles:
                    ego_vehicle_found = False
                    carla_vehicles = CarlaDataProvider.get_world().get_actors(
                    ).filter('vehicle.*')
                    for carla_vehicle in carla_vehicles:
                        if carla_vehicle.attributes[
                                'role_name'] == ego_vehicle.rolename:
                            ego_vehicle_found = True
                            self.ego_vehicles.append(carla_vehicle)
                            break
                    if not ego_vehicle_found:
                        ego_vehicle_missing = True
                        break

            for i, _ in enumerate(self.ego_vehicles):
                self.ego_vehicles[i].set_transform(ego_vehicles[i].transform)

        # sync state
        CarlaDataProvider.get_world().tick()

    def _load_and_wait_for_world(self, args, town, ego_vehicles=None):
        """
        Load a new CARLA world and provide data to CarlaActorPool and CarlaDataProvider
        """

        self.world = self.client.load_world(town)
        settings = self.world.get_settings()
        settings.fixed_delta_seconds = 1.0 / self.frame_rate
        settings.synchronous_mode = True
        self.world.apply_settings(settings)

        CarlaActorPool.set_client(self.client)
        CarlaActorPool.set_world(self.world)
        CarlaDataProvider.set_world(self.world)

        spectator = CarlaDataProvider.get_world().get_spectator()
        spectator.set_transform(
            carla.Transform(carla.Location(x=0, y=0, z=20),
                            carla.Rotation(pitch=-90)))

        # Wait for the world to be ready
        if self.world.get_settings().synchronous_mode:
            self.world.tick()
        else:
            self.world.wait_for_tick()

        if CarlaDataProvider.get_map().name != town:
            print("The CARLA server uses the wrong map!")
            print("This scenario requires to use map {}".format(town))
            return False

        return True

    def _load_and_run_scenario(self, args, config):
        """
        Load and run the scenario given by config
        """

        if not self._load_and_wait_for_world(args, config.town,
                                             config.ego_vehicles):
            self._cleanup()
            return

        agent_class_name = getattr(self.module_agent, 'get_entry_point')()
        try:
            self.agent_instance = getattr(self.module_agent,
                                          agent_class_name)(args.agent_config)
            config.agent = self.agent_instance
            self.sensors = [
                sensors_to_icons[sensor['type']]
                for sensor in self.agent_instance.sensors()
            ]
        except Exception as e:
            print("Could not setup required agent due to {}".format(e))
            self._cleanup()
            return

        # Prepare scenario
        print("Preparing scenario: " + config.name)

        try:
            self._prepare_ego_vehicles(config.ego_vehicles, False)
            scenario = RouteScenario(world=self.world,
                                     config=config,
                                     debug_mode=args.debug)

        except Exception as exception:
            print("The scenario cannot be loaded")
            if args.debug:
                traceback.print_exc()
            print(exception)
            self._cleanup()
            return

        # Set the appropriate weather conditions
        weather = carla.WeatherParameters(
            cloudiness=config.weather.cloudiness,
            precipitation=config.weather.precipitation,
            precipitation_deposits=config.weather.precipitation_deposits,
            wind_intensity=config.weather.wind_intensity,
            sun_azimuth_angle=config.weather.sun_azimuth_angle,
            sun_altitude_angle=config.weather.sun_altitude_angle,
            fog_density=config.weather.fog_density,
            fog_distance=config.weather.fog_distance,
            wetness=config.weather.wetness)

        self.world.set_weather(weather)

        # Set the appropriate road friction
        if config.friction is not None:
            friction_bp = self.world.get_blueprint_library().find(
                'static.trigger.friction')
            extent = carla.Location(1000000.0, 1000000.0, 1000000.0)
            friction_bp.set_attribute('friction', str(config.friction))
            friction_bp.set_attribute('extent_x', str(extent.x))
            friction_bp.set_attribute('extent_y', str(extent.y))
            friction_bp.set_attribute('extent_z', str(extent.z))

            # Spawn Trigger Friction
            transform = carla.Transform()
            transform.location = carla.Location(-10000.0, -10000.0, 0.0)
            self.world.spawn_actor(friction_bp, transform)

        try:
            # Load scenario and run it
            if args.record:
                self.client.start_recorder("{}/{}.log".format(
                    args.record, config.name))
            self.manager.load_scenario(scenario, self.agent_instance)
            self.statistics_manager.set_route(config.name, config.index,
                                              scenario.scenario)
            self.manager.run_scenario()

            # Stop scenario
            self.manager.stop_scenario()

            # register statistics
            current_stats_record = self.statistics_manager.compute_route_statistics(
                config, self.manager.scenario_duration_system,
                self.manager.scenario_duration_game)
            # save
            self.statistics_manager.save_record(current_stats_record,
                                                config.index, args.checkpoint)

            # Remove all actors
            scenario.remove_all_actors()

            settings = self.world.get_settings()
            settings.synchronous_mode = False
            settings.fixed_delta_seconds = None
            self.world.apply_settings(settings)

        except SensorConfigurationInvalid as e:
            self._cleanup(True)
            sys.exit(-1)
        except Exception as e:
            if args.debug:
                traceback.print_exc()
            print(e)

        self._cleanup()

    def run(self, args):
        """
        Run the challenge mode
        """
        route_indexer = RouteIndexer(args.routes, args.scenarios,
                                     args.repetitions)
        if args.resume:
            route_indexer.resume(args.checkpoint)
            self.statistics_manager.resume(args.checkpoint)
        else:
            self.statistics_manager.clear_record(args.checkpoint)
        while route_indexer.peek():
            # setup
            config = route_indexer.next()

            # run
            self._load_and_run_scenario(args, config)
            self._cleanup(ego=True)

            route_indexer.save_state(args.checkpoint)

        # save global statistics
        global_stats_record = self.statistics_manager.compute_global_statistics(
            route_indexer.total)
        StatisticsManager.save_global_record(global_stats_record, self.sensors,
                                             args.checkpoint)
Esempio n. 3
0
class LeaderboardEvaluator(object):
    """
    TODO: document me!
    """

    ego_vehicles = []

    # Tunable parameters
    client_timeout = 10.0  # in seconds
    wait_for_world = 20.0  # in seconds

    # modification: 20.0 -> 10.0
    frame_rate = 10.0  # in Hz

    def __init__(self,
                 args,
                 statistics_manager,
                 launch_server=False,
                 episode_max_time=10000):
        """
        Setup CARLA client and world
        Setup ScenarioManager
        """

        self.statistics_manager = statistics_manager
        self.sensors = []
        self._vehicle_lights = carla.VehicleLightState.Position | carla.VehicleLightState.LowBeam
        self.episode_max_time = episode_max_time

        # First of all, we need to create the client that will send the requests
        # to the simulator. Here we'll assume the simulator is accepting
        # requests in the localhost at port 2000.

        # This is currently set to be consistent with os.environ['HAS_DISPLAY'].
        # however, it is possible to control them separately.
        if os.environ['HAS_DISPLAY'] == '0':
            os.environ["DISPLAY"] = ''

        gpu = port_to_gpu[args.port]

        # if os.environ.get('SUDO_USER') is not None:
        #     username = os.environ['SUDO_USER']
        # else:
        #     username = os.environ['USER']

        # print('\n'*10, 'username', username, '\n'*10)
        os.environ["CUDA_VISIBLE_DEVICES"] = str(gpu)
        self.cmd_list = shlex.split(
            'sh ../carla_0994_no_rss/CarlaUE4.sh -opengl -carla-rpc-port=' +
            str(args.port) + ' -carla-streaming-port=0')

        if launch_server:
            while is_port_in_use(args.port):
                try:
                    subprocess.run('kill $(lsof -t -i :' + str(args.port) +
                                   ')',
                                   shell=True)
                    print('-' * 20, 'kill server at port', port)
                    time.sleep(2)
                except:
                    continue
            subprocess.Popen(self.cmd_list)
            print('-' * 20, 'start server at port', args.port)
            # 10s is usually enough
            time.sleep(10)

        while True:
            try:
                self.client = carla.Client(args.host, int(args.port))
                break
            except:
                logging.exception("__init__ error")
                traceback.print_exc()

        if args.timeout:
            self.client_timeout = float(args.timeout)
        self.client.set_timeout(self.client_timeout)

        dist = pkg_resources.get_distribution("carla")
        if LooseVersion(dist.version) < LooseVersion('0.9.9'):
            raise ImportError(
                "CARLA version 0.9.9 or newer required. CARLA version found: {}"
                .format(dist))

        # Load agent
        module_name = os.path.basename(args.agent).split('.')[0]
        sys.path.insert(0, os.path.dirname(args.agent))
        self.module_agent = importlib.import_module(module_name)

        # Create the ScenarioManager
        self.manager = ScenarioManager(args.debug, args.sync,
                                       args.challenge_mode, args.track,
                                       self.client_timeout,
                                       self.episode_max_time)

        # Time control for summary purposes
        self._start_time = GameTime.get_time()
        self._end_time = None

        # addition
        parent_folder = args.save_folder
        if not os.path.exists(parent_folder):
            os.mkdir(parent_folder)
        string = pathlib.Path(os.environ['ROUTES']).stem
        current_record_folder = pathlib.Path(parent_folder) / string

        if os.path.exists(str(current_record_folder)):
            import shutil
            shutil.rmtree(current_record_folder)
        current_record_folder.mkdir(exist_ok=False)
        (current_record_folder / 'rgb').mkdir()
        (current_record_folder / 'rgb_left').mkdir()
        (current_record_folder / 'rgb_right').mkdir()
        (current_record_folder / 'topdown').mkdir()
        (current_record_folder / 'rgb_with_car').mkdir()
        # if args.agent == 'leaderboard/team_code/auto_pilot.py':
        #     (current_record_folder / 'topdown').mkdir()

        self.save_path = str(current_record_folder / 'events.txt')

    def __del__(self):
        """
        Cleanup and delete actors, ScenarioManager and CARLA world
        """

        self._cleanup(True)
        if hasattr(self, 'manager') and self.manager:
            del self.manager
        if hasattr(self, 'world') and self.world:
            del self.world

        # addition: manually delete client to avoid RuntimeError: Resource temporarily unavailable
        del self.client

    def _cleanup(self, ego=False):
        """
        Remove and destroy all actors
        """

        self.client.stop_recorder()

        CarlaDataProvider.cleanup()

        for i, _ in enumerate(self.ego_vehicles):
            if self.ego_vehicles[i]:
                if ego:
                    self.ego_vehicles[i].destroy()
                self.ego_vehicles[i] = None
        self.ego_vehicles = []

        if hasattr(self, 'agent_instance') and self.agent_instance:
            self.agent_instance.destroy()
            self.agent_instance = None

    def _prepare_ego_vehicles(self, ego_vehicles, wait_for_ego_vehicles=False):
        """
        Spawn or update the ego vehicles
        """

        if not wait_for_ego_vehicles:
            for vehicle in ego_vehicles:
                self.ego_vehicles.append(
                    CarlaDataProvider.setup_actor(
                        vehicle.model,
                        vehicle.transform,
                        vehicle.rolename,
                        True,
                        color=vehicle.color,
                        vehicle_category=vehicle.category))
        else:
            ego_vehicle_missing = True
            while ego_vehicle_missing:
                self.ego_vehicles = []
                ego_vehicle_missing = False
                for ego_vehicle in ego_vehicles:
                    ego_vehicle_found = False
                    carla_vehicles = CarlaDataProvider.get_world().get_actors(
                    ).filter('vehicle.*')
                    for carla_vehicle in carla_vehicles:
                        if carla_vehicle.attributes[
                                'role_name'] == ego_vehicle.rolename:
                            ego_vehicle_found = True
                            self.ego_vehicles.append(carla_vehicle)
                            break
                    if not ego_vehicle_found:
                        ego_vehicle_missing = True
                        break

            for i, _ in enumerate(self.ego_vehicles):
                self.ego_vehicles[i].set_transform(ego_vehicles[i].transform)

        # sync state
        CarlaDataProvider.get_world().tick()

    def _load_and_wait_for_world(self, args, town, ego_vehicles=None):
        """
        Load a new CARLA world and provide data to CarlaDataProvider and CarlaDataProvider
        """

        while True:
            try:
                self.world = self.client.load_world(town)
                break
            except:
                logging.exception("_load_and_wait_for_world error")
                traceback.print_exc()

                while is_port_in_use(args.port):
                    for proc in process_iter():
                        for conns in proc.connections(kind='inet'):
                            if conns.laddr.port == args.port:
                                proc.send_signal(SIGKILL)
                                print('-' * 500, 'kill server at port',
                                      args.port)
                    time.sleep(2)

                subprocess.Popen(self.cmd_list)
                print('-' * 20, 'start server at port', args.port)
                time.sleep(10)

                self.client = carla.Client(args.host, int(args.port))

        settings = self.world.get_settings()
        settings.fixed_delta_seconds = 1.0 / self.frame_rate
        settings.synchronous_mode = True
        self.world.apply_settings(settings)

        CarlaDataProvider.set_client(self.client)
        CarlaDataProvider.set_world(self.world)
        CarlaDataProvider.set_world(self.world)

        spectator = CarlaDataProvider.get_world().get_spectator()
        spectator.set_transform(
            carla.Transform(carla.Location(x=0, y=0, z=20),
                            carla.Rotation(pitch=-90)))

        # Wait for the world to be ready
        if self.world.get_settings().synchronous_mode:
            self.world.tick()
        else:
            self.world.wait_for_tick()

        if CarlaDataProvider.get_map().name != town:
            print("The CARLA server uses the wrong map!")
            print("This scenario requires to use map {}".format(town))
            return False

        return True

    def _load_and_run_scenario(self, args, config, customized_data):
        """
        Load and run the scenario given by config
        """
        # hack:
        config.weather = WEATHERS[args.weather_index]
        config.friction = customized_data['friction']
        config.cur_server_port = customized_data['port']

        if not self._load_and_wait_for_world(args, config.town,
                                             config.ego_vehicles):
            self._cleanup()
            return

        _, route = interpolate_trajectory(self.world, config.trajectory)
        customized_data['center_transform'] = route[int(len(route) // 2)][0]
        '''
        customized non-default center transforms for actors
        ['waypoint_ratio', 'absolute_location']
        '''
        for k, v in customized_data['customized_center_transforms'].items():
            if v[0] == 'waypoint_ratio':
                ind = np.min([int(len(route) * v[1]), len(route) - 1])
                loc = route[ind][0].location
                customized_data[k] = create_transform(loc.x, loc.y, 0, 0, 0, 0)
            elif v[0] == 'absolute_location':
                customized_data[k] = create_transform(v[1], v[2], 0, 0, 0, 0)
            else:
                print('unknown key', k)

        if 'weather_index' in customized_data:
            print('-' * 100)
            print('port :', customized_data['port'])
            print('center_transform :', '(',
                  customized_data['center_transform'].location.x,
                  customized_data['center_transform'].location.y, ')')
            print('friction :', customized_data['friction'])
            print('weather_index :', customized_data['weather_index'])
            print('num_of_static :', customized_data['num_of_static'])
            print('num_of_pedestrians :',
                  customized_data['num_of_pedestrians'])
            print('num_of_vehicles :', customized_data['num_of_vehicles'])
            print('-' * 100)

        agent_class_name = getattr(self.module_agent, 'get_entry_point')()
        try:
            self.agent_instance = getattr(self.module_agent,
                                          agent_class_name)(args.agent_config)

            # addition
            # self.agent_instance.set_trajectory(config.trajectory)
            self.agent_instance.set_deviations_path(args.deviations_folder)

            config.agent = self.agent_instance
            self.sensors = [
                sensors_to_icons[sensor['type']]
                for sensor in self.agent_instance.sensors()
            ]
        except Exception as e:
            print("Could not setup required agent due to {}".format(e))
            traceback.print_exc()
            self._cleanup()
            return

        # Prepare scenario
        print("Preparing scenario: " + config.name)

        try:
            self._prepare_ego_vehicles(config.ego_vehicles, False)
            # print('\n'*10, 'RouteScenario config.cur_server_port', config.cur_server_port, '\n'*10)
            scenario = RouteScenario(world=self.world,
                                     config=config,
                                     debug_mode=args.debug,
                                     customized_data=customized_data)

        except Exception as exception:
            print("The scenario cannot be loaded")
            if args.debug:
                traceback.print_exc()
            print(exception)
            self._cleanup()
            return

        # Set the appropriate weather conditions
        weather = carla.WeatherParameters(
            cloudiness=config.weather.cloudiness,
            precipitation=config.weather.precipitation,
            precipitation_deposits=config.weather.precipitation_deposits,
            wind_intensity=config.weather.wind_intensity,
            sun_azimuth_angle=config.weather.sun_azimuth_angle,
            sun_altitude_angle=config.weather.sun_altitude_angle,
            fog_density=config.weather.fog_density,
            fog_distance=config.weather.fog_distance,
            wetness=config.weather.wetness)

        self.world.set_weather(weather)

        # Set the appropriate road friction
        if config.friction is not None:
            friction_bp = self.world.get_blueprint_library().find(
                'static.trigger.friction')
            extent = carla.Location(1000000.0, 1000000.0, 1000000.0)
            friction_bp.set_attribute('friction', str(config.friction))
            friction_bp.set_attribute('extent_x', str(extent.x))
            friction_bp.set_attribute('extent_y', str(extent.y))
            friction_bp.set_attribute('extent_z', str(extent.z))

            # Spawn Trigger Friction
            transform = carla.Transform()
            transform.location = carla.Location(-10000.0, -10000.0, 0.0)
            self.world.spawn_actor(friction_bp, transform)

        # night mode
        if config.weather.sun_altitude_angle < 0.0:
            for vehicle in scenario.ego_vehicles:
                vehicle.set_light_state(
                    carla.VehicleLightState(self._vehicle_lights))
            # addition: to turn on lights of
            actor_list = self.world.get_actors()
            vehicle_list = actor_list.filter('*vehicle*')
            for vehicle in vehicle_list:
                vehicle.set_light_state(
                    carla.VehicleLightState(self._vehicle_lights))

        try:
            # Load scenario and run it
            if args.record:
                self.client.start_recorder("{}/{}.log".format(
                    args.record, config.name))
            self.manager.load_scenario(scenario, self.agent_instance)
            self.statistics_manager.set_route(config.name, config.index,
                                              scenario.scenario)
            print('start to run scenario')
            self.manager.run_scenario()
            print('stop to run scanario')
            # Stop scenario
            self.manager.stop_scenario()
            # register statistics
            current_stats_record = self.statistics_manager.compute_route_statistics(
                config, self.manager.scenario_duration_system,
                self.manager.scenario_duration_game)
            # save
            # modification

            self.statistics_manager.save_record(current_stats_record,
                                                config.index, self.save_path)

            # Remove all actors
            scenario.remove_all_actors()

            settings = self.world.get_settings()
            settings.synchronous_mode = False
            settings.fixed_delta_seconds = None
            self.world.apply_settings(settings)
        except SensorConfigurationInvalid as e:
            self._cleanup(True)
            sys.exit(-1)
        except Exception as e:
            if args.debug:
                traceback.print_exc()
            print(e)

        self._cleanup()

    def run(self, args, customized_data):
        """
        Run the challenge mode
        """
        route_indexer = RouteIndexer(args.routes, args.scenarios,
                                     args.repetitions)
        if args.resume:
            route_indexer.resume(self.save_path)
            self.statistics_manager.resume(self.save_path)
        else:
            self.statistics_manager.clear_record(self.save_path)
        while route_indexer.peek():
            # setup
            config = route_indexer.next()
            # run
            self._load_and_run_scenario(args, config, customized_data)
            self._cleanup(ego=True)

            route_indexer.save_state(self.save_path)
        # save global statistics
        # modification
        global_stats_record = self.statistics_manager.compute_global_statistics(
            route_indexer.total)
        StatisticsManager.save_global_record(global_stats_record, self.sensors,
                                             self.save_path)
Esempio n. 4
0
class NoCrashEvaluator(object):
    """
    TODO: document me!
    """

    ego_vehicles = []

    # Tunable parameters
    client_timeout = 10.0  # in seconds
    wait_for_world = 20.0  # in seconds
    frame_rate = 20.0  # in Hz

    def __init__(self, args, statistics_manager):
        """
        Setup CARLA client and world
        Setup ScenarioManager
        """
        self.statistics_manager = statistics_manager
        self.sensors = None
        self.sensor_icons = []

        # First of all, we need to create the client that will send the requests
        # to the simulator. Here we'll assume the simulator is accepting
        # requests in the localhost at port 2000.
        self.client = carla.Client(args.host, int(args.port))
        if args.timeout:
            self.client_timeout = float(args.timeout)
        self.client.set_timeout(self.client_timeout)

        self.traffic_manager = self.client.get_trafficmanager(
            int(args.trafficManagerPort))

        dist = pkg_resources.get_distribution("carla")
        if dist.version != 'leaderboard':
            if LooseVersion(dist.version) < LooseVersion('0.9.10'):
                raise ImportError(
                    "CARLA version 0.9.10.1 or newer required. CARLA version found: {}"
                    .format(dist))

        # Load agent
        module_name = os.path.basename(args.agent).split('.')[0]
        sys.path.insert(0, os.path.dirname(args.agent))
        self.module_agent = importlib.import_module(module_name)

        # Create the ScenarioManager
        self.manager = ScenarioManager(args.timeout, args.debug > 1)

        # Time control for summary purposes
        self._start_time = GameTime.get_time()
        self._end_time = None

        # Create the agent timer
        self._agent_watchdog = Watchdog(int(float(args.timeout)))
        signal.signal(signal.SIGINT, self._signal_handler)

        self.town = args.town

    def _signal_handler(self, signum, frame):
        """
        Terminate scenario ticking when receiving a signal interrupt
        """
        if self._agent_watchdog and not self._agent_watchdog.get_status():
            raise RuntimeError("Timeout: Agent took too long to setup")
        elif self.manager:
            self.manager.signal_handler(signum, frame)

    def __del__(self):
        """
        Cleanup and delete actors, ScenarioManager and CARLA world
        """

        self._cleanup()
        if hasattr(self, 'manager') and self.manager:
            del self.manager
        if hasattr(self, 'world') and self.world:
            del self.world

    def _cleanup(self):
        """
        Remove and destroy all actors
        """

        # Simulation still running and in synchronous mode?
        if self.manager and self.manager.get_running_status() \
                and hasattr(self, 'world') and self.world:
            # Reset to asynchronous mode
            settings = self.world.get_settings()
            settings.synchronous_mode = False
            settings.fixed_delta_seconds = None
            self.world.apply_settings(settings)
            self.traffic_manager.set_synchronous_mode(False)

        if self.manager:
            self.manager.cleanup()

        CarlaDataProvider.cleanup()

        for i, _ in enumerate(self.ego_vehicles):
            if self.ego_vehicles[i]:
                self.ego_vehicles[i].destroy()
                self.ego_vehicles[i] = None
        self.ego_vehicles = []

        if self._agent_watchdog:
            self._agent_watchdog.stop()

        if hasattr(self, 'agent_instance') and self.agent_instance:
            self.agent_instance.destroy()
            self.agent_instance = None

        if hasattr(self, 'statistics_manager') and self.statistics_manager:
            self.statistics_manager.scenario = None

    def _load_and_wait_for_world(self, args):
        """
        Load a new CARLA world and provide data to CarlaDataProvider
        """

        self.world = self.client.load_world(args.town)
        settings = self.world.get_settings()
        settings.fixed_delta_seconds = 1.0 / self.frame_rate
        settings.synchronous_mode = True
        self.world.apply_settings(settings)

        self.world.reset_all_traffic_lights()

        CarlaDataProvider.set_client(self.client)
        CarlaDataProvider.set_world(self.world)
        CarlaDataProvider.set_traffic_manager_port(int(
            args.trafficManagerPort))

        self.traffic_manager.set_synchronous_mode(True)
        self.traffic_manager.set_random_device_seed(
            int(args.trafficManagerSeed))

        # Wait for the world to be ready
        if CarlaDataProvider.is_sync_mode():
            self.world.tick()
        else:
            self.world.wait_for_tick()

        if CarlaDataProvider.get_map().name != args.town:
            raise Exception("The CARLA server uses the wrong map!"
                            "This scenario requires to use map {}".format(
                                args.town))

    # def _register_statistics(self, config, checkpoint, entry_status, crash_message=""):
    #     """
    #     Computes and saved the simulation statistics
    #     """
    #     # register statistics
    #     current_stats_record = self.statistics_manager.compute_route_statistics(
    #         config,
    #         self.manager.scenario_duration_system,
    #         self.manager.scenario_duration_game,
    #         crash_message
    #     )

    #     print("\033[1m> Registering the route statistics\033[0m")
    #     self.statistics_manager.save_record(current_stats_record, config.index, checkpoint)
    #     self.statistics_manager.save_entry_status(entry_status, False, checkpoint)

    def _load_and_run_scenario(self, args, route, weather_idx, traffic_idx):
        """
        Load and run the scenario given by config.

        Depending on what code fails, the simulation will either stop the route and
        continue from the next one, or report a crash and stop.
        """
        crash_message = ""
        entry_status = "Started"

        start_idx, target_idx = route
        traffic_lvl = ['Empty', 'Regular', 'Dense'][traffic_idx]

        print(
            "\n\033[1m========= Preparing {} {}: {} to {}, weather {} ========="
            .format(args.town, traffic_lvl, start_idx, target_idx,
                    weather_idx))
        print("> Setting up the agent\033[0m")

        # Set up the user's agent, and the timer to avoid freezing the simulation
        try:
            self._agent_watchdog.start()
            agent_class_name = getattr(self.module_agent, 'get_entry_point')()
            self.agent_instance = getattr(self.module_agent,
                                          agent_class_name)(args.agent_config)

            # Check and store the sensors
            if not self.sensors:
                self.sensors = self.agent_instance.sensors()
                track = self.agent_instance.track

                AgentWrapper.validate_sensor_configuration(
                    self.sensors, track, args.track)

            self._agent_watchdog.stop()

        except SensorConfigurationInvalid as e:
            # The sensors are invalid -> set the ejecution to rejected and stop
            print("\n\033[91mThe sensor's configuration used is invalid:")
            print("> {}\033[0m\n".format(e))
            traceback.print_exc()

            crash_message = "Agent's sensors were invalid"
            entry_status = "Rejected"

            self._cleanup()
            sys.exit(-1)

        except Exception as e:
            # The agent setup has failed -> start the next route
            print("\n\033[91mCould not set up the required agent:")
            print("> {}\033[0m\n".format(e))
            traceback.print_exc()

            crash_message = "Agent couldn't be set up"

            self._cleanup()
            return

        print("\033[1m> Loading the world\033[0m")

        # Load the world and the scenario
        try:
            self._load_and_wait_for_world(args)
            scenario = NoCrashEvalScenario(world=self.world,
                                           agent=self.agent_instance,
                                           start_idx=start_idx,
                                           target_idx=target_idx,
                                           weather_idx=weather_idx,
                                           traffic_idx=traffic_idx,
                                           debug_mode=args.debug)

            self.manager.load_scenario(scenario, self.agent_instance, 0)

        except Exception as e:
            # The scenario is wrong -> set the ejecution to crashed and stop
            print("\n\033[91mThe scenario could not be loaded:")
            print("> {}\033[0m\n".format(e))
            traceback.print_exc()

            crash_message = "Simulation crashed"
            entry_status = "Crashed"

            if args.record:
                self.client.stop_recorder()

            self._cleanup()
            sys.exit(-1)

        print("\033[1m> Running the route\033[0m")

        # Run the scenario
        try:
            self.manager.run_scenario()

        except AgentError as e:
            # The agent has failed -> stop the route
            print("\n\033[91mStopping the route, the agent has crashed:")
            print("> {}\033[0m\n".format(e))
            traceback.print_exc()

            crash_message = "Agent crashed"

        except Exception as e:
            print("\n\033[91mError during the simulation:")
            print("> {}\033[0m\n".format(e))
            traceback.print_exc()

            crash_message = "Simulation crashed"
            entry_status = "Crashed"

        # Stop the scenario
        try:
            print("\033[1m> Stopping the route\033[0m")
            self.manager.stop_scenario()

            route_completion, lights_ran, duration = self.manager.get_nocrash_diagnostics(
            )
            self.statistics_manager.log(self.town, traffic_idx, weather_idx,
                                        start_idx, target_idx,
                                        route_completion, lights_ran, duration)

            if args.record:
                self.client.stop_recorder()

            # Remove all actors
            scenario.remove_all_actors()

            self._cleanup()

        except Exception as e:
            print(
                "\n\033[91mFailed to stop the scenario, the statistics might be empty:"
            )
            print("> {}\033[0m\n".format(e))
            traceback.print_exc()

            crash_message = "Simulation crashed"

        if crash_message == "Simulation crashed":
            sys.exit(-1)

    def run(self, args):
        """
        Run the challenge mode
        """

        # if args.resume:
        # route_indexer.resume(args.checkpoint)
        # self.statistics_manager.resume(args.checkpoint)
        # else:
        # self.statistics_manager.clear_record(args.checkpoint)
        # route_indexer.save_state(args.checkpoint)

        # Load routes
        with open(f'runners/suite/nocrash_{args.town}.txt', 'r') as f:
            routes = [tuple(map(int, l.split())) for l in f.readlines()]

        weathers = {'train': [1, 3, 6, 8], 'test': [10, 14]}.get(args.weather)
        traffics = [0, 1, 2]

        for traffic, route, weather in itertools.product(
                traffics, routes, weathers):
            if self.statistics_manager.is_finished(self.town, route, weather,
                                                   traffic):
                continue

            self._load_and_run_scenario(args, route, weather, traffic)

        # save global statistics
        print("\033[1m> Registering the global statistics\033[0m")
Esempio n. 5
0
class LeaderboardEvaluator(object):
    """
    TODO: document me!
    """

    ego_vehicles = []

    # Tunable parameters
    client_timeout = 20.0  # in seconds
    wait_for_world = 20.0  # in seconds

    # modification: 20.0 -> 10.0
    frame_rate = 10.0  # in Hz

    def __init__(self,
                 args,
                 statistics_manager,
                 launch_server=False,
                 episode_max_time=10000):
        """
        Setup CARLA client and world
        Setup ScenarioManager
        """

        self.statistics_manager = statistics_manager
        self.sensors = []
        self._vehicle_lights = (carla.VehicleLightState.Position
                                | carla.VehicleLightState.LowBeam)
        self.episode_max_time = episode_max_time

        # First of all, we need to create the client that will send the requests
        # to the simulator.

        # This is currently set to be consistent with os.environ['HAS_DISPLAY'].
        # however, it is possible to control them separately.
        if os.environ["HAS_DISPLAY"] == "0":
            os.environ["DISPLAY"] = ""

        gpu = port_to_gpu(int(args.port))
        os.environ["CUDA_VISIBLE_DEVICES"] = str(gpu)

        if launch_server:
            start_server(args.port)
        start_client(self, args.host, int(args.port))

        if args.timeout:
            self.client_timeout = float(args.timeout)
        self.client.set_timeout(self.client_timeout)

        dist = pkg_resources.get_distribution("carla")
        if LooseVersion(dist.version) < LooseVersion("0.9.9"):
            raise ImportError(
                "CARLA version 0.9.9 or newer required. CARLA version found: {}"
                .format(dist))

        # Load agent
        module_name = os.path.basename(args.agent).split(".")[0]
        sys.path.insert(0, os.path.dirname(args.agent))
        self.module_agent = importlib.import_module(module_name)

        # Create the ScenarioManager
        self.manager = ScenarioManager(
            args.debug,
            args.sync,
            args.challenge_mode,
            args.track,
            self.client_timeout,
            self.episode_max_time,
        )

        # Time control for summary purposes
        self._start_time = GameTime.get_time()
        self._end_time = None

        # addition
        parent_folder = args.save_folder
        if not os.path.exists(parent_folder):
            os.mkdir(parent_folder)
        string = pathlib.Path(os.environ["ROUTES"]).stem
        current_record_folder = pathlib.Path(parent_folder) / string

        if os.path.exists(str(current_record_folder)):
            import shutil

            shutil.rmtree(current_record_folder)
        current_record_folder.mkdir(exist_ok=False)
        (current_record_folder / "rgb").mkdir()
        (current_record_folder / "rgb_left").mkdir()
        (current_record_folder / "rgb_right").mkdir()
        (current_record_folder / "topdown").mkdir()
        (current_record_folder / "rgb_with_car").mkdir()
        # if args.agent == 'leaderboard/team_code/auto_pilot.py':
        #     (current_record_folder / 'topdown').mkdir()

        self.save_path = str(current_record_folder / "events.txt")

    def __del__(self):
        """
        Cleanup and delete actors, ScenarioManager and CARLA world
        """

        self._cleanup(True)
        if hasattr(self, "manager") and self.manager:
            del self.manager
        if hasattr(self, "world") and self.world:
            del self.world

        # addition: manually delete client to avoid RuntimeError: Resource temporarily unavailable
        del self.client

    def _cleanup(self, ego=False):
        """
        Remove and destroy all actors
        """

        self.client.stop_recorder()

        CarlaDataProvider.cleanup()

        for i, _ in enumerate(self.ego_vehicles):
            if self.ego_vehicles[i]:
                if ego:
                    self.ego_vehicles[i].destroy()
                self.ego_vehicles[i] = None
        self.ego_vehicles = []

        if hasattr(self, "agent_instance") and self.agent_instance:
            self.agent_instance.destroy()
            self.agent_instance = None

    def _prepare_ego_vehicles(self, ego_vehicles, wait_for_ego_vehicles=False):
        """
        Spawn or update the ego vehicles
        """

        if not wait_for_ego_vehicles:
            for vehicle in ego_vehicles:
                self.ego_vehicles.append(
                    CarlaDataProvider.setup_actor(
                        vehicle.model,
                        vehicle.transform,
                        vehicle.rolename,
                        True,
                        color=vehicle.color,
                        vehicle_category=vehicle.category,
                    ))
        else:
            ego_vehicle_missing = True
            while ego_vehicle_missing:
                self.ego_vehicles = []
                ego_vehicle_missing = False
                for ego_vehicle in ego_vehicles:
                    ego_vehicle_found = False
                    carla_vehicles = (CarlaDataProvider.get_world().get_actors(
                    ).filter("vehicle.*"))
                    for carla_vehicle in carla_vehicles:
                        if (carla_vehicle.attributes["role_name"] ==
                                ego_vehicle.rolename):
                            ego_vehicle_found = True
                            self.ego_vehicles.append(carla_vehicle)
                            break
                    if not ego_vehicle_found:
                        ego_vehicle_missing = True
                        break

            for i, _ in enumerate(self.ego_vehicles):
                self.ego_vehicles[i].set_transform(ego_vehicles[i].transform)

        # sync state
        CarlaDataProvider.get_world().tick()

    def _load_and_wait_for_world(self, args, town, ego_vehicles=None):
        """
        Load a new CARLA world and provide data to CarlaDataProvider and CarlaDataProvider
        """

        try_load_world(self, town, args.host, int(args.port))
        settings = self.world.get_settings()
        settings.fixed_delta_seconds = 1.0 / self.frame_rate
        settings.synchronous_mode = True

        self.world.apply_settings(settings)

        CarlaDataProvider.set_client(self.client)
        CarlaDataProvider.set_world(self.world)

        spectator = CarlaDataProvider.get_world().get_spectator()
        spectator.set_transform(
            carla.Transform(carla.Location(x=0, y=0, z=20),
                            carla.Rotation(pitch=-90)))

        # Wait for the world to be ready
        if self.world.get_settings().synchronous_mode:
            self.world.tick()
        else:
            self.world.wait_for_tick()

        if CarlaDataProvider.get_map().name != town:
            print("The CARLA server uses the wrong map!")
            print("This scenario requires to use map {}".format(town))
            return False

        return True

    def _load_and_run_scenario(self, args, config, customized_data):
        """
        Load and run the scenario given by config
        """
        # hack:
        if args.weather_index == -1:
            weather = customized_data["fine_grained_weather"]
        else:
            weather = WEATHERS[args.weather_index]

        config.weather = weather
        config.friction = customized_data["friction"]
        config.cur_server_port = customized_data["port"]

        if not self._load_and_wait_for_world(args, config.town,
                                             config.ego_vehicles):
            self._cleanup()
            return

        _, route = interpolate_trajectory(self.world, config.trajectory)
        customized_data["center_transform"] = route[int(len(route) // 2)][0]

        # from customized_utils import visualize_route
        # visualize_route(route)
        # transforms = []
        # for x in route:
        #     transforms.append(x[0])
        # print('len(transforms)', len(transforms))

        if customized_data['using_customized_route_and_scenario']:
            """
            customized non-default center transforms for actors
            ['waypoint_ratio', 'absolute_location']
            """
            for k, v in customized_data["customized_center_transforms"].items(
            ):
                if v[0] == "waypoint_ratio":
                    r = v[1] / 100
                    ind = np.min([int(len(route) * r), len(route) - 1])
                    loc = route[ind][0].location
                    customized_data[k] = create_transform(
                        loc.x, loc.y, 0, 0, 0, 0)
                    print("waypoint_ratio", loc.x, loc.y)
                elif v[0] == "absolute_location":
                    customized_data[k] = create_transform(
                        v[1], v[2], 0, 0, 0, 0)
                else:
                    print("unknown key", k)

            print("-" * 100)
            print("port :", customized_data["port"])
            print(
                "center_transform :",
                "(",
                customized_data["center_transform"].location.x,
                customized_data["center_transform"].location.y,
                ")",
            )
            print("friction :", customized_data["friction"])
            print("weather_index :", customized_data["weather_index"])
            print("num_of_static :", customized_data["num_of_static"])
            print("num_of_pedestrians :",
                  customized_data["num_of_pedestrians"])
            print("num_of_vehicles :", customized_data["num_of_vehicles"])
            print("-" * 100)

        agent_class_name = getattr(self.module_agent, "get_entry_point")()
        try:
            self.agent_instance = getattr(self.module_agent,
                                          agent_class_name)(args.agent_config)

            # addition
            # self.agent_instance.set_trajectory(config.trajectory)
            self.agent_instance.set_args(args)

            config.agent = self.agent_instance
            self.sensors = [
                sensors_to_icons[sensor["type"]]
                for sensor in self.agent_instance.sensors()
            ]
        except Exception as e:
            print("Could not setup required agent due to {}".format(e))
            traceback.print_exc()
            self._cleanup()
            return

        # Prepare scenario
        print("Preparing scenario: " + config.name)

        try:
            self._prepare_ego_vehicles(config.ego_vehicles, False)
            # print('\n'*10, 'RouteScenario config.cur_server_port', config.cur_server_port, '\n'*10)
            scenario = RouteScenario(
                world=self.world,
                config=config,
                debug_mode=args.debug,
                customized_data=customized_data,
            )

        except Exception as exception:
            print("The scenario cannot be loaded")
            if args.debug:
                traceback.print_exc()
            print(exception)
            self._cleanup()
            return

        # Set the appropriate weather conditions
        weather = carla.WeatherParameters(
            cloudiness=config.weather.cloudiness,
            precipitation=config.weather.precipitation,
            precipitation_deposits=config.weather.precipitation_deposits,
            wind_intensity=config.weather.wind_intensity,
            sun_azimuth_angle=config.weather.sun_azimuth_angle,
            sun_altitude_angle=config.weather.sun_altitude_angle,
            fog_density=config.weather.fog_density,
            fog_distance=config.weather.fog_distance,
            wetness=config.weather.wetness,
        )

        self.world.set_weather(weather)

        # Set the appropriate road friction
        if config.friction is not None:
            friction_bp = self.world.get_blueprint_library().find(
                "static.trigger.friction")
            extent = carla.Location(1000000.0, 1000000.0, 1000000.0)
            friction_bp.set_attribute("friction", str(config.friction))
            friction_bp.set_attribute("extent_x", str(extent.x))
            friction_bp.set_attribute("extent_y", str(extent.y))
            friction_bp.set_attribute("extent_z", str(extent.z))

            # Spawn Trigger Friction
            transform = carla.Transform()
            transform.location = carla.Location(-10000.0, -10000.0, 0.0)
            self.world.spawn_actor(friction_bp, transform)

        # night mode
        if config.weather.sun_altitude_angle < 0.0:
            for vehicle in scenario.ego_vehicles:
                vehicle.set_light_state(
                    carla.VehicleLightState(self._vehicle_lights))
            # addition: to turn on lights of
            actor_list = self.world.get_actors()
            vehicle_list = actor_list.filter("*vehicle*")
            for vehicle in vehicle_list:
                vehicle.set_light_state(
                    carla.VehicleLightState(self._vehicle_lights))

        try:
            # Load scenario and run it
            if args.record:
                self.client.start_recorder("{}/{}.log".format(
                    args.record, config.name))
            self.manager.load_scenario(scenario, self.agent_instance)
            self.statistics_manager.set_route(config.name, config.index,
                                              scenario.scenario)
            print("start to run scenario")
            self.manager.run_scenario()
            print("stop to run scanario")
            # Stop scenario
            self.manager.stop_scenario()
            # register statistics
            current_stats_record = self.statistics_manager.compute_route_statistics(
                config,
                self.manager.scenario_duration_system,
                self.manager.scenario_duration_game,
            )
            # save
            # modification

            self.statistics_manager.save_record(current_stats_record,
                                                config.index, self.save_path)

            # Remove all actors
            scenario.remove_all_actors()

            settings = self.world.get_settings()
            settings.synchronous_mode = False
            settings.fixed_delta_seconds = None
            self.world.apply_settings(settings)
        except SensorConfigurationInvalid as e:
            self._cleanup(True)
            sys.exit(-1)
        except Exception as e:
            if args.debug:
                traceback.print_exc()
            print(e)

        self._cleanup()

    def run(self, args, customized_data):
        """
        Run the challenge mode
        """
        route_indexer = RouteIndexer(args.routes, args.scenarios,
                                     args.repetitions,
                                     args.background_vehicles)
        if args.resume:
            route_indexer.resume(self.save_path)
            self.statistics_manager.resume(self.save_path)
        else:
            self.statistics_manager.clear_record(self.save_path)
        while route_indexer.peek():
            # setup
            config = route_indexer.next()
            # run
            self._load_and_run_scenario(args, config, customized_data)
            self._cleanup(ego=True)

            route_indexer.save_state(self.save_path)
        # save global statistics
        # modification
        global_stats_record = self.statistics_manager.compute_global_statistics(
            route_indexer.total)
        StatisticsManager.save_global_record(global_stats_record, self.sensors,
                                             self.save_path)