def run_scenario(self):
"""
Trigger the start of the scenario and wait for it to finish/fail
"""
print("ScenarioManager: Running scenario {}".format(
self.scenario_tree.name))
self.start_system_time = time.time()
start_game_time = GameTime.get_time()
self._watchdog = Watchdog(float(self._timeout))
self._watchdog.start()
self._running = True
while self._running:
timestamp = None
world = CarlaDataProvider.get_world()
if world:
snapshot = world.get_snapshot()
if snapshot:
timestamp = snapshot.timestamp
if timestamp:
self._tick_scenario(timestamp)
self.cleanup()
self.end_system_time = time.time()
end_game_time = GameTime.get_time()
self.scenario_duration_system = self.end_system_time - \
self.start_system_time
self.scenario_duration_game = end_game_time - start_game_time
if self.scenario_tree.status == py_trees.common.Status.FAILURE:
print("ScenarioManager: Terminated due to failure")
def __init__(self, debug_mode=False, challenge_mode=False, track=None, timeout=20.0):
"""
Init requires scenario as input
"""
self.scenario = None
self.scenario_tree = None
self.scenario_class = None
self.ego_vehicles = None
self.other_actors = None
self._debug_mode = debug_mode
self._challenge_mode = challenge_mode
self._track = track
self._agent = None
self._running = False
self._timestamp_last_run = 0.0
self._timeout = timeout
self._watchdog = Watchdog(float(self._timeout))
self.scenario_duration_system = 0.0
self.scenario_duration_game = 0.0
self.start_system_time = None
self.end_system_time = None
nvidia_smi.nvmlInit()
self.handle = nvidia_smi.nvmlDeviceGetHandleByIndex(1)
# Register the scenario tick as callback for the CARLA world
# Use the callback_id inside the signal handler to allow external interrupts
signal.signal(signal.SIGINT, self._signal_handler)
def run_scenario(self):
"""
Trigger the start of the scenario and wait for it to finish/fail
"""
self.start_system_time = time.time()
self.start_game_time = GameTime.get_time()
# Detects if the simulation is down
self._watchdog = Watchdog(self._timeout)
self._watchdog.start()
# Stop the agent from freezing the simulation
self._agent_watchdog = Watchdog(self._timeout)
self._agent_watchdog.start()
self._running = True
while self._running:
timestamp = None
world = CarlaDataProvider.get_world()
if world:
snapshot = world.get_snapshot()
if snapshot:
timestamp = snapshot.timestamp
if timestamp:
self._tick_scenario(timestamp)
def __init__(self, timeout, debug_mode=False):
"""
Setups up the parameters, which will be filled at load_scenario()
"""
self.scenario = None
self.scenario_tree = None
self.scenario_class = None
self.ego_vehicles = None
self.other_actors = None
self._debug_mode = debug_mode
self._agent = None
self._running = False
self._timestamp_last_run = 0.0
self._timeout = float(timeout)
# Used to detect if the simulation is down
watchdog_timeout = max(5, self._timeout - 2)
self._watchdog = Watchdog(watchdog_timeout)
# Avoid the agent from freezing the simulation
agent_timeout = watchdog_timeout - 1
self._agent_watchdog = Watchdog(agent_timeout)
self.scenario_duration_system = 0.0
self.scenario_duration_game = 0.0
self.start_system_time = None
self.end_system_time = None
self.end_game_time = None
# Register the scenario tick as callback for the CARLA world
# Use the callback_id inside the signal handler to allow external interrupts
signal.signal(signal.SIGINT, self.signal_handler)
def __init__(self, debug_mode=False, challenge_mode=False):
"""
Setups up the parameters, which will be filled at load_scenario()
"""
self.scenario = None
self.scenario_tree = None
self.scenario_class = None
self.ego_vehicles = None
self.other_actors = None
self._debug_mode = debug_mode
self._challenge_mode = challenge_mode
self._agent = None
self._running = False
self._timestamp_last_run = 0.0
# Used to detect the simulation is down, but doesn't create any exception
self._watchdog = Watchdog(5)
self.scenario_duration_system = 0.0
self.scenario_duration_game = 0.0
self.start_system_time = None
self.end_system_time = None
self.end_game_time = None
# Register the scenario tick as callback for the CARLA world
# Use the callback_id inside the signal handler to allow external interrupts
signal.signal(signal.SIGINT, self._signal_handler)
def __init__(self, args, statistics_manager):
"""
Setup CARLA client and world
Setup ScenarioManager
"""
self.statistics_manager = statistics_manager
self.sensors = None
self.sensor_icons = []
self._vehicle_lights = carla.VehicleLightState.Position | carla.VehicleLightState.LowBeam
# First of all, we need to create the client that will send the requests
# to the simulator. Here we'll assume the simulator is accepting
# requests in the localhost at port 2000.
self.client = carla.Client(args.host, int(args.port))
if args.timeout:
self.client_timeout = float(args.timeout)
self.client.set_timeout(self.client_timeout)
self.traffic_manager = self.client.get_trafficmanager(
int(args.trafficManagerPort))
dist = pkg_resources.get_distribution("carla")
if dist.version != 'leaderboard':
if LooseVersion(dist.version) < LooseVersion('0.9.10'):
raise ImportError(
"CARLA version 0.9.10.1 or newer required. CARLA version found: {}"
.format(dist))
# Load agent
module_name = os.path.basename(args.agent).split('.')[0]
sys.path.insert(0, os.path.dirname(args.agent))
self.module_agent = importlib.import_module(module_name)
# Create the ScenarioManager
self.manager = ScenarioManager(args.timeout, args.debug > 1)
# Time control for summary purposes
self._start_time = GameTime.get_time()
self._end_time = None
# Create the agent timer
self._agent_watchdog = Watchdog(int(float(args.timeout)))
signal.signal(signal.SIGINT, self._signal_handler)
# Set scenario class
self._scenario_class = {
'route_scenario': RouteScenario,
'train_scenario': TrainScenario,
'nocrash_train_scenario': NoCrashTrainScenario,
'nocrash_eval_scenario': NoCrashEvalScenario,
}.get(args.scenario_class)
if args.scenario_class == 'train_scenario':
self.town = args.town
else:
self.town = None
def __init__(self, timeout, debug_mode=False, timely_commands=False, frame_rate=20):
"""
Setups up the parameters, which will be filled at load_scenario()
"""
self.timely_commands = timely_commands
self.frame_rate = frame_rate
self.scenario = None
self.scenario_tree = None
self.scenario_class = None
self.ego_vehicles = None
self.other_actors = None
self._debug_mode = debug_mode
self._agent = None
self._running = False
self._timestamp_last_run = 0.0
self._timeout = float(timeout)
# Used to detect if the simulation is down
watchdog_timeout = max(5, self._timeout - 2)
self._watchdog = Watchdog(watchdog_timeout)
# Avoid the agent from freezing the simulation
agent_timeout = watchdog_timeout - 1
self._agent_watchdog = Watchdog(agent_timeout)
self.scenario_duration_system = 0.0
self.scenario_duration_game = 0.0
self.start_system_time = None
self.end_system_time = None
self.end_game_time = None
# Register the scenario tick as callback for the CARLA world
# Use the callback_id inside the signal handler to allow external interrupts
signal.signal(signal.SIGINT, self.signal_handler)
self._delayed_cmds = []
brake_control = carla.VehicleControl()
brake_control.throttle = 0
brake_control.brake = 1
brake_control.steer = 0
brake_control.reverse = False
brake_control.hand_brake = False
brake_control.manual_gear_shift = False
self._last_cmd = (0, 0, brake_control)
def __init__(self, debug_mode=False, timeout=2.0):
"""
Init requires scenario as input
"""
self.scenario = None
self.scenario_tree = None
self.scenario_class = None
self.ego_vehicles = None
self.other_actors = None
self._debug_mode = debug_mode
self._agent = None
self._running = False
self._timestamp_last_run = 0.0
self._timeout = timeout
self._watchdog = Watchdog(float(self._timeout))
self.scenario_duration_system = 0.0
self.scenario_duration_game = 0.0
self.start_system_time = None
self.end_system_time = None
def __init__(self, debug_mode=False, timeout=2.0):
"""
Setups up the parameters, which will be filled at load_scenario()
"""
self.scenario = None
self.scenario_tree = None
self.scenario_class = None
self.ego_vehicles = None
self.other_actors = None
self._debug_mode = debug_mode
self._agent = None
self._running = False
self._timestamp_last_run = 0.0
self._timeout = timeout
self._watchdog = Watchdog(float(self._timeout))
self.scenario_duration_system = 0.0
self.scenario_duration_game = 0.0
self.start_system_time = None
self.end_system_time = None
def __init__(self,
debug_mode=False,
sync_mode=False,
challenge_mode=False,
track=None,
timeout=10.0,
episode_max_time=10000):
"""
Setups up the parameters, which will be filled at load_scenario()
"""
self.scenario = None
self.scenario_tree = None
self.scenario_class = None
self.ego_vehicles = None
self.other_actors = None
self._debug_mode = debug_mode
self._challenge_mode = challenge_mode
self._track = track
self._agent = None
self._sync_mode = sync_mode
self._running = False
self._timestamp_last_run = 0.0
self._timeout = timeout
self._watchdog = Watchdog(float(self._timeout))
self.scenario_duration_system = 0.0
self.scenario_duration_game = 0.0
self.start_system_time = None
self.end_system_time = None
# modification: disable when running multi-process via dask
# Register the scenario tick as callback for the CARLA world
# Use the callback_id inside the signal handler to allow external interrupts
# signal.signal(signal.SIGINT, self._signal_handler)
self.episode_max_time = episode_max_time
class ScenarioManager(object):
"""
Basic scenario manager class. This class holds all functionality
required to start, and analyze a scenario.
The user must not modify this class.
To use the ScenarioManager:
1. Create an object via manager = ScenarioManager()
2. Load a scenario via manager.load_scenario()
3. Trigger the execution of the scenario manager.run_scenario()
This function is designed to explicitly control start and end of
the scenario execution
4. Trigger a result evaluation with manager.analyze_scenario()
5. If needed, cleanup with manager.stop_scenario()
"""
def __init__(self, debug_mode=False, sync_mode=False, timeout=2.0):
"""
Setups up the parameters, which will be filled at load_scenario()
"""
self.scenario = None
self.scenario_tree = None
self.scenario_class = None
self.ego_vehicles = None
self.other_actors = None
self._debug_mode = debug_mode
self._agent = None
self._sync_mode = sync_mode
self._running = False
self._timestamp_last_run = 0.0
self._timeout = timeout
self._watchdog = Watchdog(float(self._timeout))
self.scenario_duration_system = 0.0
self.scenario_duration_game = 0.0
self.start_system_time = None
self.end_system_time = None
def _reset(self):
"""
Reset all parameters
"""
self._running = False
self._timestamp_last_run = 0.0
self.scenario_duration_system = 0.0
self.scenario_duration_game = 0.0
self.start_system_time = None
self.end_system_time = None
GameTime.restart()
def cleanup(self):
"""
This function triggers a proper termination of a scenario
"""
if self.scenario is not None:
self.scenario.terminate()
if self._agent is not None:
self._agent.cleanup()
self._agent = None
CarlaDataProvider.cleanup()
def load_scenario(self, scenario, agent=None):
"""
Load a new scenario
"""
self._reset()
self._agent = AgentWrapper(agent) if agent else None
if self._agent is not None:
self._sync_mode = True
self.scenario_class = scenario
self.scenario = scenario.scenario
self.scenario_tree = self.scenario.scenario_tree
self.ego_vehicles = scenario.ego_vehicles
self.other_actors = scenario.other_actors
# To print the scenario tree uncomment the next line
# py_trees.display.render_dot_tree(self.scenario_tree)
if self._agent is not None:
self._agent.setup_sensors(self.ego_vehicles[0], self._debug_mode)
def run_scenario(self):
"""
Trigger the start of the scenario and wait for it to finish/fail
"""
print("ScenarioManager: Running scenario {}".format(
self.scenario_tree.name))
self.start_system_time = time.time()
start_game_time = GameTime.get_time()
self._watchdog.start()
self._running = True
while self._running:
timestamp = None
world = CarlaDataProvider.get_world()
if world:
snapshot = world.get_snapshot()
if snapshot:
timestamp = snapshot.timestamp
if timestamp:
self._tick_scenario(timestamp)
self._watchdog.stop()
self.cleanup()
self.end_system_time = time.time()
end_game_time = GameTime.get_time()
self.scenario_duration_system = self.end_system_time - \
self.start_system_time
self.scenario_duration_game = end_game_time - start_game_time
if self.scenario_tree.status == py_trees.common.Status.FAILURE:
print("ScenarioManager: Terminated due to failure")
def _tick_scenario(self, timestamp):
"""
Run next tick of scenario and the agent.
If running synchornously, it also handles the ticking of the world.
"""
if self._timestamp_last_run < timestamp.elapsed_seconds and self._running:
self._timestamp_last_run = timestamp.elapsed_seconds
self._watchdog.update()
if self._debug_mode:
print("\n--------- Tick ---------\n")
# Update game time and actor information
GameTime.on_carla_tick(timestamp)
CarlaDataProvider.on_carla_tick()
if self._agent is not None:
ego_action = self._agent()
# Tick scenario
self.scenario_tree.tick_once()
if self._debug_mode:
print("\n")
py_trees.display.print_ascii_tree(self.scenario_tree,
show_status=True)
sys.stdout.flush()
if self.scenario_tree.status != py_trees.common.Status.RUNNING:
self._running = False
if self._agent is not None:
self.ego_vehicles[0].apply_control(ego_action)
if self._sync_mode and self._running and self._watchdog.get_status():
CarlaDataProvider.get_world().tick()
def get_running_status(self):
"""
returns:
bool: False if watchdog exception occured, True otherwise
"""
return self._watchdog.get_status()
def stop_scenario(self):
"""
This function is used by the overall signal handler to terminate the scenario execution
"""
self._running = False
def analyze_scenario(self, stdout, filename, junit):
"""
This function is intended to be called from outside and provide
the final statistics about the scenario (human-readable, in form of a junit
report, etc.)
"""
failure = False
timeout = False
result = "SUCCESS"
if self.scenario.test_criteria is None:
print("Nothing to analyze, this scenario has no criteria")
return True
for criterion in self.scenario.get_criteria():
if (not criterion.optional and criterion.test_status != "SUCCESS"
and criterion.test_status != "ACCEPTABLE"):
failure = True
result = "FAILURE"
elif criterion.test_status == "ACCEPTABLE":
result = "ACCEPTABLE"
if self.scenario.timeout_node.timeout and not failure:
timeout = True
result = "TIMEOUT"
output = ResultOutputProvider(self, result, stdout, filename, junit)
output.write()
return failure or timeout
class LeaderboardEvaluator(object):
"""
TODO: document me!
"""
ego_vehicles = []
# Tunable parameters
client_timeout = 10.0 # in seconds
wait_for_world = 20.0 # in seconds
frame_rate = 20.0 # in Hz
def __init__(self, args, statistics_manager):
"""
Setup CARLA client and world
Setup ScenarioManager
"""
self.statistics_manager = statistics_manager
self.sensors = None
self.sensor_icons = []
self._vehicle_lights = carla.VehicleLightState.Position | carla.VehicleLightState.LowBeam
# First of all, we need to create the client that will send the requests
# to the simulator. Here we'll assume the simulator is accepting
# requests in the localhost at port 2000.
self.client = carla.Client(args.host, args.port)
if args.timeout:
self.client_timeout = args.timeout
self.client.set_timeout(self.client_timeout)
self.traffic_manager = self.client.get_trafficmanager(args.traffic_manager_port)
dist = pkg_resources.get_distribution("carla")
if dist.version != 'leaderboard':
if LooseVersion(dist.version) < LooseVersion('0.9.10'):
raise ImportError("CARLA version 0.9.10.1 or newer required. CARLA version found: {}".format(dist))
# Load agent
module_name = os.path.basename(args.agent).split('.')[0]
sys.path.insert(0, os.path.dirname(args.agent))
self.module_agent = importlib.import_module(module_name)
# Create the ScenarioManager
self.manager = ScenarioManager(args.timeout, args.debug > 1)
# Time control for summary purposes
self._start_time = GameTime.get_time()
self._end_time = None
# Create the agent timer
self._agent_watchdog = Watchdog(args.timeout)
signal.signal(signal.SIGINT, self._signal_handler)
def _signal_handler(self, signum, frame):
"""
Terminate scenario ticking when receiving a signal interrupt
"""
if self._agent_watchdog and not self._agent_watchdog.get_status():
raise RuntimeError("Timeout: Agent took longer than {}s to setup".format(self.client_timeout))
elif self.manager:
self.manager.signal_handler(signum, frame)
def __del__(self):
"""
Cleanup and delete actors, ScenarioManager and CARLA world
"""
self._cleanup()
if hasattr(self, 'manager') and self.manager:
del self.manager
if hasattr(self, 'world') and self.world:
del self.world
def _cleanup(self):
"""
Remove and destroy all actors
"""
# Simulation still running and in synchronous mode?
if self.manager and self.manager.get_running_status() \
and hasattr(self, 'world') and self.world:
# Reset to asynchronous mode
settings = self.world.get_settings()
settings.synchronous_mode = False
settings.fixed_delta_seconds = None
self.world.apply_settings(settings)
self.traffic_manager.set_synchronous_mode(False)
if self.manager:
self.manager.cleanup()
CarlaDataProvider.cleanup()
for i, _ in enumerate(self.ego_vehicles):
if self.ego_vehicles[i]:
self.ego_vehicles[i].destroy()
self.ego_vehicles[i] = None
self.ego_vehicles = []
if self._agent_watchdog:
self._agent_watchdog.stop()
if hasattr(self, 'agent_instance') and self.agent_instance:
self.agent_instance.destroy()
self.agent_instance = None
if hasattr(self, 'statistics_manager') and self.statistics_manager:
self.statistics_manager.scenario = None
def _prepare_ego_vehicles(self, ego_vehicles, wait_for_ego_vehicles=False):
"""
Spawn or update the ego vehicles
"""
if not wait_for_ego_vehicles:
for vehicle in ego_vehicles:
self.ego_vehicles.append(CarlaDataProvider.request_new_actor(vehicle.model,
vehicle.transform,
vehicle.rolename,
color=vehicle.color,
vehicle_category=vehicle.category))
else:
ego_vehicle_missing = True
while ego_vehicle_missing:
self.ego_vehicles = []
ego_vehicle_missing = False
for ego_vehicle in ego_vehicles:
ego_vehicle_found = False
carla_vehicles = CarlaDataProvider.get_world().get_actors().filter('vehicle.*')
for carla_vehicle in carla_vehicles:
if carla_vehicle.attributes['role_name'] == ego_vehicle.rolename:
ego_vehicle_found = True
self.ego_vehicles.append(carla_vehicle)
break
if not ego_vehicle_found:
ego_vehicle_missing = True
break
for i, _ in enumerate(self.ego_vehicles):
self.ego_vehicles[i].set_transform(ego_vehicles[i].transform)
# sync state
CarlaDataProvider.get_world().tick()
def _load_and_wait_for_world(self, args, town, ego_vehicles=None):
"""
Load a new CARLA world and provide data to CarlaDataProvider
"""
self.world = self.client.load_world(town)
settings = self.world.get_settings()
settings.fixed_delta_seconds = 1.0 / self.frame_rate
settings.synchronous_mode = True
self.world.apply_settings(settings)
self.world.reset_all_traffic_lights()
CarlaDataProvider.set_client(self.client)
CarlaDataProvider.set_world(self.world)
CarlaDataProvider.set_traffic_manager_port(args.traffic_manager_port)
self.traffic_manager.set_synchronous_mode(True)
self.traffic_manager.set_random_device_seed(args.traffic_manager_seed)
self.traffic_manager.set_hybrid_physics_mode(True)
# Wait for the world to be ready
if CarlaDataProvider.is_sync_mode():
self.world.tick()
else:
self.world.wait_for_tick()
if CarlaDataProvider.get_map().name != town:
raise Exception("The CARLA server uses the wrong map!"
"This scenario requires to use map {}".format(town))
def _register_statistics(self, config, checkpoint, entry_status, crash_message=""):
"""
Computes and saved the simulation statistics
"""
# register statistics
current_stats_record = self.statistics_manager.compute_route_statistics(
config,
self.manager.scenario_duration_system,
self.manager.scenario_duration_game,
crash_message
)
print("\033[1m> Registering the route statistics\033[0m")
self.statistics_manager.save_record(current_stats_record, config.index, checkpoint)
self.statistics_manager.save_entry_status(entry_status, False, checkpoint)
def _load_and_run_scenario(self, args, config):
"""
Load and run the scenario given by config.
Depending on what code fails, the simulation will either stop the route and
continue from the next one, or report a crash and stop.
"""
crash_message = ""
entry_status = "Started"
print("\n\033[1m========= Preparing {} (repetition {}) =========".format(config.name, config.repetition_index))
print("> Setting up the agent\033[0m")
# Prepare the statistics of the route
self.statistics_manager.set_route(config.name, config.index)
# Set up the user's agent, and the timer to avoid freezing the simulation
try:
self._agent_watchdog.start()
agent_class_name = getattr(self.module_agent, 'get_entry_point')()
self.agent_instance = getattr(self.module_agent, agent_class_name)(args.agent_config)
config.agent = self.agent_instance
# Check and store the sensors
if not self.sensors:
self.sensors = self.agent_instance.sensors()
track = self.agent_instance.track
AgentWrapper.validate_sensor_configuration(self.sensors, track, args.track)
self.sensor_icons = [sensors_to_icons[sensor['type']] for sensor in self.sensors]
self.statistics_manager.save_sensors(self.sensor_icons, args.checkpoint)
self._agent_watchdog.stop()
except SensorConfigurationInvalid as e:
# The sensors are invalid -> set the ejecution to rejected and stop
print("\n\033[91mThe sensor's configuration used is invalid:")
print("> {}\033[0m\n".format(e))
traceback.print_exc()
crash_message = "Agent's sensors were invalid"
entry_status = "Rejected"
self._register_statistics(config, args.checkpoint, entry_status, crash_message)
self._cleanup()
sys.exit(-1)
except Exception as e:
# The agent setup has failed -> start the next route
print("\n\033[91mCould not set up the required agent:")
print("> {}\033[0m\n".format(e))
traceback.print_exc()
crash_message = "Agent couldn't be set up"
self._register_statistics(config, args.checkpoint, entry_status, crash_message)
self._cleanup()
return
print("\033[1m> Loading the world\033[0m")
# Load the world and the scenario
try:
self._load_and_wait_for_world(args, config.town, config.ego_vehicles)
self._prepare_ego_vehicles(config.ego_vehicles, False)
scenario = RouteScenario(world=self.world, config=config, debug_mode=args.debug)
self.statistics_manager.set_scenario(scenario.scenario)
# Night mode
if config.weather.sun_altitude_angle < 0.0:
for vehicle in scenario.ego_vehicles:
vehicle.set_light_state(carla.VehicleLightState(self._vehicle_lights))
# Load scenario and run it
if args.record:
self.client.start_recorder("{}/{}_rep{}.log".format(args.record, config.name, config.repetition_index))
self.manager.load_scenario(scenario, self.agent_instance, config.repetition_index)
except Exception as e:
# The scenario is wrong -> set the ejecution to crashed and stop
print("\n\033[91mThe scenario could not be loaded:")
print("> {}\033[0m\n".format(e))
traceback.print_exc()
crash_message = "Simulation crashed"
entry_status = "Crashed"
self._register_statistics(config, args.checkpoint, entry_status, crash_message)
if args.record:
self.client.stop_recorder()
self._cleanup()
sys.exit(-1)
print("\033[1m> Running the route\033[0m")
# Run the scenario
try:
self.manager.run_scenario()
except AgentError as e:
# The agent has failed -> stop the route
print("\n\033[91mStopping the route, the agent has crashed:")
print("> {}\033[0m\n".format(e))
traceback.print_exc()
crash_message = "Agent crashed"
except Exception as e:
print("\n\033[91mError during the simulation:")
print("> {}\033[0m\n".format(e))
traceback.print_exc()
crash_message = "Simulation crashed"
entry_status = "Crashed"
# Stop the scenario
try:
print("\033[1m> Stopping the route\033[0m")
self.manager.stop_scenario()
self._register_statistics(config, args.checkpoint, entry_status, crash_message)
if args.record:
self.client.stop_recorder()
# Remove all actors
scenario.remove_all_actors()
self._cleanup()
except Exception as e:
print("\n\033[91mFailed to stop the scenario, the statistics might be empty:")
print("> {}\033[0m\n".format(e))
traceback.print_exc()
crash_message = "Simulation crashed"
if crash_message == "Simulation crashed":
sys.exit(-1)
def run(self, args):
"""
Run the challenge mode
"""
route_indexer = RouteIndexer(args.routes, args.scenarios, args.repetitions)
if args.resume:
route_indexer.resume(args.checkpoint)
self.statistics_manager.resume(args.checkpoint)
else:
self.statistics_manager.clear_record(args.checkpoint)
route_indexer.save_state(args.checkpoint)
while route_indexer.peek():
# setup
config = route_indexer.next()
# run
self._load_and_run_scenario(args, config)
route_indexer.save_state(args.checkpoint)
# save global statistics
print("\033[1m> Registering the global statistics\033[0m")
global_stats_record = self.statistics_manager.compute_global_statistics(route_indexer.total)
StatisticsManager.save_global_record(global_stats_record, self.sensor_icons, route_indexer.total, args.checkpoint)
class ScenarioManager(object):
"""
Basic scenario manager class. This class holds all functionality
required to start, and analyze a scenario.
The user must not modify this class.
To use the ScenarioManager:
1. Create an object via manager = ScenarioManager()
2. Load a scenario via manager.load_scenario()
3. Trigger the execution of the scenario manager.run_scenario()
This function is designed to explicitly control start and end of
the scenario execution
4. Trigger a result evaluation with manager.analyze_scenario()
5. If needed, cleanup with manager.stop_scenario()
"""
def __init__(self,
debug_mode=False,
sync_mode=False,
challenge_mode=False,
track=None,
timeout=10.0,
episode_max_time=10000):
"""
Setups up the parameters, which will be filled at load_scenario()
"""
self.scenario = None
self.scenario_tree = None
self.scenario_class = None
self.ego_vehicles = None
self.other_actors = None
self._debug_mode = debug_mode
self._challenge_mode = challenge_mode
self._track = track
self._agent = None
self._sync_mode = sync_mode
self._running = False
self._timestamp_last_run = 0.0
self._timeout = timeout
self._watchdog = Watchdog(float(self._timeout))
self.scenario_duration_system = 0.0
self.scenario_duration_game = 0.0
self.start_system_time = None
self.end_system_time = None
# modification: disable when running multi-process via dask
# Register the scenario tick as callback for the CARLA world
# Use the callback_id inside the signal handler to allow external interrupts
# signal.signal(signal.SIGINT, self._signal_handler)
self.episode_max_time = episode_max_time
def _signal_handler(self, signum, frame):
"""
Terminate scenario ticking when receiving a signal interrupt
"""
self._running = False
if not self.get_running_status():
raise RuntimeError("Timeout occured during scenario execution")
def _reset(self):
"""
Reset all parameters
"""
self._running = False
self._timestamp_last_run = 0.0
self.scenario_duration_system = 0.0
self.scenario_duration_game = 0.0
self.start_system_time = None
self.end_system_time = None
GameTime.restart()
def load_scenario(self, scenario, agent=None):
"""
Load a new scenario
"""
self._reset()
self._agent = AgentWrapper(agent,
self._challenge_mode) if agent else None
if self._agent is not None:
self._sync_mode = True
self.scenario_class = scenario
self.scenario = scenario.scenario
self.scenario_tree = self.scenario.scenario_tree
self.ego_vehicles = scenario.ego_vehicles
self.other_actors = scenario.other_actors
# modification registration now moves to carla_data_provider
# CarlaDataProvider.register_actors(self.ego_vehicles)
# CarlaDataProvider.register_actors(self.other_actors)
# To print the scenario tree uncomment the next line
# py_trees.display.render_dot_tree(self.scenario_tree)
if self._agent is not None:
self._agent.setup_sensors(self.ego_vehicles[0], self._debug_mode,
self._track)
def run_scenario(self):
"""
Trigger the start of the scenario and wait for it to finish/fail
"""
print("ScenarioManager: Running scenario {}".format(
self.scenario_tree.name))
self.start_system_time = time.time()
start_game_time = GameTime.get_time()
self._watchdog.start()
self._running = True
parent_folder = os.environ['SAVE_FOLDER']
string = pathlib.Path(os.environ['ROUTES']).stem
save_path = pathlib.Path(parent_folder) / string
step = 0
# hack: control the time of running
# debug:
s = time.time()
with (save_path / 'measurements_loc.csv').open("a") as f_out:
while self._running:
timestamp = None
world = CarlaDataProvider.get_world()
if world:
snapshot = world.get_snapshot()
if snapshot:
timestamp = snapshot.timestamp
if timestamp:
self._tick_scenario(timestamp)
# addition
if step == 0:
f_out.write('x,y,yaw\n')
# if step % 10 == 0:
loc = self.ego_vehicles[0].get_location()
ori = self.ego_vehicles[0].get_transform().rotation
f_out.write(
str(loc.x) + ',' + str(loc.y) + ',' + str(ori.yaw) + '\n')
step += 1
# hack: control the time of running
# debug:
if time.time() - s > self.episode_max_time:
print('maximum time reaches, end running')
self._running = False
break
self._watchdog.stop()
self.end_system_time = time.time()
end_game_time = GameTime.get_time()
self.scenario_duration_system = self.end_system_time - \
self.start_system_time
self.scenario_duration_game = end_game_time - start_game_time
self._console_message()
def _console_message(self):
"""
Message that will be displayed via console
"""
def get_symbol(value, desired_value, high=True):
"""
Returns a tick or a cross depending on the values
"""
tick = '\033[92m' + 'O' + '\033[0m'
cross = '\033[91m' + 'X' + '\033[0m'
multiplier = 1 if high else -1
if multiplier * value >= desired_value:
symbol = tick
else:
symbol = cross
return symbol
if self.scenario_tree.status == py_trees.common.Status.RUNNING:
# If still running, all the following is None, so no point continuing
print(
"\n> Something happened during the simulation. Was it manually shutdown?\n"
)
return
blackv = py_trees.blackboard.Blackboard()
route_completed = blackv.get("RouteCompletion")
collisions = blackv.get("Collision")
outside_route_lanes = blackv.get("OutsideRouteLanes")
stop_signs = blackv.get("RunningStop")
red_light = blackv.get("RunningRedLight")
in_route = blackv.get("InRoute")
# addition: new event
on_side_walk = blackv.get("OnSideWalk")
off_road = blackv.get("OffRoad")
wrong_lane = blackv.get("WrongLane")
# If something failed, stop
if [
x for x in (collisions, outside_route_lanes, stop_signs,
red_light, in_route) if x is None
]:
print("Nothing to analyze, this scenario has no criteria")
return
if blackv.get("RouteCompletion") >= 99:
route_completed = 100
else:
route_completed = blackv.get("RouteCompletion")
outside_route_lanes = float(outside_route_lanes)
route_symbol = get_symbol(route_completed, 100, True)
collision_symbol = get_symbol(collisions, 0, False)
outside_symbol = get_symbol(outside_route_lanes, 0, False)
red_light_symbol = get_symbol(red_light, 0, False)
stop_symbol = get_symbol(stop_signs, 0, False)
# addition: new event
on_side_walk_symbol = get_symbol(on_side_walk, 0, False)
off_road_symbol = get_symbol(off_road, 0, False)
wrong_lane_symbol = get_symbol(wrong_lane, 0, False)
if self.scenario_tree.status == py_trees.common.Status.FAILURE:
if not in_route:
message = "> FAILED: The actor deviated from the route"
else:
message = "> FAILED: The actor didn't finish the route"
elif self.scenario_tree.status == py_trees.common.Status.SUCCESS:
if route_completed == 100:
message = "> SUCCESS: Congratulations, route finished! "
else:
message = "> FAILED: The actor timed out "
else: # This should never be triggered
return
if self.scenario_tree.status != py_trees.common.Status.RUNNING:
print("\n" + message)
print("> ")
print("> Score: ")
print("> - Route Completed [{}]: {}%".format(
route_symbol, route_completed))
print("> - Outside route lanes [{}]: {}%".format(
outside_symbol, outside_route_lanes))
print("> - Collisions [{}]: {} times".format(
collision_symbol, collisions))
print("> - Red lights run [{}]: {} times".format(
red_light_symbol, red_light))
print("> - Stop signs run [{}]: {} times".format(
stop_symbol, stop_signs))
# addition: new event
print("> - On side walk [{}]: {} times".format(
on_side_walk_symbol, on_side_walk))
print("> - Off road [{}]: {} times".format(
off_road_symbol, off_road))
print("> - Wrong lane [{}]: {} times\n".format(
wrong_lane_symbol, wrong_lane))
def _tick_scenario(self, timestamp):
"""
Run next tick of scenario and the agent.
If running synchornously, it also handles the ticking of the world.
"""
if self._timestamp_last_run < timestamp.elapsed_seconds:
self._timestamp_last_run = timestamp.elapsed_seconds
self._watchdog.update()
# Update game time and actor information
GameTime.on_carla_tick(timestamp)
CarlaDataProvider.on_carla_tick()
if self._agent is not None:
ego_action = self._agent()
# Tick scenario
self.scenario_tree.tick_once()
if self._debug_mode > 1:
print("\n")
py_trees.display.print_ascii_tree(self.scenario_tree,
show_status=True)
sys.stdout.flush()
if self.scenario_tree.status != py_trees.common.Status.RUNNING:
self._running = False
if self._challenge_mode:
spectator = CarlaDataProvider.get_world().get_spectator()
ego_trans = self.ego_vehicles[0].get_transform()
spectator.set_transform(
carla.Transform(ego_trans.location + carla.Location(z=50),
carla.Rotation(pitch=-90)))
if self._agent is not None:
self.ego_vehicles[0].apply_control(ego_action)
if self._sync_mode and self._running and self._watchdog.get_status():
CarlaDataProvider.get_world().tick()
def get_running_status(self):
"""
returns:
bool: False if watchdog exception occured, True otherwise
"""
return self._watchdog.get_status()
def stop_scenario(self):
"""
This function triggers a proper termination of a scenario
"""
if self.scenario is not None:
self.scenario.terminate()
if self._agent is not None:
self._agent.cleanup()
self._agent = None
CarlaDataProvider.cleanup()
def _load_and_run_scenario(self, args, config):
"""
Load and run the scenario given by config.
Depending on what code fails, the simulation will either stop the route and
continue from the next one, or report a crash and stop.
"""
crash_message = ""
entry_status = "Started"
print(
"\n\033[1m========= Preparing {} (repetition {}) =========".format(
config.name, config.repetition_index))
print("> Setting up the agent\033[0m")
# Prepare the statistics of the route
self.statistics_manager.set_route(config.name, config.index)
# Set up the user's agent, and the timer to avoid freezing the simulation
try:
self._agent_watchdog = Watchdog(args.timeout)
self._agent_watchdog.start()
agent_class_name = getattr(self.module_agent, 'get_entry_point')()
self.agent_instance = getattr(self.module_agent,
agent_class_name)(args.agent_config)
config.agent = self.agent_instance
# Check and store the sensors
if not self.sensors:
self.sensors = self.agent_instance.sensors()
track = self.agent_instance.track
AgentWrapper.validate_sensor_configuration(
self.sensors, track, args.track)
self.sensor_icons = [
sensors_to_icons[sensor['type']] for sensor in self.sensors
]
self.statistics_manager.save_sensors(self.sensor_icons,
args.checkpoint)
self._agent_watchdog.stop()
self._agent_watchdog = None
except SensorConfigurationInvalid as e:
# The sensors are invalid -> set the ejecution to rejected and stop
print("\n\033[91mThe sensor's configuration used is invalid:")
print("> {}\033[0m\n".format(e))
traceback.print_exc()
crash_message = "Agent's sensors were invalid"
entry_status = "Rejected"
self._register_statistics(config, args.checkpoint, entry_status,
crash_message)
self._cleanup()
sys.exit(-1)
except Exception as e:
# The agent setup has failed -> start the next route
print("\n\033[91mCould not set up the required agent:")
print("> {}\033[0m\n".format(e))
traceback.print_exc()
crash_message = "Agent couldn't be set up"
self._register_statistics(config, args.checkpoint, entry_status,
crash_message)
self._cleanup()
return
print("\033[1m> Loading the world\033[0m")
# Load the world and the scenario
try:
self._load_and_wait_for_world(args, config.town,
config.ego_vehicles)
self._prepare_ego_vehicles(config.ego_vehicles, False)
scenario = RouteScenario(world=self.world,
config=config,
debug_mode=args.debug)
config.route = scenario.route
self.statistics_manager.set_scenario(scenario.scenario)
# Load scenario and run it
if args.record:
self.client.start_recorder("{}/{}_rep{}.log".format(
args.record, config.name, config.repetition_index))
self.manager.load_scenario(scenario, self.agent_instance,
config.repetition_index)
except Exception as e:
# The scenario is wrong -> set the ejecution to crashed and stop
print("\n\033[91mThe scenario could not be loaded:")
print("> {}\033[0m\n".format(e))
traceback.print_exc()
crash_message = "Simulation crashed"
entry_status = "Crashed"
self._register_statistics(config, args.checkpoint, entry_status,
crash_message)
if args.record:
self.client.stop_recorder()
self._cleanup()
sys.exit(-1)
print("\033[1m> Running the route\033[0m")
# Run the scenario
try:
self.manager.run_scenario()
except AgentError as e:
# The agent has failed -> stop the route
print("\n\033[91mStopping the route, the agent has crashed:")
print("> {}\033[0m\n".format(e))
traceback.print_exc()
crash_message = "Agent crashed"
except Exception as e:
print("\n\033[91mError during the simulation:")
print("> {}\033[0m\n".format(e))
traceback.print_exc()
crash_message = "Simulation crashed"
entry_status = "Crashed"
# Stop the scenario
try:
print("\033[1m> Stopping the route\033[0m")
self.manager.stop_scenario()
self._register_statistics(config, args.checkpoint, entry_status,
crash_message)
if args.record:
self.client.stop_recorder()
# Remove all actors
scenario.remove_all_actors()
self._cleanup()
except Exception as e:
print(
"\n\033[91mFailed to stop the scenario, the statistics might be empty:"
)
print("> {}\033[0m\n".format(e))
traceback.print_exc()
crash_message = "Simulation crashed"
if crash_message == "Simulation crashed":
sys.exit(-1)
class ScenarioManager(object):
"""
Basic scenario manager class. This class holds all functionality
required to start, run and stop a scenario.
The user must not modify this class.
To use the ScenarioManager:
1. Create an object via manager = ScenarioManager()
2. Load a scenario via manager.load_scenario()
3. Trigger the execution of the scenario manager.run_scenario()
This function is designed to explicitly control start and end of
the scenario execution
4. If needed, cleanup with manager.stop_scenario()
"""
def __init__(self, timeout, debug_mode=False, timely_commands=False, frame_rate=20):
"""
Setups up the parameters, which will be filled at load_scenario()
"""
self.timely_commands = timely_commands
self.frame_rate = frame_rate
self.scenario = None
self.scenario_tree = None
self.scenario_class = None
self.ego_vehicles = None
self.other_actors = None
self._debug_mode = debug_mode
self._agent = None
self._running = False
self._timestamp_last_run = 0.0
self._timeout = float(timeout)
# Used to detect if the simulation is down
watchdog_timeout = max(5, self._timeout - 2)
self._watchdog = Watchdog(watchdog_timeout)
# Avoid the agent from freezing the simulation
agent_timeout = watchdog_timeout - 1
self._agent_watchdog = Watchdog(agent_timeout)
self.scenario_duration_system = 0.0
self.scenario_duration_game = 0.0
self.start_system_time = None
self.end_system_time = None
self.end_game_time = None
# Register the scenario tick as callback for the CARLA world
# Use the callback_id inside the signal handler to allow external interrupts
signal.signal(signal.SIGINT, self.signal_handler)
self._delayed_cmds = []
brake_control = carla.VehicleControl()
brake_control.throttle = 0
brake_control.brake = 1
brake_control.steer = 0
brake_control.reverse = False
brake_control.hand_brake = False
brake_control.manual_gear_shift = False
self._last_cmd = (0, 0, brake_control)
def signal_handler(self, signum, frame):
"""
Terminate scenario ticking when receiving a signal interrupt
"""
self._running = False
def cleanup(self):
"""
Reset all parameters
"""
self._timestamp_last_run = 0.0
self.scenario_duration_system = 0.0
self.scenario_duration_game = 0.0
self.start_system_time = None
self.end_system_time = None
self.end_game_time = None
def load_scenario(self, scenario, agent, rep_number):
"""
Load a new scenario
"""
GameTime.restart()
agent.frame_delta_ms = int(1000 / self.frame_rate)
self._agent = AgentWrapper(agent)
self.scenario_class = scenario
self.scenario = scenario.scenario
self.scenario_tree = self.scenario.scenario_tree
self.ego_vehicles = scenario.ego_vehicles
self.other_actors = scenario.other_actors
self.repetition_number = rep_number
# To print the scenario tree uncomment the next line
# py_trees.display.render_dot_tree(self.scenario_tree)
self._agent.setup_sensors(self.ego_vehicles[0], self._debug_mode)
def run_scenario(self):
"""
Trigger the start of the scenario and wait for it to finish/fail
"""
self.start_system_time = time.time()
self.start_game_time = GameTime.get_time()
self._watchdog.start()
self._running = True
while self._running:
timestamp = None
world = CarlaDataProvider.get_world()
if world:
snapshot = world.get_snapshot()
if snapshot:
timestamp = snapshot.timestamp
if timestamp:
self._tick_scenario(timestamp)
def _tick_scenario(self, timestamp):
"""
Run next tick of scenario and the agent and tick the world.
"""
if self._timestamp_last_run < timestamp.elapsed_seconds and self._running:
self._timestamp_last_run = timestamp.elapsed_seconds
self._watchdog.update()
# Update game time and actor information
GameTime.on_carla_tick(timestamp)
CarlaDataProvider.on_carla_tick()
game_time_ms = int(GameTime.get_time() * 1000)
try:
res = self._agent()
if self.timely_commands:
assert isinstance(res, tuple), 'Agent did not return the runtime'
ego_action, runtime = res
else:
if isinstance(res, tuple):
ego_action = res[0]
else:
ego_action = res
runtime = 0
heapq.heappush(self._delayed_cmds,
(game_time_ms + runtime, game_time_ms, ego_action))
# Special exception inside the agent that isn't caused by the agent
except SensorReceivedNoData as e:
raise RuntimeError(e)
except Exception as e:
raise AgentError(e)
while len(self._delayed_cmds) > 0 and game_time_ms >= self._delayed_cmds[0][0]:
self._last_cmd = self._delayed_cmds[0]
heapq.heappop(self._delayed_cmds)
print("Command at {} using {}".format(self._last_cmd[0], self._last_cmd[1]))
self.ego_vehicles[0].apply_control(self._last_cmd[2])
# Tick scenario
self.scenario_tree.tick_once()
if self._debug_mode:
print("\n")
py_trees.display.print_ascii_tree(
self.scenario_tree, show_status=True)
sys.stdout.flush()
if self.scenario_tree.status != py_trees.common.Status.RUNNING:
self._running = False
spectator = CarlaDataProvider.get_world().get_spectator()
ego_trans = self.ego_vehicles[0].get_transform()
spectator.set_transform(carla.Transform(ego_trans.location + carla.Location(z=50),
carla.Rotation(pitch=-90)))
if self._running and self.get_running_status():
CarlaDataProvider.get_world().tick(self._timeout)
def get_running_status(self):
"""
returns:
bool: False if watchdog exception occured, True otherwise
"""
return self._watchdog.get_status()
def stop_scenario(self):
"""
This function triggers a proper termination of a scenario
"""
self._watchdog.stop()
self.end_system_time = time.time()
self.end_game_time = GameTime.get_time()
self.scenario_duration_system = self.end_system_time - self.start_system_time
self.scenario_duration_game = self.end_game_time - self.start_game_time
if self.get_running_status():
if self.scenario is not None:
self.scenario.terminate()
if self._agent is not None:
self._agent.cleanup()
self._agent = None
self.analyze_scenario()
def analyze_scenario(self):
"""
Analyzes and prints the results of the route
"""
global_result = '\033[92m'+'SUCCESS'+'\033[0m'
for criterion in self.scenario.get_criteria():
if criterion.test_status != "SUCCESS":
global_result = '\033[91m'+'FAILURE'+'\033[0m'
if self.scenario.timeout_node.timeout:
global_result = '\033[91m'+'FAILURE'+'\033[0m'
ResultOutputProvider(self, global_result)
class ScenarioManager(object):
"""
Basic scenario manager class. This class holds all functionality
required to start, run and stop a scenario.
The user must not modify this class.
To use the ScenarioManager:
1. Create an object via manager = ScenarioManager()
2. Load a scenario via manager.load_scenario()
3. Trigger the execution of the scenario manager.run_scenario()
This function is designed to explicitly control start and end of
the scenario execution
4. If needed, cleanup with manager.stop_scenario()
"""
def __init__(self, timeout, debug_mode=False):
"""
Setups up the parameters, which will be filled at load_scenario()
"""
self.scenario = None
self.scenario_tree = None
self.scenario_class = None
self.ego_vehicles = None
self.other_actors = None
self._debug_mode = debug_mode
self._agent = None
self._running = False
self._timestamp_last_run = 0.0
self._timeout = float(timeout)
self._watchdog = Watchdog(
self._timeout) # Detects if the simulation is down
self._agent_watchdog = Watchdog(
self._timeout) # Stop the agent from freezing the simulation
self.scenario_duration_system = 0.0
self.scenario_duration_game = 0.0
self.start_system_time = None
self.end_system_time = None
self.end_game_time = None
# Register the scenario tick as callback for the CARLA world
# Use the callback_id inside the signal handler to allow external interrupts
signal.signal(signal.SIGINT, self.signal_handler)
def signal_handler(self, signum, frame):
"""
Terminate scenario ticking when receiving a signal interrupt
"""
if self._agent_watchdog and not self._agent_watchdog.get_status():
raise RuntimeError(
"Agent took longer than {}s to send its command".format(
self._timeout))
elif self._watchdog and not self._watchdog.get_status():
raise RuntimeError(
"The simulation took longer than {}s to update".format(
self._timeout))
self._running = False
def cleanup(self):
"""
Reset all parameters
"""
self._timestamp_last_run = 0.0
self.scenario_duration_system = 0.0
self.scenario_duration_game = 0.0
self.start_system_time = None
self.end_system_time = None
self.end_game_time = None
self._spectator = None
def load_scenario(self, scenario, agent, rep_number):
"""
Load a new scenario
"""
GameTime.restart()
self._agent = AgentWrapper(agent)
self.scenario_class = scenario
self.scenario = scenario.scenario
self.scenario_tree = self.scenario.scenario_tree
self.ego_vehicles = scenario.ego_vehicles
self.other_actors = scenario.other_actors
self.repetition_number = rep_number
self._spectator = CarlaDataProvider.get_world().get_spectator()
# To print the scenario tree uncomment the next line
# py_trees.display.render_dot_tree(self.scenario_tree)
self._agent.setup_sensors(self.ego_vehicles[0], self._debug_mode)
def run_scenario(self):
"""
Trigger the start of the scenario and wait for it to finish/fail
"""
self.start_system_time = time.time()
self.start_game_time = GameTime.get_time()
self._watchdog.start()
self._agent_watchdog.start()
self._running = True
while self._running:
timestamp = None
world = CarlaDataProvider.get_world()
if world:
snapshot = world.get_snapshot()
if snapshot:
timestamp = snapshot.timestamp
if timestamp:
self._tick_scenario(timestamp)
def _tick_scenario(self, timestamp):
"""
Run next tick of scenario and the agent and tick the world.
"""
if self._timestamp_last_run < timestamp.elapsed_seconds and self._running:
self._timestamp_last_run = timestamp.elapsed_seconds
self._watchdog.update()
# Update game time and actor information
GameTime.on_carla_tick(timestamp)
CarlaDataProvider.on_carla_tick()
self._watchdog.pause()
try:
self._agent_watchdog.resume()
self._agent_watchdog.update()
ego_action = self._agent()
self._agent_watchdog.pause()
# Special exception inside the agent that isn't caused by the agent
except SensorReceivedNoData as e:
raise RuntimeError(e)
except Exception as e:
raise AgentError(e)
self._watchdog.resume()
self.ego_vehicles[0].apply_control(ego_action)
# Tick scenario
self.scenario_tree.tick_once()
if self._debug_mode:
print("\n")
py_trees.display.print_ascii_tree(self.scenario_tree,
show_status=True)
sys.stdout.flush()
if self.scenario_tree.status != py_trees.common.Status.RUNNING:
self._running = False
ego_trans = self.ego_vehicles[0].get_transform()
self._spectator.set_transform(
carla.Transform(ego_trans.location + carla.Location(z=50),
carla.Rotation(pitch=-90)))
if self._running and self.get_running_status():
CarlaDataProvider.get_world().tick(self._timeout)
def get_running_status(self):
"""
returns:
bool: False if watchdog exception occured, True otherwise
"""
return self._watchdog.get_status()
def stop_scenario(self):
"""
This function triggers a proper termination of a scenario
"""
self._watchdog.stop()
self._agent_watchdog.stop()
self.end_system_time = time.time()
self.end_game_time = GameTime.get_time()
self.scenario_duration_system = self.end_system_time - self.start_system_time
self.scenario_duration_game = self.end_game_time - self.start_game_time
if self.get_running_status():
if self.scenario is not None:
self.scenario.terminate()
if self._agent is not None:
self._agent.cleanup()
self._agent = None
self.analyze_scenario()
def analyze_scenario(self):
"""
Analyzes and prints the results of the route
"""
global_result = '\033[92m' + 'SUCCESS' + '\033[0m'
for criterion in self.scenario.get_criteria():
if criterion.test_status != "SUCCESS":
global_result = '\033[91m' + 'FAILURE' + '\033[0m'
if self.scenario.timeout_node.timeout:
global_result = '\033[91m' + 'FAILURE' + '\033[0m'
ResultOutputProvider(self, global_result)
class ScenarioManager(object):
"""
Basic scenario manager class. This class holds all functionality
required to start, and analyze a scenario.
The user must not modify this class.
To use the ScenarioManager:
1. Create an object via manager = ScenarioManager()
2. Load a scenario via manager.load_scenario()
3. Trigger the execution of the scenario manager.execute()
This function is designed to explicitly control start and end of
the scenario execution
4. Trigger a result evaluation with manager.analyze()
5. Cleanup with manager.stop_scenario()
"""
def __init__(self, debug_mode=False, challenge_mode=False, track=None, timeout=20.0):
"""
Init requires scenario as input
"""
self.scenario = None
self.scenario_tree = None
self.scenario_class = None
self.ego_vehicles = None
self.other_actors = None
self._debug_mode = debug_mode
self._challenge_mode = challenge_mode
self._track = track
self._agent = None
self._running = False
self._timestamp_last_run = 0.0
self._timeout = timeout
self._watchdog = Watchdog(float(self._timeout))
self.scenario_duration_system = 0.0
self.scenario_duration_game = 0.0
self.start_system_time = None
self.end_system_time = None
nvidia_smi.nvmlInit()
self.handle = nvidia_smi.nvmlDeviceGetHandleByIndex(1)
# Register the scenario tick as callback for the CARLA world
# Use the callback_id inside the signal handler to allow external interrupts
signal.signal(signal.SIGINT, self._signal_handler)
def _signal_handler(self, signum, frame):
"""
Terminate scenario ticking when receiving a signal interrupt
"""
self._running = False
if not self.get_running_status():
raise RuntimeError("Timeout occured during scenario execution")
def _reset(self):
"""
Reset all parameters
"""
self._running = False
self._timestamp_last_run = 0.0
self.scenario_duration_system = 0.0
self.scenario_duration_game = 0.0
self.start_system_time = None
self.end_system_time = None
GameTime.restart()
def load_scenario(self, scenario, agent=None):
"""
Load a new scenario
"""
self._reset()
self._agent = AgentWrapper(agent, self._challenge_mode) if agent else None
self.scenario_class = scenario
self.scenario = scenario.scenario
self.scenario_tree = self.scenario.scenario_tree
self.ego_vehicles = scenario.ego_vehicles
self.other_actors = scenario.other_actors
#print(self.other_actors)
CarlaDataProvider.register_actors(self.ego_vehicles)
CarlaDataProvider.register_actors(self.other_actors)
# To print the scenario tree uncomment the next line
# py_trees.display.render_dot_tree(self.scenario_tree)
if self._agent is not None:
self._agent.setup_sensors(self.ego_vehicles[0], self._debug_mode, self._track)
def run_scenario(self,path,path1):
"""
Trigger the start of the scenario and wait for it to finish/fail
"""
print("ScenarioManager: Running scenario {}".format(self.scenario_tree.name))
self.start_system_time = time.time()
start_game_time = GameTime.get_time()
total_risk_score = []
self._watchdog.start()
self._running = True
x = 0
while self._running:
timestamp = None
world = CarlaDataProvider.get_world()
if world:
snapshot = world.get_snapshot()
if snapshot:
timestamp = snapshot.timestamp
if timestamp:
self._tick_scenario(timestamp)
x+=1
if(x%60==1):
res = nvidia_smi.nvmlDeviceGetUtilizationRates(self.handle)
mem_res = nvidia_smi.nvmlDeviceGetMemoryInfo(self.handle)
#print(f'mem: {mem_res.used / (1024**2)} (GiB)') # usage in GiB
print(f'mem: {100 * (mem_res.used / mem_res.total):.3f}%') # percentage usage
cpu = psutil.cpu_percent()#cpu utilization stats
mem = psutil.virtual_memory()#virtual memory stats
print(f'gpu: {res.gpu}%, gpu-mem: {res.memory}%')
fields = ['GPU Utilization',
'GPU Memory',
'CPU Utilization',
'CPU Memory'
]
dict = [{'GPU Utilization':res.gpu, 'GPU Memory':100 * (mem_res.used / mem_res.total), 'CPU Utilization':cpu, 'CPU Memory':100 * (mem.used/mem.total)}]
file_exists = os.path.isfile(path1)
with open(path1, 'a') as csvfile:
# creating a csv dict writer object
writer = csv.DictWriter(csvfile, fieldnames = fields)
if not file_exists:
writer.writeheader()
writer.writerows(dict)
#total_risk_score.append(risk_score)
#print("--------------------------------------------------------------------------")
#print("Average Risk Score:%f"%(float(sum(total_risk_score))/len(total_risk_score)))
#print("--------------------------------------------------------------------------")
self._watchdog.stop()
self.end_system_time = time.time()
end_game_time = GameTime.get_time()
self.scenario_duration_system = self.end_system_time - \
self.start_system_time
self.scenario_duration_game = end_game_time - start_game_time
fields = ['Route Completed',
'Collisions'
]
route_completed, collisions = self._console_message()
dict = [{'Route Completed':route_completed, 'Collisions':collisions}]
file_exists = os.path.isfile(path)
with open(path, 'a') as csvfile:
# creating a csv dict writer object
writer = csv.DictWriter(csvfile, fieldnames = fields)
if not file_exists:
writer.writeheader()
writer.writerows(dict)
def _console_message(self):
"""
Message that will be displayed via console
"""
def get_symbol(value, desired_value, high=True):
"""
Returns a tick or a cross depending on the values
"""
tick = '\033[92m'+'O'+'\033[0m'
cross = '\033[91m'+'X'+'\033[0m'
multiplier = 1 if high else -1
if multiplier*value >= desired_value:
symbol = tick
else:
symbol = cross
return symbol
if self.scenario_tree.status == py_trees.common.Status.RUNNING:
# If still running, all the following is None, so no point continuing
print("\n> Something happened during the simulation. Was it manually shutdown?\n")
return
blackv = py_trees.blackboard.Blackboard()
route_completed = blackv.get("RouteCompletion")
collisions = blackv.get("Collision")
outside_route_lanes = blackv.get("OutsideRouteLanes")
stop_signs = blackv.get("RunningStop")
red_light = blackv.get("RunningRedLight")
in_route = blackv.get("InRoute")
# If something failed, stop
if [x for x in (collisions, outside_route_lanes, stop_signs, red_light, in_route) if x is None]:
return
if blackv.get("RouteCompletion") >= 99:
route_completed = 100
else:
route_completed = blackv.get("RouteCompletion")
outside_route_lanes = float(outside_route_lanes)
route_symbol = get_symbol(route_completed, 100, True)
collision_symbol = get_symbol(collisions, 0, False)
outside_symbol = get_symbol(outside_route_lanes, 0, False)
red_light_symbol = get_symbol(red_light, 0, False)
stop_symbol = get_symbol(stop_signs, 0, False)
if self.scenario_tree.status == py_trees.common.Status.FAILURE:
if not in_route:
message = "> FAILED: The actor deviated from the route"
else:
message = "> FAILED: The actor didn't finish the route"
elif self.scenario_tree.status == py_trees.common.Status.SUCCESS:
if route_completed == 100:
message = "> SUCCESS: Congratulations, route finished! "
else:
message = "> FAILED: The actor timed out "
else: # This should never be triggered
return
if self.scenario_tree.status != py_trees.common.Status.RUNNING:
print("\n" + message)
print("> ")
print("> Score: ")
print("> - Route Completed [{}]: {}%".format(route_symbol, route_completed))
#print("> - Outside route lanes [{}]: {}%".format(outside_symbol, outside_route_lanes))
print("> - Collisions [{}]: {} times".format(collision_symbol, math.floor(collisions)))
#print("> - Red lights run [{}]: {} times".format(red_light_symbol, red_light))
#print("> - Stop signs run [{}]: {} times\n".format(stop_symbol, stop_signs))
return route_completed, collisions
def _tick_scenario(self, timestamp):
"""
Run next tick of scenario
This function is a callback for world.on_tick()
Important:
- It has to be ensured that the scenario has not yet completed/failed
and that the time moved forward.
- A thread lock should be used to avoid that the scenario tick is performed
multiple times in parallel.
"""
if self._timestamp_last_run < timestamp.elapsed_seconds:
self._timestamp_last_run = timestamp.elapsed_seconds
self._watchdog.update()
# Update game time and actor information
GameTime.on_carla_tick(timestamp)
CarlaDataProvider.on_carla_tick()
if self._agent is not None:# This is the entry point to the agent
ego_action = self._agent()
# Tick scenario
self.scenario_tree.tick_once()
if self._debug_mode > 1:
print("\n")
py_trees.display.print_ascii_tree(
self.scenario_tree, show_status=True)
sys.stdout.flush()
if self.scenario_tree.status != py_trees.common.Status.RUNNING:
self._running = False
if self._challenge_mode:
spectator = CarlaDataProvider.get_world().get_spectator()
ego_trans = self.ego_vehicles[0].get_transform()
spectator.set_transform(carla.Transform(ego_trans.location + carla.Location(z=50),
carla.Rotation(pitch=-90)))
if self._agent is not None:
self.ego_vehicles[0].apply_control(ego_action)
if self._agent and self._running and self._watchdog.get_status():
CarlaDataProvider.get_world().tick()
#return risk_score
def get_running_status(self):
"""
returns:
bool: False if watchdog exception occured, True otherwise
"""
return self._watchdog.get_status()
def stop_scenario(self):
"""
This function triggers a proper termination of a scenario
"""
if self.scenario is not None:
self.scenario.terminate()
if self._agent is not None:
self._agent.cleanup()
self._agent = None
CarlaDataProvider.cleanup()
CarlaActorPool.cleanup()
class ScenarioManager(object):
"""
Basic scenario manager class. This class holds all functionality
required to start, and analyze a scenario.
The user must not modify this class.
To use the ScenarioManager:
1. Create an object via manager = ScenarioManager()
2. Load a scenario via manager.load_scenario()
3. Trigger the execution of the scenario manager.execute()
This function is designed to explicitly control start and end of
the scenario execution
4. Trigger a result evaluation with manager.analyze()
5. Cleanup with manager.stop_scenario()
"""
def __init__(self, debug_mode=False, timeout=2.0):
"""
Init requires scenario as input
"""
self.scenario = None
self.scenario_tree = None
self.scenario_class = None
self.ego_vehicles = None
self.other_actors = None
self._debug_mode = debug_mode
self._agent = None
self._running = False
self._timestamp_last_run = 0.0
self._timeout = timeout
self._watchdog = Watchdog(float(self._timeout))
self.scenario_duration_system = 0.0
self.scenario_duration_game = 0.0
self.start_system_time = None
self.end_system_time = None
def _reset(self):
"""
Reset all parameters
"""
self._running = False
self._timestamp_last_run = 0.0
self.scenario_duration_system = 0.0
self.scenario_duration_game = 0.0
self.start_system_time = None
self.end_system_time = None
GameTime.restart()
def cleanup(self):
"""
This function triggers a proper termination of a scenario
"""
if self.scenario is not None:
self.scenario.terminate()
if self._agent is not None:
self._agent.cleanup()
self._agent = None
CarlaDataProvider.cleanup()
CarlaActorPool.cleanup()
def load_scenario(self, scenario, agent=None):
"""
Load a new scenario
"""
self._reset()
self._agent = AgentWrapper(agent) if agent else None
self.scenario_class = scenario
self.scenario = scenario.scenario
self.scenario_tree = self.scenario.scenario_tree
self.ego_vehicles = scenario.ego_vehicles
self.other_actors = scenario.other_actors
CarlaDataProvider.register_actors(self.ego_vehicles)
CarlaDataProvider.register_actors(self.other_actors)
# To print the scenario tree uncomment the next line
# py_trees.display.render_dot_tree(self.scenario_tree)
if self._agent is not None:
self._agent.setup_sensors(self.ego_vehicles[0], self._debug_mode)
def run_scenario(self):
"""
Trigger the start of the scenario and wait for it to finish/fail
"""
print("ScenarioManager: Running scenario {}".format(self.scenario_tree.name))
self.start_system_time = time.time()
start_game_time = GameTime.get_time()
self._watchdog.start()
self._running = True
# self.fields = ['location',
# 'velocity',
# 'ego_location',
# 'ego_velocity'
# ]
while self._running:
timestamp = None
world = CarlaDataProvider.get_world()
if world:
snapshot = world.get_snapshot()
if snapshot:
timestamp = snapshot.timestamp
if timestamp:
self._tick_scenario(timestamp)
# dict = [{'location':location, 'velocity':vel, 'ego_location':ego_location, 'ego_velocity':ego_vel}]
#
# file_exists = os.path.isfile(self.distance_path)
# with open(self.distance_path, 'a') as csvfile:
# # creating a csv dict writer object
# writer = csv.DictWriter(csvfile, fieldnames = self.fields)
# if not file_exists:
# writer.writeheader()
# writer.writerows(dict)
self._watchdog.stop()
self.cleanup()
self.end_system_time = time.time()
end_game_time = GameTime.get_time()
self.scenario_duration_system = self.end_system_time - \
self.start_system_time
self.scenario_duration_game = end_game_time - start_game_time
if self.scenario_tree.status == py_trees.common.Status.FAILURE:
print("ScenarioManager: Terminated due to failure")
def _tick_scenario(self,timestamp):
"""
Run next tick of scenario
This function is a callback for world.on_tick()
Important:
- It has to be ensured that the scenario has not yet completed/failed
and that the time moved forward.
- A thread lock should be used to avoid that the scenario tick is performed
multiple times in parallel.
"""
if self._timestamp_last_run < timestamp.elapsed_seconds and self._running:
self._timestamp_last_run = timestamp.elapsed_seconds
self._watchdog.update()
if self._debug_mode:
print("\n--------- Tick ---------\n")
# Update game time and actor information
GameTime.on_carla_tick(timestamp)
CarlaDataProvider.on_carla_tick()
#print(vel)
if self._agent is not None:
ego_action = self._agent()
# Tick scenario
self.scenario_tree.tick_once()
if self._debug_mode:
print("\n")
py_trees.display.print_ascii_tree(self.scenario_tree, show_status=True)
sys.stdout.flush()
if self.scenario_tree.status != py_trees.common.Status.RUNNING:
self._running = False
if self._agent is not None:
self.ego_vehicles[0].apply_control(ego_action)
if self._agent and self._running and self._watchdog.get_status():
CarlaDataProvider.perform_carla_tick(self._timeout)
#return vel,location,ego_vel,ego_location
def get_running_status(self):
"""
returns:
bool: False if watchdog exception occured, True otherwise
"""
return self._watchdog.get_status()
def stop_scenario(self):
"""
This function is used by the overall signal handler to terminate the scenario execution
"""
self._running = False
def analyze_scenario(self, stdout, filename, junit):
"""
This function is intended to be called from outside and provide
the final statistics about the scenario (human-readable, in form of a junit
report, etc.)
"""
failure = False
timeout = False
result = "SUCCESS"
if self.scenario.test_criteria is None:
return True
for criterion in self.scenario.get_criteria():
if (not criterion.optional and
criterion.test_status != "SUCCESS" and
criterion.test_status != "ACCEPTABLE"):
failure = True
result = "FAILURE"
elif criterion.test_status == "ACCEPTABLE":
result = "ACCEPTABLE"
if self.scenario.timeout_node.timeout and not failure:
timeout = True
result = "TIMEOUT"
output = ResultOutputProvider(self, result, stdout, filename, junit)
output.write()
return failure or timeout
class NoCrashEvaluator(object):
"""
TODO: document me!
"""
ego_vehicles = []
# Tunable parameters
client_timeout = 10.0 # in seconds
wait_for_world = 20.0 # in seconds
frame_rate = 20.0 # in Hz
def __init__(self, args, statistics_manager):
"""
Setup CARLA client and world
Setup ScenarioManager
"""
self.statistics_manager = statistics_manager
self.sensors = None
self.sensor_icons = []
# First of all, we need to create the client that will send the requests
# to the simulator. Here we'll assume the simulator is accepting
# requests in the localhost at port 2000.
self.client = carla.Client(args.host, int(args.port))
if args.timeout:
self.client_timeout = float(args.timeout)
self.client.set_timeout(self.client_timeout)
self.traffic_manager = self.client.get_trafficmanager(
int(args.trafficManagerPort))
dist = pkg_resources.get_distribution("carla")
if dist.version != 'leaderboard':
if LooseVersion(dist.version) < LooseVersion('0.9.10'):
raise ImportError(
"CARLA version 0.9.10.1 or newer required. CARLA version found: {}"
.format(dist))
# Load agent
module_name = os.path.basename(args.agent).split('.')[0]
sys.path.insert(0, os.path.dirname(args.agent))
self.module_agent = importlib.import_module(module_name)
# Create the ScenarioManager
self.manager = ScenarioManager(args.timeout, args.debug > 1)
# Time control for summary purposes
self._start_time = GameTime.get_time()
self._end_time = None
# Create the agent timer
self._agent_watchdog = Watchdog(int(float(args.timeout)))
signal.signal(signal.SIGINT, self._signal_handler)
self.town = args.town
def _signal_handler(self, signum, frame):
"""
Terminate scenario ticking when receiving a signal interrupt
"""
if self._agent_watchdog and not self._agent_watchdog.get_status():
raise RuntimeError("Timeout: Agent took too long to setup")
elif self.manager:
self.manager.signal_handler(signum, frame)
def __del__(self):
"""
Cleanup and delete actors, ScenarioManager and CARLA world
"""
self._cleanup()
if hasattr(self, 'manager') and self.manager:
del self.manager
if hasattr(self, 'world') and self.world:
del self.world
def _cleanup(self):
"""
Remove and destroy all actors
"""
# Simulation still running and in synchronous mode?
if self.manager and self.manager.get_running_status() \
and hasattr(self, 'world') and self.world:
# Reset to asynchronous mode
settings = self.world.get_settings()
settings.synchronous_mode = False
settings.fixed_delta_seconds = None
self.world.apply_settings(settings)
self.traffic_manager.set_synchronous_mode(False)
if self.manager:
self.manager.cleanup()
CarlaDataProvider.cleanup()
for i, _ in enumerate(self.ego_vehicles):
if self.ego_vehicles[i]:
self.ego_vehicles[i].destroy()
self.ego_vehicles[i] = None
self.ego_vehicles = []
if self._agent_watchdog:
self._agent_watchdog.stop()
if hasattr(self, 'agent_instance') and self.agent_instance:
self.agent_instance.destroy()
self.agent_instance = None
if hasattr(self, 'statistics_manager') and self.statistics_manager:
self.statistics_manager.scenario = None
def _load_and_wait_for_world(self, args):
"""
Load a new CARLA world and provide data to CarlaDataProvider
"""
self.world = self.client.load_world(args.town)
settings = self.world.get_settings()
settings.fixed_delta_seconds = 1.0 / self.frame_rate
settings.synchronous_mode = True
self.world.apply_settings(settings)
self.world.reset_all_traffic_lights()
CarlaDataProvider.set_client(self.client)
CarlaDataProvider.set_world(self.world)
CarlaDataProvider.set_traffic_manager_port(int(
args.trafficManagerPort))
self.traffic_manager.set_synchronous_mode(True)
self.traffic_manager.set_random_device_seed(
int(args.trafficManagerSeed))
# Wait for the world to be ready
if CarlaDataProvider.is_sync_mode():
self.world.tick()
else:
self.world.wait_for_tick()
if CarlaDataProvider.get_map().name != args.town:
raise Exception("The CARLA server uses the wrong map!"
"This scenario requires to use map {}".format(
args.town))
# def _register_statistics(self, config, checkpoint, entry_status, crash_message=""):
# """
# Computes and saved the simulation statistics
# """
# # register statistics
# current_stats_record = self.statistics_manager.compute_route_statistics(
# config,
# self.manager.scenario_duration_system,
# self.manager.scenario_duration_game,
# crash_message
# )
# print("\033[1m> Registering the route statistics\033[0m")
# self.statistics_manager.save_record(current_stats_record, config.index, checkpoint)
# self.statistics_manager.save_entry_status(entry_status, False, checkpoint)
def _load_and_run_scenario(self, args, route, weather_idx, traffic_idx):
"""
Load and run the scenario given by config.
Depending on what code fails, the simulation will either stop the route and
continue from the next one, or report a crash and stop.
"""
crash_message = ""
entry_status = "Started"
start_idx, target_idx = route
traffic_lvl = ['Empty', 'Regular', 'Dense'][traffic_idx]
print(
"\n\033[1m========= Preparing {} {}: {} to {}, weather {} ========="
.format(args.town, traffic_lvl, start_idx, target_idx,
weather_idx))
print("> Setting up the agent\033[0m")
# Set up the user's agent, and the timer to avoid freezing the simulation
try:
self._agent_watchdog.start()
agent_class_name = getattr(self.module_agent, 'get_entry_point')()
self.agent_instance = getattr(self.module_agent,
agent_class_name)(args.agent_config)
# Check and store the sensors
if not self.sensors:
self.sensors = self.agent_instance.sensors()
track = self.agent_instance.track
AgentWrapper.validate_sensor_configuration(
self.sensors, track, args.track)
self._agent_watchdog.stop()
except SensorConfigurationInvalid as e:
# The sensors are invalid -> set the ejecution to rejected and stop
print("\n\033[91mThe sensor's configuration used is invalid:")
print("> {}\033[0m\n".format(e))
traceback.print_exc()
crash_message = "Agent's sensors were invalid"
entry_status = "Rejected"
self._cleanup()
sys.exit(-1)
except Exception as e:
# The agent setup has failed -> start the next route
print("\n\033[91mCould not set up the required agent:")
print("> {}\033[0m\n".format(e))
traceback.print_exc()
crash_message = "Agent couldn't be set up"
self._cleanup()
return
print("\033[1m> Loading the world\033[0m")
# Load the world and the scenario
try:
self._load_and_wait_for_world(args)
scenario = NoCrashEvalScenario(world=self.world,
agent=self.agent_instance,
start_idx=start_idx,
target_idx=target_idx,
weather_idx=weather_idx,
traffic_idx=traffic_idx,
debug_mode=args.debug)
self.manager.load_scenario(scenario, self.agent_instance, 0)
except Exception as e:
# The scenario is wrong -> set the ejecution to crashed and stop
print("\n\033[91mThe scenario could not be loaded:")
print("> {}\033[0m\n".format(e))
traceback.print_exc()
crash_message = "Simulation crashed"
entry_status = "Crashed"
if args.record:
self.client.stop_recorder()
self._cleanup()
sys.exit(-1)
print("\033[1m> Running the route\033[0m")
# Run the scenario
try:
self.manager.run_scenario()
except AgentError as e:
# The agent has failed -> stop the route
print("\n\033[91mStopping the route, the agent has crashed:")
print("> {}\033[0m\n".format(e))
traceback.print_exc()
crash_message = "Agent crashed"
except Exception as e:
print("\n\033[91mError during the simulation:")
print("> {}\033[0m\n".format(e))
traceback.print_exc()
crash_message = "Simulation crashed"
entry_status = "Crashed"
# Stop the scenario
try:
print("\033[1m> Stopping the route\033[0m")
self.manager.stop_scenario()
route_completion, lights_ran, duration = self.manager.get_nocrash_diagnostics(
)
self.statistics_manager.log(self.town, traffic_idx, weather_idx,
start_idx, target_idx,
route_completion, lights_ran, duration)
if args.record:
self.client.stop_recorder()
# Remove all actors
scenario.remove_all_actors()
self._cleanup()
except Exception as e:
print(
"\n\033[91mFailed to stop the scenario, the statistics might be empty:"
)
print("> {}\033[0m\n".format(e))
traceback.print_exc()
crash_message = "Simulation crashed"
if crash_message == "Simulation crashed":
sys.exit(-1)
def run(self, args):
"""
Run the challenge mode
"""
# if args.resume:
# route_indexer.resume(args.checkpoint)
# self.statistics_manager.resume(args.checkpoint)
# else:
# self.statistics_manager.clear_record(args.checkpoint)
# route_indexer.save_state(args.checkpoint)
# Load routes
with open(f'runners/suite/nocrash_{args.town}.txt', 'r') as f:
routes = [tuple(map(int, l.split())) for l in f.readlines()]
weathers = {'train': [1, 3, 6, 8], 'test': [10, 14]}.get(args.weather)
traffics = [0, 1, 2]
for traffic, route, weather in itertools.product(
traffics, routes, weathers):
if self.statistics_manager.is_finished(self.town, route, weather,
traffic):
continue
self._load_and_run_scenario(args, route, weather, traffic)
# save global statistics
print("\033[1m> Registering the global statistics\033[0m")