def analyze_scenario(self, stdout, filename, junit):
"""
This function is intended to be called from outside and provide
the final statistics about the scenario (human-readable, in form of a junit
report, etc.)
"""
failure = False
timeout = False
result = "SUCCESS"
if self.scenario.test_criteria is None:
return True
for criterion in self.scenario.get_criteria():
if (not criterion.optional and criterion.test_status != "SUCCESS"
and criterion.test_status != "ACCEPTABLE"):
failure = True
result = "FAILURE"
elif criterion.test_status == "ACCEPTABLE":
result = "ACCEPTABLE"
if self.scenario.timeout_node.timeout and not failure:
timeout = True
result = "TIMEOUT"
output = ResultOutputProvider(self, result, stdout, filename, junit)
output.write()
if self._challenge_mode:
ChallengeStatisticsManager.record_scenario_statistics()
return failure or timeout
def _tick_scenario(self, timestamp):
"""
Run next tick of scenario
This function is a callback for world.on_tick()
Important:
- It has to be ensured that the scenario has not yet completed/failed
and that the time moved forward.
- A thread lock should be used to avoid that the scenario tick is performed
multiple times in parallel.
"""
if self._timestamp_last_run < timestamp.elapsed_seconds and self._running:
self._timestamp_last_run = timestamp.elapsed_seconds
self._watchdog.update()
if self._debug_mode:
print("\n--------- Tick ---------\n")
# Update game time and actor information
GameTime.on_carla_tick(timestamp)
CarlaDataProvider.on_carla_tick()
if self._agent is not None:
ego_action = self._agent()
# Tick scenario
self.scenario_tree.tick_once()
if self._debug_mode:
print("\n")
py_trees.display.print_ascii_tree(self.scenario_tree,
show_status=True)
sys.stdout.flush()
if self.scenario_tree.status != py_trees.common.Status.RUNNING:
self._running = False
if self._challenge_mode:
ChallengeStatisticsManager.compute_current_statistics()
if self._agent is not None:
self.ego_vehicles[0].apply_control(ego_action)
if self._agent and self._running and self._watchdog.get_status():
CarlaDataProvider.perform_carla_tick(self._timeout)
def load_scenario(self, scenario, agent=None):
"""
Load a new scenario
"""
self._reset()
self._agent = AgentWrapper(agent, self._challenge_mode) if agent else None
self.scenario_class = scenario
self.scenario = scenario.scenario
self.scenario_tree = self.scenario.scenario_tree
self.ego_vehicles = scenario.ego_vehicles
self.other_actors = scenario.other_actors
CarlaDataProvider.register_actors(self.ego_vehicles)
CarlaDataProvider.register_actors(self.other_actors)
# To print the scenario tree uncomment the next line
# py_trees.display.render_dot_tree(self.scenario_tree)
if self._challenge_mode:
ChallengeStatisticsManager.next_scenario(self.scenario)
if self._agent is not None:
self._agent.setup_sensors(self.ego_vehicles[0], self._debug_mode)
def main():
"""
main function
"""
description = (
"CARLA Scenario Runner: Setup, Run and Evaluate scenarios using CARLA\n"
"Current version: " + str(VERSION))
parser = argparse.ArgumentParser(description=description,
formatter_class=RawTextHelpFormatter)
parser.add_argument('--host',
default='127.0.0.1',
help='IP of the host server (default: localhost)')
parser.add_argument('--port',
default='2000',
help='TCP port to listen to (default: 2000)')
parser.add_argument('--debug',
action="store_true",
help='Run with debug output')
parser.add_argument('--output',
action="store_true",
help='Provide results on stdout')
parser.add_argument('--file',
action="store_true",
help='Write results into a txt file')
parser.add_argument('--junit',
action="store_true",
help='Write results into a junit file')
parser.add_argument(
'--outputDir',
default='',
help='Directory for output files (default: this directory)')
parser.add_argument('--waitForEgo',
action="store_true",
help='Connect the scenario to an existing ego vehicle')
parser.add_argument(
'--configFile',
default='',
help='Provide an additional scenario configuration file (*.xml)')
parser.add_argument(
'--additionalScenario',
default='',
help='Provide additional scenario implementations (*.py)')
parser.add_argument(
'--reloadWorld',
action="store_true",
help='Reload the CARLA world before starting a scenario (default=True)'
)
# pylint: disable=line-too-long
parser.add_argument(
'--scenario',
help=
'Name of the scenario to be executed. Use the preposition \'group:\' to run all scenarios of one class, e.g. ControlLoss or FollowLeadingVehicle'
)
parser.add_argument('--randomize',
action="store_true",
help='Scenario parameters are randomized')
parser.add_argument('--repetitions',
default=1,
help='Number of scenario executions')
parser.add_argument('--list',
action="store_true",
help='List all supported scenarios and exit')
parser.add_argument(
'--agent',
help=
"Agent used to execute the scenario (optional). Currently only compatible with route-based scenarios."
)
parser.add_argument('--agentConfig',
type=str,
help="Path to Agent's configuration file",
default="")
parser.add_argument('--openscenario',
help='Provide an OpenSCENARIO definition')
parser.add_argument(
'--route',
help=
'Run a route as a scenario, similar to the CARLA AD challenge (input: (route_file,scenario_file,[number of route]))',
nargs='+',
type=str)
parser.add_argument('--challenge',
action="store_true",
help='Run in challenge mode')
parser.add_argument(
'--record',
action="store_true",
help='Use CARLA recording feature to create a recording of the scenario'
)
parser.add_argument('--timeout',
default="10.0",
help='Set the CARLA client timeout value in seconds')
parser.add_argument('-v',
'--version',
action='version',
version='%(prog)s ' + str(VERSION))
arguments = parser.parse_args()
# pylint: enable=line-too-long
if arguments.list:
print("Currently the following scenarios are supported:")
print(*ScenarioConfigurationParser.get_list_of_scenarios(
arguments.configFile),
sep='\n')
return 1
if not arguments.scenario and not arguments.openscenario and not arguments.route:
print("Please specify either a scenario or use the route mode\n\n")
parser.print_help(sys.stdout)
return 1
if (arguments.route
and arguments.openscenario) or (arguments.route
and arguments.scenario):
print(
"The route mode cannot be used together with a scenario (incl. OpenSCENARIO)'\n\n"
)
parser.print_help(sys.stdout)
return 1
if arguments.agent and (arguments.openscenario or arguments.scenario):
print("Agents are currently only compatible with route scenarios'\n\n")
parser.print_help(sys.stdout)
return 1
if arguments.challenge and (arguments.openscenario or arguments.scenario):
print(
"The challenge mode can only be used with route-based scenarios'\n\n"
)
parser.print_help(sys.stdout)
return 1
if arguments.route:
arguments.reloadWorld = True
scenario_runner = None
result = True
try:
scenario_runner = ScenarioRunner(arguments)
result = scenario_runner.run()
finally:
if arguments.challenge:
ChallengeStatisticsManager.report_challenge_statistics(
'results.json', arguments.debug)
if scenario_runner is not None:
scenario_runner.destroy()
del scenario_runner
return not result
def _run_challenge(self):
"""
Run the challenge mode
"""
result = False
phase_codename = os.getenv('CHALLENGE_PHASE_CODENAME', 'dev_track_3')
phase = phase_codename.split("_")[0]
repetitions = self._args.repetitions
if self._args.challenge:
weather_profiles = CarlaDataProvider.find_weather_presets()
scenario_runner_root = os.getenv('ROOT_SCENARIO_RUNNER', "./")
if phase == 'dev':
routes = '{}/srunner/challenge/routes_devtest.xml'.format(
scenario_runner_root)
repetitions = 1
elif phase == 'validation':
routes = '{}/srunner/challenge/routes_testprep.xml'.format(
scenario_runner_root)
repetitions = 3
elif phase == 'test':
routes = '{}/srunner/challenge/routes_testchallenge.xml'.format(
scenario_runner_root)
repetitions = 3
else:
# debug mode
routes = '{}/srunner/challenge/routes_debug.xml'.format(
scenario_runner_root)
repetitions = 1
if self._args.route:
routes = self._args.route[0]
scenario_file = self._args.route[1]
single_route = None
if len(self._args.route) > 2:
single_route = self._args.route[2]
# retrieve routes
route_descriptions_list = RouteParser.parse_routes_file(
routes, single_route)
# find and filter potential scenarios for each of the evaluated routes
# For each of the routes and corresponding possible scenarios to be evaluated.
if self._args.challenge:
n_routes = len(route_descriptions_list) * repetitions
ChallengeStatisticsManager.set_number_of_scenarios(n_routes)
for _, route_description in enumerate(route_descriptions_list):
for repetition in range(repetitions):
if self._args.challenge and not self._within_available_time():
error_message = 'Not enough simulation time available to continue'
print(error_message)
ChallengeStatisticsManager.record_fatal_error(
error_message)
self._cleanup()
return False
config = RouteScenarioConfiguration(route_description,
scenario_file)
if self._args.challenge:
profile = weather_profiles[repetition %
len(weather_profiles)]
config.weather = profile[0]
config.weather.sun_azimuth_angle = -1
config.weather.sun_altitude_angle = -1
result = self._load_and_run_scenario(config)
self._cleanup()
return result
def _load_and_run_scenario(self, config):
"""
Load and run the scenario given by config
"""
result = False
if not self._load_and_wait_for_world(config.town, config.ego_vehicles):
self._cleanup()
return False
if self._args.agent:
agent_class_name = self.module_agent.__name__.title().replace(
'_', '')
try:
self.agent_instance = getattr(self.module_agent,
agent_class_name)(
self._args.agentConfig)
config.agent = self.agent_instance
except Exception as e: # pylint: disable=broad-except
traceback.print_exc()
print("Could not setup required agent due to {}".format(e))
self._cleanup()
return False
# Prepare scenario
print("Preparing scenario: " + config.name)
try:
self._prepare_ego_vehicles(config.ego_vehicles)
if self._args.openscenario:
scenario = OpenScenario(world=self.world,
ego_vehicles=self.ego_vehicles,
config=config,
config_file=self._args.openscenario,
timeout=100000)
elif self._args.route:
scenario = RouteScenario(world=self.world,
config=config,
debug_mode=self._args.debug)
else:
scenario_class = self._get_scenario_class_or_fail(config.type)
scenario = scenario_class(self.world, self.ego_vehicles,
config, self._args.randomize,
self._args.debug)
except Exception as exception: # pylint: disable=broad-except
print("The scenario cannot be loaded")
traceback.print_exc()
print(exception)
self._cleanup()
return False
# Set the appropriate weather conditions
self.world.set_weather(config.weather)
# Set the appropriate road friction
if config.friction is not None:
friction_bp = self.world.get_blueprint_library().find(
'static.trigger.friction')
extent = carla.Location(1000000.0, 1000000.0, 1000000.0)
friction_bp.set_attribute('friction', str(config.friction))
friction_bp.set_attribute('extent_x', str(extent.x))
friction_bp.set_attribute('extent_y', str(extent.y))
friction_bp.set_attribute('extent_z', str(extent.z))
# Spawn Trigger Friction
transform = carla.Transform()
transform.location = carla.Location(-10000.0, -10000.0, 0.0)
self.world.spawn_actor(friction_bp, transform)
try:
# Load scenario and run it
if self._args.record:
self.client.start_recorder("{}/{}.log".format(
os.getenv('ROOT_SCENARIO_RUNNER', "./"), config.name))
self.manager.load_scenario(scenario, self.agent_instance)
self.manager.run_scenario()
# Provide outputs if required
self._analyze_scenario(config)
# Remove all actors
scenario.remove_all_actors()
result = True
except SensorConfigurationInvalid as e:
self._cleanup()
ChallengeStatisticsManager.record_fatal_error(e)
sys.exit(-1)
except Exception as e: # pylint: disable=broad-except
traceback.print_exc()
if self._args.challenge:
ChallengeStatisticsManager.set_error_message(
traceback.format_exc())
print(e)
result = False
self._cleanup()
return result
"The route mode cannot be used together with a scenario (incl. OpenSCENARIO)'\n\n"
)
PARSER.print_help(sys.stdout)
sys.exit(0)
if ARGUMENTS.agent and (ARGUMENTS.openscenario or ARGUMENTS.scenario):
print("Agents are currently only compatible with route scenarios'\n\n")
PARSER.print_help(sys.stdout)
sys.exit(0)
if ARGUMENTS.challenge and (ARGUMENTS.openscenario or ARGUMENTS.scenario):
print(
"The challenge mode can only be used with route-based scenarios'\n\n"
)
PARSER.print_help(sys.stdout)
sys.exit(0)
if ARGUMENTS.route:
ARGUMENTS.reloadWorld = True
SCENARIORUNNER = None
try:
SCENARIORUNNER = ScenarioRunner(ARGUMENTS)
SCENARIORUNNER.run(ARGUMENTS)
finally:
if ARGUMENTS.challenge:
ChallengeStatisticsManager.report_challenge_statistics(
'results.json', ARGUMENTS.debug)
if SCENARIORUNNER is not None:
del SCENARIORUNNER
def _load_and_run_scenario(self, args, config):
"""
Load and run the scenario given by config
"""
if not self._load_and_wait_for_world(args, config.town,
config.ego_vehicles):
self._cleanup()
return
if args.agent:
agent_class_name = self.module_agent.__name__.title().replace(
'_', '')
try:
self.agent_instance = getattr(
self.module_agent, agent_class_name)(args.agentConfig)
config.agent = self.agent_instance
except Exception as e:
print("Could not setup required agent due to {}".format(e))
self._cleanup()
return
# Prepare scenario
print("Preparing scenario: " + config.name)
try:
self._prepare_ego_vehicles(config.ego_vehicles, args.waitForEgo)
if args.openscenario:
scenario = OpenScenario(world=self.world,
ego_vehicles=self.ego_vehicles,
config=config,
config_file=args.openscenario,
timeout=100000)
elif args.route:
scenario = RouteScenario(world=self.world,
config=config,
debug_mode=args.debug)
else:
scenario_class = self._get_scenario_class_or_fail(config.type)
scenario = scenario_class(self.world, self.ego_vehicles,
config, args.randomize, args.debug)
except Exception as exception:
print("The scenario cannot be loaded")
if args.debug:
traceback.print_exc()
print(exception)
self._cleanup()
return
# Set the appropriate weather conditions
weather = carla.WeatherParameters(
cloudyness=config.weather.cloudyness,
precipitation=config.weather.precipitation,
precipitation_deposits=config.weather.precipitation_deposits,
wind_intensity=config.weather.wind_intensity,
sun_azimuth_angle=config.weather.sun_azimuth,
sun_altitude_angle=config.weather.sun_altitude)
self.world.set_weather(weather)
try:
# Load scenario and run it
if args.record:
self.client.start_recorder("{}/{}.log".format(
os.getenv('ROOT_SCENARIO_RUNNER', "./"), config.name))
self.manager.load_scenario(scenario, self.agent_instance)
self.manager.run_scenario()
# Stop scenario
self.manager.stop_scenario()
# Provide outputs if required
self._analyze_scenario(args, config)
# Remove all actors
scenario.remove_all_actors()
except SensorConfigurationInvalid as e:
self._cleanup(True)
ChallengeStatisticsManager.record_fatal_error(e)
sys.exit(-1)
except Exception as e:
if args.debug:
traceback.print_exc()
if args.challenge:
ChallengeStatisticsManager.set_error_message(
traceback.format_exc())
print(e)
self._cleanup()