def run(self, args):
"""
Run the challenge mode
"""
route_indexer = RouteIndexer(args.routes, args.scenarios, args.repetitions)
if args.resume:
route_indexer.resume(args.checkpoint)
self.statistics_manager.resume(args.checkpoint)
else:
self.statistics_manager.clear_record(args.checkpoint)
route_indexer.save_state(args.checkpoint)
while route_indexer.peek():
# setup
config = route_indexer.next()
# run
self._load_and_run_scenario(args, config)
route_indexer.save_state(args.checkpoint)
# save global statistics
print("\033[1m> Registering the global statistics\033[0m")
global_stats_record = self.statistics_manager.compute_global_statistics(route_indexer.total)
StatisticsManager.save_global_record(global_stats_record, self.sensor_icons, route_indexer.total, args.checkpoint)
def run(self, args, customized_data):
"""
Run the challenge mode
"""
route_indexer = RouteIndexer(args.routes, args.scenarios,
args.repetitions)
if args.resume:
route_indexer.resume(self.save_path)
self.statistics_manager.resume(self.save_path)
else:
self.statistics_manager.clear_record(self.save_path)
while route_indexer.peek():
# setup
config = route_indexer.next()
# run
self._load_and_run_scenario(args, config, customized_data)
self._cleanup(ego=True)
route_indexer.save_state(self.save_path)
# save global statistics
# modification
global_stats_record = self.statistics_manager.compute_global_statistics(
route_indexer.total)
StatisticsManager.save_global_record(global_stats_record, self.sensors,
self.save_path)
def run(self, args, filename, data_folder, route_folder, model_path):
"""
Run the challenge mode
"""
route_indexer = RouteIndexer(args.routes, args.scenarios,
args.repetitions, route_folder)
if args.resume:
route_indexer.resume(args.checkpoint)
self.statistics_manager.resume(args.checkpoint)
else:
self.statistics_manager.clear_record(args.checkpoint)
k = 0
while route_indexer.peek():
#x = time.time()
k += 1
# setup
config = route_indexer.next()
# run
self._load_and_run_scenario(args, config, data_folder,
route_folder, k, model_path)
self._cleanup(ego=True)
route_indexer.save_state(args.checkpoint)
# save global statistics
global_stats_record, route_in_km = self.statistics_manager.compute_global_statistics(
route_indexer.total)
StatisticsManager.save_global_record(global_stats_record,
route_in_km, self.sensors,
args.checkpoint, filename,
config)
def __init__(self,
args,
scenario_class,
scenario,
route,
checkpoint='simulation_results.json',
town=None,
port=1000,
tm_port=1002,
debug=False):
args = deepcopy(args)
# Inject args
args.scenario_class = scenario_class
args.town = town
args.port = port
args.trafficManagerPort = tm_port
args.scenarios = scenario
args.routes = route
args.debug = debug
args.checkpoint = checkpoint
args.record = ''
self.runner = LeaderboardEvaluator(args, StatisticsManager())
self.args = args
def main():
description = "CARLA AD Leaderboard Evaluation: evaluate your Agent in CARLA scenarios\n"
# general parameters
parser = argparse.ArgumentParser(description=description, formatter_class=RawTextHelpFormatter)
parser.add_argument('--host', default='localhost',
help='IP of the host server (default: localhost)')
parser.add_argument('--port', default='2000', help='TCP port to listen to (default: 2000)')
parser.add_argument('--trafficManagerPort', default='8000',
help='Port to use for the TrafficManager (default: 8000)')
parser.add_argument('--trafficManagerSeed', default='0',
help='Seed used by the TrafficManager (default: 0)')
parser.add_argument('--debug', type=int, help='Run with debug output', default=0)
parser.add_argument('--record', type=str, default='',
help='Use CARLA recording feature to create a recording of the scenario')
parser.add_argument('--timeout', default="120.0",
help='Set the CARLA client timeout value in seconds')
# simulation setup
parser.add_argument('--routes',
help='Name of the route to be executed. Point to the route_xml_file to be executed.',
required=True)
parser.add_argument('--scenarios',
help='Name of the scenario annotation file to be mixed with the route.',
required=True)
parser.add_argument('--repetitions',
type=int,
default=1,
help='Number of repetitions per route.')
# agent-related options
parser.add_argument("-a", "--agent", type=str, help="Path to Agent's py file to evaluate", required=True)
parser.add_argument("--agent-config", type=str, help="Path to Agent's configuration file", default="")
parser.add_argument("--track", type=str, default='SENSORS', help="Participation track: SENSORS, MAP")
parser.add_argument('--resume', type=bool, default=False, help='Resume execution from last checkpoint?')
parser.add_argument("--checkpoint", type=str,
default='./simulation_results.json',
help="Path to checkpoint used for saving statistics and resuming")
parser.add_argument("--timely-commands", type=bool, default=False, help="Delay commands by their runtime")
parser.add_argument("--frame-rate", type=int, default=20, help="Simulator frame rate")
arguments = parser.parse_args()
statistics_manager = StatisticsManager()
try:
leaderboard_evaluator = LeaderboardEvaluator(arguments, statistics_manager)
leaderboard_evaluator.run(arguments)
except Exception as e:
traceback.print_exc()
finally:
del leaderboard_evaluator
def run(self, args):
"""
Run the challenge mode
"""
# agent_class_name = getattr(self.module_agent, 'get_entry_point')()
# self.agent_instance = getattr(self.module_agent, agent_class_name)(args.agent_config)
route_indexer = RouteIndexer(args.routes, args.scenarios, args.repetitions)
if args.resume:
route_indexer.resume(args.checkpoint)
self.statistics_manager.resume(args.checkpoint)
else:
self.statistics_manager.clear_record(args.checkpoint)
route_indexer.save_state(args.checkpoint)
while route_indexer.peek():
# setup
config = route_indexer.next()
# run
self._load_and_run_scenario(args, config)
for obj in gc.get_objects():
try:
if torch.is_tensor(obj) or (hasattr(obj, 'data') and torch.is_tensor(obj.data)):
print(type(obj), obj.size())
except:
pass
route_indexer.save_state(args.checkpoint)
# save global statistics
print("\033[1m> Registering the global statistics\033[0m")
global_stats_record = self.statistics_manager.compute_global_statistics(route_indexer.total)
StatisticsManager.save_global_record(global_stats_record, self.sensor_icons, route_indexer.total, args.checkpoint)
def start_evaluation(arguments):
print("Starting the evaluation")
print(arguments)
statistics_manager = StatisticsManager()
try:
leaderboard_evaluator = LeaderboardEvaluator(arguments,
statistics_manager)
leaderboard_evaluator.run(arguments)
except Exception as e:
traceback.print_exc()
finally:
del leaderboard_evaluator
def __init__(self,
args,
scenario,
route,
port=1000,
tm_port=1002,
debug=False):
args = deepcopy(args)
# Inject args
args.scenario_class = 'route_scenario'
args.port = port
args.trafficManagerPort = tm_port
args.scenarios = scenario
args.routes = route
args.debug = debug
args.record = ''
self.runner = LeaderboardEvaluator(args, StatisticsManager())
self.args = args
def main():
arguments = specify_args()
arguments.debug = True
statistics_manager = StatisticsManager()
# Fixed Hyperparameters
# if use_actors = False, no other actors will be generated
use_actors = False
using_customized_route_and_scenario = True
multi_actors_scenarios = ['Scenario12']
arguments.scenarios = 'leaderboard/data/fuzzing_scenarios.json'
town_name = 'Town03'
scenario = 'Scenario12'
direction = 'front'
route = 0
# sample_factor is an integer between [1, 5]
sample_factor = 5
# waypoints_num_limit: the maximum number of waypoints that we consider to perturb.
waypoints_num_limit = 10
# lane_width = 3.5
max_num_of_vehicle = 2
max_num_of_pedestrians = 2
# Parameters to optimize
# Set up environment parameters
# real, [0, 1]
friction = 0.1
# integer, [0, 20]
weather_index = 2
# Laundry Stuff-------------------------------------------------------------
arguments.weather_index = weather_index
os.environ['WEATHER_INDEX'] = str(weather_index)
town_scenario_direction = town_name + '/' + scenario
folders = [os.environ['SAVE_FOLDER'], town_name, scenario]
if scenario in multi_actors_scenarios:
town_scenario_direction += '/' + direction
folders.append(direction)
cur_folder_name = make_hierarchical_dir(folders)
os.environ['SAVE_FOLDER'] = cur_folder_name
arguments.save_folder = os.environ['SAVE_FOLDER']
route_prefix = 'leaderboard/data/customized_routes/' + town_scenario_direction + '/route_'
route_str = str(route)
if route < 10:
route_str = '0' + route_str
arguments.routes = route_prefix + route_str + '.xml'
os.environ['ROUTES'] = arguments.routes
# extract waypoints along route
import xml.etree.ElementTree as ET
tree = ET.parse(arguments.routes)
route_waypoints = []
# this iteration should only go once since we only keep one route per file
for route in tree.iter("route"):
route_id = route.attrib['id']
route_town = route.attrib['town']
for waypoint in route.iter('waypoint'):
route_waypoints.append(
carla.Transform(
carla.Location(x=float(waypoint.attrib['x']),
y=float(waypoint.attrib['y']),
z=float(waypoint.attrib['z'])),
carla.Rotation(float(waypoint.attrib['pitch']),
float(waypoint.attrib['yaw']),
float(waypoint.attrib['roll']))))
# extract waypoints for the scenario
world_annotations = RouteParser.parse_annotations_file(arguments.scenarios)
info = world_annotations[town_name][0]["available_event_configurations"][0]
center = info["center"]
RouteParser.convert_waypoint_float(center)
center_location = carla.Location(float(center['x']), float(center['y']),
float(center['z']))
center_rotation = carla.Rotation(float(center['pitch']),
float(center['yaw']), 0.0)
center_transform = carla.Transform(center_location, center_rotation)
# --------------------------------------------------------------------------
if use_actors:
# Set up actors
# ego car
ego_car_waypoints_perturbation = []
for i in range(waypoints_num_limit):
dx = np.clip(np.random.normal(0, 2, 1)[0], -0.5, 0.5)
dy = np.clip(np.random.normal(0, 2, 1)[0], -0.5, 0.5)
ego_car_waypoints_perturbation.append((dx, dy))
# static
static_1_transform = center_transform
static_1 = Static(model='static.prop.barrel',
spawn_transform=static_1_transform)
static_list = [static_1]
# pedestrians
pedestrian_1_transform = create_transform(
route_waypoints[0].location.x - 2,
route_waypoints[0].location.y - 8, 0, 0,
route_waypoints[0].rotation.yaw, 0)
pedestrian_1 = Pedestrian(model='walker.pedestrian.0001',
spawn_transform=pedestrian_1_transform,
trigger_distance=20,
speed=1.5,
dist_to_travel=6,
after_trigger_behavior='stop')
pedestrian_list = [pedestrian_1]
# vehicles
waypoint_follower = True
vehicle_1_transform = create_transform(
route_waypoints[1].location.x, route_waypoints[1].location.y - 5,
0, 0, route_waypoints[1].rotation.yaw, 0)
# if waypoint_follower == False
vehicle_1_dist_to_travel = 5
vehicle_1_target_direction = carla.Vector3D(x=0.2, y=1, z=0)
# else
targeted_waypoint = create_transform(
route_waypoints[1].location.x, route_waypoints[1].location.y - 40,
0, 0, route_waypoints[1].rotation.yaw, 0)
# targeted_waypoint = route_waypoints[-1]
vehicle_1_waypoints_perturbation = []
for i in range(waypoints_num_limit):
dx = np.clip(np.random.normal(0, 2, 1)[0], -0.5, 0.5)
dy = np.clip(np.random.normal(0, 2, 1)[0], -0.5, 0.5)
vehicle_1_waypoints_perturbation.append((dx, dy))
vehicle_1 = Vehicle(
model='vehicle.audi.a2',
spawn_transform=vehicle_1_transform,
avoid_collision=True,
initial_speed=0,
trigger_distance=10,
waypoint_follower=waypoint_follower,
targeted_waypoint=targeted_waypoint,
dist_to_travel=vehicle_1_dist_to_travel,
target_direction=vehicle_1_target_direction,
targeted_speed=10,
after_trigger_behavior='stop',
color='(0, 0, 0)',
waypoints_perturbation=vehicle_1_waypoints_perturbation)
vehicle_list = [vehicle_1]
else:
static_list = []
pedestrian_list = []
vehicle_list = []
ego_car_waypoints_perturbation = []
customized_data = {
'friction': friction,
'static_list': static_list,
'pedestrian_list': pedestrian_list,
'vehicle_list': vehicle_list,
'center_transform': center_transform,
'using_customized_route_and_scenario': True,
'destination': route_waypoints[-1].location,
'sample_factor': sample_factor,
'ego_car_waypoints_perturbation': ego_car_waypoints_perturbation
}
try:
leaderboard_evaluator = LeaderboardEvaluator(arguments,
statistics_manager)
leaderboard_evaluator.run(arguments, customized_data)
except Exception as e:
traceback.print_exc()
finally:
del leaderboard_evaluator
def main():
description = "CARLA AD Leaderboard Evaluation: evaluate your Agent in CARLA scenarios\n"
# general parameters
parser = argparse.ArgumentParser(description=description,
formatter_class=RawTextHelpFormatter)
parser.add_argument('--host',
default='localhost',
help='IP of the host server (default: localhost)')
parser.add_argument('--port',
default='2000',
help='TCP port to listen to (default: 2000)')
parser.add_argument('--debug',
type=int,
help='Run with debug output',
default=0)
parser.add_argument('--spectator',
type=bool,
help='Switch spectator view on?',
default=True)
parser.add_argument(
'--record',
type=str,
default='',
help='Use CARLA recording feature to create a recording of the scenario'
)
parser.add_argument('--timeout',
default="30.0",
help='Set the CARLA client timeout value in seconds')
# simulation setup
parser.add_argument('--challenge-mode',
action="store_true",
help='Switch to challenge mode?')
parser.add_argument(
'--routes',
help=
'Name of the route to be executed. Point to the route_xml_file to be executed.',
required=True)
parser.add_argument(
'--scenarios',
help='Name of the scenario annotation file to be mixed with the route.',
required=True)
parser.add_argument('--repetitions',
type=int,
default=1,
help='Number of repetitions per route.')
# agent-related options
parser.add_argument("-a",
"--agent",
type=str,
help="Path to Agent's py file to evaluate",
required=True)
parser.add_argument("--agent-config",
type=str,
help="Path to Agent's configuration file",
default="")
parser.add_argument("--track",
type=str,
default='SENSORS',
help="Participation track: SENSORS, MAP")
parser.add_argument('--resume',
type=bool,
default=False,
help='Resume execution from last checkpoint?')
parser.add_argument(
"--checkpoint",
type=str,
default='./simulation_results.json',
help="Path to checkpoint used for saving statistics and resuming")
parser.add_argument('--simulation_number',
type=int,
help='Type the simulation folder to store the data')
parser.add_argument('--scene_number',
type=int,
default=1,
help='Type the scene number to be executed')
arguments = parser.parse_args()
statistics_manager = StatisticsManager()
try:
print(arguments.simulation_number)
path = "/home/scope/Carla/Resonate-Dynamic-Risk/resonate-carla/leaderboard/data/my_data/new-data/" #folder to save the simulation data
model_path = "/home/scope/Carla/Resonate-Dynamic-Risk/resonate-carla/leaderboard/team_code/detector_code/B-VAE-weights/" #trained model weights of the assurance monitor
os.makedirs(path + "simulation%d" % arguments.simulation_number + '/',
exist_ok=True)
filename = path + "simulation%d" % arguments.simulation_number + '/' + "simulation_data.csv" #csv to store weather data
data_folder = "/home/scope/Carla/Resonate-Dynamic-Risk/resonate-carla/leaderboard/data/my_data/new-data/simulation%d" % arguments.simulation_number #sub-folder for each scene run
route_folder = "/home/scope/Carla/Resonate-Dynamic-Risk/resonate-carla/leaderboard/data/my_routes/new-routes/simulation%d" % arguments.scene_number #scene route generated using our textX generator
leaderboard_evaluator = LeaderboardEvaluator(arguments,
statistics_manager)
leaderboard_evaluator.run(arguments, filename, data_folder,
route_folder, model_path)
fault_data_path = data_folder + "/fault_data.csv"
#data_plotting(data_folder,fault_data_path,filename)
except Exception as e:
traceback.print_exc()
finally:
del leaderboard_evaluator
def run_simulation(customized_data,
launch_server,
episode_max_time,
route_path,
route_str,
scenario_file,
ego_car_model,
background_vehicles=False,
changing_weather=False,
save_folder=None,
using_customized_route_and_scenario=True,
record_every_n_step=1):
arguments = arguments_info()
arguments.port = customized_data['port']
arguments.background_vehicles = background_vehicles
arguments.record_every_n_step = record_every_n_step
if save_folder:
arguments.save_folder = save_folder
# model path and checkpoint path
if ego_car_model == 'lbc':
arguments.agent = 'scenario_runner/team_code/image_agent.py'
arguments.agent_config = 'models/epoch=24.ckpt'
base_save_folder = 'collected_data_customized'
elif ego_car_model == 'auto_pilot':
arguments.agent = 'leaderboard/team_code/auto_pilot.py'
arguments.agent_config = ''
base_save_folder = 'collected_data_autopilot'
elif ego_car_model == 'pid_agent':
arguments.agent = 'scenario_runner/team_code/pid_agent.py'
arguments.agent_config = ''
base_save_folder = 'collected_data_pid_agent'
elif ego_car_model == 'map_model':
arguments.agent = 'scenario_runner/team_code/map_agent.py'
arguments.agent_config = 'models/stage1_default_50_epoch=16.ckpt'
base_save_folder = 'collected_data_map_model'
else:
print('unknown ego_car_model:', ego_car_model)
raise
from leaderboard.utils.statistics_manager import StatisticsManager
statistics_manager = StatisticsManager()
# Fixed Hyperparameters
sample_factor = 5
arguments.debug = 0
# Laundry Stuff-------------------------------------------------------------
arguments.weather_index = customized_data['weather_index']
os.environ['WEATHER_INDEX'] = str(customized_data['weather_index'])
# used to read scenario file
arguments.scenarios = scenario_file
# used to compose folder to save real-time data
os.environ['SAVE_FOLDER'] = arguments.save_folder
# used to read route to run; used to compose folder to save real-time data
arguments.routes = route_path
os.environ['ROUTES'] = arguments.routes
# used to record real time deviation data
arguments.deviations_folder = arguments.save_folder + '/' + pathlib.Path(
os.environ['ROUTES']).stem
# used to read real-time data
save_path = arguments.save_folder + '/' + pathlib.Path(
os.environ['ROUTES']).stem
arguments.changing_weather = changing_weather
# extract waypoints along route
import xml.etree.ElementTree as ET
tree = ET.parse(arguments.routes)
route_waypoints = []
for route in tree.iter("route"):
for waypoint in route.iter('waypoint'):
route_waypoints.append(
create_transform(float(waypoint.attrib['x']),
float(waypoint.attrib['y']),
float(waypoint.attrib['z']),
float(waypoint.attrib['pitch']),
float(waypoint.attrib['yaw']),
float(waypoint.attrib['roll'])))
# --------------------------------------------------------------------------
customized_data[
'using_customized_route_and_scenario'] = using_customized_route_and_scenario
customized_data['destination'] = route_waypoints[-1].location
customized_data['sample_factor'] = sample_factor
customized_data['number_of_attempts_to_request_actor'] = 10
try:
leaderboard_evaluator = LeaderboardEvaluator(arguments,
statistics_manager,
launch_server,
episode_max_time)
leaderboard_evaluator.run(arguments, customized_data)
except Exception as e:
traceback.print_exc()
finally:
del leaderboard_evaluator
# collect signals for estimating objectives
objectives, loc, object_type, route_completion = estimate_objectives(
save_path)
return objectives, loc, object_type, route_completion
def main():
description = "CARLA AD Leaderboard Evaluation: evaluate your Agent in CARLA scenarios\n"
# general parameters
parser = argparse.ArgumentParser(description=description,
formatter_class=RawTextHelpFormatter)
parser.add_argument('--host',
default='localhost',
help='IP of the host server (default: localhost)')
parser.add_argument('--port',
default='2000',
help='TCP port to listen to (default: 2000)')
parser.add_argument('--sync',
action='store_true',
help='Forces the simulation to run synchronously')
parser.add_argument('--debug',
type=int,
help='Run with debug output',
default=0)
parser.add_argument('--spectator',
type=bool,
help='Switch spectator view on?',
default=True)
parser.add_argument(
'--record',
type=str,
default='',
help='Use CARLA recording feature to create a recording of the scenario'
)
# modification: 30->15
parser.add_argument('--timeout',
default="30.0",
help='Set the CARLA client timeout value in seconds')
# simulation setup
parser.add_argument('--challenge-mode',
action="store_true",
help='Switch to challenge mode?')
parser.add_argument(
'--routes',
help=
'Name of the route to be executed. Point to the route_xml_file to be executed.',
required=True)
parser.add_argument(
'--scenarios',
help='Name of the scenario annotation file to be mixed with the route.',
required=True)
parser.add_argument('--repetitions',
type=int,
default=1,
help='Number of repetitions per route.')
# agent-related options
parser.add_argument("-a",
"--agent",
type=str,
help="Path to Agent's py file to evaluate",
required=True)
parser.add_argument("--agent-config",
type=str,
help="Path to Agent's configuration file",
default="")
parser.add_argument("--track",
type=str,
default='SENSORS',
help="Participation track: SENSORS, MAP")
parser.add_argument('--resume',
type=bool,
default=False,
help='Resume execution from last checkpoint?')
parser.add_argument(
"--checkpoint",
type=str,
default='./simulation_results.json',
help="Path to checkpoint used for saving statistics and resuming")
# addition
parser.add_argument("--weather-index",
type=int,
default=0,
help="see WEATHER for reference")
parser.add_argument("--save_folder",
type=str,
default='collected_data',
help="Path to save simulation data")
arguments = parser.parse_args()
# arguments.debug = True
statistics_manager = StatisticsManager()
# 0, 1, 2, 3, 10, 11, 14, 15, 19
# only 15 record vehicle's location for red light run
# weather_indexes is a subset of integers in [0, 20]
weather_indexes = [2]
routes = [i for i in range(0, 1)]
using_customized_route_and_scenario = True
multi_actors_scenarios = ['Scenario4']
if using_customized_route_and_scenario:
arguments.scenarios = 'leaderboard/data/customized_scenarios.json'
town_names = ['Town01']
scenarios = ['Scenario4']
directions = ['right']
routes = [0]
else:
route_prefix = 'leaderboard/data/routes/route_'
town_names = [None]
scenarios = [None]
directions = [None]
# if we use autopilot, we only need one run of weather index since we constantly switching weathers for diversity
if arguments.agent == 'leaderboard/team_code/auto_pilot.py':
weather_indexes = weather_indexes[:1]
time_start = time.time()
for town_name in town_names:
for scenario in scenarios:
if scenario not in multi_actors_scenarios:
directions = [None]
for dir in directions:
for weather_index in weather_indexes:
arguments.weather_index = weather_index
os.environ['WEATHER_INDEX'] = str(weather_index)
if using_customized_route_and_scenario:
town_scenario_direction = town_name + '/' + scenario
folder_1 = os.environ['SAVE_FOLDER'] + '/' + town_name
folder_2 = folder_1 + '/' + scenario
if not os.path.exists(folder_1):
os.mkdir(folder_1)
if not os.path.exists(folder_2):
os.mkdir(folder_2)
if scenario in multi_actors_scenarios:
town_scenario_direction += '/' + dir
folder_2 += '/' + dir
if not os.path.exists(folder_2):
os.mkdir(folder_2)
os.environ['SAVE_FOLDER'] = folder_2
arguments.save_folder = os.environ['SAVE_FOLDER']
route_prefix = 'leaderboard/data/customized_routes/' + town_scenario_direction + '/route_'
for route in routes:
# addition
# dx: -10~10
# dy: -5~50
# dyaw: -90~90
# d_activation_dist: -20~20
# _other_actor_target_velocity: 0~20
# actor_type: see specs.txt for a detailed options
dx = 0
dy = 0
dyaw = 0
d_activation_dist = 0
_other_actor_target_velocity = 5
actor_type = 'vehicle.diamondback.century'
if using_customized_route_and_scenario:
dx = 0
dy = 0
dyaw = -40
d_activation_dist = 0
_other_actor_target_velocity = 5
customized_data = {
'dx': dx,
'dy': dy,
'dyaw': dyaw,
'd_activation_dist': d_activation_dist,
'_other_actor_target_velocity':
_other_actor_target_velocity,
'using_customized_route_and_scenario':
using_customized_route_and_scenario,
'actor_type': actor_type
}
route_str = str(route)
if route < 10:
route_str = '0' + route_str
arguments.routes = route_prefix + route_str + '.xml'
os.environ['ROUTES'] = arguments.routes
try:
leaderboard_evaluator = LeaderboardEvaluator(
arguments, statistics_manager)
leaderboard_evaluator.run(arguments,
customized_data)
except Exception as e:
traceback.print_exc()
finally:
del leaderboard_evaluator
print('time elapsed :', time.time() - time_start)