def test_compute(self): # This is is the last one, generate many cases, corner cases, to be tested. metrics_obj = Metrics(self._metrics_parameters, [3]) # Lets start testing a general file, not from a real run # The case is basically an empty case poses_to_test = [[24, 32], [34, 36], [54, 67]] path = self._generate_test_case(poses_to_test) summary_dict = metrics_obj.compute(path) number_of_colisions_vehicles = sum_matrix( summary_dict['collision_vehicles'][1.0]) number_of_colisions_general = sum_matrix( summary_dict['collision_other'][1.0]) number_of_colisions_pedestrians = sum_matrix( summary_dict['collision_pedestrians'][1.0]) number_of_intersection_offroad = sum_matrix( summary_dict['intersection_offroad'][1.0]) number_of_intersection_otherlane = sum_matrix( summary_dict['intersection_otherlane'][1.0]) self.assertEqual(number_of_colisions_vehicles, 0) self.assertEqual(number_of_colisions_general, 0) self.assertEqual(number_of_colisions_pedestrians, 0) self.assertEqual(number_of_intersection_offroad, 0) self.assertEqual(number_of_intersection_otherlane, 0) # Now lets make a collision test on a premade file path = 'test/unit_tests/test_data/testfile_collisions' summary_dict = metrics_obj.compute(path) number_of_colisions_vehicles = sum_matrix( summary_dict['collision_vehicles'][3.0]) number_of_colisions_general = sum_matrix( summary_dict['collision_other'][3.0]) number_of_colisions_pedestrians = sum_matrix( summary_dict['collision_pedestrians'][3.0]) number_of_intersection_offroad = sum_matrix( summary_dict['intersection_offroad'][3.0]) number_of_intersection_otherlane = sum_matrix( summary_dict['intersection_otherlane'][3.0]) self.assertEqual(number_of_colisions_vehicles, 2) self.assertEqual(number_of_colisions_general, 9) self.assertEqual(number_of_colisions_pedestrians, 0) self.assertEqual(number_of_intersection_offroad, 1) self.assertEqual(number_of_intersection_otherlane, 3)
def test_compute(self): # This is is the last one, generate many cases, corner cases, to be tested. metrics_obj = Metrics(self._metrics_parameters,[3]) # Lets start testing a general file, not from a real run # The case is basically an empty case poses_to_test = [[24, 32], [34, 36], [54, 67]] path = self._generate_test_case(poses_to_test) summary_dict = metrics_obj.compute(path) number_of_colisions_vehicles = sum_matrix(summary_dict['collision_vehicles'][1.0]) number_of_colisions_general = sum_matrix(summary_dict['collision_other'][1.0]) number_of_colisions_pedestrians = sum_matrix(summary_dict['collision_pedestrians'][1.0]) number_of_intersection_offroad = sum_matrix(summary_dict['intersection_offroad'][1.0]) number_of_intersection_otherlane = sum_matrix(summary_dict['intersection_otherlane'][1.0]) self.assertEqual(number_of_colisions_vehicles, 0) self.assertEqual(number_of_colisions_general, 0) self.assertEqual(number_of_colisions_pedestrians, 0) self.assertEqual(number_of_intersection_offroad, 0) self.assertEqual(number_of_intersection_otherlane, 0) # Now lets make a collision test on a premade file path = 'test/unit_tests/test_data/testfile_collisions' summary_dict = metrics_obj.compute(path) number_of_colisions_vehicles = sum_matrix(summary_dict['collision_vehicles'][3.0]) number_of_colisions_general = sum_matrix(summary_dict['collision_other'][3.0]) number_of_colisions_pedestrians = sum_matrix(summary_dict['collision_pedestrians'][3.0]) number_of_intersection_offroad = sum_matrix(summary_dict['intersection_offroad'][3.0]) number_of_intersection_otherlane = sum_matrix(summary_dict['intersection_otherlane'][3.0]) self.assertEqual(number_of_colisions_vehicles, 2) self.assertEqual(number_of_colisions_general, 9) self.assertEqual(number_of_colisions_pedestrians, 0) self.assertEqual(number_of_intersection_offroad, 1) self.assertEqual(number_of_intersection_otherlane, 3)
def benchmark_agent(self, experiment_suite, agent, client): """ Function to benchmark the agent. It first check the log file for this benchmark. if it exist it continues from the experiment where it stopped. Args: experiment_suite agent: an agent object with the run step class implemented. client: Return: A dictionary with all the metrics computed from the agent running the set of experiments. """ # Instantiate a metric object that will be used to compute the metrics for # the benchmark afterwards. metrics_object = Metrics(experiment_suite.metrics_parameters, experiment_suite.dynamic_tasks) # Function return the current pose and task for this benchmark. start_pose, start_experiment = self._recording.get_pose_and_experiment( experiment_suite.get_number_of_poses_task()) logging.info('START') for experiment in experiment_suite.get_experiments( )[int(start_experiment):]: positions = client.load_settings( experiment.conditions).player_start_spots self._recording.log_start(experiment.task) for pose in experiment.poses[start_pose:]: for rep in range(experiment.repetitions): start_index = pose[0] end_index = pose[1] client.start_episode(start_index) self._episode_number += 1 # Print information on logging.info('======== !!!! ==========') logging.info('Episode Number: %d', self._episode_number) logging.info(' Start Position %d End Position %d ', start_index, end_index) self._recording.log_poses(start_index, end_index, experiment.Conditions.WeatherId) # Calculate the initial distance for this episode initial_distance = \ sldist( [positions[start_index].location.x, positions[start_index].location.y], [positions[end_index].location.x, positions[end_index].location.y]) time_out = experiment_suite.calculate_time_out( self._get_shortest_path(positions[start_index], positions[end_index])) # running the agent (result, reward_vec, control_vec, final_time, remaining_distance) = \ self._run_navigation_episode( agent, client, time_out, positions[end_index], str(experiment.Conditions.WeatherId) + '_' + str(experiment.task) + '_' + str(start_index) + '.' + str(end_index)) # Write the general status of the just ran episode self._recording.write_summary_results( experiment, pose, rep, initial_distance, remaining_distance, final_time, time_out, result) # Write the details of this episode. self._recording.write_measurements_results( experiment, rep, pose, reward_vec, control_vec) if result > 0: logging.info( '+++++ Target achieved in %f seconds! +++++', final_time) else: logging.info('----- Timeout! -----') start_pose = 0 self._recording.log_end() return metrics_object.compute(self._recording.path)
def benchmark_agent(self, experiment_suite, agent, client): """ Function to benchmark the agent. It first check the log file for this benchmark. if it exist it continues from the experiment where it stopped. Args: experiment_suite agent: an agent object with the run step class implemented. client: Return: A dictionary with all the metrics computed from the agent running the set of experiments. """ # Instantiate a metric object that will be used to compute the metrics for # the benchmark afterwards. data = OrderedDict() metrics_object = Metrics(experiment_suite.metrics_parameters, experiment_suite.dynamic_tasks) # Function return the current pose and task for this benchmark. start_pose, start_experiment = self._recording.get_pose_and_experiment( experiment_suite.get_number_of_poses_task()) logging.info('START') cols = [ 'Start_position', 'End_position', 'Total-Distance', 'Time', "Distance-Travelled", "Follow_lane", "Straight", "Left", "Right", "Avg-Speed", "Collision-Pedestrian", "Collision-Vehicle", "Collision-Other", "Intersection-Lane", "Intersection-Offroad", "Traffic_Light_Infraction", "Success" ] df = pd.DataFrame(columns=cols) ts = str(int(time.time())) for experiment in experiment_suite.get_experiments( )[int(start_experiment):]: positions = client.load_settings( experiment.conditions).player_start_spots self._recording.log_start(experiment.task) for pose in experiment.poses[start_pose:]: data['Start_position'] = pose[0] data['End_position'] = pose[1] #print(df) for rep in range(experiment.repetitions): start_index = pose[0] end_index = pose[1] client.start_episode(start_index) # Print information on logging.info('======== !!!! ==========') logging.info(' Start Position %d End Position %d ', start_index, end_index) self._recording.log_poses(start_index, end_index, experiment.Conditions.WeatherId) # Calculate the initial distance for this episode initial_distance = \ sldist( [positions[start_index].location.x, positions[start_index].location.y], [positions[end_index].location.x, positions[end_index].location.y]) data["Total-Distance"] = initial_distance time_out = experiment_suite.calculate_time_out( self._get_shortest_path(positions[start_index], positions[end_index])) # running the agent (result, reward_vec, control_vec, final_time, remaining_distance, data) = \ self._run_navigation_episode( agent, client, time_out, positions[end_index], str(experiment.Conditions.WeatherId) + '_' + str(experiment.task) + '_' + str(start_index) + '.' + str(end_index) , data) data["Time"] = final_time # Write the general status of the just ran episode self._recording.write_summary_results( experiment, pose, rep, initial_distance, remaining_distance, final_time, time_out, result) # Write the details of this episode. self._recording.write_measurements_results( experiment, rep, pose, reward_vec, control_vec) data['Success'] = result if result > 0: logging.info( '+++++ Target achieved in %f seconds! +++++', final_time) else: logging.info('----- Timeout! -----') #data['Start_position'] = start_index #data['End_position'] = end_index #df = df.append(data , ignore_index=True) #print(df) data["Avg-Speed"] = 3.6 * (data['Total-Distance'] / data['Time']) df = df.append(data, ignore_index=True) df = df[cols] try: df.to_csv("Test_result_" + ts + '.csv', columns=cols, index=True) except: print("File being used will save later") start_pose = 0 self._recording.log_end() return metrics_object.compute(self._recording.path)
else: logging.info('----- Collision! -----') # Write the details of this episode. print('end of the while loop') recording.write_measurements_results(experiments[exp_idx], rep, pose, reward_vec, control_vec) print('experiment ended') recording.log_end() except: break recording.write_measurements_results(experiments[exp_idx], rep, pose, reward_vec, control_vec) benchmark_summary = metrics_object.compute(recording.path) print("") print("") print( "----- Printing results for training weathers (Seen in Training) -----" ) print("") print("") print_summary(benchmark_summary, experiment_suite.train_weathers, recording.path) print("") print("") print( "----- Printing results for test weathers (Unseen in Training) -----")
# This will make life easier for calculating std dataListTable1 = [[] for i in range(len(tasksSuccessRate))] dataListTable2 = [[[] for i in range(len(tasksSuccessRate))] for i in range(len(tasksInfractions))] dataListTable3 = [[[] for i in range(len(tasksSuccessRate))] for i in range(len(tasksInfractions))] #logging.debug("Data list table 2 init: %s",dataListTable2) # calculate metrics : episodes_fully_completed for p in allPath: if addSlashFlag == True: path = args.path + '/' + extractedPath + p else: path = args.path + extractedPath + p metrics_summary = metrics_object.compute(path) number_of_tasks = len( list(metrics_summary[metrics_to_average[0]].items())[0][1]) values = metrics_summary[ metrics_to_average[0]] # episodes_fully_completed if (table1Flag): logging.debug("Working on table 1 ...") metric_sum_values = np.zeros(number_of_tasks) for w, tasks in values.items(): if w in set(weathers): count = 0 for tIdx, t in enumerate(tasks): #print(weathers[tIdx]) #float(sum(t)) / float(len(t))) metric_sum_values[count] += (float(sum(t)) / float( len(t))) * 1.0 / float(len(weathers))
def benchmark_agent(self, experiment_suite, agent, client): """ Function to benchmark the agent. It first check the log file for this benchmark. if it exist it continues from the experiment where it stopped. Args: experiment_suite agent: an agent object with the run step class implemented. client: Return: A dictionary with all the metrics computed from the agent running the set of experiments. """ # Instantiate a metric object that will be used to compute the metrics for # the benchmark afterwards. metrics_object = Metrics(experiment_suite.metrics_parameters, experiment_suite.dynamic_tasks) # Function return the current pose and task for this benchmark. start_pose, start_experiment = self._recording.get_pose_and_experiment( experiment_suite.get_number_of_poses_task()) logging.info('START') for experiment in experiment_suite.get_experiments()[int(start_experiment):]: positions = client.load_settings( experiment.conditions).player_start_spots self._recording.log_start(experiment.task) for pose in experiment.poses[start_pose:]: for rep in range(experiment.repetitions): start_index = pose[0] end_index = pose[1] client.start_episode(start_index) # Print information on logging.info('======== !!!! ==========') logging.info(' Start Position %d End Position %d ', start_index, end_index) self._recording.log_poses(start_index, end_index, experiment.Conditions.WeatherId) # Calculate the initial distance for this episode initial_distance = \ sldist( [positions[start_index].location.x, positions[start_index].location.y], [positions[end_index].location.x, positions[end_index].location.y]) time_out = experiment_suite.calculate_time_out( self._get_shortest_path(positions[start_index], positions[end_index])) # running the agent (result, reward_vec, control_vec, final_time, remaining_distance) = \ self._run_navigation_episode( agent, client, time_out, positions[end_index], str(experiment.Conditions.WeatherId) + '_' + str(experiment.task) + '_' + str(start_index) + '.' + str(end_index)) # Write the general status of the just ran episode self._recording.write_summary_results( experiment, pose, rep, initial_distance, remaining_distance, final_time, time_out, result) # Write the details of this episode. self._recording.write_measurements_results(experiment, rep, pose, reward_vec, control_vec) if result > 0: logging.info('+++++ Target achieved in %f seconds! +++++', final_time) else: logging.info('----- Timeout! -----') start_pose = 0 self._recording.log_end() return metrics_object.compute(self._recording.path)