def test_objects(self):
"""Test the object scoring."""
objects = self.eval.score.object
mission_evaluation.score_team(self.eval)
self.assertAlmostEqual(0.4, objects.characteristics)
self.assertAlmostEqual(0.4, objects.geolocation)
self.assertAlmostEqual(0.5, objects.actionable)
self.assertAlmostEqual(0.5, objects.autonomy)
self.assertAlmostEqual(0.1, objects.extra_object_penalty)
self.assertAlmostEqual(0.46, objects.score_ratio)
def test_objects(self):
"""Test the object scoring."""
objects = self.eval.score.object
mission_evaluation.score_team(self.eval)
self.assertAlmostEqual(0.4, objects.characteristics)
self.assertAlmostEqual(0.4, objects.geolocation)
self.assertAlmostEqual(0.5, objects.actionable)
self.assertAlmostEqual(0.5, objects.autonomy)
self.assertAlmostEqual(0.5, objects.interoperability)
self.assertAlmostEqual(0.1, objects.extra_object_penalty)
self.assertAlmostEqual(0.46, objects.score_ratio)
def test_timeline(self):
"""Test the timeline scoring."""
judge = self.eval.feedback.judge
timeline = self.eval.score.timeline
mission_evaluation.score_team(self.eval)
self.assertAlmostEqual(0.93333333, timeline.mission_time)
self.assertAlmostEqual(0, timeline.mission_penalty)
self.assertAlmostEqual(0, timeline.timeout)
self.assertAlmostEqual(0.74666666666, timeline.score_ratio)
judge.flight_time_sec = 60 * 20
judge.post_process_time_sec = 0
judge.used_timeout = False
mission_evaluation.score_team(self.eval)
self.assertAlmostEqual(1, timeline.mission_time)
self.assertAlmostEqual(0, timeline.mission_penalty)
self.assertAlmostEqual(1, timeline.timeout)
self.assertAlmostEqual(1, timeline.score_ratio)
judge.flight_time_sec = 60 * 25
judge.post_process_time_sec = 60 * 5
judge.used_timeout = True
mission_evaluation.score_team(self.eval)
self.assertAlmostEqual(0.5, timeline.mission_time)
self.assertAlmostEqual(0, timeline.mission_penalty)
self.assertAlmostEqual(0, timeline.timeout)
self.assertAlmostEqual(0.4, timeline.score_ratio)
judge.flight_time_sec = 60 * 30
judge.post_process_time_sec = 60 * 10
judge.used_timeout = True
mission_evaluation.score_team(self.eval)
self.assertAlmostEqual(0, timeline.mission_time)
self.assertAlmostEqual(0, timeline.mission_penalty)
self.assertAlmostEqual(0, timeline.timeout)
self.assertAlmostEqual(0, timeline.score_ratio)
judge.flight_time_sec = 60 * 35
judge.post_process_time_sec = 60 * 20
judge.used_timeout = True
mission_evaluation.score_team(self.eval)
self.assertAlmostEqual(0, timeline.mission_time)
self.assertAlmostEqual(27, timeline.mission_penalty)
self.assertAlmostEqual(0, timeline.timeout)
self.assertAlmostEqual(-27, timeline.score_ratio)
def test_timeline(self):
"""Test the timeline scoring."""
judge = self.eval.feedback.judge
timeline = self.eval.score.timeline
mission_evaluation.score_team(self.eval)
self.assertAlmostEqual(0.84888889, timeline.mission_time)
self.assertAlmostEqual(0, timeline.mission_penalty)
self.assertAlmostEqual(0, timeline.timeout)
self.assertAlmostEqual(0.67911111, timeline.score_ratio)
judge.flight_time_sec = 60 * 50
judge.used_timeout = False
mission_evaluation.score_team(self.eval)
self.assertAlmostEqual(0.0, timeline.mission_time)
self.assertAlmostEqual(4.32, timeline.mission_penalty)
self.assertAlmostEqual(1, timeline.timeout)
self.assertAlmostEqual(-4.12, timeline.score_ratio)
def test_timeline(self):
"""Test the timeline scoring."""
judge = self.eval.feedback.judge
timeline = self.eval.score.timeline
mission_evaluation.score_team(self.eval)
self.assertAlmostEqual(0.84888889, timeline.mission_time)
self.assertAlmostEqual(0, timeline.mission_penalty)
self.assertAlmostEqual(0, timeline.timeout)
self.assertAlmostEqual(0.67911111, timeline.score_ratio)
judge.flight_time_sec = 60 * 50
judge.used_timeout = False
mission_evaluation.score_team(self.eval)
self.assertAlmostEqual(0.0, timeline.mission_time)
self.assertAlmostEqual(4.32, timeline.mission_penalty)
self.assertAlmostEqual(1, timeline.timeout)
self.assertAlmostEqual(-4.12, timeline.score_ratio)
def test_non_negative(self):
"""Test that total score doesn't go negative."""
self.eval = interop_admin_api_pb2.MissionEvaluation()
self.eval.team = 'team'
feedback = self.eval.feedback
feedback.uas_telemetry_time_max_sec = 100.0
feedback.uas_telemetry_time_avg_sec = 100.0
wpt = feedback.waypoints.add()
wpt.score_ratio = 0
wpt = feedback.waypoints.add()
wpt.score_ratio = 0
obs = feedback.stationary_obstacles.add()
obs.hit = True
obs = feedback.stationary_obstacles.add()
obs.hit = True
odlcs = feedback.odlc
odlcs.score_ratio = 0
odlcs.extra_object_penalty_ratio = 1.0
t = odlcs.odlcs.add()
t.score_ratio = 0
t.classifications_score_ratio = 0
t.geolocation_score_ratio = 0
t.actionable_score_ratio = 0
t.autonomous_score_ratio = 0
judge = feedback.judge
judge.flight_time_sec = 60 * 100
judge.post_process_time_sec = 60 * 100
judge.used_timeout = True
judge.min_auto_flight_time = True
judge.safety_pilot_takeovers = 10
judge.waypoints_captured = 0
judge.out_of_bounds = 10
judge.unsafe_out_of_bounds = 5
judge.things_fell_off_uas = True
judge.crashed = False
judge.air_drop_accuracy = interop_admin_api_pb2.MissionJudgeFeedback.INSUFFICIENT_ACCURACY
judge.ugv_drove_to_location = False
judge.operational_excellence_percent = 0
mission_evaluation.score_team(self.eval)
self.assertAlmostEqual(0, self.eval.score.score_ratio)
def test_obstacles(self):
"""Test the obstacle scoring."""
feedback = self.eval.feedback
avoid = self.eval.score.obstacle_avoidance
mission_evaluation.score_team(self.eval)
self.assertTrue(avoid.telemetry_prerequisite)
self.assertAlmostEqual(0.5, avoid.stationary_obstacle)
self.assertAlmostEqual(0, avoid.moving_obstacle)
self.assertAlmostEqual(0.25, avoid.score_ratio)
feedback.stationary_obstacles[0].hit = False
feedback.moving_obstacles[0].hit = False
mission_evaluation.score_team(self.eval)
self.assertTrue(avoid.telemetry_prerequisite)
self.assertAlmostEqual(1, avoid.stationary_obstacle)
self.assertAlmostEqual(0.5, avoid.moving_obstacle)
self.assertAlmostEqual(0.75, avoid.score_ratio)
feedback.uas_telemetry_time_avg_sec = 1.01
mission_evaluation.score_team(self.eval)
self.assertFalse(avoid.telemetry_prerequisite)
self.assertAlmostEqual(0, avoid.stationary_obstacle)
self.assertAlmostEqual(0, avoid.moving_obstacle)
self.assertAlmostEqual(0, avoid.score_ratio)
def test_obstacles(self):
"""Test the obstacle scoring."""
feedback = self.eval.feedback
avoid = self.eval.score.obstacle_avoidance
mission_evaluation.score_team(self.eval)
self.assertTrue(avoid.telemetry_prerequisite)
self.assertAlmostEqual(0.5, avoid.stationary_obstacle)
self.assertAlmostEqual(0, avoid.moving_obstacle)
self.assertAlmostEqual(0.25, avoid.score_ratio)
feedback.stationary_obstacles[0].hit = False
feedback.moving_obstacles[0].hit = False
mission_evaluation.score_team(self.eval)
self.assertTrue(avoid.telemetry_prerequisite)
self.assertAlmostEqual(1, avoid.stationary_obstacle)
self.assertAlmostEqual(0.5, avoid.moving_obstacle)
self.assertAlmostEqual(0.75, avoid.score_ratio)
feedback.uas_telemetry_time_avg_sec = 1.01
mission_evaluation.score_team(self.eval)
self.assertFalse(avoid.telemetry_prerequisite)
self.assertAlmostEqual(0, avoid.stationary_obstacle)
self.assertAlmostEqual(0, avoid.moving_obstacle)
self.assertAlmostEqual(0, avoid.score_ratio)
def test_map(self):
"""Test the map scoring."""
mf = self.eval.feedback.map
me = self.eval.score.map
mf.quality = interop_admin_api_pb2.MapEvaluation.MapQuality.INSUFFICIENT
mission_evaluation.score_team(self.eval)
self.assertAlmostEqual(0.0, me.score_ratio)
mf.quality = interop_admin_api_pb2.MapEvaluation.MapQuality.MEDIUM
mission_evaluation.score_team(self.eval)
self.assertAlmostEqual(0.5, me.score_ratio)
mf.quality = interop_admin_api_pb2.MapEvaluation.MapQuality.HIGH
mission_evaluation.score_team(self.eval)
self.assertAlmostEqual(1.0, me.score_ratio)
def test_air_delivery(self):
"""Test the air delivery scoring."""
judge = self.eval.feedback.judge
air = self.eval.score.air_delivery
mission_evaluation.score_team(self.eval)
self.assertAlmostEqual(50, air.delivery_accuracy)
self.assertAlmostEqual(0.6666667, air.score_ratio)
judge.air_delivery_accuracy_ft = 200
mission_evaluation.score_team(self.eval)
self.assertAlmostEqual(200, air.delivery_accuracy)
self.assertAlmostEqual(0, air.score_ratio)
judge.air_delivery_accuracy_ft = 0
mission_evaluation.score_team(self.eval)
self.assertAlmostEqual(1, air.score_ratio)
def test_air_delivery(self):
"""Test the air delivery scoring."""
judge = self.eval.feedback.judge
air = self.eval.score.air_delivery
mission_evaluation.score_team(self.eval)
self.assertAlmostEqual(50, air.delivery_accuracy)
self.assertAlmostEqual(0.6666667, air.score_ratio)
judge.air_delivery_accuracy_ft = 200
mission_evaluation.score_team(self.eval)
self.assertAlmostEqual(200, air.delivery_accuracy)
self.assertAlmostEqual(0, air.score_ratio)
judge.air_delivery_accuracy_ft = 0
mission_evaluation.score_team(self.eval)
self.assertAlmostEqual(1, air.score_ratio)
def test_air_drop(self):
"""Test the air drop scoring."""
judge = self.eval.feedback.judge
air = self.eval.score.air_drop
mission_evaluation.score_team(self.eval)
self.assertAlmostEqual(0.25, air.drop_accuracy)
self.assertAlmostEqual(0, air.drive_to_location)
self.assertAlmostEqual(0.125, air.score_ratio)
judge.air_drop_accuracy = interop_admin_api_pb2.MissionJudgeFeedback.WITHIN_05_FT
judge.ugv_drove_to_location = True
mission_evaluation.score_team(self.eval)
self.assertAlmostEqual(1, air.drop_accuracy)
self.assertAlmostEqual(1, air.drive_to_location)
self.assertAlmostEqual(1, air.score_ratio)
judge.air_drop_accuracy = interop_admin_api_pb2.MissionJudgeFeedback.WITHIN_15_FT
judge.ugv_drove_to_location = True
mission_evaluation.score_team(self.eval)
self.assertAlmostEqual(0.5, air.drop_accuracy)
self.assertAlmostEqual(1, air.drive_to_location)
self.assertAlmostEqual(0.75, air.score_ratio)
def test_autonomous_flight(self):
"""Test the autonomous flight scoring."""
feedback = self.eval.feedback
judge = feedback.judge
flight = self.eval.score.autonomous_flight
mission_evaluation.score_team(self.eval)
self.assertTrue(flight.telemetry_prerequisite)
self.assertAlmostEqual(0.8, flight.flight)
self.assertAlmostEqual(1, flight.waypoint_capture)
self.assertAlmostEqual(0.35, flight.waypoint_accuracy)
self.assertAlmostEqual(0.3, flight.out_of_bounds_penalty)
self.assertEqual(0, flight.things_fell_off_penalty)
self.assertEqual(0, flight.crashed_penalty)
self.assertAlmostEqual(0.295, flight.score_ratio)
feedback.waypoints[1].score_ratio = 1
judge.waypoints_captured = 1
judge.safety_pilot_takeovers = 0
judge.out_of_bounds = 0
judge.unsafe_out_of_bounds = 0
mission_evaluation.score_team(self.eval)
self.assertAlmostEqual(1, flight.flight)
self.assertAlmostEqual(0.5, flight.waypoint_capture)
self.assertAlmostEqual(0.75, flight.waypoint_accuracy)
self.assertAlmostEqual(0, flight.out_of_bounds_penalty)
self.assertAlmostEqual(0.825, flight.score_ratio)
judge.things_fell_off_uas = True
judge.crashed = True
mission_evaluation.score_team(self.eval)
self.assertAlmostEqual(0.25, flight.things_fell_off_penalty)
self.assertAlmostEqual(0.35, flight.crashed_penalty)
self.assertAlmostEqual(0.225, flight.score_ratio)
judge.min_auto_flight_time = False
mission_evaluation.score_team(self.eval)
self.assertAlmostEqual(0, flight.flight)
judge.flight_time_sec = 0
mission_evaluation.score_team(self.eval)
self.assertAlmostEqual(0, flight.flight)
self.assertAlmostEqual(0, self.eval.score.score_ratio)
feedback.uas_telemetry_time_avg_sec = 2.0
mission_evaluation.score_team(self.eval)
self.assertFalse(flight.telemetry_prerequisite)
self.assertAlmostEqual(0, flight.waypoint_accuracy)
def test_total(self):
"""Test the total scoring."""
mission_evaluation.score_team(self.eval)
self.assertAlmostEqual(0.455077778, self.eval.score.score_ratio)
def test_operational(self):
"""Test the operational excellence scoring."""
operational = self.eval.score.operational_excellence
mission_evaluation.score_team(self.eval)
self.assertAlmostEqual(0.9, operational.score_ratio)
def test_autonomous_flight(self):
"""Test the autonomous flight scoring."""
feedback = self.eval.feedback
judge = feedback.judge
flight = self.eval.score.autonomous_flight
mission_evaluation.score_team(self.eval)
self.assertTrue(flight.telemetry_prerequisite)
self.assertAlmostEqual(0.8, flight.flight)
self.assertAlmostEqual(1, flight.waypoint_capture)
self.assertAlmostEqual(0.35, flight.waypoint_accuracy)
self.assertAlmostEqual(0.3, flight.out_of_bounds_penalty)
self.assertEqual(0, flight.things_fell_off_penalty)
self.assertEqual(0, flight.crashed_penalty)
self.assertAlmostEqual(0.295, flight.score_ratio)
feedback.waypoints[1].score_ratio = 1
judge.waypoints_captured = 1
judge.safety_pilot_takeovers = 0
judge.out_of_bounds = 0
judge.unsafe_out_of_bounds = 0
mission_evaluation.score_team(self.eval)
self.assertAlmostEqual(1, flight.flight)
self.assertAlmostEqual(0.5, flight.waypoint_capture)
self.assertAlmostEqual(0.75, flight.waypoint_accuracy)
self.assertAlmostEqual(0, flight.out_of_bounds_penalty)
self.assertAlmostEqual(0.825, flight.score_ratio)
judge.things_fell_off_uas = True
judge.crashed = True
mission_evaluation.score_team(self.eval)
self.assertAlmostEqual(0.25, flight.things_fell_off_penalty)
self.assertAlmostEqual(0.35, flight.crashed_penalty)
self.assertAlmostEqual(0.225, flight.score_ratio)
judge.min_auto_flight_time = False
mission_evaluation.score_team(self.eval)
self.assertAlmostEqual(0, flight.flight)
judge.flight_time_sec = 0
mission_evaluation.score_team(self.eval)
self.assertAlmostEqual(0, flight.flight)
self.assertAlmostEqual(0, self.eval.score.score_ratio)
feedback.uas_telemetry_time_avg_sec = 2.0
mission_evaluation.score_team(self.eval)
self.assertFalse(flight.telemetry_prerequisite)
self.assertAlmostEqual(0, flight.waypoint_accuracy)
def test_operational(self):
"""Test the operational excellence scoring."""
operational = self.eval.score.operational_excellence
mission_evaluation.score_team(self.eval)
self.assertAlmostEqual(0.9, operational.score_ratio)
def test_total(self):
"""Test the total scoring."""
mission_evaluation.score_team(self.eval)
self.assertAlmostEqual(0.455077778, self.eval.score.score_ratio)
