def test_evaluate(self): """Tests that the evaluation is generated correctly.""" e = OdlcEvaluator(self.submitted_odlcs, self.real_odlcs) d = e.evaluate() td = {t.real_odlc: t for t in d.odlcs} self.assertEqual(self.submit1.pk, td[self.real1.pk].submitted_odlc) self.assertEqual(True, td[self.real1.pk].image_approved) self.assertEqual(1.0, td[self.real1.pk].classifications_ratio) self.assertEqual(0.0, td[self.real1.pk].geolocation_accuracy_ft) self.assertEqual(True, td[self.real1.pk].actionable_submission) self.assertEqual(1.0, td[self.real1.pk].classifications_score_ratio) self.assertEqual(1.0, td[self.real1.pk].geolocation_score_ratio) self.assertEqual(1.0, td[self.real1.pk].actionable_score_ratio) self.assertEqual(1.0, td[self.real1.pk].autonomous_score_ratio) self.assertAlmostEqual(1.0, td[self.real1.pk].score_ratio) self.assertEqual(self.submit2.pk, td[self.real2.pk].submitted_odlc) self.assertEqual(True, td[self.real2.pk].image_approved) self.assertEqual(0.6, td[self.real2.pk].classifications_ratio) self.assertAlmostEqual(109.444, td[self.real2.pk].geolocation_accuracy_ft, places=3) self.assertEqual(False, td[self.real2.pk].actionable_submission) self.assertEqual(0.6, td[self.real2.pk].classifications_score_ratio) self.assertAlmostEqual(0.270, td[self.real2.pk].geolocation_score_ratio, places=3) self.assertEqual(0.0, td[self.real2.pk].actionable_score_ratio) self.assertAlmostEqual(0.201, td[self.real2.pk].score_ratio, places=3) self.assertEqual(True, td[self.real6.pk].description_approved) self.assertAlmostEqual(0.350, d.score_ratio, places=3) self.assertEqual(2, d.unmatched_odlc_count)
def test_evaluate_no_submitted_odlcs(self): """Tests that evaluation works with no submitted odlcs.""" e = OdlcEvaluator([], self.real_odlcs) d = e.evaluate() self.assertEqual(0, d.matched_score_ratio) self.assertEqual(0, d.unmatched_odlc_count) self.assertEqual(6, len(d.odlcs))
def test_evaluate_no_real_odlcs(self): """Tests that evaluation works with no real odlcs.""" e = OdlcEvaluator(self.submitted_odlcs, []) d = e.evaluate() self.assertEqual(0, d.matched_score_ratio) self.assertEqual(7, d.unmatched_odlc_count) self.assertAlmostEqual(-0.35, d.score_ratio, places=3) self.assertEqual(0, len(d.odlcs))
def generate_feedback(mission_config, user, team_eval): """Generates mission feedback for the given team and mission. Args: mission_config: The mission to evaluate the team against. user: The team user object for which to evaluate and provide feedback. team_eval: The team evaluation to fill. """ feedback = team_eval.feedback # Find the user's flights. flight_periods = TakeoffOrLandingEvent.flights(mission_config, user) for period in flight_periods: if period.duration() is None: team_eval.warnings.append('Infinite flight period.') uas_period_logs = [ UasTelemetry.dedupe(logs) for logs in UasTelemetry.by_time_period(user, flight_periods) ] uas_logs = list(itertools.chain.from_iterable(uas_period_logs)) if not uas_logs: team_eval.warnings.append('No UAS telemetry logs.') # Determine interop telemetry rates. telem_max, telem_avg = UasTelemetry.rates( user, flight_periods, time_period_logs=uas_period_logs) if telem_max: feedback.uas_telemetry_time_max_sec = telem_max if telem_avg: feedback.uas_telemetry_time_avg_sec = telem_avg # Determine if the uas hit the waypoints. feedback.waypoints.extend( UasTelemetry.satisfied_waypoints( mission_config.home_pos, mission_config.mission_waypoints.order_by('order'), uas_logs)) # Evaluate the object detections. user_odlcs = Odlc.objects.filter(user=user).filter( mission=mission_config.pk).all() evaluator = OdlcEvaluator(user_odlcs, mission_config.odlcs.all(), flight_periods) feedback.odlc.CopyFrom(evaluator.evaluate()) # Determine collisions with stationary. for obst in mission_config.stationary_obstacles.all(): obst_eval = feedback.stationary_obstacles.add() obst_eval.id = obst.pk obst_eval.hit = obst.evaluate_collision_with_uas(uas_logs) # Add judge feedback. try: judge_feedback = MissionJudgeFeedback.objects.get( mission=mission_config.pk, user=user.pk) feedback.judge.CopyFrom(judge_feedback.proto()) except MissionJudgeFeedback.DoesNotExist: team_eval.warnings.append('No MissionJudgeFeedback for team.')
def test_match_odlcs(self): """Tests that matching odlcs produce maximal matches.""" e = OdlcEvaluator(self.submitted_odlcs, self.real_odlcs) self.assertDictEqual({ self.submit1: self.real1, self.submit2: self.real2, self.submit5: self.real5, self.submit6: self.real6, self.real1: self.submit1, self.real2: self.submit2, self.real5: self.submit5, self.real6: self.submit6, }, e.match_odlcs(self.submitted_odlcs, self.real_odlcs))
def generate_feedback(mission_config, user, team_eval): """Generates mission feedback for the given team and mission. Args: mission_config: The mission to evaluate the team against. user: The team user object for which to evaluate and provide feedback. team_eval: The team evaluation to fill. """ feedback = team_eval.feedback # Calculate the total mission clock time. missions = MissionClockEvent.missions(user) mission_clock_time = datetime.timedelta(seconds=0) for mission in missions: duration = mission.duration() if duration is None: team_eval.warnings.append('Infinite mission clock.') else: mission_clock_time += duration feedback.mission_clock_time_sec = mission_clock_time.total_seconds() # Calculate total time in air. flight_periods = TakeoffOrLandingEvent.flights(user) if flight_periods: flight_time = reduce(lambda x, y: x + y, [p.duration() for p in flight_periods]) feedback.flight_time_sec = flight_time.total_seconds() else: feedback.flight_time_sec = 0 # Find the user's flights. for period in flight_periods: if period.duration() is None: team_eval.warnings.append('Infinite flight period.') uas_period_logs = [ UasTelemetry.dedupe(logs) for logs in UasTelemetry.by_time_period(user, flight_periods) ] uas_logs = list(itertools.chain.from_iterable(uas_period_logs)) if not uas_logs: team_eval.warnings.append('No UAS telemetry logs.') # Determine interop telemetry rates. telem_max, telem_avg = UasTelemetry.rates(user, flight_periods, time_period_logs=uas_period_logs) if telem_max: feedback.uas_telemetry_time_max_sec = telem_max if telem_avg: feedback.uas_telemetry_time_avg_sec = telem_avg # Determine if the uas went out of bounds. This must be done for # each period individually so time between periods isn't counted as # out of bounds time. Note that this calculates reported time out # of bounds, not actual or possible time spent out of bounds. out_of_bounds = datetime.timedelta(seconds=0) feedback.boundary_violations = 0 for logs in uas_period_logs: bv, bt = FlyZone.out_of_bounds(mission_config.fly_zones.all(), logs) feedback.boundary_violations += bv out_of_bounds += bt feedback.out_of_bounds_time_sec = out_of_bounds.total_seconds() # Determine if the uas hit the waypoints. feedback.waypoints.extend( UasTelemetry.satisfied_waypoints( mission_config.home_pos, mission_config.mission_waypoints.order_by('order'), uas_logs)) # Evaluate the object detections. user_odlcs = Odlc.objects.filter(user=user).all() evaluator = OdlcEvaluator(user_odlcs, mission_config.odlcs.all()) feedback.odlc.CopyFrom(evaluator.evaluate()) # Determine collisions with stationary and moving obstacles. for obst in mission_config.stationary_obstacles.all(): obst_eval = feedback.stationary_obstacles.add() obst_eval.id = obst.pk obst_eval.hit = obst.evaluate_collision_with_uas(uas_logs) for obst in mission_config.moving_obstacles.all(): obst_eval = feedback.moving_obstacles.add() obst_eval.id = obst.pk obst_eval.hit = obst.evaluate_collision_with_uas(uas_logs) # Add judge feedback. try: judge_feedback = MissionJudgeFeedback.objects.get( mission=mission_config.pk, user=user.pk) feedback.judge.CopyFrom(judge_feedback.proto()) except MissionJudgeFeedback.DoesNotExist: team_eval.warnings.append('No MissionJudgeFeedback for team.') # Sanity check mission time. judge_mission_clock = (feedback.judge.flight_time_sec + feedback.judge.post_process_time_sec) if abs(feedback.mission_clock_time_sec - judge_mission_clock) > 30: team_eval.warnings.append( 'Mission clock differs between interop and judge.')
def test_match_value(self): """Tests the match value for two odlcs.""" e = OdlcEvaluator(self.submitted_odlcs, self.real_odlcs) self.assertAlmostEqual( 1.0, e.evaluate_match(self.submit1, self.real1).score_ratio, places=3) self.assertAlmostEqual( 0.174, e.evaluate_match(self.submit2, self.real2).score_ratio, places=3) self.assertAlmostEqual( 0.0, e.evaluate_match(self.submit3, self.real3).score_ratio, places=3) self.assertAlmostEqual( 0.0, e.evaluate_match(self.submit4, self.real4).score_ratio, places=3) self.assertAlmostEqual( 0.3, e.evaluate_match(self.submit5, self.real5).score_ratio, places=3) self.assertAlmostEqual( 0.7, e.evaluate_match(self.submit6, self.real6).score_ratio, places=3) self.assertAlmostEqual( 0.240, e.evaluate_match(self.submit7, self.real1).score_ratio, places=3) self.assertAlmostEqual( 0.814, e.evaluate_match(self.submit1, self.real2).score_ratio, places=3) self.assertAlmostEqual( 0.32, e.evaluate_match(self.submit2, self.real1).score_ratio, places=3)