Example #1
0
    def evaluate_teams(self):
        """Evaluates the teams (non admin users) of the competition.

        Returns:
            A map from user to evaluate data. The evaluation data has the
            following map structure:
            {
                'waypoints_satisfied': {
                    id: Boolean,
                }
                'out_of_bounds_time': Seconds spent out of bounds,
                'interop_times': {
                    'server_info': {'max': Value, 'avg': Value},
                    'obst_info': {'max': Value, 'avg': Value},
                    'uas_telem': {'max': Value, 'avg': Value},
                },
                'stationary_obst_collision': {
                    id: Boolean
                },
                'moving_obst_collision': {
                    id: Boolean
                }
            }
        """
        # Start a results map from user to evaluation data
        results = {}

        # Fill in evaluation data for each user except admins
        users = User.objects.all()
        logger.info("Starting team evaluations.")

        for user in users:
            # Ignore admins.
            if user.is_superuser:
                continue

            logger.info("Evaluation starting for user: %s." % user.username)

            # Start the evaluation data structure.
            eval_data = results.setdefault(user, {})

            # Find the user's flights.
            flight_periods = TakeoffOrLandingEvent.flights(user)
            uas_period_logs = UasTelemetry.dedupe(UasTelemetry.by_time_period(user, flight_periods))
            uas_logs = list(itertools.chain.from_iterable(uas_period_logs))

            # Determine if the uas hit the waypoints.
            waypoints_hit = self.satisfied_waypoints(uas_logs)
            waypoints_keyed = {}
            for i, hit in enumerate(waypoints_hit):
                waypoints_keyed[i + 1] = hit
            eval_data["waypoints_satisfied"] = waypoints_keyed

            # Determine if the uas went out of bounds. This must be done for
            # each period individually so time between periods isn't counted as
            # out of bounds time. Note that this calculates reported time out
            # of bounds, not actual or possible time spent out of bounds.
            out_of_bounds_time = 0
            for logs in uas_period_logs:
                out_of_bounds_time += FlyZone.out_of_bounds(self.fly_zones.all(), logs)
            eval_data["out_of_bounds_time"] = out_of_bounds_time

            # Determine interop rates.
            interop_times = eval_data.setdefault("interop_times", {})

            server_info_times = ServerInfoAccessLog.rates(user, flight_periods)
            obstacle_times = ObstacleAccessLog.rates(user, flight_periods)
            uas_telemetry_times = UasTelemetry.rates(user, flight_periods, time_period_logs=uas_period_logs)

            interop_times["server_info"] = {"max": server_info_times[0], "avg": server_info_times[1]}
            interop_times["obst_info"] = {"max": obstacle_times[0], "avg": obstacle_times[1]}
            interop_times["uas_telem"] = {"max": uas_telemetry_times[0], "avg": uas_telemetry_times[1]}

            # Determine collisions with stationary and moving obstacles.
            stationary_collisions = eval_data.setdefault("stationary_obst_collision", {})
            for obst in self.stationary_obstacles.all():
                collision = obst.evaluate_collision_with_uas(uas_logs)
                stationary_collisions[obst.pk] = collision

            moving_collisions = eval_data.setdefault("moving_obst_collision", {})
            for obst in self.moving_obstacles.all():
                collision = obst.evaluate_collision_with_uas(uas_logs)
                moving_collisions[obst.pk] = collision

        return results
Example #2
0
    def evaluate_teams(self, users=None):
        """Evaluates the teams (non admin users) of the competition.

        Args:
            users: Optional list of users to eval. If None will evaluate all.
        Returns:
            A map from user to evaluate data. The evaluation data has the
            following map structure:
            {
                'mission_clock_time': Seconds spent on mission clock,
                'waypoints_satisfied': {
                    id: Boolean,
                }
                'out_of_bounds_time': Seconds spent out of bounds,
                'targets': Data from TargetEvaluation,
                'uas_telem_time': {
                    'max': Value,
                    'avg': Value,
                },
                'stationary_obst_collision': {
                    id: Boolean
                },
                'moving_obst_collision': {
                    id: Boolean
                }
                'warnings': [
                    "String message."
                ],
            }
        """
        # Start a results map from user to evaluation data
        results = {}

        # If not provided, eval all users.
        if users is None:
            users = User.objects.all()

        logger.info('Starting team evaluations.')
        for user in users:
            # Ignore admins.
            if user.is_superuser:
                continue
            logger.info('Evaluation starting for user: %s.' % user.username)

            # Start the evaluation data structure.
            eval_data = results.setdefault(user, {})
            warnings = []
            eval_data['warnings'] = warnings

            # Calculate the total mission clock time.
            mission_clock_time = 0
            missions = MissionClockEvent.missions(user)
            for mission in missions:
                duration = mission.duration()
                if duration is None:
                    warnings.append('Infinite duration mission clock.')
                else:
                    mission_clock_time += duration.total_seconds()
            eval_data['mission_clock_time'] = mission_clock_time

            # Find the user's flights.
            flight_periods = TakeoffOrLandingEvent.flights(user)
            for period in flight_periods:
                if period.duration() is None:
                    warnings.append('Infinite duration flight period.')
            uas_period_logs = [
                UasTelemetry.dedupe(logs)
                for logs in UasTelemetry.by_time_period(user, flight_periods)
            ]
            uas_logs = list(itertools.chain.from_iterable(uas_period_logs))
            if not uas_logs:
                warnings.append('No UAS telemetry logs.')

            # Determine if the uas hit the waypoints.
            waypoints_hit, waypoints_hit_track, waypoints_closest = \
                self.satisfied_waypoints(uas_logs)
            eval_data['waypoints_satisfied'] = waypoints_hit
            eval_data['waypoints_satisfied_track'] = waypoints_hit_track
            eval_data['waypoints_closest'] = waypoints_closest

            # Determine if the uas went out of bounds. This must be done for
            # each period individually so time between periods isn't counted as
            # out of bounds time. Note that this calculates reported time out
            # of bounds, not actual or possible time spent out of bounds.
            out_of_bounds_time = 0
            for logs in uas_period_logs:
                out_of_bounds_time += FlyZone.out_of_bounds(
                    self.fly_zones.all(), logs)
            eval_data['out_of_bounds_time'] = out_of_bounds_time

            # Evaluate the targets.
            eval_data['targets'] = {}
            user_targets = Target.objects.filter(user=user).all()
            target_sets = {
                'manual': [t for t in user_targets if not t.autonomous],
                'auto': [t for t in user_targets if t.autonomous],
            }
            for target_set, targets in target_sets.iteritems():
                evaluator = TargetEvaluator(targets, self.targets.all())
                eval_data['targets'][target_set] = evaluator.evaluation_dict()

            # Determine interop telemetry rates.
            uas_telemetry_times = UasTelemetry.rates(
                user, flight_periods, time_period_logs=uas_period_logs)
            eval_data['uas_telem_times'] = {
                'max': uas_telemetry_times[0],
                'avg': uas_telemetry_times[1]
            }

            # Determine collisions with stationary and moving obstacles.
            stationary_collisions = eval_data.setdefault(
                'stationary_obst_collision', {})
            for obst in self.stationary_obstacles.all():
                collision = obst.evaluate_collision_with_uas(uas_logs)
                stationary_collisions[obst.pk] = collision

            moving_collisions = eval_data.setdefault('moving_obst_collision',
                                                     {})
            for obst in self.moving_obstacles.all():
                collision = obst.evaluate_collision_with_uas(uas_logs)
                moving_collisions[obst.pk] = collision

        return results
Example #3
0
    def evaluate_teams(self):
        """Evaluates the teams (non admin users) of the competition.

        Returns:
            A map from user to evaluate data. The evaluation data has the
            following map structure:
            {
                'mission_clock_time': Seconds spent on mission clock,
                'waypoints_satisfied': {
                    id: Boolean,
                }
                'out_of_bounds_time': Seconds spent out of bounds,
                'targets': Data from TargetEvaluation,
                'interop_times': {
                    'server_info': {'max': Value, 'avg': Value},
                    'obst_info': {'max': Value, 'avg': Value},
                    'uas_telem': {'max': Value, 'avg': Value},
                },
                'stationary_obst_collision': {
                    id: Boolean
                },
                'moving_obst_collision': {
                    id: Boolean
                }
                'warnings': [
                    "String message."
                ],
            }
        """
        # Start a results map from user to evaluation data
        results = {}

        # Fill in evaluation data for each user except admins
        users = User.objects.all()
        logger.info('Starting team evaluations.')

        for user in users:
            # Ignore admins.
            if user.is_superuser:
                continue

            logger.info('Evaluation starting for user: %s.' % user.username)

            # Start the evaluation data structure.
            eval_data = results.setdefault(user, {})
            warnings = []
            eval_data['warnings'] = warnings

            # Calculate the total mission clock time.
            mission_clock_time = 0
            missions = MissionClockEvent.missions(user)
            for mission in missions:
                duration = mission.duration()
                if duration is None:
                    warnings.append('Infinite duration mission clock.')
                else:
                    mission_clock_time += duration.total_seconds()
            eval_data['mission_clock_time'] = mission_clock_time

            # Find the user's flights.
            flight_periods = TakeoffOrLandingEvent.flights(user)
            for period in flight_periods:
                if period.duration() is None:
                    warnings.append('Infinite duration flight period.')
            uas_period_logs = [
                UasTelemetry.dedupe(logs)
                for logs in UasTelemetry.by_time_period(user, flight_periods)
            ]
            uas_logs = list(itertools.chain.from_iterable(uas_period_logs))
            if not uas_logs:
                warnings.append('No UAS telemetry logs.')

            # Determine if the uas hit the waypoints.
            waypoints_hit, waypoints_hit_track, waypoints_closest = \
                self.satisfied_waypoints(uas_logs)
            eval_data['waypoints_satisfied'] = waypoints_hit
            eval_data['waypoints_satisfied_track'] = waypoints_hit_track
            eval_data['waypoints_closest'] = waypoints_closest

            # Determine if the uas went out of bounds. This must be done for
            # each period individually so time between periods isn't counted as
            # out of bounds time. Note that this calculates reported time out
            # of bounds, not actual or possible time spent out of bounds.
            out_of_bounds_time = 0
            for logs in uas_period_logs:
                out_of_bounds_time += FlyZone.out_of_bounds(
                    self.fly_zones.all(), logs)
            eval_data['out_of_bounds_time'] = out_of_bounds_time

            # Evaluate the targets.
            eval_data['targets'] = {}
            user_targets = Target.objects.filter(user=user).all()
            target_sets = {
                'manual': [t for t in user_targets if not t.autonomous],
                'auto': [t for t in user_targets
                         if t.autonomous][:settings.TARGET_MAX_NUM_AUTONOMOUS],
            }
            for target_set, targets in target_sets.iteritems():
                evaluator = TargetEvaluator(targets, self.targets.all())
                eval_data['targets'][target_set] = evaluator.evaluation_dict()

            # Determine interop rates.
            interop_times = eval_data.setdefault('interop_times', {})

            server_info_times = ServerInfoAccessLog.rates(user, flight_periods)
            obstacle_times = ObstacleAccessLog.rates(user, flight_periods)
            uas_telemetry_times = UasTelemetry.rates(
                user,
                flight_periods,
                time_period_logs=uas_period_logs)

            interop_times['server_info'] = {
                'max': server_info_times[0],
                'avg': server_info_times[1]
            }
            interop_times['obst_info'] = {
                'max': obstacle_times[0],
                'avg': obstacle_times[1]
            }
            interop_times['uas_telem'] = {
                'max': uas_telemetry_times[0],
                'avg': uas_telemetry_times[1]
            }

            # Determine collisions with stationary and moving obstacles.
            stationary_collisions = eval_data.setdefault(
                'stationary_obst_collision', {})
            for obst in self.stationary_obstacles.all():
                collision = obst.evaluate_collision_with_uas(uas_logs)
                stationary_collisions[obst.pk] = collision

            moving_collisions = eval_data.setdefault('moving_obst_collision',
                                                     {})
            for obst in self.moving_obstacles.all():
                collision = obst.evaluate_collision_with_uas(uas_logs)
                moving_collisions[obst.pk] = collision

        return results
Example #4
0
    def evaluate_teams(self):
        """Evaluates the teams (non admin users) of the competition.

        Returns:
            A map from user to evaluate data. The evaluation data has the
            following map structure:
            {
                'waypoints_satisfied': {
                    id: Boolean,
                }
                'out_of_bounds_time': Seconds spent out of bounds,
                'interop_times': {
                    'server_info': {'max': Value, 'avg': Value},
                    'obst_info': {'max': Value, 'avg': Value},
                    'uas_telem': {'max': Value, 'avg': Value},
                },
                'stationary_obst_collision': {
                    id: Boolean
                },
                'moving_obst_collision': {
                    id: Boolean
                }
            }
        """
        # Start a results map from user to evaluation data
        results = {}

        # Fill in evaluation data for each user except admins
        users = User.objects.all()
        logger.info('Starting team evaluations.')

        for user in users:
            # Ignore admins.
            if user.is_superuser:
                continue

            logger.info('Evaluation starting for user: %s.' % user.username)

            # Start the evaluation data structure.
            eval_data = results.setdefault(user, {})

            # Find the user's flights.
            flight_periods = TakeoffOrLandingEvent.flights(user)
            uas_period_logs = [
                UasTelemetry.dedupe(logs)
                for logs in UasTelemetry.by_time_period(user, flight_periods)
            ]
            uas_logs = list(itertools.chain.from_iterable(uas_period_logs))

            # Determine if the uas hit the waypoints.
            waypoints_hit = self.satisfied_waypoints(uas_logs)
            waypoints_keyed = {}
            for i, hit in enumerate(waypoints_hit):
                waypoints_keyed[i + 1] = hit
            eval_data['waypoints_satisfied'] = waypoints_keyed

            # Determine if the uas went out of bounds. This must be done for
            # each period individually so time between periods isn't counted as
            # out of bounds time. Note that this calculates reported time out
            # of bounds, not actual or possible time spent out of bounds.
            out_of_bounds_time = 0
            for logs in uas_period_logs:
                out_of_bounds_time += FlyZone.out_of_bounds(
                    self.fly_zones.all(), logs)
            eval_data['out_of_bounds_time'] = out_of_bounds_time

            # Determine interop rates.
            interop_times = eval_data.setdefault('interop_times', {})

            server_info_times = ServerInfoAccessLog.rates(user, flight_periods)
            obstacle_times = ObstacleAccessLog.rates(user, flight_periods)
            uas_telemetry_times = UasTelemetry.rates(
                user, flight_periods, time_period_logs=uas_period_logs)

            interop_times['server_info'] = {
                'max': server_info_times[0],
                'avg': server_info_times[1]
            }
            interop_times['obst_info'] = {
                'max': obstacle_times[0],
                'avg': obstacle_times[1]
            }
            interop_times['uas_telem'] = {
                'max': uas_telemetry_times[0],
                'avg': uas_telemetry_times[1]
            }

            # Determine collisions with stationary and moving obstacles.
            stationary_collisions = eval_data.setdefault(
                'stationary_obst_collision', {})
            for obst in self.stationary_obstacles.all():
                collision = obst.evaluate_collision_with_uas(uas_logs)
                stationary_collisions[obst.pk] = collision

            moving_collisions = eval_data.setdefault('moving_obst_collision',
                                                     {})
            for obst in self.moving_obstacles.all():
                collision = obst.evaluate_collision_with_uas(uas_logs)
                moving_collisions[obst.pk] = collision

        return results