def __init__(self, submitted_objects, real_objects):
"""Creates an evaluation of submitted objects against real objects.
Args:
submitted_objects: List of submitted Odlc objects, all from
the same user.
real_objects: List of real objects made by judges.
Raises:
AssertionError: not all submitted objects are from the same user.
"""
self.submitted_objects = submitted_objects
self.real_objects = real_objects
if self.submitted_objects:
self.user = self.submitted_objects[0].user
for t in self.submitted_objects:
if t.user != self.user:
raise AssertionError(
"All submitted objects must be from the same user")
self.flights = TakeoffOrLandingEvent.flights(self.user)
self.missions = MissionClockEvent.missions(self.user)
self.matches = self.match_odlcs(submitted_objects, real_objects)
self.unmatched = self.find_unmatched(submitted_objects, real_objects,
self.matches)
def interop_submission(self, missions=None):
"""Checks if Target meets Interoperability submission criteria.
A target counts as being submitted over interoperability system if it
was submitted and last updated while the team was on the mission clock.
Args:
missions: Optional memoized missions for this target's user. If
omitted, the missions will be looked up.
Returns:
True if target may be considered an "interoperability" submission.
"""
if missions is None:
missions = MissionClockEvent.missions(self.user)
for mission in missions:
if mission.within(self.creation_time) and mission.within(self.last_modified_time):
return True
return False
def interop_submission(self, missions=None):
"""Checks if Target meets Interoperability submission criteria.
A target counts as being submitted over interoperability system if it
was submitted and last updated while the team was on the mission clock.
Args:
missions: Optional memoized missions for this target's user. If
omitted, the missions will be looked up.
Returns:
True if target may be considered an "interoperability" submission.
"""
if missions is None:
missions = MissionClockEvent.missions(self.user)
for mission in missions:
if mission.within(self.creation_time) and \
mission.within(self.last_modified_time):
return True
return False
def __init__(self, submitted_targets, real_targets):
"""Creates an evaluation of submitted targets against real targets.
Args:
submitted_targets: List of submitted Target objects, all from
the same user.
real_targets: List of real Target objects made by judges.
Raises:
AssertionError: not all submitted targets are from the same user.
"""
self.submitted_targets = submitted_targets
self.real_targets = real_targets
if self.submitted_targets:
self.user = self.submitted_targets[0].user
for t in self.submitted_targets:
if t.user != self.user:
raise AssertionError("All submitted targets must be from the same user")
self.flights = TakeoffOrLandingEvent.flights(self.user)
self.missions = MissionClockEvent.missions(self.user)
self.matches = self.match_targets(submitted_targets, real_targets)
def evaluate_teams(self):
"""Evaluates the teams (non admin users) of the competition.
Returns:
A map from user to evaluate data. The evaluation data has the
following map structure:
{
'mission_clock_time': Seconds spent on mission clock,
'waypoints_satisfied': {
id: Boolean,
}
'out_of_bounds_time': Seconds spent out of bounds,
'targets': Data from TargetEvaluation,
'interop_times': {
'server_info': {'max': Value, 'avg': Value},
'obst_info': {'max': Value, 'avg': Value},
'uas_telem': {'max': Value, 'avg': Value},
},
'stationary_obst_collision': {
id: Boolean
},
'moving_obst_collision': {
id: Boolean
}
'warnings': [
"String message."
],
}
"""
# Start a results map from user to evaluation data
results = {}
# Fill in evaluation data for each user except admins
users = User.objects.all()
logger.info('Starting team evaluations.')
for user in users:
# Ignore admins.
if user.is_superuser:
continue
logger.info('Evaluation starting for user: %s.' % user.username)
# Start the evaluation data structure.
eval_data = results.setdefault(user, {})
warnings = []
eval_data['warnings'] = warnings
# Calculate the total mission clock time.
mission_clock_time = 0
missions = MissionClockEvent.missions(user)
for mission in missions:
duration = mission.duration()
if duration is None:
warnings.append('Infinite duration mission clock.')
else:
mission_clock_time += duration.total_seconds()
eval_data['mission_clock_time'] = mission_clock_time
# Find the user's flights.
flight_periods = TakeoffOrLandingEvent.flights(user)
for period in flight_periods:
if period.duration() is None:
warnings.append('Infinite duration flight period.')
uas_period_logs = [
UasTelemetry.dedupe(logs)
for logs in UasTelemetry.by_time_period(user, flight_periods)
]
uas_logs = list(itertools.chain.from_iterable(uas_period_logs))
if not uas_logs:
warnings.append('No UAS telemetry logs.')
# Determine if the uas hit the waypoints.
waypoints_hit, waypoints_hit_track, waypoints_closest = \
self.satisfied_waypoints(uas_logs)
eval_data['waypoints_satisfied'] = waypoints_hit
eval_data['waypoints_satisfied_track'] = waypoints_hit_track
eval_data['waypoints_closest'] = waypoints_closest
# Determine if the uas went out of bounds. This must be done for
# each period individually so time between periods isn't counted as
# out of bounds time. Note that this calculates reported time out
# of bounds, not actual or possible time spent out of bounds.
out_of_bounds_time = 0
for logs in uas_period_logs:
out_of_bounds_time += FlyZone.out_of_bounds(
self.fly_zones.all(), logs)
eval_data['out_of_bounds_time'] = out_of_bounds_time
# Evaluate the targets.
eval_data['targets'] = {}
user_targets = Target.objects.filter(user=user).all()
target_sets = {
'manual': [t for t in user_targets if not t.autonomous],
'auto': [t for t in user_targets
if t.autonomous][:settings.TARGET_MAX_NUM_AUTONOMOUS],
}
for target_set, targets in target_sets.iteritems():
evaluator = TargetEvaluator(targets, self.targets.all())
eval_data['targets'][target_set] = evaluator.evaluation_dict()
# Determine interop rates.
interop_times = eval_data.setdefault('interop_times', {})
server_info_times = ServerInfoAccessLog.rates(user, flight_periods)
obstacle_times = ObstacleAccessLog.rates(user, flight_periods)
uas_telemetry_times = UasTelemetry.rates(
user,
flight_periods,
time_period_logs=uas_period_logs)
interop_times['server_info'] = {
'max': server_info_times[0],
'avg': server_info_times[1]
}
interop_times['obst_info'] = {
'max': obstacle_times[0],
'avg': obstacle_times[1]
}
interop_times['uas_telem'] = {
'max': uas_telemetry_times[0],
'avg': uas_telemetry_times[1]
}
# Determine collisions with stationary and moving obstacles.
stationary_collisions = eval_data.setdefault(
'stationary_obst_collision', {})
for obst in self.stationary_obstacles.all():
collision = obst.evaluate_collision_with_uas(uas_logs)
stationary_collisions[obst.pk] = collision
moving_collisions = eval_data.setdefault('moving_obst_collision',
{})
for obst in self.moving_obstacles.all():
collision = obst.evaluate_collision_with_uas(uas_logs)
moving_collisions[obst.pk] = collision
return results
def evaluate_teams(self, users=None):
"""Evaluates the teams (non admin users) of the competition.
Args:
users: Optional list of users to eval. If None will evaluate all.
Returns:
A map from user to evaluate data. The evaluation data has the
following map structure:
{
'mission_clock_time': Seconds spent on mission clock,
'waypoints_satisfied': {
id: Boolean,
}
'out_of_bounds_time': Seconds spent out of bounds,
'targets': Data from TargetEvaluation,
'uas_telem_time': {
'max': Value,
'avg': Value,
},
'stationary_obst_collision': {
id: Boolean
},
'moving_obst_collision': {
id: Boolean
}
'warnings': [
"String message."
],
}
"""
# Start a results map from user to evaluation data
results = {}
# If not provided, eval all users.
if users is None:
users = User.objects.all()
logger.info('Starting team evaluations.')
for user in users:
# Ignore admins.
if user.is_superuser:
continue
logger.info('Evaluation starting for user: %s.' % user.username)
# Start the evaluation data structure.
eval_data = results.setdefault(user, {})
warnings = []
eval_data['warnings'] = warnings
# Calculate the total mission clock time.
mission_clock_time = 0
missions = MissionClockEvent.missions(user)
for mission in missions:
duration = mission.duration()
if duration is None:
warnings.append('Infinite duration mission clock.')
else:
mission_clock_time += duration.total_seconds()
eval_data['mission_clock_time'] = mission_clock_time
# Find the user's flights.
flight_periods = TakeoffOrLandingEvent.flights(user)
for period in flight_periods:
if period.duration() is None:
warnings.append('Infinite duration flight period.')
uas_period_logs = [
UasTelemetry.dedupe(logs)
for logs in UasTelemetry.by_time_period(user, flight_periods)
]
uas_logs = list(itertools.chain.from_iterable(uas_period_logs))
if not uas_logs:
warnings.append('No UAS telemetry logs.')
# Determine if the uas hit the waypoints.
waypoints_hit, waypoints_hit_track, waypoints_closest = \
self.satisfied_waypoints(uas_logs)
eval_data['waypoints_satisfied'] = waypoints_hit
eval_data['waypoints_satisfied_track'] = waypoints_hit_track
eval_data['waypoints_closest'] = waypoints_closest
# Determine if the uas went out of bounds. This must be done for
# each period individually so time between periods isn't counted as
# out of bounds time. Note that this calculates reported time out
# of bounds, not actual or possible time spent out of bounds.
out_of_bounds_time = 0
for logs in uas_period_logs:
out_of_bounds_time += FlyZone.out_of_bounds(
self.fly_zones.all(), logs)
eval_data['out_of_bounds_time'] = out_of_bounds_time
# Evaluate the targets.
eval_data['targets'] = {}
user_targets = Target.objects.filter(user=user).all()
target_sets = {
'manual': [t for t in user_targets if not t.autonomous],
'auto': [t for t in user_targets if t.autonomous],
}
for target_set, targets in target_sets.iteritems():
evaluator = TargetEvaluator(targets, self.targets.all())
eval_data['targets'][target_set] = evaluator.evaluation_dict()
# Determine interop telemetry rates.
uas_telemetry_times = UasTelemetry.rates(
user, flight_periods, time_period_logs=uas_period_logs)
eval_data['uas_telem_times'] = {
'max': uas_telemetry_times[0],
'avg': uas_telemetry_times[1]
}
# Determine collisions with stationary and moving obstacles.
stationary_collisions = eval_data.setdefault(
'stationary_obst_collision', {})
for obst in self.stationary_obstacles.all():
collision = obst.evaluate_collision_with_uas(uas_logs)
stationary_collisions[obst.pk] = collision
moving_collisions = eval_data.setdefault('moving_obst_collision',
{})
for obst in self.moving_obstacles.all():
collision = obst.evaluate_collision_with_uas(uas_logs)
moving_collisions[obst.pk] = collision
return results
