def evaluate_teams(self):
"""Evaluates the teams (non admin users) of the competition.
Returns:
A map from user to evaluate data. The evaluation data has the
following map structure:
{
'waypoints_satisfied': {
id: Boolean,
}
'out_of_bounds_time': Seconds spent out of bounds,
'interop_times': {
'server_info': {'max': Value, 'avg': Value},
'obst_info': {'max': Value, 'avg': Value},
'uas_telem': {'max': Value, 'avg': Value},
},
'stationary_obst_collision': {
id: Boolean
},
'moving_obst_collision': {
id: Boolean
}
}
"""
# Start a results map from user to evaluation data
results = {}
# Fill in evaluation data for each user except admins
users = User.objects.all()
logger.info("Starting team evaluations.")
for user in users:
# Ignore admins.
if user.is_superuser:
continue
logger.info("Evaluation starting for user: %s." % user.username)
# Start the evaluation data structure.
eval_data = results.setdefault(user, {})
# Find the user's flights.
flight_periods = TakeoffOrLandingEvent.flights(user)
uas_period_logs = UasTelemetry.dedupe(UasTelemetry.by_time_period(user, flight_periods))
uas_logs = list(itertools.chain.from_iterable(uas_period_logs))
# Determine if the uas hit the waypoints.
waypoints_hit = self.satisfied_waypoints(uas_logs)
waypoints_keyed = {}
for i, hit in enumerate(waypoints_hit):
waypoints_keyed[i + 1] = hit
eval_data["waypoints_satisfied"] = waypoints_keyed
# Determine if the uas went out of bounds. This must be done for
# each period individually so time between periods isn't counted as
# out of bounds time. Note that this calculates reported time out
# of bounds, not actual or possible time spent out of bounds.
out_of_bounds_time = 0
for logs in uas_period_logs:
out_of_bounds_time += FlyZone.out_of_bounds(self.fly_zones.all(), logs)
eval_data["out_of_bounds_time"] = out_of_bounds_time
# Determine interop rates.
interop_times = eval_data.setdefault("interop_times", {})
server_info_times = ServerInfoAccessLog.rates(user, flight_periods)
obstacle_times = ObstacleAccessLog.rates(user, flight_periods)
uas_telemetry_times = UasTelemetry.rates(user, flight_periods, time_period_logs=uas_period_logs)
interop_times["server_info"] = {"max": server_info_times[0], "avg": server_info_times[1]}
interop_times["obst_info"] = {"max": obstacle_times[0], "avg": obstacle_times[1]}
interop_times["uas_telem"] = {"max": uas_telemetry_times[0], "avg": uas_telemetry_times[1]}
# Determine collisions with stationary and moving obstacles.
stationary_collisions = eval_data.setdefault("stationary_obst_collision", {})
for obst in self.stationary_obstacles.all():
collision = obst.evaluate_collision_with_uas(uas_logs)
stationary_collisions[obst.pk] = collision
moving_collisions = eval_data.setdefault("moving_obst_collision", {})
for obst in self.moving_obstacles.all():
collision = obst.evaluate_collision_with_uas(uas_logs)
moving_collisions[obst.pk] = collision
return results
def evaluate_teams(self, users=None):
"""Evaluates the teams (non admin users) of the competition.
Args:
users: Optional list of users to eval. If None will evaluate all.
Returns:
A map from user to evaluate data. The evaluation data has the
following map structure:
{
'mission_clock_time': Seconds spent on mission clock,
'waypoints_satisfied': {
id: Boolean,
}
'out_of_bounds_time': Seconds spent out of bounds,
'targets': Data from TargetEvaluation,
'uas_telem_time': {
'max': Value,
'avg': Value,
},
'stationary_obst_collision': {
id: Boolean
},
'moving_obst_collision': {
id: Boolean
}
'warnings': [
"String message."
],
}
"""
# Start a results map from user to evaluation data
results = {}
# If not provided, eval all users.
if users is None:
users = User.objects.all()
logger.info('Starting team evaluations.')
for user in users:
# Ignore admins.
if user.is_superuser:
continue
logger.info('Evaluation starting for user: %s.' % user.username)
# Start the evaluation data structure.
eval_data = results.setdefault(user, {})
warnings = []
eval_data['warnings'] = warnings
# Calculate the total mission clock time.
mission_clock_time = 0
missions = MissionClockEvent.missions(user)
for mission in missions:
duration = mission.duration()
if duration is None:
warnings.append('Infinite duration mission clock.')
else:
mission_clock_time += duration.total_seconds()
eval_data['mission_clock_time'] = mission_clock_time
# Find the user's flights.
flight_periods = TakeoffOrLandingEvent.flights(user)
for period in flight_periods:
if period.duration() is None:
warnings.append('Infinite duration flight period.')
uas_period_logs = [
UasTelemetry.dedupe(logs)
for logs in UasTelemetry.by_time_period(user, flight_periods)
]
uas_logs = list(itertools.chain.from_iterable(uas_period_logs))
if not uas_logs:
warnings.append('No UAS telemetry logs.')
# Determine if the uas hit the waypoints.
waypoints_hit, waypoints_hit_track, waypoints_closest = \
self.satisfied_waypoints(uas_logs)
eval_data['waypoints_satisfied'] = waypoints_hit
eval_data['waypoints_satisfied_track'] = waypoints_hit_track
eval_data['waypoints_closest'] = waypoints_closest
# Determine if the uas went out of bounds. This must be done for
# each period individually so time between periods isn't counted as
# out of bounds time. Note that this calculates reported time out
# of bounds, not actual or possible time spent out of bounds.
out_of_bounds_time = 0
for logs in uas_period_logs:
out_of_bounds_time += FlyZone.out_of_bounds(
self.fly_zones.all(), logs)
eval_data['out_of_bounds_time'] = out_of_bounds_time
# Evaluate the targets.
eval_data['targets'] = {}
user_targets = Target.objects.filter(user=user).all()
target_sets = {
'manual': [t for t in user_targets if not t.autonomous],
'auto': [t for t in user_targets if t.autonomous],
}
for target_set, targets in target_sets.iteritems():
evaluator = TargetEvaluator(targets, self.targets.all())
eval_data['targets'][target_set] = evaluator.evaluation_dict()
# Determine interop telemetry rates.
uas_telemetry_times = UasTelemetry.rates(
user, flight_periods, time_period_logs=uas_period_logs)
eval_data['uas_telem_times'] = {
'max': uas_telemetry_times[0],
'avg': uas_telemetry_times[1]
}
# Determine collisions with stationary and moving obstacles.
stationary_collisions = eval_data.setdefault(
'stationary_obst_collision', {})
for obst in self.stationary_obstacles.all():
collision = obst.evaluate_collision_with_uas(uas_logs)
stationary_collisions[obst.pk] = collision
moving_collisions = eval_data.setdefault('moving_obst_collision',
{})
for obst in self.moving_obstacles.all():
collision = obst.evaluate_collision_with_uas(uas_logs)
moving_collisions[obst.pk] = collision
return results
def evaluate_teams(self):
"""Evaluates the teams (non admin users) of the competition.
Returns:
A map from user to evaluate data. The evaluation data has the
following map structure:
{
'mission_clock_time': Seconds spent on mission clock,
'waypoints_satisfied': {
id: Boolean,
}
'out_of_bounds_time': Seconds spent out of bounds,
'targets': Data from TargetEvaluation,
'interop_times': {
'server_info': {'max': Value, 'avg': Value},
'obst_info': {'max': Value, 'avg': Value},
'uas_telem': {'max': Value, 'avg': Value},
},
'stationary_obst_collision': {
id: Boolean
},
'moving_obst_collision': {
id: Boolean
}
'warnings': [
"String message."
],
}
"""
# Start a results map from user to evaluation data
results = {}
# Fill in evaluation data for each user except admins
users = User.objects.all()
logger.info('Starting team evaluations.')
for user in users:
# Ignore admins.
if user.is_superuser:
continue
logger.info('Evaluation starting for user: %s.' % user.username)
# Start the evaluation data structure.
eval_data = results.setdefault(user, {})
warnings = []
eval_data['warnings'] = warnings
# Calculate the total mission clock time.
mission_clock_time = 0
missions = MissionClockEvent.missions(user)
for mission in missions:
duration = mission.duration()
if duration is None:
warnings.append('Infinite duration mission clock.')
else:
mission_clock_time += duration.total_seconds()
eval_data['mission_clock_time'] = mission_clock_time
# Find the user's flights.
flight_periods = TakeoffOrLandingEvent.flights(user)
for period in flight_periods:
if period.duration() is None:
warnings.append('Infinite duration flight period.')
uas_period_logs = [
UasTelemetry.dedupe(logs)
for logs in UasTelemetry.by_time_period(user, flight_periods)
]
uas_logs = list(itertools.chain.from_iterable(uas_period_logs))
if not uas_logs:
warnings.append('No UAS telemetry logs.')
# Determine if the uas hit the waypoints.
waypoints_hit, waypoints_hit_track, waypoints_closest = \
self.satisfied_waypoints(uas_logs)
eval_data['waypoints_satisfied'] = waypoints_hit
eval_data['waypoints_satisfied_track'] = waypoints_hit_track
eval_data['waypoints_closest'] = waypoints_closest
# Determine if the uas went out of bounds. This must be done for
# each period individually so time between periods isn't counted as
# out of bounds time. Note that this calculates reported time out
# of bounds, not actual or possible time spent out of bounds.
out_of_bounds_time = 0
for logs in uas_period_logs:
out_of_bounds_time += FlyZone.out_of_bounds(
self.fly_zones.all(), logs)
eval_data['out_of_bounds_time'] = out_of_bounds_time
# Evaluate the targets.
eval_data['targets'] = {}
user_targets = Target.objects.filter(user=user).all()
target_sets = {
'manual': [t for t in user_targets if not t.autonomous],
'auto': [t for t in user_targets
if t.autonomous][:settings.TARGET_MAX_NUM_AUTONOMOUS],
}
for target_set, targets in target_sets.iteritems():
evaluator = TargetEvaluator(targets, self.targets.all())
eval_data['targets'][target_set] = evaluator.evaluation_dict()
# Determine interop rates.
interop_times = eval_data.setdefault('interop_times', {})
server_info_times = ServerInfoAccessLog.rates(user, flight_periods)
obstacle_times = ObstacleAccessLog.rates(user, flight_periods)
uas_telemetry_times = UasTelemetry.rates(
user,
flight_periods,
time_period_logs=uas_period_logs)
interop_times['server_info'] = {
'max': server_info_times[0],
'avg': server_info_times[1]
}
interop_times['obst_info'] = {
'max': obstacle_times[0],
'avg': obstacle_times[1]
}
interop_times['uas_telem'] = {
'max': uas_telemetry_times[0],
'avg': uas_telemetry_times[1]
}
# Determine collisions with stationary and moving obstacles.
stationary_collisions = eval_data.setdefault(
'stationary_obst_collision', {})
for obst in self.stationary_obstacles.all():
collision = obst.evaluate_collision_with_uas(uas_logs)
stationary_collisions[obst.pk] = collision
moving_collisions = eval_data.setdefault('moving_obst_collision',
{})
for obst in self.moving_obstacles.all():
collision = obst.evaluate_collision_with_uas(uas_logs)
moving_collisions[obst.pk] = collision
return results
def evaluate_teams(self):
"""Evaluates the teams (non admin users) of the competition.
Returns:
A map from user to evaluate data. The evaluation data has the
following map structure:
{
'waypoints_satisfied': {
id: Boolean,
}
'out_of_bounds_time': Seconds spent out of bounds,
'interop_times': {
'server_info': {'max': Value, 'avg': Value},
'obst_info': {'max': Value, 'avg': Value},
'uas_telem': {'max': Value, 'avg': Value},
},
'stationary_obst_collision': {
id: Boolean
},
'moving_obst_collision': {
id: Boolean
}
}
"""
# Start a results map from user to evaluation data
results = {}
# Fill in evaluation data for each user except admins
users = User.objects.all()
logger.info('Starting team evaluations.')
for user in users:
# Ignore admins.
if user.is_superuser:
continue
logger.info('Evaluation starting for user: %s.' % user.username)
# Start the evaluation data structure.
eval_data = results.setdefault(user, {})
# Find the user's flights.
flight_periods = TakeoffOrLandingEvent.flights(user)
uas_period_logs = [
UasTelemetry.dedupe(logs)
for logs in UasTelemetry.by_time_period(user, flight_periods)
]
uas_logs = list(itertools.chain.from_iterable(uas_period_logs))
# Determine if the uas hit the waypoints.
waypoints_hit = self.satisfied_waypoints(uas_logs)
waypoints_keyed = {}
for i, hit in enumerate(waypoints_hit):
waypoints_keyed[i + 1] = hit
eval_data['waypoints_satisfied'] = waypoints_keyed
# Determine if the uas went out of bounds. This must be done for
# each period individually so time between periods isn't counted as
# out of bounds time. Note that this calculates reported time out
# of bounds, not actual or possible time spent out of bounds.
out_of_bounds_time = 0
for logs in uas_period_logs:
out_of_bounds_time += FlyZone.out_of_bounds(
self.fly_zones.all(), logs)
eval_data['out_of_bounds_time'] = out_of_bounds_time
# Determine interop rates.
interop_times = eval_data.setdefault('interop_times', {})
server_info_times = ServerInfoAccessLog.rates(user, flight_periods)
obstacle_times = ObstacleAccessLog.rates(user, flight_periods)
uas_telemetry_times = UasTelemetry.rates(
user, flight_periods, time_period_logs=uas_period_logs)
interop_times['server_info'] = {
'max': server_info_times[0],
'avg': server_info_times[1]
}
interop_times['obst_info'] = {
'max': obstacle_times[0],
'avg': obstacle_times[1]
}
interop_times['uas_telem'] = {
'max': uas_telemetry_times[0],
'avg': uas_telemetry_times[1]
}
# Determine collisions with stationary and moving obstacles.
stationary_collisions = eval_data.setdefault(
'stationary_obst_collision', {})
for obst in self.stationary_obstacles.all():
collision = obst.evaluate_collision_with_uas(uas_logs)
stationary_collisions[obst.pk] = collision
moving_collisions = eval_data.setdefault('moving_obst_collision',
{})
for obst in self.moving_obstacles.all():
collision = obst.evaluate_collision_with_uas(uas_logs)
moving_collisions[obst.pk] = collision
return results