def _get_metric(self, app, num_frozen, target_fps, metric_name, **kwargs):
accuracy = app["accuracies"][num_frozen]
if metric_name == "f1":
prob_tnr = app["prob_tnrs"][num_frozen]
f1 = scheduler_util.get_f1_score(accuracy, prob_tnr,
app["event_length_ms"],
app["event_frequency"],
app["correlation_coefficient"],
self.stream_fps, target_fps,
**kwargs)
metric = 1 - f1
elif metric_name == "fnr":
metric = scheduler_util.get_false_neg_rate(
accuracy, app["event_length_ms"],
app["correlation_coefficient"], self.stream_fps, target_fps,
**kwargs)
elif metric_name == "fpr":
prob_tnr = app["prob_tnrs"][num_frozen]
metric = scheduler_util.get_false_pos_rate(
accuracy, prob_tnr, app["event_length_ms"],
app["event_frequency"], app["correlation_coefficient"],
self.stream_fps, target_fps, **kwargs)
else:
raise Exception(
"Didn't recognize metric {}. Exiting.".format(metric_name))
return metric
def get_observed_performance(self, streamer_schedule, fpses):
fps_by_app_id = self.get_fps_by_app_id(streamer_schedule, fpses)
fnrs = []
fprs = []
f1s = []
observed_schedule = []
for app, num_frozen in zip(self.apps, self.num_frozen_list):
kwargs = {}
if self.metric.endswith('-x'):
kwargs['x_vote'] = app['x_vote']
observed_fps = fps_by_app_id[app["app_id"]]
accuracy = app["accuracies"][num_frozen]
prob_tnr = app["prob_tnrs"][num_frozen]
false_neg_rate = scheduler_util.get_false_neg_rate(
accuracy,
app["event_length_ms"],
app["correlation_coefficient"],
self.stream_fps,
observed_fps,
**kwargs)
false_pos_rate = scheduler_util.get_false_pos_rate(
accuracy,
prob_tnr,
app["event_length_ms"],
app["event_frequency"],
app["correlation_coefficient"],
self.stream_fps,
observed_fps,
**kwargs)
recall = 1. - false_neg_rate
precision = 1. - false_pos_rate
f1 = 2. / (1. / recall + 1. / precision)
fnrs.append(false_neg_rate)
fprs.append(false_pos_rate)
f1s.append(f1)
observed_unit = Schedule.ScheduleUnit(app,
observed_fps,
num_frozen)
observed_schedule.append(observed_unit)
observed_cost = scheduler_util.get_cost_schedule(observed_schedule,
self.model.layer_latencies,
self.model.final_layer)
average_fnr = sum(fnrs) / float(len(fnrs))
average_fpr = sum(fprs) / float(len(fprs))
average_f1 = sum(f1s) / float(len(f1s))
return round(average_fnr, 4), round(average_fpr, 4), round(average_f1, 4), round(observed_cost, 4)