def set_max_parameters(self):
# Makes schedule which uses max sharing and max target_fps
# Sets self.schedule, self.num_frozen_list, self.target_fps_list
schedule = []
num_frozen_list = []
target_fps_list = []
for app in self.apps:
app_id = app["app_id"]
num_frozen = max(app["accuracies"].keys())
target_fps = self.stream_fps
unit = Schedule.ScheduleUnit(app, target_fps, num_frozen)
num_frozen_list.append(num_frozen)
target_fps_list.append(target_fps)
schedule.append(unit)
self.schedule = schedule
self.num_frozen_list = num_frozen_list
self.target_fps_list = target_fps_list
cost = scheduler_util.get_cost_schedule(schedule,
self.model.layer_latencies,
self.model.final_layer)
average_metric = self.set_schedule_values(schedule)
return average_metric
def get_parameter_options(self):
## Calculate all possible schedules
possible_params = []
for num_frozen in sorted(self.apps[0]["accuracies"].keys()):
for target_fps in range(1, self.stream_fps + 1):
possible_params.append((num_frozen, target_fps))
permutations = list(itertools.product(possible_params,
repeat=len(self.apps)))
# Append app_ids to parameter permutations to make schedules
# so that set parameters are associated explicitly with an app
schedules = []
app_ids = [app["app_id"] for app in self.apps]
for perm in permutations:
schedule = []
for app_id, tup in zip(app_ids, perm):
full_tup = tup + (app_id,)
for a in self.apps:
if a["app_id"] == app_id:
app = a
unit = Schedule.ScheduleUnit(app, tup[1], tup[0])
schedule.append(unit)
schedules.append(schedule)
## Calculate the metric you're minimizing for, for each schedule
metric_by_schedule = {}
for schedule in schedules:
total_metric = 0.0
for unit in schedule:
app = unit.app
num_frozen = unit.num_frozen
target_fps = unit.target_fps
metric = self.get_metric(app,
num_frozen,
target_fps)
total_metric += metric
avg_metric = total_metric / len(self.apps)
metric_by_schedule[tuple(schedule)] = round(avg_metric, 4)
## Sort schedules by metric
sorted_d = sorted(metric_by_schedule.items(), key=operator.itemgetter(1))
schedules = [tup[0] for tup in sorted_d] #implicit ordering by metric
metrics = [tup[1] for tup in sorted_d] #implicit ordering by metric
costs = [] #implicit ordering by metric
for schedule, metric in zip(schedules, metrics):
cost = scheduler_util.get_cost_schedule(schedule,
self.model.layer_latencies,
self.model.final_layer)
costs.append(cost)
return schedules, metrics, costs
def get_cost_threshold(self, streamer_schedule, fpses):
print "[get_cost_threshold] Recalculating..."
fps_by_app_id = self.get_fps_by_app_id(streamer_schedule, fpses)
observed_schedule = []
for unit in self.schedule:
target_fps = unit.target_fps
observed_fps = fps_by_app_id[unit.app_id]
observed_unit = Schedule.ScheduleUnit(unit.app, observed_fps,
unit.num_frozen)
observed_schedule.append(observed_unit)
print "[get_cost_threshold] Target FPS: ", target_fps, "Observed FPS: ", observed_fps
target_cost = scheduler_util.get_cost_schedule(
self.schedule, self.model.layer_latencies, self.model.final_layer)
observed_cost = scheduler_util.get_cost_schedule(
observed_schedule, self.model.layer_latencies,
self.model.final_layer)
print "[get_cost_threshold] Target cost: ", target_cost, " Observed cost: ", observed_cost
if abs(target_cost - observed_cost) / target_cost < 0.20:
return -1
return observed_cost
def get_observed_performance(self, streamer_schedule, fpses):
fps_by_app_id = self.get_fps_by_app_id(streamer_schedule, fpses)
fnrs = []
fprs = []
f1s = []
observed_schedule = []
for app, num_frozen in zip(self.apps, self.num_frozen_list):
kwargs = {}
if self.metric.endswith('-x'):
kwargs['x_vote'] = app['x_vote']
observed_fps = fps_by_app_id[app["app_id"]]
accuracy = app["accuracies"][num_frozen]
prob_tnr = app["prob_tnrs"][num_frozen]
false_neg_rate = scheduler_util.get_false_neg_rate(
accuracy,
app["event_length_ms"],
app["correlation_coefficient"],
self.stream_fps,
observed_fps,
**kwargs)
false_pos_rate = scheduler_util.get_false_pos_rate(
accuracy,
prob_tnr,
app["event_length_ms"],
app["event_frequency"],
app["correlation_coefficient"],
self.stream_fps,
observed_fps,
**kwargs)
recall = 1. - false_neg_rate
precision = 1. - false_pos_rate
f1 = 2. / (1. / recall + 1. / precision)
fnrs.append(false_neg_rate)
fprs.append(false_pos_rate)
f1s.append(f1)
observed_unit = Schedule.ScheduleUnit(app,
observed_fps,
num_frozen)
observed_schedule.append(observed_unit)
observed_cost = scheduler_util.get_cost_schedule(observed_schedule,
self.model.layer_latencies,
self.model.final_layer)
average_fnr = sum(fnrs) / float(len(fnrs))
average_fpr = sum(fprs) / float(len(fprs))
average_f1 = sum(f1s) / float(len(f1s))
return round(average_fnr, 4), round(average_fpr, 4), round(average_f1, 4), round(observed_cost, 4)
def stems_scheduler(self, cost_threshold, mode):
# Currently: brute forces all stems, runs DP for each stem individually
#
# run-time: O(# of stems * num_apps * budget * num_frozen * fps
#
# the last num_frozen * fps could probably be amortized but this might
# be moot since # of stems grows ~O(num_frozen! fps!)
cost_benefits = self.get_cost_benefits(mode)
target_fps_options = self._get_target_fps_options(mode)
chokepoints = sorted(set(key for app in self.apps for key in self._get_num_frozen_options(app, mode)))
_ = self.get_init_agg_func()
best_result = None
if self.verbose > 0:
print "k_steps, total stems, total stems in budget, total stems that improved over prev optimal, ops, ops per stem, updates (ops that resulted in change)"
for k in range(1, 1 + min((len(target_fps_options), len(chokepoints), len(self.apps)))):
num_stems, num_stems_in_budget, stems_improved = 0, 0, 0
ops, updates = 0, 0
for chosen_fpses in itertools.combinations(target_fps_options, k):
chosen_fpses = list(reversed(chosen_fpses))
for chosen_chokepoints in itertools.combinations(chokepoints, k):
stem = scheduler_util.SharedStem(zip(chosen_chokepoints, chosen_fpses), self.model)
num_stems += 1
if stem.cost > cost_threshold:
continue
num_stems_in_budget += 1
best_stem_sol, (ops_, updates_) = self.best_sol_for_stem(stem, cost_benefits, cost_threshold, target_fps_options)
ops += ops_
updates += updates_
if scheduler_util.sol_better(best_result, best_stem_sol):
if self.verbose > 0:
print 'improved:', stem, best_stem_sol
stems_improved += 1
best_result = best_stem_sol
# k_steps, total stems, total stems in budget, total stems that improved over prev optimal, ops, ops per stem, updates (ops that resulted in change)
if self.verbose > 0:
print k, num_stems, num_stems_in_budget, stems_improved, ops, ops / max(1, num_stems_in_budget), updates
if self.verbose > 1:
print 'Best:', best_result[:2]
best_schedule = scheduler_util.extract_schedule(best_result[2])
if self.verbose > 1:
print 'Schedule cost:', scheduler_util.get_cost_schedule(best_schedule, self.model.layer_latencies, self.model.final_layer)
avg_metric = self.set_schedule_values(best_schedule)
return avg_metric
def greedy_scheduler(self, cost_threshold):
# Makes schedule with optimal choices for num_frozen and target_fps
# Sets self.schedule, self.num_frozen_list, self.target_fps_list
cost_benefits = self.get_cost_benefits()
target_fps_options = range(1, self.stream_fps + 1)
current_schedule = []
for app in self.apps:
num_frozen_options = app["accuracies"].keys()
cheapest_target_fps = min(target_fps_options)
cheapest_num_frozen = max(num_frozen_options)
current_schedule.append(
Schedule.ScheduleUnit(app, cheapest_target_fps,
cheapest_num_frozen))
## Make moves in order of maximal cost/benefit
## which decrease the metric and fit the budget
updated = True # Stopping condition
while (updated):
updated = False
# Get next best change to schedule
# Upgrade is (target_fps, #frozen) with larger
# cost and largest cost/benefit across all apps
max_cost_benefit = 0
best_new_unit = -1
for unit in current_schedule:
cur_target_fps = unit.target_fps
cur_num_frozen = unit.num_frozen
app_id = unit.app_id
app = unit.app
num_frozen_options = app["accuracies"].keys()
cur_metric = self.get_metric(app, cur_num_frozen,
cur_target_fps)
for potential_target_fps in target_fps_options:
for potential_num_frozen in sorted(num_frozen_options):
# Skip if it is not a change
u_apps = [
u for u in current_schedule if u.app_id == app_id
]
if (u_apps[0].num_frozen == potential_num_frozen and
u_apps[0].target_fps == potential_target_fps):
continue
cost_benefit_tup = \
cost_benefits[app_id][potential_num_frozen][potential_target_fps]
cost_benefit = cost_benefit_tup[1] / float(
cost_benefit_tup[0])
potential_metric = self.get_metric(
app, potential_num_frozen, potential_target_fps)
if potential_metric < cur_metric and cost_benefit > max_cost_benefit:
# Check that move its within budget
potential_unit = Schedule.ScheduleUnit(
app, potential_target_fps,
potential_num_frozen)
potential_schedule = []
for c_unit in current_schedule:
if c_unit.app_id == potential_unit.app_id:
potential_schedule.append(potential_unit)
else:
copy_unit = Schedule.ScheduleUnit(
c_unit.app, c_unit.target_fps,
c_unit.num_frozen)
potential_schedule.append(copy_unit)
potential_sched_cost = scheduler_util.get_cost_schedule(
potential_schedule, self.model.layer_latencies,
self.model.final_layer)
if potential_sched_cost <= cost_threshold:
cost = potential_sched_cost
max_cost_benefit = cost_benefit
best_new_unit = potential_unit
best_new_schedule = potential_schedule
updated = True
if updated:
current_schedule = best_new_schedule
average_metric = self.set_schedule_values(current_schedule)
return average_metric
def hifi_scheduler(self, cost_threshold):
cost_benefits = self.get_cost_benefits()
target_fps_options = range(1, self.stream_fps + 1)
agg_func = operator.add
# for max-min
# agg_func = min
dp = {}
cc = Counter()
def relax2(curr, best_by_budget, curr_cost, curr_goodness, c_unit,
threshold):
# curr/best_by_budget: [(benefit, min_cost), (benefit_lower, min_cost_lower)]
vals = []
for prev_goodness, prev_budget, info in reversed(best_by_budget):
new_budget = prev_budget + curr_cost
# Pruning
if new_budget > threshold:
break
new_goodness = agg_func(prev_goodness, curr_goodness)
new_budget = int(new_budget * 50) / 50.
new_goodness = int(new_goodness * 1000) / 1000.
# new_budget = round(new_budget, 1)
# new_goodness = round(new_goodness, 3)
# print (new_goodness, new_budget)
vals.append((new_goodness, new_budget, {
'unit': c_unit,
'prev': info
}))
# vals.append((new_goodness, new_budget, {'schedule': info['schedule'] + [c_unit]}))
if len(curr) == 0:
return vals
elif len(vals) == 0:
return curr
# ret = scheduler_util.make_monotonic(curr + vals)
ret = scheduler_util.merge_monotonic(curr, list(reversed(vals)))
# cc[(len(curr), len(vals), len(ret))] += 1
return ret
for i, app in enumerate(self.apps):
num_frozen_options = sorted(app["accuracies"].keys())
combos = itertools.product(target_fps_options, num_frozen_options)
for c_fps, c_frozen in combos:
c_cost, c_benefit = cost_benefits[
app["app_id"]][c_frozen][c_fps]
c_benefit = 1. - c_benefit
c_unit = Schedule.ScheduleUnit(app, c_fps, c_frozen)
if i == 0:
stem = scheduler_util.SharedStem([(c_frozen, c_fps)],
self.model)
assert stem not in dp
if stem.cost + c_cost < cost_threshold:
dp[stem] = [(c_benefit, c_cost, {
'unit': c_unit,
'prev': None
})]
# dp[stem] = [(c_benefit, c_cost, {'schedule': [c_unit]})]
else:
for stem, best_by_budget in dp_prev.iteritems():
new_stem = stem.relax(c_frozen, c_fps)
assert new_stem.cost >= stem.cost
result = relax2(dp.get(new_stem, []), best_by_budget,
c_cost, c_benefit, c_unit,
cost_threshold - new_stem.cost)
if len(result) > 0:
dp[new_stem] = result
print '{} apps'.format(i + 1)
print 'Unique stems:', len(dp)
lens_budgets_by_stem = map(len, dp.values())
budgets_by_stem = Counter(lens_budgets_by_stem)
print 'Total DP values', sum(lens_budgets_by_stem)
budgets = [y[1] for x in dp.values() for y in x]
goodnesses = [y[0] for x in dp.values() for y in x]
cnt_budgets = Counter(budgets)
cnt_goodness = Counter(goodnesses)
def bucket_stats(vals):
ret = [Counter(map(int, vals))]
return ret + [
Counter(map(lambda x: int(x * k) / k, vals))
for k in [10., 100., 1000., 10000.]
]
cnt_budgets_buckets = bucket_stats(budgets)
cnt_goodness_buckets = bucket_stats(goodnesses)
print 'Unique budgets:', len(cnt_budgets)
print 'Budget buckets by int, .1, .01, .001:', map(
len, cnt_budgets_buckets)
print 'Unique goodness scores', len(cnt_goodness)
print 'Goodness buckets by int, .1, .01, .001:', map(
len, cnt_goodness_buckets)
print 'Budgets per stem', budgets_by_stem
# print 'Budgets:', ', '.join(map('{:.0f}'.format, sorted(cnt_budgets.keys(), reverse=True)))
# print 'Budgets:', sorted(map(int, cnt_budgets.keys()), reverse=True)
print 'Budgets by ints:', cnt_budgets_buckets[0]
# print 'Some budgets:', map('{:g}'.format, sorted(cnt_budgets.keys()))
# print 'Num of DP values by budget', sorted(cnt_budgets.values(), reverse=True)
# print 'Num of DP values by goodness', sorted(cnt_goodness.values(), reverse=True)
# print 'curr, vals:', cc
cc.clear()
print
dp_prev = dp
dp = {}
options = []
for stem, best_by_budget in dp_prev.iteritems():
options += [(goodness, budget + stem.cost, info)
for goodness, budget, info in best_by_budget
if budget + stem.cost <= cost_threshold]
results = scheduler_util.make_monotonic(options)
def extract_schedule(info_dct):
schedule = [info_dct['unit']]
while info_dct['prev'] is not None:
info_dct = info_dct['prev']
schedule.insert(0, info_dct['unit'])
return schedule
best_result = results[0]
print 'Best:', best_result[:2]
# best_schedule = best_result[2]['schedule']
best_schedule = extract_schedule(best_result[2])
print 'Schedule cost:', scheduler_util.get_cost_schedule(
best_schedule, self.model.layer_latencies, self.model.final_layer)
avg_metric = self.set_schedule_values(best_schedule)
return avg_metric
def hifi_scheduler(self, cost_threshold, mode, dp={}):
cost_benefits = self.get_cost_benefits(mode)
target_fps_options = self._get_target_fps_options(mode)
func_init, agg_func = self.get_init_agg_func()
dp = {}
num_apps = 0
if len(dp) > 0:
num_apps = dp["num_apps"]
dp_prev = dict(dp)
dp = {}
def relax2(curr, best_by_budget, curr_cost, curr_goodness, c_unit, threshold):
# curr/best_by_budget: [(benefit, min_cost), (benefit_lower, min_cost_lower)]
vals = []
for prev_goodness, prev_budget, info in reversed(best_by_budget):
new_budget = prev_budget + curr_cost
# Pruning
if new_budget > threshold:
# Note: break depends on reversed, otherwise must be continue.
break
new_goodness = agg_func(prev_goodness, curr_goodness)
# new_budget = math.ceil(new_budget * 50) / 50.
# new_goodness = int(new_goodness * 1000) / 1000.
vals.append((new_goodness, new_budget, {'unit': c_unit, 'prev': info}))
if len(curr) == 0:
return vals
elif len(vals) == 0:
return curr
ret = scheduler_util.merge_monotonic(curr, list(reversed(vals)))
return ret
for i, app in enumerate(self.apps):
if i < num_apps:
continue
dp.clear()
dp["num_apps"] = i+1
dp_prev_only = {k: v for k, v in dp_prev.items() if k != "num_apps"}
num_frozen_options = self._get_num_frozen_options(app, mode)
for c_frozen in num_frozen_options:
p_benefit = 0
for c_fps in target_fps_options:
c_cost, c_benefit = cost_benefits[app["app_id"]][c_frozen][c_fps]
c_benefit = func_init(1. - c_benefit)
if c_benefit <= p_benefit:
break
p_benefit = c_benefit
c_unit = Schedule.ScheduleUnit(app, c_fps, c_frozen)
if i == 0:
stem = scheduler_util.SharedStem([(c_frozen, c_fps)], self.model)
assert stem not in dp
if stem.cost + c_cost <= cost_threshold:
dp[stem] = [(c_benefit, c_cost, {'unit': c_unit, 'prev': None})]
else:
for stem, best_by_budget in dp_prev_only.iteritems():
new_stem = stem.relax(c_frozen, c_fps)
assert new_stem.cost >= stem.cost
result = relax2(dp.get(new_stem, []), best_by_budget, c_cost, c_benefit, c_unit, cost_threshold - new_stem.cost)
if len(result) > 0:
dp[new_stem] = result
if self.verbose > -1:
print '{} apps'.format(i+1),
dp_only = {k: v for k, v in dp.items() if k != "num_apps"}
print 'Unique stems:', len(dp_only)
lens_budgets_by_stem = map(len, dp_only.values())
budgets_by_stem = Counter(lens_budgets_by_stem)
print 'Total DP values', sum(lens_budgets_by_stem)
if self.verbose > 1:
budgets = [y[1] for x in dp_only.values() for y in x]
goodnesses = [y[0] for x in dp_only.values() for y in x]
cnt_budgets = Counter(budgets)
cnt_goodness = Counter(goodnesses)
def bucket_stats(vals):
ret = [Counter(map(int, vals))]
return ret + [Counter(map(lambda x: int(x * k) / k, vals)) for k in [10., 100., 1000., 10000.]]
cnt_budgets_buckets = bucket_stats(budgets)
cnt_goodness_buckets = bucket_stats(goodnesses)
print 'Unique budgets:', len(cnt_budgets)
print 'Budget buckets by int, .1, .01, .001:', map(len, cnt_budgets_buckets)
print 'Unique goodness scores', len(cnt_goodness)
print 'Goodness buckets by int, .1, .01, .001:', map(len, cnt_goodness_buckets)
print 'Budgets per stem', budgets_by_stem
# print 'Budgets:', ', '.join(map('{:.0f}'.format, sorted(cnt_budgets.keys(), reverse=True)))
# print 'Budgets:', sorted(map(int, cnt_budgets.keys()), reverse=True)
print 'Budgets by ints:', cnt_budgets_buckets[0]
# print 'Some budgets:', map('{:g}'.format, sorted(cnt_budgets.keys()))
# print 'Num of DP values by budget', sorted(cnt_budgets.values(), reverse=True)
# print 'Num of DP values by goodness', sorted(cnt_goodness.values(), reverse=True)
# print 'curr, vals:', cc
cc.clear()
print
if i > 0:
del dp_prev
gc.collect()
dp_prev = dp_only
options = []
for stem, best_by_budget in dp_prev.iteritems():
options += [(goodness, budget + stem.cost, info) for goodness, budget, info in best_by_budget if budget + stem.cost <= cost_threshold]
results = scheduler_util.make_monotonic(options)
best_result = results[0]
if self.verbose > 1:
print 'Best:', best_result[:2]
# best_schedule = best_result[2]['schedule']
best_schedule = scheduler_util.extract_schedule(best_result[2])
if self.verbose > 1:
print 'Schedule cost:', scheduler_util.get_cost_schedule(best_schedule, self.model.layer_latencies, self.model.final_layer)
avg_metric = self.set_schedule_values(best_schedule)
return avg_metric