def attach_plot_components(
video: str,
server: Server,
trace: Optional[str] = None,
bandwidth: Optional[int] = None,
no_plot: bool = False,
) -> Dict[str, LivePlot]:
"""
Attach multiple plot components with diverse metrics to react to and plot
Monitor-generated metrics.
"""
segments = get_video_chunks(video) + 1
max_playback = int(1.25 * get_approx_video_length(video))
if no_plot:
(server.add_post('/raw_qoe', do_nothing).add_post(
'/rebuffer', do_nothing).add_post('/quality', do_nothing).add_post(
'/vmaf',
do_nothing).add_post('/vmaf_qoe',
do_nothing).add_post('/bw', do_nothing))
return {}
plots = {
'raw_qoe':
LivePlot(figure_name='qoe', y_label='raw qoe', range_size=segments),
'rebuffer':
LivePlot(figure_name='rebuffer',
y_label='rebuffer',
range_size=segments),
'quality':
LivePlot(figure_name='quality', y_label='quality',
range_size=segments),
'vmaf':
LivePlot(figure_name='vmaf', y_label='vmaf', range_size=segments),
'vmaf_qoe':
LivePlot(figure_name='vmaf_qoe',
y_label='vmaf qoe',
range_size=segments),
'bw':
BandwidthPlot(
figure_name='bw',
y_label='bw estimation(mbps)',
trace=trace,
bandwidth=bandwidth,
range_size=max_playback,
),
}
(server.add_post('/raw_qoe', plots['raw_qoe']).add_post(
'/rebuffer',
plots['rebuffer']).add_post('/quality', plots['quality']).add_post(
'/vmaf', plots['vmaf']).add_post('/vmaf_qoe',
plots['vmaf_qoe']).add_post(
'/bw', plots['bw']))
return plots
async def export_plots(self) -> None:
for name, plot in self.plots.items():
xs = list(range(1, get_video_chunks(self.video)))
with open(self.plots_path, 'a') as f:
for x in xs:
out = {'x': x}
out[name] = {}
for ds_name, dataset in plot.datasets.items():
out[name][ds_name] = dataset.y[x]
f.write(json.dumps(out))
f.write('\n')
if len(self.plots) > 0:
for plot in self.plots.values():
await plot.draw()
ref_plot = list(self.plots.values())[0]
ref_plot.figure.savefig(self.draw_path)
def hetero(args: Namespace) -> None:
global run_trace, run_subexp
if args.dry:
run_trace = lambda *args, **kwargs: None
run_subexp = lambda *args, **kwargs: None
videos = ['got', 'bojack', 'guard']
root_path = str(Path("experiments") / "hetero")
os.system(f"mkdir -p {root_path}")
runner_log = open(str(Path(root_path) / 'exp.log'), 'w')
# only for rmpc at the moment
compete1 = [
('robustMpc', 'bbr2'),
('dynamic', 'bbr2'),
('robustMpc', 'cubic'),
('dynamic', 'cubic'),
]
compete2 = [
('gap', 'gap'),
]
minerva = [
('minerva', 'minerva'),
('minervann', 'minerva'),
]
for i, video1 in enumerate(videos):
for j, video2 in enumerate(videos):
if i != j:
shorter_video = video1 if get_video_chunks(
video1) < get_video_chunks(video2) else video2
experiments = []
experiment_path = str(Path(root_path) / f"{video1}_{video2}")
for run_id in range(4):
latency = 500
# robustMpc vs others
for bandwidth in [3, 2, 1]:
subpath = str(Path(experiment_path) / "versus_rmpc")
for (algo1, cc1) in compete1:
for (algo2, cc2) in compete2:
server1 = f"--algo {algo1} --name robustMpc --cc {cc1} --video {video1}"
server2 = f"--server-algo {algo2} --name abrcc --cc {cc2} --video {video2}"
path = str(
Path(subpath) /
f"{cc1}_{algo2}_{cc2}_{bandwidth}_run{run_id}"
)
if algo1 != 'robustMpc': # since we don't want to repet old experiments
path = str(
Path(subpath) /
f"{algo1}_{cc1}_{algo2}_{cc2}_{bandwidth}_run{run_id}"
)
runner_log.write(f'> {path}\n')
run_subexp(bandwidth,
latency,
path, [server1, server2],
burst=2000,
video=shorter_video,
headless=args.headless)
if cc2 == "gap":
cc2 = "gap2"
experiments.append(
Experiment(
video=shorter_video,
path=str(
Path(path) / "leader_plots.log"),
latency=latency,
bandwidth=bandwidth,
extra=[
"versus", algo1, cc1, algo2, cc2,
f"{video1}1", f"{video2}2"
],
run_id=run_id,
))
# self
subpath = str(Path(experiment_path) / "versus_self")
for (algo, cc) in compete2 + minerva:
server1 = f"--server-algo {algo} --name abrcc1 --cc {cc} --video {video1}"
server2 = f"--server-algo {algo} --name abrcc2 --cc {cc} --video {video2}"
if algo == "minerva" or algo == "minervann":
server1 += f" --algo {algo}"
server2 += f" --algo {algo}"
path = str(
Path(subpath) /
f"{algo}_{cc}_{bandwidth}_run{run_id}")
runner_log.write(f'> {path}\n')
run_subexp(bandwidth,
latency,
path, [server1, server2],
burst=2000,
video=shorter_video,
headless=args.headless)
if cc == "gap":
cc = "gap2"
experiments.append(
Experiment(
video=shorter_video,
path=str(Path(path) / "leader_plots.log"),
latency=latency,
bandwidth=bandwidth,
extra=[
"self", algo, cc, f"{video1}1",
f"{video2}"
],
run_id=run_id,
))
# robustMpc
subpath = str(Path(experiment_path) / "rmpc")
for cc1, cc2 in [('cubic', 'bbr2'), ('bbr2', 'bbr2'),
('cubic', 'cubic')]:
for algo in ['robustMpc', 'dynamic']:
server1 = f"--algo {algo} --name rmpc1 --cc {cc1} --video {video1}"
server2 = f"--algo {algo} --name rmpc2 --cc {cc2} --video {video2}"
path = str(
Path(subpath) /
f"{cc1}_{cc2}_{bandwidth}_run{run_id}")
if algo != 'robustMpc': # since we don't want to repet old experiments
path = str(
Path(subpath) /
f"{algo}_{cc1}_{cc2}_{bandwidth}_run{run_id}"
)
runner_log.write(f'> {path}\n')
run_subexp(bandwidth,
latency,
path, [server1, server2],
burst=2000,
video=shorter_video,
headless=args.headless)
extra = 'rmpc'
if algo == 'dynamic':
extra = 'dynamic'
experiments.append(
Experiment(
video=shorter_video,
path=str(
Path(path) / "leader_plots.log"),
latency=latency,
bandwidth=bandwidth,
extra=[
extra, cc1 + '1', cc2 + '2',
f"{video1}1", f"{video2}2"
],
run_id=run_id,
))
if args.dry:
print(experiments)
print(len(experiments))
else:
save_experiments(experiment_path, experiments)
generate_summary(experiment_path, experiments)
def get_metrics(self) -> List[ExperimentMetric]:
"""
Computes a list of per-experiment data point metrics.
"""
try:
segments = get_video_chunks(self.video) + 1
with open(self.path, 'r') as graph_log:
raw_qoe = defaultdict(lambda: defaultdict(int))
vmaf_qoe = defaultdict(lambda: defaultdict(int))
vmafd = defaultdict(lambda: defaultdict(int))
rebuffd = defaultdict(lambda: defaultdict(int))
def process(_input):
out = []
for line in _input.split('\n'):
vals = line.split('}{')
for i, val in enumerate(vals):
val = '{' + val if i > 0 else val
val = val + '}' if i < len(vals) - 1 else val
out.append(val)
return out
for line in process(graph_log.read()):
def proc_metric(curr_dict, metric):
if metric not in obj:
return
x = obj['x']
for name, value in obj[metric].items():
# not great
if x > 0 and x < segments - 2:
curr_dict[name][x] = value
def proc_vmaf(curr_dict, vmaf, rebuff):
x = obj['x']
for name in vmaf.keys():
if x - 1 in vmaf[name]:
curr_dict[name][x] = (
vmaf[name][x] - 100. * rebuff[name][x] -
2.5 *
abs(vmaf[name][x] - vmaf[name][x - 1]))
else:
curr_dict[name][x] = (
vmaf[name][x] - 100. * rebuff[name][x] -
2.5 * abs(vmaf[name][x] - 0))
obj = None
try:
obj = json.loads(line)
except:
pass
if obj:
proc_metric(vmafd, "vmaf")
proc_metric(rebuffd, "rebuffer")
proc_metric(raw_qoe, "raw_qoe")
x = obj['x']
try:
algo = list(obj[list(set(obj.keys()) -
{'x'})[0]].keys())[0]
if x in vmafd[algo] and x in rebuffd[algo]:
proc_vmaf(vmaf_qoe, vmafd, rebuffd)
except:
pass
out = []
for name in raw_qoe.keys():
qoe_vals = list(raw_qoe[name].values())
total_qoe = sum(qoe_vals) / len(qoe_vals)
vmaf_vals = list(vmaf_qoe[name].values())
total_vmaf = sum(vmaf_vals) / len(qoe_vals)
vmaf_vals = list(vmafd[name].values())
total_rebuf = sum(vmaf_vals) / len(vmaf_vals)
out.append(RawQoe(name, total_qoe))
out.append(VmafQoe(name, total_vmaf))
out.append(Vmaf(name, total_rebuf))
return out
except FileNotFoundError as e:
return []
async def process(self, json: JSONType) -> JSONType:
total_rebuffer = float(json['rebuffer']) / M_IN_K
# compute data
rebuffer_time = total_rebuffer - self.last_rebuffer_time
last_quality = self.last_quality
last_bit_rate = self.last_bit_rate
reward = (get_video_bit_rate(self.video, last_quality) / M_IN_K
- REBUF_PENALTY * rebuffer_time / M_IN_K
- SMOOTH_PENALTY * np.abs(get_video_bit_rate(self.video, last_quality)
- last_bit_rate) / M_IN_K)
# retrieve previous state
if len(self.s_batch) == 0:
state = [np.zeros((S_INFO, S_LEN))]
else:
state = np.array(self.s_batch[-1], copy=True)
# compute bandwidth measurement
bandwidth = max(float(json['bandwidth']), MIN_BW_EST_MBPS * M_IN_K)
chunk_fetch_time = float(json['last_fetch_time'])
# compute number of video chunks left
video_chunk_remain = get_video_chunks(self.video) - self.video_chunk_count
self.video_chunk_count += 1
# dequeue history record
state = np.roll(state, -1, axis=1)
next_video_chunk_sizes = []
for i in range(self.adim):
next_video_chunk_sizes.append(get_chunk_size(
self.video, i, self.video_chunk_count
))
total_buffer = float(json['buffer'])
# this should be S_INFO number of terms
try:
state[0, -1] = (
get_video_bit_rate(self.video, last_quality) /
get_max_video_bit_rate(self.video)
)
state[1, -1] = total_buffer / M_IN_K / BUFFER_NORM_FACTOR # s
state[2, -1] = bandwidth / M_IN_K / 8 # k byte / ms
state[3, -1] = float(chunk_fetch_time) / M_IN_K / BUFFER_NORM_FACTOR # 10 s
state[4, :self.adim] = np.array(next_video_chunk_sizes) / M_IN_K / M_IN_K # m byte
state[5, -1] = np.minimum(video_chunk_remain, CHUNK_TIL_VIDEO_END_CAP) / float(CHUNK_TIL_VIDEO_END_CAP)
print(state[:, -1])
except ZeroDivisionError:
if len(self.s_batch) == 0:
state = [np.zeros((S_INFO, S_LEN))]
else:
state = np.array(self.s_batch[-1], copy=True)
action_prob = self.actor.predict(np.reshape(state, (1, S_INFO, S_LEN)))
action_cumsum = np.cumsum(action_prob)
bit_rate = (action_cumsum > np.random.randint(1, RAND_RANGE) / float(RAND_RANGE)).argmax()
self.s_batch.append(state)
print(f"[Pensieve] > bit rate {bit_rate}")
self.last_rebuffer_time = total_rebuffer
self.last_bit_rate = get_video_bit_rate(self.video, last_quality)
self.last_quality = bit_rate
return {
'decision' : float(bit_rate),
}
async def process(self, json: JSONType) -> JSONType:
start = timeit.timeit()
print(f"[robustMpc] > serving {json}")
total_rebuffer = float(json['rebuffer']) / M_IN_K
last_quality = self.last_quality
last_bit_rate = self.last_bit_rate
rebuffer_time = total_rebuffer - self.last_rebuffer_time
smooth_diff = abs(
get_video_bit_rate(self.video, last_quality) - last_bit_rate)
# compute reward
reward = (get_video_bit_rate(self.video, last_quality) / M_IN_K -
REBUF_PENALTY * rebuffer_time / M_IN_K -
SMOOTH_PENALTY * smooth_diff / M_IN_K)
# update state
self.last_bit_rate = get_video_bit_rate(self.video, last_quality)
self.last_rebuffer_time = total_rebuffer
# compute bandwidth measurement
bandwidth = max(float(json['bandwidth']) / M_IN_K, MIN_BW_EST_MBPS)
self.bandwidths.append(bandwidth)
# compute number of video chunks left
index = json['index']
video_chunk_remain = get_video_chunks(self.video) - index
# past error
if len(self.past_bandwidth_ests) == 0:
self.past_errors.append(0)
else:
curr_error = abs(self.past_bandwidth_ests[-1] -
bandwidth) / bandwidth
self.past_errors.append(curr_error)
# past bandwidths
harmonic_bandwidth = statistics.harmonic_mean(
self.bandwidths[-HORIZON:])
max_error = max(self.past_errors[-HORIZON:])
future_bandwidth = max(harmonic_bandwidth / (1 + max_error),
MIN_BW_EST_MBPS)
self.past_bandwidth_ests.append(harmonic_bandwidth)
print(f"[robustMpc] > future bandwidth {future_bandwidth}")
# max reward
start_buffer = float(json['buffer']) / M_IN_K
print(f"[robustMpc] > current buffer {start_buffer}")
max_reward, best_rate = 0, 0
rates = get_nbr_video_bit_rate(self.video)
for full_combo in itertools.product(range(rates), repeat=HORIZON):
combo = full_combo[:min(
get_video_chunks(self.video) - index, HORIZON)]
curr_rebuffer_time = 0
curr_buffer = start_buffer
bitrate_sum = 0
smoothness_diff = 0
last_quality = self.last_quality
for position in range(len(combo)):
chunk_quality = combo[position]
curr_index = index + position
size = (8 *
get_chunk_size(self.video, chunk_quality, curr_index) /
M_IN_K**2) # in mb
download_time = size / future_bandwidth # in s
# simulate future buffer
if curr_buffer < download_time:
curr_rebuffer_time += download_time - curr_buffer
curr_buffer = 0
else:
curr_buffer -= download_time
curr_buffer += get_chunk_time(self.video, chunk_quality,
curr_index)
# linear reward for the buffer
bitrate_sum += get_video_bit_rate(self.video, chunk_quality)
smoothness_diff += abs(
get_video_bit_rate(self.video, chunk_quality) -
get_video_bit_rate(self.video, last_quality))
last_quality = chunk_quality
# total reward for whole simualtion
reward = (bitrate_sum / M_IN_K -
REBUF_PENALTY * curr_rebuffer_time -
smoothness_diff / M_IN_K)
if reward > max_reward:
max_reward = reward
best_rate = combo[0]
self.last_quality = best_rate
print(f"[robustMpc] > decision {best_rate}")
end = timeit.timeit()
print(f"[robustMpc] > time {end - start}")
return {
'decision': best_rate,
}