def black_box_function(bandwidth_lower_bound: float, bandwidth_upper_bound: float, delay: float, queue: float, loss: float, T_s: float, delay_noise: float, heuristic, model_path: str, save_dir: str = "") -> float: global black_box_function_calling_times save_dir = os.path.join(save_dir, 'config_{}'.format( black_box_function_calling_times % 15)) black_box_function_calling_times += 1 heuristic_rewards = [] rl_method_rewards = [] if loss < -4: loss = 0 else: loss = 10**loss traces = [generate_trace( duration_range=(30, 30), bandwidth_lower_bound_range=( 10**bandwidth_lower_bound, 10**bandwidth_lower_bound), bandwidth_upper_bound_range=( 10**bandwidth_upper_bound, 10**bandwidth_upper_bound), delay_range=(delay, delay), loss_rate_range=(loss, loss), queue_size_range=(queue, queue), T_s_range=(T_s, T_s), delay_noise_range=(delay_noise, delay_noise)) for _ in range(10)] # print("trace generation used {}s".format(time.time() - t_start)) save_dirs = [os.path.join(save_dir, 'trace_{}'.format(i)) for i in range(10)] for trace, save_dir in zip(traces, save_dirs): os.makedirs(save_dir, exist_ok=True) trace.dump(os.path.join(save_dir, 'trace.json')) if not heuristic: for trace in traces: heuristic_rewards.append(trace.optimal_reward) else: t_start = time.time() # save_dirs = [""] * len(traces) hret = heuristic.test_on_traces(traces, save_dirs, True, 8) for heuristic_mi_level_reward, heuristic_pkt_level_reward in hret: # heuristic_rewards.append(heuristic_mi_level_reward) heuristic_rewards.append(heuristic_pkt_level_reward) print("heuristic used {}s".format(time.time() - t_start)) t_start = time.time() rl_ret = test_on_traces(model_path, traces, save_dirs, 8, 20, record_pkt_log=False, plot_flag=True) for rl_mi_level_reward, rl_pkt_level_reward in rl_ret: # rl_method_rewards.append(rl_mi_level_reward) rl_method_rewards.append(rl_pkt_level_reward) print("rl used {}s".format(time.time() - t_start)) gap = float(np.mean(np.array(heuristic_rewards)) - np.mean(np.array(rl_method_rewards))) return gap
def main(): args = parse_args() set_seed(args.seed) if args.heuristic == 'cubic': cc = Cubic() elif args.heuristic == 'bbr_old': cc = BBR_old() elif args.heuristic == 'bbr': cc = BBR() else: raise NotImplementedError # if 'large' in args.config_file: # config = read_json_file(args.config_file) # config[0]['bandwidth_lower_bound'] = (1, 1) # config[0]['bandwidth_upper_bound'] = (1, 100) # traces = generate_traces_from_config(config, 50, 30) # else: if not args.config_file: dataset = PantheonDataset('../../data', 'all') traces = dataset.get_traces(0, 50) save_dirs = [os.path.join( args.save_dir, link_conn_type, link_name, trace_name) for link_conn_type, (link_name, trace_name) in zip(dataset.link_conn_types, dataset.trace_names)] else: traces = generate_traces(args.config_file, 50, 30) save_dirs = [os.path.join(args.save_dir, "trace_{:02d}".format(i)) for i in range(len(traces))] cc_save_dirs = [os.path.join(save_dir, cc.cc_name) for save_dir in save_dirs] cc_res = cc.test_on_traces(traces, cc_save_dirs, plot_flag=False, n_proc=16) aurora_save_dirs = [os.path.join(save_dir, 'aurora') for save_dir in save_dirs] aurora_res = test_on_traces(args.model_path, traces, aurora_save_dirs, nproc=16, seed=42, record_pkt_log=False, plot_flag=False) print(cc.cc_name, np.mean([res[1] for res in cc_res])) print('aurora', np.mean([res[1] for res in aurora_res])) for i, (trace, save_dir) in enumerate(zip(traces, save_dirs)): trace.dump(os.path.join(save_dir, 'trace_{:02d}.json'.format(i)))
def main(): args = parse_args() set_seed(args.seed) # tokens = os.path.basename(os.path.dirname(os.path.dirname(args.save_dir))).split('_') # config0_dim0_idx = int(tokens[1]) # config0_dim1_idx = int(tokens[2]) # config1_dim0_idx = int(tokens[4]) # config1_dim1_idx = int(tokens[5]) dim0, dim1 = args.dims config = read_json_file(args.config_file)[0] assert dim0 in config and dim1 in config # dim0_vals = np.linspace(config[dim0][0], config[dim0][1], 10) # dim1_vals = np.linspace(config[dim1][0], config[dim1][1], 10) dim0_vals = get_dim_vals(dim0) dim1_vals = get_dim_vals(dim1) print(dim0_vals) print(dim1_vals) traces = [] save_dirs = [] with open('heatmap_trace_cnt_ratio.npy', 'rb') as f: cnt_ratio = np.load(f) for dim0_idx, dim0_val in enumerate(dim0_vals): for dim1_idx, dim1_val in enumerate(dim1_vals): dim_vals = copy.copy(DEFAULT_VALUES) dim_vals[dim0] = dim0_val dim_vals[dim1] = dim1_val # print(i, dim0_val, dim1_val, dim_vals) cnt = 10 # if cnt_ratio[dim0_idx, dim1_idx] > 1: # cnt *= int(cnt_ratio[dim0_idx, dim1_idx]) # print(cnt) for trace_idx in range(cnt): trace = generate_trace( duration_range=(dim_vals['duration'], dim_vals['duration']), bandwidth_lower_bound_range=( dim_vals['bandwidth_lower_bound'], dim_vals['bandwidth_lower_bound']), bandwidth_upper_bound_range=( dim_vals['bandwidth_upper_bound'], dim_vals['bandwidth_upper_bound']), delay_range=(dim_vals['delay'], dim_vals['delay']), loss_rate_range=(dim_vals['loss'], dim_vals['loss']), queue_size_range=(dim_vals['queue'], dim_vals['queue']), T_s_range=(dim_vals['T_s'], dim_vals['T_s']), delay_noise_range=(dim_vals['delay_noise'], dim_vals['delay_noise'])) traces.append(trace) save_dir = os.path.join( args.save_dir, 'pair_{}_{}'.format(dim0_idx, dim1_idx), 'trace_{}'.format(trace_idx)) save_dirs.append(save_dir) os.makedirs(save_dir, exist_ok=True) trace.dump( os.path.join(save_dir, 'trace_{}.json'.format(trace_idx))) if args.cc == 'genet_bbr' or args.cc == 'genet_cubic' or args.cc == 'genet_bbr_old': genet_seed = '' for s in args.models_path.split('/'): if 'seed' in s: genet_seed = s for bo in range(0, 30, 3): # for bo_dir in natural_sort(glob.glob(os.path.join(args.models_path, "bo_*/"))): bo_dir = os.path.join(args.models_path, "bo_{}".format(bo)) step = 64800 model_path = os.path.join(bo_dir, 'model_step_{}.ckpt'.format(step)) if not os.path.exists(model_path + '.meta'): print(model_path, 'does not exist') continue print(model_path) genet_save_dirs = [ os.path.join(save_dir, args.cc, genet_seed, "bo_{}".format(bo), "step_{}".format(step)) for save_dir in save_dirs ] t_start = time.time() test_on_traces(model_path, traces, genet_save_dirs, args.nproc, 42, False, False) print('bo {}: {:.3f}'.format(bo, time.time() - t_start)) elif args.cc == 'pretrained': pretrained_save_dirs = [ os.path.join(save_dir, args.cc) for save_dir in save_dirs ] t_start = time.time() test_on_traces(args.models_path, traces, pretrained_save_dirs, args.nproc, 42, False, False) print('pretrained: {:.3f}'.format(time.time() - t_start)) elif args.cc == 'overfit_config': overfit_config_save_dirs = [ os.path.join(save_dir, args.cc) for save_dir in save_dirs ] t_start = time.time() test_on_traces(args.models_path, traces, overfit_config_save_dirs, args.nproc, 42, False, False) print('overfit_config: {:.3f}'.format(time.time() - t_start)) else: if args.cc == 'bbr': cc = BBR(False) elif args.cc == 'cubic': cc = Cubic(False) elif args.cc == 'bbr_old': cc = BBR_old(False) else: raise NotImplementedError heuristic_save_dirs = [ os.path.join(save_dir, cc.cc_name) for save_dir in save_dirs ] t_start = time.time() cc.test_on_traces(traces, heuristic_save_dirs, False, args.nproc) print('{}: {:.3f}'.format(args.cc, time.time() - t_start))
def main(): args = parse_args() # dataset = SyntheticDataset(25, '../../config/train/udr_7_dims_0826/udr_large.json', seed=42) dataset = SyntheticDataset.load_from_dir( '/datamirror/zxxia/PCC-RL/results_1006/synthetic_dataset') traces = dataset.traces # cnt = 0 # for idx, trace in enumerate(traces): # if trace.min_bw > 0.5: # cnt+=1 # print(idx) # print(cnt) # return save_dirs = [ os.path.join(args.save_dir, 'trace_{:05d}'.format(i)) for i in range(len(dataset)) ] if args.cc == 'bbr': cc = BBR(False) cc.test_on_traces( traces, [os.path.join(save_dir, cc.cc_name) for save_dir in save_dirs], True, args.nproc) elif args.cc == 'bbr_old': cc = BBR_old(False) cc.test_on_traces( traces, [os.path.join(save_dir, cc.cc_name) for save_dir in save_dirs], True, args.nproc) elif args.cc == 'cubic': cc = Cubic(False) cc.test_on_traces( traces, [os.path.join(save_dir, cc.cc_name) for save_dir in save_dirs], True, args.nproc) elif args.cc == 'real': model_path = args.models_path real_save_dirs = [ os.path.join(save_dir, args.cc, "seed_{}".format(args.seed)) for save_dir in save_dirs ] test_on_traces(model_path, traces, real_save_dirs, args.nproc, 42, False, plot_flag=True) elif args.cc == 'pretrained': udr_seed = '' for s in args.models_path.split('/'): if 'seed' in s: udr_seed = s step = 7200 while step <= 151200: if not os.path.exists( os.path.join(args.models_path, 'model_step_{}.ckpt.meta'.format(step))): break udr_save_dirs = [ os.path.join(save_dir, args.cc, udr_seed, "step_{}".format(step)) for save_dir in save_dirs ] model_path = os.path.join(args.models_path, 'model_step_{}.ckpt'.format(step)) test_on_traces(model_path, traces, udr_save_dirs, args.nproc, 42, False, True) step += 28800 elif args.cc == 'udr1' or args.cc == 'udr2' or args.cc == 'udr3' or \ args.cc == 'cl1' or args.cc == 'cl1_new' or args.cc == 'cl2' or \ args.cc == 'cl2_new' or args.cc == 'real_cellular' or \ 'udr' in args.cc or 'cl' in args.cc: # TODO: bug here when there is no validation log # original implementation # val_log = pd.read_csv(os.path.join( # args.models_path, 'validation_log.csv'), sep='\t') # for idx in np.linspace(0, len(val_log['num_timesteps']) / 2 - 1, 10): # step = int(val_log['num_timesteps'].iloc[int(idx)]) # udr_seed = '' # for s in args.models_path.split('/'): # if 'seed' in s: # udr_seed = s # udr_save_dirs = [os.path.join( # save_dir, args.cc, udr_seed, "step_{}".format(step)) for save_dir in save_dirs] # model_path = os.path.join( # args.models_path, 'model_step_{}.ckpt'.format(step)) # test_on_traces(model_path, traces, udr_save_dirs, # args.nproc, 42, False, False) # new implementation to be fair with genet udr_seed = '' for s in args.models_path.split('/'): if 'seed' in s: udr_seed = s step = 0 while step <= 720000: if not os.path.exists( os.path.join(args.models_path, 'model_step_{}.ckpt.meta'.format(step))): break udr_save_dirs = [ os.path.join(save_dir, args.cc, udr_seed, "step_{}".format(step)) for save_dir in save_dirs ] model_path = os.path.join(args.models_path, 'model_step_{}.ckpt'.format(step)) test_on_traces(model_path, traces, udr_save_dirs, args.nproc, 42, False, True) # step += (7200) * 2 step += 72000 print(step) # elif args.cc == 'genet_bbr' or args.cc == 'genet_cubic' or 'genet_bbr_old': elif 'genet' in args.cc: #== 'genet_bbr' or args.cc == 'genet_cubic' or 'genet_bbr_old': genet_seed = '' for s in args.models_path.split('/'): if 'seed' in s: genet_seed = s # for bo in range(0, 15, 3): for bo in range(0, 10): bo_dir = os.path.join(args.models_path, "bo_{}".format(bo)) for step in range(64800, 72000, 14400): # step = 64800 model_path = os.path.join(bo_dir, 'model_step_{}.ckpt'.format(step)) if not os.path.exists(model_path + '.meta'): print("skip " + model_path + '.meta') continue genet_save_dirs = [ os.path.join(save_dir, args.cc, genet_seed, "bo_{}".format(bo), "step_{}".format(step)) for save_dir in save_dirs ] # print(genet_save_dirs) test_on_traces(model_path, traces, genet_save_dirs, args.nproc, 42, False, True) else: raise ValueError
def main(): args = parse_args() dataset = SyntheticDataset( 1000, '../../config/train/udr_7_dims_0826/udr_large.json') save_dirs = [ os.path.join(args.save_dir, 'trace_{:05d}'.format(i)) for i in range(len(dataset)) ] for seed in range(60, 70, 10): bo = 9 step = 64800 # cc = BBR_old(False) # cc.test_on_traces(dataset.traces, [os.path.join(save_dir, cc.cc_name) # for save_dir in save_dirs], True, args.nproc) # cc = BBR(False) # cc.test_on_traces(dataset.traces, [os.path.join(save_dir, cc.cc_name) # for save_dir in save_dirs], True, args.nproc) # cc = Cubic(False) # cc.test_on_traces(dataset.traces, [os.path.join(save_dir, cc.cc_name) # for save_dir in save_dirs], True, args.nproc) # break # model_path = "/datamirror/zxxia/PCC-RL/results_1006/genet_better_bw/genet_bbr_old/seed_{}/bo_{}/model_step_{}.ckpt".format(seed, bo, step) # genet_save_dirs = [os.path.join( # save_dir, 'genet_bbr_old_better_bw', "seed_{}".format(seed), "bo_{}".format(bo), # "step_{}".format(step)) for save_dir in save_dirs] # test_on_traces(model_path, dataset.traces, genet_save_dirs, args.nproc, 42, False, False) # model_path = "/datamirror/zxxia/PCC-RL/results_1006/genet_random_real/0.05/seed_{}/bo_{}/model_step_{}.ckpt".format(seed, bo, step) # genet_save_dirs = [os.path.join( # save_dir, 'genet_bbr_old_5percent_real', "seed_{}".format(seed), "bo_{}".format(bo), # "step_{}".format(step)) for save_dir in save_dirs] # test_on_traces(model_path, dataset.traces, genet_save_dirs, args.nproc, 42, False, False) # model_path = "/tank/zxxia/PCC-RL/results_0928/genet_no_reward_scale/genet_bbr_old/seed_{}/bo_{}/model_step_{}.ckpt".format(seed, bo, step) # genet_save_dirs = [os.path.join( # save_dir, 'genet_bbr_old_cpu', "seed_{}".format(seed), "bo_{}".format(bo), # "step_{}".format(step)) for save_dir in save_dirs] # test_on_traces(model_path, dataset.traces, genet_save_dirs, args.nproc, 42, False, True) # step = 705600 # model_path = "/datamirror/zxxia/PCC-RL/results_1006/train_with_difficulty_metrics/metric1_new/seed_{}/model_step_{}.ckpt".format(seed, step) # genet_save_dirs = [os.path.join( # save_dir, 'cl1', "seed_{}".format(seed), # "step_{}".format(step)) for save_dir in save_dirs] # test_on_traces(model_path, dataset.traces, genet_save_dirs, args.nproc, 42, False, False) # model_path = "/datamirror/zxxia/PCC-RL/results_1006/train_with_difficulty_metrics/metric2_cubic/seed_{}/model_step_{}.ckpt".format(seed, step) # genet_save_dirs = [os.path.join( # save_dir, 'cl2', "seed_{}".format(seed), # "step_{}".format(step)) for save_dir in save_dirs] # test_on_traces(model_path, dataset.traces, genet_save_dirs, args.nproc, 42, False, False) step = 720000 # model_path = "/tank/zxxia/PCC-RL/results_0905/udr/udr_small/seed_{}/model_step_{}.ckpt".format(seed, step) # genet_save_dirs = [os.path.join( # save_dir, 'udr1', "seed_{}".format(seed), # "step_{}".format(step)) for save_dir in save_dirs] # test_on_traces(model_path, dataset.traces, genet_save_dirs, args.nproc, 42, False, False) # model_path = "/tank/zxxia/PCC-RL/results_0928/udr1/seed_{}/model_step_{}.ckpt".format(seed, step) # genet_save_dirs = [os.path.join( # save_dir, 'udr1_1023', "seed_{}".format(seed), # "step_{}".format(step)) for save_dir in save_dirs] # test_on_traces(model_path, dataset.traces, genet_save_dirs, args.nproc, 42, False, False) model_path = "/tank/zxxia/PCC-RL/results_0928/udr2/seed_{}/model_step_{}.ckpt".format( seed, step) genet_save_dirs = [ os.path.join(save_dir, 'udr2_1023', "seed_{}".format(seed), "step_{}".format(step)) for save_dir in save_dirs ] test_on_traces(model_path, dataset.traces, genet_save_dirs, args.nproc, 42, False, False)
'trace_{:05d}'.format(i)) for i in range(len(traces)) ] before_save_dirs = [ os.path.join(save_dir, 'before') for save_dir in save_dirs ] after_save_dirs = [ os.path.join(save_dir, 'after_best_pkt_level_reward') for save_dir in save_dirs ] bbr_old_save_dirs = [ os.path.join(save_dir, 'bbr_old') for save_dir in save_dirs ] if os.path.exists(os.path.join(after_save_dirs[0], 'aurora_summary.csv')): continue test_on_traces(MODEL_PATH_BEFORE, traces, before_save_dirs, 16, seed, False, True) test_on_traces(model_path_after, traces, after_save_dirs, 16, seed, False, True) after_save_dirs = [ os.path.join(save_dir, 'after') for save_dir in save_dirs ] model_path_after = os.path.join(ROOT, 'seed_{}'.format(seed), 'config_{:02d}'.format(config_id), 'model_step_{}.ckpt'.format(64800)) test_on_traces(model_path_after, traces, after_save_dirs, 16, seed, False, True) baseline = BBR_old(False) baseline.test_on_traces(traces, bbr_old_save_dirs, plot_flag=True,