def main(): parser = argparse.ArgumentParser() parser.add_argument('--nt', type=int, default=10, help='number of buckets') parser.add_argument('--train_cnt', default=16, type=int, help='number of train samples') parser.add_argument('--nvs', type=int, nargs='+', default=[2**i for i in range(3, 21)]) parser.add_argument('--prefix', type=str, default='', help='Additional prefix of out file name') args = parser.parse_args() print(args) methods = [ # memory error (for 32GB RAM) after 2048 (inclusive) (baselines.TimeVaryingGraphLasso(name='T-GLASSO'), { 'lamb': 0.1, 'beta': 1.0, 'indexOfPenalty': 1, 'max_iter': 100, 'lengthOfSlice': args.train_cnt, }), (baselines.TCorex(tcorex=TCorex, name='T-Corex (pytorch, cpu)'), { 'max_iter': 100, 'anneal': True, 'l1': 0.1, 'gamma': 0.8, 'reg_type': 'W', 'init': True, 'device': 'cpu' }), (baselines.TCorex(tcorex=TCorex, name='T-Corex (pytorch, cuda)'), { 'max_iter': 100, 'anneal': True, 'l1': 0.1, 'gamma': 0.8, 'reg_type': 'W', 'init': True, 'device': 'cuda' }), (baselines.LTGL(name='LTGL'), { 'alpha': 3.0, 'tau': 30.0, 'beta': 30.0, 'psi': 'l1', 'eta': 3.0, 'phi': 'l1', 'rho': 1.0 / np.sqrt(args.train_cnt), 'max_iter': 500, 'verbose': False }), (baselines.QUIC(name='QUIC'), { 'lamb': 0.1, 'tol': 1e-6, 'msg': 1, 'max_iter': 100 }), (baselines.BigQUIC(name='BigQUIC'), { 'lamb': 3, 'tol': 1e-3, 'verbose': 1, 'max_iter': 100 }) ] times = {} for method, params in methods[:-2]: times[method.name] = [] out_file = 'outputs/scalability/{}nt{}.train_cnt{}.json'.format( args.prefix, args.nt, args.train_cnt) make_sure_path_exists(out_file) print("Output file path = {}".format(out_file)) stop_methods = set() for nv in args.nvs: bs = min(nv, 16) n_hidden = nv // bs # generate data data, _, = load_nglf_sudden_change(nv=nv, m=n_hidden, nt=args.nt, ns=args.train_cnt, shuffle=False, from_matrix=False) for method, params in methods[:-2]: # start timing print("{}\nTiming method: {}, nv: {}".format( '-' * 80, method.name, nv)) if method.name.find('T-Corex') != -1: params['nv'] = nv params['n_hidden'] = min(n_hidden, 64) if method in stop_methods: print("\tskipped") continue try: ct = method.timeit(data, params) times[method.name].append((nv, ct)) print("\ttook {:.2f} seconds".format(ct)) # do not time this method again if ct is more than 6 hours if ct > 3600 * 6: stop_methods.add(method.name) except Exception as e: print("\tfailed with message: '{}'".format(str(e))) # save results with open(out_file, 'w') as f: json.dump(times, f) print("Results are saved in {}".format(out_file))
def main(): parser = argparse.ArgumentParser() parser.add_argument('--nt', type=int, default=10, help='number of train time periods') parser.add_argument('--train_cnt', default=12, type=int, help='number of train samples') parser.add_argument('--val_cnt', default=3, type=int, help='number of validation samples') parser.add_argument('--start_period', type=int, default=1) parser.add_argument( '--noise_var', type=float, default=1e-4, help='variance of Gaussian noise that will be added to time series') parser.add_argument('--prefix', type=str, default='', help='optional prefix of experiment name') parser.add_argument('--output_dir', type=str, default='outputs/portfolio/') parser.add_argument('--left', type=int, default=0) parser.add_argument('--right', type=int, default=-2) parser.add_argument('--seed', type=int, default=42) parser.add_argument('--device', type=str, default='cpu') args = parser.parse_args() print(args) ''' Load data ''' train_data, val_data, _, _, _, df_index = load_sp500( train_cnt=args.train_cnt, val_cnt=args.val_cnt, test_cnt=0, commodities=False, log_return=False, start_date='2000-01-01', end_date='2016-01-01', noise_var=args.noise_var, standardize=False, return_index=True, seed=args.seed) # Take last nt+1 time steps. Use first nt of them for training / validation. # The last time period is used for testing. start_period = args.start_period test_period = args.start_period + args.nt nv = train_data[0].shape[-1] test_data = np.concatenate( [train_data[test_period], val_data[test_period]], axis=0) train_data = train_data[start_period:test_period] val_data = val_data[start_period:test_period] start_date = df_index[start_period * (args.train_cnt + args.val_cnt)].date() end_date = df_index[(test_period + 1) * (args.train_cnt + args.val_cnt) - 1].date() print("Number of train/val time periods: {}".format(len(train_data))) print("Start date: {}".format(start_date)) print("End date: {}".format(end_date)) print("Test data shape: {}".format(test_data.shape)) ''' Define baselines and the grid of parameters ''' # gamma --- eps means samples only from the current bucket, while 1 means all samples tcorex_gamma_range = [0.8, 0.9] n_hidden_grid = [16, 32] # [16, 32, 64] methods = [ (baselines.LedoitWolf(name='Ledoit-Wolf'), {}), (baselines.LinearCorex(name='Linear CorEx'), { 'n_hidden': n_hidden_grid, 'max_iter': 500, 'anneal': True }), ( baselines.TCorex(tcorex=TCorex, name='T-Corex'), { 'nv': nv, 'n_hidden': n_hidden_grid, 'max_iter': 500, 'anneal': True, 'reg_params': { 'l1': [0.3, 1.0, 3.0, 10.0, 30.0], # [0.0, 0.03, 0.1, 0.3, 1.0, 3.0, 10.0], }, 'gamma': tcorex_gamma_range, 'reg_type': 'W', 'init': True, 'device': args.device, 'verbose': 1 }), ( baselines.TimeVaryingGraphLasso(name='T-GLASSO'), { 'lamb': [0.03, 0.1, 0.3, 1.0, 3.0], 'beta': [0.03, 0.1, 0.3, 1.0, 3.0, 10.0], 'indexOfPenalty': [1], # NOTE: L2 is very slow and gives bad results 'max_iter': 500, # NOTE: checked 1500 no improvement 'lengthOfSlice': args.train_cnt }), ( baselines.LTGL(name='LTGL'), { 'alpha': [0.3, 1.0, 3.0, 10.0], 'tau': [30.0, 100.0, 300.0, 1e3], 'beta': [10.0, 30.0, 100.0], 'psi': 'l1', 'eta': [0.3, 1.0, 3.0], 'phi': 'l1', # NOTE: tried Laplacian and l2 too no improvement 'rho': 1.0 / np.sqrt(args.train_cnt), 'max_iter': 500, # NOTE: tried 1000 no improvement 'verbose': False }) ] exp_name = 'nt{}.train_cnt{}.val_cnt{}.start_date{}.end_date{}.noise_var{}'.format( args.nt, args.train_cnt, args.val_cnt, start_date, end_date, args.noise_var) exp_name = args.prefix + exp_name best_results_path = "{}.results.json".format(exp_name) best_results_path = os.path.join(args.output_dir, 'best', best_results_path) make_sure_path_exists(best_results_path) best_results = {} if os.path.exists(best_results_path): with open(best_results_path, 'r') as f: best_results = json.load(f) all_results_path = "{}.results.json".format(exp_name) all_results_path = os.path.join(args.output_dir, 'all', all_results_path) make_sure_path_exists(all_results_path) all_results = {} if os.path.exists(all_results_path): with open(all_results_path, 'r') as f: all_results = json.load(f) mu_path = "{}.pkl".format(exp_name) mu_path = os.path.join(args.output_dir, 'mu', mu_path) make_sure_path_exists(mu_path) mus = {} if os.path.exists(mu_path): with open(mu_path, 'rb') as f: mus = pickle.load(f) sigma_path = "{}.pkl".format(exp_name) sigma_path = os.path.join(args.output_dir, 'sigma', sigma_path) make_sure_path_exists(sigma_path) sigmas = {} if os.path.exists(sigma_path): with open(sigma_path, 'rb') as f: sigmas = pickle.load(f) qp_solutions_path = "{}.pkl".format(exp_name) qp_solutions_path = os.path.join(args.output_dir, 'qp_solution', qp_solutions_path) make_sure_path_exists(qp_solutions_path) qp_solutions = {} if os.path.exists(qp_solutions_path): with open(qp_solutions_path, 'rb') as f: qp_solutions = pickle.load(f) test_data_path = "{}.txt".format(exp_name) test_data_path = os.path.join(args.output_dir, 'test_data', test_data_path) make_sure_path_exists(test_data_path) np.savetxt(test_data_path, test_data) for (method, params) in methods[args.left:args.right]: name = method.name best_score, best_params, best_covs, best_method, all_cur_results =\ method.select(train_data, val_data, params) mu = np.mean(train_data[-1], axis=0) if name == 'T-Corex': mu = best_method.theta[-1][0] mu = mu.astype(np.float64) sigma = best_covs[-1].astype(np.float64) # portfolio optimization using mu and sigma # solvers.qp needs: # minimize 1/2 x^T P x + q^T x # subject to Gx <= h # Ax = b # # our program: # minimize x^T Sigma x # subject to mu^T x >= r # 1^T x = 1 # x >= 0 qp_solutions[name] = {} for r in np.linspace(0.0, np.percentile(mu, 99), 100): P = 2.0 * matrix(sigma) q = matrix(0.0, (nv, 1)) G = matrix( np.concatenate([-np.eye(nv), -mu.reshape((1, -1))], axis=0)) h = matrix( np.concatenate([np.zeros((nv, 1)), -r * np.ones((1, 1))], axis=0)) A = matrix(np.ones((1, nv))) b = matrix(1.0) qp_solutions[name][r] = solvers.qp(P, q, G, h, A, b) # save qp_solutions with open(qp_solutions_path, 'wb') as f: pickle.dump(qp_solutions, f) # save mu and sigma mus[name] = mu sigmas[name] = sigma with open(mu_path, 'wb') as f: pickle.dump(mus, f) with open(sigma_path, 'wb') as f: pickle.dump(sigmas, f) # save model selection data best_results[name] = {} best_results[name]['best_params'] = best_params best_results[name]['best_val_score'] = best_score all_results[name] = all_cur_results with open(best_results_path, 'w') as f: json.dump(best_results, f) with open(all_results_path, 'w') as f: json.dump(all_results, f) print("Best results are saved in {}".format(best_results_path)) print("All results are saved in {}".format(all_results_path)) print("Means are saved in {}".format(mu_path)) print("Sigmas are saved in {}".format(sigma_path)) print("Solutions are saved in {}".format(qp_solutions_path)) print("Test data is saved in {}".format(test_data_path))
def main(): parser = argparse.ArgumentParser() parser.add_argument('--nt', type=int, help='number of buckets') parser.add_argument('--m', type=int, help='number of latent factors') parser.add_argument('--bs', type=int, help='block size') parser.add_argument('--train_cnt', default=16, type=int, help='number of train samples') parser.add_argument('--val_cnt', default=16, type=int, help='number of validation samples') parser.add_argument('--test_cnt', default=1000, type=int, help='number of test samples') parser.add_argument('--snr', type=float, default=5.0, help='signal to noise ratio') parser.add_argument('--min_std', type=float, default=0.25, help='minimum x-std') parser.add_argument('--max_std', type=float, default=4.0, help='maximum x-std') parser.add_argument('--prefix', type=str, default='', help='optional prefix of experiment name') parser.add_argument('--data_type', dest='data_type', action='store', default='modular', choices=['modular', 'general', 'sparse'], help='which dataset to load/create') parser.add_argument('--output_dir', type=str, default='outputs/quantitative/') parser.add_argument('--shuffle', dest='shuffle', action='store_true', help='whether to shuffle parent-child relation') parser.set_defaults(shuffle=False) parser.add_argument('--n_segments', type=int, default=2) parser.add_argument('--left', type=int, default=0) parser.add_argument('--right', type=int, default=-2) parser.add_argument('--seed', type=int, default=42) args = parser.parse_args() args.nv = args.m * args.bs print(args) ''' Load data ''' if args.data_type == 'modular': (data, ground_truth_covs) = load_modular_sudden_change(nv=args.nv, m=args.m, nt=args.nt, ns=args.train_cnt + args.val_cnt + args.test_cnt, snr=args.snr, min_std=args.min_std, max_std=args.max_std, shuffle=args.shuffle, n_segments=args.n_segments, seed=args.seed) else: raise ValueError("data_type={} is not implemented yet.".format(args.data_type)) train_data = [x[:args.train_cnt] for x in data] val_data = [x[args.train_cnt:args.train_cnt + args.val_cnt] for x in data] test_data = [x[-args.test_cnt:] for x in data] ''' Define baselines and the grid of parameters ''' # gamma --- eps means samples only from the current bucket, while 1-eps means all samples tcorex_gamma_range = None if 0 < args.train_cnt <= 16: tcorex_gamma_range = [0.3, 0.4, 0.5, 0.6, 0.7] if 16 < args.train_cnt <= 32: tcorex_gamma_range = [0.1, 0.3, 0.4, 0.5, 0.6] if 32 < args.train_cnt <= 64: tcorex_gamma_range = [1e-9, 0.1, 0.3, 0.4, 0.5] elif 64 < args.train_cnt: tcorex_gamma_range = [1e-9, 0.1, 0.3] methods = [ (baselines.GroundTruth(name='Ground Truth', covs=ground_truth_covs, test_data=test_data), {}), (baselines.Diagonal(name='Diagonal'), {}), (baselines.LedoitWolf(name='Ledoit-Wolf'), {}), (baselines.OAS(name='Oracle approximating shrinkage'), {}), (baselines.PCA(name='PCA'), { 'n_components': [args.m], }), (baselines.SparsePCA(name='SparsePCA'), { 'n_components': [args.m], 'alpha': [0.1, 0.3, 1.0, 3.0, 10.0, 30.0], 'ridge_alpha': [0.01], 'tol': 1e-6, 'max_iter': 500, }), (baselines.FactorAnalysis(name='Factor Analysis'), { 'n_components': [args.m], }), (baselines.GraphLasso(name='Graphical LASSO (sklearn)'), { 'alpha': [0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3], 'mode': 'lars', 'max_iter': 500, }), (baselines.LinearCorex(name='Linear CorEx'), { 'n_hidden': [args.m], 'max_iter': 500, 'anneal': True, }), (baselines.TimeVaryingGraphLasso(name='T-GLASSO'), { 'lamb': [0.03, 0.1, 0.3, 1.0, 3.0], 'beta': [0.03, 0.1, 0.3, 1.0, 3.0, 10.0], 'indexOfPenalty': [1], # NOTE: L2 is very slow, gives bad results; Laplacian gives worse results 'max_iter': 500, # NOTE: checked 1500 no improvement 'lengthOfSlice': args.train_cnt, }), (baselines.TimeVaryingGraphLasso(name='T-GLASSO (no reg)'), { 'lamb': [0.01, 0.03, 0.1, 0.3, 1.0, 3.0], 'beta': [0.0], 'indexOfPenalty': [1], 'max_iter': 500, 'lengthOfSlice': args.train_cnt, }), (baselines.TCorex(tcorex=TCorex, name='T-Corex (simple)'), { 'nv': args.nv, 'n_hidden': args.m, 'max_iter': 500, 'anneal': True, 'reg_params': { 'l1': [0.0, 0.03, 0.1, 0.3, 1.0, 3.0, 10.0], # NOTE: L1 works slightly better 'l2': [0, 0.1, 0.3, 1.0, 3.0, 10.0, 30.0, 100.0, 300.0] }, 'reg_type': 'W', 'gamma': 1e-9, 'init': False, }), (baselines.TCorex(tcorex=TCorex, name='T-Corex'), { 'nv': args.nv, 'n_hidden': [args.m], 'max_iter': 500, 'anneal': True, 'reg_params': { 'l1': [0.0, 0.03, 0.1, 0.3, 1.0, 3.0, 10.0], 'l2': [0, 0.1, 0.3, 1.0, 3.0, 10.0, 30.0, 100.0, 300.0] }, 'gamma': tcorex_gamma_range, 'reg_type': 'W', 'init': True, }), (baselines.TCorex(tcorex=TCorex, name='T-Corex (weighted objective)'), { 'nv': args.nv, 'n_hidden': [args.m], 'max_iter': 500, 'anneal': True, 'reg_params': { 'l1': [0.0, 0.001, 0.003, 0.01, 0.03], 'l2': [0, 0.1, 0.3, 1.0, 3.0, 10.0, 30.0, 100.0, 300.0] }, 'gamma': tcorex_gamma_range, 'reg_type': 'W', 'init': True, 'weighted_obj': True }), (baselines.TCorex(tcorex=TCorex, name='T-Corex (no reg)'), { 'nv': args.nv, 'n_hidden': [args.m], 'max_iter': 500, 'anneal': True, 'l1': 0.0, 'l2': 0.0, 'gamma': tcorex_gamma_range, 'reg_type': 'W', 'init': True, }), (baselines.TCorex(tcorex=TCorex, name='T-Corex (no init)'), { 'nv': args.nv, 'n_hidden': [args.m], 'max_iter': 500, 'anneal': True, 'reg_params': { 'l1': [0.0, 0.03, 0.1, 0.3, 1.0, 3.0, 10.0], 'l2': [0, 0.1, 0.3, 1.0, 3.0, 10.0, 30.0, 100.0, 300.0] }, 'gamma': tcorex_gamma_range, 'reg_type': 'W', 'init': False, }), (baselines.TCorex(tcorex=TCorexLearnable, name='T-Corex (learnable)'), { 'nv': args.nv, 'n_hidden': [args.m], 'max_iter': 500, 'anneal': True, 'reg_params': { 'l1': [0.0, 0.001, 0.003, 0.01, 0.03], 'l2': [0, 0.1, 0.3, 1.0, 3.0, 10.0, 30.0, 100.0, 300.0] }, 'reg_type': 'W', 'init': True, 'entropy_lamb': [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6], 'weighted_obj': True }), (baselines.LVGLASSO(name='LVGLASSO'), { 'alpha': [0.03, 0.1, 0.3, 1.0, 3.0, 10.0], 'tau': [1.0, 3.0, 10.0, 30.0, 100.0, 300.0], 'rho': 1.0 / np.sqrt(args.train_cnt), # NOTE works good, also rho doesn't change much 'max_iter': 500, # NOTE: tried 1000 no improvement 'verbose': False, }), (baselines.LTGL(name='LTGL'), { 'alpha': [0.3, 1.0, 3.0, 10.0], 'tau': [10.0, 30.0, 100.0, 300.0, 1e3], 'beta': [1.0, 3.0, 10.0, 30.0, 100.0], 'psi': 'l1', # tried L2, works worse 'eta': [3.0, 10.0, 30.0], 'phi': 'l1', # tried L2, works worse 'rho': 1.0 / np.sqrt(args.train_cnt), 'max_iter': 500, # NOTE: tried 1000 no improvement 'verbose': False }), (baselines.QUIC(name='QUIC'), { 'lamb': [0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3], 'tol': 1e-6, 'msg': 1, # NOTE: 0 - no verbosity; 1 - just two lines; 2 - max verbosity 'max_iter': 100, # NOTE: tried 500, no improvement, }), (baselines.BigQUIC(name='BigQUIC'), { 'lamb': [0.3, 1, 3, 10.0, 30.0], 'tol': 1e-3, 'verbose': 0, # NOTE: 0 - no verbosity; 1 - just two lines; 2 - max verbosity 'max_iter': 100, # NOTE: tried 500, no improvement }) ] exp_name = 'sudden_first_setup.{}.nt{}.m{}.bs{}.train_cnt{}.val_cnt{}.test_cnt{}'\ '.snr{:.2f}.min_std{:.2f}.max_std{:.2f}.n_segments{}'.format( args.data_type, args.nt, args.m, args.bs, args.train_cnt, args.val_cnt, args.test_cnt, args.snr, args.min_std, args.max_std, args.n_segments) exp_name = args.prefix + exp_name if args.shuffle: exp_name += '.shuffle' best_results_path = "{}.results.json".format(exp_name) best_results_path = os.path.join(args.output_dir, 'best', best_results_path) make_sure_path_exists(best_results_path) all_results_path = "{}.results.json".format(exp_name) all_results_path = os.path.join(args.output_dir, 'all', all_results_path) make_sure_path_exists(all_results_path) best_results = {} all_results = {} # read previously stored values if os.path.exists(best_results_path): with open(best_results_path, 'r') as f: best_results = json.load(f) if os.path.exists(all_results_path): with open(all_results_path, 'r') as f: all_results = json.load(f) for (method, params) in methods[args.left:args.right]: name = method.name best_score, best_params, _, _, all_cur_results = method.select(train_data, val_data, params) best_results[name] = {} best_results[name]['test_score'] = method.evaluate(test_data, best_params) best_results[name]['best_params'] = best_params best_results[name]['best_val_score'] = best_score all_results[name] = all_cur_results with open(best_results_path, 'w') as f: json.dump(best_results, f) with open(all_results_path, 'w') as f: json.dump(all_results, f) print("Best results are saved in {}".format(best_results_path)) print("All results are saved in {}".format(all_results_path))
def main(): parser = argparse.ArgumentParser() parser.add_argument('--nt', type=int, help='number of buckets') parser.add_argument('--train_cnt', default=16, type=int, help='number of train samples') parser.add_argument('--val_cnt', default=4, type=int, help='number of validation samples') parser.add_argument('--test_cnt', default=4, type=int, help='number of test samples') parser.add_argument('--commodities', dest='commodities', action='store_true', help='whether to include commodity prices too') parser.add_argument('--log_return', dest='log_return', action='store_true', help='whether to take log returns or normal returns') parser.add_argument('--standard_return', dest='log_return', action='store_false') parser.add_argument('--start_date', type=str, default='2000-01-01') parser.add_argument('--end_date', type=str, default='2016-01-01') parser.add_argument( '--noise_var', type=float, default=1e-4, help='variance of Gaussian noise that will be added to time series') parser.add_argument('--prefix', type=str, default='', help='optional prefix of experiment name') parser.add_argument('--output_dir', type=str, default='outputs/quantitative/') parser.add_argument('--left', type=int, default=0) parser.add_argument('--right', type=int, default=-2) parser.add_argument('--seed', type=int, default=42) parser.set_defaults(commodities=False) parser.set_defaults(log_return=True) args = parser.parse_args() print(args) ''' Load data ''' train_data, val_data, test_data, _, _ = load_sp500( train_cnt=args.train_cnt, val_cnt=args.val_cnt, test_cnt=args.test_cnt, commodities=args.commodities, log_return=args.log_return, start_date=args.start_date, end_date=args.end_date, noise_var=args.noise_var, seed=args.seed) # take last nt time steps nv = train_data[0].shape[-1] train_data = train_data[-args.nt:] val_data = val_data[-args.nt:] test_data = test_data[-args.nt:] ''' Define baselines and the grid of parameters ''' # gamma --- eps means samples only from the current bucket, while 1-eps means all samples tcorex_gamma_range = None if 0 < args.train_cnt <= 16: tcorex_gamma_range = [0.5, 0.6, 0.7, 0.8, 0.9] if 16 < args.train_cnt <= 32: tcorex_gamma_range = [0.4, 0.5, 0.6, 0.7, 0.8] if 32 < args.train_cnt <= 64: tcorex_gamma_range = [0.1, 0.2, 0.3, 0.4, 0.5] elif 64 < args.train_cnt: tcorex_gamma_range = [1e-9, 0.1, 0.2] n_hidden_grid = [16, 32, 64, 128] methods = [ (baselines.Diagonal(name='Diagonal'), {}), (baselines.LedoitWolf(name='Ledoit-Wolf'), {}), (baselines.OAS(name='Oracle approximating shrinkage'), {}), (baselines.PCA(name='PCA'), { 'n_components': n_hidden_grid }), ( baselines.SparsePCA(name='SparsePCA'), { 'n_components': n_hidden_grid, 'alpha': [0.1, 0.3, 1.0, 3.0, 10.0, 30.0], 'ridge_alpha': [0.01], 'tol': 1e-3, 'max_iter': 100, # NOTE: tried 500 no improvement, just slows down a lot ! }), (baselines.FactorAnalysis(name='Factor Analysis'), { 'n_components': n_hidden_grid }), (baselines.GraphLasso(name='Graphical LASSO (sklearn)'), { 'alpha': [0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3], 'mode': 'lars', 'max_iter': 500, }), (baselines.LinearCorex(name='Linear CorEx'), { 'n_hidden': n_hidden_grid, 'max_iter': 500, 'anneal': True }), ( baselines.TimeVaryingGraphLasso(name='T-GLASSO'), { 'lamb': [0.03, 0.1, 0.3, 1.0, 3.0], 'beta': [0.03, 0.1, 0.3, 1.0, 3.0, 10.0], 'indexOfPenalty': [1], # NOTE: L2 is very slow and gives bad results 'max_iter': 500, # NOTE: checked 1500 no improvement 'lengthOfSlice': args.train_cnt }), (baselines.TimeVaryingGraphLasso(name='T-GLASSO (no reg)'), { 'lamb': [0.01, 0.03, 0.1, 0.3, 1.0, 3.0], 'beta': [0.0], 'indexOfPenalty': [1], 'max_iter': 500, 'lengthOfSlice': args.train_cnt }), ( baselines.TCorex(tcorex=TCorex, name='T-Corex (simple)'), { 'nv': nv, 'n_hidden': n_hidden_grid, 'max_iter': 500, 'anneal': True, 'reg_params': { 'l1': [0.0, 0.03, 0.1, 0.3, 1.0, 3.0, 10.0], # NOTE: L1 works slightly better # 'l2': [0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1.0, 3.0, 10.0], }, 'reg_type': 'W', 'gamma': 1e-9, 'init': False, }), ( baselines.TCorex(tcorex=TCorex, name='T-Corex'), { 'nv': nv, 'n_hidden': n_hidden_grid, 'max_iter': 500, 'anneal': True, 'reg_params': { 'l1': [0.0, 0.03, 0.1, 0.3, 1.0, 3.0, 10.0], # 'l2': [0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1.0, 3.0, 10.0], }, 'gamma': tcorex_gamma_range, 'reg_type': 'W', 'init': True, }), ( baselines.TCorex(tcorex=TCorex, name='T-Corex (weighted objective)'), { 'nv': nv, 'n_hidden': n_hidden_grid, 'max_iter': 500, 'anneal': True, 'reg_params': { 'l1': [0.0, 0.001, 0.003, 0.01, 0.03], # 'l2': [0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1.0, 3.0, 10.0], }, 'gamma': tcorex_gamma_range, 'reg_type': 'W', 'init': True, 'weighted_obj': True }), (baselines.TCorex(tcorex=TCorex, name='T-Corex (no reg)'), { 'nv': nv, 'n_hidden': n_hidden_grid, 'max_iter': 500, 'anneal': True, 'l1': 0.0, 'l2': 0.0, 'gamma': tcorex_gamma_range, 'reg_type': 'W', 'init': True, }), ( baselines.TCorex(tcorex=TCorex, name='T-Corex (no init)'), { 'nv': nv, 'n_hidden': n_hidden_grid, 'max_iter': 500, 'anneal': True, 'reg_params': { 'l1': [0.0, 0.03, 0.1, 0.3, 1.0, 3.0, 10.0], # 'l2': [0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1.0, 3.0, 10.0], }, 'gamma': tcorex_gamma_range, 'reg_type': 'W', 'init': False, }), ( baselines.TCorex(tcorex=TCorexLearnable, name='T-Corex (learnable)'), { 'nv': nv, 'n_hidden': n_hidden_grid, 'max_iter': 500, 'anneal': True, 'reg_params': { 'l1': [0.0, 0.001, 0.003, 0.01, 0.03], # 'l2': [0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1.0, 3.0, 10.0], }, 'reg_type': 'W', 'init': True, 'entropy_lamb': [0.0, 0.1, 0.3, 0.5, 0.8], 'weighted_obj': True }), ( baselines.LVGLASSO(name='LVGLASSO'), { 'alpha': [0.03, 0.1, 0.3, 1.0, 3.0, 10.0], 'tau': [1.0, 3.0, 10.0, 30.0, 100.0, 300.0], 'rho': 1.0 / np.sqrt(args.train_cnt ), # NOTE works good, also rho doesn't change much 'max_iter': 500, # NOTE: tried 1000 no improvement 'verbose': False }), ( baselines.LTGL(name='LTGL'), { 'alpha': [0.3, 1.0, 3.0, 10.0], 'tau': [30.0, 100.0, 300.0, 1e3], 'beta': [10.0, 30.0, 100.0], 'psi': 'l1', 'eta': [0.3, 1.0, 3.0], 'phi': 'l1', # NOTE: tried Laplacian and l2 too no improvement 'rho': 1.0 / np.sqrt(args.train_cnt), 'max_iter': 500, # NOTE: tried 1000 no improvement 'verbose': False }) ] exp_name = 'stocks_first_setup.nt{}.train_cnt{}.val_cnt{}.test_cnt{}.start_date{}.end_date{}.noise_var{}'.format( args.nt, args.train_cnt, args.val_cnt, args.test_cnt, args.start_date, args.end_date, args.noise_var) exp_name = args.prefix + exp_name if args.commodities: exp_name += '.commodities' if args.log_return: exp_name += '.log_return' best_results_path = "{}.results.json".format(exp_name) best_results_path = os.path.join(args.output_dir, 'best', best_results_path) make_sure_path_exists(best_results_path) all_results_path = "{}.results.json".format(exp_name) all_results_path = os.path.join(args.output_dir, 'all', all_results_path) make_sure_path_exists(all_results_path) best_results = {} all_results = {} # read previously stored values if os.path.exists(best_results_path): with open(best_results_path, 'r') as f: best_results = json.load(f) if os.path.exists(all_results_path): with open(all_results_path, 'r') as f: all_results = json.load(f) for (method, params) in methods[args.left:args.right]: name = method.name best_score, best_params, _, _, all_cur_results = method.select( train_data, val_data, params) best_results[name] = {} best_results[name]['test_score'] = method.evaluate( test_data, best_params) best_results[name]['best_params'] = best_params best_results[name]['best_val_score'] = best_score all_results[name] = all_cur_results with open(best_results_path, 'w') as f: json.dump(best_results, f) with open(all_results_path, 'w') as f: json.dump(all_results, f) print("Best results are saved in {}".format(best_results_path)) print("All results are saved in {}".format(all_results_path))
def main(): parser = argparse.ArgumentParser() parser.add_argument('--n_hidden', '-m', default=64, type=int, help='number of hidden variables') parser.add_argument('--n_observed', '-p', default=128, type=int, help='number of observed variables') parser.add_argument('--snr', '-s', default=0.1, type=float, help='signal-to-noise ratio') parser.add_argument('--n_samples', '-n', default=300, type=int, help='number of samples') parser.add_argument('--num_extra_parents', default=0.1, type=float, help='average number of extra parents for each x_i') parser.add_argument( '--num_correlated_zs', default=0, type=int, help='number of zs each z_i is correlated with (besides z_i itself)') parser.add_argument('--random_scale', dest='random_scale', action='store_true', help='if true x_i will have random scales') parser.add_argument('--method', '-M', type=str, choices=['linearcorex', 'pycorex'], default='pycorex', help='which implementation of corex to use') parser.add_argument('--device', '-d', type=str, default='cpu', help='which device to use for pytorch corex') parser.add_argument('--output_dir', '-o', type=str, default='outputs/blessing/') parser.set_defaults(random_scale=False) args = parser.parse_args() print(args) p = args.n_observed m = args.n_hidden snr = args.snr n = args.n_samples assert p % m == 0 # generate some data data, _ = data_tools.generate_approximately_modular( nv=p, m=m, ns=n, snr=snr, num_extra_parents=args.num_extra_parents, num_correlated_zs=args.num_correlated_zs, random_scale=args.random_scale) # select the method if args.method == 'linearcorex': method = linearcorex.Corex(n_hidden=m, verbose=1) else: method = PyCorex(nv=p, n_hidden=m, verbose=2, max_iter=10000, tol=1e-6, device=args.device, optimizer_class=torch.optim.Adam, optimizer_params={'lr': 0.01}) # train and compute the clustering score true_clusters = np.arange(m).repeat(p // m, axis=0) method.fit(data) if args.method == 'linearcorex': pred_clusters = method.mis.argmax(axis=0) else: pred_clusters = method.clusters() score = adjusted_rand_score(labels_true=true_clusters, labels_pred=pred_clusters) print(pred_clusters, score) # save the results run_id = str(np.random.choice(10**18)) save_dict = { 'p': p, 'm': m, 'snr': snr, 'n': n, 'num_extra_parents': args.num_extra_parents, 'num_correlated_zs': args.num_correlated_zs, 'random_scale': args.random_scale, 'method': args.method, 'score': score, 'run_id': run_id } output_file = os.path.join(args.output_dir, run_id + '.pkl') make_sure_path_exists(output_file) with open(output_file, 'wb') as fout: pickle.dump(save_dict, fout)