def main():
parser = argparse.ArgumentParser()
parser.add_argument('--nt', type=int, default=10, help='number of buckets')
parser.add_argument('--train_cnt',
default=16,
type=int,
help='number of train samples')
parser.add_argument('--nvs',
type=int,
nargs='+',
default=[2**i for i in range(3, 21)])
parser.add_argument('--prefix',
type=str,
default='',
help='Additional prefix of out file name')
args = parser.parse_args()
print(args)
methods = [
# memory error (for 32GB RAM) after 2048 (inclusive)
(baselines.TimeVaryingGraphLasso(name='T-GLASSO'), {
'lamb': 0.1,
'beta': 1.0,
'indexOfPenalty': 1,
'max_iter': 100,
'lengthOfSlice': args.train_cnt,
}),
(baselines.TCorex(tcorex=TCorex, name='T-Corex (pytorch, cpu)'), {
'max_iter': 100,
'anneal': True,
'l1': 0.1,
'gamma': 0.8,
'reg_type': 'W',
'init': True,
'device': 'cpu'
}),
(baselines.TCorex(tcorex=TCorex, name='T-Corex (pytorch, cuda)'), {
'max_iter': 100,
'anneal': True,
'l1': 0.1,
'gamma': 0.8,
'reg_type': 'W',
'init': True,
'device': 'cuda'
}),
(baselines.LTGL(name='LTGL'), {
'alpha': 3.0,
'tau': 30.0,
'beta': 30.0,
'psi': 'l1',
'eta': 3.0,
'phi': 'l1',
'rho': 1.0 / np.sqrt(args.train_cnt),
'max_iter': 500,
'verbose': False
}),
(baselines.QUIC(name='QUIC'), {
'lamb': 0.1,
'tol': 1e-6,
'msg': 1,
'max_iter': 100
}),
(baselines.BigQUIC(name='BigQUIC'), {
'lamb': 3,
'tol': 1e-3,
'verbose': 1,
'max_iter': 100
})
]
times = {}
for method, params in methods[:-2]:
times[method.name] = []
out_file = 'outputs/scalability/{}nt{}.train_cnt{}.json'.format(
args.prefix, args.nt, args.train_cnt)
make_sure_path_exists(out_file)
print("Output file path = {}".format(out_file))
stop_methods = set()
for nv in args.nvs:
bs = min(nv, 16)
n_hidden = nv // bs
# generate data
data, _, = load_nglf_sudden_change(nv=nv,
m=n_hidden,
nt=args.nt,
ns=args.train_cnt,
shuffle=False,
from_matrix=False)
for method, params in methods[:-2]:
# start timing
print("{}\nTiming method: {}, nv: {}".format(
'-' * 80, method.name, nv))
if method.name.find('T-Corex') != -1:
params['nv'] = nv
params['n_hidden'] = min(n_hidden, 64)
if method in stop_methods:
print("\tskipped")
continue
try:
ct = method.timeit(data, params)
times[method.name].append((nv, ct))
print("\ttook {:.2f} seconds".format(ct))
# do not time this method again if ct is more than 6 hours
if ct > 3600 * 6:
stop_methods.add(method.name)
except Exception as e:
print("\tfailed with message: '{}'".format(str(e)))
# save results
with open(out_file, 'w') as f:
json.dump(times, f)
print("Results are saved in {}".format(out_file))
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--nt',
type=int,
default=10,
help='number of train time periods')
parser.add_argument('--train_cnt',
default=12,
type=int,
help='number of train samples')
parser.add_argument('--val_cnt',
default=3,
type=int,
help='number of validation samples')
parser.add_argument('--start_period', type=int, default=1)
parser.add_argument(
'--noise_var',
type=float,
default=1e-4,
help='variance of Gaussian noise that will be added to time series')
parser.add_argument('--prefix',
type=str,
default='',
help='optional prefix of experiment name')
parser.add_argument('--output_dir', type=str, default='outputs/portfolio/')
parser.add_argument('--left', type=int, default=0)
parser.add_argument('--right', type=int, default=-2)
parser.add_argument('--seed', type=int, default=42)
parser.add_argument('--device', type=str, default='cpu')
args = parser.parse_args()
print(args)
''' Load data '''
train_data, val_data, _, _, _, df_index = load_sp500(
train_cnt=args.train_cnt,
val_cnt=args.val_cnt,
test_cnt=0,
commodities=False,
log_return=False,
start_date='2000-01-01',
end_date='2016-01-01',
noise_var=args.noise_var,
standardize=False,
return_index=True,
seed=args.seed)
# Take last nt+1 time steps. Use first nt of them for training / validation.
# The last time period is used for testing.
start_period = args.start_period
test_period = args.start_period + args.nt
nv = train_data[0].shape[-1]
test_data = np.concatenate(
[train_data[test_period], val_data[test_period]], axis=0)
train_data = train_data[start_period:test_period]
val_data = val_data[start_period:test_period]
start_date = df_index[start_period *
(args.train_cnt + args.val_cnt)].date()
end_date = df_index[(test_period + 1) * (args.train_cnt + args.val_cnt) -
1].date()
print("Number of train/val time periods: {}".format(len(train_data)))
print("Start date: {}".format(start_date))
print("End date: {}".format(end_date))
print("Test data shape: {}".format(test_data.shape))
''' Define baselines and the grid of parameters '''
# gamma --- eps means samples only from the current bucket, while 1 means all samples
tcorex_gamma_range = [0.8, 0.9]
n_hidden_grid = [16, 32] # [16, 32, 64]
methods = [
(baselines.LedoitWolf(name='Ledoit-Wolf'), {}),
(baselines.LinearCorex(name='Linear CorEx'), {
'n_hidden': n_hidden_grid,
'max_iter': 500,
'anneal': True
}),
(
baselines.TCorex(tcorex=TCorex, name='T-Corex'),
{
'nv': nv,
'n_hidden': n_hidden_grid,
'max_iter': 500,
'anneal': True,
'reg_params': {
'l1': [0.3, 1.0, 3.0, 10.0,
30.0], # [0.0, 0.03, 0.1, 0.3, 1.0, 3.0, 10.0],
},
'gamma': tcorex_gamma_range,
'reg_type': 'W',
'init': True,
'device': args.device,
'verbose': 1
}),
(
baselines.TimeVaryingGraphLasso(name='T-GLASSO'),
{
'lamb': [0.03, 0.1, 0.3, 1.0, 3.0],
'beta': [0.03, 0.1, 0.3, 1.0, 3.0, 10.0],
'indexOfPenalty':
[1], # NOTE: L2 is very slow and gives bad results
'max_iter': 500, # NOTE: checked 1500 no improvement
'lengthOfSlice': args.train_cnt
}),
(
baselines.LTGL(name='LTGL'),
{
'alpha': [0.3, 1.0, 3.0, 10.0],
'tau': [30.0, 100.0, 300.0, 1e3],
'beta': [10.0, 30.0, 100.0],
'psi': 'l1',
'eta': [0.3, 1.0, 3.0],
'phi': 'l1', # NOTE: tried Laplacian and l2 too no improvement
'rho': 1.0 / np.sqrt(args.train_cnt),
'max_iter': 500, # NOTE: tried 1000 no improvement
'verbose': False
})
]
exp_name = 'nt{}.train_cnt{}.val_cnt{}.start_date{}.end_date{}.noise_var{}'.format(
args.nt, args.train_cnt, args.val_cnt, start_date, end_date,
args.noise_var)
exp_name = args.prefix + exp_name
best_results_path = "{}.results.json".format(exp_name)
best_results_path = os.path.join(args.output_dir, 'best',
best_results_path)
make_sure_path_exists(best_results_path)
best_results = {}
if os.path.exists(best_results_path):
with open(best_results_path, 'r') as f:
best_results = json.load(f)
all_results_path = "{}.results.json".format(exp_name)
all_results_path = os.path.join(args.output_dir, 'all', all_results_path)
make_sure_path_exists(all_results_path)
all_results = {}
if os.path.exists(all_results_path):
with open(all_results_path, 'r') as f:
all_results = json.load(f)
mu_path = "{}.pkl".format(exp_name)
mu_path = os.path.join(args.output_dir, 'mu', mu_path)
make_sure_path_exists(mu_path)
mus = {}
if os.path.exists(mu_path):
with open(mu_path, 'rb') as f:
mus = pickle.load(f)
sigma_path = "{}.pkl".format(exp_name)
sigma_path = os.path.join(args.output_dir, 'sigma', sigma_path)
make_sure_path_exists(sigma_path)
sigmas = {}
if os.path.exists(sigma_path):
with open(sigma_path, 'rb') as f:
sigmas = pickle.load(f)
qp_solutions_path = "{}.pkl".format(exp_name)
qp_solutions_path = os.path.join(args.output_dir, 'qp_solution',
qp_solutions_path)
make_sure_path_exists(qp_solutions_path)
qp_solutions = {}
if os.path.exists(qp_solutions_path):
with open(qp_solutions_path, 'rb') as f:
qp_solutions = pickle.load(f)
test_data_path = "{}.txt".format(exp_name)
test_data_path = os.path.join(args.output_dir, 'test_data', test_data_path)
make_sure_path_exists(test_data_path)
np.savetxt(test_data_path, test_data)
for (method, params) in methods[args.left:args.right]:
name = method.name
best_score, best_params, best_covs, best_method, all_cur_results =\
method.select(train_data, val_data, params)
mu = np.mean(train_data[-1], axis=0)
if name == 'T-Corex':
mu = best_method.theta[-1][0]
mu = mu.astype(np.float64)
sigma = best_covs[-1].astype(np.float64)
# portfolio optimization using mu and sigma
# solvers.qp needs:
# minimize 1/2 x^T P x + q^T x
# subject to Gx <= h
# Ax = b
#
# our program:
# minimize x^T Sigma x
# subject to mu^T x >= r
# 1^T x = 1
# x >= 0
qp_solutions[name] = {}
for r in np.linspace(0.0, np.percentile(mu, 99), 100):
P = 2.0 * matrix(sigma)
q = matrix(0.0, (nv, 1))
G = matrix(
np.concatenate([-np.eye(nv), -mu.reshape((1, -1))], axis=0))
h = matrix(
np.concatenate([np.zeros((nv, 1)), -r * np.ones((1, 1))],
axis=0))
A = matrix(np.ones((1, nv)))
b = matrix(1.0)
qp_solutions[name][r] = solvers.qp(P, q, G, h, A, b)
# save qp_solutions
with open(qp_solutions_path, 'wb') as f:
pickle.dump(qp_solutions, f)
# save mu and sigma
mus[name] = mu
sigmas[name] = sigma
with open(mu_path, 'wb') as f:
pickle.dump(mus, f)
with open(sigma_path, 'wb') as f:
pickle.dump(sigmas, f)
# save model selection data
best_results[name] = {}
best_results[name]['best_params'] = best_params
best_results[name]['best_val_score'] = best_score
all_results[name] = all_cur_results
with open(best_results_path, 'w') as f:
json.dump(best_results, f)
with open(all_results_path, 'w') as f:
json.dump(all_results, f)
print("Best results are saved in {}".format(best_results_path))
print("All results are saved in {}".format(all_results_path))
print("Means are saved in {}".format(mu_path))
print("Sigmas are saved in {}".format(sigma_path))
print("Solutions are saved in {}".format(qp_solutions_path))
print("Test data is saved in {}".format(test_data_path))
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--nt', type=int, help='number of buckets')
parser.add_argument('--m', type=int, help='number of latent factors')
parser.add_argument('--bs', type=int, help='block size')
parser.add_argument('--train_cnt', default=16, type=int, help='number of train samples')
parser.add_argument('--val_cnt', default=16, type=int, help='number of validation samples')
parser.add_argument('--test_cnt', default=1000, type=int, help='number of test samples')
parser.add_argument('--snr', type=float, default=5.0, help='signal to noise ratio')
parser.add_argument('--min_std', type=float, default=0.25, help='minimum x-std')
parser.add_argument('--max_std', type=float, default=4.0, help='maximum x-std')
parser.add_argument('--prefix', type=str, default='', help='optional prefix of experiment name')
parser.add_argument('--data_type', dest='data_type', action='store', default='modular',
choices=['modular', 'general', 'sparse'], help='which dataset to load/create')
parser.add_argument('--output_dir', type=str, default='outputs/quantitative/')
parser.add_argument('--shuffle', dest='shuffle', action='store_true',
help='whether to shuffle parent-child relation')
parser.set_defaults(shuffle=False)
parser.add_argument('--n_segments', type=int, default=2)
parser.add_argument('--left', type=int, default=0)
parser.add_argument('--right', type=int, default=-2)
parser.add_argument('--seed', type=int, default=42)
args = parser.parse_args()
args.nv = args.m * args.bs
print(args)
''' Load data '''
if args.data_type == 'modular':
(data, ground_truth_covs) = load_modular_sudden_change(nv=args.nv, m=args.m, nt=args.nt,
ns=args.train_cnt + args.val_cnt + args.test_cnt,
snr=args.snr, min_std=args.min_std,
max_std=args.max_std, shuffle=args.shuffle,
n_segments=args.n_segments, seed=args.seed)
else:
raise ValueError("data_type={} is not implemented yet.".format(args.data_type))
train_data = [x[:args.train_cnt] for x in data]
val_data = [x[args.train_cnt:args.train_cnt + args.val_cnt] for x in data]
test_data = [x[-args.test_cnt:] for x in data]
''' Define baselines and the grid of parameters '''
# gamma --- eps means samples only from the current bucket, while 1-eps means all samples
tcorex_gamma_range = None
if 0 < args.train_cnt <= 16:
tcorex_gamma_range = [0.3, 0.4, 0.5, 0.6, 0.7]
if 16 < args.train_cnt <= 32:
tcorex_gamma_range = [0.1, 0.3, 0.4, 0.5, 0.6]
if 32 < args.train_cnt <= 64:
tcorex_gamma_range = [1e-9, 0.1, 0.3, 0.4, 0.5]
elif 64 < args.train_cnt:
tcorex_gamma_range = [1e-9, 0.1, 0.3]
methods = [
(baselines.GroundTruth(name='Ground Truth',
covs=ground_truth_covs,
test_data=test_data), {}),
(baselines.Diagonal(name='Diagonal'), {}),
(baselines.LedoitWolf(name='Ledoit-Wolf'), {}),
(baselines.OAS(name='Oracle approximating shrinkage'), {}),
(baselines.PCA(name='PCA'), {
'n_components': [args.m],
}),
(baselines.SparsePCA(name='SparsePCA'), {
'n_components': [args.m],
'alpha': [0.1, 0.3, 1.0, 3.0, 10.0, 30.0],
'ridge_alpha': [0.01],
'tol': 1e-6,
'max_iter': 500,
}),
(baselines.FactorAnalysis(name='Factor Analysis'), {
'n_components': [args.m],
}),
(baselines.GraphLasso(name='Graphical LASSO (sklearn)'), {
'alpha': [0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3],
'mode': 'lars',
'max_iter': 500,
}),
(baselines.LinearCorex(name='Linear CorEx'), {
'n_hidden': [args.m],
'max_iter': 500,
'anneal': True,
}),
(baselines.TimeVaryingGraphLasso(name='T-GLASSO'), {
'lamb': [0.03, 0.1, 0.3, 1.0, 3.0],
'beta': [0.03, 0.1, 0.3, 1.0, 3.0, 10.0],
'indexOfPenalty': [1], # NOTE: L2 is very slow, gives bad results; Laplacian gives worse results
'max_iter': 500, # NOTE: checked 1500 no improvement
'lengthOfSlice': args.train_cnt,
}),
(baselines.TimeVaryingGraphLasso(name='T-GLASSO (no reg)'), {
'lamb': [0.01, 0.03, 0.1, 0.3, 1.0, 3.0],
'beta': [0.0],
'indexOfPenalty': [1],
'max_iter': 500,
'lengthOfSlice': args.train_cnt,
}),
(baselines.TCorex(tcorex=TCorex, name='T-Corex (simple)'), {
'nv': args.nv,
'n_hidden': args.m,
'max_iter': 500,
'anneal': True,
'reg_params': {
'l1': [0.0, 0.03, 0.1, 0.3, 1.0, 3.0, 10.0], # NOTE: L1 works slightly better
'l2': [0, 0.1, 0.3, 1.0, 3.0, 10.0, 30.0, 100.0, 300.0]
},
'reg_type': 'W',
'gamma': 1e-9,
'init': False,
}),
(baselines.TCorex(tcorex=TCorex, name='T-Corex'), {
'nv': args.nv,
'n_hidden': [args.m],
'max_iter': 500,
'anneal': True,
'reg_params': {
'l1': [0.0, 0.03, 0.1, 0.3, 1.0, 3.0, 10.0],
'l2': [0, 0.1, 0.3, 1.0, 3.0, 10.0, 30.0, 100.0, 300.0]
},
'gamma': tcorex_gamma_range,
'reg_type': 'W',
'init': True,
}),
(baselines.TCorex(tcorex=TCorex, name='T-Corex (weighted objective)'), {
'nv': args.nv,
'n_hidden': [args.m],
'max_iter': 500,
'anneal': True,
'reg_params': {
'l1': [0.0, 0.001, 0.003, 0.01, 0.03],
'l2': [0, 0.1, 0.3, 1.0, 3.0, 10.0, 30.0, 100.0, 300.0]
},
'gamma': tcorex_gamma_range,
'reg_type': 'W',
'init': True,
'weighted_obj': True
}),
(baselines.TCorex(tcorex=TCorex, name='T-Corex (no reg)'), {
'nv': args.nv,
'n_hidden': [args.m],
'max_iter': 500,
'anneal': True,
'l1': 0.0,
'l2': 0.0,
'gamma': tcorex_gamma_range,
'reg_type': 'W',
'init': True,
}),
(baselines.TCorex(tcorex=TCorex, name='T-Corex (no init)'), {
'nv': args.nv,
'n_hidden': [args.m],
'max_iter': 500,
'anneal': True,
'reg_params': {
'l1': [0.0, 0.03, 0.1, 0.3, 1.0, 3.0, 10.0],
'l2': [0, 0.1, 0.3, 1.0, 3.0, 10.0, 30.0, 100.0, 300.0]
},
'gamma': tcorex_gamma_range,
'reg_type': 'W',
'init': False,
}),
(baselines.TCorex(tcorex=TCorexLearnable, name='T-Corex (learnable)'), {
'nv': args.nv,
'n_hidden': [args.m],
'max_iter': 500,
'anneal': True,
'reg_params': {
'l1': [0.0, 0.001, 0.003, 0.01, 0.03],
'l2': [0, 0.1, 0.3, 1.0, 3.0, 10.0, 30.0, 100.0, 300.0]
},
'reg_type': 'W',
'init': True,
'entropy_lamb': [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6],
'weighted_obj': True
}),
(baselines.LVGLASSO(name='LVGLASSO'), {
'alpha': [0.03, 0.1, 0.3, 1.0, 3.0, 10.0],
'tau': [1.0, 3.0, 10.0, 30.0, 100.0, 300.0],
'rho': 1.0 / np.sqrt(args.train_cnt), # NOTE works good, also rho doesn't change much
'max_iter': 500, # NOTE: tried 1000 no improvement
'verbose': False,
}),
(baselines.LTGL(name='LTGL'), {
'alpha': [0.3, 1.0, 3.0, 10.0],
'tau': [10.0, 30.0, 100.0, 300.0, 1e3],
'beta': [1.0, 3.0, 10.0, 30.0, 100.0],
'psi': 'l1', # tried L2, works worse
'eta': [3.0, 10.0, 30.0],
'phi': 'l1', # tried L2, works worse
'rho': 1.0 / np.sqrt(args.train_cnt),
'max_iter': 500, # NOTE: tried 1000 no improvement
'verbose': False
}),
(baselines.QUIC(name='QUIC'), {
'lamb': [0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3],
'tol': 1e-6,
'msg': 1, # NOTE: 0 - no verbosity; 1 - just two lines; 2 - max verbosity
'max_iter': 100, # NOTE: tried 500, no improvement,
}),
(baselines.BigQUIC(name='BigQUIC'), {
'lamb': [0.3, 1, 3, 10.0, 30.0],
'tol': 1e-3,
'verbose': 0, # NOTE: 0 - no verbosity; 1 - just two lines; 2 - max verbosity
'max_iter': 100, # NOTE: tried 500, no improvement
})
]
exp_name = 'sudden_first_setup.{}.nt{}.m{}.bs{}.train_cnt{}.val_cnt{}.test_cnt{}'\
'.snr{:.2f}.min_std{:.2f}.max_std{:.2f}.n_segments{}'.format(
args.data_type, args.nt, args.m, args.bs, args.train_cnt, args.val_cnt, args.test_cnt,
args.snr, args.min_std, args.max_std, args.n_segments)
exp_name = args.prefix + exp_name
if args.shuffle:
exp_name += '.shuffle'
best_results_path = "{}.results.json".format(exp_name)
best_results_path = os.path.join(args.output_dir, 'best', best_results_path)
make_sure_path_exists(best_results_path)
all_results_path = "{}.results.json".format(exp_name)
all_results_path = os.path.join(args.output_dir, 'all', all_results_path)
make_sure_path_exists(all_results_path)
best_results = {}
all_results = {}
# read previously stored values
if os.path.exists(best_results_path):
with open(best_results_path, 'r') as f:
best_results = json.load(f)
if os.path.exists(all_results_path):
with open(all_results_path, 'r') as f:
all_results = json.load(f)
for (method, params) in methods[args.left:args.right]:
name = method.name
best_score, best_params, _, _, all_cur_results = method.select(train_data, val_data, params)
best_results[name] = {}
best_results[name]['test_score'] = method.evaluate(test_data, best_params)
best_results[name]['best_params'] = best_params
best_results[name]['best_val_score'] = best_score
all_results[name] = all_cur_results
with open(best_results_path, 'w') as f:
json.dump(best_results, f)
with open(all_results_path, 'w') as f:
json.dump(all_results, f)
print("Best results are saved in {}".format(best_results_path))
print("All results are saved in {}".format(all_results_path))
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--nt', type=int, help='number of buckets')
parser.add_argument('--train_cnt',
default=16,
type=int,
help='number of train samples')
parser.add_argument('--val_cnt',
default=4,
type=int,
help='number of validation samples')
parser.add_argument('--test_cnt',
default=4,
type=int,
help='number of test samples')
parser.add_argument('--commodities',
dest='commodities',
action='store_true',
help='whether to include commodity prices too')
parser.add_argument('--log_return',
dest='log_return',
action='store_true',
help='whether to take log returns or normal returns')
parser.add_argument('--standard_return',
dest='log_return',
action='store_false')
parser.add_argument('--start_date', type=str, default='2000-01-01')
parser.add_argument('--end_date', type=str, default='2016-01-01')
parser.add_argument(
'--noise_var',
type=float,
default=1e-4,
help='variance of Gaussian noise that will be added to time series')
parser.add_argument('--prefix',
type=str,
default='',
help='optional prefix of experiment name')
parser.add_argument('--output_dir',
type=str,
default='outputs/quantitative/')
parser.add_argument('--left', type=int, default=0)
parser.add_argument('--right', type=int, default=-2)
parser.add_argument('--seed', type=int, default=42)
parser.set_defaults(commodities=False)
parser.set_defaults(log_return=True)
args = parser.parse_args()
print(args)
''' Load data '''
train_data, val_data, test_data, _, _ = load_sp500(
train_cnt=args.train_cnt,
val_cnt=args.val_cnt,
test_cnt=args.test_cnt,
commodities=args.commodities,
log_return=args.log_return,
start_date=args.start_date,
end_date=args.end_date,
noise_var=args.noise_var,
seed=args.seed)
# take last nt time steps
nv = train_data[0].shape[-1]
train_data = train_data[-args.nt:]
val_data = val_data[-args.nt:]
test_data = test_data[-args.nt:]
''' Define baselines and the grid of parameters '''
# gamma --- eps means samples only from the current bucket, while 1-eps means all samples
tcorex_gamma_range = None
if 0 < args.train_cnt <= 16:
tcorex_gamma_range = [0.5, 0.6, 0.7, 0.8, 0.9]
if 16 < args.train_cnt <= 32:
tcorex_gamma_range = [0.4, 0.5, 0.6, 0.7, 0.8]
if 32 < args.train_cnt <= 64:
tcorex_gamma_range = [0.1, 0.2, 0.3, 0.4, 0.5]
elif 64 < args.train_cnt:
tcorex_gamma_range = [1e-9, 0.1, 0.2]
n_hidden_grid = [16, 32, 64, 128]
methods = [
(baselines.Diagonal(name='Diagonal'), {}),
(baselines.LedoitWolf(name='Ledoit-Wolf'), {}),
(baselines.OAS(name='Oracle approximating shrinkage'), {}),
(baselines.PCA(name='PCA'), {
'n_components': n_hidden_grid
}),
(
baselines.SparsePCA(name='SparsePCA'),
{
'n_components': n_hidden_grid,
'alpha': [0.1, 0.3, 1.0, 3.0, 10.0, 30.0],
'ridge_alpha': [0.01],
'tol': 1e-3,
'max_iter':
100, # NOTE: tried 500 no improvement, just slows down a lot !
}),
(baselines.FactorAnalysis(name='Factor Analysis'), {
'n_components': n_hidden_grid
}),
(baselines.GraphLasso(name='Graphical LASSO (sklearn)'), {
'alpha': [0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3],
'mode': 'lars',
'max_iter': 500,
}),
(baselines.LinearCorex(name='Linear CorEx'), {
'n_hidden': n_hidden_grid,
'max_iter': 500,
'anneal': True
}),
(
baselines.TimeVaryingGraphLasso(name='T-GLASSO'),
{
'lamb': [0.03, 0.1, 0.3, 1.0, 3.0],
'beta': [0.03, 0.1, 0.3, 1.0, 3.0, 10.0],
'indexOfPenalty':
[1], # NOTE: L2 is very slow and gives bad results
'max_iter': 500, # NOTE: checked 1500 no improvement
'lengthOfSlice': args.train_cnt
}),
(baselines.TimeVaryingGraphLasso(name='T-GLASSO (no reg)'), {
'lamb': [0.01, 0.03, 0.1, 0.3, 1.0, 3.0],
'beta': [0.0],
'indexOfPenalty': [1],
'max_iter': 500,
'lengthOfSlice': args.train_cnt
}),
(
baselines.TCorex(tcorex=TCorex, name='T-Corex (simple)'),
{
'nv': nv,
'n_hidden': n_hidden_grid,
'max_iter': 500,
'anneal': True,
'reg_params': {
'l1': [0.0, 0.03, 0.1, 0.3, 1.0, 3.0,
10.0], # NOTE: L1 works slightly better
# 'l2': [0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1.0, 3.0, 10.0],
},
'reg_type': 'W',
'gamma': 1e-9,
'init': False,
}),
(
baselines.TCorex(tcorex=TCorex, name='T-Corex'),
{
'nv': nv,
'n_hidden': n_hidden_grid,
'max_iter': 500,
'anneal': True,
'reg_params': {
'l1': [0.0, 0.03, 0.1, 0.3, 1.0, 3.0, 10.0],
# 'l2': [0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1.0, 3.0, 10.0],
},
'gamma': tcorex_gamma_range,
'reg_type': 'W',
'init': True,
}),
(
baselines.TCorex(tcorex=TCorex,
name='T-Corex (weighted objective)'),
{
'nv': nv,
'n_hidden': n_hidden_grid,
'max_iter': 500,
'anneal': True,
'reg_params': {
'l1': [0.0, 0.001, 0.003, 0.01, 0.03],
# 'l2': [0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1.0, 3.0, 10.0],
},
'gamma': tcorex_gamma_range,
'reg_type': 'W',
'init': True,
'weighted_obj': True
}),
(baselines.TCorex(tcorex=TCorex, name='T-Corex (no reg)'), {
'nv': nv,
'n_hidden': n_hidden_grid,
'max_iter': 500,
'anneal': True,
'l1': 0.0,
'l2': 0.0,
'gamma': tcorex_gamma_range,
'reg_type': 'W',
'init': True,
}),
(
baselines.TCorex(tcorex=TCorex, name='T-Corex (no init)'),
{
'nv': nv,
'n_hidden': n_hidden_grid,
'max_iter': 500,
'anneal': True,
'reg_params': {
'l1': [0.0, 0.03, 0.1, 0.3, 1.0, 3.0, 10.0],
# 'l2': [0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1.0, 3.0, 10.0],
},
'gamma': tcorex_gamma_range,
'reg_type': 'W',
'init': False,
}),
(
baselines.TCorex(tcorex=TCorexLearnable,
name='T-Corex (learnable)'),
{
'nv': nv,
'n_hidden': n_hidden_grid,
'max_iter': 500,
'anneal': True,
'reg_params': {
'l1': [0.0, 0.001, 0.003, 0.01, 0.03],
# 'l2': [0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1.0, 3.0, 10.0],
},
'reg_type': 'W',
'init': True,
'entropy_lamb': [0.0, 0.1, 0.3, 0.5, 0.8],
'weighted_obj': True
}),
(
baselines.LVGLASSO(name='LVGLASSO'),
{
'alpha': [0.03, 0.1, 0.3, 1.0, 3.0, 10.0],
'tau': [1.0, 3.0, 10.0, 30.0, 100.0, 300.0],
'rho': 1.0 /
np.sqrt(args.train_cnt
), # NOTE works good, also rho doesn't change much
'max_iter': 500, # NOTE: tried 1000 no improvement
'verbose': False
}),
(
baselines.LTGL(name='LTGL'),
{
'alpha': [0.3, 1.0, 3.0, 10.0],
'tau': [30.0, 100.0, 300.0, 1e3],
'beta': [10.0, 30.0, 100.0],
'psi': 'l1',
'eta': [0.3, 1.0, 3.0],
'phi': 'l1', # NOTE: tried Laplacian and l2 too no improvement
'rho': 1.0 / np.sqrt(args.train_cnt),
'max_iter': 500, # NOTE: tried 1000 no improvement
'verbose': False
})
]
exp_name = 'stocks_first_setup.nt{}.train_cnt{}.val_cnt{}.test_cnt{}.start_date{}.end_date{}.noise_var{}'.format(
args.nt, args.train_cnt, args.val_cnt, args.test_cnt, args.start_date,
args.end_date, args.noise_var)
exp_name = args.prefix + exp_name
if args.commodities:
exp_name += '.commodities'
if args.log_return:
exp_name += '.log_return'
best_results_path = "{}.results.json".format(exp_name)
best_results_path = os.path.join(args.output_dir, 'best',
best_results_path)
make_sure_path_exists(best_results_path)
all_results_path = "{}.results.json".format(exp_name)
all_results_path = os.path.join(args.output_dir, 'all', all_results_path)
make_sure_path_exists(all_results_path)
best_results = {}
all_results = {}
# read previously stored values
if os.path.exists(best_results_path):
with open(best_results_path, 'r') as f:
best_results = json.load(f)
if os.path.exists(all_results_path):
with open(all_results_path, 'r') as f:
all_results = json.load(f)
for (method, params) in methods[args.left:args.right]:
name = method.name
best_score, best_params, _, _, all_cur_results = method.select(
train_data, val_data, params)
best_results[name] = {}
best_results[name]['test_score'] = method.evaluate(
test_data, best_params)
best_results[name]['best_params'] = best_params
best_results[name]['best_val_score'] = best_score
all_results[name] = all_cur_results
with open(best_results_path, 'w') as f:
json.dump(best_results, f)
with open(all_results_path, 'w') as f:
json.dump(all_results, f)
print("Best results are saved in {}".format(best_results_path))
print("All results are saved in {}".format(all_results_path))
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--n_hidden',
'-m',
default=64,
type=int,
help='number of hidden variables')
parser.add_argument('--n_observed',
'-p',
default=128,
type=int,
help='number of observed variables')
parser.add_argument('--snr',
'-s',
default=0.1,
type=float,
help='signal-to-noise ratio')
parser.add_argument('--n_samples',
'-n',
default=300,
type=int,
help='number of samples')
parser.add_argument('--num_extra_parents',
default=0.1,
type=float,
help='average number of extra parents for each x_i')
parser.add_argument(
'--num_correlated_zs',
default=0,
type=int,
help='number of zs each z_i is correlated with (besides z_i itself)')
parser.add_argument('--random_scale',
dest='random_scale',
action='store_true',
help='if true x_i will have random scales')
parser.add_argument('--method',
'-M',
type=str,
choices=['linearcorex', 'pycorex'],
default='pycorex',
help='which implementation of corex to use')
parser.add_argument('--device',
'-d',
type=str,
default='cpu',
help='which device to use for pytorch corex')
parser.add_argument('--output_dir',
'-o',
type=str,
default='outputs/blessing/')
parser.set_defaults(random_scale=False)
args = parser.parse_args()
print(args)
p = args.n_observed
m = args.n_hidden
snr = args.snr
n = args.n_samples
assert p % m == 0
# generate some data
data, _ = data_tools.generate_approximately_modular(
nv=p,
m=m,
ns=n,
snr=snr,
num_extra_parents=args.num_extra_parents,
num_correlated_zs=args.num_correlated_zs,
random_scale=args.random_scale)
# select the method
if args.method == 'linearcorex':
method = linearcorex.Corex(n_hidden=m, verbose=1)
else:
method = PyCorex(nv=p,
n_hidden=m,
verbose=2,
max_iter=10000,
tol=1e-6,
device=args.device,
optimizer_class=torch.optim.Adam,
optimizer_params={'lr': 0.01})
# train and compute the clustering score
true_clusters = np.arange(m).repeat(p // m, axis=0)
method.fit(data)
if args.method == 'linearcorex':
pred_clusters = method.mis.argmax(axis=0)
else:
pred_clusters = method.clusters()
score = adjusted_rand_score(labels_true=true_clusters,
labels_pred=pred_clusters)
print(pred_clusters, score)
# save the results
run_id = str(np.random.choice(10**18))
save_dict = {
'p': p,
'm': m,
'snr': snr,
'n': n,
'num_extra_parents': args.num_extra_parents,
'num_correlated_zs': args.num_correlated_zs,
'random_scale': args.random_scale,
'method': args.method,
'score': score,
'run_id': run_id
}
output_file = os.path.join(args.output_dir, run_id + '.pkl')
make_sure_path_exists(output_file)
with open(output_file, 'wb') as fout:
pickle.dump(save_dict, fout)