def run_experiments(config):
result_folder = config.experiment.result_path
auc_logger = MLogger.CAUCLogger(order=config.experiment.dataset_set)
auc_logger.write(result_folder+"auc.res", append=False)
medianauc_logger = MLogger.CAUCLogger(order=config.experiment.dataset_set)
medianauc_logger.write(result_folder+"auc_median.res", append=False)
with open(result_folder+"metric.log", "w") as log_file:
for policy_name in config.experiment.policy_set:
for dataset_name in config.experiment.dataset_set:
env = MConfig.CEnvironment(config=config)
env.experiment.random = np.random.RandomState(config.experiment.rand_seed)
env.experiment.policy_name = policy_name
env.experiment.dataset_name = dataset_name
env.experiment.dataset = data.load_data(dataset_name, env.experiment.random, config.data.max_sz)
env.experiment.data_dim = env.experiment.dataset.all_data[0].x.shape[0]
#todo: remove following fields
env.dataset = env.experiment.dataset
config.example_dim = env.experiment.data_dim
config.q0 = config.data.q0
experiments.run_experiments(config, env,\
auclogger=auc_logger, medianauc_logger=medianauc_logger, log_file=log_file)
auc_logger.write(result_folder+"auc.res", append=True)
auc_logger.clear()
medianauc_logger.write(result_folder+"auc_median.res", append=True)
medianauc_logger.clear()
def main():
args = parse_args()
if args.all:
print("Running all experiments")
run_experiments(all=True)
else:
if args.e.lower() not in ['ada', 'dt', 'knn', 'nn', 'svm', 'all']:
raise ValueError(
"Invalid experiment, please select from following: ada, dt, knn, nn, svm"
)
else:
run_experiments(experiment=args.e)
def run_experiments(dataset_set, paras_set, algos, max_sz, result_folder, \
batch_sz, batch_rate, label_budget=None, \
debug=False):
if not os.path.isdir(result_folder):
os.makedirs(result_folder)
utils.backup_code(result_folder)
auc_logger = MLogger.CAUCLogger(order=dataset_set)
auc_logger.write(result_folder + "auc.res", append=False)
with open(result_folder + "metric.log", "w") as log_file:
for paras_name in paras_set:
for dataset_name in dataset_set:
experiments.run_experiments(data.load_data(dataset_name, max_sz), paras_name, \
dataset_name, algos, result_folder+dataset_name+"-"+paras_name, \
batch_sz, batch_rate,\
auclogger=auc_logger, log_file=log_file, label_budget=label_budget, debug=debug)
auc_logger.write(result_folder + "auc.res", append=True)
auc_logger.clear()
# -*- coding: utf-8 -*-
import numpy as np
from lvqmln import LVQMLN
from experiments import run_experiments
# Import data
train = np.load('./data/flc_scaled_train.npy')
test = np.load('./data/flc_scaled_test.npy')
x_train = train[:, :-1].astype('float32')
y_train = train[:, -1].astype('float32')
x_test = test[:, :-1].astype('float32')
y_test = test[:, -1].astype('float32')
# Build model
clf = LVQMLN(x_train.shape[1], 3, len(set(y_train)), list(set(y_train)))
run_experiments(
fname='flc_3_10_{}.pth',
clf=clf,
x_train=x_train,
y_train=y_train,
x_test=x_test,
y_test=y_test,
p_list=[1.0, 0.9, 0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2, 0.1, 0],
n=1000,
)
base_fn = base_dir + run
log_fn = base_fn + ".log"
# Run experiments
if not only_evaluation:
experiments.run_experiments(
N,
D,
Ms,
max_variance,
variance_decay_rate,
mu_beta,
sigma_beta,
n_random_iters,
base_fn,
log_fn,
n_reps,
plot_2D=False,
include_bias=False,
include_sampling=include_sampling,
include_advi=include_advi,
include_fast_sampling=include_fast_sampling,
sketch=sketch,
ftol=ftol,
full_sm=full_sm,
fast_sm=fast_sm)
### Evaluation
approximation_methods = ["Laplace", "Diagonal_Laplace", "Prior"]
if include_sampling:
# set prior over beta
print("\n\nBeginning experiments for D=%d" % D)
mu_beta, sigma_beta = np.zeros(D, dtype=np.float64), np.ones(D,
dtype=np.float64)
run = "D=%05d-MaxVar=%0.02f-VarDecayRate=%0.02f-SVDnIters=%02d-N=%05d" % (
D, max_variance, variance_decay_rate, n_random_iters, N)
base_fn = base_dir + run
log_fn = base_fn + ".log"
# Run experiments and plot
import ipdb
ipdb.set_trace()
experiments.run_experiments(N,
D,
Ms,
max_variance,
variance_decay_rate,
mu_beta,
sigma_beta,
n_random_iters,
base_fn,
log_fn,
n_reps=1,
plot_2D=True,
include_bias=include_bias,
include_sampling=include_sampling,
include_fast_sampling=include_sampling,
regenerate_data=False)
def main(_):
## import main model ##
main_model_name = '{}_{}'.format(args.target, args.arch)
if main_model_name in MODEL:
MainModel = MODEL[main_model_name]
else:
raise Exception("Unsupported target-arch pair!")
## create save dir ##
if args.action == 'baseline':
save_dir = os.path.join('save_baseline',
'{}_{}'.format(args.arch, '_'.join(args.task)))
if not os.path.exists(save_dir + '/record'):
os.makedirs(save_dir + '/record', exist_ok=True)
elif args.action == 'experiments_rl':
save_dir = os.path.join('save_experiments_rl',
'{}_{}'.format(args.arch, '_'.join(args.task)))
elif args.action == 'experiments_nonrl':
save_dir = os.path.join('save_experiments_nonrl',
'{}_{}'.format(args.arch, '_'.join(args.task)))
elif args.target == 'lm':
save_dir = os.path.join('save', '{}_{}'.format(args.target, args.arch))
else:
save_dir = os.path.join(
'save', '{}_{}_{}'.format(args.target, args.arch,
'_'.join(args.task)))
args.save_dir = save_dir
## data set ##
if 'all' in args.task:
args.task = list(range(1, 21))
else:
args.task = [int(i) for i in args.task]
train, test, words, args.story_size, args.sentence_size, args.question_size = read_babi(
args.task, args.batch_size, args.target == 'expert')
val = train.split_dataset(args.val_ratio)
print("training count: {}".format(train.count))
print("testing count: {}".format(test.count))
print("word2idx:", len(words.word2idx))
print("idx2word:", len(words.idx2word))
print("story size: {}".format(args.story_size))
print("sentence size: {}".format(args.sentence_size))
print("question size: {}".format(args.question_size))
## create params ##
params_dict = vars(args)
params_class = namedtuple('params_class', params_dict.keys())
params = params_class(**params_dict)
## check target-action-dirs ##
if params.target == 'lm':
if params.action == 'train':
if params.load_dir or params.expert_dir or params.lm_dir:
raise Exception("No dir needed while training %s model!" %
params.target)
train_process = train_lm
elif params.action == 'test':
if not params.load_dir:
raise Exception("Need a trained %s model to test!" %
params.target)
if params.expert_dir or params.lm_dir:
raise Exception(
"No other dirs needed while testing %s model!" %
params.target)
test_process = test_lm
else:
raise Exception("Unsupported action for %s!" % params.target)
elif params.target == 'expert':
if params.action == 'train':
if params.load_dir or params.expert_dir or params.lm_dir:
raise Exception("No dir needed while training %s model!" %
params.target)
train_process = train_normal
elif params.action == 'test':
if not params.load_dir:
raise Exception("Need a trained %s model to test!" %
params.target)
if params.expert_dir or params.lm_dir:
raise Exception(
"No other dirs needed while testing %s model!" %
params.target)
test_process = test_normal
else:
raise Exception("Unsupported action for %s!" % params.target)
elif params.target == 'learner':
if params.action == 'train':
if params.load_dir:
raise Exception(
"Learner in %s mode can only be trained from scratch!" %
params.action)
if (not params.expert_dir) or (not params.lm_dir):
raise Exception(
"Two auxiliary models are needed in learner %s mode! " %
params.action)
train_process = train_normal
elif params.action == 'test':
if (not params.load_dir) or (not params.expert_dir) or (
not params.lm_dir):
raise Exception("All dirs needed while in learner %s mode!" %
params.action)
test_process = test_normal
elif params.action == 'rl':
if (not params.load_dir) or (not params.expert_dir) or (
not params.lm_dir):
raise Exception("All dirs needed while in learner %s mode!" %
params.action)
else:
if params.load_dir:
raise Exception(
"Learner in %s mode can only be trained from scratch!" %
params.action)
if (not params.expert_dir) or (not params.lm_dir):
raise Exception(
"Two auxiliary models are needed in learner %s mode! " %
params.action)
## load params from load_dir ##
if not params.load_dir == '':
params_filename = os.path.join(params.load_dir, 'params.json')
load_params = load_params_dict(params_filename)
if not load_params['task'] == params.task:
raise Exception("incompatible task with load model!")
if (not load_params['target']
== params.target) or (not load_params['arch'] == params.arch):
raise Exception("incompatible main model with load model!")
params = params._replace(**load_params)
else:
if tf.gfile.Exists(params.save_dir):
tf.gfile.DeleteRecursively(params.save_dir)
os.makedirs(params.save_dir, exist_ok=True)
## load expert params from expert_dir ##
if not params.expert_dir == '':
params_filename = os.path.join(params.expert_dir, 'params.json')
load_params = load_params_dict(params_filename)
if not load_params['task'] == params.task:
raise Exception("incompatible task with expert model!")
if not load_params['target'] == 'expert':
raise Exception("dir contains no expert model!")
expert_params = params._replace(action='test',
load_dir=params.expert_dir,
**load_params)
else:
expert_params = None
## load lm params from lm_dir ##
if not params.lm_dir == '':
params_filename = os.path.join(params.lm_dir, 'params.json')
load_params = load_params_dict(params_filename)
if not load_params['target'] == 'lm':
raise Exception("dir contains no language model!")
lm_params = params._replace(action='test',
load_dir=params.lm_dir,
lm_num_steps=1,
lm_batch_size=params.batch_size,
**load_params)
else:
lm_params = None
## run action ##
if args.action == 'train':
train_process(MainModel, params, expert_params, lm_params, words,
train, val)
elif args.action == 'test':
test_process(MainModel, params, expert_params, lm_params, words, test)
elif args.action == 'rl':
main_model = MainModel(words, params, expert_params, lm_params)
main_model.rl_train(train, val)
main_model.save_params()
elif args.action == 'baseline':
run_baseline(MainModel, params, expert_params, lm_params, words, train,
val)
elif args.action == 'experiments_rl':
run_experiments(MainModel,
params,
expert_params,
lm_params,
words,
train,
val,
RL=True)
elif args.action == 'experiments_nonrl':
run_experiments(MainModel,
params,
expert_params,
lm_params,
words,
train,
val,
RL=False)
INPUT_SHAPE = np.product(x_train.shape[1:])
PROJECTION_DIM = x_train.shape[-1]
x_train = x_train.reshape(-1, INPUT_SHAPE).astype('float32')
y_train = y_train.astype('float32')
x_test = x_test.reshape(-1, INPUT_SHAPE).astype('float32')
y_test = y_test.astype('float32')
scaler = StandardScaler()
scaler.fit(x_train)
x_train = scaler.transform(x_train)
x_test = scaler.transform(x_test)
# Build model
clf = LVQMLN(x_train.shape[1],
PROJECTION_DIM,
len(set(y_train)),
list(set(y_train)),
sigmoid_beta=1,
swish_beta=0.1)
run_experiments(
fname=f'mnist_{PROJECTION_DIM}' + '_10_{}.pth',
clf=clf,
x_train=x_train,
y_train=y_train,
x_test=x_test,
y_test=y_test,
p_list=[1.0, 0.9, 0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2, 0.1, 0],
n=1000,
)
def main():
run_experiments()
BUFFER_SIZE = 10000
EPOCHS = [25, 50, 100]
_GPU_NUMBER = None
# Electric demand forecasting
run_experiments(
train_file_name='data/hourly_20140102_20191101_train.csv',
test_file_name='data/hourly_20140102_20191101_test.csv',
result_file_name='files/results/experimental_results_electricity.csv',
forecast_horizon=FORECAST_HORIZON,
past_history_ls=PAST_HISTORY,
batch_size_ls=BATCH_SIZE,
epochs_ls=EPOCHS,
tcn_params=TCN_PARAMS,
lstm_params=LSTM_PARAMS,
gpu_number=_GPU_NUMBER,
metrics_ls=METRICS,
buffer_size=1000,
seed=1,
show_plots=False,
webhook=WEBHOOK,
validation_size=0.)
# Electric vehicle power consumption forecasting
run_experiments(train_file_name='data/CECOVEL_train.csv',
test_file_name='data/CECOVEL_test.csv',
result_file_name='files/results/experimental_results_EV.csv',
forecast_horizon=FORECAST_HORIZON,
past_history_ls=PAST_HISTORY,
