def get_model(args, n_dim, r, from_ckpt=False, train=True, grocery='false'):
"""Create a model based on arguments"""
if train:
opt_params = { 'alg' : args.alg, 'lr' : args.lr, 'b1' : 0.9, 'b2' : 0.999,
'batch_size': args.batch_size, 'layers': args.layers }
else:
opt_params = default_opt
print(args.model)
# create model
if args.model == 'audiounet':
model = models.AudioUNet(from_ckpt=from_ckpt, n_dim=n_dim, r=r,
opt_params=opt_params, log_prefix=args.logname)
elif args.model == 'audiotfilm':
model = models.AudioTfilm(from_ckpt=from_ckpt, n_dim=n_dim, r=r, pool_size = args.pool_size,
strides=args.strides, opt_params=opt_params, log_prefix=args.logname)
elif args.model == 'dnn':
model = models.DNN(from_ckpt=from_ckpt, n_dim=n_dim, r=r,
opt_params=opt_params, log_prefix=args.logname)
elif args.model == 'spline':
model = models.Spline(from_ckpt=from_ckpt, n_dim=n_dim, r=r,
opt_params=opt_params, log_prefix=args.logname)
else:
raise ValueError('Invalid model')
return model
if device == 'cuda':
cudnn.benchmark = True
torch.cuda.manual_seed(args.seed)
X_train, y_train, X_test, y_test = data_loader.load_dataset('MSD')
X_train = torch.from_numpy(X_train).type(torch.FloatTensor)
X_test = torch.from_numpy(X_test).type(torch.FloatTensor)
y_train = torch.from_numpy(y_train).type(torch.FloatTensor)
y_test = torch.from_numpy(y_test).type(torch.FloatTensor)
# Model
print('==> Building model..')
dropout = args.dropout
net = models.DNN(dropout)
net = net.to(device)
tau = 0.0128363911266 # obtained from BO
lengthscale = 1e-4
reg = lengthscale**2 * (1 - dropout) / (2. * len(X_train) * tau)
optimizer = optim.Adam(net.parameters(), lr=args.lr, weight_decay=reg)
def calculate_loss(mean, std, target):
likelihood = normal.Normal(mean, std).log_prob(target)
return -likelihood.mean()
def apply_dropout(m):
def build_graph(
x_train,
y_train,
x_test,
y_test,
learning_rate,
batch_size,
output_regularization,
dropout,
decay_rate,
shallow,
l2_regularization = 0.0,
feature_dropout = 0.0,
num_basis_functions = 1000,
units_multiplier = 2,
activation = 'exu',
name_scope = 'model',
regression = False,
use_dnn = False,
trainable = True
):
"""Constructs the computation graph with specified hyperparameters."""
if regression:
ds_tensors = tf.data.Dataset.from_tensor_slices((x_train, y_train)).apply(
tf.data.experimental.shuffle_and_repeat(buffer_size=len(x_train[0])))
ds_tensors = ds_tensors.batch(batch_size)
else:
# Create a balanced dataset to handle class imbalance
ds_tensors = create_balanced_dataset(x_train, y_train, batch_size)
x_batch, (train_init_op, test_init_op) = create_iterators((x_train, x_test),
batch_size)
if use_dnn:
nn_model = models.DNN(dropout=dropout, trainable=trainable)
else:
nn_model = create_nam_model(
x_train=x_train,
dropout=dropout,
feature_dropout=feature_dropout,
activation=activation,
num_basis_functions=num_basis_functions,
shallow=shallow,
units_multiplier=units_multiplier,
trainable=trainable,
name_scope=name_scope)
global_step = tf.train.get_or_create_global_step()
learning_rate = tf.Variable(learning_rate, trainable=False)
lr_decay_op = learning_rate.assign(decay_rate * learning_rate)
optimizer = tf.train.AdamOptimizer(learning_rate)
predictions = nn_model(x_batch, training=False)
tf.logging.info(nn_model.summary())
train_vars = nn_model.trainable_variables
if regression:
loss_fn, y_pred = penalized_mse_loss, predictions
else:
# Apply sigmoid transformation for binary classification
loss_fn, y_pred = penalized_cross_entropy_loss, tf.nn.sigmoid(predictions)
loss_fn = functools.partial(
loss_fn,
output_regularization=output_regularization,
l2_regularization=l2_regularization,
use_dnn=use_dnn)
iterator = ds_tensors.make_initializable_iterator()
x1, y1 = iterator.get_next()
loss_tensor, grads = grad(nn_model, x1, y1, loss_fn, train_vars)
update_step = optimizer.apply_gradients(
zip(grads, train_vars), global_step=global_step)
avg_loss, avg_loss_update_op = tf.metrics.mean(
loss_tensor, name='avg_train_loss')
tf.summary.scalar('avg_train_loss', avg_loss)
running_mean_vars = tf.get_collection(
tf.GraphKeys.LOCAL_VARIABLES, scope='avg_train_loss')
running_vars_initializer = tf.variables_initializer(
var_list=running_mean_vars)
# Use RMSE for regression and ROC AUC for classification.
evaluation_metric = rmse_loss if regression else roc_auc_score
train_metric = functools.partial(
evaluation_metric,
y_true=y_train,
pred_tensor=y_pred,
dataset_init_op=train_init_op)
test_metric = functools.partial(
evaluation_metric,
y_true=y_test,
pred_tensor=y_pred,
dataset_init_op=test_init_op)
summary_op = tf.summary.merge_all()
graph_tensors = {
'train_op': [update_step, avg_loss_update_op],
'lr_decay_op': lr_decay_op,
'summary_op': summary_op,
'iterator_initializer': iterator.initializer,
'running_vars_initializer': running_vars_initializer,
'nn_model': nn_model,
'global_step': global_step,
}
eval_metric_scores = {'test': test_metric, 'train': train_metric}
return graph_tensors, eval_metric_scores
def train(X, Y, params, results_file_base, eval_metric, eval_dataset):
'''
Create and train a model based on the input parameters
Arguments
- X: dict, str (dataset) -> numpy.ndarray (data). shape = (num_examples, num_features)
Features. Example: X['train']
- Y: dict, str (dataset) -> numpy.ndarray (data). shape = (num_examples,)
Labels. Example: Y['train']
- params: dict, str -> various
Parameters for the model, training algorithm, and evaluation
- results_file_base: str
Base results file name. Usually includes run_id but leaves out file extension
- eval_metric: str
Metric to use for comparison between models. Example: 'auroc'
- eval_dataset: str
Dataset to use for comparison between models. Example: 'test'
Returns
- model
Prediction model
- results: dict, str (metric) -> dict (dataset) -> list of float (evaluation results)
Results of evaluation metrics of model classification predictions over iterations.
Example: results[metric][dataset]
'''
# unpack params
rand_seed = params['rand_seed']
res_freq = params['res_freq']
save_weights = params['save_weights']
loss_balance = params['loss_balance']
kernel_reg_const = params['kernel_reg_const']
batch_size = params['batch_size']
num_epochs = params['num_epochs']
node_array = np.array(params['node_array'], dtype=int)
num_features = params['num_features']
datasets = ['train', 'test']
# build model
model = models.DNN(num_features, node_array, kernel_reg_const, rand_seed)
# initialize tensorflow graph
with tf.Session() as sess:
if params['tensorboard']:
merged_summary = {}
for dataset in datasets:
loss = tf.summary.scalar('loss_' + dataset, model.loss_fn)
acc = tf.summary.scalar('acc_' + dataset, model.acc_fn)
merged_summary[dataset] = tf.summary.merge([loss, acc])
# `sess.graph` provides access to the graph used in a `tf.Session`.
tb_path = os.path.join('.', params['results_dir'],
params['assay_name'],
str(params['run_id']) + '_tb', '')
params['tensorboard_path'] = tb_path
writer = tf.summary.FileWriter(tb_path, sess.graph)
np.random.seed(rand_seed)
tf.set_random_seed(rand_seed)
sess.run(tf.global_variables_initializer())
sess.run(
tf.local_variables_initializer()
) # necessary for auc calculation: https://stackoverflow.com/questions/44422508/tensorflow-attempting-to-use-uninitialized-value-auc-auc-auc-false-positives
# calculate frequencies of positives, negatives in training set
q = 1
if loss_balance:
q = Y['train'].shape[0] / np.sum(Y['train'])
print('q: %0.3f' % q)
params['q'] = q
# keep track of loss and accuracy
best_index = 0
steps = []
loss, acc, auroc, auroc_sk, y_prob = {}, {}, {}, {}, {}
for res in [loss, acc, auroc, auroc_sk, y_prob]:
res['train'] = []
res['test'] = []
results = {
'best_index': best_index, # index of best evaluation
'steps': steps, # steps
'loss': loss, # losses at each evaluated step
'acc': acc, # accuracy at each evaluated step
'auroc': auroc, # auroc at each evaluated step
'auroc_sk':
auroc_sk, # auroc (calculated with scikit-learn) at each evaluated step
'y_prob': y_prob # y_prob of best evaluation
}
datasets = ['train', 'test']
# evaluate model based on initialized weights
step = 0
steps.append(step)
evaluate_model(X, Y, model, q, results, datasets, sess)
if params['tensorboard']:
for dataset in datasets:
s = sess.run(merged_summary[dataset],
feed_dict={
model.x: X[dataset],
model.y_labels: Y[dataset],
model.q: q
})
writer.add_summary(s, step)
params['weights_filename'] = ''
if save_weights:
weights_filename = results_file_base + '_model_weights.ckpt'
checkpoint_filename = str(params['run_id']) + '_checkpoint'
params['weights_filename'] = weights_filename
saver = tf.train.Saver()
saver.save(sess,
weights_filename,
latest_filename=checkpoint_filename)
print(
'(loss, acc, auroc, auroc_sk) - step %d,\ttrain: (%0.3f, %0.3f, %0.3f, %0.3f),\ttest: (%0.3f, %0.3f, %0.3f, %0.3f)'
% (step, results['loss']['train'][-1], results['acc']['train'][-1],
results['auroc']['train'][-1], results['auroc_sk']['train'][-1],
results['loss']['test'][-1], results['acc']['test'][-1],
results['auroc']['test'][-1], results['auroc_sk']['test'][-1]))
# training loop
num_examples = X['train'].shape[0]
num_batches_per_epoch = int(np.ceil(num_examples / batch_size))
print("Number of batches per epoch: %d " % num_batches_per_epoch)
for epoch in range(num_epochs):
# shuffle indices of training data
shuffle_indices = np.arange(num_examples)
np.random.shuffle(shuffle_indices)
for i in range(num_batches_per_epoch):
# get batch
batch_indices = shuffle_indices[i * batch_size:(i + 1) *
batch_size]
batch_x = X['train'][batch_indices]
batch_y = Y['train'][batch_indices]
# train on batch data
step += 1
sess.run(model.train_step,
feed_dict={
model.x: batch_x,
model.y_labels: batch_y,
model.q: q
})
# sess.run(model.auroc_fn, feed_dict={model.x: batch_x, model.y_labels: batch_y})
# store loss and accuracy
if step % res_freq == 0 or epoch == num_epochs - 1 and i == num_batches_per_epoch - 1:
evaluate_model(X, Y, model, q, results, datasets, sess)
steps.append(step)
print(
'(loss, acc, auroc, auroc_sk) - step %d,\ttrain: (%0.3f, %0.3f, %0.3f, %0.3f),\ttest: (%0.3f, %0.3f, %0.3f, %0.3f)'
% (step, results['loss']['train'][-1],
results['acc']['train'][-1],
results['auroc']['train'][-1],
results['auroc_sk']['train'][-1],
results['loss']['test'][-1],
results['acc']['test'][-1],
results['auroc']['test'][-1],
results['auroc_sk']['test'][-1]))
if params['tensorboard']:
for dataset in datasets:
s = sess.run(merged_summary[dataset],
feed_dict={
model.x: X[dataset],
model.y_labels: Y[dataset],
model.q: q
})
writer.add_summary(s, step)
# save variables only if eval_metric has improved
if save_weights and results[eval_metric][eval_dataset][
-1] > max(results[eval_metric][eval_dataset][:-1]):
print('saving new weights')
# save index
results['best_index'] = len(
results[eval_metric][eval_dataset]) - 1
# save y_prob
for dataset in datasets:
y_prob = sess.run(model.y_prob,
feed_dict={model.x: X[dataset]})
results['y_prob'][dataset] = y_prob
# save weights
saver.save(sess,
weights_filename,
latest_filename=checkpoint_filename)
# check for convergence
if len(results[eval_metric][eval_dataset]) > 5:
if np.std(results[eval_metric][eval_dataset]
[-5:]) < 1e-5:
print('Convergence criteria reached.')
print('Best ' + eval_dataset + ' ' + eval_metric + ': %0.3f' \
% results[eval_metric][eval_dataset][results['best_index']])
return model, results
if params['tensorboard']:
writer.close()
print('Maximum training steps reached.')
print('Best ' + eval_dataset + ' ' + eval_metric + ': %0.3f' \
% results[eval_metric][eval_dataset][results['best_index']])
return model, results
outf = 'test/'+'mcdp'
if not os.path.isdir(outf):
os.makedirs(outf)
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print("Random Seed: ", args.seed)
torch.manual_seed(args.seed)
if device == 'cuda':
torch.cuda.manual_seed(args.seed)
print('Load model')
model = models.DNN(0.05)
model = model.to(device)
model.load_state_dict(torch.load(args.pre_trained_net))
X_train, y_train, X_test, y_test = data_loader.load_dataset('MSD')
X_train = torch.from_numpy(X_train).type(torch.FloatTensor)
X_test = torch.from_numpy(X_test).type(torch.FloatTensor)
y_train = torch.from_numpy(y_train).type(torch.FloatTensor)
y_test = torch.from_numpy(y_test).type(torch.FloatTensor)
Iter_test = 100
batch_size = 512
target_scale = 10.939756
X_out = data_loader.load_dataset('boston')
X_out = torch.from_numpy(X_out).type(torch.FloatTensor)
def eval_score(results_file_base,
results_file_ext,
weights_filename,
saliency=False):
'''
Evaluate a trained model on the score dataset
Arguments
- results_file_base: str
Base results file name. Usually includes run_id but leaves out file extension
- results_file_ext: str
Results file extension, exluduing the period (e.g. 'results')
- weights_filename: str
Filename of saved Tensorflow weights
- saliency: bool, default = False
Whether to compute and plot the saliency map
'''
# read results of best run
results_file = results_file_base + '.' + results_file_ext
results_file_dtypes = results_file + '_dtypes'
# dtypes_series = pd.read_csv('dtype_series', header=None)
# dtypes_series = dtypes_series.set_index(0).squeeze()
# dtypes_dict = dtypes_series.to_dict()
df = pd.read_csv(results_file, header=0, float_precision='high',
sep='\t') # dtype=dtypes_dict
series = df.iloc[0]
params = series.to_dict()
# Get data
datasets = ['train', 'test', 'score']
metrics = ['loss', 'acc', 'auroc', 'auroc_sk']
X, Y = train.get_data(params, datasets)
# unpack params
rand_seed = params['rand_seed']
kernel_reg_const = params['kernel_reg_const']
num_features = params['num_features']
q = params['q']
node_array = params['node_array'].split(',')
for i in range(len(node_array)):
node_array[i] = int(node_array[i].strip('[] '))
node_array = np.array(node_array)
# rebuild model
model = models.DNN(num_features, node_array, kernel_reg_const, rand_seed)
# recreate results dict
loss, acc, auroc, auroc_sk, y_prob = {}, {}, {}, {}, {}
for res in [loss, acc, auroc, auroc_sk, y_prob]:
for dataset in datasets:
res[dataset] = []
results = {
'best_index': 0,
'loss': loss,
'acc': acc,
'auroc': auroc,
'auroc_sk': auroc_sk,
'y_prob': y_prob
}
# restore graph
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
saver = tf.train.Saver()
saver.restore(sess, weights_filename)
# evaluate model on all datasets, including score
train.evaluate_model(X, Y, model, q, results, datasets, sess)
for dataset in datasets:
y_prob = sess.run(model.y_prob, feed_dict={model.x: X[dataset]})
results['y_prob'][dataset] = y_prob
# plot ROC curve and save results
train.plot_ROC(X, Y, results, datasets, results_file_base)
train.save_results(X, params, results, metrics, datasets,
results_file_base)
# compute and plot saliency map
if saliency:
saliency_vecs = train.saliency(X, Y, model, sess)
train.plot_saliency(saliency_vecs, num_features, results_file_base)
