def train_and_plot(model,
X,
y,
sorted_index,
num_eps=10,
num_plots=5,
generating_func=None,
silent=False):
error_list = []
for i in range(num_eps):
errors = model.train_and_evaluate(X, y)
error_list += errors
#gauss_lr.train(X,y)
#vanilla.train(X,y)
if i % (num_eps / num_plots) == 0:
#make sure X is new data!
gauss_preds = model.predict(X)
gauss_var = model.predict_var(X)
#3lr_preds = gauss_lr.predict(X)
#lr_var = gauss_lr.predict_var(X)
#vanilla_preds = vanilla.predict(X)
if not silent:
plot_prediction(X,
gauss_preds,
sorted_index,
gauss_var,
generating_func=generating_func)
plt.plot(X, y, 'x')
plt.show()
evaluate_model(X, y, gauss_preds, var=gauss_var)
#plt.plot(np.squeeze(gauss_lr_error_list))
return error_list
def train_new_model_cli(device):
print("\n")
print("Camull-Net can be trained for two seperate tasks.")
print(
"1. NC v AD : Distinguishing between subjects from the normal cohort group and subjects with Alzheimers Disease."
)
print(
"2. sMCI v pMCI : Distinguishing between subjects with static mild cognitive impairement and progressive mild cognitive impairement."
)
print("\n")
choice = input("Please enter your choice [(1),2]:")
if choice == "": choice = 1
else: choice = int(choice)
if choice == 1: task = Task.NC_v_AD
else: task = Task.sMCI_v_pMCI
ld_helper = LoaderHelper(task)
print("\n")
epochs = input("How many epochs would you like to do? (default: 40): ")
print("\n")
uuid = ""
if epochs == "":
uuid = start(device, ld_helper, 40)
else:
try:
uuid = start(device, ld_helper, int(epochs))
except:
print("Number of epochs must be a valid number.")
print("\n")
print("A new {} model has been trained under the tag: {}".format(
str(task), uuid))
print("\n")
print(
"Would you like to evaluate it? You must do so for it to be saved to the database."
)
print("\n")
choice = input("Enter your choice [Y/n]: ")
if choice == 'Y' or 'y' or '':
choice = 1
else:
choice = 0
if (int(choice) == 1):
print("\n")
print("There are 5 folds to evaluate")
print(
"Input a fold number to evaluate or input 6 to evaluate all folds."
)
print("\n")
choice = int(input("Enter your choice [1,6]: "))
if (choice != 6):
evaluate_fold(device, uuid, ld_helper, choice, cur)
else:
evaluate_model(device, uuid, ld_helper, cur)
else:
basic_run(device) #loop back round to the start of the program.
def main_eval(args):
assert args.load_from is not None, '--load_from required in eval mode'
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
level=logging.INFO)
dataset_train, dataset_test, scaler = get_data(args)
logging.info(f'evaluation mode. Level: {args.level}')
device = torch.device(
'cuda:0') if torch.cuda.is_available() else torch.device('cpu')
n_features = dataset_train.items.shape[1]
generator, discriminator = get_models(args, n_features, device)
experiment = Experiment(args.comet_api_key,
project_name=args.comet_project_name,
workspace=args.comet_workspace)
experiment.log_parameters(vars(args))
load_model(Path(args.load_from), generator, discriminator, None, None,
device)
n_events = len(dataset_test)
steps = (args.gan_test_ratio * n_events) // args.eval_batch_size
evaluate_model(generator, experiment, dataset_test, args.eval_batch_size,
steps, args, device, scaler, 0)
def main():
args = get_arguments()
utils.print_args(args)
dataset = data_loading.get_dataset(args.dataset, args.normalize_raw,
args.normalize_reps, args.cifar_path)
print("Collected arguments and raw dataset.")
if args.network_type == 'simple':
input_shape = args.dim_red if args.dim_red is not None else dataset.get_raw_input_shape(
)
rounds.add_network_to_vector_rounds(args.rounds,
dataset,
input_shape,
args.neurons,
args.max_iter_optimization,
args.alpha_optimization,
args.n_train,
args.dim_red,
network_type='simple')
elif args.network_type == 'conv':
input_shape = dataset.get_raw_input_shape(True)
rounds.add_network_to_vector_rounds(args.rounds,
dataset,
input_shape,
args.neurons,
args.max_iter_optimization,
args.alpha_optimization,
args.n_train,
None,
network_type='conv')
else:
raise ValueError("Network type {} not supported".format(
args.network_type))
print("Finished getting Representations")
print("Getting represented dataset:")
print("Getting training examples...")
x, y = dataset.get_training_examples(args.n_train,
dim_reduction=args.dim_red,
print_progress=True)
print("Getting test examples...")
x_test, y_test = dataset.get_test_examples(args.n_test,
dim_reduction=args.dim_red,
print_progress=True)
print("Getting final linear separator")
w = svm.get_linear_separator(x,
y,
type_of_classifier='sdca',
verbose=2,
alpha=args.alpha_evaluation,
max_iter=args.max_iter_evaluation)
performance = evaluation.evaluate_model(w, x, y)
print("train performance is {}".format(performance))
performance = evaluation.evaluate_model(w, x_test, y_test)
print("test performance is {}".format(performance))
def single_run(args, dataset):
model = NeuMF(args, dataset.num_users, dataset.num_items)
model.build_model()
sess = tf.Session()
init = tf.global_variables_initializer()
sess.run(init)
t1 = time()
ahit, andcg = evaluate_model(sess, model, dataset, args.topk)
best_hr, best_ndcg, best_iter = ahit, andcg, -1
print('Init: HR = %.4f, NDCG = %.4f\t [%.1f s]' % (ahit, andcg, time() - t1))
tf.summary.scalar('loss', best_hr)
summaryMerged = tf.summary.merge_all()
writer = tf.summary.FileWriter('./graph_tensorboard', sess.graph)
for epoch in range(args.epochs):
t1 = time()
train_users, train_items, train_labels, num_inst = \
dataset.make_training_instances(args.num_neg_inst)
# print(train_labels[0:20])
loss_per_epoch, error_per_epoch = 0, 0
for ite in range((num_inst - 1) // args.batch_size + 1):
start_idx = ite * args.batch_size
end_idx = min((ite + 1) * args.batch_size, num_inst)
cur_user_indices = train_users[start_idx: end_idx]
cur_item_indices = train_items[start_idx: end_idx]
cur_label = train_labels[start_idx: end_idx]
_, loss, error = sess.run(
[model.train_step, model.loss, model.raw_error],
{model.user_indices: cur_user_indices,
model.item_indices: cur_item_indices,
model.ratings: cur_label})
loss_per_epoch += loss
error_per_epoch += error
summary = sess.run(summaryMerged)
writer.add_summary(summary, epoch)
error_per_epoch /= num_inst
t2 = time()
if epoch % args.verbose == 0:
ahit, andcg = evaluate_model(sess, model, dataset, args.topk)
print('epoch %d\t[%.1f s]: HR= %.4f\tNDCG= %.4f\tloss= %.4f\terror'
'= %.4f\t[%.1f s]' % (
epoch, t2 - t1, ahit, andcg, loss_per_epoch,
error_per_epoch, time() - t2))
if ahit > best_hr:
best_hr = ahit
best_iter = epoch
if andcg > best_ndcg:
best_ndcg = andcg
print("End. Best Epoch %d: HR = %.4f, NDCG = %.4f. " % (
best_iter, best_hr, best_ndcg))
def main():
starting_time = time.time()
train_sents, test_sents = open_data()
X_train, y_train, X_test, y_test = define_train_test(train_sents, test_sents)
crf = train_model(X_train, y_train)
evaluation.evaluate_model(crf, X_test, y_test)
evaluation.label_transitions(Counter(crf.transition_features_).most_common())
evaluation.feature_transitions(Counter(crf.state_features_).most_common())
#best_crf = evaluation.hyperparameter_optimization(X_train, y_train, list(crf.classes_))
#evaluation.evaluate_model(best_crf, X_test, y_test)
save_model(crf)
print('Execution time:', round(time.time() - starting_time, 3), 'seconds')
def val_model(model, epoch_i):
if config["kde_val"]:
# Eval KDE
ev.evaluate_model(model,
data=val_data,
config=config,
epoch_i=epoch_i,
kde=True,
make_plots=False)
# Eval true
return ev.evaluate_model(model,
data=val_data,
config=config,
epoch_i=epoch_i)
def main():
#CN v AD
ld_helper = loader_helper(task=Task.CN_v_AD)
#uuid = train_camull(ld_helper, epochs=40)
#evaluate_model(device, "c51bf83c4455416e8bc8b1ebbc8b75ca", ld_helper)
#transfer learning for pMCI v sMCI
ld_helper.change_task(Task.sMCI_v_pMCI)
model = load_model(
"camull",
"../weights/CN_v_AD/c51bf83c4455416e8bc8b1ebbc8b75ca/fold_5_weights-2020-04-29_13:17:33"
)
uuid = train_camull(ld_helper, model=model, epochs=40)
evaluate_model(device, uuid, ld_helper)
def handle_annotated_english_tweets(config, folder):
tweets_train_X, tweets_train_y, tweets_eval_X, tweets_eval_y, tweets_test_X, tweets_test_y = get_english_annotated_tweets_sets(
config, folder)
softmax_values, predicted_classes = evaluate_model(config, folder,
tweets_test_X,
tweets_test_y, None,
None, True)
def test(self, topK, epoch_num=0):
total_ahit, total_andcg = [], []
for i in xrange(self.testset.epoch): #
# user_list,pos_items, neg_items, mask_frame, mask, feats_idx= self.trainset.get_batch(i)
user_list, mask_frame, mask, feats_idx, pos_items = self.testset.get_batch(
i)
feat = self.video_features.take(feats_idx, axis=0)
[user_vector, V_matrix, time_attent,
img_attent] = self.test_model(user_list, mask_frame, mask, feat)
#V_value = np.asarray(V_matrix.eval())
score_maxtrix = np.dot(user_vector, V_matrix.T)
# index_top_K = score_maxtrix.argsort()[:,-topK:][:,::-1]
ahit, andcg = evaluation.evaluate_model(score_maxtrix,
self.testDataset2, topK,
user_list, pos_items)
total_ahit.append(ahit)
total_andcg.append(andcg)
#save_attent(time_attent, img_attent, i)
print(i)
#print type(time_attent)
#print type(img_attent)
#print img_attent.shape
img_attent = lower_dim(img_attent)
feats_idx = lower_dim(feats_idx)
try:
time_attent0 = np.loadtxt("Output/epoch_" + str(epoch_num) +
"_time_attent.csv",
delimiter=",")
user_list0 = np.loadtxt("Output/epoch_" + str(epoch_num) +
"_user_list.csv",
delimiter=",")
img_attent0 = np.loadtxt("Output/epoch_" + str(epoch_num) +
"_img_attent.csv",
delimiter=",")
feats_idx0 = np.loadtxt("Output/epoch_" + str(epoch_num) +
"_feats_idx.csv",
delimiter=",")
#print time_attent0
time_attent = np.concatenate([time_attent0, time_attent])
user_list = np.concatenate([user_list0, user_list])
img_attent = np.concatenate([img_attent0, img_attent])
feats_idx = np.concatenate([feats_idx0, feats_idx])
except Exception, e:
#print 'time_attent.csv does note exist, creating'
print Exception, ":", e
np.savetxt("Output/epoch_" + str(epoch_num) + "_time_attent.csv",
time_attent,
delimiter=",")
np.savetxt("Output/epoch_" + str(epoch_num) + "_user_list.csv",
user_list,
delimiter=",")
np.savetxt("Output/epoch_" + str(epoch_num) + "_img_attent.csv",
img_attent,
delimiter=",")
np.savetxt("Output/epoch_" + str(epoch_num) + "_feats_idx.csv",
feats_idx,
delimiter=",")
def demographic_parity_train(model, dr, vdr, vvector, args):
feat, rel = dr.data
N = len(rel)
from utils import get_optimizer
optimizer = get_optimizer(model.parameters(),
args.lr[0],
args.optimizer,
weight_decay=args.weight_decay[0])
for epoch in range(args.epochs[0]):
for i in range(N):
feat, rel = shuffle_combined(feat, rel)
optimizer.zero_grad()
curr_feats = feat[i]
scores = model(torchify(curr_feats)).squeeze()
probs = torch.nn.Softmax(dim=0)(scores)
if np.sum(rel[i]) == 0:
continue
normalized_rels = rel[i] # / np.sum(rel[i])
# np.random.shuffle(normalized_rels)
ranking_loss = -torch.sum(
torch.FloatTensor(normalized_rels) * torch.log(probs))
# print(scores, probs,
# torch.log(probs), normalized_rels,
# torch.log(probs) * torch.FloatTensor(normalized_rels),
# ranking_loss)
exposures = vvector[0] * probs
groups = curr_feats[:, args.group_feat_id]
if np.all(groups == 0) or np.all(groups == 1):
fairness_loss = 0.0
else:
avg_exposure_0 = torch.sum(
torch.FloatTensor(1 - groups) *
exposures) / torch.sum(1 - torch.FloatTensor(groups))
avg_exposure_1 = torch.sum(
torch.FloatTensor(groups) * exposures) / torch.sum(
torch.FloatTensor(groups))
# print(avg_exposure_0, avg_exposure_1)
fairness_loss = torch.pow(
torch.clamp(avg_exposure_1 - avg_exposure_0, min=0), 2)
loss = args.lambda_reward * ranking_loss + args.lambda_group_fairness * fairness_loss
loss.backward()
optimizer.step()
# break
if i % args.evaluate_interval == 0 and i != 0:
results = evaluate_model(model,
vdr,
fairness_evaluation=False,
group_fairness_evaluation=True,
deterministic=True,
args=args,
num_sample_per_query=100)
print(results)
return model
def eval_params(w, bias, dr, D, det=False, args=None, intercept=True):
# Given the model weights, this function evaluates the model
model = LinearModel(D=D)
model.w.weight.data = torch.FloatTensor([w])
if intercept:
model.w.bias.data = torch.FloatTensor([bias])
return evaluate_model(model,
dr,
deterministic=det,
group_fairness_evaluation=True,
args=args,
fairness_evaluation=True)
def experiment_rbm_learning():
"""Train and test an RBM with normal toy data"""
trainingset = generate_dataset(N_train)
testset = generate_dataset(N_test)
model = StackedRBMs(trainingset.layer_sizes)
# Training
train_unsupervised(model, trainingset)
# Testing
results = evaluate_model(model, testset)
return results
def experiment_mlp_learning():
"""Test if an MLP learns the desired features"""
trainingset = generate_dataset(N_train)
testset = generate_dataset(N_test)
model = MLP(trainingset.layer_sizes)
# Training
train_supervised(model, trainingset)
# Testing
results = evaluate_model(model, testset)
return results
def experiment_rbm_power():
"""Test if an RBM has the representational power to learn the desired
features"""
trainingset = generate_dataset(N_train)
testset = generate_dataset(N_test)
model = StackedRBMs(trainingset.layer_sizes)
# Train the model using also the (not so) hidden labels
train_with_hidden_labels(model, trainingset)
# Testing
results = evaluate_model(model, testset)
return results
def train_and_evaluate():
# reload weights from restore_file if specified
if args.restore_file is not None:
restore_path = os.path.join(
args.tensorboard_dir, args.restore_file + '.pth.tar')
if os.path.exists(restore_path):
logging.info("Restoring parameters from {}".format(restore_path))
utils.load_checkpoint(restore_path, model.module, optimizer)
best_val_f1 = 0.0
for epoch in range(1, model_params.num_epochs+1):
# Run one epoch
logging.info("Epoch {}/{}".format(epoch, model_params.num_epochs))
# compute number of batches in one epoch (one full pass over the training set)
train(epoch)
# Evaluate for one epoch on validation set
val_metrics = evaluate_model(model.module, val_dataset, report_dir=args.tensorboard_dir)
logging.info(val_metrics)
val_f1 = val_metrics['Weighted Macro F1']
is_best = val_f1 >= best_val_f1
# Save weights
utils.save_checkpoint({'epoch': epoch,
'state_dict': model.module.state_dict(),
'optim_dict': optimizer.state_dict()},
is_best=is_best,
checkpoint=args.tensorboard_dir)
# If best_eval, best_save_path
if is_best:
logging.info("- Found new best macro f1")
best_val_f1 = val_f1
# Save best val metrics in a json file in the model directory
best_json_path = os.path.join(
args.tensorboard_dir, "metrics_val_best_weights.json")
utils.save_dict_to_json(val_metrics, best_json_path)
last_report_path = os.path.join(args.tensorboard_dir, 'report.txt')
best_report_path = os.path.join(args.tensorboard_dir, 'best_report.txt')
if os.path.exists(last_report_path):
shutil.copy(last_report_path, best_report_path)
# Save latest val metrics in a json file in the model directory
last_json_path = os.path.join(
args.tensorboard_dir, "metrics_val_last_weights.json")
utils.save_dict_to_json(val_metrics, last_json_path)
def main():
args, params = read_params_and_args()
params = adjust_params(args, params)
# prepare data (download and extract)
if args.prep_data:
data_handler.prepare_data(params)
# evaluation
elif args.eval:
evaluation.evaluate_model(args, params)
elif args.eval_bbox:
evaluation.evaluate_iobb(args, params, args.bbox_image,
args.bbox_disease)
else:
model = networks.init_unified_net(args.model, params)
optimizer = torch.optim.Adam(params=model.parameters(),
lr=params['lr'])
tracker = helper.init_comet(args, params) if args.use_comet else None
trainer.train(args, params, model, optimizer, tracker)
def experiment_mlp_power():
"""Try to find 'ideal' parameters to test if the MLP has the
representational power to required to learn the desired features.
"""
trainingset = generate_dataset(N_train)
testset = generate_dataset(N_test)
model = MLP(trainingset.layer_sizes)
# Train the model using also the (not so) hidden labels
train_with_hidden_labels(model, trainingset)
# Testing
results = evaluate_model(model, testset)
return results
def evaluate(model, shortname, fullname, with_aux):
shortname = shortname + ('_aux' if with_aux else '')
s = '' if with_aux else 'out'
print(
f'Training "{fullname} ({shortname})" model with{s} auxiliary training loss'
)
torch.manual_seed(args.seed)
loss, acc = evaluate_model(model,
num_epochs=args.epochs,
num_rounds=args.rounds,
with_aux_classes=with_aux,
aux_loss_factor=args.aux_loss_factor)
losses.append(loss)
accuracies.append(acc)
model_names.append(shortname)
def experiment_mlp_power_uncorrelated_features():
"""A variation of experiment_mlp_power.
Train the hidden layer with an 'uncorrelated' dataset, so it can not
cheat and learn to detect features by looking for commonly co-occurring
features.
"""
uncorrelated_trainingset = generate_uncorrelated_dataset(N_train)
trainingset = generate_dataset(N_train)
testset = generate_dataset(N_test)
model = MLP(trainingset.layer_sizes)
model.train_bottom(*uncorrelated_trainingset.get_layers()[:-1])
model.train_top(*trainingset.get_layers()[1:])
results = evaluate_model(model, testset)
return results
def experiment_rbm_power_stability():
"""Test if 'ideal' parameters for us also form an optimum for the RBMs
We first do the test for representational power, and then continue with
unsupervised training to see whether the parameters stay around the same
values.
"""
# First, train with hidden labels to test representational power
results0 = experiment_rbm_power()
model = results0['model']
testset = results0['testset']
# Then, unsupervised training to see if parameters stay or change
trainingset1 = generate_dataset(N_train)
train_unsupervised(model, trainingset1)
results1 = evaluate_model(model, testset)
return results0, results1
def configure(self, resolution, min_fps):
"""
Evaluate model performance and select model with desired fps
Parameters:
fps (int): Minimum frames per second for executing model
"""
model_found = False
for args in MODEL_RANKING:
perf = evaluate_model(args, resolution)
if perf.fps < min_fps:
self.upsampler = Upsampler(args)
model_found = True
break
self.configured = False
return model_found
def evaluate_and_store_results(y_test,
y_pred,
model,
example_based_results,
label_based_results,
print_results=True):
# example based evaluation
accuracy_score, precision_score, \
recall_score, f1_score, hamming_loss, classfication_report = evaluate_model(y_test, y_pred, print_results=print_results)
# label based evaluation
accuracy_per_label, precision_per_label, \
recall_per_label, f1_per_label = evaluate_per_label(y_test, y_pred, print_results=True)
example_based_results = example_based_results.append(
{
'representation': representation,
'estimator': str(estimator),
'model': model,
'accuracy': accuracy_score,
'precision': precision_score,
'recall': recall_score,
'f1': f1_score,
'hamming_loss': hamming_loss
},
ignore_index=True)
label_based_results = label_based_results.append(
{
'representation': representation,
'estimator': str(estimator),
'model': model,
'accuracy': accuracy_per_label,
'precision': precision_per_label,
'recall': recall_per_label,
'f1': f1_per_label
},
ignore_index=True)
return example_based_results, label_based_results
def evaluate(self, mat, masked_mat, top_n):
R_hat = self.user_vectors @ self.item_vectors.T
MAP, rec_at_k, mpr_all, mpr_mask = evaluate_model(
R_hat, mat, masked_mat, top_n)
return MAP, rec_at_k, mpr_all, mpr_mask
from keras.callbacks import EarlyStopping
# create the models
model = Sequential()
model.add(Embedding(len(word_index) + 1, 128, input_shape=(280, )))
model.add(Conv1D(128, 16, activation='relu'))
model.add(MaxPooling1D(pool_size=8))
model.add(Flatten())
model.add(Dropout(0.5))
model.add(Dense(100, input_shape=(X_train.shape[1], )))
model.add(Activation('sigmoid'))
model.add(Dropout(0.1))
model.add(Dense(y_train.shape[1], activation="softmax"))
model.compile(loss="categorical_crossentropy",
optimizer=Adam(),
metrics=["accuracy"])
model.summary()
history = model.fit(X_train,
y_train,
batch_size=32,
epochs=5,
shuffle=True,
verbose=1)
# evaluation
evaluate_model(model, X_test, y_test)
def main():
parser = create_parser()
args = parser.parse_args()
if args.setup:
create_directories()
if args.debug:
dataset = DATASETS['debug']
args.dataset = "debug"
features, _, labels, _ = preprocess_data(args.patch_size,
args.distribution,
dataset=dataset)
#print(features, 'debug')
#print("length of features: ",type(features), len(features),'element.shape: ',features[0][0])
features_train, features_test = features[:100], features[100:120]
labels_train, labels_test = labels[:100], labels[100:120]
elif args.train_model or args.evaluate_model or args.preprocess_data:
dataset = DATASETS[args.dataset]
#print(dataset.values())
load_from_cache = not args.preprocess_data
try:
features_train, features_test, labels_train, labels_test = preprocess_data(
args.patch_size,
args.distribution,
dataset=dataset,
only_cache=load_from_cache)
#print(features_train, 'train_model or evaluate_model or preprocess_data')
print("Length of features_train: ", len(features_train))
except IOError:
print("Cache file does not exist. Please run again with -p flag.")
sys.exit(1)
if args.visualise:
visualise_labels(labels_train, args.patch_size, LABELS_DIR)
visualise_labels(labels_test, args.patch_size, LABELS_DIR)
if not args.model_id:
timestamp = time.strftime("%d_%m_%Y_%H%M")
model_id = "{}_{}_{}".format(timestamp, args.dataset,
args.architecture)
else:
model_id = args.model_id
if args.init_model or args.train_model or args.evaluate_model:
model_dir = os.path.join(OUTPUT_DIR, model_id)
save_makedirs(model_dir)
# Hyperparameters for the model. Since there are so many of them it is
# more convenient to set them in the source code as opposed to passing
# them as arguments to the Command Line Interface. We use a list of tuples instead of a
# dict since we want to print the hyperparameters and for that purpose
# keep them in the predefined order.
hyperparameters = [
("architecture", args.architecture),
# Hyperparameters for the first convolutional layer.
("nb_filters_1", 64),
("filter_size_1", 9),
("stride_1", (2, 2)),
# Hyperparameter for the first pooling layer.
("pool_size_1", (2, 2)),
# Hyperparameter for the second convolutional layer (when
# two layer architecture is used).
("nb_filters_2", 128),
("filter_size_2", 5),
("stride_2", (1, 1)),
# Hyperparameters for Stochastic Gradient Descent.
("learning_rate", 0.05),
("momentum", 0.9),
("decay", 0.0)
]
hyperparameters_mnih = [
("architecture", args.architecture),
# Hyperparameters for the first convolutional layer.
("nb_filters_1", 64),
("filter_size_1", 16),
("stride_1", (4, 4)),
# Hyperparameter for the first pooling layer.
("pool_size_1", (2, 2)),
("pool_stride", 1),
# Hyperparameter for the second convolutional layer).
("nb_filters_2", 112),
("filter_size_2", 4),
("stride_2", (1, 1)),
# Hyperparameter for the third convolutional layer).
("nb_filters_3", 80),
("filter_size_3", 3),
("stride_3", (1, 1)),
# Hyperparameters for Stochastic Gradient Descent.
("learning_rate", 0.05),
("momentum", 0.9),
("decay", 0.0)
]
if args.init_model:
model = init_model(args.patch_size, model_id,
**dict(hyperparameters_mnih))
save_model_summary(hyperparameters_mnih, model, model_dir)
elif args.train_model or args.evaluate_model:
hyperparameters = dict(hyperparameters_mnih)
model = load_model(model_id)
model = compile_model(model, hyperparameters["learning_rate"],
hyperparameters['momentum'],
hyperparameters["decay"])
if args.train_model:
model = train_model(model,
features_train,
labels_train,
args.patch_size,
model_id,
model_dir,
nb_epoch=args.epochs,
checkpoints=args.checkpoints,
tensorboard=args.tensorboard,
earlystop=args.earlystop)
if args.evaluate_model:
evaluate_model(model,
features_test,
labels_test,
args.patch_size,
model_dir,
out_format=args.out_format)
def main():
#CN v AD
ld_helper = loader_helper(task=Task.CN_v_AD)
uuid = train_camull(ld_helper, epochs=40)
evaluate_model(device, uuid, ld_helper)
def evaluate_a_model(device):
print("Which task would you like to evaluate?")
print("1) NC v AD")
print("2) sMCI v pMCI")
choice = int(input("Please enter your input [1..2]: "))
task = Task
if choice == 1:
task = Task.NC_v_AD
else:
task = Task.sMCI_v_pMCI
print("Here are 10 of your most recent unevaluated {} models.".format(
str(task)))
print("\n")
print(" id | model uuid | Time | model task")
model_uuids = fetch_models_from_db(task)
target_uuid = ""
if not model_uuids == []:
valid = False
while (not valid):
choice = input(
"Please enter the model number [1, 10] or the uuid that you would like to choose:"
)
try:
target_uuid = model_uuids[int(choice) - 1]
valid = True
except:
for i in range(len(model_uuids)):
if choice == model_uuids:
target_uuid = choice
break
print("Please enter a valid input.")
ld_helper = LoaderHelper(task)
print("There are 5 folds to evaluate")
print(
"Input a fold number to evaluate or input 6 to evaluate all folds."
)
print("\n")
choice = int(input("Enter your choice [1,6]: "))
if (choice != 6):
evaluate_fold(device,
target_uuid,
ld_helper,
choice,
commit_to_db=True)
remove_from_db(target_uuid)
else:
evaluate_model(device, target_uuid, ld_helper, cur)
remove_from_db(target_uuid)
print(
"The model has been evaluated. Check the graphs folder and look at the metrics in the database."
)
basic_run(device)
else:
print("\n")
print("No models available. Please train a new model.")
choice = input("Would you like to train a new model[Y/n]?: ")
if choice == 'y' or 'Y' or '': train_new_model_cli
else: basic_run
def main():
config = get_config()
# Set all random seeds
seed_all(config["seed"])
# Figure out what device to use, (GP needs data in cpu-memory)
if (not config["cpu"]) and (not config["model"]
== "gp") and torch.cuda.is_available():
device = torch.device("cuda")
else:
device = torch.device("cpu")
# Read data
dataset = dataset_list.get_dataset_spec(config["dataset"])()
train_data, val_data, test_data = dataset.load(device)
config["x_dim"] = dataset.x_dim
config["y_dim"] = dataset.y_dim
# Init wandb
if config["name"] == None:
config["name"] = "{}-{}-{}".format(config["model"], config["dataset"],
time.strftime("%H-%M"))
tags = []
if config["test"]:
tags.append("test")
# Also setting config
wandb.init(project=constants.WANDB_PROJECT,
config=config,
name=config["name"],
tags=tags)
# Set computational device (should not be synched to wandb)
config["device"] = device
# Load model
model = models[config["model"]](config)
# Train model
if config["train"]:
# Create evaluation function to feed to model for use during training
def val_model(model, epoch_i):
if config["kde_val"]:
# Eval KDE
ev.evaluate_model(model,
data=val_data,
config=config,
epoch_i=epoch_i,
kde=True,
make_plots=False)
# Eval true
return ev.evaluate_model(model,
data=val_data,
config=config,
epoch_i=epoch_i)
model.train(train_data, config, val_func=val_model)
# Test model
if config["test"]:
# determine kernel scaling from evaluation set
_, best_ks = utils.kde_eval(model, val_data, config)
print("Kernel Scale: {}".format(best_ks))
model.kernel_scale = best_ks # Store best kernel scale in model
# Get true (according to model) log-likelihood
true_evaluation_vals = ev.evaluate_model(
model, test_data, config=config,
make_plots=True) # Make plots only this run
# Average testing using KDE over multiple random seeds
eval_list = [] # list of dicts mapping eval_metric -> value
for i in range(config["test_runs"]):
seed_all(constants.TEST_SEEDS[i])
evaluation_vals = ev.evaluate_model(model,
test_data,
config=config,
make_plots=False,
kde=True)
eval_list.append(evaluation_vals)
wandb_test_dict = {}
for key in eval_list[0].keys():
eval_values = [val_dict[key] for val_dict in eval_list]
wandb_test_dict["test_{}_mean".format(constants.EVAL_NAME_MAP[key])] =\
np.mean(eval_values)
wandb_test_dict["test_{}_std_dev".format(constants.EVAL_NAME_MAP[key])] =\
np.std(eval_values)
if key in true_evaluation_vals:
wandb_test_dict["test_{}_true".format(constants.EVAL_NAME_MAP[key])] =\
true_evaluation_vals[key]
wandb.log(wandb_test_dict)
test_print_string = "\t".join([])
print("Test metrics over {} seeds".format(config["test_runs"]))
for key in eval_list[0].keys():
wandb_key = "test_{}_".format(constants.EVAL_NAME_MAP[key])
print("{}: {:.5}±{:.5}".format(
key,
wandb_test_dict[wandb_key + "mean"],
wandb_test_dict[wandb_key + "std_dev"],
))
if key in true_evaluation_vals:
print("true {}: {:.5}".format(
key, wandb_test_dict[wandb_key + "true"]))
if config["eval_div"]:
divergence = ev.estimate_divergences(model, config, {
"train": train_data,
"val": val_data,
"test": test_data,
})
nthread=1,
subsample=0.7500000000000001)),
Normalizer(norm="max"),
StackingEstimator(
estimator=LogisticRegression(C=0.5, dual=False, penalty="l1")),
StackingEstimator(
estimator=LogisticRegression(C=0.5, dual=False, penalty="l1")),
Binarizer(threshold=0.25),
XGBClassifier(learning_rate=0.001,
max_depth=1,
min_child_weight=10,
n_estimators=100,
nthread=1,
subsample=1.0))
exported_pipeline.fit(training_features, training_target)
y_pred = exported_pipeline.predict(testing_features).astype(np.float64)
y_prob = exported_pipeline.predict_proba(testing_features)[:, 1].astype(
np.float64)
evaluate_model(y_test.astype(np.float64),
y_pred,
y_prob,
model_name='tpot',
country=COUNTRY,
predict_pov_rate=True,
store_model=True,
show=False,
model=exported_pipeline)
print("done")
