def validation_phase_mAP(history, main_dict, model, val_set, predict_name, epoch): val_dict, pred_annList = au.validate(model, val_set, predict_method=predict_name, n_val=len(val_set), return_annList=True) val_dict["predict_name"] = predict_name val_dict["epoch"] = epoch val_dict["time"] = datetime.datetime.now().strftime("%b %d, 20%y") # Update history history["val"] += [val_dict] path = main_dict["path_train_model"].replace(".pth", "_{}.pth".format(epoch)) ms.save_model(path, model) # ms.copy_code_best(main_dict) # # Higher is better # if (history["best_model"] == {} or # history["best_model"]["0.5"] <= val_dict["0.5"]): # history["best_model"] = val_dict # ms.save_best_model(main_dict, model) # ms.save_pkl(main_dict["path_best_annList"], pred_annList) # ms.copy_code_best(main_dict) return history
def main(): if keras.backend.image_dim_ordering() != 'th': keras.backend.set_image_dim_ordering('th') print "INFO: temporarily set 'image_dim_ordering' to 'th'" sess = get_session() keras.backend.set_session(sess) (train_xs, train_ys), (test_xs, test_ys) = data_cifar10.load_cifar10() print 'Loaded cifar10 data' x = tf.placeholder(tf.float32, shape=(None, 3, 32, 32)) y = tf.placeholder(tf.float32, shape=(None, 10)) model, model_name = resnet_cifar10.resnet_cifar10(repetations=3) predictions = model(x) tf_model_train(sess, x, y, predictions, train_xs, train_ys, test_xs, test_ys, data_augmentor=data_cifar10.augment_batch) save_model(model, model_name)
def eval(args, model, test_loader, info, print=print): print('Eval ' + time.ctime()) model.eval() test_loss = 0 correct = 0 data_count = 0 for data, target in test_loader: data_count += data.size(0) indx_target = target.clone() if args.cuda: data, target = data.cuda(), target.cuda() data, target = Variable(data, volatile=True), Variable(target) output = model(data) test_loss += F.cross_entropy(output, target).data[0] pred = output.data.max(1)[ 1] # get the index of the max log-probability correct += pred.cpu().eq(indx_target).sum() test_loss = test_loss / len( test_loader) # average over number of mini-batch acc = 100. * correct / data_count print('\tTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)'.format( test_loss, correct, data_count, acc)) if acc > info['best_acc']: new_file = os.path.join(args.logdir, 'best-{}.pth'.format(info['epoch'])) misc.save_model(model, new_file, old_file=info['old_file'], verbose=True) info['best_acc'] = acc info['old_file'] = new_file
def adam_pretrain(model, model_name, train_xs, train_ys, num_epoch, test_xs, test_ys): model.compile(optimizer=keras.optimizers.Adam(), loss='categorical_crossentropy', metrics=['accuracy']) model.fit(train_xs, train_ys, batch_size=128, nb_epoch=num_epoch, validation_data=(test_xs, test_ys), shuffle=True) model_name = '%s_adam_pretrain' % model_name save_model(model, model_name) model = load_model(model_name) return model
def run(args, model, data_loader, info, optimizer=None, print=print): is_train = (not optimizer is None) print('%s %d - %s' % (is_train and 'Train' or 'Eval', info['epoch'], time.ctime())) if is_train: model.train() else: model.eval() if is_train and info['epoch'] in args.decreasing_lr: optimizer.param_groups[0]['lr'] *= args.lr_decreasing_rate msg = None total_loss = 0 total_correct = 0 data_count = 0 for batch_idx, (data, target) in enumerate(data_loader): data_count += data.size(0) indx_target = target.clone() if args.cuda: data, target = data.cuda(), target.cuda() data, target = Variable(data), Variable(target) if is_train: optimizer.zero_grad() output = model(data) loss = F.cross_entropy(output, target) if is_train: loss.backward() optimizer.step() total_loss += loss.data[0] pred = output.data.max(1)[ 1] # get the index of the max log-probability total_correct += pred.cpu().eq(indx_target).sum() if (not is_train) or (batch_idx % args.log_interval == 0 and batch_idx > 0): loss = total_loss / data_count acc = 100. * total_correct / data_count msg = 'Loss:{:.3f},Acc:{}/{}({:.3f}%)'.format( total_loss / data_count, total_correct, data_count, 100. * total_correct / data_count) if is_train: msg += 'lr:{:.5f}'.format(optimizer.param_groups[0]['lr']) progress_bar(batch_idx, len(data_loader), msg) if (not is_train) and acc > info['best_acc']: new_file = os.path.join(args.logdir, 'best-{}.pth'.format(info['epoch'])) misc.save_model(model, new_file, old_file=info['old_file'], verbose=True) info['best_acc'] = acc info['old_file'] = new_file
def main(net_type): if keras.backend.image_dim_ordering() != 'th': keras.backend.set_image_dim_ordering('th') print "INFO: temporarily set 'image_dim_ordering' to 'th'" sess = get_session() keras.backend.set_session(sess) (train_xs, train_ys), (test_xs, test_ys) = data_cifar10.load_cifar10() print 'Loaded cifar10 data' x = tf.placeholder(tf.float32, shape=(None, 3, 32, 32)) y = tf.placeholder(tf.float32, shape=(None, 10)) model, model_name = resnet_cifar10.resnet_cifar10(repetations=3, net_type=net_type) if net_type == 'squared_resnet': model = adam_pretrain(model, model_name, train_xs, train_ys, 1, test_xs, test_ys) predictions = model(x) tf_model_train(sess, x, y, predictions, train_xs, train_ys, test_xs, test_ys, data_augmentor=data_cifar10.augment_batch) save_model(model, model_name) # Craft adversarial examples using Fast Gradient Sign Method (FGSM) adv_x = fgsm(x, predictions, eps=0.3) test_xs_adv, = batch_eval(sess, [x], [adv_x], [test_xs]) assert test_xs_adv.shape[0] == 10000, test_xs_adv.shape # Evaluate the accuracy of the MNIST model on adversarial examples accuracy = tf_model_eval(sess, x, y, predictions, test_xs_adv, test_ys) print'Test accuracy on adversarial examples: ' + str(accuracy) print "Repeating the process, using adversarial training" # Redefine TF model graph model_2, _ = resnet_cifar10.resnet_cifar10(repetations=3, net_type=net_type) predictions_2 = model_2(x) adv_x_2 = fgsm(x, predictions_2, eps=0.3) predictions_2_adv = model_2(adv_x_2) # Perform adversarial training tf_model_train(sess, x, y, predictions_2, train_xs, train_ys, test_xs, test_ys, predictions_adv=predictions_2_adv, data_augmentor=data_cifar10.augment_batch) save_model(model, model_name+'_adv') # Craft adversarial examples using Fast Gradient Sign Method (FGSM) on # the new model, which was trained using adversarial training test_xs_adv_2, = batch_eval(sess, [x], [adv_x_2], [test_xs]) assert test_xs_adv_2.shape[0] == 10000, test_xs_adv_2.shape # Evaluate the accuracy of the adversarially trained model on adversarial examples accuracy_adv = tf_model_eval(sess, x, y, predictions_2, test_xs_adv_2, test_ys) print'Test accuracy on adversarial examples: ' + str(accuracy_adv)
def supervised_learning_steps(method,scoring,data_type,task,model,params,X_train,y_train,n_iter): gs = grid_search_cv(model, params, X_train, y_train, scoring=scoring, n_iter = n_iter) y_pred = gs.predict(X_train) y_pred[y_pred < 0] = 0 if task: results=calculate_classification_metrics(y_train, y_pred) print("Acc: %.3f, F1: %.3f, AUC: %.3f, AUPR: %.3f" % (results[0], results[1], results[2], results[3])) else: results=calculate_regression_metrics(y_train,y_pred) print("MAE: %.3f, MSE: %.3f, R2: %.3f, Pearson R: %.3f, Spearman R: %.3f" % (results[0], results[1], results[2], results[3], results[4])) print('Parameters') print('----------') for p,v in gs.best_estimator_.get_params().items(): print(p, ":", v) print('-' * 80) if task: save_model(gs, "%s_models/%s_%s_classifier_gs.pk" % (method,method,data_type)) save_model(gs.best_estimator_, "%s_models/%s_%s_classifier_best_estimator.pk" %(method,method,data_type)) else: save_model(gs, "%s_models/%s_%s_regressor_gs.pk" % (method,method,data_type)) save_model(gs.best_estimator_, "%s_models/%s_%s_regressor_best_estimator.pk" %(method,method,data_type)) return(gs)
n_iter = 200 scaler = preprocessing.MinMaxScaler() X_train_copy = scaler.fit_transform(X_train) if classification_task: svm_gs = supervised_learning_steps("svm", "roc_auc", data_type, classification_task, model, param_svm, X_train_copy, y_train, n_iter) else: svm_gs = supervised_learning_steps("svm", "r2", data_type, classification_task, model, param_svm, X_train_copy, y_train, n_iter) svm_gs.cv_results_ save_model(scaler, "%s_models/%s_%s_scaling_gs.pk" % ("svm", "svm", data_type)) # - svm_gs = load_model("svm_models/svm__LS_Drug_LS_Protein_regressor_gs.pk") scaler = load_model("svm_models/svm__LS_Drug_LS_Protein_scaling_gs.pk") svm_best = svm_gs.best_estimator_ # + np.max(svm_gs.cv_results_['mean_test_score']) file_list = [ "../data/Test_Compound_Viral_interactions_for_Supervised_Learning_with_LS_LS.csv", "../data/Test_Compound_Viral_interactions_for_Supervised_Learning_with_MFP_LS.csv" ] filename = file_list[input_option]