def train(self, data, verbose=True): """ Train all models and return the best one. Models are evaluated and ranked according to their ROC-AUC on a validation data set. Parameters ---------- data: pysster.Data A Data object providing training and validation data sets. verbose: bool If True, progress information (train/val loss) will be printed throughout the training. Returns ------- results: tuple(pysster.Model, str) The best performing model and an overview table of all models are returned. """ best_model_path = "{}/{}".format( gettempdir(), ''.join(random.choice(string.ascii_uppercase) for _ in range(20))) aucs = [] max_auroc = -1 for i, candidate in enumerate(self.candidates): model = Model(candidate, data) model.train(data, verbose) predictions = model.predict(data, "val") labels = data.get_labels("val") report = utils.performance_report(labels, predictions) roc_auc = np.sum(report[:, 0:-1] * report[:, -1, np.newaxis], axis=0) roc_auc = (roc_auc / np.sum(report[:, -1]))[3] aucs.append(roc_auc) if aucs[-1] > max_auroc: max_auroc = aucs[-1] utils.save_model(model, best_model_path) K.clear_session() K.reset_uids() if not verbose: continue print("\n=== Summary ===") print("Model {}/{} = {:.5f} weighted avg roc-auc".format( i + 1, len(self.candidates), aucs[i])) for param in candidate: if not param in ["input_shape"]: print(" - {}: {}".format(param, candidate[param])) # load the best model (and remove it from disc) model = utils.load_model(best_model_path) remove(best_model_path) remove("{}.h5".format(best_model_path)) # save a formatted summary of all trained models table = self._grid_search_table(aucs) return model, table
def test_utils_save_load_model(self): utils.save_model(self.m1, gettempdir() + "/model") self.assertTrue(isfile(gettempdir() + "/model")) self.assertTrue(isfile(gettempdir() + "/model.h5")) model = utils.load_model(gettempdir() + "/model") self.assertTrue(self.m1.params == model.params) self.assertTrue(self.m1.model.get_config() == model.model.get_config()) for x in range(6): self.assertTrue( np.allclose(self.m1.model.get_weights()[x], model.model.get_weights()[x])) remove(gettempdir() + "/model") remove(gettempdir() + "/model.h5")
def measure_rbp(entry): import os from time import time from pysster import utils output_folder = entry[4] + "_pysster/" if not os.path.isdir(output_folder): os.makedirs(output_folder) start = time() # predict secondary structures utils.predict_structures(entry[0], entry[0] + ".struct", annotate=True) utils.predict_structures(entry[1], entry[1] + ".struct", annotate=True) utils.predict_structures(entry[2], entry[2] + ".struct", annotate=True) utils.predict_structures(entry[3], entry[3] + ".struct", annotate=True) from pysster.Data import Data from pysster.Model import Model # load data data = Data([entry[0] + ".struct", entry[1] + ".struct"], ("ACGU", "HIMS")) data.train_val_test_split( 0.8, 0.1999 ) # we need to have at least one test sequence, even though we have a separate test object # training params = {"kernel_len": 8} model = Model(params, data) model.train(data) # load and predict test data data_test = Data([entry[2] + ".struct", entry[3] + ".struct"], ("ACGU", "HIMS")) predictions = model.predict(data_test, "all") stop = time() print("{}, time in seconds: {}".format(entry[4], stop - start)) # performance evaluation labels = data_test.get_labels("all") utils.plot_roc(labels, predictions, output_folder + "roc.pdf") utils.plot_prec_recall(labels, predictions, output_folder + "prec.pdf") # get motifs activations = model.get_max_activations(data_test, "all") _ = model.visualize_all_kernels(activations, data_test, output_folder) # save model to drive utils.save_model(model, "{}model.pkl".format(output_folder))
##Perfomance evaluation predictions = model.predict(data, "test") predictions labels = data.get_labels("test") labels utils.plot_roc(labels, predictions, output_folder + "roc.png") utils.plot_prec_recall(labels, predictions, output_folder + "prec.png") print(utils.get_performance_report(labels, predictions)) Image(output_folder + "roc.png") Image(output_folder + "prec.png") activations = model.get_max_activations(data, "test") logos = model.visualize_all_kernels(activations, data, output_folder) Image(output_folder + "motif_kernel_13.png") Image(output_folder + "activations_kernel_13.png") Image(output_folder + "position_kernel_13.png") Image(output_folder + "data/alu.png") utils.save_as_meme([logo[0] for logo in logos], output_folder + "motifs_seq.meme") utils.save_as_meme([logo[1] for logo in logos], output_folder + "motifs_struct.meme") model.plot_clustering(activations, output_folder + "clustering.png") Image(output_folder + "clustering.png") utils.save_data(data, output_folder + "data.pkl") utils.save_model(model, output_folder + "model.pkl")
def train(self, data, pr_auc=False, verbose=True): """ Train all models and return the best one. Models are evaluated and ranked according to their ROC-AUC or PR-AUC (precision-recall) on a validation data set. Parameters ---------- data: pysster.Data A Data object providing training and validation data sets. pr_auc: bool If True, the area under the precision-recall curve will be maximized instead of the area under the ROC curve verbose: bool If True, progress information (train/val loss) will be printed throughout the training. Returns ------- results: tuple(pysster.Model, str) The best performing model and an overview table of all models are returned. """ best_model_path = "{}/{}".format( gettempdir(), ''.join(random.choice(string.ascii_uppercase) for _ in range(20))) if True == pr_auc: metric_idx = 4 metric_name = "pre-auc" else: metric_idx = 3 metric_name = "roc-auc" metric = [] max_metric = -1 for i, candidate in enumerate(self.candidates): model = Model(candidate, data) model.train(data, verbose) predictions = model.predict(data, "val") labels = data.get_labels("val") report = utils.performance_report(labels, predictions) metric_val = np.sum(report[:, 0:-1] * report[:, -1, np.newaxis], axis=0) metric_val = (metric_val / np.sum(report[:, -1]))[metric_idx] metric.append(metric_val) if metric[-1] > max_metric: max_metric = metric[-1] utils.save_model(model, best_model_path) K.clear_session() K.reset_uids() if not verbose: continue print("\n=== Summary ===") print("Model {}/{} = {:.5f} weighted avg {}".format( i + 1, len(self.candidates), metric[i], metric_name)) for param in candidate: if not param in ["input_shape"]: print(" - {}: {}".format(param, candidate[param])) # load the best model (and remove it from disc) model = utils.load_model(best_model_path) remove(best_model_path) remove("{}.h5".format(best_model_path)) # save a formatted summary of all trained models table = self._grid_search_table(metric, metric_name) return model, table
def main(): RBPs = [("data/pum2.train.positive.fasta", "data/pum2.train.negative.fasta", "data/pum2.test.positive.fasta", "data/pum2.test.negative.fasta", "PUM2"), ("data/qki.train.positive.fasta", "data/qki.train.negative.fasta", "data/qki.test.positive.fasta", "data/qki.test.negative.fasta", "QKI"), ("data/igf2bp123.train.positive.fasta", "data/igf2bp123.train.negative.fasta", "data/igf2bp123.test.positive.fasta", "data/igf2bp123.test.negative.fasta", "IGF2BP123"), ("data/srsf1.train.positive.fasta", "data/srsf1.train.negative.fasta", "data/srsf1.test.positive.fasta", "data/srsf1.test.negative.fasta", "SRSF1"), ("data/taf2n.train.positive.fasta", "data/taf2n.train.negative.fasta", "data/taf2n.test.positive.fasta", "data/taf2n.test.negative.fasta", "TAF2N"), ("data/nova.train.positive.fasta", "data/nova.train.negative.fasta", "data/nova.test.positive.fasta", "data/nova.test.negative.fasta", "NOVA")] for entry in RBPs: output_folder = entry[4] + "_pysster/" if not os.path.isdir(output_folder): os.makedirs(output_folder) start = time() # predict secondary structures utils.predict_structures(entry[0], entry[0]+".struct.gz", annotate=True) utils.predict_structures(entry[1], entry[1]+".struct.gz", annotate=True) utils.predict_structures(entry[2], entry[2]+".struct.gz", annotate=True) utils.predict_structures(entry[3], entry[3]+".struct.gz", annotate=True) # load data data = Data([entry[0]+".struct.gz", entry[1]+".struct.gz"], ("ACGU", "HIMS")) data.train_val_test_split(0.8, 0.1999) # we need to have at least one test sequence, even though we don't need it print(data.get_summary()) # training params = {"kernel_len": 8} model = Model(params, data) model.train(data) # load and predict test data data_test = Data([entry[2]+".struct.gz", entry[3]+".struct.gz"], ("ACGU", "HIMS")) predictions = model.predict(data_test, "all") stop = time() print("{}, time in seconds: {}".format(entry[4], stop-start)) # performance evaluation labels = data_test.get_labels("all") utils.plot_roc(labels, predictions, output_folder+"roc.pdf") utils.plot_prec_recall(labels, predictions, output_folder+"prec.pdf") print(utils.get_performance_report(labels, predictions)) # get motifs activations = model.get_max_activations(data_test, "all") logos, scores = [], [] for kernel in range(model.params["kernel_num"]): logo, score = model.visualize_kernel(activations, data_test, kernel, output_folder) logos.append(logo) scores.append(score) # sort motifs by importance score sorted_idx = [i[0] for i in sorted(enumerate(scores), key=lambda x:x[1])] with open(output_folder+"kernel_scores.txt", "wt") as handle: for x in sorted_idx: print("kernel {:>3}: {:.3f}".format(x, scores[x])) handle.write("kernel {:>3}: {:.3f}\n".format(x, scores[x])) # save model to drive utils.save_model(model, "{}model.pkl".format(output_folder))