def predict(self, seq, batch_size=512): """Make model prediction Args: seq: numpy array of one-hot-encoded array of sequences batch_size: batch size Returns: dictionary key=task and value=prediction for the task """ if self.bias_model is not None: # TODO - what is this? seq, = self.bias_model.predict((seq, ), batch_size) preds = self.model.predict(seq, batch_size=batch_size) if len(self.model.output) == 2 * len(self.tasks): # extract the profile and count predictions profile_preds = {task: softmax(preds[task_i]) for task_i, task in enumerate(self.tasks)} count_preds = {task: preds[len(self.tasks) + task_i] for task_i, task in enumerate(self.tasks)} # compute the scaling factor if self.preproc is None: scales = {task: np.exp(count_preds[task]) - 1 for task in self.tasks} else: scales = {task: np.exp(self.preproc.objects[f'profile/{task}'].steps[1][1].inverse_transform(count_preds[task])) - 1 for task in self.tasks} # scaled profile (counts per base) return {task: profile_preds[task] * scales[task][:, np.newaxis] for task in self.tasks} else: return {task: preds[task_i] for task_i, task in enumerate(self.tasks)}
def postproc(preds, tasks): ntasks = len(tasks) preds[:len(tasks)] = [softmax(p) for p in preds[:ntasks]] preds_dict = dict(profile=preds[:ntasks], counts=preds[len(tasks):]) scales = pred2scale_strands(preds_dict, tasks) return { task: preds_dict['profile'][i] * scales[task][:, np.newaxis] for i, task in enumerate(tasks) }
def get_ism_score(onehot_data): from collections import OrderedDict from basepair.functions import softmax # create mutations onehot_data = np.array(onehot_data)[0] # print("one hot data: "+str(np.array(onehot_data).shape)) mutated_seqs = [] for sample in onehot_data: # print("sample: "+str(np.array(sample).shape)) for pos in range(len(sample)): for base in range(4): mutated = sample.copy() mutated[pos] = np.zeros((1, 4)) mutated[pos][base] = 1 mutated_seqs.append(mutated) mutated_seqs = np.array(mutated_seqs) # print("mutated seqs: "+str(mutated_seqs.shape)) # get predictions raw_predictions = keras_model.predict(mutated_seqs, batch_size=32) # get scores attribs = [] for sample_idx, sample in enumerate(onehot_data): temp_attribs = [] for pos in range(len(sample)): temp_attribs.append([]) for base in range(4): if pred_summary == 'count': relevant_output = raw_predictions[ len(self.tasks) + task_id][(sample_idx * len(sample) * 4) + (pos * 4) + base] temp_attribs[pos].append( relevant_output[strand_id]) else: relevant_output = raw_predictions[task_id][ (sample_idx * len(sample) * 4) + (pos * 4) + base] temp_attribs[pos].append( np.sum( softmax([relevant_output[:, strand_id]]) * [relevant_output[:, strand_id]])) temp_attribs = np.array(temp_attribs) avg_scores = np.mean(temp_attribs, axis=1, keepdims=True) # this is ACGT axis temp_attribs -= avg_scores # print("sample attribs: "+str(temp_attribs.shape)) attribs.append(temp_attribs) attribs = np.array([attribs]) # print("all attribs: "+str(attribs.shape)) # attribs = np.swapaxes(np.array(attribs), -1, -2) return attribs
def __call__(self, y_true, preds): profile_preds = { task: softmax(preds[task_i]) for task_i, task in enumerate(self.tasks) } count_preds = { task: preds[len(self.tasks) + task_i].sum(axis=-1) for task_i, task in enumerate(self.tasks) } profile_true = {task: y_true[f'profile/{task}'] for task in self.tasks} counts_true = { task: y_true[f'counts/{task}'].sum(axis=-1) for task in self.tasks } return ({ "profile": profile_true, "counts": counts_true }, { "profile": profile_preds, "counts": count_preds })
def evaluate(model_dir, output_dir=None, gpu=0, exclude_metrics=False, splits=['train', 'valid'], model_path=None, data=None, hparams=None, dataspec=None, preprocessor=None): """ Args: model_dir: path to the model directory splits: For which data splits to compute the evaluation metrics model_metrics: if True, metrics computed using mode.evaluate(..) """ if gpu is not None: create_tf_session(gpu) if dataspec is not None: ds = DataSpec.load(dataspec) else: ds = DataSpec.load(os.path.join(model_dir, "dataspec.yaml")) if hparams is not None: hp = HParams.load(hparams) else: hp = HParams.load(os.path.join(model_dir, "hparams.yaml")) if model_path is not None: model = load_model(model_path) else: model = load_model(os.path.join(model_dir, "model.h5")) if output_dir is None: output_dir = os.path.join(model_dir, "eval") train, valid, test = load_data(model_dir, dataspec=dataspec, hparams=hparams, data=data, preprocessor=preprocessor) data = dict(train=train, valid=valid, test=test) metrics = {} profile_metrics = [] os.makedirs(os.path.join(output_dir, "plots"), exist_ok=True) for split in tqdm(splits): y_pred = model.predict(data[split][0]) y_true = data[split][1] if not exclude_metrics: eval_metrics_values = model.evaluate(data[split][0], data[split][1]) eval_metrics = dict(zip(_listify(model.metrics_names), _listify(eval_metrics_values))) eval_metrics = {split + "/" + k.replace("_", "/"): v for k, v in eval_metrics.items()} metrics = {**eval_metrics, **metrics} for task in ds.task_specs: # Counts yp = y_pred[ds.task2idx(task, "counts")].sum(axis=-1) yt = y_true["counts/" + task].sum(axis=-1) # compute the correlation rp = pearsonr(yt, yp)[0] rs = spearmanr(yt, yp)[0] metrics = {**metrics, split + f"/counts/{task}/pearsonr": rp, split + f"/counts/{task}/spearmanr": rs, } fig = plt.figure(figsize=(5, 5)) plt.scatter(yp, yt, alpha=0.5) plt.xlabel("Predicted") plt.ylabel("Observed") plt.title(f"R_pearson={rp:.2f}, R_spearman={rs:.2f}") plt.savefig(os.path.join(output_dir, f"plots/counts.{split}.{task}.png")) # Profile yp = softmax(y_pred[ds.task2idx(task, "profile")]) yt = y_true["profile/" + task] df = eval_profile(yt, yp, pos_min_threshold=hp.evaluate.pos_min_threshold, neg_max_threshold=hp.evaluate.neg_max_threshold, required_min_pos_counts=hp.evaluate.required_min_pos_counts, binsizes=hp.evaluate.binsizes) df['task'] = task df['split'] = split # Evaluate for the smallest binsize auprc_min = df[df.binsize == min(hp.evaluate.binsizes)].iloc[0].auprc metrics[split + f'/profile/{task}/auprc'] = auprc_min profile_metrics.append(df) # Write the count metrics write_json(metrics, os.path.join(output_dir, "metrics.json")) # write the profile metrics dfm = pd.concat(profile_metrics) dfm.to_csv(os.path.join(output_dir, "profile_metrics.tsv"), sep='\t', index=False) return dfm, metrics