if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Run motif simulation')
parser.add_argument('exp', help='which experiment to run the simulation for')
parser.add_argument('--gpu', default=0, type=int, help='Which GPU to use')
args = parser.parse_args()
exp = args.exp
# imp_score = args.imp_score
ddir = get_data_dir()
# load the model
logger.info("Loading model")
if args.gpu is not None:
create_tf_session(args.gpu)
model_dir = models_dir / exp
bpnet = BPNetSeqModel.from_mdir(model_dir)
output_dir = model_dir / 'perturbation-analysis'
pairs = get_motif_pairs(motifs)
dfi = multiple_load_instances({task: model_dir / f'deeplift/{task}/out/profile/wn/instances.parq'
for task in bpnet.tasks},
motifs=motifs)
dfi = filter_nonoverlapping_intervals(dfi)
# dfi = dfi.iloc[:1000]
tasks = bpnet.tasks
def evaluate(model_dir,
output_dir=None,
gpu=0,
exclude_metrics=False,
splits=['train', 'valid'],
model_path=None,
data=None,
hparams=None,
dataspec=None,
preprocessor=None):
"""
Args:
model_dir: path to the model directory
splits: For which data splits to compute the evaluation metrics
model_metrics: if True, metrics computed using mode.evaluate(..)
"""
if gpu is not None:
create_tf_session(gpu)
if dataspec is not None:
ds = DataSpec.load(dataspec)
else:
ds = DataSpec.load(os.path.join(model_dir, "dataspec.yaml"))
if hparams is not None:
hp = HParams.load(hparams)
else:
hp = HParams.load(os.path.join(model_dir, "hparams.yaml"))
if model_path is not None:
model = load_model(model_path)
else:
model = load_model(os.path.join(model_dir, "model.h5"))
if output_dir is None:
output_dir = os.path.join(model_dir, "eval")
train, valid, test = load_data(model_dir,
dataspec=dataspec,
hparams=hparams,
data=data,
preprocessor=preprocessor)
data = dict(train=train, valid=valid, test=test)
metrics = {}
profile_metrics = []
os.makedirs(os.path.join(output_dir, "plots"),
exist_ok=True)
for split in tqdm(splits):
y_pred = model.predict(data[split][0])
y_true = data[split][1]
if not exclude_metrics:
eval_metrics_values = model.evaluate(data[split][0],
data[split][1])
eval_metrics = dict(zip(_listify(model.metrics_names),
_listify(eval_metrics_values)))
eval_metrics = {split + "/" + k.replace("_", "/"): v
for k, v in eval_metrics.items()}
metrics = {**eval_metrics, **metrics}
for task in ds.task_specs:
# Counts
yp = y_pred[ds.task2idx(task, "counts")].sum(axis=-1)
yt = y_true["counts/" + task].sum(axis=-1)
# compute the correlation
rp = pearsonr(yt, yp)[0]
rs = spearmanr(yt, yp)[0]
metrics = {**metrics,
split + f"/counts/{task}/pearsonr": rp,
split + f"/counts/{task}/spearmanr": rs,
}
fig = plt.figure(figsize=(5, 5))
plt.scatter(yp, yt, alpha=0.5)
plt.xlabel("Predicted")
plt.ylabel("Observed")
plt.title(f"R_pearson={rp:.2f}, R_spearman={rs:.2f}")
plt.savefig(os.path.join(output_dir, f"plots/counts.{split}.{task}.png"))
# Profile
yp = softmax(y_pred[ds.task2idx(task, "profile")])
yt = y_true["profile/" + task]
df = eval_profile(yt, yp,
pos_min_threshold=hp.evaluate.pos_min_threshold,
neg_max_threshold=hp.evaluate.neg_max_threshold,
required_min_pos_counts=hp.evaluate.required_min_pos_counts,
binsizes=hp.evaluate.binsizes)
df['task'] = task
df['split'] = split
# Evaluate for the smallest binsize
auprc_min = df[df.binsize == min(hp.evaluate.binsizes)].iloc[0].auprc
metrics[split + f'/profile/{task}/auprc'] = auprc_min
profile_metrics.append(df)
# Write the count metrics
write_json(metrics, os.path.join(output_dir, "metrics.json"))
# write the profile metrics
dfm = pd.concat(profile_metrics)
dfm.to_csv(os.path.join(output_dir, "profile_metrics.tsv"),
sep='\t', index=False)
return dfm, metrics
def imp_score_seqmodel(model_dir,
output_file,
dataspec=None,
peak_width=1000,
seq_width=None,
intp_pattern='*', # specifies which imp. scores to compute
# skip_trim=False, # skip trimming the output
method="deeplift",
batch_size=512,
max_batches=-1,
shuffle_seq=False,
memfrac=0.45,
num_workers=10,
h5_chunk_size=512,
exclude_chr='',
include_chr='',
overwrite=False,
skip_bias=False,
gpu=None):
"""Run importance scores for a BPNet model
Args:
model_dir: path to the model directory
output_file: output file path (HDF5 format)
method: which importance scoring method to use ('grad', 'deeplift' or 'ism')
split: for which dataset split to compute the importance scores
h5_chunk_size: hdf5 chunk size.
exclude_chr: comma-separated list of chromosomes to exclude
overwrite: if True, overwrite the output directory
skip_bias: if True, don't store the bias tracks in teh output
gpu (int): which GPU to use locally. If None, GPU is not used
"""
add_file_logging(os.path.dirname(output_file), logger, 'modisco-score-seqmodel')
if gpu is not None:
create_tf_session(gpu, per_process_gpu_memory_fraction=memfrac)
else:
# Don't use any GPU's
os.environ['CUDA_VISIBLE_DEVICES'] = ''
if os.path.exists(output_file):
if overwrite:
os.remove(output_file)
else:
raise ValueError(f"File exists {output_file}. Use overwrite=True to overwrite it")
if seq_width is None:
logger.info("Using seq_width = peak_width")
seq_width = peak_width
# make sure these are int's
seq_width = int(seq_width)
peak_width = int(peak_width)
# Split
intp_patterns = intp_pattern.split(",")
# Allow chr inclusion / exclusion
if exclude_chr:
exclude_chr = exclude_chr.split(",")
else:
exclude_chr = None
if include_chr:
include_chr = include_chr.split(",")
else:
include_chr = None
logger.info("Loading the config files")
model_dir = Path(model_dir)
if dataspec is None:
# Specify dataspec
dataspec = model_dir / "dataspec.yaml"
ds = DataSpec.load(dataspec)
logger.info("Creating the dataset")
from basepair.datasets import StrandedProfile
dl_valid = StrandedProfile(ds,
incl_chromosomes=include_chr,
excl_chromosomes=exclude_chr,
peak_width=peak_width,
seq_width=seq_width,
shuffle=False,
taskname_first=True, # Required to work nicely with imp-score
target_transformer=None)
# Setup importance score trimming
if seq_width > peak_width:
# Trim
# make sure we can nicely trim the peak
logger.info("Trimming the output")
assert (seq_width - peak_width) % 2 == 0
trim_start = (seq_width - peak_width) // 2
trim_end = seq_width - trim_start
assert trim_end - trim_start == peak_width
elif seq_width == peak_width:
trim_start = 0
trim_end = peak_width
else:
raise ValueError("seq_width < peak_width")
seqmodel = SeqModel.from_mdir(model_dir)
# get all possible interpretation names
# make sure they match the specified glob
intp_names = [name for name, _ in seqmodel.get_intp_tensors(preact_only=False)
if fnmatch_any(name, intp_patterns)]
logger.info(f"Using the following interpretation targets:")
for n in intp_names:
print(n)
writer = HDF5BatchWriter(output_file, chunk_size=h5_chunk_size)
for i, batch in enumerate(tqdm(dl_valid.batch_iter(batch_size=batch_size, num_workers=num_workers))):
# store the original batch containing 'inputs' and 'targets'
wdict = batch
if skip_bias:
wdict['inputs'] = {'seq': wdict['inputs']['seq']} # ignore all other inputs
if max_batches > 0:
logging.info(f"max_batches: {max_batches} exceeded. Stopping the computation")
if i > max_batches:
break
if shuffle_seq:
# Di-nucleotide shuffle the sequences
batch['inputs']['seq'] = onehot_dinucl_shuffle(batch['inputs']['seq'])
for name in intp_names:
hyp_imp = seqmodel.imp_score(batch['inputs']['seq'],
name=name,
method=method,
batch_size=None) # don't second-batch
# put importance scores to the dictionary
# also trim the importance scores appropriately so that
# the output will always be w.r.t. the peak center
wdict[f"/hyp_imp/{name}"] = hyp_imp[:, trim_start:trim_end]
# trim the sequence as well
if isinstance(wdict['inputs'], dict):
# Trim the sequence
wdict['inputs']['seq'] = wdict['inputs']['seq'][:, trim_start:trim_end]
else:
wdict['inputs'] = wdict['inputs'][:, trim_start:trim_end]
writer.batch_write(wdict)
writer.close()
def imp_score(model_dir,
output_file,
method="grad",
split='all',
batch_size=512,
num_workers=10,
h5_chunk_size=512,
max_batches=-1,
shuffle_seq=False,
memfrac=0.45,
exclude_chr='',
overwrite=False,
gpu=None):
"""Run importance scores for a BPNet model
Args:
model_dir: path to the model directory
output_file: output file path (HDF5 format)
method: which importance scoring method to use ('grad', 'deeplift' or 'ism')
split: for which dataset split to compute the importance scores
h5_chunk_size: hdf5 chunk size.
exclude_chr: comma-separated list of chromosomes to exclude
overwrite: if True, overwrite the output directory
gpu (int): which GPU to use locally. If None, GPU is not used
"""
add_file_logging(os.path.dirname(output_file), logger, 'modisco-score')
if gpu is not None:
create_tf_session(gpu, per_process_gpu_memory_fraction=memfrac)
else:
# Don't use any GPU's
os.environ['CUDA_VISIBLE_DEVICES'] = ''
if os.path.exists(output_file):
if overwrite:
os.remove(output_file)
else:
raise ValueError(f"File exists {output_file}. Use overwrite=True to overwrite it")
if exclude_chr:
exclude_chr = exclude_chr.split(",")
else:
exclude_chr = []
# load the config files
logger.info("Loading the config files")
model_dir = Path(model_dir)
hp = HParams.load(model_dir / "hparams.yaml")
ds = DataSpec.load(model_dir / "dataspec.yaml")
tasks = list(ds.task_specs)
# validate that the correct dataset was used
if hp.data.name != 'get_StrandedProfile_datasets':
logger.warn("hp.data.name != 'get_StrandedProfile_datasets'")
if split == 'valid':
assert len(exclude_chr) == 0
incl_chromosomes = hp.data.kwargs['valid_chr']
excl_chromosomes = None
elif split == 'test':
assert len(exclude_chr) == 0
incl_chromosomes = hp.data.kwargs['test_chr']
excl_chromosomes = None
elif split == 'train':
assert len(exclude_chr) == 0
incl_chromosomes = None
excl_chromosomes = hp.data.kwargs['valid_chr'] + hp.data.kwargs['test_chr'] + hp.data.kwargs.get('exclude_chr', [])
elif split == 'all':
incl_chromosomes = None
excl_chromosomes = hp.data.kwargs.get('exclude_chr', []) + exclude_chr
logger.info("Excluding chromosomes: {excl_chromosomes}")
else:
raise ValueError("split needs to be from {train,valid,test,all}")
logger.info("Creating the dataset")
from basepair.datasets import StrandedProfile
seq_len = hp.data.kwargs['peak_width']
dl_valid = StrandedProfile(ds,
incl_chromosomes=incl_chromosomes,
excl_chromosomes=excl_chromosomes,
peak_width=seq_len,
shuffle=False,
target_transformer=None)
bpnet = BPNet.from_mdir(model_dir)
writer = HDF5BatchWriter(output_file, chunk_size=h5_chunk_size)
for i, batch in enumerate(tqdm(dl_valid.batch_iter(batch_size=batch_size, num_workers=num_workers))):
if max_batches > 0:
logging.info(f"max_batches: {max_batches} exceeded. Stopping the computation")
if i > max_batches:
break
# append the bias model predictions
# (batch['inputs'], batch['targets']) = bm((batch['inputs'], batch['targets']))
# store the original batch containing 'inputs' and 'targets'
wdict = batch
if shuffle_seq:
# Di-nucleotide shuffle the sequences
if 'seq' in batch['inputs']:
batch['inputs']['seq'] = onehot_dinucl_shuffle(batch['inputs']['seq'])
else:
batch['inputs'] = onehot_dinucl_shuffle(batch['inputs'])
# loop through all tasks, pred_summary and strands
for task_i, task in enumerate(tasks):
for pred_summary in ['count', 'weighted']:
# figure out the number of channels
nstrands = batch['targets'][f'profile/{task}'].shape[-1]
strand_hash = ["pos", "neg"]
for strand_i in range(nstrands):
hyp_imp = bpnet.imp_score(batch['inputs'],
task=task,
strand=strand_hash[strand_i],
method=method,
pred_summary=pred_summary,
batch_size=None) # don't second-batch
# put importance scores to the dictionary
wdict[f"/hyp_imp/{task}/{pred_summary}/{strand_i}"] = hyp_imp
writer.batch_write(wdict)
writer.close()
def modisco_run(
imp_scores,
output_dir,
null_imp_scores=None,
hparams=None,
override_hparams="",
grad_type="weighted",
subset_tasks=None,
filter_subset_tasks=False,
filter_npy=None,
exclude_chr="",
seqmodel=False, # interpretation glob
# hparams=None,
num_workers=10,
max_strand_distance=0.1,
overwrite=False,
skip_dist_filter=False,
use_all_seqlets=False,
merge_tasks=False,
gpu=None,
):
"""
Run modisco
Args:
imp_scores: path to the hdf5 file of importance scores
null_imp_scores: Path to the null importance scores
grad_type: for which output to compute the importance scores
hparams: None, modisco hyper - parameeters: either a path to modisco.yaml or
a ModiscoHParams object
override_hparams: hyper - parameters overriding the settings in the hparams file
output_dir: output file directory
filter_npy: path to a npy file containing a boolean vector used for subsetting
exclude_chr: comma-separated list of chromosomes to exclude
seqmodel: If enabled, then the importance scores came from `imp-score-seqmodel`
subset_tasks: comma-separated list of task names to use as a subset
filter_subset_tasks: if True, run modisco only in the regions for that TF
hparams: hyper - parameter file
summary: which summary statistic to use for the profile gradients
skip_dist_filter: if True, distances are not used to filter
use_all_seqlets: if True, don't restrict the number of seqlets
split: On which data split to compute the results
merge_task: if True, importance scores for the tasks will be merged
gpu: which gpu to use. If None, don't use any GPU's
Note: when using subset_tasks, modisco will run on all the importance scores. If you wish
to run it only for the importance scores for a particular task you should subset it to
the peak regions of interest using `filter_npy`
"""
plt.switch_backend('agg')
add_file_logging(output_dir, logger, 'modisco-run')
import os
if gpu is not None:
create_tf_session(gpu)
else:
# Don't use any GPU's
os.environ['CUDA_VISIBLE_DEVICES'] = ''
os.environ['MKL_THREADING_LAYER'] = 'GNU'
# import theano
import modisco
import modisco.tfmodisco_workflow.workflow
if seqmodel:
assert '/' in grad_type
if subset_tasks == '':
logger.warn("subset_tasks == ''. Not using subset_tasks")
subset_tasks = None
if subset_tasks == 'all':
# Use all subset tasks e.g. don't subset
subset_tasks = None
if subset_tasks is not None:
subset_tasks = subset_tasks.split(",")
if len(subset_tasks) == 0:
raise ValueError("Provide one or more subset_tasks. Found None")
if filter_subset_tasks and subset_tasks is None:
print("Using filter_subset_tasks=False since `subset_tasks` is None")
filter_subset_tasks = False
if exclude_chr:
exclude_chr = exclude_chr.split(",")
else:
exclude_chr = []
output_dir = Path(output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
output_path = output_dir / "modisco.h5"
remove_exists(output_path, overwrite)
output_distances = output_dir / "strand_distances.h5"
remove_exists(output_distances, overwrite)
if filter_npy is not None:
filter_npy = os.path.abspath(filter_npy)
# save the hyper-parameters
write_json(
dict(
imp_scores=os.path.abspath(imp_scores),
grad_type=grad_type,
output_dir=str(output_dir),
subset_tasks=subset_tasks,
filter_subset_tasks=filter_subset_tasks,
hparams=hparams,
null_imp_scores=null_imp_scores,
# TODO - pack into hyper-parameters as well?
filter_npy=filter_npy,
exclude_chr=",".join(exclude_chr),
skip_dist_filter=skip_dist_filter,
use_all_seqlets=use_all_seqlets,
max_strand_distance=max_strand_distance,
gpu=gpu),
os.path.join(output_dir, "kwargs.json"))
print("-" * 40)
# parse the hyper-parameters
if hparams is None:
print(f"Using default hyper-parameters")
hp = ModiscoHParams()
else:
if isinstance(hparams, str):
print(f"Loading hyper-parameters from file: {hparams}")
hp = ModiscoHParams.load(hparams)
else:
assert isinstance(hparams, ModiscoHParams)
hp = hparams
if override_hparams:
print(f"Overriding the following hyper-parameters: {override_hparams}")
hp = tf.contrib.training.HParams(
**hp.get_modisco_kwargs()).parse(override_hparams)
if use_all_seqlets:
hp.max_seqlets_per_metacluster = None
# save the hyper-parameters
print("Using the following hyper-parameters for modisco:")
print("-" * 40)
related_dump_yaml(ModiscoHParams(**hp.values()),
os.path.join(output_dir, "hparams.yaml"),
verbose=True)
print("-" * 40)
# TODO - replace with imp_scores
d = HDF5Reader.load(imp_scores)
if 'hyp_imp' not in d:
# backcompatibility
d['hyp_imp'] = d['grads']
if seqmodel:
tasks = list(d['targets'])
else:
tasks = list(d['targets']['profile'])
if subset_tasks is not None:
# validate that all the `subset_tasks`
# are present in `tasks`
for st in subset_tasks:
if st not in tasks:
raise ValueError(
f"subset task {st} not found in tasks: {tasks}")
logger.info(
f"Using the following tasks: {subset_tasks} instead of the original tasks: {tasks}"
)
tasks = subset_tasks
if isinstance(d['inputs'], dict):
one_hot = d['inputs']['seq']
else:
one_hot = d['inputs']
n = len(one_hot)
# --------------------
# apply filters
if not skip_dist_filter:
print("Using profile prediction for the strand filtering")
grad_type_filtered = 'weighted'
distances = np.array([
np.array([
correlation(
np.ravel(d['hyp_imp'][task][grad_type_filtered][0][i]),
np.ravel(d['hyp_imp'][task][grad_type_filtered][1][i]))
for i in range(n)
]) for task in tasks
if len(d['hyp_imp'][task][grad_type_filtered]) == 2
]).T.mean(axis=-1) # average the distances across tasks
dist_filter = distances < max_strand_distance
print(f"Fraction of sequences kept: {dist_filter.mean()}")
HDF5BatchWriter.dump(output_distances, {
"distances": distances,
"included_samples": dist_filter
})
else:
dist_filter = np.ones((n, ), dtype=bool)
# add also the filter numpy
if filter_npy is not None:
print(f"Loading a filter file from {filter_npy}")
filter_vec = np.load(filter_npy)
dist_filter = dist_filter & filter_vec
if filter_subset_tasks:
assert subset_tasks is not None
interval_from_task = pd.Series(d['metadata']['interval_from_task'])
print(
f"Subsetting the intervals accoring to subset_tasks: {subset_tasks}"
)
print(f"Number of original regions: {dist_filter.sum()}")
dist_filter = dist_filter & interval_from_task.isin(
subset_tasks).values
print(
f"Number of filtered regions after filter_subset_tasks: {dist_filter.sum()}"
)
# filter by chromosome
if exclude_chr:
logger.info(f"Excluding chromosomes: {exclude_chr}")
chromosomes = d['metadata']['range']['chr']
dist_filter = dist_filter & (
~pd.Series(chromosomes).isin(exclude_chr)).values
# -------------------------------------------------------------
# setup importance scores
if seqmodel:
thr_one_hot = one_hot[dist_filter]
thr_hypothetical_contribs = {
f"{task}/{gt}":
d['hyp_imp'][task][gt.split("/")[0]][gt.split("/")[1]][dist_filter]
for task in tasks for gt in grad_type.split(",")
}
thr_contrib_scores = {
f"{task}/{gt}":
thr_hypothetical_contribs[f"{task}/{gt}"] * thr_one_hot
for task in tasks for gt in grad_type.split(",")
}
task_names = [
f"{task}/{gt}" for task in tasks for gt in grad_type.split(",")
]
else:
if merge_tasks:
thr_one_hot = np.concatenate([
one_hot[dist_filter] for task in tasks
for gt in grad_type.split(",")
])
thr_hypothetical_contribs = {
"merged":
np.concatenate([
mean(d['hyp_imp'][task][gt])[dist_filter] for task in tasks
for gt in grad_type.split(",")
])
}
thr_contrib_scores = {
"merged": thr_hypothetical_contribs['merged'] * thr_one_hot
}
task_names = ['merged']
else:
thr_one_hot = one_hot[dist_filter]
thr_hypothetical_contribs = {
f"{task}/{gt}": mean(d['hyp_imp'][task][gt])[dist_filter]
for task in tasks for gt in grad_type.split(",")
}
thr_contrib_scores = {
f"{task}/{gt}":
thr_hypothetical_contribs[f"{task}/{gt}"] * thr_one_hot
for task in tasks for gt in grad_type.split(",")
}
task_names = [
f"{task}/{gt}" for task in tasks for gt in grad_type.split(",")
]
if null_imp_scores is not None:
logger.info(f"Using null_imp_scores: {null_imp_scores}")
null_isf = ImpScoreFile(null_imp_scores)
null_per_pos_scores = {
f"{task}/{gt}": v.sum(axis=-1)
for gt in grad_type.split(",")
for task, v in null_isf.get_contrib(imp_score=gt).items()
if task in tasks
}
else:
# default Null distribution. Requires modisco 5.0
logger.info(f"Using default null_imp_scores")
null_per_pos_scores = modisco.coordproducers.LaplaceNullDist(
num_to_samp=10000)
# -------------------------------------------------------------
# run modisco
tfmodisco_results = modisco.tfmodisco_workflow.workflow.TfModiscoWorkflow(
# Modisco defaults
sliding_window_size=hp.sliding_window_size,
flank_size=hp.flank_size,
target_seqlet_fdr=hp.target_seqlet_fdr,
min_passing_windows_frac=hp.min_passing_windows_frac,
max_passing_windows_frac=hp.max_passing_windows_frac,
min_metacluster_size=hp.min_metacluster_size,
max_seqlets_per_metacluster=hp.max_seqlets_per_metacluster,
seqlets_to_patterns_factory=modisco.tfmodisco_workflow.
seqlets_to_patterns.TfModiscoSeqletsToPatternsFactory(
trim_to_window_size=hp.trim_to_window_size, # default: 30
initial_flank_to_add=hp.initial_flank_to_add, # default: 10
kmer_len=hp.kmer_len, # default: 8
num_gaps=hp.num_gaps, # default: 3
num_mismatches=hp.num_mismatches, # default: 2
n_cores=num_workers,
final_min_cluster_size=hp.final_min_cluster_size) # default: 30
)(
task_names=task_names,
contrib_scores=thr_contrib_scores, # -> task score
hypothetical_contribs=thr_hypothetical_contribs,
one_hot=thr_one_hot,
null_per_pos_scores=null_per_pos_scores)
# -------------------------------------------------------------
# save the results
grp = h5py.File(output_path)
tfmodisco_results.save_hdf5(grp)
