def modisco_score(modisco_dir,
imp_scores,
output_tsv,
output_seqlets_pkl=None,
seqlet_len=25,
n_cores=1,
method="rank",
trim_pattern=False):
"""Find seqlet instances using modisco
"""
add_file_logging(os.path.dirname(output_tsv), logger, 'modisco-score')
mr, tasks, grad_type = load_modisco_results(modisco_dir)
# load importance scores we want to score
d = HDF5Reader.load(imp_scores)
if 'hyp_imp' not in d:
# backcompatibility
d['hyp_imp'] = d['grads']
if isinstance(d['inputs'], dict):
one_hot = d['inputs']['seq']
else:
one_hot = d['inputs']
hypothetical_contribs = {
f"{task}/{gt}": mean(d['hyp_imp'][task][gt])
for task in tasks for gt in grad_type.split(",")
}
contrib_scores = {
f"{task}/{gt}": hypothetical_contribs[f"{task}/{gt}"] * one_hot
for task in tasks for gt in grad_type.split(",")
}
seqlets = find_instances(mr,
tasks,
contrib_scores,
hypothetical_contribs,
one_hot,
seqlet_len=seqlet_len,
n_cores=n_cores,
method=method,
trim_pattern=trim_pattern)
if len(seqlets) == 0:
print("ERROR: no seqlets found!!")
return [], None
if output_seqlets_pkl:
write_pkl(seqlets, output_seqlets_pkl)
df = labelled_seqlets2df(seqlets)
dfm = pd.DataFrame(d['metadata']['range'])
dfm.columns = ["example_" + v for v in dfm.columns]
df = df.merge(dfm,
left_on="example_idx",
how='left',
right_on="example_id")
df.to_csv(output_tsv, sep='\t')
return seqlets, df
def modisco_table(modisco_dir, output_dir, report_url=None, impsf=None):
"""Write the pattern table to as .html and .csv
"""
plt.switch_backend('agg')
from basepair.modisco.table import ModiscoData, modisco_table, write_modisco_table
from basepair.modisco.motif_clustering import hirearchically_reorder_table
add_file_logging(output_dir, logger, 'modisco-table')
print("Loading required data")
data = ModiscoData.load(modisco_dir, imp_scores_h5=None, impsf=impsf)
print("Generating the table")
df = modisco_table(data)
print("Writing the results")
write_modisco_table(df, output_dir, report_url, 'pattern_table')
print("Writing clustered table")
write_modisco_table(hirearchically_reorder_table(df, data.tasks),
output_dir, report_url, 'pattern_table.sorted')
print("Writing footprints")
profiles = OrderedDict([(pattern, {
task: data.get_profile_wide(pattern, task).mean(axis=0)
for task in data.tasks
}) for pattern in data.mr.patterns()])
write_pkl(profiles, Path(output_dir) / 'footprints.pkl')
print("Done!")
def modisco_centroid_seqlet_matches2(modisco_dir,
imp_scores,
output_dir=None,
trim_frac=0.08,
n_jobs=1,
data_class='profile'):
"""Write pattern matches to .csv
"""
from basepair.modisco.table import ModiscoData, ModiscoDataSingleBinary
if (output_dir is None):
output_dir = modisco_dir
add_file_logging(output_dir, logger, 'centroid-seqlet-matches')
logger.info("Loading required data")
if (data_class == 'profile'):
data = ModiscoData.load(modisco_dir, imp_scores)
else:
data = ModiscoDataSingleBinary.load(modisco_dir)
logger.info("Generating the table")
df = data.get_centroid_seqlet_matches(trim_frac=trim_frac, n_jobs=n_jobs)
df.to_csv(os.path.join(output_dir, 'centroid_seqlet_matches.csv'))
return None
def modisco2bed(modisco_dir, output_dir, trim_frac=0.08):
from pybedtools import Interval
from basepair.modisco.results import ModiscoResult
add_file_logging(output_dir, logger, 'modisco2bed')
ranges = load_ranges(modisco_dir)
example_intervals = [
Interval(row.chrom, row.start, row.end)
for i, row in ranges.iterrows()
]
r = ModiscoResult(os.path.join(modisco_dir, "modisco.h5"))
r.export_seqlets_bed(output_dir,
example_intervals=example_intervals,
position='absolute',
trim_frac=trim_frac)
r.close()
def modisco_centroid_seqlet_matches(modisco_dir,
imp_scores,
output_dir,
trim_frac=0.08,
n_jobs=1,
impsf=None):
"""Write pattern matches to .csv
"""
from basepair.modisco.table import ModiscoData
assert os.path.exists(output_dir)
add_file_logging(output_dir, logger, 'centroid-seqlet-matches')
logger.info("Loading required data")
data = ModiscoData.load(modisco_dir, imp_scores, impsf=impsf)
logger.info("Generating the table")
df = data.get_centroid_seqlet_matches(trim_frac=trim_frac, n_jobs=n_jobs)
df.to_csv(os.path.join(output_dir, 'centroid_seqlet_matches.csv'))
def imp_score(model_dir,
output_file,
method="grad",
split='all',
batch_size=512,
num_workers=10,
h5_chunk_size=512,
max_batches=-1,
shuffle_seq=False,
memfrac=0.45,
exclude_chr='',
overwrite=False,
gpu=None):
"""Run importance scores for a BPNet model
Args:
model_dir: path to the model directory
output_file: output file path (HDF5 format)
method: which importance scoring method to use ('grad', 'deeplift' or 'ism')
split: for which dataset split to compute the importance scores
h5_chunk_size: hdf5 chunk size.
exclude_chr: comma-separated list of chromosomes to exclude
overwrite: if True, overwrite the output directory
gpu (int): which GPU to use locally. If None, GPU is not used
"""
add_file_logging(os.path.dirname(output_file), logger, 'modisco-score')
if gpu is not None:
create_tf_session(gpu, per_process_gpu_memory_fraction=memfrac)
else:
# Don't use any GPU's
os.environ['CUDA_VISIBLE_DEVICES'] = ''
if os.path.exists(output_file):
if overwrite:
os.remove(output_file)
else:
raise ValueError(f"File exists {output_file}. Use overwrite=True to overwrite it")
if exclude_chr:
exclude_chr = exclude_chr.split(",")
else:
exclude_chr = []
# load the config files
logger.info("Loading the config files")
model_dir = Path(model_dir)
hp = HParams.load(model_dir / "hparams.yaml")
ds = DataSpec.load(model_dir / "dataspec.yaml")
tasks = list(ds.task_specs)
# validate that the correct dataset was used
if hp.data.name != 'get_StrandedProfile_datasets':
logger.warn("hp.data.name != 'get_StrandedProfile_datasets'")
if split == 'valid':
assert len(exclude_chr) == 0
incl_chromosomes = hp.data.kwargs['valid_chr']
excl_chromosomes = None
elif split == 'test':
assert len(exclude_chr) == 0
incl_chromosomes = hp.data.kwargs['test_chr']
excl_chromosomes = None
elif split == 'train':
assert len(exclude_chr) == 0
incl_chromosomes = None
excl_chromosomes = hp.data.kwargs['valid_chr'] + hp.data.kwargs['test_chr'] + hp.data.kwargs.get('exclude_chr', [])
elif split == 'all':
incl_chromosomes = None
excl_chromosomes = hp.data.kwargs.get('exclude_chr', []) + exclude_chr
logger.info("Excluding chromosomes: {excl_chromosomes}")
else:
raise ValueError("split needs to be from {train,valid,test,all}")
logger.info("Creating the dataset")
from basepair.datasets import StrandedProfile
seq_len = hp.data.kwargs['peak_width']
dl_valid = StrandedProfile(ds,
incl_chromosomes=incl_chromosomes,
excl_chromosomes=excl_chromosomes,
peak_width=seq_len,
shuffle=False,
target_transformer=None)
bpnet = BPNet.from_mdir(model_dir)
writer = HDF5BatchWriter(output_file, chunk_size=h5_chunk_size)
for i, batch in enumerate(tqdm(dl_valid.batch_iter(batch_size=batch_size, num_workers=num_workers))):
if max_batches > 0:
logging.info(f"max_batches: {max_batches} exceeded. Stopping the computation")
if i > max_batches:
break
# append the bias model predictions
# (batch['inputs'], batch['targets']) = bm((batch['inputs'], batch['targets']))
# store the original batch containing 'inputs' and 'targets'
wdict = batch
if shuffle_seq:
# Di-nucleotide shuffle the sequences
if 'seq' in batch['inputs']:
batch['inputs']['seq'] = onehot_dinucl_shuffle(batch['inputs']['seq'])
else:
batch['inputs'] = onehot_dinucl_shuffle(batch['inputs'])
# loop through all tasks, pred_summary and strands
for task_i, task in enumerate(tasks):
for pred_summary in ['count', 'weighted']:
# figure out the number of channels
nstrands = batch['targets'][f'profile/{task}'].shape[-1]
strand_hash = ["pos", "neg"]
for strand_i in range(nstrands):
hyp_imp = bpnet.imp_score(batch['inputs'],
task=task,
strand=strand_hash[strand_i],
method=method,
pred_summary=pred_summary,
batch_size=None) # don't second-batch
# put importance scores to the dictionary
wdict[f"/hyp_imp/{task}/{pred_summary}/{strand_i}"] = hyp_imp
writer.batch_write(wdict)
writer.close()
def imp_score_seqmodel(model_dir,
output_file,
dataspec=None,
peak_width=1000,
seq_width=None,
intp_pattern='*', # specifies which imp. scores to compute
# skip_trim=False, # skip trimming the output
method="deeplift",
batch_size=512,
max_batches=-1,
shuffle_seq=False,
memfrac=0.45,
num_workers=10,
h5_chunk_size=512,
exclude_chr='',
include_chr='',
overwrite=False,
skip_bias=False,
gpu=None):
"""Run importance scores for a BPNet model
Args:
model_dir: path to the model directory
output_file: output file path (HDF5 format)
method: which importance scoring method to use ('grad', 'deeplift' or 'ism')
split: for which dataset split to compute the importance scores
h5_chunk_size: hdf5 chunk size.
exclude_chr: comma-separated list of chromosomes to exclude
overwrite: if True, overwrite the output directory
skip_bias: if True, don't store the bias tracks in teh output
gpu (int): which GPU to use locally. If None, GPU is not used
"""
add_file_logging(os.path.dirname(output_file), logger, 'modisco-score-seqmodel')
if gpu is not None:
create_tf_session(gpu, per_process_gpu_memory_fraction=memfrac)
else:
# Don't use any GPU's
os.environ['CUDA_VISIBLE_DEVICES'] = ''
if os.path.exists(output_file):
if overwrite:
os.remove(output_file)
else:
raise ValueError(f"File exists {output_file}. Use overwrite=True to overwrite it")
if seq_width is None:
logger.info("Using seq_width = peak_width")
seq_width = peak_width
# make sure these are int's
seq_width = int(seq_width)
peak_width = int(peak_width)
# Split
intp_patterns = intp_pattern.split(",")
# Allow chr inclusion / exclusion
if exclude_chr:
exclude_chr = exclude_chr.split(",")
else:
exclude_chr = None
if include_chr:
include_chr = include_chr.split(",")
else:
include_chr = None
logger.info("Loading the config files")
model_dir = Path(model_dir)
if dataspec is None:
# Specify dataspec
dataspec = model_dir / "dataspec.yaml"
ds = DataSpec.load(dataspec)
logger.info("Creating the dataset")
from basepair.datasets import StrandedProfile
dl_valid = StrandedProfile(ds,
incl_chromosomes=include_chr,
excl_chromosomes=exclude_chr,
peak_width=peak_width,
seq_width=seq_width,
shuffle=False,
taskname_first=True, # Required to work nicely with imp-score
target_transformer=None)
# Setup importance score trimming
if seq_width > peak_width:
# Trim
# make sure we can nicely trim the peak
logger.info("Trimming the output")
assert (seq_width - peak_width) % 2 == 0
trim_start = (seq_width - peak_width) // 2
trim_end = seq_width - trim_start
assert trim_end - trim_start == peak_width
elif seq_width == peak_width:
trim_start = 0
trim_end = peak_width
else:
raise ValueError("seq_width < peak_width")
seqmodel = SeqModel.from_mdir(model_dir)
# get all possible interpretation names
# make sure they match the specified glob
intp_names = [name for name, _ in seqmodel.get_intp_tensors(preact_only=False)
if fnmatch_any(name, intp_patterns)]
logger.info(f"Using the following interpretation targets:")
for n in intp_names:
print(n)
writer = HDF5BatchWriter(output_file, chunk_size=h5_chunk_size)
for i, batch in enumerate(tqdm(dl_valid.batch_iter(batch_size=batch_size, num_workers=num_workers))):
# store the original batch containing 'inputs' and 'targets'
wdict = batch
if skip_bias:
wdict['inputs'] = {'seq': wdict['inputs']['seq']} # ignore all other inputs
if max_batches > 0:
logging.info(f"max_batches: {max_batches} exceeded. Stopping the computation")
if i > max_batches:
break
if shuffle_seq:
# Di-nucleotide shuffle the sequences
batch['inputs']['seq'] = onehot_dinucl_shuffle(batch['inputs']['seq'])
for name in intp_names:
hyp_imp = seqmodel.imp_score(batch['inputs']['seq'],
name=name,
method=method,
batch_size=None) # don't second-batch
# put importance scores to the dictionary
# also trim the importance scores appropriately so that
# the output will always be w.r.t. the peak center
wdict[f"/hyp_imp/{name}"] = hyp_imp[:, trim_start:trim_end]
# trim the sequence as well
if isinstance(wdict['inputs'], dict):
# Trim the sequence
wdict['inputs']['seq'] = wdict['inputs']['seq'][:, trim_start:trim_end]
else:
wdict['inputs'] = wdict['inputs'][:, trim_start:trim_end]
writer.batch_write(wdict)
writer.close()
def modisco_instances_to_bed(modisco_h5,
instances_parq,
imp_score_h5,
output_dir,
trim_frac=0.08):
from basepair.modisco.pattern_instances import load_instances
add_file_logging(output_dir, logger, 'modisco-instances-to-bed')
output_dir = Path(output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
mr = ModiscoResult(modisco_h5)
mr.open()
print("load task_id")
d = HDF5Reader(imp_score_h5)
d.open()
if 'hyp_imp' not in d.f.keys():
# backcompatibility
d['hyp_imp'] = d['grads']
id_hash = pd.DataFrame({
"peak_id":
d.f['/metadata/interval_from_task'][:],
"example_idx":
np.arange(d.f['/metadata/interval_from_task'].shape[0])
})
# load the instances data frame
print("load all instances")
df = load_instances(instances_parq, motifs=None, dedup=True)
# import pdb
# pdb.set_trace()
df = df.merge(id_hash, on="example_idx") # append peak_id
patterns = df.pattern.unique().tolist()
pattern_pssms = {
pattern: mr.get_pssm(*pattern.split("/"))
for pattern in patterns
}
append_pattern_loc(df, pattern_pssms, trim_frac=trim_frac)
# write out the results
example_cols = [
'example_chr', 'example_start', 'example_end', 'example_id', 'peak_id'
]
df_examples = df[example_cols].drop_duplicates().sort_values(
["example_chr", "example_start"])
df_examples.to_csv(output_dir / "scored_regions.bed",
sep='\t',
header=False,
index=False)
df["pattern_start_rel"] = df.pattern_start + df.example_start
df["pattern_end_rel"] = df.pattern_end + df.example_start
df["strand"] = df.revcomp.astype(bool).map({True: "-", False: "+"})
# TODO - update this - ?
pattern_cols = [
'example_chr', 'pattern_start_rel', 'pattern_end_rel', 'example_id',
'percnormed_score', 'strand', 'peak_id', 'seqlet_score'
]
(output_dir /
"README").write_text("score_regions.bed columns: " +
", ".join(example_cols) + "\n" +
"metacluster_<>/pattern_<>.bed columns: " +
", ".join(pattern_cols))
df_pattern = df[pattern_cols]
for pattern in df.pattern.unique():
out_path = output_dir / (pattern + ".bed.gz")
out_path.parent.mkdir(parents=True, exist_ok=True)
dfp = df_pattern[df.pattern == pattern].drop_duplicates().sort_values(
["example_chr", "pattern_start_rel"])
dfp.to_csv(out_path,
compression='gzip',
sep='\t',
header=False,
index=False)
def modisco_score2(modisco_dir,
output_file,
trim_frac=0.08,
imp_scores=None,
importance=None,
ignore_filter=False,
n_jobs=20):
"""Modisco score instances
Args:
modisco_dir: modisco directory - used to obtain centroid_seqlet_matches.csv and modisco.h5
output_file: output file path for the tsv file. If the suffix is
tsv.gz, then also gzip the file
trim_frac: how much to trim the pattern when scanning
imp_scores: hdf5 file of importance scores (contains `importance` score)
if None, then load the default importance scores from modisco
importance: which importance scores to use
n_jobs: number of parallel jobs to use
Writes a gzipped tsv file(tsv.gz)
"""
add_file_logging(os.path.dirname(output_file), logger, 'modisco-score2')
modisco_dir = Path(modisco_dir)
modisco_kwargs = read_json(f"{modisco_dir}/kwargs.json")
if importance is None:
importance = modisco_kwargs['grad_type']
# Centroid matches
cm_path = modisco_dir / 'centroid_seqlet_matches.csv'
if not cm_path.exists():
logger.info(f"Generating centroid matches to {cm_path.resolve()}")
modisco_centroid_seqlet_matches(modisco_dir,
imp_scores,
modisco_dir,
trim_frac=trim_frac,
n_jobs=n_jobs)
logger.info(f"Loading centroid matches from {cm_path.resolve()}")
dfm_norm = pd.read_csv(cm_path)
mr = ModiscoResult(modisco_dir / "modisco.h5")
mr.open()
tasks = mr.tasks()
# HACK prune the tasks of importance (in case it's present)
tasks = [t.replace(f"/{importance}", "") for t in tasks]
logger.info(f"Using tasks: {tasks}")
if imp_scores is not None:
logger.info(f"Loading the importance scores from: {imp_scores}")
imp = ImpScoreFile(imp_scores, default_imp_score=importance)
else:
imp = ImpScoreFile.from_modisco_dir(
modisco_dir, ignore_include_samples=ignore_filter)
seq, contrib, hyp_contrib, profile, ranges = imp.get_all()
logger.info("Scanning for patterns")
dfl = []
for pattern_name in tqdm(mr.patterns()):
pattern = mr.get_pattern(pattern_name).trim_seq_ic(trim_frac)
match, importance = pattern.scan_importance(contrib,
hyp_contrib,
tasks,
n_jobs=n_jobs,
verbose=False)
seq_match = pattern.scan_seq(seq, n_jobs=n_jobs, verbose=False)
dfm = pattern.get_instances(
tasks,
match,
importance,
seq_match,
norm_df=dfm_norm[dfm_norm.pattern == pattern_name],
verbose=False,
plot=False)
dfl.append(dfm)
logger.info("Merging")
# merge and write the results
dfp = pd.concat(dfl)
# append the ranges
logger.info("Append ranges")
ranges.columns = ["example_" + v for v in ranges.columns]
dfp = dfp.merge(ranges, on="example_idx", how='left')
logger.info("Table info")
dfp.info()
logger.info(
f"Writing the resuling pd.DataFrame of shape {dfp.shape} to {output_file}"
)
# write to a parquet file
dfp.to_parquet(output_file, partition_on=['pattern'], engine='fastparquet')
logger.info("Done!")
def modisco_plot(
modisco_dir,
output_dir,
# filter_npy=None,
# ignore_dist_filter=False,
figsize=(10, 10),
impsf=None):
"""Plot the results of a modisco run
Args:
modisco_dir: modisco directory
output_dir: Output directory for writing the results
figsize: Output figure size
impsf: [optional] modisco importance score file (ImpScoreFile)
"""
plt.switch_backend('agg')
add_file_logging(output_dir, logger, 'modisco-plot')
from basepair.plot.vdom import write_heatmap_pngs
from basepair.plot.profiles import plot_profiles
from basepair.utils import flatten
output_dir = Path(output_dir)
output_dir.parent.mkdir(parents=True, exist_ok=True)
# load modisco
mr = ModiscoResult(f"{modisco_dir}/modisco.h5")
if impsf is not None:
d = impsf
else:
d = ImpScoreFile.from_modisco_dir(modisco_dir)
logger.info("Loading the importance scores")
d.cache() # load all
thr_one_hot = d.get_seq()
# thr_hypothetical_contribs
tracks = d.get_profiles()
thr_hypothetical_contribs = dict()
thr_contrib_scores = dict()
# TODO - generalize this
thr_hypothetical_contribs['weighted'] = d.get_hyp_contrib()
thr_contrib_scores['weighted'] = d.get_contrib()
tasks = d.get_tasks()
# Count importance (if it exists)
if d.contains_imp_score("counts/pre-act"):
count_imp_score = "counts/pre-act"
thr_hypothetical_contribs['count'] = d.get_hyp_contrib(
imp_score=count_imp_score)
thr_contrib_scores['count'] = d.get_contrib(imp_score=count_imp_score)
elif d.contains_imp_score("count"):
count_imp_score = "count"
thr_hypothetical_contribs['count'] = d.get_hyp_contrib(
imp_score=count_imp_score)
thr_contrib_scores['count'] = d.get_contrib(imp_score=count_imp_score)
else:
# Don't do anything
pass
thr_hypothetical_contribs = OrderedDict(
flatten(thr_hypothetical_contribs, separator='/'))
thr_contrib_scores = OrderedDict(flatten(thr_contrib_scores,
separator='/'))
# # load importance scores
# modisco_kwargs = read_json(f"{modisco_dir}/kwargs.json")
# d = HDF5Reader.load(modisco_kwargs['imp_scores'])
# if 'hyp_imp' not in d:
# # backcompatibility
# d['hyp_imp'] = d['grads']
# tasks = list(d['targets']['profile'])
# if isinstance(d['inputs'], dict):
# one_hot = d['inputs']['seq']
# else:
# one_hot = d['inputs']
# # load used strand distance filter
# included_samples = load_included_samples(modisco_dir)
# grad_type = "count,weighted" # always plot both importance scores
# thr_hypothetical_contribs = OrderedDict([(f"{gt}/{task}", mean(d['hyp_imp'][task][gt])[included_samples])
# for task in tasks
# for gt in grad_type.split(",")])
# thr_one_hot = one_hot[included_samples]
# thr_contrib_scores = OrderedDict([(f"{gt}/{task}", thr_hypothetical_contribs[f"{gt}/{task}"] * thr_one_hot)
# for task in tasks
# for gt in grad_type.split(",")])
# tracks = OrderedDict([(task, d['targets']['profile'][task][included_samples]) for task in tasks])
# -------------------------------------------------
all_seqlets = mr.seqlets()
all_patterns = mr.patterns()
if len(all_patterns) == 0:
print("No patterns found")
return
# 1. Plots with tracks and contrib scores
print("Writing results for contribution scores")
plot_profiles(all_seqlets,
thr_one_hot,
tracks=tracks,
importance_scores=thr_contrib_scores,
legend=False,
flip_neg=True,
rotate_y=0,
seq_height=.5,
patterns=all_patterns,
n_bootstrap=100,
fpath_template=str(output_dir /
"{pattern}/agg_profile_contribcores"),
mkdir=True,
figsize=figsize)
# 2. Plots only with hypothetical contrib scores
print("Writing results for hypothetical contribution scores")
plot_profiles(all_seqlets,
thr_one_hot,
tracks={},
importance_scores=thr_hypothetical_contribs,
legend=False,
flip_neg=True,
rotate_y=0,
seq_height=1,
patterns=all_patterns,
n_bootstrap=100,
fpath_template=str(output_dir /
"{pattern}/agg_profile_hypcontribscores"),
figsize=figsize)
print("Plotting heatmaps")
for pattern in tqdm(all_patterns):
write_heatmap_pngs(all_seqlets[pattern],
d,
tasks,
pattern,
output_dir=str(output_dir / pattern))
mr.close()
def modisco_run(
imp_scores,
output_dir,
null_imp_scores=None,
hparams=None,
override_hparams="",
grad_type="weighted",
subset_tasks=None,
filter_subset_tasks=False,
filter_npy=None,
exclude_chr="",
seqmodel=False, # interpretation glob
# hparams=None,
num_workers=10,
max_strand_distance=0.1,
overwrite=False,
skip_dist_filter=False,
use_all_seqlets=False,
merge_tasks=False,
gpu=None,
):
"""
Run modisco
Args:
imp_scores: path to the hdf5 file of importance scores
null_imp_scores: Path to the null importance scores
grad_type: for which output to compute the importance scores
hparams: None, modisco hyper - parameeters: either a path to modisco.yaml or
a ModiscoHParams object
override_hparams: hyper - parameters overriding the settings in the hparams file
output_dir: output file directory
filter_npy: path to a npy file containing a boolean vector used for subsetting
exclude_chr: comma-separated list of chromosomes to exclude
seqmodel: If enabled, then the importance scores came from `imp-score-seqmodel`
subset_tasks: comma-separated list of task names to use as a subset
filter_subset_tasks: if True, run modisco only in the regions for that TF
hparams: hyper - parameter file
summary: which summary statistic to use for the profile gradients
skip_dist_filter: if True, distances are not used to filter
use_all_seqlets: if True, don't restrict the number of seqlets
split: On which data split to compute the results
merge_task: if True, importance scores for the tasks will be merged
gpu: which gpu to use. If None, don't use any GPU's
Note: when using subset_tasks, modisco will run on all the importance scores. If you wish
to run it only for the importance scores for a particular task you should subset it to
the peak regions of interest using `filter_npy`
"""
plt.switch_backend('agg')
add_file_logging(output_dir, logger, 'modisco-run')
import os
if gpu is not None:
create_tf_session(gpu)
else:
# Don't use any GPU's
os.environ['CUDA_VISIBLE_DEVICES'] = ''
os.environ['MKL_THREADING_LAYER'] = 'GNU'
# import theano
import modisco
import modisco.tfmodisco_workflow.workflow
if seqmodel:
assert '/' in grad_type
if subset_tasks == '':
logger.warn("subset_tasks == ''. Not using subset_tasks")
subset_tasks = None
if subset_tasks == 'all':
# Use all subset tasks e.g. don't subset
subset_tasks = None
if subset_tasks is not None:
subset_tasks = subset_tasks.split(",")
if len(subset_tasks) == 0:
raise ValueError("Provide one or more subset_tasks. Found None")
if filter_subset_tasks and subset_tasks is None:
print("Using filter_subset_tasks=False since `subset_tasks` is None")
filter_subset_tasks = False
if exclude_chr:
exclude_chr = exclude_chr.split(",")
else:
exclude_chr = []
output_dir = Path(output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
output_path = output_dir / "modisco.h5"
remove_exists(output_path, overwrite)
output_distances = output_dir / "strand_distances.h5"
remove_exists(output_distances, overwrite)
if filter_npy is not None:
filter_npy = os.path.abspath(filter_npy)
# save the hyper-parameters
write_json(
dict(
imp_scores=os.path.abspath(imp_scores),
grad_type=grad_type,
output_dir=str(output_dir),
subset_tasks=subset_tasks,
filter_subset_tasks=filter_subset_tasks,
hparams=hparams,
null_imp_scores=null_imp_scores,
# TODO - pack into hyper-parameters as well?
filter_npy=filter_npy,
exclude_chr=",".join(exclude_chr),
skip_dist_filter=skip_dist_filter,
use_all_seqlets=use_all_seqlets,
max_strand_distance=max_strand_distance,
gpu=gpu),
os.path.join(output_dir, "kwargs.json"))
print("-" * 40)
# parse the hyper-parameters
if hparams is None:
print(f"Using default hyper-parameters")
hp = ModiscoHParams()
else:
if isinstance(hparams, str):
print(f"Loading hyper-parameters from file: {hparams}")
hp = ModiscoHParams.load(hparams)
else:
assert isinstance(hparams, ModiscoHParams)
hp = hparams
if override_hparams:
print(f"Overriding the following hyper-parameters: {override_hparams}")
hp = tf.contrib.training.HParams(
**hp.get_modisco_kwargs()).parse(override_hparams)
if use_all_seqlets:
hp.max_seqlets_per_metacluster = None
# save the hyper-parameters
print("Using the following hyper-parameters for modisco:")
print("-" * 40)
related_dump_yaml(ModiscoHParams(**hp.values()),
os.path.join(output_dir, "hparams.yaml"),
verbose=True)
print("-" * 40)
# TODO - replace with imp_scores
d = HDF5Reader.load(imp_scores)
if 'hyp_imp' not in d:
# backcompatibility
d['hyp_imp'] = d['grads']
if seqmodel:
tasks = list(d['targets'])
else:
tasks = list(d['targets']['profile'])
if subset_tasks is not None:
# validate that all the `subset_tasks`
# are present in `tasks`
for st in subset_tasks:
if st not in tasks:
raise ValueError(
f"subset task {st} not found in tasks: {tasks}")
logger.info(
f"Using the following tasks: {subset_tasks} instead of the original tasks: {tasks}"
)
tasks = subset_tasks
if isinstance(d['inputs'], dict):
one_hot = d['inputs']['seq']
else:
one_hot = d['inputs']
n = len(one_hot)
# --------------------
# apply filters
if not skip_dist_filter:
print("Using profile prediction for the strand filtering")
grad_type_filtered = 'weighted'
distances = np.array([
np.array([
correlation(
np.ravel(d['hyp_imp'][task][grad_type_filtered][0][i]),
np.ravel(d['hyp_imp'][task][grad_type_filtered][1][i]))
for i in range(n)
]) for task in tasks
if len(d['hyp_imp'][task][grad_type_filtered]) == 2
]).T.mean(axis=-1) # average the distances across tasks
dist_filter = distances < max_strand_distance
print(f"Fraction of sequences kept: {dist_filter.mean()}")
HDF5BatchWriter.dump(output_distances, {
"distances": distances,
"included_samples": dist_filter
})
else:
dist_filter = np.ones((n, ), dtype=bool)
# add also the filter numpy
if filter_npy is not None:
print(f"Loading a filter file from {filter_npy}")
filter_vec = np.load(filter_npy)
dist_filter = dist_filter & filter_vec
if filter_subset_tasks:
assert subset_tasks is not None
interval_from_task = pd.Series(d['metadata']['interval_from_task'])
print(
f"Subsetting the intervals accoring to subset_tasks: {subset_tasks}"
)
print(f"Number of original regions: {dist_filter.sum()}")
dist_filter = dist_filter & interval_from_task.isin(
subset_tasks).values
print(
f"Number of filtered regions after filter_subset_tasks: {dist_filter.sum()}"
)
# filter by chromosome
if exclude_chr:
logger.info(f"Excluding chromosomes: {exclude_chr}")
chromosomes = d['metadata']['range']['chr']
dist_filter = dist_filter & (
~pd.Series(chromosomes).isin(exclude_chr)).values
# -------------------------------------------------------------
# setup importance scores
if seqmodel:
thr_one_hot = one_hot[dist_filter]
thr_hypothetical_contribs = {
f"{task}/{gt}":
d['hyp_imp'][task][gt.split("/")[0]][gt.split("/")[1]][dist_filter]
for task in tasks for gt in grad_type.split(",")
}
thr_contrib_scores = {
f"{task}/{gt}":
thr_hypothetical_contribs[f"{task}/{gt}"] * thr_one_hot
for task in tasks for gt in grad_type.split(",")
}
task_names = [
f"{task}/{gt}" for task in tasks for gt in grad_type.split(",")
]
else:
if merge_tasks:
thr_one_hot = np.concatenate([
one_hot[dist_filter] for task in tasks
for gt in grad_type.split(",")
])
thr_hypothetical_contribs = {
"merged":
np.concatenate([
mean(d['hyp_imp'][task][gt])[dist_filter] for task in tasks
for gt in grad_type.split(",")
])
}
thr_contrib_scores = {
"merged": thr_hypothetical_contribs['merged'] * thr_one_hot
}
task_names = ['merged']
else:
thr_one_hot = one_hot[dist_filter]
thr_hypothetical_contribs = {
f"{task}/{gt}": mean(d['hyp_imp'][task][gt])[dist_filter]
for task in tasks for gt in grad_type.split(",")
}
thr_contrib_scores = {
f"{task}/{gt}":
thr_hypothetical_contribs[f"{task}/{gt}"] * thr_one_hot
for task in tasks for gt in grad_type.split(",")
}
task_names = [
f"{task}/{gt}" for task in tasks for gt in grad_type.split(",")
]
if null_imp_scores is not None:
logger.info(f"Using null_imp_scores: {null_imp_scores}")
null_isf = ImpScoreFile(null_imp_scores)
null_per_pos_scores = {
f"{task}/{gt}": v.sum(axis=-1)
for gt in grad_type.split(",")
for task, v in null_isf.get_contrib(imp_score=gt).items()
if task in tasks
}
else:
# default Null distribution. Requires modisco 5.0
logger.info(f"Using default null_imp_scores")
null_per_pos_scores = modisco.coordproducers.LaplaceNullDist(
num_to_samp=10000)
# -------------------------------------------------------------
# run modisco
tfmodisco_results = modisco.tfmodisco_workflow.workflow.TfModiscoWorkflow(
# Modisco defaults
sliding_window_size=hp.sliding_window_size,
flank_size=hp.flank_size,
target_seqlet_fdr=hp.target_seqlet_fdr,
min_passing_windows_frac=hp.min_passing_windows_frac,
max_passing_windows_frac=hp.max_passing_windows_frac,
min_metacluster_size=hp.min_metacluster_size,
max_seqlets_per_metacluster=hp.max_seqlets_per_metacluster,
seqlets_to_patterns_factory=modisco.tfmodisco_workflow.
seqlets_to_patterns.TfModiscoSeqletsToPatternsFactory(
trim_to_window_size=hp.trim_to_window_size, # default: 30
initial_flank_to_add=hp.initial_flank_to_add, # default: 10
kmer_len=hp.kmer_len, # default: 8
num_gaps=hp.num_gaps, # default: 3
num_mismatches=hp.num_mismatches, # default: 2
n_cores=num_workers,
final_min_cluster_size=hp.final_min_cluster_size) # default: 30
)(
task_names=task_names,
contrib_scores=thr_contrib_scores, # -> task score
hypothetical_contribs=thr_hypothetical_contribs,
one_hot=thr_one_hot,
null_per_pos_scores=null_per_pos_scores)
# -------------------------------------------------------------
# save the results
grp = h5py.File(output_path)
tfmodisco_results.save_hdf5(grp)