def create_read_filtering_plots(config_file, config, args):
# get the filtering counts
note = config.get('note', None)
read_filtering_counts = filenames.get_riboseq_read_filtering_counts(
config['riboseq_data'], note=note)
overwrite_str = ""
if args.overwrite:
overwrite_str = "--overwrite"
logging_str = logging_utils.get_logging_options_string(args)
cpus_str = "--num-cpus {}".format(args.num_cpus)
cmd = "get-all-read-filtering-counts {} {} {} {} {}".format(
config_file, read_filtering_counts, overwrite_str, cpus_str,
logging_str)
in_files = [config_file]
out_files = [read_filtering_counts]
shell_utils.call_if_not_exists(cmd,
out_files,
in_files=in_files,
overwrite=args.overwrite,
call=True)
# and visualize them
read_filtering_image = filenames.get_riboseq_read_filtering_counts_image(
config['riboseq_data'], note=note, image_type=args.image_type)
title = "Read filtering counts"
title_str = "--title {}".format(shlex.quote(title))
cmd = "visualize-read-filtering-counts {} {} {} --config {}".format(
read_filtering_counts, read_filtering_image, title_str, config_file)
in_files = [read_filtering_counts]
out_files = [read_filtering_image]
shell_utils.call_if_not_exists(cmd,
out_files,
in_files=in_files,
overwrite=args.overwrite,
call=True)
# and visualize the filtering without the rrna
n = "no-rrna-{}".format(note)
read_filtering_image = filenames.get_riboseq_read_filtering_counts_image(
config['riboseq_data'], note=n, image_type=args.image_type)
title = "Read filtering counts, no ribosomal matches"
title_str = "--title {}".format(shlex.quote(title))
cmd = "visualize-read-filtering-counts {} {} {} --config {} --without-rrna".format(
read_filtering_counts, read_filtering_image, title_str, config_file)
in_files = [read_filtering_counts]
out_files = [read_filtering_image]
shell_utils.call_if_not_exists(cmd,
out_files,
in_files=in_files,
overwrite=args.overwrite,
call=True)
def create_figures(config_file, config, name, offsets_df, args):
""" This function creates all of the figures in the preprocessing report
for the given dataset.
"""
logging_str = logging_utils.get_logging_options_string(args)
note = config.get('note', None)
note_str = filenames.get_note_string(note)
# keep multimappers?
is_unique = not ('keep_riboseq_multimappers' in config)
image_type_str = "--image-type {}".format(args.image_type)
min_read_length = int(offsets_df['length'].min())
max_read_length = int(offsets_df['length'].max())
min_read_length_str = "--min-read-length {}".format(min_read_length)
max_read_length_str = "--max-read-length {}".format(max_read_length)
msg = "{}: Getting and visualizing read length distribution".format(name)
logger.info(msg)
# all aligned reads
genome_bam = filenames.get_riboseq_bam(config['riboseq_data'],
name,
note=note)
# uniquely aligned reads
unique_filename = filenames.get_riboseq_bam(config['riboseq_data'],
name,
is_unique=is_unique,
note=note)
# the read length counts
read_length_distribution = filenames.get_riboseq_read_length_distribution(
config['riboseq_data'], name, note=note)
# the plots
cmd = "get-read-length-distribution {} {} --out {} {}".format(
genome_bam, unique_filename, read_length_distribution, logging_str)
in_files = [genome_bam, unique_filename]
out_files = [read_length_distribution]
shell_utils.call_if_not_exists(cmd,
out_files,
in_files=in_files,
overwrite=args.overwrite,
call=True)
# visualize all read counts
title = None
if 'riboseq_sample_name_map' in config:
title = config['riboseq_sample_name_map'].get(name)
if title is None:
title = "{}{}".format(name, note_str)
title_str = "{}, All aligned reads".format(title)
title_str = "--title={}".format(shlex.quote(title_str))
# get the basename for the distribution file
unique_str = filenames.get_unique_string(False)
sample_name = "{}{}{}".format(name, note_str, unique_str)
read_length_distribution_image = filenames.get_riboseq_read_length_distribution_image(
config['riboseq_data'],
name,
is_unique=False,
note=note,
image_type=args.image_type)
cmd = "plot-read-length-distribution {} {} {} {} {} {}".format(
read_length_distribution, sample_name, read_length_distribution_image,
title_str, min_read_length_str, max_read_length_str)
in_files = [read_length_distribution]
out_files = [read_length_distribution_image]
shell_utils.call_if_not_exists(cmd,
out_files,
in_files=in_files,
overwrite=args.overwrite,
call=True)
# visualize unique read counts
# we already have the title
title_str = "{}, Uniquely aligned reads".format(title)
title_str = "--title={}".format(shlex.quote(title_str))
unique_read_length_distribution_image = filenames.get_riboseq_read_length_distribution_image(
config['riboseq_data'],
name,
is_unique=is_unique,
note=note,
image_type=args.image_type)
# get the basename for the distribution file
unique_str = filenames.get_unique_string(True)
sample_name = "{}{}{}".format(name, note_str, unique_str)
cmd = "plot-read-length-distribution {} {} {} {} {} {}".format(
read_length_distribution, sample_name,
unique_read_length_distribution_image, title_str, min_read_length_str,
max_read_length_str)
in_files = [read_length_distribution]
out_files = [unique_read_length_distribution_image]
shell_utils.call_if_not_exists(cmd,
out_files,
in_files=in_files,
overwrite=args.overwrite,
call=True)
# visualize the metagene profiles
msg = "{}: Visualizing metagene profiles and Bayes' factors".format(name)
logger.info(msg)
metagene_profiles = filenames.get_metagene_profiles(config['riboseq_data'],
name,
is_unique=is_unique,
note=note)
profile_bayes_factor = filenames.get_metagene_profiles_bayes_factors(
config['riboseq_data'], name, is_unique=is_unique, note=note)
mp_df = pd.read_csv(metagene_profiles)
for length in range(min_read_length, max_read_length + 1):
mask_length = offsets_df['length'] == length
# make sure we had some reads of that length
if sum(mask_length) == 0:
continue
length_row = offsets_df[mask_length].iloc[0]
# make sure we have enough reads to visualize
if length_row[
'highest_peak_profile_sum'] < args.min_visualization_count:
continue
# visualize the metagene profile
metagene_profile_image = filenames.get_metagene_profile_image(
config['riboseq_data'],
name,
image_type=args.image_type,
is_unique=is_unique,
length=length,
note=note)
title_str = "{}. length: {}".format(title, length)
title_str = "--title {}".format(shlex.quote(title_str))
cmd = ("create-read-length-metagene-profile-plot {} {} {} {}".format(
metagene_profiles, length, metagene_profile_image, title_str))
in_files = [metagene_profiles]
out_files = [metagene_profile_image]
shell_utils.call_if_not_exists(cmd,
out_files,
in_files=in_files,
overwrite=args.overwrite,
call=True)
# and the Bayes' factor
if args.show_read_length_bfs:
metagene_profile_image = filenames.get_metagene_profile_bayes_factor_image(
config['riboseq_data'],
name,
image_type=args.image_type,
is_unique=is_unique,
length=length,
note=note)
title_str = "Metagene profile Bayes' factors: {}. length: {}".format(
title, length)
title_str = "--title {}".format(shlex.quote(title_str))
fontsize_str = "--font-size 15"
cmd = ("visualize-metagene-profile-bayes-factor {} {} {} {} {}".
format(profile_bayes_factor, length, metagene_profile_image,
title_str, fontsize_str))
in_files = [profile_bayes_factor]
out_files = [metagene_profile_image]
shell_utils.call_if_not_exists(cmd,
out_files,
in_files=in_files,
overwrite=args.overwrite,
call=True)
# the orf-type metagene profiles
if args.show_orf_periodicity:
msg = "{}: Visualizing the ORF type metagene profiles".format(title)
logger.info(msg)
try:
lengths, offsets = ribo_utils.get_periodic_lengths_and_offsets(
config,
name,
is_unique=is_unique,
default_params=metagene_options)
except FileNotFoundError:
msg = ("Could not parse out lengths and offsets for sample: {}. "
"Skipping".format(name))
logger.error(msg)
return
orfs_genomic = filenames.get_orfs(config['genome_base_path'],
config['genome_name'],
note=config.get('orf_note'))
profiles = filenames.get_riboseq_profiles(config['riboseq_data'],
name,
length=lengths,
offset=offsets,
is_unique=is_unique,
note=note)
title_str = "{}, ORF-type periodicity".format(title)
title_str = "--title {}".format(shlex.quote(title_str))
orf_type_profile_base = filenames.get_orf_type_profile_base(
config['riboseq_data'],
name,
length=lengths,
offset=offsets,
is_unique=is_unique,
note=note,
subfolder='orf-profiles')
strand = "+"
orf_type_profiles_forward = [
filenames.get_orf_type_profile_image(orf_type_profile_base,
orf_type, strand,
args.image_type)
for orf_type in ribo_utils.orf_types
]
strand = "-"
orf_type_profiles_reverse = [
filenames.get_orf_type_profile_image(orf_type_profile_base,
orf_type, strand,
args.image_type)
for orf_type in ribo_utils.orf_types
]
cmd = ("visualize-orf-type-metagene-profiles {} {} {} {} {} {}".format(
orfs_genomic, profiles, orf_type_profile_base, title_str,
image_type_str, logging_str))
in_files = [orfs_genomic, profiles]
out_files = orf_type_profiles_forward + orf_type_profiles_reverse
shell_utils.call_if_not_exists(cmd,
out_files,
in_files=in_files,
overwrite=args.overwrite)
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description="""This is a helper script to submit a set of
samples to SLURM. It can also be used to run a set of samples sequentially. Due to limitations
on the config file specification, all of the samples must use the same reference indices
obtained by running 'create-base-genome-profile.""")
parser.add_argument('config', help="The (yaml) configuration file")
parser.add_argument('--tmp', help="The temp directory", default=None)
parser.add_argument('--overwrite',
help="""If this flag is present, existing files
will be overwritten.""",
action='store_true')
parser.add_argument('--profiles-only',
help="""If this flag is present, then only
the pre-processing part of the pipeline will be called, i.e. profiles
will be created for each sample specified in the config file, but no predictions
will be made.""",
action='store_true')
parser.add_argument('--merge-replicates',
help="""If this flag is present, then
the ORF profiles from the replicates will be merged before making the final
predictions""",
action='store_true')
parser.add_argument('--run-replicates',
help="""If this flag is given with the
--merge-replicates flag, then both the replicates and the individual
samples will be run. This flag has no effect if --merge-replicates is not
given.""",
action='store_true')
parser.add_argument('-k',
'--keep-intermediate-files',
help="""If this flag is given,
then all intermediate files will be kept; otherwise, they will be
deleted. This feature is implemented piecemeal. If the --do-not-call flag
is given, then nothing will be deleted.""",
action='store_true')
slurm.add_sbatch_options(parser,
num_cpus=default_num_cpus,
mem=default_mem)
logging_utils.add_logging_options(parser)
pgrm_utils.add_star_options(parser, star_executable)
pgrm_utils.add_flexbar_options(parser)
args = parser.parse_args()
logging_utils.update_logging(args)
config = yaml.load(open(args.config), Loader=yaml.FullLoader)
# check that all of the necessary programs are callable
programs = [
'flexbar', args.star_executable, 'samtools', 'bowtie2',
'create-base-genome-profile', 'remove-multimapping-reads',
'extract-metagene-profiles', 'estimate-metagene-profile-bayes-factors',
'select-periodic-offsets', 'extract-orf-profiles',
'estimate-orf-bayes-factors', 'select-final-prediction-set',
'create-orf-profiles', 'predict-translated-orfs', 'run-rpbp-pipeline'
]
shell_utils.check_programs_exist(programs)
required_keys = [
'riboseq_data', 'riboseq_samples', 'ribosomal_index', 'star_index',
'genome_base_path', 'genome_name', 'fasta', 'gtf'
]
utils.check_keys_exist(config, required_keys)
# handle all option strings to call the pipeline script
logging_str = logging_utils.get_logging_options_string(args)
star_str = pgrm_utils.get_star_options_string(args)
flexbar_str = pgrm_utils.get_flexbar_options_string(args)
# handle do_not_call so that we do call the pipeline script, but that it does not run anything
call = not args.do_not_call
do_not_call_str = ""
if not call:
do_not_call_str = "--do-not-call"
args.do_not_call = False
overwrite_str = ""
if args.overwrite:
overwrite_str = "--overwrite"
mem_str = "--mem {}".format(shlex.quote(args.mem))
keep_intermediate_str = ""
if args.keep_intermediate_files:
keep_intermediate_str = "--keep-intermediate-files"
# check if we only want to create the profiles, in this case
# we call run-rpbp-pipeline with the --profiles-only option
profiles_only_str = ""
if args.profiles_only:
if args.merge_replicates:
msg = (
"The --profiles-only option was given, this option has"
"precedence, and it will override the --merge-replicates option!"
)
logger.warning(msg)
args.merge_replicates = False
profiles_only_str = "--profiles-only"
# if we merge the replicates, then we only use the rpbp script to create
# the ORF profiles, but we still make predictions
if args.merge_replicates and not args.run_replicates:
profiles_only_str = "--profiles-only"
if args.run_replicates and not args.merge_replicates:
msg = (
"The --run-replicates option was given without the --merge-replicates "
"option. It will be ignored.")
logger.warning(msg)
# collect the job_ids in case we are using slurm and need to merge replicates
rep_to_condition = ribo_utils.get_riboseq_replicates_reverse_map(config)
job_ids_mapping = defaultdict(list)
sample_names = sorted(config['riboseq_samples'].keys())
for sample_name in sample_names:
data = config['riboseq_samples'][sample_name]
tmp_str = ""
if args.tmp is not None:
tmp = os.path.join(args.tmp, "{}_rpbp".format(sample_name))
tmp_str = "--tmp {}".format(tmp)
cmd = "run-rpbp-pipeline {} {} {} --num-cpus {} {} {} {} {} {} {} {} {} {}".format(
data, args.config, sample_name, args.num_cpus, mem_str, tmp_str,
do_not_call_str, overwrite_str, profiles_only_str,
keep_intermediate_str, logging_str, star_str, flexbar_str)
job_id = slurm.check_sbatch(cmd, args=args)
job_ids_mapping[rep_to_condition[sample_name]].append(job_id)
# now, if we are running the "standard" pipeline, we are done
if not args.merge_replicates:
return
# otherwise, we need to merge the replicates for each condition
riboseq_replicates = ribo_utils.get_riboseq_replicates(config)
merge_replicates_str = "--merge-replicates"
for condition_name in sorted(riboseq_replicates.keys()):
# then we predict the ORFs
cmd = "predict-translated-orfs {} {} --num-cpus {} {} {} {} {}".format(
args.config, condition_name, args.num_cpus, do_not_call_str,
overwrite_str, logging_str, merge_replicates_str)
job_ids = job_ids_mapping[condition_name]
slurm.check_sbatch(cmd, args=args, dependencies=job_ids)
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description="""This script runs the Rp-Bp pipelines
on a given sample. It requires a YAML config file that includes a number of keys.
Please see the documentation for a complete description.""")
parser.add_argument('raw_data', help="The raw data file (fastq[.gz])")
parser.add_argument('config', help="The (yaml) configuration file")
parser.add_argument(
'name', help="The name for the dataset, used in the created files")
parser.add_argument('--tmp', help="The temp directory", default=None)
parser.add_argument('--overwrite',
help="If this flag is present, existing files "
"will be overwritten.",
action='store_true')
parser.add_argument('--profiles-only',
help="""If this flag is present, then only
the ORF profiles will be created""",
action='store_true')
parser.add_argument('-k',
'--keep-intermediate-files',
help="""If this flag is given,
then all intermediate files will be kept; otherwise, they will be
deleted. This feature is implemented piecemeal. If the --do-not-call flag
is given, then nothing will be deleted.""",
action='store_true')
slurm.add_sbatch_options(parser,
num_cpus=default_num_cpus,
mem=default_mem)
logging_utils.add_logging_options(parser)
pgrm_utils.add_star_options(parser, star_executable)
pgrm_utils.add_flexbar_options(parser)
args = parser.parse_args()
logging_utils.update_logging(args)
config = yaml.load(open(args.config), Loader=yaml.FullLoader)
# check that all of the necessary programs are callable
programs = [
'flexbar', args.star_executable, 'samtools', 'bowtie2',
'create-base-genome-profile', 'remove-multimapping-reads',
'extract-metagene-profiles', 'estimate-metagene-profile-bayes-factors',
'select-periodic-offsets', 'extract-orf-profiles',
'estimate-orf-bayes-factors', 'select-final-prediction-set',
'create-orf-profiles', 'predict-translated-orfs'
]
shell_utils.check_programs_exist(programs)
required_keys = [
'riboseq_data', 'ribosomal_index', 'star_index', 'genome_base_path',
'genome_name', 'fasta', 'gtf'
]
utils.check_keys_exist(config, required_keys)
# if using slurm, submit the script, but we cannot use sys.argv directly
# as the shell strips the quotes around the arguments
if args.use_slurm:
cmd = "{}".format(' '.join("'" + s + "'" if '"' in s else s
for s in sys.argv))
slurm.check_sbatch(cmd, args=args)
return
# handle all option strings to call programs
logging_str = logging_utils.get_logging_options_string(args)
star_str = pgrm_utils.get_star_options_string(args)
flexbar_str = pgrm_utils.get_flexbar_options_string(args)
# handle do_not_call so that we do call the preprocessing script,
# but that it does not run anything
call = not args.do_not_call
do_not_call_str = ""
if not call:
do_not_call_str = "--do-not-call"
overwrite_str = ""
if args.overwrite:
overwrite_str = "--overwrite"
keep_intermediate_str = ""
if args.keep_intermediate_files:
keep_intermediate_str = "--keep-intermediate-files"
tmp_str = ""
if args.tmp is not None:
tmp_str = "--tmp {}".format(shlex.quote(args.tmp))
mem_str = "--mem {}".format(shlex.quote(args.mem))
cmd = ("create-orf-profiles {} {} {} --num-cpus {} {} {} {} {} {} {} {} {}"
.format(args.raw_data, args.config, args.name, args.num_cpus,
mem_str, do_not_call_str, overwrite_str,
keep_intermediate_str, logging_str, tmp_str, star_str,
flexbar_str))
shell_utils.check_call(cmd)
# check if we only want to create the profiles
if args.profiles_only:
return
# then we predict the ORFs
cmd = ("predict-translated-orfs {} {} --num-cpus {} {} {} {}".format(
args.config, args.name, args.num_cpus, do_not_call_str, overwrite_str,
logging_str))
shell_utils.check_call(cmd)
def get_orfs(gtf, args, config, is_annotated=False, is_de_novo=False):
""" Process a GTF file into its ORFs.
"""
call = not args.do_not_call
chr_name_file = os.path.join(config['star_index'], 'chrName.txt')
chr_name_str = "--chr-name-file {}".format(chr_name_file)
logging_str = logging_utils.get_logging_options_string(args)
cpus_str = "--num-cpus {}".format(args.num_cpus)
# extract a BED12 of the annotated ORFs
transcript_bed = filenames.get_bed(config['genome_base_path'],
config['genome_name'],
is_merged=False,
is_annotated=is_annotated,
is_de_novo=is_de_novo)
cmd = ("gtf-to-bed12 {} {} {} {} {}".format(gtf, transcript_bed,
chr_name_str, cpus_str,
logging_str))
in_files = [gtf]
out_files = [transcript_bed]
shell_utils.call_if_not_exists(cmd,
out_files,
in_files=in_files,
overwrite=args.overwrite,
call=call)
# extract the transcript fasta
transcript_fasta = filenames.get_transcript_fasta(
config['genome_base_path'],
config['genome_name'],
is_annotated=is_annotated,
is_de_novo=is_de_novo)
cmd = ("extract-bed-sequences {} {} {} {}".format(transcript_bed,
config['fasta'],
transcript_fasta,
logging_str))
in_files = [transcript_bed, config['fasta']]
out_files = [transcript_fasta]
shell_utils.call_if_not_exists(cmd,
out_files,
in_files=in_files,
overwrite=args.overwrite,
call=call)
# extract ORFs from the transcripts using genomic coordinates
orfs_genomic = filenames.get_orfs(config['genome_base_path'],
config['genome_name'],
note=config.get('orf_note'),
is_annotated=is_annotated,
is_de_novo=is_de_novo)
start_codons_str = utils.get_config_argument(config,
'start_codons',
default=default_start_codons)
stop_codons_str = utils.get_config_argument(config,
'stop_codons',
default=default_stop_codons)
cmd = "extract-orf-coordinates {} {} {} {} {} {} {}".format(
transcript_bed, transcript_fasta, orfs_genomic, cpus_str,
start_codons_str, stop_codons_str, logging_str)
in_files = [transcript_fasta, transcript_bed]
out_files = [orfs_genomic]
shell_utils.call_if_not_exists(cmd,
out_files,
in_files=in_files,
overwrite=args.overwrite,
call=call)
# write the ORF exons, used to label the ORFs
exons_file = filenames.get_exons(config['genome_base_path'],
config['genome_name'],
note=config.get('orf_note'),
is_annotated=is_annotated,
is_de_novo=is_de_novo)
cmd = ("split-bed12-blocks {} {} --num-cpus {} {}".format(
orfs_genomic, exons_file, args.num_cpus, logging_str))
in_files = [orfs_genomic]
out_files = [exons_file]
shell_utils.call_if_not_exists(cmd,
out_files,
in_files=in_files,
overwrite=args.overwrite,
call=call)
# label the ORFs
labeled_orfs = filenames.get_labels(config['genome_base_path'],
config['genome_name'],
note=config.get('orf_note'),
is_annotated=is_annotated,
is_de_novo=is_de_novo)
annotated_bed = filenames.get_bed(config['genome_base_path'],
config['genome_name'],
is_merged=False,
is_annotated=True)
orf_exons_str = '--orf-exons {}'.format(exons_file)
de_novo_str = ""
if is_de_novo:
de_novo_str = '--label-prefix "novel_" --filter --nonoverlapping-label "novel"'
cmd = "label-orfs {} {} {} {} {} {} {}".format(annotated_bed, orfs_genomic,
labeled_orfs, orf_exons_str,
de_novo_str, logging_str,
cpus_str)
in_files = [annotated_bed, orfs_genomic, exons_file]
# ** this function overwrites the input file `orfs_genomic`
out_files = [labeled_orfs]
shell_utils.call_if_not_exists(cmd,
out_files,
in_files=in_files,
overwrite=args.overwrite,
call=call)
def main():
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description="""This script runs all of the processing necessary to
produce the signals used for ORF translation prediction. In particular, it creates the
metagene profiles, selected the periodic fragments and generate the ORF profiles.""")
parser.add_argument('raw_data', help="The raw data file (fastq[.gz])")
parser.add_argument('config', help="The (yaml) configuration file")
parser.add_argument('name', help="The name for the dataset, used in the created files")
parser.add_argument('-p', '--num-cpus', help="The number of processors to use",
type=int, default=default_num_cpus)
parser.add_argument('--mem', help="The amount of RAM to request", default=default_mem)
parser.add_argument('--tmp', help="The location for temp files", default=None)
parser.add_argument('--do-not-call', action='store_true')
parser.add_argument('--overwrite', help="""If this flag is present, existing files
will be overwritten.""", action='store_true')
parser.add_argument('-k', '--keep-intermediate-files', help="""If this flag is given,
then all intermediate files will be kept; otherwise, they will be deleted.
This feature is implemented piecemeal. If the --do-not-call flag is given,
then nothing will be deleted.""", action='store_true')
logging_utils.add_logging_options(parser)
pgrm_utils.add_star_options(parser, star_executable)
pgrm_utils.add_flexbar_options(parser)
args = parser.parse_args()
logging_utils.update_logging(args)
msg = "[create-orf-profiles]: {}".format(' '.join(sys.argv))
logger.info(msg)
config = yaml.load(open(args.config), Loader=yaml.FullLoader)
# check that all of the necessary programs are callable
programs = [
'flexbar',
args.star_executable,
'samtools',
'bowtie2',
'create-base-genome-profile',
'remove-multimapping-reads',
'extract-metagene-profiles',
'estimate-metagene-profile-bayes-factors',
'select-periodic-offsets',
'extract-orf-profiles'
]
shell_utils.check_programs_exist(programs)
required_keys = [
'riboseq_data',
'ribosomal_index',
'gtf',
'genome_base_path',
'genome_name'
]
utils.check_keys_exist(config, required_keys)
note = config.get('note', None)
models_base = config.get('models_base', default_models_base)
logging_str = logging_utils.get_logging_options_string(args)
star_str = pgrm_utils.get_star_options_string(args)
flexbar_str = pgrm_utils.get_flexbar_options_string(args)
# handle do_not_call so that we do call the preprocessing script,
# but that it does not run anything
call = not args.do_not_call
do_not_call_argument = ""
if not call:
do_not_call_argument = "--do-not-call"
overwrite_argument = ""
if args.overwrite:
overwrite_argument = "--overwrite"
keep_intermediate_str = ""
if args.keep_intermediate_files:
keep_intermediate_str = "--keep-intermediate-files"
tmp_str = ""
if args.tmp is not None:
tmp_str = "--tmp {}".format(args.tmp)
mem_str = "--mem {}".format(shlex.quote(args.mem))
# check if we want to keep multimappers
is_unique = not ('keep_riboseq_multimappers' in config)
riboseq_raw_data = args.raw_data
riboseq_bam_filename = filenames.get_riboseq_bam(config['riboseq_data'],
args.name,
is_unique=is_unique,
note=note)
cmd = ("create-base-genome-profile {} {} {} --num-cpus {} {} {} {} {} {} {} {} {}".format(
riboseq_raw_data,
args.config,
args.name,
args.num_cpus,
do_not_call_argument,
overwrite_argument,
logging_str,
star_str,
tmp_str,
flexbar_str,
keep_intermediate_str,
mem_str))
# There could be cases where we start somewhere in the middle of creating
# the base genome profile. So even if the "raw data" is not available,
# we still want to call the base pipeline.
# in_files = [riboseq_raw_data]
in_files = []
out_files = [riboseq_bam_filename]
# we always call this, and pass --do-not-call through
shell_utils.call_if_not_exists(cmd, out_files, in_files=in_files,
overwrite=args.overwrite, call=True)
# Extract the metagene profiles
start_upstream_str = utils.get_config_argument(config,
'metagene_start_upstream',
'start-upstream',
default=metagene_options['metagene_start_upstream'])
start_downstream_str = utils.get_config_argument(config,
'metagene_start_downstream',
'start-downstream',
default=metagene_options['metagene_start_downstream'])
end_upstream_str = utils.get_config_argument(config,
'metagene_end_upstream',
'end-upstream',
default=metagene_options['metagene_end_upstream'])
end_downstream_str = utils.get_config_argument(config,
'metagene_end_downstream',
'end-downstream',
default=metagene_options['metagene_end_downstream'])
metagene_profiles = filenames.get_metagene_profiles(config['riboseq_data'],
args.name,
is_unique=is_unique,
note=note)
# use the canonical transcripts for extracting the metagene profiles
transcript_bed = filenames.get_bed(config['genome_base_path'],
config['genome_name'],
is_merged=False,
is_annotated=True)
cmd = ("extract-metagene-profiles {} {} {} --num-cpus {} {} {} {} {} {}".format(
riboseq_bam_filename,
transcript_bed,
metagene_profiles,
args.num_cpus,
logging_str,
start_upstream_str,
start_downstream_str,
end_upstream_str,
end_downstream_str))
in_files = [riboseq_bam_filename, transcript_bed]
out_files = [metagene_profiles]
file_checkers = {
metagene_profiles: utils.check_gzip_file
}
shell_utils.call_if_not_exists(cmd, out_files, in_files=in_files,
file_checkers=file_checkers,
overwrite=args.overwrite, call=call)
# estimate the periodicity for each offset for all read lengths
metagene_profile_bayes_factors = filenames.get_metagene_profiles_bayes_factors(
config['riboseq_data'],
args.name,
is_unique=is_unique,
note=note)
periodic_models = filenames.get_models(models_base, 'periodic')
non_periodic_models = filenames.get_models(models_base, 'nonperiodic')
periodic_models_str = ' '.join(periodic_models)
non_periodic_models_str = ' '.join(non_periodic_models)
periodic_models_str = "--periodic-models {}".format(periodic_models_str)
non_periodic_models_str = "--nonperiodic-models {}".format(non_periodic_models_str)
periodic_offset_start_str = utils.get_config_argument(config,
'periodic_offset_start',
default=metagene_options['periodic_offset_start'])
periodic_offset_end_str = utils.get_config_argument(config,
'periodic_offset_end',
default=metagene_options['periodic_offset_end'])
metagene_profile_length_str = utils.get_config_argument(config,
'metagene_profile_length',
default=metagene_options['metagene_profile_length'])
seed_str = utils.get_config_argument(config,
'seed',
default=metagene_options['seed'])
chains_str = utils.get_config_argument(config,
'chains',
default=metagene_options['chains'])
iterations_str = utils.get_config_argument(config,
'metagene_iterations',
'iterations',
default=metagene_options['metagene_iterations'])
cmd = ("estimate-metagene-profile-bayes-factors {} {} --num-cpus {} {} {} "
"{} {} {} {} {} {} {}".format(metagene_profiles,
metagene_profile_bayes_factors,
args.num_cpus,
periodic_models_str,
non_periodic_models_str,
periodic_offset_start_str,
periodic_offset_end_str,
metagene_profile_length_str,
seed_str,
chains_str,
iterations_str,
logging_str))
in_files = [metagene_profiles]
in_files.extend(periodic_models)
in_files.extend(non_periodic_models)
out_files = [metagene_profile_bayes_factors]
file_checkers = {
metagene_profile_bayes_factors: utils.check_gzip_file
}
shell_utils.call_if_not_exists(cmd, out_files, in_files=in_files,
file_checkers=file_checkers,
overwrite=args.overwrite, call=call)
# select the best read lengths for constructing the signal
periodic_offsets = filenames.get_periodic_offsets(config['riboseq_data'],
args.name,
is_unique=is_unique,
note=note)
cmd = "select-periodic-offsets {} {}".format(metagene_profile_bayes_factors,
periodic_offsets)
in_files = [metagene_profile_bayes_factors]
out_files = [periodic_offsets]
file_checkers = {
periodic_offsets: utils.check_gzip_file
}
shell_utils.call_if_not_exists(cmd, out_files, in_files=in_files,
file_checkers=file_checkers,
overwrite=args.overwrite, call=call)
# get the lengths and offsets which meet the required criteria from the config file
lengths, offsets = ribo_utils.get_periodic_lengths_and_offsets(config,
args.name,
args.do_not_call,
is_unique=is_unique,
default_params=metagene_options)
if len(lengths) == 0:
msg = ("No periodic read lengths and offsets were found. Try relaxing "
"min_metagene_profile_count, min_metagene_bf_mean, max_metagene_bf_var, "
"and/or min_metagene_bf_likelihood. Quitting.")
logger.critical(msg)
return
lengths_str = ' '.join(lengths)
offsets_str = ' '.join(offsets)
seqname_prefix_str = utils.get_config_argument(config, 'seqname_prefix')
# extract the riboseq profiles for each orf
unique_filename = filenames.get_riboseq_bam(config['riboseq_data'],
args.name,
is_unique=is_unique,
note=note)
profiles_filename = filenames.get_riboseq_profiles(config['riboseq_data'],
args.name,
length=lengths,
offset=offsets,
is_unique=is_unique,
note=note)
orfs_genomic = filenames.get_orfs(config['genome_base_path'],
config['genome_name'],
note=config.get('orf_note'))
exons_file = filenames.get_exons(config['genome_base_path'],
config['genome_name'],
note=config.get('orf_note'))
cmd = ("extract-orf-profiles {} {} {} {} --lengths {} --offsets {} {} {} --num-cpus {} ".format(
unique_filename,
orfs_genomic,
exons_file,
profiles_filename,
lengths_str,
offsets_str,
logging_str,
seqname_prefix_str,
args.num_cpus))
in_files = [orfs_genomic, exons_file, unique_filename]
out_files = [profiles_filename]
# todo: implement a file checker for mtx files
shell_utils.call_if_not_exists(cmd, out_files, in_files=in_files,
overwrite=args.overwrite, call=call)
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description=
"This script identifies the orf peptide matches for all samples in "
"a project.")
parser.add_argument('config', help="The (yaml) config file")
parser.add_argument('--peptide-filter-field',
help="The field to use for "
"filtering the peptides from MaxQuant",
default=default_peptide_filter_field)
parser.add_argument('--peptide-filter-value',
help="All peptides with a value "
"greater than the filter value will be removed",
type=float,
default=default_peptide_filter_value)
parser.add_argument('--peptide-separator',
help="The separator in the "
"peptide file",
default=default_peptide_separator)
parser.add_argument(
'--note',
help="If this option is given, it will be used in "
"the output filenames.\n\nN.B. This REPLACES the note in the config file.",
default=default_note)
slurm.add_sbatch_options(parser)
logging_utils.add_logging_options(parser)
args = parser.parse_args()
logging_utils.update_logging(args)
logging_str = logging_utils.get_logging_options_string(args)
config = yaml.load(open(args.config), Loader=yaml.FullLoader)
call = not args.do_not_call
programs = ['get-orf-peptide-matches']
shell_utils.check_programs_exist(programs)
required_keys = [
'peptide_files', 'peptide_cell_type_analysis', 'riboseq_data',
'riboseq_samples'
]
utils.check_keys_exist(config, required_keys)
note_str = config.get('note', None)
out_note_str = note_str
if args.note is not None and len(args.note) > 0:
out_note_str = args.note
args_dict = vars(args)
peptide_filter_field_str = utils.get_config_argument(
args_dict, 'peptides_filter_field')
peptide_filter_value_str = utils.get_config_argument(
args_dict, 'peptides_filter_value')
peptide_separator_str = utils.get_config_argument(args_dict,
'peptide_separator')
num_cpus_str = utils.get_config_argument(args_dict, 'num_cpus')
cell_types = ribo_utils.get_riboseq_cell_type_samples(config)
for cell_type, peptide_files in config['peptide_cell_type_analysis'].items(
):
if cell_type not in cell_types:
msg = (
"Could not find cell_type specification. Please check the config "
"file: {}".format(cell_type))
logger.warning(msg)
continue
cell_type_protein = ribo_filenames.get_riboseq_cell_type_protein(
config['riboseq_data'], cell_type, is_filtered=True, note=note_str)
if not os.path.exists(cell_type_protein):
msg = ("Could not find cell_type protein fasta. Skipping: {}".
format(cell_type_protein))
logger.warning(msg)
continue
for peptide_file in peptide_files:
if peptide_file not in config['peptide_files']:
msg = (
"Could not find peptide_file specification. Please check "
"the config file: {}".format(peptide_file))
logger.warning(msg)
continue
peptide_txt_file = config['peptide_files'][peptide_file]
if not os.path.exists(peptide_txt_file):
msg = ("Could not find peptide.txt file. Skipping: {}".format(
peptide_txt_file))
logger.warning(msg)
continue
peptide_matches = ribo_filenames.get_riboseq_peptide_matches(
config['riboseq_data'],
cell_type,
peptide_file,
is_filtered=True,
note=out_note_str)
cmd = "get-orf-peptide-matches {} {} {} {} {} {} {} {}".format(
cell_type_protein, peptide_txt_file, peptide_matches,
num_cpus_str, peptide_filter_field_str,
peptide_filter_value_str, peptide_separator_str, logging_str)
slurm.check_sbatch(cmd, args=args)
def main():
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description="Extract the ORF profiles for each specified read length "
"and offset independently, creating one sparse matrix file (mtx) for "
"each read length. These are then collected into a 'sparse tensor'.")
parser.add_argument('config', help="The yaml config file.")
parser.add_argument('name', help="The name of either one of the 'riboseq_samples'"
"or 'riboseq_biological_replicates' from the config file.")
parser.add_argument('out', help="The output (txt.gz) file. N.B. The output uses"
"base-0 indexing, contrary to the unsmoothed ORF profiles, which are written"
"using the matrix market format (base-1 indexing).")
parser.add_argument('-c', '--is-condition', help="If this flag is present, "
"then 'name' will be taken to be a condition name. The profiles for "
"all relevant replicates of the condition will be created.",
action='store_true')
parser.add_argument('--add-ids', help="If this flag is present, "
"then orf_ids will be added to the final output.", action='store_true')
slurm.add_sbatch_options(parser)
logging_utils.add_logging_options(parser)
args = parser.parse_args()
logging_utils.update_logging(args)
logging_str = logging_utils.get_logging_options_string(args)
cpus_str = "--num-cpus {}".format(args.num_cpus)
msg = "[create-read-length-orf-profiles]: {}".format(' '.join(sys.argv))
logger.info(msg)
msg = "Reading config file"
logger.info(msg)
config = yaml.load(open(args.config), Loader=yaml.FullLoader)
# pull out what we need from the config file
is_unique = not ('keep_riboseq_multimappers' in config)
seqname_str = utils.get_config_argument(config, 'seqname_prefix')
note = config.get('note', None)
orf_note = config.get('orf_note', None)
orfs = filenames.get_orfs(
config['genome_base_path'],
config['genome_name'],
note=orf_note
)
exons = filenames.get_exons(
config['genome_base_path'],
config['genome_name'],
note=orf_note,
is_orf=True
)
# make sure the necessary files exist
required_files = [orfs, exons]
msg = "[create-read-length-orf-profiles]: Some input files were missing: "
utils.check_files_exist(required_files, msg=msg)
# process one sample or all samples from condition
names = [args.name]
is_condition_str = ""
if args.is_condition:
is_condition_str = "--is-condition"
riboseq_replicates = ribo_utils.get_riboseq_replicates(config)
names = [n for n in riboseq_replicates[args.name]]
job_ids = []
for name in names:
msg = "Processing sample: {}".format(name)
logger.info(msg)
# now the relevant files
bam = filenames.get_riboseq_bam(
config['riboseq_data'],
name,
is_unique=is_unique,
note=note
)
# make sure the necessary files exist
required_files = [bam]
msg = "[create-read-length-orf-profiles]: Some input files were missing: "
utils.check_files_exist(required_files, msg=msg)
# get the lengths and offsets which meet the required criteria from the config file
lengths, offsets = ribo_utils.get_periodic_lengths_and_offsets(
config,
name,
is_unique=is_unique
)
if len(lengths) == 0:
msg = ("No periodic read lengths and offsets were found. Try relaxing "
"min_metagene_profile_count, min_metagene_bf_mean, "
"max_metagene_bf_var, and/or min_metagene_bf_likelihood. Qutting.")
logger.critical(msg)
return
for length, offset in zip(lengths, offsets):
lengths_str = "--lengths {}".format(length)
offsets_str = "--offsets {}".format(offset)
mtx = filenames.get_riboseq_profiles(
config['riboseq_data'],
name,
length=[length],
offset=[offset],
is_unique=is_unique,
note=note
)
cmd = "extract-orf-profiles {} {} {} {} {} {} {} {}".format(
bam,
orfs,
exons,
mtx,
lengths_str,
offsets_str,
seqname_str,
cpus_str,
logging_str
)
job_id = slurm.check_sbatch(cmd, args=args)
job_ids.append(job_id)
add_ids_str = ""
if args.add_ids:
add_ids_str = "--add-ids"
cmd = "collect-read-length-orf-profiles {} {} {} {} {}".format(
args.config,
args.name,
args.out,
is_condition_str,
add_ids_str,
logging_str
)
slurm.check_sbatch(cmd, args=args, dependencies=job_ids)
def _create_figures(name_pretty_name_is_replicate, config, args):
""" This function creates all of the figures in the prediction report
for the given dataset.
"""
name, pretty_name, is_replicate = name_pretty_name_is_replicate
# keep multimappers?
is_unique = not ('keep_riboseq_multimappers' in config)
# by default, we will not include chisq
chisq_values = [False]
if args.show_chisq:
chisq_values = [True, False]
filtered_values = [True]
if args.show_unfiltered_orfs:
filtered_values = [True, False]
grouped_values = [True, False]
logging_str = logging_utils.get_logging_options_string(args)
note_str = config.get('note', None)
out_note_str = config.get('note', None)
if args.note is not None and len(args.note) > 0:
out_note_str = args.note
image_type_str = "--image-type {}".format(args.image_type)
num_cpus_str = "--num-cpus {}".format(args.num_cpus)
fraction = config.get('smoothing_fraction', None)
reweighting_iterations = config.get('smoothing_reweighting_iterations',
None)
# if this is a replicate, we do not worry about lengths and offsets
if is_replicate:
lengths = None
offsets = None
else:
try:
lengths, offsets = ribo_utils.get_periodic_lengths_and_offsets(
config,
name,
is_unique=is_unique,
default_params=metagene_options)
except FileNotFoundError:
msg = ("Could not parse out lengths and offsets for sample: {}. "
"Skipping".format(name))
logger.error(msg)
return
unsmoothed_profiles = filenames.get_riboseq_profiles(
config['riboseq_data'],
name,
length=lengths,
offset=offsets,
is_unique=is_unique,
note=note_str,
is_smooth=False)
msg = "{}: creating the ORF types bar charts".format(name)
logger.debug(msg)
it = itertools.product(grouped_values, chisq_values, filtered_values)
for is_grouped, is_chisq, is_filtered in it:
is_grouped_str = ""
if is_grouped:
is_grouped_str = ", Grouped"
is_filtered_str = ""
if is_filtered:
is_filtered_str = ", Filtered"
if is_chisq:
title_str = "{}{}{}, Rp-$\chi^2$".format(pretty_name,
is_grouped_str,
is_filtered_str)
title_str = shlex.quote(title_str)
title_str = "--title {}".format(title_str)
f = None
rw = None
orfs = filenames.get_riboseq_predicted_orfs(
config['riboseq_data'],
name,
length=lengths,
offset=offsets,
is_unique=is_unique,
note=note_str,
is_chisq=True,
is_filtered=is_filtered)
else:
title_str = "{}{}{}, Rp-Bp".format(pretty_name, is_grouped_str,
is_filtered_str)
title_str = shlex.quote(title_str)
title_str = "--title {}".format(title_str)
f = fraction
rw = reweighting_iterations
orfs = filenames.get_riboseq_predicted_orfs(
config['riboseq_data'],
name,
length=lengths,
offset=offsets,
is_unique=is_unique,
note=note_str,
fraction=f,
reweighting_iterations=rw,
is_filtered=is_filtered)
use_groups_str = ""
if is_grouped:
use_groups_str = "--use-groups"
orf_types_bar_chart = filenames.get_orf_types_bar_chart(
config['riboseq_data'],
name,
length=lengths,
offset=offsets,
is_unique=is_unique,
note=out_note_str,
image_type=args.image_type,
fraction=f,
reweighting_iterations=rw,
is_grouped=is_grouped,
is_chisq=is_chisq,
is_filtered=is_filtered)
cmd = "create-orf-types-bar-chart {} {} {} {}".format(
orfs, orf_types_bar_chart, title_str, use_groups_str)
in_files = [orfs]
out_files = [orf_types_bar_chart]
shell_utils.call_if_not_exists(cmd,
out_files,
in_files=in_files,
overwrite=args.overwrite)
msg = "{}: creating the ORF length distributions line graph".format(name)
logger.debug(msg)
uniprot_str = ""
uniprot_label_str = ""
if os.path.exists(args.uniprot):
uniprot_str = "--uniprot {}".format(args.uniprot)
uniprot_label_str = shlex.quote(args.uniprot_label)
uniprot_label_str = "--uniprot-label {}".format(uniprot_label_str)
for is_grouped in grouped_values:
for is_chisq in chisq_values:
if is_chisq:
title_str = "{}, Rp-$\chi^2$".format(pretty_name)
title_str = shlex.quote(title_str)
title_str = "--title {}".format(title_str)
f = None
rw = None
orfs = filenames.get_riboseq_predicted_orfs(
config['riboseq_data'],
name,
length=lengths,
offset=offsets,
is_unique=is_unique,
note=note_str,
is_chisq=True)
else:
title_str = "{}, Rp-Bp".format(pretty_name)
title_str = shlex.quote(title_str)
title_str = "--title {}".format(title_str)
f = fraction
rw = reweighting_iterations
orfs = filenames.get_riboseq_predicted_orfs(
config['riboseq_data'],
name,
length=lengths,
offset=offsets,
is_unique=is_unique,
note=note_str,
fraction=f,
reweighting_iterations=rw)
use_groups_str = ""
if is_grouped:
use_groups_str = "--use-groups"
orf_length_line_graph = filenames.get_orf_length_distribution_line_graph(
config['riboseq_data'],
name,
length=lengths,
offset=offsets,
is_unique=is_unique,
note=out_note_str,
image_type=args.image_type,
fraction=f,
reweighting_iterations=rw,
is_grouped=is_grouped,
is_chisq=is_chisq)
cmd = (
"create-orf-length-distribution-line-graph {} {} {} {} {} {}".
format(orfs, orf_length_line_graph, title_str, use_groups_str,
uniprot_str, uniprot_label_str))
in_files = [orfs]
out_files = [orf_length_line_graph]
shell_utils.call_if_not_exists(cmd,
out_files,
in_files=in_files,
overwrite=args.overwrite)
if args.show_orf_periodicity:
msg = "{}: creating the ORF type metagene profiles".format(name)
logger.debug(msg)
for is_chisq in chisq_values:
if is_chisq:
title_str = "{}, Rp-$\chi^2$".format(pretty_name)
title_str = shlex.quote(title_str)
title_str = "--title {}".format(title_str)
f = None
rw = None
is_smooth = False
profiles = unsmoothed_profiles
orfs = filenames.get_riboseq_predicted_orfs(
config['riboseq_data'],
name,
length=lengths,
offset=offsets,
is_unique=is_unique,
note=note_str,
is_chisq=True,
is_filtered=is_filtered)
else:
title_str = "{}, Rp-Bp".format(pretty_name)
title_str = shlex.quote(title_str)
title_str = "--title {}".format(title_str)
f = fraction
rw = reweighting_iterations
is_smooth = False
profiles = unsmoothed_profiles
orfs = filenames.get_riboseq_predicted_orfs(
config['riboseq_data'],
name,
length=lengths,
offset=offsets,
is_unique=is_unique,
note=note_str,
fraction=f,
reweighting_iterations=rw)
orf_type_profile_base = filenames.get_orf_type_profile_base(
config['riboseq_data'],
name,
length=lengths,
offset=offsets,
is_unique=is_unique,
note=out_note_str,
fraction=f,
reweighting_iterations=rw,
is_chisq=is_chisq)
strand = "+"
orf_type_profiles_forward = [
filenames.get_orf_type_profile_image(orf_type_profile_base,
orf_type, strand,
args.image_type)
for orf_type in ribo_utils.orf_types
]
strand = "-"
orf_type_profiles_reverse = [
filenames.get_orf_type_profile_image(orf_type_profile_base,
orf_type, strand,
args.image_type)
for orf_type in ribo_utils.orf_types
]
cmd = ("visualize-orf-type-metagene-profiles {} {} {} {} {} {}".
format(orfs, profiles, orf_type_profile_base, title_str,
image_type_str, logging_str))
in_files = [orfs]
out_files = orf_type_profiles_forward + orf_type_profiles_reverse
shell_utils.call_if_not_exists(cmd,
out_files,
in_files=in_files,
overwrite=args.overwrite)