def find_matching_orfs_group(peptides, orfs):
""" A helper function to call find_matching_orfs on a pd.GroupBy of peptides.
"""
ret = parallel.apply_df_simple(peptides, find_matching_orfs, orfs)
#progress_bar=True)
ret = [r for r in ret if r is not None]
if len(ret) == 0:
return None
return pd.concat(ret)
def find_matching_orfs_group(peptides, orfs):
""" A helper function to call find_matching_orfs on a pd.GroupBy of peptides.
"""
ret = parallel.apply_df_simple(peptides, find_matching_orfs, orfs)
#progress_bar=True)
ret = [r for r in ret if r is not None]
if len(ret) == 0:
return None
return pd.concat(ret)
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description=
"This script extracts the differential micropeptides from two "
"conditions. Please see the documentation in redmine for more details.\n\n"
"Please see the pyensembl (https://github.com/hammerlab/pyensembl) "
"documentation for more information about the ensembl release and species."
)
parser.add_argument('config', help="The (yaml) config file")
parser.add_argument('name_a', help="The name of the first condition")
parser.add_argument('name_b', help="The name of the second condition")
parser.add_argument('out', help="The output (.csv.gz or .xlsx) file")
parser.add_argument(
'-a',
'--append-sheet',
help="If this flag is given, "
"then a worksheet with the name '<name_a>,<name_b>' will be appended "
"to the .xlsx file given by out (if it exists)",
action='store_true')
parser.add_argument(
'-f',
'--filter',
help="If this flag is present, then "
"the output will be filtered to include only the differential "
"micropeptides with the highest KL-divergence and read coverage",
action='store_true')
parser.add_argument(
'--read-filter-percent',
help="If the the --filter flag "
"is given, then only the top --read-filter-percent micropeptides will "
"be considered for the final output. They still must meet the KL-"
"divergence filtering criteria.",
type=float,
default=default_read_filter_percent)
parser.add_argument(
'--kl-filter-percent',
help="If the the --filter flag "
"is given, then only the top --read-kl-percent micropeptides will "
"be considered for the final output. They still must meet the read "
"coverage filtering criteria.",
type=float,
default=default_kl_filter_percent)
parser.add_argument(
'--id-matches',
help="This is a list of files which "
"contain ORF identifiers to compare to the differential micropeptides. "
"For each of the files given, two columns will be added to the output "
"which indicate if either A or B appear in the respective file. Each "
"file should have a single ORF identifier on each line and contain "
"nothing else.",
nargs='*',
default=default_id_matches)
parser.add_argument(
'--id-match-names',
help="A name to include in the "
"output file for each --id-matches file. The number of names must "
"match the number of files.",
nargs='*',
default=default_id_match_names)
parser.add_argument(
'--overlaps',
help="This is a list of bed12+ files "
"which will be compared to the differential micropeptides. Two columns "
"(one for A, one for B) will be added to the output which indicate if "
"the respective micropeptides overlap a feature in each file by at "
"least 1 bp.",
nargs='*',
default=default_overlaps)
parser.add_argument(
'--overlap-names',
help="A name to include in the "
"output file for each --overlaps file. The number of names must match "
"the number of files.",
nargs='*',
default=default_overlap_names)
parser.add_argument(
'-r',
'--ensembl-release',
help="The version of Ensembl "
"to use when mapping transcript identifiers to gene identifiers",
type=int,
default=default_ensembl_release)
parser.add_argument(
'-s',
'--ensembl-species',
help="The Ensembl species "
"to use when mapping transcript identifiers to gene identifiers",
default=default_ensembl_species)
parser.add_argument(
'--a-is-single-sample',
help="By default, this script "
"assumes the predictions come from merged replicates. If name_a is from "
"a single sample, this flag should be given. It is necessary to find "
"the correct filenames.",
action='store_true')
parser.add_argument(
'--b-is-single-sample',
help="By default, this script "
"assumes the predictions come from merged replicates. If name_b is from "
"a single sample, this flag should be given. It is necessary to find "
"the correct filenames.",
action='store_true')
parser.add_argument('--fields-to-keep',
help="The fields to keep from the "
"Bayes factor file for each condition",
nargs='*',
default=default_fields_to_keep)
parser.add_argument('--max-micropeptide-len',
help="The maximum (inclusive) "
"length of ORFs considered as micropeptides",
type=int,
default=default_max_micropeptide_len)
parser.add_argument(
'--do-not-fix-tcons',
help="By default, the \"TCONS_\" "
"identifiers from StringTie, etc., do not parse correctly; this script "
"update the identifiers so that will parse correctly unless instructed not "
"to. The script is likely to crash if the identifiers are not fixed.",
action='store_true')
logging_utils.add_logging_options(parser)
args = parser.parse_args()
logging_utils.update_logging(args)
msg = "Loading ensembl database"
logger.info(msg)
ensembl = pyensembl.EnsemblRelease(release=args.ensembl_release,
species=args.ensembl_species)
ensembl.db
msg = "Checking the id-match and overlaps files"
logger.info(msg)
if len(args.id_matches) != len(args.id_match_names):
msg = ("The number of --id-matches files and --id-match-names do not "
"match. {} files and {} names".format(len(args.id_matches),
len(args.id_match_names)))
raise ValueError(msg)
if len(args.overlaps) != len(args.overlap_names):
msg = ("The number of --overlaps files and --overlaps-names do not "
"match. {} files and {} names".format(len(args.overlaps),
len(args.overlap_names)))
raise ValueError(msg)
utils.check_files_exist(args.id_matches)
utils.check_files_exist(args.overlaps)
if args.filter:
msg = "Validating filter percentages"
logger.info(msg)
math_utils.check_range(args.read_filter_percent,
0,
1,
variable_name="--read-filter-percent")
math_utils.check_range(args.kl_filter_percent,
0,
1,
variable_name="--kl-filter-percent")
msg = "Extracting file names"
logger.info(msg)
config = yaml.load(open(args.config))
note_str = config.get('note', None)
# keep multimappers?
is_unique = not ('keep_riboseq_multimappers' in config)
# and the smoothing parameters
fraction = config.get('smoothing_fraction', None)
reweighting_iterations = config.get('smoothing_reweighting_iterations',
None)
lengths_a = None
offsets_a = None
if args.a_is_single_sample:
lengths_a, offsets_a = ribo_utils.get_periodic_lengths_and_offsets(
config, args.name_a, is_unique=is_unique)
bayes_factors_a = filenames.get_riboseq_bayes_factors(
config['riboseq_data'],
args.name_a,
length=lengths_a,
offset=offsets_a,
is_unique=is_unique,
note=note_str,
fraction=fraction,
reweighting_iterations=reweighting_iterations)
if not os.path.exists(bayes_factors_a):
msg = ("Could not find the Bayes factor file for {}. ({}). Quitting.".
format(args.name_a, bayes_factors_a))
raise FileNotFoundError(msg)
predicted_orfs_a = filenames.get_riboseq_predicted_orfs(
config['riboseq_data'],
args.name_a,
length=lengths_a,
offset=offsets_a,
is_unique=is_unique,
note=note_str,
fraction=fraction,
reweighting_iterations=reweighting_iterations,
is_filtered=True,
is_chisq=False)
if not os.path.exists(predicted_orfs_a):
msg = (
"Could not find the predictions bed file for {}. ({}). Quitting.".
format(args.name_a, predicted_orfs_a))
raise FileNotFoundError(msg)
lengths_b = None
offsets_b = None
if args.b_is_single_sample:
lengths_b, offsets_b = ribo_utils.get_periodic_lengths_and_offsets(
config, args.name_b, is_unique=is_unique)
bayes_factors_b = filenames.get_riboseq_bayes_factors(
config['riboseq_data'],
args.name_b,
length=lengths_b,
offset=offsets_b,
is_unique=is_unique,
note=note_str,
fraction=fraction,
reweighting_iterations=reweighting_iterations)
if not os.path.exists(bayes_factors_b):
msg = ("Could not find the Bayes factor file for {}. ({}). Quitting.".
format(args.name_b, bayes_factors_b))
raise FileNotFoundError(msg)
predicted_orfs_b = filenames.get_riboseq_predicted_orfs(
config['riboseq_data'],
args.name_b,
length=lengths_b,
offset=offsets_b,
is_unique=is_unique,
note=note_str,
fraction=fraction,
reweighting_iterations=reweighting_iterations,
is_filtered=True,
is_chisq=False)
if not os.path.exists(predicted_orfs_b):
msg = (
"Could not find the predictions bed file for {}. ({}). Quitting.".
format(args.name_b, predicted_orfs_b))
raise FileNotFoundError(msg)
exons_file = filenames.get_exons(config['genome_base_path'],
config['genome_name'],
note=config.get('orf_note'))
if not os.path.exists(exons_file):
msg = "Could not find the exons file ({}). Quitting.".format(
exons_file)
raise FileNotFoundError(msg)
msg = "Reading the exons"
logger.info(msg)
exons = bed_utils.read_bed(exons_file)
msg = "Reading the BF files"
logger.info(msg)
bf_df_a = bed_utils.read_bed(bayes_factors_a)
bf_df_b = bed_utils.read_bed(bayes_factors_b)
msg = "Reading the predictions files"
logger.info(msg)
bed_df_a = bed_utils.read_bed(predicted_orfs_a)
bed_df_b = bed_utils.read_bed(predicted_orfs_b)
differential_micropeptide_dfs = []
# extract micropeptides
msg = "Extracting micropeptides"
logger.info(msg)
m_micropeptides_a = bed_df_a['orf_len'] <= args.max_micropeptide_len
m_micropeptides_b = bed_df_b['orf_len'] <= args.max_micropeptide_len
micropeptides_a = bed_df_a[m_micropeptides_a]
micropeptides_b = bed_df_b[m_micropeptides_b]
long_orfs_a = bed_df_a[~m_micropeptides_a]
long_orfs_b = bed_df_b[~m_micropeptides_b]
msg = "Finding micropeptides in A with no overlap in B"
logger.info(msg)
micropeptides_a_no_match_b = bed_utils.subtract_bed(micropeptides_a,
bed_df_b,
exons=exons)
micropeptides_a_no_match_b_df = pd.DataFrame()
micropeptides_a_no_match_b_df['A'] = list(micropeptides_a_no_match_b)
micropeptides_a_no_match_b_df['B'] = None
micropeptides_a_no_match_b_df['kl'] = np.inf
micropeptides_a_no_match_b_df['overlap_type'] = 'micro_a_only'
differential_micropeptide_dfs.append(micropeptides_a_no_match_b_df)
msg = "Finding micropeptides in B with no overlap in A"
logger.info(msg)
micropeptides_b_no_match_a = bed_utils.subtract_bed(micropeptides_b,
bed_df_a,
exons=exons)
micropeptides_b_no_match_a_df = pd.DataFrame()
micropeptides_b_no_match_a_df['B'] = list(micropeptides_b_no_match_a)
micropeptides_b_no_match_a_df['A'] = None
micropeptides_b_no_match_a_df['kl'] = np.inf
micropeptides_b_no_match_a_df['overlap_type'] = 'micro_b_only'
differential_micropeptide_dfs.append(micropeptides_b_no_match_a_df)
msg = "Finding overlapping micropeptides"
logger.info(msg)
micropeptides_a_micropeptides_b_df = get_overlap_df(
micropeptides_a, micropeptides_b, 'micro_a_micro_b', bf_df_a, bf_df_b)
differential_micropeptide_dfs.append(micropeptides_a_micropeptides_b_df)
micropeptides_a_long_b_df = get_overlap_df(micropeptides_a, long_orfs_b,
'micro_a_long_b', bf_df_a,
bf_df_b)
differential_micropeptide_dfs.append(micropeptides_a_long_b_df)
micropeptides_b_long_a_df = get_overlap_df(long_orfs_a, micropeptides_b,
'long_a_micro_b', bf_df_a,
bf_df_b)
differential_micropeptide_dfs.append(micropeptides_b_long_a_df)
differential_micropeptides_df = pd.concat(differential_micropeptide_dfs)
msg = "Adding read count information"
logger.info(msg)
res = differential_micropeptides_df.merge(bf_df_a[args.fields_to_keep],
left_on='A',
right_on='id',
how='left')
to_rename = {f: "{}_A".format(f) for f in args.fields_to_keep}
res = res.rename(columns=to_rename)
res = res.drop('id_A', axis=1)
res = res.merge(bf_df_b[args.fields_to_keep],
left_on='B',
right_on='id',
how='left')
to_rename = {f: "{}_B".format(f) for f in args.fields_to_keep}
res = res.rename(columns=to_rename)
res = res.drop('id_B', axis=1)
id_columns = ['A', 'B']
res = res.drop_duplicates(subset=id_columns)
if not args.do_not_fix_tcons:
# replace TCONS_ with TCONS
res['A'] = res['A'].str.replace("TCONS_", "TCONS")
res['B'] = res['B'].str.replace("TCONS_", "TCONS")
msg = "Extracting the genes and their biotypes using pyensembl"
logger.info(msg)
ensembl = pyensembl.EnsemblRelease(release=args.ensembl_release,
species=args.ensembl_species)
ensembl_transcript_ids = set(ensembl.transcript_ids())
biotypes_a = parallel.apply_df_simple(res, get_transcript_and_biotype, 'A',
ensembl, ensembl_transcript_ids)
biotypes_b = parallel.apply_df_simple(res, get_transcript_and_biotype, 'B',
ensembl, ensembl_transcript_ids)
biotypes_a = utils.remove_nones(biotypes_a)
biotypes_b = utils.remove_nones(biotypes_b)
biotypes_a = pd.DataFrame(biotypes_a)
biotypes_b = pd.DataFrame(biotypes_b)
res = res.merge(biotypes_a, on='A', how='left')
res = res.merge(biotypes_b, on='B', how='left')
msg = "Pulling annotations from mygene.info"
logger.info(msg)
# pull annotations from mygene
gene_info_a = mygene_utils.query_mygene(res['gene_id_A'])
gene_info_b = mygene_utils.query_mygene(res['gene_id_B'])
# and add the mygene info
res = res.merge(gene_info_a,
left_on='gene_id_A',
right_on='gene_id',
how='left')
to_rename = {f: "{}_A".format(f) for f in gene_info_a.columns}
to_rename.pop('gene_id')
res = res.rename(columns=to_rename)
res = res.drop('gene_id', axis=1)
res = res.merge(gene_info_b,
left_on='gene_id_B',
right_on='gene_id',
how='left')
to_rename = {f: "{}_B".format(f) for f in gene_info_a.columns}
to_rename.pop('gene_id')
res = res.rename(columns=to_rename)
res = res.drop('gene_id', axis=1)
msg = "Removing duplicates"
logger.info(msg)
id_columns = ['A', 'B']
res = res.drop_duplicates(subset=id_columns)
msg = "Adding --id-matches columns"
logger.info(msg)
for (id_match_file, name) in zip(args.id_matches, args.id_match_names):
res = add_id_matches(res, id_match_file, name)
msg = "Adding --overlaps columns"
logger.info(msg)
for (overlap_file, name) in zip(args.overlaps, args.overlap_names):
res = add_overlaps(res, overlap_file, name, bed_df_a, bed_df_b, exons)
msg = "Sorting by in-frame reads"
logger.info(msg)
res['x_1_sum_A'] = res['x_1_sum_A'].fillna(0)
res['x_1_sum_B'] = res['x_1_sum_B'].fillna(0)
res['x_1_sum'] = res['x_1_sum_A'] + res['x_1_sum_B']
res = res.sort_values('x_1_sum', ascending=False)
if args.filter:
msg = "Filtering the micropeptides by read coverage and KL-divergence"
logger.info(msg)
x_1_sum_ranks = res['x_1_sum'].rank(method='min',
na_option='top',
ascending=False)
num_x_1_sum_ranks = x_1_sum_ranks.max()
max_good_x_1_sum_rank = num_x_1_sum_ranks * args.read_filter_percent
m_good_x_1_sum_rank = x_1_sum_ranks <= max_good_x_1_sum_rank
msg = ("Number of micropeptides passing read filter: {}".format(
sum(m_good_x_1_sum_rank)))
logger.debug(msg)
kl_ranks = res['kl'].rank(method='dense',
na_option='top',
ascending=False)
num_kl_ranks = kl_ranks.max()
max_good_kl_rank = num_kl_ranks * args.kl_filter_percent
m_good_kl_rank = kl_ranks <= max_good_kl_rank
msg = ("Number of micropeptides passing KL filter: {}".format(
sum(m_good_kl_rank)))
logger.debug(msg)
m_both_filters = m_good_x_1_sum_rank & m_good_kl_rank
msg = ("Number of micropeptides passing both filters: {}".format(
sum(m_both_filters)))
logger.debug(msg)
res = res[m_both_filters]
msg = "Writing differential micropeptides to disk"
logger.info(msg)
if args.append_sheet is None:
utils.write_df(res, args.out, index=False)
else:
sheet_name = "{},{}".format(args.name_a, args.name_b)
utils.append_to_xlsx(res, args.out, sheet=sheet_name, index=False)
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description="Validate the algorithm selection performance of the "
"predictions made using test-as-auto-sklearn using "
"autofolio.validation.validate.Validator.")
parser.add_argument('scenario', help="The ASlib scenario")
parser.add_argument('predictions',
help="The predictions file, from "
"test-as-auto-sklearn")
parser.add_argument('--config',
help="A (yaml) config file which "
"specifies options controlling the learner behavior")
logging_utils.add_logging_options(parser)
args = parser.parse_args()
logging_utils.update_logging(args)
msg = "Loading ASlib scenario"
logger.info(msg)
scenario = ASlibScenario()
scenario.read_scenario(args.scenario)
if args.config is not None:
msg = "Loading yaml config file"
logger.info(msg)
config = yaml.load(open(args.config))
else:
config = {}
config['allowed_feature_groups'] = [scenario.feature_group_dict.keys()]
# either way, update the scenario with the features used during training
scenario.used_feature_groups = config['allowed_feature_groups']
msg = "Reading predictions"
logger.info(msg)
predictions = pd.read_csv(args.predictions)
msg = "Selecting the algorithm with smallest prediction for each instance"
logger.info(msg)
algorithm_selections = pandas_utils.get_group_extreme(
predictions, "predicted", ex_type="min", group_fields="instance_id")
msg = "Creating the schedules for the validator"
logger.info(msg)
schedules = parallel.apply_df_simple(algorithm_selections, _get_schedule,
scenario.algorithm_cutoff_time)
schedules = utils.merge_dicts(*schedules)
val = Validator()
performance_type = scenario.performance_type[0]
if performance_type == "runtime":
stats = val.validate_runtime(schedules=schedules,
test_scenario=scenario)
elif performance_type == "solution_quality":
stats = val.validate_quality(schedules=schedules,
test_scenario=scenario)
else:
msg = "Unknown performance type: {}".format(performance_type)
raise ValueError(msg)
msg = "=== RESULTS ==="
logger.info(msg)
stats.show()
def split_all_blocks(bed):
exons = parallel.apply_df_simple(bed, bed_utils.split_bed12_blocks)
exons = collection_utils.flatten_lists(exons)
exons = pd.DataFrame(exons)
return exons
def parse_attributes_group(rows):
res = parallel.apply_df_simple(rows, gtf_utils.parse_gtf_attributes)
res = pd.DataFrame(res)
return res
def main():
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description="This script extracts the differential micropeptides from two "
"conditions. Please see the documentation in redmine for more details.\n\n"
"Please see the pyensembl (https://github.com/hammerlab/pyensembl) "
"documentation for more information about the ensembl release and species.")
parser.add_argument('config', help="The (yaml) config file")
parser.add_argument('name_a', help="The name of the first condition")
parser.add_argument('name_b', help="The name of the second condition")
parser.add_argument('out', help="The output (.csv.gz or .xlsx) file")
parser.add_argument('-a', '--append-sheet', help="If this flag is given, "
"then a worksheet with the name '<name_a>,<name_b>' will be appended "
"to the .xlsx file given by out (if it exists)", action='store_true')
parser.add_argument('-f', '--filter', help="If this flag is present, then "
"the output will be filtered to include only the differential "
"micropeptides with the highest KL-divergence and read coverage",
action='store_true')
parser.add_argument('--read-filter-percent', help="If the the --filter flag "
"is given, then only the top --read-filter-percent micropeptides will "
"be considered for the final output. They still must meet the KL-"
"divergence filtering criteria.", type=float,
default=default_read_filter_percent)
parser.add_argument('--kl-filter-percent', help="If the the --filter flag "
"is given, then only the top --read-kl-percent micropeptides will "
"be considered for the final output. They still must meet the read "
"coverage filtering criteria.", type=float,
default=default_kl_filter_percent)
parser.add_argument('--id-matches', help="This is a list of files which "
"contain ORF identifiers to compare to the differential micropeptides. "
"For each of the files given, two columns will be added to the output "
"which indicate if either A or B appear in the respective file. Each "
"file should have a single ORF identifier on each line and contain "
"nothing else.", nargs='*', default=default_id_matches)
parser.add_argument('--id-match-names', help="A name to include in the "
"output file for each --id-matches file. The number of names must "
"match the number of files.", nargs='*', default=default_id_match_names)
parser.add_argument('--overlaps', help="This is a list of bed12+ files "
"which will be compared to the differential micropeptides. Two columns "
"(one for A, one for B) will be added to the output which indicate if "
"the respective micropeptides overlap a feature in each file by at "
"least 1 bp.", nargs='*', default=default_overlaps)
parser.add_argument('--overlap-names', help="A name to include in the "
"output file for each --overlaps file. The number of names must match "
"the number of files.", nargs='*', default=default_overlap_names)
parser.add_argument('-r', '--ensembl-release', help="The version of Ensembl "
"to use when mapping transcript identifiers to gene identifiers",
type=int, default=default_ensembl_release)
parser.add_argument('-s', '--ensembl-species', help="The Ensembl species "
"to use when mapping transcript identifiers to gene identifiers",
default=default_ensembl_species)
parser.add_argument('--a-is-single-sample', help="By default, this script "
"assumes the predictions come from merged replicates. If name_a is from "
"a single sample, this flag should be given. It is necessary to find "
"the correct filenames.", action='store_true')
parser.add_argument('--b-is-single-sample', help="By default, this script "
"assumes the predictions come from merged replicates. If name_b is from "
"a single sample, this flag should be given. It is necessary to find "
"the correct filenames.", action='store_true')
parser.add_argument('--fields-to-keep', help="The fields to keep from the "
"Bayes factor file for each condition", nargs='*',
default=default_fields_to_keep)
parser.add_argument('--max-micropeptide-len', help="The maximum (inclusive) "
"length of ORFs considered as micropeptides", type=int,
default=default_max_micropeptide_len)
parser.add_argument('--do-not-fix-tcons', help="By default, the \"TCONS_\" "
"identifiers from StringTie, etc., do not parse correctly; this script "
"update the identifiers so that will parse correctly unless instructed not "
"to. The script is likely to crash if the identifiers are not fixed.",
action='store_true')
logging_utils.add_logging_options(parser)
args = parser.parse_args()
logging_utils.update_logging(args)
msg = "Loading ensembl database"
logger.info(msg)
ensembl = pyensembl.EnsemblRelease(release=args.ensembl_release,
species=args.ensembl_species)
ensembl.db
msg = "Checking the id-match and overlaps files"
logger.info(msg)
if len(args.id_matches) != len(args.id_match_names):
msg = ("The number of --id-matches files and --id-match-names do not "
"match. {} files and {} names".format(len(args.id_matches),
len(args.id_match_names)))
raise ValueError(msg)
if len(args.overlaps) != len(args.overlap_names):
msg = ("The number of --overlaps files and --overlaps-names do not "
"match. {} files and {} names".format(len(args.overlaps),
len(args.overlap_names)))
raise ValueError(msg)
utils.check_files_exist(args.id_matches)
utils.check_files_exist(args.overlaps)
if args.filter:
msg = "Validating filter percentages"
logger.info(msg)
math_utils.check_range(args.read_filter_percent, 0, 1,
variable_name="--read-filter-percent")
math_utils.check_range(args.kl_filter_percent, 0, 1,
variable_name="--kl-filter-percent")
msg = "Extracting file names"
logger.info(msg)
config = yaml.load(open(args.config))
note_str = config.get('note', None)
# keep multimappers?
is_unique = not ('keep_riboseq_multimappers' in config)
# and the smoothing parameters
fraction = config.get('smoothing_fraction', None)
reweighting_iterations = config.get('smoothing_reweighting_iterations', None)
lengths_a = None
offsets_a = None
if args.a_is_single_sample:
lengths_a, offsets_a = ribo_utils.get_periodic_lengths_and_offsets(config,
args.name_a, is_unique=is_unique)
bayes_factors_a = filenames.get_riboseq_bayes_factors(config['riboseq_data'], args.name_a,
length=lengths_a, offset=offsets_a, is_unique=is_unique, note=note_str,
fraction=fraction, reweighting_iterations=reweighting_iterations)
if not os.path.exists(bayes_factors_a):
msg = ("Could not find the Bayes factor file for {}. ({}). Quitting.".
format(args.name_a, bayes_factors_a))
raise FileNotFoundError(msg)
predicted_orfs_a = filenames.get_riboseq_predicted_orfs(config['riboseq_data'],
args.name_a, length=lengths_a, offset=offsets_a, is_unique=is_unique, note=note_str,
fraction=fraction, reweighting_iterations=reweighting_iterations,
is_filtered=True, is_chisq=False)
if not os.path.exists(predicted_orfs_a):
msg = ("Could not find the predictions bed file for {}. ({}). Quitting.".
format(args.name_a, predicted_orfs_a))
raise FileNotFoundError(msg)
lengths_b = None
offsets_b = None
if args.b_is_single_sample:
lengths_b, offsets_b = ribo_utils.get_periodic_lengths_and_offsets(config,
args.name_b, is_unique=is_unique)
bayes_factors_b = filenames.get_riboseq_bayes_factors(config['riboseq_data'], args.name_b,
length=lengths_b, offset=offsets_b, is_unique=is_unique, note=note_str,
fraction=fraction, reweighting_iterations=reweighting_iterations)
if not os.path.exists(bayes_factors_b):
msg = ("Could not find the Bayes factor file for {}. ({}). Quitting.".
format(args.name_b, bayes_factors_b))
raise FileNotFoundError(msg)
predicted_orfs_b = filenames.get_riboseq_predicted_orfs(config['riboseq_data'],
args.name_b, length=lengths_b, offset=offsets_b, is_unique=is_unique, note=note_str,
fraction=fraction, reweighting_iterations=reweighting_iterations,
is_filtered=True, is_chisq=False)
if not os.path.exists(predicted_orfs_b):
msg = ("Could not find the predictions bed file for {}. ({}). Quitting.".
format(args.name_b, predicted_orfs_b))
raise FileNotFoundError(msg)
exons_file = filenames.get_exons(config['genome_base_path'], config['genome_name'],
note=config.get('orf_note'), is_orf=True)
if not os.path.exists(exons_file):
msg = "Could not find the exons file ({}). Quitting.".format(exons_file)
raise FileNotFoundError(msg)
msg = "Reading the exons"
logger.info(msg)
exons = bed_utils.read_bed(exons_file)
msg = "Reading the BF files"
logger.info(msg)
bf_df_a = bed_utils.read_bed(bayes_factors_a)
bf_df_b = bed_utils.read_bed(bayes_factors_b)
msg = "Reading the predictions files"
logger.info(msg)
bed_df_a = bed_utils.read_bed(predicted_orfs_a)
bed_df_b = bed_utils.read_bed(predicted_orfs_b)
differential_micropeptide_dfs = []
# extract micropeptides
msg = "Extracting micropeptides"
logger.info(msg)
m_micropeptides_a = bed_df_a['orf_len'] <= args.max_micropeptide_len
m_micropeptides_b = bed_df_b['orf_len'] <= args.max_micropeptide_len
micropeptides_a = bed_df_a[m_micropeptides_a]
micropeptides_b = bed_df_b[m_micropeptides_b]
long_orfs_a = bed_df_a[~m_micropeptides_a]
long_orfs_b = bed_df_b[~m_micropeptides_b]
msg = "Finding micropeptides in A with no overlap in B"
logger.info(msg)
micropeptides_a_no_match_b = bed_utils.subtract_bed(micropeptides_a, bed_df_b, exons=exons)
micropeptides_a_no_match_b_df = pd.DataFrame()
micropeptides_a_no_match_b_df['A'] = list(micropeptides_a_no_match_b)
micropeptides_a_no_match_b_df['B'] = None
micropeptides_a_no_match_b_df['kl'] = np.inf
micropeptides_a_no_match_b_df['overlap_type'] = 'micro_a_only'
differential_micropeptide_dfs.append(micropeptides_a_no_match_b_df)
msg = "Finding micropeptides in B with no overlap in A"
logger.info(msg)
micropeptides_b_no_match_a = bed_utils.subtract_bed(micropeptides_b, bed_df_a, exons=exons)
micropeptides_b_no_match_a_df = pd.DataFrame()
micropeptides_b_no_match_a_df['B'] = list(micropeptides_b_no_match_a)
micropeptides_b_no_match_a_df['A'] = None
micropeptides_b_no_match_a_df['kl'] = np.inf
micropeptides_b_no_match_a_df['overlap_type'] = 'micro_b_only'
differential_micropeptide_dfs.append(micropeptides_b_no_match_a_df)
msg = "Finding overlapping micropeptides"
logger.info(msg)
micropeptides_a_micropeptides_b_df = get_overlap_df(micropeptides_a,
micropeptides_b, 'micro_a_micro_b', bf_df_a, bf_df_b)
differential_micropeptide_dfs.append(micropeptides_a_micropeptides_b_df)
micropeptides_a_long_b_df = get_overlap_df(micropeptides_a, long_orfs_b,
'micro_a_long_b', bf_df_a, bf_df_b)
differential_micropeptide_dfs.append(micropeptides_a_long_b_df)
micropeptides_b_long_a_df = get_overlap_df(long_orfs_a, micropeptides_b,
'long_a_micro_b', bf_df_a, bf_df_b)
differential_micropeptide_dfs.append(micropeptides_b_long_a_df)
differential_micropeptides_df = pd.concat(differential_micropeptide_dfs)
msg = "Adding read count information"
logger.info(msg)
res = differential_micropeptides_df.merge(bf_df_a[args.fields_to_keep],
left_on='A', right_on='id', how='left')
to_rename = {f: "{}_A".format(f) for f in args.fields_to_keep}
res = res.rename(columns=to_rename)
res = res.drop('id_A', axis=1)
res = res.merge(bf_df_b[args.fields_to_keep], left_on='B', right_on='id', how='left')
to_rename = {f: "{}_B".format(f) for f in args.fields_to_keep}
res = res.rename(columns=to_rename)
res = res.drop('id_B', axis=1)
id_columns = ['A', 'B']
res = res.drop_duplicates(subset=id_columns)
if not args.do_not_fix_tcons:
# replace TCONS_ with TCONS
res['A'] = res['A'].str.replace("TCONS_", "TCONS")
res['B'] = res['B'].str.replace("TCONS_", "TCONS")
msg = "Extracting the genes and their biotypes using pyensembl"
logger.info(msg)
ensembl = pyensembl.EnsemblRelease(release=args.ensembl_release,
species=args.ensembl_species)
ensembl_transcript_ids = set(ensembl.transcript_ids())
biotypes_a = parallel.apply_df_simple(res, get_transcript_and_biotype,
'A', ensembl, ensembl_transcript_ids)
biotypes_b = parallel.apply_df_simple(res, get_transcript_and_biotype,
'B', ensembl, ensembl_transcript_ids)
biotypes_a = utils.remove_nones(biotypes_a)
biotypes_b = utils.remove_nones(biotypes_b)
biotypes_a = pd.DataFrame(biotypes_a)
biotypes_b = pd.DataFrame(biotypes_b)
res = res.merge(biotypes_a, on='A', how='left')
res = res.merge(biotypes_b, on='B', how='left')
msg = "Pulling annotations from mygene.info"
logger.info(msg)
# pull annotations from mygene
gene_info_a = mygene_utils.query_mygene(res['gene_id_A'])
gene_info_b = mygene_utils.query_mygene(res['gene_id_B'])
# and add the mygene info
res = res.merge(gene_info_a, left_on='gene_id_A', right_on='gene_id',
how='left')
to_rename = {f: "{}_A".format(f) for f in gene_info_a.columns}
to_rename.pop('gene_id')
res = res.rename(columns=to_rename)
res = res.drop('gene_id', axis=1)
res = res.merge(gene_info_b, left_on='gene_id_B',
right_on='gene_id', how='left')
to_rename = {f: "{}_B".format(f) for f in gene_info_a.columns}
to_rename.pop('gene_id')
res = res.rename(columns=to_rename)
res = res.drop('gene_id', axis=1)
msg = "Removing duplicates"
logger.info(msg)
id_columns = ['A', 'B']
res = res.drop_duplicates(subset=id_columns)
msg = "Adding --id-matches columns"
logger.info(msg)
for (id_match_file, name) in zip(args.id_matches, args.id_match_names):
res = add_id_matches(res, id_match_file, name)
msg = "Adding --overlaps columns"
logger.info(msg)
for (overlap_file, name) in zip(args.overlaps, args.overlap_names):
res = add_overlaps(res, overlap_file, name, bed_df_a, bed_df_b, exons)
msg = "Sorting by in-frame reads"
logger.info(msg)
res['x_1_sum_A'] = res['x_1_sum_A'].fillna(0)
res['x_1_sum_B'] = res['x_1_sum_B'].fillna(0)
res['x_1_sum'] = res['x_1_sum_A'] + res['x_1_sum_B']
res = res.sort_values('x_1_sum', ascending=False)
if args.filter:
msg = "Filtering the micropeptides by read coverage and KL-divergence"
logger.info(msg)
x_1_sum_ranks = res['x_1_sum'].rank(method='min', na_option='top',
ascending=False)
num_x_1_sum_ranks = x_1_sum_ranks.max()
max_good_x_1_sum_rank = num_x_1_sum_ranks * args.read_filter_percent
m_good_x_1_sum_rank = x_1_sum_ranks <= max_good_x_1_sum_rank
msg = ("Number of micropeptides passing read filter: {}".format(
sum(m_good_x_1_sum_rank)))
logger.debug(msg)
kl_ranks = res['kl'].rank(method='dense', na_option='top', ascending=False)
num_kl_ranks = kl_ranks.max()
max_good_kl_rank = num_kl_ranks * args.kl_filter_percent
m_good_kl_rank = kl_ranks <= max_good_kl_rank
msg = ("Number of micropeptides passing KL filter: {}".format(
sum(m_good_kl_rank)))
logger.debug(msg)
m_both_filters = m_good_x_1_sum_rank & m_good_kl_rank
msg = ("Number of micropeptides passing both filters: {}".format(
sum(m_both_filters)))
logger.debug(msg)
res = res[m_both_filters]
msg = "Writing differential micropeptides to disk"
logger.info(msg)
if args.append_sheet is None:
pandas_utils.write_df(res, args.out, index=False)
else:
sheet_name = "{},{}".format(args.name_a, args.name_b)
utils.append_to_xlsx(res, args.out, sheet=sheet_name, index=False)
def get_transcript_ids(gtf_entries):
ret = parallel.apply_df_simple(gtf_entries, get_transcript_id)
return ret
def main():
# make sure we write the config file in a user-friendly order
setup_yaml()
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description="Create symlinks and partial config file for samples and "
"replicates from a csv sample sheet. The csv file must include the "
"following columns: sample_filename, condition, sample_type. "
"Optionally, it can also use the following columns in the filenames: "
"cell_type, replicate_name, lane. The format of the symlinks and sample "
"names is as follows:<condition>.<sample_type>[.cell-type-<cell_type>]"
"[.rep-<replicate>][.lane-<lane>]. The optional parts are skipped if "
"they are not present in the sample sheet. The script first "
"concatenates samples with the same condition, sample type, cell type "
"and replicate identifiers (but with different lanes). The "
"\"biological_replicates\" group samples with the same condition, "
"sample type and cell type.\n\nThe sample sheet can also contain "
"additional columns, but they are ignored.")
parser.add_argument(
'sample_sheet',
help="The csv sample sheet. It can "
"also be an excel file if it has the file extension \"xls\" or "
"\"xlsx\" or hdf5 if the filetype is \"hdf\", \"hdf5\", \"h5\" or "
"\"he5\".")
parser.add_argument('out',
help="The (partial) yaml config file created "
"based on the sample sheet")
parser.add_argument(
'--sample-sheet-file-type',
help="The file type of "
"the sample sheet. By default (\"AUTO\"), this is guessed based on "
"the extension.",
choices=sample_sheet_file_type_choices,
default=default_sample_sheet_file_type)
parser.add_argument(
'--sheet-name',
help="For excel and hdf5 files, the "
"name of the sheet in the workbook which contains the sample sheet.",
default=default_sheet_name)
parser.add_argument('--riboseq-sample-types',
help="The \"sample_type\"s "
"to treat as riboseq samples",
nargs='*',
default=default_riboseq_sample_types)
parser.add_argument('--rnaseq-sample-types',
help="The \"sample_type\"s "
"to treat as rna-seq samples",
nargs='*',
default=default_rnaseq_sample_types)
parser.add_argument(
'--overwrite',
help="If this flag is given, then "
"files at the symlink locations will be overwritten. N.B. THIS COULD "
"DESTROY THE ORIGINAL DATA FILES! BE CAREFUL!!!",
action='store_true')
parser.add_argument(
'--no-symlinks',
help="If this flag is given, then "
"symlinks will not be created. Namely, only the yaml config file will "
"be written.",
action='store_true')
logging_utils.add_logging_options(parser)
args = parser.parse_args()
logging_utils.update_logging(args)
msg = "Reading sample sheet"
logger.info(msg)
# make sure we convert to the correct data type
converters = {
"condition": str,
"sample_type": str,
"cell_type": str,
"replicate": str,
"lane": str,
"random_field": str
}
sample_sheet = pandas_utils.read_df(args.sample_sheet,
filetype=args.sample_sheet_file_type,
skip_blank_lines=True,
sheet=args.sheet_name,
converters=converters)
# make sure we have the necessary columns
if 'condition' not in sample_sheet.columns:
msg = "\"condition\" is not present in the sample sheet"
raise ValueError(msg)
if 'sample_type' not in sample_sheet.columns:
msg = "\"sample_type\" is not present in the sample sheet"
raise ValueError(msg)
if 'sample_filename' not in sample_sheet.columns:
msg = "\"sample_filename\" is not present in the sample sheet"
raise ValueError(msg)
msg = "Creating filenames"
logger.info(msg)
sample_sheet['sample_name'] = parallel.apply_df_simple(
sample_sheet, _get_sample_name_helper)
sample_sheet['filename'] = parallel.apply_df_simple(
sample_sheet, _get_sample_filename_helper)
if not args.no_symlinks:
msg = "Creating symlinks"
logger.info(msg)
parallel.apply_df_simple(sample_sheet, _create_symlink, args.overwrite)
sample_sheet['replicate_name'] = parallel.apply_df_simple(
sample_sheet, _get_replicate_name_helper)
sample_sheet['replicate_text'] = parallel.apply_df_simple(
sample_sheet, _get_replicate_text_helper)
sample_sheet['replicate_filename'] = parallel.apply_df_simple(
sample_sheet, _get_replicate_filename_helper)
# check if we were given lanes
if 'lane' in sample_sheet.columns:
msg = "Pooling samples for each replicate from different lanes"
logger.info(msg)
replicate_groups = sample_sheet.groupby('replicate_name')
if not args.no_symlinks:
replicate_groups.apply(pool_lanes)
msg = "Extracting replicate names for config"
logger.info(msg)
# finally, create the yaml config file
m_riboseq = sample_sheet['sample_type'].isin(args.riboseq_sample_types)
riboseq_samples = pandas_utils.dataframe_to_dict(sample_sheet[m_riboseq],
'replicate_name',
'replicate_filename')
riboseq_sample_text = pandas_utils.dataframe_to_dict(
sample_sheet[m_riboseq], 'replicate_name', 'replicate_text')
m_rnaseq = sample_sheet['sample_type'].isin(args.rnaseq_sample_types)
rnaseq_samples = pandas_utils.dataframe_to_dict(sample_sheet[m_rnaseq],
'replicate_name',
'replicate_filename')
rnaseq_sample_text = pandas_utils.dataframe_to_dict(
sample_sheet[m_rnaseq], 'replicate_name', 'replicate_text')
msg = "Grouping replicates by condition"
logger.info(msg)
# so there is not a random magic number later...
num_procs = 1
sample_sheet['full_condition_name'] = parallel.apply_df_simple(
sample_sheet, _get_full_condition_name_helper)
ribo_groups = sample_sheet[m_riboseq].groupby('full_condition_name')
ribo_condition_groups = parallel.apply_parallel_groups(
ribo_groups, num_procs, get_condition_replicates)
ribo_condition_groups = utils.merge_dicts(*ribo_condition_groups)
ribo_condition_text = parallel.apply_parallel_groups(
ribo_groups, num_procs, get_condition_text)
ribo_condition_text = utils.merge_dicts(*ribo_condition_text)
rna_groups = sample_sheet[m_rnaseq].groupby('full_condition_name')
rna_condition_groups = parallel.apply_parallel_groups(
rna_groups, num_procs, get_condition_replicates)
rna_condition_groups = utils.merge_dicts(*rna_condition_groups)
rna_condition_text = parallel.apply_parallel_groups(
rna_groups, num_procs, get_condition_text)
rna_condition_text = utils.merge_dicts(*rna_condition_text)
msg = "Writing partial config file"
logger.info(msg)
config = collections.OrderedDict(
[('project_name', "<PROJECT_NAME>"), ('note', "<NOTE>"),
('gtf', "<GTF>"), ('fasta', "<FASTA>"),
('star_index', "<STAR_INDEX>"),
('ribosomal_index', "<RIBOSOMAL_INDEX>"),
('ribosomal_fasta', "<RIBOSOMAL_FASTA>"),
('genome_base_path', "<GENOME_BASE_PATH>"),
('genome_name', "<GENOME_NAME>"), ('orf_note', "<ORF_NOTE>"),
('adapter_file', "<RIBO_ADAPTER_FILE>"),
('rna_adapter_file', "<RNA_ADAPTER_FILE>"),
('riboseq_data', "<RIBO_DATA_PATH>"),
('rnaseq_data', "<RNA_DATA_PATH>"),
('riboseq_samples', riboseq_samples),
('rnaseq_samples', rnaseq_samples),
('riboseq_biological_replicates', ribo_condition_groups),
('rnaseq_biological_replicates', rna_condition_groups),
('riboseq_sample_name_map', riboseq_sample_text),
('rnaseq_sample_name_map', rnaseq_sample_text),
('riboseq_condition_name_map', ribo_condition_text),
('rnaseq_condition_name_map', rna_condition_text)])
with open(args.out, 'w') as out:
yaml.dump(config, out, default_flow_style=False)
