def main(argv=None):
"""script main.
parses command line options in sys.argv, unless *argv* is given.
"""
if argv is None:
argv = sys.argv
# setup command line parser
parser = U.OptionParser(version="%prog version: $Id$",
usage=usage,
description=globals()["__doc__"])
group = U.OptionGroup(parser, "dedup-specific options")
group.add_option("--output-stats",
dest="stats",
type="string",
default=False,
help="Specify location to output stats")
parser.add_option_group(group)
# add common options (-h/--help, ...) and parse command line
(options, args) = U.Start(parser, argv=argv)
U.validateSamOptions(options, group=False)
if options.random_seed:
np.random.seed(options.random_seed)
if options.stdin != sys.stdin:
in_name = options.stdin.name
options.stdin.close()
else:
raise ValueError("Input on standard in not currently supported")
if options.stdout != sys.stdout:
if options.no_sort_output:
out_name = options.stdout.name
else:
out_name = U.getTempFilename(dir=options.tmpdir)
sorted_out_name = options.stdout.name
options.stdout.close()
else:
if options.no_sort_output:
out_name = "-"
else:
out_name = U.getTempFilename(dir=options.tmpdir)
sorted_out_name = "-"
if not options.no_sort_output: # need to determine the output format for sort
if options.out_sam:
sort_format = "sam"
else:
sort_format = "bam"
if options.in_sam:
in_mode = "r"
else:
in_mode = "rb"
if options.out_sam:
out_mode = "wh"
else:
out_mode = "wb"
if options.stats and options.ignore_umi:
raise ValueError("'--output-stats' and '--ignore-umi' options"
" cannot be used together")
infile = pysam.Samfile(in_name, in_mode)
outfile = pysam.Samfile(out_name, out_mode, template=infile)
if options.paired:
outfile = sam_methods.TwoPassPairWriter(infile, outfile)
nInput, nOutput, input_reads, output_reads = 0, 0, 0, 0
if options.detection_method:
bam_features = detect_bam_features(infile.filename)
if not bam_features[options.detection_method]:
if sum(bam_features.values()) == 0:
raise ValueError(
"There are no bam tags available to detect multimapping. "
"Do not set --multimapping-detection-method")
else:
raise ValueError(
"The chosen method of detection for multimapping (%s) "
"will not work with this bam. Multimapping can be detected"
" for this bam using any of the following: %s" %
(options.detection_method, ",".join(
[x for x in bam_features if bam_features[x]])))
gene_tag = options.gene_tag
metacontig2contig = None
if options.chrom:
inreads = infile.fetch(reference=options.chrom)
else:
if options.per_contig and options.gene_transcript_map:
metacontig2contig = sam_methods.getMetaContig2contig(
infile, options.gene_transcript_map)
metatag = "MC"
inreads = sam_methods.metafetcher(infile, metacontig2contig,
metatag)
gene_tag = metatag
else:
inreads = infile.fetch()
# set up ReadCluster functor with methods specific to
# specified options.method
processor = network.ReadDeduplicator(options.method)
bundle_iterator = sam_methods.get_bundles(
options, metacontig_contig=metacontig2contig)
if options.stats:
# set up arrays to hold stats data
stats_pre_df_dict = {"UMI": [], "counts": []}
stats_post_df_dict = {"UMI": [], "counts": []}
pre_cluster_stats = []
post_cluster_stats = []
pre_cluster_stats_null = []
post_cluster_stats_null = []
topology_counts = collections.Counter()
node_counts = collections.Counter()
read_gn = umi_methods.random_read_generator(
infile.filename,
chrom=options.chrom,
barcode_getter=bundle_iterator.barcode_getter)
for bundle, key, status in bundle_iterator(inreads):
nInput += sum([bundle[umi]["count"] for umi in bundle])
while nOutput >= output_reads + 100000:
output_reads += 100000
U.info("Written out %i reads" % output_reads)
while nInput >= input_reads + 1000000:
input_reads += 1000000
U.info("Parsed %i input reads" % input_reads)
if options.stats:
# generate pre-dudep stats
average_distance = umi_methods.get_average_umi_distance(
bundle.keys())
pre_cluster_stats.append(average_distance)
cluster_size = len(bundle)
random_umis = read_gn.getUmis(cluster_size)
average_distance_null = umi_methods.get_average_umi_distance(
random_umis)
pre_cluster_stats_null.append(average_distance_null)
if options.ignore_umi:
for umi in bundle:
nOutput += 1
outfile.write(bundle[umi]["read"])
else:
# dedup using umis and write out deduped bam
reads, umis, umi_counts = processor(bundle=bundle,
threshold=options.threshold)
for read in reads:
outfile.write(read)
nOutput += 1
if options.stats:
# collect pre-dudupe stats
stats_pre_df_dict['UMI'].extend(bundle)
stats_pre_df_dict['counts'].extend(
[bundle[UMI]['count'] for UMI in bundle])
# collect post-dudupe stats
post_cluster_umis = [
bundle_iterator.barcode_getter(x)[0] for x in reads
]
stats_post_df_dict['UMI'].extend(umis)
stats_post_df_dict['counts'].extend(umi_counts)
average_distance = umi_methods.get_average_umi_distance(
post_cluster_umis)
post_cluster_stats.append(average_distance)
cluster_size = len(post_cluster_umis)
random_umis = read_gn.getUmis(cluster_size)
average_distance_null = umi_methods.get_average_umi_distance(
random_umis)
post_cluster_stats_null.append(average_distance_null)
outfile.close()
if not options.no_sort_output:
# sort the output
pysam.sort("-o", sorted_out_name, "-O", sort_format, out_name)
os.unlink(out_name) # delete the tempfile
if options.stats:
# generate the stats dataframe
stats_pre_df = pd.DataFrame(stats_pre_df_dict)
stats_post_df = pd.DataFrame(stats_post_df_dict)
# tally the counts per umi per position
pre_counts = collections.Counter(stats_pre_df["counts"])
post_counts = collections.Counter(stats_post_df["counts"])
counts_index = list(
set(pre_counts.keys()).union(set(post_counts.keys())))
counts_index.sort()
with U.openFile(options.stats + "_per_umi_per_position.tsv",
"w") as outf:
outf.write("counts\tinstances_pre\tinstances_post\n")
for count in counts_index:
values = (count, pre_counts[count], post_counts[count])
outf.write("\t".join(map(str, values)) + "\n")
# aggregate stats pre/post per UMI
agg_pre_df = aggregateStatsDF(stats_pre_df)
agg_post_df = aggregateStatsDF(stats_post_df)
agg_df = pd.merge(agg_pre_df,
agg_post_df,
how='left',
left_index=True,
right_index=True,
sort=True,
suffixes=["_pre", "_post"])
# TS - if count value not observed either pre/post-dedup,
# merge will leave an empty cell and the column will be cast as a float
# see http://pandas.pydata.org/pandas-docs/dev/missing_data.html
# --> Missing data casting rules and indexing
# so, back fill with zeros and convert back to int
agg_df = agg_df.fillna(0).astype(int)
agg_df.index = [x.decode() for x in agg_df.index]
agg_df.index.name = 'UMI'
agg_df.to_csv(options.stats + "_per_umi.tsv", sep="\t")
# bin distances into integer bins
max_ed = int(
max(
map(max, [
pre_cluster_stats, post_cluster_stats,
pre_cluster_stats_null, post_cluster_stats_null
])))
cluster_bins = range(-1, int(max_ed) + 2)
def bin_clusters(cluster_list, bins=cluster_bins):
''' take list of floats and return bins'''
return np.digitize(cluster_list, bins, right=True)
def tallyCounts(binned_cluster, max_edit_distance):
''' tally counts per bin '''
return np.bincount(binned_cluster, minlength=max_edit_distance + 3)
pre_cluster_binned = bin_clusters(pre_cluster_stats)
post_cluster_binned = bin_clusters(post_cluster_stats)
pre_cluster_null_binned = bin_clusters(pre_cluster_stats_null)
post_cluster_null_binned = bin_clusters(post_cluster_stats_null)
edit_distance_df = pd.DataFrame(
{
"unique":
tallyCounts(pre_cluster_binned, max_ed),
"unique_null":
tallyCounts(pre_cluster_null_binned, max_ed),
options.method:
tallyCounts(post_cluster_binned, max_ed),
"%s_null" % options.method:
tallyCounts(post_cluster_null_binned, max_ed),
"edit_distance":
cluster_bins
},
columns=[
"unique", "unique_null", options.method,
"%s_null" % options.method, "edit_distance"
])
# TS - set lowest bin (-1) to "Single_UMI"
edit_distance_df['edit_distance'][0] = "Single_UMI"
edit_distance_df.to_csv(options.stats + "_edit_distance.tsv",
index=False,
sep="\t")
# write footer and output benchmark information.
U.info("Reads: %s" % ", ".join([
"%s: %s" % (x[0], x[1])
for x in bundle_iterator.read_events.most_common()
]))
U.info("Number of reads out: %i" % nOutput)
if not options.ignore_umi: # otherwise processor has not been used
U.info("Total number of positions deduplicated: %i" %
processor.UMIClusterer.positions)
if processor.UMIClusterer.positions > 0:
U.info("Mean number of unique UMIs per position: %.2f" %
(float(processor.UMIClusterer.total_umis_per_position) /
processor.UMIClusterer.positions))
U.info("Max. number of unique UMIs per position: %i" %
processor.UMIClusterer.max_umis_per_position)
else:
U.warn("The BAM did not contain any valid "
"reads/read pairs for deduplication")
U.Stop()
def main(argv=None):
"""script main.
parses command line options in sys.argv, unless *argv* is given.
"""
if argv is None:
argv = sys.argv
# setup command line parser
parser = U.OptionParser(version="%prog version: $Id$",
usage=globals()["__doc__"])
group = U.OptionGroup(parser, "count-specific options")
parser.add_option("--wide-format-cell-counts",
dest="wide_format_cell_counts",
action="store_true",
default=False,
help=("output the cell counts in a wide format "
"(rows=genes, columns=cells)"))
parser.add_option_group(group)
# add common options (-h/--help, ...) and parse command line
(options, args) = U.Start(parser, argv=argv, add_group_dedup_options=False)
options.per_gene = True # hardcodes counting to per-gene only
U.validateSamOptions(options, group=False)
if options.random_seed:
np.random.seed(options.random_seed)
if options.stdin != sys.stdin:
in_name = options.stdin.name
options.stdin.close()
else:
raise ValueError("Input on standard in not currently supported")
if options.in_sam:
in_mode = "r"
else:
in_mode = "rb"
infile = pysam.Samfile(in_name, in_mode)
# write out to tempfile and then sort to stdout
tmpfilename = U.getTempFilename(dir=options.tmpdir)
tmpfile = U.openFile(tmpfilename, mode="w")
nInput, nOutput, input_reads = 0, 0, 0
gene_tag = options.gene_tag
metacontig2contig = None
if options.chrom:
inreads = infile.fetch(reference=options.chrom)
else:
if options.gene_transcript_map:
metacontig2contig = umi_methods.getMetaContig2contig(
infile, options.gene_transcript_map)
metatag = "MC"
inreads = umi_methods.metafetcher(infile, metacontig2contig,
metatag)
gene_tag = metatag
else:
inreads = infile.fetch()
bundle_iterator = umi_methods.get_bundles(
options, only_count_reads=True, metacontig_contig=metacontig2contig)
for bundle, key, status in bundle_iterator(inreads):
if status == "single_read":
continue
gene, cell = key
umis = bundle.keys()
counts = {umi: bundle[umi]["count"] for umi in umis}
nInput += sum(counts.values())
while nInput >= input_reads + 1000000:
input_reads += 1000000
U.info("Parsed %i input reads" % input_reads)
# set up UMIClusterer functor with methods specific to
# specified options.method
processor = network.UMIClusterer(options.method)
# group the umis
groups = processor(umis, counts, threshold=options.threshold)
gene_count = len(groups)
if options.per_cell:
tmpfile.write("%s\n" % "\t".join(
(gene, cell.decode(), str(gene_count))))
else:
tmpfile.write("%s\n" % "\t".join((gene, str(gene_count))))
nOutput += gene_count
tmpfile.close()
if options.per_cell:
gene_counts_dict = {}
with U.openFile(tmpfilename, mode="r") as inf:
genes = set()
cells = set()
for line in inf:
gene, cell, gene_count = line.strip().split("\t")
genes.add(gene)
cells.add(cell)
if gene not in gene_counts_dict:
gene_counts_dict[gene] = {}
gene_counts_dict[gene][cell] = gene_count
if options.wide_format_cell_counts: # write out in wide format
options.stdout.write("%s\t%s\n" %
("gene", "\t".join(sorted(cells))))
for gene in sorted(genes):
counts = []
for cell in sorted(cells):
if cell in gene_counts_dict[gene]:
counts.append(gene_counts_dict[gene][cell])
else:
counts.append(0)
options.stdout.write("%s\t%s\n" %
(gene, "\t".join(map(str, counts))))
else: # write out in long format
options.stdout.write("%s\t%s\t%s\n" % ("gene", "cell", "count"))
for gene in sorted(genes):
for cell in sorted(list(gene_counts_dict[gene].keys())):
options.stdout.write(
"%s\t%s\t%s\n" %
(gene, cell, gene_counts_dict[gene][cell]))
else:
options.stdout.write("%s\t%s\n" % ("gene", "count"))
with U.openFile(tmpfilename, mode="r") as inf:
for line in inf:
options.stdout.write(line)
os.unlink(tmpfilename)
# output reads events and benchmark information.
for event in bundle_iterator.read_events.most_common():
U.info("%s: %s" % (event[0], event[1]))
U.info("Number of (post deduplication) reads counted: %i" % nOutput)
U.Stop()
def main(argv=None):
"""script main.
parses command line options in sys.argv, unless *argv* is given.
"""
if argv is None:
argv = sys.argv
# setup command line parser
parser = U.OptionParser(version="%prog version: $Id$",
usage=globals()["__doc__"])
group = U.OptionGroup(parser, "group-specific options")
group.add_option(
"--group-out",
dest="tsv",
type="string",
help="Outfile name for file mapping read id to read group",
default=None)
group.add_option(
"--output-bam",
dest="output_bam",
action="store_true",
default=False,
help=("output a bam file with read groups tagged using the UG tag"
"[default=%default]"))
group.add_option(
"--output-unmapped",
dest="output_unmapped",
action="store_true",
default=False,
help=("Retain all unmapped reads in output[default=%default]"))
parser.add_option("--umi-group-tag",
dest="umi_group_tag",
type="string",
help="tag for the outputted umi group",
default='BX')
parser.add_option_group(group)
# add common options (-h/--help, ...) and parse command line
(options, args) = U.Start(parser, argv=argv)
U.validateSamOptions(options)
if options.stdin != sys.stdin:
in_name = options.stdin.name
options.stdin.close()
else:
raise ValueError("Input on standard in not currently supported")
if options.stdout != sys.stdout:
if options.no_sort_output:
out_name = options.stdout.name
else:
out_name = U.getTempFilename()
sorted_out_name = options.stdout.name
options.stdout.close()
assert options.output_bam, (
"To output a bam you must include --output-bam option")
else:
if options.no_sort_output:
out_name = "-"
else:
out_name = U.getTempFilename()
sorted_out_name = "-"
if not options.no_sort_output: # need to determine the output format for sort
if options.out_sam:
sort_format = "sam"
else:
sort_format = "bam"
if options.in_sam:
in_mode = "r"
else:
in_mode = "rb"
if options.out_sam:
out_mode = "wh"
else:
out_mode = "wb"
infile = pysam.Samfile(in_name, in_mode)
if options.output_bam:
outfile = pysam.Samfile(out_name, out_mode, template=infile)
else:
outfile = None
if options.tsv:
mapping_outfile = U.openFile(options.tsv, "w")
mapping_outfile.write("%s\n" % "\t".join([
"read_id", "contig", "position", "gene", "umi", "umi_count",
"final_umi", "final_umi_count", "unique_id"
]))
nInput, nOutput, unique_id, input_reads, output_reads = 0, 0, 0, 0, 0
gene_tag = options.gene_tag
metacontig2contig = None
if options.chrom:
inreads = infile.fetch(reference=options.chrom)
else:
if options.per_gene and options.gene_transcript_map:
metacontig2contig = umi_methods.getMetaContig2contig(
infile, options.gene_transcript_map)
metatag = "MC"
inreads = umi_methods.metafetcher(infile, metacontig2contig,
metatag)
gene_tag = metatag
else:
inreads = infile.fetch(until_eof=options.output_unmapped)
bundle_iterator = umi_methods.get_bundles(
options,
all_reads=True,
return_read2=True,
return_unmapped=options.output_unmapped,
metacontig_contig=metacontig2contig)
for bundle, key, status in bundle_iterator(inreads):
# write out read2s and unmapped (if these options are set)
if status == 'single_read':
# bundle is just a single read here
nInput += 1
if outfile:
outfile.write(bundle)
nOutput += 1
continue
umis = bundle.keys()
counts = {umi: bundle[umi]["count"] for umi in umis}
nInput += sum(counts.values())
while nOutput >= output_reads + 10000:
output_reads += 10000
U.info("Written out %i reads" % output_reads)
while nInput >= input_reads + 1000000:
input_reads += 1000000
U.info("Parsed %i input reads" % input_reads)
# set up UMIClusterer functor with methods specific to
# specified options.method
processor = network.UMIClusterer(options.method)
# group the umis
groups = processor(umis, counts, threshold=options.threshold)
for umi_group in groups:
top_umi = umi_group[0]
group_count = sum(counts[umi] for umi in umi_group)
for umi in umi_group:
reads = bundle[umi]['read']
for read in reads:
if outfile:
# Add the 'UG' tag to the read
read.tags += [('UG', unique_id)]
read.tags += [(options.umi_group_tag, top_umi)]
outfile.write(read)
if options.tsv:
if options.per_gene:
gene = read.get_tag(gene_tag)
else:
gene = "NA"
mapping_outfile.write("%s\n" % "\t".join(
map(str,
(read.query_name, read.reference_name,
umi_methods.get_read_position(
read, options.soft_clip_threshold)[1],
gene, umi.decode(), counts[umi],
top_umi.decode(), group_count, unique_id))))
nOutput += 1
unique_id += 1
if outfile:
outfile.close()
if not options.no_sort_output:
# sort the output
pysam.sort("-o", sorted_out_name, "-O", sort_format, out_name)
os.unlink(out_name) # delete the tempfile
if options.tsv:
mapping_outfile.close()
# write footer and output benchmark information.
U.info("Reads: %s" % ", ".join([
"%s: %s" % (x[0], x[1])
for x in bundle_iterator.read_events.most_common()
]))
U.info("Number of reads out: %i, Number of groups: %i" %
(nOutput, unique_id))
U.Stop()
def main(argv=None):
"""script main.
parses command line options in sys.argv, unless *argv* is given.
"""
if argv is None:
argv = sys.argv
# setup command line parser
parser = U.OptionParser(version="%prog version: $Id$",
usage=globals()["__doc__"])
group = U.OptionGroup(parser, "group-specific options")
group.add_option("--group-out", dest="tsv", type="string",
help="Outfile name for file mapping read id to read group",
default=None)
group.add_option("--output-bam", dest="output_bam", action="store_true",
default=False,
help=("output a bam file with read groups tagged using the UG tag"
"[default=%default]"))
group.add_option("--output-unmapped", dest="output_unmapped", action="store_true",
default=False,
help=("Retain all unmapped reads in output[default=%default]"))
parser.add_option("--umi-group-tag", dest="umi_group_tag",
type="string", help="tag for the outputted umi group",
default='BX')
parser.add_option_group(group)
# add common options (-h/--help, ...) and parse command line
(options, args) = U.Start(parser, argv=argv)
U.validateSamOptions(options)
if options.stdin != sys.stdin:
in_name = options.stdin.name
options.stdin.close()
else:
raise ValueError("Input on standard in not currently supported")
if options.stdout != sys.stdout:
if options.no_sort_output:
out_name = options.stdout.name
else:
out_name = U.getTempFilename()
sorted_out_name = options.stdout.name
options.stdout.close()
assert options.output_bam, (
"To output a bam you must include --output-bam option")
else:
if options.no_sort_output:
out_name = "-"
else:
out_name = U.getTempFilename()
sorted_out_name = "-"
if not options.no_sort_output: # need to determine the output format for sort
if options.out_sam:
sort_format = "sam"
else:
sort_format = "bam"
if options.in_sam:
in_mode = "r"
else:
in_mode = "rb"
if options.out_sam:
out_mode = "wh"
else:
out_mode = "wb"
infile = pysam.Samfile(in_name, in_mode)
if options.output_bam:
outfile = pysam.Samfile(out_name, out_mode, template=infile)
else:
outfile = None
if options.tsv:
mapping_outfile = U.openFile(options.tsv, "w")
mapping_outfile.write("%s\n" % "\t".join(
["read_id", "contig", "position", "gene", "umi", "umi_count",
"final_umi", "final_umi_count", "unique_id"]))
nInput, nOutput, unique_id, input_reads, output_reads = 0, 0, 0, 0, 0
gene_tag = options.gene_tag
metacontig2contig = None
if options.chrom:
inreads = infile.fetch(reference=options.chrom)
else:
if options.per_gene and options.gene_transcript_map:
metacontig2contig = umi_methods.getMetaContig2contig(
infile, options.gene_transcript_map)
metatag = "MC"
inreads = umi_methods.metafetcher(infile, metacontig2contig, metatag)
gene_tag = metatag
else:
inreads = infile.fetch(until_eof=options.output_unmapped)
bundle_iterator = umi_methods.get_bundles(
options,
all_reads=True,
return_read2=True,
return_unmapped=options.output_unmapped,
metacontig_contig=metacontig2contig)
for bundle, key, status in bundle_iterator(inreads):
# write out read2s and unmapped (if these options are set)
if status == 'single_read':
# bundle is just a single read here
nInput += 1
if outfile:
outfile.write(bundle)
nOutput += 1
continue
umis = bundle.keys()
counts = {umi: bundle[umi]["count"] for umi in umis}
nInput += sum(counts.values())
while nOutput >= output_reads + 10000:
output_reads += 10000
U.info("Written out %i reads" % output_reads)
while nInput >= input_reads + 1000000:
input_reads += 1000000
U.info("Parsed %i input reads" % input_reads)
# set up UMIClusterer functor with methods specific to
# specified options.method
processor = network.UMIClusterer(options.method)
# group the umis
groups = processor(
umis,
counts,
threshold=options.threshold)
for umi_group in groups:
top_umi = umi_group[0]
group_count = sum(counts[umi] for umi in umi_group)
for umi in umi_group:
reads = bundle[umi]['read']
for read in reads:
if outfile:
# Add the 'UG' tag to the read
read.tags += [('UG', unique_id)]
read.tags += [(options.umi_group_tag, top_umi)]
outfile.write(read)
if options.tsv:
if options.per_gene:
gene = read.get_tag(gene_tag)
else:
gene = "NA"
mapping_outfile.write("%s\n" % "\t".join(map(str, (
read.query_name, read.reference_name,
umi_methods.get_read_position(
read, options.soft_clip_threshold)[1],
gene,
umi.decode(),
counts[umi],
top_umi.decode(),
group_count,
unique_id))))
nOutput += 1
unique_id += 1
if outfile:
outfile.close()
if not options.no_sort_output:
# sort the output
pysam.sort("-o", sorted_out_name, "-O", sort_format, out_name)
os.unlink(out_name) # delete the tempfile
if options.tsv:
mapping_outfile.close()
# write footer and output benchmark information.
U.info(
"Reads: %s" % ", ".join(["%s: %s" % (x[0], x[1]) for x in
bundle_iterator.read_events.most_common()]))
U.info("Number of reads out: %i, Number of groups: %i" %
(nOutput, unique_id))
U.Stop()
