def main(argv=None):
"""script main.
parses command line options in sys.argv, unless *argv* is given.
"""
if argv is None:
argv = sys.argv
# setup command line parser
parser = U.OptionParser(version="%prog version: $Id$",
usage=globals()["__doc__"])
# add common options (-h/--help, ...) and parse command line
(options, args) = U.Start(parser, argv=argv, add_group_dedup_options=False)
nInput, nOutput = 0, 0
# set the method with which to extract umis from reads
umi_getter = partial(umi_methods.get_umi_read_string, sep=options.umi_sep)
options.stdout.write("%s\t%s\n" % ("gene", "count"))
# set up UMIClusterer functor with methods specific to
# specified options.method
processor = network.UMIClusterer(options.method)
for gene, counts in umi_methods.get_gene_count_tab(options.stdin,
umi_getter=umi_getter):
umis = counts.keys()
nInput += sum(counts.values())
# group the umis
groups = processor(umis, counts, threshold=options.threshold)
gene_count = len(groups)
options.stdout.write("%s\t%i\n" % (gene, gene_count))
nOutput += gene_count
U.info("Number of reads counted: %i" % nOutput)
U.Stop()
def main(argv=None):
"""script main.
parses command line options in sys.argv, unless *argv* is given.
"""
if argv is None:
argv = sys.argv
# setup command line parser
parser = U.OptionParser(version="%prog version: $Id$",
usage=globals()["__doc__"])
parser.add_option("-i",
"--in-sam",
dest="in_sam",
action="store_true",
help="Input file is in sam format [default=%default]",
default=False)
parser.add_option(
"-o",
"--out-sam",
dest="out_sam",
action="store_true",
help="Output alignments in sam format [default=%default]",
default=False)
parser.add_option("--ignore-umi",
dest="ignore_umi",
action="store_true",
help="Ignore UMI and dedup"
" only on position",
default=False)
parser.add_option("--umi-separator",
dest="umi_sep",
type="string",
help="separator between read id and UMI",
default="_")
parser.add_option("--umi-tag",
dest="umi_tag",
type="string",
help="tag containing umi",
default='RX')
parser.add_option("--extract-umi-method",
dest="get_umi_method",
type="choice",
choices=("read_id", "tag"),
default="read_id",
help="where is the read UMI encoded? [default=%default]")
parser.add_option("--subset",
dest="subset",
type="float",
help="Use only a fraction of reads, specified by subset",
default=None)
parser.add_option("--spliced-is-unique",
dest="spliced",
action="store_true",
help="Treat a spliced read as different to an unspliced"
" one [default=%default]",
default=False)
parser.add_option("--soft-clip-threshold",
dest="soft",
type="float",
help="number of bases clipped from 5' end before"
"read is counted as spliced [default=%default]",
default=4)
parser.add_option("--edit-distance-threshold",
dest="threshold",
type="int",
default=1,
help="Edit distance theshold at which to join two UMIs"
"when clustering. [default=%default]")
parser.add_option("--chrom",
dest="chrom",
type="string",
help="Restrict to one chromosome",
default=None)
parser.add_option("--paired",
dest="paired",
action="store_true",
default=False,
help="paired BAM. [default=%default]")
parser.add_option("--method",
dest="method",
type="choice",
choices=("adjacency", "directional", "percentile",
"unique", "cluster"),
default="directional",
help="method to use for umi deduping [default=%default]")
parser.add_option("--output-stats",
dest="stats",
type="string",
default=False,
help="Specify location to output stats")
parser.add_option(
"--whole-contig",
dest="whole_contig",
action="store_true",
default=False,
help=
"Read whole contig before outputting bundles: guarantees that no reads"
"are missed, but increases memory usage")
parser.add_option("--multimapping-detection-method",
dest="detection_method",
type="choice",
choices=("NH", "X0", "XT"),
default=None,
help=("Some aligners identify multimapping using bam "
"tags. Setting this option to NH, X0 or XT will "
"use these tags when selecting the best read "
"amongst reads with the same position and umi "
"[default=%default]"))
parser.add_option("--mapping-quality",
dest="mapping_quality",
type="int",
help="Minimum mapping quality for a read to be retained"
" [default=%default]",
default=0)
parser.add_option(
"--read-length",
dest="read_length",
action="store_true",
default=False,
help=("use read length in addition to position and UMI"
"to identify possible duplicates [default=%default]"))
parser.add_option("--per-contig",
dest="per_contig",
action="store_true",
default=False,
help=("dedup per contig (field 3 in BAM; RNAME),"
" e.g for transcriptome where contig = gene"))
parser.add_option("--per-gene",
dest="per_gene",
action="store_true",
default=False,
help=("Deduplicate per gene,"
"e.g for transcriptome where contig = transcript"
"must also provide a transript to gene map with"
"--gene-transcript-map [default=%default]"))
parser.add_option("--gene-transcript-map",
dest="gene_transcript_map",
type="string",
help="file mapping transcripts to genes (tab separated)",
default=None)
parser.add_option("--gene-tag",
dest="gene_tag",
type="string",
help=("Deduplicate per gene where gene is"
"defined by this bam tag [default=%default]"),
default=None)
parser.add_option(
"--skip-tags-regex",
dest="skip_regex",
type="string",
help=("Used with --gene-tag. "
"Ignore reads where the gene-tag matches this regex"),
default="^[__|Unassigned]")
# add common options (-h/--help, ...) and parse command line
(options, args) = U.Start(parser, argv=argv)
if options.random_seed:
np.random.seed(options.random_seed)
if options.stdin != sys.stdin:
in_name = options.stdin.name
options.stdin.close()
else:
raise ValueError("Input on standard in not currently supported")
if options.stdout != sys.stdout:
out_name = options.stdout.name
options.stdout.close()
else:
out_name = "-"
if options.in_sam:
in_mode = "r"
else:
in_mode = "rb"
if options.out_sam:
out_mode = "wh"
else:
out_mode = "wb"
if options.stats:
if options.ignore_umi:
raise ValueError("'--output-stats' and '--ignore-umi' options"
" cannot be used together")
if options.per_gene:
if not options.gene_transcript_map and not options.gene_map:
raise ValueError(
"--per-gene option requires --gene-transcript-map "
"or --gene-tag")
try:
re.compile(options.skip_regex)
except re.error:
raise ValueError("skip-regex '%s' is not a "
"valid regex" % options.skip_regex)
infile = pysam.Samfile(in_name, in_mode)
outfile = pysam.Samfile(out_name, out_mode, template=infile)
if options.paired:
outfile = umi_methods.TwoPassPairWriter(infile, outfile)
nInput, nOutput = 0, 0
if options.detection_method:
bam_features = detect_bam_features(infile.filename)
if not bam_features[options.detection_method]:
if sum(bam_features.values()) == 0:
raise ValueError(
"There are no bam tags available to detect multimapping. "
"Do not set --multimapping-detection-method")
else:
raise ValueError(
"The chosen method of detection for multimapping (%s) "
"will not work with this bam. Multimapping can be detected"
" for this bam using any of the following: %s" %
(options.detection_method, ",".join(
[x for x in bam_features if bam_features[x]])))
# set the method with which to extract umis from reads
if options.get_umi_method == "read_id":
umi_getter = partial(umi_methods.get_umi_read_id, sep=options.umi_sep)
elif options.get_umi_method == "tag":
umi_getter = partial(umi_methods.get_umi_tag, tag=options.umi_tag)
else:
raise ValueError("Unknown umi extraction method")
if options.stats:
# set up arrays to hold stats data
stats_pre_df_dict = {"UMI": [], "counts": []}
stats_post_df_dict = {"UMI": [], "counts": []}
pre_cluster_stats = []
post_cluster_stats = []
pre_cluster_stats_null = []
post_cluster_stats_null = []
topology_counts = collections.Counter()
node_counts = collections.Counter()
read_gn = umi_methods.random_read_generator(infile.filename,
chrom=options.chrom,
umi_getter=umi_getter)
if options.chrom:
inreads = infile.fetch(reference=options.chrom)
else:
if options.per_gene and options.gene_transcript_map:
metacontig2contig = umi_methods.getMetaContig2contig(
infile, options.gene_transcript_map)
metatag = "MC"
inreads = umi_methods.metafetcher(infile, metacontig2contig,
metatag)
gene_tag = metatag
else:
inreads = infile.fetch()
gene_tag = options.gene_tag
for bundle, read_events, status in umi_methods.get_bundles(
inreads,
ignore_umi=options.ignore_umi,
subset=options.subset,
quality_threshold=options.mapping_quality,
paired=options.paired,
spliced=options.spliced,
soft_clip_threshold=options.soft,
per_contig=options.per_contig,
gene_tag=options.gene_tag,
skip_regex=options.skip_regex,
whole_contig=options.whole_contig,
read_length=options.read_length,
detection_method=options.detection_method,
umi_getter=umi_getter,
all_reads=False,
return_read2=False,
return_unmapped=False):
nInput += sum([bundle[umi]["count"] for umi in bundle])
if nOutput % 10000 == 0:
U.debug("Outputted %i" % nOutput)
if nInput % 1000000 == 0:
U.debug("Read %i input reads" % nInput)
if options.stats:
# generate pre-dudep stats
average_distance = umi_methods.get_average_umi_distance(
bundle.keys())
pre_cluster_stats.append(average_distance)
cluster_size = len(bundle)
random_umis = read_gn.getUmis(cluster_size)
average_distance_null = umi_methods.get_average_umi_distance(
random_umis)
pre_cluster_stats_null.append(average_distance_null)
if options.ignore_umi:
for umi in bundle:
nOutput += 1
outfile.write(bundle[umi]["read"])
else:
# set up ReadCluster functor with methods specific to
# specified options.method
processor = network.ReadDeduplicator(options.method)
# dedup using umis and write out deduped bam
reads, umis, umi_counts = processor(bundle=bundle,
threshold=options.threshold)
for read in reads:
outfile.write(read)
nOutput += 1
if options.stats:
# collect pre-dudupe stats
stats_pre_df_dict['UMI'].extend(bundle)
stats_pre_df_dict['counts'].extend(
[bundle[UMI]['count'] for UMI in bundle])
# collect post-dudupe stats
post_cluster_umis = [umi_getter(x) for x in reads]
stats_post_df_dict['UMI'].extend(umis)
stats_post_df_dict['counts'].extend(umi_counts)
average_distance = umi_methods.get_average_umi_distance(
post_cluster_umis)
post_cluster_stats.append(average_distance)
cluster_size = len(post_cluster_umis)
random_umis = read_gn.getUmis(cluster_size)
average_distance_null = umi_methods.get_average_umi_distance(
random_umis)
post_cluster_stats_null.append(average_distance_null)
outfile.close()
if options.stats:
# generate the stats dataframe
stats_pre_df = pd.DataFrame(stats_pre_df_dict)
stats_post_df = pd.DataFrame(stats_post_df_dict)
# tally the counts per umi per position
pre_counts = collections.Counter(stats_pre_df["counts"])
post_counts = collections.Counter(stats_post_df["counts"])
counts_index = list(
set(pre_counts.keys()).union(set(post_counts.keys())))
counts_index.sort()
with U.openFile(options.stats + "_per_umi_per_position.tsv",
"w") as outf:
outf.write("counts\tinstances_pre\tinstances_post\n")
for count in counts_index:
values = (count, pre_counts[count], post_counts[count])
outf.write("\t".join(map(str, values)) + "\n")
# aggregate stats pre/post per UMI
agg_pre_df = aggregateStatsDF(stats_pre_df)
agg_post_df = aggregateStatsDF(stats_post_df)
agg_df = pd.merge(agg_pre_df,
agg_post_df,
how='left',
left_index=True,
right_index=True,
sort=True,
suffixes=["_pre", "_post"])
# TS - if count value not observed either pre/post-dedup,
# merge will leave an empty cell and the column will be cast as a float
# see http://pandas.pydata.org/pandas-docs/dev/missing_data.html
# --> Missing data casting rules and indexing
# so, back fill with zeros and convert back to int
agg_df = agg_df.fillna(0).astype(int)
agg_df.index = [x.decode() for x in agg_df.index]
agg_df.index.name = 'UMI'
agg_df.to_csv(options.stats + "_per_umi.tsv", sep="\t")
# bin distances into integer bins
max_ed = int(
max(
map(max, [
pre_cluster_stats, post_cluster_stats,
pre_cluster_stats_null, post_cluster_stats_null
])))
cluster_bins = range(-1, int(max_ed) + 2)
def bin_clusters(cluster_list, bins=cluster_bins):
''' take list of floats and return bins'''
return np.digitize(cluster_list, bins, right=True)
def tallyCounts(binned_cluster, max_edit_distance):
''' tally counts per bin '''
return np.bincount(binned_cluster, minlength=max_edit_distance + 3)
pre_cluster_binned = bin_clusters(pre_cluster_stats)
post_cluster_binned = bin_clusters(post_cluster_stats)
pre_cluster_null_binned = bin_clusters(pre_cluster_stats_null)
post_cluster_null_binned = bin_clusters(post_cluster_stats_null)
edit_distance_df = pd.DataFrame({
"unique":
tallyCounts(pre_cluster_binned, max_ed),
"unique_null":
tallyCounts(pre_cluster_null_binned, max_ed),
options.method:
tallyCounts(post_cluster_binned, max_ed),
"%s_null" % options.method:
tallyCounts(post_cluster_null_binned, max_ed),
"edit_distance":
cluster_bins
})
# TS - set lowest bin (-1) to "Single_UMI"
edit_distance_df['edit_distance'][0] = "Single_UMI"
edit_distance_df.to_csv(options.stats + "_edit_distance.tsv",
index=False,
sep="\t")
# write footer and output benchmark information.
U.info(
"%s" %
", ".join(["%s: %s" % (x[0], x[1])
for x in read_events.most_common()]))
U.info("Number of reads out: %i" % nOutput)
U.Stop()
def main(argv=None):
"""script main.
parses command line options in sys.argv, unless *argv* is given.
"""
if argv is None:
argv = sys.argv
# setup command line parser
parser = U.OptionParser(version="%prog version: $Id$",
usage=globals()["__doc__"])
group = U.OptionGroup(parser, "group-specific options")
group.add_option(
"--group-out",
dest="tsv",
type="string",
help="Outfile name for file mapping read id to read group",
default=None)
group.add_option(
"--output-bam",
dest="output_bam",
action="store_true",
default=False,
help=("output a bam file with read groups tagged using the UG tag"
"[default=%default]"))
group.add_option(
"--output-unmapped",
dest="output_unmapped",
action="store_true",
default=False,
help=("Retain all unmapped reads in output[default=%default]"))
parser.add_option("--umi-group-tag",
dest="umi_group_tag",
type="string",
help="tag for the outputted umi group",
default='BX')
parser.add_option_group(group)
# add common options (-h/--help, ...) and parse command line
(options, args) = U.Start(parser, argv=argv)
U.validateSamOptions(options)
if options.stdin != sys.stdin:
in_name = options.stdin.name
options.stdin.close()
else:
raise ValueError("Input on standard in not currently supported")
if options.stdout != sys.stdout:
if options.no_sort_output:
out_name = options.stdout.name
else:
out_name = U.getTempFilename()
sorted_out_name = options.stdout.name
options.stdout.close()
assert options.output_bam, (
"To output a bam you must include --output-bam option")
else:
if options.no_sort_output:
out_name = "-"
else:
out_name = U.getTempFilename()
sorted_out_name = "-"
if not options.no_sort_output: # need to determine the output format for sort
if options.out_sam:
sort_format = "sam"
else:
sort_format = "bam"
if options.in_sam:
in_mode = "r"
else:
in_mode = "rb"
if options.out_sam:
out_mode = "wh"
else:
out_mode = "wb"
infile = pysam.Samfile(in_name, in_mode)
if options.output_bam:
outfile = pysam.Samfile(out_name, out_mode, template=infile)
else:
outfile = None
if options.tsv:
mapping_outfile = U.openFile(options.tsv, "w")
mapping_outfile.write("%s\n" % "\t".join([
"read_id", "contig", "position", "gene", "umi", "umi_count",
"final_umi", "final_umi_count", "unique_id"
]))
nInput, nOutput, unique_id, input_reads, output_reads = 0, 0, 0, 0, 0
gene_tag = options.gene_tag
metacontig2contig = None
if options.chrom:
inreads = infile.fetch(reference=options.chrom)
else:
if options.per_gene and options.gene_transcript_map:
metacontig2contig = umi_methods.getMetaContig2contig(
infile, options.gene_transcript_map)
metatag = "MC"
inreads = umi_methods.metafetcher(infile, metacontig2contig,
metatag)
gene_tag = metatag
else:
inreads = infile.fetch(until_eof=options.output_unmapped)
bundle_iterator = umi_methods.get_bundles(
options,
all_reads=True,
return_read2=True,
return_unmapped=options.output_unmapped,
metacontig_contig=metacontig2contig)
for bundle, key, status in bundle_iterator(inreads):
# write out read2s and unmapped (if these options are set)
if status == 'single_read':
# bundle is just a single read here
nInput += 1
if outfile:
outfile.write(bundle)
nOutput += 1
continue
umis = bundle.keys()
counts = {umi: bundle[umi]["count"] for umi in umis}
nInput += sum(counts.values())
while nOutput >= output_reads + 10000:
output_reads += 10000
U.info("Written out %i reads" % output_reads)
while nInput >= input_reads + 1000000:
input_reads += 1000000
U.info("Parsed %i input reads" % input_reads)
# set up UMIClusterer functor with methods specific to
# specified options.method
processor = network.UMIClusterer(options.method)
# group the umis
groups = processor(umis, counts, threshold=options.threshold)
for umi_group in groups:
top_umi = umi_group[0]
group_count = sum(counts[umi] for umi in umi_group)
for umi in umi_group:
reads = bundle[umi]['read']
for read in reads:
if outfile:
# Add the 'UG' tag to the read
read.tags += [('UG', unique_id)]
read.tags += [(options.umi_group_tag, top_umi)]
outfile.write(read)
if options.tsv:
if options.per_gene:
gene = read.get_tag(gene_tag)
else:
gene = "NA"
mapping_outfile.write("%s\n" % "\t".join(
map(str,
(read.query_name, read.reference_name,
umi_methods.get_read_position(
read, options.soft_clip_threshold)[1],
gene, umi.decode(), counts[umi],
top_umi.decode(), group_count, unique_id))))
nOutput += 1
unique_id += 1
if outfile:
outfile.close()
if not options.no_sort_output:
# sort the output
pysam.sort("-o", sorted_out_name, "-O", sort_format, out_name)
os.unlink(out_name) # delete the tempfile
if options.tsv:
mapping_outfile.close()
# write footer and output benchmark information.
U.info("Reads: %s" % ", ".join([
"%s: %s" % (x[0], x[1])
for x in bundle_iterator.read_events.most_common()
]))
U.info("Number of reads out: %i, Number of groups: %i" %
(nOutput, unique_id))
U.Stop()
def main(argv=None):
"""script main.
parses command line options in sys.argv, unless *argv* is given.
"""
if argv is None:
argv = sys.argv
# setup command line parser
parser = U.OptionParser(version="%prog version: $Id$",
usage=globals()["__doc__"])
parser.add_option("-p",
"--bc-pattern",
dest="pattern",
type="string",
help="Barcode pattern")
parser.add_option("--bc-pattern2",
dest="pattern2",
type="string",
help="Barcode pattern for paired reads")
parser.add_option("--3prime",
dest="prime3",
action="store_true",
help="barcode is on 3' end of read.")
parser.add_option("--read2-in",
dest="read2_in",
type="string",
help="file name for read pairs")
parser.add_option("--extract-method",
dest="extract_method",
type="choice",
choices=["string", "regex"],
help=("How to extract the umi +/- cell barcodes, Choose "
"from 'string' or 'regex'"))
parser.add_option("--plot-prefix",
dest="plot_prefix",
type="string",
help=("Prefix for plots to visualise the automated "
"detection of the number of 'true' cell barcodes"))
parser.add_option("--subset-reads",
dest="subset_reads",
type="int",
help=("Use the first N reads to automatically identify "
"the true cell barcodes. If N is greater than the "
"number of reads, all reads will be used"))
parser.add_option("--error-correct-threshold",
dest="error_correct_threshold",
type="int",
help=("Hamming distance for correction of "
"barcodes to whitelist barcodes"))
parser.add_option("--method",
dest="method",
choices=["reads", "umis"],
help=("Use reads or unique umi counts per cell"))
parser.add_option("--expect-cells",
dest="expect_cells",
type="int",
help=("Prior expectation on the upper limit on the "
"number of cells sequenced"))
parser.add_option("--set-cell-number",
dest="cell_number",
type="int",
help=("Specify the number of cell barcodes to accept"))
parser.set_defaults(method="reads",
extract_method="string",
filter_cell_barcodes=False,
whitelist_tsv=None,
blacklist_tsv=None,
error_correct_threshold=1,
pattern=None,
pattern2=None,
read2_in=None,
plot_prefix=None,
subset_reads=100000000,
expect_cells=False,
cell_number=False)
# add common options (-h/--help, ...) and parse command line
(options, args) = U.Start(parser,
argv=argv,
add_group_dedup_options=False,
add_sam_options=False)
if options.expect_cells and options.cell_number:
U.error("Cannot supply both --expect-cells and "
"--cell-number options")
if not options.pattern and not options.pattern2:
if not options.read2_in:
U.error("Must supply --bc-pattern for single-end")
else:
U.error("Must supply --bc-pattern and/or --bc-pattern2 "
"if paired-end ")
if options.pattern2:
if not options.read2_in:
U.error("must specify a paired fastq ``--read2-in``")
if not options.pattern2:
options.pattern2 = options.pattern
extract_cell = False
extract_umi = False
# If the pattern is a regex we can compile the regex(es) prior to
# ExtractFilterAndUpdate instantiation
if options.extract_method == "regex":
if options.pattern:
try:
options.pattern = regex.compile(options.pattern)
except regex.error:
U.error("barcode_regex '%s' is not a "
"valid regex" % options.pattern)
if options.pattern2:
try:
options.pattern2 = regex.compile(options.barcode_regex2)
except regex.Error:
U.error("barcode_regex2 '%s' is not a "
"valid regex" % options.barcode_regex2)
# check whether the regex contains a umi group(s) and cell groups(s)
if options.extract_method == "regex":
if options.pattern:
for group in options.pattern.groupindex:
if group.startswith("cell_"):
extract_cell = True
elif group.startswith("umi_"):
extract_umi = True
if options.pattern2:
for group in options.pattern2.groupindex:
if group.startswith("cell_"):
extract_cell = True
elif group.startswith("umi_"):
extract_umi = True
# check whether the pattern string contains umi/cell bases
elif options.extract_method == "string":
if options.pattern:
if "C" in options.pattern:
extract_cell = True
if "N" in options.pattern:
extract_umi = True
if options.pattern2:
if "C" in options.pattern2:
extract_cell = True
if "N" in options.pattern2:
extract_umi = True
if not extract_umi:
if options.extract_method == "string":
U.error("barcode pattern(s) do not include any umi bases "
"(marked with 'Ns') %s, %s" %
(options.pattern, options.pattern2))
elif options.extract_method == "regex":
U.error("barcode regex(es) do not include any umi groups "
"(starting with 'umi_') %s, %s" (options.pattern,
options.pattern2))
if not extract_cell:
if options.extract_method == "string":
U.error("barcode pattern(s) do not include any cell bases "
"(marked with 'Cs') %s, %s" %
(options.pattern, options.pattern2))
elif options.extract_method == "regex":
U.error("barcode regex(es) do not include any cell groups "
"(starting with 'cell_') %s, %s" (options.pattern,
options.pattern2))
read1s = umi_methods.fastqIterate(options.stdin)
# set up read extractor
ReadExtractor = umi_methods.ExtractFilterAndUpdate(
method=options.extract_method,
pattern=options.pattern,
pattern2=options.pattern2,
prime3=options.prime3,
extract_cell=extract_cell)
cell_barcode_counts = collections.Counter()
n_reads = 0
n_cell_barcodes = 0
# if using the umis method, need to keep a set of umis observed
if options.method == "umis":
cell_barcode_umis = collections.defaultdict(set)
# variables for progress monitor
displayMax = 100000
U.info("Starting barcode extraction")
if not options.read2_in:
for read1 in read1s:
# Update display in every 100kth iteration
if n_reads % displayMax == 0:
U.info("Parsed {} reads".format(n_reads))
n_reads += 1
barcode_values = ReadExtractor.getBarcodes(read1)
if barcode_values is None:
continue
else:
cell, umi, _, _, _, _, _ = barcode_values
if options.method == "umis":
cell_barcode_umis[cell].add(umi)
else:
cell_barcode_counts[cell] += 1
n_cell_barcodes += 1
if options.subset_reads:
if n_cell_barcodes > options.subset_reads:
break
else:
read2s = umi_methods.fastqIterate(U.openFile(options.read2_in))
for read1, read2 in izip(read1s, read2s):
# Update display in every 100kth iteration
if n_reads % displayMax == 0:
U.info("Parsed {} reads".format(n_reads))
n_reads += 1
barcode_values = ReadExtractor.getBarcodes(read1, read2)
if barcode_values is None:
continue
else:
cell, umi, _, _, _, _, _ = barcode_values
if options.method == "umis":
cell_barcode_umis[cell].add(umi)
else:
cell_barcode_counts[cell] += 1
n_cell_barcodes += 1
if options.subset_reads:
if n_reads > options.subset_reads:
break
U.info("Starting - whitelist determination")
if options.method == "umis":
for cell in cell_barcode_umis:
cell_barcode_counts[cell] = len(cell_barcode_umis[cell])
if options.cell_number and options.cell_number > len(cell_barcode_counts):
raise ValueError(
"--set-cell-barcode option specifies more cell barcodes than the "
"number of observed cell barcodes. This may be because "
"--subset-reads was set to a value too low to capture reads from "
"all cells. %s cell barcodes observed from %s parsed reads. "
"Expected>= %s cell barcodes" %
(len(cell_barcode_counts), options.subset_reads,
options.cell_number))
cell_whitelist, true_to_false_map = umi_methods.getCellWhitelist(
cell_barcode_counts, options.expect_cells, options.cell_number,
options.error_correct_threshold, options.plot_prefix)
U.info("Writing out whitelist")
for barcode in sorted(list(cell_whitelist)):
if true_to_false_map:
corrected_barcodes = ",".join(sorted(true_to_false_map[barcode]))
corrected_barcode_counts = ",".join(
map(str, [
cell_barcode_counts[x]
for x in sorted(true_to_false_map[barcode])
]))
else:
corrected_barcodes, corrected_barcode_counts = "", ""
options.stdout.write(
"%s\t%s\t%s\t%s\n" %
(barcode, corrected_barcodes, cell_barcode_counts[barcode],
corrected_barcode_counts))
U.info("Parsed %i reads" % n_reads)
U.info("%i reads matched the barcode pattern" % n_cell_barcodes)
U.info("Found %i unique cell barcodes" % len(cell_barcode_counts))
U.Stop()
def main(argv=None):
"""script main.
parses command line options in sys.argv, unless *argv* is given.
"""
if argv is None:
argv = sys.argv
# setup command line parser
parser = U.OptionParser(version="%prog version: $Id$",
usage=globals()["__doc__"])
parser.add_option("-i",
"--in-sam",
dest="in_sam",
action="store_true",
help="Input file is in sam format [default=%default]",
default=False)
parser.add_option(
"-o",
"--out-sam",
dest="out_sam",
action="store_true",
help="Output alignments in sam format [default=%default]",
default=False)
parser.add_option("--umi-separator",
dest="umi_sep",
type="string",
help="separator between read id and UMI",
default="_")
parser.add_option("--umi-tag",
dest="umi_tag",
type="string",
help="tag containing umi",
default='RX')
parser.add_option("--umi-group-tag",
dest="umi_group_tag",
type="string",
help="tag for the outputted umi group",
default='BX')
parser.add_option("--extract-umi-method",
dest="get_umi_method",
type="choice",
choices=("read_id", "tag"),
default="read_id",
help="where is the read UMI encoded? [default=%default]")
parser.add_option("--subset",
dest="subset",
type="float",
help="Use only a fraction of reads, specified by subset",
default=None)
parser.add_option("--spliced-is-unique",
dest="spliced",
action="store_true",
help="Treat a spliced read as different to an unspliced"
" one [default=%default]",
default=False)
parser.add_option("--soft-clip-threshold",
dest="soft",
type="float",
help="number of bases clipped from 5' end before"
"read is counted as spliced [default=%default]",
default=4)
parser.add_option("--edit-distance-threshold",
dest="threshold",
type="int",
default=1,
help="Edit distance theshold at which to join two UMIs"
"when clustering. [default=%default]")
parser.add_option("--chrom",
dest="chrom",
type="string",
help="Restrict to one chromosome",
default=None)
parser.add_option("--paired",
dest="paired",
action="store_true",
default=False,
help="paired BAM. [default=%default]")
parser.add_option("--method",
dest="method",
type="choice",
choices=("adjacency", "directional", "unique",
"cluster"),
default="directional",
help="method to use for umi deduping [default=%default]")
parser.add_option("--per-contig",
dest="per_contig",
action="store_true",
default=False,
help=("dedup per contig (field 3 in BAM; RNAME),"
" e.g for transcriptome where contig = gene"))
parser.add_option("--per-gene",
dest="per_gene",
action="store_true",
default=False,
help=("Deduplicate per gene,"
"e.g for transcriptome where contig = transcript"
"must also provide a transript to gene map with"
"--gene-transcript-map [default=%default]"))
parser.add_option("--gene-transcript-map",
dest="gene_transcript_map",
type="string",
help="file mapping transcripts to genes (tab separated)",
default=None)
parser.add_option("--gene-tag",
dest="gene_tag",
type="string",
help=("Deduplicate per gene where gene is"
"defined by this bam tag [default=%default]"),
default=None)
parser.add_option(
"--read-length",
dest="read_length",
action="store_true",
default=False,
help=("use read length in addition to position and UMI"
"to identify possible duplicates [default=%default]"))
parser.add_option("--mapping-quality",
dest="mapping_quality",
type="int",
help="Minimum mapping quality for a read to be retained"
" [default=%default]",
default=0)
parser.add_option(
"--output-unmapped",
dest="output_unmapped",
action="store_true",
default=False,
help=("Retain all unmapped reads in output[default=%default]"))
parser.add_option(
"--group-out",
dest="tsv",
type="string",
help="Outfile name for file mapping read id to read group",
default=None)
parser.add_option(
"--output-bam",
dest="output_bam",
action="store_true",
default=False,
help=("output a bam file with read groups tagged using the UG tag"
"[default=%default]"))
parser.add_option(
"--skip-tags-regex",
dest="skip_regex",
type="string",
help=("Used with --gene-tag. "
"Ignore reads where the gene-tag matches this regex"),
default="^[__|Unassigned]")
# add common options (-h/--help, ...) and parse command line
(options, args) = U.Start(parser, argv=argv)
if options.stdin != sys.stdin:
in_name = options.stdin.name
options.stdin.close()
else:
raise ValueError("Input on standard in not currently supported")
if options.stdout != sys.stdout:
out_name = options.stdout.name
options.stdout.close()
assert options.output_bam, (
"To output a bam you must include --output-bam option")
else:
out_name = "-"
if options.in_sam:
in_mode = "r"
else:
in_mode = "rb"
if options.out_sam:
out_mode = "wh"
else:
out_mode = "wb"
if options.per_gene:
if not options.gene_transcript_map:
raise ValueError(
"--per-gene option requires --gene-transcript-map")
infile = pysam.Samfile(in_name, in_mode)
if options.output_bam:
outfile = pysam.Samfile(out_name, out_mode, template=infile)
else:
outfile = None
if options.tsv:
mapping_outfile = U.openFile(options.tsv, "w")
mapping_outfile.write("%s\n" % "\t".join([
"read_id", "contig", "position", "gene", "umi", "umi_count",
"final_umi", "final_umi_count", "unique_id"
]))
# set the method with which to extract umis from reads
if options.get_umi_method == "read_id":
umi_getter = partial(umi_methods.get_umi_read_id, sep=options.umi_sep)
elif options.get_umi_method == "tag":
umi_getter = partial(umi_methods.get_umi_tag, tag=options.umi_tag)
else:
raise ValueError("Unknown umi extraction method")
nInput, nOutput, unique_id = 0, 0, 0
if options.chrom:
inreads = infile.fetch(reference=options.chrom)
gene_tag = options.gene_tag
else:
if options.per_gene and options.gene_transcript_map:
metacontig2contig = umi_methods.getMetaContig2contig(
infile, options.gene_transcript_map)
metatag = "MC"
inreads = umi_methods.metafetcher(infile, metacontig2contig,
metatag)
gene_tag = metatag
else:
inreads = infile.fetch(until_eof=options.output_unmapped)
gene_tag = options.gene_tag
for bundle, read_events, status in umi_methods.get_bundles(
inreads,
ignore_umi=False,
subset=options.subset,
quality_threshold=options.mapping_quality,
paired=options.paired,
spliced=options.spliced,
soft_clip_threshold=options.soft,
per_contig=options.per_contig,
gene_tag=gene_tag,
skip_regex=options.skip_regex,
read_length=options.read_length,
umi_getter=umi_getter,
all_reads=True,
return_read2=True,
return_unmapped=options.output_unmapped):
# write out read2s and unmapped if option set
if status == 'single_read':
# bundle is just a single read here
outfile.write(bundle)
nInput += 1
nOutput += 1
continue
umis = bundle.keys()
counts = {umi: bundle[umi]["count"] for umi in umis}
nInput += sum(counts.values())
if nOutput % 10000 == 0:
U.debug("Outputted %i" % nOutput)
if nInput % 1000000 == 0:
U.debug("Read %i input reads" % nInput)
# set up UMIClusterer functor with methods specific to
# specified options.method
processor = network.UMIClusterer(options.method)
# group the umis
groups = processor(umis, counts, threshold=options.threshold)
for umi_group in groups:
top_umi = umi_group[0]
group_count = sum(counts[umi] for umi in umi_group)
for umi in umi_group:
reads = bundle[umi]['read']
for read in reads:
if outfile:
# Add the 'UG' tag to the read
read.tags += [('UG', unique_id)]
read.tags += [(options.umi_group_tag, top_umi)]
outfile.write(read)
if options.tsv:
if options.per_gene:
gene = read.get_tag(gene_tag)
else:
gene = "NA"
mapping_outfile.write("%s\n" % "\t".join(
map(str,
(read.query_name, read.reference_name,
umi_methods.get_read_position(
read, options.soft)[1],
gene, umi.decode(), counts[umi],
top_umi.decode(), group_count, unique_id))))
nOutput += 1
unique_id += 1
if outfile:
outfile.close()
if options.tsv:
mapping_outfile.close()
# write footer and output benchmark information.
U.info(
"Reads: %s" %
", ".join(["%s: %s" % (x[0], x[1])
for x in read_events.most_common()]))
U.info("Number of reads out: %i, Number of groups: %i" %
(nOutput, unique_id))
U.Stop()
def main(argv=None):
"""script main.
parses command line options in sys.argv, unless *argv* is given.
"""
if argv is None:
argv = sys.argv
# setup command line parser
parser = U.OptionParser(version="%prog version: $Id$",
usage=usage,
description=globals()["__doc__"])
group = U.OptionGroup(parser, "whitelist-specific options")
group.add_option("--plot-prefix",
dest="plot_prefix", type="string",
help=("Prefix for plots to visualise the automated "
"detection of the number of 'true' cell barcodes"))
group.add_option("--subset-reads",
dest="subset_reads", type="int",
help=("Use the first N reads to automatically identify "
"the true cell barcodes. If N is greater than the "
"number of reads, all reads will be used. "
"Default is 100,000,000"))
group.add_option("--error-correct-threshold",
dest="error_correct_threshold",
type="int",
help=("Hamming distance for correction of barcodes to "
"whitelist barcodes. This value will also be used "
"for error detection above the knee if required "
"(--ed-above-threshold)"))
group.add_option("--method",
dest="method",
choices=["reads", "umis"],
help=("Use reads or unique umi counts per cell"))
group.add_option("--knee-method",
dest="knee_method",
choices=["distance", "density"],
help=("Use distance or density methods for detection of knee"))
group.add_option("--expect-cells",
dest="expect_cells",
type="int",
help=("Prior expectation on the upper limit on the "
"number of cells sequenced"))
group.add_option("--allow-threshold-error",
dest="allow_threshold_error", action="store_true",
help=("Don't select a threshold. Will still "
"output the plots if requested (--plot-prefix)"))
group.add_option("--set-cell-number",
dest="cell_number",
type="int",
help=("Specify the number of cell barcodes to accept"))
parser.add_option("--ed-above-threshold",
dest="ed_above_threshold", type="choice",
choices=["discard", "correct"],
help=("Detect CBs above the threshold which may be "
"sequence errors from another CB and either "
"'discard' or 'correct'. Default=discard"))
parser.add_option_group(group)
parser.set_defaults(method="reads",
knee_method="distance",
extract_method="string",
whitelist_tsv=None,
blacklist_tsv=None,
error_correct_threshold=1,
pattern=None,
pattern2=None,
read2_in=None,
plot_prefix=None,
subset_reads=100000000,
expect_cells=False,
allow_threshold_error=False,
cell_number=False,
ed_above_threshold=None,
ignore_suffix=False)
# add common options (-h/--help, ...) and parse command line
(options, args) = U.Start(parser, argv=argv,
add_extract_options=True,
add_group_dedup_options=False,
add_umi_grouping_options=False,
add_sam_options=False)
if options.filtered_out and not options.extract_method == "regex":
U.error("Reads will not be filtered unless extract method is"
"set to regex (--extract-method=regex)")
if options.expect_cells:
if options.knee_method == "distance":
U.error("Cannot use --expect-cells with 'distance' knee "
"method. Switch to --knee-method=density if you want to "
"provide an expectation for the number of "
"cells. Alternatively, if you know the number of cell "
"barcodes, use --cell-number")
if options.cell_number:
U.error("Cannot supply both --expect-cells and "
"--cell-number options")
extract_cell, extract_umi = U.validateExtractOptions(options)
if not extract_cell:
if options.extract_method == "string":
U.error("barcode pattern(s) do not include any cell bases "
"(marked with 'Cs') %s, %s" % (
options.pattern, options.pattern2))
elif options.extract_method == "regex":
U.error("barcode regex(es) do not include any cell groups "
"(starting with 'cell_') %s, %s" (
options.pattern, options.pattern2))
read1s = umi_methods.fastqIterate(options.stdin)
# set up read extractor
ReadExtractor = extract_methods.ExtractFilterAndUpdate(
method=options.extract_method,
pattern=options.pattern,
pattern2=options.pattern2,
prime3=options.prime3,
extract_cell=extract_cell)
cell_barcode_counts = collections.Counter()
n_reads = 0
n_cell_barcodes = 0
# if using the umis method, need to keep a set of umis observed
if options.method == "umis":
cell_barcode_umis = collections.defaultdict(set)
# variables for progress monitor
displayMax = 100000
U.info("Starting barcode extraction")
if options.filtered_out:
filtered_out = U.openFile(options.filtered_out, "w")
if not options.read2_in:
for read1 in read1s:
# Update display in every 100kth iteration
if n_reads % displayMax == 0:
U.info("Parsed {} reads".format(n_reads))
n_reads += 1
barcode_values = ReadExtractor.getBarcodes(read1)
if barcode_values is None:
if options.filtered_out:
filtered_out.write(str(read1) + "\n")
continue
else:
cell, umi, _, _, _, _, _ = barcode_values
if options.method == "umis":
cell_barcode_umis[cell].add(umi)
else:
cell_barcode_counts[cell] += 1
n_cell_barcodes += 1
if options.subset_reads:
if n_cell_barcodes > options.subset_reads:
break
else:
if options.filtered_out2:
filtered_out2 = U.openFile(options.filtered_out2, "w")
read2s = umi_methods.fastqIterate(U.openFile(options.read2_in))
for read1, read2 in izip(read1s, read2s):
# Update display in every 100kth iteration
if n_reads % displayMax == 0:
U.info("Parsed {} reads".format(n_reads))
n_reads += 1
barcode_values = ReadExtractor.getBarcodes(read1, read2)
if barcode_values is None:
if options.filtered_out:
filtered_out.write(str(read1) + "\n")
if options.filtered_out2:
filtered_out2.write(str(read2) + "\n")
continue
else:
cell, umi, _, _, _, _, _ = barcode_values
if options.method == "umis":
cell_barcode_umis[cell].add(umi)
else:
cell_barcode_counts[cell] += 1
n_cell_barcodes += 1
if options.subset_reads:
if n_reads > options.subset_reads:
break
U.info("Starting - whitelist determination")
if options.method == "umis":
for cell in cell_barcode_umis:
cell_barcode_counts[cell] = len(cell_barcode_umis[cell])
if options.cell_number and options.cell_number > len(cell_barcode_counts):
raise ValueError(
"--set-cell-barcode option specifies more cell barcodes than the "
"number of observed cell barcodes. This may be because "
"--subset-reads was set to a value too low to capture reads from "
"all cells. %s cell barcodes observed from %s parsed reads. "
"Expected>= %s cell barcodes" % (
len(cell_barcode_counts),
options.subset_reads,
options.cell_number))
cell_whitelist, true_to_false_map = whitelist_methods.getCellWhitelist(
cell_barcode_counts,
options.knee_method,
options.expect_cells,
options.cell_number,
options.error_correct_threshold,
options.plot_prefix)
if cell_whitelist:
U.info("Top %s cell barcodes passed the selected threshold" %
len(cell_whitelist))
if options.ed_above_threshold:
cell_whitelist, true_to_false_map = whitelist_methods.errorDetectAboveThreshold(
cell_barcode_counts,
cell_whitelist,
true_to_false_map,
errors=options.error_correct_threshold,
resolution_method=options.ed_above_threshold)
if cell_whitelist:
U.info("Writing out whitelist")
total_correct_barcodes = 0
total_corrected_barcodes = 0
for barcode in sorted(list(cell_whitelist)):
total_correct_barcodes += cell_barcode_counts[barcode]
if true_to_false_map:
corrected_barcodes = ",".join(
sorted(true_to_false_map[barcode]))
correct_barcode_counts = [cell_barcode_counts[x] for x in
sorted(true_to_false_map[barcode])]
total_corrected_barcodes += sum(correct_barcode_counts)
corrected_barcode_counts = ",".join(
map(str, correct_barcode_counts))
else:
corrected_barcodes, corrected_barcode_counts = "", ""
options.stdout.write("%s\t%s\t%s\t%s\n" % (
barcode, corrected_barcodes, cell_barcode_counts[barcode],
corrected_barcode_counts))
else:
msg = ("No local minima was accepted. Recommend checking the plot "
"output and counts per local minima (requires `--plot-prefix`"
"option) and then re-running with manually selected threshold "
"(`--set-cell-number` option)")
if options.allow_threshold_error:
U.info(msg)
else:
U.error(msg)
U.info("Parsed %i reads" % n_reads)
U.info("%i reads matched the barcode pattern" % n_cell_barcodes)
U.info("Found %i unique cell barcodes" % len(cell_barcode_counts))
if cell_whitelist:
U.info("Found %i total reads matching the selected cell barcodes" %
total_correct_barcodes)
U.info("Found %i total reads which can be error corrected to the "
"selected cell barcodes" % total_corrected_barcodes)
if options.filtered_out:
filtered_out.close()
if options.filtered_out2:
filtered_out2.close()
U.Stop()
def main(argv=None):
"""script main.
parses command line options in sys.argv, unless *argv* is given.
"""
if argv is None:
argv = sys.argv
# setup command line parser
parser = U.OptionParser(version="%prog version: $Id$",
usage=globals()["__doc__"])
group = U.OptionGroup(parser, "count-specific options")
parser.add_option("--wide-format-cell-counts",
dest="wide_format_cell_counts",
action="store_true",
default=False,
help=("output the cell counts in a wide format "
"(rows=genes, columns=cells)"))
parser.add_option_group(group)
# add common options (-h/--help, ...) and parse command line
(options, args) = U.Start(parser, argv=argv, add_group_dedup_options=False)
options.per_gene = True # hardcodes counting to per-gene only
U.validateSamOptions(options, group=False)
if options.random_seed:
np.random.seed(options.random_seed)
if options.stdin != sys.stdin:
in_name = options.stdin.name
options.stdin.close()
else:
raise ValueError("Input on standard in not currently supported")
if options.in_sam:
in_mode = "r"
else:
in_mode = "rb"
infile = pysam.Samfile(in_name, in_mode)
# write out to tempfile and then sort to stdout
tmpfilename = U.getTempFilename(dir=options.tmpdir)
tmpfile = U.openFile(tmpfilename, mode="w")
nInput, nOutput, input_reads = 0, 0, 0
gene_tag = options.gene_tag
metacontig2contig = None
if options.chrom:
inreads = infile.fetch(reference=options.chrom)
else:
if options.gene_transcript_map:
metacontig2contig = umi_methods.getMetaContig2contig(
infile, options.gene_transcript_map)
metatag = "MC"
inreads = umi_methods.metafetcher(infile, metacontig2contig,
metatag)
gene_tag = metatag
else:
inreads = infile.fetch()
bundle_iterator = umi_methods.get_bundles(
options, only_count_reads=True, metacontig_contig=metacontig2contig)
for bundle, key, status in bundle_iterator(inreads):
if status == "single_read":
continue
gene, cell = key
umis = bundle.keys()
counts = {umi: bundle[umi]["count"] for umi in umis}
nInput += sum(counts.values())
while nInput >= input_reads + 1000000:
input_reads += 1000000
U.info("Parsed %i input reads" % input_reads)
# set up UMIClusterer functor with methods specific to
# specified options.method
processor = network.UMIClusterer(options.method)
# group the umis
groups = processor(umis, counts, threshold=options.threshold)
gene_count = len(groups)
if options.per_cell:
tmpfile.write("%s\n" % "\t".join(
(gene, cell.decode(), str(gene_count))))
else:
tmpfile.write("%s\n" % "\t".join((gene, str(gene_count))))
nOutput += gene_count
tmpfile.close()
if options.per_cell:
gene_counts_dict = {}
with U.openFile(tmpfilename, mode="r") as inf:
genes = set()
cells = set()
for line in inf:
gene, cell, gene_count = line.strip().split("\t")
genes.add(gene)
cells.add(cell)
if gene not in gene_counts_dict:
gene_counts_dict[gene] = {}
gene_counts_dict[gene][cell] = gene_count
if options.wide_format_cell_counts: # write out in wide format
options.stdout.write("%s\t%s\n" %
("gene", "\t".join(sorted(cells))))
for gene in sorted(genes):
counts = []
for cell in sorted(cells):
if cell in gene_counts_dict[gene]:
counts.append(gene_counts_dict[gene][cell])
else:
counts.append(0)
options.stdout.write("%s\t%s\n" %
(gene, "\t".join(map(str, counts))))
else: # write out in long format
options.stdout.write("%s\t%s\t%s\n" % ("gene", "cell", "count"))
for gene in sorted(genes):
for cell in sorted(list(gene_counts_dict[gene].keys())):
options.stdout.write(
"%s\t%s\t%s\n" %
(gene, cell, gene_counts_dict[gene][cell]))
else:
options.stdout.write("%s\t%s\n" % ("gene", "count"))
with U.openFile(tmpfilename, mode="r") as inf:
for line in inf:
options.stdout.write(line)
os.unlink(tmpfilename)
# output reads events and benchmark information.
for event in bundle_iterator.read_events.most_common():
U.info("%s: %s" % (event[0], event[1]))
U.info("Number of (post deduplication) reads counted: %i" % nOutput)
U.Stop()
def main(argv=None):
if argv is None:
argv = sys.argv
# setup command line parser
parser = U.OptionParser(version="%prog version: $Id$",
usage=usage,
description=globals()["__doc__"])
group = U.OptionGroup(parser, "RSEM preparation specific options")
group.add_option(
"--tags",
dest="tags",
type="string",
default="UG,BX",
help="Comma-seperated list of tags to transfer from read1 to read2")
group.add_option("--sam",
dest="sam",
action="store_true",
default=False,
help="input and output SAM rather than BAM")
parser.add_option_group(group)
# add common options (-h/--help, ...) and parse command line
(options, args) = U.Start(parser,
argv=argv,
add_group_dedup_options=False,
add_umi_grouping_options=False,
add_sam_options=False)
skipped_stats = Counter()
if options.stdin != sys.stdin:
in_name = options.stdin.name
options.stdin.close()
else:
in_name = "-"
if options.sam:
mode = ""
else:
mode = "b"
inbam = pysam.AlignmentFile(in_name, "r" + mode)
if options.stdout != sys.stdout:
out_name = options.stdout.name
options.stdout.close()
else:
out_name = "-"
outbam = pysam.AlignmentFile(out_name, "w" + mode, template=inbam)
options.tags = options.tags.split(",")
for template in chunk_bam(inbam):
assert len(set(r.query_name for r in template)) == 1
current_template = {True: defaultdict(list), False: defaultdict(list)}
for read in template:
key = (read.reference_name, read.pos, not read.is_secondary)
current_template[read.is_read1][key].append(read)
output = set()
for read in template:
mate = None
# if this read is a non_primary alignment, we first want to check if it has a mate
# with the non-primary alignment flag set.
mate_key_primary = (True)
mate_key_secondary = (read.next_reference_name,
read.next_reference_start, False)
# First look for a read that has the same primary/secondary status
# as read (i.e. secondary mate for secondary read, and primary mate
# for primary read)
mate_key = (read.next_reference_name, read.next_reference_start,
read.is_secondary)
mate = pick_mate(read, current_template, mate_key)
# If none was found then look for the opposite (primary mate of secondary
# read or seconadary mate of primary read)
if mate is None:
mate_key = (read.next_reference_name,
read.next_reference_start, not read.is_secondary)
mate = pick_mate(read, current_template, mate_key)
# If we still don't have a mate, then their can't be one?
if mate is None:
skipped_stats["no_mate"] += 1
U.warn("Alignment {} has no mate -- skipped".format("\t".join(
map(str, [
read.query_name, read.flag, read.reference_name,
int(read.pos)
]))))
continue
# because we might want to make changes to the read, but not have those changes reflected
# if we need the read again,we copy the read. This is only way I can find to do this.
read = pysam.AlignedSegment().from_dict(read.to_dict(),
read.header)
mate = pysam.AlignedSegment().from_dict(mate.to_dict(),
read.header)
# Make it so that if our read is secondary, the mate is also secondary. We don't make the
# mate primary if the read is primary because we would otherwise end up with mulitple
# primary alignments.
if read.is_secondary:
mate.is_secondary = True
# In a situation where there is already one mate for each read, then we will come across
# each pair twice - once when we scan read1 and once when we scan read2. Thus we need
# to make sure we don't output something already output.
if read.is_read1:
mate = copy_tags(options.tags, read, mate)
output_key = str(read) + str(mate)
if output_key not in output:
output.add(output_key)
outbam.write(read)
outbam.write(mate)
skipped_stats["pairs_output"] += 1
elif read.is_read2:
read = copy_tags(options.tags, mate, read)
output_key = str(mate) + str(read)
if output_key not in output:
output.add(output_key)
outbam.write(mate)
outbam.write(read)
skipped_stats["pairs_output"] += 1
else:
skipped_stats["skipped_not_read12"] += 1
U.warn("Alignment {} is neither read1 nor read2 -- skipped".
format("\t".join(
map(str, [
read.query_name, read.flag, read.reference_name,
int(read.pos)
]))))
continue
if not out_name == "-":
outbam.close()
U.info("Total pairs output: {}, Pairs skipped - no mates: {},"
" Pairs skipped - not read1 or 2: {}".format(
skipped_stats["pairs_output"], skipped_stats["no_mate"],
skipped_stats["skipped_not_read12"]))
U.Stop()
def main(argv=None):
"""script main.
parses command line options in sys.argv, unless *argv* is given.
"""
if argv is None:
argv = sys.argv
# setup command line parser
parser = U.OptionParser(version="%prog version: $Id$",
usage=globals()["__doc__"])
parser.add_option("-i",
"--in-sam",
dest="in_sam",
action="store_true",
help="Input file is in sam format [default=%default]",
default=False)
parser.add_option(
"-o",
"--out-sam",
dest="out_sam",
action="store_true",
help="Output alignments in sam format [default=%default]",
default=False)
parser.add_option("--umi-separator",
dest="umi_sep",
type="string",
help="separator between read id and UMI",
default="_")
parser.add_option("--umi-tag",
dest="umi_tag",
type="string",
help="tag containing umi",
default='RX')
parser.add_option("--umi-group-tag",
dest="umi_group_tag",
type="string",
help="tag for the outputted umi group",
default='BX')
parser.add_option("--extract-umi-method",
dest="get_umi_method",
type="choice",
choices=("read_id", "tag"),
default="read_id",
help="where is the read UMI encoded? [default=%default]")
parser.add_option("--subset",
dest="subset",
type="float",
help="Use only a fraction of reads, specified by subset",
default=None)
parser.add_option("--spliced-is-unique",
dest="spliced",
action="store_true",
help="Treat a spliced read as different to an unspliced"
" one [default=%default]",
default=False)
parser.add_option("--soft-clip-threshold",
dest="soft",
type="float",
help="number of bases clipped from 5' end before"
"read is counted as spliced [default=%default]",
default=4)
parser.add_option("--edit-distance-threshold",
dest="threshold",
type="int",
default=1,
help="Edit distance theshold at which to join two UMIs"
"when clustering. [default=%default]")
parser.add_option("--chrom",
dest="chrom",
type="string",
help="Restrict to one chromosome",
default=None)
parser.add_option("--paired",
dest="paired",
action="store_true",
default=False,
help="paired BAM. [default=%default]")
parser.add_option("--method",
dest="method",
type="choice",
choices=("adjacency", "directional", "unique",
"cluster"),
default="directional",
help="method to use for umi deduping [default=%default]")
parser.add_option("--per-contig",
dest="per_contig",
action="store_true",
default=False,
help=("dedup per contig,"
" e.g for transcriptome where contig = gene"))
parser.add_option(
"--whole-contig",
dest="whole_contig",
action="store_true",
default=False,
help=
"Read whole contig before outputting bundles: guarantees that no reads"
"are missed, but increases memory usage")
parser.add_option(
"--read-length",
dest="read_length",
action="store_true",
default=False,
help=("use read length in addition to position and UMI"
"to identify possible duplicates [default=%default]"))
parser.add_option("--mapping-quality",
dest="mapping_quality",
type="int",
help="Minimum mapping quality for a read to be retained"
" [default=%default]",
default=0)
parser.add_option(
"--group-out",
dest="tsv",
type="string",
help="Outfile name for file mapping read id to read group",
default=None)
parser.add_option(
"--output-bam",
dest="output_bam",
action="store_true",
default=False,
help=("output a bam file with read groups tagged using the UG tag"
"[default=%default]"))
# add common options (-h/--help, ...) and parse command line
(options, args) = U.Start(parser, argv=argv)
if options.stdin != sys.stdin:
in_name = options.stdin.name
options.stdin.close()
else:
raise ValueError("Input on standard in not currently supported")
if options.stdout != sys.stdout:
out_name = options.stdout.name
options.stdout.close()
assert options.output_bam, (
"To output a bam you must include --output-bam option")
else:
out_name = "-"
if options.in_sam:
in_mode = "r"
else:
in_mode = "rb"
if options.out_sam:
out_mode = "w"
else:
out_mode = "wb"
infile = pysam.Samfile(in_name, in_mode)
if options.output_bam:
outfile = pysam.Samfile(out_name, out_mode, template=infile)
if options.paired:
outfile = umi_methods.TwoPassPairWriter(infile, outfile, tags=True)
else:
outfile = None
if options.tsv:
mapping_outfile = U.openFile(options.tsv, "w")
mapping_outfile.write(
"read_id\tcontig\tposition\tumi\tumi_count\tfinal_umi\tfinal_umi_count\tunique_id\n"
)
# set the method with which to extract umis from reads
if options.get_umi_method == "read_id":
umi_getter = partial(umi_methods.get_umi_read_id, sep=options.umi_sep)
elif options.get_umi_method == "tag":
umi_getter = partial(umi_methods.get_umi_tag, tag=options.umi_tag)
else:
raise ValueError("Unknown umi extraction method")
nInput, nOutput, unique_id = 0, 0, 0
read_events = collections.Counter()
for bundle, read_events in umi_methods.get_bundles(
infile,
read_events,
ignore_umi=False,
subset=options.subset,
quality_threshold=options.mapping_quality,
paired=options.paired,
chrom=options.chrom,
spliced=options.spliced,
soft_clip_threshold=options.soft,
per_contig=options.per_contig,
whole_contig=options.whole_contig,
read_length=options.read_length,
umi_getter=umi_getter,
all_reads=True):
nInput += sum([bundle[umi]["count"] for umi in bundle])
if nOutput % 10000 == 0:
U.debug("Outputted %i" % nOutput)
if nInput % 1000000 == 0:
U.debug("Read %i input reads" % nInput)
# set up ReadCluster functor with methods specific to
# specified options.method
processor = network.ReadClusterer(options.method)
bundle, groups, counts = processor(bundle=bundle,
threshold=options.threshold,
stats=True,
deduplicate=False)
for umi_group in groups:
top_umi = umi_group[0]
group_count = sum(counts[umi] for umi in umi_group)
for umi in umi_group:
reads = bundle[umi]['read']
for read in reads:
if outfile:
if options.paired:
# if paired, we need to supply the tags to
# add to the paired read
outfile.write(read, unique_id, top_umi)
else:
# Add the 'UG' tag to the read
read.tags += [('UG', unique_id)]
read.tags += [(options.umi_group_tag, top_umi)]
outfile.write(read)
if options.tsv:
mapping_outfile.write("%s\n" % "\t".join(
map(str, (read.query_name, read.reference_name,
umi_methods.get_read_position(
read, options.soft)[1], umi.decode(),
counts[umi], top_umi.decode(),
group_count, unique_id))))
nOutput += 1
unique_id += 1
if outfile:
outfile.close()
if options.tsv:
mapping_outfile.close()
# write footer and output benchmark information.
U.info(
"Reads: %s" %
", ".join(["%s: %s" % (x[0], x[1])
for x in read_events.most_common()]))
U.info("Number of reads out: %i, Number of groups: %i" %
(nOutput, unique_id))
U.Stop()
def main(argv=None):
"""script main.
parses command line options in sys.argv, unless *argv* is given.
"""
if argv is None:
argv = sys.argv
# setup command line parser
parser = U.OptionParser(version="%prog version: $Id$",
usage=usage,
description=globals()["__doc__"])
group = U.OptionGroup(parser, "extract-specific options")
# (Experimental option) Retain the UMI in the sequence read"
group.add_option("--retain-umi", dest="retain_umi", action="store_true",
help=optparse.SUPPRESS_HELP)
group.add_option("--read2-out", dest="read2_out", type="string",
help="file to output processed paired read to")
group.add_option("--read2-stdout", dest="read2_stdout",
action="store_true",
help="Paired reads, send read2 to stdout, discarding read1")
group.add_option("--quality-filter-threshold",
dest="quality_filter_threshold", type="int",
help=("Remove reads where any UMI base quality score "
"falls below this threshold"))
group.add_option("--quality-filter-mask",
dest="quality_filter_mask", type="int",
help=("If a UMI base has a quality below this threshold, "
"replace the base with 'N'"))
group.add_option("--quality-encoding",
dest="quality_encoding", type="choice",
choices=["phred33", "phred64", "solexa"],
help=("Quality score encoding. Choose from 'phred33'"
"[33-77] 'phred64' [64-106] or 'solexa' [59-106]"))
group.add_option("--filter-cell-barcode",
dest="filter_cell_barcode",
action="store_true",
help=optparse.SUPPRESS_HELP)
group.add_option("--error-correct-cell",
dest="error_correct_cell",
action="store_true",
help=("Correct errors in the cell barcode"))
group.add_option("--whitelist",
dest="whitelist", type="string",
help=("A whitelist of accepted cell barcodes"))
group.add_option("--blacklist",
dest="blacklist", type="string",
help=("A blacklist of rejected cell barcodes"))
group.add_option("--filter-umi",
dest="filter_umi",
action="store_true",
#help="Filter the UMIs"
help=optparse.SUPPRESS_HELP)
group.add_option("--umi-whitelist", dest="umi_whitelist",
type="string", default=None,
#help="A whitelist of accepted UMIs [default=%default]"
help=optparse.SUPPRESS_HELP)
group.add_option("--umi-whitelist-paired", dest="umi_whitelist_paired",
type="string", default=None,
#help="A whitelist of accepted UMIs for read2[default=%default]"
help=optparse.SUPPRESS_HELP)
group.add_option("--correct-umi-threshold", dest="correct_umi_threshold",
type="int", default=0,
#help="Correct errors in UMIs to the whitelist(s) provided"
#"if within threshold [default=%default]"
help=optparse.SUPPRESS_HELP)
group.add_option("--umi-correct-log", dest="umi_correct_log",
type="string", default=None,
#help="File logging UMI error correction",
help=optparse.SUPPRESS_HELP)
group.add_option("--subset-reads", "--reads-subset",
dest="reads_subset", type="int",
help=("Only extract from the first N reads. If N is "
"greater than the number of reads, all reads will "
"be used"))
group.add_option("--reconcile-pairs",
dest="reconcile", action="store_true",
help=("Allow the presences of reads in read2 input that "
"are not present in read1 input. This allows cell "
"barcode filtering of read1s without "
"considering read2s"))
parser.add_option_group(group)
group = U.OptionGroup(parser, "[EXPERIMENTAl] barcode extraction options")
group.add_option("--either-read", dest="either_read", action="store_true",
help="UMI may be on either read (see "
"--either-read-resolve) for options to resolve cases where"
"UMI is on both reads")
group.add_option("--either-read-resolve",
dest="either_read_resolve", type="choice",
choices=["discard", "quality"],
help=("How to resolve instances where both reads "
"contain a UMI but using --either-read."
"Choose from 'discard' or 'quality'"
"(use highest quality). default=dicard"))
parser.add_option_group(group)
parser.set_defaults(extract_method="string",
filter_cell_barcodes=False,
whitelist=None,
blacklist=None,
error_correct_cell=False,
pattern=None,
pattern2=None,
read2_in=None,
read2_out=False,
read2_stdout=False,
quality_filter_threshold=None,
quality_encoding=None,
reconcile=False,
either_read=False,
either_read_resolve="discard",
ignore_suffix=False)
# add common options (-h/--help, ...) and parse command line
(options, args) = U.Start(parser, argv=argv,
add_extract_options=True,
add_group_dedup_options=False,
add_umi_grouping_options=False,
add_sam_options=False)
if options.filter_cell_barcode:
U.info('Use of --whitelist ensures cell barcodes are filtered. '
'--filter-cell-barcode is no longer required and may be '
'removed in future versions.')
if options.whitelist is not None:
options.filter_cell_barcode = True
if options.retain_umi and not options.extract_method == "regex":
U.error("option --retain-umi only works with --extract-method=regex")
if (options.filtered_out and not options.extract_method == "regex" and
whitelist is None):
U.error("Reads will not be filtered unless extract method is"
"set to regex (--extract-method=regex) or cell"
"barcodes are filtered (--whitelist)")
if options.quality_filter_threshold or options.quality_filter_mask:
if not options.quality_encoding:
U.error("must provide a quality encoding (--quality-"
"encoding) to filter UMIs by quality (--quality"
"-filter-threshold) or mask low quality bases "
"with (--quality-filter-mask)")
extract_cell, extract_umi = U.validateExtractOptions(options)
if options.either_read:
if extract_cell:
U.error("Option to extract from either read (--either-read) "
"is not currently compatible with cell barcode extraction")
if not options.extract_method == "regex":
U.error("Option to extract from either read (--either-read)"
"requires --extract-method=regex")
if not options.pattern or not options.pattern2:
U.error("Option to extract from either read (--either-read)"
"requires --bc-pattern=[PATTERN1] and"
"--bc-pattern2=[PATTERN2]")
if options.filter_umi:
if not options.umi_whitelist:
U.error("must provide a UMI whitelist (--umi-whitelist) if using "
"--filter-umi option")
if options.pattern2 and not options.umi_whitelist_paired:
U.error("must provide a UMI whitelist for paired end "
"(--umi-whitelist-paired) if using --filter-umi option"
"with paired end data")
if not extract_umi:
if options.extract_method == "string":
U.error("barcode pattern(s) do not include any umi bases "
"(marked with 'Ns') %s, %s" % (
options.pattern, options.pattern2))
elif options.extract_method == "regex":
U.error("barcode regex(es) do not include any umi groups "
"(starting with 'umi_') %s, %s" (
options.pattern, options.pattern2))
if options.whitelist:
if not extract_cell:
if options.extract_method == "string":
U.error("barcode pattern(s) do not include any cell bases "
"(marked with 'Cs') %s, %s" % (
options.pattern, options.pattern2))
elif options.extract_method == "regex":
U.error("barcode regex(es) do not include any cell groups "
"(starting with 'cell_') %s, %s" (
options.pattern, options.pattern2))
read1s = umi_methods.fastqIterate(options.stdin)
# set up read extractor
ReadExtractor = extract_methods.ExtractFilterAndUpdate(
options.extract_method,
options.pattern,
options.pattern2,
options.prime3,
extract_cell,
options.quality_encoding,
options.quality_filter_threshold,
options.quality_filter_mask,
options.filter_umi,
options.filter_cell_barcode,
options.retain_umi,
options.either_read,
options.either_read_resolve)
if options.filter_umi:
umi_whitelist, false_to_true_map = whitelist_methods.getUserDefinedBarcodes(
options.umi_whitelist,
options.umi_whitelist_paired,
deriveErrorCorrection=True,
threshold=options.correct_umi_threshold)
U.info("Length of whitelist: %i" % len(umi_whitelist))
U.info("Length of 'correctable' whitelist: %i" % len(false_to_true_map))
ReadExtractor.umi_whitelist = umi_whitelist
ReadExtractor.umi_false_to_true_map = false_to_true_map
ReadExtractor.umi_whitelist_counts = collections.defaultdict(
lambda: collections.Counter())
if options.whitelist:
cell_whitelist, false_to_true_map = whitelist_methods.getUserDefinedBarcodes(
options.whitelist,
getErrorCorrection=options.error_correct_cell)
ReadExtractor.cell_whitelist = cell_whitelist
ReadExtractor.false_to_true_map = false_to_true_map
if options.blacklist:
blacklist = set()
with U.openFile(options.blacklist, "r") as inf:
for line in inf:
blacklist.add(line.strip().split("\t")[0])
ReadExtractor.cell_blacklist = blacklist
# variables for progress monitor
progCount = 0
displayMax = 100000
U.info("Starting barcode extraction")
if options.filtered_out:
filtered_out = U.openFile(options.filtered_out, "w")
if options.read2_in is None:
for read in read1s:
# incrementing count for monitoring progress
progCount += 1
# Update display in every 100kth iteration
if progCount % displayMax == 0:
U.info("Parsed {} reads".format(progCount))
new_read = ReadExtractor(read)
if options.reads_subset:
if (ReadExtractor.read_counts['Input Reads'] >
options.reads_subset):
break
if not new_read:
if options.filtered_out:
filtered_out.write(str(read) + "\n")
continue
options.stdout.write(str(new_read) + "\n")
else:
if options.filtered_out2:
filtered_out2 = U.openFile(options.filtered_out2, "w")
read2s = umi_methods.fastqIterate(U.openFile(options.read2_in))
if options.read2_out:
read2_out = U.openFile(options.read2_out, "w")
if options.reconcile:
strict = False
else:
strict = True
for read1, read2 in umi_methods.joinedFastqIterate(
read1s, read2s, strict, options.ignore_suffix):
# incrementing count for monitoring progress
progCount += 1
# Update display in every 100kth iteration
if progCount % displayMax == 0:
U.info("Parsed {} reads".format(progCount))
sys.stdout.flush()
reads = ReadExtractor(read1, read2)
if options.reads_subset:
if (ReadExtractor.read_counts['Input Reads'] >
options.reads_subset):
break
if not reads:
if options.filtered_out:
filtered_out.write(str(read1) + "\n")
if options.filtered_out2:
filtered_out2.write(str(read2) + "\n")
continue
else:
new_read1, new_read2 = reads
if options.read2_stdout:
options.stdout.write(str(new_read2) + "\n")
else:
options.stdout.write(str(new_read1) + "\n")
if options.read2_out:
read2_out.write(str(new_read2) + "\n")
if options.read2_out:
read2_out.close()
if options.filtered_out:
filtered_out.close()
if options.filtered_out2:
filtered_out2.close()
for k, v in ReadExtractor.getReadCounts().most_common():
U.info("%s: %s" % (k, v))
if options.umi_correct_log:
with U.openFile(options.umi_correct_log, "w") as outf:
outf.write("umi\tcount_no_errors\tcount_errors\n")
for umi, counts in ReadExtractor.umi_whitelist_counts.items():
outf.write("%s\t%i\t%i\n" % (
umi, counts["no_error"], counts["error"]))
outf.close()
U.Stop()
def main(argv=None):
"""script main.
parses command line options in sys.argv, unless *argv* is given.
"""
if argv is None:
argv = sys.argv
# setup command line parser
parser = U.OptionParser(version="%prog version: $Id$",
usage=globals()["__doc__"])
parser.add_option("--split-barcode",
dest="split",
action="store_true",
help="barcode is split across read pair")
parser.add_option("-p",
"--bc-pattern",
dest="pattern",
type="string",
help="Barcode pattern. Ns are random bases X's fixed")
parser.add_option("--bc-pattern2",
dest="pattern2",
type="string",
help="Barcode pattern. Ns are random bases X's fixed")
parser.add_option("--read2-in",
dest="read2_in",
type="string",
help="file name for read pairs")
parser.add_option("--3prime",
dest="prime3",
action="store_true",
help="barcode is on 3' end of read")
parser.add_option("--read2-out",
dest="read2_out",
type="string",
help="file to output processed paired read to")
parser.add_option("--quality-filter-threshold",
dest="quality_filter_threshold",
type="int",
help=("Remove reads where any UMI base quality score "
"falls below this threshold"))
parser.add_option("--quality-encoding",
dest="quality_encoding",
type="choice",
choices=["phred33", "phred64", "solexa"],
help=("Quality score encoding. Choose from phred33"
"[33-77] phred64 [64-106] or solexa [59-106]"))
parser.add_option("--supress-stats",
dest="stats",
action="store_false",
help="Suppress the writing of stats to the log")
parser.set_defaults(split=False,
pattern=None,
pattern2=None,
read2_in=None,
read2_out=None,
prime3=False,
stats=True,
quality_filter_threshold=None,
quality_encoding=None)
# add common options (-h/--help, ...) and parse command line
(options, args) = U.Start(parser, argv=argv)
# check options
if not options.pattern:
raise ValueError("must specify a pattern using ``--bc-pattern``")
if options.split:
if not options.read2_in:
raise ValueError("must specify a paired fastq ``--read2-in``")
if not options.pattern2:
options.pattern2 = options.pattern
if options.read2_in:
if not options.read2_out:
raise ValueError("must specify an output for the paired end "
"``--read2-out``")
if options.quality_filter_threshold:
if not options.quality_encoding:
raise ValueError("must provide a quality encoding to filter UMIs "
"by quality ``--quality-encoding``")
# Initialise the processor
processor = Extractor(options.pattern, options.pattern2,
options.quality_filter_threshold,
options.quality_encoding, options.prime3)
read1s = fastqIterate(options.stdin)
if options.read2_in is None:
for read in read1s:
new_1 = processor(read)
if new_1:
options.stdout.write(str(new_1) + "\n")
else:
read2s = fastqIterate(U.openFile(options.read2_in))
read2_out = U.openFile(options.read2_out, "w")
for read1, read2 in izip(read1s, read2s):
new_1, new_2 = processor(read1, read2)
if new_1:
options.stdout.write(str(new_1) + "\n")
read2_out.write(str(new_2) + "\n")
# write footer and output benchmark information.
if options.stats:
options.stdlog.write("\t".join(["Barcode", "UMI", "Sample", "Count"]) +
"\n")
for id in processor.bc_count:
options.stdlog.write("\t".join(id +
(str(processor.bc_count[id]), )) +
"\n")
U.Stop()
def main(argv=None):
"""script main.
parses command line options in sys.argv, unless *argv* is given.
"""
if argv is None:
argv = sys.argv
# setup command line parser
parser = U.OptionParser(version="%prog version: $Id$",
usage=globals()["__doc__"])
parser.add_option("-p",
"--bc-pattern",
dest="pattern",
type="string",
help="Barcode pattern")
parser.add_option("--bc-pattern2",
dest="pattern2",
type="string",
help="Barcode pattern for paired reads")
parser.add_option("--3prime",
dest="prime3",
action="store_true",
help="barcode is on 3' end of read.")
parser.add_option("--read2-in",
dest="read2_in",
type="string",
help="file name for read pairs")
parser.add_option("--read2-out",
dest="read2_out",
type="string",
help="file to output processed paired read to")
parser.add_option(
"--read2-out-only",
dest="read2_out_only",
action="store_true",
help="Paired reads, only output the second read in the pair")
parser.add_option("--quality-filter-threshold",
dest="quality_filter_threshold",
type="int",
help=("Remove reads where any UMI base quality score "
"falls below this threshold"))
parser.add_option(
"--quality-filter-mask",
dest="quality_filter_mask",
type="int",
help=("If a UMI base has a quality below this threshold, "
"replace the base with 'N'"))
parser.add_option("--quality-encoding",
dest="quality_encoding",
type="choice",
choices=["phred33", "phred64", "solexa"],
help=("Quality score encoding. Choose from 'phred33'"
"[33-77] 'phred64' [64-106] or 'solexa' [59-106]"))
parser.add_option("--extract-method",
dest="extract_method",
type="choice",
choices=["string", "regex"],
help=("How to extract the umi +/- cell barcodes, Choose "
"from 'string' or 'regex'"))
parser.add_option("--filter-cell-barcode",
dest="filter_cell_barcode",
action="store_true",
help="Filter the cell barcodes")
parser.add_option("--error-correct-cell",
dest="error_correct_cell",
action="store_true",
help=("Correct errors in the cell barcode"))
parser.add_option("--error-correct-threshold",
dest="error_correct_threshold",
type="int",
help=("Hamming distance allowed for correction"))
parser.add_option("--plot-prefix",
dest="plot_prefix",
type="string",
help=("Prefix for plots to visualise the automated "
"detection of the number of 'true' cell barcodes"))
parser.add_option("--output-whitelist",
dest="output_whitelist",
type="string",
help=("Write out the automatically generated whitelist"))
parser.add_option("--whitelist-tsv",
dest="whitelist_tsv",
type="string",
help=("A whitelist of accepted cell barcodes"))
parser.add_option("--blacklist-tsv",
dest="blacklist_tsv",
type="string",
help=("A blacklist of accepted cell barcodes"))
parser.add_option(
"--cell-barcode-subset",
dest="cell_barcode_subset",
type="int",
help=("Use only the first N reads to automatically "
"identify the true cell barcodes. If N is greater "
"than the number of reads, all reads will be used"))
parser.add_option("--reads-subset",
dest="reads_subset",
type="int",
help=("Only extract from the first N reads. If N is "
"greater than the number of reads, all reads will "
"be used"))
parser.add_option(
"--reconcile-pairs",
dest="reconcile",
action="store_true",
help=("Allow the presences of reads in read2 input that are"
"not present in read1 input. This allows cell barcode"
"filtering of read1s without considering read2s"))
parser.set_defaults(extract_method="string",
filter_cell_barcodes=False,
whitelist_tsv=None,
blacklist_tsv=None,
error_correct_cell=False,
error_correct_threshold=1,
pattern=None,
pattern2=None,
read2_in=None,
read2_out=False,
read2_out_only=False,
quality_filter_threshold=None,
quality_encoding=None,
plot_prefix=None,
output_whitelist=None,
cell_barcode_subset=50000000,
reconcile=False)
# add common options (-h/--help, ...) and parse command line
(options, args) = U.Start(parser, argv=argv)
if options.quality_filter_threshold or options.quality_filter_mask:
if not options.quality_encoding:
U.error("must provide a quality encoding (--quality-"
"encoding) to filter UMIs by quality (--quality"
"-filter-threshold) or mask low quality bases "
"with (--quality-filter-mask)")
if not options.pattern and not options.pattern2:
if not options.read2_in:
U.error("Must supply --bc-pattern for single-end")
else:
U.error("Must supply --bc-pattern and/or --bc-pattern "
"if paired-end ")
if options.pattern2:
if not options.read2_in:
U.error("must specify a paired fastq ``--read2-in``")
if not options.pattern2:
options.pattern2 = options.pattern
extract_cell = False
extract_umi = False
# If the pattern is a regex we can compile the regex(es) prior to
# ExtractFilterAndUpdate instantiation
if options.extract_method == "regex":
if options.pattern:
try:
options.pattern = regex.compile(options.pattern)
except regex.error:
U.error("barcode_regex '%s' is not a "
"valid regex" % options.pattern)
if options.pattern2:
try:
options.pattern2 = regex.compile(options.barcode_regex2)
except regex.Error:
U.error("barcode_regex2 '%s' is not a "
"valid regex" % options.barcode_regex2)
# check whether the regex contains a umi group(s) and cell groups(s)
if options.extract_method == "regex":
if options.pattern:
for group in options.pattern.groupindex:
if group.startswith("cell_"):
extract_cell = True
elif group.startswith("umi_"):
extract_umi = True
if options.pattern2:
for group in options.pattern2.groupindex:
if group.startswith("cell_"):
extract_cell = True
elif group.startswith("umi_"):
extract_umi = True
# check whether the pattern string contains umi/cell bases
elif options.extract_method == "string":
if options.pattern:
if "C" in options.pattern:
extract_cell = True
if "N" in options.pattern:
extract_umi = True
if options.pattern2:
if "C" in options.pattern2:
extract_cell = True
if "N" in options.pattern2:
extract_umi = True
if options.whitelist_tsv:
if options.blacklist_tsv:
U.error("Do not supply a blacklist and a whitelist. Just "
"remove the blacklist barcodes from the whitelist!")
if not extract_umi:
if options.extract_method == "string":
U.error("barcode pattern(s) do not include any umi bases "
"(marked with 'Ns') %s, %s" %
(options.pattern, options.pattern2))
elif options.extract_method == "regex":
U.error("barcode regex(es) do not include any umi groups "
"(starting with 'umi_') %s, %s" (options.pattern,
options.pattern2))
if options.stdin == sys.stdin:
if not options.whitelist_tsv and options.filter_cell_barcode:
U.error(
"cannot support reading from stdin if correcting cell barcode")
read1s = umi_methods.fastqIterate(U.openFile(options.stdin))
else:
read1s = umi_methods.fastqIterate(U.openFile(options.stdin.name))
# set up read extractor
ReadExtractor = umi_methods.ExtractFilterAndUpdate(
options.extract_method, options.pattern, options.pattern2,
options.prime3, extract_cell, options.quality_encoding,
options.quality_filter_threshold, options.quality_filter_mask,
options.filter_cell_barcode)
if options.filter_cell_barcode:
if (not options.whitelist_tsv) or options.error_correct_cell:
cell_barcode_counts = collections.Counter()
n_reads = 0
if not options.read2_in:
for read1 in read1s:
n_reads += 1
cell_barcode = ReadExtractor.getCellBarcode(read1)
if cell_barcode:
cell_barcode_counts[cell_barcode] += 1
if options.cell_barcode_subset:
if (n_reads > options.cell_barcode_subset):
break
else:
read2s = umi_methods.fastqIterate(U.openFile(options.read2_in))
for read1, read2 in izip(read1s, read2s):
n_reads += 1
cell_barcode = ReadExtractor.getCellBarcode(read1, read2)
if cell_barcode:
cell_barcode_counts[cell_barcode] += 1
if options.cell_barcode_subset:
if (n_reads > options.cell_barcode_subset):
break
if options.blacklist_tsv:
cell_blacklist = umi_methods.getUserDefinedBarcodes(
options.blacklist_tsv)
for cell in cell_blacklist:
del cell_barcode_counts[cell]
if options.whitelist_tsv:
cell_whitelist = umi_methods.getUserDefinedBarcodes(
options.whitelist_tsv)
error_correct_mappings = umi_methods.getErrorCorrectMappings(
cell_barcode_counts.keys(), cell_whitelist,
options.error_correct_threshold)
else:
# getCellWhitelist has not been properly defined yet!
cell_whitelist, error_correct_mappings = umi_methods.getCellWhitelist(
cell_barcode_counts, options.error_correct_threshold,
options.plot_prefix)
# re-make the reads1s iterator
read1s = umi_methods.fastqIterate(U.openFile(options.stdin.name))
else:
cell_whitelist = umi_methods.getUserDefinedBarcodes(
options.whitelist_tsv)
error_correct_mappings = None, None
false_to_true_map, true_to_false_map = error_correct_mappings
if options.output_whitelist:
with U.openFile(options.output_whitelist, "w") as outf:
columns = [
"barcode", "count", "corrected_barcodes",
"corrected_barcode_counts"
]
outf.write("\t".join(columns) + "\n")
for barcode in sorted(list(cell_whitelist)):
if true_to_false_map:
corrected_barcodes = ",".join(
sorted(true_to_false_map[barcode]))
corrected_barcode_counts = ",".join(
map(str, [
cell_barcode_counts[x]
for x in sorted(true_to_false_map[barcode])
]))
else:
corrected_barcodes, corrected_barcode_counts = "", ""
outf.write("%s\t%s\t%s\t%s\n" %
(barcode, cell_barcode_counts[barcode],
corrected_barcodes, corrected_barcode_counts))
ReadExtractor.cell_whitelist = cell_whitelist
ReadExtractor.false_to_true_map = false_to_true_map
if options.read2_in is None:
for read in read1s:
new_read = ReadExtractor(read)
if options.reads_subset:
if (ReadExtractor.read_counts['Input Reads'] >
options.reads_subset):
break
if not new_read:
continue
options.stdout.write(str(new_read) + "\n")
else:
read2s = umi_methods.fastqIterate(U.openFile(options.read2_in))
if options.read2_out:
read2_out = U.openFile(options.read2_out, "w")
if options.reconcile:
strict = False
else:
strict = True
for read1, read2 in umi_methods.joinedFastqIterate(
read1s, read2s, strict):
reads = ReadExtractor(read1, read2)
if options.reads_subset:
if (ReadExtractor.read_counts['Input Reads'] >
options.reads_subset):
break
if not reads:
continue
else:
new_read1, new_read2 = reads
if not options.read2_out_only:
options.stdout.write(str(new_read1) + "\n")
if options.read2_out:
read2_out.write(str(new_read2) + "\n")
if options.read2_out:
read2_out.close()
for k, v in ReadExtractor.getReadCounts().most_common():
U.info("%s: %s" % (k, v))
U.Stop()
def main(argv=None):
"""script main.
parses command line options in sys.argv, unless *argv* is given.
"""
if argv is None:
argv = sys.argv
# setup command line parser
parser = U.OptionParser(version="%prog version: $Id$",
usage=globals()["__doc__"])
parser.add_option("-i",
"--in-sam",
dest="in_sam",
action="store_true",
help="Input file is in sam format",
default=False)
parser.add_option("-o",
"--out-sam",
dest="out_sam",
action="store_true",
help="Output alignments in sam format",
default=False)
parser.add_option("--ignore-umi",
dest="ignore_umi",
action="store_true",
help="Ignore UMI and dedup only on position",
default=False)
parser.add_option("--subset",
dest="subset",
type="string",
help="Use only a fraction of reads, specified by subset",
default=1.1)
parser.add_option("--spliced-is-unique",
dest="spliced",
action="store_true",
help="Treat a spliced read as different to an unspliced"
" one",
default=False)
parser.add_option("--soft-clip-threshold",
dest="soft",
type="float",
help="number of bases clipped from 5' end before"
"read os counted as spliced",
default=4)
parser.add_option("--edit-distance-threshold",
dest="threshold",
type="int",
help="Edit distance theshold at which to join two UMIs"
"when clustering",
default=1)
parser.add_option("--chrom",
dest="chrom",
type="string",
help="Restrict to one chromosome",
default=None)
parser.add_option("--paired",
dest="paired",
action="store_true",
default=False,
help="Use second-in-pair position when deduping")
parser.add_option("--method",
dest="method",
type="choice",
choices=("adjacency", "directional-adjacency",
"percentile", "unique", "cluster"),
default="directional-adjacency",
help="method to use for umi deduping")
parser.add_option("--output-stats",
dest="stats",
type="string",
default=False,
help="Specify location to output stats")
parser.add_option("--further-stats",
dest="further_stats",
action="store_true",
default=False,
help="Output further stats")
parser.add_option("--per-contig",
dest="per_contig",
action="store_true",
default=False,
help=("dedup per contig,"
" e.g for transcriptome where contig = gene"))
parser.add_option(
"--whole-contig",
dest="whole_contig",
action="store_true",
default=False,
help=
"Read whole contig before outputting bundles: guarantees that no reads"
"are missed, but increases memory usage")
parser.add_option("--multimapping-detection-method",
dest="detection_method",
type="choice",
choices=("NH", "X0", "XT"),
default=None,
help=("Some aligners identify multimapping using bam "
"tags. Setting this option to NH, X0 or XT will "
"use these tags when selecting the best read "
"amongst reads with the same position and umi"))
parser.add_option("--mapping-quality",
dest="mapping_quality",
type="int",
help="Minimum mapping quality for a read to be retained",
default=0)
parser.add_option("--read-length",
dest="read_length",
action="store_true",
default=False,
help=("use read length in addition to position and UMI"
"to identify possible duplicates"))
# add common options (-h/--help, ...) and parse command line
(options, args) = U.Start(parser, argv=argv)
if options.stdin != sys.stdin:
in_name = options.stdin.name
options.stdin.close()
else:
raise ValueError("Input on standard in not currently supported")
if options.stdout != sys.stdout:
out_name = options.stdout.name
options.stdout.close()
else:
out_name = "-"
if options.in_sam:
in_mode = "r"
else:
in_mode = "rb"
if options.out_sam:
out_mode = "w"
else:
out_mode = "wb"
if options.stats:
if options.ignore_umi:
raise ValueError("'--output-stats' and '--ignore-umi' options"
" cannot be used together")
if options.further_stats:
if not options.stats:
raise ValueError("'--further-stats' options requires "
"'--output-stats' option")
if options.method not in ["cluster", "adjacency"]:
raise ValueError("'--further-stats' only enabled with 'cluster' "
"and 'adjacency' methods")
infile = pysam.Samfile(in_name, in_mode)
outfile = pysam.Samfile(out_name, out_mode, template=infile)
if options.paired:
outfile = TwoPassPairWriter(infile, outfile)
nInput, nOutput = 0, 0
if options.detection_method:
bam_features = detect_bam_features(infile.filename)
if not bam_features[options.detection_method]:
if sum(bam_features.values()) == 0:
raise ValueError(
"There are no bam tags available to detect multimapping. "
"Do not set --multimapping-detection-method")
else:
raise ValueError(
"The chosen method of detection for multimapping (%s) "
"will not work with this bam. Multimapping can be detected"
" for this bam using any of the following: %s" %
(options.detection_method, ",".join(
[x for x in bam_features if bam_features[x]])))
if options.stats:
# set up arrays to hold stats data
stats_pre_df_dict = {"UMI": [], "counts": []}
stats_post_df_dict = {"UMI": [], "counts": []}
pre_cluster_stats = []
post_cluster_stats = []
pre_cluster_stats_null = []
post_cluster_stats_null = []
topology_counts = collections.Counter()
node_counts = collections.Counter()
read_gn = random_read_generator(infile.filename, chrom=options.chrom)
for bundle in get_bundles(infile,
ignore_umi=options.ignore_umi,
subset=float(options.subset),
quality_threshold=options.mapping_quality,
paired=options.paired,
chrom=options.chrom,
spliced=options.spliced,
soft_clip_threshold=options.soft,
per_contig=options.per_contig,
whole_contig=options.whole_contig,
read_length=options.read_length,
detection_method=options.detection_method):
nInput += sum([bundle[umi]["count"] for umi in bundle])
if nOutput % 10000 == 0:
U.debug("Outputted %i" % nOutput)
if nInput % 1000000 == 0:
U.debug("Read %i input reads" % nInput)
if options.stats:
# generate pre-dudep stats
average_distance = get_average_umi_distance(bundle.keys())
pre_cluster_stats.append(average_distance)
cluster_size = len(bundle)
random_umis = read_gn.getUmis(cluster_size)
average_distance_null = get_average_umi_distance(random_umis)
pre_cluster_stats_null.append(average_distance_null)
if options.ignore_umi:
for umi in bundle:
nOutput += 1
outfile.write(bundle[umi]["read"])
else:
# set up ClusterAndReducer functor with methods specific to
# specified options.method
processor = ClusterAndReducer(options.method)
# dedup using umis and write out deduped bam
reads, umis, umi_counts, topologies, nodes = processor(
bundle, options.threshold, options.stats,
options.further_stats)
for read in reads:
outfile.write(read)
nOutput += 1
if options.stats:
# collect pre-dudupe stats
stats_pre_df_dict['UMI'].extend(bundle)
stats_pre_df_dict['counts'].extend(
[bundle[UMI]['count'] for UMI in bundle])
# collect post-dudupe stats
post_cluster_umis = [x.qname.split("_")[-1] for x in reads]
stats_post_df_dict['UMI'].extend(umis)
stats_post_df_dict['counts'].extend(umi_counts)
average_distance = get_average_umi_distance(post_cluster_umis)
post_cluster_stats.append(average_distance)
cluster_size = len(post_cluster_umis)
random_umis = read_gn.getUmis(cluster_size)
average_distance_null = get_average_umi_distance(random_umis)
post_cluster_stats_null.append(average_distance_null)
if options.further_stats:
for c_type, count in topologies.most_common():
topology_counts[c_type] += count
for c_type, count in nodes.most_common():
node_counts[c_type] += count
outfile.close()
if options.stats:
stats_pre_df = pd.DataFrame(stats_pre_df_dict)
stats_post_df = pd.DataFrame(stats_post_df_dict)
# generate histograms of counts per UMI at each position
UMI_counts_df_pre = pd.DataFrame(
stats_pre_df.pivot_table(columns=stats_pre_df["counts"],
values="counts",
aggfunc=len))
UMI_counts_df_post = pd.DataFrame(
stats_post_df.pivot_table(columns=stats_post_df["counts"],
values="counts",
aggfunc=len))
UMI_counts_df_pre.columns = ["instances"]
UMI_counts_df_post.columns = ["instances"]
UMI_counts_df = pd.merge(UMI_counts_df_pre,
UMI_counts_df_post,
how='left',
left_index=True,
right_index=True,
sort=True,
suffixes=["_pre", "_post"])
# TS - if count value not observed either pre/post-dedup,
# merge will leave an empty cell and the column will be cast as a float
# see http://pandas.pydata.org/pandas-docs/dev/missing_data.html
# --> Missing data casting rules and indexing
# so, back fill with zeros and convert back to int
UMI_counts_df = UMI_counts_df.fillna(0).astype(int)
UMI_counts_df.to_csv(options.stats + "_per_umi_per_position.tsv",
sep="\t")
# aggregate stats pre/post per UMI
agg_pre_df = aggregateStatsDF(stats_pre_df)
agg_post_df = aggregateStatsDF(stats_post_df)
agg_df = pd.merge(agg_pre_df,
agg_post_df,
how='left',
left_index=True,
right_index=True,
sort=True,
suffixes=["_pre", "_post"])
# TS - see comment above regarding missing values
agg_df = agg_df.fillna(0).astype(int)
agg_df.to_csv(options.stats + "_per_umi.tsv", sep="\t")
# bin distances into integer bins
max_ed = int(
max(
map(max, [
pre_cluster_stats, post_cluster_stats,
pre_cluster_stats_null, post_cluster_stats_null
])))
cluster_bins = range(-1, int(max_ed) + 2)
def bin_clusters(cluster_list, bins=cluster_bins):
''' take list of floats and return bins'''
return np.digitize(cluster_list, bins, right=True)
def tallyCounts(binned_cluster, max_edit_distance):
''' tally counts per bin '''
return np.bincount(binned_cluster, minlength=max_edit_distance + 3)
pre_cluster_binned = bin_clusters(pre_cluster_stats)
post_cluster_binned = bin_clusters(post_cluster_stats)
pre_cluster_null_binned = bin_clusters(pre_cluster_stats_null)
post_cluster_null_binned = bin_clusters(post_cluster_stats_null)
edit_distance_df = pd.DataFrame({
"unique":
tallyCounts(pre_cluster_binned, max_ed),
"unique_null":
tallyCounts(pre_cluster_null_binned, max_ed),
options.method:
tallyCounts(post_cluster_binned, max_ed),
"%s_null" % options.method:
tallyCounts(post_cluster_null_binned, max_ed),
"edit_distance":
cluster_bins
})
# TS - set lowest bin (-1) to "Single_UMI"
edit_distance_df['edit_distance'][0] = "Single_UMI"
edit_distance_df.to_csv(options.stats + "_edit_distance.tsv",
index=False,
sep="\t")
if options.further_stats:
with U.openFile(options.stats + "_topologies.tsv", "w") as outf:
outf.write("\n".join([
"\t".join((x, str(y)))
for x, y in topology_counts.most_common()
]) + "\n")
with U.openFile(options.stats + "_nodes.tsv", "w") as outf:
outf.write("\n".join([
"\t".join(map(str, (x, y)))
for x, y in node_counts.most_common()
]) + "\n")
# write footer and output benchmark information.
U.info("Number of reads in: %i, Number of reads out: %i" %
(nInput, nOutput))
U.Stop()
def main(argv=None):
"""script main.
parses command line options in sys.argv, unless *argv* is given.
"""
if argv is None:
argv = sys.argv
# setup command line parser
parser = U.OptionParser(version="%prog version: $Id$",
usage=usage,
description=globals()["__doc__"])
group = U.OptionGroup(parser, "count_tab-specific options")
group.add_option("--barcode-separator",
dest="bc_sep",
type="string",
help="separator between read id and UMI "
" and (optionally) the cell barcode",
default="_")
group.add_option("--per-cell",
dest="per_cell",
action="store_true",
help="Readname includes cell barcode as well as UMI in "
"format: read[sep]UMI[sep]CB")
parser.add_option_group(group)
# add common options (-h/--help, ...) and parse command line
(options, args) = U.Start(parser,
argv=argv,
add_group_dedup_options=False,
add_sam_options=False)
nInput, nOutput = 0, 0
# set the method with which to extract umis from reads
if options.per_cell:
bc_getter = partial(sam_methods.get_cell_umi_read_string,
sep=options.bc_sep)
else:
bc_getter = partial(sam_methods.get_umi_read_string,
sep=options.bc_sep)
if options.per_cell:
options.stdout.write("%s\t%s\t%s\n" % ("cell", "gene", "count"))
else:
options.stdout.write("%s\t%s\n" % ("gene", "count"))
# set up UMIClusterer functor with methods specific to
# specified options.method
processor = network.UMIClusterer(options.method)
for gene, counts in sam_methods.get_gene_count_tab(options.stdin,
bc_getter=bc_getter):
for cell in counts.keys():
umis = counts[cell].keys()
nInput += sum(counts[cell].values())
# group the umis
groups = processor(counts[cell], threshold=options.threshold)
gene_count = len(groups)
if options.per_cell:
options.stdout.write("%s\t%s\t%i\n" % (cell, gene, gene_count))
else:
options.stdout.write("%s\t%i\n" % (gene, gene_count))
nOutput += gene_count
U.info("Number of reads counted: %i" % nOutput)
U.Stop()
def main(argv=None):
"""script main.
parses command line options in sys.argv, unless *argv* is given.
"""
if argv is None:
argv = sys.argv
# setup command line parser
parser = U.OptionParser(version="%prog version: $Id$",
usage=globals()["__doc__"])
parser.add_option("-p", "--bc-pattern", dest="pattern", type="string",
help="Barcode pattern")
parser.add_option("--bc-pattern2", dest="pattern2", type="string",
help="Barcode pattern for paired reads")
parser.add_option("--3prime", dest="prime3", action="store_true",
help="barcode is on 3' end of read.")
parser.add_option("--read2-in", dest="read2_in", type="string",
help="file name for read pairs")
parser.add_option("--read2-out", dest="read2_out", type="string",
help="file to output processed paired read to")
parser.add_option("--read2-stdout", dest="read2_stdout",
action="store_true",
help="Paired reads, send read2 to stdout, discarding read1")
parser.add_option("--quality-filter-threshold",
dest="quality_filter_threshold", type="int",
help=("Remove reads where any UMI base quality score "
"falls below this threshold"))
parser.add_option("--quality-filter-mask",
dest="quality_filter_mask", type="int",
help=("If a UMI base has a quality below this threshold, "
"replace the base with 'N'"))
parser.add_option("--quality-encoding",
dest="quality_encoding", type="choice",
choices=["phred33", "phred64", "solexa"],
help=("Quality score encoding. Choose from 'phred33'"
"[33-77] 'phred64' [64-106] or 'solexa' [59-106]"))
parser.add_option("--extract-method",
dest="extract_method", type="choice",
choices=["string", "regex"],
help=("How to extract the umi +/- cell barcodes, Choose "
"from 'string' or 'regex'"))
parser.add_option("--filter-cell-barcode",
dest="filter_cell_barcode",
action="store_true",
help="Filter the cell barcodes")
parser.add_option("--error-correct-cell",
dest="error_correct_cell",
action="store_true",
help=("Correct errors in the cell barcode"))
parser.add_option("--whitelist",
dest="whitelist", type="string",
help=("A whitelist of accepted cell barcodes"))
parser.add_option("--blacklist",
dest="blacklist", type="string",
help=("A blacklist of accepted cell barcodes"))
parser.add_option("--reads-subset",
dest="reads_subset", type="int",
help=("Only extract from the first N reads. If N is "
"greater than the number of reads, all reads will "
"be used"))
parser.add_option("--reconcile-pairs",
dest="reconcile", action="store_true",
help=("Allow the presences of reads in read2 input that are"
"not present in read1 input. This allows cell barcode"
"filtering of read1s without considering read2s"))
parser.set_defaults(extract_method="string",
filter_cell_barcodes=False,
whitelist=None,
blacklist=None,
error_correct_cell=False,
pattern=None,
pattern2=None,
read2_in=None,
read2_out=False,
read2_stdout=False,
quality_filter_threshold=None,
quality_encoding=None,
reconcile=False)
# add common options (-h/--help, ...) and parse command line
(options, args) = U.Start(parser, argv=argv,
add_group_dedup_options=False,
add_sam_options=False)
if options.quality_filter_threshold or options.quality_filter_mask:
if not options.quality_encoding:
U.error("must provide a quality encoding (--quality-"
"encoding) to filter UMIs by quality (--quality"
"-filter-threshold) or mask low quality bases "
"with (--quality-filter-mask)")
if not options.pattern and not options.pattern2:
if not options.read2_in:
U.error("Must supply --bc-pattern for single-end")
else:
U.error("Must supply --bc-pattern and/or --bc-pattern "
"if paired-end ")
if options.pattern2:
if not options.read2_in:
U.error("must specify a paired fastq ``--read2-in``")
if not options.pattern2:
options.pattern2 = options.pattern
extract_cell = False
extract_umi = False
# If the pattern is a regex we can compile the regex(es) prior to
# ExtractFilterAndUpdate instantiation
if options.extract_method == "regex":
if options.pattern:
try:
options.pattern = regex.compile(options.pattern)
except regex.error:
U.error("barcode_regex '%s' is not a "
"valid regex" % options.pattern)
if options.pattern2:
try:
options.pattern2 = regex.compile(options.pattern2)
except regex.Error:
U.error("barcode_regex2 '%s' is not a "
"valid regex" % options.pattern2)
# check whether the regex contains a umi group(s) and cell groups(s)
if options.extract_method == "regex":
if options.pattern:
for group in options.pattern.groupindex:
if group.startswith("cell_"):
extract_cell = True
elif group.startswith("umi_"):
extract_umi = True
if options.pattern2:
for group in options.pattern2.groupindex:
if group.startswith("cell_"):
extract_cell = True
elif group.startswith("umi_"):
extract_umi = True
# check whether the pattern string contains umi/cell bases
elif options.extract_method == "string":
if options.pattern:
if "C" in options.pattern:
extract_cell = True
if "N" in options.pattern:
extract_umi = True
if options.pattern2:
if "C" in options.pattern2:
extract_cell = True
if "N" in options.pattern2:
extract_umi = True
if not extract_umi:
if options.extract_method == "string":
U.error("barcode pattern(s) do not include any umi bases "
"(marked with 'Ns') %s, %s" % (
options.pattern, options.pattern2))
elif options.extract_method == "regex":
U.error("barcode regex(es) do not include any umi groups "
"(starting with 'umi_') %s, %s" (
options.pattern, options.pattern2))
if options.filter_cell_barcodes:
if not options.whitelist:
U.error("must provide a whitelist (--whitelist) if using "
"--filter-cell-barcode option")
if not extract_cell:
if options.extract_method == "string":
U.error("barcode pattern(s) do not include any cell bases "
"(marked with 'Cs') %s, %s" % (
options.pattern, options.pattern2))
elif options.extract_method == "regex":
U.error("barcode regex(es) do not include any cell groups "
"(starting with 'cell_') %s, %s" (
options.pattern, options.pattern2))
read1s = umi_methods.fastqIterate(options.stdin)
# set up read extractor
ReadExtractor = umi_methods.ExtractFilterAndUpdate(
options.extract_method,
options.pattern,
options.pattern2,
options.prime3,
extract_cell,
options.quality_encoding,
options.quality_filter_threshold,
options.quality_filter_mask,
options.filter_cell_barcode)
if options.filter_cell_barcode:
cell_whitelist, false_to_true_map = umi_methods.getUserDefinedBarcodes(
options.whitelist, options.error_correct_cell)
ReadExtractor.cell_whitelist = cell_whitelist
ReadExtractor.false_to_true_map = false_to_true_map
if options.blacklist:
blacklist = set()
with U.openFile(options.blacklist, "r") as inf:
for line in inf:
blacklist.add(line.strip().split("\t")[0])
ReadExtractor.cell_blacklist = blacklist
# variables for progress monitor
progCount = 0
displayMax = 100000
U.info("Starting barcode extraction")
if options.read2_in is None:
for read in read1s:
# incrementing count for monitoring progress
progCount += 1
# Update display in every 100kth iteration
if progCount % displayMax == 0:
U.info("Parsed {} reads".format(progCount))
new_read = ReadExtractor(read)
if options.reads_subset:
if (ReadExtractor.read_counts['Input Reads'] >
options.reads_subset):
break
if not new_read:
continue
options.stdout.write(str(new_read) + "\n")
else:
read2s = umi_methods.fastqIterate(U.openFile(options.read2_in))
if options.read2_out:
read2_out = U.openFile(options.read2_out, "w")
if options.reconcile:
strict = False
else:
strict = True
for read1, read2 in umi_methods.joinedFastqIterate(
read1s, read2s, strict):
# incrementing count for monitoring progress
progCount += 1
# Update display in every 100kth iteration
if progCount % displayMax == 0:
U.info("Parsed {} reads".format(progCount))
sys.stdout.flush()
reads = ReadExtractor(read1, read2)
if options.reads_subset:
if (ReadExtractor.read_counts['Input Reads'] >
options.reads_subset):
break
if not reads:
continue
else:
new_read1, new_read2 = reads
if options.read2_stdout:
options.stdout.write(str(new_read2) + "\n")
else:
options.stdout.write(str(new_read1) + "\n")
if options.read2_out:
read2_out.write(str(new_read2) + "\n")
if options.read2_out:
read2_out.close()
for k, v in ReadExtractor.getReadCounts().most_common():
U.info("%s: %s" % (k, v))
U.Stop()
def main(argv=None):
"""script main.
parses command line options in sys.argv, unless *argv* is given.
"""
if argv is None:
argv = sys.argv
# setup command line parser
parser = U.OptionParser(version="%prog version: $Id$",
usage=usage,
description=globals()["__doc__"])
group = U.OptionGroup(parser, "dedup-specific options")
group.add_option("--output-stats",
dest="stats",
type="string",
default=False,
help="Specify location to output stats")
parser.add_option_group(group)
# add common options (-h/--help, ...) and parse command line
(options, args) = U.Start(parser, argv=argv)
U.validateSamOptions(options, group=False)
if options.random_seed:
np.random.seed(options.random_seed)
if options.stdin != sys.stdin:
in_name = options.stdin.name
options.stdin.close()
else:
raise ValueError("Input on standard in not currently supported")
if options.stdout != sys.stdout:
if options.no_sort_output:
out_name = options.stdout.name
else:
out_name = U.getTempFilename(dir=options.tmpdir)
sorted_out_name = options.stdout.name
options.stdout.close()
else:
if options.no_sort_output:
out_name = "-"
else:
out_name = U.getTempFilename(dir=options.tmpdir)
sorted_out_name = "-"
if not options.no_sort_output: # need to determine the output format for sort
if options.out_sam:
sort_format = "sam"
else:
sort_format = "bam"
if options.in_sam:
in_mode = "r"
else:
in_mode = "rb"
if options.out_sam:
out_mode = "wh"
else:
out_mode = "wb"
if options.stats and options.ignore_umi:
raise ValueError("'--output-stats' and '--ignore-umi' options"
" cannot be used together")
infile = pysam.Samfile(in_name, in_mode)
outfile = pysam.Samfile(out_name, out_mode, template=infile)
if options.paired:
outfile = sam_methods.TwoPassPairWriter(infile, outfile)
nInput, nOutput, input_reads, output_reads = 0, 0, 0, 0
if options.detection_method:
bam_features = detect_bam_features(infile.filename)
if not bam_features[options.detection_method]:
if sum(bam_features.values()) == 0:
raise ValueError(
"There are no bam tags available to detect multimapping. "
"Do not set --multimapping-detection-method")
else:
raise ValueError(
"The chosen method of detection for multimapping (%s) "
"will not work with this bam. Multimapping can be detected"
" for this bam using any of the following: %s" %
(options.detection_method, ",".join(
[x for x in bam_features if bam_features[x]])))
gene_tag = options.gene_tag
metacontig2contig = None
if options.chrom:
inreads = infile.fetch(reference=options.chrom)
else:
if options.per_contig and options.gene_transcript_map:
metacontig2contig = sam_methods.getMetaContig2contig(
infile, options.gene_transcript_map)
metatag = "MC"
inreads = sam_methods.metafetcher(infile, metacontig2contig,
metatag)
gene_tag = metatag
else:
inreads = infile.fetch()
# set up ReadCluster functor with methods specific to
# specified options.method
processor = network.ReadDeduplicator(options.method)
bundle_iterator = sam_methods.get_bundles(
options, metacontig_contig=metacontig2contig)
if options.stats:
# set up arrays to hold stats data
stats_pre_df_dict = {"UMI": [], "counts": []}
stats_post_df_dict = {"UMI": [], "counts": []}
pre_cluster_stats = []
post_cluster_stats = []
pre_cluster_stats_null = []
post_cluster_stats_null = []
topology_counts = collections.Counter()
node_counts = collections.Counter()
read_gn = umi_methods.random_read_generator(
infile.filename,
chrom=options.chrom,
barcode_getter=bundle_iterator.barcode_getter)
for bundle, key, status in bundle_iterator(inreads):
nInput += sum([bundle[umi]["count"] for umi in bundle])
while nOutput >= output_reads + 100000:
output_reads += 100000
U.info("Written out %i reads" % output_reads)
while nInput >= input_reads + 1000000:
input_reads += 1000000
U.info("Parsed %i input reads" % input_reads)
if options.stats:
# generate pre-dudep stats
average_distance = umi_methods.get_average_umi_distance(
bundle.keys())
pre_cluster_stats.append(average_distance)
cluster_size = len(bundle)
random_umis = read_gn.getUmis(cluster_size)
average_distance_null = umi_methods.get_average_umi_distance(
random_umis)
pre_cluster_stats_null.append(average_distance_null)
if options.ignore_umi:
for umi in bundle:
nOutput += 1
outfile.write(bundle[umi]["read"])
else:
# dedup using umis and write out deduped bam
reads, umis, umi_counts = processor(bundle=bundle,
threshold=options.threshold)
for read in reads:
outfile.write(read)
nOutput += 1
if options.stats:
# collect pre-dudupe stats
stats_pre_df_dict['UMI'].extend(bundle)
stats_pre_df_dict['counts'].extend(
[bundle[UMI]['count'] for UMI in bundle])
# collect post-dudupe stats
post_cluster_umis = [
bundle_iterator.barcode_getter(x)[0] for x in reads
]
stats_post_df_dict['UMI'].extend(umis)
stats_post_df_dict['counts'].extend(umi_counts)
average_distance = umi_methods.get_average_umi_distance(
post_cluster_umis)
post_cluster_stats.append(average_distance)
cluster_size = len(post_cluster_umis)
random_umis = read_gn.getUmis(cluster_size)
average_distance_null = umi_methods.get_average_umi_distance(
random_umis)
post_cluster_stats_null.append(average_distance_null)
outfile.close()
if not options.no_sort_output:
# sort the output
pysam.sort("-o", sorted_out_name, "-O", sort_format, out_name)
os.unlink(out_name) # delete the tempfile
if options.stats:
# generate the stats dataframe
stats_pre_df = pd.DataFrame(stats_pre_df_dict)
stats_post_df = pd.DataFrame(stats_post_df_dict)
# tally the counts per umi per position
pre_counts = collections.Counter(stats_pre_df["counts"])
post_counts = collections.Counter(stats_post_df["counts"])
counts_index = list(
set(pre_counts.keys()).union(set(post_counts.keys())))
counts_index.sort()
with U.openFile(options.stats + "_per_umi_per_position.tsv",
"w") as outf:
outf.write("counts\tinstances_pre\tinstances_post\n")
for count in counts_index:
values = (count, pre_counts[count], post_counts[count])
outf.write("\t".join(map(str, values)) + "\n")
# aggregate stats pre/post per UMI
agg_pre_df = aggregateStatsDF(stats_pre_df)
agg_post_df = aggregateStatsDF(stats_post_df)
agg_df = pd.merge(agg_pre_df,
agg_post_df,
how='left',
left_index=True,
right_index=True,
sort=True,
suffixes=["_pre", "_post"])
# TS - if count value not observed either pre/post-dedup,
# merge will leave an empty cell and the column will be cast as a float
# see http://pandas.pydata.org/pandas-docs/dev/missing_data.html
# --> Missing data casting rules and indexing
# so, back fill with zeros and convert back to int
agg_df = agg_df.fillna(0).astype(int)
agg_df.index = [x.decode() for x in agg_df.index]
agg_df.index.name = 'UMI'
agg_df.to_csv(options.stats + "_per_umi.tsv", sep="\t")
# bin distances into integer bins
max_ed = int(
max(
map(max, [
pre_cluster_stats, post_cluster_stats,
pre_cluster_stats_null, post_cluster_stats_null
])))
cluster_bins = range(-1, int(max_ed) + 2)
def bin_clusters(cluster_list, bins=cluster_bins):
''' take list of floats and return bins'''
return np.digitize(cluster_list, bins, right=True)
def tallyCounts(binned_cluster, max_edit_distance):
''' tally counts per bin '''
return np.bincount(binned_cluster, minlength=max_edit_distance + 3)
pre_cluster_binned = bin_clusters(pre_cluster_stats)
post_cluster_binned = bin_clusters(post_cluster_stats)
pre_cluster_null_binned = bin_clusters(pre_cluster_stats_null)
post_cluster_null_binned = bin_clusters(post_cluster_stats_null)
edit_distance_df = pd.DataFrame(
{
"unique":
tallyCounts(pre_cluster_binned, max_ed),
"unique_null":
tallyCounts(pre_cluster_null_binned, max_ed),
options.method:
tallyCounts(post_cluster_binned, max_ed),
"%s_null" % options.method:
tallyCounts(post_cluster_null_binned, max_ed),
"edit_distance":
cluster_bins
},
columns=[
"unique", "unique_null", options.method,
"%s_null" % options.method, "edit_distance"
])
# TS - set lowest bin (-1) to "Single_UMI"
edit_distance_df['edit_distance'][0] = "Single_UMI"
edit_distance_df.to_csv(options.stats + "_edit_distance.tsv",
index=False,
sep="\t")
# write footer and output benchmark information.
U.info("Reads: %s" % ", ".join([
"%s: %s" % (x[0], x[1])
for x in bundle_iterator.read_events.most_common()
]))
U.info("Number of reads out: %i" % nOutput)
if not options.ignore_umi: # otherwise processor has not been used
U.info("Total number of positions deduplicated: %i" %
processor.UMIClusterer.positions)
if processor.UMIClusterer.positions > 0:
U.info("Mean number of unique UMIs per position: %.2f" %
(float(processor.UMIClusterer.total_umis_per_position) /
processor.UMIClusterer.positions))
U.info("Max. number of unique UMIs per position: %i" %
processor.UMIClusterer.max_umis_per_position)
else:
U.warn("The BAM did not contain any valid "
"reads/read pairs for deduplication")
U.Stop()
def main(argv=None):
"""script main.
parses command line options in sys.argv, unless *argv* is given.
"""
if argv is None:
argv = sys.argv
# setup command line parser
parser = U.OptionParser(version="%prog version: $Id$",
usage=globals()["__doc__"])
parser.add_option("-i",
"--in-sam",
dest="in_sam",
action="store_true",
help="Input file is in sam format [default=%default]",
default=False)
parser.add_option("--umi-separator",
dest="umi_sep",
type="string",
help="separator between read id and UMI",
default="_")
parser.add_option("--umi-tag",
dest="umi_tag",
type="string",
help="tag containing umi",
default='RX')
parser.add_option("--extract-umi-method",
dest="get_umi_method",
type="choice",
choices=("read_id", "tag"),
default="read_id",
help="where is the read UMI encoded? [default=%default]")
parser.add_option("--subset",
dest="subset",
type="float",
help="Use only a fraction of reads, specified by subset",
default=None)
parser.add_option("--edit-distance-threshold",
dest="threshold",
type="int",
default=1,
help="Edit distance theshold at which to join two UMIs"
"when clustering. [default=%default]")
parser.add_option("--chrom",
dest="chrom",
type="string",
help="Restrict to one chromosome",
default=None)
parser.add_option("--paired",
dest="paired",
action="store_true",
default=False,
help="paired BAM. [default=%default]")
parser.add_option("--method",
dest="method",
type="choice",
choices=("adjacency", "directional", "percentile",
"unique", "cluster"),
default="directional",
help="method to use for umi deduping [default=%default]")
parser.add_option("--mapping-quality",
dest="mapping_quality",
type="int",
help="Minimum mapping quality for a read to be retained"
" [default=%default]",
default=0)
parser.add_option("--per-contig",
dest="per_contig",
action="store_true",
default=False,
help=("dedup per contig (field 3 in BAM; RNAME),"
" e.g for transcriptome where contig = gene"))
parser.add_option("--per-gene",
dest="per_gene",
action="store_true",
default=False,
help=("Deduplicate per gene,"
"e.g for transcriptome where contig = transcript"
"must also provide a transript to gene map with"
"--gene-transcript-map [default=%default]"))
parser.add_option("--gene-transcript-map",
dest="gene_transcript_map",
type="string",
help="file mapping transcripts to genes (tab separated)",
default=None)
parser.add_option("--gene-tag",
dest="gene_tag",
type="string",
help=("Deduplicate per gene where gene is"
"defined by this bam tag [default=%default]"),
default=None)
parser.add_option(
"--skip-tags-regex",
dest="skip_regex",
type="string",
help=("Used with --gene-tag. "
"Ignore reads where the gene-tag matches this regex"),
default="^[__|Unassigned]")
# add common options (-h/--help, ...) and parse command line
(options, args) = U.Start(parser, argv=argv)
if options.random_seed:
np.random.seed(options.random_seed)
if options.stdin != sys.stdin:
in_name = options.stdin.name
options.stdin.close()
else:
raise ValueError("Input on standard in not currently supported")
if options.in_sam:
in_mode = "r"
else:
in_mode = "rb"
if options.per_gene:
if not options.gene_transcript_map and not options.gene_tag:
raise ValueError(
"--per-gene option requires --gene-transcript-map "
"or --gene-tag")
try:
re.compile(options.skip_regex)
except re.error:
raise ValueError("skip-regex '%s' is not a "
"valid regex" % options.skip_regex)
infile = pysam.Samfile(in_name, in_mode)
nInput, nOutput = 0, 0
# set the method with which to extract umis from reads
if options.get_umi_method == "read_id":
umi_getter = partial(umi_methods.get_umi_read_id, sep=options.umi_sep)
elif options.get_umi_method == "tag":
umi_getter = partial(umi_methods.get_umi_tag, tag=options.umi_tag)
else:
raise ValueError("Unknown umi extraction method")
if options.chrom:
inreads = infile.fetch(reference=options.chrom)
else:
if options.per_gene and options.gene_transcript_map:
metacontig2contig = umi_methods.getMetaContig2contig(
infile, options.gene_transcript_map)
metatag = "MC"
inreads = umi_methods.metafetcher(infile, metacontig2contig,
metatag)
gene_tag = metatag
else:
inreads = infile.fetch()
gene_tag = options.gene_tag
options.stdout.write("%s\t%s\n" % ("gene", "count"))
for gene, bundle, read_events in umi_methods.get_gene_count(
inreads,
subset=options.subset,
quality_threshold=options.mapping_quality,
paired=options.paired,
per_contig=options.per_contig,
gene_tag=options.gene_tag,
skip_regex=options.skip_regex,
umi_getter=umi_getter):
umis = bundle.keys()
counts = {umi: bundle[umi]["count"] for umi in umis}
nInput += sum(counts.values())
# set up UMIClusterer functor with methods specific to
# specified options.method
processor = network.UMIClusterer(options.method)
# group the umis
groups = processor(umis, counts, threshold=options.threshold)
gene_count = len(groups)
options.stdout.write("%s\t%i\n" % (gene, gene_count))
nOutput += gene_count
# output reads events and benchmark information.
for event in read_events.most_common():
U.info("%s: %s" % (event[0], event[1]))
U.info("Number of reads counted: %i" % nOutput)
U.Stop()
