def main(args, outs):
reference_star_path = cr_utils.get_reference_star_path(args.reference_path)
star_index = cr_transcriptome.build_star_index(reference_star_path)
chroms = star_index[0][0]
gene_index = cr_reference.GeneIndex.load_pickle(cr_utils.get_reference_genes_index(args.reference_path))
barcode_whitelist = cr_utils.load_barcode_whitelist(args.barcode_whitelist)
barcode_dist = cr_utils.load_barcode_dist(args.barcode_counts, barcode_whitelist, args.gem_group)
reporter = cr_report.Reporter(reference_path=args.reference_path,
high_conf_mapq=cr_constants.STAR_DEFAULT_HIGH_CONF_MAPQ,
gene_index=gene_index,
chroms=chroms,
barcode_whitelist=barcode_whitelist,
barcode_dist=barcode_dist,
gem_groups=args.gem_groups,
umi_length=cr_chem.get_umi_length(args.chemistry_def),
umi_min_qual_threshold=args.umi_min_qual_threshold)
reporter.attach_bcs_init()
outs.num_alignments = process_alignments(args.chunk_genome_input, args.chunk_trimmed_input, outs.output, args.bam_comments, reporter, gene_index, star_index, args)
reporter.attach_bcs_finalize()
reporter.save(outs.chunked_reporter)
def main(args, outs):
np.random.seed(0)
unique_gem_groups = np.unique(args.gem_groups).tolist()
reporter = vdj_report.VdjReporter(gem_groups=unique_gem_groups)
cell_barcodes = set()
bc_support = defaultdict(int)
# Load barcode whitelist
barcode_whitelist = cr_utils.load_barcode_whitelist(args.barcode_whitelist)
all_gem_groups = sorted(set(args.gem_groups))
if args.recovered_cells:
recovered_cells = args.recovered_cells
else:
recovered_cells = cr_constants.DEFAULT_TOP_BARCODE_CUTOFF * len(
all_gem_groups)
for gem_group in all_gem_groups:
if barcode_whitelist is None:
break
# Load barcode raw read count distribution
barcode_dist = cr_utils.load_barcode_dist(args.barcode_counts,
barcode_whitelist,
gem_group,
proportions=False)
counts = np.array(barcode_dist.values())
# Append gem group to barcode seqs
barcodes = np.array([
cr_utils.format_barcode_seq(seq, gem_group)
for seq in barcode_dist.keys()
])
# Call cell barcodes
gg_bc_support, gg_cell_bcs, rpu_threshold, umi_threshold, confidence = call_cell_barcodes(
args.umi_info, int(gem_group))
# Record the RPU and UMI thresholds
reporter._get_metric_attr('vdj_filter_bcs_rpu_threshold',
gem_group).set_value(rpu_threshold)
reporter._get_metric_attr('vdj_filter_bcs_umi_threshold',
gem_group).set_value(umi_threshold)
reporter._get_metric_attr('vdj_filter_bcs_confidence',
gem_group).set_value(confidence)
if len(gg_bc_support) > 0:
if args.force_cells is not None:
sorted_bcs = map(
lambda kv: kv[0],
sorted(gg_bc_support.items(),
key=lambda kv: kv[1],
reverse=True))
gg_cell_bcs = sorted_bcs[:min(len(sorted_bcs), args.force_cells
)]
# Update set of BCs called as cells
cell_barcodes.update(set(gg_cell_bcs))
# Sum BC support
for bc, count in gg_bc_support.iteritems():
bc_support[bc] += count
# Load the extract_reads summary to get the total raw reads
total_read_pairs = cr_utils.get_metric_from_json(
args.extract_reads_summary, 'total_read_pairs')
reporter.vdj_filter_barcodes_cb(cell_barcodes, barcodes, counts,
total_read_pairs, recovered_cells)
save_cell_barcodes_json(cell_barcodes, outs.cell_barcodes)
with open(outs.barcode_support, 'w') as f:
f.write('barcode,count\n')
for k, v in bc_support.iteritems():
f.write('%s,%d\n' % (k, v))
write_barcode_umi_summary(args.umi_info, reporter,
outs.barcode_umi_summary,
args.min_readpairs_per_umi, cell_barcodes)
reporter.report_summary_json(outs.summary)
def main(args, outs):
unique_gem_groups = np.unique(args.gem_groups).tolist()
reporter = vdj_report.VdjReporter(gem_groups=unique_gem_groups)
cell_barcodes = set()
bc_support = {}
# Load barcode whitelist
barcode_whitelist = cr_utils.load_barcode_whitelist(args.barcode_whitelist)
all_gem_groups = sorted(set(args.gem_groups))
if args.recovered_cells:
recovered_cells = args.recovered_cells
else:
recovered_cells = cr_constants.DEFAULT_TOP_BARCODE_CUTOFF * len(
all_gem_groups)
for gem_group in all_gem_groups:
if barcode_whitelist is None:
break
# Load barcode raw read count distribution
barcode_dist = cr_utils.load_barcode_dist(args.barcode_counts,
barcode_whitelist,
gem_group,
proportions=False)
counts = np.array(barcode_dist.values())
# Append gem group to barcode seqs
barcodes = np.array([
cr_utils.format_barcode_seq(seq, gem_group)
for seq in barcode_dist.keys()
])
# Call cell barcodes
gg_bc_support, gg_cell_bcs, threshold = call_cell_barcodes(
args.umi_summary, int(gem_group), args.min_umis,
args.readpairs_per_umi_nx, args.readpairs_per_umi_ratio)
# Record the threshold
reporter._get_metric_attr(
'vdj_filtered_bc_contig_kth_umi_readpair_threshold',
gem_group).set_value(threshold)
if len(gg_bc_support) > 0:
if args.force_cells is not None:
sorted_bcs = map(
lambda kv: kv[0],
sorted(gg_bc_support.items(),
key=lambda kv: kv[1],
reverse=True))
gg_cell_bcs = sorted_bcs[:min(len(sorted_bcs), args.force_cells
)]
cell_barcodes.update(set(gg_cell_bcs))
bc_support.update(gg_bc_support)
# Load the extract_reads summary to get the total raw reads
total_read_pairs = cr_utils.get_metric_from_json(
args.extract_reads_summary, 'total_read_pairs')
# Load the assembly metrics summary to get the total assemblable reads
assemblable_read_pairs_by_bc = cr_utils.get_metric_from_json(
args.assemble_metrics_summary, 'assemblable_read_pairs_by_bc')
assemblable_read_pairs = sum(
assemblable_read_pairs_by_bc.get(bc, 0) for bc in cell_barcodes)
reporter.vdj_filter_barcodes_cb(cell_barcodes, barcodes, counts,
total_read_pairs,
assemblable_read_pairs,
recovered_cells)
save_cell_barcodes_json(cell_barcodes, outs.cell_barcodes)
with open(outs.barcode_support, 'w') as f:
f.write('barcode,count\n')
for k, v in bc_support.iteritems():
f.write('%s,%d\n' % (k, v))
write_barcode_umi_summary(args.umi_info, reporter,
outs.barcode_umi_summary,
args.min_readpairs_per_umi, cell_barcodes)
reporter.report_summary_json(outs.summary)
def main(args, outs):
# Load barcode whitelist
if args.barcode_whitelist is not None:
barcode_whitelist = cr_utils.load_barcode_whitelist(
args.barcode_whitelist)
reporter = vdj_report.VdjReporter()
# Load barcode count distribution
barcode_dist = cr_utils.load_barcode_dist(args.barcode_counts,
barcode_whitelist,
args.gem_group,
args.library_type)
if args.barcode_whitelist is not None:
barcode_whitelist_set = set(barcode_whitelist)
else:
barcode_whitelist_set = None
in_read1_fastq = cr_io.open_maybe_gzip(args.read1_chunk)
in_read2_fastq = cr_io.open_maybe_gzip(
args.read2_chunk) if args.read2_chunk else []
outs.corrected_bcs += h5_constants.LZ4_SUFFIX
out_file = cr_io.open_maybe_gzip(outs.corrected_bcs, 'w')
bc_counter = cr_fastq.BarcodeCounter(args.barcode_whitelist,
outs.corrected_barcode_counts)
# Correct barcodes, add processed bc tag to fastq
read_pair_iter = itertools.izip_longest(tk_fasta.read_generator_fastq(in_read1_fastq), \
tk_fasta.read_generator_fastq(in_read2_fastq))
for read1, read2 in itertools.islice(read_pair_iter, args.initial_reads):
read1_header = cr_fastq.AugmentedFastqHeader(read1[0])
raw_bc = read1_header.get_tag(cr_constants.RAW_BARCODE_TAG)
bc_qual = read1_header.get_tag(cr_constants.RAW_BARCODE_QUAL_TAG)
processed_bc = None
if raw_bc:
if barcode_whitelist_set is not None and raw_bc not in barcode_whitelist_set:
processed_bc = cr_stats.correct_bc_error(
args.barcode_confidence_threshold, raw_bc, bc_qual,
barcode_dist)
else:
# Disallow Ns in no-whitelist case
if 'N' in raw_bc:
processed_bc = None
else:
processed_bc = raw_bc
if processed_bc:
bc_counter.count(None, processed_bc, None)
# Add gem group to barcode sequence
processed_bc = cr_utils.format_barcode_seq(
processed_bc, gem_group=args.gem_group)
reporter.vdj_barcode_cb(raw_bc, processed_bc)
out_file.write('%s\n' %
(processed_bc if processed_bc is not None else ''))
in_read1_fastq.close()
if in_read2_fastq:
in_read2_fastq.close()
out_file.close()
bc_counter.close()
reporter.save(outs.chunked_reporter)
def main(args, outs):
# Load barcode whitelist
if args.barcode_whitelist is not None:
barcode_whitelist = cr_utils.load_barcode_whitelist(
args.barcode_whitelist)
reporter = vdj_report.VdjReporter()
# Load barcode count distribution
barcode_dist = cr_utils.load_barcode_dist(args.barcode_counts,
barcode_whitelist,
args.gem_group)
if args.barcode_whitelist is not None:
barcode_whitelist_set = set(barcode_whitelist)
else:
barcode_whitelist_set = None
in_read1_fastq = open(args.read1_chunk)
in_read2_fastq = open(args.read2_chunk)
out_read1_fastq = open(outs.corrected_read1s, 'w')
out_read2_fastq = open(outs.corrected_read2s, 'w')
bc_counter = cr_fastq.BarcodeCounter(args.barcode_whitelist,
outs.corrected_barcode_counts)
# Correct barcodes, add processed bc tag to fastq
read_pair_iter = itertools.izip(tk_fasta.read_generator_fastq(in_read1_fastq), \
tk_fasta.read_generator_fastq(in_read2_fastq))
for read1, read2 in itertools.islice(read_pair_iter, args.initial_reads):
read1_header = cr_fastq.AugmentedFastqHeader(read1[0])
read2_header = cr_fastq.AugmentedFastqHeader(read2[0])
raw_bc = read1_header.get_tag(cr_constants.RAW_BARCODE_TAG)
bc_qual = read1_header.get_tag(cr_constants.RAW_BARCODE_QUAL_TAG)
if raw_bc:
if barcode_whitelist_set is not None and raw_bc not in barcode_whitelist_set:
processed_bc = cr_stats.correct_bc_error(
args.barcode_confidence_threshold, raw_bc, bc_qual,
barcode_dist)
else:
# Disallow Ns in no-whitelist case
if 'N' in raw_bc:
processed_bc = None
else:
processed_bc = raw_bc
if processed_bc:
bc_counter.count(None, processed_bc, None)
# Add gem group to barcode sequence
processed_bc = cr_utils.format_barcode_seq(
processed_bc, gem_group=args.gem_group)
read1_header.set_tag(cr_constants.PROCESSED_BARCODE_TAG,
processed_bc)
read2_header.set_tag(cr_constants.PROCESSED_BARCODE_TAG,
processed_bc)
reporter.vdj_barcode_cb(raw_bc, processed_bc)
tk_fasta.write_read_fastq(out_read1_fastq, read1_header.to_string(),
read1[1], read1[2])
tk_fasta.write_read_fastq(out_read2_fastq, read2_header.to_string(),
read2[1], read2[2])
in_read1_fastq.close()
in_read2_fastq.close()
out_read1_fastq.close()
out_read2_fastq.close()
bc_counter.close()
reporter.save(outs.chunked_reporter)