def save_barcode_summary_h5(self, filename):
""" Generate a minimal barcode summary h5 without going through the reporter.
NOTE: only use this if all genomes have the same set of barcodes, i.e. a raw matrix.
"""
bc_sequences = None
bc_table_cols = {}
total_conf_mapped_deduped_reads = None
for (genome, matrix) in self.matrices.iteritems():
if bc_sequences is None:
bc_sequences = np.array(matrix.bcs)
bc_table_cols[cr_constants.H5_BC_SEQUENCE_COL] = bc_sequences
conf_mapped_deduped_reads_key = cr_utils.format_barcode_summary_h5_key(genome,
cr_constants.TRANSCRIPTOME_REGION, cr_constants.CONF_MAPPED_DEDUPED_READ_TYPE)
conf_mapped_deduped_reads = matrix.get_reads_per_bc()
if len(bc_sequences) != len(conf_mapped_deduped_reads):
raise ValueError('Cannot write barcode summary since different genomes have different number of barcodes!')
bc_table_cols[conf_mapped_deduped_reads_key] = conf_mapped_deduped_reads
# Track total counts (across genomes)
if total_conf_mapped_deduped_reads is None:
total_conf_mapped_deduped_reads = conf_mapped_deduped_reads.copy()
else:
total_conf_mapped_deduped_reads += conf_mapped_deduped_reads
# Record the 'multi'-prefixed (aka total) counts
# for the web summary to display the barcode rank plot.
key = cr_utils.format_barcode_summary_h5_key(cr_constants.MULTI_REFS_PREFIX,
cr_constants.TRANSCRIPTOME_REGION,
cr_constants.CONF_MAPPED_DEDUPED_READ_TYPE)
bc_table_cols[key] = total_conf_mapped_deduped_reads
cr_utils.write_h5(filename, bc_table_cols)
def plot_barcode_rank(chart, sample_properties, sample_data):
""" Generate the RNA counter barcode rank plot """
if sample_properties.get(
'genomes'
) is None or sample_data.barcode_summary is None or sample_data.cell_barcodes is None:
return None
if len(sample_properties['genomes']) == 0:
return None
# UMI counts per BC across all genomes present
if len(sample_properties['genomes']) > 1:
genome = lib_constants.MULTI_REFS_PREFIX
else:
genome = sample_properties['genomes'][0]
gex_prefix = rna_library.get_library_type_metric_prefix(
lib_constants.GENE_EXPRESSION_LIBRARY_TYPE)
key = cr_utils.format_barcode_summary_h5_key(
gex_prefix, genome, cr_constants.TRANSCRIPTOME_REGION,
cr_constants.CONF_MAPPED_DEDUPED_READ_TYPE)
if key in sample_data.barcode_summary:
counts_per_bc, plot_segments = sample_data.counter_barcode_rank_plot_data(
key)
return _plot_counter_barcode_rank(chart, counts_per_bc, plot_segments)
else:
# Not guaranteed to exist, depending on pipeline
pass
def plot_barcode_rank(chart, sample_properties, sample_data):
""" Generate the RNA counter barcode rank plot """
if sample_properties.get(
'genomes') is None or sample_data.barcode_summary is None:
return None
if len(sample_properties['genomes']) == 0:
return None
# UMI counts per BC across all genomes present
if len(sample_properties['genomes']) > 1:
genome = cr_constants.MULTI_REFS_PREFIX
else:
genome = sample_properties['genomes'][0]
key = cr_utils.format_barcode_summary_h5_key(
genome, cr_constants.TRANSCRIPTOME_REGION,
cr_constants.CONF_MAPPED_DEDUPED_READ_TYPE)
if key in sample_data.barcode_summary:
counts_per_bc = sample_data.barcode_summary[key][:]
return _plot_barcode_rank(chart, counts_per_bc, sample_data.num_cells)
else:
# Not guaranteed to exist, depending on pipeline
pass
def plot_barcode_rank(chart, sample_properties, sample_data):
""" Generate the RNA counter barcode rank plot """
if sample_properties.get('genomes') is None or sample_data.barcode_summary is None:
return None
if len(sample_properties['genomes']) == 0:
return None
counts_per_bc = []
for genome in sample_properties['genomes']:
key = cr_utils.format_barcode_summary_h5_key(genome, cr_constants.TRANSCRIPTOME_REGION, cr_constants.CONF_MAPPED_DEDUPED_READ_TYPE)
if key in sample_data.barcode_summary:
counts_per_bc.append(sample_data.barcode_summary[key][:])
else:
# Not guaranteed to exist, depending on pipeline
return
counts_per_bc = np.concatenate(counts_per_bc)
return _plot_barcode_rank(chart, counts_per_bc, sample_data.num_cells)
def save_barcode_summary_h5(self, filename):
""" Generate a minimal barcode summary h5 without going through the reporter.
NOTE: only use this if all genomes have the same set of barcodes, i.e. a raw matrix.
"""
bc_sequences = None
bc_table_cols = {}
for (genome, matrix) in self.matrices.iteritems():
if bc_sequences is None:
bc_sequences = np.array(matrix.bcs)
bc_table_cols[cr_constants.H5_BC_SEQUENCE_COL] = bc_sequences
conf_mapped_deduped_reads_key = cr_utils.format_barcode_summary_h5_key(
genome, cr_constants.TRANSCRIPTOME_REGION,
cr_constants.CONF_MAPPED_DEDUPED_READ_TYPE)
conf_mapped_deduped_reads = matrix.get_reads_per_bc()
if len(bc_sequences) != len(conf_mapped_deduped_reads):
raise ValueError(
'Cannot write barcode summary since different genomes have different number of barcodes!'
)
bc_table_cols[
conf_mapped_deduped_reads_key] = conf_mapped_deduped_reads
cr_utils.write_h5(filename, bc_table_cols)
def report(self, genome, barcode_summary_h5, recovered_cells, cell_bc_seqs):
d = {}
filtered_mat = self.select_barcodes_by_seq(cell_bc_seqs)
cell_bc_indices = self.bcs_to_ints(cell_bc_seqs)
n_cell_bcs = len(cell_bc_seqs)
# Don't compute metrics if no cells detected
if n_cell_bcs == 0:
return d
# Compute matrix density
d['filtered_gene_bc_matrix_density'] = tk_stats.robust_divide(filtered_mat.m.getnnz(), filtered_mat.m.shape[0]*filtered_mat.m.shape[1])
reads_per_gene = filtered_mat.get_reads_per_gene()
top_genes_with_reads = {filtered_mat.int_to_gene_id(i): int(count) for i, count in filtered_mat._topN(reads_per_gene)}
d['filtered_bcs_top_genes_with_reads'] = top_genes_with_reads
unique_bcs_per_gene = filtered_mat._sum(filtered_mat.m >= cr_constants.MIN_READS_PER_BARCODE, axis=1)
top_genes_with_unique_bcs = {filtered_mat.int_to_gene_id(i): int(count) for i, count in filtered_mat._topN(unique_bcs_per_gene)}
d['filtered_bcs_top_genes_with_unique_bcs'] = top_genes_with_unique_bcs
# Total genes and counts
total_genes_detected = np.count_nonzero(reads_per_gene)
total_counts = int(reads_per_gene.sum())
d['filtered_bcs_total_unique_genes_detected'] = total_genes_detected
d['filtered_bcs_total_counts'] = total_counts
def _summarize_per_barcode(a):
mean = np.mean(a)
stddev = np.std(a)
return {
'mean': mean,
'median': np.median(a),
'cv': tk_stats.robust_divide(float(stddev), float(mean)),
'iqr': np.percentile(a, 75) - np.percentile(a, 25),
}
# Unique genes per bc
unique_genes_per_bc = filtered_mat._sum(filtered_mat.m >= cr_constants.MIN_READS_PER_GENE, axis=0)
unique_genes_stats = _summarize_per_barcode(unique_genes_per_bc)
for stat, value in unique_genes_stats.iteritems():
d['filtered_bcs_%s_unique_genes_detected' % stat] = value
# Counts per bc
counts_per_bc_stats = _summarize_per_barcode(filtered_mat._sum(filtered_mat.m, axis=0))
for stat, value in counts_per_bc_stats.iteritems():
d['filtered_bcs_%s_counts' % stat] = value
# Cumulative fraction of counts going to top bcs
d['filtered_bcs_cum_frac'] = tk_stats.robust_divide(filtered_mat.m.sum(), self.m.sum())
# cDNA PCR duplication in top bcs
dupe_candidate_h5_key = cr_utils.format_barcode_summary_h5_key(genome, cr_constants.TRANSCRIPTOME_REGION, cr_constants.CONF_MAPPED_BC_READ_TYPE)
if dupe_candidate_h5_key in barcode_summary_h5:
n_reads = barcode_summary_h5[dupe_candidate_h5_key][list(cell_bc_indices)].sum()
n_deduped_reads = filtered_mat.m.sum()
else:
n_reads = 0
n_deduped_reads = 0
d['filtered_bcs_%s_dupe_reads_frac' % cr_constants.CDNA_PCR_DUPE_TYPE] = 1 - tk_stats.robust_divide(n_deduped_reads, n_reads)
# Reads per top bc for the various read types (computed over top bcs)
for read_type in cr_constants.MATRIX_REPORT_READ_TYPES:
# Compute (n_reads)/(n_bcs) over all bcs and over top bcs
per_bc_metric = 'filtered_bcs_%s_reads_per_filtered_bc' % read_type
# Cumulative fraction of reads going to top bcs
frac_metric = 'filtered_bcs_%s_reads_cum_frac' % read_type
if read_type in cr_constants.MATRIX_USE_MATRIX_FOR_READ_TYPE:
n_reads = filtered_mat.m.sum()
n_all_reads = self.m.sum()
else:
h5_key = cr_utils.format_barcode_summary_h5_key(genome, cr_constants.TRANSCRIPTOME_REGION, read_type)
if h5_key in barcode_summary_h5:
n_reads = barcode_summary_h5[h5_key][list(cell_bc_indices)].sum()
n_all_reads = barcode_summary_h5[h5_key][()].sum()
else:
n_reads = 0
n_all_reads = 0
d[per_bc_metric] = tk_stats.robust_divide(n_reads, n_cell_bcs)
d[frac_metric] = tk_stats.robust_divide(n_reads, n_all_reads)
return d
def _report_genome_agnostic_metrics(matrix, barcode_summary_h5,
recovered_cells, cell_bc_seqs, total_reads,
total_conf_mapped_reads, library_prefix):
"""Report metrics that are computed across all barcodes and all genomes"""
d = {}
matrix = matrix.view()
genomes = matrix.get_genomes()
if len(genomes) == 0:
# For genomeless features, use "multi" in place of the genome prefix
genomes = [lib_constants.MULTI_REFS_PREFIX]
# Get number of cell bcs across all genomes
cell_bcs_union = reduce(lambda a, x: a | set(x), cell_bc_seqs.itervalues(),
set())
n_cell_bcs_union = len(cell_bcs_union)
d['filtered_bcs_transcriptome_union'] = n_cell_bcs_union
d['%s_filtered_bcs' % lib_constants.MULTI_REFS_PREFIX] = n_cell_bcs_union
# Report reads/cell across all genomes
mean_reads_per_cell = tk_stats.robust_divide(total_reads, n_cell_bcs_union)
d['%s_%s_total_raw_reads_per_filtered_bc' %
(lib_constants.MULTI_REFS_PREFIX,
cr_constants.TRANSCRIPTOME_REGION)] = mean_reads_per_cell
# Create a feature-barcode dual whose name makes sense
d['reads_per_cell'] = mean_reads_per_cell
d['%s_%s_total_conf_mapped_reads_per_filtered_bc' %
(lib_constants.MULTI_REFS_PREFIX,
cr_constants.TRANSCRIPTOME_REGION)] = tk_stats.robust_divide(
total_conf_mapped_reads, n_cell_bcs_union)
# Split the matrix by genome
if genomes[0] != lib_constants.MULTI_REFS_PREFIX:
genome_matrices = OrderedDict(
((g, matrix.select_features_by_genome(g)) for g in genomes))
else:
# Genomeless feature types
genome_matrices = OrderedDict(
((lib_constants.MULTI_REFS_PREFIX, matrix), ))
# Total UMI counts across all matrices and all filtered barcodes
total_umi_counts = 0
for mat in genome_matrices.itervalues():
total_umi_counts += mat.select_barcodes_by_seq(cell_bcs_union).sum()
# Deviation from cell load
if recovered_cells is None:
d['%s_filtered_bcs_difference_from_recovered_cells' %
lib_constants.MULTI_REFS_PREFIX] = 0
d['%s_filtered_bcs_relative_difference_from_recovered_cells' %
lib_constants.MULTI_REFS_PREFIX] = 0
else:
d['%s_filtered_bcs_difference_from_recovered_cells' %
lib_constants.MULTI_REFS_PREFIX] = int(n_cell_bcs_union) - int(
recovered_cells)
d['%s_filtered_bcs_relative_difference_from_recovered_cells' %
lib_constants.MULTI_REFS_PREFIX] = tk_stats.robust_divide(
n_cell_bcs_union - recovered_cells, recovered_cells)
# Duplicate these metrics across genomes for backwards-compat
for genome in genomes:
d['%s_total_raw_reads_per_filtered_bc' %
genome] = tk_stats.robust_divide(total_reads, n_cell_bcs_union)
d['%s_total_conf_mapped_reads_per_filtered_bc' %
genome] = tk_stats.robust_divide(total_conf_mapped_reads,
n_cell_bcs_union)
for read_type in cr_constants.MATRIX_REPORT_READ_TYPES:
metric = '%s_total_%s_reads_per_filtered_bc' % (genome, read_type)
if read_type in cr_constants.MATRIX_USE_MATRIX_FOR_READ_TYPE:
n_reads = total_umi_counts
else:
# Note the extra underscore after the library_type prefix.
# This is induced by the Reporter framework.
h5_keys = [
cr_utils.format_barcode_summary_h5_key(
library_prefix, g, cr_constants.TRANSCRIPTOME_REGION,
read_type) for g in genomes
]
h5_keys = [x for x in h5_keys if x in barcode_summary_h5]
n_reads = sum(
np.array(barcode_summary_h5[h5_key]).sum()
for h5_key in h5_keys)
d[metric] = tk_stats.robust_divide(n_reads, n_cell_bcs_union)
# Report frac reads in cells across all genomes
total_conf_mapped_reads_in_cells = 0
total_conf_mapped_barcoded_reads = 0
for genome, g_mat in genome_matrices.iteritems():
h5_key = '%s_%s_%s_%s_reads' % (library_prefix, genome,
cr_constants.TRANSCRIPTOME_REGION,
cr_constants.CONF_MAPPED_BC_READ_TYPE)
cmb_reads = barcode_summary_h5[h5_key][:]
cell_bc_indices = _get_barcode_summary_h5_indices(
barcode_summary_h5, cell_bcs_union)
total_conf_mapped_reads_in_cells += cmb_reads[cell_bc_indices].sum()
total_conf_mapped_barcoded_reads += cmb_reads.sum()
frac_reads_in_cells = tk_stats.robust_divide(
total_conf_mapped_reads_in_cells, total_conf_mapped_barcoded_reads)
d['multi_filtered_bcs_conf_mapped_barcoded_reads_cum_frac'] = frac_reads_in_cells
# Create a feature-barcode dual whose name makes sense
d['feature_reads_in_cells'] = frac_reads_in_cells
# Compute fraction of reads usable (conf mapped, barcoded, filtered barcode)
unique_barcodes = set(cell_bcs_union)
in_unique_barcodes_vectorized = np.vectorize(
lambda x: x in unique_barcodes)
filtered_bc_h5_row = in_unique_barcodes_vectorized(
np.array(barcode_summary_h5['bc_sequence']))
usable_reads = 0
for genome in genomes:
h5_key = cr_utils.format_barcode_summary_h5_key(
library_prefix, genome, cr_constants.TRANSCRIPTOME_REGION,
cr_constants.CONF_MAPPED_BC_READ_TYPE)
if h5_key not in barcode_summary_h5:
continue
usable_reads += (filtered_bc_h5_row *
np.array(barcode_summary_h5[h5_key])).sum()
# Fraction reads usable
d['%s_transcriptome_usable_reads_frac' %
lib_constants.MULTI_REFS_PREFIX] = tk_stats.robust_divide(
usable_reads, total_reads)
# Create a feature barcoding dual whose name makes sense
d['frac_feature_reads_usable'] = tk_stats.robust_divide(
usable_reads, total_reads)
# Usable reads
d['%s_usable_reads' % lib_constants.MULTI_REFS_PREFIX] = usable_reads
# Usable reads per cell
reads_usable_per_cell = tk_stats.robust_divide(usable_reads,
n_cell_bcs_union)
d['%s_usable_reads_per_filtered_bc' %
lib_constants.MULTI_REFS_PREFIX] = reads_usable_per_cell
# Create a feature barcoding dual whose name makes sense
d['feature_reads_usable_per_cell'] = reads_usable_per_cell
# Compute matrix density
filtered_mat = matrix.select_barcodes_by_seq(cell_bcs_union)
total_nonzero_entries = filtered_mat.get_num_nonzero()
filtered_shape = filtered_mat.get_shape()
total_entries = filtered_shape[0] * filtered_shape[1]
print total_entries, total_nonzero_entries, filtered_shape
d['%s_filtered_gene_bc_matrix_density' %
lib_constants.MULTI_REFS_PREFIX] = tk_stats.robust_divide(
total_nonzero_entries, total_entries)
return d
def _report(matrix, genome, barcode_summary_h5, recovered_cells, cell_bc_seqs,
library_prefix):
d = {}
matrix = matrix.view()
filtered_mat = matrix.select_barcodes_by_seq(cell_bc_seqs)
filtered_mat_shape = filtered_mat.get_shape()
cell_bc_indices = _get_barcode_summary_h5_indices(barcode_summary_h5,
cell_bc_seqs)
n_cell_bcs = len(cell_bc_seqs)
# Don't compute metrics if no cells detected
if n_cell_bcs == 0:
return d
# Compute matrix density
d['filtered_gene_bc_matrix_density'] = tk_stats.robust_divide(
filtered_mat.get_num_nonzero(),
filtered_mat_shape[0] * filtered_mat_shape[1])
counts_per_gene = filtered_mat.sum(axis=1)
genes_top_n = min(cr_constants.TOP_N, len(counts_per_gene))
top_genes_with_counts = {
filtered_mat.int_to_feature_id(i): int(count)
for i, count in cr_matrix.top_n(counts_per_gene, genes_top_n)
}
d['filtered_bcs_top_genes_with_reads'] = top_genes_with_counts
unique_bcs_per_gene = filtered_mat.count_ge(
axis=1, threshold=cr_constants.MIN_COUNTS_PER_BARCODE)
top_genes_with_unique_bcs = {
filtered_mat.int_to_feature_id(i): int(count)
for i, count in cr_matrix.top_n(unique_bcs_per_gene, genes_top_n)
}
d['filtered_bcs_top_genes_with_unique_bcs'] = top_genes_with_unique_bcs
# Total genes and counts
total_genes_detected = np.count_nonzero(counts_per_gene)
total_counts = int(counts_per_gene.sum())
d['filtered_bcs_total_unique_genes_detected'] = total_genes_detected
d['filtered_bcs_total_counts'] = total_counts
def _summarize_per_barcode(a):
mean = np.mean(a)
stddev = np.std(a)
return {
'mean': mean,
'median': np.median(a),
'cv': tk_stats.robust_divide(float(stddev), float(mean)),
'iqr': np.percentile(a, 75) - np.percentile(a, 25),
}
# Unique genes per bc
unique_genes_per_bc = filtered_mat.count_ge(
axis=0, threshold=cr_constants.MIN_COUNTS_PER_GENE)
unique_genes_stats = _summarize_per_barcode(unique_genes_per_bc)
for stat, value in unique_genes_stats.iteritems():
d['filtered_bcs_%s_unique_genes_detected' % stat] = value
# Counts per bc
counts_per_bc = filtered_mat.sum(axis=0)
counts_per_bc_stats = _summarize_per_barcode(counts_per_bc)
for stat, value in counts_per_bc_stats.iteritems():
d['filtered_bcs_%s_counts' % stat] = value
# Cumulative fraction of counts going to top bcs
filt_total_umis = filtered_mat.sum()
raw_total_umis = matrix.sum()
d['filtered_bcs_cum_frac'] = tk_stats.robust_divide(
filt_total_umis, raw_total_umis)
# cDNA PCR duplication in top bcs
dupe_candidate_h5_key = cr_utils.format_barcode_summary_h5_key(
library_prefix, genome, cr_constants.TRANSCRIPTOME_REGION,
cr_constants.CONF_MAPPED_BC_READ_TYPE)
if dupe_candidate_h5_key in barcode_summary_h5:
n_reads = barcode_summary_h5[dupe_candidate_h5_key][:][
cell_bc_indices].sum()
n_deduped_reads = filt_total_umis
else:
n_reads = 0
n_deduped_reads = 0
d['filtered_bcs_%s_dupe_reads_frac' %
cr_constants.CDNA_PCR_DUPE_TYPE] = 1 - tk_stats.robust_divide(
n_deduped_reads, n_reads)
# Reads per top bc for the various read types (computed over top bcs)
for read_type in cr_constants.MATRIX_REPORT_READ_TYPES:
# Compute (n_reads)/(n_bcs) over all bcs and over top bcs
per_bc_metric = 'filtered_bcs_%s_reads_per_filtered_bc' % read_type
# Cumulative fraction of reads going to top bcs
frac_metric = 'filtered_bcs_%s_reads_cum_frac' % read_type
if read_type in cr_constants.MATRIX_USE_MATRIX_FOR_READ_TYPE:
n_reads = filt_total_umis
n_all_reads = raw_total_umis
else:
h5_key = cr_utils.format_barcode_summary_h5_key(
library_prefix, genome, cr_constants.TRANSCRIPTOME_REGION,
read_type)
if h5_key in barcode_summary_h5:
counts = barcode_summary_h5[h5_key][:]
n_reads = counts[cell_bc_indices].sum()
n_all_reads = counts.sum()
else:
n_reads = 0
n_all_reads = 0
d[per_bc_metric] = tk_stats.robust_divide(n_reads, n_cell_bcs)
d[frac_metric] = tk_stats.robust_divide(n_reads, n_all_reads)
return d