def filter_molecule_table( input_df: pd.DataFrame, output_directory: str, min_umi_per_cell: int = 10, min_avg_reads_per_umi: float = 2.0, min_reads_per_umi: int = -1, intbc_prop_thresh: float = 0.5, intbc_umi_thresh: int = 10, intbc_dist_thresh: int = 1, doublet_threshold: float = 0.35, allow_allele_conflicts: bool = False, plot: bool = False, ) -> pd.DataFrame: """Filters and corrects a molecule table of cellBC-UMI pairs. Performs the following steps on the alignments in a DataFrame: 1. Filters out UMIs with read count < `min_reads_per_umi`. If `min_reads_per_umi` is less than 0, a dynamic threshold is calculated as `(99th percentile of read counts) // 10`. 2. Filters out cellBCs with unique UMIs < `min_umi_per_cell` and average read count per UMI < `min_avg_reads_per_umi`. 3. Error corrects intBCs by changing intBCs with low UMI counts to intBCs with the same allele and a close sequence 4. Filters out cellBCs that contain too much conflicting allele information as intra-lineage doublets 5. Chooses one allele for each cellBC-intBC pair, by selecting the most common. This is not performed when `allow_allele_conflicts` is True. Args: input_df: A molecule table, i.e. cellBC-UMI pairs. Note that each cellBC should only contain one instance of each UMI output_directory: The output directory path to store plots min_umi_per_cell: The threshold specifying the minimum number of UMIs in a cell needed to be retained during filtering min_avg_reads_per_umi: The threshold specifying the minimum coverage (i.e. average) reads per UMI in a cell needed in order for that cell to be retained during filtering min_reads_per_umi: The threshold specifying the minimum read count needed for a UMI to be retained during filtering. Set dynamically if value is < 0. intbc_prop_thresh: The threshold specifying the maximum proportion of the total UMI counts for a intBC to be corrected to another intbc_umi_thresh: The threshold specifying the maximum UMI count for an intBC needs to be corrected to another intbc_dist_thresh: The threshold specifying the maximum Levenshtein Distance between sequences for an intBC to be corrected to another doublet_threshold: The threshold specifying the maximum proportion of conflicting alleles information allowed to for an intBC to be retained in doublet filtering. Set to None to skip doublet filtering allow_allele_conflicts: Whether or not to allow multiple alleles to be assigned to each cellBC-intBC pair. For fully single-cell data, this option should be set to False, since each cell is expected to have a single allele state for each intBC. However, this option should be set to True for chemistries that may result in multiple physical cells being captured for each barcode. plot: Indicates whether to plot the change in intBC and cellBC counts across filtering stages Returns: A filtered and corrected allele table of cellBC-UMI-allele groups """ input_df["status"] = "good" input_df.sort_values("readCount", ascending=False, inplace=True) rc_profile, upi_profile, upc_profile = {}, {}, {} logger.info("Logging initial stats...") if plot: ( rc_profile["Init"], upi_profile["Init"], upc_profile["Init"], ) = utilities.record_stats(input_df) if min_reads_per_umi < 0: R = input_df["readCount"] if list(R): min_reads_per_umi = np.percentile(R, 99) // 10 else: min_reads_per_umi = 0 logger.info(f"Filtering UMIs with less than {min_reads_per_umi} reads...") filtered_df = utilities.filter_umis(input_df, min_reads_per_umi=min_reads_per_umi) if plot: ( rc_profile["Filtered_UMI"], upi_profile["Filtered_UMI"], upc_profile["Filtered_UMI"], ) = utilities.record_stats(filtered_df) logger.info( f"Filtering out cellBCs with fewer than {min_umi_per_cell} UMIs and" f"less than {min_avg_reads_per_umi} average reads per UMI...") filtered_df = utilities.filter_cells( filtered_df, min_umi_per_cell=min_umi_per_cell, min_avg_reads_per_umi=min_avg_reads_per_umi, ) if plot: ( rc_profile["CellFilter"], upi_profile["CellFilter"], upc_profile["CellFilter"], ) = utilities.record_stats(filtered_df) if intbc_dist_thresh > 0: logger.info("Error correcting intBCs...") filtered_df = utilities.error_correct_intbc( filtered_df, prop=intbc_prop_thresh, umi_count_thresh=intbc_umi_thresh, dist_thresh=intbc_dist_thresh, ) if plot: ( rc_profile["Process_intBC"], upi_profile["Process_intBC"], upc_profile["Process_intBC"], ) = utilities.record_stats(filtered_df) if doublet_threshold and not allow_allele_conflicts: logger.info( f"Filtering out intra-lineage group doublets with proportion {doublet_threshold}..." ) filtered_df = doublet_utils.filter_intra_doublets( filtered_df, prop=doublet_threshold) if not allow_allele_conflicts: logger.info("Mapping remaining intBC conflicts...") filtered_df = map_utils.map_intbcs(filtered_df) if plot: ( rc_profile["Final"], upi_profile["Final"], upc_profile["Final"], ) = utilities.record_stats(filtered_df) # Count total filtered cellBCs cellBC_count = 0 for name, grp in filtered_df.groupby(["cellBC"]): cellBC_count += 1 if plot: stages = [ "Init", "CellFilter", "Filtered_UMI", "Process_intBC", "Final", ] # Plot Read Per UMI Histogram h = plt.figure(figsize=(14, 10)) for n in stages: ax = plt.hist(rc_profile[n], label=n, histtype="step", log=True, bins=200) plt.legend() plt.ylabel("Frequency") plt.xlabel("Number of Reads") plt.title("Reads Per UMI") plt.savefig(os.path.join(output_directory, "reads_per_umi.png")) plt.close() h = plt.figure(figsize=(14, 10)) for n in stages: ax = plt.plot(upc_profile[n], label=n) plt.legend() plt.ylabel("Number of UMIs") plt.xlabel("Rank Order") plt.xscale("log", basex=10) plt.yscale("log", basey=10) plt.title("UMIs per CellBC") plt.savefig(os.path.join(output_directory, "umis_per_cellbc.png")) plt.close() h = plt.figure(figsize=(14, 10)) for n in stages: ax = plt.hist(upi_profile[n], label=n, histtype="step", log=True, bins=200) plt.legend() plt.ylabel("Frequency") plt.xlabel("Number of UMIs") plt.title("UMIs per intBC") plt.savefig(os.path.join(output_directory, "umis_per_intbc.png")) plt.close() logger.info( f"Overall, filtered {cellBC_count} cells, with {filtered_df.shape[0]} UMIs." ) filtered_df.set_index("readName", inplace=True) filtered_df.reset_index(inplace=True) return filtered_df
def call_lineage_groups( input_df: pd.DataFrame, output_directory: str, min_umi_per_cell: int = 10, min_avg_reads_per_umi: float = 2.0, min_cluster_prop: float = 0.005, min_intbc_thresh: float = 0.05, inter_doublet_threshold: float = 0.35, kinship_thresh: float = 0.25, plot: bool = False, ) -> pd.DataFrame: """Assigns cells to their clonal populations. Performs multiple rounds of filtering and assigning to lineage groups: 1. Iteratively generates putative lineage groups by forming intBC groups for each lineage group and then assigning cells based on how many intBCs they share with each intBC group (kinship). 2. Refines these putative groups by removing non-informative intBCs and reassigning cells through kinship. 3. Removes all inter-lineage doublets, defined as cells that have relatively equal kinship scores across multiple lineages and whose assignments are therefore ambigious. 4. Finally, performs one more round of filtering non-informative intBCs and cellBCs with low UMI counts before returning a final table of lineage assignments, allele information, and read and umi counts for each sample. Args: input_df: The allele table of cellBC-UMI-allele groups to be annotated with lineage assignments output_directory: The folder to store the final table as well as plots min_umi_per_cell: The threshold specifying the minimum number of UMIs a cell needs in order to not be filtered during filtering min_avg_reads_per_umi: The threshold specifying the minimum coverage (i.e. average) reads per UMI in a cell needed in order for that cell not to be filtered during filtering min_cluster_prop: The minimum cluster size in the putative lineage assignment step, as a proportion of the number of cells min_intbc_thresh: The threshold specifying the minimum proportion of cells in a lineage group that need to have an intBC in order for it be retained during filtering. Also specifies the minimum proportion of cells that share an intBC with the most frequent intBC in forming putative lineage groups inter_doublet_threshold: The threshold specifying the minimum proportion of kinship a cell shares with its assigned lineage group out of all lineage groups for it to be retained during doublet filtering kinship_thresh: The threshold specifying the minimum proportion of intBCs shared between a cell and the intBC set of a lineage group needed to assign that cell to that lineage group in putative assignment plot: Indicates whether to generate plots Returns: None, saves output allele table to file. """ logger.info( f"{input_df.shape[0]} UMIs (rows), with {input_df.shape[1]} attributes (columns)" ) logger.info(str(len(input_df["cellBC"].unique())) + " Cells") # Create a pivot_table piv = pd.pivot_table(input_df, index="cellBC", columns="intBC", values="UMI", aggfunc="count") piv = piv.div(piv.sum(axis=1), axis=0) # Reorder piv columns by binarized intBC frequency pivbin = piv.copy() pivbin[pivbin > 0] = 1 intBC_sums = pivbin.sum(0) ordered_intBCs = intBC_sums.sort_values(ascending=False).index.tolist() piv = piv[ordered_intBCs] min_clust_size = int(min_cluster_prop * piv.shape[0]) logger.info("Assigning initial lineage groups...") logger.info(f"Clustering with minimum cluster size {min_clust_size}...") piv_assigned = lineage_utils.assign_lineage_groups( piv, min_clust_size, min_intbc_thresh=min_intbc_thresh, kinship_thresh=kinship_thresh, ) logger.info("Refining lineage groups...") logger.info( "Redefining lineage groups by removing low proportion intBCs...") master_LGs, master_intBCs = lineage_utils.filter_intbcs_lg_sets( piv_assigned, min_intbc_thresh=min_intbc_thresh) logger.info("Reassigning cells to refined lineage groups by kinship...") kinship_scores = lineage_utils.score_lineage_kinships( piv_assigned, master_LGs, master_intBCs) logger.info("Annotating alignment table with refined lineage groups...") allele_table = lineage_utils.annotate_lineage_groups( input_df, kinship_scores, master_intBCs) if inter_doublet_threshold: logger.info( f"Filtering out inter-lineage group doublets with proportion {inter_doublet_threshold}..." ) allele_table = doublet_utils.filter_inter_doublets( allele_table, rule=inter_doublet_threshold) logger.info( "Filtering out low proportion intBCs in finalized lineage groups...") filtered_lgs = lineage_utils.filter_intbcs_final_lineages( allele_table, min_intbc_thresh=min_intbc_thresh) allele_table = lineage_utils.filtered_lineage_group_to_allele_table( filtered_lgs) logger.debug("Final lineage group assignments:") for n, g in allele_table.groupby(["lineageGrp"]): logger.debug(f"LG {n}: " + str(len(g["cellBC"].unique())) + " cells") logger.info("Filtering out low UMI cell barcodes...") allele_table = utilities.filter_cells( allele_table, min_umi_per_cell=int(min_umi_per_cell), min_avg_reads_per_umi=min_avg_reads_per_umi, ) allele_table["lineageGrp"] = allele_table["lineageGrp"].astype(int) if plot: logger.info("Producing Plots...") at_pivot_I = pd.pivot_table( allele_table, index="cellBC", columns="intBC", values="UMI", aggfunc="count", ) at_pivot_I.fillna(value=0, inplace=True) at_pivot_I[at_pivot_I > 0] = 1 logger.info("Producing pivot table heatmap...") lineage_utils.plot_overlap_heatmap(allele_table, at_pivot_I, output_directory) logger.info("Plotting filtered lineage group pivot table heatmap...") lineage_utils.plot_overlap_heatmap_lg(allele_table, at_pivot_I, output_directory) return allele_table
def resolve_umi_sequence( molecule_table: pd.DataFrame, output_directory: str, min_umi_per_cell: int = 10, min_avg_reads_per_umi: float = 2.0, plot: bool = True, ) -> pd.DataFrame: """Resolve a consensus sequence for each UMI. This procedure will perform UMI and cellBC filtering on the basis of reads per UMI and UMIs per cell and then assign the most abundant sequence to each UMI if there is a set of conflicting sequences per UMI. Args: molecule_table: molecule table to resolve output_directory: Directory to store results min_umi_per_cell: The threshold specifying the minimum number of UMIs in a cell needed to be retained during filtering min_avg_reads_per_umi: The threshold specifying the minimum coverage (i.e. average) reads per UMI in a cell needed for that cell to be retained during filtering Returns: A molecule table with unique mappings between cellBC-UMI pairs. """ if plot: # -------------------- Plot # of sequences per UMI -------------------- # equivClass_group = (molecule_table.groupby(["cellBC", "UMI"]).agg({ "grpFlag": "count" }).sort_values("grpFlag", ascending=False).reset_index()) _ = plt.figure(figsize=(8, 5)) plt.hist( equivClass_group["grpFlag"], bins=range(1, equivClass_group["grpFlag"].max()), ) plt.title("Unique Seqs per cellBC+UMI") plt.yscale("log", basey=10) plt.xlabel("Number of Unique Seqs") plt.ylabel("Count (Log)") plt.savefig(os.path.join(output_directory, "seqs_per_equivClass.png")) plt.close() # ----------------- Select most abundant sequence ------------------ # mt_filter = {} total_numReads = {} top_reads = {} second_reads = {} first_reads = {} unique_pairs = molecule_table.groupby(["cellBC", "UMI"], sort=False) for _, group in tqdm( unique_pairs, total=len(unique_pairs.size()), desc="Resolving UMI sequences", ): # base case - only one sequence if group.shape[0] == 1: good_readName = group["readName"].iloc[0] mt_filter[good_readName] = False total_numReads[good_readName] = group["readCount"] top_reads[good_readName] = group["readCount"] # more commonly - many sequences for a given UMI else: group_sort = group.sort_values("readCount", ascending=False, ignore_index=True) good_readName = group_sort["readName"].iloc[0] # keep the first entry (highest readCount) mt_filter[good_readName] = False total_numReads[good_readName] = group_sort["readCount"].sum() top_reads[good_readName] = group_sort["readCount"].iloc[0] second_reads[good_readName] = group_sort["readCount"].iloc[1] first_reads[good_readName] = group_sort["readCount"].iloc[0] # mark remaining UMIs for filtering for i in range(1, group.shape[0]): bad_readName = group_sort["readName"].iloc[i] mt_filter[bad_readName] = True # apply the filter using the hash table created above molecule_table["filter"] = molecule_table["readName"].map(mt_filter) n_filtered = molecule_table[molecule_table["filter"]].shape[0] logger.info(f"Filtered out {n_filtered} reads.") # filter based on status & reindex filt_molecule_table = molecule_table[~molecule_table["filter"]].copy() filt_molecule_table.drop(columns=["filter"], inplace=True) if plot: # ---------------- Plot Diagnositics after Resolving ---------------- # h = plt.figure(figsize=(14, 10)) plt.plot(list(top_reads.values()), list(total_numReads.values()), "r.") plt.ylabel("Total Reads") plt.xlabel("Number Reads for Picked Sequence") plt.title("Total vs. Top Reads for Picked Sequence") plt.savefig( os.path.join(output_directory, "total_vs_top_reads_pickSeq.png")) plt.close() h = plt.figure(figsize=(14, 10)) plt.plot(list(first_reads.values()), list(second_reads.values()), "r.") plt.ylabel("Number Reads for Second Best Sequence") plt.xlabel("Number Reads for Picked Sequence") plt.title("Second Best vs. Top Reads for Picked Sequence") plt.savefig( os.path.join(output_directory, "second_vs_top_reads_pickSeq.png")) plt.close() filt_molecule_table = utilities.filter_cells( filt_molecule_table, min_umi_per_cell=min_umi_per_cell, min_avg_reads_per_umi=min_avg_reads_per_umi, ) return filt_molecule_table