def main(): tic = time.perf_counter() gn = Granatum() df = gn.pandas_from_assay(gn.get_import('assay')) mingenes = gn.get_arg('min_genes_per_cell') maxgenes = gn.get_arg('max_genes_per_cell') mt_percent = gn.get_arg('mt_genes_percent')/100.0 uniquegenecount = df.astype(bool).sum(axis=0) totalgenecount = df.sum(axis=0) mtrows = df[df.index.str.startswith('MT')] mtgenecount = mtrows.sum(axis=0) mtpercent = mtgenecount.div(totalgenecount) colsmatching = uniquegenecount.T[(uniquegenecount.T >= mingenes) & (uniquegenecount.T <= maxgenes) & (mtpercent.T <= mt_percent)].index.values adata = df.loc[:, colsmatching] num_orig_cells = uniquegenecount.T.index.size num_filtered_cells = len(colsmatching) num_lt_min = uniquegenecount.T[(uniquegenecount.T < mingenes)].index.size num_gt_max = uniquegenecount.T[(uniquegenecount.T > maxgenes)].index.size num_gt_mt = uniquegenecount.T[(mtpercent.T > mt_percent)].index.size gn.add_result("Number of cells is now {} out of {} original cells with {} below min genes, {} above max genes, and {} above mt percentage threshold.".format(num_filtered_cells, num_orig_cells, num_lt_min, num_gt_max, num_gt_mt), "markdown") plt.figure() plt.subplot(2, 1, 1) plt.title('Unique gene count distribution') sns.distplot(uniquegenecount, bins=int(200), color = 'darkblue', kde_kws={'linewidth': 2}) plt.ylabel('Frequency') plt.xlabel('Gene count') plt.subplot(2, 1, 2) plt.title('MT Percent Distribution') sns.distplot(mtpercent*100.0, bins=int(200), color = 'darkblue', kde_kws={'linewidth': 2}) plt.ylabel('Frequency') plt.xlabel('MT Percent') plt.tight_layout() caption = ( 'The distribution of expression levels for each cell with various metrics.' ) gn.add_current_figure_to_results(caption, zoom=1, dpi=75) gn.export(gn.assay_from_pandas(adata), "Filtered Cells Assay", dynamic=False) toc = time.perf_counter() time_passed = round(toc - tic, 2) timing = "* Finished cell filtering step in {} seconds*".format(time_passed) gn.add_result(timing, "markdown") gn.commit()
def main(): gn = Granatum() df = gn.pandas_from_assay(gn.get_import("assay")) epsilon = gn.get_arg('epsilon') min_cells_expressed = gn.get_arg('min_cells_expressed') filter_df = pd.DataFrame({'gene': df.index}) filter_df['sum_expr'] = [sum(df.values[i, :]) for i in range(df.shape[0])] filter_df['avg_expr'] = filter_df['sum_expr'] / df.shape[1] filter_df['num_expressed_genes'] = [ sum([x > epsilon for x in df.values[i, :]]) for i in range(df.shape[0]) ] filter_df[ 'removed'] = filter_df['num_expressed_genes'] < min_cells_expressed new_df = df.loc[np.logical_not(filter_df['removed'].values), :] gn.add_result( "\n".join([ "Number of genes before filtering: **{}**".format(df.shape[0]), "", "Number of genes after filtering: **{}**".format(new_df.shape[0]), ]), type="markdown", ) if filter_df.shape[0] > 0: filter_df_deleted = filter_df.loc[filter_df['removed'].values, :].drop( 'removed', axis=1) gn.add_result( { 'title': f"Removed genes ({filter_df_deleted.shape[0]})", 'orient': 'split', 'columns': filter_df_deleted.columns.values.tolist(), 'data': filter_df_deleted.values.tolist(), }, data_type='table', ) else: gn.add_result( f"No genes were removed. All {df.shape[0]} genes were kept. " f"See attachment **gene_selection.csv** for detail.", 'markdown', ) gn.export(filter_df.to_csv(index=False), 'gene_selection.csv', kind='raw', meta=None, raw=True) gn.export(gn.assay_from_pandas(new_df), "Filtered Assay", dynamic=False) gn.commit()
def main(): tic = time.perf_counter() gn = Granatum() assay = gn.pandas_from_assay(gn.get_import('assay')) groups = gn.get_import('groups') reflabels = gn.get_import('reflabels') remove_cells = gn.get_arg('remove_cells') inv_map = {} for k, v in groups.items(): inv_map[v] = inv_map.get(v, []) + [k] inv_map_ref = {} for k, v in reflabels.items(): inv_map_ref[v] = inv_map_ref.get(v, []) + [k] group_relabel = {} mislabelled_cells = [] for k, v in inv_map.items(): vset = set(v) label_scores = {} for kref, vref in inv_map_ref.items(): label_scores[kref] = len(set(vref).intersection(vset)) group_relabel[k] = max(label_scores, key=label_scores.get) mislabelled_cells = mislabelled_cells + list( vset.difference(set(inv_map_ref[group_relabel[k]]))) if remove_cells: gn.add_result( "Dropping {} mislabelled cells".format(len(mislabelled_cells)), "markdown") assay = assay.drop(mislabelled_cells, axis=1) groups = { key: val for key, val in groups.items() if not key in mislabelled_cells } for cell in groups: groups[cell] = group_relabel[groups[cell]] toc = time.perf_counter() time_passed = round(toc - tic, 2) gn.export_statically(gn.assay_from_pandas(assay), "Corresponded assay") gn.export_statically(groups, "Corresponded labels") timing = "* Finished sample coloring step in {} seconds*".format( time_passed) gn.add_result(timing, "markdown") gn.commit()
def main(): gn = Granatum() df = gn.pandas_from_assay(gn.get_import('assay')) n_steps = gn.get_arg('n_steps') min_theta = gn.get_arg('min_theta') max_theta = gn.get_arg('max_theta') jammit = JAMMIT.from_dfs([df]) jammit.scan( thetas=np.linspace(min_theta, max_theta, n_steps), calculate_fdr=True, n_perms=10, verbose=1, convergence_threshold=0.000000001, ) jammit_result = jammit.format(columns=['theta', 'alpha', 'n_sigs', 'fdr']) jammit_result['theta'] = jammit_result['theta'].round(3) jammit_result['alpha'] = jammit_result['alpha'].round(3) plt.plot(jammit_result['alpha'], jammit_result['fdr']) plt.xlabel('alpha') plt.ylabel('FDR') gn.add_current_figure_to_results('FDR plotted against alpha', height=400) gn.add_result( { 'pageSize': n_steps, 'orient': 'split', 'columns': [{ 'name': h, 'type': 'number', 'round': 3 } for h in jammit_result.columns], 'data': jammit_result.values.tolist(), }, data_type='table', ) gn.commit()
def main(): gn = Granatum() sample_coords = gn.get_import("viz_data") df = gn.pandas_from_assay(gn.get_import("assay")) gene_ids = parse(gn.get_arg("gene_ids")) groups = gn.get_import("groups") alpha = 1.0 - gn.get_arg("confint") / 100.0 min_zscore = st.norm.ppf(gn.get_arg("confint")) min_dist = 0.1 coords = sample_coords.get("coords") dim_names = sample_coords.get("dimNames") inv_map = {} for k, v in groups.items(): inv_map[v] = inv_map.get(v, []) + [k] for gene in gene_ids: plt.figure() # First form a statistic for all values, also puts out plot params = plot_fits(df.loc[gene, :].dropna().to_list(), color="r", alpha=alpha, min_dist=min_dist, min_zscore=min_zscore, label="All") for k, v in inv_map.items(): plt.subplot(1, 1, 1) plt.title('Gene expression level distribution for each cluster') plot_predict(df.loc[gene, v].dropna().to_list(), params, label=k) # sns.distplot(df.loc[gene,:].to_list(), bins=int(100), color = 'darkblue', kde_kws={'linewidth': 2}) plt.ylabel('Frequency') plt.xlabel('Gene expression') plt.legend() plt.tight_layout() caption = ( "The distribution of expression levels for gene {}.".format(gene)) gn.add_current_figure_to_results(caption, zoom=1, dpi=75) gn.commit()
def main(): tic = time.perf_counter() gn = Granatum() df = gn.pandas_from_assay(gn.get_import('assay')) n_neighbors = gn.get_arg('n_neighbors') min_dist = gn.get_arg('min_dist') metric = gn.get_arg('metric') random_seed = gn.get_arg('random_seed') embedding = umap.UMAP(n_neighbors=n_neighbors, min_dist=min_dist, metric=metric, random_state=random_seed).fit_transform(df.values.T) plt.figure() plt.scatter(embedding[:, 0], embedding[:, 1], min(5000 / df.shape[0], 36.0)) plt.xlabel('UMAP dim. 1') plt.ylabel('UMAP dim. 2') plt.tight_layout() gn.add_current_figure_to_results('UMAP plot: each dot represents a cell', dpi=75) pca_export = { 'dimNames': ['UMAP dim. 1', 'UMAP dim. 2'], 'coords': { sample_id: embedding[i, :].tolist() for i, sample_id in enumerate(df.columns) }, } gn.export_statically(pca_export, 'UMAP coordinates') toc = time.perf_counter() time_passed = round(toc - tic, 2) timing = "* Finished UMAP step in {} seconds*".format(time_passed) gn.add_result(timing, "markdown") gn.commit()
def main(): gn = Granatum() df = gn.pandas_from_assay(gn.get_import("assay")) frob_norm = np.linalg.norm(df.values) df = df / frob_norm gn.add_result( f"""\ The original assay had Frobenius norm of {frob_norm}, after normalization its Frobenius norm is now {np.linalg.norm(df.values)}""", 'markdown', ) gn.export(gn.assay_from_pandas(df), "Frobenius normalized assay", dynamic=False) gn.commit()
def main(): gn = Granatum() assay_df = gn.pandas_from_assay(gn.get_import('assay')) grdict = gn.get_import('groupVec') phe_dict = pd.Series(gn.get_import('groupVec')) groups = set(parse(gn.get_arg('groups'))) inv_map = {} for k, v in grdict.items(): if v in groups: inv_map[v] = inv_map.get(v, []) + [k] cells = [] for k, v in inv_map.items(): cells.extend(v) assay_df = assay_df.loc[:, cells] assay_df = assay_df.sparse.to_dense().fillna(0) #assay_mat = r['as.matrix'](pandas2ri.py2ri(assay_df)) # assay_mat = r['as.matrix'](conversion.py2rpy(assay_df)) phe_vec = phe_dict[assay_df.columns] r.source('./drive_DESeq2.R') ret_r = r['run_DESeq'](assay_df, phe_vec) ret_r_as_df = r['as.data.frame'](ret_r) # ret_py_df = pandas2ri.ri2py(ret_r_as_df) # TODO: maybe rename the columns to be more self-explanatory? result_df = ret_r_as_df result_df = result_df.sort_values('padj') result_df.index.name = 'gene' gn.add_pandas_df(result_df.reset_index(), description='The result table as returned by DESeq2.') gn.export(result_df.to_csv(), 'DESeq2_results.csv', raw=True) significant_genes = result_df.loc[ result_df['padj'] < 0.05]['log2FoldChange'].to_dict() gn.export(significant_genes, 'Significant genes', kind='geneMeta') gn.commit()
def main(): tic = time.perf_counter() gn = Granatum() assay = gn.pandas_from_assay(gn.get_import('assay')) # Groups is {"cell":"cluster} groups = gn.get_import('groups') certainty = gn.get_arg('certainty') alpha = 1 - certainty / 100.0 min_zscore = st.norm.ppf(gn.get_arg("certainty") / 100.0) min_dist = 0.1 # Likely we want to filter genes before we get started, namely if we cannot create a good statistic norms_df = assay.apply(np.linalg.norm, axis=1) assay = assay.loc[norms_df.T >= min_dist, :] inv_map = {} inv_map_rest = {} for k, v in groups.items(): inv_map[v] = inv_map.get(v, []) + [k] clist = inv_map_rest.get(v, list(assay.columns)) clist.remove(k) inv_map_rest[v] = clist # Inv map is {"cluster": ["cell"]} print("Completed setup", flush=True) cols = list(inv_map.keys()) colnames = [] for coli in cols: for colj in cols: if coli != colj: colnames.append("{} vs {}".format(coli, colj)) for coli in cols: colnames.append("{} vs rest".format(coli)) # Instead of scoring into a dataframe, let's analyze each statistically # Dict (gene) of dict (cluster) of dict (statistics) # { "gene_name" : { "cluster_name" : { statistics data } }} # Export would be percentage more/less expressed in "on" state # For example gene "XIST" expresses at least 20% more in cluster 1 vs cluster 4 with 95% certainty total_genes = len(assay.index) print("Executing parallel for {} genes".format(total_genes), flush=True) results = Parallel( n_jobs=math.floor(multiprocessing.cpu_count() * 2 * 9 / 10))( delayed(compref)(gene, assay.loc[gene, :], colnames, inv_map, inv_map_rest, alpha, min_dist, min_zscore) for gene in tqdm(list(assay.index))) result = pd.concat(results, axis=0) gn.export_statically(gn.assay_from_pandas(result.T), 'Differential expression sets') gn.export(result.to_csv(), 'differential_gene_sets.csv', kind='raw', meta=None, raw=True) toc = time.perf_counter() time_passed = round(toc - tic, 2) timing = "* Finished differential expression sets step in {} seconds*".format( time_passed) gn.add_result(timing, "markdown") gn.commit()
def main(): gn = Granatum() sample_coords = gn.get_import("viz_data") df = gn.pandas_from_assay(gn.get_import("assay")) gene_ids = gn.get_arg("gene_ids") overlay_genes = gn.get_arg("overlay_genes") max_colors = gn.get_arg("max_colors") min_level = gn.get_arg("min_level") max_level = gn.get_arg("max_level") convert_to_zscore = gn.get_arg("convert_to_zscore") min_marker_area = gn.get_arg("min_marker_area") max_marker_area = gn.get_arg("max_marker_area") min_alpha = gn.get_arg("min_alpha") max_alpha = gn.get_arg("max_alpha") grey_level = gn.get_arg("grey_level") coords = sample_coords.get("coords") dim_names = sample_coords.get("dimNames") cmaps = [] if overlay_genes: if max_colors == "": numcolors = len(gene_ids.split(',')) cycol = cycle('bgrcmk') for i in range(numcolors): cmaps = cmaps + [ LinearSegmentedColormap("fire", produce_cdict(next(cycol), grey=grey_level, min_alpha=min_alpha, max_alpha=max_alpha), N=256) ] else: for col in max_colors.split(','): col = col.strip() cmaps = cmaps + [ LinearSegmentedColormap("fire", produce_cdict(col, grey=grey_level, min_alpha=min_alpha, max_alpha=max_alpha), N=256) ] else: if max_colors == "": cmaps = cmaps + [LinearSegmentedColormap("fire", cdict, N=256)] else: for col in max_colors.split(','): col = col.strip() cmaps = cmaps + [ LinearSegmentedColormap("fire", produce_cdict(col, grey=grey_level, min_alpha=min_alpha, max_alpha=max_alpha), N=256) ] colorbar_height = 10 plot_height = 650 num_cbars = 1 if overlay_genes: num_cbars = len(gene_ids.split(',')) cbar_height_ratio = plot_height / (num_cbars * colorbar_height) fig, ax = plt.subplots( 1 + num_cbars, 1, gridspec_kw={'height_ratios': [cbar_height_ratio] + [1] * num_cbars}) gene_index = -1 for gene_id in gene_ids.split(','): gene_id = gene_id.strip() gene_index = gene_index + 1 if gene_id in df.index: if not overlay_genes: plt.clf() fig, ax = plt.subplots( 1 + num_cbars, 1, gridspec_kw={ 'height_ratios': [cbar_height_ratio] + [1] * num_cbars }) transposed_df = df.T mean = transposed_df[gene_id].mean() stdev = transposed_df[gene_id].std(ddof=0) if convert_to_zscore: scatter_df = pd.DataFrame( { "x": [a[0] for a in coords.values()], "y": [a[1] for a in coords.values()], "value": (df.loc[gene_id, :] - mean) / stdev }, index=coords.keys()) else: scatter_df = pd.DataFrame( { "x": [a[0] for a in coords.values()], "y": [a[1] for a in coords.values()], "value": df.loc[gene_id, :] }, index=coords.keys()) values_df = np.clip(scatter_df["value"], min_level, max_level, out=None) min_value = np.nanmin(values_df) max_value = np.nanmax(values_df) scaled_marker_size = (max_marker_area - min_marker_area) * ( values_df - min_value) / (max_value - min_value) + min_marker_area scaled_marker_size = scaled_marker_size * scaled_marker_size # s = 5000 / scatter_df.shape[0] scatter = ax[0].scatter( x=scatter_df["x"], y=scatter_df["y"], s=scaled_marker_size, c=values_df, cmap=cmaps[gene_index % len(cmaps)]) #Amp_3.mpl_colormap) cbar = fig.colorbar(scatter, cax=ax[1 + (gene_index % num_cbars)], orientation='horizontal', aspect=40) cbar.set_label(gene_id, rotation=0) ax[0].set_xlabel(dim_names[0]) ax[0].set_ylabel(dim_names[1]) if not overlay_genes: gn.add_current_figure_to_results( "Scatter-plot of {} expression".format(gene_id), dpi=75) else: # if the gene ID entered is not present in the assay # Communicate it to the user and output a table of available gene ID's description = 'The selected gene is not present in the assay. See the step that generated the assay' genes_in_assay = pd.DataFrame( df.index.tolist(), columns=['Gene unavailable in assay: choose from below']) gn.add_pandas_df(genes_in_assay, description) if overlay_genes: gn.add_current_figure_to_results( "Scatter-plot of {} expression".format(gene_ids), height=650 + 100 * len(gene_ids.split(',')), dpi=75) gn.commit()
def main(): gn = Granatum() df = gn.pandas_from_assay(gn.get_import('assay')) alpha = gn.get_arg('alpha') jammit = JAMMIT.from_dfs([df]) res = jammit.run_for_one_alpha( alpha, verbose=1, convergence_threshold=0.000000001, ) u = res['u'] v = res['v'] gn.export(dict(zip(df.index, u)), 'Genes loadings', kind='geneMeta') gn.export(dict(zip(df.columns, v)), 'Sample scores', kind='sampleMeta') gene_df = pd.DataFrame({ 'id_': df.index, 'abs_loading': abs(u), 'loading': u }) gene_df = gene_df[['id_', 'abs_loading', 'loading']] gene_df = gene_df.loc[gene_df['loading'].abs() > EPSILON] gene_df = gene_df.sort_values('abs_loading', ascending=False) gn.add_result( { 'title': f"Signal genes ({len(gene_df)})", 'orient': 'split', 'columns': gene_df.columns.values.tolist(), 'data': gene_df.values.tolist(), }, data_type='table', ) gn.export(gene_df.to_csv(index=False), 'signal_genes.csv', kind='raw', meta=None, raw=True) sample_df = pd.DataFrame({ 'id_': df.columns, 'abs_score': abs(v), 'score': v }) sample_df = sample_df[['id_', 'abs_score', 'score']] sample_df = sample_df.loc[sample_df['score'].abs() > EPSILON] sample_df = sample_df.sort_values('abs_score', ascending=False) gn.add_result( { 'title': f"Signal samples ({len(sample_df)})", 'orient': 'split', 'columns': sample_df.columns.values.tolist(), 'data': sample_df.values.tolist(), }, data_type='table', ) gn.export(sample_df.to_csv(index=False), 'signal_samples.csv', kind='raw', meta=None, raw=True) subset_df = df.loc[gene_df['id_'], sample_df['id_']] gn.export(gn.assay_from_pandas(subset_df), 'Assay with only signal genes and samples', kind='assay') sns.clustermap(subset_df, cmap='RdBu') gn.add_current_figure_to_results( description='Cluster map of the signal genes and signal samples', zoom=2, width=750, height=850, dpi=50, ) plt.close() plt.figure() plt.scatter(range(len(u)), u, s=2, c='red') plt.xlabel('index') plt.ylabel('value in u') gn.add_current_figure_to_results( description= 'The *u* vector (loadings for genes) plotted as a scatter plot.', zoom=2, width=750, height=450, dpi=50, ) plt.close() plt.figure() plt.plot(range(len(v)), v) plt.scatter(range(len(v)), v, s=6, c='red') plt.xlabel('index') plt.ylabel('value in v') gn.add_current_figure_to_results( description= 'The *v* vector (scores for samples) plotted as a line plot.', zoom=2, width=750, height=450, dpi=50, ) plt.close() # gn.export_current_figure( # 'cluster_map.pdf', # zoom=2, # width=750, # height=850, # dpi=50, # ) gn.commit()
def main(): tic = time.perf_counter() gn = Granatum() assay = gn.pandas_from_assay(gn.get_import('assay')) groups = gn.get_import('groups') min_zscore = gn.get_arg('min_zscore') max_zscore = gn.get_arg('max_zscore') min_expression_variation = gn.get_arg('min_expression_variation') inv_map = {} for k, v in groups.items(): inv_map[v] = inv_map.get(v, []) + [k] low_mean_dfs = [] high_mean_dfs = [] mean_dfs = [] std_dfs = [] colnames = [] for k, v in inv_map.items(): group_values = assay.loc[:, v] lowbound_clust = {} highbound_clust = {} for index, row in group_values.iterrows(): meanbounds = sms.DescrStatsW(row).tconfint_mean() lowbound_clust[index] = meanbounds[0] highbound_clust[index] = meanbounds[1] low_mean_dfs.append(pd.DataFrame.from_dict(lowbound_clust, orient="index", columns=[k])) high_mean_dfs.append(pd.DataFrame.from_dict(highbound_clust, orient="index", columns=[k])) mean_dfs.append(group_values.mean(axis=1)) std_dfs.append(group_values.std(axis=1)) colnames.append(k) mean_df = pd.concat(mean_dfs, axis=1) mean_df.columns = colnames low_mean_df = pd.concat(low_mean_dfs, axis=1) low_mean_df.columns = colnames high_mean_df = pd.concat(high_mean_dfs, axis=1) high_mean_df.columns = colnames std_df = pd.concat(std_dfs, axis=1) std_df.columns = colnames print(std_df) minvalues = std_df.min(axis=1).to_frame() minvalues.columns=["min"] print("Minvalues>>") print(minvalues, flush=True) genes_below_min = list((minvalues[minvalues["min"]<min_expression_variation]).index) print("{} out of {}".format(len(genes_below_min), len(minvalues.index)), flush=True) mean_df = mean_df.drop(genes_below_min, axis=0) low_mean_df = low_mean_df.drop(genes_below_min, axis=0) high_mean_df = high_mean_df.drop(genes_below_min, axis=0) std_df = std_df.drop(genes_below_min, axis=0) assay = assay.drop(genes_below_min, axis=0) print("Filtered assay to get {} columns by {} rows".format(len(assay.columns), len(assay.index)), flush=True) mean_rest_dfs = [] std_rest_dfs = [] colnames = [] for k, v in inv_map.items(): rest_v = list(set(list(assay.columns)).difference(set(v))) mean_rest_dfs.append(assay.loc[:, rest_v].mean(axis=1)) std_rest_dfs.append(assay.loc[:, rest_v].std(axis=1)) colnames.append(k) mean_rest_df = pd.concat(mean_rest_dfs, axis=1) mean_rest_df.columns = colnames std_rest_df = pd.concat(std_rest_dfs, axis=1) std_rest_df.columns = colnames zscore_dfs = [] cols = colnames colnames = [] for coli in cols: for colj in cols: if coli != colj: # Here we should check significance # Fetch most realistic mean comparison set, what is smallest difference between two ranges mean_diff_overlap_low_high = (low_mean_df[coli]-high_mean_df[colj]) mean_diff_overlap_high_low = (high_mean_df[coli]-low_mean_df[colj]) diff_df = mean_diff_overlap_low_high.combine(mean_diff_overlap_high_low, range_check) zscore_dfs.append((diff_df/(std_df[colj]+std_df[coli]/4)).fillna(0).clip(-max_zscore, max_zscore)) colnames.append("{} vs {}".format(coli, colj)) for coli in cols: zscore_dfs.append(((mean_df[coli]-mean_rest_df[colj])/(std_rest_df[colj]+std_rest_df[coli]/4)).fillna(0).clip(-max_zscore, max_zscore)) colnames.append("{} vs rest".format(coli)) zscore_df = pd.concat(zscore_dfs, axis=1) zscore_df.columns = colnames norms_df = zscore_df.apply(np.linalg.norm, axis=1) colsmatching = norms_df.T[(norms_df.T >= min_zscore)].index.values return_df = zscore_df.T[colsmatching] gn.export_statically(gn.assay_from_pandas(return_df), 'Differential expression sets') gn.export(return_df.T.to_csv(), 'differential_gene_sets.csv', kind='raw', meta=None, raw=True) toc = time.perf_counter() time_passed = round(toc - tic, 2) timing = "* Finished differential expression sets step in {} seconds*".format(time_passed) gn.add_result(timing, "markdown") gn.commit()
def main(): gn = Granatum() tb1 = gn.pandas_from_assay(gn.get_import('assay1')) tb2 = gn.pandas_from_assay(gn.get_import('assay2')) label1 = gn.get_arg('label1') label2 = gn.get_arg('label2') direction = gn.get_arg('direction') normalization = gn.get_arg('normalization') if direction == 'samples': tb1 = tb1.T tb2 = tb2.T overlapped_index = set(tb1.index) & set(tb2.index) tb1.index = [ f"{label1}_{x}" if x in overlapped_index else x for x in tb1.index ] tb2.index = [ f"{label2}_{x}" if x in overlapped_index else x for x in tb2.index ] if normalization == 'none': tb = pd.concat([tb1, tb2], axis=0) elif normalization == 'frobenius': ntb1 = np.linalg.norm(tb1) ntb2 = np.linalg.norm(tb2) ntb = np.mean([ntb1, ntb2]) fct1 = ntb / ntb1 fct2 = ntb / ntb2 tb = pd.concat([tb1 * fct1, tb2 * fct2], axis=0) gn.add_markdown(f"""\ Normalization info: - Assay **{label1}** is multiplied by {fct1} - Assay **{label2}** is multiplied by {fct2} """) elif normalization == 'mean': ntb1 = np.mean(tb1) ntb2 = np.mean(tb2) ntb = np.mean([ntb1, ntb2]) fct1 = ntb / ntb1 fct2 = ntb / ntb2 tb = pd.concat([tb1 * fct1, tb2 * fct2], axis=0) gn.add_markdown(f"""\ Normalization info:", - Assay **{label1}** is multiplied by {fct1} - Assay **{label2}** is multiplied by {fct2} """) else: raise ValueError() if direction == 'samples': tb = tb.T gn.add_markdown(f"""\ You combined the following assays: - Assay 1 (with {tb1.shape[0]} genes and {tb1.shape[1]} cells) - Assay 2 (with {tb2.shape[0]} genes and {tb2.shape[1]} cells) into: - Combined Assay (with {tb.shape[0]} genes and {tb.shape[1]} cells) """) gn.export_statically(gn.assay_from_pandas(tb), 'Combined assay') if direction == 'samples': meta_type = 'sampleMeta' elif direction == 'genes': meta_type = 'geneMeta' else: raise ValueError() gn.export( { **{x: label1 for x in tb1.index}, **{x: label2 for x in tb2.index} }, 'Assay label', meta_type) gn.commit()
def main(): tic = time.perf_counter() gn = Granatum() assay = gn.pandas_from_assay(gn.get_import('assay')) groups = gn.get_import('groups') inv_map = {} for k, v in groups.items(): inv_map[v] = inv_map.get(v, []) + [k] drop_set = parse(gn.get_arg('drop_set')) merge_set_1 = parse(gn.get_arg('merge_set_1')) merge_set_2 = parse(gn.get_arg('merge_set_2')) merge_set_3 = parse(gn.get_arg('merge_set_3')) relabel_set_1 = gn.get_arg('relabel_set_1') relabel_set_2 = gn.get_arg('relabel_set_2') relabel_set_3 = gn.get_arg('relabel_set_3') if len(merge_set_1) > 0: if relabel_set_1 == "": relabel_set_1 = " + ".join(merge_set_1) if len(merge_set_2) > 0: if relabel_set_2 == "": relabel_set_2 = " + ".join(merge_set_2) if len(merge_set_3) > 0: if relabel_set_3 == "": relabel_set_3 = " + ".join(merge_set_3) try: for ds in drop_set: cells = inv_map[ds] gn.add_result( "Dropping {} cells that match {}".format(len(cells), ds), "markdown") assay = assay.drop(cells, axis=1) groups = {key: val for key, val in groups.items() if val != ds} except Exception as e: gn.add_result( "Error found in drop set, remember it should be comma separated: {}" .format(e), "markdown") try: if len(merge_set_1) > 0: merge_set_1_cells = [] for ms1 in merge_set_1: merge_set_1_cells = merge_set_1_cells + inv_map[ms1] for cell in merge_set_1_cells: groups[cell] = relabel_set_1 if len(merge_set_2) > 0: merge_set_2_cells = [] for ms2 in merge_set_2: merge_set_2_cells = merge_set_2_cells + inv_map[ms2] for cell in merge_set_2_cells: groups[cell] = relabel_set_2 if len(merge_set_3) > 0: merge_set_3_cells = [] for ms3 in merge_set_3: merge_set_3_cells = merge_set_3_cells + inv_map[ms3] for cell in merge_set_3_cells: groups[cell] = relabel_set_3 except Exception as e: gn.add_result( "Error found in merge sets, remember it should be comma separated: {}" .format(e), "markdown") toc = time.perf_counter() time_passed = round(toc - tic, 2) gn.export_statically(gn.assay_from_pandas(assay), "Label adjusted assay") gn.export_statically(groups, "Adjusted labels") timing = "* Finished sample coloring step in {} seconds*".format( time_passed) gn.add_result(timing, "markdown") gn.commit()
def main(): tic = time.perf_counter() gn = Granatum() clustersvsgenes = gn.pandas_from_assay(gn.get_import('clustersvsgenes')) gset_group_id = gn.get_arg('gset_group_id') min_zscore = gn.get_arg('min_zscore') clustercomparisonstotest = list(clustersvsgenes.index) # Load all gene sets gsets = load_gsets(gset_group_id) G = nx.MultiDiGraph() clusternames = list(clustersvsgenes.T.columns) individualclusters = [ n[:n.index(" vs rest")] for n in clusternames if n.endswith("vs rest") ] print(individualclusters, flush=True) for cl in individualclusters: G.add_node(cl) # {pathway : {"cluster1":score1, "cluster2":score2}, pathway2 : {}} resultsmap = {} relabels = {} keys = {} urlsforkeys = {} currentkeyindex = 0 for gset in gsets: urlsforkeys[gset["name"]] = gset["url"] for cluster in clustercomparisonstotest: try: resultdf = clustersvsgenes.loc[cluster, gset["gene_ids"]] resultdf = np.nan_to_num(resultdf) score = np.nanmean(resultdf) if score >= min_zscore: keys[gset["name"]] = keys.get(gset["name"], currentkeyindex + 1) print("Score = {}".format(score), flush=True) olddict = resultsmap.get(gset["name"], {}) olddict[cluster] = score resultsmap[gset["name"]] = olddict from_to = re.split(' vs ', cluster) if from_to[1] != 'rest': G.add_weighted_edges_from( [(from_to[1], from_to[0], score * 2.0)], label=str(keys[gset["name"]]), penwidth=str(score * 2.0)) else: relabel_dict = relabels.get(from_to[0], "") if relabel_dict == "": relabel_dict = from_to[0] + ": " + str( keys[gset["name"]]) else: relabel_dict = relabel_dict + ", " + str( keys[gset["name"]]) relabels[from_to[0]] = relabel_dict currentkeyindex = max(currentkeyindex, keys[gset["name"]]) except Exception as inst: print("Key error with {}".format(gset["name"]), flush=True) print("Exception: {}".format(inst), flush=True) print("Relabels {}".format(relabels), flush=True) G = nx.relabel_nodes(G, relabels) pos = nx.spring_layout(G) edge_labels = nx.get_edge_attributes(G, 'label') write_dot(G, 'plot.dot') os.system("dot plot.dot -Tpng -Gdpi=600 > plot.png") with open('plot.png', "rb") as f: image_b64 = b64encode(f.read()).decode("utf-8") gn.results.append({ "type": "png", "width": 650, "height": 480, "description": 'Network of clusters based on expression', "data": image_b64, }) footnote = "" for k, v in sorted(keys.items(), key=lambda item: item[1]): newstr = "{}: [{}]({})".format(v, k, urlsforkeys[k]) if footnote == "": footnote = newstr else: footnote = footnote + " \n" + newstr gn.add_result(footnote, "markdown") # gn.export(return_df.T.to_csv(), 'differential_gene_sets.csv', kind='raw', meta=None, raw=True) toc = time.perf_counter() time_passed = round(toc - tic, 2) timing = "* Finished differential expression sets step in {} seconds*".format( time_passed) gn.add_result(timing, "markdown") gn.commit()
def main(): tic = time.perf_counter() gn = Granatum() clustersvsgenes = gn.pandas_from_assay(gn.get_import('clustersvsgenes')) max_dist = gn.get_arg('max_dist') min_zscore = gn.get_arg('min_zscore') clustercomparisonstotest = list(clustersvsgenes.index) G = nx.MultiDiGraph() clusternames = list(clustersvsgenes.T.columns) individualclusters = [ n[:n.index(" vs rest")] for n in clusternames if n.endswith("vs rest") ] print(individualclusters, flush=True) for cl in individualclusters: G.add_node(cl) # {pathway : {"cluster1":score1, "cluster2":score2}, pathway2 : {}} # resultsmap = {} relabels = {} keys = {} currentkeyindex = 0 maxexpression = np.max(np.max(clustersvsgenes)) print("Max expression = {}".format(maxexpression)) print("Number to analyze = {}".format( len(clustersvsgenes.columns) * len(clustercomparisonstotest)), flush=True) gene_count = 0 for gene_id in clustersvsgenes.columns: gene_count = gene_count + 1 print("Genecount = {}/{}".format(gene_count, len(clustersvsgenes.columns)), flush=True) add_all_edges_for_current_gene = True for cluster in clustercomparisonstotest: score = clustersvsgenes.loc[cluster, gene_id] if score >= min_zscore: add_edges = True if not gene_id in keys: # First check if within distance of another group closestkey = None closestkeyvalue = 1.0e12 for key in keys: gene_values = clustersvsgenes.loc[:, gene_id] ref_values = clustersvsgenes.loc[:, key] sc = np.sqrt( np.nansum(np.square(gene_values - ref_values)) / len(gene_values)) if sc <= max_dist and sc < closestkeyvalue: closestkeyvalue = sc closestkey = key break if closestkey == None: keys[gene_id] = currentkeyindex + 1 else: keys[gene_id] = keys[closestkey] add_edges = False add_all_edges_for_current_gene = False print("Found a near gene: {}".format(closestkey), flush=True) else: add_edges = add_all_edges_for_current_gene # print("Score = {}".format(score), flush=True) # olddict = resultsmap.get(gene_id, {}) # olddict[cluster] = score # resultsmap[gene_id] = olddict if add_edges: from_to = re.split(' vs ', cluster) if from_to[1] != 'rest': G.add_weighted_edges_from( [(from_to[1], from_to[0], score / maxexpression * 1.0)], label=str(keys[gene_id]), penwidth=str(score / maxexpression * 1.0)) else: relabel_dict = relabels.get(from_to[0], "") if relabel_dict == "": relabel_dict = from_to[0] + ": " + str( keys[gene_id]) else: relabel_dict = relabel_dict + ", " + str( keys[gene_id]) relabels[from_to[0]] = relabel_dict currentkeyindex = max(currentkeyindex, keys[gene_id]) print("Relabels {}".format(relabels), flush=True) G = nx.relabel_nodes(G, relabels) pos = nx.spring_layout(G) edge_labels = nx.get_edge_attributes(G, 'label') write_dot(G, 'plot.dot') os.system('dot plot.dot -Kcirco -Tpng -Gsize="6,6" -Gdpi=600 > plot.png') with open('plot.png', "rb") as f: image_b64 = b64encode(f.read()).decode("utf-8") gn.results.append({ "type": "png", "width": 650, "height": 480, "description": 'Network of clusters based on expression', "data": image_b64, }) footnote = "" inv_map = {} for k, v in keys.items(): inv_map[v] = inv_map.get(v, []) + [k] for k, v in sorted(inv_map.items(), key=lambda item: item[0]): newv = map(lambda gene: "[{}]({})".format(gene, geturl(gene)), v) vliststr = ", ".join(newv) newstr = "{}: {} {}".format( k, (clustersvsgenes.loc[clustersvsgenes[v[0]] > min_zscore, v[0]]).to_dict(), vliststr) if footnote == "": footnote = newstr else: footnote = footnote + " \n" + newstr gn.add_result(footnote, "markdown") # gn.export(return_df.T.to_csv(), 'differential_gene_sets.csv', kind='raw', meta=None, raw=True) toc = time.perf_counter() time_passed = round(toc - tic, 2) timing = "* Finished differential expression sets step in {} seconds*".format( time_passed) gn.add_result(timing, "markdown") gn.commit()