def main(): tic = time.perf_counter() gn = Granatum() df = gn.pandas_from_assay(gn.get_import('assay')) mingenes = gn.get_arg('min_genes_per_cell') maxgenes = gn.get_arg('max_genes_per_cell') mt_percent = gn.get_arg('mt_genes_percent')/100.0 uniquegenecount = df.astype(bool).sum(axis=0) totalgenecount = df.sum(axis=0) mtrows = df[df.index.str.startswith('MT')] mtgenecount = mtrows.sum(axis=0) mtpercent = mtgenecount.div(totalgenecount) colsmatching = uniquegenecount.T[(uniquegenecount.T >= mingenes) & (uniquegenecount.T <= maxgenes) & (mtpercent.T <= mt_percent)].index.values adata = df.loc[:, colsmatching] num_orig_cells = uniquegenecount.T.index.size num_filtered_cells = len(colsmatching) num_lt_min = uniquegenecount.T[(uniquegenecount.T < mingenes)].index.size num_gt_max = uniquegenecount.T[(uniquegenecount.T > maxgenes)].index.size num_gt_mt = uniquegenecount.T[(mtpercent.T > mt_percent)].index.size gn.add_result("Number of cells is now {} out of {} original cells with {} below min genes, {} above max genes, and {} above mt percentage threshold.".format(num_filtered_cells, num_orig_cells, num_lt_min, num_gt_max, num_gt_mt), "markdown") plt.figure() plt.subplot(2, 1, 1) plt.title('Unique gene count distribution') sns.distplot(uniquegenecount, bins=int(200), color = 'darkblue', kde_kws={'linewidth': 2}) plt.ylabel('Frequency') plt.xlabel('Gene count') plt.subplot(2, 1, 2) plt.title('MT Percent Distribution') sns.distplot(mtpercent*100.0, bins=int(200), color = 'darkblue', kde_kws={'linewidth': 2}) plt.ylabel('Frequency') plt.xlabel('MT Percent') plt.tight_layout() caption = ( 'The distribution of expression levels for each cell with various metrics.' ) gn.add_current_figure_to_results(caption, zoom=1, dpi=75) gn.export(gn.assay_from_pandas(adata), "Filtered Cells Assay", dynamic=False) toc = time.perf_counter() time_passed = round(toc - tic, 2) timing = "* Finished cell filtering step in {} seconds*".format(time_passed) gn.add_result(timing, "markdown") gn.commit()
def main(): gn = Granatum() df = gn.pandas_from_assay(gn.get_import("assay")) epsilon = gn.get_arg('epsilon') min_cells_expressed = gn.get_arg('min_cells_expressed') filter_df = pd.DataFrame({'gene': df.index}) filter_df['sum_expr'] = [sum(df.values[i, :]) for i in range(df.shape[0])] filter_df['avg_expr'] = filter_df['sum_expr'] / df.shape[1] filter_df['num_expressed_genes'] = [ sum([x > epsilon for x in df.values[i, :]]) for i in range(df.shape[0]) ] filter_df[ 'removed'] = filter_df['num_expressed_genes'] < min_cells_expressed new_df = df.loc[np.logical_not(filter_df['removed'].values), :] gn.add_result( "\n".join([ "Number of genes before filtering: **{}**".format(df.shape[0]), "", "Number of genes after filtering: **{}**".format(new_df.shape[0]), ]), type="markdown", ) if filter_df.shape[0] > 0: filter_df_deleted = filter_df.loc[filter_df['removed'].values, :].drop( 'removed', axis=1) gn.add_result( { 'title': f"Removed genes ({filter_df_deleted.shape[0]})", 'orient': 'split', 'columns': filter_df_deleted.columns.values.tolist(), 'data': filter_df_deleted.values.tolist(), }, data_type='table', ) else: gn.add_result( f"No genes were removed. All {df.shape[0]} genes were kept. " f"See attachment **gene_selection.csv** for detail.", 'markdown', ) gn.export(filter_df.to_csv(index=False), 'gene_selection.csv', kind='raw', meta=None, raw=True) gn.export(gn.assay_from_pandas(new_df), "Filtered Assay", dynamic=False) gn.commit()
def main(): gn = Granatum() adata = gn.ann_data_from_assay(gn.get_import("assay")) num_top_comps = gn.get_arg("num_top_comps") sc.pp.pca(adata, 20) variance_ratios = adata.uns["pca"]["variance_ratio"] pc_labels = ["PC{}".format(x + 1) for x in range(len(variance_ratios))] plt.figure() plt.bar(pc_labels, variance_ratios) plt.tight_layout() gn.add_current_figure_to_results( "Explained variance (ratio) by each Principal Component (PC)", height=350, dpi=75) X_pca = adata.obsm["X_pca"] for i, j in combinations(range(num_top_comps), 2): xlabel = "PC{}".format(i + 1) ylabel = "PC{}".format(j + 1) plt.figure() plt.scatter(X_pca[:, i], X_pca[:, j], s=5000 / adata.shape[0]) plt.xlabel(xlabel) plt.ylabel(ylabel) plt.tight_layout() gn.add_current_figure_to_results("PC{} vs. PC{}".format(i + 1, j + 1), dpi=75) pca_export = { "dimNames": [xlabel, ylabel], "coords": { sample_id: X_pca[k, [i, j]].tolist() for k, sample_id in enumerate(adata.obs_names) }, } gn.export(pca_export, "PC{} vs. PC{}".format(i + 1, j + 1), kind="sampleCoords", meta={}) gn.commit()
def main(): gn = Granatum() assay = gn.get_import('assay') args_for_init = { 'selected_embedding': gn.get_arg('selectedEmbedding'), 'selected_clustering': gn.get_arg('selectedClustering'), 'n_components': gn.get_arg('nComponents'), 'n_clusters': gn.get_arg('nClusters'), 'find_best_number_of_cluster': gn.get_arg('findBestNumberOfCluster'), } args_for_fit = { 'matrix': np.transpose(np.array(assay.get('matrix'))), 'sample_ids': assay.get('sampleIds'), } granatum_clustering = GranatumDeepClustering(**args_for_init) fit_results = granatum_clustering.fit(**args_for_fit) fit_exp = fit_results.get('clusters') gn.export_statically(fit_exp, 'Cluster assignment') newdictstr = ['"'+str(k)+'"'+", "+str(v) for k, v in fit_exp.items()] gn.export("\n".join(newdictstr), 'Cluster assignment.csv', kind='raw', meta=None, raw=True) md_str = f"""\ ## Results * Cluster array: `{fit_results.get('clusters_array')}` * Cluster array: `{fit_results.get('clusters_array')}` * nClusters: {fit_results.get('n_clusters')} * Number of components: {fit_results.get('n_components')} * Outliers: {fit_results.get('outliers')}""" # gn.add_result(md_str, 'markdown') gn.add_result( { 'orient': 'split', 'columns': ['Sample ID', 'Cluster Assignment'], 'data': [{'Sample ID':x, 'Cluster Assignment':y} for x, y in zip(assay.get('sampleIds'), fit_results.get('clusters_array'))], }, 'table', ) gn.commit()
def main(): gn = Granatum() adata = gn.ann_data_from_assay(gn.get_import("assay")) num_cells_to_sample = gn.get_arg("num_cells_to_sample") random_seed = gn.get_arg("random_seed") np.random.seed(random_seed) num_cells_before = adata.shape[0] num_genes_before = adata.shape[1] if num_cells_to_sample > 0 and num_cells_to_sample < 1: num_cells_to_sample = round(num_cells_before * num_cells_to_sample) else: num_cells_to_sample = round(num_cells_to_sample) if num_cells_to_sample > num_cells_before: num_cells_to_sample = num_cells_before if num_cells_to_sample < 1: num_cells_to_sample = 1 sampled_cells_idxs = np.sort(np.random.choice(num_cells_before, num_cells_to_sample, replace=False)) adata = adata[sampled_cells_idxs, :] gn.add_result( "\n".join( [ "The assay before down-sampling has **{}** cells and {} genes.".format( num_cells_before, num_genes_before ), "", "The assay after down-sampling has **{}** cells and {} genes.".format(adata.shape[0], adata.shape[1]), ] ), type="markdown", ) gn.export(gn.assay_from_ann_data(adata), "Down-sampled Assay", dynamic=False) gn.commit()
def main(): gn = Granatum() gene_scores = gn.get_import("gene_scores") species = gn.get_arg("species") gset_group_id = gn.get_arg("gset_group_id") n_repeats = gn.get_arg("n_repeats") alterChoice = gn.get_arg("alterChoice") if alterChoice=="pos": gene_scores = dict(filter(lambda elem: elem[1] >= 0.0, gene_scores.items())) elif alterChoice=="neg": gene_scores = dict(filter(lambda elem: elem[1] < 0.0, gene_scores.items())) gene_scores = { k: abs(v) for k, v in gene_scores.items() } gene_ids = gene_scores.keys() gene_scores = gene_scores.values() gene_id_type = guess_gene_id_type(list(gene_ids)[:5]) if gene_id_type != 'symbol': gene_ids = convert_gene_ids(gene_ids, gene_id_type, 'symbol', species) if species == "human": pass elif species == "mouse": gene_ids = zgsea.to_human_homolog(gene_ids, "mouse") else: raise ValueError() result_df = zgsea.gsea(gene_ids, gene_scores, gset_group_id, n_repeats=n_repeats) if result_df is None: gn.add_markdown('No gene set is enriched with your given genes.') else: result_df = result_df[["gset_name", "gset_size", "nes", "p_val", "fdr"]] gn.add_pandas_df(result_df) gn.export(result_df.to_csv(index=False), 'gsea_results.csv', kind='raw', meta=None, raw=True) newdict = dict(zip(result_df.index.tolist(), result_df['nes'].tolist())) print(newdict, flush=True) gn.export(newdict, 'nes', 'geneMeta') gn.commit()
def main(): gn = Granatum() adata = gn.ann_data_from_assay(gn.get_import("assay")) min_cells_expressed = gn.get_arg("min_cells_expressed") min_mean = gn.get_arg("min_mean") max_mean = gn.get_arg("max_mean") min_disp = gn.get_arg("min_disp") max_disp = gn.get_arg("max_disp") num_genes_before = adata.shape[1] sc.pp.filter_genes(adata, min_cells=min_cells_expressed) filter_result = sc.pp.filter_genes_dispersion( adata.X, flavor='seurat', min_mean=math.log(min_mean), max_mean=math.log(max_mean), min_disp=min_disp, max_disp=max_disp, ) adata = adata[:, filter_result.gene_subset] sc.pl.filter_genes_dispersion(filter_result) gn.add_current_figure_to_results( "Each dot represent a gene. The gray dots are the removed genes. The x-axis is log-transformed.", zoom=3, dpi=50, height=400, ) gn.add_result( "\n".join( [ "Number of genes before filtering: **{}**".format(num_genes_before), "", "Number of genes after filtering: **{}**".format(adata.shape[1]), ] ), type="markdown", ) gn.export(gn.assay_from_ann_data(adata), "Filtered Assay", dynamic=False) gn.commit()
def main(): gn = Granatum() df = gn.pandas_from_assay(gn.get_import("assay")) frob_norm = np.linalg.norm(df.values) df = df / frob_norm gn.add_result( f"""\ The original assay had Frobenius norm of {frob_norm}, after normalization its Frobenius norm is now {np.linalg.norm(df.values)}""", 'markdown', ) gn.export(gn.assay_from_pandas(df), "Frobenius normalized assay", dynamic=False) gn.commit()
def main(): gn = Granatum() assay_df = gn.pandas_from_assay(gn.get_import('assay')) grdict = gn.get_import('groupVec') phe_dict = pd.Series(gn.get_import('groupVec')) groups = set(parse(gn.get_arg('groups'))) inv_map = {} for k, v in grdict.items(): if v in groups: inv_map[v] = inv_map.get(v, []) + [k] cells = [] for k, v in inv_map.items(): cells.extend(v) assay_df = assay_df.loc[:, cells] assay_df = assay_df.sparse.to_dense().fillna(0) #assay_mat = r['as.matrix'](pandas2ri.py2ri(assay_df)) # assay_mat = r['as.matrix'](conversion.py2rpy(assay_df)) phe_vec = phe_dict[assay_df.columns] r.source('./drive_DESeq2.R') ret_r = r['run_DESeq'](assay_df, phe_vec) ret_r_as_df = r['as.data.frame'](ret_r) # ret_py_df = pandas2ri.ri2py(ret_r_as_df) # TODO: maybe rename the columns to be more self-explanatory? result_df = ret_r_as_df result_df = result_df.sort_values('padj') result_df.index.name = 'gene' gn.add_pandas_df(result_df.reset_index(), description='The result table as returned by DESeq2.') gn.export(result_df.to_csv(), 'DESeq2_results.csv', raw=True) significant_genes = result_df.loc[ result_df['padj'] < 0.05]['log2FoldChange'].to_dict() gn.export(significant_genes, 'Significant genes', kind='geneMeta') gn.commit()
def main(): tic = time.perf_counter() gn = Granatum() assay = gn.pandas_from_assay(gn.get_import('assay')) # Groups is {"cell":"cluster} groups = gn.get_import('groups') certainty = gn.get_arg('certainty') alpha = 1 - certainty / 100.0 min_zscore = st.norm.ppf(gn.get_arg("certainty") / 100.0) min_dist = 0.1 # Likely we want to filter genes before we get started, namely if we cannot create a good statistic norms_df = assay.apply(np.linalg.norm, axis=1) assay = assay.loc[norms_df.T >= min_dist, :] inv_map = {} inv_map_rest = {} for k, v in groups.items(): inv_map[v] = inv_map.get(v, []) + [k] clist = inv_map_rest.get(v, list(assay.columns)) clist.remove(k) inv_map_rest[v] = clist # Inv map is {"cluster": ["cell"]} print("Completed setup", flush=True) cols = list(inv_map.keys()) colnames = [] for coli in cols: for colj in cols: if coli != colj: colnames.append("{} vs {}".format(coli, colj)) for coli in cols: colnames.append("{} vs rest".format(coli)) # Instead of scoring into a dataframe, let's analyze each statistically # Dict (gene) of dict (cluster) of dict (statistics) # { "gene_name" : { "cluster_name" : { statistics data } }} # Export would be percentage more/less expressed in "on" state # For example gene "XIST" expresses at least 20% more in cluster 1 vs cluster 4 with 95% certainty total_genes = len(assay.index) print("Executing parallel for {} genes".format(total_genes), flush=True) results = Parallel( n_jobs=math.floor(multiprocessing.cpu_count() * 2 * 9 / 10))( delayed(compref)(gene, assay.loc[gene, :], colnames, inv_map, inv_map_rest, alpha, min_dist, min_zscore) for gene in tqdm(list(assay.index))) result = pd.concat(results, axis=0) gn.export_statically(gn.assay_from_pandas(result.T), 'Differential expression sets') gn.export(result.to_csv(), 'differential_gene_sets.csv', kind='raw', meta=None, raw=True) toc = time.perf_counter() time_passed = round(toc - tic, 2) timing = "* Finished differential expression sets step in {} seconds*".format( time_passed) gn.add_result(timing, "markdown") gn.commit()
def main(): gn = Granatum() df = gn.pandas_from_assay(gn.get_import('assay')) alpha = gn.get_arg('alpha') jammit = JAMMIT.from_dfs([df]) res = jammit.run_for_one_alpha( alpha, verbose=1, convergence_threshold=0.000000001, ) u = res['u'] v = res['v'] gn.export(dict(zip(df.index, u)), 'Genes loadings', kind='geneMeta') gn.export(dict(zip(df.columns, v)), 'Sample scores', kind='sampleMeta') gene_df = pd.DataFrame({ 'id_': df.index, 'abs_loading': abs(u), 'loading': u }) gene_df = gene_df[['id_', 'abs_loading', 'loading']] gene_df = gene_df.loc[gene_df['loading'].abs() > EPSILON] gene_df = gene_df.sort_values('abs_loading', ascending=False) gn.add_result( { 'title': f"Signal genes ({len(gene_df)})", 'orient': 'split', 'columns': gene_df.columns.values.tolist(), 'data': gene_df.values.tolist(), }, data_type='table', ) gn.export(gene_df.to_csv(index=False), 'signal_genes.csv', kind='raw', meta=None, raw=True) sample_df = pd.DataFrame({ 'id_': df.columns, 'abs_score': abs(v), 'score': v }) sample_df = sample_df[['id_', 'abs_score', 'score']] sample_df = sample_df.loc[sample_df['score'].abs() > EPSILON] sample_df = sample_df.sort_values('abs_score', ascending=False) gn.add_result( { 'title': f"Signal samples ({len(sample_df)})", 'orient': 'split', 'columns': sample_df.columns.values.tolist(), 'data': sample_df.values.tolist(), }, data_type='table', ) gn.export(sample_df.to_csv(index=False), 'signal_samples.csv', kind='raw', meta=None, raw=True) subset_df = df.loc[gene_df['id_'], sample_df['id_']] gn.export(gn.assay_from_pandas(subset_df), 'Assay with only signal genes and samples', kind='assay') sns.clustermap(subset_df, cmap='RdBu') gn.add_current_figure_to_results( description='Cluster map of the signal genes and signal samples', zoom=2, width=750, height=850, dpi=50, ) plt.close() plt.figure() plt.scatter(range(len(u)), u, s=2, c='red') plt.xlabel('index') plt.ylabel('value in u') gn.add_current_figure_to_results( description= 'The *u* vector (loadings for genes) plotted as a scatter plot.', zoom=2, width=750, height=450, dpi=50, ) plt.close() plt.figure() plt.plot(range(len(v)), v) plt.scatter(range(len(v)), v, s=6, c='red') plt.xlabel('index') plt.ylabel('value in v') gn.add_current_figure_to_results( description= 'The *v* vector (scores for samples) plotted as a line plot.', zoom=2, width=750, height=450, dpi=50, ) plt.close() # gn.export_current_figure( # 'cluster_map.pdf', # zoom=2, # width=750, # height=850, # dpi=50, # ) gn.commit()
def main(): tic = time.perf_counter() gn = Granatum() assay = gn.get_import('assay') sample_ids = assay.get('sampleIds') group_dict = gn.get_import('groupVec') group_vec = pd.Categorical([group_dict.get(x) for x in sample_ids]) num_groups = len(group_vec.categories) figheight = 400 * (math.floor((num_groups - 1) / 7) + 1) adata = sc.AnnData(np.array(assay.get('matrix')).transpose()) adata.var_names = assay.get('geneIds') adata.obs_names = assay.get('sampleIds') adata.obs['groupVec'] = group_vec sc.pp.neighbors(adata, n_neighbors=20, use_rep='X', method='gauss') try: sc.tl.rank_genes_groups(adata, 'groupVec', n_genes=100000) sc.pl.rank_genes_groups(adata, n_genes=20) gn.add_current_figure_to_results('One-vs-rest marker genes', dpi=75, height=figheight) gn._pickle(adata, 'adata') rg_res = adata.uns['rank_genes_groups'] for group in rg_res['names'].dtype.names: genes_names = [str(x[group]) for x in rg_res['names']] scores = [float(x[group]) for x in rg_res['scores']] newdict = dict(zip(genes_names, scores)) gn.export(newdict, 'Marker score ({} vs. rest)'.format(group), kind='geneMeta') newdictstr = [ '"' + str(k) + '"' + ", " + str(v) for k, v in newdict.items() ] gn.export("\n".join(newdictstr), 'Marker score {} vs rest.csv'.format(group), kind='raw', meta=None, raw=True) # cluster_assignment = dict(zip(adata.obs_names, adata.obs['louvain'].values.tolist())) # gn.export_statically(cluster_assignment, 'cluster_assignment') toc = time.perf_counter() time_passed = round(toc - tic, 2) timing = "* Finished marker gene identification step in {} seconds*".format( time_passed) gn.add_result(timing, "markdown") gn.commit() except Exception as e: plt.figure() plt.text(0.01, 0.5, 'Incompatible group vector due to insufficent cells') plt.text(0.01, 0.3, 'Please retry the step with a different group vector') plt.axis('off') gn.add_current_figure_to_results('One-vs-rest marker genes') gn.add_result('Error = {}'.format(e), "markdown") gn.commit()
def main(): tic = time.perf_counter() gn = Granatum() assay_file = gn.get_uploaded_file_path("assayFile") sample_meta_file = gn.get_uploaded_file_path("sampleMetaFile") file_format = gn.get_arg("fileFormat") file_format_meta = gn.get_arg("fileFormatMeta") species = gn.get_arg("species") # Share the email address among other gboxes using a pickle dump # email_address = gn.get_arg("email_address") shared = {"email_address": email_address} with open(gn.swd + "/shared.pkl", "wb") as fp: pickle.dump(shared, fp) if file_format == "und": file_format = Path(assay_file).suffix[1:] if file_format == "csv": tb = pd.read_csv(assay_file, sep=",", index_col=0, engine='c', memory_map=True) elif file_format == "tsv": tb = pd.read_csv(assay_file, sep="\t", index_col=0, engine='c', memory_map=True) elif file_format.startswith("xls"): tb = pd.read_excel(assay_file, index_col=0) elif file_format == "zip": os.system("zip -d {} __MACOSX/\\*".format(assay_file)) os.system("unzip -p {} > {}.csv".format(assay_file, assay_file)) tb = pd.read_csv("{}.csv".format(assay_file), sep=",", index_col=0, engine='c', memory_map=True) elif file_format == "gz": os.system("gunzip -c {} > {}.csv".format(assay_file, assay_file)) tb = pd.read_csv("{}.csv".format(assay_file), sep=",", index_col=0, engine='c', memory_map=True) else: gn.error("Unknown file format: {}".format(file_format)) sample_ids = tb.columns.values.tolist() gene_ids = tb.index.values.tolist() gene_id_type = guess_gene_id_type(gene_ids[:5]) whether_convert_id = gn.get_arg("whether_convert_id") if whether_convert_id: to_id_type = gn.get_arg("to_id_type") add_info = gn.get_arg("add_info") # if there are duplicated ids, pick the first row # TODO: Need to have a more sophisticated handling of duplicated ids gene_ids, new_meta = convert_gene_ids(gene_ids, gene_id_type, to_id_type, species, return_new_meta=True) # TODO: remove NaN rows # TODO: combine duplicated rows if add_info: for col_name, col in new_meta.iteritems(): gn.export(col.to_dict(), col_name, "geneMeta") assay_export_name = "[A]{}".format(basename(assay_file)) exported_assay = { "matrix": tb.values.tolist(), "sampleIds": sample_ids, "geneIds": gene_ids, } gn.export(exported_assay, assay_export_name, "assay") entry_preview = '\n'.join( [', '.join(x) for x in tb.values[:10, :10].astype(str).tolist()]) gn.add_result( f"""\ The assay has **{tb.shape[0]}** genes (with inferred ID type: {biomart_col_dict[gene_id_type]}) and **{tb.shape[1]}** samples. The first few rows and columns: ``` {entry_preview} ``` """, "markdown", ) meta_rows = [] if sample_meta_file is not None: if file_format_meta == "und": file_format_meta = Path(sample_meta_file).suffix[1:] if file_format_meta == "csv": sample_meta_tb = pd.read_csv(sample_meta_file) elif file_format_meta == "tsv": sample_meta_tb = pd.read_csv(sample_meta_file, sep="\t") elif file_format_meta.startswith("xls"): sample_meta_tb = pd.read_excel(sample_meta_file) elif file_format_meta == "zip": os.system("unzip -p {} > {}.csv".format(sample_meta_file, sample_meta_file)) sample_meta_tb = pd.read_csv("{}.csv".format(sample_meta_file)) elif file_format_meta == "gz": os.system("gunzip -c {} > {}.csv".format(sample_meta_file, sample_meta_file)) sample_meta_tb = pd.read_csv("{}.csv".format(sample_meta_file)) else: gn.error("Unknown file format: {}".format(file_format)) for meta_name in sample_meta_tb.columns: meta_output_name = "[M]{}".format(meta_name) sample_meta_dict = dict( zip(sample_ids, sample_meta_tb[meta_name].values.tolist())) gn.export(sample_meta_dict, meta_output_name, "sampleMeta") num_sample_values = 5 sample_values = ", ".join(sample_meta_tb[meta_name].astype( str).values[0:num_sample_values].tolist()) num_omitted_values = len( sample_meta_tb[meta_name]) - num_sample_values if num_omitted_values > 0: etc = ", ... and {} more entries".format(num_omitted_values) else: etc = "" meta_rows.append({ 'meta_name': meta_name, 'sample_values': str(sample_values) + etc, }) # meta_message = '\n'.join( # "* Sample meta with name **{meta_name}** is accepted ({sample_values}).".format(**x) for x in meta_rows # ) # gn.add_result(meta_message, "markdown") # gn.add_result({'columns': []}, 'table') # TODO: SAVE assay pickle toc = time.perf_counter() time_passed = round(toc - tic, 2) timing = "* Finished upload step in {} seconds*".format(time_passed) gn.add_result(timing, "markdown") gn.commit()
def main(): tic = time.perf_counter() gn = Granatum() assay = gn.pandas_from_assay(gn.get_import('assay')) groups = gn.get_import('groups') min_zscore = gn.get_arg('min_zscore') max_zscore = gn.get_arg('max_zscore') min_expression_variation = gn.get_arg('min_expression_variation') inv_map = {} for k, v in groups.items(): inv_map[v] = inv_map.get(v, []) + [k] low_mean_dfs = [] high_mean_dfs = [] mean_dfs = [] std_dfs = [] colnames = [] for k, v in inv_map.items(): group_values = assay.loc[:, v] lowbound_clust = {} highbound_clust = {} for index, row in group_values.iterrows(): meanbounds = sms.DescrStatsW(row).tconfint_mean() lowbound_clust[index] = meanbounds[0] highbound_clust[index] = meanbounds[1] low_mean_dfs.append(pd.DataFrame.from_dict(lowbound_clust, orient="index", columns=[k])) high_mean_dfs.append(pd.DataFrame.from_dict(highbound_clust, orient="index", columns=[k])) mean_dfs.append(group_values.mean(axis=1)) std_dfs.append(group_values.std(axis=1)) colnames.append(k) mean_df = pd.concat(mean_dfs, axis=1) mean_df.columns = colnames low_mean_df = pd.concat(low_mean_dfs, axis=1) low_mean_df.columns = colnames high_mean_df = pd.concat(high_mean_dfs, axis=1) high_mean_df.columns = colnames std_df = pd.concat(std_dfs, axis=1) std_df.columns = colnames print(std_df) minvalues = std_df.min(axis=1).to_frame() minvalues.columns=["min"] print("Minvalues>>") print(minvalues, flush=True) genes_below_min = list((minvalues[minvalues["min"]<min_expression_variation]).index) print("{} out of {}".format(len(genes_below_min), len(minvalues.index)), flush=True) mean_df = mean_df.drop(genes_below_min, axis=0) low_mean_df = low_mean_df.drop(genes_below_min, axis=0) high_mean_df = high_mean_df.drop(genes_below_min, axis=0) std_df = std_df.drop(genes_below_min, axis=0) assay = assay.drop(genes_below_min, axis=0) print("Filtered assay to get {} columns by {} rows".format(len(assay.columns), len(assay.index)), flush=True) mean_rest_dfs = [] std_rest_dfs = [] colnames = [] for k, v in inv_map.items(): rest_v = list(set(list(assay.columns)).difference(set(v))) mean_rest_dfs.append(assay.loc[:, rest_v].mean(axis=1)) std_rest_dfs.append(assay.loc[:, rest_v].std(axis=1)) colnames.append(k) mean_rest_df = pd.concat(mean_rest_dfs, axis=1) mean_rest_df.columns = colnames std_rest_df = pd.concat(std_rest_dfs, axis=1) std_rest_df.columns = colnames zscore_dfs = [] cols = colnames colnames = [] for coli in cols: for colj in cols: if coli != colj: # Here we should check significance # Fetch most realistic mean comparison set, what is smallest difference between two ranges mean_diff_overlap_low_high = (low_mean_df[coli]-high_mean_df[colj]) mean_diff_overlap_high_low = (high_mean_df[coli]-low_mean_df[colj]) diff_df = mean_diff_overlap_low_high.combine(mean_diff_overlap_high_low, range_check) zscore_dfs.append((diff_df/(std_df[colj]+std_df[coli]/4)).fillna(0).clip(-max_zscore, max_zscore)) colnames.append("{} vs {}".format(coli, colj)) for coli in cols: zscore_dfs.append(((mean_df[coli]-mean_rest_df[colj])/(std_rest_df[colj]+std_rest_df[coli]/4)).fillna(0).clip(-max_zscore, max_zscore)) colnames.append("{} vs rest".format(coli)) zscore_df = pd.concat(zscore_dfs, axis=1) zscore_df.columns = colnames norms_df = zscore_df.apply(np.linalg.norm, axis=1) colsmatching = norms_df.T[(norms_df.T >= min_zscore)].index.values return_df = zscore_df.T[colsmatching] gn.export_statically(gn.assay_from_pandas(return_df), 'Differential expression sets') gn.export(return_df.T.to_csv(), 'differential_gene_sets.csv', kind='raw', meta=None, raw=True) toc = time.perf_counter() time_passed = round(toc - tic, 2) timing = "* Finished differential expression sets step in {} seconds*".format(time_passed) gn.add_result(timing, "markdown") gn.commit()
def main(): gn = Granatum() tb1 = gn.pandas_from_assay(gn.get_import('assay1')) tb2 = gn.pandas_from_assay(gn.get_import('assay2')) label1 = gn.get_arg('label1') label2 = gn.get_arg('label2') direction = gn.get_arg('direction') normalization = gn.get_arg('normalization') if direction == 'samples': tb1 = tb1.T tb2 = tb2.T overlapped_index = set(tb1.index) & set(tb2.index) tb1.index = [ f"{label1}_{x}" if x in overlapped_index else x for x in tb1.index ] tb2.index = [ f"{label2}_{x}" if x in overlapped_index else x for x in tb2.index ] if normalization == 'none': tb = pd.concat([tb1, tb2], axis=0) elif normalization == 'frobenius': ntb1 = np.linalg.norm(tb1) ntb2 = np.linalg.norm(tb2) ntb = np.mean([ntb1, ntb2]) fct1 = ntb / ntb1 fct2 = ntb / ntb2 tb = pd.concat([tb1 * fct1, tb2 * fct2], axis=0) gn.add_markdown(f"""\ Normalization info: - Assay **{label1}** is multiplied by {fct1} - Assay **{label2}** is multiplied by {fct2} """) elif normalization == 'mean': ntb1 = np.mean(tb1) ntb2 = np.mean(tb2) ntb = np.mean([ntb1, ntb2]) fct1 = ntb / ntb1 fct2 = ntb / ntb2 tb = pd.concat([tb1 * fct1, tb2 * fct2], axis=0) gn.add_markdown(f"""\ Normalization info:", - Assay **{label1}** is multiplied by {fct1} - Assay **{label2}** is multiplied by {fct2} """) else: raise ValueError() if direction == 'samples': tb = tb.T gn.add_markdown(f"""\ You combined the following assays: - Assay 1 (with {tb1.shape[0]} genes and {tb1.shape[1]} cells) - Assay 2 (with {tb2.shape[0]} genes and {tb2.shape[1]} cells) into: - Combined Assay (with {tb.shape[0]} genes and {tb.shape[1]} cells) """) gn.export_statically(gn.assay_from_pandas(tb), 'Combined assay') if direction == 'samples': meta_type = 'sampleMeta' elif direction == 'genes': meta_type = 'geneMeta' else: raise ValueError() gn.export( { **{x: label1 for x in tb1.index}, **{x: label2 for x in tb2.index} }, 'Assay label', meta_type) gn.commit()
def main(): gn = Granatum() gene_scores_dict = gn.get_import("gene_scores") species = gn.get_arg("species") gset_group_id = gn.get_arg("gset_group_id") threshold = gn.get_arg("threshold") use_abs = gn.get_arg("use_abs") background = gn.get_arg("background") gene_ids = list(gene_scores_dict.keys()) gene_scores = list(gene_scores_dict.values()) gene_id_type = guess_gene_id_type(list(gene_ids)[:5]) if gene_id_type != 'symbol': gene_ids = convert_gene_ids(gene_ids, gene_id_type, 'symbol', species) if species == "human": pass elif species == "mouse": gene_ids = zgsea.to_human_homolog(gene_ids, "mouse") # problem is that gene_ids is NAN after this else: raise ValueError() if use_abs: input_list = np.array(gene_ids)[ np.abs(np.array(gene_scores)) >= threshold] else: input_list = np.array(gene_ids)[np.array(gene_scores) >= threshold] print(input_list) gn.add_result( f"""\ Number of genes after thresholding: {len(input_list)} (out of original {len(gene_ids)}). Please see the attachment `list_of_genes.csv` for the list of genes considered in this enrichment analysis.""", 'markdown', ) gn.export(pd.Series(input_list).to_csv(index=False), 'list_of_genes.csv', kind='raw', meta=None, raw=True) if background == 'all': background_list = get_all_genes('human') elif background == 'from_gene_sets': background_list = None elif background == 'from_input': background_list = gene_ids else: raise ValueError() result_df = zgsea.simple_fisher(input_list, gset_group_id, background_list=background_list) result_df = result_df.sort_values('fdr') result_df = result_df[[ 'gene_set_name', 'size', 'p_val', 'fdr', 'odds_ratio', 'n_overlaps', 'overlapping_genes', ]] result_df.columns = [ 'Gene set', 'Gene set size', 'p-value', 'FDR', 'Odds ratio', 'Number of overlapping genes', 'Overlapping genes', ] gn.add_pandas_df(result_df) gn.export(result_df.to_csv(index=False), 'enrichment_results.csv', kind='raw', meta=None, raw=True) gn.commit()