def get_proportion(exp, nonzero, singleExp): labels = v5.exp_to_celltypes(exp) celltypes = [] for label in labels: celltypes.append(co.get_term_name(label)) for descendant in co.get_descendents(label): celltypes.append(co.get_term_name(descendant)) # print(nonzero.columns) expright = 0 ll = 0 # print(nonzero) for i in range(ll, len(nonzero['0'])): if nonzero['0'][i] == exp and nonzero['1'][i] in celltypes: expright += nonzero['2'][i] ll = i if singleExp: expright = expright / 2 # if expright > 1: # expright = expright/2 return expright
def study_view(study): """ Returns a pandas dataframe of cell types in a study ranked from greatest to least. Good for viewing in a notebook. """ print(study) nonzero = pd.read_csv(decon_temp + 'nonzero_' + study + '.tsv', sep='\t') acc_list = get_exp_list(nonzero, study) # order of experiments in study type_list = [[ co.get_term_name(celltype) for celltype in exp_to_celltypes(exp) ] for exp in acc_list] type_list = ['; '.join(exp) for exp in type_list] view = [] for exp in acc_list: exp_cells = [] ll = 0 for i in range(ll, len(nonzero['0'])): if nonzero['0'][i] == exp: exp_cells.append((nonzero['2'][i], nonzero['1'][i])) ll = i exp_cells.sort(reverse=True) view.append(exp_cells) view = pd.DataFrame(view, index=acc_list) view.insert(0, "labeled_type", type_list) return view
def study_type(study): ancestor_dict = {} exps = v5.study_to_exps(study) for exp in exps: ancestors = v5.exp_to_celltypes(exp) for celltype in v5.exp_to_celltypes(exp): ancestors += co.get_ancestors(celltype) ancestor_dict[exp] = ancestors if len(exps) == 1: ancestors = ancestor_dict[exps[0]] else: first = True for exp in ancestor_dict: if first: ancestors = set(ancestor_dict[exp]) first = False continue ancestors = ancestors | set(ancestor_dict[exp]) ancestors = list(ancestors) common = [] for term_id in co.get_terms_without_children(list(ancestors)): common.append(co.get_term_name(term_id)) common = '; '.join(common) return common
def create_reference(index_list): """ Given a set of studies from which to draw from, creates a reference matrix. """ cell_types = [] for i in index_list: cell_types += exp_to_celltypes(exp_acc[i]) # TEST if 'CL:0000236' in cell_types: print("Yes!") else: print("No!") cell_types = list(set(cell_types)) leaves = co.get_terms_without_children(cell_types) if 'CL:0000236' in leaves: print("Yes! Two") else: print("No! Two") leaf_index = {} for leaf in leaves: leaf_index[leaf] = [ exp_to_index(celltype) for celltype in celltype_to_exp(leaf) ] # TEST # TODO: this could use some clarification # explain what's going on with column stack and why you picked it first = True for leaf in leaves: signatures = get_signatures(leaf_index[leaf]) if len(leaf_index[leaf]) == 1: average = signatures else: average = np.mean(signatures, axis=1) if first: a = average first = False else: a = np.column_stack([a, average]) leaves = [co.get_term_name(leaf) for leaf in leaves] return {"gene_ids": gene_ids, "reference": a, "cell_types": leaves}
def scatterplot(ax, exp, celltype, remove_study = False): study = v5.exp_to_study(exp) if remove_study: if not is_type_available(celltype, study): print("Cell type ({}) not provided in reference ".format(celltype) + "matrix for this study.") return query = get_query_expression(exp) exp_list = v5.study_to_exps(study) reference = get_reference_expression(celltype, exp_list, remove_study) ax.scatter(reference[0], query) ax.set(xlim=(0, 450000), ylim=(0, 450000)) diag_line, = ax.plot(ax.get_xlim(), ax.get_ylim(), ls="--", c=".3") xstick = np.arange(0, 450000 ,100000) ax.set_xticks(xstick) ax.set_yticks(xstick) ax.set(xlabel = "{} ({} experiments)".format(co.get_term_name(celltype), reference[1]), ylabel = exp)
def decon_study(study, test_dataset=False, TM=False, TrainedNoiseModel_1=None, TrainedNoiseModel_2=None): """ Deconvolutes given study and creates a tsv with results. Attempts to write a tsv with nonzero proportions. """ cpm = h5py.File(file_name, 'r') ext = study # for ease of readability? not sure if it makes things more or # # less confusing # # TODO : perhaps split into multiple functions? if test_dataset: studies = np.array(cpm.get('study')).astype(str)[0:150] else: studies = np.array(cpm.get('study')).astype(str) cpm.close() a = set(studies) # set of all studies s = set([study]) # set with single element - study of interest m = a - s # set of all studies except study of interest # converts sets into lists of indices (artifact of old process) query_list = [] reference_list = [] for i in range(len(studies)): if studies[i] in m: reference_list.append(i) else: # reference_list.append(i) # What if query's cell type not in reference ?I think we need this. query_list.append(i) # handle single experiment-studies by duplicating query, as DeconRNASeq # requires at least two queries to run single_exp = len(query_list) == 1 if single_exp: query_list *= 2 # # write tsvs, build dictionary info = write_tsvs(reference_list, query_list, ext) info["labeled_type"] = [[ co.get_term_name(celltype) for celltype in exp_to_celltypes(exp_acc[i]) ] for i in query_list] # TEST info["study_id"] = study query_file = decon_temp_shell + "query_" + ext + ".tsv" reference_file = decon_temp_shell + "reference_" + ext + ".tsv" decon_fileNormal = decon_temp_shell + "results_Normal_" + ext + ".tsv" decon_fileWeight = decon_temp_shell + "results_Weight_" + ext + ".tsv" # # Deconvolution result = deconrnaseq(query_file, reference_file, use_scale=False) result[0].to_csv(decon_fileNormal, sep="\t") if TM == True: result = deconrnaseqweightCore(query_file, reference_file, use_scale=False, Trainmodel_1=TrainedNoiseModel_1, Trainmodel_2=TrainedNoiseModel_2) result[0].to_csv(decon_fileWeight, sep="\t") # delete duplicate query from results file # if single_exp: # trim_single_exp(ext) # generate nonzero tsv for each study. if successfully completed, delete # query and reference tsvs try: # if single_exp: # study_nonzero_single_exp(info) # else: study_nonzero(info) # os.remove(query_file) # os.remove(reference_file) except IndexError: print(str(IndexError)) # TODO: get a better error message. # why does this print '<class IndexError>'? # I'm keeping this return statement for debugging purposes # TODO : delete eventually return info
def multi_core(cell_exp_count, studies, exp_acc, gene_ids, countspermillion, qualified_cell_type_name, cell_type_file, qualified_cell_type): exp_acc_list = list(exp_acc) cell_types_selected = qualified_cell_type # Construct the noise added reference matrix reference_matrix = [] select_study_list = {} for i in qualified_cell_type: tmp_exp = V5.celltype_to_exp(i) select_sample = random.choice(tmp_exp) select_study_list[i] = V5.exp_to_study(select_sample) exp_index = exp_acc_list.index(select_sample) reference_matrix.append(exp_index) # print(select_study_list) # Build the noise added reference matrix for i in range(len(reference_matrix)): if i == 0: reference = countspermillion[reference_matrix[i]] else: tmp = countspermillion[reference_matrix[i]] reference = np.vstack((reference, tmp)) reference_noise = reference # Build the reference matrix reference_noise_free = [] for i in range(len(qualified_cell_type)): tmp_exp = V5.celltype_to_exp(qualified_cell_type[i]) tmp_ref = [] # Since all cell type will be included, therefore we can simply using the previous one` for j in tmp_exp: # Only one study wll be chosen to construct the noisy reference, therefore using != if V5.exp_to_study(j) != select_study_list[qualified_cell_type[i]]: tmp_ref.append(exp_acc_list.index(j)) for j in range(len(tmp_ref)): if j == 0: reference = countspermillion[tmp_ref[j]] else: tmp = countspermillion[tmp_ref[j]] reference = np.vstack((reference, tmp)) if len(tmp_ref) > 1: ref_mean = np.mean(reference, axis=0) else: ref_mean = reference reference_noise_free.append(ref_mean) reference_noise_free_np = np.array(reference_noise_free) signature_np = np.transpose(reference_noise_free_np) reference_noise_np = reference_noise.copy() signature_noise_np = np.transpose(reference_noise_np) # signature_temp = signature_np.copy() # Transform to pandas signature_np = signature_np.transpose() signature_noise_np = signature_noise_np.transpose() signature_pd = pd.DataFrame(data=signature_np, columns=gene_ids, index=qualified_cell_type_name) signature_noise_np_pd = pd.DataFrame( data=signature_noise_np, columns=gene_ids, index=[co.get_term_name(i) for i in qualified_cell_type]) # Save the signature and noisy signature for future analysis signature_pd.to_csv('~/IndependentStudy/Data/SignatureSimulation/' + str(cell_exp_count) + '_signature.tsv', sep='\t') signature_noise_np_pd.to_csv( '~/IndependentStudy/Data/SignatureSimulation/' + str(cell_exp_count) + '_signature_noise.tsv', sep='\t') # Build the variance data set # Eliminate the redundant cell type in all exp cell_type_specific_file = {} for i in cell_type_file: cell_type_specific_file[i] = co.get_terms_without_children( cell_type_file[i]) # Build the exp to study check dictionary studyexpMap = {} expstudyMap = {} for i in range(len(exp_acc)): expstudyMap[exp_acc[i]] = studies[i] if studies[i] not in studyexpMap: studyexpMap[studies[i]] = [exp_acc[i]] else: studyexpMap[studies[i]].append(exp_acc[i]) # Build the variance matrix variance_matrix = [] cell_types_48 = [] for cell_co in range(len(cell_types_selected)): # Get the cell type cellExpDict = {} for i in cell_type_specific_file: if cell_types_selected[cell_co] in cell_type_specific_file[i]: cellExpDict[i] = [cell_types_selected[cell_co]] # cell type specific Exp to Study dictionary expPerStudy = [] keys = list(cellExpDict.keys()) # print(keys) studyList = [] for i in keys: if expstudyMap[i] not in studyList: studyList.append(expstudyMap[i]) expPerStudy.append(i) else: continue tmp_exp_study = {} for i in cellExpDict.keys(): if expstudyMap[i] not in tmp_exp_study.keys(): tmp_exp_study[expstudyMap[i]] = [i] else: tmp_exp_study[expstudyMap[i]].append(i) # Get the within study variance # Generate the mean profile tmp_mean = [] within_study_var = [] # Build the exp expression matrix # print(tmp_exp_study.items()) for j in tmp_exp_study.items(): # print(select_study_list[cell_types_selected[cell_co]]) # print(j[0]) if j[0] not in select_study_list[cell_types_selected[cell_co]]: # Garb the cell index specific_cell_exp_index = [] for i in range(len(exp_acc)): if exp_acc[i] in j[1]: specific_cell_exp_index.append(i) else: continue specific_cell_exp_signature = get_signatures( specific_cell_exp_index, countspermillion) # Generate the cell_type specific mean (j[1] is a tuple), tmp_mean consist study mean if len(j[1]) == 1: tmp_mean.append(specific_cell_exp_signature) else: tmp_mean.append( np.mean(specific_cell_exp_signature, axis=1)) # Calculate the residue (if j[1] > 1) if len(j[1]) > 1: tmp_residue_list = [] for index in specific_cell_exp_index: tmp_exp = get_signatures([index], countspermillion) tmp_residue = np.abs( tmp_exp - np.mean(specific_cell_exp_signature, axis=1)) tmp_residue_list.append(tmp_residue) # Construct the within study variance tmp_residue_list = np.array(tmp_residue_list) within_study_var.append(np.var(tmp_residue_list, axis=0)) else: within_study_var.append( np.zeros(specific_cell_exp_signature.shape[0])) else: continue cell_types_48 += tmp_mean within_study_var = np.array(within_study_var) # Construct the study variance tmp_mean = np.array(tmp_mean) study_variance = np.var(tmp_mean, axis=0) # We assume variance sum law here total_variance = np.zeros(study_variance.shape[0]) total_variance = total_variance + study_variance for i in within_study_var: total_variance = total_variance + i variance_matrix.append(total_variance) variance_matrix = np.array(variance_matrix) print(variance_matrix.shape) os.system("touch " + '~/IndependentStudy/Data/Variance/' + str(cell_exp_count) + '_variance.txt') np.savetxt('/ua/shi235/IndependentStudy/Data/Variance/' + str(cell_exp_count) + '_variance.txt', variance_matrix, delimiter="\t")
cpm.close() qualified_cell_type = [ 'CL:1000274', 'CL:0002618', 'CL:0000501', 'CL:0000765', 'CL:2000001', 'CL:0002341', 'CL:0000583', 'CL:0000127', 'CL:0002631', 'CL:0000936', 'CL:0002327', 'CL:0000023', 'CL:0000216', 'CL:0000557', 'CL:0000018', 'CL:0000905', 'CL:0000182', 'CL:0000895', 'CL:0000096', 'CL:0002340', 'CL:0011001', 'CL:0000050', 'CL:0002633', 'CL:0000232', 'CL:0000019', 'CL:0000792', 'CL:0002063', 'CL:0000836', 'CL:0000904', 'CL:0002399', 'CL:0000233', 'CL:0002038', 'CL:0000788', 'CL:0000900', 'CL:0000670', 'CL:0002057', 'CL:0000351', 'CL:0001069', 'CL:0000091', 'CL:0000359', 'CL:0010004', 'CL:0000171', 'CL:0000169', 'CL:0000017', 'CL:0000623', 'CL:0001057', 'CL:0002394', 'CL:0000129' ] qualified_cell_type_name = [ co.get_term_name(i) for i in qualified_cell_type ] with open('cell_types.json', 'r') as type_file: cell_type_file = json.load(type_file) print("Start!") now = time.time() Parallel(n_jobs=50)(delayed(multi_core)( i, studies, exp_acc, gene_ids, countspermillion, qualified_cell_type_name, cell_type_file, qualified_cell_type) for i in range(100)) # for i in range(5): # print(i) # multi_core(i, studies, exp_acc, gene_ids, countspermillion, qualified_cell_type_name) print("Finished in", time.time() - now, "sec")
def exp_to_celltypes(exp): ids = v5.exp_to_celltypes(exp) celltypes = [co.get_term_name(id) for id in ids] return '; '.join(celltypes)