def get_proportion(exp, nonzero, singleExp):
labels = v5.exp_to_celltypes(exp)
celltypes = []
for label in labels:
celltypes.append(co.get_term_name(label))
for descendant in co.get_descendents(label):
celltypes.append(co.get_term_name(descendant))
# print(nonzero.columns)
expright = 0
ll = 0
# print(nonzero)
for i in range(ll, len(nonzero['0'])):
if nonzero['0'][i] == exp and nonzero['1'][i] in celltypes:
expright += nonzero['2'][i]
ll = i
if singleExp:
expright = expright / 2
# if expright > 1:
# expright = expright/2
return expright
def study_view(study):
"""
Returns a pandas dataframe of cell types in a study ranked from greatest to
least. Good for viewing in a notebook.
"""
print(study)
nonzero = pd.read_csv(decon_temp + 'nonzero_' + study + '.tsv', sep='\t')
acc_list = get_exp_list(nonzero, study) # order of experiments in study
type_list = [[
co.get_term_name(celltype) for celltype in exp_to_celltypes(exp)
] for exp in acc_list]
type_list = ['; '.join(exp) for exp in type_list]
view = []
for exp in acc_list:
exp_cells = []
ll = 0
for i in range(ll, len(nonzero['0'])):
if nonzero['0'][i] == exp:
exp_cells.append((nonzero['2'][i], nonzero['1'][i]))
ll = i
exp_cells.sort(reverse=True)
view.append(exp_cells)
view = pd.DataFrame(view, index=acc_list)
view.insert(0, "labeled_type", type_list)
return view
def study_type(study):
ancestor_dict = {}
exps = v5.study_to_exps(study)
for exp in exps:
ancestors = v5.exp_to_celltypes(exp)
for celltype in v5.exp_to_celltypes(exp):
ancestors += co.get_ancestors(celltype)
ancestor_dict[exp] = ancestors
if len(exps) == 1:
ancestors = ancestor_dict[exps[0]]
else:
first = True
for exp in ancestor_dict:
if first:
ancestors = set(ancestor_dict[exp])
first = False
continue
ancestors = ancestors | set(ancestor_dict[exp])
ancestors = list(ancestors)
common = []
for term_id in co.get_terms_without_children(list(ancestors)):
common.append(co.get_term_name(term_id))
common = '; '.join(common)
return common
def create_reference(index_list):
"""
Given a set of studies from which to draw from, creates a reference matrix.
"""
cell_types = []
for i in index_list:
cell_types += exp_to_celltypes(exp_acc[i]) # TEST
if 'CL:0000236' in cell_types:
print("Yes!")
else:
print("No!")
cell_types = list(set(cell_types))
leaves = co.get_terms_without_children(cell_types)
if 'CL:0000236' in leaves:
print("Yes! Two")
else:
print("No! Two")
leaf_index = {}
for leaf in leaves:
leaf_index[leaf] = [
exp_to_index(celltype) for celltype in celltype_to_exp(leaf)
] # TEST
# TODO: this could use some clarification
# explain what's going on with column stack and why you picked it
first = True
for leaf in leaves:
signatures = get_signatures(leaf_index[leaf])
if len(leaf_index[leaf]) == 1:
average = signatures
else:
average = np.mean(signatures, axis=1)
if first:
a = average
first = False
else:
a = np.column_stack([a, average])
leaves = [co.get_term_name(leaf) for leaf in leaves]
return {"gene_ids": gene_ids, "reference": a, "cell_types": leaves}
def scatterplot(ax, exp, celltype, remove_study = False):
study = v5.exp_to_study(exp)
if remove_study:
if not is_type_available(celltype, study):
print("Cell type ({}) not provided in reference ".format(celltype)
+ "matrix for this study.")
return
query = get_query_expression(exp)
exp_list = v5.study_to_exps(study)
reference = get_reference_expression(celltype, exp_list, remove_study)
ax.scatter(reference[0], query)
ax.set(xlim=(0, 450000), ylim=(0, 450000))
diag_line, = ax.plot(ax.get_xlim(), ax.get_ylim(), ls="--", c=".3")
xstick = np.arange(0, 450000 ,100000)
ax.set_xticks(xstick)
ax.set_yticks(xstick)
ax.set(xlabel = "{} ({} experiments)".format(co.get_term_name(celltype),
reference[1]), ylabel = exp)
def decon_study(study,
test_dataset=False,
TM=False,
TrainedNoiseModel_1=None,
TrainedNoiseModel_2=None):
"""
Deconvolutes given study and creates a tsv with results. Attempts to write
a tsv with nonzero proportions.
"""
cpm = h5py.File(file_name, 'r')
ext = study # for ease of readability? not sure if it makes things more or
# # less confusing
# # TODO : perhaps split into multiple functions?
if test_dataset:
studies = np.array(cpm.get('study')).astype(str)[0:150]
else:
studies = np.array(cpm.get('study')).astype(str)
cpm.close()
a = set(studies) # set of all studies
s = set([study]) # set with single element - study of interest
m = a - s # set of all studies except study of interest
# converts sets into lists of indices (artifact of old process)
query_list = []
reference_list = []
for i in range(len(studies)):
if studies[i] in m:
reference_list.append(i)
else:
# reference_list.append(i) # What if query's cell type not in reference ?I think we need this.
query_list.append(i)
# handle single experiment-studies by duplicating query, as DeconRNASeq
# requires at least two queries to run
single_exp = len(query_list) == 1
if single_exp:
query_list *= 2
# # write tsvs, build dictionary
info = write_tsvs(reference_list, query_list, ext)
info["labeled_type"] = [[
co.get_term_name(celltype) for celltype in exp_to_celltypes(exp_acc[i])
] for i in query_list] # TEST
info["study_id"] = study
query_file = decon_temp_shell + "query_" + ext + ".tsv"
reference_file = decon_temp_shell + "reference_" + ext + ".tsv"
decon_fileNormal = decon_temp_shell + "results_Normal_" + ext + ".tsv"
decon_fileWeight = decon_temp_shell + "results_Weight_" + ext + ".tsv"
# # Deconvolution
result = deconrnaseq(query_file, reference_file, use_scale=False)
result[0].to_csv(decon_fileNormal, sep="\t")
if TM == True:
result = deconrnaseqweightCore(query_file,
reference_file,
use_scale=False,
Trainmodel_1=TrainedNoiseModel_1,
Trainmodel_2=TrainedNoiseModel_2)
result[0].to_csv(decon_fileWeight, sep="\t")
# delete duplicate query from results file
# if single_exp:
# trim_single_exp(ext)
# generate nonzero tsv for each study. if successfully completed, delete
# query and reference tsvs
try:
# if single_exp:
# study_nonzero_single_exp(info)
# else:
study_nonzero(info)
# os.remove(query_file)
# os.remove(reference_file)
except IndexError:
print(str(IndexError)) # TODO: get a better error message.
# why does this print '<class IndexError>'?
# I'm keeping this return statement for debugging purposes
# TODO : delete eventually
return info
def multi_core(cell_exp_count, studies, exp_acc, gene_ids, countspermillion,
qualified_cell_type_name, cell_type_file, qualified_cell_type):
exp_acc_list = list(exp_acc)
cell_types_selected = qualified_cell_type
# Construct the noise added reference matrix
reference_matrix = []
select_study_list = {}
for i in qualified_cell_type:
tmp_exp = V5.celltype_to_exp(i)
select_sample = random.choice(tmp_exp)
select_study_list[i] = V5.exp_to_study(select_sample)
exp_index = exp_acc_list.index(select_sample)
reference_matrix.append(exp_index)
# print(select_study_list)
# Build the noise added reference matrix
for i in range(len(reference_matrix)):
if i == 0:
reference = countspermillion[reference_matrix[i]]
else:
tmp = countspermillion[reference_matrix[i]]
reference = np.vstack((reference, tmp))
reference_noise = reference
# Build the reference matrix
reference_noise_free = []
for i in range(len(qualified_cell_type)):
tmp_exp = V5.celltype_to_exp(qualified_cell_type[i])
tmp_ref = []
# Since all cell type will be included, therefore we can simply using the previous one`
for j in tmp_exp:
# Only one study wll be chosen to construct the noisy reference, therefore using !=
if V5.exp_to_study(j) != select_study_list[qualified_cell_type[i]]:
tmp_ref.append(exp_acc_list.index(j))
for j in range(len(tmp_ref)):
if j == 0:
reference = countspermillion[tmp_ref[j]]
else:
tmp = countspermillion[tmp_ref[j]]
reference = np.vstack((reference, tmp))
if len(tmp_ref) > 1:
ref_mean = np.mean(reference, axis=0)
else:
ref_mean = reference
reference_noise_free.append(ref_mean)
reference_noise_free_np = np.array(reference_noise_free)
signature_np = np.transpose(reference_noise_free_np)
reference_noise_np = reference_noise.copy()
signature_noise_np = np.transpose(reference_noise_np)
# signature_temp = signature_np.copy()
# Transform to pandas
signature_np = signature_np.transpose()
signature_noise_np = signature_noise_np.transpose()
signature_pd = pd.DataFrame(data=signature_np,
columns=gene_ids,
index=qualified_cell_type_name)
signature_noise_np_pd = pd.DataFrame(
data=signature_noise_np,
columns=gene_ids,
index=[co.get_term_name(i) for i in qualified_cell_type])
# Save the signature and noisy signature for future analysis
signature_pd.to_csv('~/IndependentStudy/Data/SignatureSimulation/' +
str(cell_exp_count) + '_signature.tsv',
sep='\t')
signature_noise_np_pd.to_csv(
'~/IndependentStudy/Data/SignatureSimulation/' + str(cell_exp_count) +
'_signature_noise.tsv',
sep='\t')
# Build the variance data set
# Eliminate the redundant cell type in all exp
cell_type_specific_file = {}
for i in cell_type_file:
cell_type_specific_file[i] = co.get_terms_without_children(
cell_type_file[i])
# Build the exp to study check dictionary
studyexpMap = {}
expstudyMap = {}
for i in range(len(exp_acc)):
expstudyMap[exp_acc[i]] = studies[i]
if studies[i] not in studyexpMap:
studyexpMap[studies[i]] = [exp_acc[i]]
else:
studyexpMap[studies[i]].append(exp_acc[i])
# Build the variance matrix
variance_matrix = []
cell_types_48 = []
for cell_co in range(len(cell_types_selected)):
# Get the cell type
cellExpDict = {}
for i in cell_type_specific_file:
if cell_types_selected[cell_co] in cell_type_specific_file[i]:
cellExpDict[i] = [cell_types_selected[cell_co]]
# cell type specific Exp to Study dictionary
expPerStudy = []
keys = list(cellExpDict.keys())
# print(keys)
studyList = []
for i in keys:
if expstudyMap[i] not in studyList:
studyList.append(expstudyMap[i])
expPerStudy.append(i)
else:
continue
tmp_exp_study = {}
for i in cellExpDict.keys():
if expstudyMap[i] not in tmp_exp_study.keys():
tmp_exp_study[expstudyMap[i]] = [i]
else:
tmp_exp_study[expstudyMap[i]].append(i)
# Get the within study variance
# Generate the mean profile
tmp_mean = []
within_study_var = []
# Build the exp expression matrix
# print(tmp_exp_study.items())
for j in tmp_exp_study.items():
# print(select_study_list[cell_types_selected[cell_co]])
# print(j[0])
if j[0] not in select_study_list[cell_types_selected[cell_co]]:
# Garb the cell index
specific_cell_exp_index = []
for i in range(len(exp_acc)):
if exp_acc[i] in j[1]:
specific_cell_exp_index.append(i)
else:
continue
specific_cell_exp_signature = get_signatures(
specific_cell_exp_index, countspermillion)
# Generate the cell_type specific mean (j[1] is a tuple), tmp_mean consist study mean
if len(j[1]) == 1:
tmp_mean.append(specific_cell_exp_signature)
else:
tmp_mean.append(
np.mean(specific_cell_exp_signature, axis=1))
# Calculate the residue (if j[1] > 1)
if len(j[1]) > 1:
tmp_residue_list = []
for index in specific_cell_exp_index:
tmp_exp = get_signatures([index], countspermillion)
tmp_residue = np.abs(
tmp_exp -
np.mean(specific_cell_exp_signature, axis=1))
tmp_residue_list.append(tmp_residue)
# Construct the within study variance
tmp_residue_list = np.array(tmp_residue_list)
within_study_var.append(np.var(tmp_residue_list, axis=0))
else:
within_study_var.append(
np.zeros(specific_cell_exp_signature.shape[0]))
else:
continue
cell_types_48 += tmp_mean
within_study_var = np.array(within_study_var)
# Construct the study variance
tmp_mean = np.array(tmp_mean)
study_variance = np.var(tmp_mean, axis=0)
# We assume variance sum law here
total_variance = np.zeros(study_variance.shape[0])
total_variance = total_variance + study_variance
for i in within_study_var:
total_variance = total_variance + i
variance_matrix.append(total_variance)
variance_matrix = np.array(variance_matrix)
print(variance_matrix.shape)
os.system("touch " + '~/IndependentStudy/Data/Variance/' +
str(cell_exp_count) + '_variance.txt')
np.savetxt('/ua/shi235/IndependentStudy/Data/Variance/' +
str(cell_exp_count) + '_variance.txt',
variance_matrix,
delimiter="\t")
cpm.close()
qualified_cell_type = [
'CL:1000274', 'CL:0002618', 'CL:0000501', 'CL:0000765', 'CL:2000001',
'CL:0002341', 'CL:0000583', 'CL:0000127', 'CL:0002631', 'CL:0000936',
'CL:0002327', 'CL:0000023', 'CL:0000216', 'CL:0000557', 'CL:0000018',
'CL:0000905', 'CL:0000182', 'CL:0000895', 'CL:0000096', 'CL:0002340',
'CL:0011001', 'CL:0000050', 'CL:0002633', 'CL:0000232', 'CL:0000019',
'CL:0000792', 'CL:0002063', 'CL:0000836', 'CL:0000904', 'CL:0002399',
'CL:0000233', 'CL:0002038', 'CL:0000788', 'CL:0000900', 'CL:0000670',
'CL:0002057', 'CL:0000351', 'CL:0001069', 'CL:0000091', 'CL:0000359',
'CL:0010004', 'CL:0000171', 'CL:0000169', 'CL:0000017', 'CL:0000623',
'CL:0001057', 'CL:0002394', 'CL:0000129'
]
qualified_cell_type_name = [
co.get_term_name(i) for i in qualified_cell_type
]
with open('cell_types.json', 'r') as type_file:
cell_type_file = json.load(type_file)
print("Start!")
now = time.time()
Parallel(n_jobs=50)(delayed(multi_core)(
i, studies, exp_acc, gene_ids, countspermillion,
qualified_cell_type_name, cell_type_file, qualified_cell_type)
for i in range(100))
# for i in range(5):
# print(i)
# multi_core(i, studies, exp_acc, gene_ids, countspermillion, qualified_cell_type_name)
print("Finished in", time.time() - now, "sec")
def exp_to_celltypes(exp):
ids = v5.exp_to_celltypes(exp)
celltypes = [co.get_term_name(id) for id in ids]
return '; '.join(celltypes)
