def cal_1D(comb, i):
res = pd.DataFrame(columns=['taxa', 'pm', 'pvalue'])
run_log = pd.DataFrame(columns=['taxa', 'numpyindex'])
cc_list = []
ct_list = []
tt_list = []
tc_list = []
cc_kde_list = []
ct_kde_list = []
tt_kde_list = []
tc_kde_list = []
count = 0
for m in comb:
try:
m = list(m)
pm_score, pvalue, cc, ct, tt, tc, cc_kde, ct_kde, tt_kde, tc_kde = cal_pm_socre_nD(
m, demodata)
if (pm_score is None) or (cc_kde is None) or (ct_kde is None) or (
tt_kde is None) or (tc_kde is None):
continue
res.loc[count, 'taxa'] = m[0]
res.loc[count, 'pm'] = pm_score
res.loc[count, 'pvalue'] = pvalue
run_log.loc[count, 'taxa'] = m[0]
run_log.loc[count, 'numpyindex'] = count
count = count + 1
cc_list.append(cc)
ct_list.append(ct)
tt_list.append(tt)
tc_list.append(tc)
cc_kde_list.append(cc_kde)
ct_kde_list.append(ct_kde)
tt_kde_list.append(tt_kde)
tc_kde_list.append(tc_kde)
if count % 3 == 0:
finishedtask = {
'hasfinished': count,
'isfinished': False,
}
load_save_project.save_dict(
logs_place + '/' + str(i) + '_task_1D.json', finishedtask)
except:
pass
finishedtask = {
'hasfinished': count,
'isfinished': True,
}
load_save_project.save_dict(logs_place + '/' + str(i) + '_task_1D.json',
finishedtask)
np.save(logs_place + '/' + str(i) + '_cc_list_1D.npy', cc_list)
np.save(logs_place + '/' + str(i) + '_ct_list_1D.npy', ct_list)
np.save(logs_place + '/' + str(i) + '_tt_list_1D.npy', tt_list)
np.save(logs_place + '/' + str(i) + '_tc_list_1D.npy', tc_list)
np.save(logs_place + '/' + str(i) + '_cc_kde_list_1D.npy', cc_kde_list)
np.save(logs_place + '/' + str(i) + '_ct_kde_list_1D.npy', ct_kde_list)
np.save(logs_place + '/' + str(i) + '_tt_kde_list_1D.npy', tt_kde_list)
np.save(logs_place + '/' + str(i) + '_tc_kde_list_1D.npy', tc_kde_list)
run_log.to_csv(logs_place + '/' + str(i) + 'runlog_1D.csv')
return res
return res
from multiprocessing import Pool
if __name__ == '__main__':
success_1D = False
pm_res=None
try:
comb = []
for a in range(len(genuslist)):
if genuslist[a] == condition:
continue
comb.append([genuslist[a]])
totaltask = {'totalcomb': len(comb),
'isfinished': False
}
load_save_project.save_dict(logs_place + '/totaltask_1D.json', totaltask)
# print(datetime.datetime.now())
cl = np.array_split(np.asarray(range(len(comb))), processers, axis=0)
comb = np.asarray(comb)
res_list = []
p = Pool(processers)
for i in range(processers):
res_list.append(p.apply_async(cal_1D, args=(comb[cl[i]], i,)))
p.close()
p.join()
pm_res = pd.concat([i.get() for i in res_list])
tmpPvalue = pm_res['pvalue']
# fdrcorrection:
# This covers Benjamini/Hochberg for independent or positively correlated and Benjamini/Yekutieli
# for general or negatively correlated tests. Both are available in the function multipletests, as method=`fdr_bh`, resp. fdr_by.
if fdr == 'ON':