def run(args):
# load the data
df = gtm.load_file_and_avg(args.data_file)
genes = df['gene'].values
found_genes, geneTS = gtm.get_gene_TS(df, genes)
dfr = gtm.load_file_and_avg(args.rand_data_file)
genesr = dfr['gene'].values
found_genesr, geneTSr = gtm.get_gene_TS(dfr, genesr)
n = geneTSr.shape[0]
args_dict = ct.load_kwargs_file(argsfile=args.args_file)
print args_dict
if args.rowlist_file != None:
with open(args.rowlist_file, 'rU') as f:
rowlist = eval(f.readline())
else:
rowlist = range(n)
if args.test == "e":
beta_tuple, all_res_df, use_df = ct.enet_granger_causality_row_cv(geneTS, geneTS, rowlist, **args_dict)
with open(args.output_name, 'w') as outfile:
pickle.dump(beta_tuple, outfile)
all_res_df.to_csv(args.output_all_name, sep="\t", index=False)
use_df.to_csv(args.output_use_name, sep="\t", index=False)
param_df = use_df[["alpha", "lambda.min", "Row"]]
rand_beta_tuple, rand_all_res_df, rand_use_df = ct.enet_granger_causality_row_load(geneTSr, geneTS, rowlist, param_df, **args_dict)
with open(args.output_rand_name, 'w') as outfile:
pickle.dump(rand_beta_tuple, outfile)
rand_all_res_df.to_csv(args.output_rand_all_name, sep="\t", index=False)
rand_use_df.to_csv(args.output_rand_use_name, sep="\t", index=False)
print "HIIIIIII"
print "Output written to ", args.output_name
print "All results written to ", args.output_all_name
print "Used params written to ", args.output_use_name
print "Rand output written to ", args.output_rand_name
print "All rand results written to ", args.output_rand_all_name
print "Used rand params written to ", args.output_rand_use_name
def run(args):
# load the data
df = gtm.load_file_and_avg(args.data_file)
genes = df['gene'].values
found_genes, geneTS = gtm.get_gene_TS(df, genes)
dfr = gtm.load_file_and_avg(args.rand_data_file)
genesr = dfr['gene'].values
found_genesr, geneTSr = gtm.get_gene_TS(dfr, genesr)
n = geneTSr.shape[0]
args_dict = ct.load_kwargs_file(argsfile=args.args_file)
print args_dict
if args.rowlist_file != None:
with open(args.rowlist_file, 'rU') as f:
rowlist = eval(f.readline())
else:
rowlist = range(n)
if args.test == "e":
beta_tuple, all_res_df, use_df = ct.enet_granger_causality_row_cv(
geneTS, geneTS, rowlist, **args_dict)
with open(args.output_name, 'w') as outfile:
pickle.dump(beta_tuple, outfile)
all_res_df.to_csv(args.output_all_name, sep="\t", index=False)
use_df.to_csv(args.output_use_name, sep="\t", index=False)
param_df = use_df[["alpha", "lambda.min", "Row"]]
rand_beta_tuple, rand_all_res_df, rand_use_df = ct.enet_granger_causality_row_load(
geneTSr, geneTS, rowlist, param_df, **args_dict)
with open(args.output_rand_name, 'w') as outfile:
pickle.dump(rand_beta_tuple, outfile)
rand_all_res_df.to_csv(args.output_rand_all_name,
sep="\t",
index=False)
rand_use_df.to_csv(args.output_rand_use_name, sep="\t", index=False)
print "HIIIIIII"
print "Output written to ", args.output_name
print "All results written to ", args.output_all_name
print "Used params written to ", args.output_use_name
print "Rand output written to ", args.output_rand_name
print "All rand results written to ", args.output_rand_all_name
print "Used rand params written to ", args.output_rand_use_name
def run(args):
# load the data
df = gtm.load_file_and_avg(args.data_file)
genes = df['gene'].values
found_genes, geneTS = gtm.get_gene_TS(df, genes)
args_dict = load_kwargs_file(argsfile=args.args_file)
if args.pairlist_file == None:
pairlist = None
else:
pairlist = np.load(open(args.pairlist_file))
print args_dict
if args.test == 'g':
output = ct.pairwise_granger_causality_all(geneTS, pairlist, **args_dict)
with open(args.output_name, 'w') as outfile:
pickle.dump(output, outfile)
print "HELLOOOOOOOO"
print "Output written to ", args.output_name
def run(args):
# load the data
df = gtm.load_file_and_avg(args.data_file)
genes = df['gene'].values
found_genes, geneTS = gtm.get_gene_TS(df, genes)
args_dict = load_kwargs_file(argsfile=args.args_file)
if args.pairlist_file == None:
pairlist = None
else:
pairlist = np.load(open(args.pairlist_file))
print args_dict
if args.test == 'g':
output = ct.pairwise_granger_causality_all(geneTS, pairlist,
**args_dict)
with open(args.output_name, 'w') as outfile:
pickle.dump(output, outfile)
print "HELLOOOOOOOO"
print "Output written to ", args.output_name
def main():
tstart = time.time()
input_file = args.input_file
out_file_prefix = args.out_file_prefix
start_index = args.start_index
end_index = args.end_index
df = gtm.load_file_and_avg(input_file)
genes = df['gene'][start_index:end_index].values
found_genes, geneTS = gtm.get_gene_TS(df, genes)
cause_type = args.cause_type
if cause_type == 'g':
model_orders = range(args.model_order_min, args.model_order_max + 1)
threshold = args.p_threshold
p_matr_list = []
sig_matr_list = []
for model_order in model_orders:
t_gc = time.time()
p_matr = pairwise_granger_causality_all(
geneTS,
model_order=model_order,
use_processes=args.use_processes,
procnum=args.procnum)
print "Time for granger causality", time.time() - t_gc
sig_matr = p_matr < threshold
p_matr_list.append(p_matr)
sig_matr_list.append(sig_matr)
all_sig_matr, all_sig_num, not_sig_num = gtm.compare_sig_matr(
sig_matr_list=sig_matr_list)
print "Total number of significant pairs ", all_sig_num + not_sig_num
print "Pairs significant across all matrices ", all_sig_num, all_sig_num * 1.0 / (
all_sig_num + not_sig_num)
out_file_name = out_file_prefix + "_GC.p"
pickle.dump([
model_orders, p_matr_list, sig_matr_list,
(all_sig_matr, all_sig_num, not_sig_num)
], open(out_file_name, "w"))
print "Results written to", out_file_name
# compare the significant matrices
# save the output p matrices
print "Total time used ", time.time() - tstart
def main():
tstart = time.time()
input_file = args.input_file
out_file_prefix = args.out_file_prefix
start_index = args.start_index
end_index = args.end_index
df = gtm.load_file_and_avg(input_file)
genes = df['gene'][start_index:end_index].values
found_genes, geneTS = gtm.get_gene_TS(df, genes)
cause_type = args.cause_type
if cause_type == 'g':
model_orders = range(args.model_order_min, args.model_order_max + 1)
threshold = args.p_threshold
p_matr_list = []
sig_matr_list = []
for model_order in model_orders:
t_gc = time.time()
p_matr = pairwise_granger_causality_all(geneTS, model_order=model_order, use_processes=args.use_processes, procnum=args.procnum)
print "Time for granger causality", time.time() - t_gc
sig_matr = p_matr < threshold
p_matr_list.append(p_matr)
sig_matr_list.append(sig_matr)
all_sig_matr, all_sig_num, not_sig_num = gtm.compare_sig_matr(sig_matr_list=sig_matr_list)
print "Total number of significant pairs ", all_sig_num + not_sig_num
print "Pairs significant across all matrices ", all_sig_num, all_sig_num * 1.0 / (all_sig_num + not_sig_num)
out_file_name = out_file_prefix + "_GC.p"
pickle.dump([model_orders, p_matr_list, sig_matr_list, (all_sig_matr, all_sig_num, not_sig_num)], open(out_file_name, "w"))
print "Results written to", out_file_name
# compare the significant matrices
# save the output p matrices
print "Total time used ", time.time() - tstart
def run(args):
data_file = args.data_file.split('/')[-1]
rand_data_file = args.rand_data_file.split('/')[-1]
df = gtm.load_file_and_avg(data_file)
genes = df['gene'].values
n = len(genes)
script_filenames = []
output_filenames = []
output_rand_filenames = []
if args.test == "e":
all_res_filenames = []
use_filenames = []
all_res_rand_filenames = []
use_rand_filenames = []
else:
all_res_filenames = None
use_filenames = None
all_res_rand_filenames = None
use_rand_filenames = None
partition_rows = pj.partition_inputs(range(n), args.job_num)
for partition_row, i in zip(partition_rows, range(len(partition_rows))):
script_filename = args.output_name + "-script-" + str(i) + ".sh"
script_filenames.append(script_filename)
output_filename = args.output_name + "-" + str(i) + ".p"
output_filenames.append(output_filename)
output_rand_filename = args.output_name + "-randomized-" + str(i) + ".p"
output_rand_filenames.append(output_rand_filename)
# prepare the job associated with this
row_filename = args.output_name + "-row-" + str(i) + ".txt"
command_string = "python run_causal_rand_row.py -d " + data_file + " -rd " + rand_data_file + \
" -a " + args.args_file.split('/')[-1] + " -t " + args.test + " -rl " + \
str(row_filename) + " -o " + output_filename + " -or " + output_rand_filename
if args.test == "e":
all_res_filename = args.output_name + "-all-params-" + str(i) + ".txt"
all_res_filenames.append(all_res_filename)
use_filename = args.output_name + "-used-params-" + str(i) + ".txt"
use_filenames.append(use_filename)
all_res_rand_filename = args.output_name + "-all-params-randomized-" + str(i) + ".txt"
all_res_rand_filenames.append(all_res_rand_filename)
use_rand_filename = args.output_name + "-used-params-randomized-" + str(i) + ".txt"
use_rand_filenames.append(use_rand_filename)
command_string += " -oa " + all_res_filename + " -ou " + use_filename + " -ora " + all_res_rand_filename + " -oru " + use_rand_filename
with open(row_filename, 'w') as rowfile:
rowfile.write(str(partition_row) + "\n")
print "Partition row written to ", row_filename
with open(script_filename, 'w') as outputfile:
outputfile.write("#!/bin/bash\n")
outputfile.write("module load python/2.7\n")
outputfile.write("module load python/2.7/scipy-mkl\n")
outputfile.write("module load python/2.7/numpy-mkl\n")
outputfile.write("module load anaconda\n")
outputfile.write(command_string)
outputfile.write("\n")
os.chmod(script_filename, 0777)
print "Script written to ", script_filename
integrated_name_dict = {}
integrated_name_dict["Output"] = args.output_name + ".p"
integrated_name_dict["Rand-Output"] = args.output_name + "-randomized.p"
integrated_name_dict["All-Params"] = args.output_name + "-all-params.txt"
integrated_name_dict["Use-Params"] = args.output_name + "-use-params.txt"
integrated_name_dict["All-Rand-Params"] = args.output_name + "-all-params-randomized.txt"
integrated_name_dict["Use-Rand-Params"] = args.output_name + "-use-params-randomized.txt"
with open("script_list.txt", 'w') as scriptfile:
for script_filename in script_filenames:
scriptfile.write(script_filename + "\n")
print "Script list written to script_list.txt"
# list of matrices to integrate
output_matr_dict = {"Output": output_filenames, "Rand-Output": output_rand_filenames}
output_matr_df = pd.DataFrame(output_matr_dict)
output_matr_df.to_csv("output_matr_list.txt", sep="\t", index=False)
print "Output matrices written to output_matr_list.txt"
int_matr_dict = dict([(x, integrated_name_dict[x]) for x in ["Output", "Rand-Output"]])
int_matr_df = pd.DataFrame(int_matr_dict, index=[0])
int_matr_df.to_csv("int_matr_list.txt", sep="\t", index=False)
print "integrated matrices written to int_matr_list.txt"
if args.test == "e":
# lists of dataframes (param files) to integrate
# These will only be integrated if
output_df_dict = {}
output_df_lists = [all_res_filenames, use_filenames, all_res_rand_filenames, use_rand_filenames]
output_df_names = ["All-Params", "Use-Params", "All-Rand-Params", "Use-Rand-Params"]
for out_list, out_name in zip(output_df_lists, output_df_names):
if out_list != None:
output_df_dict[out_name] = out_list
output_df_df = pd.DataFrame(output_df_dict)
output_df_df.to_csv("output_df_list.txt", sep="\t", index=False)
print "output dfs written to output_df_list.txt"
int_df_dict = dict([(x, integrated_name_dict[x]) for x in set(output_df_names).intersection(output_df_dict.keys())])
int_df_df = pd.DataFrame(int_df_dict, index=[0])
int_df_df.to_csv("int_df_list.txt", sep="\t", index=False)
print "Integrated dfs written to int_df_list.txt"
with open("integrate_outputs.sh", 'w') as ifile:
if args.test == "e":
# here , "a" means the axis to integrate by
ifile.write("python integrate_outputs_rand_row.py -i output_matr_list.txt -t m -o int_matr_list.txt -a 1 && " + \
"python integrate_outputs_rand_row.py -i output_df_list.txt -t d -o int_df_list.txt\n")
else:
ifile.write("python integrate_outputs_rand_row.py -i output_matr_list.txt -t m -o int_matr_list.txt -a 1\n")
print "Integration script written to integrate_outputs.sh"
os.chmod("integrate_outputs.sh", 0777)
with open("fdr_control.sh", 'w') as ffile:
fdr_string = "python fdr_control.py -m " + integrated_name_dict["Output"] + " -rm " + integrated_name_dict["Rand-Output"] + \
" -d " + data_file + " -rd " + rand_data_file + " -n " + args.output_name + " -f \"" + str(args.fdr) + "\" " + \
" -c " + str(args.coef_num) + " -mn " + str(1) + " -pp " + args.output_name + "-all-beta-histogram "
ffile.write(fdr_string + " -sb e && " + fdr_string + " -sb n\n")
print "FDR CONTROL script written to fdr_control.sh"
os.chmod("fdr_control.sh", 0777)
if args.parallel_num > 0:
print "Parallel Number (# processes per job): " + str(args.parallel_num)
script_groups = pj.partition_inputs(script_filenames, number=int(math.ceil(len(script_filenames) * 1.0/args.parallel_num)))
print "Number of script groups ", len(script_groups)
parallel_scripts = []
for i, script_group in zip(range(len(script_groups)), script_groups):
appended_script_filenames = ["./" + script_filename for script_filename in script_group]
parallel_script = " & ".join(appended_script_filenames)
print "Parallel Script ", i, ":", parallel_script
parallel_scripts.append(parallel_script)
with open("parallel_script_list.txt", 'w') as scriptfile:
for parallel_script in parallel_scripts:
scriptfile.write(parallel_script + "\n")
print "Parallel script list written to parallel_script_list.txt"
def run(args):
data_file = args.data_file.split('/')[-1]
rand_data_file = args.rand_data_file.split('/')[-1]
df = gtm.load_file_and_avg(data_file)
genes = df['gene'].values
n = len(genes)
script_filenames = []
output_filenames = []
output_rand_filenames = []
if args.test == "e":
all_res_filenames = []
use_filenames = []
all_res_rand_filenames = []
use_rand_filenames = []
else:
all_res_filenames = None
use_filenames = None
all_res_rand_filenames = None
use_rand_filenames = None
partition_rows = pj.partition_inputs(range(n), args.job_num)
for partition_row, i in zip(partition_rows, range(len(partition_rows))):
script_filename = args.output_name + "-script-" + str(i) + ".sh"
script_filenames.append(script_filename)
output_filename = args.output_name + "-" + str(i) + ".p"
output_filenames.append(output_filename)
output_rand_filename = args.output_name + "-randomized-" + str(
i) + ".p"
output_rand_filenames.append(output_rand_filename)
# prepare the job associated with this
row_filename = args.output_name + "-row-" + str(i) + ".txt"
command_string = "python run_causal_rand_row.py -d " + data_file + " -rd " + rand_data_file + \
" -a " + args.args_file.split('/')[-1] + " -t " + args.test + " -rl " + \
str(row_filename) + " -o " + output_filename + " -or " + output_rand_filename
if args.test == "e":
all_res_filename = args.output_name + "-all-params-" + str(
i) + ".txt"
all_res_filenames.append(all_res_filename)
use_filename = args.output_name + "-used-params-" + str(i) + ".txt"
use_filenames.append(use_filename)
all_res_rand_filename = args.output_name + "-all-params-randomized-" + str(
i) + ".txt"
all_res_rand_filenames.append(all_res_rand_filename)
use_rand_filename = args.output_name + "-used-params-randomized-" + str(
i) + ".txt"
use_rand_filenames.append(use_rand_filename)
command_string += " -oa " + all_res_filename + " -ou " + use_filename + " -ora " + all_res_rand_filename + " -oru " + use_rand_filename
with open(row_filename, 'w') as rowfile:
rowfile.write(str(partition_row) + "\n")
print "Partition row written to ", row_filename
with open(script_filename, 'w') as outputfile:
outputfile.write("#!/bin/bash\n")
outputfile.write("module load python/2.7\n")
outputfile.write("module load python/2.7/scipy-mkl\n")
outputfile.write("module load python/2.7/numpy-mkl\n")
outputfile.write("module load anaconda\n")
outputfile.write(command_string)
outputfile.write("\n")
os.chmod(script_filename, 0777)
print "Script written to ", script_filename
integrated_name_dict = {}
integrated_name_dict["Output"] = args.output_name + ".p"
integrated_name_dict["Rand-Output"] = args.output_name + "-randomized.p"
integrated_name_dict["All-Params"] = args.output_name + "-all-params.txt"
integrated_name_dict["Use-Params"] = args.output_name + "-use-params.txt"
integrated_name_dict[
"All-Rand-Params"] = args.output_name + "-all-params-randomized.txt"
integrated_name_dict[
"Use-Rand-Params"] = args.output_name + "-use-params-randomized.txt"
with open("script_list.txt", 'w') as scriptfile:
for script_filename in script_filenames:
scriptfile.write(script_filename + "\n")
print "Script list written to script_list.txt"
# list of matrices to integrate
output_matr_dict = {
"Output": output_filenames,
"Rand-Output": output_rand_filenames
}
output_matr_df = pd.DataFrame(output_matr_dict)
output_matr_df.to_csv("output_matr_list.txt", sep="\t", index=False)
print "Output matrices written to output_matr_list.txt"
int_matr_dict = dict([(x, integrated_name_dict[x])
for x in ["Output", "Rand-Output"]])
int_matr_df = pd.DataFrame(int_matr_dict, index=[0])
int_matr_df.to_csv("int_matr_list.txt", sep="\t", index=False)
print "integrated matrices written to int_matr_list.txt"
if args.test == "e":
# lists of dataframes (param files) to integrate
# These will only be integrated if
output_df_dict = {}
output_df_lists = [
all_res_filenames, use_filenames, all_res_rand_filenames,
use_rand_filenames
]
output_df_names = [
"All-Params", "Use-Params", "All-Rand-Params", "Use-Rand-Params"
]
for out_list, out_name in zip(output_df_lists, output_df_names):
if out_list != None:
output_df_dict[out_name] = out_list
output_df_df = pd.DataFrame(output_df_dict)
output_df_df.to_csv("output_df_list.txt", sep="\t", index=False)
print "output dfs written to output_df_list.txt"
int_df_dict = dict([
(x, integrated_name_dict[x])
for x in set(output_df_names).intersection(output_df_dict.keys())
])
int_df_df = pd.DataFrame(int_df_dict, index=[0])
int_df_df.to_csv("int_df_list.txt", sep="\t", index=False)
print "Integrated dfs written to int_df_list.txt"
with open("integrate_outputs.sh", 'w') as ifile:
if args.test == "e":
# here , "a" means the axis to integrate by
ifile.write("python integrate_outputs_rand_row.py -i output_matr_list.txt -t m -o int_matr_list.txt -a 1 && " + \
"python integrate_outputs_rand_row.py -i output_df_list.txt -t d -o int_df_list.txt\n")
else:
ifile.write(
"python integrate_outputs_rand_row.py -i output_matr_list.txt -t m -o int_matr_list.txt -a 1\n"
)
print "Integration script written to integrate_outputs.sh"
os.chmod("integrate_outputs.sh", 0777)
with open("fdr_control.sh", 'w') as ffile:
fdr_string = "python fdr_control.py -m " + integrated_name_dict["Output"] + " -rm " + integrated_name_dict["Rand-Output"] + \
" -d " + data_file + " -rd " + rand_data_file + " -n " + args.output_name + " -f \"" + str(args.fdr) + "\" " + \
" -c " + str(args.coef_num) + " -mn " + str(1) + " -pp " + args.output_name + "-all-beta-histogram "
ffile.write(fdr_string + " -sb e && " + fdr_string + " -sb n\n")
print "FDR CONTROL script written to fdr_control.sh"
os.chmod("fdr_control.sh", 0777)
if args.parallel_num > 0:
print "Parallel Number (# processes per job): " + str(
args.parallel_num)
script_groups = pj.partition_inputs(
script_filenames,
number=int(
math.ceil(len(script_filenames) * 1.0 / args.parallel_num)))
print "Number of script groups ", len(script_groups)
parallel_scripts = []
for i, script_group in zip(range(len(script_groups)), script_groups):
appended_script_filenames = [
"./" + script_filename for script_filename in script_group
]
parallel_script = " & ".join(appended_script_filenames)
print "Parallel Script ", i, ":", parallel_script
parallel_scripts.append(parallel_script)
with open("parallel_script_list.txt", 'w') as scriptfile:
for parallel_script in parallel_scripts:
scriptfile.write(parallel_script + "\n")
print "Parallel script list written to parallel_script_list.txt"
def run(args):
df = gtm.load_file_and_avg(args.data_file)
genes = df['gene'].values
n = len(genes)
script_filenames = []
output_filenames = []
partition_pairs = lazy_partition_pairs(n, n, args.job_num)
for partition_pair, i in zip(partition_pairs, range(len(partition_pairs))):
script_filename = args.output_name + "-script-" + str(i) + ".sh"
script_filenames.append(script_filename)
output_filename = args.output_name + "-" + str(i) + ".p"
output_filenames.append(output_filename)
# prepare the job associated with this
pair_filename = args.output_name + "-pair-" + str(i) + ".txt"
command_string = "python run_causal.py -d " + args.data_file.split('/')[-1] + " -a " + args.args_file.split('/')[-1] + " -t " + args.test + " -pp " + \
str(pair_filename) + " -o " + output_filename
if args.test == "gp":
command_string += " -d2 " + args.data_file2.split('/')[-1]
with open(pair_filename, 'w') as pairfile:
pairfile.write(str(partition_pair) + "\n")
print "Partition pair written to ", pair_filename
with open(script_filename, 'w') as outputfile:
outputfile.write("#!/bin/bash\n")
outputfile.write("module load python/2.7\n")
outputfile.write("module load python/2.7/scipy-mkl\n")
outputfile.write("module load python/2.7/numpy-mkl\n")
outputfile.write("module load anaconda\n")
outputfile.write(command_string)
outputfile.write("\n")
os.chmod(script_filename, 0777)
print "Script written to ", script_filename
# submit the jobs soon
with open("script_list.txt", 'w') as scriptfile:
for script_filename in script_filenames:
scriptfile.write(script_filename + "\n")
print "Script list written to script_list.txt"
with open("output_list.txt", 'w') as outputfile:
for output_filename in output_filenames:
outputfile.write(output_filename + "\n")
print "Output list written to output_list.txt"
with open("integrate_outputs.sh", 'w') as ifile:
integrated_filename = args.output_name + ".p"
ifile.write("python integrate_outputs.py -i output_list.txt -o " + integrated_filename + " -n " + str(n) + "\n")
print "Integration script written to integrate_outputs.sh"
os.chmod("integrate_outputs.sh", 0777)
if args.parallel_num > 0:
print "Parallel Number (# processes per job): " + str(args.parallel_num)
script_groups = partition_inputs(script_filenames, number=len(script_filenames)/args.parallel_num)
print "Number of script groups ", len(script_groups)
parallel_scripts = []
for i, script_group in zip(range(len(script_groups)), script_groups):
appended_script_filenames = ["./" + script_filename for script_filename in script_group]
parallel_script = " & ".join(appended_script_filenames)
print "Parallel Script ", i, ":", parallel_script
parallel_scripts.append(parallel_script)
with open("parallel_script_list.txt", 'w') as scriptfile:
for parallel_script in parallel_scripts:
scriptfile.write(parallel_script + "\n")
print "Parallel script list written to parallel_script_list.txt"
def run(args):
data = gtm.load_file_and_avg(args.original_data)
rand_data = gtm.load_file_and_avg(args.randomized_data)
matr = pickle.load(open(args.original_matrix, 'rB'))[:, :,
args.coef_num - 1]
rand_matr = pickle.load(open(args.randomized_matrix,
'rB'))[:, :, args.coef_num - 1]
if args.stratify_by not in {"e", "n"}:
raise ValueError(
"Stratify_by must be either 'e' for effect or 'n' for none")
else:
if args.stratify_by == "e":
stratify_by = "effect"
elif args.stratify_by == "n":
stratify_by = "none"
print
print "Beginning FDR control, stratifying the matrix by ", stratify_by
genes = data["gene"]
rand_genes = rand_data["gene"]
if (genes != rand_genes).any():
raise ValueError("Genes are not the same!")
print "Original matrix for ", args.name, "saved to", args.name + "-unshuffled-matrix.txt"
gtm.save_gene_matrix(matrix=matr,
filename=args.name + "-unshuffled-matrix.txt",
genes=genes)
print "Randomized matrix for ", args.name, "saved to", args.name + "-shuffled-matrix.txt"
gtm.save_gene_matrix(matrix=rand_matr,
filename=args.name + "-shuffled-matrix.txt",
genes=rand_genes)
if args.plot_prefix != None:
plot_betas(matr.flatten(),
rand_matr.flatten(),
filename=args.plot_prefix)
plot_betas(matr.flatten(),
rand_matr.flatten(),
filename=args.plot_prefix + "_zoom-in-95",
zoom_in_percentile=95)
if args.cap_by != None:
print "First capping original and randomized matrix"
matr = cap_matr(matr, args.cap_by, name="Original")
rand_matr = cap_matr(rand_matr, args.cap_by, name="Randomized")
print "Using original"
print "Trying to have an FDR of ", args.fdr
print args.name
functions = [get_abs_thresh, get_pos_neg_thresh]
types = ["abs-thresh", "pos-neg-thresh"]
# whether to take absolute value of given matrices
absoluted = [True, True]
for function, t, a in zip(functions, types, absoluted):
print
print "*******************"
print t
print "*******************"
print "making matrix"
out_prefix = args.name + "-unshuffled-" + t + "-FDR-" + str(
args.fdr) + "-stratby-" + stratify_by
thresh_matr, threshes = function(matr,
rand_matr,
args.fdr,
stratify_by=stratify_by)
matr_df = gtm.save_gene_matrix(out_prefix + "-matrix.txt", thresh_matr,
genes)
pickle.dump(threshes, open(out_prefix + "-threshes.p", 'w'))
print "Matrix written to ", out_prefix + "-matrix.txt"
print "Threshes written to ", out_prefix + "-threshes.p"
#write_readme(thresh_matr, out_prefix, args.fdr, out_prefix + '-README.txt', out_prefix + "-matrix")
if args.make_network:
print "making network"
net_df = nh.matr_to_net(matr_df,
args.name + "-sb-" + args.stratify_by,
make_pair=False)
net_df.to_csv(out_prefix + "-network.txt", sep="\t", index=False)
print "Network written to ", out_prefix + "-network.txt"
if absoluted:
print "Making absoluted matrix "
abs_matr = np.absolute(thresh_matr)
abs_prefix = args.name + "-unshuffled-" + t + "-absoluted-FDR-" + str(
args.fdr) + "-stratby-" + stratify_by
abs_df = gtm.save_gene_matrix(abs_prefix + "-matrix", abs_matr,
genes)
#write_readme(abs_matr, abs_prefix, args.fdr, abs_prefix + '-README.txt', abs_prefix + "-matrix")
if args.make_network:
print "Making absoluted network"
abs_net_df = nh.matr_to_net(abs_df,
args.name + "-sb-" +
args.stratify_by,
make_pair=False)
abs_net_df.to_csv(abs_prefix + "-network.txt",
sep="\t",
index=False)
print "Network written to ", abs_prefix + "-network.txt"
print "FINISHED"
print "#################################################"
print
def run(args):
data = gtm.load_file_and_avg(args.original_data)
rand_data = gtm.load_file_and_avg(args.randomized_data)
matr = pickle.load(open(args.original_matrix, 'rB'))[:, :, args.coef_num - 1]
rand_matr = pickle.load(open(args.randomized_matrix, 'rB'))[:, :, args.coef_num - 1]
if args.stratify_by not in {"e", "n"}:
raise ValueError("Stratify_by must be either 'e' for effect or 'n' for none")
else:
if args.stratify_by == "e":
stratify_by = "effect"
elif args.stratify_by == "n":
stratify_by = "none"
genes = data["gene"]
rand_genes = rand_data["gene"]
if (genes != rand_genes).any():
raise ValueError("Genes are not the same!")
print "Original matrix for ", args.name, "saved to", args.name + "-unshuffled-matrix.txt"
gtm.save_gene_matrix(matrix=matr, filename=args.name + "-unshuffled-matrix.txt", genes=genes)
print "Randomized matrix for ", args.name, "saved to", args.name + "-shuffled-matrix.txt"
gtm.save_gene_matrix(matrix=rand_matr, filename=args.name + "-shuffled-matrix.txt", genes=rand_genes)
if args.plot_prefix != None:
plot_betas(matr.flatten(), rand_matr.flatten(), filename=args.plot_prefix)
plot_betas(matr.flatten(), rand_matr.flatten(), filename=args.plot_prefix + "_zoom-in-95", zoom_in_percentile=95)
print "Using original"
print "Trying to have an FDR of ", args.fdr
print args.name
functions = [get_abs_thresh, get_pos_thresh, get_neg_thresh, get_pos_neg_thresh]
types = ["abs-thresh", "pos-thresh", "neg-thresh", "pos-neg-thresh"]
# whether to take absolute value of given matrices
absoluted = [True, False, False, True]
for function, t, a in zip(functions, types, absoluted):
out_prefix = args.name + "-unshuffled-" + t + "-FDR-" + str(args.fdr) + "-stratby-" + stratify_by
thresh_matr, threshes = function(matr, rand_matr, args.fdr, stratify_by = stratify_by)
matr_df = gtm.save_gene_matrix(out_prefix + "-matrix.txt", thresh_matr, genes)
pickle.dump(threshes, open(out_prefix + "-threshes.p", 'w'))
print "Matrix written to ", out_prefix + "-matrix.txt"
print "Threshes written to ", out_prefix + "-threshes.p"
write_readme(thresh_matr, out_prefix, args.fdr, out_prefix + '-README.txt', out_prefix + "-matrix")
if args.make_network:
net_df = nh.matr_to_net(matr_df, args.name, make_pair=False)
net_df.to_csv(out_prefix + "-network.txt", sep="\t", index=False)
print "Network written to ", out_prefix + "-network.txt"
if absoluted:
abs_matr = np.absolute(thresh_matr)
abs_prefix = args.name + "-unshuffled-" + t + "-absoluted-FDR-" + str(args.fdr) + "-stratby-" + stratify_by
abs_df = gtm.save_gene_matrix(abs_prefix + "-matrix", abs_matr, genes)
write_readme(abs_matr, abs_prefix, args.fdr, abs_prefix + '-README.txt', abs_prefix + "-matrix")
if args.make_network:
abs_net_df = nh.matr_to_net(abs_df, args.name, make_pair=False)
abs_net_df.to_csv(abs_prefix + "-network.txt", sep="\t", index=False)
print "Network written to ", abs_prefix + "-network.txt"
def run(args):
df = gtm.load_file_and_avg(args.data_file)
genes = df['gene'].values
n = len(genes)
script_filenames = []
output_filenames = []
partition_pairs = lazy_partition_pairs(n, n, args.job_num)
for partition_pair, i in zip(partition_pairs, range(len(partition_pairs))):
script_filename = args.output_name + "-script-" + str(i) + ".sh"
script_filenames.append(script_filename)
output_filename = args.output_name + "-" + str(i) + ".p"
output_filenames.append(output_filename)
# prepare the job associated with this
pair_filename = args.output_name + "-pair-" + str(i) + ".txt"
command_string = "python run_causal.py -d " + args.data_file.split('/')[-1] + " -a " + args.args_file.split('/')[-1] + " -t " + args.test + " -pp " + \
str(pair_filename) + " -o " + output_filename
if args.test == "gp":
command_string += " -d2 " + args.data_file2.split('/')[-1]
with open(pair_filename, 'w') as pairfile:
pairfile.write(str(partition_pair) + "\n")
print "Partition pair written to ", pair_filename
with open(script_filename, 'w') as outputfile:
outputfile.write("#!/bin/bash\n")
outputfile.write("module load python/2.7\n")
outputfile.write("module load python/2.7/scipy-mkl\n")
outputfile.write("module load python/2.7/numpy-mkl\n")
outputfile.write("module load anaconda\n")
outputfile.write(command_string)
outputfile.write("\n")
os.chmod(script_filename, 0777)
print "Script written to ", script_filename
# submit the jobs soon
with open("script_list.txt", 'w') as scriptfile:
for script_filename in script_filenames:
scriptfile.write(script_filename + "\n")
print "Script list written to script_list.txt"
with open("output_list.txt", 'w') as outputfile:
for output_filename in output_filenames:
outputfile.write(output_filename + "\n")
print "Output list written to output_list.txt"
with open("integrate_outputs.sh", 'w') as ifile:
integrated_filename = args.output_name + ".p"
ifile.write("python integrate_outputs.py -i output_list.txt -o " +
integrated_filename + " -n " + str(n) + "\n")
print "Integration script written to integrate_outputs.sh"
os.chmod("integrate_outputs.sh", 0777)
if args.parallel_num > 0:
print "Parallel Number (# processes per job): " + str(
args.parallel_num)
script_groups = partition_inputs(script_filenames,
number=len(script_filenames) /
args.parallel_num)
print "Number of script groups ", len(script_groups)
parallel_scripts = []
for i, script_group in zip(range(len(script_groups)), script_groups):
appended_script_filenames = [
"./" + script_filename for script_filename in script_group
]
parallel_script = " & ".join(appended_script_filenames)
print "Parallel Script ", i, ":", parallel_script
parallel_scripts.append(parallel_script)
with open("parallel_script_list.txt", 'w') as scriptfile:
for parallel_script in parallel_scripts:
scriptfile.write(parallel_script + "\n")
print "Parallel script list written to parallel_script_list.txt"