def run_correlation(run_parameters):
""" perform feature prioritization
Args:
run_parameters: parameter set dictionary.
"""
max_cpu = run_parameters["max_cpu"]
run_parameters["results_tmp_directory"] = kn.create_dir(
run_parameters["results_directory"], 'tmp')
phenotype_df = kn.get_spreadsheet_df(
run_parameters["phenotype_name_full_path"])
spreadsheet_df = kn.get_spreadsheet_df(
run_parameters["spreadsheet_name_full_path"])
phenotype_df = phenotype_df.T
len_phenotype = len(phenotype_df.index)
array_of_jobs = range(0, len_phenotype)
for i in range(0, len_phenotype, max_cpu):
jobs_id = array_of_jobs[i:i + max_cpu]
number_of_jobs = len(jobs_id)
zipped_arguments = dstutil.zip_parameters(run_parameters,
spreadsheet_df, phenotype_df,
jobs_id)
dstutil.parallelize_processes_locally(run_correlation_worker,
zipped_arguments, number_of_jobs)
write_phenotype_data_all(run_parameters)
kn.remove_dir(run_parameters["results_tmp_directory"])
def run_bootstrap_net_correlation(run_parameters):
""" perform gene prioritization using bootstrap sampling and network smoothing
Args:
run_parameters: parameter set dictionary.
"""
run_parameters["results_tmp_directory"] = kn.create_dir(run_parameters["results_directory"], 'tmp')
gg_network_name_full_path = run_parameters['gg_network_name_full_path']
network_mat, unique_gene_names = kn.get_sparse_network_matrix(gg_network_name_full_path)
network_mat = normalize(network_mat, norm="l1", axis=0)
phenotype_df = kn.get_spreadsheet_df(run_parameters["phenotype_name_full_path"])
spreadsheet_df = kn.get_spreadsheet_df(run_parameters["spreadsheet_name_full_path"])
spreadsheet_genes_as_input = spreadsheet_df.index.values
phenotype_df = phenotype_df.T
spreadsheet_df = kn.update_spreadsheet_df(spreadsheet_df, unique_gene_names)
spreadsheet_df = zscore_dataframe(spreadsheet_df)
sample_smooth, iterations = kn.smooth_matrix_with_rwr(spreadsheet_df.as_matrix(), network_mat.T, run_parameters)
spreadsheet_df = pd.DataFrame(sample_smooth, index=spreadsheet_df.index, columns=spreadsheet_df.columns)
baseline_array = np.ones(network_mat.shape[0]) / network_mat.shape[0]
baseline_array = kn.smooth_matrix_with_rwr(baseline_array, network_mat, run_parameters)[0]
number_of_jobs = len(phenotype_df.index)
jobs_id = range(0, number_of_jobs)
zipped_arguments = dstutil.zip_parameters(run_parameters, spreadsheet_df, phenotype_df, network_mat,
spreadsheet_genes_as_input, baseline_array, jobs_id)
dstutil.parallelize_processes_locally(run_bootstrap_net_correlation_worker, zipped_arguments, number_of_jobs)
write_phenotype_data_all(run_parameters)
kn.remove_dir(run_parameters["results_tmp_directory"])
def run_correlation(run_parameters):
""" perform gene prioritization
Args:
run_parameters: parameter set dictionary.
"""
run_parameters["results_tmp_directory"] = kn.create_dir(run_parameters["results_directory"], 'tmp')
results_tmp_directory = run_parameters["results_tmp_directory" ]
phenotype_name_full_path = run_parameters["phenotype_name_full_path" ]
spreadsheet_name_full_path = run_parameters["spreadsheet_name_full_path"]
spreadsheet_df = kn.get_spreadsheet_df(spreadsheet_name_full_path)
phenotype_df = kn.get_spreadsheet_df(phenotype_name_full_path )
phenotype_df = phenotype_df.T
number_of_jobs = len(phenotype_df.index)
jobs_id = range(0, number_of_jobs)
zipped_arguments = dstutil.zip_parameters( run_parameters
, spreadsheet_df
, phenotype_df
, jobs_id
)
dstutil.parallelize_processes_locally( run_correlation_worker
, zipped_arguments
, number_of_jobs
)
write_phenotype_data_all(run_parameters )
kn.remove_dir (results_tmp_directory)
def run_bootstrap_correlation(run_parameters):
""" perform gene prioritization using bootstrap sampling
Args:
run_parameters: parameter set dictionary.
"""
max_cpu = run_parameters["max_cpu"]
run_parameters["results_tmp_directory"] = kn.create_dir(
run_parameters["results_directory"], 'tmp')
results_tmp_directory = run_parameters["results_tmp_directory"]
n_bootstraps = run_parameters["number_of_bootstraps"]
results_tmp_directory = run_parameters["results_tmp_directory"]
phenotype_df = kn.get_spreadsheet_df(
run_parameters["phenotype_name_full_path"])
spreadsheet_df = kn.get_spreadsheet_df(
run_parameters["spreadsheet_name_full_path"])
phenotype_df = phenotype_df.T
#-----------------------------------------------------------------------------------------
# Partition the phenotype dataframe (partition size = MaxCPU)
#-----------------------------------------------------------------------------------------
len_phenotype = len(phenotype_df.index)
array_of_jobs = range(0, len_phenotype)
if (len_phenotype <= max_cpu):
jobs_id = array_of_jobs
number_of_jobs = len(jobs_id)
#-----------------------------------------------------------------------------------------
zipped_arguments = dstutil.zip_parameters(run_parameters,
spreadsheet_df, phenotype_df,
n_bootstraps, jobs_id)
dstutil.parallelize_processes_locally(run_bootstrap_correlation_worker,
zipped_arguments, number_of_jobs)
write_phenotype_data_all(run_parameters)
#-----------------------------------------------------------------------------------------
else:
for i in range(0, len_phenotype, max_cpu):
jobs_id = array_of_jobs[i:i + max_cpu]
number_of_jobs = len(jobs_id)
#-----------------------------------------------------------------------------------------
zipped_arguments = dstutil.zip_parameters(run_parameters,
spreadsheet_df, phenotype_df,
n_bootstraps, jobs_id)
dstutil.parallelize_processes_locally(run_bootstrap_correlation_worker,
zipped_arguments, number_of_jobs)
write_phenotype_data_all(run_parameters)
#-----------------------------------------------------------------------------------------
kn.remove_dir(results_tmp_directory)
def update_tmp_directory(run_parameters, tmp_dir):
""" Update tmp_directory value in rum_parameters dictionary
Args:
run_parameters: run_parameters as the dictionary config
tmp_dir: temporary directory prefix subjected to different functions
Returns:
run_parameters: an updated run_parameters
"""
if (run_parameters['processing_method'] == 'distribute'):
run_parameters["tmp_directory"] = kn.create_dir(
run_parameters['cluster_shared_volumn'], tmp_dir)
else:
run_parameters["tmp_directory"] = kn.create_dir(
run_parameters["run_directory"], tmp_dir)
return run_parameters
def run_bootstrap_correlation(run_parameters):
""" perform feature prioritization using bootstrap sampling
Args:
run_parameters: parameter set dictionary.
"""
run_parameters["results_tmp_directory"] = kn.create_dir(run_parameters["results_directory"], 'tmp')
phenotype_df = kn.get_spreadsheet_df(run_parameters["phenotype_name_full_path"])
spreadsheet_df = kn.get_spreadsheet_df(run_parameters["spreadsheet_name_full_path"])
phenotype_df = phenotype_df.T
n_bootstraps = run_parameters["number_of_bootstraps"]
number_of_jobs = len(phenotype_df.index)
jobs_id = range(0, number_of_jobs)
zipped_arguments = dstutil.zip_parameters(run_parameters, spreadsheet_df, phenotype_df, n_bootstraps, jobs_id)
dstutil.parallelize_processes_locally(run_bootstrap_correlation_worker, zipped_arguments, number_of_jobs)
write_phenotype_data_all(run_parameters)
kn.remove_dir(run_parameters["results_tmp_directory"])
def test_create_dir_AND_remove_dir(self):
""" assert that the functions work togeather to create and remove a directory
even when files have been added
"""
dir_name = 'tmp_test'
dir_path = self.run_parameters['test_directory']
new_directory_name = kn.create_dir(dir_path, dir_name)
self.assertTrue(os.path.exists(new_directory_name), msg='create_dir function exception')
A = np.random.rand(10, 10)
time_stamp = '123456789'
a_name = os.path.join(new_directory_name, 'temp_test' + time_stamp)
with open(a_name, 'wb') as fh:
A.dump(fh)
A_back = np.load(a_name)
if os.path.isfile(a_name):
os.remove(a_name)
A_diff = A - A_back
A_diff = A_diff.sum()
self.assertEqual(A_diff, 0, msg='write / read directory exception')
kn.remove_dir(new_directory_name)
self.assertFalse(os.path.exists(new_directory_name), msg='remove_dir function exception')
def run_net_correlation(run_parameters):
""" perform gene prioritization with network smoothing
Args:
run_parameters: parameter set dictionary.
"""
max_cpu = run_parameters["max_cpu"]
run_parameters["results_tmp_directory"] = kn.create_dir(
run_parameters["results_directory"], 'tmp')
gg_network_name_full_path = run_parameters['gg_network_name_full_path']
network_mat, unique_gene_names = kn.get_sparse_network_matrix(
gg_network_name_full_path)
network_mat = normalize(network_mat, norm="l1", axis=0)
phenotype_df = kn.get_spreadsheet_df(
run_parameters["phenotype_name_full_path"])
spreadsheet_df = kn.get_spreadsheet_df(
run_parameters["spreadsheet_name_full_path"])
spreadsheet_genes_as_input = spreadsheet_df.index.values
phenotype_df = phenotype_df.T
spreadsheet_df = kn.update_spreadsheet_df(spreadsheet_df,
unique_gene_names)
spreadsheet_df = zscore_dataframe(spreadsheet_df)
sample_smooth, iterations = kn.smooth_matrix_with_rwr(
spreadsheet_df.values, network_mat.T, run_parameters)
spreadsheet_df = pd.DataFrame(sample_smooth,
index=spreadsheet_df.index,
columns=spreadsheet_df.columns)
baseline_array = np.ones(network_mat.shape[0]) / network_mat.shape[0]
baseline_array = kn.smooth_matrix_with_rwr(baseline_array, network_mat,
run_parameters)[0]
#-----------------------------------------------------------------------------------------
# Partition the phenotype dataframe (partition size = MaxCPU)
#-----------------------------------------------------------------------------------------
len_phenotype = len(phenotype_df.index)
array_of_jobs = range(0, len_phenotype)
if (len_phenotype <= max_cpu):
jobs_id = array_of_jobs
number_of_jobs = len(jobs_id)
zipped_arguments = dstutil.zip_parameters(run_parameters,
spreadsheet_df, phenotype_df,
network_mat,
spreadsheet_genes_as_input,
baseline_array, jobs_id)
dstutil.parallelize_processes_locally(run_net_correlation_worker,
zipped_arguments, number_of_jobs)
write_phenotype_data_all(run_parameters)
#-----------------------------------------------------------------------------------------
else:
for i in range(0, len_phenotype, max_cpu):
jobs_id = array_of_jobs[i:i + max_cpu]
number_of_jobs = len(jobs_id)
#-----------------------------------------------------------------------------------------
zipped_arguments = dstutil.zip_parameters(run_parameters,
spreadsheet_df, phenotype_df,
network_mat,
spreadsheet_genes_as_input,
baseline_array, jobs_id)
dstutil.parallelize_processes_locally(run_net_correlation_worker,
zipped_arguments, number_of_jobs)
write_phenotype_data_all(run_parameters)
#-----------------------------------------------------------------------------------------
kn.remove_dir(run_parameters["results_tmp_directory"])
def setUp(self):
self.run_parameters = tstdata.get_test_paramters_dictionary()
self.run_parameters["run_directory"] = '.'
self.run_parameters["tmp_directory"] = kn.create_dir(
self.run_parameters["run_directory"], 'tmp_cc_net_nmf')
self.run_parameters["use_now_name"] = 0
