selection_complement = list(set(range(mat.shape[0])) - set(selection))

mat1 = mat[selection]

mat2 = mat[selection_complement]

selected_otu_filter = [ otu for index, otu in enumerate(otus) if index in picked_indices ]

complement_otu_filter = [ otu for index, otu in enumerate(otus) if index not in picked_indices ]

if prev_otu_filter is not None:

complement_otu_filter = [ otu for otu in complement_otu_filter if otu in prev_otu_filter ]

selected_samp_filter = [ samp for index, samp in enumerate(samples) if index in picked_indices ]

complement_samp_filter = [ samp for index, samp in enumerate(samples) if index not in picked_indices ]

if prev_samp_filter is not None:

complement_samp_filter = [ samp for samp in complement_samp_filter if samp in prev_samp_filter ]

if globals()['__full_otus_dist'] is None:

full_otus_list, _ = __prepare_otu_filters_from_indices(range(len(otus)), otus)

globals()['__full_otus_dist'] = ctwc__metadata_analysis.calculate_otus_distribution(full_otus_list, range(len(otus)), table)

if globals()['__full_samples_dist'] is None:

full_samples_list, _ = __prepare_sample_filters_from_indices(range(len(samples)), samples)

globals()['__full_samples_dist'] = ctwc__metadata_analysis.calculate_samples_distribution(full_samples_list)

mn = 1.0

tpl = (None, None)

for k_1 in p_vals:

if k_1 in ctwc__metadata_analysis.OTU_RANKS_TO_SKIP:

continue

for k_2 in p_vals[k_1]:

if p_vals[k_1][k_2] < mn:

mn = p_vals[k_1][k_2]

tpl = (k_1, k_2)

filtered_p_vals = {}

for k_1 in p_vals:

for k_2 in p_vals[k_1]:

p = p_vals[k_1][k_2]

if p < t:

if filtered_p_vals.has_key(k_1):

filtered_p_vals[k_1][k_2] = p

else:

filtered_p_vals = {k_2: p}

INFO("{0}: {1}".format(title, desc))

INFO("Input size: {0} {1}".format(len(otus) if otu_filter is None else len(otu_filter),

len(samples) if sample_filter is None else len(sample_filter)))

if is_rows:

return __run_iteration__rows(title, desc, data, tree, samples, otus, otu_filter, sample_filter, table, prev)

else:

data = np.copy(data_in)

if otu_filter is not None:

if otus is not list:

otus = otus.tolist()

rows_filter = [ otus.index(otu) for otu in otu_filter ]

mask = np.ones(data.shape, dtype=bool)

mask[rows_filter] = False

data[mask] = 0.0

if sample_filter is not None:

if samples is not list:

samples = samples.tolist()

cols_filter = [ samples.index(samp) for samp in sample_filter ]

mask = np.ones(data.shape, dtype=bool)

mask[ :, cols_filter ] = False

data[mask] = 0

mask = np.ones(data.shape, dtype=bool)

mask[picked_indices] = False

data[mask] = 0

samp_indices = numpy.where(data.any(axis=1))[0]

data = np.copy(data_in)

if otu_filter is not None:

if otus is not list:

otus = otus.tolist()

rows_filter = [ otus.index(otu) for otu in otu_filter ]

mask = np.ones(data.shape, dtype=bool)

mask[rows_filter] = False

data[mask] = 0.0

if sample_filter is not None:

if samples is not list:

samples = samples.tolist()

cols_filter = [ samples.index(samp) for samp in sample_filter ]

mask = np.ones(data.shape, dtype=bool)

mask[ :, cols_filter ] = False

data[mask] = 0

mask = np.ones(data.shape, dtype=bool)

mask[ :, picked_indices ] = False

data[mask] = 0

otu_indices = numpy.where(data.any(axis=0))[0]

output = {}

for field in sel_dist:

for val in sel_dist[field][1]:

sel_val = sel_dist[field][1][val]

ref_val = ref_dist[field][1][val]

ratio = sel_val / ref_val

if ratio > t:

output[val] = sel_val, ref_val, ratio

rows_dist, _ = ctwc__distance_matrix.get_distance_matrices(data,

tree,

samples,

otus,

sample_filter=sample_filter,

otu_filter=otu_filter,

skip_cols=True)

picked_indices, last_rank, _, _, _, _ = ctwc__cluster_rank.filter_rows_by_top_rank(data,

rows_dist,

prev,

otus)

selected_otu_filter, complement_otu_filter = __prepare_otu_filters_from_indices(picked_indices, otus, otu_filter)

res_file = create_or_open_results_file(title)

add_line_to_results_file(res_file, "{0} - {1} OTUs X {2} Samples".format(title,

len(otus) if otu_filter is None else len(otu_filter),

len(samples) if sample_filter is None else len(sample_filter)))

add_line_to_results_file(res_file, get_iteration_path_string(title_to_iteration(title)))

add_line_to_results_file(res_file, "-"*BANNER_LEN)

sorted_rows_mat = __sort_matrix_rows_by_selection(rows_dist, picked_indices)

sorted_mat = __sort_matrix_cols_by_selection(sorted_rows_mat, picked_indices)

ctwc__plot.plot_mat(sorted_mat, header="{0}: {1}".format(title, "OTUs Distance Matrix - sorted"))

if table is not None:

taxonomies = ctwc__metadata_analysis.get_taxa_by_otu_indices(picked_indices, table)

picked_otus = ctwc__data_handler.get_otus_by_indices(picked_indices, table)

INFO("Selected {0} OTUs".format(len(picked_indices)))

add_line_to_results_file(res_file, "Selected {0} OTUs:".format(len(picked_indices)))

add_line_to_results_file(res_file, "-"*BANNER_LEN)

for taxonomy in taxonomies:

DEBUG(taxonomy)

add_line_to_results_file(res_file, taxonomy)

add_line_to_results_file(res_file, "Selected OTU IDs:")

add_line_to_results_file(res_file, "-"*BANNER_LEN)

for picked_otu in picked_otus:

DEBUG(picked_otu)

add_line_to_results_file(res_file, picked_otu)

add_line_to_results_file(res_file, "Included Samples:")

add_line_to_results_file(res_file, "-"*BANNER_LEN)

included_samples, included_samples_indices = get_samples_for_otus(

data, picked_indices, otu_filter, sample_filter, table, samples, otus

)

for sample in included_samples:

add_line_to_results_file(res_file, sample)

ref_dist = __get_full_otus_dist(otus, table)

sel_dist = ctwc__metadata_analysis.calculate_otus_distribution(selected_otu_filter, picked_indices, table)

p_vals = ctwc__metadata_analysis.calculate_otus_p_values(sel_dist, ref_dist)

DEBUG("P Values: {0}".format(p_vals))

keys, pv = get_top_p_val(p_vals)

INFO("Top P Value: {0}, keys: {1} {2}".format(pv, keys[0], keys[1]))

coupled_sample_filter, _ = __prepare_sample_filters_from_indices(included_samples_indices, samples, sample_filter)

if len(coupled_sample_filter) > 0:

coupled_ref_dist = __get_full_sample_dist(samples)

coupled_sel_dist = ctwc__metadata_analysis.calculate_samples_distribution(coupled_sample_filter)

enriched = get_enriched_keys_over_threshold(coupled_sel_dist, coupled_ref_dist, COUPLED_ENRICHMENT_THRESHOLD)

if len(enriched) > 0:

add_line_to_results_file(res_file, "Enriched Samples:")

add_line_to_results_file(res_file, "-"*BANNER_LEN)

for key in enriched:

add_line_to_results_file(res_file, "{0}: Selection: {1} Reference: {2} Enrichment: {3}".format(

key, enriched[key][0], enriched[key][1], enriched[key][2])

)

num_otus = len(selected_otu_filter)

num_samples = len(samples) if sample_filter == None else len(sample_filter)

res_file.close()

_, cols_dist = ctwc__distance_matrix.get_distance_matrices(data,

tree,

samples,

otus,

otu_filter=otu_filter,

sample_filter=sample_filter,

skip_rows=True)

picked_indices, last_rank, _, _, _, _ = ctwc__cluster_rank.filter_cols_by_top_rank(data,

cols_dist,

prev,

samples)

selected_sample_filter, complement_sample_filter = __prepare_sample_filters_from_indices(picked_indices, samples, sample_filter)

res_file = create_or_open_results_file(title)

add_line_to_results_file(res_file, "{0} - {1} OTUs X {2} Samples".format(title,

len(otus) if otu_filter is None else len(otu_filter),

len(samples) if sample_filter is None else len(sample_filter)))

add_line_to_results_file(res_file, get_iteration_path_string(title_to_iteration(title)))

add_line_to_results_file(res_file, "-"*BANNER_LEN)

sorted_rows_mat = __sort_matrix_rows_by_selection(cols_dist, picked_indices)

sorted_mat = __sort_matrix_cols_by_selection(sorted_rows_mat, picked_indices)

ctwc__plot.plot_mat(sorted_mat, header="{0}: {1}".format(title, "Samples Distance Matrix - sorted"))

if table is not None:

INFO("Selected {0} samples".format(len(picked_indices)))

add_line_to_results_file(res_file, "Selected {0} samples:".format(len(picked_indices)))

add_line_to_results_file(res_file, "-"*BANNER_LEN)

picked_samples = ctwc__data_handler.get_samples_by_indices(picked_indices, table)

for samp in picked_samples:

add_line_to_results_file(res_file, samp)

DEBUG(samp)

ref_dist = __get_full_sample_dist(samples)

sel_dist = ctwc__metadata_analysis.calculate_samples_distribution(selected_sample_filter)

p_vals = ctwc__metadata_analysis.calculate_samples_p_values(sel_dist, ref_dist)

DEBUG("P Values: {0}".format(p_vals))

keys, pv = get_top_p_val(p_vals)

INFO("Top P Value: {0}, keys: {1} {2}".format(pv, keys[0], keys[1]))

included_otus, included_otus_indices = get_otus_for_samples(

data, picked_indices, otu_filter, sample_filter, table, samples, otus

)

coupled_otu_filter, _ = __prepare_otu_filters_from_indices(included_otus_indices, otus, otu_filter)

add_line_to_results_file(res_file, "Included OTUs:")

add_line_to_results_file(res_file, "-"*BANNER_LEN)

for otu in included_otus:

add_line_to_results_file(res_file, otu)

if len(coupled_otu_filter) > 0:

coupled_ref_dist = __get_full_otus_dist(otus, table)

coupled_sel_dist = ctwc__metadata_analysis.calculate_otus_distribution(

coupled_otu_filter, included_otus_indices, table

)

enriched = get_enriched_keys_over_threshold(coupled_sel_dist, coupled_ref_dist, COUPLED_ENRICHMENT_THRESHOLD)

if len(enriched) > 0:

add_line_to_results_file(res_file, "Enriched OTUs:")

add_line_to_results_file(res_file, "-"*BANNER_LEN)

for key in enriched:

add_line_to_results_file(res_file, "{0}: Selection: {1} Reference: {2} Enrichment: {3}".format(

key, enriched[key][0], enriched[key][1], enriched[key][2])

)

num_otus = len(otus) if otu_filter == None else len(otu_filter)

num_samples = len(selected_sample_filter)

res_file.close()

if iteration_ind == "0":

return [ "1", "2", None, None ]

else:

iteration_results = dict()

__ctwc_recursive_iteration(data, tree, samples, otus, table, iteration_results = iteration_results)

import csv

for elem in iteration_results:

filename = Q_VALUES_ITERATION_FILENAME.format(make_camel_from_string(elem))

with open(filename, 'wb') as csv_file:

csv_writer = csv.writer(csv_file)

iteration_results[elem] = (iteration_results[elem][RES_IND_INPUT],

ctwc__metadata_analysis.correct_p_vals(iteration_results[elem][RES_IND_P_VAL]),

iteration_results[elem][RES_IND_SEL_DIST],

iteration_results[elem][RES_IND_REF_DIST],

iteration_results[elem][RES_IND_NUM_SELECTED],

iteration_results[elem][RES_IND_NUM_TOTAL])

for k in iteration_results[elem][RES_IND_P_VAL]:

ctwc__metadata_analysis.save_q_values_to_csv_for_iteration(csv_writer,

k,

iteration_results[elem][RES_IND_P_VAL],

iteration_results[elem][RES_IND_SEL_DIST],

iteration_results[elem][RES_IND_REF_DIST],

iteration_results[elem][RES_IND_NUM_SELECTED],

iteration_results[elem][RES_IND_NUM_TOTAL])

with open(filename, 'r') as q_vals:

res_file = create_or_open_results_file(elem)

lines = q_vals.readlines()

add_line_to_results_file(res_file, "-"*BANNER_LEN)

add_line_to_results_file(res_file, "Filtered Q values:")

add_line_to_results_file(res_file, "-"*BANNER_LEN)

for line in lines:

add_line_to_results_file(res_file, line.strip())

res_file.close()

#ctwc__plot.wait_for_user()

otu_filter = None,

otu_complement = None,

sample_filter = None,

sample_complement = None,

iteration_ind = "0",

iteration_results = dict()):

if __is_iteration_max_depth(iteration_ind):

return

THRESH = RECURSIVE_THRESHOLD

run_on_sample_selection = sample_filter is None or len(sample_filter) > THRESH

run_on_otu_selection = otu_filter is None or len(otu_filter) > THRESH

run_on_otu_comp = (

(otu_complement is not None and len(otu_complement) > THRESH) and

(sample_filter is None or len(sample_filter) > THRESH)

)

run_on_sample_comp = (

(sample_complement is not None and len(sample_complement) > THRESH) and

(otu_filter is None or (otu_filter) > THRESH)

)

num_total_samples = len(samples)

num_total_otus = len(otus)

if run_on_sample_selection and run_on_otu_selection:

step = __ctwc_recursive__get_next_step(iteration_ind, 0)

title = iteration_to_title(step)

result, step_samp_filter, step_samp_complement, p_vals, dist = run_iteration(title, "Pick samples...",

data,

tree,

samples,

otus,

otu_filter,

sample_filter,

table,

False,

0 if sample_filter is None else len(sample_filter))

sel_dist, ref_dist = dist

iteration_results[title] = (result, p_vals, sel_dist, ref_dist, len(step_samp_filter), num_total_samples)

if sample_filter is None or (len(step_samp_filter) > 0 and len(step_samp_filter) < len(sample_filter)):

__ctwc_recursive_iteration(data, tree, samples, otus, table,

otu_filter,

otu_complement,

step_samp_filter,

step_samp_complement,

step,

iteration_results)

if run_on_otu_selection and run_on_sample_selection:

step = __ctwc_recursive__get_next_step(iteration_ind, 1)

title = iteration_to_title(step)

result, step_otu_filter, step_otu_complement, p_vals, dist = run_iteration(title, "Pick OTUs...",

data,

tree,

samples,

otus,

otu_filter,

sample_filter,

table,

True,

0 if otu_filter is None else len(otu_filter))

sel_dist, ref_dist = dist

iteration_results[title] = (result, p_vals, sel_dist, ref_dist, len(step_otu_filter), num_total_otus)

if otu_filter is None or (len(step_otu_filter) > 0 and len(step_otu_filter) < len(otu_filter)):

__ctwc_recursive_iteration(data, tree, samples, otus, table,

step_otu_filter,

step_otu_complement,

sample_filter,

sample_complement,

step,

iteration_results)

if run_on_sample_comp and run_on_otu_selection:

step = __ctwc_recursive__get_next_step(iteration_ind, 2)

title = iteration_to_title(step)

result, step_samp_filter, step_samp_complement, p_vals, dist = run_iteration(title, "Pick samples from complement...",

data,

tree,

samples,

otus,

otu_filter,

sample_complement,

table,

False)

sel_dist, ref_dist = dist

iteration_results[title] = (result, p_vals, sel_dist, ref_dist, len(step_samp_filter), num_total_samples)

if len(step_samp_filter) < len(sample_complement) and len(step_samp_filter) > 0:

__ctwc_recursive_iteration(data, tree, samples, otus, table,

otu_filter,

otu_complement,

step_samp_filter,

step_samp_complement,

step,

iteration_results)

if run_on_sample_comp:

step = __ctwc_recursive__get_next_step(iteration_ind, 5)

title = iteration_to_title(step)

result, step_otu_filter, step_otu_complement, p_vals, dist = run_iteration(title, "Pick OTUs from samples complement...",

data,

tree,

samples,

otus,

None,

sample_complement,

table,

True)

sel_dist, ref_dist = dist

iteration_results[title] = (result, p_vals, sel_dist, ref_dist, len(step_otu_filter), num_total_otus)

if otu_filter is None or (len(step_otu_filter) < len(otu_filter) and len(step_otu_filter) > 0):

__ctwc_recursive_iteration(data, tree, samples, otus, table,

step_otu_filter,

step_otu_complement,

sample_filter,

sample_complement,

step,

iteration_results)

if run_on_otu_comp and run_on_sample_selection:

step = __ctwc_recursive__get_next_step(iteration_ind, 3)

title = iteration_to_title(step)

result, step_otu_filter, step_otu_complement, p_vals, dist = run_iteration(title, "Pick OTUs from complement...",

data,

tree,

samples,

otus,

otu_complement,

sample_filter,

table,

True)

sel_dist, ref_dist = dist

iteration_results[title] = (result, p_vals, sel_dist, ref_dist, len(step_otu_filter), num_total_otus)

if len(step_otu_filter) < len(otu_complement) and len(step_otu_filter) > 0:

__ctwc_recursive_iteration(data, tree, samples, otus, table,

step_otu_filter,

step_otu_complement,

sample_filter,

sample_complement,

step,

iteration_results)

if run_on_otu_comp:

step = __ctwc_recursive__get_next_step(iteration_ind, 4)

title = iteration_to_title(step)

result, step_samp_filter, step_samp_complement, p_vals, dist = run_iteration(title, "Pick samples from OTUs complement...",

data,

tree,

samples,

otus,

otu_complement,

None,

table,

False)

sel_dist, ref_dist = dist

iteration_results[title] = (result, p_vals, sel_dist, ref_dist, len(step_samp_filter), num_total_samples)

if sample_filter is None or (len(step_samp_filter) < len(sample_filter) and len(step_samp_filter) > 0):

__ctwc_recursive_iteration(data, tree, samples, otus, table,

otu_filter,

otu_complement,

step_samp_filter,

step_samp_complement,

step,

try:

ctwc__plot.init()

np.seterr(all="ignore")

samples, otus, tree, data, table = ctwc__distance_matrix.get_data(use_real_data=True, full_set=True)

#samples, otus, tree, data, table = ctwc__distance_matrix.get_data(use_real_data=False, full_set=False)

output = ctwc_recursive_select(data, tree, samples, otus, table)

write_cluster_summary_for_all_files_in_path(CLUSTER_OUTPUT_FILE)

INFO("Full data size: {0} X {1}".format(data.shape[0], data.shape[1]))

for elem in output:

keys, pv = get_top_p_val(output[elem][RES_IND_P_VAL])

#INFO("{0}: {1} X {2} - P Value {3} Keys {4}".format(elem, output[elem][RES_IND_INPUT][0], output[elem][RES_IND_INPUT][1], pv, keys))

except Exception as ex:

ERROR("Failed with exception: {}, stack trace".format(str(ex)))

ERROR("Calling stack:")