def run_ba_cov_neutral_sims(shape=1, scale=1, G=50, N=50, iter1=1000, iter2=1000): df_out = open(pt.get_path() + '/data/simulations/ba_cov_neutral_sims.txt', 'w') df_out.write('\t'.join([ 'N', 'G', 'lamba_mean', 'lambda_neutral', 'Cov', 'Iteration', 'dist_percent' ]) + '\n') covs = [0.2] mean_gamma = shape * scale neutral_range = np.logspace(-2, 1, num=20, endpoint=True, base=10.0) neutral_range = neutral_range[::-1] for neutral_ in neutral_range: for cov in covs: for i in range(iter1): C = pt.get_ba_cov_matrix(G, cov) lambda_genes = np.random.gamma(shape=shape, scale=scale, size=G) lambda_genes_null = np.asarray([neutral_] * G) test_cov_adapt = np.stack( [pt.get_count_pop(lambda_genes, C=C) for x in range(N)], axis=0) # matrix with diaganol values equal to one test_cov_neutral = np.stack([ pt.get_count_pop(lambda_genes_null, C=np.identity(G)) for x in range(N) ], axis=0) test_cov = test_cov_adapt + test_cov_neutral X = pt.hellinger_transform(test_cov) pca = PCA() pca_fit = pca.fit_transform(X) euc_dist = pt.get_mean_pairwise_euc_distance(pca_fit) euc_dists = [] for j in range(iter2): #X_j = pt.hellinger_transform(pt.random_matrix(test_cov)) X_j = pt.hellinger_transform( pt.get_random_matrix(test_cov)) pca_fit_j = pca.fit_transform(X_j) euc_dists.append( pt.get_mean_pairwise_euc_distance(pca_fit_j)) euc_percent = len([k for k in euc_dists if k < euc_dist ]) / len(euc_dists) print(neutral_, cov, i, euc_percent) df_out.write('\t'.join([ str(N), str(G), str(mean_gamma), str(neutral_), str(cov), str(i), str(euc_percent) ]) + '\n') df_out.close()
def run_all_sims(): df_out = open(pt.get_path() + '/data/simulations/cov_euc.txt', 'w') n_pops = 20 n_genes = 50 lambda_genes = np.random.gamma(shape=3, scale=1, size=n_genes) covs = [0.5, 0, -0.5] df_out.write('\t'.join(['Covariance', 'Iteration', 'z_score']) + '\n') for cov in covs: for i in range(100): print(str(cov) + ' ' + str(i)) test_cov = np.stack( [get_count_pop(lambda_genes, cov=cov) for x in range(n_pops)], axis=0) X = pt.hellinger_transform(test_cov) pca = PCA() pca_fit = pca.fit_transform(X) euc_dist = pt.get_euclidean_distance(pca_fit) sim_eucs = [] for j in range(1000): #if j % 100 == 0: # print(j) X_j = pt.hellinger_transform(pt.random_matrix(test_cov)) pca_fit_j = pca.fit_transform(X_j) sim_eucs.append(pt.get_euclidean_distance(pca_fit_j)) z_score = (euc_dist - np.mean(sim_eucs)) / np.std(sim_eucs) df_out.write('\t'.join([str(cov), str(i), str(z_score)]) + '\n') df_out.close()
def run_ntwrk_cov_sims(var=1, cov=0.25): df_out = open( pt.get_path() + '/data/simulations/cov_ntwrk_euc_pos_only_010.txt', 'w') n_pops = 20 n_genes = 250 lambda_genes = np.random.gamma(shape=3, scale=1, size=n_genes) df_out.write('\t'.join(['Cov', 'Iteration', 'z_score']) + '\n') C = np.loadtxt(pt.get_path() + '/data/modular_ntwrk_mu_010.txt', delimiter='\t') #, dtype='int') #print(np.mean(np.sum(ntwrk, axis =1))) C = C * cov np.fill_diagonal(C, var) for i in range(100): test_cov = np.stack( [get_count_pop(lambda_genes, cov=C) for x in range(n_pops)], axis=0) X = pt.hellinger_transform(test_cov) pca = PCA() pca_fit = pca.fit_transform(X) euc_dist = pt.get_mean_pairwise_euc_distance(pca_fit) sim_eucs = [] for j in range(1000): X_j = pt.hellinger_transform(pt.random_matrix(test_cov)) pca_fit_j = pca.fit_transform(X_j) sim_eucs.append(pt.get_mean_pairwise_euc_distance(pca_fit_j)) z_score = (euc_dist - np.mean(sim_eucs)) / np.std(sim_eucs) print(str(cov), ' ', str(i), ' ', str(z_score)) df_out.write('\t'.join([str(cov), str(i), str(z_score)]) + '\n') df_out.close()
def run_block_cov_sims(): df_out = open( pt.get_path() + '/data/simulations/cov_block_euc_pos_only.txt', 'w') n_pops = 20 n_genes = 50 lambda_genes = np.random.gamma(shape=3, scale=1, size=n_genes) df_out.write('\t'.join(['Cov', 'Iteration', 'z_score']) + '\n') #covs = [0.1, 0.3, 0.4, 0.5, 0.6, 0.7, 0.9] covs = [-0.9] for cov in covs: C = get_block_cov(n_genes, pos_cov=cov, neg_cov=cov) print(np.all(np.linalg.eigvals(C) > 0)) print(C) for i in range(100): test_cov = np.stack( [get_count_pop(lambda_genes, cov=C) for x in range(n_pops)], axis=0) X = pt.hellinger_transform(test_cov) pca = PCA() pca_fit = pca.fit_transform(X) euc_dist = pt.get_mean_pairwise_euc_distance(pca_fit) sim_eucs = [] for j in range(1000): X_j = pt.hellinger_transform(pt.random_matrix(test_cov)) pca_fit_j = pca.fit_transform(X_j) sim_eucs.append(pt.get_mean_pairwise_euc_distance(pca_fit_j)) z_score = (euc_dist - np.mean(sim_eucs)) / np.std(sim_eucs) print(str(cov), ' ', str(i), ' ', str(z_score)) df_out.write('\t'.join([str(cov), str(i), str(z_score)]) + '\n') df_out.close()
def two_treats_sim(iter1=1000, iter2=1000, alpha=0.05): genes = 10 pops1 = pops2 = 10 shape = 1 scale = 1 muts1 = muts2 = 20 to_reshuffle = [0, 5, 10, 15, 20] for reshuf in to_reshuffle: p_vales = [] for i in range(iter1): #print(i) rates = np.random.gamma(shape, scale=scale, size=genes) rates1 = rates.copy() # permute rates shuffle(rates[:reshuf]) rates2 = rates.copy() list_dicts1 = [ Counter( np.random.choice(genes, size=muts1, replace=True, p=rates1 / sum(rates1))) for i in range(pops1) ] list_dicts2 = [ Counter( np.random.choice(genes, size=muts2, replace=True, p=rates2 / sum(rates2))) for i in range(pops2) ] df1 = pd.DataFrame(list_dicts1) df2 = pd.DataFrame(list_dicts2) df = pd.concat([df1, df2]) df = df.fillna(0) count_matrix = df.values groups = [ np.asarray(list(range(0, pops1))), np.asarray(list(range(pops1, pops1 + pops2))) ] pca = PCA() X = pt.hellinger_transform(count_matrix) pca_fit = pca.fit_transform(X) F = get_F_stat_pairwise(pca_fit, groups) F_list = [] for j in range(iter2): count_matrix_n0 = pt.random_matrix(count_matrix) X_n0 = pt.hellinger_transform(count_matrix_n0) pca_fit_n0 = pca.fit_transform(X_n0) F_list.append(get_F_stat_pairwise(pca_fit_n0, groups)) p_vales.append( (len([x for x in F_list if x > F]) + 1) / (iter2 + 1)) power = (len([k for k in p_vales if k < alpha]) + 1) / (iter1 + 1) print('Reshuffle = ' + str(reshuf) + ', Power ' + str(power))
def rndm_sample_tenaillon(iter1=1000, iter2=1000): df_path = pt.get_path() + '/data/Tenaillon_et_al/gene_by_pop.txt' df = pd.read_csv(df_path, sep='\t', header='infer', index_col=0) df_np = df.values gene_names = df.columns.values n_rows = df_np.shape[0] df_out = open(pt.get_path() + '/data/Tenaillon_et_al/sample_size_sim.txt', 'w') df_out.write( '\t'.join(['N', 'G', 'Iteration', 'dist_percent', 'z_score']) + '\n') Ns = list(range(2, 40, 2)) for N in Ns: for i in range(iter1): #df_np_i = df_np[np.random.choice(n_rows, N, replace=False), :] #df_np_i = df_np_i[: , ~np.all(df_np_i == 0, axis=0)] #df_i = df.sample(N) df_np_i = df_np[np.random.randint(n_rows, size=N), :] gene_bool = np.all(df_np_i == 0, axis=0) # flip around to select gene_size gene_names_i = list( compress(gene_names, list(map(operator.not_, gene_bool)))) df_np_i = df_np_i[:, ~np.all(df_np_i == 0, axis=0)] #df_i = df_i.loc[:, (df_i != 0).any(axis=0)] np.seterr(divide='ignore') df_np_i_delta = pt.likelihood_matrix_array( df_np_i, gene_names_i, 'Tenaillon_et_al').get_likelihood_matrix() X = pt.hellinger_transform(df_np_i_delta) pca = PCA() pca_fit = pca.fit_transform(X) euc_dist = pt.get_mean_pairwise_euc_distance(pca_fit) euc_dists = [] for j in range(iter2): #df_np_i_j = pt.random_matrix(df_np_i) df_np_i_j = pt.get_random_matrix(df_np_i) np.seterr(divide='ignore') df_np_i_j_delta = pt.likelihood_matrix_array( df_np_i_j, gene_names_i, 'Tenaillon_et_al').get_likelihood_matrix() #df_i_j = pd.DataFrame(data=pt.random_matrix(df_np_i_j), index=df_i.index, columns=df_i.columns) #df_i_j_delta = pt.likelihood_matrix(df_i_j, 'Tenaillon_et_al').get_likelihood_matrix() X_j = pt.hellinger_transform(df_np_i_j_delta) pca_fit_j = pca.fit_transform(X_j) euc_dists.append(pt.get_mean_pairwise_euc_distance(pca_fit_j)) G = df_np_i.shape[1] euc_percent = len([k for k in euc_dists if k < euc_dist ]) / len(euc_dists) z_score = (euc_dist - np.mean(euc_dists)) / np.std(euc_dists) print(str(N), str(i), str(G), str(euc_percent), str(z_score)) df_out.write('\t'.join([ str(N), str(G), str(i), str(euc_percent), str(z_score) ]) + '\n') df_out.close()
def run_ba_cov_sims(gene_list, pop_list, out_name, iter1=1000, iter2=1000): df_out = open(pt.get_path() + '/data/simulations/' + out_name + '.txt', 'w') df_out.write('\t'.join(['N', 'G', 'Cov', 'Iteration', 'dist_percent']) + '\n') covs = [0.1, 0.15, 0.2] for G in gene_list: for N in pop_list: for cov in covs: for i in range(iter1): C = pt.get_ba_cov_matrix(G, cov) while True: lambda_genes = np.random.gamma(shape=1, scale=1, size=G) test_cov = np.stack([ pt.get_count_pop(lambda_genes, cov=C) for x in range(N) ], axis=0) #test_cov_row_sum = test_cov.sum(axis=1) if (np.any(test_cov.sum(axis=1) == 0)) == False: break #if np.count_nonzero(test_cov_row_sum) == len(test_cov_row_sum): # break X = pt.hellinger_transform(test_cov) pca = PCA() pca_fit = pca.fit_transform(X) euc_dist = pt.get_mean_pairwise_euc_distance(pca_fit) euc_dists = [] for j in range(iter2): X_j = pt.hellinger_transform( pt.get_random_matrix(test_cov)) #X_j = pt.hellinger_transform(pt.random_matrix(test_cov)) pca_fit_j = pca.fit_transform(X_j) euc_dists.append( pt.get_mean_pairwise_euc_distance(pca_fit_j)) euc_percent = len([k for k in euc_dists if k < euc_dist ]) / len(euc_dists) print(N, G, cov, i, euc_percent) df_out.write('\t'.join( [str(N), str(G), str(cov), str(i), str(euc_percent)]) + '\n') df_out.close()
def run_pca_sample_size_permutation(iter=10000, analysis='PCA', k=3): df_path = pt.get_path() + '/data/Tenaillon_et_al/gene_by_pop.txt' df = pd.read_csv(df_path, sep='\t', header='infer', index_col=0) df_array = df.as_matrix() sample_sizes = np.linspace(2, df.shape[0], num=20, dtype=int) df_out = open( pt.get_path() + '/data/Tenaillon_et_al/sample_size_permute_' + analysis + '.txt', 'w') column_headers = [ 'Sample_size', 'Iteration', 'MCD', 'mean_angle', 'delta_L' ] df_out.write('\t'.join(column_headers) + '\n') for sample_size in sample_sizes: print("Sample size = " + str(sample_size)) for i in range(iter): print("Sample size = " + str(sample_size) + ' Iteration = ' + str(i)) df_sample = df.sample(n=sample_size) #df_sample = df_sample.loc[:, (df_sample != 0).any(axis=0)] df_sample_delta = pt.likelihood_matrix( df_sample, 'Tenaillon_et_al').get_likelihood_matrix() df_sample_delta = df_sample_delta.loc[:, (df_sample_delta != 0).any( axis=0)] X = pt.hellinger_transform(df_sample_delta) pca = PCA() df_sample_delta_out = pca.fit_transform(X) mcd = pt.get_mean_centroid_distance(df_sample_delta_out, k=k) mean_angle = pt.get_mean_angle(df_sample_delta_out, k=k) mean_length = pt.get_euclidean_distance(df_sample_delta_out, k=k) df_out.write('\t'.join([ str(sample_size), str(i), str(mcd), str(mean_angle), str(mean_length) ]) + '\n') df_out.close()
def run_pca_permutation(iter=10000, analysis='PCA', dataset='tenaillon'): if dataset == 'tenaillon': k = 3 df_path = pt.get_path() + '/data/Tenaillon_et_al/gene_by_pop.txt' df = pd.read_csv(df_path, sep='\t', header='infer', index_col=0) df_array = df.as_matrix() df_out = open( pt.get_path() + '/data/Tenaillon_et_al/permute_' + analysis + '.txt', 'w') column_headers = [ 'Iteration', 'MCD', 'mean_angle', 'mean_dist', 'delta_L', 'x_stat' ] df_out.write('\t'.join(column_headers) + '\n') for i in range(iter): print(i) df_rndm = pd.DataFrame(data=pt.random_matrix(df_array), index=df.index, columns=df.columns) df_rndm_delta = pt.likelihood_matrix( df_rndm, 'Tenaillon_et_al').get_likelihood_matrix() if analysis == 'PCA': X = pt.hellinger_transform(df_rndm_delta) pca = PCA() df_rndm_delta_out = pca.fit_transform(X) #df_pca = pd.DataFrame(data=X_pca, index=df.index) mean_angle = pt.get_mean_angle(df_rndm_delta_out, k=k) mcd = pt.get_mean_centroid_distance(df_rndm_delta_out, k=k) mean_length = pt.get_euc_magnitude_diff(df_rndm_delta_out, k=k) mean_dist = pt.get_mean_pairwise_euc_distance(df_rndm_delta_out, k=k) x_stat = pt.get_x_stat(pca.explained_variance_[:-1]) df_out.write('\t'.join([ str(i), str(mcd), str(mean_angle), str(mean_dist), str(mean_length), str(x_stat) ]) + '\n') df_out.close() elif dataset == 'good': k = 5 df_path = pt.get_path() + '/data/Good_et_al/gene_by_pop.txt' df = pd.read_csv(df_path, sep='\t', header='infer', index_col=0) to_exclude = pt.complete_nonmutator_lines() to_exclude.append('p5') df_nonmut = df[df.index.str.contains('|'.join(to_exclude))] # remove columns with all zeros df_nonmut = df_nonmut.loc[:, (df_nonmut != 0).any(axis=0)] time_points = [int(x.split('_')[1]) for x in df_nonmut.index.values] time_points_set = sorted( list(set([int(x.split('_')[1]) for x in df_nonmut.index.values]))) df_nonmut_array = df_nonmut.as_matrix() time_points_positions = {} for x in time_points_set: time_points_positions[x] = [ i for i, j in enumerate(time_points) if j == x ] df_final = df_nonmut.iloc[time_points_positions[time_points_set[-1]]] df_out = open( pt.get_path() + '/data/Good_et_al/permute_' + analysis + '.txt', 'w') #column_headers = ['Iteration', 'Generation', 'MCD'] column_headers = [ 'Iteration', 'Generation', 'MCD', 'mean_angle', 'delta_L', 'mean_dist' ] df_out.write('\t'.join(column_headers) + '\n') for i in range(iter): print("Iteration " + str(i)) matrix_0 = df_nonmut.iloc[time_points_positions[ time_points_set[0]]] matrix_0_rndm = pt.random_matrix(matrix_0.as_matrix()) df_rndm_list = [ pd.DataFrame(data=matrix_0_rndm, index=matrix_0.index, columns=matrix_0.columns) ] # skip first time step for j, tp in enumerate(time_points_set[0:]): if j == 0: continue df_tp_minus1 = df_nonmut[df_nonmut.index.str.contains( '_' + str(time_points_set[j - 1]))] df_tp = df_nonmut[df_nonmut.index.str.contains('_' + str(tp))] matrix_diff = df_tp.as_matrix() - df_tp_minus1.as_matrix() matrix_0_rndm = matrix_0_rndm + pt.random_matrix(matrix_diff) df_0_rndm = pd.DataFrame(data=matrix_0_rndm, index=df_tp.index, columns=df_tp.columns) df_rndm_list.append(df_0_rndm) df_rndm = pd.concat(df_rndm_list) df_rndm_delta = pt.likelihood_matrix( df_rndm, 'Good_et_al').get_likelihood_matrix() if analysis == 'PCA': X = pt.hellinger_transform(df_rndm_delta) pca = PCA() matrix_rndm_delta_out = pca.fit_transform(X) elif analysis == 'cMDS': matrix_rndm_delta_bc = np.sqrt( pt.get_bray_curtis(df_rndm_delta.as_matrix())) matrix_rndm_delta_out = pt.cmdscale(matrix_rndm_delta_bc)[0] else: print("Analysis argument not accepted") continue df_rndm_delta_out = pd.DataFrame(data=matrix_rndm_delta_out, index=df_rndm_delta.index) for tp in time_points_set: df_rndm_delta_out_tp = df_rndm_delta_out[ df_rndm_delta_out.index.str.contains('_' + str(tp))] df_rndm_delta_out_tp_matrix = df_rndm_delta_out_tp.as_matrix() mean_angle = pt.get_mean_angle(df_rndm_delta_out_tp_matrix, k=k) mcd = pt.get_mean_centroid_distance( df_rndm_delta_out_tp_matrix, k=k) mean_length = pt.get_euc_magnitude_diff( df_rndm_delta_out_tp_matrix, k=k) mean_dist = pt.get_mean_pairwise_euc_distance( df_rndm_delta_out_tp_matrix, k=k) df_out.write('\t'.join([ str(i), str(tp), str(mcd), str(mean_angle), str(mean_length), str(mean_dist) ]) + '\n') df_out.close()
def run_ba_ntwk_cov_sims(): df_out = open(pt.get_path() + '/data/simulations/cov_ba_ntwrk_ev.txt', 'w') n_pops = 100 n_genes = 50 ntwk = nx.barabasi_albert_graph(n_genes, 2) ntwk_np = nx.to_numpy_matrix(ntwk) lambda_genes = np.random.gamma(shape=3, scale=1, size=n_genes) df_out.write('\t'.join([ 'Cov', 'Iteration', 'euc_z_score', 'euc_percent', 'eig_percent', 'mcd_percent_k1', 'mcd_percent_k3' ]) + '\n') covs = [0.05, 0.1, 0.15, 0.2] #covs = [0.2, 0.7] for cov in covs: C = ntwk_np * cov np.fill_diagonal(C, 1) #z_scores = [] #eig_percents = [] #euc_percents = [] #centroid_percents_k1 = [] #centroid_percents_k3 = [] for i in range(1000): test_cov = np.stack( [get_count_pop(lambda_genes, cov=C) for x in range(n_pops)], axis=0) X = pt.hellinger_transform(test_cov) pca = PCA() pca_fit = pca.fit_transform(X) euc_dist = pt.get_mean_pairwise_euc_distance(pca_fit) euc_dists = [] eig = pt.get_x_stat(pca.explained_variance_[:-1]) mcd_k1 = pt.get_mean_centroid_distance(pca_fit, k=1) mcd_k3 = pt.get_mean_centroid_distance(pca_fit, k=3) eigs = [] centroid_dists_k1 = [] centroid_dists_k3 = [] for j in range(1000): X_j = pt.hellinger_transform(pt.random_matrix(test_cov)) #pca_j = PCA() #pca_fit_j = pca_j.fit_transform(X_j) pca_fit_j = pca.fit_transform(X_j) euc_dists.append(pt.get_mean_pairwise_euc_distance(pca_fit_j)) centroid_dists_k1.append( pt.get_mean_centroid_distance(pca_fit_j, k=1)) centroid_dists_k3.append( pt.get_mean_centroid_distance(pca_fit_j, k=3)) eigs.append(pt.get_x_stat(pca.explained_variance_[:-1])) #eigs.append( pt.get_x_stat(pca_j.explained_variance_[:-1]) ) z_score = (euc_dist - np.mean(euc_dists)) / np.std(euc_dists) euc_percent = len([k for k in euc_dists if k < euc_dist ]) / len(euc_dists) eig_percent = len([k for k in eigs if k < eig]) / len(eigs) centroid_percent_k1 = len([ k for k in centroid_dists_k1 if k < mcd_k1 ]) / len(centroid_dists_k1) centroid_percent_k3 = len([ k for k in centroid_dists_k3 if k < mcd_k3 ]) / len(centroid_dists_k3) #eig_percents.append(eig_percent) #euc_percents.append(euc_percent) #z_scores.append(z_score) print(cov, i, z_score, euc_percent, eig_percent) df_out.write('\t'.join([ str(cov), str(i), str(z_score), str(euc_percent), str(eig_percent), str(centroid_percent_k1), str(centroid_percent_k3) ]) + '\n') #print(cov, np.all(np.linalg.eigvals(C) > 0), np.mean(z_scores)) df_out.close()
def plot_permutation(dataset = 'good', analysis = 'PCA', alpha = 0.05): df_path = pt.get_path() + '/data/Good_et_al/gene_by_pop.txt' df = pd.read_csv(df_path, sep = '\t', header = 'infer', index_col = 0) to_exclude = pt.complete_nonmutator_lines() to_exclude.append('p5') df_nonmut = df[df.index.str.contains('|'.join( to_exclude))] # remove columns with all zeros df_nonmut = df_nonmut.loc[:, (df_nonmut != 0).any(axis=0)] df_delta = pt.likelihood_matrix(df_nonmut, 'Good_et_al').get_likelihood_matrix() if analysis == 'PCA': X = pt.hellinger_transform(df_delta) pca = PCA() df_out = pca.fit_transform(X) elif analysis == 'cMDS': df_delta_bc = np.sqrt(pt.get_scipy_bray_curtis(df_delta.as_matrix())) df_out = pt.cmdscale(df_delta_bc)[0] time_points = [ int(x.split('_')[1]) for x in df_nonmut.index.values] time_points_set = sorted(list(set([ int(x.split('_')[1]) for x in df_nonmut.index.values]))) df_rndm_delta_out = pd.DataFrame(data=df_out, index=df_delta.index) mcds = [] for tp in time_points_set: df_rndm_delta_out_tp = df_rndm_delta_out[df_rndm_delta_out.index.str.contains('_' + str(tp))] mcds.append(pt.get_mean_pairwise_euc_distance(df_rndm_delta_out_tp.as_matrix(), k=3)) mcd_perm_path = pt.get_path() + '/data/Good_et_al/permute_' + analysis + '.txt' mcd_perm = pd.read_csv(mcd_perm_path, sep = '\t', header = 'infer', index_col = 0) mcd_perm_x = np.sort(list(set(mcd_perm.Generation.tolist()))) lower_ci = [] upper_ci = [] mean_mcds = [] std_mcds = [] lower_z_ci = [] upper_z_ci = [] for x in mcd_perm_x: mcd_perm_y = mcd_perm.loc[mcd_perm['Generation'] == x] mcd_perm_y_sort = np.sort(mcd_perm_y.mean_dist.tolist()) mean_mcd_perm_y = np.mean(mcd_perm_y_sort) std_mcd_perm_y = np.std(mcd_perm_y_sort) mean_mcds.append(mean_mcd_perm_y) std_mcds.append(std_mcd_perm_y) lower_ci.append(mean_mcd_perm_y - mcd_perm_y_sort[int(len(mcd_perm_y_sort) * alpha)]) upper_ci.append(abs(mean_mcd_perm_y - mcd_perm_y_sort[int(len(mcd_perm_y_sort) * (1 - alpha))])) # z-scores mcd_perm_y_sort_z = [ ((i - mean_mcd_perm_y) / std_mcd_perm_y) for i in mcd_perm_y_sort] lower_z_ci.append(abs(mcd_perm_y_sort_z[int(len(mcd_perm_y_sort_z) * alpha)])) upper_z_ci.append(abs(mcd_perm_y_sort_z[int(len(mcd_perm_y_sort_z) * (1 - alpha))])) fig = plt.figure() plt.figure(1) plt.subplot(211) plt.errorbar(mcd_perm_x, mean_mcds, yerr = [lower_ci, upper_ci], fmt = 'o', alpha = 0.5, \ barsabove = True, marker = '.', mfc = 'k', mec = 'k', c = 'k', zorder=1) plt.scatter(time_points_set, mcds, c='#175ac6', marker = 'o', s = 70, \ edgecolors='#244162', linewidth = 0.6, alpha = 0.5, zorder=2)#, edgecolors='none') #plt.xlabel("Time (generations)", fontsize = 16) #plt.ylabel("Mean \n Euclidean distance", fontsize = 14) plt.ylabel("Mean pair-wise \n Euclidean \n distance, " + r'$ \left \langle d \right \rangle$', fontsize = 14) plt.figure(1) plt.subplot(212) plt.errorbar(mcd_perm_x, [0] * len(mcd_perm_x), yerr = [lower_z_ci, upper_z_ci], fmt = 'o', alpha = 0.5, \ barsabove = True, marker = '.', mfc = 'k', mec = 'k', c = 'k', zorder=1) # zip mean, std, and measured values to make z-scores zip_list = list(zip(mean_mcds, std_mcds, mcds)) z_scores = [((i[2] - i[0]) / i[1]) for i in zip_list ] plt.scatter(time_points_set, z_scores, c='#175ac6', marker = 'o', s = 70, \ edgecolors='#244162', linewidth = 0.6, alpha = 0.5, zorder=2)#, edgecolors='none') plt.ylim(-2.2, 2.2) #plt.axhline(0, color = 'k', lw = 2, ls = '-') #plt.axhline(-1, color = 'dimgrey', lw = 2, ls = '--') #plt.axhline(-2, color = 'dimgrey', lw = 2, ls = ':') plt.xlabel("Time (generations)", fontsize = 16) plt.ylabel("Standardized mean \n pair-wise Euclidean \n distance, " + r'$ z_{\left \langle d \right \rangle}$', fontsize = 14) #plt.ylabel("Standardized mean \n Euclidean distance", fontsize = 14) fig.tight_layout() fig.savefig(pt.get_path() + '/figs/permutation_scatter_good.png', bbox_inches = "tight", pad_inches = 0.4, dpi = 600) plt.close()
def hist_tenaillon_multi(k = 3): df_path = pt.get_path() + '/data/Tenaillon_et_al/gene_by_pop.txt' df = pd.read_csv(df_path, sep = '\t', header = 'infer', index_col = 0) df_delta = pt.likelihood_matrix(df, 'Tenaillon_et_al').get_likelihood_matrix() X = pt.hellinger_transform(df_delta) pca = PCA() df_out = pca.fit_transform(X) df_null_path = pt.get_path() + '/data/Tenaillon_et_al/permute_PCA.txt' df_null = pd.read_csv(df_null_path, sep = '\t', header = 'infer', index_col = 0) mean_angle = pt.get_mean_angle(df_out, k = k) mcd = pt.get_mean_centroid_distance(df_out, k=k) #mean_length = pt.get_euclidean_distance(df_out, k=k) mean_dist = pt.get_mean_pairwise_euc_distance(df_out, k=k) x_stat = pt.get_x_stat(pca.explained_variance_[:-1]) fig = plt.figure() ax1 = plt.subplot2grid((2, 2), (0, 0), colspan=1) ax1.axhline(y=0, color='k', linestyle=':', alpha = 0.8, zorder=1) ax1.axvline(x=0, color='k', linestyle=':', alpha = 0.8, zorder=2) ax1.scatter(0, 0, marker = "o", edgecolors='none', c = 'darkgray', s = 120, zorder=3) ax1.scatter(df_out[:,0], df_out[:,1], marker = "o", edgecolors='#244162', c = '#175ac6', alpha = 0.4, s = 60, zorder=4) ax1.set_xlim([-0.75,0.75]) ax1.set_ylim([-0.75,0.75]) ax1.set_xlabel('PCA 1 (' + str(round(pca.explained_variance_ratio_[0],3)*100) + '%)' , fontsize = 14) ax1.set_ylabel('PCA 2 (' + str(round(pca.explained_variance_ratio_[1],3)*100) + '%)' , fontsize = 14) ax2 = plt.subplot2grid((2, 2), (0, 1), colspan=1) mcd_list = df_null.MCD.tolist() #ax2.hist(mcd_list, bins=30, histtype='stepfilled', normed=True, alpha=0.6, color='b') ax2.hist(mcd_list,bins=30, weights=np.zeros_like(mcd_list) + 1. / len(mcd_list), alpha=0.8, color = '#175ac6') ax2.axvline(mcd, color = 'red', lw = 3) ax2.set_xlabel("Mean centroid distance, " + r'$ \left \langle \delta_{c} \right \rangle$', fontsize = 14) ax2.set_ylabel("Frequency", fontsize = 16) mcd_list.append(mcd) relative_position_mcd = sorted(mcd_list).index(mcd) / (len(mcd_list) -1) if relative_position_mcd > 0.5: p_score_mcd = 1 - relative_position_mcd else: p_score_mcd = relative_position_mcd print('mean centroid distance p-score = ' + str(round(p_score_mcd, 3))) ax2.text(0.366, 0.088, r'$p < 0.05$', fontsize = 10) ax3 = plt.subplot2grid((2, 2), (1, 0), colspan=1) delta_L_list = df_null.mean_dist.tolist() #ax3.hist(delta_L_list, bins=30, histtype='stepfilled', normed=True, alpha=0.6, color='b') ax3.hist(delta_L_list,bins=30, weights=np.zeros_like(delta_L_list) + 1. / len(delta_L_list), alpha=0.8, color = '#175ac6') ax3.axvline(mean_dist, color = 'red', lw = 3) ax3.set_xlabel("Mean pair-wise \n Euclidean distance, " + r'$ \left \langle d \right \rangle$', fontsize = 14) ax3.set_ylabel("Frequency", fontsize = 16) delta_L_list.append(mean_dist) relative_position_delta_L = sorted(delta_L_list).index(mean_dist) / (len(delta_L_list) -1) if relative_position_delta_L > 0.5: p_score_delta_L = 1 - relative_position_delta_L else: p_score_delta_L = relative_position_delta_L print('mean difference in distances p-score = ' + str(round(p_score_delta_L, 3))) ax3.text(0.50, 0.09, r'$p < 0.05$', fontsize = 10) ax4 = plt.subplot2grid((2, 2), (1, 1), colspan=1) ax4_values = df_null.x_stat.values ax4_values = ax4_values[np.logical_not(np.isnan(ax4_values))] #ax4.hist(ax4_values, bins=30, histtype='stepfilled', normed=True, alpha=0.6, color='b') ax4.hist(ax4_values, bins=30, weights=np.zeros_like(ax4_values) + 1. / len(ax4_values), alpha=0.8, color = '#175ac6') print(np.mean(ax4_values)) print(stats.mode(ax4_values)) ax4.axvline(x_stat, color = 'red', lw = 3) ax4.set_xlabel(r'$F_{1}$', fontsize = 14) ax4.set_ylabel("Frequency", fontsize = 16) mean_angle_list = ax4_values.tolist() mean_angle_list.append(mean_angle) relative_position_angle = sorted(mean_angle_list).index(mean_angle) / (len(mean_angle_list) -1) print(x_stat) print( len([x for x in mean_angle_list if x > x_stat])/ sum(mean_angle_list) ) if relative_position_angle > 0.5: p_score_angle = 1 - relative_position_angle else: p_score_angle = relative_position_angle print('F_{1} statistic p-score = ' + str(round(p_score_angle, 3))) ax4.text(19.1, 0.09, r'$p \nless 0.05$', fontsize = 10) plt.tight_layout() fig_name = pt.get_path() + '/figs/fig1.png' fig.savefig(fig_name, bbox_inches = "tight", pad_inches = 0.4, dpi = 600) plt.close()