def test_nsl(): n_variants = 1000 n_haplotypes = 20 h = np.random.randint(0, 2, size=(n_variants, n_haplotypes)).astype('i1') for use_threads in True, False: score = nsl(h, use_threads=use_threads) assert_is_instance(score, np.ndarray) eq((n_variants, ), score.shape) eq(np.dtype('f8'), score.dtype)
def nsl(haplotype, pos_vec=None, window=None): """ Compute the standardize number of segregating sites by length (nSl) for each variant, comparing the reference and alternate alleles, after Ferrer-Admetlla et al. (2014) if windowed stat, provide pos_vec too. """ nsl_stats = allel.nsl(haplotype) nsl_stand, bins = allel.standardize_by_allele_count( nsl_stats, haplotype.count_alleles().T[1], diagnostics=False) if window: dn = pd.DataFrame(nsl_stand, columns=["nSL"]) dn["pos_cat"] = pd.cut(pos_vec, window, labels=range(1, window + 1)) dng = dn.groupby("pos_cat").nSL.mean() return dng else: return nsl_stand
for c in chromlist: callset = h5py.File("PNG.phased.autosomal.recode.{}.h5".format(c), mode='r') samples = callset['samples'][:] sample_name = [sid.decode() for sid in samples.tolist()] g = allel.GenotypeChunkedArray(callset["calldata/GT"]) h = g.to_haplotypes() pos = allel.SortedIndex(callset["variants/POS"][:]) acc = h.count_alleles()[:, 1] # ihs ihs = allel.ihs(h, pos, include_edges=True) ihs_std = allel.standardize_by_allele_count(ihs, acc) plt.plot(pos, -np.log10(ihs_std[0])) nan = ~np.isnan(ihs) ihs_real = ihs[nan] pos_ihs = pos[nan] # nsl nsl = allel.nsl(h) nsl_std = allel.standardize_by_allele_count(nsl, acc) plt.plot(pos, -np.log10(nsl_std[0])) nan = ~np.isnan(ihs) nsl_real = ihs[nan] pos_nsl = pos[nan] seldict[c] = (ihs_std[0], nsl_std[0]) ## ehh is site dependent site dependent #ehh = allel.ehh_decay(h) #nan = ~np.isnan(ihs) #ehh_real = ihs[nan] #pos_ehh = pos[nan] # H12
def image_simulation(path1,path2,S, N, file_name, NCHROMS, threshold, apply_threshold,sort,maj_min): """ Generates images from iterations of simulation files - Deals with both txt files and gzip txt files - Calculates summary statistics for each iteration Keyword Arguments: apply_threshold (Boolean) -- Whether or not to apply p-threshold col_order (Boolean) -- Whether or not to order the columns file_name (string) -- The name of the simulation file being processed( either txt or txt.gz) NCHROMS (int) -- N (int) -- N parameter of simulation n_alleles (array) -- Number of alleles at each genome position path1 (string) -- Path to directory where the simulation files exist path2 (string) -- Path to directory where produced image should be stored S (float) -- Selection Co-efficient of simulation threshold (float) -- Threshold value ? row_order (Boolean) -- Whether or not to order the columns maj_min (Boolean) -- Whether or not to colour my major/minor alleles Returns: simulation_error (list) -- List of erronous simulation files and the iteration with error in statistics_list (list) -- List of dictionaries containing summary statitics of simulations """ global once global nsl simulation_error = [] statistics_list = [] dim = [] ################################################## #############OPENING THE SIMULATION FILES######### ################################################## #Suffix of g_zip files (Compressed) gzip_suffix = ".gz" #Suffix of txt files (Uncompressed) txt_suffix = ".txt" #we import and open the file if file_name.endswith(gzip_suffix): with gzip.open(path1 + file_name, 'rb') as f: file = f.read() if type(file) == str: #gzip files might need to be processed to be in correct format file = file.splitlines(True) elif file_name.endswith(txt_suffix): file = open(path1 + file_name).readlines() ################################################## ##########INDEXING THE FILES BY INTERATION######## ################################################## #we look for the caracter // inside the file find = [] for i, string in enumerate(file): if string == '//\n': find.append(i+3) ################################################## ###GENERATE ONE IMAGE PER SIMULATION ITERATION#### ################################################## for ITER, pointer in enumerate(find): try: ########################### ####CREATE CHROM MATRIX#### ########################### n_columns = len(list(file[pointer]))-1 croms = np.zeros((NCHROMS,n_columns),dtype=int) for j in range(NCHROMS): f = list(file[pointer + j]) del f[-1] position_it = file[pointer - 1].split() del position_it[0] position_it = np.array(position_it, dtype='float') position_it = position_it*N F = np.array(f,dtype=int) if j == 0: crom_array = F else: crom_array = np.vstack((crom_array,F)) croms[j,:]=F n_pos = np.size(croms,1) ########################### #####APPLY THRESHOLD####### ########################### if apply_threshold == True: #Count the number of derived alleles at each position count = croms.sum(axis=0,dtype=float) #Calculate the frrequency of the drived allele for each position freq = count/float(NCHROMS) for i in range(n_pos): if freq[i] > 0.5: freq[i] = 1-freq[i] #freq is now a vector that contains the minor allele frequency for each position #we delete the positions in which the minor allele frequency is <= threshold positions = np.where(freq<=threshold) croms,n_pos,freq = delete_simulation(n_pos,croms,freq,positions) ########################### ###COLOUR BY MAJOR/MINOR### ########################### if maj_min == True: #Calculate the Major and the minor allele for each position of the matrix/array #Traspose the matrix/array transponse_array_croms = np.transpose(croms) #Record the Major and Minor allele for each allelic position maj_allele = [] minor_allele = [] for i in range(len(transponse_array_croms)): freq_data = np.unique(transponse_array_croms[i], return_counts = True) index_max = np.argmax(freq_data[1]) if index_max == 0: maj_allele.append(0) minor_allele.append(1) if index_max == 1: maj_allele.append(1) minor_allele.append(0) #Black and white image: #Simulation File: 0 = ancestrial, 1 = Derived (White encoded by 1, Black encoded by 0) #If the major allele is 0, we want to change 0 with 1 and vice verasa (1 = Major, 0 = Minor) #If the major allele is 1, no changes need to be made as 1 would by default be coded to be white matrix_maj_min_col = np.ones((n_pos,NCHROMS),dtype=int) for row in range(len(transponse_array_croms)): if maj_allele[row] == 1: matrix_maj_min_col[row,:] = transponse_array_croms[row] if maj_allele[row] == 0: matrix_maj_min_col[row,:] = matrix_maj_min_col[row,:] - transponse_array_croms[row] #Transpose the matrix so that the rows are the NCHROM and the columns are n_pos croms = np.transpose(matrix_maj_min_col) if maj_min == False: #Black and white image: #Simulation File: 0 = ancestrial, 1 = Derived (White encoded by 1, Black encoded by 0) #We want the opposite(ancestrial = white & derived = black) : hence we need to change 0 with 1 and vice versa before producing the image all1 = np.ones((NCHROMS,n_pos)) croms = all1 - croms ########################### ####ORDER ROWS/COLUMNS##### ########################### if sort == 2: #Sort the matrix by row (chromosome) croms = order_data(croms) if sort == 3: #Sort the matrix by column (genetic posistion) croms_transpose = croms.transpose() croms_transpose = order_data(croms_transpose) croms = croms_transpose.transpose() if sort == 4: #First: sort the matrix by row (chromosome) croms = order_data(croms) #Second: sort the matrix by column (genetic posistion) croms_transpose = croms.transpose() croms_transpose = order_data(croms_transpose) croms = croms_transpose.transpose() ###################### ###IMAGE GENERATION### ###################### #Create image from the simulations bw_croms_uint8 = np.uint8(croms) bw_croms_im = Image.fromarray (bw_croms_uint8*255, mode = 'L') dim.append(bw_croms_im.size[0]) #img..selection_coefficients..NREF..ITER.bmp" string = path2 + file_name + "_"+ str(ITER+1) + str(maj_min)+ str(sort) + ".bmp" bw_croms_im.save(string) ###################### ##Summary Statistics## ###################### ####THINK: DO I NEED TO CHANGE THIS IF THERE IS A MINOR/MAJOR ALLELE CONVERSION n_position_it = np.size(crom_array,1) freq_crom = crom_array.sum(axis=0)/NCHROMS freq_crom = np.array(freq_crom) positions_1 = np.where(freq_crom<0.50) mask_1 = np.ones(n_position_it, dtype=bool) mask_1[positions_1[0]] = False freq_crom = freq_crom[mask_1] n_positions_1 = np.size(freq_crom) #Calculating the summary statistics haplos = np.transpose(crom_array) h = allel.HaplotypeArray(haplos) #tajimasd ac = h.count_alleles() TjD = allel.stats.tajima_d(ac) #watterson theta_hat_w = allel.stats.watterson_theta(position_it, ac) #nsl nsl = allel.nsl(h) nsl = nsl[mask_1] size = np.size(nsl) if size == 0: nsl_max = 0 else: nsl_max = np.max(nsl) #dictionary to store the statistics statistics_dictionary = {'simulation_file': file_name, 'Selection coefficient':str(S),'Population size':str(N),'Iteration':str(ITER+1), 'Tajimas D':TjD,'Watterson':theta_hat_w,'nsl':nsl_max} statistics_list.append(statistics_dictionary) except: simulation_error.append(pointer) continue return(simulation_error,statistics_list,dim)
for c in chromlist: callset = h5py.File("PNG.phased.autosomal.recode.{}.h5".format(c), mode='r') samples = callset['samples'][:] sample_name = [sid.decode() for sid in samples.tolist()] g = allel.GenotypeChunkedArray(callset["calldata/GT"]) h = g.to_haplotypes() pos = allel.SortedIndex(callset["variants/POS"][:]) acc = h.count_alleles()[:, 1] # ihs ihs = allel.ihs(h, pos, include_edges=True) ihs_std = allel.standardize_by_allele_count(ihs, acc) plt.plot(pos, -np.log10(ihs_std[0])) nan = ~np.isnan(ihs) ihs_real = ihs[nan] pos_ihs = pos[nan] # nsl nsl = allel.nsl(h) nsl_std = allel.standardize_by_allele_count(nsl, acc) plt.plot(pos, -np.log10(nsl_std[0])) nan = ~np.isnan(ihs) nsl_real = ihs[nan] pos_nsl = pos[nan] seldict[c] = (ihs_std[0], nsl_std[0]) ## ehh is site dependent site dependent #ehh = allel.ehh_decay(h) #nan = ~np.isnan(ihs) #ehh_real = ihs[nan] #pos_ehh = pos[nan] # H12
def statistics (S, N, file_name, NCHROMS): global once global nsl #importo il file file = open("/home/lucrezialorenzon/Simulations/Results_decompressed/" + file_name).readlines() #cerco il carattere // nel file find = [] for i, string in enumerate(file): if string == '//\n': find.append(i+3) for ITER,pointer in enumerate(find): #croms è la matrice totale for j in range(NCHROMS): f = list(file[pointer + j]) del f[-1] pos = file[pointer - 1].split() del pos[0] pos = np.array(pos, dtype='float') pos = pos*100000 #perchè abbiamo simulato una regione di 100000 posizioni F = np.array(f,dtype=int) if j == 0: croms = F else: croms = np.vstack((croms,F)) #n_pos è il numero di posizioni n_pos = np.size(croms,1) freq = croms.sum(axis=0)/NCHROMS freq = np.array(freq) positions_1 = np.where(freq<0.70) mask_1 = np.ones(n_pos, dtype=bool) mask_1[positions_1[0]] = False freq = freq[mask_1] n_pos_1 = np.size(freq) positions_2 = np.where(freq>0.90) mask_2 = np.ones(n_pos_1, dtype=bool) mask_2[positions_2[0]] = False freq = freq[mask_2] #SUMMARY STATISTICS haplos = np.transpose(croms) h = allel.HaplotypeArray(haplos) #tajimasd ac = h.count_alleles() TjD = allel.stats.tajima_d(ac) #watterson theta_hat_w = allel.stats.watterson_theta(pos, ac) #nsl nsl = allel.nsl(h) nsl = nsl[mask_1] nsl = nsl[mask_2] size = np.size(nsl) if size == 0: nsl_max = 0 else: nsl_max = np.max(nsl) #scrivo su file csv f = open("/home/lucrezialorenzon/Simulations/summarystatistics.csv",'a+') with f: header = ['Selection coefficient','Population size','Iteration','Tajimas D','Watterson','nsl'] writer = csv.DictWriter(f,fieldnames=header) if once == 0: writer.writeheader() writer.writerow({'Selection coefficient':str(S),'Population size':str(N),'Iteration':str(ITER+1), 'Tajimas D':TjD,'Watterson':theta_hat_w,'nsl':nsl_max}) once = 1 else: writer.writerow({'Selection coefficient':str(S),'Population size':str(N),'Iteration':str(ITER+1), 'Tajimas D':TjD,'Watterson':theta_hat_w,'nsl':nsl_max})