def local(geno_name, harmonizer_path, vcf_path, legend_path, fasta_path): # Using 1000 Genomes as a reference(based off Perl script by W.Rayner, 2015, wrayner @ well.ox.ac.uk) # -Removes SNPs with MAF < 5% in study dataset # -Removes SNPs not in 1000 Genomes Phase 3 # -Removes all A/T G/C SNPs with MAF > 40% in the reference data set # -Removes all SNPs with an AF difference >0.2, between reference and dataset frequency file, frequency file is # expected to be a plink frequency file with the same number of SNPs as the bim file # -Removes duplicates that may be introduced with the position update # -Removes indels #Need to figure out how to do this. Or even if it is necessary with plink files. # -Removes SNPs with HWE p-value < 0.01 # -Updates the reference allele to match 1000G # -Outputs new files per chromosome, in plink bed/bim/fam format. # Needed modules import sys import csv import gzip try: import pandas as pd except (ImportError, ModuleNotFoundError): import genodownload genodownload.getpandas() import pandas as pd try: import numpy as np except (ImportError, ModuleNotFoundError): import genodownload genodownload.getnumpy() import numpy as np # Get current working directory. orig_wd = os.getcwd() # Make new folder where the harmonized files will be located. if not os.path.exists('Harmonized_To_1000G'): os.makedirs('Harmonized_To_1000G') # Copy genotype files to new folder. shutil.copy2(geno_name + '.bed', 'Harmonized_To_1000G') shutil.copy2(geno_name + '.bim', 'Harmonized_To_1000G') shutil.copy2(geno_name + '.fam', 'Harmonized_To_1000G') # Switch to this directory. os.chdir('Harmonized_To_1000G') # Make the lists that we're going to need, since this is on a per chromosome basis. vcf_file_names = ['ALL.chr%d.phase3_shapeit2_mvncall_integrated_v5a.20130502.genotypes.vcf.gz' % x for x in range(1, 23)] vcf_file_names.extend(['ALL.chrX.phase3_shapeit2_mvncall_integrated_v1b.20130502.genotypes.vcf.gz']) legend_file_names = ['1000GP_Phase3_chr%d.legend.gz' % x for x in range(1, 23)] legend_file_names.extend(['1000GP_Phase3_chrX_NONPAR.legend.gz']) harmonized_geno_names = [geno_name + '_chr%d_Harmonized' % x for x in range(1, 24)] id_updates = [s + '_idUpdates.txt' for s in harmonized_geno_names] id_update_names = [s + '_idUpdates.txt' for s in harmonized_geno_names] snp_logs = [s + '_snpLog.log' for s in harmonized_geno_names] snp_log_names = [s + '_snpLog.log' for s in harmonized_geno_names] freq_file_names = [geno_name + '_chr%d_Harmonized.frq' % x for x in range(1, 24)] af_diff_removed_by_chr = ['chr%d_SNPsRemoved_AFDiff' % x for x in range(1, 24)] final_snps_by_chr = ['chr%d_SNPsKept' % x for x in range(1, 24)] final_snp_lists = ['chr%d_SNPsKept.txt' % x for x in range(1, 24)] af_checked_names = [geno_name + '_chr%d_HarmonizedTo1000G' % x for x in range(1, 24)] # Harmonize for each chromosome for i in range(0, len(vcf_file_names)): # Call genotype harmonizer for autosomes if i < 22: # Remove SNPs with HWE p-value < 0.01 and SNPs with MAF < 0.05 subprocess.check_output([plink, '--bfile', geno_name, '--chr', str(i+1), '--hardy', '--hwe', '0.01', '--maf', '0.05', '--make-bed', '--out', geno_name + '_MAF_HWE_Filter_chr' + str(i+1)]) subprocess.check_output('java -Xmx1g -jar "' + harmonizer_path + '/GenotypeHarmonizer.jar" $* --input ' + geno_name + '_MAF_HWE_Filter_chr' + str(i+1) + ' --ref "' + os.path.join(vcf_path, vcf_file_names[i]) + '" --refType VCF --update-id --debug --mafAlign 0 --update-reference-allele ' '--outputType PLINK_BED --output ' + harmonized_geno_names[i], shell=True) subprocess.call(rm + geno_name + '_MAF_HWE_Filter_chr' + str(i + 1) + '.*', shell=True) id_updates[i] = pd.read_csv(id_update_names[i], sep='\t', header=0, dtype={'chr': str, 'pos': int, 'originalId': str, 'newId': str}) snp_logs[i] = pd.read_table(snp_log_names[i], sep='\t', header=0, dtype={'chr': str, 'pos': int, 'id': str, 'alleles': str, 'action': str, 'message': str}) elif i == 22: # Special handling for chrX # Make list of females fam_file = pd.read_csv(geno_name + '.fam', sep=' ', header=None) females = fam_file.loc[fam_file[4] == 2] females[1].to_csv(geno_name + '_Females.txt', sep='\t', header=None, index=False) # Make hwe statistics using just females subprocess.check_output([plink, '--bfile', geno_name, '--chr', 'X', '--hardy', '--keep', geno_name + '_Females.txt', '--out', geno_name + '_chr23']) # Get list of SNPs with HWE p-values < 0.01 hwe = pd.read_csv(geno_name + '_chr23.hwe', sep='\t', header=None, skiprows=1) hweremove = hwe.loc[hwe[8] <= 0.01] hweremove[1].to_csv(geno_name + '_chr23_RemHWE.txt', sep='\t', header=None, index=False) # Remove these from plink file subprocess.check_output([plink, '--bfile', geno_name, '--chr', 'X', '--maf', '0.05', '--exclude', geno_name + '_chr23_RemHWE.txt', '--make-bed', '--out', geno_name + '_MAF_HWE_Filter_chr23']) # Read chrX file into pandas bim_file = pd.read_csv(geno_name + '_MAF_HWE_Filter_chr23.bim', sep='\t', header=None) # Replace '23' with 'X', which is how genotype harmonizer calls X bim_file.iloc[:, 0].replace(23, 'X', inplace=True) # Write new genotype bim_file.to_csv(geno_name + '_MAF_HWE_Filter_chr23.bim', sep='\t', header=False, index=False, na_rep='NA') # Call genotype harmonizer for X chromosome subprocess.check_output('java -Xmx1g -jar "' + harmonizer_path + '/GenotypeHarmonizer.jar" $* --input ' + geno_name + '_MAF_HWE_Filter_chr23 --ref "' + os.path.join(vcf_path, vcf_file_names[i]) + '" --refType VCF --update-id --debug --mafAlign 0 --update-reference-allele ' '--outputType PLINK_BED --output ' + harmonized_geno_names[i], shell=True) subprocess.call(rm + geno_name + '_MAF_HWE_Filter_chr23.*', shell=True) subprocess.call(rm + geno_name + '_chr23*', shell=True) subprocess.call(rm + geno_name + '_chr23_RemHWE.txt*', shell=True) id_updates[i] = pd.read_csv(id_update_names[i], sep='\t', header=0, dtype={'chr': str, 'pos': int, 'originalId': str, 'newId': str}) snp_logs[i] = pd.read_csv(snp_log_names[i], sep='\t', header=0, dtype={'chr': str, 'pos': int, 'id': str, 'alleles': str, 'action': str, 'message': str}) else: print(Fore.RED + Style.BRIGHT) sys.exit("Something is wrong with the number/name of reference files") # Concatenate all of the id updates into one file. all_id_updates = pd.concat([id_updates[0], id_updates[1], id_updates[2], id_updates[3], id_updates[4], id_updates[5], id_updates[6], id_updates[7], id_updates[8], id_updates[9], id_updates[10], id_updates[11], id_updates[12], id_updates[13], id_updates[14], id_updates[15], id_updates[16], id_updates[17], id_updates[18], id_updates[19], id_updates[20], id_updates[21], id_updates[22]]) # Write list to text file. all_id_updates.to_csv('Harmonization_ID_Updates.txt', sep='\t', header=True, index=False) # Remove the clutter if os.path.getsize('Harmonization_ID_Updates.txt') > 0: for f in id_update_names: subprocess.call([rm, f]) all_snp_logs = pd.concat([snp_logs[0], snp_logs[1], snp_logs[2], snp_logs[3], snp_logs[4], snp_logs[5], snp_logs[6], snp_logs[7], snp_logs[8], snp_logs[9], snp_logs[10], snp_logs[11], snp_logs[12], snp_logs[13], snp_logs[14], snp_logs[15], snp_logs[16], snp_logs[17], snp_logs[18], snp_logs[19], snp_logs[20], snp_logs[21], snp_logs[22]]) # Write list to text file. all_snp_logs.to_csv('Harmonization_SNP_Logs.txt', sep='\t', header=True, index=False) # Remove the clutter if os.path.getsize('Harmonization_SNP_Logs.txt') > 0: for f in snp_log_names: subprocess.call([rm, f]) # Now for each that was just harmonized, remove all SNPs with an allele (AF) difference > 0.2 since we are going to # use a global reference population between study dataset and all superpopulation allele frequencies. IF within 0.2 # of any superpopulation frequency, keep variant. Frequency file is expected to be a Plink frequency file with the # same number of variants as the bim file. for i in range(0, len(harmonized_geno_names)): # Create plink file subprocess.check_output([plink, '--bfile', harmonized_geno_names[i], '--freq', '--out', harmonized_geno_names[i]]) # Read freq file into python freq_file = pd.read_csv(freq_file_names[i], sep='\s+', header=0, usecols=[0, 1, 2, 3, 4], dtype={'CHR': int, 'SNP': str, 'A1': str, 'A2': str, 'MAF': float}) # Rename columns of freq file. freq_file.rename(columns={'A1': 'dataset_a1', 'A2': 'dataset_a2', 'MAF': 'dataset_a1_frq'}, inplace=True) # Calculate the frequency of the second allele, since it's not given in the freq file. freq_file['dataset_a2_frq'] = 1 - freq_file['dataset_a1_frq'] # Read in bim file. bim_file = pd.read_csv(harmonized_geno_names[i] + '.bim', sep='\s+', header=None, usecols=[0, 1, 3], names=['CHR', 'SNP', 'position']) # Merge frequency file with bim file to get position for each SNP freq_file_with_position = pd.merge(left=freq_file, right=bim_file, how='inner', on=['CHR', 'SNP']) # Read in legend file. legend_file = pd.read_csv(os.path.join(legend_path, legend_file_names[i]), compression="gzip", sep=" ", header=0, dtype={'id': str, 'position': int, 'a0': str, 'a1': str, 'TYPE': str, 'AFR': float, 'AMR': float, 'EAS': float, 'EUR': float, 'SAS': float, 'ALL': float}) # Rename columns of legend file legend_file.rename(columns={'id': 'reference_id', 'a0': 'reference_a0', 'a1': 'reference_a1'}, inplace=True) # To Remove A/T or G/C SNPs in reference file that have an MAF > 40%, first identify which are AT/GC SNPs. legend_file['ATGC_SNP'] = np.where( ((legend_file['reference_a0'] == 'A') & (legend_file['reference_a1'] == 'T')) | ((legend_file['reference_a0'] == 'T') & (legend_file['reference_a1'] == 'A')) | ((legend_file['reference_a0'] == 'C') & (legend_file['reference_a1'] == 'G')) | ((legend_file['reference_a0'] == 'G') & (legend_file['reference_a1'] == 'C')), 'ATGC', 'Fine') # For each superpopulation, create a new column with the MAF. If the AF in that column is less than 0.5, then # that is the MAF, if not then 1-AF is MAF legend_file['AFR_MAF'] = np.where(legend_file['AFR'] < 0.5, legend_file['AFR'], 1 - legend_file['AFR']) legend_file['AMR_MAF'] = np.where(legend_file['AMR'] < 0.5, legend_file['AMR'], 1 - legend_file['AMR']) legend_file['EAS_MAF'] = np.where(legend_file['EAS'] < 0.5, legend_file['EAS'], 1 - legend_file['EAS']) legend_file['EUR_MAF'] = np.where(legend_file['EUR'] < 0.5, legend_file['EUR'], 1 - legend_file['EUR']) legend_file['SAS_MAF'] = np.where(legend_file['SAS'] < 0.5, legend_file['SAS'], 1 - legend_file['SAS']) # Create column in legend file with decision about whether to keep or remove SNP, if it is an ATGC SNP where # the MAF in all superpopulations is greater than 40%, then remove it. legend_file['MAF_Decision'] = np.where((legend_file['ATGC_SNP'] == 'ATGC') & (legend_file['AFR_MAF'] > 0.4) & (legend_file['AMR_MAF'] > 0.4) & (legend_file['EAS_MAF'] > 0.4) & (legend_file['EUR_MAF'] > 0.4) & (legend_file['SAS_MAF'] > 0.4), 'Remove', 'Keep') # Make new legend file with just the SNPs that pass this threshold. legend_file = legend_file[legend_file['MAF_Decision'] == 'Keep'] legend_file.drop(labels=['ATGC_SNP', 'AFR_MAF', 'AMR_MAF', 'EAS_MAF', 'EUR_MAF', 'SAS_MAF', 'MAF_Decision'], axis=1, inplace=True) # Merge freq file with positions with legend file to get overlap. This file contains only SNPs with matches in # 1000G merged_file = pd.merge(left=freq_file_with_position, right=legend_file, how='inner', on='position') merged_file = merged_file[merged_file['TYPE'] == 'Biallelic_SNP'] # The MAF column in the frq file is the allele frequency of the A1 allele (which is usually minor, but # possibly not in my case because I just updated the reference to match 1000G. # The AF columns in the legend file are the allele frequencies of the a1 allele in that file. # Need to match based on A1 alleles (hopefully this has already been done in the harmonization step, then # calculate the allele frequency difference. # For each population group, match the alleles where they aren't flipped (i.e. in the same order in house and # ref datasets.. merged_file['AFR_Match_Diff'] = np.where((merged_file['dataset_a1'] == merged_file['reference_a1']) & (merged_file['dataset_a2'] == merged_file['reference_a0']), abs(merged_file['dataset_a1_frq'] - merged_file['AFR']), '') merged_file['AMR_Match_Diff'] = np.where((merged_file['dataset_a1'] == merged_file['reference_a1']) & (merged_file['dataset_a2'] == merged_file['reference_a0']), abs(merged_file['dataset_a1_frq'] - merged_file['AMR']), '') merged_file['EAS_Match_Diff'] = np.where((merged_file['dataset_a1'] == merged_file['reference_a1']) & (merged_file['dataset_a2'] == merged_file['reference_a0']), abs(merged_file['dataset_a1_frq'] - merged_file['EAS']), '') merged_file['EUR_Match_Diff'] = np.where((merged_file['dataset_a1'] == merged_file['reference_a1']) & (merged_file['dataset_a2'] == merged_file['reference_a0']), abs(merged_file['dataset_a1_frq'] - merged_file['EUR']), '') merged_file['SAS_Match_Diff'] = np.where((merged_file['dataset_a1'] == merged_file['reference_a1']) & (merged_file['dataset_a2'] == merged_file['reference_a0']), abs(merged_file['dataset_a1_frq'] - merged_file['SAS']), '') # For each population group, match the flipped alleles. merged_file['AFR_FlipMatch_Diff'] = np.where((merged_file['dataset_a1'] == merged_file['reference_a0']) & (merged_file['dataset_a2'] == merged_file['reference_a1']), abs(merged_file['dataset_a2_frq'] - merged_file['AFR']), '') merged_file['AMR_FlipMatch_Diff'] = np.where((merged_file['dataset_a1'] == merged_file['reference_a0']) & (merged_file['dataset_a2'] == merged_file['reference_a1']), abs(merged_file['dataset_a2_frq'] - merged_file['AMR']), '') merged_file['EAS_FlipMatch_Diff'] = np.where((merged_file['dataset_a1'] == merged_file['reference_a0']) & (merged_file['dataset_a2'] == merged_file['reference_a1']), abs(merged_file['dataset_a2_frq'] - merged_file['EAS']), '') merged_file['EUR_FlipMatch_Diff'] = np.where((merged_file['dataset_a1'] == merged_file['reference_a0']) & (merged_file['dataset_a2'] == merged_file['reference_a1']), abs(merged_file['dataset_a2_frq'] - merged_file['EUR']), '') merged_file['SAS_FlipMatch_Diff'] = np.where((merged_file['dataset_a1'] == merged_file['reference_a0']) & (merged_file['dataset_a2'] == merged_file['reference_a1']), abs(merged_file['dataset_a2_frq'] - merged_file['SAS']), '') # Add the nonflipped and flipped columns together to find out which files have an allele frequency # difference > 0.2 merged_file['AFR_Diff'] = merged_file['AFR_Match_Diff'] + merged_file['AFR_FlipMatch_Diff'] merged_file['AMR_Diff'] = merged_file['AMR_Match_Diff'] + merged_file['AMR_FlipMatch_Diff'] merged_file['EAS_Diff'] = merged_file['EAS_Match_Diff'] + merged_file['EAS_FlipMatch_Diff'] merged_file['EUR_Diff'] = merged_file['EUR_Match_Diff'] + merged_file['EUR_FlipMatch_Diff'] merged_file['SAS_Diff'] = merged_file['SAS_Match_Diff'] + merged_file['SAS_FlipMatch_Diff'] # Delete these columns because we don't need them anymore merged_file.drop(labels=['AFR_Match_Diff', 'AFR_FlipMatch_Diff', 'AMR_Match_Diff', 'AMR_FlipMatch_Diff', 'EAS_Match_Diff', 'EAS_FlipMatch_Diff', 'EUR_Match_Diff', 'EUR_FlipMatch_Diff', 'SAS_Match_Diff', 'SAS_FlipMatch_Diff'], axis=1, inplace=True) # Make allele freuqency columns numeric. merged_file[['AFR_Diff', 'AMR_Diff', 'EAS_Diff', 'EUR_Diff', 'SAS_Diff']] = \ merged_file[['AFR_Diff', 'AMR_Diff', 'EAS_Diff', 'EUR_Diff', 'SAS_Diff']].apply(pd.to_numeric, errors='coerce') # Make new column 'AF_Decision' where you remove alleles that have allele frequency differences > 0.2 from all # population groups. merged_file['AF_Decision'] = np.where((merged_file['AFR_Diff'] > 0.2) & (merged_file['AMR_Diff'] > 0.2) & (merged_file['EAS_Diff'] > 0.2) & (merged_file['EUR_Diff'] > 0.2) & (merged_file['SAS_Diff'] > 0.2), 'Remove', 'Keep') # Drop duplicate SNPs merged_file.drop_duplicates(subset=['SNP'], keep=False, inplace=True) # Write file for each chromosome of the SNPs that we've removed in this step. af_diff_removed_by_chr[i] = merged_file[merged_file['AF_Decision'] == 'Remove'] # Write list for each chromosome of final SNPs that we are keeping. final_snps_by_chr[i] = merged_file[merged_file['AF_Decision'] == 'Keep'] # Write list for each chromosome, because we're going to use it to filter the chromosomes to create new files. final_snps_by_chr[i]['SNP'].to_csv(final_snp_lists[i], sep='\t', header=False, index=False) # Make plink files for each chromosomes. Need bed file for merging subprocess.check_output([plink, '--bfile', harmonized_geno_names[i], '--extract', final_snp_lists[i], '--make-bed', '--out', af_checked_names[i]]) # Remove extra files that we don't need anymore. These were files that were harmonized, but not checked for # allele frequency differences. if os.path.getsize(af_checked_names[i] + '.bim') > 0: os.remove(final_snp_lists[i]) subprocess.call(rm + harmonized_geno_names[i] + '.*', shell=True) # Done with one chromosome. print('Finished with chr' + str(i + 1)) # Make a big list of all SNPs removed and all SNPs kept just for reference purposes. all_snps_removed = pd.concat([af_diff_removed_by_chr[0], af_diff_removed_by_chr[1], af_diff_removed_by_chr[2], af_diff_removed_by_chr[3], af_diff_removed_by_chr[4], af_diff_removed_by_chr[5], af_diff_removed_by_chr[6], af_diff_removed_by_chr[7], af_diff_removed_by_chr[8], af_diff_removed_by_chr[9], af_diff_removed_by_chr[10], af_diff_removed_by_chr[11], af_diff_removed_by_chr[12], af_diff_removed_by_chr[13], af_diff_removed_by_chr[14], af_diff_removed_by_chr[15], af_diff_removed_by_chr[16], af_diff_removed_by_chr[17], af_diff_removed_by_chr[18], af_diff_removed_by_chr[19], af_diff_removed_by_chr[20], af_diff_removed_by_chr[21], af_diff_removed_by_chr[22]]) # Write list to text file. all_snps_removed.to_csv('SNPs_Removed_AFCheck.txt', sep='\t', header=True, index=False) # Make one big list of all SNPs kept all_snps_kept = pd.concat([final_snps_by_chr[0], final_snps_by_chr[1], final_snps_by_chr[2], final_snps_by_chr[3], final_snps_by_chr[4], final_snps_by_chr[5], final_snps_by_chr[6], final_snps_by_chr[7], final_snps_by_chr[8], final_snps_by_chr[9], final_snps_by_chr[10], final_snps_by_chr[11], final_snps_by_chr[12], final_snps_by_chr[13], final_snps_by_chr[14], final_snps_by_chr[15], final_snps_by_chr[16], final_snps_by_chr[17], final_snps_by_chr[18], final_snps_by_chr[19], final_snps_by_chr[20], final_snps_by_chr[21], final_snps_by_chr[22]]) # Write this list to a text file. all_snps_kept.to_csv('SNPs_Kept_AFCheck.txt', sep='\t', header=True, index=False) # Merge harmonized dataset genotypes with open("HouseMergeList.txt", "w") as f: wr = csv.writer(f, delimiter="\n") wr.writerow(af_checked_names) subprocess.check_output([plink, '--merge-list', 'HouseMergeList.txt', '--geno', '0.01', '--make-bed', '--out', geno_name + '_HarmonizedTo1000G']) if os.path.getsize(geno_name + '_HarmonizedTo1000G.bim') > 0: for i in range(0, len(af_checked_names)): subprocess.call(rm + str(af_checked_names[i]) + '.*', shell=True) else: print(Fore.RED + Style.BRIGHT) sys.exit("For some reason the house gentoypes did not merge. You should try it manually. Then, you will need " "to use the program snpflip to make sure the snps are on the same strand as the reference.") # Check to make sure the snps are on the same strand as the reference # First need to change the chromosome names to match the fasta file so they can match. # Read chrX file into pandas bim_file = pd.read_csv(geno_name + '_HarmonizedTo1000G.bim', sep='\t', header=None, dtype={0: str, 1: str, 2: int, 3: int, 4: str, 5: str}) # Replace '23' with 'X', which is how the fasta file calls X bim_file.iloc[:, 0].replace('23', 'X', inplace=True) # Replace '24' with 'Y', which is how the fasta file calls Y bim_file.iloc[:, 0].replace('24', 'Y', inplace=True) # Replace '26' with 'MT' which is how the fasta file calls the mitochondrial DNA bim_file.iloc[:, 0].replace('26', 'MT', inplace=True) # Write new genotype bim_file.to_csv(geno_name + '_HarmonizedTo1000G.bim', sep='\t', header=False, index=False, na_rep='NA') # Fasta file needs to be unzipped for snpflip to work. if os.path.exists(os.path.join(fasta_path, 'human_g1k_v37.fasta')): pass elif os.path.exists(os.path.join(fasta_path, 'human_g1k_v37.fasta.gz')): try: with gzip.open(os.path.join(fasta_path, 'human_g1k_v37.fasta.gz'), 'rb') as f_in, \ open(os.path.join(fasta_path, 'human_g1k_v37.fasta'), 'wb') as f_out: shutil.copyfileobj(f_in, f_out) except: if system_check in ("Linux", "Darwin"): os.system('gunzip -c ' + os.path.join(fasta_path, 'human_g1k_v37.fasta.gz') + ' > ' + os.path.join(fasta_path, 'human_g1k_v37.fasta')) elif system_check == "Windows": zip_path = [] for r, d, f in os.walk(os.path.join('C:\\', 'Program Files')): for files in f: if files == "7zG.exe": zip_path = os.path.join(r, files) subprocess.check_output([zip_path, 'e', os.path.join(fasta_path, 'human_gik_v37.fasta.gz')]) else: sys.exit("Quitting because I cannot find the fasta file. You must have this for snpflip to run.") # Determine if they have snpflip, if not download it. if os.path.exists(os.path.join(bindir, 'snpflip')): pass else: import genodownload genodownload.snpflip() snpflip_path = [os.path.join(bindir, 'snpflip')] # Perform flip check. subprocess.check_output('python ' + snpflip_path[0] + ' --fasta-genome "' + os.path.join(fasta_path, 'human_g1k_v37.fasta') + '" --bim-file ' + geno_name + '_HarmonizedTo1000G.bim --output-prefix ' + geno_name + '_HarmonizedTo1000G', shell=True) # If SNPs exist that are on the reverse strand, then flip them. # Currently ignores snps that are ambiguous, since I already removed those that would be hard to phase. Could change # this later. if os.path.getsize(geno_name + '_HarmonizedTo1000G.reverse') > 0: subprocess.check_output([plink, '--bfile', geno_name + '_HarmonizedTo1000G', '--flip', geno_name + '_HarmonizedTo1000G.reverse', '--make-bed', '--out', geno_name + '_HarmonizedTo1000G_StrandChecked']) # If .reverse doesn't exist, still make a new file to signify that you've done the strand check. else: subprocess.check_output([plink, '--bfile', geno_name + '_HarmonizedTo1000G', '--make-bed', '--out', geno_name + '_HarmonizedTo1000G_StrandChecked']) # Finished shutil.copy2(geno_name + '_HarmonizedTo1000G_StrandChecked.bed', orig_wd) shutil.copy2(geno_name + '_HarmonizedTo1000G_StrandChecked.bim', orig_wd) shutil.copy2(geno_name + '_HarmonizedTo1000G_StrandChecked.fam', orig_wd) print("Finished with harmonization")
def merge1000g(harmonized_name, harmonized_path): import os import sys import subprocess # Get original working directory orig_wd = os.getcwd() # Make sure user has harmonized first. print(Fore.BLUE + Style.BRIGHT) merge_proceed = input("You must harmonize your data with 1000G before this step. Have you already done this? " "(y/n): ").lower() print(Style.RESET_ALL) if merge_proceed in ("y", "yes"): import csv import shutil import glob try: import pandas as pd except (ImportError, ModuleNotFoundError): import genodownload genodownload.getpandas() import pandas as pd # Create new directory for storing these files. if not os.path.exists('Merged_With_1000G'): os.makedirs('Merged_With_1000G') # Copy plink to new folder. if not glob.glob(r'plink*'): sys.exit("We need plink to run this part of the script. Please run step 1 if you do not have plink, or " "make sure that it is in this directory.") else: for file in glob.glob(r'plink*'): print(file) shutil.copy(file, 'Merged_With_1000G') # Making sure they still have the 1000G vcf files. They should if they have just harmonized, but they might # have deleted them or something. print(Fore.MAGENTA + Style.BRIGHT) vcf_exists = input('Have you already downloaded the 1000G Phase3 VCF files? (y/n): ').lower() print(Style.RESET_ALL) if vcf_exists in ('y', 'yes'): # Getting user's path to VCF files print(Fore.GREEN) vcf_path = input('Please enter the pathname of where your 1000G vcf files are ' '(i.e. C:\\Users\\Julie White\\Box Sync\\1000GP\\VCF\\ etc.): ') print(Style.RESET_ALL) elif vcf_exists in ('n', 'no'): # Get module where downloading instructions are. import genodownload # From that module, call download 1000G Phase 3 VCF genodownload.vcf_1000g_phase3() # Saving VCF path vcf_path = os.path.join(os.getcwd(), '1000G_Phase3_VCF') else: sys.exit("Please answer yes or no. Quitting now because no VCF files.") # Names of per chromosome files that we're going to merge into one big file. ref_file_names = ['ALL.chr%d.phase3_shapeit2_mvncall_integrated_v5a.20130502.genotypes.vcf.gz' % x for x in range(1, 23)] ref_file_names.extend(['ALL.chrX.phase3_shapeit2_mvncall_integrated_v1b.20130502.genotypes.vcf.gz']) chr_1000g_phase3_names = ['chr%d_1000G_Phase3' % x for x in range(1, 24)] # Merge 1000G chr data into one plink formatted file, need to convert from vcf files - but only taking the snps # that are in the house dataset # Read in SNPs_Kept file from harmonization process if os.path.exists(os.path.join(harmonized_path, 'SNPs_Kept.txt')): house_snps_kept = pd.read_csv(os.path.join(harmonized_path, 'SNPs_Kept.txt'), header=0, sep='\t') # Keep only the 'SNP' column house_snps_kept = house_snps_kept.loc[:, ['SNP']] # Write that column to a file to be used by plink house_snps_kept.to_csv('Merged_With_1000G/SNPs_Kept_List.txt', sep='\t', header=False, index=False) else: sys.exit("Quitting because I cannot find a file called 'SNPs_Kept.txt' at " + harmonized_path + ". This is a product of the harmonization process and is necessary for " "merging with 1000G.") # Change to directory where we're going to merge the files. os.chdir('Merged_With_1000G') # Convert vcf files to plink format. for i in range(0, len(ref_file_names)): subprocess.check_output([plink, '--vcf', os.path.join(vcf_path, ref_file_names[i]), '--double-id', '--biallelic-only', 'strict', '--vcf-require-gt', '--extract', 'SNPs_Kept_List.txt', '--make-bed', '--out', chr_1000g_phase3_names[i]]) subprocess.call(rm + '*~', shell=True) # Create list of files to be merged into one large file. with open("1000GMergeList.txt", "w") as f: wr = csv.writer(f, delimiter="\n") wr.writerow(chr_1000g_phase3_names) # Use plink to merge those files into one large file. subprocess.check_output([plink, '--merge-list', '1000GMergeList.txt', '--geno', '0.01', '--make-bed', '--out', '1000G_Phase3']) # Read in log file from merge. logfile = pd.DataFrame() # Format the log file into a pandas dataframe. with open('1000G_Phase3.log', 'r') as f: for line in f: logfile = pd.concat([logfile, pd.DataFrame([tuple(line.strip().split(" "))])], ignore_index=True) # If the logfile contains warnings, write a text file '1000G_MergeWarnings.txt' with the SNPs that threw # warnings. rsid_warnings = [] if logfile[0].str.contains('Warning:').any(): rsid_warnings = logfile.loc[logfile[0] == 'Warning:', 6].str.split("'", expand=True) rsid_warnings[1].dropna(how='any').to_csv('1000G_MergeWarnings.txt', sep='\t', header=False, index=False) # If the text file 1000G_MergeWarnings exists... if os.path.exists('1000G_MergeWarnings.txt'): # If merge warnings and missnps exist, exclude both from 1000G completely (there are plenty of other snps) if os.path.exists('1000G_Phase3-merge.missnp'): # Read in missnp file missnp = pd.read_csv('1000G_Phase3-merge.missnp', sep='\t', header=None) # Merge the warning snps with the missnps warnings_missnp = pd.concat([rsid_warnings, missnp], axis=0) # Drop duplicates and write to a file to be used in plink. warnings_missnp[1].dropna(how='any').to_csv('1000G_warnings_missnp.txt', sep='\t', header=False, index=False) # Remove these snps from plink files and create new plink files. for i in range(0, len(chr_1000g_phase3_names)): subprocess.check_output([plink, '--bfile', chr_1000g_phase3_names[i], '--exclude', '1000G_warnings_missnp.txt', '--geno', '0.01', '--make-bed', '--out', chr_1000g_phase3_names[i]]) # Remove old plink files. subprocess.call(rm + '*~', shell=True) # Retry the merge subprocess.check_output([plink, '--merge-list', '1000GMergeList.txt', '--geno', '0.01', '--make-bed', '--out', '1000G_Phase3']) # The merge should be successful this time, but the user should double check. print("Successfully merged 1000G, though you should double-check the log file to be sure.") else: # If only merge warnings exist, exclude from 1000G completely # Use plink to exclude the merge warning snps. for i in range(0, len(chr_1000g_phase3_names)): subprocess.check_output([plink, '--bfile', chr_1000g_phase3_names[i], '--exclude', '1000G_MergeWarnings.txt', '--geno', '0.01', '--make-bed', '--out', chr_1000g_phase3_names[i]]) # Remove old plink files. subprocess.call(rm + '*~', shell=True) # Try merge again. subprocess.check_output([plink, '--merge-list', '1000GMergeList.txt', '--geno', '0.01', '--make-bed', '--out', '1000G_Phase3']) # Merge should be successful this time, but the user should double check. print("Successfully merged 1000G, though you should double-check the log file to be sure.") # If only the missnps exist, remove them in 1000G. elif os.path.exists('1000G_Phase3-merge.missnp') and os.path.getsize('1000G_MergeWarnings.txt') == 0: # Use plink to remove the missnps for i in range(0, len(chr_1000g_phase3_names)): subprocess.check_output([plink, '--bfile', chr_1000g_phase3_names[i], '--exclude', '1000G_Phase3-merge.missnp', '--geno', '0.01', '--make-bed', '--out', chr_1000g_phase3_names[i]]) # Remove old plink files subprocess.call(rm + '*~', shell=True) # Retry the merge subprocess.check_output([plink, '--merge-list', '1000GMergeList.txt', '--geno', '0.01', '--make-bed', '--out', '1000G_Phase3']) # Merge should be successful this time, but the user should double check. print("Successfully merged 1000G, though you should double check the log file to be sure.") elif os.path.exists('1000G_Phase3.bim'): # Merge was successful. print("Successfully merged 1000G.") # Remove the per chromosome 1000G files, since they're just taking up space now. for i in range(0, len(chr_1000g_phase3_names)): subprocess.call(rm + chr_1000g_phase3_names[i] + '.*', shell=True) else: # If merge didn't work for reasons other than merge warnings and missnps. print(Fore.RED + Style.BRIGHT) sys.exit("Unable to merge 1000G chromosome files. You should try to merge them on your own.") # Merge of house data and 1000G # # Copying harmonized to 1000G files to this folder. shutil.copy2(os.path.join(orig_wd, harmonized_name + '.bed'), os.getcwd()) shutil.copy2(os.path.join(orig_wd, harmonized_name + '.bim'), os.getcwd()) shutil.copy2(os.path.join(orig_wd, harmonized_name + '.fam'), os.getcwd()) # Perform initial merge subprocess.check_output([plink, '--bfile', harmonized_name, '--bmerge', '1000G_Phase3', '--geno', '0.01', '--make-bed', '--out', harmonized_name + '_1000G']) # Read in log file to see if anything went wrong logfile = pd.DataFrame() # Concatenate log file to pandas dataframe. with open(harmonized_name + '_1000G.log', 'r') as f: for line in f: logfile = pd.concat([logfile, pd.DataFrame([tuple(line.strip().split(" "))])], ignore_index=True) # Check to see if the logfile has warnings if logfile[0].str.contains('Warning:').any(): # Identify the snps that made the warning. rsid_warnings = logfile.loc[logfile[0] == 'Warning:', 6].str.split("'", expand=True) # Put those SNPs in a file so we can remove them. rsid_warnings[1].dropna(how='any').to_csv(harmonized_name + '_1000G_MergeWarnings.txt', sep='\t', header=False, index=False) # If there are merge warnings if os.path.exists(harmonized_name + '_1000G_MergeWarnings.txt'): # and if there are triallelic snps that need to be flipped (missnp) if os.path.exists(harmonized_name + '_1000G-merge.missnp'): # If merge warnings and missnps exist, exclude warning snps from both 1000G and house dataset subprocess.check_output([plink, '--bfile', '1000G_Phase3', '--exclude', harmonized_name + '_1000G_MergeWarnings.txt', '--geno', '0.01', '--make-bed', '--out', '1000G_Phase3']) # Flip missnps in house dataset. subprocess.check_output([plink, '--bfile', harmonized_name, '--exclude', harmonized_name + '_1000G_MergeWarnings.txt', '--flip', harmonized_name + '_1000G-merge.missnp', '--geno', '0.01', '--make-bed', '--out', harmonized_name]) # Remove old files. subprocess.call(rm + '*~', shell=True) # Retry merge. subprocess.check_output([plink, '--bfile', harmonized_name, '--bmerge', '1000G_Phase3', '--geno', '0.01', '--make-bed', '--out', harmonized_name + '_1000G_merge2']) else: # If only mergewarnings exists, exclude warning snps from both 1000G and house dataset. subprocess.check_output([plink, '--bfile', '1000G_Phase3', '--exclude', harmonized_name + '_1000G_MergeWarnings.txt', '--geno', '0.01', '--make-bed', '--out', '1000G_Phase3']) # Exclude warning snps from house dataset. subprocess.check_output([plink, '--bfile', harmonized_name, '--exclude', harmonized_name + '_1000G_MergeWarnings.txt', '--geno', '0.01', '--make-bed', '--out', harmonized_name]) # Remove old plink files. subprocess.call(rm + '*~', shell=True) # Retry merge subprocess.check_output([plink, '--bfile', harmonized_name, '--bmerge', '1000G_Phase3', '--geno', '0.01', '--make-bed', '--out', harmonized_name + '_1000G_merge2']) # If only the missnps exist, flip them in the house dataset. elif os.path.exists(harmonized_name + '_1000G-merge.missnp') \ and not os.path.exists(harmonized_name + '_1000G_MergeWarnings.txt'): # Flip in house dataset. subprocess.check_output([plink, '--bfile', harmonized_name, '--flip', harmonized_name + '_1000G-merge.missnp', '--geno', '0.01', '--make-bed', '--out', harmonized_name]) # Remove old plink files. subprocess.call(rm + '*~', shell=True) # Retry the merge. subprocess.check_output([plink, '--bfile', harmonized_name, '--bmerge', '1000G_Phase3', '--geno', '0.01', '--make-bed', '--out', harmonized_name + '_1000G_merge2']) elif os.path.exists(harmonized_name + '_1000G.bim'): # If mergewarnings and missnips don't exist, then hopefully the merge happened successfully on the first # try. print("Successfully merged house dataset with 1000G on the first try, though you should double check. I " "can't predict every error.") # Copy successfully merged files to original working directory. shutil.copy2(harmonized_name + '_1000G.bed', orig_wd) shutil.copy2(harmonized_name + '_1000G.bim', orig_wd) shutil.copy2(harmonized_name + '_1000G.fam', orig_wd) shutil.copy2(harmonized_name + '_1000G.log', orig_wd) # Change back to original working directory. os.chdir(orig_wd) else: print(Fore.RED + Style.BRIGHT) sys.exit("The house dataset did not merge properly with 1000G, but not because of SNP merge " "warnings or SNPs that needed to be flipped. I'm sorry, you'll have to perform the merge on your " "own.") # If we had to perform a second merge because of one of the reasons above. if os.path.exists(harmonized_name + '_1000G_merge2.log'): # Read in log file logfile = pd.DataFrame() # Convert log file to pandas dataframe with open(harmonized_name + '_1000G_merge2.log', 'r') as f: for line in f: logfile = pd.concat([logfile, pd.DataFrame([tuple(line.strip().split(" "))])], ignore_index=True) # Check to see if the logfile has warnings if logfile[0].str.contains('Warning:').any(): # Figure out what snps caused the warnings. rsid_warnings = logfile.loc[logfile[0] == 'Warning:', 6].str.split("'", expand=True) # Write these warnings to a text file for plink to use. rsid_warnings[1].dropna(how='any').to_csv(harmonized_name + '_1000G_merge2_warnings.txt', sep='\t', header=False, index=False) # If warnings exist if os.path.exists(harmonized_name + '_1000G_merge2_warnings.txt'): # And if there are still triallelic snps if os.path.exists(harmonized_name + '_1000G_merge2-merge.missnp'): # Import missnp file to pandas missnp = pd.read_csv(harmonized_name + '_1000G_merge2-merge.missnp', sep='\t', header=None) # Merge this with merge warnings snps warnings_missnp = pd.concat([rsid_warnings, missnp], axis=0) # Write to text file for plink to use. warnings_missnp[1].dropna(how='any').to_csv(harmonized_name + '_1000G_merge2_warnings_missnp.txt', sep='\t', header=False, index=False) # Remove all of these snps from 1000G dataset. subprocess.check_output([plink, '--bfile', '1000G_Phase3', '--exclude', harmonized_name + '_1000G_merge2_warnings_missnp.txt', '--geno', '0.01', '--make-bed', '--out', '1000G_Phase3']) # Remove all of these snps from house dataset. subprocess.check_output([plink, '--bfile', harmonized_name, '--exclude', harmonized_name + '_1000G_merge2_warnings_missnp.txt', '--geno', '0.01', '--make-bed', '--out', harmonized_name]) # Remove old plink files. subprocess.call(rm + '*~', shell=True) # Try merge a third time. subprocess.check_output([plink, '--bfile', harmonized_name, '--bmerge', '1000G_Phase3', '--geno', '0.01', '--make-bed', '--out', harmonized_name + '_1000G_merge3']) else: # If only merge warnings still exist, exclude from both 1000G and house dataset. # Exclude from 1000G dataset. subprocess.check_output([plink, '--bfile', '1000G_Phase3', '--exclude', harmonized_name + '_1000G_merge2_warnings.txt', '--geno', '0.01', '--make-bed', '--out', '1000G_Phase3']) # Exclude from house dataset subprocess.check_output([plink, '--bfile', harmonized_name, '--exclude', harmonized_name + '_1000G_merge2_warnings.txt', '--geno', '0.01', '--make-bed', '--out', harmonized_name]) # Remove old plink files. subprocess.call(rm + '*~', shell=True) # Retry merge a third time. subprocess.check_output([plink, '--bfile', harmonized_name, '--bmerge', '1000G_Phase3', '--geno', '0.01', '--make-bed', '--out', harmonized_name + '_1000G_merge3']) # If only the missnps still exist, remove them in both datasets. elif os.path.exists(harmonized_name + '_1000G_merge2-merge.missnp') \ and not os.path.exists(harmonized_name + '_1000G_merge2_warnings.txt'): # Exclude from house dataset. subprocess.check_output([plink, '--bfile', harmonized_name, '--exclude', harmonized_name + '_1000G_merge2-merge.missnp', '--geno', '0.01', '--make-bed', '--out', harmonized_name]) # Exclude from 1000G dataset. subprocess.check_output([plink, '--bfile', '1000G_Phase3', '--exclude', harmonized_name + '_1000G_merge2-merge.missnp', '--geno', '0.01', '--make-bed', '--out', '1000G_Phase3']) # Remove old files. subprocess.call(rm + '*~', shell=True) # Retry merge a third time. subprocess.check_output([plink, '--bfile', harmonized_name, '--bmerge', '1000G_Phase3', '--geno', '0.01', '--make-bed', '--out', harmonized_name + '_1000G_merge3']) # If we don't find warnings or missnps elif os.path.exists(harmonized_name + '_1000G_merge2.bim'): # Merge should have happened, but user should check. print("Successfully merged house dataset with 1000G on 2nd try, though you should double check. I " "can't predict every error.") # Copy merged files to original working directory shutil.copy2(harmonized_name + '_1000G_merge2.bed', orig_wd) shutil.copy2(harmonized_name + '_1000G_merge2.bim', orig_wd) shutil.copy2(harmonized_name + '_1000G_merge2.fam', orig_wd) shutil.copy2(harmonized_name + '_1000G_merge2.log', orig_wd) # Change back to original working directory. os.chdir(orig_wd) # If the merge didn't happen, but not because of merge warnings or missnps. else: print(Fore.RED + Style.BRIGHT) sys.exit("House dataset and 1000G did not successfully merge 2nd time, though not because of merge " "warnings or SNPs that needed to be flipped. You'll have to perform the merge on your own. " "I'm sorry!") if os.path.exists(harmonized_name + '_1000G_merge3.log'): # If merge 3 log exists, read it into pandas dataframe. logfile = pd.DataFrame() with open(harmonized_name + '_1000G_merge3.log', 'r') as f: for line in f: logfile = pd.concat([logfile, pd.DataFrame([tuple(line.strip().split(" "))])], ignore_index=True) # If the bim file doesn't exist, try to figure out why. if not os.path.exists(harmonized_name + '_1000G_merge3.bim'): # If logfile still has warnings, or there are still missnps, the user will need to identify them and # take care of them manually. if logfile[0].str.contains('Warning:').any() \ and os.path.exists(harmonized_name + '_1000G_merge3-merge.missnp'): print(Fore.RED + Style.BRIGHT) sys.exit("I'm sorry, the logfile still has warnings, even after removing snps that " "threw errors in the first two tries. There are also still snps with 3+ variants present, " "even after flipping some and removing the ones that the flip didn't solve. You'll have " "to manually deal with these using the merge3 log and the merge3-merge.missnp file.") # Check to see if the logfile still has warnings elif logfile[0].str.contains('Warning:').any(): print(Fore.RED + Style.BRIGHT) sys.exit("I'm sorry, the logfile still has warnings, even after removing snps that threw errors in " "the first two tries. You'll have to manually deal with these using the merge3 log file.") # If triallelic snps still exist elif os.path.exists(harmonized_name + '_1000G_merge3-merge.missnp'): print(Fore.RED + Style.BRIGHT) sys.exit("I'm sorry, there are still snps with 3+ variants present, even after flipping some and " "removing the ones that the flip didn't solve. You'll have to deal with these manually " "using the merge3-merge.missnp file") # If the bim file exists, then the merge should have happened correctly. if os.path.exists(harmonized_name + '_1000G_merge3.bim'): print("House dataset and 1000G merged correctly on 3rd try. Though you should double-check the log " "file, I can't predict every error.") # Copy files to original working directory. shutil.copy2(harmonized_name + '_1000G_merge3.bed', orig_wd) shutil.copy2(harmonized_name + '_1000G_merge3.bim', orig_wd) shutil.copy2(harmonized_name + '_1000G_merge3.fam', orig_wd) shutil.copy2(harmonized_name + '_1000G_merge3.log', orig_wd) # Change back to original working directory. os.chdir(orig_wd) # End the program if the user did not harmonize first. elif merge_proceed in ('n', 'no'): sys.exit("Please harmonize your genotypes with 1000G first.") # End the program if the user did not give a correctly formatted answer. else: sys.exit('Please answer yes or no.')
def het(geno_name): # Identifies individuals with extreme heterozygosity values (more than +- 3 SD) # Getting extra required modules try: import pandas as pd except (ImportError, ModuleNotFoundError): import genodownload genodownload.getpandas() import pandas as pd try: import numpy as np except (ImportError, ModuleNotFoundError): import genodownload genodownload.getnumpy() import numpy as np import subprocess # Use plink to calculate the heterozygosity, paying attention to geno and mind. subprocess.check_output([ plink, '--bfile', geno_name, '--geno', '0.1', '--mind', '0.1', '--het', '--out', geno_name ]) # Read het file into pandas het_file = pd.read_csv(geno_name + '.het', sep='\s+', header=0) # Create new column with formula: (N(NM)-O(HOM))/N(NM) het_file['HET'] = (het_file['N(NM)'] - het_file['O(HOM)']) / het_file['N(NM)'] # Getting standard deviation and average of HET column het_sd = np.std(het_file['HET']) het_avg = np.mean(het_file['HET']) # Add label 'keep' to people within 3*SD of the average het value, give 'remove' to everyone else. het_file['HET_Filter'] = np.where( (het_file['HET'] > het_avg - 3 * het_sd) & (het_file['HET'] < het_avg + 3 * het_sd), 'Keep', 'Remove') # Write this file so the user has it. het_file.to_csv(geno_name + '.het', sep='\t', header=True, index=False) # Make a list of the people who pass the filter. het_keep = het_file[het_file['HET_Filter'] == 'Keep'] # Write this file so that we can use it later to filter people out. het_keep[['FID', 'IID']].to_csv(geno_name + '_KeptAfterHetCheck.txt', sep='\t', header=False, index=False) # Make a list of the people who fail the filter. het_rem = het_file[het_file['HET_Filter'] == 'Remove'] # Write this to file so we have the record. het_rem.to_csv(geno_name + '_RemAfterHetCheck.txt', sep='\t', header=True, index=False) # Make new plink file with people passing het check. subprocess.check_output([ plink, '--bfile', geno_name, '--keep', geno_name + '_KeptAfterHetCheck.txt', '--geno', '0.1', '--make-bed', '--out', geno_name + '_HetChecked' ]) print( "Done. Your new file of people with non-extreme heterozygosity values will be called " + geno_name + "_HetChecked")
import gzip import shutil from os.path import expanduser try: import argparse except (ImportError, ModuleNotFoundError): import genodownload genodownload.getargparse() import argparse try: import pandas as pd except (ImportError, ModuleNotFoundError): import genodownload genodownload.getpandas() import pandas as pd try: import numpy as np except (ImportError, ModuleNotFoundError): import genodownload genodownload.getnumpy() import numpy as np try: import colorama from colorama import init, Fore, Style init() except ImportError: import genodownload