#!/users/nanje/miniconda3/bin/python
#-*- coding: utf-8 -*-
import os, sys, re
from measureAcc.measureAccuracy import CookHLA_measureAcc
if __name__ == '__main__':
"""
measureAccuracy.py (v3.5)
- takes (1) imputed result and (2) answer file in '*.Marked.chped' format
- takes Allele group file.
(Alleles classifed based on the nucleotide sequence of exon 2,3,(4) of each HLA gene)
- calculates accuracy.
"""
[_answer, _imputed, _out] = sys.argv[1:]
CookHLA_measureAcc(_answer, _imputed, _out)
def __init__(self,
idx_process,
MHC,
_reference,
_out,
_hg,
_window,
_overlap,
_ne,
_nthreads,
_AdaptiveGeneticMap,
_Average_Erate,
_LINKAGE2BEAGLE,
_BEAGLE2LINKAGE,
_BEAGLE2VCF,
_VCF2BEAGLE,
_PLINK,
_BEAGLE5,
_answer=None,
f_save_intermediates=False,
_HapMap_Map=None,
f_measureAcc_v2=False):
### Class variables
# General
self.idx_process = idx_process
self.__save_intermediates = f_save_intermediates
self.FLAG_AdaptiveGeneticMap = _AdaptiveGeneticMap and _Average_Erate # (***) Deciding whether to use Adaptive genetic map or not.
# Prefixes
self.OUTPUT_dir = os.path.dirname(_out)
self.OUTPUT_dir_ref = join(self.OUTPUT_dir,
os.path.basename(_reference))
self.OUTPUT_dir_GM = join(
self.OUTPUT_dir, os.path.basename(
_AdaptiveGeneticMap)) if self.FLAG_AdaptiveGeneticMap else None
# self.HLA_IMPUTED_Result_MHC = join(os.path.dirname(MHC), "HLA_IMPUTED_Result.{}".format(os.path.basename(MHC)))
self.HLA_IMPUTED_Result_MHC = MHC + '.HLA_IMPUTATION_OUT'
# Result
self.raw_IMP_Reuslt = None
self.HLA_IMPUTATION_OUT = None # Final Imputation output ('*.imputed.alleles').
self.accuracy = None
# Dependencies
self.LINKAGE2BEAGLE = _LINKAGE2BEAGLE
self.BEAGLE2LINKAGE = _BEAGLE2LINKAGE
self.BEAGLE2VCF = _BEAGLE2VCF
self.VCF2BEAGLE = _VCF2BEAGLE
self.PLINK = _PLINK
self.BEAGLE5 = _BEAGLE5
# Adaptive Genetic Map
self.__AGM__ = _AdaptiveGeneticMap
self.__AVER__ = _Average_Erate
# created in 'CONVERT_IN'
self.refined_REF_markers = None # used in 'CONVERT_OUT'
self.refined_Genetic_Map = None # used in 'IMPUTE'
self.GCchangeBGL = None # used in 'CONVERT_OUT'
# HapMap_Map
if _HapMap_Map:
self.HapMap_Map = _HapMap_Map
self.OUTPUT_dir_GM = join(self.OUTPUT_dir,
os.path.basename(_HapMap_Map))
###### < Main - 'CONVERT_IN', 'IMPUTE', 'CONVERT_OUT' > ######
### (1) CONVERT_IN
# self.CONVERT_IN(MHC, _reference, _out, _hg, _aver_erate=self.__AVER__, _Genetic_Map=self.__AGM__)
[MHC_QC_VCF,
REF_PHASED_VCF] = self.CONVERT_IN(MHC,
_reference,
_out,
_hg,
_Genetic_Map=self.__AGM__)
# [Temporary Hard coding]
# MHC_QC_VCF = "/Users/wansun/Git_Projects/CookHLA/tests/_3_CookHLA/20190605_onlyAGM/_3_HM_CEU_T1DGC_REF.MHC.QC.vcf"
# REF_PHASED_VCF = "/Users/wansun/Git_Projects/CookHLA/tests/_3_CookHLA/20190605_onlyAGM/T1DGC_REF.phased.vcf"
# self.refined_Genetic_Map = '/Users/wansun/Git_Projects/CookHLA/tests/_3_CookHLA/20190605_onlyAGM/CEU_T1DGC.mach_step.avg.clpsB.refined.map'
# print("CONVERT_IN :\n{}\n{}".format(MHC_QC_VCF, REF_PHASED_VCF))
### (2) IMPUTE
if _HapMap_Map:
self.raw_IMP_Reuslt = self.IMPUTE_HapMap_Map(
_out, MHC_QC_VCF, REF_PHASED_VCF, _window, _overlap, _ne,
_nthreads)
else:
self.raw_IMP_Reuslt = self.IMPUTE(_out, MHC_QC_VCF, REF_PHASED_VCF,
self.__AVER__,
self.refined_Genetic_Map,
_window, _overlap, _ne,
_nthreads)
# [Temporary Hard coding]
# self.raw_IMP_Reuslt = '/Users/wansun/Git_Projects/CookHLA/tests/_3_CookHLA/20190605_onlyAGM/_3_HM_CEU_T1DGC_REF.QC.imputation_out.vcf'
# self.refined_REF_markers = '/Users/wansun/Git_Projects/CookHLA/tests/_3_CookHLA/20190605_onlyAGM/T1DGC_REF.refined.markers'
# self.GCchangeBGL = '/Users/wansun/Git_Projects/CookHLA/tests/_3_CookHLA/20190605_onlyAGM/_3_HM_CEU_T1DGC_REF.MHC.QC.GCchange.bgl'
# print("raw Imputed Reuslt :\n{}".format(self.raw_IMP_Reuslt))
### (3) CONVERT_OUT
self.HLA_IMPUTATION_OUT = self.CONVERT_OUT(MHC, _reference, _out,
self.raw_IMP_Reuslt)
# [Temporary Hard coding]
# self.IMP_Result = '/Users/wansun/Git_Projects/CookHLA/tests/_3_CookHLA/20190605_onlyAGM/HLA_IMPUTED_Result._3_HM_CEU_T1DGC_REF.MHC.imputed.alleles'
# print("\n\nImputation Result : {}".format(self.IMP_Result))
###### < Get Accuracy > ######
if bool(_answer):
print(std_MAIN_PROCESS_NAME +
"Calculating accuracy of each HLA gene. (answer: '{}')".
format(_answer))
if not os.path.exists(_answer):
print(
std_WARNING_MAIN_PROCESS_NAME +
"Given answer file doesn't exist. Please check '--answer/-an' argument again.\n"
"Skipping calculating imputation accuracy.")
elif os.path.getsize(_answer) == 0:
print(
std_WARNING_MAIN_PROCESS_NAME +
"Given answer file doesn't have any content. Please check '--answer/-an' argument again.\n"
"Skipping calculating imputation accuracy.")
else:
if f_measureAcc_v2:
# measureAcc_v2
self.accuracy = measureAccuracy(
_answer,
self.HLA_IMPUTATION_OUT,
'all',
outfile=self.HLA_IMPUTATION_OUT + '.accuracy',
__only4digits=True)
else:
# measureAcc_v3.5
measureAcc_start = time()
t = CookHLA_measureAcc(_answer, self.HLA_IMPUTATION_OUT,
self.HLA_IMPUTATION_OUT)
self.accuracy = t.accuracy
measureAcc_end = time()
measureAcc_time = (measureAcc_end - measureAcc_start) / 60
print("\nAccuracy : {}".format(self.accuracy))
print("measureAccuracy time: {}(min)\n".format(
measureAcc_time))
###### < Get Accuracy > ######
# Reference panel
RUN_Bash('rm {}'.format(REF_PHASED_VCF))
RUN_Bash('rm {}'.format(self.OUTPUT_dir_ref + '.GCchange.bgl.phased'))
RUN_Bash('rm {}'.format(self.OUTPUT_dir_ref + '.GCchange.markers'))
if self.FLAG_AdaptiveGeneticMap:
RUN_Bash('rm {}'.format(self.refined_Genetic_Map))
def __init__(self,
idx_process,
MHC,
_reference,
_out,
_hg,
_nthreads,
_AdaptiveGeneticMap,
_Average_Erate,
_LINKAGE2BEAGLE,
_BEAGLE2LINKAGE,
_BEAGLE2VCF,
_VCF2BEAGLE,
_PLINK,
_BEAGLE4,
_CSH,
_answer=None,
f_save_intermediates=False,
_MultP=1,
_given_prephased=None,
f_prephasing=False,
f_remove_raw_IMP_results=False,
f_measureAcc_v2=False):
### General
self.idx_process = idx_process
self.__save_intermediates = f_save_intermediates
self.FLAG_AdaptiveGeneticMap = _AdaptiveGeneticMap and _Average_Erate # (***) Deciding whether to use Adaptive genetic map or not.
# Prefixes
self.OUTPUT_dir = os.path.dirname(_out)
self.OUTPUT_dir_ref = join(self.OUTPUT_dir,
os.path.basename(_reference))
self.OUTPUT_dir_GM = join(
self.OUTPUT_dir, os.path.basename(
_AdaptiveGeneticMap)) if self.FLAG_AdaptiveGeneticMap else None
# Result
self.Exon234_Panel = None
self.dict_ExonN_Panel = {_exonN: None for _exonN in __EXON__}
self.dict_ExonN_AGM = {_exonN: None for _exonN in __EXON__}
self.dict_IMP_Result = {
_exonN: {_overlap: None
for _overlap in __overlap__}
for _exonN in __EXON__
}
self.accuracy = None
self.HLA_IMPUTATION_OUT = None
self.dict_DOUBLED_PHASED_RESULT = {_exonN: None for _exonN in __EXON__}
self.dict_REF_PHASED_VCF = {_exonN: None for _exonN in __EXON__}
# Dependencies
self.LINKAGE2BEAGLE = _LINKAGE2BEAGLE
self.BEAGLE2LINKAGE = _BEAGLE2LINKAGE
self.BEAGLE2VCF = _BEAGLE2VCF
self.VCF2BEAGLE = _VCF2BEAGLE
self.PLINK = _PLINK
self.BEAGLE4 = _BEAGLE4
# created in 'CONVERT_IN'
# self.refined_REF_markers = None # used in 'CONVERT_OUT'
# self.refined_Genetic_Map = None # used in 'IMPUTE'
# self.GCchangeBGL = None # used in 'CONVERT_OUT'
# Adaptive Genetic Map
self.__AGM__ = _AdaptiveGeneticMap if self.FLAG_AdaptiveGeneticMap else None
self.__AVER__ = _Average_Erate if self.FLAG_AdaptiveGeneticMap else None
###### < Reference panel for Exon 2, 3, 4 > ######
multiple_panels = HLA_MultipleRefs(_reference,
self.OUTPUT_dir_ref,
_hg,
self.BEAGLE2LINKAGE,
self.BEAGLE2VCF,
self.PLINK,
_MultP=_MultP,
__AGM__=self.__AGM__,
_out_AGM=self.OUTPUT_dir_GM)
self.dict_ExonN_Panel = multiple_panels.ExonN_Panel
self.Exon234_Panel = multiple_panels.EXON234_Panel
self.dict_ExonN_AGM = multiple_panels.ExonN_AGM if self.FLAG_AdaptiveGeneticMap else {
_exonN: None
for _exonN in __EXON__
}
# [Temporary Hard-coding]
# self.Exon234_Panel = '/Users/wansun/Git_Projects/CookHLA/tests/_3_CookHLA/20190716_BOTH/T1DGC_REF.exon234'
#
# self.dict_ExonN_Panel['exon2'] = '/Users/wansun/Git_Projects/CookHLA/tests/_3_CookHLA/20190716_BOTH/T1DGC_REF.exon2'
# self.dict_ExonN_Panel['exon3'] = '/Users/wansun/Git_Projects/CookHLA/tests/_3_CookHLA/20190716_BOTH/T1DGC_REF.exon3'
# self.dict_ExonN_Panel['exon4'] = '/Users/wansun/Git_Projects/CookHLA/tests/_3_CookHLA/20190716_BOTH/T1DGC_REF.exon4'
#
# self.dict_ExonN_AGM['exon2'] = '/Users/wansun/Git_Projects/CookHLA/tests/_3_CookHLA/20190716_BOTH/CEU_T1DGC.mach_step.avg.clpsB.exon2.txt'
# self.dict_ExonN_AGM['exon3'] = '/Users/wansun/Git_Projects/CookHLA/tests/_3_CookHLA/20190716_BOTH/CEU_T1DGC.mach_step.avg.clpsB.exon3.txt'
# self.dict_ExonN_AGM['exon4'] = '/Users/wansun/Git_Projects/CookHLA/tests/_3_CookHLA/20190716_BOTH/CEU_T1DGC.mach_step.avg.clpsB.exon4.txt'
###### < Main - 'CONVERT_IN', 'IMPUTE', 'CONVERT_OUT' > ######
### (1) CONVERT_IN
IMPUTATION_INPUT = self.CONVERT_IN(MHC,
self.Exon234_Panel,
_out,
_hg,
_given_prephased=_given_prephased,
f_prephasing=f_prephasing)
# Only one time of pre-phasing with Exon234 reference panel.
### (2) Imputation
if _MultP == 1:
imputation_serial_start = time()
## Serial implementation of main.
for _exonN in __EXON__:
for _overlap in __overlap__:
self.dict_IMP_Result[_exonN][_overlap] = \
self.IMPUTE(MHC, _out, IMPUTATION_INPUT, self.dict_ExonN_Panel[_exonN] + '.phased.vcf',
_overlap, _exonN, _nthreads, self.__AVER__, self.dict_ExonN_AGM[_exonN], f_prephasing=f_prephasing)
imputation_serial_end = time()
imputation_serial_time = (imputation_serial_end -
imputation_serial_start) / 60
print(std_MAIN_PROCESS_NAME +
"Total imputation time of Serial implementation: {}(min)\n".
format(imputation_serial_time))
else:
## Parallel implementation of main.
imputation_parallel_start = time()
pool = mp.Pool(processes=_MultP if _MultP <= 9 else 9)
dict_Pool = {
_exonN: {
_overlap: pool.apply_async(
self.IMPUTE,
(MHC, _out, IMPUTATION_INPUT,
self.dict_ExonN_Panel[_exonN] + '.phased.vcf',
_overlap, _exonN, _nthreads, self.__AVER__,
self.dict_ExonN_AGM[_exonN], f_prephasing))
for _overlap in __overlap__
}
for _exonN in __EXON__
}
pool.close()
pool.join()
for _exonN in __EXON__:
for _overlap in __overlap__:
self.dict_IMP_Result[_exonN][_overlap] = dict_Pool[_exonN][
_overlap].get()
imputation_parallel_end = time()
imputation_parallel_time = (imputation_parallel_end -
imputation_parallel_start) / 60
print(
std_MAIN_PROCESS_NAME +
"Total imputation time of Parallel implementation (with {} core(s)): {}(min)\n"
.format(_MultP, imputation_parallel_time))
self.idx_process += 1
# [Temporary Hard-coding]
# self.dict_IMP_Result['exon2'][3000] = '/home/wanson/Git_Projects/CookHLA/tests/_3_CookHLA/20190731_MM_AGM/HM_CEU_T1DGC_REF.MM.AGM.noprephasing.exon2.overlap3000.MHC.QC.double.imputation_out.vcf'
# self.dict_IMP_Result['exon2'][4000] = '/home/wanson/Git_Projects/CookHLA/tests/_3_CookHLA/20190731_MM_AGM/HM_CEU_T1DGC_REF.MM.AGM.noprephasing.exon2.overlap4000.MHC.QC.double.imputation_out.vcf'
# self.dict_IMP_Result['exon2'][5000] = '/home/wanson/Git_Projects/CookHLA/tests/_3_CookHLA/20190731_MM_AGM/HM_CEU_T1DGC_REF.MM.AGM.noprephasing.exon2.overlap5000.MHC.QC.double.imputation_out.vcf'
# self.dict_IMP_Result['exon3'][3000] = '/home/wanson/Git_Projects/CookHLA/tests/_3_CookHLA/20190731_MM_AGM/HM_CEU_T1DGC_REF.MM.AGM.noprephasing.exon3.overlap3000.MHC.QC.double.imputation_out.vcf'
# self.dict_IMP_Result['exon3'][4000] = '/home/wanson/Git_Projects/CookHLA/tests/_3_CookHLA/20190731_MM_AGM/HM_CEU_T1DGC_REF.MM.AGM.noprephasing.exon3.overlap4000.MHC.QC.double.imputation_out.vcf'
# self.dict_IMP_Result['exon3'][5000] = '/home/wanson/Git_Projects/CookHLA/tests/_3_CookHLA/20190731_MM_AGM/HM_CEU_T1DGC_REF.MM.AGM.noprephasing.exon3.overlap5000.MHC.QC.double.imputation_out.vcf'
# self.dict_IMP_Result['exon4'][3000] = '/home/wanson/Git_Projects/CookHLA/tests/_3_CookHLA/20190731_MM_AGM/HM_CEU_T1DGC_REF.MM.AGM.noprephasing.exon4.overlap3000.MHC.QC.double.imputation_out.vcf'
# self.dict_IMP_Result['exon4'][4000] = '/home/wanson/Git_Projects/CookHLA/tests/_3_CookHLA/20190731_MM_AGM/HM_CEU_T1DGC_REF.MM.AGM.noprephasing.exon4.overlap4000.MHC.QC.double.imputation_out.vcf'
# self.dict_IMP_Result['exon4'][5000] = '/home/wanson/Git_Projects/CookHLA/tests/_3_CookHLA/20190731_MM_AGM/HM_CEU_T1DGC_REF.MM.AGM.noprephasing.exon4.overlap5000.MHC.QC.double.imputation_out.vcf'
### (3) CONVERT_OUT
self.HLA_IMPUTATION_OUT = self.CONVERT_OUT(self.dict_IMP_Result,
MHC + '.HLA_IMPUTATION_OUT',
_CSH,
f_prephasing=f_prephasing)
print(std_MAIN_PROCESS_NAME +
'IMPUTATION_OUT:\n{}'.format(self.HLA_IMPUTATION_OUT))
## Acquiring accuracy
if bool(_answer):
print(std_MAIN_PROCESS_NAME +
"Calculating accuracy of each HLA gene. (answer: '{}')".
format(_answer))
if not os.path.exists(_answer):
print(
std_WARNING_MAIN_PROCESS_NAME +
"Given answer file doesn't exist. Please check '--answer/-an' argument again.\n"
"Skipping calculating imputation accuracy.")
elif os.path.getsize(_answer) == 0:
print(
std_WARNING_MAIN_PROCESS_NAME +
"Given answer file doesn't have any content. Please check '--answer/-an' argument again.\n"
"Skipping calculating imputation accuracy.")
else:
if f_measureAcc_v2:
# measureAcc_v2
self.accuracy = measureAccuracy(
_answer,
self.HLA_IMPUTATION_OUT,
'all',
outfile=self.HLA_IMPUTATION_OUT + '.accuracy',
__only4digits=True)
else:
# measureAcc_v3.5
measureAcc_start = time()
t = CookHLA_measureAcc(_answer, self.HLA_IMPUTATION_OUT,
self.HLA_IMPUTATION_OUT)
self.accuracy = t.accuracy
measureAcc_end = time()
measureAcc_time = (measureAcc_end - measureAcc_start) / 60
print("\nAccuracy : {}".format(self.accuracy))
print("measureAccuracy time: {}(min)\n".format(
measureAcc_time))
### General Removal
if not self.__save_intermediates:
# 'Exon234 panel'
RUN_Bash('rm {}'.format(self.Exon234_Panel + '.bed'))
RUN_Bash('rm {}'.format(self.Exon234_Panel + '.bim'))
RUN_Bash('rm {}'.format(self.Exon234_Panel + '.fam'))
RUN_Bash('rm {}'.format(self.Exon234_Panel + '.FRQ.frq'))
RUN_Bash('rm {}'.format(self.Exon234_Panel + '.markers'))
RUN_Bash('rm {}'.format(self.Exon234_Panel + '.bgl.phased'))
RUN_Bash('rm {}'.format(self.Exon234_Panel + '.GCchange.markers'))
RUN_Bash('rm {}'.format(self.Exon234_Panel +
'.GCchange.bgl.phased'))
RUN_Bash('rm {}'.format(self.Exon234_Panel + '.phased.vcf'))
RUN_Bash(
'rm {}'.format(self.Exon234_Panel +
'.refined.markers')) # only in Exon234 panel
# 'Exon 2,3,4 panel'
for _exonN in __EXON__:
RUN_Bash('rm {}'.format(self.dict_ExonN_Panel[_exonN] +
'.bed'))
RUN_Bash('rm {}'.format(self.dict_ExonN_Panel[_exonN] +
'.bim'))
RUN_Bash('rm {}'.format(self.dict_ExonN_Panel[_exonN] +
'.fam'))
RUN_Bash('rm {}'.format(self.dict_ExonN_Panel[_exonN] +
'.FRQ.frq'))
RUN_Bash('rm {}'.format(self.dict_ExonN_Panel[_exonN] +
'.markers'))
RUN_Bash('rm {}'.format(self.dict_ExonN_Panel[_exonN] +
'.bgl.phased'))
RUN_Bash('rm {}'.format(self.dict_ExonN_Panel[_exonN] +
'.GCchange.markers'))
RUN_Bash('rm {}'.format(self.dict_ExonN_Panel[_exonN] +
'.GCchange.bgl.phased'))
RUN_Bash('rm {}'.format(self.dict_ExonN_Panel[_exonN] +
'.phased.vcf'))
# 'Exon 2,3,4 AGM'
RUN_Bash('rm {}'.format(multiple_panels.EXON234_AGM))
for _exonN in __EXON__:
RUN_Bash('rm {}'.format(self.dict_ExonN_AGM[_exonN]))
# # 'CONVERT_IN'
# RUN_Bash('rm {}'.format(MHC + '.QC.nopheno.ped'))
# RUN_Bash('rm {}'.format(MHC + '.QC.dat'))
# 'CONVERT_OUT'
for _exonN in __EXON__:
for _overlap in __overlap__:
for _hla in HLA_names:
RUN_Bash('rm {}'.format(
self.dict_IMP_Result[_exonN][_overlap] +
'.HLA_{}'.format(_hla)))
if f_remove_raw_IMP_results:
RUN_Bash('rm {}'.format(
self.dict_IMP_Result[_exonN][_overlap]))
RUN_Bash(
'rm {}'.format(self.dict_IMP_Result[_exonN]
[_overlap].rstrip('.vcf') +
'.log'))
#!/users/nanje/miniconda3/bin/python
import sys
sys.path.append('/scratch3/users/nanje/MHC-Imputation-Accuracy/templates')
from measureAcc.measureAccuracy import CookHLA_measureAcc
import argparse,time
parser = argparse.ArgumentParser()
parser.add_argument("--answer_file", default="${answer_file}", help="")
parser.add_argument("--bglphased", default="${bglphased}", help="")
parser.add_argument("--output", default="${output}", help="")
args = parser.parse_args()
if __name__ == '__main__':
CookHLA_measureAcc(args.answer_file, args.bglphased, args.output)
#!/users/nanje/miniconda3/bin/python3.6
import sys
sys.path.append('/scratch3/users/nanje/MHC-Imputation-Accuracy/templates')
from measureAcc.measureAccuracy import CookHLA_measureAcc
import argparse, time
parser = argparse.ArgumentParser()
parser.add_argument("--answer_file", default="${answer_file}", help="")
parser.add_argument("--alleles", default="${alleles}", help="")
parser.add_argument("--output", default="${output}", help="")
args = parser.parse_args()
if __name__ == '__main__':
CookHLA_measureAcc(args.answer_file, args.alleles, args.output)