def plot_QTdb_filtered_Result_with_syntax_filter(RFfolder,
TargetRecordList,
ResultFilterType,
showExpertLabels=False):
# exit
# ==========================
# plot prediction result
# ==========================
reslist = getresultfilelist(RFfolder)
qtdb = QTloader()
non_result_extensions = ['out', 'json', 'log', 'txt']
for fi, fname in enumerate(reslist):
# block *.out
file_extension = fname.split('.')[-1]
if file_extension in non_result_extensions:
continue
print 'file name:', fname
currecname = os.path.split(fname)[-1]
print currecname
#if currecname == 'result_sel820':
#pdb.set_trace()
if TargetRecordList is not None:
if currecname not in TargetRecordList:
continue
# load signal and reslist
with open(fname, 'r') as fin:
(recID, reslist) = pickle.load(fin)
# filter result of QT
resfilter = ResultFilter(reslist)
if len(ResultFilterType) >= 1 and ResultFilterType[0] == 'G':
reslist = resfilter.group_local_result(cp_del_thres=1)
reslist_syntax = reslist
if len(ResultFilterType) >= 2 and ResultFilterType[1] == 'S':
reslist_syntax = resfilter.syntax_filter(reslist)
# empty signal result
#if reslist is None or len(reslist) == 0:
#continue
#pdb.set_trace()
sigstruct = qtdb.load(recID)
if showExpertLabels == True:
# Expert Label AdditionalPlot
ExpertRes = qtdb.getexpertlabeltuple(recID)
ExpertPoslist = map(lambda x: x[0], ExpertRes)
AdditionalPlot = [
['kd', 'Expert Labels', ExpertPoslist],
]
else:
AdditionalPlot = None
# plot res
#resploter = ECGResultPloter(sigstruct['sig'],reslist)
#resploter.plot(plotTitle = 'QT database',plotShow = True,plotFig = 2)
# syntax_filter
resploter_syntax = ECGResultPloter(sigstruct['sig'], reslist_syntax)
resploter_syntax.plot(plotTitle='QT Record {}'.format(recID),
plotShow=True,
AdditionalPlot=AdditionalPlot)
def plot_QTdb_filtered_Result_with_syntax_filter():
# exit
RFfolder = os.path.join(\
projhomepath,\
'TestResult',\
'pc',\
'r5')
TargetRecordList = [
'result_sel39',
'result_sel41',
'result_sel48',
] #'sel38','sel42','result_sel821','result_sel14046']
# ==========================
# plot prediction result
# ==========================
reslist = glob.glob(os.path.join(\
RFfolder,'*'))
qtdb = QTloader()
non_result_extensions = ['out', 'json', 'log']
for fi, fname in enumerate(reslist):
# block *.out
file_extension = fname.split('.')[-1]
if file_extension in non_result_extensions:
continue
print 'file name:', fname
currecname = os.path.split(fname)[-1]
print currecname
#if currecname == 'result_sel820':
#pdb.set_trace()
if currecname not in TargetRecordList:
pass
continue
# load signal and reslist
with open(fname, 'r') as fin:
(recID, reslist) = pickle.load(fin)
# filter result of QT
resfilter = ResultFilter(reslist)
reslist = resfilter.group_local_result(cp_del_thres=1)
reslist_syntax = resfilter.syntax_filter(reslist)
# empty signal result
#if reslist is None or len(reslist) == 0:
#continue
#pdb.set_trace()
sigstruct = qtdb.load(recID)
# plot res
#resploter = ECGResultPloter(sigstruct['sig'],reslist)
#resploter.plot(plotTitle = 'QT database',plotShow = True,plotFig = 2)
# syntax_filter
resploter_syntax = ECGResultPloter(sigstruct['sig'], reslist_syntax)
resploter_syntax.plot(plotTitle='QT database syntax_filter',
plotShow=True)
def plotMITdbTestResult():
RFfolder = os.path.join(\
projhomepath,\
'TestResult',\
'pc',\
'r3')
TargetRecordList = [
'sel38',
'sel42',
]
# ==========================
# plot prediction result
# ==========================
reslist = glob.glob(os.path.join(\
RFfolder,'*'))
for fi, fname in enumerate(reslist):
# block *.out
# filter file name
if fname[-4:] == '.out' or '.json' in fname:
continue
currecname = os.path.split(fname)[-1]
if currecname not in TargetRecordList:
pass
if not currecname.startswith('result'):
continue
print 'processing file', fname, '...'
with open(fname, 'r') as fin:
(recID, reslist) = pickle.load(fin)
print 'pickle file loaded.'
# load signal from MITdb
print 'loading signal data from MITdb...'
mitdb = MITdbLoader()
rawsig = mitdb.load(recID)
print 'signal loaded.'
# filter result list
resfilter = ResultFilter(reslist)
reslist = resfilter.groupresult()
# plot res
resploter = ECGResultPloter(rawsig, reslist)
dispRange = (20000, 21000)
savefolderpath = os.path.join(curfolderpath, 'tmp', 'MITdbTestResult')
# debug
#pdb.set_trace()
#resploter.plot()
#resploter.plotAndsave(os.path.join(savefolderpath,recID),plotTitle = 'ID:{},Range:{}'.format(recID,dispRange),dispRange = dispRange)
resploter.plot(plotTitle=recID)
pdb.set_trace()
def debug_FN_FP(self,rawsig,reslist,FN,FP):
# =================================
# group result labels
resgrper = ECGGrouper(reslist)
ft_reslist = resgrper.group_local_result(reslist)
# delete white point
reslist = filter(lambda x:x[1]=='R',reslist)
# R wave pos only
ft_reslist = filter(lambda x:x[1]=='R',ft_reslist)
# filtering
ft_reslist = resgrper.syntax_filter(ft_reslist,Max_Len_Ratio = 1.0/5.0)
# =================================
FN_plot_line = ['rd','False Negtive',FN['pos']]
FP_plot_line = ['ks','False Negtive',FP['pos']]
#res_plot_line = ['ko','filtered R',map(lambda x:x[0],ft_reslist)]
# plot res
resploter = ECGResultPloter(rawsig,reslist)
resploter.plot(plotTitle = 'Detection Result:'+FN['recname'][0],AdditionalPlot = [FN_plot_line,FP_plot_line,])
def plotMITdbTestResult():
RFfolder = os.path.join(\
projhomepath,\
'TestResult',\
'pc',\
'r3')
TargetRecordList = ['sel38','sel42',]
# ==========================
# plot prediction result
# ==========================
reslist = glob.glob(os.path.join(\
RFfolder,'*'))
for fi,fname in enumerate(reslist):
# block *.out
# filter file name
if fname[-4:] == '.out' or '.json' in fname:
continue
currecname = os.path.split(fname)[-1]
if currecname not in TargetRecordList:
pass
if not currecname.startswith('result'):
continue
print 'processing file name:',fname
with open(fname,'r') as fin:
(recID,reslist) = pickle.load(fin)
# load signal
mitdb = MITdbLoader()
rawsig = mitdb.load(recID)
# plot res
# group result labels
resgrper = ECGGrouper(reslist)
reslist = resgrper.group_local_result(reslist)
resploter = ECGResultPloter(rawsig,reslist)
dispRange = (20000,21000)
savefolderpath = ur'E:\ECGResults\MITdbTestResult\pic'
# debug
#resploter.plot()
resploter.plot(plotTitle = 'ID:{},Range:{}'.format(recID,dispRange),dispRange = dispRange)
def EvalQTdbResults(resultfilelist, OutputFolder):
if resultfilelist == None or len(resultfilelist) == 0:
print "Empty result file list!"
return None
FN = {'pos': [], 'label': [], 'recname': []}
FP = {'pos': [], 'label': [], 'recname': []}
Err = {'err': [], 'pos': [], 'label': [], 'recname': []}
#========================================
# select best round to compare with refs
#========================================
bRselector = BestRoundSelector()
#InvalidRecordList = conf['InvalidRecords']
# for each record test result
for fi, fname in enumerate(resultfilelist):
print 'json load :', fname
with open(fname, 'rU') as fin:
Results = json.load(fin)
Results = Results[0]
# skip invalid records
currecordname = Results[0]
#if currecordname in InvalidRecordList:
#continue
# ==================================
# filter result of QT
# ==================================
reslist = Results[1]
resfilter = ResultFilter(reslist)
reslist = resfilter.group_local_result(cp_del_thres=1)
reslist = resfilter.syntax_filter(reslist)
fResults = (Results[0], reslist)
fResults = [
fResults,
]
# show filtered results & raw results
#for recname , recRes in Results:
# Evaluate prediction result statistics
#
ECGstats = ECGstatistics(fResults)
pErr, pFN = ECGstats.eval(debug=False)
# get False Positive
pFP = ECGstats.pFP
# one test Error stat
print '[picle filename]:{}'.format(fname)
print '[{}] files left.'.format(len(resultfilelist) - fi)
evallabellist, evalstats = ECGstatistics.dispstat0(pFN=pFN, pErr=pErr)
# select best Round
numofFN = len(pFN['pos'])
if numofFN == 0:
ExtraInfo = 'Best Round ResultFileName[{}]\nTestSet :{}\n#False Negtive:{}\n'.format(
fname, [x[0] for x in Results], numofFN)
bRselector.input(evallabellist, evalstats, ExtraInfo=ExtraInfo)
# ==============================================
for kk in Err:
Err[kk].extend(pErr[kk])
for kk in FN:
FN[kk].extend(pFN[kk])
for kk in FP:
FP[kk].extend(pFP[kk])
#====================================
# write to log file
#EvalLogfilename = os.path.join(curfolderpath,'res.log')
output_log_filename = os.path.join(OutputFolder, 'RecordResults.log')
EvalLogfilename = output_log_filename
# display error stat for each label & save results to logfile
ECGstatistics.dispstat0(
pFN=FN,
pErr=Err,
LogFileName=EvalLogfilename,
LogText='Statistics of Results in FilePath [{}]'.format(
os.path.split(resultfilelist[0])[0]),
OutputFolder=OutputFolder)
with open(os.path.join(curfolderpath, 'Err.txt'), 'w') as fout:
pickle.dump(Err, fout)
# find best round
bRselector.dispBestRound()
bRselector.dumpBestRound(EvalLogfilename)
ECGstats.stat_record_analysis(pErr=Err,
pFN=FN,
LogFileName=EvalLogfilename)
# write csv file
outputfilename = os.path.join(OutputFolder, 'FalsePositive.csv')
ECGstats.FP2CSV(FP, Err, outputfilename)
# False Negtive
outputfilename = os.path.join(OutputFolder, 'FalseNegtive.csv')
ECGstats.FN2CSV(FN, Err, outputfilename)
file_extension = fname.split('.')[-1]
if file_extension in non_result_extensions:
continue
print 'file name:', fname
currecname = os.path.split(fname)[-1]
print currecname
#if currecname == 'result_sel820':
#pdb.set_trace()
if currecname not in TargetRecordList:
pass
#continue
# load signal and reslist
with open(fname, 'r') as fin:
(recID, reslist) = pickle.load(fin)
# filter result of QT
resfilter = ResultFilter(reslist)
reslist = resfilter.group_local_result(cp_del_thres=1)
# empty signal result
#if reslist is None or len(reslist) == 0:
#continue
#pdb.set_trace()
sigstruct = qtdb.load(recID)
# plot figure
# plot res
resploter = ECGResultPloter(sigstruct['sig'], reslist)
resploter.plot(plotTitle='QT database')
#==========================
#show evaluation statistics
#==========================
#TestN_Eval(RFfolder)
def getFN_FP(self,reslist,mit_expertlabel_list,recname,fs = 250):
FN = {
'pos':[],
'label':[],
'recname':[]
}
FP = {
'pos':[],
'label':[],
'recname':[]
}
match_dist_thres = fs/5
# load signal
cFN = 0
#mitdb = MITdbLoader()
#rawsig = mitdb.load(recID)
# plot res
# group result labels
resgrper = ECGGrouper(reslist)
reslist = resgrper.group_local_result(reslist)
# R wave pos only
reslist = filter(lambda x:x[1]=='R',reslist)
# filtering
reslist = resgrper.syntax_filter(reslist,Max_Len_Ratio = 1.0/5.0)
res_poslist,res_labellist = zip(*reslist)
# ------------------------------------------------
# for each expert label find its closest match,if
# distance of this match exceeds match_dist_thres,
# then it is a classification dismatch.
# ------------------------------------------------
N_reslist = len(res_poslist)
N_TP = 0
matched_pred_ind_set = set()
matched_reslist_indexlist = []
for expmpos in mit_expertlabel_list:
# find the closest match in reslist
close_ind = bisect.bisect_left(res_poslist,expmpos)
if close_ind>= N_reslist:
FN['pos'].append(expmpos)
FN['label'].append('R')
FN['recname'].append(recname)
cFN += 1
continue
matchdist = abs(res_poslist[close_ind]-expmpos)
if close_ind>0:
# may find better match
if abs(res_poslist[close_ind-1]-expmpos)<matchdist:
matchdist = abs(res_poslist[close_ind-1]-expmpos)
close_ind -= 1
if matchdist>match_dist_thres or close_ind in matched_pred_ind_set:
FN['pos'].append(expmpos)
FN['label'].append('R')
FN['recname'].append(recname)
cFN+=1
continue
# find a right match:
N_TP += 1
matched_pred_ind_set.add(close_ind)
matched_reslist_indexlist.append(close_ind)
# find False Positive
matched_reslist_indexlist.sort()
start_ind = 0
for m_ind in matched_reslist_indexlist:
while start_ind<N_reslist and start_ind != m_ind:
# a FP
FP['pos'].append(res_poslist[start_ind])
FP['label'].append('R')
FP['recname'].append(recname)
start_ind += 1
start_ind += 1
return (FN,FP,N_TP)
def FN_stat(csvfilepath = r'MITdb_FalseNegtive_stat.csv'):
# analysis false negtive status
RFfolder = os.path.join(\
projhomepath,\
'TestResult',\
'pc',\
'r3')
# write FN statistics to csv file
csv = CSVwriter(csvfilepath)
title_line = ['record name','False Negtive number','Expert Label Number']
csv.output([title_line,])
TargetRecordList = ['sel38','sel42',]
# ==========================
# analysis FN
# ==========================
match_dist_thres = 50
nFN = 0
nExpLabel = 0
reslist = glob.glob(os.path.join(RFfolder,'*'))
N_reslist = len(reslist)
for fi,fname in enumerate(reslist):
# block *.out
# filter file name
if fname[-4:] == '.out' or '.json' in fname:
continue
currecname = os.path.split(fname)[-1]
if currecname not in TargetRecordList:
pass
if not currecname.startswith('result'):
continue
print 'processing file:',fname
print '{} files left.'.format(N_reslist-fi-1)
with open(fname,'r') as fin:
(recID,reslist) = pickle.load(fin)
# load signal
cFN = 0
cExpLabel = 0
mitdb = MITdbLoader()
rawsig = mitdb.load(recID)
# plot res
# group result labels
resgrper = ECGGrouper(reslist)
reslist = resgrper.group_local_result(reslist)
res_poslist,res_labellist = zip(*reslist)
#--------------------------------------
# for each expert label find its closest match,if distance of this match exceeds match_dist_thres,then it is a classification dismatch.
N_res = len(res_poslist)
cExpLabel = len(mitdb.markpos)
for expmpos in mitdb.markpos:
# find the closest match in reslist
close_ind = bisect.bisect_left(res_poslist,expmpos)
matchdist = abs(res_poslist[close_ind]-expmpos)
if close_ind+1<N_res:
matchdist = min(matchdist,abs(res_poslist[close_ind+1]-expmpos))
if matchdist>match_dist_thres:
cFN+=1
# to csv
csv.output([[currecname,cFN,cExpLabel],])
# total stat
nFN += cFN
nExpLabel += cExpLabel
print
print 'record[{}] FN rate:{},PD rate:{}'.format(fname,float(cFN)/cExpLabel,1-float(cFN)/cExpLabel)
# =================================
# total FN
# =================================
print 'Total FN rate:{},PD rate:{}'.format(float(nFN)/nExpLabel,1.0-float(nFN)/nExpLabel)
def EvalQTdbResults(resultfilelist, OutputFolder):
if resultfilelist == None or len(resultfilelist) == 0:
print "Empty result file list!"
return None
FN = {'pos': [], 'label': [], 'recname': []}
FP = {'pos': [], 'label': [], 'recname': []}
Err = {'err': [], 'pos': [], 'label': [], 'recname': []}
#========================================
# select best round to compare with refs
#========================================
bRselector = BestRoundSelector()
#InvalidRecordList = conf['InvalidRecords']
# for each record test result
for fi, fname in enumerate(resultfilelist):
print 'pickle load :', fname
with open(fname, 'rU') as fin:
Results = pickle.load(fin)
# skip invalid records
currecordname = Results[0]
#if currecordname in InvalidRecordList:
#continue
# ==================================
# filter result of QT
# ==================================
reslist = Results[1]
resfilter = ResultFilter(reslist)
reslist = resfilter.group_local_result(cp_del_thres=1)
#reslist = resfilter.syntax_filter(reslist)
fResults = (Results[0], reslist)
fResults = [
fResults,
]
# show filtered results & raw results
#for recname , recRes in Results:
# Evaluate prediction result statistics
#
ECGstats = ECGstatistics(fResults)
pErr, pFN = ECGstats.eval(debug=False)
# get False Positive
pFP = ECGstats.pFP
# one test Error stat
print '[picle filename]:{}'.format(fname)
print '[{}] files left.'.format(len(resultfilelist) - fi)
evallabellist, evalstats = ECGstatistics.dispstat0(pFN=pFN, pErr=pErr)
# select best Round
numofFN = len(pFN['pos'])
if numofFN == 0:
ExtraInfo = 'Best Round ResultFileName[{}]\nTestSet :{}\n#False Negtive:{}\n'.format(
fname, [x[0] for x in Results], numofFN)
bRselector.input(evallabellist, evalstats, ExtraInfo=ExtraInfo)
# ==============================================
for kk in Err:
Err[kk].extend(pErr[kk])
for kk in FN:
FN[kk].extend(pFN[kk])
for kk in FP:
FP[kk].extend(pFP[kk])
return (FN, FP, Err)