def plot_QTdb_filtered_Result_with_syntax_filter(RFfolder, TargetRecordList, ResultFilterType, showExpertLabels=False): # exit # ========================== # plot prediction result # ========================== reslist = getresultfilelist(RFfolder) qtdb = QTloader() non_result_extensions = ['out', 'json', 'log', 'txt'] for fi, fname in enumerate(reslist): # block *.out file_extension = fname.split('.')[-1] if file_extension in non_result_extensions: continue print 'file name:', fname currecname = os.path.split(fname)[-1] print currecname #if currecname == 'result_sel820': #pdb.set_trace() if TargetRecordList is not None: if currecname not in TargetRecordList: continue # load signal and reslist with open(fname, 'r') as fin: (recID, reslist) = pickle.load(fin) # filter result of QT resfilter = ResultFilter(reslist) if len(ResultFilterType) >= 1 and ResultFilterType[0] == 'G': reslist = resfilter.group_local_result(cp_del_thres=1) reslist_syntax = reslist if len(ResultFilterType) >= 2 and ResultFilterType[1] == 'S': reslist_syntax = resfilter.syntax_filter(reslist) # empty signal result #if reslist is None or len(reslist) == 0: #continue #pdb.set_trace() sigstruct = qtdb.load(recID) if showExpertLabels == True: # Expert Label AdditionalPlot ExpertRes = qtdb.getexpertlabeltuple(recID) ExpertPoslist = map(lambda x: x[0], ExpertRes) AdditionalPlot = [ ['kd', 'Expert Labels', ExpertPoslist], ] else: AdditionalPlot = None # plot res #resploter = ECGResultPloter(sigstruct['sig'],reslist) #resploter.plot(plotTitle = 'QT database',plotShow = True,plotFig = 2) # syntax_filter resploter_syntax = ECGResultPloter(sigstruct['sig'], reslist_syntax) resploter_syntax.plot(plotTitle='QT Record {}'.format(recID), plotShow=True, AdditionalPlot=AdditionalPlot)
def plot_QTdb_filtered_Result_with_syntax_filter(): # exit RFfolder = os.path.join(\ projhomepath,\ 'TestResult',\ 'pc',\ 'r5') TargetRecordList = [ 'result_sel39', 'result_sel41', 'result_sel48', ] #'sel38','sel42','result_sel821','result_sel14046'] # ========================== # plot prediction result # ========================== reslist = glob.glob(os.path.join(\ RFfolder,'*')) qtdb = QTloader() non_result_extensions = ['out', 'json', 'log'] for fi, fname in enumerate(reslist): # block *.out file_extension = fname.split('.')[-1] if file_extension in non_result_extensions: continue print 'file name:', fname currecname = os.path.split(fname)[-1] print currecname #if currecname == 'result_sel820': #pdb.set_trace() if currecname not in TargetRecordList: pass continue # load signal and reslist with open(fname, 'r') as fin: (recID, reslist) = pickle.load(fin) # filter result of QT resfilter = ResultFilter(reslist) reslist = resfilter.group_local_result(cp_del_thres=1) reslist_syntax = resfilter.syntax_filter(reslist) # empty signal result #if reslist is None or len(reslist) == 0: #continue #pdb.set_trace() sigstruct = qtdb.load(recID) # plot res #resploter = ECGResultPloter(sigstruct['sig'],reslist) #resploter.plot(plotTitle = 'QT database',plotShow = True,plotFig = 2) # syntax_filter resploter_syntax = ECGResultPloter(sigstruct['sig'], reslist_syntax) resploter_syntax.plot(plotTitle='QT database syntax_filter', plotShow=True)
def plotMITdbTestResult(): RFfolder = os.path.join(\ projhomepath,\ 'TestResult',\ 'pc',\ 'r3') TargetRecordList = [ 'sel38', 'sel42', ] # ========================== # plot prediction result # ========================== reslist = glob.glob(os.path.join(\ RFfolder,'*')) for fi, fname in enumerate(reslist): # block *.out # filter file name if fname[-4:] == '.out' or '.json' in fname: continue currecname = os.path.split(fname)[-1] if currecname not in TargetRecordList: pass if not currecname.startswith('result'): continue print 'processing file', fname, '...' with open(fname, 'r') as fin: (recID, reslist) = pickle.load(fin) print 'pickle file loaded.' # load signal from MITdb print 'loading signal data from MITdb...' mitdb = MITdbLoader() rawsig = mitdb.load(recID) print 'signal loaded.' # filter result list resfilter = ResultFilter(reslist) reslist = resfilter.groupresult() # plot res resploter = ECGResultPloter(rawsig, reslist) dispRange = (20000, 21000) savefolderpath = os.path.join(curfolderpath, 'tmp', 'MITdbTestResult') # debug #pdb.set_trace() #resploter.plot() #resploter.plotAndsave(os.path.join(savefolderpath,recID),plotTitle = 'ID:{},Range:{}'.format(recID,dispRange),dispRange = dispRange) resploter.plot(plotTitle=recID) pdb.set_trace()
def debug_FN_FP(self,rawsig,reslist,FN,FP): # ================================= # group result labels resgrper = ECGGrouper(reslist) ft_reslist = resgrper.group_local_result(reslist) # delete white point reslist = filter(lambda x:x[1]=='R',reslist) # R wave pos only ft_reslist = filter(lambda x:x[1]=='R',ft_reslist) # filtering ft_reslist = resgrper.syntax_filter(ft_reslist,Max_Len_Ratio = 1.0/5.0) # ================================= FN_plot_line = ['rd','False Negtive',FN['pos']] FP_plot_line = ['ks','False Negtive',FP['pos']] #res_plot_line = ['ko','filtered R',map(lambda x:x[0],ft_reslist)] # plot res resploter = ECGResultPloter(rawsig,reslist) resploter.plot(plotTitle = 'Detection Result:'+FN['recname'][0],AdditionalPlot = [FN_plot_line,FP_plot_line,])
def plotMITdbTestResult(): RFfolder = os.path.join(\ projhomepath,\ 'TestResult',\ 'pc',\ 'r3') TargetRecordList = ['sel38','sel42',] # ========================== # plot prediction result # ========================== reslist = glob.glob(os.path.join(\ RFfolder,'*')) for fi,fname in enumerate(reslist): # block *.out # filter file name if fname[-4:] == '.out' or '.json' in fname: continue currecname = os.path.split(fname)[-1] if currecname not in TargetRecordList: pass if not currecname.startswith('result'): continue print 'processing file name:',fname with open(fname,'r') as fin: (recID,reslist) = pickle.load(fin) # load signal mitdb = MITdbLoader() rawsig = mitdb.load(recID) # plot res # group result labels resgrper = ECGGrouper(reslist) reslist = resgrper.group_local_result(reslist) resploter = ECGResultPloter(rawsig,reslist) dispRange = (20000,21000) savefolderpath = ur'E:\ECGResults\MITdbTestResult\pic' # debug #resploter.plot() resploter.plot(plotTitle = 'ID:{},Range:{}'.format(recID,dispRange),dispRange = dispRange)
def EvalQTdbResults(resultfilelist, OutputFolder): if resultfilelist == None or len(resultfilelist) == 0: print "Empty result file list!" return None FN = {'pos': [], 'label': [], 'recname': []} FP = {'pos': [], 'label': [], 'recname': []} Err = {'err': [], 'pos': [], 'label': [], 'recname': []} #======================================== # select best round to compare with refs #======================================== bRselector = BestRoundSelector() #InvalidRecordList = conf['InvalidRecords'] # for each record test result for fi, fname in enumerate(resultfilelist): print 'json load :', fname with open(fname, 'rU') as fin: Results = json.load(fin) Results = Results[0] # skip invalid records currecordname = Results[0] #if currecordname in InvalidRecordList: #continue # ================================== # filter result of QT # ================================== reslist = Results[1] resfilter = ResultFilter(reslist) reslist = resfilter.group_local_result(cp_del_thres=1) reslist = resfilter.syntax_filter(reslist) fResults = (Results[0], reslist) fResults = [ fResults, ] # show filtered results & raw results #for recname , recRes in Results: # Evaluate prediction result statistics # ECGstats = ECGstatistics(fResults) pErr, pFN = ECGstats.eval(debug=False) # get False Positive pFP = ECGstats.pFP # one test Error stat print '[picle filename]:{}'.format(fname) print '[{}] files left.'.format(len(resultfilelist) - fi) evallabellist, evalstats = ECGstatistics.dispstat0(pFN=pFN, pErr=pErr) # select best Round numofFN = len(pFN['pos']) if numofFN == 0: ExtraInfo = 'Best Round ResultFileName[{}]\nTestSet :{}\n#False Negtive:{}\n'.format( fname, [x[0] for x in Results], numofFN) bRselector.input(evallabellist, evalstats, ExtraInfo=ExtraInfo) # ============================================== for kk in Err: Err[kk].extend(pErr[kk]) for kk in FN: FN[kk].extend(pFN[kk]) for kk in FP: FP[kk].extend(pFP[kk]) #==================================== # write to log file #EvalLogfilename = os.path.join(curfolderpath,'res.log') output_log_filename = os.path.join(OutputFolder, 'RecordResults.log') EvalLogfilename = output_log_filename # display error stat for each label & save results to logfile ECGstatistics.dispstat0( pFN=FN, pErr=Err, LogFileName=EvalLogfilename, LogText='Statistics of Results in FilePath [{}]'.format( os.path.split(resultfilelist[0])[0]), OutputFolder=OutputFolder) with open(os.path.join(curfolderpath, 'Err.txt'), 'w') as fout: pickle.dump(Err, fout) # find best round bRselector.dispBestRound() bRselector.dumpBestRound(EvalLogfilename) ECGstats.stat_record_analysis(pErr=Err, pFN=FN, LogFileName=EvalLogfilename) # write csv file outputfilename = os.path.join(OutputFolder, 'FalsePositive.csv') ECGstats.FP2CSV(FP, Err, outputfilename) # False Negtive outputfilename = os.path.join(OutputFolder, 'FalseNegtive.csv') ECGstats.FN2CSV(FN, Err, outputfilename)
file_extension = fname.split('.')[-1] if file_extension in non_result_extensions: continue print 'file name:', fname currecname = os.path.split(fname)[-1] print currecname #if currecname == 'result_sel820': #pdb.set_trace() if currecname not in TargetRecordList: pass #continue # load signal and reslist with open(fname, 'r') as fin: (recID, reslist) = pickle.load(fin) # filter result of QT resfilter = ResultFilter(reslist) reslist = resfilter.group_local_result(cp_del_thres=1) # empty signal result #if reslist is None or len(reslist) == 0: #continue #pdb.set_trace() sigstruct = qtdb.load(recID) # plot figure # plot res resploter = ECGResultPloter(sigstruct['sig'], reslist) resploter.plot(plotTitle='QT database') #========================== #show evaluation statistics #========================== #TestN_Eval(RFfolder)
def getFN_FP(self,reslist,mit_expertlabel_list,recname,fs = 250): FN = { 'pos':[], 'label':[], 'recname':[] } FP = { 'pos':[], 'label':[], 'recname':[] } match_dist_thres = fs/5 # load signal cFN = 0 #mitdb = MITdbLoader() #rawsig = mitdb.load(recID) # plot res # group result labels resgrper = ECGGrouper(reslist) reslist = resgrper.group_local_result(reslist) # R wave pos only reslist = filter(lambda x:x[1]=='R',reslist) # filtering reslist = resgrper.syntax_filter(reslist,Max_Len_Ratio = 1.0/5.0) res_poslist,res_labellist = zip(*reslist) # ------------------------------------------------ # for each expert label find its closest match,if # distance of this match exceeds match_dist_thres, # then it is a classification dismatch. # ------------------------------------------------ N_reslist = len(res_poslist) N_TP = 0 matched_pred_ind_set = set() matched_reslist_indexlist = [] for expmpos in mit_expertlabel_list: # find the closest match in reslist close_ind = bisect.bisect_left(res_poslist,expmpos) if close_ind>= N_reslist: FN['pos'].append(expmpos) FN['label'].append('R') FN['recname'].append(recname) cFN += 1 continue matchdist = abs(res_poslist[close_ind]-expmpos) if close_ind>0: # may find better match if abs(res_poslist[close_ind-1]-expmpos)<matchdist: matchdist = abs(res_poslist[close_ind-1]-expmpos) close_ind -= 1 if matchdist>match_dist_thres or close_ind in matched_pred_ind_set: FN['pos'].append(expmpos) FN['label'].append('R') FN['recname'].append(recname) cFN+=1 continue # find a right match: N_TP += 1 matched_pred_ind_set.add(close_ind) matched_reslist_indexlist.append(close_ind) # find False Positive matched_reslist_indexlist.sort() start_ind = 0 for m_ind in matched_reslist_indexlist: while start_ind<N_reslist and start_ind != m_ind: # a FP FP['pos'].append(res_poslist[start_ind]) FP['label'].append('R') FP['recname'].append(recname) start_ind += 1 start_ind += 1 return (FN,FP,N_TP)
def FN_stat(csvfilepath = r'MITdb_FalseNegtive_stat.csv'): # analysis false negtive status RFfolder = os.path.join(\ projhomepath,\ 'TestResult',\ 'pc',\ 'r3') # write FN statistics to csv file csv = CSVwriter(csvfilepath) title_line = ['record name','False Negtive number','Expert Label Number'] csv.output([title_line,]) TargetRecordList = ['sel38','sel42',] # ========================== # analysis FN # ========================== match_dist_thres = 50 nFN = 0 nExpLabel = 0 reslist = glob.glob(os.path.join(RFfolder,'*')) N_reslist = len(reslist) for fi,fname in enumerate(reslist): # block *.out # filter file name if fname[-4:] == '.out' or '.json' in fname: continue currecname = os.path.split(fname)[-1] if currecname not in TargetRecordList: pass if not currecname.startswith('result'): continue print 'processing file:',fname print '{} files left.'.format(N_reslist-fi-1) with open(fname,'r') as fin: (recID,reslist) = pickle.load(fin) # load signal cFN = 0 cExpLabel = 0 mitdb = MITdbLoader() rawsig = mitdb.load(recID) # plot res # group result labels resgrper = ECGGrouper(reslist) reslist = resgrper.group_local_result(reslist) res_poslist,res_labellist = zip(*reslist) #-------------------------------------- # for each expert label find its closest match,if distance of this match exceeds match_dist_thres,then it is a classification dismatch. N_res = len(res_poslist) cExpLabel = len(mitdb.markpos) for expmpos in mitdb.markpos: # find the closest match in reslist close_ind = bisect.bisect_left(res_poslist,expmpos) matchdist = abs(res_poslist[close_ind]-expmpos) if close_ind+1<N_res: matchdist = min(matchdist,abs(res_poslist[close_ind+1]-expmpos)) if matchdist>match_dist_thres: cFN+=1 # to csv csv.output([[currecname,cFN,cExpLabel],]) # total stat nFN += cFN nExpLabel += cExpLabel print print 'record[{}] FN rate:{},PD rate:{}'.format(fname,float(cFN)/cExpLabel,1-float(cFN)/cExpLabel) # ================================= # total FN # ================================= print 'Total FN rate:{},PD rate:{}'.format(float(nFN)/nExpLabel,1.0-float(nFN)/nExpLabel)
def EvalQTdbResults(resultfilelist, OutputFolder): if resultfilelist == None or len(resultfilelist) == 0: print "Empty result file list!" return None FN = {'pos': [], 'label': [], 'recname': []} FP = {'pos': [], 'label': [], 'recname': []} Err = {'err': [], 'pos': [], 'label': [], 'recname': []} #======================================== # select best round to compare with refs #======================================== bRselector = BestRoundSelector() #InvalidRecordList = conf['InvalidRecords'] # for each record test result for fi, fname in enumerate(resultfilelist): print 'pickle load :', fname with open(fname, 'rU') as fin: Results = pickle.load(fin) # skip invalid records currecordname = Results[0] #if currecordname in InvalidRecordList: #continue # ================================== # filter result of QT # ================================== reslist = Results[1] resfilter = ResultFilter(reslist) reslist = resfilter.group_local_result(cp_del_thres=1) #reslist = resfilter.syntax_filter(reslist) fResults = (Results[0], reslist) fResults = [ fResults, ] # show filtered results & raw results #for recname , recRes in Results: # Evaluate prediction result statistics # ECGstats = ECGstatistics(fResults) pErr, pFN = ECGstats.eval(debug=False) # get False Positive pFP = ECGstats.pFP # one test Error stat print '[picle filename]:{}'.format(fname) print '[{}] files left.'.format(len(resultfilelist) - fi) evallabellist, evalstats = ECGstatistics.dispstat0(pFN=pFN, pErr=pErr) # select best Round numofFN = len(pFN['pos']) if numofFN == 0: ExtraInfo = 'Best Round ResultFileName[{}]\nTestSet :{}\n#False Negtive:{}\n'.format( fname, [x[0] for x in Results], numofFN) bRselector.input(evallabellist, evalstats, ExtraInfo=ExtraInfo) # ============================================== for kk in Err: Err[kk].extend(pErr[kk]) for kk in FN: FN[kk].extend(pFN[kk]) for kk in FP: FP[kk].extend(pFP[kk]) return (FN, FP, Err)