from upcall.accuracy_measure import map_build_day_file import glob, os from upcall.config import Config from upcall.full_process import full_process_interfaceDSP, full_process_interface from upcall.RunDetectorDSPdependency import MakeSoundStream #import matplotlib.pyplot as plt # parameter config = Config() DATA_DIR = r'/home/ys587/__Data' # truth data: TruthFolder = r'/home/ys587/__Data/__TruthLabelVAPlusStAndrew' TruthFolder = r'/cache/kpalmer/quick_ssd/data/dclmmpa2013/LogsWithNewSNR/RavenST/NOPPSet2' day_file_map = map_build_day_file(TruthFolder) # testing sound stream of St Andrew DCL SoundPath = os.path.join(DATA_DIR, r'DCL_St_Andrew/Sound_3_days') day_list = sorted(glob.glob(os.path.join(TruthFolder + '/', '*'))) SoundFileLoc = '/cache/kpalmer/quick_ssd/data/dclmmpa2013/Testing/Upcalls_NOPPset2' file_days = os.listdir(SoundFileLoc) file_days = [os.path.join(SoundFileLoc, s) for s in file_days] # Make soundstream of all folders sample_stream = MakeSoundStream(file_days) # Test expt_label = 'Test detection' classifier_file = r'weights-improvement-38-0.8361.hdf5'
def calc_avg_prec(truth_folder, seltab_detect_path, config):
map_day_file = map_build_day_file(truth_folder)
day_list = sorted(glob.glob(os.path.join(
seltab_detect_path, '*.txt'))) # find selection table for all days
# turn off pd "cope of a slice" warning
pd.set_option('mode.chained_assignment', None)
df_detect_list = []
df_truth_list = []
for dd in day_list:
basename = os.path.basename(dd)
print('Current day: ' + basename)
#print('Corresponding truth file: ', end='')
date_str = get_date_str(basename)
truth_file = map_day_file[date_str]
#print(truth_file)
# (a) dd, the detection file versus (b) truth_file
df_detect0 = pd.read_table(dd)
df_truth0 = pd.read_table(truth_file)
# sort by the column "Begin Time (s)"
df_detect0 = df_detect0.sort_values(r'Begin Time (s)')
df_detect0 = df_detect0.reset_index(drop=True)
df_detect0[r'Center Time (s)'] = 0.5 * df_detect0[
r'Begin Time (s)'] + 0.5 * df_detect0[r'End Time (s)']
df_detect0[r'date'] = date_str
df_detect_list.append(df_detect0)
df_truth0 = df_truth0.sort_values(r'Begin Time (s)')
df_truth0 = df_truth0.reset_index(drop=True)
df_truth0[r'Center Time (s)'] = 0.5 * df_truth0[
r'Begin Time (s)'] + 0.5 * df_truth0[r'End Time (s)']
df_truth0[r'date'] = date_str
df_truth_list.append(df_truth0)
df_detect = pd.concat(df_detect_list)
del df_detect_list
df_truth = pd.concat(df_truth_list)
del df_truth_list
#df_truth = df_truth.reset_index(drop=True)
#dects = []
#[dects.append(row) for index, row in df_detect.iterrows()]
gts = []
[gts.append(row) for index, row in df_truth.iterrows()]
# dects => df_detect; gts => df_truth
npos = df_truth.shape[0]
# sort detections by decreasing confidence
df_detect = df_detect.sort_values(by=['Score'],
ascending=False) # sort by score
TP = np.zeros(df_detect.shape[0])
FP = np.zeros(df_detect.shape[0])
det0 = Counter([cc[r'date'] for cc in gts])
det = {}
for key, val in det0.items():
det[key] = np.zeros(val)
for dd in range(df_detect.shape[0]):
# for dd in range(500):
df_truth_same_date = df_truth.loc[
(df_truth['date'] == df_detect.iloc[dd]['date']) &
(np.abs(df_truth[r'Center Time (s)'] -
df_detect.iloc[dd][r'Center Time (s)']) < config.SEP_THRE)]
df_truth_same_date[r'delta time'] = np.abs(
df_truth_same_date[r'Center Time (s)'] -
df_detect.iloc[dd][r'Center Time (s)'])
if dd % 200 == 0:
print('#', end='', flush=True)
hit_num = df_truth_same_date.shape[0]
if hit_num > 0:
if hit_num == 1:
ind_hit = df_truth_same_date.index[0]
else: # >= 2
ind_hit = df_truth_same_date[r'delta time'].idxmin(axis=0)
if det[df_detect.iloc[dd]['date']][ind_hit] == 0:
TP[dd] = 1
det[df_detect.iloc[dd]['date']][ind_hit] = 1
else:
FP[dd] = 1
else:
FP[dd] = 1
print('')
acc_FP = np.cumsum(FP)
acc_TP = np.cumsum(TP)
rec = acc_TP / npos
prec = np.divide(acc_TP, (acc_FP + acc_TP))
[ap_11pt, mpre, mrec, _] = ElevenPointInterpolatedAP(rec, prec)
print('AP_11pt is: ' + str(ap_11pt))
[ap_avg, mpre, mrec, _] = CalculateAveragePrecision(rec, prec)
print('AP_avg is: ' + str(ap_avg))
# plt.plot(mrec, mpre)
return ap_11pt, ap_avg, mpre, mrec
from upcall.config import Config from upcall.full_process import full_process_interfaceDSP, full_process_interface from upcall.RunDetectorDSPdependency import MakeSoundStream #import matplotlib.pyplot as plt start_time = time.time() # parameter config = Config() DATA_DIR = r'/home/ys587/__Data' # truth data: #TruthFolder = r'/home/ys587/__Data/__TruthLabelVAPlusStAndrew' truth_folder = os.path.join(DATA_DIR, 'DCL_St_Andrew/Sound_3_days_seltab') day_file_map = map_build_day_file(truth_folder) # testing sound stream of St Andrew DCL #SoundPath = os.path.join(DATA_DIR, r'DCL_St_Andrew/Sound_3_days_short_test') SoundPath = os.path.join(DATA_DIR, r'DCL_St_Andrew/Sound_3_days') #SoundPath = '/cache/kpalmer/quick_ssd/data/dclmmpa2013/Testing/Upcalls_NOPPset2' SoundFileLoc = file_days = os.listdir(SoundPath) file_days = [os.path.join(SoundPath, s) for s in file_days] # Make soundstream of all folders sample_stream = MakeSoundStream(file_days) # Test expt_label = 'evaluate' #classifier_file = r'weights-improvement-38-0.8361.hdf5' #classifier_path = r'/home/ys587/__Data/DetectionTestCase/' classifier_path = '/home/ys587/tensorflow3/__ModelOutput/__DCLDE2018/ModelSearch201805_DCL/LeNet_Dropout_Layer_p2/weights-improvement-56-0.9133.hdf5'
Created on 7/11/19
@author: atoultaro
"""
import os
import glob
from upcall.config import Config
from upcall.accuracy_measure import calc_TP_FP_FN, map_build_day_file, \
get_date_str
result_path = os.path.join(
'/home/ys587/__ExptResult/cv_lenet_dropout_conv_train_Data_2_augment',
'__full_data_large')
DATA_DIR = r'/home/ys587/__Data' # Data
truth_folder = os.path.join(DATA_DIR, '__large_test_dataset/__truth_seltab')
config = Config(dataset='Data_2', model='lenet_dropout_conv')
config.NUM_RUNS = 1
for rr in range(config.NUM_RUNS):
day_list = sorted(
glob.glob(
os.path.join(result_path, 'Run' + str(rr), '__TP_FN_FP',
'*.txt'))) # find selection table for all days
print(rr)
for dd in day_list:
basename = os.path.basename(dd)
map_day_file = map_build_day_file(truth_folder)
truth_file = map_day_file[get_date_str(basename)]
accu_result, _ = calc_TP_FP_FN(dd, truth_file, result_path,
config) # TP, FP, TP2, FN