def main(ground_truth_filename, subset, prediction_filename, output_folder, is_thumos14):
if not is_thumos14:
if subset == 'testing':
# ActivityNet testing
characteristic_names_to_bins = {'context-size': (range(-1,7), ['0','1','2','3','4','5','6']),
'context-distance': (range(-1,4), ['Inf','N','M','F']),
'agreement': (np.linspace(0,1.0,6), ['XW','W','M','H','XH']),
'coverage': (np.linspace(0,1.0,6), ['XS','S','M','L','XL']),
'length': (np.array([0,30,60,120,180,np.inf]), ['XS','S','M','L','XL']),
'num-instances': (np.array([-1,1,4,8,np.inf]), ['XS','S','M','L'])}
elif subset == 'validation':
# ActivityNet validation
characteristic_names_to_bins = {'coverage': (np.linspace(0,1.0,6), ['XS','S','M','L','XL']),
'length': (np.array([0,30,60,120,180,np.inf]), ['XS','S','M','L','XL']),
'num-instances': (np.array([-1,1,4,8,np.inf]), ['XS','S','M','L'])}
else:
raise RuntimeError('%s is not a valid subset' % subset)
tiou_thresholds = np.linspace(0.5, 0.95, 10)
else:
# THUMOS14
characteristic_names_to_bins = {'coverage': (np.array([0,0.02,0.04,0.06,0.08,1]), ['XS','S','M','L','XL']),
'length': (np.array([0,3,6,12,18,np.inf]), ['XS','S','M','L','XL']),
'num-instances': (np.array([-1,1,40,80,np.inf]), ['XS','S','M','L'])}
tiou_thresholds = [0.5]
fp_error_analysis = ActionDetectorDiagnosis(ground_truth_filename=ground_truth_filename,
prediction_filename=prediction_filename,
tiou_thresholds=tiou_thresholds,
limit_factor=10,
min_tiou_thr=0.1,
subset=subset,
verbose=True,
check_status=True,
load_extra_annotations=True,
characteristic_names_to_bins=characteristic_names_to_bins,
normalize_ap=True,
minimum_normalized_precision_threshold_for_detection=0.0
)
fp_error_analysis.evaluate()
fp_error_analysis.diagnose()
plot_fp_analysis(fp_error_analysis=fp_error_analysis,
save_filename=os.path.join(output_folder,'false_positive_analysis.pdf'))
def main(ground_truth_filename, subset, prediction_filename, output_folder,
is_thumos14):
os.makedirs(output_folder, exist_ok=True)
if not is_thumos14:
if subset == 'testing':
# ActivityNet testing
characteristic_names_to_bins = {
'context-size':
(range(-1, 7), ['0', '1', '2', '3', '4', '5', '6']),
'context-distance': (range(-1, 4), ['Inf', 'N', 'M', 'F']),
'agreement': (np.linspace(0, 1.0,
6), ['XW', 'W', 'M', 'H', 'XH']),
'coverage': (np.linspace(0, 1.0,
6), ['XS', 'S', 'M', 'L', 'XL']),
'length': (np.array([0, 30, 60, 120, 180,
np.inf]), ['XS', 'S', 'M', 'L', 'XL']),
'num-instances': (np.array([-1, 1, 4, 8,
np.inf]), ['XS', 'S', 'M', 'L'])
}
colors = [
'#7fc97f', '#beaed4', '#fdc086', '#386cb0', '#f0027f',
'#bf5b17'
]
characteristic_names = [
'context-size', 'context-distance', 'agreement', 'coverage',
'length', 'num-instances'
]
characteristic_names_in_text = [
'Context Size', 'Context Distance', 'Agreement', 'Coverage',
'Length', '\# Instances'
]
characteristic_names_delta_positions = [
1.1, -1.4, 0.25, 0.5, 1, -0.2
]
figsize = (25, 6)
num_grids = 4
elif subset == 'validation':
# ActivityNet validation
characteristic_names_to_bins = {
'coverage': (np.linspace(0, 1.0,
6), ['XS', 'S', 'M', 'L', 'XL']),
'length': (np.array([0, 30, 60, 120, 180,
np.inf]), ['XS', 'S', 'M', 'L', 'XL']),
'num-instances': (np.array([-1, 1, 4, 8,
np.inf]), ['XS', 'S', 'M', 'L'])
}
colors = ['#386cb0', '#f0027f', '#bf5b17']
characteristic_names = ['coverage', 'length', 'num-instances']
characteristic_names_in_text = [
'Coverage', 'Length', '\# Instances'
]
characteristic_names_delta_positions = [0.5, 1, -0.2]
figsize = (17.5, 6)
num_grids = 3
else:
raise RuntimeError('%s is not a valid subset' % subset)
tiou_thresholds = np.linspace(0.5, 0.95, 10)
else:
# THUMOS14
characteristic_names_to_bins = {
'coverage': (np.array([0, 0.02, 0.04, 0.06, 0.08,
1]), ['XS', 'S', 'M', 'L', 'XL']),
'length': (np.array([0, 3, 6, 12, 18,
np.inf]), ['XS', 'S', 'M', 'L', 'XL']),
'num-instances': (np.array([-1, 1, 40, 80,
np.inf]), ['XS', 'S', 'M', 'L'])
}
colors = ['#386cb0', '#f0027f', '#bf5b17']
characteristic_names = ['coverage', 'length', 'num-instances']
characteristic_names_in_text = ['Coverage', 'Length', '\# Instances']
characteristic_names_delta_positions = [0.5, 1, -0.2]
figsize = (17.5, 6)
num_grids = 3
tiou_thresholds = [0.5]
sensitivity_analysis = ActionDetectorDiagnosis(
ground_truth_filename=ground_truth_filename,
prediction_filename=prediction_filename,
tiou_thresholds=tiou_thresholds,
limit_factor=None,
min_tiou_thr=0.1,
subset=subset,
verbose=True,
load_extra_annotations=True,
characteristic_names_to_bins=characteristic_names_to_bins,
normalize_ap=True,
minimum_normalized_precision_threshold_for_detection=0.0)
sensitivity_analysis.evaluate()
plot_sensitivity_analysis(
sensitivity_analysis=sensitivity_analysis,
save_filename=os.path.join(output_folder, 'sensitivity_analysis.pdf'),
colors=colors,
characteristic_names=characteristic_names,
characteristic_names_in_text=characteristic_names_in_text,
characteristic_names_delta_positions=
characteristic_names_delta_positions,
figsize=figsize,
num_grids=num_grids)