def _do_python_eval(json_dataset, salt, output_dir='output'):
info = voc_info(json_dataset)
year = info['year']
anno_path = info['anno_path']
image_set_path = info['image_set_path']
devkit_path = info['devkit_path']
cachedir = os.path.join(devkit_path, 'annotations_cache')
aps = []
# The PASCAL VOC metric changed in 2010
use_07_metric = True if int(year) < 2010 else False
logger.info('VOC07 metric? ' + ('Yes' if use_07_metric else 'No'))
if not os.path.isdir(output_dir):
os.mkdir(output_dir)
for _, cls in enumerate(json_dataset.classes):
if cls == '__background__':
continue
filename = _get_voc_results_file_template(json_dataset,
salt).format(cls)
rec, prec, ap = voc_eval(filename,
anno_path,
image_set_path,
cls,
cachedir,
ovthresh=0.5,
use_07_metric=use_07_metric)
aps += [ap]
logger.info('AP for {} = {:.4f}'.format(cls, ap))
res_file = os.path.join(output_dir, cls + '_pr.pkl')
save_object({'rec': rec, 'prec': prec, 'ap': ap}, res_file)
logger.info('Mean AP = {:.4f}'.format(np.mean(aps)))
logger.info('~~~~~~~~')
logger.info('Results:')
for ap in aps:
logger.info('{:.3f}'.format(ap))
logger.info('{:.3f}'.format(np.mean(aps)))
logger.info('~~~~~~~~')
logger.info('')
logger.info('----------------------------------------------------------')
logger.info('Results computed with the **unofficial** Python eval code.')
logger.info('Results should be very close to the official MATLAB code.')
logger.info('Use `./tools/reval.py --matlab ...` for your paper.')
logger.info('-- Thanks, The Management')
logger.info('----------------------------------------------------------')
def _do_python_eval(self, output_dir='output'):
annopath = os.path.join(self._devkit_path, 'VOC' + self._year,
'Annotations', '{:s}.xml')
imagesetfile = os.path.join(self._devkit_path, 'VOC' + self._year,
'ImageSets', 'Main',
self._image_set + '.txt')
cachedir = os.path.join(self._devkit_path, 'annotations_cache')
aps = []
# The PASCAL VOC metric changed in 2010
use_07_metric = True if int(self._year) < 2010 else False
print('VOC07 metric? ' + ('Yes' if use_07_metric else 'No'))
if not os.path.isdir(output_dir):
os.mkdir(output_dir)
for i, cls in enumerate(self._classes):
if cls == '__background__':
continue
filename = self._get_voc_results_file_template().format(cls)
rec, prec, ap = voc_eval(filename,
annopath,
imagesetfile,
cls,
cachedir,
ovthresh=0.5,
use_07_metric=use_07_metric,
use_diff=self.config['use_diff'])
aps += [ap]
print(('AP for {} = {:.4f}'.format(cls, ap)))
with open(os.path.join(output_dir, cls + '_pr.pkl'), 'wb') as f:
pickle.dump({'rec': rec, 'prec': prec, 'ap': ap}, f)
print(('Mean AP = {:.4f}'.format(np.mean(aps))))
print('~~~~~~~~')
print('Results:')
for ap in aps:
print(('{:.3f}'.format(ap)))
print(('{:.3f}'.format(np.mean(aps))))
print('~~~~~~~~')
print('')
print('--------------------------------------------------------------')
print('Results computed with the **unofficial** Python eval code.')
print('Results should be very close to the official MATLAB eval code.')
print('Recompute with `./tools/reval.py --matlab ...` for your paper.')
print('-- Thanks, The Management')
print('--------------------------------------------------------------')
def _do_python_eval(json_dataset, salt, output_dir='output'):
info = voc_info(json_dataset)
year = info['year']
anno_path = info['anno_path']
image_set_path = info['image_set_path']
devkit_path = info['devkit_path']
cachedir = os.path.join(devkit_path, 'annotations_cache')
aps = []
# The PASCAL VOC metric changed in 2010
use_07_metric = True if int(year) < 2010 else False
logger.info('VOC07 metric? ' + ('Yes' if use_07_metric else 'No'))
if not os.path.isdir(output_dir):
os.mkdir(output_dir)
for _, cls in enumerate(json_dataset.classes):
if cls == '__background__':
continue
filename = _get_voc_results_file_template(
json_dataset, salt).format(cls)
rec, prec, ap = voc_eval(
filename, anno_path, image_set_path, cls, cachedir, ovthresh=0.5,
use_07_metric=use_07_metric)
aps += [ap]
logger.info('AP for {} = {:.4f}'.format(cls, ap))
res_file = os.path.join(output_dir, cls + '_pr.pkl')
save_object({'rec': rec, 'prec': prec, 'ap': ap}, res_file)
logger.info('Mean AP = {:.4f}'.format(np.mean(aps)))
logger.info('~~~~~~~~')
logger.info('Results:')
for ap in aps:
logger.info('{:.3f}'.format(ap))
logger.info('{:.3f}'.format(np.mean(aps)))
logger.info('~~~~~~~~')
logger.info('')
logger.info('----------------------------------------------------------')
logger.info('Results computed with the **unofficial** Python eval code.')
logger.info('Results should be very close to the official MATLAB code.')
logger.info('Use `./tools/reval.py --matlab ...` for your paper.')
logger.info('-- Thanks, The Management')
logger.info('----------------------------------------------------------')
def _do_python_eval(self, output_dir = 'output'):
annopath = os.path.join(self._data_path, 'xml', '{:s}.xml')
cachedir = os.path.join(self._data_path, 'annotations_cache', self._image_set)
aps = []
# The PASCAL VOC metric changed in 2010
# use_07_metric = True if int(self._year) < 2010 else False
use_07_metric = False
print 'VOC07 metric? ' + ('Yes' if use_07_metric else 'No')
if not os.path.isdir(output_dir):
os.mkdir(output_dir)
# for i, cls in enumerate(self._classes):
for i, cls in enumerate(['sofa']): # Only evaluate sofa
if cls == '__background__':
continue
filename = self._get_voc_results_file_template().format(cls)
rec, prec, ap = voc_eval(
filename, annopath, self._image_set_file, cls, cachedir, ovthresh=0.5,
use_07_metric=use_07_metric)
aps += [ap]
print('AP for {} = {:.4f}'.format(cls, ap))
with open(os.path.join(output_dir, cls + '_pr.pkl'), 'w') as f:
cPickle.dump({'rec': rec, 'prec': prec, 'ap': ap}, f)
print('Mean AP = {:.4f}'.format(np.mean(aps)))
print('~~~~~~~~')
print('Results:')
for ap in aps:
print('{:.3f}'.format(ap))
print('{:.3f}'.format(np.mean(aps)))
print('~~~~~~~~')
print('')
print('--------------------------------------------------------------')
print('Results computed with the **unofficial** Python eval code.')
print('Results should be very close to the official MATLAB eval code.')
print('Recompute with `./tools/reval.py --matlab ...` for your paper.')
print('-- Thanks, The Management')
print('--------------------------------------------------------------')
def _do_python_eval(self, output_dir='output'):
annopath = os.path.join(self._devkit_path, 'RGZ' + self._year,
'Annotations', '{:s}.xml')
imagesetfile = os.path.join(self._devkit_path, 'RGZ' + self._year,
'ImageSets', 'Main',
self._image_set + '.txt')
cachedir = os.path.join(self._devkit_path, 'annotations_cache')
aps = []
# The PASCAL VOC metric changed in 2010
use_07_metric = True if int(self._year) < 2010 else False
print('VOC07 metric? ' + ('Yes' if use_07_metric else 'No'))
if not os.path.isdir(output_dir):
os.mkdir(output_dir)
for i, cls in enumerate(self._classes):
if cls == '__background__':
continue
filename = self._get_voc_results_file_template().format(cls)
rec, prec, ap = voc_eval(filename,
annopath,
imagesetfile,
cls,
cachedir,
ovthresh=0.5,
use_07_metric=use_07_metric)
aps += [ap]
print('AP for {} = {:.4f}'.format(cls, ap))
with open(os.path.join(output_dir, cls + '_pr.pkl'), 'wb') as f:
cPickle.dump({'rec': rec, 'prec': prec, 'ap': ap}, f)
print('Mean AP = {:.4f}'.format(np.mean(aps)))
print('~~~~~~~~')
print('Results:')
for ap in aps:
print('{:.3f}'.format(ap))
print('{:.3f}'.format(np.mean(aps)))
print('~~~~~~~~')
print('')
def _do_python_eval(self, output_dir='output', path=None):
annopath = os.path.join(self._devkit_path, 'segmentation_detection',
'xml_annotations', '{:s}.xml')
imagesetfile = os.path.join(self._devkit_path, 'train_val_test',
self._image_set + '.txt')
cachedir = os.path.join(self._devkit_path, 'annotations_cache')
# The PASCAL VOC metric changed in 2010
use_07_metric = True if int(self._year) < 2010 else False
print('VOC07 metric? ' + ('Yes' if use_07_metric else 'No'))
if not os.path.isdir(output_dir):
os.mkdir(output_dir)
ovthresh_list = [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95]
for ovthresh in ovthresh_list:
res_file = open(
os.path.join('/DeepWatershedDetection' + path,
'res-' + str(ovthresh) + '.txt'), "w+")
aps = []
sum_aps, present = 0, 0
for i, cls in enumerate(self._classes):
if cls in [
'clef15', 'noteheadDoubleWholeSmall', 'flag64thUp',
'articTenutoBelow', 'rest64th', 'flag8thDownSmall',
'restHBar', 'caesura', 'restLonga', 'restMaxima',
'rest32nd', 'dynamicRinforzando2',
'noteheadWholeSmall', 'dynamicPPP', 'flag128thUp',
'flag64thDown', 'articStaccatissimoAbove',
'articStaccatissimoBelow', 'restDoubleWhole',
'noteheadDoubleWhole', 'articMarcatoBelow',
'fingering5', 'fingering4', 'fingering1', 'fingering0',
'fingering3', 'fingering2', 'timeSig9', 'timeSig5',
'timeSig0', 'flag128thDown', 'timeSig16', 'timeSig12',
'dynamicPPPPP', 'rest128th', 'dynamicFortePiano',
'flag32ndUp', 'noteheadHalfSmall', 'flag8thUpSmall',
'articMarcatoAbove'
]:
continue
filename = self._get_voc_results_file_template().format(cls)
rec, prec, ap = voc_eval(filename,
annopath,
imagesetfile,
cls,
cachedir,
ovthresh=ovthresh,
use_07_metric=use_07_metric)
aps += [ap]
print(('AP for {} = {:.4f}'.format(cls, ap)))
with open(os.path.join(output_dir, cls + '_pr.pkl'),
'wb') as f:
pickle.dump({'rec': rec, 'prec': prec, 'ap': ap}, f)
if math.isnan(ap):
res_file.write(cls + " " + str(0) + "\n")
else:
res_file.write(cls + " " + '{:.3f}'.format(ap) + "\n")
sum_aps += ap
present += 1
res_file.write('\n\n\n')
res_file.write("Mean Average Precision: " +
str(sum_aps / float(present)))
res_file.close()
print(('Mean AP = {:.4f}'.format(np.mean(aps))))
print('~~~~~~~~')
print('Results:')
print(('{:.3f}'.format(np.mean(aps))))
print('~~~~~~~~')
print('')
print('--------------------------------------------------------------')
print('Results computed with the **unofficial** Python eval code.')
print('Results should be very close to the official MATLAB eval code.')
print('Recompute with `./tools/reval.py --matlab ...` for your paper.')
print('-- Thanks, The Management')
print('--------------------------------------------------------------')
def _do_python_eval(self, output_dir='output', path=None):
annopath = os.path.join(self._devkit_path, 'segmentation_detection',
'xml_annotations', '{:s}.xml')
imagesetfile = os.path.join(self._devkit_path, 'train_val_test',
self._image_set + '.txt')
cachedir = os.path.join(self._devkit_path, 'annotations_cache')
# The PASCAL VOC metric changed in 2010
use_07_metric = True if int(self._year) < 2010 else False
print('VOC07 metric? ' + ('Yes' if use_07_metric else 'No'))
if not os.path.isdir(output_dir):
os.mkdir(output_dir)
ovthresh_list = [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95]
for ovthresh in ovthresh_list:
aps = []
for i, cls in enumerate(self._classes):
if cls == '__background__':
continue
filename = self._get_voc_results_file_template().format(cls)
rec, prec, ap = voc_eval(filename,
annopath,
imagesetfile,
cls,
cachedir,
ovthresh=ovthresh,
use_07_metric=use_07_metric)
aps += [ap]
print(('AP for {} = {:.4f}'.format(cls, ap)))
with open(os.path.join(output_dir, cls + '_pr.pkl'),
'wb') as f:
pickle.dump({'rec': rec, 'prec': prec, 'ap': ap}, f)
print(('Mean AP = {:.4f}'.format(np.mean(aps))))
print('~~~~~~~~')
print('Results:')
# open the file where we want to save the results
if path is not None:
res_file = open(
os.path.join('/DeepWatershedDetection' + path,
'res-' + str(ovthresh) + '.txt'), "w+")
len_ap = len(aps)
sum_aps = 0
present = 0
for i in range(len_ap):
print(('{:.3f}'.format(aps[i])))
if i not in [
68, 34, 35, 36, 90, 102, 39, 42, 75, 45, 48, 99,
20, 117, 118, 89, 25, 26, 74
]:
if math.isnan(aps[i]):
res_file.write(str(0) + "\n")
else:
res_file.write(('{:.3f}'.format(aps[i])) + "\n")
sum_aps += aps[i]
present += 1
res_file.write('\n\n\n')
res_file.write("Mean Average Precision: " +
str(sum_aps / float(present)))
res_file.close()
print(('{:.3f}'.format(np.mean(aps))))
print('~~~~~~~~')
print('')
print('--------------------------------------------------------------')
print('Results computed with the **unofficial** Python eval code.')
print('Results should be very close to the official MATLAB eval code.')
print('Recompute with `./tools/reval.py --matlab ...` for your paper.')
print('-- Thanks, The Management')
print('--------------------------------------------------------------')
def _do_python_eval(json_dataset, salt, output_dir='output'):
info = voc_info(json_dataset)
year = info['year']
anno_path = info['anno_path']
image_set_path = info['image_set_path']
devkit_path = info['devkit_path']
#cachedir = os.path.join(devkit_path, 'annotations_cache')
cachedir = os.path.join('data/VOC' + year, 'annotations_cache')
aps_50 = []
cls_ls = []
# The PASCAL VOC metric changed in 2010
use_07_metric = True if int(year) < 2010 else False
logger.info('VOC07 metric? ' + ('Yes' if use_07_metric else 'No'))
if not os.path.isdir(output_dir):
os.mkdir(output_dir)
# load gt
with open(image_set_path, 'r') as f:
lines = f.readlines()
imagenames = [x.strip() for x in lines]
# load annots
recs = {}
with open('./data/fsod/annotations/fsod_test.json', 'r') as f:
now_dict = json.load(f)
anno = pd.DataFrame.from_dict(now_dict['annotations'])
category = pd.DataFrame.from_dict(now_dict['categories'])
# we need image_id and category
for i, imagename in enumerate(imagenames):
image_now_ls = []
image_id, category_id, ep = imagename.strip().split('_')
category_name = category.loc[category['id'] == int(category_id),
'name'].tolist()[0]
now_df = anno.loc[(anno['image_id'] == int(image_id)) &
(anno['category_id'] == int(category_id)
), :] #anno[anno['image_id'] == int(image_id)]
obj_dict = now_df.to_dict('index')
for key, value in obj_dict.items():
value[
'name'] = category_name + '_' + ep #category_name #category[category['id'] == value['category_id']]['name'].values[0]
value['bbox'] = [
value['bbox'][0], value['bbox'][1],
value['bbox'][0] + value['bbox'][2],
value['bbox'][1] + value['bbox'][3]
]
#print(value['name'])
value['truncated'] = 0
value['difficult'] = 0
image_now_ls.append(value)
recs[imagename] = image_now_ls
if i % 100 == 0:
logger.info('Reading annotation for {:d}/{:d}'.format(
i + 1, len(imagenames)))
for _, cls in enumerate(json_dataset.classes):
if cls == '__background__':
continue
filename = _get_voc_results_file_template(json_dataset,
salt).format(cls)
rec, prec, ap = voc_eval(filename,
anno_path,
recs,
imagenames,
cls,
cachedir,
ovthresh=0.5,
use_07_metric=use_07_metric)
aps_50 += [ap]
cls_ls += [cls]
#logger.info('AP for {} = {:.4f}'.format(cls, ap))
res_file = os.path.join(output_dir, cls + '_50_pr.pkl')
save_object({'rec': rec, 'prec': prec, 'ap': ap}, res_file)
logger.info('Mean AP50 = {:.4f}'.format(np.mean(aps_50)))
logger.info('~~~~~~~~')
logger.info('AP50 Results:')
for ap_id, ap in enumerate(aps_50):
logger.info('AP50 for {} = {:.4f}'.format(cls_ls[ap_id], ap))
logger.info('~~~~~~~~')
logger.info('')
logger.info('----------------------------------------------------------')
logger.info('Results computed with the **unofficial** Python eval code.')
logger.info('Results should be very close to the official MATLAB code.')
logger.info('Use `./tools/reval.py --matlab ...` for your paper.')
logger.info('-- Thanks, The Management')
logger.info('----------------------------------------------------------')
aps_75 = []
cls_ls = []
for _, cls in enumerate(json_dataset.classes):
if cls == '__background__':
continue
filename = _get_voc_results_file_template(json_dataset,
salt).format(cls)
rec, prec, ap = voc_eval(filename,
anno_path,
recs,
imagenames,
cls,
cachedir,
ovthresh=0.75,
use_07_metric=use_07_metric)
aps_75 += [ap]
cls_ls += [cls]
#logger.info('AP for {} = {:.4f}'.format(cls, ap))
res_file = os.path.join(output_dir, cls + '_75_pr.pkl')
save_object({'rec': rec, 'prec': prec, 'ap': ap}, res_file)
logger.info('Mean AP75 = {:.4f}'.format(np.mean(aps_75)))
logger.info('~~~~~~~~')
logger.info('AP75 Results:')
for ap_id, ap in enumerate(aps_75):
logger.info('AP75 for {} = {:.4f}'.format(cls_ls[ap_id], ap))
logger.info('~~~~~~~~')
logger.info('Mean AP50 = {:.4f}'.format(np.mean(aps_50)))
logger.info('Mean AP75 = {:.4f}'.format(np.mean(aps_75)))
logger.info('~~~~~~~~')
logger.info('')
logger.info('----------------------------------------------------------')
logger.info('Results computed with the **unofficial** Python eval code.')
logger.info('Results should be very close to the official MATLAB code.')
logger.info('Use `./tools/reval.py --matlab ...` for your paper.')
logger.info('-- Thanks, The Management')
logger.info('----------------------------------------------------------')
def _do_python_eval(self, output_dir='output', path=None):
annopath = os.path.join(self._devkit_path, 'MUSICMA++_' + self._year,
'xml_annotations', '{:s}.xml')
imagesetfile = os.path.join(self._devkit_path, 'train_val_test',
self._image_set + '.txt')
cachedir = os.path.join(self._devkit_path, 'annotations_cache')
aps = []
# The PASCAL VOC metric changed in 2010
use_07_metric = True if int(self._year) < 2010 else False
print('VOC07 metric? ' + ('Yes' if use_07_metric else 'No'))
if not os.path.isdir(output_dir):
os.mkdir(output_dir)
# pdb.set_trace()
ovthresh_list = [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95]
for ovthresh in ovthresh_list:
res_file = open(
os.path.join('/DeepWatershedDetection' + path,
'res-' + str(ovthresh) + '.txt'), "w+")
aps = []
sum_aps, present = 0, 0
# pdb.set_trace()
for i, cls in enumerate(self._classes):
if cls in [
'tuple_bracket_line', 'other-clef',
'dotted_horizontal_spanner', 'letter_other',
'trill_wobble', 'numeral_0', 'letter_E', 'letter_S',
'curved-line_(tie-or-slur)', 'letter_g',
'arpeggio_wobble', 'transposition_text'
]: # or cls[:3] == 'let':
continue
# pdb.set_trace()
filename = self._get_voc_results_file_template().format(cls)
rec, prec, ap = voc_eval(filename,
annopath,
imagesetfile,
cls,
cachedir,
ovthresh=ovthresh,
use_07_metric=use_07_metric)
aps += [ap]
print(('AP for {} = {:.4f}'.format(cls, ap)))
with open(os.path.join(output_dir, cls + '_pr.pkl'),
'wb') as f:
pickle.dump({'rec': rec, 'prec': prec, 'ap': ap}, f)
if math.isnan(ap):
res_file.write(cls + " " + ap + "\n")
else:
res_file.write(cls + " " + '{:.3f}'.format(ap) + "\n")
sum_aps += ap
present += 1
res_file.write('\n\n\n')
res_file.write("Mean Average Precision: " +
str(sum_aps / float(present)))
res_file.close()
print(('Mean AP = {:.4f}'.format(np.mean(aps))))
print('~~~~~~~~')
print('Results:')
print(('{:.3f}'.format(np.mean(aps))))
print('~~~~~~~~')
print('')
print('--------------------------------------------------------------')
print('Results computed with the **unofficial** Python eval code.')
print('Results should be very close to the official MATLAB eval code.')
print('Recompute with `./tools/reval.py --matlab ...` for your paper.')
print('-- Thanks, The Management')
print('--------------------------------------------------------------')
