def analysis_for_exter_class_overlap(self,
model_path,
model,
WIDTH,
HEIGHT,
top=12):
if model is None:
model, _ = load_model(model_path)
model.eval()
# use 100 images/class to calculate the class center
test_pictures = get_training_set_list(self.test_dir,
train_limit=100,
random_training_set=False)
self._get_avg_feature_for_all([model],
weight_ls=[1.0],
WIDTH=WIDTH,
HEIGHT=HEIGHT,
test_pictures=test_pictures)
all_test_pkls_dir = './results/temp/%s_all_test_pkls' % self.prefix.split(
'/')[0]
feature_map_for_all = self._calc_true_avg_feature(all_test_pkls_dir)
exter_class_distance_dict, _ = self._calc_exter_class_distance(
feature_map_for_all)
distance_dict = self._calc_inter_distance(all_test_pkls_dir,
feature_map_for_all)
variance_dict = self._calc_variance_each_class(distance_dict)
# use exter class distance + class variance to calculate the overlap rate of class pairs
overlap_rate_dict = {}
variance_dict_len = len(variance_dict.keys())
for class_id in range(1, variance_dict_len + 2):
for class_id_second in range(class_id + 1, variance_dict_len + 1):
key = '%d-%d' % (class_id, class_id_second)
if key not in exter_class_distance_dict:
key = '%s-%s' % (class_id_second, class_id)
if key not in exter_class_distance_dict:
continue
overlap_rate = (variance_dict[str(class_id)] +
variance_dict[str(class_id_second)] -
exter_class_distance_dict[key]
) / exter_class_distance_dict[key]
overlap_rate_dict[key] = overlap_rate
overlap_rate_dict_ls = sorted(overlap_rate_dict.items(),
key=lambda x: x[1])
overlap_rate_dict_ls.reverse()
_exter_class_top = []
for key, _ in overlap_rate_dict_ls[:top]:
first_id = key.split('-')[0]
second_id = key.split('-')[1]
if first_id not in _exter_class_top:
_exter_class_top.append(first_id)
if second_id not in _exter_class_top:
_exter_class_top.append(second_id)
return _exter_class_top, overlap_rate_dict_ls
def analysis_for_inter_exter_acc(self,
model_path,
WIDTH,
HEIGHT,
model_type=None):
model, _ = load_model(model_path)
model.eval()
if model_type is not None:
if model_type == 'resnet':
WIDTH = HEIGHT = 224
elif model_type == 'inception3':
WIDTH = HEIGHT = 299
feature_map = get_feature_map_average(
[model],
sample_num_each_cls=5,
sample_file_dir=self.sample_file_dir,
temp_prefix=self.prefix.split('/')[0],
weight_ls=[1.0],
WIDTH=WIDTH,
HEIGHT=HEIGHT)
acc = predict_pictures(feature_map, [model],
[i for i in range(1, self.num_of_classes)],
test_dir=self.test_dir,
temp_prefix=self.prefix.split('/')[0],
seen='seen',
weight_ls=[1.0],
WIDTH=WIDTH,
HEIGHT=HEIGHT)
print('accuracy in analysis_for_inter_exter_acc:',
'%.2f' % (acc * 100))
class_acc = self.calc_predict_result(
class_count_dict_path='./results/temp/%s_class_count_dict_seen.pkl'
% self.prefix.split('/')[0],
predict_dict_path='./results/temp/%s_predict_label_dict_seen.pkl' %
self.prefix.split('/')[0])
all_test_pkls_dir = './results/temp/%s_all_test_pkls' % self.prefix.split(
'/')[0]
feature_map_for_all = self._calc_true_avg_feature(all_test_pkls_dir)
exter_class_distance_dict, _ = self._calc_exter_class_distance(
feature_map_for_all)
distance_dict = self._calc_inter_distance(all_test_pkls_dir)
variance_dict = self._calc_variance_each_class(distance_dict)
self.visualize_exter_class_distance(exter_class_distance_dict)
self.visualize_class_variance(variance_dict, class_acc)
def __init__(self,
model_path,
_range,
sample_file_dir,
test_dir,
prefix,
WIDTH,
HEIGHT,
sample_num_each_cls=5):
# tablewares_mapping_path=None) :
self.models, _ = load_model(model_path)
self.models = [self.models]
self._range = _range
self.sample_file_dir = sample_file_dir
self.test_dir = test_dir
self.prefix = prefix
self.sample_num_each_cls = sample_num_each_cls
def __init__(self,
model_type='resnet',
load_model_path=None,
num_of_classes=0,
update_conv_layers=.05,
method='metric'):
self.model_type = model_type
self.load_model_path = load_model_path
self.num_of_classes = num_of_classes
self.update_conv_layers = update_conv_layers
self.method = method
if model_type == 'resnet':
self.model, self.optim_policy = load_model(
model_path=load_model_path, num_of_classes=num_of_classes)
if model_type == 'inception3':
self.model, self.optim_policy = load_inception3(
model_path=load_model_path)
print('model size: {:.5f}M'.format(
sum(p.numel() for p in self.model.parameters()) / 1e6))
# exclude_pictures = set(test_pictures) - set(keep_normalized_picture_ls)
# test_exclude_dir = '/home/ubuntu/Program/Dish_recognition/dataset/test_exclude/%6.6f' % time.time()
# if not os.path.exists(test_exclude_dir):
# os.makedirs(test_exclude_dir)
# for image in exclude_pictures:
# shutil.move( os.path.join(self.test_dir, image),
# os.path.join(test_exclude_dir, image) )
# print('test pictures are made more balanced for testing. now get pictures:', len(os.listdir(self.test_dir)))
# keep_classes_set = set(class_count_dict.keys()) - set(exclude_class_ls)
# keep_test_classes = [x for x in keep_classes_set]
# print('Testing', len(keep_test_classes), 'classes.', keep_test_classes)
if __name__ == '__main__':
"""
Example of evaluate a model
"""
model, _ = load_model('/home/ubuntu/Program/xhq/TablewareFinetunePro-V3/model/model_fine-tuned.tar')
tester = Tester(model_path = None,
model=model,
test_dir = '/home/ubuntu/Program/xhq/TablewareFinetunePro-V3/test_set',
sample_file_dir = '/home/ubuntu/Program/xhq/TablewareFinetunePro-V3/base_sample',
prefix='tableware',
input_w=300,
input_h=300)
acc = tester.evaluate_with_models()
print('Accuracy:', acc)
'test pictures are made more balanced for testing. now get pictures:',
len(os.listdir(test_dir)))
keep_classes_set = set(class_count_dict.keys()) - set(exclude_class_ls)
return [x for x in keep_classes_set]
if __name__ == '__main__':
"""
Example of evaluate a model
"""
o_model_path = '/home/ubuntu/Program/Dish_recognition/program/model/chawdoe-hardsample-o-o-run-1-augmentation/a_o_o_8/121_' \
'resnet_metric_conv0.05.tar'
n10_model_path = '/home/ubuntu/Program/Dish_recognition/program/model/chawdoe-hardsample-(o->n10)-n-run-1-augmentation/a_o->n10_2/' \
'chawdoe-hardsample-(o->n10)-n-run-1-augmentation_metric.tar'
model, _ = load_model(model_path=n10_model_path)
acc = evaluate_with_models(
[model],
_range=43,
test_dir='/home/ubuntu/Program/fsl/54c/test/',
sample_file_dir='/home/ubuntu/Program/fsl/54c/n10/',
temp_prefix='feng-oldModel-new54ctest-n10',
balance_testset=False,
WIDTH=300,
HEIGHT=300)
print(acc)
"""
Example of predict a picture
"""
# model = load_model(model_path='model/pytorch-ckpt/model_best.pth.tar')
# feature_map = '/path/to/feature_map'