def test_resnet_resnet50(self):
with _test_eager_guard():
model = resnet50(pretrained=False)
egr_data = paddle.to_tensor(self.data)
egr_data.stop_gradient = False
egr_out = model(egr_data)
egr_preds = paddle.argmax(egr_out, axis=1)
egr_label_onehot = paddle.nn.functional.one_hot(
paddle.to_tensor(egr_preds), num_classes=egr_out.shape[1])
egr_target = paddle.sum(egr_out * egr_label_onehot, axis=1)
egr_g = paddle.grad(outputs=egr_target, inputs=egr_out)[0]
egr_g_numpy = egr_g.numpy()
self.assertEqual(list(egr_g_numpy.shape), list(egr_out.shape))
model = resnet50(pretrained=False)
data = paddle.to_tensor(self.data)
data.stop_gradient = False
out = model(data)
preds = paddle.argmax(out, axis=1)
label_onehot = paddle.nn.functional.one_hot(paddle.to_tensor(preds),
num_classes=out.shape[1])
target = paddle.sum(out * label_onehot, axis=1)
g = paddle.grad(outputs=target, inputs=out)[0]
g_numpy = g.numpy()
self.assertEqual(list(g_numpy.shape), list(out.shape))
self.assertTrue(np.array_equal(egr_out, out))
self.assertTrue(np.array_equal(egr_g_numpy, g_numpy))
def test_shape(self):
paddle_model = resnet50(pretrained=True)
img_path = 'imgs/catdog.jpg'
algo = it.OcclusionInterpreter(paddle_model, device='cpu')
exp = algo.interpret(img_path,
sliding_window_shapes=(1, 32, 32),
strides=(1, 32, 32),
resize_to=64,
crop_to=64,
visual=False)
result = np.array([*exp.shape])
assert_arrays_almost_equal(self, result, np.array([1, 3, 64, 64]))
def get_resnet50_model():
main = paddle.static.Program()
startup = paddle.static.Program()
with paddle.static.program_guard(main, startup):
image = paddle.static.data(
name="image", shape=[None, 3, 224, 224], dtype="float32")
label = paddle.static.data(name="label", shape=[None, 1], dtype="int64")
model = resnet50()
loss_fn = CrossEntropyLoss()
pred = model(image)
loss = loss_fn(pred, label)
optimizer = paddle.optimizer.Adam(learning_rate=1e-3)
optimizer.minimize(loss)
return main, startup, image, label, loss
def test_algo(self):
from paddle.vision.models import resnet34, resnet50, resnet101, mobilenet_v2
# Here we use four models to give an illustration. Using more models shows more impressive results.
list_models = {
'resnet34': resnet34(pretrained=False),
'resnet50': resnet50(pretrained=False),
'resnet101': resnet101(pretrained=False),
'mobilenet_v2': mobilenet_v2(pretrained=False)
}
consensus = it.ConsensusInterpreter(it.SmoothGradInterpreter,
list_models.values(),
device='gpu:0')
img_path = np.random.randint(0,
255,
size=(1, 64, 64, 3),
dtype=np.uint8)
exp = consensus.interpret(img_path, n_samples=5)
def test_algo(self):
paddle_model = resnet50(pretrained=True)
np.random.seed(42)
img_path = np.random.randint(0,
255,
size=(1, 64, 64, 3),
dtype=np.uint8)
algo = it.OcclusionInterpreter(paddle_model, device='cpu')
exp = algo.interpret(img_path,
sliding_window_shapes=(1, 32, 32),
strides=(1, 32, 32),
resize_to=64,
crop_to=64,
visual=False)
result = np.array([exp.mean(), exp.std(), exp.min(), exp.max()])
desired = np.array([-0.11518122, 0.16784915, -0.46479446, 0.11298946])
assert_arrays_almost_equal(self, result, desired)
def train():
paddle.set_device('gpu')
model = resnet50()
paddle.summary(model, (1, 3, 32, 32))
transform = Compose([
paddle.vision.transforms.Transpose(),
paddle.vision.transforms.Normalize(0, 255.),
])
cifar10 = Cifar10(mode='train', transform=transform)
loader = paddle.io.DataLoader(cifar10,
shuffle=True,
batch_size=BATCH_SIZE,
num_workers=10)
for epoch in range(EPOCH_NUM):
for batch_id, data in enumerate(loader()):
out = model(data[0])
out = paddle.mean(out)
if batch_id % 10 == 0:
print("Epoch {}: batch {}, out {}".format(
epoch, batch_id, out.numpy()))
def test_save(self):
import matplotlib
matplotlib.use('agg') # non-GUI, for skipping.
paddle_model = resnet50(pretrained=True)
np.random.seed(42)
img_path = np.random.randint(0,
255,
size=(1, 64, 64, 3),
dtype=np.uint8)
algo = it.OcclusionInterpreter(paddle_model, device='cpu')
exp = algo.interpret(img_path,
sliding_window_shapes=(1, 33, 33),
strides=(1, 32, 32),
resize_to=64,
crop_to=64,
visual=True,
save_path='tmp.jpg')
result = np.array([exp.mean(), exp.std(), exp.min(), exp.max()])
desired = np.array([-0.12583457, 0.20028804, -0.8618185, 0.2236681])
assert_arrays_almost_equal(self, result, desired)
os.remove('tmp.jpg')
train_datasets = pdl.vision.datasets.Flowers(mode='train')
val_datasets = pdl.vision.datasets.Flowers(mode='test')
train_reader = Reader(train_datasets)
val_reader = Reader(val_datasets)
train_reader = pdl.io.DataLoader(train_reader,
batch_size=1,
num_workers=0,
use_shared_memory=True)
val_reader = pdl.io.DataLoader(val_reader,
batch_size=1,
num_workers=4,
use_shared_memory=False)
net = pdl.nn.Sequential(
resnet50(pretrained=True, num_classes=1000, with_pool=True),
pdl.nn.Linear(1000, 102))
# 定于输入层
input_define = pdl.static.InputSpec(shape=[-1, 3, 256, 256],
dtype="float32",
name="img")
# 定义Label输入层，用于计算损失和准确率
label_define = pdl.static.InputSpec(shape=[-1, 1], dtype="int64", name="label")
# 实例化网络对象并定义优化器等训练逻辑
model = pdl.Model(net, inputs=input_define, labels=label_define)
# 此处使用SGD优化器，可尝试使用Adam，收敛效果更好更快速
optimizer = pdl.optimizer.SGD(learning_rate=0.001,
parameters=model.parameters())
# 损失函数使用交叉熵，评价指标使用准确率
# 其中Top-k表示推理得到的概率分布中，概率最高的前k个推理结果中是否包含正确标签，如果包含则视为正确，这里的1，2，3分别计算k为1~3的情况
def setUp(self):
model = resnet50()
sp_net_config = supernet(expand_ratio=[0.25, 0.5, 1.0])
self.model = Convert(sp_net_config).convert(model)
self.images = paddle.randn(shape=[2, 3, 224, 224], dtype='float32')
self._test_clear_search_space()
def setUp(self):
self.model = resnet50(pretrained=False)
'''
import paddle
# 从paddle.vision.models 模块中import 残差网络，VGG网络，LeNet网络
from paddle.vision.models import resnet50, vgg16, LeNet
from paddle.vision.datasets import Cifar10
from paddle.optimizer import Momentum
from paddle.regularizer import L2Decay
from paddle.nn import CrossEntropyLoss
from paddle.metric import Accuracy
from paddle.vision.transforms import Transpose
# 确保从paddle.vision.datasets.Cifar10中加载的图像数据是np.ndarray类型
paddle.vision.set_image_backend('cv2')
# 调用resnet50模型
model = paddle.Model(resnet50(pretrained=False, num_classes=10))
# 使用Cifar10数据集
train_dataset = Cifar10(mode='train', transform=Transpose())
val_dataset = Cifar10(mode='test', transform=Transpose())
# 定义优化器
optimizer = Momentum(learning_rate=0.01,
momentum=0.9,
weight_decay=L2Decay(1e-4),
parameters=model.parameters())
# 进行训练前准备
model.prepare(optimizer, CrossEntropyLoss(), Accuracy(topk=(1, 5)))
# 启动训练
model.fit(train_dataset,
val_dataset,
epochs=50,
from paddle.vision.transforms import Compose, Resize, ToTensor
from paddle.vision.models import resnet50
from paddle.vision.datasets import Cifar100
# 导入RIFLE模块
from paddle_rifle.rifle import RIFLECallback
# 定义数据预处理
transform = Compose([Resize(224), ToTensor()])
# 加载Cifar100数据集
train_data = Cifar100(transform=transform)
test_data = Cifar100(mode="test", transform=transform)
# 加载Resnet50
net = resnet50(True, num_classes=100)
# 获取Resnet50的输出层
fc_layer = net.fc
"""
# 自定义网络场景下的输出层获取示例
class Net(paddle.nn.Layer):
def __init__(self):
super(Net, self).__init__()
self.layer1 = xxx
self.layer2 = xxx
self.输出层 = paddle.nn.Linear(...)
...
# 实例化Net类
net = Net()
def main():
args = parse_args()
# device setup
paddle.set_device("gpu")
# load model
model = resnet50(pretrained=True)
model.eval()
# set model's intermediate outputs
outputs = []
def hook(module, input, output):
outputs.append(output)
model.layer1[-1].register_forward_post_hook(hook)
model.layer2[-1].register_forward_post_hook(hook)
model.layer3[-1].register_forward_post_hook(hook)
model.avgpool.register_forward_post_hook(hook)
os.makedirs(os.path.join(args.save_path, 'temp'), exist_ok=True)
fig, ax = plt.subplots(1, 2, figsize=(20, 10))
fig_img_rocauc = ax[0]
fig_pixel_rocauc = ax[1]
total_roc_auc = []
total_pixel_roc_auc = []
for class_name in mvtec.CLASS_NAMES:
train_dataset = mvtec.MVTecDataset(class_name=class_name,
is_train=True)
train_dataloader = DataLoader(train_dataset, batch_size=32)
test_dataset = mvtec.MVTecDataset(class_name=class_name,
is_train=False)
test_dataloader = DataLoader(test_dataset, batch_size=1)
train_outputs = OrderedDict([('layer1', []), ('layer2', []),
('layer3', []), ('avgpool', [])])
test_outputs = OrderedDict([('layer1', []), ('layer2', []),
('layer3', []), ('avgpool', [])])
# extract train set features
train_feature_filepath = os.path.join(args.save_path, 'temp',
'train_%s.pkl' % class_name)
if not os.path.exists(train_feature_filepath):
# extract train set features
for (x, y, mask) in tqdm(
train_dataloader,
'| feature extraction | train | %s |' % class_name):
with paddle.no_grad():
pred = model(x)
for k, v in zip(train_outputs.keys(), outputs):
train_outputs[k].append(v)
# initialize hook outputs
outputs = []
# transfer to np.array in order to save by pickle
for k, v in train_outputs.items():
train_outputs[k] = paddle.concat(v, 0).numpy()
with open(train_feature_filepath, 'wb') as f:
pickle.dump(train_outputs, f)
# transfer back to paddle.Tensor() in order to continue to compute in test stage
for k, v in train_outputs.items():
train_outputs[k] = paddle.to_tensor(v)
else:
print('load train set feature from: %s' % train_feature_filepath)
with open(train_feature_filepath, 'rb') as f:
train_outputs = pickle.load(f)
for k, v in train_outputs.items():
train_outputs[k] = paddle.to_tensor(v)
gt_list = []
gt_mask_list = []
test_imgs = []
# extract test set features
for (x, y,
mask) in tqdm(test_dataloader,
'| feature extraction | test | %s |' % class_name):
test_imgs.extend(x.cpu().detach().numpy())
gt_list.extend(y.cpu().detach().numpy())
gt_mask_list.extend(mask.cpu().detach().numpy())
# model prediction
with paddle.no_grad():
pred = model(x)
for k, v in zip(test_outputs.keys(), outputs):
test_outputs[k].append(v)
# initialize hook outputs
outputs = []
for k, v in test_outputs.items():
test_outputs[k] = paddle.concat(v, 0)
# calculate distance matrix
dist_matrix = calc_dist_matrix(
paddle.flatten(test_outputs['avgpool'], 1),
paddle.flatten(train_outputs['avgpool'], 1))
# select K nearest neighbor and take average
topk_values, topk_indexes = paddle.topk(dist_matrix,
k=args.top_k,
largest=False)
scores = paddle.mean(topk_values, 1).cpu().detach().numpy()
# calculate image-level ROC AUC score
fpr, tpr, _ = roc_curve(gt_list, scores)
roc_auc = roc_auc_score(gt_list, scores)
total_roc_auc.append(roc_auc)
print('%s ROCAUC: %.3f' % (class_name, roc_auc))
fig_img_rocauc.plot(fpr,
tpr,
label='%s ROCAUC: %.3f' % (class_name, roc_auc))
score_map_list = []
for t_idx in tqdm(range(test_outputs['avgpool'].shape[0]),
'| localization | test | %s |' % class_name):
score_maps = []
for layer_name in ['layer1', 'layer2', 'layer3']: # for each layer
# construct a gallery of features at all pixel locations of the K nearest neighbors
topk_feat_map = paddle.stack([
train_outputs[layer_name][idx]
for idx in topk_indexes[t_idx].numpy().tolist()
])
test_feat_map = test_outputs[layer_name][t_idx:t_idx + 1]
feat_gallery = paddle.transpose(topk_feat_map, [0, 3, 2, 1])
feat_gallery = paddle.flatten(
feat_gallery, start_axis=0,
stop_axis=2).unsqueeze(-1).unsqueeze(-1)
# calculate distance matrix
dist_matrix_list = []
for d_idx in range(feat_gallery.shape[0] // 100):
dist = paddle.nn.PairwiseDistance()
dist_matrix = dist(
feat_gallery[d_idx * 100:d_idx * 100 + 100],
test_feat_map)
dist_matrix_list.append(dist_matrix)
dist_matrix = paddle.concat(dist_matrix_list, 0)
# k nearest features from the gallery (k=1)
score_map = paddle.min(dist_matrix, axis=0)
score_map = F.interpolate(score_map.unsqueeze(0).unsqueeze(0),
size=[224, 224],
mode='bilinear',
align_corners=False)
score_maps.append(score_map)
# average distance between the features
score_map = paddle.mean(paddle.concat(score_maps, 0), axis=0)
# apply gaussian smoothing on the score map
score_map = gaussian_filter(
score_map.squeeze().cpu().detach().numpy(), sigma=4)
score_map_list.append(score_map)
flatten_gt_mask_list = np.concatenate(gt_mask_list).ravel().astype(int)
flatten_score_map_list = np.concatenate(score_map_list).ravel()
# calculate per-pixel level ROCAUC
fpr, tpr, _ = roc_curve(flatten_gt_mask_list, flatten_score_map_list)
per_pixel_rocauc = roc_auc_score(flatten_gt_mask_list,
flatten_score_map_list)
total_pixel_roc_auc.append(per_pixel_rocauc)
print('%s pixel ROCAUC: %.3f' % (class_name, per_pixel_rocauc))
fig_pixel_rocauc.plot(fpr,
tpr,
label='%s ROCAUC: %.3f' %
(class_name, per_pixel_rocauc))
# get optimal threshold
precision, recall, thresholds = precision_recall_curve(
flatten_gt_mask_list, flatten_score_map_list)
a = 2 * precision * recall
b = precision + recall
f1 = np.divide(a, b, out=np.zeros_like(a), where=b != 0)
threshold = thresholds[np.nanargmax(f1)]
# visualize localization result
visualize_loc_result(test_imgs,
gt_mask_list,
score_map_list,
threshold,
args.save_path,
class_name,
vis_num=5)
print('Average ROCAUC: %.3f' % np.nanmean(total_roc_auc))
fig_img_rocauc.title.set_text('Average image ROCAUC: %.3f' %
np.nanmean(total_roc_auc))
fig_img_rocauc.legend(loc="lower right")
print('Average pixel ROCUAC: %.3f' % np.nanmean(total_pixel_roc_auc))
fig_pixel_rocauc.title.set_text('Average pixel ROCAUC: %.3f' %
np.nanmean(total_pixel_roc_auc))
fig_pixel_rocauc.legend(loc="lower right")
fig.tight_layout()
fig.savefig(os.path.join(args.save_path, 'roc_curve.png'), dpi=100)
def main(args):
# data
def sort_key_func(x):
# for imagenet val set.
return x.split('/')[-1]
if '*' in args.data_list:
data_list = args.data_list.replace('\\', '')
files = glob(data_list)
np.random.seed(0)
files = np.random.permutation(files)
list_image_paths = files[:args.num_images]
print(args.data_list)
print(len(list_image_paths))
# model
model_init_args = {'pretrained': True, 'num_classes': args.num_classes}
if args.model_weights is not None:
model_init_args['pretrained'] = False
if args.model.lower() == 'resnet50':
if 'lrp' == args.it:
from tutorials.assets.lrp_model import resnet50_lrp
paddle_model = resnet50_lrp(**model_init_args)
else:
paddle_model = resnet50(**model_init_args)
elif args.model.lower() == 'resnet101':
paddle_model = resnet101(**model_init_args)
else:
paddle_model = resnet50(**model_init_args)
## load weights if given
if args.model_weights is not None:
state_dict = paddle.load(args.model_weights)
paddle_model.set_dict(state_dict)
print("Load weights from", args.model_weights)
# interpreter instance
to_test_list = {
'lime': it.LIMECVInterpreter,
'gradcam': it.GradCAMInterpreter,
'intgrad': it.IntGradCVInterpreter,
'smoothgrad': it.SmoothGradInterpreter,
'gradshap': it.GradShapCVInterpreter,
'scorecam': it.ScoreCAMInterpreter,
'glime': it.GLIMECVInterpreter,
'lrp': it.LRPCVInterpreter
}
interpreter = to_test_list[args.it](paddle_model, device=args.device)
# interpreter configs
it_configs = args.it_configs
# evaluation configs
eval_configs = args.eval_configs
# image resize config.
# depreciated set: {"resize_to": 256, "crop_to": 224}
if args.img_resize_config is None:
img_resize_config = {"resize_to": 224, "crop_to": 224}
else:
img_resize_config = args.img_resize_config
if 'glime' == args.it:
interpreter.set_global_weights(args.global_weights)
num_limit_adapter = {}
if args.eval_num_limit_adapter is not None:
lime_results = dict(
np.load(args.eval_num_limit_adapter, allow_pickle=True))
img_path_list = list(lime_results.keys())
for i in range(len(img_path_list)):
img_path = img_path_list[i]
b = lime_results[img_path].item()
num_limit_adapter[img_path] = len(np.unique(b['segmentation']))
# evaluator instance
del_ins_evaluator = it.DeletionInsertion(paddle_model, device=args.device)
pert_evaluator = it.Perturbation(paddle_model,
device=args.device,
compute_MoRF=False)
# compute exp
del_scores = []
ins_scores = []
LeRF_scores = []
eval_results = {}
i = 1
if os.path.exists(f'./work_dirs/{get_exp_id(args)}.npz'):
eval_results = dict(
np.load(f'./work_dirs/{get_exp_id(args)}.npz', allow_pickle=True))
for img_path in tqdm(list_image_paths, leave=True, position=0):
if args.it == 'lime' or args.it == 'glime':
if img_path in eval_results:
exp = eval_results[img_path].item()
else:
exp = interpreter.interpret(img_path,
**it_configs,
**img_resize_config,
visual=False)
if hasattr(interpreter, 'lime_results'):
exp = interpreter.lime_results
else:
exp = interpreter.interpret(img_path,
**it_configs,
**img_resize_config,
visual=False)
if img_path in num_limit_adapter:
eval_configs['limit_number_generated_samples'] = num_limit_adapter[
img_path]
print(img_path, 'update eval_configs:', eval_configs)
results = del_ins_evaluator.evaluate(img_path, exp, **eval_configs,
**img_resize_config)
del_scores.append(results['del_probas'].mean())
ins_scores.append(results['ins_probas'].mean())
# print(results['del_probas'])
# print(results['ins_probas'])
# results = pert_evaluator.evaluate(img_path, exp, **eval_configs)
# LeRF_scores.append(results['LeRF_score'])
# print(results['LeRF_probas'])
if args.it == 'lime' or args.it == 'glime':
exp['del_probas'] = results['del_probas']
exp['ins_probas'] = results['ins_probas']
eval_results[img_path] = copy.deepcopy(exp)
np.savez(f'./work_dirs/{get_exp_id(args)}.npz', **eval_results)
else:
eval_results[img_path] = {
'del_probas': results['del_probas'],
'ins_probas': results['ins_probas']
}
np.savez(f'./work_dirs/{get_exp_id(args)}.npz', **eval_results)
if i % 20 == 0:
print("Del score:\t", sum(del_scores) / len(del_scores))
print("Ins score:\t", sum(ins_scores) / len(ins_scores))
logging.info(f"{i}")
logging.info(
f"Del score:\t {sum(del_scores) / len(del_scores): .5f}")
logging.info(
f"Ins score:\t {sum(ins_scores) / len(ins_scores): .5f}")
i += 1
print("Del score:\t", sum(del_scores) / len(del_scores))
print("Ins score:\t", sum(ins_scores) / len(ins_scores))
# print("LeRF score:\t", sum(LeRF_scores) / len(LeRF_scores))
logging.info(f"Del score:\t {sum(del_scores) / len(del_scores): .5f}")
logging.info(f"Ins score:\t {sum(ins_scores) / len(ins_scores): .5f}")
# logging.info(f"LeRF score:\t {sum(LeRF_scores) / len(LeRF_scores): .5f}")
logging.info(
f"{sum(del_scores) / len(del_scores): .5f} \t {sum(ins_scores) / len(ins_scores): .5f}"
)
# logging.info(f"{sum(del_scores) / len(del_scores): .5f} \t {sum(ins_scores) / len(ins_scores): .5f} \t {sum(LeRF_scores) / len(LeRF_scores): .5f}")
return
