def convert_model(data='voc', load_name='weights/voc/voc_pretrained.npy'): if data == 'coco': classes = load_classes('data/coco.names') classes = tuple(classes + ['__background__']) elif 'voc' in data: classes = ('aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', 'car', 'cat', 'chair', 'cow', 'diningtable', 'dog', 'horse', 'motorbike', 'person', 'pottedplant', 'sheep', 'sofa', 'train', 'tvmonitor', '__background__') model = resnet(classes, 50, pretrained=False, class_agnostic=False) print("load checkpoint %s" % (load_name)) if load_name.endswith('.pt'): checkpoint = torch.load(load_name) checkpoint['model'] = { k: v for k, v in checkpoint['model'].items() if model.state_dict()[k].numel() == v.numel() } model.load_state_dict(checkpoint['model'], strict=True) elif load_name.endswith('.npy'): checkpoint = np.load(load_name, allow_pickle=True).item() model_dict = { k: torch.from_numpy(checkpoint[k]) for k in checkpoint.keys() if model.state_dict()[k].numel() == torch.from_numpy(checkpoint[k]).numel() } model.load_state_dict(model_dict, strict=True) if load_name.endswith('.pt'): params = {} m_dict = model.state_dict() for key in m_dict.keys(): params[key] = m_dict[key].detach().cpu().numpy() np.save(load_name.replace('pt', 'npy'), params) elif load_name.endswith('.npy'): chkpt = { 'model': model.module.state_dict() if type(model) is nn.parallel.DistributedDataParallel else model.state_dict(), } torch.save(chkpt, load_name.replace('npy', 'pt')) print('save successfully')
num_boxes = num_boxes.cuda() gt_boxes = gt_boxes.cuda() # make variable im_data = Variable(im_data) im_info = Variable(im_info) num_boxes = Variable(num_boxes) gt_boxes = Variable(gt_boxes) if args.cuda: cfg.CUDA = True # initilize the network here. if args.net == 'res101': FPN = resnet(imdb.classes, 101, pretrained=True, class_agnostic=args.class_agnostic) elif args.net == 'res50': FPN = resnet(imdb.classes, 50, pretrained=True, class_agnostic=args.class_agnostic) elif args.net == 'res152': FPN = resnet(imdb.classes, 152, pretrained=True, class_agnostic=args.class_agnostic) else: print("network is not defined") pdb.set_trace()
print('{:d} roidb entries'.format(len(roidb))) if args.exp_name is not None: input_dir = args.load_dir + "/" + args.net + "/" + args.dataset + '/' + args.exp_name else: input_dir = args.load_dir + "/" + args.net + "/" + args.dataset if not os.path.exists(input_dir): raise Exception('There is no input directory for loading network from ' + input_dir) load_name = os.path.join(input_dir, 'fpn_{}_{}_{}.pth'.format(args.checksession, args.checkepoch, args.checkpoint)) # initilize the network here. if args.net == 'vgg16': fpn = vgg16(imdb.classes, pretrained=False, class_agnostic=args.class_agnostic) elif args.net == 'res101': fpn = resnet(imdb.classes, 101, pretrained=False, class_agnostic=args.class_agnostic) elif args.net == 'res50': fpn = resnet(imdb.classes, 50, pretrained=False, class_agnostic=args.class_agnostic) elif args.net == 'res152': fpn = resnet(imdb.classes, 152, pretrained=False, class_agnostic=args.class_agnostic) else: print("network is not defined") pdb.set_trace() fpn.create_architecture() print("load checkpoint %s" % (load_name)) checkpoint = torch.load(load_name) fpn.load_state_dict(checkpoint['model']) if 'pooling_mode' in checkpoint.keys(): cfg.POOLING_MODE = checkpoint['pooling_mode']
input_dir, 'fpn_{}_{}_{}.pth'.format(args.checksession, args.checkepoch, args.checkpoint)) pascal_classes = np.asarray([ '__background__', 'defect0', 'defect1', 'defect2', 'defect3', 'defect4', 'defect5', 'defect6', 'defect7', 'defect8', 'defect9' ]) # initilize the network here. if args.net == 'vgg16': fasterRCNN = vgg16(pascal_classes, pretrained=False, class_agnostic=args.class_agnostic) elif args.net == 'res101': fasterRCNN = resnet(pascal_classes, 101, pretrained=False, class_agnostic=args.class_agnostic) elif args.net == 'res50': fasterRCNN = resnet(pascal_classes, 50, pretrained=False, class_agnostic=args.class_agnostic) elif args.net == 'res152': fasterRCNN = resnet(pascal_classes, 152, pretrained=False, class_agnostic=args.class_agnostic) else: print("network is not defined") pdb.set_trace()
source = args.source cfg_from_file('cfgs/res50.yml') cfg.TRAIN.USE_FLIPPED = False cfg.POOLING_MODE = 'align' init_seeds(cfg.RNG_SEED) if data == 'coco': classes = load_classes('data/coco.names') classes = tuple(classes + ['__background__']) elif 'voc' in data: classes = ('aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', 'car', 'cat', 'chair', 'cow', 'diningtable', 'dog', 'horse', 'motorbike', 'person', 'pottedplant', 'sheep', 'sofa', 'train', 'tvmonitor', '__background__') model = resnet(classes, 50, pretrained=False, class_agnostic=False) print("load checkpoint %s" % (load_name)) if load_name.endswith('.pt'): checkpoint = torch.load(load_name) checkpoint['model'] = { k: v for k, v in checkpoint['model'].items() if model.state_dict()[k].numel() == v.numel() } model.load_state_dict(checkpoint['model'], strict=True) elif load_name.endswith('.npy'): checkpoint = np.load(load_name, allow_pickle=True).item() model_dict = { k: torch.from_numpy(checkpoint[k]) for k in checkpoint.keys() if model.state_dict()[k].numel() == torch.from_numpy(checkpoint[k]).numel()
def test_adv(step_size=0.01, num_steps=0, dataset='coco', batch_size=1, weights='weights/voc_pretrained.npy', save=False, grad_cam=False): if save: p_t1 = 'detect_adv_normal' if not os.path.exists(p_t1): os.makedirs(p_t1) cfg_file = 'cfgs/res50.yml' cfg_from_file(cfg_file) cfg.POOLING_MODE = 'align' cfg.TRAIN.USE_FLIPPED = False init_seeds(cfg.RNG_SEED) if dataset == "pascal_voc": imdb_name = "voc_2007_trainval" imdbval_name = "voc_2007_test" set_cfgs = [ 'ANCHOR_SCALES', '[8, 16, 32]', 'ANCHOR_RATIOS', '[0.5,1,2]' ] elif dataset == "pascal_voc_0712": imdb_name = "voc_2007_trainval+voc_2012_trainval" imdbval_name = "voc_2007_test" set_cfgs = [ 'ANCHOR_SCALES', '[8, 16, 32]', 'ANCHOR_RATIOS', '[0.5,1,2]' ] elif dataset == "coco": imdb_name = "coco_2014_train+coco_2014_valminusminival" imdbval_name = "coco_2014_minival" set_cfgs = [ 'ANCHOR_SCALES', '[4, 8, 16, 32]', 'ANCHOR_RATIOS', '[0.5,1,2]' ] elif dataset == "imagenet": imdb_name = "imagenet_train" imdbval_name = "imagenet_val" set_cfgs = [ 'ANCHOR_SCALES', '[8, 16, 32]', 'ANCHOR_RATIOS', '[0.5,1,2]' ] elif dataset == "vg": imdb_name = "vg_150-50-50_minitrain" imdbval_name = "vg_150-50-50_minival" set_cfgs = [ 'ANCHOR_SCALES', '[4, 8, 16, 32]', 'ANCHOR_RATIOS', '[0.5,1,2]' ] imdb, roidb, ratio_list, ratio_index = combined_roidb(imdbval_name, training=False) imdb.competition_mode(on=True) print('{:d} roidb entries'.format(len(roidb))) model = resnet(imdb.classes, 50, pretrained=False, class_agnostic=False) print("load checkpoint %s" % (weights)) if weights.endswith('.pt'): checkpoint = torch.load(weights) checkpoint['model'] = { k: v for k, v in checkpoint['model'].items() if model.state_dict()[k].numel() == v.numel() } model.load_state_dict(checkpoint['model'], strict=True) elif weights.endswith('.npy'): checkpoint = np.load(weights, allow_pickle=True).item() model_dict = { k: torch.from_numpy(checkpoint[k]) for k in checkpoint.keys() if model.state_dict()[k].numel() == torch.from_numpy(checkpoint[k]).numel() } model.load_state_dict(model_dict, strict=True) # load_state_dict(fpn.state_dict(), checkpoint['state_dict']) model.cuda().eval() del checkpoint print('load model successfully!') if not grad_cam: for param in model.parameters(): param.requires_grad = False model_adv = PGD(model) max_per_image = 100 vis = False thresh = 0.001 iou_thre = 0.5 colors = [[random.randint(0, 255) for _ in range(3)] for _ in range(len(imdb.classes))] num_images = len(imdb.image_index) all_boxes = [[[] for _ in xrange(num_images)] for _ in xrange(imdb.num_classes)] save_name = 'v1' output_dir = get_output_dir(imdb, save_name) dataset = roibatchLoader(roidb, ratio_list, ratio_index, batch_size, \ imdb.num_classes, training=False, normalize=False) dataloader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=False, num_workers=0, pin_memory=True) data_iter = iter(dataloader) empty_array = np.transpose(np.array([[], [], [], [], []]), (1, 0)) im_data = torch.FloatTensor(1).cuda() im_info = torch.FloatTensor(1).cuda() num_boxes = torch.LongTensor(1).cuda() gt_boxes = torch.FloatTensor(1).cuda() for i in range(num_images): data = next(data_iter) with torch.no_grad(): im_data.resize_(data[0].size()).copy_(data[0]) im_info.resize_(data[1].size()).copy_(data[1]) gt_boxes.resize_(data[2].size()).copy_(data[2]) num_boxes.resize_(data[3].size()).copy_(data[3]) if vis or save: im = cv2.imread(imdb.image_path_at(i)) im2show = np.copy(im) with torch.enable_grad(): if num_steps * step_size > 0: im_adv = model_adv.adv_sample_infer(im_data, im_info, gt_boxes, num_boxes, step_size, num_steps=num_steps) else: im_adv = im_data if save: file_name = imdb.image_path_at(i).split('/')[-1] if grad_cam: model.eval() rois, cls_prob, bbox_pred, \ rpn_loss_cls, rpn_loss_box, \ RCNN_loss_cls, RCNN_loss_bbox, \ rois_label, conv_output = model(im_adv, im_info, gt_boxes, num_boxes, grad_cam=True) one_hot_output = torch.zeros_like(cls_prob) one_hot_output[0][:, 0:-1] = 1 model.zero_grad() cls_prob.backward(gradient=one_hot_output, retain_graph=True) guided_gradients = model.gradients.cpu().data.numpy()[0] target = conv_output.cpu().data.numpy()[0] ws = np.mean(guided_gradients, axis=(1, 2)) # take averages for each gradient # create empty numpy array for cam cam = np.ones(target.shape[1:], dtype=np.float32) # multiply each weight with its conv output and then, sum for l, w in enumerate(ws): cam += w * target[l, :, :] cam = np.maximum(cam, 0) cam = (cam - np.min(cam)) / (np.max(cam) - np.min(cam) ) # normalize between 0-1 cam = np.uint8(cam * 255) # scale between 0-255 to visualize im_rgb = cv2.cvtColor(im2show, cv2.COLOR_BGR2RGB) cam = np.uint8( Image.fromarray(cam).resize( (im_rgb.shape[1], im_rgb.shape[0]), Image.ANTIALIAS)) / 255 original_image = Image.fromarray(im_rgb) save_class_activation_images(original_image, cam, file_name) rois, cls_prob, bbox_pred, \ rpn_loss_cls, rpn_loss_box, \ RCNN_loss_cls, RCNN_loss_bbox, \ rois_label = model(im_adv, im_info, gt_boxes, num_boxes) scores = cls_prob.data boxes = rois.data[:, :, 1:5] box_deltas = bbox_pred.data box_deltas = box_deltas.view(-1, 4) * torch.FloatTensor(cfg.TRAIN.BBOX_NORMALIZE_STDS).cuda() \ + torch.FloatTensor(cfg.TRAIN.BBOX_NORMALIZE_MEANS).cuda() box_deltas = box_deltas.view(batch_size, -1, 4 * (imdb.num_classes - 1)) pred_boxes = bbox_transform_inv(boxes, box_deltas, 1) pred_boxes = clip_boxes(pred_boxes, im_info.data, 1) scores = scores.squeeze() pred_boxes /= data[1][0][2].item() pred_boxes = pred_boxes.squeeze() nms_cfg = {'type': 'nms', 'iou_threshold': iou_thre} det_bboxes, det_labels = multiclass_nms(pred_boxes, scores, thresh, nms_cfg, max_per_image) keep = det_bboxes[:, 4] > thresh det_bboxes = det_bboxes[keep] det_labels = det_labels[keep] for j in xrange(0, imdb.num_classes - 1): inds = torch.nonzero(det_labels == j, as_tuple=False).view(-1) # if there is det if inds.numel() > 0: cls_dets = det_bboxes[inds] if vis or save: im2show = vis_detections(im2show, imdb.classes[j], cls_dets.cpu().numpy(), color=colors[int(j)]) all_boxes[j][i] = cls_dets.cpu().numpy() else: all_boxes[j][i] = empty_array # Limit to max_per_image detections *over all classes* if max_per_image > 0: image_scores = np.hstack([ all_boxes[j][i][:, -1] for j in xrange(0, imdb.num_classes - 1) ]) if len(image_scores) > max_per_image: image_thresh = np.sort(image_scores)[-max_per_image] for j in xrange(0, imdb.num_classes - 1): keep = np.where(all_boxes[j][i][:, -1] >= image_thresh)[0] all_boxes[j][i] = all_boxes[j][i][keep, :] if save: cv2.imwrite(os.path.join(p_t1, file_name.replace('jpg', 'png')), im2show) elif vis: cv2.imwrite('result.png', im2show) if i % 200 == 0: print(i, 'waiting.....') det_file = os.path.join(output_dir, 'detections.pkl') with open(det_file, 'wb') as f: pickle.dump(all_boxes, f, pickle.HIGHEST_PROTOCOL) print('Evaluating detections') mAP = imdb.evaluate_detections(all_boxes, output_dir) return mAP
args.set_cfgs = [ 'ANCHOR_SCALES', '[4, 8, 16, 32]', 'ANCHOR_RATIOS', '[0.5,1,2]', 'MAX_NUM_GT_BOXES', '50' ] init_seeds(cfg.RNG_SEED) cfg_from_file(args.cfg_file) cfg.TRAIN.USE_FLIPPED = False cfg.POOLING_MODE = 'align' imdb, roidb, ratio_list, ratio_index = combined_roidb(args.imdb_name) imdb.competition_mode(on=True) print('{:d} roidb entries'.format(len(roidb))) model = resnet(imdb.classes, 50, pretrained=False, class_agnostic=False) train_size = len(roidb) sampler_batch = sampler(train_size, batch_size) dataset = roibatchLoader(roidb, ratio_list, ratio_index, batch_size, \ imdb.num_classes, training=True) dataloader = torch.utils.data.DataLoader( dataset, batch_size=batch_size, sampler=sampler_batch, num_workers=min([os.cpu_count(), batch_size, 16])) # initilize the tensor holder here. im_data = torch.FloatTensor(1) im_info = torch.FloatTensor(1)