def test():
import os
im_file = 'demo/004545.jpg'
image = cv2.imread(im_file)
detector = FasterRCNN()
network.load_net('/media/longc/Data/models/VGGnet_fast_rcnn_iter_70000.h5',
detector)
detector.cuda()
print('load model successfully!')
# network.save_net(r'/media/longc/Data/models/VGGnet_fast_rcnn_iter_70000.h5', detector)
# print('save model succ')
t = Timer()
t.tic()
# image = np.zeros(shape=[600, 800, 3], dtype=np.uint8) + 255
dets, scores, classes = detector.detect(image, 0.3)
runtime = t.toc()
print('total spend: {}s'.format(runtime))
im2show = np.copy(image)
for i, det in enumerate(dets):
if scores[i] < 0.3:
continue
det = tuple(int(x) for x in det)
cv2.rectangle(im2show, det[0:2], det[2:4], (255, 205, 51), 2)
cv2.putText(im2show,
'%s: %.3f' % (classes[i], scores[i]),
(det[0], det[1] + 15),
cv2.FONT_HERSHEY_PLAIN,
1.0, (0, 0, 255),
thickness=1)
cv2.imwrite(os.path.join('demo', 'out.jpg'), im2show)
def test():
import os
im_file = 'demo/004545.jpg'
# im_file = 'data/VOCdevkit2007/VOC2007/JPEGImages/009036.jpg'
# im_file = '/media/longc/Data/data/2DMOT2015/test/ETH-Crossing/img1/000100.jpg'
image = cv2.imread(im_file)
model_file = 'models/saved_model3/faster_rcnn_200000.pth'
# model_file = '/media/longc/Data/models/faster_rcnn_pytorch3/faster_rcnn_100000.h5'
# model_file = '/media/longc/Data/models/faster_rcnn_pytorch2/faster_rcnn_2000.h5'
detector = FasterRCNN()
detector.load_state_dict(torch.load(model_file))
detector.cuda()
detector.eval()
print('load model successfully!')
# network.save_net(r'/media/longc/Data/models/VGGnet_fast_rcnn_iter_70000.h5', detector)
# print('save model succ')
t = Timer()
t.tic()
# image = np.zeros(shape=[600, 800, 3], dtype=np.uint8) + 255
dets, scores, classes = detector.detect(image, 0.7)
runtime = t.toc()
print('total spend: {}s'.format(runtime))
im2show = np.copy(image)
for i, det in enumerate(dets):
det = tuple(int(x) for x in det)
cv2.rectangle(im2show, det[0:2], det[2:4], (255, 205, 51), 2)
cv2.putText(im2show, '%s: %.3f' % (classes[i], scores[i]), (det[0], det[1] + 15), cv2.FONT_HERSHEY_PLAIN,
1.0, (0, 0, 255), thickness=1)
cv2.imwrite(os.path.join('demo', 'out.jpg'), im2show)
cv2.imshow('demo', im2show)
cv2.waitKey(0)
def __init__(self, weights=None):
if weights is None:
if not os.path.exists('weights'):
os.mkdir('weights')
download_url = 'https://github.com/ArnholdInstitute/ColdSpots/releases/download/1.0/faster-rcnn.zip'
if not os.path.exists('weights/faster-rcnn'):
print('Downloading weights for faster-rcnn')
if not os.path.exists(os.path.join('weights/faster-rcnn.zip')):
check_output([
'wget', download_url, '-O', 'weights/faster-rcnn.zip'
])
print('Unzipping...')
check_output(
['unzip', 'weights/faster-rcnn.zip', '-d', 'weights'])
description = json.load(
open('weights/faster-rcnn/description.json'))
weights = os.path.join('weights/faster-rcnn',
description['weights'])
print('Building model...')
self.model = FasterRCNNModel(classes=['__backround__', 'building'],
debug=False)
network.load_net(weights, self.model)
self.model.cuda()
self.model.eval()
def build_extractor(model_file, classes=None):
if classes is None:
extractor = FasterRCNN()
else:
extractor = FasterRCNN(classes)
extractor.cuda()
extractor.eval()
network.load_net(model_file, extractor)
print('load model successfully!')
return extractor
def main():
global args
print "Loading training set and testing set..."
# train_set = visual_genome(args.dataset_option, 'train')
test_set = visual_genome('small', 'test')
object_classes = test_set.object_classes
print "Done."
# train_loader = torch.utils.data.DataLoader(train_set, batch_size=1, shuffle=True, num_workers=8, pin_memory=True)
test_loader = torch.utils.data.DataLoader(test_set, batch_size=1, shuffle=False, num_workers=8, pin_memory=True)
net = FasterRCNN(use_kmeans_anchors=args.use_kmeans_anchors, n_classes=len(object_classes), model=args.base_model)
network.load_net('./output/detection/Faster_RCNN_small_vgg_12epoch_epoch_11.h5', net)
# network.load_net('./output/detection/RPN_object1_best.h5', net)
# network.set_trainable(net.features, requires_grad=False)
net.cuda()
# Testing
recall = test(test_loader, net)
print('Recall: '
'object: {recall: .3f}%'.format(recall=recall*100))
def test():
import os
im_file = 'demo/004545.jpg'
# im_file = 'data/VOCdevkit2007/VOC2007/JPEGImages/009036.jpg'
# im_file = '/media/longc/Data/data/2DMOT2015/test/ETH-Crossing/img1/000100.jpg'
image = cv2.imread(im_file)
# model_file = './VGGnet_fast_rcnn_iter_70000.h5'
# model_file = '/media/longc/Data/models/faster_rcnn_pytorch3/faster_rcnn_100000.h5'
# model_file = '/media/longc/Data/models/faster_rcnn_pytorch2/faster_rcnn_2000.h5'
model_file = './models/saved_model_max/faster_rcnn_100000.h5'
detector = FasterRCNN()
network.load_net(model_file, detector)
detector.cuda()
detector.eval()
print('load model successfully!')
# network.save_net(r'/media/longc/Data/models/VGGnet_fast_rcnn_iter_70000.h5', detector)
# print('save model succ')
t = Timer()
t.tic()
# image = np.zeros(shape=[600, 800, 3], dtype=np.uint8) + 255
dets, scores, classes = detector.detect(image, 0.7)
runtime = t.toc()
print('total spend: {}s'.format(runtime))
im2show = np.copy(image)
img = mpimg.imread(im_file)
# Create figure and axes
fig, ax = plt.subplots(1)
# Display the image
ax.imshow(img)
# Create a Rectangle patch
for i, det in enumerate(dets):
w = det[2] - det[0]
h = det[3] - det[1]
rect = patches.Rectangle(det[0:2],
w,
h,
linewidth=1,
edgecolor='r',
facecolor='none')
# text
plt.text(det[0], det[1], '%s: %.3f' % (classes[i], scores[i]))
# Add the patch to the Axes
ax.add_patch(rect)
plt.show()
print('aa')
def load_model(model_file_path):
detector = FasterRCNN()
network.load_net(model_file_path, detector)
detector.cuda()
detector.eval()
print('load model successfully!')
return detector
def test(visualize=False):
import os
im_file = 'data/cervix/train/Type_2/1381.jpg'
im_name = im_file.split('/')[-1]
image = cv2.imread(im_file)
# model_file = 'models/VGGnet_fast_rcnn_iter_70000.h5'
model_file = 'models/saved_model3/faster_rcnn_100000.h5'
expm = model_file.split('/')[-1].split('.')[0]
expm_dir = os.path.join('demo', expm)
if not os.path.exists(expm_dir):
os.makedirs(expm_dir)
detector = FasterRCNN()
network.load_net(model_file, detector)
detector.cuda()
detector.eval(
) # set model in evaluation mode, has effect on Dropout and Batchnorm. Use train() to set train mode.
print('load model successfully!')
# network.save_net(r'/media/longc/Data/models/VGGnet_fast_rcnn_iter_70000.h5', detector)
# print('save model succ')
t = Timer()
t.tic()
# image = np.zeros(shape=[600, 800, 3], dtype=np.uint8) + 255
dets, scores, classes = detector.detect(image, 0.7)
runtime = t.toc()
print('total spend: {}s'.format(runtime))
im2show = np.copy(image)
for i, det in enumerate(dets):
det = tuple(int(x) for x in det)
cv2.rectangle(im2show, det[0:2], det[2:4], (255, 205, 51), 4)
cv2.putText(im2show,
'%s: %.3f' % (classes[i], scores[i]),
(det[0], det[1] + 15),
cv2.FONT_HERSHEY_PLAIN,
1.0, (0, 0, 255),
thickness=1)
cv2.imwrite(os.path.join('demo', expm, im_name), im2show)
if visualize:
im2show = cv2.resize(im2show,
None,
None,
fx=0.15,
fy=0.15,
interpolation=cv2.INTER_LINEAR)
cv2.imshow('demo', im2show)
cv2.waitKey(0)
def analysis_video(self, result_dir):
self.statusbar_stringvar.set('Analysis..Please wait..')
model_file = 'model.h5'
detector = FasterRCNN()
network.load_net(model_file, detector)
detector.cuda()
detector.eval()
print('load model successfully!')
info_dict = {}
info_dict['pictures'] = []
for index in range(len(self.image_list)):
accuracy = 0.
pic_info = {}
pic_info['objects'] = []
dets, scores, classes = detector.detect(self.image_list[index], 0.8)
im2show = np.copy(self.image_list[index])
for i, det in enumerate(dets):
object_info = {}
det = tuple(int(x) for x in det)
cv2.rectangle(im2show, det[0:2], det[2:4], (255, 205, 51), 2)
cv2.putText(im2show, '%s: %.3f' % (classes[i], scores[i]), (det[0], det[1] + 15), cv2.FONT_HERSHEY_PLAIN,
1.0, (0, 0, 255), thickness=1)
accuracy += scores[i]
#object info initial
object_info['name'] = classes[i]
object_info['accuracy'] = scores[i]
object_info['bbox'] = det
pic_info['objects'].append(object_info)
# pic_info initial
pic_info['filename'] = os.path.basename(self.video_path).split('.')[0] + '_' + str(index + 1) + '.jpg'
pic_info['size'] = im2show.shape
info_dict['pictures'].append(pic_info)
cv2.imwrite(os.path.join(result_dir, pic_info['filename']), im2show)
self.view_table.update(index + 1, **{
'name': pic_info['filename'],
'accuracy': accuracy / len(classes),
'state': 'yes'
})
self.statusbar_stringvar.set('Analysis done!')
return info_dict
def __init__(self, classes, n_action_classes, n_action_nonagent_roles,
**kwargs):
super(HoiModel, self).__init__()
print "Constructing HOI Model"
faster_rcnn_config = kwargs.get("faster_rcnn_config", None)
if faster_rcnn_config is not None:
cf.cfg_from_file(faster_rcnn_config)
faster_rcnn_cle = kwargs.get("faster_rcnn_command_line", None)
if faster_rcnn_cle is not None:
cf.cfg_from_list(faster_rcnn_cle)
assert(cf.cfg["NCLASSES"] == len(classes)), \
"inconsistent FasterRCNN settings"
self.detection_branch = FasterRCNN(classes=classes)
self.human_centric_branch = HumanCentricBranch(
n_action_classes, n_action_nonagent_roles)
self.interaction_branch = InteractionBranch(n_action_nonagent_roles)
.format(i + 1, num_images, detect_time, nms_time)
# if vis:
# cv2.imshow('test', im2show)
# cv2.waitKey(1)
if sav:
cv2.imwrite(output_dir_detections + str(i) + '.png', im2show)
with open(det_file, 'wb') as f:
cPickle.dump(all_boxes, f, cPickle.HIGHEST_PROTOCOL)
print 'Evaluating detections'
imdb.evaluate_detections(all_boxes, output_dir)
if __name__ == '__main__':
# load data
imdb = get_imdb(imdb_name)
imdb.competition_mode(on=True)
# load net
net = FasterRCNN(classes=imdb.classes, debug=False)
network.load_net(trained_model, net)
print('load model successfully!')
net.cuda()
net.eval()
# evaluation
test_net(net, imdb, max_per_image, thresh=thresh, vis=vis)
print 'im_detect: {:d}/{:d} {:.3f}s {:.3f}s' \
.format(i + 1, num_images, detect_time, nms_time)
if vis:
cv2.imshow('test', im2show)
cv2.waitKey(1)
with open(det_file, 'wb') as f:
cPickle.dump(all_boxes, f, cPickle.HIGHEST_PROTOCOL)
print 'Evaluating detections'
imdb.evaluate_detections(all_boxes, output_dir)
if __name__ == '__main__':
# load data
imdb = get_imdb(imdb_name)
imdb.competition_mode(on=True)
# load net
net = FasterRCNN(classes=imdb.classes, debug=False, backbone='VGG')
network.load_net(trained_model, net)
print('load model successfully!')
net.cuda()
net.eval()
# evaluation
test_net(save_name, net, imdb, max_per_image, thresh=thresh, vis=vis)
# load config cfg_from_file(cfg_file) # overrides default config params lr = cfg.TRAIN.LEARNING_RATE momentum = cfg.TRAIN.MOMENTUM weight_decay = cfg.TRAIN.WEIGHT_DECAY disp_interval = cfg.TRAIN.DISPLAY log_interval = cfg.TRAIN.LOG_IMAGE_ITERS # load data imdb = get_imdb(imdb_name) rdl_roidb.prepare_roidb(imdb) roidb = imdb.roidb data_layer = RoIDataLayer(roidb, imdb.num_classes) # load net net = FasterRCNN(classes=imdb.classes, debug=_DEBUG) network.weights_normal_init(net, dev=0.01) # network.load_pretrained_npy(net, pretrained_model) network.load_pretrained_model(net, 'vgg16') # model_file = '/media/longc/Data/models/VGGnet_fast_rcnn_iter_70000.h5' # model_file = 'models/saved_model3/faster_rcnn_60000.h5' # network.load_net(model_file, net) # exp_name = 'vgg16_02-19_13-24' # start_step = 60001 # lr /= 10. # network.weights_normal_init([net.bbox_fc, net.score_fc, net.fc6, net.fc7], dev=0.01) net.cuda() net.train() # set model in train mode, has effect on Dropout and Batchnorm. Use eval() to set evaluation mode. params = list(net.parameters())
bboxes_pick[i, 3],
cur_scores[i]
)
cur_rec += str_out
cur_rec += "\n"
else:
cur_rec = "{} {}\n".format(file_path, 0)
out_file.write(cur_rec)
if __name__ == "__main__":
ctx = mx.gpu(9)
net = FasterRCNN(
len(cfg.anchor_ratios) * len(cfg.anchor_scales),
cfg.num_classes,
pretrained_model="vgg16",
feature_name="vgg0_conv12_fwd_output",
# pretrained_model="mobilenetv2_0.25",
# feature_name="mobilenetv20_features_linearbottleneck12_batchnorm2_fwd_output",
ctx=ctx)
net.init_params(ctx)
net.collect_params().load("/world/data-gpu-112/zhanglinghan/face-detect-faster-rcnn-mx/faster-rcnn-vgg16-9anchors/faster-rcnn-vgg16-9anchors-280000.gluonmodel", ctx)
global f_path
path_lst = os.listdir("/world/data-c27/face_landmarks_hourglass/detection-input")
path_lst.sort()
for f_name in path_lst:
f_path = os.path.join("/world/data-c27/face_landmarks_hourglass/detection-input", f_name)
print("processing {}".format(f_path))
f_path_out = os.path.join("/world/data-c27/face_landmarks_hourglass/detection-output", f_name.strip())
benchmark(net, ctx, f_path_out)
def train():
args = parse_args()
args.decay_lrs = cfg.TRAIN.DECAY_LRS
cfg.USE_GPU_NMS = True if args.use_cuda else False
assert args.batch_size == 1, 'Only support single batch'
lr = cfg.TRAIN.LEARNING_RATE
momentum = cfg.TRAIN.MOMENTUM
weight_decay = cfg.TRAIN.WEIGHT_DECAY
gamma = cfg.TRAIN.GAMMA
# initial tensorboardX writer
if args.use_tfboard:
if args.exp_name == 'default':
writer = SummaryWriter()
else:
writer = SummaryWriter('runs/' + args.exp_name)
if args.dataset == 'voc07trainval':
args.imdb_name = 'voc_2007_trainval'
args.imdbval_name = 'voc_2007_test'
elif args.dataset == 'voc0712trainval':
args.imdb_name = 'voc_2007_trainval+voc_2012_trainval'
args.imdbval_name = 'voc_2007_test'
else:
raise NotImplementedError
if args.net == 'res50':
fname = 'resnet50-caffe.pth'
elif args.net == 'res101':
fname = 'resnet101-caffe.pth'
else:
raise NotImplementedError
args.pretrained_model = os.path.join('data', 'pretrained', fname)
output_dir = args.output_dir
if not os.path.exists(output_dir):
os.makedirs(output_dir)
# dataset_cachefile = os.path.join(output_dir, 'dataset.pickle')
# if not os.path.exists(dataset_cachefile):
# imdb, roidb = combined_roidb(args.imdb_name)
# cache = [imdb, roidb]
# with open(dataset_cachefile, 'wb') as f:
# pickle.dump(cache, f)
# print('save dataset cache')
# else:
# with open(dataset_cachefile, 'rb') as f:
# cache = pickle.load(f)
# imdb, roidb = cache[0], cache[1]
# print('loaded dataset from cache')
imdb, roidb = combined_roidb(args.imdb_name)
train_dataset = RoiDataset(roidb)
train_dataloader = DataLoader(train_dataset, args.batch_size, shuffle=True)
model = FasterRCNN(backbone=args.net, pretrained=args.pretrained_model)
print('model loaded')
# if cfg.PRETRAINED_RPN:
# rpn_model_path = 'output/rpn.pth'
# model.load_state_dict(torch.load(rpn_model_path)['model'])
# print('loaded rpn!')
# optimizer
params = []
for key, value in dict(model.named_parameters()).items():
if value.requires_grad:
if 'bias' in key:
params += [{'params': [value], 'lr': lr * (cfg.TRAIN.DOUBLE_BIAS + 1), \
'weight_decay': cfg.TRAIN.BIAS_DECAY and weight_decay or 0}]
else:
params += [{
'params': [value],
'lr': lr,
'weight_decay': weight_decay
}]
optimizer = SGD(params, momentum=momentum)
if args.use_cuda:
model = model.cuda()
model.train()
iters_per_epoch = int(len(train_dataset) / args.batch_size)
# start training
for epoch in range(args.start_epoch, args.max_epochs + 1):
loss_temp = 0
rpn_tp, rpn_tn, rpn_fg, rpn_bg = 0, 0, 0, 0
rcnn_tp, rcnn_tn, rcnn_fg, rcnn_bg = 0, 0, 0, 0
tic = time.time()
train_data_iter = iter(train_dataloader)
if epoch in args.decay_lrs:
lr = lr * gamma
adjust_learning_rate(optimizer, lr)
print('adjust learning rate to {}'.format(lr))
for step in range(iters_per_epoch):
im_data, gt_boxes, im_info = next(train_data_iter)
if args.use_cuda:
im_data = im_data.cuda()
gt_boxes = gt_boxes.cuda()
im_info = im_info.cuda()
im_data_variable = Variable(im_data)
output = model(im_data_variable, gt_boxes, im_info)
rois, _, _, \
rcnn_cls_loss, rcnn_box_loss, \
rpn_cls_loss, rpn_box_loss, _train_info = output
loss = rcnn_cls_loss.mean() + rcnn_box_loss.mean() +\
rpn_cls_loss.mean() + rpn_box_loss.mean()
optimizer.zero_grad()
loss.backward()
optimizer.step()
loss_temp += loss.item()
if cfg.VERBOSE:
rpn_tp += _train_info['rpn_tp']
rpn_tn += _train_info['rpn_tn']
rpn_fg += _train_info['rpn_num_fg']
rpn_bg += _train_info['rpn_num_bg']
rcnn_tp += _train_info['rcnn_tp']
rcnn_tn += _train_info['rcnn_tn']
rcnn_fg += _train_info['rcnn_num_fg']
rcnn_bg += _train_info['rcnn_num_bg']
if (step + 1) % args.display_interval == 0:
toc = time.time()
loss_temp /= args.display_interval
rpn_cls_loss_v = rpn_cls_loss.mean().item()
rpn_box_loss_v = rpn_box_loss.mean().item()
rcnn_cls_loss_v = rcnn_cls_loss.mean().item()
rcnn_box_loss_v = rcnn_box_loss.mean().item()
print("[epoch %2d][step %4d/%4d] loss: %.4f, lr: %.2e, time cost %.1fs" \
% (epoch, step+1, iters_per_epoch, loss_temp, lr, toc - tic))
print("\t\t\t rpn_cls_loss_v: %.4f, rpn_box_loss_v: %.4f\n\t\t\t "
"rcnn_cls_loss_v: %.4f, rcnn_box_loss_v: %.4f" \
% (rpn_cls_loss_v, rpn_box_loss_v, rcnn_cls_loss_v, rcnn_box_loss_v))
if cfg.VERBOSE:
print('\t\t\t RPN : [FG/BG] [%d/%d], FG: %.4f, BG: %.4f' %
(rpn_fg, rpn_bg, float(rpn_tp) / rpn_fg,
float(rpn_tn) / rpn_bg))
print('\t\t\t RCNN: [FG/BG] [%d/%d], FG: %.4f, BG: %.4f' %
(rcnn_fg, rcnn_bg, float(rcnn_tp) / rcnn_fg,
float(rcnn_tn) / rcnn_bg))
if args.use_tfboard:
n_iter = (epoch - 1) * iters_per_epoch + step + 1
writer.add_scalar('losses/loss', loss_temp, n_iter)
writer.add_scalar('losses/rpn_cls_loss_v', rpn_cls_loss_v,
n_iter)
writer.add_scalar('losses/rpn_box_loss_v', rpn_box_loss_v,
n_iter)
writer.add_scalar('losses/rcnn_cls_loss_v',
rcnn_cls_loss_v, n_iter)
writer.add_scalar('losses/rcnn_box_loss_v',
rcnn_box_loss_v, n_iter)
if cfg.VERBOSE:
writer.add_scalar('rpn/fg_acc',
float(rpn_tp) / rpn_fg, n_iter)
writer.add_scalar('rpn/bg_acc',
float(rpn_tn) / rpn_bg, n_iter)
writer.add_scalar('rcnn/fg_acc',
float(rcnn_tp) / rcnn_fg, n_iter)
writer.add_scalar('rcnn/bg_acc',
float(rcnn_tn) / rcnn_bg, n_iter)
loss_temp = 0
rpn_tp, rpn_tn, rpn_fg, rpn_bg = 0, 0, 0, 0
rcnn_tp, rcnn_tn, rcnn_fg, rcnn_bg = 0, 0, 0, 0
tic = time.time()
if epoch % args.save_interval == 0:
save_name = os.path.join(
output_dir, 'faster_{}_epoch_{}.pth'.format(args.net, epoch))
torch.save({
'model': model.state_dict(),
'epoch': epoch,
'lr': lr
}, save_name)
from faster_rcnn.rpn_proposal import proposal_train
def train_transformation(data, label):
data, label = random_flip(data, label)
data = imagenetNormalize(data)
return data, label
train_dataset = VOCDataset(annotation_dir=cfg.annotation_dir,
img_dir=cfg.img_dir,
dataset_index=cfg.dataset_index,
transform=train_transformation,
resize_func=img_resize)
train_datait = mx.gluon.data.DataLoader(train_dataset, batch_size=1, shuffle=True)
ctx = mx.gpu(0)
net = FasterRCNN(len(cfg.anchor_ratios) * len(cfg.anchor_scales), cfg.num_classes)
net.init_params(ctx)
trainer = mx.gluon.trainer.Trainer(net.collect_params(),
'sgd',
{'learning_rate': 0.001,
'wd': 0.0005,
'momentum': 0.9})
anchors_count = len(cfg.anchor_ratios) * len(cfg.anchor_scales)
first_iter_saved_toggle = True
for epoch in range(1, 21):
for it, (data, label) in enumerate(train_datait):
data = data.as_in_context(ctx)
_n, _c, h, w = data.shape
label = label.as_in_context(ctx).reshape((1, -1, 5))
return data, label
test_dataset = VOCDataset(annotation_dir=cfg.test_annotation_dir,
img_dir=cfg.test_img_dir,
dataset_index=cfg.test_dataset_index,
transform=test_transformation,
resize_func=None)
# test_datait = mx.gluon.data.DataLoader(test_dataset, batch_size=1, shuffle=False)
ctx = mx.gpu(0)
args = parse_args()
print("Load model: {}".format(args.model_file))
net = FasterRCNN(len(cfg.anchor_ratios) * len(cfg.anchor_scales),
cfg.num_classes,
feature_name=args.feature_name)
net.init_params(ctx)
net.collect_params().load(args.model_file, ctx)
det_result = []
prograss_bar = tqdm(total=len(test_dataset))
for it, data_id in enumerate(range(len(test_dataset))):
data, label = test_dataset[data_id]
data = data.asnumpy()
data = data.transpose(1, 2, 0)
data, scale = img_resize(data)
data = data.transpose(2, 0, 1)
data_id = test_dataset.dataset_index[data_id]
data = mx.nd.array(data, ctx=ctx)
_c, h, w = data.shape
nms_time = _t['misc'].toc(average=False)
print 'im_detect: {:d}/{:d} {:.3f}s {:.3f}s' \
.format(i + 1, num_images, detect_time, nms_time)
df = pandas.DataFrame(all_boxes)
df.columns = ['x1', 'y1', 'x2', 'y2', 'score', 'image_id']
df.to_csv('predictions.csv', index=False)
print('Total time: %.4f, per image: %.4f' %
(total_time, total_time / num_images))
if __name__ == '__main__':
# load net
net = FasterRCNN(classes=['__backround__', 'building'], debug=False)
network.load_net(args.weights, net)
print('load model successfully!')
net.cuda()
net.eval()
val_data = json.load(open(args.test_boxes))
# evaluation
test_net(net,
val_data,
max_per_image,
thresh=thresh,
vis=vis,
data_dir='../data')
def test():
# Set up dataloader
data_loader = DAVIS_seq_dataloader(split='val')
model_file = './model/VGGnet_fast_rcnn_iter_70000.h5'
detector = FasterRCNN()
network.load_net(model_file, detector)
detector.cuda()
detector.eval()
print('Load Faster R-CNN model successfully!')
# unet_model = './model/vgg_unet_1e-4_500.h5'
# unet = UNet()
# network.load_net(unet_model, unet)
# unet.cuda()
# network.weights_normal_init(unet, dev=0.01)
# unet.load_from_faster_rcnn_h5(h5py.File(model_file))
criterion_bce = torch.nn.BCELoss().cuda()
weight_decay = 5e-5
# optimizer = torch.optim.SGD(list(unet.parameters())[26:], lr=1e-4, weight_decay=weight_decay)
# print('Load U-Net model successfully!')
crop_set = []
# Iterate
for i in range(data_loader.num_seq):
# Get the first frame info
seq = data_loader.seq_list[data_loader.out_pointer]
seq_len = data_loader.seq_len[seq]
img_blobs, seg_blobs = data_loader.get_next_minibatch()
img = img_blobs[0,:,:,:]
im_data, im_scales = detector.get_image_blob(img)
im_info = np.array([[im_data.shape[1], im_data.shape[2], im_scales[0]]], dtype=np.float32)
# Get the category of the object in the first frame
rmin, rmax, cmin, cmax = bbox(seg_blobs[0,:,:,0])
features, rois = detector(im_data, im_info, rpn_only=True)
new_rois_np = np.array([[0, cmin, rmin, cmax, rmax]], dtype=np.float32)
new_rois_t = torch.from_numpy(new_rois_np).cuda()
new_rois = Variable(new_rois_t, requires_grad=False)
pooled_features = detector.roi_pool(features, new_rois)
x = pooled_features.view(pooled_features.size()[0], -1)
x = detector.fc6(x)
x = detector.fc7(x)
cls_score = detector.score_fc(x)
cls_prob = F.softmax(cls_score)
bbox_pred = detector.bbox_fc(x)
cls_prob_np = cls_prob.cpu().data.numpy()
bbox_pred_np = bbox_pred.cpu().data.numpy()
cls_idx = cls_prob_np.argmax()
cls_conf = cls_prob_np.max()
# Overfit U-Net with the first frame
# for i in range(100):
# unet.train()
# img_t = torch.from_numpy(img_blobs).permute(0,3,1,2).float().cuda()
# img_v = Variable(img_t, requires_grad=False)
# seg_t = torch.from_numpy(seg_blobs).permute(0,3,1,2).float().cuda()
# seg_v = Variable(seg_t, requires_grad=False)
# pred = unet(img_v)
# loss = criterion_bce(pred, seg_v)
# pred_view = pred.view(-1, 1)
# seg_view = seg_v.view(-1, 1)
# EPS = 1e-6
# loss = 0.6 * seg_view.mul(torch.log(pred_view+EPS)) + 0.4 * seg_view.mul(-1).add(1).mul(torch.log(1-pred+EPS))
# loss = -torch.mean(loss)
# loss_val = loss.data[0]
# optimizer.zero_grad()
# loss.backward()
# optimizer.step()
# print('{}/100: {}'.format(i, loss_val))
# unet.eval()
# Merge region proposals overlapping with last frame proposal
for j in range(1, seq_len):
img_blobs, _ = data_loader.get_next_minibatch()
img = img_blobs[0,:,:,:]
im_data, im_scales = detector.get_image_blob(img)
# 300 x 5, the first elements are useless here
features, rois = detector(im_data, im_info, rpn_only=True)
x1, y1, x2, y2 = merge_rois((rmin, rmax, cmin, cmax), rois.cpu().data.numpy(), thres=0.75)
# Have overlapping proposals
if x1 is not None:
# Send to following layers to refine the bbox
new_rois_np = np.array([[0, x1, y1, x2, y2]], dtype=np.float32)
new_rois_t = torch.from_numpy(new_rois_np).cuda()
new_rois = Variable(new_rois_t, requires_grad=False)
pooled_features = detector.roi_pool(features, new_rois)
x = pooled_features.view(pooled_features.size()[0], -1)
x = detector.fc6(x)
x = detector.fc7(x)
cls_score = detector.score_fc(x)
cls_prob = F.softmax(cls_score)
bbox_pred = detector.bbox_fc(x)
cls_prob_np = cls_prob.cpu().data.numpy()
bbox_pred_np = bbox_pred.cpu().data.numpy()
# Only regress bbox when confidence is greater than 0.8
if cls_prob_np.max() > 0.8 and cls_prob_np.argmax() != 0:
keep = cls_prob_np.argmax()
pred_boxes, scores, classes = detector.interpret_faster_rcnn(cls_prob, bbox_pred, new_rois, im_info, im_data.shape, 0.8)
cx = (x1 + x2) / 2
cy = (y1 + y2) / 2
width = x2 - x1 + 1
height = y2 - y1 + 1
dx = bbox_pred_np[0,keep*4+0]
dy = bbox_pred_np[0,keep*4+1]
dw = bbox_pred_np[0,keep*4+2]
dh = bbox_pred_np[0,keep*4+3]
pred_x = dx * width + cx
pred_y = dy * height + cy
pred_w = np.exp(dw) * width
pred_h = np.exp(dh) * height
x1 = pred_x - pred_w / 2
x2 = pred_x + pred_w / 2
y1 = pred_y - pred_h / 2
y2 = pred_y + pred_h / 2
# No overlapping proposals
if x1 is None:
# Using Faster R-CNN again to find potential objects
dets, scores, classes = detector.detect(img, 0.6)
# Cannot find any salient object
if dets.shape[0] == 0:
x1, y1, x2, y2 = cmin, rmin, cmax, rmax
else:
x1 = dets[:,0]
y1 = dets[:,1]
x2 = dets[:,2]
y2 = dets[:,3]
pred_area = (x2 - x1 + 1) * (y2 - y1 + 1)
init_area = (cmax - cmin + 1) * (rmax - rmin + 1)
xx1 = np.maximum(x1, cmin)
xx2 = np.minimum(x2, cmax)
yy1 = np.maximum(y1, rmin)
yy2 = np.minimum(y2, rmax)
inter = (xx2 - xx1 + 1) * (yy2 - yy1 + 1)
ovr = inter / (pred_area + init_area - inter)
# If there is overlapping, choose the largest IoU bbox
try:
ovr = ovr[ovr > 0.3]
ovr_idx = np.argsort(ovr)[-1]
x1 = dets[ovr_idx,0]
y1 = dets[ovr_idx,1]
x2 = dets[ovr_idx,2]
y2 = dets[ovr_idx,3]
# Else, choose the highest objectness score one
except:
if cls_idx == 0:
temp_idx = scores.argmax()
x1 = dets[temp_idx,0]
y1 = dets[temp_idx,1]
x2 = dets[temp_idx,2]
y2 = dets[temp_idx,3]
else:
cx = (x1 + x2) / 2
cy = (y1 + y2) / 2
cc = (cmin + cmax) / 2
cr = (rmin + rmax) / 2
dist = np.sqrt(np.square(cx-cc) + np.square(cy-cr))
dist_idx = np.argsort(dist)
for di in dist_idx:
if classes[di] == _CLASSES[cls_idx]:
x1 = dets[di,0]
y1 = dets[di,1]
x2 = dets[di,2]
y2 = dets[di,3]
# Crop the region and send it to U-Net
try:
x1 = int(max(x1, 0))
x2 = int(min(x2, im_data.shape[2]))
y1 = int(max(y1, 0))
y2 = int(min(y2, im_data.shape[1]))
except:
x1 = int(max(x1[0], 0))
x2 = int(min(x2[0], im_data.shape[2]))
y1 = int(max(y1[0], 0))
y2 = int(min(y2[0], im_data.shape[1]))
# MEAN_PIXEL = np.array([103.939, 116.779, 123.68])
# crop = img_blobs[:, y1:y2+1, x1:x2+1, :] - MEAN_PIXEL
# crop = img_blobs[:,:,:,:] - MEAN_PIXEL
# crop_v = Variable(torch.from_numpy(crop).permute(0, 3, 1, 2).cuda(), requires_grad=False)
# pred = unet(crop_v)
# pred_np = pred.cpu().data.numpy()[0,0,:,:]
# pred_np[pred_np < 0.5] = 0
# pred_np[pred_np >= 0.5] = 1
# pred_np = pred_np * 255
# res = pred_np.astype(int)
# cv2.imwrite('test.png', res)
if y2 - y1 <= 1 or x2 - x1 <= 1:
ipdb.set_trace()
cv2.imwrite(os.path.join('demo', 'crop_{}_{}.png'.format(i, j)), img[y1:y2+1,x1:x2+1,:])
rmin = y1
rmax = y2
cmin = x1
cmax = x2
im2show = np.copy(img)
cv2.rectangle(im2show, (int(x1),int(y1)), (int(x2),int(y2)), (0, 255, 0), 2)
cv2.imwrite(os.path.join('demo', '{}_{}.jpg'.format(i, j)), im2show)
temp = [i, j, x1, y1, x2, y2]
crop_set.append(temp)
# Save
crop_set = np.array(crop_set)
np.save('crop', crop_set)
# load config
cfg_from_file(cfg_file)
lr = cfg.TRAIN.LEARNING_RATE
momentum = cfg.TRAIN.MOMENTUM
weight_decay = cfg.TRAIN.WEIGHT_DECAY
disp_interval = cfg.TRAIN.DISPLAY
# load data
imdb = get_imdb(imdb_name)
rdl_roidb.prepare_roidb(imdb)
roidb = imdb.roidb
data_layer = RoIDataLayer(roidb, imdb.num_classes)
# load net
net = FasterRCNN(classes=imdb.classes, debug=_DEBUG, training=True)
net.cuda()
net.train()
net.apply(weight_init) #-- parameters initialize
#----download resnet101 weights-----
pretrained_state = torch.load(pretrained_model)
net.resnet.load_state_dict({
k: v
for k, v in pretrained_state.items() if k in net.resnet.state_dict()
})
for p in net.resnet.conv1.parameters():
p.requires_grad = False
for p in net.resnet.bn1.parameters():
p.requires_grad = False
# load config cfg_from_file(cfg_file) lr = cfg.TRAIN.LEARNING_RATE momentum = cfg.TRAIN.MOMENTUM weight_decay = cfg.TRAIN.WEIGHT_DECAY disp_interval = cfg.TRAIN.DISPLAY log_interval = cfg.TRAIN.LOG_IMAGE_ITERS # load data imdb = get_imdb(imdb_name) rdl_roidb.prepare_roidb(imdb) roidb = imdb.roidb data_layer = RoIDataLayer(roidb, imdb.num_classes) # load net net = FasterRCNN(classes=imdb.classes, debug=_DEBUG) network.weights_normal_init(net, dev=0.01) network.load_pretrained_npy(net, pretrained_model) # model_file = '/media/longc/Data/models/VGGnet_fast_rcnn_iter_70000.h5' # model_file = 'models/saved_model3/faster_rcnn_60000.h5' # network.load_net(model_file, net) # exp_name = 'vgg16_02-19_13-24' # start_step = 60001 # lr /= 10. # network.weights_normal_init([net.bbox_fc, net.score_fc, net.fc6, net.fc7], dev=0.01) net.cuda() net.train() params = list(net.parameters()) # optimizer = torch.optim.Adam(params[-8:], lr=lr)
class FasterRCNN:
def __init__(self, weights=None):
if weights is None:
if not os.path.exists('weights'):
os.mkdir('weights')
download_url = 'https://github.com/ArnholdInstitute/ColdSpots/releases/download/1.0/faster-rcnn.zip'
if not os.path.exists('weights/faster-rcnn'):
print('Downloading weights for faster-rcnn')
if not os.path.exists(os.path.join('weights/faster-rcnn.zip')):
check_output([
'wget', download_url, '-O', 'weights/faster-rcnn.zip'
])
print('Unzipping...')
check_output(
['unzip', 'weights/faster-rcnn.zip', '-d', 'weights'])
description = json.load(
open('weights/faster-rcnn/description.json'))
weights = os.path.join('weights/faster-rcnn',
description['weights'])
print('Building model...')
self.model = FasterRCNNModel(classes=['__backround__', 'building'],
debug=False)
network.load_net(weights, self.model)
self.model.cuda()
self.model.eval()
def close_session(self):
pass
def predict_image(self, image, threshold, eval_mode=False):
"""
Infer buildings for a single image.
Inputs:
image :: n x m x 3 ndarray - Should be in RGB format
"""
if type(image) is str:
image = cv2.imread(image)
else:
image = image[:, :, (2, 1, 0)] # RGB -> BGR
im_data, im_scales = self.model.get_image_blob(image)
im_info = np.array(
[[im_data.shape[1], im_data.shape[2], im_scales[0]]],
dtype=np.float32)
t0 = time.time()
cls_prob, bbox_pred, rois = self.model(im_data, im_info)
runtime = time.time() - t0
scores = cls_prob.data.cpu().numpy()
boxes = rois.data.cpu().numpy()[:, 1:5] / im_info[0][2]
if cfg.TEST.BBOX_REG:
# Apply bounding-box regression deltas
box_deltas = bbox_pred.data.cpu().numpy()
pred_boxes = bbox_transform_inv(boxes, box_deltas)
pred_boxes = clip_boxes(pred_boxes, image.shape)
else:
# Simply repeat the boxes, once for each class
pred_boxes = np.tile(boxes, (1, scores.shape[1]))
current = np.concatenate(
[
pred_boxes[:, 4:8], # (skip the background class)
np.expand_dims(scores[:, 1], 1)
],
axis=1)
suppressed = current[py_cpu_nms(current.astype(np.float32), 0.3)]
suppressed = pandas.DataFrame(
suppressed, columns=['x1', 'y1', 'x2', 'y2', 'score'])
if eval_mode:
return suppressed[
suppressed['score'] >= threshold], suppressed, runtime
else:
return suppressed[suppressed['score'] >= threshold]
def predict_all(self, test_boxes_file, threshold, data_dir=None):
test_boxes = json.load(open(test_boxes_file))
if data_dir is None:
data_dir = os.path.join(os.path.dirname(test_boxes_file))
total_time = 0.0
for i, anno in enumerate(test_boxes):
orig_img = cv2.imread(
'%s/%s' % (data_dir, anno['image_path']))[:, :, (2, 1, 0)]
pred, all_rects, time = self.predict_image(orig_img,
threshold,
eval_mode=True)
pred['image_id'] = i
all_rects['image_id'] = i
yield pred, all_rects, test_boxes[i]
# create VGG model for state featurization
print("Loading image embedding model...")
if args.image_embedding_model_type == "resnet":
im_emb_model = ResNet50()
elif args.image_embedding_model_type == "vgg":
im_emb_model = VGG16()
else:
print("--image_embedding_model_type must be either resnet or vgg")
sys.exit(0)
print("Done!")
# create Faster-RCNN model for state featurization
print("Loading Fast-RCNN...")
model_file = 'VGGnet_fast_rcnn_iter_70000.h5'
model_frcnn = FasterRCNN()
network.load_net(model_file, model_frcnn)
model_frcnn.cuda()
model_frcnn.eval()
print("Done!")
# create DQN's for the next object, predicates, and attributes
print("Creating DQN models...")
DQN_next_object_main = DQN_MLP(2048*3+9600 + parameters["maximum_num_entities_per_image"], 1)
DQN_next_object_target = DQN_MLP(2048*3+9600 + parameters["maximum_num_entities_per_image"], 1)
DQN_predicate_main = DQN_MLP(2048*3+9600 + len(semantic_action_graph.predicate_nodes), 1)
DQN_predicate_target = DQN_MLP(2048*3+9600 + len(semantic_action_graph.predicate_nodes), 1)
DQN_attribute_main = DQN_MLP(2048*3+9600 + len(semantic_action_graph.attribute_nodes), 1)
DQN_attribute_target = DQN_MLP(2048*3+9600 + len(semantic_action_graph.attribute_nodes), 1)
print("Done!")
# load config
cfg_from_file(cfg_file)
lr = cfg.TRAIN.LEARNING_RATE
momentum = cfg.TRAIN.MOMENTUM
weight_decay = cfg.TRAIN.WEIGHT_DECAY
disp_interval = cfg.TRAIN.DISPLAY
log_interval = cfg.TRAIN.LOG_IMAGE_ITERS
# load data
imdb = get_imdb(imdb_name)
rdl_roidb.prepare_roidb(imdb)
roidb = imdb.roidb
data_layer = RoIDataLayer(roidb, imdb.num_classes)
# load net
net = FasterRCNN(classes=imdb.classes, debug=_DEBUG)
network.weights_normal_init(net, dev=0.01)
#network.load_pretrained_npy(net, pretrained_model)
# model_file = '/media/longc/Data/models/VGGnet_fast_rcnn_iter_70000.h5'
# model_file = 'models/saved_model3/faster_rcnn_60000.h5'
# network.load_net(model_file, net)
# exp_name = 'vgg16_02-19_13-24'
# start_step = 60001
# lr /= 10.
# network.weights_normal_init([net.bbox_fc, net.score_fc, net.fc6, net.fc7], dev=0.01)
if os.path.exists('pretrained_vgg.pkl'):
pret_net = pkl.load(open('pretrained_vgg.pkl','rb'))
else:
pret_net = model_zoo.load_url('https://download.pytorch.org/models/vgg16-397923af.pth')
pkl.dump(pret_net, open('pretrained_vgg.pkl','wb'), pkl.HIGHEST_PROTOCOL)
# load config cfg_from_file(cfg_file) lr = cfg.TRAIN.LEARNING_RATE momentum = cfg.TRAIN.MOMENTUM weight_decay = cfg.TRAIN.WEIGHT_DECAY disp_interval = cfg.TRAIN.DISPLAY log_interval = cfg.TRAIN.LOG_IMAGE_ITERS # load data imdb = get_imdb(imdb_name) rdl_roidb.prepare_roidb(imdb) roidb = imdb.roidb data_layer = RoIDataLayer(roidb, imdb.num_classes) # load net net = FasterRCNN(classes=imdb.classes, debug=_DEBUG) network.weights_normal_init(net, dev=0.01) network.load_pretrained_npy(net, pretrained_model) # model_file = '/media/longc/Data/models/VGGnet_fast_rcnn_iter_70000.h5' # model_file = 'models/saved_model3/faster_rcnn_60000.h5' # network.load_net(model_file, net) # exp_name = 'vgg16_02-19_13-24' # start_step = 60001 # lr /= 10. # network.weights_normal_init([net.bbox_fc, net.score_fc, net.fc6, net.fc7], dev=0.01) net.cuda() net.train() params = list(net.parameters()) # optimizer = torch.optim.Adam(params[-8:], lr=lr)
def train():
args = parse_args()
lr = args.lr
decay_lrs = args.decay_lrs
momentum = args.momentum
weight_decay = args.weight_decay
bais_decay = args.bais_decay
gamma = args.gamma
cfg.USE_GPU_NMS = True if args.use_gpu else False
if args.use_tfboard:
writer = SummaryWriter()
# load data
print('load data')
if args.dataset == 'voc07trainval':
dataset_name = 'voc_2007_trainval'
elif args.dataset == 'voc12trainval':
dataset_name = 'voc_2012_trainval'
elif args.dataset == 'voc0712trainval':
dataset_name = 'voc_2007_trainval+voc_2012_trainval'
else:
raise NotImplementedError
imdb, roidb = combined_roidb(dataset_name)
train_dataset = RoiDataset(roidb)
train_dataloader = DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True)
iter_per_epoch = int(len(train_dataset) / args.batch_size)
# prepare model
print('load model')
model = FasterRCNN(backbone=args.backbone)
params = []
for key, value in dict(model.named_parameters()).items():
if value.requires_grad:
if 'bias' in key:
params += [{
'params': [value],
'lr': lr * (cfg.TRAIN.DOUBLE_BIAS + 1),
'weight_decay': bais_decay and weight_decay or 0
}]
else:
params += [{
'params': [value],
'lr': lr,
'weight_decay': weight_decay
}]
if args.use_gpu:
model = model.cuda()
model.train()
# define optimizer
optimizer = SGD(params, momentum=momentum)
# training
print('start training...')
for epoch in range(args.epochs):
start_time = time.time()
train_data_iter = iter(train_dataloader)
temp_loss = 0
rpn_tp, rpn_tn, rpn_fg, rpn_bg = 0, 0, 0, 0
faster_rcnn_tp, faster_rcnn_tn, faster_rcnn_fg, faster_rcnn_bg = 0, 0, 0, 0
if epoch in decay_lrs:
lr = lr * gamma
adjust_lr(optimizer, lr)
print('adjusting learning rate to {}'.format(lr))
for step in range(iter_per_epoch):
im_data, gt_boxes, im_info = next(train_data_iter)
if args.use_gpu:
im_data = im_data.cuda()
gt_boxes = gt_boxes.cuda()
im_info = im_info.cuda()
im_data_variable = Variable(im_data)
outputs = model(im_data_variable, gt_boxes, im_info)
rois, _, _, faster_rcnn_cls_loss, faster_rcnn_reg_loss, \
rpn_cls_loss, rpn_reg_loss, _train_info = outputs
loss = faster_rcnn_cls_loss.mean() + faster_rcnn_reg_loss.mean() + \
rpn_cls_loss.mean() + rpn_reg_loss.mean()
optimizer.zero_grad()
loss.backward()
optimizer.step()
temp_loss += loss.item()
if cfg.VERBOSE:
rpn_tp += _train_info['rpn_tp']
rpn_tn += _train_info['rpn_tn']
rpn_fg += _train_info['rpn_num_fg']
rpn_bg += _train_info['rpn_num_bg']
faster_rcnn_tp += _train_info['faster_rcnn_tp']
faster_rcnn_tn += _train_info['faster_rcnn_tn']
faster_rcnn_fg += _train_info['faster_rcnn_num_fg']
faster_rcnn_bg += _train_info['faster_rcnn_num_bg']
if (step + 1) % args.display_interval == 0:
end_time = time.time()
temp_loss /= args.display_interval
rpn_cls_loss_m = rpn_cls_loss.mean().item()
rpn_reg_loss_m = rpn_reg_loss.mean().item()
faster_rcnn_cls_loss_m = faster_rcnn_cls_loss.mean().item()
faster_rcnn_reg_loss_m = faster_rcnn_reg_loss.mean().item()
print('[epoch %2d][step %4d/%4d] loss: %.4f, time_cost: %.1f' %
(epoch, step + 1, iter_per_epoch, temp_loss,
end_time - start_time))
print(
'loss: rpn_cls_loss_m: %.4f, rpn_reg_loss_m: %.4f, faster_rcnn_cls_loss_m: %.4f, faster_rcnn_reg_loss_m: %.4f'
% (rpn_cls_loss_m, rpn_reg_loss_m, faster_rcnn_cls_loss_m,
faster_rcnn_reg_loss_m))
if args.use_tfboard:
n_iter = epoch * iter_per_epoch + step + 1
writer.add_scalar('losses/loss', temp_loss, n_iter)
writer.add_scalar('losses/rpn_cls_loss_m', rpn_cls_loss_m,
n_iter)
writer.add_scalar('losses/rpn_reg_loss_m', rpn_reg_loss_m,
n_iter)
writer.add_scalar('losses/faster_rcnn_cls_loss_m',
faster_rcnn_cls_loss_m, n_iter)
writer.add_scalar('losses/faster_rcnn_reg_loss_m',
faster_rcnn_reg_loss_m, n_iter)
if cfg.VERBOSE:
writer.add_scalar('rpn/fg_acc',
float(rpn_tp) / rpn_fg, n_iter)
writer.add_scalar('rpn/bg_acc',
float(rpn_tn) / rpn_bg, n_iter)
writer.add_scalar(
'rcnn/fg_acc',
float(faster_rcnn_tp) / faster_rcnn_fg, n_iter)
writer.add_scalar(
'rcnn/bg_acc',
float(faster_rcnn_tn) / faster_rcnn_bg, n_iter)
temp_loss = 0
rpn_tp, rpn_tn, rpn_fg, rpn_bg = 0, 0, 0, 0
faster_rcnn_tp, faster_rcnn_tn, faster_rcnn_fg, faster_rcnn_bg = 0, 0, 0, 0
start_time = time.time()
if not os.path.exists(args.output_dir):
os.mkdir(args.output_dir)
if epoch % args.save_interval == 0:
save_name = os.path.join(
args.output_dir, 'faster_rcnn101_epoch_{}.pth'.format(epoch))
torch.save({
'model': model.state_dict(),
'epoch': epoch,
'lr': lr
}, save_name)
# ------------
if rand_seed is not None:
np.random.seed(rand_seed)
# load config
cfg_from_file(cfg_file)
lr = cfg.TRAIN.LEARNING_RATE
momentum = cfg.TRAIN.MOMENTUM
weight_decay = cfg.TRAIN.WEIGHT_DECAY
disp_interval = cfg.TRAIN.DISPLAY
log_interval = cfg.TRAIN.LOG_IMAGE_ITERS
# load net
net = FasterRCNN(classes=['__background__', 'building'], debug=_DEBUG)
network.weights_normal_init(net, dev=0.01)
network.load_pretrained_npy(net, pretrained_model)
net.cuda()
net.train()
params = list(net.parameters())
optimizer = torch.optim.SGD(params[8:],
lr=lr,
momentum=momentum,
weight_decay=weight_decay)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
def main():
global args, logger
train_ds = train_dataset()
# only support batch_size = 1 so far
train_datait = mx.gluon.data.DataLoader(train_ds,
batch_size=1,
shuffle=True)
ctx = [mx.gpu(i) for i in range(len(args.gpus.split(",")))]
ctx = ctx[0]
net = FasterRCNN(len(cfg.anchor_ratios) * len(cfg.anchor_scales),
cfg.num_classes,
feature_name=args.feature_name)
net.init_params(ctx)
if args.pretrained_model != "":
net.collect_params().load(args.pretrained_model, ctx)
logger.info("loading {}".format(args.pretrained_model))
lr_schdl = mx.lr_scheduler.FactorScheduler(step=20000, factor=0.9)
trainer = mx.gluon.trainer.Trainer(net.collect_params(),
'sgd',
optimizer_params={
'learning_rate': args.learning_rate,
'wd': args.weight_decay,
"lr_scheduler": lr_schdl,
'momentum': 0.9
})
anchors_count = len(cfg.anchor_ratios) * len(cfg.anchor_scales)
for epoch in range(0, args.epochs):
last_iter_end_timestamp = time.time()
for it, (data, label) in enumerate(train_datait):
data_loaed_time = time.time()
data = data.as_in_context(ctx)
_n, _c, h, w = data.shape
label = label.as_in_context(ctx).reshape((1, -1, 5))
with mx.autograd.record():
rpn_cls, rpn_reg, f = net.rpn(data)
f_height = f.shape[2]
f_width = f.shape[3]
rpn_cls_gt, rpn_reg_gt = rpn_gt_opr(rpn_reg.shape, label, ctx,
h, w)
rpn_bbox_sampled, rcnn_reg_target, rcnn_cls_target = proposal_train(
rpn_cls, rpn_reg, label, f.shape, data.shape, ctx)
# RPN Loss part
# Reshape and transpose to the shape of gt
rpn_cls = rpn_cls.reshape((1, -1, 2, f_height, f_width))
rpn_cls = mx.nd.transpose(rpn_cls, (0, 1, 3, 4, 2))
rpn_reg = mx.nd.transpose(
rpn_reg.reshape((1, -1, 4, f_height, f_width)),
(0, 1, 3, 4, 2))
mask = (rpn_cls_gt == 1).reshape(
(1, anchors_count, f_height, f_width, 1)).broadcast_to(
(1, anchors_count, f_height, f_width, 4))
rpn_loss_reg = mx.nd.sum(
mx.nd.smooth_l1(
(rpn_reg - rpn_reg_gt) * mask, 3.0)) / mx.nd.sum(mask)
rpn_loss_cls = softmax_celoss_with_ignore(
rpn_cls.reshape((-1, 2)), rpn_cls_gt.reshape((-1, )), -1)
# RCNN part
# add batch dimension
rpn_bbox_sampled = mx.nd.concatenate([
mx.nd.zeros(
(rpn_bbox_sampled.shape[0], 1), ctx), rpn_bbox_sampled
],
axis=1)
f = mx.nd.ROIPooling(f, rpn_bbox_sampled, (7, 7),
1.0 / 16) # VGG16 based spatial stride=16
rcnn_cls, rcnn_reg = net.rcnn(f)
mask = (rcnn_cls_target > 0).reshape(
(rcnn_cls_target.shape[0], 1)).broadcast_to(
(rcnn_cls_target.shape[0], 4 * cfg.num_classes))
rcnn_loss_reg = mx.nd.sum(
mx.nd.smooth_l1((rcnn_reg - rcnn_reg_target) * mask,
1.0)) / mx.nd.sum(mask)
rcnn_loss_cls = mx.nd.softmax_cross_entropy(
rcnn_cls, rcnn_cls_target) / rcnn_cls.shape[0]
loss = rpn_loss_cls + rpn_loss_reg + rcnn_loss_cls + rcnn_loss_reg
loss.backward()
trainer.step(data.shape[0])
logger.info(
"Epoch {} Iter {:>6d}: loss={:>6.5f}, rpn_loss_cls={:>6.5f}, rpn_loss_reg={:>6.5f}, rcnn_loss_cls={:>6.5f}, rcnn_loss_reg={:>6.5f}, lr={:>6.5f}"
.format(epoch, it, loss.asscalar(), rpn_loss_cls.asscalar(),
rpn_loss_reg.asscalar(), rcnn_loss_cls.asscalar(),
rcnn_loss_reg.asscalar(), trainer.learning_rate))
net.collect_params().save(
os.path.join(args.save_path, "lastest.gluonmodel"))
if epoch % args.save_interval == 0:
save_schema = os.path.split(args.save_path)[1] + "-{}"
net.collect_params().save(
os.path.join(args.save_path,
save_schema.format(epoch) + ".gluonmodel"))
# load config
cfg_from_file(cfg_file)
lr = cfg.TRAIN.LEARNING_RATE
momentum = cfg.TRAIN.MOMENTUM
weight_decay = cfg.TRAIN.WEIGHT_DECAY
disp_interval = cfg.TRAIN.DISPLAY
log_interval = cfg.TRAIN.LOG_IMAGE_ITERS
# load data
imdb = VisualGenome(split=0, num_im=50)
roidb = imdb.roidb
data_layer = RoIDataLayer(roidb, imdb.num_classes)
# load net
net = FasterRCNN(classes=imdb.classes, debug=_DEBUG)
network.weights_normal_init(net, dev=0.01)
network.load_net(pretrained_model, net)
# network.load_pretrained_npy(net, 'checkpoints/VGG_imagenet.npy')
net.cuda()
net.train()
params = list(net.parameters())
print("Params are {}".format(
'\n'.join(['{}: {}'.format(n, p.size()) for n,p in net.named_parameters()]))
)
# optimizer = torch.optim.Adam(params, lr=0.001, eps=1e-4, weight_decay=weight_decay)
optimizer = torch.optim.SGD(params[8:], lr=lr, momentum=momentum, weight_decay=weight_decay)
if not os.path.exists(output_dir):
thickness=1)
im_name = os.path.basename(image_file)
print(os.path.join('demo/det_results', im_name))
cv2.imwrite(os.path.join('demo/det_results', im_name), im2show)
#cv2.imshow('demo', im2show)
#cv2.waitKey(0)
def folder_test(net, folder):
txt_file = folder + 'JPEGImages/file_name.txt'
with open(txt_file) as f:
for line in f:
img_path = folder + 'JPEGImages/' + line.strip('\n') + '.JPG'
anno_path = folder + 'Annotations/' + line.strip('\n') + '.xml'
image_test(net, img_path, anno_path)
if __name__ == '__main__':
model_file = 'models/saved_model3/faster_rcnn_100000.h5'
detector = FasterRCNN()
network.load_net(model_file, detector)
detector.cuda()
detector.eval()
print('load model successfully!')
#image_file = 'demo/000001.JPG'
#image_test(detector, image_file, None)
folder = '/data/jmtian/PlateData/PVW_WRM_CUT/'
folder_test(detector, folder)
