def train(train_list, model, criterion, optimizer, epoch, writer, LOG):
losses = AverageMeter()
percep1 = AverageMeter()
percep2 = AverageMeter()
maaloss = AverageMeter()
batch_time = AverageMeter()
data_time = AverageMeter()
train_loader = torch.utils.data.DataLoader(dataset.listDataset(
train_list,
shuffle=True,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]),
]),
phase='train',
seen=model.seen,
batch_size=args.batch_size,
num_workers=args.workers,
epoch=epoch),
batch_size=args.batch_size)
model.train()
end = time.time()
mae = 0.0
for i, (img, target) in enumerate(train_loader):
data_time.update(time.time() - end)
a = time.time()
img = img.cuda()
target = target.type(torch.FloatTensor).unsqueeze(0).cuda()
output, x64, gt64, x256, gt256 = model(img, target)
b = time.time()
maa, p1, p2 = criterion(output, target, x64, gt64, x256, gt256)
loss = maa + p1 + p2
c = time.time()
mae += abs(output.sum().item() - target.sum().item())
losses.update(loss.item(), img.shape[0])
maaloss.update(maa.item(), img.shape[0])
percep1.update(p1.item(), img.shape[0])
percep2.update(p2.item(), img.shape[0])
optimizer.zero_grad()
loss.backward()
optimizer.step()
d = time.time()
batch_time.update(time.time() - end)
end = time.time()
if i % (args.print_freq * 7) == 0:
LOG.log(
'TRAIN - Epoch: [{0}][{1}/{2}] '
'Time {batch_time.val:.3f} '
'Data {data_time.val:.3f} '
'Loss {loss.avg:.4f} '
'Maa:{maaloss.avg:.4f} Percep1:{ploss1.avg:.4f} Percep2:{ploss2.avg:.4f}\t'
.format(epoch,
i,
len(train_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses,
maaloss=maaloss,
ploss1=percep1,
ploss2=percep2))
cprint('b-a:%.3f @ c-b:%.3f @d-c:%.3f' % (b - a, c - b, d - c),
color='green')
mae = mae / len(train_loader)
writer.add_scalar('trainmae', mae, epoch)
writer.add_scalar('trainloss', losses.avg, epoch)
LOG.log('train - epoch:%d - mae:%.3f\n' % (epoch, mae), color='yellow')
return model
def valid(datacfg,
darknetcfg,
learnetcfg,
weightfile,
outfile,
use_baserw=False):
options = read_data_cfg(datacfg)
valid_images = options['valid']
metadict = options['meta']
# name_list = options['names']
# backup = cfg.backup
ckpt = weightfile.split('/')[-1].split('.')[0]
backup = weightfile.split('/')[-2]
ckpt_pre = '/ene_' if use_baserw else '/ene'
prefix = 'results/' + backup.split('/')[-1] + ckpt_pre + ckpt
print('saving to: ' + prefix)
# prefix = 'results/' + weightfile.split('/')[1]
# names = load_class_names(name_list)
with open(valid_images) as fp:
tmp_files = fp.readlines()
valid_files = [item.rstrip() for item in tmp_files]
m = Darknet(darknetcfg, learnetcfg)
m.print_network()
m.load_weights(weightfile)
m.cuda()
m.eval()
valid_dataset = dataset.listDataset(valid_images,
shape=(m.width, m.height),
shuffle=False,
transform=transforms.Compose([
transforms.ToTensor(),
]))
valid_batchsize = 2
assert (valid_batchsize > 1)
kwargs = {'num_workers': 4, 'pin_memory': True}
valid_loader = torch.utils.data.DataLoader(valid_dataset,
batch_size=valid_batchsize,
shuffle=False,
**kwargs)
if False:
metaset = dataset.MetaDataset(metafiles=metadict,
train=False,
ensemble=True)
metaloader = torch.utils.data.DataLoader(metaset,
batch_size=len(metaset),
shuffle=False,
**kwargs)
metaloader = iter(metaloader)
n_cls = len(metaset.classes)
print('===> Generating dynamic weights...')
metax, mask = metaloader.next()
metax, mask = metax.cuda(), mask.cuda()
metax, mask = Variable(metax, volatile=True), Variable(mask,
volatile=True)
dynamic_weights = m.meta_forward(metax, mask)
for i in range(len(dynamic_weights)):
assert dynamic_weights[i].size(0) == sum(metaset.meta_cnts)
inds = np.cumsum([0] + metaset.meta_cnts)
new_weight = []
for j in range(len(metaset.meta_cnts)):
new_weight.append(
torch.mean(dynamic_weights[i][inds[j]:inds[j + 1]], dim=0))
dynamic_weights[i] = torch.stack(new_weight)
print(dynamic_weights[i].shape)
else:
metaset = dataset.MetaDataset(metafiles=metadict,
train=False,
ensemble=True,
with_ids=True)
metaloader = torch.utils.data.DataLoader(metaset,
batch_size=64,
shuffle=False,
**kwargs)
# metaloader = iter(metaloader)
n_cls = len(metaset.classes)
enews = [0.0] * n_cls
cnt = [0.0] * n_cls
print('===> Generating dynamic weights...')
kkk = 0
for metax, mask, clsids in metaloader:
print('===> {}/{}'.format(kkk, len(metaset) // 64))
kkk += 1
metax, mask = metax.cuda(), mask.cuda()
metax, mask = Variable(metax,
volatile=True), Variable(mask,
volatile=True)
dws = m.meta_forward(metax, mask)
dw = dws[0]
for ci, c in enumerate(clsids):
enews[c] = enews[c] * cnt[c] / (cnt[c] +
1) + dw[ci] / (cnt[c] + 1)
cnt[c] += 1
dynamic_weights = [torch.stack(enews)]
# import pickle
# with open('data/rws/voc_novel2_.pkl', 'wb') as f:
# tmp = [x.data.cpu().numpy() for x in dynamic_weights]
# pickle.dump(tmp, f)
# import pdb; pdb.set_trace()
if use_baserw:
import pickle
# f = 'data/rws/voc_novel{}_.pkl'.format(cfg.novelid)
f = 'data/rws/voc_novel{}_.pkl'.format(0)
print('===> Loading from {}...'.format(f))
with open(f, 'rb') as f:
# with open('data/rws/voc_novel0_.pkl', 'rb') as f:
rws = pickle.load(f)
rws = [Variable(torch.from_numpy(rw)).cuda() for rw in rws]
tki = cfg._real_base_ids
for i in range(len(rws)):
dynamic_weights[i][tki] = rws[i][tki]
# dynamic_weights[i] = rws[i]
# pdb.set_trace()
if not os.path.exists(prefix):
# os.mkdir(prefix)
os.makedirs(prefix)
fps = [0] * n_cls
for i, cls_name in enumerate(metaset.classes):
buf = '%s/%s%s.txt' % (prefix, outfile, cls_name)
fps[i] = open(buf, 'w')
lineId = -1
conf_thresh = 0.005
nms_thresh = 0.45
for batch_idx, (data, target) in enumerate(valid_loader):
data = data.cuda()
data = Variable(data, volatile=True)
output = m.detect_forward(data, dynamic_weights)
if isinstance(output, tuple):
output = (output[0].data, output[1].data)
else:
output = output.data
# import pdb; pdb.set_trace()
batch_boxes = get_region_boxes_v2(output, n_cls, conf_thresh,
m.num_classes, m.anchors,
m.num_anchors, 0, 1)
if isinstance(output, tuple):
bs = output[0].size(0)
else:
assert output.size(0) % n_cls == 0
bs = output.size(0) // n_cls
for b in range(bs):
lineId = lineId + 1
imgpath = valid_dataset.lines[lineId].rstrip()
print(imgpath)
imgid = os.path.basename(imgpath).split('.')[0]
width, height = get_image_size(imgpath)
for i in range(n_cls):
# oi = i * bs + b
oi = b * n_cls + i
boxes = batch_boxes[oi]
boxes = nms(boxes, nms_thresh)
for box in boxes:
x1 = (box[0] - box[2] / 2.0) * width
y1 = (box[1] - box[3] / 2.0) * height
x2 = (box[0] + box[2] / 2.0) * width
y2 = (box[1] + box[3] / 2.0) * height
det_conf = box[4]
for j in range((len(box) - 5) / 2):
cls_conf = box[5 + 2 * j]
cls_id = box[6 + 2 * j]
prob = det_conf * cls_conf
fps[i].write('%s %f %f %f %f %f\n' %
(imgid, prob, x1, y1, x2, y2))
for i in range(n_cls):
fps[i].close()
def train(train_list, model, criterion, optimizer, epoch):
global train_loss
losses = AverageMeter()
batch_time = AverageMeter()
data_time = AverageMeter()
train_loader = torch.utils.data.DataLoader(dataset.listDataset(
train_list,
shuffle=True,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]),
]),
train=True,
seen=model.seen,
batch_size=args.batch_size,
num_workers=args.workers),
batch_size=args.batch_size)
print('epoch %d, processed %d samples, lr %.10f' %
(epoch, epoch * len(train_loader.dataset), args.lr))
model.train()
end = time.time()
for i, (img, target) in enumerate(train_loader):
data_time.update(time.time() - end)
img = img.cuda()
img = Variable(img)
output = model(img)
target = target.type(torch.FloatTensor).unsqueeze(0).cuda()
target = Variable(target)
#############print if you want########
# print('gtruth',target.detach().cpu().sum())
# print('count',output.detach().cpu().sum())
#################################
loss = criterion(output, target)
losses.update(loss.item(), img.size(0))
optimizer.zero_grad()
loss.backward()
optimizer.step()
batch_time.update(time.time() - end)
end = time.time()
if i % args.print_freq == 0:
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'.format(
epoch,
i,
len(train_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses))
def valid(datacfg, cfgfile, weightfile, outfile):
options = read_data_cfg(datacfg)
valid_images = options['valid']
name_list = options['names']
prefix = 'results'
names = load_class_names(name_list)
with open(valid_images) as fp:
tmp_files = fp.readlines()
valid_files = [item.rstrip() for item in tmp_files]
m = Darknet(cfgfile)
m.print_network()
m.load_weights(weightfile)
m.cuda()
m.eval()
valid_dataset = dataset.listDataset(valid_images,
shape=(m.width, m.height),
shuffle=False,
transform=transforms.Compose([
transforms.ToTensor(),
]))
valid_batchsize = 2
assert (valid_batchsize > 1)
kwargs = {'num_workers': 4, 'pin_memory': True}
valid_loader = torch.utils.data.DataLoader(valid_dataset,
batch_size=valid_batchsize,
shuffle=False,
**kwargs)
fps = [0] * m.num_classes
if not os.path.exists('results'):
os.mkdir('results')
for i in range(m.num_classes):
buf = '%s/%s%s.txt' % (prefix, outfile, names[i])
fps[i] = open(buf, 'w')
lineId = -1
conf_thresh = 0.005
nms_thresh = 0.45
if m.net_name() == 'region': # region_layer
shape = (0, 0)
else:
shape = (m.width, m.height)
for _, (data, target, org_w, org_h) in enumerate(valid_loader):
data = data.cuda()
output = m(data)
batch_boxes = get_all_boxes(output,
shape,
conf_thresh,
m.num_classes,
only_objectness=0,
validation=True)
for i in range(len(batch_boxes)):
lineId += 1
fileId = os.path.basename(valid_files[lineId]).split('.')[0]
#width, height = get_image_size(valid_files[lineId])
width, height = float(org_w[i]), float(org_h[i])
print(valid_files[lineId])
boxes = batch_boxes[i]
correct_yolo_boxes(boxes, width, height, m.width, m.height)
boxes = nms(boxes, nms_thresh)
for box in boxes:
x1 = (box[0] - box[2] / 2.0) * width
y1 = (box[1] - box[3] / 2.0) * height
x2 = (box[0] + box[2] / 2.0) * width
y2 = (box[1] + box[3] / 2.0) * height
det_conf = box[4]
for j in range((len(box) - 5) // 2):
cls_conf = box[5 + 2 * j]
cls_id = box[6 + 2 * j]
prob = det_conf * cls_conf
fps[cls_id].write('%s %f %f %f %f %f\n' %
(fileId, prob, x1, y1, x2, y2))
for i in range(m.num_classes):
fps[i].close()
def train(train_list, model, criterion, optimizer, epoch):
losses = AverageMeter()
batch_time = AverageMeter()
data_time = AverageMeter()
train_loader = torch.utils.data.DataLoader(dataset.listDataset(
train_list,
shuffle=True,
crop=True,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]),
]),
train=True,
seen=model.seen,
batch_size=args.batch_size,
num_workers=args.workers),
batch_size=args.batch_size)
log_text = 'epoch %d, processed %d samples, lr %.10f' % (
epoch, epoch * len(train_loader.dataset), args.lr)
log_print(log_text, color='green', attrs=['bold'])
model.train()
end = time.time()
#BATCH OF HEMP< OFFSET< SCALE
#img, thm_tgt, pff_tgt, size_tgt
for i, (img_list, target_list) in enumerate(train_loader):
for j, patch in enumerate(img_list):
data_time.update(time.time() - end)
img = img_list[j]
target = target_list[j]
img = img.cuda()
img = Variable(img)
output = model(img)
target = target.type(
torch.FloatTensor).unsqueeze(0).cuda() #TARGET to cuda
target = Variable(target)
loss = criterion(output,
target) #list of criterion [focal loss, mse]
# focal_loss = vri[0](cri[0])
#loss =
losses.update(float(loss.item()), img.size(0))
optimizer.zero_grad()
loss.backward()
optimizer.step()
batch_time.update(time.time() - end)
end = time.time()
if i % args.print_freq == 0:
log_text = ((
'Epoch: [{0}][{1}/{2}]\t'
'Patch {patch_num:d}\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t').format(
epoch,
i,
len(train_loader),
patch_num=j,
batch_time=batch_time,
data_time=data_time,
loss=losses))
log_print(log_text, color='green', attrs=['bold'])
writer.add_scalar('train_loss', losses.avg, epoch)
def trainRPNPP(model, optimizer, epoch, coco):
losses = AverageMeter()
batch_time = AverageMeter()
data_time = AverageMeter()
train_loader = torch.utils.data.DataLoader(dataset.listDataset(
args.ilsvrc,
args.youtube,
args.data_type,
shuffle=True,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]),
]),
train=True,
batch_size=args.batch_size,
num_workers=args.workers,
coco=coco,
rpnpp=True),
batch_size=args.batch_size)
model.train()
end = time.time()
for i, (z, x, regression_target, conf_target) in enumerate(train_loader):
data_time.update(time.time() - end)
z = z.cuda()
z = Variable(z)
x = x.cuda()
x = Variable(x)
pred_score, pred_regression = model(z, x)
# print(pred_score.size(), pred_regression.size())
b, a2, h, w = pred_score.size()
pred_conf = pred_score.reshape(-1, 2, 5 * h * w).permute(0, 2, 1)
pred_offset = pred_regression.reshape(-1, 4,
5 * h * w).permute(0, 2, 1)
regression_target = regression_target.type(torch.FloatTensor).cuda()
conf_target = conf_target.type(torch.LongTensor).cuda()
# print(pred_conf.size(), conf_target.size())
cls_loss = rpn_cross_entropy(pred_conf, conf_target)
reg_loss = rpn_smoothL1(pred_offset, regression_target, conf_target)
loss = cls_loss + reg_loss
losses.update(loss.item(), x.size(0))
optimizer.zero_grad()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), 10)
optimizer.step()
batch_time.update(time.time() - end)
end = time.time()
if i % args.print_freq == 0:
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'.format(
epoch,
i,
len(train_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses))
def train(epoch):
global processed_batches
t0 = time.time()
if ngpus > 1:
cur_model = model.module
else:
cur_model = model
train_loader = torch.utils.data.DataLoader(
dataset.listDataset(trainlist, shape=(init_width, init_height),
shuffle=True,
transform=transforms.Compose([
transforms.ToTensor(),
]),
train=True,
seen=cur_model.seen,
batch_size=batch_size,
num_workers=num_workers),
batch_size=batch_size, shuffle=False, **kwargs)
lr = adjust_learning_rate(optimizer, processed_batches)
logging('epoch %d, processed %d samples, lr %f' % (epoch, epoch * len(train_loader.dataset), lr))
model.train()
t1 = time.time()
avg_time = torch.zeros(9)
for batch_idx, (data, target) in enumerate(train_loader):
t2 = time.time()
adjust_learning_rate(optimizer, processed_batches)
processed_batches = processed_batches + 1
#if (batch_idx+1) % dot_interval == 0:
# sys.stdout.write('.')
if use_cuda:
data = data.cuda()
#target= target.cuda()
t3 = time.time()
data, target = Variable(data), Variable(target)
t4 = time.time()
optimizer.zero_grad()
t5 = time.time()
output = model(data)
t6 = time.time()
region_loss.seen = region_loss.seen + data.data.size(0)
loss = region_loss(output, target)
t7 = time.time()
loss.backward()
t8 = time.time()
optimizer.step()
t9 = time.time()
if False and batch_idx > 1:
avg_time[0] = avg_time[0] + (t2-t1)
avg_time[1] = avg_time[1] + (t3-t2)
avg_time[2] = avg_time[2] + (t4-t3)
avg_time[3] = avg_time[3] + (t5-t4)
avg_time[4] = avg_time[4] + (t6-t5)
avg_time[5] = avg_time[5] + (t7-t6)
avg_time[6] = avg_time[6] + (t8-t7)
avg_time[7] = avg_time[7] + (t9-t8)
avg_time[8] = avg_time[8] + (t9-t1)
print('-------------------------------')
print(' load data : %f' % (avg_time[0]/(batch_idx)))
print(' cpu to cuda : %f' % (avg_time[1]/(batch_idx)))
print('cuda to variable : %f' % (avg_time[2]/(batch_idx)))
print(' zero_grad : %f' % (avg_time[3]/(batch_idx)))
print(' forward feature : %f' % (avg_time[4]/(batch_idx)))
print(' forward loss : %f' % (avg_time[5]/(batch_idx)))
print(' backward : %f' % (avg_time[6]/(batch_idx)))
print(' step : %f' % (avg_time[7]/(batch_idx)))
print(' total : %f' % (avg_time[8]/(batch_idx)))
t1 = time.time()
print('')
t1 = time.time()
logging('training with %f samples/s' % (len(train_loader.dataset)/(t1-t0)))
if (epoch+1) % save_interval == 0:
logging('save weights to %s/%06d.weights' % (backupdir, epoch+1))
cur_model.seen = (epoch + 1) * len(train_loader.dataset)
cur_model.save_weights('%s/%06d.weights' % (backupdir, epoch+1))
weightfile = '../yolov2-tiny-bnn/weights/yolov2-tiny-voc.weights'
options = read_data_cfg(datacfg)
valid_images = options['valid']
with open(valid_images) as fp:
tmp_files = fp.readlines()
valid_files = [item.rstrip() for item in tmp_files]
m = Darknet(cfgfile)
m.print_network()
m.load_weights(weightfile)
m.cuda()
m.eval()
valid_dataset = dataset.listDataset(valid_images,
shape=(m.width, m.height),
shuffle=False,
transform=transforms.Compose([
transforms.ToTensor(),
]))
valid_batchsize = 2
assert (valid_batchsize > 1)
kwargs = {'num_workers': 0, 'pin_memory': True}
valid_loader = torch.utils.data.DataLoader(valid_dataset,
batch_size=valid_batchsize,
shuffle=False,
**kwargs)
conf_thresh = 0.005
nms_thresh = 0.45
metrics = []
labels = []
def train(epoch):
global processed_batches
t0 = time.time()
cur_model = curmodel()
init_width = cur_model.width
init_height = cur_model.height
kwargs = {
'num_workers': num_workers,
'pin_memory': True
} if use_cuda else {}
#kwargs = {'num_workers': 0, 'pin_memory': True} if use_cuda else {}
train_loader = torch.utils.data.DataLoader(
dataset.listDataset(
trainlist,
imgRoot=image_folder,
wdatalist=wdata,
odweight=odw,
shape=(init_width, init_height),
shuffle=False, #True,
transform=transforms.Compose([
transforms.ToTensor(),
]),
train=True,
seen=cur_model.seen,
batch_size=batch_size,
num_workers=num_workers),
collate_fn=dataset.custom_collate,
batch_size=batch_size,
shuffle=False,
**kwargs)
processed_batches = cur_model.seen // batch_size
#lr = adjust_learning_rate(optimizer, processed_batches)
for param_group in optimizer.param_groups:
lr = param_group['lr']
logging('[%03d] processed %d samples, lr %e' %
(epoch, epoch * len(train_loader.dataset), lr))
#logging('[%03d] processed %d samples' % (epoch, epoch * len(train_loader.dataset)))
model.train()
t1 = time.time()
avg_time = torch.zeros(9)
for batch_idx, (data, target) in enumerate(train_loader):
t2 = time.time()
#adjust_learning_rate(optimizer, processed_batches)
processed_batches = processed_batches + 1
#if (batch_idx+1) % dot_interval == 0:
# sys.stdout.write('.')
t3 = time.time()
data, target = data.to(device), target.to(device)
t4 = time.time()
optimizer.zero_grad()
t5 = time.time()
output = model(data)
t6 = time.time()
org_loss = []
for i, l in enumerate(loss_layers):
l.seen = l.seen + data.data.size(0)
ol = l(output[i]['x'], target)
org_loss.append(ol)
t7 = time.time()
#for i, l in enumerate(reversed(org_loss)):
# l.backward(retain_graph=True if i < len(org_loss)-1 else False)
# org_loss.reverse()
sum(org_loss).backward()
nn.utils.clip_grad_norm_(model.parameters(), 10000)
#for p in model.parameters():
# p.data.add_(-lr, p.grad.data)
t8 = time.time()
optimizer.step()
t9 = time.time()
if False and batch_idx > 1:
avg_time[0] = avg_time[0] + (t2 - t1)
avg_time[1] = avg_time[1] + (t3 - t2)
avg_time[2] = avg_time[2] + (t4 - t3)
avg_time[3] = avg_time[3] + (t5 - t4)
avg_time[4] = avg_time[4] + (t6 - t5)
avg_time[5] = avg_time[5] + (t7 - t6)
avg_time[6] = avg_time[6] + (t8 - t7)
avg_time[7] = avg_time[7] + (t9 - t8)
avg_time[8] = avg_time[8] + (t9 - t1)
print('-------------------------------')
print(' load data : %f' % (avg_time[0] / (batch_idx)))
print(' cpu to cuda : %f' % (avg_time[1] / (batch_idx)))
print('cuda to variable : %f' % (avg_time[2] / (batch_idx)))
print(' zero_grad : %f' % (avg_time[3] / (batch_idx)))
print(' forward feature : %f' % (avg_time[4] / (batch_idx)))
print(' forward loss : %f' % (avg_time[5] / (batch_idx)))
print(' backward : %f' % (avg_time[6] / (batch_idx)))
print(' step : %f' % (avg_time[7] / (batch_idx)))
print(' total : %f' % (avg_time[8] / (batch_idx)))
t1 = time.time()
del data, target
org_loss.clear()
gc.collect()
print('')
t1 = time.time()
nsamples = len(train_loader.dataset)
logging('[%03d] training with %f samples/s' % (epoch, nsamples /
(t1 - t0)))
return nsamples
def validate(Pre_data, model, args):
print('begin test')
batch_size = 1
test_loader = torch.utils.data.DataLoader(dataset.listDataset(
Pre_data,
args['task_id'],
shuffle=False,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]),
]),
args=args,
train=False),
batch_size=1)
model.eval()
mae = 0.0
mse = 0.0
visi = []
index = 0
if not os.path.exists('./local_eval/loc_file'):
os.makedirs('./local_eval/loc_file')
'''output coordinates'''
f_loc = open("./local_eval/A_localization.txt", "w+")
for i, (fname, img, fidt_map, kpoint) in enumerate(test_loader):
count = 0
img = img.cuda()
if len(img.shape) == 5:
img = img.squeeze(0)
if len(fidt_map.shape) == 5:
fidt_map = fidt_map.squeeze(0)
if len(img.shape) == 3:
img = img.unsqueeze(0)
if len(fidt_map.shape) == 3:
fidt_map = fidt_map.unsqueeze(0)
with torch.no_grad():
d6 = model(img)
'''return counting and coordinates'''
count, pred_kpoint, f_loc = LMDS_counting(d6, i + 1, f_loc, fname)
point_map = generate_point_map(pred_kpoint, f_loc, rate=1)
if args['visual'] == True:
if not os.path.exists(args['task_id'] + '_box/'):
os.makedirs(args['task_id'] + '_box/')
ori_img, box_img = generate_bounding_boxes(pred_kpoint, fname)
show_fidt = show_map(d6.data.cpu().numpy())
gt_show = show_map(fidt_map.data.cpu().numpy())
res = np.hstack(
(ori_img, gt_show, show_fidt, point_map, box_img))
cv2.imwrite(args['task_id'] + '_box/' + fname[0], res)
gt_count = torch.sum(kpoint).item()
mae += abs(gt_count - count)
mse += abs(gt_count - count) * abs(gt_count - count)
if i % 1 == 0:
print('{fname} Gt {gt:.2f} Pred {pred}'.format(fname=fname[0],
gt=gt_count,
pred=count))
visi.append([
img.data.cpu().numpy(),
d6.data.cpu().numpy(),
fidt_map.data.cpu().numpy(), fname
])
index += 1
mae = mae * 1.0 / (len(test_loader) * batch_size)
mse = math.sqrt(mse / (len(test_loader)) * batch_size)
nni.report_intermediate_result(mae)
print(' \n* MAE {mae:.3f}\n'.format(mae=mae),
'* MSE {mse:.3f}'.format(mse=mse))
return mae, visi
def valid(datacfg, cfgfile, weightfile, outfile):
options = read_data_cfg(datacfg)
valid_images = options['valid']
name_list = options['names']
prefix = 'results'
names = load_class_names(name_list)
with open(valid_images) as fp:
tmp_files = fp.readlines()
valid_files = [item.rstrip() for item in tmp_files]
m = Darknet(cfgfile)
# m.load_state_dict(torch.load("darknet_bayes_60.pkl"))
# m = torch.nn.DataParallel(m).cuda()
m.load_state_dict(torch.load(weightfile))
m.print_network()
# m.load_weights(weightfile)
m.cuda()
m.eval()
#################################################
"""
layers = [m.models[0].conv1,
m.models[2].conv2,
m.models[4].conv3,
m.models[6].conv4,
m.models[8].conv5,
m.models[10].conv6,
m.models[12].conv7,
m.models[13].conv8,
m.models[14].conv9]
thresholds = [-3,-3,-3,-3,-3,-3,-3,-3,-3]
sig,exp = compute_compression_rate(layers, m.get_masks(thresholds))
print("Test error after with reduced bit precision:")
weights,biases = compute_reduced_weights(layers, m.get_masks(thresholds),sig,exp)
for layer, weight,bias in zip(layers, weights,biases):
if True:
layer.post_weight_mu.data = torch.Tensor(weight).cuda()
layer.post_bias_mu = torch.Tensor(bias).cuda()
else:
layer.post_weight_mu.data = torch.Tensor(weight)
for layer in layers: layer.deterministic = True
"""
##########################################
valid_dataset = dataset.listDataset(valid_images,
shape=(m.width, m.height),
shuffle=False,
transform=transforms.Compose([
transforms.ToTensor(),
]))
valid_batchsize = 2
assert (valid_batchsize > 1)
kwargs = {'num_workers': 4, 'pin_memory': True}
valid_loader = torch.utils.data.DataLoader(valid_dataset,
batch_size=valid_batchsize,
shuffle=False,
**kwargs)
fps = [0] * m.num_classes
if not os.path.exists('results'):
os.mkdir('results')
for i in range(m.num_classes):
buf = '%s/%s%s.txt' % (prefix, outfile, names[i])
fps[i] = open(buf, 'w')
lineId = -1
conf_thresh = 0.005
nms_thresh = 0.45
for batch_idx, (data, target) in enumerate(valid_loader):
data = data.cuda()
data = Variable(data, volatile=True)
output = m(data).data
batch_boxes = get_region_boxes(output, conf_thresh, m.num_classes,
m.anchors, m.num_anchors, 0, 1)
for i in range(output.size(0)):
lineId = lineId + 1
fileId = os.path.basename(valid_files[lineId]).split('.')[0]
width, height = get_image_size(valid_files[lineId])
print(valid_files[lineId])
boxes = batch_boxes[i]
boxes = nms(boxes, nms_thresh)
for box in boxes:
x1 = (box[0] - box[2] / 2.0) * width
y1 = (box[1] - box[3] / 2.0) * height
x2 = (box[0] + box[2] / 2.0) * width
y2 = (box[1] + box[3] / 2.0) * height
det_conf = box[4]
for j in range(int((len(box) - 5) / 2)):
cls_conf = box[5 + 2 * j]
cls_id = box[6 + 2 * j]
prob = det_conf * cls_conf
fps[cls_id].write('%s %f %f %f %f %f\n' %
(fileId, prob, x1, y1, x2, y2))
for i in range(m.num_classes):
fps[i].close()
os.system('mkdir {0}'.format(opt.experiment))
opt.manualSeed = random.randint(1, 10000) # fix seed
print("Random Seed: ", opt.manualSeed)
random.seed(opt.manualSeed)
np.random.seed(opt.manualSeed)
torch.manual_seed(opt.manualSeed)
cudnn.benchmark = True
if torch.cuda.is_available() and not opt.cuda:
print(
"WARNING: You have a CUDA device, so you should probably run with --cuda"
)
train_dataset = dataset.listDataset(list_file=opt.trainlist)
assert train_dataset
if opt.random_sample:
sampler = dataset.randomSequentialSampler(train_dataset, opt.batchSize)
else:
sampler = None
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=opt.batchSize,
shuffle=False,
sampler=sampler,
num_workers=int(opt.workers),
collate_fn=dataset.alignCollate(
imgH=opt.imgH,
imgW=opt.imgW,
keep_ratio=opt.keep_ratio))
print(len(train_loader))
def valid(datacfg, cfgfile, weightfile, weightfile_distance,
weightfile_rotation, outfile):
def truths_length(truths):
for i in range(50):
if truths[i][1] == 0:
return i
# Parse configuration files
options = read_data_cfg(datacfg)
valid_images = options['valid']
#meshname = options['mesh']
backupdir = options['backup']
name = options['name']
if not os.path.exists(backupdir):
makedirs(backupdir)
# Parameters
prefix = 'results'
seed = int(time.time())
#gpus = '0,1,2,3' # Specify which gpus to use
gpus = options['gpus']
test_width = 416
test_height = 416
torch.manual_seed(seed)
use_cuda = True
if use_cuda:
os.environ['CUDA_VISIBLE_DEVICES'] = gpus
torch.cuda.manual_seed(seed)
save = False
testtime = True
use_cuda = True
num_classes = 1
testing_samples = 0.0
eps = 1e-5
notpredicted = 0
conf_thresh = 0.1
nms_thresh = 0.4
match_thresh = 0.5
if save:
makedirs(backupdir + '/test')
makedirs(backupdir + '/test/gt')
makedirs(backupdir + '/test/pr')
# To save
testing_error_trans = 0.0
testing_error_angle = 0.0
testing_error_pixel = 0.0
errs_2d = []
errs_3d = []
errs_trans = []
errs_angle = []
errs_corner2D = []
preds_trans = []
preds_rot = []
preds_corners2D = []
gts_trans = []
gts_rot = []
gts_corners2D = []
# Get validation file names
with open(valid_images) as fp:
tmp_files = fp.readlines()
valid_files = [item.rstrip() for item in tmp_files]
# Specicy model, load pretrained weights, pass to GPU and set the module in evaluation mode
model = Darknet(cfgfile)
model_2 = DistanceNet()
model_3 = RotationNet()
#model.print_network()
model.load_weights(weightfile)
model_2.load_state_dict(torch.load(weightfile_distance))
model_3.load_state_dict(torch.load(weightfile_rotation))
model.cuda()
model.eval()
model_2.cuda()
model_3.cuda()
# Get the parser for the test dataset
valid_dataset = dataset.listDataset(valid_images,
shape=(test_width, test_height),
shuffle=False,
transform=transforms.Compose([
transforms.ToTensor(),
]))
valid_batchsize = 1
# Specify the number of workers for multiple processing, get the dataloader for the test dataset
kwargs = {'num_workers': 4, 'pin_memory': True}
test_loader = torch.utils.data.DataLoader(valid_dataset,
batch_size=valid_batchsize,
shuffle=False,
**kwargs)
logging(" Testing {}...".format(name))
logging(" Number of test samples: %d" % len(test_loader.dataset))
# Iterate through test batches (Batch size for test data is 1)
count = 0
z = np.zeros((3, 1))
out_file = open('data/valid_results.txt', 'w')
for batch_idx, (data, target, gt_T, gt_R) in enumerate(test_loader):
t1 = time.time()
# Pass data to GPU
if use_cuda:
data = data.cuda()
target = target.cuda()
# Wrap tensors in Variable class, set volatile=True for inference mode and to use minimal memory during inference
data = Variable(data, volatile=True)
# Forward pass
output = model(data).data
# Using confidence threshold, eliminate low-confidence predictions
all_boxes_1 = get_region_boxes(output.data, conf_thresh, num_classes)
all_boxes = np.reshape(np.array(all_boxes_1), (-1, 15))
all_boxes = torch.from_numpy(all_boxes).float()
if use_cuda:
all_boxes = all_boxes.cuda()
output_T = model_2(all_boxes).float()
if use_cuda:
gt_T = gt_T.cuda()
gt_R = gt_R.cuda()
input_final = torch.cat((output_T, all_boxes), 1)
output_R = model_3(input_final)
control_points = torch.FloatTensor([[[0.1, 0, 0]], [[0, 0.1, 0]],
[[-0.1, 0, 0]], [[0, -0.1,
0]]]).cuda()
gt_proj = torch.zeros(output.size(0), 4, 3).cuda()
pred_proj = torch.zeros(output.size(0), 4, 3).cuda()
# Iterate through all images in the batch
for j in range(output.size(0)):
# For each image, get all the predictions
boxes = all_boxes_1[j]
plot_boxes_polygon(data[j],
boxes,
"",
"",
savename='data/Results/batch%s_%s.jpeg' %
(str(batch_idx), str(j)),
class_names=['QR_Code'])
if boxes == [[]]:
continue
(x, y, z, w) = (gt_R[j][0], gt_R[j][1], gt_R[j][2], gt_R[j][3])
gt_rotation = torch.Tensor([[
2 * (x**2 + w**2) - 1, 2 * (x * y + z * w), 2 * (x * z - y * w)
], [
2 * (x * y - z * w), 2 * (y**2 + w**2) - 1, 2 * (y * z + x * w)
], [
2 * (x * z + y * w), 2 * (y * z - x * w), 2 * (z**2 + w**2) - 1
]])
gt_rotation = gt_rotation.cuda()
(x, y, z, w) = (output_R[j][0], output_R[j][1], output_R[j][2],
output_R[j][3])
pred_rotation = torch.Tensor([[
2 * (x**2 + w**2) - 1, 2 * (x * y + z * w), 2 * (x * z - y * w)
], [
2 * (x * y - z * w), 2 * (y**2 + w**2) - 1, 2 * (y * z + x * w)
], [
2 * (x * z + y * w), 2 * (y * z - x * w), 2 * (z**2 + w**2) - 1
]])
pred_rotation = pred_rotation.cuda()
for i in range(4):
gt_proj[j][i] = gt_T[j] + torch.mm(
gt_rotation, control_points[i].t()).view(-1)
pred_proj[j][i] = output_T[j] + torch.mm(
pred_rotation, control_points[i].t()).view(-1)
loss = 0
loss_tmp = torch.zeros(4, 4).cuda()
loss_m = torch.zeros(4).cuda()
for m in range(4):
for n in range(4):
loss_tmp[m][n] = nn.MSELoss(size_average=True)(
pred_proj[j][m], gt_proj[j][n])
loss_m, index = torch.min(loss_tmp, dim=1)
loss = torch.mean(loss_m)
out_file.write('batch%d_%d,%04f\n' % (batch_idx, j, loss))
def train(Pre_data, model, criterion, optimizer, epoch, args, scheduler):
losses = AverageMeter()
batch_time = AverageMeter()
data_time = AverageMeter()
train_loader = torch.utils.data.DataLoader(dataset.listDataset(
Pre_data,
args['save_path'],
shuffle=True,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]),
]),
train=True,
batch_size=args['batch_size'],
num_workers=args['workers'],
args=args),
batch_size=args['batch_size'],
drop_last=False)
args['lr'] = optimizer.param_groups[0]['lr']
print('epoch %d, processed %d samples, lr %.10f' %
(epoch, epoch * len(train_loader.dataset), args['lr']))
model.train()
end = time.time()
for i, (fname, img, gt_count) in enumerate(train_loader):
data_time.update(time.time() - end)
img = img.cuda()
out1 = model(img)
gt_count = gt_count.type(torch.FloatTensor).cuda().unsqueeze(1)
# print(out1.shape, kpoint.shape)
loss = criterion(out1, gt_count)
losses.update(loss.item(), img.size(0))
optimizer.zero_grad()
loss.backward()
optimizer.step()
batch_time.update(time.time() - end)
end = time.time()
if i % args['print_freq'] == 0:
print('4_Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'.format(
epoch,
i,
len(train_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses))
scheduler.step()
def valid(datacfg, modelcfg, weightfile):
def truths_length(truths, max_num_gt=50):
for i in range(max_num_gt):
if truths[i][1] == 0:
return i
# Parse configuration files
data_options = read_data_cfg(datacfg)
valid_images = data_options['valid']
meshname = data_options['mesh']
backupdir = data_options['backup']
name = data_options['name']
gpus = data_options['gpus']
fx = float(data_options['fx'])
fy = float(data_options['fy'])
u0 = float(data_options['u0'])
v0 = float(data_options['v0'])
im_width = int(data_options['width'])
im_height = int(data_options['height'])
if not os.path.exists(backupdir):
makedirs(backupdir)
# Parameters
seed = int(time.time())
os.environ['CUDA_VISIBLE_DEVICES'] = gpus
torch.cuda.manual_seed(seed)
save = False
testtime = True
num_classes = 1
testing_samples = 0.0
if save:
makedirs(backupdir + '/test')
makedirs(backupdir + '/test/gt')
makedirs(backupdir + '/test/pr')
# To save
testing_error_trans = 0.0
testing_error_angle = 0.0
testing_error_pixel = 0.0
errs_2d = []
errs_3d = []
errs_trans = []
errs_angle = []
errs_corner2D = []
preds_trans = []
preds_rot = []
preds_corners2D = []
gts_trans = []
gts_rot = []
gts_corners2D = []
# Read object model information, get 3D bounding box corners
mesh = MeshPly(meshname)
vertices = np.c_[np.array(mesh.vertices),
np.ones((len(mesh.vertices), 1))].transpose()
corners3D = get_3D_corners(vertices)
try:
diam = float(options['diam'])
except:
diam = calc_pts_diameter(np.array(mesh.vertices))
# Read intrinsic camera parameters
intrinsic_calibration = get_camera_intrinsic(u0, v0, fx, fy)
# Get validation file names
with open(valid_images) as fp:
tmp_files = fp.readlines()
valid_files = [item.rstrip() for item in tmp_files]
# Specicy model, load pretrained weights, pass to GPU and set the module in evaluation mode
model = Darknet(modelcfg)
model.print_network()
model.load_weights(weightfile)
model.cuda()
model.eval()
test_width = model.test_width
test_height = model.test_height
num_keypoints = model.num_keypoints
num_labels = num_keypoints * 2 + 3 # +2 for width, height, +1 for class label
# Get the parser for the test dataset
valid_dataset = dataset.listDataset(valid_images,
shape=(test_width, test_height),
shuffle=False,
transform=transforms.Compose([
transforms.ToTensor(),
]))
# Specify the number of workers for multiple processing, get the dataloader for the test dataset
kwargs = {'num_workers': 4, 'pin_memory': True}
test_loader = torch.utils.data.DataLoader(valid_dataset,
batch_size=1,
shuffle=False,
**kwargs)
logging(" Testing {}...".format(name))
logging(" Number of test samples: %d" % len(test_loader.dataset))
# Iterate through test batches (Batch size for test data is 1)
count = 0
for batch_idx, (data, target) in enumerate(test_loader):
t1 = time.time()
# Pass data to GPU
# import IPython; IPython.embed()
data = data.cuda()
# target = target.cuda()
# Wrap tensors in Variable class, set volatile=True for inference mode and to use minimal memory during inference
data = Variable(data, volatile=True)
t2 = time.time()
# Forward pass
output = model(data).data
t3 = time.time()
# Using confidence threshold, eliminate low-confidence predictions
all_boxes = get_region_boxes(output, num_classes, num_keypoints)
all_boxes = [t.cpu() for t in all_boxes]
t4 = time.time()
# Evaluation
# Iterate through all batch elements
for box_pr, target in zip([all_boxes], [target[0]]):
# For each image, get all the targets (for multiple object pose estimation, there might be more than 1 target per image)
truths = target.view(-1, num_labels)
# Get how many objects are present in the scene
num_gts = truths_length(truths)
# Iterate through each ground-truth object
for k in range(num_gts):
box_gt = list()
for j in range(1, 2 * num_keypoints + 1):
box_gt.append(truths[k][j])
box_gt.extend([1.0, 1.0])
box_gt.append(truths[k][0])
# Denormalize the corner predictions
corners2D_gt = np.array(np.reshape(box_gt[:18], [-1, 2]),
dtype='float32')
corners2D_pr = np.array(np.reshape(box_pr[:18], [-1, 2]),
dtype='float32')
corners2D_gt[:, 0] = corners2D_gt[:, 0] * im_width
corners2D_gt[:, 1] = corners2D_gt[:, 1] * im_height
corners2D_pr[:, 0] = corners2D_pr[:, 0] * im_width
corners2D_pr[:, 1] = corners2D_pr[:, 1] * im_height
preds_corners2D.append(corners2D_pr)
gts_corners2D.append(corners2D_gt)
# Compute corner prediction error
corner_norm = np.linalg.norm(corners2D_gt - corners2D_pr,
axis=1)
corner_dist = np.mean(corner_norm)
errs_corner2D.append(corner_dist)
# Compute [R|t] by pnp
R_gt, t_gt = pnp(
np.array(np.transpose(
np.concatenate((np.zeros((3, 1)), corners3D[:3, :]),
axis=1)),
dtype='float32'), corners2D_gt,
np.array(intrinsic_calibration, dtype='float32'))
R_pr, t_pr = pnp(
np.array(np.transpose(
np.concatenate((np.zeros((3, 1)), corners3D[:3, :]),
axis=1)),
dtype='float32'), corners2D_pr,
np.array(intrinsic_calibration, dtype='float32'))
# Compute translation error
trans_dist = np.sqrt(np.sum(np.square(t_gt - t_pr)))
errs_trans.append(trans_dist)
# Compute angle error
angle_dist = calcAngularDistance(R_gt, R_pr)
errs_angle.append(angle_dist)
# Compute pixel error
Rt_gt = np.concatenate((R_gt, t_gt), axis=1)
Rt_pr = np.concatenate((R_pr, t_pr), axis=1)
proj_2d_gt = compute_projection(vertices, Rt_gt,
intrinsic_calibration)
proj_2d_pred = compute_projection(vertices, Rt_pr,
intrinsic_calibration)
norm = np.linalg.norm(proj_2d_gt - proj_2d_pred, axis=0)
pixel_dist = np.mean(norm)
errs_2d.append(pixel_dist)
# Compute 3D distances
transform_3d_gt = compute_transformation(vertices, Rt_gt)
transform_3d_pred = compute_transformation(vertices, Rt_pr)
norm3d = np.linalg.norm(transform_3d_gt - transform_3d_pred,
axis=0)
vertex_dist = np.mean(norm3d)
errs_3d.append(vertex_dist)
# Sum errors
testing_error_trans += trans_dist
testing_error_angle += angle_dist
testing_error_pixel += pixel_dist
testing_samples += 1
count = count + 1
if save:
preds_trans.append(t_pr)
gts_trans.append(t_gt)
preds_rot.append(R_pr)
gts_rot.append(R_gt)
np.savetxt(
backupdir + '/test/gt/R_' + valid_files[count][-8:-3] +
'txt', np.array(R_gt, dtype='float32'))
np.savetxt(
backupdir + '/test/gt/t_' + valid_files[count][-8:-3] +
'txt', np.array(t_gt, dtype='float32'))
np.savetxt(
backupdir + '/test/pr/R_' + valid_files[count][-8:-3] +
'txt', np.array(R_pr, dtype='float32'))
np.savetxt(
backupdir + '/test/pr/t_' + valid_files[count][-8:-3] +
'txt', np.array(t_pr, dtype='float32'))
np.savetxt(
backupdir + '/test/gt/corners_' +
valid_files[count][-8:-3] + 'txt',
np.array(corners2D_gt, dtype='float32'))
np.savetxt(
backupdir + '/test/pr/corners_' +
valid_files[count][-8:-3] + 'txt',
np.array(corners2D_pr, dtype='float32'))
t5 = time.time()
# Compute 2D projection error, 6D pose error, 5cm5degree error
px_threshold = 5 # 5 pixel threshold for 2D reprojection error is standard in recent sota 6D object pose estimation works
eps = 1e-5
acc = len(np.where(
np.array(errs_2d) <= px_threshold)[0]) * 100. / (len(errs_2d) + eps)
acc5cm5deg = len(
np.where((np.array(errs_trans) <= 0.05)
& (np.array(errs_angle) <= 5))[0]) * 100. / (len(errs_trans) +
eps)
acc3d10 = len(np.where(
np.array(errs_3d) <= diam * 0.1)[0]) * 100. / (len(errs_3d) + eps)
acc5cm5deg = len(
np.where((np.array(errs_trans) <= 0.05)
& (np.array(errs_angle) <= 5))[0]) * 100. / (len(errs_trans) +
eps)
corner_acc = len(np.where(np.array(errs_corner2D) <= px_threshold)
[0]) * 100. / (len(errs_corner2D) + eps)
mean_err_2d = np.mean(errs_2d)
mean_corner_err_2d = np.mean(errs_corner2D)
nts = float(testing_samples)
if testtime:
print('-----------------------------------')
print(' tensor to cuda : %f' % (t2 - t1))
print(' forward pass : %f' % (t3 - t2))
print('get_region_boxes : %f' % (t4 - t3))
print(' prediction time : %f' % (t4 - t1))
print(' eval : %f' % (t5 - t4))
print('-----------------------------------')
# Print test statistics
logging('Results of {}'.format(name))
logging(' Acc using {} px 2D Projection = {:.2f}%'.format(
px_threshold, acc))
logging(' Acc using 10% threshold - {} vx 3D Transformation = {:.2f}%'.
format(diam * 0.1, acc3d10))
logging(' Acc using 5 cm 5 degree metric = {:.2f}%'.format(acc5cm5deg))
logging(
" Mean 2D pixel error is %f, Mean vertex error is %f, mean corner error is %f"
% (mean_err_2d, np.mean(errs_3d), mean_corner_err_2d))
logging(
' Translation error: %f m, angle error: %f degree, pixel error: % f pix'
% (testing_error_trans / nts, testing_error_angle / nts,
testing_error_pixel / nts))
if save:
predfile = backupdir + '/predictions_linemod_' + name + '.mat'
scipy.io.savemat(
predfile, {
'R_gts': gts_rot,
't_gts': gts_trans,
'corner_gts': gts_corners2D,
'R_prs': preds_rot,
't_prs': preds_trans,
'corner_prs': preds_corners2D
})
cuda_str = 'cuda:' + str(config['cuda']['gpu_id'])
device = torch.device(cuda_str if config['cuda']['using_cuda'] else "cpu")
# make dataset
transform = transforms.Compose([transforms.ToTensor()])
encoder = utils.LabelEncoder(
labels=config['hyperparameters']['labels'],
is_ignore_case=config['hyperparameters']['is_ignore_case'])
img_width = config['hyperparameters']['imgWidth']
img_height = config['hyperparameters']['imgHeight']
train_dataset = dataset.listDataset(root_path=config['data']['train'],
transform=transform,
encoder=encoder,
img_size=(img_width, img_height))
valid_dataset = dataset.listDataset(root_path=config['data']['valid'],
transform=transform,
encoder=encoder,
img_size=(img_width, img_height))
assert train_dataset
assert valid_dataset
train_loader = torch.utils.data.DataLoader(
train_dataset,
batch_size=config['hyperparameters']['batch_size'],
shuffle=True,
num_workers=0,
collate_fn=train_dataset.collate_fn)
def train(model, criterion, optimizer, epoch, coco):
losses = AverageMeter()
batch_time = AverageMeter()
data_time = AverageMeter()
train_loader = torch.utils.data.DataLoader(dataset.listDataset(
args.ilsvrc,
args.youtube,
args.data_type,
shuffle=True,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
]),
train=True,
batch_size=args.batch_size,
num_workers=args.workers,
coco=coco),
batch_size=args.batch_size)
model.train()
end = time.time()
for i, (z, x, template, gt_box) in enumerate(train_loader):
data_time.update(time.time() - end)
z = z.cuda()
z = Variable(z)
x = x.cuda()
x = Variable(x)
template = template.type(torch.FloatTensor).cuda()
template = Variable(template)
oup = model(z, x)
if isinstance(model, SiamFC) or isinstance(model, SiamVGG):
loss = criterion(oup, template)
elif isinstance(model, SiamFCRes22):
loss = model.train_loss(oup, template)
losses.update(loss.item(), x.size(0))
optimizer.zero_grad()
loss.backward()
if isinstance(model, SiamFCRes22):
torch.nn.utils.clip_grad_norm_(model.parameters(),
10) # gradient clip
if is_valid_number(loss.item()):
optimizer.step()
# optimizer.step()
batch_time.update(time.time() - end)
end = time.time()
if i % args.print_freq == 0:
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'.format(
epoch,
i,
len(train_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses))
def valid(datacfg, cfgfile, weightfile, outfile):
options = read_data_cfg(datacfg)
valid_images = options["valid"]
name_list = options["names"]
prefix = "results"
names = load_class_names(name_list)
with open(valid_images) as fp:
tmp_files = fp.readlines()
valid_files = [item.rstrip() for item in tmp_files]
m = Darknet(cfgfile)
m.print_network()
m.load_weights(weightfile)
m.cuda()
m.eval()
valid_dataset = dataset.listDataset(valid_images,
shape=(m.width, m.height),
shuffle=False,
transform=transforms.Compose([
transforms.ToTensor(),
]))
valid_batchsize = 2
assert (valid_batchsize > 1)
kwargs = {"num_workers": 4, "pin_memory": True}
valid_loader = torch.utils.data.DataLoader(valid_dataset,
batch_size=valid_batchsize,
shuffle=False,
**kwargs)
fps = [0] * m.num_classes
if not os.path.exists("results"):
os.mkdir("results")
for i in range(m.num_classes):
buf = "%s/%s%s.txt" % (prefix, outfile, names[i])
fps[i] = open(buf, "w")
lineId = -1
conf_thresh = 0.005
nms_thresh = 0.45
for batch_idx, (data, target) in enumerate(valid_loader):
data = data.cuda()
data = Variable(data, volatile=True)
output = m(data).data
batch_boxes = get_region_boxes(output, conf_thresh, m.num_classes,
m.anchors, m.num_anchors, 0, 1)
for i in range(output.size(0)):
lineId = lineId + 1
fileId = os.path.basename(valid_files[lineId]).split(".")[0]
width, height = get_image_size(valid_files[lineId])
print(valid_files[lineId])
boxes = batch_boxes[i]
boxes = nms(boxes, nms_thresh)
for box in boxes:
x1 = (box[0] - box[2] / 2.0) * width
y1 = (box[1] - box[3] / 2.0) * height
x2 = (box[0] + box[2] / 2.0) * width
y2 = (box[1] + box[3] / 2.0) * height
det_conf = box[4]
for j in range((len(box) - 5) / 2):
cls_conf = box[5 + 2 * j]
cls_id = box[6 + 2 * j]
prob = det_conf * cls_conf
fps[cls_id].write("%s %f %f %f %f %f\n" %
(fileId, prob, x1, y1, x2, y2))
for i in range(m.num_classes):
fps[i].close()
model.load_weights(weightfile)
model.print_network()
region_loss.seen = model.seen
processed_batches = model.seen/batch_size
init_width = model.width
init_height = model.height
init_epoch = model.seen/nsamples
kwargs = {'num_workers': num_workers, 'pin_memory': True} if use_cuda else {}
test_loader = torch.utils.data.DataLoader(
dataset.listDataset(testlist, shape=(init_width, init_height),
shuffle=False,
transform=transforms.Compose([
transforms.ToTensor(),
]), train=False),
batch_size=batch_size, shuffle=False, **kwargs)
if use_cuda:
if ngpus > 1:
model = torch.nn.DataParallel(model).cuda()
else:
model = model.cuda()
params_dict = dict(model.named_parameters())
params = []
for key, value in params_dict.items():
if key.find('.bn') >= 0 or key.find('.bias') >= 0:
params += [{'params': [value], 'weight_decay': 0.0}]
os.system('mkdir {0}'.format(opt.experiment))
opt.manualSeed = random.randint(1, 10000) # fix seed
print("Random Seed: ", opt.manualSeed)
random.seed(opt.manualSeed)
np.random.seed(opt.manualSeed)
torch.manual_seed(opt.manualSeed)
cudnn.benchmark = True
if torch.cuda.is_available() and not opt.cuda:
print(
"WARNING: You have a CUDA device, so you should probably run with --cuda"
)
train_dataset = dataset.listDataset(list_file=opt.trainlist)
assert train_dataset
if not opt.random_sample:
sampler = dataset.randomSequentialSampler(train_dataset, opt.batchSize)
else:
sampler = None
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=opt.batchSize,
shuffle=False,
sampler=sampler,
num_workers=int(opt.workers),
collate_fn=dataset.alignCollate(
imgH=opt.imgH,
imgW=opt.imgW,
keep_ratio=opt.keep_ratio))
def train(train_list, model, criterion, optimizer, epoch,ring,ring_weight):
ring = torch.from_numpy(ring).type(torch.FloatTensor).cuda()
ring_weight = torch.from_numpy(ring_weight).type(torch.FloatTensor).cuda()
ring = Variable(ring)
ring_weight = Variable(ring_weight)
losses = AverageMeter()
batch_time = AverageMeter()
data_time = AverageMeter()
train_loader = torch.utils.data.DataLoader(
dataset.listDataset(train_list,
shuffle=True,
transform=transforms.Compose([
transforms.ToTensor(),transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]),
]),
train=True,
seen=model.seen,
batch_size=args.batch_size,
num_workers=args.workers),
batch_size=args.batch_size)
print('epoch %d, processed %d samples, lr %.10f' % (epoch, epoch * len(train_loader.dataset), args.lr))
model.train()
end = time.time()
for i,(img, target,signal,signal_down)in enumerate(train_loader):
data_time.update(time.time() - end)
img = img.cuda()
img = Variable(img)
sig_down = signal_down.type(torch.FloatTensor).unsqueeze(0).cuda()
sig_down = Variable(sig_down)
sig = signal.type(torch.FloatTensor).unsqueeze(0).cuda()
sig = Variable(sig)
#compute round people
round_count = torch.zeros([24]).type(torch.FloatTensor).cuda()
round_count = Variable(round_count)
for k in range(24):
round_count[k] = torch.mul(sig,ring[k]).sum()
sig = round_count
# #random generate a mask that mask out a ring
# rand = random.randint(0,174)
# rand2 = random.randint(0,349)
# for q in range(3):
# img[:,q,rand:rand+50,rand2:rand2+50] = 0
output = model(img,sig_down)
target = target.type(torch.FloatTensor).unsqueeze(0).cuda()
target = Variable(target)
##multiply ring weight
total_sum = torch.tensor([0]).type(torch.FloatTensor).cuda()
total_sum = Variable(total_sum)
for k in range(24):
ring_sum = torch.abs(torch.mul(output,ring[k]).sum()-torch.mul(target,ring[k]).sum())
total_sum += ring_sum*ring_weight[k]
loss = criterion(output,target)+ 0.005 * total_sum
losses.update(loss.item())
optimizer.zero_grad()
loss.backward()
optimizer.step()
batch_time.update(time.time() - end)
end = time.time()
if i % args.print_freq == 0:
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
.format(
epoch, i, len(train_loader), batch_time=batch_time,
data_time=data_time, loss=losses))
model.load_weights(weightfile)
model.print_network()
region_loss.seen = model.seen
processed_batches = model.seen / batch_size
init_width = model.width
init_height = model.height
init_epoch = int(model.seen / nsamples)
kwargs = {
'num_workers': num_workers,
'pin_memory': True
} if use_cuda else {}
test_loader = torch.utils.data.DataLoader(dataset.listDataset(
testlist, shape=(init_width, init_height), shuffle=False, train=False),
batch_size=batch_size,
shuffle=False,
**kwargs)
if use_cuda:
cudnn.enabled = True
cudnn.benchmark = True
if ngpus > 1:
model = torch.nn.DataParallel(model).cuda()
else:
model = model.cuda()
params_dict = dict(model.named_parameters())
params = []
for key, value in params_dict.items():
def valid(datacfg, cfgfile, weightfile, outfile):
def truths_length(truths):
for i in range(50):
if truths[i][1] == 0:
return i
# Parse configuration files
options = read_data_cfg(datacfg)
valid_images = options['valid']
meshname = options['mesh']
backupdir = options['backup']
name = options['name']
if not os.path.exists(backupdir):
makedirs(backupdir)
# Parameters
prefix = 'results'
seed = int(time.time())
gpus = '0' # Specify which gpus to use
test_width = 544
test_height = 544
torch.manual_seed(seed)
use_cuda = True
if use_cuda:
os.environ['CUDA_VISIBLE_DEVICES'] = gpus
torch.cuda.manual_seed(seed)
save = False
testtime = True
use_cuda = True
num_classes = 1
testing_samples = 0.0
eps = 1e-5
notpredicted = 0
conf_thresh = 0.1
nms_thresh = 0.4
match_thresh = 0.5
if save:
makedirs(backupdir + '/test')
makedirs(backupdir + '/test/gt')
makedirs(backupdir + '/test/pr')
# To save
testing_error_trans = 0.0
testing_error_angle = 0.0
testing_error_pixel = 0.0
errs_2d = []
errs_3d = []
errs_trans = []
errs_angle = []
errs_corner2D = []
preds_trans = []
preds_rot = []
preds_corners2D = []
gts_trans = []
gts_rot = []
gts_corners2D = []
edges_corners = [[0, 1], [0, 2], [0, 4], [1, 3], [1, 5], [2, 3], [2, 6],
[3, 7], [4, 5], [4, 6], [5, 7], [6, 7]]
# Read object model information, get 3D bounding box corners
mesh = MeshPly(meshname)
vertices = np.c_[np.array(mesh.vertices),
np.ones((len(mesh.vertices), 1))].transpose()
corners3D = get_3D_corners(vertices)
# diam = calc_pts_diameter(np.array(mesh.vertices))
diam = float(options['diam'])
# Read intrinsic camera parameters
internal_calibration = get_camera_intrinsic()
# Get validation file names
with open(valid_images) as fp:
tmp_files = fp.readlines()
valid_files = [item.rstrip() for item in tmp_files]
# Specicy model, load pretrained weights, pass to GPU and set the module in evaluation mode
model = Darknet(cfgfile)
model.print_network()
model.load_weights(weightfile)
model.cuda()
model.eval()
# Get the parser for the test dataset
valid_dataset = dataset.listDataset(valid_images,
shape=(test_width, test_height),
shuffle=False,
transform=transforms.Compose([
transforms.ToTensor(),
]))
valid_batchsize = 1
# Specify the number of workers for multiple processing, get the dataloader for the test dataset
kwargs = {'num_workers': 4, 'pin_memory': True}
test_loader = torch.utils.data.DataLoader(valid_dataset,
batch_size=valid_batchsize,
shuffle=False,
**kwargs)
logging(" Testing {}...".format(name))
logging(" Number of test samples: %d" % len(test_loader.dataset))
# Iterate through test batches (Batch size for test data is 1)
count = 0
z = np.zeros((3, 1))
for batch_idx, (data, target) in enumerate(test_loader):
# Images
img = data[0, :, :, :]
img = img.numpy().squeeze()
img = np.transpose(img, (1, 2, 0))
t1 = time.time()
# Pass data to GPU
if use_cuda:
data = data.cuda()
target = target.cuda()
# Wrap tensors in Variable class, set volatile=True for inference mode and to use minimal memory during inference
data = Variable(data, volatile=True)
t2 = time.time()
# Forward pass
output = model(data).data
t3 = time.time()
# Using confidence threshold, eliminate low-confidence predictions
all_boxes = get_region_boxes(output, conf_thresh, num_classes)
t4 = time.time()
# Iterate through all images in the batch
for i in range(output.size(0)):
# For each image, get all the predictions
boxes = all_boxes[i]
# For each image, get all the targets (for multiple object pose estimation, there might be more than 1 target per image)
truths = target[i].view(-1, 21)
# Get how many object are present in the scene
num_gts = truths_length(truths)
# Iterate through each ground-truth object
for k in range(num_gts):
box_gt = [
truths[k][1], truths[k][2], truths[k][3], truths[k][4],
truths[k][5], truths[k][6], truths[k][7], truths[k][8],
truths[k][9], truths[k][10], truths[k][11], truths[k][12],
truths[k][13], truths[k][14], truths[k][15], truths[k][16],
truths[k][17], truths[k][18], 1.0, 1.0, truths[k][0]
]
best_conf_est = -1
# If the prediction has the highest confidence, choose it as our prediction for single object pose estimation
for j in range(len(boxes)):
if (boxes[j][18] > best_conf_est):
match = corner_confidence9(
box_gt[:18], torch.FloatTensor(boxes[j][:18]))
box_pr = boxes[j]
best_conf_est = boxes[j][18]
# Denormalize the corner predictions
corners2D_gt = np.array(np.reshape(box_gt[:18], [9, 2]),
dtype='float32')
corners2D_pr = np.array(np.reshape(box_pr[:18], [9, 2]),
dtype='float32')
corners2D_gt[:, 0] = corners2D_gt[:, 0] * 640
corners2D_gt[:, 1] = corners2D_gt[:, 1] * 480
corners2D_pr[:, 0] = corners2D_pr[:, 0] * 640
corners2D_pr[:, 1] = corners2D_pr[:, 1] * 480
preds_corners2D.append(corners2D_pr)
gts_corners2D.append(corners2D_gt)
# Compute corner prediction error
corner_norm = np.linalg.norm(corners2D_gt - corners2D_pr,
axis=1)
corner_dist = np.mean(corner_norm)
errs_corner2D.append(corner_dist)
# Compute [R|t] by pnp
R_gt, t_gt = pnp(
np.array(np.transpose(
np.concatenate((np.zeros((3, 1)), corners3D[:3, :]),
axis=1)),
dtype='float32'), corners2D_gt,
np.array(internal_calibration, dtype='float32'))
R_pr, t_pr = pnp(
np.array(np.transpose(
np.concatenate((np.zeros((3, 1)), corners3D[:3, :]),
axis=1)),
dtype='float32'), corners2D_pr,
np.array(internal_calibration, dtype='float32'))
if save:
preds_trans.append(t_pr)
gts_trans.append(t_gt)
preds_rot.append(R_pr)
gts_rot.append(R_gt)
np.savetxt(
backupdir + '/test/gt/R_' + valid_files[count][-8:-3] +
'txt', np.array(R_gt, dtype='float32'))
np.savetxt(
backupdir + '/test/gt/t_' + valid_files[count][-8:-3] +
'txt', np.array(t_gt, dtype='float32'))
np.savetxt(
backupdir + '/test/pr/R_' + valid_files[count][-8:-3] +
'txt', np.array(R_pr, dtype='float32'))
np.savetxt(
backupdir + '/test/pr/t_' + valid_files[count][-8:-3] +
'txt', np.array(t_pr, dtype='float32'))
np.savetxt(
backupdir + '/test/gt/corners_' +
valid_files[count][-8:-3] + 'txt',
np.array(corners2D_gt, dtype='float32'))
np.savetxt(
backupdir + '/test/pr/corners_' +
valid_files[count][-8:-3] + 'txt',
np.array(corners2D_pr, dtype='float32'))
# Compute translation error
trans_dist = np.sqrt(np.sum(np.square(t_gt - t_pr)))
errs_trans.append(trans_dist)
# Compute angle error
angle_dist = calcAngularDistance(R_gt, R_pr)
errs_angle.append(angle_dist)
# Compute pixel error
Rt_gt = np.concatenate((R_gt, t_gt), axis=1)
Rt_pr = np.concatenate((R_pr, t_pr), axis=1)
proj_2d_gt = compute_projection(vertices, Rt_gt,
internal_calibration)
proj_2d_pred = compute_projection(vertices, Rt_pr,
internal_calibration)
proj_corners_gt = np.transpose(
compute_projection(corners3D, Rt_gt, internal_calibration))
proj_corners_pr = np.transpose(
compute_projection(corners3D, Rt_pr, internal_calibration))
norm = np.linalg.norm(proj_2d_gt - proj_2d_pred, axis=0)
pixel_dist = np.mean(norm)
errs_2d.append(pixel_dist)
# Visualize
fig = plt.figure()
plt.xlim((0, 640))
plt.ylim((0, 480))
plt.imshow(scipy.misc.imresize(img, (480, 640)))
# Projections
for edge in edges_corners:
plt.plot(proj_corners_gt[edge, 0],
proj_corners_gt[edge, 1],
color='g',
linewidth=3.0)
plt.plot(proj_corners_pr[edge, 0],
proj_corners_pr[edge, 1],
color='b',
linewidth=3.0)
plt.gca().invert_yaxis()
# plt.show()
plt.savefig(outfile + '/output_' + str(count) + '_.png',
bbox_inches='tight')
fig.canvas.draw()
count = count + 1
# Compute 3D distances
transform_3d_gt = compute_transformation(vertices, Rt_gt)
transform_3d_pred = compute_transformation(vertices, Rt_pr)
norm3d = np.linalg.norm(transform_3d_gt - transform_3d_pred,
axis=0)
vertex_dist = np.mean(norm3d)
errs_3d.append(vertex_dist)
# Sum errors
testing_error_trans += trans_dist
testing_error_angle += angle_dist
testing_error_pixel += pixel_dist
testing_samples += 1
count = count + 1
t5 = time.time()
# Compute 2D projection error, 6D pose error, 5cm5degree error
px_threshold = 5
acc = len(np.where(
np.array(errs_2d) <= px_threshold)[0]) * 100. / (len(errs_2d) + eps)
acc5cm5deg = len(
np.where((np.array(errs_trans) <= 0.05)
& (np.array(errs_angle) <= 5))[0]) * 100. / (len(errs_trans) +
eps)
acc3d10 = len(np.where(
np.array(errs_3d) <= diam * 0.1)[0]) * 100. / (len(errs_3d) + eps)
acc5cm5deg = len(
np.where((np.array(errs_trans) <= 0.05)
& (np.array(errs_angle) <= 5))[0]) * 100. / (len(errs_trans) +
eps)
corner_acc = len(np.where(np.array(errs_corner2D) <= px_threshold)
[0]) * 100. / (len(errs_corner2D) + eps)
mean_err_2d = np.mean(errs_2d)
mean_corner_err_2d = np.mean(errs_corner2D)
nts = float(testing_samples)
if testtime:
print('-----------------------------------')
print(' tensor to cuda : %f' % (t2 - t1))
print(' predict : %f' % (t3 - t2))
print('get_region_boxes : %f' % (t4 - t3))
print(' eval : %f' % (t5 - t4))
print(' total : %f' % (t5 - t1))
print('-----------------------------------')
# Print test statistics
logging('Results of {}'.format(name))
logging(' Acc using {} px 2D Projection = {:.2f}%'.format(
px_threshold, acc))
logging(' Acc using 10% threshold - {} vx 3D Transformation = {:.2f}%'.
format(diam * 0.1, acc3d10))
logging(' Acc using 5 cm 5 degree metric = {:.2f}%'.format(acc5cm5deg))
logging(
" Mean 2D pixel error is %f, Mean vertex error is %f, mean corner error is %f"
% (mean_err_2d, np.mean(errs_3d), mean_corner_err_2d))
logging(
' Translation error: %f m, angle error: %f degree, pixel error: % f pix'
% (testing_error_trans / nts, testing_error_angle / nts,
testing_error_pixel / nts))
if save:
predfile = backupdir + '/predictions_linemod_' + name + '.mat'
scipy.io.savemat(
predfile, {
'R_gts': gts_rot,
't_gts': gts_trans,
'corner_gts': gts_corners2D,
'R_prs': preds_rot,
't_prs': preds_trans,
'corner_prs': preds_corners2D
})
def valid(datacfg, darknetcfg, learnetcfg, weightfile, outfile):
options = read_data_cfg(datacfg)
valid_images = options['valid']
metadict = options['meta']
# name_list = options['names']
# backup = cfg.backup
ckpt = weightfile.split('/')[-1].split('.')[0]
backup = weightfile.split('/')[-2]
prefix = 'results/' + backup.split('/')[-1] + '/e' + ckpt
print('saving to: ' + prefix)
# prefix = 'results/' + weightfile.split('/')[1]
# names = load_class_names(name_list)
with open(valid_images) as fp:
tmp_files = fp.readlines()
valid_files = [item.rstrip() for item in tmp_files]
m = Darknet(darknetcfg, learnetcfg)
m.print_network()
m.load_weights(weightfile)
m.cuda()
m.eval()
valid_dataset = dataset.listDataset(valid_images, shape=(m.width, m.height),
shuffle=False,
transform=transforms.Compose([
transforms.ToTensor(),
]))
valid_batchsize = 2
assert (valid_batchsize > 1)
kwargs = {'num_workers': 4, 'pin_memory': True}
valid_loader = torch.utils.data.DataLoader(
valid_dataset, batch_size=valid_batchsize, shuffle=False, **kwargs)
metaset = dataset.MetaDataset(metafiles=metadict, train=False)
metaloader = torch.utils.data.DataLoader(
metaset,
batch_size=metaset.batch_size,
shuffle=False,
**kwargs
)
metaloader = iter(metaloader)
n_cls = len(metaset.classes)
if not os.path.exists(prefix):
# os.mkdir(prefix)
os.makedirs(prefix)
fps = [0] * n_cls
for i, cls_name in enumerate(metaset.classes):
buf = '%s/%s%s.txt' % (prefix, outfile, cls_name)
fps[i] = open(buf, 'w')
lineId = -1
conf_thresh = 0.005
nms_thresh = 0.45
for batch_idx, (data, target) in enumerate(valid_loader):
metax, mask = metaloader.next()
# print(ids)
data = data.cuda()
mask = mask.cuda()
metax = metax.cuda()
data = Variable(data, volatile=True)
mask = Variable(mask, volatile=True)
metax = Variable(metax, volatile=True)
output = m(data, metax, mask)
if isinstance(output, tuple):
output = (output[0].data, output[1].data)
else:
output = output.data
batch_boxes = get_region_boxes_v2(output, n_cls, conf_thresh, m.num_classes, m.anchors, m.num_anchors, 0, 1)
if isinstance(output, tuple):
bs = output[0].size(0)
else:
assert output.size(0) % n_cls == 0
bs = output.size(0) // n_cls
for b in range(bs):
lineId = lineId + 1
imgpath = valid_dataset.lines[lineId].rstrip()
print(imgpath)
imgid = os.path.basename(imgpath).split('.')[0]
width, height = get_image_size(imgpath)
for i in range(n_cls):
# oi = i * bs + b
oi = b * n_cls + i
boxes = batch_boxes[oi]
boxes = nms(boxes, nms_thresh)
for box in boxes:
x1 = (box[0] - box[2] / 2.0) * width
y1 = (box[1] - box[3] / 2.0) * height
x2 = (box[0] + box[2] / 2.0) * width
y2 = (box[1] + box[3] / 2.0) * height
det_conf = box[4]
for j in range((len(box) - 5) / 2):
cls_conf = box[5 + 2 * j]
cls_id = box[6 + 2 * j]
prob = det_conf * cls_conf
fps[i].write('%s %f %f %f %f %f\n' % (imgid, prob, x1, y1, x2, y2))
for i in range(n_cls):
fps[i].close()
def train(epoch):
global processed_batches
# Initialize timer
t0 = time.time()
# Get the dataloader for training dataset
train_loader = torch.utils.data.DataLoader(dataset.listDataset(
trainlist,
shape=(init_width, init_height),
shuffle=True,
transform=transforms.Compose([
transforms.ToTensor(),
]),
train=True,
seen=model.seen,
batch_size=batch_size,
num_workers=num_workers,
bg_file_names=bg_file_names),
batch_size=batch_size,
shuffle=False,
**kwargs)
# TRAINING
lr = adjust_learning_rate(optimizer, processed_batches)
logging('epoch %d, processed %d samples, lr %f' %
(epoch, epoch * len(train_loader.dataset), lr))
# Start training
model.train()
t1 = time.time()
avg_time = torch.zeros(9)
niter = 0
# Iterate through batches
for batch_idx, (data, target) in enumerate(train_loader):
t2 = time.time()
# adjust learning rate
adjust_learning_rate(optimizer, processed_batches)
processed_batches = processed_batches + 1
# Pass the data to GPU
if use_cuda:
data = data.cuda()
t3 = time.time()
# Wrap tensors in Variable class for automatic differentiation
data, target = Variable(data), Variable(target)
t4 = time.time()
# Zero the gradients before running the backward pass
optimizer.zero_grad()
t5 = time.time()
# Forward pass
output = model(data)
t6 = time.time()
model.seen = model.seen + data.data.size(0)
region_loss.seen = region_loss.seen + data.data.size(0)
# Compute loss, grow an array of losses for saving later on
loss = region_loss(output, target, epoch)
training_iters.append(
epoch * math.ceil(len(train_loader.dataset) / float(batch_size)) +
niter)
training_losses.append(convert2cpu(loss.data))
niter += 1
t7 = time.time()
# Backprop: compute gradient of the loss with respect to model parameters
loss.backward()
t8 = time.time()
# Update weights
optimizer.step()
t9 = time.time()
# Print time statistics
if False and batch_idx > 1:
avg_time[0] = avg_time[0] + (t2 - t1)
avg_time[1] = avg_time[1] + (t3 - t2)
avg_time[2] = avg_time[2] + (t4 - t3)
avg_time[3] = avg_time[3] + (t5 - t4)
avg_time[4] = avg_time[4] + (t6 - t5)
avg_time[5] = avg_time[5] + (t7 - t6)
avg_time[6] = avg_time[6] + (t8 - t7)
avg_time[7] = avg_time[7] + (t9 - t8)
avg_time[8] = avg_time[8] + (t9 - t1)
print('-------------------------------')
print(' load data : %f' % (avg_time[0] / (batch_idx)))
print(' cpu to cuda : %f' % (avg_time[1] / (batch_idx)))
print('cuda to variable : %f' % (avg_time[2] / (batch_idx)))
print(' zero_grad : %f' % (avg_time[3] / (batch_idx)))
print(' forward feature : %f' % (avg_time[4] / (batch_idx)))
print(' forward loss : %f' % (avg_time[5] / (batch_idx)))
print(' backward : %f' % (avg_time[6] / (batch_idx)))
print(' step : %f' % (avg_time[7] / (batch_idx)))
print(' total : %f' % (avg_time[8] / (batch_idx)))
t1 = time.time()
t1 = time.time()
return epoch * math.ceil(
len(train_loader.dataset) / float(batch_size)) + niter - 1
def train(train_list, teacher_model, student_model, criterion, optimizer,
epoch):
losses = AverageMeter()
batch_time = AverageMeter()
data_time = AverageMeter()
# _train_loader = dataset.listDataset(train_list,
# shuffle=True,
# transform=transforms.Compose([
# transforms.ToTensor(),
# transforms.Normalize(mean=[0.485, 0.456, 0.406], # from img net ?
# std=[0.229, 0.224, 0.225]),
# ]),
# train=True,
# # seen=model.seen,
# )
# print(" line 196 ..........................")
# train_loader = DataLoader( _train_loader,
# batch_size=args.batch_size,
# shuffle=True,
# num_workers=6)
train_loader = torch.utils.data.DataLoader(
dataset.listDataset(
train_list,
shuffle=True,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]),
]),
train=True,
# seen=model.seen,
batch_size=args.batch_size,
num_workers=args.workers),
batch_size=args.batch_size)
print(
f'epoch {epoch}, processed {(train_loader.__len__)} samples, lr {args.lr} \n\n'
)
student_model.train()
end = time.time()
# print(f"train loader dataset: \n", str(train_loader))
for i, (img, target) in enumerate(train_loader):
data_time.update(time.time() - end)
# img = torch.randn(1,3,400,800).float()
img = img.cuda()
img = Variable(img)
target = target.type(torch.FloatTensor).unsqueeze(1).cuda()
target = Variable(target)
print(f"img ********************************* TRAIN.PY : {img.shape}")
with torch.no_grad():
teacher_output, teacher_kd_list, teacher_resize_list = teacher_model(
img)
# print(" line 224 ............______________________________________________________________-.")
student_output, student_kd_list, student_resize_list = student_model(
img)
print(
f"target ****************************** TRAIN.PY: {target.shape}")
print("target = ", target.sum().item())
print("teacher output: ", teacher_output.sum())
# break
loss = criterion(student_kd_list, teacher_kd_list, student_resize_list,
teacher_resize_list, student_output, teacher_output,
target)
losses.update(loss.item(), img.size(0))
optimizer.zero_grad()
loss.backward()
optimizer.step()
batch_time.update(time.time() - end)
end = time.time()
if i % args.print_freq == 0:
print('\n\n\nEpoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'.format(
epoch,
i,
len(train_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses))
print(
" accuracy: ",
student_output.sum().data / target.sum().data * 100, " % \n\n")
def valid(datacfg, cfgfile, weightfile, outfile):
options = read_data_cfg(datacfg)
valid_images = options['valid']
# backup = cfg.backup
backup = weightfile.split('/')[-2]
ckpt = weightfile.split('/')[-1].split('.')[0]
prefix = 'results/' + backup.split('/')[-1] + '/e' + ckpt
print('saving to: ' + prefix)
names = cfg.classes
with open(valid_images) as fp:
tmp_files = fp.readlines()
valid_files = [item.rstrip() for item in tmp_files]
m = Darknet(cfgfile)
m.print_network()
m.load_weights(weightfile)
m.cuda()
m.eval()
valid_dataset = dataset.listDataset(valid_images,
shape=(m.width, m.height),
shuffle=False,
transform=transforms.Compose([
transforms.ToTensor(),
]))
valid_batchsize = 2
assert (valid_batchsize > 1)
kwargs = {'num_workers': 4, 'pin_memory': True}
valid_loader = torch.utils.data.DataLoader(valid_dataset,
batch_size=valid_batchsize,
shuffle=False,
**kwargs)
fps = [0] * m.num_classes
if not os.path.exists(prefix):
# os.mkdir(prefix)
os.makedirs(prefix)
for i in range(m.num_classes):
buf = '%s/%s%s.txt' % (prefix, outfile, names[i])
fps[i] = open(buf, 'w')
lineId = -1
conf_thresh = 0.005
nms_thresh = 0.45
for batch_idx, (data, target) in enumerate(valid_loader):
data = data.cuda()
data = Variable(data, volatile=True)
output = m(data).data
batch_boxes = get_region_boxes(output, conf_thresh, m.num_classes,
m.anchors, m.num_anchors, 0, 1)
for i in range(output.size(0)):
lineId = lineId + 1
fileId = os.path.basename(valid_files[lineId]).split('.')[0]
width, height = get_image_size(valid_files[lineId])
print(valid_files[lineId])
boxes = batch_boxes[i]
boxes = nms(boxes, nms_thresh)
for box in boxes:
x1 = (box[0] - box[2] / 2.0) * width
y1 = (box[1] - box[3] / 2.0) * height
x2 = (box[0] + box[2] / 2.0) * width
y2 = (box[1] + box[3] / 2.0) * height
det_conf = box[4]
# import pdb
# pdb.set_trace()
for j in range((len(box) - 5) / 2):
cls_conf = box[5 + 2 * j]
cls_id = box[6 + 2 * j]
prob = det_conf * cls_conf
fps[cls_id].write('%s %f %f %f %f %f\n' %
(fileId, prob, x1, y1, x2, y2))
# fps[cls_id].write('%s %f %f %f %f %f %f\n' % (fileId, det_conf, cls_conf, x1, y1, x2, y2))
for i in range(m.num_classes):
fps[i].close()
def train(epoch):
global processed_batches
global seen
global region_loss_seen
t0 = time.time()
def batch_iter(it, batch_size):
def list2np(t):
imgs, labels = zip(*t)
retimgs = np.zeros((len(imgs),) + imgs[0].shape, dtype=np.float32)
retlabels = np.zeros(
(len(labels),) + labels[0].shape, dtype=np.float32)
for i, img in enumerate(imgs):
retimgs[i, :, :, :] = img
for i, label in enumerate(labels):
retlabels[i, :] = label
return retimgs, retlabels
retlist = []
for i, item in enumerate(it):
retlist.append(item)
if i % batch_size == batch_size - 1:
ret = list2np(retlist)
# Don't train for batches that contain no labels
if not (np.sum(ret[1]) == 0):
yield ret
retlist = []
# Excess data is discarded
if len(retlist) > 0:
ret = list2np(retlist)
# Don't train for batches that contain no labels
if not (np.sum(ret[1]) == 0):
yield ret
train_loader_base = dataset.listDataset(args.train, args, shape=(init_width, init_height),
shuffle=True,
train=True,
seen=seen,
batch_size=batch_size,
num_workers=num_workers)
train_loader = batch_iter(iter(train_loader_base), batch_size=batch_size)
lr = adjust_learning_rate(solver_convweights, processed_batches)
lr = adjust_learning_rate(solver_others, processed_batches)
logging('epoch %d, processed %d samples, lr %f' %
(epoch, epoch * len(train_loader_base), lr))
yolo_x_nnabla, yolo_features_nnabla, yolo_vars, yolo_tvars, loss_nnabla = create_network(
batch_size, init_height, init_width, args)
t1 = time.time()
step_called = False
solver_convweights.zero_grad()
solver_others.zero_grad()
for batch_idx, (data_tensor, target_tensor) in enumerate(train_loader):
dn = data_tensor
tn = target_tensor
if dn.shape != yolo_x_nnabla.shape:
del(yolo_x_nnabla)
del(yolo_features_nnabla)
yolo_x_nnabla, yolo_features_nnabla, yolo_vars, yolo_tvars, loss_nnabla = create_network(
dn.shape[0], dn.shape[2], dn.shape[3], args)
yolo_x_nnabla.d = dn
yolo_features_nnabla.forward()
for v in yolo_vars:
v.forward()
region_loss_seen += data_tensor.shape[0]
nGT, nCorrect, nProposals = region_loss.forward_nnabla(
args, region_loss_seen, yolo_features_nnabla, target_tensor, yolo_vars, yolo_tvars)
loss_nnabla.forward()
loss_nnabla.backward()
print('%d: nGT %d, recall %d, proposals %d, loss: total %f' %
(region_loss_seen, nGT, nCorrect, nProposals, loss_nnabla.d))
yolo_features_nnabla.backward(grad=None, clear_buffer=True)
if batch_idx % args.accum_times == args.accum_times - 1:
step_called = True
solver_convweights.weight_decay(args.decay*batch_size)
solver_convweights.update()
solver_convweights.zero_grad()
solver_others.update()
solver_others.zero_grad()
processed_batches = processed_batches + 1
adjust_learning_rate(solver_convweights, processed_batches)
adjust_learning_rate(solver_others, processed_batches)
t1 = time.time()
if not step_called:
solver_convweights.weight_decay(args.decay*batch_size)
solver_convweights.update()
solver_convweights.zero_grad()
solver_others.update()
solver_others.zero_grad()
print()
t1 = time.time()
logging('training with %f samples/s' % (len(train_loader_base)/(t1-t0)))
if (epoch+1) % args.save_interval == 0:
logging('save weights to %s/%06d.h5' % (args.output, epoch+1))
seen = (epoch + 1) * len(train_loader_base)
nn.save_parameters('%s/%06d.h5' % (args.output, epoch+1))
testing_errors_angle = []
testing_errors_pixel = []
testing_accuracies = []
# Get the intrinsic camerea matrix, mesh, vertices and corners of the model
mesh = MeshPly(meshname)
vertices = np.c_[np.array(mesh.vertices), np.ones((len(mesh.vertices), 1))].transpose()
corners3D = get_3D_corners(vertices)
internal_calibration = get_camera_intrinsic()
# Specify the number of workers
kwargs = {'num_workers': num_workers, 'pin_memory': True} if use_cuda else {}
# Get the dataloader for test data
test_loader = torch.utils.data.DataLoader(dataset.listDataset(testlist, shape=(test_width, test_height),
shuffle=False,
transform=transforms.Compose([transforms.ToTensor(),]),
train=False),
batch_size=1, shuffle=False, **kwargs)
# Pass the model to GPU
if use_cuda:
model = model.cuda() # model = torch.nn.DataParallel(model, device_ids=[0]).cuda() # Multiple GPU parallelism
# Get the optimizer
params_dict = dict(model.named_parameters())
params = []
for key, value in params_dict.items():
if key.find('.bn') >= 0 or key.find('.bias') >= 0:
params += [{'params': [value], 'weight_decay': 0.0}]
else:
params += [{'params': [value], 'weight_decay': decay*batch_size}]
def valid(datacfg, cfgfile, weightfile, outfile):
def truths_length(truths):
for i in range(50):
if truths[i][1] == 0:
return i
# Parse configuration files
options = read_data_cfg(datacfg)
valid_images = options['valid']
meshname = options['mesh']
backupdir = options['backup']
name = options['name']
if not os.path.exists(backupdir):
makedirs(backupdir)
# Parameters
prefix = 'results'
seed = int(time.time())
gpus = '0' # Specify which gpus to use
test_width = 544
test_height = 544
torch.manual_seed(seed)
use_cuda = True
visualize = True
if use_cuda:
os.environ['CUDA_VISIBLE_DEVICES'] = gpus
torch.cuda.manual_seed(seed)
save = False
testtime = True
use_cuda = True
num_classes = 1
eps = 1e-5
notpredicted = 0
conf_thresh = 0.1
nms_thresh = 0.4
match_thresh = 0.5
edges_corners = [[0, 1], [0, 2], [0, 4], [1, 3], [1, 5], [2, 3], [2, 6],
[3, 7], [4, 5], [4, 6], [5, 7], [6, 7]]
if save:
makedirs(backupdir + '/test')
makedirs(backupdir + '/test/gt')
makedirs(backupdir + '/test/pr')
# Read object model information, get 3D bounding box corners
mesh = MeshPly(meshname)
vertices = np.c_[np.array(mesh.vertices),
np.ones((len(mesh.vertices), 1))].transpose()
corners3D = get_3D_corners(vertices)
# diam = calc_pts_diameter(np.array(mesh.vertices))
diam = float(options['diam'])
# Read intrinsic camera parameters
internal_calibration = get_camera_intrinsic()
# Get validation file names
with open(valid_images) as fp:
tmp_files = fp.readlines()
valid_files = [item.rstrip() for item in tmp_files]
# Specicy model, load pretrained weights, pass to GPU and set the module in evaluation mode
model = Darknet(cfgfile)
# model.print_network()
model.load_weights(weightfile)
model.cuda()
model.eval()
# Get the parser for the test dataset
valid_dataset = dataset.listDataset(valid_images,
shape=(test_width, test_height),
shuffle=False,
transform=transforms.Compose([
transforms.ToTensor(),
]))
valid_batchsize = 1
# Specify the number of workers for multiple processing, get the dataloader for the test dataset
kwargs = {'num_workers': 4, 'pin_memory': True}
test_loader = torch.utils.data.DataLoader(valid_dataset,
batch_size=valid_batchsize,
shuffle=False,
**kwargs)
logging(" Testing {}...".format(name))
logging(" Number of test samples: %d" % len(test_loader.dataset))
# Iterate through test batches (Batch size for test data is 1)
count = 0
z = np.zeros((3, 1))
print("--------------------------------------------")
for batch_idx, (data, target) in enumerate(test_loader):
# Images
print("-----------------idx---------------------------", batch_idx)
print("-------------------data-------------------------", data.shape)
print("--------------------target------------------------",
target.shape)
img = data[0, :, :, :]
img = img.numpy().squeeze()
img = np.transpose(img, (1, 2, 0))
# Pass data to GPU
if use_cuda:
data = data.cuda()
target = target.cuda()
# Wrap tensors in Variable class, set volatile=True for inference mode and to use minimal memory during inference
data = Variable(data, volatile=True)
# Forward pass
output = model(data).data
# Using confidence threshold, eliminate low-confidence predictions
all_boxes = get_region_boxes(output, conf_thresh, num_classes)
# Iterate through all images in the batch
for i in range(output.size(0)):
# For each image, get all the predictions
boxes = all_boxes[i]
# For each image, get all the targets (for multiple object pose estimation, there might be more than 1 target per image)
truths = target[i].view(-1, 21)
# Get how many object are present in the scene
num_gts = truths_length(truths)
# Iterate through each ground-truth object
for k in range(num_gts):
box_gt = [
truths[k][1], truths[k][2], truths[k][3], truths[k][4],
truths[k][5], truths[k][6], truths[k][7], truths[k][8],
truths[k][9], truths[k][10], truths[k][11], truths[k][12],
truths[k][13], truths[k][14], truths[k][15], truths[k][16],
truths[k][17], truths[k][18], 1.0, 1.0, truths[k][0]
]
best_conf_est = -1
# If the prediction has the highest confidence, choose it as our prediction for single object pose estimation
for j in range(len(boxes)):
if (boxes[j][18] > best_conf_est):
match = corner_confidence9(
box_gt[:18], torch.FloatTensor(boxes[j][:18]))
box_pr = boxes[j]
best_conf_est = boxes[j][18]
# Denormalize the corner predictions
corners2D_gt = np.array(np.reshape(box_gt[:18], [9, 2]),
dtype='float32')
corners2D_pr = np.array(np.reshape(box_pr[:18], [9, 2]),
dtype='float32')
corners2D_gt[:, 0] = corners2D_gt[:, 0] * 1920
corners2D_gt[:, 1] = corners2D_gt[:, 1] * 1080
corners2D_pr[:, 0] = corners2D_pr[:, 0] * 1920
corners2D_pr[:, 1] = corners2D_pr[:, 1] * 1080
# corners2D_gt[:, 0] = corners2D_gt[:, 0] * 640
# corners2D_gt[:, 1] = corners2D_gt[:, 1] * 480
# corners2D_pr[:, 0] = corners2D_pr[:, 0] * 640
# corners2D_pr[:, 1] = corners2D_pr[:, 1] * 480
preds_corners2D.append(corners2D_pr)
gts_corners2D.append(corners2D_gt)
# print("corners gt:\n",gts_corners2D)
# print("corners pr:\n", preds_corners2D)
# Compute [R|t] by pnp
R_gt, t_gt = pnp(
np.array(np.transpose(
np.concatenate((np.zeros((3, 1)), corners3D[:3, :]),
axis=1)),
dtype='float32'), corners2D_gt,
np.array(internal_calibration, dtype='float32'))
R_pr, t_pr = pnp(
np.array(np.transpose(
np.concatenate((np.zeros((3, 1)), corners3D[:3, :]),
axis=1)),
dtype='float32'), corners2D_pr,
np.array(internal_calibration, dtype='float32'))
#Eulerangle
# # Compute pixel error
Rt_gt = np.concatenate((R_gt, t_gt), axis=1)
Rt_pr = np.concatenate((R_pr, t_pr), axis=1)
proj_2d_gt = compute_projection(vertices, Rt_gt,
internal_calibration)
proj_2d_pred = compute_projection(vertices, Rt_pr,
internal_calibration)
proj_corners_gt = np.transpose(
compute_projection(corners3D, Rt_gt, internal_calibration))
proj_corners_pr = np.transpose(
compute_projection(corners3D, Rt_pr, internal_calibration))
if visualize:
# Visualize
plt.xlim((0, 1920))
plt.ylim((0, 1080))
plt.imshow(scipy.misc.imresize(img, (1080, 1920)))
plt.plot(corners2D_gt[1][0], corners2D_gt[1][1], 'r*')
plt.plot(corners2D_gt[2][0], corners2D_gt[2][1], 'r*')
plt.plot(corners2D_gt[3][0], corners2D_gt[3][1], 'r*')
plt.plot(corners2D_gt[4][0], corners2D_gt[4][1], 'r*')
plt.plot(corners2D_gt[5][0], corners2D_gt[5][1], 'r*')
plt.plot(corners2D_gt[6][0], corners2D_gt[6][1], 'r*')
plt.plot(corners2D_gt[7][0], corners2D_gt[7][1], 'r*')
plt.plot(corners2D_gt[8][0], corners2D_gt[8][1], 'r*')
# plt.plot(corners2D_pr[0][0], corners2D_pr[0][1], 'b*')
# plt.xlim((0, 640))
# plt.ylim((0, 480))
# plt.imshow(scipy.misc.imresize(img, (480, 640)))
# Projections
for edge in edges_corners:
# plt.plot(proj_corners_gt[edge, 0], proj_corners_gt[edge, 1], color='g', linewidth=2.0)
plt.plot(corners2D_gt[edge, 0],
corners2D_gt[edge, 1],
color='r',
linewidth=2.0)
plt.plot(proj_corners_pr[edge, 0],
proj_corners_pr[edge, 1],
color='b',
linewidth=2.0)
plt.gca().invert_yaxis()
# print(corners2D_pr[0])
# print(corners2D_pr)
# print(EulerAngle)
# plt.show()
print("_________")
plt.show()
