def train(train_list, model, criterion, optimizer, epoch, writer, LOG): losses = AverageMeter() percep1 = AverageMeter() percep2 = AverageMeter() maaloss = AverageMeter() batch_time = AverageMeter() data_time = AverageMeter() train_loader = torch.utils.data.DataLoader(dataset.listDataset( train_list, shuffle=True, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), ]), phase='train', seen=model.seen, batch_size=args.batch_size, num_workers=args.workers, epoch=epoch), batch_size=args.batch_size) model.train() end = time.time() mae = 0.0 for i, (img, target) in enumerate(train_loader): data_time.update(time.time() - end) a = time.time() img = img.cuda() target = target.type(torch.FloatTensor).unsqueeze(0).cuda() output, x64, gt64, x256, gt256 = model(img, target) b = time.time() maa, p1, p2 = criterion(output, target, x64, gt64, x256, gt256) loss = maa + p1 + p2 c = time.time() mae += abs(output.sum().item() - target.sum().item()) losses.update(loss.item(), img.shape[0]) maaloss.update(maa.item(), img.shape[0]) percep1.update(p1.item(), img.shape[0]) percep2.update(p2.item(), img.shape[0]) optimizer.zero_grad() loss.backward() optimizer.step() d = time.time() batch_time.update(time.time() - end) end = time.time() if i % (args.print_freq * 7) == 0: LOG.log( 'TRAIN - Epoch: [{0}][{1}/{2}] ' 'Time {batch_time.val:.3f} ' 'Data {data_time.val:.3f} ' 'Loss {loss.avg:.4f} ' 'Maa:{maaloss.avg:.4f} Percep1:{ploss1.avg:.4f} Percep2:{ploss2.avg:.4f}\t' .format(epoch, i, len(train_loader), batch_time=batch_time, data_time=data_time, loss=losses, maaloss=maaloss, ploss1=percep1, ploss2=percep2)) cprint('b-a:%.3f @ c-b:%.3f @d-c:%.3f' % (b - a, c - b, d - c), color='green') mae = mae / len(train_loader) writer.add_scalar('trainmae', mae, epoch) writer.add_scalar('trainloss', losses.avg, epoch) LOG.log('train - epoch:%d - mae:%.3f\n' % (epoch, mae), color='yellow') return model
def valid(datacfg, darknetcfg, learnetcfg, weightfile, outfile, use_baserw=False): options = read_data_cfg(datacfg) valid_images = options['valid'] metadict = options['meta'] # name_list = options['names'] # backup = cfg.backup ckpt = weightfile.split('/')[-1].split('.')[0] backup = weightfile.split('/')[-2] ckpt_pre = '/ene_' if use_baserw else '/ene' prefix = 'results/' + backup.split('/')[-1] + ckpt_pre + ckpt print('saving to: ' + prefix) # prefix = 'results/' + weightfile.split('/')[1] # names = load_class_names(name_list) with open(valid_images) as fp: tmp_files = fp.readlines() valid_files = [item.rstrip() for item in tmp_files] m = Darknet(darknetcfg, learnetcfg) m.print_network() m.load_weights(weightfile) m.cuda() m.eval() valid_dataset = dataset.listDataset(valid_images, shape=(m.width, m.height), shuffle=False, transform=transforms.Compose([ transforms.ToTensor(), ])) valid_batchsize = 2 assert (valid_batchsize > 1) kwargs = {'num_workers': 4, 'pin_memory': True} valid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=valid_batchsize, shuffle=False, **kwargs) if False: metaset = dataset.MetaDataset(metafiles=metadict, train=False, ensemble=True) metaloader = torch.utils.data.DataLoader(metaset, batch_size=len(metaset), shuffle=False, **kwargs) metaloader = iter(metaloader) n_cls = len(metaset.classes) print('===> Generating dynamic weights...') metax, mask = metaloader.next() metax, mask = metax.cuda(), mask.cuda() metax, mask = Variable(metax, volatile=True), Variable(mask, volatile=True) dynamic_weights = m.meta_forward(metax, mask) for i in range(len(dynamic_weights)): assert dynamic_weights[i].size(0) == sum(metaset.meta_cnts) inds = np.cumsum([0] + metaset.meta_cnts) new_weight = [] for j in range(len(metaset.meta_cnts)): new_weight.append( torch.mean(dynamic_weights[i][inds[j]:inds[j + 1]], dim=0)) dynamic_weights[i] = torch.stack(new_weight) print(dynamic_weights[i].shape) else: metaset = dataset.MetaDataset(metafiles=metadict, train=False, ensemble=True, with_ids=True) metaloader = torch.utils.data.DataLoader(metaset, batch_size=64, shuffle=False, **kwargs) # metaloader = iter(metaloader) n_cls = len(metaset.classes) enews = [0.0] * n_cls cnt = [0.0] * n_cls print('===> Generating dynamic weights...') kkk = 0 for metax, mask, clsids in metaloader: print('===> {}/{}'.format(kkk, len(metaset) // 64)) kkk += 1 metax, mask = metax.cuda(), mask.cuda() metax, mask = Variable(metax, volatile=True), Variable(mask, volatile=True) dws = m.meta_forward(metax, mask) dw = dws[0] for ci, c in enumerate(clsids): enews[c] = enews[c] * cnt[c] / (cnt[c] + 1) + dw[ci] / (cnt[c] + 1) cnt[c] += 1 dynamic_weights = [torch.stack(enews)] # import pickle # with open('data/rws/voc_novel2_.pkl', 'wb') as f: # tmp = [x.data.cpu().numpy() for x in dynamic_weights] # pickle.dump(tmp, f) # import pdb; pdb.set_trace() if use_baserw: import pickle # f = 'data/rws/voc_novel{}_.pkl'.format(cfg.novelid) f = 'data/rws/voc_novel{}_.pkl'.format(0) print('===> Loading from {}...'.format(f)) with open(f, 'rb') as f: # with open('data/rws/voc_novel0_.pkl', 'rb') as f: rws = pickle.load(f) rws = [Variable(torch.from_numpy(rw)).cuda() for rw in rws] tki = cfg._real_base_ids for i in range(len(rws)): dynamic_weights[i][tki] = rws[i][tki] # dynamic_weights[i] = rws[i] # pdb.set_trace() if not os.path.exists(prefix): # os.mkdir(prefix) os.makedirs(prefix) fps = [0] * n_cls for i, cls_name in enumerate(metaset.classes): buf = '%s/%s%s.txt' % (prefix, outfile, cls_name) fps[i] = open(buf, 'w') lineId = -1 conf_thresh = 0.005 nms_thresh = 0.45 for batch_idx, (data, target) in enumerate(valid_loader): data = data.cuda() data = Variable(data, volatile=True) output = m.detect_forward(data, dynamic_weights) if isinstance(output, tuple): output = (output[0].data, output[1].data) else: output = output.data # import pdb; pdb.set_trace() batch_boxes = get_region_boxes_v2(output, n_cls, conf_thresh, m.num_classes, m.anchors, m.num_anchors, 0, 1) if isinstance(output, tuple): bs = output[0].size(0) else: assert output.size(0) % n_cls == 0 bs = output.size(0) // n_cls for b in range(bs): lineId = lineId + 1 imgpath = valid_dataset.lines[lineId].rstrip() print(imgpath) imgid = os.path.basename(imgpath).split('.')[0] width, height = get_image_size(imgpath) for i in range(n_cls): # oi = i * bs + b oi = b * n_cls + i boxes = batch_boxes[oi] boxes = nms(boxes, nms_thresh) for box in boxes: x1 = (box[0] - box[2] / 2.0) * width y1 = (box[1] - box[3] / 2.0) * height x2 = (box[0] + box[2] / 2.0) * width y2 = (box[1] + box[3] / 2.0) * height det_conf = box[4] for j in range((len(box) - 5) / 2): cls_conf = box[5 + 2 * j] cls_id = box[6 + 2 * j] prob = det_conf * cls_conf fps[i].write('%s %f %f %f %f %f\n' % (imgid, prob, x1, y1, x2, y2)) for i in range(n_cls): fps[i].close()
def train(train_list, model, criterion, optimizer, epoch): global train_loss losses = AverageMeter() batch_time = AverageMeter() data_time = AverageMeter() train_loader = torch.utils.data.DataLoader(dataset.listDataset( train_list, shuffle=True, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), ]), train=True, seen=model.seen, batch_size=args.batch_size, num_workers=args.workers), batch_size=args.batch_size) print('epoch %d, processed %d samples, lr %.10f' % (epoch, epoch * len(train_loader.dataset), args.lr)) model.train() end = time.time() for i, (img, target) in enumerate(train_loader): data_time.update(time.time() - end) img = img.cuda() img = Variable(img) output = model(img) target = target.type(torch.FloatTensor).unsqueeze(0).cuda() target = Variable(target) #############print if you want######## # print('gtruth',target.detach().cpu().sum()) # print('count',output.detach().cpu().sum()) ################################# loss = criterion(output, target) losses.update(loss.item(), img.size(0)) optimizer.zero_grad() loss.backward() optimizer.step() batch_time.update(time.time() - end) end = time.time() if i % args.print_freq == 0: print('Epoch: [{0}][{1}/{2}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Data {data_time.val:.3f} ({data_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t'.format( epoch, i, len(train_loader), batch_time=batch_time, data_time=data_time, loss=losses))
def valid(datacfg, cfgfile, weightfile, outfile): options = read_data_cfg(datacfg) valid_images = options['valid'] name_list = options['names'] prefix = 'results' names = load_class_names(name_list) with open(valid_images) as fp: tmp_files = fp.readlines() valid_files = [item.rstrip() for item in tmp_files] m = Darknet(cfgfile) m.print_network() m.load_weights(weightfile) m.cuda() m.eval() valid_dataset = dataset.listDataset(valid_images, shape=(m.width, m.height), shuffle=False, transform=transforms.Compose([ transforms.ToTensor(), ])) valid_batchsize = 2 assert (valid_batchsize > 1) kwargs = {'num_workers': 4, 'pin_memory': True} valid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=valid_batchsize, shuffle=False, **kwargs) fps = [0] * m.num_classes if not os.path.exists('results'): os.mkdir('results') for i in range(m.num_classes): buf = '%s/%s%s.txt' % (prefix, outfile, names[i]) fps[i] = open(buf, 'w') lineId = -1 conf_thresh = 0.005 nms_thresh = 0.45 if m.net_name() == 'region': # region_layer shape = (0, 0) else: shape = (m.width, m.height) for _, (data, target, org_w, org_h) in enumerate(valid_loader): data = data.cuda() output = m(data) batch_boxes = get_all_boxes(output, shape, conf_thresh, m.num_classes, only_objectness=0, validation=True) for i in range(len(batch_boxes)): lineId += 1 fileId = os.path.basename(valid_files[lineId]).split('.')[0] #width, height = get_image_size(valid_files[lineId]) width, height = float(org_w[i]), float(org_h[i]) print(valid_files[lineId]) boxes = batch_boxes[i] correct_yolo_boxes(boxes, width, height, m.width, m.height) boxes = nms(boxes, nms_thresh) for box in boxes: x1 = (box[0] - box[2] / 2.0) * width y1 = (box[1] - box[3] / 2.0) * height x2 = (box[0] + box[2] / 2.0) * width y2 = (box[1] + box[3] / 2.0) * height det_conf = box[4] for j in range((len(box) - 5) // 2): cls_conf = box[5 + 2 * j] cls_id = box[6 + 2 * j] prob = det_conf * cls_conf fps[cls_id].write('%s %f %f %f %f %f\n' % (fileId, prob, x1, y1, x2, y2)) for i in range(m.num_classes): fps[i].close()
def train(train_list, model, criterion, optimizer, epoch): losses = AverageMeter() batch_time = AverageMeter() data_time = AverageMeter() train_loader = torch.utils.data.DataLoader(dataset.listDataset( train_list, shuffle=True, crop=True, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), ]), train=True, seen=model.seen, batch_size=args.batch_size, num_workers=args.workers), batch_size=args.batch_size) log_text = 'epoch %d, processed %d samples, lr %.10f' % ( epoch, epoch * len(train_loader.dataset), args.lr) log_print(log_text, color='green', attrs=['bold']) model.train() end = time.time() #BATCH OF HEMP< OFFSET< SCALE #img, thm_tgt, pff_tgt, size_tgt for i, (img_list, target_list) in enumerate(train_loader): for j, patch in enumerate(img_list): data_time.update(time.time() - end) img = img_list[j] target = target_list[j] img = img.cuda() img = Variable(img) output = model(img) target = target.type( torch.FloatTensor).unsqueeze(0).cuda() #TARGET to cuda target = Variable(target) loss = criterion(output, target) #list of criterion [focal loss, mse] # focal_loss = vri[0](cri[0]) #loss = losses.update(float(loss.item()), img.size(0)) optimizer.zero_grad() loss.backward() optimizer.step() batch_time.update(time.time() - end) end = time.time() if i % args.print_freq == 0: log_text = (( 'Epoch: [{0}][{1}/{2}]\t' 'Patch {patch_num:d}\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Data {data_time.val:.3f} ({data_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t').format( epoch, i, len(train_loader), patch_num=j, batch_time=batch_time, data_time=data_time, loss=losses)) log_print(log_text, color='green', attrs=['bold']) writer.add_scalar('train_loss', losses.avg, epoch)
def trainRPNPP(model, optimizer, epoch, coco): losses = AverageMeter() batch_time = AverageMeter() data_time = AverageMeter() train_loader = torch.utils.data.DataLoader(dataset.listDataset( args.ilsvrc, args.youtube, args.data_type, shuffle=True, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), ]), train=True, batch_size=args.batch_size, num_workers=args.workers, coco=coco, rpnpp=True), batch_size=args.batch_size) model.train() end = time.time() for i, (z, x, regression_target, conf_target) in enumerate(train_loader): data_time.update(time.time() - end) z = z.cuda() z = Variable(z) x = x.cuda() x = Variable(x) pred_score, pred_regression = model(z, x) # print(pred_score.size(), pred_regression.size()) b, a2, h, w = pred_score.size() pred_conf = pred_score.reshape(-1, 2, 5 * h * w).permute(0, 2, 1) pred_offset = pred_regression.reshape(-1, 4, 5 * h * w).permute(0, 2, 1) regression_target = regression_target.type(torch.FloatTensor).cuda() conf_target = conf_target.type(torch.LongTensor).cuda() # print(pred_conf.size(), conf_target.size()) cls_loss = rpn_cross_entropy(pred_conf, conf_target) reg_loss = rpn_smoothL1(pred_offset, regression_target, conf_target) loss = cls_loss + reg_loss losses.update(loss.item(), x.size(0)) optimizer.zero_grad() loss.backward() torch.nn.utils.clip_grad_norm_(model.parameters(), 10) optimizer.step() batch_time.update(time.time() - end) end = time.time() if i % args.print_freq == 0: print('Epoch: [{0}][{1}/{2}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Data {data_time.val:.3f} ({data_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t'.format( epoch, i, len(train_loader), batch_time=batch_time, data_time=data_time, loss=losses))
def train(epoch): global processed_batches t0 = time.time() if ngpus > 1: cur_model = model.module else: cur_model = model train_loader = torch.utils.data.DataLoader( dataset.listDataset(trainlist, shape=(init_width, init_height), shuffle=True, transform=transforms.Compose([ transforms.ToTensor(), ]), train=True, seen=cur_model.seen, batch_size=batch_size, num_workers=num_workers), batch_size=batch_size, shuffle=False, **kwargs) lr = adjust_learning_rate(optimizer, processed_batches) logging('epoch %d, processed %d samples, lr %f' % (epoch, epoch * len(train_loader.dataset), lr)) model.train() t1 = time.time() avg_time = torch.zeros(9) for batch_idx, (data, target) in enumerate(train_loader): t2 = time.time() adjust_learning_rate(optimizer, processed_batches) processed_batches = processed_batches + 1 #if (batch_idx+1) % dot_interval == 0: # sys.stdout.write('.') if use_cuda: data = data.cuda() #target= target.cuda() t3 = time.time() data, target = Variable(data), Variable(target) t4 = time.time() optimizer.zero_grad() t5 = time.time() output = model(data) t6 = time.time() region_loss.seen = region_loss.seen + data.data.size(0) loss = region_loss(output, target) t7 = time.time() loss.backward() t8 = time.time() optimizer.step() t9 = time.time() if False and batch_idx > 1: avg_time[0] = avg_time[0] + (t2-t1) avg_time[1] = avg_time[1] + (t3-t2) avg_time[2] = avg_time[2] + (t4-t3) avg_time[3] = avg_time[3] + (t5-t4) avg_time[4] = avg_time[4] + (t6-t5) avg_time[5] = avg_time[5] + (t7-t6) avg_time[6] = avg_time[6] + (t8-t7) avg_time[7] = avg_time[7] + (t9-t8) avg_time[8] = avg_time[8] + (t9-t1) print('-------------------------------') print(' load data : %f' % (avg_time[0]/(batch_idx))) print(' cpu to cuda : %f' % (avg_time[1]/(batch_idx))) print('cuda to variable : %f' % (avg_time[2]/(batch_idx))) print(' zero_grad : %f' % (avg_time[3]/(batch_idx))) print(' forward feature : %f' % (avg_time[4]/(batch_idx))) print(' forward loss : %f' % (avg_time[5]/(batch_idx))) print(' backward : %f' % (avg_time[6]/(batch_idx))) print(' step : %f' % (avg_time[7]/(batch_idx))) print(' total : %f' % (avg_time[8]/(batch_idx))) t1 = time.time() print('') t1 = time.time() logging('training with %f samples/s' % (len(train_loader.dataset)/(t1-t0))) if (epoch+1) % save_interval == 0: logging('save weights to %s/%06d.weights' % (backupdir, epoch+1)) cur_model.seen = (epoch + 1) * len(train_loader.dataset) cur_model.save_weights('%s/%06d.weights' % (backupdir, epoch+1))
weightfile = '../yolov2-tiny-bnn/weights/yolov2-tiny-voc.weights' options = read_data_cfg(datacfg) valid_images = options['valid'] with open(valid_images) as fp: tmp_files = fp.readlines() valid_files = [item.rstrip() for item in tmp_files] m = Darknet(cfgfile) m.print_network() m.load_weights(weightfile) m.cuda() m.eval() valid_dataset = dataset.listDataset(valid_images, shape=(m.width, m.height), shuffle=False, transform=transforms.Compose([ transforms.ToTensor(), ])) valid_batchsize = 2 assert (valid_batchsize > 1) kwargs = {'num_workers': 0, 'pin_memory': True} valid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=valid_batchsize, shuffle=False, **kwargs) conf_thresh = 0.005 nms_thresh = 0.45 metrics = [] labels = []
def train(epoch): global processed_batches t0 = time.time() cur_model = curmodel() init_width = cur_model.width init_height = cur_model.height kwargs = { 'num_workers': num_workers, 'pin_memory': True } if use_cuda else {} #kwargs = {'num_workers': 0, 'pin_memory': True} if use_cuda else {} train_loader = torch.utils.data.DataLoader( dataset.listDataset( trainlist, imgRoot=image_folder, wdatalist=wdata, odweight=odw, shape=(init_width, init_height), shuffle=False, #True, transform=transforms.Compose([ transforms.ToTensor(), ]), train=True, seen=cur_model.seen, batch_size=batch_size, num_workers=num_workers), collate_fn=dataset.custom_collate, batch_size=batch_size, shuffle=False, **kwargs) processed_batches = cur_model.seen // batch_size #lr = adjust_learning_rate(optimizer, processed_batches) for param_group in optimizer.param_groups: lr = param_group['lr'] logging('[%03d] processed %d samples, lr %e' % (epoch, epoch * len(train_loader.dataset), lr)) #logging('[%03d] processed %d samples' % (epoch, epoch * len(train_loader.dataset))) model.train() t1 = time.time() avg_time = torch.zeros(9) for batch_idx, (data, target) in enumerate(train_loader): t2 = time.time() #adjust_learning_rate(optimizer, processed_batches) processed_batches = processed_batches + 1 #if (batch_idx+1) % dot_interval == 0: # sys.stdout.write('.') t3 = time.time() data, target = data.to(device), target.to(device) t4 = time.time() optimizer.zero_grad() t5 = time.time() output = model(data) t6 = time.time() org_loss = [] for i, l in enumerate(loss_layers): l.seen = l.seen + data.data.size(0) ol = l(output[i]['x'], target) org_loss.append(ol) t7 = time.time() #for i, l in enumerate(reversed(org_loss)): # l.backward(retain_graph=True if i < len(org_loss)-1 else False) # org_loss.reverse() sum(org_loss).backward() nn.utils.clip_grad_norm_(model.parameters(), 10000) #for p in model.parameters(): # p.data.add_(-lr, p.grad.data) t8 = time.time() optimizer.step() t9 = time.time() if False and batch_idx > 1: avg_time[0] = avg_time[0] + (t2 - t1) avg_time[1] = avg_time[1] + (t3 - t2) avg_time[2] = avg_time[2] + (t4 - t3) avg_time[3] = avg_time[3] + (t5 - t4) avg_time[4] = avg_time[4] + (t6 - t5) avg_time[5] = avg_time[5] + (t7 - t6) avg_time[6] = avg_time[6] + (t8 - t7) avg_time[7] = avg_time[7] + (t9 - t8) avg_time[8] = avg_time[8] + (t9 - t1) print('-------------------------------') print(' load data : %f' % (avg_time[0] / (batch_idx))) print(' cpu to cuda : %f' % (avg_time[1] / (batch_idx))) print('cuda to variable : %f' % (avg_time[2] / (batch_idx))) print(' zero_grad : %f' % (avg_time[3] / (batch_idx))) print(' forward feature : %f' % (avg_time[4] / (batch_idx))) print(' forward loss : %f' % (avg_time[5] / (batch_idx))) print(' backward : %f' % (avg_time[6] / (batch_idx))) print(' step : %f' % (avg_time[7] / (batch_idx))) print(' total : %f' % (avg_time[8] / (batch_idx))) t1 = time.time() del data, target org_loss.clear() gc.collect() print('') t1 = time.time() nsamples = len(train_loader.dataset) logging('[%03d] training with %f samples/s' % (epoch, nsamples / (t1 - t0))) return nsamples
def validate(Pre_data, model, args): print('begin test') batch_size = 1 test_loader = torch.utils.data.DataLoader(dataset.listDataset( Pre_data, args['task_id'], shuffle=False, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), ]), args=args, train=False), batch_size=1) model.eval() mae = 0.0 mse = 0.0 visi = [] index = 0 if not os.path.exists('./local_eval/loc_file'): os.makedirs('./local_eval/loc_file') '''output coordinates''' f_loc = open("./local_eval/A_localization.txt", "w+") for i, (fname, img, fidt_map, kpoint) in enumerate(test_loader): count = 0 img = img.cuda() if len(img.shape) == 5: img = img.squeeze(0) if len(fidt_map.shape) == 5: fidt_map = fidt_map.squeeze(0) if len(img.shape) == 3: img = img.unsqueeze(0) if len(fidt_map.shape) == 3: fidt_map = fidt_map.unsqueeze(0) with torch.no_grad(): d6 = model(img) '''return counting and coordinates''' count, pred_kpoint, f_loc = LMDS_counting(d6, i + 1, f_loc, fname) point_map = generate_point_map(pred_kpoint, f_loc, rate=1) if args['visual'] == True: if not os.path.exists(args['task_id'] + '_box/'): os.makedirs(args['task_id'] + '_box/') ori_img, box_img = generate_bounding_boxes(pred_kpoint, fname) show_fidt = show_map(d6.data.cpu().numpy()) gt_show = show_map(fidt_map.data.cpu().numpy()) res = np.hstack( (ori_img, gt_show, show_fidt, point_map, box_img)) cv2.imwrite(args['task_id'] + '_box/' + fname[0], res) gt_count = torch.sum(kpoint).item() mae += abs(gt_count - count) mse += abs(gt_count - count) * abs(gt_count - count) if i % 1 == 0: print('{fname} Gt {gt:.2f} Pred {pred}'.format(fname=fname[0], gt=gt_count, pred=count)) visi.append([ img.data.cpu().numpy(), d6.data.cpu().numpy(), fidt_map.data.cpu().numpy(), fname ]) index += 1 mae = mae * 1.0 / (len(test_loader) * batch_size) mse = math.sqrt(mse / (len(test_loader)) * batch_size) nni.report_intermediate_result(mae) print(' \n* MAE {mae:.3f}\n'.format(mae=mae), '* MSE {mse:.3f}'.format(mse=mse)) return mae, visi
def valid(datacfg, cfgfile, weightfile, outfile): options = read_data_cfg(datacfg) valid_images = options['valid'] name_list = options['names'] prefix = 'results' names = load_class_names(name_list) with open(valid_images) as fp: tmp_files = fp.readlines() valid_files = [item.rstrip() for item in tmp_files] m = Darknet(cfgfile) # m.load_state_dict(torch.load("darknet_bayes_60.pkl")) # m = torch.nn.DataParallel(m).cuda() m.load_state_dict(torch.load(weightfile)) m.print_network() # m.load_weights(weightfile) m.cuda() m.eval() ################################################# """ layers = [m.models[0].conv1, m.models[2].conv2, m.models[4].conv3, m.models[6].conv4, m.models[8].conv5, m.models[10].conv6, m.models[12].conv7, m.models[13].conv8, m.models[14].conv9] thresholds = [-3,-3,-3,-3,-3,-3,-3,-3,-3] sig,exp = compute_compression_rate(layers, m.get_masks(thresholds)) print("Test error after with reduced bit precision:") weights,biases = compute_reduced_weights(layers, m.get_masks(thresholds),sig,exp) for layer, weight,bias in zip(layers, weights,biases): if True: layer.post_weight_mu.data = torch.Tensor(weight).cuda() layer.post_bias_mu = torch.Tensor(bias).cuda() else: layer.post_weight_mu.data = torch.Tensor(weight) for layer in layers: layer.deterministic = True """ ########################################## valid_dataset = dataset.listDataset(valid_images, shape=(m.width, m.height), shuffle=False, transform=transforms.Compose([ transforms.ToTensor(), ])) valid_batchsize = 2 assert (valid_batchsize > 1) kwargs = {'num_workers': 4, 'pin_memory': True} valid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=valid_batchsize, shuffle=False, **kwargs) fps = [0] * m.num_classes if not os.path.exists('results'): os.mkdir('results') for i in range(m.num_classes): buf = '%s/%s%s.txt' % (prefix, outfile, names[i]) fps[i] = open(buf, 'w') lineId = -1 conf_thresh = 0.005 nms_thresh = 0.45 for batch_idx, (data, target) in enumerate(valid_loader): data = data.cuda() data = Variable(data, volatile=True) output = m(data).data batch_boxes = get_region_boxes(output, conf_thresh, m.num_classes, m.anchors, m.num_anchors, 0, 1) for i in range(output.size(0)): lineId = lineId + 1 fileId = os.path.basename(valid_files[lineId]).split('.')[0] width, height = get_image_size(valid_files[lineId]) print(valid_files[lineId]) boxes = batch_boxes[i] boxes = nms(boxes, nms_thresh) for box in boxes: x1 = (box[0] - box[2] / 2.0) * width y1 = (box[1] - box[3] / 2.0) * height x2 = (box[0] + box[2] / 2.0) * width y2 = (box[1] + box[3] / 2.0) * height det_conf = box[4] for j in range(int((len(box) - 5) / 2)): cls_conf = box[5 + 2 * j] cls_id = box[6 + 2 * j] prob = det_conf * cls_conf fps[cls_id].write('%s %f %f %f %f %f\n' % (fileId, prob, x1, y1, x2, y2)) for i in range(m.num_classes): fps[i].close()
os.system('mkdir {0}'.format(opt.experiment)) opt.manualSeed = random.randint(1, 10000) # fix seed print("Random Seed: ", opt.manualSeed) random.seed(opt.manualSeed) np.random.seed(opt.manualSeed) torch.manual_seed(opt.manualSeed) cudnn.benchmark = True if torch.cuda.is_available() and not opt.cuda: print( "WARNING: You have a CUDA device, so you should probably run with --cuda" ) train_dataset = dataset.listDataset(list_file=opt.trainlist) assert train_dataset if opt.random_sample: sampler = dataset.randomSequentialSampler(train_dataset, opt.batchSize) else: sampler = None train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=opt.batchSize, shuffle=False, sampler=sampler, num_workers=int(opt.workers), collate_fn=dataset.alignCollate( imgH=opt.imgH, imgW=opt.imgW, keep_ratio=opt.keep_ratio)) print(len(train_loader))
def valid(datacfg, cfgfile, weightfile, weightfile_distance, weightfile_rotation, outfile): def truths_length(truths): for i in range(50): if truths[i][1] == 0: return i # Parse configuration files options = read_data_cfg(datacfg) valid_images = options['valid'] #meshname = options['mesh'] backupdir = options['backup'] name = options['name'] if not os.path.exists(backupdir): makedirs(backupdir) # Parameters prefix = 'results' seed = int(time.time()) #gpus = '0,1,2,3' # Specify which gpus to use gpus = options['gpus'] test_width = 416 test_height = 416 torch.manual_seed(seed) use_cuda = True if use_cuda: os.environ['CUDA_VISIBLE_DEVICES'] = gpus torch.cuda.manual_seed(seed) save = False testtime = True use_cuda = True num_classes = 1 testing_samples = 0.0 eps = 1e-5 notpredicted = 0 conf_thresh = 0.1 nms_thresh = 0.4 match_thresh = 0.5 if save: makedirs(backupdir + '/test') makedirs(backupdir + '/test/gt') makedirs(backupdir + '/test/pr') # To save testing_error_trans = 0.0 testing_error_angle = 0.0 testing_error_pixel = 0.0 errs_2d = [] errs_3d = [] errs_trans = [] errs_angle = [] errs_corner2D = [] preds_trans = [] preds_rot = [] preds_corners2D = [] gts_trans = [] gts_rot = [] gts_corners2D = [] # Get validation file names with open(valid_images) as fp: tmp_files = fp.readlines() valid_files = [item.rstrip() for item in tmp_files] # Specicy model, load pretrained weights, pass to GPU and set the module in evaluation mode model = Darknet(cfgfile) model_2 = DistanceNet() model_3 = RotationNet() #model.print_network() model.load_weights(weightfile) model_2.load_state_dict(torch.load(weightfile_distance)) model_3.load_state_dict(torch.load(weightfile_rotation)) model.cuda() model.eval() model_2.cuda() model_3.cuda() # Get the parser for the test dataset valid_dataset = dataset.listDataset(valid_images, shape=(test_width, test_height), shuffle=False, transform=transforms.Compose([ transforms.ToTensor(), ])) valid_batchsize = 1 # Specify the number of workers for multiple processing, get the dataloader for the test dataset kwargs = {'num_workers': 4, 'pin_memory': True} test_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=valid_batchsize, shuffle=False, **kwargs) logging(" Testing {}...".format(name)) logging(" Number of test samples: %d" % len(test_loader.dataset)) # Iterate through test batches (Batch size for test data is 1) count = 0 z = np.zeros((3, 1)) out_file = open('data/valid_results.txt', 'w') for batch_idx, (data, target, gt_T, gt_R) in enumerate(test_loader): t1 = time.time() # Pass data to GPU if use_cuda: data = data.cuda() target = target.cuda() # Wrap tensors in Variable class, set volatile=True for inference mode and to use minimal memory during inference data = Variable(data, volatile=True) # Forward pass output = model(data).data # Using confidence threshold, eliminate low-confidence predictions all_boxes_1 = get_region_boxes(output.data, conf_thresh, num_classes) all_boxes = np.reshape(np.array(all_boxes_1), (-1, 15)) all_boxes = torch.from_numpy(all_boxes).float() if use_cuda: all_boxes = all_boxes.cuda() output_T = model_2(all_boxes).float() if use_cuda: gt_T = gt_T.cuda() gt_R = gt_R.cuda() input_final = torch.cat((output_T, all_boxes), 1) output_R = model_3(input_final) control_points = torch.FloatTensor([[[0.1, 0, 0]], [[0, 0.1, 0]], [[-0.1, 0, 0]], [[0, -0.1, 0]]]).cuda() gt_proj = torch.zeros(output.size(0), 4, 3).cuda() pred_proj = torch.zeros(output.size(0), 4, 3).cuda() # Iterate through all images in the batch for j in range(output.size(0)): # For each image, get all the predictions boxes = all_boxes_1[j] plot_boxes_polygon(data[j], boxes, "", "", savename='data/Results/batch%s_%s.jpeg' % (str(batch_idx), str(j)), class_names=['QR_Code']) if boxes == [[]]: continue (x, y, z, w) = (gt_R[j][0], gt_R[j][1], gt_R[j][2], gt_R[j][3]) gt_rotation = torch.Tensor([[ 2 * (x**2 + w**2) - 1, 2 * (x * y + z * w), 2 * (x * z - y * w) ], [ 2 * (x * y - z * w), 2 * (y**2 + w**2) - 1, 2 * (y * z + x * w) ], [ 2 * (x * z + y * w), 2 * (y * z - x * w), 2 * (z**2 + w**2) - 1 ]]) gt_rotation = gt_rotation.cuda() (x, y, z, w) = (output_R[j][0], output_R[j][1], output_R[j][2], output_R[j][3]) pred_rotation = torch.Tensor([[ 2 * (x**2 + w**2) - 1, 2 * (x * y + z * w), 2 * (x * z - y * w) ], [ 2 * (x * y - z * w), 2 * (y**2 + w**2) - 1, 2 * (y * z + x * w) ], [ 2 * (x * z + y * w), 2 * (y * z - x * w), 2 * (z**2 + w**2) - 1 ]]) pred_rotation = pred_rotation.cuda() for i in range(4): gt_proj[j][i] = gt_T[j] + torch.mm( gt_rotation, control_points[i].t()).view(-1) pred_proj[j][i] = output_T[j] + torch.mm( pred_rotation, control_points[i].t()).view(-1) loss = 0 loss_tmp = torch.zeros(4, 4).cuda() loss_m = torch.zeros(4).cuda() for m in range(4): for n in range(4): loss_tmp[m][n] = nn.MSELoss(size_average=True)( pred_proj[j][m], gt_proj[j][n]) loss_m, index = torch.min(loss_tmp, dim=1) loss = torch.mean(loss_m) out_file.write('batch%d_%d,%04f\n' % (batch_idx, j, loss))
def train(Pre_data, model, criterion, optimizer, epoch, args, scheduler): losses = AverageMeter() batch_time = AverageMeter() data_time = AverageMeter() train_loader = torch.utils.data.DataLoader(dataset.listDataset( Pre_data, args['save_path'], shuffle=True, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), ]), train=True, batch_size=args['batch_size'], num_workers=args['workers'], args=args), batch_size=args['batch_size'], drop_last=False) args['lr'] = optimizer.param_groups[0]['lr'] print('epoch %d, processed %d samples, lr %.10f' % (epoch, epoch * len(train_loader.dataset), args['lr'])) model.train() end = time.time() for i, (fname, img, gt_count) in enumerate(train_loader): data_time.update(time.time() - end) img = img.cuda() out1 = model(img) gt_count = gt_count.type(torch.FloatTensor).cuda().unsqueeze(1) # print(out1.shape, kpoint.shape) loss = criterion(out1, gt_count) losses.update(loss.item(), img.size(0)) optimizer.zero_grad() loss.backward() optimizer.step() batch_time.update(time.time() - end) end = time.time() if i % args['print_freq'] == 0: print('4_Epoch: [{0}][{1}/{2}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Data {data_time.val:.3f} ({data_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t'.format( epoch, i, len(train_loader), batch_time=batch_time, data_time=data_time, loss=losses)) scheduler.step()
def valid(datacfg, modelcfg, weightfile): def truths_length(truths, max_num_gt=50): for i in range(max_num_gt): if truths[i][1] == 0: return i # Parse configuration files data_options = read_data_cfg(datacfg) valid_images = data_options['valid'] meshname = data_options['mesh'] backupdir = data_options['backup'] name = data_options['name'] gpus = data_options['gpus'] fx = float(data_options['fx']) fy = float(data_options['fy']) u0 = float(data_options['u0']) v0 = float(data_options['v0']) im_width = int(data_options['width']) im_height = int(data_options['height']) if not os.path.exists(backupdir): makedirs(backupdir) # Parameters seed = int(time.time()) os.environ['CUDA_VISIBLE_DEVICES'] = gpus torch.cuda.manual_seed(seed) save = False testtime = True num_classes = 1 testing_samples = 0.0 if save: makedirs(backupdir + '/test') makedirs(backupdir + '/test/gt') makedirs(backupdir + '/test/pr') # To save testing_error_trans = 0.0 testing_error_angle = 0.0 testing_error_pixel = 0.0 errs_2d = [] errs_3d = [] errs_trans = [] errs_angle = [] errs_corner2D = [] preds_trans = [] preds_rot = [] preds_corners2D = [] gts_trans = [] gts_rot = [] gts_corners2D = [] # Read object model information, get 3D bounding box corners mesh = MeshPly(meshname) vertices = np.c_[np.array(mesh.vertices), np.ones((len(mesh.vertices), 1))].transpose() corners3D = get_3D_corners(vertices) try: diam = float(options['diam']) except: diam = calc_pts_diameter(np.array(mesh.vertices)) # Read intrinsic camera parameters intrinsic_calibration = get_camera_intrinsic(u0, v0, fx, fy) # Get validation file names with open(valid_images) as fp: tmp_files = fp.readlines() valid_files = [item.rstrip() for item in tmp_files] # Specicy model, load pretrained weights, pass to GPU and set the module in evaluation mode model = Darknet(modelcfg) model.print_network() model.load_weights(weightfile) model.cuda() model.eval() test_width = model.test_width test_height = model.test_height num_keypoints = model.num_keypoints num_labels = num_keypoints * 2 + 3 # +2 for width, height, +1 for class label # Get the parser for the test dataset valid_dataset = dataset.listDataset(valid_images, shape=(test_width, test_height), shuffle=False, transform=transforms.Compose([ transforms.ToTensor(), ])) # Specify the number of workers for multiple processing, get the dataloader for the test dataset kwargs = {'num_workers': 4, 'pin_memory': True} test_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=1, shuffle=False, **kwargs) logging(" Testing {}...".format(name)) logging(" Number of test samples: %d" % len(test_loader.dataset)) # Iterate through test batches (Batch size for test data is 1) count = 0 for batch_idx, (data, target) in enumerate(test_loader): t1 = time.time() # Pass data to GPU # import IPython; IPython.embed() data = data.cuda() # target = target.cuda() # Wrap tensors in Variable class, set volatile=True for inference mode and to use minimal memory during inference data = Variable(data, volatile=True) t2 = time.time() # Forward pass output = model(data).data t3 = time.time() # Using confidence threshold, eliminate low-confidence predictions all_boxes = get_region_boxes(output, num_classes, num_keypoints) all_boxes = [t.cpu() for t in all_boxes] t4 = time.time() # Evaluation # Iterate through all batch elements for box_pr, target in zip([all_boxes], [target[0]]): # For each image, get all the targets (for multiple object pose estimation, there might be more than 1 target per image) truths = target.view(-1, num_labels) # Get how many objects are present in the scene num_gts = truths_length(truths) # Iterate through each ground-truth object for k in range(num_gts): box_gt = list() for j in range(1, 2 * num_keypoints + 1): box_gt.append(truths[k][j]) box_gt.extend([1.0, 1.0]) box_gt.append(truths[k][0]) # Denormalize the corner predictions corners2D_gt = np.array(np.reshape(box_gt[:18], [-1, 2]), dtype='float32') corners2D_pr = np.array(np.reshape(box_pr[:18], [-1, 2]), dtype='float32') corners2D_gt[:, 0] = corners2D_gt[:, 0] * im_width corners2D_gt[:, 1] = corners2D_gt[:, 1] * im_height corners2D_pr[:, 0] = corners2D_pr[:, 0] * im_width corners2D_pr[:, 1] = corners2D_pr[:, 1] * im_height preds_corners2D.append(corners2D_pr) gts_corners2D.append(corners2D_gt) # Compute corner prediction error corner_norm = np.linalg.norm(corners2D_gt - corners2D_pr, axis=1) corner_dist = np.mean(corner_norm) errs_corner2D.append(corner_dist) # Compute [R|t] by pnp R_gt, t_gt = pnp( np.array(np.transpose( np.concatenate((np.zeros((3, 1)), corners3D[:3, :]), axis=1)), dtype='float32'), corners2D_gt, np.array(intrinsic_calibration, dtype='float32')) R_pr, t_pr = pnp( np.array(np.transpose( np.concatenate((np.zeros((3, 1)), corners3D[:3, :]), axis=1)), dtype='float32'), corners2D_pr, np.array(intrinsic_calibration, dtype='float32')) # Compute translation error trans_dist = np.sqrt(np.sum(np.square(t_gt - t_pr))) errs_trans.append(trans_dist) # Compute angle error angle_dist = calcAngularDistance(R_gt, R_pr) errs_angle.append(angle_dist) # Compute pixel error Rt_gt = np.concatenate((R_gt, t_gt), axis=1) Rt_pr = np.concatenate((R_pr, t_pr), axis=1) proj_2d_gt = compute_projection(vertices, Rt_gt, intrinsic_calibration) proj_2d_pred = compute_projection(vertices, Rt_pr, intrinsic_calibration) norm = np.linalg.norm(proj_2d_gt - proj_2d_pred, axis=0) pixel_dist = np.mean(norm) errs_2d.append(pixel_dist) # Compute 3D distances transform_3d_gt = compute_transformation(vertices, Rt_gt) transform_3d_pred = compute_transformation(vertices, Rt_pr) norm3d = np.linalg.norm(transform_3d_gt - transform_3d_pred, axis=0) vertex_dist = np.mean(norm3d) errs_3d.append(vertex_dist) # Sum errors testing_error_trans += trans_dist testing_error_angle += angle_dist testing_error_pixel += pixel_dist testing_samples += 1 count = count + 1 if save: preds_trans.append(t_pr) gts_trans.append(t_gt) preds_rot.append(R_pr) gts_rot.append(R_gt) np.savetxt( backupdir + '/test/gt/R_' + valid_files[count][-8:-3] + 'txt', np.array(R_gt, dtype='float32')) np.savetxt( backupdir + '/test/gt/t_' + valid_files[count][-8:-3] + 'txt', np.array(t_gt, dtype='float32')) np.savetxt( backupdir + '/test/pr/R_' + valid_files[count][-8:-3] + 'txt', np.array(R_pr, dtype='float32')) np.savetxt( backupdir + '/test/pr/t_' + valid_files[count][-8:-3] + 'txt', np.array(t_pr, dtype='float32')) np.savetxt( backupdir + '/test/gt/corners_' + valid_files[count][-8:-3] + 'txt', np.array(corners2D_gt, dtype='float32')) np.savetxt( backupdir + '/test/pr/corners_' + valid_files[count][-8:-3] + 'txt', np.array(corners2D_pr, dtype='float32')) t5 = time.time() # Compute 2D projection error, 6D pose error, 5cm5degree error px_threshold = 5 # 5 pixel threshold for 2D reprojection error is standard in recent sota 6D object pose estimation works eps = 1e-5 acc = len(np.where( np.array(errs_2d) <= px_threshold)[0]) * 100. / (len(errs_2d) + eps) acc5cm5deg = len( np.where((np.array(errs_trans) <= 0.05) & (np.array(errs_angle) <= 5))[0]) * 100. / (len(errs_trans) + eps) acc3d10 = len(np.where( np.array(errs_3d) <= diam * 0.1)[0]) * 100. / (len(errs_3d) + eps) acc5cm5deg = len( np.where((np.array(errs_trans) <= 0.05) & (np.array(errs_angle) <= 5))[0]) * 100. / (len(errs_trans) + eps) corner_acc = len(np.where(np.array(errs_corner2D) <= px_threshold) [0]) * 100. / (len(errs_corner2D) + eps) mean_err_2d = np.mean(errs_2d) mean_corner_err_2d = np.mean(errs_corner2D) nts = float(testing_samples) if testtime: print('-----------------------------------') print(' tensor to cuda : %f' % (t2 - t1)) print(' forward pass : %f' % (t3 - t2)) print('get_region_boxes : %f' % (t4 - t3)) print(' prediction time : %f' % (t4 - t1)) print(' eval : %f' % (t5 - t4)) print('-----------------------------------') # Print test statistics logging('Results of {}'.format(name)) logging(' Acc using {} px 2D Projection = {:.2f}%'.format( px_threshold, acc)) logging(' Acc using 10% threshold - {} vx 3D Transformation = {:.2f}%'. format(diam * 0.1, acc3d10)) logging(' Acc using 5 cm 5 degree metric = {:.2f}%'.format(acc5cm5deg)) logging( " Mean 2D pixel error is %f, Mean vertex error is %f, mean corner error is %f" % (mean_err_2d, np.mean(errs_3d), mean_corner_err_2d)) logging( ' Translation error: %f m, angle error: %f degree, pixel error: % f pix' % (testing_error_trans / nts, testing_error_angle / nts, testing_error_pixel / nts)) if save: predfile = backupdir + '/predictions_linemod_' + name + '.mat' scipy.io.savemat( predfile, { 'R_gts': gts_rot, 't_gts': gts_trans, 'corner_gts': gts_corners2D, 'R_prs': preds_rot, 't_prs': preds_trans, 'corner_prs': preds_corners2D })
cuda_str = 'cuda:' + str(config['cuda']['gpu_id']) device = torch.device(cuda_str if config['cuda']['using_cuda'] else "cpu") # make dataset transform = transforms.Compose([transforms.ToTensor()]) encoder = utils.LabelEncoder( labels=config['hyperparameters']['labels'], is_ignore_case=config['hyperparameters']['is_ignore_case']) img_width = config['hyperparameters']['imgWidth'] img_height = config['hyperparameters']['imgHeight'] train_dataset = dataset.listDataset(root_path=config['data']['train'], transform=transform, encoder=encoder, img_size=(img_width, img_height)) valid_dataset = dataset.listDataset(root_path=config['data']['valid'], transform=transform, encoder=encoder, img_size=(img_width, img_height)) assert train_dataset assert valid_dataset train_loader = torch.utils.data.DataLoader( train_dataset, batch_size=config['hyperparameters']['batch_size'], shuffle=True, num_workers=0, collate_fn=train_dataset.collate_fn)
def train(model, criterion, optimizer, epoch, coco): losses = AverageMeter() batch_time = AverageMeter() data_time = AverageMeter() train_loader = torch.utils.data.DataLoader(dataset.listDataset( args.ilsvrc, args.youtube, args.data_type, shuffle=True, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ]), train=True, batch_size=args.batch_size, num_workers=args.workers, coco=coco), batch_size=args.batch_size) model.train() end = time.time() for i, (z, x, template, gt_box) in enumerate(train_loader): data_time.update(time.time() - end) z = z.cuda() z = Variable(z) x = x.cuda() x = Variable(x) template = template.type(torch.FloatTensor).cuda() template = Variable(template) oup = model(z, x) if isinstance(model, SiamFC) or isinstance(model, SiamVGG): loss = criterion(oup, template) elif isinstance(model, SiamFCRes22): loss = model.train_loss(oup, template) losses.update(loss.item(), x.size(0)) optimizer.zero_grad() loss.backward() if isinstance(model, SiamFCRes22): torch.nn.utils.clip_grad_norm_(model.parameters(), 10) # gradient clip if is_valid_number(loss.item()): optimizer.step() # optimizer.step() batch_time.update(time.time() - end) end = time.time() if i % args.print_freq == 0: print('Epoch: [{0}][{1}/{2}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Data {data_time.val:.3f} ({data_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t'.format( epoch, i, len(train_loader), batch_time=batch_time, data_time=data_time, loss=losses))
def valid(datacfg, cfgfile, weightfile, outfile): options = read_data_cfg(datacfg) valid_images = options["valid"] name_list = options["names"] prefix = "results" names = load_class_names(name_list) with open(valid_images) as fp: tmp_files = fp.readlines() valid_files = [item.rstrip() for item in tmp_files] m = Darknet(cfgfile) m.print_network() m.load_weights(weightfile) m.cuda() m.eval() valid_dataset = dataset.listDataset(valid_images, shape=(m.width, m.height), shuffle=False, transform=transforms.Compose([ transforms.ToTensor(), ])) valid_batchsize = 2 assert (valid_batchsize > 1) kwargs = {"num_workers": 4, "pin_memory": True} valid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=valid_batchsize, shuffle=False, **kwargs) fps = [0] * m.num_classes if not os.path.exists("results"): os.mkdir("results") for i in range(m.num_classes): buf = "%s/%s%s.txt" % (prefix, outfile, names[i]) fps[i] = open(buf, "w") lineId = -1 conf_thresh = 0.005 nms_thresh = 0.45 for batch_idx, (data, target) in enumerate(valid_loader): data = data.cuda() data = Variable(data, volatile=True) output = m(data).data batch_boxes = get_region_boxes(output, conf_thresh, m.num_classes, m.anchors, m.num_anchors, 0, 1) for i in range(output.size(0)): lineId = lineId + 1 fileId = os.path.basename(valid_files[lineId]).split(".")[0] width, height = get_image_size(valid_files[lineId]) print(valid_files[lineId]) boxes = batch_boxes[i] boxes = nms(boxes, nms_thresh) for box in boxes: x1 = (box[0] - box[2] / 2.0) * width y1 = (box[1] - box[3] / 2.0) * height x2 = (box[0] + box[2] / 2.0) * width y2 = (box[1] + box[3] / 2.0) * height det_conf = box[4] for j in range((len(box) - 5) / 2): cls_conf = box[5 + 2 * j] cls_id = box[6 + 2 * j] prob = det_conf * cls_conf fps[cls_id].write("%s %f %f %f %f %f\n" % (fileId, prob, x1, y1, x2, y2)) for i in range(m.num_classes): fps[i].close()
model.load_weights(weightfile) model.print_network() region_loss.seen = model.seen processed_batches = model.seen/batch_size init_width = model.width init_height = model.height init_epoch = model.seen/nsamples kwargs = {'num_workers': num_workers, 'pin_memory': True} if use_cuda else {} test_loader = torch.utils.data.DataLoader( dataset.listDataset(testlist, shape=(init_width, init_height), shuffle=False, transform=transforms.Compose([ transforms.ToTensor(), ]), train=False), batch_size=batch_size, shuffle=False, **kwargs) if use_cuda: if ngpus > 1: model = torch.nn.DataParallel(model).cuda() else: model = model.cuda() params_dict = dict(model.named_parameters()) params = [] for key, value in params_dict.items(): if key.find('.bn') >= 0 or key.find('.bias') >= 0: params += [{'params': [value], 'weight_decay': 0.0}]
os.system('mkdir {0}'.format(opt.experiment)) opt.manualSeed = random.randint(1, 10000) # fix seed print("Random Seed: ", opt.manualSeed) random.seed(opt.manualSeed) np.random.seed(opt.manualSeed) torch.manual_seed(opt.manualSeed) cudnn.benchmark = True if torch.cuda.is_available() and not opt.cuda: print( "WARNING: You have a CUDA device, so you should probably run with --cuda" ) train_dataset = dataset.listDataset(list_file=opt.trainlist) assert train_dataset if not opt.random_sample: sampler = dataset.randomSequentialSampler(train_dataset, opt.batchSize) else: sampler = None train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=opt.batchSize, shuffle=False, sampler=sampler, num_workers=int(opt.workers), collate_fn=dataset.alignCollate( imgH=opt.imgH, imgW=opt.imgW, keep_ratio=opt.keep_ratio))
def train(train_list, model, criterion, optimizer, epoch,ring,ring_weight): ring = torch.from_numpy(ring).type(torch.FloatTensor).cuda() ring_weight = torch.from_numpy(ring_weight).type(torch.FloatTensor).cuda() ring = Variable(ring) ring_weight = Variable(ring_weight) losses = AverageMeter() batch_time = AverageMeter() data_time = AverageMeter() train_loader = torch.utils.data.DataLoader( dataset.listDataset(train_list, shuffle=True, transform=transforms.Compose([ transforms.ToTensor(),transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), ]), train=True, seen=model.seen, batch_size=args.batch_size, num_workers=args.workers), batch_size=args.batch_size) print('epoch %d, processed %d samples, lr %.10f' % (epoch, epoch * len(train_loader.dataset), args.lr)) model.train() end = time.time() for i,(img, target,signal,signal_down)in enumerate(train_loader): data_time.update(time.time() - end) img = img.cuda() img = Variable(img) sig_down = signal_down.type(torch.FloatTensor).unsqueeze(0).cuda() sig_down = Variable(sig_down) sig = signal.type(torch.FloatTensor).unsqueeze(0).cuda() sig = Variable(sig) #compute round people round_count = torch.zeros([24]).type(torch.FloatTensor).cuda() round_count = Variable(round_count) for k in range(24): round_count[k] = torch.mul(sig,ring[k]).sum() sig = round_count # #random generate a mask that mask out a ring # rand = random.randint(0,174) # rand2 = random.randint(0,349) # for q in range(3): # img[:,q,rand:rand+50,rand2:rand2+50] = 0 output = model(img,sig_down) target = target.type(torch.FloatTensor).unsqueeze(0).cuda() target = Variable(target) ##multiply ring weight total_sum = torch.tensor([0]).type(torch.FloatTensor).cuda() total_sum = Variable(total_sum) for k in range(24): ring_sum = torch.abs(torch.mul(output,ring[k]).sum()-torch.mul(target,ring[k]).sum()) total_sum += ring_sum*ring_weight[k] loss = criterion(output,target)+ 0.005 * total_sum losses.update(loss.item()) optimizer.zero_grad() loss.backward() optimizer.step() batch_time.update(time.time() - end) end = time.time() if i % args.print_freq == 0: print('Epoch: [{0}][{1}/{2}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Data {data_time.val:.3f} ({data_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' .format( epoch, i, len(train_loader), batch_time=batch_time, data_time=data_time, loss=losses))
model.load_weights(weightfile) model.print_network() region_loss.seen = model.seen processed_batches = model.seen / batch_size init_width = model.width init_height = model.height init_epoch = int(model.seen / nsamples) kwargs = { 'num_workers': num_workers, 'pin_memory': True } if use_cuda else {} test_loader = torch.utils.data.DataLoader(dataset.listDataset( testlist, shape=(init_width, init_height), shuffle=False, train=False), batch_size=batch_size, shuffle=False, **kwargs) if use_cuda: cudnn.enabled = True cudnn.benchmark = True if ngpus > 1: model = torch.nn.DataParallel(model).cuda() else: model = model.cuda() params_dict = dict(model.named_parameters()) params = [] for key, value in params_dict.items():
def valid(datacfg, cfgfile, weightfile, outfile): def truths_length(truths): for i in range(50): if truths[i][1] == 0: return i # Parse configuration files options = read_data_cfg(datacfg) valid_images = options['valid'] meshname = options['mesh'] backupdir = options['backup'] name = options['name'] if not os.path.exists(backupdir): makedirs(backupdir) # Parameters prefix = 'results' seed = int(time.time()) gpus = '0' # Specify which gpus to use test_width = 544 test_height = 544 torch.manual_seed(seed) use_cuda = True if use_cuda: os.environ['CUDA_VISIBLE_DEVICES'] = gpus torch.cuda.manual_seed(seed) save = False testtime = True use_cuda = True num_classes = 1 testing_samples = 0.0 eps = 1e-5 notpredicted = 0 conf_thresh = 0.1 nms_thresh = 0.4 match_thresh = 0.5 if save: makedirs(backupdir + '/test') makedirs(backupdir + '/test/gt') makedirs(backupdir + '/test/pr') # To save testing_error_trans = 0.0 testing_error_angle = 0.0 testing_error_pixel = 0.0 errs_2d = [] errs_3d = [] errs_trans = [] errs_angle = [] errs_corner2D = [] preds_trans = [] preds_rot = [] preds_corners2D = [] gts_trans = [] gts_rot = [] gts_corners2D = [] edges_corners = [[0, 1], [0, 2], [0, 4], [1, 3], [1, 5], [2, 3], [2, 6], [3, 7], [4, 5], [4, 6], [5, 7], [6, 7]] # Read object model information, get 3D bounding box corners mesh = MeshPly(meshname) vertices = np.c_[np.array(mesh.vertices), np.ones((len(mesh.vertices), 1))].transpose() corners3D = get_3D_corners(vertices) # diam = calc_pts_diameter(np.array(mesh.vertices)) diam = float(options['diam']) # Read intrinsic camera parameters internal_calibration = get_camera_intrinsic() # Get validation file names with open(valid_images) as fp: tmp_files = fp.readlines() valid_files = [item.rstrip() for item in tmp_files] # Specicy model, load pretrained weights, pass to GPU and set the module in evaluation mode model = Darknet(cfgfile) model.print_network() model.load_weights(weightfile) model.cuda() model.eval() # Get the parser for the test dataset valid_dataset = dataset.listDataset(valid_images, shape=(test_width, test_height), shuffle=False, transform=transforms.Compose([ transforms.ToTensor(), ])) valid_batchsize = 1 # Specify the number of workers for multiple processing, get the dataloader for the test dataset kwargs = {'num_workers': 4, 'pin_memory': True} test_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=valid_batchsize, shuffle=False, **kwargs) logging(" Testing {}...".format(name)) logging(" Number of test samples: %d" % len(test_loader.dataset)) # Iterate through test batches (Batch size for test data is 1) count = 0 z = np.zeros((3, 1)) for batch_idx, (data, target) in enumerate(test_loader): # Images img = data[0, :, :, :] img = img.numpy().squeeze() img = np.transpose(img, (1, 2, 0)) t1 = time.time() # Pass data to GPU if use_cuda: data = data.cuda() target = target.cuda() # Wrap tensors in Variable class, set volatile=True for inference mode and to use minimal memory during inference data = Variable(data, volatile=True) t2 = time.time() # Forward pass output = model(data).data t3 = time.time() # Using confidence threshold, eliminate low-confidence predictions all_boxes = get_region_boxes(output, conf_thresh, num_classes) t4 = time.time() # Iterate through all images in the batch for i in range(output.size(0)): # For each image, get all the predictions boxes = all_boxes[i] # For each image, get all the targets (for multiple object pose estimation, there might be more than 1 target per image) truths = target[i].view(-1, 21) # Get how many object are present in the scene num_gts = truths_length(truths) # Iterate through each ground-truth object for k in range(num_gts): box_gt = [ truths[k][1], truths[k][2], truths[k][3], truths[k][4], truths[k][5], truths[k][6], truths[k][7], truths[k][8], truths[k][9], truths[k][10], truths[k][11], truths[k][12], truths[k][13], truths[k][14], truths[k][15], truths[k][16], truths[k][17], truths[k][18], 1.0, 1.0, truths[k][0] ] best_conf_est = -1 # If the prediction has the highest confidence, choose it as our prediction for single object pose estimation for j in range(len(boxes)): if (boxes[j][18] > best_conf_est): match = corner_confidence9( box_gt[:18], torch.FloatTensor(boxes[j][:18])) box_pr = boxes[j] best_conf_est = boxes[j][18] # Denormalize the corner predictions corners2D_gt = np.array(np.reshape(box_gt[:18], [9, 2]), dtype='float32') corners2D_pr = np.array(np.reshape(box_pr[:18], [9, 2]), dtype='float32') corners2D_gt[:, 0] = corners2D_gt[:, 0] * 640 corners2D_gt[:, 1] = corners2D_gt[:, 1] * 480 corners2D_pr[:, 0] = corners2D_pr[:, 0] * 640 corners2D_pr[:, 1] = corners2D_pr[:, 1] * 480 preds_corners2D.append(corners2D_pr) gts_corners2D.append(corners2D_gt) # Compute corner prediction error corner_norm = np.linalg.norm(corners2D_gt - corners2D_pr, axis=1) corner_dist = np.mean(corner_norm) errs_corner2D.append(corner_dist) # Compute [R|t] by pnp R_gt, t_gt = pnp( np.array(np.transpose( np.concatenate((np.zeros((3, 1)), corners3D[:3, :]), axis=1)), dtype='float32'), corners2D_gt, np.array(internal_calibration, dtype='float32')) R_pr, t_pr = pnp( np.array(np.transpose( np.concatenate((np.zeros((3, 1)), corners3D[:3, :]), axis=1)), dtype='float32'), corners2D_pr, np.array(internal_calibration, dtype='float32')) if save: preds_trans.append(t_pr) gts_trans.append(t_gt) preds_rot.append(R_pr) gts_rot.append(R_gt) np.savetxt( backupdir + '/test/gt/R_' + valid_files[count][-8:-3] + 'txt', np.array(R_gt, dtype='float32')) np.savetxt( backupdir + '/test/gt/t_' + valid_files[count][-8:-3] + 'txt', np.array(t_gt, dtype='float32')) np.savetxt( backupdir + '/test/pr/R_' + valid_files[count][-8:-3] + 'txt', np.array(R_pr, dtype='float32')) np.savetxt( backupdir + '/test/pr/t_' + valid_files[count][-8:-3] + 'txt', np.array(t_pr, dtype='float32')) np.savetxt( backupdir + '/test/gt/corners_' + valid_files[count][-8:-3] + 'txt', np.array(corners2D_gt, dtype='float32')) np.savetxt( backupdir + '/test/pr/corners_' + valid_files[count][-8:-3] + 'txt', np.array(corners2D_pr, dtype='float32')) # Compute translation error trans_dist = np.sqrt(np.sum(np.square(t_gt - t_pr))) errs_trans.append(trans_dist) # Compute angle error angle_dist = calcAngularDistance(R_gt, R_pr) errs_angle.append(angle_dist) # Compute pixel error Rt_gt = np.concatenate((R_gt, t_gt), axis=1) Rt_pr = np.concatenate((R_pr, t_pr), axis=1) proj_2d_gt = compute_projection(vertices, Rt_gt, internal_calibration) proj_2d_pred = compute_projection(vertices, Rt_pr, internal_calibration) proj_corners_gt = np.transpose( compute_projection(corners3D, Rt_gt, internal_calibration)) proj_corners_pr = np.transpose( compute_projection(corners3D, Rt_pr, internal_calibration)) norm = np.linalg.norm(proj_2d_gt - proj_2d_pred, axis=0) pixel_dist = np.mean(norm) errs_2d.append(pixel_dist) # Visualize fig = plt.figure() plt.xlim((0, 640)) plt.ylim((0, 480)) plt.imshow(scipy.misc.imresize(img, (480, 640))) # Projections for edge in edges_corners: plt.plot(proj_corners_gt[edge, 0], proj_corners_gt[edge, 1], color='g', linewidth=3.0) plt.plot(proj_corners_pr[edge, 0], proj_corners_pr[edge, 1], color='b', linewidth=3.0) plt.gca().invert_yaxis() # plt.show() plt.savefig(outfile + '/output_' + str(count) + '_.png', bbox_inches='tight') fig.canvas.draw() count = count + 1 # Compute 3D distances transform_3d_gt = compute_transformation(vertices, Rt_gt) transform_3d_pred = compute_transformation(vertices, Rt_pr) norm3d = np.linalg.norm(transform_3d_gt - transform_3d_pred, axis=0) vertex_dist = np.mean(norm3d) errs_3d.append(vertex_dist) # Sum errors testing_error_trans += trans_dist testing_error_angle += angle_dist testing_error_pixel += pixel_dist testing_samples += 1 count = count + 1 t5 = time.time() # Compute 2D projection error, 6D pose error, 5cm5degree error px_threshold = 5 acc = len(np.where( np.array(errs_2d) <= px_threshold)[0]) * 100. / (len(errs_2d) + eps) acc5cm5deg = len( np.where((np.array(errs_trans) <= 0.05) & (np.array(errs_angle) <= 5))[0]) * 100. / (len(errs_trans) + eps) acc3d10 = len(np.where( np.array(errs_3d) <= diam * 0.1)[0]) * 100. / (len(errs_3d) + eps) acc5cm5deg = len( np.where((np.array(errs_trans) <= 0.05) & (np.array(errs_angle) <= 5))[0]) * 100. / (len(errs_trans) + eps) corner_acc = len(np.where(np.array(errs_corner2D) <= px_threshold) [0]) * 100. / (len(errs_corner2D) + eps) mean_err_2d = np.mean(errs_2d) mean_corner_err_2d = np.mean(errs_corner2D) nts = float(testing_samples) if testtime: print('-----------------------------------') print(' tensor to cuda : %f' % (t2 - t1)) print(' predict : %f' % (t3 - t2)) print('get_region_boxes : %f' % (t4 - t3)) print(' eval : %f' % (t5 - t4)) print(' total : %f' % (t5 - t1)) print('-----------------------------------') # Print test statistics logging('Results of {}'.format(name)) logging(' Acc using {} px 2D Projection = {:.2f}%'.format( px_threshold, acc)) logging(' Acc using 10% threshold - {} vx 3D Transformation = {:.2f}%'. format(diam * 0.1, acc3d10)) logging(' Acc using 5 cm 5 degree metric = {:.2f}%'.format(acc5cm5deg)) logging( " Mean 2D pixel error is %f, Mean vertex error is %f, mean corner error is %f" % (mean_err_2d, np.mean(errs_3d), mean_corner_err_2d)) logging( ' Translation error: %f m, angle error: %f degree, pixel error: % f pix' % (testing_error_trans / nts, testing_error_angle / nts, testing_error_pixel / nts)) if save: predfile = backupdir + '/predictions_linemod_' + name + '.mat' scipy.io.savemat( predfile, { 'R_gts': gts_rot, 't_gts': gts_trans, 'corner_gts': gts_corners2D, 'R_prs': preds_rot, 't_prs': preds_trans, 'corner_prs': preds_corners2D })
def valid(datacfg, darknetcfg, learnetcfg, weightfile, outfile): options = read_data_cfg(datacfg) valid_images = options['valid'] metadict = options['meta'] # name_list = options['names'] # backup = cfg.backup ckpt = weightfile.split('/')[-1].split('.')[0] backup = weightfile.split('/')[-2] prefix = 'results/' + backup.split('/')[-1] + '/e' + ckpt print('saving to: ' + prefix) # prefix = 'results/' + weightfile.split('/')[1] # names = load_class_names(name_list) with open(valid_images) as fp: tmp_files = fp.readlines() valid_files = [item.rstrip() for item in tmp_files] m = Darknet(darknetcfg, learnetcfg) m.print_network() m.load_weights(weightfile) m.cuda() m.eval() valid_dataset = dataset.listDataset(valid_images, shape=(m.width, m.height), shuffle=False, transform=transforms.Compose([ transforms.ToTensor(), ])) valid_batchsize = 2 assert (valid_batchsize > 1) kwargs = {'num_workers': 4, 'pin_memory': True} valid_loader = torch.utils.data.DataLoader( valid_dataset, batch_size=valid_batchsize, shuffle=False, **kwargs) metaset = dataset.MetaDataset(metafiles=metadict, train=False) metaloader = torch.utils.data.DataLoader( metaset, batch_size=metaset.batch_size, shuffle=False, **kwargs ) metaloader = iter(metaloader) n_cls = len(metaset.classes) if not os.path.exists(prefix): # os.mkdir(prefix) os.makedirs(prefix) fps = [0] * n_cls for i, cls_name in enumerate(metaset.classes): buf = '%s/%s%s.txt' % (prefix, outfile, cls_name) fps[i] = open(buf, 'w') lineId = -1 conf_thresh = 0.005 nms_thresh = 0.45 for batch_idx, (data, target) in enumerate(valid_loader): metax, mask = metaloader.next() # print(ids) data = data.cuda() mask = mask.cuda() metax = metax.cuda() data = Variable(data, volatile=True) mask = Variable(mask, volatile=True) metax = Variable(metax, volatile=True) output = m(data, metax, mask) if isinstance(output, tuple): output = (output[0].data, output[1].data) else: output = output.data batch_boxes = get_region_boxes_v2(output, n_cls, conf_thresh, m.num_classes, m.anchors, m.num_anchors, 0, 1) if isinstance(output, tuple): bs = output[0].size(0) else: assert output.size(0) % n_cls == 0 bs = output.size(0) // n_cls for b in range(bs): lineId = lineId + 1 imgpath = valid_dataset.lines[lineId].rstrip() print(imgpath) imgid = os.path.basename(imgpath).split('.')[0] width, height = get_image_size(imgpath) for i in range(n_cls): # oi = i * bs + b oi = b * n_cls + i boxes = batch_boxes[oi] boxes = nms(boxes, nms_thresh) for box in boxes: x1 = (box[0] - box[2] / 2.0) * width y1 = (box[1] - box[3] / 2.0) * height x2 = (box[0] + box[2] / 2.0) * width y2 = (box[1] + box[3] / 2.0) * height det_conf = box[4] for j in range((len(box) - 5) / 2): cls_conf = box[5 + 2 * j] cls_id = box[6 + 2 * j] prob = det_conf * cls_conf fps[i].write('%s %f %f %f %f %f\n' % (imgid, prob, x1, y1, x2, y2)) for i in range(n_cls): fps[i].close()
def train(epoch): global processed_batches # Initialize timer t0 = time.time() # Get the dataloader for training dataset train_loader = torch.utils.data.DataLoader(dataset.listDataset( trainlist, shape=(init_width, init_height), shuffle=True, transform=transforms.Compose([ transforms.ToTensor(), ]), train=True, seen=model.seen, batch_size=batch_size, num_workers=num_workers, bg_file_names=bg_file_names), batch_size=batch_size, shuffle=False, **kwargs) # TRAINING lr = adjust_learning_rate(optimizer, processed_batches) logging('epoch %d, processed %d samples, lr %f' % (epoch, epoch * len(train_loader.dataset), lr)) # Start training model.train() t1 = time.time() avg_time = torch.zeros(9) niter = 0 # Iterate through batches for batch_idx, (data, target) in enumerate(train_loader): t2 = time.time() # adjust learning rate adjust_learning_rate(optimizer, processed_batches) processed_batches = processed_batches + 1 # Pass the data to GPU if use_cuda: data = data.cuda() t3 = time.time() # Wrap tensors in Variable class for automatic differentiation data, target = Variable(data), Variable(target) t4 = time.time() # Zero the gradients before running the backward pass optimizer.zero_grad() t5 = time.time() # Forward pass output = model(data) t6 = time.time() model.seen = model.seen + data.data.size(0) region_loss.seen = region_loss.seen + data.data.size(0) # Compute loss, grow an array of losses for saving later on loss = region_loss(output, target, epoch) training_iters.append( epoch * math.ceil(len(train_loader.dataset) / float(batch_size)) + niter) training_losses.append(convert2cpu(loss.data)) niter += 1 t7 = time.time() # Backprop: compute gradient of the loss with respect to model parameters loss.backward() t8 = time.time() # Update weights optimizer.step() t9 = time.time() # Print time statistics if False and batch_idx > 1: avg_time[0] = avg_time[0] + (t2 - t1) avg_time[1] = avg_time[1] + (t3 - t2) avg_time[2] = avg_time[2] + (t4 - t3) avg_time[3] = avg_time[3] + (t5 - t4) avg_time[4] = avg_time[4] + (t6 - t5) avg_time[5] = avg_time[5] + (t7 - t6) avg_time[6] = avg_time[6] + (t8 - t7) avg_time[7] = avg_time[7] + (t9 - t8) avg_time[8] = avg_time[8] + (t9 - t1) print('-------------------------------') print(' load data : %f' % (avg_time[0] / (batch_idx))) print(' cpu to cuda : %f' % (avg_time[1] / (batch_idx))) print('cuda to variable : %f' % (avg_time[2] / (batch_idx))) print(' zero_grad : %f' % (avg_time[3] / (batch_idx))) print(' forward feature : %f' % (avg_time[4] / (batch_idx))) print(' forward loss : %f' % (avg_time[5] / (batch_idx))) print(' backward : %f' % (avg_time[6] / (batch_idx))) print(' step : %f' % (avg_time[7] / (batch_idx))) print(' total : %f' % (avg_time[8] / (batch_idx))) t1 = time.time() t1 = time.time() return epoch * math.ceil( len(train_loader.dataset) / float(batch_size)) + niter - 1
def train(train_list, teacher_model, student_model, criterion, optimizer, epoch): losses = AverageMeter() batch_time = AverageMeter() data_time = AverageMeter() # _train_loader = dataset.listDataset(train_list, # shuffle=True, # transform=transforms.Compose([ # transforms.ToTensor(), # transforms.Normalize(mean=[0.485, 0.456, 0.406], # from img net ? # std=[0.229, 0.224, 0.225]), # ]), # train=True, # # seen=model.seen, # ) # print(" line 196 ..........................") # train_loader = DataLoader( _train_loader, # batch_size=args.batch_size, # shuffle=True, # num_workers=6) train_loader = torch.utils.data.DataLoader( dataset.listDataset( train_list, shuffle=True, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), ]), train=True, # seen=model.seen, batch_size=args.batch_size, num_workers=args.workers), batch_size=args.batch_size) print( f'epoch {epoch}, processed {(train_loader.__len__)} samples, lr {args.lr} \n\n' ) student_model.train() end = time.time() # print(f"train loader dataset: \n", str(train_loader)) for i, (img, target) in enumerate(train_loader): data_time.update(time.time() - end) # img = torch.randn(1,3,400,800).float() img = img.cuda() img = Variable(img) target = target.type(torch.FloatTensor).unsqueeze(1).cuda() target = Variable(target) print(f"img ********************************* TRAIN.PY : {img.shape}") with torch.no_grad(): teacher_output, teacher_kd_list, teacher_resize_list = teacher_model( img) # print(" line 224 ............______________________________________________________________-.") student_output, student_kd_list, student_resize_list = student_model( img) print( f"target ****************************** TRAIN.PY: {target.shape}") print("target = ", target.sum().item()) print("teacher output: ", teacher_output.sum()) # break loss = criterion(student_kd_list, teacher_kd_list, student_resize_list, teacher_resize_list, student_output, teacher_output, target) losses.update(loss.item(), img.size(0)) optimizer.zero_grad() loss.backward() optimizer.step() batch_time.update(time.time() - end) end = time.time() if i % args.print_freq == 0: print('\n\n\nEpoch: [{0}][{1}/{2}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Data {data_time.val:.3f} ({data_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t'.format( epoch, i, len(train_loader), batch_time=batch_time, data_time=data_time, loss=losses)) print( " accuracy: ", student_output.sum().data / target.sum().data * 100, " % \n\n")
def valid(datacfg, cfgfile, weightfile, outfile): options = read_data_cfg(datacfg) valid_images = options['valid'] # backup = cfg.backup backup = weightfile.split('/')[-2] ckpt = weightfile.split('/')[-1].split('.')[0] prefix = 'results/' + backup.split('/')[-1] + '/e' + ckpt print('saving to: ' + prefix) names = cfg.classes with open(valid_images) as fp: tmp_files = fp.readlines() valid_files = [item.rstrip() for item in tmp_files] m = Darknet(cfgfile) m.print_network() m.load_weights(weightfile) m.cuda() m.eval() valid_dataset = dataset.listDataset(valid_images, shape=(m.width, m.height), shuffle=False, transform=transforms.Compose([ transforms.ToTensor(), ])) valid_batchsize = 2 assert (valid_batchsize > 1) kwargs = {'num_workers': 4, 'pin_memory': True} valid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=valid_batchsize, shuffle=False, **kwargs) fps = [0] * m.num_classes if not os.path.exists(prefix): # os.mkdir(prefix) os.makedirs(prefix) for i in range(m.num_classes): buf = '%s/%s%s.txt' % (prefix, outfile, names[i]) fps[i] = open(buf, 'w') lineId = -1 conf_thresh = 0.005 nms_thresh = 0.45 for batch_idx, (data, target) in enumerate(valid_loader): data = data.cuda() data = Variable(data, volatile=True) output = m(data).data batch_boxes = get_region_boxes(output, conf_thresh, m.num_classes, m.anchors, m.num_anchors, 0, 1) for i in range(output.size(0)): lineId = lineId + 1 fileId = os.path.basename(valid_files[lineId]).split('.')[0] width, height = get_image_size(valid_files[lineId]) print(valid_files[lineId]) boxes = batch_boxes[i] boxes = nms(boxes, nms_thresh) for box in boxes: x1 = (box[0] - box[2] / 2.0) * width y1 = (box[1] - box[3] / 2.0) * height x2 = (box[0] + box[2] / 2.0) * width y2 = (box[1] + box[3] / 2.0) * height det_conf = box[4] # import pdb # pdb.set_trace() for j in range((len(box) - 5) / 2): cls_conf = box[5 + 2 * j] cls_id = box[6 + 2 * j] prob = det_conf * cls_conf fps[cls_id].write('%s %f %f %f %f %f\n' % (fileId, prob, x1, y1, x2, y2)) # fps[cls_id].write('%s %f %f %f %f %f %f\n' % (fileId, det_conf, cls_conf, x1, y1, x2, y2)) for i in range(m.num_classes): fps[i].close()
def train(epoch): global processed_batches global seen global region_loss_seen t0 = time.time() def batch_iter(it, batch_size): def list2np(t): imgs, labels = zip(*t) retimgs = np.zeros((len(imgs),) + imgs[0].shape, dtype=np.float32) retlabels = np.zeros( (len(labels),) + labels[0].shape, dtype=np.float32) for i, img in enumerate(imgs): retimgs[i, :, :, :] = img for i, label in enumerate(labels): retlabels[i, :] = label return retimgs, retlabels retlist = [] for i, item in enumerate(it): retlist.append(item) if i % batch_size == batch_size - 1: ret = list2np(retlist) # Don't train for batches that contain no labels if not (np.sum(ret[1]) == 0): yield ret retlist = [] # Excess data is discarded if len(retlist) > 0: ret = list2np(retlist) # Don't train for batches that contain no labels if not (np.sum(ret[1]) == 0): yield ret train_loader_base = dataset.listDataset(args.train, args, shape=(init_width, init_height), shuffle=True, train=True, seen=seen, batch_size=batch_size, num_workers=num_workers) train_loader = batch_iter(iter(train_loader_base), batch_size=batch_size) lr = adjust_learning_rate(solver_convweights, processed_batches) lr = adjust_learning_rate(solver_others, processed_batches) logging('epoch %d, processed %d samples, lr %f' % (epoch, epoch * len(train_loader_base), lr)) yolo_x_nnabla, yolo_features_nnabla, yolo_vars, yolo_tvars, loss_nnabla = create_network( batch_size, init_height, init_width, args) t1 = time.time() step_called = False solver_convweights.zero_grad() solver_others.zero_grad() for batch_idx, (data_tensor, target_tensor) in enumerate(train_loader): dn = data_tensor tn = target_tensor if dn.shape != yolo_x_nnabla.shape: del(yolo_x_nnabla) del(yolo_features_nnabla) yolo_x_nnabla, yolo_features_nnabla, yolo_vars, yolo_tvars, loss_nnabla = create_network( dn.shape[0], dn.shape[2], dn.shape[3], args) yolo_x_nnabla.d = dn yolo_features_nnabla.forward() for v in yolo_vars: v.forward() region_loss_seen += data_tensor.shape[0] nGT, nCorrect, nProposals = region_loss.forward_nnabla( args, region_loss_seen, yolo_features_nnabla, target_tensor, yolo_vars, yolo_tvars) loss_nnabla.forward() loss_nnabla.backward() print('%d: nGT %d, recall %d, proposals %d, loss: total %f' % (region_loss_seen, nGT, nCorrect, nProposals, loss_nnabla.d)) yolo_features_nnabla.backward(grad=None, clear_buffer=True) if batch_idx % args.accum_times == args.accum_times - 1: step_called = True solver_convweights.weight_decay(args.decay*batch_size) solver_convweights.update() solver_convweights.zero_grad() solver_others.update() solver_others.zero_grad() processed_batches = processed_batches + 1 adjust_learning_rate(solver_convweights, processed_batches) adjust_learning_rate(solver_others, processed_batches) t1 = time.time() if not step_called: solver_convweights.weight_decay(args.decay*batch_size) solver_convweights.update() solver_convweights.zero_grad() solver_others.update() solver_others.zero_grad() print() t1 = time.time() logging('training with %f samples/s' % (len(train_loader_base)/(t1-t0))) if (epoch+1) % args.save_interval == 0: logging('save weights to %s/%06d.h5' % (args.output, epoch+1)) seen = (epoch + 1) * len(train_loader_base) nn.save_parameters('%s/%06d.h5' % (args.output, epoch+1))
testing_errors_angle = [] testing_errors_pixel = [] testing_accuracies = [] # Get the intrinsic camerea matrix, mesh, vertices and corners of the model mesh = MeshPly(meshname) vertices = np.c_[np.array(mesh.vertices), np.ones((len(mesh.vertices), 1))].transpose() corners3D = get_3D_corners(vertices) internal_calibration = get_camera_intrinsic() # Specify the number of workers kwargs = {'num_workers': num_workers, 'pin_memory': True} if use_cuda else {} # Get the dataloader for test data test_loader = torch.utils.data.DataLoader(dataset.listDataset(testlist, shape=(test_width, test_height), shuffle=False, transform=transforms.Compose([transforms.ToTensor(),]), train=False), batch_size=1, shuffle=False, **kwargs) # Pass the model to GPU if use_cuda: model = model.cuda() # model = torch.nn.DataParallel(model, device_ids=[0]).cuda() # Multiple GPU parallelism # Get the optimizer params_dict = dict(model.named_parameters()) params = [] for key, value in params_dict.items(): if key.find('.bn') >= 0 or key.find('.bias') >= 0: params += [{'params': [value], 'weight_decay': 0.0}] else: params += [{'params': [value], 'weight_decay': decay*batch_size}]
def valid(datacfg, cfgfile, weightfile, outfile): def truths_length(truths): for i in range(50): if truths[i][1] == 0: return i # Parse configuration files options = read_data_cfg(datacfg) valid_images = options['valid'] meshname = options['mesh'] backupdir = options['backup'] name = options['name'] if not os.path.exists(backupdir): makedirs(backupdir) # Parameters prefix = 'results' seed = int(time.time()) gpus = '0' # Specify which gpus to use test_width = 544 test_height = 544 torch.manual_seed(seed) use_cuda = True visualize = True if use_cuda: os.environ['CUDA_VISIBLE_DEVICES'] = gpus torch.cuda.manual_seed(seed) save = False testtime = True use_cuda = True num_classes = 1 eps = 1e-5 notpredicted = 0 conf_thresh = 0.1 nms_thresh = 0.4 match_thresh = 0.5 edges_corners = [[0, 1], [0, 2], [0, 4], [1, 3], [1, 5], [2, 3], [2, 6], [3, 7], [4, 5], [4, 6], [5, 7], [6, 7]] if save: makedirs(backupdir + '/test') makedirs(backupdir + '/test/gt') makedirs(backupdir + '/test/pr') # Read object model information, get 3D bounding box corners mesh = MeshPly(meshname) vertices = np.c_[np.array(mesh.vertices), np.ones((len(mesh.vertices), 1))].transpose() corners3D = get_3D_corners(vertices) # diam = calc_pts_diameter(np.array(mesh.vertices)) diam = float(options['diam']) # Read intrinsic camera parameters internal_calibration = get_camera_intrinsic() # Get validation file names with open(valid_images) as fp: tmp_files = fp.readlines() valid_files = [item.rstrip() for item in tmp_files] # Specicy model, load pretrained weights, pass to GPU and set the module in evaluation mode model = Darknet(cfgfile) # model.print_network() model.load_weights(weightfile) model.cuda() model.eval() # Get the parser for the test dataset valid_dataset = dataset.listDataset(valid_images, shape=(test_width, test_height), shuffle=False, transform=transforms.Compose([ transforms.ToTensor(), ])) valid_batchsize = 1 # Specify the number of workers for multiple processing, get the dataloader for the test dataset kwargs = {'num_workers': 4, 'pin_memory': True} test_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=valid_batchsize, shuffle=False, **kwargs) logging(" Testing {}...".format(name)) logging(" Number of test samples: %d" % len(test_loader.dataset)) # Iterate through test batches (Batch size for test data is 1) count = 0 z = np.zeros((3, 1)) print("--------------------------------------------") for batch_idx, (data, target) in enumerate(test_loader): # Images print("-----------------idx---------------------------", batch_idx) print("-------------------data-------------------------", data.shape) print("--------------------target------------------------", target.shape) img = data[0, :, :, :] img = img.numpy().squeeze() img = np.transpose(img, (1, 2, 0)) # Pass data to GPU if use_cuda: data = data.cuda() target = target.cuda() # Wrap tensors in Variable class, set volatile=True for inference mode and to use minimal memory during inference data = Variable(data, volatile=True) # Forward pass output = model(data).data # Using confidence threshold, eliminate low-confidence predictions all_boxes = get_region_boxes(output, conf_thresh, num_classes) # Iterate through all images in the batch for i in range(output.size(0)): # For each image, get all the predictions boxes = all_boxes[i] # For each image, get all the targets (for multiple object pose estimation, there might be more than 1 target per image) truths = target[i].view(-1, 21) # Get how many object are present in the scene num_gts = truths_length(truths) # Iterate through each ground-truth object for k in range(num_gts): box_gt = [ truths[k][1], truths[k][2], truths[k][3], truths[k][4], truths[k][5], truths[k][6], truths[k][7], truths[k][8], truths[k][9], truths[k][10], truths[k][11], truths[k][12], truths[k][13], truths[k][14], truths[k][15], truths[k][16], truths[k][17], truths[k][18], 1.0, 1.0, truths[k][0] ] best_conf_est = -1 # If the prediction has the highest confidence, choose it as our prediction for single object pose estimation for j in range(len(boxes)): if (boxes[j][18] > best_conf_est): match = corner_confidence9( box_gt[:18], torch.FloatTensor(boxes[j][:18])) box_pr = boxes[j] best_conf_est = boxes[j][18] # Denormalize the corner predictions corners2D_gt = np.array(np.reshape(box_gt[:18], [9, 2]), dtype='float32') corners2D_pr = np.array(np.reshape(box_pr[:18], [9, 2]), dtype='float32') corners2D_gt[:, 0] = corners2D_gt[:, 0] * 1920 corners2D_gt[:, 1] = corners2D_gt[:, 1] * 1080 corners2D_pr[:, 0] = corners2D_pr[:, 0] * 1920 corners2D_pr[:, 1] = corners2D_pr[:, 1] * 1080 # corners2D_gt[:, 0] = corners2D_gt[:, 0] * 640 # corners2D_gt[:, 1] = corners2D_gt[:, 1] * 480 # corners2D_pr[:, 0] = corners2D_pr[:, 0] * 640 # corners2D_pr[:, 1] = corners2D_pr[:, 1] * 480 preds_corners2D.append(corners2D_pr) gts_corners2D.append(corners2D_gt) # print("corners gt:\n",gts_corners2D) # print("corners pr:\n", preds_corners2D) # Compute [R|t] by pnp R_gt, t_gt = pnp( np.array(np.transpose( np.concatenate((np.zeros((3, 1)), corners3D[:3, :]), axis=1)), dtype='float32'), corners2D_gt, np.array(internal_calibration, dtype='float32')) R_pr, t_pr = pnp( np.array(np.transpose( np.concatenate((np.zeros((3, 1)), corners3D[:3, :]), axis=1)), dtype='float32'), corners2D_pr, np.array(internal_calibration, dtype='float32')) #Eulerangle # # Compute pixel error Rt_gt = np.concatenate((R_gt, t_gt), axis=1) Rt_pr = np.concatenate((R_pr, t_pr), axis=1) proj_2d_gt = compute_projection(vertices, Rt_gt, internal_calibration) proj_2d_pred = compute_projection(vertices, Rt_pr, internal_calibration) proj_corners_gt = np.transpose( compute_projection(corners3D, Rt_gt, internal_calibration)) proj_corners_pr = np.transpose( compute_projection(corners3D, Rt_pr, internal_calibration)) if visualize: # Visualize plt.xlim((0, 1920)) plt.ylim((0, 1080)) plt.imshow(scipy.misc.imresize(img, (1080, 1920))) plt.plot(corners2D_gt[1][0], corners2D_gt[1][1], 'r*') plt.plot(corners2D_gt[2][0], corners2D_gt[2][1], 'r*') plt.plot(corners2D_gt[3][0], corners2D_gt[3][1], 'r*') plt.plot(corners2D_gt[4][0], corners2D_gt[4][1], 'r*') plt.plot(corners2D_gt[5][0], corners2D_gt[5][1], 'r*') plt.plot(corners2D_gt[6][0], corners2D_gt[6][1], 'r*') plt.plot(corners2D_gt[7][0], corners2D_gt[7][1], 'r*') plt.plot(corners2D_gt[8][0], corners2D_gt[8][1], 'r*') # plt.plot(corners2D_pr[0][0], corners2D_pr[0][1], 'b*') # plt.xlim((0, 640)) # plt.ylim((0, 480)) # plt.imshow(scipy.misc.imresize(img, (480, 640))) # Projections for edge in edges_corners: # plt.plot(proj_corners_gt[edge, 0], proj_corners_gt[edge, 1], color='g', linewidth=2.0) plt.plot(corners2D_gt[edge, 0], corners2D_gt[edge, 1], color='r', linewidth=2.0) plt.plot(proj_corners_pr[edge, 0], proj_corners_pr[edge, 1], color='b', linewidth=2.0) plt.gca().invert_yaxis() # print(corners2D_pr[0]) # print(corners2D_pr) # print(EulerAngle) # plt.show() print("_________") plt.show()