def create_object(config):
# set seed value
config.manualSeed = random.randint(1, 10000)
random.seed(config.manualSeed)
torch.manual_seed(config.manualSeed)
torch.cuda.manual_seed_all(config.manualSeed)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
# load dataset
train_loader, test_loader = getattr(dataset_factory,
config.dataloader.name)(config)
# model & loss func & optimizer
nets = []
criterions = []
optimizers = []
for model_args in config.models:
# model
net = getattr(model_fuctory, model_args.name)(**model_args.args)
net = net.cuda(config.gpu.id)
# load weight
if model_args.load_weight.path is not None:
utils.load_model(net, model_args.load_weight.path,
model_args.load_weight.model_id)
nets += [net]
# loss function
criterions = []
for row in config.losses:
r = []
for loss in row:
criterion = getattr(loss_func, loss.name)(loss.args)
criterion = criterion.cuda(config.gpu.id)
r += [criterion]
criterions += [loss_func.TotalLoss(r)]
# optimizer
optimizer = getattr(torch.optim, model_args.optim.name)
optimizers += [optimizer(net.parameters(), **model_args.optim.args)]
# trainer
trainer = getattr(trainer_module, config.trainer.name)(config)
# logger
logs = utils.LogManagers(len(config.models), len(train_loader.dataset),
config.trainer.epochs,
config.dataloader.batch_size)
return trainer, nets, criterions, optimizers, train_loader, test_loader, logs
def test_net(args):
print("Init...")
_, _, val_loader, _ = lib.build_dataloader(args)
model = lib.build_model(args)
load_model(model, args)
args.cuda = not args.no_cuda and torch.cuda.is_available()
if args.cuda:
model.cuda()
if args.label_smoothing:
criterion = cross_entropy_with_label_smoothing
else:
criterion = nn.CrossEntropyLoss()
print("Start testing...")
val(model, val_loader, criterion, args.test_model, args)
def __init__(self, opt):
self.model = TensorMask(backbone=opt.backbone,
num_cls=opt.num_class,
base_window=opt.base_window,
freezeBN=opt.frezeBN,
freezeLayers=opt.frezeLayer,
align_corners=opt.align_corners)
self.model = load_model(self.model, opt.weights)
self.model.eval()
self.model.cuda()
self.mean = COCO.mean
self.std = COCO.std
self.opt = opt
self.strides = np.array(
[self.opt.base_stride * 2**i for i in range(self.opt.k + 1)])
self.windows = np.array(
[self.opt.base_window * lamda for lamda in self.strides], np.int32)
self.output_size = np.array(
list(
zip(self.opt.input_w // self.strides,
self.opt.input_h // self.strides)))
self.num_det = [
output_w * output_h for output_w, output_h in self.output_size
]
self.det_offset = np.cumsum(self.num_det)
def main():
args = parse_arguments()
if args.task not in ['haystack', 'form', 'issue', 'target', 'multi']:
print("User error: --task must be either " +\
"haystack, form, issue, target or multi.")
sys.exit(0)
# Read the article as string into variable
with open(args.article, 'r') as file:
text = file.read().replace('\n', '')
model, tokenizer = load_model(args)
# Tokenize article as input to model
text_tokenized = torch.tensor([tokenizer.encode(text)])
results = predict(model, text_tokenized, args)
print_results(results, args)
if args.output_path != '':
store_results(results, args)
models = ["/vol/pfc/data/models/sequence_2_oversample_25/train_sequence_2_vs_seq3_25_200000_iter_200000.caffemodel"]
means = ["/vol/pfc/data/means/Sequences_2_Oversample_mean.npy"]
deploys=["/vol/pfc/prototxt/sequence2oversample/train_sequence_25_test3_lmdb_deploy.prototxt"]
len_model=len(models)
len_sequences=len(sequences)
print "inicio"
data_evaluate2=[]
for i in range(0,len_model):
model=models[i]
deploy=deploys[i]
print 'evalua el modelo: '+str(model)
net=utils.load_model(model,deploy)
transform=utils.load_transform(net,means[i])
data_evaluate2.append([])
for j in range(0,len_sequences):
print 'evalua sequence '+str(sequences[j])
data=utils.load_sequence(sequences[j])
data_evaluate2[i].append(utils.process_sequence(net,data,transform))
print "end "+str(model)
import pickle
X= np.array(data_evaluate2)
output = open('evaluate_20_200000_vs_1_3.pkl', 'wb')
pickle.dump(X, output)
output.close()
opt.batch_size = opt.batch_size * len(opt.gpus)
opt.save_dir = os.path.join(opt.save_dir,opt.exp_id)
logger = Logger(opt)
model = TensorMask(backbone=opt.backbone , num_cls=opt.num_class ,
base_window= opt.base_window ,
freezeBN=opt.frezeBN,freezeLayers=opt.frezeLayer,
align_corners= opt.align_corners)
optimizer = optimer.SGD([{'params':filter(lambda x:len(x.size()) == 4 ,model.parameters()),'weight_decay':0.0001 },
{'params': filter(lambda x:len(x.size()) <4,model.parameters())}],
lr=opt.lr,warm_up=1000,momentum=0.9,nesterov=True)
start_epoch = 0
if opt.weights != '' :
model, optimizer, start_epoch = load_model(
model, opt.weights, optimizer, opt.resume, opt.lr, opt.lr_step)
trainer = Trainer(opt,model,optimizer)
trainer.set_device(opt.gpus,opt.device)
print('Setting up data...')
val_loader = torch.utils.data.DataLoader(
COCO(cfg=opt, split='val',augment=False),
batch_size=8,
shuffle=False,
num_workers=8,
pin_memory=True
)
train_loader = torch.utils.data.DataLoader(
COCO(cfg=opt, split='train',augment=True),
batch_size=opt.batch_size,
shuffle=True,
def main(config):
val_data_loader = DataLoader(PluckerData3D_precompute(phase='valid',
config=configs),
batch_size=1,
shuffle=False,
drop_last=False,
num_workers=1)
# no gradients
with torch.no_grad():
# Model initialization
Model = load_model("PluckerNetKnn")
model = Model(config)
# limited GPU
if config.gpu_inds > -1:
torch.cuda.set_device(config.gpu_inds)
device = torch.device('cuda', config.gpu_inds)
else:
device = torch.device(
'cuda' if torch.cuda.is_available() else 'cpu')
model.to(device)
# load the weights
if config.weights:
checkpoint = torch.load(config.weights)
model.load_state_dict(checkpoint['state_dict'])
logging.info(model)
# evaluation model
model.eval()
num_data = 0
data_timer, matching_timer = Timer(), Timer()
tot_num_data = len(val_data_loader.dataset)
data_loader_iter = val_data_loader.__iter__()
# collecting the errors in rotation, errors in tranlsation, num of inliers, inlier ratios
measure_list = ["err_q", "err_t", "inlier_ratio"]
eval_res = {}
for measure in measure_list:
eval_res[measure] = np.zeros(tot_num_data)
for batch_idx in range(tot_num_data):
data_timer.tic()
matches, plucker1, plucker2, R_gt, t_gt = data_loader_iter.next()
data_timer.toc()
nb_plucker = matches.size(1)
# you can comment this line, as my GPU is short of memory
if nb_plucker > 3000 or nb_plucker < 2:
continue
matches, plucker1, plucker2 = matches.to(device), plucker1.to(
device), plucker2.to(device)
# Compute output
matching_timer.tic()
prob_matrix, prior1, prior2 = model(plucker1, plucker2)
matching_timer.toc()
# compute the topK correspondences
k = min(200, round(plucker1.size(1) * plucker2.size(1)))
_, P_topk_i = torch.topk(prob_matrix.flatten(start_dim=-2),
k=k,
dim=-1,
largest=True,
sorted=True)
plucker1_indices = P_topk_i / prob_matrix.size(
-1) # bxk (integer division)
plucker2_indices = P_topk_i % prob_matrix.size(-1) # bxk
# in case cannot be estimated
err_q = np.pi
err_t = np.inf
inlier_ratio = 0
nb_inliers_gt = np.where(
matches[0, :].cpu().numpy() > 0)[0].shape[0]
# more than 3 3D-3D matches
if k > 3:
# let's check the inliner ratios within the topK matches
# retrieve the inlier/outlier 1/0 logit
inlier_inds = matches[:, plucker1_indices,
plucker2_indices].cpu().numpy()
inlier_ratio = np.sum(inlier_inds) / k * 100.0
# compute the rotation and translation error
plucker1_topK = plucker1[0, plucker1_indices[
0, :k], :].cpu().numpy()
plucker2_topK = plucker2[0, plucker2_indices[
0, :k], :].cpu().numpy()
if config.dataset == "structured3D" or config.dataset == "semantic3D":
dis_threshold = 0.5
else:
dis_threshold = 1e-1
best_rot, best_trans, best_ic, best_ic_mask = run_ransac(
plucker1_topK.T,
plucker2_topK.T,
inlier_threshold=dis_threshold)
if best_rot is None or best_trans is None:
err_q, err_t = np.pi, np.inf
else:
err_q, err_t = evaluate_R_t(best_rot, best_trans,
R_gt[0, :, :].numpy(),
t_gt.numpy())
num_data += 1
torch.cuda.empty_cache()
eval_res["err_q"][batch_idx] = err_q
eval_res["err_t"][batch_idx] = err_t
eval_res["inlier_ratio"][batch_idx] = inlier_ratio
logging.info(' '.join([
f"Validation iter {num_data} / {tot_num_data} : Data Loading Time: {data_timer.avg:.3f},",
f"Matching Time: {matching_timer.avg:.3f},",
f"err_rot: {err_q:.3f}, err_t: {err_t:.3f}, inlier_ratio: {inlier_ratio:.3f}, nb_matches: {k}, nb_inliers_gt: {nb_inliers_gt}, nb_plucker:{nb_plucker}",
]))
data_timer.reset()
# after checking all the validation samples, let's calculate statistics
recall = recalls(eval_res)
logging.info(' '.join([
f"recall_rot: {recall[0]:.3f}, med. rot. : {recall[1]:.3f}, med. trans. : {recall[2]:.3f}, avg. inlier ratio: {recall[3]:.3f},",
]))
def identify(name):
classifier = utils.load_model()
name = utils.preprocess(name)
print classifier.classify(utils.feature_extract(name))
def __init__(self,
backbone='resnet18',
norm_layer=nn.BatchNorm2d,
cfg=None,
aux_layers=True):
super(Segment, self).__init__()
self.cfg = cfg
self.aux_layers = aux_layers
if backbone == 'resnet18':
channels = [64, 128, 256, 512]
self.backbone_rgb = resnet18(in_channel=3, norm_layer=norm_layer)
self.backbone_d = resnet18(in_channel=1, norm_layer=norm_layer)
backbone_rgb = load_model(self.backbone_rgb,
'model_zoo/resnet18-5c106cde.pth')
backbone_d = load_model(self.backbone_d,
'model_zoo/resnet18-5c106cde.pth',
depth_input=True)
elif backbone == 'resnet34':
channels = [64, 128, 256, 512] # resnet34
self.backbone_rgb = resnet34(in_channel=3, norm_layer=norm_layer)
self.backbone_d = resnet34(in_channel=1, norm_layer=norm_layer)
backbone_rgb = load_model(self.backbone_rgb,
'model_zoo/resnet34-333f7ec4.pth')
backbone_d = load_model(self.backbone_rgb,
'model_zoo/resnet34-333f7ec4.pth',
depth_input=True)
elif backbone == 'resnet50':
channels = [256, 512, 1024, 2048]
self.backbone_rgb = resnet50(in_channel=3, norm_layer=norm_layer)
self.backbone_d = resnet50(in_channel=1, norm_layer=norm_layer)
backbone_rgb = load_model(self.backbone_rgb,
'model_zoo/resnet50-19c8e357.pth')
backbone_d = load_model(self.backbone_rgb,
'model_zoo/resnet50-19c8e357.pth',
depth_input=True)
else:
raise Exception("backbone:%s does not support!" % backbone)
if backbone_rgb is None:
print(
"Warning: the model_zoo of {} does no exist!".format(backbone))
else:
self.backbone_rgb = backbone_rgb
self.backbone_d = backbone_d
# fusion modules
self.cmat5 = CMAT(channels[3], True, ratio=8)
self.cmat4 = CMAT(channels[2], True, ratio=8)
self.cmat3 = CMAT(channels[1], True, ratio=8)
self.cmat2 = CMAT(channels[0], True, ratio=8)
# low-level & high-level
self.fam54_1 = FAM(256, 256)
self.fam43_1 = FAM(256, 256)
self.fam32_1 = FAM(256, 256)
self.fam54_2 = FAM(256, 256)
self.fam43_2 = FAM(256, 256)
self.fam32_2 = FAM(256, 256)
# fusion, TBD
self.fusion = Fusion(256)
if self.aux_layers:
self.linear5_1 = nn.Conv2d(256,
1,
kernel_size=3,
stride=1,
padding=1)
self.linear4_1 = nn.Conv2d(256,
1,
kernel_size=3,
stride=1,
padding=1)
self.linear3_1 = nn.Conv2d(256,
1,
kernel_size=3,
stride=1,
padding=1)
self.linear2_1 = nn.Conv2d(256,
1,
kernel_size=3,
stride=1,
padding=1)
self.linear5_2 = nn.Conv2d(256,
1,
kernel_size=3,
stride=1,
padding=1)
self.linear4_2 = nn.Conv2d(256,
1,
kernel_size=3,
stride=1,
padding=1)
self.linear3_2 = nn.Conv2d(256,
1,
kernel_size=3,
stride=1,
padding=1)
self.linear2_2 = nn.Conv2d(256,
1,
kernel_size=3,
stride=1,
padding=1)
self.linear_out = nn.Conv2d(256, 1, kernel_size=3, stride=1, padding=1)
self.gap1 = nn.AdaptiveAvgPool2d(1)
self.gap2 = nn.AdaptiveAvgPool2d(1)
self.fc = nn.Sequential(
nn.Linear(channels[-1] * 2, 512),
##nn.Dropout(p=0.3),
nn.ReLU(True),
nn.Linear(512, 256 + 1),
nn.Sigmoid(),
)
self.initialize()
EPOCHs = 500
SMALL = True
DATASETMODE = 'autoencoderSpike'
MODEL_PTH = 'vae'
LR = 0.0001
netParams = snn.params('network_specs/vae.yaml')
print(netParams)
device = torch.device("cuda" if USE_CUDA else "cpu")
# Create network instance.
model = VAE(netParams, hidden_size=100, latent_size=2).to(device)
# Load model
load_model(MODEL_PTH, model)
# Define optimizer module.
optimizer = torch.optim.Adam(model.parameters(), lr=LR, amsgrad=True)
# Learning stats instance.
stats = snn.learningStats()
trainingSet = SMNIST(datasetPath=netParams['training']['path']['in'],
samplingTime=netParams['simulation']['Ts'],
sampleLength=netParams['simulation']['tSample'],
mode=DATASETMODE,
small=SMALL)
testingSet = SMNIST(datasetPath=netParams['training']['path']['in'],
samplingTime=netParams['simulation']['Ts'],
sampleLength=netParams['simulation']['tSample'],
def main(exps,
model_type,
aux,
data,
num_steps,
num_samples=16,
hdim=10,
num_test=100,
sigma=0.03,
prior="normal",
reverse=False,
evalu=False,
plot_posterior=False):
run = True
print 'model: ' + model_type + aux
print 'data: ' + data
if 'data' == 'continuous':
print 'sigma: ' + str(sigma)
permute = True
model, generator = utils.load_model(model_type, aux)
if exps == "train":
print('run train')
if data == 'continuous':
X = utils.load_mnist('train')[:50000]
X += rng.uniform(0, 1, size=X.shape) / 256
if data == 'binary':
X = utils.fixbinary_mnist('train')[:50000]
elif exps == "valid":
print('run valid')
if data == 'continuous':
X = utils.load_mnist('test')[:5000]
X += rng.uniform(0, 1, size=X.shape) / 256
if data == 'binary':
X = utils.fixbinary_mnist('test')[:5000]
elif exps == "test":
print('run test')
if data == 'continuous':
X = utils.load_mnist('test')[5000:]
X += rng.uniform(0, 1, size=X.shape) / 256
if data == 'binary':
X = utils.fixbinary_mnist('test')
elif exps == "BDMC":
print('run BDMC')
exact_h = np.random.RandomState(seed=2039).normal(
0, 1, size=[num_simulated_samples, hdim]).astype(np.float32)
X = (generator(exact_h)).eval()
if data == 'binary':
X = operator.le(np.random.uniform(size=X.shape),
X).astype(np.float32)
else:
X += np.random.RandomState(seed=1429).normal(0,
np.sqrt(sigma),
size=X.shape).astype(
np.float32)
elif exps[:4] == 'post':
plot_posterior = 1
print('run posterior sampling')
if exps[-3:] == 'tra':
X, y = utils.load_mnist('train', label=True)[:50000]
elif exps[-3:] == 'val':
X, y = utils.load_mnist('test', label=True)[:5000]
X = X[y == int(exps[4])]
if plot_posterior:
directory_name = "vis/" + model_type + aux + 'num_steps' + str(
num_steps) + 'sigma' + str(sigma) + exps + 'posterior_sample/'
if os.path.exists(directory_name):
finalstate = np.load(directory_name + 'final_state.npy')
pf = np.load(directory_name + 'pf.npy')
lld = 0
run = False
##Shuffle the data
if permute:
permutation = np.random.RandomState(seed=2919).permutation(X.shape[0])
X = X[permutation][:num_test]
if reverse:
exact_h = exact_h[permutation][:num_test]
results = {'lld': [], 'pf': []}
iwae_time = None
if evalu:
print('IWAE evalution...')
t_start = time.time()
batch = np.reshape(X, (num_test, 28 * 28))
utils.estimate_lld(model, batch, in_sam)
t_end = time.time()
iwae_time = t_end - t_start
print('IWAE Eval time: ' + str(iwae_time) + ' seconds')
if reverse and run:
print('run BDMC reverse')
t_start = time.time()
lld, pf, finalstate = samplers.run_reverse_ais(generator, X, exact_h,
num_steps, sigma, hdim,
L, eps, data, prior)
t_end = time.time()
ais_time = t_end - t_start
print('BDMC backward Eval time: ' + str(ais_time) + ' seconds')
elif run:
print('num_test: ' + str(num_test))
t_start = time.time()
lld, pf, finalstate = samplers.run_ais(generator, X, num_samples,
num_steps, sigma, hdim, L, eps,
data, prior)
t_end = time.time()
ais_time = t_end - t_start
print('AIS forward Eval time: ' + str(ais_time) + ' seconds')
results['lld'].append(lld)
results['pf'].append(pf)
if plot_posterior:
directory_name = "vis/" + model_type + aux + 'num_steps' + str(
num_steps) + 'sigma' + str(sigma) + exps + 'posterior_sample/'
if not os.path.exists(directory_name):
os.makedirs(directory_name)
post_img = (generator(finalstate)).eval()
post_img = post_img.reshape(num_samples, num_test, 28, 28)
img_size = int(np.sqrt(num_test))
exppf = np.exp(pf - np.max(pf, axis=0))
sampling_prob = exppf / np.sum(exppf, axis=0)
choices = []
for i in range(num_test):
choices.append(rng.choice(num_samples, 3, p=sampling_prob[:, i]))
choices = np.vstack(choices)
for i in range(3):
utils.plot_gen(post_img[choices[:, i],
np.arange(num_test)],
directory_name + model_type + aux +
'posterior_sample' + str(i) + "num_steps" +
str(num_steps) + 'sigma' + str(sigma),
n_ex=num_test,
dim=(10, 3))
np.save(directory_name + 'final_state.npy', finalstate)
np.save(directory_name + 'pf.npy', pf)
if exps == 'post2tra':
return X, post_img[choices[:, 0], np.arange(num_test)]
return {'res': results, 'ais_time': ais_time, 'iwae_time': iwae_time}
def train_net(gpu, ngpus_per_node, args):
args.gpu = gpu
if args.gpu is not None:
print("Use GPU: {} for training".format(args.gpu))
if args.distributed:
if args.dist_url == "env://" and args.rank == -1:
args.rank = int(os.environ["RANK"])
if args.multiprocessing_distributed:
# For multiprocessing distributed training, rank needs to be the
# global rank among all the processes
args.rank = args.rank * ngpus_per_node + gpu
dist.init_process_group(backend=args.dist_backend,
init_method=args.dist_url,
world_size=args.world_size,
rank=args.rank)
verbose = args.multiprocessing_distributed and args.rank % ngpus_per_node == 0
if verbose and not args.test:
log_writer = tensorboardX.SummaryWriter(args.log_dir)
else:
log_writer = None
model = lib.build_model(args)
print('Parameters:', sum([np.prod(p.size()) for p in model.parameters()]))
optimizer = lib.build_optimizer(args, model)
if args.distributed:
torch.cuda.set_device(args.gpu)
model.cuda(args.gpu)
args.batch_size = int(args.batch_size / ngpus_per_node)
args.val_batch_size = int(args.val_batch_size / ngpus_per_node)
args.num_workers = int(
(args.num_workers + ngpus_per_node - 1) / ngpus_per_node)
model = torch.nn.parallel.DistributedDataParallel(
model, device_ids=[args.gpu])
epoch = 0
if args.resume:
epoch = resume_model(model, optimizer, args)
if args.label_smoothing:
criterion = cross_entropy_with_label_smoothing
else:
criterion = nn.CrossEntropyLoss()
cudnn.benchmark = True
train_loader, train_sampler, val_loader, val_sampler = lib.build_dataloader(
args, args.distributed)
if args.test:
load_model(model, args)
val(model,
val_loader,
val_sampler,
criterion,
epoch,
verbose=verbose,
args=args)
return
if verbose:
print("Start training...")
while epoch < args.epochs:
train(model,
train_sampler,
train_loader,
optimizer,
criterion,
epoch,
log_writer,
args,
verbose=verbose)
if (epoch + 1) % args.save_epochs == 0:
dist_save_model(model, optimizer, epoch, ngpus_per_node, args)
if (epoch + 1) % args.test_epochs == 0:
val(model,
val_loader,
val_sampler,
criterion,
epoch,
args,
log_writer,
verbose=verbose)
epoch += 1
dist_save_model(model, optimizer, epoch - 1, ngpus_per_node, args)
def __init__(self, config, data_loader, val_data_loader=None):
# Model initialization
Model = load_model("PluckerNetKnn")
self.model = Model(config)
if config.weights:
checkpoint = torch.load(config.weights)
self.model.load_state_dict(checkpoint['state_dict'])
logging.info(self.model)
#
self.config = config
self.max_epoch = config.train_epoches
self.save_freq = config.train_save_freq_epoch
self.val_max_iter = config.val_max_iter
self.val_epoch_freq = config.val_epoch_freq
self.best_val_metric = config.best_val_metric
self.best_val_epoch = -np.inf
self.best_val = -np.inf
if config.use_gpu and not torch.cuda.is_available():
logging.warning(
'Warning: There\'s no CUDA support on this machine, '
'training is performed on CPU.')
raise ValueError('GPU not available, but cuda flag set')
# limited GPU
if config.gpu_inds > -1:
torch.cuda.set_device(config.gpu_inds)
self.device = torch.device('cuda', config.gpu_inds)
else:
self.device = torch.device(
'cuda' if torch.cuda.is_available() else 'cpu')
self.optimizer = getattr(optim,
config.optimizer)(self.model.parameters(),
lr=config.train_lr,
betas=(0.9, 0.999))
#
self.scheduler = optim.lr_scheduler.ExponentialLR(
self.optimizer, config.exp_gamma)
#
self.start_epoch = config.train_start_epoch
# concat dataset name
self.checkpoint_dir = os.path.join(config.out_dir, config.dataset,
config.model_nb)
#
ensure_dir(self.checkpoint_dir)
json.dump(config,
open(os.path.join(self.checkpoint_dir, 'config.json'), 'w'),
indent=4,
sort_keys=False)
#
self.iter_size = config.iter_size
self.batch_size = data_loader.batch_size
self.data_loader = data_loader
self.val_data_loader = val_data_loader
self.test_valid = True if self.val_data_loader is not None else False
self.model = self.model.to(self.device)
self.writer = SummaryWriter(logdir=self.checkpoint_dir)
#
if config.resume is not None:
if osp.isfile(config.resume):
logging.info("=> loading checkpoint '{}'".format(
config.resume))
state = torch.load(config.resume)
self.start_epoch = state['epoch']
self.model.load_state_dict(state['state_dict'])
self.scheduler.load_state_dict(state['scheduler'])
self.optimizer.load_state_dict(state['optimizer'])
if 'best_val' in state.keys():
self.best_val = state['best_val']
self.best_val_epoch = state['best_val_epoch']
self.best_val_metric = state['best_val_metric']
else:
raise ValueError(
f"=> no checkpoint found at '{config.resume}'")
def train(nb_epoch,
batch_size,
store_name,
image_path,
soft_labels=False,
gaussian_std=1.5,
resume=False,
start_epoch=0,
model_path=None):
batch_size = int(batch_size)
nb_epoch = int(nb_epoch)
start_epoch = int(start_epoch)
exp_dir = store_name
try:
os.stat(exp_dir)
except:
os.makedirs(exp_dir)
use_cuda = torch.cuda.is_available()
# GPU
# Data
print('==> Preparing data..')
transform_train = transforms.Compose([
transforms.Scale((550, 550)),
transforms.RandomCrop(448, padding=8),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
])
transform_test = transforms.Compose([
transforms.Scale((550, 550)),
transforms.CenterCrop(448),
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
])
trainset = YearBuiltFolder(image_path,
calc_perf=True,
soft_labels=soft_labels,
gaussian_std=gaussian_std,
transforms=transform_train)
train_weights = np.array(trainset.train_weights)
train_sampler = torch.utils.data.WeightedRandomSampler(train_weights,
len(train_weights),
replacement=True)
if use_cuda:
device = 'cuda'
else:
device = 'cpu'
trainloader = torch.utils.data.DataLoader(trainset,
batch_size=batch_size,
shuffle=(train_sampler == None),
num_workers=4,
sampler=train_sampler)
# Testing is on the same data but without the transforms. This whole class
# only givey training performance and the user has to use the detect script
# to check the performance.
train_set_without_transforms = YearBuiltFolder(image_path,
calc_perf=True,
soft_labels=soft_labels,
gaussian_std=gaussian_std,
transforms=transform_test)
train_test_loader = torch.utils.data.DataLoader(
train_set_without_transforms,
batch_size=3,
shuffle=True,
num_workers=4)
# Model
if resume:
net = torch.load(model_path)
else:
net = load_model(model_name='resnet50_pmg',
pretrain=True,
require_grad=True,
num_classes=len(trainset.classes))
netp = torch.nn.DataParallel(net)
net.to(device)
if soft_labels:
loss = nn.KLDivLoss(reduction='batchmean')
else:
loss = nn.CrossEntropyLoss()
sm = nn.LogSoftmax(dim=1)
optimizer = optim.SGD([{
'params': net.classifier_concat.parameters(),
'lr': 0.002
}, {
'params': net.conv_block1.parameters(),
'lr': 0.002
}, {
'params': net.classifier1.parameters(),
'lr': 0.002
}, {
'params': net.conv_block2.parameters(),
'lr': 0.002
}, {
'params': net.classifier2.parameters(),
'lr': 0.002
}, {
'params': net.conv_block3.parameters(),
'lr': 0.002
}, {
'params': net.classifier3.parameters(),
'lr': 0.002
}, {
'params': net.features.parameters(),
'lr': 0.0002
}],
momentum=0.9,
weight_decay=5e-4)
max_train_acc = 0
lr = [0.002, 0.002, 0.002, 0.002, 0.002, 0.002, 0.002, 0.0002]
for epoch in range(start_epoch, nb_epoch):
print('\nEpoch: %d' % epoch)
net.train()
train_loss = 0
train_loss1 = 0
train_loss2 = 0
train_loss3 = 0
train_loss4 = 0
correct = 0
total = 0
idx = 0
for batch_idx, (inputs, targets, indexes) in enumerate(trainloader):
idx = batch_idx
if inputs.shape[0] < batch_size:
continue
if use_cuda:
inputs, targets = inputs.to(device), targets.to(device)
inputs, targets = Variable(inputs), Variable(targets)
# update learning rate
for nlr in range(len(optimizer.param_groups)):
optimizer.param_groups[nlr]['lr'] = cosine_anneal_schedule(
epoch, nb_epoch, lr[nlr])
# Step 1
optimizer.zero_grad()
inputs1 = jigsaw_generator(inputs, 8)
output_1, _, _, _ = netp(inputs1)
loss1 = loss(sm(output_1), targets) * 1
loss1.backward()
optimizer.step()
# Step 2
optimizer.zero_grad()
inputs2 = jigsaw_generator(inputs, 4)
_, output_2, _, _ = netp(inputs2)
loss2 = loss(sm(output_2), targets) * 1
loss2.backward()
optimizer.step()
# Step 3
optimizer.zero_grad()
inputs3 = jigsaw_generator(inputs, 2)
_, _, output_3, _ = netp(inputs3)
loss3 = loss(sm(output_3), targets) * 1
loss3.backward()
optimizer.step()
# Step 4
optimizer.zero_grad()
_, _, _, output_concat = netp(inputs)
concat_loss = loss(sm(output_concat), targets) * 2
concat_loss.backward()
optimizer.step()
# training log
_, predicted = torch.max(output_concat.data, 1)
if soft_labels:
_, targets = torch.max(targets.data, 1)
total += targets.size(0)
correct += predicted.eq(targets.data).cpu().sum()
train_loss += (loss1.item() + loss2.item() + loss3.item() +
concat_loss.item())
train_loss1 += loss1.item()
train_loss2 += loss2.item()
train_loss3 += loss3.item()
train_loss4 += concat_loss.item()
if batch_idx % 50 == 0:
print(
'Step: %d | Loss1: %.3f | Loss2: %.5f | Loss3: %.5f | Loss_concat: %.5f | Loss: %.3f | Acc: %.3f%% (%d/%d)'
% (batch_idx, train_loss1 / (batch_idx + 1), train_loss2 /
(batch_idx + 1), train_loss3 /
(batch_idx + 1), train_loss4 /
(batch_idx + 1), train_loss / (batch_idx + 1),
100. * float(correct) / total, correct, total))
train_acc = 100. * float(correct) / total
train_loss = train_loss / (idx + 1)
with open(os.path.join(exp_dir, 'results_train.txt'), 'a') as file:
file.write(
'Iteration %d | train_acc = %.5f | train_loss = %.5f | Loss1: %.3f | Loss2: %.5f | Loss3: %.5f | Loss_concat: %.5f |\n'
% (epoch, train_acc, train_loss, train_loss1 /
(idx + 1), train_loss2 / (idx + 1), train_loss3 /
(idx + 1), train_loss4 / (idx + 1)))
# if epoch < 5 or epoch >= 80:
print('Start testing')
if soft_labels:
train_acc, train_acc_com, train_loss, y_gt, y_pred = test_softlabels(
net, loss, train_test_loader)
else:
train_acc, train_acc_com, train_loss, y_gt, y_pred = test(
net, loss, train_test_loader)
if train_acc_com > max_train_acc:
max_train_acc = train_acc_com
net.cpu()
torch.save(net, os.path.join(store_name, 'model_best.pth'))
net.to(device)
with open(os.path.join(exp_dir, 'results_test.txt'), 'a') as file:
file.write(
'Iteration %d, test_acc = %.5f, test_acc_combined = %.5f, test_loss = %.6f\n'
% (epoch, train_acc, train_acc_com, train_loss))
net.cpu()
torch.save(
net, os.path.join(store_name, 'model_epoch_{}.pth'.format(epoch)))
net.to(device)
def main(config):
# loading the test data
val_data_loader = make_data_loader(configs, "valid", 1, num_threads=configs.val_num_thread, shuffle=False)
# no gradients
with torch.no_grad():
# Model initialization
Model = load_model(config.model)
model = Model(config)
# limited GPU
if config.gpu_inds > -1:
torch.cuda.set_device(config.gpu_inds)
device = torch.device('cuda', config.gpu_inds)
else:
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model.to(device)
# load the weights
if config.weights:
checkpoint = torch.load(config.weights)
model.load_state_dict(checkpoint['state_dict'])
logging.info(model)
# evaluation model
model.eval()
num_data = 0
data_timer, matching_timer = Timer(), Timer()
tot_num_data = len(val_data_loader.dataset)
data_loader_iter = val_data_loader.__iter__()
# collecting the errors in rotation, errors in tranlsation, num of inliers, inlier ratios
measure_list = ["err_q", "err_t", "inlier_ratio"]
nb_outlier_ratios = config.outlier_ratios.shape[0]
eval_res = {}
for measure in measure_list:
eval_res[measure] = np.zeros((nb_outlier_ratios, tot_num_data))
# for gpu memory consideration
max_nb_points = 20000
for batch_idx in range(tot_num_data):
data_timer.tic()
matches, numInliers, input_3d_xyz, input_2d_xy, R_gt, t_gt, _ = data_loader_iter.next()
data_timer.toc()
p3d_cur = input_3d_xyz
p2d_cur = input_2d_xy
matches_cur = matches
# for each outlier_ratio level
for oind in range(nb_outlier_ratios):
outlier_ratio = config.outlier_ratios[oind]
if outlier_ratio > 0:
# adding outliers
nb_added_outlier_2d = min(int(round(numInliers.item() * outlier_ratio)), max_nb_points - numInliers.item())
nb_added_outlier_3d = min(int(round(numInliers.item() * outlier_ratio)), max_nb_points - numInliers.item())
if config.outlier_type == "synthetic":
# add synthetic outliers
p2d_outlier, p3d_outlier = generate_outliers_synthetic(input_2d_xy, input_3d_xyz, nb_added_outlier_2d, nb_added_outlier_3d)
elif config.outlier_type == "real":
# add real outliers, you can load real-outlier from the non-matchable-2d/3d.pkl
raise NameError('Please modify dataloader')
# p2d_outlier, p3d_outlier = generate_outliers_real(p2d_outlier_all, p3d_outlier_all, nb_added_outlier_2d, nb_added_outlier_3d)
else:
raise NameError('Invalid outlier type')
# padding the outliers
p2d_cur = torch.cat((input_2d_xy, p2d_outlier), -2)
p3d_cur = torch.cat((input_3d_xyz, p3d_outlier), -2)
# Fill ground-truth matching indexes with outliers
b = p2d_cur.size(0)
m = p3d_cur.size(-2)
n = p2d_cur.size(-2)
matches_cur = matches.new_full((b,m,n),0.0)
matches_cur[:,:numInliers.item(),:numInliers.item()] = matches
# print([p3d_cur.size(1), p2d_cur.size(1)])
p2d_cur, p3d_cur, R_gt, t_gt, matches_cur = p2d_cur.to(device), p3d_cur.to(device), R_gt.to(
device), t_gt.to(device), matches_cur.to(device)
# Compute output
matching_timer.tic()
prob_matrix = model(p3d_cur, p2d_cur)
matching_timer.toc()
# compute the topK correspondences
# note cv2.solvePnPRansac is not stable, sometimes wrong!
# please use https://github.com/k88joshi/posest, and cite the original paper
k = min(2000, round(p3d_cur.size(1) * p2d_cur.size(1))) # Choose at most 2000 points in the testing stage
_, P_topk_i = torch.topk(prob_matrix.flatten(start_dim=-2), k=k, dim=-1, largest=True, sorted=True)
p3d_indices = P_topk_i / prob_matrix.size(-1) # bxk (integer division)
p2d_indices = P_topk_i % prob_matrix.size(-1) # bxk
# let's check the inliner ratios within the topK matches
# retrieve the inlier/outlier 1/0 logit
inlier_inds = matches_cur[:, p3d_indices, p2d_indices].cpu().numpy()
# inlier ratio
inlier_ratio = np.sum(inlier_inds) / k * 100.0
# in case cannot be estimated
err_q = np.pi
err_t = np.inf
# more than 5 2D-3D matches
if k > 5:
# compute the rotation and translation error
K = np.float32(np.array([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]))
dist_coeff = np.float32(np.array([0.0, 0.0, 0.0, 0.0]))
# RANSAC p3p
p2d_np = p2d_cur[0, p2d_indices[0, :k], :].cpu().numpy()
p3d_np = p3d_cur[0, p3d_indices[0, :k], :].cpu().numpy()
retval, rvec, tvec, inliers = cv2.solvePnPRansac(
p3d_np, p2d_np, K, dist_coeff,
iterationsCount=1000,
reprojectionError=0.01,
flags=cv2.SOLVEPNP_P3P)
if rvec is not None and tvec is not None:
R_est, _ = cv2.Rodrigues(rvec)
t_est = tvec
err_q, err_t = evaluate_R_t(R_est, t_est, R_gt[0, :, :].cpu().numpy(), t_gt.cpu().numpy())
torch.cuda.empty_cache()
eval_res["err_q"][oind, batch_idx] = err_q
eval_res["err_t"][oind, batch_idx] = err_t
eval_res["inlier_ratio"][oind, batch_idx] = inlier_ratio
logging.info(' '.join([
f"Validation iter {num_data} / {tot_num_data} : Data Loading Time: {data_timer.avg:.3f},",
f"outlier ratio: {outlier_ratio:.3f},",
f"Matching Time: {matching_timer.avg:.3f},",
f"err_rot: {err_q:.3f}, err_t: {err_t:.3f}, inlier_ratio: {inlier_ratio:.3f},",
]))
data_timer.reset()
num_data += 1
# after checking all the validation samples, let's calculate statistics
# for each outlier_ratio level
for oind in range(nb_outlier_ratios):
outlier_ratio = config.outlier_ratios[oind]
eval_res_cur = {}
for measure in measure_list:
eval_res_cur[measure] = eval_res[measure][oind,:]
recall = recalls(eval_res_cur)
logging.info(' '.join([
f" outlier_ratio: {outlier_ratio:.3f}, recall_rot: {recall[0]:.3f}, med. rot. : {recall[1]:.3f}, med. trans. : {recall[2]:.3f}, avg. inlier ratio: {recall[3]:.3f},",
]))
from lib.utils import load_model,save_model
from lib.coco import COCO
import numpy as np
import torch
import os
import cv2
os.environ['CUDA_VISIBLE_DEVICES'] = '3'
model = TensorMask(backbone=opt.backbone , num_cls=opt.num_class ,
base_window= opt.base_window ,
freezeBN=opt.frezeBN,freezeLayers=opt.frezeLayer,
align_corners=opt.align_corners)
opt.test = True
opt.weights = 'exp/coco_person/model_last.pth'
model = load_model(model, opt.weights)
model.eval()
model.cuda()
val_loader = torch.utils.data.DataLoader(
COCO(cfg=opt, split='val',augment=False),
batch_size=1,
shuffle=False,
num_workers=1,
pin_memory=True
)
strides = np.array([opt.base_stride * 2 ** i for i in range(opt.k + 1)])
windows = np.array([opt.base_window * lamda for lamda in strides], np.int32)
output_size = np.array(list(zip(opt.input_w // strides, opt.input_h // strides)))
num_det = [output_w * output_h for output_w, output_h in output_size]
det_offset = np.cumsum(num_det)