class Visualier():
"""Visulization, plot the logs during training process"""
def __init__(self, num_classes=10):
port = 8097
self.loss_logger = VisdomPlotLogger('line',
port=port,
win="Loss",
opts={'title': 'Loss Logger'})
self.acc_logger = VisdomPlotLogger('line',
port=port,
win="acc",
opts={'title': 'Accuracy Logger'})
self.confusion_logger = VisdomLogger('heatmap',
port=port,
win="confusion",
opts={
'title':
'Confusion matrix',
'columnnames':
list(range(num_classes)),
'rownames':
list(range(num_classes))
})
def plot(self, train_acc, train_err, val_acc, val_err, confusion, epoch):
self.loss_logger.log(epoch, train_err, name="train")
self.acc_logger.log(epoch, train_acc, name="train")
self.loss_logger.log(epoch, val_err, name="val")
self.acc_logger.log(epoch, val_acc, name="val")
self.confusion_logger.log(confusion)
print("epoch: [%d/%d]" % (epoch, args.n_epoches))
print('Training loss: %.4f, accuracy: %.2f%%' % (train_err, train_acc))
print('Validation loss: %.4f, accuracy: %.2f%%' % (val_err, val_acc))
class ConfusionVisdom(object):
'''Plot test confusion matrix in a VisdomLogger
'''
def __init__(self, num_classes, title='TBD'):
self._confusion = VisdomLogger(
'heatmap',
opts={
'title': '{:s} Confusion Matrix'.format(title),
'columnnames': list(range(num_classes)),
'rownames': list(range(num_classes))
})
check_visdom_server(self._confusion.viz)
def log(self, confusion, train=None):
assert train is not None,\
'train should be True or False, not {}'.format(train)
if train:
pass
else:
try:
self._confusion.log(confusion)
except BaseException as e:
check_visdom_server(self._confusion.viz)
print(e)
print("***Retry ConfusionVisdom")
self.log(confusion, train)
def log_image(im_logger: VisdomLogger,
img: torch.Tensor,
batch_size: int = None):
if batch_size is None:
batch_size = img.shape[0]
grid_img = make_grid(img.detach().cpu(),
nrow=int(batch_size**0.5),
normalize=True,
range=(0, 1)).numpy()
im_logger.log(grid_img)
return grid_img
def test_only(model,train_dataloader, val_dataloader, optimizer,
loss_fn, metrics, params, model_dir,logger,restore_file=None):
# reload weights from restore_file if specified
if restore_file is not None:
logging.info("Restoring parameters from {}".format(restore_file))
checkpoint = utils.load_checkpoint(restore_file, model, optimizer)
best_val_acc = checkpoint['best_val_acc']
params.current_epoch = checkpoint['epoch']
print('best_val_acc=',best_val_acc)
print(optimizer.state_dict()['param_groups'][0]['lr'], checkpoint['epoch'])
train_confusion_logger = VisdomLogger('heatmap', port=port,
opts={'title': params.experiment_path + 'train_Confusion matrix',
'columnnames': columnnames, 'rownames': rownames},env='Test')
test_confusion_logger = VisdomLogger('heatmap', port=port,
opts={'title': params.experiment_path + 'test_Confusion matrix',
'columnnames': columnnames, 'rownames': rownames},env='Test')
diff_confusion_logger = VisdomLogger('heatmap', port=port,
opts={'title': params.experiment_path + 'diff_Confusion matrix',
'columnnames': columnnames, 'rownames': rownames},env='Test')
# Evaluate for one epoch on validation set
# model.train()
model.eval()
train_metrics, train_confusion_meter = evaluate(model, loss_fn, train_dataloader, metrics, params, logger)
train_confusion_logger.log(train_confusion_meter.value())
model.eval()
val_metrics,test_confusion_meter = evaluate(model, loss_fn, val_dataloader, metrics, params, logger)
test_confusion_logger.log(test_confusion_meter.value())
diff_confusion_meter = train_confusion_meter.value()-test_confusion_meter.value()
diff_confusion_logger.log(diff_confusion_meter)
pass
def main(checkpoint_path, batch_size, normalized,
visdom_port):
checkpoint_path = Path(checkpoint_path)
snapshot_path = checkpoint_path.parent.parent.parent / 'snapshot.json'
with snapshot_path.open('r') as f:
snapshot_dict = json.load(f)
mat_id_to_label = snapshot_dict['mat_id_to_label']
label_to_mat_id = {int(v): int(k) for k, v in mat_id_to_label.items()}
num_classes = len(label_to_mat_id) + 1
print(f'Loading model checkpoint from {checkpoint_path!r}')
checkpoint = torch.load(checkpoint_path)
model = RendNet3(num_classes=num_classes,
num_roughness_classes=20,
num_substances=len(SUBSTANCES),
base_model=resnet.resnet18(pretrained=False))
model.load_state_dict(checkpoint['state_dict'])
model.train(False)
model = model.cuda()
validation_dataset = rendering_dataset.MaterialRendDataset(
snapshot_dict,
snapshot_dict['examples']['validation'],
shape=(384, 384),
image_transform=transforms.inference_image_transform(INPUT_SIZE),
mask_transform=transforms.inference_mask_transform(INPUT_SIZE))
validation_loader = DataLoader(
validation_dataset, batch_size=batch_size,
num_workers=8,
shuffle=False,
pin_memory=True,
collate_fn=rendering_dataset.collate_fn)
pred_counts = collections.defaultdict(collections.Counter)
# switch to evaluate mode
model.eval()
confusion_meter = tnt.meter.ConfusionMeter(
k=num_classes, normalized=normalized)
pbar = tqdm(validation_loader)
for batch_idx, batch_dict in enumerate(pbar):
input_tensor = batch_dict['image'].cuda()
labels = batch_dict['material_label'].cuda()
# compute output
output = model.forward(input_tensor)
pbar.set_description(f"{output['material'].size()}")
# _, pred = output['material'].topk(k=1, dim=1, largest=True, sorted=True)
confusion_meter.add(output['material'].cpu(), labels.cpu())
with session_scope() as sess:
materials = sess.query(models.Material).filter_by(enabled=True).all()
material_id_to_name = {m.id: m.name for m in materials}
mat_by_id = {m.id: m for m in materials}
class_names = ['background']
class_names.extend([
mat_by_id[label_to_mat_id[i]].name for i in range(1, num_classes)
])
print(len(class_names), )
confusion_matrix = confusion_meter.value()
# sorted_confusion_matrix = confusion_matrix[:, inds]
# sorted_confusion_matrix = sorted_confusion_matrix[inds, :]
# sorted_class_names = [class_names[i] for i in inds]
confusion_logger = VisdomLogger(
'heatmap', opts={
'title': 'Confusion matrix',
'columnnames': class_names,
'rownames': class_names,
'xtickfont': {'size': 8},
'ytickfont': {'size': 8},
},
env='brdf-classifier-confusion',
port=visdom_port)
confusion_logger.log(confusion_matrix)
torch.save(model.state_dict(), 'epochs/epoch_%d.pt' % state['epoch'])
# Reconstruction visualization.
test_sample = next(iter(get_iterator(False)))
ground_truth = (test_sample[0].unsqueeze(1).float() / 255.0)
<<<<<<< HEAD
_, reconstructions = model(Variable(ground_truth))
=======
_, reconstructions = model(Variable(ground_truth).cuda())
>>>>>>> 3da462b9351869b0342b95d99fef37ab3e45a309
reconstruction = reconstructions.cpu().view_as(ground_truth).data
ground_truth_logger.log(
make_grid(ground_truth, nrow=int(BATCH_SIZE ** 0.5), normalize=True, range=(0, 1)).numpy())
reconstruction_logger.log(
make_grid(reconstruction, nrow=int(BATCH_SIZE ** 0.5), normalize=True, range=(0, 1)).numpy())
# def on_start(state):
# state['epoch'] = 327
#
# engine.hooks['on_start'] = on_start
engine.hooks['on_sample'] = on_sample
engine.hooks['on_forward'] = on_forward
engine.hooks['on_start_epoch'] = on_start_epoch
engine.hooks['on_end_epoch'] = on_end_epoch
engine.train(processor, get_iterator(True), maxepoch=NUM_EPOCHS, optimizer=optimizer)
loss.backward()
optimzier.step()
schduler.step()
loss_.append(loss.item())
del loss
if opt.local_rank == 0:
print('[Epoch:{}/{}][batch:{}/{}][loss:{}][learning rate:{}]'.format(epoch,opt.n_epochs,batch,b_r,loss_[-1],optimzier.state_dict()['param_groups'][0]['lr']))
a = np.random.randint(0,31,fout.shape[0])
a_ = np.array(range(fout.shape[0]))
output = fout.detach().cpu()[a_,a,:,:]*255
output = output[:,None,:,:]
output = torch.cat([output,output,output],1)
output_ = make_grid(output,nrow=int(5)).numpy()
if opt.local_rank == 0:
train_logger.log(output_)
Loss_logger.log(epoch,np.mean(np.array(loss_)))
psnr_g = []
with torch.no_grad():
for hsi,hsi_g,hsi_resize,msi in Hyper_test:
spenet.eval()
hsi_g = hsi_g.cuda().float()
hsi_resize = hsi_resize.cuda().float()
hsi_resize = torch.nn.functional.interpolate(hsi_resize,scale_factor=(8,8),mode='bilinear')
# hsi = renet(hsi_resize)
hsi = hsi_resize
# hsi = [a.cuda().float() for a in hsi]
msi = [a.cuda().float() for a in msi]
hsi_spe,scale,refined = spenet(hsi,msi[-1])
# hsi_e = get_spe_gt(hsi_g)
# hsi_2 = hsi_spe[-1][:,:31,:,:]
loss = focal_loss(classes, focal_label) + margin_loss(
classes, margin_label)
loss.backward()
optimizer.step()
# save the metrics
meter_loss.add(loss.detach().cpu().item())
meter_accuracy.add(classes.detach().cpu(), target)
meter_confusion.add(classes.detach().cpu(), target)
if current_step % NUM_STEPS == 0:
# print the information about train
train_loss_logger.log(current_step // NUM_STEPS,
meter_loss.value()[0])
train_accuracy_logger.log(current_step // NUM_STEPS,
meter_accuracy.value()[0])
train_confusion_logger.log(meter_confusion.value())
results['train_loss'].append(meter_loss.value()[0])
results['train_accuracy'].append(meter_accuracy.value()[0])
print('[Step %d] Training Loss: %.4f Accuracy: %.2f%%' %
(current_step // NUM_STEPS, meter_loss.value()[0],
meter_accuracy.value()[0]))
reset_meters()
# test model periodically
model.eval()
with torch.no_grad():
for data, target in test_iterator:
focal_label, margin_label = target, torch.eye(
num_class).index_select(dim=0, index=target)
if torch.cuda.is_available():
data, focal_label, margin_label = data.to(
def test_gradient(self):
for mesh in self.meshes:
object = sl.Object(mesh)
self.scene.add_object(object)
pose = th.tensor([[0.0596, 0.8315, -0.5523, -0.0651],
[0.4715, 0.4642, 0.7498, -0.06036],
[0.8798, -0.3051, -0.3644, 0.80551],
[0.0000, 0.0000, 0.0000, 1.0000]])
U, S, V = th.svd(pose[:3, :3])
pose[:3, :3] = th.matmul(U, V.t())
object.set_pose(pose)
renderer = sl.RenderPass()
result = renderer.render(self.scene)
gt_mask = (result.instance_index() != 0).float()
rgb = result.rgb().detach()
gt_rgb = rgb[:, :, :3]
gt_rgb_np = gt_rgb.cpu().numpy()
gt_pose = 0
if VIS:
import visdom
from torchnet.logger import VisdomLogger
env_name = 'stillleben'
vis = visdom.Visdom(port=8097, env=env_name)
vis.close(env=env_name)
gt_logger = VisdomLogger('image',
env=env_name,
port=8097,
opts=dict(title='gt'))
img_logger = VisdomLogger('image',
env=env_name,
port=8097,
opts=dict(title='rgb'))
for param in range(6):
for obj in self.scene.objects:
new_pose = obj.pose().clone()
gt_pose = obj.pose().clone()
GT_DELTA = th.zeros(6)
GT_DELTA[param] = 0.01
new_pose = sl.diff.apply_pose_delta(gt_pose, GT_DELTA)
obj.set_pose(new_pose)
rendered_result = renderer.render(self.scene)
rendered_rgb = rendered_result.rgb()
rendered_rgb = rendered_rgb[:, :, :3]
rgb_np = rendered_rgb.cpu().numpy()
if VIS:
img_logger.log(rgb_np.transpose(2, 0, 1))
gt_logger.log(gt_rgb_np.transpose(2, 0, 1))
gt_mask = th.ones_like(gt_mask)
grad_wrt_img, l = get_gaussian_pyramid_comparison(
gt_rgb.float() / 255.0,
rendered_rgb.float() / 255.0, gt_mask.squeeze())
delta = sl.diff.backpropagate_gradient_to_poses(
self.scene, rendered_result, grad_wrt_img)
print('GT delta', GT_DELTA)
print('Delta', delta)
self.assertGreater(delta[0][param], 0)
loss_D_B = (loss_D_real + loss_D_fake) * 0.5
loss_D_B.backward()
optimizer_D_B.step()
loss_meters['loss_G_meter'].add(loss_G.item())
loss_meters['loss_G_identity_meter'].add(
(loss_identity_A + loss_identity_B).item())
loss_meters['loss_G_GAN_meter'].add(
(loss_GAN_A2B + loss_GAN_B2A).item())
loss_meters['loss_G_cycle_meter'].add(
(loss_cycle_ABA + loss_cycle_BAB).item())
loss_meters['loss_D_meter'].add((loss_D_A + loss_D_B).item())
real_A_im_logger.log(
make_grid(real_A.detach().cpu(),
nrow=int(opt.batchSize**0.5),
normalize=True,
range=(0, 1)).numpy())
real_B_im_logger.log(
make_grid(real_B.detach().cpu(),
nrow=int(opt.batchSize**0.5),
normalize=True,
range=(0, 1)).numpy())
fake_A_im_logger.log(
make_grid(fake_A.detach().cpu(),
nrow=int(opt.batchSize**0.5),
normalize=True,
range=(0, 1)).numpy())
fake_B_im_logger.log(
make_grid(fake_B.detach().cpu(),
nrow=int(opt.batchSize**0.5),
'rownames':
list(range(args.num_classes))
},
env=args.env_name)
ground_truth_logger = VisdomLogger('image',
opts={'title': 'Ground Truth'},
env=args.env_name)
reconstruction_logger = VisdomLogger('image',
opts={'title': 'Reconstruction'},
env=args.env_name)
weight_folder = 'weights/{}'.format(args.env_name.replace(' ', '_'))
if not os.path.isdir(weight_folder):
os.mkdir(weight_folder)
setting_logger.log(str(args))
print("# parameters:", sum(param.numel() for param in model.parameters()))
optimizer = torch.optim.Adam(model.parameters(), lr=args.lr)
scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, 'max', patience=1)
train_dataset = datasets.MNIST(root='./data/',
train=True,
transform=transforms.ToTensor(),
download=True)
test_dataset = datasets.MNIST(root='./data/',
train=False,
transform=transforms.ToTensor())
if os.path.isfile('loss.log'):
with open("loss.log", "r") as lossfile:
loss_list = []
for loss in lossfile:
loss_list.append(loss)
while len(loss_list) > args.load_loss:
loss_list.pop(0)
for loss in loss_list:
train_loss_logger.log(epoch_offset, float(loss))
epoch_offset += 1
ground_truth_logger_left = VisdomLogger('image', opts={'title': 'Ground Truth, left'}, env=args.env_name)
ground_truth_logger_right = VisdomLogger('image', opts={'title': 'Ground Truth, right'}, env=args.env_name)
reconstruction_logger_left = VisdomLogger('image', opts={'title': 'Reconstruction, left'}, env=args.env_name)
reconstruction_logger_right = VisdomLogger('image', opts={'title': 'Reconstruction, right'}, env=args.env_name)
setting_logger.log(str(args))
"""
Load training data
"""
train_dataset = util.MyImageFolder(root='../../data/dumps/', transform=transforms.ToTensor(), target_transform=transforms.ToTensor())
train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=args.batch_size, num_workers=args.num_workers, shuffle=True, drop_last=True)
steps = len(train_dataset) // args.batch_size
"""
Training Loop
"""
for epoch in range(args.num_epochs):
def test_only(model,
train_dataloader,
val_dataloader,
optimizer,
loss_fn,
metrics,
params,
model_dir,
logger,
restore_file=None):
if 'CAD' in params.dataset_name:
# modify the model for CAD-60 for fine-tuning
out_channel = params.model_args['out_channel']
window_size = params.model_args['window_size']
model.fc7 = nn.Sequential(
nn.Linear(
(out_channel * 4) * (window_size // 16) * (window_size // 16),
256), # 4*4 for window=64; 8*8 for window=128
nn.ReLU(),
nn.Dropout2d(p=0.5))
model.fc8 = nn.Linear(256, 12)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = model.to(device)
# reload weights from restore_file if specified
if restore_file is not None:
logging.info("Restoring parameters from {}".format(restore_file))
checkpoint = utils.load_checkpoint(restore_file, model, optimizer)
best_val_acc = checkpoint['best_val_acc']
params.current_epoch = checkpoint['epoch']
print('best_val_acc=', best_val_acc)
print(optimizer.state_dict()['param_groups'][0]['lr'],
checkpoint['epoch'])
train_confusion_logger = VisdomLogger('heatmap',
port=port,
opts={
'title': params.experiment_path +
'train_Confusion matrix',
'columnnames': columnnames,
'rownames': rownames
},
env='Test')
test_confusion_logger = VisdomLogger('heatmap',
port=port,
opts={
'title': params.experiment_path +
'test_Confusion matrix',
'columnnames': columnnames,
'rownames': rownames
},
env='Test')
diff_confusion_logger = VisdomLogger('heatmap',
port=port,
opts={
'title': params.experiment_path +
'diff_Confusion matrix',
'columnnames': columnnames,
'rownames': rownames
},
env='Test')
# Evaluate for one epoch on validation set
# model.train()
model.eval()
train_metrics, train_confusion_meter = evaluate(model, loss_fn,
train_dataloader, metrics,
params, logger)
train_confusion_logger.log(train_confusion_meter.value())
model.eval()
val_metrics, test_confusion_meter = evaluate(model, loss_fn,
val_dataloader, metrics,
params, logger)
test_confusion_logger.log(test_confusion_meter.value())
diff_confusion_meter = train_confusion_meter.value(
) - test_confusion_meter.value()
diff_confusion_logger.log(diff_confusion_meter)
pass
# x1.grad += args.lambda_ * x1_grad
# x2.grad += args.lambda_ * x2_grad
optimizer.step()
meter_accuracy.add(out.data, labels.data)
meter_loss.add(loss.data[0])
pbar.set_postfix(loss=meter_loss.value()[0],
acc=meter_accuracy.value()[0])
pbar.update()
loss = meter_loss.value()[0]
acc = meter_accuracy.value()[0]
if epoch == 0:
setting_logger.log(str(args))
train_loss_logger.log(epoch, loss)
train_error_logger.log(epoch, acc)
print("\nEpoch{} Train acc:{:4}, loss:{:4}".format(
epoch, acc, loss))
scheduler.step(loss)
torch.save(model.state_dict(),
weight_folder + "/model_{}.pth".format(epoch))
reset_meters()
# Test
print('Testing...')
model.eval()
for i, data in enumerate(test_loader):
def train():
end = time.time()
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
env_name = args.experiment_id
img_a_logger = VisdomLogger('images', env=env_name, port=8097, opts=dict(title='img_a'))
img_b_logger = VisdomLogger('images', env=env_name, port=8097, opts=dict(title='img_b'))
rnd_logger = [VisdomLogger('images', env=env_name, port=8097, opts=dict(title='rnd_rgb_{}'.format(_i))) for _i in range(4)]
for i in range(start_iter, args.num_iterations + 1):
# adjust learning rate and sigma_val (decay after 150k iter)
lr = adjust_learning_rate([optimizer], args.learning_rate, i, method=args.lr_type)
model.set_sigma(adjust_sigma(args.sigma_val, i))
# load images from multi-view
images_a, images_b, viewpoints_a, viewpoints_b = dataset_train.get_random_batch(args.batch_size)
images_a = images_a.cuda()
images_b = images_b.cuda()
viewpoints_a = viewpoints_a.cuda()
viewpoints_b = viewpoints_b.cuda()
# soft render images
render_images, laplacian_loss, flatten_loss = model([images_a, images_b],
[viewpoints_a, viewpoints_b],
task='train')
img_a_logger.log(F.interpolate(images_a.detach().clone(), scale_factor=4).squeeze()[:,:3])
img_b_logger.log(F.interpolate(images_b.detach().clone(), scale_factor=4).squeeze()[:,:3])
for _i in range(4):
rnd_logger[_i].log(F.interpolate(render_images[_i].detach().clone(), scale_factor=4).squeeze()[:,:3])
import ipdb; ipdb.set_trace()
laplacian_loss = laplacian_loss.mean()
flatten_loss = flatten_loss.mean()
# compute loss
loss = multiview_iou_loss(render_images, images_a, images_b) + \
args.lambda_laplacian * laplacian_loss + \
args.lambda_flatten * flatten_loss
losses.update(loss.data.item(), images_a.size(0))
# compute gradient and optimize
optimizer.zero_grad()
loss.backward()
optimizer.step()
batch_time.update(time.time() - end)
end = time.time()
# save checkpoint
if i % args.save_freq == 0:
model_path = os.path.join(directory_output, 'checkpoint_%07d.pth.tar'%i)
torch.save({
'model': model.state_dict(),
'optimizer': optimizer.state_dict(),
}, model_path)
# save demo images
if i % args.demo_freq == 0:
demo_image = images_a[0:1]
demo_path = os.path.join(directory_output, 'demo_%07d.obj'%i)
demo_v, demo_f = model.reconstruct(demo_image)
srf.save_obj(demo_path, demo_v[0], demo_f[0])
imageio.imsave(os.path.join(image_output, '%07d_fake.png' % i), img_cvt(render_images[0][0]))
imageio.imsave(os.path.join(image_output, '%07d_input.png' % i), img_cvt(images_a[0]))
# print
if i % args.print_freq == 0:
print('Iter: [{0}/{1}]\t'
'Time {batch_time.val:.3f}\t'
'Loss {loss.val:.3f}\t'
'lr {lr:.6f}\t'
'sv {sv:.6f}\t'.format(i, args.num_iterations,
batch_time=batch_time, loss=losses,
lr=lr, sv=model.renderer.rasterizer.sigma_val))
train_loss_logger.log(epoch + epoch_offset,
loss - loss_recon,
name='loss')
test_loss_logger.log(epoch + epoch_offset, test_loss, name='loss')
"""
loss_relation = loss_recon/(loss-loss_recon)
if loss_relation > 0.25 and epoch>15:
fac = 0.25/loss_relation
print("Loss relation = {}. Recon-factor reduced from {} to {}".format(loss_relation, args.recon_factor, args.recon_factor*fac))
args.recon_factor *= fac
"""
if not args.disable_recon:
ground_truth_logger_left.log(
make_grid(imgs,
nrow=int(args.batch_size**0.5),
normalize=True,
range=(0, 1)).cpu().numpy())
reconstruction_logger_left.log(
make_grid(recon.data,
nrow=int(args.batch_size**0.5),
normalize=True,
range=(0, 1)).cpu().numpy())
train_loss_logger.log(epoch + epoch_offset,
loss_recon,
name='recon')
test_loss_logger.log(epoch + epoch_offset,
test_loss_recon,
name='recon')
with open("loss.log", "a") as myfile:
class statBase():
def __init__(self, args):
self.args = args
self.lossAvg = tnt.meter.AverageValueMeter()
#self.lossSparseMu = tnt.meter.AverageValueMeter()
#self.lossSparseVar = tnt.meter.AverageValueMeter()
self.train_loss_logger = VisdomPlotLogger('line', opts={'title': 'Train Loss'}, env='PoseCapsules')
self.test_loss_logger = VisdomPlotLogger('line', opts={'title': 'Test Loss'}, env='PoseCapsules')
self.recon_sum = 0
self.rout_id = 1
if not self.args.disable_recon:
self.reconLossAvg = tnt.meter.AverageValueMeter()
self.ground_truth_logger_left = VisdomLogger('image', opts={'title': 'Ground Truth, left'}, env='PoseCapsules')
self.reconstruction_logger_left = VisdomLogger('image', opts={'title': 'Reconstruction, left'}, env='PoseCapsules')
if self.args.regularize:
self.regularizeLossAvg = tnt.meter.AverageValueMeter()
self.logsigAvg = tnt.meter.AverageValueMeter()
self.costmeanAvg = tnt.meter.AverageValueMeter()
self.costAvg = tnt.meter.AverageValueMeter()
self.aAvg = tnt.meter.AverageValueMeter()
def reset(self):
self.lossAvg.reset()
if not self.args.disable_recon:
self.reconLossAvg.reset()
if self.args.regularize:
self.regularizeLossAvg.reset()
self.logsigAvg.reset()
self.costmeanAvg.reset()
self.costAvg.reset()
self.aAvg.reset()
def log(self, pbar, output, labels, dict = OrderedDict(), stat=None):
if not self.args.disable_loss:
dict['loss'] = self.lossAvg.value()[0]
if stat is not None:
self.logsigAvg.add(stat[-self.rout_id*4 + 0])
self.costmeanAvg.add(stat[-self.rout_id*4 + 1])
self.costAvg.add(stat[-self.rout_id*4 + 2])
self.aAvg.add(stat[-self.rout_id*4 + 3])
stat.clear()
dict['logsig'] = self.logsigAvg.value()[0]
dict['costmean'] = self.costmeanAvg.value()[0]
dict['cost'] = self.costAvg.value()[0]
dict['a'] = self.aAvg.value()[0]
#dict['mloss'] = self.lossSparseMu.value()[0]
#dict['vloss'] = self.lossSparseVar.value()[0]
if not self.args.disable_recon:
#pbar.set_postfix(loss=self.lossAvg.value()[0], refresh=False)
#else:
dict['reconloss'] = self.reconLossAvg.value()[0]
dict['rsum'] = self.recon_sum
#pbar.set_postfix(loss=self.lossAvg.value()[0], rloss=self.reconLossAvg.value()[0], rsum=self.recon_sum, refresh=False)
if self.args.regularize:
dict['reguloss'] = self.regularizeLossAvg.value()[0]
pbar.set_postfix(dict, refresh=False)
def endTrainLog(self, epoch, groundtruth_image=None, recon_image=None):
#self.train_loss = self.lossAvg.value()[0]
if not self.args.disable_loss:
self.train_loss_logger.log(epoch, self.lossAvg.value()[0], name='loss')
with open("train.log", "a") as myfile:
myfile.write(str(self.lossAvg.value()[0]) + '\n')
if not self.args.disable_recon:
if groundtruth_image is not None:
self.ground_truth_logger_left.log(make_grid(groundtruth_image, nrow=int(self.args.batch_size ** 0.5), normalize=True, range=(0, 1)).cpu().numpy())
if recon_image is not None:
self.reconstruction_logger_left.log(make_grid(recon_image.data, nrow=int(self.args.batch_size ** 0.5), normalize=True, range=(0, 1)).cpu().numpy())
#self.train_recon_loss = self.reconLossAvg.value()[0]
self.train_loss_logger.log(epoch, self.reconLossAvg.value()[0], name='recon')
#if self.args.regularize:
# self.train_regularize_loss = self.regularizeLossAvg.value()[0]
def endTestLog(self, epoch):
#loss = self.lossAvg.value()[0]
if not self.args.disable_loss:
self.test_loss_logger.log(epoch, self.lossAvg.value()[0], name='loss')
with open("test.log", "a") as myfile:
myfile.write(str(self.lossAvg.value()[0]) + '\n')
if not self.args.disable_recon:
self.test_loss_logger.log(epoch, self.reconLossAvg.value()[0], name='recon')
def load_loss(self, history_count):
if os.path.isfile('train.log'):
with open("train.log", "r") as lossfile:
loss_list = []
for loss in lossfile:
loss_list.append(loss)
while len(loss_list) > history_count:
loss_list.pop(0)
epoch = -len(loss_list)
for loss in loss_list:
self.train_loss_logger.log(epoch, float(loss), name='loss')
epoch += 1
if os.path.isfile('test.log'):
with open("test.log", "r") as lossfile:
loss_list = []
for loss in lossfile:
loss_list.append(loss)
while len(loss_list) > history_count:
loss_list.pop(0)
epoch = -len(loss_list)
for loss in loss_list:
self.test_loss_logger.log(epoch, float(loss), name='loss')
epoch += 1
class_criterion_B, train_loader, test_loader, opt)
if opt.pretrained:
trained_model = load_weight(fullModel, opt.pretrained, verbose=True)
# -- Evaluation
nTestImages = reid_set.test_inds # [2 ** (n+1) for n in range(5)]
cmc, simMat, _, avgSame, avgDiff = compute_cmc(reid_set, nTestImages,
trained_model, 128)
print(cmc)
print(simMat)
print(avgSame, avgDiff)
sim_logger = VisdomLogger('heatmap',
port=8097,
opts={
'title': 'simMat',
'columnnames': list(range(len(simMat[0]))),
'rownames': list(range(len(simMat)))
})
cmc_logger = VisdomPlotLogger("line", win="cmc_curve")
for i, v in enumerate(cmc):
cmc_logger.log(i, v, name="cmc_curve")
sim_logger.log(simMat)
log.info("Saving results...")
with open("cmc.pkl", 'w') as f:
pickle.dump(cmc, f)
with open("simMat.pkl", 'w') as f:
pickle.dump(simMat, f)
def test_only(model,
train_dataloader,
val_dataloader,
optimizer,
loss_fn,
metrics,
params,
model_dir,
logger,
restore_file=None):
# reload weights from restore_file if specified
if restore_file is not None:
logging.info("Restoring parameters from {}".format(restore_file))
checkpoint = utils.load_checkpoint(restore_file, model, optimizer)
# checkpoint = utils.load_checkpoint(restore_file, model, optimizer)
best_val_acc = checkpoint['best_val_acc']
params.current_epoch = checkpoint['epoch']
print('best_val_acc=', best_val_acc)
print(optimizer.state_dict()['param_groups'][0]['lr'],
checkpoint['epoch'])
train_confusion_logger = VisdomLogger('heatmap',
port=port,
opts={
'title': params.experiment_path +
'train_Confusion matrix',
'columnnames': columnnames,
'rownames': rownames
},
env='Test')
test_confusion_logger = VisdomLogger('heatmap',
port=port,
opts={
'title': params.experiment_path +
'test_Confusion matrix',
'columnnames': columnnames,
'rownames': rownames
},
env='Test')
diff_confusion_logger = VisdomLogger('heatmap',
port=port,
opts={
'title': params.experiment_path +
'diff_Confusion matrix',
'columnnames': columnnames,
'rownames': rownames
},
env='Test')
# Evaluate for one epoch on validation set
# model.train()
model.eval()
train_metrics, train_confusion_meter, train_middle_val, train_middle_label = evaluate(
model, loss_fn, train_dataloader, metrics, params, logger)
train_confusion_logger.log(train_confusion_meter.value())
model.eval()
val_metrics, test_confusion_meter, test_middle_val, test_middle_label = evaluate(
model, loss_fn, val_dataloader, metrics, params, logger)
test_confusion_logger.log(test_confusion_meter.value())
diff_confusion_meter = train_confusion_meter.value(
) - test_confusion_meter.value()
diff_confusion_logger.log(diff_confusion_meter)
#---------------------------DWnet Testing------------------------------------------
lr = .001
C = 2**(-30)
s = 0.7
N2 = 1 # Number of windows of feature nodes
N3 = 7250 # Number of enhancement nodes
epochs = 10 # Number of epochs
en_list = []
test_acc_list = []
test_time_list = []
#FeaturesPerWindow = []
train_label = train_middle_label[0].reshape([64, 1])
print('train_label:', train_label.shape, train_middle_label[1].shape)
for i in range(len(train_middle_label) - 1):
train_label = np.vstack(
(train_label, np.reshape(train_middle_label[i + 1], [-1, 1])))
print('Shape of train label is:', train_label.shape)
np.savetxt('DWnet_result/train_label.txt', train_label)
Y = np.zeros([train_label.shape[0], 60])
for i in range(train_label.shape[0]):
Y[i, train_label[i]] = 1
np.savetxt('DWnet_result/Y.txt', Y)
test_label = np.reshape(test_middle_label[0], [64, 1])
for i in range(len(test_middle_label) - 1):
test_label = np.vstack(
(test_label, np.reshape(test_middle_label[i + 1], [-1, 1])))
print('Shape of test label is:', test_label.shape)
#print(train_middle_val[0])
np.savetxt('DWnet_result/test_label.txt', test_label)
Y_TEST = np.zeros([test_label.shape[0], 60])
for i in range(test_label.shape[0]):
Y_TEST[i, test_label[i]] = 1
np.savetxt('DWnet_result/Y_TEST.txt', Y_TEST)
epoch = 10
for en in range(epoch):
print('Enhancement epochs at: ', en,
'; The number of enhancement nodes is: ', N3)
# N3 = N3 + 50
FeaturesPerWindow = []
train_start_time = time.time()
for i in range(N2):
print(type(train_middle_val), len(train_middle_val),
train_middle_val[0].shape)
train_val = train_middle_val[0]
for elem in range(len(train_middle_val) - 1):
train_val = np.vstack((train_val, train_middle_val[elem + 1]))
# print(train_val)
print('Train_val shape is:', train_val.shape)
FeaturesExtract = train_val
print('FeaturesExtract shape is: ', FeaturesExtract.shape)
FeaturesPerWindow.append(FeaturesExtract)
N1 = FeaturesPerWindow[0].shape[1]
FeaturesPerWindow = np.array(FeaturesPerWindow)
print('Train features shape is: ', FeaturesPerWindow.shape)
FeaturesPerWindow = np.reshape(
FeaturesPerWindow,
newshape=[-1, FeaturesPerWindow[0].shape[1] * N2])
Features_bias = np.hstack([
FeaturesPerWindow, 0.1 * np.ones(((FeaturesPerWindow.shape[0]), 1))
])
if N1 * N2 >= N3:
# random.seed(67797325)
random.seed(int(time.time()))
wh = La.orth(2 * rnd.randn(N2 * N1 + 1, N3)) - 1
else:
# random.seed(67797325)
random.seed(int(time.time()))
wh = La.orth((2 * rnd.randn(N2 * N1 + 1, N3).T) - 1).T
print(wh)
print(wh.shape)
EnhancementFeature = np.dot(Features_bias, wh)
print(EnhancementFeature.shape)
l2 = s / np.max(EnhancementFeature)
EnhancementFeature = tansig(EnhancementFeature * l2)
FeatureLayerOutput = np.hstack([FeaturesPerWindow, EnhancementFeature])
beta = pinv(FeatureLayerOutput, C)
print(FeatureLayerOutput.shape)
print(beta.shape)
OutputWeights = np.dot(beta, Y)
print(OutputWeights.shape)
xx = np.dot(FeatureLayerOutput, OutputWeights)
TrainingAccuracy = show_accuracy(xx, Y)
train_end_time = time.time()
train_time = train_end_time - train_start_time
print('Training time is' + str(train_time) + 's')
print('Training accurate is', TrainingAccuracy * 100, '%')
'''Testing Process'''
test_start_time = time.time()
TestFeatures = []
for j in range(N2):
test_val = test_middle_val[0]
for elem in range(len(test_middle_val) - 1):
test_val = np.vstack((test_val, test_middle_val[elem + 1]))
test_pred = test_val
TestFeatures.append(test_pred)
TestFeatures = np.array(TestFeatures)
TestFeatures = np.reshape(TestFeatures,
newshape=[-1, TestFeatures[0].shape[1] * N2])
print(TestFeatures.shape)
'''Test Enhancement Nodes'''
TestFeaturesBias = np.hstack(
[TestFeatures, 0.1 * np.ones(((TestFeatures.shape[0]), 1))])
TestEnhancementFeatures = np.dot(TestFeaturesBias, wh)
TestEnhancementFeatures = tansig(TestEnhancementFeatures * l2)
FinalFeatures = np.hstack([TestFeatures, TestEnhancementFeatures])
TestOutput = np.dot(FinalFeatures, OutputWeights)
TestingAccuracy = show_accuracy(TestOutput, Y_TEST)
test_end_time = time.time()
test_time = test_end_time - test_start_time
np.savetxt('./DWnet_result/pred.txt', TestOutput)
np.savetxt('./DWnet_result/label.txt', Y_TEST)
print('Testing time is' + str(test_time) + 's')
print('Testing accurate is', TestingAccuracy * 100, '%')
en_list.append(N3)
test_time_list.append(test_time)
test_acc_list.append(TestingAccuracy)
# if (en+1) % 50 == 0:
np.savetxt('params/enhance_' + str(en + 1) + '.txt', en_list)
np.savetxt('params/test_time_' + str(en + 1) + '.txt', test_time_list)
np.savetxt('params/test_acc_' + str(en + 1) + '.txt', test_acc_list)
pass
def train_valid_loop(train_loader,
dev_loader,
test_loader,
args,
model,
fold=None):
# --------------------------------------------------------------------------
# TRAIN/VALID LOOP
logger.info('-' * 100)
stats = {
'timer': utils.Timer(),
'epoch': 0,
'best_valid': 0,
'best_epoch': 0,
'fold': fold
}
start_epoch = 0
if args.visdom:
# add visdom logger code
port = args.visdom_port
train_loss_logger = VisdomPlotLogger(
'line', port=port, opts={'title': f'{args.model_name} Train Loss'})
train_metric_logger = VisdomPlotLogger(
'line',
port=port,
opts={'title': f'{args.model_name} Train Class Accuracy'})
idx2label = {i: label for label, i in model.label_dict.items()}
label_names = [idx2label[i] for i in range(model.args.label_size)]
train_confusion_logger = VisdomLogger(
'heatmap',
port=port,
opts={
'title': f'{args.model_name} Train Confusion Matrix',
'columnnames': label_names,
'rownames': label_names
})
valid_metric_logger = VisdomPlotLogger(
'line',
port=port,
opts={'title': f'{args.model_name} Valid Class Accuracy'})
valid_confusion_logger = VisdomLogger(
'heatmap',
port=port,
opts={
'title': f'{args.model_name} Valid Confusion Matrix',
'columnnames': label_names,
'rownames': label_names
})
train_confusion_meter = tnt.meter.ConfusionMeter(model.args.label_size,
normalized=True)
valid_confusion_meter = tnt.meter.ConfusionMeter(model.args.label_size,
normalized=True)
else:
train_confusion_meter = None
valid_confusion_meter = None
try:
for epoch in range(start_epoch, args.num_epochs):
stats['epoch'] = epoch
# Train
loss = train(args, train_loader, model, stats)
stats['train_loss'] = loss
# Validate train
train_res, train_cfm = validate(
args,
train_loader,
model,
stats,
mode='train',
confusion_meter=train_confusion_meter)
for m in train_res:
stats['train_' + m] = train_res[m]
# Validate dev
val_res, valid_cfm = validate(
args,
dev_loader,
model,
stats,
mode='dev',
confusion_meter=valid_confusion_meter)
for m in train_res:
stats['dev_' + m] = val_res[m]
if args.visdom:
train_loss_logger.log(epoch, loss)
train_metric_logger.log(epoch, train_res[args.valid_metric])
train_confusion_logger.log(train_cfm)
valid_metric_logger.log(epoch, val_res[args.valid_metric])
valid_confusion_logger.log(valid_cfm)
train_confusion_meter.reset()
valid_confusion_meter.reset()
# Save best valid
if val_res[args.valid_metric] > stats['best_valid']:
logger.info(
colored(
f'Best valid: {args.valid_metric} = {val_res[args.valid_metric]*100:.2f}% ',
'yellow') +
colored(
f'(epoch {stats["epoch"]}, {model.updates} updates)',
'yellow'))
fold_info = f'.fold_{fold}' if fold is not None else ''
model.save(args.model_file + fold_info)
stats['best_valid'] = val_res[args.valid_metric]
stats['best_epoch'] = epoch
logger.info('-' * 100)
if args.stats_file:
with open(args.stats_file, 'w') as f:
out_stats = stats.copy()
out_stats['timer'] = out_stats['timer'].time()
if fold is None:
del out_stats['fold']
f.write(json.dumps(out_stats) + '\n')
if epoch - stats['best_epoch'] >= args.early_stopping:
logger.info(
colored(
f'No improvement for {args.early_stopping} epochs, stop training.',
'red'))
break
except KeyboardInterrupt:
logger.info(colored(f'User ended training. stop.', 'red'))
logger.info('Load best model...')
model = EntityClassifier.load(args.model_file + fold_info, args)
# device = torch.device(f"cuda:{args.gpu}" if args.cuda else "cpu")
# model.to(device)
model.cuda()
stats['epoch'] = stats['best_epoch']
if fold is not None:
mode = f'fold {fold} test'
else:
mode = 'test'
test_result, _ = validate(args, test_loader, model, stats, mode=mode)
return test_result
def test_only(model,
train_dataloader,
val_dataloader,
optimizer,
loss_fn,
metrics,
params,
model_dir,
logger,
restore_file=None):
# reload weights from restore_file if specified
if restore_file is not None:
logging.info("Restoring parameters from {}".format(restore_file))
checkpoint = utils.load_checkpoint(restore_file, model, optimizer)
best_val_acc = checkpoint['best_val_acc']
params.current_epoch = checkpoint['epoch']
print('best_val_acc=', best_val_acc)
print(optimizer.state_dict()['param_groups'][0]['lr'],
checkpoint['epoch'])
train_confusion_logger = VisdomLogger('heatmap',
port=port,
opts={
'title': params.experiment_path +
'train_Confusion matrix',
'columnnames': columnnames,
'rownames': rownames
},
env='Test')
test_confusion_logger = VisdomLogger('heatmap',
port=port,
opts={
'title': params.experiment_path +
'test_Confusion matrix',
'columnnames': columnnames,
'rownames': rownames
},
env='Test')
diff_confusion_logger = VisdomLogger('heatmap',
port=port,
opts={
'title': params.experiment_path +
'diff_Confusion matrix',
'columnnames': columnnames,
'rownames': rownames
},
env='Test')
# Evaluate for one epoch on validation set
# model.train()
model.eval()
train_metrics, train_confusion_meter, train_saved_feature = evaluate(
model, loss_fn, train_dataloader, metrics, params, logger)
train_confusion_logger.log(train_confusion_meter.value())
train_saved_feature_path = os.path.join(
feature_path,
params.data_type1 + '_' + params.data_type2 + "_train.pkl")
with open(train_saved_feature_path, 'wb') as f:
pickle.dump(train_saved_feature, f)
model.eval()
val_metrics, test_confusion_meter, test_saved_feature = evaluate(
model, loss_fn, val_dataloader, metrics, params, logger)
test_confusion_logger.log(test_confusion_meter.value())
diff_confusion_meter = train_confusion_meter.value(
) - test_confusion_meter.value()
diff_confusion_logger.log(diff_confusion_meter)
val_saved_feature_path = os.path.join(
feature_path, params.data_type1 + '_' + params.data_type2 + "_val.pkl")
with open(val_saved_feature_path, 'wb') as f:
pickle.dump(test_saved_feature, f)
# save the AIF metrics
print('save the AIF metrics ing--------------------')
AIF1 = model.adat_metrices1_c.data.cpu().numpy()
with open('saved_KI_AIF.pkl', 'wb') as f:
pickle.dump(AIF1, f)
AIF_r1 = model.adat_metrices_r1.data.cpu().numpy()
with open('saved_KI_AIF_lay_r1.pkl', 'wb') as f:
pickle.dump(AIF_r1, f)
AIF_r2 = model.adat_metrices3.data.cpu().numpy()
with open('saved_KI_AIF_lay_r2.pkl', 'wb') as f:
pickle.dump(AIF_r2, f)
print('save the AIF metrics end--------------------')
pass
