def __init__(self,
num_instances,
latent_dim,
tracing_steps,
has_params=False,
fit_single_srn=True,
use_unet_renderer=False,
freeze_networks=False):
super().__init__()
self.latent_dim = latent_dim
self.has_params = has_params
self.num_hidden_units_phi = 256
self.phi_layers = 4 # includes the in and out layers
self.rendering_layers = 5 # includes the in and out layers
self.sphere_trace_steps = tracing_steps
self.freeze_networks = freeze_networks
self.fit_single_srn = fit_single_srn
if self.fit_single_srn: # Fit a single scene with a single SRN (no hypernetworks)
self.phi = pytorch_prototyping.FCBlock(
hidden_ch=self.num_hidden_units_phi,
num_hidden_layers=self.phi_layers - 2,
in_features=6,
out_features=self.num_hidden_units_phi)
self.ray_marcher = custom_layers.Raymarcher2(
num_feature_channels=self.num_hidden_units_phi,
raymarch_steps=self.sphere_trace_steps)
self.phi_scene = PhiScene()
if use_unet_renderer:
self.pixel_generator = custom_layers.DeepvoxelsRenderer(
nf0=32,
in_channels=self.num_hidden_units_phi,
input_resolution=128,
img_sidelength=128)
else:
self.pixel_generator = pytorch_prototyping.FCBlock(
hidden_ch=self.num_hidden_units_phi,
num_hidden_layers=self.rendering_layers - 1,
in_features=self.num_hidden_units_phi,
out_features=3,
outermost_linear=True)
if self.freeze_networks:
all_network_params = (list(self.pixel_generator.parameters()) +
list(self.ray_marcher.parameters()) +
list(self.hyper_phi.parameters()))
for param in all_network_params:
param.requires_grad = False
# Losses
self.l2_loss = nn.MSELoss(reduction="mean")
# List of logs
self.logs = list()
print(self)
print("Number of parameters:")
util.print_network(self)
def define_tsnet(name, num_class, depth, cuda=True):
if name == 'resnet':
net = ResNet(num_class, depth)
elif name == 'RN_dml':
net = ResNet_DML(num_class, depth)
else:
raise Exception('model name does not exist.')
if cuda:
net = torch.nn.DataParallel(net).cuda()
util.print_network(net, name)
return net
def define_tsnet(name, num_class, cuda=True):
if name == 'resnet20':
net = resnet20(num_class=num_class)
elif name == 'resnet110':
net = resnet110(num_class=num_class)
else:
raise Exception('model name does not exist.')
if cuda:
net = torch.nn.DataParallel(net).cuda()
util.print_network(net)
return net
def __init__(self,
latent_dim,
tracing_steps,
has_params=False,
fit_single_srn=True,
freeze_networks=False):
super().__init__()
self.latent_dim = latent_dim
self.has_params = has_params
self.num_hidden_units_phi = 256
self.phi_layers = 4 # includes the in and out layers
self.rendering_layers = 5 # includes the in and out layers
self.sphere_trace_steps = tracing_steps
self.freeze_networks = freeze_networks
self.fit_single_srn = fit_single_srn
self.phi2 = pytorch_prototyping.FCBlock(
hidden_ch=self.num_hidden_units_phi,
num_hidden_layers=self.phi_layers - 2,
in_features=3,
out_features=self.num_hidden_units_phi)
self.ray_marcher = DepthCompute()
self.phi1 = PhiScene()
self.pixel_generator = pytorch_prototyping.FCBlock(
hidden_ch=self.num_hidden_units_phi,
num_hidden_layers=self.rendering_layers - 1,
in_features=self.num_hidden_units_phi,
out_features=3,
outermost_linear=True)
# Losses
self.l2_loss = nn.MSELoss(reduction="mean")
# List of logs
self.logs = list()
print(self)
print("Number of parameters:")
util.print_network(self)
def __init__(self, opt):
self.opt = opt
self.gpu_ids = opt.gpu_ids
self.is_train = opt.is_train
self.device = torch.device('cuda:{}'.format(
self.gpu_ids[0])) if self.gpu_ids else torch.device('cpu')
self.save_dir = join(opt.checkpoints_dir, opt.name)
self.optimizer = None
self.edge_features = None
self.labels = None
self.mesh = None
self.soft_label = None
self.loss = None
#
self.nclasses = opt.nclasses
# load/define networks
self.net = define_classifier(opt.input_nc, opt.ncf, opt.ninput_edges,
opt.nclasses, opt, self.gpu_ids, opt.arch,
opt.init_type, opt.init_gain)
self.net.train(self.is_train)
self.criterion = define_loss(opt).to(self.device)
if self.is_train:
if opt.optim == 'RMSprop':
self.optimizer = torch.optim.RMSprop(
self.net.parameters(),
lr=opt.lr,
weight_decay=opt.reg_weight)
else:
self.optimizer = torch.optim.Adam(self.net.parameters(),
lr=opt.lr,
betas=(opt.beta1, 0.999))
self.scheduler = get_scheduler(self.optimizer, opt)
print_network(self.net)
if not self.is_train or opt.continue_train:
self.load_network(opt.which_epoch)
def define_tsnet(name, num_class, cuda=True):
if name == 'resnet20':
net = resnet20(num_class=num_class)
elif name == 'resnet32':
net = resnet32(num_class=num_class)
elif name == 'resnet110':
net = resnet110(num_class=num_class)
elif name == 'resnet1202':
net = resnet1202(num_class=num_class)
elif name == 'resnext29_16_64':
net = resnext29_16_64(num_class=num_class)
elif name == 'resnext29_8_64':
net = resnext29_8_64(num_class=num_class)
elif name == 'densenetBC100':
net = densenet.DenseNet100(num_classes=num_class)
elif name == 'densenetBC250':
net = densenet.DenseNet250(num_classes=num_class)
else:
raise Exception('model name does not exist.')
if cuda:
net = torch.nn.DataParallel(net).cuda()
util.print_network(net)
return net
render_net = nn.DataParallel(render_net)
interpolater = nn.DataParallel(interpolater)
# set to training mode
texture_mapper.train()
render_net.train()
interpolater.train()
# collect all networks
part_list = [texture_mapper_module, render_net_module]
part_name_list = ['texture_mapper', 'render_net']
print("*" * 100)
print("Number of parameters:")
print("texture mapper:")
opt.num_params_texture_mapper = util.print_network(texture_mapper)
print("render net:")
opt.num_params_render_net = util.print_network(render_net)
print("*" * 100)
def main():
print('Start buffering data for training views...')
view_dataset.buffer_all()
view_dataloader = DataLoader(view_dataset,
batch_size=opt.batch_size,
shuffle=True,
num_workers=8)
print('Start buffering data for validation views...')
view_val_dataset.buffer_all()
def define_translator(k, cuda=True):
net = translator(k)
if cuda:
net = torch.nn.DataParallel(net).cuda()
util.print_network(net)
return net
def define_paraphraser(k, cuda=True):
net = paraphraser(k)
if cuda:
net = torch.nn.DataParallel(net).cuda()
util.print_network(net)
return net
def __init__(self,
img_sidelength,
lifting_img_dims,
frustrum_img_dims,
grid_dims,
num_grid_feats=64,
nf0=64,
use_occlusion_net=True):
''' Initializes the DeepVoxels model.
:param img_sidelength: The sidelength of the input images (for instance 512)
:param lifting_img_dims: The dimensions of the feature map to be lifted.
:param frustrum_img_dims: The dimensions of the canonical view volume that DeepVoxels are resampled to.
:param grid_dims: The dimensions of the deepvoxels grid.
:param grid_dims: The number of featres in the outermost layer of U-Nets.
:param use_occlusion_net: Whether to use the OcclusionNet or not.
'''
super().__init__()
self.use_occlusion_net = use_occlusion_net
self.grid_dims = grid_dims
self.norm = nn.BatchNorm2d
self.lifting_img_dims = lifting_img_dims
self.frustrum_img_dims = frustrum_img_dims
self.grid_dims = grid_dims
# The frustrum depth is the number of voxels in the depth dimension of the canonical viewing volume.
# It's calculated as the length of the diagonal of the DeepVoxels grid.
self.frustrum_depth = int(np.ceil(np.sqrt(3) * grid_dims[-1]))
self.nf0 = nf0 # Number of features to use in the outermost layer of all U-Nets
self.n_grid_feats = num_grid_feats # Number of features in the DeepVoxels grid.
self.occnet_nf = 4 # Number of features to use in the 3D unet of the occlusion subnetwork
# Feature extractor is an asymmetric UNet: Straight downsampling to 64x64, then a UNet with skip connections
self.feature_extractor = FeatureExtractor(nf0=self.nf0,
out_channels=self.n_grid_feats,
input_resolution=img_sidelength,
output_sidelength=self.frustrum_img_dims[0])
# Rendering net is an asymmetric UNet: UNet with skip connections and then straight upsampling
self.rendering_net = RenderingNet(nf0=self.nf0,
in_channels=self.n_grid_feats,
input_resolution=self.frustrum_img_dims[0],
img_sidelength=img_sidelength)
if self.use_occlusion_net:
self.occlusion_net = OcclusionNet(nf0=self.n_grid_feats,
occnet_nf=self.occnet_nf,
frustrum_dims=[self.frustrum_img_dims[0], self.frustrum_img_dims[1],
self.frustrum_depth])
print(self.occlusion_net)
else:
self.depth_collapse_net = nn.Sequential(
Conv2dSame(self.n_grid_feats * self.frustrum_depth,
out_channels=self.nf0 * self.grid_dims[-1] // 2,
kernel_size=3,
bias=False),
self.norm(self.nf0 * self.grid_dims[-1] // 2),
nn.ReLU(True),
Conv2dSame(self.nf0 * self.grid_dims[-1] // 2,
out_channels=self.nf0 * self.grid_dims[-1] // 8,
kernel_size=3,
bias=False),
self.norm(self.nf0 * self.grid_dims[-1] // 8),
nn.ReLU(True),
Conv2dSame(self.nf0 * self.grid_dims[-1] // 8,
out_channels=self.nf0,
kernel_size=3,
bias=False),
self.norm(self.nf0),
nn.ReLU(True),
)
print(self.frustrum_collapse_net)
# The deepvoxels grid is registered as a buffer - meaning, it is safed together with model parameters, but is
# not trainable.
self.register_buffer("deepvoxels",
torch.zeros(
(1, self.n_grid_feats, self.grid_dims[0], self.grid_dims[1], self.grid_dims[2])))
self.integration_net = IntegrationNet(self.n_grid_feats,
use_dropout=True,
coord_conv=True,
per_feature=False,
grid_dim=grid_dims[-1])
self.inpainting_net = Unet3d(in_channels=self.n_grid_feats + 3,
out_channels=self.n_grid_feats,
num_down=2,
nf0=self.n_grid_feats,
max_channels=4 * self.n_grid_feats)
print(100 * "*")
print("inpainting_net")
util.print_network(self.inpainting_net)
print(self.inpainting_net)
print("rendering net")
util.print_network(self.rendering_net)
print(self.rendering_net)
print("feature extraction net")
util.print_network(self.feature_extractor)
print(self.feature_extractor)
print(100 * "*")
# Coordconv volumes
coord_conv_volume = np.mgrid[-self.grid_dims[0] // 2:self.grid_dims[0] // 2,
-self.grid_dims[1] // 2:self.grid_dims[1] // 2,
-self.grid_dims[2] // 2:self.grid_dims[2] // 2]
coord_conv_volume = np.stack(coord_conv_volume, axis=0).astype(np.float32)
coord_conv_volume = coord_conv_volume / self.grid_dims[0]
self.coord_conv_volume = torch.Tensor(coord_conv_volume).float().cuda()[None, :, :, :, :]
start = checkpoint['epoch'] + 1
else:
G_A = model.Generator(config)
G_B = model.Generator(config)
D_A = model.Discriminator(config)
D_B = model.Discriminator(config)
start = 1
if use_cuda:
G_A = G_A.cuda()
G_B = G_B.cuda()
D_A = D_A.cuda()
D_B = D_B.cuda()
cudnn.benchmark = True
util.print_network(G_A)
util.print_network(D_A)
G_A.train()
G_B.train()
D_A.train()
D_B.train()
MSE_Loss = torch.nn.MSELoss()
L1_Loss = torch.nn.L1Loss()
G_A_Optimizer = Adam(G_A.parameters(), lr=config.learning_rate, betas=(0.5, 0.999))
G_B_Optimizer = Adam(G_B.parameters(), lr=config.learning_rate, betas=(0.5, 0.999))
D_A_Optimizer = Adam(D_A.parameters(), lr=config.learning_rate, betas=(0.5, 0.999))
D_B_Optimizer = Adam(D_B.parameters(), lr=config.learning_rate, betas=(0.5, 0.999))
a_loader = util.get_loader(config, config.dataset_A + '/train')
def __init__(self,
num_instances,
latent_dim,
tracing_steps,
has_params=False,
fit_single_srn=False,
use_unet_renderer=False,
freeze_networks=False):
super().__init__()
self.latent_dim = latent_dim
self.has_params = has_params
self.num_hidden_units_phi = 256
self.phi_layers = 4 # includes the in and out layers
self.rendering_layers = 5 # includes the in and out layers
self.sphere_trace_steps = tracing_steps
self.freeze_networks = freeze_networks
self.fit_single_srn = fit_single_srn
if self.fit_single_srn: # Fit a single scene with a single SRN (no hypernetworks)
self.phi = pytorch_prototyping.FCBlock(hidden_ch=self.num_hidden_units_phi,
num_hidden_layers=self.phi_layers - 2,
in_features=3,
out_features=self.num_hidden_units_phi)
else:
# Auto-decoder: each scene instance gets its own code vector z
self.latent_codes = nn.Embedding(num_instances, latent_dim).cuda()
nn.init.normal_(self.latent_codes.weight, mean=0, std=0.01)
self.hyper_phi = hyperlayers.HyperFC(hyper_in_ch=self.latent_dim,
hyper_num_hidden_layers=1,
hyper_hidden_ch=self.latent_dim,
hidden_ch=self.num_hidden_units_phi,
num_hidden_layers=self.phi_layers - 2,
in_ch=3,
out_ch=self.num_hidden_units_phi)
self.ray_marcher = custom_layers.Raymarcher(num_feature_channels=self.num_hidden_units_phi,
raymarch_steps=self.sphere_trace_steps)
if use_unet_renderer:
self.pixel_generator = custom_layers.DeepvoxelsRenderer(nf0=32, in_channels=self.num_hidden_units_phi,
input_resolution=128, img_sidelength=128)
else:
self.pixel_generator = pytorch_prototyping.FCBlock(hidden_ch=self.num_hidden_units_phi,
num_hidden_layers=self.rendering_layers - 1,
in_features=self.num_hidden_units_phi,
out_features=3,
outermost_linear=True)
if self.freeze_networks:
all_network_params = (self.pixel_generator.parameters()
+ self.ray_marcher.parameters()
+ self.hyper_phi.parameters())
for param in all_network_params:
param.requires_grad = False
# Losses
self.l2_loss = nn.MSELoss(reduction="mean")
# List of logs
self.logs = list()
print(self)
print("Number of parameters:")
util.print_network(self)
fake_fixed = net_g(fixed)
state = {
'net_g': net_g,
'net_d': net_d,
'epoch': epoch,
}
if not os.path.isdir('checkpoint'):
os.mkdir('checkpoint')
torch.save(state, './checkpoint/dcgan.nn')
if (epoch > 15 and epoch % config.save_frequency == 0) or (epoch <= 15):
save_image(
util.denorm(fake_fixed).data,
'%s/fixed_%d.jpg' % (config.result_path, epoch))
save_image(
util.denorm(fake).data,
'%s/fake_%d.jpg' % (config.result_path, epoch))
print('-- Models and test images saved.')
epoch_time = (time.time() - epoch_time) / 60
time_remain = (epoch_time * (config.final_epoch - epoch)) / 60
print('-- Epoch completed. Epoch Time: %.2f min, Time Est: %.2f hour.' %
(epoch_time, time_remain))
if not os.path.exists(config.result_path):
os.makedirs(config.result_path)
util.print_network(net_g)
util.print_network(net_d)
print('-- Start Training')
for epoch in range(start, config.final_epoch):
train(start, epoch, config)
def main(args):
# select benchmark
c = args.data
# load data
inc_feat, feats, times, censored_train_num, censored_test_num, uncensored_train_num, uncensored_test_num = read_ic(
args)
# initialize model parameter
model = CNFNet(args)
optimizer = optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
print_network(model)
uncensor_loader = DataLoader(uncensored_train_num,
batch_size=args.batch_size,
shuffle=True,
drop_last=True)
censor_loader = DataLoader(censored_train_num,
batch_size=args.batch_size,
shuffle=True,
drop_last=True)
args.num_instance = len(times)
def train(model):
for epoch in range(args.epochs):
for step, un_ids in enumerate(uncensor_loader):
t = time.time()
c_ids = next(iter(censor_loader))
model.train()
output = model(
itemgetter(*un_ids)(inc_feat),
itemgetter(*c_ids)(inc_feat), feats[un_ids], feats[c_ids])
loss_train, ret_str = model.combo_loss(
output, times[un_ids].view(-1, 1))
optimizer.zero_grad()
loss_train.backward()
optimizer.step()
print('Epoch: {:02d}/{:04d}'.format(epoch, step + 1),
'loss_train: {:.4f}'.format(loss_train.item()),
'time: {:.4f}s'.format(time.time() - t), ret_str)
############
# Train model
############
t_total = time.time()
train(model)
# print training info
# plot_metric(range(len(train_loss)), train_loss, eval_loss,
# '{}_{}_{}_train'.format(c, args.sa_loss, args.censor_ratio),
# '{}_{}_{}_eval'.format(c, args.sa_loss, args.censor_ratio))
print("Optimization Finished!")
print("Total time elapsed: {:.4f}s".format(time.time() - t_total))
def test(censored_test_num, uncensored_test_num):
model.eval()
test_un_ids = list(chunks(uncensored_test_num, args.batch_size))
test_c_ids = list(chunks(censored_test_num, args.batch_size))
output_un = padding_and_trim_reg(model, test_un_ids, inc_feat, feats,
args.batch_size)
output_c_pos = padding_and_trim_class(model, test_c_ids, inc_feat,
feats, args.batch_size)
output_c_neg = padding_and_trim_class(model, test_un_ids, inc_feat,
feats, args.batch_size)
prefix = "{}_{}_{}_{}_{}".format('DSAG', c, args.thres, args.alpha,
args.beta)
# uncensor test
time_un_test = times[torch.cat(test_un_ids)].data.numpy()
time_un_pred = output_un
# print(time_un_test, time_un_pred}
print("test loss: {} ".format(
mean_squared_error(time_un_pred, time_un_test)))
plot_metric(range(time_un_test.size), time_un_pred, time_un_test,
'{}_pred'.format(prefix), '{}_real'.format(prefix))
pk.dump(print_metric(time_un_test, time_un_pred),
open('{}_metric.pk'.format(prefix), 'wb'))
pk.dump(time_un_test, open('{}_test_list.pk'.format(prefix), 'wb'))
pk.dump(time_un_pred, open('{}_pred_list.pk'.format(prefix), 'wb'))
# censor test
censor_size, uncensor_size = censored_test_num.shape[
0], uncensored_test_num.shape[0]
pred = np.concatenate([output_c_pos, output_c_neg])
censor_labels = torch.cat(
(torch.FloatTensor([0]).repeat(uncensor_size),
torch.FloatTensor([1]).repeat(censor_size)))
confuse_mat = confusion_matrix(pred, censor_labels)
class_report = classification_report(pred, censor_labels)
pk.dump(confuse_mat, open('{}_confusion_matrix.pk'.format(prefix),
'wb'))
pk.dump(class_report, open('{}_class_report.pk'.format(prefix), 'wb'))
############
# Testing
############
test(censored_test_num, uncensored_test_num)
pretrained (bool): If True, returns a model pre-trained.
"""
model = VGG(depth=16, **kwargs)
if pretrained:
model.load_state_dict((torch.load(path))['state_dict'])
return model
def vgg19(pretrained=False, path=None, **kwargs):
"""
Constructs a VGG19 model.
Args:
pretrained (bool): If True, returns a model pre-trained.
"""
model = VGG(depth=19, **kwargs)
if pretrained:
model.load_state_dict((torch.load(path))['state_dict'])
return model
if __name__ == '__main__':
import torch
input = torch.FloatTensor(1, 3, 32, 32)
net = VGG()
from util import print_network
print_network(net, name='1')
out = net(input)
print(out.size())
criterionGAN = GANLoss().to(device)
discriminator = PatchDiscriminator(input_nc=3).to(device)
# Optimizers
optimizerD = torch.optim.Adam(discriminator.parameters(), lr=opt.lr)
optimizerG = torch.optim.Adam(model.parameters(), lr=opt.lr)
print("*" * 100)
print("Frustrum depth")
print(frustrum_depth)
print("Near plane")
print(near_plane)
print("Intrinsic")
print(lift_intrinsic)
print("Number of discriminator parameters:")
util.print_network(discriminator)
print("Number of generator parameters:")
util.print_network(model)
print("*" * 100)
def train():
discriminator.train()
model.train()
if opt.checkpoint:
util.custom_load(model, opt.checkpoint, discriminator)
# Create the training dataset loader
train_dataset = NovelViewTriplets(root_dir=opt.data_root,
img_size=input_image_dims,
def __init__(self,
img_sidelength,
lifting_img_dims,
frustrum_img_dims,
grid_dims,
num_grid_feats=64,
nf0=64,
use_occlusion_net=True):
''' Initializes the DeepVoxels model.
:param img_sidelength: The sidelength of the input images (for instance 512)
:param lifting_img_dims: The dimensions of the feature map to be lifted.
:param frustrum_img_dims: The dimensions of the canonical view volume that DeepVoxels are resampled to.
:param grid_dims: The dimensions of the deepvoxels grid.
:param grid_dims: The number of featres in the outermost layer of U-Nets.
:param use_occlusion_net: Whether to use the OcclusionNet or not.
'''
super().__init__()
self.use_occlusion_net = use_occlusion_net
self.grid_dims = grid_dims
self.norm = nn.BatchNorm2d
self.lifting_img_dims = lifting_img_dims
self.frustrum_img_dims = frustrum_img_dims
self.grid_dims = grid_dims
# The frustrum depth is the number of voxels in the depth dimension of the canonical viewing volume.
# It's calculated as the length of the diagonal of the DeepVoxels grid.
self.frustrum_depth = int(np.ceil(1.5 * grid_dims[-1]))
self.nf0 = nf0 # Number of features to use in the outermost layer of all U-Nets
self.n_grid_feats = num_grid_feats # Number of features in the DeepVoxels grid.
self.occnet_nf = 4 # Number of features to use in the 3D unet of the occlusion subnetwork
num_downs = util.num_divisible_by_2(img_sidelength) - 1
# Feature extractor is an asymmetric UNet: Straight downsampling to 64x64, then a UNet with skip connections
self.feature_extractor = nn.Sequential(
# DownsamplingNet([self.nf0 * (2 ** i) for i in range(num_downs - 5)],
# in_channels=3,
# use_dropout=False,
# norm=self.norm),
# AlexNetConv4()
# Unet(in_channels=self.nf0 * (2 ** (num_downs - 6)),
# out_channels=self.n_grid_feats,
# nf0=self.nf0 * (2 ** (num_downs - 6)),
# use_dropout=False,
# max_channels=8*self.nf0,
# num_down=5,
# norm=self.norm)
Conv2dPad(in_channels = 256, out_channels = 64, kernel_size = 1, padding_size = 4)
)
# Rendering net is an asymmetric UNet: UNet with skip connections and then straight upsampling
self.rendering_net = nn.Sequential(
Unet(in_channels=self.n_grid_feats,
out_channels=4 * self.nf0,
use_dropout=True,
dropout_prob=0.1,
nf0=self.nf0 * (2 ** (num_downs - 6)),
max_channels=8 * self.nf0,
num_down=5,
norm=self.norm), # from 64 to 2 and back
UpsamplingNet([4 * self.nf0, self.nf0] + max(0, num_downs - 7) * [self.nf0],
in_channels=4 * self.nf0, # 4*self.nf0
use_dropout=True,
dropout_prob=0.1,
norm=self.norm,
upsampling_mode='transpose'),
Conv2dSame(self.nf0, out_channels=self.nf0 // 2, kernel_size=3, bias=False),
nn.BatchNorm2d(self.nf0 // 2),
nn.ReLU(True),
Conv2dSame(self.nf0 // 2, out_channels=3, kernel_size=3),
nn.Tanh()
)
if self.use_occlusion_net:
self.occlusion_net = OcclusionNet(nf0=self.n_grid_feats,
occnet_nf=self.occnet_nf,
frustrum_dims=[self.frustrum_img_dims[0], self.frustrum_img_dims[1],
self.frustrum_depth])
print(self.occlusion_net)
else:
self.depth_collapse_net = nn.Sequential(
Conv2dSame(self.n_grid_feats * self.frustrum_depth,
out_channels=self.nf0 * self.grid_dims[-1] // 2,
kernel_size=3,
bias=False),
self.norm(self.nf0 * self.grid_dims[-1] // 2),
nn.ReLU(True),
Conv2dSame(self.nf0 * self.grid_dims[-1] // 2,
out_channels=self.nf0 * self.grid_dims[-1] // 8,
kernel_size=3,
bias=False),
self.norm(self.nf0 * self.grid_dims[-1] // 8),
nn.ReLU(True),
Conv2dSame(self.nf0 * self.grid_dims[-1] // 8,
out_channels=self.nf0,
kernel_size=3,
bias=False),
self.norm(self.nf0),
nn.ReLU(True),
)
print(self.frustrum_collapse_net)
# The deepvoxels grid is registered as a buffer - meaning, it is safed together with model parameters, but is
# not trainable.
self.register_buffer("deepvoxels",
torch.zeros(
(1, self.n_grid_feats, self.grid_dims[0], self.grid_dims[1], self.grid_dims[2])))
self.integration_net = IntegrationNet(self.n_grid_feats,
use_dropout=True,
coord_conv=True,
per_feature=False,
grid_dim=grid_dims[-1])
self.inpainting_net = Unet3d(in_channels=self.n_grid_feats + 3,
out_channels=self.n_grid_feats,
num_down=2,
nf0=self.n_grid_feats,
max_channels=4 * self.n_grid_feats)
print(100 * "*")
print("inpainting_net")
util.print_network(self.inpainting_net)
print(self.inpainting_net)
print("rendering net")
util.print_network(self.rendering_net)
print(self.rendering_net)
print("feature extraction net")
util.print_network(self.feature_extractor)
print(self.feature_extractor)
print(100 * "*")
# Coordconv volumes
coord_conv_volume = np.mgrid[-self.grid_dims[0] // 2:self.grid_dims[0] // 2,
-self.grid_dims[1] // 2:self.grid_dims[1] // 2,
-self.grid_dims[2] // 2:self.grid_dims[2] // 2]
coord_conv_volume = np.stack(coord_conv_volume, axis=0).astype(np.float32)
coord_conv_volume = coord_conv_volume / self.grid_dims[0]
# self.coord_conv_volume = torch.Tensor(coord_conv_volume).float().cuda()[None, :, :, :, :]
self.coord_conv_volume = torch.Tensor(coord_conv_volume).float()[None, :, :, :, :]
print('-- Resuming From Checkpoint')
assert os.path.isdir(
'checkpoint'), '-- Error: No checkpoint directory found!'
checkpoint = torch.load('./checkpoint/music.nn')
net = checkpoint['net']
best_acc = checkpoint['acc']
start_epoch = checkpoint['epoch']
else:
net = model.net(config)
start_epoch = 1
if use_cuda:
net = net.cuda()
cudnn.benchmark = True
util.print_network(net)
net.train()
best_acc = 0
criterion = torch.nn.CrossEntropyLoss()
Optimizer = torch.optim.SGD(net.parameters(),
lr=config.learning_rate,
momentum=0.9)
trainloader = util.get_loader(config, './train')
testloader = util.get_loader(config, './test')
def train(epoch):
print('-- Current Training Epoch %d' % epoch)
net.train()
train_loss = 0
