def _compute_prob_feas(self, X: Tensor, means: Tensor, sigmas: Tensor) -> Tensor:
r"""Compute feasibility probability for each batch of X.
Args:
X: A `(b) x 1 x d`-dim Tensor of `(b)` t-batches of `d`-dim design
points each.
means: A `(b) x t`-dim Tensor of means.
sigmas: A `(b) x t`-dim Tensor of standard deviations.
Returns:
A `(b) x 1 x 1`-dim tensor of feasibility probabilities
Note: This function does case-work for upper bound, lower bound, and both-sided
bounds. Another way to do it would be to use 'inf' and -'inf' for the
one-sided bounds and use the logic for the both-sided case. But this
causes an issue with autograd since we get 0 * inf.
TODO: Investigate further.
"""
output_shape = list(X.shape)
output_shape[-1] = 1
prob_feas = torch.ones(output_shape, device=X.device, dtype=X.dtype)
if len(self.constraint_lower_inds) > 0:
normal_lower = _construct_dist(means, sigmas, self.constraint_lower_inds)
prob_l = 1 - normal_lower.cdf(self.constraint_lower)
prob_feas = prob_feas.mul(torch.prod(prob_l, dim=-1, keepdim=True))
if len(self.constraint_upper_inds) > 0:
normal_upper = _construct_dist(means, sigmas, self.constraint_upper_inds)
prob_u = normal_upper.cdf(self.constraint_upper)
prob_feas = prob_feas.mul(torch.prod(prob_u, dim=-1, keepdim=True))
if len(self.constraint_both_inds) > 0:
normal_both = _construct_dist(means, sigmas, self.constraint_both_inds)
prob_u = normal_both.cdf(self.constraint_both[:, 1])
prob_l = normal_both.cdf(self.constraint_both[:, 0])
prob_feas = prob_feas.mul(torch.prod(prob_u - prob_l, dim=-1, keepdim=True))
return prob_feas
def forward(self,inputs):
if isinstance(self.output_shape,int):
size = [self.output_shape]
else:
size = list(self.output_shape)
assert torch.prod(torch.LongTensor(list(inputs.size())[1:])).item() == torch.prod(torch.LongTensor(size)).item()
size = list(size)
size = tuple([-1] + size)
outputs = inputs.view(size)
return outputs
def iou_matrix(yx_min1, yx_max1, yx_min2, yx_max2, min=float(np.finfo(np.float32).eps)):
"""
Calculates the IoU of two lists of bounding boxes.
:author 申瑞珉 (Ruimin Shen)
:param yx_min1: The top left coordinates (y, x) of the first list (size [N1, 2]) of bounding boxes.
:param yx_max1: The bottom right coordinates (y, x) of the first list (size [N1, 2]) of bounding boxes.
:param yx_min2: The top left coordinates (y, x) of the second list (size [N2, 2]) of bounding boxes.
:param yx_max2: The bottom right coordinates (y, x) of the second list (size [N2, 2]) of bounding boxes.
:return: The matrix (size [N1, N2]) of the IoU.
"""
intersect_area = intersection_area(yx_min1, yx_max1, yx_min2, yx_max2)
area1 = torch.prod(yx_max1 - yx_min1, -1).unsqueeze(-1)
area2 = torch.prod(yx_max2 - yx_min2, -1).unsqueeze(-2)
union_area = torch.clamp(area1 + area2 - intersect_area, min=min)
return intersect_area / union_area
def vector_to_parameters(vec, parameters):
r"""Convert one vector to the parameters
Arguments:
vec (Variable): a single vector represents the parameters of a model.
parameters (Iterable[Variable]): an iterator of Variables that are the
parameters of a model.
"""
# Ensure vec of type Variable
if not isinstance(vec, Variable):
raise TypeError('expected torch.autograd.Variable, but got: {}'
.format(torch.typename(vec)))
# Flag for the device where the parameter is located
param_device = None
# Pointer for slicing the vector for each parameter
pointer = 0
for param in parameters:
# Ensure the parameters are located in the same device
param_device = _check_param_device(param, param_device)
# The length of the parameter
num_param = torch.prod(torch.LongTensor(list(param.size())))
# Slice the vector, reshape it, and replace the old data of the parameter
param.data = vec[pointer:pointer + num_param].view(param.size()).data
# Increment the pointer
pointer += num_param
def batch_iou_pair(yx_min1, yx_max1, yx_min2, yx_max2, min=float(np.finfo(np.float32).eps)):
"""
Pairwisely calculates the IoU of two lists (at the same size M) of bounding boxes for N independent batches.
:author 申瑞珉 (Ruimin Shen)
:param yx_min1: The top left coordinates (y, x) of the first lists (size [N, M, 2]) of bounding boxes.
:param yx_max1: The bottom right coordinates (y, x) of the first lists (size [N, M, 2]) of bounding boxes.
:param yx_min2: The top left coordinates (y, x) of the second lists (size [N, M, 2]) of bounding boxes.
:param yx_max2: The bottom right coordinates (y, x) of the second lists (size [N, M, 2]) of bounding boxes.
:return: The lists (size [N, M]) of the IoU.
"""
yx_min = torch.max(yx_min1, yx_min2)
yx_max = torch.min(yx_max1, yx_max2)
size = torch.clamp(yx_max - yx_min, min=0)
intersect_area = torch.prod(size, -1)
area1 = torch.prod(yx_max1 - yx_min1, -1)
area2 = torch.prod(yx_max2 - yx_min2, -1)
union_area = torch.clamp(area1 + area2 - intersect_area, min=min)
return intersect_area / union_area
def get_normal_loglik(x, mean, std, scale = False):
recon_losses = \
Normal(mean, std + 1e-8).log_prob(x)
if scale:
factor = torch.prod(torch.Tensor([x.size()])) / x.size()[0]
else:
factor = 1.0
return (recon_losses / factor).view(x.size(0), -1).sum(1)
def reset_parameters(self):
n_trainable_params, n_nontrainable_params = 0, 0
for p in self.model.parameters():
n_params = torch.prod(torch.tensor(p.shape)) # 参数数量
if p.requires_grad:
n_trainable_params += n_params
if len(p.shape) > 1:
self.opt.initializer(p) # 参数初始化
else:
n_nontrainable_params += n_params
print('n_trainable_params: {0}, n_nontrainable_params: {1}'.format(n_trainable_params, n_nontrainable_params))
def hook(module, input, output):
class_name = str(module.__class__).split('.')[-1].split("'")[0]
module_idx = len(summary)
m_key = '%s-%i' % (class_name, module_idx+1)
summary[m_key] = OrderedDict()
summary[m_key]['input_shape'] = list(input[0].size())
summary[m_key]['input_shape'][0] = -1
summary[m_key]['output_shape'] = list(output.size())
summary[m_key]['output_shape'][0] = -1
params = 0
if hasattr(module, 'weight'):
params += th.prod(th.LongTensor(list(module.weight.size())))
if module.weight.requires_grad:
summary[m_key]['trainable'] = True
else:
summary[m_key]['trainable'] = False
if hasattr(module, 'bias'):
params += th.prod(th.LongTensor(list(module.bias.size())))
summary[m_key]['nb_params'] = params
def hook(module, input, output):
class_name = str(module.__class__.__name__)
instance_index = 1
if class_name not in layer_instances:
layer_instances[class_name] = instance_index
else:
instance_index = layer_instances[class_name] + 1
layer_instances[class_name] = instance_index
layer_name = class_name + "_" + str(instance_index)
params = 0
if hasattr(module, "weight"):
weight_size = module.weight.data.size()
weight_params = torch.prod(torch.LongTensor(list(weight_size)))
params += weight_params.item()
if hasattr(module, "bias"):
try:
bias_size = module.bias.data.size()
bias_params = torch.prod(torch.LongTensor(list(bias_size)))
params += bias_params.item()
except:
pass
flops = "Not Available"
if class_name.find("Conv")!= -1 and hasattr(module,"weight") :
flops = (
torch.prod(torch.LongTensor(list(module.weight.data.size()))) * output.size(2) * output.size(
3)).item()
elif isinstance(module, nn.Linear):
flops = (torch.prod(torch.LongTensor(list(output.size()))) * input[0].size(1)).item()
summary.append(
ModuleDetails(name=layer_name, input_size=list(input[0].size()), output_size=list(output.size()),
num_parameters=params, multiply_adds=flops))
def fit_positive(rows, cols, yx_min, yx_max, anchors):
device_id = anchors.get_device() if torch.cuda.is_available() else None
batch_size, num, _ = yx_min.size()
num_anchors, _ = anchors.size()
valid = torch.prod(yx_min < yx_max, -1)
center = (yx_min + yx_max) / 2
ij = torch.floor(center)
i, j = torch.unbind(ij.long(), -1)
index = i * cols + j
anchors2 = anchors / 2
iou_matrix = utils.iou.torch.iou_matrix((yx_min - center).view(-1, 2), (yx_max - center).view(-1, 2), -anchors2, anchors2).view(batch_size, -1, num_anchors)
iou, index_anchor = iou_matrix.max(-1)
_positive = []
cells = rows * cols
for valid, index, index_anchor in zip(torch.unbind(valid), torch.unbind(index), torch.unbind(index_anchor)):
index, index_anchor = (t[valid] for t in (index, index_anchor))
t = utils.ensure_device(torch.ByteTensor(cells, num_anchors).zero_(), device_id)
t[index, index_anchor] = 1
_positive.append(t)
return torch.stack(_positive)
def forward(self,xlist,coordlist):
# xlist: n x k x 1x 96 x 96 x 96
# coordlist: n x k x 3 x 24 x 24 x 24
xsize = xlist.size()
corrdsize = coordlist.size()
xlist = xlist.view(-1,xsize[2],xsize[3],xsize[4],xsize[5])
coordlist = coordlist.view(-1,corrdsize[2],corrdsize[3],corrdsize[4],corrdsize[5])
noduleFeat,nodulePred = self.NoduleNet(xlist,coordlist)
nodulePred = nodulePred.contiguous().view(corrdsize[0],corrdsize[1],-1)
featshape = noduleFeat.size()#nk x 128 x 24 x 24 x24
centerFeat = self.pool(noduleFeat[:,:,featshape[2]/2-1:featshape[2]/2+1,
featshape[3]/2-1:featshape[3]/2+1,
featshape[4]/2-1:featshape[4]/2+1])
centerFeat = centerFeat[:,:,0,0,0]
out = self.dropout(centerFeat)
out = self.Relu(self.fc1(out))
out = torch.sigmoid(self.fc2(out))
out = out.view(xsize[0],xsize[1])
base_prob = torch.sigmoid(self.baseline)
casePred = 1-torch.prod(1-out,dim=1)*(1-base_prob.expand(out.size()[0]))
return nodulePred,casePred,out
def _parameter_and_gradient_statistics_to_tensorboard(self, # pylint: disable=invalid-name
epoch: int,
batch_grad_norm: float) -> None:
"""
Send the mean and std of all parameters and gradients to tensorboard, as well
as logging the average gradient norm.
"""
# Log parameter values to Tensorboard
for name, param in self._model.named_parameters():
self._tensorboard.add_train_scalar("parameter_mean/" + name,
param.data.mean(),
epoch)
self._tensorboard.add_train_scalar("parameter_std/" + name, param.data.std(), epoch)
if param.grad is not None:
if is_sparse(param.grad):
# pylint: disable=protected-access
grad_data = param.grad.data._values()
else:
grad_data = param.grad.data
# skip empty gradients
if torch.prod(torch.tensor(grad_data.shape)).item() > 0: # pylint: disable=not-callable
self._tensorboard.add_train_scalar("gradient_mean/" + name,
grad_data.mean(),
epoch)
self._tensorboard.add_train_scalar("gradient_std/" + name,
grad_data.std(),
epoch)
else:
# no gradient for a parameter with sparse gradients
logger.info("No gradient for %s, skipping tensorboard logging.", name)
# norm of gradients
if batch_grad_norm is not None:
self._tensorboard.add_train_scalar("gradient_norm",
batch_grad_norm,
epoch)
def test_prod(self):
x = Variable(torch.randn(1, 2, 3, 4), requires_grad=True)
self.assertONNX(lambda x: torch.prod(x), x)
def __init__(self, latent_dim: int, out_dim: Tuple[int]) -> None:
super(latent_to_features, self).__init__()
self.reshape_ = out_dim
self.fc = nn.Linear(latent_dim, torch.prod(tt(out_dim)).item())
def initialize(self, image, info: dict) -> dict:
# Initialize some stuff
self.frame_num = 1
if not self.params.has('device'):
self.params.device = 'cuda' if self.params.use_gpu else 'cpu'
# Initialize network
self.initialize_features()
# The DiMP network
self.net = self.params.net
# Time initialization
tic = time.time()
# Convert image
im = numpy_to_torch(image)
# Get target position and size
state = info['init_bbox']
self.pos = torch.Tensor(
[state[1] + (state[3] - 1) / 2, state[0] + (state[2] - 1) / 2])
self.target_sz = torch.Tensor([state[3], state[2]])
# Get object id
self.object_id = info.get('object_ids', [None])[0]
self.id_str = '' if self.object_id is None else ' {}'.format(
self.object_id)
# Set sizes
self.image_sz = torch.Tensor([im.shape[2], im.shape[3]])
sz = self.params.image_sample_size
sz = torch.Tensor([sz, sz] if isinstance(sz, int) else sz)
if self.params.get('use_image_aspect_ratio', False):
sz = self.image_sz * sz.prod().sqrt() / self.image_sz.prod().sqrt()
stride = self.params.get('feature_stride', 32)
sz = torch.round(sz / stride) * stride
self.img_sample_sz = sz
self.img_support_sz = self.img_sample_sz
# Set search area
search_area = torch.prod(self.target_sz *
self.params.search_area_scale).item()
self.target_scale = math.sqrt(
search_area) / self.img_sample_sz.prod().sqrt()
# Target size in base scale
self.base_target_sz = self.target_sz / self.target_scale
# Setup scale factors
if not self.params.has('scale_factors'):
self.params.scale_factors = torch.ones(1)
elif isinstance(self.params.scale_factors, (list, tuple)):
self.params.scale_factors = torch.Tensor(self.params.scale_factors)
# Setup scale bounds
self.min_scale_factor = torch.max(10 / self.base_target_sz)
self.max_scale_factor = torch.min(self.image_sz / self.base_target_sz)
# Extract and transform sample
init_backbone_feat = self.generate_init_samples(im)
# Initialize classifier
self.init_classifier(init_backbone_feat)
# Initialize IoUNet
if self.params.get('use_iou_net', True):
self.init_iou_net(init_backbone_feat)
out = {'time': time.time() - tic}
return out
def product(self, tensor_in, axis=None):
return torch.prod(tensor_in) if axis is None else torch.prod(
tensor_in, axis)
import torch
from main import Resnet50Classifier
if __name__ == "__main__":
net = Resnet50Classifier().to("cuda:0")
criterion = torch.nn.NLLLoss()
image = torch.randn((1, 3, 224, 224))
label = torch.LongTensor([10])
output = net(image.to("cuda:0")).cpu()
loss = criterion(output, label)
loss.backward()
total_nn_size = 0
for name, p in list(net.named_parameters()):
layer_size = (p.grad.element_size() * torch.prod(torch.LongTensor(list(p.grad.size())))).numpy()
total_nn_size += layer_size
print(f"NN size in mbytes: {total_nn_size * 1e-6}")
def test_resplit(self):
# resplitting with same axis, should leave everything unchanged
shape = (ht.MPI_WORLD.size, ht.MPI_WORLD.size)
data = ht.zeros(shape, split=None)
data.resplit_(None)
self.assertIsInstance(data, ht.DNDarray)
self.assertEqual(data.shape, shape)
self.assertEqual(data.lshape, shape)
self.assertEqual(data.split, None)
# resplitting with same axis, should leave everything unchanged
shape = (ht.MPI_WORLD.size, ht.MPI_WORLD.size)
data = ht.zeros(shape, split=1)
data.resplit_(1)
self.assertIsInstance(data, ht.DNDarray)
self.assertEqual(data.shape, shape)
self.assertEqual(data.lshape, (data.comm.size, 1))
self.assertEqual(data.split, 1)
# splitting an unsplit tensor should result in slicing the tensor locally
shape = (ht.MPI_WORLD.size, ht.MPI_WORLD.size)
data = ht.zeros(shape)
data.resplit_(-1)
self.assertIsInstance(data, ht.DNDarray)
self.assertEqual(data.shape, shape)
self.assertEqual(data.lshape, (data.comm.size, 1))
self.assertEqual(data.split, 1)
# unsplitting, aka gathering a tensor
shape = (ht.MPI_WORLD.size + 1, ht.MPI_WORLD.size)
data = ht.ones(shape, split=0)
data.resplit_(None)
self.assertIsInstance(data, ht.DNDarray)
self.assertEqual(data.shape, shape)
self.assertEqual(data.lshape, shape)
self.assertEqual(data.split, None)
# assign and entirely new split axis
shape = (ht.MPI_WORLD.size + 2, ht.MPI_WORLD.size + 1)
data = ht.ones(shape, split=0)
data.resplit_(1)
self.assertIsInstance(data, ht.DNDarray)
self.assertEqual(data.shape, shape)
self.assertEqual(data.lshape[0], ht.MPI_WORLD.size + 2)
self.assertTrue(data.lshape[1] == 1 or data.lshape[1] == 2)
self.assertEqual(data.split, 1)
# test sorting order of resplit
a_tensor = self.reference_tensor.copy()
N = ht.MPI_WORLD.size
# split along axis = 0
a_tensor.resplit_(axis=0)
local_shape = (1, N + 1, 2 * N)
local_tensor = self.reference_tensor[ht.MPI_WORLD.rank, :, :]
self.assertEqual(a_tensor.lshape, local_shape)
self.assertTrue(
(a_tensor._DNDarray__array == local_tensor._DNDarray__array).all())
# unsplit
a_tensor.resplit_(axis=None)
self.assertTrue((a_tensor._DNDarray__array ==
self.reference_tensor._DNDarray__array).all())
# split along axis = 1
a_tensor.resplit_(axis=1)
if ht.MPI_WORLD.rank == 0:
local_shape = (N, 2, 2 * N)
local_tensor = self.reference_tensor[:, 0:2, :]
else:
local_shape = (N, 1, 2 * N)
local_tensor = self.reference_tensor[:, ht.MPI_WORLD.rank +
1:ht.MPI_WORLD.rank + 2, :]
self.assertEqual(a_tensor.lshape, local_shape)
self.assertTrue(
(a_tensor._DNDarray__array == local_tensor._DNDarray__array).all())
# unsplit
a_tensor.resplit_(axis=None)
self.assertTrue((a_tensor._DNDarray__array ==
self.reference_tensor._DNDarray__array).all())
# split along axis = 2
a_tensor.resplit_(axis=2)
local_shape = (N, N + 1, 2)
local_tensor = self.reference_tensor[:, :, 2 * ht.MPI_WORLD.rank:2 *
ht.MPI_WORLD.rank + 2]
self.assertEqual(a_tensor.lshape, local_shape)
self.assertTrue(
(a_tensor._DNDarray__array == local_tensor._DNDarray__array).all())
expected = torch.ones((ht.MPI_WORLD.size, 100),
dtype=torch.int64,
device=self.device.torch_device)
data = ht.array(expected, split=1)
data.resplit_(None)
self.assertTrue(torch.equal(data._DNDarray__array, expected))
self.assertFalse(data.is_distributed())
self.assertIsNone(data.split)
self.assertEqual(data.dtype, ht.int64)
self.assertEqual(data._DNDarray__array.dtype, expected.dtype)
expected = torch.zeros((100, ht.MPI_WORLD.size),
dtype=torch.uint8,
device=self.device.torch_device)
data = ht.array(expected, split=0)
data.resplit_(None)
self.assertTrue(torch.equal(data._DNDarray__array, expected))
self.assertFalse(data.is_distributed())
self.assertIsNone(data.split)
self.assertEqual(data.dtype, ht.uint8)
self.assertEqual(data._DNDarray__array.dtype, expected.dtype)
# "in place"
length = torch.tensor([i + 20 for i in range(2)],
device=self.device.torch_device)
test = torch.arange(torch.prod(length),
dtype=torch.float64,
device=self.device.torch_device).reshape(
[i + 20 for i in range(2)])
a = ht.array(test, split=1)
a.resplit_(axis=0)
self.assertTrue(ht.equal(a, ht.array(test, split=0)))
self.assertEqual(a.split, 0)
self.assertEqual(a.dtype, ht.float64)
del a
test = torch.arange(torch.prod(length),
device=self.device.torch_device)
a = ht.array(test, split=0)
a.resplit_(axis=None)
self.assertTrue(ht.equal(a, ht.array(test, split=None)))
self.assertEqual(a.split, None)
self.assertEqual(a.dtype, ht.int64)
del a
a = ht.array(test, split=None)
a.resplit_(axis=0)
self.assertTrue(ht.equal(a, ht.array(test, split=0)))
self.assertEqual(a.split, 0)
self.assertEqual(a.dtype, ht.int64)
del a
a = ht.array(test, split=0)
resplit_a = ht.manipulations.resplit(a, axis=None)
self.assertTrue(ht.equal(resplit_a, ht.array(test, split=None)))
self.assertEqual(resplit_a.split, None)
self.assertEqual(resplit_a.dtype, ht.int64)
del a
a = ht.array(test, split=None)
resplit_a = ht.manipulations.resplit(a, axis=0)
self.assertTrue(ht.equal(resplit_a, ht.array(test, split=0)))
self.assertEqual(resplit_a.split, 0)
self.assertEqual(resplit_a.dtype, ht.int64)
del a
def initialize(self, image, state, gt, *args, **kwargs):
if len(gt) == 8:
ww = gt[2] - gt[0]
hh = gt[7] - gt[1]
else:
ww = gt[2]
hh = gt[3]
# Initialize some stuff
self.frame_num = 1
if not hasattr(self.params, 'device'):
self.params.device = 'cuda' if self.params.use_gpu else 'cpu'
if ww < 25 and hh < 25:
self.feature_sz = TensorList([torch.Tensor([28., 28.])])
self.output_layer = TensorList(['layer2'])
else:
self.feature_sz = TensorList([torch.Tensor([14., 14.])])
# self.output_layer = TensorList(['layer3'])
self.output_layer = TensorList(['layer3'])
# Initialize some stuff
if not hasattr(self.params, 'device'):
self.params.device = 'cuda' if self.params.use_gpu else 'cpu'
# Initialize features
self.initialize_features(self.output_layer)
# Check if image is color
self.params.features.set_is_color(image.shape[2] == 3)
# Get feature specific params
self.fparams = self.params.features.get_fparams('feature_params')
self.time = 0
tic = time.time()
# Get position and size
self.pos = torch.Tensor(
[state[1] + (state[3] - 1) / 2, state[0] + (state[2] - 1) / 2])
self.target_sz = torch.Tensor([state[3], state[2]])
if state[3] > 50 or state[2] > 50:
self.target_sz = torch.Tensor(
[state[3] - state[3] / 8, state[2] - state[2] / 4])
else:
self.target_sz = torch.Tensor([state[3], state[2]])
# Set search area
self.target_scale = 1.0
search_area = torch.prod(self.target_sz *
self.params.search_area_scale).item()
if search_area > self.params.max_image_sample_size:
self.target_scale = math.sqrt(search_area /
self.params.max_image_sample_size)
elif search_area < self.params.min_image_sample_size:
self.target_scale = math.sqrt(search_area /
self.params.min_image_sample_size)
# Check if IoUNet is used
self.use_iou_net = getattr(self.params, 'use_iou_net', True)
# Target size in base scale
self.base_target_sz = self.target_sz / self.target_scale
# Use odd square search area and set sizes
feat_max_stride = max(self.params.features.stride())
if getattr(self.params, 'search_area_shape', 'square') == 'square':
self.img_sample_sz = torch.round(
torch.sqrt(
torch.prod(self.base_target_sz *
self.params.search_area_scale))) * torch.ones(2)
elif self.params.search_area_shape == 'initrect':
self.img_sample_sz = torch.round(self.base_target_sz *
self.params.search_area_scale)
else:
raise ValueError('Unknown search area shape')
if self.params.feature_size_odd:
self.img_sample_sz += feat_max_stride - self.img_sample_sz % (
2 * feat_max_stride)
else:
self.img_sample_sz += feat_max_stride - (
self.img_sample_sz + feat_max_stride) % (2 * feat_max_stride)
# Set sizes
self.img_support_sz = self.img_sample_sz
self.feature_sz = self.params.features.size(self.img_sample_sz)
self.output_sz = self.params.score_upsample_factor * self.img_support_sz # Interpolated size of the output
self.kernel_size = self.fparams.attribute('kernel_size')
self.iou_img_sample_sz = self.img_sample_sz
# Optimization options
self.params.precond_learning_rate = self.fparams.attribute(
'learning_rate')
if self.params.CG_forgetting_rate is None or max(
self.params.precond_learning_rate) >= 1:
self.params.direction_forget_factor = 0
else:
self.params.direction_forget_factor = (
1 - max(self.params.precond_learning_rate)
)**self.params.CG_forgetting_rate
self.output_window = None
if getattr(self.params, 'window_output', False):
if getattr(self.params, 'use_clipped_window', False):
self.output_window = dcf.hann2d_clipped(
self.output_sz.long(),
self.output_sz.long() * self.params.effective_search_area /
self.params.search_area_scale,
centered=False).to(self.params.device)
else:
self.output_window = dcf.hann2d(self.output_sz.long(),
centered=False).to(
self.params.device)
# Initialize some learning things
self.init_learning()
# Convert image
im = numpy_to_torch(image)
self.im = im # For debugging only
# Setup scale bounds
self.image_sz = torch.Tensor([im.shape[2], im.shape[3]])
self.min_scale_factor = torch.max(10 / self.base_target_sz)
self.max_scale_factor = torch.min(self.image_sz / self.base_target_sz)
# Extract and transform sample
x = self.generate_init_samples(im)
# Initialize iounet
if self.use_iou_net:
self.init_iou_net()
# Initialize projection matrix
self.init_projection_matrix(x)
# Transform to get the training sample
train_x = self.preprocess_sample(x)
# Generate label function
init_y = self.init_label_function(train_x)
# Init memory
self.init_memory(train_x)
# Init optimizer and do initial optimization
self.init_optimization(train_x, init_y)
self.pos_iounet = self.pos.clone()
self.time += time.time() - tic
self.pool1 = torch.nn.AdaptiveMaxPool2d((1, 224))
self.pool2 = torch.nn.AdaptiveMaxPool2d((224, 1))
def test_prod(self):
x = torch.randn(1, 2, 3, 4, requires_grad=True)
self.assertONNX(lambda x: torch.prod(x), x)
def bboxes_iou(bboxes_a,
bboxes_b,
xyxy=True,
GIoU=False,
DIoU=False,
CIoU=False):
"""Calculate the Intersection of Unions (IoUs) between bounding boxes.
IoU is calculated as a ratio of area of the intersection
and area of the union.
Args:
bbox_a (array): An array whose shape is :math:`(N, 4)`.
:math:`N` is the number of bounding boxes.
The dtype should be :obj:`numpy.float32`.
bbox_b (array): An array similar to :obj:`bbox_a`,
whose shape is :math:`(K, 4)`.
The dtype should be :obj:`numpy.float32`.
Returns:
array:
An array whose shape is :math:`(N, K)`. \
An element at index :math:`(n, k)` contains IoUs between \
:math:`n` th bounding box in :obj:`bbox_a` and :math:`k` th bounding \
box in :obj:`bbox_b`.
from: https://github.com/chainer/chainercv
https://github.com/ultralytics/yolov3/blob/eca5b9c1d36e4f73bf2f94e141d864f1c2739e23/utils/utils.py#L262-L282
"""
if bboxes_a.shape[1] != 4 or bboxes_b.shape[1] != 4:
raise IndexError
if xyxy:
# intersection top left
tl = torch.max(bboxes_a[:, None, :2], bboxes_b[:, :2])
# intersection bottom right
br = torch.min(bboxes_a[:, None, 2:], bboxes_b[:, 2:])
# convex (smallest enclosing box) top left and bottom right
con_tl = torch.min(bboxes_a[:, None, :2], bboxes_b[:, :2])
con_br = torch.max(bboxes_a[:, None, 2:], bboxes_b[:, 2:])
# centerpoint distance squared
rho2 = ((bboxes_a[:, None, 0] + bboxes_a[:, None, 2]) -
(bboxes_b[:, 0] + bboxes_b[:, 2]))**2 / 4 + (
(bboxes_a[:, None, 1] + bboxes_a[:, None, 3]) -
(bboxes_b[:, 1] + bboxes_b[:, 3]))**2 / 4
w1 = bboxes_a[:, 2] - bboxes_a[:, 0]
h1 = bboxes_a[:, 3] - bboxes_a[:, 1]
w2 = bboxes_b[:, 2] - bboxes_b[:, 0]
h2 = bboxes_b[:, 3] - bboxes_b[:, 1]
area_a = torch.prod(bboxes_a[:, 2:] - bboxes_a[:, :2], 1)
area_b = torch.prod(bboxes_b[:, 2:] - bboxes_b[:, :2], 1)
else:
# intersection top left
tl = torch.max((bboxes_a[:, None, :2] - bboxes_a[:, None, 2:] / 2),
(bboxes_b[:, :2] - bboxes_b[:, 2:] / 2))
# intersection bottom right
br = torch.min((bboxes_a[:, None, :2] + bboxes_a[:, None, 2:] / 2),
(bboxes_b[:, :2] + bboxes_b[:, 2:] / 2))
# convex (smallest enclosing box) top left and bottom right
con_tl = torch.min((bboxes_a[:, None, :2] - bboxes_a[:, None, 2:] / 2),
(bboxes_b[:, :2] - bboxes_b[:, 2:] / 2))
con_br = torch.max((bboxes_a[:, None, :2] + bboxes_a[:, None, 2:] / 2),
(bboxes_b[:, :2] + bboxes_b[:, 2:] / 2))
# centerpoint distance squared
rho2 = ((bboxes_a[:, None, :2] - bboxes_b[:, :2])**2 / 4).sum(dim=-1)
w1 = bboxes_a[:, 2]
h1 = bboxes_a[:, 3]
w2 = bboxes_b[:, 2]
h2 = bboxes_b[:, 3]
area_a = torch.prod(bboxes_a[:, 2:], 1)
area_b = torch.prod(bboxes_b[:, 2:], 1)
en = (tl < br).type(tl.type()).prod(dim=2)
area_i = torch.prod(br - tl, 2) * en # * ((tl < br).all())
area_u = area_a[:, None] + area_b - area_i
iou = area_i / area_u
if GIoU or DIoU or CIoU:
if GIoU: # Generalized IoU https://arxiv.org/pdf/1902.09630.pdf
area_c = torch.prod(con_br - con_tl, 2) # convex area
return iou - (area_c - area_u) / area_c # GIoU
if DIoU or CIoU: # Distance or Complete IoU https://arxiv.org/abs/1911.08287v1
# convex diagonal squared
c2 = torch.pow(con_br - con_tl, 2).sum(dim=2) + 1e-16
if DIoU:
return iou - rho2 / c2 # DIoU
elif CIoU: # https://github.com/Zzh-tju/DIoU-SSD-pytorch/blob/master/utils/box/box_utils.py#L47
v = (4 / math.pi**2) * torch.pow(
torch.atan(w1 / h1).unsqueeze(1) - torch.atan(w2 / h2), 2)
with torch.no_grad():
alpha = v / (1 - iou + v)
return iou - (rho2 / c2 + v * alpha) # CIoU
return iou
def ms_ssim_loss(input,
target,
max_val,
filter_size=11,
k1=0.01,
k2=0.03,
sigma=1.5,
size_average=None,
reduce=None,
reduction='mean'):
# type: (Tensor, Tensor, float, int, float, float, float, Optional[bool], Optional[bool], str) -> Tensor
r"""ms_ssim_loss(input, target, max_val, filter_size, k1, k2,
sigma, size_average=None, reduce=None, reduction='mean') -> Tensor
Measures the multi-scale structural similarity index (MS-SSIM) error.
See :class:`~torch.nn.MSSSIMLoss` for details.
"""
if input.size() != target.size():
raise ValueError(
'Expected input size ({}) to match target size ({}).'.format(
input.size(0), target.size(0)))
if input.device != target.device:
raise RuntimeError(
f'The input device {input.device} and target device {target.device} do not match.'
)
dim = input.dim()
if dim == 2: # Expand does not copy data.
input = input.expand(1, 1, input.dim(-2), input.dim(-1))
target = target.expand(1, 1, target.dim(-2), target.dim(-1))
elif dim == 3:
input = input.expand(1, input.dim(-3), input.dim(-2), input.dim(-1))
target = target.expand(1, target.dim(-3), target.dim(-2),
target.dim(-1))
elif dim != 4:
raise ValueError('Expected 2, 3, or 4 dimensions (got {})'.format(dim))
if size_average is not None or reduce is not None:
reduction = _Reduction.legacy_get_string(size_average, reduce)
channel = input.size(1)
kernel = _fspecial_gaussian(filter_size, channel, sigma, input.device)
# It would be nice if this did not have to be initialized every time that the function was called.
# Copying from host (CPU) to device (GPU) is a major bottleneck.
weights = torch.tensor([0.0448, 0.2856, 0.3001, 0.2363, 0.1333],
device=input.device)
weights = weights.unsqueeze(-1).unsqueeze(-1)
levels = weights.size(0)
mssim = list()
mcs = list()
for _ in range(levels):
ssim, cs = _ssim(input, target, max_val, k1, k2, channel, kernel)
ssim = ssim.mean((2, 3))
cs = cs.mean((2, 3))
mssim.append(ssim)
mcs.append(cs)
input = avg_pool2d(input, (2, 2))
target = avg_pool2d(target, (2, 2))
mssim = torch.stack(mssim)
mcs = torch.stack(mcs)
p1 = mcs**weights
p2 = mssim**weights
ret = torch.prod(p1[:-1], 0) * p2[-1]
if reduction != 'none':
ret = torch.mean(ret) if reduction == 'mean' else torch.sum(ret)
return ret
def __init__(self,
d,
input_dim=1,
output_dim=1,
latent_size=8,
img_size=28,
DEVICE='cuda'):
super(SparseVAE, self).__init__()
self.d = d
self.opt_basis = nn.Parameter(torch.zeros(self.d).to(DEVICE),
requires_grad=True)
self.latent_size = latent_size
self.img_size = img_size
self.input_dim = input_dim
self.output_dim = output_dim
self.linear_size = int(16 * (img_size / 4)**2)
self.layers = torch.nn.ModuleDict()
self.layers['enc_conv1'] = nn.Conv2d(self.input_dim,
32,
kernel_size=5,
stride=1,
padding=2,
bias=False)
self.layers['enc_bn1'] = nn.BatchNorm2d(32, track_running_stats=True)
self.layers['enc_act1'] = nn.ELU()
self.layers['enc_conv2'] = nn.Conv2d(32,
32,
kernel_size=4,
stride=2,
padding=1,
bias=False)
self.layers['enc_bn2'] = nn.BatchNorm2d(32, track_running_stats=True)
self.layers['enc_act2'] = nn.ELU()
self.layers['enc_conv3'] = nn.Conv2d(32,
64,
kernel_size=3,
stride=2,
padding=1,
bias=False)
self.layers['enc_bn3'] = nn.BatchNorm2d(64, track_running_stats=True)
self.layers['enc_act3'] = nn.ELU()
self.layers['enc_conv4'] = nn.Conv2d(64,
16,
kernel_size=5,
stride=1,
padding=2,
bias=True)
self.layers['dec_conv1'] = nn.ConvTranspose2d(16,
32,
kernel_size=4,
stride=1,
padding=2,
output_padding=0,
bias=False)
self.layers['dec_bn1'] = nn.BatchNorm2d(32, track_running_stats=True)
self.layers['dec_act1'] = nn.ELU()
self.layers['dec_conv2'] = nn.ConvTranspose2d(32,
16,
kernel_size=5,
stride=2,
padding=1,
output_padding=1,
bias=False)
self.layers['dec_bn2'] = nn.BatchNorm2d(16, track_running_stats=True)
self.layers['dec_act2'] = nn.ELU()
self.layers['dec_conv3'] = nn.ConvTranspose2d(16,
16,
kernel_size=5,
stride=2,
padding=2,
output_padding=1,
bias=False)
self.layers['dec_bn3'] = nn.BatchNorm2d(16, track_running_stats=True)
self.layers['dec_act3'] = nn.ELU()
self.layers['dec_conv4'] = nn.ConvTranspose2d(16,
self.output_dim,
kernel_size=3,
stride=1,
padding=1,
output_padding=0,
bias=True)
self.layers['fc_mu_logvar'] = nn.Linear(self.linear_size,
2 * self.latent_size)
self.layers['fc_dec'] = nn.Linear(self.latent_size, self.linear_size)
self.config = {}
self.config['fc_mu_logvar'] = 'linear'
self.config['fc_dec'] = 'linear'
self.config['enc_conv1'] = 'conv2d'
self.config['enc_bn1'] = 'batch_norm'
self.config['enc_act1'] = 'elu'
self.config['enc_conv2'] = 'conv2d'
self.config['enc_bn2'] = 'batch_norm'
self.config['enc_act2'] = 'elu'
self.config['enc_conv3'] = 'conv2d'
self.config['enc_bn3'] = 'batch_norm'
self.config['enc_act3'] = 'elu'
self.config['enc_conv4'] = 'conv2d'
self.config['dec_conv1'] = 'conv_transpose2d'
self.config['dec_bn1'] = 'batch_norm'
self.config['dec_act1'] = 'elu'
self.config['dec_conv2'] = 'conv_transpose2d'
self.config['dec_bn2'] = 'batch_norm'
self.config['dec_act2'] = 'elu'
self.config['dec_conv3'] = 'conv_transpose2d'
self.config['dec_bn3'] = 'batch_norm'
self.config['dec_act3'] = 'elu'
self.config['dec_conv4'] = 'conv_transpose2d'
# get full dimension
self.D = 0
for name, layer in self.layers.items():
if hasattr(layer, 'weight'):
self.D += torch.prod(torch.tensor(layer.weight.size()))
if layer.bias is not None:
self.D += torch.prod(torch.tensor(layer.bias.size()))
layer.requires_grad = False # none of the layers will be updated
self.DEVICE = DEVICE
self.D = self.D.item()
self.get_projection_matrix()
def validate(loader, model, criterion, criterion_seg,
criterion_line_class, criterion_horizon, M_inv, epoch=0):
# Define container to keep track of metric and loss
losses = AverageMeter()
avg_area = AverageMeter()
avg_trapezium_rule = AverageMeter()
acc_hor_tot = AverageMeter()
acc_line_tot = AverageMeter()
# Evaluate model
model.eval()
# Only forward pass, hence no gradients needed
with torch.no_grad():
# Start validation loop
for i, (input_data, gt, params, idx, gt_line, gt_horizon, index) in tqdm(enumerate(loader)):
if not args.no_cuda:
input_data, params = input_data.cuda(non_blocking=True), params.cuda(non_blocking=True)
input_data = input_data.float()
gt0, gt1, gt2, gt3 = params[:, 0, :], params[:, 1, :], params[:, 2, :], params[:, 3, :]
# Evaluate model
try:
beta0, beta1, beta2, beta3, weightmap_zeros, M, \
output_net, outputs_line, outputs_horizon = model(input_data, args.end_to_end)
'''
beta0_np = beta0.cpu().numpy()
beta1_np = beta1.cpu().numpy()
left_lane = polynomial(beta0)
right_lane = polynomial(beta1)
l_a1_np = left_lane.a1.cpu().numpy()
l_b1_np = left_lane.b1.cpu().numpy()
l_c1_np = left_lane.c1.cpu().numpy()
r_a1_np = right_lane.a1.cpu().numpy()
r_b1_np = right_lane.b1.cpu().numpy()
r_c1_np = right_lane.c1.cpu().numpy()
input_data_np = input_data.cpu().numpy()
gt_np = gt.cpu().numpy()
print(type(input_data_np))
print('input_data.shape: {}'.format(input_data_np.shape))
print('gt.shape: {}'.format(gt_np.shape))
print('beta0.shape: {}'.format(beta0_np.shape))
l_x_pts = np.arange(0, 512)
r_x_pts = np.arange(0, 512)
for index in range(input_data_np.shape[0]):
processed_img = input_data_np[index]
gt_img = gt_np[index]
processed_img = np.moveaxis(processed_img, 0, -1)
r,g,b = cv2.split(processed_img)
rgb_img = cv2.merge([b,g,r])
l_y_pts = l_a1_np[index]*l_x_pts**2 + l_b1_np[index]*l_x_pts + l_c1_np[index]
r_y_pts = r_a1_np[index]*r_x_pts**2 + r_b1_np[index]*r_x_pts + r_c1_np[index]
l_pts = np.vstack((l_x_pts,l_y_pts)).astype(np.int32).T
r_pts = np.vstack((r_x_pts,r_y_pts)).astype(np.int32).T
cv2.polylines(rgb_img, [l_pts], False, (0, 255, 0), 1)
cv2.polylines(rgb_img, [r_pts], False, (0, 255, 0), 1)
cv2.imshow("input image", rgb_img)
cv2.waitKey(1)
'''
except RuntimeError as e:
print("Batch with idx {} skipped due to singular matrix".format(idx.numpy()))
print(e)
continue
# Compute losses on parameters or segmentation
if args.end_to_end:
loss = criterion(beta0, gt0) + criterion(beta1, gt1)
if args.nclasses > 3:
# Masks to create zero in the loss when lane line is not present
mask_llhs = torch.prod(gt2 != 0, 1) \
.unsqueeze(1).unsqueeze(1).expand_as(beta2).type(torch.FloatTensor)
mask_rrhs = torch.prod(gt3 != 0, 1) \
.unsqueeze(1).unsqueeze(1).expand_as(beta3).type(torch.FloatTensor)
if not args.no_cuda:
mask_llhs = mask_llhs.cuda()
mask_rrhs = mask_rrhs.cuda()
beta2 = beta2*mask_llhs
beta3 = beta3*mask_rrhs
# add losses of further lane lines
loss += criterion(beta2, gt2) + criterion(beta3, gt3)
else:
gt = gt.cuda(non_blocking=True)
loss = criterion_seg(output_net, gt)
area = criterion(beta0, gt0) + criterion(beta1, gt1)
avg_area.update(area.item(), input_data.size(0))
# Horizon task & Line classification task
if args.clas:
gt_horizon, gt_line = gt_horizon.cuda(non_blocking=True), \
gt_line.cuda(non_blocking=True)
horizon_pred = torch.round(nn.Sigmoid()(outputs_horizon))
acc = torch.eq(horizon_pred, gt_horizon)
acc_hor = torch.sum(acc).float()/(args.resize*args.batch_size)
acc_hor_tot.update(acc_hor.item())
_, line_pred = torch.max(outputs_line, 1)
acc = torch.eq(line_pred, gt_line)
acc_line = torch.sum(acc).float()/(args.nclasses*args.batch_size)
acc_line_tot.update(acc_line.item())
loss_horizon = criterion_horizon(outputs_horizon, gt_horizon)
loss_line = criterion_line_class(outputs_line, gt_line)
loss = loss*args.weight_fit + (loss_line + loss_horizon)*args.weight_class
else:
line_pred = gt_line
# Exact area computation
gt_left_lines = polynomial(gt0.cpu())
gt_right_lines = polynomial(gt1.cpu())
pred_left_lines = polynomial(beta0.cpu())
pred_right_lines = polynomial(beta1.cpu())
trap_left = pred_left_lines.trapezoidal(gt_left_lines)
trap_right = pred_right_lines.trapezoidal(gt_right_lines)
avg_trapezium_rule.update(((trap_left + trap_right)/2).mean().item(), input_data.size(0))
losses.update(loss.item(), input_data.size(0))
#Write predictions to json file
if args.clas:
num_el = input_data.size(0)
if args.nclasses > 2:
params_batch = torch.cat((beta0, beta1, beta2, beta3),2) \
.transpose(1, 2).data.tolist()
else:
params_batch = torch.cat((beta0, beta1),2).transpose(1, 2).data.tolist()
line_type = line_pred.data.tolist()
horizon_pred = horizon_pred.data.tolist()
with open(val_set_path, 'w') as jsonFile:
for j in range(num_el):
im_id = index[j]
json_line = valid_set_labels[im_id]
line_id = line_type[j]
horizon_est = horizon_pred[j]
params = params_batch[j]
json_line["params"] = params
json_line["line_id"] = line_id
json_line["horizon_est"] = horizon_est
json.dump(json_line, jsonFile)
jsonFile.write('\n')
# Print info
if (i + 1) % args.print_freq == 0:
print('Test: [{0}/{1}]\t'
'Loss {loss.val:.8f} ({loss.avg:.8f})\t'
'Area {metric.val:.8f} ({metric.avg:.8f})'.format(
i+1, len(loader), loss=losses, metric=avg_area))
# Plot weightmap and curves
batch_size = input_data.cpu().numpy().shape[0]
print('saving results for batch {}'.format(i))
for idx in range(batch_size):
print('\tsaving results for image {} of batch {}'.format(idx, i))
save_weightmap('valid', M, M_inv,
weightmap_zeros, beta0, beta1, beta2, beta3,
gt0, gt1, gt2, gt3, line_pred, gt, idx, i, input_data,
args.no_ortho, args.resize, args.save_path)
# Compute x, y coordinates for accuracy later
if args.clas and args.nclasses > 3:
write_lsq_results(val_set_path, ls_result_path, args.nclasses,
False, False, args.resize, no_ortho=args.no_ortho)
acc_seg = LaneEval.bench_one_submit(ls_result_path, val_set_path)
print("===> Average ACC_SEG on val is {:.8}".format(acc_seg[0]))
if args.evaluate:
print("===> Average {}-loss on validation set is {:.8}".format(crit_string,
losses.avg))
print("===> Average exact area on validation set is {:.8}".format(
avg_trapezium_rule.avg))
if not args.end_to_end:
print("===> Average area**2 on validation set is {:.8}".format(avg_area.avg))
if args.clas:
print("===> Average HORIZON ACC on val is {:.8}".format(acc_hor_tot.avg))
print("===> Average LINE ACC on val is {:.8}".format(acc_hor_tot.avg))
return losses.avg, avg_area.avg, avg_trapezium_rule.avg, acc_hor_tot.avg, acc_line_tot.avg
def main():
global args
global mse_policy
parser = define_args()
args = parser.parse_args()
if not args.end_to_end:
assert args.pretrained == False
mse_policy = args.loss_policy == 'homography_mse'
if args.clas:
assert args.nclasses == 4
# Check GPU availability
if not args.no_cuda and not torch.cuda.is_available():
raise Exception("No gpu available for usage")
torch.backends.cudnn.benchmark = args.cudnn
# Define save path
save_id = 'Mod_{}_opt_{}_loss_{}_lr_{}_batch_{}_end2end_{}_lanes_{}_resize_{}_pretrain{}_clas{}' \
.format(args.mod, args.optimizer,
args.loss_policy,
args.learning_rate,
args.batch_size,
args.end_to_end,
args.nclasses,
args.resize,
args.pretrained,
args.clas)
# Get homography
M_inv = get_homography(args.resize)
# Dataloader for training and validation set
train_loader, valid_loader, valid_idx = get_loader(args.num_train,
args.json_file, args.image_dir,
args.gt_dir,
args.flip_on, args.batch_size,
shuffle=True, num_workers=args.nworkers,
end_to_end=args.end_to_end,
resize=args.resize,
split_percentage=args.split_percentage)
# Define network
model = Net(args)
define_init_weights(model, args.weight_init)
if not args.no_cuda:
# Load model on gpu before passing params to optimizer
model = model.cuda()
# Define optimizer and scheduler
optimizer = define_optim(args.optimizer, model.parameters(),
args.learning_rate, args.weight_decay)
scheduler = define_scheduler(optimizer, args)
# Define loss criteria for multiple tasks
criterion, criterion_seg = define_loss_crit(args)
criterion_line_class = nn.CrossEntropyLoss().cuda()
criterion_horizon = nn.BCEWithLogitsLoss().cuda()
# Name
global crit_string
crit_string = 'AREA**2' if args.end_to_end else 'ENTROPY'
if args.clas:
crit_string = 'TOT LOSS'
# Logging setup
best_epoch = 0
lowest_loss = np.inf
log_file_name = 'log_train_start_0.txt'
args.save_path = os.path.join(args.save_path, save_id)
mkdir_if_missing(args.save_path)
mkdir_if_missing(os.path.join(args.save_path, 'example/'))
mkdir_if_missing(os.path.join(args.save_path, 'example/train'))
mkdir_if_missing(os.path.join(args.save_path, 'example/valid'))
# Computes the file with lane data of the validation set
validation_set_path = os.path.join(args.save_path , 'validation_set.json')
load_valid_set_file_all(valid_idx, validation_set_path, args.image_dir)
global valid_set_labels
global val_set_path
global ls_result_path
valid_set_labels = [json.loads(line) for line in open(validation_set_path).readlines()]
val_set_path = os.path.join(args.save_path, 'validation_set_dst.json')
ls_result_path = os.path.join(args.save_path, 'ls_result.json')
# Tensorboard writer
if not args.no_tb:
global writer
writer = SummaryWriter(os.path.join(args.save_path, 'Tensorboard/'))
# Train, evaluate or resume
args.resume = first_run(args.save_path)
if args.resume and not args.test_mode and not args.evaluate:
path = os.path.join(args.save_path, 'checkpoint_model_epoch_{}.pth.tar'.format(
int(args.resume)))
if os.path.isfile(path):
log_file_name = 'log_train_start_{}.txt'.format(args.resume)
# Redirect stdout
sys.stdout = Logger(os.path.join(args.save_path, log_file_name))
print("=> loading checkpoint '{}'".format(args.resume))
checkpoint = torch.load(path)
args.start_epoch = checkpoint['epoch']
lowest_loss = checkpoint['loss']
best_epoch = checkpoint['best epoch']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded checkpoint '{}' (epoch {})"
.format(args.resume, checkpoint['epoch']))
else:
log_file_name = 'log_train_start_0.txt'
# Redirect stdout
sys.stdout = Logger(os.path.join(args.save_path, log_file_name))
print("=> no checkpoint found at '{}'".format(path))
# Only evaluate
elif args.evaluate:
best_file_name = glob.glob(os.path.join(args.save_path, 'model_best*'))[0]
if os.path.isfile(best_file_name):
sys.stdout = Logger(os.path.join(args.save_path, 'Evaluate.txt'))
print("=> loading checkpoint '{}'".format(best_file_name))
checkpoint = torch.load(best_file_name)
model.load_state_dict(checkpoint['state_dict'])
else:
print("=> no checkpoint found at '{}'".format(best_file_name))
validate(valid_loader, model, criterion, criterion_seg,
criterion_line_class, criterion_horizon, M_inv)
return
elif args.test_mode:
best_file_name = glob.glob(os.path.join(args.save_path, 'model_best*'))[0]
if os.path.isfile(best_file_name):
sys.stdout = Logger(os.path.join(args.save_path, 'Evaluate.txt'))
print("=> loading checkpoint '{}'".format(best_file_name))
checkpoint = torch.load(best_file_name)
model.load_state_dict(checkpoint['state_dict'])
else:
print("=> no checkpoint found at '{}'".format(best_file_name))
test(valid_loader, model, criterion, criterion_seg,
criterion_line_class, criterion_horizon, M_inv)
return
# Start training from clean slate
else:
# Redirect stdout
sys.stdout = Logger(os.path.join(args.save_path, log_file_name))
# INIT MODEL
print(40*"="+"\nArgs:{}\n".format(args)+40*"=")
print("Init model: '{}'".format(args.mod))
print("Number of parameters in model {} is {:.3f}M".format(
args.mod.upper(), sum(tensor.numel() for tensor in model.parameters())/1e6))
# Start training and validation for nepochs
for epoch in range(args.start_epoch, args.nepochs):
print("\n => Start train set for EPOCH {}".format(epoch + 1))
# Adjust learning rate
if args.lr_policy is not None and args.lr_policy != 'plateau':
scheduler.step()
lr = optimizer.param_groups[0]['lr']
print('lr is set to {}'.format(lr))
if args.pretrained:
if (epoch < args.pretrain_epochs):
args.end_to_end = False
print("Pretraining so set args.end_to_end to {}".format(args.end_to_end))
else:
args.end_to_end = True
# Define container objects to keep track of multiple losses/metrics
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
avg_area = AverageMeter()
exact_area = AverageMeter()
# Specfiy operation modus
model.train()
# compute timing
end = time.time()
# Start training loop
for i, (input, gt, params, idx, gt_line, gt_horizon) in tqdm(enumerate(train_loader)):
# Time dataloader
data_time.update(time.time() - end)
# Put inputs on gpu if possible
if not args.no_cuda:
input, params = input.cuda(non_blocking=True), params.cuda(non_blocking=True)
input = input.float()
assert params.size(1) == 4
gt0, gt1, gt2, gt3 = params[:, 0, :], params[:, 1, :], params[:, 2, :], params[:, 3, :]
# Run model
try:
beta0, beta1, beta2, beta3, weightmap_zeros, M, \
output_net, outputs_line, outputs_horizon = model(input, args.end_to_end)
except RuntimeError as e:
print("Batch with idx {} skipped due to singular matrix".format(idx.numpy()))
print(e)
continue
# Compute losses on parameters or on segmentation
if args.end_to_end:
loss = criterion(beta0, gt0) + criterion(beta1, gt1)
if args.nclasses > 3:
# Masks to create zero in the loss when lane line is not present
mask_llhs = torch.prod(gt2 != 0, 1) \
.unsqueeze(1).unsqueeze(1).expand_as(beta2).type(torch.FloatTensor)
mask_rrhs = torch.prod(gt3 != 0, 1) \
.unsqueeze(1).unsqueeze(1).expand_as(beta3).type(torch.FloatTensor)
if not args.no_cuda:
mask_llhs = mask_llhs.cuda()
mask_rrhs = mask_rrhs.cuda()
beta2 = beta2*mask_llhs
beta3 = beta3*mask_rrhs
# add losses of further lane lines
loss += criterion(beta2, gt2) + criterion(beta3, gt3)
else:
gt = gt.cuda(non_blocking=True)
loss = criterion_seg(output_net, gt)
with torch.no_grad():
area = criterion(beta0, gt0) + criterion(beta1, gt1)
avg_area.update(area.item(), input.size(0))
# Horizon task & Line classification task
if args.clas:
gt_horizon, gt_line = gt_horizon.cuda(non_blocking=True), \
gt_line.cuda(non_blocking=True)
_, line_pred = torch.max(outputs_line, 1)
loss_horizon = criterion_horizon(outputs_horizon, gt_horizon)
loss_line = criterion_line_class(outputs_line, gt_line)
loss = loss*args.weight_fit + (loss_line + loss_horizon)*args.weight_class
else:
line_pred = gt_line
losses.update(loss.item(), input.size(0))
# Clip gradients (usefull for instabilities or mistakes in ground truth)
if args.clip_grad_norm != 0:
nn.utils.clip_grad_norm(model.parameters(), args.clip_grad_norm)
# Setup backward pass
optimizer.zero_grad()
loss.backward()
optimizer.step()
# Time trainig iteration
batch_time.update(time.time() - end)
end = time.time()
# Exact AREA computation for egolines on cpu
with torch.no_grad():
gt_left_lines = polynomial(gt0.cpu())
gt_right_lines = polynomial(gt1.cpu())
pred_left_lines = polynomial(beta0.cpu())
pred_right_lines = polynomial(beta1.cpu())
trap_left = pred_left_lines.trapezoidal(gt_left_lines)
trap_right = pred_right_lines.trapezoidal(gt_right_lines)
exact_area.update(((trap_left + trap_right)/2).mean().item(), input.size(0))
# Print info
if (i + 1) % args.print_freq == 0:
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Loss {loss.val:.8f} ({loss.avg:.8f})'.format(
epoch+1, i+1, len(train_loader), batch_time=batch_time,
data_time=data_time, loss=losses))
# Plot weightmap and curves
if (i + 1) % args.save_freq == 0:
save_weightmap('train', M, M_inv,
weightmap_zeros, beta0, beta1, beta2, beta3,
gt0, gt1, gt2, gt3, line_pred, gt, 0, i, input,
args.no_ortho, args.resize, args.save_path)
losses_valid, avg_area_valid, avg_exact_area, \
acc_hor_tot, acc_line_tot = validate(valid_loader,
model, criterion,
criterion_seg,
criterion_line_class,
criterion_horizon,
M_inv,
epoch)
if not args.end_to_end:
print("===> Average AREA**2 from segmentation on training set is {:.8f}" \
.format(avg_area.avg))
print("===> Average AREA**2 from segmetnation validation set is {:.8f}" \
.format(avg_area_valid))
print("===> Average {}-loss on training set is {:.8f}".format(crit_string, losses.avg))
print("===> Average {}-loss on validation set is {:.8f}".format(crit_string, losses_valid))
print("===> Average exact area on training set is {:.8f}".format(exact_area.avg))
print("===> Average exact area on validation set is {:.8f}".format(avg_exact_area))
if args.clas:
print("===> Average HORIZON ACC on val is {:.8}".format(acc_hor_tot))
print("===> Average LINE ACC on val is {:.8}".format(acc_line_tot))
print("===> Last best {}-loss was {:.8f} in epoch {}".format(
crit_string, lowest_loss, best_epoch))
if not args.no_tb:
if args.end_to_end:
writer.add_scalars('Loss/Area**2', {'Training': losses.avg}, epoch)
writer.add_scalars('Loss/Area**2', {'Validation': losses_valid}, epoch)
else:
writer.add_scalars('Loss/Area**2', {'Training': avg_area.avg}, epoch)
writer.add_scalars('Loss/Area**2', {'Validation': avg_area_valid}, epoch)
writer.add_scalars('CROSS-ENTROPY', {'Training': losses.avg}, epoch)
writer.add_scalars('CROSS-ENTROPY', {'Validation': losses_valid}, epoch)
writer.add_scalars('Metric', {'Training': exact_area.avg}, epoch)
writer.add_scalars('Metric', {'Validation': avg_exact_area}, epoch)
total_score = avg_exact_area
# Adjust learning_rate if loss plateaued
if args.lr_policy == 'plateau':
scheduler.step(total_score)
lr = optimizer.param_groups[0]['lr']
print('LR plateaued, hence is set to {}'.format(lr))
# File to keep latest epoch
with open(os.path.join(args.save_path, 'first_run.txt'), 'w') as f:
f.write(str(epoch))
# Save model
to_save = False
if total_score < lowest_loss:
to_save = True
best_epoch = epoch+1
lowest_loss = total_score
save_checkpoint({
'epoch': epoch + 1,
'best epoch': best_epoch,
'arch': args.mod,
'state_dict': model.state_dict(),
'loss': lowest_loss,
'optimizer': optimizer.state_dict()}, to_save, epoch)
if not args.no_tb:
writer.close()
def prod(a, dim=-1):
return torch.prod(a, dim=dim)
def test_reduced_prod_keepdim(self):
x = torch.randn(1, 2, 3, 4, requires_grad=True)
self.assertONNX(lambda x: torch.prod(x, dim=2, keepdim=True), x)
def get_fourier_fn_k(new_k, U_shape):
# only torch one because we need gradient
h_k = torch.tensor(get_h_k(new_k, U_shape))
return lambda x: (1 / h_k) * torch.prod(torch.cos(x * torch.tensor(new_k)))
def test_reduced_prod_dtype(self):
x = torch.randn(1, 2, 3, 4, requires_grad=True)
self.assertONNXRaisesRegex(
RuntimeError, 'Couldn\'t export operator aten::prod',
lambda x: torch.prod(x, dim=0, dtype=torch.double), x)
def forward(self, x):
shape = torch.prod(torch.tensor(x.shape[1:])).item()
return x.view(-1, shape)
def count_maxpool(m, x, y):
kernel_ops = torch.prod(torch.Tensor([m.kernel_size])) - 1
num_elements = y.numel()
total_ops = kernel_ops * num_elements
return int(total_ops)
print("Using MLTN")
model = MLTN(input_dim=dim,
output_dim=output_dim,
nCh=nCh,
kernel=kernel,
bn=args.bn,
dropout=args.dropout,
bond_dim=args.bond_dim,
feature_dim=feature_dim,
adaptive_mode=adaptive_mode,
periodic_bc=periodic_bc,
virtual_dim=1)
elif args.tenetx:
print("Tensornet-X baseline")
# pdb.set_trace()
model = MPS(input_dim=torch.prod(dim),
output_dim=output_dim,
bond_dim=args.bond_dim,
feature_dim=feature_dim,
adaptive_mode=adaptive_mode,
periodic_bc=periodic_bc,
tenetx=args.tenetx)
elif args.densenet:
print("Densenet Baseline!")
model = DenseNet(depth=40,
growthRate=12,
reduction=0.5,
bottleneck=True,
nClasses=output_dim)
elif args.mlp:
print("MLP Baseline!")
def forward(self,inputs):
size = torch.prod(torch.LongTensor(list(inputs.size())[1:])).item()
return inputs.view((-1,size))
def validate(model,
pair_dataloader,
device,
criterion,
n_iterations=1,
read_level_info=False):
"""
extract all the features from a given dataset
"""
model.eval()
all_y_true = None
all_y_pred = []
all_y_pred_read = []
start = time.time()
with torch.no_grad():
for n in range(n_iterations):
y_true_tmp = []
y_pred_tmp = []
y_pred_read_tmp = []
for batch in pair_dataloader:
y_true = batch.pop('y').to(device).flatten()
X = {key: val.to(device) for key, val in batch.items()}
y_pred = model(X).detach().cpu().numpy()
if all_y_true is None:
y_true_tmp.extend(y_true.detach().cpu().numpy())
if (len(y_pred.shape) == 1) or (y_pred.shape[1] == 1):
y_pred_tmp.extend(y_pred.flatten())
else:
y_pred_tmp.extend(y_pred[:, 1])
if read_level_info:
y_pred_read = model.get_read_probability(X)
y_pred_read = 1 - torch.prod(1 - y_pred_read, axis=1)
y_pred_read = y_pred_read.detach().cpu().numpy()
y_pred_read_tmp.extend(y_pred_read)
if all_y_true is None:
all_y_true = y_true_tmp
all_y_pred.append(y_pred_tmp)
if read_level_info:
all_y_pred_read.append(y_pred_read_tmp)
compute_time = time.time() - start
y_pred_avg = np.mean(all_y_pred, axis=0)
all_y_true = np.array(all_y_true).flatten()
val_results = {}
val_results['y_pred'] = all_y_pred
val_results['y_true'] = all_y_true
val_results['compute_time'] = compute_time
val_results['accuracy'] = get_accuracy(all_y_true,
(y_pred_avg.flatten() > 0.5) * 1)
val_results['roc_auc'] = get_roc_auc(all_y_true, y_pred_avg)
val_results['pr_auc'] = get_pr_auc(all_y_true, y_pred_avg)
val_results["avg_loss"] = criterion(torch.Tensor(y_pred_avg),
torch.Tensor(all_y_true)).item()
if len(all_y_pred_read) > 0:
all_y_pred_read = np.mean(all_y_pred_read, axis=0)
all_y_pred_read = np.array(all_y_pred_read)
val_results['accuracy_read'] = get_accuracy(
all_y_true.flatten(), (all_y_pred_read.flatten() > 0.5) * 1)
val_results['roc_auc_read'] = get_roc_auc(all_y_true, all_y_pred_read)
val_results['pr_auc_read'] = get_pr_auc(all_y_true, all_y_pred_read)
val_results["avg_loss_read"] = criterion(
torch.Tensor(all_y_pred_read), torch.Tensor(all_y_true)).item()
return val_results
def getProb(sample, pVec):
#sample is 0-1 set and pVec is a probability distribution
temp = pVec * sample + (1 - pVec) * (1 - sample)
return torch.prod(temp, 1)
def ms_ssim(X, Y,
data_range=255,
size_average=True,
win_size=11,
win_sigma=1.5,
win=None,
weights=None,
K=(0.01, 0.03)):
r""" interface of ms-ssim
Args:
X (torch.Tensor): a batch of images, (N,C,H,W)
Y (torch.Tensor): a batch of images, (N,C,H,W)
data_range (float or int, optional): value range of input images. (usually 1.0 or 255)
size_average (bool, optional): if size_average=True, ssim of all images will be averaged as a scalar
win_size: (int, optional): the size of gauss kernel
win_sigma: (float, optional): sigma of normal distribution
win (torch.Tensor, optional): 1-D gauss kernel. if None, a new kernel will be created according to win_size and win_sigma
weights (list, optional): weights for different levels
K (list or tuple, optional): scalar constants (K1, K2). Try a larger K2 constant (e.g. 0.4) if you get a negative or NaN results_nuclear_real_denoise.
Returns:
torch.Tensor: ms-ssim results_nuclear_real_denoise
"""
if len(X.shape) != 4:
raise ValueError('Input images should be 4-d tensors.')
if not X.type() == Y.type():
raise ValueError('Input images should have the same dtype.')
if not X.shape == Y.shape:
raise ValueError('Input images should have the same dimensions.')
if win is not None: # set win_size
win_size = win.shape[-1]
if not (win_size % 2 == 1):
raise ValueError('Window size should be odd.')
smaller_side = min(X.shape[-2:])
assert smaller_side > (win_size - 1) * (2 ** 4), \
"Image size should be larger than %d due to the 4 downsamplings in ms-ssim" % ((win_size - 1) * (2 ** 4))
if weights is None:
weights = [0.0448, 0.2856, 0.3001, 0.2363, 0.1333]
weights = torch.FloatTensor(weights).to(X.device, dtype=X.dtype)
if win is None:
win = _fspecial_gauss_1d(win_size, win_sigma)
win = win.repeat(X.shape[1], 1, 1, 1)
levels = weights.shape[0]
mcs = []
for i in range(levels):
ssim_per_channel, cs = _ssim(X, Y,
win=win,
data_range=data_range,
size_average=False,
K=K)
if i < levels - 1:
mcs.append(torch.relu(cs))
padding = (X.shape[2] % 2, X.shape[3] % 2)
X = F.avg_pool2d(X, kernel_size=2, padding=padding)
Y = F.avg_pool2d(Y, kernel_size=2, padding=padding)
ssim_per_channel = torch.relu(ssim_per_channel) # (batch, channel)
mcs_and_ssim = torch.stack(mcs + [ssim_per_channel], dim=0) # (level, batch, channel)
ms_ssim_val = torch.prod(mcs_and_ssim ** weights.view(-1, 1, 1), dim=0)
if size_average:
return ms_ssim_val.mean()
else:
return ms_ssim_val.mean(1)
def Run(self, variables):
all_F = variables['all_F']
num_objects = variables['info']['num_objs']
num_frames = variables['info']['num_frames']
height = variables['info']['height']
width = variables['info']['width']
prev_targets = variables['prev_targets']
scribbles = variables['scribbles']
target = scribbles['annotated_frame']
loss = 0
masks = torch.zeros(num_objects, num_frames, height, width)
for n_obj in range(1, num_objects + 1):
# variables for current obj
all_E_n = variables['mask_objs'][n_obj-1:n_obj].data if variables['mask_objs'][n_obj-1:n_obj] is not None \
else variables['mask_objs'][n_obj-1:n_obj]
a_ref = variables['ref'][n_obj - 1]
prev_E_n = all_E_n.clone()
all_M_n = (variables['all_M'] == n_obj).long()
# divide scribbles for current object
n_scribble = copy.deepcopy(scribbles)
n_scribble['scribbles'][target] = []
for p in scribbles['scribbles'][target]:
if p['object_id'] == n_obj:
n_scribble['scribbles'][target].append(copy.deepcopy(p))
n_scribble['scribbles'][target][-1]['object_id'] = 1
else:
if p['object_id'] == 0:
n_scribble['scribbles'][target].append(
copy.deepcopy(p))
scribble_mask = scribbles2mask(n_scribble, (height, width))[target]
scribble_mask_N = (prev_E_n[0, target].cpu() >
0.5) & (torch.tensor(scribble_mask) == 0)
scribble_mask[scribble_mask == 0] = -1
scribble_mask[scribble_mask_N] = 0
scribble_mask = Dilate_mask(scribble_mask, 1)
# interaction
tar_P, tar_N = ToCudaPN(scribble_mask)
all_E_n[:, target], batch_CE, ref = self.model_I(
all_F[:, :, target], all_E_n[:, target], tar_P, tar_N,
all_M_n[:, target], [1, 1, 1, 1, 1]) # [batch, 256,512,2]
loss += batch_CE
# propagation
left_end, right_end, weight = Get_weight(target,
prev_targets,
num_frames,
at_least=-1)
# Prop_forward
next_a_ref = None
for n in range(target + 1, right_end + 1): #[1,2,...,N-1]
all_E_n[:, n], batch_CE, next_a_ref = self.model_P(
ref, a_ref, all_F[:, :, n], prev_E_n[:, n],
all_E_n[:, n - 1], all_M_n[:,
n], [1, 1, 1, 1, 1], next_a_ref)
loss += batch_CE
# Prop_backward
for n in reversed(range(left_end, target)):
all_E_n[:, n], batch_CE, next_a_ref = self.model_P(
ref, a_ref, all_F[:, :, n], prev_E_n[:, n],
all_E_n[:, n + 1], all_M_n[:,
n], [1, 1, 1, 1, 1], next_a_ref)
loss += batch_CE
for f in range(num_frames):
all_E_n[:, f, :, :] = weight[f] * all_E_n[:, f, :, :] + (
1 - weight[f]) * prev_E_n[:, f, :, :]
masks[n_obj - 1] = all_E_n[0]
variables['ref'][n_obj - 1] = next_a_ref
loss /= num_objects
em = torch.zeros(1, num_objects + 1, num_frames, height,
width).to(masks.device)
em[0, 0, :, :, :] = torch.prod(1 - masks, dim=0) # bg prob
em[0, 1:num_objects + 1, :, :] = masks # obj prob
em = torch.clamp(em, 1e-7, 1 - 1e-7)
all_E = torch.log((em / (1 - em)))
all_E = F.softmax(all_E, dim=1)
final_masks = all_E[0].max(0)[1].float()
variables['prev_targets'].append(target)
variables['masks'] = final_masks
variables['mask_objs'] = masks
variables['probs'] = all_E
variables['loss'] = loss
def __init__(self, input_dim: Tuple[int], latent_dim: int = 2) -> None:
super(features_to_latent, self).__init__()
self.reshape_ = torch.prod(tt(input_dim))
self.fc_latent = nn.Linear(self.reshape_, latent_dim)
def evaluate_true(self, X: Tensor) -> Tensor:
part1 = torch.sum(X**2 / 4000.0, dim=1)
d = X.shape[1]
part2 = -(torch.prod(
torch.cos(X / torch.sqrt(X.new(range(1, d + 1))).view(1, -1))))
return part1 + part2 + 1.0
def backward(ctx, grad_output):
'''
There are two algorithms to do this. The first one
is very efficient, but works only when there are no
nonzero elements in the input.
The second one is much more complex, but it doesn't
assume anything on the input. The main downside is
that it takes time O(n^2), where n = input.size(self.dim)
(i.e. the length of the cumulative product). This is in
contrast to the forward pass and the efficient algorithm,
which are both O(n).
The second algorithm is a simple application of the chain
rule. If x is an n-dimensional vector, and y = cumprod(x),
and F is the final cost, then
dF / dx_k = sum_j (dF / dy_j) * (dy_j / dx_k) (1)
The term dF / dy_j is just grad_output[j] (assuming again
everything is one-dimensional).
The term (dy_j / dx_k) is easilly seen to be
if j >= k
dy_j / dx_k = prod_{1 <= i <= j, i != k} x_i
else:
dy_j / dx_k = 0
Note that the indicator (j>=k) can be taken out
by replacing the sum in (1) with a sum from
j = k to n.
Thus,
df / dx_k = sum_{k <= j <= n} grad_output[j] * (dy_j / dx_k)
with
dy_j / dx_k = prod_{1 <= i <= j, i != k} x_i (2)
Note that this last term is just the cumulative product
with k omitted. Thus, if x_k (the input) is nonzero, we can
just express this as
dy_j / dx_k = (prod_{1 <= i <= j} x_i) / x_k
= y_j / x_k
So therefore,
df / dx_k = sum_{k <= j <= n} grad_output[j] * y_j / x_k
so
grad_output = sum_scan_exclusiv(grad_output * output) / input
If the input is nonzero, we need to calculate the dy_j / dx_k
by using the formula (2), called in the code omitted_products.
The way the code calculates it is simply by noting that
prod_{1 <= i <= j, i != k} x_i
= (prod_{1 <= i <= k} x_i) * (prod_{k + 1 <= i <= j} x_i)
the first term is calculated as prods_until_k, which since
doesn't depend in j is easy to vectorize.
The second term (indexed by j) is the cumulative product of
x_{k+1}, x_{k+2}, ..., x_n, and it's named in the code
prods_from_k_pkus_1, and it's calculated as a cumprod.
In order to vectorize this properly, we need to add to
omitted_products the dimensions where k > j, and therefore
dy_j / dx_k = 0, which is done right after the assert.
'''
input, = ctx.saved_variables
dim_size = input.size(ctx.dim)
if dim_size == 1:
return grad_output, None
# Simple case with nonzero elements in the input
if (input != 0).data.all():
output = torch.cumprod(input, dim=ctx.dim)
return sum_scan_exclusive(output * grad_output, dim=ctx.dim) / input, None
positive_dim = ctx.dim if ctx.dim >= 0 else input.dim() + ctx.dim
dim_padding = (slice(None, None),) * (positive_dim)
ones_size = list(input.size())
ones_size[ctx.dim] = 1
ones = Variable(input.data.new([1]).expand(ones_size))
grad_input = Variable(grad_output.data.new(input.size()).zero_())
for k in range(dim_size):
if k == 0:
prods_from_k_plus_1 = torch.cumprod(
input[dim_padding + (slice(k + 1, None),)],
dim=ctx.dim
)
omitted_products = torch.cat(
(ones, prods_from_k_plus_1),
dim=ctx.dim
)
elif k == dim_size - 1:
prods_until_k = torch.prod(
input[dim_padding + (slice(None, k),)],
dim=ctx.dim,
keepdim=True
)
omitted_products = prods_until_k
else:
prods_until_k = torch.prod(
input[dim_padding + (slice(None, k),)],
dim=ctx.dim,
keepdim=True
)
prods_from_k_plus_1 = torch.cumprod(
input[dim_padding + (slice(k + 1, None),)],
dim=ctx.dim
)
omitted_products = prods_until_k.expand_as(
prods_from_k_plus_1) * prods_from_k_plus_1
omitted_products = torch.cat(
(prods_until_k, omitted_products), ctx.dim)
# At this point omitted_products is the same size
# as input, except on the dimension dim where it's
# dim_size - k
assert omitted_products.size(ctx.dim) == dim_size - k
# should we implement copy_ or _set_item in variable?
index = tuple(slice(None, None) for _ in range(positive_dim)) + (k,)
grad_input[index] = torch.sum(
grad_output[dim_padding + (slice(k, None),)] * omitted_products,
dim=ctx.dim)
return grad_input, None
def forward(self, x):
"""
此方法中实现了DNM的前向传播，输入X为二维矩阵
"""
X_orig = torch.as_tensor(x, device='cuda', dtype=torch.float32)
# x_split = torch.split(X_orig,self.max_Neuron,dim=1)
y = torch.tensor([], device='cuda')
X_orig = X_orig.unsqueeze(dim=1)
X_orig = X_orig.expand(X_orig.shape[0], self.M,
X_orig.shape[2]) # 若输入X为多维矩阵，此处应修改
y_temp_orig = F.leaky_relu(
self.k1 * 1.0 * (torch.mul(X_orig, self.W_orig) - self.q_orig))
y_temp_orig2 = F.leaky_relu(
self.k1 * 1.0 * (torch.mul(X_orig, self.W2_orig) - self.q2_orig))
y_temp_orig3 = F.leaky_relu(
self.k1 * 1.0 * (torch.mul(X_orig, self.W3_orig) - self.q3_orig))
y_split = torch.split(y_temp_orig, self.BN_internal, dim=2)
batchnorm = nn.BatchNorm1d(9)
batchnorm.cuda()
y_group_out = torch.ones(y_temp_orig.shape[0],
y_temp_orig.shape[1],
1,
device='cuda')
y_split2 = torch.split(y_temp_orig2, self.BN_internal, dim=2)
batchnorm2 = nn.BatchNorm1d(9)
y_group_out2 = torch.ones(y_temp_orig2.shape[0], y_temp_orig2.shape[1],
1)
y_split3 = torch.split(y_temp_orig3, self.BN_internal, dim=2)
batchnorm3 = nn.BatchNorm1d(9)
y_group_out3 = torch.ones(y_temp_orig3.shape[0], y_temp_orig3.shape[1],
1)
for i in range(len(y_split)):
y_split_temp = y_split[i]
y_group_prod = y_group_out * torch.prod(
y_split_temp, dim=2, keepdim=True
) # 将不同特征的维度相乘 此处因为sigmoid激活导致数据分布在0-1之间，在特征多时会导致相乘后为0 方法一：前面降维：lstm降维或全连接层降维，方法二：自剪特征降维
y_group_out = batchnorm(y_group_prod)
y_orig_prod = torch.squeeze(y_group_out, dim=2)
y1_orig_sum = torch.sum(y_orig_prod, dim=1, keepdim=True) # 并相加
y2_orig_final = torch.sigmoid(self.k2 * 1.0 * (y1_orig_sum - 0.5))
y = torch.cat([y, y2_orig_final], dim=1)
for i in range(len(y_split2)):
y_split_temp2 = y_split2[i]
y_group_prod2 = y_group_out * torch.prod(
y_split_temp2, dim=2, keepdim=True
) # 将不同特征的维度相乘 此处因为sigmoid激活导致数据分布在0-1之间，在特征多时会导致相乘后为0 方法一：前面降维：lstm降维或全连接层降维，方法二：自剪特征降维
y_group_out2 = batchnorm(y_group_prod2)
y_orig_prod2 = torch.squeeze(y_group_out2, dim=2)
y1_orig_sum2 = torch.sum(y_orig_prod2, dim=1, keepdim=True) # 并相加
y2_orig_final2 = torch.sigmoid(self.k2 * 1.0 * (y1_orig_sum2 - 0.5))
y2 = torch.cat([y, y2_orig_final2], dim=1)
for i in range(len(y_split3)):
y_split_temp3 = y_split3[i]
y_group_prod3 = y_group_out * torch.prod(
y_split_temp3, dim=2, keepdim=True
) # 将不同特征的维度相乘 此处因为sigmoid激活导致数据分布在0-1之间，在特征多时会导致相乘后为0 方法一：前面降维：lstm降维或全连接层降维，方法二：自剪特征降维
y_group_out3 = batchnorm(y_group_prod3)
y_orig_prod3 = torch.squeeze(y_group_out3, dim=2)
y1_orig_sum3 = torch.sum(y_orig_prod3, dim=1, keepdim=True) # 并相加
y2_orig_final3 = torch.sigmoid(self.k2 * 1.0 * (y1_orig_sum3 - 0.5))
y3 = torch.cat([y, y2_orig_final3], dim=1)
y_hat = torch.cat([y2_orig_final, y2_orig_final2, y2_orig_final3],
dim=1)
# print("W的梯度：{} q的梯度: {}".format(self.W_orig.grad,self.q_orig.grad))
return y_hat
def test_reduced_prod_keepdim(self):
x = Variable(torch.randn(1, 2, 3, 4), requires_grad=True)
self.assertONNX(lambda x: torch.prod(x, dim=2, keepdim=True), x)
def __init__(
self,
env,
train_tasks,
eval_tasks,
latent_dim,
nets,
policy_lr=3e-4,
qf_lr=1e-3,
vf_lr=1e-3,
alpha = 0.15,
automatic_entropy_tuning = True,
lr=3e-4,
context_lr=3e-4,
kl_lambda=1.,
policy_mean_reg_weight=1e-3,
policy_std_reg_weight=1e-3,
policy_pre_activation_weight=0.,
optimizer_class=optim.Adam,
recurrent=False,
use_information_bottleneck=True,
sparse_rewards=False,
#soft_target_tau=1e-2,
soft_target_tau=0.005,
plotter=None,
render_eval_paths=False,
**kwargs
):
super().__init__(
env=env,
agent=nets[0],
train_tasks=train_tasks,
eval_tasks=eval_tasks,
**kwargs
)
self.soft_target_tau = soft_target_tau
self.policy_mean_reg_weight = policy_mean_reg_weight
self.policy_std_reg_weight = policy_std_reg_weight
self.policy_pre_activation_weight = policy_pre_activation_weight
self.plotter = plotter
self.render_eval_paths = render_eval_paths
self.alpha = alpha
self.automatic_entropy_tuning = automatic_entropy_tuning
self.recurrent = recurrent
self.latent_dim = latent_dim
# self.qf_criterion = nn.MSELoss()
# self.vf_criterion = nn.MSELoss()
self.vib_criterion = nn.MSELoss()
self.l2_reg_criterion = nn.MSELoss()
self.kl_lambda = kl_lambda
self.use_information_bottleneck = use_information_bottleneck
self.sparse_rewards = sparse_rewards
# self.qf1, self.qf2, self.vf = nets[1:]
self.critic,self.critic_target = nets[1:]#q1,q2,target q1,target q2
# self.target_vf = self.vf.copy()
self.critic_optimizer = Adam(self.critic.parameters(), lr=lr)
hard_update(self.critic_target,self.critic)
self.policy_optimizer = Adam(self.agent.policy.parameters(), lr=policy_lr)
self.target_entropy = -torch.prod(torch.Tensor(env.action_space.shape)).to("cuda").item() # torch.prod(input) : 返回所有元素的乘积
self.log_alpha = torch.zeros(1, requires_grad=True, device="cuda")
self.alpha_optim = Adam([self.log_alpha], lr=lr)
# self.policy_optimizer = optimizer_class(
# self.agent.policy.parameters(),
# lr=policy_lr,
# )
# self.qf1_optimizer = optimizer_class(
# self.qf1.parameters(),
# lr=qf_lr,
# )
# self.qf2_optimizer = optimizer_class(
# self.qf2.parameters(),
# lr=qf_lr,
# )
# self.vf_optimizer = optimizer_class(
# self.vf.parameters(),
# lr=vf_lr,
# )
self.context_optimizer = optimizer_class(
self.agent.context_encoder.parameters(),
lr=context_lr,
)
def prod(self, value, axis=None):
if isinstance(axis, (tuple, list)):
for dim in reversed(sorted(axis)):
value = torch.prod(value, dim=dim)
return value
return torch.prod(value, dim=axis)
def prod(x, axis=None):
if axis is None:
return torch.prod(x)
return torch.prod(x, dim=axis)
def dummy_predict(model, X):
# Add column to X that is a product of previous elements.
mean = torch.cat([X, torch.prod(X, dim=1).reshape(-1, 1)], dim=1)
cov = torch.zeros(mean.shape[0], mean.shape[1], mean.shape[1])
return mean, cov
y = torch.squeeze(x); print(y.shape) # torch.Size([2, 2, 2]); 所有维度是 1 的去掉 y = torch.squeeze(x, 0); print(y.shape) # torch.Size([2, 1, 2, 1, 2]); 只对第 0 维操作 y = torch.squeeze(x, 1); print(y.shape) # torch.Size([2, 2, 1, 2]); 只对第 1 维操作 ## Operation; 和 np 的区别就是 torch 需要变量都是 tensor 类型的 # 求和以及按索引求和: torch.sum() torch.Tensor.indexadd() # 元素乘积: torch.prod(input) # 求均值、方差、极值: torch.mean() torch.var() torch.max() torch.min() # 在 NLP 领域经常用到的: # 平方根倒数、线性插值、双曲正切 torch.rsqrt(input) torch.lerp(star,end,weight) torch.tanh(input, out=None) a, b = torch.Tensor([1, 2]), torch.Tensor([3, 4]) print(torch.add(a, b), a.add_(b)) print(torch.max(torch.Tensor([1, 2, 5, 3]))) # tensor(5.); Returns the maximum value of all elements in the :attr:`input` tensor. print(torch.max(torch.arange(1, 13).view(3, 4), 1)) # (tensor([ 4., 8., 12.]), tensor([ 3, 3, 3])); the latter is the indexs _, idx = torch.max(torch.arange(1, 13).view(3, 4), 1); print(idx) # tensor([ 3, 3, 3]) print(torch.prod(torch.Tensor([2, 3, 4]))) # tensor(24.); Returns the product of all elements ## log a = torch.Tensor([[1, 2], [3, 4]]) print(torch.log(a)) # tensor([[0.0000, 0.6931], [1.0986, 1.3863]]) ## min max sum a = torch.tensor([[1, 2, 3], [4, 5, 6]]); print(a) # tensor([[1, 2, 3], [4, 5, 6]]); (B, L) b = torch.tensor([[1, 3, 2], [4, 6, 4]]); print(b) # tensor([[1, 3, 2], [4, 6, 4]]) print(torch.min(a, b)) # tensor([[1, 2, 2], [4, 5, 4]]); 相同位置依次比较 print(torch.max(a, b)) # tensor([[1, 3, 3], [4, 6, 6]]) print(torch.max(a, dim=1)) # (tensor([3, 6]), tensor([2, 2])); 第二个是 index print(a.max(), a.max(0), a.max(1)) # tensor(6); (tensor([4, 5, 6]), tensor([1, 1, 1])); (tensor([3, 6]), tensor([2, 2])) print(a.max(), a.max(0)[1], a.max(1)[1]) # tensor(6) tensor([1, 1, 1]) tensor([2, 2]); [1] is index print(torch.sum(a, 0)) # tensor([5, 7, 9]) print(torch.sum(a, 1)) # tensor([ 6, 15])