def __init__(self):
self.ps = U.Params(params).init_comps()
self.pre = None
self.post = None
i = torch.constant([0.0] * (4 * 10), shape=(4, 10))
self.src_b = torch.Variable(initial_value=i)
i = torch.constant([0.0] * (4 * 10), shape=(4, 10))
self.mem_b = torch.Variable(initial_value=i)
def test_with_owner():
a = L.Attend(Owner())
a.build([(4, 10, 16), (), (4, 18, 16), ()])
src = torch.constant([0.0] * (4 * 10 * 16), shape=(4, 10, 16))
bias = torch.constant([0.0] * (4 * 10), shape=(4, 10))
bias = torch.expand_dims(torch.expand_dims(bias, axis=1), axis=3)
mem = torch.constant([0.0] * (4 * 15 * 16), shape=(4, 15, 16))
ctx = torch.constant([0.0] * (4 * 15 * 16), shape=(4, 15, 16))
a.call([src, bias, mem, ctx])
def test_owner_none():
a = L.Attend(Owner())
a.build([(4, 10, 16)])
src = torch.constant([0.0] * (4 * 10 * 16), shape=(4, 10, 16))
a.call([src])
bias = torch.constant([0.0] * (4 * 10), shape=(4, 10))
bias = torch.expand_dims(torch.expand_dims(bias, axis=1), axis=3)
a.call([src, bias])
ctx = torch.constant([0.0] * (4 * 15 * 16), shape=(4, 15, 16))
a.call([src, bias, None, ctx])
def loss_info_dropout(model, images_v, labels_v, train, criterion):
"""
Criterion is Softmax-CE loss
We add to cost function regularization on the noise (alphas) via the KL terms
In all experiments we divide the KLdivergence term by the number of training
samples, so that for beta = 1 the scaling of the KL-divergence term in similar to
the one used by Variational Dropout
"""
beta = 3.0
x_output_v, kl_terms = model(images_v, train)
loss_v = None
if train:
kl_terms = [kl.sum(dim=1).mean()
for kl in kl_terms] # kl had dims (batch_sz,4096)
if not kl_terms:
kl_terms = [torch.constant(0.)]
N = images_v.size(0)
Lz = (kl_terms[0] + kl_terms[1]) * 1. / N # sum the list
# size_average = True : By default, the losses are averaged over observations for each minibatch.
Lx = criterion(x_output_v, labels_v)
if np.random.randint(0, 100) < 1: #print 1% of time
print(' [KL loss term: {}'.format(beta * Lz.data[0]))
print(' [CE loss term: {}'.format(Lx.data[0]))
loss_v = Lx + beta * Lz # PyTorch implicitly includes weight_decay * L2 in the loss
return loss_v, x_output_v
def test_tokembed():
e = TokEmbed(ps)
e.build((1, 5))
src = torch.constant([1, 2, 0, 3, 0], shape=(1, 5))
e.call(src)
ps.one_hot = True
e = TokEmbed(ps)
e.build((1, 5))
e.call(src)
def top_logp(self, ctx, bias, i):
cfg = self.cfg
y = torch.zeros((
cfg.batch_size,
cfg.beam_size,
cfg.num_toks,
))
y += torch.expand_dims(self.logp, axis=2)
b = torch.range(cfg.batch_size)
ii = torch.constant([i] * cfg.batch_size)
for j in range(cfg.beam_size):
jj = torch.constant([j] * cfg.batch_size)
sel = torch.stack([b, jj, ii])
yj = self.to_logp(self.tgt[:, j, :], ctx, bias, i)[1]
y = torch.tensor_scatter_nd_add(y, sel, yj)
y = torch.reshape(y, (-1, cfg.beam_size * cfg.num_toks))
logp, idx = torch.top_k(y, k=2 * cfg.beam_size)
return logp, idx
def conj_kspace(image_in, name="kspace_conj"):
"""Conjugate k-space data."""
image_out = torch.reverse(image_in, axis=[1])
image_out = torch.reverse(image_out, axis=[2])
mod = np.zeros((1, 1, 1, image_in.get_shape().as_list()[-1]))
mod[:, :, :, 1::2] = -1
mod = torch.constant(mod, dtype=torch.float32)
image_out = torch.multiply(image_out, mod)
return image_out
def build(self, input_shape):
cfg = self.cfg
tgt = input_shape[0]
assert tgt[0] == cfg.batch_size
y = torch.constant([[0.0] + [-float("inf")] * (cfg.beam_size - 1)])
self._logp = torch.tile(y, [cfg.batch_size, 1])
sh = (cfg.batch_size, cfg.beam_size)
self._score = torch.ones(shape=sh) * utils.big_neg
self._flag = torch.zeros(dtype="bool", shape=sh)
return super().build(input_shape)
def test_w_grad():
e = TokEmbed(ps)
e.build((None, 3))
ins = torch.constant([[0, 1, 0]], dtype="int32")
with torch.GradientTape() as tape:
out = e(ins)
print("===", out, e.weights)
gs = tape.gradient(out, e.weights)
opt = adagrad.AdagradOptimizer(0.1)
opt.apply_gradients(zip(gs, e.weights))
print("###", len(gs), 1)
def append_tok(self, idx, i, **kw):
cfg = self.cfg
k = 2 * cfg.beam_size
b = torch.range(cfg.batch_size * k) // k
b = torch.reshape(b, (cfg.batch_size, k))
beam = idx // cfg.num_toks
sel = torch.stack([b, beam], axis=2)
y = torch.gather_nd(self.tgt, sel)
ii = torch.constant([i] * cfg.batch_size * k)
ii = torch.reshape(ii, (cfg.batch_size, k))
sel = torch.stack([b, beam, ii], axis=2)
u = torch.expand_dims(idx % cfg.num_toks, axis=2)
tgt = torch.tensor_scatter_nd_update(y, sel, u)
return tgt
def pose2mat(quat, translation):
# [R t] [R1 t1] = [R*R1 R*t1+t]
# [0 1] [ 0 1] = [ 0 1 ]
x, y, z, w = quat[:, 0], quat[:, 1], quat[:, 2], quat[:, 3]
B = quat.size(0)
w2, x2, y2, z2 = w.pow(2), x.pow(2), y.pow(2), z.pow(2)
wx, wy, wz = w * x, w * y, w * z
xy, xz, yz = x * y, x * z, y * z
t1, t2, t3 = translation[:, 0], translation[:, 1], translation[:, 2]
l0 = torch.constant(0.0)
l1 = torch.constant(1.0)
mat = torch.stack([
w2 + x2 - y2 - z2, 2 * xy - 2 * wz, 2 * wy + 2 * xz, t1,
2 * wz + 2 * xy, w2 - x2 + y2 - z2, 2 * yz - 2 * wx, t2,
2 * xz - 2 * wy, 2 * wx + 2 * yz, w2 - x2 - y2 + z2, t3, l0, l0, l0, l1
],
dim=1).reshape(B, 4, 4)
return mat
def __init__(self, minval=0.0001, maxval=0.0001):
self.maxvalval = maxval
self.maxval = torch.constant(maxval)
self.minvalval = minval
self.minval = torch.constant(minval)
def __init__(self, maxval=4.0):
self.maxvalval = maxval
self.maxval = torch.constant(maxval)
def build(self, input_shape, dtype=torch.float64):
print("building")
self.input_shapes = input_shape
self.len_input = len(self.input_shapes)
self.connections = self.input_shapes[-1]
if self.dendrite_mode == self.modes[1]: # sparse
self.connections -= self.dendrite_shift
elif self.dendrite_mode == self.modes[2]: # overlap:
self.connections += self.dendrite_shift
if self.dendrites is None:
self.segmenter() # list of dendrites per neuron
if self.version == 4:
self.dendrites = torch.constant(self.dendrites)
self.pre_dendrites = self.connections * self.units # neurons*previous_layer_neurons
if self.version != 1:
dwshape = [self.units, self.seql]
else:
dwshape = [self.seql, self.units]
# dwshape=[self.units,self.seql,*[1 for _ in range(self.len_input-1)]]
# self.num_dendrites=self.pre_dendrites/self.dendrite_size
# if self.bigger_dendrite:
# self.num_dendrites=math.floor(self.num_dendrites)
# else:
# self.num_dendrites=math.ceil(self.num_dendrites)
# input_shape = tensor_shape.TensorShape(input_shape)
if self.version == 2:
if len(self.input_shapes) > 2:
part_inshape = (*self.input_shapes[1:-1], -1)
else:
part_inshape = (-1, )
self.debuildshape = (self.units * self.connections, *part_inshape)
self.deseqshape = (self.units * self.connections, )
self.rebuildshape = (self.units, self.seql, *part_inshape)
print('line228')
if self.weight_twice:
"""if self.uniqueW==2:#useless since all input are there once, could also work with sparse
print([self.dendrite_size,self.seql, self.units])
self.kernel=self.add_variable('Weight',shape=[*[1 for _ in range(self.len_input-1)],self.dendrite_size,self.seql, self.units],
initializer=self.Weight_initializer,regularizer=self.Weight_regularizer,
constraint=self.Weight_constraint,dtype=self.dtype,
trainable=True)"""
if self.uniqueW:
kernel = torch.empty(*[1 for _ in range(self.len_input - 1)],
self.input_shapes[-1],
self.units,
dtype=dtype)
else:
kernel = torch.empty(1, self.units, dtype=dtype)
finit.kaiming_normal(kernel)
self.kernel = nn.Parameter(kernel)
self.register_parameter('kernel', self.kernel)
self.params.append(self.kernel)
print('line246')
dw = torch.empty(*dwshape, dtype=dtype)
finit.kaiming_normal(dw)
self.dendriticW = nn.Parameter(dw)
self.params.append(self.dendriticW)
print(self.dendriticW)
print("added dendw")
if self.use_bias:
if self.weight_twice:
if self.uniqueW:
b = torch.empty(self.input_shapes[-1],
self.units,
dtype=dtype)
else:
b = torch.empty(1, self.units, dtype=dtype)
try:
finit.kaiming_normal_(b)
except:
finit.xavier_normal_(b)
self.bias = nn.Parameter(b)
self.register_parameter('Bias', self.bias)
self.params.append(self.bias)
if self.uniqueW:
db = torch.empty(self.seql, self.units, dtype=dtype)
else:
db = torch.empty(1, self.units, dtype=dtype)
finit.kaiming_normal_(db)
self.dendriticB = nn.Parameter(db)
self.params.append(self.dendriticB)
self.register_parameter('dendritic_B', self.dendriticB)
print("supered")
#self.register_parameter('dentritic_W', self.dendriticW)
self.built = True
print('builded')
def initalize_data(self,
a,
phi,
b,
prior_W,
prior_H,
Beta,
K0,
use_val_set,
dtype=torch.float32):
self.V = np.array(
self.V
) #when gets called in a loop as in run_parameter_sweep this can get updated to a torch tensor in a previous iteration which breaks some numpy functions
if K0 == None:
self.K0 = self.M
self.number_of_active_components = self.M
else:
self.K0 = K0
self.number_of_active_components = self.K0
if self.objective.lower() == 'poisson':
self.phi = torch.tensor(phi, dtype=dtype, requires_grad=False)
else:
self.phi = torch.tensor(np.var(self.V) * phi,
dtype=dtype,
requires_grad=False)
if use_val_set:
torch.manual_seed(0) #get the same mask each time
self.mask = (torch.rand(self.V.shape) > 0.2).type(
self.dtype
) #create mask, randomly mask ~20% of data in shape V. Only used when passed
else:
self.mask = torch.ones(self.V.shape, dtype=self.dtype)
self.a = a
self.prior_W = prior_W
self.prior_H = prior_H
self.C = []
self.b = b
W0 = np.multiply(
np.random.uniform(size=[self.M, self.K0]) + self.eps_.numpy(),
np.sqrt(self.V_max))
H0 = np.multiply(
np.random.uniform(size=[self.K0, self.N]) + self.eps_.numpy(),
np.sqrt(self.V_max))
L0 = np.sum(W0, axis=0) + np.sum(H0, axis=1)
self.W = torch.tensor(W0, dtype=self.dtype, requires_grad=False)
self.H = torch.tensor(H0, dtype=self.dtype, requires_grad=False)
self.Lambda = torch.tensor(L0,
dtype=torch.float32,
requires_grad=False)
# calculate default b as described in Tan and Fevotte (2012)
if self.b == None or self.b == 'None':
# L1 ARD
if self.prior_H == 'L1' and self.prior_W == 'L1':
self.bcpu = np.sqrt(
np.true_divide(
(self.a - 1) * (self.a - 2) * np.mean(self.V),
self.K0))
self.b = torch.tensor(np.sqrt(
np.true_divide(
(self.a - 1) * (self.a - 2) * np.mean(self.V),
self.K0)),
dtype=self.dtype,
requires_grad=False)
self.C = torch.tensor(self.N + self.M + self.a + 1,
dtype=self.dtype,
requires_grad=False)
# L2 ARD
elif self.prior_H == 'L2' and self.prior_W == 'L2':
self.bcpu = np.true_divide(
np.pi * (self.a - 1) * np.mean(self.V), 2 * self.K0)
self.b = torch.tensor(np.true_divide(
np.pi * (self.a - 1) * np.mean(self.V), 2 * self.K0),
dtype=self.dtype,
requires_grad=False)
self.C = torch.tensor((self.N + self.M) * 0.5 + self.a + 1,
dtype=self.dtype,
requires_grad=False)
# L1 - L2 ARD
elif self.prior_H == 'L1' and self.prior_W == 'L2':
self.bcpu = np.true_divide(
np.mean(self.V) * np.sqrt(2) * gamma(self.a - 3 / 2),
self.K0 * np.sqrt(np.pi) * gamma(self.a))
self.b = torch.tensor(np.true_divide(
np.mean(self.V) * np.sqrt(2) * gamma(self.a - 3 / 2),
self.K0 * np.sqrt(np.pi) * gamma(self.a)),
dtype=self.dtype,
requires_grad=False)
self.C = torch.tensor(self.N + self.M / 2 + self.a + 1,
dtype=self.dtype)
elif self.prior_H == 'L2' and self.prior_W == 'L1':
self.bcpu = np.true_divide(
np.mean(self.V) * np.sqrt(2) * gamma(self.a - 3 / 2),
self.K0 * np.sqrt(np.pi) * gamma(self.a))
self.b = torch.tensor(np.true_divide(
np.mean(self.V) * np.sqrt(2) * gamma(self.a - 3 / 2),
self.K0 * np.sqrt(np.pi) * gamma(self.a)),
dtype=self.dtype,
requires_grad=False)
self.C = torch.tensor(self.N / 2 + self.M + self.a + 1,
dtype=self.dtype)
else:
self.bcpu = self.b
self.b = torch.tensor(self.b,
dtype=self.dtype,
requires_grad=False)
if self.prior_H == 'L1' and self.prior_W == 'L1':
self.C = torch.tensor(self.N + self.M + self.a + 1,
dtype=self.dtype,
requires_grad=False)
# L2 ARD
elif self.prior_H == 'L2' and self.prior_W == 'L2':
self.C = torch.tensor((self.N + self.M) * 0.5 + self.a + 1,
dtype=self.dtype,
requires_grad=False)
# L1 - L2 ARD
elif self.prior_H == 'L1' and self.prior_W == 'L2':
self.C = torch.tensor(self.N + self.M / 2 + self.a + 1,
dtype=self.dtype,
requires_grad=False)
elif self.prior_H == 'L2' and self.prior_W == 'L1':
self.C = torch.constant(self.N / 2 + self.M + self.a + 1,
dtype=self.dtype,
requires_grad=False)
self.V = torch.tensor(self.V, dtype=self.dtype, requires_grad=False)
print('NMF data and parameters set.')
def test_shift():
a = L.Attend(Owner())
x = torch.constant([1, 2, 3, 4, 5, 6], shape=(1, 1, 2, 3))
torch.print(x)
x = a.shift(x)
torch.print(x)
def initalize_data(self,
a,
phi,
b,
prior_W,
prior_H,
Beta,
K0,
dtype=torch.float32):
if K0 == None:
self.K0 = self.M
self.number_of_active_components = self.M
else:
self.K0 = K0
self.number_of_active_components = self.K0
if self.objective.lower() == 'poisson':
self.phi = torch.tensor(phi, dtype=dtype, requires_grad=False)
else:
self.phi = torch.tensor(np.var(self.V) * phi,
dtype=dtype,
requires_grad=False)
self.a = a
self.prior_W = prior_W
self.prior_H = prior_H
self.C = []
self.b = b
W0 = np.multiply(
np.random.uniform(size=[self.M, self.K0]) + self.eps_.numpy(),
np.sqrt(self.V_max))
H0 = np.multiply(
np.random.uniform(size=[self.K0, self.N]) + self.eps_.numpy(),
np.sqrt(self.V_max))
L0 = np.sum(W0, axis=0) + np.sum(H0, axis=1)
self.W = torch.tensor(W0, dtype=self.dtype, requires_grad=False)
self.H = torch.tensor(H0, dtype=self.dtype, requires_grad=False)
self.Lambda = torch.tensor(L0,
dtype=torch.float32,
requires_grad=False)
# calculate default b as described in Tan and Fevotte (2012)
if self.b == None or self.b == 'None':
# L1 ARD
if self.prior_H == 'L1' and self.prior_W == 'L1':
self.bcpu = np.sqrt(
np.true_divide(
(self.a - 1) * (self.a - 2) * np.mean(self.V),
self.K0))
self.b = torch.tensor(np.sqrt(
np.true_divide(
(self.a - 1) * (self.a - 2) * np.mean(self.V),
self.K0)),
dtype=self.dtype,
requires_grad=False)
self.C = torch.tensor(self.N + self.M + self.a + 1,
dtype=self.dtype,
requires_grad=False)
# L2 ARD
elif self.prior_H == 'L2' and self.prior_W == 'L2':
self.bcpu = np.true_divide(
np.pi * (self.a - 1) * np.mean(self.V), 2 * self.K0)
self.b = torch.tensor(np.true_divide(
np.pi * (self.a - 1) * np.mean(self.V), 2 * self.K0),
dtype=self.dtype,
requires_grad=False)
self.C = torch.tensor((self.N + self.M) * 0.5 + self.a + 1,
dtype=self.dtype,
requires_grad=False)
# L1 - L2 ARD
elif self.prior_H == 'L1' and self.prior_W == 'L2':
self.bcpu = np.true_divide(
np.mean(self.V) * np.sqrt(2) * gamma(self.a - 3 / 2),
self.K0 * np.sqrt(np.pi) * gamma(self.a))
self.b = torch.tensor(np.true_divide(
np.mean(self.V) * np.sqrt(2) * gamma(self.a - 3 / 2),
self.K0 * np.sqrt(np.pi) * gamma(self.a)),
dtype=self.dtype,
requires_grad=False)
self.C = torch.tensor(self.N + self.M / 2 + self.a + 1,
dtype=self.dtype)
elif self.prior_H == 'L2' and self.prior_W == 'L1':
self.bcpu = np.true_divide(
np.mean(self.V) * np.sqrt(2) * gamma(self.a - 3 / 2),
self.K0 * np.sqrt(np.pi) * gamma(self.a))
self.b = torch.tensor(np.true_divide(
np.mean(self.V) * np.sqrt(2) * gamma(self.a - 3 / 2),
self.K0 * np.sqrt(np.pi) * gamma(self.a)),
dtype=self.dtype,
requires_grad=False)
self.C = torch.tensor(self.N / 2 + self.M + self.a + 1,
dtype=self.dtype)
else:
self.bcpu = self.b
self.b = torch.tensor(self.b,
dtype=self.dtype,
requires_grad=False)
if self.prior_H == 'L1' and self.prior_W == 'L1':
self.C = torch.tensor(self.N + self.M + self.a + 1,
dtype=self.dtype,
requires_grad=False)
# L2 ARD
elif self.prior_H == 'L2' and self.prior_W == 'L2':
self.C = torch.tensor((self.N + self.M) * 0.5 + self.a + 1,
dtype=self.dtype,
requires_grad=False)
# L1 - L2 ARD
elif self.prior_H == 'L1' and self.prior_W == 'L2':
self.C = torch.tensor(self.N + self.M / 2 + self.a + 1,
dtype=self.dtype,
requires_grad=False)
elif self.prior_H == 'L2' and self.prior_W == 'L1':
self.C = torch.constant(self.N / 2 + self.M + self.a + 1,
dtype=self.dtype,
requires_grad=False)
self.V = torch.tensor(self.V, dtype=self.dtype, requires_grad=False)
print('NMF data and parameters set.')
def testComplexShading(self):
"""Test specular highlights, colors, and multiple lights per image."""
model_transforms = camera_utils.euler_matrices(
[[-20.0, 0.0, 60.0], [45.0, 60.0, 0.0]])[:, :3, :3]
vertices_world_space = torch.matmul(
torch.stack([self.cube_vertices, self.cube_vertices]),
model_transforms.transpose())
normals_world_space = torch.matmul(
torch.stack([self.cube_normals, self.cube_normals]),
model_transforms.transpose())
# camera position:
eye = torch.tensor([[0.0, 0.0, 6.0], [0.0, 0.2, 18.0]], dtype=torch.float32)
center = torch.tensor([[0.0, 0.0, 0.0], [0.1, -0.1, 0.1]], dtype=torch.float32)
world_up = torch.constant([[0.0, 1.0, 0.0], [0.1, 1.0, 0.15]], dtype=torch.float32)
fov_y = torch.tensor([40.0, 13.3], dtype=torch.float32)
near_clip = 0.1
far_clip = 25.0
image_width = 640
image_height = 480
light_positions = torch.tensor([[[0.0, 0.0, 6.0], [1.0, 2.0, 6.0]],
[[0.0, -2.0, 4.0], [1.0, 3.0, 4.0]]])
light_intensities = torch.tensor(
[[[1.0, 1.0, 1.0], [1.0, 1.0, 1.0]],
[[2.0, 0.0, 1.0], [0.0, 2.0, 1.0]]],
dtype=torch.float32)
vertex_diffuse_colors = torch.tensor(2*[[[1.0, 0.0, 0.0],
[0.0, 1.0, 0.0],
[0.0, 0.0, 1.0],
[1.0, 1.0, 1.0],
[1.0, 1.0, 0.0],
[1.0, 0.0, 1.0],
[0.0, 1.0, 1.0],
[0.5, 0.5, 0.5]]],
dtype=torch.float32)
vertex_specular_colors = torch.tensor(2*[[[0.0, 1.0, 0.0],
[0.0, 0.0, 1.0],
[1.0, 1.0, 1.0],
[1.0, 1.0, 0.0],
[1.0, 0.0, 1.0],
[0.0, 1.0, 1.0],
[0.5, 0.5, 0.5],
[1.0, 0.0, 0.0]]],
dtype=torch.float32)
shininess_coefficients = 6.0 * torch.ones([2, 8], dtype=torch.float32)
ambient_color = torch.tensor([[0.0, 0.0, 0.0], [0.1, 0.1, 0.2]], dtype=torch.float32)
renders = mesh_renderer.mesh_renderer(
vertices_world_space,
self.cube_triangles,
normals_world_space,
vertex_diffuse_colors,
eye,
center,
world_up,
light_positions,
light_intensities,
image_width,
image_height,
vertex_specular_colors,
shininess_coefficients,
ambient_color,
fov_y,
near_clip,
far_clip)
tonemapped_renders = torch.cat([
mesh_renderer.tone_mapper(renders[:, :, :, 0:3], 0.7),
renders[:, :, :, 3:4]
],
dim=3)
# Check that shininess coefficient broadcasting works by also rendering
# with a scalar shininess coefficient, and ensuring the result is identical:
broadcasted_renders = mesh_renderer.mesh_renderer(
vertices_world_space,
self.cube_triangles,
normals_world_space,
vertex_diffuse_colors,
eye,
center,
world_up,
light_positions,
light_intensities,
image_width,
image_height,
vertex_specular_colors,
6.0,
ambient_color,
fov_y,
near_clip,
far_clip)
tonemapped_broadcasted_renders = torch.cat([
mesh_renderer.tone_mapper(broadcasted_renders[:, :, :, 0:3], 0.7),
broadcasted_renders[:, :, :, 3:4]
],
dim=3)
for image_id in range(renders.shape[0]):
target_image_name = "Colored_Cube_%i.png" % image_id
baseline_image_path = os.path.join(self.test_data_directory,
target_image_name)
test_utils.expect_image_file_and_render_are_near(
self, baseline_image_path, tonemapped_renders[image_id, :, :, :])
test_utils.expect_image_file_and_render_are_near(
self, baseline_image_path, tonemapped_broadcasted_renders[image_id, :, :, :])