def test_kwarg_only(self) -> None:
with capture_logs() as logs:
x = LoggingTensor(torch.ones(1))
y = LoggingTensor(torch.ones(1, 1))
z = LoggingTensor(torch.ones(1))
log_input("x", x)
log_input("y", y)
log_input("z", z)
torch.addmv(x, y, z)
torch.addmv(x, y, z, beta=1)
torch.addmv(x, y, z, beta=2)
torch.addmv(x, y, z, alpha=2)
torch.addmv(x, y, z, beta=2, alpha=2)
# The expectation is that beta/alpha don't show up when they're
# defaulted. This is even if the user explicitly specified it.
self.assertExpectedInline('\n'.join(logs), '''\
$0 = input('x')
$1 = input('y')
$2 = input('z')
$3 = torch._ops.aten.addmv($0, $1, $2)
$4 = torch._ops.aten.addmv($0, $1, $2)
$5 = torch._ops.aten.addmv($0, $1, $2, beta=2)
$6 = torch._ops.aten.addmv($0, $1, $2, alpha=2)
$7 = torch._ops.aten.addmv($0, $1, $2, beta=2, alpha=2)''')
def forward(self, input_, hx=None):
"""
An Elman RNN cell with tanh or ReLU non-linearity.
h' = tanh/relu(w_{ih} x + b_{ih} + w_{hh} h + b_{hh})
"""
# print(self.d_rec)
if hx is None:
hx = input_.new_zeros(self.hidden_size, requires_grad=False)
c_prev = hx
w_x = torch.addmv(self.bias_ih, self.weight_ih, input_)
w_h = torch.addmv(self.bias_hh, self.weight_hh, hx)
w_w = (w_x + w_h)
c_tilda = self.tanh(w_w[0:self.hidden_size])
gate = self.sigmoid(w_w[self.hidden_size:2 * self.hidden_size])
# inp = torch.mv(self.weight_ir, input_)
# prevh = torch.mv(self.weight_hr, (r * hx))
c = ((1 - gate) * c_prev) + (gate * c_tilda)
h = c
return h, gate
def native_update(Beta,
Delta,
Sigma,
nonnegative=True,
batch=None):
""" Native Pytorch Implementation Of HALS Update
Parameter:
Beta:
Delta:
Sigma:
nonnegative:
batch: batch size
"""
# TODO Implement Batching To Mitigate GPU Bottleneck
for ndx in range(Beta.shape[0]):
tmp_scale = 1.0 / Sigma[ndx, ndx].item()
torch.addmv(Delta[ndx],
Beta.t(),
Sigma[ndx],
alpha=-1 * tmp_scale,
beta=tmp_scale,
out=Delta[ndx])
Beta[ndx].add_(Delta[ndx])
if nonnegative:
torch.nn.functional.relu(Beta[ndx],
inplace=True)
def forward(self, input_, hx=None):
"""
begin{array}{ll}
i = sigma(W_{ii} x + b_{ii} + W_{hi} h + b_{hi}) \\
f = sigma(W_{if} x + b_{if} + W_{hf} h + b_{hf}) \\
g = tanh(W_{ig} x + b_{ig} + W_{hg} h + b_{hg}) \\
o = sigma(W_{io} x + b_{io} + W_{ho} h + b_{ho}) \\
c' = f * c + i * g \\
h' = o * tanh(c') \\
end{array}
"""
if hx is None:
hx = input_.new_zeros(self.hidden_size, requires_grad=False)
hx = (hx, hx)
use_gate = rectify(self.igate)
if (self.igate > 1.0):
# use_gate = self.igate - 1.0
print('ho')
hprev, cprev = hx
w_x = torch.addmv(self.bias_ih, self.weight_ih, input_)
w_h = torch.addmv(self.bias_hh, self.weight_hh, hprev)
w_w = w_x + w_h
i = self.sigmoid(w_w[0:self.hidden_size])
f = self.sigmoid(w_w[self.hidden_size:2 * self.hidden_size])
o = self.sigmoid(w_w[2 * self.hidden_size:3 * self.hidden_size])
g = self.tanh(w_w[3 * self.hidden_size:4 * self.hidden_size])
c = (f * cprev) + (i * g)
h = o * self.tanh(c)
return (h, c), o
def forward(self, input_, hx=None):
"""
An Elman RNN cell with tanh or ReLU non-linearity.
h' = tanh/relu(w_{ih} x + b_{ih} + w_{hh} h + b_{hh})
"""
# print(self.d_rec)
# print (self.rgate)
if hx is None:
hx = input_.new_zeros(self.hidden_size, requires_grad=False)
#dale_hh = torch.mm(self.relu(self.weight_hh), self.d_rec)
if (self.bias):
w_x = torch.addmv(self.bias_ih, self.weight_ih, input_)
#w_h = torch.addmv(self.bias_hh, dale_hh, hx)
w_h = torch.addmv(self.bias_hh, self.weight_hh, hx)
else:
w_x = torch.mv(self.weight_ih, input_)
#w_h = torch.mv(dale_hh, hx)
w_h = torch.mv(self.weight_hh, hx)
w_w = ((self.rgate) * hx) + ((1 - (self.rgate)) * (w_x + w_h))
h = self.relu(w_w)
return h
def get_elig(self, task_indicator_pm):
ANDmat = self.ANDmat
b_AND = self.b_AND
ORmat = self.ORmat # nb_or x
b_OR = self.b_OR
indicator = task_indicator_pm.type(torch.float)
ANDout = torch.addmv(-b_AND, ANDmat, indicator).sign().ne(-1).type(torch.float) #sign(A x indic + b) (+1 or 0)
elig_hard = torch.addmv(-b_OR, ORmat, ANDout).sign().ne(-1)
return elig_hard
def forward(ctx, add_vector, matrix, vector, alpha=1, beta=1, inplace=False):
ctx.alpha = alpha
ctx.beta = beta
ctx.add_vector_size = add_vector.size()
ctx.save_for_backward(matrix, vector)
output = _get_output(ctx, add_vector, inplace=inplace)
return torch.addmv(alpha, add_vector, beta,
matrix, vector, out=output)
def forward(ctx, add_vector, matrix, vector, alpha=1, beta=1, inplace=False):
ctx.alpha = alpha
ctx.beta = beta
ctx.add_vector_size = add_vector.size()
ctx.save_for_backward(matrix, vector)
output = _get_output(ctx, add_vector, inplace=inplace)
return torch.addmv(alpha, add_vector, beta,
matrix, vector, out=output)
def forward(self, add_vector, matrix, vector):
self.save_for_backward(matrix, vector)
output = self._get_output(add_vector)
return torch.addmv(self.alpha,
add_vector,
self.beta,
matrix,
vector,
out=output)
def forward(self, x): xsize = x.size() out = torch.addmv(self.bias, x.view(-1, xsize[-1]), self.w) if self.bias else x.view(-1, xsize[-1]).mv(self.w) rsize = list(xsize) rsize[-1] = 1 return out.view(rsize)
def forward(self, x):
xsize = x.size()
out = torch.addmv(self.bias, x.view(-1, xsize[-1]), self.w)
xsize = list(xsize)
xsize[-1] = 1
return ((torch.abs(self.k) + self.minv) *
(self.act(out) + 1)).view(xsize)
def blas_lapack_ops(self):
m = torch.randn(3, 3)
a = torch.randn(10, 3, 4)
b = torch.randn(10, 4, 3)
v = torch.randn(3)
return (
torch.addbmm(m, a, b),
torch.addmm(torch.randn(2, 3), torch.randn(2, 3),
torch.randn(3, 3)),
torch.addmv(torch.randn(2), torch.randn(2, 3), torch.randn(3)),
torch.addr(torch.zeros(3, 3), v, v),
torch.baddbmm(m, a, b),
torch.bmm(a, b),
torch.chain_matmul(torch.randn(3, 3), torch.randn(3, 3),
torch.randn(3, 3)),
# torch.cholesky(a), # deprecated
torch.cholesky_inverse(torch.randn(3, 3)),
torch.cholesky_solve(torch.randn(3, 3), torch.randn(3, 3)),
torch.dot(v, v),
torch.eig(m),
torch.geqrf(a),
torch.ger(v, v),
torch.inner(m, m),
torch.inverse(m),
torch.det(m),
torch.logdet(m),
torch.slogdet(m),
torch.lstsq(m, m),
torch.lu(m),
torch.lu_solve(m, *torch.lu(m)),
torch.lu_unpack(*torch.lu(m)),
torch.matmul(m, m),
torch.matrix_power(m, 2),
# torch.matrix_rank(m),
torch.matrix_exp(m),
torch.mm(m, m),
torch.mv(m, v),
# torch.orgqr(a, m),
# torch.ormqr(a, m, v),
torch.outer(v, v),
torch.pinverse(m),
# torch.qr(a),
torch.solve(m, m),
torch.svd(a),
# torch.svd_lowrank(a),
# torch.pca_lowrank(a),
# torch.symeig(a), # deprecated
# torch.lobpcg(a, b), # not supported
torch.trapz(m, m),
torch.trapezoid(m, m),
torch.cumulative_trapezoid(m, m),
# torch.triangular_solve(m, m),
torch.vdot(v, v),
)
def init_fusion(self):
print("Fusing BN-FC")
bn_weight_var = torch.mul(
self.batch_norm.weight.data,
torch.rsqrt(self.batch_norm.running_var + self.batch_norm.eps))
bias_coeff = self.batch_norm.bias.data - torch.mul(
self.batch_norm.running_mean, bn_weight_var)
self.linear.bias.data = torch.addmv(self.linear.bias.data,
self.linear.weight.data,
bias_coeff)
self.linear.weight.data = self.linear.weight.data * bn_weight_var.expand_as(
self.linear.weight.data)
def forward(self, input_, hx = None):
"""
An Elman RNN cell with tanh or ReLU non-linearity.
h' = tanh/relu(w_{ih} x + b_{ih} + w_{hh} h + b_{hh})
"""
# print(self.d_rec)
if hx is None:
hx = input_.new_zeros(self.hidden_size, requires_grad=False)
w_x = torch.addmv(self.bias_ih, self.weight_ih, input_)
w_h = torch.addmv(self.bias_hh, self.weight_hh, hx)
w_w = (w_x + w_h)
z = self.sigmoid(w_w[0 : self.hidden_size])
r = self.sigmoid(w_w[self.hidden_size : 2*self.hidden_size])
inp = torch.mv(self.weight_ir, input_)
prevh = torch.mv(self.weight_hr, (r * hx))
h = ((1 - z) * hx) + (z * self.relu(inp + prevh + self.bias_r))
return h, z, r
def forward(self, input_, hx=None):
if hx is None:
hx = input_.new_zeros(self.hidden_size, requires_grad=False)
# dale_hh = torch.mm(self.relu(self.weight_hh), self.d_rec)
if (self.bias):
w_x = torch.addmv(self.bias_ih, self.weight_ih, input_)
else:
w_x = torch.mv(self.weight_ih, input_)
w_w = ((self.rgate) * hx) + ((1 - (self.rgate)) * (w_x))
h = self.relu(w_w)
return h
def exact_predictive_mean(self, test_mean, test_train_covar):
"""
Computes the posterior predictive covariance of a GP
Args:
test_mean (:obj:`torch.tensor`): The test prior mean
test_train_covar (:obj:`gpytorch.lazy.LazyTensor`): Covariance matrix between test and train inputs
Returns:
:obj:`torch.tensor`: The predictive posterior mean of the test points
"""
# For efficiency - we can use addmv in the 2d case
if test_train_covar.dim() == 2:
res = torch.addmv(test_mean, delazify(test_train_covar), self.mean_cache)
# In other cases - we'll use the standard infrastructure
else:
res = (test_train_covar @ self.mean_cache.unsqueeze(-1)).squeeze(-1)
res = res + test_mean
return res
def step(self, A_batch, b_batch):
# helper variables
x = self._x
step_size = self._step_size
m = A_batch.shape[0] # number of rows = batch size
# compute linear system coefficients
P_batch = torch.addmm(torch.eye(m, dtype=A_batch.dtype), A_batch, A_batch.t(), beta=m, alpha=step_size)
rhs = torch.addmv(b_batch, A_batch, x)
# solve positive-definite linear system using Cholesky factorization
P_factor = torch.cholesky(P_batch)
rhs_chol = rhs.unsqueeze(1)
s_star = torch.cholesky_solve(rhs_chol, P_factor)
# perform step
step_dir = torch.mm(A_batch.t(), s_star)
x.sub_(step_size * step_dir.reshape(x.shape))
# return the losses w.r.t the params before making the step
return 0.5 * (rhs ** 2)
print(torch.clamp(x, min=0.5)) print(torch.clamp(x, max=0.5)) print(torch.trunc(x)) # truncated integer values print(torch.frac(x)) # fractional portion of each element print(x.add(1)) print(torch.exp(x)) print(torch.expm1(x)) print(torch.logit(x)) print(torch.mul(x, 100)) print(torch.addcdiv(t, t1, t2, value=0.1)) # t + value * t1 / t2 print(torch.addcmul(t, t1, t2, value=0.1)) # t + value * t1 * t2 print(torch.addmm(M, mat1, mat2)) # beta * M + alpha * mat1 * mat2 print(torch.matmul(mat1, mat2)) # mat1 * mat2 print(torch.mm(mat1, mat2)) # mat1 * mat2 print(torch.matrix_power(mat1, 2)) # mat1 * mat1 print(torch.addmv(x, mat1, x)) # β x+α (mat * x) print(torch.mv(mat1, x)) # mat * vec print(torch.outer(x, x)) # vec1⊗vec2 print(torch.renorm(mat1, 1, 0, 5)) input_ = torch.tensor([10000., 1e-07]) other_ = torch.tensor([10000.1, 1e-08]) print(torch.floor_divide(input_, other_)) # trunc(input_ / other_) print(torch.allclose(input_, other_)) # ∣input−other∣≤atol+rtol×∣other∣ print(torch.isclose(input_, other_)) # ∣input−other∣≤atol+rtol×∣other∣ print(mat1) print(torch.where(mat1 > 0, mat1, -mat1)) print(torch.amax(mat1, 0)) # 按列 print(torch.amax(mat1, 1)) # 按行 print(torch.max(mat1, 0)) # 按列 print(torch.max(mat1, 1)) # 按行 print(torch.argmax(mat1)) # 所有元素
# 8. Matrix, vector multiplication # ================================================================== # # Dot product dot_product_result = torch.dot(torch.Tensor([4, 2]), torch.Tensor([3, 1])) # Matrix X vector mat = torch.randn(2, 4) vec = torch.randn(4) result_1 = torch.mv(mat, vec) # Matrix + Matrix X vector M = torch.randn(2) mat = torch.randn(2, 3) vec = torch.randn(3) result_2 = torch.addmv(M, mat, vec) # Matrix, Matrix products mat1 = torch.randn(2, 3) mat2 = torch.randn(3, 4) result = torch.mm(mat1, mat2) # Outer product of vectors v1 = torch.arange(1, 4) # Size 3 v2 = torch.arange(1, 3) # Size 2 result = torch.ger(v1, v2) """ Other matrix Operations **************************************** torch.cross - cross product
def exact_posterior_mean(self, test_mean, alpha):
if isinstance(self.var, LazyVariable):
return self.var.matmul(alpha) + test_mean
return torch.addmv(test_mean, self.var, alpha)
#上三角矩阵 torch.triu(a) #对矩阵mat1和mat2进行矩阵乘操作。矩阵mat加到最终结果。如果mat1 是一个 n×m张量，mat2 是一个 m×p张量， # 那么out和mat的形状为n×p。 alpha 和 beta 分别是两个矩阵 mat1@mat2和mat的比例因子，即out=(beta∗M)+(alpha∗mat1@mat2) M = torch.ones(2, 2) mat1 = torch.Tensor([[1, 2], [3, 4]]) mat2 = torch.Tensor([[1, 2], [3, 4]]) torch.addmm(M, mat1, mat2) #torch.addmv,对矩阵mat和向量vec对进行相乘操作。向量tensor加到最终结果。如果mat 是一个 n×m维矩阵，vec 是一个 m维向量， # 那么out和mat的为n元向量。 可选参数_alpha_ 和 beta 分别是 mat∗vec和mat的比例因子，即out=(beta∗tensor)+(alpha∗(mat@vec)) M = torch.randn(2) mat = torch.randn(2, 3) vec = torch.randn(3) torch.addmv(M, mat, vec) #点乘 torch.dot(torch.Tensor([1, 2]), torch.Tensor([1, 2])) #特征分解 torch.eig(torch.randn(4, 4), eigenvectors=True) #最小二乘解 A = torch.Tensor([[1, 1, 1], [2, 3, 4], [3, 5, 2], [4, 2, 5], [5, 4, 3]]) B = torch.Tensor([[-10, -3], [12, 14], [14, 12], [16, 16], [18, 16]]) X, _ = torch.gels(B, A) X #计算线性方程组AX=B的解 A = torch.Tensor([[6.80, -2.11, 5.66, 5.97, 8.23],
def forward(self, x, y, z):
out1 = x.addmv(y, z, beta=0.1, alpha=0.2)
out2 = torch.addmv(x, y, z, beta=0.1, alpha=0.2)
return out1, out2
