class OptNetLatent(nn.Module):
def __init__(self, n, Qpenalty, nLatent, nineq, trueInit=False):
super().__init__()
nx = (n**2)**3
self.fc_in = nn.Linear(nx, nLatent)
self.Q = Variable(Qpenalty * torch.eye(nLatent).cuda())
self.G = Parameter(torch.Tensor(nineq, nLatent).uniform_(-1, 1).cuda())
self.z = Parameter(torch.zeros(nLatent).cuda())
self.s = Parameter(torch.ones(nineq).cuda())
self.fc_out = nn.Linear(nLatent, nx)
def forward(self, puzzles):
nBatch = puzzles.size(0)
x = puzzles.view(nBatch, -1)
x = self.fc_in(x)
e = Variable(torch.Tensor())
h = self.G.mv(self.z) + self.s
x = QPFunction(verbose=False)(
self.Q,
x,
self.G,
h,
e,
e,
)
x = self.fc_out(x)
x = x.view_as(puzzles)
return x
class OptNetIneq(nn.Module):
def __init__(self, n, Qpenalty, nineq):
super().__init__()
nx = (n**2)**3
self.Q = Variable(Qpenalty * torch.eye(nx).double().cuda())
self.G1 = Variable(-torch.eye(nx).double().cuda())
self.h1 = Variable(torch.zeros(nx).double().cuda())
# if trueInit:
# self.A = Parameter(torch.DoubleTensor(get_sudoku_matrix(n)).cuda())
# else:
# # t = get_sudoku_matrix(n)
# # self.A = Parameter(torch.rand(t.shape).double().cuda())
# # import IPython, sys; IPython.embed(); sys.exit(-1)
self.A = Parameter(torch.rand(50, nx).double().cuda())
self.G2 = Parameter(
torch.Tensor(128, nx).uniform_(-1, 1).double().cuda())
self.z2 = Parameter(torch.zeros(nx).double().cuda())
self.s2 = Parameter(torch.ones(128).double().cuda())
# self.b = Variable(torch.ones(self.A.size(0)).double().cuda())
def forward(self, puzzles):
nBatch = puzzles.size(0)
p = -puzzles.view(nBatch, -1)
h2 = self.G2.mv(self.z2) + self.s2
G = torch.cat((self.G1, self.G2), 0)
h = torch.cat((self.h1, h2), 0)
e = Variable(torch.Tensor())
return QPFunction(verbose=False)(self.Q, p.double(), G, h, e,
e).float().view_as(puzzles)
class OptNet(nn.Module):
def __init__(self, nFeatures, nHidden, nCls, bn, nineq=1, neq=0, eps=1e-4):
super().__init__()
self.nFeatures = nFeatures
self.nHidden = nHidden
self.bn = bn
self.nCls = nCls
self.nineq = nineq
self.neq = neq
self.eps = eps
if bn:
self.bn1 = nn.BatchNorm1d(nHidden)
self.bn2 = nn.BatchNorm1d(nCls)
self.fc1 = nn.Linear(nFeatures, nHidden)
self.fc2 = nn.Linear(nHidden, nCls)
self.M = Variable(torch.tril(torch.ones(nCls, nCls)).cuda())
self.L = Parameter(torch.tril(torch.rand(nCls, nCls).cuda()))
self.G = Parameter(torch.Tensor(nineq, nCls).uniform_(-1, 1).cuda())
self.z0 = Parameter(torch.zeros(nCls).cuda())
self.s0 = Parameter(torch.ones(nineq).cuda())
print("M=")
print(self.M)
print("L=")
print(self.L)
print("G=")
print(self.G)
print("z0=")
print(self.z0)
print("s0=")
print(self.s0)
def forward(self, x):
nBatch = x.size(0)
# FC-ReLU-(BN)-FC-ReLU-(BN)-QP-Softmax
x = x.view(nBatch, -1)
x = F.relu(self.fc1(x))
if self.bn:
x = self.bn1(x)
x = F.relu(self.fc2(x))
if self.bn:
x = self.bn2(x)
L = self.M * self.L
Q = L.mm(L.t()) + self.eps * Variable(torch.eye(self.nCls)).cuda()
h = self.G.mv(self.z0) + self.s0
e = Variable(torch.Tensor())
x = QPFunction(verbose=False)(Q, x, G, h, e, e)
return F.log_softmax(x)
class MatchingLSTMCell(RNNCellBase):
# TODO: xz. modify this description, this is copied from LSTMCell
r"""A long short-term memory (LSTM) cell.
.. math::
\begin{array}{ll}
i = sigmoid(W_{mi} x + b_{ii} + W_{hi} h + b_{hi}) \\
f = sigmoid(W_{if} x + b_{if} + W_{hf} h + b_{hf}) \\
g = \tanh(W_{ig} x + b_{ig} + W_{hc} h + b_{hg}) \\
o = sigmoid(W_{io} x + b_{io} + W_{ho} h + b_{ho}) \\
c' = f * c + i * g \\
h' = o * \tanh(c_t) \\
\end{array}
Args:
input_size: The number of expected features in the input x
hidden_size: The number of features in the hidden state h
bias: If `False`, then the layer does not use bias weights `b_ih` and `b_hh`. Default: True
"""
def __init__(self, input_size, hidden_size):
super(MatchingLSTMCell, self).__init__()
self.input_size = input_size
self.hidden_size = hidden_size
# 4 for c, i, f, o
self.weight_ih = Parameter(torch.Tensor(4*hidden_size, input_size))
self.weight_hh = Parameter(torch.Tensor(4*hidden_size, hidden_size))
self.bias_ih = Parameter(torch.Tensor(4*hidden_size))
self.reset_parameters()
def reset_parameters(self):
stdv = 1.0 / math.sqrt(self.hidden_size)
for weight in self.parameters():
weight.data.uniform_(-stdv, stdv)
def forward(self, input, hx, cx):
gates = self.weight_ih.mv(input) + self.weight_hh.mv(hx) + self.bias_ih
cgate, igate, fgate, ogate = gates.chunk(4, 0)
# non linear
cgate = F.tanh(cgate)
igate = F.sigmoid(igate)
fgate = F.sigmoid(fgate)
ogate = F.sigmoid(ogate)
cy = (fgate * cx) + (igate * cgate)
hy = ogate * F.tanh(cy)
return hy, cy
class OptNetEq(nn.Module):
def __init__(self, n, Qpenalty, qp_solver, trueInit=False):
super().__init__()
self.qp_solver = qp_solver
nx = (n**2)**3
self.Q = Variable(Qpenalty * torch.eye(nx).double().cuda())
self.Q_idx = spa.csc_matrix(self.Q.detach().cpu().numpy()).nonzero()
self.G = Variable(-torch.eye(nx).double().cuda())
self.h = Variable(torch.zeros(nx).double().cuda())
t = get_sudoku_matrix(n)
if trueInit:
self.A = Parameter(torch.DoubleTensor(t).cuda())
else:
self.A = Parameter(torch.rand(t.shape).double().cuda())
self.log_z0 = Parameter(torch.zeros(nx).double().cuda())
# self.b = Variable(torch.ones(self.A.size(0)).double().cuda())
if self.qp_solver == 'osqpth':
t = torch.cat((self.A, self.G), dim=0)
self.AG_idx = spa.csc_matrix(t.detach().cpu().numpy()).nonzero()
# @profile
def forward(self, puzzles):
nBatch = puzzles.size(0)
p = -puzzles.view(nBatch, -1)
b = self.A.mv(self.log_z0.exp())
if self.qp_solver == 'qpth':
y = QPFunction(verbose=-1)(self.Q, p.double(), self.G, self.h,
self.A, b).float().view_as(puzzles)
elif self.qp_solver == 'osqpth':
_l = torch.cat((b,
torch.full(self.h.shape,
float('-inf'),
device=self.h.device,
dtype=self.h.dtype)),
dim=0)
_u = torch.cat((b, self.h), dim=0)
Q_data = self.Q[self.Q_idx[0], self.Q_idx[1]]
AG = torch.cat((self.A, self.G), dim=0)
AG_data = AG[self.AG_idx[0], self.AG_idx[1]]
y = OSQP(self.Q_idx,
self.Q.shape,
self.AG_idx,
AG.shape,
diff_mode=DiffModes.FULL)(Q_data, p.double(), AG_data, _l,
_u).float().view_as(puzzles)
else:
assert False
return y
class OptNet(nn.Module):
def __init__(self, nFeatures, args):
super(OptNet, self).__init__()
nHidden, neq, nineq = 2 * nFeatures - 1, 0, 2 * nFeatures - 2
assert (neq == 0)
self.fc1 = nn.Linear(nFeatures, nHidden)
self.M = Variable(torch.tril(torch.ones(nHidden, nHidden)).cuda())
if args.tvInit:
Q = 1e-8 * torch.eye(nHidden)
Q[:nFeatures, :nFeatures] = torch.eye(nFeatures)
self.L = Parameter(torch.potrf(Q))
D = torch.zeros(nFeatures - 1, nFeatures)
D[:nFeatures - 1, :nFeatures - 1] = torch.eye(nFeatures - 1)
D[:nFeatures - 1, 1:nFeatures] -= torch.eye(nFeatures - 1)
G_ = block(((D, -torch.eye(nFeatures - 1)),
(-D, -torch.eye(nFeatures - 1))))
self.G = Parameter(G_)
self.s0 = Parameter(
torch.ones(2 * nFeatures - 2) +
1e-6 * torch.randn(2 * nFeatures - 2))
G_pinv = (G_.t().mm(G_) + 1e-5 * torch.eye(nHidden)).inverse().mm(
G_.t())
self.z0 = Parameter(-G_pinv.mv(self.s0.data) +
1e-6 * torch.randn(nHidden))
lam = 21.21
W_fc1, b_fc1 = self.fc1.weight, self.fc1.bias
W_fc1.data[:, :] = 1e-3 * torch.randn(
(2 * nFeatures - 1, nFeatures))
# W_fc1.data[:,:] = 0.0
W_fc1.data[:nFeatures, :nFeatures] += -torch.eye(nFeatures)
# b_fc1.data[:] = torch.zeros(2*nFeatures-1)
b_fc1.data[:] = 0.0
b_fc1.data[nFeatures:2 * nFeatures - 1] = lam
else:
self.L = Parameter(torch.tril(torch.rand(nHidden, nHidden)))
self.G = Parameter(torch.Tensor(nineq, nHidden).uniform_(-1, 1))
self.z0 = Parameter(torch.zeros(nHidden))
self.s0 = Parameter(torch.ones(nineq))
self.nFeatures = nFeatures
self.nHidden = nHidden
self.neq = neq
self.nineq = nineq
self.args = args
def cuda(self):
# TODO: Is there a more automatic way?
for x in [self.L, self.G, self.z0, self.s0]:
x.data = x.data.cuda()
return super().cuda()
def forward(self, x):
nBatch = x.size(0)
x = self.fc1(x)
L = self.M * self.L
Q = L.mm(
L.t()) + self.args.eps * Variable(torch.eye(self.nHidden)).cuda()
Q = Q.unsqueeze(0).expand(nBatch, self.nHidden, self.nHidden)
G = self.G.unsqueeze(0).expand(nBatch, self.nineq, self.nHidden)
h = self.G.mv(self.z0) + self.s0
h = h.unsqueeze(0).expand(nBatch, self.nineq)
e = Variable(torch.Tensor())
x = QPFunction()(Q, x, G, h, e, e)
x = x[:, :self.nFeatures]
return x
class OptNet(nn.Module):
def __init__(self, nFeatures, nHidden, nCls, bn, nineq=1, neq=0, eps=1e-4):
super(OptNet, self).__init__()
self.device = torch.device("cuda")
self.nFeatures = nFeatures
self.nHidden = nHidden
self.bn = bn
self.nCls = nCls
self.nineq = nineq
self.neq = neq
self.eps = eps
if bn:
self.bn1 = nn.BatchNorm1d(nHidden)
self.bn2 = nn.BatchNorm1d(nCls)
self.fc1 = nn.Linear(nFeatures, nHidden)
self.fc2 = nn.Linear(nHidden, nCls)
X = torch.tril(torch.ones(nCls, nCls))
self.M = Variable(X.cuda())
self.L = Parameter(torch.tril(torch.rand(nCls, nCls).cuda()))
self.p = Parameter(torch.Tensor(1, nCls).uniform_(-1, 1).cuda())
self.G = Parameter(torch.Tensor(nineq, nCls).uniform_(-1, 1).cuda())
#self.A =Parameter(torch.Tensor(neq,nCls).uniform_(-1,1).cuda())
#self.b=
self.z0 = Parameter(torch.zeros(nCls).cuda())
self.s0 = Parameter(torch.ones(nineq).cuda())
def forward(self, x):
nBatch = x.size(0)
# FC-ReLU-(BN)-FC-ReLU-(BN)-QP-Softmax
x = x.view(nBatch, -1)
x = x.unsqueeze(0)
x = x.float()
tmp = self.fc1(x)
x = F.relu(tmp)
x = x.squeeze(2)
#if self.bn:
#x = self.bn1(x)
#x = F.relu(self.fc2(x))
#if self.bn:
#x = self.bn2(x)
L = self.M * self.L
Q = L.mm(L.t()) + self.eps * Variable(torch.eye(self.nCls)).cuda()
p = self.p.double()
h = self.G.mv(self.z0) + self.s0
G = self.G.double()
Q = Q.double()
h = h.double()
print(Q.size(), p.size(), G.size(), h.size())
e = Variable(torch.Tensor())
x = QPFunction(verbose=True)(Q, p, G, h, e, e).cuda()
print(x)
return F.log_softmax(x, dim=1)
