class OptNetLatent(nn.Module): def __init__(self, n, Qpenalty, nLatent, nineq, trueInit=False): super().__init__() nx = (n**2)**3 self.fc_in = nn.Linear(nx, nLatent) self.Q = Variable(Qpenalty * torch.eye(nLatent).cuda()) self.G = Parameter(torch.Tensor(nineq, nLatent).uniform_(-1, 1).cuda()) self.z = Parameter(torch.zeros(nLatent).cuda()) self.s = Parameter(torch.ones(nineq).cuda()) self.fc_out = nn.Linear(nLatent, nx) def forward(self, puzzles): nBatch = puzzles.size(0) x = puzzles.view(nBatch, -1) x = self.fc_in(x) e = Variable(torch.Tensor()) h = self.G.mv(self.z) + self.s x = QPFunction(verbose=False)( self.Q, x, self.G, h, e, e, ) x = self.fc_out(x) x = x.view_as(puzzles) return x
class OptNetIneq(nn.Module): def __init__(self, n, Qpenalty, nineq): super().__init__() nx = (n**2)**3 self.Q = Variable(Qpenalty * torch.eye(nx).double().cuda()) self.G1 = Variable(-torch.eye(nx).double().cuda()) self.h1 = Variable(torch.zeros(nx).double().cuda()) # if trueInit: # self.A = Parameter(torch.DoubleTensor(get_sudoku_matrix(n)).cuda()) # else: # # t = get_sudoku_matrix(n) # # self.A = Parameter(torch.rand(t.shape).double().cuda()) # # import IPython, sys; IPython.embed(); sys.exit(-1) self.A = Parameter(torch.rand(50, nx).double().cuda()) self.G2 = Parameter( torch.Tensor(128, nx).uniform_(-1, 1).double().cuda()) self.z2 = Parameter(torch.zeros(nx).double().cuda()) self.s2 = Parameter(torch.ones(128).double().cuda()) # self.b = Variable(torch.ones(self.A.size(0)).double().cuda()) def forward(self, puzzles): nBatch = puzzles.size(0) p = -puzzles.view(nBatch, -1) h2 = self.G2.mv(self.z2) + self.s2 G = torch.cat((self.G1, self.G2), 0) h = torch.cat((self.h1, h2), 0) e = Variable(torch.Tensor()) return QPFunction(verbose=False)(self.Q, p.double(), G, h, e, e).float().view_as(puzzles)
class OptNet(nn.Module): def __init__(self, nFeatures, nHidden, nCls, bn, nineq=1, neq=0, eps=1e-4): super().__init__() self.nFeatures = nFeatures self.nHidden = nHidden self.bn = bn self.nCls = nCls self.nineq = nineq self.neq = neq self.eps = eps if bn: self.bn1 = nn.BatchNorm1d(nHidden) self.bn2 = nn.BatchNorm1d(nCls) self.fc1 = nn.Linear(nFeatures, nHidden) self.fc2 = nn.Linear(nHidden, nCls) self.M = Variable(torch.tril(torch.ones(nCls, nCls)).cuda()) self.L = Parameter(torch.tril(torch.rand(nCls, nCls).cuda())) self.G = Parameter(torch.Tensor(nineq, nCls).uniform_(-1, 1).cuda()) self.z0 = Parameter(torch.zeros(nCls).cuda()) self.s0 = Parameter(torch.ones(nineq).cuda()) print("M=") print(self.M) print("L=") print(self.L) print("G=") print(self.G) print("z0=") print(self.z0) print("s0=") print(self.s0) def forward(self, x): nBatch = x.size(0) # FC-ReLU-(BN)-FC-ReLU-(BN)-QP-Softmax x = x.view(nBatch, -1) x = F.relu(self.fc1(x)) if self.bn: x = self.bn1(x) x = F.relu(self.fc2(x)) if self.bn: x = self.bn2(x) L = self.M * self.L Q = L.mm(L.t()) + self.eps * Variable(torch.eye(self.nCls)).cuda() h = self.G.mv(self.z0) + self.s0 e = Variable(torch.Tensor()) x = QPFunction(verbose=False)(Q, x, G, h, e, e) return F.log_softmax(x)
class MatchingLSTMCell(RNNCellBase): # TODO: xz. modify this description, this is copied from LSTMCell r"""A long short-term memory (LSTM) cell. .. math:: \begin{array}{ll} i = sigmoid(W_{mi} x + b_{ii} + W_{hi} h + b_{hi}) \\ f = sigmoid(W_{if} x + b_{if} + W_{hf} h + b_{hf}) \\ g = \tanh(W_{ig} x + b_{ig} + W_{hc} h + b_{hg}) \\ o = sigmoid(W_{io} x + b_{io} + W_{ho} h + b_{ho}) \\ c' = f * c + i * g \\ h' = o * \tanh(c_t) \\ \end{array} Args: input_size: The number of expected features in the input x hidden_size: The number of features in the hidden state h bias: If `False`, then the layer does not use bias weights `b_ih` and `b_hh`. Default: True """ def __init__(self, input_size, hidden_size): super(MatchingLSTMCell, self).__init__() self.input_size = input_size self.hidden_size = hidden_size # 4 for c, i, f, o self.weight_ih = Parameter(torch.Tensor(4*hidden_size, input_size)) self.weight_hh = Parameter(torch.Tensor(4*hidden_size, hidden_size)) self.bias_ih = Parameter(torch.Tensor(4*hidden_size)) self.reset_parameters() def reset_parameters(self): stdv = 1.0 / math.sqrt(self.hidden_size) for weight in self.parameters(): weight.data.uniform_(-stdv, stdv) def forward(self, input, hx, cx): gates = self.weight_ih.mv(input) + self.weight_hh.mv(hx) + self.bias_ih cgate, igate, fgate, ogate = gates.chunk(4, 0) # non linear cgate = F.tanh(cgate) igate = F.sigmoid(igate) fgate = F.sigmoid(fgate) ogate = F.sigmoid(ogate) cy = (fgate * cx) + (igate * cgate) hy = ogate * F.tanh(cy) return hy, cy
class OptNetEq(nn.Module): def __init__(self, n, Qpenalty, qp_solver, trueInit=False): super().__init__() self.qp_solver = qp_solver nx = (n**2)**3 self.Q = Variable(Qpenalty * torch.eye(nx).double().cuda()) self.Q_idx = spa.csc_matrix(self.Q.detach().cpu().numpy()).nonzero() self.G = Variable(-torch.eye(nx).double().cuda()) self.h = Variable(torch.zeros(nx).double().cuda()) t = get_sudoku_matrix(n) if trueInit: self.A = Parameter(torch.DoubleTensor(t).cuda()) else: self.A = Parameter(torch.rand(t.shape).double().cuda()) self.log_z0 = Parameter(torch.zeros(nx).double().cuda()) # self.b = Variable(torch.ones(self.A.size(0)).double().cuda()) if self.qp_solver == 'osqpth': t = torch.cat((self.A, self.G), dim=0) self.AG_idx = spa.csc_matrix(t.detach().cpu().numpy()).nonzero() # @profile def forward(self, puzzles): nBatch = puzzles.size(0) p = -puzzles.view(nBatch, -1) b = self.A.mv(self.log_z0.exp()) if self.qp_solver == 'qpth': y = QPFunction(verbose=-1)(self.Q, p.double(), self.G, self.h, self.A, b).float().view_as(puzzles) elif self.qp_solver == 'osqpth': _l = torch.cat((b, torch.full(self.h.shape, float('-inf'), device=self.h.device, dtype=self.h.dtype)), dim=0) _u = torch.cat((b, self.h), dim=0) Q_data = self.Q[self.Q_idx[0], self.Q_idx[1]] AG = torch.cat((self.A, self.G), dim=0) AG_data = AG[self.AG_idx[0], self.AG_idx[1]] y = OSQP(self.Q_idx, self.Q.shape, self.AG_idx, AG.shape, diff_mode=DiffModes.FULL)(Q_data, p.double(), AG_data, _l, _u).float().view_as(puzzles) else: assert False return y
class OptNet(nn.Module): def __init__(self, nFeatures, args): super(OptNet, self).__init__() nHidden, neq, nineq = 2 * nFeatures - 1, 0, 2 * nFeatures - 2 assert (neq == 0) self.fc1 = nn.Linear(nFeatures, nHidden) self.M = Variable(torch.tril(torch.ones(nHidden, nHidden)).cuda()) if args.tvInit: Q = 1e-8 * torch.eye(nHidden) Q[:nFeatures, :nFeatures] = torch.eye(nFeatures) self.L = Parameter(torch.potrf(Q)) D = torch.zeros(nFeatures - 1, nFeatures) D[:nFeatures - 1, :nFeatures - 1] = torch.eye(nFeatures - 1) D[:nFeatures - 1, 1:nFeatures] -= torch.eye(nFeatures - 1) G_ = block(((D, -torch.eye(nFeatures - 1)), (-D, -torch.eye(nFeatures - 1)))) self.G = Parameter(G_) self.s0 = Parameter( torch.ones(2 * nFeatures - 2) + 1e-6 * torch.randn(2 * nFeatures - 2)) G_pinv = (G_.t().mm(G_) + 1e-5 * torch.eye(nHidden)).inverse().mm( G_.t()) self.z0 = Parameter(-G_pinv.mv(self.s0.data) + 1e-6 * torch.randn(nHidden)) lam = 21.21 W_fc1, b_fc1 = self.fc1.weight, self.fc1.bias W_fc1.data[:, :] = 1e-3 * torch.randn( (2 * nFeatures - 1, nFeatures)) # W_fc1.data[:,:] = 0.0 W_fc1.data[:nFeatures, :nFeatures] += -torch.eye(nFeatures) # b_fc1.data[:] = torch.zeros(2*nFeatures-1) b_fc1.data[:] = 0.0 b_fc1.data[nFeatures:2 * nFeatures - 1] = lam else: self.L = Parameter(torch.tril(torch.rand(nHidden, nHidden))) self.G = Parameter(torch.Tensor(nineq, nHidden).uniform_(-1, 1)) self.z0 = Parameter(torch.zeros(nHidden)) self.s0 = Parameter(torch.ones(nineq)) self.nFeatures = nFeatures self.nHidden = nHidden self.neq = neq self.nineq = nineq self.args = args def cuda(self): # TODO: Is there a more automatic way? for x in [self.L, self.G, self.z0, self.s0]: x.data = x.data.cuda() return super().cuda() def forward(self, x): nBatch = x.size(0) x = self.fc1(x) L = self.M * self.L Q = L.mm( L.t()) + self.args.eps * Variable(torch.eye(self.nHidden)).cuda() Q = Q.unsqueeze(0).expand(nBatch, self.nHidden, self.nHidden) G = self.G.unsqueeze(0).expand(nBatch, self.nineq, self.nHidden) h = self.G.mv(self.z0) + self.s0 h = h.unsqueeze(0).expand(nBatch, self.nineq) e = Variable(torch.Tensor()) x = QPFunction()(Q, x, G, h, e, e) x = x[:, :self.nFeatures] return x
class OptNet(nn.Module): def __init__(self, nFeatures, nHidden, nCls, bn, nineq=1, neq=0, eps=1e-4): super(OptNet, self).__init__() self.device = torch.device("cuda") self.nFeatures = nFeatures self.nHidden = nHidden self.bn = bn self.nCls = nCls self.nineq = nineq self.neq = neq self.eps = eps if bn: self.bn1 = nn.BatchNorm1d(nHidden) self.bn2 = nn.BatchNorm1d(nCls) self.fc1 = nn.Linear(nFeatures, nHidden) self.fc2 = nn.Linear(nHidden, nCls) X = torch.tril(torch.ones(nCls, nCls)) self.M = Variable(X.cuda()) self.L = Parameter(torch.tril(torch.rand(nCls, nCls).cuda())) self.p = Parameter(torch.Tensor(1, nCls).uniform_(-1, 1).cuda()) self.G = Parameter(torch.Tensor(nineq, nCls).uniform_(-1, 1).cuda()) #self.A =Parameter(torch.Tensor(neq,nCls).uniform_(-1,1).cuda()) #self.b= self.z0 = Parameter(torch.zeros(nCls).cuda()) self.s0 = Parameter(torch.ones(nineq).cuda()) def forward(self, x): nBatch = x.size(0) # FC-ReLU-(BN)-FC-ReLU-(BN)-QP-Softmax x = x.view(nBatch, -1) x = x.unsqueeze(0) x = x.float() tmp = self.fc1(x) x = F.relu(tmp) x = x.squeeze(2) #if self.bn: #x = self.bn1(x) #x = F.relu(self.fc2(x)) #if self.bn: #x = self.bn2(x) L = self.M * self.L Q = L.mm(L.t()) + self.eps * Variable(torch.eye(self.nCls)).cuda() p = self.p.double() h = self.G.mv(self.z0) + self.s0 G = self.G.double() Q = Q.double() h = h.double() print(Q.size(), p.size(), G.size(), h.size()) e = Variable(torch.Tensor()) x = QPFunction(verbose=True)(Q, p, G, h, e, e).cuda() print(x) return F.log_softmax(x, dim=1)