def _build_net(self): return (nn.Sequential() .add(nn.Concat(0) .add(nn.Linear(2, 5)) .add(nn.Linear(2, 5))) .add(nn.ReLU()) .add(nn.Linear(10, 20)))
def build_net(self): # [1.0] first layer first_layer = nn.ConcatTable() # [1.1] feed forward neural net, produce v1, v2 feedforward = nn.Sequential() feedforward.add(nn.Linear(self.input_size, self.hidden_layer_size)) feedforward.add(nn.PReLU()) # add hidden layers for i in range(self.hidden_layer_count - 1): feedforward.add( nn.Linear(self.hidden_layer_size, self.hidden_layer_size)) feedforward.add(nn.PReLU()) feedforward.add(nn.Linear(self.hidden_layer_size, self.output_size)) # [1.2] right part, discard pot_size, produce r1, r2 right_part = nn.Sequential() right_part.add(nn.Narrow(1, 0, self.output_size)) first_layer.add(feedforward) first_layer.add(right_part) # [2.0] outer net force counterfactual values satisfy 0-sum property second_layer = nn.ConcatTable() # accept v1,v2; ignore r1, r2 left_part2 = nn.Sequential() left_part2.add(nn.SelectTable(0)) # accept, r1,r2, v1,v2; produce -0.5k=-0.5(r1v1 + r2v2) right_part2 = nn.Sequential() right_part2.add(nn.DotProduct()) right_part2.add(nn.Unsqueeze(1)) right_part2.add(nn.Replicate(self.output_size, 1)) right_part2.add(nn.Squeeze(2)) right_part2.add(nn.MulConstant(-0.5)) second_layer.add(left_part2) second_layer.add(right_part2) final_mlp = nn.Sequential() final_mlp.add(first_layer) final_mlp.add(second_layer) # accept v1,v2 and -0.5k, product v1-0.5k, v2-0.5k final_mlp.add(nn.CAddTable()) return final_mlp
def test_Concat(self): input = torch.randn(4, 2) num_modules = random.randint(2, 5) linears = [nn.Linear(2, 5) for i in range(num_modules)] m = nn.Concat(0) for l in linears: m.add(l) l.zeroGradParameters() l.weight.fill_(1) l.bias.fill_(0) # Check that these don't raise errors m.__repr__() str(m) output = m.forward(input) output2 = input.sum(1).expand(4, 5).repeat(num_modules, 1) self.assertEqual(output2, output) gradInput = m.backward(input, torch.ones(output2.size())) gradInput2 = torch.ones(4, 2).fill_(num_modules * 5) self.assertEqual(gradInput, gradInput2) gradWeight = input.sum(0).expand(5, 2) for l in linears: self.assertEqual(gradWeight, l.gradWeight)
def run_multi_embedding_test(module_type: MultiEmbModuleType, conf: MultiEmbConfig, batch_size, scenario: MultiEmbScenario, device, num_iters=10): dims = conf.value sizes = ([1000, 5000, 10000, 50000] * 10)[:len(conf.value)] if module_type == MultiEmbModuleType.FAST_MULTI: emb = FastMultiEmbedding(sizes, dims).to(device) elif module_type == MultiEmbModuleType.FAST: embs = [FastEmbedding(size, dim).to(device) for size, dim in zip(sizes, dims)] def emb(multi_indices): return torch.cat([e(multi_indices[:, idx]) for (idx, e) in enumerate(embs)], 1) else: embs = [nn.Embedding(size, dim).to(device) for size, dim in zip(sizes, dims)] def emb(multi_indices): return torch.cat([e(multi_indices[:, idx]) for (idx, e) in enumerate(embs)], 1) linear = nn.Linear(sum(dims), 1).to(device) indices = torch.cat([create_indices(batch_size, num_embs, scenario.value[1](idx, dim)).view(-1, 1) for idx, (dim, num_embs) in enumerate(zip(dims, sizes))], 1).to(device) on_cuda = device.type == 'cuda' def run(): x = emb(indices) linear.forward(x).sum().backward() if on_cuda: torch.cuda.synchronize() return measure_execution_time(num_iters, run)
def run_embedding_test(emb_module, num_embeddings, embedding_dim, batch_size, scenario: EmbScenario, device, num_iters=10): emb = emb_module(num_embeddings, embedding_dim).to(device) linear = nn.Linear(embedding_dim, 1).to(device) indices = create_indices(batch_size, num_embeddings, scenario).to(device) on_cuda = device.type == 'cuda' def run(): x = emb.forward(indices) linear.forward(x).sum().backward() if on_cuda: torch.cuda.synchronize() return measure_execution_time(num_iters, run)
) else 'torch.FloatTensor' # two-dimensional SparseConvNet model = nn.Sequential() sparseModel = scn.Sequential() denseModel = nn.Sequential() model.add(sparseModel).add(denseModel) sparseModel.add(scn.ValidConvolution(2, 3, 16, 3, False)) sparseModel.add(scn.MaxPooling(2, 3, 2)) sparseModel.add( scn.SparseResNet( 2, 16, [['b', 16, 2, 1], ['b', 32, 2, 2], ['b', 48, 2, 2], ['b', 96, 2, 2]])) sparseModel.add(scn.Convolution(2, 96, 128, 4, 1, False)) sparseModel.add(scn.BatchNormReLU(128)) sparseModel.add(scn.SparseToDense(2)) denseModel.add(nn.View(-1, 128)) denseModel.add(nn.Linear(128, 3755)) model.type(dtype) print(model) spatial_size = sparseModel.suggestInputSize(torch.LongTensor([1, 1])) print('input spatial size', spatial_size) dataset = getIterators(spatial_size, 63, 3) scn.ClassificationTrainValidate(model, dataset, { 'nEpochs': 100, 'initial_LR': 0.1, 'LR_decay': 0.05, 'weightDecay': 1e-4 })
denseModel = nn.Sequential() model.add(sparseModel).add(denseModel) sparseModel.add(scn.ValidConvolution(2, 3, 16, 3, False))\ .add(scn.SparseDenseNet(2, 16, [ {'pool': 'MP', 'compression': 0}, {'nExtraLayers': 2, 'growthRate': 16}, {'pool': 'BN-R-C-AP', 'compression': 0}, {'nExtraLayers': 2, 'growthRate': 16}, {'pool': 'BN-R-C-AP', 'compression': 0}, {'nExtraLayers': 2, 'growthRate': 16}, {'pool': 'BN-R-C-AP', 'compression': 0}, {'nExtraLayers': 2, 'growthRate': 16}])) n_out = sparseModel.modules[-1].nOutputPlanes sparseModel.add(scn.Convolution(2, n_out, 256, 4, 1, False)) sparseModel.add(scn.BatchNormReLU(256)) sparseModel.add(scn.SparseToDense(2)) denseModel.add(nn.View(-1, 256)) denseModel.add(nn.Linear(256, 3755)) model.type(dtype) print(model) spatial_size = sparseModel.suggestInputSize(torch.LongTensor([1, 1])) print('input spatial size', spatial_size) dataset = getIterators(spatial_size, 64, 2) scn.ClassificationTrainValidate(model, dataset, { 'nEpochs': 100, 'initial_LR': 0.1, 'LR_decay': 0.05, 'weightDecay': 1e-4 })
def test_linear(self): self.input = self.input.flatten() input_size = self.input.shape[0] self._test_single_layer(nn.Linear(input_size, 3, True), decimal=5)
dtype = 'torch.cuda.FloatTensor' # Uncomment this to run on GPU # two-dimensional SparseConvNet model = nn.Sequential() sparseModel = scn.Sequential() denseModel = nn.Sequential() model.add(sparseModel).add(denseModel) sparseModel.add( scn.SparseVggNet(2, 3, [[ 'C', 8, ], ['C', 8], 'MP', ['C', 16], ['C', 16], 'MP', ['C', 16, 8], ['C', 16, 8], 'MP', ['C', 24, 8], ['C', 24, 8], 'MP'])) sparseModel.add(scn.Convolution(2, 32, 64, 5, 1, False)) sparseModel.add(scn.BatchNormReLU(64)) sparseModel.add(scn.SparseToDense(2)) denseModel.add(nn.View(-1, 64)) denseModel.add(nn.Linear(64, 183)) model.type(dtype) print(model) spatial_size = sparseModel.suggestInputSize(torch.LongTensor([1, 1])) print('input spatial size', spatial_size) dataset = getIterators(spatial_size, 63, 3) scn.ClassificationTrainValidate(model, dataset, { 'nEpochs': 100, 'initial_LR': 0.1, 'LR_decay': 0.05, 'weightDecay': 1e-4 })
nn.Conv2d(32, 96, (5, 5), (1, 1), (2, 2)), nn.ReLU(), nn.MaxPool2d((3, 3), (1, 1), (1, 1), ceil_mode=True), nn.Conv2d(256, 64, (1, 1)), nn.ReLU(), nn.MaxPool2d((3, 3), (2, 2), (0, 0), ceil_mode=True), nn.Conv2d(480, 192, (1, 1)), nn.ReLU(), nn.Conv2d(480, 96, (1, 1)), nn.ReLU(), nn.Conv2d(96, 208, (3, 3), (1, 1), (1, 1)), nn.ReLU(), nn.Conv2d(480, 16, (1, 1)), nn.ReLU(), nn.Conv2d(16, 48, (5, 5), (1, 1), (2, 2)), nn.ReLU(), nn.MaxPool2d((3, 3), (1, 1), (1, 1), ceil_mode=True), nn.Conv2d(480, 64, (1, 1)), nn.ReLU(), nn.AvgPool2d((5, 5), (3, 3), (0, 0), ceil_mode=True), #AvgPool2d, nn.Conv2d(512, 128, (1, 1)), nn.ReLU(), Lambda(lambda x: x.view(x.size(0), -1)), # View, nn.Sequential(Lambda(lambda x: x.view(1, -1) if 1 == len(x.size()) else x), nn.Linear(2048, 1024)), # Linear, nn.ReLU(), nn.Dropout(0.7), nn.Sequential(Lambda(lambda x: x.view(1, -1) if 1 == len(x.size()) else x), nn.Linear(1024, 365)), # Linear, )