def _build_net(self):
return (nn.Sequential()
.add(nn.Concat(0)
.add(nn.Linear(2, 5))
.add(nn.Linear(2, 5)))
.add(nn.ReLU())
.add(nn.Linear(10, 20)))
def build_net(self):
# [1.0] first layer
first_layer = nn.ConcatTable()
# [1.1] feed forward neural net, produce v1, v2
feedforward = nn.Sequential()
feedforward.add(nn.Linear(self.input_size, self.hidden_layer_size))
feedforward.add(nn.PReLU())
# add hidden layers
for i in range(self.hidden_layer_count - 1):
feedforward.add(
nn.Linear(self.hidden_layer_size, self.hidden_layer_size))
feedforward.add(nn.PReLU())
feedforward.add(nn.Linear(self.hidden_layer_size, self.output_size))
# [1.2] right part, discard pot_size, produce r1, r2
right_part = nn.Sequential()
right_part.add(nn.Narrow(1, 0, self.output_size))
first_layer.add(feedforward)
first_layer.add(right_part)
# [2.0] outer net force counterfactual values satisfy 0-sum property
second_layer = nn.ConcatTable()
# accept v1,v2; ignore r1, r2
left_part2 = nn.Sequential()
left_part2.add(nn.SelectTable(0))
# accept, r1,r2, v1,v2; produce -0.5k=-0.5(r1v1 + r2v2)
right_part2 = nn.Sequential()
right_part2.add(nn.DotProduct())
right_part2.add(nn.Unsqueeze(1))
right_part2.add(nn.Replicate(self.output_size, 1))
right_part2.add(nn.Squeeze(2))
right_part2.add(nn.MulConstant(-0.5))
second_layer.add(left_part2)
second_layer.add(right_part2)
final_mlp = nn.Sequential()
final_mlp.add(first_layer)
final_mlp.add(second_layer)
# accept v1,v2 and -0.5k, product v1-0.5k, v2-0.5k
final_mlp.add(nn.CAddTable())
return final_mlp
def test_Concat(self):
input = torch.randn(4, 2)
num_modules = random.randint(2, 5)
linears = [nn.Linear(2, 5) for i in range(num_modules)]
m = nn.Concat(0)
for l in linears:
m.add(l)
l.zeroGradParameters()
l.weight.fill_(1)
l.bias.fill_(0)
# Check that these don't raise errors
m.__repr__()
str(m)
output = m.forward(input)
output2 = input.sum(1).expand(4, 5).repeat(num_modules, 1)
self.assertEqual(output2, output)
gradInput = m.backward(input, torch.ones(output2.size()))
gradInput2 = torch.ones(4, 2).fill_(num_modules * 5)
self.assertEqual(gradInput, gradInput2)
gradWeight = input.sum(0).expand(5, 2)
for l in linears:
self.assertEqual(gradWeight, l.gradWeight)
def run_multi_embedding_test(module_type: MultiEmbModuleType, conf: MultiEmbConfig, batch_size, scenario: MultiEmbScenario, device, num_iters=10):
dims = conf.value
sizes = ([1000, 5000, 10000, 50000] * 10)[:len(conf.value)]
if module_type == MultiEmbModuleType.FAST_MULTI:
emb = FastMultiEmbedding(sizes, dims).to(device)
elif module_type == MultiEmbModuleType.FAST:
embs = [FastEmbedding(size, dim).to(device) for size, dim in zip(sizes, dims)]
def emb(multi_indices):
return torch.cat([e(multi_indices[:, idx]) for (idx, e) in enumerate(embs)], 1)
else:
embs = [nn.Embedding(size, dim).to(device) for size, dim in zip(sizes, dims)]
def emb(multi_indices):
return torch.cat([e(multi_indices[:, idx]) for (idx, e) in enumerate(embs)], 1)
linear = nn.Linear(sum(dims), 1).to(device)
indices = torch.cat([create_indices(batch_size, num_embs, scenario.value[1](idx, dim)).view(-1, 1)
for idx, (dim, num_embs) in enumerate(zip(dims, sizes))], 1).to(device)
on_cuda = device.type == 'cuda'
def run():
x = emb(indices)
linear.forward(x).sum().backward()
if on_cuda:
torch.cuda.synchronize()
return measure_execution_time(num_iters, run)
def run_embedding_test(emb_module, num_embeddings, embedding_dim, batch_size, scenario: EmbScenario, device, num_iters=10):
emb = emb_module(num_embeddings, embedding_dim).to(device)
linear = nn.Linear(embedding_dim, 1).to(device)
indices = create_indices(batch_size, num_embeddings, scenario).to(device)
on_cuda = device.type == 'cuda'
def run():
x = emb.forward(indices)
linear.forward(x).sum().backward()
if on_cuda:
torch.cuda.synchronize()
return measure_execution_time(num_iters, run)
) else 'torch.FloatTensor'
# two-dimensional SparseConvNet
model = nn.Sequential()
sparseModel = scn.Sequential()
denseModel = nn.Sequential()
model.add(sparseModel).add(denseModel)
sparseModel.add(scn.ValidConvolution(2, 3, 16, 3, False))
sparseModel.add(scn.MaxPooling(2, 3, 2))
sparseModel.add(
scn.SparseResNet(
2, 16,
[['b', 16, 2, 1], ['b', 32, 2, 2], ['b', 48, 2, 2], ['b', 96, 2, 2]]))
sparseModel.add(scn.Convolution(2, 96, 128, 4, 1, False))
sparseModel.add(scn.BatchNormReLU(128))
sparseModel.add(scn.SparseToDense(2))
denseModel.add(nn.View(-1, 128))
denseModel.add(nn.Linear(128, 3755))
model.type(dtype)
print(model)
spatial_size = sparseModel.suggestInputSize(torch.LongTensor([1, 1]))
print('input spatial size', spatial_size)
dataset = getIterators(spatial_size, 63, 3)
scn.ClassificationTrainValidate(model, dataset, {
'nEpochs': 100,
'initial_LR': 0.1,
'LR_decay': 0.05,
'weightDecay': 1e-4
})
denseModel = nn.Sequential()
model.add(sparseModel).add(denseModel)
sparseModel.add(scn.ValidConvolution(2, 3, 16, 3, False))\
.add(scn.SparseDenseNet(2, 16, [
{'pool': 'MP', 'compression': 0},
{'nExtraLayers': 2, 'growthRate': 16},
{'pool': 'BN-R-C-AP', 'compression': 0},
{'nExtraLayers': 2, 'growthRate': 16},
{'pool': 'BN-R-C-AP', 'compression': 0},
{'nExtraLayers': 2, 'growthRate': 16},
{'pool': 'BN-R-C-AP', 'compression': 0},
{'nExtraLayers': 2, 'growthRate': 16}]))
n_out = sparseModel.modules[-1].nOutputPlanes
sparseModel.add(scn.Convolution(2, n_out, 256, 4, 1, False))
sparseModel.add(scn.BatchNormReLU(256))
sparseModel.add(scn.SparseToDense(2))
denseModel.add(nn.View(-1, 256))
denseModel.add(nn.Linear(256, 3755))
model.type(dtype)
print(model)
spatial_size = sparseModel.suggestInputSize(torch.LongTensor([1, 1]))
print('input spatial size', spatial_size)
dataset = getIterators(spatial_size, 64, 2)
scn.ClassificationTrainValidate(model, dataset, {
'nEpochs': 100,
'initial_LR': 0.1,
'LR_decay': 0.05,
'weightDecay': 1e-4
})
def test_linear(self):
self.input = self.input.flatten()
input_size = self.input.shape[0]
self._test_single_layer(nn.Linear(input_size, 3, True), decimal=5)
dtype = 'torch.cuda.FloatTensor' # Uncomment this to run on GPU
# two-dimensional SparseConvNet
model = nn.Sequential()
sparseModel = scn.Sequential()
denseModel = nn.Sequential()
model.add(sparseModel).add(denseModel)
sparseModel.add(
scn.SparseVggNet(2, 3, [[
'C',
8,
], ['C', 8], 'MP', ['C', 16], ['C', 16], 'MP', ['C', 16, 8], ['C', 16, 8],
'MP', ['C', 24, 8], ['C', 24, 8], 'MP']))
sparseModel.add(scn.Convolution(2, 32, 64, 5, 1, False))
sparseModel.add(scn.BatchNormReLU(64))
sparseModel.add(scn.SparseToDense(2))
denseModel.add(nn.View(-1, 64))
denseModel.add(nn.Linear(64, 183))
model.type(dtype)
print(model)
spatial_size = sparseModel.suggestInputSize(torch.LongTensor([1, 1]))
print('input spatial size', spatial_size)
dataset = getIterators(spatial_size, 63, 3)
scn.ClassificationTrainValidate(model, dataset, {
'nEpochs': 100,
'initial_LR': 0.1,
'LR_decay': 0.05,
'weightDecay': 1e-4
})
nn.Conv2d(32, 96, (5, 5), (1, 1), (2, 2)),
nn.ReLU(),
nn.MaxPool2d((3, 3), (1, 1), (1, 1), ceil_mode=True),
nn.Conv2d(256, 64, (1, 1)),
nn.ReLU(),
nn.MaxPool2d((3, 3), (2, 2), (0, 0), ceil_mode=True),
nn.Conv2d(480, 192, (1, 1)),
nn.ReLU(),
nn.Conv2d(480, 96, (1, 1)),
nn.ReLU(),
nn.Conv2d(96, 208, (3, 3), (1, 1), (1, 1)),
nn.ReLU(),
nn.Conv2d(480, 16, (1, 1)),
nn.ReLU(),
nn.Conv2d(16, 48, (5, 5), (1, 1), (2, 2)),
nn.ReLU(),
nn.MaxPool2d((3, 3), (1, 1), (1, 1), ceil_mode=True),
nn.Conv2d(480, 64, (1, 1)),
nn.ReLU(),
nn.AvgPool2d((5, 5), (3, 3), (0, 0), ceil_mode=True), #AvgPool2d,
nn.Conv2d(512, 128, (1, 1)),
nn.ReLU(),
Lambda(lambda x: x.view(x.size(0), -1)), # View,
nn.Sequential(Lambda(lambda x: x.view(1, -1) if 1 == len(x.size()) else x),
nn.Linear(2048, 1024)), # Linear,
nn.ReLU(),
nn.Dropout(0.7),
nn.Sequential(Lambda(lambda x: x.view(1, -1) if 1 == len(x.size()) else x),
nn.Linear(1024, 365)), # Linear,
)
