def test_simple_resnet_graph():
simple_archi: Architecture = Architecture(
preprocess=lambda x: x,
layers=[
LinearLayer(4, 4),
LinearLayer(4, 4),
LinearLayer(4, 4),
LinearLayer(4, 10),
SoftMaxLayer(),
],
layer_links=[(-1, 0), (0, 1), (1, 2), (1, 3), (2, 3), (3, 4)],
)
simple_archi.build_matrices()
simple_example = torch.ones(4)
graph = Graph.from_architecture_and_data_point(simple_archi, simple_example)
adjacency_matrix = graph.get_adjacency_matrix().todense()
assert np.shape(adjacency_matrix) == (26, 26)
assert len(graph.get_edge_list()) == 128
assert len(np.where(adjacency_matrix > 0)[0]) == 128 * 2
print(graph.get_edge_list())
print(simple_archi.get_pre_softmax_idx())
def test_simple_cnn_multi_channels():
simple_archi = Architecture(
preprocess=lambda x: x,
layers=[
# 2 input channels
# 3 output channels
ConvLayer(2, 3, 2, input_shape=(3, 4)),
LinearLayer(18, 1),
],
)
simple_archi.build_matrices()
simple_example = torch.tensor(
[
[
[[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]],
[[2, 4, 6, 8], [10, 12, 14, 16], [18, 20, 22, 24]],
]
]
)
for param in simple_archi.parameters():
print(f"Kernel size is {list(param.shape)}")
m = simple_archi.get_graph_values(simple_example)
# Shape should be 6*3 out_channels = 18 x 12*2 in_channels = 24
assert np.shape(m[(-1, 0)]) == (18, 24)
assert np.shape(m[(0, 1)]) == (1, 18)
graph = Graph.from_architecture_and_data_point(simple_archi, simple_example)
adjacency_matrix = graph.get_adjacency_matrix()
assert np.shape(adjacency_matrix) == (18 + 24 + 1, 18 + 24 + 1)
def _cifar_resnet(layer_sizes):
conv1 = conv3x3_layer(3, 16, name="IntroConv")
bn1 = BatchNorm2d(16, activ=F.relu)
layers = [conv1, bn1]
edges = [(-1, 0), (0, 1)]
inplanes = 16
layer1, inplanes = _basic_layer(inplanes, 16, layer_sizes[0], name="L1")
layer2, inplanes = _basic_layer(inplanes,
32,
layer_sizes[1],
stride=2,
name="L2")
layer3, inplanes = _basic_layer(inplanes,
64,
layer_sizes[2],
stride=2,
name="L3")
for layer_block in [layer1, layer2, layer3]:
for block_layers, block_edges in layer_block:
start_idx = max([v for u, v in edges])
mapped_edges = [(u + start_idx + 1, v + start_idx + 1)
for u, v in block_edges]
layers.extend(block_layers)
edges.extend(mapped_edges)
avgpool = AdaptativeAvgPool2dLayer(output_size=(1, 1))
fc = LinearLayer(64, 100)
start_idx = max([v for u, v in edges])
layers.extend([avgpool, fc])
last_edges = [(start_idx, start_idx + 1), (start_idx + 1, start_idx + 2)]
edges.extend(last_edges)
return layers, edges
bias=False,
),
BatchNorm2d(channels=512, activ=F.relu),
ConvLayer(
in_channels=512,
out_channels=512,
kernel_size=3,
stride=1,
padding=1,
bias=False,
),
BatchNorm2d(channels=512),
ReluLayer(),
# End part
AvgPool2dLayer(kernel_size=4),
LinearLayer(512, 10),
SoftMaxLayer(),
# Layer to reduce dimension in residual blocks
ConvLayer(
in_channels=64,
out_channels=128,
kernel_size=1,
stride=2,
padding=0,
bias=False,
),
ConvLayer(
in_channels=128,
out_channels=256,
kernel_size=1,
stride=2,
mnist_preprocess2,
mnist_preprocess,
mnist_preprocess_cnn_05,
)
fashion_mnist_lenet = Architecture(
name="fashion_mnist_lenet",
preprocess=mnist_preprocess2,
layers=[
ConvLayer(1, 10, 5, activ=F.relu, bias=True,
name="conv1"), # output 6 * 28 * 28
MaxPool2dLayer(2),
ConvLayer(10, 20, 5, activ=F.relu, bias=True,
name="conv2"), # output 6 * 28 * 28
MaxPool2dLayer(2),
LinearLayer(320, 50, activ=F.relu, name="fc1"),
# DropOut(),
LinearLayer(50, 10, name="fc2"),
SoftMaxLayer(),
],
)
# Same model as above but with the pixels between 0 and 1
# (instead of -0.5 / 0.5)
fashion_mnist_lenet_05 = Architecture(
name="fashion_mnist_lenet_05",
preprocess=mnist_preprocess_cnn_05,
layers=[
ConvLayer(1, 10, 5, activ=F.relu, bias=True,
name="conv1"), # output 6 * 28 * 28
MaxPool2dLayer(2),
import torch
import pytest
import numpy as np
from tda.models.layers import (
LinearLayer,
ConvLayer,
AvgPool2dLayer,
AdaptativeAvgPool2dLayer,
MaxPool2dLayer,
)
@pytest.mark.parametrize(
"layer,input_shape",
[(LinearLayer(3, 2, activ=None, bias=False), (3, 1)),
(
ConvLayer(
in_channels=2, out_channels=3, kernel_size=2, input_shape=(7, 7)),
(1, 2, 7, 7),
),
(
ConvLayer(
in_channels=1,
out_channels=1,
kernel_size=2,
stride=2,
input_shape=(4, 4),
),
(1, 1, 4, 4),
),
(
return x.view(-1, 1, 32, 32)
def svhn_preprocess_resize(x):
x = x.reshape(-1, 3, 32, 32)
x = F.interpolate(x, scale_factor=7.5)
x = x.reshape(-1, 3, 240, 240)
return x
svhn_cnn_simple = Architecture(
name="simple_cnn_svhn",
preprocess=svhn_preprocess,
layers=[
ConvLayer(3, 8, 5), # output 8 * 28 * 28
ConvLayer(8, 3, 5), # output 3 * 24 * 24
LinearLayer(3 * 24 * 24, 500),
LinearLayer(500, 256),
LinearLayer(256, 10),
SoftMaxLayer(),
],
)
svhn_lenet = Architecture(
name="svhn_lenet",
preprocess=svhn_preprocess,
layers=[
ConvLayer(3, 6, 5, activ=F.relu), # output 6 * 28 * 28
MaxPool2dLayer(2),
ConvLayer(6, 16, 5, activ=F.relu),
MaxPool2dLayer(2), # output 16 * 5 * 5
LinearLayer(16 * 5 * 5, 120, activ=F.relu),
return x.view(-1, 28 * 28)
def mnist_preprocess2(x):
return x.view(-1, 1, 28, 28)
def mnist_preprocess_cnn_05(x):
return x.view(-1, 1, 28, 28) - 0.5
mnist_mlp = Architecture(
name="simple_fcn_mnist",
preprocess=mnist_preprocess,
layers=[
LinearLayer(28 * 28, 500),
LinearLayer(500, 256),
LinearLayer(256, 10),
SoftMaxLayer(),
],
)
mnist_mlp_relu = Architecture(
name="simple_fcn_mnist_relu",
preprocess=mnist_preprocess,
layers=[
LinearLayer(28 * 28, 500, activ=F.relu),
LinearLayer(500, 256, activ=F.relu),
LinearLayer(256, 10),
SoftMaxLayer(),
],
import torch.nn.functional as F
from tda.models.layers import (
ConvLayer,
MaxPool2dLayer,
DropOut,
LinearLayer,
SoftMaxLayer,
)
from .architecture import Architecture
toy_mlp = Architecture(
name="toy_mlp",
#preprocess=mnist_preprocess,
layers=[
LinearLayer(2, 4, activ=F.relu, name="fc1"),
LinearLayer(4, 2, name="fc2"),
SoftMaxLayer(),
],
)
toy_mlp2 = Architecture(
name="toy_mlp2",
#preprocess=mnist_preprocess,
layers=[
LinearLayer(2, 8, activ=F.relu, name="fc1"),
LinearLayer(8, 2, name="fc2"),
SoftMaxLayer(),
],
)
# End
# 136
ConvLayer(
in_channels=320,
out_channels=1280,
kernel_size=1,
stride=1,
padding=0,
grouped_channels=False,
bias=False,
p=dropout_conv,
),
# 137
AdaptativeAvgPool2dLayer(output_size=(1, 1)),
# 138
LinearLayer(1280, 200, p=dropout_linear),
# 139
SoftMaxLayer(),
#############
#############
# Skip layers
# For block 2
# 140
AvgPool2dLayer(kernel_size=2, stride=2),
# 141
ConvLayer(
in_channels=16,
out_channels=24,
kernel_size=1,
stride=1,
padding=0,
MaxPool2dLayer,
LinearLayer,
SoftMaxLayer,
)
from .architecture import Architecture
from .svhn_models import svhn_preprocess
cifar_lenet = Architecture(
name="cifar_lenet",
preprocess=svhn_preprocess,
layers=[
ConvLayer(3, 6, 5, activ=F.relu), # output 6 * 28 * 28
MaxPool2dLayer(2),
ConvLayer(6, 16, 5, activ=F.relu),
MaxPool2dLayer(2), # output 16 * 5 * 5
LinearLayer(16 * 5 * 5, 120, activ=F.relu),
LinearLayer(120, 84, activ=F.relu),
LinearLayer(84, 10),
SoftMaxLayer(),
],
)
cifar_toy_resnet = Architecture(
name="cifar_toy_resnet",
preprocess=svhn_preprocess,
layers=[
ConvLayer(3, 6, 5, activ=F.relu), # output 6 * 28 * 28
MaxPool2dLayer(2),
ConvLayer(6, 16, 5, activ=F.relu),
MaxPool2dLayer(2), # output 16 * 5 * 5
LinearLayer(16 * 5 * 5, 120, activ=F.relu),