def test_simple_resnet_graph(): simple_archi: Architecture = Architecture( preprocess=lambda x: x, layers=[ LinearLayer(4, 4), LinearLayer(4, 4), LinearLayer(4, 4), LinearLayer(4, 10), SoftMaxLayer(), ], layer_links=[(-1, 0), (0, 1), (1, 2), (1, 3), (2, 3), (3, 4)], ) simple_archi.build_matrices() simple_example = torch.ones(4) graph = Graph.from_architecture_and_data_point(simple_archi, simple_example) adjacency_matrix = graph.get_adjacency_matrix().todense() assert np.shape(adjacency_matrix) == (26, 26) assert len(graph.get_edge_list()) == 128 assert len(np.where(adjacency_matrix > 0)[0]) == 128 * 2 print(graph.get_edge_list()) print(simple_archi.get_pre_softmax_idx())
def test_simple_cnn_multi_channels(): simple_archi = Architecture( preprocess=lambda x: x, layers=[ # 2 input channels # 3 output channels ConvLayer(2, 3, 2, input_shape=(3, 4)), LinearLayer(18, 1), ], ) simple_archi.build_matrices() simple_example = torch.tensor( [ [ [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]], [[2, 4, 6, 8], [10, 12, 14, 16], [18, 20, 22, 24]], ] ] ) for param in simple_archi.parameters(): print(f"Kernel size is {list(param.shape)}") m = simple_archi.get_graph_values(simple_example) # Shape should be 6*3 out_channels = 18 x 12*2 in_channels = 24 assert np.shape(m[(-1, 0)]) == (18, 24) assert np.shape(m[(0, 1)]) == (1, 18) graph = Graph.from_architecture_and_data_point(simple_archi, simple_example) adjacency_matrix = graph.get_adjacency_matrix() assert np.shape(adjacency_matrix) == (18 + 24 + 1, 18 + 24 + 1)
def _cifar_resnet(layer_sizes): conv1 = conv3x3_layer(3, 16, name="IntroConv") bn1 = BatchNorm2d(16, activ=F.relu) layers = [conv1, bn1] edges = [(-1, 0), (0, 1)] inplanes = 16 layer1, inplanes = _basic_layer(inplanes, 16, layer_sizes[0], name="L1") layer2, inplanes = _basic_layer(inplanes, 32, layer_sizes[1], stride=2, name="L2") layer3, inplanes = _basic_layer(inplanes, 64, layer_sizes[2], stride=2, name="L3") for layer_block in [layer1, layer2, layer3]: for block_layers, block_edges in layer_block: start_idx = max([v for u, v in edges]) mapped_edges = [(u + start_idx + 1, v + start_idx + 1) for u, v in block_edges] layers.extend(block_layers) edges.extend(mapped_edges) avgpool = AdaptativeAvgPool2dLayer(output_size=(1, 1)) fc = LinearLayer(64, 100) start_idx = max([v for u, v in edges]) layers.extend([avgpool, fc]) last_edges = [(start_idx, start_idx + 1), (start_idx + 1, start_idx + 2)] edges.extend(last_edges) return layers, edges
bias=False, ), BatchNorm2d(channels=512, activ=F.relu), ConvLayer( in_channels=512, out_channels=512, kernel_size=3, stride=1, padding=1, bias=False, ), BatchNorm2d(channels=512), ReluLayer(), # End part AvgPool2dLayer(kernel_size=4), LinearLayer(512, 10), SoftMaxLayer(), # Layer to reduce dimension in residual blocks ConvLayer( in_channels=64, out_channels=128, kernel_size=1, stride=2, padding=0, bias=False, ), ConvLayer( in_channels=128, out_channels=256, kernel_size=1, stride=2,
mnist_preprocess2, mnist_preprocess, mnist_preprocess_cnn_05, ) fashion_mnist_lenet = Architecture( name="fashion_mnist_lenet", preprocess=mnist_preprocess2, layers=[ ConvLayer(1, 10, 5, activ=F.relu, bias=True, name="conv1"), # output 6 * 28 * 28 MaxPool2dLayer(2), ConvLayer(10, 20, 5, activ=F.relu, bias=True, name="conv2"), # output 6 * 28 * 28 MaxPool2dLayer(2), LinearLayer(320, 50, activ=F.relu, name="fc1"), # DropOut(), LinearLayer(50, 10, name="fc2"), SoftMaxLayer(), ], ) # Same model as above but with the pixels between 0 and 1 # (instead of -0.5 / 0.5) fashion_mnist_lenet_05 = Architecture( name="fashion_mnist_lenet_05", preprocess=mnist_preprocess_cnn_05, layers=[ ConvLayer(1, 10, 5, activ=F.relu, bias=True, name="conv1"), # output 6 * 28 * 28 MaxPool2dLayer(2),
import torch import pytest import numpy as np from tda.models.layers import ( LinearLayer, ConvLayer, AvgPool2dLayer, AdaptativeAvgPool2dLayer, MaxPool2dLayer, ) @pytest.mark.parametrize( "layer,input_shape", [(LinearLayer(3, 2, activ=None, bias=False), (3, 1)), ( ConvLayer( in_channels=2, out_channels=3, kernel_size=2, input_shape=(7, 7)), (1, 2, 7, 7), ), ( ConvLayer( in_channels=1, out_channels=1, kernel_size=2, stride=2, input_shape=(4, 4), ), (1, 1, 4, 4), ), (
return x.view(-1, 1, 32, 32) def svhn_preprocess_resize(x): x = x.reshape(-1, 3, 32, 32) x = F.interpolate(x, scale_factor=7.5) x = x.reshape(-1, 3, 240, 240) return x svhn_cnn_simple = Architecture( name="simple_cnn_svhn", preprocess=svhn_preprocess, layers=[ ConvLayer(3, 8, 5), # output 8 * 28 * 28 ConvLayer(8, 3, 5), # output 3 * 24 * 24 LinearLayer(3 * 24 * 24, 500), LinearLayer(500, 256), LinearLayer(256, 10), SoftMaxLayer(), ], ) svhn_lenet = Architecture( name="svhn_lenet", preprocess=svhn_preprocess, layers=[ ConvLayer(3, 6, 5, activ=F.relu), # output 6 * 28 * 28 MaxPool2dLayer(2), ConvLayer(6, 16, 5, activ=F.relu), MaxPool2dLayer(2), # output 16 * 5 * 5 LinearLayer(16 * 5 * 5, 120, activ=F.relu),
return x.view(-1, 28 * 28) def mnist_preprocess2(x): return x.view(-1, 1, 28, 28) def mnist_preprocess_cnn_05(x): return x.view(-1, 1, 28, 28) - 0.5 mnist_mlp = Architecture( name="simple_fcn_mnist", preprocess=mnist_preprocess, layers=[ LinearLayer(28 * 28, 500), LinearLayer(500, 256), LinearLayer(256, 10), SoftMaxLayer(), ], ) mnist_mlp_relu = Architecture( name="simple_fcn_mnist_relu", preprocess=mnist_preprocess, layers=[ LinearLayer(28 * 28, 500, activ=F.relu), LinearLayer(500, 256, activ=F.relu), LinearLayer(256, 10), SoftMaxLayer(), ],
import torch.nn.functional as F from tda.models.layers import ( ConvLayer, MaxPool2dLayer, DropOut, LinearLayer, SoftMaxLayer, ) from .architecture import Architecture toy_mlp = Architecture( name="toy_mlp", #preprocess=mnist_preprocess, layers=[ LinearLayer(2, 4, activ=F.relu, name="fc1"), LinearLayer(4, 2, name="fc2"), SoftMaxLayer(), ], ) toy_mlp2 = Architecture( name="toy_mlp2", #preprocess=mnist_preprocess, layers=[ LinearLayer(2, 8, activ=F.relu, name="fc1"), LinearLayer(8, 2, name="fc2"), SoftMaxLayer(), ], )
# End # 136 ConvLayer( in_channels=320, out_channels=1280, kernel_size=1, stride=1, padding=0, grouped_channels=False, bias=False, p=dropout_conv, ), # 137 AdaptativeAvgPool2dLayer(output_size=(1, 1)), # 138 LinearLayer(1280, 200, p=dropout_linear), # 139 SoftMaxLayer(), ############# ############# # Skip layers # For block 2 # 140 AvgPool2dLayer(kernel_size=2, stride=2), # 141 ConvLayer( in_channels=16, out_channels=24, kernel_size=1, stride=1, padding=0,
MaxPool2dLayer, LinearLayer, SoftMaxLayer, ) from .architecture import Architecture from .svhn_models import svhn_preprocess cifar_lenet = Architecture( name="cifar_lenet", preprocess=svhn_preprocess, layers=[ ConvLayer(3, 6, 5, activ=F.relu), # output 6 * 28 * 28 MaxPool2dLayer(2), ConvLayer(6, 16, 5, activ=F.relu), MaxPool2dLayer(2), # output 16 * 5 * 5 LinearLayer(16 * 5 * 5, 120, activ=F.relu), LinearLayer(120, 84, activ=F.relu), LinearLayer(84, 10), SoftMaxLayer(), ], ) cifar_toy_resnet = Architecture( name="cifar_toy_resnet", preprocess=svhn_preprocess, layers=[ ConvLayer(3, 6, 5, activ=F.relu), # output 6 * 28 * 28 MaxPool2dLayer(2), ConvLayer(6, 16, 5, activ=F.relu), MaxPool2dLayer(2), # output 16 * 5 * 5 LinearLayer(16 * 5 * 5, 120, activ=F.relu),