Exemplo n.º 1
0
    def __init__(
        self,
        feature_scale=4,  # to reduce dimensionality
        in_resolution=256,
        output_channels=3,
        is_deconv=True,
        upper_billinear=False,
        lower_billinear=False,
        in_channels=3,
        is_batchnorm=True,
        skip_background=True,
        num_joints=17,
        nb_dims=3,  # ecoding transformation
        encoderType='UNet',
        num_encoding_layers=5,
        dimension_bg=256,
        dimension_fg=256,
        dimension_3d=3 * 64,  # needs to be devidable by 3
        latent_dropout=0.3,
        shuffle_fg=True,
        shuffle_3d=True,
        from_latent_hidden_layers=0,
        n_hidden_to3Dpose=2,
        subbatch_size=4,
        implicit_rotation=False,
        nb_stage=1,  # number of U-net stacks
        output_types=[
            '3D', 'img_crop', 'shuffled_pose', 'shuffled_appearance'
        ],
        num_cameras=4,
    ):
        super(unet, self).__init__()
        self.in_resolution = in_resolution
        self.is_deconv = is_deconv
        self.in_channels = in_channels
        self.is_batchnorm = is_batchnorm
        self.feature_scale = feature_scale
        self.nb_stage = nb_stage
        self.dimension_bg = dimension_bg
        self.dimension_fg = dimension_fg
        self.dimension_3d = dimension_3d
        self.shuffle_fg = shuffle_fg
        self.shuffle_3d = shuffle_3d
        self.num_encoding_layers = num_encoding_layers
        self.output_types = output_types
        self.encoderType = encoderType
        assert dimension_3d % 3 == 0
        self.implicit_rotation = implicit_rotation
        self.num_cameras = num_cameras

        self.skip_connections = False
        self.skip_background = skip_background
        self.subbatch_size = subbatch_size
        self.latent_dropout = latent_dropout

        #filters = [64, 128, 256, 512, 1024]
        self.filters = [64, 128, 256, 512, 512, 512]  # HACK
        self.filters = [int(x / self.feature_scale) for x in self.filters]
        self.bottleneck_resolution = in_resolution // (2**(
            num_encoding_layers - 1))
        num_output_features = self.bottleneck_resolution**2 * self.filters[
            num_encoding_layers - 1]
        print('bottleneck_resolution', self.bottleneck_resolution,
              'num_output_features', num_output_features)

        ####################################
        ############ encoder ###############
        if self.encoderType == "ResNet":
            self.encoder = resnet_VNECT_3Donly.resnet50(
                pretrained=True,
                input_key='img_crop',
                output_keys=['latent_3d', '2D_heat'],
                input_width=in_resolution,
                num_classes=self.dimension_fg + self.dimension_3d)

        ns = 0
        setattr(
            self, 'conv_1_stage' + str(ns),
            unetConv2(self.in_channels,
                      self.filters[0],
                      self.is_batchnorm,
                      padding=1))
        setattr(self, 'pool_1_stage' + str(ns), nn.MaxPool2d(kernel_size=2))
        for li in range(
                2, num_encoding_layers
        ):  # note, first layer(li==1) is already created, last layer(li==num_encoding_layers) is created externally
            setattr(
                self, 'conv_' + str(li) + '_stage' + str(ns),
                unetConv2(self.filters[li - 2],
                          self.filters[li - 1],
                          self.is_batchnorm,
                          padding=1))
            setattr(self, 'pool_' + str(li) + '_stage' + str(ns),
                    nn.MaxPool2d(kernel_size=2))

        if from_latent_hidden_layers:
            setattr(
                self, 'conv_' + str(num_encoding_layers) + '_stage' + str(ns),
                nn.Sequential(
                    unetConv2(self.filters[num_encoding_layers - 2],
                              self.filters[num_encoding_layers - 1],
                              self.is_batchnorm,
                              padding=1), nn.MaxPool2d(kernel_size=2)))
        else:
            setattr(
                self, 'conv_' + str(num_encoding_layers) + '_stage' + str(ns),
                unetConv2(self.filters[num_encoding_layers - 2],
                          self.filters[num_encoding_layers - 1],
                          self.is_batchnorm,
                          padding=1))

        ####################################
        ############ background ###############
        if skip_background:
            setattr(
                self, 'conv_1_stage_bg' + str(ns),
                unetConv2(self.in_channels,
                          self.filters[0],
                          self.is_batchnorm,
                          padding=1))

        ###########################################################
        ############ latent transformation and pose ###############
        assert self.dimension_fg < self.filters[num_encoding_layers - 1]
        num_output_features_3d = self.bottleneck_resolution**2 * (
            self.filters[num_encoding_layers - 1] - self.dimension_fg)
        #setattr(self, 'fc_1_stage' + str(ns), Linear(num_output_features, 1024))
        setattr(self, 'fc_1_stage' + str(ns), Linear(self.dimension_3d, 128))
        setattr(self, 'fc_2_stage' + str(ns), Linear(128,
                                                     num_joints * nb_dims))

        self.to_pose = MLP.MLP_fromLatent(d_in=self.dimension_3d,
                                          d_hidden=2048,
                                          d_out=51,
                                          n_hidden=n_hidden_to3Dpose,
                                          dropout=0.5)

        self.to_3d = nn.Sequential(
            Linear(num_output_features, self.dimension_3d),
            Dropout(inplace=True,
                    p=self.latent_dropout)  # removing dropout degrades results
        )

        if self.implicit_rotation:
            print("WARNING: doing implicit rotation!")
            rotation_encoding_dimension = 128
            self.encode_angle = nn.Sequential(
                Linear(3 * 3, rotation_encoding_dimension // 2),
                Dropout(inplace=True, p=self.latent_dropout),
                ReLU(inplace=False),
                Linear(rotation_encoding_dimension // 2,
                       rotation_encoding_dimension),
                Dropout(inplace=True, p=self.latent_dropout),
                ReLU(inplace=False),
                Linear(rotation_encoding_dimension,
                       rotation_encoding_dimension),
            )

            self.rotate_implicitely = nn.Sequential(
                Linear(self.dimension_3d + rotation_encoding_dimension,
                       self.dimension_3d),
                Dropout(inplace=True, p=self.latent_dropout),
                ReLU(inplace=False))

        if from_latent_hidden_layers:
            hidden_layer_dimension = 1024
            if self.dimension_fg > 0:
                self.to_fg = nn.Sequential(
                    Linear(num_output_features, 256),  # HACK pooling 
                    Dropout(inplace=True, p=self.latent_dropout),
                    ReLU(inplace=False),
                    Linear(256, self.dimension_fg),
                    Dropout(inplace=True, p=self.latent_dropout),
                    ReLU(inplace=False))
            self.from_latent = nn.Sequential(
                Linear(self.dimension_3d, hidden_layer_dimension),
                Dropout(inplace=True, p=self.latent_dropout),
                ReLU(inplace=False),
                Linear(hidden_layer_dimension, num_output_features_3d),
                Dropout(inplace=True, p=self.latent_dropout),
                ReLU(inplace=False))
        else:
            if self.dimension_fg > 0:
                self.to_fg = nn.Sequential(
                    Linear(num_output_features, self.dimension_fg),
                    Dropout(inplace=True, p=self.latent_dropout),
                    ReLU(inplace=False))
            self.from_latent = nn.Sequential(
                Linear(self.dimension_3d, num_output_features_3d),
                Dropout(inplace=True, p=self.latent_dropout),
                ReLU(inplace=False))

        ####################################
        ############ decoder ###############
        upper_conv = self.is_deconv and not upper_billinear
        lower_conv = self.is_deconv and not lower_billinear
        if self.skip_connections:
            for li in range(1, num_encoding_layers - 1):
                setattr(
                    self, 'upconv_' + str(li) + '_stage' + str(ns),
                    unetUp(self.filters[num_encoding_layers - li],
                           self.filters[num_encoding_layers - li - 1],
                           upper_conv,
                           padding=1))
                #setattr(self, 'upconv_2_stage' + str(ns), unetUp(self.filters[2], self.filters[1], upper_conv, padding=1))
        else:
            for li in range(1, num_encoding_layers - 1):
                setattr(
                    self, 'upconv_' + str(li) + '_stage' + str(ns),
                    unetUpNoSKip(self.filters[num_encoding_layers - li],
                                 self.filters[num_encoding_layers - li - 1],
                                 upper_conv,
                                 padding=1))
            #setattr(self, 'upconv_2_stage' + str(ns), unetUpNoSKip(self.filters[2], self.filters[1], upper_conv, padding=1))

        if self.skip_connections or self.skip_background:
            setattr(
                self,
                'upconv_' + str(num_encoding_layers - 1) + '_stage' + str(ns),
                unetUp(self.filters[1], self.filters[0], lower_conv,
                       padding=1))
        else:
            setattr(
                self,
                'upconv_' + str(num_encoding_layers - 1) + '_stage' + str(ns),
                unetUpNoSKip(self.filters[1],
                             self.filters[0],
                             lower_conv,
                             padding=1))

        setattr(self, 'final_stage' + str(ns),
                nn.Conv2d(self.filters[0], output_channels, 1))

        self.relu = ReLU(inplace=True)
        self.relu2 = ReLU(inplace=False)
        self.dropout = Dropout(inplace=True, p=0.3)
Exemplo n.º 2
0
 def __init__(self, embedding_size):
     super(MobileFaceNet, self).__init__()
     self.conv1 = Conv_block(3,
                             64,
                             kernel=(3, 3),
                             stride=(2, 2),
                             padding=(1, 1))
     self.conv2_dw = Conv_block(64,
                                64,
                                kernel=(3, 3),
                                stride=(1, 1),
                                padding=(1, 1),
                                groups=64)
     self.conv_23 = Depth_Wise(64,
                               64,
                               kernel=(3, 3),
                               stride=(2, 2),
                               padding=(1, 1),
                               groups=128)
     self.conv_3 = Residual(64,
                            num_block=4,
                            groups=128,
                            kernel=(3, 3),
                            stride=(1, 1),
                            padding=(1, 1))
     self.conv_34 = Depth_Wise(64,
                               128,
                               kernel=(3, 3),
                               stride=(2, 2),
                               padding=(1, 1),
                               groups=256)
     self.conv_4 = Residual(128,
                            num_block=6,
                            groups=256,
                            kernel=(3, 3),
                            stride=(1, 1),
                            padding=(1, 1))
     self.conv_45 = Depth_Wise(128,
                               128,
                               kernel=(3, 3),
                               stride=(2, 2),
                               padding=(1, 1),
                               groups=512)
     self.conv_5 = Residual(128,
                            num_block=2,
                            groups=256,
                            kernel=(3, 3),
                            stride=(1, 1),
                            padding=(1, 1))
     self.conv_6_sep = Conv_block(128,
                                  512,
                                  kernel=(1, 1),
                                  stride=(1, 1),
                                  padding=(0, 0))
     self.conv_6_dw = Linear_block(512,
                                   512,
                                   groups=512,
                                   kernel=(7, 7),
                                   stride=(1, 1),
                                   padding=(0, 0))
     self.conv_6_flatten = Flatten()
     self.linear = Linear(512, embedding_size, bias=False)
     self.bn = BatchNorm1d(embedding_size)
Exemplo n.º 3
0
    def __init__(
        self,
        in_channels: int,
        hidden_channels: int,
        num_layers: int,
        out_channels: Optional[int] = None,
        dropout: float = 0.0,
        act: Union[str, Callable, None] = "relu",
        norm: Optional[torch.nn.Module] = None,
        jk: Optional[str] = None,
        act_first: bool = False,
        act_kwargs: Optional[Dict[str, Any]] = None,
        **kwargs,
    ):
        super().__init__()

        from class_resolver.contrib.torch import activation_resolver

        self.in_channels = in_channels
        self.hidden_channels = hidden_channels
        self.num_layers = num_layers

        self.dropout = dropout
        self.act = activation_resolver.make(act, act_kwargs)
        self.jk_mode = jk
        self.act_first = act_first

        if out_channels is not None:
            self.out_channels = out_channels
        else:
            self.out_channels = hidden_channels

        self.convs = ModuleList()
        self.convs.append(
            self.init_conv(in_channels, hidden_channels, **kwargs))
        for _ in range(num_layers - 2):
            self.convs.append(
                self.init_conv(hidden_channels, hidden_channels, **kwargs))
        if out_channels is not None and jk is None:
            self._is_conv_to_out = True
            self.convs.append(
                self.init_conv(hidden_channels, out_channels, **kwargs))
        else:
            self.convs.append(
                self.init_conv(hidden_channels, hidden_channels, **kwargs))

        self.norms = None
        if norm is not None:
            self.norms = ModuleList()
            for _ in range(num_layers - 1):
                self.norms.append(copy.deepcopy(norm))
            if jk is not None:
                self.norms.append(copy.deepcopy(norm))

        if jk is not None and jk != 'last':
            self.jk = JumpingKnowledge(jk, hidden_channels, num_layers)

        if jk is not None:
            if jk == 'cat':
                in_channels = num_layers * hidden_channels
            else:
                in_channels = hidden_channels
            self.lin = Linear(in_channels, self.out_channels)
Exemplo n.º 4
0
    def __init__(
        self, out_h, out_w, feat_dim, blocks_args=None, global_params=None
    ):
        super().__init__()
        assert isinstance(blocks_args, list), "blocks_args should be a list"
        assert len(blocks_args) > 0, "block args must be greater than 0"
        self._global_params = global_params
        self._blocks_args = blocks_args

        # Batch norm parameters
        bn_mom = 1 - self._global_params.batch_norm_momentum
        bn_eps = self._global_params.batch_norm_epsilon

        # Get stem static or dynamic convolution depending on image size
        image_size = global_params.image_size
        Conv2d = get_same_padding_conv2d(image_size=image_size)

        # Stem
        in_channels = 3  # rgb
        out_channels = round_filters(
            32, self._global_params
        )  # number of output channels
        # self._conv_stem = Conv2d(in_channels, out_channels, kernel_size=3, stride=2, bias=False)
        self._conv_stem = Conv2d(
            in_channels, out_channels, kernel_size=3, stride=1, bias=False
        )
        self._bn0 = nn.BatchNorm2d(
            num_features=out_channels, momentum=bn_mom, eps=bn_eps
        )
        image_size = calculate_output_image_size(image_size, 2)

        # Build blocks
        self._blocks = nn.ModuleList([])
        for block_args in self._blocks_args:

            # Update block input and output filters based on depth multiplier.
            block_args = block_args._replace(
                input_filters=round_filters(
                    block_args.input_filters, self._global_params
                ),
                output_filters=round_filters(
                    block_args.output_filters, self._global_params
                ),
                num_repeat=round_repeats(
                    block_args.num_repeat, self._global_params
                ),
            )

            # The first block needs to take care of stride and filter size increase.
            self._blocks.append(
                MBConvBlock(
                    block_args, self._global_params, image_size=image_size
                )
            )
            image_size = calculate_output_image_size(
                image_size, block_args.stride
            )
            if (
                block_args.num_repeat > 1
            ):  # modify block_args to keep same output size
                block_args = block_args._replace(
                    input_filters=block_args.output_filters, stride=1
                )
            for _ in range(block_args.num_repeat - 1):
                self._blocks.append(
                    MBConvBlock(
                        block_args, self._global_params, image_size=image_size
                    )
                )
                # image_size = calculate_output_image_size(image_size, block_args.stride)  # stride = 1

        # Head
        in_channels = block_args.output_filters  # output of final block
        out_channels = round_filters(1280, self._global_params)
        # out_channels = round_filters(512, self._global_params)
        Conv2d = get_same_padding_conv2d(image_size=image_size)
        self._conv_head = Conv2d(
            in_channels, out_channels, kernel_size=1, bias=False
        )
        self._bn1 = nn.BatchNorm2d(
            num_features=out_channels, momentum=bn_mom, eps=bn_eps
        )

        # Final linear layer
        self._avg_pooling = nn.AdaptiveAvgPool2d(1)
        self._dropout = nn.Dropout(self._global_params.dropout_rate)
        self._fc = nn.Linear(out_channels, self._global_params.num_classes)
        self._swish = MemoryEfficientSwish()
        self.output_layer = Sequential(
            BatchNorm2d(1280),
            # BatchNorm2d(512),
            Dropout(self._global_params.dropout_rate),
            Flatten(),
            Linear(1280 * out_h * out_w, feat_dim),
            # Linear(512 * out_h * out_w, feat_dim),
            BatchNorm1d(feat_dim),
        )
Exemplo n.º 5
0
    def __init__(self, input_size, output_size, hidsize=16):
        super(MLP, self).__init__()

        self.fc1 = Linear(input_size, hidsize)
        self.fc2 = Linear(hidsize, output_size)
Exemplo n.º 6
0
from torch.nn import Linear, ReLU
from torch.nn import MSELoss
import numpy as np

N, D_in, H, D_out = 1000, 64, 100, 1
x = np.random.randn(N, D_in)
y = np.random.randn(N, D_out)

x = torch.from_numpy(x).float()
y = torch.from_numpy(y).float()

print(x.shape, y.shape)

# every layer takes the input and the output dimensions as well

model = torch.nn.Sequential(Linear(D_in, H), ReLU(), Linear(H, D_out))

loss_fn = MSELoss(reduction='sum')

learning_rate = 1e-4
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)

for t in range(10000):

    y_pred = model(x)
    loss = loss_fn(y_pred, y)
    print("at iter ", t, " loss is ", loss.item())
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()
    def __init__(self,
                 resolution: Union[int, List[int]],
                 in_channels: int,
                 out_channels: int,
                 conv_type: str = 'graph',
                 kernel_size: int = 3,
                 sampling: str = 'healpix',
                 knn: int = 10,
                 pool_method: str = 'max',
                 kernel_size_pooling: int = 4,
                 periodic: Union[int, bool, None] = None,
                 ratio: Union[int, float, bool, None] = None):
        super().__init__()

        conv_type = conv_type.lower()
        sampling = sampling.lower()
        pool_method = pool_method.lower()
        assert conv_type != 'image' or periodic is not None
        periodic = bool(periodic)

        ### Delete in the future
        self.test_mode = False
        if resolution == [46, 92] and pool_method in ['max', 'avg']:
            resolution = [48, 92]
            self.test_mode = True

        if not isinstance(resolution, Iterable):
            resolution = [resolution]
        resolution = np.array(resolution)
        self.sphere_graph = UNetSpherical.build_graph([resolution], sampling,
                                                      knn)[0]
        coarsening = int(np.sqrt(kernel_size_pooling))
        resolutions = [
            resolution, resolution // coarsening,
            resolution // coarsening // coarsening
        ]
        self.graphs = []
        if conv_type == 'graph':
            self.graphs = UNetSpherical.build_graph(resolutions, sampling, knn)
            self.laplacians = UNetSpherical.get_laplacian_kernels(self.graphs)
        elif conv_type == 'image':
            self.laplacians = [None] * 20
        else:
            raise ValueError(f'{conv_type} convolution is not supported')

        ### Delete in the future
        if self.test_mode:
            self.sphere_graph = pygsp.graphs.SphereEquiangular(nlat=46,
                                                               nlon=92,
                                                               poles=0)

        # Pooling - unpooling
        if pool_method in ('interp', 'maxval', 'maxarea', 'learn'):
            assert conv_type == 'graph'
            self.pool1, self.unpool1 = PoolUnpoolBlock.getGeneralPoolUnpoolLayer(
                self.graphs[0], self.graphs[1], pool_method)
            self.pool2, self.unpool2 = PoolUnpoolBlock.getGeneralPoolUnpoolLayer(
                self.graphs[1], self.graphs[2], pool_method)
        else:
            self.pool1, self.unpool1 = PoolUnpoolBlock.getPoolUnpoolLayer(
                sampling,
                pool_method,
                kernel_size=kernel_size_pooling,
                ratio=ratio)
            self.pool2, self.unpool2 = PoolUnpoolBlock.getPoolUnpoolLayer(
                sampling,
                pool_method,
                kernel_size=kernel_size_pooling,
                ratio=ratio)

        # Encoding block 1
        self.conv11 = ConvBlock(in_channels,
                                32 * 2,
                                kernel_size,
                                conv_type,
                                True,
                                True,
                                laplacian=self.laplacians[0],
                                periodic=periodic,
                                ratio=ratio)
        self.conv13 = ConvBlock(32 * 2,
                                64 * 2,
                                kernel_size,
                                conv_type,
                                True,
                                True,
                                laplacian=self.laplacians[0],
                                periodic=periodic,
                                ratio=ratio)

        self.conv1_res = Linear(in_channels, 64 * 2)

        # Encoding block 2
        self.conv21 = ConvBlock(64 * 2,
                                96 * 2,
                                kernel_size,
                                conv_type,
                                True,
                                True,
                                laplacian=self.laplacians[1],
                                periodic=periodic,
                                ratio=ratio)
        self.conv23 = ConvBlock(96 * 2,
                                128 * 2,
                                kernel_size,
                                conv_type,
                                True,
                                True,
                                laplacian=self.laplacians[1],
                                periodic=periodic,
                                ratio=ratio)

        self.conv2_res = Linear(64 * 2, 128 * 2)

        # Encoding block 3
        self.conv31 = ConvBlock(128 * 2,
                                256 * 2,
                                kernel_size,
                                conv_type,
                                True,
                                True,
                                laplacian=self.laplacians[2],
                                periodic=periodic,
                                ratio=ratio)
        self.conv33 = ConvBlock(256 * 2,
                                128 * 2,
                                kernel_size,
                                conv_type,
                                True,
                                True,
                                laplacian=self.laplacians[2],
                                periodic=periodic,
                                ratio=ratio)

        self.conv3_res = Linear(128 * 2, 128 * 2)

        # Decoding block 2
        self.uconv21 = ConvBlock(256 * 2,
                                 128 * 2,
                                 kernel_size,
                                 conv_type,
                                 True,
                                 True,
                                 laplacian=self.laplacians[1],
                                 periodic=periodic,
                                 ratio=ratio)
        self.uconv22 = ConvBlock(128 * 2,
                                 64 * 2,
                                 kernel_size,
                                 conv_type,
                                 True,
                                 True,
                                 laplacian=self.laplacians[1],
                                 periodic=periodic,
                                 ratio=ratio)

        # Decoding block 1
        self.uconv11 = ConvBlock(128 * 2,
                                 64 * 2,
                                 kernel_size,
                                 conv_type,
                                 True,
                                 True,
                                 laplacian=self.laplacians[0],
                                 periodic=periodic,
                                 ratio=ratio)
        self.uconv12 = ConvBlock(64 * 2,
                                 32 * 2,
                                 kernel_size,
                                 conv_type,
                                 True,
                                 True,
                                 laplacian=self.laplacians[0],
                                 periodic=periodic,
                                 ratio=ratio)
        self.uconv13 = ConvBlock(32 * 2 * 2,
                                 out_channels,
                                 kernel_size,
                                 conv_type,
                                 False,
                                 False,
                                 laplacian=self.laplacians[0],
                                 periodic=periodic,
                                 ratio=ratio)
axes[0].legend()

import torch.nn.functional as F

x = torch.linspace(-3, 3, 1000, requires_grad=True)
Y = torch.nn.functional.relu(x)
y = torch.sum(F.relu(x))
y.backward()

axes[1].plot(x.detach().numpy(), Y.detach().numpy(), label='function')
axes[1].plot(x.detach().numpy(), x.grad.detach().numpy(), label='derivative')
axes[1].legend()

plt.show()

from torch.nn import Linear

print("\n Test Linear model \n")
x = torch.tensor([[1.0, 1.0]])

torch.manual_seed(1)
model = Linear(in_features=2, out_features=2)
print(list(model.parameters()))
print("class result:", str(model(x)))

w = torch.tensor([[0.3643, -0.1371], [-0.3121, 0.3319]])
b = torch.tensor([[-0.6657, 0.4241]])
f = torch.mm(x, w) + b

print("result:", str(f))
Exemplo n.º 9
0
Arquivo: rgnn.py Projeto: ahoyosid/ogb
    def __init__(self,
                 model: str,
                 in_channels: int,
                 out_channels: int,
                 hidden_channels: int,
                 num_relations: int,
                 num_layers: int,
                 heads: int = 4,
                 dropout: float = 0.5):
        super().__init__()
        self.save_hyperparameters()
        self.model = model.lower()
        self.num_relations = num_relations
        self.dropout = dropout

        self.convs = ModuleList()
        self.norms = ModuleList()
        self.skips = ModuleList()

        if self.model == 'rgat':
            self.convs.append(
                ModuleList([
                    GATConv(in_channels,
                            hidden_channels // heads,
                            heads,
                            add_self_loops=False) for _ in range(num_relations)
                ]))

            for _ in range(num_layers - 1):
                self.convs.append(
                    ModuleList([
                        GATConv(hidden_channels,
                                hidden_channels // heads,
                                heads,
                                add_self_loops=False)
                        for _ in range(num_relations)
                    ]))

        elif self.model == 'rgraphsage':
            self.convs.append(
                ModuleList([
                    SAGEConv(in_channels, hidden_channels, root_weight=False)
                    for _ in range(num_relations)
                ]))

            for _ in range(num_layers - 1):
                self.convs.append(
                    ModuleList([
                        SAGEConv(hidden_channels,
                                 hidden_channels,
                                 root_weight=False)
                        for _ in range(num_relations)
                    ]))

        for _ in range(num_layers):
            self.norms.append(BatchNorm1d(hidden_channels))

        self.skips.append(Linear(in_channels, hidden_channels))
        for _ in range(num_layers - 1):
            self.skips.append(Linear(hidden_channels, hidden_channels))

        self.mlp = Sequential(
            Linear(hidden_channels, hidden_channels),
            BatchNorm1d(hidden_channels),
            ReLU(inplace=True),
            Dropout(p=self.dropout),
            Linear(hidden_channels, out_channels),
        )

        self.train_acc = Accuracy()
        self.val_acc = Accuracy()
        self.test_acc = Accuracy()
Exemplo n.º 10
0
    :param model:
    :type model:
    :param constant:
    :type constant:"""
    for m in model.modules():
        if isinstance(m, (Conv2d, Linear)):
            constant_(m.weight, constant)
            constant_(m.bias, constant)


def normal_init(model: Module, mean: float = 0, std: float = 1.0):
    """

    :param model:
    :param mean:
    :param std:
    :return:
    """
    for m in model.modules():
        if isinstance(m, (Conv2d, Linear)):
            torch.nn.init.normal_(m.weight, mean, std)
            torch.nn.init.normal_(m.bias, mean, std)


if __name__ == "__main__":
    a = Linear(3, 3)
    fan_in_init(a)
    xavier_init(a)
    constant_init(a)
Exemplo n.º 11
0
 def __init__(self, channel, reduction=16):
     super(SELayer, self).__init__()
     self.avg_pool = AdaptiveAvgPool2d(1)
     self.fc = Sequential(Linear(channel, channel // reduction),
                          ReLU(inplace=True),
                          Linear(channel // reduction, channel), Sigmoid())
Exemplo n.º 12
0
    def __init__(self, n_output=1,num_features_xd=78, num_features_xt=25,
                 n_filters=32, embed_dim=128, output_dim=128, dropout=0.2):

        super(GINConvNet, self).__init__()

        dim = 32
        self.dropout = nn.Dropout(dropout)
        self.relu = nn.ReLU()
        self.n_output = n_output
        # convolution layers
        nn1 = Sequential(Linear(num_features_xd, dim), ReLU(), Linear(dim, dim))
        self.conv1 = GINConv(nn1)
        self.bn1 = torch.nn.BatchNorm1d(dim)

        nn2 = Sequential(Linear(dim, dim), ReLU(), Linear(dim, dim))
        self.conv2 = GINConv(nn2)
        self.bn2 = torch.nn.BatchNorm1d(dim)

        nn3 = Sequential(Linear(dim, dim), ReLU(), Linear(dim, dim))
        self.conv3 = GINConv(nn3)
        self.bn3 = torch.nn.BatchNorm1d(dim)

        nn4 = Sequential(Linear(dim, dim), ReLU(), Linear(dim, dim))
        self.conv4 = GINConv(nn4)
        self.bn4 = torch.nn.BatchNorm1d(dim)

        nn5 = Sequential(Linear(dim, dim), ReLU(), Linear(dim, dim))
        self.conv5 = GINConv(nn5)
        self.bn5 = torch.nn.BatchNorm1d(dim)

        self.fc1_xd = Linear(dim, output_dim)

        # 1D convolution on protein sequence  # mut features
        self.embedding_xt_mut = nn.Embedding(num_features_xt + 1, embed_dim)
        self.conv_xt_mut_1 = nn.Conv1d(in_channels=1000, out_channels=n_filters, kernel_size=8)

        # 1D convolution on protein sequence # mut features
        self.embedding_xt_meth = nn.Embedding(num_features_xt + 1, embed_dim)
        self.conv_xt_meth_1 = nn.Conv1d(in_channels=1000, out_channels=n_filters, kernel_size=8)


        # cell line mut feature
        self.conv_xt_mut_1 = nn.Conv1d(in_channels=1, out_channels=n_filters, kernel_size=8)
        self.pool_xt_mut_1 = nn.MaxPool1d(3)
        self.conv_xt_mut_2 = nn.Conv1d(in_channels=n_filters, out_channels=n_filters*2, kernel_size=8)
        self.pool_xt_mut_2 = nn.MaxPool1d(3)
        self.conv_xt_mut_3 = nn.Conv1d(in_channels=n_filters*2, out_channels=n_filters*4, kernel_size=8)
        self.pool_xt_mut_3 = nn.MaxPool1d(3)
        self.fc1_xt_mut = nn.Linear(1280, output_dim)

        # cell line meth feature
        self.conv_xt_meth_1 = nn.Conv1d(in_channels=1, out_channels=n_filters, kernel_size=8)
        self.pool_xt_meth_1 = nn.MaxPool1d(3)
        self.conv_xt_meth_2 = nn.Conv1d(in_channels=n_filters, out_channels=n_filters*2, kernel_size=8)
        self.pool_xt_meth_2 = nn.MaxPool1d(3)
        self.conv_xt_meth_3 = nn.Conv1d(in_channels=n_filters*2, out_channels=n_filters*4, kernel_size=8)
        self.pool_xt_meth_3 = nn.MaxPool1d(3)
        self.fc1_xt_meth = nn.Linear(83584, output_dim)

        # combined layers
        self.fc1 = nn.Linear(3*output_dim, 1024)
        self.fc2 = nn.Linear(1024, 128)
        self.out = nn.Linear(128, n_output)

        # activation and regularization
        self.relu = nn.ReLU()
        self.dropout = nn.Dropout(0.5)
Exemplo n.º 13
0
    def __init__(self, in_channels, num_classes):
        super(Output, self).__init__()
        self.conv = BasicConv2d(in_channels, 128, kernel_size=1)

        self.fc1 = Linear(2048, 1024)
        self.fc2 = Linear(1024, num_classes)
Exemplo n.º 14
0
    def __init__(self, D_key, D_query):
        super(AttentionLayer, self).__init__()

        self.W_k = Linear(D_key, D_query, bias=False)
        self.W_q = Linear(D_key + D_query, D_query, bias=False)
Exemplo n.º 15
0
    def __init__(self, args):
        super(VAE, self).__init__(args)

        self.args = args

        # encoder: q(z2 | x)
        self.q_z2_layers = nn.ModuleList()

        self.q_z2_layers.append( GatedDense(np.prod(self.args.input_size), 300) )
        self.q_z2_layers.append( GatedDense(300, 300) )

        self.q_z2_mean = Linear(300, self.args.z2_size)
        self.q_z2_logvar = NonLinear(300, self.args.z2_size, activation=nn.Hardtanh(min_val=-6.,max_val=2.))

        # encoder: q(z1 | x, z2)
        self.q_z1_layers_x = nn.ModuleList()
        self.q_z1_layers_z2 = nn.ModuleList()
        self.q_z1_layers_joint = nn.ModuleList()

        # x
        self.q_z1_layers_x.append( GatedDense(np.prod(self.args.input_size), 300) )
        # z1
        self.q_z1_layers_z2.append( GatedDense(self.args.z2_size, 300) )
        # joint
        self.q_z1_layers_joint.append( GatedDense(2 * 300, 300) )

        self.q_z1_mean = Linear(300, self.args.z1_size)
        self.q_z1_logvar = NonLinear(300, self.args.z1_size, activation=nn.Hardtanh(min_val=-6.,max_val=2.))

        # decoder: p(z1 | z2)
        self.p_z1_layers = nn.ModuleList()

        self.p_z1_layers.append( GatedDense(self.args.z2_size, 300) )

        self.p_z1_layers.append( GatedDense(300, 300) )

        self.p_z1_mean = Linear(300, self.args.z1_size)
        self.p_z1_logvar = NonLinear(300, self.args.z1_size, activation=nn.Hardtanh(min_val=-6.,max_val=2.))

        # decoder: p(x | z1, z2)
        self.p_x_layers_z1 = nn.ModuleList()
        self.p_x_layers_z2 = nn.ModuleList()
        self.p_x_layers_joint = nn.ModuleList()

        # z1
        self.p_x_layers_z1.append( GatedDense(self.args.z1_size, 300) )
        # z2
        self.p_x_layers_z2.append( GatedDense(self.args.z2_size, 300) )
        # joint
        self.p_x_layers_joint.append( GatedDense(2 * 300, 300) )

        if self.args.input_type == 'binary':
            self.p_x_mean = NonLinear(300, np.prod(self.args.input_size), activation=nn.Sigmoid())
        elif self.args.input_type == 'gray' or self.args.input_type == 'continuous':
            self.p_x_mean = NonLinear(300, np.prod(self.args.input_size), activation=nn.Sigmoid())
            self.p_x_logvar = NonLinear(300, np.prod(self.args.input_size), activation=nn.Hardtanh(min_val=-4.5,max_val=0))

        # weights initialization
        for m in self.modules():
            if isinstance(m, nn.Linear):
                he_init(m)

        # add pseudo-inputs if VampPrior
        if self.args.prior == 'vampprior':
            self.add_pseudoinputs()
Exemplo n.º 16
0
class GNN_ae(
        torch.nn.Module
):  # a baseline for no-coarsening unsupervised learning, essentially an auto-encoder
    def __init__(self, dataset, hidden, num_layers=2, ratio=0.5):
        super(GNN_ae, self).__init__()

        self.embed_block1 = DenseGCNConv(dataset.num_features, hidden)

        self.embed_block2 = DenseGCNConv(hidden, dataset.num_features)
        # self.embed_block2 = GNNBlock(hidden, hidden, dataset.num_features)

        self.num_layers = num_layers

        self.jump = JumpingKnowledge(mode='cat')
        self.lin1 = Linear(hidden, hidden)
        self.lin2 = Linear(hidden, dataset.num_classes)

    def reset_parameters(self):
        self.embed_block1.reset_parameters()
        self.embed_block2.reset_parameters()

        self.jump.reset_parameters()
        self.lin1.reset_parameters()
        self.lin2.reset_parameters()

    def forward(self, data, epsilon=0.01, opt_epochs=100):
        x, adj, mask = data.x, data.adj, data.mask
        batch_num_nodes = data.mask.sum(-1)

        new_adjs = [adj]
        Ss = []

        x1 = F.relu(self.embed_block1(x, adj, mask, add_loop=True))
        xs = [x1.mean(dim=1)]
        new_adj = adj
        coarse_x = x1
        # coarse_x, new_adj, S = self.coarse_block1(x1, adj, batch_num_nodes)
        # new_adjs.append(new_adj)
        # Ss.append(S)

        x2 = self.embed_block2(
            coarse_x, new_adj, mask, add_loop=True
        )  #should not add ReLu, otherwise x2 could be all zero.
        # xs.append(x2.mean(dim=1))
        opt_loss = 0.0
        for i in range(len(x)):
            opt_loss += F.mse_loss(x2[i], x[i])

        return xs, new_adjs, Ss, opt_loss

    def get_nonzero_rows(self, M):  # M is a matrix
        # row_ind = M.sum(-1).nonzero().squeeze() #nonzero has bugs in Pytorch 1.2.0.........
        #So we use other methods to take place of it
        MM, MM_ind = torch.abs(M.sum(-1)).sort()
        N = (torch.abs(M.sum(-1)) > 0).sum()
        return M[MM_ind[:N]]

    def predict(self, xs):
        x = self.jump(xs)
        x = F.relu(self.lin1(x))
        x = F.dropout(x, p=0.5, training=self.training)
        x = self.lin2(x)
        return F.log_softmax(x, dim=-1)

    def __repr__(self):
        return self.__class__.__name__
Exemplo n.º 17
0
 def __init__(self, wrap_fsdp):
     super().__init__()
     self.inner = Linear(*INNER_SHAPE)
     if wrap_fsdp:
         self.inner = FSDP(self.inner)
     self.outer = Linear(*OUTER_SHAPE)
 def __init__(self):
     super().__init__()
     self.conv1 = GCNConv(3, 16)
     self.conv2 = GCNConv(16, 16)
     self.fc1 = Sequential(Linear(16, 16), ReLU(), Dropout(0.2),
                           Linear(16, 7))
Exemplo n.º 19
0
class RECT_L(torch.nn.Module):
    r"""The RECT model, *i.e.* its supervised RECT-L part, from the
    `"Network Embedding with Completely-imbalanced Labels"
    <https://arxiv.org/abs/2007.03545>`_ paper.
    In particular, a GCN model is trained that reconstructs semantic class
    knowledge.

    .. note::

        For an example of using RECT, see `examples/rect.py
        <https://github.com/pyg-team/pytorch_geometric/blob/master/examples/
        rect.py>`_.

    Args:
        in_channels (int): Size of each input sample.
        hidden_channels (int): Intermediate size of each sample.
        normalize (bool, optional): Whether to add self-loops and compute
            symmetric normalization coefficients on the fly.
            (default: :obj:`True`)
        dropout (float, optional): The dropout probability.
            (default: :obj:`0.0`)
    """
    def __init__(self,
                 in_channels: int,
                 hidden_channels: int,
                 normalize: bool = True,
                 dropout: float = 0.0):
        super().__init__()
        self.in_channels = in_channels
        self.hidden_channels = hidden_channels
        self.dropout = dropout

        self.conv = GCNConv(in_channels, hidden_channels, normalize=normalize)
        self.lin = Linear(hidden_channels, in_channels)

        self.reset_parameters()

    def reset_parameters(self):
        self.conv.reset_parameters()
        self.lin.reset_parameters()
        torch.nn.init.xavier_uniform_(self.lin.weight.data)

    def forward(self,
                x: Tensor,
                edge_index: Adj,
                edge_weight: OptTensor = None) -> Tensor:
        """"""
        x = self.conv(x, edge_index, edge_weight)
        x = F.dropout(x, p=self.dropout, training=self.training)
        return self.lin(x)

    @torch.no_grad()
    def embed(self,
              x: Tensor,
              edge_index: Adj,
              edge_weight: OptTensor = None) -> Tensor:
        return self.conv(x, edge_index, edge_weight)

    @torch.no_grad()
    def get_semantic_labels(self, x: Tensor, y: Tensor,
                            mask: Tensor) -> Tensor:
        """Replaces the original labels by their class-centers."""
        y = y[mask]
        mean = scatter(x[mask], y, dim=0, reduce='mean')
        return mean[y]

    def __repr__(self) -> str:
        return (f'{self.__class__.__name__}({self.in_channels}, '
                f'{self.hidden_channels})')
Exemplo n.º 20
0
    def __init__(self):
        super(Net, self).__init__()

        #name
        self.name = "DirCNNh2"
        #optimizer
        self.lr = 0.001
        self.optimizer_name = 'Adam-Exp'

        #data
        #self.data_name = "ModelNet10"
        self.data_name = "Geometry"
        self.batch_size = 20
        self.nr_points = 1024
        self.nr_classes = 10 if self.data_name == 'ModelNet10' else 40

        #train_info
        self.max_epochs = 301
        self.save_every = 100

        #model
        self.k = 20
        self.l = 7
        
        # DD1
        self.in_size = 3
        self.out_size = 64
        layers = []
        layers.append(Linear(self.in_size, 64))
        layers.append(ReLU())
        layers.append(torch.nn.BatchNorm1d(64))
        layers.append(Linear(64 , 64))
        layers.append(ReLU())
        layers.append(torch.nn.BatchNorm1d(64))
        layers.append(Linear(64, self.out_size))
        layers.append(ReLU())
        layers.append(torch.nn.BatchNorm1d(self.out_size))
        dense3dnet = Sequential(*layers)
        self.dd = DD(l = self.l,
                        k = self.k,
                        mlp = dense3dnet,
                        conv_p  = True,
                        conv_fc = False,
                        conv_fn = False,
                        out_3d  = True)

        # DD2
        self.in_size_2 = 64 * 6 
        self.out_size_2 = 128
        layers2 = []
        layers2.append(Linear(self.in_size_2, self.out_size_2))
        layers2.append(ReLU())
        layers2.append(torch.nn.BatchNorm1d(self.out_size_2))
        dense3dnet2 = Sequential(*layers2)
        self.dd2 = DDm(l = self.l,
                        k = self.k,
                        mlp = dense3dnet2,
                        conv_p  = False,
                        conv_fc = True,
                        conv_fn = True,
                        out_3d  = False)


        self.nn1 = torch.nn.Linear(self.out_size_2, 1024)
        self.bn1 = torch.nn.BatchNorm1d(1024)
        self.nn2 = torch.nn.Linear(1024, 512)
        self.bn2 = torch.nn.BatchNorm1d(512)
        self.nn3 = torch.nn.Linear(512, 265)
        self.bn3 = torch.nn.BatchNorm1d(265)
        self.nn4 = torch.nn.Linear(265, self.nr_classes)

        self.sm = torch.nn.LogSoftmax(dim=1)
#make the prediction of y = 2*x -1 at x = [1, 2]
yhat = forward(x)
print("The prediction: ", yhat)
""" PRACTICE """
x = torch.tensor([[1.0], [2.0], [3.0]])
yhat = forward(x)
print("My result: ", yhat)
""" CLASS LINEAR """
# import class linear
from torch.nn import Linear
torch.manual_seed(1)

# create Linear Regression Model, and print out the parameters

lr = Linear(in_features=1, out_features=1, bias=True)
print("Parameters w and b: ", list(lr.parameters()))

#make the prediction at x = [[1.]]
x = torch.tensor([[1.0]])
yhat = lr(x)
print("The prediction: ", yhat)

# create the prediction using Linear Model
x = torch.tensor([[1.0], [2.0]])
yhat = lr(x)
print("The prediction: ", yhat)
""" PRACTICE """
x = torch.tensor([[1.0], [2.0], [3.0]])
yhat = lr(x)
print("THe prediction: ", yhat)
Exemplo n.º 22
0
 def __init__(self, vocab_size, embedding_dim, context_size):
     super(NGramLanguageModeler, self).__init__()
     self.embeddings = Embedding(vocab_size, embedding_dim)
     self.linear1 = Linear(context_size * embedding_dim, 128)
     self.linear2 = Linear(128, vocab_size)
Exemplo n.º 23
0
import time

import torch
from torch.nn import CrossEntropyLoss, Flatten, Linear, Sequential
from torch.nn.utils.convert_parameters import parameters_to_vector

from backpack.hessianfree.hvp import hessian_vector_product
from backpack.utils.convert_parameters import vector_to_parameter_list
from backpack.utils.examples import load_mnist_data

B = 4
X, y = load_mnist_data(B)

model = Sequential(
    Flatten(),
    Linear(784, 10),
)
lossfunc = CrossEntropyLoss()

print("# 1) Hessian matrix with automatic differentiation | B =", B)

loss = lossfunc(model(X), y)

num_params = sum(p.numel() for p in model.parameters())
hessian = torch.zeros(num_params, num_params)

start = time.time()
for i in range(num_params):
    # GGN-vector product with i.th unit vector yields the i.th row
    e_i = torch.zeros(num_params)
    e_i[i] = 1.0
Exemplo n.º 24
0
 def __init__(self, config):
     super().__init__()
     self.transform_nn = Linear(config.hidden_size, config.hidden_size)
     self.dropout = nn.Dropout(config_args["classifier_dropout_prob"])
     self.activation = nn.ReLU()
     self.out_proj = nn.Linear(config.hidden_size, config.num_labels)
Exemplo n.º 25
0
def linear_model(data_path: Path,
                 output_path: Path,
                 mode: Mode,
                 num_input: int,
                 radius: int,
                 num_output: int,
                 epochs: int,
                 lr: float = typer.Argument(1e-3),
                 lambda_: float = typer.Argument(1e-4)):

    data = json.load(open(data_path, 'r'))

    info = [{} for _ in data]

    X = torch.stack([
        morgan_count_fingerprint(d['id'], num_input, radius,
                                 bitInfo=info[i]).tensor()
        for i, d in enumerate(data)
    ]).float()

    Y = torch.stack([torch.tensor(d['prediction']['output']) for d in data])

    print("X = ", X.numpy())
    print("Y = ",
          Y.numpy() if mode != 'classification' else Y.argmax(dim=-1).numpy())

    create_path(output_path)

    interpretable_model = Sequential(Linear(num_input, num_output))

    optimizer = torch.optim.SGD(interpretable_model.parameters(), lr=lr)
    EPS = 1e-15
    with tq(total=epochs) as pbar:
        for epoch in range(epochs):
            optimizer.zero_grad()

            out = interpretable_model(X).squeeze()
            W = torch.cat(
                [w.flatten() for w in interpretable_model[0].parameters()])
            reg = lambda_ * torch.norm(W, 1)
            loss = F.mse_loss(out, Y) + reg

            description = f"Loss: {loss.item():.4f}"

            if mode == 'classification':
                y1 = F.softmax(out, dim=-1).max(dim=1)[1]
                y2 = F.softmax(Y, dim=-1).max(dim=1)[1]

                acc = accuracy(y1, y2)
                description = description + f", Accuracy: {acc:.2f})"

                if acc == 1:
                    break
            else:
                acc = loss.item()

            loss.backward()
            optimizer.step()

            pbar.update(1)
            pbar.set_description(description)

    weight = interpretable_model[0].weight
    torch.set_printoptions(precision=2)
    np.set_printoptions(precision=2)
    w_abs = weight.abs().detach().numpy()

    for c in range(num_output):
        print(f"Feature importance for class {c}:")
        print(f"max: {w_abs[c].max()}")
        print(f"mean: {w_abs[c].mean()}")
        print(f"std: {w_abs[c].std()}")

    np.save(f"{output_path}/W.npy", weight.detach().numpy())
    np.save(f"{output_path}/morgan_envs.npy", np.array(info))

    for i, d in enumerate(data):
        mol = mol_from_smiles(d['id'])
        d2d = Draw.rdMolDraw2D.MolDraw2DSVG(300, 300)
        d2d.drawOptions().addAtomIndices = True
        d2d.DrawMolecule(mol)
        d2d.FinishDrawing()
        open(f"{output_path}/mol-with-indexes.svg",
             "w").write(d2d.GetDrawingText())
Exemplo n.º 26
0
 def __init__(self):
     super(model, self).__init__()
     self.input = Sequential(Linear(out_features=64, in_features=784),
                             ReLU())
     self.out = Sequential(Linear(in_features=64, out_features=784),
                           Sigmoid())
Exemplo n.º 27
0
class BasicGNN(torch.nn.Module):
    r"""An abstract class for implementing basic GNN models.

    Args:
        in_channels (int): Size of each input sample.
        hidden_channels (int): Size of each hidden sample.
        num_layers (int): Number of message passing layers.
        out_channels (int, optional): If not set to :obj:`None`, will apply a
            final linear transformation to convert hidden node embeddings to
            output size :obj:`out_channels`. (default: :obj:`None`)
        dropout (float, optional): Dropout probability. (default: :obj:`0.`)
        act (str or Callable, optional): The non-linear activation function to
            use. (default: :obj:`"relu"`)
        norm (torch.nn.Module, optional): The normalization operator to use.
            (default: :obj:`None`)
        jk (str, optional): The Jumping Knowledge mode. If specified, the model
            will additionally apply a final linear transformation to transform
            node embeddings to the expected output feature dimensionality.
            (:obj:`None`, :obj:`"last"`, :obj:`"cat"`, :obj:`"max"`,
            :obj:`"lstm"`). (default: :obj:`None`)
        act_first (bool, optional): If set to :obj:`True`, activation is
            applied before normalization. (default: :obj:`False`)
        act_kwargs (Dict[str, Any], optional): Arguments passed to the
            respective activation function defined by :obj:`act`.
            (default: :obj:`None`)
        **kwargs (optional): Additional arguments of the underlying
            :class:`torch_geometric.nn.conv.MessagePassing` layers.
    """
    def __init__(
        self,
        in_channels: int,
        hidden_channels: int,
        num_layers: int,
        out_channels: Optional[int] = None,
        dropout: float = 0.0,
        act: Union[str, Callable, None] = "relu",
        norm: Optional[torch.nn.Module] = None,
        jk: Optional[str] = None,
        act_first: bool = False,
        act_kwargs: Optional[Dict[str, Any]] = None,
        **kwargs,
    ):
        super().__init__()

        from class_resolver.contrib.torch import activation_resolver

        self.in_channels = in_channels
        self.hidden_channels = hidden_channels
        self.num_layers = num_layers

        self.dropout = dropout
        self.act = activation_resolver.make(act, act_kwargs)
        self.jk_mode = jk
        self.act_first = act_first

        if out_channels is not None:
            self.out_channels = out_channels
        else:
            self.out_channels = hidden_channels

        self.convs = ModuleList()
        self.convs.append(
            self.init_conv(in_channels, hidden_channels, **kwargs))
        for _ in range(num_layers - 2):
            self.convs.append(
                self.init_conv(hidden_channels, hidden_channels, **kwargs))
        if out_channels is not None and jk is None:
            self._is_conv_to_out = True
            self.convs.append(
                self.init_conv(hidden_channels, out_channels, **kwargs))
        else:
            self.convs.append(
                self.init_conv(hidden_channels, hidden_channels, **kwargs))

        self.norms = None
        if norm is not None:
            self.norms = ModuleList()
            for _ in range(num_layers - 1):
                self.norms.append(copy.deepcopy(norm))
            if jk is not None:
                self.norms.append(copy.deepcopy(norm))

        if jk is not None and jk != 'last':
            self.jk = JumpingKnowledge(jk, hidden_channels, num_layers)

        if jk is not None:
            if jk == 'cat':
                in_channels = num_layers * hidden_channels
            else:
                in_channels = hidden_channels
            self.lin = Linear(in_channels, self.out_channels)

    def init_conv(self, in_channels: int, out_channels: int,
                  **kwargs) -> MessagePassing:
        raise NotImplementedError

    def reset_parameters(self):
        for conv in self.convs:
            conv.reset_parameters()
        for norm in self.norms or []:
            norm.reset_parameters()
        if hasattr(self, 'jk'):
            self.jk.reset_parameters()
        if hasattr(self, 'lin'):
            self.lin.reset_parameters()

    def forward(self, x: Tensor, edge_index: Adj, *args, **kwargs) -> Tensor:
        """"""
        xs: List[Tensor] = []
        for i in range(self.num_layers):
            x = self.convs[i](x, edge_index, *args, **kwargs)
            if i == self.num_layers - 1 and self.jk_mode is None:
                break
            if self.act_first:
                x = self.act(x)
            if self.norms is not None:
                x = self.norms[i](x)
            if not self.act_first:
                x = self.act(x)
            x = F.dropout(x, p=self.dropout, training=self.training)
            if hasattr(self, 'jk'):
                xs.append(x)

        x = self.jk(xs) if hasattr(self, 'jk') else x
        x = self.lin(x) if hasattr(self, 'lin') else x
        return x

    def __repr__(self) -> str:
        return (f'{self.__class__.__name__}({self.in_channels}, '
                f'{self.out_channels}, num_layers={self.num_layers})')
Exemplo n.º 28
0
    def __init__(self, vocab_size, ext_vocab_size, emb_dim, enc_hidden_dim,
                 c_dim, z_dim, att_hidden_dim, states_sc_hidden=150,
                 cgate_hidden_dim=None):
        """
        :param vocab_size: the number of words by the generator.
        :param ext_vocab_size: the extended number of words that is accessible
            by the copy mechanism.
        :param emb_dim: the number of dimensions in word embeddings.
        :param enc_hidden_dim: GRU encoder's hidden dimension.
        :param c_dim: dimension of the group representation.
        :param z_dim: dimension of the review representation.
        :param att_hidden_dim: hidden dimension of hidden dimension of
            feed-forward networks.
        :param states_sc_hidden: hidden dimension of the score function used
            for production of the group representation c.
        :param cgate_hidden_dim: copy gate hidden dimension.
        """
        assert vocab_size <= ext_vocab_size

        super(CopyCat, self).__init__()
        self._embds = Embedding(ext_vocab_size, emb_dim)

        self._encoder = GruEncoder(input_dim=emb_dim, hidden_dim=enc_hidden_dim)

        # POINTER-GENERATOR NETWORK #

        #   generation network - computes generation distribution over words
        dec_inp_dim = z_dim + enc_hidden_dim
        gen_network = Sequential()
        gen_network.add_module("lin_proj", Linear(dec_inp_dim, emb_dim))
        gen_network.add_module("out_embds", OutEmbds(self._embds, vocab_size))
        gen_network.add_module("softmax", Softmax(dim=-1))

        #   copy gate - computes probability of copying a word
        c_ffnn = Ffnn(z_dim + enc_hidden_dim + emb_dim,
                      hidden_dim=cgate_hidden_dim,
                      non_linearity=Tanh(), output_dim=1)
        copy_gate = Sequential()
        copy_gate.add_module("ffnn", c_ffnn)
        copy_gate.add_module("sigmoid", Sigmoid())

        pgn = PointerGenNetwork(gen=gen_network, copy_gate=copy_gate,
                                ext_vocab_size=ext_vocab_size)

        # COMPLETE DECODER (GRU + ATTENTION + COPY-GEN) #

        #   attention for decoder over encoder's hidden states
        dec_att = Attention(query_dim=z_dim, hidden_dim=att_hidden_dim,
                            value_dim=enc_hidden_dim,
                            non_linearity=Tanh())
        self._keys_creator = dec_att.create_keys
        self._decoder = GruPointerDecoder(input_dim=emb_dim + z_dim,
                                          hidden_dim=z_dim,
                                          contxt_dim=enc_hidden_dim,
                                          att_module=dec_att,
                                          pointer_gen_module=pgn,
                                          cat_contx_to_inp=True,
                                          pass_extra_feat_to_pg=False)

        # NETWORKS THAT PRODUCES LATENT VARIABLES' MUS AND SIGMAS #

        # z inference network
        self._z_inf_network = MuSigmaFfnn(input_dim=enc_hidden_dim + c_dim,
                                          non_linearity=Tanh(),
                                          output_dim=z_dim)
        # z prior network
        self._z_prior_network = MuSigmaFfnn(input_dim=c_dim, output_dim=z_dim)

        # c inference network
        #   scores each state to obtain group context vector
        states_dim = enc_hidden_dim + emb_dim
        self._c_states_scoring = Ffnn(input_dim=states_dim,
                                      hidden_dim=states_sc_hidden,
                                      output_dim=1, non_linearity=Tanh())

        self._c_inf_network = MuSigmaFfnn(input_dim=states_dim,
                                          output_dim=c_dim)
Exemplo n.º 29
0
 def _build_N_net(input_shape, output_shape):
     net = torch.nn.Sequential(Linear(input_shape, 128), ReLU(),
                               Linear(128, output_shape))
     return net.cuda()
Exemplo n.º 30
0
 def __init__(self, i, o):
     super(Residual, self).__init__()
     self.fc = Linear(i, o)
     self.bn = BatchNorm1d(o)
     self.relu = ReLU()