def __init__( self, input_dim: int, num_relations: int, output_dim: Optional[int] = None, use_bias: bool = True, activation: Hint[nn.Module] = None, activation_kwargs: Optional[Mapping[str, Any]] = None, self_loop_dropout: float = 0.2, decomposition: Hint[Decomposition] = None, decomposition_kwargs: Optional[Mapping[str, Any]] = None, ): """ Initialize the layer. :param input_dim: >0 the input dimension :param num_relations: the number of relations :param output_dim: >0 the output dimension. If none is given, use the input dimension. :param use_bias: whether to use a trainable bias :param activation: the activation function to use. Defaults to None, i.e., the identity function serves as activation. :param activation_kwargs: additional keyword-based arguments passed to the activation function for instantiation :param self_loop_dropout: 0 <= self_loop_dropout <= 1 the dropout to use for self-loops :param decomposition: the decomposition to use, cf. Decomposition and decomposition_resolver :param decomposition_kwargs: the keyword-based arguments passed to the decomposition for instantiation """ super().__init__() # cf. https://github.com/MichSchli/RelationPrediction/blob/c77b094fe5c17685ed138dae9ae49b304e0d8d89/code/encoders/message_gcns/gcn_basis.py#L22-L24 # noqa: E501 # there are separate decompositions for forward and backward relations. # the self-loop weight is not decomposed. self.fwd = decomposition_resolver.make( query=decomposition, pos_kwargs=decomposition_kwargs, input_dim=input_dim, num_relations=num_relations, ) output_dim = self.fwd.output_dim self.bwd = decomposition_resolver.make( query=decomposition, pos_kwargs=decomposition_kwargs, input_dim=input_dim, num_relations=num_relations, ) self.w_self_loop = nn.Parameter(torch.empty(input_dim, output_dim)) self.bias = nn.Parameter(torch.empty(output_dim)) if use_bias else None self.dropout = nn.Dropout(p=self_loop_dropout) if activation is not None: activation = activation_resolver.make(query=activation, pos_kwargs=activation_kwargs) self.activation = activation
def __init__( self, channel_list: Optional[Union[List[int], int]] = None, *, in_channels: Optional[int] = None, hidden_channels: Optional[int] = None, out_channels: Optional[int] = None, num_layers: Optional[int] = None, dropout: float = 0., act: str = "relu", batch_norm: bool = True, act_first: bool = False, act_kwargs: Optional[Dict[str, Any]] = None, batch_norm_kwargs: Optional[Dict[str, Any]] = None, bias: bool = True, relu_first: bool = False, ): super().__init__() from class_resolver.contrib.torch import activation_resolver act_first = act_first or relu_first # Backward compatibility. batch_norm_kwargs = batch_norm_kwargs or {} if isinstance(channel_list, int): in_channels = channel_list if in_channels is not None: assert num_layers >= 1 channel_list = [hidden_channels] * (num_layers - 1) channel_list = [in_channels] + channel_list + [out_channels] assert isinstance(channel_list, (tuple, list)) assert len(channel_list) >= 2 self.channel_list = channel_list self.dropout = dropout self.act = activation_resolver.make(act, act_kwargs) self.act_first = act_first self.lins = torch.nn.ModuleList() pairwise = zip(channel_list[:-1], channel_list[1:]) for in_channels, out_channels in pairwise: self.lins.append(Linear(in_channels, out_channels, bias=bias)) self.norms = torch.nn.ModuleList() for hidden_channels in channel_list[1:-1]: if batch_norm: norm = BatchNorm1d(hidden_channels, **batch_norm_kwargs) else: norm = Identity() self.norms.append(norm) self.reset_parameters()
def __init__( self, entity_embedding_dim: int, literal_embedding_dim: int, input_dropout: float = 0.0, gate_activation: HintOrType[nn.Module] = nn.Sigmoid, gate_activation_kwargs: Optional[Mapping[str, Any]] = None, linlayer_activation: HintOrType[nn.Module] = nn.Tanh, linlayer_activation_kwargs: Optional[Mapping[str, Any]] = None, ) -> None: """Instantiate the :class:`torch.nn.Module`. :param entity_embedding_dim: The dimension of the entity representations. :param literal_embedding_dim: The dimension of the literals. :param gate_activation: An optional, pre-instantiated activation module, like :class:`torch.nn.Sigmoid`, used on the gate output. :param linlayer_activation_kwargs: An optional, pre-instantiated activation module, like :class:`torch.nn.Tanh`, used on the linear layer output. """ super().__init__() self.gate_activation = activation_resolver.make( gate_activation, gate_activation_kwargs) self.combination_linear_layer = nn.Linear( entity_embedding_dim + literal_embedding_dim, entity_embedding_dim, ) self.gate_entity_layer = nn.Linear( entity_embedding_dim, entity_embedding_dim, bias=False, ) self.gate_literal_layer = nn.Linear( literal_embedding_dim, entity_embedding_dim, bias=False, ) self.bias = nn.Parameter(torch.zeros(entity_embedding_dim)) self.linlayer_activation = activation_resolver.make( linlayer_activation, linlayer_activation_kwargs) self.dropout = nn.Dropout(input_dropout)
def __init__( self, entity_embedding_dim: int, literal_embedding_dim: int, input_dropout: float = 0.0, activation: HintOrType[nn.Module] = None, activation_kwargs: Optional[Mapping[str, Any]] = None, ) -> None: """Instantiate the :class:`torch.nn.Sequential`. :param entity_embedding_dim: The dimension of the entity representations to which literals are concatenated :param literal_embedding_dim: The dimension of the literals that are concatenated :param input_dropout: The dropout probability of an element to be zeroed. :param activation: An optional, pre-instantiated activation module, like :class:`torch.nn.Tanh`. """ linear = nn.Linear(entity_embedding_dim + literal_embedding_dim, entity_embedding_dim) dropout = nn.Dropout(input_dropout) if activation: activation_instance = activation_resolver.make( activation, activation_kwargs) super().__init__(linear, dropout, activation_instance) else: super().__init__(linear, dropout)
def __init__( self, input_dim: int, output_dim: Optional[int] = None, dropout: float = 0.0, use_bias: bool = True, use_relation_bias: bool = False, composition: Hint[CompositionModule] = None, attention_heads: int = 4, attention_dropout: float = 0.1, activation: Hint[nn.Module] = nn.Identity, activation_kwargs: Optional[Mapping[str, Any]] = None, edge_weighting: HintType[EdgeWeighting] = SymmetricEdgeWeighting, ): """ Initialize the module. :param input_dim: The input dimension. :param output_dim: The output dimension. If None, equals the input dimension. :param dropout: The dropout to use for forward and backward edges. :param use_bias: # TODO: do we really need this? it comes before a mandatory batch norm layer Whether to use bias. :param use_relation_bias: Whether to use a bias for the relation transformation. :param composition: The composition function. :param attention_heads: Number of attention heads when using the attention weighting :param attention_dropout: Dropout for the attention message weighting :param activation: The activation to use. :param activation_kwargs: Additional key-word based arguments passed to the activation. """ super().__init__() # normalize output dimension output_dim = output_dim or input_dim # entity-relation composition self.composition = composition_resolver.make(composition) # edge weighting self.edge_weighting: EdgeWeighting = edge_weight_resolver.make( edge_weighting, output_dim=output_dim, attn_drop=attention_dropout, num_heads=attention_heads) # message passing weights self.w_loop = nn.Parameter(data=torch.empty(input_dim, output_dim)) self.w_fwd = nn.Parameter(data=torch.empty(input_dim, output_dim)) self.w_bwd = nn.Parameter(data=torch.empty(input_dim, output_dim)) # linear relation transformation self.w_rel = nn.Linear(in_features=input_dim, out_features=output_dim, bias=use_relation_bias) # layer-specific self-loop relation representation self.self_loop = nn.Parameter(data=torch.empty(1, input_dim)) # other components self.drop = nn.Dropout(dropout) self.bn = nn.BatchNorm1d(output_dim) self.bias = Bias(output_dim) if use_bias else None self.activation = activation_resolver.make( query=activation, pos_kwargs=activation_kwargs) # initialize self.reset_parameters()
def __init__( self, in_channels: int, hidden_channels: int, num_layers: int, out_channels: Optional[int] = None, dropout: float = 0.0, act: Union[str, Callable, None] = "relu", norm: Optional[torch.nn.Module] = None, jk: Optional[str] = None, act_first: bool = False, act_kwargs: Optional[Dict[str, Any]] = None, **kwargs, ): super().__init__() from class_resolver.contrib.torch import activation_resolver self.in_channels = in_channels self.hidden_channels = hidden_channels self.num_layers = num_layers self.dropout = dropout self.act = activation_resolver.make(act, act_kwargs) self.jk_mode = jk self.act_first = act_first if out_channels is not None: self.out_channels = out_channels else: self.out_channels = hidden_channels self.convs = ModuleList() self.convs.append( self.init_conv(in_channels, hidden_channels, **kwargs)) for _ in range(num_layers - 2): self.convs.append( self.init_conv(hidden_channels, hidden_channels, **kwargs)) if out_channels is not None and jk is None: self.convs.append( self.init_conv(hidden_channels, out_channels, **kwargs)) else: self.convs.append( self.init_conv(hidden_channels, hidden_channels, **kwargs)) self.norms = None if norm is not None: self.norms = ModuleList() for _ in range(num_layers - 1): self.norms.append(copy.deepcopy(norm)) if jk is not None: self.norms.append(copy.deepcopy(norm)) if jk is not None and jk != 'last': self.jk = JumpingKnowledge(jk, hidden_channels, num_layers) if jk is not None: if jk == 'cat': in_channels = num_layers * hidden_channels else: in_channels = hidden_channels self.lin = Linear(in_channels, self.out_channels)