def __init__(self, in_features: int, out_features: int, phm_dim: int, phm_rule: Union[None, nn.ParameterList],
learn_phm: True, bias: bool = True,
add_self_loops: bool = True,
w_init: str = "phm", c_init: str = "standard",
aggr: str = "softmax", same_dim: bool = True,
msg_encoder: str = "identity",
**kwargs) -> None:
super(PHMConvSoftmax, self).__init__(aggr=None)
self.in_features = in_features
self.out_features = out_features
self.phm_dim = phm_dim
self.phm_rule = phm_rule
self.learn_phm = learn_phm
self.bias = bias
self.add_self_loops = add_self_loops
self.w_init = w_init
self.c_init = c_init
self.aggr = aggr
self.same_dim = same_dim
self.transform = PHMLinear(in_features=in_features, out_features=out_features, phm_rule=phm_rule,
phm_dim=phm_dim, bias=bias,
w_init=w_init, c_init=c_init, learn_phm=learn_phm)
self.initial_beta = kwargs.get("initial_beta")
self.learn_beta = kwargs.get("learn_beta")
self.beta = nn.Parameter(torch.tensor(self.initial_beta), requires_grad=self.learn_beta)
self.msg_encoder_str = msg_encoder
self.msg_encoder = get_module_activation(activation=msg_encoder)
self.reset_parameters()
def __init__(self, in_features: int, out_features: int, phm_dim: int, phm_rule: Union[None, nn.ParameterList],
learn_phm: True, bias: bool = True,
add_self_loops: bool = True,
w_init: str = "phm", c_init: str = "standard",
aggr: str = "add", same_dim: bool = True,
msg_encoder: str = "identity") -> None:
super(PHMConv, self).__init__(aggr=aggr)
self.in_features = in_features
self.out_features = out_features
self.phm_dim = phm_dim
self.phm_rule = phm_rule
self.learn_phm = learn_phm
self.bias = bias
self.add_self_loops = add_self_loops
self.w_init = w_init
self.c_init = c_init
self.aggr = aggr
self.same_dim = same_dim
self.transform = PHMLinear(in_features=in_features, out_features=out_features, phm_rule=phm_rule,
phm_dim=phm_dim, bias=bias, w_init=w_init, c_init=c_init,
learn_phm=learn_phm)
self.msg_encoder_str = msg_encoder
self.msg_encoder = get_module_activation(activation=msg_encoder)
self.reset_parameters()
def __init__(self, phm_dim: int, in_features: int, out_features: int, learn_phm: bool, init: str,
phm_rule, aggregators: List[str], scalers: Optional[List[str]],
deg: Optional[torch.Tensor]) -> None:
super(PNAAggregator, self).__init__()
self.in_features = in_features
self.out_features = out_features
self.phm_dim = phm_dim
self.aggregators_l = aggregators
self.aggregators = [AGGREGATORS[aggr] for aggr in aggregators]
self.scalers_l = scalers
if scalers:
self.scalers = [SCALERS[scale] for scale in scalers]
out_trafo_dim = in_features*(len(aggregators) * len(scalers))
self.deg = deg.to(torch.float)
self.avg_deg: Dict[str, float] = {
'lin': self.deg.mean().item(),
'log': (self.deg + 1).log().mean().item(),
'exp': self.deg.exp().mean().item(),
}
else:
self.scalers = None
self.avg_deg = None
out_trafo_dim = in_features*len(aggregators)
self.transform = PHMLinear(in_features=out_trafo_dim, out_features=out_features, bias=True,
phm_dim=phm_dim, phm_rule=phm_rule, learn_phm=learn_phm, init=init)
self.reset_parameters()
def __init__(self, in_features: int, out_features: int,
phm_dim: int, phm_rule: Union[None, nn.ParameterList], learn_phm: bool,
bias: bool,
activation: str, norm: Optional[str],
w_init: str, c_init: str,
deg: torch.Tensor,
aggregators: List[str] = ['mean', 'min', 'max', 'std'],
scalers: List[str] = ['identity', 'amplification', 'attenuation'],
post_layers: int = 1,
msg_encoder: str = "relu",
**kwargs):
super(PHMPNAConvSimple, self).__init__(aggr=None, node_dim=0, **kwargs)
self.in_features = in_features
self.out_features = out_features
self.bias_flag = bias
self.activation_str = activation
self.norm = norm
self.phm_dim = phm_dim
self.phm_rule = phm_rule
self.w_init = w_init
self.c_init = c_init
self.learn_phm = learn_phm
self.aggregators_l = aggregators
self.scalers_l = scalers
self.aggregators = [AGGREGATORS[aggr] for aggr in aggregators]
self.scalers = [SCALERS[scale] for scale in scalers]
self.F_in = in_features
self.F_out = self.out_features
self.deg = deg.to(torch.float)
self.avg_deg: Dict[str, float] = {
'lin': self.deg.mean().item(),
'log': (self.deg + 1).log().mean().item(),
'exp': self.deg.exp().mean().item(),
}
in_features = (len(aggregators) * len(scalers)) * self.F_in
modules = [PHMLinear(in_features=in_features, out_features=self.F_out, bias=self.bias_flag,
phm_dim=self.phm_dim, phm_rule=self.phm_rule,
w_init=self.w_init, c_init=self.c_init)]
self.post_layers = post_layers
for _ in range(post_layers - 1):
if self.norm:
modules += [PHMNorm(num_features=self.F_out, phm_dim=self.phm_dim, type="naive-batch-norm")]
modules += [get_module_activation(self.activation_str)]
modules += [PHMLinear(in_features=self.F_out, out_features=self.F_out, bias=self.bias_flag,
phm_dim=self.phm_dim, phm_rule=self.phm_rule,
w_init=self.w_init, c_init=self.c_init)]
self.transform = nn.Sequential(*modules)
self.msg_encoder_str = msg_encoder
self.msg_encoder = get_module_activation(activation=msg_encoder)
self.reset_parameters()
class PHMSoftAttentionPooling(nn.Module):
def __init__(self, embed_dim: int, phm_dim: int, phm_rule: Union[None, nn.ParameterList],
learn_phm: bool = True,
bias: bool = True, w_init: str = "phm", c_init: str = "standard",
real_trafo: str = "linear"):
super(PHMSoftAttentionPooling, self).__init__()
self.embed_dim = embed_dim
self.phm_dim = phm_dim
self.w_init = w_init
self.c_init = c_init
self.phm_rule = phm_rule
self.learn_phm = learn_phm
self.real_trafo_type = real_trafo
self.bias = bias
self.linear = PHMLinear(in_features=self.embed_dim, out_features=self.embed_dim, phm_dim=self.phm_dim,
phm_rule=phm_rule,
learn_phm=learn_phm, w_init=w_init, c_init=c_init,
bias=bias)
self.real_trafo = RealTransformer(type=self.real_trafo_type, phm_dim=self.phm_dim,
in_features=self.embed_dim, bias=True)
self.sigmoid = nn.Sigmoid()
self.sum_pooling = PHMGlobalSumPooling(phm_dim=self.phm_dim)
self.reset_parameters()
def reset_parameters(self):
self.real_trafo.reset_parameters()
self.linear.reset_parameters()
def forward(self, x: torch.Tensor, batch: Batch) -> torch.Tensor:
out = self.linear(x) # get logits
out = self.real_trafo(out) # "transform" to real-valued
out = self.sigmoid(out) # get "probabilities"
#x = torch.stack([*x.split(split_size=self.embed_dim, dim=-1)], dim=0)
x = x.reshape(x.size(0), self.phm_dim, self.embed_dim)
# apply element-wise hadamard product through broadcasting
out = out.unsqueeze(dim=1)
x = out * x
x = x.reshape(x.size(0), self.phm_dim*self.embed_dim)
x = self.sum_pooling(x, batch=batch)
return x
def __repr__(self):
return "{}(embed_dim={}, phm_dim={}, phm_rule={}, learn_phm={}," \
"bias={}, init='{}', real_trafo='{}')".format(self.__class__.__name__,
self.embed_dim,
self.phm_dim,
self.phm_rule,
self.learn_phm,
self.bias,
self.init,
self.real_trafo_type)
class PNAAggregator(nn.Module):
""" Principal Neighbourhood Aggregator inherits from nn.Module in case we want to further parametrize. """
def __init__(self, phm_dim: int, in_features: int, out_features: int, learn_phm: bool, init: str,
phm_rule, aggregators: List[str], scalers: Optional[List[str]],
deg: Optional[torch.Tensor]) -> None:
super(PNAAggregator, self).__init__()
self.in_features = in_features
self.out_features = out_features
self.phm_dim = phm_dim
self.aggregators_l = aggregators
self.aggregators = [AGGREGATORS[aggr] for aggr in aggregators]
self.scalers_l = scalers
if scalers:
self.scalers = [SCALERS[scale] for scale in scalers]
out_trafo_dim = in_features*(len(aggregators) * len(scalers))
self.deg = deg.to(torch.float)
self.avg_deg: Dict[str, float] = {
'lin': self.deg.mean().item(),
'log': (self.deg + 1).log().mean().item(),
'exp': self.deg.exp().mean().item(),
}
else:
self.scalers = None
self.avg_deg = None
out_trafo_dim = in_features*len(aggregators)
self.transform = PHMLinear(in_features=out_trafo_dim, out_features=out_features, bias=True,
phm_dim=phm_dim, phm_rule=phm_rule, learn_phm=learn_phm, init=init)
self.reset_parameters()
def reset_parameters(self):
self.transform.reset_parameters()
def forward(self, x: torch.Tensor, idx: torch.Tensor, dim_size: Optional[int] = None, dim: int = 0) -> torch.Tensor:
outs = [aggr(x, idx, dim_size) for aggr in self.aggregators]
# concatenate the different aggregator results, considering the shape of the hypercomplex components.
out = phm_cat(tensors=outs, phm_dim=self.phm_dim, dim=-1)
if self.scalers is not None:
deg = degree(idx, dim_size, dtype=x.dtype).view(-1, 1)
# concatenate the different aggregator results, considering the shape of the hypercomplex components.
outs = [scaler(out, deg, self.avg_deg) for scaler in self.scalers]
out = phm_cat(tensors=outs, phm_dim=self.phm_dim, dim=-1)
out = self.transform(out)
return out
def __init__(self, embed_dim: int, phm_dim: int, phm_rule: Union[None, nn.ParameterList],
learn_phm: bool = True,
bias: bool = True, w_init: str = "phm", c_init: str = "standard",
real_trafo: str = "linear"):
super(PHMSoftAttentionPooling, self).__init__()
self.embed_dim = embed_dim
self.phm_dim = phm_dim
self.w_init = w_init
self.c_init = c_init
self.phm_rule = phm_rule
self.learn_phm = learn_phm
self.real_trafo_type = real_trafo
self.bias = bias
self.linear = PHMLinear(in_features=self.embed_dim, out_features=self.embed_dim, phm_dim=self.phm_dim,
phm_rule=phm_rule,
learn_phm=learn_phm, w_init=w_init, c_init=c_init,
bias=bias)
self.real_trafo = RealTransformer(type=self.real_trafo_type, phm_dim=self.phm_dim,
in_features=self.embed_dim, bias=True)
self.sigmoid = nn.Sigmoid()
self.sum_pooling = PHMGlobalSumPooling(phm_dim=self.phm_dim)
self.reset_parameters()
def __init__(self,
in_features: int,
phm_dim: int,
phm_rule: Union[None, nn.ParameterList],
hidden_layers: list,
out_features: int,
activation: str,
bias: bool,
norm: str,
w_init: str,
c_init: str,
dropout: Union[float, list],
learn_phm: bool = True,
same_dropout: bool = False,
real_trafo: str = "linear") -> None:
super(PHMDownstreamNet, self).__init__()
self.in_features = in_features
self.out_features = out_features
self.learn_phm = learn_phm
self.phm_rule = phm_rule
self.phm_dim = phm_dim
self.hidden_layers = hidden_layers
self.activation_str = activation
self.activation_func = get_module_activation(activation)
self.w_init = w_init
self.c_init = c_init
self.bias = bias
self.dropout = [dropout] * len(hidden_layers) if isinstance(
dropout, float) else dropout
assert len(self.dropout) == len(self.hidden_layers), "dropout list must be of the same size " \
"as number of hidden layer"
self.norm_type = norm
self.same_dropout = same_dropout
# affine linear layers
# input -> first hidden layer
self.affine = [
PHMLinear(in_features=in_features,
phm_dim=self.phm_dim,
phm_rule=phm_rule,
out_features=self.hidden_layers[0],
learn_phm=learn_phm,
bias=bias,
w_init=w_init,
c_init=c_init)
]
# hidden layers
self.affine += [
PHMLinear(in_features=self.hidden_layers[i],
out_features=self.hidden_layers[i + 1],
phm_dim=self.phm_dim,
learn_phm=learn_phm,
phm_rule=phm_rule,
bias=bias,
w_init=w_init,
c_init=c_init)
for i in range(len(self.hidden_layers) - 1)
]
# output layer
self.affine += [
PHMLinear(in_features=self.hidden_layers[-1],
out_features=self.out_features,
phm_rule=phm_rule,
phm_dim=self.phm_dim,
learn_phm=learn_phm,
w_init=w_init,
c_init=c_init,
bias=bias)
]
self.affine = nn.ModuleList(self.affine)
# transform the output quaternionic vector to real vector with Real_Transformer module
self.real_trafo_type = real_trafo
self.real_trafo = RealTransformer(type=self.real_trafo_type,
in_features=self.out_features,
phm_dim=self.phm_dim,
bias=True)
# normalizations
self.norm_flag = False
if self.norm_type:
norm_type = self.norm_type
self.norm = [
PHMNorm(num_features=dim, phm_dim=self.phm_dim, type=norm_type)
for dim in self.hidden_layers
]
self.norm = nn.ModuleList(self.norm)
self.norm_flag = True
self.reset_parameters()
class PHMConv(MessagePassing):
r"""
Parametrized Hypercomplex Graphconvolution operator that uses edge-attributes.
Transformation is a linear layer.
"""
def __init__(self, in_features: int, out_features: int, phm_dim: int, phm_rule: Union[None, nn.ParameterList],
learn_phm: True, bias: bool = True,
add_self_loops: bool = True,
w_init: str = "phm", c_init: str = "standard",
aggr: str = "add", same_dim: bool = True,
msg_encoder: str = "identity") -> None:
super(PHMConv, self).__init__(aggr=aggr)
self.in_features = in_features
self.out_features = out_features
self.phm_dim = phm_dim
self.phm_rule = phm_rule
self.learn_phm = learn_phm
self.bias = bias
self.add_self_loops = add_self_loops
self.w_init = w_init
self.c_init = c_init
self.aggr = aggr
self.same_dim = same_dim
self.transform = PHMLinear(in_features=in_features, out_features=out_features, phm_rule=phm_rule,
phm_dim=phm_dim, bias=bias, w_init=w_init, c_init=c_init,
learn_phm=learn_phm)
self.msg_encoder_str = msg_encoder
self.msg_encoder = get_module_activation(activation=msg_encoder)
self.reset_parameters()
def reset_parameters(self):
self.transform.reset_parameters()
def forward(self, x: torch.Tensor, edge_index: Adj, edge_attr: torch.Tensor, size: Size = None) -> torch.Tensor:
if self.add_self_loops:
x_c = x.clone()
# propagate messages
x = self.propagate(edge_index=edge_index, x=x, edge_attr=edge_attr, size=size)
if self.same_dim:
x = self.transform(x)
if self.add_self_loops:
x += x_c
else:
if self.add_self_loops:
x += x_c
x = self.transform(x)
return x
def message(self, x_j: torch.Tensor, edge_attr: torch.Tensor) -> torch.Tensor:
assert x_j.size(-1) == edge_attr.size(-1)
return self.msg_encoder(x_j + edge_attr)
def __repr__(self):
return "{}(in_features={}, out_features={}, phm_dim={}, phm_rule={}," \
" learn_phm={}, bias={}, add_self_loops={}, " \
", w_init='{}', c_init='{}', aggr='{}')".format(self.__class__.__name__,
self.in_features,
self.out_features,
self.phm_dim,
self.phm_rule,
self.learn_phm,
self.bias,
self.add_self_loops,
self.w_init, self.c_init,
self.aggr)