def __init__(self,
in_channels: int,
out_channels: int,
edge_pos_emb=False,
aggr: str = 'softmax',
t: float = 1.0,
learn_t: bool = False,
p: float = 1.0,
learn_p: bool = False,
msg_norm: bool = False,
learn_msg_scale: bool = False,
norm: str = 'batch',
num_layers: int = 2,
eps: float = 1e-7,
**kwargs):
kwargs.setdefault('aggr', None)
super(GENConv, self).__init__(**kwargs)
self.in_channels = in_channels
self.out_channels = out_channels
self.aggr = aggr
self.eps = eps
assert aggr in ['softmax', 'softmax_sg', 'power']
channels = [in_channels]
for i in range(num_layers - 1):
channels.append(in_channels * 2)
channels.append(out_channels)
self.mlp = MLP(channels, norm=norm)
self.msg_norm = MessageNorm(learn_msg_scale) if msg_norm else None
self.initial_t = t
self.initial_p = p
if learn_t and aggr == 'softmax':
self.t = Parameter(torch.Tensor([t]), requires_grad=True)
else:
self.t = t
if learn_p:
self.p = Parameter(torch.Tensor([p]), requires_grad=True)
else:
self.p = p
if edge_pos_emb:
self.edge_enc = TrajPositionalEncoding(d_model=out_channels,
max_len=110)
else:
self.edge_enc = None
def __init__(self,
in_channels: int,
out_channels: int,
aggr: str = 'softmax',
t: float = 1.0,
learn_t: bool = False,
p: float = 1.0,
learn_p: bool = False,
msg_norm: bool = False,
learn_msg_scale: bool = False,
norm: str = 'batch',
num_layers: int = 2,
eps: float = 1e-7,
**kwargs):
super(GENConv, self).__init__(aggr=None, **kwargs)
self.in_channels = in_channels
self.out_channels = out_channels
self.aggr = aggr
self.eps = eps
assert aggr in ['softmax', 'softmax_sg', 'power', 'stat']
channels = [in_channels]
for i in range(num_layers - 1):
channels.append(in_channels * 2)
channels.append(out_channels)
self.mlp = MLP(channels, norm=norm)
self.msg_norm = MessageNorm(learn_msg_scale) if msg_norm else None
self.initial_t = t
self.initial_p = p
if learn_t and aggr == 'softmax':
self.t = Parameter(torch.Tensor([t]), requires_grad=True)
else:
self.t = t
if learn_p:
self.p = Parameter(torch.Tensor([p]), requires_grad=True)
else:
self.p = p
self.lin_stat = nn.Linear(4, 1)
def __init__(self,
in_channels: int,
out_channels: int,
aggr: str = 'softmax',
t: float = 1.0,
learn_t: bool = False,
p: float = 1.0,
learn_p: bool = False,
msg_norm: bool = False,
learn_msg_scale: bool = False,
norm: str = 'batch',
num_layers: int = 2,
eps: float = 1e-7,
**kwargs):
# Backward compatibility:
aggr = 'softmax' if aggr == 'softmax_sg' else aggr
aggr = 'powermean' if aggr == 'power' else aggr
aggr_kwargs = {}
if aggr == 'softmax':
aggr_kwargs = dict(t=t, learn=learn_t)
elif aggr == 'powermean':
aggr_kwargs = dict(p=p, learn=learn_p)
super().__init__(aggr=aggr, aggr_kwargs=aggr_kwargs, **kwargs)
self.in_channels = in_channels
self.out_channels = out_channels
self.eps = eps
channels = [in_channels]
for i in range(num_layers - 1):
channels.append(in_channels * 2)
channels.append(out_channels)
self.mlp = MLP(channels, norm=norm)
self.msg_norm = MessageNorm(learn_msg_scale) if msg_norm else None
def __init__(self, in_channels, out_channels, edge_channels=1, **kwargs):
super(GateConv, self).__init__(aggr='add', **kwargs)
self.in_channels = in_channels
self.out_channels = out_channels
self.edge_channels = edge_channels
self.linear_n = nn.Parameter(torch.Tensor(in_channels, out_channels))
self.linear_e = nn.Parameter(torch.Tensor(edge_channels, out_channels))
# self.linear_attn = LinearMultiHeadedAttention(h=)
self.tfm_encoder = make_transformer_encoder(num_layers=2,
hidden_size=out_channels *
2,
ff_size=out_channels * 2,
num_att_heads=8)
self.msg_norm = MessageNorm(learn_scale=True)
self.linear_msg = nn.Linear(out_channels * 2, out_channels)
self.linear_aggr = nn.Linear(out_channels, out_channels)
self.reset_parameters()
class GENConv(MessagePassing):
r"""The GENeralized Graph Convolution (GENConv) from the `"DeeperGCN: All
You Need to Train Deeper GCNs" <https://arxiv.org/abs/2006.07739>`_ paper.
Supports SoftMax & PowerMean aggregation. The message construction is:
.. math::
\mathbf{x}_i^{\prime} = \mathrm{MLP} \left( \mathbf{x}_i +
\mathrm{AGG} \left( \left\{
\mathrm{ReLU} \left( \mathbf{x}_j + \mathbf{e_{ji}} \right) +\epsilon
: j \in \mathcal{N}(i) \right\} \right)
\right)
.. note::
For an example of using :obj:`GENConv`, see
`examples/ogbn_proteins_deepgcn.py
<https://github.com/pyg-team/pytorch_geometric/blob/master/examples/
ogbn_proteins_deepgcn.py>`_.
Args:
in_channels (int or tuple): Size of each input sample, or :obj:`-1` to
derive the size from the first input(s) to the forward method.
A tuple corresponds to the sizes of source and target
dimensionalities.
out_channels (int): Size of each output sample.
aggr (str, optional): The aggregation scheme to use (:obj:`"softmax"`,
:obj:`"softmax_sg"`, :obj:`"power"`, :obj:`"add"`, :obj:`"mean"`,
:obj:`max`). (default: :obj:`"softmax"`)
t (float, optional): Initial inverse temperature for softmax
aggregation. (default: :obj:`1.0`)
learn_t (bool, optional): If set to :obj:`True`, will learn the value
:obj:`t` for softmax aggregation dynamically.
(default: :obj:`False`)
p (float, optional): Initial power for power mean aggregation.
(default: :obj:`1.0`)
learn_p (bool, optional): If set to :obj:`True`, will learn the value
:obj:`p` for power mean aggregation dynamically.
(default: :obj:`False`)
msg_norm (bool, optional): If set to :obj:`True`, will use message
normalization. (default: :obj:`False`)
learn_msg_scale (bool, optional): If set to :obj:`True`, will learn the
scaling factor of message normalization. (default: :obj:`False`)
norm (str, optional): Norm layer of MLP layers (:obj:`"batch"`,
:obj:`"layer"`, :obj:`"instance"`) (default: :obj:`batch`)
num_layers (int, optional): The number of MLP layers.
(default: :obj:`2`)
eps (float, optional): The epsilon value of the message construction
function. (default: :obj:`1e-7`)
**kwargs (optional): Additional arguments of
:class:`torch_geometric.nn.conv.GenMessagePassing`.
"""
def __init__(self, in_channels: int, out_channels: int,
aggr: str = 'softmax', t: float = 1.0, learn_t: bool = False,
p: float = 1.0, learn_p: bool = False, msg_norm: bool = False,
learn_msg_scale: bool = False, norm: str = 'batch',
num_layers: int = 2, eps: float = 1e-7, **kwargs):
kwargs.setdefault('aggr', None)
super().__init__(**kwargs)
self.in_channels = in_channels
self.out_channels = out_channels
self.aggr = aggr
self.eps = eps
assert aggr in ['softmax', 'softmax_sg', 'power', 'add', 'mean', 'max']
channels = [in_channels]
for i in range(num_layers - 1):
channels.append(in_channels * 2)
channels.append(out_channels)
self.mlp = MLP(channels, norm=norm)
self.msg_norm = MessageNorm(learn_msg_scale) if msg_norm else None
self.initial_t = t
self.initial_p = p
if learn_t and aggr == 'softmax':
self.t = Parameter(torch.Tensor([t]), requires_grad=True)
else:
self.t = t
if learn_p:
self.p = Parameter(torch.Tensor([p]), requires_grad=True)
else:
self.p = p
def reset_parameters(self):
reset(self.mlp)
if self.msg_norm is not None:
self.msg_norm.reset_parameters()
if self.t and isinstance(self.t, Tensor):
self.t.data.fill_(self.initial_t)
if self.p and isinstance(self.p, Tensor):
self.p.data.fill_(self.initial_p)
def forward(self, x: Union[Tensor, OptPairTensor], edge_index: Adj,
edge_attr: OptTensor = None, size: Size = None) -> Tensor:
""""""
if isinstance(x, Tensor):
x: OptPairTensor = (x, x)
# Node and edge feature dimensionalites need to match.
if isinstance(edge_index, Tensor):
if edge_attr is not None:
assert x[0].size(-1) == edge_attr.size(-1)
elif isinstance(edge_index, SparseTensor):
edge_attr = edge_index.storage.value()
if edge_attr is not None:
assert x[0].size(-1) == edge_attr.size(-1)
# propagate_type: (x: OptPairTensor, edge_attr: OptTensor)
out = self.propagate(edge_index, x=x, edge_attr=edge_attr, size=size)
if self.msg_norm is not None:
out = self.msg_norm(x[0], out)
x_r = x[1]
if x_r is not None:
out += x_r
return self.mlp(out)
def message(self, x_j: Tensor, edge_attr: OptTensor) -> Tensor:
msg = x_j if edge_attr is None else x_j + edge_attr
return F.relu(msg) + self.eps
def aggregate(self, inputs: Tensor, index: Tensor,
dim_size: Optional[int] = None) -> Tensor:
if self.aggr == 'softmax':
out = scatter_softmax(inputs * self.t, index, dim=self.node_dim)
return scatter(inputs * out, index, dim=self.node_dim,
dim_size=dim_size, reduce='sum')
elif self.aggr == 'softmax_sg':
out = scatter_softmax(inputs * self.t, index,
dim=self.node_dim).detach()
return scatter(inputs * out, index, dim=self.node_dim,
dim_size=dim_size, reduce='sum')
elif self.aggr == 'power':
min_value, max_value = 1e-7, 1e1
torch.clamp_(inputs, min_value, max_value)
out = scatter(torch.pow(inputs, self.p), index, dim=self.node_dim,
dim_size=dim_size, reduce='mean')
torch.clamp_(out, min_value, max_value)
return torch.pow(out, 1 / self.p)
else:
return scatter(inputs, index, dim=self.node_dim, dim_size=dim_size,
reduce=self.aggr)
def __repr__(self) -> str:
return (f'{self.__class__.__name__}({self.in_channels}, '
f'{self.out_channels}, aggr={self.aggr})')
class GENConv(MessagePassing):
r"""The GENeralized Graph Convolution (GENConv) from the `"DeeperGCN: All
You Need to Train Deeper GCNs" <https://arxiv.org/abs/2006.07739>`_ paper.
Supports SoftMax & PowerMean aggregation. The message construction is:
.. math::
\mathbf{x}_i^{\prime} = \mathrm{MLP} \left( \mathbf{x}_i +
\mathrm{AGG} \left( \left\{
\mathrm{ReLU} \left( \mathbf{x}_j + \mathbf{e_{ji}} \right) +\epsilon
: j \in \mathcal{N}(i) \right\} \right)
\right)
.. note::
For an example of using :obj:`GENConv`, see
`examples/ogbn_proteins_deepgcn.py
<https://github.com/pyg-team/pytorch_geometric/blob/master/examples/
ogbn_proteins_deepgcn.py>`_.
Args:
in_channels (int or tuple): Size of each input sample, or :obj:`-1` to
derive the size from the first input(s) to the forward method.
A tuple corresponds to the sizes of source and target
dimensionalities.
out_channels (int): Size of each output sample.
aggr (str, optional): The aggregation scheme to use (:obj:`"softmax"`,
:obj:`"powermean"`, :obj:`"add"`, :obj:`"mean"`, :obj:`max`).
(default: :obj:`"softmax"`)
t (float, optional): Initial inverse temperature for softmax
aggregation. (default: :obj:`1.0`)
learn_t (bool, optional): If set to :obj:`True`, will learn the value
:obj:`t` for softmax aggregation dynamically.
(default: :obj:`False`)
p (float, optional): Initial power for power mean aggregation.
(default: :obj:`1.0`)
learn_p (bool, optional): If set to :obj:`True`, will learn the value
:obj:`p` for power mean aggregation dynamically.
(default: :obj:`False`)
msg_norm (bool, optional): If set to :obj:`True`, will use message
normalization. (default: :obj:`False`)
learn_msg_scale (bool, optional): If set to :obj:`True`, will learn the
scaling factor of message normalization. (default: :obj:`False`)
norm (str, optional): Norm layer of MLP layers (:obj:`"batch"`,
:obj:`"layer"`, :obj:`"instance"`) (default: :obj:`batch`)
num_layers (int, optional): The number of MLP layers.
(default: :obj:`2`)
eps (float, optional): The epsilon value of the message construction
function. (default: :obj:`1e-7`)
**kwargs (optional): Additional arguments of
:class:`torch_geometric.nn.conv.GenMessagePassing`.
Shapes:
- **input:**
node features :math:`(|\mathcal{V}|, F_{in})` or
:math:`((|\mathcal{V_s}|, F_{in}), (|\mathcal{V_t}|, F_{t})`
if bipartite,
edge indices :math:`(2, |\mathcal{E}|)`,
edge attributes :math:`(|\mathcal{E}|, D)` *(optional)*
- **output:** node features :math:`(|\mathcal{V}|, F_{out})` or
:math:`(|\mathcal{V}_t|, F_{out})` if bipartite
"""
def __init__(self,
in_channels: int,
out_channels: int,
aggr: str = 'softmax',
t: float = 1.0,
learn_t: bool = False,
p: float = 1.0,
learn_p: bool = False,
msg_norm: bool = False,
learn_msg_scale: bool = False,
norm: str = 'batch',
num_layers: int = 2,
eps: float = 1e-7,
**kwargs):
# Backward compatibility:
aggr = 'softmax' if aggr == 'softmax_sg' else aggr
aggr = 'powermean' if aggr == 'power' else aggr
aggr_kwargs = {}
if aggr == 'softmax':
aggr_kwargs = dict(t=t, learn=learn_t)
elif aggr == 'powermean':
aggr_kwargs = dict(p=p, learn=learn_p)
super().__init__(aggr=aggr, aggr_kwargs=aggr_kwargs, **kwargs)
self.in_channels = in_channels
self.out_channels = out_channels
self.eps = eps
channels = [in_channels]
for i in range(num_layers - 1):
channels.append(in_channels * 2)
channels.append(out_channels)
self.mlp = MLP(channels, norm=norm)
self.msg_norm = MessageNorm(learn_msg_scale) if msg_norm else None
def reset_parameters(self):
reset(self.mlp)
self.aggr_module.reset_parameters()
if self.msg_norm is not None:
self.msg_norm.reset_parameters()
def forward(self,
x: Union[Tensor, OptPairTensor],
edge_index: Adj,
edge_attr: OptTensor = None,
size: Size = None) -> Tensor:
""""""
if isinstance(x, Tensor):
x: OptPairTensor = (x, x)
# Node and edge feature dimensionalites need to match.
if isinstance(edge_index, Tensor):
if edge_attr is not None:
assert x[0].size(-1) == edge_attr.size(-1)
elif isinstance(edge_index, SparseTensor):
edge_attr = edge_index.storage.value()
if edge_attr is not None:
assert x[0].size(-1) == edge_attr.size(-1)
# propagate_type: (x: OptPairTensor, edge_attr: OptTensor)
out = self.propagate(edge_index, x=x, edge_attr=edge_attr, size=size)
if self.msg_norm is not None:
out = self.msg_norm(x[0], out)
x_r = x[1]
if x_r is not None:
out += x_r
return self.mlp(out)
def message(self, x_j: Tensor, edge_attr: OptTensor) -> Tensor:
msg = x_j if edge_attr is None else x_j + edge_attr
return msg.relu() + self.eps
def __repr__(self) -> str:
return (f'{self.__class__.__name__}({self.in_channels}, '
f'{self.out_channels}, aggr={self.aggr})')
class GENConv(MessagePassing):
r"""The GENeralized Graph Convolution (GENConv) from the `"DeeperGCN: All
You Need to Train Deeper GCNs" <https://arxiv.org/abs/2006.07739>`_ paper.
Supports SoftMax & PowerMean aggregation. The message construction is:
.. math::
\mathbf{x}_i^{\prime} = \mathrm{MLP} \left( \mathbf{x}_i +
\mathrm{AGG} \left( \left\{
\mathrm{ReLU} \left( \mathbf{x}_j + \mathbf{e_{ji}} \right) +\epsilon
: j \in \mathcal{N}(i) \right\} \right)
\right)
.. note::
For an example of using :obj:`GENConv`, see
`examples/ogbn_proteins_deepgcn.py
<https://github.com/rusty1s/pytorch_geometric/blob/master/examples/
ogbn_proteins_deepgcn.py>`_.
Args:
in_channels (int): Size of each input sample.
out_channels (int): Size of each output sample.
aggr (str, optional): The aggregation scheme to use (:obj:`"softmax"`,
:obj:`"softmax_sg"`, :obj:`"power"`, :obj:`"add"`, :obj:`"mean"`,
:obj:`max`). (default: :obj:`"softmax"`)
t (float, optional): Initial inverse temperature for softmax
aggregation. (default: :obj:`1.0`)
learn_t (bool, optional): If set to :obj:`True`, will learn the value
:obj:`t` for softmax aggregation dynamically.
(default: :obj:`False`)
p (float, optional): Initial power for power mean aggregation.
(default: :obj:`1.0`)
learn_p (bool, optional): If set to :obj:`True`, will learn the value
:obj:`p` for power mean aggregation dynamically.
(default: :obj:`False`)
msg_norm (bool, optional): If set to :obj:`True`, will use message
normalization. (default: :obj:`False`)
learn_msg_scale (bool, optional): If set to :obj:`True`, will learn the
scaling factor of message normalization. (default: :obj:`False`)
norm (str, optional): Norm layer of MLP layers (:obj:`"batch"`,
:obj:`"layer"`, :obj:`"instance"`) (default: :obj:`batch`)
num_layers (int, optional): The number of MLP layers.
(default: :obj:`2`)
eps (float, optional): The epsilon value of the message construction
function. (default: :obj:`1e-7`)
**kwargs (optional): Additional arguments of
:class:`torch_geometric.nn.conv.GenMessagePassing`.
"""
def __init__(self,
in_channels: int,
out_channels: int,
edge_pos_emb=False,
aggr: str = 'softmax',
t: float = 1.0,
learn_t: bool = False,
p: float = 1.0,
learn_p: bool = False,
msg_norm: bool = False,
learn_msg_scale: bool = False,
norm: str = 'batch',
num_layers: int = 2,
eps: float = 1e-7,
**kwargs):
kwargs.setdefault('aggr', None)
super(GENConv, self).__init__(**kwargs)
self.in_channels = in_channels
self.out_channels = out_channels
self.aggr = aggr
self.eps = eps
assert aggr in ['softmax', 'softmax_sg', 'power']
channels = [in_channels]
for i in range(num_layers - 1):
channels.append(in_channels * 2)
channels.append(out_channels)
self.mlp = MLP(channels, norm=norm)
self.msg_norm = MessageNorm(learn_msg_scale) if msg_norm else None
self.initial_t = t
self.initial_p = p
if learn_t and aggr == 'softmax':
self.t = Parameter(torch.Tensor([t]), requires_grad=True)
else:
self.t = t
if learn_p:
self.p = Parameter(torch.Tensor([p]), requires_grad=True)
else:
self.p = p
if edge_pos_emb:
self.edge_enc = TrajPositionalEncoding(d_model=out_channels,
max_len=110)
else:
self.edge_enc = None
def reset_parameters(self):
reset(self.mlp)
if self.msg_norm is not None:
self.msg_norm.reset_parameters()
if self.t and isinstance(self.t, Tensor):
self.t.data.fill_(self.initial_t)
if self.p and isinstance(self.p, Tensor):
self.p.data.fill_(self.initial_p)
def forward(self,
x: Union[Tensor, OptPairTensor],
edge_index: Adj,
edge_attr: OptTensor = None,
edge_attr_len: OptTensor = None,
size: Size = None,
tm_index=None,
duplicates_idx=None) -> Tensor:
""""""
if isinstance(x, Tensor):
x: OptPairTensor = (x, x)
# add_self_loops
if tm_index is not None:
edge_index = add_self_loops(edge_index, tm_index, verbose=False)
# propagate_type: (x: OptPairTensor, edge_attr: OptTensor)
out = self.propagate(edge_index,
x=x,
edge_attr=edge_attr,
edge_attr_len=edge_attr_len,
size=size,
duplicates_idx=duplicates_idx)
if self.msg_norm is not None:
out = self.msg_norm(x[0], out)
x_r = x[1]
if x_r is not None:
out += x_r
return self.mlp(out)
def message(self, x_j: Tensor, edge_attr: OptTensor,
edge_attr_len: OptTensor, duplicates_idx: OptTensor) -> Tensor:
# print('in message')
# print('x_j.size(), edge_attr.size()', x_j.size(), edge_attr.size())
just_made_dup = False
if duplicates_idx is None: # 1st layer, so make duplicates_idx
just_made_dup = True
edge_indice, cnt_ = torch.unique(
edge_attr,
return_counts=True,
)
duplicates_idx = [
[idx] * (c - 1)
for idx, c in zip(edge_indice[cnt_ != 1], cnt_[cnt_ != 1])
]
duplicates_idx = [_ for li in duplicates_idx for _ in li]
# duplicates_idx is not None
x_j = torch.cat((x_j, x_j[duplicates_idx, ]),
dim=0) # ((E+num_duplicates),H)
# print("duplicates_idx: ", duplicates_idx)
# print("x_j: ", x_j)
if just_made_dup:
for i, idx in enumerate(duplicates_idx):
edge_attr[torch.arange(edge_attr.size(0))[edge_attr == idx]
[-1]] = x_j.size(0) - len(duplicates_idx) + i
# print('x_j.size(), edge_attr.size()', x_j.size(), edge_attr.size())
if self.edge_enc:
# print("Adding edge_enc ... ")
edge_pos_emb = self.edge_enc(edge_attr_len)
# print(edge_pos_emb.size())
x_j[edge_attr] += edge_pos_emb.squeeze()
# print("x_j size", x_j.size())
self.duplicates_idx = duplicates_idx
return F.relu(x_j) + self.eps, duplicates_idx
def aggregate(self,
inputs,
index: Tensor,
dim_size: Optional[int] = None) -> Tensor:
inputs, duplicates_idx = inputs
index = torch.cat((index, index[duplicates_idx, ]), dim=0)
# print('duplicates_idx: ',duplicates_idx)
if self.aggr == 'softmax':
out = scatter_softmax(inputs * self.t, index, dim=self.node_dim)
return scatter(inputs * out,
index,
dim=self.node_dim,
dim_size=dim_size,
reduce='sum')
elif self.aggr == 'softmax_sg':
out = scatter_softmax(inputs * self.t, index,
dim=self.node_dim).detach()
return scatter(inputs * out,
index,
dim=self.node_dim,
dim_size=dim_size,
reduce='sum')
else:
min_value, max_value = 1e-7, 1e1
torch.clamp_(inputs, min_value, max_value)
out = scatter(torch.pow(inputs, self.p),
index,
dim=self.node_dim,
dim_size=dim_size,
reduce='mean')
torch.clamp_(out, min_value, max_value)
return torch.pow(out, 1 / self.p)
def __repr__(self):
return '{}({}, {}, aggr={})'.format(self.__class__.__name__,
self.in_channels,
self.out_channels, self.aggr)