def __init__(self, num_layers, in_dim, num_hidden, num_classes, heads,
activation, feat_drop, attn_drop, negative_slope, residual,
readout, device):
super(Classifier, self).__init__()
self.num_layers = num_layers
self.layers = nn.ModuleList()
self.activation = activation
self.readout = readout
self.device = device
# input projection (no residual)
self.layers.append(
conv.GATConv(in_dim, num_hidden, heads[0], feat_drop, attn_drop,
negative_slope, False, self.activation))
# hidden layers
for l in range(1, num_layers):
# due to multi-head, the in_dim = num_hidden * num_heads
self.layers.append(
conv.GATConv(num_hidden * heads[l - 1], num_hidden, heads[l],
feat_drop, attn_drop, negative_slope, residual,
self.activation))
# output projection
self.layers.append(
conv.GATConv(num_hidden * heads[-2], num_hidden, heads[-1],
feat_drop, attn_drop, negative_slope, residual, None))
# last layer
self.classify = nn.Linear(num_hidden, num_classes)
def __init__(self, in_feats, n_hidden, n_classes, n_layers, num_heads,
activation, dropout):
super().__init__()
self.n_layers = n_layers
self.n_hidden = n_hidden
self.n_classes = n_classes
self.layers = nn.ModuleList()
self.layers.append(
dglnn.GATConv((in_feats, in_feats),
n_hidden,
num_heads=num_heads,
feat_drop=0.,
attn_drop=0.,
activation=activation,
negative_slope=0.2))
for i in range(1, n_layers - 1):
self.layers.append(
dglnn.GATConv((n_hidden * num_heads, n_hidden * num_heads),
n_hidden,
num_heads=num_heads,
feat_drop=0.,
attn_drop=0.,
activation=activation,
negative_slope=0.2))
self.layers.append(
dglnn.GATConv((n_hidden * num_heads, n_hidden * num_heads),
n_classes,
num_heads=num_heads,
feat_drop=0.,
attn_drop=0.,
activation=None,
negative_slope=0.2))
def __init__(self, in_feat, hidden_feat, out_feat, rel_names):
super().__init__()
self.conv1 = dglnn.HeteroGraphConv({
rel: dglnn.GATConv(in_feat, hidden_feat, num_heads=4)
for rel in rel_names
})
self.conv2 = dglnn.HeteroGraphConv({
rel: dglnn.GATConv(hidden_feat, out_feat, num_heads=4)
for rel in rel_names
})
def test_gat_conv():
ctx = F.ctx()
g = dgl.rand_graph(100, 1000)
gat = nn.GATConv(5, 2, 4)
feat = F.randn((100, 5))
gat = gat.to(ctx)
h = gat(g, feat)
assert h.shape == (100, 4, 2)
g = dgl.bipartite(sp.sparse.random(100, 200, density=0.1))
gat = nn.GATConv((5, 10), 2, 4)
feat = (F.randn((100, 5)), F.randn((200, 10)))
gat = gat.to(ctx)
h = gat(g, feat)
assert h.shape == (200, 4, 2)
def test_gat_conv():
ctx = F.ctx()
g = dgl.DGLGraph(sp.sparse.random(100, 100, density=0.1), readonly=True)
gat = nn.GATConv(5, 2, 4)
feat = F.randn((100, 5))
gat = gat.to(ctx)
h = gat(g, feat)
assert h.shape[-1] == 2 and h.shape[-2] == 4
def test_gat_conv(g, idtype):
g = g.astype(idtype).to(F.ctx())
ctx = F.ctx()
gat = nn.GATConv(5, 2, 4)
feat = F.randn((g.number_of_nodes(), 5))
gat = gat.to(ctx)
h = gat(g, feat)
assert h.shape == (g.number_of_nodes(), 4, 2)
def __init__(self,
in_feats,
n_hidden,
n_classes,
n_layers,
num_heads,
activation):
super().__init__()
self.n_layers = n_layers
self.n_hidden = n_hidden
self.n_classes = n_classes
self.layers = nn.ModuleList()
self.layers.append(dglnn.GATConv((in_feats, in_feats), n_hidden, num_heads=num_heads, activation=activation))
for i in range(1, n_layers - 1):
self.layers.append(dglnn.GATConv((n_hidden * num_heads, n_hidden * num_heads), n_hidden,
num_heads=num_heads, activation=activation))
self.layers.append(dglnn.GATConv((n_hidden * num_heads, n_hidden * num_heads), n_classes,
num_heads=num_heads, activation=None))
def test_gat_conv_bi(g, idtype, out_dim, num_heads):
g = g.astype(idtype).to(F.ctx())
ctx = F.ctx()
gat = nn.GATConv(5, out_dim, num_heads)
feat = (F.randn((g.number_of_src_nodes(), 5)), F.randn((g.number_of_dst_nodes(), 5)))
gat = gat.to(ctx)
h = gat(g, feat)
assert h.shape == (g.number_of_dst_nodes(), num_heads, out_dim)
_, a = gat(g, feat, get_attention=True)
assert a.shape == (g.number_of_edges(), num_heads, 1)
def test_gat_conv(g, idtype):
g = g.astype(idtype).to(F.ctx())
ctx = F.ctx()
gat = nn.GATConv(5, 2, 4)
feat = F.randn((g.number_of_nodes(), 5))
gat = gat.to(ctx)
h = gat(g, feat)
assert h.shape == (g.number_of_nodes(), 4, 2)
_, a = gat(g, feat, get_attention=True)
assert a.shape == (g.number_of_edges(), 4, 1)
def test_gat_conv(g, idtype, out_dim, num_heads):
g = g.astype(idtype).to(F.ctx())
ctx = F.ctx()
gat = nn.GATConv(5, out_dim, num_heads)
feat = F.randn((g.number_of_src_nodes(), 5))
gat = gat.to(ctx)
h = gat(g, feat)
# test pickle
th.save(gat, tmp_buffer)
assert h.shape == (g.number_of_dst_nodes(), num_heads, out_dim)
_, a = gat(g, feat, get_attention=True)
assert a.shape == (g.number_of_edges(), num_heads, 1)
# test residual connection
gat = nn.GATConv(5, out_dim, num_heads, residual=True)
gat = gat.to(ctx)
h = gat(g, feat)
def __init__(self,
layer_type,
block_type,
activation,
normalization=None,
**core_layer_hyperparms):
super(GNNBasicBlock, self).__init__()
self.layer_type = layer_type
self.block_type = block_type
if self.layer_type in ['gcn', 'gcn_res']:
self.core_layer_type = 'gcn'
self.core_layer = dglnn.GraphConv(
in_feats=core_layer_hyperparms['in_channels'],
out_feats=core_layer_hyperparms['out_channels'],
bias=core_layer_hyperparms['bias'])
elif self.layer_type in ['gat', 'gat_res']:
self.core_layer_type = 'gat'
self.core_layer = dglnn.GATConv(
in_feats=core_layer_hyperparms['in_channels'],
out_feats=int(core_layer_hyperparms['out_channels'] /
core_layer_hyperparms['num_heads']),
num_heads=core_layer_hyperparms['num_heads'],
feat_drop=core_layer_hyperparms['feat_drop'],
attn_drop=core_layer_hyperparms['attn_drop'])
elif self.layer_type in ['sage', 'sage_res']:
self.core_layer_type = 'sage'
self.core_layer = dglnn.SAGEConv(
in_feats=core_layer_hyperparms['in_channels'],
out_feats=core_layer_hyperparms['out_channels'],
aggregator_type='mean',
bias=core_layer_hyperparms['bias'])
else:
raise NotImplementedError
acti_type, acti_hyperparam = activation
if acti_type == 'relu':
self.activation = nn.ReLU(inplace=acti_hyperparam)
elif acti_type == 'lkrelu':
self.activation = nn.LeakyReLU(negative_slope=acti_hyperparam)
elif acti_type == 'elu':
self.activation = nn.ELU(inplace=acti_hyperparam)
elif acti_type == 'no':
self.activation = None
else:
raise NotImplementedError
if 'n' in block_type.split('_'):
self.node_norm = get_normalization(
norm_type=normalization,
num_channels=core_layer_hyperparms['out_channels'])
self.block_type_str = self.get_block_type_str()
def __init__(self, input_size: int, hidden_size: int):
"""
参数：
input_size 输入尺寸
hidden_size 输出尺寸/隐藏尺寸
若是GAT，则至少还有num_heads
输出与隐藏状态之间经过线性变换
"""
super(GRUCell_GAT, self).__init__() #父类初始化函数
#生成卷积层, num_heads为1 不使用多头注意力
self.ConvIR = dglnn.GATConv(input_size + hidden_size,
hidden_size,
1,
allow_zero_in_degree=True)
self.ConvIZ = dglnn.GATConv(input_size + hidden_size,
hidden_size,
1,
allow_zero_in_degree=True)
self.ConvIH = dglnn.GATConv(input_size + hidden_size,
hidden_size,
1,
allow_zero_in_degree=True)
def __init__(self, in_feats, hidden_size, n_head_list, extra_feats):
super().__init__()
self.n_conv = len(n_head_list)
self.gat_list = nn.ModuleList()
for i in range(self.n_conv):
n_head = n_head_list[i]
if i == 0:
layer = dglnn.GATConv(in_feats, hidden_size, n_head)
else:
n_head_last = n_head_list[i - 1]
layer = dglnn.GATConv(n_head_last * hidden_size, hidden_size,
n_head)
torch.nn.init.normal_(layer.attn_l, std=0.5)
torch.nn.init.normal_(layer.attn_r, std=0.5)
torch.nn.init.normal_(layer.fc.weight, std=0.5)
self.gat_list.append(layer)
self.readout = WeightedAverage(hidden_size * n_head_list[-1])
self.mlp = MLPModel(hidden_size * n_head_list[-1] + extra_feats, 1,
[2 * hidden_size, hidden_size])
def __init__(self):
super(StochasticNetwork, self).__init__()
if config.NETWORK == 'SAGE':
self.layers = [
dglnn.SAGEConv(config.IN_FEATURES,
config.HIDDEN_FEATURES,
aggregator_type='mean',
feat_drop=config.DROPOUT),
dglnn.SAGEConv(config.HIDDEN_FEATURES,
config.HIDDEN_FEATURES,
aggregator_type='mean',
feat_drop=config.DROPOUT)
]
elif config.NETWORK == 'GAT':
self.layers = [
dglnn.GATConv(config.IN_FEATURES,
config.HIDDEN_FEATURES,
feat_drop=config.DROPOUT,
attn_drop=config.ATTN_DROPOUT,
num_heads=config.ATTN_HEADS),
dglnn.GATConv(config.ATTN_HEADS * config.HIDDEN_FEATURES,
config.HIDDEN_FEATURES,
feat_drop=config.DROPOUT,
num_heads=1)
]
elif config.NETWORK == 'GIN':
self.mlp1 = MLP(1, config.IN_FEATURES, config.HIDDEN_FEATURES,
config.HIDDEN_FEATURES)
self.mlp2 = MLP(1, config.HIDDEN_FEATURES, config.HIDDEN_FEATURES,
config.HIDDEN_FEATURES)
self.layers = [
dglnn.GINConv(apply_func=self.mlp1, aggregator_type='mean'),
dglnn.GINConv(apply_func=self.mlp2, aggregator_type='mean'),
]
self.layers = torch.nn.ModuleList(self.layers)
self.final = nn.Linear(config.HIDDEN_FEATURES, 2)
def test_gat_conv():
ctx = F.ctx()
g = dgl.rand_graph(100, 1000)
gat = nn.GATConv(5, 2, 4)
feat = F.randn((100, 5))
gat = gat.to(ctx)
h = gat(g, feat)
assert h.shape == (100, 4, 2)
g = dgl.bipartite(sp.sparse.random(100, 200, density=0.1))
gat = nn.GATConv((5, 10), 2, 4)
feat = (F.randn((100, 5)), F.randn((200, 10)))
gat = gat.to(ctx)
h = gat(g, feat)
assert h.shape == (200, 4, 2)
g = dgl.graph(sp.sparse.random(100, 100, density=0.001))
seed_nodes = th.unique(g.edges()[1])
block = dgl.to_block(g, seed_nodes)
gat = nn.GATConv(5, 2, 4)
feat = F.randn((block.number_of_src_nodes(), 5))
gat = gat.to(ctx)
h = gat(block, feat)
assert h.shape == (block.number_of_dst_nodes(), 4, 2)
def __init__(self,
dim_in,
dim_out,
dim_t,
numr,
nume,
g,
dropout=0,
deepth=2,
sampling=None,
granularity=1,
r_limit=None):
super(EmbModule, self).__init__()
self.dim_in = dim_in
self.dim_out = dim_out
self.dim_t = dim_t
self.numr = numr
self.nume = nume
self.deepth = deepth
self.g = g
self.granularity = granularity
mods = dict()
mods['time_enc'] = TimeEnc(dim_t, nume)
mods['entity_emb'] = nn.Embedding(nume, dim_in)
if r_limit is None:
r_limit = numr
for l in range(self.deepth):
mods['norm' + str(l)] = nn.LayerNorm(dim_in + dim_t)
# mods['dropout' + str(l)] = nn.Dropout(dropout)
conv_dict = dict()
for r in range(r_limit):
conv_dict['r' + str(r)] = dglnn.GATConv(dim_in + dim_t,
dim_out // 4,
4,
feat_drop=dropout,
attn_drop=dropout,
residual=False)
conv_dict['-r' + str(r)] = dglnn.GATConv(dim_in + dim_t,
dim_out // 4,
4,
feat_drop=dropout,
attn_drop=dropout,
residual=False)
conv_dict['self'] = dglnn.GATConv(dim_in + dim_t,
dim_out // 4,
4,
feat_drop=dropout,
attn_drop=dropout,
residual=False)
# conv_dict['r' + str(r)] = dglnn.GraphConv(dim_in + dim_t, dim_out)
# conv_dict['-r' + str(r)] = dglnn.GraphConv(dim_in + dim_t, dim_out)
# conv_dict['self'] = dglnn.GraphConv(dim_in + dim_t, dim_out)
mods['conv' + str(l)] = dglnn.HeteroGraphConv(conv_dict,
aggregate='mean')
mods['act' + str(l)] = nn.ReLU()
dim_in = dim_out
self.mods = nn.ModuleDict(mods)
if sampling is not None:
fanouts = [int(d) for d in sampling.split('/')]
self.sampler = dgl.dataloading.MultiLayerNeighborSampler(
fanouts=fanouts)
else:
self.sampler = dgl.dataloading.MultiLayerFullNeighborSampler(
self.deepth)
def __init__(self, args, config):
super(NumericHGN, self).__init__()
self.args = args
self.config = config
self.encoder = ContextEncoder(self.args, config)
self.bi_attn = BiAttention(args, self.config.hidden_size)
self.bi_attn_linear = nn.Linear(self.config.hidden_size * 4,
self.config.hidden_size)
self.bi_lstm = nn.LSTM(self.config.hidden_size,
self.config.hidden_size,
bidirectional=True)
self.para_node_mlp = nn.Linear(self.config.hidden_size * 2,
self.config.hidden_size)
self.sent_node_mlp = nn.Linear(self.config.hidden_size * 2,
self.config.hidden_size)
self.ent_node_mlp = nn.Linear(self.config.hidden_size * 2,
self.config.hidden_size)
# https://docs.dgl.ai/api/python/nn.pytorch.html#dgl.nn.pytorch.HeteroGraphConv
self.gat = dglnn.HeteroGraphConv(
{
"ps":
dglnn.GATConv(self.config.hidden_size,
self.config.hidden_size,
num_heads=1),
"sp":
dglnn.GATConv(self.config.hidden_size,
self.config.hidden_size,
num_heads=1),
"se":
dglnn.GATConv(self.config.hidden_size,
self.config.hidden_size,
num_heads=1),
"es":
dglnn.GATConv(self.config.hidden_size,
self.config.hidden_size,
num_heads=1),
"pp":
dglnn.GATConv(self.config.hidden_size,
self.config.hidden_size,
num_heads=1),
"ss":
dglnn.GATConv(self.config.hidden_size,
self.config.hidden_size,
num_heads=1),
"qp":
dglnn.GATConv(self.config.hidden_size,
self.config.hidden_size,
num_heads=1),
"pq":
dglnn.GATConv(self.config.hidden_size,
self.config.hidden_size,
num_heads=1),
"qe":
dglnn.GATConv(self.config.hidden_size,
self.config.hidden_size,
num_heads=1),
"eq":
dglnn.GATConv(self.config.hidden_size,
self.config.hidden_size,
num_heads=1),
# TODO: Need (i) bi-directional edges and (ii) more edge types (e.g., question-paragraph, paragraph-paragraph, etc.)
},
aggregate='sum'
) # TODO: May need to change aggregate function (test it!) - ‘sum’, ‘max’, ‘min’, ‘mean’, ‘stack’.
self.gated_attn = GatedAttention(self.args, self.config)
self.para_mlp = nn.Sequential(
nn.Linear(self.config.hidden_size, self.config.hidden_size),
nn.Linear(self.config.hidden_size, args.num_paragraphs))
self.sent_mlp = nn.Sequential(
nn.Linear(self.config.hidden_size, self.config.hidden_size),
nn.Linear(self.config.hidden_size, args.num_sentences))
self.ent_mlp = nn.Sequential(
nn.Linear(self.config.hidden_size, self.config.hidden_size),
nn.Linear(self.config.hidden_size, args.num_entities))
self.span_mlp = nn.Sequential(
nn.Linear(self.config.hidden_size * 4, self.config.hidden_size),
nn.Linear(self.config.hidden_size, self.config.num_labels))
self.answer_type_mlp = nn.Sequential(
nn.Linear(self.config.hidden_size * 4, self.config.hidden_size),
nn.Linear(self.config.hidden_size, 3))
