def make_model_newast(src_vocab, tgt_vocab, ast_vocab, kg_embed, N=6, d_model=512, d_ff=2048, d_intermediate=512, h=8, dropout=0.1):
"从超参数构造模型"
c = copy.deepcopy
attn = MultiHeadedAttention(h, d_model)
attn_ent = MultiHeadedAttention(h, d_model)
attn_ast = MultiHeadedAttention(h, d_model)
ff = PositionwiseFeedForward(d_model, d_ff, dropout)
position = PositionalEncoding(d_model, dropout)
with open(kg_embed, "r", encoding='utf-8') as f:
lines = json.loads(f.read())
vecs = list()
# vecs.append([0] * 100) # CLS
for (i, line) in enumerate(lines):
if line == "ent_embeddings":
for vec in lines[line]:
vec = [float(x) for x in vec]
vecs.append(vec)
embed = torch.FloatTensor(vecs)
model = EncoderDecoder4newAST(
Encoder4KG(EncoderLayer4KG(d_model, d_intermediate, c(attn), c(attn_ent), c(ff), dropout), N),
Encoder4newAST(EncoderLayer4newAST(d_model, c(attn_ast), c(ff), dropout), N),
Decoder(DecoderLayer(d_model, c(attn), c(attn), c(ff), dropout), N),
nn.Sequential(Embeddings(d_model, src_vocab), c(position)),
torch.nn.Embedding.from_pretrained(embed),
nn.Sequential(Embeddings(d_model, 5000), c(position)),
nn.Sequential(Embeddings(d_model, tgt_vocab), c(position)),
Generator(d_model, tgt_vocab))
# 从代码来看，使用 Glorot / fan_avg初始化参数很重要。
# 对参数进行均匀分布初始化
for p in model.parameters():
if p.dim() > 1:
nn.init.xavier_uniform_(p)
return model
def make_model(src_vocab,
tgt_vocab,
N=6,
d_model=512,
d_ff=2048,
h=8,
dropout=0.1):
"从超参数构造模型"
c = copy.deepcopy
attn = MultiHeadedAttention(h, d_model)
ff = PositionwiseFeedForward(d_model, d_ff, dropout)
position = PositionalEncoding(d_model, dropout)
model = EncoderDecoder(
Encoder(EncoderLayer(d_model, c(attn), c(ff), dropout), N),
Decoder(DecoderLayer(d_model, c(attn), c(attn), c(ff), dropout), N),
nn.Sequential(Embeddings(d_model, src_vocab), c(position)),
nn.Sequential(Embeddings(d_model, tgt_vocab), c(position)),
Generator(d_model, tgt_vocab))
# 从代码来看，使用 Glorot / fan_avg初始化参数很重要。
# 对参数进行均匀分布初始化
for p in model.parameters():
if p.dim() > 1:
nn.init.xavier_uniform_(p)
return model
def make_model_ast(src_vocab, tgt_vocab, voc_size, device, kg_embed, ast_embed, N=6, d_model=512, d_ff=2048,
d_intermediate=512, h=8, dropout=0.1, embedding_dim=512, hidden_size=512):
# "????????
c = copy.deepcopy
attn = MultiHeadedAttention(h, d_model)
attn_ent = MultiHeadedAttention(h, d_model)
ff = PositionwiseFeedForward(d_model, d_ff, dropout)
position = PositionalEncoding(d_model, dropout)
with open(kg_embed, "r", encoding='utf-8') as f:
lines = json.loads(f.read())
vecs = list()
# vecs.append([0] * 100) # CLS
for (i, line) in enumerate(lines):
if line == "ent_embeddings":
for vec in lines[line]:
vec = [float(x) for x in vec]
vecs.append(vec)
embed = torch.FloatTensor(vecs)
ast_size = voc_size
path_listtensor = list()
lstm = LSTM(voc_size, embedding_dim, hidden_size)
# bilstm = BiLSTM(voc_size, embedding_dim, hidden_size, device)
# for k in ast_embed.keys():
# pathlist = ast_embed[k]
# path_tensor = torch.zeros(voc_size).long()
# path_tensor = Variable(path_tensor).to(device)
# for p in pathlist:
# outpath = torch.zeros(voc_size).long()
# outpath = Variable(outpath).to(device)
# p.unsqueeze(0)
# hidden = lstm.init_hidden().to(device)
# cell_state = lstm.init_cell_state().to(device)
# #cell_state_b = lstm.init_cell_state().to(device)
# i = 0
# while i < (list(p.size())[0]):
# # output, hiddenout, cell_state, cell_state_b = bilstm.forward(p[i], p[list(p.size())[0] - 1 - i], hidden, cell_state, cell_state_b)
# output, hidden, cell_state = lstm(p[i], hidden, cell_state)
# i += 1
# outpath += output
# path_tensor += outpath
#
# path_float = list()
# for x in path_tensor:
# path_float.append(float(x))
# path_listtensor.append(path_float)
# ast_embed = torch.FloatTensor(path_listtensor)
model = EncoderDecoder4AST(
Encoder4AST(EncoderLayer4AST(d_model, d_intermediate, ast_size, c(attn), c(attn_ent), c(ff), dropout, voc_size,
embedding_dim, hidden_size, device), N),
Decoder(DecoderLayer(d_model, c(attn), c(attn), c(ff), dropout), N),
nn.Sequential(Embeddings(d_model, src_vocab), c(position)),
torch.nn.Embedding.from_pretrained(embed),
#nn.Sequential(Embeddings(d_model, voc_size)),
nn.Sequential(Embeddings(d_model, tgt_vocab), c(position)),
Generator(d_model, tgt_vocab))
# ?????,?? Glorot / fan_avg?????????
# ????????????
for p in model.parameters():
if p.dim() > 1:
nn.init.xavier_uniform_(p)
return model