def __init__(self,
vocab_size,
embedding_size=128,
hidden_dropout_prob=0.1,
max_position_embeddings=512,
type_vocab_size=16,
padding_idx=0):
super().__init__()
self.word_embeddings = nn.Embedding(
vocab_size, embedding_size, padding_idx=padding_idx)
self.position_embeddings = nn.Embedding(max_position_embeddings,
embedding_size)
self.token_type_embeddings = nn.Embedding(type_vocab_size,
embedding_size)
self.layer_norm = nn.LayerNorm(embedding_size)
self.dropout = nn.Dropout(hidden_dropout_prob)
def __init__(self, cfg, name=None):
"""
Fundamental pretrained Ernie model
"""
log.debug('init ErnieModel with config: %s' % repr(cfg))
nn.Layer.__init__(self)
d_model = cfg['hidden_size']
d_emb = cfg.get('emb_size', cfg['hidden_size'])
d_vocab = cfg['vocab_size']
d_pos = cfg['max_position_embeddings']
d_sent = cfg.get("sent_type_vocab_size") or cfg['type_vocab_size']
self.n_head = cfg['num_attention_heads']
self.return_additional_info = cfg.get('return_additional_info', False)
initializer = nn.initializer.TruncatedNormal(
std=cfg['initializer_range'])
self.ln = _build_ln(d_model, name=append_name(name, 'pre_encoder'))
self.word_emb = nn.Embedding(
d_vocab,
d_emb,
weight_attr=P.ParamAttr(name=append_name(name, 'word_embedding'),
initializer=initializer))
self.pos_emb = nn.Embedding(
d_pos,
d_emb,
weight_attr=P.ParamAttr(name=append_name(name, 'pos_embedding'),
initializer=initializer))
self.sent_emb = nn.Embedding(
d_sent,
d_emb,
weight_attr=P.ParamAttr(name=append_name(name, 'sent_embedding'),
initializer=initializer))
prob = cfg['hidden_dropout_prob']
self.dropout = nn.Dropout(p=prob)
self.encoder_stack = ErnieEncoderStack(cfg,
append_name(name, 'encoder'))
if cfg.get('has_pooler', True):
self.pooler = _build_linear(
cfg['hidden_size'],
cfg['hidden_size'],
append_name(name, 'pooled_fc'),
initializer,
)
else:
self.pooler = None
self.train()
def __init__(self,
vocab_size,
hidden_size=768,
hidden_dropout_prob=0.1,
max_position_embeddings=512,
type_vocab_size=16,
pad_token_id=0):
super(RobertaEmbeddings, self).__init__()
self.word_embeddings = nn.Embedding(vocab_size,
hidden_size,
padding_idx=pad_token_id)
self.position_embeddings = nn.Embedding(max_position_embeddings,
hidden_size)
self.token_type_embeddings = nn.Embedding(type_vocab_size, hidden_size)
self.layer_norm = nn.LayerNorm(hidden_size)
self.dropout = nn.Dropout(hidden_dropout_prob)
self.padding_idx = pad_token_id
def __init__(self, d_model, vocab, padding_idx, scale_embedding):
super(Embeddings, self).__init__()
self.embedding = nn.Embedding(vocab, d_model, padding_idx=padding_idx)
w0 = np.random.normal(0.0, d_model**-0.5,
(vocab, d_model)).astype(np.float32)
self.embedding.weight.set_value(w0)
self.d_model = d_model
self.scale_embedding = scale_embedding
def __init__(self,
vocab_size,
d_model,
hidden_dropout_prob,
memory_len,
max_position_embeddings=512,
type_vocab_size=3,
padding_idx=0):
super(ErnieDocEmbeddings, self).__init__()
self.word_emb = nn.Embedding(vocab_size, d_model)
self.pos_emb = nn.Embedding(max_position_embeddings * 2 + memory_len,
d_model)
self.token_type_emb = nn.Embedding(type_vocab_size, d_model)
self.memory_len = memory_len
self.dropouts = nn.LayerList(
[nn.Dropout(hidden_dropout_prob) for i in range(3)])
self.norms = nn.LayerList([nn.LayerNorm(d_model) for i in range(3)])
def __init__(self, vocab_size, embed_dim, hidden_size, num_layers):
super(Seq2SeqEncoder, self).__init__()
self.embedder = nn.Embedding(vocab_size, embed_dim)
self.lstm = nn.LSTM(input_size=embed_dim,
hidden_size=hidden_size,
num_layers=num_layers,
dropout=0.2 if num_layers > 1 else 0.)
def __init__(self):
super(SimpleNet, self).__init__()
self.word_embeddings = nn.Embedding(vocab_size, hidden_size)
self.softmax_weight = self.create_parameter(
shape=[hidden_size, vocab_size])
self.softmax_bias = self.create_parameter(shape=[vocab_size],
is_bias=False)
def __init__(self,
vocab_size,
hidden_size=768,
hidden_dropout_prob=0.1,
max_position_embeddings=512,
max_2d_position_embeddings=1024,
layer_norm_eps=1e-12,
pad_token_id=0,
type_vocab_size=16):
super(LayoutLMEmbeddings, self).__init__()
self.word_embeddings = nn.Embedding(vocab_size,
hidden_size,
padding_idx=pad_token_id)
self.position_embeddings = nn.Embedding(max_position_embeddings,
hidden_size)
# gry add for layoutlm
self.x_position_embeddings = nn.Embedding(max_2d_position_embeddings,
hidden_size)
self.y_position_embeddings = nn.Embedding(max_2d_position_embeddings,
hidden_size)
self.h_position_embeddings = nn.Embedding(max_2d_position_embeddings,
hidden_size)
self.w_position_embeddings = nn.Embedding(max_2d_position_embeddings,
hidden_size)
# end of gry add for layoutlm
#self.token_type_embeddings = nn.Embedding(type_vocab_size, hidden_size, padding_idx=pad_token_id)
self.token_type_embeddings = nn.Embedding(type_vocab_size, hidden_size)
self.layer_norm = nn.LayerNorm(hidden_size, epsilon=layer_norm_eps)
self.dropout = nn.Dropout(hidden_dropout_prob)
self.register_buffer(
"position_ids",
paddle.arange(max_position_embeddings).expand((1, -1)))
def __init__(self, config):
super(LayoutXLMEmbeddings, self).__init__()
self.word_embeddings = nn.Embedding(
config["vocab_size"], config["hidden_size"], padding_idx=0)
self.position_embeddings = nn.Embedding(
config["max_position_embeddings"], config["hidden_size"])
# gry add for layoutxlm
self.x_position_embeddings = nn.Embedding(
config["max_2d_position_embeddings"], config["coordinate_size"])
self.y_position_embeddings = nn.Embedding(
config["max_2d_position_embeddings"], config["coordinate_size"])
self.h_position_embeddings = nn.Embedding(
config["max_2d_position_embeddings"], config["coordinate_size"])
self.w_position_embeddings = nn.Embedding(
config["max_2d_position_embeddings"], config["coordinate_size"])
# end of gry add for layoutxlm
self.token_type_embeddings = nn.Embedding(config["type_vocab_size"],
config["hidden_size"])
self.LayerNorm = nn.LayerNorm(
config["hidden_size"], epsilon=config["layer_norm_eps"])
self.dropout = nn.Dropout(config["hidden_dropout_prob"])
self.register_buffer(
"position_ids",
paddle.arange(config["max_position_embeddings"]).expand((1, -1)))
def __init__(self, cfg, name=None):
super(RelationalTransformerBlock, self).__init__()
d_model = cfg['hidden_size']
n_heads = cfg['num_attention_heads']
self.attn = RelationalAttentionLayer(cfg,
name=new_name(
name, 'multi_head_att'))
self.ln1 = _build_ln(d_model, name=new_name(name, 'post_att'))
self.ffn = PositionwiseFeedForwardLayer(cfg,
name=new_name(name, 'ffn'))
self.ln2 = _build_ln(d_model, name=new_name(name, 'post_ffn'))
prob = cfg.get('intermediate_dropout_prob', cfg['hidden_dropout_prob'])
self.dropout = nn.Dropout(p=prob)
# 假设 k/v 的
rel_hidden = d_model // n_heads
self.relation_k_emb = nn.Embedding(cfg['num_relations'], rel_hidden)
self.relation_v_emb = nn.Embedding(cfg['num_relations'], rel_hidden)
def __init__(self, config):
"""
Model initialization
Args:
config: model configuration, see config.yaml for more detail
"""
super(MemN2N, self).__init__()
self.nwords = config.nwords
self.init_hid = config.init_hid
self.init_std = config.init_std
self.nhop = config.nhop
self.edim = config.edim
self.mem_size = config.mem_size
self.lindim = config.lindim
self.max_grad_norm = config.max_grad_norm
self.batch_size = config.batch_size
self.checkpoint_dir = config.checkpoint_dir
normal_attr = paddle.framework.ParamAttr(
initializer=paddle.nn.initializer.Normal(std=self.init_std))
self.A = nn.Embedding(self.nwords, self.edim, weight_attr=normal_attr)
self.C = nn.Embedding(self.nwords, self.edim, weight_attr=normal_attr)
# Temporal Encoding
self.T_A = nn.Embedding(self.mem_size,
self.edim,
weight_attr=normal_attr)
self.T_C = nn.Embedding(self.mem_size,
self.edim,
weight_attr=normal_attr)
# Linear mapping for q
self.H = nn.Linear(self.edim,
self.edim,
weight_attr=normal_attr,
bias_attr=False)
# output mapping
self.W = nn.Linear(self.edim,
self.nwords,
weight_attr=normal_attr,
bias_attr=False)
def __init__(self, bond_names, embed_dim):
super(BondEmbedding, self).__init__()
self.bond_names = bond_names
self.embed_list = nn.LayerList()
for name in self.bond_names:
embed = nn.Embedding(CompoundKit.get_bond_feature_size(name) + 5,
embed_dim,
weight_attr=nn.initializer.XavierUniform())
self.embed_list.append(embed)
def __init__(self):
super(ModelLinear2, self).__init__()
with supernet(expand_ratio=None) as ofa_super:
models = []
models += [nn.Embedding(num_embeddings=64, embedding_dim=64)]
models += [nn.Linear(64, 128)]
models += [nn.LayerNorm(128)]
models += [nn.Linear(128, 256)]
models = ofa_super.convert(models)
self.models = paddle.nn.Sequential(*models)
def __init__(self, emb_dim, max_length, bos_idx=0):
super(PositionalEmbedding, self).__init__()
self.emb_dim = emb_dim
self.pos_encoder = nn.Embedding(
num_embeddings=max_length,
embedding_dim=self.emb_dim,
weight_attr=paddle.ParamAttr(
initializer=paddle.nn.initializer.Assign(
position_encoding_init(max_length, self.emb_dim))))
def __init__(self,
vocab_size,
hidden_size=768,
hidden_dropout_prob=0.1,
max_position_embeddings=512,
type_vocab_size=16,
initializer_range=0.02):
super(GPT2Embeddings, self).__init__()
self.word_embeddings = nn.Embedding(
vocab_size,
hidden_size,
weight_attr=paddle.ParamAttr(initializer=nn.initializer.Normal(
mean=0.0, std=initializer_range)))
self.position_embeddings = nn.Embedding(
max_position_embeddings,
hidden_size,
weight_attr=paddle.ParamAttr(initializer=nn.initializer.Normal(
mean=0.0, std=initializer_range)))
self.dropout = nn.Dropout(hidden_dropout_prob)
def __init__(self, vocab, hidden_size, latent_size, depthT, depthG):
super(JTNNVAE, self).__init__()
self.vocab = vocab
self.hidden_size = hidden_size
self.latent_size = latent_size = int(latent_size / 2)
self.jtnn = JTNNEncoder(hidden_size, depthT, nn.Embedding(vocab.size(), hidden_size))
self.decoder = JTNNDecoder(vocab, hidden_size, latent_size, nn.Embedding(vocab.size(), hidden_size))
self.jtmpn = JTMPN(hidden_size, depthG)
self.mpn = MPN(hidden_size, depthG)
self.A_assm = nn.Linear(latent_size, hidden_size, bias_attr=False)
self.assm_loss = nn.CrossEntropyLoss(reduction='sum')
self.T_mean = nn.Linear(hidden_size, latent_size)
self.T_var = nn.Linear(hidden_size, latent_size)
self.G_mean = nn.Linear(hidden_size, latent_size)
self.G_var = nn.Linear(hidden_size, latent_size)
def __init__(self, vocab_size, emb_dim, bos_id=0):
super(WordEmbedding, self).__init__()
self.emb_dim = emb_dim
self.word_embedding = nn.Embedding(
num_embeddings=vocab_size,
embedding_dim=emb_dim,
padding_idx=bos_id,
weight_attr=paddle.ParamAttr(
initializer=nn.initializer.Normal(0., emb_dim**-0.5)))
def __init__(self,
vocab_size,
hidden_size=768,
hidden_dropout_prob=0.1,
max_position_embeddings=512,
type_vocab_size=16,
use_relative_position=True):
super(NeZhaEmbeddings, self).__init__()
self.use_relative_position = use_relative_position
self.word_embeddings = nn.Embedding(vocab_size, hidden_size)
if not use_relative_position:
self.position_embeddings = nn.Embedding(max_position_embeddings,
hidden_size)
self.token_type_embeddings = nn.Embedding(type_vocab_size, hidden_size)
self.layer_norm = nn.LayerNorm(hidden_size)
self.dropout = nn.Dropout(hidden_dropout_prob)
def __init__(self, emb_dim, max_length):
super(PositionalEmbedding, self).__init__()
self.emb_dim = emb_dim
self.pos_encoder = nn.Embedding(num_embeddings=max_length,
embedding_dim=self.emb_dim)
self.pos_encoder.weight.set_value(
position_encoding_init(max_length,
self.emb_dim,
dtype=paddle.get_default_dtype()))
def __init__(self):
super(ModelOriginLinear, self).__init__()
models = []
models += [nn.Embedding(num_embeddings=64, embedding_dim=64)]
models += [nn.Linear(64, 128)]
models += [nn.LayerNorm(128)]
models += [nn.Linear(128, 256)]
models += [nn.Linear(256, 256)]
self.models = paddle.nn.Sequential(*models)
def __init__(self, hidden_dim, cut_dist, activation=F.relu):
super(SpatialInputLayer, self).__init__()
self.cut_dist = cut_dist
self.dist_embedding_layer = nn.Embedding(int(cut_dist) - 1,
hidden_dim,
sparse=True)
self.dist_input_layer = DenseLayer(hidden_dim,
hidden_dim,
activation,
bias=True)
def __init__(self, max_d=100, max_p=1000, n_filter=5, embed_dim=128, p_features=25, d_features=64, d_filter=4, p_filter=8, num_filters=32):
super(DeepdtaModel,self).__init__()
# Basic config
self.max_drug = max_d
self.max_protein = max_p
self.embed_dim = embed_dim
self.BN = nn.BatchNorm(1024)
self.dropout = nn.Dropout(p=0.1)
self.relu =nn.ReLU()
self.p_embedding = nn.Embedding(p_features, embed_dim)
self.d_embedding = nn.Embedding(d_features, embed_dim)
# Protein CNN
self.p_conv = nn.Sequential(
nn.Conv1D(in_channels=self.embed_dim, out_channels=num_filters, kernel_size=p_filter),
nn.ReLU(),
nn.Conv1D(in_channels=num_filters, out_channels=num_filters*2, kernel_size=p_filter),
nn.ReLU(),
nn.Conv1D(in_channels=num_filters*2, out_channels=num_filters*3, kernel_size=p_filter),
nn.ReLU(),
nn.MaxPool1D(kernel_size=1000-3*p_filter+3)
)
# Drug CNN
self.d_conv = nn.Sequential(
nn.Conv1D(in_channels=self.embed_dim, out_channels=num_filters, kernel_size=d_filter),
nn.ReLU(),
nn.Conv1D(in_channels=num_filters, out_channels=num_filters*2, kernel_size=d_filter),
nn.ReLU(),
nn.Conv1D(in_channels=num_filters*2, out_channels=num_filters*3, kernel_size=d_filter),
nn.ReLU(),
nn.MaxPool1D(kernel_size=100-3*d_filter+3)
)
#Decoder
self.fc1 = nn.Linear(192,1024)
self.fc2 = nn.Linear(1024,1024)
self.fc3 = nn.Linear(1024,512)
self.fc4 = nn.Linear(512,1)
def __init__(self,
vocab_size,
emb_dim=128,
hidden_size=256,
kernel_size=9,
n_layers=35,
padding_idx=0,
dropout_rate=0.1,
epsilon=1e-6):
super(ResnetEncoderModel, self).__init__()
self.hidden_size = hidden_size
self.n_layers = n_layers
self.token_embedding = nn.Embedding(vocab_size,
emb_dim,
padding_idx=padding_idx)
max_pos_len = 3000
self.pos_embedding = nn.Embedding(max_pos_len,
emb_dim,
padding_idx=padding_idx)
self.layer_norm = nn.BatchNorm1D(emb_dim, data_format="NLC")
self.dropout = nn.Dropout(dropout_rate)
self.padded_conv = nn.Sequential(
nn.Conv1D(in_channels=emb_dim, out_channels=hidden_size, kernel_size=kernel_size, padding="same", \
data_format="NLC", weight_attr=nn.initializer.KaimingNormal()),
nn.BatchNorm1D(hidden_size, data_format="NLC"),
nn.GELU(),
nn.Dropout(p=dropout_rate)
)
self.residual_block_1 = ResnetBasicBlock(inplanes=hidden_size,
planes=hidden_size,
kernel_size=kernel_size,
dropout_rate=dropout_rate)
self.residual_block_n = nn.Sequential()
for i in range(1, n_layers):
self.residual_block_n.add_sublayer("residual_block_%d" % i, \
ResnetBasicBlock(inplanes=hidden_size, planes=hidden_size, kernel_size=kernel_size, dilation=2, dropout_rate=dropout_rate))
self.apply(self.init_weights)
def __init__(self, vocab_size, type_size, max_position_seq_len, num_layers,
n_head, hidden_size, attn_dropout, act_dropout):
super(Plato2Encoder, self).__init__()
self.n_head = n_head
self.word_embedding_layer = nn.Embedding(vocab_size, hidden_size)
self.sent_embedding_layer = nn.Embedding(type_size, hidden_size)
self.pos_embedding_layer = nn.Embedding(max_position_seq_len,
hidden_size)
self.encoder_layers = []
for i in range(num_layers):
encoder_layer = Plato2EncoderLayer(n_head, hidden_size,
attn_dropout, act_dropout)
self.encoder_layers.append(encoder_layer)
self.add_sublayer('layers.' + str(i), encoder_layer)
self.post_encoder_layer_norm = nn.LayerNorm(hidden_size)
self.dropout_layer = nn.Dropout(act_dropout)
def __init__(self,
feat,
n_feats,
n_words,
pad_index=0,
feat_pad_index=0,
n_char_embed=50,
n_feat_embed=60,
n_lstm_char_embed=100,
n_embed=300,
embed_dropout=0.33,
n_lstm_hidden=300,
n_lstm_layers=3,
lstm_dropout=0.33):
super(LSTMEncoder, self).__init__()
self.pad_index = pad_index
if feat == "char":
self.feat_embed = CharLSTMEncoder(
n_chars=n_feats,
n_embed=n_char_embed,
n_out=n_lstm_char_embed,
pad_index=feat_pad_index,
)
feat_embed_size = n_lstm_char_embed
else:
self.feat_embed = nn.Embedding(num_embeddings=n_feats,
embedding_dim=n_feat_embed)
feat_embed_size = n_feat_embed
self.word_embed = nn.Embedding(num_embeddings=n_words,
embedding_dim=n_embed)
self.embed_dropout = IndependentDropout(p=embed_dropout)
self.lstm = nn.LSTM(input_size=n_embed + feat_embed_size,
hidden_size=n_lstm_hidden,
num_layers=n_lstm_layers,
dropout=lstm_dropout,
direction="bidirectional")
self.lstm_dropout = SharedDropout(p=lstm_dropout)
self.mlp_input_size = n_lstm_hidden * 2
def __init__(self, num_nodes, embed_size=16, neg_num=5, sparse=False):
super(SkipGramModel, self).__init__()
self.num_nodes = num_nodes
self.neg_num = neg_num
embed_init = nn.initializer.Uniform(
low=-1. / math.sqrt(embed_size), high=1. / math.sqrt(embed_size))
emb_attr = paddle.ParamAttr(name="node_embedding")
self.emb = nn.Embedding(
num_nodes, embed_size, sparse=sparse, weight_attr=emb_attr)
self.loss = paddle.nn.BCEWithLogitsLoss(reduction="none")
def __init__(self,
hidden_size,
vocab_size,
class_num=2,
num_steps=128,
num_layers=1,
init_scale=0.1,
dropout=None):
# 参数含义如下：
# 1.hidden_size，表示embedding-size，hidden和cell向量的维度
# 2.vocab_size，模型可以考虑的词表大小
# 3.class_num，情感类型个数，可以是2分类，也可以是多分类
# 4.num_steps，表示这个情感分析模型最大可以考虑的句子长度
# 5.num_layers，表示网络的层数
# 6.init_scale，表示网络内部的参数的初始化范围
# 长短时记忆网络内部用了很多Tanh，Sigmoid等激活函数，这些函数对数值精度非常敏感，
# 因此我们一般只使用比较小的初始化范围，以保证效果
super(SentimentClassifier, self).__init__()
self.hidden_size = hidden_size
self.vocab_size = vocab_size
self.class_num = class_num
self.init_scale = init_scale
self.num_layers = num_layers
self.num_steps = num_steps
self.dropout = dropout
# 声明一个embedding层，用来把句子中的每个词转换为向量
self.embedding = nn.Embedding(
num_embeddings=vocab_size,
embedding_dim=hidden_size,
sparse=False,
weight_attr=paddle.ParamAttr(
initializer=paddle.nn.initializer.Uniform(low=-init_scale,
high=init_scale)))
# 声明一个LSTM模型，用来把每个句子抽象成向量
self.simple_lstm_rnn = nn.LSTM(input_size=hidden_size,
hidden_size=hidden_size,
num_layers=num_layers)
# 在得到一个句子的向量表示后，需要根据这个向量表示对这个句子进行分类
# 一般来说，可以把这个句子的向量表示乘以一个大小为[self.hidden_size, self.class_num]的W参数，
# 并加上一个大小为[self.class_num]的b参数，从而达到把句子向量映射到分类结果的目的
# 我们需要声明最终在使用句子向量映射到具体情感类别过程中所需要使用的参数
# 这个参数的大小一般是[self.hidden_size, self.class_num]
self.cls_fc = nn.Linear(in_features=self.hidden_size,
out_features=self.class_num,
weight_attr=None,
bias_attr=None)
self.dropout_layer = nn.Dropout(p=self.dropout,
mode='upscale_in_train')
def __init__(self,
word_emb_dim,
hidden_size,
vocab_size,
num_labels,
emb_lr=2.0,
crf_lr=0.2,
with_start_stop_tag=True):
super(BiGruCrf, self).__init__()
self.word_emb_dim = word_emb_dim
self.vocab_size = vocab_size
self.num_labels = num_labels
self.hidden_size = hidden_size
self.emb_lr = emb_lr
self.crf_lr = crf_lr
self.init_bound = 0.1
self.word_embedding = nn.Embedding(
num_embeddings=self.vocab_size,
embedding_dim=self.word_emb_dim,
weight_attr=paddle.ParamAttr(learning_rate=self.emb_lr,
initializer=nn.initializer.Uniform(
low=-self.init_bound,
high=self.init_bound)))
self.gru = nn.GRU(
input_size=self.word_emb_dim,
hidden_size=self.hidden_size,
num_layers=2,
direction='bidirectional',
weight_ih_attr=paddle.ParamAttr(
initializer=nn.initializer.Uniform(low=-self.init_bound,
high=self.init_bound),
regularizer=paddle.regularizer.L2Decay(coeff=1e-4)),
weight_hh_attr=paddle.ParamAttr(
initializer=nn.initializer.Uniform(low=-self.init_bound,
high=self.init_bound),
regularizer=paddle.regularizer.L2Decay(coeff=1e-4)))
self.fc = nn.Linear(
in_features=self.hidden_size * 2,
out_features=self.num_labels + 2 \
if with_start_stop_tag else self.num_labels,
weight_attr=paddle.ParamAttr(
initializer=nn.initializer.Uniform(
low=-self.init_bound, high=self.init_bound),
regularizer=paddle.regularizer.L2Decay(coeff=1e-4)))
self.crf = LinearChainCrf(self.num_labels, self.crf_lr,
with_start_stop_tag)
self.crf_loss = LinearChainCrfLoss(self.crf)
self.viterbi_decoder = ViterbiDecoder(self.crf.transitions,
with_start_stop_tag)
def __init__(self,
vocab_size,
emb_size=128,
hidden_dropout_prob=0.1,
max_position_embeddings=512,
type_vocab_size=2,
pad_token_id=0,
rel_pos_size=None,
num_attention_heads=None):
super(ErnieGramEmbeddings, self).__init__()
self.word_embeddings = nn.Embedding(
vocab_size, emb_size, padding_idx=pad_token_id)
self.position_embeddings = nn.Embedding(max_position_embeddings,
emb_size)
self.token_type_embeddings = nn.Embedding(type_vocab_size, emb_size)
if rel_pos_size and num_attention_heads:
self.rel_pos_embeddings = nn.Embedding(rel_pos_size,
num_attention_heads)
self.layer_norm = nn.LayerNorm(emb_size)
self.dropout = nn.Dropout(hidden_dropout_prob)
def __init_weight(self):
self.num_users = self.dataset.n_users
self.num_items = self.dataset.m_items
self.latent_dim = self.config['latent_dim_rec']
self.n_layers = self.config['lightGCN_n_layers']
self.gcn = NGCFConv(self.latent_dim, self.latent_dim, self.n_layers)
weight_attr1 = paddle.framework.ParamAttr(
initializer=paddle.nn.initializer.XavierUniform())
weight_attr2 = paddle.framework.ParamAttr(
initializer=paddle.nn.initializer.XavierUniform())
self.embedding_user = nn.Embedding(num_embeddings=self.num_users,
embedding_dim=self.latent_dim,
weight_attr=weight_attr1)
self.embedding_item = nn.Embedding(num_embeddings=self.num_items,
embedding_dim=self.latent_dim,
weight_attr=weight_attr2)
self.f = nn.Sigmoid()
num_nodes = self.dataset.n_users + self.dataset.m_items
edges = paddle.to_tensor(self.dataset.trainEdge, dtype='int64')
self.Graph = pgl.Graph(num_nodes=num_nodes, edges=edges)
print(f"lgn is already to go(dropout:{self.config['dropout']})")
