def __init__(self,
vocab_size,
embedding_dim=256,
hidden_size=512,
num_layers=1,
kernel_size=3,
dropout=0.25,
max_length=100):
super().__init__()
assert kernel_size % 2 == 1, f"Kernel size must be odd, got {kernel_size}"
self.embedding_dim = embedding_dim
self.vocab_size = vocab_size
self.max_length = max_length
self.num_layers = num_layers
self.scale = torch.sqrt(torch.FloatTensor([0.5]))
self.token_embedding = Embedding(vocab_size, embedding_dim)
self.position_embedding = Embedding(max_length, embedding_dim)
self.embed2hidden = Linear(embedding_dim, hidden_size)
self.hidden2embed = Linear(hidden_size, embedding_dim)
convs = [
EncoderConv(hidden_size, 2 * hidden_size, kernel_size,
(kernel_size - 1) // 2, dropout)
for _ in range(num_layers)
]
self.convs = Sequential(*convs)
self.dropout = Dropout(dropout)
def __init__(self,
hidden_dim=512,
embedding_dim=256,
vocab_size=10000,
num_layers=10,
kernel_size=3,
dropout=0.25,
PAD_token=0,
max_len=50):
super().__init__()
self.kernel_size = kernel_size
self.PAD_token = PAD_token
self.num_layers = num_layers
self.vocab_size = vocab_size
self.max_len = max_len
self.token_embedding = Embedding(vocab_size, embedding_dim)
self.position_embedding = Embedding(max_len, embedding_dim)
self.embedd2hidden = Linear(embedding_dim, hidden_dim)
self.hidden2embedd = Linear(hidden_dim, embedding_dim)
self.attention_layer = Attention(embedding_dim, hidden_dim)
self.decoder_conv = DecoderConv(kernel_size, dropout, PAD_token)
self.convs = ModuleList([
Conv1d(in_channels=hidden_dim,
out_channels=2 * hidden_dim,
kernel_size=kernel_size) for _ in range(num_layers)
])
self.out = Linear(embedding_dim, vocab_size)
self.dropout = Dropout(dropout)
def __init__(self, config, num_classes, vocab_size):
super().__init__(config=config)
w_embed_dim = config.word_embedding_dim
filter_sizes = config.window_sizes
filter_num = config.filter_num
max_filter_size = max(config.window_sizes)
self.fix_embedding = config.fix_embedding
position_embed_dim = config.position_embedding_dim
self.word_embed = Embedding(vocab_size, w_embed_dim)
self.position_embed = Embedding(2 * max_filter_size + 1,
position_embed_dim)
self.full_embed_dim = w_embed_dim + position_embed_dim
self.convs1 = ModuleList([
Conv2d(1, filter_num, (fs, self.full_embed_dim))
for fs in filter_sizes
])
self.dropout = nn.Dropout(config.dropout)
self.linear = nn.Linear(len(filter_sizes) * filter_num, num_classes)
# Add batch size dimension.
self.positions = torch.IntTensor(range(2 * max_filter_size + 1))
def __init__(self, input_embeddings: str, input_vocabulary_size: int,
input_embeddings_size: int, clear_text: bool,
tokenize_model: str):
super().__init__()
if clear_text:
assert tokenize_model is not None
from pytorch_pretrained_bert import BertTokenizer
self.bert_tokenizer = BertTokenizer.from_pretrained(
tokenize_model, do_lower_case=False)
input_vocabulary_size = len(self.bert_tokenizer.vocab)
self.lut_embeddings = Embedding(
num_embeddings=input_vocabulary_size,
embedding_dim=input_embeddings_size,
padding_idx=pad_token_index)
self._is_fixed = False
else:
self.bert_tokenizer = None
if input_embeddings is not None:
self.lut_embeddings = Embedding.from_pretrained(
embeddings=input_embeddings, freeze=True)
self._is_fixed = True
else:
self.lut_embeddings = Embedding(
num_embeddings=input_vocabulary_size,
embedding_dim=input_embeddings_size,
padding_idx=pad_token_index)
self._is_fixed = False
self._output_dim = input_embeddings_size
def __init__(self, vocab_size, embed_dim):
super(BayesianSG, self).__init__()
# Sizes
self.vocab_size = vocab_size
self.embed_dim = embed_dim
# Priors
self.prior_locs = Embedding(vocab_size, embed_dim)
self.prior_scales = Embedding(vocab_size, embed_dim)
# Inference
self.embeddings = Embedding(vocab_size, embed_dim)
self.encoder = Linear(2 * embed_dim, 2 * embed_dim)
self.affine_loc = Linear(2 * embed_dim, embed_dim)
self.affine_scale = Linear(2 * embed_dim, embed_dim)
self.std_normal = MultivariateNormal(torch.zeros(embed_dim), torch.eye(embed_dim))
# Generation
self.affine_vocab = Linear(embed_dim, vocab_size)
# Functions
self.softmax = Softmax(dim=1)
self.softplus = Softplus()
self.relu = ReLU()
def __init__(self):
super(Net, self).__init__()
self.embedder1 = Embedding(num_embeddings=316, embedding_dim=16)
self.embedder2 = Embedding(num_embeddings=289, embedding_dim=8)
self.cnn1_1 = Conv2d(in_channels=1,
out_channels=128,
kernel_size=(3, 16),
stride=1,
padding=(1, 0))
self.cnn1_2 = Conv2d(in_channels=1,
out_channels=128,
kernel_size=(4, 16),
stride=1)
self.cnn1_3 = Conv2d(in_channels=1,
out_channels=128,
kernel_size=(5, 16),
stride=1,
padding=(2, 0))
self.cnn2 = Conv2d(in_channels=1,
out_channels=128,
kernel_size=(4, 8),
stride=4)
self.lstm = LSTM(input_size=512,
hidden_size=100,
bidirectional=True,
batch_first=True)
self.lin1 = Linear(200, 64)
self.lin2 = Linear(64, 32)
self.lin3 = Linear(32, 1)
def __init__(self, num_classes: int, dim: int, dropout_rate: float = 0.1):
super(ComplexEmbedding, self).__init__()
self.amplitudes = Embedding(num_embeddings=num_classes,
embedding_dim=dim)
self.frequencies = Embedding(num_embeddings=num_classes,
embedding_dim=dim)
self.dropout = Dropout(dropout_rate)
def __init__(self):
super(Net_without_BiLSTM, self).__init__()
self.embedder1 = Embedding(num_embeddings=316, embedding_dim=16)
self.embedder2 = Embedding(num_embeddings=289, embedding_dim=8)
self.cnn1_1 = Conv2d(in_channels=1,
out_channels=128,
kernel_size=(3, 16),
stride=1,
padding=(1, 0))
self.cnn1_2 = Conv2d(in_channels=1,
out_channels=128,
kernel_size=(4, 16),
stride=1)
self.cnn1_3 = Conv2d(in_channels=1,
out_channels=128,
kernel_size=(5, 16),
stride=1,
padding=(2, 0))
self.cnn2 = Conv2d(in_channels=1,
out_channels=128,
kernel_size=(4, 8),
stride=4)
self.lin1 = Linear(512000, 32)
self.lin2 = Linear(32, 16)
self.lin3 = Linear(16, 1)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.save_hyperparameters()
kwargs = self.sanetize_kwargs(kwargs)
self.node_emb = Embedding(kwargs["node_vocab"], kwargs["node_dim"])
self.edge_emb = Embedding(kwargs["edge_vocab"], kwargs["edge_dim"])
self.convs = ModuleList()
self.batch_norms = ModuleList()
for _ in range(kwargs["num_layers"]):
conv = PNAConv(
in_channels=kwargs["node_dim"],
out_channels=kwargs["node_dim"],
aggregators=kwargs["aggregators"],
scalers=kwargs["scalers"],
deg=torch.tensor(kwargs["deg"]),
edge_dim=kwargs["edge_dim"],
towers=kwargs["towers"],
pre_layers=kwargs["pre_layers"],
post_layers=kwargs["post_layers"],
divide_input=kwargs["divide_input"],
)
self.convs.append(conv)
self.batch_norms.append(BatchNorm(kwargs["node_dim"]))
self.mlp = Sequential(
Linear(kwargs["node_dim"], kwargs["edge_dim"]),
ReLU(),
Linear(kwargs["edge_dim"], kwargs["hidden_channels"]),
ReLU(),
Linear(kwargs["hidden_channels"], kwargs["num_classes"]),
)
def __init__(self,
num_embeddings=12,
embedding_dim=64,
num_layers=1,
num_heads=1,
num_classes=10,
dropout=0.2,
edge_encoding=EDGE_ENCODING_TYPE.RELATIVE_POSITION,
log=False,
name=None,
use_cuda=False):
super(TransformerModel, self).__init__(name=name, log=log, use_cuda=use_cuda)
if edge_encoding == EDGE_ENCODING_TYPE.RELATIVE_POSITION or edge_encoding==EDGE_ENCODING_TYPE.GRAPH_RELATIVE:
self.edges_embedding = ModuleList([Embedding(5, embedding_dim), Embedding(5, embedding_dim)])
else:
self.edges_embedding = None
self.embeddings = Embedding(num_embeddings=num_embeddings, embedding_dim=embedding_dim, padding_idx=0)
self.softmax = Softmax(dim=-1)
self.l_out = Linear(in_features=embedding_dim, out_features=num_classes)
self.attention_layers = ModuleList(
[TransformerLayer(in_features=embedding_dim, hidden_dim=embedding_dim, num_heads=num_heads, dropout=dropout, edge_encoding=edge_encoding) for _ in range(num_layers)]
)
def __init__(self, cfg):
super(GRU_model, self).__init__()
self.input_dim = cfg.model.input_size
self.hidden_dim = cfg.model.hidden_size
self.output_dim = cfg.model.output_size
self.n_layers = cfg.model.n_layers
self.cfg = cfg
self.accuracy = pl.metrics.Accuracy()
self.hidden_state = 0
self.embed_seq = Embedding(26, cfg.model.embed_dim_seq)
self.embed_creature = Embedding(10, cfg.model.embed_dim_creature)
self.gru = nn.GRU(input_size=(cfg.model.embed_dim_seq+cfg.model.embed_dim_creature),hidden_size=cfg.model.hidden_size, num_layers=self.n_layers, batch_first=True, dropout=cfg.model.dropout, bidirectional=cfg.model.bi)
self.num_directions = (2 if cfg.model.bi else 1)
self.fc_1 = nn.Sequential(
nn.Linear(in_features=cfg.model.hidden_size*self.num_directions, out_features=20)
#nn.BatchNorm1d(20),
#nn.ReLU(),
#nn.Dropout(p=cfg.model.dropout),
#nn.Linear(in_features=20, out_features=20)
)
self.correct = []
self.count = []
self.correct_test = []
self.count_test = []
def __init__(self):
super().__init__()
self.node_emb = Embedding(21, 75)
self.edge_emb = Embedding(4, 50)
aggregators = ['mean', 'min', 'max', 'std']
scalers = ['identity', 'amplification', 'attenuation']
self.convs = ModuleList()
self.batch_norms = ModuleList()
for _ in range(4):
conv = PNAConv(in_channels=75,
out_channels=75,
aggregators=aggregators,
scalers=scalers,
deg=deg,
edge_dim=50,
towers=5,
pre_layers=1,
post_layers=1,
divide_input=False)
self.convs.append(conv)
self.batch_norms.append(BatchNorm(75))
self.mlp = Sequential(Linear(75, 50), ReLU(), Linear(50, 25), ReLU(),
Linear(25, 1))
def __init__(self, config):
super().__init__()
self.word_embeddings = Embedding(config.vocab_size, config.embedding_size, padding_idx=0)
self.position_embeddings = Embedding(config.max_position_embeddings, config.embedding_size)
self.token_type_embeddings = Embedding(config.type_vocab_size, config.embedding_size)
self.LayerNorm = LayerNorm(config.embedding_size, eps=1e-12)
self.dropout = Dropout(config.dropout_prob)
def __init__(self, categories_nums, num_layers=2, hidden=64, features_num=16, num_class=2, node_num=10000):
super(GCN, self).__init__()
embed_size = 8
self.dropout_p = 0.1
id_embed_size = 8
self.id_embedding = Embedding(categories_nums[0], id_embed_size)
self.lin0_id_emb = Linear(id_embed_size, id_embed_size)
self.embeddings = torch.nn.ModuleList()
for max_nums in categories_nums[1:]:
self.embeddings.append(Embedding(max_nums, embed_size))
n = max(0,len(categories_nums)-1)
if n>0:
self.lin0_emb = Linear(embed_size*n, embed_size*n)
if features_num>0:
self.lin0 = Linear(features_num, hidden)
self.ln0 = torch.nn.LayerNorm(id_embed_size+embed_size*n+hidden)
self.conv1 = GCNConv(id_embed_size+embed_size*n+hidden, hidden)
else:
self.ln0 = torch.nn.LayerNorm(id_embed_size+embed_size*n)
self.conv1 = GCNConv(id_embed_size+embed_size*n, hidden)
self.ln1 = torch.nn.LayerNorm(hidden)
self.conv2 = GCNConv(hidden, hidden)
self.ln2 = torch.nn.LayerNorm(hidden)
self.lin1 = Linear(hidden, num_class)
def __init__(self,
emb_dim,
hidden_dim,
rank_dim,
n_layers,
dropout):
super(Net, self).__init__()
self.node_emb = Embedding(21, emb_dim)
self.edge_emb = Embedding(4, emb_dim)
self.n_layers = n_layers
self.dropout = dropout
aggregators = ['mean', 'min', 'max', 'std']
scalers = ['identity', 'amplification', 'attenuation']
self.convs = ModuleList()
self.pool = graph_cp_pooling(hidden_dim)
self.batch_norms = ModuleList()
for _ in range(n_layers):
#conv = PNAConv(in_channels=75, out_channels=75,
# aggregators=aggregators, scalers=scalers, deg=deg,
# edge_dim=50, towers=5, pre_layers=1, post_layers=1,
# divide_input=False)
conv = GCNConv(emb_dim=emb_dim, hidden_dim=hidden_dim, rank_dim=rank_dim)
self.convs.append(conv)
self.batch_norms.append(BatchNorm(hidden_dim))
self.mlp = Sequential(Linear(hidden_dim, 50), ReLU(), Linear(50, 25), ReLU(),
Linear(25, 1))
def __init__(
self,
src_vocab_size: int,
tgt_vocab_size: int,
tgt_sequence_size: int,
word_emb_dim: int = 32,
nhead: int = 1,
num_encoder_layers: int = 2,
num_decoder_layers: int = 2,
dim_feedforward: int = 128,
dropout_prob: float = 0.1,
) -> None:
super(Transformer, self).__init__()
# para
self.word_emb_dim = word_emb_dim
self.nhead = nhead
self.tgt_sequence_size = tgt_sequence_size
# layers
self.src_word_emb = Embedding(src_vocab_size, word_emb_dim)
self.tgt_word_emb = Embedding(tgt_vocab_size, word_emb_dim)
self.position_encoding = PositionalEncoding(word_emb_dim, dropout_prob)
encoder_layer = TransformerEncoderLayer(word_emb_dim, nhead, dim_feedforward, dropout_prob)
encoder_norm = LayerNorm(word_emb_dim)
self.encoder = TransformerEncoder(encoder_layer, num_encoder_layers, encoder_norm)
decoder_layer = TransformerDecoderLayer(word_emb_dim, nhead, dim_feedforward, dropout_prob)
decoder_norm = LayerNorm(word_emb_dim)
self.decoder = TransformerDecoder(decoder_layer, num_decoder_layers, decoder_norm)
self.generator = Linear(word_emb_dim * tgt_sequence_size, tgt_vocab_size)
def __init__(self, config, dissimilarity):
super().__init__()
self.ent_emb_dim = config.entities_embedding_dimension
self.rel_emb_dim = config.relations_embedding_dimension
self.number_entities = config.number_entities
self.number_relations = config.number_relations
self.norm_type = config.norm_type
self.dissimilarity = dissimilarity
# initialize embedding objects
self.entity_embeddings = Embedding(self.number_entities,
self.ent_emb_dim)
self.relation_embeddings = Embedding(self.number_relations,
self.rel_emb_dim)
# fill the embedding weights with Xavier initialized values
self.entity_embeddings.weight = Parameter(
xavier_uniform_(
empty(size=(self.number_entities, self.ent_emb_dim))))
self.relation_embeddings.weight = Parameter(
xavier_uniform_(
empty(size=(self.number_relations, self.rel_emb_dim))))
# normalize the embeddings
self.entity_embeddings.weight.data = normalize(
self.entity_embeddings.weight.data, p=self.norm_type, dim=1)
self.relation_embeddings.weight.data = normalize(
self.relation_embeddings.weight.data, p=self.norm_type, dim=1)
def __init__(self, config):
super(BiDAF, self).__init__()
self.config = config
self.logits = None
self.yp = None
self.dc, self.dw, self.dco = config.char_emb_size, config.word_emb_size, \
config.char_out_size
self.N, self.M, self.JX, self.JQ, self.VW, self.VC, self.d, self.W = \
config.batch_size, config.max_num_sents, config.max_sent_size, \
config.max_ques_size, config.word_vocab_size, config.char_vocab_size, \
config.hidden_size, config.max_word_size
self.word_embed = Embedding(config.word_vocab_size, \
config.glove_vec_size)
self.char_embed = Embedding(config.char_vocab_size, \
config.char_emb_size)
# char-level convs
filter_sizes = list(map(int, config.out_channel_dims.split(',')))
heights = list(map(int, config.filter_heights.split(',')))
self.filter_sizes = filter_sizes
self.heights = heights
self.multiconv_1d = L.MultiConv1D(config.is_train, config.keep_prob)
self.multiconv_1d_qq = L.MultiConv1D(config.is_train, config.keep_prob)
if config.use_char_emb:
highway_outsize = self.dco + self.dw
else:
highway_outsize = self.dw
self.highway = L.HighwayNet(config.highway_num_layers, highway_outsize)
self.prepro = L.BiEncoder(config,
highway_outsize,
hidden_size=config.hidden_size)
self.prepro_x = L.BiEncoder(config,
highway_outsize,
hidden_size=config.hidden_size)
self.attention_layer = L.AttentionLayer(config, self.JX, self.M,
self.JQ,
2 * config.hidden_size)
# Because p0 = torch.cat([h, u_a, torch.mul(h, u_a), torch.mul(h, h_a)], 3) and the last dim of
# these matrices are d.
self.g0_biencoder = L.BiEncoder(config,
8 * config.hidden_size,
hidden_size=config.hidden_size)
self.g1_biencoder = L.BiEncoder(config,
2 * config.hidden_size,
hidden_size=config.hidden_size)
# p0: 8 * d. g1: 2 * d
self.g1_logits = L.GetLogits(config,
10 * config.hidden_size,
input_keep_prob=config.input_keep_prob,
function=config.answer_func)
# p0: [60, 1, 161, 800], g1: [60, 1, 161, 200], a1: [60, 1, 161, 200], g1 * a1: [60, 1, 161, 200]
self.g2_biencoder = L.BiEncoder(config,
14 * config.hidden_size,
hidden_size=config.hidden_size)
# p0: 8 * d. g2: 2 * d
self.g2_logits = L.GetLogits(config,
10 * config.hidden_size,
input_keep_prob=config.input_keep_prob,
function=config.answer_func)
def __init__(self,
categories_nums,
features_num=16,
num_class=2,
sparse=False):
super(GAT_noLN, self).__init__()
if sparse:
print('---sparse---')
hidden = 16
heads = 8
else:
print('---no sparse---')
hidden = 8
heads = 4
dropout = 0.1
self.dropout_p = dropout
embed_size = 8
id_embed_size = 8
self.id_embedding = Embedding(categories_nums[0], id_embed_size)
self.lin0_id_emb = Linear(id_embed_size, id_embed_size)
self.embeddings = torch.nn.ModuleList()
for max_nums in categories_nums[1:]:
self.embeddings.append(Embedding(max_nums, embed_size))
n = max(0, len(categories_nums) - 1)
if n > 0:
self.lin0_emb = Linear(embed_size * n, embed_size * n)
if features_num > 0:
self.lin0 = Linear(features_num, hidden)
self.ln0 = torch.nn.LayerNorm(id_embed_size + embed_size * n +
hidden)
self.conv1 = GATConv(id_embed_size + embed_size * n + hidden,
hidden,
heads=heads,
concat=True,
dropout=dropout)
else:
self.ln0 = torch.nn.LayerNorm(id_embed_size + embed_size * n)
self.conv1 = GATConv(id_embed_size + embed_size * n,
hidden,
heads=heads,
concat=True,
dropout=dropout)
self.ln1 = torch.nn.LayerNorm(hidden * heads)
self.conv2 = GATConv(hidden * heads,
hidden,
heads=heads,
concat=True,
dropout=dropout)
self.ln2 = torch.nn.LayerNorm(hidden * heads)
self.lin1 = Linear(hidden * heads, num_class)
def __init__(self,
args,
train_dataset,
dataset,
hidden_channels,
num_layers,
max_z,
GNN=GCNConv,
k=0.6,
use_feature=False,
dataset_name=None,
node_embedding=None):
super(WLGNN_model, self).__init__()
self.use_feature = use_feature
self.node_embedding = node_embedding
self.args = args
if k <= 1: # Transform percentile to number.
if args.dynamic_train:
sampled_train = train_dataset[:1000]
else:
sampled_train = train_dataset
num_nodes = sorted([g.num_nodes for g in sampled_train])
k = num_nodes[int(math.ceil(k * len(num_nodes))) - 1]
k = max(10, k)
self.k = int(k)
self.max_z = max_z
if "social" in dataset_name: self.w_z = 50000
else: self.w_z = max_z
self.w_embedding = Embedding(self.w_z, hidden_channels)
self.z1_embedding = Embedding(self.max_z, hidden_channels)
self.z2_embedding = Embedding(self.max_z, hidden_channels)
self.convs = ModuleList()
initial_channels = hidden_channels * 3
if self.use_feature:
initial_channels += dataset.num_features * 2
if self.node_embedding is not None:
initial_channels += node_embedding.embedding_dim
self.convs.append(GNN(initial_channels, hidden_channels))
for i in range(0, num_layers - 1):
self.convs.append(GNN(hidden_channels, hidden_channels))
self.convs.append(GNN(hidden_channels, 1))
conv1d_channels = [16, 32]
total_latent_dim = hidden_channels * num_layers + 1
conv1d_kws = [total_latent_dim, 5]
self.conv1 = Conv1d(1, conv1d_channels[0], conv1d_kws[0],
conv1d_kws[0])
self.maxpool1d = MaxPool1d(2, 2)
self.conv2 = Conv1d(conv1d_channels[0], conv1d_channels[1],
conv1d_kws[1], 1)
dense_dim = int((self.k - 2) / 2 + 1)
dense_dim = (dense_dim - conv1d_kws[1] + 1) * conv1d_channels[1]
self.lin1 = Linear(dense_dim, 128)
self.lin2 = Linear(128, 1)
def __init__(self, layers, **kwargs):
super(MATGCN, self).__init__()
self.n_nodes, out_timesteps = kwargs['n_nodes'], kwargs[
'out_timesteps']
self.d_embed = Embedding(7, len(layers) * self.n_nodes * out_timesteps)
self.h_embed = Embedding(24,
len(layers) * self.n_nodes * out_timesteps)
self.layers = ModuleList(
[MATGCNLayer(**layer, **kwargs) for layer in layers])
def __init__(self, vocab_size, embed_dim):
super(Skipgram, self).__init__()
self.vocab_size = vocab_size
self.embed_dim = embed_dim
self.input_embeds = Embedding(vocab_size, embed_dim, sparse=False)
self.output_embeds = Embedding(vocab_size, embed_dim, sparse=False)
self.init_emb()
def __init__(self, hparams):
super().__init__()
self.hparams = hparams
self.encoder_embeddings = Embedding(hparams.encoder_num_embeddings, hparams.embedding_dim)
self.encoder_cell = TPN2FEncoderCell(hparams)
self.mapper = ReasoningModule(hparams)
self.decoder_relation_embeddings = Embedding(hparams.relation_number, hparams.embedding_dim)
self.decoder_argument_embeddings = Embedding(hparams.argument_number, hparams.embedding_dim)
self.decoder_cell = TPN2FDecoderCell(hparams)
def __init__(self, params: EncoderDecoderParams):
super(EncoderDecoder, self).__init__()
# word embed layer
self._embeddings_words = self._load_pre_trained(params.EMBED_WORDS_pre_trained, params.GPU) if \
params.EMBED_WORDS_use_pre_trained else Embedding(params.EMBED_WORDS_vocab_dim, params.EMBED_WORDS_dim)
self._embeddings_pos = Embedding(params.EMBED_POS_vocab_dim,
params.EMBED_POS_dim)
self._embeddings_ner = Embedding(params.EMBED_NER_vocab_dim,
params.EMBED_NER_dim)
self._dropout = Dropout(p=params.LSTM_dropout_0)
# ========================================= ENCODER =========================================
self._encoder_layer_0 = LSTM(
1 + params.EMBED_WORDS_dim + params.EMBED_CHR_dim +
params.EMBED_POS_dim + params.EMBED_NER_dim,
params.LSTM_hidden_dim,
num_layers=1,
batch_first=True,
bidirectional=True)
self._encoder_layer_1 = LSTM(1 + params.EMBED_WORDS_dim +
params.EMBED_CHR_dim +
(2 * params.LSTM_hidden_dim),
params.LSTM_hidden_dim,
num_layers=1,
batch_first=True,
bidirectional=True)
self._encoder_layer_2 = LSTM(1 + params.EMBED_WORDS_dim +
params.EMBED_CHR_dim +
(2 * params.LSTM_hidden_dim),
params.LSTM_hidden_dim,
num_layers=1,
batch_first=True,
bidirectional=True)
# ========================================= DECODER =========================================
self._decoder_layer_0 = LSTM(params.EMBED_WORDS_dim +
params.EMBED_CHR_dim +
(8 * params.LSTM_hidden_dim),
params.LSTM_hidden_dim,
num_layers=1,
batch_first=True,
bidirectional=True)
self._decoder_layer_1 = LSTM(params.EMBED_WORDS_dim +
params.EMBED_CHR_dim +
(2 * params.LSTM_hidden_dim),
params.LSTM_hidden_dim,
num_layers=1,
batch_first=True,
bidirectional=True)
self._decoder_layer_2 = LSTM(params.EMBED_WORDS_dim +
params.EMBED_CHR_dim +
(2 * params.LSTM_hidden_dim),
params.LSTM_hidden_dim,
num_layers=1,
batch_first=True,
bidirectional=True)
def __init__(self,
hparams: "SentenceEmbeddings.Options",
statistics,
plugins=None):
super().__init__()
self.hparams = hparams
self.mode = hparams.mode
self.plugins = ModuleDict(plugins) if plugins is not None else {}
# embedding
input_dims = {}
if hparams.dim_word != 0:
self.word_embeddings = Embedding(len(statistics.words),
hparams.dim_word,
padding_idx=0)
self.word_dropout = FeatureDropout2(hparams.word_dropout)
input_dims["word"] = hparams.dim_word
if hparams.dim_postag != 0:
self.pos_embeddings = Embedding(len(statistics.postags),
hparams.dim_postag,
padding_idx=0)
self.pos_dropout = FeatureDropout2(hparams.postag_dropout)
input_dims["postag"] = hparams.dim_postag
if hparams.dim_char > 0:
self.bilm = None
self.character_lookup = Embedding(len(statistics.characters),
hparams.dim_char_input)
self.char_embeded = CharacterEmbedding.get(
hparams.character_embedding,
dim_char_input=hparams.dim_char_input,
input_size=hparams.dim_char)
if not hparams.replace_unk_with_chars:
input_dims["char"] = hparams.dim_char
else:
assert hparams.dim_word == hparams.dim_char
else:
self.character_lookup = None
for name, plugin in self.plugins.items():
input_dims[name] = plugin.output_dim
if hparams.mode == "concat":
self.output_dim = sum(input_dims.values())
else:
assert hparams.mode == "add"
uniq_input_dims = list(set(input_dims.values()))
if len(uniq_input_dims) != 1:
raise ValueError(f"Different input dims: {input_dims}")
print(input_dims)
self.output_dim = uniq_input_dims[0]
self.input_layer_norm = LayerNorm(self.output_dim, eps=1e-6) \
if hparams.input_layer_norm else None
def __init__(
self,
weight_file: str,
uid_column: CategoricalColumnWithIdentity,
iid_column: CategoricalColumnWithIdentity,
pos_state_len_column: CategoricalColumnWithIdentity,
pos_state_column: CategoricalColumnWithIdentity,
pos_next_state_len_column: CategoricalColumnWithIdentity,
pos_next_state_column: CategoricalColumnWithIdentity,
neg_state_len_column: CategoricalColumnWithIdentity,
neg_state_column: CategoricalColumnWithIdentity,
neg_next_state_len_column: CategoricalColumnWithIdentity,
neg_next_state_column: CategoricalColumnWithIdentity,
rl_sample_column: CategoricalColumnWithIdentity,
emb_size: int,
hidden_size: int,
):
super().__init__()
self.emb_size = emb_size
self.hidden_size = hidden_size
self.uid_column = uid_column
self.iid_column = iid_column
self.pos_state_len_column = pos_state_len_column
self.pos_state_column = pos_state_column
self.pos_next_state_len_column = pos_next_state_len_column
self.pos_next_state_column = pos_next_state_column
self.neg_state_len_column = neg_state_len_column
self.neg_state_column = neg_state_column
self.neg_next_state_len_column = neg_next_state_len_column
self.neg_next_state_column = neg_next_state_column
self.rl_sample_column = rl_sample_column
self.weight_file = weight_file
self.i_embeddings = Embedding(self.iid_column.category_num,
self.emb_size)
self.u_embeddings = Embedding(self.uid_column.category_num,
self.emb_size)
self.long_mlp = MLP(self.emb_size * 2, [self.emb_size] * 3,
activation="relu",
dropout=0.2)
self.pos_rnn = GRU(input_size=self.emb_size,
hidden_size=self.hidden_size,
batch_first=True)
self.pos_mlp = MLP(self.hidden_size + self.emb_size,
[self.emb_size] * 3,
activation="relu",
dropout=0.2)
self.mlp = Dense(self.emb_size * 2,
self.emb_size,
activation="relu",
dropout=0.2)
self.prediction = Linear(self.emb_size, 1, bias=False)
def __init__(
self,
input_size, # 输入特征纬度，如768
label_num,
role_num,
dropout=0.5,
embedding_dim=50,
action_stack_dim=60):
super().__init__()
stack_dim = input_size
# 保存 head_forward 输出的当前 PRED 的 representation
self.current_head = Keeper(stack_dim)
# 保存本句所有 token 的输入 representation
self.buffer = Buffer(input_size)
self.left_candidates = StackLSTM(input_size, stack_dim,
dropout) # sigma left
self.left_undecided = StackLSTM(input_size, stack_dim,
dropout) # alpha left
self.right_candidates = StackLSTM(input_size, stack_dim, dropout)
self.right_undecided = StackLSTM(input_size, stack_dim, dropout)
self.output_stack = StackLSTM(input_size, action_stack_dim)
self.action_stack = StackLSTM(embedding_dim, action_stack_dim, dropout)
self.action_embedding = Embedding(len(Action), embedding_dim)
self.label_embedding = Embedding(label_num, embedding_dim)
self.action_helper = ActionHelper()
self.action_generator = ActionGenerator(input_size, len(Action))
hidden_input_dim = input_size + stack_dim * 5 + action_stack_dim * 2
self.labeler = RoleLabeler(input_size, stack_dim, action_stack_dim,
role_num)
# 8个 prototype 的 embedding 作为输入
self.hidden_forward = NonLinear(hidden_input_dim,
input_size,
activation=Tanh())
# PRED 的输入 representation 和 embedding 作为输入
self.head_forward = NonLinear(input_size + embedding_dim,
stack_dim,
activation=Tanh())
self.distributions_head_forward = NonLinear(stack_dim * 2 +
embedding_dim,
len(Action),
activation=Softmax(-1))
self.distributions_role_forward = NonLinear(stack_dim * 2 +
embedding_dim,
role_num,
activation=Softmax(-1))
self.dropout = Dropout(p=dropout)
def __init__(self, encoder, decoder, generator, rnn_size, src_vocab_size,
tgt_vocab_size):
super(EncoderDecoder, self).__init__()
self.encoder = encoder
self.decoder = decoder
self.generator = generator
self.src_embedding_pe = Sequential(Embedding(src_vocab_size, rnn_size),
PositionalEncoding(rnn_size))
self.tgt_embedding_pe = Sequential(Embedding(tgt_vocab_size, rnn_size),
PositionalEncoding(rnn_size))
def __init__(self,
vocab: Vocabulary,
word_embedding: Dict[str, Any],
depsawr: torch.nn.Module = None,
transform_dim: int = 0,
pos_dim: int = 50,
indicator_dim: int = 50,
encoder: Dict[str, Any] = None,
dropout: float = 0.33,
label_namespace: str = "labels",
top_k: int = 1,
**kwargs) -> None:
super().__init__()
self.word_embedding = build_word_embedding(num_embeddings=len(
vocab['words']),
vocab=vocab,
dropout=dropout,
**word_embedding)
feat_dim: int = self.word_embedding.output_dim
if transform_dim > 0:
self.word_transform = NonLinear(feat_dim, transform_dim)
feat_dim: int = transform_dim
else:
self.word_transform = None
if depsawr:
dep_dim = kwargs.pop('dep_dim', 300)
self.depsawr_forward = depsawr.forward
self.projections = ModuleList(
[NonLinear(i, dep_dim) for i in depsawr.dims])
self.depsawr_mix = ScalarMix(len(depsawr.dims), True)
feat_dim += dep_dim
else:
self.depsawr_forward = None
self.pos_embedding = Embedding(len(vocab['upostag']), pos_dim, 0)
self.indicator_embedding = Embedding(2, indicator_dim)
feat_dim += (pos_dim + indicator_dim)
if encoder is not None:
self.encoder = build_encoder(feat_dim, dropout=dropout, **encoder)
feat_dim = self.encoder.output_dim
else:
self.encoder = None
self.tag_projection_layer = torch.nn.Linear(
feat_dim, len(vocab[label_namespace]))
self.word_dropout = WordDropout(dropout)
self.crf = ConditionalRandomField(len(vocab[label_namespace]),
include_start_end_transitions=False)
self.top_k = top_k
self.metric = SRLMetric(vocab[label_namespace]['_'])
def __init__(self,
hidden_channels=128,
num_filters=128,
num_interactions=6,
num_gaussians=50,
cutoff=10.0,
readout='add',
dipole=False,
mean=None,
std=None,
atomref=None):
super(SchNet, self).__init__()
if ase is None:
raise ImportError('Package `ase` could not be found.')
assert readout in ['add', 'sum', 'mean']
self.hidden_channels = hidden_channels
self.num_filters = num_filters
self.num_interactions = num_interactions
self.num_gaussians = num_gaussians
self.cutoff = cutoff
self.readout = readout
self.dipole = dipole
self.readout = 'add' if self.dipole else self.readout
self.mean = mean
self.std = std
self.scale = None
atomic_mass = torch.from_numpy(ase.data.atomic_masses)
self.register_buffer('atomic_mass', atomic_mass)
self.embedding = Embedding(100, hidden_channels)
self.distance_expansion = GaussianSmearing(0.0, cutoff, num_gaussians)
self.interactions = ModuleList()
for _ in range(num_interactions):
block = InteractionBlock(hidden_channels, num_gaussians,
num_filters, cutoff)
self.interactions.append(block)
self.lin1 = Linear(hidden_channels, hidden_channels // 2)
self.act = ShiftedSoftplus()
self.lin2 = Linear(hidden_channels // 2, 1)
self.register_buffer('initial_atomref', atomref)
self.atomref = None
if atomref is not None:
self.atomref = Embedding(100, 1)
self.atomref.weight.data.copy_(atomref)
self.reset_parameters()
