def from_random(cls, vocab_size: int, emb_size: int, class_count: int,
mode: str, activation: str):
embedding_bag = EmbeddingBag(num_embeddings=vocab_size,
embedding_dim=emb_size,
mode=mode)
return cls(embedding_bag, class_count, activation)
def __init__(self, class_size, vectors):
super(NBoW, self).__init__()
# 単語ベクトルの平均を用いて特殊ベクトルを作成するレイヤー
self.nbow_layer = EmbeddingBag(vectors.size(0), vectors.size(1))
# 単語ベクトルの初期値として学習した単語の意味ベクトルを用いる
self.nbow_layer.weight = Parameter(vectors)
# 各クラスに対応するスコアの出力を行うレイヤー
self.output_layer = Linear(vectors.size(1), class_size)
def __init__(self, field_sizes, embed_dim):
super(LinearEmbedder, self).__init__()
self.embedding = EmbeddingBag(sum(field_sizes) + 1,
embed_dim,
mode='sum')
self.offsets = np.array((0, *np.cumsum(field_sizes)[:-1]),
dtype=np.long)
self.bias = Parameter(torch.zeros((embed_dim, )))
def __init__(self, class_size, vectors):
super(NBoW, self).__init__()
self.nbow_layer = EmbeddingBag(vectors.size(0), vectors.size(1))
self.nbow_layer.weight = Parameter(vectors)
self.output_layer = Linear(vectors.size(1), class_size)
def __init__(self, vocab_size, embedding_dim, hidden_dim, num_class):
super(MLP, self).__init__()
self.embedding = EmbeddingBag(vocab_size, embedding_dim)
self.linear1 = Linear(embedding_dim, hidden_dim)
self.activate = torch.relu
self.linear2 = Linear(hidden_dim, num_class)
from torch.nn import Module, Embedding, Parameter
from torch.autograd import Variable
from torch.utils.data import Dataset, DataLoader
from torch.utils.data.sampler import SubsetRandomSampler
import random
from sklearn.metrics import r2_score
EMBS_FILE = 'embs.pickle'
BATCH_SIZE = 100000
CUDA = True
NUM_EPOCHS = 20
with open(EMBS_FILE, 'rb') as f:
emb_params = pickle.load(f)
num_entities, dim = emb_params.size()
emb = EmbeddingBag(num_entities, dim, mode='sum')
emb.weight = Parameter(emb_params)
data = []
model = torch.nn.Sequential(
torch.nn.Linear(100, 100),
torch.nn.LeakyReLU(),
torch.nn.Linear(100, 10),
torch.nn.LeakyReLU(),
torch.nn.Linear(10, 10),
torch.nn.LeakyReLU(),
torch.nn.Linear(10, 10),
torch.nn.LeakyReLU(),
torch.nn.Linear(10, 10),
torch.nn.LeakyReLU(),
torch.nn.Linear(10, 10),
def from_pre_trained(cls, vocab: Vocab, class_count: int, mode: str,
update_vectors: bool, activation: str):
embedding_bag = EmbeddingBag.from_pretrained(
vocab.vectors, mode=mode, freeze=(not update_vectors))
return cls(embedding_bag, class_count, activation)
def __init__(
self,
vocab: Vocabulary,
source_embedder: TextFieldEmbedder,
source_encoder: Seq2SeqEncoder,
max_decoding_steps: int,
dialog_acts_encoder: FeedForward = None,
attention: Attention = None,
attention_function: SimilarityFunction = None,
n_dialog_acts: int = None,
beam_size: int = None,
target_namespace: str = "tokens",
target_embedding_dim: int = None,
scheduled_sampling_ratio: float = 0.0,
use_bleu: bool = True,
use_dialog_acts: bool = True,
regularizers: Optional[RegularizerApplicator] = None,
) -> None:
super().__init__(vocab, regularizers)
self._target_namespace = target_namespace
self._scheduled_sampling_ratio = scheduled_sampling_ratio
# We need the start symbol to provide as the input at the first
# timestep of decoding, and end symbol as a way to indicate the end
# of the decoded sequence.
self._start_index = self.vocab.get_token_index(START_SYMBOL,
self._target_namespace)
self._end_index = self.vocab.get_token_index(END_SYMBOL,
self._target_namespace)
if use_bleu:
pad_index = self.vocab.get_token_index(self.vocab._padding_token,
self._target_namespace)
self._bleu = BLEU(exclude_indices={
pad_index, self._end_index, self._start_index
})
else:
self._bleu = None
# At prediction time, we use a beam search to find the most
# likely sequence of target tokens.
beam_size = beam_size or 1
self._max_decoding_steps = max_decoding_steps
self._beam_search = BeamSearch(self._end_index,
max_steps=max_decoding_steps,
beam_size=beam_size)
# Dense embedding of source (Facts) vocab tokens.
self._source_embedder = source_embedder
# Encodes the sequence of source embeddings into a sequence of hidden states.
self._source_encoder = source_encoder
if use_dialog_acts:
# Dense embedding of dialog acts.
da_embedding_dim = dialog_acts_encoder.get_input_dim()
self._dialog_acts_embedder = EmbeddingBag(n_dialog_acts,
da_embedding_dim)
# Encodes dialog acts
self._dialog_acts_encoder = dialog_acts_encoder
else:
self._dialog_acts_embedder = None
self._dialog_acts_encoder = None
num_classes = self.vocab.get_vocab_size(self._target_namespace)
# Attention mechanism applied to the encoder output for each step.
if attention:
if attention_function:
raise ConfigurationError(
"You can only specify an attention module or an "
"attention function, but not both.")
self._attention = attention
elif attention_function:
self._attention = LegacyAttention(attention_function)
else:
self._attention = None
# Dense embedding of vocab words in the target space.
target_embedding_dim = target_embedding_dim or source_embedder.get_output_dim(
)
self._target_embedder = Embedding(num_classes, target_embedding_dim)
# Decoder output dim needs to be the same as the encoder output dim
# since we initialize the hidden state of the decoder with the final
# hidden state of the encoder.
self._encoder_output_dim = self._source_encoder.get_output_dim()
if use_dialog_acts:
self._merge_encoder = Sequential(
Linear(
self._source_encoder.get_output_dim() +
self._dialog_acts_encoder.get_output_dim(),
self._encoder_output_dim,
))
self._decoder_output_dim = self._encoder_output_dim
if self._attention:
# If using attention, a weighted average over encoder outputs will
# be concatenated to the previous target embedding to form the input
# to the decoder at each time step.
self._decoder_input_dim = self._decoder_output_dim + target_embedding_dim
else:
# Otherwise, the input to the decoder is just the previous target embedding.
self._decoder_input_dim = target_embedding_dim
# We'll use an LSTM cell as the recurrent cell that produces a hidden state
# for the decoder at each time step.
# TODO (pradeep): Do not hardcode decoder cell type.
self._decoder_cell = LSTMCell(self._decoder_input_dim,
self._decoder_output_dim)
# We project the hidden state from the decoder into the output vocabulary space
# in order to get log probabilities of each target token, at each time step.
self._output_projection_layer = Linear(self._decoder_output_dim,
num_classes)
def from_pre_trained(cls, vocab: Vocab, class_count: int):
embedding_bag = EmbeddingBag.from_pretrained(vocab.vectors)
return cls(embedding_bag, class_count)
def from_random(cls, vocab_size: int, emb_size: int, class_count: int):
embedding_bag = EmbeddingBag(num_embeddings=vocab_size,
embedding_dim=emb_size)
return cls(embedding_bag, class_count)
print('Shuffling indices')
random.shuffle(indices)
valid_indices = indices[:10000]
train_indices = indices[10001:]
print('Creating samplers')
train_dataset = TensorDataset(features[train_indices], labels[train_indices])
valid_dataset = TensorDataset(features[valid_indices], labels[valid_indices])
train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
valid_loader = DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=True)
num_entities = len(node_ids)
emb = EmbeddingBag(num_entities, DIM, mode='sum')
optim = torch.optim.Adam(emb.parameters(), lr=10e-4)
if CUDA:
emb = emb.cuda()
rev_node_ids = dict((x[1], x[0]) for x in node_ids.items())
def compute_loss(batch):
ids, labels = batch
labels = torch.FloatTensor(labels.numpy().tolist())
ids = Variable(ids)
labels = Variable(labels)
if CUDA:
ids = ids.cuda()
labels = labels.cuda()