def from_random(cls, vocab_size: int, emb_size: int, class_count: int, mode: str, activation: str): embedding_bag = EmbeddingBag(num_embeddings=vocab_size, embedding_dim=emb_size, mode=mode) return cls(embedding_bag, class_count, activation)
def __init__(self, class_size, vectors): super(NBoW, self).__init__() # 単語ベクトルの平均を用いて特殊ベクトルを作成するレイヤー self.nbow_layer = EmbeddingBag(vectors.size(0), vectors.size(1)) # 単語ベクトルの初期値として学習した単語の意味ベクトルを用いる self.nbow_layer.weight = Parameter(vectors) # 各クラスに対応するスコアの出力を行うレイヤー self.output_layer = Linear(vectors.size(1), class_size)
def __init__(self, field_sizes, embed_dim): super(LinearEmbedder, self).__init__() self.embedding = EmbeddingBag(sum(field_sizes) + 1, embed_dim, mode='sum') self.offsets = np.array((0, *np.cumsum(field_sizes)[:-1]), dtype=np.long) self.bias = Parameter(torch.zeros((embed_dim, )))
def __init__(self, class_size, vectors): super(NBoW, self).__init__() self.nbow_layer = EmbeddingBag(vectors.size(0), vectors.size(1)) self.nbow_layer.weight = Parameter(vectors) self.output_layer = Linear(vectors.size(1), class_size)
def __init__(self, vocab_size, embedding_dim, hidden_dim, num_class): super(MLP, self).__init__() self.embedding = EmbeddingBag(vocab_size, embedding_dim) self.linear1 = Linear(embedding_dim, hidden_dim) self.activate = torch.relu self.linear2 = Linear(hidden_dim, num_class)
from torch.nn import Module, Embedding, Parameter from torch.autograd import Variable from torch.utils.data import Dataset, DataLoader from torch.utils.data.sampler import SubsetRandomSampler import random from sklearn.metrics import r2_score EMBS_FILE = 'embs.pickle' BATCH_SIZE = 100000 CUDA = True NUM_EPOCHS = 20 with open(EMBS_FILE, 'rb') as f: emb_params = pickle.load(f) num_entities, dim = emb_params.size() emb = EmbeddingBag(num_entities, dim, mode='sum') emb.weight = Parameter(emb_params) data = [] model = torch.nn.Sequential( torch.nn.Linear(100, 100), torch.nn.LeakyReLU(), torch.nn.Linear(100, 10), torch.nn.LeakyReLU(), torch.nn.Linear(10, 10), torch.nn.LeakyReLU(), torch.nn.Linear(10, 10), torch.nn.LeakyReLU(), torch.nn.Linear(10, 10), torch.nn.LeakyReLU(), torch.nn.Linear(10, 10),
def from_pre_trained(cls, vocab: Vocab, class_count: int, mode: str, update_vectors: bool, activation: str): embedding_bag = EmbeddingBag.from_pretrained( vocab.vectors, mode=mode, freeze=(not update_vectors)) return cls(embedding_bag, class_count, activation)
def __init__( self, vocab: Vocabulary, source_embedder: TextFieldEmbedder, source_encoder: Seq2SeqEncoder, max_decoding_steps: int, dialog_acts_encoder: FeedForward = None, attention: Attention = None, attention_function: SimilarityFunction = None, n_dialog_acts: int = None, beam_size: int = None, target_namespace: str = "tokens", target_embedding_dim: int = None, scheduled_sampling_ratio: float = 0.0, use_bleu: bool = True, use_dialog_acts: bool = True, regularizers: Optional[RegularizerApplicator] = None, ) -> None: super().__init__(vocab, regularizers) self._target_namespace = target_namespace self._scheduled_sampling_ratio = scheduled_sampling_ratio # We need the start symbol to provide as the input at the first # timestep of decoding, and end symbol as a way to indicate the end # of the decoded sequence. self._start_index = self.vocab.get_token_index(START_SYMBOL, self._target_namespace) self._end_index = self.vocab.get_token_index(END_SYMBOL, self._target_namespace) if use_bleu: pad_index = self.vocab.get_token_index(self.vocab._padding_token, self._target_namespace) self._bleu = BLEU(exclude_indices={ pad_index, self._end_index, self._start_index }) else: self._bleu = None # At prediction time, we use a beam search to find the most # likely sequence of target tokens. beam_size = beam_size or 1 self._max_decoding_steps = max_decoding_steps self._beam_search = BeamSearch(self._end_index, max_steps=max_decoding_steps, beam_size=beam_size) # Dense embedding of source (Facts) vocab tokens. self._source_embedder = source_embedder # Encodes the sequence of source embeddings into a sequence of hidden states. self._source_encoder = source_encoder if use_dialog_acts: # Dense embedding of dialog acts. da_embedding_dim = dialog_acts_encoder.get_input_dim() self._dialog_acts_embedder = EmbeddingBag(n_dialog_acts, da_embedding_dim) # Encodes dialog acts self._dialog_acts_encoder = dialog_acts_encoder else: self._dialog_acts_embedder = None self._dialog_acts_encoder = None num_classes = self.vocab.get_vocab_size(self._target_namespace) # Attention mechanism applied to the encoder output for each step. if attention: if attention_function: raise ConfigurationError( "You can only specify an attention module or an " "attention function, but not both.") self._attention = attention elif attention_function: self._attention = LegacyAttention(attention_function) else: self._attention = None # Dense embedding of vocab words in the target space. target_embedding_dim = target_embedding_dim or source_embedder.get_output_dim( ) self._target_embedder = Embedding(num_classes, target_embedding_dim) # Decoder output dim needs to be the same as the encoder output dim # since we initialize the hidden state of the decoder with the final # hidden state of the encoder. self._encoder_output_dim = self._source_encoder.get_output_dim() if use_dialog_acts: self._merge_encoder = Sequential( Linear( self._source_encoder.get_output_dim() + self._dialog_acts_encoder.get_output_dim(), self._encoder_output_dim, )) self._decoder_output_dim = self._encoder_output_dim if self._attention: # If using attention, a weighted average over encoder outputs will # be concatenated to the previous target embedding to form the input # to the decoder at each time step. self._decoder_input_dim = self._decoder_output_dim + target_embedding_dim else: # Otherwise, the input to the decoder is just the previous target embedding. self._decoder_input_dim = target_embedding_dim # We'll use an LSTM cell as the recurrent cell that produces a hidden state # for the decoder at each time step. # TODO (pradeep): Do not hardcode decoder cell type. self._decoder_cell = LSTMCell(self._decoder_input_dim, self._decoder_output_dim) # We project the hidden state from the decoder into the output vocabulary space # in order to get log probabilities of each target token, at each time step. self._output_projection_layer = Linear(self._decoder_output_dim, num_classes)
def from_pre_trained(cls, vocab: Vocab, class_count: int): embedding_bag = EmbeddingBag.from_pretrained(vocab.vectors) return cls(embedding_bag, class_count)
def from_random(cls, vocab_size: int, emb_size: int, class_count: int): embedding_bag = EmbeddingBag(num_embeddings=vocab_size, embedding_dim=emb_size) return cls(embedding_bag, class_count)
print('Shuffling indices') random.shuffle(indices) valid_indices = indices[:10000] train_indices = indices[10001:] print('Creating samplers') train_dataset = TensorDataset(features[train_indices], labels[train_indices]) valid_dataset = TensorDataset(features[valid_indices], labels[valid_indices]) train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True) valid_loader = DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=True) num_entities = len(node_ids) emb = EmbeddingBag(num_entities, DIM, mode='sum') optim = torch.optim.Adam(emb.parameters(), lr=10e-4) if CUDA: emb = emb.cuda() rev_node_ids = dict((x[1], x[0]) for x in node_ids.items()) def compute_loss(batch): ids, labels = batch labels = torch.FloatTensor(labels.numpy().tolist()) ids = Variable(ids) labels = Variable(labels) if CUDA: ids = ids.cuda() labels = labels.cuda()