class CBOW: def __init__(self, vocab_size: int, hidden_size: int, window_size: int, corpus: List[int]) -> None: W_in = 0.01 * np.random.randn(vocab_size, hidden_size).astype(float) W_out = 0.01 * np.random.randn(vocab_size, hidden_size).astype(float) self.in_layers = [] for i in range(2 * window_size): layer = Embedding(W_in) self.in_layers.append(layer) self.ns_loss = NegativeSamplingLoss(W_out, corpus, power=0.75, sample_size=5) layers = self.in_layers + [self.ns_loss] self.params, self.grads = [], [] for layer in layers: self.params += layer.params self.grads += layer.grads self.word_vecs = W_in def forward(self, contexts: np.ndarray, target: np.ndarray) -> float: h = 0 for i, layer in enumerate(self.in_layers): h += layer.forward(contexts[:, i]) h *= 1 / len(self.in_layers) loss = self.ns_loss.forward(h, target) return loss def backward(self, dout: int=1) -> None: dout = self.ns_loss.backward(dout) dout *= 1 / len(self.in_layers) for layer in self.in_layers: layer.backward(dout) return None
class CBOW: def __init__(self, vocab_size, hidden_size, window_size, corpus): V, H = vocab_size, hidden_size W_in = 0.01 * np.random.randn(V, H).astype("f") W_out = 0.01 * np.random.randn(V, H).astype("f") # レイヤの生成 self.in_layers = [] for i in range(2 * window_size): layer = Embedding(W_in) self.in_layers.append(layer) # 負例は5つサンプリングするように設定 self.ns_loss = NegativeSamplingLoss(W_out, corpus, power=0.75, sample_size=5) layers = self.in_layers + [self.ns_loss] self.params, self.grads = [], [] for layer in layers: self.params += layer.params self.grads += layer.grads # メンバ変数に単語の分散表現を設定 self.word_vecs = W_in def forward(self, contexts, target): # 中間層の計算 h = 0 for i, layer in enumerate(self.in_layers): h += layer.forward(contexts[:, i]) h *= 1 / len(self.in_layers) # lossの計算 loss = self.ns_loss.forward(h, target) return loss def backward(self, dout=1): dh = self.ns_loss.backward(dout) dh *= 1 / len(self.in_layers) for layer in self.in_layers: layer.backward(dh) return None
class CBOW: def __init__(self, vocab_size, hidden_size, window_size, corpus): V, H = vocab_size, hidden_size # 가중치 초기화 W_in = 0.01 * np.random.randn(V, H).astype('f') W_out = 0.01 * np.random.randn(V, H).astype('f') # 레이어 생성 self.in_layers = [] for i in range(2 * window_size): layer = Embedding(W_in) # Embedding 계층 사용 self.in_layers.append(layer) self.ns_loss = NegativeSamplingLoss(W_out, corpus, power=0.75, sample_size=5) # 모든 가중치와 기울기를 배열에 모은다. layers = self.in_layers + [self.ns_loss] self.params, self.grads = [], [] for layer in layers: self.params += layer.params self.grads += layer.grads # 인스턴스 변수에 단어의 분산 표현을 저장한다. self.word_vecs1 = W_in self.word_vecs2 = W_out def forward(self, contexts, target): h = 0 for i, layer in enumerate(self.in_layers): h += layer.forward(contexts[:, i]) h *= 1 / len(self.in_layers) # average loss = self.ns_loss.forward(h, target) return loss def backward(self, dout=1): dout = self.ns_loss.backward(dout) dout *= 1 / len(self.in_layers) for layer in self.in_layers: layer.backward(dout) return None
class CBOW: def __init__(self, vocab_size, hidden_size, window_size, corpus): V, H = vocab_size, hidden_size # init weights W_in = 0.01 * np.random.randn(V, H).astype('f') W_out = 0.01 * np.random.randn(V, H).astype('f') # init layers self.in_layers = [] for i in range(2 * window_size): layer = Embedding(W_in) # use Embedding layer self.in_layers.append(layer) self.ns_loss = NegativeSamplingLoss(W_out, corpus, power=0.75, sample_size=5) # collect all params and grads in list layers = self.in_layers + [self.ns_loss] self.params, self.grads = [], [] for layer in layers: self.params += layer.params self.grads += layer.grads # save dispersion representation of word in instance variable self.word_vecs = W_in def forward(self, contexts, target): h = 0 for i, layer in enumerate(self.in_layers): h += layer.forward(contexts[:, i]) h *= 1 / len(self.in_layers) loss = self.ns_loss.forward(h, target) return loss def backward(self, dout=1): dout = self.ns_loss.backward(dout) dout *= 1 / len(self.in_layers) for layer in self.in_layers: layer.backward(dout) return None
class CBOW: def __init__(self, vocab_size, hidden_size, window_size, corpus): V, H = vocab_size, hidden_size # 重みの初期化 W_in = 0.01 * np.random.randn(V, H).astype('f') W_out = 0.01 * np.random.randn(V, H).astype('f') # レイヤの生成 self.in_layers = [] for i in range(2 * window_size): layer = Embedding(W_in) # Embeddingレイヤを使用 self.in_layers.append(layer) self.ns_loss = NegativeSamplingLoss(W_out, corpus, power=0.75, sample_size=5) # 全ての重みと勾配を配列にまとめる layers = self.in_layers + [self.ns_loss] self.params, self.grads = [], [] for layer in layers: self.params += layer.params self.grads += layer.grads # メンバ変数に単語の分散表現を設定 self.word_vecs = W_in def forward(self, contexts, target): h = 0 for i, layer in enumerate(self.in_layers): h += layer.forward(contexts[:, i]) h *= 1 / len(self.in_layers) loss = self.ns_loss.forward(h, target) return loss def backward(self, dout=1): dout = self.ns_loss.backward(dout) dout *= 1 / len(self.in_layers) for layer in self.in_layers: layer.backward(dout) return None
class CBOW: def __init__(self, vocab_size, hidden_size, window_size, corpus): V, H = vocab_size, hidden_size # 初始化權重 W_in = 0.01 * np.random.randn(V, H).astype('f') W_out = 0.01 * np.random.randn(V, H).astype('f') # 產生各層 self.in_layers = [] for i in range(2 * window_size): layer = Embedding(W_in) # 使用Embedding層 self.in_layers.append(layer) self.ns_loss = NegativeSamplingLoss(W_out, corpus, power=0.75, sample_size=5) # 把所有權重與梯度整合成清單 layers = self.in_layers + [self.ns_loss] self.params, self.grads = [], [] for layer in layers: self.params += layer.params self.grads += layer.grads # 在成員變數設定字詞的分散式表示 self.word_vecs = W_in def forward(self, contexts, target): h = 0 for i, layer in enumerate(self.in_layers): h += layer.forward(contexts[:, i]) h *= 1 / len(self.in_layers) loss = self.ns_loss.forward(h, target) return loss def backward(self, dout=1): dout = self.ns_loss.backward(dout) dout *= 1 / len(self.in_layers) for layer in self.in_layers: layer.backward(dout) return None