def __init__(self, vocab_size, hidden_size, window_size, corpus):
V, H = vocab_size, hidden_size
# init weights
W_in = 0.01 * np.random.randn(V, H).astype('f')
W_out = 0.01 * np.random.randn(V, H).astype('f')
# init layers
self.in_layers = []
for i in range(2 * window_size):
layer = Embedding(W_in) # use Embedding layer
self.in_layers.append(layer)
self.ns_loss = NegativeSamplingLoss(W_out,
corpus,
power=0.75,
sample_size=5)
# collect all params and grads in list
layers = self.in_layers + [self.ns_loss]
self.params, self.grads = [], []
for layer in layers:
self.params += layer.params
self.grads += layer.grads
# save dispersion representation of word in instance variable
self.word_vecs = W_in
def __init__(self, vocab_size, hidden_size, window_size, corpus):
V, H = vocab_size, hidden_size
# 重みの初期化
W_in = 0.01 * np.random.randn(V, H).astype('f')
W_out = 0.01 * np.random.randn(V, H).astype('f')
# レイヤの生成
self.in_layers = []
for i in range(2 * window_size):
layer = Embedding(W_in) # Embeddingレイヤを使用
self.in_layers.append(layer)
self.ns_loss = NegativeSamplingLoss(W_out,
corpus,
power=0.75,
sample_size=5)
# 全ての重みと勾配を配列にまとめる
layers = self.in_layers + [self.ns_loss]
self.params, self.grads = [], []
for layer in layers:
self.params += layer.params
self.grads += layer.grads
# メンバ変数に単語の分散表現を設定
self.word_vecs = W_in
def __init__(self, vocab_size, hidden_size, window_size, corpus):
V, H = vocab_size, hidden_size
# 가중치 초기화
W_in = 0.01 * np.random.randn(V, H).astype('f')
W_out = 0.01 * np.random.randn(V, H).astype('f')
# 레이어 생성
self.in_layers = []
for i in range(2 * window_size):
layer = Embedding(W_in) # Embedding 계층 사용
self.in_layers.append(layer)
self.ns_loss = NegativeSamplingLoss(W_out,
corpus,
power=0.75,
sample_size=5)
# 모든 가중치와 기울기를 배열에 모은다.
layers = self.in_layers + [self.ns_loss]
self.params, self.grads = [], []
for layer in layers:
self.params += layer.params
self.grads += layer.grads
# 인스턴스 변수에 단어의 분산 표현을 저장한다.
self.word_vecs1 = W_in
self.word_vecs2 = W_out
class CBOW:
def __init__(self, vocab_size: int, hidden_size: int, window_size: int, corpus: List[int]) -> None:
W_in = 0.01 * np.random.randn(vocab_size, hidden_size).astype(float)
W_out = 0.01 * np.random.randn(vocab_size, hidden_size).astype(float)
self.in_layers = []
for i in range(2 * window_size):
layer = Embedding(W_in)
self.in_layers.append(layer)
self.ns_loss = NegativeSamplingLoss(W_out, corpus, power=0.75, sample_size=5)
layers = self.in_layers + [self.ns_loss]
self.params, self.grads = [], []
for layer in layers:
self.params += layer.params
self.grads += layer.grads
self.word_vecs = W_in
def forward(self, contexts: np.ndarray, target: np.ndarray) -> float:
h = 0
for i, layer in enumerate(self.in_layers):
h += layer.forward(contexts[:, i])
h *= 1 / len(self.in_layers)
loss = self.ns_loss.forward(h, target)
return loss
def backward(self, dout: int=1) -> None:
dout = self.ns_loss.backward(dout)
dout *= 1 / len(self.in_layers)
for layer in self.in_layers:
layer.backward(dout)
return None
def __init__(self, vocab_size, hidden_size, window_size, corpus):
V, H = vocab_size, hidden_size
W_in = 0.01 * np.random.randn(V, H).astype("f")
W_out = 0.01 * np.random.randn(V, H).astype("f")
# レイヤの生成
self.in_layers = []
for i in range(2 * window_size):
layer = Embedding(W_in)
self.in_layers.append(layer)
# 負例は5つサンプリングするように設定
self.ns_loss = NegativeSamplingLoss(W_out,
corpus,
power=0.75,
sample_size=5)
layers = self.in_layers + [self.ns_loss]
self.params, self.grads = [], []
for layer in layers:
self.params += layer.params
self.grads += layer.grads
# メンバ変数に単語の分散表現を設定
self.word_vecs = W_in
def __init__(self, vocab_size, hidden_size, window_size, corpus):
V, H = vocab_size, hidden_size
rn = np.random.randn
# 가중치 초기화
W_in = 0.01 * rn(V, H).astype('f')
W_out = 0.01 * rn(V, H).astype('f')
# 계층 생성
self.in_layer = Embedding(W_in)
self.loss_layers = []
for i in range(2 * window_size):
layer = NegativeSamplingLoss(W_out,
corpus,
power=0.75,
sample_size=5)
self.loss_layers.append(layer)
# 모든 가중치와 기울기를 리스트에 모은다.
layers = [self.in_layer] + self.loss_layers
self.params, self.grads = [], []
for layer in layers:
self.params += layer.params
self.grads += layer.grads
# 인스턴스 변수에 단어의 분산 표현을 저장한다.
self.word_vecs = W_in
def __init__(self, vocab_size, hidden_size, window_size, corpus):
V, H = vocab_size, hidden_size
rn = np.random.randn
# 初始化權重
W_in = 0.01 * rn(V, H).astype('f')
W_out = 0.01 * rn(V, H).astype('f')
# 產生各層
self.in_layer = Embedding(W_in)
self.loss_layers = []
for i in range(2 * window_size):
layer = NegativeSamplingLoss(W_out,
corpus,
power=0.75,
sample_size=5)
self.loss_layers.append(layer)
# 把所有權重與梯度整合成清單
layers = [self.in_layer] + self.loss_layers
self.params, self.grads = [], []
for layer in layers:
self.params += layer.params
self.grads += layer.grads
# 在成員變數設定字詞的分散式表示
self.word_vecs = W_in
def __init__(self, vocab_size: int, hidden_size: int, window_size: int, corpus: List[int]) -> None:
W_in = 0.01 * np.random.randn(vocab_size, hidden_size).astype(float)
W_out = 0.01 * np.random.randn(vocab_size, hidden_size).astype(float)
self.in_layers = []
for i in range(2 * window_size):
layer = Embedding(W_in)
self.in_layers.append(layer)
self.ns_loss = NegativeSamplingLoss(W_out, corpus, power=0.75, sample_size=5)
layers = self.in_layers + [self.ns_loss]
self.params, self.grads = [], []
for layer in layers:
self.params += layer.params
self.grads += layer.grads
self.word_vecs = W_in
class CBOW:
def __init__(self, vocab_size, hidden_size, window_size, corpus):
V, H = vocab_size, hidden_size
W_in = 0.01 * np.random.randn(V, H).astype("f")
W_out = 0.01 * np.random.randn(V, H).astype("f")
# レイヤの生成
self.in_layers = []
for i in range(2 * window_size):
layer = Embedding(W_in)
self.in_layers.append(layer)
# 負例は5つサンプリングするように設定
self.ns_loss = NegativeSamplingLoss(W_out,
corpus,
power=0.75,
sample_size=5)
layers = self.in_layers + [self.ns_loss]
self.params, self.grads = [], []
for layer in layers:
self.params += layer.params
self.grads += layer.grads
# メンバ変数に単語の分散表現を設定
self.word_vecs = W_in
def forward(self, contexts, target):
# 中間層の計算
h = 0
for i, layer in enumerate(self.in_layers):
h += layer.forward(contexts[:, i])
h *= 1 / len(self.in_layers)
# lossの計算
loss = self.ns_loss.forward(h, target)
return loss
def backward(self, dout=1):
dh = self.ns_loss.backward(dout)
dh *= 1 / len(self.in_layers)
for layer in self.in_layers:
layer.backward(dh)
return None
class CBOW:
def __init__(self, vocab_size, hidden_size, window_size, corpus):
V, H = vocab_size, hidden_size
# 가중치 초기화
W_in = 0.01 * np.random.randn(V, H).astype('f')
W_out = 0.01 * np.random.randn(V, H).astype('f')
# 레이어 생성
self.in_layers = []
for i in range(2 * window_size):
layer = Embedding(W_in) # Embedding 계층 사용
self.in_layers.append(layer)
self.ns_loss = NegativeSamplingLoss(W_out,
corpus,
power=0.75,
sample_size=5)
# 모든 가중치와 기울기를 배열에 모은다.
layers = self.in_layers + [self.ns_loss]
self.params, self.grads = [], []
for layer in layers:
self.params += layer.params
self.grads += layer.grads
# 인스턴스 변수에 단어의 분산 표현을 저장한다.
self.word_vecs1 = W_in
self.word_vecs2 = W_out
def forward(self, contexts, target):
h = 0
for i, layer in enumerate(self.in_layers):
h += layer.forward(contexts[:, i])
h *= 1 / len(self.in_layers) # average
loss = self.ns_loss.forward(h, target)
return loss
def backward(self, dout=1):
dout = self.ns_loss.backward(dout)
dout *= 1 / len(self.in_layers)
for layer in self.in_layers:
layer.backward(dout)
return None
class CBOW:
def __init__(self, vocab_size, hidden_size, window_size, corpus):
V, H = vocab_size, hidden_size
# init weights
W_in = 0.01 * np.random.randn(V, H).astype('f')
W_out = 0.01 * np.random.randn(V, H).astype('f')
# init layers
self.in_layers = []
for i in range(2 * window_size):
layer = Embedding(W_in) # use Embedding layer
self.in_layers.append(layer)
self.ns_loss = NegativeSamplingLoss(W_out,
corpus,
power=0.75,
sample_size=5)
# collect all params and grads in list
layers = self.in_layers + [self.ns_loss]
self.params, self.grads = [], []
for layer in layers:
self.params += layer.params
self.grads += layer.grads
# save dispersion representation of word in instance variable
self.word_vecs = W_in
def forward(self, contexts, target):
h = 0
for i, layer in enumerate(self.in_layers):
h += layer.forward(contexts[:, i])
h *= 1 / len(self.in_layers)
loss = self.ns_loss.forward(h, target)
return loss
def backward(self, dout=1):
dout = self.ns_loss.backward(dout)
dout *= 1 / len(self.in_layers)
for layer in self.in_layers:
layer.backward(dout)
return None
class CBOW:
def __init__(self, vocab_size, hidden_size, window_size, corpus):
V, H = vocab_size, hidden_size
# 重みの初期化
W_in = 0.01 * np.random.randn(V, H).astype('f')
W_out = 0.01 * np.random.randn(V, H).astype('f')
# レイヤの生成
self.in_layers = []
for i in range(2 * window_size):
layer = Embedding(W_in) # Embeddingレイヤを使用
self.in_layers.append(layer)
self.ns_loss = NegativeSamplingLoss(W_out,
corpus,
power=0.75,
sample_size=5)
# 全ての重みと勾配を配列にまとめる
layers = self.in_layers + [self.ns_loss]
self.params, self.grads = [], []
for layer in layers:
self.params += layer.params
self.grads += layer.grads
# メンバ変数に単語の分散表現を設定
self.word_vecs = W_in
def forward(self, contexts, target):
h = 0
for i, layer in enumerate(self.in_layers):
h += layer.forward(contexts[:, i])
h *= 1 / len(self.in_layers)
loss = self.ns_loss.forward(h, target)
return loss
def backward(self, dout=1):
dout = self.ns_loss.backward(dout)
dout *= 1 / len(self.in_layers)
for layer in self.in_layers:
layer.backward(dout)
return None
def __init__(self, vocab_size, hidden_size, window_size, corpus):
V, H = vocab_size, hidden_size
W_in = 0.01 * cp.random.randn(V, H).astype('f')
W_out = 0.01 * cp.random.randn(V, H).astype('f')
self.in_layers = []
for i in range(2 * window_size):
layer = Embedding(W_in)
self.in_layers.append(layer)
self.ns_loss = NegativeSamplingLoss(W_out,
corpus,
power=0.75,
sample_size=5)
layers = self.in_layers + [self.ns_loss]
self.params, self.grads = [], []
for layer in layers:
self.params += layer.params
self.grads += layer.grads
self.word_vecs = W_in
class CBOW:
def __init__(self, vocab_size, hidden_size, window_size, corpus):
V, H = vocab_size, hidden_size
# 初始化權重
W_in = 0.01 * np.random.randn(V, H).astype('f')
W_out = 0.01 * np.random.randn(V, H).astype('f')
# 產生各層
self.in_layers = []
for i in range(2 * window_size):
layer = Embedding(W_in) # 使用Embedding層
self.in_layers.append(layer)
self.ns_loss = NegativeSamplingLoss(W_out, corpus, power=0.75, sample_size=5)
# 把所有權重與梯度整合成清單
layers = self.in_layers + [self.ns_loss]
self.params, self.grads = [], []
for layer in layers:
self.params += layer.params
self.grads += layer.grads
# 在成員變數設定字詞的分散式表示
self.word_vecs = W_in
def forward(self, contexts, target):
h = 0
for i, layer in enumerate(self.in_layers):
h += layer.forward(contexts[:, i])
h *= 1 / len(self.in_layers)
loss = self.ns_loss.forward(h, target)
return loss
def backward(self, dout=1):
dout = self.ns_loss.backward(dout)
dout *= 1 / len(self.in_layers)
for layer in self.in_layers:
layer.backward(dout)
return None
