def test_02_sigmoid(self): # 2p
""" Tests if logistic sigmoid is calculated correctly."""
self.assertAlmostEqual(utils.sigmoid(0), 0.5, delta=0.001)
self.assertAlmostEqual(utils.sigmoid(1), 0.731, delta=0.001)
self.assertAlmostEqual(utils.sigmoid(-1), 0.269, delta=0.001)
self.assertAlmostEqual(utils.sigmoid(10), 1.0, delta=0.001)
self.assertAlmostEqual(utils.sigmoid(-10), 0.0, delta=0.001)
def update(self, target_id, context_id, label, learning_rate):
"""
Performs a gradient update for one instance.
:param target_id: Row number of target word to predict.
:param context_id: Row number of context word.
:param label: Indicator whether pair is co-occurrence from corpus.
:param learning_rate: Multiplier for the magnitude of gradient step.
:return: Log-likelihood of of current example *before* the update.
"""
ctxt_vec = self.context_word_matrix[context_id]
tgt_vec = self.target_word_matrix[target_id]
ctxt_vec_cp = np.copy(ctxt_vec)
tgt_vec_cp = np.copy(tgt_vec)
prob_pos = utils.sigmoid(ctxt_vec.T.dot(tgt_vec))
self.context_word_matrix[context_id] = ctxt_vec_cp + (learning_rate * (label - utils.sigmoid(ctxt_vec_cp.T.dot(tgt_vec_cp))) * tgt_vec_cp)
self.target_word_matrix[target_id] = tgt_vec_cp + (learning_rate * (label - utils.sigmoid(ctxt_vec_cp.T.dot(tgt_vec_cp))) * ctxt_vec_cp)
return math.log(prob_pos) if label else math.log(1 - prob_pos)
def test_04_skipgram_update_2(self):
""" Tests whether update on negative tuple is performed correctly."""
# Check if "positive_and_negative_cooccurrences" isimplemented
self.assertIsNotNone(utils.positive_and_negative_cooccurrences([], 1, 0, {}))
sg = skipgram.SkipGram(["b", "a", "c", "b", "c", "c"], window_size=1, neg_samples_factor=0, vocab_size=3,
num_dims=5)
sg.context_word_matrix = np.array([[1, 0, 1], [0, 1, 0], [0, 0, -2]], dtype='float64')
sg.target_word_matrix = np.array([[-1, 0, -1], [1, 0, 1], [1, 1, 1]], dtype='float64')
ll = sg.update(context_id=0, target_id=1, label=False, learning_rate=1.0)
self.assertTrue((sg.context_word_matrix[0] == sg.target_word_matrix[1]).all())
expected_updated = (1 - utils.sigmoid(2)) * np.array([1., 0., 1.], dtype='float64')
self.assertTrue((sg.context_word_matrix[0] == expected_updated).all())
self.assertAlmostEqual(ll, -2.127, delta=0.001)
def update(self, target_id, context_id, label,
learning_rate): # TODO: Exercise 4.
"""
Performs a gradient update for one instance.
:param target_id: Row number of target word to predict.
:param context_id: Row number of context word.
:param label: Indicator whether pair is co-occurrence from corpus.
:param learning_rate: Multiplier for the magnitude of gradient step.
:return: Log-likelihood of of current example *before* the update.
"""
ctxt_vec = self.context_word_matrix[context_id]
tgt_vec = self.target_word_matrix[target_id]
prob_pos = utils.sigmoid(tgt_vec.dot(ctxt_vec))
nabla_v = (label - prob_pos) * tgt_vec
nabla_w = (label - prob_pos) * ctxt_vec
self.context_word_matrix[context_id] += learning_rate * nabla_v
self.target_word_matrix[target_id] += learning_rate * nabla_w
return math.log(prob_pos) if label else math.log(1 - prob_pos)