def next_state(self, previous_state, user_response,
slate_docs):
chosen_docs = user_response.get('choice')
chosen_doc_features = selectors.get_chosen(slate_docs, chosen_docs)
provider_id = chosen_doc_features.get('provider_id')
provider_id_one_hot = tf.one_hot(
provider_id, self._num_providers, dtype=tf.float32)
return Value(
provider_pulls=(previous_state.get('provider_pulls') +
provider_id_one_hot))
def next_state(self, previous_state, user_response, slate_docs):
"""Increases click counts of content providers of consumed documents."""
chosen_docs = user_response.get("choice")
chosen_doc_features = selectors.get_chosen(slate_docs, chosen_docs)
provider_id = chosen_doc_features.get("provider_id")
provider_id_one_hot = tf.one_hot(provider_id,
self._num_providers,
dtype=tf.float32)
provider_click_count = (
self._discount * previous_state.get("provider_click_count") +
tf.reduce_sum(provider_id_one_hot, 0))
return Value(
provider_click_count=ed.Deterministic(provider_click_count))
def next_response(self, previous_state, slate_docs):
"""The response value after the initial value."""
affinities = self._affinity_model.affinities(
previous_state.get('interest.state'),
slate_docs.get('doc_features')).get('affinities')
choice = self._choice_model.choice(affinities + 2.0)
chosen_doc_idx = choice.get('choice')
# Calculate consumption time. Negative quality documents generate more
# engagement but ultimately lead to negative interest evolution.
doc_quality = slate_docs.get('doc_quality')
consumed_fraction = tf.sigmoid(-doc_quality)
doc_length = slate_docs.get('doc_length')
consumed_time = consumed_fraction * doc_length
chosen_doc_responses = selector_lib.get_chosen(
Value(consumed_time=consumed_time), chosen_doc_idx)
return chosen_doc_responses.union(choice)
def next_state(self, previous_state, user_response, slate_docs):
"""The state value after the initial value."""
user_interests = previous_state.get('user_interests')
chosen_docs = user_response.get('choice')
chosen_doc_features = selectors.get_chosen(slate_docs, chosen_docs)
doc_features = chosen_doc_features.get('doc_features')
# Define similarities to be affinities(user_interest, doc_features) + 2.
similarities = self._utility_model.affinities(
user_interests, doc_features, False).get('affinities') + 2.0
return Value(
utilities=ed.Normal(loc=similarities,
scale=self._utility_stddev,
validate_args=True),
user_interests=ed.Independent(
tfd.Deterministic(user_interests + self._interest_step_size *
(user_interests - doc_features)),
reinterpreted_batch_ndims=1))
def test_states(self):
self._user = ie_user.InterestEvolutionUser(
self._config, no_click_mass=self._no_click_mass)
init_state = self._user.initial_state()
user_interests = init_state.get('interest').get('state')
np.testing.assert_array_equal(
[self._config['num_users'], self._config['num_topics']],
np.shape(user_interests))
# Create a dummy response and check user interest shift.
doc_features = [[[1., 0., 0., 0., 0.]], [[1., 0., 0., 0., 0.]],
[[0., 0., 1., 0., 0.]], [[0., 1., 0., 0., 0.]],
[[0., 0., 1., 0., 0.]]]
slate_docs = Value(
doc_id=ed.Deterministic(
loc=tf.constant([[1], [2], [3], [4], [5]])),
doc_topic=ed.Deterministic(
loc=tf.constant([[0], [1], [2], [3], [4]])),
doc_quality=ed.Deterministic(
loc=tf.constant([[0.], [0.], [0.], [0.], [0.]])),
doc_features=ed.Deterministic(loc=tf.constant(doc_features)),
doc_length=ed.Deterministic(
loc=tf.constant([[1.], [1.], [1.], [1.], [1.]])),
)
mock_response = Value(
choice=ed.Deterministic(
loc=tf.zeros((self._num_users, ), dtype=tf.int32)),
consumed_time=ed.Deterministic(loc=tf.ones((self._num_users, ))))
next_state = self._user.next_state(init_state, mock_response,
slate_docs)
chosen_docs = mock_response.get('choice')
chosen_doc_features = selector_lib.get_chosen(slate_docs, chosen_docs)
response_doc_quality = chosen_doc_features.get('doc_quality')
response_doc_features = chosen_doc_features.get('doc_features')
expected_direction = response_doc_quality * (response_doc_features -
user_interests)
expected_user_interests_update = (self._interest_step_size *
expected_direction)
expected_user_interests = user_interests + expected_user_interests_update
expected_user_interests = (
4.0 * self._user._max_user_affinity *
(tf.sigmoid(expected_user_interests /
self._user._max_user_affinity) - 0.5))
self.assertAllClose(expected_user_interests,
next_state.get('interest').get('state'))
def next_state(self, previous_state, user_response, slate_docs):
"""The state value after the initial value."""
chosen_docs = user_response.get('choice')
chosen_doc_features = selector_lib.get_chosen(slate_docs, chosen_docs)
# Calculate utilities.
user_interests = previous_state.get('interest.state')
doc_features = chosen_doc_features.get('doc_features')
# User interests are increased/decreased towards the consumed document's
# topic proportinal to the document quality.
direction = tf.expand_dims(chosen_doc_features.get('doc_quality'),
axis=-1) * (doc_features - user_interests)
linear_update = self._interest_model.next_state(
previous_state.get('interest'),
Value(input=direction,
condition=tf.less(user_response.get('consumed_time'), 0.)))
# We squash the interest vector to avoid infinite blow-up using the function
# 4 * M * (sigmoid(X/M) - 0.5) which is roughly linear around the origin and
# softly saturates at +/-2M. These constants are not intended to be tunable.
next_interest = Value(
state=4.0 * self._max_user_affinity *
(tf.sigmoid(linear_update.get('state') / self._max_user_affinity) -
0.5)).union(linear_update.prefixed_with('linear_update'))
return next_interest.prefixed_with('interest')
def next_state(self, previous_state, user_response, slate_docs):
"""The state value after the initial value."""
chosen_doc_idx = user_response.get("choice")
chosen_doc_features = selector_lib.get_chosen(slate_docs,
chosen_doc_idx)
# Update doc_id history.
doc_consumed = tf.reshape(chosen_doc_features.get("doc_id"),
[self._num_users])
# We update histories of only users who chose a doc.
no_choice = tf.equal(user_response.get("choice"),
self._slate_size)[Ellipsis, tf.newaxis]
next_doc_id_history = self._doc_history.next_state(
previous_state.get("doc_history"),
Value(input=doc_consumed,
condition=no_choice)).prefixed_with("doc_history")
# Update consumed time.
time_consumed = tf.reshape(user_response.get("consumed_time"),
[self._num_users])
next_ctime_history = self._ctime_history.next_state(
previous_state.get("ctime_history"),
Value(input=time_consumed,
condition=no_choice)).prefixed_with("ctime_history")
return next_doc_id_history.union(next_ctime_history)