def next_state(self, previous_state, user_response, slate_docs): chosen_docs = user_response.get('choice') chosen_doc_features = selectors.get_chosen(slate_docs, chosen_docs) provider_id = chosen_doc_features.get('provider_id') provider_id_one_hot = tf.one_hot( provider_id, self._num_providers, dtype=tf.float32) return Value( provider_pulls=(previous_state.get('provider_pulls') + provider_id_one_hot))
def next_state(self, previous_state, user_response, slate_docs): """Increases click counts of content providers of consumed documents.""" chosen_docs = user_response.get("choice") chosen_doc_features = selectors.get_chosen(slate_docs, chosen_docs) provider_id = chosen_doc_features.get("provider_id") provider_id_one_hot = tf.one_hot(provider_id, self._num_providers, dtype=tf.float32) provider_click_count = ( self._discount * previous_state.get("provider_click_count") + tf.reduce_sum(provider_id_one_hot, 0)) return Value( provider_click_count=ed.Deterministic(provider_click_count))
def next_response(self, previous_state, slate_docs): """The response value after the initial value.""" affinities = self._affinity_model.affinities( previous_state.get('interest.state'), slate_docs.get('doc_features')).get('affinities') choice = self._choice_model.choice(affinities + 2.0) chosen_doc_idx = choice.get('choice') # Calculate consumption time. Negative quality documents generate more # engagement but ultimately lead to negative interest evolution. doc_quality = slate_docs.get('doc_quality') consumed_fraction = tf.sigmoid(-doc_quality) doc_length = slate_docs.get('doc_length') consumed_time = consumed_fraction * doc_length chosen_doc_responses = selector_lib.get_chosen( Value(consumed_time=consumed_time), chosen_doc_idx) return chosen_doc_responses.union(choice)
def next_state(self, previous_state, user_response, slate_docs): """The state value after the initial value.""" user_interests = previous_state.get('user_interests') chosen_docs = user_response.get('choice') chosen_doc_features = selectors.get_chosen(slate_docs, chosen_docs) doc_features = chosen_doc_features.get('doc_features') # Define similarities to be affinities(user_interest, doc_features) + 2. similarities = self._utility_model.affinities( user_interests, doc_features, False).get('affinities') + 2.0 return Value( utilities=ed.Normal(loc=similarities, scale=self._utility_stddev, validate_args=True), user_interests=ed.Independent( tfd.Deterministic(user_interests + self._interest_step_size * (user_interests - doc_features)), reinterpreted_batch_ndims=1))
def test_states(self): self._user = ie_user.InterestEvolutionUser( self._config, no_click_mass=self._no_click_mass) init_state = self._user.initial_state() user_interests = init_state.get('interest').get('state') np.testing.assert_array_equal( [self._config['num_users'], self._config['num_topics']], np.shape(user_interests)) # Create a dummy response and check user interest shift. doc_features = [[[1., 0., 0., 0., 0.]], [[1., 0., 0., 0., 0.]], [[0., 0., 1., 0., 0.]], [[0., 1., 0., 0., 0.]], [[0., 0., 1., 0., 0.]]] slate_docs = Value( doc_id=ed.Deterministic( loc=tf.constant([[1], [2], [3], [4], [5]])), doc_topic=ed.Deterministic( loc=tf.constant([[0], [1], [2], [3], [4]])), doc_quality=ed.Deterministic( loc=tf.constant([[0.], [0.], [0.], [0.], [0.]])), doc_features=ed.Deterministic(loc=tf.constant(doc_features)), doc_length=ed.Deterministic( loc=tf.constant([[1.], [1.], [1.], [1.], [1.]])), ) mock_response = Value( choice=ed.Deterministic( loc=tf.zeros((self._num_users, ), dtype=tf.int32)), consumed_time=ed.Deterministic(loc=tf.ones((self._num_users, )))) next_state = self._user.next_state(init_state, mock_response, slate_docs) chosen_docs = mock_response.get('choice') chosen_doc_features = selector_lib.get_chosen(slate_docs, chosen_docs) response_doc_quality = chosen_doc_features.get('doc_quality') response_doc_features = chosen_doc_features.get('doc_features') expected_direction = response_doc_quality * (response_doc_features - user_interests) expected_user_interests_update = (self._interest_step_size * expected_direction) expected_user_interests = user_interests + expected_user_interests_update expected_user_interests = ( 4.0 * self._user._max_user_affinity * (tf.sigmoid(expected_user_interests / self._user._max_user_affinity) - 0.5)) self.assertAllClose(expected_user_interests, next_state.get('interest').get('state'))
def next_state(self, previous_state, user_response, slate_docs): """The state value after the initial value.""" chosen_docs = user_response.get('choice') chosen_doc_features = selector_lib.get_chosen(slate_docs, chosen_docs) # Calculate utilities. user_interests = previous_state.get('interest.state') doc_features = chosen_doc_features.get('doc_features') # User interests are increased/decreased towards the consumed document's # topic proportinal to the document quality. direction = tf.expand_dims(chosen_doc_features.get('doc_quality'), axis=-1) * (doc_features - user_interests) linear_update = self._interest_model.next_state( previous_state.get('interest'), Value(input=direction, condition=tf.less(user_response.get('consumed_time'), 0.))) # We squash the interest vector to avoid infinite blow-up using the function # 4 * M * (sigmoid(X/M) - 0.5) which is roughly linear around the origin and # softly saturates at +/-2M. These constants are not intended to be tunable. next_interest = Value( state=4.0 * self._max_user_affinity * (tf.sigmoid(linear_update.get('state') / self._max_user_affinity) - 0.5)).union(linear_update.prefixed_with('linear_update')) return next_interest.prefixed_with('interest')
def next_state(self, previous_state, user_response, slate_docs): """The state value after the initial value.""" chosen_doc_idx = user_response.get("choice") chosen_doc_features = selector_lib.get_chosen(slate_docs, chosen_doc_idx) # Update doc_id history. doc_consumed = tf.reshape(chosen_doc_features.get("doc_id"), [self._num_users]) # We update histories of only users who chose a doc. no_choice = tf.equal(user_response.get("choice"), self._slate_size)[Ellipsis, tf.newaxis] next_doc_id_history = self._doc_history.next_state( previous_state.get("doc_history"), Value(input=doc_consumed, condition=no_choice)).prefixed_with("doc_history") # Update consumed time. time_consumed = tf.reshape(user_response.get("consumed_time"), [self._num_users]) next_ctime_history = self._ctime_history.next_state( previous_state.get("ctime_history"), Value(input=time_consumed, condition=no_choice)).prefixed_with("ctime_history") return next_doc_id_history.union(next_ctime_history)