def __init__(self,
user_sampler,
seed=None,
slate_size=1,
affinity_update_delta=1.0,
topic_affinity_update_threshold=3.0):
"""Defines the dynamic user model.
Args:
user_sampler: Object of Class UserSampler.
seed: Random seed for the user model.
slate_size: Maximum number of items that can be presented to the user.
affinity_update_delta: Delta for updating user's preference for genres
whose movies are rated >= topic_affinity_update_threshold.
topic_affinity_update_threshold: Rating threshold above which user's
preferences for the genre's is updated.
"""
super().__init__(
slate_size=slate_size,
user_sampler=user_sampler,
response_model_ctor=Response)
self._response_model_ctor = Response
self.affinity_update_delta = affinity_update_delta
self.topic_affinity_update_threshold = topic_affinity_update_threshold
self._rng = np.random.RandomState(seed)
self.choice_model = recsim_choice.MultinomialLogitChoiceModel(
choice_features={'no_click_mass': -float('Inf')})
def _choice_model_ctor():
# env_config['choice_feature'] decides the mass of no-click option when the
# user chooses one document from the recommended slate. It is a dictionary
# with key='no_click_mass' and value=logit of no-click. If empty,
# the MultinomialLogitChoice Model sets the logit of no-click to be -inf,
# and thus the user has to click one document from the recommended slate.
return choice_model.MultinomialLogitChoiceModel(
env_config['choice_features'])
def test_multinomial_logit_default(self):
mnl_model = choice_model.MultinomialLogitChoiceModel(choice_features={})
mnl_model.score_documents(
self._user_state, np.array([[0.8, 0.6], [0.6, 0.8]]))
# The logits for two documents are 1 and 0.96 respectively. When computing
# softmax logits, we subtract the largest value, which is 1 here. So the
# score is softmax([0, -0.04]) = [0.51, 0.49]
self.assertAlmostEqual(mnl_model._scores[0], 0.510, delta=0.001)
self.assertAlmostEqual(mnl_model._scores[1], 0.490, delta=0.001)
self.assertEqual(mnl_model._score_no_click, 0)
def test_multinomial_logit_no_click_mass(self):
choice_features = dict(no_click_mass=1.0)
mnl_model = choice_model.MultinomialLogitChoiceModel(
choice_features=choice_features)
# The logits for two documents are 1 and 0.96 respectively. No click mass
# is 1.0. When computing
# softmax logits, we subtract the largest value, which is 1 here. So the
# score is softmax([0, -0.04, 0]) = [0.337, 0.325, 0.338]
mnl_model.score_documents(self._user_state,
np.array([[0.8, 0.6], [0.6, 0.8]]))
self.assertAlmostEqual(mnl_model._scores[0], 0.338, delta=0.001)
self.assertAlmostEqual(mnl_model._scores[1], 0.325, delta=0.001)
self.assertAlmostEqual(mnl_model._score_no_click, 0.338, delta=0.001)
def __init__(self,
action_space,
belief_state,
choice_model=cm.MultinomialLogitChoiceModel({'no_click_mass': 5
})):
"""Initializes a new greedy pCTR agent.
Args:
action_space: A gym.spaces object that specifies the format of actions
belief_state: An instantiation of AbstractUserState assumed by the agent
choice_model: An instantiation of AbstractChoiceModel assumed by the agent
Default to a multinomial logit choice model with no_click_mass = 5.
"""
super(GreedyPCTRAgent, self).__init__(action_space)
self._choice_model = choice_model
self._belief_state = belief_state
def __init__(
self,
slate_size,
user_sampler,
response_model_ctor,
choice_model_ctor=lambda: choice_model.MultinomialLogitChoiceModel({})
):
"""Initializes a UserModel.
Args:
slate_size: Number of items that the agent suggests.
user_sampler: A UserSampler responsible for providing new users every time
reset is called.
response_model_ctor: A response_model class that generates user response
to recommendations.
choice_model_ctor: A function that returns a ChoiceModel that will
determine which doc in the slate the user interacts with.
"""
super(UserModel, self).__init__(
slate_size=slate_size,
user_sampler=user_sampler,
response_model_ctor=response_model_ctor,
)
self.choice_model = choice_model_ctor()