def _collect_stats(args):
"""
Function that is executed in a separate process.
:param args: arguments of the process to be executed.
:return: a vector of CTR for these confidence values:
0th: Q0.500
1st: Q0.025
snd: Q0.975
"""
start = time.time()
print(f"START: Num of Users: {args['num_offline_users']}")
stats = recogym.test_agent(
deepcopy(args['env']),
deepcopy(args['agent']),
args['num_offline_users'],
args['num_online_users'],
args['num_organic_offline_users'],
args['num_epochs'],
args['epoch_with_random_reset'],
args['with_cache'],
)
print(
f"END: Num of Offline Users: {args['num_offline_users']} ({time.time() - start}s)"
)
return stats
logs_a = c_env.generate_logs(users)
c_env = deepcopy(env)
logs_b = c_env.generate_logs(users)
return np.mean(
logs_a[~np.isnan(logs_b.v)].v == logs_b[~np.isnan(logs_b.v)].v) == 1.
#return logs_a.equals(logs_b) # this can return false.. it isn't clear why atm ... mostl likely types differ..
if __name__ == "__main__":
for env_name in env_test.keys():
env = gym.make(env_name)
env.init_gym(env_test[env_name])
if is_env_deterministic(env):
print(f"{env_name} appears deterministic")
else:
print(f"{env_name} is NOT deterministic")
for agent_name in agent_test.keys():
agent = agent_test[agent_name]
a = test_agent(deepcopy(env), deepcopy(agent), samples, eval_size,
organic_size)
print(f"{agent_name} runs")
b = test_agent(deepcopy(env), deepcopy(agent), samples, eval_size,
organic_size)
if a == b:
print(f"{agent_name} appears deterministic")
else:
print(f"{agent_name} is NOT deterministic")
}))
simple_agent = SimpleFarmerAgent(Configuration(garden_env_1_args))
wait_agent = WaitAgent(Configuration(garden_env_1_args))
# Now we have instances of our two agents. We can use the `test_agent` method from RecoGym and compare there performance.
#
# To use `test_agent`, one must provide a copy of the current env, a copy of the agent class, the number of training users and the number of testing users.
# In[6]:
# Credible interval of the CTR median and 0.025 0.975 quantile.
agent_success = recogym.test_agent(deepcopy(env), deepcopy(agent_rand), 1000, 1000)
print(f"RandomAgent success is {agent_success}")
# In[7]:
agent_success = recogym.test_agent(deepcopy(env), deepcopy(wait_agent), 1000, 1000)
print(f"WaitAgent success is {agent_success}")
agent_success, plots = recogym.test_agent(deepcopy(env), deepcopy(simple_agent), 1000, 1000, plotting = False)