def test_experiment_runs_with_default_parameters(self):
# Tests that the experiment can run.
test_experiment = college_admission.CollegeExperiment(
agent_type='fixed', num_steps=10)
result = test_experiment.run_experiment()
# Tests that the result is a valid json string.
result = json.loads(result)
def test_short_run_final_threshold(self):
# Accuracy with static agent should be around 0.55 (true threshold)
# with TEST_PARAMS at equilibrium.
test_experiment = college_admission.CollegeExperiment(
num_steps=30,
burnin=25,
agent_type='static',
env_config=TEST_PARAMS)
result = test_experiment.run_experiment()
result = json.loads(result)
self.assertTrue(
np.isclose(result['metric_results']['final_threshold'],
0.55,
atol=1e-2))
def run_baseline_experiment(feature_mu=None):
"""Run baseline experiment without noise."""
results = {}
agent_types = ['fixed', 'static', 'continuous', 'robust']
thresholds = np.arange(0, 1, THRESHOLD_SPACING)
if feature_mu is None:
feature_mu = [0.5, 0.5]
if len(feature_mu) != 2:
raise ValueError('Expected feature_mu to be of length 2.')
env_config_params = copy.deepcopy(ENV_PARAMS)
env_config_params.update({
'feature_params':
params.GMM(mix_weight=[0.5, 0.5], mu=feature_mu,
sigma=[0.1, 0.1])
})
for agent_type in agent_types:
results[agent_type] = {}
for threshold in thresholds:
results[agent_type][threshold] = {}
if agent_type != 'fixed' and threshold > 0:
continue
num_steps = FIXED_AGENT_NUMSTEPS if agent_type == 'fixed' else FLAGS.num_steps
college_experiment = college_admission.CollegeExperiment(
num_steps=num_steps,
env_config=env_config_params,
agent_type=agent_type,
agent_threshold=threshold,
burnin=FLAGS.burnin,
epsilon_greedy=FLAGS.epsilon_greedy,
initial_epsilon_prob=FLAGS.initial_epsilon_prob)
json_dump = college_experiment.run_experiment()
exp_result = json.loads(json_dump)
exp_params = copy.deepcopy(attr.asdict(college_experiment))
exp_result.update({'exp_params': exp_params})
if FLAGS.verbose:
log_results(exp_result)
with open(
os.path.join(FLAGS.output_dir, 'experiment_results.json'), 'a+') as f:
core.to_json(exp_result, f)
f.write('\n---------------------------------------\n')
results[agent_type][threshold] = exp_result
return results, thresholds
def run_noisy_experiment(noise_dist='gaussian',
noisy_features=False,
noisy_threshold=False,
feature_mu=None,
stdevs=None):
"""Noisy experiment runs."""
results = {}
deltas = {}
agent_types = ['fixed', 'static', 'continuous', 'robust']
thresholds = np.arange(0, 1, THRESHOLD_SPACING)
if noise_dist == 'beta':
logging.info('Using Beta Noise Distribution.')
stdevs = np.arange(2, 9, 1)
mu = 2
max_value = 0.7
min_value = 0
else:
logging.info('Using Gaussian Noise Distribution.')
mu = 0
max_value = 0.35
min_value = -0.35
if feature_mu is None:
feature_mu = [0.5, 0.5]
if len(feature_mu) != 2:
raise ValueError('Expected feature_mu to be of length 2.')
if stdevs is None:
stdevs = STDEV_RANGE_DEFAULTS
for sd in stdevs:
env_config_params = copy.deepcopy(ENV_PARAMS)
env_config_params.update({
'noise_dist':
noise_dist,
'noise_params':
params.BoundedGaussian(
max=max_value, min=min_value, mu=mu, sigma=sd),
'noisy_features':
noisy_features,
'noisy_threshold':
noisy_threshold,
'feature_params':
params.GMM(mix_weight=[0.5, 0.5], mu=feature_mu, sigma=[0.1, 0.1]),
})
logging.info('Stdev %f', sd)
results[sd] = {}
for agent_type in agent_types:
results[sd][agent_type] = {}
for threshold in thresholds:
results[sd][agent_type][threshold] = {}
if agent_type != 'fixed' and threshold > 0:
continue
num_steps = FIXED_AGENT_NUMSTEPS if agent_type == 'fixed' else FLAGS.num_steps
college_experiment = college_admission.CollegeExperiment(
num_steps=num_steps,
env_config=env_config_params,
agent_type=agent_type,
agent_threshold=threshold,
burnin=FLAGS.burnin,
epsilon_greedy=FLAGS.epsilon_greedy,
initial_epsilon_prob=FLAGS.initial_epsilon_prob)
json_dump = college_experiment.run_experiment()
exp_result = json.loads(json_dump)
exp_params = copy.deepcopy(attr.asdict(college_experiment))
exp_result.update({'exp_params': exp_params})
if FLAGS.verbose:
log_results(exp_result)
with open(
os.path.join(FLAGS.output_dir, 'experiment_results.json'),
'w+') as f:
core.to_json(exp_result, f)
f.write('\n---------------------------------------\n')
results[sd][agent_type][threshold] = exp_result
deltas[sd] = (
results[sd]['continuous'][0.0]['metric_results']['final_threshold'] -
results[sd]['static'][0.0]['metric_results']['final_threshold'])
return results, thresholds, deltas, stdevs