def test_manipulate_features_no_max_control(self):
"""Tests that features are manipulated as expected no gaming control."""
env = college_admission.CollegeAdmissionsEnv(
user_params={
"num_applicants": 6,
"gaming": True,
"gaming_control": np.inf,
"noise_params": params.BoundedGaussian(max=0, mu=0, min=0, sigma=0),
"group_cost": {0: 3, 1: 4},
}
)
agent = random_agents.RandomAgent(
env.action_space,
None,
env.observation_space,
default_action={"threshold": np.array(0.8), "epsilon_prob": np.array(0)},
)
env.set_scalar_reward(agent.reward_fn)
action = agent.initial_action()
env.step(action)
env.state.test_scores_x = [0.1, 0.3, 0.6, 0.7, 0.7, 0.9]
env.state.applicant_groups = [0, 1, 1, 1, 0, 0]
env.state.true_eligible = [0, 0, 1, 1, 0, 1]
expected_changed_scores = [0.1, 0.3, 0.8, 0.8, 0.8, 0.9]
expected_individual_burden = self._return_individual_burden(env, agent)
changed_scores, individual_burden = env._manipulate_features(env.state, action)
self.assertTrue(np.all(np.isclose(expected_changed_scores, changed_scores, atol=1e-4)))
self.assertTrue(
np.all(np.isclose(individual_burden, expected_individual_burden, atol=1e-4))
)
def test_agent_returns_correct_threshold(self):
env = college_admission.CollegeAdmissionsEnv(
user_params={
'gaming':
False,
'subsidize':
False,
'noise_params':
params.BoundedGaussian(max=0.3, min=0, sigma=0, mu=0.1),
'feature_params':
params.GMM(
mix_weight=[0.5, 0.5], mu=[0.5, 0.5], sigma=[0.1, 0.1])
})
agent = college_admission_jury.NaiveJury(
action_space=env.action_space,
observation_space=env.observation_space,
reward_fn=(lambda x: 0),
threshold=0,
burnin=9,
freeze_classifier_after_burnin=True)
test_util.run_test_simulation(env=env,
agent=agent,
num_steps=10,
stackelberg=True)
learned_threshold = env.history[-1].action['threshold']
self.assertTrue(np.isclose(learned_threshold, 0.55, atol=1e-2))
def error_raised_when_noise_params_wrong(self):
env = college_admission.CollegeAdmissionsEnv(
user_params={
'noise_params':
params.BoundedGaussian(min=0, max=0.3, mu=0, sigma=0.00001),
})
with self.assertRaises(ValueError):
env._add_noise()
def test_one_sided_noise_generated_correctly(self):
env = college_admission.CollegeAdmissionsEnv(
user_params={
"num_applicants": 4,
"noise_params": params.BoundedGaussian(min=0, max=0.3, mu=0.2, sigma=0.00001),
}
)
noise = env._add_noise(env.state.rng)
self.assertTrue(np.isclose(0.2, noise, atol=1e-3))
def feature_noise_propagates_to_labels(self):
env = college_admission.CollegeAdmissionsEnv(
user_params={
"num_applicants": 10,
"noise_params": params.BoundedGaussian(min=0.5, max=0.5, mu=0, sigma=1),
}
)
env.state.rng = np.random.RandomState(seed=100)
env._sample_next_state_vars(env.state)
scores = np.array(env.state.test_scores_x)
eligible = np.array(env.state.true_eligible)
# Check that at least one "eligible" candidate has a lower score than an
# ineligible one.
self.assertLess(np.min(scores[eligible == 1]), np.max(scores[eligible == 0]))
class Params(core.Params):
"""College Admissions Params."""
# Number of applicants that appear each round.
num_applicants = attr.ib(default=20) # type: int
# Min and Max values of the test score for each applicant.
score_params = attr.ib(default=params.MinMax(max=1,
min=0)) # type: params.MinMax
# Stopping params.
# Max steps to run the interaction for.
max_steps = attr.ib(default=100000) # type: int
# Tunable Params.
# Parameters of the population feature distribution.
feature_params = attr.ib(
default=params.GMM(
mix_weight=[0.6, 0.4], mu=[0.8, 0.6],
sigma=[0.1, 0.3])) # type: params.GMM
# To calculate true thresholds, as mu + factor * mu (factor can be -ve).
true_threshold_factor = attr.ib(default=0.1) # type: float
# Params for the noise ditribution for noise added to threshold.
noise_params = attr.ib(
default=params.BoundedGaussian(
max=0.3, min=-0.3, mu=0.1,
sigma=0.01)) # type: params.BoundedGaussian
# Scalar multiplier for cost (of a feature) per group.
# Note: We codify B's disadvantage by requiring Cost A < Cost B.
# pylint: disable=g-long-lambda
group_cost = attr.ib(factory=lambda: {
0: 0.8,
1: 1.2
}) # type: Mapping[int, float]
# pylint: enable=g-long-lambda
gaming = attr.ib(default=True) # type: bool
# Control amount of gaming. If inf, then allow maximum allowable by cost fn.
# else if a value in [0, 1], applicant can only change by that many points.
gaming_control = attr.ib(default=np.inf) # type: float
# Whether to subsidize
subsidize = attr.ib(default=False) # type: bool
# Group 2 is subsidized by only allowing it to bear subsidy_beta cost.
# subsidy_beta can be a value in [0, 1]. Percentage of cost B actually pays,
# higher this is higher is the cost paid by B.
subsidy_beta = attr.ib(default=0.8) # type: float
# Distribution of noise. One of beta or gaussian.
noise_dist = attr.ib(default='gaussian') # type: Text
# Add noise to unmanipulated features.
noisy_features = attr.ib(default=False) # type: bool
# Add noise to observed threshold.
noisy_threshold = attr.ib(default=False) # type: bool
import json
from absl.testing import absltest
import params
from experiments import college_admission
import numpy as np
TEST_PARAMS = {
'num_applicants':
20,
'feature_params':
params.GMM(mix_weight=[0.5, 0.5], mu=[0.5, 0.5], sigma=[0.1, 0.1]),
'noise_params':
params.BoundedGaussian(max=0.0, min=-0, mu=0, sigma=0),
# Scalar multiplier for cost (of a feature) per group.
# Note: We codify B's disadvantage by requiring Cost A < Cost B.
'group_cost': {
0: 5,
1: 10
},
'gaming':
True,
'gaming_control':
np.inf
}
class CollegeAdmissionTest(absltest.TestCase):
def test_experiment_runs_with_default_parameters(self):
def run_noisy_experiment(noise_dist='gaussian',
noisy_features=False,
noisy_threshold=False,
feature_mu=None,
stdevs=None):
"""Noisy experiment runs."""
results = {}
deltas = {}
agent_types = ['fixed', 'static', 'continuous', 'robust']
thresholds = np.arange(0, 1, THRESHOLD_SPACING)
if noise_dist == 'beta':
logging.info('Using Beta Noise Distribution.')
stdevs = np.arange(2, 9, 1)
mu = 2
max_value = 0.7
min_value = 0
else:
logging.info('Using Gaussian Noise Distribution.')
mu = 0
max_value = 0.35
min_value = -0.35
if feature_mu is None:
feature_mu = [0.5, 0.5]
if len(feature_mu) != 2:
raise ValueError('Expected feature_mu to be of length 2.')
if stdevs is None:
stdevs = STDEV_RANGE_DEFAULTS
for sd in stdevs:
env_config_params = copy.deepcopy(ENV_PARAMS)
env_config_params.update({
'noise_dist':
noise_dist,
'noise_params':
params.BoundedGaussian(
max=max_value, min=min_value, mu=mu, sigma=sd),
'noisy_features':
noisy_features,
'noisy_threshold':
noisy_threshold,
'feature_params':
params.GMM(mix_weight=[0.5, 0.5], mu=feature_mu, sigma=[0.1, 0.1]),
})
logging.info('Stdev %f', sd)
results[sd] = {}
for agent_type in agent_types:
results[sd][agent_type] = {}
for threshold in thresholds:
results[sd][agent_type][threshold] = {}
if agent_type != 'fixed' and threshold > 0:
continue
num_steps = FIXED_AGENT_NUMSTEPS if agent_type == 'fixed' else FLAGS.num_steps
college_experiment = college_admission.CollegeExperiment(
num_steps=num_steps,
env_config=env_config_params,
agent_type=agent_type,
agent_threshold=threshold,
burnin=FLAGS.burnin,
epsilon_greedy=FLAGS.epsilon_greedy,
initial_epsilon_prob=FLAGS.initial_epsilon_prob)
json_dump = college_experiment.run_experiment()
exp_result = json.loads(json_dump)
exp_params = copy.deepcopy(attr.asdict(college_experiment))
exp_result.update({'exp_params': exp_params})
if FLAGS.verbose:
log_results(exp_result)
with open(
os.path.join(FLAGS.output_dir, 'experiment_results.json'),
'w+') as f:
core.to_json(exp_result, f)
f.write('\n---------------------------------------\n')
results[sd][agent_type][threshold] = exp_result
deltas[sd] = (
results[sd]['continuous'][0.0]['metric_results']['final_threshold'] -
results[sd]['static'][0.0]['metric_results']['final_threshold'])
return results, thresholds, deltas, stdevs
