def test_oracle(tmp_dir):
hps = hps_module.HyperParameters()
hps.Choice('iyo_koiyo', values=[1, 2, 3, 4, 5, 6], ordered=False)
oracle_tst = OracleTest(objective=['mse', 'auc_roc_score'],
max_trials=50,
hyperparameters=hps)
assert oracle_tst.objective == [
Objective(name='mse', direction='min'),
Objective(name='auc_roc_score', direction='min')
]
trial1 = oracle_tst.create_trial(tuner_id='114514')
trial2 = oracle_tst.create_trial(tuner_id='114514')
oracle_tst.set_project_dir(directory=tmp_dir,
project_name='test',
overwrite=False)
oracle_tst.save()
assert os.path.exists(os.path.join(tmp_dir,
oracle_tst._get_oracle_fname()))
oracle_tst._save_trial(trial1)
oracle_tst._save_trial(trial2)
assert os.path.exists(
os.path.join(oracle_tst._project_dir, f'trial_{trial1.trial_id}'))
assert os.path.exists(
os.path.join(oracle_tst._project_dir, f'trial_{trial2.trial_id}'))
oracle_tst.reload()
assert all(_id in oracle_tst.trials
for _id in [trial1.trial_id, trial2.trial_id])
def test_RandomSelectInteraction(self):
# Step 1: Test constructor and get_state
tf.keras.backend.reset_uids(
) # prevent get_state() from getting uid based on the interactor's previous calls
p = {}
interactor = RandomSelectInteraction(**p)
ans_state = interactor.get_state()
sol_state = {
'name': 'random_select_interaction_1',
}
assert ans_state == sol_state
# Step 2: Test set_state
p = {}
interactor.set_state(p)
ans_state = interactor.get_state()
sol_state = {
'name': 'random_select_interaction_1',
}
assert ans_state == sol_state
# Step 3: Test build and associated functions
reps = 100 # Arrange
sol = [tf.constant([[1, 2, 3]], dtype='float32')
] * reps # extra list compensates for batch dimension
ans = list()
hp = hp_module.HyperParameters()
interactor = RandomSelectInteraction()
for _ in range(reps):
set_seed()
ans.append(interactor.build(hp, self.inputs)) # Act
assert all([all(tf.equal(a, s)[0]) for a, s in zip(ans, sol)
]) # Assert: [0] unwraps the batch dimension
def test_ConcatenateInteraction(self):
# Step 1: Test constructor and get_state
tf.keras.backend.reset_uids(
) # prevent get_state() from getting uid based on the interactor's previous calls
p = {}
interactor = ConcatenateInteraction(**p)
ans_state = interactor.get_state()
sol_state = {
'name': 'concatenate_interaction_1',
}
assert ans_state == sol_state
# Step 2: Test set_state
p = {}
interactor.set_state(p)
ans_state = interactor.get_state()
sol_state = {
'name': 'concatenate_interaction_1',
}
assert ans_state == sol_state
# Step 3: Test build and associated functions
sol = tf.constant([[1, 2, 3, 4, 5, 6]], dtype='float32') # Arrange
hp = hp_module.HyperParameters()
interactor = ConcatenateInteraction()
ans = interactor.build(hp, self.inputs) # Act
assert all(tf.equal(ans, sol)[0]) # Assert
def test_DenseFeatureMapper(self):
# test constructor and get_state
p = {'num_of_fields': 10, 'embedding_dim': 4}
mapper = DenseFeatureMapper(**p)
sol_get_state = {
'name': 'dense_feature_mapper_1',
'num_of_fields': 10,
'embedding_dim': 4
}
assert mapper.get_state() == sol_get_state
# test set_state
p = {
'num_of_fields': self.input_shape,
'embedding_dim': self.embed_dim
}
sol_set_state = {
'name': 'dense_feature_mapper_1',
'num_of_fields': self.input_shape,
'embedding_dim': self.embed_dim
}
mapper.set_state(p)
ans_set_state = mapper.get_state()
assert ans_set_state == sol_set_state
# test build
hp = hp_module.HyperParameters()
output = mapper.build(hp, self.tensor_inputs) # Act
assert len(nest.flatten(output)) == 1
assert output.shape == (self.batch, self.input_shape, self.embed_dim)
def test_name_scope():
hp = hp_module.HyperParameters()
hp.Choice('choice', [1, 2, 3], default=2)
with hp.name_scope('scope1'):
hp.Choice('choice', [4, 5, 6], default=5)
with hp.name_scope('scope2'):
hp.Choice('choice', [7, 8, 9], default=8)
hp.Int('range', min_value=0, max_value=10, default=0)
assert hp.values == {
'choice': 2,
'scope1/choice': 5,
'scope1/scope2/choice': 8,
'scope1/range': 0
}
assert hp.get_value_in_nested_format() == {
'choice': 2,
'scope1': {
'choice': 5,
'scope2': {
'choice': 8
},
'range': 0,
},
}
def test_LatentFactorMapper(self):
# test constructor and get_state
p = {'column_id': 0, 'num_of_entities': 3, 'embedding_dim': 4}
mapper = LatentFactorMapper(**p)
sol_get_state = {
'name': 'latent_factor_mapper_1',
'column_id': 0,
'num_of_entities': 3,
'embedding_dim': 4
}
assert mapper.get_state() == sol_get_state
# test set_state
p = {
'column_id': self.column_id,
'num_of_entities': 10,
'embedding_dim': self.embed_dim
}
sol_set_state = {
'name': 'latent_factor_mapper_1',
'column_id': self.column_id,
'num_of_entities': 10,
'embedding_dim': self.embed_dim
}
mapper.set_state(p)
ans_set_state = mapper.get_state()
assert ans_set_state == sol_set_state
# test build
hp = hp_module.HyperParameters()
output = mapper.build(hp, self.tensor_inputs)
assert len(nest.flatten(output)) == 1
assert output.shape == (
self.batch, self.embed_dim
) # LatentFactorMapper does not have input shape dimension
def __init__(self,
objective,
max_trials=None,
hyperparameters=None,
allow_new_entries=True,
tune_new_entries=True):
self.objective = _format_objective(objective)
self.max_trials = max_trials
if not hyperparameters:
if not tune_new_entries:
raise ValueError(
'If you set `tune_new_entries=False`, you must'
'specify the search space via the '
'`hyperparameters` argument.')
if not allow_new_entries:
raise ValueError(
'If you set `allow_new_entries=False`, you must'
'specify the search space via the '
'`hyperparameters` argument.')
self.hyperparameters = hp_module.HyperParameters()
else:
self.hyperparameters = hyperparameters
self.allow_new_entries = allow_new_entries
self.tune_new_entries = tune_new_entries
# trial_id -> Trial
self.trials = {}
# tuner_id -> Trial
self.ongoing_trials = {}
# Set in `BaseTuner` via `set_project_dir`.
self._directory = None
self._project_name = None
def test_nested_conditional_scopes_and_name_scopes():
hp = hp_module.HyperParameters()
a = hp.Choice('a', [1, 2, 3], default=2)
with hp.conditional_scope('a', [1, 3]):
b = hp.Choice('b', [4, 5, 6])
with hp.conditional_scope('b', 6):
c = hp.Choice('c', [7, 8, 9])
with hp.name_scope('d'):
e = hp.Choice('e', [10, 11, 12])
with hp.conditional_scope('a', 2):
f = hp.Choice('f', [13, 14, 15])
assert hp.values == {
'a': 2,
'a=1,3/b': 4,
'a=1,3/b=6/c': 7,
'a=1,3/b=6/d/e': 10,
'a=2/f': 13
}
# Assignment to an active conditional hyperparameter returns the value.
assert a == 2
assert f == 13
# Assignment to a non-active conditional hyperparameter returns `None`.
assert b is None
assert c is None
assert e is None
def test_parent_name():
hp = hp_module.HyperParameters()
hp.Choice('a', [1, 2, 3], default=2)
b1 = hp.Int('b', 0, 10, parent_name='a', parent_values=1, default=5)
b2 = hp.Int('b', 0, 100, parent_name='a', parent_values=2, default=4)
assert b1 is None
assert b2 == 4
assert hp.values == {'a': 2, 'a=1/b': 5, 'a=2/b': 4}
def test_ElementwiseInteraction(self):
# Step 1: Test constructor and get_state
tf.keras.backend.reset_uids(
) # prevent get_state() from getting uid based on the interactor's previous calls
p = {
'elementwise_type': 'average',
}
interactor = ElementwiseInteraction(**p)
ans_state = interactor.get_state()
sol_state = {
'name': 'elementwise_interaction_1',
'elementwise_type': 'average',
}
assert ans_state == sol_state
# Step 2: Test set_state
p = {
'elementwise_type': 'multiply',
}
interactor.set_state(p)
ans_state = interactor.get_state()
sol_state = {
'name': 'elementwise_interaction_1',
'elementwise_type': 'multiply',
}
assert ans_state == sol_state
# Step 3: Test build and associated functions
# Step 3.1: Test elementwise sum
hp = hp_module.HyperParameters() # Arrange
sol = tf.constant([[5, 7, 9]], dtype='float32')
interactor = ElementwiseInteraction('sum') # Arrange
ans = interactor.build(hp, self.inputs) # Act
assert all(tf.equal(ans, sol)[0]) # Assert
# Step 3.2: Test elementwise average
sol = tf.constant([[2.5, 3.5, 4.5]], dtype='float32') # Arrange
interactor = ElementwiseInteraction('average')
ans = interactor.build(hp, self.inputs) # Act
assert all(tf.equal(ans, sol)[0]) # Assert
# Step 3.3: Test elementwise multiply (Hadamard product)
sol = tf.constant([[4, 10, 18]], dtype='float32') # Arrange
interactor = ElementwiseInteraction('multiply')
ans = interactor.build(hp, self.inputs) # Act
assert all(tf.equal(ans, sol)[0]) # Assert
# Step 3.4: Test elementwise max
sol = tf.constant([[4, 5, 6]], dtype='float32') # Arrange
interactor = ElementwiseInteraction('max')
ans = interactor.build(hp, self.inputs) # Act
assert all(tf.equal(ans, sol)[0]) # Assert
# Step 3.5: Test elementwise min
sol = tf.constant([[1, 2, 3]], dtype='float32') # Arrange
interactor = ElementwiseInteraction('min')
ans = interactor.build(hp, self.inputs) # Act
assert all(tf.equal(ans, sol)[0]) # Assert
def test_concatenate(self):
"""
Test class ConcatenateInteraction in interactor.py
"""
sol = tf.constant([1, 2, 3, 4, 5, 6]) # Arrange
hp = hp_module.HyperParameters()
interactor = ConcatenateInteraction()
ans = interactor.build(hp, self.inputs) # Act
assert all(tf.equal(ans, sol)) # Assert
def test_elementwise_add(self):
"""
Test class ElementwiseAddInteraction in interactor.py
"""
sol = tf.constant([5, 7, 9]) # Arrange
hp = hp_module.HyperParameters()
interactor = ElementwiseAddInteraction()
ans = interactor.build(hp, self.inputs) # Act
assert all(tf.equal(ans, sol)) # Assert
def test_HyperInteraction(self):
# Step 1: Test constructor and get_state
tf.keras.backend.reset_uids(
) # prevent get_state() from getting uid based on the interactor's previous calls
p = {
'meta_interactor_num': 3,
'interactor_type': 'ConcatenateInteraction',
}
interactor = HyperInteraction(**p)
ans_state = interactor.get_state()
sol_state = {
'name': 'hyper_interaction_1',
'meta_interactor_num': 3,
'interactor_type': 'ConcatenateInteraction',
'name2interactor': {
'RandomSelectInteraction': RandomSelectInteraction,
'ConcatenateInteraction': ConcatenateInteraction,
'InnerProductInteraction': InnerProductInteraction,
'ElementwiseInteraction': ElementwiseInteraction,
'MLPInteraction': MLPInteraction,
'FMInteraction': FMInteraction,
'CrossNetInteraction': CrossNetInteraction,
'SelfAttentionInteraction': SelfAttentionInteraction,
}
}
assert ans_state == sol_state
# Step 2: Test set_state
p = {
'meta_interactor_num': 6,
'interactor_type': 'MLPInteraction',
}
interactor.set_state(p)
ans_state = interactor.get_state()
sol_state = {
'name': 'hyper_interaction_1',
'meta_interactor_num': 6,
'interactor_type': 'MLPInteraction',
'name2interactor': {
'RandomSelectInteraction': RandomSelectInteraction,
'ConcatenateInteraction': ConcatenateInteraction,
'InnerProductInteraction': InnerProductInteraction,
'ElementwiseInteraction': ElementwiseInteraction,
'MLPInteraction': MLPInteraction,
'FMInteraction': FMInteraction,
'CrossNetInteraction': CrossNetInteraction,
'SelfAttentionInteraction': SelfAttentionInteraction,
}
}
assert ans_state == sol_state
# Step 3: Test build and associated functions
hp = hp_module.HyperParameters()
ans = interactor.build(hp, self.inputs) # Act
sol = 1
assert len(tf.nest.flatten(ans)) == sol
def test_graph_basics():
input_node = Input(shape=(30, ))
output_node = input_node
output_node = MLPInteraction()(output_node)
output_node = RatingPredictionOptimizer()(output_node)
graph = graph_module.PlainGraph(input_node, output_node)
model = graph.build_keras_graph().build(hp_module.HyperParameters())
assert model.input_shape == (None, 30)
assert model.output_shape == (None, )
def get_hyperparameters(self):
"""Get the tunable hyperperparameters from all the blocks in this pipeline."""
hps = hp_module.HyperParameters()
for block in self._blocks:
params_dict = block.hyperparameters
if params_dict:
with hps.name_scope(block.name):
for param_name, single_hp in params_dict.items():
hps.register(param_name, single_hp.__class__.__name__,
single_hp.get_config())
return hps
def test_merge():
hp = hp_module.HyperParameters()
hp.Int('a', 0, 100)
hp.Float('b', min_value=0.5, max_value=9.5, default=2)
hp2 = hp_module.HyperParameters()
hp2.Int('a', 3, 4, default=3)
hp.Int('c', 10, 100, default=30)
hp.merge(hp2)
assert hp.get('a') == 3
assert hp.get('b') == 2
assert hp.get('c') == 30
hp3 = hp_module.HyperParameters()
hp3.Float('a', 3.5, 4.5)
hp3.Choice('d', [1, 2, 3], default=1)
hp.merge(hp3, overwrite=False)
assert hp.get('a') == 3
assert hp.get('b') == 2
assert hp.get('c') == 30
assert hp.get('d') == 1
def test_trial():
hps = hps_module.HyperParameters()
hps.Int('a', 0, 10, default=3)
trial = trial_module.Trial(hps, trial_id='trial1', status='COMPLETED')
trial.metrics.register('score', direction='max')
trial.metrics.update('score', 10, step=1)
assert len(trial.hyperparameters.space) == 1
_trail = trial_module.Trial.from_state(trial.get_state())
assert _trail.hyperparameters.get('a') == 3
assert _trail.trial_id == 'trial1'
assert _trail.score is None
assert _trail.best_step is None
assert _trail.metrics.get_best_value('score') == 10
assert _trail.metrics.get_history('score') == [
metric.MetricObservation(10, step=1)
]
def test_conditional_scope():
hp = hp_module.HyperParameters()
hp.Choice('choice', [1, 2, 3], default=2)
with hp.conditional_scope('choice', [1, 3]):
child1 = hp.Choice('child_choice', [4, 5, 6])
with hp.conditional_scope('choice', 2):
child2 = hp.Choice('child_choice', [7, 8, 9])
assert hp.values == {
'choice': 2,
'choice=1,3/child_choice': 4,
'choice=2/child_choice': 7
}
# Assignment to a non-active conditional hyperparameter returns `None`.
assert child1 is None
# Assignment to an active conditional hyperparameter returns the value.
assert child2 == 7
def test_hyperparameters():
hp = hp_module.HyperParameters()
assert hp.values == {}
assert hp.space == []
hp.Choice('choice', [1, 2, 3], default=2)
assert hp.values == {'choice': 2}
assert len(hp.space) == 1
assert hp.space[0].name == 'choice'
hp.values['choice'] = 3
assert hp.get('choice') == 3
hp = hp.copy()
assert hp.values == {'choice': 3}
assert len(hp.space) == 1
assert hp.space[0].name == 'choice'
with pytest.raises(ValueError, match='Unknown parameter'):
hp.get('wrong')
def test_MLPInteraction(self):
# initialize
tf.keras.backend.reset_uids(
) # prevent get_state() from getting uid based on the interactor's previous calls
hp = hp_module.HyperParameters()
p = {
'units': 16,
'num_layers': 2,
'use_batchnorm': False,
'dropout_rate': 0.25
} # units, num_layer, use_batchnorm, dropout
interactor = MLPInteraction(**p)
# test get_state()
sol_get_state = {
'name': 'mlp_interaction_1',
'units': 16,
'num_layers': 2,
'use_batchnorm': False,
'dropout_rate': 0.25
}
assert interactor.get_state() == sol_get_state
# test set_state()
p = {
'units': 32,
'num_layers': 1,
'use_batchnorm': True,
'dropout_rate': 0.0
}
sol_set_state = {
'name': 'mlp_interaction_1',
'units': 32,
'num_layers': 1,
'use_batchnorm': True,
'dropout_rate': 0.0
}
interactor.set_state(p)
ans_set_state = interactor.get_state()
assert ans_set_state == sol_set_state
# output shape
output = interactor.build(hp, self.inputs) # Act
assert len(tf.nest.flatten(output)) == 1
def test_SelfAttentionInteraction(self):
# Step 1: Test constructor and get_state
tf.keras.backend.reset_uids(
) # prevent get_state() from getting uid based on the interactor's previous calls
p = {
'embedding_dim': 8,
'att_embedding_dim': 8,
'head_num': 2,
'residual': True,
}
interactor = SelfAttentionInteraction(**p)
ans_state = interactor.get_state()
sol_state = {
'name': 'self_attention_interaction_1',
'embedding_dim': 8,
'att_embedding_dim': 8,
'head_num': 2,
'residual': True,
}
assert ans_state == sol_state
# Step 2: Test set_state
p = {
'embedding_dim': 16,
'att_embedding_dim': 16,
'head_num': 4,
'residual': False,
}
interactor.set_state(p)
ans_state = interactor.get_state()
sol_state = {
'name': 'self_attention_interaction_1',
'embedding_dim': 16,
'att_embedding_dim': 16,
'head_num': 4,
'residual': False,
}
assert ans_state == sol_state
# Step 3: Test build and associated functions
hp = hp_module.HyperParameters()
ans = interactor.build(hp, self.inputs) # Act
sol = 1
assert len(tf.nest.flatten(ans)) == sol
def test_CrossNetInteraction(self):
tf.keras.backend.reset_uids(
) # prevent get_state() from getting uid based on the interactor's previous calls
hp = hp_module.HyperParameters()
p = {'layer_num': 1} # embedding_dim
interactor = CrossNetInteraction(**p)
# test get_state()
sol_get_state = {'name': 'cross_net_interaction_1', 'layer_num': 1}
assert interactor.get_state() == sol_get_state
# test set_state()
p = {'layer_num': 2}
sol_set_state = {'name': 'cross_net_interaction_1', 'layer_num': 2}
interactor.set_state(p)
ans_set_state = interactor.get_state()
assert ans_set_state == sol_set_state
# output shape
output = interactor.build(hp, self.inputs) # Act
assert len(tf.nest.flatten(output)) == 1
def test_SparseFeatureMapper(self):
# test constructor and get_state
p = {'num_of_fields': 10, 'hash_size': [2, 4, 10], 'embedding_dim': 4}
mapper = SparseFeatureMapper(**p)
sol_get_state = {
'name': 'sparse_feature_mapper_1',
'num_of_fields': 10,
'hash_size': [2, 4, 10],
'embedding_dim': 4
}
assert mapper.get_state() == sol_get_state
# test set_state
hash_size = self.df_inputs.nunique().tolist()
p = {
'num_of_fields': self.input_shape,
'hash_size': hash_size,
'embedding_dim': self.embed_dim
}
sol_set_state = {
'name': 'sparse_feature_mapper_1',
'num_of_fields': self.input_shape,
'hash_size': hash_size,
'embedding_dim': self.embed_dim
}
mapper.set_state(p)
ans_set_state = mapper.get_state()
assert ans_set_state == sol_set_state
# test build
hp = hp_module.HyperParameters()
tensor_inputs = [tf.convert_to_tensor(self.df_inputs.values)]
mapper = SparseFeatureMapper(**p)
output = mapper.build(hp, tensor_inputs) # Act
assert len(nest.flatten(output)) == 1
assert output.shape == (self.batch, self.input_shape, self.embed_dim)
def test_RatingPredictionOptimizer(self):
hp = hp_module.HyperParameters()
optimizer = RatingPredictionOptimizer()
output = optimizer.build(hp, self.inputs)
assert len(nest.flatten(output)) == 1
assert output.shape == self.batch
def test_CTRPredictionOptimizer(self):
hp = hp_module.HyperParameters() # Arrange
optimizer = CTRPredictionOptimizer()
output = optimizer.build(hp, self.inputs) # Act
assert len(tf.nest.flatten(output)) == 1 # Assert
assert output.shape == (self.batch, 1)
def test_get_with_conditional_scopes():
hp = hp_module.HyperParameters()
hp.Choice('a', [1, 2, 3], default=2)
assert hp.get('a') == 2
with hp.conditional_scope('a', 2):
assert hp.get('a') == 2
