def test_oracle(tmp_dir): hps = hps_module.HyperParameters() hps.Choice('iyo_koiyo', values=[1, 2, 3, 4, 5, 6], ordered=False) oracle_tst = OracleTest(objective=['mse', 'auc_roc_score'], max_trials=50, hyperparameters=hps) assert oracle_tst.objective == [ Objective(name='mse', direction='min'), Objective(name='auc_roc_score', direction='min') ] trial1 = oracle_tst.create_trial(tuner_id='114514') trial2 = oracle_tst.create_trial(tuner_id='114514') oracle_tst.set_project_dir(directory=tmp_dir, project_name='test', overwrite=False) oracle_tst.save() assert os.path.exists(os.path.join(tmp_dir, oracle_tst._get_oracle_fname())) oracle_tst._save_trial(trial1) oracle_tst._save_trial(trial2) assert os.path.exists( os.path.join(oracle_tst._project_dir, f'trial_{trial1.trial_id}')) assert os.path.exists( os.path.join(oracle_tst._project_dir, f'trial_{trial2.trial_id}')) oracle_tst.reload() assert all(_id in oracle_tst.trials for _id in [trial1.trial_id, trial2.trial_id])
def test_RandomSelectInteraction(self): # Step 1: Test constructor and get_state tf.keras.backend.reset_uids( ) # prevent get_state() from getting uid based on the interactor's previous calls p = {} interactor = RandomSelectInteraction(**p) ans_state = interactor.get_state() sol_state = { 'name': 'random_select_interaction_1', } assert ans_state == sol_state # Step 2: Test set_state p = {} interactor.set_state(p) ans_state = interactor.get_state() sol_state = { 'name': 'random_select_interaction_1', } assert ans_state == sol_state # Step 3: Test build and associated functions reps = 100 # Arrange sol = [tf.constant([[1, 2, 3]], dtype='float32') ] * reps # extra list compensates for batch dimension ans = list() hp = hp_module.HyperParameters() interactor = RandomSelectInteraction() for _ in range(reps): set_seed() ans.append(interactor.build(hp, self.inputs)) # Act assert all([all(tf.equal(a, s)[0]) for a, s in zip(ans, sol) ]) # Assert: [0] unwraps the batch dimension
def test_ConcatenateInteraction(self): # Step 1: Test constructor and get_state tf.keras.backend.reset_uids( ) # prevent get_state() from getting uid based on the interactor's previous calls p = {} interactor = ConcatenateInteraction(**p) ans_state = interactor.get_state() sol_state = { 'name': 'concatenate_interaction_1', } assert ans_state == sol_state # Step 2: Test set_state p = {} interactor.set_state(p) ans_state = interactor.get_state() sol_state = { 'name': 'concatenate_interaction_1', } assert ans_state == sol_state # Step 3: Test build and associated functions sol = tf.constant([[1, 2, 3, 4, 5, 6]], dtype='float32') # Arrange hp = hp_module.HyperParameters() interactor = ConcatenateInteraction() ans = interactor.build(hp, self.inputs) # Act assert all(tf.equal(ans, sol)[0]) # Assert
def test_DenseFeatureMapper(self): # test constructor and get_state p = {'num_of_fields': 10, 'embedding_dim': 4} mapper = DenseFeatureMapper(**p) sol_get_state = { 'name': 'dense_feature_mapper_1', 'num_of_fields': 10, 'embedding_dim': 4 } assert mapper.get_state() == sol_get_state # test set_state p = { 'num_of_fields': self.input_shape, 'embedding_dim': self.embed_dim } sol_set_state = { 'name': 'dense_feature_mapper_1', 'num_of_fields': self.input_shape, 'embedding_dim': self.embed_dim } mapper.set_state(p) ans_set_state = mapper.get_state() assert ans_set_state == sol_set_state # test build hp = hp_module.HyperParameters() output = mapper.build(hp, self.tensor_inputs) # Act assert len(nest.flatten(output)) == 1 assert output.shape == (self.batch, self.input_shape, self.embed_dim)
def test_name_scope(): hp = hp_module.HyperParameters() hp.Choice('choice', [1, 2, 3], default=2) with hp.name_scope('scope1'): hp.Choice('choice', [4, 5, 6], default=5) with hp.name_scope('scope2'): hp.Choice('choice', [7, 8, 9], default=8) hp.Int('range', min_value=0, max_value=10, default=0) assert hp.values == { 'choice': 2, 'scope1/choice': 5, 'scope1/scope2/choice': 8, 'scope1/range': 0 } assert hp.get_value_in_nested_format() == { 'choice': 2, 'scope1': { 'choice': 5, 'scope2': { 'choice': 8 }, 'range': 0, }, }
def test_LatentFactorMapper(self): # test constructor and get_state p = {'column_id': 0, 'num_of_entities': 3, 'embedding_dim': 4} mapper = LatentFactorMapper(**p) sol_get_state = { 'name': 'latent_factor_mapper_1', 'column_id': 0, 'num_of_entities': 3, 'embedding_dim': 4 } assert mapper.get_state() == sol_get_state # test set_state p = { 'column_id': self.column_id, 'num_of_entities': 10, 'embedding_dim': self.embed_dim } sol_set_state = { 'name': 'latent_factor_mapper_1', 'column_id': self.column_id, 'num_of_entities': 10, 'embedding_dim': self.embed_dim } mapper.set_state(p) ans_set_state = mapper.get_state() assert ans_set_state == sol_set_state # test build hp = hp_module.HyperParameters() output = mapper.build(hp, self.tensor_inputs) assert len(nest.flatten(output)) == 1 assert output.shape == ( self.batch, self.embed_dim ) # LatentFactorMapper does not have input shape dimension
def __init__(self, objective, max_trials=None, hyperparameters=None, allow_new_entries=True, tune_new_entries=True): self.objective = _format_objective(objective) self.max_trials = max_trials if not hyperparameters: if not tune_new_entries: raise ValueError( 'If you set `tune_new_entries=False`, you must' 'specify the search space via the ' '`hyperparameters` argument.') if not allow_new_entries: raise ValueError( 'If you set `allow_new_entries=False`, you must' 'specify the search space via the ' '`hyperparameters` argument.') self.hyperparameters = hp_module.HyperParameters() else: self.hyperparameters = hyperparameters self.allow_new_entries = allow_new_entries self.tune_new_entries = tune_new_entries # trial_id -> Trial self.trials = {} # tuner_id -> Trial self.ongoing_trials = {} # Set in `BaseTuner` via `set_project_dir`. self._directory = None self._project_name = None
def test_nested_conditional_scopes_and_name_scopes(): hp = hp_module.HyperParameters() a = hp.Choice('a', [1, 2, 3], default=2) with hp.conditional_scope('a', [1, 3]): b = hp.Choice('b', [4, 5, 6]) with hp.conditional_scope('b', 6): c = hp.Choice('c', [7, 8, 9]) with hp.name_scope('d'): e = hp.Choice('e', [10, 11, 12]) with hp.conditional_scope('a', 2): f = hp.Choice('f', [13, 14, 15]) assert hp.values == { 'a': 2, 'a=1,3/b': 4, 'a=1,3/b=6/c': 7, 'a=1,3/b=6/d/e': 10, 'a=2/f': 13 } # Assignment to an active conditional hyperparameter returns the value. assert a == 2 assert f == 13 # Assignment to a non-active conditional hyperparameter returns `None`. assert b is None assert c is None assert e is None
def test_parent_name(): hp = hp_module.HyperParameters() hp.Choice('a', [1, 2, 3], default=2) b1 = hp.Int('b', 0, 10, parent_name='a', parent_values=1, default=5) b2 = hp.Int('b', 0, 100, parent_name='a', parent_values=2, default=4) assert b1 is None assert b2 == 4 assert hp.values == {'a': 2, 'a=1/b': 5, 'a=2/b': 4}
def test_ElementwiseInteraction(self): # Step 1: Test constructor and get_state tf.keras.backend.reset_uids( ) # prevent get_state() from getting uid based on the interactor's previous calls p = { 'elementwise_type': 'average', } interactor = ElementwiseInteraction(**p) ans_state = interactor.get_state() sol_state = { 'name': 'elementwise_interaction_1', 'elementwise_type': 'average', } assert ans_state == sol_state # Step 2: Test set_state p = { 'elementwise_type': 'multiply', } interactor.set_state(p) ans_state = interactor.get_state() sol_state = { 'name': 'elementwise_interaction_1', 'elementwise_type': 'multiply', } assert ans_state == sol_state # Step 3: Test build and associated functions # Step 3.1: Test elementwise sum hp = hp_module.HyperParameters() # Arrange sol = tf.constant([[5, 7, 9]], dtype='float32') interactor = ElementwiseInteraction('sum') # Arrange ans = interactor.build(hp, self.inputs) # Act assert all(tf.equal(ans, sol)[0]) # Assert # Step 3.2: Test elementwise average sol = tf.constant([[2.5, 3.5, 4.5]], dtype='float32') # Arrange interactor = ElementwiseInteraction('average') ans = interactor.build(hp, self.inputs) # Act assert all(tf.equal(ans, sol)[0]) # Assert # Step 3.3: Test elementwise multiply (Hadamard product) sol = tf.constant([[4, 10, 18]], dtype='float32') # Arrange interactor = ElementwiseInteraction('multiply') ans = interactor.build(hp, self.inputs) # Act assert all(tf.equal(ans, sol)[0]) # Assert # Step 3.4: Test elementwise max sol = tf.constant([[4, 5, 6]], dtype='float32') # Arrange interactor = ElementwiseInteraction('max') ans = interactor.build(hp, self.inputs) # Act assert all(tf.equal(ans, sol)[0]) # Assert # Step 3.5: Test elementwise min sol = tf.constant([[1, 2, 3]], dtype='float32') # Arrange interactor = ElementwiseInteraction('min') ans = interactor.build(hp, self.inputs) # Act assert all(tf.equal(ans, sol)[0]) # Assert
def test_concatenate(self): """ Test class ConcatenateInteraction in interactor.py """ sol = tf.constant([1, 2, 3, 4, 5, 6]) # Arrange hp = hp_module.HyperParameters() interactor = ConcatenateInteraction() ans = interactor.build(hp, self.inputs) # Act assert all(tf.equal(ans, sol)) # Assert
def test_elementwise_add(self): """ Test class ElementwiseAddInteraction in interactor.py """ sol = tf.constant([5, 7, 9]) # Arrange hp = hp_module.HyperParameters() interactor = ElementwiseAddInteraction() ans = interactor.build(hp, self.inputs) # Act assert all(tf.equal(ans, sol)) # Assert
def test_HyperInteraction(self): # Step 1: Test constructor and get_state tf.keras.backend.reset_uids( ) # prevent get_state() from getting uid based on the interactor's previous calls p = { 'meta_interactor_num': 3, 'interactor_type': 'ConcatenateInteraction', } interactor = HyperInteraction(**p) ans_state = interactor.get_state() sol_state = { 'name': 'hyper_interaction_1', 'meta_interactor_num': 3, 'interactor_type': 'ConcatenateInteraction', 'name2interactor': { 'RandomSelectInteraction': RandomSelectInteraction, 'ConcatenateInteraction': ConcatenateInteraction, 'InnerProductInteraction': InnerProductInteraction, 'ElementwiseInteraction': ElementwiseInteraction, 'MLPInteraction': MLPInteraction, 'FMInteraction': FMInteraction, 'CrossNetInteraction': CrossNetInteraction, 'SelfAttentionInteraction': SelfAttentionInteraction, } } assert ans_state == sol_state # Step 2: Test set_state p = { 'meta_interactor_num': 6, 'interactor_type': 'MLPInteraction', } interactor.set_state(p) ans_state = interactor.get_state() sol_state = { 'name': 'hyper_interaction_1', 'meta_interactor_num': 6, 'interactor_type': 'MLPInteraction', 'name2interactor': { 'RandomSelectInteraction': RandomSelectInteraction, 'ConcatenateInteraction': ConcatenateInteraction, 'InnerProductInteraction': InnerProductInteraction, 'ElementwiseInteraction': ElementwiseInteraction, 'MLPInteraction': MLPInteraction, 'FMInteraction': FMInteraction, 'CrossNetInteraction': CrossNetInteraction, 'SelfAttentionInteraction': SelfAttentionInteraction, } } assert ans_state == sol_state # Step 3: Test build and associated functions hp = hp_module.HyperParameters() ans = interactor.build(hp, self.inputs) # Act sol = 1 assert len(tf.nest.flatten(ans)) == sol
def test_graph_basics(): input_node = Input(shape=(30, )) output_node = input_node output_node = MLPInteraction()(output_node) output_node = RatingPredictionOptimizer()(output_node) graph = graph_module.PlainGraph(input_node, output_node) model = graph.build_keras_graph().build(hp_module.HyperParameters()) assert model.input_shape == (None, 30) assert model.output_shape == (None, )
def get_hyperparameters(self): """Get the tunable hyperperparameters from all the blocks in this pipeline.""" hps = hp_module.HyperParameters() for block in self._blocks: params_dict = block.hyperparameters if params_dict: with hps.name_scope(block.name): for param_name, single_hp in params_dict.items(): hps.register(param_name, single_hp.__class__.__name__, single_hp.get_config()) return hps
def test_merge(): hp = hp_module.HyperParameters() hp.Int('a', 0, 100) hp.Float('b', min_value=0.5, max_value=9.5, default=2) hp2 = hp_module.HyperParameters() hp2.Int('a', 3, 4, default=3) hp.Int('c', 10, 100, default=30) hp.merge(hp2) assert hp.get('a') == 3 assert hp.get('b') == 2 assert hp.get('c') == 30 hp3 = hp_module.HyperParameters() hp3.Float('a', 3.5, 4.5) hp3.Choice('d', [1, 2, 3], default=1) hp.merge(hp3, overwrite=False) assert hp.get('a') == 3 assert hp.get('b') == 2 assert hp.get('c') == 30 assert hp.get('d') == 1
def test_trial(): hps = hps_module.HyperParameters() hps.Int('a', 0, 10, default=3) trial = trial_module.Trial(hps, trial_id='trial1', status='COMPLETED') trial.metrics.register('score', direction='max') trial.metrics.update('score', 10, step=1) assert len(trial.hyperparameters.space) == 1 _trail = trial_module.Trial.from_state(trial.get_state()) assert _trail.hyperparameters.get('a') == 3 assert _trail.trial_id == 'trial1' assert _trail.score is None assert _trail.best_step is None assert _trail.metrics.get_best_value('score') == 10 assert _trail.metrics.get_history('score') == [ metric.MetricObservation(10, step=1) ]
def test_conditional_scope(): hp = hp_module.HyperParameters() hp.Choice('choice', [1, 2, 3], default=2) with hp.conditional_scope('choice', [1, 3]): child1 = hp.Choice('child_choice', [4, 5, 6]) with hp.conditional_scope('choice', 2): child2 = hp.Choice('child_choice', [7, 8, 9]) assert hp.values == { 'choice': 2, 'choice=1,3/child_choice': 4, 'choice=2/child_choice': 7 } # Assignment to a non-active conditional hyperparameter returns `None`. assert child1 is None # Assignment to an active conditional hyperparameter returns the value. assert child2 == 7
def test_hyperparameters(): hp = hp_module.HyperParameters() assert hp.values == {} assert hp.space == [] hp.Choice('choice', [1, 2, 3], default=2) assert hp.values == {'choice': 2} assert len(hp.space) == 1 assert hp.space[0].name == 'choice' hp.values['choice'] = 3 assert hp.get('choice') == 3 hp = hp.copy() assert hp.values == {'choice': 3} assert len(hp.space) == 1 assert hp.space[0].name == 'choice' with pytest.raises(ValueError, match='Unknown parameter'): hp.get('wrong')
def test_MLPInteraction(self): # initialize tf.keras.backend.reset_uids( ) # prevent get_state() from getting uid based on the interactor's previous calls hp = hp_module.HyperParameters() p = { 'units': 16, 'num_layers': 2, 'use_batchnorm': False, 'dropout_rate': 0.25 } # units, num_layer, use_batchnorm, dropout interactor = MLPInteraction(**p) # test get_state() sol_get_state = { 'name': 'mlp_interaction_1', 'units': 16, 'num_layers': 2, 'use_batchnorm': False, 'dropout_rate': 0.25 } assert interactor.get_state() == sol_get_state # test set_state() p = { 'units': 32, 'num_layers': 1, 'use_batchnorm': True, 'dropout_rate': 0.0 } sol_set_state = { 'name': 'mlp_interaction_1', 'units': 32, 'num_layers': 1, 'use_batchnorm': True, 'dropout_rate': 0.0 } interactor.set_state(p) ans_set_state = interactor.get_state() assert ans_set_state == sol_set_state # output shape output = interactor.build(hp, self.inputs) # Act assert len(tf.nest.flatten(output)) == 1
def test_SelfAttentionInteraction(self): # Step 1: Test constructor and get_state tf.keras.backend.reset_uids( ) # prevent get_state() from getting uid based on the interactor's previous calls p = { 'embedding_dim': 8, 'att_embedding_dim': 8, 'head_num': 2, 'residual': True, } interactor = SelfAttentionInteraction(**p) ans_state = interactor.get_state() sol_state = { 'name': 'self_attention_interaction_1', 'embedding_dim': 8, 'att_embedding_dim': 8, 'head_num': 2, 'residual': True, } assert ans_state == sol_state # Step 2: Test set_state p = { 'embedding_dim': 16, 'att_embedding_dim': 16, 'head_num': 4, 'residual': False, } interactor.set_state(p) ans_state = interactor.get_state() sol_state = { 'name': 'self_attention_interaction_1', 'embedding_dim': 16, 'att_embedding_dim': 16, 'head_num': 4, 'residual': False, } assert ans_state == sol_state # Step 3: Test build and associated functions hp = hp_module.HyperParameters() ans = interactor.build(hp, self.inputs) # Act sol = 1 assert len(tf.nest.flatten(ans)) == sol
def test_CrossNetInteraction(self): tf.keras.backend.reset_uids( ) # prevent get_state() from getting uid based on the interactor's previous calls hp = hp_module.HyperParameters() p = {'layer_num': 1} # embedding_dim interactor = CrossNetInteraction(**p) # test get_state() sol_get_state = {'name': 'cross_net_interaction_1', 'layer_num': 1} assert interactor.get_state() == sol_get_state # test set_state() p = {'layer_num': 2} sol_set_state = {'name': 'cross_net_interaction_1', 'layer_num': 2} interactor.set_state(p) ans_set_state = interactor.get_state() assert ans_set_state == sol_set_state # output shape output = interactor.build(hp, self.inputs) # Act assert len(tf.nest.flatten(output)) == 1
def test_SparseFeatureMapper(self): # test constructor and get_state p = {'num_of_fields': 10, 'hash_size': [2, 4, 10], 'embedding_dim': 4} mapper = SparseFeatureMapper(**p) sol_get_state = { 'name': 'sparse_feature_mapper_1', 'num_of_fields': 10, 'hash_size': [2, 4, 10], 'embedding_dim': 4 } assert mapper.get_state() == sol_get_state # test set_state hash_size = self.df_inputs.nunique().tolist() p = { 'num_of_fields': self.input_shape, 'hash_size': hash_size, 'embedding_dim': self.embed_dim } sol_set_state = { 'name': 'sparse_feature_mapper_1', 'num_of_fields': self.input_shape, 'hash_size': hash_size, 'embedding_dim': self.embed_dim } mapper.set_state(p) ans_set_state = mapper.get_state() assert ans_set_state == sol_set_state # test build hp = hp_module.HyperParameters() tensor_inputs = [tf.convert_to_tensor(self.df_inputs.values)] mapper = SparseFeatureMapper(**p) output = mapper.build(hp, tensor_inputs) # Act assert len(nest.flatten(output)) == 1 assert output.shape == (self.batch, self.input_shape, self.embed_dim)
def test_RatingPredictionOptimizer(self): hp = hp_module.HyperParameters() optimizer = RatingPredictionOptimizer() output = optimizer.build(hp, self.inputs) assert len(nest.flatten(output)) == 1 assert output.shape == self.batch
def test_CTRPredictionOptimizer(self): hp = hp_module.HyperParameters() # Arrange optimizer = CTRPredictionOptimizer() output = optimizer.build(hp, self.inputs) # Act assert len(tf.nest.flatten(output)) == 1 # Assert assert output.shape == (self.batch, 1)
def test_get_with_conditional_scopes(): hp = hp_module.HyperParameters() hp.Choice('a', [1, 2, 3], default=2) assert hp.get('a') == 2 with hp.conditional_scope('a', 2): assert hp.get('a') == 2