def test_generator_with_dropouts(self): # Force graph mode with tf.compat.v1.Graph().as_default(): spec = phoenix_spec_pb2.PhoenixSpec( problem_type=phoenix_spec_pb2.PhoenixSpec.DNN) spec.search_type = phoenix_spec_pb2.PhoenixSpec.NONADAPTIVE_RANDOM_SEARCH spec.is_input_shared = True generator = search_candidate_generator.SearchCandidateGenerator( phoenix_spec=spec, metadata=ml_metadata_db.MLMetaData(phoenix_spec=spec, study_name='', study_owner='')) input_tensor = tf.zeros([20, 32, 32, 3]) fake_config = collections.namedtuple('RunConfig', ['model_dir', 'is_chief']) run_config = fake_config(model_dir=flags.FLAGS.test_tmpdir + '/1', is_chief=True) _ = generator.generate( features={}, input_layer_fn=lambda: None, trial_mode=trial_utils.TrialMode.NO_PRIOR, shared_input_tensor=input_tensor, shared_lengths=None, logits_dimension=10, hparams=hp.HParams(initial_architecture=['CONVOLUTION_3X3'], dropout_rate=0.3), run_config=run_config, is_training=True, trials=[]) all_nodes = [ node.name for node in tf.compat.v1.get_default_graph().as_graph_def().node ] self.assertAllInSet(_DROPOUT_GRAPH_NODE, all_nodes)
def test_construct_tower_with_transfer_learning( self, transfer_learning_type=transfer_learning_spec_pb2.TransferLearningSpec. NO_TRANSFER_LEARNING): # convolutions and then flatten plate. architecture = np.array([1, 3, 34]) str_signature = "_1334" input_tensor = tf.zeros([100, 32, 32, 3]) tower_name = "test_tower" transfer_learning_spec = transfer_learning_spec_pb2.TransferLearningSpec( transfer_learning_type=transfer_learning_type) phoenix_spec = phoenix_spec_pb2.PhoenixSpec( problem_type=phoenix_spec_pb2.PhoenixSpec.CNN, transfer_learning_spec=transfer_learning_spec) _ = architecture_utils.construct_tower( phoenix_spec=phoenix_spec, input_tensor=input_tensor, tower_name=tower_name, architecture=architecture, is_training=True, lengths=None, logits_dimension=10, hparams=hp.HParams(), model_directory=self.get_temp_dir(), is_frozen=False, dropout_rate=None) tensors = architecture_utils.get_tower_variables(tower_name) for tensor in tensors: if (transfer_learning_type == transfer_learning_spec_pb2. TransferLearningSpec.NO_TRANSFER_LEARNING): self.assertEndsWith(tensor.op.name, str_signature) else: self.assertNotEndsWith(tensor.op.name, str_signature)
def _create_checkpoint(self, towers, trial_id): with self.test_session(graph=tf.Graph()) as sess: architecture = np.array([1, 3, 34]) input_tensor = tf.zeros([100, 32, 32, 3]) phoenix_spec = phoenix_spec_pb2.PhoenixSpec( problem_type=phoenix_spec_pb2.PhoenixSpec.CNN) dirname = os.path.join(flags.FLAGS.test_tmpdir, str(trial_id)) if dirname and not tf.io.gfile.exists(dirname): tf.io.gfile.makedirs(dirname) for tower in towers: _ = architecture_utils.construct_tower( phoenix_spec=phoenix_spec, input_tensor=input_tensor, tower_name=str(tower) + '_0', architecture=architecture, is_training=True, lengths=None, logits_dimension=10, hparams=hp.HParams(), model_directory=dirname, is_frozen=False, dropout_rate=None) architecture_utils.set_number_of_towers(tower, 1) architecture_utils.set_number_of_towers('replay_generator', 0) directory = flags.FLAGS.test_tmpdir saver = tf.compat.v1.train.Saver() sess.run(tf.compat.v1.global_variables_initializer()) sess.run(tf.compat.v1.local_variables_initializer()) saver.save(sess, os.path.join(directory, str(trial_id)) + '/ckpt')
def test_tpu(self): # Force graph mode with tf.compat.v1.Graph().as_default(): learning_rate_spec = { 'learning_rate': 0.001, 'gradient_max_norm': 3 } spec = phoenix_spec_pb2.PhoenixSpec( problem_type=phoenix_spec_pb2.PhoenixSpec.DNN) task_manager_instance = task_manager.TaskManager(spec) logits = tf.keras.layers.Dense(10)(tf.zeros([20, 10])) logits_spec = architecture_utils.LogitsSpec(logits=logits) features = {'x': tf.zeros([10, 10])} loss_fn = loss_fns.make_multi_class_loss_fn() _ = task_manager_instance.create_model_spec( features=features, params=hp.HParams(optimizer='sgd'), learning_rate_spec=learning_rate_spec, train_logits_specs=[logits_spec], eval_logits_spec=logits_spec, labels=tf.ones([20], dtype=tf.int32), loss_fn=loss_fn, mode=tf.estimator.ModeKeys.TRAIN, lengths=None, use_tpu=True, predictions_fn=_default_predictions_fn) self.assertNotEmpty([ node.name for node in tf.compat.v1.get_default_graph().as_graph_def().node if 'CrossReplicaSum' in node.name ])
def test_learning_spec_on_predict(self, learning_rate_spec, not_containing): spec = phoenix_spec_pb2.PhoenixSpec( problem_type=phoenix_spec_pb2.PhoenixSpec.DNN) task_manager_instance = task_manager.TaskManager(spec) logits = tf.keras.layers.Dense(10)(tf.zeros([20, 10])) logits_spec = architecture_utils.LogitsSpec(logits=logits) features = {'x': tf.zeros([10, 10])} loss_fn = loss_fns.make_multi_class_loss_fn() model = task_manager_instance.create_model_spec( features=features, params=hp.HParams(optimizer='sgd'), learning_rate_spec=learning_rate_spec, train_logits_specs=[logits_spec], eval_logits_spec=logits_spec, labels=tf.ones([20], dtype=tf.int32), loss_fn=loss_fn, mode=tf.estimator.ModeKeys.PREDICT, lengths=None, use_tpu=False, predictions_fn=_default_predictions_fn) for phrase in not_containing: self.assertEmpty([ node.name for node in tf.compat.v1.get_default_graph().as_graph_def().node if phrase in node.name ]) self.assertLen(model.predictions, 3) self.assertIn('probabilities', model.predictions) self.assertIn('log_probabilities', model.predictions) self.assertIn('predictions', model.predictions) self.assertIsNone(model.loss)
def test_ensembler(self, combining_type, search_type, priors, logits, output_graph): # Force graph mode with tf.compat.v1.Graph().as_default(): spec = phoenix_spec_pb2.PhoenixSpec() spec.ensemble_spec.combining_type = combining_type spec.ensemble_spec.ensemble_search_type = search_type ensembler_instance = ensembler.Ensembler(spec) priors_logits_specs = [] search_logits_specs = [] if priors: for _ in range(priors): spec = architecture_utils.LogitsSpec( logits=tf.zeros([20, 10])) priors_logits_specs.append(spec) if logits: spec = architecture_utils.LogitsSpec( logits=tf.keras.layers.Dense(10)(tf.zeros([20, 10]))) search_logits_specs.append(spec) _ = ensembler_instance.bundle_logits( priors_logits_specs=priors_logits_specs, search_logits_specs=search_logits_specs, logits_dimension=10) nodes = tf.compat.v1.get_default_graph().as_graph_def().node logging.info([node.name for node in nodes]) self.assertCountEqual([n.name for n in nodes], output_graph)
def test_architecture(self): # Force graph mode with tf.compat.v1.Graph().as_default(): learning_rate_spec = {'learning_rate': 0.001, 'gradient_max_norm': 3} spec = phoenix_spec_pb2.PhoenixSpec( problem_type=phoenix_spec_pb2.PhoenixSpec.CNN) text_format.Merge( """ multi_task_spec { label_name: "label1" number_of_classes: 10 architecture: "FIXED_OUTPUT_FULLY_CONNECTED_128" } multi_task_spec { label_name: "label2" number_of_classes: 10 architecture: "FIXED_OUTPUT_FULLY_CONNECTED_256" architecture: "FIXED_OUTPUT_FULLY_CONNECTED_512" } """, spec) task_manager_instance = task_manager.TaskManager(spec) logits = tf.keras.layers.Dense(10)(tf.zeros([20, 10])) logits_spec = architecture_utils.LogitsSpec(logits=logits) features = {'x': tf.zeros([10, 10])} loss_fn = loss_fns.make_multi_class_loss_fn() model = task_manager_instance.create_model_spec( features=features, params=hp.HParams(optimizer='sgd'), learning_rate_spec=learning_rate_spec, train_logits_specs=[logits_spec], eval_logits_spec=logits_spec, labels={ 'label1': tf.ones([20], dtype=tf.int32), 'label2': tf.ones([20], dtype=tf.int32) }, loss_fn=loss_fn, model_directory=self.get_temp_dir(), mode=tf.estimator.ModeKeys.TRAIN, lengths=None, use_tpu=False, predictions_fn=_default_predictions_fn) self.assertNotEmpty([ node.name for node in tf.compat.v1.get_default_graph().as_graph_def().node if 'task_label1_tower/1_FIXED_OUTPUT_FULLY_CONNECTED_128' in node.name ]) self.assertNotEmpty([ node.name for node in tf.compat.v1.get_default_graph().as_graph_def().node if 'task_label2_tower/1_FIXED_OUTPUT_FULLY_CONNECTED_256' in node.name ]) self.assertNotEmpty([ node.name for node in tf.compat.v1.get_default_graph().as_graph_def().node if 'task_label2_tower/2_FIXED_OUTPUT_FULLY_CONNECTED_512' in node.name ]) self.assertLen(model.predictions, 3 * (1 + 2)) self.assertIn('probabilities', model.predictions) self.assertIn('log_probabilities', model.predictions) self.assertIn('predictions', model.predictions)
def test_construct_network(self, dropout, expected_logits): # Force graph mode with tf.compat.v1.Graph().as_default(): tf.random.set_seed(1234) # convolutions and then flatten plate. architecture = np.array([1, 3, 34]) input_tensor = tf.compat.v1.placeholder(dtype=tf.float32, shape=[None, 32, 32, 3], name="input") phoenix_spec = phoenix_spec_pb2.PhoenixSpec( problem_type=phoenix_spec_pb2.PhoenixSpec.CNN) tower_spec = architecture_utils.construct_tower( phoenix_spec=phoenix_spec, input_tensor=input_tensor, tower_name="test_tower", architecture=architecture, is_training=True, lengths=None, logits_dimension=10, hparams=hp.HParams(), model_directory=self.get_temp_dir(), is_frozen=False, dropout_rate=dropout) np.random.seed(42) test_input = np.random.random([1, 32, 32, 3]) with tf.compat.v1.Session() as sess: sess.run([ tf.compat.v1.global_variables_initializer(), tf.compat.v1.local_variables_initializer() ]) logits_val = sess.run(tower_spec.logits_spec.logits, feed_dict={input_tensor: test_input}) self.assertAllClose(expected_logits, logits_val, rtol=1e-3)
def _create_phoenix_spec(self, problem_type): """Creates a new phoenix.PhoenixSpec for the given problem type.""" spec_path = os.path.join(FLAGS.test_srcdir, _SPEC_PATH_TEMPLATE.format(problem_type)) spec = phoenix_spec_pb2.PhoenixSpec() with tf.io.gfile.GFile(spec_path, "r") as f: text_format.Merge(f.read(), spec) return spec
def test_preconditions(self, kwargs_to_pass=None): if "phoenix_spec" not in kwargs_to_pass: kwargs_to_pass["phoenix_spec"] = phoenix_spec_pb2.PhoenixSpec() kwargs_to_pass["input_layer_fn"] = lambda: None kwargs_to_pass["study_owner"] = "fake_owner" kwargs_to_pass["study_name"] = "fake_name" with self.assertRaises(AssertionError): phoenix.Phoenix(**kwargs_to_pass)
def test_wrong_dict_weight_feature(self, weight_is_a_feature): learning_rate_spec = {'learning_rate': 0.001, 'gradient_max_norm': 3} spec = phoenix_spec_pb2.PhoenixSpec( problem_type=phoenix_spec_pb2.PhoenixSpec.DNN) text_format.Merge( """ multi_task_spec { label_name: "label1" number_of_classes: 10 weight_feature_name: "weight1" weight_is_a_feature: %s } multi_task_spec { label_name: "label2" number_of_classes: 10 weight_feature_name: "weight2" weight_is_a_feature: %s } """ % (str(weight_is_a_feature), str(weight_is_a_feature)), spec) labels = { 'label1': tf.ones([20], dtype=tf.int32), 'label2': tf.ones([20], dtype=tf.int32), } # Fix the size of the dict labels to bypass the assertion. if not weight_is_a_feature: labels.update({ 'not_used': tf.ones([20], dtype=tf.int32), 'not_used2': tf.ones([20], dtype=tf.int32) }) weights = { 'weight1': tf.constant([2] * 20), 'weight2': tf.constant([3] * 20) } features = {'x': tf.zeros([10, 10])} if not weight_is_a_feature: features.update(weights) task_manager_instance = task_manager.TaskManager(spec) logits = tf.keras.layers.Dense(10)(tf.zeros([20, 10])) logits_spec = architecture_utils.LogitsSpec(logits=logits) with self.assertRaises(KeyError): loss_fn = loss_fns.make_multi_class_loss_fn() _ = task_manager_instance.create_model_spec( features=features, params=hp.HParams(optimizer='sgd'), learning_rate_spec=learning_rate_spec, train_logits_specs=[logits_spec], eval_logits_spec=logits_spec, labels=labels, loss_fn=loss_fn, model_directory=self.get_temp_dir(), mode=tf.estimator.ModeKeys.TRAIN, lengths=None, use_tpu=False, predictions_fn=_default_predictions_fn)
def test_weight_feature(self, is_multitask, weight_is_a_feature): # Force graph mode with tf.compat.v1.Graph().as_default(): learning_rate_spec = { 'learning_rate': 0.001, 'gradient_max_norm': 3 } spec = phoenix_spec_pb2.PhoenixSpec( problem_type=phoenix_spec_pb2.PhoenixSpec.DNN) labels = tf.ones([20], dtype=tf.int32) if is_multitask: text_format.Merge( """ multi_task_spec { label_name: "label1" number_of_classes: 10 weight_feature_name: "weight1" weight_is_a_feature: %s } multi_task_spec { label_name: "label2" number_of_classes: 10 weight_feature_name: "weight2" weight_is_a_feature: %s } """ % (str(weight_is_a_feature), str(weight_is_a_feature)), spec) labels = { 'label1': tf.ones([20], dtype=tf.int32), 'label2': tf.ones([20], dtype=tf.int32) } weights = { 'weight1': tf.constant([2] * 20), 'weight2': tf.constant([3] * 20) } features = {'x': tf.zeros([10, 10])} if weight_is_a_feature: features.update(weights) elif isinstance(labels, dict): labels.update(weights) task_manager_instance = task_manager.TaskManager(spec) logits = tf.keras.layers.Dense(10)(tf.zeros([20, 10])) logits_spec = architecture_utils.LogitsSpec(logits=logits) loss_fn = loss_fns.make_multi_class_loss_fn() _ = task_manager_instance.create_model_spec( features=features, params=hp.HParams(optimizer='sgd'), learning_rate_spec=learning_rate_spec, train_logits_specs=[logits_spec], eval_logits_spec=logits_spec, labels=labels, loss_fn=loss_fn, mode=tf.estimator.ModeKeys.TRAIN, lengths=None, use_tpu=False, predictions_fn=_default_predictions_fn)
def _create_spec(ensemble_type='none'): output = phoenix_spec_pb2.PhoenixSpec() if ensemble_type == 'adaptive': output.ensemble_spec.ensemble_search_type = ( ensembling_spec_pb2.EnsemblingSpec.ADAPTIVE_ENSEMBLE_SEARCH) if ensemble_type == 'nonadaptive': output.ensemble_spec.ensemble_search_type = ( ensembling_spec_pb2.EnsemblingSpec.NONADAPTIVE_ENSEMBLE_SEARCH) return output
def __init__(self, data, spec): self._data = data if isinstance(spec, str): self._spec = phoenix_spec_pb2.PhoenixSpec() with tf.io.gfile.GFile(spec, "r") as f: text_format.Parse(f.read(), self._spec) else: self._spec = spec self._tuner_id = "tuner-1"
def _create_spec(problem_type, complexity_thresholds=None, max_depth=None, min_depth=None): output = phoenix_spec_pb2.PhoenixSpec() if complexity_thresholds is not None: output.increase_complexity_minimum_trials[:] = complexity_thresholds if max_depth is not None: output.maximum_depth = max_depth if min_depth is not None: output.minimum_depth = min_depth output.problem_type = problem_type return output
def test_init_variables(self, new_architecture, expected_output, new_tower_name="test_tower"): # Force graph mode with tf.compat.v1.Graph().as_default(): directory = self.get_temp_dir() architecture = np.array([1, 3, 34]) phoenix_spec = phoenix_spec_pb2.PhoenixSpec( problem_type=phoenix_spec_pb2.PhoenixSpec.CNN) with self.test_session(graph=tf.Graph()) as sess: input_tensor = tf.zeros([100, 32, 32, 3]) _ = architecture_utils.construct_tower( phoenix_spec=phoenix_spec, input_tensor=input_tensor, tower_name="test_tower", architecture=architecture, is_training=True, lengths=None, logits_dimension=10, model_directory=self.get_temp_dir(), hparams=hp.HParams(), is_frozen=False, dropout_rate=None) saver = tf.compat.v1.train.Saver() sess.run(tf.compat.v1.global_variables_initializer()) sess.run(tf.compat.v1.local_variables_initializer()) saver.save(sess, directory + "/ckpt") with self.test_session(graph=tf.Graph()) as sess: input_tensor = tf.zeros([100, 32, 32, 3]) _ = architecture_utils.construct_tower( phoenix_spec=phoenix_spec, input_tensor=input_tensor, tower_name=new_tower_name, architecture=new_architecture, is_training=True, lengths=None, logits_dimension=10, hparams=hp.HParams(), model_directory=self.get_temp_dir(), is_frozen=False, dropout_rate=None) snapshotting_variables = architecture_utils.init_variables( tf.train.latest_checkpoint(directory), "Phoenix/test_tower", "Phoenix/{}".format(new_tower_name)) self.assertCountEqual(snapshotting_variables, expected_output)
def create_spec(problem_type, complexity_thresholds=None, max_depth=None, min_depth=None, blocks_to_use=None): """Creates a phoenix_spec_pb2.PhoenixSpec with the given options.""" output = phoenix_spec_pb2.PhoenixSpec() if complexity_thresholds is not None: output.increase_complexity_minimum_trials[:] = complexity_thresholds if max_depth is not None: output.maximum_depth = max_depth if min_depth is not None: output.minimum_depth = min_depth output.problem_type = problem_type if blocks_to_use: output.blocks_to_use[:] = blocks_to_use return output
def test_intermixed_prior_graph(self): # Force graph mode with tf.compat.v1.Graph().as_default(): spec = phoenix_spec_pb2.PhoenixSpec( problem_type=phoenix_spec_pb2.PhoenixSpec.DNN) spec.ensemble_spec.ensemble_search_type = ( ensembling_spec_pb2.EnsemblingSpec. INTERMIXED_NONADAPTIVE_ENSEMBLE_SEARCH) spec.ensemble_spec.intermixed_search.width = 2 spec.ensemble_spec.intermixed_search.try_ensembling_every = 4 spec.ensemble_spec.intermixed_search.num_trials_to_consider = 3 spec.is_input_shared = True generator = prior_generator.PriorGenerator( phoenix_spec=spec, metadata=ml_metadata_db.MLMetaData(phoenix_spec=spec, study_name='', study_owner='')) fake_config = collections.namedtuple('RunConfig', ['model_dir']) # Should be multplication of 4. run_config = fake_config(model_dir=flags.FLAGS.test_tmpdir + '/10000') tf.io.gfile.makedirs(run_config.model_dir) # Best three trials checkpoint are generated. If the generator chooses # the suboptimal (wrong) trials, the test will fail. self._create_checkpoint(['search_generator'], 2) self._create_checkpoint(['search_generator'], 3) self._create_checkpoint(['search_generator'], 5) logits, _ = generator.first_time_chief_generate( features={}, input_layer_fn=lambda: None, trial_mode=trial_utils.TrialMode.ENSEMBLE_SEARCH, shared_input_tensor=tf.zeros([100, 32, 32, 3]), shared_lengths=None, logits_dimension=10, hparams={}, run_config=run_config, is_training=True, trials=trial_utils.create_test_trials_intermixed( flags.FLAGS.test_tmpdir)) self.assertLen(logits, 2) all_nodes = [ node.name for node in tf.compat.v1.get_default_graph().as_graph_def().node ] self.assertAllInSet(_NONADAPTIVE_GRAPH_NODES, all_nodes)
def _create_spec(replay_towers=0, ensemble_type="none"): output = phoenix_spec_pb2.PhoenixSpec() output.search_type = phoenix_spec_pb2.PhoenixSpec.NONADAPTIVE_RANDOM_SEARCH if replay_towers: output.replay.CopyFrom(phoenix_spec_pb2.ArchitectureReplay()) for _ in range(replay_towers): output.replay.towers.add() if ensemble_type == "adaptive": output.ensemble_spec.ensemble_search_type = ( ensembling_spec_pb2.EnsemblingSpec.ADAPTIVE_ENSEMBLE_SEARCH) if ensemble_type == "nonadaptive": output.ensemble_spec.ensemble_search_type = ( ensembling_spec_pb2.EnsemblingSpec.NONADAPTIVE_ENSEMBLE_SEARCH) if ensemble_type == "intermix": output.ensemble_spec.ensemble_search_type = ( ensembling_spec_pb2.EnsemblingSpec. INTERMIXED_NONADAPTIVE_ENSEMBLE_SEARCH) return output
def test_generator_with_distillation_and_intermixed(self): # Force graph mode with tf.compat.v1.Graph().as_default(): spec = phoenix_spec_pb2.PhoenixSpec( problem_type=phoenix_spec_pb2.PhoenixSpec.CNN) spec.is_input_shared = True spec.search_type = phoenix_spec_pb2.PhoenixSpec.NONADAPTIVE_RANDOM_SEARCH spec.ensemble_spec.ensemble_search_type = ( ensembling_spec_pb2.EnsemblingSpec. INTERMIXED_NONADAPTIVE_ENSEMBLE_SEARCH) spec.ensemble_spec.intermixed_search.width = 2 spec.ensemble_spec.intermixed_search.try_ensembling_every = 4 spec.ensemble_spec.intermixed_search.num_trials_to_consider = 3 spec.distillation_spec.distillation_type = ( distillation_spec_pb2.DistillationSpec.DistillationType. MSE_LOGITS) generator = search_candidate_generator.SearchCandidateGenerator( phoenix_spec=spec, metadata=ml_metadata_db.MLMetaData(phoenix_spec=spec, study_name='', study_owner='')) fake_config = collections.namedtuple('RunConfig', ['model_dir', 'is_chief']) run_config = fake_config(model_dir=flags.FLAGS.test_tmpdir + '/10000', is_chief=True) self._create_checkpoint(['search_generator'], 2) self._create_checkpoint(['search_generator'], 3) self._create_checkpoint(['search_generator'], 5) input_tensor = tf.zeros([20, 32, 32, 3]) _ = generator.generate( features={}, input_layer_fn=lambda: None, trial_mode=trial_utils.TrialMode.DISTILLATION, shared_input_tensor=input_tensor, shared_lengths=None, logits_dimension=10, hparams=hp.HParams(initial_architecture=['CONVOLUTION_3X3']), run_config=run_config, is_training=True, trials=trial_utils.create_test_trials_intermixed( flags.FLAGS.test_tmpdir))
def test_get_suggestion(self, get_architecture, randint, completed_trials, initial_architecture, expected_architecture, replicate_cell): get_architecture.return_value = initial_architecture randint.return_value = 2 hparams = hp.HParams( new_block_type="FIXED_CHANNEL_CONVOLUTION_16") # id=1. # Use problem_type=DNN so we don't have to worry about the final # architecture being correct. spec = phoenix_spec_pb2.PhoenixSpec( problem_type=phoenix_spec_pb2.PhoenixSpec.DNN, maximum_depth=11, num_blocks_in_cell=4, reduction_block_type="AVERAGE_POOL_2X2", replicate_cell=replicate_cell, beam_size=3) algorithm = constrained_descent.ConstrainedDescent( spec, self._metadata) trials = [] for i in range(completed_trials): trials.append( trial_module.Trial({ "id": i, "model_dir": "/tmp/" + str(i), "status": "COMPLETED", "trial_infeasible": False, "final_measurement": { "objective_value": 100 - i } })) actual_architecture, fork_trial = algorithm.get_suggestion( trials, hparams) # We use a beam of size 3 and always choose the last trial to be the best. self.assertEqual(fork_trial, completed_trials - 3) self.assertTrue( search_test_utils.is_mutation_or_equal(actual_architecture, expected_architecture))
def test_generator_with_snapshot(self): # Force graph mode with tf.compat.v1.Graph().as_default(): spec = phoenix_spec_pb2.PhoenixSpec( problem_type=phoenix_spec_pb2.PhoenixSpec.CNN) spec.search_type = phoenix_spec_pb2.PhoenixSpec.ADAPTIVE_COORDINATE_DESCENT spec.transfer_learning_spec.transfer_learning_type = ( transfer_learning_spec_pb2.TransferLearningSpec. SNAPSHOT_TRANSFER_LEARNING) spec.is_input_shared = True generator = search_candidate_generator.SearchCandidateGenerator( phoenix_spec=spec, metadata=ml_metadata_db.MLMetaData(phoenix_spec=spec, study_name='', study_owner='')) input_tensor = tf.zeros([20, 32, 32, 3]) fake_config = collections.namedtuple('RunConfig', ['model_dir', 'is_chief']) tf.io.gfile.makedirs(flags.FLAGS.test_tmpdir + '/3') run_config = fake_config(model_dir=flags.FLAGS.test_tmpdir + '/3', is_chief=True) self._create_checkpoint(['search_generator'], 2) _ = generator.generate( features={}, input_layer_fn=lambda: None, trial_mode=trial_utils.TrialMode.ENSEMBLE_SEARCH, shared_input_tensor=input_tensor, shared_lengths=None, logits_dimension=10, hparams=hp.HParams(initial_architecture=['CONVOLUTION_3X3'], dropout_rate=0.3, new_block_type='CONVOLUTION_3X3'), run_config=run_config, is_training=True, trials=_create_trials(flags.FLAGS.test_tmpdir)) all_nodes = [ node.name for node in tf.compat.v1.get_default_graph().as_graph_def().node ] self.assertAllInSet(_DROPOUT_GRAPH_NODE, all_nodes)
def write_replay_spec(model_dir, filename, original_spec, search_architecture, hparams): """Writes a replay spec to retrain the same model.""" # Ensure the same search space as the original run replay_spec = copy.deepcopy(original_spec) # Remove user suggestions replay_spec.ClearField('user_suggestions') # If this is already a replay config replay_spec.ClearField('replay') dependency_file = os.path.join(model_dir, _PREVIOUS_DEPENDENCIES_FILENAME) if tf.io.gfile.exists(dependency_file): with tf.io.gfile.GFile(dependency_file, 'r') as f: data = f.read().split('\n') for dependency in data: spec = phoenix_spec_pb2.PhoenixSpec() with tf.io.gfile.GFile(os.path.join(dependency, filename), 'r') as f: text_format.Parse(f.read(), spec) for i, _ in enumerate(spec.replay.towers): replay_spec.replay.towers.add().CopyFrom(spec.replay.towers[i]) # Currently we support non-ensembling models. search_tower = replay_spec.replay.towers.add() # Removing the "context" hparam. This hparam is added for tpu, by the tpu team # It is not compatible with the function "to_proto" used below. copy_hparams = hp.HParams(**hparams.values()) if hasattr(copy_hparams, 'context'): copy_hparams.del_hparam('context') # We sometimes (for some search algorithms) override the hparam # initial_architecture. Updating with the ground truth. # TODO(b/172564129): Make initial_architecture consistent everywhere. copy_hparams.set_hparam('initial_architecture', search_architecture) search_tower.hparams.CopyFrom(copy_hparams.to_proto()) search_tower.architecture[:] = search_architecture with tf.io.gfile.GFile(os.path.join(model_dir, filename), 'w') as f: f.write(str(replay_spec))
def test_nonadaptive_prior(self, width, consider): # Force graph mode with tf.compat.v1.Graph().as_default(): spec = phoenix_spec_pb2.PhoenixSpec( problem_type=phoenix_spec_pb2.PhoenixSpec.DNN) spec.ensemble_spec.ensemble_search_type = ( ensembling_spec_pb2.EnsemblingSpec.NONADAPTIVE_ENSEMBLE_SEARCH) spec.ensemble_spec.nonadaptive_search.width = width spec.ensemble_spec.nonadaptive_search.num_trials_to_consider = consider spec.is_input_shared = True generator = prior_generator.PriorGenerator( phoenix_spec=spec, metadata=ml_metadata_db.MLMetaData(phoenix_spec=spec, study_name='', study_owner='')) fake_config = collections.namedtuple('RunConfig', ['model_dir']) run_config = fake_config(model_dir=flags.FLAGS.test_tmpdir + '/10000') tf.io.gfile.makedirs(run_config.model_dir) # Best three trials checkpoint are generated. If the generator chooses # the suboptimal (wrong) trials, the test will fail. self._create_checkpoint(['search_generator'], 3) self._create_checkpoint(['search_generator'], 4) self._create_checkpoint(['search_generator'], 5) logits, _ = generator.first_time_chief_generate( features={}, input_layer_fn=lambda: None, trial_mode=trial_utils.TrialMode.ENSEMBLE_SEARCH, shared_input_tensor=tf.zeros([100, 32, 32, 3]), shared_lengths=None, logits_dimension=10, hparams={}, run_config=run_config, is_training=True, trials=_create_trials(flags.FLAGS.test_tmpdir)) self.assertLen(logits, min(width, consider))
def _replay_spec(): output = phoenix_spec_pb2.PhoenixSpec() output.replay.CopyFrom(phoenix_spec_pb2.ArchitectureReplay()) return output
def test_multitask(self, learning_rate_spec, contains_node, not_containing): # Force graph mode with tf.compat.v1.Graph().as_default(): spec = phoenix_spec_pb2.PhoenixSpec( problem_type=phoenix_spec_pb2.PhoenixSpec.DNN) text_format.Merge( """ multi_task_spec { label_name: "label1" number_of_classes: 10 } multi_task_spec { label_name: "label2" number_of_classes: 10 } """, spec) task_manager_instance = task_manager.TaskManager(spec) logits = tf.keras.layers.Dense(10)(tf.zeros([20, 10])) logits_spec = architecture_utils.LogitsSpec(logits=logits) features = {'x': tf.zeros([10, 10])} loss_fn = loss_fns.make_multi_class_loss_fn() model = task_manager_instance.create_model_spec( features=features, params=hp.HParams(optimizer='sgd'), learning_rate_spec=learning_rate_spec, train_logits_specs=[logits_spec], eval_logits_spec=logits_spec, labels={ 'label1': tf.ones([20], dtype=tf.int32), 'label2': tf.ones([20], dtype=tf.int32) }, loss_fn=loss_fn, mode=tf.estimator.ModeKeys.TRAIN, lengths=None, use_tpu=False, predictions_fn=_default_predictions_fn) self.assertNotEmpty([ node.name for node in tf.compat.v1.get_default_graph().as_graph_def().node if contains_node in node.name ]) for phrase in not_containing: self.assertEmpty([ node.name for node in tf.compat.v1.get_default_graph().as_graph_def().node if phrase in node.name ]) self.assertLen(model.predictions, 3 * (1 + 2)) self.assertContainsSubset([ 'probabilities', 'probabilities/label1', 'probabilities/label2', 'log_probabilities', 'log_probabilities/label1', 'log_probabilities/label2', 'predictions', 'predictions/label1', 'predictions/label2', ], model.predictions.keys())
def test_import_tower(self, shared_input): np.random.seed(42) test_input = np.random.random([1, 32, 32, 3]) # Force graph mode with tf.compat.v1.Graph().as_default(): directory = self.get_temp_dir() architecture = np.array([1, 3, 34]) phoenix_spec = phoenix_spec_pb2.PhoenixSpec( problem_type=phoenix_spec_pb2.PhoenixSpec.CNN) phoenix_spec.is_input_shared = shared_input features = {} shared_input_tensor = None with self.test_session(graph=tf.Graph()) as sess: input_tensor_1 = tf.compat.v1.placeholder( dtype=tf.float32, shape=[None, 32, 32, 3], name="input_1") tf.random.set_seed(1234) tower_spec_1 = architecture_utils.construct_tower( phoenix_spec=phoenix_spec, input_tensor=input_tensor_1, tower_name="test_tower", architecture=architecture, is_training=True, lengths=None, logits_dimension=10, hparams=hp.HParams(), model_directory=self.get_temp_dir(), is_frozen=False, dropout_rate=None) saver = tf.compat.v1.train.Saver() sess.run(tf.compat.v1.global_variables_initializer()) sess.run(tf.compat.v1.local_variables_initializer()) logits_val_1 = sess.run(tower_spec_1.logits_spec.logits, feed_dict={input_tensor_1: test_input}) saver.save(sess, directory + "/ckpt") with self.test_session(graph=tf.Graph()) as sess: input_tensor_2 = tf.compat.v1.placeholder( dtype=tf.float32, shape=[None, 32, 32, 3], name="input_2") if shared_input: shared_input_tensor = input_tensor_2 def _input_layer_fn(features, is_training, scope_name="Phoenix/Input", lengths_feature_name=None): del features, is_training, scope_name, lengths_feature_name return None, None else: features = {"x": input_tensor_2} def _input_layer_fn(features, is_training, scope_name="Phoenix/Input", lengths_feature_name=None): del is_training, lengths_feature_name with tf.compat.v1.variable_scope(scope_name): return tf.cast(features["x"], dtype=tf.float32), None tf.random.set_seed(1234) tower_spec_2 = architecture_utils.import_tower( features=features, input_layer_fn=_input_layer_fn, phoenix_spec=phoenix_spec, shared_input_tensor=shared_input_tensor, original_tower_name="test_tower", new_tower_name="imported_tower", model_directory=directory, new_model_directory=self.get_temp_dir(), is_training=True, logits_dimension=10, shared_lengths=None, force_snapshot=False, force_freeze=False) sess.run(tf.compat.v1.global_variables_initializer()) sess.run(tf.compat.v1.local_variables_initializer()) logits_val_2 = sess.run(tower_spec_2.logits_spec.logits, feed_dict={input_tensor_2: test_input}) self.assertAllClose(logits_val_1, logits_val_2, rtol=1e-3)
class ArchitectureUtilsTest(parameterized.TestCase, tf.test.TestCase): def test_get_blocks_search_space(self): hps = architecture_utils.get_blocks_search_space() self.assertIn("TUNABLE_SVDF_output_size", hps) self.assertIn("TUNABLE_SVDF_rank", hps) self.assertIn("TUNABLE_SVDF_projection_size", hps) self.assertIn("TUNABLE_SVDF_memory_size", hps) hps = architecture_utils.get_blocks_search_space(["TUNABLE_SVDF"]) self.assertIn("TUNABLE_SVDF_output_size", hps) self.assertIn("TUNABLE_SVDF_rank", hps) self.assertIn("TUNABLE_SVDF_projection_size", hps) self.assertIn("TUNABLE_SVDF_memory_size", hps) def test_get_block_hparams(self): hp_ = architecture_utils.get_block_hparams( hp.HParams(TUNABLE_SVDF_target=10, non_target=15), "TUNABLE_SVDF") self.assertLen(hp_.values(), 1) self.assertEqual(hp_.target, 10) def test_store_and_get_hparams(self): hp_ = hp.HParams(hello="world", context="toberemoved") dirname = self.get_temp_dir() architecture_utils.store_hparams_to_dir(hp_, dirname, "tower") hp_replica = architecture_utils.get_hparams_from_dir(dirname, "tower") self.assertLen(hp_replica.values(), 1) self.assertEqual(hp_replica.hello, "world") @parameterized.named_parameters( { "testcase_name": "empty", "input_dir": "", "expected_output": None, "spec": phoenix_spec_pb2.PhoenixSpec(), }, { "testcase_name": "normal", "input_dir": "some/random/path/512", "expected_output": 512, "spec": phoenix_spec_pb2.PhoenixSpec(), }, { "testcase_name": "normal_tfx", "input_dir": "/some/random/Trial-000034/active/Run-0000000", "expected_output": 34, "spec": phoenix_spec_pb2.PhoenixSpec(), }, { "testcase_name": "hybrid", "input_dir": "/some/random/Trial-000034/active/Run-0000000/7", "expected_output": 7, "spec": phoenix_spec_pb2.PhoenixSpec(), }, { "testcase_name": "replay", "input_dir": "/some/random/Trial-000034/active/Run-0000000/7", "expected_output": 7, "spec": _replay_spec(), }) def test_get_trial_id(self, input_dir, expected_output, spec): output = architecture_utils.DirectoryHandler.get_trial_id( input_dir, spec) self.assertEqual(output, expected_output) @parameterized.named_parameters( { "testcase_name": "empty", "input_trial": trial.Trial({"model_dir": ""}), "expected_output": "" }, { "testcase_name": "normal", "input_trial": trial.Trial({"model_dir": "random/path"}), "expected_output": "random/path" }) def test_get_trial_dir(self, input_trial, expected_output): output = architecture_utils.DirectoryHandler.trial_dir(input_trial) self.assertEqual(output, expected_output) @parameterized.named_parameters( { "testcase_name": "empty", "initial_architecture": [], "fixed_architecture": [blocks.BlockType.PLATE_REDUCTION_FLATTEN] }, { "testcase_name": "flatten", "initial_architecture": [blocks.BlockType.PLATE_REDUCTION_FLATTEN], "fixed_architecture": [blocks.BlockType.PLATE_REDUCTION_FLATTEN] }, { "testcase_name": "downsample_flatten", "initial_architecture": [blocks.BlockType.DOWNSAMPLE_FLATTEN], "fixed_architecture": [blocks.BlockType.DOWNSAMPLE_FLATTEN] }, { "testcase_name": "basic_case", "initial_architecture": [ blocks.BlockType.PLATE_REDUCTION_FLATTEN, blocks.BlockType.FULLY_CONNECTED, blocks.BlockType.CONVOLUTION_3X3 ], "fixed_architecture": [ blocks.BlockType.CONVOLUTION_3X3, blocks.BlockType.PLATE_REDUCTION_FLATTEN, blocks.BlockType.FULLY_CONNECTED ] }, { "testcase_name": "basic_downsample_case", "initial_architecture": [ blocks.BlockType.DOWNSAMPLE_FLATTEN, blocks.BlockType.FULLY_CONNECTED, blocks.BlockType.CONVOLUTION_3X3 ], "fixed_architecture": [ blocks.BlockType.CONVOLUTION_3X3, blocks.BlockType.DOWNSAMPLE_FLATTEN, blocks.BlockType.FULLY_CONNECTED ] }, { "testcase_name": "advanced_case", "initial_architecture": [ blocks.BlockType.PLATE_REDUCTION_FLATTEN, blocks.BlockType.FULLY_CONNECTED, blocks.BlockType.FULLY_CONNECTED_PYRAMID, blocks.BlockType.CONVOLUTION_3X3, blocks.BlockType.DOWNSAMPLE_CONVOLUTION_3X3 ], "fixed_architecture": [ blocks.BlockType.CONVOLUTION_3X3, blocks.BlockType.DOWNSAMPLE_CONVOLUTION_3X3, blocks.BlockType.PLATE_REDUCTION_FLATTEN, blocks.BlockType.FULLY_CONNECTED, blocks.BlockType.FULLY_CONNECTED_PYRAMID ] }) def test_fix_architecture_order(self, initial_architecture, fixed_architecture): out_architecture = architecture_utils.fix_architecture_order( initial_architecture, phoenix_spec_pb2.PhoenixSpec.CNN) self.assertAllEqual(fixed_architecture, out_architecture) @parameterized.named_parameters( { "testcase_name": "empty_rnn", "problem_type": phoenix_spec_pb2.PhoenixSpec.RNN_ALL_ACTIVATIONS, "new_block": blocks.BlockType.LSTM_128, "initial_architecture": np.array([]), "expected_architecture": [blocks.BlockType.LSTM_128] }, { "testcase_name": "empty_dnn", "problem_type": phoenix_spec_pb2.PhoenixSpec.DNN, "new_block": blocks.BlockType.FIXED_OUTPUT_FULLY_CONNECTED_128, "initial_architecture": np.array([]), "expected_architecture": [blocks.BlockType.FIXED_OUTPUT_FULLY_CONNECTED_128] }) def test_increase_structure_depth(self, problem_type, initial_architecture, new_block, expected_architecture): out_architecture = architecture_utils.increase_structure_depth( initial_architecture, new_block, problem_type) self.assertAllEqual(expected_architecture, out_architecture) @parameterized.named_parameters( { "testcase_name": "test1", "architecture": np.array([1, 2, 3, 4]), }, { "testcase_name": "test2", "architecture": np.array([2, 3, 4, 5]), }) def test_set_get_architecture(self, architecture): # Force graph mode with tf.compat.v1.Graph().as_default(): directory = self.get_temp_dir() with self.test_session(graph=tf.Graph()) as sess: architecture_utils.set_architecture(architecture) saver = tf.compat.v1.train.Saver() sess.run(tf.compat.v1.global_variables_initializer()) sess.run(tf.compat.v1.local_variables_initializer()) saver.save(sess, directory + "/ckpt") output_architecture = architecture_utils.get_architecture( directory) self.assertAllEqual(output_architecture, architecture) @parameterized.named_parameters( { "testcase_name": "no_architecture", "architecture": None, }, { "testcase_name": "has_architecture", "architecture": np.array([2, 3, 4, 5]), }) def test_get_architecture_size(self, architecture): # Force graph mode with tf.compat.v1.Graph().as_default(): with self.test_session(graph=tf.Graph()): tower_name = "tower" if architecture is not None: architecture_utils.set_architecture(architecture, tower_name=tower_name) size = architecture_utils.get_architecture_size( tower_name=tower_name) if architecture is not None: self.assertEqual(size, architecture.size) else: self.assertIsNone(size) @parameterized.named_parameters( { "testcase_name": "regular_network", "dropout": -1, "expected_logits": [[ -0.1481, 0.3328, 0.3028, 0.5652, 0.6860, 0.06171, 0.09998, -0.2622, 0.2186, -0.1322 ]] }, { "testcase_name": "dropout_network", "dropout": 0.1, "expected_logits": [[ 0.7001, -0.06655, -0.1711, 0.1274, -0.8175, 0.2932, 0.06242, 0.2182, -0.06626, 0.7882 ]], }) def test_construct_network(self, dropout, expected_logits): # Force graph mode with tf.compat.v1.Graph().as_default(): tf.random.set_seed(1234) # convolutions and then flatten plate. architecture = np.array([1, 3, 34]) input_tensor = tf.compat.v1.placeholder(dtype=tf.float32, shape=[None, 32, 32, 3], name="input") phoenix_spec = phoenix_spec_pb2.PhoenixSpec( problem_type=phoenix_spec_pb2.PhoenixSpec.CNN) tower_spec = architecture_utils.construct_tower( phoenix_spec=phoenix_spec, input_tensor=input_tensor, tower_name="test_tower", architecture=architecture, is_training=True, lengths=None, logits_dimension=10, hparams=hp.HParams(), model_directory=self.get_temp_dir(), is_frozen=False, dropout_rate=dropout) np.random.seed(42) test_input = np.random.random([1, 32, 32, 3]) with tf.compat.v1.Session() as sess: sess.run([ tf.compat.v1.global_variables_initializer(), tf.compat.v1.local_variables_initializer() ]) logits_val = sess.run(tower_spec.logits_spec.logits, feed_dict={input_tensor: test_input}) self.assertAllClose(expected_logits, logits_val, rtol=1e-3) @parameterized.named_parameters( { "testcase_name": "disabled", "transfer_learning_type": transfer_learning_spec_pb2.TransferLearningSpec. NO_TRANSFER_LEARNING }, { "testcase_name": "enabled", "transfer_learning_type": transfer_learning_spec_pb2.TransferLearningSpec. UNIFORM_AVERAGE_TRANSFER_LEARNING }) def test_construct_tower_with_transfer_learning( self, transfer_learning_type=transfer_learning_spec_pb2.TransferLearningSpec. NO_TRANSFER_LEARNING): # convolutions and then flatten plate. architecture = np.array([1, 3, 34]) str_signature = "_1334" input_tensor = tf.zeros([100, 32, 32, 3]) tower_name = "test_tower" transfer_learning_spec = transfer_learning_spec_pb2.TransferLearningSpec( transfer_learning_type=transfer_learning_type) phoenix_spec = phoenix_spec_pb2.PhoenixSpec( problem_type=phoenix_spec_pb2.PhoenixSpec.CNN, transfer_learning_spec=transfer_learning_spec) _ = architecture_utils.construct_tower( phoenix_spec=phoenix_spec, input_tensor=input_tensor, tower_name=tower_name, architecture=architecture, is_training=True, lengths=None, logits_dimension=10, hparams=hp.HParams(), model_directory=self.get_temp_dir(), is_frozen=False, dropout_rate=None) tensors = architecture_utils.get_tower_variables(tower_name) for tensor in tensors: if (transfer_learning_type == transfer_learning_spec_pb2. TransferLearningSpec.NO_TRANSFER_LEARNING): self.assertEndsWith(tensor.op.name, str_signature) else: self.assertNotEndsWith(tensor.op.name, str_signature) @parameterized.named_parameters( { "testcase_name": "shared_input", "shared_input": True }, { "testcase_name": "not_shared_input", "shared_input": False }) def test_import_tower(self, shared_input): np.random.seed(42) test_input = np.random.random([1, 32, 32, 3]) # Force graph mode with tf.compat.v1.Graph().as_default(): directory = self.get_temp_dir() architecture = np.array([1, 3, 34]) phoenix_spec = phoenix_spec_pb2.PhoenixSpec( problem_type=phoenix_spec_pb2.PhoenixSpec.CNN) phoenix_spec.is_input_shared = shared_input features = {} shared_input_tensor = None with self.test_session(graph=tf.Graph()) as sess: input_tensor_1 = tf.compat.v1.placeholder( dtype=tf.float32, shape=[None, 32, 32, 3], name="input_1") tf.random.set_seed(1234) tower_spec_1 = architecture_utils.construct_tower( phoenix_spec=phoenix_spec, input_tensor=input_tensor_1, tower_name="test_tower", architecture=architecture, is_training=True, lengths=None, logits_dimension=10, hparams=hp.HParams(), model_directory=self.get_temp_dir(), is_frozen=False, dropout_rate=None) saver = tf.compat.v1.train.Saver() sess.run(tf.compat.v1.global_variables_initializer()) sess.run(tf.compat.v1.local_variables_initializer()) logits_val_1 = sess.run(tower_spec_1.logits_spec.logits, feed_dict={input_tensor_1: test_input}) saver.save(sess, directory + "/ckpt") with self.test_session(graph=tf.Graph()) as sess: input_tensor_2 = tf.compat.v1.placeholder( dtype=tf.float32, shape=[None, 32, 32, 3], name="input_2") if shared_input: shared_input_tensor = input_tensor_2 def _input_layer_fn(features, is_training, scope_name="Phoenix/Input", lengths_feature_name=None): del features, is_training, scope_name, lengths_feature_name return None, None else: features = {"x": input_tensor_2} def _input_layer_fn(features, is_training, scope_name="Phoenix/Input", lengths_feature_name=None): del is_training, lengths_feature_name with tf.compat.v1.variable_scope(scope_name): return tf.cast(features["x"], dtype=tf.float32), None tf.random.set_seed(1234) tower_spec_2 = architecture_utils.import_tower( features=features, input_layer_fn=_input_layer_fn, phoenix_spec=phoenix_spec, shared_input_tensor=shared_input_tensor, original_tower_name="test_tower", new_tower_name="imported_tower", model_directory=directory, new_model_directory=self.get_temp_dir(), is_training=True, logits_dimension=10, shared_lengths=None, force_snapshot=False, force_freeze=False) sess.run(tf.compat.v1.global_variables_initializer()) sess.run(tf.compat.v1.local_variables_initializer()) logits_val_2 = sess.run(tower_spec_2.logits_spec.logits, feed_dict={input_tensor_2: test_input}) self.assertAllClose(logits_val_1, logits_val_2, rtol=1e-3) @parameterized.named_parameters( { "testcase_name": "cnn", "input_tensor_shape": [20, 20], "spec": _create_spec(phoenix_spec_pb2.PhoenixSpec.CNN), "output_shape": [20, 7], }, { "testcase_name": "dnn", "input_tensor_shape": [20, 20], "spec": _create_spec(phoenix_spec_pb2.PhoenixSpec.DNN), "output_shape": [20, 7], }, { "testcase_name": "rnn_all_activations", "input_tensor_shape": [20, 20, 20], "spec": _create_spec( phoenix_spec_pb2.PhoenixSpec.RNN_ALL_ACTIVATIONS), "output_shape": [20, 20, 7], }, { "testcase_name": "rnn_last_activations", "input_tensor_shape": [20, 20, 20], "spec": _create_spec( phoenix_spec_pb2.PhoenixSpec.RNN_LAST_ACTIVATIONS), "output_shape": [20, 7], "lengths": list(range(20)) }, { "testcase_name": "rnn_last_activations_no_length", "input_tensor_shape": [20, 20, 20], "spec": _create_spec( phoenix_spec_pb2.PhoenixSpec.RNN_LAST_ACTIVATIONS), "output_shape": [20, 20, 7], }, { "testcase_name": "nasnet_aux_head", "input_tensor_shape": [20, 20], "spec": _create_spec(phoenix_spec_pb2.PhoenixSpec.CNN), "output_shape": [20, 7], "extra_block": 6, # Downsample convolution. "extra_block_shape": [20, 20, 20, 20], "use_auxiliary_head": True }, { "testcase_name": "deep_skip_head", "input_tensor_shape": [20, 20], "spec": _create_spec(phoenix_spec_pb2.PhoenixSpec.CNN), "output_shape": [20, 7], "extra_block": 16, # Flatten. "use_auxiliary_head": True }) def test_create_tower_spec( self, input_tensor_shape, spec, output_shape, lengths=None, extra_block=14, # Fully connected. extra_block_shape=None, use_auxiliary_head=False): # Force graph mode with tf.compat.v1.Graph().as_default(): spec.use_auxiliary_head = use_auxiliary_head spec.auxiliary_head_loss_weight = .4 input_tensors = [ tf.zeros(extra_block_shape or (1, )), # Raw features -- discarded. tf.zeros(extra_block_shape or input_tensor_shape), # Extra block. tf.zeros(extra_block_shape or input_tensor_shape), # Extra block. tf.zeros(input_tensor_shape) # Fully connected block. ] fake_architecture = [extra_block, extra_block, 14] logits_spec, _, _ = architecture_utils.create_tower_spec( phoenix_spec=spec, inputs=input_tensors, architecture=fake_architecture, dimension=7, is_frozen=False, lengths=lengths, allow_auxiliary_head=use_auxiliary_head) self.assertAllEqual(output_shape, logits_spec.logits.shape) if use_auxiliary_head: self.assertIsNotNone(logits_spec.aux_logits) self.assertNear(logits_spec.aux_logits_weight, spec.auxiliary_head_loss_weight, 1e-6) self.assertNear(logits_spec.logits_weight, 1.0, 1e-6) @parameterized.named_parameters( { "testcase_name": "same_graph_snapshotting", "new_architecture": np.array([1, 3, 34]), "expected_output": [ "Phoenix/test_tower/1_FIXED_CHANNEL_CONVOLUTION_16_1/" "Conv/biases", "Phoenix/test_tower/1_FIXED_CHANNEL_CONVOLUTION_16_1/" "Conv/weights", "Phoenix/test_tower/2_FIXED_CHANNEL_CONVOLUTION_64_13/" "Conv/biases", "Phoenix/test_tower/2_FIXED_CHANNEL_CONVOLUTION_64_13/" "Conv/weights", "Phoenix/test_tower/last_dense_1334/dense/bias", "Phoenix/test_tower/last_dense_1334/dense/kernel", "Phoenix/test_tower/2_FIXED_CHANNEL_CONVOLUTION_64_13/BatchNorm/" "beta", "Phoenix/test_tower/1_FIXED_CHANNEL_CONVOLUTION_16_1/BatchNorm/" "moving_variance", "Phoenix/test_tower/2_FIXED_CHANNEL_CONVOLUTION_64_13/BatchNorm/" "moving_variance", "Phoenix/test_tower/1_FIXED_CHANNEL_CONVOLUTION_16_1/BatchNorm/" "moving_mean", "Phoenix/test_tower/1_FIXED_CHANNEL_CONVOLUTION_16_1/BatchNorm/" "beta", "Phoenix/test_tower/2_FIXED_CHANNEL_CONVOLUTION_64_13/BatchNorm/" "moving_mean", ] }, { "testcase_name": "same_graph_snapshotting_new_towername", "new_architecture": np.array([1, 3, 34]), "expected_output": [ "Phoenix/test_tower/1_FIXED_CHANNEL_CONVOLUTION_16_1/" "Conv/biases", "Phoenix/test_tower/1_FIXED_CHANNEL_CONVOLUTION_16_1/" "Conv/weights", "Phoenix/test_tower/2_FIXED_CHANNEL_CONVOLUTION_64_13/" "Conv/biases", "Phoenix/test_tower/2_FIXED_CHANNEL_CONVOLUTION_64_13/" "Conv/weights", "Phoenix/test_tower/last_dense_1334/dense/bias", "Phoenix/test_tower/last_dense_1334/dense/kernel", "Phoenix/test_tower/2_FIXED_CHANNEL_CONVOLUTION_64_13/BatchNorm/" "beta", "Phoenix/test_tower/1_FIXED_CHANNEL_CONVOLUTION_16_1/BatchNorm/" "moving_variance", "Phoenix/test_tower/2_FIXED_CHANNEL_CONVOLUTION_64_13/BatchNorm/" "moving_variance", "Phoenix/test_tower/1_FIXED_CHANNEL_CONVOLUTION_16_1/BatchNorm/" "moving_mean", "Phoenix/test_tower/1_FIXED_CHANNEL_CONVOLUTION_16_1/BatchNorm/" "beta", "Phoenix/test_tower/2_FIXED_CHANNEL_CONVOLUTION_64_13/BatchNorm/" "moving_mean", ], "new_tower_name": "test_tower_2" }, { "testcase_name": "changing_second", "new_architecture": np.array([1, 2, 34]), "expected_output": [ "Phoenix/test_tower/1_FIXED_CHANNEL_CONVOLUTION_16_1/" "Conv/weights", "Phoenix/test_tower/1_FIXED_CHANNEL_CONVOLUTION_16_1/Conv/biases", "Phoenix/test_tower/1_FIXED_CHANNEL_CONVOLUTION_16_1/BatchNorm/" "moving_mean", "Phoenix/test_tower/1_FIXED_CHANNEL_CONVOLUTION_16_1/BatchNorm/" "moving_variance", "Phoenix/test_tower/1_FIXED_CHANNEL_CONVOLUTION_16_1/BatchNorm/" "beta" ] }) def test_init_variables(self, new_architecture, expected_output, new_tower_name="test_tower"): # Force graph mode with tf.compat.v1.Graph().as_default(): directory = self.get_temp_dir() architecture = np.array([1, 3, 34]) phoenix_spec = phoenix_spec_pb2.PhoenixSpec( problem_type=phoenix_spec_pb2.PhoenixSpec.CNN) with self.test_session(graph=tf.Graph()) as sess: input_tensor = tf.zeros([100, 32, 32, 3]) _ = architecture_utils.construct_tower( phoenix_spec=phoenix_spec, input_tensor=input_tensor, tower_name="test_tower", architecture=architecture, is_training=True, lengths=None, logits_dimension=10, model_directory=self.get_temp_dir(), hparams=hp.HParams(), is_frozen=False, dropout_rate=None) saver = tf.compat.v1.train.Saver() sess.run(tf.compat.v1.global_variables_initializer()) sess.run(tf.compat.v1.local_variables_initializer()) saver.save(sess, directory + "/ckpt") with self.test_session(graph=tf.Graph()) as sess: input_tensor = tf.zeros([100, 32, 32, 3]) _ = architecture_utils.construct_tower( phoenix_spec=phoenix_spec, input_tensor=input_tensor, tower_name=new_tower_name, architecture=new_architecture, is_training=True, lengths=None, logits_dimension=10, hparams=hp.HParams(), model_directory=self.get_temp_dir(), is_frozen=False, dropout_rate=None) snapshotting_variables = architecture_utils.init_variables( tf.train.latest_checkpoint(directory), "Phoenix/test_tower", "Phoenix/{}".format(new_tower_name)) self.assertCountEqual(snapshotting_variables, expected_output)
def test_write_replay_spec(self): original_spec = phoenix_spec_pb2.PhoenixSpec() text_format.Parse( """ minimum_depth: 2 maximum_depth: 20 problem_type: CNN search_type: ADAPTIVE_COORDINATE_DESCENT user_suggestions { architecture: "LSTM_128" } """, original_spec) _write_dependency_reply(self.get_temp_dir()) trial_utils.write_replay_spec( model_dir=self.get_temp_dir(), filename="replay_config.pbtxt", original_spec=original_spec, search_architecture=["RNN_128", "RNN_128"], hparams=hp.HParams( learning_rate=0.001, initial_architecture=["not_cor_arc", "not_cor_arc2"], context="should_be_deleted")) output_spec = phoenix_spec_pb2.PhoenixSpec() with tf.io.gfile.GFile(self.get_temp_dir() + "/replay_config.pbtxt", "r") as f: text_format.Parse(f.read(), output_spec) expected_spec = phoenix_spec_pb2.PhoenixSpec() text_format.Parse( """ minimum_depth: 2 maximum_depth: 20 problem_type: CNN search_type: ADAPTIVE_COORDINATE_DESCENT replay { towers { architecture: "RNN_128" architecture: "RNN_128" hparams { hparam { key: "learning_rate" value { float_value: 0.021 } } hparam { key: "initial_architecture" value { bytes_list { value: "RNN_256" value: "RNN_128" } } } } } towers { architecture: "RNN_128" architecture: "RNN_128" hparams { hparam { key: "learning_rate" value { float_value: 0.001 } } hparam { key: "initial_architecture" value { bytes_list { value: "RNN_128" value: "RNN_128" } } } } } } """, expected_spec) self.assertEqual(output_spec, expected_spec)
def main(unused_argv): filename = FLAGS.phoenix_spec_filename spec = phoenix_spec_pb2.PhoenixSpec() with tf.io.gfile.GFile(filename, "r") as f: text_format.Merge(f.read(), spec) dataset_provider = get_dataset_provider() loss_fn, metric_fn, predictions_fn = ( loss_and_metric_and_predictions_fn(dataset_provider)) metadata = None if (FLAGS.optimization_goal != "minimize" or FLAGS.optimization_metric != "loss"): metadata = ml_metadata_db.MLMetaData( phoenix_spec=spec, study_name=FLAGS.experiment_name, study_owner=FLAGS.experiment_owner, optimization_goal=FLAGS.optimization_goal, optimization_metric=FLAGS.optimization_metric) phoenix_instance = phoenix.Phoenix( phoenix_spec=spec, input_layer_fn=dataset_provider.get_input_layer_fn(spec.problem_type), logits_dimension=dataset_provider.number_of_classes(), study_name=FLAGS.experiment_name, study_owner=FLAGS.experiment_owner, loss_fn=loss_fn, metric_fn=metric_fn, predictions_fn=predictions_fn, metadata=metadata) # Replay only!! if spec.HasField("replay"): hparams = hp.HParams.from_proto(spec.replay.towers[0].hparams) run_train_and_eval(hparams, FLAGS.model_dir, phoenix_instance, dataset_provider, train_steps=FLAGS.phoenix_train_steps, eval_steps=FLAGS.phoenix_eval_steps, batch_size=FLAGS.phoenix_batch_size) return 0 tuner_id = FLAGS.tuner_id or "phoenix-tuner-%d" % random.randint( 0, 10000000) hyperparameters = phoenix.Phoenix.get_keras_hyperparameters_space( spec, FLAGS.phoenix_train_steps) if FLAGS.hypertuning_method == "random": oracle = kerastuner.tuners.randomsearch.RandomSearchOracle( objective="loss", max_trials=FLAGS.experiment_max_num_trials, seed=73, hyperparameters=hyperparameters, allow_new_entries=True, tune_new_entries=True) else: oracle = kerastuner.tuners.bayesian.BayesianOptimizationOracle( objective="loss", hyperparameters=hyperparameters, max_trials=FLAGS.experiment_max_num_trials) # pylint: disable=protected-access oracle._set_project_dir(FLAGS.model_dir, FLAGS.experiment_name, overwrite=True) # pylint: enable=protected-access data_provider = get_dataset_provider() while run_parameterized_train_and_eval( phoenix_instance=phoenix_instance, oracle=oracle, tuner_id=tuner_id, root_dir=FLAGS.model_dir, max_trials=FLAGS.experiment_max_num_trials, data_provider=data_provider, train_steps=FLAGS.phoenix_train_steps, eval_steps=FLAGS.phoenix_eval_steps, batch_size=FLAGS.phoenix_batch_size): pass