def test_generator_with_dropouts(self):
# Force graph mode
with tf.compat.v1.Graph().as_default():
spec = phoenix_spec_pb2.PhoenixSpec(
problem_type=phoenix_spec_pb2.PhoenixSpec.DNN)
spec.search_type = phoenix_spec_pb2.PhoenixSpec.NONADAPTIVE_RANDOM_SEARCH
spec.is_input_shared = True
generator = search_candidate_generator.SearchCandidateGenerator(
phoenix_spec=spec,
metadata=ml_metadata_db.MLMetaData(phoenix_spec=spec,
study_name='',
study_owner=''))
input_tensor = tf.zeros([20, 32, 32, 3])
fake_config = collections.namedtuple('RunConfig',
['model_dir', 'is_chief'])
run_config = fake_config(model_dir=flags.FLAGS.test_tmpdir + '/1',
is_chief=True)
_ = generator.generate(
features={},
input_layer_fn=lambda: None,
trial_mode=trial_utils.TrialMode.NO_PRIOR,
shared_input_tensor=input_tensor,
shared_lengths=None,
logits_dimension=10,
hparams=hp.HParams(initial_architecture=['CONVOLUTION_3X3'],
dropout_rate=0.3),
run_config=run_config,
is_training=True,
trials=[])
all_nodes = [
node.name for node in
tf.compat.v1.get_default_graph().as_graph_def().node
]
self.assertAllInSet(_DROPOUT_GRAPH_NODE, all_nodes)
def test_construct_tower_with_transfer_learning(
self,
transfer_learning_type=transfer_learning_spec_pb2.TransferLearningSpec.
NO_TRANSFER_LEARNING):
# convolutions and then flatten plate.
architecture = np.array([1, 3, 34])
str_signature = "_1334"
input_tensor = tf.zeros([100, 32, 32, 3])
tower_name = "test_tower"
transfer_learning_spec = transfer_learning_spec_pb2.TransferLearningSpec(
transfer_learning_type=transfer_learning_type)
phoenix_spec = phoenix_spec_pb2.PhoenixSpec(
problem_type=phoenix_spec_pb2.PhoenixSpec.CNN,
transfer_learning_spec=transfer_learning_spec)
_ = architecture_utils.construct_tower(
phoenix_spec=phoenix_spec,
input_tensor=input_tensor,
tower_name=tower_name,
architecture=architecture,
is_training=True,
lengths=None,
logits_dimension=10,
hparams=hp.HParams(),
model_directory=self.get_temp_dir(),
is_frozen=False,
dropout_rate=None)
tensors = architecture_utils.get_tower_variables(tower_name)
for tensor in tensors:
if (transfer_learning_type == transfer_learning_spec_pb2.
TransferLearningSpec.NO_TRANSFER_LEARNING):
self.assertEndsWith(tensor.op.name, str_signature)
else:
self.assertNotEndsWith(tensor.op.name, str_signature)
def _create_checkpoint(self, towers, trial_id):
with self.test_session(graph=tf.Graph()) as sess:
architecture = np.array([1, 3, 34])
input_tensor = tf.zeros([100, 32, 32, 3])
phoenix_spec = phoenix_spec_pb2.PhoenixSpec(
problem_type=phoenix_spec_pb2.PhoenixSpec.CNN)
dirname = os.path.join(flags.FLAGS.test_tmpdir, str(trial_id))
if dirname and not tf.io.gfile.exists(dirname):
tf.io.gfile.makedirs(dirname)
for tower in towers:
_ = architecture_utils.construct_tower(
phoenix_spec=phoenix_spec,
input_tensor=input_tensor,
tower_name=str(tower) + '_0',
architecture=architecture,
is_training=True,
lengths=None,
logits_dimension=10,
hparams=hp.HParams(),
model_directory=dirname,
is_frozen=False,
dropout_rate=None)
architecture_utils.set_number_of_towers(tower, 1)
architecture_utils.set_number_of_towers('replay_generator', 0)
directory = flags.FLAGS.test_tmpdir
saver = tf.compat.v1.train.Saver()
sess.run(tf.compat.v1.global_variables_initializer())
sess.run(tf.compat.v1.local_variables_initializer())
saver.save(sess, os.path.join(directory, str(trial_id)) + '/ckpt')
def test_tpu(self):
# Force graph mode
with tf.compat.v1.Graph().as_default():
learning_rate_spec = {
'learning_rate': 0.001,
'gradient_max_norm': 3
}
spec = phoenix_spec_pb2.PhoenixSpec(
problem_type=phoenix_spec_pb2.PhoenixSpec.DNN)
task_manager_instance = task_manager.TaskManager(spec)
logits = tf.keras.layers.Dense(10)(tf.zeros([20, 10]))
logits_spec = architecture_utils.LogitsSpec(logits=logits)
features = {'x': tf.zeros([10, 10])}
loss_fn = loss_fns.make_multi_class_loss_fn()
_ = task_manager_instance.create_model_spec(
features=features,
params=hp.HParams(optimizer='sgd'),
learning_rate_spec=learning_rate_spec,
train_logits_specs=[logits_spec],
eval_logits_spec=logits_spec,
labels=tf.ones([20], dtype=tf.int32),
loss_fn=loss_fn,
mode=tf.estimator.ModeKeys.TRAIN,
lengths=None,
use_tpu=True,
predictions_fn=_default_predictions_fn)
self.assertNotEmpty([
node.name for node in
tf.compat.v1.get_default_graph().as_graph_def().node
if 'CrossReplicaSum' in node.name
])
def test_learning_spec_on_predict(self, learning_rate_spec,
not_containing):
spec = phoenix_spec_pb2.PhoenixSpec(
problem_type=phoenix_spec_pb2.PhoenixSpec.DNN)
task_manager_instance = task_manager.TaskManager(spec)
logits = tf.keras.layers.Dense(10)(tf.zeros([20, 10]))
logits_spec = architecture_utils.LogitsSpec(logits=logits)
features = {'x': tf.zeros([10, 10])}
loss_fn = loss_fns.make_multi_class_loss_fn()
model = task_manager_instance.create_model_spec(
features=features,
params=hp.HParams(optimizer='sgd'),
learning_rate_spec=learning_rate_spec,
train_logits_specs=[logits_spec],
eval_logits_spec=logits_spec,
labels=tf.ones([20], dtype=tf.int32),
loss_fn=loss_fn,
mode=tf.estimator.ModeKeys.PREDICT,
lengths=None,
use_tpu=False,
predictions_fn=_default_predictions_fn)
for phrase in not_containing:
self.assertEmpty([
node.name for node in
tf.compat.v1.get_default_graph().as_graph_def().node
if phrase in node.name
])
self.assertLen(model.predictions, 3)
self.assertIn('probabilities', model.predictions)
self.assertIn('log_probabilities', model.predictions)
self.assertIn('predictions', model.predictions)
self.assertIsNone(model.loss)
def test_ensembler(self, combining_type, search_type, priors, logits,
output_graph):
# Force graph mode
with tf.compat.v1.Graph().as_default():
spec = phoenix_spec_pb2.PhoenixSpec()
spec.ensemble_spec.combining_type = combining_type
spec.ensemble_spec.ensemble_search_type = search_type
ensembler_instance = ensembler.Ensembler(spec)
priors_logits_specs = []
search_logits_specs = []
if priors:
for _ in range(priors):
spec = architecture_utils.LogitsSpec(
logits=tf.zeros([20, 10]))
priors_logits_specs.append(spec)
if logits:
spec = architecture_utils.LogitsSpec(
logits=tf.keras.layers.Dense(10)(tf.zeros([20, 10])))
search_logits_specs.append(spec)
_ = ensembler_instance.bundle_logits(
priors_logits_specs=priors_logits_specs,
search_logits_specs=search_logits_specs,
logits_dimension=10)
nodes = tf.compat.v1.get_default_graph().as_graph_def().node
logging.info([node.name for node in nodes])
self.assertCountEqual([n.name for n in nodes], output_graph)
def test_architecture(self):
# Force graph mode
with tf.compat.v1.Graph().as_default():
learning_rate_spec = {'learning_rate': 0.001, 'gradient_max_norm': 3}
spec = phoenix_spec_pb2.PhoenixSpec(
problem_type=phoenix_spec_pb2.PhoenixSpec.CNN)
text_format.Merge(
"""
multi_task_spec {
label_name: "label1"
number_of_classes: 10
architecture: "FIXED_OUTPUT_FULLY_CONNECTED_128"
}
multi_task_spec {
label_name: "label2"
number_of_classes: 10
architecture: "FIXED_OUTPUT_FULLY_CONNECTED_256"
architecture: "FIXED_OUTPUT_FULLY_CONNECTED_512"
}
""", spec)
task_manager_instance = task_manager.TaskManager(spec)
logits = tf.keras.layers.Dense(10)(tf.zeros([20, 10]))
logits_spec = architecture_utils.LogitsSpec(logits=logits)
features = {'x': tf.zeros([10, 10])}
loss_fn = loss_fns.make_multi_class_loss_fn()
model = task_manager_instance.create_model_spec(
features=features,
params=hp.HParams(optimizer='sgd'),
learning_rate_spec=learning_rate_spec,
train_logits_specs=[logits_spec],
eval_logits_spec=logits_spec,
labels={
'label1': tf.ones([20], dtype=tf.int32),
'label2': tf.ones([20], dtype=tf.int32)
},
loss_fn=loss_fn,
model_directory=self.get_temp_dir(),
mode=tf.estimator.ModeKeys.TRAIN,
lengths=None,
use_tpu=False,
predictions_fn=_default_predictions_fn)
self.assertNotEmpty([
node.name
for node in tf.compat.v1.get_default_graph().as_graph_def().node
if 'task_label1_tower/1_FIXED_OUTPUT_FULLY_CONNECTED_128' in node.name
])
self.assertNotEmpty([
node.name
for node in tf.compat.v1.get_default_graph().as_graph_def().node
if 'task_label2_tower/1_FIXED_OUTPUT_FULLY_CONNECTED_256' in node.name
])
self.assertNotEmpty([
node.name
for node in tf.compat.v1.get_default_graph().as_graph_def().node
if 'task_label2_tower/2_FIXED_OUTPUT_FULLY_CONNECTED_512' in node.name
])
self.assertLen(model.predictions, 3 * (1 + 2))
self.assertIn('probabilities', model.predictions)
self.assertIn('log_probabilities', model.predictions)
self.assertIn('predictions', model.predictions)
def test_construct_network(self, dropout, expected_logits):
# Force graph mode
with tf.compat.v1.Graph().as_default():
tf.random.set_seed(1234)
# convolutions and then flatten plate.
architecture = np.array([1, 3, 34])
input_tensor = tf.compat.v1.placeholder(dtype=tf.float32,
shape=[None, 32, 32, 3],
name="input")
phoenix_spec = phoenix_spec_pb2.PhoenixSpec(
problem_type=phoenix_spec_pb2.PhoenixSpec.CNN)
tower_spec = architecture_utils.construct_tower(
phoenix_spec=phoenix_spec,
input_tensor=input_tensor,
tower_name="test_tower",
architecture=architecture,
is_training=True,
lengths=None,
logits_dimension=10,
hparams=hp.HParams(),
model_directory=self.get_temp_dir(),
is_frozen=False,
dropout_rate=dropout)
np.random.seed(42)
test_input = np.random.random([1, 32, 32, 3])
with tf.compat.v1.Session() as sess:
sess.run([
tf.compat.v1.global_variables_initializer(),
tf.compat.v1.local_variables_initializer()
])
logits_val = sess.run(tower_spec.logits_spec.logits,
feed_dict={input_tensor: test_input})
self.assertAllClose(expected_logits, logits_val, rtol=1e-3)
def _create_phoenix_spec(self, problem_type):
"""Creates a new phoenix.PhoenixSpec for the given problem type."""
spec_path = os.path.join(FLAGS.test_srcdir,
_SPEC_PATH_TEMPLATE.format(problem_type))
spec = phoenix_spec_pb2.PhoenixSpec()
with tf.io.gfile.GFile(spec_path, "r") as f:
text_format.Merge(f.read(), spec)
return spec
def test_preconditions(self, kwargs_to_pass=None):
if "phoenix_spec" not in kwargs_to_pass:
kwargs_to_pass["phoenix_spec"] = phoenix_spec_pb2.PhoenixSpec()
kwargs_to_pass["input_layer_fn"] = lambda: None
kwargs_to_pass["study_owner"] = "fake_owner"
kwargs_to_pass["study_name"] = "fake_name"
with self.assertRaises(AssertionError):
phoenix.Phoenix(**kwargs_to_pass)
def test_wrong_dict_weight_feature(self, weight_is_a_feature):
learning_rate_spec = {'learning_rate': 0.001, 'gradient_max_norm': 3}
spec = phoenix_spec_pb2.PhoenixSpec(
problem_type=phoenix_spec_pb2.PhoenixSpec.DNN)
text_format.Merge(
"""
multi_task_spec {
label_name: "label1"
number_of_classes: 10
weight_feature_name: "weight1"
weight_is_a_feature: %s
}
multi_task_spec {
label_name: "label2"
number_of_classes: 10
weight_feature_name: "weight2"
weight_is_a_feature: %s
}
""" % (str(weight_is_a_feature), str(weight_is_a_feature)), spec)
labels = {
'label1': tf.ones([20], dtype=tf.int32),
'label2': tf.ones([20], dtype=tf.int32),
}
# Fix the size of the dict labels to bypass the assertion.
if not weight_is_a_feature:
labels.update({
'not_used': tf.ones([20], dtype=tf.int32),
'not_used2': tf.ones([20], dtype=tf.int32)
})
weights = {
'weight1': tf.constant([2] * 20),
'weight2': tf.constant([3] * 20)
}
features = {'x': tf.zeros([10, 10])}
if not weight_is_a_feature:
features.update(weights)
task_manager_instance = task_manager.TaskManager(spec)
logits = tf.keras.layers.Dense(10)(tf.zeros([20, 10]))
logits_spec = architecture_utils.LogitsSpec(logits=logits)
with self.assertRaises(KeyError):
loss_fn = loss_fns.make_multi_class_loss_fn()
_ = task_manager_instance.create_model_spec(
features=features,
params=hp.HParams(optimizer='sgd'),
learning_rate_spec=learning_rate_spec,
train_logits_specs=[logits_spec],
eval_logits_spec=logits_spec,
labels=labels,
loss_fn=loss_fn,
model_directory=self.get_temp_dir(),
mode=tf.estimator.ModeKeys.TRAIN,
lengths=None,
use_tpu=False,
predictions_fn=_default_predictions_fn)
def test_weight_feature(self, is_multitask, weight_is_a_feature):
# Force graph mode
with tf.compat.v1.Graph().as_default():
learning_rate_spec = {
'learning_rate': 0.001,
'gradient_max_norm': 3
}
spec = phoenix_spec_pb2.PhoenixSpec(
problem_type=phoenix_spec_pb2.PhoenixSpec.DNN)
labels = tf.ones([20], dtype=tf.int32)
if is_multitask:
text_format.Merge(
"""
multi_task_spec {
label_name: "label1"
number_of_classes: 10
weight_feature_name: "weight1"
weight_is_a_feature: %s
}
multi_task_spec {
label_name: "label2"
number_of_classes: 10
weight_feature_name: "weight2"
weight_is_a_feature: %s
}
""" % (str(weight_is_a_feature), str(weight_is_a_feature)), spec)
labels = {
'label1': tf.ones([20], dtype=tf.int32),
'label2': tf.ones([20], dtype=tf.int32)
}
weights = {
'weight1': tf.constant([2] * 20),
'weight2': tf.constant([3] * 20)
}
features = {'x': tf.zeros([10, 10])}
if weight_is_a_feature:
features.update(weights)
elif isinstance(labels, dict):
labels.update(weights)
task_manager_instance = task_manager.TaskManager(spec)
logits = tf.keras.layers.Dense(10)(tf.zeros([20, 10]))
logits_spec = architecture_utils.LogitsSpec(logits=logits)
loss_fn = loss_fns.make_multi_class_loss_fn()
_ = task_manager_instance.create_model_spec(
features=features,
params=hp.HParams(optimizer='sgd'),
learning_rate_spec=learning_rate_spec,
train_logits_specs=[logits_spec],
eval_logits_spec=logits_spec,
labels=labels,
loss_fn=loss_fn,
mode=tf.estimator.ModeKeys.TRAIN,
lengths=None,
use_tpu=False,
predictions_fn=_default_predictions_fn)
def _create_spec(ensemble_type='none'):
output = phoenix_spec_pb2.PhoenixSpec()
if ensemble_type == 'adaptive':
output.ensemble_spec.ensemble_search_type = (
ensembling_spec_pb2.EnsemblingSpec.ADAPTIVE_ENSEMBLE_SEARCH)
if ensemble_type == 'nonadaptive':
output.ensemble_spec.ensemble_search_type = (
ensembling_spec_pb2.EnsemblingSpec.NONADAPTIVE_ENSEMBLE_SEARCH)
return output
def __init__(self, data, spec):
self._data = data
if isinstance(spec, str):
self._spec = phoenix_spec_pb2.PhoenixSpec()
with tf.io.gfile.GFile(spec, "r") as f:
text_format.Parse(f.read(), self._spec)
else:
self._spec = spec
self._tuner_id = "tuner-1"
def _create_spec(problem_type,
complexity_thresholds=None,
max_depth=None,
min_depth=None):
output = phoenix_spec_pb2.PhoenixSpec()
if complexity_thresholds is not None:
output.increase_complexity_minimum_trials[:] = complexity_thresholds
if max_depth is not None:
output.maximum_depth = max_depth
if min_depth is not None:
output.minimum_depth = min_depth
output.problem_type = problem_type
return output
def test_init_variables(self,
new_architecture,
expected_output,
new_tower_name="test_tower"):
# Force graph mode
with tf.compat.v1.Graph().as_default():
directory = self.get_temp_dir()
architecture = np.array([1, 3, 34])
phoenix_spec = phoenix_spec_pb2.PhoenixSpec(
problem_type=phoenix_spec_pb2.PhoenixSpec.CNN)
with self.test_session(graph=tf.Graph()) as sess:
input_tensor = tf.zeros([100, 32, 32, 3])
_ = architecture_utils.construct_tower(
phoenix_spec=phoenix_spec,
input_tensor=input_tensor,
tower_name="test_tower",
architecture=architecture,
is_training=True,
lengths=None,
logits_dimension=10,
model_directory=self.get_temp_dir(),
hparams=hp.HParams(),
is_frozen=False,
dropout_rate=None)
saver = tf.compat.v1.train.Saver()
sess.run(tf.compat.v1.global_variables_initializer())
sess.run(tf.compat.v1.local_variables_initializer())
saver.save(sess, directory + "/ckpt")
with self.test_session(graph=tf.Graph()) as sess:
input_tensor = tf.zeros([100, 32, 32, 3])
_ = architecture_utils.construct_tower(
phoenix_spec=phoenix_spec,
input_tensor=input_tensor,
tower_name=new_tower_name,
architecture=new_architecture,
is_training=True,
lengths=None,
logits_dimension=10,
hparams=hp.HParams(),
model_directory=self.get_temp_dir(),
is_frozen=False,
dropout_rate=None)
snapshotting_variables = architecture_utils.init_variables(
tf.train.latest_checkpoint(directory),
"Phoenix/test_tower", "Phoenix/{}".format(new_tower_name))
self.assertCountEqual(snapshotting_variables, expected_output)
def create_spec(problem_type,
complexity_thresholds=None,
max_depth=None,
min_depth=None,
blocks_to_use=None):
"""Creates a phoenix_spec_pb2.PhoenixSpec with the given options."""
output = phoenix_spec_pb2.PhoenixSpec()
if complexity_thresholds is not None:
output.increase_complexity_minimum_trials[:] = complexity_thresholds
if max_depth is not None:
output.maximum_depth = max_depth
if min_depth is not None:
output.minimum_depth = min_depth
output.problem_type = problem_type
if blocks_to_use:
output.blocks_to_use[:] = blocks_to_use
return output
def test_intermixed_prior_graph(self):
# Force graph mode
with tf.compat.v1.Graph().as_default():
spec = phoenix_spec_pb2.PhoenixSpec(
problem_type=phoenix_spec_pb2.PhoenixSpec.DNN)
spec.ensemble_spec.ensemble_search_type = (
ensembling_spec_pb2.EnsemblingSpec.
INTERMIXED_NONADAPTIVE_ENSEMBLE_SEARCH)
spec.ensemble_spec.intermixed_search.width = 2
spec.ensemble_spec.intermixed_search.try_ensembling_every = 4
spec.ensemble_spec.intermixed_search.num_trials_to_consider = 3
spec.is_input_shared = True
generator = prior_generator.PriorGenerator(
phoenix_spec=spec,
metadata=ml_metadata_db.MLMetaData(phoenix_spec=spec,
study_name='',
study_owner=''))
fake_config = collections.namedtuple('RunConfig', ['model_dir'])
# Should be multplication of 4.
run_config = fake_config(model_dir=flags.FLAGS.test_tmpdir +
'/10000')
tf.io.gfile.makedirs(run_config.model_dir)
# Best three trials checkpoint are generated. If the generator chooses
# the suboptimal (wrong) trials, the test will fail.
self._create_checkpoint(['search_generator'], 2)
self._create_checkpoint(['search_generator'], 3)
self._create_checkpoint(['search_generator'], 5)
logits, _ = generator.first_time_chief_generate(
features={},
input_layer_fn=lambda: None,
trial_mode=trial_utils.TrialMode.ENSEMBLE_SEARCH,
shared_input_tensor=tf.zeros([100, 32, 32, 3]),
shared_lengths=None,
logits_dimension=10,
hparams={},
run_config=run_config,
is_training=True,
trials=trial_utils.create_test_trials_intermixed(
flags.FLAGS.test_tmpdir))
self.assertLen(logits, 2)
all_nodes = [
node.name for node in
tf.compat.v1.get_default_graph().as_graph_def().node
]
self.assertAllInSet(_NONADAPTIVE_GRAPH_NODES, all_nodes)
def _create_spec(replay_towers=0, ensemble_type="none"):
output = phoenix_spec_pb2.PhoenixSpec()
output.search_type = phoenix_spec_pb2.PhoenixSpec.NONADAPTIVE_RANDOM_SEARCH
if replay_towers:
output.replay.CopyFrom(phoenix_spec_pb2.ArchitectureReplay())
for _ in range(replay_towers):
output.replay.towers.add()
if ensemble_type == "adaptive":
output.ensemble_spec.ensemble_search_type = (
ensembling_spec_pb2.EnsemblingSpec.ADAPTIVE_ENSEMBLE_SEARCH)
if ensemble_type == "nonadaptive":
output.ensemble_spec.ensemble_search_type = (
ensembling_spec_pb2.EnsemblingSpec.NONADAPTIVE_ENSEMBLE_SEARCH)
if ensemble_type == "intermix":
output.ensemble_spec.ensemble_search_type = (
ensembling_spec_pb2.EnsemblingSpec.
INTERMIXED_NONADAPTIVE_ENSEMBLE_SEARCH)
return output
def test_generator_with_distillation_and_intermixed(self):
# Force graph mode
with tf.compat.v1.Graph().as_default():
spec = phoenix_spec_pb2.PhoenixSpec(
problem_type=phoenix_spec_pb2.PhoenixSpec.CNN)
spec.is_input_shared = True
spec.search_type = phoenix_spec_pb2.PhoenixSpec.NONADAPTIVE_RANDOM_SEARCH
spec.ensemble_spec.ensemble_search_type = (
ensembling_spec_pb2.EnsemblingSpec.
INTERMIXED_NONADAPTIVE_ENSEMBLE_SEARCH)
spec.ensemble_spec.intermixed_search.width = 2
spec.ensemble_spec.intermixed_search.try_ensembling_every = 4
spec.ensemble_spec.intermixed_search.num_trials_to_consider = 3
spec.distillation_spec.distillation_type = (
distillation_spec_pb2.DistillationSpec.DistillationType.
MSE_LOGITS)
generator = search_candidate_generator.SearchCandidateGenerator(
phoenix_spec=spec,
metadata=ml_metadata_db.MLMetaData(phoenix_spec=spec,
study_name='',
study_owner=''))
fake_config = collections.namedtuple('RunConfig',
['model_dir', 'is_chief'])
run_config = fake_config(model_dir=flags.FLAGS.test_tmpdir +
'/10000',
is_chief=True)
self._create_checkpoint(['search_generator'], 2)
self._create_checkpoint(['search_generator'], 3)
self._create_checkpoint(['search_generator'], 5)
input_tensor = tf.zeros([20, 32, 32, 3])
_ = generator.generate(
features={},
input_layer_fn=lambda: None,
trial_mode=trial_utils.TrialMode.DISTILLATION,
shared_input_tensor=input_tensor,
shared_lengths=None,
logits_dimension=10,
hparams=hp.HParams(initial_architecture=['CONVOLUTION_3X3']),
run_config=run_config,
is_training=True,
trials=trial_utils.create_test_trials_intermixed(
flags.FLAGS.test_tmpdir))
def test_get_suggestion(self, get_architecture, randint, completed_trials,
initial_architecture, expected_architecture,
replicate_cell):
get_architecture.return_value = initial_architecture
randint.return_value = 2
hparams = hp.HParams(
new_block_type="FIXED_CHANNEL_CONVOLUTION_16") # id=1.
# Use problem_type=DNN so we don't have to worry about the final
# architecture being correct.
spec = phoenix_spec_pb2.PhoenixSpec(
problem_type=phoenix_spec_pb2.PhoenixSpec.DNN,
maximum_depth=11,
num_blocks_in_cell=4,
reduction_block_type="AVERAGE_POOL_2X2",
replicate_cell=replicate_cell,
beam_size=3)
algorithm = constrained_descent.ConstrainedDescent(
spec, self._metadata)
trials = []
for i in range(completed_trials):
trials.append(
trial_module.Trial({
"id": i,
"model_dir": "/tmp/" + str(i),
"status": "COMPLETED",
"trial_infeasible": False,
"final_measurement": {
"objective_value": 100 - i
}
}))
actual_architecture, fork_trial = algorithm.get_suggestion(
trials, hparams)
# We use a beam of size 3 and always choose the last trial to be the best.
self.assertEqual(fork_trial, completed_trials - 3)
self.assertTrue(
search_test_utils.is_mutation_or_equal(actual_architecture,
expected_architecture))
def test_generator_with_snapshot(self):
# Force graph mode
with tf.compat.v1.Graph().as_default():
spec = phoenix_spec_pb2.PhoenixSpec(
problem_type=phoenix_spec_pb2.PhoenixSpec.CNN)
spec.search_type = phoenix_spec_pb2.PhoenixSpec.ADAPTIVE_COORDINATE_DESCENT
spec.transfer_learning_spec.transfer_learning_type = (
transfer_learning_spec_pb2.TransferLearningSpec.
SNAPSHOT_TRANSFER_LEARNING)
spec.is_input_shared = True
generator = search_candidate_generator.SearchCandidateGenerator(
phoenix_spec=spec,
metadata=ml_metadata_db.MLMetaData(phoenix_spec=spec,
study_name='',
study_owner=''))
input_tensor = tf.zeros([20, 32, 32, 3])
fake_config = collections.namedtuple('RunConfig',
['model_dir', 'is_chief'])
tf.io.gfile.makedirs(flags.FLAGS.test_tmpdir + '/3')
run_config = fake_config(model_dir=flags.FLAGS.test_tmpdir + '/3',
is_chief=True)
self._create_checkpoint(['search_generator'], 2)
_ = generator.generate(
features={},
input_layer_fn=lambda: None,
trial_mode=trial_utils.TrialMode.ENSEMBLE_SEARCH,
shared_input_tensor=input_tensor,
shared_lengths=None,
logits_dimension=10,
hparams=hp.HParams(initial_architecture=['CONVOLUTION_3X3'],
dropout_rate=0.3,
new_block_type='CONVOLUTION_3X3'),
run_config=run_config,
is_training=True,
trials=_create_trials(flags.FLAGS.test_tmpdir))
all_nodes = [
node.name for node in
tf.compat.v1.get_default_graph().as_graph_def().node
]
self.assertAllInSet(_DROPOUT_GRAPH_NODE, all_nodes)
def write_replay_spec(model_dir, filename, original_spec, search_architecture,
hparams):
"""Writes a replay spec to retrain the same model."""
# Ensure the same search space as the original run
replay_spec = copy.deepcopy(original_spec)
# Remove user suggestions
replay_spec.ClearField('user_suggestions')
# If this is already a replay config
replay_spec.ClearField('replay')
dependency_file = os.path.join(model_dir, _PREVIOUS_DEPENDENCIES_FILENAME)
if tf.io.gfile.exists(dependency_file):
with tf.io.gfile.GFile(dependency_file, 'r') as f:
data = f.read().split('\n')
for dependency in data:
spec = phoenix_spec_pb2.PhoenixSpec()
with tf.io.gfile.GFile(os.path.join(dependency, filename),
'r') as f:
text_format.Parse(f.read(), spec)
for i, _ in enumerate(spec.replay.towers):
replay_spec.replay.towers.add().CopyFrom(spec.replay.towers[i])
# Currently we support non-ensembling models.
search_tower = replay_spec.replay.towers.add()
# Removing the "context" hparam. This hparam is added for tpu, by the tpu team
# It is not compatible with the function "to_proto" used below.
copy_hparams = hp.HParams(**hparams.values())
if hasattr(copy_hparams, 'context'):
copy_hparams.del_hparam('context')
# We sometimes (for some search algorithms) override the hparam
# initial_architecture. Updating with the ground truth.
# TODO(b/172564129): Make initial_architecture consistent everywhere.
copy_hparams.set_hparam('initial_architecture', search_architecture)
search_tower.hparams.CopyFrom(copy_hparams.to_proto())
search_tower.architecture[:] = search_architecture
with tf.io.gfile.GFile(os.path.join(model_dir, filename), 'w') as f:
f.write(str(replay_spec))
def test_nonadaptive_prior(self, width, consider):
# Force graph mode
with tf.compat.v1.Graph().as_default():
spec = phoenix_spec_pb2.PhoenixSpec(
problem_type=phoenix_spec_pb2.PhoenixSpec.DNN)
spec.ensemble_spec.ensemble_search_type = (
ensembling_spec_pb2.EnsemblingSpec.NONADAPTIVE_ENSEMBLE_SEARCH)
spec.ensemble_spec.nonadaptive_search.width = width
spec.ensemble_spec.nonadaptive_search.num_trials_to_consider = consider
spec.is_input_shared = True
generator = prior_generator.PriorGenerator(
phoenix_spec=spec,
metadata=ml_metadata_db.MLMetaData(phoenix_spec=spec,
study_name='',
study_owner=''))
fake_config = collections.namedtuple('RunConfig', ['model_dir'])
run_config = fake_config(model_dir=flags.FLAGS.test_tmpdir +
'/10000')
tf.io.gfile.makedirs(run_config.model_dir)
# Best three trials checkpoint are generated. If the generator chooses
# the suboptimal (wrong) trials, the test will fail.
self._create_checkpoint(['search_generator'], 3)
self._create_checkpoint(['search_generator'], 4)
self._create_checkpoint(['search_generator'], 5)
logits, _ = generator.first_time_chief_generate(
features={},
input_layer_fn=lambda: None,
trial_mode=trial_utils.TrialMode.ENSEMBLE_SEARCH,
shared_input_tensor=tf.zeros([100, 32, 32, 3]),
shared_lengths=None,
logits_dimension=10,
hparams={},
run_config=run_config,
is_training=True,
trials=_create_trials(flags.FLAGS.test_tmpdir))
self.assertLen(logits, min(width, consider))
def _replay_spec():
output = phoenix_spec_pb2.PhoenixSpec()
output.replay.CopyFrom(phoenix_spec_pb2.ArchitectureReplay())
return output
def test_multitask(self, learning_rate_spec, contains_node,
not_containing):
# Force graph mode
with tf.compat.v1.Graph().as_default():
spec = phoenix_spec_pb2.PhoenixSpec(
problem_type=phoenix_spec_pb2.PhoenixSpec.DNN)
text_format.Merge(
"""
multi_task_spec {
label_name: "label1"
number_of_classes: 10
}
multi_task_spec {
label_name: "label2"
number_of_classes: 10
}
""", spec)
task_manager_instance = task_manager.TaskManager(spec)
logits = tf.keras.layers.Dense(10)(tf.zeros([20, 10]))
logits_spec = architecture_utils.LogitsSpec(logits=logits)
features = {'x': tf.zeros([10, 10])}
loss_fn = loss_fns.make_multi_class_loss_fn()
model = task_manager_instance.create_model_spec(
features=features,
params=hp.HParams(optimizer='sgd'),
learning_rate_spec=learning_rate_spec,
train_logits_specs=[logits_spec],
eval_logits_spec=logits_spec,
labels={
'label1': tf.ones([20], dtype=tf.int32),
'label2': tf.ones([20], dtype=tf.int32)
},
loss_fn=loss_fn,
mode=tf.estimator.ModeKeys.TRAIN,
lengths=None,
use_tpu=False,
predictions_fn=_default_predictions_fn)
self.assertNotEmpty([
node.name for node in
tf.compat.v1.get_default_graph().as_graph_def().node
if contains_node in node.name
])
for phrase in not_containing:
self.assertEmpty([
node.name for node in
tf.compat.v1.get_default_graph().as_graph_def().node
if phrase in node.name
])
self.assertLen(model.predictions, 3 * (1 + 2))
self.assertContainsSubset([
'probabilities',
'probabilities/label1',
'probabilities/label2',
'log_probabilities',
'log_probabilities/label1',
'log_probabilities/label2',
'predictions',
'predictions/label1',
'predictions/label2',
], model.predictions.keys())
def test_import_tower(self, shared_input):
np.random.seed(42)
test_input = np.random.random([1, 32, 32, 3])
# Force graph mode
with tf.compat.v1.Graph().as_default():
directory = self.get_temp_dir()
architecture = np.array([1, 3, 34])
phoenix_spec = phoenix_spec_pb2.PhoenixSpec(
problem_type=phoenix_spec_pb2.PhoenixSpec.CNN)
phoenix_spec.is_input_shared = shared_input
features = {}
shared_input_tensor = None
with self.test_session(graph=tf.Graph()) as sess:
input_tensor_1 = tf.compat.v1.placeholder(
dtype=tf.float32, shape=[None, 32, 32, 3], name="input_1")
tf.random.set_seed(1234)
tower_spec_1 = architecture_utils.construct_tower(
phoenix_spec=phoenix_spec,
input_tensor=input_tensor_1,
tower_name="test_tower",
architecture=architecture,
is_training=True,
lengths=None,
logits_dimension=10,
hparams=hp.HParams(),
model_directory=self.get_temp_dir(),
is_frozen=False,
dropout_rate=None)
saver = tf.compat.v1.train.Saver()
sess.run(tf.compat.v1.global_variables_initializer())
sess.run(tf.compat.v1.local_variables_initializer())
logits_val_1 = sess.run(tower_spec_1.logits_spec.logits,
feed_dict={input_tensor_1: test_input})
saver.save(sess, directory + "/ckpt")
with self.test_session(graph=tf.Graph()) as sess:
input_tensor_2 = tf.compat.v1.placeholder(
dtype=tf.float32, shape=[None, 32, 32, 3], name="input_2")
if shared_input:
shared_input_tensor = input_tensor_2
def _input_layer_fn(features,
is_training,
scope_name="Phoenix/Input",
lengths_feature_name=None):
del features, is_training, scope_name, lengths_feature_name
return None, None
else:
features = {"x": input_tensor_2}
def _input_layer_fn(features,
is_training,
scope_name="Phoenix/Input",
lengths_feature_name=None):
del is_training, lengths_feature_name
with tf.compat.v1.variable_scope(scope_name):
return tf.cast(features["x"],
dtype=tf.float32), None
tf.random.set_seed(1234)
tower_spec_2 = architecture_utils.import_tower(
features=features,
input_layer_fn=_input_layer_fn,
phoenix_spec=phoenix_spec,
shared_input_tensor=shared_input_tensor,
original_tower_name="test_tower",
new_tower_name="imported_tower",
model_directory=directory,
new_model_directory=self.get_temp_dir(),
is_training=True,
logits_dimension=10,
shared_lengths=None,
force_snapshot=False,
force_freeze=False)
sess.run(tf.compat.v1.global_variables_initializer())
sess.run(tf.compat.v1.local_variables_initializer())
logits_val_2 = sess.run(tower_spec_2.logits_spec.logits,
feed_dict={input_tensor_2: test_input})
self.assertAllClose(logits_val_1, logits_val_2, rtol=1e-3)
class ArchitectureUtilsTest(parameterized.TestCase, tf.test.TestCase):
def test_get_blocks_search_space(self):
hps = architecture_utils.get_blocks_search_space()
self.assertIn("TUNABLE_SVDF_output_size", hps)
self.assertIn("TUNABLE_SVDF_rank", hps)
self.assertIn("TUNABLE_SVDF_projection_size", hps)
self.assertIn("TUNABLE_SVDF_memory_size", hps)
hps = architecture_utils.get_blocks_search_space(["TUNABLE_SVDF"])
self.assertIn("TUNABLE_SVDF_output_size", hps)
self.assertIn("TUNABLE_SVDF_rank", hps)
self.assertIn("TUNABLE_SVDF_projection_size", hps)
self.assertIn("TUNABLE_SVDF_memory_size", hps)
def test_get_block_hparams(self):
hp_ = architecture_utils.get_block_hparams(
hp.HParams(TUNABLE_SVDF_target=10, non_target=15), "TUNABLE_SVDF")
self.assertLen(hp_.values(), 1)
self.assertEqual(hp_.target, 10)
def test_store_and_get_hparams(self):
hp_ = hp.HParams(hello="world", context="toberemoved")
dirname = self.get_temp_dir()
architecture_utils.store_hparams_to_dir(hp_, dirname, "tower")
hp_replica = architecture_utils.get_hparams_from_dir(dirname, "tower")
self.assertLen(hp_replica.values(), 1)
self.assertEqual(hp_replica.hello, "world")
@parameterized.named_parameters(
{
"testcase_name": "empty",
"input_dir": "",
"expected_output": None,
"spec": phoenix_spec_pb2.PhoenixSpec(),
}, {
"testcase_name": "normal",
"input_dir": "some/random/path/512",
"expected_output": 512,
"spec": phoenix_spec_pb2.PhoenixSpec(),
}, {
"testcase_name": "normal_tfx",
"input_dir": "/some/random/Trial-000034/active/Run-0000000",
"expected_output": 34,
"spec": phoenix_spec_pb2.PhoenixSpec(),
}, {
"testcase_name": "hybrid",
"input_dir": "/some/random/Trial-000034/active/Run-0000000/7",
"expected_output": 7,
"spec": phoenix_spec_pb2.PhoenixSpec(),
}, {
"testcase_name": "replay",
"input_dir": "/some/random/Trial-000034/active/Run-0000000/7",
"expected_output": 7,
"spec": _replay_spec(),
})
def test_get_trial_id(self, input_dir, expected_output, spec):
output = architecture_utils.DirectoryHandler.get_trial_id(
input_dir, spec)
self.assertEqual(output, expected_output)
@parameterized.named_parameters(
{
"testcase_name": "empty",
"input_trial": trial.Trial({"model_dir": ""}),
"expected_output": ""
}, {
"testcase_name": "normal",
"input_trial": trial.Trial({"model_dir": "random/path"}),
"expected_output": "random/path"
})
def test_get_trial_dir(self, input_trial, expected_output):
output = architecture_utils.DirectoryHandler.trial_dir(input_trial)
self.assertEqual(output, expected_output)
@parameterized.named_parameters(
{
"testcase_name": "empty",
"initial_architecture": [],
"fixed_architecture": [blocks.BlockType.PLATE_REDUCTION_FLATTEN]
}, {
"testcase_name": "flatten",
"initial_architecture": [blocks.BlockType.PLATE_REDUCTION_FLATTEN],
"fixed_architecture": [blocks.BlockType.PLATE_REDUCTION_FLATTEN]
}, {
"testcase_name": "downsample_flatten",
"initial_architecture": [blocks.BlockType.DOWNSAMPLE_FLATTEN],
"fixed_architecture": [blocks.BlockType.DOWNSAMPLE_FLATTEN]
}, {
"testcase_name":
"basic_case",
"initial_architecture": [
blocks.BlockType.PLATE_REDUCTION_FLATTEN,
blocks.BlockType.FULLY_CONNECTED,
blocks.BlockType.CONVOLUTION_3X3
],
"fixed_architecture": [
blocks.BlockType.CONVOLUTION_3X3,
blocks.BlockType.PLATE_REDUCTION_FLATTEN,
blocks.BlockType.FULLY_CONNECTED
]
}, {
"testcase_name":
"basic_downsample_case",
"initial_architecture": [
blocks.BlockType.DOWNSAMPLE_FLATTEN,
blocks.BlockType.FULLY_CONNECTED,
blocks.BlockType.CONVOLUTION_3X3
],
"fixed_architecture": [
blocks.BlockType.CONVOLUTION_3X3,
blocks.BlockType.DOWNSAMPLE_FLATTEN,
blocks.BlockType.FULLY_CONNECTED
]
}, {
"testcase_name":
"advanced_case",
"initial_architecture": [
blocks.BlockType.PLATE_REDUCTION_FLATTEN,
blocks.BlockType.FULLY_CONNECTED,
blocks.BlockType.FULLY_CONNECTED_PYRAMID,
blocks.BlockType.CONVOLUTION_3X3,
blocks.BlockType.DOWNSAMPLE_CONVOLUTION_3X3
],
"fixed_architecture": [
blocks.BlockType.CONVOLUTION_3X3,
blocks.BlockType.DOWNSAMPLE_CONVOLUTION_3X3,
blocks.BlockType.PLATE_REDUCTION_FLATTEN,
blocks.BlockType.FULLY_CONNECTED,
blocks.BlockType.FULLY_CONNECTED_PYRAMID
]
})
def test_fix_architecture_order(self, initial_architecture,
fixed_architecture):
out_architecture = architecture_utils.fix_architecture_order(
initial_architecture, phoenix_spec_pb2.PhoenixSpec.CNN)
self.assertAllEqual(fixed_architecture, out_architecture)
@parameterized.named_parameters(
{
"testcase_name": "empty_rnn",
"problem_type": phoenix_spec_pb2.PhoenixSpec.RNN_ALL_ACTIVATIONS,
"new_block": blocks.BlockType.LSTM_128,
"initial_architecture": np.array([]),
"expected_architecture": [blocks.BlockType.LSTM_128]
}, {
"testcase_name":
"empty_dnn",
"problem_type":
phoenix_spec_pb2.PhoenixSpec.DNN,
"new_block":
blocks.BlockType.FIXED_OUTPUT_FULLY_CONNECTED_128,
"initial_architecture":
np.array([]),
"expected_architecture":
[blocks.BlockType.FIXED_OUTPUT_FULLY_CONNECTED_128]
})
def test_increase_structure_depth(self, problem_type, initial_architecture,
new_block, expected_architecture):
out_architecture = architecture_utils.increase_structure_depth(
initial_architecture, new_block, problem_type)
self.assertAllEqual(expected_architecture, out_architecture)
@parameterized.named_parameters(
{
"testcase_name": "test1",
"architecture": np.array([1, 2, 3, 4]),
}, {
"testcase_name": "test2",
"architecture": np.array([2, 3, 4, 5]),
})
def test_set_get_architecture(self, architecture):
# Force graph mode
with tf.compat.v1.Graph().as_default():
directory = self.get_temp_dir()
with self.test_session(graph=tf.Graph()) as sess:
architecture_utils.set_architecture(architecture)
saver = tf.compat.v1.train.Saver()
sess.run(tf.compat.v1.global_variables_initializer())
sess.run(tf.compat.v1.local_variables_initializer())
saver.save(sess, directory + "/ckpt")
output_architecture = architecture_utils.get_architecture(
directory)
self.assertAllEqual(output_architecture, architecture)
@parameterized.named_parameters(
{
"testcase_name": "no_architecture",
"architecture": None,
}, {
"testcase_name": "has_architecture",
"architecture": np.array([2, 3, 4, 5]),
})
def test_get_architecture_size(self, architecture):
# Force graph mode
with tf.compat.v1.Graph().as_default():
with self.test_session(graph=tf.Graph()):
tower_name = "tower"
if architecture is not None:
architecture_utils.set_architecture(architecture,
tower_name=tower_name)
size = architecture_utils.get_architecture_size(
tower_name=tower_name)
if architecture is not None:
self.assertEqual(size, architecture.size)
else:
self.assertIsNone(size)
@parameterized.named_parameters(
{
"testcase_name":
"regular_network",
"dropout":
-1,
"expected_logits": [[
-0.1481, 0.3328, 0.3028, 0.5652, 0.6860, 0.06171, 0.09998,
-0.2622, 0.2186, -0.1322
]]
}, {
"testcase_name":
"dropout_network",
"dropout":
0.1,
"expected_logits": [[
0.7001, -0.06655, -0.1711, 0.1274, -0.8175, 0.2932, 0.06242,
0.2182, -0.06626, 0.7882
]],
})
def test_construct_network(self, dropout, expected_logits):
# Force graph mode
with tf.compat.v1.Graph().as_default():
tf.random.set_seed(1234)
# convolutions and then flatten plate.
architecture = np.array([1, 3, 34])
input_tensor = tf.compat.v1.placeholder(dtype=tf.float32,
shape=[None, 32, 32, 3],
name="input")
phoenix_spec = phoenix_spec_pb2.PhoenixSpec(
problem_type=phoenix_spec_pb2.PhoenixSpec.CNN)
tower_spec = architecture_utils.construct_tower(
phoenix_spec=phoenix_spec,
input_tensor=input_tensor,
tower_name="test_tower",
architecture=architecture,
is_training=True,
lengths=None,
logits_dimension=10,
hparams=hp.HParams(),
model_directory=self.get_temp_dir(),
is_frozen=False,
dropout_rate=dropout)
np.random.seed(42)
test_input = np.random.random([1, 32, 32, 3])
with tf.compat.v1.Session() as sess:
sess.run([
tf.compat.v1.global_variables_initializer(),
tf.compat.v1.local_variables_initializer()
])
logits_val = sess.run(tower_spec.logits_spec.logits,
feed_dict={input_tensor: test_input})
self.assertAllClose(expected_logits, logits_val, rtol=1e-3)
@parameterized.named_parameters(
{
"testcase_name":
"disabled",
"transfer_learning_type":
transfer_learning_spec_pb2.TransferLearningSpec.
NO_TRANSFER_LEARNING
}, {
"testcase_name":
"enabled",
"transfer_learning_type":
transfer_learning_spec_pb2.TransferLearningSpec.
UNIFORM_AVERAGE_TRANSFER_LEARNING
})
def test_construct_tower_with_transfer_learning(
self,
transfer_learning_type=transfer_learning_spec_pb2.TransferLearningSpec.
NO_TRANSFER_LEARNING):
# convolutions and then flatten plate.
architecture = np.array([1, 3, 34])
str_signature = "_1334"
input_tensor = tf.zeros([100, 32, 32, 3])
tower_name = "test_tower"
transfer_learning_spec = transfer_learning_spec_pb2.TransferLearningSpec(
transfer_learning_type=transfer_learning_type)
phoenix_spec = phoenix_spec_pb2.PhoenixSpec(
problem_type=phoenix_spec_pb2.PhoenixSpec.CNN,
transfer_learning_spec=transfer_learning_spec)
_ = architecture_utils.construct_tower(
phoenix_spec=phoenix_spec,
input_tensor=input_tensor,
tower_name=tower_name,
architecture=architecture,
is_training=True,
lengths=None,
logits_dimension=10,
hparams=hp.HParams(),
model_directory=self.get_temp_dir(),
is_frozen=False,
dropout_rate=None)
tensors = architecture_utils.get_tower_variables(tower_name)
for tensor in tensors:
if (transfer_learning_type == transfer_learning_spec_pb2.
TransferLearningSpec.NO_TRANSFER_LEARNING):
self.assertEndsWith(tensor.op.name, str_signature)
else:
self.assertNotEndsWith(tensor.op.name, str_signature)
@parameterized.named_parameters(
{
"testcase_name": "shared_input",
"shared_input": True
}, {
"testcase_name": "not_shared_input",
"shared_input": False
})
def test_import_tower(self, shared_input):
np.random.seed(42)
test_input = np.random.random([1, 32, 32, 3])
# Force graph mode
with tf.compat.v1.Graph().as_default():
directory = self.get_temp_dir()
architecture = np.array([1, 3, 34])
phoenix_spec = phoenix_spec_pb2.PhoenixSpec(
problem_type=phoenix_spec_pb2.PhoenixSpec.CNN)
phoenix_spec.is_input_shared = shared_input
features = {}
shared_input_tensor = None
with self.test_session(graph=tf.Graph()) as sess:
input_tensor_1 = tf.compat.v1.placeholder(
dtype=tf.float32, shape=[None, 32, 32, 3], name="input_1")
tf.random.set_seed(1234)
tower_spec_1 = architecture_utils.construct_tower(
phoenix_spec=phoenix_spec,
input_tensor=input_tensor_1,
tower_name="test_tower",
architecture=architecture,
is_training=True,
lengths=None,
logits_dimension=10,
hparams=hp.HParams(),
model_directory=self.get_temp_dir(),
is_frozen=False,
dropout_rate=None)
saver = tf.compat.v1.train.Saver()
sess.run(tf.compat.v1.global_variables_initializer())
sess.run(tf.compat.v1.local_variables_initializer())
logits_val_1 = sess.run(tower_spec_1.logits_spec.logits,
feed_dict={input_tensor_1: test_input})
saver.save(sess, directory + "/ckpt")
with self.test_session(graph=tf.Graph()) as sess:
input_tensor_2 = tf.compat.v1.placeholder(
dtype=tf.float32, shape=[None, 32, 32, 3], name="input_2")
if shared_input:
shared_input_tensor = input_tensor_2
def _input_layer_fn(features,
is_training,
scope_name="Phoenix/Input",
lengths_feature_name=None):
del features, is_training, scope_name, lengths_feature_name
return None, None
else:
features = {"x": input_tensor_2}
def _input_layer_fn(features,
is_training,
scope_name="Phoenix/Input",
lengths_feature_name=None):
del is_training, lengths_feature_name
with tf.compat.v1.variable_scope(scope_name):
return tf.cast(features["x"],
dtype=tf.float32), None
tf.random.set_seed(1234)
tower_spec_2 = architecture_utils.import_tower(
features=features,
input_layer_fn=_input_layer_fn,
phoenix_spec=phoenix_spec,
shared_input_tensor=shared_input_tensor,
original_tower_name="test_tower",
new_tower_name="imported_tower",
model_directory=directory,
new_model_directory=self.get_temp_dir(),
is_training=True,
logits_dimension=10,
shared_lengths=None,
force_snapshot=False,
force_freeze=False)
sess.run(tf.compat.v1.global_variables_initializer())
sess.run(tf.compat.v1.local_variables_initializer())
logits_val_2 = sess.run(tower_spec_2.logits_spec.logits,
feed_dict={input_tensor_2: test_input})
self.assertAllClose(logits_val_1, logits_val_2, rtol=1e-3)
@parameterized.named_parameters(
{
"testcase_name": "cnn",
"input_tensor_shape": [20, 20],
"spec": _create_spec(phoenix_spec_pb2.PhoenixSpec.CNN),
"output_shape": [20, 7],
},
{
"testcase_name": "dnn",
"input_tensor_shape": [20, 20],
"spec": _create_spec(phoenix_spec_pb2.PhoenixSpec.DNN),
"output_shape": [20, 7],
},
{
"testcase_name": "rnn_all_activations",
"input_tensor_shape": [20, 20, 20],
"spec": _create_spec(
phoenix_spec_pb2.PhoenixSpec.RNN_ALL_ACTIVATIONS),
"output_shape": [20, 20, 7],
},
{
"testcase_name": "rnn_last_activations",
"input_tensor_shape": [20, 20, 20],
"spec": _create_spec(
phoenix_spec_pb2.PhoenixSpec.RNN_LAST_ACTIVATIONS),
"output_shape": [20, 7],
"lengths": list(range(20))
},
{
"testcase_name": "rnn_last_activations_no_length",
"input_tensor_shape": [20, 20, 20],
"spec": _create_spec(
phoenix_spec_pb2.PhoenixSpec.RNN_LAST_ACTIVATIONS),
"output_shape": [20, 20, 7],
},
{
"testcase_name": "nasnet_aux_head",
"input_tensor_shape": [20, 20],
"spec": _create_spec(phoenix_spec_pb2.PhoenixSpec.CNN),
"output_shape": [20, 7],
"extra_block": 6, # Downsample convolution.
"extra_block_shape": [20, 20, 20, 20],
"use_auxiliary_head": True
},
{
"testcase_name": "deep_skip_head",
"input_tensor_shape": [20, 20],
"spec": _create_spec(phoenix_spec_pb2.PhoenixSpec.CNN),
"output_shape": [20, 7],
"extra_block": 16, # Flatten.
"use_auxiliary_head": True
})
def test_create_tower_spec(
self,
input_tensor_shape,
spec,
output_shape,
lengths=None,
extra_block=14, # Fully connected.
extra_block_shape=None,
use_auxiliary_head=False):
# Force graph mode
with tf.compat.v1.Graph().as_default():
spec.use_auxiliary_head = use_auxiliary_head
spec.auxiliary_head_loss_weight = .4
input_tensors = [
tf.zeros(extra_block_shape
or (1, )), # Raw features -- discarded.
tf.zeros(extra_block_shape
or input_tensor_shape), # Extra block.
tf.zeros(extra_block_shape
or input_tensor_shape), # Extra block.
tf.zeros(input_tensor_shape) # Fully connected block.
]
fake_architecture = [extra_block, extra_block, 14]
logits_spec, _, _ = architecture_utils.create_tower_spec(
phoenix_spec=spec,
inputs=input_tensors,
architecture=fake_architecture,
dimension=7,
is_frozen=False,
lengths=lengths,
allow_auxiliary_head=use_auxiliary_head)
self.assertAllEqual(output_shape, logits_spec.logits.shape)
if use_auxiliary_head:
self.assertIsNotNone(logits_spec.aux_logits)
self.assertNear(logits_spec.aux_logits_weight,
spec.auxiliary_head_loss_weight, 1e-6)
self.assertNear(logits_spec.logits_weight, 1.0, 1e-6)
@parameterized.named_parameters(
{
"testcase_name":
"same_graph_snapshotting",
"new_architecture":
np.array([1, 3, 34]),
"expected_output": [
"Phoenix/test_tower/1_FIXED_CHANNEL_CONVOLUTION_16_1/"
"Conv/biases",
"Phoenix/test_tower/1_FIXED_CHANNEL_CONVOLUTION_16_1/"
"Conv/weights",
"Phoenix/test_tower/2_FIXED_CHANNEL_CONVOLUTION_64_13/"
"Conv/biases",
"Phoenix/test_tower/2_FIXED_CHANNEL_CONVOLUTION_64_13/"
"Conv/weights",
"Phoenix/test_tower/last_dense_1334/dense/bias",
"Phoenix/test_tower/last_dense_1334/dense/kernel",
"Phoenix/test_tower/2_FIXED_CHANNEL_CONVOLUTION_64_13/BatchNorm/"
"beta",
"Phoenix/test_tower/1_FIXED_CHANNEL_CONVOLUTION_16_1/BatchNorm/"
"moving_variance",
"Phoenix/test_tower/2_FIXED_CHANNEL_CONVOLUTION_64_13/BatchNorm/"
"moving_variance",
"Phoenix/test_tower/1_FIXED_CHANNEL_CONVOLUTION_16_1/BatchNorm/"
"moving_mean",
"Phoenix/test_tower/1_FIXED_CHANNEL_CONVOLUTION_16_1/BatchNorm/"
"beta",
"Phoenix/test_tower/2_FIXED_CHANNEL_CONVOLUTION_64_13/BatchNorm/"
"moving_mean",
]
}, {
"testcase_name":
"same_graph_snapshotting_new_towername",
"new_architecture":
np.array([1, 3, 34]),
"expected_output": [
"Phoenix/test_tower/1_FIXED_CHANNEL_CONVOLUTION_16_1/"
"Conv/biases",
"Phoenix/test_tower/1_FIXED_CHANNEL_CONVOLUTION_16_1/"
"Conv/weights",
"Phoenix/test_tower/2_FIXED_CHANNEL_CONVOLUTION_64_13/"
"Conv/biases",
"Phoenix/test_tower/2_FIXED_CHANNEL_CONVOLUTION_64_13/"
"Conv/weights",
"Phoenix/test_tower/last_dense_1334/dense/bias",
"Phoenix/test_tower/last_dense_1334/dense/kernel",
"Phoenix/test_tower/2_FIXED_CHANNEL_CONVOLUTION_64_13/BatchNorm/"
"beta",
"Phoenix/test_tower/1_FIXED_CHANNEL_CONVOLUTION_16_1/BatchNorm/"
"moving_variance",
"Phoenix/test_tower/2_FIXED_CHANNEL_CONVOLUTION_64_13/BatchNorm/"
"moving_variance",
"Phoenix/test_tower/1_FIXED_CHANNEL_CONVOLUTION_16_1/BatchNorm/"
"moving_mean",
"Phoenix/test_tower/1_FIXED_CHANNEL_CONVOLUTION_16_1/BatchNorm/"
"beta",
"Phoenix/test_tower/2_FIXED_CHANNEL_CONVOLUTION_64_13/BatchNorm/"
"moving_mean",
],
"new_tower_name":
"test_tower_2"
}, {
"testcase_name":
"changing_second",
"new_architecture":
np.array([1, 2, 34]),
"expected_output": [
"Phoenix/test_tower/1_FIXED_CHANNEL_CONVOLUTION_16_1/"
"Conv/weights",
"Phoenix/test_tower/1_FIXED_CHANNEL_CONVOLUTION_16_1/Conv/biases",
"Phoenix/test_tower/1_FIXED_CHANNEL_CONVOLUTION_16_1/BatchNorm/"
"moving_mean",
"Phoenix/test_tower/1_FIXED_CHANNEL_CONVOLUTION_16_1/BatchNorm/"
"moving_variance",
"Phoenix/test_tower/1_FIXED_CHANNEL_CONVOLUTION_16_1/BatchNorm/"
"beta"
]
})
def test_init_variables(self,
new_architecture,
expected_output,
new_tower_name="test_tower"):
# Force graph mode
with tf.compat.v1.Graph().as_default():
directory = self.get_temp_dir()
architecture = np.array([1, 3, 34])
phoenix_spec = phoenix_spec_pb2.PhoenixSpec(
problem_type=phoenix_spec_pb2.PhoenixSpec.CNN)
with self.test_session(graph=tf.Graph()) as sess:
input_tensor = tf.zeros([100, 32, 32, 3])
_ = architecture_utils.construct_tower(
phoenix_spec=phoenix_spec,
input_tensor=input_tensor,
tower_name="test_tower",
architecture=architecture,
is_training=True,
lengths=None,
logits_dimension=10,
model_directory=self.get_temp_dir(),
hparams=hp.HParams(),
is_frozen=False,
dropout_rate=None)
saver = tf.compat.v1.train.Saver()
sess.run(tf.compat.v1.global_variables_initializer())
sess.run(tf.compat.v1.local_variables_initializer())
saver.save(sess, directory + "/ckpt")
with self.test_session(graph=tf.Graph()) as sess:
input_tensor = tf.zeros([100, 32, 32, 3])
_ = architecture_utils.construct_tower(
phoenix_spec=phoenix_spec,
input_tensor=input_tensor,
tower_name=new_tower_name,
architecture=new_architecture,
is_training=True,
lengths=None,
logits_dimension=10,
hparams=hp.HParams(),
model_directory=self.get_temp_dir(),
is_frozen=False,
dropout_rate=None)
snapshotting_variables = architecture_utils.init_variables(
tf.train.latest_checkpoint(directory),
"Phoenix/test_tower", "Phoenix/{}".format(new_tower_name))
self.assertCountEqual(snapshotting_variables, expected_output)
def test_write_replay_spec(self):
original_spec = phoenix_spec_pb2.PhoenixSpec()
text_format.Parse(
"""
minimum_depth: 2
maximum_depth: 20
problem_type: CNN
search_type: ADAPTIVE_COORDINATE_DESCENT
user_suggestions {
architecture: "LSTM_128"
}
""", original_spec)
_write_dependency_reply(self.get_temp_dir())
trial_utils.write_replay_spec(
model_dir=self.get_temp_dir(),
filename="replay_config.pbtxt",
original_spec=original_spec,
search_architecture=["RNN_128", "RNN_128"],
hparams=hp.HParams(
learning_rate=0.001,
initial_architecture=["not_cor_arc", "not_cor_arc2"],
context="should_be_deleted"))
output_spec = phoenix_spec_pb2.PhoenixSpec()
with tf.io.gfile.GFile(self.get_temp_dir() + "/replay_config.pbtxt",
"r") as f:
text_format.Parse(f.read(), output_spec)
expected_spec = phoenix_spec_pb2.PhoenixSpec()
text_format.Parse(
"""
minimum_depth: 2
maximum_depth: 20
problem_type: CNN
search_type: ADAPTIVE_COORDINATE_DESCENT
replay {
towers {
architecture: "RNN_128"
architecture: "RNN_128"
hparams {
hparam {
key: "learning_rate"
value {
float_value: 0.021
}
}
hparam {
key: "initial_architecture"
value {
bytes_list {
value: "RNN_256"
value: "RNN_128"
}
}
}
}
}
towers {
architecture: "RNN_128"
architecture: "RNN_128"
hparams {
hparam {
key: "learning_rate"
value {
float_value: 0.001
}
}
hparam {
key: "initial_architecture"
value {
bytes_list {
value: "RNN_128"
value: "RNN_128"
}
}
}
}
}
}
""", expected_spec)
self.assertEqual(output_spec, expected_spec)
def main(unused_argv):
filename = FLAGS.phoenix_spec_filename
spec = phoenix_spec_pb2.PhoenixSpec()
with tf.io.gfile.GFile(filename, "r") as f:
text_format.Merge(f.read(), spec)
dataset_provider = get_dataset_provider()
loss_fn, metric_fn, predictions_fn = (
loss_and_metric_and_predictions_fn(dataset_provider))
metadata = None
if (FLAGS.optimization_goal != "minimize"
or FLAGS.optimization_metric != "loss"):
metadata = ml_metadata_db.MLMetaData(
phoenix_spec=spec,
study_name=FLAGS.experiment_name,
study_owner=FLAGS.experiment_owner,
optimization_goal=FLAGS.optimization_goal,
optimization_metric=FLAGS.optimization_metric)
phoenix_instance = phoenix.Phoenix(
phoenix_spec=spec,
input_layer_fn=dataset_provider.get_input_layer_fn(spec.problem_type),
logits_dimension=dataset_provider.number_of_classes(),
study_name=FLAGS.experiment_name,
study_owner=FLAGS.experiment_owner,
loss_fn=loss_fn,
metric_fn=metric_fn,
predictions_fn=predictions_fn,
metadata=metadata)
# Replay only!!
if spec.HasField("replay"):
hparams = hp.HParams.from_proto(spec.replay.towers[0].hparams)
run_train_and_eval(hparams,
FLAGS.model_dir,
phoenix_instance,
dataset_provider,
train_steps=FLAGS.phoenix_train_steps,
eval_steps=FLAGS.phoenix_eval_steps,
batch_size=FLAGS.phoenix_batch_size)
return 0
tuner_id = FLAGS.tuner_id or "phoenix-tuner-%d" % random.randint(
0, 10000000)
hyperparameters = phoenix.Phoenix.get_keras_hyperparameters_space(
spec, FLAGS.phoenix_train_steps)
if FLAGS.hypertuning_method == "random":
oracle = kerastuner.tuners.randomsearch.RandomSearchOracle(
objective="loss",
max_trials=FLAGS.experiment_max_num_trials,
seed=73,
hyperparameters=hyperparameters,
allow_new_entries=True,
tune_new_entries=True)
else:
oracle = kerastuner.tuners.bayesian.BayesianOptimizationOracle(
objective="loss",
hyperparameters=hyperparameters,
max_trials=FLAGS.experiment_max_num_trials)
# pylint: disable=protected-access
oracle._set_project_dir(FLAGS.model_dir,
FLAGS.experiment_name,
overwrite=True)
# pylint: enable=protected-access
data_provider = get_dataset_provider()
while run_parameterized_train_and_eval(
phoenix_instance=phoenix_instance,
oracle=oracle,
tuner_id=tuner_id,
root_dir=FLAGS.model_dir,
max_trials=FLAGS.experiment_max_num_trials,
data_provider=data_provider,
train_steps=FLAGS.phoenix_train_steps,
eval_steps=FLAGS.phoenix_eval_steps,
batch_size=FLAGS.phoenix_batch_size):
pass
