def test_merge_hparams(self):
params = phoenix._merge_hparams(
hp.HParams(learning_rate=1, untouched=4),
hp.HParams(learning_rate=2, new_param=3))
self.assertEqual(params.learning_rate, 2)
self.assertEqual(params.untouched, 4)
self.assertEqual(params.new_param, 3)
def test_construct_tower_with_transfer_learning(
self,
transfer_learning_type=transfer_learning_spec_pb2.TransferLearningSpec.
NO_TRANSFER_LEARNING):
# convolutions and then flatten plate.
architecture = np.array([1, 3, 34])
str_signature = "_1334"
input_tensor = tf.zeros([100, 32, 32, 3])
tower_name = "test_tower"
transfer_learning_spec = transfer_learning_spec_pb2.TransferLearningSpec(
transfer_learning_type=transfer_learning_type)
phoenix_spec = phoenix_spec_pb2.PhoenixSpec(
problem_type=phoenix_spec_pb2.PhoenixSpec.CNN,
transfer_learning_spec=transfer_learning_spec)
_ = architecture_utils.construct_tower(
phoenix_spec=phoenix_spec,
input_tensor=input_tensor,
tower_name=tower_name,
architecture=architecture,
is_training=True,
lengths=None,
logits_dimension=10,
hparams=hp.HParams(),
model_directory=self.get_temp_dir(),
is_frozen=False,
dropout_rate=None)
tensors = architecture_utils.get_tower_variables(tower_name)
for tensor in tensors:
if (transfer_learning_type == transfer_learning_spec_pb2.
TransferLearningSpec.NO_TRANSFER_LEARNING):
self.assertEndsWith(tensor.op.name, str_signature)
else:
self.assertNotEndsWith(tensor.op.name, str_signature)
def test_get_suggestion(self, get_architecture, problem_type):
# Return value (architectures) for the various trials.
get_architecture.side_effect = [
np.array([1, 2, 3, 4]),
np.array([2, 3, 4, 1]),
np.array([3, 4, 1, 2]),
np.array([4, 1, 2, 3]),
np.array([1, 1, 1, 1]),
np.array([2, 2, 2, 2]),
np.array([3, 3, 3, 3]),
np.array([2, 3, 2, 3]),
np.array([3, 4, 3, 4])
]
algorithm = categorical_harmonica.Harmonica(test_utils.create_spec(
problem_type,
blocks_to_use=[
"FIXED_CHANNEL_CONVOLUTION_16", "FIXED_CHANNEL_CONVOLUTION_32",
"FIXED_CHANNEL_CONVOLUTION_64", "CONVOLUTION_3X3"
],
min_depth=4),
num_random_samples=10,
seed=73)
output_architecture, _ = algorithm.get_suggestion(
_create_trials(), hp.HParams())
expected_output = [3, 3, 3, 3]
if problem_type == phoenix_spec_pb2.PhoenixSpec.CNN:
expected_output += [34]
self.assertAllEqual(expected_output, output_architecture)
def test_head(self, head_fn, label_fn=None, loss_fn=None, metric_fn=None):
batch_size = 8
input_shape = [batch_size, 32, 32, 3]
hparams = hp.HParams(
initial_architecture=[
"FIXED_CHANNEL_CONVOLUTION_16", "FIXED_CHANNEL_CONVOLUTION_64",
"PLATE_REDUCTION_FLATTEN"
],
new_block_type="FIXED_CHANNEL_CONVOLUTION_32",
learning_rate=
1000.0, # Approximating constant so we'll never diverge.
optimizer="sgd")
instance = self._create_phoenix_instance(problem_type="cnn",
input_shape=input_shape,
head=head_fn(),
loss_fn=loss_fn,
metric_fn=metric_fn)
run_config = tf.estimator.RunConfig(model_dir=self.get_temp_dir() +
"/1")
def input_fn():
features = {"zeros": tf.zeros(input_shape)}
if getattr(head_fn(), "_weight_column", None):
features["weights"] = tf.ones(batch_size) * .5
labels = label_fn() if label_fn else tf.zeros(batch_size,
dtype=tf.int32)
return features, labels
estimator = instance.get_estimator(run_config, hparams, 10)
estimator.train(input_fn=input_fn, max_steps=10)
eval_result = estimator.evaluate(input_fn=input_fn, steps=10)
self.assertAlmostEqual(0., eval_result["loss"], places=3)
def test_store_and_get_hparams(self):
hp_ = hp.HParams(hello="world", context="toberemoved")
dirname = self.get_temp_dir()
architecture_utils.store_hparams_to_dir(hp_, dirname, "tower")
hp_replica = architecture_utils.get_hparams_from_dir(dirname, "tower")
self.assertLen(hp_replica.values(), 1)
self.assertEqual(hp_replica.hello, "world")
def get_hparams_from_dir(model_directory, tower_name):
hparams = hparam_pb2.HParamDef()
with tf.io.gfile.GFile(os.path.join(model_directory, tower_name),
"r") as f:
text_format.Parse(f.read(), hparams)
output = hp.HParams(hparam_def=hparams)
return output
def _create_checkpoint(self, towers, trial_id):
with self.test_session(graph=tf.Graph()) as sess:
architecture = np.array([1, 3, 34])
input_tensor = tf.zeros([100, 32, 32, 3])
phoenix_spec = phoenix_spec_pb2.PhoenixSpec(
problem_type=phoenix_spec_pb2.PhoenixSpec.CNN)
dirname = os.path.join(flags.FLAGS.test_tmpdir, str(trial_id))
if dirname and not tf.io.gfile.exists(dirname):
tf.io.gfile.makedirs(dirname)
for tower in towers:
_ = architecture_utils.construct_tower(
phoenix_spec=phoenix_spec,
input_tensor=input_tensor,
tower_name=str(tower) + '_0',
architecture=architecture,
is_training=True,
lengths=None,
logits_dimension=10,
hparams=hp.HParams(),
model_directory=dirname,
is_frozen=False,
dropout_rate=None)
architecture_utils.set_number_of_towers(tower, 1)
architecture_utils.set_number_of_towers('replay_generator', 0)
directory = flags.FLAGS.test_tmpdir
saver = tf.compat.v1.train.Saver()
sess.run(tf.compat.v1.global_variables_initializer())
sess.run(tf.compat.v1.local_variables_initializer())
saver.save(sess, os.path.join(directory, str(trial_id)) + '/ckpt')
def test_generator_with_dropouts(self):
# Force graph mode
with tf.compat.v1.Graph().as_default():
spec = phoenix_spec_pb2.PhoenixSpec(
problem_type=phoenix_spec_pb2.PhoenixSpec.DNN)
spec.search_type = phoenix_spec_pb2.PhoenixSpec.NONADAPTIVE_RANDOM_SEARCH
spec.is_input_shared = True
generator = search_candidate_generator.SearchCandidateGenerator(
phoenix_spec=spec,
metadata=ml_metadata_db.MLMetaData(phoenix_spec=spec,
study_name='',
study_owner=''))
input_tensor = tf.zeros([20, 32, 32, 3])
fake_config = collections.namedtuple('RunConfig',
['model_dir', 'is_chief'])
run_config = fake_config(model_dir=flags.FLAGS.test_tmpdir + '/1',
is_chief=True)
_ = generator.generate(
features={},
input_layer_fn=lambda: None,
trial_mode=trial_utils.TrialMode.NO_PRIOR,
shared_input_tensor=input_tensor,
shared_lengths=None,
logits_dimension=10,
hparams=hp.HParams(initial_architecture=['CONVOLUTION_3X3'],
dropout_rate=0.3),
run_config=run_config,
is_training=True,
trials=[])
all_nodes = [
node.name for node in
tf.compat.v1.get_default_graph().as_graph_def().node
]
self.assertAllInSet(_DROPOUT_GRAPH_NODE, all_nodes)
def test_learning_spec_on_predict(self, learning_rate_spec,
not_containing):
spec = phoenix_spec_pb2.PhoenixSpec(
problem_type=phoenix_spec_pb2.PhoenixSpec.DNN)
task_manager_instance = task_manager.TaskManager(spec)
logits = tf.keras.layers.Dense(10)(tf.zeros([20, 10]))
logits_spec = architecture_utils.LogitsSpec(logits=logits)
features = {'x': tf.zeros([10, 10])}
loss_fn = loss_fns.make_multi_class_loss_fn()
model = task_manager_instance.create_model_spec(
features=features,
params=hp.HParams(optimizer='sgd'),
learning_rate_spec=learning_rate_spec,
train_logits_specs=[logits_spec],
eval_logits_spec=logits_spec,
labels=tf.ones([20], dtype=tf.int32),
loss_fn=loss_fn,
mode=tf.estimator.ModeKeys.PREDICT,
lengths=None,
use_tpu=False,
predictions_fn=_default_predictions_fn)
for phrase in not_containing:
self.assertEmpty([
node.name for node in
tf.compat.v1.get_default_graph().as_graph_def().node
if phrase in node.name
])
self.assertLen(model.predictions, 3)
self.assertIn('probabilities', model.predictions)
self.assertIn('log_probabilities', model.predictions)
self.assertIn('predictions', model.predictions)
self.assertIsNone(model.loss)
def test_architecture(self):
# Force graph mode
with tf.compat.v1.Graph().as_default():
learning_rate_spec = {'learning_rate': 0.001, 'gradient_max_norm': 3}
spec = phoenix_spec_pb2.PhoenixSpec(
problem_type=phoenix_spec_pb2.PhoenixSpec.CNN)
text_format.Merge(
"""
multi_task_spec {
label_name: "label1"
number_of_classes: 10
architecture: "FIXED_OUTPUT_FULLY_CONNECTED_128"
}
multi_task_spec {
label_name: "label2"
number_of_classes: 10
architecture: "FIXED_OUTPUT_FULLY_CONNECTED_256"
architecture: "FIXED_OUTPUT_FULLY_CONNECTED_512"
}
""", spec)
task_manager_instance = task_manager.TaskManager(spec)
logits = tf.keras.layers.Dense(10)(tf.zeros([20, 10]))
logits_spec = architecture_utils.LogitsSpec(logits=logits)
features = {'x': tf.zeros([10, 10])}
loss_fn = loss_fns.make_multi_class_loss_fn()
model = task_manager_instance.create_model_spec(
features=features,
params=hp.HParams(optimizer='sgd'),
learning_rate_spec=learning_rate_spec,
train_logits_specs=[logits_spec],
eval_logits_spec=logits_spec,
labels={
'label1': tf.ones([20], dtype=tf.int32),
'label2': tf.ones([20], dtype=tf.int32)
},
loss_fn=loss_fn,
model_directory=self.get_temp_dir(),
mode=tf.estimator.ModeKeys.TRAIN,
lengths=None,
use_tpu=False,
predictions_fn=_default_predictions_fn)
self.assertNotEmpty([
node.name
for node in tf.compat.v1.get_default_graph().as_graph_def().node
if 'task_label1_tower/1_FIXED_OUTPUT_FULLY_CONNECTED_128' in node.name
])
self.assertNotEmpty([
node.name
for node in tf.compat.v1.get_default_graph().as_graph_def().node
if 'task_label2_tower/1_FIXED_OUTPUT_FULLY_CONNECTED_256' in node.name
])
self.assertNotEmpty([
node.name
for node in tf.compat.v1.get_default_graph().as_graph_def().node
if 'task_label2_tower/2_FIXED_OUTPUT_FULLY_CONNECTED_512' in node.name
])
self.assertLen(model.predictions, 3 * (1 + 2))
self.assertIn('probabilities', model.predictions)
self.assertIn('log_probabilities', model.predictions)
self.assertIn('predictions', model.predictions)
def test_tpu(self):
# Force graph mode
with tf.compat.v1.Graph().as_default():
learning_rate_spec = {
'learning_rate': 0.001,
'gradient_max_norm': 3
}
spec = phoenix_spec_pb2.PhoenixSpec(
problem_type=phoenix_spec_pb2.PhoenixSpec.DNN)
task_manager_instance = task_manager.TaskManager(spec)
logits = tf.keras.layers.Dense(10)(tf.zeros([20, 10]))
logits_spec = architecture_utils.LogitsSpec(logits=logits)
features = {'x': tf.zeros([10, 10])}
loss_fn = loss_fns.make_multi_class_loss_fn()
_ = task_manager_instance.create_model_spec(
features=features,
params=hp.HParams(optimizer='sgd'),
learning_rate_spec=learning_rate_spec,
train_logits_specs=[logits_spec],
eval_logits_spec=logits_spec,
labels=tf.ones([20], dtype=tf.int32),
loss_fn=loss_fn,
mode=tf.estimator.ModeKeys.TRAIN,
lengths=None,
use_tpu=True,
predictions_fn=_default_predictions_fn)
self.assertNotEmpty([
node.name for node in
tf.compat.v1.get_default_graph().as_graph_def().node
if 'CrossReplicaSum' in node.name
])
def test_construct_network(self, dropout, expected_logits):
# Force graph mode
with tf.compat.v1.Graph().as_default():
tf.random.set_seed(1234)
# convolutions and then flatten plate.
architecture = np.array([1, 3, 34])
input_tensor = tf.compat.v1.placeholder(dtype=tf.float32,
shape=[None, 32, 32, 3],
name="input")
phoenix_spec = phoenix_spec_pb2.PhoenixSpec(
problem_type=phoenix_spec_pb2.PhoenixSpec.CNN)
tower_spec = architecture_utils.construct_tower(
phoenix_spec=phoenix_spec,
input_tensor=input_tensor,
tower_name="test_tower",
architecture=architecture,
is_training=True,
lengths=None,
logits_dimension=10,
hparams=hp.HParams(),
model_directory=self.get_temp_dir(),
is_frozen=False,
dropout_rate=dropout)
np.random.seed(42)
test_input = np.random.random([1, 32, 32, 3])
with tf.compat.v1.Session() as sess:
sess.run([
tf.compat.v1.global_variables_initializer(),
tf.compat.v1.local_variables_initializer()
])
logits_val = sess.run(tower_spec.logits_spec.logits,
feed_dict={input_tensor: test_input})
self.assertAllClose(expected_logits, logits_val, rtol=1e-3)
def test_init_variables(self,
new_architecture,
expected_output,
new_tower_name="test_tower"):
# Force graph mode
with tf.compat.v1.Graph().as_default():
directory = self.get_temp_dir()
architecture = np.array([1, 3, 34])
phoenix_spec = phoenix_spec_pb2.PhoenixSpec(
problem_type=phoenix_spec_pb2.PhoenixSpec.CNN)
with self.test_session(graph=tf.Graph()) as sess:
input_tensor = tf.zeros([100, 32, 32, 3])
_ = architecture_utils.construct_tower(
phoenix_spec=phoenix_spec,
input_tensor=input_tensor,
tower_name="test_tower",
architecture=architecture,
is_training=True,
lengths=None,
logits_dimension=10,
model_directory=self.get_temp_dir(),
hparams=hp.HParams(),
is_frozen=False,
dropout_rate=None)
saver = tf.compat.v1.train.Saver()
sess.run(tf.compat.v1.global_variables_initializer())
sess.run(tf.compat.v1.local_variables_initializer())
saver.save(sess, directory + "/ckpt")
with self.test_session(graph=tf.Graph()) as sess:
input_tensor = tf.zeros([100, 32, 32, 3])
_ = architecture_utils.construct_tower(
phoenix_spec=phoenix_spec,
input_tensor=input_tensor,
tower_name=new_tower_name,
architecture=new_architecture,
is_training=True,
lengths=None,
logits_dimension=10,
hparams=hp.HParams(),
model_directory=self.get_temp_dir(),
is_frozen=False,
dropout_rate=None)
snapshotting_variables = architecture_utils.init_variables(
tf.train.latest_checkpoint(directory),
"Phoenix/test_tower", "Phoenix/{}".format(new_tower_name))
self.assertCountEqual(snapshotting_variables, expected_output)
def store_hparams_to_dir(hparams, model_directory, tower_name):
# Removing the "context" hparam. This hparam is added for tpu, by the tpu team
# It is not compatible with the function "to_proto" used below.
copy_hparams = hp.HParams(**hparams.values())
if hasattr(copy_hparams, "context"):
copy_hparams.del_hparam("context")
with tf.io.gfile.GFile(os.path.join(model_directory, tower_name),
"w") as f:
f.write(str(copy_hparams.to_proto()))
def test_wrong_dict_weight_feature(self, weight_is_a_feature):
learning_rate_spec = {'learning_rate': 0.001, 'gradient_max_norm': 3}
spec = phoenix_spec_pb2.PhoenixSpec(
problem_type=phoenix_spec_pb2.PhoenixSpec.DNN)
text_format.Merge(
"""
multi_task_spec {
label_name: "label1"
number_of_classes: 10
weight_feature_name: "weight1"
weight_is_a_feature: %s
}
multi_task_spec {
label_name: "label2"
number_of_classes: 10
weight_feature_name: "weight2"
weight_is_a_feature: %s
}
""" % (str(weight_is_a_feature), str(weight_is_a_feature)), spec)
labels = {
'label1': tf.ones([20], dtype=tf.int32),
'label2': tf.ones([20], dtype=tf.int32),
}
# Fix the size of the dict labels to bypass the assertion.
if not weight_is_a_feature:
labels.update({
'not_used': tf.ones([20], dtype=tf.int32),
'not_used2': tf.ones([20], dtype=tf.int32)
})
weights = {
'weight1': tf.constant([2] * 20),
'weight2': tf.constant([3] * 20)
}
features = {'x': tf.zeros([10, 10])}
if not weight_is_a_feature:
features.update(weights)
task_manager_instance = task_manager.TaskManager(spec)
logits = tf.keras.layers.Dense(10)(tf.zeros([20, 10]))
logits_spec = architecture_utils.LogitsSpec(logits=logits)
with self.assertRaises(KeyError):
loss_fn = loss_fns.make_multi_class_loss_fn()
_ = task_manager_instance.create_model_spec(
features=features,
params=hp.HParams(optimizer='sgd'),
learning_rate_spec=learning_rate_spec,
train_logits_specs=[logits_spec],
eval_logits_spec=logits_spec,
labels=labels,
loss_fn=loss_fn,
model_directory=self.get_temp_dir(),
mode=tf.estimator.ModeKeys.TRAIN,
lengths=None,
use_tpu=False,
predictions_fn=_default_predictions_fn)
def test_weight_feature(self, is_multitask, weight_is_a_feature):
# Force graph mode
with tf.compat.v1.Graph().as_default():
learning_rate_spec = {
'learning_rate': 0.001,
'gradient_max_norm': 3
}
spec = phoenix_spec_pb2.PhoenixSpec(
problem_type=phoenix_spec_pb2.PhoenixSpec.DNN)
labels = tf.ones([20], dtype=tf.int32)
if is_multitask:
text_format.Merge(
"""
multi_task_spec {
label_name: "label1"
number_of_classes: 10
weight_feature_name: "weight1"
weight_is_a_feature: %s
}
multi_task_spec {
label_name: "label2"
number_of_classes: 10
weight_feature_name: "weight2"
weight_is_a_feature: %s
}
""" % (str(weight_is_a_feature), str(weight_is_a_feature)), spec)
labels = {
'label1': tf.ones([20], dtype=tf.int32),
'label2': tf.ones([20], dtype=tf.int32)
}
weights = {
'weight1': tf.constant([2] * 20),
'weight2': tf.constant([3] * 20)
}
features = {'x': tf.zeros([10, 10])}
if weight_is_a_feature:
features.update(weights)
elif isinstance(labels, dict):
labels.update(weights)
task_manager_instance = task_manager.TaskManager(spec)
logits = tf.keras.layers.Dense(10)(tf.zeros([20, 10]))
logits_spec = architecture_utils.LogitsSpec(logits=logits)
loss_fn = loss_fns.make_multi_class_loss_fn()
_ = task_manager_instance.create_model_spec(
features=features,
params=hp.HParams(optimizer='sgd'),
learning_rate_spec=learning_rate_spec,
train_logits_specs=[logits_spec],
eval_logits_spec=logits_spec,
labels=labels,
loss_fn=loss_fn,
mode=tf.estimator.ModeKeys.TRAIN,
lengths=None,
use_tpu=False,
predictions_fn=_default_predictions_fn)
def test_get_suggestion(self,
get_architecture,
spec,
init_architecture,
completed_trials,
new_block,
should_increase_depth,
expected_fork_architecture,
increase_complexity_probability=1.0):
spec.increase_complexity_probability = increase_complexity_probability
get_architecture.return_value = np.array([1, 2, 3, 4])
algorithm = coordinate_descent.CoordinateDescent(spec, self._metadata)
hparams = hp.HParams(initial_architecture=init_architecture,
new_block_type=new_block)
trials = []
for i in range(completed_trials):
trials.append(
trial_module.Trial({
"id": i,
"model_dir": "/tmp/" + str(i),
"status": "COMPLETED",
"trial_infeasible": False,
"final_measurement": {
"objective_value": 100 - i
}
}))
# Adding one infeasible to make sure we don't fork from it.
trials.append(
trial_module.Trial({
"id": 99,
"model_dir": "/tmp/99",
"status": "COMPLETED",
"trial_infeasible": True,
"final_measurement": {
"objective_value": 0
}
}))
logging.info(trials)
output_architecture, fork_trial = algorithm.get_suggestion(
trials, hparams)
if completed_trials:
self.assertEqual(fork_trial, completed_trials - 1)
if should_increase_depth:
self.assertAllEqual(
output_architecture,
np.append(expected_fork_architecture,
blocks.BlockType[new_block]))
else:
self.assertEqual(output_architecture.shape,
expected_fork_architecture.shape)
self.assertTrue(
search_test_utils.is_mutation_or_equal(
expected_fork_architecture, output_architecture))
def get_block_hparams(hparams, block_name):
if not hparams:
return None
return hp.HParams(
**{
k[len(block_name + "_"):]: v
for k, v in hparams.values().items()
if k.startswith(block_name + "_")
})
def _merge_hparams(original_hparams, overrides):
"""Merges to hp.HParams objects."""
# make a copy
hparams = hp.HParams(**original_hparams.values())
existing_ones = {k: v for k, v in overrides.values().items() if k in hparams}
new_ones = {k: v for k, v in overrides.values().items() if k not in hparams}
hparams.override_from_dict(existing_ones)
for k, v in new_ones.items():
hparams.add_hparam(k, v)
return hparams
def run_parameterized_train_and_eval(phoenix_instance, oracle, tuner_id,
root_dir, max_trials, data_provider,
train_steps, eval_steps, batch_size):
"""Train, getting parameters from a tuner.
Args:
phoenix_instance: a phoenix.Phoenix object.
oracle: a kerastuner oracle.
tuner_id: identifier of the tuner (integer).
root_dir: the root directory to save the models.
max_trials: the maximal number of trials allowed.
data_provider: The data provider object.
train_steps: The number of training steps.
eval_steps: The number of evaluation steps.
batch_size: The batch size (integer).
Returns:
True if the tuner provided a trial to run, False if the tuner
has run out of trials.
"""
trial = oracle.create_trial(tuner_id)
trial_dir = get_trial_dir(root_dir, tuner_id)
my_id = int(os.path.basename(trial_dir))
if my_id > max_trials:
return False
hparams = trial.hyperparameters
phoenix_hparams = aggregate_initial_architecture(hparams.values)
logging.info("Tuner id: %s", tuner_id)
logging.info("Training with the following hyperparameters: ")
logging.info(phoenix_hparams)
with _set_model_dir_for_run_config(model_dir=trial_dir):
evaluation_metrics = run_train_and_eval(
hparams=hp.HParams(**phoenix_hparams),
model_dir=trial_dir,
phoenix_instance=phoenix_instance,
data_provider=data_provider,
train_steps=train_steps,
eval_steps=eval_steps,
batch_size=batch_size)
oracle.update_trial(trial_id=trial.trial_id,
metrics=evaluation_metrics,
step=evaluation_metrics["global_step"])
oracle.end_trial(trial.trial_id,
kerastuner.engine.trial.TrialStatus.COMPLETED)
oracle.update_space(trial.hyperparameters)
# Display needs the updated trial scored by the Oracle.
# self._display.on_trial_end(self.oracle.get_trial(trial.trial_id))
oracle.save()
return True
def test_get_suggestion_beam_size_gt_one(self, get_architecture):
# Trials should be ranked by trial_id. That is, we should only ever fork
# from the first 2 trials.
beam_size = 2
trial_to_arch = {
0: np.array([1, 2, 3]),
1: np.array([4, 5, 6]),
2: np.array([7, 8, 9]),
3: np.array([10, 11, 12])
}
get_architecture.side_effect = lambda idx: trial_to_arch[int(idx)]
spec = search_test_utils.create_spec(phoenix_spec_pb2.PhoenixSpec.DNN)
spec.beam_size = beam_size
spec.increase_complexity_minimum_trials.append(0)
algorithm = coordinate_descent.CoordinateDescent(spec, self._metadata)
hparams = hp.HParams(
new_block_type="FIXED_CHANNEL_CONVOLUTION_16") # enum: 1
# Create one fake trial for each architecture.
trials = []
for i in range(3):
trials.append(
trial_module.Trial({
"id": i,
"model_dir": str(i),
"status": "COMPLETED",
"trial_infeasible": False,
"final_measurement": {
"objective_value": i
}
}))
# Adding one infeasible to make sure we don't fork from it.
trials.append(
trial_module.Trial({
"id": 4,
"model_dir": "4",
"status": "COMPLETED",
"trial_infeasible": True,
"final_measurement": {
"objective_value": 0
}
}))
logging.info(trials)
# Since forking is random, fork 1000 times then check that we forked from
# only the trials we care about.
forked = set()
for i in range(1000):
_, fork_trial = algorithm.get_suggestion(trials, hparams)
forked.add(int(fork_trial))
self.assertEqual(forked, {0, 1})
def _get_suggestion(architectures,
blocks_to_use,
losses,
grow=False,
remove_outliers=False,
pass_flatten=False):
"""Testing subroutine to handle boilerplate Trial construction, dirs, etc."""
# TODO(b/172564129): Figure out how to use mock decorator for free functions.
with mock.patch("model_search.architecture"
".architecture_utils.get_architecture") as mock_get_arch:
blocks_strs = [blocks.BlockType(b).name for b in blocks_to_use]
spec = search_test_utils.create_spec(
phoenix_spec_pb2.PhoenixSpec.CNN,
blocks_to_use=blocks_strs,
)
spec.search_type = phoenix_spec_pb2.PhoenixSpec.LINEAR_MODEL
spec.increase_complexity_probability = 1.0 if grow else 0.0
spec.linear_model.remove_outliers = remove_outliers
spec.linear_model.trials_before_fit = 1
algorithm = linear_model.LinearModel(spec)
mock_get_arch.side_effect = lambda idx: architectures[int(idx)]
trials = []
for i, loss in enumerate(losses):
if isinstance(loss, (np.floating, np.integer)):
loss = loss.item()
trials.append(
trial_module.Trial({
"id": i,
"model_dir": str(i),
"status": "COMPLETED",
"trial_infeasible": False,
"final_measurement": {
"objective_value": loss
}
}))
hparams = hp.HParams(new_block_type=NEW_BLOCK)
# Second return val fork_trial is a nonsense concept for LinearModel.
output_architecture, _ = algorithm.get_suggestion(trials, hparams)
if not pass_flatten:
output_architecture = np.array(
[b for b in output_architecture if b not in blocks.FLATTEN_TYPES])
return output_architecture
def test_generator_with_distillation_and_intermixed(self):
# Force graph mode
with tf.compat.v1.Graph().as_default():
spec = phoenix_spec_pb2.PhoenixSpec(
problem_type=phoenix_spec_pb2.PhoenixSpec.CNN)
spec.is_input_shared = True
spec.search_type = phoenix_spec_pb2.PhoenixSpec.NONADAPTIVE_RANDOM_SEARCH
spec.ensemble_spec.ensemble_search_type = (
ensembling_spec_pb2.EnsemblingSpec.
INTERMIXED_NONADAPTIVE_ENSEMBLE_SEARCH)
spec.ensemble_spec.intermixed_search.width = 2
spec.ensemble_spec.intermixed_search.try_ensembling_every = 4
spec.ensemble_spec.intermixed_search.num_trials_to_consider = 3
spec.distillation_spec.distillation_type = (
distillation_spec_pb2.DistillationSpec.DistillationType.
MSE_LOGITS)
generator = search_candidate_generator.SearchCandidateGenerator(
phoenix_spec=spec,
metadata=ml_metadata_db.MLMetaData(phoenix_spec=spec,
study_name='',
study_owner=''))
fake_config = collections.namedtuple('RunConfig',
['model_dir', 'is_chief'])
run_config = fake_config(model_dir=flags.FLAGS.test_tmpdir +
'/10000',
is_chief=True)
self._create_checkpoint(['search_generator'], 2)
self._create_checkpoint(['search_generator'], 3)
self._create_checkpoint(['search_generator'], 5)
input_tensor = tf.zeros([20, 32, 32, 3])
_ = generator.generate(
features={},
input_layer_fn=lambda: None,
trial_mode=trial_utils.TrialMode.DISTILLATION,
shared_input_tensor=input_tensor,
shared_lengths=None,
logits_dimension=10,
hparams=hp.HParams(initial_architecture=['CONVOLUTION_3X3']),
run_config=run_config,
is_training=True,
trials=trial_utils.create_test_trials_intermixed(
flags.FLAGS.test_tmpdir))
def write_replay_spec(model_dir, filename, original_spec, search_architecture,
hparams):
"""Writes a replay spec to retrain the same model."""
# Ensure the same search space as the original run
replay_spec = copy.deepcopy(original_spec)
# Remove user suggestions
replay_spec.ClearField('user_suggestions')
# If this is already a replay config
replay_spec.ClearField('replay')
dependency_file = os.path.join(model_dir, _PREVIOUS_DEPENDENCIES_FILENAME)
if tf.io.gfile.exists(dependency_file):
with tf.io.gfile.GFile(dependency_file, 'r') as f:
data = f.read().split('\n')
for dependency in data:
spec = phoenix_spec_pb2.PhoenixSpec()
with tf.io.gfile.GFile(os.path.join(dependency, filename),
'r') as f:
text_format.Parse(f.read(), spec)
for i, _ in enumerate(spec.replay.towers):
replay_spec.replay.towers.add().CopyFrom(spec.replay.towers[i])
# Currently we support non-ensembling models.
search_tower = replay_spec.replay.towers.add()
# Removing the "context" hparam. This hparam is added for tpu, by the tpu team
# It is not compatible with the function "to_proto" used below.
copy_hparams = hp.HParams(**hparams.values())
if hasattr(copy_hparams, 'context'):
copy_hparams.del_hparam('context')
# We sometimes (for some search algorithms) override the hparam
# initial_architecture. Updating with the ground truth.
# TODO(b/172564129): Make initial_architecture consistent everywhere.
copy_hparams.set_hparam('initial_architecture', search_architecture)
search_tower.hparams.CopyFrom(copy_hparams.to_proto())
search_tower.architecture[:] = search_architecture
with tf.io.gfile.GFile(os.path.join(model_dir, filename), 'w') as f:
f.write(str(replay_spec))
def test_get_suggestion(self, get_architecture, randint, completed_trials,
initial_architecture, expected_architecture,
replicate_cell):
get_architecture.return_value = initial_architecture
randint.return_value = 2
hparams = hp.HParams(
new_block_type="FIXED_CHANNEL_CONVOLUTION_16") # id=1.
# Use problem_type=DNN so we don't have to worry about the final
# architecture being correct.
spec = phoenix_spec_pb2.PhoenixSpec(
problem_type=phoenix_spec_pb2.PhoenixSpec.DNN,
maximum_depth=11,
num_blocks_in_cell=4,
reduction_block_type="AVERAGE_POOL_2X2",
replicate_cell=replicate_cell,
beam_size=3)
algorithm = constrained_descent.ConstrainedDescent(
spec, self._metadata)
trials = []
for i in range(completed_trials):
trials.append(
trial_module.Trial({
"id": i,
"model_dir": "/tmp/" + str(i),
"status": "COMPLETED",
"trial_infeasible": False,
"final_measurement": {
"objective_value": 100 - i
}
}))
actual_architecture, fork_trial = algorithm.get_suggestion(
trials, hparams)
# We use a beam of size 3 and always choose the last trial to be the best.
self.assertEqual(fork_trial, completed_trials - 3)
self.assertTrue(
search_test_utils.is_mutation_or_equal(actual_architecture,
expected_architecture))
def test_generator_with_snapshot(self):
# Force graph mode
with tf.compat.v1.Graph().as_default():
spec = phoenix_spec_pb2.PhoenixSpec(
problem_type=phoenix_spec_pb2.PhoenixSpec.CNN)
spec.search_type = phoenix_spec_pb2.PhoenixSpec.ADAPTIVE_COORDINATE_DESCENT
spec.transfer_learning_spec.transfer_learning_type = (
transfer_learning_spec_pb2.TransferLearningSpec.
SNAPSHOT_TRANSFER_LEARNING)
spec.is_input_shared = True
generator = search_candidate_generator.SearchCandidateGenerator(
phoenix_spec=spec,
metadata=ml_metadata_db.MLMetaData(phoenix_spec=spec,
study_name='',
study_owner=''))
input_tensor = tf.zeros([20, 32, 32, 3])
fake_config = collections.namedtuple('RunConfig',
['model_dir', 'is_chief'])
tf.io.gfile.makedirs(flags.FLAGS.test_tmpdir + '/3')
run_config = fake_config(model_dir=flags.FLAGS.test_tmpdir + '/3',
is_chief=True)
self._create_checkpoint(['search_generator'], 2)
_ = generator.generate(
features={},
input_layer_fn=lambda: None,
trial_mode=trial_utils.TrialMode.ENSEMBLE_SEARCH,
shared_input_tensor=input_tensor,
shared_lengths=None,
logits_dimension=10,
hparams=hp.HParams(initial_architecture=['CONVOLUTION_3X3'],
dropout_rate=0.3,
new_block_type='CONVOLUTION_3X3'),
run_config=run_config,
is_training=True,
trials=_create_trials(flags.FLAGS.test_tmpdir))
all_nodes = [
node.name for node in
tf.compat.v1.get_default_graph().as_graph_def().node
]
self.assertAllInSet(_DROPOUT_GRAPH_NODE, all_nodes)
def test_get_block_hparams(self):
hp_ = architecture_utils.get_block_hparams(
hp.HParams(TUNABLE_SVDF_target=10, non_target=15), "TUNABLE_SVDF")
self.assertLen(hp_.values(), 1)
self.assertEqual(hp_.target, 10)
def test_multitask(self, learning_rate_spec, contains_node,
not_containing):
# Force graph mode
with tf.compat.v1.Graph().as_default():
spec = phoenix_spec_pb2.PhoenixSpec(
problem_type=phoenix_spec_pb2.PhoenixSpec.DNN)
text_format.Merge(
"""
multi_task_spec {
label_name: "label1"
number_of_classes: 10
}
multi_task_spec {
label_name: "label2"
number_of_classes: 10
}
""", spec)
task_manager_instance = task_manager.TaskManager(spec)
logits = tf.keras.layers.Dense(10)(tf.zeros([20, 10]))
logits_spec = architecture_utils.LogitsSpec(logits=logits)
features = {'x': tf.zeros([10, 10])}
loss_fn = loss_fns.make_multi_class_loss_fn()
model = task_manager_instance.create_model_spec(
features=features,
params=hp.HParams(optimizer='sgd'),
learning_rate_spec=learning_rate_spec,
train_logits_specs=[logits_spec],
eval_logits_spec=logits_spec,
labels={
'label1': tf.ones([20], dtype=tf.int32),
'label2': tf.ones([20], dtype=tf.int32)
},
loss_fn=loss_fn,
mode=tf.estimator.ModeKeys.TRAIN,
lengths=None,
use_tpu=False,
predictions_fn=_default_predictions_fn)
self.assertNotEmpty([
node.name for node in
tf.compat.v1.get_default_graph().as_graph_def().node
if contains_node in node.name
])
for phrase in not_containing:
self.assertEmpty([
node.name for node in
tf.compat.v1.get_default_graph().as_graph_def().node
if phrase in node.name
])
self.assertLen(model.predictions, 3 * (1 + 2))
self.assertContainsSubset([
'probabilities',
'probabilities/label1',
'probabilities/label2',
'log_probabilities',
'log_probabilities/label1',
'log_probabilities/label2',
'predictions',
'predictions/label1',
'predictions/label2',
], model.predictions.keys())
def test_get_suggestion(self, spec, initial_architecture,
expected_architecture):
algorithm = identity.Identity(spec)
output_architecture, _ = algorithm.get_suggestion(
[], hp.HParams(initial_architecture=initial_architecture))
self.assertAllEqual(expected_architecture, output_architecture)
def test_import_tower(self, shared_input):
np.random.seed(42)
test_input = np.random.random([1, 32, 32, 3])
# Force graph mode
with tf.compat.v1.Graph().as_default():
directory = self.get_temp_dir()
architecture = np.array([1, 3, 34])
phoenix_spec = phoenix_spec_pb2.PhoenixSpec(
problem_type=phoenix_spec_pb2.PhoenixSpec.CNN)
phoenix_spec.is_input_shared = shared_input
features = {}
shared_input_tensor = None
with self.test_session(graph=tf.Graph()) as sess:
input_tensor_1 = tf.compat.v1.placeholder(
dtype=tf.float32, shape=[None, 32, 32, 3], name="input_1")
tf.random.set_seed(1234)
tower_spec_1 = architecture_utils.construct_tower(
phoenix_spec=phoenix_spec,
input_tensor=input_tensor_1,
tower_name="test_tower",
architecture=architecture,
is_training=True,
lengths=None,
logits_dimension=10,
hparams=hp.HParams(),
model_directory=self.get_temp_dir(),
is_frozen=False,
dropout_rate=None)
saver = tf.compat.v1.train.Saver()
sess.run(tf.compat.v1.global_variables_initializer())
sess.run(tf.compat.v1.local_variables_initializer())
logits_val_1 = sess.run(tower_spec_1.logits_spec.logits,
feed_dict={input_tensor_1: test_input})
saver.save(sess, directory + "/ckpt")
with self.test_session(graph=tf.Graph()) as sess:
input_tensor_2 = tf.compat.v1.placeholder(
dtype=tf.float32, shape=[None, 32, 32, 3], name="input_2")
if shared_input:
shared_input_tensor = input_tensor_2
def _input_layer_fn(features,
is_training,
scope_name="Phoenix/Input",
lengths_feature_name=None):
del features, is_training, scope_name, lengths_feature_name
return None, None
else:
features = {"x": input_tensor_2}
def _input_layer_fn(features,
is_training,
scope_name="Phoenix/Input",
lengths_feature_name=None):
del is_training, lengths_feature_name
with tf.compat.v1.variable_scope(scope_name):
return tf.cast(features["x"],
dtype=tf.float32), None
tf.random.set_seed(1234)
tower_spec_2 = architecture_utils.import_tower(
features=features,
input_layer_fn=_input_layer_fn,
phoenix_spec=phoenix_spec,
shared_input_tensor=shared_input_tensor,
original_tower_name="test_tower",
new_tower_name="imported_tower",
model_directory=directory,
new_model_directory=self.get_temp_dir(),
is_training=True,
logits_dimension=10,
shared_lengths=None,
force_snapshot=False,
force_freeze=False)
sess.run(tf.compat.v1.global_variables_initializer())
sess.run(tf.compat.v1.local_variables_initializer())
logits_val_2 = sess.run(tower_spec_2.logits_spec.logits,
feed_dict={input_tensor_2: test_input})
self.assertAllClose(logits_val_1, logits_val_2, rtol=1e-3)
