def test_learning_spec_on_predict(self, learning_rate_spec,
not_containing):
spec = phoenix_spec_pb2.PhoenixSpec(
problem_type=phoenix_spec_pb2.PhoenixSpec.DNN)
task_manager_instance = task_manager.TaskManager(spec)
logits = tf.keras.layers.Dense(10)(tf.zeros([20, 10]))
logits_spec = architecture_utils.LogitsSpec(logits=logits)
features = {'x': tf.zeros([10, 10])}
loss_fn = loss_fns.make_multi_class_loss_fn()
model = task_manager_instance.create_model_spec(
features=features,
params=hp.HParams(optimizer='sgd'),
learning_rate_spec=learning_rate_spec,
train_logits_specs=[logits_spec],
eval_logits_spec=logits_spec,
labels=tf.ones([20], dtype=tf.int32),
loss_fn=loss_fn,
mode=tf.estimator.ModeKeys.PREDICT,
lengths=None,
use_tpu=False,
predictions_fn=_default_predictions_fn)
for phrase in not_containing:
self.assertEmpty([
node.name for node in
tf.compat.v1.get_default_graph().as_graph_def().node
if phrase in node.name
])
self.assertLen(model.predictions, 3)
self.assertIn('probabilities', model.predictions)
self.assertIn('log_probabilities', model.predictions)
self.assertIn('predictions', model.predictions)
self.assertIsNone(model.loss)
def test_tpu(self):
# Force graph mode
with tf.compat.v1.Graph().as_default():
learning_rate_spec = {
'learning_rate': 0.001,
'gradient_max_norm': 3
}
spec = phoenix_spec_pb2.PhoenixSpec(
problem_type=phoenix_spec_pb2.PhoenixSpec.DNN)
task_manager_instance = task_manager.TaskManager(spec)
logits = tf.keras.layers.Dense(10)(tf.zeros([20, 10]))
logits_spec = architecture_utils.LogitsSpec(logits=logits)
features = {'x': tf.zeros([10, 10])}
loss_fn = loss_fns.make_multi_class_loss_fn()
_ = task_manager_instance.create_model_spec(
features=features,
params=hp.HParams(optimizer='sgd'),
learning_rate_spec=learning_rate_spec,
train_logits_specs=[logits_spec],
eval_logits_spec=logits_spec,
labels=tf.ones([20], dtype=tf.int32),
loss_fn=loss_fn,
mode=tf.estimator.ModeKeys.TRAIN,
lengths=None,
use_tpu=True,
predictions_fn=_default_predictions_fn)
self.assertNotEmpty([
node.name for node in
tf.compat.v1.get_default_graph().as_graph_def().node
if 'CrossReplicaSum' in node.name
])
def test_architecture(self):
# Force graph mode
with tf.compat.v1.Graph().as_default():
learning_rate_spec = {'learning_rate': 0.001, 'gradient_max_norm': 3}
spec = phoenix_spec_pb2.PhoenixSpec(
problem_type=phoenix_spec_pb2.PhoenixSpec.CNN)
text_format.Merge(
"""
multi_task_spec {
label_name: "label1"
number_of_classes: 10
architecture: "FIXED_OUTPUT_FULLY_CONNECTED_128"
}
multi_task_spec {
label_name: "label2"
number_of_classes: 10
architecture: "FIXED_OUTPUT_FULLY_CONNECTED_256"
architecture: "FIXED_OUTPUT_FULLY_CONNECTED_512"
}
""", spec)
task_manager_instance = task_manager.TaskManager(spec)
logits = tf.keras.layers.Dense(10)(tf.zeros([20, 10]))
logits_spec = architecture_utils.LogitsSpec(logits=logits)
features = {'x': tf.zeros([10, 10])}
loss_fn = loss_fns.make_multi_class_loss_fn()
model = task_manager_instance.create_model_spec(
features=features,
params=hp.HParams(optimizer='sgd'),
learning_rate_spec=learning_rate_spec,
train_logits_specs=[logits_spec],
eval_logits_spec=logits_spec,
labels={
'label1': tf.ones([20], dtype=tf.int32),
'label2': tf.ones([20], dtype=tf.int32)
},
loss_fn=loss_fn,
model_directory=self.get_temp_dir(),
mode=tf.estimator.ModeKeys.TRAIN,
lengths=None,
use_tpu=False,
predictions_fn=_default_predictions_fn)
self.assertNotEmpty([
node.name
for node in tf.compat.v1.get_default_graph().as_graph_def().node
if 'task_label1_tower/1_FIXED_OUTPUT_FULLY_CONNECTED_128' in node.name
])
self.assertNotEmpty([
node.name
for node in tf.compat.v1.get_default_graph().as_graph_def().node
if 'task_label2_tower/1_FIXED_OUTPUT_FULLY_CONNECTED_256' in node.name
])
self.assertNotEmpty([
node.name
for node in tf.compat.v1.get_default_graph().as_graph_def().node
if 'task_label2_tower/2_FIXED_OUTPUT_FULLY_CONNECTED_512' in node.name
])
self.assertLen(model.predictions, 3 * (1 + 2))
self.assertIn('probabilities', model.predictions)
self.assertIn('log_probabilities', model.predictions)
self.assertIn('predictions', model.predictions)
def test_weight_feature(self, is_multitask, weight_is_a_feature):
# Force graph mode
with tf.compat.v1.Graph().as_default():
learning_rate_spec = {
'learning_rate': 0.001,
'gradient_max_norm': 3
}
spec = phoenix_spec_pb2.PhoenixSpec(
problem_type=phoenix_spec_pb2.PhoenixSpec.DNN)
labels = tf.ones([20], dtype=tf.int32)
if is_multitask:
text_format.Merge(
"""
multi_task_spec {
label_name: "label1"
number_of_classes: 10
weight_feature_name: "weight1"
weight_is_a_feature: %s
}
multi_task_spec {
label_name: "label2"
number_of_classes: 10
weight_feature_name: "weight2"
weight_is_a_feature: %s
}
""" % (str(weight_is_a_feature), str(weight_is_a_feature)), spec)
labels = {
'label1': tf.ones([20], dtype=tf.int32),
'label2': tf.ones([20], dtype=tf.int32)
}
weights = {
'weight1': tf.constant([2] * 20),
'weight2': tf.constant([3] * 20)
}
features = {'x': tf.zeros([10, 10])}
if weight_is_a_feature:
features.update(weights)
elif isinstance(labels, dict):
labels.update(weights)
task_manager_instance = task_manager.TaskManager(spec)
logits = tf.keras.layers.Dense(10)(tf.zeros([20, 10]))
logits_spec = architecture_utils.LogitsSpec(logits=logits)
loss_fn = loss_fns.make_multi_class_loss_fn()
_ = task_manager_instance.create_model_spec(
features=features,
params=hp.HParams(optimizer='sgd'),
learning_rate_spec=learning_rate_spec,
train_logits_specs=[logits_spec],
eval_logits_spec=logits_spec,
labels=labels,
loss_fn=loss_fn,
mode=tf.estimator.ModeKeys.TRAIN,
lengths=None,
use_tpu=False,
predictions_fn=_default_predictions_fn)
def test_wrong_dict_weight_feature(self, weight_is_a_feature):
learning_rate_spec = {'learning_rate': 0.001, 'gradient_max_norm': 3}
spec = phoenix_spec_pb2.PhoenixSpec(
problem_type=phoenix_spec_pb2.PhoenixSpec.DNN)
text_format.Merge(
"""
multi_task_spec {
label_name: "label1"
number_of_classes: 10
weight_feature_name: "weight1"
weight_is_a_feature: %s
}
multi_task_spec {
label_name: "label2"
number_of_classes: 10
weight_feature_name: "weight2"
weight_is_a_feature: %s
}
""" % (str(weight_is_a_feature), str(weight_is_a_feature)), spec)
labels = {
'label1': tf.ones([20], dtype=tf.int32),
'label2': tf.ones([20], dtype=tf.int32),
}
# Fix the size of the dict labels to bypass the assertion.
if not weight_is_a_feature:
labels.update({
'not_used': tf.ones([20], dtype=tf.int32),
'not_used2': tf.ones([20], dtype=tf.int32)
})
weights = {
'weight1': tf.constant([2] * 20),
'weight2': tf.constant([3] * 20)
}
features = {'x': tf.zeros([10, 10])}
if not weight_is_a_feature:
features.update(weights)
task_manager_instance = task_manager.TaskManager(spec)
logits = tf.keras.layers.Dense(10)(tf.zeros([20, 10]))
logits_spec = architecture_utils.LogitsSpec(logits=logits)
with self.assertRaises(KeyError):
loss_fn = loss_fns.make_multi_class_loss_fn()
_ = task_manager_instance.create_model_spec(
features=features,
params=hp.HParams(optimizer='sgd'),
learning_rate_spec=learning_rate_spec,
train_logits_specs=[logits_spec],
eval_logits_spec=logits_spec,
labels=labels,
loss_fn=loss_fn,
model_directory=self.get_temp_dir(),
mode=tf.estimator.ModeKeys.TRAIN,
lengths=None,
use_tpu=False,
predictions_fn=_default_predictions_fn)
def test_model_fn_binary_class(self, problem, input_shape,
label_vocabulary, labels_fn, hparams):
num_classes = 2
loss_fn = loss_fns.make_multi_class_loss_fn(
label_vocabulary=label_vocabulary)
instance = self._create_phoenix_instance(
problem_type=problem,
input_shape=input_shape,
logits_dimension=num_classes,
label_vocabulary=label_vocabulary,
loss_fn=loss_fn)
run_config = tf.estimator.RunConfig(model_dir=self.get_temp_dir() +
"/1")
def input_fn():
features = {"zeros": tf.zeros(input_shape)}
labels = labels_fn()
return features, labels
estimator = instance.get_estimator(run_config, hparams, 10)
estimator.train(input_fn=input_fn, max_steps=10)
eval_result = estimator.evaluate(input_fn=input_fn, steps=10)
self.assertIn("auc_roc", eval_result)
def test_multitask(self, learning_rate_spec, contains_node,
not_containing):
# Force graph mode
with tf.compat.v1.Graph().as_default():
spec = phoenix_spec_pb2.PhoenixSpec(
problem_type=phoenix_spec_pb2.PhoenixSpec.DNN)
text_format.Merge(
"""
multi_task_spec {
label_name: "label1"
number_of_classes: 10
}
multi_task_spec {
label_name: "label2"
number_of_classes: 10
}
""", spec)
task_manager_instance = task_manager.TaskManager(spec)
logits = tf.keras.layers.Dense(10)(tf.zeros([20, 10]))
logits_spec = architecture_utils.LogitsSpec(logits=logits)
features = {'x': tf.zeros([10, 10])}
loss_fn = loss_fns.make_multi_class_loss_fn()
model = task_manager_instance.create_model_spec(
features=features,
params=hp.HParams(optimizer='sgd'),
learning_rate_spec=learning_rate_spec,
train_logits_specs=[logits_spec],
eval_logits_spec=logits_spec,
labels={
'label1': tf.ones([20], dtype=tf.int32),
'label2': tf.ones([20], dtype=tf.int32)
},
loss_fn=loss_fn,
mode=tf.estimator.ModeKeys.TRAIN,
lengths=None,
use_tpu=False,
predictions_fn=_default_predictions_fn)
self.assertNotEmpty([
node.name for node in
tf.compat.v1.get_default_graph().as_graph_def().node
if contains_node in node.name
])
for phrase in not_containing:
self.assertEmpty([
node.name for node in
tf.compat.v1.get_default_graph().as_graph_def().node
if phrase in node.name
])
self.assertLen(model.predictions, 3 * (1 + 2))
self.assertContainsSubset([
'probabilities',
'probabilities/label1',
'probabilities/label2',
'log_probabilities',
'log_probabilities/label1',
'log_probabilities/label2',
'predictions',
'predictions/label1',
'predictions/label2',
], model.predictions.keys())
def __init__(self,
phoenix_spec,
input_layer_fn,
study_owner,
study_name,
head=None,
logits_dimension=None,
label_vocabulary=None,
loss_fn=None,
metric_fn=None,
predictions_fn=None,
metadata=None):
"""Constructs a Phoenix instance.
Args:
phoenix_spec: A `PhoenixSpec` proto with the spec for the run.
input_layer_fn: A function that converts feature Tensors to input layer.
See learning.autolx.model_search.data.Provider.get_input_layer_fn
for details.
study_owner: A string holding the ldap of the study owner. We use tuner
platforms to conduct the various architectures training. This field
specifies the study owner.
study_name: A string holding the study name.
head: A head to use with Phoenix for creating the loss and eval metrics.
If no head is given, Phoenix falls back to using the loss_fn and
metric_fn. N.B.: Phoenix creates its own EstimatorSpec so everything
besides the loss and eval metrics returned by head will be ignored.
logits_dimension: An int holding the dimension of the output. Must be
provided if head is None. Will be ignored if head is not None.
label_vocabulary: List or tuple with labels vocabulary. Needed only if the
labels are of type string. This list is used by the loss function if
loss_fn is not provided. It is also used in the metric function to
create the accuracy metric ops. Use only with multiclass classification
problems.
loss_fn: A function to compute the loss. Ignored if `head` is not None.
Must accept as inputs a `labels` Tensor, a `logits` Tensor, and
optionally a `weights` Tensor. `weights` must either be rank 0 or have
the same rank as labels. If None, Phoenix defaults to using softmax
cross-entropy.
metric_fn: Metrics for Tensorboard. Ignored if `head` is not None.
metric_fn takes `label` and `predictions` as input, and outputs a
dictionary of (tensor, update_op) tuples. `label` is a Tensor (in the
single task case) or a dict of Tensors (in the case of multi-task, where
the key of the dicts correspond to the task names). `predictions` is a
dict of Tensors. In the single task case, it consists of `predictions`,
`probabilities`, and `log_probabilities`. In the multi-task case, it
consists of the same keys as that of the single task case, but also
those corresponding to each task (e.g., predictions/task_name_1). See
`metric_fns` for more detail. If `metric_fn` is None, it will include a
metric for the number of parameters, accuracy (if logit_dimensions >=
2), and AUC metrics (if logit_dimensions == 2).
predictions_fn: A function to convert eval logits to the
`predictions` dictionary passed to metric_fn. If `None`, defaults to
computing 'predictions', 'probabilities', and 'log_probabilities'.
metadata: An object that implements metadata api in
learning.adanets.phoenix.metadata.Metadata
"""
# Check Phoenix preconditions and fail early if any of them are broken.
if phoenix_spec.multi_task_spec:
# TODO(b/172564129): Add support for head and custom loss_fns in
# multi-task.
assert not head, "head is not supported for multi-task."
if head:
msg = "Do not specify {} when using head as head already contains it."
assert not logits_dimension, msg.format("logits_dimension")
assert not label_vocabulary, msg.format("label_vocabulary")
assert not loss_fn, msg.format("loss_fn")
assert not metric_fn, msg.format("metric_fn")
# Check ensemble search / distillation preconditions.
ensemble_spec = phoenix_spec.ensemble_spec
distillation_spec = phoenix_spec.distillation_spec
if trial_utils.has_distillation(
distillation_spec) and trial_utils.has_ensemble_search(
ensemble_spec
) and not trial_utils.is_intermixed_ensemble_search(ensemble_spec):
ensemble_search_spec = (
ensemble_spec.nonadaptive_search
if trial_utils.is_nonadaptive_ensemble_search(ensemble_spec) else
ensemble_spec.adaptive_search)
if (distillation_spec.minimal_pool_size ==
ensemble_search_spec.minimal_pool_size):
logging.warning("minimal_pool_size is the same for ensemble spec and "
"distillation spec, so distillation will be ignored.")
self._phoenix_spec = phoenix_spec
self._input_layer_fn = input_layer_fn
self._ensembler = ensembler.Ensembler(phoenix_spec)
self._distiller = distillation.Distiller(phoenix_spec.distillation_spec)
self._study_owner = study_owner
self._study_name = study_name
self._head = head
self._logits_dimension = (
self._head.logits_dimension if head else logits_dimension)
self._label_vocabulary = label_vocabulary
if self._label_vocabulary:
assert self._logits_dimension == len(self._label_vocabulary)
self._loss_fn = loss_fn or loss_fns.make_multi_class_loss_fn(
label_vocabulary=label_vocabulary)
self._user_specified_metric_fn = metric_fn
self._predictions_fn = (predictions_fn or _default_predictions_fn)
if metadata is None:
self._metadata = ml_metadata_db.MLMetaData(phoenix_spec, study_name,
study_owner)
else:
self._metadata = metadata
self._task_manager = task_manager.TaskManager(phoenix_spec)
self._controller = controller.InProcessController(
phoenix_spec=phoenix_spec, metadata=self._metadata)
class PhoenixTest(parameterized.TestCase, tf.test.TestCase):
def _create_phoenix_spec(self, problem_type):
"""Creates a new phoenix.PhoenixSpec for the given problem type."""
spec_path = os.path.join(FLAGS.test_srcdir,
_SPEC_PATH_TEMPLATE.format(problem_type))
spec = phoenix_spec_pb2.PhoenixSpec()
with tf.io.gfile.GFile(spec_path, "r") as f:
text_format.Merge(f.read(), spec)
return spec
def _create_phoenix_instance(self,
problem_type,
input_shape=None,
head=None,
logits_dimension=None,
label_vocabulary=None,
loss_fn=None,
metric_fn=None):
"""Creates a phoenix.Phoenix instance with mostly default parameters.
Args:
problem_type: The problem type (e.g. cnn, rnn, etc) for which to create a
PhoenixSpec.
input_shape: The shape of the input Tensor (including batch size).
head: The head to pass to the Phoenix instance.
logits_dimension: The shape of the logits to pass to the Phoenix instance.
label_vocabulary: The label_vocabulary to pass to the Phoenix instance.
loss_fn: The loss_fn to pass to the Phoenix instance.
metric_fn: The metric_fn to pass to the Phoenix instance.
Returns:
A newly created phoenix.Phoenix instance.
"""
spec = self._create_phoenix_spec(problem_type)
input_shape = input_shape or [8, 32, 32, 3]
logits_dimension = (10 if not logits_dimension and not head else
logits_dimension)
def _input_layer_fn(features, is_training, scope_name,
lengths_feature_name):
del is_training
del scope_name
input_layer = tf.cast(features["zeros"], dtype=tf.float32)
if (spec.problem_type
== phoenix_spec_pb2.PhoenixSpec.RNN_ALL_ACTIVATIONS
or spec.problem_type
== phoenix_spec_pb2.PhoenixSpec.RNN_LAST_ACTIVATIONS):
lengths = features[lengths_feature_name]
else:
lengths = None
return input_layer, lengths
return phoenix.Phoenix(phoenix_spec=spec,
input_layer_fn=_input_layer_fn,
study_owner="test_owner",
study_name="test_name",
head=head,
logits_dimension=logits_dimension,
label_vocabulary=label_vocabulary,
loss_fn=loss_fn,
metric_fn=metric_fn,
metadata=mock_metadata.MockMetaData())
@parameterized.named_parameters(
{
"testcase_name": "multi_task_with_head",
"kwargs_to_pass": {
"phoenix_spec":
phoenix_spec_pb2.PhoenixSpec(
multi_task_spec=[phoenix_spec_pb2.TaskSpec()]),
"head":
tf.estimator.BinaryClassHead(),
}
}, {
"testcase_name": "head_with_logits_dimension",
"kwargs_to_pass": {
"head": tf.estimator.BinaryClassHead(),
"logits_dimension": 10,
}
}, {
"testcase_name": "head_with_label_vocabulary",
"kwargs_to_pass": {
"head": tf.estimator.BinaryClassHead(),
"label_vocabulary": {
"the_answer": 42
},
}
}, {
"testcase_name": "head_with_loss_fn",
"kwargs_to_pass": {
"head": tf.estimator.BinaryClassHead(),
"loss_fn": lambda loss: 10,
}
}, {
"testcase_name": "head_with_metrics_fn",
"kwargs_to_pass": {
"head": tf.estimator.BinaryClassHead(),
"metric_fn": lambda labels, predictions, weights: 10,
}
})
def test_preconditions(self, kwargs_to_pass=None):
if "phoenix_spec" not in kwargs_to_pass:
kwargs_to_pass["phoenix_spec"] = phoenix_spec_pb2.PhoenixSpec()
kwargs_to_pass["input_layer_fn"] = lambda: None
kwargs_to_pass["study_owner"] = "fake_owner"
kwargs_to_pass["study_name"] = "fake_name"
with self.assertRaises(AssertionError):
phoenix.Phoenix(**kwargs_to_pass)
@parameterized.named_parameters({
"testcase_name": "cnn",
"problem": "cnn"
}, {
"testcase_name": "dnn",
"problem": "dnn"
}, {
"testcase_name": "rnn_all",
"problem": "rnn_all"
}, {
"testcase_name": "rnn_last",
"problem": "rnn_last"
})
def test_get_keras_hyperparameters_space(self, problem):
spec = self._create_phoenix_spec(problem)
_ = phoenix.Phoenix.get_keras_hyperparameters_space(phoenix_spec=spec,
train_steps=10000)
@parameterized.named_parameters(
{
"testcase_name":
"cnn",
"problem":
"cnn",
"input_shape": [20, 32, 32, 3],
"hparams":
hp.HParams(initial_architecture=[
"FIXED_CHANNEL_CONVOLUTION_16", "FIXED_CHANNEL_CONVOLUTION_64",
"PLATE_REDUCTION_FLATTEN"
],
learning_rate=0.01,
optimizer="sgd"),
"label_shape": [20]
}, {
"testcase_name":
"dnn",
"problem":
"dnn",
"input_shape": [20, 32],
"hparams":
hp.HParams(initial_architecture=[
"FIXED_OUTPUT_FULLY_CONNECTED_128",
"FIXED_OUTPUT_FULLY_CONNECTED_128",
],
learning_rate=0.01,
optimizer="sgd"),
"label_shape": [20]
}, {
"testcase_name":
"rnn_all",
"problem":
"rnn_all",
"input_shape": [20, 32, 32],
"hparams":
hp.HParams(initial_architecture=[
"RNN_CELL_64",
"RNN_CELL_64",
],
learning_rate=0.01,
optimizer="sgd"),
"label_shape": [20, 32]
}, {
"testcase_name":
"rnn_last",
"problem":
"rnn_last",
"input_shape": [20, 32, 32],
"hparams":
hp.HParams(initial_architecture=[
"RNN_CELL_64",
"RNN_CELL_64",
],
learning_rate=0.01,
optimizer="sgd"),
"label_shape": [20]
})
def test_model_fn(self, problem, input_shape, hparams, label_shape):
# Force graph mode
with tf.compat.v1.Graph().as_default():
instance = self._create_phoenix_instance(problem, input_shape)
features = {
"zeros": tf.zeros(input_shape),
"lengths": tf.ones([20])
}
labels = tf.ones(label_shape, dtype=tf.int32)
run_config = tf.estimator.RunConfig(model_dir=self.get_temp_dir() +
"/1")
model_fn = instance._make_model_fn(run_config, train_steps=100)
_ = model_fn(features, labels, tf.estimator.ModeKeys.TRAIN,
hparams)
@parameterized.named_parameters(
{
"testcase_name":
"cnn",
"problem":
"cnn",
"input_shape": [20, 32, 32, 3],
"hparams":
hp.HParams(initial_architecture=[
"FIXED_CHANNEL_CONVOLUTION_16", "FIXED_CHANNEL_CONVOLUTION_64",
"PLATE_REDUCTION_FLATTEN"
]),
}, {
"testcase_name":
"dnn",
"problem":
"dnn",
"input_shape": [20, 32],
"hparams":
hp.HParams(initial_architecture=[
"FIXED_OUTPUT_FULLY_CONNECTED_128",
"FIXED_OUTPUT_FULLY_CONNECTED_128",
]),
}, {
"testcase_name":
"rnn_all",
"problem":
"rnn_all",
"input_shape": [20, 32, 32],
"hparams":
hp.HParams(initial_architecture=[
"RNN_CELL_64",
"RNN_CELL_64",
]),
}, {
"testcase_name":
"rnn_last",
"problem":
"rnn_last",
"input_shape": [20, 32, 32],
"hparams":
hp.HParams(initial_architecture=[
"RNN_CELL_64",
"RNN_CELL_64",
],
learning_rate=0.01,
optimizer="sgd"),
})
def test_model_fn_predict(self, problem, input_shape, hparams):
instance = self._create_phoenix_instance(problem, input_shape)
features = {"zeros": tf.zeros(input_shape), "lengths": tf.ones([20])}
run_config = tf.estimator.RunConfig(model_dir=self.get_temp_dir() +
"/1")
model_fn = instance._make_model_fn(run_config, train_steps=100)
spec = model_fn(features, None, tf.estimator.ModeKeys.PREDICT, hparams)
self.assertIsNone(spec.loss)
self.assertNotEqual(spec.predictions, None)
@parameterized.named_parameters(
{
"testcase_name": "no_scaled_training",
"use_parameter_scaled_training": False,
"train_steps": 10000,
"maximum_depth": 100,
"architecture": np.array([1, 2, 3]),
"expected": 1
},
{
"testcase_name": "no_architecture",
"use_parameter_scaled_training": False,
"train_steps": 10000,
"maximum_depth": 100,
"architecture": None,
"expected": 1
},
{
"testcase_name": "scaled_training",
"use_parameter_scaled_training": True,
"train_steps": 10000,
"maximum_depth": 10,
"architecture": np.array([1, 2, 3]),
"expected": 3
},
)
def test_increment_global_step(self, use_parameter_scaled_training,
train_steps, maximum_depth, architecture,
expected):
# Force graph mode
with tf.compat.v1.Graph().as_default():
with self.test_session(graph=tf.Graph()) as sess:
tower_name = "tower"
if architecture is not None:
architecture_utils.set_architecture(architecture,
tower_name=tower_name)
spec = self._create_phoenix_spec(problem_type="cnn")
spec.maximum_depth = maximum_depth
spec.use_parameter_scaled_training = use_parameter_scaled_training
instance = phoenix.Phoenix(phoenix_spec=spec,
input_layer_fn=lambda: None,
logits_dimension=0,
study_name="test",
study_owner="test")
global_step = tf.compat.v1.train.get_or_create_global_step()
sess.run(tf.compat.v1.global_variables_initializer())
sess.run(tf.compat.v1.local_variables_initializer())
before = sess.run(global_step)
op = instance._increment_global_step(train_op=tf.no_op(),
train_steps=train_steps,
tower_name=tower_name)
sess.run(op)
after = sess.run(global_step)
self.assertEqual(before, 0)
self.assertEqual(after, expected)
def test_merge_hparams(self):
params = phoenix._merge_hparams(
hp.HParams(learning_rate=1, untouched=4),
hp.HParams(learning_rate=2, new_param=3))
self.assertEqual(params.learning_rate, 2)
self.assertEqual(params.untouched, 4)
self.assertEqual(params.new_param, 3)
@parameterized.named_parameters(
{
"testcase_name":
"cnn",
"problem":
"cnn",
"input_shape": [20, 32, 32, 3],
"hparams":
hp.HParams(initial_architecture=[
"FIXED_CHANNEL_CONVOLUTION_16", "FIXED_CHANNEL_CONVOLUTION_64",
"PLATE_REDUCTION_FLATTEN"
],
learning_rate=0.01,
new_block_type="FIXED_CHANNEL_CONVOLUTION_16",
optimizer="sgd"),
}, {
"testcase_name":
"dnn",
"problem":
"dnn",
"input_shape": [20, 32],
"hparams":
hp.HParams(initial_architecture=[
"FIXED_OUTPUT_FULLY_CONNECTED_128",
"FIXED_OUTPUT_FULLY_CONNECTED_128",
],
learning_rate=0.01,
new_block_type="FIXED_OUTPUT_FULLY_CONNECTED_128",
optimizer="sgd"),
}, {
"testcase_name":
"dnn_no_vocab",
"problem":
"dnn",
"input_shape": [20, 32],
"hparams":
hp.HParams(initial_architecture=[
"FIXED_OUTPUT_FULLY_CONNECTED_128",
"FIXED_OUTPUT_FULLY_CONNECTED_128",
],
learning_rate=0.01,
new_block_type="FIXED_OUTPUT_FULLY_CONNECTED_128",
optimizer="sgd"),
"vocab": [],
"loss_fn":
loss_fns.make_multi_class_loss_fn(label_vocabulary=[
"CLASS1", "CLASS2", "CLASS3", "CLASS4", "CLASS5"
]),
})
def test_model_fn_with_vocab(self,
problem,
input_shape,
hparams,
vocab=None,
loss_fn=None):
if vocab is None:
vocab = ["CLASS1", "CLASS2", "CLASS3", "CLASS4", "CLASS5"]
instance = self._create_phoenix_instance(problem_type=problem,
input_shape=input_shape,
logits_dimension=5,
label_vocabulary=vocab,
loss_fn=loss_fn)
run_config = tf.estimator.RunConfig(model_dir=self.get_temp_dir() +
"/1")
def input_fn():
features = {"zeros": tf.zeros(input_shape)}
labels = tf.constant(["CLASS1", "CLASS1"] * 10, dtype=tf.string)
return features, labels
estimator = instance.get_estimator(run_config, hparams, 10)
estimator.train(input_fn=input_fn, max_steps=10)
estimator.evaluate(input_fn=input_fn, steps=10)
@parameterized.named_parameters(
{
"testcase_name":
"dnn_regression",
"problem":
"dnn",
"input_shape": [10, 32],
"hparams":
hp.HParams(initial_architecture=[
"FIXED_OUTPUT_FULLY_CONNECTED_128",
"FIXED_OUTPUT_FULLY_CONNECTED_128",
],
learning_rate=0.01,
new_block_type="FIXED_OUTPUT_FULLY_CONNECTED_128",
optimizer="sgd"),
"loss_fn":
loss_fns.make_regression_loss_fn(),
}, {
"testcase_name":
"cnn_regression",
"problem":
"dnn",
"input_shape": [10, 32, 32, 3],
"hparams":
hp.HParams(initial_architecture=[
"FIXED_OUTPUT_FULLY_CONNECTED_128",
"FIXED_OUTPUT_FULLY_CONNECTED_128",
],
learning_rate=0.01,
new_block_type="FIXED_OUTPUT_FULLY_CONNECTED_128",
optimizer="sgd"),
"loss_fn":
loss_fns.make_regression_loss_fn(),
})
def test_model_fn_regression(self,
problem,
input_shape,
hparams,
loss_fn=None,
metric_fn=None):
instance = self._create_phoenix_instance(problem_type=problem,
input_shape=input_shape,
logits_dimension=1,
loss_fn=loss_fn,
metric_fn=metric_fn)
run_config = tf.estimator.RunConfig(model_dir=self.get_temp_dir() +
"/1")
def input_fn():
features = {"zeros": tf.zeros(input_shape)}
labels = tf.constant([[0.]] * 10, dtype=tf.float32)
return features, labels
estimator = instance.get_estimator(run_config, hparams, 10)
estimator.train(input_fn=input_fn, max_steps=10)
eval_result = estimator.evaluate(input_fn=input_fn, steps=10)
self.assertAlmostEqual(0., eval_result["loss"])
@parameterized.named_parameters(
{
"testcase_name":
"cnn_custom_metric",
"problem":
"cnn",
"input_shape": [20, 32, 32, 3],
"hparams":
hp.HParams(initial_architecture=[
"FIXED_CHANNEL_CONVOLUTION_16", "FIXED_CHANNEL_CONVOLUTION_64",
"PLATE_REDUCTION_FLATTEN"
],
learning_rate=0.01,
new_block_type="FIXED_CHANNEL_CONVOLUTION_16",
optimizer="sgd"),
}, {
"testcase_name":
"dnn_custom_metric",
"problem":
"dnn",
"input_shape": [20, 32],
"hparams":
hp.HParams(initial_architecture=[
"FIXED_OUTPUT_FULLY_CONNECTED_128",
"FIXED_OUTPUT_FULLY_CONNECTED_128",
],
learning_rate=0.01,
new_block_type="FIXED_OUTPUT_FULLY_CONNECTED_128",
optimizer="sgd"),
})
def test_model_fn_custom_metric_fn(self, problem, input_shape, hparams):
num_classes = 5
def metric_fn(labels, predictions, weights):
del weights
return {
"mean_per_class_accuracy":
(tf.compat.v1.metrics.mean_per_class_accuracy(
labels, predictions["predictions"], num_classes))
}
instance = self._create_phoenix_instance(problem_type=problem,
input_shape=input_shape,
logits_dimension=num_classes,
metric_fn=metric_fn)
run_config = tf.estimator.RunConfig(model_dir=self.get_temp_dir() +
"/1")
def input_fn():
features = {"zeros": tf.zeros(input_shape)}
labels = tf.constant([0] * 20, dtype=tf.int64)
return features, labels
estimator = instance.get_estimator(run_config, hparams, 10)
estimator.train(input_fn=input_fn, max_steps=10)
eval_result = estimator.evaluate(input_fn=input_fn, steps=10)
self.assertIn("mean_per_class_accuracy", eval_result)
@parameterized.named_parameters(
{
"testcase_name":
"dnn",
"problem":
"dnn",
"input_shape": [10, 32],
"label_vocabulary":
None,
"labels_fn":
lambda: tf.constant([0] * 10, dtype=tf.int64),
"hparams":
hp.HParams(initial_architecture=[
"FIXED_OUTPUT_FULLY_CONNECTED_128",
"FIXED_OUTPUT_FULLY_CONNECTED_128",
],
learning_rate=0.01,
new_block_type="FIXED_OUTPUT_FULLY_CONNECTED_128",
optimizer="sgd"),
}, {
"testcase_name":
"dnn_vocab",
"problem":
"dnn",
"input_shape": [10, 32],
"label_vocabulary": ["ZERO", "ONE"],
"labels_fn":
lambda: tf.constant(["ZERO"] * 10, dtype=tf.string),
"hparams":
hp.HParams(initial_architecture=[
"FIXED_OUTPUT_FULLY_CONNECTED_128",
"FIXED_OUTPUT_FULLY_CONNECTED_128",
],
learning_rate=0.01,
new_block_type="FIXED_OUTPUT_FULLY_CONNECTED_128",
optimizer="sgd"),
}, {
"testcase_name":
"cnn",
"problem":
"cnn",
"input_shape": [10, 32, 32, 3],
"label_vocabulary":
None,
"labels_fn":
lambda: tf.constant([0] * 10, dtype=tf.int64),
"hparams":
hp.HParams(initial_architecture=[
"FIXED_CHANNEL_CONVOLUTION_16", "FIXED_CHANNEL_CONVOLUTION_64",
"PLATE_REDUCTION_FLATTEN"
],
learning_rate=0.01,
new_block_type="FIXED_CHANNEL_CONVOLUTION_16",
optimizer="sgd"),
}, {
"testcase_name":
"cnn_vocab",
"problem":
"cnn",
"input_shape": [10, 32, 32, 3],
"label_vocabulary": ["ZERO", "ONE"],
"labels_fn":
lambda: tf.constant(["ZERO"] * 10, dtype=tf.string),
"hparams":
hp.HParams(initial_architecture=[
"FIXED_CHANNEL_CONVOLUTION_16", "FIXED_CHANNEL_CONVOLUTION_64",
"PLATE_REDUCTION_FLATTEN"
],
learning_rate=0.01,
new_block_type="FIXED_CHANNEL_CONVOLUTION_16",
optimizer="sgd"),
})
def test_model_fn_binary_class(self, problem, input_shape,
label_vocabulary, labels_fn, hparams):
num_classes = 2
loss_fn = loss_fns.make_multi_class_loss_fn(
label_vocabulary=label_vocabulary)
instance = self._create_phoenix_instance(
problem_type=problem,
input_shape=input_shape,
logits_dimension=num_classes,
label_vocabulary=label_vocabulary,
loss_fn=loss_fn)
run_config = tf.estimator.RunConfig(model_dir=self.get_temp_dir() +
"/1")
def input_fn():
features = {"zeros": tf.zeros(input_shape)}
labels = labels_fn()
return features, labels
estimator = instance.get_estimator(run_config, hparams, 10)
estimator.train(input_fn=input_fn, max_steps=10)
eval_result = estimator.evaluate(input_fn=input_fn, steps=10)
self.assertIn("auc_roc", eval_result)
# pylint: disable=unnecessary-lambda
@parameterized.named_parameters(
{
"testcase_name": "regression",
"head_fn": lambda: tf.estimator.RegressionHead()
},
{
"testcase_name": "binary_classification",
"head_fn": lambda: tf.estimator.BinaryClassHead()
},
{
"testcase_name": "multi_class_classification",
"head_fn": lambda: tf.estimator.MultiClassHead(n_classes=10)
},
{
"testcase_name": "multi_label_classification",
"head_fn": lambda: tf.estimator.MultiLabelHead(n_classes=10),
# (batch_size, logits_dimension)
"label_fn": lambda: tf.zeros((8, 10), dtype=tf.int32)
},
{
"testcase_name":
"multi_class_classification_with_vocab_and_weights",
# pylint: disable=g-long-lambda
"head_fn":
lambda: tf.estimator.MultiClassHead(n_classes=3,
label_vocabulary=
["ONE", "TWO", "THREE"],
weight_column="weights"),
# pylint: enable=g-long-lambda
"label_fn":
lambda: tf.constant(["ONE"] * 8, dtype=tf.string), # batch_size
})
# pylint: enable=unnecessary-lambda
def test_head(self, head_fn, label_fn=None, loss_fn=None, metric_fn=None):
batch_size = 8
input_shape = [batch_size, 32, 32, 3]
hparams = hp.HParams(
initial_architecture=[
"FIXED_CHANNEL_CONVOLUTION_16", "FIXED_CHANNEL_CONVOLUTION_64",
"PLATE_REDUCTION_FLATTEN"
],
new_block_type="FIXED_CHANNEL_CONVOLUTION_32",
learning_rate=
1000.0, # Approximating constant so we'll never diverge.
optimizer="sgd")
instance = self._create_phoenix_instance(problem_type="cnn",
input_shape=input_shape,
head=head_fn(),
loss_fn=loss_fn,
metric_fn=metric_fn)
run_config = tf.estimator.RunConfig(model_dir=self.get_temp_dir() +
"/1")
def input_fn():
features = {"zeros": tf.zeros(input_shape)}
if getattr(head_fn(), "_weight_column", None):
features["weights"] = tf.ones(batch_size) * .5
labels = label_fn() if label_fn else tf.zeros(batch_size,
dtype=tf.int32)
return features, labels
estimator = instance.get_estimator(run_config, hparams, 10)
estimator.train(input_fn=input_fn, max_steps=10)
eval_result = estimator.evaluate(input_fn=input_fn, steps=10)
self.assertAlmostEqual(0., eval_result["loss"], places=3)
