def test_get_suggestion(self,
get_architecture,
spec,
init_architecture,
completed_trials,
new_block,
should_increase_depth,
expected_fork_architecture,
increase_complexity_probability=1.0):
spec.increase_complexity_probability = increase_complexity_probability
get_architecture.return_value = np.array([1, 2, 3, 4])
algorithm = coordinate_descent.CoordinateDescent(spec, self._metadata)
hparams = hp.HParams(initial_architecture=init_architecture,
new_block_type=new_block)
trials = []
for i in range(completed_trials):
trials.append(
trial_module.Trial({
"id": i,
"model_dir": "/tmp/" + str(i),
"status": "COMPLETED",
"trial_infeasible": False,
"final_measurement": {
"objective_value": 100 - i
}
}))
# Adding one infeasible to make sure we don't fork from it.
trials.append(
trial_module.Trial({
"id": 99,
"model_dir": "/tmp/99",
"status": "COMPLETED",
"trial_infeasible": True,
"final_measurement": {
"objective_value": 0
}
}))
logging.info(trials)
output_architecture, fork_trial = algorithm.get_suggestion(
trials, hparams)
if completed_trials:
self.assertEqual(fork_trial, completed_trials - 1)
if should_increase_depth:
self.assertAllEqual(
output_architecture,
np.append(expected_fork_architecture,
blocks.BlockType[new_block]))
else:
self.assertEqual(output_architecture.shape,
expected_fork_architecture.shape)
self.assertTrue(
search_test_utils.is_mutation_or_equal(
expected_fork_architecture, output_architecture))
def test_get_suggestion_beam_size_gt_one(self, get_architecture):
# Trials should be ranked by trial_id. That is, we should only ever fork
# from the first 2 trials.
beam_size = 2
trial_to_arch = {
0: np.array([1, 2, 3]),
1: np.array([4, 5, 6]),
2: np.array([7, 8, 9]),
3: np.array([10, 11, 12])
}
get_architecture.side_effect = lambda idx: trial_to_arch[int(idx)]
spec = search_test_utils.create_spec(phoenix_spec_pb2.PhoenixSpec.DNN)
spec.beam_size = beam_size
spec.increase_complexity_minimum_trials.append(0)
algorithm = coordinate_descent.CoordinateDescent(spec, self._metadata)
hparams = hp.HParams(
new_block_type="FIXED_CHANNEL_CONVOLUTION_16") # enum: 1
# Create one fake trial for each architecture.
trials = []
for i in range(3):
trials.append(
trial_module.Trial({
"id": i,
"model_dir": str(i),
"status": "COMPLETED",
"trial_infeasible": False,
"final_measurement": {
"objective_value": i
}
}))
# Adding one infeasible to make sure we don't fork from it.
trials.append(
trial_module.Trial({
"id": 4,
"model_dir": "4",
"status": "COMPLETED",
"trial_infeasible": True,
"final_measurement": {
"objective_value": 0
}
}))
logging.info(trials)
# Since forking is random, fork 1000 times then check that we forked from
# only the trials we care about.
forked = set()
for i in range(1000):
_, fork_trial = algorithm.get_suggestion(trials, hparams)
forked.add(int(fork_trial))
self.assertEqual(forked, {0, 1})
def create_test_trials_intermixed(root_dir):
"""Creates fake trials used for testing."""
trials = [{
'model_dir': os.path.join(root_dir, str(2)),
'id': 2,
'status': 'COMPLETED',
'trial_infeasible': False,
'final_measurement': {
'objective_value': 0.94
},
}, {
'model_dir': os.path.join(root_dir, str(3)),
'id': 3,
'status': 'COMPLETED',
'trial_infeasible': False,
'final_measurement': {
'objective_value': 0.7
},
}, {
'model_dir': os.path.join(root_dir, str(5)),
'id': 5,
'status': 'COMPLETED',
'trial_infeasible': False,
'final_measurement': {
'objective_value': 0.3
},
}]
return [trial_module.Trial(t) for t in trials]
def _create_trials():
trials = [
trial.Trial({
"id": 99,
"model_dir": "99",
"status": "COMPLETED",
"trial_infeasible": False,
"final_measurement": {
"objective_value": 0.97
},
}),
trial.Trial({
"id": 97,
"model_dir": "97",
"status": "COMPLETED",
"trial_infeasible": False,
"final_measurement": {
"objective_value": 0.94
},
}),
trial.Trial({
"id": 96,
"model_dir": "96",
"status": "COMPLETED",
"trial_infeasible": True,
"final_measurement": {
"objective_value": 0.7
},
}),
trial.Trial({
"id": 95,
"model_dir": "95",
"status": "COMPLETED",
"trial_infeasible": False,
"final_measurement": {
"objective_value": 0.72
},
}),
]
return trials
def _convert_to_trial_object(self, trial_str):
"""Returns a Trial object like object."""
evaluation_dictionary = json.loads(
trial_str.properties["evaluation"].string_value)
data_as_json = {
"id": trial_str.properties["id"].int_value,
"status": trial_str.properties["state"].string_value,
"final_measurement": {
"objective_value":
evaluation_dictionary[self._optimization_metric]
},
"model_dir": trial_str.properties["model_dir"].string_value,
"trial_infeasible": False
}
return trial.Trial(data_as_json)
def _get_suggestion(architectures,
blocks_to_use,
losses,
grow=False,
remove_outliers=False,
pass_flatten=False):
"""Testing subroutine to handle boilerplate Trial construction, dirs, etc."""
# TODO(b/172564129): Figure out how to use mock decorator for free functions.
with mock.patch("model_search.architecture"
".architecture_utils.get_architecture") as mock_get_arch:
blocks_strs = [blocks.BlockType(b).name for b in blocks_to_use]
spec = search_test_utils.create_spec(
phoenix_spec_pb2.PhoenixSpec.CNN,
blocks_to_use=blocks_strs,
)
spec.search_type = phoenix_spec_pb2.PhoenixSpec.LINEAR_MODEL
spec.increase_complexity_probability = 1.0 if grow else 0.0
spec.linear_model.remove_outliers = remove_outliers
spec.linear_model.trials_before_fit = 1
algorithm = linear_model.LinearModel(spec)
mock_get_arch.side_effect = lambda idx: architectures[int(idx)]
trials = []
for i, loss in enumerate(losses):
if isinstance(loss, (np.floating, np.integer)):
loss = loss.item()
trials.append(
trial_module.Trial({
"id": i,
"model_dir": str(i),
"status": "COMPLETED",
"trial_infeasible": False,
"final_measurement": {
"objective_value": loss
}
}))
hparams = hp.HParams(new_block_type=NEW_BLOCK)
# Second return val fork_trial is a nonsense concept for LinearModel.
output_architecture, _ = algorithm.get_suggestion(trials, hparams)
if not pass_flatten:
output_architecture = np.array(
[b for b in output_architecture if b not in blocks.FLATTEN_TYPES])
return output_architecture
def test_loss_weighted_transfer_learning_hook(
self,
shapes,
dtypes,
values,
losses,
expected,
max_completed_trials=1000000):
# Force graph mode
with tf.compat.v1.Graph().as_default():
root_dir = self.get_temp_dir()
n_prev = self._create_previous_trials(root_dir, shapes, dtypes,
values)
# Warm start the current trial for transfer learning and check output.
with self.test_session(graph=tf.Graph()) as sess:
var = tf.compat.v1.get_variable(
"var",
shape=shapes[0],
dtype=tf.float32,
initializer=tf.compat.v1.zeros_initializer)
completed_trials = []
for i, loss in enumerate(losses):
trial = trial_module.Trial({
"model_dir":
os.path.join(root_dir, str(i)),
"id":
i,
"final_measurement": {
"objective_value": loss
}
})
completed_trials.append(trial)
hook = transfer_learning.LossWeightedAverageTransferLearningHook(
vars_to_warm_start=[var],
current_trial_id=n_prev,
completed_trials=completed_trials,
discount_factor=1.,
max_completed_trials=max_completed_trials,
model_dir=os.path.join(root_dir, str(n_prev)))
hook.begin()
sess.run(tf.compat.v1.global_variables_initializer())
hook.after_create_session(sess, None)
actual = sess.run(var)
self.assertAllClose(actual, expected)
def _create_trials(root_dir):
trials = [{
'model_dir': os.path.join(root_dir, str(1)),
'id': 1,
'status': 'COMPLETED',
'trial_infeasible': False,
'final_measurement': {
'objective_value': 0.97
},
}, {
'model_dir': os.path.join(root_dir, str(2)),
'id': 2,
'status': 'COMPLETED',
'trial_infeasible': False,
'final_measurement': {
'objective_value': 0.94
},
}, {
'model_dir': os.path.join(root_dir, str(3)),
'id': 3,
'status': 'COMPLETED',
'trial_infeasible': False,
'final_measurement': {
'objective_value': 0.7
},
}, {
'model_dir': os.path.join(root_dir, str(4)),
'id': 4,
'status': 'COMPLETED',
'trial_infeasible': False,
'final_measurement': {
'objective_value': 0.72
},
}, {
'model_dir': os.path.join(root_dir, str(5)),
'id': 5,
'status': 'COMPLETED',
'trial_infeasible': False,
'final_measurement': {
'objective_value': 0.3
},
}]
return [trial_module.Trial(t) for t in trials]
def test_get_suggestion(self, get_architecture, randint, completed_trials,
initial_architecture, expected_architecture,
replicate_cell):
get_architecture.return_value = initial_architecture
randint.return_value = 2
hparams = hp.HParams(
new_block_type="FIXED_CHANNEL_CONVOLUTION_16") # id=1.
# Use problem_type=DNN so we don't have to worry about the final
# architecture being correct.
spec = phoenix_spec_pb2.PhoenixSpec(
problem_type=phoenix_spec_pb2.PhoenixSpec.DNN,
maximum_depth=11,
num_blocks_in_cell=4,
reduction_block_type="AVERAGE_POOL_2X2",
replicate_cell=replicate_cell,
beam_size=3)
algorithm = constrained_descent.ConstrainedDescent(
spec, self._metadata)
trials = []
for i in range(completed_trials):
trials.append(
trial_module.Trial({
"id": i,
"model_dir": "/tmp/" + str(i),
"status": "COMPLETED",
"trial_infeasible": False,
"final_measurement": {
"objective_value": 100 - i
}
}))
actual_architecture, fork_trial = algorithm.get_suggestion(
trials, hparams)
# We use a beam of size 3 and always choose the last trial to be the best.
self.assertEqual(fork_trial, completed_trials - 3)
self.assertTrue(
search_test_utils.is_mutation_or_equal(actual_architecture,
expected_architecture))
class ArchitectureUtilsTest(parameterized.TestCase, tf.test.TestCase):
def test_get_blocks_search_space(self):
hps = architecture_utils.get_blocks_search_space()
self.assertIn("TUNABLE_SVDF_output_size", hps)
self.assertIn("TUNABLE_SVDF_rank", hps)
self.assertIn("TUNABLE_SVDF_projection_size", hps)
self.assertIn("TUNABLE_SVDF_memory_size", hps)
hps = architecture_utils.get_blocks_search_space(["TUNABLE_SVDF"])
self.assertIn("TUNABLE_SVDF_output_size", hps)
self.assertIn("TUNABLE_SVDF_rank", hps)
self.assertIn("TUNABLE_SVDF_projection_size", hps)
self.assertIn("TUNABLE_SVDF_memory_size", hps)
def test_get_block_hparams(self):
hp_ = architecture_utils.get_block_hparams(
hp.HParams(TUNABLE_SVDF_target=10, non_target=15), "TUNABLE_SVDF")
self.assertLen(hp_.values(), 1)
self.assertEqual(hp_.target, 10)
def test_store_and_get_hparams(self):
hp_ = hp.HParams(hello="world", context="toberemoved")
dirname = self.get_temp_dir()
architecture_utils.store_hparams_to_dir(hp_, dirname, "tower")
hp_replica = architecture_utils.get_hparams_from_dir(dirname, "tower")
self.assertLen(hp_replica.values(), 1)
self.assertEqual(hp_replica.hello, "world")
@parameterized.named_parameters(
{
"testcase_name": "empty",
"input_dir": "",
"expected_output": None,
"spec": phoenix_spec_pb2.PhoenixSpec(),
}, {
"testcase_name": "normal",
"input_dir": "some/random/path/512",
"expected_output": 512,
"spec": phoenix_spec_pb2.PhoenixSpec(),
}, {
"testcase_name": "normal_tfx",
"input_dir": "/some/random/Trial-000034/active/Run-0000000",
"expected_output": 34,
"spec": phoenix_spec_pb2.PhoenixSpec(),
}, {
"testcase_name": "hybrid",
"input_dir": "/some/random/Trial-000034/active/Run-0000000/7",
"expected_output": 7,
"spec": phoenix_spec_pb2.PhoenixSpec(),
}, {
"testcase_name": "replay",
"input_dir": "/some/random/Trial-000034/active/Run-0000000/7",
"expected_output": 7,
"spec": _replay_spec(),
})
def test_get_trial_id(self, input_dir, expected_output, spec):
output = architecture_utils.DirectoryHandler.get_trial_id(
input_dir, spec)
self.assertEqual(output, expected_output)
@parameterized.named_parameters(
{
"testcase_name": "empty",
"input_trial": trial.Trial({"model_dir": ""}),
"expected_output": ""
}, {
"testcase_name": "normal",
"input_trial": trial.Trial({"model_dir": "random/path"}),
"expected_output": "random/path"
})
def test_get_trial_dir(self, input_trial, expected_output):
output = architecture_utils.DirectoryHandler.trial_dir(input_trial)
self.assertEqual(output, expected_output)
@parameterized.named_parameters(
{
"testcase_name": "empty",
"initial_architecture": [],
"fixed_architecture": [blocks.BlockType.PLATE_REDUCTION_FLATTEN]
}, {
"testcase_name": "flatten",
"initial_architecture": [blocks.BlockType.PLATE_REDUCTION_FLATTEN],
"fixed_architecture": [blocks.BlockType.PLATE_REDUCTION_FLATTEN]
}, {
"testcase_name": "downsample_flatten",
"initial_architecture": [blocks.BlockType.DOWNSAMPLE_FLATTEN],
"fixed_architecture": [blocks.BlockType.DOWNSAMPLE_FLATTEN]
}, {
"testcase_name":
"basic_case",
"initial_architecture": [
blocks.BlockType.PLATE_REDUCTION_FLATTEN,
blocks.BlockType.FULLY_CONNECTED,
blocks.BlockType.CONVOLUTION_3X3
],
"fixed_architecture": [
blocks.BlockType.CONVOLUTION_3X3,
blocks.BlockType.PLATE_REDUCTION_FLATTEN,
blocks.BlockType.FULLY_CONNECTED
]
}, {
"testcase_name":
"basic_downsample_case",
"initial_architecture": [
blocks.BlockType.DOWNSAMPLE_FLATTEN,
blocks.BlockType.FULLY_CONNECTED,
blocks.BlockType.CONVOLUTION_3X3
],
"fixed_architecture": [
blocks.BlockType.CONVOLUTION_3X3,
blocks.BlockType.DOWNSAMPLE_FLATTEN,
blocks.BlockType.FULLY_CONNECTED
]
}, {
"testcase_name":
"advanced_case",
"initial_architecture": [
blocks.BlockType.PLATE_REDUCTION_FLATTEN,
blocks.BlockType.FULLY_CONNECTED,
blocks.BlockType.FULLY_CONNECTED_PYRAMID,
blocks.BlockType.CONVOLUTION_3X3,
blocks.BlockType.DOWNSAMPLE_CONVOLUTION_3X3
],
"fixed_architecture": [
blocks.BlockType.CONVOLUTION_3X3,
blocks.BlockType.DOWNSAMPLE_CONVOLUTION_3X3,
blocks.BlockType.PLATE_REDUCTION_FLATTEN,
blocks.BlockType.FULLY_CONNECTED,
blocks.BlockType.FULLY_CONNECTED_PYRAMID
]
})
def test_fix_architecture_order(self, initial_architecture,
fixed_architecture):
out_architecture = architecture_utils.fix_architecture_order(
initial_architecture, phoenix_spec_pb2.PhoenixSpec.CNN)
self.assertAllEqual(fixed_architecture, out_architecture)
@parameterized.named_parameters(
{
"testcase_name": "empty_rnn",
"problem_type": phoenix_spec_pb2.PhoenixSpec.RNN_ALL_ACTIVATIONS,
"new_block": blocks.BlockType.LSTM_128,
"initial_architecture": np.array([]),
"expected_architecture": [blocks.BlockType.LSTM_128]
}, {
"testcase_name":
"empty_dnn",
"problem_type":
phoenix_spec_pb2.PhoenixSpec.DNN,
"new_block":
blocks.BlockType.FIXED_OUTPUT_FULLY_CONNECTED_128,
"initial_architecture":
np.array([]),
"expected_architecture":
[blocks.BlockType.FIXED_OUTPUT_FULLY_CONNECTED_128]
})
def test_increase_structure_depth(self, problem_type, initial_architecture,
new_block, expected_architecture):
out_architecture = architecture_utils.increase_structure_depth(
initial_architecture, new_block, problem_type)
self.assertAllEqual(expected_architecture, out_architecture)
@parameterized.named_parameters(
{
"testcase_name": "test1",
"architecture": np.array([1, 2, 3, 4]),
}, {
"testcase_name": "test2",
"architecture": np.array([2, 3, 4, 5]),
})
def test_set_get_architecture(self, architecture):
# Force graph mode
with tf.compat.v1.Graph().as_default():
directory = self.get_temp_dir()
with self.test_session(graph=tf.Graph()) as sess:
architecture_utils.set_architecture(architecture)
saver = tf.compat.v1.train.Saver()
sess.run(tf.compat.v1.global_variables_initializer())
sess.run(tf.compat.v1.local_variables_initializer())
saver.save(sess, directory + "/ckpt")
output_architecture = architecture_utils.get_architecture(
directory)
self.assertAllEqual(output_architecture, architecture)
@parameterized.named_parameters(
{
"testcase_name": "no_architecture",
"architecture": None,
}, {
"testcase_name": "has_architecture",
"architecture": np.array([2, 3, 4, 5]),
})
def test_get_architecture_size(self, architecture):
# Force graph mode
with tf.compat.v1.Graph().as_default():
with self.test_session(graph=tf.Graph()):
tower_name = "tower"
if architecture is not None:
architecture_utils.set_architecture(architecture,
tower_name=tower_name)
size = architecture_utils.get_architecture_size(
tower_name=tower_name)
if architecture is not None:
self.assertEqual(size, architecture.size)
else:
self.assertIsNone(size)
@parameterized.named_parameters(
{
"testcase_name":
"regular_network",
"dropout":
-1,
"expected_logits": [[
-0.1481, 0.3328, 0.3028, 0.5652, 0.6860, 0.06171, 0.09998,
-0.2622, 0.2186, -0.1322
]]
}, {
"testcase_name":
"dropout_network",
"dropout":
0.1,
"expected_logits": [[
0.7001, -0.06655, -0.1711, 0.1274, -0.8175, 0.2932, 0.06242,
0.2182, -0.06626, 0.7882
]],
})
def test_construct_network(self, dropout, expected_logits):
# Force graph mode
with tf.compat.v1.Graph().as_default():
tf.random.set_seed(1234)
# convolutions and then flatten plate.
architecture = np.array([1, 3, 34])
input_tensor = tf.compat.v1.placeholder(dtype=tf.float32,
shape=[None, 32, 32, 3],
name="input")
phoenix_spec = phoenix_spec_pb2.PhoenixSpec(
problem_type=phoenix_spec_pb2.PhoenixSpec.CNN)
tower_spec = architecture_utils.construct_tower(
phoenix_spec=phoenix_spec,
input_tensor=input_tensor,
tower_name="test_tower",
architecture=architecture,
is_training=True,
lengths=None,
logits_dimension=10,
hparams=hp.HParams(),
model_directory=self.get_temp_dir(),
is_frozen=False,
dropout_rate=dropout)
np.random.seed(42)
test_input = np.random.random([1, 32, 32, 3])
with tf.compat.v1.Session() as sess:
sess.run([
tf.compat.v1.global_variables_initializer(),
tf.compat.v1.local_variables_initializer()
])
logits_val = sess.run(tower_spec.logits_spec.logits,
feed_dict={input_tensor: test_input})
self.assertAllClose(expected_logits, logits_val, rtol=1e-3)
@parameterized.named_parameters(
{
"testcase_name":
"disabled",
"transfer_learning_type":
transfer_learning_spec_pb2.TransferLearningSpec.
NO_TRANSFER_LEARNING
}, {
"testcase_name":
"enabled",
"transfer_learning_type":
transfer_learning_spec_pb2.TransferLearningSpec.
UNIFORM_AVERAGE_TRANSFER_LEARNING
})
def test_construct_tower_with_transfer_learning(
self,
transfer_learning_type=transfer_learning_spec_pb2.TransferLearningSpec.
NO_TRANSFER_LEARNING):
# convolutions and then flatten plate.
architecture = np.array([1, 3, 34])
str_signature = "_1334"
input_tensor = tf.zeros([100, 32, 32, 3])
tower_name = "test_tower"
transfer_learning_spec = transfer_learning_spec_pb2.TransferLearningSpec(
transfer_learning_type=transfer_learning_type)
phoenix_spec = phoenix_spec_pb2.PhoenixSpec(
problem_type=phoenix_spec_pb2.PhoenixSpec.CNN,
transfer_learning_spec=transfer_learning_spec)
_ = architecture_utils.construct_tower(
phoenix_spec=phoenix_spec,
input_tensor=input_tensor,
tower_name=tower_name,
architecture=architecture,
is_training=True,
lengths=None,
logits_dimension=10,
hparams=hp.HParams(),
model_directory=self.get_temp_dir(),
is_frozen=False,
dropout_rate=None)
tensors = architecture_utils.get_tower_variables(tower_name)
for tensor in tensors:
if (transfer_learning_type == transfer_learning_spec_pb2.
TransferLearningSpec.NO_TRANSFER_LEARNING):
self.assertEndsWith(tensor.op.name, str_signature)
else:
self.assertNotEndsWith(tensor.op.name, str_signature)
@parameterized.named_parameters(
{
"testcase_name": "shared_input",
"shared_input": True
}, {
"testcase_name": "not_shared_input",
"shared_input": False
})
def test_import_tower(self, shared_input):
np.random.seed(42)
test_input = np.random.random([1, 32, 32, 3])
# Force graph mode
with tf.compat.v1.Graph().as_default():
directory = self.get_temp_dir()
architecture = np.array([1, 3, 34])
phoenix_spec = phoenix_spec_pb2.PhoenixSpec(
problem_type=phoenix_spec_pb2.PhoenixSpec.CNN)
phoenix_spec.is_input_shared = shared_input
features = {}
shared_input_tensor = None
with self.test_session(graph=tf.Graph()) as sess:
input_tensor_1 = tf.compat.v1.placeholder(
dtype=tf.float32, shape=[None, 32, 32, 3], name="input_1")
tf.random.set_seed(1234)
tower_spec_1 = architecture_utils.construct_tower(
phoenix_spec=phoenix_spec,
input_tensor=input_tensor_1,
tower_name="test_tower",
architecture=architecture,
is_training=True,
lengths=None,
logits_dimension=10,
hparams=hp.HParams(),
model_directory=self.get_temp_dir(),
is_frozen=False,
dropout_rate=None)
saver = tf.compat.v1.train.Saver()
sess.run(tf.compat.v1.global_variables_initializer())
sess.run(tf.compat.v1.local_variables_initializer())
logits_val_1 = sess.run(tower_spec_1.logits_spec.logits,
feed_dict={input_tensor_1: test_input})
saver.save(sess, directory + "/ckpt")
with self.test_session(graph=tf.Graph()) as sess:
input_tensor_2 = tf.compat.v1.placeholder(
dtype=tf.float32, shape=[None, 32, 32, 3], name="input_2")
if shared_input:
shared_input_tensor = input_tensor_2
def _input_layer_fn(features,
is_training,
scope_name="Phoenix/Input",
lengths_feature_name=None):
del features, is_training, scope_name, lengths_feature_name
return None, None
else:
features = {"x": input_tensor_2}
def _input_layer_fn(features,
is_training,
scope_name="Phoenix/Input",
lengths_feature_name=None):
del is_training, lengths_feature_name
with tf.compat.v1.variable_scope(scope_name):
return tf.cast(features["x"],
dtype=tf.float32), None
tf.random.set_seed(1234)
tower_spec_2 = architecture_utils.import_tower(
features=features,
input_layer_fn=_input_layer_fn,
phoenix_spec=phoenix_spec,
shared_input_tensor=shared_input_tensor,
original_tower_name="test_tower",
new_tower_name="imported_tower",
model_directory=directory,
new_model_directory=self.get_temp_dir(),
is_training=True,
logits_dimension=10,
shared_lengths=None,
force_snapshot=False,
force_freeze=False)
sess.run(tf.compat.v1.global_variables_initializer())
sess.run(tf.compat.v1.local_variables_initializer())
logits_val_2 = sess.run(tower_spec_2.logits_spec.logits,
feed_dict={input_tensor_2: test_input})
self.assertAllClose(logits_val_1, logits_val_2, rtol=1e-3)
@parameterized.named_parameters(
{
"testcase_name": "cnn",
"input_tensor_shape": [20, 20],
"spec": _create_spec(phoenix_spec_pb2.PhoenixSpec.CNN),
"output_shape": [20, 7],
},
{
"testcase_name": "dnn",
"input_tensor_shape": [20, 20],
"spec": _create_spec(phoenix_spec_pb2.PhoenixSpec.DNN),
"output_shape": [20, 7],
},
{
"testcase_name": "rnn_all_activations",
"input_tensor_shape": [20, 20, 20],
"spec": _create_spec(
phoenix_spec_pb2.PhoenixSpec.RNN_ALL_ACTIVATIONS),
"output_shape": [20, 20, 7],
},
{
"testcase_name": "rnn_last_activations",
"input_tensor_shape": [20, 20, 20],
"spec": _create_spec(
phoenix_spec_pb2.PhoenixSpec.RNN_LAST_ACTIVATIONS),
"output_shape": [20, 7],
"lengths": list(range(20))
},
{
"testcase_name": "rnn_last_activations_no_length",
"input_tensor_shape": [20, 20, 20],
"spec": _create_spec(
phoenix_spec_pb2.PhoenixSpec.RNN_LAST_ACTIVATIONS),
"output_shape": [20, 20, 7],
},
{
"testcase_name": "nasnet_aux_head",
"input_tensor_shape": [20, 20],
"spec": _create_spec(phoenix_spec_pb2.PhoenixSpec.CNN),
"output_shape": [20, 7],
"extra_block": 6, # Downsample convolution.
"extra_block_shape": [20, 20, 20, 20],
"use_auxiliary_head": True
},
{
"testcase_name": "deep_skip_head",
"input_tensor_shape": [20, 20],
"spec": _create_spec(phoenix_spec_pb2.PhoenixSpec.CNN),
"output_shape": [20, 7],
"extra_block": 16, # Flatten.
"use_auxiliary_head": True
})
def test_create_tower_spec(
self,
input_tensor_shape,
spec,
output_shape,
lengths=None,
extra_block=14, # Fully connected.
extra_block_shape=None,
use_auxiliary_head=False):
# Force graph mode
with tf.compat.v1.Graph().as_default():
spec.use_auxiliary_head = use_auxiliary_head
spec.auxiliary_head_loss_weight = .4
input_tensors = [
tf.zeros(extra_block_shape
or (1, )), # Raw features -- discarded.
tf.zeros(extra_block_shape
or input_tensor_shape), # Extra block.
tf.zeros(extra_block_shape
or input_tensor_shape), # Extra block.
tf.zeros(input_tensor_shape) # Fully connected block.
]
fake_architecture = [extra_block, extra_block, 14]
logits_spec, _, _ = architecture_utils.create_tower_spec(
phoenix_spec=spec,
inputs=input_tensors,
architecture=fake_architecture,
dimension=7,
is_frozen=False,
lengths=lengths,
allow_auxiliary_head=use_auxiliary_head)
self.assertAllEqual(output_shape, logits_spec.logits.shape)
if use_auxiliary_head:
self.assertIsNotNone(logits_spec.aux_logits)
self.assertNear(logits_spec.aux_logits_weight,
spec.auxiliary_head_loss_weight, 1e-6)
self.assertNear(logits_spec.logits_weight, 1.0, 1e-6)
@parameterized.named_parameters(
{
"testcase_name":
"same_graph_snapshotting",
"new_architecture":
np.array([1, 3, 34]),
"expected_output": [
"Phoenix/test_tower/1_FIXED_CHANNEL_CONVOLUTION_16_1/"
"Conv/biases",
"Phoenix/test_tower/1_FIXED_CHANNEL_CONVOLUTION_16_1/"
"Conv/weights",
"Phoenix/test_tower/2_FIXED_CHANNEL_CONVOLUTION_64_13/"
"Conv/biases",
"Phoenix/test_tower/2_FIXED_CHANNEL_CONVOLUTION_64_13/"
"Conv/weights",
"Phoenix/test_tower/last_dense_1334/dense/bias",
"Phoenix/test_tower/last_dense_1334/dense/kernel",
"Phoenix/test_tower/2_FIXED_CHANNEL_CONVOLUTION_64_13/BatchNorm/"
"beta",
"Phoenix/test_tower/1_FIXED_CHANNEL_CONVOLUTION_16_1/BatchNorm/"
"moving_variance",
"Phoenix/test_tower/2_FIXED_CHANNEL_CONVOLUTION_64_13/BatchNorm/"
"moving_variance",
"Phoenix/test_tower/1_FIXED_CHANNEL_CONVOLUTION_16_1/BatchNorm/"
"moving_mean",
"Phoenix/test_tower/1_FIXED_CHANNEL_CONVOLUTION_16_1/BatchNorm/"
"beta",
"Phoenix/test_tower/2_FIXED_CHANNEL_CONVOLUTION_64_13/BatchNorm/"
"moving_mean",
]
}, {
"testcase_name":
"same_graph_snapshotting_new_towername",
"new_architecture":
np.array([1, 3, 34]),
"expected_output": [
"Phoenix/test_tower/1_FIXED_CHANNEL_CONVOLUTION_16_1/"
"Conv/biases",
"Phoenix/test_tower/1_FIXED_CHANNEL_CONVOLUTION_16_1/"
"Conv/weights",
"Phoenix/test_tower/2_FIXED_CHANNEL_CONVOLUTION_64_13/"
"Conv/biases",
"Phoenix/test_tower/2_FIXED_CHANNEL_CONVOLUTION_64_13/"
"Conv/weights",
"Phoenix/test_tower/last_dense_1334/dense/bias",
"Phoenix/test_tower/last_dense_1334/dense/kernel",
"Phoenix/test_tower/2_FIXED_CHANNEL_CONVOLUTION_64_13/BatchNorm/"
"beta",
"Phoenix/test_tower/1_FIXED_CHANNEL_CONVOLUTION_16_1/BatchNorm/"
"moving_variance",
"Phoenix/test_tower/2_FIXED_CHANNEL_CONVOLUTION_64_13/BatchNorm/"
"moving_variance",
"Phoenix/test_tower/1_FIXED_CHANNEL_CONVOLUTION_16_1/BatchNorm/"
"moving_mean",
"Phoenix/test_tower/1_FIXED_CHANNEL_CONVOLUTION_16_1/BatchNorm/"
"beta",
"Phoenix/test_tower/2_FIXED_CHANNEL_CONVOLUTION_64_13/BatchNorm/"
"moving_mean",
],
"new_tower_name":
"test_tower_2"
}, {
"testcase_name":
"changing_second",
"new_architecture":
np.array([1, 2, 34]),
"expected_output": [
"Phoenix/test_tower/1_FIXED_CHANNEL_CONVOLUTION_16_1/"
"Conv/weights",
"Phoenix/test_tower/1_FIXED_CHANNEL_CONVOLUTION_16_1/Conv/biases",
"Phoenix/test_tower/1_FIXED_CHANNEL_CONVOLUTION_16_1/BatchNorm/"
"moving_mean",
"Phoenix/test_tower/1_FIXED_CHANNEL_CONVOLUTION_16_1/BatchNorm/"
"moving_variance",
"Phoenix/test_tower/1_FIXED_CHANNEL_CONVOLUTION_16_1/BatchNorm/"
"beta"
]
})
def test_init_variables(self,
new_architecture,
expected_output,
new_tower_name="test_tower"):
# Force graph mode
with tf.compat.v1.Graph().as_default():
directory = self.get_temp_dir()
architecture = np.array([1, 3, 34])
phoenix_spec = phoenix_spec_pb2.PhoenixSpec(
problem_type=phoenix_spec_pb2.PhoenixSpec.CNN)
with self.test_session(graph=tf.Graph()) as sess:
input_tensor = tf.zeros([100, 32, 32, 3])
_ = architecture_utils.construct_tower(
phoenix_spec=phoenix_spec,
input_tensor=input_tensor,
tower_name="test_tower",
architecture=architecture,
is_training=True,
lengths=None,
logits_dimension=10,
model_directory=self.get_temp_dir(),
hparams=hp.HParams(),
is_frozen=False,
dropout_rate=None)
saver = tf.compat.v1.train.Saver()
sess.run(tf.compat.v1.global_variables_initializer())
sess.run(tf.compat.v1.local_variables_initializer())
saver.save(sess, directory + "/ckpt")
with self.test_session(graph=tf.Graph()) as sess:
input_tensor = tf.zeros([100, 32, 32, 3])
_ = architecture_utils.construct_tower(
phoenix_spec=phoenix_spec,
input_tensor=input_tensor,
tower_name=new_tower_name,
architecture=new_architecture,
is_training=True,
lengths=None,
logits_dimension=10,
hparams=hp.HParams(),
model_directory=self.get_temp_dir(),
is_frozen=False,
dropout_rate=None)
snapshotting_variables = architecture_utils.init_variables(
tf.train.latest_checkpoint(directory),
"Phoenix/test_tower", "Phoenix/{}".format(new_tower_name))
self.assertCountEqual(snapshotting_variables, expected_output)
def _create_trials():
trials = [{
"id": 99,
"model_dir": "99",
"status": "COMPLETED",
"trial_infeasible": False,
"final_measurement": {
"objective_value": 0.97
},
}, {
"id": 97,
"model_dir": "97",
"status": "COMPLETED",
"trial_infeasible": False,
"final_measurement": {
"objective_value": 0.94
},
}, {
"id": 96,
"model_dir": "96",
"status": "COMPLETED",
"trial_infeasible": False,
"final_measurement": {
"objective_value": 0.7
},
}, {
"id": 95,
"model_dir": "95",
"status": "COMPLETED",
"trial_infeasible": False,
"final_measurement": {
"objective_value": 0.72
},
}, {
"id": 94,
"model_dir": "94",
"status": "COMPLETED",
"trial_infeasible": False,
"final_measurement": {
"objective_value": 0.3
},
}, {
"id": 93,
"model_dir": "93",
"status": "COMPLETED",
"trial_infeasible": False,
"final_measurement": {
"objective_value": 0.79
},
}, {
"id": 92,
"model_dir": "92",
"status": "COMPLETED",
"trial_infeasible": False,
"final_measurement": {
"objective_value": 0.39
},
}, {
"id": 91,
"model_dir": "91",
"status": "COMPLETED",
"trial_infeasible": False,
"final_measurement": {
"objective_value": 0.9
},
}, {
"id": 90,
"model_dir": "90",
"status": "COMPLETED",
"trial_infeasible": False,
"final_measurement": {
"objective_value": 0.19
},
}]
return [trial_module.Trial(t) for t in trials]
output.replay.towers.add()
if ensemble_type == "adaptive":
output.ensemble_spec.ensemble_search_type = (
ensembling_spec_pb2.EnsemblingSpec.ADAPTIVE_ENSEMBLE_SEARCH)
if ensemble_type == "nonadaptive":
output.ensemble_spec.ensemble_search_type = (
ensembling_spec_pb2.EnsemblingSpec.NONADAPTIVE_ENSEMBLE_SEARCH)
if ensemble_type == "intermix":
output.ensemble_spec.ensemble_search_type = (
ensembling_spec_pb2.EnsemblingSpec.
INTERMIXED_NONADAPTIVE_ENSEMBLE_SEARCH)
return output
_TRIALS = [trial_module.Trial({"id": id + 1}) for id in range(105)]
def _create_trials(num_trials):
return _TRIALS[:num_trials]
class ControllerTest(parameterized.TestCase, absltest.TestCase):
@parameterized.named_parameters(
{
"testcase_name": "search_replay",
"spec": _create_spec(1),
"my_id": 1,
"trials": [],
"expected_output": {
"search_generator": []
