class IdentityTest(parameterized.TestCase, tf.test.TestCase):
@parameterized.named_parameters(
{
"testcase_name":
"regular_cnn",
"spec":
test_utils.create_spec(phoenix_spec_pb2.PhoenixSpec.CNN),
"initial_architecture": [
"FIXED_OUTPUT_FULLY_CONNECTED_128",
"FIXED_CHANNEL_CONVOLUTION_32", "FIXED_CHANNEL_CONVOLUTION_64",
"CONVOLUTION_3X3"
],
"expected_architecture":
np.array([2, 3, 4, 34, 22])
}, {
"testcase_name":
"regular_dnn",
"spec":
test_utils.create_spec(phoenix_spec_pb2.PhoenixSpec.DNN),
"initial_architecture": [
"FIXED_OUTPUT_FULLY_CONNECTED_128",
"FIXED_OUTPUT_FULLY_CONNECTED_256"
],
"expected_architecture":
np.array([22, 23])
})
def test_get_suggestion(self, spec, initial_architecture,
expected_architecture):
algorithm = identity.Identity(spec)
output_architecture, _ = algorithm.get_suggestion(
[], hp.HParams(initial_architecture=initial_architecture))
self.assertAllEqual(expected_architecture, output_architecture)
def test_translate_architecture_to_assignment(self):
algorithm = categorical_harmonica.Harmonica(
test_utils.create_spec(phoenix_spec_pb2.PhoenixSpec.DNN,
blocks_to_use=[
"FIXED_CHANNEL_CONVOLUTION_16",
"FIXED_CHANNEL_CONVOLUTION_32",
"FIXED_CHANNEL_CONVOLUTION_64",
"CONVOLUTION_3X3"
],
min_depth=4))
assignment = algorithm.translate_architecture_to_feature_assignment(
np.array([2, 3, 4]))
expected_output = [[
1.00000000e+00, 6.12323400e-17, -1.00000000e+00, -1.83697020e-16,
1.00000000e+00, -1.00000000e+00, 1.00000000e+00, -1.00000000e+00,
1.00000000e+00, -1.83697020e-16, -1.00000000e+00, 5.51091060e-16,
0.00000000e+00, -1.00000000e+00, 3.67394040e-16, 1.00000000e+00
],
[
0.00000000e+00, 1.00000000e+00, 1.22464680e-16,
-1.00000000e+00, 0.00000000e+00, 1.22464680e-16,
-2.44929360e-16, 3.67394040e-16, 0.00000000e+00,
-1.00000000e+00, 3.67394040e-16, 1.00000000e+00,
1.00000000e+00, -1.83697020e-16,
-1.00000000e+00, 5.51091060e-16
]]
for i in range(len(assignment)):
self.assertAllAlmostEqual(expected_output[i], assignment[i])
def test_get_suggestion(self, get_architecture, problem_type):
# Return value (architectures) for the various trials.
get_architecture.side_effect = [
np.array([1, 2, 3, 4]),
np.array([2, 3, 4, 1]),
np.array([3, 4, 1, 2]),
np.array([4, 1, 2, 3]),
np.array([1, 1, 1, 1]),
np.array([2, 2, 2, 2]),
np.array([3, 3, 3, 3]),
np.array([2, 3, 2, 3]),
np.array([3, 4, 3, 4])
]
algorithm = categorical_harmonica.Harmonica(test_utils.create_spec(
problem_type,
blocks_to_use=[
"FIXED_CHANNEL_CONVOLUTION_16", "FIXED_CHANNEL_CONVOLUTION_32",
"FIXED_CHANNEL_CONVOLUTION_64", "CONVOLUTION_3X3"
],
min_depth=4),
num_random_samples=10,
seed=73)
output_architecture, _ = algorithm.get_suggestion(
_create_trials(), hp.HParams())
expected_output = [3, 3, 3, 3]
if problem_type == phoenix_spec_pb2.PhoenixSpec.CNN:
expected_output += [34]
self.assertAllEqual(expected_output, output_architecture)
def test_polynomial_expansion(self):
algorithm = categorical_harmonica.Harmonica(
test_utils.create_spec(phoenix_spec_pb2.PhoenixSpec.DNN))
feature_extender = PolynomialFeatures(2, interaction_only=True)
self.assertAllEqual([[1, 0, 1, 0], [1, 0.5, 1, 0.5]],
algorithm._get_polynomial_expansion(
feature_extender, np.array([[0, 1], [0.5,
1]])))
def test_get_suggestion_beam_size_gt_one(self, get_architecture):
# Trials should be ranked by trial_id. That is, we should only ever fork
# from the first 2 trials.
beam_size = 2
trial_to_arch = {
0: np.array([1, 2, 3]),
1: np.array([4, 5, 6]),
2: np.array([7, 8, 9]),
3: np.array([10, 11, 12])
}
get_architecture.side_effect = lambda idx: trial_to_arch[int(idx)]
spec = search_test_utils.create_spec(phoenix_spec_pb2.PhoenixSpec.DNN)
spec.beam_size = beam_size
spec.increase_complexity_minimum_trials.append(0)
algorithm = coordinate_descent.CoordinateDescent(spec, self._metadata)
hparams = hp.HParams(
new_block_type="FIXED_CHANNEL_CONVOLUTION_16") # enum: 1
# Create one fake trial for each architecture.
trials = []
for i in range(3):
trials.append(
trial_module.Trial({
"id": i,
"model_dir": str(i),
"status": "COMPLETED",
"trial_infeasible": False,
"final_measurement": {
"objective_value": i
}
}))
# Adding one infeasible to make sure we don't fork from it.
trials.append(
trial_module.Trial({
"id": 4,
"model_dir": "4",
"status": "COMPLETED",
"trial_infeasible": True,
"final_measurement": {
"objective_value": 0
}
}))
logging.info(trials)
# Since forking is random, fork 1000 times then check that we forked from
# only the trials we care about.
forked = set()
for i in range(1000):
_, fork_trial = algorithm.get_suggestion(trials, hparams)
forked.add(int(fork_trial))
self.assertEqual(forked, {0, 1})
def _get_suggestion(architectures,
blocks_to_use,
losses,
grow=False,
remove_outliers=False,
pass_flatten=False):
"""Testing subroutine to handle boilerplate Trial construction, dirs, etc."""
# TODO(b/172564129): Figure out how to use mock decorator for free functions.
with mock.patch("model_search.architecture"
".architecture_utils.get_architecture") as mock_get_arch:
blocks_strs = [blocks.BlockType(b).name for b in blocks_to_use]
spec = search_test_utils.create_spec(
phoenix_spec_pb2.PhoenixSpec.CNN,
blocks_to_use=blocks_strs,
)
spec.search_type = phoenix_spec_pb2.PhoenixSpec.LINEAR_MODEL
spec.increase_complexity_probability = 1.0 if grow else 0.0
spec.linear_model.remove_outliers = remove_outliers
spec.linear_model.trials_before_fit = 1
algorithm = linear_model.LinearModel(spec)
mock_get_arch.side_effect = lambda idx: architectures[int(idx)]
trials = []
for i, loss in enumerate(losses):
if isinstance(loss, (np.floating, np.integer)):
loss = loss.item()
trials.append(
trial_module.Trial({
"id": i,
"model_dir": str(i),
"status": "COMPLETED",
"trial_infeasible": False,
"final_measurement": {
"objective_value": loss
}
}))
hparams = hp.HParams(new_block_type=NEW_BLOCK)
# Second return val fork_trial is a nonsense concept for LinearModel.
output_architecture, _ = algorithm.get_suggestion(trials, hparams)
if not pass_flatten:
output_architecture = np.array(
[b for b in output_architecture if b not in blocks.FLATTEN_TYPES])
return output_architecture
def test_get_good_architecture_with_relevant_variables(self):
algorithm = categorical_harmonica.Harmonica(test_utils.create_spec(
phoenix_spec_pb2.PhoenixSpec.CNN,
blocks_to_use=[
"FIXED_CHANNEL_CONVOLUTION_16", "FIXED_CHANNEL_CONVOLUTION_32",
"FIXED_CHANNEL_CONVOLUTION_64", "CONVOLUTION_3X3"
],
min_depth=2),
degree=2,
seed=73)
expected_output = [3, 1, 34]
feature_extender = PolynomialFeatures(2, interaction_only=True)
self.assertAllEqual(
expected_output,
algorithm._get_good_architecture(
feature_extender, 20, np.array([0, 0.5, 0.6, -0.2] + [0] * 33),
set([3, 3, 3, 3])))
def test_get_good_architecture(self, problem_type):
algorithm = categorical_harmonica.Harmonica(test_utils.create_spec(
problem_type,
blocks_to_use=[
"FIXED_CHANNEL_CONVOLUTION_16", "FIXED_CHANNEL_CONVOLUTION_32",
"FIXED_CHANNEL_CONVOLUTION_64", "CONVOLUTION_3X3"
],
min_depth=2),
degree=2,
seed=73)
expected_output = [3, 3]
if problem_type == phoenix_spec_pb2.PhoenixSpec.CNN:
expected_output += [34]
feature_extender = PolynomialFeatures(2, interaction_only=True)
self.assertAllEqual(
expected_output,
algorithm._get_good_architecture(
feature_extender, 10, np.array([0, 0.5, 0.6, -0.2] + [0] * 33),
None))
def test_extract_relevant_variables_indices(self):
algorithm = categorical_harmonica.Harmonica(test_utils.create_spec(
phoenix_spec_pb2.PhoenixSpec.CNN,
blocks_to_use=[
"FIXED_CHANNEL_CONVOLUTION_16", "FIXED_CHANNEL_CONVOLUTION_32",
"FIXED_CHANNEL_CONVOLUTION_64", "CONVOLUTION_3X3"
],
min_depth=4),
degree=2)
feature_extender = PolynomialFeatures(2, interaction_only=True)
algorithm._get_polynomial_expansion(feature_extender,
np.array([[1, 2, 3, 4]]))
output = algorithm._extract_relevant_variables_indices(
feature_extender, np.array([1, 1, 1] + [0] * 7 + [1]))
logging.info(output)
self.assertAllEqual(output, set([0, 1, 2, 3]))
output = algorithm._extract_relevant_variables_indices(
feature_extender, np.array([1, 0, 0] + [0] * 7 + [1]))
self.assertAllEqual(output, set([2, 3]))
output = algorithm._extract_relevant_variables_indices(
feature_extender, np.array([1, 0, 0] + [0] * 7 + [0]))
self.assertEmpty(output)
def test_batch_sample(self, get_architecture):
algorithm = categorical_harmonica.Harmonica(test_utils.create_spec(
phoenix_spec_pb2.PhoenixSpec.DNN,
blocks_to_use=[
"FIXED_CHANNEL_CONVOLUTION_16", "FIXED_CHANNEL_CONVOLUTION_32",
"FIXED_CHANNEL_CONVOLUTION_64", "CONVOLUTION_3X3"
],
min_depth=4),
degree=1)
get_architecture.side_effect = [
np.array([1, 2, 3, 4]),
np.array([2, 3, 4, 1]),
np.array([3, 4, 1, 2]),
# Adding 34 (flatten) to see that it is ignored
np.array([4, 1, 2, 34, 3]),
np.array([1, 1, 1, 1]),
np.array([2, 2, 2, 2]),
np.array([3, 3, 3, 3]),
np.array([2, 3, 2, 3]),
np.array([3, 4, 3, 4])
]
x, y = algorithm.batch_sample(_create_trials())
# Check first and last assignments
self.assertAllAlmostEqual([
1.00000000e+00, 1.00000000e+00, 1.00000000e+00, 1.00000000e+00,
1.00000000e+00, 6.12323400e-17, -1.00000000e+00, -1.83697020e-16,
1.00000000e+00, -1.00000000e+00, 1.00000000e+00, -1.00000000e+00,
1.00000000e+00, -1.83697020e-16, -1.00000000e+00, 5.51091060e-16
], x[0])
self.assertAllAlmostEqual([
0.00000000e+00, 1.22464680e-16, -2.44929360e-16, 3.67394040e-16,
0.00000000e+00, -1.00000000e+00, 3.67394040e-16, 1.00000000e+00,
0.00000000e+00, 1.22464680e-16, -2.44929360e-16, 3.67394040e-16,
0.00000000e+00, -1.00000000e+00, 3.67394040e-16, 1.00000000e+00
], x[17])
self.assertAllEqual([
0.97, 0., 0.94, 0., 0.7, 0., 0.72, 0., 0.3, 0., 0.79, 0., 0.39, 0.,
0.9, 0., 0.19, 0.
], y)
class CoordinateDescentTest(parameterized.TestCase, tf.test.TestCase):
def setUp(self):
super(CoordinateDescentTest, self).setUp()
self._metadata = ml_metadata_db.MLMetaData(None, None, None)
@parameterized.named_parameters(
{
"testcase_name":
"increase_size",
"spec":
search_test_utils.create_spec(phoenix_spec_pb2.PhoenixSpec.DNN),
"init_architecture": [
"FIXED_CHANNEL_CONVOLUTION_32", "FIXED_CHANNEL_CONVOLUTION_64",
"CONVOLUTION_3X3"
], # np.array([2, 3, 4])
"completed_trials":
100000,
"new_block":
"FIXED_CHANNEL_CONVOLUTION_16", # enum:1
"should_increase_depth":
True,
"expected_fork_architecture":
np.array([1, 2, 3, 4])
},
{
"testcase_name":
"no_completed_trials",
"spec":
search_test_utils.create_spec(phoenix_spec_pb2.PhoenixSpec.DNN),
"init_architecture": [
"FIXED_CHANNEL_CONVOLUTION_32", "FIXED_CHANNEL_CONVOLUTION_64",
"CONVOLUTION_3X3"
], # np.array([2, 3, 4])
"completed_trials":
0,
"new_block":
"FIXED_CHANNEL_CONVOLUTION_16", # enum:1
"should_increase_depth":
False,
"expected_fork_architecture":
np.array([2, 3, 4])
},
{
"testcase_name":
"custom_depth_thresholds",
"spec":
search_test_utils.create_spec(phoenix_spec_pb2.PhoenixSpec.DNN,
[1, 2, 3, 4]),
"init_architecture": [
"FIXED_CHANNEL_CONVOLUTION_32", "FIXED_CHANNEL_CONVOLUTION_64",
"CONVOLUTION_3X3"
], # np.array([2, 3, 4])
"completed_trials":
3,
"new_block":
"FIXED_CHANNEL_CONVOLUTION_16", # enum:1
"should_increase_depth":
False,
"expected_fork_architecture":
np.array([1, 2, 3, 4])
},
{
"testcase_name":
"should_increase_depth",
"spec":
search_test_utils.create_spec(phoenix_spec_pb2.PhoenixSpec.DNN,
[1, 2, 3, 4]),
"init_architecture": [
"FIXED_CHANNEL_CONVOLUTION_32", "FIXED_CHANNEL_CONVOLUTION_64",
"CONVOLUTION_3X3"
], # np.array([2, 3, 4])
"completed_trials":
4,
"new_block":
"FIXED_CHANNEL_CONVOLUTION_16", # enum:1
"should_increase_depth":
True,
"expected_fork_architecture":
np.array([1, 2, 3, 4])
},
{
"testcase_name":
"increase_complexity_probability_eq_zero",
"spec":
search_test_utils.create_spec(phoenix_spec_pb2.PhoenixSpec.DNN,
[1, 2, 3, 4]),
"init_architecture": [
"FIXED_CHANNEL_CONVOLUTION_32", "FIXED_CHANNEL_CONVOLUTION_64",
"CONVOLUTION_3X3"
], # np.array([2, 3, 4])
"completed_trials":
4,
"new_block":
"FIXED_CHANNEL_CONVOLUTION_16", # enum:1
"should_increase_depth":
False,
"expected_fork_architecture":
np.array([1, 2, 3, 4]),
"increase_complexity_probability":
0.0,
},
)
@mock.patch("model_search.architecture.architecture_utils"
".get_architecture")
def test_get_suggestion(self,
get_architecture,
spec,
init_architecture,
completed_trials,
new_block,
should_increase_depth,
expected_fork_architecture,
increase_complexity_probability=1.0):
spec.increase_complexity_probability = increase_complexity_probability
get_architecture.return_value = np.array([1, 2, 3, 4])
algorithm = coordinate_descent.CoordinateDescent(spec, self._metadata)
hparams = hp.HParams(initial_architecture=init_architecture,
new_block_type=new_block)
trials = []
for i in range(completed_trials):
trials.append(
trial_module.Trial({
"id": i,
"model_dir": "/tmp/" + str(i),
"status": "COMPLETED",
"trial_infeasible": False,
"final_measurement": {
"objective_value": 100 - i
}
}))
# Adding one infeasible to make sure we don't fork from it.
trials.append(
trial_module.Trial({
"id": 99,
"model_dir": "/tmp/99",
"status": "COMPLETED",
"trial_infeasible": True,
"final_measurement": {
"objective_value": 0
}
}))
logging.info(trials)
output_architecture, fork_trial = algorithm.get_suggestion(
trials, hparams)
if completed_trials:
self.assertEqual(fork_trial, completed_trials - 1)
if should_increase_depth:
self.assertAllEqual(
output_architecture,
np.append(expected_fork_architecture,
blocks.BlockType[new_block]))
else:
self.assertEqual(output_architecture.shape,
expected_fork_architecture.shape)
self.assertTrue(
search_test_utils.is_mutation_or_equal(
expected_fork_architecture, output_architecture))
@mock.patch("model_search.architecture.architecture_utils"
".get_architecture")
def test_get_suggestion_beam_size_gt_one(self, get_architecture):
# Trials should be ranked by trial_id. That is, we should only ever fork
# from the first 2 trials.
beam_size = 2
trial_to_arch = {
0: np.array([1, 2, 3]),
1: np.array([4, 5, 6]),
2: np.array([7, 8, 9]),
3: np.array([10, 11, 12])
}
get_architecture.side_effect = lambda idx: trial_to_arch[int(idx)]
spec = search_test_utils.create_spec(phoenix_spec_pb2.PhoenixSpec.DNN)
spec.beam_size = beam_size
spec.increase_complexity_minimum_trials.append(0)
algorithm = coordinate_descent.CoordinateDescent(spec, self._metadata)
hparams = hp.HParams(
new_block_type="FIXED_CHANNEL_CONVOLUTION_16") # enum: 1
# Create one fake trial for each architecture.
trials = []
for i in range(3):
trials.append(
trial_module.Trial({
"id": i,
"model_dir": str(i),
"status": "COMPLETED",
"trial_infeasible": False,
"final_measurement": {
"objective_value": i
}
}))
# Adding one infeasible to make sure we don't fork from it.
trials.append(
trial_module.Trial({
"id": 4,
"model_dir": "4",
"status": "COMPLETED",
"trial_infeasible": True,
"final_measurement": {
"objective_value": 0
}
}))
logging.info(trials)
# Since forking is random, fork 1000 times then check that we forked from
# only the trials we care about.
forked = set()
for i in range(1000):
_, fork_trial = algorithm.get_suggestion(trials, hparams)
forked.add(int(fork_trial))
self.assertEqual(forked, {0, 1})
class HarmonicaTest(parameterized.TestCase, tf.test.TestCase):
def assertAllAlmostEqual(self, left, right, places=3):
for i in range(len(left)):
self.assertAlmostEqual(left[i], right[i], places=places)
def test_polynomial_expansion(self):
algorithm = categorical_harmonica.Harmonica(
test_utils.create_spec(phoenix_spec_pb2.PhoenixSpec.DNN))
feature_extender = PolynomialFeatures(2, interaction_only=True)
self.assertAllEqual([[1, 0, 1, 0], [1, 0.5, 1, 0.5]],
algorithm._get_polynomial_expansion(
feature_extender, np.array([[0, 1], [0.5,
1]])))
def test_translate_architecture_to_assignment(self):
algorithm = categorical_harmonica.Harmonica(
test_utils.create_spec(phoenix_spec_pb2.PhoenixSpec.DNN,
blocks_to_use=[
"FIXED_CHANNEL_CONVOLUTION_16",
"FIXED_CHANNEL_CONVOLUTION_32",
"FIXED_CHANNEL_CONVOLUTION_64",
"CONVOLUTION_3X3"
],
min_depth=4))
assignment = algorithm.translate_architecture_to_feature_assignment(
np.array([2, 3, 4]))
expected_output = [[
1.00000000e+00, 6.12323400e-17, -1.00000000e+00, -1.83697020e-16,
1.00000000e+00, -1.00000000e+00, 1.00000000e+00, -1.00000000e+00,
1.00000000e+00, -1.83697020e-16, -1.00000000e+00, 5.51091060e-16,
0.00000000e+00, -1.00000000e+00, 3.67394040e-16, 1.00000000e+00
],
[
0.00000000e+00, 1.00000000e+00, 1.22464680e-16,
-1.00000000e+00, 0.00000000e+00, 1.22464680e-16,
-2.44929360e-16, 3.67394040e-16, 0.00000000e+00,
-1.00000000e+00, 3.67394040e-16, 1.00000000e+00,
1.00000000e+00, -1.83697020e-16,
-1.00000000e+00, 5.51091060e-16
]]
for i in range(len(assignment)):
self.assertAllAlmostEqual(expected_output[i], assignment[i])
@mock.patch("model_search.architecture.architecture_utils"
".get_architecture")
def test_batch_sample(self, get_architecture):
algorithm = categorical_harmonica.Harmonica(test_utils.create_spec(
phoenix_spec_pb2.PhoenixSpec.DNN,
blocks_to_use=[
"FIXED_CHANNEL_CONVOLUTION_16", "FIXED_CHANNEL_CONVOLUTION_32",
"FIXED_CHANNEL_CONVOLUTION_64", "CONVOLUTION_3X3"
],
min_depth=4),
degree=1)
get_architecture.side_effect = [
np.array([1, 2, 3, 4]),
np.array([2, 3, 4, 1]),
np.array([3, 4, 1, 2]),
# Adding 34 (flatten) to see that it is ignored
np.array([4, 1, 2, 34, 3]),
np.array([1, 1, 1, 1]),
np.array([2, 2, 2, 2]),
np.array([3, 3, 3, 3]),
np.array([2, 3, 2, 3]),
np.array([3, 4, 3, 4])
]
x, y = algorithm.batch_sample(_create_trials())
# Check first and last assignments
self.assertAllAlmostEqual([
1.00000000e+00, 1.00000000e+00, 1.00000000e+00, 1.00000000e+00,
1.00000000e+00, 6.12323400e-17, -1.00000000e+00, -1.83697020e-16,
1.00000000e+00, -1.00000000e+00, 1.00000000e+00, -1.00000000e+00,
1.00000000e+00, -1.83697020e-16, -1.00000000e+00, 5.51091060e-16
], x[0])
self.assertAllAlmostEqual([
0.00000000e+00, 1.22464680e-16, -2.44929360e-16, 3.67394040e-16,
0.00000000e+00, -1.00000000e+00, 3.67394040e-16, 1.00000000e+00,
0.00000000e+00, 1.22464680e-16, -2.44929360e-16, 3.67394040e-16,
0.00000000e+00, -1.00000000e+00, 3.67394040e-16, 1.00000000e+00
], x[17])
self.assertAllEqual([
0.97, 0., 0.94, 0., 0.7, 0., 0.72, 0., 0.3, 0., 0.79, 0., 0.39, 0.,
0.9, 0., 0.19, 0.
], y)
@parameterized.named_parameters(
{
"testcase_name": "cnn",
"problem_type": phoenix_spec_pb2.PhoenixSpec.CNN
}, {
"testcase_name": "dnn",
"problem_type": phoenix_spec_pb2.PhoenixSpec.DNN
})
def test_get_good_architecture(self, problem_type):
algorithm = categorical_harmonica.Harmonica(test_utils.create_spec(
problem_type,
blocks_to_use=[
"FIXED_CHANNEL_CONVOLUTION_16", "FIXED_CHANNEL_CONVOLUTION_32",
"FIXED_CHANNEL_CONVOLUTION_64", "CONVOLUTION_3X3"
],
min_depth=2),
degree=2,
seed=73)
expected_output = [3, 3]
if problem_type == phoenix_spec_pb2.PhoenixSpec.CNN:
expected_output += [34]
feature_extender = PolynomialFeatures(2, interaction_only=True)
self.assertAllEqual(
expected_output,
algorithm._get_good_architecture(
feature_extender, 10, np.array([0, 0.5, 0.6, -0.2] + [0] * 33),
None))
@parameterized.named_parameters(
{
"testcase_name":
"regular_cnn",
"spec":
test_utils.create_spec(phoenix_spec_pb2.PhoenixSpec.CNN),
"initial_architecture": [
"FIXED_OUTPUT_FULLY_CONNECTED_128",
"FIXED_CHANNEL_CONVOLUTION_32", "FIXED_CHANNEL_CONVOLUTION_64",
"CONVOLUTION_3X3"
],
"expected_architecture":
np.array([2, 3, 4, 34, 22])
}, {
"testcase_name":
"regular_dnn",
"spec":
test_utils.create_spec(phoenix_spec_pb2.PhoenixSpec.DNN),
"initial_architecture": [
"FIXED_OUTPUT_FULLY_CONNECTED_128",
"FIXED_OUTPUT_FULLY_CONNECTED_256"
],
"expected_architecture":
np.array([22, 23])
})
def test_not_enough_data_get_suggestion(self, spec, initial_architecture,
expected_architecture):
algorithm = categorical_harmonica.Harmonica(spec)
output_architecture, _ = algorithm.get_suggestion(
[], hp.HParams(initial_architecture=initial_architecture))
self.assertAllEqual(expected_architecture, output_architecture)
@parameterized.named_parameters(
{
"testcase_name": "cnn",
"problem_type": phoenix_spec_pb2.PhoenixSpec.CNN
}, {
"testcase_name": "dnn",
"problem_type": phoenix_spec_pb2.PhoenixSpec.DNN
})
@mock.patch("model_search.architecture.architecture_utils"
".get_architecture")
def test_get_suggestion(self, get_architecture, problem_type):
# Return value (architectures) for the various trials.
get_architecture.side_effect = [
np.array([1, 2, 3, 4]),
np.array([2, 3, 4, 1]),
np.array([3, 4, 1, 2]),
np.array([4, 1, 2, 3]),
np.array([1, 1, 1, 1]),
np.array([2, 2, 2, 2]),
np.array([3, 3, 3, 3]),
np.array([2, 3, 2, 3]),
np.array([3, 4, 3, 4])
]
algorithm = categorical_harmonica.Harmonica(test_utils.create_spec(
problem_type,
blocks_to_use=[
"FIXED_CHANNEL_CONVOLUTION_16", "FIXED_CHANNEL_CONVOLUTION_32",
"FIXED_CHANNEL_CONVOLUTION_64", "CONVOLUTION_3X3"
],
min_depth=4),
num_random_samples=10,
seed=73)
output_architecture, _ = algorithm.get_suggestion(
_create_trials(), hp.HParams())
expected_output = [3, 3, 3, 3]
if problem_type == phoenix_spec_pb2.PhoenixSpec.CNN:
expected_output += [34]
self.assertAllEqual(expected_output, output_architecture)
def test_extract_relevant_variables_indices(self):
algorithm = categorical_harmonica.Harmonica(test_utils.create_spec(
phoenix_spec_pb2.PhoenixSpec.CNN,
blocks_to_use=[
"FIXED_CHANNEL_CONVOLUTION_16", "FIXED_CHANNEL_CONVOLUTION_32",
"FIXED_CHANNEL_CONVOLUTION_64", "CONVOLUTION_3X3"
],
min_depth=4),
degree=2)
feature_extender = PolynomialFeatures(2, interaction_only=True)
algorithm._get_polynomial_expansion(feature_extender,
np.array([[1, 2, 3, 4]]))
output = algorithm._extract_relevant_variables_indices(
feature_extender, np.array([1, 1, 1] + [0] * 7 + [1]))
logging.info(output)
self.assertAllEqual(output, set([0, 1, 2, 3]))
output = algorithm._extract_relevant_variables_indices(
feature_extender, np.array([1, 0, 0] + [0] * 7 + [1]))
self.assertAllEqual(output, set([2, 3]))
output = algorithm._extract_relevant_variables_indices(
feature_extender, np.array([1, 0, 0] + [0] * 7 + [0]))
self.assertEmpty(output)
def test_get_good_architecture_with_relevant_variables(self):
algorithm = categorical_harmonica.Harmonica(test_utils.create_spec(
phoenix_spec_pb2.PhoenixSpec.CNN,
blocks_to_use=[
"FIXED_CHANNEL_CONVOLUTION_16", "FIXED_CHANNEL_CONVOLUTION_32",
"FIXED_CHANNEL_CONVOLUTION_64", "CONVOLUTION_3X3"
],
min_depth=2),
degree=2,
seed=73)
expected_output = [3, 1, 34]
feature_extender = PolynomialFeatures(2, interaction_only=True)
self.assertAllEqual(
expected_output,
algorithm._get_good_architecture(
feature_extender, 20, np.array([0, 0.5, 0.6, -0.2] + [0] * 33),
set([3, 3, 3, 3])))