def test_get_operation_multipliers_respects_min_tfidf_score(
self, min_tfidf_score):
handler = tfidf_handler.TfidfDescriptionHandler(
max_num_prioritized=1000000, min_tfidf_score=min_tfidf_score)
description = 'Tile a tensor multiple times'
scores = handler.score_description(description)
benchmark = benchmark_module.Benchmark(examples=[
benchmark_module.Example(
inputs=[
[10],
[20],
],
output=[30],
),
],
constants=[],
description=description,
target_program='',
source='test',
name='test_benchmark')
multipliers = handler.get_operation_multipliers(
benchmark, settings=settings_module.default_settings())
prioritized_names = [
name for name in multipliers.keys() if multipliers[name] < 1
]
expected_prioritized_names = [
name for name in scores.keys() if scores[name] >= min_tfidf_score
]
self.assertCountEqual(prioritized_names, expected_prioritized_names)
# All multipliers must be in (0, 1] (no operation is deprioritized).
self.assertTrue(
all(0 < multiplier <= 1 for multiplier in multipliers.values()))
def test_get_operation_multiplier_respects_max_num_prioritized(
self, max_num_prioritized):
handler = tfidf_handler.TfidfDescriptionHandler(
max_num_prioritized=max_num_prioritized, min_tfidf_score=0)
benchmark = benchmark_module.Benchmark(
examples=[
benchmark_module.Example(
inputs=[
[10],
[20],
],
output=[30],
),
],
constants=[],
description='Tile a tensor multiple times',
target_program='',
source='test',
name='test_benchmark')
multipliers = handler.get_operation_multipliers(
benchmark, settings=settings_module.default_settings())
actual_num_prioritized = sum(multiplier < 1
for multiplier in multipliers.values())
self.assertEqual(actual_num_prioritized, max_num_prioritized)
# All multipliers must be in (0, 1] (no operation is deprioritized).
self.assertTrue(
all(0 < multiplier <= 1 for multiplier in multipliers.values()))
def run_value_search_from_example(
inputs: Union[List[Any], Dict[Text, Any]],
output: Any,
settings: Optional[settings_module.Settings] = None,
**kwargs) -> ValueSearchResults:
"""Performs value search for a single user-provided input-output example.
Args:
inputs: A list of inputs, or a dict mapping input names to inputs.
output: The corresponding desired output.
settings: An optional Settings object to use, or None to use defaults.
**kwargs: The kwarg 'constants' can be used to specify a list of constants,
and 'description' can be used to provide a natural language description of
the task. Other arguments are passed directly to run_value_search().
Returns:
A ValueSearchResults namedtuple.
"""
if settings is None:
settings = settings_module.default_settings()
constants = kwargs.pop('constants', None)
description = kwargs.pop('description', None)
source = kwargs.pop('source', 'From user-provided example.')
benchmark = benchmark_module.Benchmark(
examples=[benchmark_module.Example(inputs, output)],
constants=constants, # Will turn into empty list if constants=None.
description=description, # Will turn into '' if description=None.
source=source)
return run_value_search(benchmark, settings, **kwargs)
def test_apply_for_tensor(self, index, expected):
tensor = tf.constant([[1.2, 3.4], [5.6, 7.8]])
arg_values = [value.ConstantValue(tensor), value.ConstantValue(index)]
result_value = self.operation.apply(arg_values,
settings_module.default_settings())
self.assertEqual(result_value,
value.ConstantValue(tf.constant(expected)))
def setUp(self):
super(OperationValueTest, self).setUp()
function_info = tf_functions.FunctionInfo(
name='tf.reduce_max(input_tensor, axis)',
filter_group=filter_group.FilterGroup.NONE,
weight=1)
self.operation = function_operation.FunctionOperation(function_info)
self.settings = settings_module.default_settings()
def test_apply_for_list(self, index, expected):
arg_values = [
value.ConstantValue([1.2, 3.4, 5.6]),
value.ConstantValue(index)
]
result_value = self.operation.apply(arg_values,
settings_module.default_settings())
self.assertEqual(result_value, value.ConstantValue(expected))
def test_get_operation_multipliers(self, description, operation_name,
expected_multiplier):
handler = description_handler.FunctionNameDescriptionHandler()
multipliers = handler.get_operation_multipliers(
description, settings_module.default_settings())
self.assertEqual(multipliers.get(operation_name, 1.0),
expected_multiplier)
# Nothing is deprioritized.
self.assertTrue(all(0 < value < 1 for value in multipliers.values()))
def test_get_operation_multiplier_respects_max_num_prioritized(
self, max_num_prioritized):
handler = tfidf_handler.TfidfDescriptionHandler(
max_num_prioritized=max_num_prioritized, min_tfidf_score=0)
multipliers = handler.get_operation_multipliers(
'Tile a tensor multiple times',
settings=settings_module.default_settings())
actual_num_prioritized = sum(multiplier < 1
for multiplier in multipliers.values())
self.assertEqual(actual_num_prioritized, max_num_prioritized)
# All multipliers must be in (0, 1] (no operation is deprioritized).
self.assertTrue(
all(0 < multiplier <= 1 for multiplier in multipliers.values()))
def setUp(self):
super(CollectTensorDataTest, self).setUp()
self.settings = settings_module.default_settings()
operations = all_operations.get_operations()
self.unique_with_counts_operation = all_operations.find_operation_with_name(
'tf.unique_with_counts(x)', operation_list=operations)
self.indexing_operation = all_operations.find_operation_with_name(
'IndexingOperation', operation_list=operations)
self.gather_operation = all_operations.find_operation_with_name(
'tf.gather(params, indices)', operation_list=operations)
self.add_operation = all_operations.find_operation_with_name(
'tf.add(x, y)', operation_list=operations)
# Example with many operations.
in1 = value_module.InputValue([1, 1, 2, 5, 6, 5], 'in1')
in2 = value_module.InputValue([0, 10, 20, 30, 40, 50, 60, 70], 'in2')
constant_1 = value_module.ConstantValue(1)
unique = self.unique_with_counts_operation.apply([in1], self.settings)
indexed = self.indexing_operation.apply([unique, constant_1],
self.settings)
gathered = self.gather_operation.apply([in2, in1], self.settings)
self.example_value_1 = self.add_operation.apply([indexed, gathered],
self.settings)
self.assertEqual(
self.example_value_1.reconstruct_expression(),
'tf.add(tf.unique_with_counts(in1)[1], tf.gather(in2, in1))')
self.assertEqual(self.example_value_1,
value_module.OutputValue([10, 10, 21, 52, 63, 52]))
# Example with many variables and new inputs.
in3 = value_module.InputValue([1], 'in3')
in4 = value_module.InputValue([2], 'in4')
a = self.add_operation.apply([in3, new_input([10])], self.settings)
b = self.add_operation.apply([in4, in3], self.settings)
c = self.add_operation.apply([new_input([20]), in3], self.settings)
d = self.add_operation.apply([a, b], self.settings)
self.example_value_2 = self.add_operation.apply([c, d], self.settings)
self.assertEqual(
self.example_value_2.reconstruct_expression(),
'tf.add(tf.add(NEW_INPUT, in3), '
'tf.add(tf.add(in3, NEW_INPUT), tf.add(in4, in3)))')
self.assertEqual(self.example_value_2, value_module.OutputValue([35]))
def test_get_operation_multipliers_respects_min_tfidf_score(
self, min_tfidf_score):
handler = tfidf_handler.TfidfDescriptionHandler(
max_num_prioritized=1000000, min_tfidf_score=min_tfidf_score)
description = 'Tile a tensor multiple times'
scores = handler.score_description(description)
multipliers = handler.get_operation_multipliers(
description, settings=settings_module.default_settings())
prioritized_names = [
name for name in multipliers.keys() if multipliers[name] < 1
]
expected_prioritized_names = [
name for name in scores.keys() if scores[name] >= min_tfidf_score
]
self.assertCountEqual(prioritized_names, expected_prioritized_names)
# All multipliers must be in (0, 1] (no operation is deprioritized).
self.assertTrue(
all(0 < multiplier <= 1 for multiplier in multipliers.values()))
def test_get_operation_multiplier_respects_max_num_prioritized(
self, max_num_prioritized):
# Note: More operations may be prioritized if they are chosen because they
# are explicitly listed in the description.
handler = tfidf_handler.TfidfDescriptionHandler(
max_num_prioritized=max_num_prioritized,
min_tfidf_score=0)
benchmark = benchmark_module.Benchmark(
examples=[benchmark_module.Example(inputs=[1], output=1)],
description='Tiling a tensor multiple times')
multipliers = handler.get_operation_multipliers(
benchmark, settings=settings_module.default_settings())
actual_num_prioritized = sum(multiplier < 1
for multiplier in multipliers.values())
self.assertEqual(actual_num_prioritized, max_num_prioritized)
# All multipliers must be in (0, 1] (no operation is deprioritized).
self.assertTrue(all(0 < multiplier <= 1
for multiplier in multipliers.values()))
def test_get_operation_multipliers(self, description, operation_name,
expected_multiplier):
handler = description_handler.FunctionNameDescriptionHandler()
benchmark = benchmark_module.Benchmark(examples=[
benchmark_module.Example(
inputs=[
[10],
[20],
],
output=[30],
),
],
constants=[],
description=description,
target_program='',
source='test',
name='test_benchmark')
multipliers = handler.get_operation_multipliers(
benchmark, settings_module.default_settings())
self.assertEqual(multipliers.get(operation_name, 1.0),
expected_multiplier)
# Nothing is deprioritized.
self.assertTrue(all(0 < value < 1 for value in multipliers.values()))
def test_get_operation_multipliers_respects_min_tfidf_score(
self, min_tfidf_score):
# Note: An operation may violate the `min_tfidf_score` threshold if it is
# chosen because it is explicitly listed in the description.
handler = tfidf_handler.TfidfDescriptionHandler(
max_num_prioritized=1000000,
min_tfidf_score=min_tfidf_score)
description = 'Tiling a tensor multiple times'
scores = handler.score_description(description)
benchmark = benchmark_module.Benchmark(
examples=[benchmark_module.Example(inputs=[1], output=1)],
description=description)
multipliers = handler.get_operation_multipliers(
benchmark, settings=settings_module.default_settings())
prioritized_names = [name for name in multipliers.keys()
if multipliers[name] < 1]
expected_prioritized_names = [name for name in scores.keys()
if scores[name] >= min_tfidf_score]
self.assertCountEqual(prioritized_names, expected_prioritized_names)
# All multipliers must be in (0, 1] (no operation is deprioritized).
self.assertTrue(all(0 < multiplier <= 1
for multiplier in multipliers.values()))
def test_as_dict_default_works_with_json(self): settings = settings_module.default_settings() as_dict = settings.as_dict() self.assertFalse(as_dict['printing.verbose']) # Test that all defaults are JSON-serializable. self.assertNotEmpty(json.dumps(as_dict))
def setUp(self):
super(NoChangeDescriptionHandlerTest, self).setUp()
self.indexing_operation = python_operations.IndexingOperation()
self.settings = settings_module.default_settings()
def test_apply(self):
self.assertEqual(
self.operation.apply(self.arg_values,
settings_module.default_settings()),
value.OutputValue([34, 78]))
'what happened.')
# TODO(kshi): Add a link to GitHub.
# The global description handler for this session. This will be loaded during
# warm_up().
DESCRIPTION_HANDLER = None
# The tensor features model and config. These will be loaded during warm_up().
TENSOR_MODEL = None
TENSOR_CONFIG = None
# Whether warm_up() was called before.
WARMED_UP = False
# Default settings, if the user doesn't provide their own settings.
DEFAULT_SETTINGS = settings_module.default_settings()
def warm_up():
"""Loads and warms up the tensor features model."""
global DESCRIPTION_HANDLER, TENSOR_MODEL, TENSOR_CONFIG, WARMED_UP
if WARMED_UP:
return
WARMED_UP = True
# Use the default choices for the description handler and tensor model. The
# Colab interface will not allow users to change these.
DESCRIPTION_HANDLER = description_handler_factory.create_handler(
DEFAULT_SETTINGS.description_handler_name)
if (not DEFAULT_SETTINGS.tensor_model.config_path or
def test_apply(self):
self.assertEqual(
self.operation.apply(self.arg_values,
settings_module.default_settings()),
value.ConstantValue((12, )))
def setUp(self):
super(FunctionOperationTest, self).setUp()
self.settings = settings_module.default_settings()
def setUp(self):
super(OperationBaseTest, self).setUp()
self.operation = StrangeAdditionOperation()
self.settings = settings_module.default_settings()
