def test_value_search_checks_bad_solutions(self):
inputs = {'tensor': [10, 20, 30], 'index': -2}
output = 20
constants = [1, 20]
results = value_search.run_value_search_from_example(
inputs=inputs,
output=output,
constants=constants,
settings=settings_module.from_dict(
{'require_all_inputs_used': True}))
self.assertLen(results.solutions, 1)
self.assertEqual(results.solutions[0].expression, 'tensor[index]')
results = value_search.run_value_search_from_example(
inputs=inputs,
output=output,
constants=constants,
settings=settings_module.from_dict({
'require_all_inputs_used': False,
'require_one_input_used': True
}))
self.assertLen(results.solutions, 1)
self.assertEqual(results.solutions[0].expression, 'tensor[1]')
results = value_search.run_value_search_from_example(
inputs=inputs,
output=output,
constants=constants,
settings=settings_module.from_dict({
'require_all_inputs_used': False,
'require_one_input_used': False
}))
self.assertLen(results.solutions, 1)
self.assertEqual(results.solutions[0].expression, 'tf.constant(20)')
def get_settings():
"""Specifies settings for TF-Coder. Edit this function!"""
# How long to search for a solution, in seconds.
time_limit = 300
# How many solutions to find before stopping. If more than 1, the entire
# search will slow down.
number_of_solutions = 1
# Whether solutions must use all inputs, at least one input, or no such
# requirement. Choose one of "all inputs", "one input", "no restriction".
solution_requirement = 'all inputs'
assert solution_requirement in [
'all inputs', 'one input', 'no restriction'
]
return settings_module.from_dict({
'timeout':
time_limit,
'only_minimal_solutions':
False,
'max_solutions':
number_of_solutions,
'require_all_inputs_used':
solution_requirement == 'all inputs',
'require_one_input_used':
solution_requirement == 'one input',
})
def test_value_search_can_load_data(self, mock_stdout):
benchmark = all_benchmarks.find_benchmark_with_name('simple_cast')
handler = description_handler_factory.create_handler('naive_bayes')
settings = settings_module.from_dict({
'timeout':
5,
'printing.prioritized_operations':
True,
'printing.deprioritized_operations':
True,
})
tensor_config = tensor_features_model.load_config(
settings.tensor_model.config_path)
tensor_model = tensor_features_model.get_model(tensor_config)
tensor_checkpoint = tensor_features_model.create_checkpoint(
tensor_model)
tensor_checkpoint.restore(
settings.tensor_model.checkpoint_path).expect_partial()
results = value_search.run_value_search(benchmark=benchmark,
description_handler=handler,
settings=settings,
tensor_model=tensor_model,
tensor_config=tensor_config)
self.assertLen(results.solutions, 1)
self.assertIn('BOW handler prioritized tf.cast(x, dtype)',
mock_stdout.getvalue())
self.assertIn('Tensor features model prioritized tf.cast(x, dtype)',
mock_stdout.getvalue())
def test_run_value_search_works_for_small_max_weight(
self, benchmark_name, max_weight):
benchmark = all_benchmarks.find_benchmark_with_name(benchmark_name)
results = value_search.run_value_search(
benchmark=benchmark,
settings=settings_module.from_dict({'max_weight': max_weight}))
print("Note: if you're inspecting the test log, the above failure is "
"expected in test_run_value_search_works_for_small_max_weight.")
self.assertEmpty(results.solutions)
def setUp(self):
super(ValueSearchTest, self).setUp()
operations = all_operations.get_operations()
self._constant_operation = None
for operation in operations:
if operation.name == tf_functions.CONSTANT_OPERATION_NAME:
self._constant_operation = operation
break
self.assertIsNotNone(self._constant_operation)
self.settings = settings_module.from_dict({'timeout': 10})
def test_from_dict(self):
overrides = {
'timeout': 12345,
'printing.verbose': True,
'tensor_model_prioritize_threshold': 0.9,
}
settings = settings_module.from_dict(overrides)
self.assertEqual(settings.timeout, 12345)
self.assertTrue(settings.printing.verbose)
self.assertEqual(settings.tensor_model.prioritize_threshold, 0.9)
def setUp(self):
super(ColabLoggingTest, self).setUp()
self.inputs = [[1, 2, 3], [10, 20, 30, 40]]
self.output = [[11, 21, 31, 41], [12, 22, 32, 42], [13, 23, 33, 43]]
self.constants = [12, 34, (56, 78)]
self.description = 'add two vectors with broadcasting to get a matrix'
self.settings = settings_module.from_dict({
'timeout': 60,
'only_minimal_solutions': False,
'max_solutions': 2,
})
colab_logging._IPYTHON_MOCK.reset_mock()
def test_run_value_search_from_example(self):
results = value_search.run_value_search_from_example(
inputs=[[1, 2, 3], [10, 20, 30]],
output=[11, 22, 33],
settings=settings_module.from_dict({
'timeout': 5,
'max_solutions': 2
}))
self.assertLen(results.solutions, 2)
self.assertEqual(results.solutions[0].expression, 'tf.add(in1, in2)')
self.assertEqual(results.solutions[1].expression, 'tf.add(in2, in1)')
# Linter suggests self.assertLess() but it's wrong.
self.assertTrue(0.0 < results.total_time < 5.0) # pylint: disable=g-generic-assert
def test_run_value_search_works_for_simple_benchmarks(
self, simple_benchmark):
handler = description_handler_factory.create_handler('tfidf')
results = value_search.run_value_search(
benchmark=simple_benchmark,
settings=settings_module.from_dict({
'timeout': 20,
'printing.statistics': True
}),
description_handler=handler)
self.assertLen(results.solutions, 1)
self.assertIsNotNone(results.solutions[0].expression)
self.assertGreater(results.solutions[0].weight, 0)
self.assertGreater(results.total_time, 0)
self.assertNotEmpty(results.value_set)
self.assertGreaterEqual(results.statistics.total_apply_count, 0)
def test_run_value_search_works_for_important_benchmarks(
self, benchmark_name, timeout, solution):
# These benchmarks are frequently used in demos. We should ensure that they
# are solved quickly and consistently. While the exact solution may change
# as the algorithm improves, we should ensure that the solution is always
# simple and understandable.
handler = description_handler_factory.create_handler('tfidf')
benchmark = all_benchmarks.find_benchmark_with_name(benchmark_name)
results = value_search.run_value_search(
benchmark=benchmark,
description_handler=handler,
settings=settings_module.from_dict({'timeout': timeout}))
print('Time for benchmark {}: {:.2f} sec'.format(
benchmark_name, results.total_time))
self.assertLen(results.solutions, 1)
self.assertEqual(results.solutions[0].expression, solution)
def test_run_value_search_only_minimal_solutions(self,
only_minimal_solutions,
expected_solutions):
benchmark = benchmark_module.Benchmark(examples=[
benchmark_module.Example(inputs=[[1, 4], [2, 7]], output=[3, 11])
])
results = value_search.run_value_search(
benchmark=benchmark,
settings=settings_module.from_dict({
'timeout': 20,
'max_solutions': 4,
'only_minimal_solutions': only_minimal_solutions,
'max_extra_solutions_time': 20
}))
self.assertLen(results.solutions, len(expected_solutions))
self.assertEqual(
[solution.expression for solution in results.solutions],
expected_solutions)
def test_version(self):
inputs = {
'rows': [10, 20, 30],
'cols': [1, 2, 3, 4],
}
output = [[11, 12, 13, 14], [21, 22, 23, 24], [31, 32, 33, 34]]
constants = []
description = 'add two vectors with broadcasting to get a matrix'
logging_dict = colab_logging.get_problem_logging_dict(
inputs=inputs,
output=output,
constants=constants,
description=description,
settings=settings_module.from_dict({'timeout': 99}),
include_in_dataset=False,
problem_id=123)
self.assertEqual(logging_dict['tf_coder_version'],
tf_coder_version.__version__)
self.assertEqual(logging_dict['logging_version'],
logging_version.__version__)
def test_run_value_search_works_for_simple_benchmarks(
self, simple_benchmark):
handler = description_handler_factory.create_handler('tfidf')
results = value_search.run_value_search(
benchmark=simple_benchmark,
settings=settings_module.from_dict({
'timeout': 20,
'printing.statistics': True,
'require_all_inputs_used': False,
'require_one_input_used': False,
}),
description_handler=handler)
self.assertLen(results.solutions, 1)
self.assertIsNotNone(results.solutions[0].expression)
self.assertGreater(results.solutions[0].weight, 0)
self.assertGreater(results.total_time, 0)
self.assertNotEmpty(results.value_set)
self.assertGreaterEqual(results.statistics.total_apply_count, 0)
# The simple benchmarks are so simple that we can directly write down the
# correct (minimal) target program.
self.assertEqual(results.solutions[0].expression,
simple_benchmark.target_program)
def find_op(einsum_str, inputs, max_solutions=1, time_limit=300, n_samples=1):
#print("Warm up...")
colab_interface.warm_up()
#print("Warm up done")
#if n_samples == 1 :
outputs = tf.einsum(einsum_str, *inputs)
#else :
# g_inputs = [inputs]
# for i in range(n_samples):
# g_inputs.append( #Tensor With Same Shape )
# outputs = [ tf.einsum(einsum_str,*inp) for inp in g_inputs ]
settings = settings_module.from_dict({
'timeout': time_limit,
'only_minimal_solutions': False,
'max_solutions': max_solutions,
'require_all_inputs_used': True,
'require_one_input_used': False,
})
colab_interface.run_value_search_from_colab(inputs, outputs, [], "",
settings)
def test_value_search_prioritizes_operations(self):
# Make sure prioritized ops get printed out.
settings = settings_module.from_dict({
'timeout':
0.5, # We don't actually care about solving the problem.
'printing.prioritized_operations':
True,
'printing.deprioritized_operations':
True,
'tensor_model.prioritize_threshold':
0.5,
'tensor_model.deprioritize_threshold':
0.2,
})
# Create data for the mocks.
all_ops = all_operations.get_operations(include_sparse_operations=True)
op_names = [op.name for op in all_ops]
abs_index = None # Prioritized by tensor features model.
argmax_index = None # Deprioritized by tensor features model.
for i, op in enumerate(all_ops):
if op.name == 'tf.abs(x)':
abs_index = i
elif op.name == 'tf.argmax(input, axis)':
argmax_index = i
self.assertIsNotNone(abs_index)
self.assertIsNotNone(argmax_index)
operation_probs = np.repeat(0.4, len(all_ops))
operation_probs[abs_index] = 0.6 # Above prioritize threshold.
operation_probs[argmax_index] = 0.1 # Below deprioritize threshold.
operation_logits = np.expand_dims(_inverse_sigmoid(operation_probs),
axis=0)
nl_data = [
# Associate "apple" with tf.zeros.
{
'docstring': ['apple pie'],
'tf_functions': ['tf.zeros'],
'comments': [],
'names': [],
'strings': []
},
] * 100
# Mock the tensor model's predictions, the NL model's data, and print().
with mock.patch('tf_coder.models.tensor_features_model.eval_single_example',
return_value=tensor_features_model.Result(
operation_logits=operation_logits)), \
mock.patch('tf_coder.datasets.github.data_loader.load_data',
return_value=nl_data), \
mock.patch('sys.stdout', new_callable=six.StringIO) as mock_stdout:
handler = bag_of_words_handlers.NaiveBayesDescriptionHandler(
max_num_prioritized=1)
benchmark = benchmark_module.Benchmark(
# I/O example doesn't matter.
examples=[
benchmark_module.Example({'my_var': [1, 2]}, [2, 1])
],
# Description contains "apple"!
description='honeycrisp apple')
value_search.run_value_search(benchmark=benchmark,
description_handler=handler,
settings=settings,
tensor_model=mock.Mock(),
tensor_config={
'max_num_inputs': 3,
'operation_names': op_names
})
self.assertIn('BOW handler prioritized tf.zeros',
mock_stdout.getvalue())
self.assertIn('Tensor features model prioritized tf.abs(x), p=0.6',
mock_stdout.getvalue())
self.assertIn(
'Tensor features model deprioritized tf.argmax(input, axis), p=0.1',
mock_stdout.getvalue())
