def test_value_search_checks_bad_solutions(self): inputs = {'tensor': [10, 20, 30], 'index': -2} output = 20 constants = [1, 20] results = value_search.run_value_search_from_example( inputs=inputs, output=output, constants=constants, settings=settings_module.from_dict( {'require_all_inputs_used': True})) self.assertLen(results.solutions, 1) self.assertEqual(results.solutions[0].expression, 'tensor[index]') results = value_search.run_value_search_from_example( inputs=inputs, output=output, constants=constants, settings=settings_module.from_dict({ 'require_all_inputs_used': False, 'require_one_input_used': True })) self.assertLen(results.solutions, 1) self.assertEqual(results.solutions[0].expression, 'tensor[1]') results = value_search.run_value_search_from_example( inputs=inputs, output=output, constants=constants, settings=settings_module.from_dict({ 'require_all_inputs_used': False, 'require_one_input_used': False })) self.assertLen(results.solutions, 1) self.assertEqual(results.solutions[0].expression, 'tf.constant(20)')
def get_settings(): """Specifies settings for TF-Coder. Edit this function!""" # How long to search for a solution, in seconds. time_limit = 300 # How many solutions to find before stopping. If more than 1, the entire # search will slow down. number_of_solutions = 1 # Whether solutions must use all inputs, at least one input, or no such # requirement. Choose one of "all inputs", "one input", "no restriction". solution_requirement = 'all inputs' assert solution_requirement in [ 'all inputs', 'one input', 'no restriction' ] return settings_module.from_dict({ 'timeout': time_limit, 'only_minimal_solutions': False, 'max_solutions': number_of_solutions, 'require_all_inputs_used': solution_requirement == 'all inputs', 'require_one_input_used': solution_requirement == 'one input', })
def test_value_search_can_load_data(self, mock_stdout): benchmark = all_benchmarks.find_benchmark_with_name('simple_cast') handler = description_handler_factory.create_handler('naive_bayes') settings = settings_module.from_dict({ 'timeout': 5, 'printing.prioritized_operations': True, 'printing.deprioritized_operations': True, }) tensor_config = tensor_features_model.load_config( settings.tensor_model.config_path) tensor_model = tensor_features_model.get_model(tensor_config) tensor_checkpoint = tensor_features_model.create_checkpoint( tensor_model) tensor_checkpoint.restore( settings.tensor_model.checkpoint_path).expect_partial() results = value_search.run_value_search(benchmark=benchmark, description_handler=handler, settings=settings, tensor_model=tensor_model, tensor_config=tensor_config) self.assertLen(results.solutions, 1) self.assertIn('BOW handler prioritized tf.cast(x, dtype)', mock_stdout.getvalue()) self.assertIn('Tensor features model prioritized tf.cast(x, dtype)', mock_stdout.getvalue())
def test_run_value_search_works_for_small_max_weight( self, benchmark_name, max_weight): benchmark = all_benchmarks.find_benchmark_with_name(benchmark_name) results = value_search.run_value_search( benchmark=benchmark, settings=settings_module.from_dict({'max_weight': max_weight})) print("Note: if you're inspecting the test log, the above failure is " "expected in test_run_value_search_works_for_small_max_weight.") self.assertEmpty(results.solutions)
def setUp(self): super(ValueSearchTest, self).setUp() operations = all_operations.get_operations() self._constant_operation = None for operation in operations: if operation.name == tf_functions.CONSTANT_OPERATION_NAME: self._constant_operation = operation break self.assertIsNotNone(self._constant_operation) self.settings = settings_module.from_dict({'timeout': 10})
def test_from_dict(self): overrides = { 'timeout': 12345, 'printing.verbose': True, 'tensor_model_prioritize_threshold': 0.9, } settings = settings_module.from_dict(overrides) self.assertEqual(settings.timeout, 12345) self.assertTrue(settings.printing.verbose) self.assertEqual(settings.tensor_model.prioritize_threshold, 0.9)
def setUp(self): super(ColabLoggingTest, self).setUp() self.inputs = [[1, 2, 3], [10, 20, 30, 40]] self.output = [[11, 21, 31, 41], [12, 22, 32, 42], [13, 23, 33, 43]] self.constants = [12, 34, (56, 78)] self.description = 'add two vectors with broadcasting to get a matrix' self.settings = settings_module.from_dict({ 'timeout': 60, 'only_minimal_solutions': False, 'max_solutions': 2, }) colab_logging._IPYTHON_MOCK.reset_mock()
def test_run_value_search_from_example(self): results = value_search.run_value_search_from_example( inputs=[[1, 2, 3], [10, 20, 30]], output=[11, 22, 33], settings=settings_module.from_dict({ 'timeout': 5, 'max_solutions': 2 })) self.assertLen(results.solutions, 2) self.assertEqual(results.solutions[0].expression, 'tf.add(in1, in2)') self.assertEqual(results.solutions[1].expression, 'tf.add(in2, in1)') # Linter suggests self.assertLess() but it's wrong. self.assertTrue(0.0 < results.total_time < 5.0) # pylint: disable=g-generic-assert
def test_run_value_search_works_for_simple_benchmarks( self, simple_benchmark): handler = description_handler_factory.create_handler('tfidf') results = value_search.run_value_search( benchmark=simple_benchmark, settings=settings_module.from_dict({ 'timeout': 20, 'printing.statistics': True }), description_handler=handler) self.assertLen(results.solutions, 1) self.assertIsNotNone(results.solutions[0].expression) self.assertGreater(results.solutions[0].weight, 0) self.assertGreater(results.total_time, 0) self.assertNotEmpty(results.value_set) self.assertGreaterEqual(results.statistics.total_apply_count, 0)
def test_run_value_search_works_for_important_benchmarks( self, benchmark_name, timeout, solution): # These benchmarks are frequently used in demos. We should ensure that they # are solved quickly and consistently. While the exact solution may change # as the algorithm improves, we should ensure that the solution is always # simple and understandable. handler = description_handler_factory.create_handler('tfidf') benchmark = all_benchmarks.find_benchmark_with_name(benchmark_name) results = value_search.run_value_search( benchmark=benchmark, description_handler=handler, settings=settings_module.from_dict({'timeout': timeout})) print('Time for benchmark {}: {:.2f} sec'.format( benchmark_name, results.total_time)) self.assertLen(results.solutions, 1) self.assertEqual(results.solutions[0].expression, solution)
def test_run_value_search_only_minimal_solutions(self, only_minimal_solutions, expected_solutions): benchmark = benchmark_module.Benchmark(examples=[ benchmark_module.Example(inputs=[[1, 4], [2, 7]], output=[3, 11]) ]) results = value_search.run_value_search( benchmark=benchmark, settings=settings_module.from_dict({ 'timeout': 20, 'max_solutions': 4, 'only_minimal_solutions': only_minimal_solutions, 'max_extra_solutions_time': 20 })) self.assertLen(results.solutions, len(expected_solutions)) self.assertEqual( [solution.expression for solution in results.solutions], expected_solutions)
def test_version(self): inputs = { 'rows': [10, 20, 30], 'cols': [1, 2, 3, 4], } output = [[11, 12, 13, 14], [21, 22, 23, 24], [31, 32, 33, 34]] constants = [] description = 'add two vectors with broadcasting to get a matrix' logging_dict = colab_logging.get_problem_logging_dict( inputs=inputs, output=output, constants=constants, description=description, settings=settings_module.from_dict({'timeout': 99}), include_in_dataset=False, problem_id=123) self.assertEqual(logging_dict['tf_coder_version'], tf_coder_version.__version__) self.assertEqual(logging_dict['logging_version'], logging_version.__version__)
def test_run_value_search_works_for_simple_benchmarks( self, simple_benchmark): handler = description_handler_factory.create_handler('tfidf') results = value_search.run_value_search( benchmark=simple_benchmark, settings=settings_module.from_dict({ 'timeout': 20, 'printing.statistics': True, 'require_all_inputs_used': False, 'require_one_input_used': False, }), description_handler=handler) self.assertLen(results.solutions, 1) self.assertIsNotNone(results.solutions[0].expression) self.assertGreater(results.solutions[0].weight, 0) self.assertGreater(results.total_time, 0) self.assertNotEmpty(results.value_set) self.assertGreaterEqual(results.statistics.total_apply_count, 0) # The simple benchmarks are so simple that we can directly write down the # correct (minimal) target program. self.assertEqual(results.solutions[0].expression, simple_benchmark.target_program)
def find_op(einsum_str, inputs, max_solutions=1, time_limit=300, n_samples=1): #print("Warm up...") colab_interface.warm_up() #print("Warm up done") #if n_samples == 1 : outputs = tf.einsum(einsum_str, *inputs) #else : # g_inputs = [inputs] # for i in range(n_samples): # g_inputs.append( #Tensor With Same Shape ) # outputs = [ tf.einsum(einsum_str,*inp) for inp in g_inputs ] settings = settings_module.from_dict({ 'timeout': time_limit, 'only_minimal_solutions': False, 'max_solutions': max_solutions, 'require_all_inputs_used': True, 'require_one_input_used': False, }) colab_interface.run_value_search_from_colab(inputs, outputs, [], "", settings)
def test_value_search_prioritizes_operations(self): # Make sure prioritized ops get printed out. settings = settings_module.from_dict({ 'timeout': 0.5, # We don't actually care about solving the problem. 'printing.prioritized_operations': True, 'printing.deprioritized_operations': True, 'tensor_model.prioritize_threshold': 0.5, 'tensor_model.deprioritize_threshold': 0.2, }) # Create data for the mocks. all_ops = all_operations.get_operations(include_sparse_operations=True) op_names = [op.name for op in all_ops] abs_index = None # Prioritized by tensor features model. argmax_index = None # Deprioritized by tensor features model. for i, op in enumerate(all_ops): if op.name == 'tf.abs(x)': abs_index = i elif op.name == 'tf.argmax(input, axis)': argmax_index = i self.assertIsNotNone(abs_index) self.assertIsNotNone(argmax_index) operation_probs = np.repeat(0.4, len(all_ops)) operation_probs[abs_index] = 0.6 # Above prioritize threshold. operation_probs[argmax_index] = 0.1 # Below deprioritize threshold. operation_logits = np.expand_dims(_inverse_sigmoid(operation_probs), axis=0) nl_data = [ # Associate "apple" with tf.zeros. { 'docstring': ['apple pie'], 'tf_functions': ['tf.zeros'], 'comments': [], 'names': [], 'strings': [] }, ] * 100 # Mock the tensor model's predictions, the NL model's data, and print(). with mock.patch('tf_coder.models.tensor_features_model.eval_single_example', return_value=tensor_features_model.Result( operation_logits=operation_logits)), \ mock.patch('tf_coder.datasets.github.data_loader.load_data', return_value=nl_data), \ mock.patch('sys.stdout', new_callable=six.StringIO) as mock_stdout: handler = bag_of_words_handlers.NaiveBayesDescriptionHandler( max_num_prioritized=1) benchmark = benchmark_module.Benchmark( # I/O example doesn't matter. examples=[ benchmark_module.Example({'my_var': [1, 2]}, [2, 1]) ], # Description contains "apple"! description='honeycrisp apple') value_search.run_value_search(benchmark=benchmark, description_handler=handler, settings=settings, tensor_model=mock.Mock(), tensor_config={ 'max_num_inputs': 3, 'operation_names': op_names }) self.assertIn('BOW handler prioritized tf.zeros', mock_stdout.getvalue()) self.assertIn('Tensor features model prioritized tf.abs(x), p=0.6', mock_stdout.getvalue()) self.assertIn( 'Tensor features model deprioritized tf.argmax(input, axis), p=0.1', mock_stdout.getvalue())