def test_get_operation_multipliers_respects_min_tfidf_score( self, min_tfidf_score): handler = tfidf_handler.TfidfDescriptionHandler( max_num_prioritized=1000000, min_tfidf_score=min_tfidf_score) description = 'Tile a tensor multiple times' scores = handler.score_description(description) benchmark = benchmark_module.Benchmark(examples=[ benchmark_module.Example( inputs=[ [10], [20], ], output=[30], ), ], constants=[], description=description, target_program='', source='test', name='test_benchmark') multipliers = handler.get_operation_multipliers( benchmark, settings=settings_module.default_settings()) prioritized_names = [ name for name in multipliers.keys() if multipliers[name] < 1 ] expected_prioritized_names = [ name for name in scores.keys() if scores[name] >= min_tfidf_score ] self.assertCountEqual(prioritized_names, expected_prioritized_names) # All multipliers must be in (0, 1] (no operation is deprioritized). self.assertTrue( all(0 < multiplier <= 1 for multiplier in multipliers.values()))
def test_get_operation_multiplier_respects_max_num_prioritized( self, max_num_prioritized): handler = tfidf_handler.TfidfDescriptionHandler( max_num_prioritized=max_num_prioritized, min_tfidf_score=0) benchmark = benchmark_module.Benchmark( examples=[ benchmark_module.Example( inputs=[ [10], [20], ], output=[30], ), ], constants=[], description='Tile a tensor multiple times', target_program='', source='test', name='test_benchmark') multipliers = handler.get_operation_multipliers( benchmark, settings=settings_module.default_settings()) actual_num_prioritized = sum(multiplier < 1 for multiplier in multipliers.values()) self.assertEqual(actual_num_prioritized, max_num_prioritized) # All multipliers must be in (0, 1] (no operation is deprioritized). self.assertTrue( all(0 < multiplier <= 1 for multiplier in multipliers.values()))
def run_value_search_from_example( inputs: Union[List[Any], Dict[Text, Any]], output: Any, settings: Optional[settings_module.Settings] = None, **kwargs) -> ValueSearchResults: """Performs value search for a single user-provided input-output example. Args: inputs: A list of inputs, or a dict mapping input names to inputs. output: The corresponding desired output. settings: An optional Settings object to use, or None to use defaults. **kwargs: The kwarg 'constants' can be used to specify a list of constants, and 'description' can be used to provide a natural language description of the task. Other arguments are passed directly to run_value_search(). Returns: A ValueSearchResults namedtuple. """ if settings is None: settings = settings_module.default_settings() constants = kwargs.pop('constants', None) description = kwargs.pop('description', None) source = kwargs.pop('source', 'From user-provided example.') benchmark = benchmark_module.Benchmark( examples=[benchmark_module.Example(inputs, output)], constants=constants, # Will turn into empty list if constants=None. description=description, # Will turn into '' if description=None. source=source) return run_value_search(benchmark, settings, **kwargs)
def test_apply_for_tensor(self, index, expected): tensor = tf.constant([[1.2, 3.4], [5.6, 7.8]]) arg_values = [value.ConstantValue(tensor), value.ConstantValue(index)] result_value = self.operation.apply(arg_values, settings_module.default_settings()) self.assertEqual(result_value, value.ConstantValue(tf.constant(expected)))
def setUp(self): super(OperationValueTest, self).setUp() function_info = tf_functions.FunctionInfo( name='tf.reduce_max(input_tensor, axis)', filter_group=filter_group.FilterGroup.NONE, weight=1) self.operation = function_operation.FunctionOperation(function_info) self.settings = settings_module.default_settings()
def test_apply_for_list(self, index, expected): arg_values = [ value.ConstantValue([1.2, 3.4, 5.6]), value.ConstantValue(index) ] result_value = self.operation.apply(arg_values, settings_module.default_settings()) self.assertEqual(result_value, value.ConstantValue(expected))
def test_get_operation_multipliers(self, description, operation_name, expected_multiplier): handler = description_handler.FunctionNameDescriptionHandler() multipliers = handler.get_operation_multipliers( description, settings_module.default_settings()) self.assertEqual(multipliers.get(operation_name, 1.0), expected_multiplier) # Nothing is deprioritized. self.assertTrue(all(0 < value < 1 for value in multipliers.values()))
def test_get_operation_multiplier_respects_max_num_prioritized( self, max_num_prioritized): handler = tfidf_handler.TfidfDescriptionHandler( max_num_prioritized=max_num_prioritized, min_tfidf_score=0) multipliers = handler.get_operation_multipliers( 'Tile a tensor multiple times', settings=settings_module.default_settings()) actual_num_prioritized = sum(multiplier < 1 for multiplier in multipliers.values()) self.assertEqual(actual_num_prioritized, max_num_prioritized) # All multipliers must be in (0, 1] (no operation is deprioritized). self.assertTrue( all(0 < multiplier <= 1 for multiplier in multipliers.values()))
def setUp(self): super(CollectTensorDataTest, self).setUp() self.settings = settings_module.default_settings() operations = all_operations.get_operations() self.unique_with_counts_operation = all_operations.find_operation_with_name( 'tf.unique_with_counts(x)', operation_list=operations) self.indexing_operation = all_operations.find_operation_with_name( 'IndexingOperation', operation_list=operations) self.gather_operation = all_operations.find_operation_with_name( 'tf.gather(params, indices)', operation_list=operations) self.add_operation = all_operations.find_operation_with_name( 'tf.add(x, y)', operation_list=operations) # Example with many operations. in1 = value_module.InputValue([1, 1, 2, 5, 6, 5], 'in1') in2 = value_module.InputValue([0, 10, 20, 30, 40, 50, 60, 70], 'in2') constant_1 = value_module.ConstantValue(1) unique = self.unique_with_counts_operation.apply([in1], self.settings) indexed = self.indexing_operation.apply([unique, constant_1], self.settings) gathered = self.gather_operation.apply([in2, in1], self.settings) self.example_value_1 = self.add_operation.apply([indexed, gathered], self.settings) self.assertEqual( self.example_value_1.reconstruct_expression(), 'tf.add(tf.unique_with_counts(in1)[1], tf.gather(in2, in1))') self.assertEqual(self.example_value_1, value_module.OutputValue([10, 10, 21, 52, 63, 52])) # Example with many variables and new inputs. in3 = value_module.InputValue([1], 'in3') in4 = value_module.InputValue([2], 'in4') a = self.add_operation.apply([in3, new_input([10])], self.settings) b = self.add_operation.apply([in4, in3], self.settings) c = self.add_operation.apply([new_input([20]), in3], self.settings) d = self.add_operation.apply([a, b], self.settings) self.example_value_2 = self.add_operation.apply([c, d], self.settings) self.assertEqual( self.example_value_2.reconstruct_expression(), 'tf.add(tf.add(NEW_INPUT, in3), ' 'tf.add(tf.add(in3, NEW_INPUT), tf.add(in4, in3)))') self.assertEqual(self.example_value_2, value_module.OutputValue([35]))
def test_get_operation_multipliers_respects_min_tfidf_score( self, min_tfidf_score): handler = tfidf_handler.TfidfDescriptionHandler( max_num_prioritized=1000000, min_tfidf_score=min_tfidf_score) description = 'Tile a tensor multiple times' scores = handler.score_description(description) multipliers = handler.get_operation_multipliers( description, settings=settings_module.default_settings()) prioritized_names = [ name for name in multipliers.keys() if multipliers[name] < 1 ] expected_prioritized_names = [ name for name in scores.keys() if scores[name] >= min_tfidf_score ] self.assertCountEqual(prioritized_names, expected_prioritized_names) # All multipliers must be in (0, 1] (no operation is deprioritized). self.assertTrue( all(0 < multiplier <= 1 for multiplier in multipliers.values()))
def test_get_operation_multiplier_respects_max_num_prioritized( self, max_num_prioritized): # Note: More operations may be prioritized if they are chosen because they # are explicitly listed in the description. handler = tfidf_handler.TfidfDescriptionHandler( max_num_prioritized=max_num_prioritized, min_tfidf_score=0) benchmark = benchmark_module.Benchmark( examples=[benchmark_module.Example(inputs=[1], output=1)], description='Tiling a tensor multiple times') multipliers = handler.get_operation_multipliers( benchmark, settings=settings_module.default_settings()) actual_num_prioritized = sum(multiplier < 1 for multiplier in multipliers.values()) self.assertEqual(actual_num_prioritized, max_num_prioritized) # All multipliers must be in (0, 1] (no operation is deprioritized). self.assertTrue(all(0 < multiplier <= 1 for multiplier in multipliers.values()))
def test_get_operation_multipliers(self, description, operation_name, expected_multiplier): handler = description_handler.FunctionNameDescriptionHandler() benchmark = benchmark_module.Benchmark(examples=[ benchmark_module.Example( inputs=[ [10], [20], ], output=[30], ), ], constants=[], description=description, target_program='', source='test', name='test_benchmark') multipliers = handler.get_operation_multipliers( benchmark, settings_module.default_settings()) self.assertEqual(multipliers.get(operation_name, 1.0), expected_multiplier) # Nothing is deprioritized. self.assertTrue(all(0 < value < 1 for value in multipliers.values()))
def test_get_operation_multipliers_respects_min_tfidf_score( self, min_tfidf_score): # Note: An operation may violate the `min_tfidf_score` threshold if it is # chosen because it is explicitly listed in the description. handler = tfidf_handler.TfidfDescriptionHandler( max_num_prioritized=1000000, min_tfidf_score=min_tfidf_score) description = 'Tiling a tensor multiple times' scores = handler.score_description(description) benchmark = benchmark_module.Benchmark( examples=[benchmark_module.Example(inputs=[1], output=1)], description=description) multipliers = handler.get_operation_multipliers( benchmark, settings=settings_module.default_settings()) prioritized_names = [name for name in multipliers.keys() if multipliers[name] < 1] expected_prioritized_names = [name for name in scores.keys() if scores[name] >= min_tfidf_score] self.assertCountEqual(prioritized_names, expected_prioritized_names) # All multipliers must be in (0, 1] (no operation is deprioritized). self.assertTrue(all(0 < multiplier <= 1 for multiplier in multipliers.values()))
def test_as_dict_default_works_with_json(self): settings = settings_module.default_settings() as_dict = settings.as_dict() self.assertFalse(as_dict['printing.verbose']) # Test that all defaults are JSON-serializable. self.assertNotEmpty(json.dumps(as_dict))
def setUp(self): super(NoChangeDescriptionHandlerTest, self).setUp() self.indexing_operation = python_operations.IndexingOperation() self.settings = settings_module.default_settings()
def test_apply(self): self.assertEqual( self.operation.apply(self.arg_values, settings_module.default_settings()), value.OutputValue([34, 78]))
'what happened.') # TODO(kshi): Add a link to GitHub. # The global description handler for this session. This will be loaded during # warm_up(). DESCRIPTION_HANDLER = None # The tensor features model and config. These will be loaded during warm_up(). TENSOR_MODEL = None TENSOR_CONFIG = None # Whether warm_up() was called before. WARMED_UP = False # Default settings, if the user doesn't provide their own settings. DEFAULT_SETTINGS = settings_module.default_settings() def warm_up(): """Loads and warms up the tensor features model.""" global DESCRIPTION_HANDLER, TENSOR_MODEL, TENSOR_CONFIG, WARMED_UP if WARMED_UP: return WARMED_UP = True # Use the default choices for the description handler and tensor model. The # Colab interface will not allow users to change these. DESCRIPTION_HANDLER = description_handler_factory.create_handler( DEFAULT_SETTINGS.description_handler_name) if (not DEFAULT_SETTINGS.tensor_model.config_path or
def test_apply(self): self.assertEqual( self.operation.apply(self.arg_values, settings_module.default_settings()), value.ConstantValue((12, )))
def setUp(self): super(FunctionOperationTest, self).setUp() self.settings = settings_module.default_settings()
def setUp(self): super(OperationBaseTest, self).setUp() self.operation = StrangeAdditionOperation() self.settings = settings_module.default_settings()