def test_benchmark_init_assigns_name_if_not_given(self): benchmark_1 = benchmark.Benchmark( [benchmark.Example(inputs=[1], output=1)]) benchmark_2 = benchmark.Benchmark( [benchmark.Example(inputs=[2], output=2)]) self.assertEqual(benchmark_1.name, 'unnamed_benchmark_1') self.assertEqual(benchmark_2.name, 'unnamed_benchmark_2')
def test_benchmark_init_computes_num_inputs(self): examples = [ benchmark.Example(inputs=[ [1], [2], ], output=[1, 2]), benchmark.Example(inputs=[ [[1, 2], [3, 4]], [[5, 6], [7, 8]], ], output=[[1, 2], [3, 4], [5, 6], [7, 8]]), ] self.assertEqual( benchmark.Benchmark(examples, name='test').num_inputs, 2)
class BenchmarkTest(parameterized.TestCase): @parameterized.named_parameters( ('no_examples', []), ('none_output', [benchmark.Example(inputs=[[1]], output=None)]), ( 'inconsistent_num_inputs', [ benchmark.Example(inputs=[[1]], output=[1]), # 1 input. benchmark.Example(inputs=[[1], [2]], output=[3]), # 2 inputs. ])) def test_benchmark_init_raises(self, examples): with self.assertRaises(ValueError): benchmark.Benchmark(examples, name='test') def test_benchmark_init_computes_num_inputs(self): examples = [ benchmark.Example(inputs=[ [1], [2], ], output=[1, 2]), benchmark.Example(inputs=[ [[1, 2], [3, 4]], [[5, 6], [7, 8]], ], output=[[1, 2], [3, 4], [5, 6], [7, 8]]), ] self.assertEqual( benchmark.Benchmark(examples, name='test').num_inputs, 2) def test_benchmark_init_assigns_name_if_not_given(self): benchmark_1 = benchmark.Benchmark( [benchmark.Example(inputs=[1], output=1)]) benchmark_2 = benchmark.Benchmark( [benchmark.Example(inputs=[2], output=2)]) self.assertEqual(benchmark_1.name, 'unnamed_benchmark_1') self.assertEqual(benchmark_2.name, 'unnamed_benchmark_2') def test_benchmark_not_ignored_by_default(self): not_ignored_list = [ test_benchmarks.test_add(), test_benchmarks.test_cast() ] for not_ignored in not_ignored_list: self.assertFalse(not_ignored.should_ignore) self.assertIsNone(not_ignored.ignore_reason) def test_ignore_decorator(self): ignored_benchmark = test_benchmarks.inconsistent_target_program() self.assertTrue(ignored_benchmark.should_ignore) self.assertIsNotNone(ignored_benchmark.ignore_reason)
def run_value_search_from_example( inputs: Union[List[Any], Dict[Text, Any]], output: Any, settings: Optional[settings_module.Settings] = None, **kwargs) -> ValueSearchResults: """Performs value search for a single user-provided input-output example. Args: inputs: A list of inputs, or a dict mapping input names to inputs. output: The corresponding desired output. settings: An optional Settings object to use, or None to use defaults. **kwargs: The kwarg 'constants' can be used to specify a list of constants, and 'description' can be used to provide a natural language description of the task. Other arguments are passed directly to run_value_search(). Returns: A ValueSearchResults namedtuple. """ if settings is None: settings = settings_module.default_settings() constants = kwargs.pop('constants', None) description = kwargs.pop('description', None) source = kwargs.pop('source', 'From user-provided example.') benchmark = benchmark_module.Benchmark( examples=[benchmark_module.Example(inputs, output)], constants=constants, # Will turn into empty list if constants=None. description=description, # Will turn into '' if description=None. source=source) return run_value_search(benchmark, settings, **kwargs)
def test_get_operation_multipliers_respects_min_tfidf_score( self, min_tfidf_score): handler = tfidf_handler.TfidfDescriptionHandler( max_num_prioritized=1000000, min_tfidf_score=min_tfidf_score) description = 'Tile a tensor multiple times' scores = handler.score_description(description) benchmark = benchmark_module.Benchmark(examples=[ benchmark_module.Example( inputs=[ [10], [20], ], output=[30], ), ], constants=[], description=description, target_program='', source='test', name='test_benchmark') multipliers = handler.get_operation_multipliers( benchmark, settings=settings_module.default_settings()) prioritized_names = [ name for name in multipliers.keys() if multipliers[name] < 1 ] expected_prioritized_names = [ name for name in scores.keys() if scores[name] >= min_tfidf_score ] self.assertCountEqual(prioritized_names, expected_prioritized_names) # All multipliers must be in (0, 1] (no operation is deprioritized). self.assertTrue( all(0 < multiplier <= 1 for multiplier in multipliers.values()))
def google_03(): examples = [ benchmark.Example(inputs=[ tf.SparseTensor(indices=[[0, 0, 0], [0, 1, 1], [1, 1, 1], [1, 1, 2]], values=[1., 1., 1., 1.], dense_shape=[2, 2, 800]), ], output=tf.SparseTensor(indices=[[0, 0, 0], [0, 1, 1]], values=[1., 1.], dense_shape=[1, 2, 800])), ] constants = [] description = 'Slice the first dimension of a SparseTensor' # Original approach, which can't be written as one expression. """ reduced_sp_tensor = tf.sparse_retain(in1, tf.equal(in1.indices[:, 0], 0)) output = tf.SparseTensor(indices=reduced_sp_tensor.indices, values=reduced_sp_tensor.values, dense_shape=tf.concat([tf.ones(1, dtype=tf.int64), in1.dense_shape[1:]], 0)) """ # pylint: disable=pointless-string-statement target_program = 'tf.sparse.slice(in1, tf.zeros(3, dtype=tf.int64), tf.concat([[1], in1.dense_shape[1:]], 0))' source = 'Real task encountered by Googler, 10/16/2018' return benchmark.Benchmark(examples=examples, constants=constants, description=description, target_program=target_program, source=source, name='google_03')
def test_get_operation_multiplier_respects_max_num_prioritized( self, max_num_prioritized): handler = tfidf_handler.TfidfDescriptionHandler( max_num_prioritized=max_num_prioritized, min_tfidf_score=0) benchmark = benchmark_module.Benchmark( examples=[ benchmark_module.Example( inputs=[ [10], [20], ], output=[30], ), ], constants=[], description='Tile a tensor multiple times', target_program='', source='test', name='test_benchmark') multipliers = handler.get_operation_multipliers( benchmark, settings=settings_module.default_settings()) actual_num_prioritized = sum(multiplier < 1 for multiplier in multipliers.values()) self.assertEqual(actual_num_prioritized, max_num_prioritized) # All multipliers must be in (0, 1] (no operation is deprioritized). self.assertTrue( all(0 < multiplier <= 1 for multiplier in multipliers.values()))
def autopandas_03(): # The "series" and "step" columns only serve to identify where the data should # move to. The number of unique "series" and "step" values determine the # output's shape, so we provide them (3 and 5) as constants. The real data is # in the "value" column, so it's a 1D tensor. This is simply a tensor reshape # and transpose. examples = [ benchmark.Example(inputs=[ [ 1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1012, 1013, 1014 ], ], output=[[1000, 1003, 1006, 1009, 1012], [1001, 1004, 1007, 1010, 1013], [1002, 1005, 1008, 1011, 1014]]), ] constants = [3, 5] description = '' # No description for AutoPandas benchmarks. target_program = 'tf.transpose(tf.reshape(in1, (5, 3)))' source = 'SO_13647222_depth1' return benchmark.Benchmark(examples=examples, constants=constants, description=description, target_program=target_program, source=source, name='autopandas_03')
def stackoverflow_18(): examples = [ benchmark.Example( inputs=[ # shape=[2, 2, 3]. [[[1, 1, 1], [1, 0, 1]], [[1, 2, 3], [4, 5, 6]]], # shape=[3, 4]. [[1, 1, 1, 1], [1, 2, 3, 4], [5, 6, 7, 8]], # shape=[4]. [100, 200, 300, 400], ], # Shape=[sequence_length, batch_size, 4]=[2, 2, 4]. output=[[[107, 209, 311, 413], [106, 207, 308, 409]], [[118, 223, 328, 433], [139, 250, 361, 472]]]), ] constants = [] description = 'multiply 3D tensor and 2D tensor and add another tensor' target_program = 'tf.add(in3, tf.matmul(in1, in2))' source = 'https://stackoverflow.com/questions/38222126/tensorflow-efficient-way-for-tensor-multiplication' return benchmark.Benchmark(examples=examples, constants=constants, description=description, target_program=target_program, source=source, name='stackoverflow_18')
def autopandas_05(): # The problem is to sort a table using values in one column. Their code that # constructs the output actually sorts by 3 columns, but their example has # unique values for the first column, so the other two columns don't affect # the result at all. Instead of having dates and strings and numbers, we just # use numbers. Sort by column 0 in increasing order. examples = [ benchmark.Example(inputs=[ [[6, 3, 7, 8, 4], [8, 9, 4, 5, 3], [1, 5, 3, 6, 9], [2, 1, 4, 3, 2], [7, 9, 6, 2, 7], [5, 8, 0, 4, 2]], ], output=[[1, 5, 3, 6, 9], [2, 1, 4, 3, 2], [5, 8, 0, 4, 2], [6, 3, 7, 8, 4], [7, 9, 6, 2, 7], [8, 9, 4, 5, 3]]), ] constants = [] description = '' # No description for AutoPandas benchmarks. target_program = 'tf.gather(in1, tf.argsort(in1[:, 0], stable=True))' source = 'SO_49583055_depth1' return benchmark.Benchmark(examples=examples, constants=constants, description=description, target_program=target_program, source=source, name='autopandas_05')
def autopandas_14(): # The intended solution is to do a merge, but the example actually has nothing # to merge, so all that happens is an extra column is created filled with NaN. # Why are there so many problems where the example completely underspecifies # the intended transformation? Instead of ignoring the problem because it has # a merge (out of scope), let's just try to append a NaN column. examples = [ benchmark.Example( inputs=[ [[1, 0, 1, 2], [1, 1, 3, 4], [2, 0, 1, 2], [2, 1, 3, 4]], # They have another input that is used to provide the column # header for the new NaN-filled column. Tensors don't have column # headers so this input is useless. Omit it, or else TF-Coder will # be required to use it. ], output=[[1, 0, 1, 2, float('nan')], [1, 1, 3, 4, float('nan')], [2, 0, 1, 2, float('nan')], [2, 1, 3, 4, float('nan')]]), ] constants = [float('nan')] description = '' # No description for AutoPandas benchmarks. target_program = 'tf.concat((tf.cast(in1, tf.float32), tf.fill([4, 1], float("nan"))), axis=1)' source = 'SO_13576164_depth3' return benchmark.Benchmark(examples=examples, constants=constants, description=description, target_program=target_program, source=source, name='autopandas_14')
def stackoverflow_47(): examples = [ benchmark.Example(inputs=[ [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], [[True, True, True, False, False], [True, True, False, False, False], [True, True, True, True, True], [True, True, True, True, False], [True, False, False, False, False], [True, True, False, False, False]], ], output=[[0, 1, 2, 0, 0], [3, 4, 0, 0, 0], [5, 6, 7, 8, 9], [10, 11, 12, 13, 0], [14, 0, 0, 0, 0], [15, 16, 0, 0, 0]]), ] constants = [] description = 'put given values into a sequence mask' # Note: StackOverflow answer is wrong (gather index out of bounds). # Solution is simpler with an auxiliary variable: # linear_mask = tf.reshape(tf.cast(in2, tf.int32), [-1]) # output = tf.reshape(tf.gather(in1, tf.cumsum(linear_mask, exclusive=True) * linear_mask), in2.shape) target_program = 'tf.reshape(tf.gather(in1, tf.cumsum(tf.reshape(tf.cast(in2, tf.int32), [-1]), exclusive=True) * tf.reshape(tf.cast(in2, tf.int32), [-1])), in2.shape)' source = 'https://stackoverflow.com/questions/58641546/how-can-i-put-the-sequential-values-to-the-sequence-mask' return benchmark.Benchmark(examples=examples, constants=constants, description=description, target_program=target_program, source=source, name='stackoverflow_47')
def stackoverflow_30(): examples = [ benchmark.Example( inputs=[ [[1., 2.], [3., 4.], [5., 6.]], [[9., 4.], [8., 5.], [7., 6.]], ], output=[[math.sqrt(68), math.sqrt(58), math.sqrt(52)], [math.sqrt(36), math.sqrt(26), math.sqrt(20)], [math.sqrt(20), math.sqrt(10), math.sqrt(4)]]), ] constants = [] description = 'compute Euclidean distance between two tensors' # StackOverflow answer is incorrect. target_program = 'tf.sqrt(tf.reduce_sum(tf.square(tf.subtract(tf.expand_dims(in1, 1), tf.expand_dims(in2, 0))), axis=2))' source = 'https://stackoverflow.com/questions/54147780/tensorflow-how-to-calculate-the-euclidean-distance-between-two-tensor' return benchmark.Benchmark(examples=examples, constants=constants, description=description, target_program=target_program, source=source, name='stackoverflow_30')
def simple_sparse_add(): """Adding a sparse and a dense tensor to produce a sparse tensor.""" examples = [ benchmark.Example( inputs=[ tf.SparseTensor(indices=[[0, 0], [0, 1]], values=[12, 34], dense_shape=[2, 2]), [[-3, 0], [-5, 0]], ], output=tf.SparseTensor(indices=[[0, 0], [0, 1], [1, 0]], values=[9, 34, -5], dense_shape=[2, 2]), ), ] constants = [] description = 'Add sparse tensor with dense tensor' target_program = 'tf.sparse.add(in1, tf.sparse.from_dense(in2))' source = 'handwritten task' return benchmark.Benchmark(examples=examples, constants=constants, description=description, target_program=target_program, source=source, name='simple_sparse_add')
def test_add_constants_and_inputs_and_print_checks_names(self, bad_name): values_by_weight = [collections.OrderedDict() for _ in range(100)] examples = [ benchmark_module.Example(inputs={bad_name: [1, 2, 3]}, output=[1, 2]) ] benchmark = benchmark_module.Benchmark(examples, constants=[]) output_value = value_module.OutputValue(benchmark.examples[0].output) with self.assertRaises(ValueError): value_search._add_constants_and_inputs_and_print( values_by_weight, benchmark, output_value, self._constant_operation, self.settings)
def autopandas_02(): """AutoPandas benchmark. The DataFrame input: value1 value2 group1 group2 a c 1.1 7.1 c 2.0 8.0 d 3.0 9.0 b d 4.0 10.0 d 5.0 11.0 e 6.0 12.0 The DataFrame output: value1 value2 group2 c 1.1 7.1 c 2.0 8.0 d 3.0 9.0 Notice that "c", "d", and "e" are just treated as data, so we'll replace them with 1, 2, and 3, respectively. "group1" acts as a third dimension, so our input tensor will be 3D. We'll also make "group1" the innermost axis, to make the problem not super trivial in TensorFlow. Finally, I changed some numbers to rule out tf.reduce_min(axis=2). """ examples = [ benchmark.Example( inputs=[ [ [[1, 2], [1, 2], [2, 3]], # group2 [[1.1, 4], [2, 1], [3, 6]], # value1 [[7.1, 10], [8, 11], [9, 7]] ], # value2 ], output=[[1, 1, 2], [1.1, 2, 3], [7.1, 8, 9]]), ] constants = [] description = '' # No description for AutoPandas benchmarks. # Note that we don't have ops for indexing/slicing axis 2. But, TF-Coder will # find a workaround. target_program = 'in1[:, :, 0]' source = 'SO_11941492_depth1' return benchmark.Benchmark(examples=examples, constants=constants, description=description, target_program=target_program, source=source, name='autopandas_02')
def stackoverflow_12(): examples = [ benchmark.Example(inputs=[[[12, 34, 56], [33, 22, 11]]], output=[[12, 56], [33, 11]]), ] constants = [0, 1, 2] description = 'remove a column from the tensor' target_program = 'tf.gather(in1, (0, 2), axis=1, batch_dims=0)' source = 'https://stackoverflow.com/questions/47447183/remove-a-set-of-tensors-from-a-tensor-in-tensorflow' return benchmark.Benchmark(examples=examples, constants=constants, description=description, target_program=target_program, source=source, name='stackoverflow_12')
def test_run_value_handles_large_weight_constants(self): benchmark = benchmark_module.Benchmark(examples=[ benchmark_module.Example(inputs=[[1], [2]], output=[[3]]) ]) results = value_search.run_value_search(benchmark=benchmark, settings=self.settings) self.assertNotEmpty(results.solutions) self.assertEqual(results.solutions[0].expression, 'tf.add(in1, tf.expand_dims(in2, 0))') output_shape_constant = value_module.ConstantValue((1, 1)) self.assertIn(output_shape_constant, results.value_set) # Find the element in value_set equal to output_shape_constant and assert # that it's actually a ConstantValue, as opposed to an OperationValue. for value in results.value_set: if value == output_shape_constant: self.assertIsInstance(value, value_module.ConstantValue)
def stackoverflow_26(): examples = [ benchmark.Example(inputs=[[[[3, 4], [1, 2]], [[5, -2], [-10, 3]], [[10, 20], [-4, 7]]]], output=[10, -4, 33]), ] constants = [] description = 'reduction operation for multiple dimensions simultaneously' target_program = 'tf.reduce_sum(tf.reduce_sum(in1, axis=1), axis=1)' source = 'https://stackoverflow.com/questions/54294780/how-to-perform-reduce-op-on-multiple-dimensions-at-once' return benchmark.Benchmark(examples=examples, constants=constants, description=description, target_program=target_program, source=source, name='stackoverflow_26')
def stackoverflow_42(): examples = [ benchmark.Example(inputs=[ [4, 6, 2, 6, 7, 3, -3], ], output=[0, 0, 0, 0, 1, 0, 0]), ] constants = [] description = 'create a binary vector where the max element is 1' target_program = 'tf.cast(tf.equal(in1, tf.reduce_max(in1)), tf.int32)' source = 'https://stackoverflow.com/questions/54493814/binary-vector-of-max' return benchmark.Benchmark(examples=examples, constants=constants, description=description, target_program=target_program, source=source, name='stackoverflow_42')
def stackoverflow_11(): examples = [ benchmark.Example(inputs=[ [4, 0, 1, 1, 0, 4, 0, 0, 3, 4, 1], ], output=[4, 3, 0, 1, 3]), ] constants = [] description = 'count the number of occurences of each distinct number' target_program = 'tf.math.bincount(in1)' source = 'https://stackoverflow.com/questions/45194672/how-to-count-elements-in-tensorflow-tensor' return benchmark.Benchmark(examples=examples, constants=constants, description=description, target_program=target_program, source=source, name='stackoverflow_11')
def autopandas_06(): # This is very similar to autopandas_03, they're both pivot tables. examples = [ benchmark.Example(inputs=[ [4, 5, 6, 7], ], output=[[4, 5], [6, 7]]), ] constants = [2] description = '' # No description for AutoPandas benchmarks. target_program = 'tf.reshape(in1, (2, 2))' source = 'SO_13261175_depth1' return benchmark.Benchmark(examples=examples, constants=constants, description=description, target_program=target_program, source=source, name='autopandas_06')
def autopandas_06_ignored(): """AutoPandas benchmark. The input DataFrame: X Y Z 4 X1 Y2 Z3 5 X1 Y1 Z1 6 X1 Y1 Z1 7 X1 Y1 Z2 The output DataFrame: Z Z1 Z2 Z3 Y Y1 1.0 1.0 NaN Y2 NaN NaN 1.0 Basically, the Ys are row indices and the Zs are column indices. The X1 does not really matter, we just want a boolean output table (1.0=True, NaN=False) that identifies which Y and Z pairs appeared in the input table. Their example is tiny so I expanded it. This task also doesn't appear in their table of results? """ examples = [ benchmark.Example( inputs=[ # Y Z [[1, 2], [0, 0], [0, 0], [0, 1], [4, 2], [4, 3], [4, 2], [2, 1]], ], output=[[True, True, False, False], [False, False, True, False], [False, True, False, False], [False, False, False, False], [False, False, True, True]]), ] constants = [] description = '' # No description for AutoPandas benchmarks. # TODO(kshi): Note that tf.scatter_nd is not yet supported by TF-Coder!! target_program = 'tf.cast(tf.scatter_nd(in1, updates=tf.ones(8), shape=[5, 4]), tf.bool)' source = 'SO_12860421_depth1' return benchmark.Benchmark(examples=examples, constants=constants, description=description, target_program=target_program, source=source, name='autopandas_06_ignored')
def stackoverflow_15(): examples = [ benchmark.Example(inputs=[ [3, 1, 2, 0, 1, -1, 10, 1, -10], ], output=[3, 0, 2, 0, 0, -1, 10, 0, -10]), ] constants = [0, 1] description = 'set all instances of 1 to 0' target_program = 'tf.subtract(in1, tf.cast(tf.equal(in1, tf.constant(1)), tf.int32))' source = 'https://stackoverflow.com/questions/39045797/conditional-assignment-of-tensor-values-in-tensorflow' return benchmark.Benchmark(examples=examples, constants=constants, description=description, target_program=target_program, source=source, name='stackoverflow_15')
def stackoverflow_02(): examples = [ benchmark.Example(inputs=[ [5, 1, 0, 3, 0, -1, 2, -10, 2], ], output=[1, 1, 0, 1, 0, -1, 1, -10, 1]), ] constants = [1] description = 'clip values that are greater than 1' target_program = 'tf.minimum(in1, tf.constant(1))' source = 'https://stackoverflow.com/questions/46408839/tensorflow-trim-values-in-tensor' return benchmark.Benchmark(examples=examples, constants=constants, description=description, target_program=target_program, source=source, name='stackoverflow_02')
def google_20(): examples = [ benchmark.Example(inputs=[ [10, 7, 4, 3, 2, 8], ], output=[5, 3, 2, 1, 0, 4]), ] constants = [] description = 'sort a tensor and return sorted index in original order' target_program = 'tf.cast(tf.argsort(tf.argsort(in1)), tf.int32)' source = 'From an internal Google forum' return benchmark.Benchmark(examples=examples, constants=constants, description=description, target_program=target_program, source=source, name='google_20')
def google_15(): examples = [ benchmark.Example(inputs=[ [[1, 3, 5, 7], [2, 4, 6, 8]], ], output=[[1, 3, 5, 7, 0], [2, 4, 6, 8, 0]]), ] constants = [] description = 'pad a zero column' target_program = "tf.pad(in1, [[0, 0], [0, 1]], 'CONSTANT')" source = 'Real task encountered by Googler, 10/23/2019' return benchmark.Benchmark(examples=examples, constants=constants, description=description, target_program=target_program, source=source, name='google_15')
def simple_using_constant(): """A benchmark that requires a user-provided constant.""" examples = [ benchmark.Example(inputs=[ [1, 2, 3], ], output=[101, 102, 103]), ] constants = [100] description = 'Add 100 to every element' target_program = 'tf.add(in1, tf.constant(100))' source = 'handwritten task' return benchmark.Benchmark(examples=examples, constants=constants, description=description, target_program=target_program, source=source, name='simple_using_constant')
def stackoverflow_32(): examples = [ benchmark.Example(inputs=[ [[0.1, 0.6, 0.2, 0.1], [0.3, 0.1, 0.4, 0.2], [0.2, 0.1, 0.2, 0.5]], ], output=[1.3, 1.5, 2.0]), ] constants = [] description = 'weighted sum across rows, where the column index is the weight' target_program = 'tf.tensordot(in1, tf.cast(tf.range(4), tf.float32), 1)' source = 'https://stackoverflow.com/questions/48659449/how-to-compute-the-weighted-sum-of-a-tensor-in-tensorflow' return benchmark.Benchmark(examples=examples, constants=constants, description=description, target_program=target_program, source=source, name='stackoverflow_32')
def stackoverflow_46(): examples = [ benchmark.Example(inputs=[ [3, 4, 1], ], output=[0, 0, 0, 1, 1, 1, 1, 2]), ] constants = [] description = 'convert segment lengths to segment ids' target_program = 'tf.cast(tf.where(tf.sequence_mask(in1))[:, 0], tf.int32)' source = 'https://stackoverflow.com/questions/58652161/how-to-convert-2-3-4-to-0-0-1-1-1-2-2-2-2-to-utilize-tf-math-segment-sum' return benchmark.Benchmark(examples=examples, constants=constants, description=description, target_program=target_program, source=source, name='stackoverflow_46')