def test_benchmark_init_assigns_name_if_not_given(self):
benchmark_1 = benchmark.Benchmark(
[benchmark.Example(inputs=[1], output=1)])
benchmark_2 = benchmark.Benchmark(
[benchmark.Example(inputs=[2], output=2)])
self.assertEqual(benchmark_1.name, 'unnamed_benchmark_1')
self.assertEqual(benchmark_2.name, 'unnamed_benchmark_2')
def run_value_search_from_example(
inputs: Union[List[Any], Dict[Text, Any]],
output: Any,
settings: Optional[settings_module.Settings] = None,
**kwargs) -> ValueSearchResults:
"""Performs value search for a single user-provided input-output example.
Args:
inputs: A list of inputs, or a dict mapping input names to inputs.
output: The corresponding desired output.
settings: An optional Settings object to use, or None to use defaults.
**kwargs: The kwarg 'constants' can be used to specify a list of constants,
and 'description' can be used to provide a natural language description of
the task. Other arguments are passed directly to run_value_search().
Returns:
A ValueSearchResults namedtuple.
"""
if settings is None:
settings = settings_module.default_settings()
constants = kwargs.pop('constants', None)
description = kwargs.pop('description', None)
source = kwargs.pop('source', 'From user-provided example.')
benchmark = benchmark_module.Benchmark(
examples=[benchmark_module.Example(inputs, output)],
constants=constants, # Will turn into empty list if constants=None.
description=description, # Will turn into '' if description=None.
source=source)
return run_value_search(benchmark, settings, **kwargs)
def test_get_operation_multipliers_respects_min_tfidf_score(
self, min_tfidf_score):
handler = tfidf_handler.TfidfDescriptionHandler(
max_num_prioritized=1000000, min_tfidf_score=min_tfidf_score)
description = 'Tile a tensor multiple times'
scores = handler.score_description(description)
benchmark = benchmark_module.Benchmark(examples=[
benchmark_module.Example(
inputs=[
[10],
[20],
],
output=[30],
),
],
constants=[],
description=description,
target_program='',
source='test',
name='test_benchmark')
multipliers = handler.get_operation_multipliers(
benchmark, settings=settings_module.default_settings())
prioritized_names = [
name for name in multipliers.keys() if multipliers[name] < 1
]
expected_prioritized_names = [
name for name in scores.keys() if scores[name] >= min_tfidf_score
]
self.assertCountEqual(prioritized_names, expected_prioritized_names)
# All multipliers must be in (0, 1] (no operation is deprioritized).
self.assertTrue(
all(0 < multiplier <= 1 for multiplier in multipliers.values()))
def test_get_operation_multiplier_respects_max_num_prioritized(
self, max_num_prioritized):
handler = tfidf_handler.TfidfDescriptionHandler(
max_num_prioritized=max_num_prioritized, min_tfidf_score=0)
benchmark = benchmark_module.Benchmark(
examples=[
benchmark_module.Example(
inputs=[
[10],
[20],
],
output=[30],
),
],
constants=[],
description='Tile a tensor multiple times',
target_program='',
source='test',
name='test_benchmark')
multipliers = handler.get_operation_multipliers(
benchmark, settings=settings_module.default_settings())
actual_num_prioritized = sum(multiplier < 1
for multiplier in multipliers.values())
self.assertEqual(actual_num_prioritized, max_num_prioritized)
# All multipliers must be in (0, 1] (no operation is deprioritized).
self.assertTrue(
all(0 < multiplier <= 1 for multiplier in multipliers.values()))
def simple_sparse_add():
"""Adding a sparse and a dense tensor to produce a sparse tensor."""
examples = [
benchmark.Example(
inputs=[
tf.SparseTensor(indices=[[0, 0], [0, 1]],
values=[12, 34],
dense_shape=[2, 2]),
[[-3, 0], [-5, 0]],
],
output=tf.SparseTensor(indices=[[0, 0], [0, 1], [1, 0]],
values=[9, 34, -5],
dense_shape=[2, 2]),
),
]
constants = []
description = 'Add sparse tensor with dense tensor'
target_program = 'tf.sparse.add(in1, tf.sparse.from_dense(in2))'
source = 'handwritten task'
return benchmark.Benchmark(examples=examples,
constants=constants,
description=description,
target_program=target_program,
source=source,
name='simple_sparse_add')
def stackoverflow_47():
examples = [
benchmark.Example(inputs=[
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16],
[[True, True, True, False,
False], [True, True, False, False, False],
[True, True, True, True, True], [True, True, True, True, False],
[True, False, False, False, False],
[True, True, False, False, False]],
],
output=[[0, 1, 2, 0, 0], [3, 4, 0, 0, 0],
[5, 6, 7, 8, 9], [10, 11, 12, 13, 0],
[14, 0, 0, 0, 0], [15, 16, 0, 0, 0]]),
]
constants = []
description = 'put given values into a sequence mask'
# Note: StackOverflow answer is wrong (gather index out of bounds).
# Solution is simpler with an auxiliary variable:
# linear_mask = tf.reshape(tf.cast(in2, tf.int32), [-1])
# output = tf.reshape(tf.gather(in1, tf.cumsum(linear_mask, exclusive=True) * linear_mask), in2.shape)
target_program = 'tf.reshape(tf.gather(in1, tf.cumsum(tf.reshape(tf.cast(in2, tf.int32), [-1]), exclusive=True) * tf.reshape(tf.cast(in2, tf.int32), [-1])), in2.shape)'
source = 'https://stackoverflow.com/questions/58641546/how-can-i-put-the-sequential-values-to-the-sequence-mask'
return benchmark.Benchmark(examples=examples,
constants=constants,
description=description,
target_program=target_program,
source=source,
name='stackoverflow_47')
def google_03():
examples = [
benchmark.Example(inputs=[
tf.SparseTensor(indices=[[0, 0, 0], [0, 1, 1], [1, 1, 1],
[1, 1, 2]],
values=[1., 1., 1., 1.],
dense_shape=[2, 2, 800]),
],
output=tf.SparseTensor(indices=[[0, 0, 0], [0, 1,
1]],
values=[1., 1.],
dense_shape=[1, 2, 800])),
]
constants = []
description = 'Slice the first dimension of a SparseTensor'
# Original approach, which can't be written as one expression.
"""
reduced_sp_tensor = tf.sparse_retain(in1, tf.equal(in1.indices[:, 0], 0))
output = tf.SparseTensor(indices=reduced_sp_tensor.indices,
values=reduced_sp_tensor.values,
dense_shape=tf.concat([tf.ones(1, dtype=tf.int64),
in1.dense_shape[1:]], 0))
""" # pylint: disable=pointless-string-statement
target_program = 'tf.sparse.slice(in1, tf.zeros(3, dtype=tf.int64), tf.concat([[1], in1.dense_shape[1:]], 0))'
source = 'Real task encountered by Googler, 10/16/2018'
return benchmark.Benchmark(examples=examples,
constants=constants,
description=description,
target_program=target_program,
source=source,
name='google_03')
def stackoverflow_18():
examples = [
benchmark.Example(
inputs=[
# shape=[2, 2, 3].
[[[1, 1, 1], [1, 0, 1]], [[1, 2, 3], [4, 5, 6]]],
# shape=[3, 4].
[[1, 1, 1, 1], [1, 2, 3, 4], [5, 6, 7, 8]],
# shape=[4].
[100, 200, 300, 400],
],
# Shape=[sequence_length, batch_size, 4]=[2, 2, 4].
output=[[[107, 209, 311, 413], [106, 207, 308, 409]],
[[118, 223, 328, 433], [139, 250, 361, 472]]]),
]
constants = []
description = 'multiply 3D tensor and 2D tensor and add another tensor'
target_program = 'tf.add(in3, tf.matmul(in1, in2))'
source = 'https://stackoverflow.com/questions/38222126/tensorflow-efficient-way-for-tensor-multiplication'
return benchmark.Benchmark(examples=examples,
constants=constants,
description=description,
target_program=target_program,
source=source,
name='stackoverflow_18')
def autopandas_03():
# The "series" and "step" columns only serve to identify where the data should
# move to. The number of unique "series" and "step" values determine the
# output's shape, so we provide them (3 and 5) as constants. The real data is
# in the "value" column, so it's a 1D tensor. This is simply a tensor reshape
# and transpose.
examples = [
benchmark.Example(inputs=[
[
1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009,
1010, 1011, 1012, 1013, 1014
],
],
output=[[1000, 1003, 1006, 1009, 1012],
[1001, 1004, 1007, 1010, 1013],
[1002, 1005, 1008, 1011, 1014]]),
]
constants = [3, 5]
description = '' # No description for AutoPandas benchmarks.
target_program = 'tf.transpose(tf.reshape(in1, (5, 3)))'
source = 'SO_13647222_depth1'
return benchmark.Benchmark(examples=examples,
constants=constants,
description=description,
target_program=target_program,
source=source,
name='autopandas_03')
def autopandas_05():
# The problem is to sort a table using values in one column. Their code that
# constructs the output actually sorts by 3 columns, but their example has
# unique values for the first column, so the other two columns don't affect
# the result at all. Instead of having dates and strings and numbers, we just
# use numbers. Sort by column 0 in increasing order.
examples = [
benchmark.Example(inputs=[
[[6, 3, 7, 8, 4], [8, 9, 4, 5, 3], [1, 5, 3, 6, 9],
[2, 1, 4, 3, 2], [7, 9, 6, 2, 7], [5, 8, 0, 4, 2]],
],
output=[[1, 5, 3, 6, 9], [2, 1, 4, 3, 2],
[5, 8, 0, 4, 2], [6, 3, 7, 8, 4],
[7, 9, 6, 2, 7], [8, 9, 4, 5, 3]]),
]
constants = []
description = '' # No description for AutoPandas benchmarks.
target_program = 'tf.gather(in1, tf.argsort(in1[:, 0], stable=True))'
source = 'SO_49583055_depth1'
return benchmark.Benchmark(examples=examples,
constants=constants,
description=description,
target_program=target_program,
source=source,
name='autopandas_05')
def autopandas_14():
# The intended solution is to do a merge, but the example actually has nothing
# to merge, so all that happens is an extra column is created filled with NaN.
# Why are there so many problems where the example completely underspecifies
# the intended transformation? Instead of ignoring the problem because it has
# a merge (out of scope), let's just try to append a NaN column.
examples = [
benchmark.Example(
inputs=[
[[1, 0, 1, 2], [1, 1, 3, 4], [2, 0, 1, 2], [2, 1, 3, 4]],
# They have another input that is used to provide the column
# header for the new NaN-filled column. Tensors don't have column
# headers so this input is useless. Omit it, or else TF-Coder will
# be required to use it.
],
output=[[1, 0, 1, 2, float('nan')], [1, 1, 3, 4,
float('nan')],
[2, 0, 1, 2, float('nan')], [2, 1, 3, 4,
float('nan')]]),
]
constants = [float('nan')]
description = '' # No description for AutoPandas benchmarks.
target_program = 'tf.concat((tf.cast(in1, tf.float32), tf.fill([4, 1], float("nan"))), axis=1)'
source = 'SO_13576164_depth3'
return benchmark.Benchmark(examples=examples,
constants=constants,
description=description,
target_program=target_program,
source=source,
name='autopandas_14')
def stackoverflow_30():
examples = [
benchmark.Example(
inputs=[
[[1., 2.], [3., 4.], [5., 6.]],
[[9., 4.], [8., 5.], [7., 6.]],
],
output=[[math.sqrt(68),
math.sqrt(58),
math.sqrt(52)],
[math.sqrt(36),
math.sqrt(26),
math.sqrt(20)],
[math.sqrt(20), math.sqrt(10),
math.sqrt(4)]]),
]
constants = []
description = 'compute Euclidean distance between two tensors'
# StackOverflow answer is incorrect.
target_program = 'tf.sqrt(tf.reduce_sum(tf.square(tf.subtract(tf.expand_dims(in1, 1), tf.expand_dims(in2, 0))), axis=2))'
source = 'https://stackoverflow.com/questions/54147780/tensorflow-how-to-calculate-the-euclidean-distance-between-two-tensor'
return benchmark.Benchmark(examples=examples,
constants=constants,
description=description,
target_program=target_program,
source=source,
name='stackoverflow_30')
def test_add_constants_and_inputs_and_print_checks_names(self, bad_name):
values_by_weight = [collections.OrderedDict() for _ in range(100)]
examples = [
benchmark_module.Example(inputs={bad_name: [1, 2, 3]},
output=[1, 2])
]
benchmark = benchmark_module.Benchmark(examples, constants=[])
output_value = value_module.OutputValue(benchmark.examples[0].output)
with self.assertRaises(ValueError):
value_search._add_constants_and_inputs_and_print(
values_by_weight, benchmark, output_value,
self._constant_operation, self.settings)
def test_benchmark_init_computes_num_inputs(self):
examples = [
benchmark.Example(inputs=[
[1],
[2],
], output=[1, 2]),
benchmark.Example(inputs=[
[[1, 2], [3, 4]],
[[5, 6], [7, 8]],
],
output=[[1, 2], [3, 4], [5, 6], [7, 8]]),
]
self.assertEqual(
benchmark.Benchmark(examples, name='test').num_inputs, 2)
def autopandas_02():
"""AutoPandas benchmark.
The DataFrame input:
value1 value2
group1 group2
a c 1.1 7.1
c 2.0 8.0
d 3.0 9.0
b d 4.0 10.0
d 5.0 11.0
e 6.0 12.0
The DataFrame output:
value1 value2
group2
c 1.1 7.1
c 2.0 8.0
d 3.0 9.0
Notice that "c", "d", and "e" are just treated as data, so we'll replace them
with 1, 2, and 3, respectively. "group1" acts as a third dimension, so our
input tensor will be 3D. We'll also make "group1" the innermost axis, to make
the problem not super trivial in TensorFlow.
Finally, I changed some numbers to rule out tf.reduce_min(axis=2).
"""
examples = [
benchmark.Example(
inputs=[
[
[[1, 2], [1, 2], [2, 3]], # group2
[[1.1, 4], [2, 1], [3, 6]], # value1
[[7.1, 10], [8, 11], [9, 7]]
], # value2
],
output=[[1, 1, 2], [1.1, 2, 3], [7.1, 8, 9]]),
]
constants = []
description = '' # No description for AutoPandas benchmarks.
# Note that we don't have ops for indexing/slicing axis 2. But, TF-Coder will
# find a workaround.
target_program = 'in1[:, :, 0]'
source = 'SO_11941492_depth1'
return benchmark.Benchmark(examples=examples,
constants=constants,
description=description,
target_program=target_program,
source=source,
name='autopandas_02')
def stackoverflow_12():
examples = [
benchmark.Example(inputs=[[[12, 34, 56], [33, 22, 11]]],
output=[[12, 56], [33, 11]]),
]
constants = [0, 1, 2]
description = 'remove a column from the tensor'
target_program = 'tf.gather(in1, (0, 2), axis=1, batch_dims=0)'
source = 'https://stackoverflow.com/questions/47447183/remove-a-set-of-tensors-from-a-tensor-in-tensorflow'
return benchmark.Benchmark(examples=examples,
constants=constants,
description=description,
target_program=target_program,
source=source,
name='stackoverflow_12')
def test_run_value_handles_large_weight_constants(self):
benchmark = benchmark_module.Benchmark(examples=[
benchmark_module.Example(inputs=[[1], [2]], output=[[3]])
])
results = value_search.run_value_search(benchmark=benchmark,
settings=self.settings)
self.assertNotEmpty(results.solutions)
self.assertEqual(results.solutions[0].expression,
'tf.add(in1, tf.expand_dims(in2, 0))')
output_shape_constant = value_module.ConstantValue((1, 1))
self.assertIn(output_shape_constant, results.value_set)
# Find the element in value_set equal to output_shape_constant and assert
# that it's actually a ConstantValue, as opposed to an OperationValue.
for value in results.value_set:
if value == output_shape_constant:
self.assertIsInstance(value, value_module.ConstantValue)
def stackoverflow_26():
examples = [
benchmark.Example(inputs=[[[[3, 4], [1, 2]], [[5, -2], [-10, 3]],
[[10, 20], [-4, 7]]]],
output=[10, -4, 33]),
]
constants = []
description = 'reduction operation for multiple dimensions simultaneously'
target_program = 'tf.reduce_sum(tf.reduce_sum(in1, axis=1), axis=1)'
source = 'https://stackoverflow.com/questions/54294780/how-to-perform-reduce-op-on-multiple-dimensions-at-once'
return benchmark.Benchmark(examples=examples,
constants=constants,
description=description,
target_program=target_program,
source=source,
name='stackoverflow_26')
def stackoverflow_09():
examples = [
benchmark.Example(inputs=[
[37, 42, 42, 37, 28, 15, 42, 15],
],
output=[0, 1, 1, 0, 2, 3, 1, 3]),
]
constants = []
description = 'group items by value and get the group indices'
target_program = 'tf.unique_with_counts(in1)[1]'
source = 'https://stackoverflow.com/questions/53054668/assign-values-between-0-and-n-1-for-a-vector-of-length-l-with-n-different-eleme'
return benchmark.Benchmark(examples=examples,
constants=constants,
description=description,
target_program=target_program,
source=source,
name='stackoverflow_09')
def autopandas_06():
# This is very similar to autopandas_03, they're both pivot tables.
examples = [
benchmark.Example(inputs=[
[4, 5, 6, 7],
], output=[[4, 5], [6, 7]]),
]
constants = [2]
description = '' # No description for AutoPandas benchmarks.
target_program = 'tf.reshape(in1, (2, 2))'
source = 'SO_13261175_depth1'
return benchmark.Benchmark(examples=examples,
constants=constants,
description=description,
target_program=target_program,
source=source,
name='autopandas_06')
def autopandas_06_ignored():
"""AutoPandas benchmark.
The input DataFrame:
X Y Z
4 X1 Y2 Z3
5 X1 Y1 Z1
6 X1 Y1 Z1
7 X1 Y1 Z2
The output DataFrame:
Z Z1 Z2 Z3
Y
Y1 1.0 1.0 NaN
Y2 NaN NaN 1.0
Basically, the Ys are row indices and the Zs are column indices. The X1 does
not really matter, we just want a boolean output table (1.0=True, NaN=False)
that identifies which Y and Z pairs appeared in the input table.
Their example is tiny so I expanded it.
This task also doesn't appear in their table of results?
"""
examples = [
benchmark.Example(
inputs=[
# Y Z
[[1, 2], [0, 0], [0, 0], [0, 1], [4, 2], [4, 3], [4, 2],
[2, 1]],
],
output=[[True, True, False, False], [False, False, True, False],
[False, True, False, False], [False, False, False, False],
[False, False, True, True]]),
]
constants = []
description = '' # No description for AutoPandas benchmarks.
# TODO(kshi): Note that tf.scatter_nd is not yet supported by TF-Coder!!
target_program = 'tf.cast(tf.scatter_nd(in1, updates=tf.ones(8), shape=[5, 4]), tf.bool)'
source = 'SO_12860421_depth1'
return benchmark.Benchmark(examples=examples,
constants=constants,
description=description,
target_program=target_program,
source=source,
name='autopandas_06_ignored')
def stackoverflow_11():
examples = [
benchmark.Example(inputs=[
[4, 0, 1, 1, 0, 4, 0, 0, 3, 4, 1],
],
output=[4, 3, 0, 1, 3]),
]
constants = []
description = 'count the number of occurences of each distinct number'
target_program = 'tf.math.bincount(in1)'
source = 'https://stackoverflow.com/questions/45194672/how-to-count-elements-in-tensorflow-tensor'
return benchmark.Benchmark(examples=examples,
constants=constants,
description=description,
target_program=target_program,
source=source,
name='stackoverflow_11')
def stackoverflow_15():
examples = [
benchmark.Example(inputs=[
[3, 1, 2, 0, 1, -1, 10, 1, -10],
],
output=[3, 0, 2, 0, 0, -1, 10, 0, -10]),
]
constants = [0, 1]
description = 'set all instances of 1 to 0'
target_program = 'tf.subtract(in1, tf.cast(tf.equal(in1, tf.constant(1)), tf.int32))'
source = 'https://stackoverflow.com/questions/39045797/conditional-assignment-of-tensor-values-in-tensorflow'
return benchmark.Benchmark(examples=examples,
constants=constants,
description=description,
target_program=target_program,
source=source,
name='stackoverflow_15')
def google_20():
examples = [
benchmark.Example(inputs=[
[10, 7, 4, 3, 2, 8],
],
output=[5, 3, 2, 1, 0, 4]),
]
constants = []
description = 'sort a tensor and return sorted index in original order'
target_program = 'tf.cast(tf.argsort(tf.argsort(in1)), tf.int32)'
source = 'From an internal Google forum'
return benchmark.Benchmark(examples=examples,
constants=constants,
description=description,
target_program=target_program,
source=source,
name='google_20')
def google_15():
examples = [
benchmark.Example(inputs=[
[[1, 3, 5, 7], [2, 4, 6, 8]],
],
output=[[1, 3, 5, 7, 0], [2, 4, 6, 8, 0]]),
]
constants = []
description = 'pad a zero column'
target_program = "tf.pad(in1, [[0, 0], [0, 1]], 'CONSTANT')"
source = 'Real task encountered by Googler, 10/23/2019'
return benchmark.Benchmark(examples=examples,
constants=constants,
description=description,
target_program=target_program,
source=source,
name='google_15')
def stackoverflow_46():
examples = [
benchmark.Example(inputs=[
[3, 4, 1],
],
output=[0, 0, 0, 1, 1, 1, 1, 2]),
]
constants = []
description = 'convert segment lengths to segment ids'
target_program = 'tf.cast(tf.where(tf.sequence_mask(in1))[:, 0], tf.int32)'
source = 'https://stackoverflow.com/questions/58652161/how-to-convert-2-3-4-to-0-0-1-1-1-2-2-2-2-to-utilize-tf-math-segment-sum'
return benchmark.Benchmark(examples=examples,
constants=constants,
description=description,
target_program=target_program,
source=source,
name='stackoverflow_46')
def stackoverflow_02():
examples = [
benchmark.Example(inputs=[
[5, 1, 0, 3, 0, -1, 2, -10, 2],
],
output=[1, 1, 0, 1, 0, -1, 1, -10, 1]),
]
constants = [1]
description = 'clip values that are greater than 1'
target_program = 'tf.minimum(in1, tf.constant(1))'
source = 'https://stackoverflow.com/questions/46408839/tensorflow-trim-values-in-tensor'
return benchmark.Benchmark(examples=examples,
constants=constants,
description=description,
target_program=target_program,
source=source,
name='stackoverflow_02')
def simple_using_constant():
"""A benchmark that requires a user-provided constant."""
examples = [
benchmark.Example(inputs=[
[1, 2, 3],
], output=[101, 102, 103]),
]
constants = [100]
description = 'Add 100 to every element'
target_program = 'tf.add(in1, tf.constant(100))'
source = 'handwritten task'
return benchmark.Benchmark(examples=examples,
constants=constants,
description=description,
target_program=target_program,
source=source,
name='simple_using_constant')
def stackoverflow_32():
examples = [
benchmark.Example(inputs=[
[[0.1, 0.6, 0.2, 0.1], [0.3, 0.1, 0.4, 0.2], [0.2, 0.1, 0.2, 0.5]],
],
output=[1.3, 1.5, 2.0]),
]
constants = []
description = 'weighted sum across rows, where the column index is the weight'
target_program = 'tf.tensordot(in1, tf.cast(tf.range(4), tf.float32), 1)'
source = 'https://stackoverflow.com/questions/48659449/how-to-compute-the-weighted-sum-of-a-tensor-in-tensorflow'
return benchmark.Benchmark(examples=examples,
constants=constants,
description=description,
target_program=target_program,
source=source,
name='stackoverflow_32')
def stackoverflow_42():
examples = [
benchmark.Example(inputs=[
[4, 6, 2, 6, 7, 3, -3],
],
output=[0, 0, 0, 0, 1, 0, 0]),
]
constants = []
description = 'create a binary vector where the max element is 1'
target_program = 'tf.cast(tf.equal(in1, tf.reduce_max(in1)), tf.int32)'
source = 'https://stackoverflow.com/questions/54493814/binary-vector-of-max'
return benchmark.Benchmark(examples=examples,
constants=constants,
description=description,
target_program=target_program,
source=source,
name='stackoverflow_42')
