def test_get_known_descendants(self): expression = prensor_test_util.create_nested_prensor() expr = create_expression.create_expression_from_prensor(expression) expr_map = expr.get_known_descendants() self.assertIn(path.Path(["doc"]), expr_map) self.assertIn(path.Path(["doc", "bar"]), expr_map) self.assertIn(path.Path(["doc", "keep_me"]), expr_map) self.assertIn(path.Path(["user"]), expr_map) self.assertIn(path.Path(["user", "friends"]), expr_map)
def test_get_positional_index_calculate(self):
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_nested_prensor())
new_root, new_path = index.get_positional_index(
expr, path.Path(["user", "friends"]), path.get_anonymous_field())
new_field = new_root.get_descendant_or_error(new_path)
leaf_node = expression_test_util.calculate_value_slowly(new_field)
self.assertAllEqual(leaf_node.parent_index, [0, 1, 1, 2, 3])
self.assertAllEqual(leaf_node.values, [0, 0, 1, 0, 0])
def test_promote_and_calculate(self):
"""Tests promoting a leaf on a nested tree."""
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_nested_prensor())
new_root, new_path = promote.promote_anonymous(
expr, path.Path(["user", "friends"]))
new_field = new_root.get_descendant_or_error(new_path)
leaf_node = expression_test_util.calculate_value_slowly(new_field)
self.assertAllEqual(leaf_node.parent_index, [0, 1, 1, 1, 2])
self.assertAllEqual(leaf_node.values, [b"a", b"b", b"c", b"d", b"e"])
def test_nested_prensor(self):
"""Tests on a deep expression."""
pren = prensor_test_util.create_nested_prensor()
st = prensor_to_structured_tensor.prensor_to_structured_tensor(pren)
self.assertAllEqual(st.field_value(["doc", "bar"]),
[[[b"a"]], [[b"b", b"c"], [b"d"]], []])
self.assertAllEqual(st.field_value(["doc", "keep_me"]),
[[[False]], [[True], []], []])
self.assertAllEqual(st.field_value(["user", "friends"]),
[[[b"a"]], [[b"b", b"c"], [b"d"]], [[b"e"]]])
def testPlaceholderExpression(self):
pren = prensor_test_util.create_nested_prensor()
expected_pren = prensor.create_prensor_from_descendant_nodes({
path.Path([]):
prensor.RootNodeTensor(tf.constant(3, dtype=tf.int64)),
path.Path(["new_friends"]):
prensor.LeafNodeTensor(
tf.constant([0, 1, 1, 1, 2], dtype=tf.int64),
tf.constant(["a", "b", "c", "d", "e"], dtype=tf.string), True)
})
root_schema = mpp.create_schema(is_repeated=True,
children={
"doc": {
"is_repeated": True,
"children": {
"bar": {
"is_repeated": True,
"dtype": tf.string
},
"keep_me": {
"is_repeated": False,
"dtype": tf.bool
}
}
},
"user": {
"is_repeated": True,
"children": {
"friends": {
"is_repeated": True,
"dtype": tf.string
}
}
}
})
exp = placeholder.create_expression_from_schema(root_schema)
promote_exp = promote.promote(exp, path.Path(["user", "friends"]),
"new_friends")
project_exp = project.project(promote_exp,
[path.Path(["new_friends"])])
new_friends_exp = project_exp.get_descendant(path.Path(["new_friends"
]))
result = calculate.calculate_values([new_friends_exp],
feed_dict={exp: pren})
res_node = result[0]
exp_node = expected_pren.get_descendant(path.Path(["new_friends"
])).node
self.assertAllEqual(res_node.is_repeated, exp_node.is_repeated)
self.assertAllEqual(res_node.values, exp_node.values)
self.assertAllEqual(res_node.parent_index, exp_node.parent_index)
def test_calculate_promote_anonymous(self):
"""Performs promote_test.PromoteValuesTest, but with calculate_values."""
for options in options_to_test:
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_nested_prensor())
new_root, new_path = promote.promote_anonymous(
expr, path.Path(["user", "friends"]))
new_field = new_root.get_descendant_or_error(new_path)
[leaf_node] = calculate.calculate_values([new_field], options=options)
self.assertAllEqual(leaf_node.parent_index, [0, 1, 1, 1, 2])
self.assertAllEqual(leaf_node.values, [b"a", b"b", b"c", b"d", b"e"])
def test_project(self):
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_nested_prensor())
projected = expr.project(
[path.Path(["user", "friends"]),
path.Path(["doc", "keep_me"])])
self.assertIsNotNone(
projected.get_descendant(path.Path(["user", "friends"])))
self.assertIsNotNone(
projected.get_descendant(path.Path(["doc", "keep_me"])))
self.assertIsNone(projected.get_descendant(path.Path(["doc", "bar"])))
def test_promote(self):
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_nested_prensor())
new_root = expr.promote("user.friends", "new_field")
new_field = new_root.get_child_or_error("new_field")
self.assertIsNotNone(new_field)
self.assertTrue(new_field.is_repeated)
self.assertEqual(new_field.type, tf.string)
self.assertTrue(new_field.is_leaf)
leaf_node = expression_test_util.calculate_value_slowly(new_field)
self.assertEqual(leaf_node.values.dtype, tf.string)
self.assertEqual(new_field.known_field_names(), frozenset())
def test_calculate_promote_named(self):
"""Performs promote_test.PromoteValuesTest, but with calculate_values."""
for options in options_to_test:
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_nested_prensor())
new_root = promote.promote(expr, path.Path(["user", "friends"]),
"new_friends")
# projected = project.project(new_root, [path.Path(["new_friends"])])
new_field = new_root.get_child_or_error("new_friends")
[leaf_node] = calculate.calculate_values([new_field], options=options)
self.assertAllEqual(leaf_node.parent_index, [0, 1, 1, 1, 2])
self.assertAllEqual(leaf_node.values, [b"a", b"b", b"c", b"d", b"e"])
def test_map_prensor_to_prensor_with_schema(self):
original = create_expression.create_expression_from_prensor(
prensor_test_util.create_nested_prensor())
def my_prensor_op(original_prensor):
# Note that we are copying over the original root prensor node. The root
# node is ignored in the result.
return prensor.create_prensor_from_descendant_nodes({
path.Path([]):
original_prensor.node,
path.Path(["bar2"]):
original_prensor.get_child_or_error("bar").node,
path.Path(["keep_me2"]):
original_prensor.get_child_or_error("keep_me").node
})
bar2_feature = schema_pb2.Feature()
bar2_feature.value_count.max = 7
keep_me2_feature = schema_pb2.Feature()
keep_me2_feature.value_count.max = 10
# Since the top node is actually a child node, we use the child schema.
my_output_schema = map_prensor_to_prensor.create_schema(
is_repeated=True,
children={
"bar2": {
"is_repeated": True,
"dtype": tf.string,
"schema_feature": bar2_feature
},
"keep_me2": {
"is_repeated": False,
"dtype": tf.bool,
"schema_feature": keep_me2_feature
}
})
result = map_prensor_to_prensor.map_prensor_to_prensor(
root_expr=original,
source=path.Path(["doc"]),
paths_needed=[path.Path(["bar"]),
path.Path(["keep_me"])],
prensor_op=my_prensor_op,
output_schema=my_output_schema)
doc_result = result.get_child_or_error("doc")
bar2_result = doc_result.get_child_or_error("bar2")
self.assertEqual(bar2_result.schema_feature.value_count.max, 7)
keep_me2_result = doc_result.get_child_or_error("keep_me2")
self.assertEqual(keep_me2_result.schema_feature.value_count.max, 10)
def test_promote(self):
"""Tests promote.promote(...), and indirectly tests set_path."""
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_nested_prensor())
new_root = promote.promote(expr, path.Path(["user", "friends"]),
"new_field")
new_field = new_root.get_child_or_error("new_field")
self.assertIsNotNone(new_field)
self.assertTrue(new_field.is_repeated)
self.assertEqual(new_field.type, tf.string)
self.assertTrue(new_field.is_leaf)
leaf_node = expression_test_util.calculate_value_slowly(new_field)
self.assertEqual(leaf_node.values.dtype, tf.string)
self.assertEqual(new_field.known_field_names(), frozenset())
def test_promote_and_calculate(self):
"""Tests get_sparse_tensors on a deep tree."""
with self.session(use_gpu=False) as sess:
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_nested_prensor())
new_root, new_path = promote.promote_anonymous(
expr, path.Path(["user", "friends"]))
new_field = new_root.get_descendant_or_error(new_path)
leaf_node = expression_test_util.calculate_value_slowly(new_field)
parent_index, values = sess.run(
[leaf_node.parent_index, leaf_node.values])
self.assertAllEqual(parent_index, [0, 1, 1, 1, 2])
self.assertAllEqual(values, [b"a", b"b", b"c", b"d", b"e"])
def test_prensor_is_composite_tensor(self):
for pren in [
prensor_test_util.create_nested_prensor(),
prensor_test_util.create_big_prensor(),
prensor_test_util.create_deep_prensor()
]:
flattened_tensors = tf.nest.flatten(pren, expand_composites=True)
self.assertIsInstance(flattened_tensors, list)
for t in flattened_tensors:
self.assertIsInstance(t, tf.Tensor)
packed_pren = tf.nest.pack_sequence_as(pren,
flattened_tensors,
expand_composites=True)
self._assert_prensor_equals(pren, packed_pren)
def test_get_index_from_end(self):
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_nested_prensor())
new_root, new_path = index.get_index_from_end(
expr, path.Path(["user", "friends"]), path.get_anonymous_field())
new_field = new_root.get_descendant_or_error(new_path)
self.assertTrue(new_field.is_repeated)
self.assertEqual(new_field.type, tf.int64)
self.assertTrue(new_field.is_leaf)
self.assertTrue(new_field.calculation_equal(new_field))
self.assertFalse(new_field.calculation_equal(expr))
leaf_node = expression_test_util.calculate_value_slowly(new_field)
self.assertEqual(leaf_node.values.dtype, tf.int64)
self.assertEqual(new_field.known_field_names(), frozenset())
def test_get_index_from_end_calculate(self):
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_nested_prensor())
new_root, new_path = index.get_index_from_end(
expr, path.Path(["user", "friends"]), path.get_anonymous_field())
print("test_get_index_from_end_calculate: new_path: {}".format(new_path))
new_field = new_root.get_descendant_or_error(new_path)
print("test_get_index_from_end_calculate: new_field: {}".format(
str(new_field)))
leaf_node = expression_test_util.calculate_value_slowly(new_field)
print("test_get_index_from_end_calculate: leaf_node: {}".format(
str(leaf_node)))
self.assertAllEqual(leaf_node.parent_index, [0, 1, 1, 2, 3])
self.assertAllEqual(leaf_node.values, [-1, -2, -1, -1, -1])
def test_prensor_to_ragged_tensors(self):
for options in _OPTIONS_TO_TEST:
pren = prensor_test_util.create_nested_prensor()
ragged_tensor_map = pren.get_ragged_tensors(options=options)
string_tensor_map = {
str(k): v
for k, v in ragged_tensor_map.items()
}
string_np_map = self.evaluate(string_tensor_map)
self.assertAllEqual(string_np_map["doc.bar"].to_list(),
[[[b"a"]], [[b"b", b"c"], [b"d"]], []])
self.assertAllEqual(string_np_map["doc.keep_me"].to_list(),
[[[False]], [[True], []], []])
self.assertAllEqual(string_np_map["user.friends"].to_list(),
[[[b"a"]], [[b"b", b"c"], [b"d"]], [[b"e"]]])
def test_is_leaf(self):
"""Tests get_sparse_tensors on a deep expression."""
expression = prensor_test_util.create_nested_prensor()
self.assertTrue(
expression.get_descendant_or_error(path.Path(["doc",
"bar"])).is_leaf)
self.assertFalse(
expression.get_descendant_or_error(path.Path(["doc"])).is_leaf)
self.assertFalse(
expression.get_descendant_or_error(path.Path([])).is_leaf)
self.assertTrue(
expression.get_descendant_or_error(path.Path(["user",
"friends"])).is_leaf)
self.assertTrue(
expression.get_descendant_or_error(path.Path(["doc",
"keep_me"])).is_leaf)
def test_add_paths(self):
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_nested_prensor())
new_root = expression_add.add_paths(
expr, {
path.Path(["user", "friends_copy"]):
expr.get_descendant_or_error(path.Path(["user", "friends"]))
})
new_field = new_root.get_descendant_or_error(
path.Path(["user", "friends_copy"]))
self.assertIsNotNone(new_field)
self.assertTrue(new_field.is_repeated)
self.assertEqual(new_field.type, tf.string)
self.assertTrue(new_field.is_leaf)
leaf_node = expression_test_util.calculate_value_slowly(new_field)
self.assertEqual(leaf_node.values.dtype, tf.string)
self.assertEqual(new_field.known_field_names(), frozenset())
def test_get_ragged_tensors(self):
"""Tests get_ragged_tensors on a deep expression."""
for options in options_to_test:
expression = prensor_test_util.create_nested_prensor()
ragged_tensor_map = prensor_util.get_ragged_tensors(
expression, options)
string_tensor_map = {
str(k): v
for k, v in ragged_tensor_map.items()
}
string_np_map = self.evaluate(string_tensor_map)
self.assertAllEqual(string_np_map["doc.bar"].to_list(),
[[[b"a"]], [[b"b", b"c"], [b"d"]], []])
self.assertAllEqual(string_np_map["doc.keep_me"].to_list(),
[[[False]], [[True], []], []])
self.assertAllEqual(string_np_map["user.friends"].to_list(),
[[[b"a"]], [[b"b", b"c"], [b"d"]], [[b"e"]]])
def test_add_to_already_existing_path(self):
with self.assertRaises(ValueError):
root = create_expression.create_expression_from_prensor(
prensor_test_util.create_nested_prensor())
root_1 = expression_add.add_paths(
root, {
path.Path(["user", "friends_2"]):
root.get_descendant_or_error(path.Path(["user", "friends"
]))
})
root_2 = expression_add.add_paths(
root_1, {
path.Path(["user", "friends_3"]):
root_1.get_descendant_or_error(
path.Path(["user", "friends_2"]))
})
expression_add.add_to(root,
{path.Path(["user", "friends"]): root_2})
def test_promote_anonymous(self):
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_nested_prensor())
new_root, new_field = promote.promote_anonymous(
expr, path.Path(["user", "friends"]))
new_field = new_root.get_descendant_or_error(new_field)
self.assertTrue(new_field.is_repeated)
self.assertEqual(new_field.type, tf.string)
self.assertTrue(new_field.is_leaf)
self.assertFalse(new_field.calculation_is_identity())
self.assertTrue(new_field.calculation_equal(new_field))
self.assertFalse(new_field.calculation_equal(expr))
leaf_node = expression_test_util.calculate_value_slowly(new_field)
self.assertEqual(leaf_node.values.dtype, tf.string)
self.assertEqual(new_field.known_field_names(), frozenset())
sources = new_field.get_source_expressions()
self.assertLen(sources, 2)
self.assertIs(
expr.get_descendant_or_error(path.Path(["user", "friends"])),
sources[0])
self.assertIs(expr.get_child_or_error("user"), sources[1])
def test_prensor_to_sparse_tensors(self):
for options in _OPTIONS_TO_TEST:
pren = prensor_test_util.create_nested_prensor()
sparse_tensor_map = pren.get_sparse_tensors(options=options)
string_tensor_map = {
str(k): v
for k, v in sparse_tensor_map.items()
}
self.assertAllEqual(string_tensor_map["doc.bar"].indices,
[[0, 0, 0], [1, 0, 0], [1, 0, 1], [1, 1, 0]])
self.assertAllEqual(string_tensor_map["doc.bar"].values,
[b"a", b"b", b"c", b"d"])
self.assertAllEqual(string_tensor_map["doc.keep_me"].indices,
[[0, 0], [1, 0]])
self.assertAllEqual(string_tensor_map["doc.keep_me"].values,
[False, True])
self.assertAllEqual(
string_tensor_map["user.friends"].indices,
[[0, 0, 0], [1, 0, 0], [1, 0, 1], [1, 1, 0], [2, 0, 0]])
self.assertAllEqual(string_tensor_map["user.friends"].values,
[b"a", b"b", b"c", b"d", b"e"])
def test_add_to(self):
root = create_expression.create_expression_from_prensor(
prensor_test_util.create_nested_prensor())
root_1 = expression_add.add_paths(
root, {
path.Path(["user", "friends_2"]):
root.get_descendant_or_error(path.Path(["user", "friends"]))
})
root_2 = expression_add.add_paths(
root_1, {
path.Path(["user", "friends_3"]):
root_1.get_descendant_or_error(path.Path(["user", "friends_2"
]))
})
root_3 = expression_add.add_to(
root, {path.Path(["user", "friends_3"]): root_2})
new_field = root_3.get_descendant_or_error(
path.Path(["user", "friends_3"]))
self.assertIsNotNone(new_field)
self.assertTrue(new_field.is_repeated)
self.assertEqual(new_field.type, tf.string)
leaf_node = expression_test_util.calculate_value_slowly(new_field)
self.assertEqual(leaf_node.values.dtype, tf.string)
def test_map_prensor_to_prensor(self):
original = create_expression.create_expression_from_prensor(
prensor_test_util.create_nested_prensor())
def my_prensor_op(original_prensor):
# Note that we are copying over the original root prensor node. The root
# node is ignored in the result.
return prensor.create_prensor_from_descendant_nodes({
path.Path([]):
original_prensor.node,
path.Path(["bar2"]):
original_prensor.get_child_or_error("bar").node,
path.Path(["keep_me2"]):
original_prensor.get_child_or_error("keep_me").node
})
# Since the top node is actually a child node, we use the child schema.
my_output_schema = map_prensor_to_prensor.create_schema(
is_repeated=True,
children={
"bar2": {
"is_repeated": True,
"dtype": tf.string
},
"keep_me2": {
"is_repeated": False,
"dtype": tf.bool
}
})
result = map_prensor_to_prensor.map_prensor_to_prensor(
root_expr=original,
source=path.Path(["doc"]),
paths_needed=[path.Path(["bar"]),
path.Path(["keep_me"])],
prensor_op=my_prensor_op,
output_schema=my_output_schema)
doc_result = result.get_child_or_error("doc")
bar_result = doc_result.get_child_or_error("bar")
keep_me_result = doc_result.get_child_or_error("keep_me")
bar2_result = doc_result.get_child_or_error("bar2")
keep_me2_result = doc_result.get_child_or_error("keep_me2")
self.assertIsNone(doc_result.get_child("missing_field"))
self.assertTrue(bar_result.is_repeated)
self.assertTrue(bar2_result.is_repeated)
self.assertEqual(bar_result.type, tf.string)
self.assertEqual(bar2_result.type, tf.string)
self.assertFalse(keep_me_result.is_repeated)
self.assertFalse(keep_me2_result.is_repeated)
self.assertEqual(keep_me_result.type, tf.bool)
self.assertEqual(keep_me2_result.type, tf.bool)
[prensor_result] = calculate.calculate_prensors([result])
doc_value = prensor_result.get_child_or_error("doc")
self.assertAllEqual([0, 1, 1], doc_value.node.parent_index)
bar2_value = doc_value.get_child_or_error("bar2")
self.assertAllEqual([0, 1, 1, 2], bar2_value.node.parent_index)
self.assertAllEqual([b"a", b"b", b"c", b"d"], bar2_value.node.values)
keep_me2_value = doc_value.get_child_or_error("keep_me2")
self.assertAllEqual([0, 1], keep_me2_value.node.parent_index)
self.assertAllEqual([False, True], keep_me2_value.node.values)
