def test_promote_and_calculate_substructure(self):
"""Tests promoting substructure on a tree with depth of 4."""
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_four_layer_prensor())
new_root, new_path = promote.promote_anonymous(
expr, path.Path(["event", "doc", "nested_child"]))
new_nested_child = new_root.get_descendant_or_error(new_path)
bar_expr = new_root.get_descendant_or_error(new_path.get_child("bar"))
keep_me_expr = new_root.get_descendant_or_error(
new_path.get_child("keep_me"))
# the promoted nested_child's parent index is changed.
nested_child_node = expression_test_util.calculate_value_slowly(
new_nested_child)
self.assertAllEqual(nested_child_node.parent_index, [0, 1, 1, 1])
self.assertTrue(nested_child_node.is_repeated)
# bar's parent index should be unchanged.
bar_node = expression_test_util.calculate_value_slowly(bar_expr)
self.assertAllEqual(bar_node.parent_index, [0, 1, 1, 2])
self.assertAllEqual(bar_node.values, [b"a", b"b", b"c", b"d"])
self.assertTrue(bar_node.is_repeated)
# keep_me's parent index should be unchanged.
keep_me_node = expression_test_util.calculate_value_slowly(
keep_me_expr)
self.assertAllEqual(keep_me_node.parent_index, [0, 1])
self.assertAllEqual(keep_me_node.values, [False, True])
self.assertFalse(keep_me_node.is_repeated)
def test_reroot_and_create_proto_index(self):
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_big_prensor()).reroot(
"doc").create_proto_index("proto_index")
proto_index = expr.get_child("proto_index")
new_field = expr.get_child("bar")
leaf_node = expression_test_util.calculate_value_slowly(new_field)
proto_index_node = expression_test_util.calculate_value_slowly(
proto_index)
self.assertIsNotNone(new_field)
self.assertTrue(new_field.is_repeated)
self.assertEqual(new_field.type, tf.string)
self.assertTrue(new_field.is_leaf)
self.assertEqual(new_field.known_field_names(), frozenset())
self.assertEqual(leaf_node.values.dtype, tf.string)
self.assertIsNotNone(proto_index)
self.assertFalse(proto_index.is_repeated)
self.assertEqual(proto_index.type, tf.int64)
self.assertTrue(proto_index.is_leaf)
self.assertEqual(proto_index.known_field_names(), frozenset())
self.assertEqual(proto_index_node.values.dtype, tf.int64)
self.assertAllEqual([b"a", b"b", b"c", b"d"], leaf_node.values)
self.assertAllEqual([0, 1, 1, 2], leaf_node.parent_index)
self.assertAllEqual([0, 1, 1], proto_index_node.values)
self.assertAllEqual([0, 1, 2], proto_index_node.parent_index)
def test_promote_substructure(self):
"""Tests promote.promote(...) of substructure."""
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_deep_prensor())
new_root = promote.promote(expr, path.Path(["event", "doc"]),
"new_field")
new_field = new_root.get_child_or_error("new_field")
self.assertIsNotNone(new_field)
self.assertTrue(new_field.is_repeated)
self.assertEqual(new_field.known_field_names(),
frozenset(["bar", "keep_me"]))
bar_expr = new_field.get_child_or_error("bar")
self.assertIsNotNone(bar_expr)
self.assertTrue(bar_expr.is_repeated)
self.assertEqual(bar_expr.type, tf.string)
self.assertTrue(bar_expr.is_leaf)
keep_me_expr = new_field.get_child_or_error("keep_me")
self.assertIsNotNone(keep_me_expr)
self.assertFalse(keep_me_expr.is_repeated)
self.assertEqual(keep_me_expr.type, tf.bool)
self.assertTrue(keep_me_expr.is_leaf)
child_node = expression_test_util.calculate_value_slowly(new_field)
self.assertEqual(child_node.size, 3)
self.assertTrue(child_node.is_repeated)
bar_node = expression_test_util.calculate_value_slowly(bar_expr)
self.assertEqual(bar_node.values.dtype, tf.string)
keep_me_node = expression_test_util.calculate_value_slowly(
keep_me_expr)
self.assertEqual(keep_me_node.values.dtype, tf.bool)
def test_broadcast_substructure(self):
"""Tests broadcast of a submessage.
The result of broadcasting `user` into `doc` looks like:
{
foo: 9,
foorepeated: [9],
doc: [{bar:["a"], keep_me:False, new_user: [{friends:["a"]}]}],
user: [{friends:["a"]}]
},
{
foo: 8,
foorepeated: [8, 7],
doc: [
{
bar: ["b","c"],
keep_me: True,
new_user: [{friends:["b", "c"]},{friends:["d"]}]
},
{
bar: ["d"],
new_user: [{friends:["b", "c"]},{friends:["d"]}]
}
],
user: [{friends:["b", "c"]},{friends:["d"]}],
},
{
foo: 7,
foorepeated: [6],
user: [{friends:["e"]}]
}
"""
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_big_prensor())
new_root = broadcast.broadcast(expr, path.Path(["user"]), "doc", "new_user")
new_user = new_root.get_child("doc").get_child("new_user")
self.assertIsNotNone(new_user)
self.assertTrue(new_user.is_repeated)
self.assertIsNone(new_user.type)
self.assertFalse(new_user.is_leaf)
new_user_node = expression_test_util.calculate_value_slowly(new_user)
self.assertAllEqual(new_user_node.parent_index, [0, 1, 1, 2, 2])
self.assertAllEqual(new_user_node.index_to_value, [0, 1, 2, 1, 2])
new_friends = new_user.get_child("friends")
self.assertIsNotNone(new_friends)
self.assertTrue(new_friends.is_repeated)
self.assertEqual(new_friends.type, tf.string)
self.assertTrue(new_friends.is_leaf)
new_friends_node = expression_test_util.calculate_value_slowly(new_friends)
self.assertEqual(new_friends_node.values.dtype, tf.string)
self.assertAllEqual(new_friends_node.values,
["a", "b", "c", "d", "b", "c", "d"])
self.assertAllEqual(new_friends_node.parent_index, [0, 1, 1, 2, 3, 3, 4])
def test_map_many_values(self):
with self.session(use_gpu=False) as sess:
expr = create_expression.create_expression_from_prensor(
prensor.create_prensor_from_descendant_nodes({
path.Path([]):
prensor_test_util.create_root_node(3),
path.Path(["foo"]):
prensor_test_util.create_optional_leaf_node([0, 2, 3],
[9, 8, 7]),
path.Path(["bar"]):
prensor_test_util.create_optional_leaf_node([0, 2, 3],
[10, 20, 30])
}))
new_root, p = map_values.map_many_values(expr, path.Path([]),
["foo", "bar"],
lambda x, y: x + y,
tf.int64, "new_field")
leaf_node = expression_test_util.calculate_value_slowly(
new_root.get_descendant_or_error(p))
[parent_index,
values] = sess.run([leaf_node.parent_index, leaf_node.values])
self.assertAllEqual(parent_index, [0, 2, 3])
self.assertAllEqual(values, [19, 28, 37])
def test_create_expression_from_proto_and_calculate_event_id_value(self):
"""Tests get_sparse_tensors on a deep tree."""
expr = proto_test_util._get_expression_from_session_empty_user_info()
event_id_value = expression_test_util.calculate_value_slowly(
expr.get_descendant_or_error(path.Path(["event", "event_id"])))
self.assertAllEqual(event_id_value.parent_index, [0, 1, 2, 4])
self.assertAllEqual(event_id_value.values, [b"A", b"B", b"C", b"D"])
def test_create_expression_from_proto_and_calculate_root_value(self):
"""Tests get_sparse_tensors on a deep tree."""
expr = proto_test_util._get_expression_from_session_empty_user_info()
root_value = expression_test_util.calculate_value_slowly(expr)
# For some reason, this fails on tf.eager. It could be because it is
# a scalar, I don't know.
self.assertEqual(self.evaluate(root_value.size), 2)
def test_create_proto_index_directly_reroot_at_action(self):
sessions = [
"""
event {
action {}
action {}
}
event {}
event { action {} }
""", "", """
event {}
event {
action {}
action {}
}
event { }
"""
]
expr = proto_test_util.text_to_expression(sessions, test_pb2.Session)
reroot_expr = expr.reroot("event.action")
# Reroot with a depth > 1 (all the other cases are depth == 1)
proto_index_directly_reroot_at_action = (
reroot_expr.create_proto_index(
"proto_index_directly_reroot_at_action").get_child_or_error(
"proto_index_directly_reroot_at_action"))
self.assertFalse(proto_index_directly_reroot_at_action.is_repeated)
result = expression_test_util.calculate_value_slowly(
proto_index_directly_reroot_at_action)
self.assertAllEqual(result.parent_index, [0, 1, 2, 3, 4])
self.assertAllEqual(result.values, [0, 0, 0, 2, 2])
def test_create_expression_from_proto_with_any(self):
"""Test an any field."""
expr = _get_expression_with_any()
any_expr = expr.get_child_or_error("my_any")
simple_expr = expr.get_descendant_or_error(
path.Path([
"my_any", "(type.googleapis.com/struct2tensor.test.AllSimple)"
]))
self.assertFalse(simple_expr.is_repeated)
self.assertIsNone(simple_expr.type)
self.assertFalse(simple_expr.is_leaf)
self.assertFalse(simple_expr.calculation_is_identity())
self.assertTrue(simple_expr.calculation_equal(simple_expr))
self.assertFalse(simple_expr.calculation_equal(expr))
child_node = expression_test_util.calculate_value_slowly(simple_expr)
self.assertEqual(child_node.parent_index.dtype, tf.int64)
self.assertEqual(
simple_expr.known_field_names(),
frozenset({
"optional_string", "optional_uint64", "repeated_uint64",
"repeated_int32", "repeated_string", "optional_int32",
"optional_float", "repeated_int64", "optional_uint32",
"repeated_float", "repeated_uint32", "optional_double",
"optional_int64", "repeated_double"
}))
sources = simple_expr.get_source_expressions()
self.assertLen(sources, 1)
self.assertIs(any_expr, sources[0])
def test_create_expression_from_proto_and_calculate_root_value(self):
"""Tests get_sparse_tensors on a deep tree."""
with self.session(use_gpu=False) as sess:
expr = proto_test_util._get_expression_from_session_empty_user_info(
)
root_value = expression_test_util.calculate_value_slowly(expr)
size = sess.run(root_value.size)
self.assertEqual(size, 2)
def test_size_anonymous(self):
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_big_prensor())
new_root, new_path = size.size_anonymous(expr,
path.Path(["doc", "bar"]))
new_field = new_root.get_descendant_or_error(new_path)
leaf_node = expression_test_util.calculate_value_slowly(new_field)
self.assertAllEqual(leaf_node.parent_index, [0, 1, 2])
self.assertAllEqual(leaf_node.values, [1, 2, 1])
def test_create_has_field(self):
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_big_prensor())
new_root = expr.create_has_field("doc.keep_me", "result")
new_field = new_root.get_descendant_or_error(
path.Path(["doc", "result"]))
leaf_node = expression_test_util.calculate_value_slowly(new_field)
self.assertAllEqual(leaf_node.parent_index, [0, 1, 2])
self.assertAllEqual(leaf_node.values, [True, True, False])
def test_get_positional_index_calculate(self):
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_nested_prensor())
new_root, new_path = index.get_positional_index(
expr, path.Path(["user", "friends"]), path.get_anonymous_field())
new_field = new_root.get_descendant_or_error(new_path)
leaf_node = expression_test_util.calculate_value_slowly(new_field)
self.assertAllEqual(leaf_node.parent_index, [0, 1, 1, 2, 3])
self.assertAllEqual(leaf_node.values, [0, 0, 1, 0, 0])
def test_create_expression_from_proto_and_calculate_event_value(self):
"""Tests get_sparse_tensors on a deep tree."""
with self.session(use_gpu=False) as sess:
expr = proto_test_util._get_expression_from_session_empty_user_info(
)
event_value = expression_test_util.calculate_value_slowly(
expr.get_child_or_error("event"))
parent_index = sess.run(event_value.parent_index)
self.assertAllEqual(parent_index, [0, 0, 0, 1, 1])
def test_size_missing_value(self):
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_big_prensor())
new_root = size.size(expr, path.Path(["doc", "keep_me"]), "result")
new_field = new_root.get_descendant_or_error(
path.Path(["doc", "result"]))
leaf_node = expression_test_util.calculate_value_slowly(new_field)
self.assertAllEqual(leaf_node.parent_index, [0, 1, 2])
self.assertAllEqual(leaf_node.values, [1, 1, 0])
def test_promote_and_calculate(self):
"""Tests promoting a leaf on a nested tree."""
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_nested_prensor())
new_root, new_path = promote.promote_anonymous(
expr, path.Path(["user", "friends"]))
new_field = new_root.get_descendant_or_error(new_path)
leaf_node = expression_test_util.calculate_value_slowly(new_field)
self.assertAllEqual(leaf_node.parent_index, [0, 1, 1, 1, 2])
self.assertAllEqual(leaf_node.values, [b"a", b"b", b"c", b"d", b"e"])
def test_create_expression_from_proto_and_calculate_event_value(
self, use_string_view):
"""Tests get_sparse_tensors on a deep tree."""
expr = proto_test_util._get_expression_from_session_empty_user_info()
event_value = expression_test_util.calculate_value_slowly(
expr.get_child_or_error("event"),
options=self._get_calculate_options(use_string_view))
self.assertAllEqual(event_value.parent_index, [0, 0, 0, 1, 1])
if use_string_view:
self._check_string_view()
def test_create_expression_from_proto_with_any(self):
"""Test an any field."""
expr = _get_expression_with_any()
simple_expr = expr.get_descendant_or_error(
path.Path([
"my_any", "(type.googleapis.com/struct2tensor.test.AllSimple)"
]))
child_node = expression_test_util.calculate_value_slowly(
simple_expr).parent_index
self.assertAllEqual(child_node, [0, 2])
def test_broadcast_and_calculate(self):
"""Tests get_sparse_tensors on a deep tree."""
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_big_prensor())
new_root, new_path = broadcast.broadcast_anonymous(
expr, path.Path(["foo"]), "user")
new_field = new_root.get_descendant_or_error(new_path)
leaf_node = expression_test_util.calculate_value_slowly(new_field)
self.assertAllEqual(leaf_node.parent_index, [0, 1, 2, 3])
self.assertAllEqual(leaf_node.values, [9, 8, 8, 7])
def test_create_size_field(self):
with self.session(use_gpu=False) as sess:
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_big_prensor())
new_root = expr.create_size_field("doc.bar", "result")
new_field = new_root.get_descendant_or_error(path.Path(["doc", "result"]))
leaf_node = expression_test_util.calculate_value_slowly(new_field)
[parent_index,
values] = sess.run([leaf_node.parent_index, leaf_node.values])
self.assertAllEqual(parent_index, [0, 1, 2])
self.assertAllEqual(values, [1, 2, 1])
def test_create_expression_from_proto_with_any_missing_message(self):
"""Test an any field with a message that is absent."""
expr = _get_expression_with_any()
simple_expr = expr.get_descendant_or_error(
path.Path([
"my_any",
"(type.googleapis.com/struct2tensor.test.SpecialUserInfo)"
]))
child_node = expression_test_util.calculate_value_slowly(
simple_expr).parent_index
self.assertAllEqual(child_node, [])
def _test_runner(options):
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_simple_prensor())
new_root = map_prensor.map_sparse_tensor(expr, path.Path([]),
[path.Path(["foo"])],
lambda x: x * 2, True,
tf.int32, "foo_doubled")
leaf_node = expression_test_util.calculate_value_slowly(
new_root.get_descendant_or_error(path.Path(["foo_doubled"])),
options=options)
self.evaluate(leaf_node.parent_index)
self.evaluate(leaf_node.values)
def test_map_field_values_test(self):
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_simple_prensor())
new_root = expr.map_field_values("foo", lambda x: x * 2, tf.int64,
"foo_doubled")
leaf_node = expression_test_util.calculate_value_slowly(
new_root.get_descendant_or_error(path.Path(["foo_doubled"])))
self.assertAllEqual(leaf_node.parent_index, [0, 1, 2])
self.assertAllEqual(leaf_node.values, [18, 16, 14])
def test_promote(self):
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_nested_prensor())
new_root = expr.promote("user.friends", "new_field")
new_field = new_root.get_child_or_error("new_field")
self.assertIsNotNone(new_field)
self.assertTrue(new_field.is_repeated)
self.assertEqual(new_field.type, tf.string)
self.assertTrue(new_field.is_leaf)
leaf_node = expression_test_util.calculate_value_slowly(new_field)
self.assertEqual(leaf_node.values.dtype, tf.string)
self.assertEqual(new_field.known_field_names(), frozenset())
def test_map_values_anonymous(self):
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_simple_prensor())
new_root, p = map_values.map_values_anonymous(expr, path.Path(["foo"]),
lambda x: x * 2,
tf.int64)
leaf_node = expression_test_util.calculate_value_slowly(
new_root.get_descendant_or_error(p))
self.assertAllEqual(leaf_node.parent_index, [0, 1, 2])
self.assertAllEqual(leaf_node.values, [18, 16, 14])
def test_promote_and_calculate_leaf_then_substructure(self):
"""Tests promoting of leaf and then a substructure."""
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_four_layer_prensor())
new_root, new_bar_path = promote.promote_anonymous(
expr, path.Path(["event", "doc", "nested_child", "bar"]))
new_root, new_path = promote.promote_anonymous(
new_root, path.Path(["event", "doc"]))
new_doc = new_root.get_descendant_or_error(new_path)
new_bar = new_root.get_descendant_or_error(
new_path.concat(new_bar_path.suffix(2)))
bar_expr = new_root.get_descendant_or_error(
new_path.concat(path.Path(["nested_child", "bar"])))
keep_me_expr = new_root.get_descendant_or_error(
new_path.concat(path.Path(["nested_child", "keep_me"])))
new_doc_node = expression_test_util.calculate_value_slowly(new_doc)
self.assertAllEqual(new_doc_node.parent_index, [0, 1, 1])
self.assertTrue(new_doc_node.is_repeated)
# new_bar's parent index is changed (from the first promote).
# The second promote should not change new_bar's parent index.
new_bar_node = expression_test_util.calculate_value_slowly(new_bar)
self.assertAllEqual(new_bar_node.parent_index, [0, 1, 1, 1])
self.assertAllEqual(new_bar_node.values, [b"a", b"b", b"c", b"d"])
self.assertTrue(new_bar_node.is_repeated)
# bar's parent index should be unchanged.
bar_node = expression_test_util.calculate_value_slowly(bar_expr)
self.assertAllEqual(bar_node.parent_index, [0, 1, 1, 2])
self.assertAllEqual(bar_node.values, [b"a", b"b", b"c", b"d"])
self.assertTrue(bar_node.is_repeated)
# keep_me's parent index should be unchanged.
keep_me_node = expression_test_util.calculate_value_slowly(
keep_me_expr)
self.assertAllEqual(keep_me_node.parent_index, [0, 1])
self.assertAllEqual(keep_me_node.values, [False, True])
self.assertFalse(keep_me_node.is_repeated)
def test_broadcast(self):
"""Tests broadcast.broadcast(...), and indirectly tests set_path."""
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_big_prensor())
new_root = expr.broadcast("foo", "user", "new_field")
new_field = new_root.get_child("user").get_child("new_field")
self.assertIsNotNone(new_field)
self.assertFalse(new_field.is_repeated)
self.assertEqual(new_field.type, tf.int32)
self.assertTrue(new_field.is_leaf)
leaf_node = expression_test_util.calculate_value_slowly(new_field)
self.assertEqual(leaf_node.values.dtype, tf.int32)
self.assertEqual(new_field.known_field_names(), frozenset())
def test_user_info_with_extension(self):
expr = _get_user_info_with_extension()
ext_expr = expr.get_child_or_error(
"(struct2tensor.test.MyExternalExtension.ext)")
self.assertFalse(ext_expr.is_repeated)
self.assertIsNone(ext_expr.type)
self.assertFalse(ext_expr.is_leaf)
self.assertFalse(ext_expr.calculation_is_identity())
self.assertTrue(ext_expr.calculation_equal(ext_expr))
self.assertFalse(ext_expr.calculation_equal(expr))
child_node = expression_test_util.calculate_value_slowly(ext_expr)
self.assertEqual(child_node.parent_index.dtype, tf.int64)
self.assertEqual(ext_expr.known_field_names(), frozenset({"special"}))
def test_promote_substructure_then_leaf(self):
"""Tests expr.promote(...) of substructure and then a leaf."""
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_deep_prensor())
new_root = (expr.promote(path.Path(["event", "doc"]),
"new_field").promote(
path.Path(["new_field", "bar"]),
"new_bar"))
new_bar = new_root.get_child_or_error("new_bar")
self.assertIsNotNone(new_bar)
self.assertTrue(new_bar.is_repeated)
self.assertEqual(new_bar.type, tf.string)
self.assertTrue(new_bar.is_leaf)
new_field_bar = new_root.get_descendant_or_error(
path.Path(["new_field", "bar"]))
self.assertIsNotNone(new_field_bar)
self.assertTrue(new_bar.is_repeated)
self.assertEqual(new_bar.type, tf.string)
self.assertTrue(new_bar.is_leaf)
new_field_keep_me = new_root.get_descendant_or_error(
path.Path(["new_field", "keep_me"]))
self.assertIsNotNone(new_field_keep_me)
self.assertFalse(new_field_keep_me.is_repeated)
self.assertEqual(new_field_keep_me.type, tf.bool)
self.assertTrue(new_field_keep_me.is_leaf)
bar_node = expression_test_util.calculate_value_slowly(new_bar)
self.assertEqual(bar_node.values.dtype, tf.string)
new_field_bar_node = expression_test_util.calculate_value_slowly(
new_field_bar)
self.assertEqual(new_field_bar_node.values.dtype, tf.string)
new_field_keep_me_node = expression_test_util.calculate_value_slowly(
new_field_keep_me)
self.assertEqual(new_field_keep_me_node.values.dtype, tf.bool)
def test_map_sparse_tensor(self):
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_simple_prensor())
new_root = map_prensor.map_sparse_tensor(expr, path.Path([]),
[path.Path(["foo"])],
lambda x: x * 2, False,
tf.int32, "foo_doubled")
leaf_node = expression_test_util.calculate_value_slowly(
new_root.get_descendant_or_error(path.Path(["foo_doubled"])))
self.assertAllEqual(leaf_node.parent_index, [0, 1, 2])
self.assertAllEqual(leaf_node.values, [18, 16, 14])
