def test_prensor_children_ordered(self):
def _recursively_check_sorted(p):
self.assertEqual(list(p.get_children().keys()),
sorted(p.get_children().keys()))
for c in p.get_children().values():
_recursively_check_sorted(c)
for pren in [
prensor_test_util.create_nested_prensor(),
prensor_test_util.create_big_prensor(),
prensor_test_util.create_deep_prensor()
]:
_recursively_check_sorted(pren)
p = prensor.create_prensor_from_descendant_nodes({
path.Path([]):
prensor_test_util.create_root_node(1),
path.Path(["d"]):
prensor_test_util.create_optional_leaf_node([0], [True]),
path.Path(["c"]):
prensor_test_util.create_optional_leaf_node([0], [True]),
path.Path(["b"]):
prensor_test_util.create_optional_leaf_node([0], [True]),
path.Path(["a"]):
prensor_test_util.create_optional_leaf_node([0], [True]),
})
self.assertEqual(["a", "b", "c", "d"], list(p.get_children().keys()))
def test_promote_substructure(self):
"""Tests promote.promote(...) of substructure."""
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_deep_prensor())
new_root = promote.promote(expr, path.Path(["event", "doc"]),
"new_field")
new_field = new_root.get_child_or_error("new_field")
self.assertIsNotNone(new_field)
self.assertTrue(new_field.is_repeated)
self.assertEqual(new_field.known_field_names(),
frozenset(["bar", "keep_me"]))
bar_expr = new_field.get_child_or_error("bar")
self.assertIsNotNone(bar_expr)
self.assertTrue(bar_expr.is_repeated)
self.assertEqual(bar_expr.type, tf.string)
self.assertTrue(bar_expr.is_leaf)
keep_me_expr = new_field.get_child_or_error("keep_me")
self.assertIsNotNone(keep_me_expr)
self.assertFalse(keep_me_expr.is_repeated)
self.assertEqual(keep_me_expr.type, tf.bool)
self.assertTrue(keep_me_expr.is_leaf)
child_node = expression_test_util.calculate_value_slowly(new_field)
self.assertEqual(child_node.size, 3)
self.assertTrue(child_node.is_repeated)
bar_node = expression_test_util.calculate_value_slowly(bar_expr)
self.assertEqual(bar_node.values.dtype, tf.string)
keep_me_node = expression_test_util.calculate_value_slowly(
keep_me_expr)
self.assertEqual(keep_me_node.values.dtype, tf.bool)
def test_promote_substructure_with_schema(self):
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_deep_prensor()).apply_schema(
prensor_test_util.create_deep_prensor_schema())
original_schema = expr.get_descendant_or_error(
path.Path(["event", "doc"])).schema_feature
new_root, new_field_path = promote.promote_anonymous(
expr, path.Path(["event", "doc"]))
new_field = new_root.get_descendant_or_error(new_field_path)
new_schema_feature = new_field.schema_feature
self.assertIsNotNone(new_schema_feature)
# The struct_domain of this feature should not be changed.
self.assertProtoEquals(new_schema_feature.struct_domain,
original_schema.struct_domain)
bar_schema = new_root.get_descendant_or_error(
new_field_path.concat(path.Path(["bar"]))).schema_feature
self.assertIsNotNone(bar_schema)
self.assertEqual(bar_schema.string_domain.value[0], "a")
keep_me_schema = new_root.get_descendant_or_error(
new_field_path.concat(path.Path(["keep_me"]))).schema_feature
self.assertIsNotNone(keep_me_schema)
self.assertEqual(keep_me_schema.presence.min_count, 1)
def test_reroot_and_create_proto_index_deep(self):
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_deep_prensor())
new_root = reroot.reroot(expr, path.Path(["event", "doc"]))
proto_index = reroot.create_proto_index_field(
new_root, "proto_index").get_child("proto_index")
new_field = new_root.get_child("bar")
leaf_node = expression_test_util.calculate_value_slowly(new_field)
proto_index_node = expression_test_util.calculate_value_slowly(
proto_index)
self.assertIsNotNone(new_field)
self.assertTrue(new_field.is_repeated)
self.assertEqual(new_field.type, tf.string)
self.assertTrue(new_field.is_leaf)
self.assertEqual(new_field.known_field_names(), frozenset())
self.assertEqual(leaf_node.values.dtype, tf.string)
self.assertIsNotNone(proto_index)
self.assertFalse(proto_index.is_repeated)
self.assertEqual(proto_index.type, tf.int64)
self.assertTrue(proto_index.is_leaf)
self.assertEqual(proto_index.known_field_names(), frozenset())
self.assertEqual(proto_index_node.values.dtype, tf.int64)
self.assertAllEqual([b"a", b"b", b"c", b"d"], leaf_node.values)
self.assertAllEqual([0, 1, 1, 2], leaf_node.parent_index)
self.assertAllEqual([0, 1, 1], proto_index_node.values)
self.assertAllEqual([0, 1, 2], proto_index_node.parent_index)
def test_prensor_is_composite_tensor(self):
for pren in [
prensor_test_util.create_nested_prensor(),
prensor_test_util.create_big_prensor(),
prensor_test_util.create_deep_prensor()
]:
flattened_tensors = tf.nest.flatten(pren, expand_composites=True)
self.assertIsInstance(flattened_tensors, list)
for t in flattened_tensors:
self.assertIsInstance(t, tf.Tensor)
packed_pren = tf.nest.pack_sequence_as(pren,
flattened_tensors,
expand_composites=True)
self._assert_prensor_equals(pren, packed_pren)
def test_promote_substructure_then_leaf(self):
"""Tests expr.promote(...) of substructure and then a leaf."""
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_deep_prensor())
new_root = (expr.promote(path.Path(["event", "doc"]),
"new_field").promote(
path.Path(["new_field", "bar"]),
"new_bar"))
new_bar = new_root.get_child_or_error("new_bar")
self.assertIsNotNone(new_bar)
self.assertTrue(new_bar.is_repeated)
self.assertEqual(new_bar.type, tf.string)
self.assertTrue(new_bar.is_leaf)
new_field_bar = new_root.get_descendant_or_error(
path.Path(["new_field", "bar"]))
self.assertIsNotNone(new_field_bar)
self.assertTrue(new_bar.is_repeated)
self.assertEqual(new_bar.type, tf.string)
self.assertTrue(new_bar.is_leaf)
new_field_keep_me = new_root.get_descendant_or_error(
path.Path(["new_field", "keep_me"]))
self.assertIsNotNone(new_field_keep_me)
self.assertFalse(new_field_keep_me.is_repeated)
self.assertEqual(new_field_keep_me.type, tf.bool)
self.assertTrue(new_field_keep_me.is_leaf)
bar_node = expression_test_util.calculate_value_slowly(new_bar)
self.assertEqual(bar_node.values.dtype, tf.string)
new_field_bar_node = expression_test_util.calculate_value_slowly(
new_field_bar)
self.assertEqual(new_field_bar_node.values.dtype, tf.string)
new_field_keep_me_node = expression_test_util.calculate_value_slowly(
new_field_keep_me)
self.assertEqual(new_field_keep_me_node.values.dtype, tf.bool)
def test_deep_prensor(self):
"""Test a prensor with three layers: root, event, and doc.
a prensor expression representing:
{foo:9, foorepeated:[9], user:[{friends:["a"]}],
event:{doc:[{bar:["a"], keep_me:False}]}}
{foo:8, foorepeated:[8,7],
event:{doc:[{bar:["b","c"], keep_me:True},{bar:["d"]}]},
user:[{friends:["b", "c"]}, {friends:["d"]}]}
{foo:7, foorepeated:[6], user:[friends:["e"]], event:{}}
"""
pren = prensor_test_util.create_deep_prensor()
st = prensor_to_structured_tensor.prensor_to_structured_tensor(pren)
self.assertAllEqual(st.field_value("foo"), [[9], [8], [7]])
self.assertAllEqual(st.field_value(["foorepeated"]),
[[9], [8, 7], [6]])
self.assertAllEqual(st.field_value(["user", "friends"]),
[[[b"a"]], [[b"b", b"c"], [b"d"]], [[b"e"]]])
self.assertAllEqual(st.field_value(["event", "doc", "bar"]),
[[[[b"a"]]], [[[b"b", b"c"], [b"d"]]], [[]]])
self.assertAllEqual(st.field_value(["event", "doc", "keep_me"]),
[[[[False]]], [[[True], []]], [[]]])
def test_depth_limit_2(self):
"""Tests depth_limit with a limit of 2.
Starting with a prensor expression representing:
{foo:9, foorepeated:[9], user:[{friends:["a"]}],
event:{doc:[{bar:["a"], keep_me:False}]}}
{foo:8, foorepeated:[8,7],
event:{doc:[{bar:["b","c"], keep_me:True},{bar:["d"]}]},
user:[{friends:["b", "c"]}, {friends:["d"]}]}
{foo:7, foorepeated:[6], user:[friends:["e"]], event:{}}
After depth_limit.limit_depth(expr, 2), you lose event.doc.bar
and event.doc.keep_me:
{foo:9, foorepeated:[9], user:[{friends:["a"]}], event:{doc:[{}]}}
{foo:8, foorepeated:[8,7], event:{doc:[{},{}]},
user:[{friends:["b", "c"]}, {friends:["d"]}]}
{foo:7, foorepeated:[6], user:[friends:["e"]], event:{}}
"""
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_deep_prensor())
new_root = depth_limit.limit_depth(expr, 2)
self.assertIsNone(
new_root.get_descendant(path.Path(["event", "doc", "bar"])))
self.assertIsNone(
new_root.get_descendant(path.Path(["event", "doc", "keep_me"])))
self.assertIsNotNone(new_root.get_descendant(path.Path(["foo"])))
self.assertIsNotNone(
new_root.get_descendant(path.Path(["foorepeated"])))
self.assertIsNotNone(new_root.get_descendant(path.Path(["user"])))
self.assertIsNotNone(
new_root.get_descendant(path.Path(["user", "friends"])))
self.assertIsNotNone(new_root.get_descendant(path.Path(["event"])))
self.assertIsNotNone(
new_root.get_descendant(path.Path(["event", "doc"])))
def test_broadcast_substructure_deep(self):
"""Tests broadcast of a submessage.
The result of broadcasting `event` into `user` looks like:
{
foo: 9,
foorepeated: [9],
user: [{
friends: ["a"],
new_event: [{
doc:[{
bar: ["a"],
keep_me:False
}]
}]
}],
event: [{doc:[{bar:["a"], keep_me:False}]}]
},
{
foo: 8,
foorepeated: [8,7],
user: [{
friends: ["b", "c"],
new_event: [{
doc:[{
bar: ["b","c"],
keep_me:True
},
{
bar:["d"]
}]
}]
},
{
friends: ["d"],
new_event: [{
doc:[{
bar: ["b","c"],
keep_me: True
},
{
bar: ["d"]
}]
}]
}],
event: [{doc:[{bar:["b","c"], keep_me:True},{bar:["d"]}]}]
},
{
foo:7,
foorepeated: [6],
user: [{
friends:["e"],
new_event: [{}]
}],
event: [{}]
}
"""
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_deep_prensor())
new_root = broadcast.broadcast(expr, path.Path(["event"]), "user",
"new_event")
new_event = new_root.get_child("user").get_child("new_event")
self.assertIsNotNone(new_event)
self.assertTrue(new_event.is_repeated)
self.assertIsNone(new_event.type)
self.assertFalse(new_event.is_leaf)
new_event_node = expression_test_util.calculate_value_slowly(new_event)
self.assertAllEqual(new_event_node.parent_index, [0, 1, 2, 3])
self.assertAllEqual(new_event_node.index_to_value, [0, 1, 1, 2])
new_doc = new_event.get_child("doc")
self.assertIsNotNone(new_doc)
self.assertTrue(new_doc.is_repeated)
self.assertIsNone(new_doc.type)
self.assertFalse(new_doc.is_leaf)
new_doc_node = expression_test_util.calculate_value_slowly(new_doc)
self.assertAllEqual(new_doc_node.parent_index, [0, 1, 1, 2, 2])
self.assertAllEqual(new_doc_node.index_to_value, [0, 1, 2, 1, 2])
new_bar = new_doc.get_child("bar")
self.assertIsNotNone(new_bar)
self.assertTrue(new_doc.is_repeated)
self.assertEqual(new_bar.type, tf.string)
self.assertTrue(new_bar.is_leaf)
new_bar_node = expression_test_util.calculate_value_slowly(new_bar)
self.assertAllEqual(new_bar_node.values,
["a", "b", "c", "d", "b", "c", "d"])
self.assertAllEqual(new_bar_node.parent_index, [0, 1, 1, 2, 3, 3, 4])
new_keep_me = new_doc.get_child("keep_me")
self.assertIsNotNone(new_keep_me)
self.assertFalse(new_keep_me.is_repeated)
self.assertEqual(new_keep_me.type, tf.bool)
self.assertTrue(new_keep_me.is_leaf)
new_keep_me_node = expression_test_util.calculate_value_slowly(new_keep_me)
self.assertAllEqual(new_keep_me_node.values,
[False, True, True])
self.assertAllEqual(new_keep_me_node.parent_index, [0, 1, 3])
