def test_prensor_children_ordered(self): def _recursively_check_sorted(p): self.assertEqual(list(p.get_children().keys()), sorted(p.get_children().keys())) for c in p.get_children().values(): _recursively_check_sorted(c) for pren in [ prensor_test_util.create_nested_prensor(), prensor_test_util.create_big_prensor(), prensor_test_util.create_deep_prensor() ]: _recursively_check_sorted(pren) p = prensor.create_prensor_from_descendant_nodes({ path.Path([]): prensor_test_util.create_root_node(1), path.Path(["d"]): prensor_test_util.create_optional_leaf_node([0], [True]), path.Path(["c"]): prensor_test_util.create_optional_leaf_node([0], [True]), path.Path(["b"]): prensor_test_util.create_optional_leaf_node([0], [True]), path.Path(["a"]): prensor_test_util.create_optional_leaf_node([0], [True]), }) self.assertEqual(["a", "b", "c", "d"], list(p.get_children().keys()))
def test_promote_substructure(self): """Tests promote.promote(...) of substructure.""" expr = create_expression.create_expression_from_prensor( prensor_test_util.create_deep_prensor()) new_root = promote.promote(expr, path.Path(["event", "doc"]), "new_field") new_field = new_root.get_child_or_error("new_field") self.assertIsNotNone(new_field) self.assertTrue(new_field.is_repeated) self.assertEqual(new_field.known_field_names(), frozenset(["bar", "keep_me"])) bar_expr = new_field.get_child_or_error("bar") self.assertIsNotNone(bar_expr) self.assertTrue(bar_expr.is_repeated) self.assertEqual(bar_expr.type, tf.string) self.assertTrue(bar_expr.is_leaf) keep_me_expr = new_field.get_child_or_error("keep_me") self.assertIsNotNone(keep_me_expr) self.assertFalse(keep_me_expr.is_repeated) self.assertEqual(keep_me_expr.type, tf.bool) self.assertTrue(keep_me_expr.is_leaf) child_node = expression_test_util.calculate_value_slowly(new_field) self.assertEqual(child_node.size, 3) self.assertTrue(child_node.is_repeated) bar_node = expression_test_util.calculate_value_slowly(bar_expr) self.assertEqual(bar_node.values.dtype, tf.string) keep_me_node = expression_test_util.calculate_value_slowly( keep_me_expr) self.assertEqual(keep_me_node.values.dtype, tf.bool)
def test_promote_substructure_with_schema(self): expr = create_expression.create_expression_from_prensor( prensor_test_util.create_deep_prensor()).apply_schema( prensor_test_util.create_deep_prensor_schema()) original_schema = expr.get_descendant_or_error( path.Path(["event", "doc"])).schema_feature new_root, new_field_path = promote.promote_anonymous( expr, path.Path(["event", "doc"])) new_field = new_root.get_descendant_or_error(new_field_path) new_schema_feature = new_field.schema_feature self.assertIsNotNone(new_schema_feature) # The struct_domain of this feature should not be changed. self.assertProtoEquals(new_schema_feature.struct_domain, original_schema.struct_domain) bar_schema = new_root.get_descendant_or_error( new_field_path.concat(path.Path(["bar"]))).schema_feature self.assertIsNotNone(bar_schema) self.assertEqual(bar_schema.string_domain.value[0], "a") keep_me_schema = new_root.get_descendant_or_error( new_field_path.concat(path.Path(["keep_me"]))).schema_feature self.assertIsNotNone(keep_me_schema) self.assertEqual(keep_me_schema.presence.min_count, 1)
def test_reroot_and_create_proto_index_deep(self): expr = create_expression.create_expression_from_prensor( prensor_test_util.create_deep_prensor()) new_root = reroot.reroot(expr, path.Path(["event", "doc"])) proto_index = reroot.create_proto_index_field( new_root, "proto_index").get_child("proto_index") new_field = new_root.get_child("bar") leaf_node = expression_test_util.calculate_value_slowly(new_field) proto_index_node = expression_test_util.calculate_value_slowly( proto_index) self.assertIsNotNone(new_field) self.assertTrue(new_field.is_repeated) self.assertEqual(new_field.type, tf.string) self.assertTrue(new_field.is_leaf) self.assertEqual(new_field.known_field_names(), frozenset()) self.assertEqual(leaf_node.values.dtype, tf.string) self.assertIsNotNone(proto_index) self.assertFalse(proto_index.is_repeated) self.assertEqual(proto_index.type, tf.int64) self.assertTrue(proto_index.is_leaf) self.assertEqual(proto_index.known_field_names(), frozenset()) self.assertEqual(proto_index_node.values.dtype, tf.int64) self.assertAllEqual([b"a", b"b", b"c", b"d"], leaf_node.values) self.assertAllEqual([0, 1, 1, 2], leaf_node.parent_index) self.assertAllEqual([0, 1, 1], proto_index_node.values) self.assertAllEqual([0, 1, 2], proto_index_node.parent_index)
def test_prensor_is_composite_tensor(self): for pren in [ prensor_test_util.create_nested_prensor(), prensor_test_util.create_big_prensor(), prensor_test_util.create_deep_prensor() ]: flattened_tensors = tf.nest.flatten(pren, expand_composites=True) self.assertIsInstance(flattened_tensors, list) for t in flattened_tensors: self.assertIsInstance(t, tf.Tensor) packed_pren = tf.nest.pack_sequence_as(pren, flattened_tensors, expand_composites=True) self._assert_prensor_equals(pren, packed_pren)
def test_promote_substructure_then_leaf(self): """Tests expr.promote(...) of substructure and then a leaf.""" expr = create_expression.create_expression_from_prensor( prensor_test_util.create_deep_prensor()) new_root = (expr.promote(path.Path(["event", "doc"]), "new_field").promote( path.Path(["new_field", "bar"]), "new_bar")) new_bar = new_root.get_child_or_error("new_bar") self.assertIsNotNone(new_bar) self.assertTrue(new_bar.is_repeated) self.assertEqual(new_bar.type, tf.string) self.assertTrue(new_bar.is_leaf) new_field_bar = new_root.get_descendant_or_error( path.Path(["new_field", "bar"])) self.assertIsNotNone(new_field_bar) self.assertTrue(new_bar.is_repeated) self.assertEqual(new_bar.type, tf.string) self.assertTrue(new_bar.is_leaf) new_field_keep_me = new_root.get_descendant_or_error( path.Path(["new_field", "keep_me"])) self.assertIsNotNone(new_field_keep_me) self.assertFalse(new_field_keep_me.is_repeated) self.assertEqual(new_field_keep_me.type, tf.bool) self.assertTrue(new_field_keep_me.is_leaf) bar_node = expression_test_util.calculate_value_slowly(new_bar) self.assertEqual(bar_node.values.dtype, tf.string) new_field_bar_node = expression_test_util.calculate_value_slowly( new_field_bar) self.assertEqual(new_field_bar_node.values.dtype, tf.string) new_field_keep_me_node = expression_test_util.calculate_value_slowly( new_field_keep_me) self.assertEqual(new_field_keep_me_node.values.dtype, tf.bool)
def test_deep_prensor(self): """Test a prensor with three layers: root, event, and doc. a prensor expression representing: {foo:9, foorepeated:[9], user:[{friends:["a"]}], event:{doc:[{bar:["a"], keep_me:False}]}} {foo:8, foorepeated:[8,7], event:{doc:[{bar:["b","c"], keep_me:True},{bar:["d"]}]}, user:[{friends:["b", "c"]}, {friends:["d"]}]} {foo:7, foorepeated:[6], user:[friends:["e"]], event:{}} """ pren = prensor_test_util.create_deep_prensor() st = prensor_to_structured_tensor.prensor_to_structured_tensor(pren) self.assertAllEqual(st.field_value("foo"), [[9], [8], [7]]) self.assertAllEqual(st.field_value(["foorepeated"]), [[9], [8, 7], [6]]) self.assertAllEqual(st.field_value(["user", "friends"]), [[[b"a"]], [[b"b", b"c"], [b"d"]], [[b"e"]]]) self.assertAllEqual(st.field_value(["event", "doc", "bar"]), [[[[b"a"]]], [[[b"b", b"c"], [b"d"]]], [[]]]) self.assertAllEqual(st.field_value(["event", "doc", "keep_me"]), [[[[False]]], [[[True], []]], [[]]])
def test_depth_limit_2(self): """Tests depth_limit with a limit of 2. Starting with a prensor expression representing: {foo:9, foorepeated:[9], user:[{friends:["a"]}], event:{doc:[{bar:["a"], keep_me:False}]}} {foo:8, foorepeated:[8,7], event:{doc:[{bar:["b","c"], keep_me:True},{bar:["d"]}]}, user:[{friends:["b", "c"]}, {friends:["d"]}]} {foo:7, foorepeated:[6], user:[friends:["e"]], event:{}} After depth_limit.limit_depth(expr, 2), you lose event.doc.bar and event.doc.keep_me: {foo:9, foorepeated:[9], user:[{friends:["a"]}], event:{doc:[{}]}} {foo:8, foorepeated:[8,7], event:{doc:[{},{}]}, user:[{friends:["b", "c"]}, {friends:["d"]}]} {foo:7, foorepeated:[6], user:[friends:["e"]], event:{}} """ expr = create_expression.create_expression_from_prensor( prensor_test_util.create_deep_prensor()) new_root = depth_limit.limit_depth(expr, 2) self.assertIsNone( new_root.get_descendant(path.Path(["event", "doc", "bar"]))) self.assertIsNone( new_root.get_descendant(path.Path(["event", "doc", "keep_me"]))) self.assertIsNotNone(new_root.get_descendant(path.Path(["foo"]))) self.assertIsNotNone( new_root.get_descendant(path.Path(["foorepeated"]))) self.assertIsNotNone(new_root.get_descendant(path.Path(["user"]))) self.assertIsNotNone( new_root.get_descendant(path.Path(["user", "friends"]))) self.assertIsNotNone(new_root.get_descendant(path.Path(["event"]))) self.assertIsNotNone( new_root.get_descendant(path.Path(["event", "doc"])))
def test_broadcast_substructure_deep(self): """Tests broadcast of a submessage. The result of broadcasting `event` into `user` looks like: { foo: 9, foorepeated: [9], user: [{ friends: ["a"], new_event: [{ doc:[{ bar: ["a"], keep_me:False }] }] }], event: [{doc:[{bar:["a"], keep_me:False}]}] }, { foo: 8, foorepeated: [8,7], user: [{ friends: ["b", "c"], new_event: [{ doc:[{ bar: ["b","c"], keep_me:True }, { bar:["d"] }] }] }, { friends: ["d"], new_event: [{ doc:[{ bar: ["b","c"], keep_me: True }, { bar: ["d"] }] }] }], event: [{doc:[{bar:["b","c"], keep_me:True},{bar:["d"]}]}] }, { foo:7, foorepeated: [6], user: [{ friends:["e"], new_event: [{}] }], event: [{}] } """ expr = create_expression.create_expression_from_prensor( prensor_test_util.create_deep_prensor()) new_root = broadcast.broadcast(expr, path.Path(["event"]), "user", "new_event") new_event = new_root.get_child("user").get_child("new_event") self.assertIsNotNone(new_event) self.assertTrue(new_event.is_repeated) self.assertIsNone(new_event.type) self.assertFalse(new_event.is_leaf) new_event_node = expression_test_util.calculate_value_slowly(new_event) self.assertAllEqual(new_event_node.parent_index, [0, 1, 2, 3]) self.assertAllEqual(new_event_node.index_to_value, [0, 1, 1, 2]) new_doc = new_event.get_child("doc") self.assertIsNotNone(new_doc) self.assertTrue(new_doc.is_repeated) self.assertIsNone(new_doc.type) self.assertFalse(new_doc.is_leaf) new_doc_node = expression_test_util.calculate_value_slowly(new_doc) self.assertAllEqual(new_doc_node.parent_index, [0, 1, 1, 2, 2]) self.assertAllEqual(new_doc_node.index_to_value, [0, 1, 2, 1, 2]) new_bar = new_doc.get_child("bar") self.assertIsNotNone(new_bar) self.assertTrue(new_doc.is_repeated) self.assertEqual(new_bar.type, tf.string) self.assertTrue(new_bar.is_leaf) new_bar_node = expression_test_util.calculate_value_slowly(new_bar) self.assertAllEqual(new_bar_node.values, ["a", "b", "c", "d", "b", "c", "d"]) self.assertAllEqual(new_bar_node.parent_index, [0, 1, 1, 2, 3, 3, 4]) new_keep_me = new_doc.get_child("keep_me") self.assertIsNotNone(new_keep_me) self.assertFalse(new_keep_me.is_repeated) self.assertEqual(new_keep_me.type, tf.bool) self.assertTrue(new_keep_me.is_leaf) new_keep_me_node = expression_test_util.calculate_value_slowly(new_keep_me) self.assertAllEqual(new_keep_me_node.values, [False, True, True]) self.assertAllEqual(new_keep_me_node.parent_index, [0, 1, 3])