def test_promote_substructure_with_schema(self):
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_deep_prensor()).apply_schema(
prensor_test_util.create_deep_prensor_schema())
original_schema = expr.get_descendant_or_error(
path.Path(["event", "doc"])).schema_feature
new_root, new_field_path = promote.promote_anonymous(
expr, path.Path(["event", "doc"]))
new_field = new_root.get_descendant_or_error(new_field_path)
new_schema_feature = new_field.schema_feature
self.assertIsNotNone(new_schema_feature)
# The struct_domain of this feature should not be changed.
self.assertProtoEquals(new_schema_feature.struct_domain,
original_schema.struct_domain)
bar_schema = new_root.get_descendant_or_error(
new_field_path.concat(path.Path(["bar"]))).schema_feature
self.assertIsNotNone(bar_schema)
self.assertEqual(bar_schema.string_domain.value[0], "a")
keep_me_schema = new_root.get_descendant_or_error(
new_field_path.concat(path.Path(["keep_me"]))).schema_feature
self.assertIsNotNone(keep_me_schema)
self.assertEqual(keep_me_schema.presence.min_count, 1)
def test_promote_and_calculate_substructure(self):
"""Tests promoting substructure on a tree with depth of 4."""
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_four_layer_prensor())
new_root, new_path = promote.promote_anonymous(
expr, path.Path(["event", "doc", "nested_child"]))
new_nested_child = new_root.get_descendant_or_error(new_path)
bar_expr = new_root.get_descendant_or_error(new_path.get_child("bar"))
keep_me_expr = new_root.get_descendant_or_error(
new_path.get_child("keep_me"))
# the promoted nested_child's parent index is changed.
nested_child_node = expression_test_util.calculate_value_slowly(
new_nested_child)
self.assertAllEqual(nested_child_node.parent_index, [0, 1, 1, 1])
self.assertTrue(nested_child_node.is_repeated)
# bar's parent index should be unchanged.
bar_node = expression_test_util.calculate_value_slowly(bar_expr)
self.assertAllEqual(bar_node.parent_index, [0, 1, 1, 2])
self.assertAllEqual(bar_node.values, [b"a", b"b", b"c", b"d"])
self.assertTrue(bar_node.is_repeated)
# keep_me's parent index should be unchanged.
keep_me_node = expression_test_util.calculate_value_slowly(
keep_me_expr)
self.assertAllEqual(keep_me_node.parent_index, [0, 1])
self.assertAllEqual(keep_me_node.values, [False, True])
self.assertFalse(keep_me_node.is_repeated)
def test_promote_with_schema_dense_fraction(self):
"""Test when min_fraction is not 1."""
s = prensor_test_util.create_big_prensor_schema()
feature_dict = {feature.name: feature for feature in s.feature}
user_feature = feature_dict["user"]
user_feature.value_count.min = 3
user_feature.value_count.max = 3
user_feature.presence.min_fraction = 1
user_dict = {
feature.name: feature
for feature in user_feature.struct_domain.feature
}
friends_feature = user_dict["friends"]
friends_feature.presence.min_fraction = 0.9
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_big_prensor()).apply_schema(s)
new_root, new_field = promote.promote_anonymous(
expr, path.Path(["user", "friends"]))
new_field = new_root.get_descendant_or_error(new_field)
new_schema_feature = new_field.schema_feature
self.assertIsNotNone(new_schema_feature)
self.assertEqual(new_schema_feature.presence.min_fraction, 0.3)
def test_promote_optional_child_of_repeated(self):
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_nested_prensor())
new_root, new_field = promote.promote_anonymous(
expr, path.Path(["doc", "keep_me"]))
new_expr = new_root.get_descendant_or_error(new_field)
self.assertTrue(new_expr.is_repeated)
def test_promote_and_calculate(self):
"""Tests promoting a leaf on a nested tree."""
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_nested_prensor())
new_root, new_path = promote.promote_anonymous(
expr, path.Path(["user", "friends"]))
new_field = new_root.get_descendant_or_error(new_path)
leaf_node = expression_test_util.calculate_value_slowly(new_field)
self.assertAllEqual(leaf_node.parent_index, [0, 1, 1, 1, 2])
self.assertAllEqual(leaf_node.values, [b"a", b"b", b"c", b"d", b"e"])
def test_promote_with_schema(self):
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_big_prensor()).apply_schema(
prensor_test_util.create_big_prensor_schema())
new_root, new_field = promote.promote_anonymous(
expr, path.Path(["user", "friends"]))
new_field = new_root.get_descendant_or_error(new_field)
new_schema_feature = new_field.schema_feature
self.assertIsNotNone(new_schema_feature)
self.assertEqual(new_schema_feature.string_domain.value[0], "a")
def test_calculate_promote_anonymous(self):
"""Performs promote_test.PromoteValuesTest, but with calculate_values."""
for options in options_to_test:
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_nested_prensor())
new_root, new_path = promote.promote_anonymous(
expr, path.Path(["user", "friends"]))
new_field = new_root.get_descendant_or_error(new_path)
[leaf_node] = calculate.calculate_values([new_field], options=options)
self.assertAllEqual(leaf_node.parent_index, [0, 1, 1, 1, 2])
self.assertAllEqual(leaf_node.values, [b"a", b"b", b"c", b"d", b"e"])
def test_promote_and_calculate_leaf_then_substructure(self):
"""Tests promoting of leaf and then a substructure."""
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_four_layer_prensor())
new_root, new_bar_path = promote.promote_anonymous(
expr, path.Path(["event", "doc", "nested_child", "bar"]))
new_root, new_path = promote.promote_anonymous(
new_root, path.Path(["event", "doc"]))
new_doc = new_root.get_descendant_or_error(new_path)
new_bar = new_root.get_descendant_or_error(
new_path.concat(new_bar_path.suffix(2)))
bar_expr = new_root.get_descendant_or_error(
new_path.concat(path.Path(["nested_child", "bar"])))
keep_me_expr = new_root.get_descendant_or_error(
new_path.concat(path.Path(["nested_child", "keep_me"])))
new_doc_node = expression_test_util.calculate_value_slowly(new_doc)
self.assertAllEqual(new_doc_node.parent_index, [0, 1, 1])
self.assertTrue(new_doc_node.is_repeated)
# new_bar's parent index is changed (from the first promote).
# The second promote should not change new_bar's parent index.
new_bar_node = expression_test_util.calculate_value_slowly(new_bar)
self.assertAllEqual(new_bar_node.parent_index, [0, 1, 1, 1])
self.assertAllEqual(new_bar_node.values, [b"a", b"b", b"c", b"d"])
self.assertTrue(new_bar_node.is_repeated)
# bar's parent index should be unchanged.
bar_node = expression_test_util.calculate_value_slowly(bar_expr)
self.assertAllEqual(bar_node.parent_index, [0, 1, 1, 2])
self.assertAllEqual(bar_node.values, [b"a", b"b", b"c", b"d"])
self.assertTrue(bar_node.is_repeated)
# keep_me's parent index should be unchanged.
keep_me_node = expression_test_util.calculate_value_slowly(
keep_me_expr)
self.assertAllEqual(keep_me_node.parent_index, [0, 1])
self.assertAllEqual(keep_me_node.values, [False, True])
self.assertFalse(keep_me_node.is_repeated)
def promote_and_broadcast_anonymous(root, origin, new_parent):
"""Promotes then broadcasts the origin until its parent is new_parent."""
least_common_ancestor = origin.get_least_common_ancestor(new_parent)
new_expr, new_path = root, origin
while new_path.get_parent() != least_common_ancestor:
new_expr, new_path = promote.promote_anonymous(new_expr, new_path)
while new_path.get_parent() != new_parent:
new_parent_step = new_parent.field_list[len(new_path) - 1]
new_expr, new_path = broadcast.broadcast_anonymous(
new_expr, new_path, new_parent_step)
return new_expr, new_path
def test_promote_and_calculate(self):
"""Tests get_sparse_tensors on a deep tree."""
with self.session(use_gpu=False) as sess:
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_nested_prensor())
new_root, new_path = promote.promote_anonymous(
expr, path.Path(["user", "friends"]))
new_field = new_root.get_descendant_or_error(new_path)
leaf_node = expression_test_util.calculate_value_slowly(new_field)
parent_index, values = sess.run(
[leaf_node.parent_index, leaf_node.values])
self.assertAllEqual(parent_index, [0, 1, 1, 1, 2])
self.assertAllEqual(values, [b"a", b"b", b"c", b"d", b"e"])
def test_promote_anonymous(self):
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_nested_prensor())
new_root, new_field = promote.promote_anonymous(
expr, path.Path(["user", "friends"]))
new_field = new_root.get_descendant_or_error(new_field)
self.assertTrue(new_field.is_repeated)
self.assertEqual(new_field.type, tf.string)
self.assertTrue(new_field.is_leaf)
self.assertFalse(new_field.calculation_is_identity())
self.assertTrue(new_field.calculation_equal(new_field))
self.assertFalse(new_field.calculation_equal(expr))
leaf_node = expression_test_util.calculate_value_slowly(new_field)
self.assertEqual(leaf_node.values.dtype, tf.string)
self.assertEqual(new_field.known_field_names(), frozenset())
sources = new_field.get_source_expressions()
self.assertLen(sources, 2)
self.assertIs(
expr.get_descendant_or_error(path.Path(["user", "friends"])),
sources[0])
self.assertIs(expr.get_child_or_error("user"), sources[1])
def _check_lifecycle_stage(a, b):
s = prensor_test_util.create_big_prensor_schema()
feature_dict = {feature.name: feature for feature in s.feature}
user_feature = feature_dict["user"]
if a is not None:
user_feature.lifecycle_stage = a
user_dict = {
feature.name: feature
for feature in user_feature.struct_domain.feature
}
friends_feature = user_dict["friends"]
if b is not None:
friends_feature.lifecycle_stage = b
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_big_prensor()).apply_schema(s)
new_root, new_field = promote.promote_anonymous(
expr, path.Path(["user", "friends"]))
new_field = new_root.get_descendant_or_error(new_field)
return new_field.schema_feature.lifecycle_stage
def test_promote_with_schema_dense_parent(self):
s = prensor_test_util.create_big_prensor_schema()
feature_dict = {feature.name: feature for feature in s.feature}
user_feature = feature_dict["user"]
user_feature.value_count.min = 3
user_feature.value_count.max = 3
user_feature.presence.min_fraction = 1
user_feature.lifecycle_stage = schema_pb2.LifecycleStage.ALPHA
user_dict = {
feature.name: feature
for feature in user_feature.struct_domain.feature
}
friends_feature = user_dict["friends"]
friends_feature.value_count.min = 2
friends_feature.value_count.max = 2
friends_feature.presence.min_fraction = 1
friends_feature.presence.min_count = 10
friends_feature.lifecycle_stage = schema_pb2.LifecycleStage.BETA
friends_feature.distribution_constraints.min_domain_mass = 0.5
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_big_prensor()).apply_schema(s)
new_root, new_field = promote.promote_anonymous(
expr, path.Path(["user", "friends"]))
[new_field] = new_root.get_descendant_or_error(
new_field).get_source_expressions()
new_schema_feature = new_field.schema_feature
self.assertIsNotNone(new_schema_feature)
self.assertEqual(new_schema_feature.string_domain.value[0], "a")
self.assertEqual(new_schema_feature.value_count.max, 6)
self.assertEqual(new_schema_feature.value_count.min, 6)
self.assertEqual(new_schema_feature.presence.min_fraction, 1)
self.assertEqual(new_schema_feature.presence.min_count, 3)
self.assertEqual(new_schema_feature.lifecycle_stage,
schema_pb2.LifecycleStage.ALPHA)
self.assertEqual(
new_schema_feature.distribution_constraints.min_domain_mass, 0.5)
