def test_promote_substructure_with_schema(self): expr = create_expression.create_expression_from_prensor( prensor_test_util.create_deep_prensor()).apply_schema( prensor_test_util.create_deep_prensor_schema()) original_schema = expr.get_descendant_or_error( path.Path(["event", "doc"])).schema_feature new_root, new_field_path = promote.promote_anonymous( expr, path.Path(["event", "doc"])) new_field = new_root.get_descendant_or_error(new_field_path) new_schema_feature = new_field.schema_feature self.assertIsNotNone(new_schema_feature) # The struct_domain of this feature should not be changed. self.assertProtoEquals(new_schema_feature.struct_domain, original_schema.struct_domain) bar_schema = new_root.get_descendant_or_error( new_field_path.concat(path.Path(["bar"]))).schema_feature self.assertIsNotNone(bar_schema) self.assertEqual(bar_schema.string_domain.value[0], "a") keep_me_schema = new_root.get_descendant_or_error( new_field_path.concat(path.Path(["keep_me"]))).schema_feature self.assertIsNotNone(keep_me_schema) self.assertEqual(keep_me_schema.presence.min_count, 1)
def test_promote_and_calculate_substructure(self): """Tests promoting substructure on a tree with depth of 4.""" expr = create_expression.create_expression_from_prensor( prensor_test_util.create_four_layer_prensor()) new_root, new_path = promote.promote_anonymous( expr, path.Path(["event", "doc", "nested_child"])) new_nested_child = new_root.get_descendant_or_error(new_path) bar_expr = new_root.get_descendant_or_error(new_path.get_child("bar")) keep_me_expr = new_root.get_descendant_or_error( new_path.get_child("keep_me")) # the promoted nested_child's parent index is changed. nested_child_node = expression_test_util.calculate_value_slowly( new_nested_child) self.assertAllEqual(nested_child_node.parent_index, [0, 1, 1, 1]) self.assertTrue(nested_child_node.is_repeated) # bar's parent index should be unchanged. bar_node = expression_test_util.calculate_value_slowly(bar_expr) self.assertAllEqual(bar_node.parent_index, [0, 1, 1, 2]) self.assertAllEqual(bar_node.values, [b"a", b"b", b"c", b"d"]) self.assertTrue(bar_node.is_repeated) # keep_me's parent index should be unchanged. keep_me_node = expression_test_util.calculate_value_slowly( keep_me_expr) self.assertAllEqual(keep_me_node.parent_index, [0, 1]) self.assertAllEqual(keep_me_node.values, [False, True]) self.assertFalse(keep_me_node.is_repeated)
def test_promote_with_schema_dense_fraction(self): """Test when min_fraction is not 1.""" s = prensor_test_util.create_big_prensor_schema() feature_dict = {feature.name: feature for feature in s.feature} user_feature = feature_dict["user"] user_feature.value_count.min = 3 user_feature.value_count.max = 3 user_feature.presence.min_fraction = 1 user_dict = { feature.name: feature for feature in user_feature.struct_domain.feature } friends_feature = user_dict["friends"] friends_feature.presence.min_fraction = 0.9 expr = create_expression.create_expression_from_prensor( prensor_test_util.create_big_prensor()).apply_schema(s) new_root, new_field = promote.promote_anonymous( expr, path.Path(["user", "friends"])) new_field = new_root.get_descendant_or_error(new_field) new_schema_feature = new_field.schema_feature self.assertIsNotNone(new_schema_feature) self.assertEqual(new_schema_feature.presence.min_fraction, 0.3)
def test_promote_optional_child_of_repeated(self): expr = create_expression.create_expression_from_prensor( prensor_test_util.create_nested_prensor()) new_root, new_field = promote.promote_anonymous( expr, path.Path(["doc", "keep_me"])) new_expr = new_root.get_descendant_or_error(new_field) self.assertTrue(new_expr.is_repeated)
def test_promote_and_calculate(self): """Tests promoting a leaf on a nested tree.""" expr = create_expression.create_expression_from_prensor( prensor_test_util.create_nested_prensor()) new_root, new_path = promote.promote_anonymous( expr, path.Path(["user", "friends"])) new_field = new_root.get_descendant_or_error(new_path) leaf_node = expression_test_util.calculate_value_slowly(new_field) self.assertAllEqual(leaf_node.parent_index, [0, 1, 1, 1, 2]) self.assertAllEqual(leaf_node.values, [b"a", b"b", b"c", b"d", b"e"])
def test_promote_with_schema(self): expr = create_expression.create_expression_from_prensor( prensor_test_util.create_big_prensor()).apply_schema( prensor_test_util.create_big_prensor_schema()) new_root, new_field = promote.promote_anonymous( expr, path.Path(["user", "friends"])) new_field = new_root.get_descendant_or_error(new_field) new_schema_feature = new_field.schema_feature self.assertIsNotNone(new_schema_feature) self.assertEqual(new_schema_feature.string_domain.value[0], "a")
def test_calculate_promote_anonymous(self): """Performs promote_test.PromoteValuesTest, but with calculate_values.""" for options in options_to_test: expr = create_expression.create_expression_from_prensor( prensor_test_util.create_nested_prensor()) new_root, new_path = promote.promote_anonymous( expr, path.Path(["user", "friends"])) new_field = new_root.get_descendant_or_error(new_path) [leaf_node] = calculate.calculate_values([new_field], options=options) self.assertAllEqual(leaf_node.parent_index, [0, 1, 1, 1, 2]) self.assertAllEqual(leaf_node.values, [b"a", b"b", b"c", b"d", b"e"])
def test_promote_and_calculate_leaf_then_substructure(self): """Tests promoting of leaf and then a substructure.""" expr = create_expression.create_expression_from_prensor( prensor_test_util.create_four_layer_prensor()) new_root, new_bar_path = promote.promote_anonymous( expr, path.Path(["event", "doc", "nested_child", "bar"])) new_root, new_path = promote.promote_anonymous( new_root, path.Path(["event", "doc"])) new_doc = new_root.get_descendant_or_error(new_path) new_bar = new_root.get_descendant_or_error( new_path.concat(new_bar_path.suffix(2))) bar_expr = new_root.get_descendant_or_error( new_path.concat(path.Path(["nested_child", "bar"]))) keep_me_expr = new_root.get_descendant_or_error( new_path.concat(path.Path(["nested_child", "keep_me"]))) new_doc_node = expression_test_util.calculate_value_slowly(new_doc) self.assertAllEqual(new_doc_node.parent_index, [0, 1, 1]) self.assertTrue(new_doc_node.is_repeated) # new_bar's parent index is changed (from the first promote). # The second promote should not change new_bar's parent index. new_bar_node = expression_test_util.calculate_value_slowly(new_bar) self.assertAllEqual(new_bar_node.parent_index, [0, 1, 1, 1]) self.assertAllEqual(new_bar_node.values, [b"a", b"b", b"c", b"d"]) self.assertTrue(new_bar_node.is_repeated) # bar's parent index should be unchanged. bar_node = expression_test_util.calculate_value_slowly(bar_expr) self.assertAllEqual(bar_node.parent_index, [0, 1, 1, 2]) self.assertAllEqual(bar_node.values, [b"a", b"b", b"c", b"d"]) self.assertTrue(bar_node.is_repeated) # keep_me's parent index should be unchanged. keep_me_node = expression_test_util.calculate_value_slowly( keep_me_expr) self.assertAllEqual(keep_me_node.parent_index, [0, 1]) self.assertAllEqual(keep_me_node.values, [False, True]) self.assertFalse(keep_me_node.is_repeated)
def promote_and_broadcast_anonymous(root, origin, new_parent): """Promotes then broadcasts the origin until its parent is new_parent.""" least_common_ancestor = origin.get_least_common_ancestor(new_parent) new_expr, new_path = root, origin while new_path.get_parent() != least_common_ancestor: new_expr, new_path = promote.promote_anonymous(new_expr, new_path) while new_path.get_parent() != new_parent: new_parent_step = new_parent.field_list[len(new_path) - 1] new_expr, new_path = broadcast.broadcast_anonymous( new_expr, new_path, new_parent_step) return new_expr, new_path
def test_promote_and_calculate(self): """Tests get_sparse_tensors on a deep tree.""" with self.session(use_gpu=False) as sess: expr = create_expression.create_expression_from_prensor( prensor_test_util.create_nested_prensor()) new_root, new_path = promote.promote_anonymous( expr, path.Path(["user", "friends"])) new_field = new_root.get_descendant_or_error(new_path) leaf_node = expression_test_util.calculate_value_slowly(new_field) parent_index, values = sess.run( [leaf_node.parent_index, leaf_node.values]) self.assertAllEqual(parent_index, [0, 1, 1, 1, 2]) self.assertAllEqual(values, [b"a", b"b", b"c", b"d", b"e"])
def test_promote_anonymous(self): expr = create_expression.create_expression_from_prensor( prensor_test_util.create_nested_prensor()) new_root, new_field = promote.promote_anonymous( expr, path.Path(["user", "friends"])) new_field = new_root.get_descendant_or_error(new_field) self.assertTrue(new_field.is_repeated) self.assertEqual(new_field.type, tf.string) self.assertTrue(new_field.is_leaf) self.assertFalse(new_field.calculation_is_identity()) self.assertTrue(new_field.calculation_equal(new_field)) self.assertFalse(new_field.calculation_equal(expr)) leaf_node = expression_test_util.calculate_value_slowly(new_field) self.assertEqual(leaf_node.values.dtype, tf.string) self.assertEqual(new_field.known_field_names(), frozenset()) sources = new_field.get_source_expressions() self.assertLen(sources, 2) self.assertIs( expr.get_descendant_or_error(path.Path(["user", "friends"])), sources[0]) self.assertIs(expr.get_child_or_error("user"), sources[1])
def _check_lifecycle_stage(a, b): s = prensor_test_util.create_big_prensor_schema() feature_dict = {feature.name: feature for feature in s.feature} user_feature = feature_dict["user"] if a is not None: user_feature.lifecycle_stage = a user_dict = { feature.name: feature for feature in user_feature.struct_domain.feature } friends_feature = user_dict["friends"] if b is not None: friends_feature.lifecycle_stage = b expr = create_expression.create_expression_from_prensor( prensor_test_util.create_big_prensor()).apply_schema(s) new_root, new_field = promote.promote_anonymous( expr, path.Path(["user", "friends"])) new_field = new_root.get_descendant_or_error(new_field) return new_field.schema_feature.lifecycle_stage
def test_promote_with_schema_dense_parent(self): s = prensor_test_util.create_big_prensor_schema() feature_dict = {feature.name: feature for feature in s.feature} user_feature = feature_dict["user"] user_feature.value_count.min = 3 user_feature.value_count.max = 3 user_feature.presence.min_fraction = 1 user_feature.lifecycle_stage = schema_pb2.LifecycleStage.ALPHA user_dict = { feature.name: feature for feature in user_feature.struct_domain.feature } friends_feature = user_dict["friends"] friends_feature.value_count.min = 2 friends_feature.value_count.max = 2 friends_feature.presence.min_fraction = 1 friends_feature.presence.min_count = 10 friends_feature.lifecycle_stage = schema_pb2.LifecycleStage.BETA friends_feature.distribution_constraints.min_domain_mass = 0.5 expr = create_expression.create_expression_from_prensor( prensor_test_util.create_big_prensor()).apply_schema(s) new_root, new_field = promote.promote_anonymous( expr, path.Path(["user", "friends"])) [new_field] = new_root.get_descendant_or_error( new_field).get_source_expressions() new_schema_feature = new_field.schema_feature self.assertIsNotNone(new_schema_feature) self.assertEqual(new_schema_feature.string_domain.value[0], "a") self.assertEqual(new_schema_feature.value_count.max, 6) self.assertEqual(new_schema_feature.value_count.min, 6) self.assertEqual(new_schema_feature.presence.min_fraction, 1) self.assertEqual(new_schema_feature.presence.min_count, 3) self.assertEqual(new_schema_feature.lifecycle_stage, schema_pb2.LifecycleStage.ALPHA) self.assertEqual( new_schema_feature.distribution_constraints.min_domain_mass, 0.5)