def test_promote_substructure(self):
"""Tests promote.promote(...) of substructure."""
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_deep_prensor())
new_root = promote.promote(expr, path.Path(["event", "doc"]),
"new_field")
new_field = new_root.get_child_or_error("new_field")
self.assertIsNotNone(new_field)
self.assertTrue(new_field.is_repeated)
self.assertEqual(new_field.known_field_names(),
frozenset(["bar", "keep_me"]))
bar_expr = new_field.get_child_or_error("bar")
self.assertIsNotNone(bar_expr)
self.assertTrue(bar_expr.is_repeated)
self.assertEqual(bar_expr.type, tf.string)
self.assertTrue(bar_expr.is_leaf)
keep_me_expr = new_field.get_child_or_error("keep_me")
self.assertIsNotNone(keep_me_expr)
self.assertFalse(keep_me_expr.is_repeated)
self.assertEqual(keep_me_expr.type, tf.bool)
self.assertTrue(keep_me_expr.is_leaf)
child_node = expression_test_util.calculate_value_slowly(new_field)
self.assertEqual(child_node.size, 3)
self.assertTrue(child_node.is_repeated)
bar_node = expression_test_util.calculate_value_slowly(bar_expr)
self.assertEqual(bar_node.values.dtype, tf.string)
keep_me_node = expression_test_util.calculate_value_slowly(
keep_me_expr)
self.assertEqual(keep_me_node.values.dtype, tf.bool)
def test_any_path(self):
my_any_0 = test_any_pb2.MessageWithAny()
my_value_0 = test_pb2.AllSimple()
my_value_0.optional_int32 = 17
my_any_0.my_any.Pack(my_value_0)
expr = proto.create_expression_from_proto(
[my_any_0.SerializeToString()], test_any_pb2.MessageWithAny.DESCRIPTOR)
new_root = promote.promote(
expr,
path.Path(
["my_any", "(struct2tensor.test.AllSimple)", "optional_int32"]),
"new_int32")
new_field = new_root.get_descendant_or_error(
path.Path(["my_any", "new_int32"]))
result = calculate_with_source_paths.calculate_prensors_with_source_paths(
[new_field])
prensor_result, proto_summary_result = result
self.assertLen(prensor_result, 1)
self.assertLen(proto_summary_result, 1)
leaf_node = prensor_result[0].node
self.assertAllEqual(leaf_node.parent_index, [0])
self.assertAllEqual(leaf_node.values, [17])
list_of_paths = proto_summary_result[0].paths
expected = [
path.Path(
["my_any", "(struct2tensor.test.AllSimple)", "optional_int32"])
]
self.equal_ignore_order(list_of_paths, expected)
def test_calculate_prensors_with_source_paths_with_transform(self):
"""Tests get_sparse_tensors on a deep tree with a transformed field."""
expr = proto_test_util._get_expression_from_session_empty_user_info()
# Let's make it non-trivial by transforming the data.
def _reverse(parent_indices, values):
return parent_indices, tf.reverse(values, axis=[-1])
expr = proto.create_transformed_field(expr, path.Path(["event"]),
"reversed_event", _reverse)
new_root = promote.promote(
expr, path.Path(["reversed_event", "action", "doc_id"]),
"action_doc_ids")
# A poor-man's reroot.
new_field = new_root.get_descendant_or_error(
path.Path(["reversed_event", "action_doc_ids"]))
result = calculate_with_source_paths.calculate_prensors_with_source_paths(
[new_field])
prensor_result, proto_summary_result = result
self.assertLen(prensor_result, 1)
self.assertLen(proto_summary_result, 1)
leaf_node = prensor_result[0].node
self.assertAllEqual(leaf_node.parent_index,
[0, 0, 0, 1, 2, 2, 3, 4, 4])
self.assertAllEqual(
leaf_node.values,
[b"h", b"i", b"j", b"g", b"e", b"f", b"c", b"a", b"b"])
list_of_paths = proto_summary_result[0].paths
expected = [path.Path(["event", "action", "doc_id"])]
self.equal_ignore_order(list_of_paths, expected)
def test_calculate_prensors_with_source_paths(self):
"""Tests get_sparse_tensors on a deep tree."""
with self.session(use_gpu=False) as sess:
expr = proto_test_util._get_expression_from_session_empty_user_info(
)
# Let's make it non-trivial by transforming the data.
new_root = promote.promote(
expr, path.Path(["event", "action", "doc_id"]),
"action_doc_ids")
# A poor-man's reroot.
new_field = new_root.get_descendant_or_error(
path.Path(["event", "action_doc_ids"]))
result = calculate_with_source_paths.calculate_prensors_with_source_paths(
[new_field])
prensor_result, proto_summary_result = result
self.assertLen(prensor_result, 1)
self.assertLen(proto_summary_result, 1)
leaf_node = prensor_result[0].node
[parent_index,
values] = sess.run([leaf_node.parent_index, leaf_node.values])
self.assertAllEqual(parent_index, [0, 0, 1, 2, 2, 3, 4, 4, 4])
self.assertAllEqual(
values, [b"a", b"b", b"c", b"e", b"f", b"g", b"h", b"i", b"j"])
list_of_paths = proto_summary_result[0].paths
expected = [path.Path(["event", "action", "doc_id"])]
self.equal_ignore_order(list_of_paths, expected)
def testPlaceholderExpression(self):
pren = prensor_test_util.create_nested_prensor()
expected_pren = prensor.create_prensor_from_descendant_nodes({
path.Path([]):
prensor.RootNodeTensor(tf.constant(3, dtype=tf.int64)),
path.Path(["new_friends"]):
prensor.LeafNodeTensor(
tf.constant([0, 1, 1, 1, 2], dtype=tf.int64),
tf.constant(["a", "b", "c", "d", "e"], dtype=tf.string), True)
})
root_schema = mpp.create_schema(is_repeated=True,
children={
"doc": {
"is_repeated": True,
"children": {
"bar": {
"is_repeated": True,
"dtype": tf.string
},
"keep_me": {
"is_repeated": False,
"dtype": tf.bool
}
}
},
"user": {
"is_repeated": True,
"children": {
"friends": {
"is_repeated": True,
"dtype": tf.string
}
}
}
})
exp = placeholder.create_expression_from_schema(root_schema)
promote_exp = promote.promote(exp, path.Path(["user", "friends"]),
"new_friends")
project_exp = project.project(promote_exp,
[path.Path(["new_friends"])])
new_friends_exp = project_exp.get_descendant(path.Path(["new_friends"
]))
result = calculate.calculate_values([new_friends_exp],
feed_dict={exp: pren})
res_node = result[0]
exp_node = expected_pren.get_descendant(path.Path(["new_friends"
])).node
self.assertAllEqual(res_node.is_repeated, exp_node.is_repeated)
self.assertAllEqual(res_node.values, exp_node.values)
self.assertAllEqual(res_node.parent_index, exp_node.parent_index)
def test_calculate_promote_named(self):
"""Performs promote_test.PromoteValuesTest, but with calculate_values."""
for options in options_to_test:
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_nested_prensor())
new_root = promote.promote(expr, path.Path(["user", "friends"]),
"new_friends")
# projected = project.project(new_root, [path.Path(["new_friends"])])
new_field = new_root.get_child_or_error("new_friends")
[leaf_node] = calculate.calculate_values([new_field], options=options)
self.assertAllEqual(leaf_node.parent_index, [0, 1, 1, 1, 2])
self.assertAllEqual(leaf_node.values, [b"a", b"b", b"c", b"d", b"e"])
def test_promote(self):
"""Tests promote.promote(...), and indirectly tests set_path."""
expr = create_expression.create_expression_from_prensor(
prensor_test_util.create_nested_prensor())
new_root = promote.promote(expr, path.Path(["user", "friends"]),
"new_field")
new_field = new_root.get_child_or_error("new_field")
self.assertIsNotNone(new_field)
self.assertTrue(new_field.is_repeated)
self.assertEqual(new_field.type, tf.string)
self.assertTrue(new_field.is_leaf)
leaf_node = expression_test_util.calculate_value_slowly(new_field)
self.assertEqual(leaf_node.values.dtype, tf.string)
self.assertEqual(new_field.known_field_names(), frozenset())
def testPromoteAndProjectExpression(self):
filenames = [
"struct2tensor/testdata/parquet_testdata/dremel_example.parquet"
]
batch_size = 2
exp = parquet.create_expression_from_parquet_file(filenames)
new_exp = promote.promote(exp, path.Path(["Name", "Language", "Code"]),
"new_code")
new_code_project_exp = project.project(
new_exp, [path.Path(["Name", "new_code"])])
docid_project_exp = project.project(exp, [path.Path(["DocId"])])
pqds = parquet.calculate_parquet_values(
[new_code_project_exp, docid_project_exp], exp, filenames,
batch_size)
new_code_expected = prensor.create_prensor_from_descendant_nodes({
path.Path([]):
prensor.RootNodeTensor(tf.constant(2, dtype=tf.int64)),
path.Path(["Name"]):
prensor.ChildNodeTensor(tf.constant([0, 0, 0, 1], dtype=tf.int64),
True),
path.Path(["Name", "new_code"]):
prensor.LeafNodeTensor(tf.constant([0, 0, 2], dtype=tf.int64),
tf.constant([b"en-us", b"en", b"en-gb"]),
True)
})
docid_expected = prensor.create_prensor_from_descendant_nodes({
path.Path([]):
prensor.RootNodeTensor(tf.constant(2, dtype=tf.int64)),
path.Path(["DocId"]):
prensor.LeafNodeTensor(tf.constant([0, 1], dtype=tf.int64),
tf.constant([10, 20], dtype=tf.int64),
False)
})
for ele in pqds:
new_code_pren = ele[0]
docid_pren = ele[1]
self._assertPrensorEqual(new_code_pren, new_code_expected)
self._assertPrensorEqual(docid_pren, docid_expected)
