def test_calculate_prensors_with_source_paths_with_transform(self):
"""Tests get_sparse_tensors on a deep tree with a transformed field."""
expr = proto_test_util._get_expression_from_session_empty_user_info()
# Let's make it non-trivial by transforming the data.
def _reverse(parent_indices, values):
return parent_indices, tf.reverse(values, axis=[-1])
expr = proto.create_transformed_field(expr, path.Path(["event"]),
"reversed_event", _reverse)
new_root = promote.promote(
expr, path.Path(["reversed_event", "action", "doc_id"]),
"action_doc_ids")
# A poor-man's reroot.
new_field = new_root.get_descendant_or_error(
path.Path(["reversed_event", "action_doc_ids"]))
result = calculate_with_source_paths.calculate_prensors_with_source_paths(
[new_field])
prensor_result, proto_summary_result = result
self.assertLen(prensor_result, 1)
self.assertLen(proto_summary_result, 1)
leaf_node = prensor_result[0].node
self.assertAllEqual(leaf_node.parent_index,
[0, 0, 0, 1, 2, 2, 3, 4, 4])
self.assertAllEqual(
leaf_node.values,
[b"h", b"i", b"j", b"g", b"e", b"f", b"c", b"a", b"b"])
list_of_paths = proto_summary_result[0].paths
expected = [path.Path(["event", "action", "doc_id"])]
self.equal_ignore_order(list_of_paths, expected)
def test_create_expression_from_proto_and_calculate_root_value(self):
"""Tests get_sparse_tensors on a deep tree."""
expr = proto_test_util._get_expression_from_session_empty_user_info()
root_value = expression_test_util.calculate_value_slowly(expr)
# For some reason, this fails on tf.eager. It could be because it is
# a scalar, I don't know.
self.assertEqual(self.evaluate(root_value.size), 2)
def test_create_expression_from_proto_and_calculate_event_id_value(self):
"""Tests get_sparse_tensors on a deep tree."""
expr = proto_test_util._get_expression_from_session_empty_user_info()
event_id_value = expression_test_util.calculate_value_slowly(
expr.get_descendant_or_error(path.Path(["event", "event_id"])))
self.assertAllEqual(event_id_value.parent_index, [0, 1, 2, 4])
self.assertAllEqual(event_id_value.values, [b"A", b"B", b"C", b"D"])
def test_calculate_prensors_with_source_paths(self):
"""Tests get_sparse_tensors on a deep tree."""
with self.session(use_gpu=False) as sess:
expr = proto_test_util._get_expression_from_session_empty_user_info(
)
# Let's make it non-trivial by transforming the data.
new_root = promote.promote(
expr, path.Path(["event", "action", "doc_id"]),
"action_doc_ids")
# A poor-man's reroot.
new_field = new_root.get_descendant_or_error(
path.Path(["event", "action_doc_ids"]))
result = calculate_with_source_paths.calculate_prensors_with_source_paths(
[new_field])
prensor_result, proto_summary_result = result
self.assertLen(prensor_result, 1)
self.assertLen(proto_summary_result, 1)
leaf_node = prensor_result[0].node
[parent_index,
values] = sess.run([leaf_node.parent_index, leaf_node.values])
self.assertAllEqual(parent_index, [0, 0, 1, 2, 2, 3, 4, 4, 4])
self.assertAllEqual(
values, [b"a", b"b", b"c", b"e", b"f", b"g", b"h", b"i", b"j"])
list_of_paths = proto_summary_result[0].paths
expected = [path.Path(["event", "action", "doc_id"])]
self.equal_ignore_order(list_of_paths, expected)
def test_create_expression_from_proto_and_calculate_root_value(self):
"""Tests get_sparse_tensors on a deep tree."""
with self.session(use_gpu=False) as sess:
expr = proto_test_util._get_expression_from_session_empty_user_info(
)
root_value = expression_test_util.calculate_value_slowly(expr)
size = sess.run(root_value.size)
self.assertEqual(size, 2)
def test_create_query_and_calculate_event_value(self):
"""Calculating a child value in a proto tests dependencies."""
for options in options_to_test:
expr = proto_test_util._get_expression_from_session_empty_user_info()
[event_value
] = calculate.calculate_values([expr.get_child_or_error("event")],
options=options)
self.assertAllEqual(event_value.parent_index, [0, 0, 0, 1, 1])
def test_create_expression_from_proto_and_calculate_event_value(self):
"""Tests get_sparse_tensors on a deep tree."""
with self.session(use_gpu=False) as sess:
expr = proto_test_util._get_expression_from_session_empty_user_info(
)
event_value = expression_test_util.calculate_value_slowly(
expr.get_child_or_error("event"))
parent_index = sess.run(event_value.parent_index)
self.assertAllEqual(parent_index, [0, 0, 0, 1, 1])
def test_create_expression_from_proto_and_calculate_event_value(
self, use_string_view):
"""Tests get_sparse_tensors on a deep tree."""
expr = proto_test_util._get_expression_from_session_empty_user_info()
event_value = expression_test_util.calculate_value_slowly(
expr.get_child_or_error("event"),
options=self._get_calculate_options(use_string_view))
self.assertAllEqual(event_value.parent_index, [0, 0, 0, 1, 1])
if use_string_view:
self._check_string_view()
def test_create_query_modify_and_calculate_event_value(self):
"""Calculating a child value in a proto tests dependencies."""
for options in options_to_test:
root = proto_test_util._get_expression_from_session_empty_user_info()
root_2 = expression_add.add_paths(
root, {path.Path(["event_copy"]): root.get_child_or_error("event")})
[event_value
] = calculate.calculate_values([root_2.get_child_or_error("event_copy")],
options=options)
self.assertAllEqual(event_value.parent_index, [0, 0, 0, 1, 1])
def test_transformed_field_values(self):
expr = proto_test_util._get_expression_from_session_empty_user_info()
reversed_events_expr = proto.create_transformed_field(
expr, path.Path(["event"]), "reversed_event", _reverse_values)
result = expression_test_util.calculate_list_map(
reversed_events_expr.project(["reversed_event.action.doc_id"]),
self)
self.assertAllEqual(
result["reversed_event.action.doc_id"],
[[[[b"h"], [b"i"], [b"j"]], [[b"g"]], [[b"e"], [b"f"]]],
[[[b"c"], []], [[b"a"], [b"b"]]]])
def test_create_reversed_field_nested(self):
expr = proto_test_util._get_expression_from_session_empty_user_info()
first_reverse = proto.create_transformed_field(expr, path.Path(["event"]),
"reversed_event",
_reverse_values)
second_reverse = proto.create_transformed_field(
first_reverse, path.Path(["reversed_event", "action"]),
"reversed_action", _reverse_values)
leaf_expr = second_reverse.get_descendant_or_error(
path.Path(["reversed_event", "reversed_action", "doc_id"]))
leaf_tensor = expression_test_util.calculate_value_slowly(leaf_expr)
self.assertEqual(leaf_tensor.parent_index.dtype, tf.int64)
self.assertEqual(leaf_tensor.values.dtype, tf.string)
def test_create_transformed_field(self):
expr = proto_test_util._get_expression_from_session_empty_user_info()
reversed_events_expr = proto.create_transformed_field(
expr, path.Path(["event"]), "reversed_event", _reverse_values)
source_events = expr.get_child_or_error("event")
dest_events = reversed_events_expr.get_child_or_error("reversed_event")
self.assertTrue(dest_events.is_repeated)
self.assertFalse(dest_events.is_leaf)
self.assertEqual(source_events.type, dest_events.type)
leaf_expr = reversed_events_expr.get_descendant_or_error(
path.Path(["reversed_event", "action", "doc_id"]))
leaf_tensor = expression_test_util.calculate_value_slowly(leaf_expr)
self.assertEqual(leaf_tensor.parent_index.dtype, tf.int64)
self.assertEqual(leaf_tensor.values.dtype, tf.string)
def test_create_expression_from_proto_with_root(self):
expr = proto_test_util._get_expression_from_session_empty_user_info()
self.assertTrue(expr.is_repeated)
self.assertIsNone(expr.type)
self.assertFalse(expr.is_leaf)
self.assertFalse(expr.calculation_is_identity())
self.assertTrue(expr.calculation_equal(expr))
self.assertFalse(expr.calculation_equal(expr.get_child_or_error("event")))
root_node = expression_test_util.calculate_value_slowly(expr)
self.assertEqual(root_node.size.dtype, tf.int64)
self.assertEqual(expr.known_field_names(),
frozenset({"event", "session_id", "session_info"}))
sources = expr.get_source_expressions()
self.assertEmpty(sources)
def test_transformed_field_values_with_transformed_parent(
self, use_string_view):
expr = proto_test_util._get_expression_from_session_empty_user_info()
first_reversed_expr = proto.create_transformed_field(
expr, path.Path(["event"]), "reversed_event", _reverse_values)
second_reversed_expr = proto.create_transformed_field(
first_reversed_expr, path.Path(["reversed_event", "action"]),
"reversed_action", _reverse_values)
result = expression_test_util.calculate_list_map(
second_reversed_expr.project(["reversed_event.reversed_action.doc_id"]),
self,
options=self._get_calculate_options(use_string_view))
self.assertAllEqual(result["reversed_event.reversed_action.doc_id"],
[[[[b"b"], [b"a"], []], [[b"c"]], [[b"f"], [b"e"]]],
[[[b"g"], [b"j"]], [[b"i"], [b"h"]]]])
if use_string_view:
self._check_string_view()
def test_create_expression_from_proto_with_event(self):
expr = proto_test_util._get_expression_from_session_empty_user_info()
event_expr = expr.get_child_or_error("event")
self.assertTrue(event_expr.is_repeated)
self.assertIsNone(event_expr.type)
self.assertFalse(event_expr.is_leaf)
self.assertFalse(event_expr.calculation_is_identity())
self.assertTrue(event_expr.calculation_equal(event_expr))
self.assertFalse(event_expr.calculation_equal(expr))
child_node = expression_test_util.calculate_value_slowly(event_expr)
self.assertEqual(child_node.parent_index.dtype, tf.int64)
self.assertEqual(
event_expr.known_field_names(),
frozenset({
"event_id", "query", "query_token", "action", "user_info",
"action_mask"
}))
sources = event_expr.get_source_expressions()
self.assertLen(sources, 1)
self.assertIs(expr, sources[0])
def test_missing_extension(self):
"""Tests a missing extension on a deep tree."""
expr = proto_test_util._get_expression_from_session_empty_user_info()
missing_expr = expr.get_child("(ext.NotPresent)")
self.assertIsNone(missing_expr)
