def test_slice_end(self):
with self.session(use_gpu=False) as sess:
root = create_expression.create_expression_from_prensor(
prensor_test_util.create_big_prensor())
root_2 = slice_expression.slice_expression(root,
path.Path(["doc"]),
"new_doc", None, 1)
result = prensor_value.materialize(
calculate.calculate_prensors([root_2])[0], sess)
self.assertAllEqual(
result.get_descendant_or_error(path.Path(
["new_doc"])).node.parent_index, [0, 1])
self.assertAllEqual(
result.get_descendant_or_error(
path.Path(["new_doc", "keep_me"])).node.parent_index,
[0, 1])
self.assertAllEqual(
result.get_descendant_or_error(
path.Path(["new_doc", "keep_me"])).node.values,
[False, True])
self.assertAllEqual(
result.get_descendant_or_error(path.Path(
["new_doc", "bar"])).node.parent_index, [0, 1, 1])
self.assertAllEqual(
result.get_descendant_or_error(path.Path(["new_doc",
"bar"])).node.values,
[b"a", b"b", b"c"])
def test_materialize(self):
"""Tests get_sparse_tensors on a deep expression."""
with self.session(use_gpu=False) as sess:
pren = prensor_test_util.create_nested_prensor()
mat = prensor_value.materialize(pren, sess)
self.assertAllEqual(
mat.get_descendant_or_error(path.Path(["doc",
"bar"])).node.values,
[b"a", b"b", b"c", b"d"])
self.assertAllEqual(
mat.get_descendant_or_error(path.Path(["doc", "bar"
])).node.parent_index,
[0, 1, 1, 2])
self.assertAllEqual(
mat.get_descendant_or_error(path.Path(["doc"
])).node.parent_index,
[0, 1, 1])
self.assertAllEqual(
mat.get_descendant_or_error(path.Path([])).node.size, 3)
self.assertAllEqual(
mat.get_descendant_or_error(path.Path(["doc", "keep_me"
])).node.parent_index,
[0, 1])
self.assertAllEqual(
mat.get_descendant_or_error(path.Path(["doc", "keep_me"
])).node.values,
[False, True])
def test_filter_by_child_create_nested_prensor_2(self):
"""Tests filter_by_child.
In particular, it checks for the case where parent_index != self index.
"""
with self.session(use_gpu=False) as sess:
root = create_expression.create_expression_from_prensor(
_create_nested_prensor_2())
root_2 = filter_expression.filter_by_child(root,
path.create_path("doc"),
"keep_me", "new_doc")
result = prensor_value.materialize(
calculate.calculate_prensors([root_2])[0], sess)
self.assertAllEqual(
result.get_descendant_or_error(path.Path(
["new_doc"])).node.parent_index, [1])
self.assertAllEqual(
result.get_descendant_or_error(
path.Path(["new_doc", "keep_me"])).node.parent_index, [0])
self.assertAllEqual(
result.get_descendant_or_error(
path.Path(["new_doc", "keep_me"])).node.values, [True])
self.assertAllEqual(
result.get_descendant_or_error(path.Path(
["new_doc", "bar"])).node.parent_index, [0, 0])
self.assertAllEqual(
result.get_descendant_or_error(path.Path(["new_doc",
"bar"])).node.values,
[b"b", b"c"])
def test_slice_mask_begin_negative(self):
with self.session(use_gpu=False) as sess:
root = create_expression.create_expression_from_prensor(
prensor_test_util.create_big_prensor())
root_2, new_path = slice_expression._get_slice_mask(
root, path.Path(["doc"]), -1, None)
result = prensor_value.materialize(
calculate.calculate_prensors([root_2])[0], sess)
self.assertAllEqual(
result.get_descendant_or_error(new_path).node.parent_index,
[0, 1, 1])
self.assertAllEqual(
result.get_descendant_or_error(new_path).node.values,
[True, False, True])
def test_filter_by_sibling(self):
r"""Tests filter_by_sibling.
Beginning with the struct:
-----*----------------------------------------------------
/ \ \
root0 root1----------------------- root2 (empty)
/ \ / \ \ \
| keep_my_sib0:False | keep_my_sib1:True | keep_my_sib2:False
doc0----- doc1--------------- doc2--------
| \ \ \ \ \
bar:"a" keep_me:False bar:"b" bar:"c" keep_me:True bar:"d"
Filter doc with keep_my_sib:
End with the struct (suppressing original doc):
-----*----------------------------------------------------
/ \ \
root0 root1------------------ root2 (empty)
\ / \ \
keep_my_sib0:False | keep_my_sib1:True keep_my_sib2:False
new_doc0-----------
\ \ \
bar:"b" bar:"c" keep_me:True
"""
with self.session(use_gpu=False) as sess:
root = create_expression.create_expression_from_prensor(
_create_nested_prensor())
root_2 = filter_expression.filter_by_sibling(
root, path.create_path("doc"), "keep_my_sib", "new_doc")
result_prensor = calculate.calculate_prensors([root_2])[0]
result = prensor_value.materialize(result_prensor, sess)
self.assertAllEqual(
result.get_descendant_or_error(path.Path(
["new_doc"])).node.parent_index, [1])
self.assertAllEqual(
result.get_descendant_or_error(
path.Path(["new_doc", "keep_me"])).node.parent_index, [0])
self.assertAllEqual(
result.get_descendant_or_error(
path.Path(["new_doc", "keep_me"])).node.values, [True])
self.assertAllEqual(
result.get_descendant_or_error(path.Path(
["new_doc", "bar"])).node.parent_index, [0, 0])
self.assertAllEqual(
result.get_descendant_or_error(path.Path(["new_doc",
"bar"])).node.values,
[b"b", b"c"])
def test_slice_begin(self):
"""Test slice with only begin specified.
Starts with:
{
foo:9,
foorepeated:[9],
doc:[{
bar:["a"],
keep_me:False
}],
user:[
{
friends:["a"]
}]
}
{foo:8,
foorepeated:[8,7],
doc:[{
bar:["b","c"],
keep_me:True
},{
bar:["d"]
}],
user:[{
friends:["b", "c"]
},{
friends:["d"]
}],
}
{foo:7,
foorepeated:[6],
user:[{friends:["e"]}]}
Creates new_doc by slicing doc[1:]:
{foo:9,
foorepeated:[9],
doc:[{
bar:["a"],
keep_me:False
}],
user:[{
friends:["a"]
}]}
{foo:8,
foorepeated:[8,7],
doc:[{
bar:["b","c"],
keep_me:True
},{
bar:["d"]
}],
new_doc[{
bar:["d"]
}],
user:[{
friends:["b", "c"]
},{
friends:["d"]}],}
{foo:7,
foorepeated:[6],
user:[{
friends:["e"]
}]}
"""
with self.session(use_gpu=False) as sess:
root = create_expression.create_expression_from_prensor(
prensor_test_util.create_big_prensor())
root_2 = slice_expression.slice_expression(root,
path.Path(["doc"]),
"new_doc", 1, None)
result = prensor_value.materialize(
calculate.calculate_prensors([root_2])[0], sess)
self.assertAllEqual(
result.get_descendant_or_error(path.Path(
["new_doc"])).node.parent_index, [1])
self.assertAllEqual(
result.get_descendant_or_error(
path.Path(["new_doc", "keep_me"])).node.parent_index, [])
self.assertAllEqual(
result.get_descendant_or_error(
path.Path(["new_doc", "keep_me"])).node.values, [])
self.assertAllEqual(
result.get_descendant_or_error(path.Path(
["new_doc", "bar"])).node.parent_index, [0])
self.assertAllEqual(
result.get_descendant_or_error(path.Path(["new_doc",
"bar"])).node.values,
[b"d"])
def test_map_prensor_to_prensor(self):
original = create_expression.create_expression_from_prensor(
prensor_test_util.create_nested_prensor())
def my_prensor_op(original_prensor):
# Note that we are copying over the original root prensor node. The root
# node is ignored in the result.
return prensor.create_prensor_from_descendant_nodes({
path.Path([]):
original_prensor.node,
path.Path(["bar2"]):
original_prensor.get_child_or_error("bar").node,
path.Path(["keep_me2"]):
original_prensor.get_child_or_error("keep_me").node
})
# Since the top node is actually a child node, we use the child schema.
my_output_schema = map_prensor_to_prensor.create_schema(
is_repeated=True,
children={
"bar2": {
"is_repeated": True,
"dtype": tf.string
},
"keep_me2": {
"is_repeated": False,
"dtype": tf.bool
}
})
result = map_prensor_to_prensor.map_prensor_to_prensor(
root_expr=original,
source=path.Path(["doc"]),
paths_needed=[path.Path(["bar"]),
path.Path(["keep_me"])],
prensor_op=my_prensor_op,
output_schema=my_output_schema)
doc_result = result.get_child_or_error("doc")
bar_result = doc_result.get_child_or_error("bar")
keep_me_result = doc_result.get_child_or_error("keep_me")
bar2_result = doc_result.get_child_or_error("bar2")
keep_me2_result = doc_result.get_child_or_error("keep_me2")
self.assertIsNone(doc_result.get_child("missing_field"))
self.assertTrue(bar_result.is_repeated)
self.assertTrue(bar2_result.is_repeated)
self.assertEqual(bar_result.type, tf.string)
self.assertEqual(bar2_result.type, tf.string)
self.assertFalse(keep_me_result.is_repeated)
self.assertFalse(keep_me2_result.is_repeated)
self.assertEqual(keep_me_result.type, tf.bool)
self.assertEqual(keep_me2_result.type, tf.bool)
[prensor_result] = calculate.calculate_prensors([result])
with self.session(use_gpu=False) as sess:
value = prensor_value.materialize(prensor_result, sess)
doc_value = value.get_child_or_error("doc")
self.assertAllEqual([0, 1, 1], doc_value.node.parent_index)
bar2_value = doc_value.get_child_or_error("bar2")
self.assertAllEqual([0, 1, 1, 2], bar2_value.node.parent_index)
self.assertAllEqual([b"a", b"b", b"c", b"d"],
bar2_value.node.values)
keep_me2_value = doc_value.get_child_or_error("keep_me2")
self.assertAllEqual([0, 1], keep_me2_value.node.parent_index)
self.assertAllEqual([False, True], keep_me2_value.node.values)
