def calculate_value_slowly(
expr: expression.Expression,
destinations: Optional[Sequence[expression.Expression]] = None,
options: Optional[calculate_options.Options] = None
) -> prensor.NodeTensor:
"""A calculation of the node tensor of an expression, without optimization.
This will not do any common subexpression elimination or caching of
node tensors, and will likely be very slow for larger operations.
Args:
expr: The expression to calculate.
destinations: Where the calculation will be used (None implies directly)
options: Calculation options for individual calculations
Returns:
The node tensor of the expression.
"""
new_options = calculate_options.get_default_options(
) if options is None else options
source_node_tensors = [
calculate_value_slowly(x, [expr], new_options)
for x in expr.get_source_expressions()
]
real_dest = [] if destinations is None else destinations
return expr.calculate(source_node_tensors, real_dest, new_options)
def calculate_list_map(expr: expression.Expression, evaluator):
"""Calculate a map from paths to nested lists, representing the leafs."""
[my_prensor] = calculate.calculate_prensors([expr])
ragged_tensor_map = prensor_util.get_ragged_tensors(
my_prensor, calculate_options.get_default_options())
string_tensor_map = {str(k): v for k, v in ragged_tensor_map.items()}
string_np_map = evaluator.evaluate(string_tensor_map)
return {k: v.to_list() for k, v in string_np_map.items()}
def calculate_list_map(expr: expression.Expression,
evaluator,
options: Optional[calculate_options.Options] = None):
"""Calculate a map from paths to nested lists, representing the leafs."""
[my_prensor] = calculate.calculate_prensors([expr], options=options)
if not options:
options = calculate_options.get_default_options()
ragged_tensor_map = my_prensor.get_ragged_tensors(options)
string_tensor_map = {str(k): v for k, v in ragged_tensor_map.items()}
string_np_map = evaluator.evaluate(string_tensor_map)
return {k: v.to_list() for k, v in string_np_map.items()}
def get_ragged_tensors(
t: prensor.Prensor,
options: calculate_options.Options = calculate_options.get_default_options(
)
) -> Mapping[path.Path, tf.RaggedTensor]:
"""Gets ragged tensors for all the leaves of the prensor expression.
Args:
t: The Prensor to extract tensors from.
options: used to pass options for calculating ragged tensors.
Returns:
A map from paths to ragged tensors.
"""
return {
p: _get_ragged_tensor_from_leaf_node_path(v, options)
for p, v in _get_leaf_node_paths(t).items()
}
def _get_ragged_tensor_from_leaf_node_path(
nodes: _LeafNodePath,
options: calculate_options.Options = calculate_options.get_default_options(
)
) -> tf.RaggedTensor:
"""Gets a ragged tensor from a leaf node path."""
if not nodes.middle:
return from_value_rowids_bridge(nodes.tail.values,
value_rowids=nodes.tail.parent_index,
nrows=nodes.head.size,
validate=options.ragged_checks)
deeper_ragged = _get_ragged_tensor_from_leaf_node_path(
_get_leaf_node_path_suffix(nodes), options)
first_child_node = nodes.middle[0]
return from_value_rowids_bridge(deeper_ragged,
value_rowids=first_child_node.parent_index,
nrows=nodes.head.size,
validate=options.ragged_checks)
def calculate_values_with_graph(
expressions,
options = None
):
"""Calculates the values of the expressions, and the graph used.
Note that this does not return prensors, but instead a list of NodeTensors.
Args:
expressions: a list of expressions to calculate.
options: options for calculations, passed to calculate(...).
Returns:
the list of values and the graph used to calculate them.
"""
if options is None:
options = calculate_options.get_default_options()
expression_graph = _create_graph(expressions, options)
return ([expression_graph.get_value_or_die(x) for x in expressions],
expression_graph)
def get_sparse_tensors(
t: prensor.Prensor,
options: calculate_options.Options = calculate_options.get_default_options(
)
) -> Mapping[path.Path, tf.SparseTensor]:
"""Gets sparse tensors for all the leaves of the prensor expression.
Args:
t: The Prensor to extract tensors from.
options: Currently unused.
Returns:
A map from paths to sparse tensors.
"""
del options
return {
p: _get_sparse_tensor_from_leaf_node_path(v)
for p, v in _get_leaf_node_paths(t).items()
}
def test_create_proto_index_directly_reroot_at_action_sparse_dense(self):
sessions = [
"""
event {
action {}
action {}
}
event {}
event { action {} }
""", "", """
event {}
event {
action {}
action {}
}
event { }
"""
]
with self.session(use_gpu=False) as sess:
expr = proto_test_util.text_to_expression(sessions, test_pb2.Session)
reroot_expr = expr.reroot("event.action")
# Reroot with a depth > 1 (all the other cases are depth == 1)
[prensor_tree] = calculate.calculate_prensors([
reroot_expr.create_proto_index(
"proto_index_directly_reroot_at_action")
])
proto_index_node = prensor_tree.get_child_or_error(
"proto_index_directly_reroot_at_action").node
self.assertFalse(proto_index_node.is_repeated)
sparse_tensors = prensor_util.get_sparse_tensors(
prensor_tree, calculate_options.get_default_options())
proto_index_directly_reroot_at_action = sparse_tensors[path.Path(
["proto_index_directly_reroot_at_action"])]
[sparse_value, dense_value] = sess.run([
proto_index_directly_reroot_at_action,
tf.sparse_tensor_to_dense(proto_index_directly_reroot_at_action)
])
self.assertAllEqual(sparse_value.values, [0, 0, 0, 2, 2])
self.assertAllEqual(sparse_value.indices, [[0], [1], [2], [3], [4]])
self.assertAllEqual(sparse_value.dense_shape, [5])
self.assertAllEqual(dense_value, [0, 0, 0, 2, 2])
def get_ragged_tensor(
t: prensor.Prensor,
p: path.Path,
options: calculate_options.Options = calculate_options.get_default_options(
)
) -> tf.RaggedTensor:
"""Get a ragged tensor for a path.
All steps are represented in the ragged tensor.
Args:
t: The Prensor to extract tensors from.
p: the path to a leaf node in `t`.
options: used to pass options for calculating ragged tensors.
Returns:
A ragged tensor containing values of the leaf node, preserving the
structure along the path. Raises an error if the path is not found.
"""
leaf_node_path = _get_leaf_node_path(p, t)
return _get_ragged_tensor_from_leaf_node_path(leaf_node_path, options)
def calculate_values_with_graph(
expressions,
options = None,
feed_dict = None
):
"""Calculates the values of the expressions, and the graph used.
Note that this does not return prensors, but instead a list of NodeTensors.
Args:
expressions: a list of expressions to calculate.
options: options for calculations, passed to calculate(...).
feed_dict: a dictionary, mapping expression to prensor that will be used
as the initial expression in the expression graph.
Returns:
the list of values and the graph used to calculate them.
"""
if options is None:
options = calculate_options.get_default_options()
expression_graph = _create_graph(expressions, options, feed_dict=feed_dict)
return ([expression_graph.get_value_or_die(x) for x in expressions],
expression_graph)
def _get_sparse_tensor(
t: Prensor,
p: path.Path,
options: calculate_options.Options = calculate_options.get_default_options(
)
) -> tf.SparseTensor:
"""Gets a sparse tensor for path p.
Note that any optional fields are not registered as dimensions, as they can't
be represented in a sparse tensor.
Args:
t: The Prensor to extract tensors from.
p: The path to a leaf node in `t`.
options: Currently unused.
Returns:
A sparse tensor containing values of the leaf node, preserving the
structure along the path. Raises an error if the path is not found.
"""
del options
leaf_node_path = _get_leaf_node_path(p, t)
return _get_sparse_tensor_from_leaf_node_path(leaf_node_path)
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests for struct2tensor.prensor."""
from struct2tensor import calculate_options
from struct2tensor import path
from struct2tensor import prensor
from struct2tensor.test import prensor_test_util
import tensorflow as tf
from tensorflow.python.framework import test_util # pylint: disable=g-direct-tensorflow-import
_OPTIONS_TO_TEST = [
calculate_options.get_default_options(),
calculate_options.get_options_with_minimal_checks()
]
@test_util.run_all_in_graph_and_eager_modes
class PrensorTest(tf.test.TestCase):
def _assert_prensor_equals(self, lhs, rhs):
if isinstance(lhs.node, prensor.RootNodeTensor):
self.assertIsInstance(rhs.node, prensor.RootNodeTensor)
self.assertIs(lhs.node.size, rhs.node.size)
elif isinstance(lhs.node, prensor.ChildNodeTensor):
self.assertIsInstance(rhs.node, prensor.ChildNodeTensor)
self.assertIs(lhs.node.parent_index, rhs.node.parent_index)
self.assertEqual(lhs.node.is_repeated, rhs.node.is_repeated)
else:
def _test_assert_raises(self, test_runner):
with self.assertRaises(tf.errors.InvalidArgumentError):
test_runner(calculate_options.get_default_options())
test_runner(calculate_options.get_options_with_minimal_checks())
def _get_calculate_options(self, use_string_view): options = calculate_options.get_default_options() options.use_string_view = use_string_view return options
