def test_get_known_descendants(self): expression = prensor_test_util.create_nested_prensor() expr = create_expression.create_expression_from_prensor(expression) expr_map = expr.get_known_descendants() self.assertIn(path.Path(["doc"]), expr_map) self.assertIn(path.Path(["doc", "bar"]), expr_map) self.assertIn(path.Path(["doc", "keep_me"]), expr_map) self.assertIn(path.Path(["user"]), expr_map) self.assertIn(path.Path(["user", "friends"]), expr_map)
def test_get_positional_index_calculate(self): expr = create_expression.create_expression_from_prensor( prensor_test_util.create_nested_prensor()) new_root, new_path = index.get_positional_index( expr, path.Path(["user", "friends"]), path.get_anonymous_field()) new_field = new_root.get_descendant_or_error(new_path) leaf_node = expression_test_util.calculate_value_slowly(new_field) self.assertAllEqual(leaf_node.parent_index, [0, 1, 1, 2, 3]) self.assertAllEqual(leaf_node.values, [0, 0, 1, 0, 0])
def test_promote_and_calculate(self): """Tests promoting a leaf on a nested tree.""" expr = create_expression.create_expression_from_prensor( prensor_test_util.create_nested_prensor()) new_root, new_path = promote.promote_anonymous( expr, path.Path(["user", "friends"])) new_field = new_root.get_descendant_or_error(new_path) leaf_node = expression_test_util.calculate_value_slowly(new_field) self.assertAllEqual(leaf_node.parent_index, [0, 1, 1, 1, 2]) self.assertAllEqual(leaf_node.values, [b"a", b"b", b"c", b"d", b"e"])
def test_nested_prensor(self): """Tests on a deep expression.""" pren = prensor_test_util.create_nested_prensor() st = prensor_to_structured_tensor.prensor_to_structured_tensor(pren) self.assertAllEqual(st.field_value(["doc", "bar"]), [[[b"a"]], [[b"b", b"c"], [b"d"]], []]) self.assertAllEqual(st.field_value(["doc", "keep_me"]), [[[False]], [[True], []], []]) self.assertAllEqual(st.field_value(["user", "friends"]), [[[b"a"]], [[b"b", b"c"], [b"d"]], [[b"e"]]])
def testPlaceholderExpression(self): pren = prensor_test_util.create_nested_prensor() expected_pren = prensor.create_prensor_from_descendant_nodes({ path.Path([]): prensor.RootNodeTensor(tf.constant(3, dtype=tf.int64)), path.Path(["new_friends"]): prensor.LeafNodeTensor( tf.constant([0, 1, 1, 1, 2], dtype=tf.int64), tf.constant(["a", "b", "c", "d", "e"], dtype=tf.string), True) }) root_schema = mpp.create_schema(is_repeated=True, children={ "doc": { "is_repeated": True, "children": { "bar": { "is_repeated": True, "dtype": tf.string }, "keep_me": { "is_repeated": False, "dtype": tf.bool } } }, "user": { "is_repeated": True, "children": { "friends": { "is_repeated": True, "dtype": tf.string } } } }) exp = placeholder.create_expression_from_schema(root_schema) promote_exp = promote.promote(exp, path.Path(["user", "friends"]), "new_friends") project_exp = project.project(promote_exp, [path.Path(["new_friends"])]) new_friends_exp = project_exp.get_descendant(path.Path(["new_friends" ])) result = calculate.calculate_values([new_friends_exp], feed_dict={exp: pren}) res_node = result[0] exp_node = expected_pren.get_descendant(path.Path(["new_friends" ])).node self.assertAllEqual(res_node.is_repeated, exp_node.is_repeated) self.assertAllEqual(res_node.values, exp_node.values) self.assertAllEqual(res_node.parent_index, exp_node.parent_index)
def test_calculate_promote_anonymous(self): """Performs promote_test.PromoteValuesTest, but with calculate_values.""" for options in options_to_test: expr = create_expression.create_expression_from_prensor( prensor_test_util.create_nested_prensor()) new_root, new_path = promote.promote_anonymous( expr, path.Path(["user", "friends"])) new_field = new_root.get_descendant_or_error(new_path) [leaf_node] = calculate.calculate_values([new_field], options=options) self.assertAllEqual(leaf_node.parent_index, [0, 1, 1, 1, 2]) self.assertAllEqual(leaf_node.values, [b"a", b"b", b"c", b"d", b"e"])
def test_project(self): expr = create_expression.create_expression_from_prensor( prensor_test_util.create_nested_prensor()) projected = expr.project( [path.Path(["user", "friends"]), path.Path(["doc", "keep_me"])]) self.assertIsNotNone( projected.get_descendant(path.Path(["user", "friends"]))) self.assertIsNotNone( projected.get_descendant(path.Path(["doc", "keep_me"]))) self.assertIsNone(projected.get_descendant(path.Path(["doc", "bar"])))
def test_promote(self): expr = create_expression.create_expression_from_prensor( prensor_test_util.create_nested_prensor()) new_root = expr.promote("user.friends", "new_field") new_field = new_root.get_child_or_error("new_field") self.assertIsNotNone(new_field) self.assertTrue(new_field.is_repeated) self.assertEqual(new_field.type, tf.string) self.assertTrue(new_field.is_leaf) leaf_node = expression_test_util.calculate_value_slowly(new_field) self.assertEqual(leaf_node.values.dtype, tf.string) self.assertEqual(new_field.known_field_names(), frozenset())
def test_calculate_promote_named(self): """Performs promote_test.PromoteValuesTest, but with calculate_values.""" for options in options_to_test: expr = create_expression.create_expression_from_prensor( prensor_test_util.create_nested_prensor()) new_root = promote.promote(expr, path.Path(["user", "friends"]), "new_friends") # projected = project.project(new_root, [path.Path(["new_friends"])]) new_field = new_root.get_child_or_error("new_friends") [leaf_node] = calculate.calculate_values([new_field], options=options) self.assertAllEqual(leaf_node.parent_index, [0, 1, 1, 1, 2]) self.assertAllEqual(leaf_node.values, [b"a", b"b", b"c", b"d", b"e"])
def test_map_prensor_to_prensor_with_schema(self): original = create_expression.create_expression_from_prensor( prensor_test_util.create_nested_prensor()) def my_prensor_op(original_prensor): # Note that we are copying over the original root prensor node. The root # node is ignored in the result. return prensor.create_prensor_from_descendant_nodes({ path.Path([]): original_prensor.node, path.Path(["bar2"]): original_prensor.get_child_or_error("bar").node, path.Path(["keep_me2"]): original_prensor.get_child_or_error("keep_me").node }) bar2_feature = schema_pb2.Feature() bar2_feature.value_count.max = 7 keep_me2_feature = schema_pb2.Feature() keep_me2_feature.value_count.max = 10 # Since the top node is actually a child node, we use the child schema. my_output_schema = map_prensor_to_prensor.create_schema( is_repeated=True, children={ "bar2": { "is_repeated": True, "dtype": tf.string, "schema_feature": bar2_feature }, "keep_me2": { "is_repeated": False, "dtype": tf.bool, "schema_feature": keep_me2_feature } }) result = map_prensor_to_prensor.map_prensor_to_prensor( root_expr=original, source=path.Path(["doc"]), paths_needed=[path.Path(["bar"]), path.Path(["keep_me"])], prensor_op=my_prensor_op, output_schema=my_output_schema) doc_result = result.get_child_or_error("doc") bar2_result = doc_result.get_child_or_error("bar2") self.assertEqual(bar2_result.schema_feature.value_count.max, 7) keep_me2_result = doc_result.get_child_or_error("keep_me2") self.assertEqual(keep_me2_result.schema_feature.value_count.max, 10)
def test_promote(self): """Tests promote.promote(...), and indirectly tests set_path.""" expr = create_expression.create_expression_from_prensor( prensor_test_util.create_nested_prensor()) new_root = promote.promote(expr, path.Path(["user", "friends"]), "new_field") new_field = new_root.get_child_or_error("new_field") self.assertIsNotNone(new_field) self.assertTrue(new_field.is_repeated) self.assertEqual(new_field.type, tf.string) self.assertTrue(new_field.is_leaf) leaf_node = expression_test_util.calculate_value_slowly(new_field) self.assertEqual(leaf_node.values.dtype, tf.string) self.assertEqual(new_field.known_field_names(), frozenset())
def test_promote_and_calculate(self): """Tests get_sparse_tensors on a deep tree.""" with self.session(use_gpu=False) as sess: expr = create_expression.create_expression_from_prensor( prensor_test_util.create_nested_prensor()) new_root, new_path = promote.promote_anonymous( expr, path.Path(["user", "friends"])) new_field = new_root.get_descendant_or_error(new_path) leaf_node = expression_test_util.calculate_value_slowly(new_field) parent_index, values = sess.run( [leaf_node.parent_index, leaf_node.values]) self.assertAllEqual(parent_index, [0, 1, 1, 1, 2]) self.assertAllEqual(values, [b"a", b"b", b"c", b"d", b"e"])
def test_prensor_is_composite_tensor(self): for pren in [ prensor_test_util.create_nested_prensor(), prensor_test_util.create_big_prensor(), prensor_test_util.create_deep_prensor() ]: flattened_tensors = tf.nest.flatten(pren, expand_composites=True) self.assertIsInstance(flattened_tensors, list) for t in flattened_tensors: self.assertIsInstance(t, tf.Tensor) packed_pren = tf.nest.pack_sequence_as(pren, flattened_tensors, expand_composites=True) self._assert_prensor_equals(pren, packed_pren)
def test_get_index_from_end(self): expr = create_expression.create_expression_from_prensor( prensor_test_util.create_nested_prensor()) new_root, new_path = index.get_index_from_end( expr, path.Path(["user", "friends"]), path.get_anonymous_field()) new_field = new_root.get_descendant_or_error(new_path) self.assertTrue(new_field.is_repeated) self.assertEqual(new_field.type, tf.int64) self.assertTrue(new_field.is_leaf) self.assertTrue(new_field.calculation_equal(new_field)) self.assertFalse(new_field.calculation_equal(expr)) leaf_node = expression_test_util.calculate_value_slowly(new_field) self.assertEqual(leaf_node.values.dtype, tf.int64) self.assertEqual(new_field.known_field_names(), frozenset())
def test_get_index_from_end_calculate(self): expr = create_expression.create_expression_from_prensor( prensor_test_util.create_nested_prensor()) new_root, new_path = index.get_index_from_end( expr, path.Path(["user", "friends"]), path.get_anonymous_field()) print("test_get_index_from_end_calculate: new_path: {}".format(new_path)) new_field = new_root.get_descendant_or_error(new_path) print("test_get_index_from_end_calculate: new_field: {}".format( str(new_field))) leaf_node = expression_test_util.calculate_value_slowly(new_field) print("test_get_index_from_end_calculate: leaf_node: {}".format( str(leaf_node))) self.assertAllEqual(leaf_node.parent_index, [0, 1, 1, 2, 3]) self.assertAllEqual(leaf_node.values, [-1, -2, -1, -1, -1])
def test_prensor_to_ragged_tensors(self): for options in _OPTIONS_TO_TEST: pren = prensor_test_util.create_nested_prensor() ragged_tensor_map = pren.get_ragged_tensors(options=options) string_tensor_map = { str(k): v for k, v in ragged_tensor_map.items() } string_np_map = self.evaluate(string_tensor_map) self.assertAllEqual(string_np_map["doc.bar"].to_list(), [[[b"a"]], [[b"b", b"c"], [b"d"]], []]) self.assertAllEqual(string_np_map["doc.keep_me"].to_list(), [[[False]], [[True], []], []]) self.assertAllEqual(string_np_map["user.friends"].to_list(), [[[b"a"]], [[b"b", b"c"], [b"d"]], [[b"e"]]])
def test_is_leaf(self): """Tests get_sparse_tensors on a deep expression.""" expression = prensor_test_util.create_nested_prensor() self.assertTrue( expression.get_descendant_or_error(path.Path(["doc", "bar"])).is_leaf) self.assertFalse( expression.get_descendant_or_error(path.Path(["doc"])).is_leaf) self.assertFalse( expression.get_descendant_or_error(path.Path([])).is_leaf) self.assertTrue( expression.get_descendant_or_error(path.Path(["user", "friends"])).is_leaf) self.assertTrue( expression.get_descendant_or_error(path.Path(["doc", "keep_me"])).is_leaf)
def test_add_paths(self): expr = create_expression.create_expression_from_prensor( prensor_test_util.create_nested_prensor()) new_root = expression_add.add_paths( expr, { path.Path(["user", "friends_copy"]): expr.get_descendant_or_error(path.Path(["user", "friends"])) }) new_field = new_root.get_descendant_or_error( path.Path(["user", "friends_copy"])) self.assertIsNotNone(new_field) self.assertTrue(new_field.is_repeated) self.assertEqual(new_field.type, tf.string) self.assertTrue(new_field.is_leaf) leaf_node = expression_test_util.calculate_value_slowly(new_field) self.assertEqual(leaf_node.values.dtype, tf.string) self.assertEqual(new_field.known_field_names(), frozenset())
def test_get_ragged_tensors(self): """Tests get_ragged_tensors on a deep expression.""" for options in options_to_test: expression = prensor_test_util.create_nested_prensor() ragged_tensor_map = prensor_util.get_ragged_tensors( expression, options) string_tensor_map = { str(k): v for k, v in ragged_tensor_map.items() } string_np_map = self.evaluate(string_tensor_map) self.assertAllEqual(string_np_map["doc.bar"].to_list(), [[[b"a"]], [[b"b", b"c"], [b"d"]], []]) self.assertAllEqual(string_np_map["doc.keep_me"].to_list(), [[[False]], [[True], []], []]) self.assertAllEqual(string_np_map["user.friends"].to_list(), [[[b"a"]], [[b"b", b"c"], [b"d"]], [[b"e"]]])
def test_add_to_already_existing_path(self): with self.assertRaises(ValueError): root = create_expression.create_expression_from_prensor( prensor_test_util.create_nested_prensor()) root_1 = expression_add.add_paths( root, { path.Path(["user", "friends_2"]): root.get_descendant_or_error(path.Path(["user", "friends" ])) }) root_2 = expression_add.add_paths( root_1, { path.Path(["user", "friends_3"]): root_1.get_descendant_or_error( path.Path(["user", "friends_2"])) }) expression_add.add_to(root, {path.Path(["user", "friends"]): root_2})
def test_promote_anonymous(self): expr = create_expression.create_expression_from_prensor( prensor_test_util.create_nested_prensor()) new_root, new_field = promote.promote_anonymous( expr, path.Path(["user", "friends"])) new_field = new_root.get_descendant_or_error(new_field) self.assertTrue(new_field.is_repeated) self.assertEqual(new_field.type, tf.string) self.assertTrue(new_field.is_leaf) self.assertFalse(new_field.calculation_is_identity()) self.assertTrue(new_field.calculation_equal(new_field)) self.assertFalse(new_field.calculation_equal(expr)) leaf_node = expression_test_util.calculate_value_slowly(new_field) self.assertEqual(leaf_node.values.dtype, tf.string) self.assertEqual(new_field.known_field_names(), frozenset()) sources = new_field.get_source_expressions() self.assertLen(sources, 2) self.assertIs( expr.get_descendant_or_error(path.Path(["user", "friends"])), sources[0]) self.assertIs(expr.get_child_or_error("user"), sources[1])
def test_prensor_to_sparse_tensors(self): for options in _OPTIONS_TO_TEST: pren = prensor_test_util.create_nested_prensor() sparse_tensor_map = pren.get_sparse_tensors(options=options) string_tensor_map = { str(k): v for k, v in sparse_tensor_map.items() } self.assertAllEqual(string_tensor_map["doc.bar"].indices, [[0, 0, 0], [1, 0, 0], [1, 0, 1], [1, 1, 0]]) self.assertAllEqual(string_tensor_map["doc.bar"].values, [b"a", b"b", b"c", b"d"]) self.assertAllEqual(string_tensor_map["doc.keep_me"].indices, [[0, 0], [1, 0]]) self.assertAllEqual(string_tensor_map["doc.keep_me"].values, [False, True]) self.assertAllEqual( string_tensor_map["user.friends"].indices, [[0, 0, 0], [1, 0, 0], [1, 0, 1], [1, 1, 0], [2, 0, 0]]) self.assertAllEqual(string_tensor_map["user.friends"].values, [b"a", b"b", b"c", b"d", b"e"])
def test_add_to(self): root = create_expression.create_expression_from_prensor( prensor_test_util.create_nested_prensor()) root_1 = expression_add.add_paths( root, { path.Path(["user", "friends_2"]): root.get_descendant_or_error(path.Path(["user", "friends"])) }) root_2 = expression_add.add_paths( root_1, { path.Path(["user", "friends_3"]): root_1.get_descendant_or_error(path.Path(["user", "friends_2" ])) }) root_3 = expression_add.add_to( root, {path.Path(["user", "friends_3"]): root_2}) new_field = root_3.get_descendant_or_error( path.Path(["user", "friends_3"])) self.assertIsNotNone(new_field) self.assertTrue(new_field.is_repeated) self.assertEqual(new_field.type, tf.string) leaf_node = expression_test_util.calculate_value_slowly(new_field) self.assertEqual(leaf_node.values.dtype, tf.string)
def test_map_prensor_to_prensor(self): original = create_expression.create_expression_from_prensor( prensor_test_util.create_nested_prensor()) def my_prensor_op(original_prensor): # Note that we are copying over the original root prensor node. The root # node is ignored in the result. return prensor.create_prensor_from_descendant_nodes({ path.Path([]): original_prensor.node, path.Path(["bar2"]): original_prensor.get_child_or_error("bar").node, path.Path(["keep_me2"]): original_prensor.get_child_or_error("keep_me").node }) # Since the top node is actually a child node, we use the child schema. my_output_schema = map_prensor_to_prensor.create_schema( is_repeated=True, children={ "bar2": { "is_repeated": True, "dtype": tf.string }, "keep_me2": { "is_repeated": False, "dtype": tf.bool } }) result = map_prensor_to_prensor.map_prensor_to_prensor( root_expr=original, source=path.Path(["doc"]), paths_needed=[path.Path(["bar"]), path.Path(["keep_me"])], prensor_op=my_prensor_op, output_schema=my_output_schema) doc_result = result.get_child_or_error("doc") bar_result = doc_result.get_child_or_error("bar") keep_me_result = doc_result.get_child_or_error("keep_me") bar2_result = doc_result.get_child_or_error("bar2") keep_me2_result = doc_result.get_child_or_error("keep_me2") self.assertIsNone(doc_result.get_child("missing_field")) self.assertTrue(bar_result.is_repeated) self.assertTrue(bar2_result.is_repeated) self.assertEqual(bar_result.type, tf.string) self.assertEqual(bar2_result.type, tf.string) self.assertFalse(keep_me_result.is_repeated) self.assertFalse(keep_me2_result.is_repeated) self.assertEqual(keep_me_result.type, tf.bool) self.assertEqual(keep_me2_result.type, tf.bool) [prensor_result] = calculate.calculate_prensors([result]) doc_value = prensor_result.get_child_or_error("doc") self.assertAllEqual([0, 1, 1], doc_value.node.parent_index) bar2_value = doc_value.get_child_or_error("bar2") self.assertAllEqual([0, 1, 1, 2], bar2_value.node.parent_index) self.assertAllEqual([b"a", b"b", b"c", b"d"], bar2_value.node.values) keep_me2_value = doc_value.get_child_or_error("keep_me2") self.assertAllEqual([0, 1], keep_me2_value.node.parent_index) self.assertAllEqual([False, True], keep_me2_value.node.values)