def test_promote_substructure(self): """Tests promote.promote(...) of substructure.""" expr = create_expression.create_expression_from_prensor( prensor_test_util.create_deep_prensor()) new_root = promote.promote(expr, path.Path(["event", "doc"]), "new_field") new_field = new_root.get_child_or_error("new_field") self.assertIsNotNone(new_field) self.assertTrue(new_field.is_repeated) self.assertEqual(new_field.known_field_names(), frozenset(["bar", "keep_me"])) bar_expr = new_field.get_child_or_error("bar") self.assertIsNotNone(bar_expr) self.assertTrue(bar_expr.is_repeated) self.assertEqual(bar_expr.type, tf.string) self.assertTrue(bar_expr.is_leaf) keep_me_expr = new_field.get_child_or_error("keep_me") self.assertIsNotNone(keep_me_expr) self.assertFalse(keep_me_expr.is_repeated) self.assertEqual(keep_me_expr.type, tf.bool) self.assertTrue(keep_me_expr.is_leaf) child_node = expression_test_util.calculate_value_slowly(new_field) self.assertEqual(child_node.size, 3) self.assertTrue(child_node.is_repeated) bar_node = expression_test_util.calculate_value_slowly(bar_expr) self.assertEqual(bar_node.values.dtype, tf.string) keep_me_node = expression_test_util.calculate_value_slowly( keep_me_expr) self.assertEqual(keep_me_node.values.dtype, tf.bool)
def test_any_path(self): my_any_0 = test_any_pb2.MessageWithAny() my_value_0 = test_pb2.AllSimple() my_value_0.optional_int32 = 17 my_any_0.my_any.Pack(my_value_0) expr = proto.create_expression_from_proto( [my_any_0.SerializeToString()], test_any_pb2.MessageWithAny.DESCRIPTOR) new_root = promote.promote( expr, path.Path( ["my_any", "(struct2tensor.test.AllSimple)", "optional_int32"]), "new_int32") new_field = new_root.get_descendant_or_error( path.Path(["my_any", "new_int32"])) result = calculate_with_source_paths.calculate_prensors_with_source_paths( [new_field]) prensor_result, proto_summary_result = result self.assertLen(prensor_result, 1) self.assertLen(proto_summary_result, 1) leaf_node = prensor_result[0].node self.assertAllEqual(leaf_node.parent_index, [0]) self.assertAllEqual(leaf_node.values, [17]) list_of_paths = proto_summary_result[0].paths expected = [ path.Path( ["my_any", "(struct2tensor.test.AllSimple)", "optional_int32"]) ] self.equal_ignore_order(list_of_paths, expected)
def test_calculate_prensors_with_source_paths_with_transform(self): """Tests get_sparse_tensors on a deep tree with a transformed field.""" expr = proto_test_util._get_expression_from_session_empty_user_info() # Let's make it non-trivial by transforming the data. def _reverse(parent_indices, values): return parent_indices, tf.reverse(values, axis=[-1]) expr = proto.create_transformed_field(expr, path.Path(["event"]), "reversed_event", _reverse) new_root = promote.promote( expr, path.Path(["reversed_event", "action", "doc_id"]), "action_doc_ids") # A poor-man's reroot. new_field = new_root.get_descendant_or_error( path.Path(["reversed_event", "action_doc_ids"])) result = calculate_with_source_paths.calculate_prensors_with_source_paths( [new_field]) prensor_result, proto_summary_result = result self.assertLen(prensor_result, 1) self.assertLen(proto_summary_result, 1) leaf_node = prensor_result[0].node self.assertAllEqual(leaf_node.parent_index, [0, 0, 0, 1, 2, 2, 3, 4, 4]) self.assertAllEqual( leaf_node.values, [b"h", b"i", b"j", b"g", b"e", b"f", b"c", b"a", b"b"]) list_of_paths = proto_summary_result[0].paths expected = [path.Path(["event", "action", "doc_id"])] self.equal_ignore_order(list_of_paths, expected)
def test_calculate_prensors_with_source_paths(self): """Tests get_sparse_tensors on a deep tree.""" with self.session(use_gpu=False) as sess: expr = proto_test_util._get_expression_from_session_empty_user_info( ) # Let's make it non-trivial by transforming the data. new_root = promote.promote( expr, path.Path(["event", "action", "doc_id"]), "action_doc_ids") # A poor-man's reroot. new_field = new_root.get_descendant_or_error( path.Path(["event", "action_doc_ids"])) result = calculate_with_source_paths.calculate_prensors_with_source_paths( [new_field]) prensor_result, proto_summary_result = result self.assertLen(prensor_result, 1) self.assertLen(proto_summary_result, 1) leaf_node = prensor_result[0].node [parent_index, values] = sess.run([leaf_node.parent_index, leaf_node.values]) self.assertAllEqual(parent_index, [0, 0, 1, 2, 2, 3, 4, 4, 4]) self.assertAllEqual( values, [b"a", b"b", b"c", b"e", b"f", b"g", b"h", b"i", b"j"]) list_of_paths = proto_summary_result[0].paths expected = [path.Path(["event", "action", "doc_id"])] self.equal_ignore_order(list_of_paths, expected)
def testPlaceholderExpression(self): pren = prensor_test_util.create_nested_prensor() expected_pren = prensor.create_prensor_from_descendant_nodes({ path.Path([]): prensor.RootNodeTensor(tf.constant(3, dtype=tf.int64)), path.Path(["new_friends"]): prensor.LeafNodeTensor( tf.constant([0, 1, 1, 1, 2], dtype=tf.int64), tf.constant(["a", "b", "c", "d", "e"], dtype=tf.string), True) }) root_schema = mpp.create_schema(is_repeated=True, children={ "doc": { "is_repeated": True, "children": { "bar": { "is_repeated": True, "dtype": tf.string }, "keep_me": { "is_repeated": False, "dtype": tf.bool } } }, "user": { "is_repeated": True, "children": { "friends": { "is_repeated": True, "dtype": tf.string } } } }) exp = placeholder.create_expression_from_schema(root_schema) promote_exp = promote.promote(exp, path.Path(["user", "friends"]), "new_friends") project_exp = project.project(promote_exp, [path.Path(["new_friends"])]) new_friends_exp = project_exp.get_descendant(path.Path(["new_friends" ])) result = calculate.calculate_values([new_friends_exp], feed_dict={exp: pren}) res_node = result[0] exp_node = expected_pren.get_descendant(path.Path(["new_friends" ])).node self.assertAllEqual(res_node.is_repeated, exp_node.is_repeated) self.assertAllEqual(res_node.values, exp_node.values) self.assertAllEqual(res_node.parent_index, exp_node.parent_index)
def test_calculate_promote_named(self): """Performs promote_test.PromoteValuesTest, but with calculate_values.""" for options in options_to_test: expr = create_expression.create_expression_from_prensor( prensor_test_util.create_nested_prensor()) new_root = promote.promote(expr, path.Path(["user", "friends"]), "new_friends") # projected = project.project(new_root, [path.Path(["new_friends"])]) new_field = new_root.get_child_or_error("new_friends") [leaf_node] = calculate.calculate_values([new_field], options=options) self.assertAllEqual(leaf_node.parent_index, [0, 1, 1, 1, 2]) self.assertAllEqual(leaf_node.values, [b"a", b"b", b"c", b"d", b"e"])
def test_promote(self): """Tests promote.promote(...), and indirectly tests set_path.""" expr = create_expression.create_expression_from_prensor( prensor_test_util.create_nested_prensor()) new_root = promote.promote(expr, path.Path(["user", "friends"]), "new_field") new_field = new_root.get_child_or_error("new_field") self.assertIsNotNone(new_field) self.assertTrue(new_field.is_repeated) self.assertEqual(new_field.type, tf.string) self.assertTrue(new_field.is_leaf) leaf_node = expression_test_util.calculate_value_slowly(new_field) self.assertEqual(leaf_node.values.dtype, tf.string) self.assertEqual(new_field.known_field_names(), frozenset())
def testPromoteAndProjectExpression(self): filenames = [ "struct2tensor/testdata/parquet_testdata/dremel_example.parquet" ] batch_size = 2 exp = parquet.create_expression_from_parquet_file(filenames) new_exp = promote.promote(exp, path.Path(["Name", "Language", "Code"]), "new_code") new_code_project_exp = project.project( new_exp, [path.Path(["Name", "new_code"])]) docid_project_exp = project.project(exp, [path.Path(["DocId"])]) pqds = parquet.calculate_parquet_values( [new_code_project_exp, docid_project_exp], exp, filenames, batch_size) new_code_expected = prensor.create_prensor_from_descendant_nodes({ path.Path([]): prensor.RootNodeTensor(tf.constant(2, dtype=tf.int64)), path.Path(["Name"]): prensor.ChildNodeTensor(tf.constant([0, 0, 0, 1], dtype=tf.int64), True), path.Path(["Name", "new_code"]): prensor.LeafNodeTensor(tf.constant([0, 0, 2], dtype=tf.int64), tf.constant([b"en-us", b"en", b"en-gb"]), True) }) docid_expected = prensor.create_prensor_from_descendant_nodes({ path.Path([]): prensor.RootNodeTensor(tf.constant(2, dtype=tf.int64)), path.Path(["DocId"]): prensor.LeafNodeTensor(tf.constant([0, 1], dtype=tf.int64), tf.constant([10, 20], dtype=tf.int64), False) }) for ele in pqds: new_code_pren = ele[0] docid_pren = ele[1] self._assertPrensorEqual(new_code_pren, new_code_expected) self._assertPrensorEqual(docid_pren, docid_expected)