def test_create_temporary_table_from_descriptor(self): from pyflink.table.schema import Schema t_env = self.t_env catalog = t_env.get_current_catalog() database = t_env.get_current_database() schema = Schema.new_builder().column("f0", DataTypes.INT()).build() t_env.create_temporary_table( "T", TableDescriptor.for_connector("fake") .schema(schema) .option("a", "Test") .build()) self.assertFalse(t_env.get_catalog(catalog).table_exists(ObjectPath(database, "T"))) gateway = get_gateway() catalog_table = CatalogBaseTable( t_env._j_tenv.getCatalogManager() .getTable(gateway.jvm.ObjectIdentifier.of(catalog, database, "T")) .get() .getTable()) self.assertEqual(schema, catalog_table.get_unresolved_schema()) self.assertEqual("fake", catalog_table.get_options().get("connector")) self.assertEqual("Test", catalog_table.get_options().get("a"))
def get_j_obj_to_py_obj_rules(): """ Provides rules to transform from Java Objects to Python objects. Batch/StreamOperators are not included. :return: """ def get_all_subclasses(cls): return set(cls.__subclasses__()).union( [s for c in cls.__subclasses__() for s in get_all_subclasses(c)]) from ..vector import VectorIterator from pyflink.table.catalog import ObjectPath rules = { "org.apache.flink.api.java.tuple.Tuple2": lambda v: (j_value_to_py_value(v.f0), j_value_to_py_value(v.f1)), 'com.alibaba.alink.common.linalg.DenseVector$DenseVectorIterator': VectorIterator, 'com.alibaba.alink.common.linalg.SparseVector$SparseVectorVectorIterator': VectorIterator, 'org.apache.flink.table.catalog.ObjectPath': lambda v: ObjectPath(j_object_path=v) } from ..bases.j_obj_wrapper import JavaObjectWrapper py_classes = get_all_subclasses(JavaObjectWrapper) for py_cls in py_classes: if hasattr(py_cls, '_j_cls_name'): # noinspection PyProtectedMember rules[py_cls._j_cls_name] = py_cls return rules
def get_catalog_object(self): tempdir = tempfile.TemporaryDirectory().name os.makedirs(tempdir, exist_ok=True) url = os.path.join(tempdir, "test_sqlite_db") sqlite = SqliteCatalog("sqlite_test_catalog", "test_sqlite_db", "3.19.3", url) object_path = ObjectPath("test_sqlite_db", "test_table") catalog_object = CatalogObject(sqlite, object_path) return sqlite, catalog_object
def setUp(self): super(CatalogTestBase, self).setUp() gateway = get_gateway() self.catalog = Catalog(gateway.jvm.GenericInMemoryCatalog(self.test_catalog_name)) self.path1 = ObjectPath(self.db1, self.t1) self.path2 = ObjectPath(self.db2, self.t2) self.path3 = ObjectPath(self.db1, self.t2) self.path4 = ObjectPath(self.db1, self.t3) self.non_exist_db_path = ObjectPath.from_string("non.exist") self.non_exist_object_path = ObjectPath.from_string("db1.nonexist")
def test_partition_exists(self): self.catalog.create_database(self.db1, self.create_db(), False) self.catalog.create_table(self.path1, self.create_partitioned_table(), False) self.catalog.create_partition(self.path1, self.create_partition_spec(), self.create_partition(), False) self.assertTrue(self.catalog.partition_exists(self.path1, self.create_partition_spec())) self.assertFalse(self.catalog.partition_exists(self.path2, self.create_partition_spec())) self.assertFalse(self.catalog.partition_exists(ObjectPath.from_string("non.exist"), self.create_partition_spec()))
def test_create_table_from_descriptor(self): from pyflink.table.schema import Schema catalog = self.t_env.get_current_catalog() database = self.t_env.get_current_database() schema = Schema.new_builder().column("f0", DataTypes.INT()).build() self.t_env.create_table( "T", TableDescriptor.for_connector("fake").schema(schema).option( "a", "Test").build()) object_path = ObjectPath(database, "T") self.assertTrue( self.t_env.get_catalog(catalog).table_exists(object_path)) catalog_table = self.t_env.get_catalog(catalog).get_table(object_path) self.assertEqual(schema, catalog_table.get_unresolved_schema()) self.assertEqual("fake", catalog_table.get_options().get("connector")) self.assertEqual("Test", catalog_table.get_options().get("a"))
class CatalogTestBase(PyFlinkTestCase): db1 = "db1" db2 = "db2" non_exist_database = "non-exist-db" t1 = "t1" t2 = "t2" t3 = "t3" test_catalog_name = "test-catalog" test_comment = "test comment" def setUp(self): super(CatalogTestBase, self).setUp() gateway = get_gateway() self.catalog = Catalog( gateway.jvm.GenericInMemoryCatalog(self.test_catalog_name)) self.path1 = ObjectPath(self.db1, self.t1) self.path2 = ObjectPath(self.db2, self.t2) self.path3 = ObjectPath(self.db1, self.t2) self.path4 = ObjectPath(self.db1, self.t3) self.non_exist_db_path = ObjectPath.from_string("non.exist") self.non_exist_object_path = ObjectPath.from_string("db1.nonexist") def check_catalog_database_equals(self, cd1, cd2): self.assertEqual(cd1.get_comment(), cd2.get_comment()) self.assertEqual(cd1.get_properties(), cd2.get_properties()) def check_catalog_table_equals(self, t1, t2): self.assertEqual(t1.get_schema(), t2.get_schema()) self.assertEqual(t1.get_properties(), t2.get_properties()) self.assertEqual(t1.get_comment(), t2.get_comment()) def check_catalog_view_equals(self, v1, v2): self.assertEqual(v1.get_schema(), v1.get_schema()) self.assertEqual(v1.get_properties(), v2.get_properties()) self.assertEqual(v1.get_comment(), v2.get_comment()) def check_catalog_function_equals(self, f1, f2): self.assertEqual(f1.get_class_name(), f2.get_class_name()) self.assertEqual(f1.get_properties(), f2.get_properties()) def check_catalog_partition_equals(self, p1, p2): self.assertEqual(p1.get_properties(), p2.get_properties()) @staticmethod def create_db(): gateway = get_gateway() j_database = gateway.jvm.CatalogDatabaseImpl( {"k1": "v1"}, CatalogTestBase.test_comment) return CatalogDatabase(j_database) @staticmethod def create_another_db(): gateway = get_gateway() j_database = gateway.jvm.CatalogDatabaseImpl( {"k2": "v2"}, "this is another database.") return CatalogDatabase(j_database) @staticmethod def create_table_schema(): return TableSchema( ["first", "second", "third"], [DataTypes.STRING(), DataTypes.INT(), DataTypes.STRING()]) @staticmethod def create_another_table_schema(): return TableSchema( ["first2", "second", "third"], [DataTypes.STRING(), DataTypes.STRING(), DataTypes.STRING()]) @staticmethod def get_batch_table_properties(): return {"is_streaming": "false"} @staticmethod def get_streaming_table_properties(): return {"is_streaming": "true"} @staticmethod def create_partition_keys(): return ["second", "third"] @staticmethod def create_table(): gateway = get_gateway() table_schema = CatalogTestBase.create_table_schema() j_table = gateway.jvm.GenericCatalogTable( table_schema._j_table_schema, CatalogTestBase.get_batch_table_properties(), CatalogTestBase.test_comment) return CatalogBaseTable(j_table) @staticmethod def create_another_table(): gateway = get_gateway() table_schema = CatalogTestBase.create_another_table_schema() j_table = gateway.jvm.GenericCatalogTable( table_schema._j_table_schema, CatalogTestBase.get_batch_table_properties(), CatalogTestBase.test_comment) return CatalogBaseTable(j_table) @staticmethod def create_stream_table(): gateway = get_gateway() table_schema = CatalogTestBase.create_table_schema() j_table = gateway.jvm.GenericCatalogTable( table_schema._j_table_schema, CatalogTestBase.get_streaming_table_properties(), CatalogTestBase.test_comment) return CatalogBaseTable(j_table) @staticmethod def create_partitioned_table(): gateway = get_gateway() table_schema = CatalogTestBase.create_table_schema() j_table = gateway.jvm.GenericCatalogTable( table_schema._j_table_schema, CatalogTestBase.create_partition_keys(), CatalogTestBase.get_batch_table_properties(), CatalogTestBase.test_comment) return CatalogBaseTable(j_table) @staticmethod def create_another_partitioned_table(): gateway = get_gateway() table_schema = CatalogTestBase.create_another_table_schema() j_table = gateway.jvm.GenericCatalogTable( table_schema._j_table_schema, CatalogTestBase.create_partition_keys(), CatalogTestBase.get_batch_table_properties(), CatalogTestBase.test_comment) return CatalogBaseTable(j_table) @staticmethod def create_view(): gateway = get_gateway() table_schema = CatalogTestBase.create_table_schema() j_view = gateway.jvm.GenericCatalogView( "select * from t1", "select * from test-catalog.db1.t1", table_schema._j_table_schema, {}, "This is a view") return CatalogBaseTable(j_view) @staticmethod def create_another_view(): gateway = get_gateway() table_schema = CatalogTestBase.create_another_table_schema() j_view = gateway.jvm.GenericCatalogView( "select * from t2", "select * from test-catalog.db2.t2", table_schema._j_table_schema, {}, "This is another view") return CatalogBaseTable(j_view) @staticmethod def create_function(): gateway = get_gateway() j_function = gateway.jvm.GenericCatalogFunction("MyFunction", {}) return CatalogFunction(j_function) @staticmethod def create_another_function(): gateway = get_gateway() j_function = gateway.jvm.GenericCatalogFunction( "MyAnotherFunction", {}) return CatalogFunction(j_function) @staticmethod def create_partition_spec(): gateway = get_gateway() j_partition_spec = gateway.jvm.CatalogPartitionSpec({ "third": "2000", "second": "bob" }) return CatalogPartitionSpec(j_partition_spec) @staticmethod def create_another_partition_spec(): gateway = get_gateway() j_partition_spec = gateway.jvm.CatalogPartitionSpec({ "third": "2010", "second": "bob" }) return CatalogPartitionSpec(j_partition_spec) @staticmethod def create_partition(): gateway = get_gateway() j_partition = gateway.jvm.GenericCatalogPartition( CatalogTestBase.get_batch_table_properties(), "Generic batch table") return CatalogPartition(j_partition) @staticmethod def create_partition_spec_subset(): gateway = get_gateway() j_partition_spec = gateway.jvm.CatalogPartitionSpec({"second": "bob"}) return CatalogPartitionSpec(j_partition_spec) @staticmethod def create_another_partition_spec_subset(): gateway = get_gateway() j_partition_spec = gateway.jvm.CatalogPartitionSpec({"third": "2000"}) return CatalogPartitionSpec(j_partition_spec) @staticmethod def create_invalid_partition_spec_subset(): gateway = get_gateway() j_partition_spec = gateway.jvm.CatalogPartitionSpec({"third": "2010"}) return CatalogPartitionSpec(j_partition_spec) def test_create_db(self): self.assertFalse(self.catalog.database_exists(self.db1)) catalog_db = self.create_db() self.catalog.create_database(self.db1, catalog_db, False) self.assertTrue(self.catalog.database_exists(self.db1)) self.check_catalog_database_equals(catalog_db, self.catalog.get_database(self.db1)) def test_create_db_database_already_exist_exception(self): self.catalog.create_database(self.db1, self.create_db(), False) with self.assertRaises(DatabaseAlreadyExistException): self.catalog.create_database(self.db1, self.create_db(), False) def test_create_db_database_already_exist_ignored(self): catalog_db = self.create_db() self.catalog.create_database(self.db1, catalog_db, False) dbs = self.catalog.list_databases() self.check_catalog_database_equals(catalog_db, self.catalog.get_database(self.db1)) self.assertEqual(2, len(dbs)) self.assertEqual( {self.db1, self.catalog.get_default_database()}, set(dbs)) self.catalog.create_database(self.db1, self.create_another_db(), True) self.check_catalog_database_equals(catalog_db, self.catalog.get_database(self.db1)) self.assertEqual(2, len(dbs)) self.assertEqual( {self.db1, self.catalog.get_default_database()}, set(dbs)) def test_get_db_database_not_exist_exception(self): with self.assertRaises(DatabaseNotExistException): self.catalog.get_database("nonexistent") def test_drop_db(self): self.catalog.create_database(self.db1, self.create_db(), False) self.assertTrue(self.catalog.database_exists(self.db1)) self.catalog.drop_database(self.db1, False) self.assertFalse(self.catalog.database_exists(self.db1)) def test_drop_db_database_not_exist_exception(self): with self.assertRaises(DatabaseNotExistException): self.catalog.drop_database(self.db1, False) def test_drop_db_database_not_exist_ignore(self): self.catalog.drop_database(self.db1, True) def test_drop_db_database_not_empty_exception(self): self.catalog.create_database(self.db1, self.create_db(), False) self.catalog.create_table(self.path1, self.create_table(), False) with self.assertRaises(DatabaseNotEmptyException): self.catalog.drop_database(self.db1, True) def test_alter_db(self): db = self.create_db() self.catalog.create_database(self.db1, db, False) new_db = self.create_another_db() self.catalog.alter_database(self.db1, new_db, False) new_properties = self.catalog.get_database(self.db1).get_properties() old_properties = db.get_properties() self.assertFalse( all(k in new_properties for k in old_properties.keys())) self.check_catalog_database_equals(new_db, self.catalog.get_database(self.db1)) def test_alter_db_database_not_exist_exception(self): with self.assertRaises(DatabaseNotExistException): self.catalog.alter_database("nonexistent", self.create_db(), False) def test_alter_db_database_not_exist_ignored(self): self.catalog.alter_database("nonexistent", self.create_db(), True) self.assertFalse(self.catalog.database_exists("nonexistent")) def test_db_exists(self): self.assertFalse(self.catalog.database_exists("nonexistent")) self.catalog.create_database(self.db1, self.create_db(), False) self.assertTrue(self.catalog.database_exists(self.db1)) def test_create_table_streaming(self): self.catalog.create_database(self.db1, self.create_db(), False) table = self.create_stream_table() self.catalog.create_table(self.path1, table, False) self.check_catalog_table_equals(table, self.catalog.get_table(self.path1)) def test_create_table_batch(self): self.catalog.create_database(self.db1, self.create_db(), False) # Non - partitioned table table = self.create_table() self.catalog.create_table(self.path1, table, False) table_created = self.catalog.get_table(self.path1) self.check_catalog_table_equals(table, table_created) self.assertEqual(self.test_comment, table_created.get_description()) tables = self.catalog.list_tables(self.db1) self.assertEqual(1, len(tables)) self.assertEqual(self.path1.get_object_name(), tables[0]) self.catalog.drop_table(self.path1, False) # Partitioned table self.table = self.create_partitioned_table() self.catalog.create_table(self.path1, table, False) self.check_catalog_table_equals(table, self.catalog.get_table(self.path1)) tables = self.catalog.list_tables(self.db1) self.assertEqual(1, len(tables)) self.assertEqual(self.path1.get_object_name(), tables[0]) def test_create_table_database_not_exist_exception(self): self.assertFalse(self.catalog.database_exists(self.db1)) with self.assertRaises(DatabaseNotExistException): self.catalog.create_table(self.non_exist_object_path, self.create_table(), False) def test_create_table_table_already_exist_exception(self): self.catalog.create_database(self.db1, self.create_db(), False) self.catalog.create_table(self.path1, self.create_table(), False) with self.assertRaises(TableAlreadyExistException): self.catalog.create_table(self.path1, self.create_table(), False) def test_create_table_table_already_exist_ignored(self): self.catalog.create_database(self.db1, self.create_db(), False) table = self.create_table() self.catalog.create_table(self.path1, table, False) self.check_catalog_table_equals(table, self.catalog.get_table(self.path1)) self.catalog.create_table(self.path1, self.create_another_table(), True) self.check_catalog_table_equals(table, self.catalog.get_table(self.path1)) def test_get_table_table_not_exist_exception(self): self.catalog.create_database(self.db1, self.create_db(), False) with self.assertRaises(TableNotExistException): self.catalog.get_table(self.non_exist_object_path) def test_get_table_table_not_exist_exception_no_db(self): with self.assertRaises(TableNotExistException): self.catalog.get_table(self.non_exist_object_path) def test_drop_table_non_partitioned_table(self): self.catalog.create_database(self.db1, self.create_db(), False) self.catalog.create_table(self.path1, self.create_table(), False) self.assertTrue(self.catalog.table_exists(self.path1)) self.catalog.drop_table(self.path1, False) self.assertFalse(self.catalog.table_exists(self.path1)) def test_drop_table_table_not_exist_exception(self): with self.assertRaises(TableNotExistException): self.catalog.drop_table(self.non_exist_db_path, False) def test_drop_table_table_not_exist_ignored(self): self.catalog.create_database(self.db1, self.create_db(), False) self.catalog.drop_table(self.non_exist_object_path, True) def test_alter_table(self): self.catalog.create_database(self.db1, self.create_db(), False) # Non - partitioned table table = self.create_table() self.catalog.create_table(self.path1, table, False) self.check_catalog_table_equals(table, self.catalog.get_table(self.path1)) new_table = self.create_another_table() self.catalog.alter_table(self.path1, new_table, False) self.assertNotEqual(table, self.catalog.get_table(self.path1)) self.check_catalog_table_equals(new_table, self.catalog.get_table(self.path1)) self.catalog.drop_table(self.path1, False) # Partitioned table table = self.create_partitioned_table() self.catalog.create_table(self.path1, table, False) self.check_catalog_table_equals(table, self.catalog.get_table(self.path1)) new_table = self.create_another_partitioned_table() self.catalog.alter_table(self.path1, new_table, False) self.check_catalog_table_equals(new_table, self.catalog.get_table(self.path1)) # View view = self.create_view() self.catalog.create_table(self.path3, view, False) self.check_catalog_view_equals(view, self.catalog.get_table(self.path3)) new_view = self.create_another_view() self.catalog.alter_table(self.path3, new_view, False) self.assertNotEqual(view, self.catalog.get_table(self.path3)) self.check_catalog_view_equals(new_view, self.catalog.get_table(self.path3)) def test_alter_table_table_not_exist_exception(self): with self.assertRaises(TableNotExistException): self.catalog.alter_table(self.non_exist_db_path, self.create_table(), False) def test_alter_table_table_not_exist_ignored(self): self.catalog.create_database(self.db1, self.create_db(), False) self.catalog.alter_table(self.non_exist_object_path, self.create_table(), True) self.assertFalse(self.catalog.table_exists(self.non_exist_object_path)) def test_rename_table_non_partitioned_table(self): self.catalog.create_database(self.db1, self.create_db(), False) table = self.create_table() self.catalog.create_table(self.path1, table, False) self.check_catalog_table_equals(table, self.catalog.get_table(self.path1)) self.catalog.rename_table(self.path1, self.t2, False) self.check_catalog_table_equals(table, self.catalog.get_table(self.path3)) self.assertFalse(self.catalog.table_exists(self.path1)) def test_rename_table_table_not_exist_exception(self): self.catalog.create_database(self.db1, self.create_db(), False) with self.assertRaises(TableNotExistException): self.catalog.rename_table(self.path1, self.t2, False) def test_rename_table_table_not_exist_exception_ignored(self): self.catalog.create_database(self.db1, self.create_db(), False) self.catalog.rename_table(self.path1, self.t2, True) def test_rename_table_table_already_exist_exception(self): self.catalog.create_database(self.db1, self.create_db(), False) table = self.create_table() self.catalog.create_table(self.path1, table, False) self.catalog.create_table(self.path3, self.create_another_table(), False) with self.assertRaises(TableAlreadyExistException): self.catalog.rename_table(self.path1, self.t2, False) def test_list_tables(self): self.catalog.create_database(self.db1, self.create_db(), False) self.catalog.create_table(self.path1, self.create_table(), False) self.catalog.create_table(self.path3, self.create_table(), False) self.catalog.create_table(self.path4, self.create_view(), False) self.assertEqual(3, len(self.catalog.list_tables(self.db1))) self.assertEqual(1, len(self.catalog.list_views(self.db1))) def test_table_exists(self): self.catalog.create_database(self.db1, self.create_db(), False) self.assertFalse(self.catalog.table_exists(self.path1)) self.catalog.create_table(self.path1, self.create_table(), False) self.assertTrue(self.catalog.table_exists(self.path1)) def test_create_view(self): self.catalog.create_database(self.db1, self.create_db(), False) self.assertFalse(self.catalog.table_exists(self.path1)) view = self.create_view() self.catalog.create_table(self.path1, view, False) self.check_catalog_view_equals(view, self.catalog.get_table(self.path1)) def test_create_view_database_not_exist_exception(self): self.assertFalse(self.catalog.database_exists(self.db1)) with self.assertRaises(DatabaseNotExistException): self.catalog.create_table(self.non_exist_object_path, self.create_view(), False) def test_create_view_table_already_exist_exception(self): self.catalog.create_database(self.db1, self.create_db(), False) self.catalog.create_table(self.path1, self.create_view(), False) with self.assertRaises(TableAlreadyExistException): self.catalog.create_table(self.path1, self.create_view(), False) def test_create_view_table_already_exist_ignored(self): self.catalog.create_database(self.db1, self.create_db(), False) view = self.create_view() self.catalog.create_table(self.path1, view, False) self.check_catalog_view_equals(view, self.catalog.get_table(self.path1)) self.catalog.create_table(self.path1, self.create_another_view(), True) self.check_catalog_view_equals(view, self.catalog.get_table(self.path1)) def test_drop_view(self): self.catalog.create_database(self.db1, self.create_db(), False) self.catalog.create_table(self.path1, self.create_view(), False) self.assertTrue(self.catalog.table_exists(self.path1)) self.catalog.drop_table(self.path1, False) self.assertFalse(self.catalog.table_exists(self.path1)) def test_alter_view(self): self.catalog.create_database(self.db1, self.create_db(), False) view = self.create_view() self.catalog.create_table(self.path1, view, False) self.check_catalog_view_equals(view, self.catalog.get_table(self.path1)) new_view = self.create_another_view() self.catalog.alter_table(self.path1, new_view, False) self.check_catalog_view_equals(new_view, self.catalog.get_table(self.path1)) def test_alter_view_table_not_exist_exception(self): with self.assertRaises(TableNotExistException): self.catalog.alter_table(self.non_exist_db_path, self.create_table(), False) def test_alter_view_table_not_exist_ignored(self): self.catalog.create_database(self.db1, self.create_db(), False) self.catalog.alter_table(self.non_exist_object_path, self.create_view(), True) self.assertFalse(self.catalog.table_exists(self.non_exist_object_path)) def test_list_view(self): self.catalog.create_database(self.db1, self.create_db(), False) self.assertTrue(0 == len(self.catalog.list_tables(self.db1))) self.catalog.create_table(self.path1, self.create_view(), False) self.catalog.create_table(self.path3, self.create_table(), False) self.assertEqual(2, len(self.catalog.list_tables(self.db1))) self.assertEqual( {self.path1.get_object_name(), self.path3.get_object_name()}, set(self.catalog.list_tables(self.db1))) self.assertEqual([self.path1.get_object_name()], self.catalog.list_views(self.db1)) def test_rename_view(self): self.catalog.create_database(self.db1, self.create_db(), False) self.catalog.create_table(self.path1, self.create_view(), False) self.assertTrue(self.catalog.table_exists(self.path1)) self.catalog.rename_table(self.path1, self.t2, False) self.assertFalse(self.catalog.table_exists(self.path1)) self.assertTrue(self.catalog.table_exists(self.path3)) def test_create_function(self): self.catalog.create_database(self.db1, self.create_db(), False) self.assertFalse(self.catalog.function_exists(self.path1)) self.catalog.create_function(self.path1, self.create_function(), False) self.assertTrue(self.catalog.function_exists(self.path1)) def test_create_function_database_not_exist_exception(self): self.assertFalse(self.catalog.database_exists(self.db1)) with self.assertRaises(DatabaseNotExistException): self.catalog.create_function(self.path1, self.create_function(), False) def test_create_functin_function_already_exist_function(self): self.catalog.create_database(self.db1, self.create_db(), False) self.catalog.create_function(self.path1, self.create_function(), False) self.assertTrue(self.catalog.function_exists(self.path1)) # test 'ignoreIfExist' flag self.catalog.create_function(self.path1, self.create_another_function(), True) with self.assertRaises(FunctionAlreadyExistException): self.catalog.create_function(self.path1, self.create_function(), False) def test_alter_function(self): self.catalog.create_database(self.db1, self.create_db(), False) func = self.create_function() self.catalog.create_function(self.path1, func, False) self.check_catalog_function_equals( func, self.catalog.get_function(self.path1)) new_func = self.create_another_function() self.catalog.alter_function(self.path1, new_func, False) actual = self.catalog.get_function(self.path1) self.assertFalse(func.get_class_name() == (actual.get_class_name())) self.check_catalog_function_equals(new_func, actual) def test_alter_function_function_not_exist_exception(self): with self.assertRaises(FunctionNotExistException): self.catalog.alter_function(self.non_exist_object_path, self.create_function(), False) def test_alter_function_function_not_exist_ignored(self): self.catalog.create_database(self.db1, self.create_db(), False) self.catalog.alter_function(self.non_exist_object_path, self.create_function(), True) self.assertFalse( self.catalog.function_exists(self.non_exist_object_path)) def test_list_functions(self): self.catalog.create_database(self.db1, self.create_db(), False) func = self.create_function() self.catalog.create_function(self.path1, func, False) self.assertEqual(self.path1.get_object_name(), self.catalog.list_functions(self.db1)[0]) def test_list_functions_database_not_exist_exception(self): with self.assertRaises(DatabaseNotExistException): self.catalog.list_functions(self.db1) def test_get_function_function_not_exist_exception(self): self.catalog.create_database(self.db1, self.create_db(), False) with self.assertRaises(FunctionNotExistException): self.catalog.get_function(self.non_exist_object_path) def test_get_function_function_not_exist_exception_no_db(self): with self.assertRaises(FunctionNotExistException): self.catalog.get_function(self.non_exist_object_path) def test_drop_function(self): self.catalog.create_database(self.db1, self.create_db(), False) self.catalog.create_function(self.path1, self.create_function(), False) self.assertTrue(self.catalog.function_exists(self.path1)) self.catalog.drop_function(self.path1, False) self.assertFalse(self.catalog.function_exists(self.path1)) def test_drop_function_function_not_exist_exception(self): with self.assertRaises(FunctionNotExistException): self.catalog.drop_function(self.non_exist_db_path, False) def test_drop_function_function_not_exist_ignored(self): self.catalog.create_database(self.db1, self.create_db(), False) self.catalog.drop_function(self.non_exist_object_path, True) self.catalog.drop_database(self.db1, False) def test_create_partition(self): self.catalog.create_database(self.db1, self.create_db(), False) self.catalog.create_table(self.path1, self.create_partitioned_table(), False) self.assertTrue(0 == len(self.catalog.list_partitions(self.path1))) self.catalog.create_partition(self.path1, self.create_partition_spec(), self.create_partition(), False) self.check_catalog_partition_equals( self.create_partition(), self.catalog.get_partition(self.path1, self.create_partition_spec())) self.catalog.create_partition(self.path1, self.create_another_partition_spec(), self.create_partition(), False) self.check_catalog_partition_equals( self.create_partition(), self.catalog.get_partition(self.path1, self.create_another_partition_spec())) def test_create_partition_table_not_exist_exception(self): self.catalog.create_database(self.db1, self.create_db(), False) with self.assertRaises(TableNotExistException): self.catalog.create_partition(self.path1, self.create_partition_spec(), self.create_partition(), False) def test_create_partition_table_not_partitoned_exception(self): self.catalog.create_database(self.db1, self.create_db(), False) self.catalog.create_table(self.path1, self.create_table(), False) with self.assertRaises(TableNotPartitionedException): self.catalog.create_partition(self.path1, self.create_partition_spec(), self.create_partition(), False) def test_create_partition_partition_spec_invalid_exception(self): self.catalog.create_database(self.db1, self.create_db(), False) table = self.create_partitioned_table() self.catalog.create_table(self.path1, table, False) partition_spec = self.create_invalid_partition_spec_subset() with self.assertRaises(PartitionSpecInvalidException): self.catalog.create_partition(self.path1, partition_spec, self.create_partition(), False) def test_create_partition_partition_already_exists_exception(self): self.catalog.create_database(self.db1, self.create_db(), False) self.catalog.create_table(self.path1, self.create_partitioned_table(), False) partition = self.create_partition() self.catalog.create_partition(self.path1, self.create_partition_spec(), partition, False) partition_spec = self.create_partition_spec() with self.assertRaises(PartitionAlreadyExistsException): self.catalog.create_partition(self.path1, partition_spec, self.create_partition(), False) def test_create_partition_partition_already_exists_ignored(self): self.catalog.create_database(self.db1, self.create_db(), False) self.catalog.create_table(self.path1, self.create_partitioned_table(), False) partition_spec = self.create_partition_spec() self.catalog.create_partition(self.path1, partition_spec, self.create_partition(), False) self.catalog.create_partition(self.path1, partition_spec, self.create_partition(), True) def test_drop_partition(self): self.catalog.create_database(self.db1, self.create_db(), False) self.catalog.create_table(self.path1, self.create_partitioned_table(), False) self.catalog.create_partition(self.path1, self.create_partition_spec(), self.create_partition(), False) self.catalog.drop_partition(self.path1, self.create_partition_spec(), False) self.assertEqual([], self.catalog.list_partitions(self.path1)) def test_drop_partition_table_not_exist(self): self.catalog.create_database(self.db1, self.create_db(), False) partition_spec = self.create_partition_spec() with self.assertRaises(PartitionNotExistException): self.catalog.drop_partition(self.path1, partition_spec, False) def test_drop_partition_table_not_partitioned(self): self.catalog.create_database(self.db1, self.create_db(), False) self.catalog.create_table(self.path1, self.create_table(), False) partition_spec = self.create_partition_spec() with self.assertRaises(PartitionNotExistException): self.catalog.drop_partition(self.path1, partition_spec, False) def test_drop_partition_partition_spec_invalid(self): self.catalog.create_database(self.db1, self.create_db(), False) table = self.create_partitioned_table() self.catalog.create_table(self.path1, table, False) partition_spec = self.create_invalid_partition_spec_subset() with self.assertRaises(PartitionNotExistException): self.catalog.drop_partition(self.path1, partition_spec, False) def test_drop_partition_patition_not_exist(self): self.catalog.create_database(self.db1, self.create_db(), False) self.catalog.create_table(self.path1, self.create_partitioned_table(), False) partition_spec = self.create_partition_spec() with self.assertRaises(PartitionNotExistException): self.catalog.drop_partition(self.path1, partition_spec, False) def test_drop_partition_patition_not_exist_ignored(self): self.catalog.create_database(self.db1, self.create_db(), False) self.catalog.create_table(self.path1, self.create_partitioned_table(), False) self.catalog.drop_partition(self.path1, self.create_partition_spec(), True) def test_alter_partition(self): self.catalog.create_database(self.db1, self.create_db(), False) self.catalog.create_table(self.path1, self.create_partitioned_table(), False) self.catalog.create_partition(self.path1, self.create_partition_spec(), self.create_partition(), False) cp = self.catalog.get_partition(self.path1, self.create_partition_spec()) self.check_catalog_partition_equals(self.create_partition(), cp) self.assertIsNone(cp.get_properties().get("k")) gateway = get_gateway() j_partition = gateway.jvm.GenericCatalogPartition( { "is_streaming": "false", "k": "v" }, "Generic batch table") another = CatalogPartition(j_partition) self.catalog.alter_partition(self.path1, self.create_partition_spec(), another, False) cp = self.catalog.get_partition(self.path1, self.create_partition_spec()) self.check_catalog_partition_equals(another, cp) self.assertEqual("v", cp.get_properties().get("k")) def test_alter_partition_table_not_exist(self): self.catalog.create_database(self.db1, self.create_db(), False) partition_spec = self.create_partition_spec() with self.assertRaises(PartitionNotExistException): self.catalog.alter_partition(self.path1, partition_spec, self.create_partition(), False) def test_alter_partition_table_not_partitioned(self): self.catalog.create_database(self.db1, self.create_db(), False) self.catalog.create_table(self.path1, self.create_table(), False) partition_spec = self.create_partition_spec() with self.assertRaises(PartitionNotExistException): self.catalog.alter_partition(self.path1, partition_spec, self.create_partition(), False) def test_alter_partition_partition_spec_invalid(self): self.catalog.create_database(self.db1, self.create_db(), False) table = self.create_partitioned_table() self.catalog.create_table(self.path1, table, False) partition_spec = self.create_invalid_partition_spec_subset() with self.assertRaises(PartitionNotExistException): self.catalog.alter_partition(self.path1, partition_spec, self.create_partition(), False) def test_alter_partition_partition_not_exist(self): self.catalog.create_database(self.db1, self.create_db(), False) self.catalog.create_table(self.path1, self.create_partitioned_table(), False) partition_spec = self.create_partition_spec() with self.assertRaises(PartitionNotExistException): self.catalog.alter_partition(self.path1, partition_spec, self.create_partition(), False) def test_alter_partition_partition_not_exist_ignored(self): self.catalog.create_database(self.db1, self.create_db(), False) self.catalog.create_table(self.path1, self.create_partitioned_table(), False) self.catalog.alter_partition(self.path1, self.create_partition_spec(), self.create_partition(), True) def test_get_partition_table_not_exists(self): partition_spec = self.create_partition_spec() with self.assertRaises(PartitionNotExistException): self.catalog.get_partition(self.path1, partition_spec) def test_get_partition_table_not_partitioned(self): self.catalog.create_database(self.db1, self.create_db(), False) self.catalog.create_table(self.path1, self.create_table(), False) partition_spec = self.create_partition_spec() with self.assertRaises(PartitionNotExistException): self.catalog.get_partition(self.path1, partition_spec) def test_get_partition_partition_spec_invalid_invalid_partition_spec(self): self.catalog.create_database(self.db1, self.create_db(), False) table = self.create_partitioned_table() self.catalog.create_table(self.path1, table, False) partition_spec = self.create_invalid_partition_spec_subset() with self.assertRaises(PartitionNotExistException): self.catalog.get_partition(self.path1, partition_spec) def test_get_partition_partition_spec_invalid_size_not_equal(self): self.catalog.create_database(self.db1, self.create_db(), False) table = self.create_partitioned_table() self.catalog.create_table(self.path1, table, False) partition_spec = self.create_partition_spec_subset() with self.assertRaises(PartitionNotExistException): self.catalog.get_partition(self.path1, partition_spec) def test_get_partition_partition_not_exist(self): self.catalog.create_database(self.db1, self.create_db(), False) self.catalog.create_table(self.path1, self.create_partitioned_table(), False) partition_spec = self.create_partition_spec() with self.assertRaises(PartitionNotExistException): self.catalog.get_partition(self.path1, partition_spec) def test_partition_exists(self): self.catalog.create_database(self.db1, self.create_db(), False) self.catalog.create_table(self.path1, self.create_partitioned_table(), False) self.catalog.create_partition(self.path1, self.create_partition_spec(), self.create_partition(), False) self.assertTrue( self.catalog.partition_exists(self.path1, self.create_partition_spec())) self.assertFalse( self.catalog.partition_exists(self.path2, self.create_partition_spec())) self.assertFalse( self.catalog.partition_exists(ObjectPath.from_string("non.exist"), self.create_partition_spec())) def test_list_partition_partial_spec(self): self.catalog.create_database(self.db1, self.create_db(), False) self.catalog.create_table(self.path1, self.create_partitioned_table(), False) self.catalog.create_partition(self.path1, self.create_partition_spec(), self.create_partition(), False) self.catalog.create_partition(self.path1, self.create_another_partition_spec(), self.create_partition(), False) self.assertEqual( 2, len( self.catalog.list_partitions( self.path1, self.create_partition_spec_subset()))) self.assertEqual( 1, len( self.catalog.list_partitions( self.path1, self.create_another_partition_spec_subset())))