class TestPartitionParams(TestcaseBase): """ Test case of partition interface in parameters""" @pytest.mark.tags(CaseLabel.L0) # @pytest.mark.parametrize("partition_name, description", # [(cf.gen_unique_str(prefix), cf.gen_unique_str("desc_"))]) def test_partition_default(self): """ target: verify create a partition method: 1. create a partition expected: 1. create successfully """ # create collection collection_w = self.init_collection_wrap() # create partition partition_name = cf.gen_unique_str(prefix) description = cf.gen_unique_str("desc_") self.init_partition_wrap( collection_w, partition_name, description=description, check_task=CheckTasks.check_partition_property, check_items={ "name": partition_name, "description": description, "is_empty": True, "num_entities": 0 }) # check that the partition has been created assert collection_w.has_partition(partition_name)[0] @pytest.mark.tags(CaseLabel.L1) # @pytest.mark.xfail(reason="issue #5375") @pytest.mark.parametrize("partition_name", [""]) def test_partition_empty_name(self, partition_name): """ target: verify create a partition with empyt name method: 1. create a partition empty none name expected: 1. raise exception """ # create a collection collection_w = self.init_collection_wrap() # create partition self.partition_wrap.init_partition( collection_w.collection, partition_name, check_task=CheckTasks.err_res, check_items={ ct.err_code: 1, ct.err_msg: "Partition name should not be empty" }) @pytest.mark.tags(CaseLabel.L1) # @pytest.mark.parametrize("partition_name, description", [(cf.gen_unique_str(prefix), "")]) def test_partition_empty_description(self): """ target: verify create a partition with empty description method: 1. create a partition with empty description expected: 1. create successfully """ # create collection collection_w = self.init_collection_wrap() # init partition partition_name = cf.gen_unique_str(prefix) description = "" self.init_partition_wrap( collection_w, partition_name, description=description, check_task=CheckTasks.check_partition_property, check_items={ "name": partition_name, "description": description, "is_empty": True, "num_entities": 0 }) # check that the partition has been created assert collection_w.has_partition(partition_name)[0] @pytest.mark.tags(CaseLabel.L1) # @pytest.mark.parametrize("partition_name, description", # [(cf.gen_str_by_length(255), cf.gen_str_by_length(2048))]) def test_partition_max_description_length(self): """ target: verify create a partition with 255 length name and 1024 length description method: 1. create a partition with 255 length name and 1024 length description expected: 1. create successfully """ # create collection collection_w = self.init_collection_wrap() # init partition partition_name = cf.gen_str_by_length(255) description = cf.gen_str_by_length(2048) self.init_partition_wrap( collection_w, partition_name, description=description, check_task=CheckTasks.check_partition_property, check_items={ "name": partition_name, "description": description, "is_empty": True }) @pytest.mark.tags(CaseLabel.L1) # @pytest.mark.parametrize("collection_name, partition_name, description", # [(cf.gen_unique_str(), cf.gen_unique_str(prefix), cf.gen_unique_str())]) def test_partition_dup_name(self): """ target: verify create partitions with duplicate name method: 1. create partitions with duplicate name expected: 1. create successfully 2. the same partition returned with diff object id """ # create a collection collection_w = self.init_collection_wrap() # create two partitions partition_name = cf.gen_unique_str(prefix) description = cf.gen_unique_str() partition_w1 = self.init_partition_wrap(collection_w, partition_name, description) partition_w2 = self.init_partition_wrap(collection_w, partition_name, description) # public check func to be extracted assert id(partition_w1.partition) != id(partition_w2.partition) assert partition_w1.name == partition_w2.name assert partition_w1.description == partition_w2.description @pytest.mark.tags(CaseLabel.L1) @pytest.mark.parametrize("description", ct.get_invalid_strs) # @pytest.mark.parametrize("partition_name", [cf.gen_unique_str(prefix)]) def test_partition_special_chars_description(self, description): """ target: verify create a partition with special characters in description method: 1. create a partition with special characters in description expected: 1. create successfully """ # create collection collection_w = self.init_collection_wrap() # create partition partition_name = cf.gen_unique_str(prefix) self.init_partition_wrap( collection_w, partition_name, description=description, check_task=CheckTasks.check_partition_property, check_items={ "name": partition_name, "description": description, "is_empty": True, "num_entities": 0 }) assert collection_w.has_partition(partition_name)[0] @pytest.mark.tags(CaseLabel.L0) def test_partition_default_name(self): """ target: verify create a partition with default name method: 1. get the _default partition 2. create a partition with _default name expected: 1. the same partition returned """ # create collection collection_w = self.init_collection_wrap() # check that the default partition exists assert collection_w.has_partition(ct.default_partition_name)[0] # check that can get the _default partition collection, _ = collection_w.partition(ct.default_partition_name) # check that init the _default partition object partition_w = self.init_partition_wrap(collection_w, ct.default_partition_name) assert collection.name == partition_w.name @pytest.mark.tags(CaseLabel.L1) # @pytest.mark.parametrize("partition_name", [cf.gen_str_by_length(256)]) def test_partition_maxlength_name(self): """ target: verify create a partition with maxlength(256) name method: 1. create a partition with max length names expected: 1. raise exception """ # create collection collection_w = self.init_collection_wrap() # create partition partition_name = cf.gen_str_by_length(256) self.partition_wrap.init_partition(collection_w.collection, partition_name, check_task=CheckTasks.err_res, check_items={ ct.err_code: 1, 'err_msg': "is illegal" }) @pytest.mark.tags(CaseLabel.L1) @pytest.mark.parametrize("partition_name", ct.get_invalid_strs) def test_partition_invalid_name(self, partition_name): """ target: verify create a partition with invalid name method: 1. create a partition with invalid names expected: 1. raise exception """ # create collection collection_w = self.init_collection_wrap() # create partition self.partition_wrap.init_partition(collection_w.collection, partition_name, check_task=CheckTasks.err_res, check_items={ ct.err_code: 1, 'err_msg': "is illegal" }) # TODO: need an error code issue #5144 and assert independently @pytest.mark.tags(CaseLabel.L1) # @pytest.mark.parametrize("partition_name", [cf.gen_unique_str(prefix)]) def test_partition_none_collection(self): """ target: verify create a partition with none collection method: 1. create a partition with none collection expected: 1. raise exception """ # create partition with collection is None partition_name = cf.gen_unique_str(prefix) self.partition_wrap.init_partition(collection=None, name=partition_name, check_task=CheckTasks.err_res, check_items={ ct.err_code: 1, ct.err_msg: "must be pymilvus.Collection" }) @pytest.mark.tags(CaseLabel.L1) # @pytest.mark.parametrize("partition_name", [cf.gen_unique_str(prefix)]) def test_partition_drop(self): """ target: verify drop a partition in one collection method: 1. create a partition in one collection 2. drop the partition expected: 1. drop successfully """ # create collection collection_w = self.init_collection_wrap() # create partition partition_name = cf.gen_unique_str(prefix) partition_w = self.init_partition_wrap(collection_w, partition_name) # check that the partition exists assert collection_w.has_partition(partition_name)[0] # drop partition partition_w.drop() # check that the partition not exists assert not collection_w.has_partition(partition_name)[0] @pytest.mark.tags(CaseLabel.L1) # @pytest.mark.parametrize("search_vectors", [cf.gen_vectors(1, ct.default_dim)]) def test_partition_release(self): """ target: verify release partition method: 1. create a collection and several partitions 2. insert data into each partition 3. flush and load the partitions 4. release partition1 5. release partition1 twice expected: 1. the released partition is released 2. the other partition is not released """ # create collection collection_w = self.init_collection_wrap() # create two partitions partition_w1 = self.init_partition_wrap(collection_w) partition_w2 = self.init_partition_wrap(collection_w) # insert data to two partition partition_w1.insert(cf.gen_default_list_data()) partition_w2.insert(cf.gen_default_list_data()) # load two partitions partition_w1.load() partition_w2.load() # search two partitions search_vectors = cf.gen_vectors(1, ct.default_dim) res1, _ = partition_w1.search( data=search_vectors, anns_field=ct.default_float_vec_field_name, params={"nprobe": 32}, limit=1) res2, _ = partition_w2.search( data=search_vectors, anns_field=ct.default_float_vec_field_name, params={"nprobe": 32}, limit=1) assert len(res1) == 1 and len(res2) == 1 # release the first partition partition_w1.release() # check result res1, _ = partition_w1.search( data=search_vectors, anns_field=ct.default_float_vec_field_name, params={"nprobe": 32}, limit=1, check_task=ct.CheckTasks.err_res, check_items={ ct.err_code: 1, ct.err_msg: "partitions have been released" }) res2, _ = partition_w2.search( data=search_vectors, anns_field=ct.default_float_vec_field_name, params={"nprobe": 32}, limit=1) assert len(res2) == 1 @pytest.mark.tags(CaseLabel.L1) # @pytest.mark.parametrize("partition_name", [cf.gen_unique_str(prefix)]) @pytest.mark.parametrize("data", [ cf.gen_default_dataframe_data(10), cf.gen_default_list_data(10), cf.gen_default_tuple_data(10) ]) def test_partition_insert(self, data): """ target: verify insert multi entities by dataFrame method: 1. create a collection and a partition 2. partition.insert(data) 3. insert data again expected: 1. insert data successfully """ nums = 10 # create collection collection_w = self.init_collection_wrap() # create partition partition_name = cf.gen_unique_str(prefix) partition_w = self.init_partition_wrap( collection_w, partition_name, check_task=CheckTasks.check_partition_property, check_items={ "name": partition_name, "is_empty": True, "num_entities": 0 }) # insert data partition_w.insert(data) # self._connect().flush([collection_w.name]) # don't need flush for issue #5737 assert not partition_w.is_empty assert partition_w.num_entities == nums # insert data partition_w.insert(data) # self._connect().flush([collection_w.name]) assert not partition_w.is_empty assert partition_w.num_entities == (nums + nums)
class TestPartitionParams(TestcaseBase): """ Test case of partition interface in parameters""" @pytest.mark.tags(CaseLabel.L0) def test_partition_default(self): """ target: verify create a partition method: create a partition expected: create successfully """ # create collection collection_w = self.init_collection_wrap() # create partition partition_name = cf.gen_unique_str(prefix) description = cf.gen_unique_str("desc_") self.init_partition_wrap( collection_w, partition_name, description=description, check_task=CheckTasks.check_partition_property, check_items={ "name": partition_name, "description": description, "is_empty": True, "num_entities": 0 }) # check that the partition has been created assert collection_w.has_partition(partition_name)[0] @pytest.mark.tags(CaseLabel.L2) @pytest.mark.parametrize("partition_name", [""]) def test_partition_empty_name(self, partition_name): """ target: verify create a partition with empty name method: create a partition with empty name expected: raise exception """ # create a collection collection_w = self.init_collection_wrap() # create partition self.partition_wrap.init_partition( collection_w.collection, partition_name, check_task=CheckTasks.err_res, check_items={ ct.err_code: 1, ct.err_msg: "Partition name should not be empty" }) @pytest.mark.tags(CaseLabel.L2) def test_partition_empty_description(self): """ target: verify create a partition with empty description method: create a partition with empty description expected: create successfully """ # create collection collection_w = self.init_collection_wrap() # init partition partition_name = cf.gen_unique_str(prefix) description = "" self.init_partition_wrap( collection_w, partition_name, description=description, check_task=CheckTasks.check_partition_property, check_items={ "name": partition_name, "description": description, "is_empty": True, "num_entities": 0 }) # check that the partition has been created assert collection_w.has_partition(partition_name)[0] @pytest.mark.tags(CaseLabel.L2) def test_partition_max_description_length(self): """ target: verify create a partition with 255 length name and 1024 length description method: create a partition with 255 length name and 1024 length description expected: create successfully """ # create collection collection_w = self.init_collection_wrap() # init partition partition_name = cf.gen_str_by_length(255) description = cf.gen_str_by_length(2048) self.init_partition_wrap( collection_w, partition_name, description=description, check_task=CheckTasks.check_partition_property, check_items={ "name": partition_name, "description": description, "is_empty": True }) @pytest.mark.tags(CaseLabel.L1) def test_partition_dup_name(self): """ target: verify create partitions with duplicate names method: create partitions with duplicate names expected: 1. create successfully 2. the same partition returned with diff object ids """ # create a collection collection_w = self.init_collection_wrap() # create two partitions partition_name = cf.gen_unique_str(prefix) description = cf.gen_unique_str() partition_w1 = self.init_partition_wrap(collection_w, partition_name, description) partition_w2 = self.init_partition_wrap(collection_w, partition_name, description) # public check func to be extracted assert id(partition_w1.partition) != id(partition_w2.partition) assert partition_w1.name == partition_w2.name assert partition_w1.description == partition_w2.description @pytest.mark.tags(CaseLabel.L2) @pytest.mark.parametrize("description", ct.get_invalid_strs) def test_partition_special_chars_description(self, description): """ target: verify create a partition with special characters in description method: create a partition with special characters in description expected: create successfully """ # create collection collection_w = self.init_collection_wrap() # create partition partition_name = cf.gen_unique_str(prefix) self.init_partition_wrap( collection_w, partition_name, description=description, check_task=CheckTasks.check_partition_property, check_items={ "name": partition_name, "description": description, "is_empty": True, "num_entities": 0 }) assert collection_w.has_partition(partition_name)[0] @pytest.mark.tags(CaseLabel.L0) def test_partition_default_name(self): """ target: verify create a partition with default name method: 1. get the _default partition 2. create a partition with _default name expected: the same partition returned """ # create collection collection_w = self.init_collection_wrap() # check that the default partition exists assert collection_w.has_partition(ct.default_partition_name)[0] # check that can get the _default partition collection, _ = collection_w.partition(ct.default_partition_name) # check that init the _default partition object partition_w = self.init_partition_wrap(collection_w, ct.default_partition_name) assert collection.name == partition_w.name @pytest.mark.tags(CaseLabel.L2) def test_partition_max_length_name(self): """ target: verify create a partition with max length(256) name method: create a partition with max length name expected: raise exception """ # create collection collection_w = self.init_collection_wrap() # create partition partition_name = cf.gen_str_by_length(256) self.partition_wrap.init_partition(collection_w.collection, partition_name, check_task=CheckTasks.err_res, check_items={ ct.err_code: 1, 'err_msg': "is illegal" }) @pytest.mark.tags(CaseLabel.L2) @pytest.mark.parametrize("partition_name", ct.get_invalid_strs) def test_partition_invalid_name(self, partition_name): """ target: verify create a partition with invalid name method: create a partition with invalid names expected: raise exception """ # create collection collection_w = self.init_collection_wrap() # create partition self.partition_wrap.init_partition(collection_w.collection, partition_name, check_task=CheckTasks.err_res, check_items={ ct.err_code: 1, 'err_msg': "is illegal" }) # TODO: need an error code issue #5144 and assert independently @pytest.mark.tags(CaseLabel.L2) def test_partition_none_collection(self): """ target: verify create a partition with none collection method: create a partition with none collection expected: raise exception """ # create partition with collection is None partition_name = cf.gen_unique_str(prefix) self.partition_wrap.init_partition(collection=None, name=partition_name, check_task=CheckTasks.err_res, check_items={ ct.err_code: 1, ct.err_msg: "must be pymilvus.Collection" }) @pytest.mark.tags(CaseLabel.L1) def test_partition_drop(self): """ target: verify drop a partition in one collection method: 1. create a partition in one collection 2. drop the partition expected: drop successfully """ # create collection collection_w = self.init_collection_wrap() # create partition partition_name = cf.gen_unique_str(prefix) partition_w = self.init_partition_wrap(collection_w, partition_name) # check that the partition exists assert collection_w.has_partition(partition_name)[0] # drop partition partition_w.drop() # check that the partition not exists assert not collection_w.has_partition(partition_name)[0] @pytest.mark.tags(CaseLabel.L2) def test_load_partiton_respectively(self): """ target: test release the partition after load partition method: load partition1 and load another partition expected: raise exception """ self._connect() collection_w = self.init_collection_wrap() partition_w1 = self.init_partition_wrap(collection_w) partition_w2 = self.init_partition_wrap(collection_w) partition_w1.insert(cf.gen_default_list_data()) partition_w2.insert(cf.gen_default_list_data()) partition_w1.load() error = { ct.err_code: 1, ct.err_msg: f'load the partition after load collection is not supported' } partition_w2.load(check_task=CheckTasks.err_res, check_items=error) @pytest.mark.tags(CaseLabel.L2) def test_load_partitions_after_release(self): """ target: test release the partition after load partition method: load partitions and release partitions expected: no exception """ self._connect() collection_w = self.init_collection_wrap() partition_w1 = self.init_partition_wrap(collection_w, name="partition_w1") partition_w2 = self.init_partition_wrap(collection_w, name="partition_w2") partition_w1.insert(cf.gen_default_list_data()) partition_w2.insert(cf.gen_default_list_data()) partition_names = ["partition_w1", "partition_w2"] collection_w.load(partition_names) collection_w.release(partition_names) @pytest.mark.tags(CaseLabel.L2) def test_load_partition_after_load_partition(self): """ target: test release the partition after load partition method: load partition1 and release the partition1 load partition2 expected: no exception """ self._connect() collection_w = self.init_collection_wrap() partition_w1 = self.init_partition_wrap(collection_w) partition_w2 = self.init_partition_wrap(collection_w) partition_w1.insert(cf.gen_default_list_data()) partition_w2.insert(cf.gen_default_list_data()) partition_w1.load() partition_w1.release() partition_w2.load() @pytest.fixture(scope="function", params=ct.get_invalid_strs) def get_non_number_replicas(self, request): if request.param == 1: pytest.skip("1 is valid replica number") if request.param is None: pytest.skip("None is valid replica number") yield request.param @pytest.mark.tags(CaseLabel.L2) def test_load_partition_replica_non_number(self, get_non_number_replicas): """ target: test load partition with non-number replicas method: load with non-number replicas expected: raise exceptions """ # create, insert self._connect() collection_w = self.init_collection_wrap() partition_w = self.init_partition_wrap(collection_w) partition_w.insert(cf.gen_default_list_data(nb=100)) # load with non-number replicas error = {ct.err_code: 0, ct.err_msg: f"but expected one of: int, long"} partition_w.load(replica_number=get_non_number_replicas, check_task=CheckTasks.err_res, check_items=error) @pytest.mark.tags(CaseLabel.L2) @pytest.mark.parametrize("replicas", [0, -1, None]) def test_load_replica_invalid_number(self, replicas): """ target: test load partition with invalid replica number method: load with invalid replica number expected: raise exception """ # create, insert self._connect() collection_w = self.init_collection_wrap() partition_w = self.init_partition_wrap(collection_w) partition_w.insert(cf.gen_default_list_data()) assert partition_w.num_entities == ct.default_nb partition_w.load(replica_number=replicas) p_replicas = partition_w.get_replicas()[0] assert len(p_replicas.groups) == 1 query_res, _ = partition_w.query( expr=f"{ct.default_int64_field_name} in [0]") assert len(query_res) == 1 @pytest.mark.tags(CaseLabel.L2) def test_load_replica_greater_than_querynodes(self): """ target: test load with replicas that greater than querynodes method: load with 3 replicas (2 querynode) expected: Verify load successfully and 1 available replica """ # create, insert self._connect() collection_w = self.init_collection_wrap() partition_w = self.init_partition_wrap(collection_w) partition_w.insert(cf.gen_default_list_data()) assert partition_w.num_entities == ct.default_nb # load with 2 replicas error = { ct.err_code: 1, ct.err_msg: f"no enough nodes to create replicas" } partition_w.load(replica_number=3, check_task=CheckTasks.err_res, check_items=error) @pytest.mark.tags(CaseLabel.ClusterOnly) def test_load_replica_change(self): """ target: test load replica change method: 1.load with replica 1 2.load with a new replica number 3.release partition 4.load with a new replica expected: The second time successfully loaded with a new replica number """ # create, insert self._connect() collection_w = self.init_collection_wrap() partition_w = self.init_partition_wrap(collection_w) partition_w.insert(cf.gen_default_list_data()) assert partition_w.num_entities == ct.default_nb partition_w.load(replica_number=1) collection_w.query(expr=f"{ct.default_int64_field_name} in [0]", check_task=CheckTasks.check_query_results, check_items={'exp_res': [{ 'int64': 0 }]}) error = { ct.err_code: 5, ct.err_msg: f"Should release first then reload with the new number of replicas" } partition_w.load(replica_number=2, check_task=CheckTasks.err_res, check_items=error) partition_w.release() partition_w.load(replica_number=2) collection_w.query(expr=f"{ct.default_int64_field_name} in [0]", check_task=CheckTasks.check_query_results, check_items={'exp_res': [{ 'int64': 0 }]}) two_replicas, _ = collection_w.get_replicas() assert len(two_replicas.groups) == 2 # verify loaded segments included 2 replicas and twice num entities seg_info, _ = self.utility_wrap.get_query_segment_info( collection_w.name) seg_ids = list(map(lambda seg: seg.segmentID, seg_info)) num_entities = list(map(lambda seg: seg.num_rows, seg_info)) assert reduce(lambda x, y: x ^ y, seg_ids) == 0 assert reduce(lambda x, y: x + y, num_entities) == ct.default_nb * 2 @pytest.mark.tags(CaseLabel.ClusterOnly) def test_partition_replicas_change_cross_partitions(self): """ target: test load with different replicas between partitions method: 1.Create two partitions and insert data 2.Load two partitions with different replicas expected: Raise an exception """ # Create two partitions and insert data collection_w = self.init_collection_wrap() partition_w1 = self.init_partition_wrap(collection_w) partition_w2 = self.init_partition_wrap(collection_w) partition_w1.insert(cf.gen_default_dataframe_data()) partition_w2.insert(cf.gen_default_dataframe_data(start=ct.default_nb)) assert collection_w.num_entities == ct.default_nb * 2 # load with different replicas partition_w1.load(replica_number=1) partition_w1.release() partition_w2.load(replica_number=2) # verify different have same replicas replicas_1, _ = partition_w1.get_replicas() replicas_2, _ = partition_w2.get_replicas() group1_ids = list(map(lambda g: g.id, replicas_1.groups)) group2_ids = list(map(lambda g: g.id, replicas_1.groups)) assert group1_ids.sort() == group2_ids.sort() # verify loaded segments included 2 replicas and 1 partition seg_info, _ = self.utility_wrap.get_query_segment_info( collection_w.name) seg_ids = list(map(lambda seg: seg.segmentID, seg_info)) num_entities = list(map(lambda seg: seg.num_rows, seg_info)) assert reduce(lambda x, y: x ^ y, seg_ids) == 0 assert reduce(lambda x, y: x + y, num_entities) == ct.default_nb * 2 @pytest.mark.tags(CaseLabel.L1) def test_partition_release(self): """ target: verify release partition method: 1. create a collection and two partitions 2. insert data into each partition 3. flush and load the partition1 4. release partition1 5. release partition2 expected: 1. the 1st partition is released 2. the 2nd partition is released """ # create collection collection_w = self.init_collection_wrap() # create two partitions partition_w1 = self.init_partition_wrap(collection_w) partition_w2 = self.init_partition_wrap(collection_w) # insert data to two partition partition_w1.insert(cf.gen_default_list_data()) partition_w2.insert(cf.gen_default_list_data()) # load two partitions partition_w1.load() # search partition1 search_vectors = cf.gen_vectors(1, ct.default_dim) res1, _ = partition_w1.search( data=search_vectors, anns_field=ct.default_float_vec_field_name, params={"nprobe": 32}, limit=1) assert len(res1) == 1 # release the first partition partition_w1.release() partition_w2.release() # check result res1, _ = partition_w1.search( data=search_vectors, anns_field=ct.default_float_vec_field_name, params={"nprobe": 32}, limit=1, check_task=ct.CheckTasks.err_res, check_items={ ct.err_code: 1, ct.err_msg: "partitions have been released" }) @pytest.mark.tags(CaseLabel.L1) @pytest.mark.parametrize("data", [ cf.gen_default_dataframe_data(10), cf.gen_default_list_data(10), cf.gen_default_tuple_data(10) ]) def test_partition_insert(self, data): """ target: verify insert entities multiple times method: 1. create a collection and a partition 2. partition.insert(data) 3. insert data again expected: insert data successfully """ nums = 10 # create collection collection_w = self.init_collection_wrap() # create partition partition_name = cf.gen_unique_str(prefix) partition_w = self.init_partition_wrap( collection_w, partition_name, check_task=CheckTasks.check_partition_property, check_items={ "name": partition_name, "is_empty": True, "num_entities": 0 }) # insert data partition_w.insert(data) # self._connect().flush([collection_w.name]) # don't need flush for issue #5737 assert not partition_w.is_empty assert partition_w.num_entities == nums # insert data partition_w.insert(data) # self._connect().flush([collection_w.name]) assert not partition_w.is_empty assert partition_w.num_entities == (nums + nums)