def test_collection_by_DataFrame(self):
from pymilvus import Collection
from pymilvus import FieldSchema, CollectionSchema
from pymilvus import DataType
coll_name = gen_collection_name()
fields = [
FieldSchema("int64", DataType.INT64),
FieldSchema("float", DataType.FLOAT),
FieldSchema("float_vector", DataType.FLOAT_VECTOR, dim=128)
]
prefix = "pymilvus.client.grpc_handler.GrpcHandler"
collection_schema = CollectionSchema(fields, primary_field="int64")
with mock.patch(f"{prefix}.__init__", return_value=None):
with mock.patch(f"{prefix}._wait_for_channel_ready",
return_value=None):
connections.connect()
with mock.patch(f"{prefix}.create_collection", return_value=None):
with mock.patch(f"{prefix}.has_collection", return_value=False):
collection = Collection(name=coll_name,
schema=collection_schema)
with mock.patch(f"{prefix}.create_collection", return_value=None):
with mock.patch(f"{prefix}.has_collection", return_value=True):
with mock.patch(f"{prefix}.describe_collection",
return_value=collection_schema.to_dict()):
collection = Collection(name=coll_name)
with mock.patch(f"{prefix}.drop_collection", return_value=None):
with mock.patch(f"{prefix}.describe_index", return_value=None):
collection.drop()
def test_partition():
connections.connect(alias="default")
print("create collection")
collection = Collection(name=gen_unique_str(), schema=gen_default_fields())
print("create partition")
partition = Partition(collection, name=gen_unique_str())
print(list_collections())
assert has_partition(collection.name, partition.name) is True
data = gen_data(default_nb)
print("insert data to partition")
partition.insert(data)
assert partition.is_empty is False
assert partition.num_entities == default_nb
print("load partition")
partition.load()
topK = 5
search_params = {"metric_type": "L2", "params": {"nprobe": 10}}
print("search partition")
res = partition.search(data[2][-2:], "float_vector", search_params, topK, "count > 100")
for hits in res:
for hit in hits:
print(hit)
print("release partition")
partition.release()
print("drop partition")
partition.drop()
print("drop collection")
collection.drop()
def test_collection_by_DataFrame(self):
from pymilvus import Collection, connections
from pymilvus import FieldSchema, CollectionSchema
from pymilvus import DataType
coll_name = gen_collection_name()
fields = [
FieldSchema("int64", DataType.INT64),
FieldSchema("float", DataType.FLOAT),
FieldSchema("float_vector", DataType.FLOAT_VECTOR, dim=128)
]
collection_schema = CollectionSchema(fields, primary_field="int64")
with mock.patch("pymilvus.Milvus.__init__", return_value=None):
connections.connect()
with mock.patch("pymilvus.Milvus.create_collection",
return_value=None):
with mock.patch("pymilvus.Milvus.has_collection",
return_value=False):
collection = Collection(name=coll_name,
schema=collection_schema)
with mock.patch("pymilvus.Milvus.create_collection",
return_value=None):
with mock.patch("pymilvus.Milvus.has_collection",
return_value=True):
with mock.patch("pymilvus.Milvus.describe_collection",
return_value=collection_schema.to_dict()):
collection = Collection(name=coll_name)
with mock.patch("pymilvus.Milvus.drop_collection", return_value=None):
with mock.patch("pymilvus.Milvus.describe_index",
return_value=None):
collection.drop()
def connection(self, host, port):
connections.add_connection(default={"host": host, "port": port})
connections.connect(alias='default')
if connections.has_connection("default") is False:
raise Exception("no connections")
self.host = host
self.port = port
def test_expand_index_node(self):
"""
target: test expand indexNode from 1 to 2
method: 1.deploy two indexNode
2.create index with two indexNode
3.expand indexNode from 1 to 2
4.create index with one indexNode
expected: The cost of one indexNode is about twice that of two indexNodes
"""
release_name = "scale-index"
milvusOp, host, port = scale_common.deploy_default_milvus(release_name)
# connect
connections.add_connection(default={"host": host, "port": port})
connections.connect(alias='default')
data = cf.gen_default_dataframe_data(nb)
# create
c_name = "index_scale_one"
collection_w = ApiCollectionWrapper()
# collection_w.init_collection(name=c_name)
collection_w.init_collection(name=c_name,
schema=cf.gen_default_collection_schema())
# insert
loop = 10
for i in range(loop):
collection_w.insert(data)
assert collection_w.num_entities == nb * loop
# create index on collection one and two
start = datetime.datetime.now()
collection_w.create_index(ct.default_float_vec_field_name,
default_index_params)
assert collection_w.has_index()[0]
t0 = datetime.datetime.now() - start
log.debug(f't0: {t0}')
collection_w.drop_index()
assert not collection_w.has_index()[0]
# expand indexNode from 1 to 2
milvusOp.upgrade(release_name,
{'spec.components.indexNode.replicas': 2},
constants.NAMESPACE)
milvusOp.wait_for_healthy(release_name, constants.NAMESPACE)
start = datetime.datetime.now()
collection_w.create_index(ct.default_float_vec_field_name,
default_index_params)
assert collection_w.has_index()[0]
t1 = datetime.datetime.now() - start
log.debug(f't1: {t1}')
assert round(t0 / t1) == 2
def connection(self, host, port):
connections.add_connection(default={"host": host, "port": port})
connections.connect(alias='default')
if connections.has_connection("default") is False:
raise Exception("no connections")
self.host = host
self.port = port
self.instance_name = get_milvus_instance_name(
constants.CHAOS_NAMESPACE, host)
def task_5(data_size, host):
"""
task_5_:
before upgrade: create collection and insert data without flush
after upgrade: get collection, load with multi replicas, search, insert data with flush, load with multi replicas and search
"""
prefix = "task_5_"
connections.connect(host=host, port=19530, timeout=60)
get_collections(prefix)
create_collections_and_insert_data(prefix, flush=False, count=data_size)
def test_shrink_index_node(self):
"""
target: test shrink indexNode from 2 to 1
method: 1.deploy two indexNode
2.create index with two indexNode
3.shrink indexNode from 2 to 1
4.create index with 1 indexNode
expected: The cost of one indexNode is about twice that of two indexNodes
"""
release_name = "scale-index"
env = HelmEnv(release_name=release_name, indexNode=2)
host = env.helm_install_cluster_milvus()
# connect
connections.add_connection(default={"host": host, "port": 19530})
connections.connect(alias='default')
data = cf.gen_default_dataframe_data(nb)
# create
c_name = "index_scale_one"
collection_w = ApiCollectionWrapper()
# collection_w.init_collection(name=c_name)
collection_w.init_collection(name=c_name,
schema=cf.gen_default_collection_schema())
# insert
loop = 10
for i in range(loop):
collection_w.insert(data)
assert collection_w.num_entities == nb * loop
# create index on collection one and two
start = datetime.datetime.now()
collection_w.create_index(ct.default_float_vec_field_name,
default_index_params)
assert collection_w.has_index()[0]
t0 = datetime.datetime.now() - start
log.debug(f'two indexNodes: {t0}')
collection_w.drop_index()
assert not collection_w.has_index()[0]
# expand indexNode from 2 to 1
env.helm_upgrade_cluster_milvus(indexNode=1)
start = datetime.datetime.now()
collection_w.create_index(ct.default_float_vec_field_name,
default_index_params)
assert collection_w.has_index()[0]
t1 = datetime.datetime.now() - start
log.debug(f'one indexNode: {t1}')
log.debug(t1 / t0)
assert round(t1 / t0) == 2
def task_1(data_size, host):
"""
task_1:
before upgrade: create collection and insert data with flush, load and search
after upgrade: get collection, load, search, insert data with flush, create index, load, and search
"""
prefix = "task_1_"
connections.connect(host=host, port=19530, timeout=60)
get_collections(prefix)
create_collections_and_insert_data(prefix, data_size)
load_and_search(prefix)
def task_1():
"""
task_1:
before reinstall: create collection and insert data, load and search
after reinstall: get collection, load, search, create index, load, and search
"""
prefix = "task_1_"
connections.connect(host="127.0.0.1", port=19530, timeout=60)
get_collections(prefix)
load_and_search(prefix)
create_collections_and_insert_data(prefix)
def task_4(data_size, host):
"""
task_4_:
before upgrade: create collection, insert data, flush, and create index
after upgrade: get collection, load with multi replicas, search, insert data, load with multi replicas and search
"""
prefix = "task_4_"
connections.connect(host=host, port=19530, timeout=60)
get_collections(prefix)
create_collections_and_insert_data(prefix, flush=True, count=data_size)
create_index(prefix)
def task_3(data_size, host):
"""
task_3:
before upgrade: create collection, insert data, flush, create index, load with one replicas and search
after upgrade: get collection, load, search, insert data, create index, release, load with multi replicas, and search
"""
prefix = "task_3_"
connections.connect(host=host, port=19530, timeout=60)
get_collections(prefix)
create_collections_and_insert_data(prefix, data_size)
create_index(prefix)
load_and_search(prefix)
def task_5(data_size, host):
"""
task_5_:
before upgrade: create collection and insert data without flush
after upgrade: get collection, load with multi replicas, search, insert data with flush, load with multi replicas and search
"""
prefix = "task_5_"
connections.connect(host=host, port=19530, timeout=60)
col_list = get_collections(prefix, check=True)
assert len(col_list) == len(all_index_types)
load_and_search(prefix, replicas=NUM_REPLICAS)
create_collections_and_insert_data(prefix, flush=True, count=data_size)
load_and_search(prefix, replicas=NUM_REPLICAS)
def task_2(data_zise, host):
"""
task_2:
before reinstall: create collection, insert data and create index, load and search
after reinstall: get collection, load, search, insert data, create index, load, and search
"""
prefix = "task_2_"
connections.connect(host=host, port=19530, timeout=60)
get_collections(prefix)
load_and_search(prefix)
create_collections_and_insert_data(prefix, data_zise)
create_index(prefix)
load_and_search(prefix)
def task_2(data_size, host):
"""
task_2:
before upgrade: create collection, insert data and create index, load , search, and insert data without flush
after upgrade: get collection, load, search, insert data, create index, load, and search
"""
prefix = "task_2_"
connections.connect(host=host, port=19530, timeout=60)
get_collections(prefix)
create_collections_and_insert_data(prefix, data_size)
create_index(prefix)
load_and_search(prefix)
create_collections_and_insert_data(prefix, flush=False, count=data_size)
def task_2():
"""
task_2:
before upgrade: create collection, insert data and create index, load and search
after upgrade: get collection, load, search, insert data, create index, load, and search
"""
prefix = "task_2_"
connections.connect(host="127.0.0.1", port=19530, timeout=60)
get_collections(prefix)
load_and_search(prefix)
create_collections_and_insert_data(prefix)
create_index(prefix)
load_and_search(prefix)
def task_1(data_size, host):
"""
task_1:
before upgrade: create collection and insert data with flush, load and search
after upgrade: get collection, load, search, insert data with flush, create index, load, and search
"""
prefix = "task_1_"
connections.connect(host=host, port=19530, timeout=60)
col_list = get_collections(prefix, check=True)
assert len(col_list) == len(all_index_types)
load_and_search(prefix)
create_collections_and_insert_data(prefix, data_size)
create_index(prefix)
load_and_search(prefix)
def task_3(data_size, host):
"""
task_3:
before upgrade: create collection, insert data, flush, create index, load with one replicas and search
after upgrade: get collection, load, search, insert data, create index, release, load with multi replicas, and search
"""
prefix = "task_3_"
connections.connect(host=host, port=19530, timeout=60)
col_list = get_collections(prefix, check=True)
assert len(col_list) == len(all_index_types)
load_and_search(prefix)
create_collections_and_insert_data(prefix, count=data_size)
create_index(prefix)
load_and_search(prefix, replicas=NUM_REPLICAS)
def connection(self, host, port):
connections.add_connection(default={"host": host, "port": port})
conn = connections.connect(alias='default')
if conn is None:
raise Exception("no connections")
self.host = host
self.port = port
return conn
def test_expand_query_node(self):
release_name = "scale-query"
env = HelmEnv(release_name=release_name)
host = env.helm_install_cluster_milvus()
# connect
connections.add_connection(default={"host": host, "port": 19530})
connections.connect(alias='default')
# create
c_name = "query_scale_one"
collection_w = ApiCollectionWrapper()
collection_w.init_collection(name=c_name,
schema=cf.gen_default_collection_schema())
# insert
data = cf.gen_default_list_data(ct.default_nb)
mutation_res, _ = collection_w.insert(data)
assert mutation_res.insert_count == ct.default_nb
# # create index
# collection_w.create_index(ct.default_float_vec_field_name, default_index_params)
# assert collection_w.has_index()[0]
# assert collection_w.index()[0] == Index(collection_w.collection, ct.default_float_vec_field_name,
# default_index_params)
collection_w.load()
# vectors = [[random.random() for _ in range(ct.default_dim)] for _ in range(5)]
res1, _ = collection_w.search(data[-1][:5],
ct.default_float_vec_field_name,
ct.default_search_params,
ct.default_limit)
# scale queryNode pod
env.helm_upgrade_cluster_milvus(queryNode=2)
c_name_2 = "query_scale_two"
collection_w2 = ApiCollectionWrapper()
collection_w2.init_collection(
name=c_name_2, schema=cf.gen_default_collection_schema())
collection_w2.insert(data)
assert collection_w2.num_entities == ct.default_nb
collection_w2.load()
res2, _ = collection_w2.search(data[-1][:5],
ct.default_float_vec_field_name,
ct.default_search_params,
ct.default_limit)
assert res1[0].ids == res2[0].ids
def get_milvus_instance_name(namespace, host, port="19530"):
"""
get milvus instance name after connection
:param namespace: the namespace where the release
:type namespace: str
:param host: milvus host ip
:type host: str
:param port: milvus port
:type port: str
:example:
>>> milvus_instance_name = get_milvus_instance_name("chaos-testing", "10.96.250.111")
"milvus-multi-querynode"
"""
connections.add_connection(_default={"host": host, "port": port})
connections.connect(alias='_default')
ms = MilvusSys()
query_node_ip = ms.query_nodes[0]["infos"]['hardware_infos']["ip"].split(
":")[0]
pod_name = ""
if ms.deploy_mode == "STANDALONE":
# get all pods which label is app.kubernetes.io/name=milvus and component=standalone
ip_name_pairs = get_pod_ip_name_pairs(
namespace, "app.kubernetes.io/name=milvus, component=standalone")
pod_name = ip_name_pairs[query_node_ip]
if ms.deploy_mode == "DISTRIBUTED":
# get all pods which label is app.kubernetes.io/name=milvus and component=querynode
ip_name_pairs = get_pod_ip_name_pairs(
namespace, "app.kubernetes.io/name=milvus, component=querynode")
pod_name = ip_name_pairs[query_node_ip]
init_k8s_client_config()
api_instance = client.CoreV1Api()
try:
api_response = api_instance.read_namespaced_pod(namespace=namespace,
name=pod_name)
except ApiException as e:
log.error(
"Exception when calling CoreV1Api->list_namespaced_pod: %s\n" % e)
raise Exception(str(e))
milvus_instance_name = api_response.metadata.labels[
"app.kubernetes.io/instance"]
return milvus_instance_name
def test_expand_data_node(self):
"""
target: test create and insert api after expand dataNode pod
method: 1.create collection a and insert df
2.expand dataNode pod from 1 to 2
3.verify collection a property and verify create and insert of new collection
expected: two collection create and insert op are both correctly
"""
# deploy all nodes one pod cluster milvus with helm
release_name = "scale-data"
env = HelmEnv(release_name=release_name)
host = env.helm_install_cluster_milvus()
# connect
connections.add_connection(default={"host": host, "port": 19530})
connections.connect(alias='default')
# create
c_name = cf.gen_unique_str(prefix)
collection_w = ApiCollectionWrapper()
collection_w.init_collection(name=c_name,
schema=cf.gen_default_collection_schema())
# # insert
data = cf.gen_default_list_data(ct.default_nb)
mutation_res, _ = collection_w.insert(data)
assert mutation_res.insert_count == ct.default_nb
# scale dataNode to 2 pods
env.helm_upgrade_cluster_milvus(dataNode=2)
# after scale, assert data consistent
assert utility.has_collection(c_name)
assert collection_w.num_entities == ct.default_nb
# assert new operations
new_cname = cf.gen_unique_str(prefix)
new_collection_w = ApiCollectionWrapper()
new_collection_w.init_collection(
name=new_cname, schema=cf.gen_default_collection_schema())
new_mutation_res, _ = new_collection_w.insert(data)
assert new_mutation_res.insert_count == ct.default_nb
assert new_collection_w.num_entities == ct.default_nb
# assert old collection ddl
mutation_res_2, _ = collection_w.insert(data)
assert mutation_res.insert_count == ct.default_nb
assert collection_w.num_entities == ct.default_nb * 2
collection_w.drop()
new_collection_w.drop()
def test_expand_data_node(self):
"""
target: test create and insert api after expand dataNode pod
method: 1.create collection a and insert df
2.expand dataNode pod from 1 to 2
3.verify collection a property and verify create and insert of new collection
expected: two collection create and insert op are both correctly
"""
release_name = "scale-data"
milvusOp, host, port = scale_common.deploy_default_milvus(release_name)
# connect
connections.add_connection(default={"host": host, "port": port})
connections.connect(alias='default')
# create
c_name = cf.gen_unique_str(prefix)
collection_w = ApiCollectionWrapper()
collection_w.init_collection(name=c_name, schema=cf.gen_default_collection_schema())
# # insert
data = cf.gen_default_list_data()
mutation_res, _ = collection_w.insert(data)
assert mutation_res.insert_count == ct.default_nb
# scale dataNode to 2 pods
milvusOp.upgrade(release_name, {'spec.components.dataNode.replicas': 2}, constants.NAMESPACE)
milvusOp.wait_for_healthy(release_name, constants.NAMESPACE)
# after scale, assert data consistent
assert utility.has_collection(c_name)
assert collection_w.num_entities == ct.default_nb
# assert new operations
new_cname = cf.gen_unique_str(prefix)
new_collection_w = ApiCollectionWrapper()
new_collection_w.init_collection(name=new_cname, schema=cf.gen_default_collection_schema())
new_mutation_res, _ = new_collection_w.insert(data)
assert new_mutation_res.insert_count == ct.default_nb
assert new_collection_w.num_entities == ct.default_nb
# assert old collection ddl
mutation_res_2, _ = collection_w.insert(data)
assert mutation_res.insert_count == ct.default_nb
assert collection_w.num_entities == ct.default_nb*2
collection_w.drop()
new_collection_w.drop()
def test_auto_load_balance(self):
"""
"""
log.info(f"start to install milvus")
release_name, host, port = install_milvus(
"test-auto-load-balance") # todo add release name
self.release_name = release_name
assert host is not None
conn = connections.connect("default", host=host, port=port)
assert conn is not None
self.health_checkers = {
Op.create: CreateChecker(),
Op.insert: InsertFlushChecker(),
Op.flush: InsertFlushChecker(flush=True),
Op.index: IndexChecker(),
Op.search: SearchChecker(),
Op.query: QueryChecker()
}
cc.start_monitor_threads(self.health_checkers)
# wait
sleep(constants.WAIT_PER_OP * 10)
all_collections = list_collections()
for c in all_collections:
seg_info = utility.get_query_segment_info(c)
seg_distribution = cf.get_segment_distribution(seg_info)
for k in seg_distribution.keys():
log.info(
f"collection {c}'s segment distribution in node {k} is {seg_distribution[k]['sealed']}"
)
# first assert
log.info("first assert")
assert_statistic(self.health_checkers)
# scale up
log.info("scale up milvus")
scale_up_milvus(self.release_name)
# reset counting
cc.reset_counting(self.health_checkers)
sleep(constants.WAIT_PER_OP * 10)
all_collections = list_collections()
for c in all_collections:
seg_info = utility.get_query_segment_info(c)
seg_distribution = cf.get_segment_distribution(seg_info)
for k in seg_distribution.keys():
log.info(
f"collection {c}'s sealed segment distribution in node {k} is {seg_distribution[k]['sealed']}"
)
# second assert
log.info("second assert")
assert_statistic(self.health_checkers)
# TODO assert segment distribution
# assert all expectations
assert_expectations()
def prepare_collection(self, host, port):
""" dim 128, 1000,000 entities loaded needed memory 3-5 Gi"""
connections.connect("default", host=host, port=19530)
collection_w = ApiCollectionWrapper()
c_name = "stress_replicas_2"
collection_w.init_collection(name=c_name,
schema=cf.gen_default_collection_schema(dim=self.dim))
# insert 10 sealed segments
for i in range(20):
t0 = datetime.datetime.now()
df = cf.gen_default_dataframe_data(nb=self.nb, dim=self.dim)
res = collection_w.insert(df)[0]
assert res.insert_count == self.nb
log.info(f'After {i + 1} insert, num_entities: {collection_w.num_entities}')
tt = datetime.datetime.now() - t0
log.info(f"{i} insert and flush data cost: {tt}")
log.debug(collection_w.num_entities)
return collection_w
def e2e_milvus(host, c_name, collection_exist=False):
# connect
connections.add_connection(default={"host": host, "port": 19530})
connections.connect(alias='default')
# create
collection_w = ApiCollectionWrapper()
if collection_exist:
collection_w.init_collection(name=c_name)
else:
collection_w.init_collection(name=c_name,
schema=cf.gen_default_collection_schema())
# insert
data = cf.gen_default_list_data(ct.default_nb)
mutation_res, _ = collection_w.insert(data)
assert mutation_res.insert_count == ct.default_nb
log.debug(collection_w.num_entities)
# create index
collection_w.create_index(ct.default_float_vec_field_name,
ct.default_index)
assert collection_w.has_index()[0]
assert collection_w.index()[0] == Index(collection_w.collection,
ct.default_float_vec_field_name,
ct.default_index)
# search
collection_w.load()
search_res, _ = collection_w.search(data[-1][:ct.default_nq],
ct.default_float_vec_field_name,
ct.default_search_params,
ct.default_limit)
assert len(search_res[0]) == ct.default_limit
log.debug(search_res[0][0].id)
# query
ids = search_res[0].ids[0]
term_expr = f'{ct.default_int64_field_name} in [{ids}]'
query_res, _ = collection_w.query(term_expr, output_fields=["*", "%"])
assert query_res[0][ct.default_int64_field_name] == ids
def e2e_milvus(host, c_name):
""" e2e milvus """
log.debug(f'pid: {os.getpid()}')
# connect
connections.add_connection(default={"host": host, "port": 19530})
connections.connect(alias='default')
# create
collection_w = ApiCollectionWrapper()
collection_w.init_collection(name=c_name,
schema=cf.gen_default_collection_schema())
# insert
df = cf.gen_default_dataframe_data()
mutation_res, _ = collection_w.insert(df)
assert mutation_res.insert_count == ct.default_nb
log.debug(collection_w.num_entities)
# create index
collection_w.create_index(ct.default_float_vec_field_name,
ct.default_index)
assert collection_w.has_index()[0]
assert collection_w.index()[0] == Index(collection_w.collection,
ct.default_float_vec_field_name,
ct.default_index)
# search
collection_w.load()
search_res, _ = collection_w.search(cf.gen_vectors(1, dim=ct.default_dim),
ct.default_float_vec_field_name,
ct.default_search_params,
ct.default_limit)
assert len(search_res[0]) == ct.default_limit
log.debug(search_res[0].ids)
# query
ids = search_res[0].ids[0]
term_expr = f'{ct.default_int64_field_name} in [{ids}]'
query_res, _ = collection_w.query(term_expr, output_fields=["*", "%"])
assert query_res[0][ct.default_int64_field_name] == ids
def test_shrink_data_node(self):
"""
target: test shrink dataNode from 2 to 1
method: 1.create collection and insert df 2. shrink dataNode 3.insert df
expected: verify the property of collection which channel on shrink pod
"""
release_name = "scale-data"
env = HelmEnv(release_name=release_name, dataNode=2)
host = env.helm_install_cluster_milvus(
image_pull_policy=constants.IF_NOT_PRESENT)
# connect
connections.add_connection(default={"host": host, "port": 19530})
connections.connect(alias='default')
c_name = "data_scale_one"
data = cf.gen_default_list_data(ct.default_nb)
collection_w = ApiCollectionWrapper()
collection_w.init_collection(name=c_name,
schema=cf.gen_default_collection_schema())
mutation_res, _ = collection_w.insert(data)
assert mutation_res.insert_count == ct.default_nb
assert collection_w.num_entities == ct.default_nb
c_name_2 = "data_scale_two"
collection_w2 = ApiCollectionWrapper()
collection_w2.init_collection(
name=c_name_2, schema=cf.gen_default_collection_schema())
mutation_res2, _ = collection_w2.insert(data)
assert mutation_res2.insert_count == ct.default_nb
assert collection_w2.num_entities == ct.default_nb
env.helm_upgrade_cluster_milvus(dataNode=1)
assert collection_w.num_entities == ct.default_nb
mutation_res2, _ = collection_w2.insert(data)
assert collection_w2.num_entities == ct.default_nb * 2
collection_w.drop()
collection_w2.drop()
def save_all_checker_collections(host="127.0.0.1", prefix="Checker"):
# create connection
connections.connect(host=host, port="19530")
all_collections = list_collections()
if prefix is None:
all_collections = [c_name for c_name in all_collections]
else:
all_collections = [
c_name for c_name in all_collections if prefix in c_name
]
m = defaultdict(list)
for c_name in all_collections:
prefix = c_name.split("_")[0]
if len(m[prefix]) <= 10:
m[prefix].append(c_name)
selected_collections = []
for v in m.values():
selected_collections.extend(v)
data = {"all": selected_collections}
print("selected_collections is")
print(selected_collections)
with open("/tmp/ci_logs/all_collections.json", "w") as f:
f.write(json.dumps(data))
def test_customize_segment_size(self, size_id):
"""
steps
1. [test_milvus_install]: set up milvus with customized simd configured
2. [test_simd_compat_e2e]: verify milvus is working well
4. [test_milvus_cleanup]: delete milvus instances in teardown
"""
size = customize_segment_sizes[size_id]
log.info(f"start to install milvus with segment size {size}")
release_name, host, port = _install_milvus(size)
self.release_name = release_name
assert host is not None
conn = connections.connect("default", host=host, port=port)
assert conn is not None
mil = MilvusSys(alias="default")
log.info(f"milvus build version: {mil.build_version}")
