def test_shrink_index_node(self):
"""
target: test shrink indexNode from 2 to 1
method: 1.deploy two indexNode
2.create index with two indexNode
3.shrink indexNode from 2 to 1
4.create index with 1 indexNode
expected: The cost of one indexNode is about twice that of two indexNodes
"""
release_name = "scale-index"
env = HelmEnv(release_name=release_name, indexNode=2)
env.helm_install_cluster_milvus()
# connect
connections.add_connection(default={
"host": '10.98.0.8',
"port": 19530
})
connections.connect(alias='default')
data = cf.gen_default_dataframe_data(nb)
# create
c_name = "index_scale_one"
collection_w = ApiCollectionWrapper()
# collection_w.init_collection(name=c_name)
collection_w.init_collection(name=c_name,
schema=cf.gen_default_collection_schema())
# insert
loop = 10
for i in range(loop):
collection_w.insert(data)
assert collection_w.num_entities == nb * loop
# create index on collection one and two
start = datetime.datetime.now()
collection_w.create_index(ct.default_float_vec_field_name,
default_index_params)
assert collection_w.has_index()[0]
t0 = datetime.datetime.now() - start
log.debug(f'two indexNodes: {t0}')
collection_w.drop_index()
assert not collection_w.has_index()[0]
# expand indexNode from 1 to 2
# pdb.set_trace()
env.helm_upgrade_cluster_milvus(indexNode=1)
start = datetime.datetime.now()
collection_w.create_index(ct.default_float_vec_field_name,
default_index_params)
assert collection_w.has_index()[0]
t1 = datetime.datetime.now() - start
log.debug(f'one indexNode: {t1}')
log.debug(t1 / t0)
assert round(t1 / t0) == 2
def test_chaos_memory_stress_indexnode(self, connection, chaos_yaml):
"""
target: test inject memory stress into indexnode
method: 1.Deploy milvus and limit indexnode memory resource 3 / 4Gi
2.Create collection and insert some data
3.Inject memory stress chaos 512Mi
4.Create index
expected:
"""
# init collection and insert
nb = 256000 # vector size: 512*4*nb about 512Mi and create index need 2.8Gi memory
dim = 512
# c_name = cf.gen_unique_str('chaos_memory')
c_name = 'chaos_memory_gKs8aSUu'
index_params = {"index_type": "IVF_SQ8", "metric_type": "L2", "params": {"nlist": 128}}
collection_w = ApiCollectionWrapper()
collection_w.init_collection(name=c_name,
schema=cf.gen_default_collection_schema(dim=dim), shards_num=1)
# insert 256000 512 dim entities, size 512Mi
for i in range(2):
t0_insert = datetime.datetime.now()
df = cf.gen_default_dataframe_data(nb=nb // 2, dim=dim)
res = collection_w.insert(df)[0]
assert res.insert_count == nb // 2
# log.info(f'After {i + 1} insert, num_entities: {collection_w.num_entities}')
tt_insert = datetime.datetime.now() - t0_insert
log.info(f"{i} insert data cost: {tt_insert}")
# flush
t0_flush = datetime.datetime.now()
assert collection_w.num_entities == nb
tt_flush = datetime.datetime.now() - t0_flush
log.info(f'flush {nb * 10} entities cost: {tt_flush}')
log.info(collection_w.indexes[0].params)
if collection_w.has_index()[0]:
collection_w.drop_index()
# indexNode start build index, inject chaos memory stress
chaos_config = gen_experiment_config(chaos_yaml)
log.debug(chaos_config)
chaos_res = CusResource(kind=chaos_config['kind'],
group=constants.CHAOS_GROUP,
version=constants.CHAOS_VERSION,
namespace=constants.CHAOS_NAMESPACE)
chaos_res.create(chaos_config)
log.debug("inject chaos")
# create index
t0_index = datetime.datetime.now()
index, _ = collection_w.create_index(field_name=ct.default_float_vec_field_name,
index_params=index_params)
tt_index = datetime.datetime.now() - t0_index
log.info(f"create index cost: {tt_index}")
log.info(collection_w.indexes[0].params)
def test_expand_index_node(self):
"""
target: test expand indexNode from 1 to 2
method: 1.deploy two indexNode
2.create index with two indexNode
3.expand indexNode from 1 to 2
4.create index with one indexNode
expected: The cost of one indexNode is about twice that of two indexNodes
"""
release_name = "scale-index"
milvusOp, host, port = scale_common.deploy_default_milvus(release_name)
# connect
connections.add_connection(default={"host": host, "port": port})
connections.connect(alias='default')
data = cf.gen_default_dataframe_data(nb)
# create
c_name = "index_scale_one"
collection_w = ApiCollectionWrapper()
# collection_w.init_collection(name=c_name)
collection_w.init_collection(name=c_name,
schema=cf.gen_default_collection_schema())
# insert
loop = 10
for i in range(loop):
collection_w.insert(data)
assert collection_w.num_entities == nb * loop
# create index on collection one and two
start = datetime.datetime.now()
collection_w.create_index(ct.default_float_vec_field_name,
default_index_params)
assert collection_w.has_index()[0]
t0 = datetime.datetime.now() - start
log.debug(f't0: {t0}')
collection_w.drop_index()
assert not collection_w.has_index()[0]
# expand indexNode from 1 to 2
milvusOp.upgrade(release_name,
{'spec.components.indexNode.replicas': 2},
constants.NAMESPACE)
milvusOp.wait_for_healthy(release_name, constants.NAMESPACE)
start = datetime.datetime.now()
collection_w.create_index(ct.default_float_vec_field_name,
default_index_params)
assert collection_w.has_index()[0]
t1 = datetime.datetime.now() - start
log.debug(f't1: {t1}')
assert round(t0 / t1) == 2
def test_expand_index_node(self):
"""
target: test expand indexNode from 1 to 2
method: 1.deploy two indexNode
2.create index with two indexNode
3.expand indexNode from 1 to 2
4.create index with one indexNode
expected: The cost of one indexNode is about twice that of two indexNodes
"""
release_name = "expand-index"
image_tag = get_latest_tag()
image = f'{constants.IMAGE_REPOSITORY}:{image_tag}'
init_replicas = 1
expand_replicas = 2
data_config = {
'metadata.namespace': constants.NAMESPACE,
'spec.mode': 'cluster',
'metadata.name': release_name,
'spec.components.image': image,
'spec.components.proxy.serviceType': 'LoadBalancer',
'spec.components.indexNode.replicas': init_replicas,
'spec.components.dataNode.replicas': 2,
'spec.config.common.retentionDuration': 60
}
mic = MilvusOperator()
mic.install(data_config)
if mic.wait_for_healthy(release_name, constants.NAMESPACE, timeout=1800):
host = mic.endpoint(release_name, constants.NAMESPACE).split(':')[0]
else:
# If deploy failed and want to uninsatll mic
# log.warning(f'Deploy {release_name} timeout and ready to uninstall')
# mic.uninstall(release_name, namespace=constants.NAMESPACE)
raise MilvusException(message=f'Milvus healthy timeout 1800s')
try:
# connect
connections.add_connection(default={"host": host, "port": 19530})
connections.connect(alias='default')
# create collection
c_name = "index_scale_one"
collection_w = ApiCollectionWrapper()
collection_w.init_collection(name=c_name, schema=cf.gen_default_collection_schema())
# insert data
data = cf.gen_default_dataframe_data(nb)
loop = 100
for i in range(loop):
collection_w.insert(data, timeout=60)
assert collection_w.num_entities == nb * loop
# create index
# Note that the num of segments and the num of indexNode are related to indexing time
start = datetime.datetime.now()
collection_w.create_index(ct.default_float_vec_field_name, default_index_params)
assert collection_w.has_index()[0]
t0 = datetime.datetime.now() - start
log.info(f'Create index on {init_replicas} indexNode cost t0: {t0}')
# drop index
collection_w.drop_index()
assert not collection_w.has_index()[0]
# expand indexNode
mic.upgrade(release_name, {'spec.components.indexNode.replicas': expand_replicas}, constants.NAMESPACE)
mic.wait_for_healthy(release_name, constants.NAMESPACE)
wait_pods_ready(constants.NAMESPACE, f"app.kubernetes.io/instance={release_name}")
# create index again
start = datetime.datetime.now()
collection_w.create_index(ct.default_float_vec_field_name, default_index_params)
assert collection_w.has_index()[0]
t1 = datetime.datetime.now() - start
log.info(f'Create index on {expand_replicas} indexNode cost t1: {t1}')
collection_w.drop_index()
start = datetime.datetime.now()
collection_w.create_index(ct.default_float_vec_field_name, default_index_params)
assert collection_w.has_index()[0]
t2 = datetime.datetime.now() - start
log.info(f'Create index on {expand_replicas} indexNode cost t2: {t2}')
log.debug(f't2 is {t2}, t0 is {t0}, t0/t2 is {t0 / t2}')
# assert round(t0 / t2) == 2
except Exception as e:
raise Exception(str(e))
finally:
label = f"app.kubernetes.io/instance={release_name}"
log.info('Start to export milvus pod logs')
read_pod_log(namespace=constants.NAMESPACE, label_selector=label, release_name=release_name)
mic.uninstall(release_name, namespace=constants.NAMESPACE)
def test_shrink_index_node(self):
"""
target: test shrink indexNode from 2 to 1
method: 1.deploy two indexNode
2.create index with two indexNode
3.shrink indexNode from 2 to 1
4.create index with 1 indexNode
expected: The cost of one indexNode is about twice that of two indexNodes
"""
release_name = "shrink-index"
image_tag = get_latest_tag()
image = f'{constants.IMAGE_REPOSITORY}:{image_tag}'
data_config = {
'metadata.namespace': constants.NAMESPACE,
'metadata.name': release_name,
'spec.components.image': image,
'spec.components.proxy.serviceType': 'LoadBalancer',
'spec.components.indexNode.replicas': 2,
'spec.components.dataNode.replicas': 2,
'spec.config.dataCoord.enableCompaction': True,
'spec.config.dataCoord.enableGarbageCollection': True
}
mic = MilvusOperator()
mic.install(data_config)
if mic.wait_for_healthy(release_name,
constants.NAMESPACE,
timeout=1800):
host = mic.endpoint(release_name,
constants.NAMESPACE).split(':')[0]
else:
raise MilvusException(message=f'Milvus healthy timeout 1800s')
try:
# connect
connections.add_connection(default={"host": host, "port": 19530})
connections.connect(alias='default')
data = cf.gen_default_dataframe_data(nb)
# create
c_name = "index_scale_one"
collection_w = ApiCollectionWrapper()
# collection_w.init_collection(name=c_name)
collection_w.init_collection(
name=c_name, schema=cf.gen_default_collection_schema())
# insert
loop = 10
for i in range(loop):
collection_w.insert(data)
assert collection_w.num_entities == nb * loop
# create index on collection one and two
start = datetime.datetime.now()
collection_w.create_index(ct.default_float_vec_field_name,
default_index_params)
assert collection_w.has_index()[0]
t0 = datetime.datetime.now() - start
log.info(f'Create index on 2 indexNode cost t0: {t0}')
collection_w.drop_index()
assert not collection_w.has_index()[0]
# shrink indexNode from 2 to 1
mic.upgrade(release_name,
{'spec.components.indexNode.replicas': 1},
constants.NAMESPACE)
mic.wait_for_healthy(release_name, constants.NAMESPACE)
wait_pods_ready(constants.NAMESPACE,
f"app.kubernetes.io/instance={release_name}")
start = datetime.datetime.now()
collection_w.create_index(ct.default_float_vec_field_name,
default_index_params)
assert collection_w.has_index()[0]
t1 = datetime.datetime.now() - start
log.info(f'Create index on 1 indexNode cost t1: {t1}')
collection_w.drop_index()
start = datetime.datetime.now()
collection_w.create_index(ct.default_float_vec_field_name,
default_index_params)
assert collection_w.has_index()[0]
t2 = datetime.datetime.now() - start
log.info(f'Create index on 1 indexNode cost t2: {t2}')
log.debug(f'one indexNode: {t2}')
log.debug(f't2 is {t2}, t0 is {t0}, t2/t0 is {t2 / t0}')
# assert round(t2 / t0) == 2
except Exception as e:
raise Exception(str(e))
finally:
label = f"app.kubernetes.io/instance={release_name}"
log.info('Start to export milvus pod logs')
read_pod_log(namespace=constants.NAMESPACE,
label_selector=label,
release_name=release_name)
mic.uninstall(release_name, namespace=constants.NAMESPACE)
def test_expand_index_node(self):
"""
target: test expand indexNode from 1 to 2
method: 1.deploy two indexNode
2.create index with two indexNode
3.expand indexNode from 1 to 2
4.create index with one indexNode
expected: The cost of one indexNode is about twice that of two indexNodes
"""
release_name = "scale-index"
image = f'{constants.IMAGE_REPOSITORY}:{constants.IMAGE_TAG}'
data_config = {
'metadata.namespace': constants.NAMESPACE,
'metadata.name': release_name,
'spec.components.image': image,
'spec.components.proxy.serviceType': 'LoadBalancer',
'spec.components.indexNode.replicas': 1,
'spec.components.dataNode.replicas': 2,
'spec.config.dataCoord.enableCompaction': True,
'spec.config.dataCoord.enableGarbageCollection': True
}
mic = MilvusOperator()
mic.install(data_config)
healthy = mic.wait_for_healthy(release_name,
constants.NAMESPACE,
timeout=1200)
log.info(f"milvus healthy: {healthy}")
host = mic.endpoint(release_name, constants.NAMESPACE).split(':')[0]
# host = '10.98.0.8'
# connect
connections.add_connection(default={"host": host, "port": 19530})
connections.connect(alias='default')
data = cf.gen_default_dataframe_data(nb)
# create
c_name = "index_scale_one"
collection_w = ApiCollectionWrapper()
# collection_w.init_collection(name=c_name)
collection_w.init_collection(name=c_name,
schema=cf.gen_default_collection_schema())
# insert
loop = 100
for i in range(loop):
collection_w.insert(data, timeout=60)
assert collection_w.num_entities == nb * loop
# create index on collection
# note that the num of segments and the num of indexNode are related to indexing time
collection_w.drop_index()
start = datetime.datetime.now()
collection_w.create_index(ct.default_float_vec_field_name,
default_index_params)
assert collection_w.has_index()[0]
t0 = datetime.datetime.now() - start
log.debug(f't0: {t0}')
collection_w.drop_index()
assert not collection_w.has_index()[0]
# expand indexNode from 1 to 2
mic.upgrade(release_name, {'spec.components.indexNode.replicas': 2},
constants.NAMESPACE)
time.sleep(60)
mic.wait_for_healthy(release_name, constants.NAMESPACE)
start = datetime.datetime.now()
collection_w.create_index(ct.default_float_vec_field_name,
default_index_params)
assert collection_w.has_index()[0]
t1 = datetime.datetime.now() - start
log.debug(f't1: {t1}')
assert round(t0 / t1) == 2
