def test_scale_proxy(self):
"""
target: test milvus operation after proxy expand
method: 1.deploy 1 proxy replicas
2.milvus e2e test in parallel
3.expand proxy pod from 1 to 5
4.milvus e2e test
5.shrink proxy from 5 to 2
expected: 1.verify data consistent and func work
"""
# deploy milvus cluster with one proxy
release_name = "scale-proxy"
image = f'{constants.IMAGE_REPOSITORY}:{constants.IMAGE_TAG}'
data_config = {
'metadata.namespace': constants.NAMESPACE,
'metadata.name': release_name,
'spec.components.image': image,
'spec.components.proxy.serviceType': 'LoadBalancer',
'spec.components.proxy.replicas': 1,
'spec.components.dataNode.replicas': 2,
'spec.config.dataCoord.enableCompaction': True,
'spec.config.dataCoord.enableGarbageCollection': True
}
mic = MilvusOperator()
mic.install(data_config)
healthy = mic.wait_for_healthy(release_name, constants.NAMESPACE, timeout=1200)
log.info(f"milvus healthy: {healthy}")
host = mic.endpoint(release_name, constants.NAMESPACE).split(':')[0]
# host = "10.98.0.7"
c_name = cf.gen_unique_str(prefix)
self.e2e_milvus_parallel(5, host, c_name)
log.info('Milvus test before expand')
# expand proxy replicas from 1 to 5
mic.upgrade(release_name, {'spec.components.proxy.replicas': 5}, constants.NAMESPACE)
wait_pods_ready(constants.NAMESPACE, f"app.kubernetes.io/instance={release_name}")
self.e2e_milvus_parallel(5, host, c_name)
log.info('Milvus test after expand')
# expand proxy replicas from 5 to 2
mic.upgrade(release_name, {'spec.components.proxy.replicas': 2}, constants.NAMESPACE)
wait_pods_ready(constants.NAMESPACE, f"app.kubernetes.io/instance={release_name}")
self.e2e_milvus_parallel(2, host, c_name)
log.info('Milvus test after shrink')
def test_scale_data_node(self):
"""
target: test scale dataNode
method: 1.deploy milvus cluster with 2 dataNode
2.create collection with shards_num=5
3.continuously insert new data (daemon thread)
4.expand dataNode from 2 to 5
5.create new collection with shards_num=2
6.continuously insert new collection new data (daemon thread)
7.shrink dataNode from 5 to 3
expected: Verify milvus remains healthy, Insert and flush successfully during scale
Average dataNode memory usage
"""
release_name = "scale-data"
image_tag = get_latest_tag()
image = f'{constants.IMAGE_REPOSITORY}:{image_tag}'
fail_count = 0
data_config = {
'metadata.namespace': constants.NAMESPACE,
'metadata.name': release_name,
'spec.components.image': image,
'spec.components.proxy.serviceType': 'LoadBalancer',
'spec.components.dataNode.replicas': 2,
'spec.config.dataCoord.enableCompaction': True,
'spec.config.dataCoord.enableGarbageCollection': True
}
mic = MilvusOperator()
mic.install(data_config)
if mic.wait_for_healthy(release_name,
constants.NAMESPACE,
timeout=1200):
host = mic.endpoint(release_name,
constants.NAMESPACE).split(':')[0]
else:
# log.warning(f'Deploy {release_name} timeout and ready to uninstall')
# mic.uninstall(release_name, namespace=constants.NAMESPACE)
raise BaseException(f'Milvus healthy timeout 1200s')
try:
# connect
connections.add_connection(default={"host": host, "port": 19530})
connections.connect(alias='default')
# create
c_name = cf.gen_unique_str("scale_query")
# c_name = 'scale_query_DymS7kI4'
collection_w = ApiCollectionWrapper()
collection_w.init_collection(
name=c_name,
schema=cf.gen_default_collection_schema(),
shards_num=5)
tmp_nb = 10000
def do_insert():
while True:
tmp_df = cf.gen_default_dataframe_data(tmp_nb)
collection_w.insert(tmp_df)
log.debug(collection_w.num_entities)
t_insert = threading.Thread(target=do_insert, args=(), daemon=True)
t_insert.start()
# scale dataNode to 5
mic.upgrade(release_name, {'spec.components.dataNode.replicas': 5},
constants.NAMESPACE)
mic.wait_for_healthy(release_name, constants.NAMESPACE)
wait_pods_ready(constants.NAMESPACE,
f"app.kubernetes.io/instance={release_name}")
log.debug("Expand dataNode test finished")
# create new collection and insert
new_c_name = cf.gen_unique_str("scale_query")
collection_w_new = ApiCollectionWrapper()
collection_w_new.init_collection(
name=new_c_name,
schema=cf.gen_default_collection_schema(),
shards_num=2)
def do_new_insert():
while True:
tmp_df = cf.gen_default_dataframe_data(tmp_nb)
collection_w_new.insert(tmp_df)
log.debug(collection_w_new.num_entities)
t_insert_new = threading.Thread(target=do_new_insert,
args=(),
daemon=True)
t_insert_new.start()
# scale dataNode to 3
mic.upgrade(release_name, {'spec.components.dataNode.replicas': 3},
constants.NAMESPACE)
mic.wait_for_healthy(release_name, constants.NAMESPACE)
wait_pods_ready(constants.NAMESPACE,
f"app.kubernetes.io/instance={release_name}")
log.debug(collection_w.num_entities)
time.sleep(300)
log.debug("Shrink dataNode test finished")
except Exception as e:
log.error(str(e))
fail_count += 1
# raise Exception(str(e))
finally:
log.info(f'Test finished with {fail_count} fail request')
assert fail_count <= 1
label = f"app.kubernetes.io/instance={release_name}"
log.info('Start to export milvus pod logs')
read_pod_log(namespace=constants.NAMESPACE,
label_selector=label,
release_name=release_name)
mic.uninstall(release_name, namespace=constants.NAMESPACE)
def test_shrink_index_node(self):
"""
target: test shrink indexNode from 2 to 1
method: 1.deploy two indexNode
2.create index with two indexNode
3.shrink indexNode from 2 to 1
4.create index with 1 indexNode
expected: The cost of one indexNode is about twice that of two indexNodes
"""
release_name = "shrink-index"
image_tag = get_latest_tag()
image = f'{constants.IMAGE_REPOSITORY}:{image_tag}'
data_config = {
'metadata.namespace': constants.NAMESPACE,
'metadata.name': release_name,
'spec.components.image': image,
'spec.components.proxy.serviceType': 'LoadBalancer',
'spec.components.indexNode.replicas': 2,
'spec.components.dataNode.replicas': 2,
'spec.config.dataCoord.enableCompaction': True,
'spec.config.dataCoord.enableGarbageCollection': True
}
mic = MilvusOperator()
mic.install(data_config)
if mic.wait_for_healthy(release_name,
constants.NAMESPACE,
timeout=1800):
host = mic.endpoint(release_name,
constants.NAMESPACE).split(':')[0]
else:
raise MilvusException(message=f'Milvus healthy timeout 1800s')
try:
# connect
connections.add_connection(default={"host": host, "port": 19530})
connections.connect(alias='default')
data = cf.gen_default_dataframe_data(nb)
# create
c_name = "index_scale_one"
collection_w = ApiCollectionWrapper()
# collection_w.init_collection(name=c_name)
collection_w.init_collection(
name=c_name, schema=cf.gen_default_collection_schema())
# insert
loop = 10
for i in range(loop):
collection_w.insert(data)
assert collection_w.num_entities == nb * loop
# create index on collection one and two
start = datetime.datetime.now()
collection_w.create_index(ct.default_float_vec_field_name,
default_index_params)
assert collection_w.has_index()[0]
t0 = datetime.datetime.now() - start
log.info(f'Create index on 2 indexNode cost t0: {t0}')
collection_w.drop_index()
assert not collection_w.has_index()[0]
# shrink indexNode from 2 to 1
mic.upgrade(release_name,
{'spec.components.indexNode.replicas': 1},
constants.NAMESPACE)
mic.wait_for_healthy(release_name, constants.NAMESPACE)
wait_pods_ready(constants.NAMESPACE,
f"app.kubernetes.io/instance={release_name}")
start = datetime.datetime.now()
collection_w.create_index(ct.default_float_vec_field_name,
default_index_params)
assert collection_w.has_index()[0]
t1 = datetime.datetime.now() - start
log.info(f'Create index on 1 indexNode cost t1: {t1}')
collection_w.drop_index()
start = datetime.datetime.now()
collection_w.create_index(ct.default_float_vec_field_name,
default_index_params)
assert collection_w.has_index()[0]
t2 = datetime.datetime.now() - start
log.info(f'Create index on 1 indexNode cost t2: {t2}')
log.debug(f'one indexNode: {t2}')
log.debug(f't2 is {t2}, t0 is {t0}, t2/t0 is {t2 / t0}')
# assert round(t2 / t0) == 2
except Exception as e:
raise Exception(str(e))
finally:
label = f"app.kubernetes.io/instance={release_name}"
log.info('Start to export milvus pod logs')
read_pod_log(namespace=constants.NAMESPACE,
label_selector=label,
release_name=release_name)
mic.uninstall(release_name, namespace=constants.NAMESPACE)
def test_scale_query_node(self):
"""
target: test scale queryNode
method: 1.deploy milvus cluster with 1 queryNode
2.prepare work (connect, create, insert, index and load)
3.continuously search (daemon thread)
4.expand queryNode from 2 to 5
5.continuously insert new data (daemon thread)
6.shrink queryNode from 5 to 3
expected: Verify milvus remains healthy and search successfully during scale
"""
release_name = "scale-query"
query_config = {
'metadata.namespace': constants.NAMESPACE,
'metadata.name': release_name,
'spec.components.image': 'harbor.zilliz.cc/milvus/milvus:master-20211202-ed546d0',
'spec.components.proxy.serviceType': 'LoadBalancer',
'spec.components.queryNode.replicas': 1,
'spec.config.dataCoord.enableCompaction': True,
'spec.config.dataCoord.enableGarbageCollection': True
}
mic = MilvusOperator()
mic.install(query_config)
healthy = mic.wait_for_healthy(release_name, constants.NAMESPACE, timeout=1200)
log.info(f"milvus healthy: {healthy}")
host = mic.endpoint(release_name, constants.NAMESPACE).split(':')[0]
# host = "10.98.0.8"
# connect
connections.add_connection(default={"host": host, "port": 19530})
connections.connect(alias='default')
# create
c_name = cf.gen_unique_str("scale_query")
# c_name = 'scale_query_DymS7kI4'
collection_w = ApiCollectionWrapper()
collection_w.init_collection(name=c_name, schema=cf.gen_default_collection_schema(), shards_num=2)
# insert two segments
for i in range(3):
df = cf.gen_default_dataframe_data(nb)
collection_w.insert(df)
log.debug(collection_w.num_entities)
# create index
collection_w.create_index(ct.default_float_vec_field_name, default_index_params)
assert collection_w.has_index()[0]
assert collection_w.index()[0] == Index(collection_w.collection, ct.default_float_vec_field_name,
default_index_params)
# load
collection_w.load()
# scale queryNode to 5
mic.upgrade(release_name, {'spec.components.queryNode.replicas': 5}, constants.NAMESPACE)
# continuously search
def do_search():
while True:
search_res, _ = collection_w.search(cf.gen_vectors(1, ct.default_dim),
ct.default_float_vec_field_name,
ct.default_search_params, ct.default_limit)
log.debug(search_res[0].ids)
assert len(search_res[0].ids) == ct.default_limit
t_search = threading.Thread(target=do_search, args=(), daemon=True)
t_search.start()
# wait new QN running, continuously insert
# time.sleep(10)
healthy = mic.wait_for_healthy(release_name, constants.NAMESPACE, timeout=1200)
log.info(f"milvus healthy after scale up: {healthy}")
# wait_pods_ready(constants.NAMESPACE, f"app.kubernetes.io/instance={release_name}")
def do_insert():
while True:
tmp_df = cf.gen_default_dataframe_data(1000)
collection_w.insert(tmp_df)
t_insert = threading.Thread(target=do_insert, args=(), daemon=True)
t_insert.start()
log.debug(collection_w.num_entities)
time.sleep(20)
log.debug("Expand querynode test finished")
mic.upgrade(release_name, {'spec.components.queryNode.replicas': 3}, constants.NAMESPACE)
time.sleep(60)
wait_pods_ready(constants.NAMESPACE, f"app.kubernetes.io/instance={release_name}")
log.debug(collection_w.num_entities)
time.sleep(60)
log.debug("Shrink querynode test finished")
def test_chaos_data_consist(self, connection, chaos_yaml):
"""
target: verify data consistence after chaos injected and recovered
method: 1. create a collection, insert some data, search and query
2. inject a chaos object
3. reconnect to service
4. verify a) data entities persists, index persists,
b) search and query results persist
expected: collection data and results persist
"""
c_name = cf.gen_unique_str('chaos_collection_')
nb = 5000
i_name = cf.gen_unique_str('chaos_index_')
index_params = {
"index_type": "IVF_SQ8",
"metric_type": "L2",
"params": {
"nlist": 64
}
}
# create
t0 = datetime.datetime.now()
collection_w = ApiCollectionWrapper()
collection_w.init_collection(name=c_name,
schema=cf.gen_default_collection_schema())
tt = datetime.datetime.now() - t0
log.info(f"assert create: {tt}")
assert collection_w.name == c_name
# insert
data = cf.gen_default_list_data(nb=nb)
t0 = datetime.datetime.now()
_, res = collection_w.insert(data)
tt = datetime.datetime.now() - t0
log.info(f"assert insert: {tt}")
assert res
# flush
t0 = datetime.datetime.now()
assert collection_w.num_entities == nb
tt = datetime.datetime.now() - t0
log.info(f"assert flush: {tt}")
# search
collection_w.load()
search_vectors = cf.gen_vectors(1, ct.default_dim)
t0 = datetime.datetime.now()
search_params = {"metric_type": "L2", "params": {"nprobe": 16}}
search_res, _ = collection_w.search(
data=search_vectors,
anns_field=ct.default_float_vec_field_name,
param=search_params,
limit=1)
tt = datetime.datetime.now() - t0
log.info(f"assert search: {tt}")
assert len(search_res) == 1
# index
t0 = datetime.datetime.now()
index, _ = collection_w.create_index(
field_name=ct.default_float_vec_field_name,
index_params=index_params,
name=i_name)
tt = datetime.datetime.now() - t0
log.info(f"assert index: {tt}")
assert len(collection_w.indexes) == 1
# query
term_expr = f'{ct.default_int64_field_name} in [1001,1201,999,99]'
t0 = datetime.datetime.now()
query_res, _ = collection_w.query(term_expr)
tt = datetime.datetime.now() - t0
log.info(f"assert query: {tt}")
assert len(query_res) == 4
# reboot a pod
reboot_pod(chaos_yaml)
# parse chaos object
chaos_config = cc.gen_experiment_config(chaos_yaml)
meta_name = chaos_config.get('metadata', None).get('name', None)
# wait all pods ready
log.info(
f"wait for pods in namespace {constants.CHAOS_NAMESPACE} with label app.kubernetes.io/instance={meta_name}"
)
wait_pods_ready(constants.CHAOS_NAMESPACE,
f"app.kubernetes.io/instance={meta_name}")
log.info(
f"wait for pods in namespace {constants.CHAOS_NAMESPACE} with label release={meta_name}"
)
wait_pods_ready(constants.CHAOS_NAMESPACE, f"release={meta_name}")
log.info("all pods are ready")
# reconnect if needed
sleep(constants.WAIT_PER_OP * 3)
reconnect(connections, alias='default')
# verify collection persists
assert utility.has_collection(c_name)
log.info("assert collection persists")
collection_w2 = ApiCollectionWrapper()
collection_w2.init_collection(c_name)
# verify data persist
assert collection_w2.num_entities == nb
log.info("assert data persists")
# verify index persists
assert collection_w2.has_index(i_name)
log.info("assert index persists")
# verify search results persist
collection_w2.load()
search_res, _ = collection_w.search(
data=search_vectors,
anns_field=ct.default_float_vec_field_name,
param=search_params,
limit=1)
tt = datetime.datetime.now() - t0
log.info(f"assert search: {tt}")
assert len(search_res) == 1
# verify query results persist
query_res2, _ = collection_w2.query(term_expr)
assert len(query_res2) == len(query_res)
log.info("assert query result persists")
def test_multi_replicas_with_only_one_group_available(
self, chaos_type, failed_node_type, failed_group_scope,
is_streaming):
# start the monitor threads to check the milvus ops
log.info("*********************Chaos Test Start**********************")
log.info("Test config")
log.info(cc.gen_experiment_config(config_file_name))
# log.info(f"chaos_yaml: {chaos_yaml}")
log.info(connections.get_connection_addr('default'))
if is_streaming is False:
del self.health_checkers[Op.insert]
cc.start_monitor_threads(self.health_checkers)
# get replicas info
release_name = self.instance_name
querynode_id_pod_pair = get_querynode_info(release_name)
log.info(querynode_id_pod_pair)
group_list = []
shard_leader_list = []
replicas_info, _ = self.health_checkers[
Op.search].c_wrap.get_replicas()
for g in replicas_info.groups:
group_list.append(list(g.group_nodes))
for shard in g.shards:
shard_leader_list.append(shard.shard_leader)
# keep only one group in healthy status, other groups will be unhealthy by injecting pod failure chaos,
# In the effected groups, each group has one pod is in pod failure status
target_pod_list = []
target_group = []
group_list = sorted(group_list, key=lambda x: -len(x))
if failed_group_scope == "one":
target_group = random.sample(group_list, 1)
if failed_group_scope == "except_one":
target_group = random.sample(group_list, len(group_list) - 1)
if failed_group_scope == "all":
target_group = group_list[:]
for g in target_group:
target_nodes = []
if failed_node_type == "shard_leader":
target_nodes = list(set(g) & set(shard_leader_list))
if failed_node_type == "non_shard_leader":
target_nodes = list(set(g) - set(shard_leader_list))
if len(target_nodes) == 0:
log.info("there is no node satisfied, chose one randomly")
target_nodes = [random.choice(g)]
for target_node in target_nodes:
pod = querynode_id_pod_pair[target_node]
target_pod_list.append(pod)
log.info(f"target_pod_list: {target_pod_list}")
chaos_config = cc.gen_experiment_config(
f"{str(Path(__file__).absolute().parent)}/chaos_objects/template/{chaos_type}-by-pod-list.yaml"
)
chaos_config['metadata'][
'name'] = f"test-multi-replicase-{int(time.time())}"
meta_name = chaos_config.get('metadata', None).get('name', None)
chaos_config['spec']['selector']['pods'][
'chaos-testing'] = target_pod_list
self._chaos_config = chaos_config # cache the chaos config for tear down
log.info(f"chaos_config: {chaos_config}")
# wait 20s
sleep(constants.WAIT_PER_OP * 2)
# replicas info
replicas_info, _ = self.health_checkers[
Op.search].c_wrap.get_replicas()
log.info(
f"replicas_info for search collection {self.health_checkers[Op.search].c_wrap.name}: {replicas_info}"
)
# assert statistic:all ops 100% succ
log.info("******1st assert before chaos: ")
assert_statistic(self.health_checkers)
# apply chaos object
chaos_res = CusResource(kind=chaos_config['kind'],
group=constants.CHAOS_GROUP,
version=constants.CHAOS_VERSION,
namespace=constants.CHAOS_NAMESPACE)
chaos_res.create(chaos_config)
log.info("chaos injected")
sleep(constants.WAIT_PER_OP * 2)
# reset counting
cc.reset_counting(self.health_checkers)
# wait 120s
sleep(constants.CHAOS_DURATION)
log.info(f'Alive threads: {threading.enumerate()}')
# node info
querynode_id_pod_pair = get_querynode_info(release_name)
log.info(querynode_id_pod_pair)
# replicas info
replicas_info, _ = self.health_checkers[
Op.search].c_wrap.get_replicas()
log.info(
f"replicas_info for search collection {self.health_checkers[Op.search].c_wrap.name}: {replicas_info}"
)
replicas_info, _ = self.health_checkers[Op.query].c_wrap.get_replicas()
log.info(
f"replicas_info for query collection {self.health_checkers[Op.query].c_wrap.name}: {replicas_info}"
)
# assert statistic
log.info("******2nd assert after chaos injected: ")
expectations = {Op.search: constants.SUCC, Op.query: constants.SUCC}
if failed_group_scope == "all":
expectations = {
Op.search: constants.FAIL,
Op.query: constants.FAIL
}
assert_statistic(self.health_checkers, expectations=expectations)
# delete chaos
chaos_res.delete(meta_name)
log.info("chaos deleted")
sleep(2)
# wait all pods ready
log.info(
f"wait for pods in namespace {constants.CHAOS_NAMESPACE} with label app.kubernetes.io/instance={release_name}"
)
ready_1 = wait_pods_ready(
constants.CHAOS_NAMESPACE,
f"app.kubernetes.io/instance={release_name}")
log.info(
f"wait for pods in namespace {constants.CHAOS_NAMESPACE} with label release={release_name}"
)
ready_2 = wait_pods_ready(constants.CHAOS_NAMESPACE,
f"release={release_name}")
if ready_1 and ready_2:
log.info("all pods are ready")
# reconnect if needed
sleep(constants.WAIT_PER_OP * 2)
# cc.reconnect(connections, alias='default')
# reset counting again
cc.reset_counting(self.health_checkers)
# wait 50s (varies by feature)
sleep(constants.WAIT_PER_OP * 5)
# node info
querynode_id_pod_pair = get_querynode_info(release_name)
log.info(querynode_id_pod_pair)
sleep(30)
# replicas info
replicas_info, _ = self.health_checkers[
Op.search].c_wrap.get_replicas()
log.info(
f"replicas_info for collection {self.health_checkers[Op.search].c_wrap.name}: {replicas_info}"
)
replicas_info, _ = self.health_checkers[Op.query].c_wrap.get_replicas()
log.info(
f"replicas_info for collection {self.health_checkers[Op.query].c_wrap.name}: {replicas_info}"
)
# assert statistic: all ops success again
log.info("******3rd assert after chaos deleted: ")
assert_statistic(self.health_checkers)
# assert all expectations
assert_expectations()
log.info(
"*********************Chaos Test Completed**********************")
def test_scale_data_node(self):
"""
target: test scale dataNode
method: 1.deploy milvus cluster with 2 dataNode
2.create collection with shards_num=5
3.continuously insert new data (daemon thread)
4.expand dataNode from 2 to 5
5.create new collection with shards_num=2
6.continuously insert new collection new data (daemon thread)
7.shrink dataNode from 5 to 3
expected: Verify milvus remains healthy, Insert and flush successfully during scale
Average dataNode memory usage
"""
release_name = "scale-data"
image = f'{constants.IMAGE_REPOSITORY}:{constants.IMAGE_TAG}'
data_config = {
'metadata.namespace': constants.NAMESPACE,
'metadata.name': release_name,
'spec.components.image': image,
'spec.components.proxy.serviceType': 'LoadBalancer',
'spec.components.dataNode.replicas': 2,
'spec.config.dataCoord.enableCompaction': True,
'spec.config.dataCoord.enableGarbageCollection': True
}
mic = MilvusOperator()
mic.install(data_config)
healthy = mic.wait_for_healthy(release_name,
constants.NAMESPACE,
timeout=1200)
log.info(f"milvus healthy: {healthy}")
host = mic.endpoint(release_name, constants.NAMESPACE).split(':')[0]
# host = '10.98.0.4'
# connect
connections.add_connection(default={"host": host, "port": 19530})
connections.connect(alias='default')
# create
c_name = cf.gen_unique_str("scale_query")
# c_name = 'scale_query_DymS7kI4'
collection_w = ApiCollectionWrapper()
collection_w.init_collection(name=c_name,
schema=cf.gen_default_collection_schema(),
shards_num=5)
tmp_nb = 10000
def do_insert():
while True:
tmp_df = cf.gen_default_dataframe_data(tmp_nb)
collection_w.insert(tmp_df)
log.debug(collection_w.num_entities)
t_insert = threading.Thread(target=do_insert, args=(), daemon=True)
t_insert.start()
# scale dataNode to 5
mic.upgrade(release_name, {'spec.components.dataNode.replicas': 5},
constants.NAMESPACE)
time.sleep(300)
log.debug("Expand dataNode test finished")
# create new collection and insert
new_c_name = cf.gen_unique_str("scale_query")
collection_w_new = ApiCollectionWrapper()
collection_w_new.init_collection(
name=new_c_name,
schema=cf.gen_default_collection_schema(),
shards_num=2)
def do_new_insert():
while True:
tmp_df = cf.gen_default_dataframe_data(tmp_nb)
collection_w_new.insert(tmp_df)
log.debug(collection_w_new.num_entities)
t_insert_new = threading.Thread(target=do_new_insert,
args=(),
daemon=True)
t_insert_new.start()
# scale dataNode to 3
mic.upgrade(release_name, {'spec.components.dataNode.replicas': 3},
constants.NAMESPACE)
wait_pods_ready(constants.NAMESPACE,
f"app.kubernetes.io/instance={release_name}")
log.debug(collection_w.num_entities)
time.sleep(300)
log.debug("Shrink dataNode test finished")
def test_bulk_load(self, chaos_type, target_component):
# start the monitor threads to check the milvus ops
log.info("*********************Chaos Test Start**********************")
log.info(connections.get_connection_addr('default'))
release_name = self.instance_name
cc.start_monitor_threads(self.health_checkers)
chaos_config = cc.gen_experiment_config(
f"{str(Path(__file__).absolute().parent)}/chaos_objects/{chaos_type}/chaos_{target_component}_{chaos_type}.yaml"
)
chaos_config['metadata']['name'] = f"test-bulk-load-{int(time.time())}"
kind = chaos_config['kind']
meta_name = chaos_config.get('metadata', None).get('name', None)
update_key_value(chaos_config, "release", release_name)
update_key_value(chaos_config, "app.kubernetes.io/instance",
release_name)
self._chaos_config = chaos_config # cache the chaos config for tear down
log.info(f"chaos_config: {chaos_config}")
# wait 20s
sleep(constants.WAIT_PER_OP * 10)
# assert statistic:all ops 100% succ
log.info("******1st assert before chaos: ")
assert_statistic(self.health_checkers)
# apply chaos object
chaos_res = CusResource(kind=chaos_config['kind'],
group=constants.CHAOS_GROUP,
version=constants.CHAOS_VERSION,
namespace=constants.CHAOS_NAMESPACE)
chaos_res.create(chaos_config)
log.info("chaos injected")
sleep(constants.WAIT_PER_OP * 10)
# reset counting
cc.reset_counting(self.health_checkers)
# wait 120s
sleep(constants.CHAOS_DURATION)
log.info(f'Alive threads: {threading.enumerate()}')
# assert statistic
log.info("******2nd assert after chaos injected: ")
assert_statistic(self.health_checkers,
expectations={
Op.bulk_load: constants.FAIL,
})
# delete chaos
chaos_res.delete(meta_name)
log.info("chaos deleted")
sleep(2)
# wait all pods ready
log.info(
f"wait for pods in namespace {constants.CHAOS_NAMESPACE} with label app.kubernetes.io/instance={release_name}"
)
wait_pods_ready(constants.CHAOS_NAMESPACE,
f"app.kubernetes.io/instance={release_name}")
log.info(
f"wait for pods in namespace {constants.CHAOS_NAMESPACE} with label release={release_name}"
)
wait_pods_ready(constants.CHAOS_NAMESPACE, f"release={release_name}")
log.info("all pods are ready")
# reconnect if needed
sleep(constants.WAIT_PER_OP * 2)
cc.reconnect(connections, alias='default')
# recheck failed tasks in third assert
self.health_checkers[Op.bulk_load].recheck_failed_task = True
# reset counting again
cc.reset_counting(self.health_checkers)
# wait 50s (varies by feature)
sleep(constants.WAIT_PER_OP * 10)
# assert statistic: all ops success again
log.info("******3rd assert after chaos deleted: ")
assert_statistic(self.health_checkers)
# assert all expectations
assert_expectations()
log.info(
"*********************Chaos Test Completed**********************")
def test_scale_proxy(self):
"""
target: test milvus operation after proxy expand
method: 1.deploy 1 proxy replicas
2.milvus e2e test in parallel
3.expand proxy pod from 1 to 5
4.milvus e2e test
5.shrink proxy from 5 to 2
expected: 1.verify data consistent and func work
"""
# deploy milvus cluster with one proxy
fail_count = 0
release_name = "scale-proxy"
image_tag = get_latest_tag()
image = f'{constants.IMAGE_REPOSITORY}:{image_tag}'
data_config = {
'metadata.namespace': constants.NAMESPACE,
'metadata.name': release_name,
'spec.components.image': image,
'spec.components.proxy.serviceType': 'LoadBalancer',
'spec.components.proxy.replicas': 1,
'spec.components.dataNode.replicas': 2,
'spec.config.dataCoord.enableCompaction': True,
'spec.config.dataCoord.enableGarbageCollection': True
}
mic = MilvusOperator()
mic.install(data_config)
if mic.wait_for_healthy(release_name,
constants.NAMESPACE,
timeout=1200):
host = mic.endpoint(release_name,
constants.NAMESPACE).split(':')[0]
else:
# log.warning(f'Deploy {release_name} timeout and ready to uninstall')
# mic.uninstall(release_name, namespace=constants.NAMESPACE)
raise BaseException(f'Milvus healthy timeout 1200s')
try:
c_name = cf.gen_unique_str(prefix)
self.e2e_milvus_parallel(5, host, c_name)
log.info('Milvus test before expand')
# expand proxy replicas from 1 to 5
mic.upgrade(release_name, {'spec.components.proxy.replicas': 5},
constants.NAMESPACE)
mic.wait_for_healthy(release_name, constants.NAMESPACE)
wait_pods_ready(constants.NAMESPACE,
f"app.kubernetes.io/instance={release_name}")
self.e2e_milvus_parallel(5, host, c_name)
log.info('Milvus test after expand')
# expand proxy replicas from 5 to 2
mic.upgrade(release_name, {'spec.components.proxy.replicas': 2},
constants.NAMESPACE)
mic.wait_for_healthy(release_name, constants.NAMESPACE)
wait_pods_ready(constants.NAMESPACE,
f"app.kubernetes.io/instance={release_name}")
self.e2e_milvus_parallel(2, host, c_name)
log.info('Milvus test after shrink')
except Exception as e:
log.error(str(e))
fail_count += 1
# raise Exception(str(e))
finally:
log.info(f'Test finished with {fail_count} fail request')
assert fail_count <= 1
label = f"app.kubernetes.io/instance={release_name}"
log.info('Start to export milvus pod logs')
read_pod_log(namespace=constants.NAMESPACE,
label_selector=label,
release_name=release_name)
mic.uninstall(release_name, namespace=constants.NAMESPACE)
def test_scale_query_node(self):
"""
target: test scale queryNode
method: 1.deploy milvus cluster with 1 queryNode
2.prepare work (connect, create, insert, index and load)
3.continuously search (daemon thread)
4.expand queryNode from 2 to 5
5.continuously insert new data (daemon thread)
6.shrink queryNode from 5 to 3
expected: Verify milvus remains healthy and search successfully during scale
"""
release_name = "scale-query"
image_tag = get_latest_tag()
image = f'{constants.IMAGE_REPOSITORY}:{image_tag}'
query_config = {
'metadata.namespace': constants.NAMESPACE,
'spec.mode': 'cluster',
'metadata.name': release_name,
'spec.components.image': image,
'spec.components.proxy.serviceType': 'LoadBalancer',
'spec.components.queryNode.replicas': 1,
'spec.config.common.retentionDuration': 60
}
mic = MilvusOperator()
mic.install(query_config)
if mic.wait_for_healthy(release_name,
constants.NAMESPACE,
timeout=1800):
host = mic.endpoint(release_name,
constants.NAMESPACE).split(':')[0]
else:
raise MilvusException(message=f'Milvus healthy timeout 1800s')
try:
# connect
connections.add_connection(default={"host": host, "port": 19530})
connections.connect(alias='default')
# create
c_name = cf.gen_unique_str("scale_query")
# c_name = 'scale_query_DymS7kI4'
collection_w = ApiCollectionWrapper()
collection_w.init_collection(
name=c_name,
schema=cf.gen_default_collection_schema(),
shards_num=2)
# insert two segments
for i in range(3):
df = cf.gen_default_dataframe_data(nb)
collection_w.insert(df)
log.debug(collection_w.num_entities)
# create index
collection_w.create_index(ct.default_float_vec_field_name,
default_index_params)
assert collection_w.has_index()[0]
assert collection_w.index()[0] == Index(
collection_w.collection, ct.default_float_vec_field_name,
default_index_params)
# load
collection_w.load()
# scale queryNode to 5
mic.upgrade(release_name,
{'spec.components.queryNode.replicas': 5},
constants.NAMESPACE)
@counter
def do_search():
""" do search """
search_res, is_succ = collection_w.search(
cf.gen_vectors(1, ct.default_dim),
ct.default_float_vec_field_name,
ct.default_search_params,
ct.default_limit,
check_task=CheckTasks.check_nothing)
assert len(search_res) == 1
return search_res, is_succ
def loop_search():
""" continuously search """
while True:
do_search()
threading.Thread(target=loop_search, args=(), daemon=True).start()
# wait new QN running, continuously insert
mic.wait_for_healthy(release_name, constants.NAMESPACE)
wait_pods_ready(constants.NAMESPACE,
f"app.kubernetes.io/instance={release_name}")
@counter
def do_insert():
""" do insert """
return collection_w.insert(cf.gen_default_dataframe_data(1000),
check_task=CheckTasks.check_nothing)
def loop_insert():
""" loop insert """
while True:
do_insert()
threading.Thread(target=loop_insert, args=(), daemon=True).start()
log.debug(collection_w.num_entities)
time.sleep(20)
log.debug("Expand querynode test finished")
mic.upgrade(release_name,
{'spec.components.queryNode.replicas': 3},
constants.NAMESPACE)
mic.wait_for_healthy(release_name, constants.NAMESPACE)
wait_pods_ready(constants.NAMESPACE,
f"app.kubernetes.io/instance={release_name}")
log.debug(collection_w.num_entities)
time.sleep(60)
scale_common.check_succ_rate(do_search)
scale_common.check_succ_rate(do_insert)
log.debug("Shrink querynode test finished")
except Exception as e:
raise Exception(str(e))
finally:
label = f"app.kubernetes.io/instance={release_name}"
log.info('Start to export milvus pod logs')
read_pod_log(namespace=constants.NAMESPACE,
label_selector=label,
release_name=release_name)
mic.uninstall(release_name, namespace=constants.NAMESPACE)
def test_scale_in_query_node_less_than_replicas(self):
"""
target: test scale in cluster and querynode < replica
method: 1.Deploy cluster with 3 querynodes
2.Create and insert data, flush
3.Load collection with 2 replica number
4.Scale in querynode from 3 to 1 and query
5.Scale out querynode from 1 back to 3
expected: Verify search successfully after scale out
"""
release_name = "scale-in-query"
image_tag = get_latest_tag()
image = f'{constants.IMAGE_REPOSITORY}:{image_tag}'
query_config = {
'metadata.namespace': constants.NAMESPACE,
'metadata.name': release_name,
'spec.mode': 'cluster',
'spec.components.image': image,
'spec.components.proxy.serviceType': 'LoadBalancer',
'spec.components.queryNode.replicas': 2,
'spec.config.common.retentionDuration': 60
}
mic = MilvusOperator()
mic.install(query_config)
if mic.wait_for_healthy(release_name,
constants.NAMESPACE,
timeout=1800):
host = mic.endpoint(release_name,
constants.NAMESPACE).split(':')[0]
else:
raise MilvusException(message=f'Milvus healthy timeout 1800s')
try:
# prepare collection
connections.connect("scale-in", host=host, port=19530)
utility_w = ApiUtilityWrapper()
collection_w = ApiCollectionWrapper()
collection_w.init_collection(
name=cf.gen_unique_str("scale_in"),
schema=cf.gen_default_collection_schema(),
using="scale-in")
collection_w.insert(cf.gen_default_dataframe_data())
assert collection_w.num_entities == ct.default_nb
# load multi replicas and search success
collection_w.load(replica_number=2)
search_res, is_succ = collection_w.search(
cf.gen_vectors(1, ct.default_dim),
ct.default_float_vec_field_name, ct.default_search_params,
ct.default_limit)
assert len(search_res[0].ids) == ct.default_limit
log.info("Search successfully after load with 2 replicas")
log.debug(collection_w.get_replicas()[0])
log.debug(
utility_w.get_query_segment_info(collection_w.name,
using="scale-in"))
# scale in querynode from 2 to 1, less than replica number
log.debug("Scale in querynode from 2 to 1")
mic.upgrade(release_name,
{'spec.components.queryNode.replicas': 1},
constants.NAMESPACE)
mic.wait_for_healthy(release_name, constants.NAMESPACE)
wait_pods_ready(constants.NAMESPACE,
f"app.kubernetes.io/instance={release_name}")
# search and not assure success
collection_w.search(cf.gen_vectors(1, ct.default_dim),
ct.default_float_vec_field_name,
ct.default_search_params,
ct.default_limit,
check_task=CheckTasks.check_nothing)
log.debug(
collection_w.get_replicas(
check_task=CheckTasks.check_nothing)[0])
# scale querynode from 1 back to 2
mic.upgrade(release_name,
{'spec.components.queryNode.replicas': 2},
constants.NAMESPACE)
mic.wait_for_healthy(release_name, constants.NAMESPACE)
wait_pods_ready(constants.NAMESPACE,
f"app.kubernetes.io/instance={release_name}")
# verify search success
collection_w.search(cf.gen_vectors(1, ct.default_dim),
ct.default_float_vec_field_name,
ct.default_search_params, ct.default_limit)
# Verify replica info is correct
replicas = collection_w.get_replicas()[0]
assert len(replicas.groups) == 2
for group in replicas.groups:
assert len(group.group_nodes) == 1
# Verify loaded segment info is correct
seg_info = utility_w.get_query_segment_info(collection_w.name,
using="scale-in")[0]
num_entities = 0
for seg in seg_info:
assert len(seg.nodeIds) == 2
num_entities += seg.num_rows
assert num_entities == ct.default_nb
except Exception as e:
raise Exception(str(e))
finally:
label = f"app.kubernetes.io/instance={release_name}"
log.info('Start to export milvus pod logs')
read_pod_log(namespace=constants.NAMESPACE,
label_selector=label,
release_name=release_name)
mic.uninstall(release_name, namespace=constants.NAMESPACE)
def test_scale_query_node_replicas(self):
"""
target: test scale out querynode when load multi replicas
method: 1.Deploy cluster with 5 querynodes
2.Create collection with 2 shards
3.Insert 10 segments and flushed
4.Load collection with 2 replicas
5.Scale out querynode from 5 to 6 while search and insert growing data
expected: Verify search succ rate is 100%
"""
release_name = "scale-replica"
image_tag = get_latest_tag()
image = f'{constants.IMAGE_REPOSITORY}:{image_tag}'
query_config = {
'metadata.namespace': constants.NAMESPACE,
'metadata.name': release_name,
'spec.mode': 'cluster',
'spec.components.image': image,
'spec.components.proxy.serviceType': 'LoadBalancer',
'spec.components.queryNode.replicas': 5,
'spec.config.common.retentionDuration': 60
}
mic = MilvusOperator()
mic.install(query_config)
if mic.wait_for_healthy(release_name,
constants.NAMESPACE,
timeout=1800):
host = mic.endpoint(release_name,
constants.NAMESPACE).split(':')[0]
else:
raise MilvusException(message=f'Milvus healthy timeout 1800s')
try:
scale_querynode = random.choice([6, 7, 4, 3])
connections.connect("scale-replica", host=host, port=19530)
collection_w = ApiCollectionWrapper()
collection_w.init_collection(
name=cf.gen_unique_str("scale_out"),
schema=cf.gen_default_collection_schema(),
using='scale-replica',
shards_num=3)
# insert 10 sealed segments
for i in range(5):
df = cf.gen_default_dataframe_data(nb=nb, start=i * nb)
collection_w.insert(df)
assert collection_w.num_entities == (i + 1) * nb
collection_w.load(replica_number=2)
@counter
def do_search():
""" do search """
search_res, is_succ = collection_w.search(
cf.gen_vectors(1, ct.default_dim),
ct.default_float_vec_field_name,
ct.default_search_params,
ct.default_limit,
check_task=CheckTasks.check_nothing)
assert len(search_res) == 1
return search_res, is_succ
def loop_search():
""" continuously search """
while True:
do_search()
threading.Thread(target=loop_search, args=(), daemon=True).start()
# scale out
mic.upgrade(
release_name,
{'spec.components.queryNode.replicas': scale_querynode},
constants.NAMESPACE)
mic.wait_for_healthy(release_name, constants.NAMESPACE)
wait_pods_ready(constants.NAMESPACE,
f"app.kubernetes.io/instance={release_name}")
log.debug("Scale out querynode success")
time.sleep(100)
scale_common.check_succ_rate(do_search)
log.debug("Scale out test finished")
except Exception as e:
raise Exception(str(e))
finally:
label = f"app.kubernetes.io/instance={release_name}"
log.info('Start to export milvus pod logs')
read_pod_log(namespace=constants.NAMESPACE,
label_selector=label,
release_name=release_name)
mic.uninstall(release_name, namespace=constants.NAMESPACE)
def test_scale_query_node(self):
"""
target: test scale queryNode
method: 1.deploy milvus cluster with 1 queryNode
2.prepare work (connect, create, insert, index and load)
3.continuously search (daemon thread)
4.expand queryNode from 2 to 5
5.continuously insert new data (daemon thread)
6.shrink queryNode from 5 to 3
expected: Verify milvus remains healthy and search successfully during scale
"""
fail_count = 0
release_name = "scale-query"
image_tag = get_latest_tag()
image = f'{constants.IMAGE_REPOSITORY}:{image_tag}'
query_config = {
'metadata.namespace': constants.NAMESPACE,
'metadata.name': release_name,
'spec.components.image': image,
'spec.components.proxy.serviceType': 'LoadBalancer',
'spec.components.queryNode.replicas': 1,
'spec.config.dataCoord.enableCompaction': True,
'spec.config.dataCoord.enableGarbageCollection': True
}
mic = MilvusOperator()
mic.install(query_config)
if mic.wait_for_healthy(release_name, constants.NAMESPACE, timeout=1200):
host = mic.endpoint(release_name, constants.NAMESPACE).split(':')[0]
else:
# log.warning(f'Deploy {release_name} timeout and ready to uninstall')
# mic.uninstall(release_name, namespace=constants.NAMESPACE)
raise BaseException(f'Milvus healthy timeout 1200s')
try:
# connect
connections.add_connection(default={"host": host, "port": 19530})
connections.connect(alias='default')
# create
c_name = cf.gen_unique_str("scale_query")
# c_name = 'scale_query_DymS7kI4'
collection_w = ApiCollectionWrapper()
collection_w.init_collection(name=c_name, schema=cf.gen_default_collection_schema(), shards_num=2)
# insert two segments
for i in range(3):
df = cf.gen_default_dataframe_data(nb)
collection_w.insert(df)
log.debug(collection_w.num_entities)
# create index
collection_w.create_index(ct.default_float_vec_field_name, default_index_params)
assert collection_w.has_index()[0]
assert collection_w.index()[0] == Index(collection_w.collection, ct.default_float_vec_field_name,
default_index_params)
# load
collection_w.load()
# scale queryNode to 5
mic.upgrade(release_name, {'spec.components.queryNode.replicas': 5}, constants.NAMESPACE)
# continuously search
def do_search():
while True:
search_res, _ = collection_w.search(cf.gen_vectors(1, ct.default_dim),
ct.default_float_vec_field_name,
ct.default_search_params, ct.default_limit)
log.debug(search_res[0].ids)
assert len(search_res[0].ids) == ct.default_limit
t_search = threading.Thread(target=do_search, args=(), daemon=True)
t_search.start()
# wait new QN running, continuously insert
mic.wait_for_healthy(release_name, constants.NAMESPACE)
wait_pods_ready(constants.NAMESPACE, f"app.kubernetes.io/instance={release_name}")
def do_insert():
while True:
tmp_df = cf.gen_default_dataframe_data(1000)
collection_w.insert(tmp_df)
t_insert = threading.Thread(target=do_insert, args=(), daemon=True)
t_insert.start()
log.debug(collection_w.num_entities)
time.sleep(20)
log.debug("Expand querynode test finished")
mic.upgrade(release_name, {'spec.components.queryNode.replicas': 3}, constants.NAMESPACE)
mic.wait_for_healthy(release_name, constants.NAMESPACE)
wait_pods_ready(constants.NAMESPACE, f"app.kubernetes.io/instance={release_name}")
log.debug(collection_w.num_entities)
time.sleep(60)
log.debug("Shrink querynode test finished")
except Exception as e:
log.error(str(e))
fail_count += 1
# raise Exception(str(e))
finally:
log.info(f'Test finished with {fail_count} fail request')
assert fail_count <= 1
label = f"app.kubernetes.io/instance={release_name}"
log.info('Start to export milvus pod logs')
read_pod_log(namespace=constants.NAMESPACE, label_selector=label, release_name=release_name)
mic.uninstall(release_name, namespace=constants.NAMESPACE)
def test_expand_index_node(self):
"""
target: test expand indexNode from 1 to 2
method: 1.deploy two indexNode
2.create index with two indexNode
3.expand indexNode from 1 to 2
4.create index with one indexNode
expected: The cost of one indexNode is about twice that of two indexNodes
"""
release_name = "expand-index"
image_tag = get_latest_tag()
image = f'{constants.IMAGE_REPOSITORY}:{image_tag}'
init_replicas = 1
expand_replicas = 2
data_config = {
'metadata.namespace': constants.NAMESPACE,
'spec.mode': 'cluster',
'metadata.name': release_name,
'spec.components.image': image,
'spec.components.proxy.serviceType': 'LoadBalancer',
'spec.components.indexNode.replicas': init_replicas,
'spec.components.dataNode.replicas': 2,
'spec.config.common.retentionDuration': 60
}
mic = MilvusOperator()
mic.install(data_config)
if mic.wait_for_healthy(release_name, constants.NAMESPACE, timeout=1800):
host = mic.endpoint(release_name, constants.NAMESPACE).split(':')[0]
else:
# If deploy failed and want to uninsatll mic
# log.warning(f'Deploy {release_name} timeout and ready to uninstall')
# mic.uninstall(release_name, namespace=constants.NAMESPACE)
raise MilvusException(message=f'Milvus healthy timeout 1800s')
try:
# connect
connections.add_connection(default={"host": host, "port": 19530})
connections.connect(alias='default')
# create collection
c_name = "index_scale_one"
collection_w = ApiCollectionWrapper()
collection_w.init_collection(name=c_name, schema=cf.gen_default_collection_schema())
# insert data
data = cf.gen_default_dataframe_data(nb)
loop = 100
for i in range(loop):
collection_w.insert(data, timeout=60)
assert collection_w.num_entities == nb * loop
# create index
# Note that the num of segments and the num of indexNode are related to indexing time
start = datetime.datetime.now()
collection_w.create_index(ct.default_float_vec_field_name, default_index_params)
assert collection_w.has_index()[0]
t0 = datetime.datetime.now() - start
log.info(f'Create index on {init_replicas} indexNode cost t0: {t0}')
# drop index
collection_w.drop_index()
assert not collection_w.has_index()[0]
# expand indexNode
mic.upgrade(release_name, {'spec.components.indexNode.replicas': expand_replicas}, constants.NAMESPACE)
mic.wait_for_healthy(release_name, constants.NAMESPACE)
wait_pods_ready(constants.NAMESPACE, f"app.kubernetes.io/instance={release_name}")
# create index again
start = datetime.datetime.now()
collection_w.create_index(ct.default_float_vec_field_name, default_index_params)
assert collection_w.has_index()[0]
t1 = datetime.datetime.now() - start
log.info(f'Create index on {expand_replicas} indexNode cost t1: {t1}')
collection_w.drop_index()
start = datetime.datetime.now()
collection_w.create_index(ct.default_float_vec_field_name, default_index_params)
assert collection_w.has_index()[0]
t2 = datetime.datetime.now() - start
log.info(f'Create index on {expand_replicas} indexNode cost t2: {t2}')
log.debug(f't2 is {t2}, t0 is {t0}, t0/t2 is {t0 / t2}')
# assert round(t0 / t2) == 2
except Exception as e:
raise Exception(str(e))
finally:
label = f"app.kubernetes.io/instance={release_name}"
log.info('Start to export milvus pod logs')
read_pod_log(namespace=constants.NAMESPACE, label_selector=label, release_name=release_name)
mic.uninstall(release_name, namespace=constants.NAMESPACE)
def test_scale_proxy(self):
"""
target: test milvus operation after proxy expand
method: 1.deploy 1 proxy replicas
2.milvus e2e test in parallel
3.expand proxy pod from 1 to 5
4.milvus e2e test
5.shrink proxy from 5 to 2
expected: 1.verify data consistent and func work
"""
# deploy milvus cluster with one proxy
fail_count = 0
release_name = "scale-proxy"
image_tag = get_latest_tag()
image = f'{constants.IMAGE_REPOSITORY}:{image_tag}'
data_config = {
'metadata.namespace': constants.NAMESPACE,
'metadata.name': release_name,
'spec.mode': 'cluster',
'spec.components.image': image,
'spec.components.proxy.serviceType': 'LoadBalancer',
'spec.components.proxy.replicas': 1,
'spec.components.dataNode.replicas': 2,
'spec.config.common.retentionDuration': 60
}
mic = MilvusOperator()
mic.install(data_config)
if mic.wait_for_healthy(release_name, constants.NAMESPACE, timeout=1800):
host = mic.endpoint(release_name, constants.NAMESPACE).split(':')[0]
else:
raise MilvusException(message=f'Milvus healthy timeout 1800s')
try:
c_name = cf.gen_unique_str("proxy_scale")
e2e_milvus_parallel(2, host, c_name)
log.info('Milvus test before expand')
# expand proxy replicas from 1 to 5
mic.upgrade(release_name, {'spec.components.proxy.replicas': 5}, constants.NAMESPACE)
mic.wait_for_healthy(release_name, constants.NAMESPACE)
wait_pods_ready(constants.NAMESPACE, f"app.kubernetes.io/instance={release_name}")
e2e_milvus_parallel(5, host, c_name)
log.info('Milvus test after expand')
# expand proxy replicas from 5 to 2
mic.upgrade(release_name, {'spec.components.proxy.replicas': 2}, constants.NAMESPACE)
mic.wait_for_healthy(release_name, constants.NAMESPACE)
wait_pods_ready(constants.NAMESPACE, f"app.kubernetes.io/instance={release_name}")
e2e_milvus_parallel(2, host, c_name)
log.info('Milvus test after shrink')
connections.connect('default', host=host, port=19530)
collection_w = ApiCollectionWrapper()
collection_w.init_collection(name=c_name)
"""
total start 2+5+2 process to run e2e, each time insert default_nb data, But one of the 2 processes started
for the first time did not insert due to collection creation exception. So actually insert eight times
"""
assert collection_w.num_entities == 8 * default_nb
except Exception as e:
log.error(str(e))
fail_count += 1
# raise Exception(str(e))
finally:
log.info(f'Test finished with {fail_count} fail request')
assert fail_count <= 1
label = f"app.kubernetes.io/instance={release_name}"
log.info('Start to export milvus pod logs')
read_pod_log(namespace=constants.NAMESPACE, label_selector=label, release_name=release_name)
mic.uninstall(release_name, namespace=constants.NAMESPACE)
def test_chaos(self, chaos_yaml):
# start the monitor threads to check the milvus ops
log.info("*********************Chaos Test Start**********************")
log.info(connections.get_connection_addr('default'))
cc.start_monitor_threads(self.health_checkers)
# parse chaos object
chaos_config = cc.gen_experiment_config(chaos_yaml)
meta_name = chaos_config.get('metadata', None).get('name', None)
release_name = meta_name
chaos_config_str = json.dumps(chaos_config)
chaos_config_str = chaos_config_str.replace("milvus-chaos", release_name)
chaos_config = json.loads(chaos_config_str)
self._chaos_config = chaos_config # cache the chaos config for tear down
log.info(f"chaos_config: {chaos_config}")
# parse the test expectations in testcases.yaml
if self.parser_testcase_config(chaos_yaml, chaos_config) is False:
log.error("Fail to get the testcase info in testcases.yaml")
assert False
# init report
dir_name = "./reports"
file_name = f"./reports/{meta_name}.log"
if not os.path.exists(dir_name):
os.makedirs(dir_name)
# wait 20s
sleep(constants.WAIT_PER_OP * 2)
# assert statistic:all ops 100% succ
log.info("******1st assert before chaos: ")
assert_statistic(self.health_checkers)
with open(file_name, "a+") as f:
ts = time.strftime("%Y-%m-%d %H:%M:%S")
f.write(f"{meta_name}-{ts}\n")
f.write("1st assert before chaos:\n")
f.write(record_results(self.health_checkers))
# apply chaos object
chaos_res = CusResource(kind=chaos_config['kind'],
group=constants.CHAOS_GROUP,
version=constants.CHAOS_VERSION,
namespace=constants.CHAOS_NAMESPACE)
chaos_res.create(chaos_config)
log.info("chaos injected")
log.info(f"chaos information: {chaos_res.get(meta_name)}")
sleep(constants.WAIT_PER_OP * 2)
# reset counting
cc.reset_counting(self.health_checkers)
# wait 40s
sleep(constants.CHAOS_DURATION)
log.info(f'Alive threads: {threading.enumerate()}')
# assert statistic
log.info("******2nd assert after chaos injected: ")
assert_statistic(self.health_checkers,
expectations={Op.create: self.expect_create,
Op.insert: self.expect_insert,
Op.flush: self.expect_flush,
Op.index: self.expect_index,
Op.search: self.expect_search,
Op.query: self.expect_query
})
with open(file_name, "a+") as f:
f.write("2nd assert after chaos injected:\n")
f.write(record_results(self.health_checkers))
# delete chaos
chaos_res.delete(meta_name)
log.info("chaos deleted")
log.info(f'Alive threads: {threading.enumerate()}')
sleep(2)
# wait all pods ready
log.info(f"wait for pods in namespace {constants.CHAOS_NAMESPACE} with label app.kubernetes.io/instance={meta_name}")
wait_pods_ready(constants.CHAOS_NAMESPACE, f"app.kubernetes.io/instance={meta_name}")
log.info(f"wait for pods in namespace {constants.CHAOS_NAMESPACE} with label release={meta_name}")
wait_pods_ready(constants.CHAOS_NAMESPACE, f"release={meta_name}")
log.info("all pods are ready")
# reconnect if needed
sleep(constants.WAIT_PER_OP * 2)
cc.reconnect(connections, alias='default')
# reset counting again
cc.reset_counting(self.health_checkers)
# wait 50s (varies by feature)
sleep(constants.WAIT_PER_OP * 5)
# assert statistic: all ops success again
log.info("******3rd assert after chaos deleted: ")
assert_statistic(self.health_checkers)
with open(file_name, "a+") as f:
f.write("3rd assert after chaos deleted:\n")
f.write(record_results(self.health_checkers))
# assert all expectations
assert_expectations()
log.info("*********************Chaos Test Completed**********************")
def test_scale_data_node(self):
"""
target: test scale dataNode
method: 1.deploy milvus cluster with 2 dataNode
2.create collection with shards_num=5
3.continuously insert new data (daemon thread)
4.expand dataNode from 2 to 5
5.create new collection with shards_num=2
6.continuously insert new collection new data (daemon thread)
7.shrink dataNode from 5 to 3
expected: Verify milvus remains healthy, Insert and flush successfully during scale
Average dataNode memory usage
"""
release_name = "scale-data"
image_tag = get_latest_tag()
image = f'{constants.IMAGE_REPOSITORY}:{image_tag}'
data_config = {
'metadata.namespace': constants.NAMESPACE,
'spec.mode': 'cluster',
'metadata.name': release_name,
'spec.components.image': image,
'spec.components.proxy.serviceType': 'LoadBalancer',
'spec.components.dataNode.replicas': 2,
'spec.config.common.retentionDuration': 60
}
mic = MilvusOperator()
mic.install(data_config)
if mic.wait_for_healthy(release_name,
constants.NAMESPACE,
timeout=1800):
host = mic.endpoint(release_name,
constants.NAMESPACE).split(':')[0]
else:
raise MilvusException(message=f'Milvus healthy timeout 1800s')
try:
# connect
connections.add_connection(default={"host": host, "port": 19530})
connections.connect(alias='default')
# create
c_name = cf.gen_unique_str("scale_data")
collection_w = ApiCollectionWrapper()
collection_w.init_collection(
name=c_name,
schema=cf.gen_default_collection_schema(),
shards_num=4)
tmp_nb = 10000
@counter
def do_insert():
""" do insert and flush """
insert_res, is_succ = collection_w.insert(
cf.gen_default_dataframe_data(tmp_nb))
log.debug(collection_w.num_entities)
return insert_res, is_succ
def loop_insert():
""" loop do insert """
while True:
do_insert()
threading.Thread(target=loop_insert, args=(), daemon=True).start()
# scale dataNode to 5
mic.upgrade(release_name, {'spec.components.dataNode.replicas': 5},
constants.NAMESPACE)
mic.wait_for_healthy(release_name, constants.NAMESPACE)
wait_pods_ready(constants.NAMESPACE,
f"app.kubernetes.io/instance={release_name}")
log.debug("Expand dataNode test finished")
# create new collection and insert
new_c_name = cf.gen_unique_str("scale_data")
collection_w_new = ApiCollectionWrapper()
collection_w_new.init_collection(
name=new_c_name,
schema=cf.gen_default_collection_schema(),
shards_num=3)
@counter
def do_new_insert():
""" do new insert """
insert_res, is_succ = collection_w_new.insert(
cf.gen_default_dataframe_data(tmp_nb))
log.debug(collection_w_new.num_entities)
return insert_res, is_succ
def loop_new_insert():
""" loop new insert """
while True:
do_new_insert()
threading.Thread(target=loop_new_insert, args=(),
daemon=True).start()
# scale dataNode to 3
mic.upgrade(release_name, {'spec.components.dataNode.replicas': 3},
constants.NAMESPACE)
mic.wait_for_healthy(release_name, constants.NAMESPACE)
wait_pods_ready(constants.NAMESPACE,
f"app.kubernetes.io/instance={release_name}")
log.debug(collection_w.num_entities)
time.sleep(300)
scale_common.check_succ_rate(do_insert)
scale_common.check_succ_rate(do_new_insert)
log.debug("Shrink dataNode test finished")
except Exception as e:
log.error(str(e))
# raise Exception(str(e))
finally:
label = f"app.kubernetes.io/instance={release_name}"
log.info('Start to export milvus pod logs')
read_pod_log(namespace=constants.NAMESPACE,
label_selector=label,
release_name=release_name)
mic.uninstall(release_name, namespace=constants.NAMESPACE)
