def test_auto_load_balance(self):
"""
"""
log.info(f"start to install milvus")
release_name, host, port = install_milvus(
"test-auto-load-balance") # todo add release name
self.release_name = release_name
assert host is not None
conn = connections.connect("default", host=host, port=port)
assert conn is not None
self.health_checkers = {
Op.create: CreateChecker(),
Op.insert: InsertFlushChecker(),
Op.flush: InsertFlushChecker(flush=True),
Op.index: IndexChecker(),
Op.search: SearchChecker(),
Op.query: QueryChecker()
}
cc.start_monitor_threads(self.health_checkers)
# wait
sleep(constants.WAIT_PER_OP * 10)
all_collections = list_collections()
for c in all_collections:
seg_info = utility.get_query_segment_info(c)
seg_distribution = cf.get_segment_distribution(seg_info)
for k in seg_distribution.keys():
log.info(
f"collection {c}'s segment distribution in node {k} is {seg_distribution[k]['sealed']}"
)
# first assert
log.info("first assert")
assert_statistic(self.health_checkers)
# scale up
log.info("scale up milvus")
scale_up_milvus(self.release_name)
# reset counting
cc.reset_counting(self.health_checkers)
sleep(constants.WAIT_PER_OP * 10)
all_collections = list_collections()
for c in all_collections:
seg_info = utility.get_query_segment_info(c)
seg_distribution = cf.get_segment_distribution(seg_info)
for k in seg_distribution.keys():
log.info(
f"collection {c}'s sealed segment distribution in node {k} is {seg_distribution[k]['sealed']}"
)
# second assert
log.info("second assert")
assert_statistic(self.health_checkers)
# TODO assert segment distribution
# assert all expectations
assert_expectations()
def init_health_checkers(self):
checkers = {
Op.create: CreateChecker(),
Op.insert: InsertFlushChecker(),
Op.flush: InsertFlushChecker(flush=True),
Op.index: IndexChecker(),
Op.search: SearchChecker(),
Op.query: QueryChecker()
}
self.health_checkers = checkers
def init_health_checkers(self, collection_name=None):
c_name = collection_name
checkers = {
Op.insert: InsertFlushChecker(collection_name=c_name),
Op.flush: InsertFlushChecker(collection_name=c_name, flush=True),
Op.index: IndexChecker(collection_name=c_name),
Op.search: SearchChecker(collection_name=c_name),
Op.query: QueryChecker(collection_name=c_name),
Op.delete: DeleteChecker(collection_name=c_name),
}
self.health_checkers = checkers
def test_chaos_memory_stress_etcd(self, chaos_yaml):
"""
target: test inject memory stress into all etcd pods
method: 1.Deploy milvus and limit etcd memory resource 1Gi witl all mode
2.Continuously and concurrently do milvus operations
3.Inject memory stress chaos 51024Mi
4.After duration, delete chaos stress
expected: Verify milvus operation succ rate
"""
mic_checkers = {
Op.create: CreateChecker(),
Op.insert: InsertFlushChecker(),
Op.flush: InsertFlushChecker(flush=True),
Op.index: IndexChecker(),
Op.search: SearchChecker(),
Op.query: QueryChecker()
}
# start thread keep running milvus op
start_monitor_threads(mic_checkers)
# parse chaos object
chaos_config = cc.gen_experiment_config(chaos_yaml)
# duration = chaos_config["spec"]["duration"]
meta_name = chaos_config.get('metadata').get('name')
duration = chaos_config.get('spec').get('duration')
# apply chaos object
chaos_res = CusResource(kind=chaos_config['kind'],
group=constants.CHAOS_GROUP,
version=constants.CHAOS_VERSION,
namespace=constants.CHAOS_NAMESPACE)
chaos_res.create(chaos_config)
log.info("Chaos injected")
# convert string duration time to a int number in seconds
if isinstance(duration, str):
duration = duration.replace('h', '*3600+').replace(
'm', '*60+').replace('s', '*1+') + '+0'
else:
log.error("Duration must be string type")
# Delete experiment after it's over
timer = threading.Timer(interval=eval(duration),
function=chaos_res.delete,
args=(meta_name, False))
timer.start()
timer.join()
# output milvus op succ rate
for k, ch in mic_checkers.items():
log.debug(f'Succ rate of {k.value}: {ch.succ_rate()}')
assert ch.succ_rate() == 1.0
def init_health_checkers(self):
c_name = cf.gen_unique_str('MultiReplicasChecker_')
replicas_num = 2
shards_num = 2
checkers = {
Op.insert:
InsertFlushChecker(collection_name=c_name, shards_num=shards_num),
Op.search:
SearchChecker(collection_name=c_name,
shards_num=shards_num,
replica_number=replicas_num),
Op.query:
QueryChecker(collection_name=c_name,
shards_num=shards_num,
replica_number=replicas_num)
}
self.health_checkers = checkers