def queue_worker(queue):
while not queue.empty():
q = queue.get()
suite = q["suite"]
server_host = q["server_host"]
image_type = q["image_type"]
image_tag = q["image_tag"]
with open(suite) as f:
suite_dict = full_load(f)
f.close()
logger.debug(suite_dict)
run_type, run_params = parser.operations_parser(suite_dict)
collections = run_params["collections"]
for collection in collections:
# run tests
server_config = collection["server"]
logger.debug(server_config)
runner = K8sRunner()
if runner.init_env(server_config, server_host, image_type,
image_tag):
logger.debug("Start run tests")
try:
runner.run(run_type, collection)
except Exception as e:
logger.error(str(e))
logger.error(traceback.format_exc())
finally:
runner.clean_up()
else:
logger.error("Runner init failed")
logger.debug("All task finished in queue: %s" % server_host)
def queue_worker(queue):
"""
Using queue to make sure only one test process on each host.
Workers can be run concurrently on different host
"""
while not queue.empty():
q = queue.get()
suite = q["suite"]
server_host = q["server_host"]
deploy_mode = q["deploy_mode"]
image_type = q["image_type"]
image_tag = q["image_tag"]
with open(suite) as f:
suite_dict = full_load(f)
f.close()
logger.debug(suite_dict)
run_type, run_params = parser.operations_parser(suite_dict)
collections = run_params["collections"]
for collection in collections:
# run tests
server_config = collection["server"] if "server" in collection else None
logger.debug(server_config)
runner = K8sRunner()
if runner.init_env(server_config, server_host, deploy_mode, image_type, image_tag):
logger.debug("Start run tests")
try:
runner.run(run_type, collection)
except Exception as e:
logger.error(str(e))
logger.error(traceback.format_exc())
finally:
time.sleep(10)
runner.clean_up()
else:
logger.error("Runner init failed")
if server_host:
logger.debug("All task finished in queue: %s" % server_host)
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--image', help='use the given image')
parser.add_argument('--local',
action='store_true',
help='use local milvus server')
parser.add_argument("--run-count",
default=1,
type=positive_int,
help="run each db operation times")
# performance / stability / accuracy test
parser.add_argument("--run-type",
default="performance",
help="run type, default performance")
parser.add_argument('--suites',
metavar='FILE',
help='load test suites from FILE',
default='suites.yaml')
parser.add_argument('--ip',
help='server ip param for local mode',
default='127.0.0.1')
parser.add_argument('--port',
help='server port param for local mode',
default='19530')
args = parser.parse_args()
operations = None
# Get all benchmark test suites
if args.suites:
with open(args.suites) as f:
suites_dict = load(f)
f.close()
# With definition order
operations = operations_parser(suites_dict, run_type=args.run_type)
# init_env()
run_params = {"run_count": args.run_count}
if args.image:
# for docker mode
if args.local:
logger.error(
"Local mode and docker mode are incompatible arguments")
sys.exit(-1)
# Docker pull image
if not utils.pull_image(args.image):
raise Exception('Image %s pull failed' % image)
# TODO: Check milvus server port is available
logger.info("Init: remove all containers created with image: %s" %
args.image)
utils.remove_all_containers(args.image)
runner = DockerRunner(args.image)
for operation_type in operations:
logger.info("Start run test, test type: %s" % operation_type)
run_params["params"] = operations[operation_type]
runner.run({operation_type: run_params}, run_type=args.run_type)
logger.info("Run params: %s" % str(run_params))
if args.local:
# for local mode
ip = args.ip
port = args.port
runner = LocalRunner(ip, port)
for operation_type in operations:
logger.info("Start run local mode test, test type: %s" %
operation_type)
run_params["params"] = operations[operation_type]
runner.run({operation_type: run_params}, run_type=args.run_type)
logger.info("Run params: %s" % str(run_params))
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("--hostname", default="eros", help="server host name")
parser.add_argument("--image-tag", default="", help="image tag")
parser.add_argument("--image-type", default="", help="image type")
# parser.add_argument(
# "--run-count",
# default=1,
# type=positive_int,
# help="run times for each test")
# # performance / stability / accuracy test
# parser.add_argument(
# "--run-type",
# default="search_performance",
# help="run type, default performance")
parser.add_argument('--suite',
metavar='FILE',
help='load test suite from FILE',
default='suites/suite.yaml')
parser.add_argument('--local',
action='store_true',
help='use local milvus server')
parser.add_argument('--host',
help='server host ip param for local mode',
default='127.0.0.1')
parser.add_argument('--port',
help='server port param for local mode',
default='19530')
args = parser.parse_args()
# Get all benchmark test suites
if args.suite:
with open(args.suite) as f:
suite_dict = full_load(f)
f.close()
# With definition order
run_type, run_params = operations_parser(suite_dict)
# init_env()
# run_params = {"run_count": args.run_count}
if args.image_tag:
namespace = "milvus"
logger.debug(args)
# for docker mode
if args.local:
logger.error("Local mode and docker mode are incompatible")
sys.exit(-1)
# Docker pull image
# if not utils.pull_image(args.image):
# raise Exception('Image %s pull failed' % image)
# TODO: Check milvus server port is available
# logger.info("Init: remove all containers created with image: %s" % args.image)
# utils.remove_all_containers(args.image)
# runner = DockerRunner(args)
tables = run_params["tables"]
for table in tables:
# run tests
server_config = table["server"]
logger.debug(server_config)
runner = K8sRunner()
if runner.init_env(server_config, args):
logger.debug("Start run tests")
try:
runner.run(run_type, table)
except Exception as e:
logger.error(str(e))
logger.error(traceback.format_exc())
finally:
runner.clean_up()
else:
logger.error("Runner init failed")
# for operation_type in operations:
# logger.info("Start run test, test type: %s" % operation_type)
# run_params["params"] = operations[operation_type]
# runner.run({operation_type: run_params}, run_type=args.run_type)
# logger.info("Run params: %s" % str(run_params))
if args.local:
# for local mode
host = args.host
port = args.port
runner = LocalRunner(host, port)
for operation_type in operations:
logger.info("Start run local mode test, test type: %s" %
operation_type)
run_params["params"] = operations[operation_type]
runner.run({operation_type: run_params}, run_type=args.run_type)
logger.info("Run params: %s" % str(run_params))
def main():
arg_parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
# helm mode with scheduler
arg_parser.add_argument(
"--image-version",
default="",
help="image version")
arg_parser.add_argument(
"--schedule-conf",
metavar='FILE',
default='',
help="load test schedule from FILE")
arg_parser.add_argument(
"--deploy-mode",
default='',
help="single or shards")
# local mode
arg_parser.add_argument(
'--local',
action='store_true',
help='use local milvus server')
arg_parser.add_argument(
'--host',
help='server host ip param for local mode',
default='127.0.0.1')
arg_parser.add_argument(
'--port',
help='server port param for local mode',
default='19530')
arg_parser.add_argument(
'--suite',
metavar='FILE',
help='load test suite from FILE',
default='')
args = arg_parser.parse_args()
if args.schedule_conf:
if args.local:
raise Exception("Helm mode with scheduler and other mode are incompatible")
if not args.image_version:
raise Exception("Image version not given")
image_version = args.image_version
deploy_mode = args.deploy_mode
with open(args.schedule_conf) as f:
schedule_config = full_load(f)
f.close()
queues = []
# server_names = set()
server_names = []
for item in schedule_config:
server_host = item["server"] if "server" in item else ""
suite_params = item["suite_params"]
server_names.append(server_host)
q = Queue()
for suite_param in suite_params:
suite = "suites/"+suite_param["suite"]
image_type = suite_param["image_type"]
image_tag = get_image_tag(image_version, image_type)
q.put({
"suite": suite,
"server_host": server_host,
"deploy_mode": deploy_mode,
"image_tag": image_tag,
"image_type": image_type
})
queues.append(q)
logging.error(queues)
thread_num = len(server_names)
processes = []
for i in range(thread_num):
x = Process(target=queue_worker, args=(queues[i], ))
processes.append(x)
x.start()
time.sleep(10)
for x in processes:
x.join()
elif args.local:
# for local mode
host = args.host
port = args.port
suite = args.suite
with open(suite) as f:
suite_dict = full_load(f)
f.close()
logger.debug(suite_dict)
run_type, run_params = parser.operations_parser(suite_dict)
collections = run_params["collections"]
if len(collections) > 1:
raise Exception("Multi collections not supported in Local Mode")
collection = collections[0]
runner = LocalRunner(host, port)
logger.info("Start run local mode test, test type: %s" % run_type)
runner.run(run_type, collection)
