class CartCoRpcOneNodeTest(Test):
"""
Runs basic CaRT CoRPC tests
:avocado: tags=all,corpc,one_node
"""
def setUp(self):
""" Test setup """
print("Running setup\n")
self.utils = CartUtils()
self.env = self.utils.get_env(self)
def tearDown(self):
""" Test teardown """
print("Run TearDown\n")
def test_cart_corpc(self):
"""
Test CaRT CoRPC
:avocado: tags=all,corpc,one_node
"""
cmd = self.utils.build_cmd(self, self.env, "srv")
print("\nTest cmd : %s\n" % cmd)
self.utils.launch_test(self, cmd)
class CartRpcOneNodeTest(Test):
"""
Runs basic CaRT RPC tests
:avocado: tags=all,rpc,one_node
"""
def setUp(self):
""" Test setup """
print("Running setup\n")
self.utils = CartUtils()
self.env = self.utils.get_env(self)
def tearDown(self):
""" Test tear down """
print("Run TearDown\n")
def test_cart_rpc(self):
"""
Test CaRT RPC
:avocado: tags=all,rpc,one_node
"""
urifile = self.utils.create_uri_file()
srvcmd = self.utils.build_cmd(self, self.env, "srv", True, urifile)
clicmd = self.utils.build_cmd(self, self.env, "cli", False, urifile)
self.utils.launch_srv_cli_test(self, srvcmd, clicmd)
self.utils.log_check(self)
class CartRpcOneNodeTest(Test):
"""
Runs basic CaRT RPC tests
:avocado: tags=all,cart,pr,rpc,one_node
"""
def setUp(self):
""" Test setup """
print("Running setup\n")
self.utils = CartUtils()
self.env = self.utils.get_env(self)
def tearDown(self):
""" Test tear down """
print("Run TearDown\n")
def test_cart_rpc(self):
"""
Test CaRT RPC
:avocado: tags=all,cart,pr,rpc,one_node
"""
srvcmd = self.utils.build_cmd(self, self.env, "test_servers")
clicmd = self.utils.build_cmd(self, self.env, "test_clients")
self.utils.launch_srv_cli_test(self, srvcmd, clicmd)
self.utils.log_check(self)
class GroupTest(Test):
"""
Runs GroupTests for primary and secondary resizeable groups
:avocado: tags=all,group_test,one_node,no_pmix
"""
def setUp(self):
""" Test setup """
print("Running setup\n")
self.utils = CartUtils()
self.env = self.utils.get_env(self)
def tearDown(self):
""" Test tear down """
print("Run TearDown\n")
def test_group(self):
"""
Test CaRT NoPmix Launcher
:avocado: tags=all,group_test,one_node
"""
log_mask = self.params.get("D_LOG_MASK", "/run/defaultENV/")
crt_phy_addr = self.params.get("CRT_PHY_ADDR_STR", "/run/defaultENV/")
ofi_interface = self.params.get("OFI_INTERFACE", "/run/defaultENV/")
srv_cmd = self.utils.build_cmd(self, self.env, "srv")
cmd = srv_cmd
self.utils.launch_test(self, cmd)
class GroupTest(Test):
"""
Runs GroupTests for primary and secondary resizeable groups
:avocado: tags=all,cart,pr,group_test,one_node,no_pmix
"""
def setUp(self):
""" Test setup """
print("Running setup\n")
self.utils = CartUtils()
self.env = self.utils.get_env(self)
def tearDown(self):
""" Test tear down """
print("Run TearDown\n")
def test_group(self):
"""
Test CaRT NoPmix Launcher
:avocado: tags=all,cart,pr,group_test,one_node
"""
srv_cmd = self.utils.build_cmd(self, self.env, "test_servers")
cmd = srv_cmd
self.utils.launch_test(self, cmd)
class CartBarrierFiveNodeTest(Test):
"""
Runs basic CaRT barrier tests
:avocado: tags=all,barrier,five_node
"""
def setUp(self):
""" Test setup """
print("Running setup\n")
self.utils = CartUtils()
self.env = self.utils.get_env(self)
def tearDown(self):
""" Test tear down """
print("Run TearDown\n")
def test_cart_barrier(self):
"""
Test CaRT barrier
:avocado: tags=all,barrier,five_node
"""
cmd = self.utils.build_cmd(self, self.env, "srv")
print("\nTest cmd : %s\n" % cmd)
self.utils.launch_test(self, cmd)
class CartCoRpcTwoNodeTest(Test):
"""
Runs basic CaRT CoRPC tests
:avocado: tags=all,cart,pr,corpc,two_node
"""
def setUp(self):
""" Test setup """
print("Running setup\n")
self.utils = CartUtils()
self.env = self.utils.get_env(self)
def tearDown(self):
""" Test tear down """
print("Run TearDown\n")
def test_cart_corpc(self):
"""
Test CaRT CoRPC
:avocado: tags=all,cart,pr,corpc,two_node
"""
cmd = self.utils.build_cmd(self, self.env, "test_servers")
self.utils.launch_test(self, cmd)
class CartRpcOneNodeTest(TestWithoutServers):
"""
Runs basic CaRT RPC tests
:avocado: recursive
"""
def setUp(self):
""" Test setup """
print("Running setup\n")
self.utils = CartUtils()
self.env = self.utils.get_env(self)
def tearDown(self):
""" Tear down """
self.report_timeout()
self._teardown_errors.extend(self.utils.cleanup_processes())
super(CartRpcOneNodeTest, self).tearDown()
def test_cart_rpc(self):
"""
Test CaRT RPC
:avocado: tags=all,cart,pr,daily_regression,rpc,one_node
"""
srvcmd = self.utils.build_cmd(self, self.env, "test_servers")
clicmd = self.utils.build_cmd(self, self.env, "test_clients")
self.utils.launch_srv_cli_test(self, srvcmd, clicmd)
self.utils.log_check(self)
class CartCoRpcOneNodeTest(Test):
"""
Runs CaRT ghost rank RPC test
#:avocado: tags=all,cart,pr,ghost_rank_rpc,one_node
"""
def setUp(self):
""" Test setup """
print("Running setup\n")
self.utils = CartUtils()
self.env = self.utils.get_env(self)
def tearDown(self):
""" Test teardown """
print("Run TearDown\n")
def test_cart_ghost_rank_rpc(self):
"""
Test ghost rank RPC
#:avocado: tags=all,cart,pr,ghost_rank_rpc,one_node
"""
cmd = self.utils.build_cmd(self, self.env, "test_servers")
self.utils.launch_test(self, cmd)
class CartCoRpcOneNodeTest(TestWithoutServers):
"""
Runs CaRT ghost rank RPC test
:avocado: recursive
"""
def setUp(self):
""" Test setup """
print("Running setup\n")
self.utils = CartUtils()
self.env = self.utils.get_env(self)
def tearDown(self):
""" Tear down """
self.report_timeout()
self._teardown_errors.extend(self.utils.cleanup_processes())
super(CartCoRpcOneNodeTest, self).tearDown()
def test_cart_ghost_rank_rpc(self):
"""
Test ghost rank RPC
#:avocado: tags=all,cart,pr,daily_regression,ghost_rank_rpc,one_node
"""
cmd = self.utils.build_cmd(self, self.env, "test_servers")
self.utils.launch_test(self, cmd)
class CartCoRpcFiveNodeTest(TestWithoutServers):
"""
Runs basic CaRT CoRPC tests
:avocado: recursive
"""
def setUp(self):
""" Test setup """
print("Running setup\n")
self.utils = CartUtils()
self.env = self.utils.get_env(self)
def tearDown(self):
""" Tear down """
super(CartCoRpcFiveNodeTest, self).tearDown()
self.utils.cleanup_processes()
def test_cart_corpc(self):
"""
Test CaRT CoRPC
:avocado: tags=all,cart,pr,daily_regression,corpc,five_node
"""
cmd = self.utils.build_cmd(self, self.env, "test_servers")
self.utils.launch_test(self, cmd)
class GroupTest(TestWithoutServers):
"""
Runs GroupTests for primary and secondary resizeable groups
:avocado: recursive
"""
def setUp(self):
""" Test setup """
print("Running setup\n")
self.utils = CartUtils()
self.env = self.utils.get_env(self)
def tearDown(self):
""" Tear down """
super(GroupTest, self).tearDown()
self.utils.cleanup_processes()
def test_group(self):
"""
Test CaRT NoPmix Launcher
:avocado: tags=all,cart,pr,daily_regression,group_test,one_node
"""
srv_cmd = self.utils.build_cmd(self, self.env, "test_servers")
cmd = srv_cmd
self.utils.launch_test(self, cmd)
class CartCtlOneNodeTest(Test):
"""
Runs basic CaRT ctl tests
:avocado: tags=all,ctl,one_node
"""
def setUp(self):
""" Test setup """
print("Running setup\n")
self.utils = CartUtils()
self.env = self.utils.get_env(self)
def tearDown(self):
""" Test tear down """
print("Run TearDown\n")
def test_cart_ctl(self):
"""
Test CaRT ctl
:avocado: tags=all,ctl,one_node
"""
urifile = self.utils.create_uri_file()
srvcmd = self.utils.build_cmd(self, self.env, "srv", True, urifile)
print("\nServer cmd : %s\n" % srvcmd)
try:
srv_rtn = self.utils.launch_cmd_bg(self, srvcmd)
except Exception as e:
print("Exception in launching server : {}".format(e))
self.fail("Test failed.\n")
# Verify the server is still running.
if not self.utils.check_process(srv_rtn):
procrtn = self.utils.stop_process(srv_rtn)
self.fail("Server did not launch, return code %s" \
% procrtn)
clicmd = self.utils.build_cmd(self, self.env, "cli1", False, urifile)
print("\nClient cmd : %s\n" % clicmd)
self.utils.launch_test(self, clicmd, srv_rtn)
clicmd = self.utils.build_cmd(self, self.env, "cli2", False, urifile)
print("\nClient cmd : %s\n" % clicmd)
self.utils.launch_test(self, clicmd, srv_rtn)
clicmd = self.utils.build_cmd(self, self.env, "cli3", False, urifile)
print("\nClient cmd : %s\n" % clicmd)
self.utils.launch_test(self, clicmd, srv_rtn)
clicmd = self.utils.build_cmd(self, self.env, "cli4", False, urifile)
print("\nClient cmd : %s\n" % clicmd)
self.utils.launch_test(self, clicmd, srv_rtn)
clicmd = self.utils.build_cmd(self, self.env, "cli5", False, urifile)
print("\nClient cmd : %s\n" % clicmd)
self.utils.launch_test(self, clicmd, srv_rtn)
clicmd = self.utils.build_cmd(self, self.env, "cli6", False, urifile)
print("\nClient cmd : %s\n" % clicmd)
self.utils.launch_test(self, clicmd, srv_rtn)
class CartCtlOneNodeTest(Test):
"""
Runs basic CaRT ctl tests
:avocado: tags=all,cart,pr,ctl,one_node
"""
def setUp(self):
""" Test setup """
print("Running setup\n")
self.utils = CartUtils()
self.env = self.utils.get_env(self)
def tearDown(self):
""" Test tear down """
print("Run TearDown\n")
@skipForTicket("DAOS-5547")
def test_cart_ctl(self):
"""
Test CaRT ctl
:avocado: tags=all,cart,pr,ctl,one_node
"""
srvcmd = self.utils.build_cmd(self, self.env, "test_servers")
try:
srv_rtn = self.utils.launch_cmd_bg(self, srvcmd)
# pylint: disable=broad-except
except Exception as e:
self.utils.print("Exception in launching server : {}".format(e))
self.fail("Test failed.\n")
# Verify the server is still running.
if not self.utils.check_process(srv_rtn):
procrtn = self.utils.stop_process(srv_rtn)
self.fail("Server did not launch, return code %s" \
% procrtn)
clicmd = self.utils.build_cmd(self, self.env, "test_clients_1")
self.utils.launch_test(self, clicmd, srv_rtn)
clicmd = self.utils.build_cmd(self, self.env, "test_clients_2")
self.utils.launch_test(self, clicmd, srv_rtn)
clicmd = self.utils.build_cmd(self, self.env, "test_clients_3")
self.utils.launch_test(self, clicmd, srv_rtn)
clicmd = self.utils.build_cmd(self, self.env, "test_clients_4")
self.utils.launch_test(self, clicmd, srv_rtn)
clicmd = self.utils.build_cmd(self, self.env, "test_clients_5")
self.utils.launch_test(self, clicmd, srv_rtn)
clicmd = self.utils.build_cmd(self, self.env, "test_clients_6")
self.utils.launch_test(self, clicmd, srv_rtn)
class CartNoPmixOneNodeTest(Test):
"""
Runs basic CaRT no_pmix tests
:avocado: tags=all,no_pmix,one_node
"""
def setUp(self):
""" Test setup """
print("Running setup\n")
self.utils = CartUtils()
self.env = self.utils.get_env(self)
crt_phy_addr = self.params.get("CRT_PHY_ADDR_STR", '/run/defaultENV/')
ofi_interface = self.params.get("OFI_INTERFACE", '/run/defaultENV/')
ofi_ctx_num = self.params.get("CRT_CTX_NUM", '/run/defaultENV/')
ofi_share_addr = self.params.get("CRT_CTX_SHARE_ADDR",
'/run/defaultENV/')
self.pass_env = {
"CRT_PHY_ADDR_STR": crt_phy_addr,
"OFI_INTERFACE": ofi_interface,
"CRT_CTX_SHARE_ADDR": ofi_share_addr,
"CRT_CTX_NUM": ofi_ctx_num
}
def tearDown(self):
""" Test tear down """
print("Run TearDown\n")
def test_cart_no_pmix(self):
"""
Test CaRT NoPmix
:avocado: tags=all,no_pmix,one_node
"""
cmd = self.params.get("tst_bin", '/run/tests/*/')
self.utils.print("\nTest cmd : %s\n" % cmd)
test_env = self.pass_env
p = subprocess.Popen([cmd], env=test_env, stdout=subprocess.PIPE)
rc = self.utils.wait_process(p, 10)
if rc != 0:
self.utils.print(
"Error waiting for process. returning {}".format(rc))
self.fail("Test failed.\n")
self.utils.print("Finished waiting for {}".format(p))
class CartNoPmixOneNodeTest(TestWithoutServers):
"""
Runs basic CaRT no_pmix tests
:avocado: recursive
"""
def setUp(self):
""" Test setup """
print("Running setup\n")
self.utils = CartUtils()
self.env = self.utils.get_env(self)
crt_phy_addr = self.params.get("CRT_PHY_ADDR_STR", '/run/defaultENV/')
ofi_interface = self.params.get("OFI_INTERFACE", '/run/defaultENV/')
ofi_ctx_num = self.params.get("CRT_CTX_NUM", '/run/defaultENV/')
ofi_share_addr = self.params.get("CRT_CTX_SHARE_ADDR",
'/run/defaultENV/')
self.pass_env = {
"CRT_PHY_ADDR_STR": crt_phy_addr,
"OFI_INTERFACE": ofi_interface,
"CRT_CTX_SHARE_ADDR": ofi_share_addr,
"CRT_CTX_NUM": ofi_ctx_num
}
def tearDown(self):
""" Tear down """
self.report_timeout()
self._teardown_errors.extend(self.utils.cleanup_processes())
super(CartNoPmixOneNodeTest, self).tearDown()
def test_cart_no_pmix(self):
"""
Test CaRT NoPmix
:avocado: tags=all,cart,pr,daily_regression,no_pmix,one_node
"""
cmd = self.params.get("tst_bin", '/run/tests/*/')
self.utils.print("\nTest cmd : %s\n" % cmd)
test_env = self.pass_env
p = subprocess.Popen([cmd], env=test_env, stdout=subprocess.PIPE)
rc = self.utils.wait_process(p, 30)
if rc != 0:
self.utils.print("Error waiting for process.")
self.utils.print("returning {}".format(rc))
self.fail("Test failed.\n")
self.utils.print("Finished waiting for {}".format(p))
class CartCtlFiveNodeTest(TestWithoutServers):
"""
Runs basic CaRT ctl tests
:avocado: recursive
"""
def setUp(self):
""" Test setup """
print("Running setup\n")
self.utils = CartUtils()
self.env = self.utils.get_env(self)
def tearDown(self):
""" Tear down """
super(CartCtlFiveNodeTest, self).tearDown()
self.utils.cleanup_processes()
def test_cart_ctl(self):
"""
Test CaRT ctl
:avocado: tags=all,cart,pr,daily_regression,ctl,five_node
"""
srvcmd = self.utils.build_cmd(self, self.env, "test_servers")
try:
srv_rtn = self.utils.launch_cmd_bg(self, srvcmd)
# pylint: disable=broad-except
except Exception as e:
self.utils.print("Exception in launching server : {}".format(e))
self.fail("Test failed.\n")
# Verify the server is still running.
if not self.utils.check_process(srv_rtn):
procrtn = self.utils.stop_process(srv_rtn)
self.fail("Server did not launch, return code %s" \
% procrtn)
time.sleep(5)
for index in range(2):
clicmd = self.utils.build_cmd(self,
self.env,
"test_clients",
index=index)
self.utils.launch_test(self, clicmd, srv_rtn)
self.utils.stop_process(srv_rtn)
class CartSelfThreeNodeTest(TestWithoutServers):
"""
Runs basic CaRT self test
:avocado: recursive
"""
def setUp(self):
""" Test setup """
print("Running setup\n")
self.utils = CartUtils()
self.env = self.utils.get_env(self)
def tearDown(self):
""" Tear down """
self.report_timeout()
self._teardown_errors.extend(self.utils.cleanup_processes())
super(CartSelfThreeNodeTest, self).tearDown()
def test_cart_selftest(self):
"""
Test CaRT Self Test
:avocado: tags=all,cart,pr,daily_regression,selftest,three_node
"""
srvcmd = self.utils.build_cmd(self, self.env, "test_servers")
try:
srv_rtn = self.utils.launch_cmd_bg(self, srvcmd)
# pylint: disable=broad-except
except Exception as e:
self.utils.print("Exception in launching server : {}".format(e))
self.fail("Test failed.\n")
# Verify the server is still running.
if not self.utils.check_process(srv_rtn):
procrtn = self.utils.stop_process(srv_rtn)
self.fail("Server did not launch, return code %s" \
% procrtn)
for index in range(3):
clicmd = self.utils.build_cmd(
self, self.env, "test_clients", index=index)
self.utils.launch_test(self, clicmd, srv_rtn)
# Give few seconds for servers to fully shut down before exiting
# from this test.
if not self.utils.wait_process(srv_rtn, 5):
self.utils.stop_process(srv_rtn)
class CartCtlFiveNodeTest(Test):
"""
Runs basic CaRT ctl tests
:avocado: tags=all,ctl,five_node
"""
def setUp(self):
""" Test setup """
print("Running setup\n")
self.utils = CartUtils()
self.env = self.utils.get_env(self)
def tearDown(self):
""" Test tear down """
print("Run TearDown\n")
def test_cart_ctl(self):
"""
Test CaRT ctl
:avocado: tags=all,ctl,five_node
"""
srvcmd = self.utils.build_cmd(self, self.env, "srv")
try:
srv_rtn = self.utils.launch_cmd_bg(self, srvcmd)
except Exception as e:
self.utils.print("Exception in launching server : {}".format(e))
self.fail("Test failed.\n")
# Verify the server is still running.
if not self.utils.check_process(srv_rtn):
procrtn = self.utils.stop_process(srv_rtn)
self.fail("Server did not launch, return code %s" \
% procrtn)
time.sleep(5)
clicmd = self.utils.build_cmd(self, self.env, "cli1")
self.utils.launch_test(self, clicmd, srv_rtn)
clicmd = self.utils.build_cmd(self, self.env, "cli2")
self.utils.launch_test(self, clicmd, srv_rtn)
self.utils.stop_process(srv_rtn)
class CartRpcOneNodeSwimNotificationOnRankEvictionTest(TestWithoutServers):
"""
Runs basic CaRT RPC tests
:avocado: recursive
"""
def setUp(self):
""" Test setup """
print("Running setup\n")
self.utils = CartUtils()
self.env = self.utils.get_env(self)
def tearDown(self):
"""Tear down."""
print("tearDown() start")
super(CartRpcOneNodeSwimNotificationOnRankEvictionTest,
self).tearDown()
def test_cart_rpc(self):
"""
Test CaRT RPC
:avocado: tags=all,cart,pr,rpc,one_node,swim_rank_eviction
"""
srvcmd = self.utils.build_cmd(self, self.env, "test_servers")
try:
srv_rtn = self.utils.launch_cmd_bg(self, srvcmd)
# pylint: disable=broad-except
except Exception as e:
self.utils.print("Exception in launching server : {}".format(e))
self.fail("Test failed.\n")
# Verify the server is still running.
if not self.utils.check_process(srv_rtn):
procrtn = self.utils.stop_process(srv_rtn)
self.fail("Server did not launch, return code %s" \
% procrtn)
for index in range(6):
clicmd = self.utils.build_cmd(self,
self.env,
"test_clients",
index=index)
self.utils.launch_test(self, clicmd, srv_rtn)
class CartSelfThreeNodeTest(Test):
"""
Runs basic CaRT self test
:avocado: tags=all,cart,pr,selftest,three_node
"""
def setUp(self):
""" Test setup """
print("Running setup\n")
self.utils = CartUtils()
self.env = self.utils.get_env(self)
def tearDown(self):
""" Test tear down """
print("Run TearDown\n")
def test_cart_selftest(self):
"""
Test CaRT Self Test
:avocado: tags=all,cart,pr,selftest,three_node
"""
srvcmd = self.utils.build_cmd(self, self.env, "test_servers")
try:
srv_rtn = self.utils.launch_cmd_bg(self, srvcmd)
# pylint: disable=broad-except
except Exception as e:
self.utils.print("Exception in launching server : {}".format(e))
self.fail("Test failed.\n")
# Verify the server is still running.
if not self.utils.check_process(srv_rtn):
procrtn = self.utils.stop_process(srv_rtn)
self.fail("Server did not launch, return code %s" \
% procrtn)
for index in range(3):
clicmd = self.utils.build_cmd(
self, self.env, "test_clients", index=index)
self.utils.launch_test(self, clicmd, srv_rtn)
class CartNoPmixLauncherOneNodeTest(TestWithoutServers):
"""
Runs basic CaRT no_pmix_launcher tests
:avocado: recursive
"""
def setUp(self):
""" Test setup """
print("Running setup\n")
self.utils = CartUtils()
self.env = self.utils.get_env(self)
def tearDown(self):
""" Tear down """
self.report_timeout()
self._teardown_errors.extend(self.utils.cleanup_processes())
super(CartNoPmixLauncherOneNodeTest, self).tearDown()
def test_cart_no_pmix_launcher(self):
"""
Test CaRT NoPmix Launcher
:avocado: tags=all,cart,pr,daily_regression,no_pmix_launcher,one_node
"""
cli_bin = self.params.get("test_clients_bin", '/run/tests/*/')
cli_arg = self.params.get("test_clients_arg", '/run/tests/*/')
cli_ppn = self.params.get("test_clients_ppn", '/run/tests/*/')
log_mask = self.params.get("D_LOG_MASK", "/run/defaultENV/")
crt_phy_addr = self.params.get("CRT_PHY_ADDR_STR", "/run/defaultENV/")
ofi_interface = self.params.get("OFI_INTERFACE", "/run/defaultENV/")
srv_cmd = self.utils.build_cmd(self, self.env, "test_servers")
cmd = srv_cmd + " : -np {}".format(cli_ppn)
cmd += " -x CRT_PHY_ADDR_STR={}".format(crt_phy_addr)
cmd += " -x OFI_INTERFACE={}".format(ofi_interface)
cmd += " -x D_LOG_MASK={}".format(log_mask)
cmd += " {}".format(cli_bin)
cmd += " {}".format(cli_arg)
self.utils.launch_test(self, cmd)
class CartNoPmixLauncherOneNodeTest(Test):
"""
Runs basic CaRT no_pmix_launcher tests
:avocado: tags=all,no_pmix_launcher,one_node
"""
def setUp(self):
""" Test setup """
print("Running setup\n")
self.utils = CartUtils()
self.env = self.utils.get_env(self)
def tearDown(self):
""" Test tear down """
print("Run TearDown\n")
def test_cart_no_pmix_launcher(self):
"""
Test CaRT NoPmix Launcher
:avocado: tags=all,no_pmix_launcher,one_node
"""
cli_bin = self.params.get("cli_bin", '/run/tests/*/')
cli_arg = self.params.get("cli_arg", '/run/tests/*/')
cli_ppn = self.params.get("cli_ppn", '/run/tests/*/')
log_mask = self.params.get("D_LOG_MASK", "/run/defaultENV/")
crt_phy_addr = self.params.get("CRT_PHY_ADDR_STR",
"/run/defaultENV/")
ofi_interface = self.params.get("OFI_INTERFACE", "/run/defaultENV/")
srv_cmd = self.utils.build_cmd(self, self.env, "srv")
cmd = srv_cmd + " : -np {}".format(cli_ppn)
cmd += " -x CRT_PHY_ADDR_STR={}".format(crt_phy_addr)
cmd += " -x OFI_INTERFACE={}".format(ofi_interface)
cmd += " -x D_LOG_MASK={}".format(log_mask)
cmd += " {}".format(cli_bin)
cmd += " {}".format(cli_arg)
print("\nTest cmd : %s\n" % cmd)
self.utils.launch_test(self, cmd)
class CartSelfTwoNodeTest(Test):
"""
Runs basic CaRT self test
:avocado: tags=all,selftest,two_node
"""
def setUp(self):
""" Test setup """
print("Running setup\n")
self.utils = CartUtils()
self.env = self.utils.get_env(self)
def tearDown(self):
""" Test tear down """
print("Run TearDown\n")
def test_cart_selftest(self):
"""
Test CaRT Self Test
:avocado: tags=all,selftest,two_node
"""
urifile = self.utils.create_uri_file()
srvcmd = self.utils.build_cmd(self, self.env, "srv", True, urifile)
clicmd = self.utils.build_cmd(self, self.env, "cli", False, urifile)
print("\nServer cmd : %s\n" % srvcmd)
print("\nClient cmd : %s\n" % clicmd)
srv_rtn = self.utils.launch_cmd_bg(self, srvcmd)
# Verify the server is still running.
if not self.utils.check_process(srv_rtn):
procrtn = self.utils.stop_process(srv_rtn)
self.fail("Server did not launch, return code %s" % procrtn)
self.utils.launch_test(self, clicmd, srv_rtn)
self.utils.stop_process(srv_rtn)
class CartCoRpcOneNodeTest(TestWithoutServers):
"""
Runs CaRT ghost rank RPC test
:avocado: recursive
"""
def setUp(self):
""" Test setup """
print("Running setup\n")
self.utils = CartUtils()
self.env = self.utils.get_env(self)
def test_cart_ghost_rank_rpc(self):
"""
Test ghost rank RPC
#:avocado: tags=all,cart,pr,daily_regression,ghost_rank_rpc,one_node
"""
cmd = self.utils.build_cmd(self, self.env, "test_servers")
self.utils.launch_test(self, cmd)
class CartCoRpcTwoNodeTest(TestWithoutServers):
"""
Runs basic CaRT CoRPC tests
:avocado: recursive
"""
def setUp(self):
""" Test setup """
print("Running setup\n")
self.utils = CartUtils()
self.env = self.utils.get_env(self)
def test_cart_corpc(self):
"""
Test CaRT CoRPC
:avocado: tags=all,cart,pr,daily_regression,corpc,two_node
"""
cmd = self.utils.build_cmd(self, self.env, "test_servers")
self.utils.launch_test(self, cmd)
class CartRpcTwoNodeTest(TestWithoutServers):
"""
Runs basic CaRT RPC tests
:avocado: recursive
"""
def setUp(self):
""" Test setup """
print("Running setup\n")
self.utils = CartUtils()
self.env = self.utils.get_env(self)
def test_cart_rpc(self):
"""
Test CaRT RPC
:avocado: tags=all,cart,pr,daily_regression,rpc,two_node
"""
srvcmd = self.utils.build_cmd(self, self.env, "test_servers")
clicmd = self.utils.build_cmd(self, self.env, "test_clients")
self.utils.launch_srv_cli_test(self, srvcmd, clicmd)
self.utils.log_check(self)
class CartGroupTiersThreeNodeTest(Test):
"""
Runs basic CaRT group tier tests
:avocado: tags=all,group_tiers,three_node
"""
def setUp(self):
""" Test setup """
print("Running setup\n")
self.utils = CartUtils()
self.env = self.utils.get_env(self)
def tearDown(self):
""" Test tear down """
print("Run TearDown\n")
def test_group_tier(self):
"""
Test CaRT group_tier
:avocado: tags=all,multi_tier,group_tiers,three_node
"""
urifile1 = self.utils.create_uri_file()
srvcmd = self.utils.build_cmd(self, self.env, "srv1", True, urifile1)
urifile2 = self.utils.create_uri_file()
srv2cmd = self.utils.build_cmd(self, self.env, "srv2", True, urifile2)
try:
srv2_rtn = self.utils.launch_cmd_bg(self, srv2cmd)
except Exception as e:
self.utils.print("Exception in launching server : {}".format(e))
self.fail("Test failed.\n")
time.sleep(8)
try:
srv_rtn = self.utils.launch_cmd_bg(self, srvcmd)
except Exception as e:
self.utils.print("Exception in launching server : {}".format(e))
self.fail("Test failed.\n")
time.sleep(4)
clicmd = self.utils.build_cmd(self, self.env, "cli1", False, urifile2)
self.utils.launch_test(self, clicmd, srv_rtn, srv2_rtn)
clicmd = self.utils.build_cmd(self, self.env, "cli2", False, urifile1)
self.utils.launch_test(self, clicmd, srv_rtn, srv2_rtn)
# Stop the server
self.utils.print("Stopping server process 2 {}".format(srv2_rtn))
procrtn2 = self.utils.stop_process(srv2_rtn)
self.utils.print("Stopping server process 1 {}".format(srv_rtn))
procrtn1 = self.utils.stop_process(srv_rtn)
if procrtn2 or procrtn1:
self.fail("Test failed. \
server 1 ret code {} \
server 2 ret code {}".format(procrtn1, procrtn2))
class CartIvTwoNodeTest(TestWithoutServers):
"""
Runs basic CaRT tests on one-node
:avocado: recursive
"""
def setUp(self):
""" Test setup """
print("Running setup\n")
self.utils = CartUtils()
self.env = self.utils.get_env(self)
def _verify_action(self, action):
"""verify the action"""
if (('operation' not in action) or
('rank' not in action) or
('key' not in action)):
self.utils.print("Error happened during action check")
raise ValueError("Each action must contain an operation," \
" rank, and key")
if len(action['key']) != 2:
self.utils.print("Error key should be tuple of (rank, idx)")
raise ValueError("key should be a tuple of (rank, idx)")
def _verify_fetch_operation(self, action):
"""verify fetch operation"""
if (('return_code' not in action) or
('expected_value' not in action)):
self.utils.print("Error: fetch operation was malformed")
raise ValueError("Fetch operation malformed")
def _iv_test_actions(self, cmd, actions):
#pylint: disable=too-many-locals
"""Go through each action and perform the test"""
for action in actions:
clicmd = cmd
command = 'tests/iv_client'
self._verify_action(action)
operation = action['operation']
rank = int(action['rank'])
key_rank = int(action['key'][0])
key_idx = int(action['key'][1])
if "fetch" in operation:
self._verify_fetch_operation(action)
expected_rc = int(action['return_code'])
# Create a temporary file for iv_client to write the results to
log_path_dir = os.environ['HOME']
if os.environ['DAOS_TEST_SHARED_DIR']:
log_path_dir = os.environ['DAOS_TEST_SHARED_DIR']
log_fd, log_path = tempfile.mkstemp(dir=log_path_dir)
command = " {!s} -o '{!s}' -r '{!s}' -k '{!s}:{!s}' -l '{!s}'" \
.format(command, operation, rank, key_rank, key_idx,
log_path)
clicmd += command
self.utils.print("\nClient cmd : %s\n" % clicmd)
cli_rtn = subprocess.call(shlex.split(clicmd))
if cli_rtn != 0:
raise ValueError('Error code {!s} running command "{!s}"' \
.format(cli_rtn, command))
# Read the result into test_result and remove the temp file
log_file = open(log_path)
test_result = json.load(log_file)
log_file.close()
os.close(log_fd)
os.remove(log_path)
# Parse return code and make sure it matches
if expected_rc != test_result["return_code"]:
raise ValueError("Fetch returned return code {!s} != " \
"expected value {!s}".format(
test_result["return_code"],
expected_rc))
# Other values will be invalid if return code is failure
if expected_rc != 0:
continue
# Check that returned key matches expected one
if not _check_key(key_rank, key_idx, test_result["key"]):
raise ValueError("Fetch returned unexpected key")
# Check that returned value matches expected one
if not _check_value(action['expected_value'],
test_result["value"]):
raise ValueError("Fetch returned unexpected value")
if "update" in operation:
if 'value' not in action:
raise ValueError("Update operation requires value")
command = " {!s} -o '{!s}' -r '{!s}' -k '{!s}:{!s}' -v '{!s}'" \
.format(command, operation, rank, key_rank, key_idx,
action['value'])
clicmd += command
self.utils.print("\nClient cmd : %s\n" % clicmd)
cli_rtn = subprocess.call(shlex.split(clicmd))
if cli_rtn != 0:
raise ValueError('Error code {!s} running command "{!s}"' \
.format(cli_rtn, command))
if "invalidate" in operation:
command = " {!s} -o '{!s}' -r '{!s}' -k '{!s}:{!s}'".format(
command, operation, rank, key_rank, key_idx)
clicmd += command
self.utils.print("\nClient cmd : %s\n" % clicmd)
cli_rtn = subprocess.call(shlex.split(clicmd))
if cli_rtn != 0:
raise ValueError('Error code {!s} running command "{!s}"' \
.format(cli_rtn, command))
def test_cart_iv(self):
"""
Test CaRT IV
:avocado: tags=all,cart,pr,daily_regression,iv,two_node
"""
srvcmd = self.utils.build_cmd(self, self.env, "test_servers")
try:
srv_rtn = self.utils.launch_cmd_bg(self, srvcmd)
# pylint: disable=broad-except
except Exception as e:
self.utils.print("Exception in launching server : {}".format(e))
self.fail("Test failed.\n")
# Verify the server is still running.
if not self.utils.check_process(srv_rtn):
procrtn = self.utils.stop_process(srv_rtn)
self.fail("Server did not launch, return code %s" \
% procrtn)
actions = [
# Fetch, expect fail, no variable yet
{"operation":"fetch", "rank":0, "key":(0, 42), "return_code":-1,
"expected_value":""},
# Add variable 0:42
{"operation":"update", "rank":0, "key":(0, 42), "value":"potato"},
# Fetch the value and verify it
{"operation":"fetch", "rank":0, "key":(0, 42), "return_code":0,
"expected_value":"potato"},
# Invalidate the value
{"operation":"invalidate", "rank":0, "key":(0, 42)},
# Fetch the value again expecting failure
{"operation":"fetch", "rank":0, "key":(0, 42), "return_code":-1,
"expected_value":""},
]
time.sleep(2)
failed = False
clicmd = self.utils.build_cmd(self, self.env, "test_clients")
########## Launch Client Actions ##########
try:
self._iv_test_actions(clicmd, actions)
except ValueError as exception:
failed = True
self.utils.print("TEST FAILED: %s" % str(exception))
########## Shutdown Servers ##########
num_servers = self.utils.get_srv_cnt(self, "test_servers")
srv_ppn = self.params.get("test_servers_ppn", '/run/tests/*/')
# Note: due to CART-408 issue, rank 0 needs to shutdown last
# Request each server shut down gracefully
for rank in reversed(range(1, int(srv_ppn) * num_servers)):
clicmd += " -o shutdown -r " + str(rank)
self.utils.print("\nClient cmd : %s\n" % clicmd)
try:
subprocess.call(shlex.split(clicmd))
# pylint: disable=broad-except
except Exception as e:
failed = True
self.utils.print("Exception in launching client : {}".format(e))
time.sleep(1)
# Shutdown rank 0 separately
clicmd += " -o shutdown -r 0"
self.utils.print("\nClient cmd : %s\n" % clicmd)
try:
subprocess.call(shlex.split(clicmd))
# pylint: disable=broad-except
except Exception as e:
failed = True
self.utils.print("Exception in launching client : {}".format(e))
time.sleep(2)
# Stop the server if it is still running
if self.utils.check_process(srv_rtn):
# Return value is meaningless with --continuous
self.utils.stop_process(srv_rtn)
if failed:
self.fail("Test failed.\n")
class CartNoPmixOneNodeTest(Test):
"""
Runs basic CaRT no_pmix tests on one-node
:avocado: tags=all,no_pmix,one_node
"""
def setUp(self):
""" Test setup """
print("Running setup\n")
self.utils = CartUtils()
self.env = self.utils.get_env(self)
crt_phy_addr = self.params.get("CRT_PHY_ADDR_STR", '/run/defaultENV/')
ofi_interface = self.params.get("OFI_INTERFACE", '/run/defaultENV/')
ofi_ctx_num = self.params.get("CRT_CTX_NUM", '/run/defaultENV/')
ofi_share_addr = self.params.get("CRT_CTX_SHARE_ADDR",
'/run/defaultENV/')
self.pass_env = {"CRT_PHY_ADDR_STR": crt_phy_addr,
"OFI_INTERFACE": ofi_interface,
"CRT_CTX_SHARE_ADDR": ofi_share_addr,
"CRT_CTX_NUM": ofi_ctx_num}
def tearDown(self):
""" Test tear down """
print("Run TearDown\n")
def test_cart_no_pmix(self):
"""
Test CaRT NoPmix
:avocado: tags=all,no_pmix,one_node
"""
test_bin = self.params.get("tst_bin", '/run/tests/*/')
self.utils.print("\nTest cmd : %s\n" % test_bin)
ranks = [1, 2, 3, 10, 4]
master_rank = 10
for x in ranks:
tmp_file = "/tmp/no_pmix_rank{}.uri_info".format(x)
if os.path.exists(tmp_file):
os.remove(tmp_file)
process_other = []
arg_master = ",".join(map(str, ranks))
test_env = self.pass_env
p1 = subprocess.Popen([test_bin, '{}'.format(master_rank),
'-m {}'.format(arg_master)], env=test_env,
stdout=subprocess.PIPE)
for rank in ranks:
if rank is master_rank:
continue
p = subprocess.Popen([test_bin, '{}'.format(rank)],
env=test_env, stdout=subprocess.PIPE)
process_other.append(p)
for x in process_other:
rc = self.utils.wait_process(x, 10)
if rc != 0:
self.utils.print("Error waiting for process. returning {}".format(rc))
return rc
self.utils.print("Finished waiting for {}".format(x))
rc = self.utils.wait_process(p1, 10)
if rc != 0:
self.utils.print("error waiting for master process {}".format(rc))
return rc
self.utils.print("everything finished successfully")
