class CartCoRpcOneNodeTest(TestWithoutServers):
"""
Runs CaRT ghost rank RPC test
:avocado: recursive
"""
def setUp(self):
""" Test setup """
print("Running setup\n")
self.utils = CartUtils()
self.env = self.utils.get_env(self)
def tearDown(self):
""" Tear down """
super(CartCoRpcOneNodeTest, self).tearDown()
self.utils.cleanup_processes()
def test_cart_ghost_rank_rpc(self):
"""
Test ghost rank RPC
#:avocado: tags=all,cart,pr,daily_regression,ghost_rank_rpc,one_node
"""
cmd = self.utils.build_cmd(self, self.env, "test_servers")
self.utils.launch_test(self, cmd)
class GroupTest(TestWithoutServers):
"""
Runs GroupTests for primary and secondary resizeable groups
:avocado: recursive
"""
def setUp(self):
""" Test setup """
print("Running setup\n")
self.utils = CartUtils()
self.env = self.utils.get_env(self)
def tearDown(self):
""" Tear down """
super(GroupTest, self).tearDown()
self.utils.cleanup_processes()
def test_group(self):
"""
Test CaRT NoPmix Launcher
:avocado: tags=all,cart,pr,daily_regression,group_test,one_node
"""
srv_cmd = self.utils.build_cmd(self, self.env, "test_servers")
cmd = srv_cmd
self.utils.launch_test(self, cmd)
class CartCoRpcTwoNodeTest(TestWithoutServers):
"""
Runs basic CaRT CoRPC tests
:avocado: recursive
"""
def setUp(self):
""" Test setup """
print("Running setup\n")
self.utils = CartUtils()
self.env = self.utils.get_env(self)
def tearDown(self):
""" Tear down """
super(CartCoRpcTwoNodeTest, self).tearDown()
self.utils.cleanup_processes()
def test_cart_corpc(self):
"""
Test CaRT CoRPC
:avocado: tags=all,cart,pr,daily_regression,corpc,two_node
"""
cmd = self.utils.build_cmd(self, self.env, "test_servers")
self.utils.launch_test(self, cmd)
class CartRpcOneNodeTest(TestWithoutServers):
"""
Runs basic CaRT RPC tests
:avocado: recursive
"""
def setUp(self):
""" Test setup """
print("Running setup\n")
self.utils = CartUtils()
self.env = self.utils.get_env(self)
def tearDown(self):
""" Tear down """
super(CartRpcOneNodeTest, self).tearDown()
self.utils.cleanup_processes()
def test_cart_rpc(self):
"""
Test CaRT RPC
:avocado: tags=all,cart,pr,daily_regression,rpc,one_node
"""
srvcmd = self.utils.build_cmd(self, self.env, "test_servers")
clicmd = self.utils.build_cmd(self, self.env, "test_clients")
self.utils.launch_srv_cli_test(self, srvcmd, clicmd)
self.utils.log_check(self)
class CartNoPmixOneNodeTest(TestWithoutServers):
"""
Runs basic CaRT no_pmix tests
:avocado: recursive
"""
def setUp(self):
""" Test setup """
print("Running setup\n")
self.utils = CartUtils()
self.env = self.utils.get_env(self)
crt_phy_addr = self.params.get("CRT_PHY_ADDR_STR", '/run/defaultENV/')
ofi_interface = self.params.get("OFI_INTERFACE", '/run/defaultENV/')
ofi_ctx_num = self.params.get("CRT_CTX_NUM", '/run/defaultENV/')
ofi_share_addr = self.params.get("CRT_CTX_SHARE_ADDR",
'/run/defaultENV/')
self.pass_env = {
"CRT_PHY_ADDR_STR": crt_phy_addr,
"OFI_INTERFACE": ofi_interface,
"CRT_CTX_SHARE_ADDR": ofi_share_addr,
"CRT_CTX_NUM": ofi_ctx_num
}
def tearDown(self):
""" Tear down """
super(CartNoPmixOneNodeTest, self).tearDown()
self.utils.cleanup_processes()
def test_cart_no_pmix(self):
"""
Test CaRT NoPmix
:avocado: tags=all,cart,pr,daily_regression,no_pmix,one_node
"""
cmd = self.params.get("tst_bin", '/run/tests/*/')
self.utils.print("\nTest cmd : %s\n" % cmd)
test_env = self.pass_env
p = subprocess.Popen([cmd], env=test_env, stdout=subprocess.PIPE)
rc = self.utils.wait_process(p, 10)
if rc != 0:
self.utils.print("Error waiting for process.")
self.utils.print("returning {}".format(rc))
self.fail("Test failed.\n")
self.utils.print("Finished waiting for {}".format(p))
class CartCtlFiveNodeTest(TestWithoutServers):
"""
Runs basic CaRT ctl tests
:avocado: recursive
"""
def setUp(self):
""" Test setup """
print("Running setup\n")
self.utils = CartUtils()
self.env = self.utils.get_env(self)
def tearDown(self):
""" Tear down """
super(CartCtlFiveNodeTest, self).tearDown()
self.utils.cleanup_processes()
def test_cart_ctl(self):
"""
Test CaRT ctl
:avocado: tags=all,cart,pr,daily_regression,ctl,five_node
"""
srvcmd = self.utils.build_cmd(self, self.env, "test_servers")
try:
srv_rtn = self.utils.launch_cmd_bg(self, srvcmd)
# pylint: disable=broad-except
except Exception as e:
self.utils.print("Exception in launching server : {}".format(e))
self.fail("Test failed.\n")
# Verify the server is still running.
if not self.utils.check_process(srv_rtn):
procrtn = self.utils.stop_process(srv_rtn)
self.fail("Server did not launch, return code %s" \
% procrtn)
time.sleep(5)
for index in range(2):
clicmd = self.utils.build_cmd(self,
self.env,
"test_clients",
index=index)
self.utils.launch_test(self, clicmd, srv_rtn)
self.utils.stop_process(srv_rtn)
class CartNoPmixLauncherOneNodeTest(TestWithoutServers):
"""
Runs basic CaRT no_pmix_launcher tests
:avocado: recursive
"""
def setUp(self):
""" Test setup """
print("Running setup\n")
self.utils = CartUtils()
self.env = self.utils.get_env(self)
def tearDown(self):
""" Tear down """
super(CartNoPmixLauncherOneNodeTest, self).tearDown()
self.utils.cleanup_processes()
def test_cart_no_pmix_launcher(self):
"""
Test CaRT NoPmix Launcher
:avocado: tags=all,cart,pr,daily_regression,no_pmix_launcher,one_node
"""
cli_bin = self.params.get("test_clients_bin", '/run/tests/*/')
cli_arg = self.params.get("test_clients_arg", '/run/tests/*/')
cli_ppn = self.params.get("test_clients_ppn", '/run/tests/*/')
log_mask = self.params.get("D_LOG_MASK", "/run/defaultENV/")
crt_phy_addr = self.params.get("CRT_PHY_ADDR_STR", "/run/defaultENV/")
ofi_interface = self.params.get("OFI_INTERFACE", "/run/defaultENV/")
srv_cmd = self.utils.build_cmd(self, self.env, "test_servers")
cmd = srv_cmd + " : -np {}".format(cli_ppn)
cmd += " -x CRT_PHY_ADDR_STR={}".format(crt_phy_addr)
cmd += " -x OFI_INTERFACE={}".format(ofi_interface)
cmd += " -x D_LOG_MASK={}".format(log_mask)
cmd += " {}".format(cli_bin)
cmd += " {}".format(cli_arg)
self.utils.launch_test(self, cmd)
class CartSelfThreeNodeTest(TestWithoutServers):
"""
Runs basic CaRT self test
:avocado: recursive
"""
def setUp(self):
""" Test setup """
print("Running setup\n")
self.utils = CartUtils()
self.env = self.utils.get_env(self)
def tearDown(self):
""" Tear down """
self.report_timeout()
self._teardown_errors.extend(self.utils.cleanup_processes())
super(CartSelfThreeNodeTest, self).tearDown()
def test_cart_selftest(self):
"""
Test CaRT Self Test
:avocado: tags=all,cart,pr,daily_regression,selftest,three_node
"""
srvcmd = self.utils.build_cmd(self, self.env, "test_servers")
try:
srv_rtn = self.utils.launch_cmd_bg(self, srvcmd)
# pylint: disable=broad-except
except Exception as e:
self.utils.print("Exception in launching server : {}".format(e))
self.fail("Test failed.\n")
# Verify the server is still running.
if not self.utils.check_process(srv_rtn):
procrtn = self.utils.stop_process(srv_rtn)
self.fail("Server did not launch, return code %s" \
% procrtn)
for index in range(3):
clicmd = self.utils.build_cmd(
self, self.env, "test_clients", index=index)
self.utils.launch_test(self, clicmd, srv_rtn)
# Give few seconds for servers to fully shut down before exiting
# from this test.
if not self.utils.wait_process(srv_rtn, 5):
self.utils.stop_process(srv_rtn)
class CartIvTwoNodeTest(TestWithoutServers):
"""
Runs basic CaRT tests on one-node
:avocado: recursive
"""
def setUp(self):
""" Test setup """
print("Running setup\n")
self.utils = CartUtils()
self.env = self.utils.get_env(self)
def tearDown(self):
""" Tear down """
super(CartIvTwoNodeTest, self).tearDown()
self.utils.cleanup_processes()
def _verify_action(self, action):
"""verify the action"""
if (('operation' not in action) or
('rank' not in action) or
('key' not in action)):
self.utils.print("Error happened during action check")
raise ValueError("Each action must contain an operation," \
" rank, and key")
if len(action['key']) != 2:
self.utils.print("Error key should be tuple of (rank, idx)")
raise ValueError("key should be a tuple of (rank, idx)")
def _verify_fetch_operation(self, action):
"""verify fetch operation"""
if (('return_code' not in action) or
('expected_value' not in action)):
self.utils.print("Error: fetch operation was malformed")
raise ValueError("Fetch operation malformed")
def _iv_test_actions(self, cmd, actions):
#pylint: disable=too-many-locals
"""Go through each action and perform the test"""
for action in actions:
clicmd = cmd
command = 'tests/iv_client'
self._verify_action(action)
operation = action['operation']
rank = int(action['rank'])
key_rank = int(action['key'][0])
key_idx = int(action['key'][1])
if "fetch" in operation:
self._verify_fetch_operation(action)
expected_rc = int(action['return_code'])
# Create a temporary file for iv_client to write the results to
log_path_dir = os.environ['HOME']
if os.environ['DAOS_TEST_SHARED_DIR']:
log_path_dir = os.environ['DAOS_TEST_SHARED_DIR']
log_fd, log_path = tempfile.mkstemp(dir=log_path_dir)
command = " {!s} -o '{!s}' -r '{!s}' -k '{!s}:{!s}' -l '{!s}'" \
.format(command, operation, rank, key_rank, key_idx,
log_path)
clicmd += command
self.utils.print("\nClient cmd : %s\n" % clicmd)
cli_rtn = subprocess.call(shlex.split(clicmd))
if cli_rtn != 0:
raise ValueError('Error code {!s} running command "{!s}"' \
.format(cli_rtn, command))
# Read the result into test_result and remove the temp file
log_file = open(log_path)
test_result = json.load(log_file)
log_file.close()
os.close(log_fd)
os.remove(log_path)
# Parse return code and make sure it matches
if expected_rc != test_result["return_code"]:
raise ValueError("Fetch returned return code {!s} != " \
"expected value {!s}".format(
test_result["return_code"],
expected_rc))
# Other values will be invalid if return code is failure
if expected_rc != 0:
continue
# Check that returned key matches expected one
if not _check_key(key_rank, key_idx, test_result["key"]):
raise ValueError("Fetch returned unexpected key")
# Check that returned value matches expected one
if not _check_value(action['expected_value'],
test_result["value"]):
raise ValueError("Fetch returned unexpected value")
if "update" in operation:
if 'value' not in action:
raise ValueError("Update operation requires value")
command = " {!s} -o '{!s}' -r '{!s}' -k '{!s}:{!s}' -v '{!s}'" \
.format(command, operation, rank, key_rank, key_idx,
action['value'])
clicmd += command
self.utils.print("\nClient cmd : %s\n" % clicmd)
cli_rtn = subprocess.call(shlex.split(clicmd))
if cli_rtn != 0:
raise ValueError('Error code {!s} running command "{!s}"' \
.format(cli_rtn, command))
if "invalidate" in operation:
command = " {!s} -o '{!s}' -r '{!s}' -k '{!s}:{!s}'".format(
command, operation, rank, key_rank, key_idx)
clicmd += command
self.utils.print("\nClient cmd : %s\n" % clicmd)
cli_rtn = subprocess.call(shlex.split(clicmd))
if cli_rtn != 0:
raise ValueError('Error code {!s} running command "{!s}"' \
.format(cli_rtn, command))
def test_cart_iv(self):
"""
Test CaRT IV
:avocado: tags=all,cart,pr,daily_regression,iv,two_node
"""
srvcmd = self.utils.build_cmd(self, self.env, "test_servers")
try:
srv_rtn = self.utils.launch_cmd_bg(self, srvcmd)
# pylint: disable=broad-except
except Exception as e:
self.utils.print("Exception in launching server : {}".format(e))
self.fail("Test failed.\n")
# Verify the server is still running.
if not self.utils.check_process(srv_rtn):
procrtn = self.utils.stop_process(srv_rtn)
self.fail("Server did not launch, return code %s" \
% procrtn)
actions = [
# Fetch, expect fail, no variable yet
{"operation":"fetch", "rank":0, "key":(0, 42), "return_code":-1,
"expected_value":""},
# Add variable 0:42
{"operation":"update", "rank":0, "key":(0, 42), "value":"potato"},
# Fetch the value and verify it
{"operation":"fetch", "rank":0, "key":(0, 42), "return_code":0,
"expected_value":"potato"},
# Invalidate the value
{"operation":"invalidate", "rank":0, "key":(0, 42)},
# Fetch the value again expecting failure
{"operation":"fetch", "rank":0, "key":(0, 42), "return_code":-1,
"expected_value":""},
]
time.sleep(2)
failed = False
clicmd = self.utils.build_cmd(self, self.env, "test_clients")
########## Launch Client Actions ##########
try:
self._iv_test_actions(clicmd, actions)
except ValueError as exception:
failed = True
self.utils.print("TEST FAILED: %s" % str(exception))
########## Shutdown Servers ##########
num_servers = self.utils.get_srv_cnt(self, "test_servers")
srv_ppn = self.params.get("test_servers_ppn", '/run/tests/*/')
# Note: due to CART-408 issue, rank 0 needs to shutdown last
# Request each server shut down gracefully
for rank in reversed(range(1, int(srv_ppn) * num_servers)):
clicmd += " -o shutdown -r " + str(rank)
self.utils.print("\nClient cmd : %s\n" % clicmd)
try:
subprocess.call(shlex.split(clicmd))
# pylint: disable=broad-except
except Exception as e:
failed = True
self.utils.print("Exception in launching client : {}".format(e))
time.sleep(1)
# Shutdown rank 0 separately
clicmd += " -o shutdown -r 0"
self.utils.print("\nClient cmd : %s\n" % clicmd)
try:
subprocess.call(shlex.split(clicmd))
# pylint: disable=broad-except
except Exception as e:
failed = True
self.utils.print("Exception in launching client : {}".format(e))
time.sleep(2)
# Stop the server if it is still running
if self.utils.check_process(srv_rtn):
# Return value is meaningless with --continuous
self.utils.stop_process(srv_rtn)
if failed:
self.fail("Test failed.\n")
class CartIvOneNodeTest(TestWithoutServers):
"""
Runs basic CaRT tests on one-node
:avocado: recursive
"""
def setUp(self):
""" Test setup """
print("Running setup\n")
self.utils = CartUtils()
self.env = self.utils.get_env(self)
def tearDown(self):
""" Tear down """
super(CartIvOneNodeTest, self).tearDown()
self.utils.cleanup_processes()
def _verify_action(self, action):
"""verify the action"""
if (('operation' not in action) or ('rank' not in action)
or ('key' not in action)):
self.utils.print("Error happened during action check")
raise ValueError("Each action must contain an operation," \
" rank, and key")
if len(action['key']) != 2:
self.utils.print("Error key should be tuple of (rank, idx)")
raise ValueError("key should be a tuple of (rank, idx)")
def _verify_fetch_operation(self, action):
"""verify fetch operation"""
if (('return_code' not in action) or ('expected_value' not in action)):
self.utils.print("Error: fetch operation was malformed")
raise ValueError("Fetch operation malformed")
def _iv_test_actions(self, cmd, actions):
#pylint: disable=too-many-locals
"""Go through each action and perform the test"""
for action in actions:
clicmd = cmd
command = 'tests/iv_client'
self._verify_action(action)
operation = action['operation']
rank = int(action['rank'])
key_rank = int(action['key'][0])
key_idx = int(action['key'][1])
if "fetch" in operation:
self._verify_fetch_operation(action)
expected_rc = int(action['return_code'])
# Create a temporary file for iv_client to write the results to
log_path_dir = os.environ['HOME']
if os.environ['DAOS_TEST_SHARED_DIR']:
log_path_dir = os.environ['DAOS_TEST_SHARED_DIR']
log_fd, log_path = tempfile.mkstemp(dir=log_path_dir)
# try writing to an unwritable spot
# log_path = "/"
command = " {!s} -o '{!s}' -r '{!s}' -k '{!s}:{!s}' -l '{!s}'" \
.format(command, operation, rank, key_rank, key_idx,
log_path)
clicmd += command
self.utils.print("\nClient cmd : %s\n" % clicmd)
cli_rtn = subprocess.call(shlex.split(clicmd))
if cli_rtn != 0:
raise ValueError('Error code {!s} running command "{!s}"' \
.format(cli_rtn, command))
# Read the result into test_result and remove the temp file
log_file = open(log_path)
# Try to induce "No JSON object could be decoded" error
#
# 1.
# with open(log_path, "a") as myfile:
# myfile.write("some-invalid-junk-appended-to-json")
#
# 2.
# codecs.open(log_file, "w", "unicode").write('')
# DEBUGGING: dump contents of JSON file to screen
with open(log_path, 'r') as f:
print(f.read())
test_result = json.load(log_file)
log_file.close()
os.close(log_fd)
os.remove(log_path)
# Parse return code and make sure it matches
if expected_rc != test_result["return_code"]:
raise ValueError("Fetch returned return code {!s} != " \
"expected value {!s}".format(
test_result["return_code"],
expected_rc))
# Other values will be invalid if return code is failure
if expected_rc != 0:
continue
# Check that returned key matches expected one
if not _check_key(key_rank, key_idx, test_result["key"]):
raise ValueError("Fetch returned unexpected key")
# Check that returned value matches expected one
if not _check_value(action['expected_value'],
test_result["value"]):
raise ValueError("Fetch returned unexpected value")
if "update" in operation:
if 'value' not in action:
raise ValueError("Update operation requires value")
command = " {!s} -o '{!s}' -r '{!s}' -k '{!s}:{!s}' -v '{!s}'"\
.format(command, operation, rank, key_rank, key_idx,
action['value'])
if 'sync' in action:
command = "{!s} -s '{!s}'".format(command, action['sync'])
if 'sync' not in action:
command = "{!s} -s '{!s}'".format(command, "none")
clicmd += command
self.utils.print("\nClient cmd : %s\n" % clicmd)
cli_rtn = subprocess.call(shlex.split(clicmd))
if cli_rtn != 0:
raise ValueError('Error code {!s} running command "{!s}"' \
.format(cli_rtn, command))
if "invalidate" in operation:
command = " {!s} -o '{!s}' -r '{!s}' -k '{!s}:{!s}' " \
.format(command, operation, rank, key_rank,\
key_idx)
if 'sync' in action:
command = "{!s} -s '{!s}'".format(command, action['sync'])
if 'sync' not in action:
command = "{!s} -s '{!s}'".format(command, "none")
clicmd += command
self.utils.print("\nClient cmd : %s\n" % clicmd)
cli_rtn = subprocess.call(shlex.split(clicmd))
if cli_rtn != 0:
raise ValueError('Error code {!s} running command "{!s}"' \
.format(cli_rtn, command))
if "set_grp_version" in operation:
command = " {!s} -o '{!s}' -r '{!s}' -v '{!s}' "\
.format(command, operation, rank,
action['version'])
if 'time' in action:
command = " {!s} -m '{!s}'"\
.format(command, action['time'])
if 'time' not in action:
command = " {!s} -m '{!s}'"\
.format(command, 0)
clicmd += command
self.utils.print("\nClient cmd : %s\n" % clicmd)
cli_rtn = subprocess.call(shlex.split(clicmd))
if cli_rtn != 0:
raise ValueError('Error code {!s} running command "{!s}"' \
.format(cli_rtn, command))
if "get_grp_version" in operation:
command = " {!s} -o '{!s}' -r '{!s}' " \
.format(command, operation, rank)
clicmd += command
self.utils.print("\nClient cmd : %s\n" % clicmd)
cli_rtn = subprocess.call(shlex.split(clicmd))
if cli_rtn != 0:
raise ValueError('Error code {!s} running command "{!s}"' \
.format(cli_rtn, command))
def test_cart_iv(self):
"""
Test CaRT IV
:avocado: tags=all,cart,pr,daily_regression,iv,one_node
"""
srvcmd = self.utils.build_cmd(self, self.env, "test_servers")
try:
srv_rtn = self.utils.launch_cmd_bg(self, srvcmd)
# pylint: disable=broad-except
except Exception as e:
self.utils.print("Exception in launching server : {}".format(e))
self.fail("Test failed.\n")
# Verify the server is still running.
if not self.utils.check_process(srv_rtn):
procrtn = self.utils.stop_process(srv_rtn)
self.fail("Server did not launch, return code %s" \
% procrtn)
actions = [
# ******************
# Fetch, to expect fail, no variable yet
# Make sure everything goes to the top rank
{
"operation": "fetch",
"rank": 0,
"key": (0, 42),
"return_code": -1,
"expected_value": ""
},
{
"operation": "fetch",
"rank": 1,
"key": (0, 42),
"return_code": -1,
"expected_value": ""
},
{
"operation": "fetch",
"rank": 4,
"key": (0, 42),
"return_code": -1,
"expected_value": ""
},
#
# ****
# Add variable 0:42
{
"operation": "update",
"rank": 0,
"key": (0, 42),
"value": "potato"
},
#
# ****
# Fetch the value from each server and verify it
{
"operation": "fetch",
"rank": 0,
"key": (0, 42),
"return_code": 0,
"expected_value": "potato"
},
{
"operation": "fetch",
"rank": 1,
"key": (0, 42),
"return_code": 0,
"expected_value": "potato"
},
{
"operation": "fetch",
"rank": 2,
"key": (0, 42),
"return_code": 0,
"expected_value": "potato"
},
{
"operation": "fetch",
"rank": 3,
"key": (0, 42),
"return_code": 0,
"expected_value": "potato"
},
{
"operation": "fetch",
"rank": 4,
"key": (0, 42),
"return_code": 0,
"expected_value": "potato"
},
#
# ******************
# Invalidate the value
{
"operation": "invalidate",
"rank": 0,
"key": (0, 42),
"sync": "eager_update"
},
#
# ****
# Fetch the value again from each server, expecting failure
# Reverse order of fetch just in case.
{
"operation": "fetch",
"rank": 4,
"key": (0, 42),
"return_code": -1,
"expected_value": ""
},
{
"operation": "fetch",
"rank": 3,
"key": (0, 42),
"return_code": -1,
"expected_value": ""
},
{
"operation": "fetch",
"rank": 2,
"key": (0, 42),
"return_code": -1,
"expected_value": ""
},
{
"operation": "fetch",
"rank": 1,
"key": (0, 42),
"return_code": -1,
"expected_value": ""
},
{
"operation": "fetch",
"rank": 0,
"key": (0, 42),
"return_code": -1,
"expected_value": ""
},
#
######################
# Testing version number conflicts.
######################
# Test of version skew on fetch between rank 0 and rank 1.
# Make sure we can set version numbers.
{
"operation": "set_grp_version",
"rank": 0,
"key": (0, 42),
"version": "0xdeadc0de",
"return_code": 0,
"expected_value": ""
},
{
"operation": "get_grp_version",
"rank": 0,
"key": (0, 42),
"return_code": 0,
"expected_value": "0xdeadc0de"
},
{
"operation": "set_grp_version",
"rank": 0,
"key": (0, 42),
"version": "",
"return_code": 0,
"expected_value": ""
},
#
# ******************
# Test of version skew on fetch between rank 0 and rank 1.
# From parent to child and from child to parent
# Don't setup a iv variable.
# Modify version number on root 0.
# Do fetch in both direction for and test for failure.
# First, do test for normal failure.
{
"operation": "fetch",
"rank": 0,
"key": (1, 42),
"return_code": -1,
"expected_value": ""
},
{
"operation": "set_grp_version",
"rank": 0,
"key": (0, 42),
"version": "0xdeadc0de",
"return_code": 0,
"expected_value": ""
},
{
"operation": "fetch",
"rank": 0,
"key": (1, 42),
"return_code": -1036,
"expected_value": ""
},
{
"operation": "fetch",
"rank": 1,
"key": (0, 42),
"return_code": -1036,
"expected_value": ""
},
{
"operation": "set_grp_version",
"rank": 0,
"key": (0, 42),
"version": "0x0",
"return_code": 0,
"expected_value": ""
},
{
"operation": "invalidate",
"rank": 1,
"key": (1, 42)
},
#
# ******************
# Test of version skew on fetch between rank 0 and rank 1.
# Create iv variable on rank 1.
# Fetch from rank 0.
# Change version on rank 0 while request in flight,
# Not an error:
# Used for testing to ensure we donot break something
# that should work.
# Version change occurs in iv_test_fetch_iv
# Need to invalidate on both nodes, stale data.
{
"operation": "update",
"rank": 1,
"key": (1, 42),
"value": "beans"
},
{
"operation": "set_grp_version",
"rank": 0,
"key": (0, 42),
"time": 1,
"version": "0xc001c001",
"return_code": 0,
"expected_value": ""
},
{
"operation": "fetch",
"rank": 0,
"key": (1, 42),
"return_code": 0,
"expected_value": "beans"
},
{
"operation": "set_grp_version",
"rank": 0,
"key": (1, 42),
"version": "0",
"return_code": 0,
"expected_value": ""
},
{
"operation": "invalidate",
"rank": 1,
"key": (1, 42)
},
{
"operation": "invalidate",
"rank": 0,
"key": (1, 42)
},
#
# ******************
# Test of version skew on fetch between rank 0 and rank 1.
# From parent to child.
# Create a iv variable on second server (child).
# Setup second server to change version after it receives
# the rpc request.
# Fetch variable from the first server.
# Tests version-check in crt_hdlr_iv_fetch_aux.
# Version change in iv_pre_fetch
#
{
"operation": "update",
"rank": 1,
"key": (1, 42),
"value": "carrot"
},
{
"operation": "set_grp_version",
"rank": 1,
"key": (1, 42),
"time": 2,
"version": "0xdeadc0de",
"return_code": 0,
"expected_value": ""
},
{
"operation": "fetch",
"rank": 0,
"key": (1, 42),
"return_code": -1036,
"expected_value": ""
},
{
"operation": "set_grp_version",
"rank": 1,
"key": (1, 42),
"version": "0x0",
"return_code": 0,
"expected_value": ""
},
{
"operation": "invalidate",
"rank": 1,
"key": (1, 42)
},
#
# ******************
# Test invalidate with synchronization.
# First create an iv value from rank 0 to rank 4.
# Then verify that all ranks can see it.
# Then remove it and verify that no ranks has a local copy
# Need to know that this works prior to changing version
# Verifies eager_notify works with invalidate.
#
{
"operation": "update",
"rank": 0,
"key": (4, 42),
"value": "turnip"
},
{
"operation": "fetch",
"rank": 1,
"key": (4, 42),
"return_code": 0,
"expected_value": "turnip"
},
{
"operation": "fetch",
"rank": 0,
"key": (4, 42),
"return_code": 0,
"expected_value": "turnip"
},
{
"operation": "fetch",
"rank": 3,
"key": (4, 42),
"return_code": 0,
"expected_value": "turnip"
},
{
"operation": "fetch",
"rank": 2,
"key": (4, 42),
"return_code": 0,
"expected_value": "turnip"
},
{
"operation": "fetch",
"rank": 4,
"key": (4, 42),
"return_code": 0,
"expected_value": "turnip"
},
#
{
"operation": "invalidate",
"rank": 4,
"key": (4, 42),
"sync": "eager_notify",
"return_code": 0
},
#
# Check for stale state.
{
"operation": "fetch",
"rank": 4,
"key": (4, 42),
"return_code": -1,
"expected_value": ""
},
{
"operation": "fetch",
"rank": 1,
"key": (4, 42),
"return_code": -1,
"expected_value": ""
},
{
"operation": "fetch",
"rank": 0,
"key": (4, 42),
"return_code": -1,
"expected_value": ""
},
{
"operation": "fetch",
"rank": 2,
"key": (4, 42),
"return_code": -1,
"expected_value": ""
},
{
"operation": "fetch",
"rank": 3,
"key": (4, 42),
"return_code": -1,
"expected_value": ""
},
#
# ******************
# Test of version skew on update with synchronization
# when version on child process is different
# Change version on rank 4
# Create iv variable on rank 0 using sync.
# Should return error and no iv variable created.
# Make sure nothing is left behind on other nodes.
#
{
"operation": "set_grp_version",
"rank": 4,
"key": (4, 42),
"version": "0xdeadc0de",
"return_code": 0,
"expected_value": ""
},
{
"operation": "update",
"rank": 0,
"key": (0, 42),
"value": "beans",
"sync": "eager_update",
"return_code": -1036
},
# Even though previous failure, leaves stale state on ranks
{
"operation": "fetch",
"rank": 0,
"key": (0, 42),
"return_code": 0,
"expected_value": "beans"
},
{
"operation": "fetch",
"rank": 1,
"key": (0, 42),
"return_code": 0,
"expected_value": "beans"
},
{
"operation": "fetch",
"rank": 4,
"key": (0, 42),
"return_code": -1036,
"expected_value": ""
},
#
# Clean up. Make sure no stale state left
{
"operation": "invalidate",
"rank": 0,
"key": (0, 42),
"sync": "eager_notify",
"return_code": 0
},
{
"operation": "fetch",
"rank": 0,
"key": (0, 42),
"return_code": -1,
"expected_value": ""
},
{
"operation": "set_grp_version",
"rank": 4,
"key": (4, 42),
"version": "0x0",
"return_code": 0,
"expected_value": ""
},
{
"operation": "fetch",
"rank": 1,
"key": (0, 42),
"return_code": -1,
"expected_value": ""
},
]
time.sleep(2)
failed = False
clicmd = self.utils.build_cmd(self, self.env, "test_clients")
########## Launch Client Actions ##########
try:
self._iv_test_actions(clicmd, actions)
except ValueError as exception:
failed = True
traceback.print_stack()
self.utils.print("TEST FAILED: %s" % str(exception))
########## Shutdown Servers ##########
num_servers = self.utils.get_srv_cnt(self, "test_servers")
srv_ppn = self.params.get("test_servers_ppn", '/run/tests/*/')
# Note: due to CART-408 issue, rank 0 needs to shutdown last
# Request each server shut down gracefully
for rank in reversed(range(1, int(srv_ppn) * num_servers)):
clicmd += " -o shutdown -r " + str(rank)
self.utils.print("\nClient cmd : %s\n" % clicmd)
try:
subprocess.call(shlex.split(clicmd))
# pylint: disable=broad-except
except Exception as e:
failed = True
self.utils.print(
"Exception in launching client : {}".format(e))
time.sleep(1)
# Shutdown rank 0 separately
clicmd += " -o shutdown -r 0"
self.utils.print("\nClient cmd : %s\n" % clicmd)
try:
subprocess.call(shlex.split(clicmd))
# pylint: disable=broad-except
except Exception as e:
failed = True
self.utils.print("Exception in launching client : {}".format(e))
time.sleep(2)
# Stop the server if it is still running
if self.utils.check_process(srv_rtn):
# Return value is meaningless with --continuous
self.utils.stop_process(srv_rtn)
if failed:
self.fail("Test failed.\n")
