class CartCtlFiveNodeTest(TestWithoutServers): """ Runs basic CaRT ctl tests :avocado: recursive """ def setUp(self): """ Test setup """ print("Running setup\n") self.utils = CartUtils() self.env = self.utils.get_env(self) def tearDown(self): """ Tear down """ super(CartCtlFiveNodeTest, self).tearDown() self.utils.cleanup_processes() def test_cart_ctl(self): """ Test CaRT ctl :avocado: tags=all,cart,pr,daily_regression,ctl,five_node """ srvcmd = self.utils.build_cmd(self, self.env, "test_servers") try: srv_rtn = self.utils.launch_cmd_bg(self, srvcmd) # pylint: disable=broad-except except Exception as e: self.utils.print("Exception in launching server : {}".format(e)) self.fail("Test failed.\n") # Verify the server is still running. if not self.utils.check_process(srv_rtn): procrtn = self.utils.stop_process(srv_rtn) self.fail("Server did not launch, return code %s" \ % procrtn) time.sleep(5) for index in range(2): clicmd = self.utils.build_cmd(self, self.env, "test_clients", index=index) self.utils.launch_test(self, clicmd, srv_rtn) self.utils.stop_process(srv_rtn)
class CartCtlOneNodeTest(Test): """ Runs basic CaRT ctl tests :avocado: tags=all,ctl,one_node """ def setUp(self): """ Test setup """ print("Running setup\n") self.utils = CartUtils() self.env = self.utils.get_env(self) def tearDown(self): """ Test tear down """ print("Run TearDown\n") def test_cart_ctl(self): """ Test CaRT ctl :avocado: tags=all,ctl,one_node """ srvcmd = self.utils.build_cmd(self, self.env, "srv") try: srv_rtn = self.utils.launch_cmd_bg(self, srvcmd) except Exception as e: self.utils.print("Exception in launching server : {}".format(e)) self.fail("Test failed.\n") # Verify the server is still running. if not self.utils.check_process(srv_rtn): procrtn = self.utils.stop_process(srv_rtn) self.fail("Server did not launch, return code %s" \ % procrtn) clicmd = self.utils.build_cmd(self, self.env, "cli1") self.utils.launch_test(self, clicmd, srv_rtn) clicmd = self.utils.build_cmd(self, self.env, "cli2") self.utils.launch_test(self, clicmd, srv_rtn) clicmd = self.utils.build_cmd(self, self.env, "cli3") self.utils.launch_test(self, clicmd, srv_rtn) clicmd = self.utils.build_cmd(self, self.env, "cli4") self.utils.launch_test(self, clicmd, srv_rtn) clicmd = self.utils.build_cmd(self, self.env, "cli5") self.utils.launch_test(self, clicmd, srv_rtn) clicmd = self.utils.build_cmd(self, self.env, "cli6") self.utils.launch_test(self, clicmd, srv_rtn)
class CartSelfThreeNodeTest(TestWithoutServers): """ Runs basic CaRT self test :avocado: recursive """ def setUp(self): """ Test setup """ print("Running setup\n") self.utils = CartUtils() self.env = self.utils.get_env(self) def tearDown(self): """ Tear down """ self.report_timeout() self._teardown_errors.extend(self.utils.cleanup_processes()) super(CartSelfThreeNodeTest, self).tearDown() def test_cart_selftest(self): """ Test CaRT Self Test :avocado: tags=all,cart,pr,daily_regression,selftest,three_node """ srvcmd = self.utils.build_cmd(self, self.env, "test_servers") try: srv_rtn = self.utils.launch_cmd_bg(self, srvcmd) # pylint: disable=broad-except except Exception as e: self.utils.print("Exception in launching server : {}".format(e)) self.fail("Test failed.\n") # Verify the server is still running. if not self.utils.check_process(srv_rtn): procrtn = self.utils.stop_process(srv_rtn) self.fail("Server did not launch, return code %s" \ % procrtn) for index in range(3): clicmd = self.utils.build_cmd( self, self.env, "test_clients", index=index) self.utils.launch_test(self, clicmd, srv_rtn) # Give few seconds for servers to fully shut down before exiting # from this test. if not self.utils.wait_process(srv_rtn, 5): self.utils.stop_process(srv_rtn)
class CartRpcOneNodeSwimNotificationOnRankEvictionTest(TestWithoutServers): """ Runs basic CaRT RPC tests :avocado: recursive """ def setUp(self): """ Test setup """ print("Running setup\n") self.utils = CartUtils() self.env = self.utils.get_env(self) def tearDown(self): """Tear down.""" print("tearDown() start") super(CartRpcOneNodeSwimNotificationOnRankEvictionTest, self).tearDown() def test_cart_rpc(self): """ Test CaRT RPC :avocado: tags=all,cart,pr,rpc,one_node,swim_rank_eviction """ srvcmd = self.utils.build_cmd(self, self.env, "test_servers") try: srv_rtn = self.utils.launch_cmd_bg(self, srvcmd) # pylint: disable=broad-except except Exception as e: self.utils.print("Exception in launching server : {}".format(e)) self.fail("Test failed.\n") # Verify the server is still running. if not self.utils.check_process(srv_rtn): procrtn = self.utils.stop_process(srv_rtn) self.fail("Server did not launch, return code %s" \ % procrtn) for index in range(6): clicmd = self.utils.build_cmd(self, self.env, "test_clients", index=index) self.utils.launch_test(self, clicmd, srv_rtn)
class CartSelfThreeNodeTest(Test): """ Runs basic CaRT self test :avocado: tags=all,cart,pr,selftest,three_node """ def setUp(self): """ Test setup """ print("Running setup\n") self.utils = CartUtils() self.env = self.utils.get_env(self) def tearDown(self): """ Test tear down """ print("Run TearDown\n") def test_cart_selftest(self): """ Test CaRT Self Test :avocado: tags=all,cart,pr,selftest,three_node """ srvcmd = self.utils.build_cmd(self, self.env, "test_servers") try: srv_rtn = self.utils.launch_cmd_bg(self, srvcmd) # pylint: disable=broad-except except Exception as e: self.utils.print("Exception in launching server : {}".format(e)) self.fail("Test failed.\n") # Verify the server is still running. if not self.utils.check_process(srv_rtn): procrtn = self.utils.stop_process(srv_rtn) self.fail("Server did not launch, return code %s" \ % procrtn) for index in range(3): clicmd = self.utils.build_cmd( self, self.env, "test_clients", index=index) self.utils.launch_test(self, clicmd, srv_rtn)
class CartNoPmixOneNodeTest(TestWithoutServers): """ Runs basic CaRT no_pmix tests :avocado: recursive """ def setUp(self): """ Test setup """ print("Running setup\n") self.utils = CartUtils() self.env = self.utils.get_env(self) crt_phy_addr = self.params.get("CRT_PHY_ADDR_STR", '/run/defaultENV/') ofi_interface = self.params.get("OFI_INTERFACE", '/run/defaultENV/') ofi_ctx_num = self.params.get("CRT_CTX_NUM", '/run/defaultENV/') ofi_share_addr = self.params.get("CRT_CTX_SHARE_ADDR", '/run/defaultENV/') self.pass_env = { "CRT_PHY_ADDR_STR": crt_phy_addr, "OFI_INTERFACE": ofi_interface, "CRT_CTX_SHARE_ADDR": ofi_share_addr, "CRT_CTX_NUM": ofi_ctx_num } def tearDown(self): """ Tear down """ self.report_timeout() self._teardown_errors.extend(self.utils.cleanup_processes()) super(CartNoPmixOneNodeTest, self).tearDown() def test_cart_no_pmix(self): """ Test CaRT NoPmix :avocado: tags=all,cart,pr,daily_regression,no_pmix,one_node """ cmd = self.params.get("tst_bin", '/run/tests/*/') self.utils.print("\nTest cmd : %s\n" % cmd) test_env = self.pass_env p = subprocess.Popen([cmd], env=test_env, stdout=subprocess.PIPE) rc = self.utils.wait_process(p, 30) if rc != 0: self.utils.print("Error waiting for process.") self.utils.print("returning {}".format(rc)) self.fail("Test failed.\n") self.utils.print("Finished waiting for {}".format(p))
class CartIvOneNodeTest(TestWithoutServers): """ Runs basic CaRT tests on one-node :avocado: recursive """ def setUp(self): """ Test setup """ print("Running setup\n") self.utils = CartUtils() self.env = self.utils.get_env(self) def tearDown(self): """ Tear down """ super(CartIvOneNodeTest, self).tearDown() self.utils.cleanup_processes() def _verify_action(self, action): """verify the action""" if (('operation' not in action) or ('rank' not in action) or ('key' not in action)): self.utils.print("Error happened during action check") raise ValueError("Each action must contain an operation," \ " rank, and key") if len(action['key']) != 2: self.utils.print("Error key should be tuple of (rank, idx)") raise ValueError("key should be a tuple of (rank, idx)") def _verify_fetch_operation(self, action): """verify fetch operation""" if (('return_code' not in action) or ('expected_value' not in action)): self.utils.print("Error: fetch operation was malformed") raise ValueError("Fetch operation malformed") def _iv_test_actions(self, cmd, actions): #pylint: disable=too-many-locals """Go through each action and perform the test""" for action in actions: clicmd = cmd command = 'tests/iv_client' self._verify_action(action) operation = action['operation'] rank = int(action['rank']) key_rank = int(action['key'][0]) key_idx = int(action['key'][1]) if "fetch" in operation: self._verify_fetch_operation(action) expected_rc = int(action['return_code']) # Create a temporary file for iv_client to write the results to log_path_dir = os.environ['HOME'] if os.environ['DAOS_TEST_SHARED_DIR']: log_path_dir = os.environ['DAOS_TEST_SHARED_DIR'] log_fd, log_path = tempfile.mkstemp(dir=log_path_dir) # try writing to an unwritable spot # log_path = "/" command = " {!s} -o '{!s}' -r '{!s}' -k '{!s}:{!s}' -l '{!s}'" \ .format(command, operation, rank, key_rank, key_idx, log_path) clicmd += command self.utils.print("\nClient cmd : %s\n" % clicmd) cli_rtn = subprocess.call(shlex.split(clicmd)) if cli_rtn != 0: raise ValueError('Error code {!s} running command "{!s}"' \ .format(cli_rtn, command)) # Read the result into test_result and remove the temp file log_file = open(log_path) # Try to induce "No JSON object could be decoded" error # # 1. # with open(log_path, "a") as myfile: # myfile.write("some-invalid-junk-appended-to-json") # # 2. # codecs.open(log_file, "w", "unicode").write('') # DEBUGGING: dump contents of JSON file to screen with open(log_path, 'r') as f: print(f.read()) test_result = json.load(log_file) log_file.close() os.close(log_fd) os.remove(log_path) # Parse return code and make sure it matches if expected_rc != test_result["return_code"]: raise ValueError("Fetch returned return code {!s} != " \ "expected value {!s}".format( test_result["return_code"], expected_rc)) # Other values will be invalid if return code is failure if expected_rc != 0: continue # Check that returned key matches expected one if not _check_key(key_rank, key_idx, test_result["key"]): raise ValueError("Fetch returned unexpected key") # Check that returned value matches expected one if not _check_value(action['expected_value'], test_result["value"]): raise ValueError("Fetch returned unexpected value") if "update" in operation: if 'value' not in action: raise ValueError("Update operation requires value") command = " {!s} -o '{!s}' -r '{!s}' -k '{!s}:{!s}' -v '{!s}'"\ .format(command, operation, rank, key_rank, key_idx, action['value']) if 'sync' in action: command = "{!s} -s '{!s}'".format(command, action['sync']) if 'sync' not in action: command = "{!s} -s '{!s}'".format(command, "none") clicmd += command self.utils.print("\nClient cmd : %s\n" % clicmd) cli_rtn = subprocess.call(shlex.split(clicmd)) if cli_rtn != 0: raise ValueError('Error code {!s} running command "{!s}"' \ .format(cli_rtn, command)) if "invalidate" in operation: command = " {!s} -o '{!s}' -r '{!s}' -k '{!s}:{!s}' " \ .format(command, operation, rank, key_rank,\ key_idx) if 'sync' in action: command = "{!s} -s '{!s}'".format(command, action['sync']) if 'sync' not in action: command = "{!s} -s '{!s}'".format(command, "none") clicmd += command self.utils.print("\nClient cmd : %s\n" % clicmd) cli_rtn = subprocess.call(shlex.split(clicmd)) if cli_rtn != 0: raise ValueError('Error code {!s} running command "{!s}"' \ .format(cli_rtn, command)) if "set_grp_version" in operation: command = " {!s} -o '{!s}' -r '{!s}' -v '{!s}' "\ .format(command, operation, rank, action['version']) if 'time' in action: command = " {!s} -m '{!s}'"\ .format(command, action['time']) if 'time' not in action: command = " {!s} -m '{!s}'"\ .format(command, 0) clicmd += command self.utils.print("\nClient cmd : %s\n" % clicmd) cli_rtn = subprocess.call(shlex.split(clicmd)) if cli_rtn != 0: raise ValueError('Error code {!s} running command "{!s}"' \ .format(cli_rtn, command)) if "get_grp_version" in operation: command = " {!s} -o '{!s}' -r '{!s}' " \ .format(command, operation, rank) clicmd += command self.utils.print("\nClient cmd : %s\n" % clicmd) cli_rtn = subprocess.call(shlex.split(clicmd)) if cli_rtn != 0: raise ValueError('Error code {!s} running command "{!s}"' \ .format(cli_rtn, command)) def test_cart_iv(self): """ Test CaRT IV :avocado: tags=all,cart,pr,daily_regression,iv,one_node """ srvcmd = self.utils.build_cmd(self, self.env, "test_servers") try: srv_rtn = self.utils.launch_cmd_bg(self, srvcmd) # pylint: disable=broad-except except Exception as e: self.utils.print("Exception in launching server : {}".format(e)) self.fail("Test failed.\n") # Verify the server is still running. if not self.utils.check_process(srv_rtn): procrtn = self.utils.stop_process(srv_rtn) self.fail("Server did not launch, return code %s" \ % procrtn) actions = [ # ****************** # Fetch, to expect fail, no variable yet # Make sure everything goes to the top rank { "operation": "fetch", "rank": 0, "key": (0, 42), "return_code": -1, "expected_value": "" }, { "operation": "fetch", "rank": 1, "key": (0, 42), "return_code": -1, "expected_value": "" }, { "operation": "fetch", "rank": 4, "key": (0, 42), "return_code": -1, "expected_value": "" }, # # **** # Add variable 0:42 { "operation": "update", "rank": 0, "key": (0, 42), "value": "potato" }, # # **** # Fetch the value from each server and verify it { "operation": "fetch", "rank": 0, "key": (0, 42), "return_code": 0, "expected_value": "potato" }, { "operation": "fetch", "rank": 1, "key": (0, 42), "return_code": 0, "expected_value": "potato" }, { "operation": "fetch", "rank": 2, "key": (0, 42), "return_code": 0, "expected_value": "potato" }, { "operation": "fetch", "rank": 3, "key": (0, 42), "return_code": 0, "expected_value": "potato" }, { "operation": "fetch", "rank": 4, "key": (0, 42), "return_code": 0, "expected_value": "potato" }, # # ****************** # Invalidate the value { "operation": "invalidate", "rank": 0, "key": (0, 42), "sync": "eager_update" }, # # **** # Fetch the value again from each server, expecting failure # Reverse order of fetch just in case. { "operation": "fetch", "rank": 4, "key": (0, 42), "return_code": -1, "expected_value": "" }, { "operation": "fetch", "rank": 3, "key": (0, 42), "return_code": -1, "expected_value": "" }, { "operation": "fetch", "rank": 2, "key": (0, 42), "return_code": -1, "expected_value": "" }, { "operation": "fetch", "rank": 1, "key": (0, 42), "return_code": -1, "expected_value": "" }, { "operation": "fetch", "rank": 0, "key": (0, 42), "return_code": -1, "expected_value": "" }, # ###################### # Testing version number conflicts. ###################### # Test of version skew on fetch between rank 0 and rank 1. # Make sure we can set version numbers. { "operation": "set_grp_version", "rank": 0, "key": (0, 42), "version": "0xdeadc0de", "return_code": 0, "expected_value": "" }, { "operation": "get_grp_version", "rank": 0, "key": (0, 42), "return_code": 0, "expected_value": "0xdeadc0de" }, { "operation": "set_grp_version", "rank": 0, "key": (0, 42), "version": "", "return_code": 0, "expected_value": "" }, # # ****************** # Test of version skew on fetch between rank 0 and rank 1. # From parent to child and from child to parent # Don't setup a iv variable. # Modify version number on root 0. # Do fetch in both direction for and test for failure. # First, do test for normal failure. { "operation": "fetch", "rank": 0, "key": (1, 42), "return_code": -1, "expected_value": "" }, { "operation": "set_grp_version", "rank": 0, "key": (0, 42), "version": "0xdeadc0de", "return_code": 0, "expected_value": "" }, { "operation": "fetch", "rank": 0, "key": (1, 42), "return_code": -1036, "expected_value": "" }, { "operation": "fetch", "rank": 1, "key": (0, 42), "return_code": -1036, "expected_value": "" }, { "operation": "set_grp_version", "rank": 0, "key": (0, 42), "version": "0x0", "return_code": 0, "expected_value": "" }, { "operation": "invalidate", "rank": 1, "key": (1, 42) }, # # ****************** # Test of version skew on fetch between rank 0 and rank 1. # Create iv variable on rank 1. # Fetch from rank 0. # Change version on rank 0 while request in flight, # Not an error: # Used for testing to ensure we donot break something # that should work. # Version change occurs in iv_test_fetch_iv # Need to invalidate on both nodes, stale data. { "operation": "update", "rank": 1, "key": (1, 42), "value": "beans" }, { "operation": "set_grp_version", "rank": 0, "key": (0, 42), "time": 1, "version": "0xc001c001", "return_code": 0, "expected_value": "" }, { "operation": "fetch", "rank": 0, "key": (1, 42), "return_code": 0, "expected_value": "beans" }, { "operation": "set_grp_version", "rank": 0, "key": (1, 42), "version": "0", "return_code": 0, "expected_value": "" }, { "operation": "invalidate", "rank": 1, "key": (1, 42) }, { "operation": "invalidate", "rank": 0, "key": (1, 42) }, # # ****************** # Test of version skew on fetch between rank 0 and rank 1. # From parent to child. # Create a iv variable on second server (child). # Setup second server to change version after it receives # the rpc request. # Fetch variable from the first server. # Tests version-check in crt_hdlr_iv_fetch_aux. # Version change in iv_pre_fetch # { "operation": "update", "rank": 1, "key": (1, 42), "value": "carrot" }, { "operation": "set_grp_version", "rank": 1, "key": (1, 42), "time": 2, "version": "0xdeadc0de", "return_code": 0, "expected_value": "" }, { "operation": "fetch", "rank": 0, "key": (1, 42), "return_code": -1036, "expected_value": "" }, { "operation": "set_grp_version", "rank": 1, "key": (1, 42), "version": "0x0", "return_code": 0, "expected_value": "" }, { "operation": "invalidate", "rank": 1, "key": (1, 42) }, # # ****************** # Test invalidate with synchronization. # First create an iv value from rank 0 to rank 4. # Then verify that all ranks can see it. # Then remove it and verify that no ranks has a local copy # Need to know that this works prior to changing version # Verifies eager_notify works with invalidate. # { "operation": "update", "rank": 0, "key": (4, 42), "value": "turnip" }, { "operation": "fetch", "rank": 1, "key": (4, 42), "return_code": 0, "expected_value": "turnip" }, { "operation": "fetch", "rank": 0, "key": (4, 42), "return_code": 0, "expected_value": "turnip" }, { "operation": "fetch", "rank": 3, "key": (4, 42), "return_code": 0, "expected_value": "turnip" }, { "operation": "fetch", "rank": 2, "key": (4, 42), "return_code": 0, "expected_value": "turnip" }, { "operation": "fetch", "rank": 4, "key": (4, 42), "return_code": 0, "expected_value": "turnip" }, # { "operation": "invalidate", "rank": 4, "key": (4, 42), "sync": "eager_notify", "return_code": 0 }, # # Check for stale state. { "operation": "fetch", "rank": 4, "key": (4, 42), "return_code": -1, "expected_value": "" }, { "operation": "fetch", "rank": 1, "key": (4, 42), "return_code": -1, "expected_value": "" }, { "operation": "fetch", "rank": 0, "key": (4, 42), "return_code": -1, "expected_value": "" }, { "operation": "fetch", "rank": 2, "key": (4, 42), "return_code": -1, "expected_value": "" }, { "operation": "fetch", "rank": 3, "key": (4, 42), "return_code": -1, "expected_value": "" }, # # ****************** # Test of version skew on update with synchronization # when version on child process is different # Change version on rank 4 # Create iv variable on rank 0 using sync. # Should return error and no iv variable created. # Make sure nothing is left behind on other nodes. # { "operation": "set_grp_version", "rank": 4, "key": (4, 42), "version": "0xdeadc0de", "return_code": 0, "expected_value": "" }, { "operation": "update", "rank": 0, "key": (0, 42), "value": "beans", "sync": "eager_update", "return_code": -1036 }, # Even though previous failure, leaves stale state on ranks { "operation": "fetch", "rank": 0, "key": (0, 42), "return_code": 0, "expected_value": "beans" }, { "operation": "fetch", "rank": 1, "key": (0, 42), "return_code": 0, "expected_value": "beans" }, { "operation": "fetch", "rank": 4, "key": (0, 42), "return_code": -1036, "expected_value": "" }, # # Clean up. Make sure no stale state left { "operation": "invalidate", "rank": 0, "key": (0, 42), "sync": "eager_notify", "return_code": 0 }, { "operation": "fetch", "rank": 0, "key": (0, 42), "return_code": -1, "expected_value": "" }, { "operation": "set_grp_version", "rank": 4, "key": (4, 42), "version": "0x0", "return_code": 0, "expected_value": "" }, { "operation": "fetch", "rank": 1, "key": (0, 42), "return_code": -1, "expected_value": "" }, ] time.sleep(2) failed = False clicmd = self.utils.build_cmd(self, self.env, "test_clients") ########## Launch Client Actions ########## try: self._iv_test_actions(clicmd, actions) except ValueError as exception: failed = True traceback.print_stack() self.utils.print("TEST FAILED: %s" % str(exception)) ########## Shutdown Servers ########## num_servers = self.utils.get_srv_cnt(self, "test_servers") srv_ppn = self.params.get("test_servers_ppn", '/run/tests/*/') # Note: due to CART-408 issue, rank 0 needs to shutdown last # Request each server shut down gracefully for rank in reversed(range(1, int(srv_ppn) * num_servers)): clicmd += " -o shutdown -r " + str(rank) self.utils.print("\nClient cmd : %s\n" % clicmd) try: subprocess.call(shlex.split(clicmd)) # pylint: disable=broad-except except Exception as e: failed = True self.utils.print( "Exception in launching client : {}".format(e)) time.sleep(1) # Shutdown rank 0 separately clicmd += " -o shutdown -r 0" self.utils.print("\nClient cmd : %s\n" % clicmd) try: subprocess.call(shlex.split(clicmd)) # pylint: disable=broad-except except Exception as e: failed = True self.utils.print("Exception in launching client : {}".format(e)) time.sleep(2) # Stop the server if it is still running if self.utils.check_process(srv_rtn): # Return value is meaningless with --continuous self.utils.stop_process(srv_rtn) if failed: self.fail("Test failed.\n")
class CartSingletonTwoNodeTest(Test): """ Runs basic CaRT singleton tests :avocado: tags=all,singleton,two_node """ def setUp(self): """ Test setup """ print("Running setup\n") self.utils = CartUtils() self.env = self.utils.get_env(self) self.tempdir = tempfile.mkdtemp() os.environ["CRT_PHY_ADDR_STR"] = self.params.get("CRT_PHY_ADDR_STR", '/run/defaultENV/') os.environ["OFI_INTERFACE"] = self.params.get("OFI_INTERFACE", '/run/defaultENV/') os.environ["CRT_CTX_NUM"] = self.params.get("cli_CRT_CTX_NUM", '/run/defaultENV/') os.environ["CRT_CTX_SHARE_ADDR"] = self.params.get("CRT_CTX_SHARE_ADDR", '/run/env_CRT_CTX_SHARE_ADDR/*/') def tearDown(self): """ Test tear down """ print("Run TearDown\n") shutil.rmtree(self.tempdir) def test_cart_singleton(self): """ Test CaRT Singleton :avocado: tags=all,singleton,two_node """ urifile = self.utils.create_uri_file() srvcmd = self.utils.build_cmd(self, self.env, "srv", True, urifile) srvcmd += " -p {} -s".format(self.tempdir) try: srv_rtn = self.utils.launch_cmd_bg(self, srvcmd) except Exception as e: self.utils.print("Exception in launching server : {}".format(e)) self.fail("Test failed.\n") time.sleep(5) # Verify the server is still running. if not self.utils.check_process(srv_rtn): procrtn = self.utils.stop_process(srv_rtn) self.fail("Server did not launch, return code %s" \ % procrtn) clicmd = self.params.get("cli_bin", '/run/tests/*/') clicmd += " -p {} -s".format(self.tempdir) self.utils.launch_test(self, clicmd, srv_rtn) # Stop the server self.utils.print("Stopping server process {}".format(srv_rtn)) procrtn = self.utils.stop_process(srv_rtn) if procrtn: self.fail("Test failed. Server ret code {}".format(procrtn)) def test_multi_tier_singleton_attach(self): """ Test CaRT Multi_tier singleton attach test on two nodes :avocado: tags=all,singleton,multi_tier,two_node """ urifile = self.utils.create_uri_file() srvcmd = self.utils.build_cmd(self, self.env, "srv", True, urifile) srvcmd += " -p {} -s -m".format(self.tempdir) srv2_bin = self.params.get("srv2_bin", '/run/tests/*/') srv2_ctx = self.params.get("srv2_CRT_CTX_NUM", '/run/defaultENV/') srv2_hos = self.params.get("srv2", '/run/hosts/*/') srv2_ppn = self.params.get("srv2_ppn", '/run/tests/*/') hostfile = self.utils.write_host_file(srv2_hos, srv2_ppn) srvcmd += " : {} -x CRT_CTX_NUM={} -N {} --hostfile {} {}".format(self.env, srv2_ctx, srv2_ppn, hostfile, srv2_bin) try: srv_rtn = self.utils.launch_cmd_bg(self, srvcmd) except Exception as e: self.utils.print("Exception in launching server : {}".format(e)) self.fail("Test failed.\n") time.sleep(10) # Verify the server is still running. if not self.utils.check_process(srv_rtn): procrtn = self.utils.stop_process(srv_rtn) self.fail("Server did not launch, return code %s" \ % procrtn) clicmd = self.params.get("cli_bin", '/run/tests/*/') clicmd += " -p {} -s -m".format(self.tempdir) self.utils.launch_test(self, clicmd, srv_rtn) # Stop the server self.utils.print("Stopping server process {}".format(srv_rtn)) procrtn = self.utils.stop_process(srv_rtn) if procrtn: self.fail("Test failed. Server ret code {}".format(procrtn)) def test_multi_tier_without_singleton_attach(self): """ Test CaRT Multi_tier without singleton attach test on two nodes :avocado: tags=all,singleton,multi_tier,two_node """ urifile = self.utils.create_uri_file() srvcmd = self.utils.build_cmd(self, self.env, "srv", True, urifile) srvcmd += " -m" srv2cmd = self.utils.build_cmd(self, self.env, "srv2", False, urifile) try: srv_rtn = self.utils.launch_cmd_bg(self, srvcmd) except Exception as e: self.utils.print("Exception in launching server : {}".format(e)) self.fail("Test failed.\n") time.sleep(10) try: srv2_rtn = self.utils.launch_cmd_bg(self, srv2cmd) except Exception as e: self.utils.print("Exception in launching server : {}".format(e)) self.fail("Test failed.\n") # Verify the server is still running. if not self.utils.check_process(srv_rtn): procrtn = self.utils.stop_process(srv_rtn) self.fail("Server did not launch, return code %s" \ % procrtn) clicmd = self.utils.build_cmd(self, self.env, "cli", False, urifile) clicmd += " -m" self.utils.launch_test(self, clicmd, srv_rtn, srv2_rtn) # Stop the server self.utils.print("Stopping server process {}".format(srv_rtn)) procrtn1 = self.utils.stop_process(srv_rtn) self.utils.print("Stopping server process {}".format(srv2_rtn)) procrtn2 = self.utils.stop_process(srv2_rtn) if procrtn1 or procrtn2: self.fail("Test failed. \ Server ret code {} {}".format(procrtn1, procrtn2))
class CartSingletonOneNodeTest(Test): """ Runs basic CaRT singleton tests :avocado: tags=all,singleton,one_node """ def setUp(self): """ Test setup """ print("Running setup\n") self.utils = CartUtils() self.env = self.utils.get_env(self) self.tempdir = tempfile.mkdtemp() def tearDown(self): """ Test tear down """ print("Run TearDown\n") shutil.rmtree(self.tempdir) def test_cart_singleton(self): """ Test CaRT Singleton :avocado: tags=all,singleton,one_node """ srvcmd = self.utils.build_cmd(self, self.env, "srv") srvcmd += " -p {} -s".format(self.tempdir) try: srv_rtn = self.utils.launch_cmd_bg(self, srvcmd) except Exception as e: self.utils.print("Exception in launching server : {}".format(e)) self.fail("Test failed.\n") time.sleep(5) # Verify the server is still running. if not self.utils.check_process(srv_rtn): procrtn = self.utils.stop_process(srv_rtn) self.fail("Server did not launch, return code %s" \ % procrtn) os.environ["CRT_PHY_ADDR_STR"] = self.params.get("CRT_PHY_ADDR_STR", '/run/defaultENV/') os.environ["OFI_INTERFACE"] = self.params.get("OFI_INTERFACE", '/run/defaultENV/') os.environ["CRT_CTX_NUM"] = self.params.get("cli_CRT_CTX_NUM", '/run/defaultENV/') os.environ["CRT_CTX_SHARE_ADDR"] = self.params.get("CRT_CTX_SHARE_ADDR", '/run/env_CRT_CTX_SHARE_ADDR/*/') clicmd = self.params.get("cli_bin", '/run/tests/*/') clicmd += " -p {} -s".format(self.tempdir) self.utils.launch_test(self, clicmd, srv_rtn) # Stop the server self.utils.print("Stopping server process {}".format(srv_rtn)) procrtn = self.utils.stop_process(srv_rtn) if procrtn: self.fail("Test failed. Server ret code {}".format(procrtn))