def setUp(self):
# Create a nice standardized filesystem name to use.
self.fs_name = "testfs"
self.remote_operations = RealRemoteOperations(self)
# Enable agent debugging
self.remote_operations.enable_agent_debug(self.TEST_SERVERS)
def setUp(self):
super(ApiTestCaseWithTestReset, self).setUp()
self.remote_operations = RealRemoteOperations(self)
storage_servers = [
s for s in self.TEST_SERVERS
if "worker" not in s.get("profile", "")
]
if self.quick_setup is False:
# Ensure that all servers are up and available
for server in storage_servers:
logger.info(
"Checking that %s is running and restarting if necessary..."
% server["fqdn"])
self.remote_operations.await_server_boot(server["fqdn"],
restart=True)
logger.info("%s is running" % server["fqdn"])
self.remote_operations.inject_log_message(
server["fqdn"], "==== "
"starting test %s "
"=====" % self)
if config.get("reset", True):
self.reset_cluster()
elif config.get("soft_reset", True):
# Reset the manager via the API
self.wait_until_true(self.api_contactable)
self.api_force_clear()
self.remote_operations.clear_ha(storage_servers)
[
self.remote_operations.unmount_lustre_targets(x)
for x in storage_servers
if not self.remote_operations.is_worker(x)
]
self.remote_operations.clear_lnet_config(self.TEST_SERVERS)
if config.get("managed"):
# Ensure that config from previous runs doesn't linger into
# this one.
self.remote_operations.remove_config(self.TEST_SERVERS)
# If there are no configuration options for a given server
# (e.g. corosync_config), then this is a noop and no config file
# is written.
self.remote_operations.write_config(self.TEST_SERVERS)
# cleanup linux devices
self.cleanup_linux_devices(storage_servers)
self.cleanup_zpools()
self.create_zpools()
# Enable agent debugging
self.remote_operations.enable_agent_debug(self.TEST_SERVERS)
def setUp(self):
# Create a nice standardized filesystem name to use.
self.fs_name = "testfs"
self.remote_operations = RealRemoteOperations(self)
# Enable agent debugging
self.remote_operations.enable_agent_debug(self.TEST_SERVERS)
self.wait_until_true(self.supervisor_controlled_processes_running)
self.initial_supervisor_controlled_process_start_times = self.get_supervisor_controlled_process_start_times(
)
def test_multicast_works(self):
import multiprocessing
import json
def run_omping(pipe, server, num_requests):
response = self.remote_operations.omping(server,
self.config_servers,
count=num_requests)
pipe.send(json.dumps(response))
num_requests = 5
if config['failover_is_configured']:
self.remote_operations = RealRemoteOperations(self)
pipe_outs = {}
processes = {}
# TODO: This is basically pdsh. Generalize it so that everyone
# can use it.
for server in self.config_servers:
pout, pin = multiprocessing.Pipe()
process = multiprocessing.Process(target=run_omping,
args=(pin, server,
num_requests))
pipe_outs[server['nodename']] = pout
processes[server['nodename']] = process
process.start()
passed = True
stdouts = []
for server in self.config_servers:
omping_result = json.loads(
pipe_outs[server['nodename']].recv())
# This tests if any of the omping pings failed after the first.
# It is fairly common for the first multicast packet to be lost
# while it is still creating the multicast tree.
pattern = re.compile('\(seq>=2 [1-9][0-9]*%\)')
if pattern.search(omping_result):
passed = False
# Store the results for aggregate reporting/logging
stdouts.append("""----------------
%s
-----------------
%s""" % (server['nodename'], omping_result))
# Make sure each omping process terminates
processes[server['nodename']].join()
aggregate_omping_results = "\n" + " ".join(
[stdout for stdout in stdouts])
logger.debug("Omping results: %s" % aggregate_omping_results)
self.assertTrue(passed, aggregate_omping_results)
def setUp(self):
# Create a nice standardized filesystem name to use.
self.fs_name = "testfs"
# connect the remote operations but otherwise...
if config.get('simulator', False):
self.remote_operations = SimulatorRemoteOperations(self, self.simulator)
else:
self.remote_operations = RealRemoteOperations(self)
# Enable agent debugging
self.remote_operations.enable_agent_debug(self.TEST_SERVERS)
self.wait_until_true(self.supervisor_controlled_processes_running)
self.initial_supervisor_controlled_process_start_times = self.get_supervisor_controlled_process_start_times()
def _check_status(address, additional_check_items):
logger.debug("*" * 40)
logger.debug("Checking status of %s" % address)
check_items = [
("Services running", "ps ax | grep python"),
("Installed Packages", "rpm -qa"),
("Verify Packages", "rpm -Va"),
("Network status", "netstat"),
] + additional_check_items
# Catch all exceptions so our caller doesn't have it's exceptions overwritten.
try:
remote_ops = RealRemoteOperations(None)
if remote_ops.host_contactable(address):
logger.debug("Host %s was contactable via SSH" % address)
for action, command in check_items:
logger.debug("#" * 40)
remote_command_result = remote_ops._ssh_address(
address, command, None)
if remote_command_result.rc == 0:
logger.debug(
"%s on %s\n%s" %
(action, address, remote_command_result.stdout.read()))
else:
logger.debug(
"Error capturing %s on %s: result %s, stdout %s, stderr %s"
% (
action,
address,
remote_command_result.rc,
remote_command_result.stdout.read(),
remote_command_result.stderr.read(),
))
else:
logger.debug("Host %s was not contactable via SSH")
except:
pass
logger.debug("*" * 40)
def setUp(self):
super(TestFirewall, self).setUp()
self.remote_operations = RealRemoteOperations(self)
def setUp(self):
super(ApiTestCaseWithTestReset, self).setUp()
if config.get('simulator', False):
# When we're running with the simulator, parts of the simulated
# Copytools use agent code, and the agent code expects to find
# a populated agent-side configuration. The safe way to handle
# this requirement is to use mock to patch in a fresh
# ConfigStore instance for each test run.
try:
from chroma_agent.config_store import ConfigStore
except ImportError:
raise ImportError(
"Cannot import agent, do you need to do a 'setup.py develop' of it?"
)
import mock # Mock is only available when running the simulator, hence local inclusion
self.mock_config = ConfigStore(tempfile.mkdtemp())
mock.patch('chroma_agent.config', self.mock_config).start()
from chroma_agent.action_plugins.settings_management import reset_agent_config, set_agent_config
reset_agent_config()
# Allow the worker to create a fifo in /tmp rather than /var/spool
set_agent_config('copytool_fifo_directory',
self.COPYTOOL_TESTING_FIFO_ROOT)
try:
from cluster_sim.simulator import ClusterSimulator
except ImportError:
raise ImportError(
"Cannot import simulator, do you need to do a 'setup.py develop' of it?"
)
# The simulator's state directory will be left behind when a test fails,
# so make sure it has a unique-per-run name
timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H%M%S")
state_path = 'simulator_state_%s.%s_%s' % (
self.__class__.__name__, self._testMethodName, timestamp)
if os.path.exists(state_path):
raise RuntimeError(
"Simulator state folder already exists at '%s'!" %
state_path)
# Hook up the agent log to a file
from chroma_agent.agent_daemon import daemon_log
handler = logging.FileHandler(
os.path.join(config.get('log_dir', '/var/log/'),
'chroma_test_agent.log'))
handler.setFormatter(
logging.Formatter('[%(asctime)s] %(message)s',
'%d/%b/%Y:%H:%M:%S'))
daemon_log.addHandler(handler)
daemon_log.setLevel(logging.DEBUG)
self.simulator = ClusterSimulator(
state_path, config['chroma_managers'][0]['server_http_url'])
volume_count = max(
[len(s['device_paths']) for s in self.config_servers])
self.simulator.setup(len(self.config_servers),
len(self.config_workers),
volume_count,
self.SIMULATOR_NID_COUNT,
self.SIMULATOR_CLUSTER_SIZE,
len(config['power_distribution_units']),
su_size=0)
self.remote_operations = SimulatorRemoteOperations(
self, self.simulator)
if self.TESTS_NEED_POWER_CONTROL:
self.simulator.power.start()
else:
self.remote_operations = RealRemoteOperations(self)
if self.quick_setup is False:
# Ensure that all servers are up and available
for server in self.TEST_SERVERS:
logger.info(
"Checking that %s is running and restarting if necessary..."
% server['fqdn'])
self.remote_operations.await_server_boot(server['fqdn'],
restart=True)
logger.info("%s is running" % server['fqdn'])
self.remote_operations.inject_log_message(
server['fqdn'], "==== "
"starting test %s "
"=====" % self)
if config.get('reset', True):
self.reset_cluster()
elif config.get('soft_reset', True):
# Reset the manager via the API
self.wait_until_true(self.api_contactable)
self.remote_operations.unmount_clients()
self.api_force_clear()
self.remote_operations.clear_ha(self.TEST_SERVERS)
self.remote_operations.clear_lnet_config(self.TEST_SERVERS)
if config.get('managed'):
# Ensure that config from previous runs doesn't linger into
# this one.
self.remote_operations.remove_config(self.TEST_SERVERS)
# If there are no configuration options for a given server
# (e.g. corosync_config), then this is a noop and no config file
# is written.
self.remote_operations.write_config(self.TEST_SERVERS)
# cleanup linux devices
self.cleanup_linux_devices(self.TEST_SERVERS)
# cleanup zfs pools
self.cleanup_zfs_pools(
self.config_servers, self.CZP_EXPORTPOOLS |
(self.CZP_RECREATEZPOOLS if config.get(
'new_zpools_each_test', False) else
self.CZP_REMOVEDATASETS), None, True)
# Enable agent debugging
self.remote_operations.enable_agent_debug(self.TEST_SERVERS)
self.wait_until_true(self.supervisor_controlled_processes_running)
self.initial_supervisor_controlled_process_start_times = self.get_supervisor_controlled_process_start_times(
)
