def test_instantiation(self):
"""Raise if instantiated directly, should return an instance of BaseWorker if inherited."""
target = transports.Target(cumin.nodeset('node1'))
with pytest.raises(TypeError):
transports.BaseWorker({}, target) # pylint: disable=abstract-class-instantiated
assert isinstance(
ConcreteBaseWorker({},
transports.Target(cumin.nodeset('node[1-2]'))),
transports.BaseWorker)
def test_init_batch_sleep(self):
"""Creating a Target instance with a batch_sleep should set it to it's value, if valid."""
target = transports.Target(self.hosts, batch_sleep=5.0)
assert target.batch_sleep == pytest.approx(5.0)
target = transports.Target(self.hosts, batch_sleep=None)
assert target.batch_sleep == pytest.approx(0.0)
with pytest.raises(transports.WorkerError):
transports.Target(self.hosts, batch_sleep=0)
with pytest.raises(transports.WorkerError):
transports.Target(self.hosts, batch_sleep=-1.0)
def test_init_batch_size(self, mocked_logger):
"""Creating a Target instance with a batch_size should set it to it's value, if valid."""
target = transports.Target(self.hosts, batch_size=5)
assert target.batch_size == 5
target = transports.Target(self.hosts, batch_size=len(self.hosts) + 1)
assert target.batch_size == len(self.hosts)
assert mocked_logger.called
target = transports.Target(self.hosts, batch_size=None)
assert target.batch_size == len(self.hosts)
with pytest.raises(transports.WorkerError,
match='must be a positive integer'):
transports.Target(self.hosts, batch_size=0)
def test_first_batch(self):
"""The first_batch property should return the first_batch of hosts."""
size = 5
target = transports.Target(self.hosts, batch_size=size)
assert len(target.first_batch) == size
assert target.first_batch == cumin.nodeset_fromlist(self.hosts[:size])
assert isinstance(target.first_batch, NodeSet)
def run(self, host, command):
hosts = query.Query(self.config).execute(host)
if not hosts:
return CommandReturn(1, None, "host is wrong or does not match rules")
target = transports.Target(hosts)
worker = transport.Transport.new(self.config, target)
worker.commands = [self.format_command(command)]
worker.handler = "sync"
# If verbose is false, suppress stdout and stderr of Cumin.
if self.options.get("verbose", False):
return_code = worker.execute()
else:
# Temporary workaround until Cumin has full support to suppress output (T212783).
stdout = transports.clustershell.sys.stdout
stderr = transports.clustershell.sys.stderr
try:
with open(os.devnull, "w") as discard_output:
transports.clustershell.sys.stdout = discard_output
transports.clustershell.sys.stderr = discard_output
return_code = worker.execute()
finally:
transports.clustershell.sys.stdout = stdout
transports.clustershell.sys.stderr = stderr
for nodes, output in worker.get_results():
if host in nodes:
result = str(bytes(output), "utf-8")
return CommandReturn(return_code, result, None)
return CommandReturn(return_code, None, None)
def run_cumin(label, hosts_query, commands, timeout=30, installer=False, ignore_exit=False):
"""Run a remote command via Cumin.
Arguments:
label -- label to identify the caller in messages and logs
hosts_query -- the query for the hosts selection to pass to cumin
commands -- the list of commands to be executed
timeout -- a timeout in seconds for each command. [optional, default: 30]
installer -- whether the host will reboot into the installer or not,
"""
if installer:
config = cumin_config_installer
if 'SSH_AUTH_SOCK' in os.environ:
del os.environ['SSH_AUTH_SOCK']
else:
config = cumin_config
hosts = query.Query(config).execute(hosts_query)
target = transports.Target(hosts)
worker = transport.Transport.new(config, target)
ok_codes = None
if ignore_exit:
ok_codes = []
worker.commands = [transports.Command(command, timeout=timeout, ok_codes=ok_codes)
for command in commands]
worker.handler = 'async'
exit_code = worker.execute()
if exit_code != 0:
raise RuntimeError('Failed to {label}'.format(label=label))
return exit_code, worker
def test_missing_worker_class():
"""Passing a transport without a defined worker_class should raise CuminError."""
module = mock.MagicMock()
del module.worker_class
with mock.patch('importlib.import_module', lambda _: module):
with pytest.raises(CuminError, match=r'worker_class'):
Transport.new({'transport': 'invalid_transport'},
transports.Target(['host1']))
def test_invalid_transport():
"""Passing an invalid transport should raise CuminError."""
with pytest.raises(
CuminError,
match=r"No module named 'cumin\.transports\.non_existent_transport'"
):
Transport.new({'transport': 'non_existent_transport'},
transports.Target(['host1']))
def test_init_batch_size_perc(self):
"""Creating a Target instance with a batch_size_ratio should set batch_size to the appropriate value."""
target = transports.Target(self.hosts, batch_size_ratio=0.5)
assert target.batch_size == 5
target = transports.Target(self.hosts, batch_size_ratio=1.0)
assert target.batch_size == len(self.hosts)
target = transports.Target(self.hosts, batch_size_ratio=None)
assert target.batch_size == len(self.hosts)
with pytest.raises(transports.WorkerError,
match='parameters are mutually exclusive'):
transports.Target(self.hosts, batch_size=1, batch_size_ratio=0.5)
with pytest.raises(transports.WorkerError,
match='has generated a batch_size of 0 hosts'):
transports.Target(self.hosts, batch_size_ratio=0.0)
def test_init(self):
"""Constructor should save config and set environment variables."""
env_dict = {'ENV_VARIABLE': 'env_value'}
config = {'transport': 'test_transport', 'environment': env_dict}
assert transports.os.environ == {}
worker = ConcreteBaseWorker(
config, transports.Target(cumin.nodeset('node[1-2]')))
assert transports.os.environ == env_dict
assert worker.config == config
def setup_method(self, _):
"""Initialize default properties and instances."""
# pylint: disable=attribute-defined-outside-init
self.worker = ConcreteBaseWorker({},
transports.Target(
cumin.nodeset('node[1-2]')))
self.commands = [
transports.Command('command1'),
transports.Command('command2')
]
def run(self, host, command):
hosts = query.Query(self.config).execute(host)
target = transports.Target(hosts)
worker = transport.Transport.new(self.config, target)
worker.commands = [self.format_command(command)]
worker.handler = 'sync'
return_code = worker.execute()
for nodes, output in worker.get_results():
if host in nodes:
result = str(bytes(output), 'utf-8')
return CommandReturn(return_code, result, None)
return CommandReturn(return_code, None, None)
def run(args, config):
"""Execute the commands on the selected hosts and print the results.
Arguments:
args: ArgumentParser instance with parsed command line arguments
config: a dictionary with the parsed configuration file
"""
hosts = get_hosts(args, config)
if not hosts:
return 0
target = transports.Target(hosts,
batch_size=args.batch_size['value'],
batch_size_ratio=args.batch_size['ratio'],
batch_sleep=args.batch_sleep)
worker = transport.Transport.new(config, target)
ok_codes = None
if args.ignore_exit_codes:
ok_codes = []
worker.commands = [
transports.Command(command, timeout=args.timeout, ok_codes=ok_codes)
for command in args.commands
]
worker.timeout = args.global_timeout
worker.handler = args.mode
worker.success_threshold = args.success_percentage / 100
exit_code = worker.execute()
if args.interactive:
# Define a help function h() that will be available in the interactive shell to print the help message.
# The name is to not shadow the Python built-in help() that might be usefult too to inspect objects.
def h(): # pylint: disable=possibly-unused-variable,invalid-name
"""Print the help message in interactive shell."""
tqdm.write(INTERACTIVE_BANNER)
code.interact(banner=INTERACTIVE_BANNER, local=locals())
elif args.output is not None:
tqdm.write(OUTPUT_SEPARATOR)
print_output(args.output, worker)
return exit_code
def test_init_batch_size_perc_range(self, ratio):
"""Creating a Target instance with an invalid batch_size_ratio should raise WorkerError."""
with pytest.raises(transports.WorkerError,
match='must be a float between 0.0 and 1.0'):
transports.Target(self.hosts, batch_size_ratio=ratio)
def _execute( # pylint: disable=too-many-arguments
self,
commands: Sequence[Union[str, Command]],
mode: str = "sync",
success_threshold: float = 1.0,
batch_size: Optional[Union[int, str]] = None,
batch_sleep: Optional[float] = None,
is_safe: bool = False,
print_output: bool = True,
print_progress_bars: bool = True,
) -> Iterator[Tuple[NodeSet, MsgTreeElem]]:
"""Lower level Cumin's execution of commands on the target nodes.
Arguments:
commands (list): the list of commands to execute on the target hosts, either a list of commands or a list
of cumin.transports.Command instances.
mode (str, optional): the Cumin's mode of execution. Accepted values: sync, async.
success_threshold (float, optional): to consider the execution successful, must be between 0.0 and 1.0.
batch_size (int, str, optional): the batch size for cumin, either as percentage (e.g. ``25%``) or absolute
number (e.g. ``5``).
batch_sleep (float, optional): the batch sleep in seconds to use in Cumin before scheduling the next host.
is_safe (bool, optional): whether the command is safe to run also in dry-run mode because it's a read-only
command that doesn't modify the state.
print_output (bool, optional): whether to print Cumin's output to stdout.
print_progress_bars (bool, optional): whether to print Cumin's progress bars to stderr.
Returns:
generator: as returned by :py:meth:`cumin.transports.BaseWorker.get_results` to iterate over the results.
Raises:
RemoteExecutionError: if the Cumin execution returns a non-zero exit code.
"""
if batch_size is None:
parsed_batch_size = {"value": None, "ratio": None}
else:
parsed_batch_size = target_batch_size(str(batch_size))
if self._use_sudo:
commands = [self._prepend_sudo(command) for command in commands]
target = transports.Target(
self._hosts,
batch_size=parsed_batch_size["value"],
batch_size_ratio=parsed_batch_size["ratio"],
batch_sleep=batch_sleep,
)
worker = transport.Transport.new(self._config, target)
worker.commands = commands
worker.handler = mode
worker.success_threshold = success_threshold
worker.progress_bars = print_progress_bars
if print_output:
worker.reporter = TqdmReporter
else:
worker.reporter = NullReporter
logger.debug(
"Executing commands %s on %d hosts: %s",
commands,
len(target.hosts),
str(target.hosts),
)
if self._dry_run and not is_safe:
return iter(()) # Empty generator
ret = worker.execute()
if ret != 0 and not self._dry_run:
raise RemoteExecutionError(ret, "Cumin execution failed")
return worker.get_results()
def test_valid_transport(transport):
"""Passing a valid transport should return an instance of BaseWorker."""
assert isinstance(
Transport.new({'transport': transport}, transports.Target(['host1'])),
transports.BaseWorker)
def test_init_list(self):
"""Creating a Target instance with a list and without optional parameter should return their defaults."""
target = transports.Target(self.hosts_list)
assert target.hosts == self.hosts
assert target.batch_size == len(self.hosts)
assert target.batch_sleep == 0.0
def test_init_invalid(self):
"""Creating a Target instance with invalid hosts should raise WorkerError."""
with pytest.raises(
transports.WorkerError,
match="must be a non-empty ClusterShell NodeSet or list"):
transports.Target(set(self.hosts_list))
def test_missing_transport():
"""Not passing a transport should raise CuminError."""
with pytest.raises(CuminError,
match=r"Missing required parameter 'transport'"):
Transport.new({}, transports.Target(['host1']))
