def test_instantiation(self): """Raise if instantiated directly, should return an instance of BaseWorker if inherited.""" target = transports.Target(cumin.nodeset('node1')) with pytest.raises(TypeError): transports.BaseWorker({}, target) # pylint: disable=abstract-class-instantiated assert isinstance( ConcreteBaseWorker({}, transports.Target(cumin.nodeset('node[1-2]'))), transports.BaseWorker)
def test_init_batch_sleep(self): """Creating a Target instance with a batch_sleep should set it to it's value, if valid.""" target = transports.Target(self.hosts, batch_sleep=5.0) assert target.batch_sleep == pytest.approx(5.0) target = transports.Target(self.hosts, batch_sleep=None) assert target.batch_sleep == pytest.approx(0.0) with pytest.raises(transports.WorkerError): transports.Target(self.hosts, batch_sleep=0) with pytest.raises(transports.WorkerError): transports.Target(self.hosts, batch_sleep=-1.0)
def test_init_batch_size(self, mocked_logger): """Creating a Target instance with a batch_size should set it to it's value, if valid.""" target = transports.Target(self.hosts, batch_size=5) assert target.batch_size == 5 target = transports.Target(self.hosts, batch_size=len(self.hosts) + 1) assert target.batch_size == len(self.hosts) assert mocked_logger.called target = transports.Target(self.hosts, batch_size=None) assert target.batch_size == len(self.hosts) with pytest.raises(transports.WorkerError, match='must be a positive integer'): transports.Target(self.hosts, batch_size=0)
def test_first_batch(self): """The first_batch property should return the first_batch of hosts.""" size = 5 target = transports.Target(self.hosts, batch_size=size) assert len(target.first_batch) == size assert target.first_batch == cumin.nodeset_fromlist(self.hosts[:size]) assert isinstance(target.first_batch, NodeSet)
def run(self, host, command): hosts = query.Query(self.config).execute(host) if not hosts: return CommandReturn(1, None, "host is wrong or does not match rules") target = transports.Target(hosts) worker = transport.Transport.new(self.config, target) worker.commands = [self.format_command(command)] worker.handler = "sync" # If verbose is false, suppress stdout and stderr of Cumin. if self.options.get("verbose", False): return_code = worker.execute() else: # Temporary workaround until Cumin has full support to suppress output (T212783). stdout = transports.clustershell.sys.stdout stderr = transports.clustershell.sys.stderr try: with open(os.devnull, "w") as discard_output: transports.clustershell.sys.stdout = discard_output transports.clustershell.sys.stderr = discard_output return_code = worker.execute() finally: transports.clustershell.sys.stdout = stdout transports.clustershell.sys.stderr = stderr for nodes, output in worker.get_results(): if host in nodes: result = str(bytes(output), "utf-8") return CommandReturn(return_code, result, None) return CommandReturn(return_code, None, None)
def run_cumin(label, hosts_query, commands, timeout=30, installer=False, ignore_exit=False): """Run a remote command via Cumin. Arguments: label -- label to identify the caller in messages and logs hosts_query -- the query for the hosts selection to pass to cumin commands -- the list of commands to be executed timeout -- a timeout in seconds for each command. [optional, default: 30] installer -- whether the host will reboot into the installer or not, """ if installer: config = cumin_config_installer if 'SSH_AUTH_SOCK' in os.environ: del os.environ['SSH_AUTH_SOCK'] else: config = cumin_config hosts = query.Query(config).execute(hosts_query) target = transports.Target(hosts) worker = transport.Transport.new(config, target) ok_codes = None if ignore_exit: ok_codes = [] worker.commands = [transports.Command(command, timeout=timeout, ok_codes=ok_codes) for command in commands] worker.handler = 'async' exit_code = worker.execute() if exit_code != 0: raise RuntimeError('Failed to {label}'.format(label=label)) return exit_code, worker
def test_missing_worker_class(): """Passing a transport without a defined worker_class should raise CuminError.""" module = mock.MagicMock() del module.worker_class with mock.patch('importlib.import_module', lambda _: module): with pytest.raises(CuminError, match=r'worker_class'): Transport.new({'transport': 'invalid_transport'}, transports.Target(['host1']))
def test_invalid_transport(): """Passing an invalid transport should raise CuminError.""" with pytest.raises( CuminError, match=r"No module named 'cumin\.transports\.non_existent_transport'" ): Transport.new({'transport': 'non_existent_transport'}, transports.Target(['host1']))
def test_init_batch_size_perc(self): """Creating a Target instance with a batch_size_ratio should set batch_size to the appropriate value.""" target = transports.Target(self.hosts, batch_size_ratio=0.5) assert target.batch_size == 5 target = transports.Target(self.hosts, batch_size_ratio=1.0) assert target.batch_size == len(self.hosts) target = transports.Target(self.hosts, batch_size_ratio=None) assert target.batch_size == len(self.hosts) with pytest.raises(transports.WorkerError, match='parameters are mutually exclusive'): transports.Target(self.hosts, batch_size=1, batch_size_ratio=0.5) with pytest.raises(transports.WorkerError, match='has generated a batch_size of 0 hosts'): transports.Target(self.hosts, batch_size_ratio=0.0)
def test_init(self): """Constructor should save config and set environment variables.""" env_dict = {'ENV_VARIABLE': 'env_value'} config = {'transport': 'test_transport', 'environment': env_dict} assert transports.os.environ == {} worker = ConcreteBaseWorker( config, transports.Target(cumin.nodeset('node[1-2]'))) assert transports.os.environ == env_dict assert worker.config == config
def setup_method(self, _): """Initialize default properties and instances.""" # pylint: disable=attribute-defined-outside-init self.worker = ConcreteBaseWorker({}, transports.Target( cumin.nodeset('node[1-2]'))) self.commands = [ transports.Command('command1'), transports.Command('command2') ]
def run(self, host, command): hosts = query.Query(self.config).execute(host) target = transports.Target(hosts) worker = transport.Transport.new(self.config, target) worker.commands = [self.format_command(command)] worker.handler = 'sync' return_code = worker.execute() for nodes, output in worker.get_results(): if host in nodes: result = str(bytes(output), 'utf-8') return CommandReturn(return_code, result, None) return CommandReturn(return_code, None, None)
def run(args, config): """Execute the commands on the selected hosts and print the results. Arguments: args: ArgumentParser instance with parsed command line arguments config: a dictionary with the parsed configuration file """ hosts = get_hosts(args, config) if not hosts: return 0 target = transports.Target(hosts, batch_size=args.batch_size['value'], batch_size_ratio=args.batch_size['ratio'], batch_sleep=args.batch_sleep) worker = transport.Transport.new(config, target) ok_codes = None if args.ignore_exit_codes: ok_codes = [] worker.commands = [ transports.Command(command, timeout=args.timeout, ok_codes=ok_codes) for command in args.commands ] worker.timeout = args.global_timeout worker.handler = args.mode worker.success_threshold = args.success_percentage / 100 exit_code = worker.execute() if args.interactive: # Define a help function h() that will be available in the interactive shell to print the help message. # The name is to not shadow the Python built-in help() that might be usefult too to inspect objects. def h(): # pylint: disable=possibly-unused-variable,invalid-name """Print the help message in interactive shell.""" tqdm.write(INTERACTIVE_BANNER) code.interact(banner=INTERACTIVE_BANNER, local=locals()) elif args.output is not None: tqdm.write(OUTPUT_SEPARATOR) print_output(args.output, worker) return exit_code
def test_init_batch_size_perc_range(self, ratio): """Creating a Target instance with an invalid batch_size_ratio should raise WorkerError.""" with pytest.raises(transports.WorkerError, match='must be a float between 0.0 and 1.0'): transports.Target(self.hosts, batch_size_ratio=ratio)
def _execute( # pylint: disable=too-many-arguments self, commands: Sequence[Union[str, Command]], mode: str = "sync", success_threshold: float = 1.0, batch_size: Optional[Union[int, str]] = None, batch_sleep: Optional[float] = None, is_safe: bool = False, print_output: bool = True, print_progress_bars: bool = True, ) -> Iterator[Tuple[NodeSet, MsgTreeElem]]: """Lower level Cumin's execution of commands on the target nodes. Arguments: commands (list): the list of commands to execute on the target hosts, either a list of commands or a list of cumin.transports.Command instances. mode (str, optional): the Cumin's mode of execution. Accepted values: sync, async. success_threshold (float, optional): to consider the execution successful, must be between 0.0 and 1.0. batch_size (int, str, optional): the batch size for cumin, either as percentage (e.g. ``25%``) or absolute number (e.g. ``5``). batch_sleep (float, optional): the batch sleep in seconds to use in Cumin before scheduling the next host. is_safe (bool, optional): whether the command is safe to run also in dry-run mode because it's a read-only command that doesn't modify the state. print_output (bool, optional): whether to print Cumin's output to stdout. print_progress_bars (bool, optional): whether to print Cumin's progress bars to stderr. Returns: generator: as returned by :py:meth:`cumin.transports.BaseWorker.get_results` to iterate over the results. Raises: RemoteExecutionError: if the Cumin execution returns a non-zero exit code. """ if batch_size is None: parsed_batch_size = {"value": None, "ratio": None} else: parsed_batch_size = target_batch_size(str(batch_size)) if self._use_sudo: commands = [self._prepend_sudo(command) for command in commands] target = transports.Target( self._hosts, batch_size=parsed_batch_size["value"], batch_size_ratio=parsed_batch_size["ratio"], batch_sleep=batch_sleep, ) worker = transport.Transport.new(self._config, target) worker.commands = commands worker.handler = mode worker.success_threshold = success_threshold worker.progress_bars = print_progress_bars if print_output: worker.reporter = TqdmReporter else: worker.reporter = NullReporter logger.debug( "Executing commands %s on %d hosts: %s", commands, len(target.hosts), str(target.hosts), ) if self._dry_run and not is_safe: return iter(()) # Empty generator ret = worker.execute() if ret != 0 and not self._dry_run: raise RemoteExecutionError(ret, "Cumin execution failed") return worker.get_results()
def test_valid_transport(transport): """Passing a valid transport should return an instance of BaseWorker.""" assert isinstance( Transport.new({'transport': transport}, transports.Target(['host1'])), transports.BaseWorker)
def test_init_list(self): """Creating a Target instance with a list and without optional parameter should return their defaults.""" target = transports.Target(self.hosts_list) assert target.hosts == self.hosts assert target.batch_size == len(self.hosts) assert target.batch_sleep == 0.0
def test_init_invalid(self): """Creating a Target instance with invalid hosts should raise WorkerError.""" with pytest.raises( transports.WorkerError, match="must be a non-empty ClusterShell NodeSet or list"): transports.Target(set(self.hosts_list))
def test_missing_transport(): """Not passing a transport should raise CuminError.""" with pytest.raises(CuminError, match=r"Missing required parameter 'transport'"): Transport.new({}, transports.Target(['host1']))