def test_tty( self, dcos_node: Node, tty: bool, ) -> None: """ If the ``tty`` parameter is set to ``True``, a TTY is created. """ filename = uuid.uuid4().hex script = textwrap.dedent( """ if [ -t 1 ] then echo True > {filename} else echo False > {filename} fi """, ).format(filename=filename) echo_result = dcos_node.run( args=[script], tty=tty, shell=True, ) assert echo_result.returncode == 0 run_result = dcos_node.run(args=['cat', filename]) assert run_result.stdout.strip().decode() == str(tty)
def test_send_file_to_directory( self, dcos_node: Node, tmpdir: local, ) -> None: """ It is possible to send a file to a cluster node to a directory that is mounted as tmpfs. See ``DockerExecTransport.send_file`` for details. """ content = str(uuid.uuid4()) file_name = 'example_file.txt' local_file = tmpdir.join(file_name) local_file.write(content) master_destination_path = Path( '/etc/{random}'.format(random=uuid.uuid4().hex), ) dcos_node.run(args=['mkdir', '--parent', str(master_destination_path)]) dcos_node.send_file( local_path=Path(str(local_file)), remote_path=master_destination_path, ) args = ['cat', str(master_destination_path / file_name)] result = dcos_node.run(args=args) assert result.stdout.decode() == content
def test_error(self, dcos_node: Node) -> None: """ Commands which return a non-0 code raise a ``CalledProcessError``. """ with pytest.raises(CalledProcessError) as excinfo: dcos_node.run(args=['rm', 'does_not_exist']) exception = excinfo.value assert exception.returncode == 1
def check_bootstrap(node: Node) -> None: # Check that bootstrap works - `dcos-cluster-id` checks the cluster id, # which demonstrates that consensus checking is working node.run( [ '/opt/mesosphere/bin/dcos-shell', '/opt/mesosphere/bin/bootstrap', 'dcos-cluster-id' ], output=Output.LOG_AND_CAPTURE, )
def test_errors(self, dcos_node: Node, output: Output) -> None: """ The ``stderr`` of a failed command is available in the raised ``subprocess.CalledProcessError``. """ args = ['rm', 'does_not_exist'] with pytest.raises(subprocess.CalledProcessError) as excinfo: dcos_node.run(args=args, shell=True, output=output) expected_message = b'No such file or directory' assert expected_message in excinfo.value.stderr
def test_log_output_live_and_tty(self, dcos_node: Node) -> None: """ A ``ValueError`` is raised if ``tty`` is ``True`` and ``log_output_live`` is ``True``. """ with pytest.raises(ValueError) as excinfo: dcos_node.run( args=['echo', '1'], log_output_live=True, tty=True, ) expected_message = '`log_output_live` and `tty` cannot both be `True`.' assert str(excinfo.value) == expected_message
def _do_backup(master: Node, backup_local_path: Path) -> None: """ Automated ZooKeeper backup procedure. Intended to be consistent with the documentation. https://jira.mesosphere.com/browse/DCOS-51647 """ master.run(args=['systemctl', 'stop', 'dcos-exhibitor']) backup_name = backup_local_path.name # This must be an existing directory on the remote server. backup_remote_path = Path('/etc/') / backup_name master.run( args=[ '/opt/mesosphere/bin/dcos-shell', 'dcos-zk', 'backup', str(backup_remote_path), '-v', ], output=Output.LOG_AND_CAPTURE, ) master.run(args=['systemctl', 'start', 'dcos-exhibitor']) master.download_file( remote_path=backup_remote_path, local_path=backup_local_path, ) master.run(args=['rm', str(backup_remote_path)])
def test_log_and_capture( self, caplog: LogCaptureFixture, dcos_node: Node, stdout_message: str, stderr_message: str, ) -> None: """ When given ``Output.LOG_AND_CAPTURE``, stderr and stdout are captured in the output as stdout. stdout and stderr are logged. """ args = ['echo', stdout_message, '&&', '>&2', 'echo', stderr_message] result = dcos_node.run( args=args, shell=True, output=Output.LOG_AND_CAPTURE, ) # stderr is merged into stdout. # This is not ideal but for now it is the case. # The order is not necessarily preserved. expected_messages = set([stdout_message, stderr_message]) result_stdout = result.stdout.strip().decode() assert set(result_stdout.split('\n')) == expected_messages first_log, second_log = caplog.records assert first_log.levelno == logging.DEBUG assert second_log.levelno == logging.DEBUG messages = set([first_log.message, second_log.message]) assert messages == expected_messages
def test_capture( self, caplog: LogCaptureFixture, dcos_node: Node, stdout_message: str, stderr_message: str, ) -> None: """ When given ``Output.CAPTURE``, stderr and stdout are captured in the output. stderr is logged. """ args = ['echo', stdout_message, '&&', '>&2', 'echo', stderr_message] result = dcos_node.run(args=args, output=Output.CAPTURE, shell=True) assert result.stdout.strip().decode() == stdout_message assert result.stderr.strip().decode() == stderr_message args_log, result_log = caplog.records assert args_log.levelno == logging.WARNING assert stdout_message in args_log.message assert stderr_message in args_log.message assert 'echo' in args_log.message assert result_log.levelno == logging.WARNING assert result_log.message == stderr_message
def test_default( self, caplog: LogCaptureFixture, dcos_node: Node, ) -> None: """ By default, stderr and stdout are captured in the output. stderr is logged. """ stdout_message = uuid.uuid4().hex stderr_message = uuid.uuid4().hex args = ['echo', stdout_message, '&&', '>&2', 'echo', stderr_message] result = dcos_node.run(args=args, shell=True) assert result.stdout.strip().decode() == stdout_message assert result.stderr.strip().decode() == stderr_message args_log, result_log = caplog.records assert args_log.levelno == logging.WARNING assert stdout_message in args_log.message assert stderr_message in args_log.message assert 'echo' in args_log.message assert result_log.levelno == logging.WARNING assert result_log.message == stderr_message
def test_tty( self, dcos_node: Node, tty: bool, ) -> None: """ If the ``tty`` parameter is set to ``True``, a TTY is created. """ filename = uuid.uuid4().hex script = textwrap.dedent( """ if [ -t 1 ] then echo True else echo False fi """, ).format(filename=filename) echo_result = dcos_node.run( args=[script], tty=tty, shell=True, ) if not sys.stdout.isatty(): # pragma: no cover reason = ('For this test to be valid, stdout must be a TTY. ' 'Use ``--capture=no / -s`` to run this test.') raise pytest.skip(reason) else: # pragma: no cover assert echo_result.returncode == 0 assert echo_result.stdout.strip().decode() == str(tty)
def test_log_and_capture_stderr( self, caplog: LogCaptureFixture, dcos_node: Node, message: str, ) -> None: """ When using ``Output.LOG_AND_CAPTURE``, stderr is logged and captured. """ args = ['>&2', 'echo', message] result = dcos_node.run( args=args, shell=True, output=Output.LOG_AND_CAPTURE, ) expected_command = ( 'Running command `/bin/sh -c >&2 echo {message}` on a node ' '`{node}`').format( message=message, node=str(dcos_node), ) assert result.stderr.strip().decode() == message command_log, first_log = caplog.records assert first_log.levelno == logging.WARN assert command_log.message == expected_command assert message == first_log.message
def _get_node_distribution(node: Node) -> Distribution: """ Given a ``Node``, return the ``Distribution`` on that node. """ cat_cmd = node.run( args=['cat /etc/*-release'], shell=True, ) version_info = cat_cmd.stdout version_info_lines = [ line for line in version_info.decode().split('\n') if '=' in line ] version_data = dict(item.split('=') for item in version_info_lines) distributions = { ('"centos"', '"7"'): Distribution.CENTOS_7, ('"rhel"', '"7.4"'): Distribution.RHEL_7, ('coreos', '1911.3.0'): Distribution.COREOS, ('coreos', '1632.3.0'): Distribution.COREOS, } distro_id = version_data['ID'].strip() distro_version_id = version_data['VERSION_ID'].strip() return distributions[(distro_id, distro_version_id)]
def test_errors( self, caplog: LogCaptureFixture, dcos_node: Node, output: Output, ) -> None: """ Errors are always logged at the error level. """ args = ['rm', 'does_not_exist'] output = Output.CAPTURE with pytest.raises(subprocess.CalledProcessError): dcos_node.run(args=args, shell=True, output=output) [record] = caplog.records assert record.levelno == logging.ERROR expected_message = 'No such file or directory' assert expected_message in record.message
def test_not_utf_8_log_and_capture( self, caplog: LogCaptureFixture, dcos_node: Node, ) -> None: """ It is possible to see output of commands which output non-utf-8 bytes using ``output.LOG_AND_CAPTURE``. """ # We expect that this will trigger a UnicodeDecodeError when run on a # node, if the result is meant to be decoded with utf-8. # It also is not so long that it will kill our terminal. args = ['head', '-c', '100', '/bin/cat'] dcos_node.run(args=args, output=Output.LOG_AND_CAPTURE) # We do not test the output, but we at least test its length for now. [log] = caplog.records assert len(log.message) >= 100
def test_stderr(self, dcos_node: Node) -> None: """ ``stderr`` is send to the result's ``stderr`` property. """ echo_result = dcos_node.run(args=['echo', '1', '1>&2'], shell=True) assert echo_result.returncode == 0 assert echo_result.stdout.strip() == b'' assert echo_result.stderr.strip() == b'1'
def test_custom_user( self, dcos_node: Node, tmpdir: local, ) -> None: """ It is possible to send a file to a cluster node as a custom user. """ testuser = str(uuid.uuid4().hex) dcos_node.run(args=['useradd', testuser]) dcos_node.run( args=['cp', '-R', '$HOME/.ssh', '/home/{}/'.format(testuser)], shell=True, ) random = str(uuid.uuid4()) local_file = tmpdir.join('example_file.txt') local_file.write(random) master_destination_dir = '/home/{testuser}/{random}'.format( testuser=testuser, random=random, ) master_destination_path = Path(master_destination_dir) / 'file.txt' dcos_node.send_file( local_path=Path(str(local_file)), remote_path=master_destination_path, user=testuser, ) args = ['stat', '-c', '"%U"', str(master_destination_path)] result = dcos_node.run(args=args, shell=True) assert result.stdout.decode().strip() == testuser # Implicitly asserts SSH connection closed by ``send_file``. dcos_node.run(args=['userdel', '-r', testuser])
def _do_backup(master: Node, backup_local_path: Path) -> None: backup_name = backup_local_path.name # This must be an existing directory on the remote server. backup_remote_path = Path("/etc/") / backup_name dcos_etcdctl_with_args = get_dcos_etcdctl() dcos_etcdctl_with_args += ["backup", str(backup_remote_path)] master.run( args=dcos_etcdctl_with_args, output=Output.LOG_AND_CAPTURE, ) master.download_file( remote_path=backup_remote_path, local_path=backup_local_path, ) master.run(args=["rm", str(backup_remote_path)])
def test_sudo( self, dcos_node: Node, tmpdir: local, ) -> None: """ It is possible to use sudo to send a file to a directory which the user does not have access to. """ testuser = str(uuid.uuid4().hex) dcos_node.run(args=['useradd', testuser]) dcos_node.run( args=['cp', '-R', '$HOME/.ssh', '/home/{}/'.format(testuser)], shell=True, ) sudoers_line = '{user} ALL=(ALL) NOPASSWD: ALL'.format(user=testuser) dcos_node.run( args=['echo "' + sudoers_line + '">> /etc/sudoers'], shell=True, ) random = str(uuid.uuid4()) local_file = tmpdir.join('example_file.txt') local_file.write(random) master_destination_dir = '/etc/{testuser}/{random}'.format( testuser=testuser, random=random, ) master_destination_path = Path(master_destination_dir) / 'file.txt' with pytest.raises(CalledProcessError): dcos_node.send_file( local_path=Path(str(local_file)), remote_path=master_destination_path, user=testuser, ) dcos_node.send_file( local_path=Path(str(local_file)), remote_path=master_destination_path, user=testuser, sudo=True, ) args = ['stat', '-c', '"%U"', str(master_destination_path)] result = dcos_node.run(args=args, shell=True) assert result.stdout.decode().strip() == 'root' # Implicitly asserts SSH connection closed by ``send_file``. dcos_node.run(args=['userdel', '-r', testuser])
def test_async(self, dcos_node: Node) -> None: """ It is possible to run commands asynchronously. """ proc_1 = dcos_node.popen( args=['(mkfifo /tmp/pipe | true)', '&&', '(cat /tmp/pipe)'], shell=True, ) proc_2 = dcos_node.popen( args=[ '(mkfifo /tmp/pipe | true)', '&&', '(echo $HOME > /tmp/pipe)', ], shell=True, ) try: # An arbitrary timeout to avoid infinite wait times. stdout, _ = proc_1.communicate(timeout=15) except TimeoutExpired: # pragma: no cover proc_1.kill() stdout, _ = proc_1.communicate() return_code_1 = proc_1.poll() # Needed to cleanly terminate second subprocess try: # An arbitrary timeout to avoid infinite wait times. proc_2.communicate(timeout=15) except TimeoutExpired: # pragma: no cover proc_2.kill() proc_2.communicate() raise return_code_2 = proc_2.poll() assert stdout.strip().decode() == '/' + dcos_node.default_user assert return_code_1 == 0 assert return_code_2 == 0 dcos_node.run(['rm', '-f', '/tmp/pipe'])
def test_error( self, caplog: LogCaptureFixture, dcos_node: Node, shell: bool, log_output_live: bool, ) -> None: """ Commands which return a non-0 code raise a ``CalledProcessError``. """ with pytest.raises(CalledProcessError) as excinfo: dcos_node.run( args=['rm', 'does_not_exist'], shell=shell, log_output_live=log_output_live, ) exception = excinfo.value assert exception.returncode == 1 error_message = ( 'rm: cannot remove ‘does_not_exist’: No such file or directory' ) if log_output_live: assert exception.stderr.strip() == b'' assert exception.stdout.decode().strip() == error_message else: assert exception.stdout.strip() == b'' assert exception.stderr.decode().strip() == error_message # The stderr output is not in the debug log output. debug_messages = set( filter( lambda record: record.levelno == logging.DEBUG, caplog.records, ), ) matching_messages = set( filter( lambda record: 'No such file' in record.getMessage(), caplog.records, ), ) assert bool(len(debug_messages & matching_messages)) is log_output_live
def test_remote_env( self, dcos_node: Node, ) -> None: """ Remote environment variables are available. """ echo_result = dcos_node.run(args=['echo', '$HOME'], shell=True) assert echo_result.returncode == 0 assert echo_result.stdout.strip() == b'/root' assert echo_result.stderr == b''
def _get_storage_driver( self, node: Node, ) -> DockerStorageDriver: """ Given a `Node`, return the `DockerStorageDriver` on that node. """ _wait_for_docker(node=node) result = node.run(args=['docker', 'info', '--format', '{{.Driver}}']) return self.DOCKER_STORAGE_DRIVERS[result.stdout.decode().strip()]
def test_literal( self, dcos_node: Node, ) -> None: """ When shell=False, preserve arguments as literal values. """ echo_result = dcos_node.run( args=['echo', 'Hello, ', '&&', 'echo', 'World!'], ) assert echo_result.returncode == 0 assert echo_result.stdout.strip() == b'Hello, && echo World!' assert echo_result.stderr == b''
def _send_tarstream_to_node_and_extract( tarstream: io.BytesIO, node: Node, remote_path: Path, ) -> None: """ Given a tarstream, send the contents to a remote path. """ tar_path = Path('/tmp/dcos_e2e_tmp.tar') with tempfile.NamedTemporaryFile() as tmp_file: tmp_file.write(tarstream.getvalue()) tmp_file.flush() node.send_file( local_path=Path(tmp_file.name), remote_path=tar_path, ) tar_args = ['tar', '-C', str(remote_path), '-xvf', str(tar_path)] node.run(args=tar_args) node.run(args=['rm', str(tar_path)])
def assert_system_unit_state(node: Node, unit_name: str, active: bool=True) -> None: result = node.run( args=["systemctl show {}".format(unit_name)], output=Output.LOG_AND_CAPTURE, shell=True, ) unit_properties = result.stdout.strip().decode() if active: assert "ActiveState=active" in unit_properties else: assert "ActiveState=inactive" in unit_properties assert "ConditionResult=no" in unit_properties
def test_send_directory( self, dcos_node: Node, tmp_path: Path, ) -> None: """ It is possible to send a directory to a cluster node as the default user. """ original_content = str(uuid.uuid4()) dir_name = 'directory' file_name = 'example_file.txt' dir_path = tmp_path / dir_name dir_path.mkdir() local_file_path = dir_path / file_name local_file_path.write_text(original_content) random = uuid.uuid4().hex master_base_dir = '/etc/{random}'.format(random=random) master_destination_dir = Path(master_base_dir) dcos_node.send_file( local_path=local_file_path, remote_path=master_destination_dir / dir_name / file_name, ) args = ['cat', str(master_destination_dir / dir_name / file_name)] result = dcos_node.run(args=args) assert result.stdout.decode() == original_content new_content = str(uuid.uuid4()) local_file_path.write_text(new_content) dcos_node.send_file( local_path=dir_path, remote_path=master_destination_dir, ) args = ['cat', str(master_destination_dir / dir_name / file_name)] result = dcos_node.run(args=args) assert result.stdout.decode() == new_content
def test_custom_user( self, dcos_node: Node, ) -> None: """ Commands can be run as a custom user. """ testuser = str(uuid.uuid4().hex) dcos_node.run(args=['useradd', testuser]) dcos_node.run( args=['cp', '-R', '$HOME/.ssh', '/home/{}/'.format(testuser)], shell=True, ) echo_result = dcos_node.popen( args=['echo', '$HOME'], user=testuser, shell=True, ) stdout, stderr = echo_result.communicate() assert echo_result.returncode == 0 assert stdout.strip().decode() == '/home/' + testuser assert stderr.strip().decode() == '' dcos_node.run(args=['userdel', '-r', testuser])
def _dcos_systemd_units(node: Node) -> List[str]: """ Return all systemd services that are started up by DC/OS. """ result = node.run( args=[ 'sudo', 'systemctl', 'show', '-p', 'Wants', 'dcos.target', '|', 'cut', '-d=', '-f2' ], shell=True, ) systemd_units_string = result.stdout.strip().decode() return str(systemd_units_string).split(' ')
def test_sudo( self, dcos_node: Node, ) -> None: """ When sudo is given as ``True``, the given command has sudo prefixed. """ testuser = str(uuid.uuid4().hex) dcos_node.run(args=['useradd', testuser]) dcos_node.run( args=['cp', '-R', '$HOME/.ssh', '/home/{}/'.format(testuser)], shell=True, ) sudoers_line = '{user} ALL=(ALL) NOPASSWD: ALL'.format(user=testuser) echo_result = dcos_node.run( args=['echo "' + sudoers_line + '">> /etc/sudoers'], shell=True, ) assert echo_result.returncode == 0 assert echo_result.stdout.strip().decode() == '' assert echo_result.stderr.strip().decode() == '' echo_result = dcos_node.run( args=['echo', '$(whoami)'], user=testuser, shell=True, ) assert echo_result.returncode == 0 assert echo_result.stdout.strip().decode() == testuser assert echo_result.stderr.strip().decode() == '' echo_result = dcos_node.run( args=['echo', '$(whoami)'], user=testuser, shell=True, sudo=True, ) assert echo_result.returncode == 0 assert echo_result.stdout.strip().decode() == 'root' assert echo_result.stderr.strip().decode() == '' dcos_node.run(args=['userdel', '-r', testuser])