def run(
self,
args: List[str],
user: str,
log_output_live: bool,
env: Dict[str, Any],
tty: bool,
ssh_key_path: Path,
public_ip_address: IPv4Address,
capture_output: bool,
) -> subprocess.CompletedProcess:
"""
Run a command on this node the given user.
Args:
args: The command to run on the node.
user: The username to communicate as.
log_output_live: If ``True``, log output live. If ``True``, stderr
is merged into stdout in the return value.
env: Environment variables to be set on the node before running
the command. A mapping of environment variable names to
values.
tty: If ``True``, allocate a pseudo-tty. This means that the users
terminal is attached to the streams of the process.
This means that the values of stdout and stderr will not be in
the returned ``subprocess.CompletedProcess``.
ssh_key_path: The path to an SSH key which can be used to SSH to
the node as the ``user`` user.
public_ip_address: The public IP address of the node.
capture_output: Whether to capture output in the result.
Returns:
The representation of the finished process.
Raises:
subprocess.CalledProcessError: The process exited with a non-zero
code.
"""
ssh_args = _compose_ssh_command(
args=args,
user=user,
env=env,
tty=tty,
ssh_key_path=ssh_key_path,
public_ip_address=public_ip_address,
)
return run_subprocess(
args=ssh_args,
log_output_live=log_output_live,
pipe_output=capture_output,
)
def __init__( # pylint: disable=super-init-not-called
self,
generate_config_path: Optional[Path],
masters: int,
agents: int,
public_agents: int,
extra_config: Dict[str, Any],
log_output_live: bool,
files_to_copy_to_installer: Dict[Path, Path],
files_to_copy_to_masters: Dict[Path, Path],
cluster_backend: DCOS_Docker,
) -> None:
"""
Create a DC/OS Docker cluster.
Args:
generate_config_path: The path to a build artifact to install.
masters: The number of master nodes to create.
agents: The number of agent nodes to create.
public_agents: The number of public agent nodes to create.
extra_config: DC/OS Docker comes with a "base" configuration.
This dictionary can contain extra installation configuration
variables.
log_output_live: If `True`, log output of subprocesses live.
If `True`, stderr is merged into stdout in the return value.
files_to_copy_to_installer: A mapping of host paths to paths on
the installer node. These are files to copy from the host to
the installer node before installing DC/OS. Currently on DC/OS
Docker the only supported paths on the installer are in the
`/genconf` directory.
files_to_copy_to_masters: A mapping of host paths to paths on the
master nodes. These are files to copy from the host to
the master nodes before installing DC/OS. On DC/OS Docker the
files are mounted, read only, to the masters.
cluster_backend: Details of the specific DC/OS Docker backend to
use.
Raises:
ValueError: There is no file at `generate_config_path`.
CalledProcessError: The step to create and install containers
exited with a non-zero code.
"""
if generate_config_path is None or not generate_config_path.exists():
raise ValueError()
self.log_output_live = log_output_live
# To avoid conflicts, we use random container names.
# We use the same random string for each container in a cluster so
# that they can be associated easily.
#
# Starting with "dcos-e2e" allows `make clean` to remove these and
# only these containers.
unique = 'dcos-e2e-{random}'.format(random=uuid.uuid4())
# We create a new instance of DC/OS Docker and we work in this
# directory.
# This helps running tests in parallel without conflicts and it
# reduces the chance of side-effects affecting sequential tests.
self._path = Path(
TemporaryDirectory(
suffix=unique,
dir=(str(cluster_backend.workspace_dir)
if cluster_backend.workspace_dir else None),
).name)
copytree(
src=str(cluster_backend.dcos_docker_path),
dst=str(self._path),
# If there is already a config, we do not copy it as it will be
# overwritten and therefore copying it is wasteful.
ignore=ignore_patterns('dcos_generate_config.sh'),
)
# Files in the DC/OS Docker directory's `genconf` directory are mounted
# to the installer at `/genconf`.
# Therefore, every file which we want to copy to `/genconf` on the
# installer is put into the genconf directory in DC/OS Docker.
# The way to fix this if we want to be able to put files anywhere is
# to add an variable to `dcos_generate_config.sh.in` which allows
# `-v` mounts.
# Then `INSTALLER_MOUNTS` can be added to DC/OS Docker.
genconf_dir = self._path / 'genconf'
# We wrap this in `Path` to work around
# https://github.com/PyCQA/pylint/issues/224.
Path(genconf_dir).mkdir(exist_ok=True)
for host_path, installer_path in files_to_copy_to_installer.items():
relative_installer_path = installer_path.relative_to('/genconf')
destination_path = genconf_dir / relative_installer_path
copyfile(src=str(host_path), dst=str(destination_path))
extra_genconf_config = ''
if extra_config:
extra_genconf_config = yaml.dump(
data=extra_config,
default_flow_style=False,
)
master_mounts = []
for host_path, master_path in files_to_copy_to_masters.items():
mount = '-v {host_path}:{master_path}:ro'.format(
host_path=host_path.absolute(),
master_path=master_path,
)
master_mounts.append(mount)
# Only overlay, overlay2, and aufs storage drivers are supported.
# This chooses the overlay2 driver if the host's driver is not
# supported for speed reasons.
client = docker.from_env(version='auto')
host_driver = client.info()['Driver']
storage_driver = host_driver if host_driver in ('overlay', 'overlay2',
'aufs') else 'overlay2'
self._master_prefix = '{unique}-master-'.format(unique=unique)
self._agent_prefix = '{unique}-agent-'.format(unique=unique)
self._public_agent_prefix = '{unique}-pub-agent-'.format(unique=unique)
variables = {
# This version of Docker supports `overlay2`.
'DOCKER_VERSION': '1.13.1',
'DOCKER_STORAGEDRIVER': storage_driver,
# Some platforms support systemd and some do not.
# Disabling support makes all platforms consistent in this aspect.
'MESOS_SYSTEMD_ENABLE_SUPPORT': 'false',
# Number of nodes.
'MASTERS': str(masters),
'AGENTS': str(agents),
'PUBLIC_AGENTS': str(public_agents),
# Container names.
'MASTER_CTR': self._master_prefix,
'AGENT_CTR': self._agent_prefix,
'PUBLIC_AGENT_CTR': self._public_agent_prefix,
'INSTALLER_CTR': '{unique}-installer'.format(unique=unique),
'INSTALLER_PORT': str(_get_open_port()),
'EXTRA_GENCONF_CONFIG': extra_genconf_config,
'MASTER_MOUNTS': ' '.join(master_mounts),
'DCOS_GENERATE_CONFIG_PATH': str(generate_config_path),
# Make sure that there are no home mounts.
# If $HOME is set to a directory we use, like `/root`, home mounts
# can cause problems.
'HOME_MOUNTS': '',
} # type: Dict[str, str]
make_args = []
for key, value in variables.items():
# See https://stackoverflow.com/a/7860705 for details on escaping
# Make variables.
escaped_value = value.replace('$', '$$')
escaped_value = escaped_value.replace('#', '\\#')
set_variable = '{key}={value}'.format(key=key, value=escaped_value)
make_args.append(set_variable)
run_subprocess(args=['make'] + make_args + ['install'],
cwd=str(self._path),
log_output_live=self.log_output_live)
def install_dcos_from_path_with_bootstrap_node(
self,
build_artifact: Path,
dcos_config: Dict[str, Any],
log_output_live: bool,
) -> None:
"""
Install DC/OS from a given build artifact.
Args:
build_artifact: The ``Path`` to a build artifact to install DC/OS
from.
dcos_config: The DC/OS configuration to use.
log_output_live: If ``True``, log output of the installation live.
Raises:
CalledProcessError: There was an error installing DC/OS on a node.
"""
config_yaml = yaml.dump(data=dcos_config)
config_file_path = self._genconf_dir / 'config.yaml'
config_file_path.write_text(data=config_yaml)
genconf_args = [
'bash',
str(build_artifact),
'--offline',
'-v',
'--genconf',
]
installer_ctr = '{cluster_id}-installer'.format(
cluster_id=self._cluster_id, )
installer_port = _get_open_port()
run_subprocess(
args=genconf_args,
env={
'PORT': str(installer_port),
'DCOS_INSTALLER_CONTAINER_NAME': installer_ctr,
},
log_output_live=log_output_live,
cwd=str(self._path),
)
for role, nodes in [
('master', self.masters),
('slave', self.agents),
('slave_public', self.public_agents),
]:
dcos_install_args = [
'/bin/bash',
str(self._bootstrap_tmp_path / 'dcos_install.sh'),
'--no-block-dcos-setup',
role,
]
for node in nodes:
try:
node.run(args=dcos_install_args)
except subprocess.CalledProcessError as ex: # pragma: no cover
LOGGER.error(ex.stdout)
LOGGER.error(ex.stderr)
raise
def install_dcos_from_path_with_bootstrap_node(
self,
dcos_installer: Path,
dcos_config: Dict[str, Any],
ip_detect_path: Path,
output: Output,
files_to_copy_to_genconf_dir: Iterable[Tuple[Path, Path]],
) -> None:
"""
Install DC/OS from a given installer.
Args:
dcos_installer: The ``Path`` to an installer to install DC/OS
from.
dcos_config: The DC/OS configuration to use.
ip_detect_path: The ``ip-detect`` script that is used for
installing DC/OS.
output: What happens with stdout and stderr.
files_to_copy_to_genconf_dir: Pairs of host paths to paths on
the installer node. These are files to copy from the host to
the installer node before installing DC/OS.
Raises:
CalledProcessError: There was an error installing DC/OS on a node.
"""
copyfile(
src=str(ip_detect_path),
dst=str(self._genconf_dir / 'ip-detect'),
)
config_yaml = yaml.dump(data=dcos_config)
config_file_path = self._genconf_dir / 'config.yaml'
config_file_path.write_text(data=config_yaml)
for host_path, installer_path in files_to_copy_to_genconf_dir:
relative_installer_path = installer_path.relative_to('/genconf')
destination_path = self._genconf_dir / relative_installer_path
if host_path.is_dir():
destination_path = destination_path / host_path.stem
copytree(src=str(host_path), dst=str(destination_path))
else:
copyfile(src=str(host_path), dst=str(destination_path))
genconf_args = [
'bash',
str(dcos_installer),
'--offline',
'-v',
'--genconf',
]
installer_ctr = '{cluster_id}-installer'.format(
cluster_id=self._cluster_id,
)
installer_port = _get_open_port()
log_output_live = {
Output.CAPTURE: False,
Output.LOG_AND_CAPTURE: True,
Output.NO_CAPTURE: False,
}[output]
capture_output = {
Output.CAPTURE: True,
Output.LOG_AND_CAPTURE: True,
Output.NO_CAPTURE: False,
}[output]
run_subprocess(
args=genconf_args,
env={
'PORT': str(installer_port),
'DCOS_INSTALLER_CONTAINER_NAME': installer_ctr,
},
log_output_live=log_output_live,
cwd=str(self._path),
pipe_output=capture_output,
)
for role, nodes in [
('master', self.masters),
('slave', self.agents),
('slave_public', self.public_agents),
]:
dcos_install_args = [
'/bin/bash',
str(self._bootstrap_tmp_path / 'dcos_install.sh'),
'--no-block-dcos-setup',
role,
]
for node in nodes:
try:
node.run(args=dcos_install_args)
except subprocess.CalledProcessError as ex: # pragma: no cover
LOGGER.error(ex.stdout)
LOGGER.error(ex.stderr)
raise
def install_dcos_from_path(
self,
build_artifact: Path,
extra_config: Dict[str, Any],
log_output_live: bool,
) -> None:
"""
Args:
build_artifact: The `Path` to a build artifact to install DC/OS
from.
extra_config: May contain extra installation configuration
variables that are applied on top of the default DC/OS
configuration of the Docker backend.
log_output_live: If `True`, log output of the installation live.
"""
ssh_user = self._default_ssh_user
def ip_list(nodes: Set[Node]) -> List[str]:
return list(map(lambda node: str(node.public_ip_address), nodes))
config = {
'agent_list': ip_list(nodes=self.agents),
'bootstrap_url': 'file://' + str(self._bootstrap_tmp_path),
# Without this, we see errors like:
# "Time is not synchronized / marked as bad by the kernel.".
# Adam saw this on Docker for Mac 17.09.0-ce-mac35.
#
# In that case this was fixable with:
# $ docker run --rm --privileged alpine hwclock -s
'check_time': 'false',
'cluster_name': 'DCOS',
'exhibitor_storage_backend': 'static',
'master_discovery': 'static',
'master_list': ip_list(nodes=self.masters),
'process_timeout': 10000,
'public_agent_list': ip_list(nodes=self.public_agents),
'resolvers': ['8.8.8.8'],
'ssh_port': 22,
'ssh_user': ssh_user,
}
config_data = {**config, **extra_config}
config_yaml = yaml.dump(data=config_data) # type: ignore
config_file_path = self._genconf_dir / 'config.yaml'
config_file_path.write_text(data=config_yaml)
genconf_args = [
'bash',
str(build_artifact),
'--offline',
'-v',
'--genconf',
]
installer_ctr = '{cluster_id}-installer'.format(
cluster_id=self._cluster_id)
installer_port = _get_open_port()
run_subprocess(
args=genconf_args,
env={
'PORT': str(installer_port),
'DCOS_INSTALLER_CONTAINER_NAME': installer_ctr,
},
log_output_live=log_output_live,
cwd=str(self._path),
)
for role, nodes in [
('master', self.masters),
('slave', self.agents),
('slave_public', self.public_agents),
]:
dcos_install_args = [
'/bin/bash',
str(self._bootstrap_tmp_path / 'dcos_install.sh'),
'--no-block-dcos-setup',
role,
]
for node in nodes:
node.run(args=dcos_install_args, user=ssh_user)