def doctor(verbose: int) -> None:
"""
Diagnose common issues which stop DC/OS E2E from working correctly.
"""
set_logging(verbosity_level=verbose)
check_functions = [
check_1_9_sed,
_check_docker_root_free_space,
_check_docker_supports_mounts,
_check_memory,
_check_mount_tmp,
_check_networking,
_check_selinux,
check_ssh,
_check_storage_driver,
_check_tmp_free_space,
# These two start ``Cluster``s, and so they come last.
_check_can_build,
# This comes last because it depends on ``_check_can_build``.
_check_can_mount_in_docker,
]
for function in check_functions:
level = function()
if level == CheckLevels.ERROR:
sys.exit(1)
_link_to_troubleshooting()
def sync_code(
cluster_id: str,
dcos_checkout_dir: str,
verbose: int,
) -> None:
"""
Sync files from a DC/OS checkout to master nodes.
This syncs integration test files and bootstrap files.
``DCOS_CHECKOUT_DIR`` should be set to the path of clone of an open source
DC/OS or DC/OS Enterprise repository.
By default the ``DCOS_CHECKOUT_DIR`` argument is set to the value of the
``DCOS_CHECKOUT_DIR`` environment variable.
If no ``DCOS_CHECKOUT_DIR`` is given, the current working directory is
used.
"""
set_logging(verbosity_level=verbose)
check_cluster_id_exists(
new_cluster_id=cluster_id,
existing_cluster_ids=existing_cluster_ids(),
)
cluster_containers = ClusterVMs(cluster_id=cluster_id)
cluster = cluster_containers.cluster
sync_code_to_masters(
cluster=cluster,
dcos_checkout_dir=Path(dcos_checkout_dir),
)
def run(
cluster_id: str,
node_args: Tuple[str],
sync_dir: Optional[Path],
dcos_login_uname: str,
dcos_login_pw: str,
no_test_env: bool,
env: Dict[str, str],
aws_region: str,
verbose: int,
node: str,
) -> None:
"""
Run an arbitrary command on a node.
This command sets up the environment so that ``pytest`` can be run.
For example, run
``dcos-aws run --cluster-id 1231599 pytest -k test_tls.py``.
Or, with sync:
``dcos-aws run --sync-dir . --cluster-id 1231599 pytest -k test_tls.py``.
To use special characters such as single quotes in your command, wrap the
whole command in double quotes.
""" # noqa: E501
set_logging(verbosity_level=verbose)
check_cluster_id_exists(
new_cluster_id=cluster_id,
existing_cluster_ids=existing_cluster_ids(aws_region=aws_region),
)
cluster_instances = ClusterInstances(
cluster_id=cluster_id,
aws_region=aws_region,
)
cluster = cluster_instances.cluster
host = _get_node(
cluster_id=cluster_id,
node_reference=node,
aws_region=aws_region,
)
if sync_dir is not None:
sync_code_to_masters(
cluster=cluster,
dcos_checkout_dir=sync_dir,
)
run_command(
args=list(node_args),
cluster=cluster,
host=host,
use_test_env=not no_test_env,
dcos_login_uname=dcos_login_uname,
dcos_login_pw=dcos_login_pw,
env=env,
transport=Transport.SSH,
)
def doctor(verbose: int) -> None:
"""
Diagnose common issues which stop DC/OS E2E from working correctly.
"""
set_logging(verbosity_level=verbose)
check_functions = [check_ssh]
for function in check_functions:
level = function()
if level == CheckLevels.ERROR:
sys.exit(1)
def wait(
ctx: click.core.Context,
cluster_id: str,
superuser_username: str,
superuser_password: str,
transport: Transport,
skip_http_checks: bool,
verbose: int,
) -> None:
"""
Wait for DC/OS to start.
"""
check_cluster_id_exists(
new_cluster_id=cluster_id,
existing_cluster_ids=existing_cluster_ids(),
)
set_logging(verbosity_level=verbose)
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
cluster_containers = ClusterContainers(
cluster_id=cluster_id,
transport=transport,
)
http_checks = not skip_http_checks
parent = ctx.parent
assert parent is not None
doctor_command_name = '{info_name} {doctor_name}'.format(
info_name=parent.info_name,
doctor_name=doctor.name,
)
show_wait_help(
is_enterprise=cluster_containers.is_enterprise,
doctor_command_name=doctor_command_name,
)
with click_spinner.spinner():
if cluster_containers.is_enterprise:
cluster_containers.cluster.wait_for_dcos_ee(
superuser_username=superuser_username,
superuser_password=superuser_password,
http_checks=http_checks,
)
return
cluster_containers.cluster.wait_for_dcos_oss(http_checks=http_checks)
def create(
agents: int,
artifact: str,
extra_config: Dict[str, Any],
masters: int,
public_agents: int,
variant: str,
workspace_dir: Optional[Path],
license_key: Optional[str],
security_mode: Optional[str],
copy_to_master: List[Tuple[Path, Path]],
cluster_id: str,
verbose: int,
enable_selinux_enforcing: bool,
genconf_dir: Optional[Path],
) -> None:
"""
Create a DC/OS cluster.
DC/OS Enterprise
\b
DC/OS Enterprise clusters require different configuration variables to DC/OS OSS.
For example, enterprise clusters require the following configuration parameters:
``superuser_username``, ``superuser_password_hash``, ``fault_domain_enabled``, ``license_key_contents``
\b
These can all be set in ``--extra-config``.
However, some defaults are provided for all but the license key.
\b
The default superuser username is ``admin``.
The default superuser password is ``admin``.
The default ``fault_domain_enabled`` is ``false``.
\b
``license_key_contents`` must be set for DC/OS Enterprise 1.11 and above.
This is set to one of the following, in order:
\b
* The ``license_key_contents`` set in ``--extra-config``.
* The contents of the path given with ``--license-key``.
* The contents of the path set in the ``DCOS_LICENSE_KEY_PATH`` environment variable.
\b
If none of these are set, ``license_key_contents`` is not given.
""" # noqa: E501
set_logging(verbosity_level=verbose)
check_cluster_id_unique(
new_cluster_id=cluster_id,
existing_cluster_ids=existing_cluster_ids(),
)
base_workspace_dir = workspace_dir or Path(tempfile.gettempdir())
workspace_dir = base_workspace_dir / uuid.uuid4().hex
workspace_dir.mkdir(parents=True)
doctor_message = 'Try `dcos-vagrant doctor` for troubleshooting help.'
artifact_path = Path(artifact).resolve()
if variant == 'auto':
variant = get_variant(
artifact_path=artifact_path,
workspace_dir=workspace_dir,
doctor_message=doctor_message,
)
enterprise = bool(variant == 'enterprise')
description = {
CLUSTER_ID_DESCRIPTION_KEY: cluster_id,
WORKSPACE_DIR_DESCRIPTION_KEY: str(workspace_dir),
VARIANT_DESCRIPTION_KEY: 'ee' if enterprise else '',
}
cluster_backend = Vagrant(
workspace_dir=workspace_dir,
virtualbox_description=json.dumps(obj=description),
)
if enterprise:
superuser_username = '******'
superuser_password = '******'
enterprise_extra_config = {
'superuser_username': superuser_username,
'superuser_password_hash': sha512_crypt.hash(superuser_password),
'fault_domain_enabled': False,
}
if license_key is not None:
key_contents = Path(license_key).read_text()
enterprise_extra_config['license_key_contents'] = key_contents
extra_config = {**enterprise_extra_config, **extra_config}
if security_mode is not None:
extra_config['security'] = security_mode
try:
cluster = Cluster(
cluster_backend=cluster_backend,
masters=masters,
agents=agents,
public_agents=public_agents,
)
except CalledProcessError as exc:
click.echo('Error creating cluster.', err=True)
click.echo(doctor_message)
sys.exit(exc.returncode)
nodes = {*cluster.masters, *cluster.agents, *cluster.public_agents}
for node in nodes:
if enable_selinux_enforcing:
node.run(args=['setenforce', '1'], sudo=True)
for node in cluster.masters:
for path_pair in copy_to_master:
local_path, remote_path = path_pair
node.send_file(
local_path=local_path,
remote_path=remote_path,
)
files_to_copy_to_genconf_dir = []
if genconf_dir is not None:
container_genconf_path = Path('/genconf')
for genconf_file in genconf_dir.glob('*'):
genconf_relative = genconf_file.relative_to(genconf_dir)
relative_path = container_genconf_path / genconf_relative
files_to_copy_to_genconf_dir.append((genconf_file, relative_path))
try:
with click_spinner.spinner():
cluster.install_dcos_from_path(
build_artifact=artifact_path,
dcos_config={
**cluster.base_config,
**extra_config,
},
ip_detect_path=cluster_backend.ip_detect_path,
files_to_copy_to_genconf_dir=files_to_copy_to_genconf_dir,
)
except CalledProcessError as exc:
click.echo('Error installing DC/OS.', err=True)
click.echo(doctor_message)
cluster.destroy()
sys.exit(exc.returncode)
def create(
ctx: click.core.Context,
agents: int,
artifact: str,
cluster_id: str,
docker_storage_driver: str,
docker_version: str,
extra_config: Dict[str, Any],
linux_distribution: str,
masters: int,
public_agents: int,
license_key: Optional[str],
security_mode: Optional[str],
copy_to_master: List[Tuple[Path, Path]],
genconf_dir: Optional[Path],
workspace_dir: Optional[Path],
custom_volume: List[Mount],
custom_master_volume: List[Mount],
custom_agent_volume: List[Mount],
custom_public_agent_volume: List[Mount],
variant: str,
transport: Transport,
wait_for_dcos: bool,
network: Network,
one_master_host_port_map: Dict[str, int],
verbose: int,
) -> None:
"""
Create a DC/OS cluster.
DC/OS Enterprise
\b
DC/OS Enterprise clusters require different configuration variables to DC/OS OSS.
For example, enterprise clusters require the following configuration parameters:
``superuser_username``, ``superuser_password_hash``, ``fault_domain_enabled``, ``license_key_contents``
\b
These can all be set in ``--extra-config``.
However, some defaults are provided for all but the license key.
\b
The default superuser username is ``admin``.
The default superuser password is ``admin``.
The default ``fault_domain_enabled`` is ``false``.
\b
``license_key_contents`` must be set for DC/OS Enterprise 1.11 and above.
This is set to one of the following, in order:
\b
* The ``license_key_contents`` set in ``--extra-config``.
* The contents of the path given with ``--license-key``.
* The contents of the path set in the ``DCOS_LICENSE_KEY_PATH`` environment variable.
\b
If none of these are set, ``license_key_contents`` is not given.
""" # noqa: E501
set_logging(verbosity_level=verbose)
check_cluster_id_unique(
new_cluster_id=cluster_id,
existing_cluster_ids=existing_cluster_ids(),
)
base_workspace_dir = workspace_dir or Path(tempfile.gettempdir())
workspace_dir = base_workspace_dir / uuid.uuid4().hex
doctor_message = 'Try `dcos-docker doctor` for troubleshooting help.'
ssh_keypair_dir = workspace_dir / 'ssh'
ssh_keypair_dir.mkdir(parents=True)
public_key_path = ssh_keypair_dir / 'id_rsa.pub'
private_key_path = ssh_keypair_dir / 'id_rsa'
write_key_pair(
public_key_path=public_key_path,
private_key_path=private_key_path,
)
artifact_path = Path(artifact).resolve()
if variant == 'auto':
variant = get_variant(
artifact_path=artifact_path,
workspace_dir=workspace_dir,
doctor_message=doctor_message,
)
enterprise = bool(variant == 'enterprise')
if enterprise:
superuser_username = '******'
superuser_password = '******'
enterprise_extra_config = {
'superuser_username': superuser_username,
'superuser_password_hash': sha512_crypt.hash(superuser_password),
'fault_domain_enabled': False,
}
if license_key is not None:
key_contents = Path(license_key).read_text()
enterprise_extra_config['license_key_contents'] = key_contents
extra_config = {**enterprise_extra_config, **extra_config}
if security_mode is not None:
extra_config['security'] = security_mode
files_to_copy_to_genconf_dir = []
if genconf_dir is not None:
container_genconf_path = Path('/genconf')
for genconf_file in genconf_dir.glob('*'):
genconf_relative = genconf_file.relative_to(genconf_dir)
relative_path = container_genconf_path / genconf_relative
files_to_copy_to_genconf_dir.append((genconf_file, relative_path))
cluster_backend = Docker(
custom_container_mounts=custom_volume,
custom_master_mounts=custom_master_volume,
custom_agent_mounts=custom_agent_volume,
custom_public_agent_mounts=custom_public_agent_volume,
linux_distribution=LINUX_DISTRIBUTIONS[linux_distribution],
docker_version=DOCKER_VERSIONS[docker_version],
storage_driver=DOCKER_STORAGE_DRIVERS.get(docker_storage_driver),
docker_container_labels={
CLUSTER_ID_LABEL_KEY: cluster_id,
WORKSPACE_DIR_LABEL_KEY: str(workspace_dir),
VARIANT_LABEL_KEY: 'ee' if enterprise else '',
},
docker_master_labels={
NODE_TYPE_LABEL_KEY: NODE_TYPE_MASTER_LABEL_VALUE,
},
docker_agent_labels={NODE_TYPE_LABEL_KEY: NODE_TYPE_AGENT_LABEL_VALUE},
docker_public_agent_labels={
NODE_TYPE_LABEL_KEY: NODE_TYPE_PUBLIC_AGENT_LABEL_VALUE,
},
workspace_dir=workspace_dir,
transport=transport,
network=network,
one_master_host_port_map=one_master_host_port_map,
)
try:
cluster = Cluster(
cluster_backend=cluster_backend,
masters=masters,
agents=agents,
public_agents=public_agents,
)
except CalledProcessError as exc:
click.echo('Error creating cluster.', err=True)
click.echo(doctor_message)
sys.exit(exc.returncode)
nodes = {
*cluster.masters,
*cluster.agents,
*cluster.public_agents,
}
for node in nodes:
node.run(
args=['echo', '', '>>', '/root/.ssh/authorized_keys'],
shell=True,
)
node.run(
args=[
'echo',
public_key_path.read_text(),
'>>',
'/root/.ssh/authorized_keys',
],
shell=True,
)
for node in cluster.masters:
for path_pair in copy_to_master:
local_path, remote_path = path_pair
node.send_file(
local_path=local_path,
remote_path=remote_path,
)
try:
with click_spinner.spinner():
cluster.install_dcos_from_path(
build_artifact=artifact_path,
dcos_config={
**cluster.base_config,
**extra_config,
},
ip_detect_path=cluster_backend.ip_detect_path,
files_to_copy_to_genconf_dir=files_to_copy_to_genconf_dir,
)
except CalledProcessError as exc:
click.echo('Error installing DC/OS.', err=True)
click.echo(doctor_message)
cluster.destroy()
sys.exit(exc.returncode)
click.echo(cluster_id)
if wait_for_dcos:
ctx.invoke(
wait,
cluster_id=cluster_id,
transport=transport,
skip_http_checks=bool(transport == Transport.DOCKER_EXEC),
)
return
started_message = (
'Cluster "{cluster_id}" has started. '
'Run "dcos-docker wait --cluster-id {cluster_id}" to wait for DC/OS '
'to become ready.').format(cluster_id=cluster_id)
click.echo(started_message, err=True)
def create(
agents: int,
artifact_url: str,
extra_config: Dict[str, Any],
masters: int,
public_agents: int,
variant: str,
workspace_dir: Optional[Path],
license_key: Optional[str],
security_mode: Optional[str],
copy_to_master: List[Tuple[Path, Path]],
verbose: int,
aws_region: str,
linux_distribution: str,
cluster_id: str,
enable_selinux_enforcing: bool,
genconf_dir: Optional[Path],
custom_tag: Dict[str, str],
) -> None:
"""
Create a DC/OS cluster.
DC/OS Enterprise
\b
DC/OS Enterprise clusters require different configuration variables to DC/OS OSS.
For example, enterprise clusters require the following configuration parameters:
``superuser_username``, ``superuser_password_hash``, ``fault_domain_enabled``, ``license_key_contents``
\b
These can all be set in ``--extra-config``.
However, some defaults are provided for all but the license key.
\b
The default superuser username is ``admin``.
The default superuser password is ``admin``.
The default ``fault_domain_enabled`` is ``false``.
\b
``license_key_contents`` must be set for DC/OS Enterprise 1.11 and above.
This is set to one of the following, in order:
\b
* The ``license_key_contents`` set in ``--extra-config``.
* The contents of the path given with ``--license-key``.
* The contents of the path set in the ``DCOS_LICENSE_KEY_PATH`` environment variable.
\b
If none of these are set, ``license_key_contents`` is not given.
""" # noqa: E501
set_logging(verbosity_level=verbose)
check_cluster_id_unique(
new_cluster_id=cluster_id,
existing_cluster_ids=existing_cluster_ids(aws_region=aws_region),
)
base_workspace_dir = workspace_dir or Path(tempfile.gettempdir())
workspace_dir = base_workspace_dir / uuid.uuid4().hex
workspace_dir.mkdir(parents=True)
ssh_keypair_dir = workspace_dir / 'ssh'
ssh_keypair_dir.mkdir(parents=True)
key_name = 'dcos-e2e-{random}'.format(random=uuid.uuid4().hex)
public_key_path = ssh_keypair_dir / 'id_rsa.pub'
private_key_path = ssh_keypair_dir / 'id_rsa'
write_key_pair(
public_key_path=public_key_path,
private_key_path=private_key_path,
)
ec2 = boto3.resource('ec2', region_name=aws_region)
ec2.import_key_pair(
KeyName=key_name,
PublicKeyMaterial=public_key_path.read_bytes(),
)
doctor_message = 'Try `dcos-aws doctor` for troubleshooting help.'
enterprise = bool(variant == 'enterprise')
ssh_user = {
Distribution.CENTOS_7: 'centos',
Distribution.COREOS: 'core',
Distribution.UBUNTU_16_04: 'ubuntu',
Distribution.RHEL_7: 'ec2-user',
}
distribution = LINUX_DISTRIBUTIONS[linux_distribution]
default_user = ssh_user[distribution]
cluster_tags = {
SSH_USER_TAG_KEY: default_user,
CLUSTER_ID_TAG_KEY: cluster_id,
WORKSPACE_DIR_TAG_KEY: str(workspace_dir),
KEY_NAME_TAG_KEY: key_name,
VARIANT_TAG_KEY: 'ee' if enterprise else '',
**custom_tag,
}
master_tags = {NODE_TYPE_TAG_KEY: NODE_TYPE_MASTER_TAG_VALUE}
agent_tags = {NODE_TYPE_TAG_KEY: NODE_TYPE_AGENT_TAG_VALUE}
public_agent_tags = {NODE_TYPE_TAG_KEY: NODE_TYPE_PUBLIC_AGENT_TAG_VALUE}
cluster_backend = AWS(
aws_key_pair=(key_name, private_key_path),
workspace_dir=workspace_dir,
aws_region=aws_region,
linux_distribution=distribution,
ec2_instance_tags=cluster_tags,
master_ec2_instance_tags=master_tags,
agent_ec2_instance_tags=agent_tags,
public_agent_ec2_instance_tags=public_agent_tags,
)
if enterprise:
superuser_username = '******'
superuser_password = '******'
enterprise_extra_config = {
'superuser_username': superuser_username,
'superuser_password_hash': sha512_crypt.hash(superuser_password),
'fault_domain_enabled': False,
}
if license_key is not None:
key_contents = Path(license_key).read_text()
enterprise_extra_config['license_key_contents'] = key_contents
extra_config = {**enterprise_extra_config, **extra_config}
if security_mode is not None:
extra_config['security'] = security_mode
try:
cluster = Cluster(
cluster_backend=cluster_backend,
masters=masters,
agents=agents,
public_agents=public_agents,
)
except CalledProcessError as exc:
click.echo('Error creating cluster.', err=True)
click.echo(doctor_message)
sys.exit(exc.returncode)
nodes = {*cluster.masters, *cluster.agents, *cluster.public_agents}
for node in nodes:
if enable_selinux_enforcing:
node.run(args=['setenforce', '1'], sudo=True)
for node in cluster.masters:
for path_pair in copy_to_master:
local_path, remote_path = path_pair
node.send_file(
local_path=local_path,
remote_path=remote_path,
)
files_to_copy_to_genconf_dir = []
if genconf_dir is not None:
container_genconf_path = Path('/genconf')
for genconf_file in genconf_dir.glob('*'):
genconf_relative = genconf_file.relative_to(genconf_dir)
relative_path = container_genconf_path / genconf_relative
files_to_copy_to_genconf_dir.append((genconf_file, relative_path))
try:
with click_spinner.spinner():
cluster.install_dcos_from_url(
build_artifact=artifact_url,
dcos_config={
**cluster.base_config,
**extra_config,
},
ip_detect_path=cluster_backend.ip_detect_path,
files_to_copy_to_genconf_dir=files_to_copy_to_genconf_dir,
)
except CalledProcessError as exc:
click.echo('Error installing DC/OS.', err=True)
click.echo(doctor_message)
cluster.destroy()
sys.exit(exc.returncode)
