def create(
ctx: click.core.Context,
agents: int,
installer: Path,
cluster_id: str,
docker_storage_driver: Optional[DockerStorageDriver],
docker_version: DockerVersion,
extra_config: Dict[str, Any],
linux_distribution: Distribution,
masters: int,
public_agents: int,
license_key: Optional[Path],
security_mode: Optional[str],
copy_to_master: List[Tuple[Path, Path]],
genconf_dir: Optional[Path],
workspace_dir: Path,
custom_volume: List[Mount],
custom_master_volume: List[Mount],
custom_agent_volume: List[Mount],
custom_public_agent_volume: List[Mount],
variant: str,
transport: Transport,
wait_for_dcos: bool,
network: Network,
one_master_host_port_map: Dict[str, int],
mount_sys_fs_cgroup: bool,
) -> None:
"""
Create a DC/OS cluster.
"""
check_cluster_id_unique(
new_cluster_id=cluster_id,
existing_cluster_ids=existing_cluster_ids(),
)
http_checks = bool(transport == Transport.SSH)
wait_command_name = command_path(sibling_ctx=ctx, command=wait)
doctor_command_name = command_path(sibling_ctx=ctx, command=doctor)
doctor_message = get_doctor_message(
doctor_command_name=doctor_command_name, )
public_key_path = workspace_dir / 'id_rsa.pub'
private_key_path = workspace_dir / 'id_rsa'
write_key_pair(
public_key_path=public_key_path,
private_key_path=private_key_path,
)
dcos_variant = get_install_variant(
given_variant=variant,
installer_path=installer,
workspace_dir=workspace_dir,
doctor_message=doctor_message,
)
# This is useful for some people to identify containers.
container_name_prefix = Docker().container_name_prefix + '-' + cluster_id
cluster_backend = Docker(
container_name_prefix=container_name_prefix,
custom_container_mounts=custom_volume,
custom_master_mounts=custom_master_volume,
custom_agent_mounts=custom_agent_volume,
custom_public_agent_mounts=custom_public_agent_volume,
linux_distribution=linux_distribution,
docker_version=docker_version,
storage_driver=docker_storage_driver,
docker_container_labels={
CLUSTER_ID_LABEL_KEY: cluster_id,
WORKSPACE_DIR_LABEL_KEY: str(workspace_dir),
},
docker_master_labels={
NODE_TYPE_LABEL_KEY: NODE_TYPE_MASTER_LABEL_VALUE,
},
docker_agent_labels={NODE_TYPE_LABEL_KEY: NODE_TYPE_AGENT_LABEL_VALUE},
docker_public_agent_labels={
NODE_TYPE_LABEL_KEY: NODE_TYPE_PUBLIC_AGENT_LABEL_VALUE,
},
workspace_dir=workspace_dir,
transport=transport,
network=network,
one_master_host_port_map=one_master_host_port_map,
mount_sys_fs_cgroup=mount_sys_fs_cgroup,
)
cluster = create_cluster(
cluster_backend=cluster_backend,
masters=masters,
agents=agents,
public_agents=public_agents,
doctor_message=doctor_message,
)
cluster_containers = ClusterContainers(
cluster_id=cluster_id,
transport=transport,
)
private_ssh_key_path = cluster_containers.ssh_key_path
private_ssh_key_path.parent.mkdir(parents=True)
private_key_path.replace(private_ssh_key_path)
add_authorized_key(cluster=cluster, public_key_path=public_key_path)
for node in cluster.masters:
for path_pair in copy_to_master:
local_path, remote_path = path_pair
node.send_file(
local_path=local_path,
remote_path=remote_path,
)
dcos_config = get_config(
cluster_representation=cluster_containers,
extra_config=extra_config,
dcos_variant=dcos_variant,
security_mode=security_mode,
license_key=license_key,
)
install_dcos_from_path(
cluster_representation=cluster_containers,
dcos_config=dcos_config,
ip_detect_path=cluster_backend.ip_detect_path,
doctor_message=doctor_message,
dcos_installer=installer,
local_genconf_dir=genconf_dir,
)
run_post_install_steps(
cluster=cluster,
cluster_id=cluster_id,
dcos_config=dcos_config,
doctor_command_name=doctor_command_name,
http_checks=http_checks,
wait_command_name=wait_command_name,
wait_for_dcos=wait_for_dcos,
)
def provision(
ctx: click.core.Context,
agents: int,
masters: int,
public_agents: int,
workspace_dir: Path,
copy_to_master: List[Tuple[Path, Path]],
aws_instance_type: str,
aws_region: str,
linux_distribution: str,
cluster_id: str,
enable_selinux_enforcing: bool,
custom_tag: Dict[str, str],
enable_spinner: bool,
) -> None:
"""
Provision an AWS cluster to install DC/OS.
"""
check_cluster_id_unique(
new_cluster_id=cluster_id,
existing_cluster_ids=existing_cluster_ids(aws_region=aws_region),
)
ssh_keypair_dir = workspace_dir / 'ssh'
ssh_keypair_dir.mkdir(parents=True)
key_name = 'key-{random}'.format(random=uuid.uuid4().hex)
public_key_path = ssh_keypair_dir / 'id_rsa.pub'
private_key_path = ssh_keypair_dir / 'id_rsa'
write_key_pair(
public_key_path=public_key_path,
private_key_path=private_key_path,
)
ec2 = boto3.resource('ec2', region_name=aws_region)
ec2.import_key_pair(
KeyName=key_name,
PublicKeyMaterial=public_key_path.read_bytes(),
)
doctor_command_name = command_path(sibling_ctx=ctx, command=doctor)
doctor_message = get_doctor_message(
doctor_command_name=doctor_command_name, )
ssh_user = {
Distribution.CENTOS_7: 'centos',
Distribution.UBUNTU_16_04: 'ubuntu',
Distribution.RHEL_7: 'ec2-user',
}
distribution = LINUX_DISTRIBUTIONS[linux_distribution]
default_user = ssh_user[distribution]
cluster_tags = {
SSH_USER_TAG_KEY: default_user,
CLUSTER_ID_TAG_KEY: cluster_id,
WORKSPACE_DIR_TAG_KEY: str(workspace_dir),
KEY_NAME_TAG_KEY: key_name,
**custom_tag,
}
master_tags = {NODE_TYPE_TAG_KEY: NODE_TYPE_MASTER_TAG_VALUE}
agent_tags = {NODE_TYPE_TAG_KEY: NODE_TYPE_AGENT_TAG_VALUE}
public_agent_tags = {NODE_TYPE_TAG_KEY: NODE_TYPE_PUBLIC_AGENT_TAG_VALUE}
cluster_backend = AWS(
aws_key_pair=(key_name, private_key_path),
workspace_dir=workspace_dir,
aws_instance_type=aws_instance_type,
aws_region=aws_region,
linux_distribution=distribution,
ec2_instance_tags=cluster_tags,
master_ec2_instance_tags=master_tags,
agent_ec2_instance_tags=agent_tags,
public_agent_ec2_instance_tags=public_agent_tags,
aws_cloudformation_stack_name=cluster_id,
)
cluster = create_cluster(
cluster_backend=cluster_backend,
masters=masters,
agents=agents,
public_agents=public_agents,
doctor_message=doctor_message,
enable_spinner=enable_spinner,
)
nodes = {*cluster.masters, *cluster.agents, *cluster.public_agents}
for node in nodes:
if enable_selinux_enforcing:
node.run(args=['setenforce', '1'], sudo=True)
for node in cluster.masters:
for path_pair in copy_to_master:
local_path, remote_path = path_pair
node.send_file(
local_path=local_path,
remote_path=remote_path,
sudo=True,
)
def create(
ctx: click.core.Context,
agents: int,
installer_url: str,
extra_config: Dict[str, Any],
masters: int,
public_agents: int,
variant: str,
workspace_dir: Optional[Path],
license_key: Optional[str],
security_mode: Optional[str],
copy_to_master: List[Tuple[Path, Path]],
verbose: int,
aws_region: str,
linux_distribution: str,
cluster_id: str,
enable_selinux_enforcing: bool,
genconf_dir: Optional[Path],
custom_tag: Dict[str, str],
wait_for_dcos: bool,
) -> None:
"""
Create a DC/OS cluster.
DC/OS Enterprise
\b
DC/OS Enterprise clusters require different configuration variables to DC/OS OSS.
For example, enterprise clusters require the following configuration parameters:
``superuser_username``, ``superuser_password_hash``, ``fault_domain_enabled``, ``license_key_contents``
\b
These can all be set in ``--extra-config``.
However, some defaults are provided for all but the license key.
\b
The default superuser username is ``admin``.
The default superuser password is ``admin``.
The default ``fault_domain_enabled`` is ``false``.
\b
``license_key_contents`` must be set for DC/OS Enterprise 1.11 and above.
This is set to one of the following, in order:
\b
* The ``license_key_contents`` set in ``--extra-config``.
* The contents of the path given with ``--license-key``.
* The contents of the path set in the ``DCOS_LICENSE_KEY_PATH`` environment variable.
\b
If none of these are set, ``license_key_contents`` is not given.
""" # noqa: E501
set_logging(verbosity_level=verbose)
check_cluster_id_unique(
new_cluster_id=cluster_id,
existing_cluster_ids=existing_cluster_ids(aws_region=aws_region),
)
base_workspace_dir = workspace_dir or Path(tempfile.gettempdir())
workspace_dir = base_workspace_dir / uuid.uuid4().hex
workspace_dir.mkdir(parents=True)
ssh_keypair_dir = workspace_dir / 'ssh'
ssh_keypair_dir.mkdir(parents=True)
key_name = 'key-{random}'.format(random=uuid.uuid4().hex)
public_key_path = ssh_keypair_dir / 'id_rsa.pub'
private_key_path = ssh_keypair_dir / 'id_rsa'
write_key_pair(
public_key_path=public_key_path,
private_key_path=private_key_path,
)
ec2 = boto3.resource('ec2', region_name=aws_region)
ec2.import_key_pair(
KeyName=key_name,
PublicKeyMaterial=public_key_path.read_bytes(),
)
doctor_message = get_doctor_message(sibling_ctx=ctx, doctor_command=doctor)
dcos_variant = get_variant(
given_variant=variant,
installer_path=None,
workspace_dir=workspace_dir,
doctor_message=doctor_message,
)
variant_tag_value = {
DCOSVariant.OSS: VARIANT_OSS_TAG_VALUE,
DCOSVariant.ENTERPRISE: VARIANT_ENTERPRISE_TAG_VALUE,
}[dcos_variant]
ssh_user = {
Distribution.CENTOS_7: 'centos',
Distribution.COREOS: 'core',
Distribution.UBUNTU_16_04: 'ubuntu',
Distribution.RHEL_7: 'ec2-user',
}
distribution = LINUX_DISTRIBUTIONS[linux_distribution]
default_user = ssh_user[distribution]
cluster_tags = {
SSH_USER_TAG_KEY: default_user,
CLUSTER_ID_TAG_KEY: cluster_id,
WORKSPACE_DIR_TAG_KEY: str(workspace_dir),
KEY_NAME_TAG_KEY: key_name,
VARIANT_TAG_KEY: variant_tag_value,
**custom_tag,
}
master_tags = {NODE_TYPE_TAG_KEY: NODE_TYPE_MASTER_TAG_VALUE}
agent_tags = {NODE_TYPE_TAG_KEY: NODE_TYPE_AGENT_TAG_VALUE}
public_agent_tags = {NODE_TYPE_TAG_KEY: NODE_TYPE_PUBLIC_AGENT_TAG_VALUE}
cluster_backend = AWS(
aws_key_pair=(key_name, private_key_path),
workspace_dir=workspace_dir,
aws_region=aws_region,
linux_distribution=distribution,
ec2_instance_tags=cluster_tags,
master_ec2_instance_tags=master_tags,
agent_ec2_instance_tags=agent_tags,
public_agent_ec2_instance_tags=public_agent_tags,
)
cluster = create_cluster(
cluster_backend=cluster_backend,
masters=masters,
agents=agents,
public_agents=public_agents,
sibling_ctx=ctx,
doctor_command=doctor,
)
nodes = {*cluster.masters, *cluster.agents, *cluster.public_agents}
for node in nodes:
if enable_selinux_enforcing:
node.run(args=['setenforce', '1'], sudo=True)
for node in cluster.masters:
for path_pair in copy_to_master:
local_path, remote_path = path_pair
node.send_file(
local_path=local_path,
remote_path=remote_path,
sudo=True,
)
files_to_copy_to_genconf_dir = []
if genconf_dir is not None:
container_genconf_path = Path('/genconf')
for genconf_file in genconf_dir.glob('*'):
genconf_relative = genconf_file.relative_to(genconf_dir)
relative_path = container_genconf_path / genconf_relative
files_to_copy_to_genconf_dir.append((genconf_file, relative_path))
dcos_config = get_config(
cluster=cluster,
extra_config=extra_config,
dcos_variant=dcos_variant,
security_mode=security_mode,
license_key=license_key,
)
try:
with click_spinner.spinner():
cluster.install_dcos_from_url(
dcos_installer=installer_url,
dcos_config=dcos_config,
ip_detect_path=cluster_backend.ip_detect_path,
files_to_copy_to_genconf_dir=files_to_copy_to_genconf_dir,
)
except CalledProcessError as exc:
click.echo('Error installing DC/OS.', err=True)
click.echo(doctor_message)
cluster.destroy()
sys.exit(exc.returncode)
superuser_username = dcos_config.get(
'superuser_username',
DEFAULT_SUPERUSER_USERNAME,
)
superuser_password = dcos_config.get(
'superuser_password',
DEFAULT_SUPERUSER_PASSWORD,
)
if wait_for_dcos:
dcos_e2e_cli.common.wait.wait_for_dcos(
dcos_variant=dcos_variant,
cluster=cluster,
superuser_username=superuser_username,
superuser_password=superuser_password,
http_checks=True,
doctor_command=doctor,
sibling_ctx=ctx,
)
return
show_cluster_started_message(
# We work on the assumption that the ``wait`` command is a sibling
# command of this one.
sibling_ctx=ctx,
wait_command=wait,
cluster_id=cluster_id,
)
click.echo(cluster_id)
def provision(
ctx: click.core.Context,
agents: int,
cluster_id: str,
docker_storage_driver: Optional[DockerStorageDriver],
docker_version: DockerVersion,
linux_distribution: Distribution,
masters: int,
public_agents: int,
workspace_dir: Path,
custom_volume: List[Mount],
custom_master_volume: List[Mount],
custom_agent_volume: List[Mount],
custom_public_agent_volume: List[Mount],
transport: Transport,
network: Network,
one_master_host_port_map: Dict[str, int],
mount_sys_fs_cgroup: bool,
enable_spinner: bool,
) -> None:
"""
Provision Docker containers to install a DC/OS cluster.
"""
check_cluster_id_unique(
new_cluster_id=cluster_id,
existing_cluster_ids=existing_cluster_ids(),
)
doctor_command_name = command_path(sibling_ctx=ctx, command=doctor)
doctor_message = get_doctor_message(
doctor_command_name=doctor_command_name, )
public_key_path = workspace_dir / 'id_rsa.pub'
private_key_path = workspace_dir / 'id_rsa'
write_key_pair(
public_key_path=public_key_path,
private_key_path=private_key_path,
)
# This is useful for some people to identify containers.
container_name_prefix = Docker().container_name_prefix + '-' + cluster_id
cluster_backend = Docker(
container_name_prefix=container_name_prefix,
custom_container_mounts=custom_volume,
custom_master_mounts=custom_master_volume,
custom_agent_mounts=custom_agent_volume,
custom_public_agent_mounts=custom_public_agent_volume,
linux_distribution=linux_distribution,
docker_version=docker_version,
storage_driver=docker_storage_driver,
docker_container_labels={
CLUSTER_ID_LABEL_KEY: cluster_id,
WORKSPACE_DIR_LABEL_KEY: str(workspace_dir),
},
docker_master_labels={
NODE_TYPE_LABEL_KEY: NODE_TYPE_MASTER_LABEL_VALUE,
},
docker_agent_labels={NODE_TYPE_LABEL_KEY: NODE_TYPE_AGENT_LABEL_VALUE},
docker_public_agent_labels={
NODE_TYPE_LABEL_KEY: NODE_TYPE_PUBLIC_AGENT_LABEL_VALUE,
},
workspace_dir=workspace_dir,
transport=transport,
network=network,
one_master_host_port_map=one_master_host_port_map,
mount_sys_fs_cgroup=mount_sys_fs_cgroup,
)
cluster = create_cluster(
cluster_backend=cluster_backend,
masters=masters,
agents=agents,
public_agents=public_agents,
doctor_message=doctor_message,
enable_spinner=enable_spinner,
)
cluster_containers = ClusterContainers(
cluster_id=cluster_id,
transport=transport,
)
private_ssh_key_path = cluster_containers.ssh_key_path
private_ssh_key_path.parent.mkdir(parents=True)
private_key_path.replace(private_ssh_key_path)
add_authorized_key(cluster=cluster, public_key_path=public_key_path)
def create(
ctx: click.core.Context,
agents: int,
installer_url: str,
extra_config: Dict[str, Any],
masters: int,
public_agents: int,
variant: str,
workspace_dir: Path,
license_key: Optional[Path],
security_mode: Optional[str],
copy_to_master: List[Tuple[Path, Path]],
aws_region: str,
linux_distribution: str,
cluster_id: str,
enable_selinux_enforcing: bool,
genconf_dir: Optional[Path],
custom_tag: Dict[str, str],
wait_for_dcos: bool,
) -> None:
"""
Create a DC/OS cluster.
"""
check_cluster_id_unique(
new_cluster_id=cluster_id,
existing_cluster_ids=existing_cluster_ids(aws_region=aws_region),
)
ssh_keypair_dir = workspace_dir / 'ssh'
ssh_keypair_dir.mkdir(parents=True)
key_name = 'key-{random}'.format(random=uuid.uuid4().hex)
public_key_path = ssh_keypair_dir / 'id_rsa.pub'
private_key_path = ssh_keypair_dir / 'id_rsa'
write_key_pair(
public_key_path=public_key_path,
private_key_path=private_key_path,
)
ec2 = boto3.resource('ec2', region_name=aws_region)
ec2.import_key_pair(
KeyName=key_name,
PublicKeyMaterial=public_key_path.read_bytes(),
)
doctor_command_name = command_path(sibling_ctx=ctx, command=doctor)
wait_command_name = command_path(sibling_ctx=ctx, command=wait)
doctor_message = get_doctor_message(
doctor_command_name=doctor_command_name, )
dcos_variant = get_install_variant(
given_variant=variant,
installer_path=None,
workspace_dir=workspace_dir,
doctor_message=doctor_message,
)
ssh_user = {
Distribution.CENTOS_7: 'centos',
Distribution.COREOS: 'core',
Distribution.UBUNTU_16_04: 'ubuntu',
Distribution.RHEL_7: 'ec2-user',
}
distribution = LINUX_DISTRIBUTIONS[linux_distribution]
default_user = ssh_user[distribution]
cluster_tags = {
SSH_USER_TAG_KEY: default_user,
CLUSTER_ID_TAG_KEY: cluster_id,
WORKSPACE_DIR_TAG_KEY: str(workspace_dir),
KEY_NAME_TAG_KEY: key_name,
**custom_tag,
}
master_tags = {NODE_TYPE_TAG_KEY: NODE_TYPE_MASTER_TAG_VALUE}
agent_tags = {NODE_TYPE_TAG_KEY: NODE_TYPE_AGENT_TAG_VALUE}
public_agent_tags = {NODE_TYPE_TAG_KEY: NODE_TYPE_PUBLIC_AGENT_TAG_VALUE}
cluster_backend = AWS(
aws_key_pair=(key_name, private_key_path),
workspace_dir=workspace_dir,
aws_region=aws_region,
linux_distribution=distribution,
ec2_instance_tags=cluster_tags,
master_ec2_instance_tags=master_tags,
agent_ec2_instance_tags=agent_tags,
public_agent_ec2_instance_tags=public_agent_tags,
)
cluster = create_cluster(
cluster_backend=cluster_backend,
masters=masters,
agents=agents,
public_agents=public_agents,
doctor_message=doctor_message,
)
nodes = {*cluster.masters, *cluster.agents, *cluster.public_agents}
for node in nodes:
if enable_selinux_enforcing:
node.run(args=['setenforce', '1'], sudo=True)
for node in cluster.masters:
for path_pair in copy_to_master:
local_path, remote_path = path_pair
node.send_file(
local_path=local_path,
remote_path=remote_path,
sudo=True,
)
cluster_instances = ClusterInstances(
cluster_id=cluster_id,
aws_region=aws_region,
)
dcos_config = get_config(
cluster_representation=cluster_instances,
extra_config=extra_config,
dcos_variant=dcos_variant,
security_mode=security_mode,
license_key=license_key,
)
install_dcos_from_url(
cluster_representation=cluster_instances,
dcos_config=dcos_config,
dcos_installer_url=installer_url,
doctor_message=doctor_message,
local_genconf_dir=genconf_dir,
ip_detect_path=cluster_backend.ip_detect_path,
)
run_post_install_steps(
cluster=cluster,
cluster_id=cluster_id,
dcos_config=dcos_config,
doctor_command_name=doctor_command_name,
http_checks=True,
wait_command_name=wait_command_name,
wait_for_dcos=wait_for_dcos,
)