def doctor(verbose: int) -> None: """ Diagnose common issues which stop DC/OS E2E from working correctly. """ set_logging(verbosity_level=verbose) check_functions = [ check_1_9_sed, _check_docker_root_free_space, _check_docker_supports_mounts, _check_memory, _check_mount_tmp, _check_networking, _check_selinux, check_ssh, _check_storage_driver, _check_tmp_free_space, # These two start ``Cluster``s, and so they come last. _check_can_build, # This comes last because it depends on ``_check_can_build``. _check_can_mount_in_docker, ] for function in check_functions: level = function() if level == CheckLevels.ERROR: sys.exit(1) _link_to_troubleshooting()
def sync_code( cluster_id: str, dcos_checkout_dir: str, verbose: int, ) -> None: """ Sync files from a DC/OS checkout to master nodes. This syncs integration test files and bootstrap files. ``DCOS_CHECKOUT_DIR`` should be set to the path of clone of an open source DC/OS or DC/OS Enterprise repository. By default the ``DCOS_CHECKOUT_DIR`` argument is set to the value of the ``DCOS_CHECKOUT_DIR`` environment variable. If no ``DCOS_CHECKOUT_DIR`` is given, the current working directory is used. """ set_logging(verbosity_level=verbose) check_cluster_id_exists( new_cluster_id=cluster_id, existing_cluster_ids=existing_cluster_ids(), ) cluster_containers = ClusterVMs(cluster_id=cluster_id) cluster = cluster_containers.cluster sync_code_to_masters( cluster=cluster, dcos_checkout_dir=Path(dcos_checkout_dir), )
def run( cluster_id: str, node_args: Tuple[str], sync_dir: Optional[Path], dcos_login_uname: str, dcos_login_pw: str, no_test_env: bool, env: Dict[str, str], aws_region: str, verbose: int, node: str, ) -> None: """ Run an arbitrary command on a node. This command sets up the environment so that ``pytest`` can be run. For example, run ``dcos-aws run --cluster-id 1231599 pytest -k test_tls.py``. Or, with sync: ``dcos-aws run --sync-dir . --cluster-id 1231599 pytest -k test_tls.py``. To use special characters such as single quotes in your command, wrap the whole command in double quotes. """ # noqa: E501 set_logging(verbosity_level=verbose) check_cluster_id_exists( new_cluster_id=cluster_id, existing_cluster_ids=existing_cluster_ids(aws_region=aws_region), ) cluster_instances = ClusterInstances( cluster_id=cluster_id, aws_region=aws_region, ) cluster = cluster_instances.cluster host = _get_node( cluster_id=cluster_id, node_reference=node, aws_region=aws_region, ) if sync_dir is not None: sync_code_to_masters( cluster=cluster, dcos_checkout_dir=sync_dir, ) run_command( args=list(node_args), cluster=cluster, host=host, use_test_env=not no_test_env, dcos_login_uname=dcos_login_uname, dcos_login_pw=dcos_login_pw, env=env, transport=Transport.SSH, )
def doctor(verbose: int) -> None: """ Diagnose common issues which stop DC/OS E2E from working correctly. """ set_logging(verbosity_level=verbose) check_functions = [check_ssh] for function in check_functions: level = function() if level == CheckLevels.ERROR: sys.exit(1)
def wait( ctx: click.core.Context, cluster_id: str, superuser_username: str, superuser_password: str, transport: Transport, skip_http_checks: bool, verbose: int, ) -> None: """ Wait for DC/OS to start. """ check_cluster_id_exists( new_cluster_id=cluster_id, existing_cluster_ids=existing_cluster_ids(), ) set_logging(verbosity_level=verbose) urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) cluster_containers = ClusterContainers( cluster_id=cluster_id, transport=transport, ) http_checks = not skip_http_checks parent = ctx.parent assert parent is not None doctor_command_name = '{info_name} {doctor_name}'.format( info_name=parent.info_name, doctor_name=doctor.name, ) show_wait_help( is_enterprise=cluster_containers.is_enterprise, doctor_command_name=doctor_command_name, ) with click_spinner.spinner(): if cluster_containers.is_enterprise: cluster_containers.cluster.wait_for_dcos_ee( superuser_username=superuser_username, superuser_password=superuser_password, http_checks=http_checks, ) return cluster_containers.cluster.wait_for_dcos_oss(http_checks=http_checks)
def create( agents: int, artifact: str, extra_config: Dict[str, Any], masters: int, public_agents: int, variant: str, workspace_dir: Optional[Path], license_key: Optional[str], security_mode: Optional[str], copy_to_master: List[Tuple[Path, Path]], cluster_id: str, verbose: int, enable_selinux_enforcing: bool, genconf_dir: Optional[Path], ) -> None: """ Create a DC/OS cluster. DC/OS Enterprise \b DC/OS Enterprise clusters require different configuration variables to DC/OS OSS. For example, enterprise clusters require the following configuration parameters: ``superuser_username``, ``superuser_password_hash``, ``fault_domain_enabled``, ``license_key_contents`` \b These can all be set in ``--extra-config``. However, some defaults are provided for all but the license key. \b The default superuser username is ``admin``. The default superuser password is ``admin``. The default ``fault_domain_enabled`` is ``false``. \b ``license_key_contents`` must be set for DC/OS Enterprise 1.11 and above. This is set to one of the following, in order: \b * The ``license_key_contents`` set in ``--extra-config``. * The contents of the path given with ``--license-key``. * The contents of the path set in the ``DCOS_LICENSE_KEY_PATH`` environment variable. \b If none of these are set, ``license_key_contents`` is not given. """ # noqa: E501 set_logging(verbosity_level=verbose) check_cluster_id_unique( new_cluster_id=cluster_id, existing_cluster_ids=existing_cluster_ids(), ) base_workspace_dir = workspace_dir or Path(tempfile.gettempdir()) workspace_dir = base_workspace_dir / uuid.uuid4().hex workspace_dir.mkdir(parents=True) doctor_message = 'Try `dcos-vagrant doctor` for troubleshooting help.' artifact_path = Path(artifact).resolve() if variant == 'auto': variant = get_variant( artifact_path=artifact_path, workspace_dir=workspace_dir, doctor_message=doctor_message, ) enterprise = bool(variant == 'enterprise') description = { CLUSTER_ID_DESCRIPTION_KEY: cluster_id, WORKSPACE_DIR_DESCRIPTION_KEY: str(workspace_dir), VARIANT_DESCRIPTION_KEY: 'ee' if enterprise else '', } cluster_backend = Vagrant( workspace_dir=workspace_dir, virtualbox_description=json.dumps(obj=description), ) if enterprise: superuser_username = '******' superuser_password = '******' enterprise_extra_config = { 'superuser_username': superuser_username, 'superuser_password_hash': sha512_crypt.hash(superuser_password), 'fault_domain_enabled': False, } if license_key is not None: key_contents = Path(license_key).read_text() enterprise_extra_config['license_key_contents'] = key_contents extra_config = {**enterprise_extra_config, **extra_config} if security_mode is not None: extra_config['security'] = security_mode try: cluster = Cluster( cluster_backend=cluster_backend, masters=masters, agents=agents, public_agents=public_agents, ) except CalledProcessError as exc: click.echo('Error creating cluster.', err=True) click.echo(doctor_message) sys.exit(exc.returncode) nodes = {*cluster.masters, *cluster.agents, *cluster.public_agents} for node in nodes: if enable_selinux_enforcing: node.run(args=['setenforce', '1'], sudo=True) for node in cluster.masters: for path_pair in copy_to_master: local_path, remote_path = path_pair node.send_file( local_path=local_path, remote_path=remote_path, ) files_to_copy_to_genconf_dir = [] if genconf_dir is not None: container_genconf_path = Path('/genconf') for genconf_file in genconf_dir.glob('*'): genconf_relative = genconf_file.relative_to(genconf_dir) relative_path = container_genconf_path / genconf_relative files_to_copy_to_genconf_dir.append((genconf_file, relative_path)) try: with click_spinner.spinner(): cluster.install_dcos_from_path( build_artifact=artifact_path, dcos_config={ **cluster.base_config, **extra_config, }, ip_detect_path=cluster_backend.ip_detect_path, files_to_copy_to_genconf_dir=files_to_copy_to_genconf_dir, ) except CalledProcessError as exc: click.echo('Error installing DC/OS.', err=True) click.echo(doctor_message) cluster.destroy() sys.exit(exc.returncode)
def create( ctx: click.core.Context, agents: int, artifact: str, cluster_id: str, docker_storage_driver: str, docker_version: str, extra_config: Dict[str, Any], linux_distribution: str, masters: int, public_agents: int, license_key: Optional[str], security_mode: Optional[str], copy_to_master: List[Tuple[Path, Path]], genconf_dir: Optional[Path], workspace_dir: Optional[Path], custom_volume: List[Mount], custom_master_volume: List[Mount], custom_agent_volume: List[Mount], custom_public_agent_volume: List[Mount], variant: str, transport: Transport, wait_for_dcos: bool, network: Network, one_master_host_port_map: Dict[str, int], verbose: int, ) -> None: """ Create a DC/OS cluster. DC/OS Enterprise \b DC/OS Enterprise clusters require different configuration variables to DC/OS OSS. For example, enterprise clusters require the following configuration parameters: ``superuser_username``, ``superuser_password_hash``, ``fault_domain_enabled``, ``license_key_contents`` \b These can all be set in ``--extra-config``. However, some defaults are provided for all but the license key. \b The default superuser username is ``admin``. The default superuser password is ``admin``. The default ``fault_domain_enabled`` is ``false``. \b ``license_key_contents`` must be set for DC/OS Enterprise 1.11 and above. This is set to one of the following, in order: \b * The ``license_key_contents`` set in ``--extra-config``. * The contents of the path given with ``--license-key``. * The contents of the path set in the ``DCOS_LICENSE_KEY_PATH`` environment variable. \b If none of these are set, ``license_key_contents`` is not given. """ # noqa: E501 set_logging(verbosity_level=verbose) check_cluster_id_unique( new_cluster_id=cluster_id, existing_cluster_ids=existing_cluster_ids(), ) base_workspace_dir = workspace_dir or Path(tempfile.gettempdir()) workspace_dir = base_workspace_dir / uuid.uuid4().hex doctor_message = 'Try `dcos-docker doctor` for troubleshooting help.' ssh_keypair_dir = workspace_dir / 'ssh' ssh_keypair_dir.mkdir(parents=True) public_key_path = ssh_keypair_dir / 'id_rsa.pub' private_key_path = ssh_keypair_dir / 'id_rsa' write_key_pair( public_key_path=public_key_path, private_key_path=private_key_path, ) artifact_path = Path(artifact).resolve() if variant == 'auto': variant = get_variant( artifact_path=artifact_path, workspace_dir=workspace_dir, doctor_message=doctor_message, ) enterprise = bool(variant == 'enterprise') if enterprise: superuser_username = '******' superuser_password = '******' enterprise_extra_config = { 'superuser_username': superuser_username, 'superuser_password_hash': sha512_crypt.hash(superuser_password), 'fault_domain_enabled': False, } if license_key is not None: key_contents = Path(license_key).read_text() enterprise_extra_config['license_key_contents'] = key_contents extra_config = {**enterprise_extra_config, **extra_config} if security_mode is not None: extra_config['security'] = security_mode files_to_copy_to_genconf_dir = [] if genconf_dir is not None: container_genconf_path = Path('/genconf') for genconf_file in genconf_dir.glob('*'): genconf_relative = genconf_file.relative_to(genconf_dir) relative_path = container_genconf_path / genconf_relative files_to_copy_to_genconf_dir.append((genconf_file, relative_path)) cluster_backend = Docker( custom_container_mounts=custom_volume, custom_master_mounts=custom_master_volume, custom_agent_mounts=custom_agent_volume, custom_public_agent_mounts=custom_public_agent_volume, linux_distribution=LINUX_DISTRIBUTIONS[linux_distribution], docker_version=DOCKER_VERSIONS[docker_version], storage_driver=DOCKER_STORAGE_DRIVERS.get(docker_storage_driver), docker_container_labels={ CLUSTER_ID_LABEL_KEY: cluster_id, WORKSPACE_DIR_LABEL_KEY: str(workspace_dir), VARIANT_LABEL_KEY: 'ee' if enterprise else '', }, docker_master_labels={ NODE_TYPE_LABEL_KEY: NODE_TYPE_MASTER_LABEL_VALUE, }, docker_agent_labels={NODE_TYPE_LABEL_KEY: NODE_TYPE_AGENT_LABEL_VALUE}, docker_public_agent_labels={ NODE_TYPE_LABEL_KEY: NODE_TYPE_PUBLIC_AGENT_LABEL_VALUE, }, workspace_dir=workspace_dir, transport=transport, network=network, one_master_host_port_map=one_master_host_port_map, ) try: cluster = Cluster( cluster_backend=cluster_backend, masters=masters, agents=agents, public_agents=public_agents, ) except CalledProcessError as exc: click.echo('Error creating cluster.', err=True) click.echo(doctor_message) sys.exit(exc.returncode) nodes = { *cluster.masters, *cluster.agents, *cluster.public_agents, } for node in nodes: node.run( args=['echo', '', '>>', '/root/.ssh/authorized_keys'], shell=True, ) node.run( args=[ 'echo', public_key_path.read_text(), '>>', '/root/.ssh/authorized_keys', ], shell=True, ) for node in cluster.masters: for path_pair in copy_to_master: local_path, remote_path = path_pair node.send_file( local_path=local_path, remote_path=remote_path, ) try: with click_spinner.spinner(): cluster.install_dcos_from_path( build_artifact=artifact_path, dcos_config={ **cluster.base_config, **extra_config, }, ip_detect_path=cluster_backend.ip_detect_path, files_to_copy_to_genconf_dir=files_to_copy_to_genconf_dir, ) except CalledProcessError as exc: click.echo('Error installing DC/OS.', err=True) click.echo(doctor_message) cluster.destroy() sys.exit(exc.returncode) click.echo(cluster_id) if wait_for_dcos: ctx.invoke( wait, cluster_id=cluster_id, transport=transport, skip_http_checks=bool(transport == Transport.DOCKER_EXEC), ) return started_message = ( 'Cluster "{cluster_id}" has started. ' 'Run "dcos-docker wait --cluster-id {cluster_id}" to wait for DC/OS ' 'to become ready.').format(cluster_id=cluster_id) click.echo(started_message, err=True)
def create( agents: int, artifact_url: str, extra_config: Dict[str, Any], masters: int, public_agents: int, variant: str, workspace_dir: Optional[Path], license_key: Optional[str], security_mode: Optional[str], copy_to_master: List[Tuple[Path, Path]], verbose: int, aws_region: str, linux_distribution: str, cluster_id: str, enable_selinux_enforcing: bool, genconf_dir: Optional[Path], custom_tag: Dict[str, str], ) -> None: """ Create a DC/OS cluster. DC/OS Enterprise \b DC/OS Enterprise clusters require different configuration variables to DC/OS OSS. For example, enterprise clusters require the following configuration parameters: ``superuser_username``, ``superuser_password_hash``, ``fault_domain_enabled``, ``license_key_contents`` \b These can all be set in ``--extra-config``. However, some defaults are provided for all but the license key. \b The default superuser username is ``admin``. The default superuser password is ``admin``. The default ``fault_domain_enabled`` is ``false``. \b ``license_key_contents`` must be set for DC/OS Enterprise 1.11 and above. This is set to one of the following, in order: \b * The ``license_key_contents`` set in ``--extra-config``. * The contents of the path given with ``--license-key``. * The contents of the path set in the ``DCOS_LICENSE_KEY_PATH`` environment variable. \b If none of these are set, ``license_key_contents`` is not given. """ # noqa: E501 set_logging(verbosity_level=verbose) check_cluster_id_unique( new_cluster_id=cluster_id, existing_cluster_ids=existing_cluster_ids(aws_region=aws_region), ) base_workspace_dir = workspace_dir or Path(tempfile.gettempdir()) workspace_dir = base_workspace_dir / uuid.uuid4().hex workspace_dir.mkdir(parents=True) ssh_keypair_dir = workspace_dir / 'ssh' ssh_keypair_dir.mkdir(parents=True) key_name = 'dcos-e2e-{random}'.format(random=uuid.uuid4().hex) public_key_path = ssh_keypair_dir / 'id_rsa.pub' private_key_path = ssh_keypair_dir / 'id_rsa' write_key_pair( public_key_path=public_key_path, private_key_path=private_key_path, ) ec2 = boto3.resource('ec2', region_name=aws_region) ec2.import_key_pair( KeyName=key_name, PublicKeyMaterial=public_key_path.read_bytes(), ) doctor_message = 'Try `dcos-aws doctor` for troubleshooting help.' enterprise = bool(variant == 'enterprise') ssh_user = { Distribution.CENTOS_7: 'centos', Distribution.COREOS: 'core', Distribution.UBUNTU_16_04: 'ubuntu', Distribution.RHEL_7: 'ec2-user', } distribution = LINUX_DISTRIBUTIONS[linux_distribution] default_user = ssh_user[distribution] cluster_tags = { SSH_USER_TAG_KEY: default_user, CLUSTER_ID_TAG_KEY: cluster_id, WORKSPACE_DIR_TAG_KEY: str(workspace_dir), KEY_NAME_TAG_KEY: key_name, VARIANT_TAG_KEY: 'ee' if enterprise else '', **custom_tag, } master_tags = {NODE_TYPE_TAG_KEY: NODE_TYPE_MASTER_TAG_VALUE} agent_tags = {NODE_TYPE_TAG_KEY: NODE_TYPE_AGENT_TAG_VALUE} public_agent_tags = {NODE_TYPE_TAG_KEY: NODE_TYPE_PUBLIC_AGENT_TAG_VALUE} cluster_backend = AWS( aws_key_pair=(key_name, private_key_path), workspace_dir=workspace_dir, aws_region=aws_region, linux_distribution=distribution, ec2_instance_tags=cluster_tags, master_ec2_instance_tags=master_tags, agent_ec2_instance_tags=agent_tags, public_agent_ec2_instance_tags=public_agent_tags, ) if enterprise: superuser_username = '******' superuser_password = '******' enterprise_extra_config = { 'superuser_username': superuser_username, 'superuser_password_hash': sha512_crypt.hash(superuser_password), 'fault_domain_enabled': False, } if license_key is not None: key_contents = Path(license_key).read_text() enterprise_extra_config['license_key_contents'] = key_contents extra_config = {**enterprise_extra_config, **extra_config} if security_mode is not None: extra_config['security'] = security_mode try: cluster = Cluster( cluster_backend=cluster_backend, masters=masters, agents=agents, public_agents=public_agents, ) except CalledProcessError as exc: click.echo('Error creating cluster.', err=True) click.echo(doctor_message) sys.exit(exc.returncode) nodes = {*cluster.masters, *cluster.agents, *cluster.public_agents} for node in nodes: if enable_selinux_enforcing: node.run(args=['setenforce', '1'], sudo=True) for node in cluster.masters: for path_pair in copy_to_master: local_path, remote_path = path_pair node.send_file( local_path=local_path, remote_path=remote_path, ) files_to_copy_to_genconf_dir = [] if genconf_dir is not None: container_genconf_path = Path('/genconf') for genconf_file in genconf_dir.glob('*'): genconf_relative = genconf_file.relative_to(genconf_dir) relative_path = container_genconf_path / genconf_relative files_to_copy_to_genconf_dir.append((genconf_file, relative_path)) try: with click_spinner.spinner(): cluster.install_dcos_from_url( build_artifact=artifact_url, dcos_config={ **cluster.base_config, **extra_config, }, ip_detect_path=cluster_backend.ip_detect_path, files_to_copy_to_genconf_dir=files_to_copy_to_genconf_dir, ) except CalledProcessError as exc: click.echo('Error installing DC/OS.', err=True) click.echo(doctor_message) cluster.destroy() sys.exit(exc.returncode)