def test_uninstall_multiple_apps():
stderr = (
b'This is a Community service. '
b'Community services are not tested '
b'for production environments. '
b'There may be bugs, incomplete features, '
b'incorrect documentation, or other discrepancies.\n'
b'By Deploying, you agree to the Terms '
b'and Conditions https://mesosphere.com/'
b'catalog-terms-conditions/#community-services\n'
b'A sample pre-installation message\n'
b'Installing Marathon app for package [helloworld] version '
b'[0.1.0]\n'
b'A sample post-installation message\n'
)
with util.temptext(b'{"name": "/helloworld-1"}') as hello1, \
util.temptext(b'{"name": "/helloworld-2"}') as hello2:
_install_helloworld(
['--yes', '--options='+hello1[1], '--app'],
stderr=stderr)
_install_helloworld(
['--yes', '--options='+hello2[1], '--app'],
stderr=stderr)
stderr = (b"Multiple apps named [helloworld] are installed: "
b"[/helloworld-1, /helloworld-2].\n"
b"Please use --app-id to specify the ID of the app "
b"to uninstall, or use --all to uninstall all apps.\n")
_uninstall_helloworld(stderr=stderr, returncode=1, uninstalled=b'')
_uninstall_helloworld(args=['--all'])
def test_uninstall_multiple_apps():
stdout = (
b'By Deploying, you agree to the Terms '
b'and Conditions https://mesosphere.com/'
b'catalog-terms-conditions/#community-services\n'
b'A sample pre-installation message\n'
b'Installing Marathon app for package [helloworld] version '
b'[0.1.0]\n'
b'A sample post-installation message\n'
)
with util.temptext(b'{"name": "/helloworld-1"}') as hello1, \
util.temptext(b'{"name": "/helloworld-2"}') as hello2:
_install_helloworld(
['--yes', '--options='+hello1[1], '--app'],
stdout=stdout)
_install_helloworld(
['--yes', '--options='+hello2[1], '--app'],
stdout=stdout)
stderr = (b"Multiple apps named [helloworld] are installed: "
b"[/helloworld-1, /helloworld-2].\n"
b"Please use --app-id to specify the ID of the app "
b"to uninstall, or use --all to uninstall all apps.\n")
_uninstall_helloworld(stderr=stderr, returncode=1, uninstalled=b'')
_uninstall_helloworld(args=['--all'])
def test_bad_install_helloworld_msg():
terms_conditions = (
b'This is a Community service. '
b'Community services are not tested '
b'for production environments. '
b'There may be bugs, incomplete features, '
b'incorrect documentation, or other discrepancies.\n'
b'By Deploying, you agree to the Terms '
b'and Conditions https://mesosphere.com/'
b'catalog-terms-conditions/#community-services\n'
b'A sample pre-installation message\n'
b'Installing Marathon app for package [helloworld] version '
b'[0.1.0]\n'
)
stderr = (
terms_conditions +
b'Installing CLI subcommand for package [helloworld] '
b'version [0.1.0]\n'
b'New commands available: http\n'
b'A sample post-installation message\n'
)
with util.temptext(b'{"name": "/foo"}') as foo, \
util.temptext(b'{"name": "/foo/bar"}') as foobar:
_install_helloworld(['--yes', '--options='+foo[1]],
stderr=stderr)
stderr = terms_conditions + b'Error: Object is not valid\n'
_install_helloworld(['--yes', '--options='+foobar[1]],
stderr=stderr,
returncode=1)
_uninstall_helloworld()
def test_bad_install_helloworld_msg():
terms_conditions = (
b'By Deploying, you agree to the Terms '
b'and Conditions https://mesosphere.com/'
b'catalog-terms-conditions/#community-services\n'
b'A sample pre-installation message\n'
b'Installing Marathon app for package [helloworld] version '
b'[0.1.0]\n'
)
stdout = (
terms_conditions +
b'Installing CLI subcommand for package [helloworld] '
b'version [0.1.0]\n'
b'New command available: dcos ' +
_executable_name(b'helloworld') +
b'\nA sample post-installation message\n'
)
with util.temptext(b'{"name": "/foo"}') as foo, \
util.temptext(b'{"name": "/foo/bar"}') as foobar:
_install_helloworld(['--yes', '--options='+foo[1]], stdout=stdout)
stderr = (b'Object is not valid\n'
b'Groups and Applications may not have the same '
b'identifier.\n')
_install_helloworld(['--yes', '--options='+foobar[1]],
stdout=terms_conditions,
stderr=stderr,
returncode=1)
_uninstall_helloworld()
def test_describe_options():
stdout = file_json(
'tests/data/package/json/test_describe_app_options.json')
stdout = json.loads(stdout.decode('utf-8'))
expected_labels = stdout.pop("labels", None)
with util.temptext(b'{"name": "hallo", "port": 80}') as options:
returncode, stdout_, stderr = exec_command(
['dcos', 'package', 'describe', '--app', '--options',
options[1], 'helloworld'])
stdout_ = json.loads(stdout_.decode('utf-8'))
actual_labels = stdout_.pop("labels", None)
for label, value in expected_labels.items():
if label in ["DCOS_PACKAGE_METADATA", "DCOS_PACKAGE_OPTIONS"]:
# We covert the metadata into a dictionary
# so that failures in equality are more descriptive
assert base64_to_dict(value) == \
base64_to_dict(actual_labels.get(label))
else:
assert value == actual_labels.get(label)
assert stdout == stdout_
assert stderr == b''
assert returncode == 0
def _store_cluster_cert(cert, no_check):
"""Store cluster certificate bundle downloaded from cluster and store
settings in core.ssl_verify
:param cert: ca cert from cluster
:type cert: str
:param no_check: whether to verify downloaded cert
:type no_check: bool
:returns: whether or not we are storing the downloaded cert bundle
:rtype: bool
"""
if not no_check:
if not _user_cert_validation(cert):
# we don't have a cert, but we still want to validate SSL
config.set_val("core.ssl_verify", "true")
return False
with util.temptext() as temp_file:
_, temp_path = temp_file
with open(temp_path, 'w') as f:
f.write(cert)
cert_path = os.path.join(
config.get_attached_cluster_path(), "dcos_ca.crt")
util.sh_copy(temp_path, cert_path)
config.set_val("core.ssl_verify", cert_path)
return True
def _store_cluster_cert(cert, no_check):
"""Store cluster certificate bundle downloaded from cluster and store
settings in core.ssl_verify
:param cert: ca cert from cluster
:type cert: str
:param no_check: whether to verify downloaded cert
:type no_check: bool
:returns: whether or not we are storing the downloaded cert bundle
:rtype: bool
"""
if not no_check and not _user_cert_validation(cert):
raise DCOSException("Couldn't get confirmation for the fingerprint.")
with util.temptext() as temp_file:
_, temp_path = temp_file
with open(temp_path, 'w') as f:
f.write(cert)
cert_path = os.path.join(
config.get_attached_cluster_path(), "dcos_ca.crt")
util.sh_copy(temp_path, cert_path)
config.set_val("core.ssl_verify", cert_path)
return True
def test_uninstall_cli_only_when_no_apps_remain():
with util.temptext(b'{"name": "/hello1"}') as opts_hello1, \
util.temptext(b'{"name": "/hello2"}') as opts_hello2:
stderr = (
b'This is a Community service. '
b'Community services are not tested '
b'for production environments. '
b'There may be bugs, incomplete features, '
b'incorrect documentation, or other discrepancies.\n'
b'By Deploying, you agree to the Terms '
b'and Conditions https://mesosphere.com/'
b'catalog-terms-conditions/#community-services\n'
b'A sample pre-installation message\n'
b'Installing Marathon app for package [helloworld] version '
b'[0.1.0]\n'
b'Installing CLI subcommand for package [helloworld] '
b'version [0.1.0]\n'
b'New commands available: http\n'
b'A sample post-installation message\n'
)
uninstall_stderr = (
b'Uninstalled package [helloworld] version [0.1.0]\n'
)
with _package(name='helloworld',
args=['--yes', '--options='+opts_hello1[1]],
stderr=stderr,
uninstall_app_id='/hello1',
uninstall_stderr=uninstall_stderr):
with _package(name='helloworld',
args=['--yes', '--options='+opts_hello2[1]],
stderr=stderr,
uninstall_app_id='/hello2',
uninstall_stderr=uninstall_stderr):
subcommand.command_executables('http')
# helloworld subcommand should still be there as there is the
# /hello1 app installed
subcommand.command_executables('http')
# helloworld subcommand should have been removed
with pytest.raises(errors.DCOSException) as exc_info:
subcommand.command_executables('helloworld')
assert str(exc_info.value) == "'helloworld' is not a dcos command."
def _install_with_binary(package_name, env_directory, binary_cli):
"""
:param package_name: the name of the package
:type package_name: str
:param env_directory: the path to the directory in which to install the
package's binary_cli
:type env_directory: str
:param binary_cli: binary cli to install
:type binary_cli: str
:rtype: None
"""
binary_url, kind = binary_cli.get("url"), binary_cli.get("kind")
try:
env_bin_dir = os.path.join(env_directory, BIN_DIRECTORY)
if kind in ["executable", "zip"]:
with util.temptext() as file_tmp:
_, binary_tmp = file_tmp
_download_and_store(binary_url, binary_tmp)
_check_hash(binary_tmp, binary_cli.get("contentHash"))
if kind == "executable":
util.ensure_dir_exists(env_bin_dir)
binary_name = "dcos-{}".format(package_name)
binary_file = os.path.join(env_bin_dir, binary_name)
shutil.move(binary_tmp, binary_file)
else:
# kind == "zip"
with zipfile.ZipFile(binary_tmp) as zf:
zf.extractall(env_directory)
# check contents for package_name/env/bin folder structure
if not os.path.exists(env_bin_dir):
msg = (
"CLI subcommand for [{}] has an unexpected format. "
"Please contact the package maintainer".format(package_name)
)
raise DCOSException(msg)
else:
msg = "CLI subcommand for [{}] is an unsupported type: {}" "Please contact the package maintainer".format(
package_name, kind
)
raise DCOSException(msg)
# make binar(ies) executable
for f in os.listdir(env_bin_dir):
binary = os.path.join(env_bin_dir, f)
if f.startswith(constants.DCOS_COMMAND_PREFIX):
st = os.stat(binary)
os.chmod(binary, st.st_mode | stat.S_IEXEC)
except DCOSException:
raise
except Exception as e:
logger.exception(e)
raise _generic_error(package_name)
return None
def _install_with_pip(package_name, env_directory, requirements):
"""
:param package_name: the name of the package
:type package_name: str
:param env_directory: the path to the directory in which to install the
package's virtual env
:type env_directory: str
:param requirements: the list of pip requirements
:type requirements: list of str
:rtype: None
"""
bin_directory = util.dcos_bin_path()
new_package_dir = not os.path.exists(env_directory)
pip_path = os.path.join(env_directory, BIN_DIRECTORY, 'pip')
if not os.path.exists(pip_path):
virtualenv_path = _find_virtualenv(bin_directory)
virtualenv_version = _execute_command([virtualenv_path, '--version'
])[0].strip().decode('utf-8')
if LooseVersion("12") > LooseVersion(virtualenv_version):
msg = ("Unable to install CLI subcommand. "
"Required program 'virtualenv' must be version 12+, "
"currently version {}\n"
"Please see installation instructions: "
"https://virtualenv.pypa.io/en/latest/installation.html"
"".format(virtualenv_version))
raise DCOSException(msg)
cmd = [_find_virtualenv(bin_directory), env_directory]
if _execute_command(cmd)[2] != 0:
raise _generic_error(package_name)
# Do not replace util.temptext NamedTemporaryFile
# otherwise bad things will happen on Windows
with util.temptext() as text_file:
fd, requirement_path = text_file
# Write the requirements to the file
with os.fdopen(fd, 'w') as requirements_file:
for line in requirements:
print(line, file=requirements_file)
cmd = [
os.path.join(env_directory, BIN_DIRECTORY, 'pip'),
'install',
'--requirement',
requirement_path,
]
if _execute_command(cmd)[2] != 0:
# We should remove the directory that we just created
if new_package_dir:
shutil.rmtree(env_directory)
raise _generic_error(package_name)
return None
def test_bad_install():
stderr = """\
Please create a JSON file with the appropriate options, and pass the \
/path/to/file as an --options argument.
"""
with util.temptext(b'{"nom": "hallo"}') as options:
args = ['--options='+options[1], '--yes']
_install_bad_helloworld(args=args, stderr=stderr)
def _install_with_binary(package_name, env_directory, binary_cli):
"""
:param package_name: the name of the package
:type package_name: str
:param env_directory: the path to the directory in which to install the
package's binary_cli
:type env_directory: str
:param binary_cli: binary cli to install
:type binary_cli: str
:rtype: None
"""
binary_url, kind = binary_cli.get("url"), binary_cli.get("kind")
try:
env_bin_dir = os.path.join(env_directory, BIN_DIRECTORY)
if kind in ["executable", "zip"]:
with util.temptext() as file_tmp:
_, binary_tmp = file_tmp
_download_and_store(binary_url, binary_tmp)
_check_hash(binary_tmp, binary_cli.get("contentHash"))
if kind == "executable":
util.ensure_dir_exists(env_bin_dir)
binary_name = "dcos-{}".format(package_name)
binary_file = os.path.join(env_bin_dir, binary_name)
shutil.move(binary_tmp, binary_file)
else:
# kind == "zip"
with zipfile.ZipFile(binary_tmp) as zf:
zf.extractall(env_directory)
# check contents for package_name/env/bin folder structure
if not os.path.exists(env_bin_dir):
msg = ("CLI subcommand for [{}] has an unexpected format. "
"Please contact the package maintainer".format(
package_name))
raise DCOSException(msg)
else:
msg = ("CLI subcommand for [{}] is an unsupported type: {}"
"Please contact the package maintainer".format(
package_name, kind))
raise DCOSException(msg)
# make binar(ies) executable
for f in os.listdir(env_bin_dir):
binary = os.path.join(env_bin_dir, f)
if (f.startswith(constants.DCOS_COMMAND_PREFIX)):
st = os.stat(binary)
os.chmod(binary, st.st_mode | stat.S_IEXEC)
except DCOSException:
raise
except Exception as e:
logger.exception(e)
raise _generic_error(package_name)
return None
def _install_with_pip(package_name, env_directory, requirements):
"""
:param package_name: the name of the package
:type package_name: str
:param env_directory: the path to the directory in which to install the
package's virtual env
:type env_directory: str
:param requirements: the list of pip requirements
:type requirements: list of str
:rtype: None
"""
bin_directory = util.dcos_bin_path()
new_package_dir = not os.path.exists(env_directory)
pip_path = os.path.join(env_directory, BIN_DIRECTORY, "pip")
if not os.path.exists(pip_path):
virtualenv_path = _find_virtualenv(bin_directory)
virtualenv_version = _execute_command([virtualenv_path, "--version"])[0].strip().decode("utf-8")
if LooseVersion("12") > LooseVersion(virtualenv_version):
msg = (
"Unable to install CLI subcommand. "
"Required program 'virtualenv' must be version 12+, "
"currently version {}\n"
"Please see installation instructions: "
"https://virtualenv.pypa.io/en/latest/installation.html"
"".format(virtualenv_version)
)
raise DCOSException(msg)
cmd = [_find_virtualenv(bin_directory), env_directory]
if _execute_command(cmd)[2] != 0:
raise _generic_error(package_name)
# Do not replace util.temptext NamedTemporaryFile
# otherwise bad things will happen on Windows
with util.temptext() as text_file:
fd, requirement_path = text_file
# Write the requirements to the file
with os.fdopen(fd, "w") as requirements_file:
for line in requirements:
print(line, file=requirements_file)
cmd = [os.path.join(env_directory, BIN_DIRECTORY, "pip"), "install", "--requirement", requirement_path]
if _execute_command(cmd)[2] != 0:
# We should remove the directory that we just created
if new_package_dir:
shutil.rmtree(env_directory)
raise _generic_error(package_name)
return None
def test_uninstall_cli_only_when_no_apps_remain():
with util.temptext(b'{"name": "/hello1"}') as opts_hello1, \
util.temptext(b'{"name": "/hello2"}') as opts_hello2:
stdout = (b'By Deploying, you agree to the Terms '
b'and Conditions https://mesosphere.com/'
b'catalog-terms-conditions/#community-services\n'
b'A sample pre-installation message\n'
b'Installing Marathon app for package [helloworld] version '
b'[0.1.0]\n'
b'Installing CLI subcommand for package [helloworld] '
b'version [0.1.0]\n'
b'New command available: dcos ' +
_executable_name(b'helloworld') +
b'\nA sample post-installation message\n')
stderr = b'Uninstalled package [helloworld] version [0.1.0]\n'
with _package(name='helloworld',
args=['--yes', '--options=' + opts_hello1[1]],
stdout=stdout,
uninstall_app_id='/hello1',
uninstall_stderr=stderr):
with _package(name='helloworld',
args=['--yes', '--options=' + opts_hello2[1]],
stdout=stdout,
uninstall_app_id='/hello2',
uninstall_stderr=stderr):
subcommand.command_executables('helloworld')
# helloworld subcommand should still be there as there is the
# /hello1 app installed
subcommand.command_executables('helloworld')
# helloworld subcommand should have been removed
try:
subcommand.command_executables('helloworld')
except errors.DCOSException as e:
assert str(e) == "'helloworld' is not a dcos command."
return
assert False, "expected command_executables('helloworld') to fail"
def setup_module():
print("SETUP")
app = {
'id':
"/{}".format(TEST_APP_NAME),
'cpus':
0.1,
'mem':
32,
'instances':
1,
'cmd':
'/opt/mesosphere/bin/dcos-shell python '
'/opt/mesosphere/active/dcos-integration-test/util/python_test_server.py $PORT0',
'env': {
'DCOS_TEST_UUID': TEST_APP_NAME,
# required for python_test_server.py to run as nobody
'HOME': '/'
},
'healthChecks': [{
'protocol': 'MESOS_HTTP',
'path': '/ping',
'portIndex': 0,
'gracePeriodSeconds': 5,
'intervalSeconds': 10,
'timeoutSeconds': 10,
'maxConsecutiveFailures': 3
}],
'portDefinitions': [{
"protocol": TEST_PORT_PROTOCOL,
"port": TEST_PORT_NUMBER,
"name": TEST_PORT_NAME
}],
'acceptedResourceRoles': ["slave_public"],
'requirePorts':
True,
}
with util.temptext() as file_tup:
f, fpath = file_tup
app_bits = json.dumps(app).encode()
os.write(f, app_bits)
os.fsync(f)
add_com = "dcos marathon app add {}".format(fpath)
subprocess.call(shlex.split(add_com))
time.sleep(5)
for _ in range(360):
ret = subprocess.call(shlex.split("dcos marathon deployment list"))
if ret != 0:
break
time.sleep(1)
def _install_with_pip(
package_name,
env_directory,
requirements):
"""
:param package_name: the name of the package
:type package_name: str
:param env_directory: the path to the directory in which to install the
package's virtual env
:type env_directory: str
:param requirements: the list of pip requirements
:type requirements: list of str
:rtype: None
"""
bin_directory = util.dcos_bin_path()
new_package_dir = not os.path.exists(env_directory)
pip_path = os.path.join(env_directory, BIN_DIRECTORY, 'pip')
if not os.path.exists(pip_path):
cmd = [_find_virtualenv(bin_directory), env_directory]
if _execute_install(cmd) != 0:
raise _generic_error(package_name)
# Do not replace util.temptext NamedTemporaryFile
# otherwise bad things will happen on Windows
with util.temptext() as text_file:
fd, requirement_path = text_file
# Write the requirements to the file
with os.fdopen(fd, 'w') as requirements_file:
for line in requirements:
print(line, file=requirements_file)
cmd = [
os.path.join(env_directory, BIN_DIRECTORY, 'pip'),
'install',
'--requirement',
requirement_path,
]
if _execute_install(cmd) != 0:
# We should remove the directory that we just created
if new_package_dir:
shutil.rmtree(env_directory)
raise _generic_error(package_name)
return None
def _install_with_pip(
package_name,
env_directory,
requirements):
"""
:param package_name: the name of the package
:type package_name: str
:param env_directory: the path to the directory in which to install the
package's virtual env
:type env_directory: str
:param requirements: the list of pip requirements
:type requirements: list of str
:rtype: None
"""
bin_directory = util.dcos_bin_path()
new_package_dir = not os.path.exists(env_directory)
pip_path = os.path.join(env_directory, BIN_DIRECTORY, 'pip')
if not os.path.exists(pip_path):
cmd = [_find_virtualenv(bin_directory), env_directory]
if _execute_install(cmd) != 0:
raise _generic_error(package_name)
# Do not replace util.temptext NamedTemporaryFile
# otherwise bad things will happen on Windows
with util.temptext() as text_file:
fd, requirement_path = text_file
# Write the requirements to the file
with os.fdopen(fd, 'w') as requirements_file:
for line in requirements:
print(line, file=requirements_file)
cmd = [
os.path.join(env_directory, BIN_DIRECTORY, 'pip'),
'install',
'--requirement',
requirement_path,
]
if _execute_install(cmd) != 0:
# We should remove the directory that we just created
if new_package_dir:
shutil.rmtree(env_directory)
raise _generic_error(package_name)
return None
def setup_module():
print("SETUP")
app = {
'id': "/{}".format(TEST_APP_NAME),
'cpus': 0.1,
'mem': 32,
'instances': 1,
'cmd': '/opt/mesosphere/bin/dcos-shell python '
'/opt/mesosphere/active/dcos-integration-test/util/python_test_server.py $PORT0',
'env': {
'DCOS_TEST_UUID': TEST_APP_NAME,
# required for python_test_server.py to run as nobody
'HOME': '/'
},
'healthChecks': [
{
'protocol': 'MESOS_HTTP',
'path': '/ping',
'portIndex': 0,
'gracePeriodSeconds': 5,
'intervalSeconds': 10,
'timeoutSeconds': 10,
'maxConsecutiveFailures': 3
}
],
'portDefinitions': [{
"protocol": TEST_PORT_PROTOCOL,
"port": TEST_PORT_NUMBER,
"name": TEST_PORT_NAME
}],
'acceptedResourceRoles': ["slave_public"],
'requirePorts': True,
}
with util.temptext() as file_tup:
f, fpath = file_tup
app_bits = json.dumps(app).encode()
os.write(f, app_bits)
os.fsync(f)
add_com = "dcos marathon app add {}".format(fpath)
subprocess.call(shlex.split(add_com))
time.sleep(5)
for _ in range(360):
ret = subprocess.call(shlex.split("dcos marathon deployment list"))
if ret != 0:
break
time.sleep(1)
def test_bad_install():
stderr = (
b'This is a Community service. '
b'Community services are not tested '
b'for production environments. '
b'There may be bugs, incomplete features, '
b'incorrect documentation, or other discrepancies.\n'
b'By Deploying, you agree to the Terms '
b'and Conditions https://mesosphere.com/'
b'catalog-terms-conditions/#community-services\n'
b'A sample pre-installation message\n'
b'Installing Marathon app for package [helloworld] version '
b'[0.1.0]\n'
b'Error: Options JSON failed validation\n'
)
with util.temptext(b'{"nom": "hallo"}') as options:
args = ['--options='+options[1], '--yes']
_install_bad_helloworld(args=args, stderr=stderr)
def _install_with_pip(package_name, env_directory, requirements):
"""
:param package_name: the name of the package
:type package_name: str
:param env_directory: the path to the directory in which to install the
package's virtual env
:type env_directory: str
:param requirements: the list of pip requirements
:type requirements: list of str
:rtype: None
"""
bin_directory = os.path.join(util.dcos_path(), BIN_DIRECTORY)
new_package_dir = not os.path.exists(env_directory)
pip_path = os.path.join(env_directory, BIN_DIRECTORY, "pip")
if not os.path.exists(pip_path):
cmd = [os.path.join(bin_directory, "virtualenv"), env_directory]
if _execute_install(cmd) != 0:
raise _generic_error(package_name)
with util.temptext() as text_file:
fd, requirement_path = text_file
# Write the requirements to the file
with os.fdopen(fd, "w") as requirements_file:
for line in requirements:
print(line, file=requirements_file)
cmd = [os.path.join(env_directory, BIN_DIRECTORY, "pip"), "install", "--requirement", requirement_path]
if _execute_install(cmd) != 0:
# We should remove the directory that we just created
if new_package_dir:
shutil.rmtree(env_directory)
raise _generic_error(package_name)
return None
def _vpn(port, config_file, user, privileged, ssh_port, host, verbose,
openvpn_container, vpn_client, docker_cmd, addroute, delroute,
disablevips, disableoverlay):
"""
VPN into a DC/OS cluster using the IP addresses found in master's
state.json
:param port: The port the HTTP proxy listens on locally
:type port: int | None
:param config_file: SSH config file
:type config_file: str | None
:param user: SSH user
:type user: str | None
:param privileged: If True, override privilege checks
:type privileged: bool
:param ssh_port: The port SSH is accessible through
:type ssh_port: int | None
:param host: The host to connect to
:type host: str | None
:param verbose: Verbose output
:type verbose: bool
:param openvpn_container: `docker pull <param>` should work
:type openvpn_container: str
:param vpn_client: Relative or absolute path to openvpn client
:type vpn_client: str
:param docker_cmd: The docker client command
:type docker_cmd: str | None
:param addroute: Add route to IPv4/IPv6 address with optional subnet
:type option: [str]
:param delroute: Delete route to IPv4/IPv6 address with optional subnet
:type option: [str]
:param disablevips: Disable auto-configuration of routes to VIPs.
:type option: bool
:param disableoverlay: Disable auto-configuration of routes to overlay networks.
:type option: bool
:returns: process return code
:rtype: int
"""
if verbose:
set_verbose()
if privileged:
os.environ[constants.privileged] = '1'
if sys.platform == 'win32':
logger.error("*** VPN is currently unsupported on Windows")
return 1
if not is_privileged():
logger.error("*** You don't have permission to run this command." +
"\n{}".format(constants.unprivileged_suggestion))
return 1
if not ((os.path.isfile(vpn_client) and os.access(vpn_client, os.X_OK)) or
shutil.which(vpn_client)):
msg = ("*** Not a valid executable: {}\n" +
"*** Check that it is a valid absolute path, relative path, or in your $PATH")
logger.error(msg.format(vpn_client))
return 1
port = validate_port(port, default=1194)
if port is None:
return 1
client = sshclient(config_file, user, ssh_port, host)
docker_cmd = resolve_docker_cmd(client, docker_cmd)
route_hosts, dns_hosts = VPNHost(client, addroute, delroute, disablevips, disableoverlay).gen_hosts()
container_name = "openvpn-{}".format(rand_str(8))
remote_openvpn_dir = "/etc/openvpn"
remote_keyfile = "{}/static.key".format(remote_openvpn_dir)
remote_clientfile = "{}/client.ovpn".format(remote_openvpn_dir)
emitter.publish("\nATTENTION: IF DNS DOESN'T WORK, add these DNS servers!")
for host in dns_hosts:
emitter.publish(host)
parsed_routes = ','.join(route_hosts)
parsed_dns = ','.join(dns_hosts)
with util.temptext() as server_tup, \
util.temptext() as key_tup, \
util.temptext() as config_tup, \
util.temptext() as client_tup:
serverfile, serverpath = server_tup
keyfile, keypath = key_tup
clientconfigfile, clientconfigpath = config_tup
clientfile, clientpath = client_tup
scom = """\
{} run --rm --cap-add=NET_ADMIN -p 0:1194 \
-e "OPENVPN_ROUTES={}" -e "OPENVPN_DNS={}" --name {} {}\
""".format(docker_cmd, parsed_routes, parsed_dns,
container_name, openvpn_container)
# FDs created when python opens a file have O_CLOEXEC set, which
# makes them invalid in new threads (cloning). So we duplicate the
# FD, which creates one without O_CLOEXEC.
serverfile_dup = os.dup(serverfile)
# XXX This FD is never closed because it would cause the vpn server
# thread to crash
vpn_server = threading.Thread(target=logging_exec,
args=(client, scom, serverfile_dup,
True),
daemon=True)
vpn_server.start()
msg = "\nWaiting for VPN server in container '{}' to come up..."
emitter.publish(msg.format(container_name))
scom = ("until "
"""[ "$(%s inspect --format='{{ .State.Running }}' """
"""%s 2>/dev/null)" = "true" ] 2>/dev/null; do sleep 0.5; """
"""done""") % (docker_cmd, container_name)
scom += (" && "
"""{} exec {} sh -c 'until [ -s {} ]; do sleep 0.5; """
"""done' """).format(docker_cmd, container_name,
remote_keyfile)
scom += (" && "
"""{} exec {} sh -c 'until [ -s {} ]; do sleep 0.5; """
"""done' """).format(docker_cmd, container_name,
remote_clientfile)
ssh_exec_fatal(client, scom)
scom = ("""%s inspect --format='"""
"""{{range $p, $conf := .NetworkSettings.Ports}}"""
"""{{(index $conf 0).HostPort}}{{end}}' %s"""
) % (docker_cmd, container_name)
remote_port = int(ssh_exec_fatal(client, scom).read().decode().strip())
def tunnel_def():
ssh_transport = client.get_transport()
forward_tunnel(port, '127.0.0.1', remote_port, ssh_transport)
tunnel = threading.Thread(target=tunnel_def, daemon=True)
tunnel.start()
container_cp(client, container_name, remote_keyfile, keyfile,
docker_cmd)
container_cp(client, container_name, remote_clientfile,
clientconfigfile, docker_cmd)
vpn_com = '{} --config {} --secret {} --port {}'
vpn_com = vpn_com.format(vpn_client, clientconfigpath, keypath, port)
logger.info("Running VPN command: {}".format(vpn_com))
emitter.publish("\nVPN server output at {}".format(serverpath))
emitter.publish("VPN client output at {}\n".format(clientpath))
ret = run_vpn(vpn_com, clientfile)
client.close()
input('Exited. Temporary files will be gone once you hit <Return>.')
return ret
def _vpn(port, option, config_file, user, master_proxy, openvpn_container):
"""
VPN into a DC/OS cluster using the IP addresses found in master's
state.json
:param port: The port the HTTP proxy listens on locally
:type port: int | None
:param option: SSH option
:type option: [str]
:param config_file: SSH config file
:type config_file: str | None
:param user: SSH user
:type user: str | None
:param master_proxy: If True, SSH-hop from a master
:type master_proxy: bool | None
:returns: process return code
:rtype: int
"""
if sys.platform == 'win32':
emitter.publish("VPN is currently unsupported on Windows")
return
if port is None:
port = 1194
vpn_client = 'openvpn'
if not distutils.spawn.find_executable(vpn_client):
msg = ("You don't seem to have the '{0}' executable. Please add it to "
"your $PATH or equivalent.")
emitter.publish(msg.format(vpn_client))
return 1
# The reason I ask for sudo here instead of when it's actually needed,
# is that there is something messing with stdin once the docker tunnel
# ssh command is run
if not is_privileged():
emitter.publish("You don't have permission to run this "
"command. Attempting to increase privileges...")
FNULL = open(os.devnull, 'w')
subprocess.call(['sudo', '-l'], stdout=FNULL, stderr=FNULL)
FNULL.close()
mesos_hosts, dns_hosts = gen_hosts(option, config_file, user, master_proxy)
container_name = "openvpn-{0}".format(rand_str(8))
remote_openvpn_dir = "/etc/openvpn"
remote_keyfile = "{0}/static.key".format(remote_openvpn_dir)
remote_clientfile = "{0}/client.ovpn".format(remote_openvpn_dir)
emitter.publish("\nATTENTION: IF DNS DOESN'T WORK, add these DNS servers!")
for host in dns_hosts:
emitter.publish(host)
parsed_routes = ','.join(mesos_hosts)
parsed_dns = ','.join(dns_hosts)
with util.temptext() as server_tup, \
util.temptext() as key_tup, \
util.temptext() as config_tup, \
util.temptext() as client_tup:
serverfile, serverpath = server_tup
keyfile, keypath = key_tup
clientconfigfile, clientconfigpath = config_tup
clientfile, clientpath = client_tup
scom = """\
docker run --rm --cap-add=NET_ADMIN -p 0:1194 \
-e "OPENVPN_ROUTES={0}" -e "OPENVPN_DNS={1}" --name {2} {3}\
""".format(parsed_routes, parsed_dns, container_name,
openvpn_container)
vpn_server = _ssh(True,
None,
option,
config_file,
user,
master_proxy,
scom,
print_command=False,
short_circuit=True,
output_dst=serverfile,
tty=False,
raw=True)
msg = "\nWaiting for VPN server in container '{0}' to come up..."
emitter.publish(msg.format(container_name))
scom = ("until "
"""[ "$(docker inspect --format='{{ .State.Running }}' """
"""%s 2>/dev/null)" = "true" ] 2>/dev/null; do sleep 0.5; """
"""done""") % (container_name)
scom += (";"
"""docker exec {0} "sh -c 'until [ -s {1} ]; do sleep 0.5; """
"""done'" """).format(container_name, remote_keyfile)
scom += (";"
"""docker exec {0} "sh -c 'until [ -s {1} ]; do sleep 0.5; """
"""done'" """).format(container_name, remote_clientfile)
_ssh(True,
None,
option,
config_file,
user,
master_proxy,
scom,
print_command=False,
short_circuit=True,
tty=False)
scom = ('"'
"""docker inspect --format='"""
"""{{range $p, $conf := .NetworkSettings.Ports}}"""
"""{{(index $conf 0).HostPort}}{{end}}' %s"""
'"') % (container_name)
remote_port = _ssh(True,
None,
option,
config_file,
user,
master_proxy,
scom,
print_command=False,
short_circuit=True,
output=True,
tty=False)
fwd_flag = '-N -L {0}:127.0.0.1:{1}'.format(port, remote_port)
tunnel = _ssh(True,
None,
option,
config_file,
user,
master_proxy,
None,
flag=[fwd_flag],
print_command=False,
short_circuit=True,
tty=False,
raw=True)
container_cp(option, config_file, user, master_proxy, container_name,
remote_keyfile, keyfile)
container_cp(option, config_file, user, master_proxy, container_name,
remote_clientfile, clientconfigfile)
vpn_com = ('openvpn ' +
'--config {0} --secret {1} --port {2} --user {3}')
vpn_com = vpn_com.format(clientconfigpath, keypath, port,
getpass.getuser())
if not is_privileged():
vpn_com = 'sudo -n {0}'.format(vpn_com)
emitter.publish("\nVPN server output at {0}".format(serverpath))
emitter.publish("VPN client output at {0}\n".format(clientpath))
ret = subprocess.call(shlex.split(vpn_com),
stdout=clientfile,
stderr=clientfile)
vpn_server.communicate()
tunnel.communicate()
return ret
def _vpn(port, config_file, user, privileged, ssh_port, host,
openvpn_container, vpn_client):
"""
VPN into a DC/OS cluster using the IP addresses found in master's
state.json
:param port: The port the HTTP proxy listens on locally
:type port: int | None
:param config_file: SSH config file
:type config_file: str | None
:param user: SSH user
:type user: str | None
:param privileged: If True, override privilege checks
:type privileged: bool
:param ssh_port: The port SSH is accessible through
:type ssh_port: int | None
:param host: The host to connect to
:type host: str | None
:param openvpn_container: `docker pull <param>` should work
:type openvpn_container: str
:param vpn_client: Relative or absolute path to openvpn client
:type vpn_client: str
:returns: process return code
:rtype: int
"""
if privileged:
os.environ[constants.privileged] = '1'
if sys.platform == 'win32':
logger.error("*** VPN is currently unsupported on Windows")
return 1
if not is_privileged():
logger.error("*** You don't have permission to run this command." +
"\n{}".format(constants.unprivileged_suggestion))
return 1
if not ((os.path.isfile(vpn_client) and os.access(vpn_client, os.X_OK))
or shutil.which(vpn_client)):
msg = "*** Not a valid executable: {}"
logger.error(msg.format(vpn_client))
return 1
port = validate_port(port, default=1194)
if port is None:
return 1
client = sshclient(config_file, user, ssh_port, host)
if not distutils.spawn.find_executable(vpn_client):
msg = ("You don't seem to have the '{}' executable. Please add it to "
"your $PATH or equivalent.")
logger.error(msg.format(vpn_client))
return 1
mesos_hosts, dns_hosts = gen_hosts(client)
container_name = "openvpn-{}".format(rand_str(8))
remote_openvpn_dir = "/etc/openvpn"
remote_keyfile = "{}/static.key".format(remote_openvpn_dir)
remote_clientfile = "{}/client.ovpn".format(remote_openvpn_dir)
emitter.publish("\nATTENTION: IF DNS DOESN'T WORK, add these DNS servers!")
for host in dns_hosts:
emitter.publish(host)
parsed_routes = ','.join(mesos_hosts)
parsed_dns = ','.join(dns_hosts)
with util.temptext() as server_tup, \
util.temptext() as key_tup, \
util.temptext() as config_tup, \
util.temptext() as client_tup:
serverfile, serverpath = server_tup
keyfile, keypath = key_tup
clientconfigfile, clientconfigpath = config_tup
clientfile, clientpath = client_tup
scom = """\
docker run --rm --cap-add=NET_ADMIN -p 0:1194 \
-e "OPENVPN_ROUTES={}" -e "OPENVPN_DNS={}" --name {} {}\
""".format(parsed_routes, parsed_dns, container_name,
openvpn_container)
# FDs created when python opens a file have O_CLOEXEC set, which
# makes them invalid in new threads (cloning). So we duplicate the
# FD, which creates one without O_CLOEXEC.
serverfile_dup = os.dup(serverfile)
# XXX This FD is never closed because it would cause the vpn server
# thread to crash
vpn_server = threading.Thread(target=logging_exec,
args=(client, scom, serverfile_dup,
True),
daemon=True)
vpn_server.start()
msg = "\nWaiting for VPN server in container '{}' to come up..."
emitter.publish(msg.format(container_name))
scom = ("until "
"""[ "$(docker inspect --format='{{ .State.Running }}' """
"""%s 2>/dev/null)" = "true" ] 2>/dev/null; do sleep 0.5; """
"""done""") % (container_name)
scom += (";"
"""docker exec {} "sh -c 'until [ -s {} ]; do sleep 0.5; """
"""done'" """).format(container_name, remote_keyfile)
scom += (";"
"""docker exec {} "sh -c 'until [ -s {} ]; do sleep 0.5; """
"""done'" """).format(container_name, remote_clientfile)
server_setup = client.exec_command(scom, get_pty=True)
ssh_exec_wait(server_setup)
scom = ("""docker inspect --format='"""
"""{{range $p, $conf := .NetworkSettings.Ports}}"""
"""{{(index $conf 0).HostPort}}{{end}}' %s""") % (
container_name)
_, query_stdout, _ = client.exec_command(scom, get_pty=True)
remote_port = int(query_stdout.read().decode().strip())
def tunnel_def():
ssh_transport = client.get_transport()
forward_tunnel(port, '127.0.0.1', remote_port, ssh_transport)
tunnel = threading.Thread(target=tunnel_def, daemon=True)
tunnel.start()
container_cp(client, container_name, remote_keyfile, keyfile)
container_cp(client, container_name, remote_clientfile,
clientconfigfile)
vpn_com = ('{} --config {} --secret {} --port {}')
vpn_com = vpn_com.format(vpn_client, clientconfigpath, keypath, port)
emitter.publish("\nVPN server output at {}".format(serverpath))
emitter.publish("VPN client output at {}\n".format(clientpath))
ret = run_vpn(vpn_com, clientfile)
client.close()
input('Exited. Temporary files will be gone once you hit <Return>.')
return ret
def _vpn(port, option, config_file, user, master_proxy, openvpn_container):
"""
VPN into a DC/OS cluster using the IP addresses found in master's
state.json
:param port: The port the HTTP proxy listens on locally
:type port: int | None
:param option: SSH option
:type option: [str]
:param config_file: SSH config file
:type config_file: str | None
:param user: SSH user
:type user: str | None
:param master_proxy: If True, SSH-hop from a master
:type master_proxy: bool | None
:returns: process return code
:rtype: int
"""
if sys.platform == 'win32':
emitter.publish("VPN is currently unsupported on Windows")
return
if port is None:
port = 1194
vpn_client = 'openvpn'
if not distutils.spawn.find_executable(vpn_client):
msg = ("You don't seem to have the '{0}' executable. Please add it to "
"your $PATH or equivalent.")
emitter.publish(msg.format(vpn_client))
return 1
# The reason I ask for sudo here instead of when it's actually needed,
# is that there is something messing with stdin once the docker tunnel
# ssh command is run
if not is_privileged():
emitter.publish("You don't have permission to run this "
"command. Attempting to increase privileges...")
FNULL = open(os.devnull, 'w')
subprocess.call(['sudo', '-l'], stdout=FNULL, stderr=FNULL)
FNULL.close()
mesos_hosts, dns_hosts = gen_hosts(option, config_file, user, master_proxy)
container_name = "openvpn-{0}".format(rand_str(8))
remote_openvpn_dir = "/etc/openvpn"
remote_keyfile = "{0}/static.key".format(remote_openvpn_dir)
remote_clientfile = "{0}/client.ovpn".format(remote_openvpn_dir)
emitter.publish("\nATTENTION: IF DNS DOESN'T WORK, add these DNS servers!")
for host in dns_hosts:
emitter.publish(host)
parsed_routes = ','.join(mesos_hosts)
parsed_dns = ','.join(dns_hosts)
with util.temptext() as server_tup, \
util.temptext() as key_tup, \
util.temptext() as config_tup, \
util.temptext() as client_tup:
serverfile, serverpath = server_tup
keyfile, keypath = key_tup
clientconfigfile, clientconfigpath = config_tup
clientfile, clientpath = client_tup
scom = """\
docker run --rm --cap-add=NET_ADMIN -p 0:1194 \
-e "OPENVPN_ROUTES={0}" -e "OPENVPN_DNS={1}" --name {2} {3}\
""".format(parsed_routes, parsed_dns, container_name,
openvpn_container)
vpn_server = _ssh(True, None, option, config_file, user, master_proxy,
scom, print_command=False, short_circuit=True,
output_dst=serverfile, tty=False, raw=True)
msg = "\nWaiting for VPN server in container '{0}' to come up..."
emitter.publish(msg.format(container_name))
scom = ("until "
"""[ "$(docker inspect --format='{{ .State.Running }}' """
"""%s 2>/dev/null)" = "true" ] 2>/dev/null; do sleep 0.5; """
"""done""") % (container_name)
scom += (";"
"""docker exec {0} "sh -c 'until [ -s {1} ]; do sleep 0.5; """
"""done'" """).format(container_name, remote_keyfile)
scom += (";"
"""docker exec {0} "sh -c 'until [ -s {1} ]; do sleep 0.5; """
"""done'" """).format(container_name, remote_clientfile)
_ssh(True, None, option, config_file, user, master_proxy, scom,
print_command=False, short_circuit=True, tty=False)
scom = ('"'
"""docker inspect --format='"""
"""{{range $p, $conf := .NetworkSettings.Ports}}"""
"""{{(index $conf 0).HostPort}}{{end}}' %s"""
'"') % (container_name)
remote_port = _ssh(True, None, option, config_file, user, master_proxy,
scom, print_command=False, short_circuit=True,
output=True, tty=False)
fwd_flag = '-N -L {0}:127.0.0.1:{1}'.format(port, remote_port)
tunnel = _ssh(True, None, option, config_file, user, master_proxy,
None, flag=[fwd_flag], print_command=False,
short_circuit=True, tty=False, raw=True)
container_cp(option, config_file, user, master_proxy,
container_name, remote_keyfile, keyfile)
container_cp(option, config_file, user, master_proxy,
container_name, remote_clientfile, clientconfigfile)
vpn_com = ('openvpn ' +
'--config {0} --secret {1} --port {2} --user {3}')
vpn_com = vpn_com.format(clientconfigpath, keypath, port,
getpass.getuser())
if not is_privileged():
vpn_com = 'sudo -n {0}'.format(vpn_com)
emitter.publish("\nVPN server output at {0}".format(serverpath))
emitter.publish("VPN client output at {0}\n".format(clientpath))
ret = subprocess.call(shlex.split(vpn_com), stdout=clientfile,
stderr=clientfile)
vpn_server.communicate()
tunnel.communicate()
return ret
def _install_with_binary(
package_name,
env_directory,
binary_cli):
"""
:param package_name: the name of the package
:type package_name: str
:param env_directory: the path to the directory in which to install the
package's binary_cli
:type env_directory: str
:param binary_cli: binary cli to install
:type binary_cli: str
:rtype: None
"""
binary_url, kind = binary_cli.get("url"), binary_cli.get("kind")
binary_url = _rewrite_binary_url(
binary_url,
config.get_config_val("core.dcos_url"))
try:
env_bin_dir = os.path.join(env_directory, 'bin')
if not os.path.exists(env_bin_dir) and util.is_windows_platform():
env_bin_dir = os.path.join(env_directory, "Scripts")
if kind in ["executable", "zip"]:
with util.temptext() as file_tmp:
_, binary_tmp = file_tmp
_download_and_store(binary_url, binary_tmp)
_check_hash(binary_tmp, binary_cli.get("contentHash"))
if kind == "executable":
util.ensure_dir_exists(env_bin_dir)
binary_name = "dcos-{}".format(package_name)
if util.is_windows_platform():
binary_name += '.exe'
binary_file = os.path.join(env_bin_dir, binary_name)
# copy to avoid windows error of moving open file
# binary_tmp will be removed by context manager
shutil.copy(binary_tmp, binary_file)
else:
# kind == "zip"
with zipfile.ZipFile(binary_tmp) as zf:
zf.extractall(env_directory)
# Check contents for package_name/env/bin folder structure.
# Don't fail if the env or Scripts folder doesn't exist,
# it is possible that a plugin.toml file specifies a custom
# path. The package format should be validated in a more thorough
# way during the refactoring of this command in Go.
if os.path.exists(env_bin_dir):
# make binar(ies) executable
for f in os.listdir(env_bin_dir):
binary = os.path.join(env_bin_dir, f)
if (f.startswith(constants.DCOS_COMMAND_PREFIX)):
st = os.stat(binary)
os.chmod(binary, st.st_mode | stat.S_IEXEC)
else:
msg = ("CLI subcommand for [{}] is an unsupported type: {}"
"Please contact the package maintainer".format(
package_name, kind))
raise DCOSException(msg)
except DCOSException:
raise
except Exception as e:
logger.exception(e)
raise _generic_error(package_name, e.message)
return None
def test_read_file_secure_with_trailing_whitespaces():
with util.temptext(b"my_secure_password \r\n") as temp_file:
path = temp_file[1]
os.chmod(path, 0o600)
password = util.read_file_secure(path)
assert password == "my_secure_password"
