def calculate_ssh_user(os_name, platform, platform_info_filename):
if platform_info_filename != '':
return load_yaml(platform_info_filename)['ssh_user']
if platform == 'aws':
return test_util.aws.OS_SSH_INFO[os_name].user
else:
raise Exception('Cannot yet calculate user for {} platform'.format(platform))
def load_config(config_path: str) -> dict:
try:
config = load_yaml(config_path)
return config
except YamlParseError as ex:
raise launch.util.LauncherError('InvalidInput', None) from ex
except FileNotFoundError as ex:
raise launch.util.LauncherError('MissingConfig', None) from ex
def load_config(config_path: str) -> dict:
try:
config = load_yaml(config_path)
return config
except YamlParseError as ex:
raise launch.util.LauncherError('InvalidInput', None) from ex
except FileNotFoundError as ex:
raise launch.util.LauncherError('MissingConfig', None) from ex
def genconf(
self, master_list, agent_list, public_agent_list, ssh_user, ssh_key,
ip_detect, platform=None, rexray_config=None, rexray_config_preset=None,
zk_host=None, expect_errors=False, add_config_path=None):
"""Runs configuration generation.
Args:
master_list: list of IPv4 addresses to be used as masters
agent_list: list of IPv4 addresses to be used as agents
public_agent_list: list of IPv4 addresses to be used as public agents
ssh_user (str): name of SSH user that has access to targets
ssh_key (str): complete public SSH key for ssh_user. Must already
be installed on tagets as authorized_key
ip_detect (str): name of preset IP-detect script
platform (str): name of the infrastructure platform
rexray_config: complete contents of REX-Ray config file. Must be a
JSON-serializable object.
rexray_config_preset (str): name of preset REX-Ray config
zk_host (optional): if provided, zk is used for exhibitor backend
expect_errors (optional): raises error if result is unexpected
add_config_path (optional): string pointing to a file with additional
config parameters to be merged or used as overide
Raises:
AssertionError: "error" present in returned json keys when error
was not expected or vice versa
"""
headers = {'content-type': 'application/json'}
payload = {
'master_list': master_list,
'agent_list': agent_list,
'public_agent_list': public_agent_list,
'ssh_user': ssh_user,
'ssh_key': ssh_key,
'ip_detect_script': self.ip_detect_script(ip_detect)}
if platform:
payload['platform'] = platform
if rexray_config:
payload['rexray_config'] = rexray_config
if rexray_config_preset:
payload['rexray_config_preset'] = rexray_config_preset
if zk_host:
payload['exhibitor_zk_hosts'] = zk_host
if add_config_path:
add_config = load_yaml(add_config_path)
payload.update(add_config)
response = requests.post(self.url + '/api/v1/configure', headers=headers, data=json.dumps(payload))
assert response.status_code == 200, "{} {}".format(response.status_code, response.content)
response_json_keys = list(response.json().keys())
if expect_errors:
assert "error" in response_json_keys
else:
assert "error" not in response_json_keys
def do_main(args):
if args['create']:
info_path = args['--info-path']
if os.path.exists(info_path):
raise LauncherError('InputConflict',
'Target info path already exists!')
config = load_yaml(args['--config-path'])
check_keys(config, [
'type', 'provider_info',
'this_is_a_temporary_config_format_do_not_put_in_production'
])
write_json(
info_path,
get_launcher(config['type'],
config['provider_info']).create(config))
return 0
info = load_json(args['--info-path'])
check_keys(info, ['type', 'provider'])
launcher = get_launcher(info['type'], info['provider'])
if args['wait']:
launcher.wait(info)
print('Cluster is ready!')
return 0
if args['describe']:
print(json_prettyprint(launcher.describe(info)))
return 0
if args['pytest']:
test_cmd = 'py.test'
if args['--env'] is not None:
if '=' in args['--env']:
# User is attempting to do an assigment with the option
raise LauncherError(
'OptionError',
"The '--env' option can only pass through environment variables "
"from the current environment. Set variables according to the shell being used."
)
var_list = args['--env'].split(',')
check_keys(os.environ, var_list)
test_cmd = ' '.join(
['{}={}'.format(e, os.environ[e])
for e in var_list]) + ' ' + test_cmd
if len(args['<pytest_extras>']) > 0:
test_cmd += ' ' + ' '.join(args['<pytest_extras>'])
launcher.test(info, test_cmd)
return 0
if args['delete']:
launcher.delete(info)
return 0
def _load_config(self):
if self.config_path is None:
return {}
try:
return load_yaml(self.config_path)
except FileNotFoundError as ex:
raise NoConfigError(
"No config file found at {}. See the DC/OS documentation for the "
"available configuration options. You can also use the GUI web installer (--web),"
"which provides a guided configuration and installation for simple "
"deployments.".format(self.config_path)) from ex
except YamlParseError as ex:
raise NoConfigError(
"Unable to load configuration file. {}".format(ex)) from ex
def _load_config(self):
if self.config_path is None:
return {}
try:
return load_yaml(self.config_path)
except FileNotFoundError as ex:
raise NoConfigError(
"No config file found at {}. See the DC/OS documentation for the "
"available configuration options.".format(self.config_path)) from ex
except OSError as ex:
raise NoConfigError(
"Failed to open config file at {}: {}. See the DC/OS documentation to learn "
"how to create a config file.".format(self.config_path, ex)) from ex
except YamlParseError as ex:
raise NoConfigError("Unable to load configuration file. {}".format(ex)) from ex
def _load_config(self):
if self.config_path is None:
return {}
try:
return load_yaml(self.config_path)
except FileNotFoundError as ex:
raise NoConfigError(
"No config file found at {}. See the DC/OS documentation for the "
"available configuration options.".format(self.config_path)) from ex
except OSError as ex:
raise NoConfigError(
"Failed to open config file at {}: {}. See the DC/OS documentation to learn "
"how to create a config file.".format(self.config_path, ex)) from ex
except YamlParseError as ex:
raise NoConfigError("Unable to load configuration file. {}".format(ex)) from ex
def do_main(args):
_handle_logging(args['--log-level'].upper())
if args['create']:
info_path = args['--info-path']
if os.path.exists(info_path):
raise LauncherError('InputConflict', 'Target info path already exists!')
config = load_yaml(args['--config-path'])
check_keys(config, ['type', 'provider_info', 'this_is_a_temporary_config_format_do_not_put_in_production'])
write_json(info_path, get_launcher(config['type'], config['provider_info']).create(config))
return 0
info = load_json(args['--info-path'])
check_keys(info, ['type', 'provider'])
launcher = get_launcher(info['type'], info['provider'])
if args['wait']:
launcher.wait(info)
print('Cluster is ready!')
return 0
if args['describe']:
print(json_prettyprint(launcher.describe(info)))
return 0
if args['pytest']:
test_cmd = 'py.test'
if args['--env'] is not None:
if '=' in args['--env']:
# User is attempting to do an assigment with the option
raise LauncherError('OptionError', "The '--env' option can only pass through environment variables "
"from the current environment. Set variables according to the shell being used.")
var_list = args['--env'].split(',')
check_keys(os.environ, var_list)
test_cmd = ' '.join(['{}={}'.format(e, os.environ[e]) for e in var_list]) + ' ' + test_cmd
if len(args['<pytest_extras>']) > 0:
test_cmd += ' ' + ' '.join(args['<pytest_extras>'])
launcher.test(info, test_cmd)
return 0
if args['delete']:
launcher.delete(info)
return 0
def _load_config(self):
if self.config_path is None:
return {}
try:
return load_yaml(self.config_path)
except FileNotFoundError as ex:
raise NoConfigError(
"No config file found at {}. See the DC/OS documentation for the "
"available configuration options. You can also use the GUI web installer (--web), "
"which provides a guided configuration and installation for simple "
"deployments.".format(self.config_path)) from ex
except OSError as ex:
raise NoConfigError(
"Failed to open config file at {}: {}. See the DC/OS documentation to learn "
"how to create a config file. You can also use the GUI web installer (--web), "
"which provides a guided configuration and installation for simple "
"deployments.".format(self.config_path, ex)) from ex
except YamlParseError as ex:
raise NoConfigError("Unable to load configuration file. {}".format(ex)) from ex
def _do_bootstrap(install, repository):
# These files should be set by the environment which initially builds
# the host (cloud-init).
repository_url = if_exists(load_string, install.get_config_filename("setup-flags/repository-url"))
def fetcher(id, target):
if repository_url is None:
raise ValidationError("ERROR: Non-local package {} but no repository url given.".format(id))
return requests_fetcher(repository_url, id, target, os.getcwd())
setup_pkg_dir = install.get_config_filename("setup-packages")
if os.path.exists(setup_pkg_dir):
raise ValidationError(
"setup-packages is no longer supported. It's functionality has been replaced with late "
"binding packages. Found setup packages dir: {}".format(setup_pkg_dir))
setup_packages_to_activate = []
# If the host has late config values, build the late config package from them.
late_config = if_exists(load_yaml, install.get_config_filename("setup-flags/late-config.yaml"))
if late_config:
pkg_id_str = late_config['late_bound_package_id']
late_values = late_config['bound_values']
print("Binding late config to late package {}".format(pkg_id_str))
print("Bound values: {}".format(late_values))
if not PackageId.is_id(pkg_id_str):
raise ValidationError("Invalid late package id: {}".format(pkg_id_str))
pkg_id = PackageId(pkg_id_str)
if pkg_id.version != "setup":
raise ValidationError("Late package must have the version setup. Bad package: {}".format(pkg_id_str))
# Collect the late config package.
with tempfile.NamedTemporaryFile() as f:
download(
f.name,
repository_url + '/packages/{0}/{1}.dcos_config'.format(pkg_id.name, pkg_id_str),
os.getcwd(),
rm_on_error=False,
)
late_package = load_yaml(f.name)
# Resolve the late package using the bound late config values.
final_late_package = resolve_late_package(late_package, late_values)
# Render the package onto the filesystem and add it to the package
# repository.
with tempfile.NamedTemporaryFile() as f:
do_gen_package(final_late_package, f.name)
repository.add(lambda _, target: extract_tarball(f.name, target), pkg_id_str)
setup_packages_to_activate.append(pkg_id_str)
# If active.json is set on the host, use that as the set of packages to
# activate. Otherwise just use the set of currently active packages (those
# active in the bootstrap tarball)
to_activate = None
active_path = install.get_config_filename("setup-flags/active.json")
if os.path.exists(active_path):
print("Loaded active packages from", active_path)
to_activate = load_json(active_path)
# Ensure all packages are local
print("Ensuring all packages in active set {} are local".format(",".join(to_activate)))
for package in to_activate:
repository.add(fetcher, package)
else:
print("Calculated active packages from bootstrap tarball")
to_activate = list(install.get_active())
package_list_filename = install.get_config_filename("setup-flags/cluster-package-list")
print("Checking for cluster packages in:", package_list_filename)
package_list_id = if_exists(load_string, package_list_filename)
if package_list_id:
print("Cluster package list:", package_list_id)
cluster_packages = _get_package_list(package_list_id, repository_url)
print("Loading cluster-packages: {}".format(cluster_packages))
for package_id_str in cluster_packages:
# Validate the package ids
pkg_id = PackageId(package_id_str)
# Fetch the packages if not local
if not repository.has_package(package_id_str):
repository.add(fetcher, package_id_str)
# Add the package to the set to activate
setup_packages_to_activate.append(package_id_str)
else:
print("No cluster-packages specified")
# Calculate the full set of final packages (Explicit activations + setup packages).
# De-duplicate using a set.
to_activate = list(set(to_activate + setup_packages_to_activate))
print("Activating packages")
install.activate(repository.load_packages(to_activate))
def genconf(self,
master_list,
agent_list,
public_agent_list,
ssh_user,
ssh_key,
ip_detect,
rexray_config=None,
rexray_config_preset=None,
zk_host=None,
expect_errors=False,
add_config_path=None,
bootstrap_url='file:///opt/dcos_install_tmp'):
"""Runs configuration generation.
Args:
master_list: list of IPv4 addresses to be used as masters
agent_list: list of IPv4 addresses to be used as agents
public_agent_list: list of IPv$ addresses to be used as public agents
ip_detect (str): name of preset IP-detect script
ssh_user (str): name of SSH user that has access to targets
ssh_key (str): complete public SSH key for ssh_user. Must already
be installed on tagets as authorized_key
rexray_config: complete contents of REX-Ray config file. Must be a
JSON-serializable object.
rexray_config_preset (str): name of preset REX-Ray config
zk_host (optional): if provided, zk is used for exhibitor backend
expect_errors (optional): raises error if result is unexpected
add_config_path (optional): string pointing to a file with additional
config parameters to be merged or used as overide
Raises:
AssertionError: "error" present in returned json keys when error
was not expected or vice versa
"""
test_config = {
'cluster_name': 'SSH Installed DC/OS',
'bootstrap_url': bootstrap_url,
'dns_search': 'mesos',
'master_discovery': 'static',
'master_list': master_list,
'ssh_user': ssh_user,
'agent_list': agent_list,
'public_agent_list': public_agent_list,
'process_timeout': MAX_STAGE_TIME
}
if rexray_config:
test_config['rexray_config'] = rexray_config
if rexray_config_preset:
test_config['rexray_config_preset'] = rexray_config_preset
if zk_host:
test_config['exhibitor_storage_backend'] = 'zookeeper'
test_config['exhibitor_zk_hosts'] = zk_host
test_config['exhibitor_zk_path'] = '/exhibitor'
else:
test_config['exhibitor_storage_backend'] = 'static'
if add_config_path:
add_config = load_yaml(add_config_path)
test_config.update(add_config)
with open('config.yaml', 'w') as config_fh:
config_fh.write(yaml.dump(test_config))
with open('ip-detect', 'w') as ip_detect_fh:
ip_detect_fh.write(self.ip_detect_script(ip_detect))
with open('ssh_key', 'w') as key_fh:
key_fh.write(ssh_key)
remote_dir = os.path.dirname(self.installer_path)
self.tunnel.remote_cmd(
['mkdir', '-p', os.path.join(remote_dir, 'genconf')])
self.tunnel.write_to_remote(
'config.yaml', os.path.join(remote_dir, 'genconf/config.yaml'))
self.tunnel.write_to_remote(
'ip-detect', os.path.join(remote_dir, 'genconf/ip-detect'))
self.tunnel.write_to_remote(
'ssh_key', os.path.join(remote_dir, 'genconf/ssh_key'))
self.tunnel.remote_cmd(
['chmod', '600',
os.path.join(remote_dir, 'genconf/ssh_key')])
self.run_cli_cmd('--genconf', expect_errors=expect_errors)
def test_load_user_config():
user_config = load_yaml("/opt/mesosphere/etc/user.config.yaml")
# Calculated parameters shouldn't be in the user config
assert 'master_quorum' not in user_config
def test_load_user_config():
user_config = load_yaml("/opt/mesosphere/etc/user.config.yaml")
# Calculated parameters shouldn't be in the user config
assert 'master_quorum' not in user_config
def genconf(
self, master_list, agent_list, public_agent_list, ssh_user, ssh_key,
ip_detect, platform=None, rexray_config=None, rexray_config_preset=None,
zk_host=None, expect_errors=False, add_config_path=None,
bootstrap_url='file:///opt/dcos_install_tmp'):
"""Runs configuration generation.
Args:
master_list: list of IPv4 addresses to be used as masters
agent_list: list of IPv4 addresses to be used as agents
public_agent_list: list of IPv$ addresses to be used as public agents
ssh_user (str): name of SSH user that has access to targets
ssh_key (str): complete public SSH key for ssh_user. Must already
be installed on tagets as authorized_key
ip_detect (str): name of preset IP-detect script
platform (str): name of the infrastructure platform
rexray_config: complete contents of REX-Ray config file. Must be a
JSON-serializable object.
rexray_config_preset (str): name of preset REX-Ray config
zk_host (optional): if provided, zk is used for exhibitor backend
expect_errors (optional): raises error if result is unexpected
add_config_path (optional): string pointing to a file with additional
config parameters to be merged or used as overide
Raises:
AssertionError: "error" present in returned json keys when error
was not expected or vice versa
"""
test_config = {
'cluster_name': 'SSH Installed DC/OS',
'bootstrap_url': bootstrap_url,
'dns_search': 'mesos',
'master_discovery': 'static',
'master_list': master_list,
'ssh_user': ssh_user,
'agent_list': agent_list,
'public_agent_list': public_agent_list,
'process_timeout': MAX_STAGE_TIME}
if platform:
test_config['platform'] = platform
if rexray_config:
test_config['rexray_config'] = rexray_config
if rexray_config_preset:
test_config['rexray_config_preset'] = rexray_config_preset
if zk_host:
test_config['exhibitor_storage_backend'] = 'zookeeper'
test_config['exhibitor_zk_hosts'] = zk_host
test_config['exhibitor_zk_path'] = '/exhibitor'
else:
test_config['exhibitor_storage_backend'] = 'static'
if add_config_path:
add_config = load_yaml(add_config_path)
test_config.update(add_config)
with open('config.yaml', 'w') as config_fh:
config_fh.write(yaml.dump(test_config))
with open('ip-detect', 'w') as ip_detect_fh:
ip_detect_fh.write(self.ip_detect_script(ip_detect))
with open('ssh_key', 'w') as key_fh:
key_fh.write(ssh_key)
remote_dir = os.path.dirname(self.installer_path)
self.tunnel.remote_cmd(['mkdir', '-p', os.path.join(remote_dir, 'genconf')])
self.tunnel.write_to_remote('config.yaml', os.path.join(remote_dir, 'genconf/config.yaml'))
self.tunnel.write_to_remote('ip-detect', os.path.join(remote_dir, 'genconf/ip-detect'))
self.tunnel.write_to_remote('ssh_key', os.path.join(remote_dir, 'genconf/ssh_key'))
self.tunnel.remote_cmd(['chmod', '600', os.path.join(remote_dir, 'genconf/ssh_key')])
self.run_cli_cmd('--genconf', expect_errors=expect_errors)
