def calculate_ssh_user(os_name, platform, platform_info_filename): if platform_info_filename != '': return load_yaml(platform_info_filename)['ssh_user'] if platform == 'aws': return test_util.aws.OS_SSH_INFO[os_name].user else: raise Exception('Cannot yet calculate user for {} platform'.format(platform))
def load_config(config_path: str) -> dict: try: config = load_yaml(config_path) return config except YamlParseError as ex: raise launch.util.LauncherError('InvalidInput', None) from ex except FileNotFoundError as ex: raise launch.util.LauncherError('MissingConfig', None) from ex
def genconf( self, master_list, agent_list, public_agent_list, ssh_user, ssh_key, ip_detect, platform=None, rexray_config=None, rexray_config_preset=None, zk_host=None, expect_errors=False, add_config_path=None): """Runs configuration generation. Args: master_list: list of IPv4 addresses to be used as masters agent_list: list of IPv4 addresses to be used as agents public_agent_list: list of IPv4 addresses to be used as public agents ssh_user (str): name of SSH user that has access to targets ssh_key (str): complete public SSH key for ssh_user. Must already be installed on tagets as authorized_key ip_detect (str): name of preset IP-detect script platform (str): name of the infrastructure platform rexray_config: complete contents of REX-Ray config file. Must be a JSON-serializable object. rexray_config_preset (str): name of preset REX-Ray config zk_host (optional): if provided, zk is used for exhibitor backend expect_errors (optional): raises error if result is unexpected add_config_path (optional): string pointing to a file with additional config parameters to be merged or used as overide Raises: AssertionError: "error" present in returned json keys when error was not expected or vice versa """ headers = {'content-type': 'application/json'} payload = { 'master_list': master_list, 'agent_list': agent_list, 'public_agent_list': public_agent_list, 'ssh_user': ssh_user, 'ssh_key': ssh_key, 'ip_detect_script': self.ip_detect_script(ip_detect)} if platform: payload['platform'] = platform if rexray_config: payload['rexray_config'] = rexray_config if rexray_config_preset: payload['rexray_config_preset'] = rexray_config_preset if zk_host: payload['exhibitor_zk_hosts'] = zk_host if add_config_path: add_config = load_yaml(add_config_path) payload.update(add_config) response = requests.post(self.url + '/api/v1/configure', headers=headers, data=json.dumps(payload)) assert response.status_code == 200, "{} {}".format(response.status_code, response.content) response_json_keys = list(response.json().keys()) if expect_errors: assert "error" in response_json_keys else: assert "error" not in response_json_keys
def do_main(args): if args['create']: info_path = args['--info-path'] if os.path.exists(info_path): raise LauncherError('InputConflict', 'Target info path already exists!') config = load_yaml(args['--config-path']) check_keys(config, [ 'type', 'provider_info', 'this_is_a_temporary_config_format_do_not_put_in_production' ]) write_json( info_path, get_launcher(config['type'], config['provider_info']).create(config)) return 0 info = load_json(args['--info-path']) check_keys(info, ['type', 'provider']) launcher = get_launcher(info['type'], info['provider']) if args['wait']: launcher.wait(info) print('Cluster is ready!') return 0 if args['describe']: print(json_prettyprint(launcher.describe(info))) return 0 if args['pytest']: test_cmd = 'py.test' if args['--env'] is not None: if '=' in args['--env']: # User is attempting to do an assigment with the option raise LauncherError( 'OptionError', "The '--env' option can only pass through environment variables " "from the current environment. Set variables according to the shell being used." ) var_list = args['--env'].split(',') check_keys(os.environ, var_list) test_cmd = ' '.join( ['{}={}'.format(e, os.environ[e]) for e in var_list]) + ' ' + test_cmd if len(args['<pytest_extras>']) > 0: test_cmd += ' ' + ' '.join(args['<pytest_extras>']) launcher.test(info, test_cmd) return 0 if args['delete']: launcher.delete(info) return 0
def _load_config(self): if self.config_path is None: return {} try: return load_yaml(self.config_path) except FileNotFoundError as ex: raise NoConfigError( "No config file found at {}. See the DC/OS documentation for the " "available configuration options. You can also use the GUI web installer (--web)," "which provides a guided configuration and installation for simple " "deployments.".format(self.config_path)) from ex except YamlParseError as ex: raise NoConfigError( "Unable to load configuration file. {}".format(ex)) from ex
def _load_config(self): if self.config_path is None: return {} try: return load_yaml(self.config_path) except FileNotFoundError as ex: raise NoConfigError( "No config file found at {}. See the DC/OS documentation for the " "available configuration options.".format(self.config_path)) from ex except OSError as ex: raise NoConfigError( "Failed to open config file at {}: {}. See the DC/OS documentation to learn " "how to create a config file.".format(self.config_path, ex)) from ex except YamlParseError as ex: raise NoConfigError("Unable to load configuration file. {}".format(ex)) from ex
def do_main(args): _handle_logging(args['--log-level'].upper()) if args['create']: info_path = args['--info-path'] if os.path.exists(info_path): raise LauncherError('InputConflict', 'Target info path already exists!') config = load_yaml(args['--config-path']) check_keys(config, ['type', 'provider_info', 'this_is_a_temporary_config_format_do_not_put_in_production']) write_json(info_path, get_launcher(config['type'], config['provider_info']).create(config)) return 0 info = load_json(args['--info-path']) check_keys(info, ['type', 'provider']) launcher = get_launcher(info['type'], info['provider']) if args['wait']: launcher.wait(info) print('Cluster is ready!') return 0 if args['describe']: print(json_prettyprint(launcher.describe(info))) return 0 if args['pytest']: test_cmd = 'py.test' if args['--env'] is not None: if '=' in args['--env']: # User is attempting to do an assigment with the option raise LauncherError('OptionError', "The '--env' option can only pass through environment variables " "from the current environment. Set variables according to the shell being used.") var_list = args['--env'].split(',') check_keys(os.environ, var_list) test_cmd = ' '.join(['{}={}'.format(e, os.environ[e]) for e in var_list]) + ' ' + test_cmd if len(args['<pytest_extras>']) > 0: test_cmd += ' ' + ' '.join(args['<pytest_extras>']) launcher.test(info, test_cmd) return 0 if args['delete']: launcher.delete(info) return 0
def _load_config(self): if self.config_path is None: return {} try: return load_yaml(self.config_path) except FileNotFoundError as ex: raise NoConfigError( "No config file found at {}. See the DC/OS documentation for the " "available configuration options. You can also use the GUI web installer (--web), " "which provides a guided configuration and installation for simple " "deployments.".format(self.config_path)) from ex except OSError as ex: raise NoConfigError( "Failed to open config file at {}: {}. See the DC/OS documentation to learn " "how to create a config file. You can also use the GUI web installer (--web), " "which provides a guided configuration and installation for simple " "deployments.".format(self.config_path, ex)) from ex except YamlParseError as ex: raise NoConfigError("Unable to load configuration file. {}".format(ex)) from ex
def _do_bootstrap(install, repository): # These files should be set by the environment which initially builds # the host (cloud-init). repository_url = if_exists(load_string, install.get_config_filename("setup-flags/repository-url")) def fetcher(id, target): if repository_url is None: raise ValidationError("ERROR: Non-local package {} but no repository url given.".format(id)) return requests_fetcher(repository_url, id, target, os.getcwd()) setup_pkg_dir = install.get_config_filename("setup-packages") if os.path.exists(setup_pkg_dir): raise ValidationError( "setup-packages is no longer supported. It's functionality has been replaced with late " "binding packages. Found setup packages dir: {}".format(setup_pkg_dir)) setup_packages_to_activate = [] # If the host has late config values, build the late config package from them. late_config = if_exists(load_yaml, install.get_config_filename("setup-flags/late-config.yaml")) if late_config: pkg_id_str = late_config['late_bound_package_id'] late_values = late_config['bound_values'] print("Binding late config to late package {}".format(pkg_id_str)) print("Bound values: {}".format(late_values)) if not PackageId.is_id(pkg_id_str): raise ValidationError("Invalid late package id: {}".format(pkg_id_str)) pkg_id = PackageId(pkg_id_str) if pkg_id.version != "setup": raise ValidationError("Late package must have the version setup. Bad package: {}".format(pkg_id_str)) # Collect the late config package. with tempfile.NamedTemporaryFile() as f: download( f.name, repository_url + '/packages/{0}/{1}.dcos_config'.format(pkg_id.name, pkg_id_str), os.getcwd(), rm_on_error=False, ) late_package = load_yaml(f.name) # Resolve the late package using the bound late config values. final_late_package = resolve_late_package(late_package, late_values) # Render the package onto the filesystem and add it to the package # repository. with tempfile.NamedTemporaryFile() as f: do_gen_package(final_late_package, f.name) repository.add(lambda _, target: extract_tarball(f.name, target), pkg_id_str) setup_packages_to_activate.append(pkg_id_str) # If active.json is set on the host, use that as the set of packages to # activate. Otherwise just use the set of currently active packages (those # active in the bootstrap tarball) to_activate = None active_path = install.get_config_filename("setup-flags/active.json") if os.path.exists(active_path): print("Loaded active packages from", active_path) to_activate = load_json(active_path) # Ensure all packages are local print("Ensuring all packages in active set {} are local".format(",".join(to_activate))) for package in to_activate: repository.add(fetcher, package) else: print("Calculated active packages from bootstrap tarball") to_activate = list(install.get_active()) package_list_filename = install.get_config_filename("setup-flags/cluster-package-list") print("Checking for cluster packages in:", package_list_filename) package_list_id = if_exists(load_string, package_list_filename) if package_list_id: print("Cluster package list:", package_list_id) cluster_packages = _get_package_list(package_list_id, repository_url) print("Loading cluster-packages: {}".format(cluster_packages)) for package_id_str in cluster_packages: # Validate the package ids pkg_id = PackageId(package_id_str) # Fetch the packages if not local if not repository.has_package(package_id_str): repository.add(fetcher, package_id_str) # Add the package to the set to activate setup_packages_to_activate.append(package_id_str) else: print("No cluster-packages specified") # Calculate the full set of final packages (Explicit activations + setup packages). # De-duplicate using a set. to_activate = list(set(to_activate + setup_packages_to_activate)) print("Activating packages") install.activate(repository.load_packages(to_activate))
def genconf(self, master_list, agent_list, public_agent_list, ssh_user, ssh_key, ip_detect, rexray_config=None, rexray_config_preset=None, zk_host=None, expect_errors=False, add_config_path=None, bootstrap_url='file:///opt/dcos_install_tmp'): """Runs configuration generation. Args: master_list: list of IPv4 addresses to be used as masters agent_list: list of IPv4 addresses to be used as agents public_agent_list: list of IPv$ addresses to be used as public agents ip_detect (str): name of preset IP-detect script ssh_user (str): name of SSH user that has access to targets ssh_key (str): complete public SSH key for ssh_user. Must already be installed on tagets as authorized_key rexray_config: complete contents of REX-Ray config file. Must be a JSON-serializable object. rexray_config_preset (str): name of preset REX-Ray config zk_host (optional): if provided, zk is used for exhibitor backend expect_errors (optional): raises error if result is unexpected add_config_path (optional): string pointing to a file with additional config parameters to be merged or used as overide Raises: AssertionError: "error" present in returned json keys when error was not expected or vice versa """ test_config = { 'cluster_name': 'SSH Installed DC/OS', 'bootstrap_url': bootstrap_url, 'dns_search': 'mesos', 'master_discovery': 'static', 'master_list': master_list, 'ssh_user': ssh_user, 'agent_list': agent_list, 'public_agent_list': public_agent_list, 'process_timeout': MAX_STAGE_TIME } if rexray_config: test_config['rexray_config'] = rexray_config if rexray_config_preset: test_config['rexray_config_preset'] = rexray_config_preset if zk_host: test_config['exhibitor_storage_backend'] = 'zookeeper' test_config['exhibitor_zk_hosts'] = zk_host test_config['exhibitor_zk_path'] = '/exhibitor' else: test_config['exhibitor_storage_backend'] = 'static' if add_config_path: add_config = load_yaml(add_config_path) test_config.update(add_config) with open('config.yaml', 'w') as config_fh: config_fh.write(yaml.dump(test_config)) with open('ip-detect', 'w') as ip_detect_fh: ip_detect_fh.write(self.ip_detect_script(ip_detect)) with open('ssh_key', 'w') as key_fh: key_fh.write(ssh_key) remote_dir = os.path.dirname(self.installer_path) self.tunnel.remote_cmd( ['mkdir', '-p', os.path.join(remote_dir, 'genconf')]) self.tunnel.write_to_remote( 'config.yaml', os.path.join(remote_dir, 'genconf/config.yaml')) self.tunnel.write_to_remote( 'ip-detect', os.path.join(remote_dir, 'genconf/ip-detect')) self.tunnel.write_to_remote( 'ssh_key', os.path.join(remote_dir, 'genconf/ssh_key')) self.tunnel.remote_cmd( ['chmod', '600', os.path.join(remote_dir, 'genconf/ssh_key')]) self.run_cli_cmd('--genconf', expect_errors=expect_errors)
def test_load_user_config(): user_config = load_yaml("/opt/mesosphere/etc/user.config.yaml") # Calculated parameters shouldn't be in the user config assert 'master_quorum' not in user_config
def genconf( self, master_list, agent_list, public_agent_list, ssh_user, ssh_key, ip_detect, platform=None, rexray_config=None, rexray_config_preset=None, zk_host=None, expect_errors=False, add_config_path=None, bootstrap_url='file:///opt/dcos_install_tmp'): """Runs configuration generation. Args: master_list: list of IPv4 addresses to be used as masters agent_list: list of IPv4 addresses to be used as agents public_agent_list: list of IPv$ addresses to be used as public agents ssh_user (str): name of SSH user that has access to targets ssh_key (str): complete public SSH key for ssh_user. Must already be installed on tagets as authorized_key ip_detect (str): name of preset IP-detect script platform (str): name of the infrastructure platform rexray_config: complete contents of REX-Ray config file. Must be a JSON-serializable object. rexray_config_preset (str): name of preset REX-Ray config zk_host (optional): if provided, zk is used for exhibitor backend expect_errors (optional): raises error if result is unexpected add_config_path (optional): string pointing to a file with additional config parameters to be merged or used as overide Raises: AssertionError: "error" present in returned json keys when error was not expected or vice versa """ test_config = { 'cluster_name': 'SSH Installed DC/OS', 'bootstrap_url': bootstrap_url, 'dns_search': 'mesos', 'master_discovery': 'static', 'master_list': master_list, 'ssh_user': ssh_user, 'agent_list': agent_list, 'public_agent_list': public_agent_list, 'process_timeout': MAX_STAGE_TIME} if platform: test_config['platform'] = platform if rexray_config: test_config['rexray_config'] = rexray_config if rexray_config_preset: test_config['rexray_config_preset'] = rexray_config_preset if zk_host: test_config['exhibitor_storage_backend'] = 'zookeeper' test_config['exhibitor_zk_hosts'] = zk_host test_config['exhibitor_zk_path'] = '/exhibitor' else: test_config['exhibitor_storage_backend'] = 'static' if add_config_path: add_config = load_yaml(add_config_path) test_config.update(add_config) with open('config.yaml', 'w') as config_fh: config_fh.write(yaml.dump(test_config)) with open('ip-detect', 'w') as ip_detect_fh: ip_detect_fh.write(self.ip_detect_script(ip_detect)) with open('ssh_key', 'w') as key_fh: key_fh.write(ssh_key) remote_dir = os.path.dirname(self.installer_path) self.tunnel.remote_cmd(['mkdir', '-p', os.path.join(remote_dir, 'genconf')]) self.tunnel.write_to_remote('config.yaml', os.path.join(remote_dir, 'genconf/config.yaml')) self.tunnel.write_to_remote('ip-detect', os.path.join(remote_dir, 'genconf/ip-detect')) self.tunnel.write_to_remote('ssh_key', os.path.join(remote_dir, 'genconf/ssh_key')) self.tunnel.remote_cmd(['chmod', '600', os.path.join(remote_dir, 'genconf/ssh_key')]) self.run_cli_cmd('--genconf', expect_errors=expect_errors)