def test_action_deploy_post(client, monkeypatch): route = '/api/v1/action/deploy' write_string('genconf/config.yaml', ssh_config_yaml) monkeypatch.setattr(dcos_installer.action_lib, '_get_bootstrap_tarball', lambda: '123') monkeypatch.setattr(dcos_installer.action_lib, '_add_copy_packages', lambda _: None) # Deploy should be already executed for action 'deploy' def mocked_json_state(arg): return { 'hosts': { '127.0.0.1': { 'host_status': 'success' }, '127.0.0.2': { 'host_status': 'success' } } } mock_json_state(monkeypatch, mocked_json_state) res = client.request(route, method='POST') assert res.json == {'status': 'deploy was already executed, skipping'} # Test start deploy mock_json_state(monkeypatch, lambda arg: False) res = client.request(route, method='POST') assert res.json == {'status': 'deploy started'}
def do_bundle_onprem(extra_files, gen_out, output_dir): # We are only being called via dcos_generate_config.sh with an output_dir assert output_dir is not None assert output_dir assert output_dir[-1] != '/' output_dir = output_dir + '/' # Copy the extra_files for filename in extra_files: shutil.copy(filename, output_dir + filename) # Copy the cluster packages for name, info in gen_out.cluster_packages.items(): copy_makedirs(info['filename'], output_dir + info['filename']) # Write an index of the cluster packages write_json(output_dir + 'cluster-package-info.json', gen_out.cluster_packages) # Write the bootstrap id write_string(output_dir + 'bootstrap.latest', gen_out.arguments['bootstrap_id']) # Make a package fetch script package_fetches = "\n".join( fetch_pkg_template.format( package_path='packages/{name}/{id}.tar.xz'.format(name=pkgpanda.PackageId(package).name, id=package), bootstrap_url='https://downloads.dcos.io/dcos/stable' ) for package in load_json("/artifacts/{}.active.json".format(gen_out.arguments['bootstrap_id']))) write_string(output_dir + 'fetch_packages.sh', fetch_all_pkgs.format(package_fetches=package_fetches))
def generate_node_upgrade_script(gen_out, installed_cluster_version, serve_dir=SERVE_DIR): # installed_cluster_version: Current installed version on the cluster # installer_version: Version we are upgrading to bootstrap_url = gen_out.arguments['bootstrap_url'] installer_version = gen.calc.entry['must']['dcos_version'] package_list = ' '.join(package['id'] for package in gen_out.cluster_packages.values()) bash_script = gen.template.parse_str(node_upgrade_template).render({ 'dcos_image_commit': util.dcos_image_commit, 'generation_date': util.template_generation_date, 'bootstrap_url': bootstrap_url, 'cluster_packages': package_list, 'installed_cluster_version': installed_cluster_version, 'installer_version': installer_version}) upgrade_script_path = '/upgrade/' + uuid.uuid4().hex subprocess.check_call(['mkdir', '-p', serve_dir + upgrade_script_path]) write_string(serve_dir + upgrade_script_path + '/dcos_node_upgrade.sh', bash_script) print("Node upgrade script URL: " + bootstrap_url + upgrade_script_path + '/dcos_node_upgrade.sh') return 0
def setup(install, repository): """Set up a fresh install of DC/OS. install: pkgpanda.Install repository: pkgpanda.Repository """ # Check for /opt/mesosphere/bootstrap. If not exists, download everything # and install /etc/systemd/system/mutli-user.target/dcos.target bootstrap_path = os.path.join(install.root, "bootstrap") if os.path.exists(bootstrap_path): # Write, enable /etc/systemd/system/dcos.target for next boot. dcos_target_dir = os.path.dirname(install.systemd_dir) try: os.makedirs(dcos_target_dir) except FileExistsError: pass write_string(os.path.join(dcos_target_dir, "dcos.target"), DCOS_TARGET_CONTENTS) _do_bootstrap(install, repository) # Enable dcos.target only after we have populated it to prevent starting # up stuff inside of it before we activate the new set of packages. if install.manage_systemd: _start_dcos_target(block_systemd=True) os.remove(bootstrap_path) # Check for /opt/mesosphere/install_progress. If found, recover the partial # update. if os.path.exists(install_root + "/install_progress"): took_action, msg = install.recover_swap_active() if not took_action: print("No recovery performed: {}".format(msg))
def mark_latest(): # Ensure latest is always written write_string(latest_name, bootstrap_id) print("bootstrap: {}".format(bootstrap_name)) print("active: {}".format(active_name)) print("latest: {}".format(latest_name)) return bootstrap_id
def main(): options = check_environment() bw = test_util.aws.BotoWrapper( region=options.aws_region, aws_access_key_id=options.aws_access_key_id, aws_secret_access_key=options.aws_secret_access_key) stack_name = 'dcos-ci-test-cf-{}'.format(random_id(10)) ssh_key = bw.create_key_pair(stack_name) write_string('ssh_key', ssh_key) log.info('Spinning up AWS CloudFormation with ID: {}'.format(stack_name)) if options.advanced: cf, ssh_info = test_util.aws.DcosZenCfStack.create( stack_name=stack_name, boto_wrapper=bw, template_url=options.template_url, private_agents=options.agents, public_agents=options.public_agents, key_pair_name=stack_name, private_agent_type='m3.xlarge', public_agent_type='m3.xlarge', master_type='m3.xlarge', vpc=options.vpc, gateway=options.gateway, private_subnet=options.private_subnet, public_subnet=options.public_subnet) else: cf, ssh_info = test_util.aws.DcosCfStack.create( stack_name=stack_name, template_url=options.template_url, private_agents=options.agents, public_agents=options.public_agents, admin_location='0.0.0.0/0', key_pair_name=stack_name, boto_wrapper=bw) time.sleep(300) # we know the cluster is not ready yet, don't poll to avoid hitting the rate limit cf.wait_for_complete() # Resiliency testing requires knowing the stack name options.test_cmd = 'AWS_STACK_NAME=' + stack_name + ' ' + options.test_cmd # hidden hook where user can supply an ssh_key for a preexisting cluster cluster = test_util.cluster.Cluster.from_cloudformation(cf, ssh_info, ssh_key) result = test_util.cluster.run_integration_tests( cluster, region=options.aws_region, aws_access_key_id=options.aws_access_key_id, aws_secret_access_key=options.aws_secret_access_key, test_cmd=options.test_cmd, ) if result == 0: log.info('Test successful! Deleting CloudFormation.') cf.delete() bw.delete_key_pair(stack_name) else: logging.warning('Test exited with an error') if options.ci_flags: result = 0 # Wipe the return code so that tests can be muted in CI sys.exit(result)
def integration_test( tunnel, test_dir, dcos_dns, master_list, agent_list, public_agent_list, test_dns_search, provider, ci_flags, aws_access_key_id='', aws_secret_access_key='', region='', add_env=None): """Runs integration test on host Args: test_dir: directory to leave test_wrapper.sh dcos_dns: string representing IP of DCOS DNS host master_list: string of comma separated master addresses agent_list: string of comma separated agent addresses test_dns_search: if set to True, test for deployed mesos DNS app ci_flags: optional additional string to be passed to test provider: (str) either onprem, aws, or azure # The following variables correspond to currently disabled tests aws_access_key_id: needed for REXRAY tests aws_secret_access_key: needed for REXRAY tests region: string indicating AWS region in which cluster is running add_env: a python dict with any number of key=value assignments to be passed to the test environment """ dns_search = 'true' if test_dns_search else 'false' test_env = [ 'DCOS_DNS_ADDRESS=http://'+dcos_dns, 'MASTER_HOSTS='+','.join(master_list), 'PUBLIC_MASTER_HOSTS='+','.join(master_list), 'SLAVE_HOSTS='+','.join(agent_list), 'PUBLIC_SLAVE_HOSTS='+','.join(public_agent_list), 'DCOS_PROVIDER='+provider, 'DNS_SEARCH='+dns_search, 'AWS_ACCESS_KEY_ID='+aws_access_key_id, 'AWS_SECRET_ACCESS_KEY='+aws_secret_access_key, 'AWS_REGION='+region] if add_env: for key, value in add_env.items(): extra_env = key + '=' + value test_env.append(extra_env) test_wrapper = """#!/bin/bash source /opt/mesosphere/environment.export {env} cd /opt/mesosphere/active/dcos-integration-test py.test -vv {ci_flags} """ test_env_str = ''.join(['export '+e+'\n' for e in test_env]) write_string('test_wrapper.sh', test_wrapper.format(env=test_env_str, ci_flags=ci_flags)) wrapper_path = join(test_dir, 'test_wrapper.sh') log.info('Running integration test...') tunnel.write_to_remote('test_wrapper.sh', wrapper_path) try: tunnel.remote_cmd(['bash', wrapper_path], stdout=sys.stdout.buffer) except CalledProcessError as e: return e.returncode return 0
def temp_data(key): temp_dir = tempfile.mkdtemp() socket_path = temp_dir + '/control_socket' key_path = temp_dir + '/key' write_string(key_path, key) os.chmod(key_path, stat.S_IREAD | stat.S_IWRITE) yield (socket_path, key_path) os.remove(key_path) # might have been deleted already if SSH exited correctly if os.path.exists(socket_path): os.remove(socket_path) os.rmdir(temp_dir)
def mock_do_build_packages(cache_repository_url, tree_variants): make_directory('packages/cache/bootstrap') write_string("packages/cache/bootstrap/bootstrap_id.bootstrap.tar.xz", "bootstrap_contents") write_json("packages/cache/bootstrap/bootstrap_id.active.json", ['a--b', 'c--d']) write_string("packages/cache/bootstrap/bootstrap.latest", "bootstrap_id") write_string("packages/cache/bootstrap/installer.bootstrap.latest", "installer_bootstrap_id") write_json("packages/cache/bootstrap/installer_bootstrap_id.active.json", ['c--d', 'e--f']) write_string("packages/cache/bootstrap/downstream.installer.bootstrap.latest", "downstream_installer_bootstrap_id") write_json("packages/cache/bootstrap/downstream_installer_bootstrap_id.active.json", []) make_directory('packages/cache/complete') write_json( "packages/cache/complete/complete.latest.json", {'bootstrap': 'bootstrap_id', 'packages': ['a--b', 'c--d']}) write_json( "packages/cache/complete/installer.complete.latest.json", {'bootstrap': 'installer_bootstrap_id', 'packages': ['c--d', 'e--f']}) write_json( "packages/cache/complete/downstream.installer.complete.latest.json", {'bootstrap': 'installer_bootstrap_id', 'packages': []}) return { None: {"bootstrap": "bootstrap_id", "packages": ["a--b", "c--d"]}, "installer": {"bootstrap": "installer_bootstrap_id", "packages": ["c--d", "e--f"]}, "downstream.installer": {"bootstrap": "downstream_installer_bootstrap_id", "packages": []} }
def mock_do_build_packages(cache_repository_url): subprocess.check_call(['mkdir', '-p', 'packages/cache/bootstrap']) write_string("packages/cache/bootstrap/bootstrap_id.bootstrap.tar.xz", "bootstrap_contents") write_json("packages/cache/bootstrap/bootstrap_id.active.json", ['a--b', 'c--d']) write_string("packages/cache/bootstrap/bootstrap.latest", "bootstrap_id") write_string("packages/cache/bootstrap/installer.bootstrap.latest", "installer_bootstrap_id") write_json("packages/cache/bootstrap/installer_bootstrap_id.active.json", ['c--d', 'e--f']) write_string("packages/cache/bootstrap/ee.installer.bootstrap.latest", "ee_installer_bootstrap_id") write_json("packages/cache/bootstrap/ee_installer_bootstrap_id.active.json", []) subprocess.check_call(['mkdir', '-p', 'packages/cache/complete']) write_json( "packages/cache/complete/complete.latest.json", {'bootstrap': 'bootstrap_id', 'packages': ['a--b', 'c--d']}) write_json( "packages/cache/complete/installer.complete.latest.json", {'bootstrap': 'installer_bootstrap_id', 'packages': ['c--d', 'e--f']}) write_json( "packages/cache/complete/ee.installer.complete.latest.json", {'bootstrap': 'installer_bootstrap_id', 'packages': []}) return { None: {"bootstrap": "bootstrap_id", "packages": ["a--b", "c--d"]}, "installer": {"bootstrap": "installer_bootstrap_id", "packages": ["c--d", "e--f"]}, "ee.installer": {"bootstrap": "ee_installer_bootstrap_id", "packages": []} }
def wait(master_count_filename): if try_shortcut(): log.info("Shortcut succeeeded, assuming local zk is in good config state, not waiting for quorum.") return log.info('Shortcut failed, waiting for exhibitor to bring up zookeeper and stabilize') if not os.path.exists(master_count_filename): log.info("master_count file doesn't exist when it should. Hard failing.") sys.exit(1) cluster_size = int(utils.read_file_line(master_count_filename)) log.info('Expected cluster size: {}'.format(cluster_size)) log.info('Waiting for ZooKeeper cluster to stabilize') try: response = requests.get(EXHIBITOR_STATUS_URL) except requests.exceptions.ConnectionError as ex: log.error('Could not connect to exhibitor: {}'.format(ex)) sys.exit(1) if response.status_code != 200: log.error('Could not get exhibitor status: {}, Status code: {}'.format( EXHIBITOR_STATUS_URL, response.status_code)) sys.exit(1) data = response.json() serving = [] leaders = [] for node in data: if node['isLeader']: leaders.append(node['hostname']) if node['description'] == 'serving': serving.append(node['hostname']) log.info( "Serving hosts: `%s`, leader: `%s`", ','.join(serving), ','.join(leaders)) if len(serving) != cluster_size or len(leaders) != 1: msg_fmt = 'Expected {} servers and 1 leader, got {} servers and {} leaders' raise Exception(msg_fmt.format(cluster_size, len(serving), len(leaders))) # Local Zookeeper is up. Config should be stable, local zookeeper happy. Stash the PID so if # there is a restart we can come up quickly without requiring a new zookeeper quorum. zk_pid_mtime = get_zk_pid_mtime() if zk_pid_mtime is not None: log.info('Stashing zk.pid mtime %s to %s', zk_pid_mtime, stash_zk_pid_stat_mtime_path) write_string(stash_zk_pid_stat_mtime_path, str(zk_pid_mtime))
def mock_do_build_packages(cache_repository_url): subprocess.check_call(["mkdir", "-p", "packages/cache/bootstrap"]) write_string("packages/cache/bootstrap/bootstrap_id.bootstrap.tar.xz", "bootstrap_contents") write_json("packages/cache/bootstrap/bootstrap_id.active.json", ["a--b", "c--d"]) write_string("packages/cache/bootstrap/bootstrap.latest", "bootstrap_id") write_string("packages/cache/bootstrap/installer.bootstrap.latest", "installer_bootstrap_id") write_json("packages/cache/bootstrap/installer_bootstrap_id.active.json", ["c--d", "e--f"]) write_string("packages/cache/bootstrap/ee.installer.bootstrap.latest", "ee_installer_bootstrap_id") write_json("packages/cache/bootstrap/ee_installer_bootstrap_id.active.json", []) return {None: "bootstrap_id", "installer": "installer_bootstrap_id", "ee.installer": "ee_installer_bootstrap_id"}
def temp_data(key: str) -> (str, str): """ Provides file paths for data required to establish the SSH tunnel Args: key: string containing the private SSH key Returns: (path_for_temp_socket_file, path_for_temp_ssh_key) """ temp_dir = tempfile.mkdtemp() socket_path = temp_dir + '/control_socket' key_path = temp_dir + '/key' write_string(key_path, key) os.chmod(key_path, stat.S_IREAD | stat.S_IWRITE) yield (socket_path, key_path) os.remove(key_path) # might have been deleted already if SSH exited correctly if os.path.exists(socket_path): os.remove(socket_path) os.rmdir(temp_dir)
def do_bundle_onprem(gen_out, output_dir): # We are only being called via dcos_generate_config.sh with an output_dir assert output_dir is not None assert output_dir assert output_dir[-1] != '/' output_dir = output_dir + '/' # Copy generated artifacts for filename in gen_out.channel_artifacts + gen_out.stable_artifacts: copy_makedirs(filename, output_dir + filename) # Write an index of the cluster packages write_json(output_dir + 'cluster-package-info.json', gen_out.cluster_packages) # Write the bootstrap id write_string(output_dir + 'bootstrap.latest', gen_out.arguments['bootstrap_id']) # Write cluster package list ID write_string(output_dir + 'cluster-package-list.latest', gen_out.arguments['cluster_package_list_id'])
def do_bundle_onprem(extra_files, gen_out, output_dir): # We are only being called via dcos_generate_config.sh with an output_dir assert output_dir is not None assert output_dir assert output_dir[-1] != '/' output_dir = output_dir + '/' # Copy the extra_files for filename in extra_files: shutil.copy(filename, output_dir + filename) # Copy the cluster packages for name, info in gen_out.cluster_packages.items(): copy_makedirs(info['filename'], output_dir + info['filename']) # Write an index of the cluster packages write_json(output_dir + 'cluster-package-info.json', gen_out.cluster_packages) # Write the bootstrap id write_string(output_dir + 'bootstrap.latest', gen_out.arguments['bootstrap_id'])
def do_bundle_onprem(extra_files, gen_out, output_dir): # We are only being called via dcos_generate_config.sh with an output_dir assert output_dir is not None assert output_dir assert output_dir[-1] != '/' output_dir = output_dir + '/' # Copy the extra_files for filename in extra_files: copy_makedirs(filename, output_dir + filename) # Copy the config packages for package_name in json.loads(gen_out.arguments['config_package_names']): filename = gen_out.cluster_packages[package_name]['filename'] copy_makedirs(filename, output_dir + filename) # Write an index of the cluster packages write_json(output_dir + 'cluster-package-info.json', gen_out.cluster_packages) # Write the bootstrap id write_string(output_dir + 'bootstrap.latest', gen_out.arguments['bootstrap_id'])
def test_action_deploy_retry(client, monkeypatch): route = '/api/v1/action/deploy' write_string('genconf/config.yaml', ssh_config_yaml) monkeypatch.setattr(dcos_installer.action_lib, '_get_bootstrap_tarball', lambda: '123') monkeypatch.setattr(dcos_installer.action_lib, '_get_cluster_package_list', lambda: '123') monkeypatch.setattr(dcos_installer.action_lib, '_add_copy_packages', lambda _: None) monkeypatch.setattr(dcos_installer.action_lib, '_read_state_file', lambda state_file: {'total_hosts': 2}) removed_hosts = list() def mocked_remove_host(state_file, host): removed_hosts.append(host) monkeypatch.setattr(dcos_installer.action_lib, '_remove_host', mocked_remove_host) # Test retry def mocked_json_state(arg): return { 'hosts': { '127.0.0.1:22': { 'host_status': 'failed', 'tags': {'role': 'master', 'dcos_install_param': 'master'}, }, '127.0.0.2:22022': { 'host_status': 'success' }, '127.0.0.3:22022': { 'host_status': 'failed', 'tags': {'role': 'agent', 'dcos_install_param': 'slave'}, } } } mock_json_state(monkeypatch, mocked_json_state) res = client.post(route, params={'retry': 'true'}, content_type='application/x-www-form-urlencoded') assert res.json == {'details': ['127.0.0.1:22', '127.0.0.3:22022'], 'status': 'retried'} assert len(set(removed_hosts)) == 2, \ "Should have had two hosts removed exactly once, removed_hosts: {}".format(removed_hosts) assert set(removed_hosts) == {'127.0.0.3:22022', '127.0.0.1:22'}
def mock_do_build_packages(cache_repository_url): subprocess.check_call(['mkdir', '-p', 'packages']) write_string("packages/bootstrap_id.bootstrap.tar.xz", "bootstrap_contents") write_json("packages/bootstrap_id.active.json", ['a--b', 'c--d']) write_string("packages/bootstrap.latest", "bootstrap_id") write_string("packages/installer.bootstrap.latest", "installer_bootstrap_id") write_json("packages/installer_bootstrap_id.active.json", ['c--d', 'e--f']) write_string("packages/ee.installer.bootstrap.latest", "ee_installer_bootstrap_id") write_json("packages/ee_installer_bootstrap_id.active.json", []) return { None: "bootstrap_id", "installer": "installer_bootstrap_id", "ee.installer": "ee_installer_bootstrap_id" }
def mock_do_build_packages(cache_repository_url, skip_build): subprocess.check_call(['mkdir', '-p', 'packages']) write_string("packages/bootstrap_id.bootstrap.tar.xz", "bootstrap_contents") write_json("packages/bootstrap_id.active.json", ['a--b', 'c--d']) write_string("packages/bootstrap.latest", "bootstrap_id") write_string("packages/installer.bootstrap.latest", "installer_bootstrap_id") write_json("packages/installer_bootstrap_id.active.json", ['c--d', 'e--f']) write_string("packages/ee.installer.bootstrap.latest", "ee_installer_bootstrap_id") write_json("packages/ee_installer_bootstrap_id.active.json", []) return { None: "bootstrap_id", "installer": "installer_bootstrap_id", "ee.installer": "ee_installer_bootstrap_id" }
def build(package_store, name, variant, clean_after_build, recursive=False): assert isinstance(package_store, PackageStore) print("Building package {} variant {}".format(name, pkgpanda.util.variant_str(variant))) tmpdir = tempfile.TemporaryDirectory(prefix="pkgpanda_repo") repository = Repository(tmpdir.name) package_dir = package_store.get_package_folder(name) def src_abs(name): return package_dir + '/' + name def cache_abs(filename): return package_store.get_package_cache_folder(name) + '/' + filename # Build pkginfo over time, translating fields from buildinfo. pkginfo = {} # Build up the docker command arguments over time, translating fields as needed. cmd = DockerCmd() assert (name, variant) in package_store.packages, \ "Programming error: name, variant should have been validated to be valid before calling build()." builder = IdBuilder(package_store.get_buildinfo(name, variant)) final_buildinfo = dict() builder.add('name', name) builder.add('variant', pkgpanda.util.variant_str(variant)) # Convert single_source -> sources if builder.has('sources'): if builder.has('single_source'): raise BuildError('Both sources and single_source cannot be specified at the same time') sources = builder.take('sources') elif builder.has('single_source'): sources = {name: builder.take('single_source')} builder.replace('single_source', 'sources', sources) else: builder.add('sources', {}) sources = dict() print("NOTICE: No sources specified") final_buildinfo['sources'] = sources # Construct the source fetchers, gather the checkout ids from them checkout_ids = dict() fetchers = dict() try: for src_name, src_info in sorted(sources.items()): # TODO(cmaloney): Switch to a unified top level cache directory shared by all packages cache_dir = package_store.get_package_cache_folder(name) + '/' + src_name check_call(['mkdir', '-p', cache_dir]) fetcher = get_src_fetcher(src_info, cache_dir, package_dir) fetchers[src_name] = fetcher checkout_ids[src_name] = fetcher.get_id() except ValidationError as ex: raise BuildError("Validation error when fetching sources for package: {}".format(ex)) for src_name, checkout_id in checkout_ids.items(): # NOTE: single_source buildinfo was expanded above so the src_name is # always correct here. # Make sure we never accidentally overwrite something which might be # important. Fields should match if specified (And that should be # tested at some point). For now disallowing identical saves hassle. assert_no_duplicate_keys(checkout_id, final_buildinfo['sources'][src_name]) final_buildinfo['sources'][src_name].update(checkout_id) # Add the sha1 of the buildinfo.json + build file to the build ids builder.update('sources', checkout_ids) build_script = src_abs(builder.take('build_script')) # TODO(cmaloney): Change dest name to build_script_sha1 builder.replace('build_script', 'build', pkgpanda.util.sha1(build_script)) builder.add('pkgpanda_version', pkgpanda.build.constants.version) extra_dir = src_abs("extra") # Add the "extra" folder inside the package as an additional source if it # exists if os.path.exists(extra_dir): extra_id = hash_folder(extra_dir) builder.add('extra_source', extra_id) final_buildinfo['extra_source'] = extra_id # Figure out the docker name. docker_name = builder.take('docker') cmd.container = docker_name # Add the id of the docker build environment to the build_ids. try: docker_id = get_docker_id(docker_name) except CalledProcessError: # docker pull the container and try again check_call(['docker', 'pull', docker_name]) docker_id = get_docker_id(docker_name) builder.update('docker', docker_id) # TODO(cmaloney): The environment variables should be generated during build # not live in buildinfo.json. pkginfo['environment'] = builder.take('environment') # Whether pkgpanda should on the host make sure a `/var/lib` state directory is available pkginfo['state_directory'] = builder.take('state_directory') if pkginfo['state_directory'] not in [True, False]: raise BuildError("state_directory in buildinfo.json must be a boolean `true` or `false`") username = None if builder.has('username'): username = builder.take('username') if not isinstance(username, str): raise BuildError("username in buildinfo.json must be either not set (no user for this" " package), or a user name string") try: pkgpanda.UserManagement.validate_username(username) except ValidationError as ex: raise BuildError("username in buildinfo.json didn't meet the validation rules. {}".format(ex)) pkginfo['username'] = username group = None if builder.has('group'): group = builder.take('group') if not isinstance(group, str): raise BuildError("group in buildinfo.json must be either not set (use default group for this user)" ", or group must be a string") try: pkgpanda.UserManagement.validate_group_name(group) except ValidationError as ex: raise BuildError("group in buildinfo.json didn't meet the validation rules. {}".format(ex)) pkginfo['group'] = group # Packages need directories inside the fake install root (otherwise docker # will try making the directories on a readonly filesystem), so build the # install root now, and make the package directories in it as we go. install_dir = tempfile.mkdtemp(prefix="pkgpanda-") active_packages = list() active_package_ids = set() active_package_variants = dict() auto_deps = set() # Final package has the same requires as the build. requires = builder.take('requires') pkginfo['requires'] = requires if builder.has("sysctl"): pkginfo["sysctl"] = builder.take("sysctl") # TODO(cmaloney): Pull generating the full set of requires a function. to_check = copy.deepcopy(requires) if type(to_check) != list: raise BuildError("`requires` in buildinfo.json must be an array of dependencies.") while to_check: requires_info = to_check.pop(0) requires_name, requires_variant = expand_require(requires_info) if requires_name in active_package_variants: # TODO(cmaloney): If one package depends on the <default> # variant of a package and 1+ others depends on a non-<default> # variant then update the dependency to the non-default variant # rather than erroring. if requires_variant != active_package_variants[requires_name]: # TODO(cmaloney): Make this contain the chains of # dependencies which contain the conflicting packages. # a -> b -> c -> d {foo} # e {bar} -> d {baz} raise BuildError( "Dependncy on multiple variants of the same package {}. variants: {} {}".format( requires_name, requires_variant, active_package_variants[requires_name])) # The variant has package {requires_name, variant} already is a # dependency, don't process it again / move on to the next. continue active_package_variants[requires_name] = requires_variant # Figure out the last build of the dependency, add that as the # fully expanded dependency. requires_last_build = package_store.get_last_build_filename(requires_name, requires_variant) if not os.path.exists(requires_last_build): if recursive: # Build the dependency build(package_store, requires_name, requires_variant, clean_after_build, recursive) else: raise BuildError("No last build file found for dependency {} variant {}. Rebuild " "the dependency".format(requires_name, requires_variant)) try: pkg_id_str = load_string(requires_last_build) auto_deps.add(pkg_id_str) pkg_buildinfo = package_store.get_buildinfo(requires_name, requires_variant) pkg_requires = pkg_buildinfo['requires'] pkg_path = repository.package_path(pkg_id_str) pkg_tar = pkg_id_str + '.tar.xz' if not os.path.exists(package_store.get_package_cache_folder(requires_name) + '/' + pkg_tar): raise BuildError( "The build tarball {} refered to by the last_build file of the dependency {} " "variant {} doesn't exist. Rebuild the dependency.".format( pkg_tar, requires_name, requires_variant)) active_package_ids.add(pkg_id_str) # Mount the package into the docker container. cmd.volumes[pkg_path] = "/opt/mesosphere/packages/{}:ro".format(pkg_id_str) os.makedirs(os.path.join(install_dir, "packages/{}".format(pkg_id_str))) # Add the dependencies of the package to the set which will be # activated. # TODO(cmaloney): All these 'transitive' dependencies shouldn't # be available to the package being built, only what depends on # them directly. to_check += pkg_requires except ValidationError as ex: raise BuildError("validating package needed as dependency {0}: {1}".format(requires_name, ex)) from ex except PackageError as ex: raise BuildError("loading package needed as dependency {0}: {1}".format(requires_name, ex)) from ex # Add requires to the package id, calculate the final package id. # NOTE: active_packages isn't fully constructed here since we lazily load # packages not already in the repository. builder.update('requires', list(active_package_ids)) version_extra = None if builder.has('version_extra'): version_extra = builder.take('version_extra') build_ids = builder.get_build_ids() version_base = hash_checkout(build_ids) version = None if builder.has('version_extra'): version = "{0}-{1}".format(version_extra, version_base) else: version = version_base pkg_id = PackageId.from_parts(name, version) # Everything must have been extracted by now. If it wasn't, then we just # had a hard error that it was set but not used, as well as didn't include # it in the caluclation of the PackageId. builder = None # Save the build_ids. Useful for verify exactly what went into the # package build hash. final_buildinfo['build_ids'] = build_ids final_buildinfo['package_version'] = version # Save the package name and variant. The variant is used when installing # packages to validate dependencies. final_buildinfo['name'] = name final_buildinfo['variant'] = variant # If the package is already built, don't do anything. pkg_path = package_store.get_package_cache_folder(name) + '/{}.tar.xz'.format(pkg_id) # Done if it exists locally if exists(pkg_path): print("Package up to date. Not re-building.") # TODO(cmaloney): Updating / filling last_build should be moved out of # the build function. write_string(package_store.get_last_build_filename(name, variant), str(pkg_id)) return pkg_path # Try downloading. dl_path = package_store.try_fetch_by_id(pkg_id) if dl_path: print("Package up to date. Not re-building. Downloaded from repository-url.") # TODO(cmaloney): Updating / filling last_build should be moved out of # the build function. write_string(package_store.get_last_build_filename(name, variant), str(pkg_id)) print(dl_path, pkg_path) assert dl_path == pkg_path return pkg_path # Fall out and do the build since it couldn't be downloaded print("Unable to download from cache. Proceeding to build") print("Building package {} with buildinfo: {}".format( pkg_id, json.dumps(final_buildinfo, indent=2, sort_keys=True))) # Clean out src, result so later steps can use them freely for building. def clean(): # Run a docker container to remove src/ and result/ cmd = DockerCmd() cmd.volumes = { package_store.get_package_cache_folder(name): "/pkg/:rw", } cmd.container = "ubuntu:14.04.4" cmd.run("package-cleaner", ["rm", "-rf", "/pkg/src", "/pkg/result"]) clean() # Only fresh builds are allowed which don't overlap existing artifacts. result_dir = cache_abs("result") if exists(result_dir): raise BuildError("result folder must not exist. It will be made when the package is " "built. {}".format(result_dir)) # 'mkpanda add' all implicit dependencies since we actually need to build. for dep in auto_deps: print("Auto-adding dependency: {}".format(dep)) # NOTE: Not using the name pkg_id because that overrides the outer one. id_obj = PackageId(dep) add_package_file(repository, package_store.get_package_path(id_obj)) package = repository.load(dep) active_packages.append(package) # Checkout all the sources int their respective 'src/' folders. try: src_dir = cache_abs('src') if os.path.exists(src_dir): raise ValidationError( "'src' directory already exists, did you have a previous build? " + "Currently all builds must be from scratch. Support should be " + "added for re-using a src directory when possible. src={}".format(src_dir)) os.mkdir(src_dir) for src_name, fetcher in sorted(fetchers.items()): root = cache_abs('src/' + src_name) os.mkdir(root) fetcher.checkout_to(root) except ValidationError as ex: raise BuildError("Validation error when fetching sources for package: {}".format(ex)) # Activate the packages so that we have a proper path, environment # variables. # TODO(cmaloney): RAII type thing for temproary directory so if we # don't get all the way through things will be cleaned up? install = Install( root=install_dir, config_dir=None, rooted_systemd=True, manage_systemd=False, block_systemd=True, fake_path=True, manage_users=False, manage_state_dir=False) install.activate(active_packages) # Rewrite all the symlinks inside the active path because we will # be mounting the folder into a docker container, and the absolute # paths to the packages will change. # TODO(cmaloney): This isn't very clean, it would be much nicer to # just run pkgpanda inside the package. rewrite_symlinks(install_dir, repository.path, "/opt/mesosphere/packages/") print("Building package in docker") # TODO(cmaloney): Run as a specific non-root user, make it possible # for non-root to cleanup afterwards. # Run the build, prepping the environment as necessary. mkdir(cache_abs("result")) # Copy the build info to the resulting tarball write_json(cache_abs("src/buildinfo.full.json"), final_buildinfo) write_json(cache_abs("result/buildinfo.full.json"), final_buildinfo) write_json(cache_abs("result/pkginfo.json"), pkginfo) # Make the folder for the package we are building. If docker does it, it # gets auto-created with root permissions and we can't actually delete it. os.makedirs(os.path.join(install_dir, "packages", str(pkg_id))) # TOOD(cmaloney): Disallow writing to well known files and directories? # Source we checked out cmd.volumes.update({ # TODO(cmaloney): src should be read only... cache_abs("src"): "/pkg/src:rw", # The build script build_script: "/pkg/build:ro", # Getting the result out cache_abs("result"): "/opt/mesosphere/packages/{}:rw".format(pkg_id), install_dir: "/opt/mesosphere:ro" }) if os.path.exists(extra_dir): cmd.volumes[extra_dir] = "/pkg/extra:ro" cmd.environment = { "PKG_VERSION": version, "PKG_NAME": name, "PKG_ID": pkg_id, "PKG_PATH": "/opt/mesosphere/packages/{}".format(pkg_id), "PKG_VARIANT": variant if variant is not None else "<default>", "NUM_CORES": multiprocessing.cpu_count() } try: # TODO(cmaloney): Run a wrapper which sources # /opt/mesosphere/environment then runs a build. Also should fix # ownership of /opt/mesosphere/packages/{pkg_id} post build. cmd.run("package-builder", [ "/bin/bash", "-o", "nounset", "-o", "pipefail", "-o", "errexit", "/pkg/build"]) except CalledProcessError as ex: raise BuildError("docker exited non-zero: {}\nCommand: {}".format(ex.returncode, ' '.join(ex.cmd))) # Clean up the temporary install dir used for dependencies. # TODO(cmaloney): Move to an RAII wrapper. check_call(['rm', '-rf', install_dir]) print("Building package tarball") # Check for forbidden services before packaging the tarball: try: check_forbidden_services(cache_abs("result"), RESERVED_UNIT_NAMES) except ValidationError as ex: raise BuildError("Package validation failed: {}".format(ex)) # TODO(cmaloney): Updating / filling last_build should be moved out of # the build function. write_string(package_store.get_last_build_filename(name, variant), str(pkg_id)) # Bundle the artifacts into the pkgpanda package tmp_name = pkg_path + "-tmp.tar.xz" make_tar(tmp_name, cache_abs("result")) os.rename(tmp_name, pkg_path) print("Package built.") if clean_after_build: clean() return pkg_path
def activate(self, packages): # Ensure the new set is reasonable. validate_compatible(packages, self.__roles) # Build the absolute paths for the running config, new config location, # and where to archive the config. active_names = self.get_active_names() active_dirs = list(map(self._make_abs, self.__well_known_dirs + ["active"])) new_names = [name + ".new" for name in active_names] new_dirs = [name + ".new" for name in active_dirs] old_names = [name + ".old" for name in active_names] # Remove all pre-existing new and old directories for name in chain(new_names, old_names): if os.path.exists(name): if os.path.isdir(name): shutil.rmtree(name) else: os.remove(name) # Make the directories for the new config for name in new_dirs: os.makedirs(name) def symlink_all(src, dest): if not os.path.isdir(src): return symlink_tree(src, dest) # Set the new LD_LIBRARY_PATH, PATH. env_contents = env_header.format("/opt/mesosphere" if self.__fake_path else self.__root) env_export_contents = env_export_header.format("/opt/mesosphere" if self.__fake_path else self.__root) active_buildinfo_full = {} dcos_service_configuration = self._get_dcos_configuration_template() # Building up the set of users sysusers = UserManagement(self.__manage_users, self.__add_users) def _get_service_files(_dir): service_files = [] for root, directories, filenames in os.walk(_dir): for filename in filter(lambda name: name.endswith(".service"), filenames): service_files.append(os.path.join(root, filename)) return service_files def _get_service_names(_dir): service_files = list(map(os.path.basename, _get_service_files(_dir))) if not service_files: return [] return list(map(lambda name: os.path.splitext(name)[0], service_files)) # Add the folders, config in each package. for package in packages: # Package folders # NOTE: Since active is at the end of the folder list it will be # removed by the zip. This is the desired behavior, since it will be # populated later. # Do the basename since some well known dirs are full paths (dcos.target.wants) # while inside the packages they are always top level directories. for new, dir_name in zip(new_dirs, self.__well_known_dirs): dir_name = os.path.basename(dir_name) pkg_dir = os.path.join(package.path, dir_name) assert os.path.isabs(new) assert os.path.isabs(pkg_dir) try: symlink_all(pkg_dir, new) # Symlink all applicable role-based config for role in self.__roles: role_dir = os.path.join(package.path, "{0}_{1}".format(dir_name, role)) symlink_all(role_dir, new) except ConflictingFile as ex: raise ValidationError("Two packages are trying to install the same file {0} or " "two roles in the set of roles {1} are causing a package " "to try activating multiple versions of the same file. " "One of the package files is {2}.".format(ex.dest, self.__roles, ex.src)) # Add to the active folder os.symlink(package.path, os.path.join(self._make_abs("active.new"), package.name)) # Add to the environment and environment.export contents env_contents += "# package: {0}\n".format(package.id) env_export_contents += "# package: {0}\n".format(package.id) for k, v in package.environment.items(): env_contents += "{0}={1}\n".format(k, v) env_export_contents += "export {0}={1}\n".format(k, v) env_contents += "\n" env_export_contents += "\n" # Add to the buildinfo try: active_buildinfo_full[package.name] = load_json(os.path.join(package.path, "buildinfo.full.json")) except FileNotFoundError: # TODO(cmaloney): These only come from setup-packages. Should update # setup-packages to add a buildinfo.full for those packages active_buildinfo_full[package.name] = None # NOTE: It is critical the state dir, the package name and the user name are all the # same. Otherwise on upgrades we might remove access to a files by changing their chown # to something incompatible. We survive the first upgrade because everything goes from # root to specific users, and root can access all user files. if package.username is not None: sysusers.add_user(package.username, package.group) # Ensure the state directory exists # TODO(cmaloney): On upgrade take a snapshot? if self.__manage_state_dir: state_dir_path = self.__state_dir_root + '/' + package.name if package.state_directory: check_call(['mkdir', '-p', state_dir_path]) if package.username: uid = sysusers.get_uid(package.username) check_call(['chown', '-R', str(uid), state_dir_path]) if package.sysctl: service_names = _get_service_names(package.path) if not service_names: raise ValueError("service name required for sysctl could not be determined for {package}".format( package=package.id)) for service in service_names: if service in package.sysctl: dcos_service_configuration["sysctl"][service] = package.sysctl[service] dcos_service_configuration_file = os.path.join(self._make_abs("etc.new"), DCOS_SERVICE_CONFIGURATION_FILE) write_json(dcos_service_configuration_file, dcos_service_configuration) # Write out the new environment file. new_env = self._make_abs("environment.new") write_string(new_env, env_contents) # Write out the new environment.export file new_env_export = self._make_abs("environment.export.new") write_string(new_env_export, env_export_contents) # Write out the buildinfo of every active package new_buildinfo_meta = self._make_abs("active.buildinfo.full.json.new") write_json(new_buildinfo_meta, active_buildinfo_full) self.swap_active(".new")
def make_bootstrap_tarball(package_store, packages, variant): # Convert filenames to package ids pkg_ids = list() for pkg_path in packages: # Get the package id from the given package path filename = os.path.basename(pkg_path) if not filename.endswith(".tar.xz"): raise BuildError("Packages must be packaged / end with a .tar.xz. Got {}".format(filename)) pkg_id = filename[:-len(".tar.xz")] pkg_ids.append(pkg_id) bootstrap_cache_dir = package_store.get_bootstrap_cache_dir() # Filename is output_name.<sha-1>.{active.json|.bootstrap.tar.xz} bootstrap_id = hash_checkout(pkg_ids) latest_name = "{}/{}bootstrap.latest".format(bootstrap_cache_dir, pkgpanda.util.variant_prefix(variant)) output_name = bootstrap_cache_dir + '/' + bootstrap_id + '.' # bootstrap tarball = <sha1 of packages in tarball>.bootstrap.tar.xz bootstrap_name = "{}bootstrap.tar.xz".format(output_name) active_name = "{}active.json".format(output_name) def mark_latest(): # Ensure latest is always written write_string(latest_name, bootstrap_id) print("bootstrap: {}".format(bootstrap_name)) print("active: {}".format(active_name)) print("latest: {}".format(latest_name)) return bootstrap_id if (os.path.exists(bootstrap_name)): print("Bootstrap already up to date, not recreating") return mark_latest() check_call(['mkdir', '-p', bootstrap_cache_dir]) # Try downloading. if package_store.try_fetch_bootstrap_and_active(bootstrap_id): print("Bootstrap already up to date, Not recreating. Downloaded from repository-url.") return mark_latest() print("Unable to download from cache. Building.") print("Creating bootstrap tarball for variant {}".format(variant)) work_dir = tempfile.mkdtemp(prefix='mkpanda_bootstrap_tmp') def make_abs(path): return os.path.join(work_dir, path) pkgpanda_root = make_abs("opt/mesosphere") repository = Repository(os.path.join(pkgpanda_root, "packages")) # Fetch all the packages to the root for pkg_path in packages: filename = os.path.basename(pkg_path) pkg_id = filename[:-len(".tar.xz")] def local_fetcher(id, target): shutil.unpack_archive(pkg_path, target, "gztar") repository.add(local_fetcher, pkg_id, False) # Activate the packages inside the repository. # Do generate dcos.target.wants inside the root so that we don't # try messing with /etc/systemd/system. install = Install( root=pkgpanda_root, config_dir=None, rooted_systemd=True, manage_systemd=False, block_systemd=True, fake_path=True, skip_systemd_dirs=True, manage_users=False, manage_state_dir=False) install.activate(repository.load_packages(pkg_ids)) # Mark the tarball as a bootstrap tarball/filesystem so that # dcos-setup.service will fire. make_file(make_abs("opt/mesosphere/bootstrap")) # Write out an active.json for the bootstrap tarball write_json(active_name, pkg_ids) # Rewrite all the symlinks to point to /opt/mesosphere rewrite_symlinks(work_dir, work_dir, "/") make_tar(bootstrap_name, pkgpanda_root) shutil.rmtree(work_dir) # Update latest last so that we don't ever use partially-built things. write_string(latest_name, bootstrap_id) print("Built bootstrap") return mark_latest()
def integration_test( tunnel, test_dir, dcos_dns, master_list, agent_list, public_agent_list, test_dns_search, provider, ci_flags, timeout=None, aws_access_key_id='', aws_secret_access_key='', region='', add_env=None): """Runs integration test on host Args: test_dir: string representing host where integration_test.py exists on test_host dcos_dns: string representing IP of DCOS DNS host master_list: string of comma separated master addresses agent_list: string of comma separated agent addresses test_dns_search: if set to True, test for deployed mesos DNS app ci_flags: optional additional string to be passed to test provider: (str) either onprem, aws, or azure # The following variables correspond to currently disabled tests aws_access_key_id: needed for REXRAY tests aws_secret_access_key: needed for REXRAY tests region: string indicating AWS region in which cluster is running add_env: a python dict with any number of key=value assignments to be passed to the test environment """ test_script = pkg_filename('integration_test.py') pytest_docker = pkg_filename('docker/py.test/Dockerfile') dns_search = 'true' if test_dns_search else 'false' test_env = [ 'DCOS_DNS_ADDRESS=http://'+dcos_dns, 'MASTER_HOSTS='+','.join(master_list), 'PUBLIC_MASTER_HOSTS='+','.join(master_list), 'SLAVE_HOSTS='+','.join(agent_list), 'PUBLIC_SLAVE_HOSTS='+','.join(public_agent_list), 'REGISTRY_HOST=127.0.0.1', 'DCOS_PROVIDER='+provider, 'DNS_SEARCH='+dns_search, 'AWS_ACCESS_KEY_ID='+aws_access_key_id, 'AWS_SECRET_ACCESS_KEY='+aws_secret_access_key, 'AWS_REGION='+region] if add_env: for key, value in add_env.items(): extra_env = key + '=' + value test_env.append(extra_env) test_env = ['export '+e+'\n' for e in test_env] test_env = ''.join(test_env) test_cmd = 'py.test -vv ' + ci_flags + ' /integration_test.py' log.info('Building py.test image') # Make a clean docker context temp_dir = tempfile.mkdtemp() cmd_script = """ #!/bin/bash set -euo pipefail; set -x {test_env} {test_cmd} """.format(test_env=test_env, test_cmd=test_cmd) write_string(join(temp_dir, 'test_wrapper.sh'), cmd_script) check_call(['cp', test_script, join(temp_dir, 'integration_test.py')]) check_call(['cp', pytest_docker, join(temp_dir, 'Dockerfile')]) check_call(['docker', 'build', '-t', 'py.test', temp_dir]) log.info('Exporting py.test image') pytest_image_tar = 'DCOS_integration_test.tar' check_call(['docker', 'save', '-o', join(temp_dir, pytest_image_tar), 'py.test']) log.info('Transferring py.test image') tunnel.write_to_remote(join(temp_dir, pytest_image_tar), join(test_dir, pytest_image_tar)) log.info('Loading py.test image on remote host') tunnel.remote_cmd(['docker', 'load', '-i', join(test_dir, pytest_image_tar)]) test_container_name = 'int_test_' + str(int(time.time())) docker_cmd = ['docker', 'run', '--net=host', '--name='+test_container_name, 'py.test'] try: with remote_port_forwarding(tunnel, agent_list+public_agent_list, join(test_dir, 'ssh_key')): log.info('Running integration test...') try: tunnel.remote_cmd(docker_cmd, timeout=timeout) except CalledProcessError: log.exception('Test failed!') if ci_flags: return False raise log.info('Successful test run!') except TimeoutExpired: log.error('Test failed due to timing out after {} seconds'.format(timeout)) raise finally: get_logs_cmd = ['docker', 'logs', test_container_name] test_log = tunnel.remote_cmd(get_logs_cmd) log_file = 'integration_test.log' with open(log_file, 'wb') as fh: fh.write(test_log) log.info('Logs from test container can be found in '+log_file) return True
def integration_test( tunnel, test_dir, dcos_dns, master_list, agent_list, public_agent_list, variant, test_dns_search, provider, ci_flags, timeout=None, aws_access_key_id='', aws_secret_access_key='', region='', add_env=None): """Runs integration test on host Args: test_dir: string representing host where integration_test.py exists on test_host dcos_dns: string representing IP of DCOS DNS host master_list: string of comma separated master addresses agent_list: string of comma separated agent addresses variant: 'ee' or 'default' test_dns_search: if set to True, test for deployed mesos DNS app ci_flags: optional additional string to be passed to test provider: (str) either onprem, aws, or azure # The following variables correspond to currently disabled tests aws_access_key_id: needed for REXRAY tests aws_secret_access_key: needed for REXRAY tests region: string indicating AWS region in which cluster is running add_env: a python dict with any number of key=value assignments to be passed to the test environment """ test_script = pkg_filename('integration_test.py') pytest_docker = pkg_filename('docker/py.test/Dockerfile') dns_search = 'true' if test_dns_search else 'false' test_env = [ 'DCOS_DNS_ADDRESS=http://'+dcos_dns, 'MASTER_HOSTS='+','.join(master_list), 'PUBLIC_MASTER_HOSTS='+','.join(master_list), 'SLAVE_HOSTS='+','.join(agent_list), 'PUBLIC_SLAVE_HOSTS='+','.join(public_agent_list), 'REGISTRY_HOST=127.0.0.1', 'DCOS_VARIANT='+variant, 'DCOS_PROVIDER='+provider, 'DNS_SEARCH='+dns_search, 'AWS_ACCESS_KEY_ID='+aws_access_key_id, 'AWS_SECRET_ACCESS_KEY='+aws_secret_access_key, 'AWS_REGION='+region] if add_env: for key, value in add_env.items(): extra_env = key + '=' + value test_env.append(extra_env) test_env = ['export '+e+'\n' for e in test_env] test_env = ''.join(test_env) test_cmd = 'py.test -vv ' + ci_flags + ' /integration_test.py' log.info('Building py.test image') # Make a clean docker context temp_dir = tempfile.mkdtemp() cmd_script = """ #!/bin/bash set -euo pipefail; set -x {test_env} {test_cmd} """.format(test_env=test_env, test_cmd=test_cmd) write_string(join(temp_dir, 'test_wrapper.sh'), cmd_script) check_call(['cp', test_script, join(temp_dir, 'integration_test.py')]) check_call(['cp', pytest_docker, join(temp_dir, 'Dockerfile')]) check_call(['docker', 'build', '-t', 'py.test', temp_dir]) log.info('Exporting py.test image') pytest_image_tar = 'DCOS_integration_test.tar' check_call(['docker', 'save', '-o', join(temp_dir, pytest_image_tar), 'py.test']) log.info('Transferring py.test image') tunnel.write_to_remote(join(temp_dir, pytest_image_tar), join(test_dir, pytest_image_tar)) log.info('Loading py.test image on remote host') tunnel.remote_cmd(['docker', 'load', '-i', join(test_dir, pytest_image_tar)]) test_container_name = 'int_test_' + str(int(time.time())) docker_cmd = ['docker', 'run', '--net=host', '--name='+test_container_name, 'py.test'] try: with remote_port_forwarding(tunnel, agent_list+public_agent_list, join(test_dir, 'ssh_key')): log.info('Running integration test...') try: tunnel.remote_cmd(docker_cmd, timeout=timeout) except CalledProcessError: log.exception('Test failed!') if ci_flags: return False raise log.info('Successful test run!') except TimeoutExpired: log.error('Test failed due to timing out after {} seconds'.format(timeout)) raise finally: get_logs_cmd = ['docker', 'logs', test_container_name] test_log = tunnel.remote_cmd(get_logs_cmd) log_file = 'integration_test.log' with open(log_file, 'wb') as fh: fh.write(test_log) log.info('Logs from test container can be found in '+log_file) return True
def main(): options = check_environment() unique_cluster_id = "dcos-ci-test-onprem-{}".format(random_id(10)) log.info("Spinning up AWS VPC with ID: {}".format(unique_cluster_id)) if options.test_install_prereqs: os_name = "cent-os-7" else: os_name = "cent-os-7-dcos-prereqs" bw = test_util.aws.BotoWrapper( region=DEFAULT_AWS_REGION, aws_access_key_id=options.aws_access_key_id, aws_secret_access_key=options.aws_secret_access_key) ssh_key = bw.create_key_pair(unique_cluster_id) # Drop the key to disk so CI can cache it as an artifact write_string('ssh_key', ssh_key) if options.cluster_ami: vpc = test_util.aws.VpcCfStack.create_from_ami( stack_name=unique_cluster_id, instance_type=options.instance_type, instance_ami=options.cluster_ami, # An instance for each cluster node plus the bootstrap. instance_count=(options.masters + options.agents + options.public_agents + 1), admin_location='0.0.0.0/0', key_pair_name=unique_cluster_id, boto_wrapper=bw) ssh_info = SshInfo(user=options.cluster_ami_ssh_user, home_dir=options.cluster_ami_ssh_home_dir) else: vpc, ssh_info = test_util.aws.VpcCfStack.create( stack_name=unique_cluster_id, instance_type=options.instance_type, instance_os=os_name, # An instance for each cluster node plus the bootstrap. instance_count=(options.masters + options.agents + options.public_agents + 1), admin_location='0.0.0.0/0', key_pair_name=unique_cluster_id, boto_wrapper=bw) vpc.wait_for_complete() cluster = test_util.cluster.Cluster.from_vpc( vpc, ssh_info, ssh_key=ssh_key, num_masters=options.masters, num_agents=options.agents, num_public_agents=options.public_agents, ) test_util.cluster.install_dcos( cluster, installer_url=options.installer_url, setup=options.do_setup, api=options.use_api, add_config_path=options.add_config_path, # If we don't want to test the prereq install, use offline mode to avoid it. installer_api_offline_mode=(not options.test_install_prereqs), install_prereqs=options.test_install_prereqs, install_prereqs_only=options.test_install_prereqs_only, ) if options.test_install_prereqs and options.test_install_prereqs_only: # install_dcos() exited after running prereqs, so we're done. vpc.delete() bw.delete_key_pair(unique_cluster_id) sys.exit(0) result = test_util.cluster.run_integration_tests( cluster, # Setting dns_search: mesos not currently supported in API region=DEFAULT_AWS_REGION, aws_access_key_id=options.aws_access_key_id, aws_secret_access_key=options.aws_secret_access_key, test_cmd=options.test_cmd, ) if result == 0: log.info("Test successful! Deleting VPC if provided in this run.") vpc.delete() bw.delete_key_pair(unique_cluster_id) else: log.info("Test failed! VPC will remain for debugging 1 hour from instantiation") if options.ci_flags: result = 0 # Wipe the return code so that tests can be muted in CI sys.exit(result)
def activate(self, packages): # Ensure the new set is reasonable. validate_compatible(packages, self.__roles) # Build the absolute paths for the running config, new config location, # and where to archive the config. active_names = self.get_active_names() active_dirs = list(map(self._make_abs, self.__well_known_dirs + ["active"])) new_names = [name + ".new" for name in active_names] new_dirs = [name + ".new" for name in active_dirs] old_names = [name + ".old" for name in active_names] # Remove all pre-existing new and old directories for name in chain(new_names, old_names): if (os.path.exists(name)): if os.path.isdir(name): shutil.rmtree(name) else: os.remove(name) # Make the directories for the new config for name in new_dirs: os.makedirs(name) # Fill in all the new contents def symlink_all(src, dest): if not os.path.isdir(src): return symlink_tree(src, dest) # Set the new LD_LIBRARY_PATH, PATH. env_contents = env_header.format("/opt/mesosphere" if self.__fake_path else self.__root) env_export_contents = env_export_header.format("/opt/mesosphere" if self.__fake_path else self.__root) active_buildinfo_full = {} # Add the folders, config in each package. for package in packages: # Package folders # NOTE: Since active is at the end of the folder list it will be # removed by the zip. This is the desired behavior, since it will be # populated later. # Do the basename since some well known dirs are full paths (dcos.target.wants) # while inside the packages they are always top level directories. for new, dir_name in zip(new_dirs, self.__well_known_dirs): dir_name = os.path.basename(dir_name) pkg_dir = os.path.join(package.path, dir_name) assert os.path.isabs(new) assert os.path.isabs(pkg_dir) try: symlink_all(pkg_dir, new) # Symlink all applicable role-based config for role in self.__roles: role_dir = os.path.join(package.path, "{0}_{1}".format(dir_name, role)) symlink_all(role_dir, new) except ConflictingFile as ex: raise ValidationError("Two packages are trying to install the same file {0} or " "two roles in the set of roles {1} are causing a package " "to try activating multiple versions of the same file. " "One of the package files is {2}.".format( ex.dest, self.__roles, ex.src)) # Add to the active folder os.symlink(package.path, os.path.join(self._make_abs("active.new"), package.name)) # Add to the environment contents env_contents += "# package: {0}\n".format(package.id) for k, v in package.environment.items(): env_contents += "{0}={1}\n".format(k, v) env_contents += "\n" # Add to the environment.export contents env_export_contents += "# package: {0}\n".format(package.id) for k, v in package.environment.items(): env_export_contents += "export {0}={1}\n".format(k, v) env_export_contents += "\n" # Add to the buildinfo try: active_buildinfo_full[package.name] = load_json(os.path.join(package.path, "buildinfo.full.json")) except FileNotFoundError: # TODO(cmaloney): These only come from setup-packages. Should update # setup-packages to add a buildinfo.full for those packages active_buildinfo_full[package.name] = None # Write out the new environment file. new_env = self._make_abs("environment.new") write_string(new_env, env_contents) # Write out the new environment.export file new_env_export = self._make_abs("environment.export.new") write_string(new_env_export, env_export_contents) # Write out the buildinfo of every active package new_buildinfo_meta = self._make_abs("active.buildinfo.full.json.new") write_json(new_buildinfo_meta, active_buildinfo_full) self.swap_active(".new")
def build(variant, package_dir, name, repository_url, clean_after_build): print("Building package {} variant {}".format(name, variant or "<default>")) tmpdir = tempfile.TemporaryDirectory(prefix="pkgpanda_repo") repository = Repository(tmpdir.name) def pkg_abs(name): return package_dir + '/' + name # Build pkginfo over time, translating fields from buildinfo. pkginfo = {} # Build up the docker command arguments over time, translating fields as needed. cmd = DockerCmd() buildinfo = load_buildinfo(package_dir, variant) if 'name' in buildinfo: raise BuildError("'name' is not allowed in buildinfo.json, it is implicitly the name of the " "folder containing the buildinfo.json") # Make sure build_script is only set on variants if 'build_script' in buildinfo and variant is None: raise BuildError("build_script can only be set on package variants") # Convert single_source -> sources try: sources = expand_single_source_alias(name, buildinfo) except ValidationError as ex: raise BuildError("Invalid buildinfo.json for package: {}".format(ex)) from ex # Save the final sources back into buildinfo so it gets written into # buildinfo.json. This also means buildinfo.json is always expanded form. buildinfo['sources'] = sources # Construct the source fetchers, gather the checkout ids from them checkout_ids = dict() fetchers = dict() try: for src_name, src_info in sorted(sources.items()): if src_info['kind'] not in pkgpanda.build.src_fetchers.all_fetchers: raise ValidationError("No known way to catch src with kind '{}'. Known kinds: {}".format( src_info['kind'], pkgpanda.src_fetchers.all_fetchers.keys())) cache_dir = pkg_abs("cache") if not os.path.exists(cache_dir): os.mkdir(cache_dir) fetchers[src_name] = pkgpanda.build.src_fetchers.all_fetchers[src_info['kind']](src_name, src_info, package_dir) checkout_ids[src_name] = fetchers[src_name].get_id() except ValidationError as ex: raise BuildError("Validation error when fetching sources for package: {}".format(ex)) for src_name, checkout_id in checkout_ids.items(): # NOTE: single_source buildinfo was expanded above so the src_name is # always correct here. # Make sure we never accidentally overwrite something which might be # important. Fields should match if specified (And that should be # tested at some point). For now disallowing identical saves hassle. assert_no_duplicate_keys(checkout_id, buildinfo['sources'][src_name]) buildinfo['sources'][src_name].update(checkout_id) # Add the sha1sum of the buildinfo.json + build file to the build ids build_ids = {"sources": checkout_ids} build_ids['build'] = pkgpanda.util.sha1(pkg_abs("build")) build_ids['pkgpanda_version'] = pkgpanda.build.constants.version build_ids['variant'] = '' if variant is None else variant extra_dir = pkg_abs("extra") # Add the "extra" folder inside the package as an additional source if it # exists if os.path.exists(extra_dir): extra_id = hash_folder(extra_dir) build_ids['extra_source'] = extra_id buildinfo['extra_source'] = extra_id # Figure out the docker name. docker_name = buildinfo.get('docker', 'dcos-builder:latest') cmd.container = docker_name # Add the id of the docker build environment to the build_ids. try: docker_id = get_docker_id(docker_name) except CalledProcessError: # docker pull the container and try again check_call(['docker', 'pull', docker_name]) docker_id = get_docker_id(docker_name) build_ids['docker'] = docker_id # TODO(cmaloney): The environment variables should be generated during build # not live in buildinfo.json. build_ids['environment'] = buildinfo.get('environment', {}) # Packages need directories inside the fake install root (otherwise docker # will try making the directories on a readonly filesystem), so build the # install root now, and make the package directories in it as we go. install_dir = tempfile.mkdtemp(prefix="pkgpanda-") active_packages = list() active_package_ids = set() active_package_variants = dict() auto_deps = set() # Verify all requires are in the repository. if 'requires' in buildinfo: # Final package has the same requires as the build. pkginfo['requires'] = buildinfo['requires'] # TODO(cmaloney): Pull generating the full set of requires a function. to_check = copy.deepcopy(buildinfo['requires']) if type(to_check) != list: raise BuildError("`requires` in buildinfo.json must be an array of dependencies.") while to_check: requires_info = to_check.pop(0) requires_name, requires_variant = expand_require(requires_info) if requires_name in active_package_variants: # TODO(cmaloney): If one package depends on the <default> # variant of a package and 1+ others depends on a non-<default> # variant then update the dependency to the non-default variant # rather than erroring. if requires_variant != active_package_variants[requires_name]: # TODO(cmaloney): Make this contain the chains of # dependencies which contain the conflicting packages. # a -> b -> c -> d {foo} # e {bar} -> d {baz} raise BuildError("Dependncy on multiple variants of the same package {}. " "variants: {} {}".format( requires_name, requires_variant, active_package_variants[requires_name])) # The variant has package {requires_name, variant} already is a # dependency, don't process it again / move on to the next. continue active_package_variants[requires_name] = requires_variant # Figure out the last build of the dependency, add that as the # fully expanded dependency. require_package_dir = os.path.normpath(pkg_abs('../' + requires_name)) last_build = require_package_dir + '/' + last_build_filename(requires_variant) if not os.path.exists(last_build): raise BuildError("No last build file found for dependency {} variant {}. Rebuild " "the dependency".format(requires_name, requires_variant)) try: pkg_id_str = load_string(last_build) auto_deps.add(pkg_id_str) pkg_buildinfo = load_buildinfo(require_package_dir, requires_variant) pkg_requires = pkg_buildinfo.get('requires', list()) pkg_path = repository.package_path(pkg_id_str) pkg_tar = pkg_id_str + '.tar.xz' if not os.path.exists(require_package_dir + '/' + pkg_tar): raise BuildError("The build tarball {} refered to by the last_build file of the " "dependency {} variant {} doesn't exist. Rebuild the dependency.".format( pkg_tar, requires_name, requires_variant)) active_package_ids.add(pkg_id_str) # Mount the package into the docker container. cmd.volumes[pkg_path] = "/opt/mesosphere/packages/{}:ro".format(pkg_id_str) os.makedirs(os.path.join(install_dir, "packages/{}".format(pkg_id_str))) # Add the dependencies of the package to the set which will be # activated. # TODO(cmaloney): All these 'transitive' dependencies shouldn't # be available to the package being built, only what depends on # them directly. to_check += pkg_requires except ValidationError as ex: raise BuildError("validating package needed as dependency {0}: {1}".format(requires_name, ex)) from ex except PackageError as ex: raise BuildError("loading package needed as dependency {0}: {1}".format(requires_name, ex)) from ex # Add requires to the package id, calculate the final package id. # NOTE: active_packages isn't fully constructed here since we lazily load # packages not already in the repository. build_ids['requires'] = list(active_package_ids) version_base = hash_checkout(build_ids) version = None if "version_extra" in buildinfo: version = "{0}-{1}".format(buildinfo["version_extra"], version_base) else: version = version_base pkg_id = PackageId.from_parts(name, version) # Save the build_ids. Useful for verify exactly what went into the # package build hash. buildinfo['build_ids'] = build_ids buildinfo['package_version'] = version # Save the package name and variant. The variant is used when installing # packages to validate dependencies. buildinfo['name'] = name buildinfo['variant'] = variant # If the package is already built, don't do anything. pkg_path = pkg_abs("{}.tar.xz".format(pkg_id)) # Done if it exists locally if exists(pkg_path): print("Package up to date. Not re-building.") # TODO(cmaloney): Updating / filling last_build should be moved out of # the build function. check_call(["mkdir", "-p", pkg_abs("cache")]) write_string(pkg_abs(last_build_filename(variant)), str(pkg_id)) return pkg_path # Try downloading. if repository_url: tmp_filename = pkg_path + '.tmp' try: # Normalize to no trailing slash for repository_url repository_url = repository_url.rstrip('/') url = repository_url + '/packages/{0}/{1}.tar.xz'.format(pkg_id.name, str(pkg_id)) print("Attempting to download", pkg_id, "from", url) download(tmp_filename, url, package_dir) os.rename(tmp_filename, pkg_path) print("Package up to date. Not re-building. Downloaded from repository-url.") # TODO(cmaloney): Updating / filling last_build should be moved out of # the build function. check_call(["mkdir", "-p", pkg_abs("cache")]) write_string(pkg_abs(last_build_filename(variant)), str(pkg_id)) return pkg_path except FetchError: try: os.remove(tmp_filename) except: pass # Fall out and do the build since the command errored. print("Unable to download from cache. Proceeding to build") print("Building package {} with buildinfo: {}".format( pkg_id, json.dumps(buildinfo, indent=2, sort_keys=True))) # Clean out src, result so later steps can use them freely for building. clean(package_dir) # Only fresh builds are allowed which don't overlap existing artifacts. result_dir = pkg_abs("result") if exists(result_dir): raise BuildError("result folder must not exist. It will be made when the package is " "built. {}".format(result_dir)) # 'mkpanda add' all implicit dependencies since we actually need to build. for dep in auto_deps: print("Auto-adding dependency: {}".format(dep)) # NOTE: Not using the name pkg_id because that overrides the outer one. id_obj = PackageId(dep) add_to_repository(repository, pkg_abs('../{0}/{1}.tar.xz'.format(id_obj.name, dep))) package = repository.load(dep) active_packages.append(package) # Checkout all the sources int their respective 'src/' folders. try: src_dir = pkg_abs('src') if os.path.exists(src_dir): raise ValidationError( "'src' directory already exists, did you have a previous build? " + "Currently all builds must be from scratch. Support should be " + "added for re-using a src directory when possible. src={}".format(src_dir)) os.mkdir(src_dir) for src_name, fetcher in sorted(fetchers.items()): root = pkg_abs('src/' + src_name) os.mkdir(root) fetcher.checkout_to(root) except ValidationError as ex: raise BuildError("Validation error when fetching sources for package: {}".format(ex)) # Copy over environment settings if 'environment' in buildinfo: pkginfo['environment'] = buildinfo['environment'] # Activate the packages so that we have a proper path, environment # variables. # TODO(cmaloney): RAII type thing for temproary directory so if we # don't get all the way through things will be cleaned up? install = Install(install_dir, None, True, False, True, True) install.activate(active_packages) # Rewrite all the symlinks inside the active path because we will # be mounting the folder into a docker container, and the absolute # paths to the packages will change. # TODO(cmaloney): This isn't very clean, it would be much nicer to # just run pkgpanda inside the package. rewrite_symlinks(install_dir, repository.path, "/opt/mesosphere/packages/") print("Building package in docker") # TODO(cmaloney): Run as a specific non-root user, make it possible # for non-root to cleanup afterwards. # Run the build, prepping the environment as necessary. mkdir(pkg_abs("result")) # Copy the build info to the resulting tarball write_json(pkg_abs("src/buildinfo.full.json"), buildinfo) write_json(pkg_abs("result/buildinfo.full.json"), buildinfo) write_json(pkg_abs("result/pkginfo.json"), pkginfo) # Make the folder for the package we are building. If docker does it, it # gets auto-created with root permissions and we can't actually delete it. os.makedirs(os.path.join(install_dir, "packages", str(pkg_id))) # TOOD(cmaloney): Disallow writing to well known files and directories? # Source we checked out cmd.volumes.update({ # TODO(cmaloney): src should be read only... pkg_abs("src"): "/pkg/src:rw", # The build script pkg_abs(buildinfo.get('build_script', 'build')): "/pkg/build:ro", # Getting the result out pkg_abs("result"): "/opt/mesosphere/packages/{}:rw".format(pkg_id), install_dir: "/opt/mesosphere:ro" }) if os.path.exists(extra_dir): cmd.volumes[extra_dir] = "/pkg/extra:ro" cmd.environment = { "PKG_VERSION": version, "PKG_NAME": name, "PKG_ID": pkg_id, "PKG_PATH": "/opt/mesosphere/packages/{}".format(pkg_id), "PKG_VARIANT": variant if variant is not None else "<default>" } try: # TODO(cmaloney): Run a wrapper which sources # /opt/mesosphere/environment then runs a build. Also should fix # ownership of /opt/mesosphere/packages/{pkg_id} post build. cmd.run([ "/bin/bash", "-o", "nounset", "-o", "pipefail", "-o", "errexit", "/pkg/build"]) except CalledProcessError as ex: raise BuildError("docker exited non-zero: {}\nCommand: {}".format(ex.returncode, ' '.join(ex.cmd))) # Clean up the temporary install dir used for dependencies. # TODO(cmaloney): Move to an RAII wrapper. check_call(['rm', '-rf', install_dir]) print("Building package tarball") # Check for forbidden services before packaging the tarball: try: check_forbidden_services(pkg_abs("result"), RESERVED_UNIT_NAMES) except ValidationError as ex: raise BuildError("Package validation failed: {}".format(ex)) # TODO(cmaloney): Updating / filling last_build should be moved out of # the build function. check_call(["mkdir", "-p", pkg_abs("cache")]) write_string(pkg_abs(last_build_filename(variant)), str(pkg_id)) # Bundle the artifacts into the pkgpanda package tmp_name = pkg_path + "-tmp.tar.xz" make_tar(tmp_name, pkg_abs("result")) os.rename(tmp_name, pkg_path) print("Package built.") if clean_after_build: clean(package_dir) return pkg_path
def make_default_config_if_needed(config_path): if os.path.exists(config_path): return write_string(config_path, config_sample)
def write_config(self): assert self.config_path is not None write_string(self.config_path, self.get_yaml_str())
def integration_test( tunnel, test_dir, dcos_dns, master_list, agent_list, public_agent_list, provider, test_dns_search=True, aws_access_key_id='', aws_secret_access_key='', region='', add_env=None, pytest_cmd='py.test -rs -vv'): """Runs integration test on host Args: test_dir: directory to leave test_wrapper.sh dcos_dns: string representing IP of DCOS DNS host master_list: string of comma separated master addresses agent_list: string of comma separated agent addresses test_dns_search: if set to True, test for deployed mesos DNS app provider: (str) either onprem, aws, or azure Optional args: aws_access_key_id: needed for REXRAY tests aws_secret_access_key: needed for REXRAY tests region: string indicating AWS region in which cluster is running add_env: a python dict with any number of key=value assignments to be passed to the test environment pytest_cmd: string representing command for py.test Returns: exit code corresponding to test_cmd run """ dns_search = 'true' if test_dns_search else 'false' test_env = [ 'DCOS_DNS_ADDRESS=http://' + dcos_dns, 'MASTER_HOSTS=' + ','.join(master_list), 'PUBLIC_MASTER_HOSTS=' + ','.join(master_list), 'SLAVE_HOSTS=' + ','.join(agent_list), 'PUBLIC_SLAVE_HOSTS=' + ','.join(public_agent_list), 'DCOS_PROVIDER=' + provider, 'DNS_SEARCH=' + dns_search, 'AWS_ACCESS_KEY_ID=' + aws_access_key_id, 'AWS_SECRET_ACCESS_KEY=' + aws_secret_access_key, 'AWS_REGION=' + region] if add_env: for key, value in add_env.items(): extra_env = key + '=' + value test_env.append(extra_env) test_env_str = ''.join(['export ' + e + '\n' for e in test_env]) test_boilerplate = """#!/bin/bash {env} cd /opt/mesosphere/active/dcos-integration-test /opt/mesosphere/bin/dcos-shell {cmd} """ write_string('test_preflight.sh', test_boilerplate.format( env=test_env_str, cmd='py.test -rs -vv --collect-only')) write_string('test_wrapper.sh', test_boilerplate.format( env=test_env_str, cmd=pytest_cmd)) pretest_path = join(test_dir, 'test_preflight.sh') log.info('Running integration test setup check...') tunnel.write_to_remote('test_preflight.sh', pretest_path) tunnel.remote_cmd(['bash', pretest_path], stdout=sys.stdout.buffer) wrapper_path = join(test_dir, 'test_wrapper.sh') log.info('Running integration test...') tunnel.write_to_remote('test_wrapper.sh', wrapper_path) try: tunnel.remote_cmd(['bash', wrapper_path], stdout=sys.stdout.buffer) except CalledProcessError as e: return e.returncode return 0
def run_test(self) -> int: stack_name = 'dcos-ci-test-upgrade-' + random_id(10) test_id = uuid.uuid4().hex healthcheck_app_id = TEST_APP_NAME_FMT.format('healthcheck-' + test_id) dns_app_id = TEST_APP_NAME_FMT.format('dns-' + test_id) with logger.scope("create vpc cf stack '{}'".format(stack_name)): bw = test_util.aws.BotoWrapper( region=self.aws_region, aws_access_key_id=self.aws_access_key_id, aws_secret_access_key=self.aws_secret_access_key) ssh_key = bw.create_key_pair(stack_name) write_string('ssh_key', ssh_key) vpc, ssh_info = test_util.aws.VpcCfStack.create( stack_name=stack_name, instance_type='m4.xlarge', instance_os='cent-os-7-dcos-prereqs', # An instance for each cluster node plus the bootstrap. instance_count=(self.num_masters + self.num_agents + self.num_public_agents + 1), admin_location='0.0.0.0/0', key_pair_name=stack_name, boto_wrapper=bw ) vpc.wait_for_complete() cluster = test_util.cluster.Cluster.from_vpc( vpc, ssh_info, ssh_key=ssh_key, num_masters=self.num_masters, num_agents=self.num_agents, num_public_agents=self.num_public_agents, ) with logger.scope("install dcos"): # Use the CLI installer to set exhibitor_storage_backend = zookeeper. test_util.cluster.install_dcos(cluster, self.stable_installer_url, api=False, add_config_path=self.config_yaml_override_install) master_list = [h.private_ip for h in cluster.masters] dcos_api_install = self.dcos_api_session_factory_install.apply( 'http://{ip}'.format(ip=cluster.masters[0].public_ip), master_list, master_list, [h.private_ip for h in cluster.agents], [h.private_ip for h in cluster.public_agents], self.default_os_user) dcos_api_install.wait_for_dcos() installed_version = dcos_api_install.get_version() healthcheck_app = create_marathon_healthcheck_app(healthcheck_app_id) dns_app = create_marathon_dns_app(dns_app_id, healthcheck_app_id) self.setup_cluster_workload(dcos_api_install, healthcheck_app, dns_app) with logger.scope("upgrade cluster"): test_util.cluster.upgrade_dcos(cluster, self.installer_url, installed_version, add_config_path=self.config_yaml_override_upgrade) with cluster.ssher.tunnel(cluster.bootstrap_host) as bootstrap_host_tunnel: bootstrap_host_tunnel.remote_cmd(['sudo', 'rm', '-rf', cluster.ssher.home_dir + '/*']) # this method invocation looks like it is the same as the one above, and that is partially correct. # the arguments to the invocation are the same, but the thing that changes is the lifecycle of the cluster # the client is being created to interact with. This client is specifically for the cluster after the # upgrade has taken place, and can account for any possible settings that may change for the client under # the hood when it probes the cluster. dcos_api_upgrade = self.dcos_api_session_factory_upgrade.apply( 'http://{ip}'.format(ip=cluster.masters[0].public_ip), master_list, master_list, [h.private_ip for h in cluster.agents], [h.private_ip for h in cluster.public_agents], self.default_os_user) dcos_api_upgrade.wait_for_dcos() # here we wait for DC/OS to be "up" so that we can auth this new client self.verify_apps_state(dcos_api_upgrade, dns_app) with logger.scope("run integration tests"): # copied from test_util/test_aws_cf.py:96 add_env = [] prefix = 'TEST_ADD_ENV_' for k, v in os.environ.items(): if k.startswith(prefix): add_env.append(k.replace(prefix, '') + '=' + v) test_cmd = ' '.join(add_env) + ' py.test -vv -s -rs ' + os.getenv('CI_FLAGS', '') result = test_util.cluster.run_integration_tests(cluster, test_cmd=test_cmd) if result == 0: self.log.info("Test successful! Deleting VPC if provided in this run.") vpc.delete() bw.delete_key_pair(stack_name) else: self.log.info("Test failed! VPC cluster will remain available for " "debugging for 2 hour after instantiation.") return result
def generate( arguments, extra_templates=list(), extra_sources=list(), extra_targets=list()): # To maintain the old API where we passed arguments rather than the new name. user_arguments = arguments arguments = None sources, targets, templates = get_dcosconfig_source_target_and_templates( user_arguments, extra_templates, extra_sources) resolver = validate_and_raise(sources, targets + extra_targets) argument_dict = get_final_arguments(resolver) late_variables = get_late_variables(resolver, sources) secret_builtins = ['expanded_config_full', 'user_arguments_full', 'config_yaml_full'] secret_variables = set(get_secret_variables(sources) + secret_builtins) masked_value = '**HIDDEN**' # Calculate values for builtin variables. user_arguments_masked = {k: (masked_value if k in secret_variables else v) for k, v in user_arguments.items()} argument_dict['user_arguments_full'] = json_prettyprint(user_arguments) argument_dict['user_arguments'] = json_prettyprint(user_arguments_masked) argument_dict['config_yaml_full'] = user_arguments_to_yaml(user_arguments) argument_dict['config_yaml'] = user_arguments_to_yaml(user_arguments_masked) # The expanded_config and expanded_config_full variables contain all other variables and their values. # expanded_config is a copy of expanded_config_full with secret values removed. Calculating these variables' values # must come after the calculation of all other variables to prevent infinite recursion. # TODO(cmaloney): Make this late-bound by gen.internals expanded_config_full = { k: v for k, v in argument_dict.items() # Omit late-bound variables whose values have not yet been calculated. if not v.startswith(gen.internals.LATE_BIND_PLACEHOLDER_START) } expanded_config_scrubbed = {k: v for k, v in expanded_config_full.items() if k not in secret_variables} argument_dict['expanded_config_full'] = format_expanded_config(expanded_config_full) argument_dict['expanded_config'] = format_expanded_config(expanded_config_scrubbed) log.debug( "Final arguments:" + json_prettyprint({ # Mask secret config values. k: (masked_value if k in secret_variables else v) for k, v in argument_dict.items() }) ) # Fill in the template parameters # TODO(cmaloney): render_templates should ideally take the template targets. rendered_templates = render_templates(templates, argument_dict) # Validate there aren't any unexpected top level directives in any of the files # (likely indicates a misspelling) for name, template in rendered_templates.items(): if name == 'dcos-services.yaml': # yaml list of the service files assert isinstance(template, list) elif name == 'cloud-config.yaml': assert template.keys() <= CLOUDCONFIG_KEYS, template.keys() elif isinstance(template, str): # Not a yaml template pass else: # yaml template file log.debug("validating template file %s", name) assert template.keys() <= PACKAGE_KEYS, template.keys() stable_artifacts = [] channel_artifacts = [] # Find all files which contain late bind variables and turn them into a "late bind package" # TODO(cmaloney): check there are no late bound variables in cloud-config.yaml late_files, regular_files = extract_files_containing_late_variables( rendered_templates['dcos-config.yaml']['package']) # put the regular files right back rendered_templates['dcos-config.yaml'] = {'package': regular_files} # Render cluster package list artifact. cluster_package_list_filename = 'package_lists/{}.package_list.json'.format( argument_dict['cluster_package_list_id'] ) os.makedirs(os.path.dirname(cluster_package_list_filename), mode=0o755, exist_ok=True) write_string(cluster_package_list_filename, argument_dict['cluster_packages']) log.info('Cluster package list: {}'.format(cluster_package_list_filename)) stable_artifacts.append(cluster_package_list_filename) def make_package_filename(package_id, extension): return 'packages/{0}/{1}{2}'.format( package_id.name, repr(package_id), extension) # Render all the cluster packages cluster_package_info = {} # Prepare late binding config, if any. late_package = build_late_package(late_files, argument_dict['config_id'], argument_dict['provider']) if late_variables: # Render the late binding package. This package will be downloaded onto # each cluster node during bootstrap and rendered into the final config # using the values from the late config file. late_package_id = PackageId(late_package['name']) late_package_filename = make_package_filename(late_package_id, '.dcos_config') os.makedirs(os.path.dirname(late_package_filename), mode=0o755) write_yaml(late_package_filename, {'package': late_package['package']}, default_flow_style=False) log.info('Package filename: {}'.format(late_package_filename)) stable_artifacts.append(late_package_filename) # Add the late config file to cloud config. The expressions in # late_variables will be resolved by the service handling the cloud # config (e.g. Amazon CloudFormation). The rendered late config file # on a cluster node's filesystem will contain the final values. rendered_templates['cloud-config.yaml']['root'].append({ 'path': '/etc/mesosphere/setup-flags/late-config.yaml', 'permissions': '0644', 'owner': 'root', # TODO(cmaloney): don't prettyprint to save bytes. # NOTE: Use yaml here simply to make avoiding painful escaping and # unescaping easier. 'content': render_yaml({ 'late_bound_package_id': late_package['name'], 'bound_values': late_variables })}) # Collect metadata for cluster packages. for package_id_str in json.loads(argument_dict['cluster_packages']): package_id = PackageId(package_id_str) package_filename = make_package_filename(package_id, '.tar.xz') cluster_package_info[package_id.name] = { 'id': package_id_str, 'filename': package_filename } # Render config packages. config_package_ids = json.loads(argument_dict['config_package_ids']) for package_id_str in config_package_ids: package_id = PackageId(package_id_str) package_filename = cluster_package_info[package_id.name]['filename'] do_gen_package(rendered_templates[package_id.name + '.yaml'], cluster_package_info[package_id.name]['filename']) stable_artifacts.append(package_filename) # Convert cloud-config to just contain write_files rather than root cc = rendered_templates['cloud-config.yaml'] # Shouldn't contain any packages. Providers should pull what they need to # late bind out of other packages via cc_package_file. assert 'package' not in cc cc_root = cc.pop('root', []) # Make sure write_files exists. assert 'write_files' not in cc cc['write_files'] = [] # Do the transform for item in cc_root: assert item['path'].startswith('/') cc['write_files'].append(item) rendered_templates['cloud-config.yaml'] = cc # Add utils that need to be defined here so they can be bound to locals. def add_services(cloudconfig, cloud_init_implementation): return add_units(cloudconfig, rendered_templates['dcos-services.yaml'], cloud_init_implementation) utils.add_services = add_services def add_stable_artifact(filename): assert filename not in stable_artifacts + channel_artifacts stable_artifacts.append(filename) utils.add_stable_artifact = add_stable_artifact def add_channel_artifact(filename): assert filename not in stable_artifacts + channel_artifacts channel_artifacts.append(filename) utils.add_channel_artifact = add_channel_artifact return Bunch({ 'arguments': argument_dict, 'cluster_packages': cluster_package_info, 'stable_artifacts': stable_artifacts, 'channel_artifacts': channel_artifacts, 'templates': rendered_templates, 'utils': utils })