def do_create(tag, build_name, reproducible_artifact_path, commit, variant_arguments, all_completes): """Create a installer script for each variant in bootstrap_dict. Writes a dcos_generate_config.<variant>.sh for each variant in bootstrap_dict to the working directory, except for the default variant's script, which is written to dcos_generate_config.sh. Returns a dict mapping variants to (genconf_version, genconf_filename) tuples. Outputs the generated dcos_generate_config.sh as it's artifacts. """ # TODO(cmaloney): Build installers in parallel. # Variants are sorted for stable ordering. for variant, bootstrap_info in sorted(variant_arguments.items(), key=lambda kv: pkgpanda.util.variant_str(kv[0])): with logger.scope("Building installer for variant: ".format(pkgpanda.util.variant_name(variant))): bootstrap_installer_name = '{}installer'.format(pkgpanda.util.variant_prefix(variant)) installer_filename = make_installer_docker(variant, all_completes[variant], all_completes[bootstrap_installer_name]) yield { 'channel_path': 'dcos_generate_config.{}sh'.format(pkgpanda.util.variant_prefix(variant)), 'local_path': installer_filename } # Build dcos-launch # TODO(cmaloney): This really doesn't belong to here, but it's the best place it fits for now. # dcos-launch works many places which aren't bash / on-premise installers. # It also isn't dependent on the "reproducible" artifacts at all. Just the commit... with logger.scope("building dcos-launch"): yield { 'channel_path': 'dcos-launch', 'local_path': make_dcos_launch() }
def do_create(tag, build_name, reproducible_artifact_path, commit, variant_arguments, all_completes): """Create a installer script for each variant in bootstrap_dict. Writes a dcos_generate_config.<variant>.sh for each variant in bootstrap_dict to the working directory, except for the default variant's script, which is written to dcos_generate_config.sh. Returns a dict mapping variants to (genconf_version, genconf_filename) tuples. Outputs the generated dcos_generate_config.sh as it's artifacts. """ # TODO(cmaloney): Build installers in parallel. # Variants are sorted for stable ordering. for variant in sorted(variant_arguments.keys(), key=lambda k: pkgpanda.util.variant_str(k)): variant_name = pkgpanda.util.variant_name(variant) bootstrap_installer_name = '{}installer'.format(pkgpanda.util.variant_prefix(variant)) if bootstrap_installer_name not in all_completes: print('WARNING: No installer tree for variant: {}'.format(variant_name)) else: with logger.scope("Building installer for variant: {}".format(variant_name)): yield { 'channel_path': 'dcos_generate_config.{}sh'.format(pkgpanda.util.variant_prefix(variant)), 'local_path': make_installer_docker(variant, all_completes[variant], all_completes[bootstrap_installer_name]) }
def setup_cluster_workload(self, dcos_api: DcosApiSession, healthcheck_app: dict, dns_app: dict): # Deploy test apps. # TODO(branden): We ought to be able to deploy these apps concurrently. See # https://mesosphere.atlassian.net/browse/DCOS-13360. with logger.scope("deploy apps"): dcos_api.marathon.deploy_app(healthcheck_app) dcos_api.marathon.ensure_deployments_complete() # This is a hack to make sure we don't deploy dns_app before the name it's # trying to resolve is available. self.wait_for_dns(dcos_api, dns_app['env']['RESOLVE_NAME']) dcos_api.marathon.deploy_app(dns_app, check_health=False) dcos_api.marathon.ensure_deployments_complete() test_apps = [healthcheck_app, dns_app] self.test_app_ids = [app['id'] for app in test_apps] self.tasks_start = {app_id: sorted(self.app_task_ids(dcos_api, app_id)) for app_id in self.test_app_ids} self.log.debug('Test app tasks at start:\n' + pprint.pformat(self.tasks_start)) for app in test_apps: assert app['instances'] == len(self.tasks_start[app['id']]) # Save the master's state of the task to compare with # the master's view after the upgrade. # See this issue for why we check for a difference: # https://issues.apache.org/jira/browse/MESOS-1718 self.task_state_start = self.get_master_task_state(dcos_api, self.tasks_start[self.test_app_ids[0]][0])
def apply_storage_commands(self, storage_commands): assert storage_commands.keys() == {'stage1', 'stage2'} if self.__noop: return with logger.scope("Uploading artifacts"): apply_storage_commands(self.__storage_providers, storage_commands)
def make_stable_artifacts(cache_repository_url): metadata = { "commit": util.dcos_image_commit, "core_artifacts": [], "packages": set() } # TODO(cmaloney): Rather than guessing / reverse-engineering all these paths # have do_build_packages get them directly from pkgpanda with logger.scope("Building packages"): try: all_completes = do_build_packages(cache_repository_url) except pkgpanda.build.BuildError as ex: logger.error("Failure building package(s): {}".format(ex)) raise # The installer and util are built bootstraps, but not a DC/OS variants. We use # iteration over the complete_dict to enumerate all variants a whole lot, # so explicity remove installer/util here so people don't accidentally hit it. # TODO: make this into a tree option complete_dict = dict() for name, info in copy.copy(all_completes).items(): if name is not None and (name.endswith('installer') or name.endswith('util')): continue complete_dict[name] = info metadata["complete_dict"] = complete_dict metadata["all_completes"] = all_completes metadata["bootstrap_dict"] = {k: v['bootstrap'] for k, v in complete_dict.items()} metadata["all_bootstraps"] = {k: v['bootstrap'] for k, v in all_completes.items()} def add_file(info): metadata["core_artifacts"].append(info) def add_package(package_id): if package_id in metadata['packages']: return metadata['packages'].add(package_id) add_file(get_package_artifact(package_id)) # Add the bootstrap, active.json, packages as reproducible_path artifacts # Add the <variant>.bootstrap.latest as a channel_path for name, info in sorted(all_completes.items(), key=lambda kv: pkgpanda.util.variant_str(kv[0])): for file in make_bootstrap_artifacts(info['bootstrap'], info['packages'], name, 'packages/cache'): add_file(file) # Add all the packages which haven't been added yet for package_id in sorted(info['packages']): add_package(package_id) # Sets aren't json serializable, so transform to a list for future use. metadata['packages'] = list(sorted(metadata['packages'])) return metadata
def make_bootstrap(package_set): with logger.scope("Making bootstrap variant: {}".format(pkgpanda.util.variant_name(package_set.variant))): package_paths = list() for name, pkg_variant in package_set.bootstrap_packages: package_paths.append(built_packages[name][pkg_variant]) if mkbootstrap: return make_bootstrap_tarball( package_store, list(sorted(package_paths)), package_set.variant)
def gen_simple_template(variant_prefix, filename, arguments, extra_source): results = gen.generate( arguments=arguments, extra_templates=[ 'aws/templates/cloudformation.json', 'aws/dcos-config.yaml', 'coreos-aws/cloud-config.yaml', 'coreos/cloud-config.yaml'], cc_package_files=[ '/etc/cfn_signal_metadata', '/etc/adminrouter.env', '/etc/ui-config.json', '/etc/dns_config', '/etc/exhibitor', '/etc/mesos-master-provider', '/etc/extra_master_addresses'], extra_sources=[aws_base_source, aws_simple_source, extra_source]) cloud_config = results.templates['cloud-config.yaml'] # Add general services cloud_config = results.utils.add_services(cloud_config, 'coreos') # Specialize for master, slave, slave_public variant_cloudconfig = {} for variant, params in cf_instance_groups.items(): cc_variant = deepcopy(cloud_config) # Specialize the dcos-cfn-signal service cc_variant = results.utils.add_units( cc_variant, yaml.safe_load(gen.template.parse_str(late_services).render(params))) # Add roles cc_variant = results.utils.add_roles(cc_variant, params['roles'] + ['aws']) # NOTE: If this gets printed in string stylerather than '|' the AWS # parameters which need to be split out for the cloudformation to # interpret end up all escaped and undoing it would be hard. variant_cloudconfig[variant] = results.utils.render_cloudconfig(cc_variant) # Render the cloudformation cloudformation = render_cloudformation( results.templates['cloudformation.json'], master_cloud_config=variant_cloudconfig['master'], slave_cloud_config=variant_cloudconfig['slave'], slave_public_cloud_config=variant_cloudconfig['slave_public']) with logger.scope("Validating CloudFormation"): validate_cf(cloudformation) yield from _as_artifact_and_pkg(variant_prefix, filename, (cloudformation, results))
def verify_apps_state(self, dcos_api: DcosApiSession, dns_app: dict): with logger.scope("verify apps state"): # nested methods here so we can "close" over external state def marathon_app_tasks_survive_upgrade(): # Verify that the tasks we started are still running. tasks_end = {app_id: sorted(self.app_task_ids(dcos_api, app_id)) for app_id in self.test_app_ids} self.log.debug('Test app tasks at end:\n' + pprint.pformat(tasks_end)) if not self.tasks_start == tasks_end: self.teamcity_msg.testFailed( "test_upgrade_vpc.marathon_app_tasks_survive_upgrade", details="expected: {}\nactual: {}".format(self.tasks_start, tasks_end)) def test_mesos_task_state_remains_consistent(): # Verify that the "state" of the task does not change. task_state_end = self.get_master_task_state(dcos_api, self.tasks_start[self.test_app_ids[0]][0]) if not self.task_state_start == task_state_end: self.teamcity_msg.testFailed( "test_upgrade_vpc.test_mesos_task_state_remains_consistent", details="expected: {}\nactual: {}".format(self.task_state_start, task_state_end)) def test_app_dns_survive_upgrade(): # Verify DNS didn't fail. marathon_framework_id = dcos_api.marathon.get('/v2/info').json()['frameworkId'] dns_app_task = dcos_api.marathon.get('/v2/apps' + dns_app['id'] + '/tasks').json()['tasks'][0] dns_log = self.parse_dns_log(dcos_api.mesos_sandbox_file( dns_app_task['slaveId'], marathon_framework_id, dns_app_task['id'], dns_app['env']['DNS_LOG_FILENAME'], )) dns_failure_times = [entry[0] for entry in dns_log if entry[1] != 'SUCCESS'] if len(dns_failure_times) > 0: message = 'Failed to resolve Marathon app hostname {} at least once.'.format( dns_app['env']['RESOLVE_NAME']) err_msg = message + ' Hostname failed to resolve at these times:\n' + '\n'.join(dns_failure_times) self.log.debug(err_msg) self.teamcity_msg.testFailed("test_upgrade_vpc.test_app_dns_survive_upgrade", details=err_msg) self.log_test("test_upgrade_vpc.marathon_app_tasks_survive_upgrade", marathon_app_tasks_survive_upgrade) self.log_test( "test_upgrade_vpc.test_mesos_task_state_remains_consistent", test_mesos_task_state_remains_consistent ) self.log_test("test_upgrade_vpc.test_app_dns_survive_upgrade", test_app_dns_survive_upgrade)
def setup_cluster_workload(self, dcos_api: DcosApiSession, healthcheck_app: dict, dns_app: dict, viplisten_app: dict, viptalk_app: dict): # Deploy test apps. # TODO(branden): We ought to be able to deploy these apps concurrently. See # https://mesosphere.atlassian.net/browse/DCOS-13360. with logger.scope("deploy apps"): dcos_api.marathon.deploy_app(viplisten_app) dcos_api.marathon.ensure_deployments_complete() # viptalk app depends on VIP from viplisten app, which may still fail # the first try immediately after ensure_deployments_complete dcos_api.marathon.deploy_app(viptalk_app, ignore_failed_tasks=True) dcos_api.marathon.ensure_deployments_complete() dcos_api.marathon.deploy_app(healthcheck_app) dcos_api.marathon.ensure_deployments_complete() # This is a hack to make sure we don't deploy dns_app before the name it's # trying to resolve is available. self.wait_for_dns(dcos_api, dns_app['env']['RESOLVE_NAME']) dcos_api.marathon.deploy_app(dns_app, check_health=False) dcos_api.marathon.ensure_deployments_complete() test_apps = [healthcheck_app, dns_app, viplisten_app, viptalk_app] self.test_app_ids = [app['id'] for app in test_apps] self.tasks_start = { app_id: sorted(self.app_task_ids(dcos_api, app_id)) for app_id in self.test_app_ids } self.log.debug('Test app tasks at start:\n' + pprint.pformat(self.tasks_start)) for app in test_apps: assert app['instances'] == len(self.tasks_start[app['id']]) # Save the master's state of the task to compare with # the master's view after the upgrade. # See this issue for why we check for a difference: # https://issues.apache.org/jira/browse/MESOS-1718 self.task_state_start = self.get_master_task_state( dcos_api, self.tasks_start[self.test_app_ids[0]][0])
def setup_cluster_workload(self, dcos_api: DcosApiSession, healthcheck_app: dict, dns_app: dict): # Deploy test apps. # TODO(branden): We ought to be able to deploy these apps concurrently. See # https://mesosphere.atlassian.net/browse/DCOS-13360. with logger.scope("deploy apps"): dcos_api.marathon.deploy_app(healthcheck_app) dcos_api.marathon.ensure_deployments_complete() # This is a hack to make sure we don't deploy dns_app before the name it's # trying to resolve is available. self.wait_for_dns(dcos_api, dns_app['env']['RESOLVE_NAME']) dcos_api.marathon.deploy_app(dns_app, check_health=False) dcos_api.marathon.ensure_deployments_complete() test_apps = [healthcheck_app, dns_app] self.test_app_ids = [app['id'] for app in test_apps] self.tasks_start = {app_id: sorted(self.app_task_ids(dcos_api, app_id)) for app_id in self.test_app_ids} self.log.debug('Test app tasks at start:\n' + pprint.pformat(self.tasks_start)) for app in test_apps: assert app['instances'] == len(self.tasks_start[app['id']])
def build(package_store: PackageStore, name: str, variant, clean_after_build, recursive=False): msg = "Building package {} variant {}".format(name, pkgpanda.util.variant_name(variant)) with logger.scope(msg): return _build(package_store, name, variant, clean_after_build, recursive)
def build_tree(package_store, mkbootstrap, tree_variant): """Build packages and bootstrap tarballs for one or all tree variants. Returns a dict mapping tree variants to bootstrap IDs. If tree_variant is None, builds all available tree variants. """ # TODO(cmaloney): Add support for circular dependencies. They are doable # long as there is a pre-built version of enough of the packages. # TODO(cmaloney): Make it so when we're building a treeinfo which has a # explicit package list we don't build all the other packages. build_order = list() visited = set() built = set() def visit(pkg_tuple: tuple): """Add a package and its requires to the build order. Raises AssertionError if pkg_tuple is in the set of visited packages. If the package has any requires, they're recursively visited and added to the build order depth-first. Then the package itself is added. """ # Visit the node for the first (and only) time. assert pkg_tuple not in visited visited.add(pkg_tuple) # Ensure all dependencies are built. Sorted for stability for require in sorted(package_store.packages[pkg_tuple]['requires']): require_tuple = expand_require(require) # If the dependency has already been built, we can move on. if require_tuple in built: continue # If the dependency has not been built but has been visited, then # there's a cycle in the dependency graph. if require_tuple in visited: raise BuildError("Circular dependency. Circular link {0} -> {1}".format(pkg_tuple, require_tuple)) if PackageId.is_id(require_tuple[0]): raise BuildError("Depending on a specific package id is not supported. Package {} " "depends on {}".format(pkg_tuple, require_tuple)) if require_tuple not in package_store.packages: raise BuildError("Package {0} require {1} not buildable from tree.".format(pkg_tuple, require_tuple)) # Add the dependency (after its dependencies, if any) to the build # order. visit(require_tuple) build_order.append(pkg_tuple) built.add(pkg_tuple) # Can't compare none to string, so expand none -> "true" / "false", then put # the string in a field after "" if none, the string if not. def key_func(elem): return elem[0], elem[1] is None, elem[1] or "" def visit_packages(package_tuples): for pkg_tuple in sorted(package_tuples, key=key_func): if pkg_tuple in visited: continue visit(pkg_tuple) if tree_variant: package_sets = [package_store.get_package_set(tree_variant)] else: package_sets = package_store.get_all_package_sets() with logger.scope("resolve package graph"): # Build all required packages for all tree variants. for package_set in package_sets: visit_packages(package_set.all_packages) built_packages = dict() for (name, variant) in build_order: built_packages.setdefault(name, dict()) # Run the build, store the built package path for later use. # TODO(cmaloney): Only build the requested variants, rather than all variants. built_packages[name][variant] = build( package_store, name, variant, True) # Build bootstrap tarballs for all tree variants. def make_bootstrap(package_set): with logger.scope("Making bootstrap variant: {}".format(pkgpanda.util.variant_name(package_set.variant))): package_paths = list() for name, pkg_variant in package_set.bootstrap_packages: package_paths.append(built_packages[name][pkg_variant]) if mkbootstrap: return make_bootstrap_tarball( package_store, list(sorted(package_paths)), package_set.variant) # Build bootstraps and and package lists for all variants. # TODO(cmaloney): Allow distinguishing between "build all" and "build the default one". complete_cache_dir = package_store.get_complete_cache_dir() check_call(['mkdir', '-p', complete_cache_dir]) results = {} for package_set in package_sets: info = { 'bootstrap': make_bootstrap(package_set), 'packages': sorted( load_string(package_store.get_last_build_filename(*pkg_tuple)) for pkg_tuple in package_set.all_packages)} write_json( complete_cache_dir + '/' + pkgpanda.util.variant_prefix(package_set.variant) + 'complete.latest.json', info) results[package_set.variant] = info return results
def make_channel_artifacts(metadata): artifacts = [] provider_data = {} providers = load_providers() for name, module in sorted(providers.items()): bootstrap_url = metadata['repository_url'] # If the particular provider has its own storage by the same name then # Use the storage provider rather if name in metadata['storage_urls']: bootstrap_url = metadata['storage_urls'][name] + metadata[ 'repository_path'] variant_arguments = dict() for bootstrap_name, bootstrap_id in metadata['bootstrap_dict'].items(): variant_arguments[bootstrap_name] = copy.deepcopy({ 'bootstrap_url': bootstrap_url, 'provider': name, 'bootstrap_id': bootstrap_id, 'bootstrap_variant': pkgpanda.util.variant_prefix(bootstrap_name) }) # Load additional default variant arguments out of gen_extra if os.path.exists('gen_extra/calc.py'): mod = importlib.machinery.SourceFileLoader( 'gen_extra.calc', 'gen_extra/calc.py').load_module() variant_arguments[bootstrap_name].update( mod.provider_template_defaults) # Add templates for the default variant. # Use keyword args to make not matching ordering a loud error around changes. with logger.scope("Creating {} deploy tools".format(module.__name__)): for built_resource in module.do_create( tag=metadata['tag'], build_name=metadata['build_name'], reproducible_artifact_path=metadata[ 'reproducible_artifact_path'], commit=metadata['commit'], variant_arguments=variant_arguments, all_bootstraps=metadata["all_bootstraps"]): assert isinstance(built_resource, dict), built_resource # Type switch if 'packages' in built_resource: for package in built_resource['packages']: artifacts.append(get_gen_package_artifact(package)) else: assert 'packages' not in built_resource artifacts.append(built_resource) # TODO(cmaloney): Check the provider artifacts adhere to the artifact template. artifacts += provider_data.get('artifacts', list()) return artifacts
def _build(package_store, name, variant, clean_after_build, recursive): assert isinstance(package_store, PackageStore) tmpdir = tempfile.TemporaryDirectory(prefix="pkgpanda_repo") repository = Repository(tmpdir.name) package_dir = package_store.get_package_folder(name) def src_abs(name): return package_dir + '/' + name def cache_abs(filename): return package_store.get_package_cache_folder(name) + '/' + filename # Build pkginfo over time, translating fields from buildinfo. pkginfo = {} # Build up the docker command arguments over time, translating fields as needed. cmd = DockerCmd() assert (name, variant) in package_store.packages, \ "Programming error: name, variant should have been validated to be valid before calling build()." builder = IdBuilder(package_store.get_buildinfo(name, variant)) final_buildinfo = dict() builder.add('name', name) builder.add('variant', pkgpanda.util.variant_str(variant)) # Convert single_source -> sources if builder.has('sources'): if builder.has('single_source'): raise BuildError('Both sources and single_source cannot be specified at the same time') sources = builder.take('sources') elif builder.has('single_source'): sources = {name: builder.take('single_source')} builder.replace('single_source', 'sources', sources) else: builder.add('sources', {}) sources = dict() print("NOTICE: No sources specified") final_buildinfo['sources'] = sources # Construct the source fetchers, gather the checkout ids from them checkout_ids = dict() fetchers = dict() try: for src_name, src_info in sorted(sources.items()): # TODO(cmaloney): Switch to a unified top level cache directory shared by all packages cache_dir = package_store.get_package_cache_folder(name) + '/' + src_name check_call(['mkdir', '-p', cache_dir]) fetcher = get_src_fetcher(src_info, cache_dir, package_dir) fetchers[src_name] = fetcher checkout_ids[src_name] = fetcher.get_id() except ValidationError as ex: raise BuildError("Validation error when fetching sources for package: {}".format(ex)) for src_name, checkout_id in checkout_ids.items(): # NOTE: single_source buildinfo was expanded above so the src_name is # always correct here. # Make sure we never accidentally overwrite something which might be # important. Fields should match if specified (And that should be # tested at some point). For now disallowing identical saves hassle. assert_no_duplicate_keys(checkout_id, final_buildinfo['sources'][src_name]) final_buildinfo['sources'][src_name].update(checkout_id) # Add the sha1 of the buildinfo.json + build file to the build ids builder.update('sources', checkout_ids) build_script = src_abs(builder.take('build_script')) # TODO(cmaloney): Change dest name to build_script_sha1 builder.replace('build_script', 'build', pkgpanda.util.sha1(build_script)) builder.add('pkgpanda_version', pkgpanda.build.constants.version) extra_dir = src_abs("extra") # Add the "extra" folder inside the package as an additional source if it # exists if os.path.exists(extra_dir): extra_id = hash_folder_abs(extra_dir, package_dir) builder.add('extra_source', extra_id) final_buildinfo['extra_source'] = extra_id # Figure out the docker name. docker_name = builder.take('docker') cmd.container = docker_name # Add the id of the docker build environment to the build_ids. try: docker_id = get_docker_id(docker_name) except CalledProcessError: # docker pull the container and try again check_call(['docker', 'pull', docker_name]) docker_id = get_docker_id(docker_name) builder.update('docker', docker_id) # TODO(cmaloney): The environment variables should be generated during build # not live in buildinfo.json. pkginfo['environment'] = builder.take('environment') # Whether pkgpanda should on the host make sure a `/var/lib` state directory is available pkginfo['state_directory'] = builder.take('state_directory') if pkginfo['state_directory'] not in [True, False]: raise BuildError("state_directory in buildinfo.json must be a boolean `true` or `false`") username = None if builder.has('username'): username = builder.take('username') if not isinstance(username, str): raise BuildError("username in buildinfo.json must be either not set (no user for this" " package), or a user name string") try: pkgpanda.UserManagement.validate_username(username) except ValidationError as ex: raise BuildError("username in buildinfo.json didn't meet the validation rules. {}".format(ex)) pkginfo['username'] = username group = None if builder.has('group'): group = builder.take('group') if not isinstance(group, str): raise BuildError("group in buildinfo.json must be either not set (use default group for this user)" ", or group must be a string") try: pkgpanda.UserManagement.validate_group_name(group) except ValidationError as ex: raise BuildError("group in buildinfo.json didn't meet the validation rules. {}".format(ex)) pkginfo['group'] = group # Packages need directories inside the fake install root (otherwise docker # will try making the directories on a readonly filesystem), so build the # install root now, and make the package directories in it as we go. install_dir = tempfile.mkdtemp(prefix="pkgpanda-") active_packages = list() active_package_ids = set() active_package_variants = dict() auto_deps = set() # Final package has the same requires as the build. requires = builder.take('requires') pkginfo['requires'] = requires if builder.has("sysctl"): pkginfo["sysctl"] = builder.take("sysctl") # TODO(cmaloney): Pull generating the full set of requires a function. to_check = copy.deepcopy(requires) if type(to_check) != list: raise BuildError("`requires` in buildinfo.json must be an array of dependencies.") while to_check: requires_info = to_check.pop(0) requires_name, requires_variant = expand_require(requires_info) if requires_name in active_package_variants: # TODO(cmaloney): If one package depends on the <default> # variant of a package and 1+ others depends on a non-<default> # variant then update the dependency to the non-default variant # rather than erroring. if requires_variant != active_package_variants[requires_name]: # TODO(cmaloney): Make this contain the chains of # dependencies which contain the conflicting packages. # a -> b -> c -> d {foo} # e {bar} -> d {baz} raise BuildError( "Dependncy on multiple variants of the same package {}. variants: {} {}".format( requires_name, requires_variant, active_package_variants[requires_name])) # The variant has package {requires_name, variant} already is a # dependency, don't process it again / move on to the next. continue active_package_variants[requires_name] = requires_variant # Figure out the last build of the dependency, add that as the # fully expanded dependency. requires_last_build = package_store.get_last_build_filename(requires_name, requires_variant) if not os.path.exists(requires_last_build): if recursive: # Build the dependency build(package_store, requires_name, requires_variant, clean_after_build, recursive) else: raise BuildError("No last build file found for dependency {} variant {}. Rebuild " "the dependency".format(requires_name, requires_variant)) try: pkg_id_str = load_string(requires_last_build) auto_deps.add(pkg_id_str) pkg_buildinfo = package_store.get_buildinfo(requires_name, requires_variant) pkg_requires = pkg_buildinfo['requires'] pkg_path = repository.package_path(pkg_id_str) pkg_tar = pkg_id_str + '.tar.xz' if not os.path.exists(package_store.get_package_cache_folder(requires_name) + '/' + pkg_tar): raise BuildError( "The build tarball {} refered to by the last_build file of the dependency {} " "variant {} doesn't exist. Rebuild the dependency.".format( pkg_tar, requires_name, requires_variant)) active_package_ids.add(pkg_id_str) # Mount the package into the docker container. cmd.volumes[pkg_path] = "/opt/mesosphere/packages/{}:ro".format(pkg_id_str) os.makedirs(os.path.join(install_dir, "packages/{}".format(pkg_id_str))) # Add the dependencies of the package to the set which will be # activated. # TODO(cmaloney): All these 'transitive' dependencies shouldn't # be available to the package being built, only what depends on # them directly. to_check += pkg_requires except ValidationError as ex: raise BuildError("validating package needed as dependency {0}: {1}".format(requires_name, ex)) from ex except PackageError as ex: raise BuildError("loading package needed as dependency {0}: {1}".format(requires_name, ex)) from ex # Add requires to the package id, calculate the final package id. # NOTE: active_packages isn't fully constructed here since we lazily load # packages not already in the repository. builder.update('requires', list(active_package_ids)) version_extra = None if builder.has('version_extra'): version_extra = builder.take('version_extra') build_ids = builder.get_build_ids() version_base = hash_checkout(build_ids) version = None if builder.has('version_extra'): version = "{0}-{1}".format(version_extra, version_base) else: version = version_base pkg_id = PackageId.from_parts(name, version) # Everything must have been extracted by now. If it wasn't, then we just # had a hard error that it was set but not used, as well as didn't include # it in the caluclation of the PackageId. builder = None # Save the build_ids. Useful for verify exactly what went into the # package build hash. final_buildinfo['build_ids'] = build_ids final_buildinfo['package_version'] = version # Save the package name and variant. The variant is used when installing # packages to validate dependencies. final_buildinfo['name'] = name final_buildinfo['variant'] = variant # If the package is already built, don't do anything. pkg_path = package_store.get_package_cache_folder(name) + '/{}.tar.xz'.format(pkg_id) # Done if it exists locally if exists(pkg_path): print("Package up to date. Not re-building.") # TODO(cmaloney): Updating / filling last_build should be moved out of # the build function. write_string(package_store.get_last_build_filename(name, variant), str(pkg_id)) return pkg_path # Try downloading. dl_path = package_store.try_fetch_by_id(pkg_id) if dl_path: print("Package up to date. Not re-building. Downloaded from repository-url.") # TODO(cmaloney): Updating / filling last_build should be moved out of # the build function. write_string(package_store.get_last_build_filename(name, variant), str(pkg_id)) print(dl_path, pkg_path) assert dl_path == pkg_path return pkg_path # Fall out and do the build since it couldn't be downloaded print("Unable to download from cache. Proceeding to build") print("Building package {} with buildinfo: {}".format( pkg_id, json.dumps(final_buildinfo, indent=2, sort_keys=True))) # Clean out src, result so later steps can use them freely for building. def clean(): # Run a docker container to remove src/ and result/ cmd = DockerCmd() cmd.volumes = { package_store.get_package_cache_folder(name): "/pkg/:rw", } cmd.container = "ubuntu:14.04.4" cmd.run("package-cleaner", ["rm", "-rf", "/pkg/src", "/pkg/result"]) clean() # Only fresh builds are allowed which don't overlap existing artifacts. result_dir = cache_abs("result") if exists(result_dir): raise BuildError("result folder must not exist. It will be made when the package is " "built. {}".format(result_dir)) # 'mkpanda add' all implicit dependencies since we actually need to build. for dep in auto_deps: print("Auto-adding dependency: {}".format(dep)) # NOTE: Not using the name pkg_id because that overrides the outer one. id_obj = PackageId(dep) add_package_file(repository, package_store.get_package_path(id_obj)) package = repository.load(dep) active_packages.append(package) # Checkout all the sources int their respective 'src/' folders. try: src_dir = cache_abs('src') if os.path.exists(src_dir): raise ValidationError( "'src' directory already exists, did you have a previous build? " + "Currently all builds must be from scratch. Support should be " + "added for re-using a src directory when possible. src={}".format(src_dir)) os.mkdir(src_dir) for src_name, fetcher in sorted(fetchers.items()): root = cache_abs('src/' + src_name) os.mkdir(root) fetcher.checkout_to(root) except ValidationError as ex: raise BuildError("Validation error when fetching sources for package: {}".format(ex)) # Activate the packages so that we have a proper path, environment # variables. # TODO(cmaloney): RAII type thing for temproary directory so if we # don't get all the way through things will be cleaned up? install = Install( root=install_dir, config_dir=None, rooted_systemd=True, manage_systemd=False, block_systemd=True, fake_path=True, manage_users=False, manage_state_dir=False) install.activate(active_packages) # Rewrite all the symlinks inside the active path because we will # be mounting the folder into a docker container, and the absolute # paths to the packages will change. # TODO(cmaloney): This isn't very clean, it would be much nicer to # just run pkgpanda inside the package. rewrite_symlinks(install_dir, repository.path, "/opt/mesosphere/packages/") print("Building package in docker") # TODO(cmaloney): Run as a specific non-root user, make it possible # for non-root to cleanup afterwards. # Run the build, prepping the environment as necessary. mkdir(cache_abs("result")) # Copy the build info to the resulting tarball write_json(cache_abs("src/buildinfo.full.json"), final_buildinfo) write_json(cache_abs("result/buildinfo.full.json"), final_buildinfo) write_json(cache_abs("result/pkginfo.json"), pkginfo) # Make the folder for the package we are building. If docker does it, it # gets auto-created with root permissions and we can't actually delete it. os.makedirs(os.path.join(install_dir, "packages", str(pkg_id))) # TOOD(cmaloney): Disallow writing to well known files and directories? # Source we checked out cmd.volumes.update({ # TODO(cmaloney): src should be read only... cache_abs("src"): "/pkg/src:rw", # The build script build_script: "/pkg/build:ro", # Getting the result out cache_abs("result"): "/opt/mesosphere/packages/{}:rw".format(pkg_id), install_dir: "/opt/mesosphere:ro" }) if os.path.exists(extra_dir): cmd.volumes[extra_dir] = "/pkg/extra:ro" cmd.environment = { "PKG_VERSION": version, "PKG_NAME": name, "PKG_ID": pkg_id, "PKG_PATH": "/opt/mesosphere/packages/{}".format(pkg_id), "PKG_VARIANT": variant if variant is not None else "<default>", "NUM_CORES": multiprocessing.cpu_count() } try: # TODO(cmaloney): Run a wrapper which sources # /opt/mesosphere/environment then runs a build. Also should fix # ownership of /opt/mesosphere/packages/{pkg_id} post build. cmd.run("package-builder", [ "/bin/bash", "-o", "nounset", "-o", "pipefail", "-o", "errexit", "/pkg/build"]) except CalledProcessError as ex: raise BuildError("docker exited non-zero: {}\nCommand: {}".format(ex.returncode, ' '.join(ex.cmd))) # Clean up the temporary install dir used for dependencies. # TODO(cmaloney): Move to an RAII wrapper. check_call(['rm', '-rf', install_dir]) with logger.scope("Build package tarball"): # Check for forbidden services before packaging the tarball: try: check_forbidden_services(cache_abs("result"), RESERVED_UNIT_NAMES) except ValidationError as ex: raise BuildError("Package validation failed: {}".format(ex)) # TODO(cmaloney): Updating / filling last_build should be moved out of # the build function. write_string(package_store.get_last_build_filename(name, variant), str(pkg_id)) # Bundle the artifacts into the pkgpanda package tmp_name = pkg_path + "-tmp.tar.xz" make_tar(tmp_name, cache_abs("result")) os.rename(tmp_name, pkg_path) print("Package built.") if clean_after_build: clean() return pkg_path
def run_test(self) -> int: stack_name = 'dcos-ci-test-upgrade-' + random_id(10) test_id = uuid.uuid4().hex healthcheck_app_id = TEST_APP_NAME_FMT.format('healthcheck-' + test_id) dns_app_id = TEST_APP_NAME_FMT.format('dns-' + test_id) with logger.scope("create vpc cf stack '{}'".format(stack_name)): bw = test_util.aws.BotoWrapper( region=self.aws_region, aws_access_key_id=self.aws_access_key_id, aws_secret_access_key=self.aws_secret_access_key) ssh_key = bw.create_key_pair(stack_name) write_string('ssh_key', ssh_key) vpc, ssh_info = test_util.aws.VpcCfStack.create( stack_name=stack_name, instance_type='m4.xlarge', instance_os='cent-os-7-dcos-prereqs', # An instance for each cluster node plus the bootstrap. instance_count=(self.num_masters + self.num_agents + self.num_public_agents + 1), admin_location='0.0.0.0/0', key_pair_name=stack_name, boto_wrapper=bw ) vpc.wait_for_complete() cluster = test_util.cluster.Cluster.from_vpc( vpc, ssh_info, ssh_key=ssh_key, num_masters=self.num_masters, num_agents=self.num_agents, num_public_agents=self.num_public_agents, ) with logger.scope("install dcos"): # Use the CLI installer to set exhibitor_storage_backend = zookeeper. # Don't install prereqs since stable breaks Docker 1.13. See # https://jira.mesosphere.com/browse/DCOS_OSS-743. test_util.cluster.install_dcos(cluster, self.stable_installer_url, api=False, install_prereqs=False, add_config_path=self.config_yaml_override_install) master_list = [h.private_ip for h in cluster.masters] dcos_api_install = self.dcos_api_session_factory_install.apply( 'http://{ip}'.format(ip=cluster.masters[0].public_ip), master_list, master_list, [h.private_ip for h in cluster.agents], [h.private_ip for h in cluster.public_agents], self.default_os_user) dcos_api_install.wait_for_dcos() installed_version = dcos_api_install.get_version() healthcheck_app = create_marathon_healthcheck_app(healthcheck_app_id) dns_app = create_marathon_dns_app(dns_app_id, healthcheck_app_id) viplisten_app = create_marathon_viplisten_app() viptalk_app = create_marathon_viptalk_app() self.setup_cluster_workload(dcos_api_install, healthcheck_app, dns_app, viplisten_app, viptalk_app) with logger.scope("upgrade cluster"): test_util.cluster.upgrade_dcos(cluster, self.installer_url, installed_version, add_config_path=self.config_yaml_override_upgrade) with cluster.ssher.tunnel(cluster.bootstrap_host) as bootstrap_host_tunnel: bootstrap_host_tunnel.remote_cmd(['sudo', 'rm', '-rf', cluster.ssher.home_dir + '/*']) # this method invocation looks like it is the same as the one above, and that is partially correct. # the arguments to the invocation are the same, but the thing that changes is the lifecycle of the cluster # the client is being created to interact with. This client is specifically for the cluster after the # upgrade has taken place, and can account for any possible settings that may change for the client under # the hood when it probes the cluster. dcos_api_upgrade = self.dcos_api_session_factory_upgrade.apply( 'http://{ip}'.format(ip=cluster.masters[0].public_ip), master_list, master_list, [h.private_ip for h in cluster.agents], [h.private_ip for h in cluster.public_agents], self.default_os_user) dcos_api_upgrade.wait_for_dcos() # here we wait for DC/OS to be "up" so that we can auth this new client self.verify_apps_state(dcos_api_upgrade, dns_app) with logger.scope("run integration tests"): # copied from test_util/test_aws_cf.py:96 add_env = [] prefix = 'TEST_ADD_ENV_' for k, v in os.environ.items(): if k.startswith(prefix): add_env.append(k.replace(prefix, '') + '=' + v) test_cmd = ' '.join(add_env) + ' py.test -vv -s -rs ' + os.getenv('CI_FLAGS', '') result = test_util.cluster.run_integration_tests(cluster, test_cmd=test_cmd) if result == 0: self.log.info("Test successful! Deleting VPC if provided in this run.") vpc.delete() bw.delete_key_pair(stack_name) else: self.log.info("Test failed! VPC cluster will remain available for " "debugging for 2 hour after instantiation.") return result
def build_tree(package_store, mkbootstrap, tree_variant): """Build packages and bootstrap tarballs for one or all tree variants. Returns a dict mapping tree variants to bootstrap IDs. If tree_variant is None, builds all available tree variants. """ # TODO(cmaloney): Add support for circular dependencies. They are doable # long as there is a pre-built version of enough of the packages. # TODO(cmaloney): Make it so when we're building a treeinfo which has a # explicit package list we don't build all the other packages. build_order = list() visited = set() built = set() def visit(pkg_tuple: tuple): """Add a package and its requires to the build order. Raises AssertionError if pkg_tuple is in the set of visited packages. If the package has any requires, they're recursively visited and added to the build order depth-first. Then the package itself is added. """ # Visit the node for the first (and only) time. assert pkg_tuple not in visited visited.add(pkg_tuple) # Ensure all dependencies are built. Sorted for stability for require in sorted(package_store.packages[pkg_tuple]['requires']): require_tuple = expand_require(require) # If the dependency has already been built, we can move on. if require_tuple in built: continue # If the dependency has not been built but has been visited, then # there's a cycle in the dependency graph. if require_tuple in visited: raise BuildError("Circular dependency. Circular link {0} -> {1}".format(pkg_tuple, require_tuple)) if PackageId.is_id(require_tuple[0]): raise BuildError("Depending on a specific package id is not supported. Package {} " "depends on {}".format(pkg_tuple, require_tuple)) if require_tuple not in package_store.packages: raise BuildError("Package {0} require {1} not buildable from tree.".format(pkg_tuple, require_tuple)) # Add the dependency (after its dependencies, if any) to the build # order. visit(require_tuple) build_order.append(pkg_tuple) built.add(pkg_tuple) # Can't compare none to string, so expand none -> "true" / "false", then put # the string in a field after "" if none, the string if not. def key_func(elem): return elem[0], elem[1] is None, elem[1] or "" def visit_packages(package_tuples): for pkg_tuple in sorted(package_tuples, key=key_func): if pkg_tuple in visited: continue visit(pkg_tuple) if tree_variant: package_sets = [package_store.get_package_set(tree_variant)] else: package_sets = package_store.get_all_package_sets() with logger.scope("resolve package graph"): # Build all required packages for all tree variants. for package_set in package_sets: visit_packages(package_set.all_packages) built_packages = dict() for (name, variant) in build_order: built_packages.setdefault(name, dict()) # Run the build, store the built package path for later use. # TODO(cmaloney): Only build the requested variants, rather than all variants. built_packages[name][variant] = build( package_store, name, variant, True) # Build bootstrap tarballs for all tree variants. def make_bootstrap(package_set): with logger.scope("Making bootstrap variant: {}".format(pkgpanda.util.variant_name(package_set.variant))): package_paths = list() for name, pkg_variant in package_set.bootstrap_packages: package_paths.append(built_packages[name][pkg_variant]) if mkbootstrap: return make_bootstrap_tarball( package_store, list(sorted(package_paths)), package_set.variant) # Build bootstraps and and package lists for all variants. # TODO(cmaloney): Allow distinguishing between "build all" and "build the default one". complete_cache_dir = package_store.get_complete_cache_dir() check_call(['mkdir', '-p', complete_cache_dir]) results = {} for package_set in package_sets: info = { 'bootstrap': make_bootstrap(package_set), 'packages': sorted( load_string(package_store.get_last_build_filename(*pkg_tuple)) for pkg_tuple in package_set.all_packages)} write_json( complete_cache_dir + '/' + pkgpanda.util.variant_prefix(package_set.variant) + 'complete.latest.json', info) results[package_set.variant] = info return results
def build(package_store: PackageStore, name: str, variant, clean_after_build, recursive=False): msg = "Building package {} variant {}".format(name, pkgpanda.util.variant_name(variant)) with logger.scope(msg): return _build(package_store, name, variant, clean_after_build, recursive)
def run_test(self) -> int: stack_name = 'dcos-ci-test-upgrade-' + random_id(10) test_id = uuid.uuid4().hex healthcheck_app_id = TEST_APP_NAME_FMT.format('healthcheck-' + test_id) dns_app_id = TEST_APP_NAME_FMT.format('dns-' + test_id) with logger.scope("create vpc cf stack '{}'".format(stack_name)): bw = test_util.aws.BotoWrapper( region=self.aws_region, aws_access_key_id=self.aws_access_key_id, aws_secret_access_key=self.aws_secret_access_key) ssh_key = bw.create_key_pair(stack_name) write_string('ssh_key', ssh_key) vpc, ssh_info = test_util.aws.VpcCfStack.create( stack_name=stack_name, instance_type='m4.xlarge', instance_os='cent-os-7-dcos-prereqs', # An instance for each cluster node plus the bootstrap. instance_count=(self.num_masters + self.num_agents + self.num_public_agents + 1), admin_location='0.0.0.0/0', key_pair_name=stack_name, boto_wrapper=bw ) vpc.wait_for_complete() cluster = test_util.cluster.Cluster.from_vpc( vpc, ssh_info, ssh_key=ssh_key, num_masters=self.num_masters, num_agents=self.num_agents, num_public_agents=self.num_public_agents, ) with logger.scope("install dcos"): # Use the CLI installer to set exhibitor_storage_backend = zookeeper. test_util.cluster.install_dcos(cluster, self.stable_installer_url, api=False, add_config_path=self.config_yaml_override_install) master_list = [h.private_ip for h in cluster.masters] dcos_api_install = self.dcos_api_session_factory_install.apply( 'http://{ip}'.format(ip=cluster.masters[0].public_ip), master_list, master_list, [h.private_ip for h in cluster.agents], [h.private_ip for h in cluster.public_agents], self.default_os_user) dcos_api_install.wait_for_dcos() installed_version = dcos_api_install.get_version() healthcheck_app = create_marathon_healthcheck_app(healthcheck_app_id) dns_app = create_marathon_dns_app(dns_app_id, healthcheck_app_id) self.setup_cluster_workload(dcos_api_install, healthcheck_app, dns_app) with logger.scope("upgrade cluster"): test_util.cluster.upgrade_dcos(cluster, self.installer_url, installed_version, add_config_path=self.config_yaml_override_upgrade) with cluster.ssher.tunnel(cluster.bootstrap_host) as bootstrap_host_tunnel: bootstrap_host_tunnel.remote_cmd(['sudo', 'rm', '-rf', cluster.ssher.home_dir + '/*']) # this method invocation looks like it is the same as the one above, and that is partially correct. # the arguments to the invocation are the same, but the thing that changes is the lifecycle of the cluster # the client is being created to interact with. This client is specifically for the cluster after the # upgrade has taken place, and can account for any possible settings that may change for the client under # the hood when it probes the cluster. dcos_api_upgrade = self.dcos_api_session_factory_upgrade.apply( 'http://{ip}'.format(ip=cluster.masters[0].public_ip), master_list, master_list, [h.private_ip for h in cluster.agents], [h.private_ip for h in cluster.public_agents], self.default_os_user) dcos_api_upgrade.wait_for_dcos() # here we wait for DC/OS to be "up" so that we can auth this new client self.verify_apps_state(dcos_api_upgrade, dns_app) with logger.scope("run integration tests"): # copied from test_util/test_aws_cf.py:96 add_env = [] prefix = 'TEST_ADD_ENV_' for k, v in os.environ.items(): if k.startswith(prefix): add_env.append(k.replace(prefix, '') + '=' + v) test_cmd = ' '.join(add_env) + ' py.test -vv -s -rs ' + os.getenv('CI_FLAGS', '') result = test_util.cluster.run_integration_tests(cluster, test_cmd=test_cmd) if result == 0: self.log.info("Test successful! Deleting VPC if provided in this run.") vpc.delete() bw.delete_key_pair(stack_name) else: self.log.info("Test failed! VPC cluster will remain available for " "debugging for 2 hour after instantiation.") return result
def make_channel_artifacts(metadata): artifacts = [] # Set logging to debug so we get gen error messages, since those are # logging.DEBUG currently to not show up when people are using `--genconf` # and friends. # TODO(cmaloney): Remove this and make the core bits of gen, code log at # the proper info / warning / etc. level. log = logging.getLogger() original_log_level = log.getEffectiveLevel() log.setLevel(logging.DEBUG) provider_data = {} providers = load_providers() for name, module in sorted(providers.items()): bootstrap_url = metadata['repository_url'] # If the particular provider has its own storage by the same name then # Use the storage provider rather if name in metadata['storage_urls']: bootstrap_url = metadata['storage_urls'][name] + metadata['repository_path'] variant_arguments = dict() for variant, variant_info in metadata['complete_dict'].items(): variant_arguments[variant] = copy.deepcopy({ 'bootstrap_url': bootstrap_url, 'provider': name, 'bootstrap_id': variant_info['bootstrap'], 'bootstrap_variant': pkgpanda.util.variant_prefix(variant), 'package_ids': json.dumps(variant_info['packages']) }) # Load additional default variant arguments out of gen_extra if os.path.exists('gen_extra/calc.py'): mod = importlib.machinery.SourceFileLoader('gen_extra.calc', 'gen_extra/calc.py').load_module() variant_arguments[variant].update(mod.provider_template_defaults) # Add templates for the default variant. # Use keyword args to make not matching ordering a loud error around changes. with logger.scope("Creating {} deploy tools".format(module.__name__)): # TODO(cmaloney): Cleanup by just having this make and pass another source. module_specific_variant_arguments = copy.deepcopy(variant_arguments) for arg_dict in module_specific_variant_arguments.values(): if module.__name__ == 'gen.build_deploy.aws': arg_dict['cloudformation_s3_url_full'] = metadata['cloudformation_s3_url_full'] elif module.__name__ == 'gen.build_deploy.azure': arg_dict['azure_download_url'] = metadata['azure_download_url'] elif module.__name__ == 'gen.build_deploy.bash': pass else: raise NotImplementedError("Unknown how to add args to deploy tool: {}".format(module.__name__)) for built_resource in module.do_create( tag=metadata['tag'], build_name=metadata['build_name'], reproducible_artifact_path=metadata['reproducible_artifact_path'], commit=metadata['commit'], variant_arguments=module_specific_variant_arguments, all_completes=metadata['all_completes']): assert isinstance(built_resource, dict), built_resource # Type switch if 'packages' in built_resource: for package in built_resource['packages']: artifacts.append(get_gen_package_artifact(package)) else: assert 'packages' not in built_resource artifacts.append(built_resource) # TODO(cmaloney): Check the provider artifacts adhere to the artifact template. artifacts += provider_data.get('artifacts', list()) log.setLevel(original_log_level) return artifacts
def make_channel_artifacts(metadata, provider_names): artifacts = [{ 'channel_path': 'version', 'local_content': DCOS_VERSION, 'content_type': 'text/plain; charset=utf-8', }] # Set logging to debug so we get gen error messages, since those are # logging.DEBUG currently to not show up when people are using `--genconf` # and friends. # TODO(cmaloney): Remove this and make the core bits of gen, code log at # the proper info / warning / etc. level. log = logging.getLogger() original_log_level = log.getEffectiveLevel() log.setLevel(logging.DEBUG) provider_data = {} providers = load_providers(provider_names) for name, module in sorted(providers.items()): bootstrap_url = metadata['repository_url'] # If the particular provider has its own storage by the same name then # Use the storage provider rather if name in metadata['storage_urls']: bootstrap_url = metadata['storage_urls'][name] + metadata[ 'repository_path'] variant_arguments = dict() for variant, variant_info in metadata['complete_dict'].items(): variant_arguments[variant] = copy.deepcopy({ 'bootstrap_url': bootstrap_url, 'provider': name, 'bootstrap_id': variant_info['bootstrap'], 'bootstrap_variant': pkgpanda.util.variant_prefix(variant), 'package_ids': json.dumps(variant_info['packages']) }) # Load additional default variant arguments out of gen_extra if os.path.exists('gen_extra/calc.py'): mod = importlib.machinery.SourceFileLoader( 'gen_extra.calc', 'gen_extra/calc.py').load_module() variant_arguments[variant].update( mod.provider_template_defaults) # Add templates for the default variant. # Use keyword args to make not matching ordering a loud error around changes. with logger.scope("Creating {} deploy tools".format(module.__name__)): # TODO(cmaloney): Cleanup by just having this make and pass another source. module_specific_variant_arguments = copy.deepcopy( variant_arguments) for arg_dict in module_specific_variant_arguments.values(): if module.__name__ == 'gen.build_deploy.aws': arg_dict['cloudformation_s3_url_full'] = metadata[ 'cloudformation_s3_url_full'] elif module.__name__ == 'gen.build_deploy.azure': arg_dict['azure_download_url'] = metadata[ 'azure_download_url'] elif module.__name__ == 'gen.build_deploy.bash': pass else: raise NotImplementedError( "Unknown how to add args to deploy tool: {}".format( module.__name__)) for built_resource in module.do_create( tag=metadata['tag'], build_name=metadata['build_name'], reproducible_artifact_path=metadata[ 'reproducible_artifact_path'], commit=metadata['commit'], variant_arguments=module_specific_variant_arguments, all_completes=metadata['all_completes']): assert isinstance(built_resource, dict), built_resource # Type switch if 'packages' in built_resource: for package in built_resource['packages']: artifacts.append(get_gen_package_artifact(package)) else: assert 'packages' not in built_resource artifacts.append(built_resource) # TODO(cmaloney): Check the provider artifacts adhere to the artifact template. artifacts += provider_data.get('artifacts', list()) log.setLevel(original_log_level) return artifacts
def _build(package_store, name, variant, clean_after_build, recursive): assert isinstance(package_store, PackageStore) tmpdir = tempfile.TemporaryDirectory(prefix="pkgpanda_repo") repository = Repository(tmpdir.name) package_dir = package_store.get_package_folder(name) def src_abs(name): return package_dir + '/' + name def cache_abs(filename): return package_store.get_package_cache_folder(name) + '/' + filename # Build pkginfo over time, translating fields from buildinfo. pkginfo = {} # Build up the docker command arguments over time, translating fields as needed. cmd = DockerCmd() assert (name, variant) in package_store.packages, \ "Programming error: name, variant should have been validated to be valid before calling build()." builder = IdBuilder(package_store.get_buildinfo(name, variant)) final_buildinfo = dict() builder.add('name', name) builder.add('variant', pkgpanda.util.variant_str(variant)) # Convert single_source -> sources if builder.has('sources'): if builder.has('single_source'): raise BuildError('Both sources and single_source cannot be specified at the same time') sources = builder.take('sources') elif builder.has('single_source'): sources = {name: builder.take('single_source')} builder.replace('single_source', 'sources', sources) else: builder.add('sources', {}) sources = dict() print("NOTICE: No sources specified") final_buildinfo['sources'] = sources # Construct the source fetchers, gather the checkout ids from them checkout_ids = dict() fetchers = dict() try: for src_name, src_info in sorted(sources.items()): # TODO(cmaloney): Switch to a unified top level cache directory shared by all packages cache_dir = package_store.get_package_cache_folder(name) + '/' + src_name check_call(['mkdir', '-p', cache_dir]) fetcher = get_src_fetcher(src_info, cache_dir, package_dir) fetchers[src_name] = fetcher checkout_ids[src_name] = fetcher.get_id() except ValidationError as ex: raise BuildError("Validation error when fetching sources for package: {}".format(ex)) for src_name, checkout_id in checkout_ids.items(): # NOTE: single_source buildinfo was expanded above so the src_name is # always correct here. # Make sure we never accidentally overwrite something which might be # important. Fields should match if specified (And that should be # tested at some point). For now disallowing identical saves hassle. assert_no_duplicate_keys(checkout_id, final_buildinfo['sources'][src_name]) final_buildinfo['sources'][src_name].update(checkout_id) # Add the sha1 of the buildinfo.json + build file to the build ids builder.update('sources', checkout_ids) build_script = src_abs(builder.take('build_script')) # TODO(cmaloney): Change dest name to build_script_sha1 builder.replace('build_script', 'build', pkgpanda.util.sha1(build_script)) builder.add('pkgpanda_version', pkgpanda.build.constants.version) extra_dir = src_abs("extra") # Add the "extra" folder inside the package as an additional source if it # exists if os.path.exists(extra_dir): extra_id = hash_folder_abs(extra_dir, package_dir) builder.add('extra_source', extra_id) final_buildinfo['extra_source'] = extra_id # Figure out the docker name. docker_name = builder.take('docker') cmd.container = docker_name # Add the id of the docker build environment to the build_ids. try: docker_id = get_docker_id(docker_name) except CalledProcessError: # docker pull the container and try again check_call(['docker', 'pull', docker_name]) docker_id = get_docker_id(docker_name) builder.update('docker', docker_id) # TODO(cmaloney): The environment variables should be generated during build # not live in buildinfo.json. pkginfo['environment'] = builder.take('environment') # Whether pkgpanda should on the host make sure a `/var/lib` state directory is available pkginfo['state_directory'] = builder.take('state_directory') if pkginfo['state_directory'] not in [True, False]: raise BuildError("state_directory in buildinfo.json must be a boolean `true` or `false`") username = None if builder.has('username'): username = builder.take('username') if not isinstance(username, str): raise BuildError("username in buildinfo.json must be either not set (no user for this" " package), or a user name string") try: pkgpanda.UserManagement.validate_username(username) except ValidationError as ex: raise BuildError("username in buildinfo.json didn't meet the validation rules. {}".format(ex)) pkginfo['username'] = username group = None if builder.has('group'): group = builder.take('group') if not isinstance(group, str): raise BuildError("group in buildinfo.json must be either not set (use default group for this user)" ", or group must be a string") try: pkgpanda.UserManagement.validate_group_name(group) except ValidationError as ex: raise BuildError("group in buildinfo.json didn't meet the validation rules. {}".format(ex)) pkginfo['group'] = group # Packages need directories inside the fake install root (otherwise docker # will try making the directories on a readonly filesystem), so build the # install root now, and make the package directories in it as we go. install_dir = tempfile.mkdtemp(prefix="pkgpanda-") active_packages = list() active_package_ids = set() active_package_variants = dict() auto_deps = set() # Final package has the same requires as the build. requires = builder.take('requires') pkginfo['requires'] = requires if builder.has("sysctl"): pkginfo["sysctl"] = builder.take("sysctl") # TODO(cmaloney): Pull generating the full set of requires a function. to_check = copy.deepcopy(requires) if type(to_check) != list: raise BuildError("`requires` in buildinfo.json must be an array of dependencies.") while to_check: requires_info = to_check.pop(0) requires_name, requires_variant = expand_require(requires_info) if requires_name in active_package_variants: # TODO(cmaloney): If one package depends on the <default> # variant of a package and 1+ others depends on a non-<default> # variant then update the dependency to the non-default variant # rather than erroring. if requires_variant != active_package_variants[requires_name]: # TODO(cmaloney): Make this contain the chains of # dependencies which contain the conflicting packages. # a -> b -> c -> d {foo} # e {bar} -> d {baz} raise BuildError( "Dependncy on multiple variants of the same package {}. variants: {} {}".format( requires_name, requires_variant, active_package_variants[requires_name])) # The variant has package {requires_name, variant} already is a # dependency, don't process it again / move on to the next. continue active_package_variants[requires_name] = requires_variant # Figure out the last build of the dependency, add that as the # fully expanded dependency. requires_last_build = package_store.get_last_build_filename(requires_name, requires_variant) if not os.path.exists(requires_last_build): if recursive: # Build the dependency build(package_store, requires_name, requires_variant, clean_after_build, recursive) else: raise BuildError("No last build file found for dependency {} variant {}. Rebuild " "the dependency".format(requires_name, requires_variant)) try: pkg_id_str = load_string(requires_last_build) auto_deps.add(pkg_id_str) pkg_buildinfo = package_store.get_buildinfo(requires_name, requires_variant) pkg_requires = pkg_buildinfo['requires'] pkg_path = repository.package_path(pkg_id_str) pkg_tar = pkg_id_str + '.tar.xz' if not os.path.exists(package_store.get_package_cache_folder(requires_name) + '/' + pkg_tar): raise BuildError( "The build tarball {} refered to by the last_build file of the dependency {} " "variant {} doesn't exist. Rebuild the dependency.".format( pkg_tar, requires_name, requires_variant)) active_package_ids.add(pkg_id_str) # Mount the package into the docker container. cmd.volumes[pkg_path] = "/opt/mesosphere/packages/{}:ro".format(pkg_id_str) os.makedirs(os.path.join(install_dir, "packages/{}".format(pkg_id_str))) # Add the dependencies of the package to the set which will be # activated. # TODO(cmaloney): All these 'transitive' dependencies shouldn't # be available to the package being built, only what depends on # them directly. to_check += pkg_requires except ValidationError as ex: raise BuildError("validating package needed as dependency {0}: {1}".format(requires_name, ex)) from ex except PackageError as ex: raise BuildError("loading package needed as dependency {0}: {1}".format(requires_name, ex)) from ex # Add requires to the package id, calculate the final package id. # NOTE: active_packages isn't fully constructed here since we lazily load # packages not already in the repository. builder.update('requires', list(active_package_ids)) version_extra = None if builder.has('version_extra'): version_extra = builder.take('version_extra') build_ids = builder.get_build_ids() version_base = hash_checkout(build_ids) version = None if builder.has('version_extra'): version = "{0}-{1}".format(version_extra, version_base) else: version = version_base pkg_id = PackageId.from_parts(name, version) # Everything must have been extracted by now. If it wasn't, then we just # had a hard error that it was set but not used, as well as didn't include # it in the caluclation of the PackageId. builder = None # Save the build_ids. Useful for verify exactly what went into the # package build hash. final_buildinfo['build_ids'] = build_ids final_buildinfo['package_version'] = version # Save the package name and variant. The variant is used when installing # packages to validate dependencies. final_buildinfo['name'] = name final_buildinfo['variant'] = variant # If the package is already built, don't do anything. pkg_path = package_store.get_package_cache_folder(name) + '/{}.tar.xz'.format(pkg_id) # Done if it exists locally if exists(pkg_path): print("Package up to date. Not re-building.") # TODO(cmaloney): Updating / filling last_build should be moved out of # the build function. write_string(package_store.get_last_build_filename(name, variant), str(pkg_id)) return pkg_path # Try downloading. dl_path = package_store.try_fetch_by_id(pkg_id) if dl_path: print("Package up to date. Not re-building. Downloaded from repository-url.") # TODO(cmaloney): Updating / filling last_build should be moved out of # the build function. write_string(package_store.get_last_build_filename(name, variant), str(pkg_id)) print(dl_path, pkg_path) assert dl_path == pkg_path return pkg_path # Fall out and do the build since it couldn't be downloaded print("Unable to download from cache. Proceeding to build") print("Building package {} with buildinfo: {}".format( pkg_id, json.dumps(final_buildinfo, indent=2, sort_keys=True))) # Clean out src, result so later steps can use them freely for building. def clean(): # Run a docker container to remove src/ and result/ cmd = DockerCmd() cmd.volumes = { package_store.get_package_cache_folder(name): "/pkg/:rw", } cmd.container = "ubuntu:14.04.4" cmd.run("package-cleaner", ["rm", "-rf", "/pkg/src", "/pkg/result"]) clean() # Only fresh builds are allowed which don't overlap existing artifacts. result_dir = cache_abs("result") if exists(result_dir): raise BuildError("result folder must not exist. It will be made when the package is " "built. {}".format(result_dir)) # 'mkpanda add' all implicit dependencies since we actually need to build. for dep in auto_deps: print("Auto-adding dependency: {}".format(dep)) # NOTE: Not using the name pkg_id because that overrides the outer one. id_obj = PackageId(dep) add_package_file(repository, package_store.get_package_path(id_obj)) package = repository.load(dep) active_packages.append(package) # Checkout all the sources int their respective 'src/' folders. try: src_dir = cache_abs('src') if os.path.exists(src_dir): raise ValidationError( "'src' directory already exists, did you have a previous build? " + "Currently all builds must be from scratch. Support should be " + "added for re-using a src directory when possible. src={}".format(src_dir)) os.mkdir(src_dir) for src_name, fetcher in sorted(fetchers.items()): root = cache_abs('src/' + src_name) os.mkdir(root) fetcher.checkout_to(root) except ValidationError as ex: raise BuildError("Validation error when fetching sources for package: {}".format(ex)) # Activate the packages so that we have a proper path, environment # variables. # TODO(cmaloney): RAII type thing for temproary directory so if we # don't get all the way through things will be cleaned up? install = Install( root=install_dir, config_dir=None, rooted_systemd=True, manage_systemd=False, block_systemd=True, fake_path=True, manage_users=False, manage_state_dir=False) install.activate(active_packages) # Rewrite all the symlinks inside the active path because we will # be mounting the folder into a docker container, and the absolute # paths to the packages will change. # TODO(cmaloney): This isn't very clean, it would be much nicer to # just run pkgpanda inside the package. rewrite_symlinks(install_dir, repository.path, "/opt/mesosphere/packages/") print("Building package in docker") # TODO(cmaloney): Run as a specific non-root user, make it possible # for non-root to cleanup afterwards. # Run the build, prepping the environment as necessary. mkdir(cache_abs("result")) # Copy the build info to the resulting tarball write_json(cache_abs("src/buildinfo.full.json"), final_buildinfo) write_json(cache_abs("result/buildinfo.full.json"), final_buildinfo) write_json(cache_abs("result/pkginfo.json"), pkginfo) # Make the folder for the package we are building. If docker does it, it # gets auto-created with root permissions and we can't actually delete it. os.makedirs(os.path.join(install_dir, "packages", str(pkg_id))) # TOOD(cmaloney): Disallow writing to well known files and directories? # Source we checked out cmd.volumes.update({ # TODO(cmaloney): src should be read only... cache_abs("src"): "/pkg/src:rw", # The build script build_script: "/pkg/build:ro", # Getting the result out cache_abs("result"): "/opt/mesosphere/packages/{}:rw".format(pkg_id), install_dir: "/opt/mesosphere:ro" }) if os.path.exists(extra_dir): cmd.volumes[extra_dir] = "/pkg/extra:ro" cmd.environment = { "PKG_VERSION": version, "PKG_NAME": name, "PKG_ID": pkg_id, "PKG_PATH": "/opt/mesosphere/packages/{}".format(pkg_id), "PKG_VARIANT": variant if variant is not None else "<default>", "NUM_CORES": multiprocessing.cpu_count() } try: # TODO(cmaloney): Run a wrapper which sources # /opt/mesosphere/environment then runs a build. Also should fix # ownership of /opt/mesosphere/packages/{pkg_id} post build. cmd.run("package-builder", [ "/bin/bash", "-o", "nounset", "-o", "pipefail", "-o", "errexit", "/pkg/build"]) except CalledProcessError as ex: raise BuildError("docker exited non-zero: {}\nCommand: {}".format(ex.returncode, ' '.join(ex.cmd))) # Clean up the temporary install dir used for dependencies. # TODO(cmaloney): Move to an RAII wrapper. check_call(['rm', '-rf', install_dir]) with logger.scope("Build package tarball"): # Check for forbidden services before packaging the tarball: try: check_forbidden_services(cache_abs("result"), RESERVED_UNIT_NAMES) except ValidationError as ex: raise BuildError("Package validation failed: {}".format(ex)) # TODO(cmaloney): Updating / filling last_build should be moved out of # the build function. write_string(package_store.get_last_build_filename(name, variant), str(pkg_id)) # Bundle the artifacts into the pkgpanda package tmp_name = pkg_path + "-tmp.tar.xz" make_tar(tmp_name, cache_abs("result")) os.rename(tmp_name, pkg_path) print("Package built.") if clean_after_build: clean() return pkg_path
def do_build_docker(name, path): with logger.scope("dcos/dcos-builder ({})".format(name)): return _do_build_docker(name, path)
def make_stable_artifacts(cache_repository_url, tree_variants): metadata = { "commit": util.dcos_image_commit, "core_artifacts": [], "packages": set() } # TODO(cmaloney): Rather than guessing / reverse-engineering all these paths # have do_build_packages get them directly from pkgpanda with logger.scope("Building packages"): try: all_completes = do_build_packages(cache_repository_url, tree_variants) except pkgpanda.build.BuildError as ex: logger.error("Failure building package(s): {}".format(ex)) raise # The installer and util are built bootstraps, but not a DC/OS variants. We use # iteration over the complete_dict to enumerate all variants a whole lot, # so explicity remove installer/util here so people don't accidentally hit it. # TODO: make this into a tree option complete_dict = dict() for name, info in copy.copy(all_completes).items(): if name is not None and (name.endswith('installer') or name.endswith('util')): continue complete_dict[name] = info metadata["complete_dict"] = complete_dict metadata["all_completes"] = all_completes metadata["bootstrap_dict"] = { k: v['bootstrap'] for k, v in complete_dict.items() } metadata["all_bootstraps"] = { k: v['bootstrap'] for k, v in all_completes.items() } def add_file(info): metadata["core_artifacts"].append(info) def add_package(package_id): if package_id in metadata['packages']: return metadata['packages'].add(package_id) add_file(get_package_artifact(package_id)) # Add the bootstrap, active.json, packages as reproducible_path artifacts # Add the <variant>.bootstrap.latest as a channel_path for name, info in sorted(all_completes.items(), key=lambda kv: pkgpanda.util.variant_str(kv[0])): for file in make_bootstrap_artifacts(info['bootstrap'], info['packages'], name, 'packages/cache'): add_file(file) # Add all the packages which haven't been added yet for package_id in sorted(info['packages']): add_package(package_id) # Sets aren't json serializable, so transform to a list for future use. metadata['packages'] = list(sorted(metadata['packages'])) return metadata
def verify_apps_state(self, dcos_api: DcosApiSession, dns_app: dict): with logger.scope("verify apps state"): # nested methods here so we can "close" over external state def marathon_app_tasks_survive_upgrade(): # Verify that the tasks we started are still running. tasks_end = { app_id: sorted(self.app_task_ids(dcos_api, app_id)) for app_id in self.test_app_ids } self.log.debug('Test app tasks at end:\n' + pprint.pformat(tasks_end)) if not self.tasks_start == tasks_end: self.teamcity_msg.testFailed( "test_upgrade_vpc.marathon_app_tasks_survive_upgrade", details="expected: {}\nactual: {}".format( self.tasks_start, tasks_end)) def test_mesos_task_state_remains_consistent(): # Verify that the "state" of the task does not change. task_state_end = self.get_master_task_state( dcos_api, self.tasks_start[0]) if not self.task_state_start == task_state_end: self.teamcity_msg.testFailed( "test_upgrade_vpc.test_mesos_task_state_remains_consistent", details="expected: {}\nactual: {}".format( self.task_state_start, task_state_end)) def test_app_dns_survive_upgrade(): # Verify DNS didn't fail. marathon_framework_id = dcos_api.marathon.get( '/v2/info').json()['frameworkId'] dns_app_task = dcos_api.marathon.get( '/v2/apps' + dns_app['id'] + '/tasks').json()['tasks'][0] dns_log = self.parse_dns_log( dcos_api.mesos_sandbox_file( dns_app_task['slaveId'], marathon_framework_id, dns_app_task['id'], dns_app['env']['DNS_LOG_FILENAME'], )) dns_failure_times = [ entry[0] for entry in dns_log if entry[1] != 'SUCCESS' ] if len(dns_failure_times) > 0: message = 'Failed to resolve Marathon app hostname {} at least once.'.format( dns_app['env']['RESOLVE_NAME']) err_msg = message + ' Hostname failed to resolve at these times:\n' + '\n'.join( dns_failure_times) self.log.debug(err_msg) self.teamcity_msg.testFailed( "test_upgrade_vpc.test_app_dns_survive_upgrade", details=err_msg) self.log_test( "test_upgrade_vpc.marathon_app_tasks_survive_upgrade", marathon_app_tasks_survive_upgrade) self.log_test( "test_upgrade_vpc.test_mesos_task_state_remains_consistent", test_mesos_task_state_remains_consistent) self.log_test("test_upgrade_vpc.test_app_dns_survive_upgrade", test_app_dns_survive_upgrade)
def do_build_docker(name, path): with logger.scope("dcos/dcos-builder ({})".format(name)): return _do_build_docker(name, path)
def make_channel_artifacts(metadata): artifacts = [] # Set logging to debug so we get gen error messages, since those are # logging.DEBUG currently to not show up when people are using `--genconf` # and friends. # TODO(cmaloney): Remove this and make the core bits of gen, code log at # the proper info / warning / etc. level. log = logging.getLogger() original_log_level = log.getEffectiveLevel() log.setLevel(logging.DEBUG) provider_data = {} providers = load_providers() for name, module in sorted(providers.items()): bootstrap_url = metadata['repository_url'] # If the particular provider has its own storage by the same name then # Use the storage provider rather if name in metadata['storage_urls']: bootstrap_url = metadata['storage_urls'][name] + metadata['repository_path'] variant_arguments = dict() for bootstrap_name, bootstrap_id in metadata['bootstrap_dict'].items(): variant_arguments[bootstrap_name] = copy.deepcopy({ 'bootstrap_url': bootstrap_url, 'provider': name, 'bootstrap_id': bootstrap_id, 'bootstrap_variant': pkgpanda.util.variant_prefix(bootstrap_name) }) # Load additional default variant arguments out of gen_extra if os.path.exists('gen_extra/calc.py'): mod = importlib.machinery.SourceFileLoader('gen_extra.calc', 'gen_extra/calc.py').load_module() variant_arguments[bootstrap_name].update(mod.provider_template_defaults) # Add templates for the default variant. # Use keyword args to make not matching ordering a loud error around changes. with logger.scope("Creating {} deploy tools".format(module.__name__)): for built_resource in module.do_create( tag=metadata['tag'], build_name=metadata['build_name'], reproducible_artifact_path=metadata['reproducible_artifact_path'], commit=metadata['commit'], variant_arguments=variant_arguments, all_bootstraps=metadata["all_bootstraps"]): assert isinstance(built_resource, dict), built_resource # Type switch if 'packages' in built_resource: for package in built_resource['packages']: artifacts.append(get_gen_package_artifact(package)) else: assert 'packages' not in built_resource artifacts.append(built_resource) # TODO(cmaloney): Check the provider artifacts adhere to the artifact template. artifacts += provider_data.get('artifacts', list()) log.setLevel(original_log_level) return artifacts