def do_gen_package(config, package_filename): # Generate the specific dcos-config package. # Version will be setup-{sha1 of contents} with gen.util.pkgpanda_package_tmpdir() as tmpdir: # Only contains package, root assert config.keys() == {"package"} # Write out the individual files for file_info in config["package"]: assert file_info.keys() <= {"path", "content", "permissions"} if is_absolute_path(file_info['path']): fileinfo_drive, fileinfo_path = os.path.splitdrive( file_info['path']) path = tmpdir + fileinfo_path else: path = tmpdir + '/' + file_info['path'] try: if os.path.dirname(path): os.makedirs(os.path.dirname(path), mode=0o755) except FileExistsError: pass with open(path, 'w') as f: f.write(file_info['content'] or '') # the file has special mode defined, handle that. if 'permissions' in file_info: assert isinstance(file_info['permissions'], str) os.chmod(path, int(file_info['permissions'], 8)) else: os.chmod(path, 0o644) gen.util.make_pkgpanda_package(tmpdir, package_filename)
def do_gen_package(config, package_filename): # Generate the specific dcos-config package. # Version will be setup-{sha1 of contents} with gen.util.pkgpanda_package_tmpdir() as tmpdir: # Only contains package, root assert config.keys() == {"package"} # Write out the individual files for file_info in config["package"]: assert file_info.keys() <= {"path", "content", "permissions"} if is_absolute_path(file_info['path']): fileinfo_drive, fileinfo_path = os.path.splitdrive(file_info['path']) path = tmpdir + fileinfo_path else: path = tmpdir + '/' + file_info['path'] try: if os.path.dirname(path): os.makedirs(os.path.dirname(path), mode=0o755) except FileExistsError: pass with open(path, 'w') as f: f.write(file_info['content'] or '') # the file has special mode defined, handle that. if 'permissions' in file_info: assert isinstance(file_info['permissions'], str) os.chmod(path, int(file_info['permissions'], 8)) else: os.chmod(path, 0o644) gen.util.make_pkgpanda_package(tmpdir, package_filename)
def build_late_package(late_files, config_id, provider): if not late_files: return None # Add a empty pkginfo.json to the late package after validating there # isn't already one. for file_info in late_files: assert file_info['path'] != '/pkginfo.json' assert is_absolute_path(file_info['path']) late_files.append({"path": "/pkginfo.json", "content": "{}"}) return { 'package': late_files, 'name': 'dcos-provider-{}-{}--setup'.format(config_id, provider) }
def build_late_package(late_files, config_id, provider): if not late_files: return None # Add a empty pkginfo.json to the late package after validating there # isn't already one. for file_info in late_files: assert file_info['path'] != '/pkginfo.json' assert is_absolute_path(file_info['path']) late_files.append({ "path": "/pkginfo.json", "content": "{}"}) return { 'package': late_files, 'name': 'dcos-provider-{}-{}--setup'.format(config_id, provider) }
def generate(arguments, extra_templates=list(), extra_sources=list(), extra_targets=list()): # To maintain the old API where we passed arguments rather than the new name. user_arguments = arguments arguments = None sources, targets, templates = get_dcosconfig_source_target_and_templates( user_arguments, extra_templates, extra_sources) resolver = validate_and_raise(sources, targets + extra_targets) argument_dict = get_final_arguments(resolver) late_variables = get_late_variables(resolver, sources) secret_builtins = [ 'expanded_config_full', 'user_arguments_full', 'config_yaml_full' ] secret_variables = set(get_secret_variables(sources) + secret_builtins) masked_value = '**HIDDEN**' # Calculate config ID after all variables are resolved, to make sure any change in config yields a new config ID. config_id = get_config_id(argument_dict) # Calculate values that depend on the config ID. config_package_names = json.loads(argument_dict['config_package_names']) package_ids = json.loads(argument_dict['package_ids']) config_package_ids = [ '{}--setup_{}'.format(name, config_id) for name in config_package_names ] cluster_packages = sorted(config_package_ids + package_ids) validate_cluster_packages(cluster_packages) cluster_package_list_id = hash_checkout(cluster_packages) # Calculate values for builtin variables. argument_dict['cluster_packages'] = json.dumps(cluster_packages) argument_dict['cluster_package_list_id'] = cluster_package_list_id user_arguments_masked = { k: (masked_value if k in secret_variables else v) for k, v in user_arguments.items() } argument_dict['user_arguments_full'] = json_prettyprint(user_arguments) argument_dict['user_arguments'] = json_prettyprint(user_arguments_masked) argument_dict['config_yaml_full'] = user_arguments_to_yaml(user_arguments) argument_dict['config_yaml'] = user_arguments_to_yaml( user_arguments_masked) # The expanded_config and expanded_config_full variables contain all other variables and their values. # expanded_config is a copy of expanded_config_full with secret values removed. Calculating these variables' values # must come after the calculation of all other variables to prevent infinite recursion. # TODO(cmaloney): Make this late-bound by gen.internals expanded_config_full = { k: v for k, v in argument_dict.items() # Omit late-bound variables whose values have not yet been calculated. if not v.startswith(gen.internals.LATE_BIND_PLACEHOLDER_START) } expanded_config_scrubbed = { k: v for k, v in expanded_config_full.items() if k not in secret_variables } argument_dict['expanded_config_full'] = format_expanded_config( expanded_config_full) argument_dict['expanded_config'] = format_expanded_config( expanded_config_scrubbed) # Initialize CA and add arguments (exhibitor_ca_certificate and exhibitor_ca_certificate_path) gen.exhibitor_tls_bootstrap.initialize_exhibitor_ca(argument_dict) log.debug("Final arguments:" + json_prettyprint({ # Mask secret config values. k: (masked_value if k in secret_variables else v) for k, v in argument_dict.items() })) # Fill in the template parameters # TODO(cmaloney): render_templates should ideally take the template targets. rendered_templates = render_templates(templates, argument_dict) # Validate there aren't any unexpected top level directives in any of the files # (likely indicates a misspelling) for name, template in rendered_templates.items(): if name == dcos_services_yaml: # yaml list of the service files assert isinstance(template, list) elif name == cloud_config_yaml: assert template.keys() <= CLOUDCONFIG_KEYS, template.keys() elif isinstance(template, str): # Not a yaml template pass else: # yaml template file log.debug("validating template file %s", name) assert template.keys() <= PACKAGE_KEYS, template.keys() stable_artifacts = [] channel_artifacts = [] # Find all files which contain late bind variables and turn them into a "late bind package" # TODO(cmaloney): check there are no late bound variables in cloud-config.yaml late_files, regular_files = extract_files_containing_late_variables( rendered_templates[dcos_config_yaml]['package']) # put the regular files right back rendered_templates[dcos_config_yaml] = {'package': regular_files} # Render cluster package list artifact. cluster_package_list_filename = 'package_lists/{}.package_list.json'.format( cluster_package_list_id) os.makedirs(os.path.dirname(cluster_package_list_filename), mode=0o755, exist_ok=True) write_string(cluster_package_list_filename, argument_dict['cluster_packages']) log.info('Cluster package list: {}'.format(cluster_package_list_filename)) stable_artifacts.append(cluster_package_list_filename) def make_package_filename(package_id, extension): return 'packages/{0}/{1}{2}'.format(package_id.name, repr(package_id), extension) # Render all the cluster packages cluster_package_info = {} # Prepare late binding config, if any. late_package = build_late_package(late_files, config_id, argument_dict['provider']) if late_variables and late_package: # Render the late binding package. This package will be downloaded onto # each cluster node during bootstrap and rendered into the final config # using the values from the late config file. late_package_id = PackageId(late_package['name']) late_package_filename = make_package_filename(late_package_id, '.dcos_config') os.makedirs(os.path.dirname(late_package_filename), mode=0o755) write_yaml(late_package_filename, {'package': late_package['package']}, default_flow_style=False) log.info('Package filename: {}'.format(late_package_filename)) stable_artifacts.append(late_package_filename) # Add the late config file to cloud config. The expressions in # late_variables will be resolved by the service handling the cloud # config (e.g. Amazon CloudFormation). The rendered late config file # on a cluster node's filesystem will contain the final values. rendered_templates[cloud_config_yaml]['root'].append({ 'path': config_dir + '/setup-flags/late-config.yaml', 'permissions': '0644', 'owner': 'root', # TODO(cmaloney): don't prettyprint to save bytes. # NOTE: Use yaml here simply to make avoiding painful escaping and # unescaping easier. 'content': render_yaml({ 'late_bound_package_id': late_package['name'], 'bound_values': late_variables }) }) # Collect metadata for cluster packages. for package_id_str in cluster_packages: package_id = PackageId(package_id_str) package_filename = make_package_filename(package_id, '.tar.xz') cluster_package_info[package_id.name] = { 'id': package_id_str, 'filename': package_filename } # Render config packages. for package_id_str in config_package_ids: package_id = PackageId(package_id_str) package_filename = cluster_package_info[package_id.name]['filename'] do_gen_package(rendered_templates[package_id.name + '.yaml'], cluster_package_info[package_id.name]['filename']) stable_artifacts.append(package_filename) # Convert cloud-config to just contain write_files rather than root cc = rendered_templates[cloud_config_yaml] # Shouldn't contain any packages. Providers should pull what they need to # late bind out of other packages via cc_package_file. assert 'package' not in cc cc_root = cc.pop('root', []) # Make sure write_files exists. assert 'write_files' not in cc cc['write_files'] = [] # Do the transform for item in cc_root: assert is_absolute_path(item['path']) cc['write_files'].append(item) rendered_templates[cloud_config_yaml] = cc # Add utils that need to be defined here so they can be bound to locals. def add_services(cloudconfig, cloud_init_implementation): return add_units(cloudconfig, rendered_templates[dcos_services_yaml], cloud_init_implementation) utils.add_services = add_services def add_stable_artifact(filename): assert filename not in stable_artifacts + channel_artifacts stable_artifacts.append(filename) utils.add_stable_artifact = add_stable_artifact def add_channel_artifact(filename): assert filename not in stable_artifacts + channel_artifacts channel_artifacts.append(filename) utils.add_channel_artifact = add_channel_artifact return Bunch({ 'arguments': argument_dict, 'cluster_packages': cluster_package_info, 'stable_artifacts': stable_artifacts, 'channel_artifacts': channel_artifacts, 'templates': rendered_templates, 'utils': utils })
def exists(self, path): assert not is_absolute_path(path) return os.path.exists(self.__full_path(path))
def generate( arguments, extra_templates=list(), extra_sources=list(), extra_targets=list()): # To maintain the old API where we passed arguments rather than the new name. user_arguments = arguments arguments = None sources, targets, templates = get_dcosconfig_source_target_and_templates( user_arguments, extra_templates, extra_sources) resolver = validate_and_raise(sources, targets + extra_targets) argument_dict = get_final_arguments(resolver) late_variables = get_late_variables(resolver, sources) secret_builtins = ['expanded_config_full', 'user_arguments_full', 'config_yaml_full'] secret_variables = set(get_secret_variables(sources) + secret_builtins) masked_value = '**HIDDEN**' # Calculate config ID after all variables are resolved, to make sure any change in config yields a new config ID. config_id = get_config_id(argument_dict) # Calculate values that depend on the config ID. config_package_names = json.loads(argument_dict['config_package_names']) package_ids = json.loads(argument_dict['package_ids']) config_package_ids = ['{}--setup_{}'.format(name, config_id) for name in config_package_names] cluster_packages = sorted(config_package_ids + package_ids) validate_cluster_packages(cluster_packages) cluster_package_list_id = hash_checkout(cluster_packages) # Calculate values for builtin variables. argument_dict['cluster_packages'] = json.dumps(cluster_packages) argument_dict['cluster_package_list_id'] = cluster_package_list_id user_arguments_masked = {k: (masked_value if k in secret_variables else v) for k, v in user_arguments.items()} argument_dict['user_arguments_full'] = json_prettyprint(user_arguments) argument_dict['user_arguments'] = json_prettyprint(user_arguments_masked) argument_dict['config_yaml_full'] = user_arguments_to_yaml(user_arguments) argument_dict['config_yaml'] = user_arguments_to_yaml(user_arguments_masked) # The expanded_config and expanded_config_full variables contain all other variables and their values. # expanded_config is a copy of expanded_config_full with secret values removed. Calculating these variables' values # must come after the calculation of all other variables to prevent infinite recursion. # TODO(cmaloney): Make this late-bound by gen.internals expanded_config_full = { k: v for k, v in argument_dict.items() # Omit late-bound variables whose values have not yet been calculated. if not v.startswith(gen.internals.LATE_BIND_PLACEHOLDER_START) } expanded_config_scrubbed = {k: v for k, v in expanded_config_full.items() if k not in secret_variables} argument_dict['expanded_config_full'] = format_expanded_config(expanded_config_full) argument_dict['expanded_config'] = format_expanded_config(expanded_config_scrubbed) log.debug( "Final arguments:" + json_prettyprint({ # Mask secret config values. k: (masked_value if k in secret_variables else v) for k, v in argument_dict.items() }) ) # Fill in the template parameters # TODO(cmaloney): render_templates should ideally take the template targets. rendered_templates = render_templates(templates, argument_dict) # Validate there aren't any unexpected top level directives in any of the files # (likely indicates a misspelling) for name, template in rendered_templates.items(): if name == dcos_services_yaml: # yaml list of the service files assert isinstance(template, list) elif name == cloud_config_yaml: assert template.keys() <= CLOUDCONFIG_KEYS, template.keys() elif isinstance(template, str): # Not a yaml template pass else: # yaml template file log.debug("validating template file %s", name) assert template.keys() <= PACKAGE_KEYS, template.keys() stable_artifacts = [] channel_artifacts = [] # Find all files which contain late bind variables and turn them into a "late bind package" # TODO(cmaloney): check there are no late bound variables in cloud-config.yaml late_files, regular_files = extract_files_containing_late_variables( rendered_templates[dcos_config_yaml]['package']) # put the regular files right back rendered_templates[dcos_config_yaml] = {'package': regular_files} # Render cluster package list artifact. cluster_package_list_filename = 'package_lists/{}.package_list.json'.format(cluster_package_list_id) os.makedirs(os.path.dirname(cluster_package_list_filename), mode=0o755, exist_ok=True) write_string(cluster_package_list_filename, argument_dict['cluster_packages']) log.info('Cluster package list: {}'.format(cluster_package_list_filename)) stable_artifacts.append(cluster_package_list_filename) def make_package_filename(package_id, extension): return 'packages/{0}/{1}{2}'.format( package_id.name, repr(package_id), extension) # Render all the cluster packages cluster_package_info = {} # Prepare late binding config, if any. late_package = build_late_package(late_files, config_id, argument_dict['provider']) if late_variables and late_package: # Render the late binding package. This package will be downloaded onto # each cluster node during bootstrap and rendered into the final config # using the values from the late config file. late_package_id = PackageId(late_package['name']) late_package_filename = make_package_filename(late_package_id, '.dcos_config') os.makedirs(os.path.dirname(late_package_filename), mode=0o755) write_yaml(late_package_filename, {'package': late_package['package']}, default_flow_style=False) log.info('Package filename: {}'.format(late_package_filename)) stable_artifacts.append(late_package_filename) # Add the late config file to cloud config. The expressions in # late_variables will be resolved by the service handling the cloud # config (e.g. Amazon CloudFormation). The rendered late config file # on a cluster node's filesystem will contain the final values. rendered_templates[cloud_config_yaml]['root'].append({ 'path': config_dir + '/setup-flags/late-config.yaml', 'permissions': '0644', 'owner': 'root', # TODO(cmaloney): don't prettyprint to save bytes. # NOTE: Use yaml here simply to make avoiding painful escaping and # unescaping easier. 'content': render_yaml({ 'late_bound_package_id': late_package['name'], 'bound_values': late_variables })}) # Collect metadata for cluster packages. for package_id_str in cluster_packages: package_id = PackageId(package_id_str) package_filename = make_package_filename(package_id, '.tar.xz') cluster_package_info[package_id.name] = { 'id': package_id_str, 'filename': package_filename } # Render config packages. for package_id_str in config_package_ids: package_id = PackageId(package_id_str) package_filename = cluster_package_info[package_id.name]['filename'] do_gen_package(rendered_templates[package_id.name + '.yaml'], cluster_package_info[package_id.name]['filename']) stable_artifacts.append(package_filename) # Convert cloud-config to just contain write_files rather than root cc = rendered_templates[cloud_config_yaml] # Shouldn't contain any packages. Providers should pull what they need to # late bind out of other packages via cc_package_file. assert 'package' not in cc cc_root = cc.pop('root', []) # Make sure write_files exists. assert 'write_files' not in cc cc['write_files'] = [] # Do the transform for item in cc_root: assert is_absolute_path(item['path']) cc['write_files'].append(item) rendered_templates[cloud_config_yaml] = cc # Add utils that need to be defined here so they can be bound to locals. def add_services(cloudconfig, cloud_init_implementation): return add_units(cloudconfig, rendered_templates[dcos_services_yaml], cloud_init_implementation) utils.add_services = add_services def add_stable_artifact(filename): assert filename not in stable_artifacts + channel_artifacts stable_artifacts.append(filename) utils.add_stable_artifact = add_stable_artifact def add_channel_artifact(filename): assert filename not in stable_artifacts + channel_artifacts channel_artifacts.append(filename) utils.add_channel_artifact = add_channel_artifact return Bunch({ 'arguments': argument_dict, 'cluster_packages': cluster_package_info, 'stable_artifacts': stable_artifacts, 'channel_artifacts': channel_artifacts, 'templates': rendered_templates, 'utils': utils })