def do_gen_package(config, package_filename):
# Generate the specific dcos-config package.
# Version will be setup-{sha1 of contents}
with gen.util.pkgpanda_package_tmpdir() as tmpdir:
# Only contains package, root
assert config.keys() == {"package"}
# Write out the individual files
for file_info in config["package"]:
assert file_info.keys() <= {"path", "content", "permissions"}
if is_absolute_path(file_info['path']):
fileinfo_drive, fileinfo_path = os.path.splitdrive(
file_info['path'])
path = tmpdir + fileinfo_path
else:
path = tmpdir + '/' + file_info['path']
try:
if os.path.dirname(path):
os.makedirs(os.path.dirname(path), mode=0o755)
except FileExistsError:
pass
with open(path, 'w') as f:
f.write(file_info['content'] or '')
# the file has special mode defined, handle that.
if 'permissions' in file_info:
assert isinstance(file_info['permissions'], str)
os.chmod(path, int(file_info['permissions'], 8))
else:
os.chmod(path, 0o644)
gen.util.make_pkgpanda_package(tmpdir, package_filename)
def do_gen_package(config, package_filename):
# Generate the specific dcos-config package.
# Version will be setup-{sha1 of contents}
with gen.util.pkgpanda_package_tmpdir() as tmpdir:
# Only contains package, root
assert config.keys() == {"package"}
# Write out the individual files
for file_info in config["package"]:
assert file_info.keys() <= {"path", "content", "permissions"}
if is_absolute_path(file_info['path']):
fileinfo_drive, fileinfo_path = os.path.splitdrive(file_info['path'])
path = tmpdir + fileinfo_path
else:
path = tmpdir + '/' + file_info['path']
try:
if os.path.dirname(path):
os.makedirs(os.path.dirname(path), mode=0o755)
except FileExistsError:
pass
with open(path, 'w') as f:
f.write(file_info['content'] or '')
# the file has special mode defined, handle that.
if 'permissions' in file_info:
assert isinstance(file_info['permissions'], str)
os.chmod(path, int(file_info['permissions'], 8))
else:
os.chmod(path, 0o644)
gen.util.make_pkgpanda_package(tmpdir, package_filename)
def build_late_package(late_files, config_id, provider):
if not late_files:
return None
# Add a empty pkginfo.json to the late package after validating there
# isn't already one.
for file_info in late_files:
assert file_info['path'] != '/pkginfo.json'
assert is_absolute_path(file_info['path'])
late_files.append({"path": "/pkginfo.json", "content": "{}"})
return {
'package': late_files,
'name': 'dcos-provider-{}-{}--setup'.format(config_id, provider)
}
def build_late_package(late_files, config_id, provider):
if not late_files:
return None
# Add a empty pkginfo.json to the late package after validating there
# isn't already one.
for file_info in late_files:
assert file_info['path'] != '/pkginfo.json'
assert is_absolute_path(file_info['path'])
late_files.append({
"path": "/pkginfo.json",
"content": "{}"})
return {
'package': late_files,
'name': 'dcos-provider-{}-{}--setup'.format(config_id, provider)
}
def generate(arguments,
extra_templates=list(),
extra_sources=list(),
extra_targets=list()):
# To maintain the old API where we passed arguments rather than the new name.
user_arguments = arguments
arguments = None
sources, targets, templates = get_dcosconfig_source_target_and_templates(
user_arguments, extra_templates, extra_sources)
resolver = validate_and_raise(sources, targets + extra_targets)
argument_dict = get_final_arguments(resolver)
late_variables = get_late_variables(resolver, sources)
secret_builtins = [
'expanded_config_full', 'user_arguments_full', 'config_yaml_full'
]
secret_variables = set(get_secret_variables(sources) + secret_builtins)
masked_value = '**HIDDEN**'
# Calculate config ID after all variables are resolved, to make sure any change in config yields a new config ID.
config_id = get_config_id(argument_dict)
# Calculate values that depend on the config ID.
config_package_names = json.loads(argument_dict['config_package_names'])
package_ids = json.loads(argument_dict['package_ids'])
config_package_ids = [
'{}--setup_{}'.format(name, config_id) for name in config_package_names
]
cluster_packages = sorted(config_package_ids + package_ids)
validate_cluster_packages(cluster_packages)
cluster_package_list_id = hash_checkout(cluster_packages)
# Calculate values for builtin variables.
argument_dict['cluster_packages'] = json.dumps(cluster_packages)
argument_dict['cluster_package_list_id'] = cluster_package_list_id
user_arguments_masked = {
k: (masked_value if k in secret_variables else v)
for k, v in user_arguments.items()
}
argument_dict['user_arguments_full'] = json_prettyprint(user_arguments)
argument_dict['user_arguments'] = json_prettyprint(user_arguments_masked)
argument_dict['config_yaml_full'] = user_arguments_to_yaml(user_arguments)
argument_dict['config_yaml'] = user_arguments_to_yaml(
user_arguments_masked)
# The expanded_config and expanded_config_full variables contain all other variables and their values.
# expanded_config is a copy of expanded_config_full with secret values removed. Calculating these variables' values
# must come after the calculation of all other variables to prevent infinite recursion.
# TODO(cmaloney): Make this late-bound by gen.internals
expanded_config_full = {
k: v
for k, v in argument_dict.items()
# Omit late-bound variables whose values have not yet been calculated.
if not v.startswith(gen.internals.LATE_BIND_PLACEHOLDER_START)
}
expanded_config_scrubbed = {
k: v
for k, v in expanded_config_full.items() if k not in secret_variables
}
argument_dict['expanded_config_full'] = format_expanded_config(
expanded_config_full)
argument_dict['expanded_config'] = format_expanded_config(
expanded_config_scrubbed)
# Initialize CA and add arguments (exhibitor_ca_certificate and exhibitor_ca_certificate_path)
gen.exhibitor_tls_bootstrap.initialize_exhibitor_ca(argument_dict)
log.debug("Final arguments:" + json_prettyprint({
# Mask secret config values.
k: (masked_value if k in secret_variables else v)
for k, v in argument_dict.items()
}))
# Fill in the template parameters
# TODO(cmaloney): render_templates should ideally take the template targets.
rendered_templates = render_templates(templates, argument_dict)
# Validate there aren't any unexpected top level directives in any of the files
# (likely indicates a misspelling)
for name, template in rendered_templates.items():
if name == dcos_services_yaml: # yaml list of the service files
assert isinstance(template, list)
elif name == cloud_config_yaml:
assert template.keys() <= CLOUDCONFIG_KEYS, template.keys()
elif isinstance(template, str): # Not a yaml template
pass
else: # yaml template file
log.debug("validating template file %s", name)
assert template.keys() <= PACKAGE_KEYS, template.keys()
stable_artifacts = []
channel_artifacts = []
# Find all files which contain late bind variables and turn them into a "late bind package"
# TODO(cmaloney): check there are no late bound variables in cloud-config.yaml
late_files, regular_files = extract_files_containing_late_variables(
rendered_templates[dcos_config_yaml]['package'])
# put the regular files right back
rendered_templates[dcos_config_yaml] = {'package': regular_files}
# Render cluster package list artifact.
cluster_package_list_filename = 'package_lists/{}.package_list.json'.format(
cluster_package_list_id)
os.makedirs(os.path.dirname(cluster_package_list_filename),
mode=0o755,
exist_ok=True)
write_string(cluster_package_list_filename,
argument_dict['cluster_packages'])
log.info('Cluster package list: {}'.format(cluster_package_list_filename))
stable_artifacts.append(cluster_package_list_filename)
def make_package_filename(package_id, extension):
return 'packages/{0}/{1}{2}'.format(package_id.name, repr(package_id),
extension)
# Render all the cluster packages
cluster_package_info = {}
# Prepare late binding config, if any.
late_package = build_late_package(late_files, config_id,
argument_dict['provider'])
if late_variables and late_package:
# Render the late binding package. This package will be downloaded onto
# each cluster node during bootstrap and rendered into the final config
# using the values from the late config file.
late_package_id = PackageId(late_package['name'])
late_package_filename = make_package_filename(late_package_id,
'.dcos_config')
os.makedirs(os.path.dirname(late_package_filename), mode=0o755)
write_yaml(late_package_filename, {'package': late_package['package']},
default_flow_style=False)
log.info('Package filename: {}'.format(late_package_filename))
stable_artifacts.append(late_package_filename)
# Add the late config file to cloud config. The expressions in
# late_variables will be resolved by the service handling the cloud
# config (e.g. Amazon CloudFormation). The rendered late config file
# on a cluster node's filesystem will contain the final values.
rendered_templates[cloud_config_yaml]['root'].append({
'path':
config_dir + '/setup-flags/late-config.yaml',
'permissions':
'0644',
'owner':
'root',
# TODO(cmaloney): don't prettyprint to save bytes.
# NOTE: Use yaml here simply to make avoiding painful escaping and
# unescaping easier.
'content':
render_yaml({
'late_bound_package_id': late_package['name'],
'bound_values': late_variables
})
})
# Collect metadata for cluster packages.
for package_id_str in cluster_packages:
package_id = PackageId(package_id_str)
package_filename = make_package_filename(package_id, '.tar.xz')
cluster_package_info[package_id.name] = {
'id': package_id_str,
'filename': package_filename
}
# Render config packages.
for package_id_str in config_package_ids:
package_id = PackageId(package_id_str)
package_filename = cluster_package_info[package_id.name]['filename']
do_gen_package(rendered_templates[package_id.name + '.yaml'],
cluster_package_info[package_id.name]['filename'])
stable_artifacts.append(package_filename)
# Convert cloud-config to just contain write_files rather than root
cc = rendered_templates[cloud_config_yaml]
# Shouldn't contain any packages. Providers should pull what they need to
# late bind out of other packages via cc_package_file.
assert 'package' not in cc
cc_root = cc.pop('root', [])
# Make sure write_files exists.
assert 'write_files' not in cc
cc['write_files'] = []
# Do the transform
for item in cc_root:
assert is_absolute_path(item['path'])
cc['write_files'].append(item)
rendered_templates[cloud_config_yaml] = cc
# Add utils that need to be defined here so they can be bound to locals.
def add_services(cloudconfig, cloud_init_implementation):
return add_units(cloudconfig, rendered_templates[dcos_services_yaml],
cloud_init_implementation)
utils.add_services = add_services
def add_stable_artifact(filename):
assert filename not in stable_artifacts + channel_artifacts
stable_artifacts.append(filename)
utils.add_stable_artifact = add_stable_artifact
def add_channel_artifact(filename):
assert filename not in stable_artifacts + channel_artifacts
channel_artifacts.append(filename)
utils.add_channel_artifact = add_channel_artifact
return Bunch({
'arguments': argument_dict,
'cluster_packages': cluster_package_info,
'stable_artifacts': stable_artifacts,
'channel_artifacts': channel_artifacts,
'templates': rendered_templates,
'utils': utils
})
def exists(self, path):
assert not is_absolute_path(path)
return os.path.exists(self.__full_path(path))
def generate(
arguments,
extra_templates=list(),
extra_sources=list(),
extra_targets=list()):
# To maintain the old API where we passed arguments rather than the new name.
user_arguments = arguments
arguments = None
sources, targets, templates = get_dcosconfig_source_target_and_templates(
user_arguments, extra_templates, extra_sources)
resolver = validate_and_raise(sources, targets + extra_targets)
argument_dict = get_final_arguments(resolver)
late_variables = get_late_variables(resolver, sources)
secret_builtins = ['expanded_config_full', 'user_arguments_full', 'config_yaml_full']
secret_variables = set(get_secret_variables(sources) + secret_builtins)
masked_value = '**HIDDEN**'
# Calculate config ID after all variables are resolved, to make sure any change in config yields a new config ID.
config_id = get_config_id(argument_dict)
# Calculate values that depend on the config ID.
config_package_names = json.loads(argument_dict['config_package_names'])
package_ids = json.loads(argument_dict['package_ids'])
config_package_ids = ['{}--setup_{}'.format(name, config_id) for name in config_package_names]
cluster_packages = sorted(config_package_ids + package_ids)
validate_cluster_packages(cluster_packages)
cluster_package_list_id = hash_checkout(cluster_packages)
# Calculate values for builtin variables.
argument_dict['cluster_packages'] = json.dumps(cluster_packages)
argument_dict['cluster_package_list_id'] = cluster_package_list_id
user_arguments_masked = {k: (masked_value if k in secret_variables else v) for k, v in user_arguments.items()}
argument_dict['user_arguments_full'] = json_prettyprint(user_arguments)
argument_dict['user_arguments'] = json_prettyprint(user_arguments_masked)
argument_dict['config_yaml_full'] = user_arguments_to_yaml(user_arguments)
argument_dict['config_yaml'] = user_arguments_to_yaml(user_arguments_masked)
# The expanded_config and expanded_config_full variables contain all other variables and their values.
# expanded_config is a copy of expanded_config_full with secret values removed. Calculating these variables' values
# must come after the calculation of all other variables to prevent infinite recursion.
# TODO(cmaloney): Make this late-bound by gen.internals
expanded_config_full = {
k: v for k, v in argument_dict.items()
# Omit late-bound variables whose values have not yet been calculated.
if not v.startswith(gen.internals.LATE_BIND_PLACEHOLDER_START)
}
expanded_config_scrubbed = {k: v for k, v in expanded_config_full.items() if k not in secret_variables}
argument_dict['expanded_config_full'] = format_expanded_config(expanded_config_full)
argument_dict['expanded_config'] = format_expanded_config(expanded_config_scrubbed)
log.debug(
"Final arguments:" + json_prettyprint({
# Mask secret config values.
k: (masked_value if k in secret_variables else v) for k, v in argument_dict.items()
})
)
# Fill in the template parameters
# TODO(cmaloney): render_templates should ideally take the template targets.
rendered_templates = render_templates(templates, argument_dict)
# Validate there aren't any unexpected top level directives in any of the files
# (likely indicates a misspelling)
for name, template in rendered_templates.items():
if name == dcos_services_yaml: # yaml list of the service files
assert isinstance(template, list)
elif name == cloud_config_yaml:
assert template.keys() <= CLOUDCONFIG_KEYS, template.keys()
elif isinstance(template, str): # Not a yaml template
pass
else: # yaml template file
log.debug("validating template file %s", name)
assert template.keys() <= PACKAGE_KEYS, template.keys()
stable_artifacts = []
channel_artifacts = []
# Find all files which contain late bind variables and turn them into a "late bind package"
# TODO(cmaloney): check there are no late bound variables in cloud-config.yaml
late_files, regular_files = extract_files_containing_late_variables(
rendered_templates[dcos_config_yaml]['package'])
# put the regular files right back
rendered_templates[dcos_config_yaml] = {'package': regular_files}
# Render cluster package list artifact.
cluster_package_list_filename = 'package_lists/{}.package_list.json'.format(cluster_package_list_id)
os.makedirs(os.path.dirname(cluster_package_list_filename), mode=0o755, exist_ok=True)
write_string(cluster_package_list_filename, argument_dict['cluster_packages'])
log.info('Cluster package list: {}'.format(cluster_package_list_filename))
stable_artifacts.append(cluster_package_list_filename)
def make_package_filename(package_id, extension):
return 'packages/{0}/{1}{2}'.format(
package_id.name,
repr(package_id),
extension)
# Render all the cluster packages
cluster_package_info = {}
# Prepare late binding config, if any.
late_package = build_late_package(late_files, config_id, argument_dict['provider'])
if late_variables and late_package:
# Render the late binding package. This package will be downloaded onto
# each cluster node during bootstrap and rendered into the final config
# using the values from the late config file.
late_package_id = PackageId(late_package['name'])
late_package_filename = make_package_filename(late_package_id, '.dcos_config')
os.makedirs(os.path.dirname(late_package_filename), mode=0o755)
write_yaml(late_package_filename, {'package': late_package['package']}, default_flow_style=False)
log.info('Package filename: {}'.format(late_package_filename))
stable_artifacts.append(late_package_filename)
# Add the late config file to cloud config. The expressions in
# late_variables will be resolved by the service handling the cloud
# config (e.g. Amazon CloudFormation). The rendered late config file
# on a cluster node's filesystem will contain the final values.
rendered_templates[cloud_config_yaml]['root'].append({
'path': config_dir + '/setup-flags/late-config.yaml',
'permissions': '0644',
'owner': 'root',
# TODO(cmaloney): don't prettyprint to save bytes.
# NOTE: Use yaml here simply to make avoiding painful escaping and
# unescaping easier.
'content': render_yaml({
'late_bound_package_id': late_package['name'],
'bound_values': late_variables
})})
# Collect metadata for cluster packages.
for package_id_str in cluster_packages:
package_id = PackageId(package_id_str)
package_filename = make_package_filename(package_id, '.tar.xz')
cluster_package_info[package_id.name] = {
'id': package_id_str,
'filename': package_filename
}
# Render config packages.
for package_id_str in config_package_ids:
package_id = PackageId(package_id_str)
package_filename = cluster_package_info[package_id.name]['filename']
do_gen_package(rendered_templates[package_id.name + '.yaml'], cluster_package_info[package_id.name]['filename'])
stable_artifacts.append(package_filename)
# Convert cloud-config to just contain write_files rather than root
cc = rendered_templates[cloud_config_yaml]
# Shouldn't contain any packages. Providers should pull what they need to
# late bind out of other packages via cc_package_file.
assert 'package' not in cc
cc_root = cc.pop('root', [])
# Make sure write_files exists.
assert 'write_files' not in cc
cc['write_files'] = []
# Do the transform
for item in cc_root:
assert is_absolute_path(item['path'])
cc['write_files'].append(item)
rendered_templates[cloud_config_yaml] = cc
# Add utils that need to be defined here so they can be bound to locals.
def add_services(cloudconfig, cloud_init_implementation):
return add_units(cloudconfig, rendered_templates[dcos_services_yaml], cloud_init_implementation)
utils.add_services = add_services
def add_stable_artifact(filename):
assert filename not in stable_artifacts + channel_artifacts
stable_artifacts.append(filename)
utils.add_stable_artifact = add_stable_artifact
def add_channel_artifact(filename):
assert filename not in stable_artifacts + channel_artifacts
channel_artifacts.append(filename)
utils.add_channel_artifact = add_channel_artifact
return Bunch({
'arguments': argument_dict,
'cluster_packages': cluster_package_info,
'stable_artifacts': stable_artifacts,
'channel_artifacts': channel_artifacts,
'templates': rendered_templates,
'utils': utils
})