def do_validate(self, include_ssh):
user_arguments = self.as_gen_format()
extra_sources = [onprem_source]
extra_targets = []
if include_ssh:
extra_sources.append(ssh.validate.source)
extra_targets.append(ssh.validate.get_target())
sources, targets, _ = gen.get_dcosconfig_source_target_and_templates(user_arguments, [], extra_sources)
targets = targets + extra_targets
resolver = gen.internals.resolve_configuration(sources, targets)
# TODO(cmaloney): kill this function and make the API return the structured
# results api as was always intended rather than the flattened / lossy other
# format. This will be an API incompatible change. The messages format was
# specifically so that there wouldn't be this sort of API incompatibility.
return normalize_config_validation(resolver.status_dict)
def validate_onprem_dcos_config_contents(onprem_dcos_config_contents, num_masters, prevalidate_onprem_config):
# TODO DCOS-14033: [gen.internals] Source validate functions are global only
if prevalidate_onprem_config != 'true':
return
user_config = yaml.load(onprem_dcos_config_contents)
# Use the default config in the installer
config = yaml.load(dcos_installer.config.config_sample)
config.update(user_config)
# This field is required and auto-added by installer, so add a dummy here
if 'bootstrap_id' not in config:
config['bootstrap_id'] = 'deadbeef'
# dummy master list to pass validation
config['master_list'] = [('10.0.0.' + str(i)) for i in range(int(num_masters))]
# Use the default config in the installer
sources, targets, templates = gen.get_dcosconfig_source_target_and_templates(
gen.stringify_configuration(config), list(), [ssh.validate.source, gen.build_deploy.bash.onprem_source])
# Copy the gen target from dcos_installer/config.py, but instead remove
# 'ssh_key_path' from the target because the validate fn in ssh_source is
# too strict I.E. we cannot validate a key if we are going to generate
# Furthermore, we cannot use the target ssh_key_path as it will automatically
# invoked the validate fn from ssh/validate.py Luckily, we can instead use
# the more idiomatic 'ssh_private_key_filename'
targets.append(Target({
'ssh_user',
'ssh_port',
'master_list',
'agent_list',
'public_agent_list',
'ssh_parallelism',
'process_timeout'}))
resolver = resolve_configuration(sources, targets)
status = resolver.status_dict
if status['status'] == 'errors':
raise AssertionError(pretty_print_validate_error(status['errors'], status['unset']))
def do_aws_cf_configure():
"""Returns error code
Generates AWS templates using a custom config.yaml
"""
# TODO(cmaloney): Move to Config class introduced in https://github.com/dcos/dcos/pull/623
config = Config(CONFIG_PATH)
gen_config = config.as_gen_format()
extra_sources = [
gen.build_deploy.aws.aws_base_source,
aws_advanced_source,
gen.build_deploy.aws.groups['master'][1]]
sources, targets, _ = gen.get_dcosconfig_source_target_and_templates(gen_config, [], extra_sources)
targets.append(get_aws_advanced_target())
resolver = gen.internals.resolve_configuration(sources, targets)
# TODO(cmaloney): kill this function and make the API return the structured
# results api as was always intended rather than the flattened / lossy other
# format. This will be an API incompatible change. The messages format was
# specifically so that there wouldn't be this sort of API incompatibility.
messages = normalize_config_validation(resolver.status_dict)
if messages:
print_messages(messages)
return 1
# TODO(cmaloney): This is really hacky but a lot simpler than merging all the config flows into
# one currently.
# Get out the calculated arguments and manually move critical calculated ones to the gen_config
# object.
# NOTE: the copying across, as well as validation is guaranteed to succeed because we've already
# done a validation run.
full_config = {k: v.value for k, v in resolver.arguments.items()}
# TODO(cmaloney): Switch to using the targets
gen_config['bootstrap_url'] = full_config['bootstrap_url']
gen_config['provider'] = full_config['provider']
gen_config['bootstrap_id'] = full_config['bootstrap_id']
gen_config['package_ids'] = full_config['package_ids']
gen_config['cloudformation_s3_url_full'] = full_config['cloudformation_s3_url_full']
# Convert the bootstrap_Variant string we have back to a bootstrap_id as used internally by all
# the tooling (never has empty string, uses None to say "no variant")
bootstrap_variant = full_config['bootstrap_variant'] if full_config['bootstrap_variant'] else None
artifacts = list()
for built_resource in list(gen.build_deploy.aws.do_create(
tag='dcos_generate_config.sh --aws-cloudformation',
build_name='Custom',
reproducible_artifact_path=full_config['reproducible_artifact_path'],
variant_arguments={bootstrap_variant: gen_config},
commit=full_config['dcos_image_commit'],
all_completes=None)):
artifacts += release.built_resource_to_artifacts(built_resource)
artifacts += list(release.make_bootstrap_artifacts(
full_config['bootstrap_id'],
json.loads(full_config['package_ids']),
bootstrap_variant,
'artifacts',
))
# Upload all the artifacts to the config-id path and then print out what
# the path that should be used is, as well as saving a local json file for
# easy machine access / processing.
repository = release.Repository(
full_config['aws_template_storage_bucket_path'],
None,
'config_id/' + full_config['config_id'])
storage_commands = repository.make_commands({'core_artifacts': [], 'channel_artifacts': artifacts})
cf_dir = GENCONF_DIR + '/cloudformation'
log.warning("Writing local copies to {}".format(cf_dir))
storage_provider = release.storage.local.LocalStorageProvider(cf_dir)
release.apply_storage_commands({'local': storage_provider}, storage_commands)
log.warning(
"Generated templates locally available at %s",
cf_dir + "/" + full_config["reproducible_artifact_path"])
# TODO(cmaloney): Print where the user can find the files locally
if full_config['aws_template_upload'] == 'false':
return 0
storage_provider = release.storage.aws.S3StorageProvider(
bucket=full_config['aws_template_storage_bucket'],
object_prefix=None,
download_url=full_config['cloudformation_s3_url'],
region_name=full_config['aws_template_storage_region_name'],
access_key_id=full_config['aws_template_storage_access_key_id'],
secret_access_key=full_config['aws_template_storage_secret_access_key'])
log.warning("Uploading to AWS")
release.apply_storage_commands({'aws': storage_provider}, storage_commands)
log.warning("AWS CloudFormation templates now available at: {}".format(
full_config['cloudformation_s3_url']))
# TODO(cmaloney): Print where the user can find the files in AWS
# TODO(cmaloney): Dump out a JSON with machine paths to make scripting easier.
return 0
def do_aws_cf_configure():
"""Returns error code
Generates AWS templates using a custom config.yaml
"""
# TODO(cmaloney): Move to Config class introduced in https://github.com/dcos/dcos/pull/623
config = Config(CONFIG_PATH)
# This process is usually ran from a docker container where default boto3 credential
# method may fail and as such, we allow passing these creds explicitly
if 'aws_template_storage_access_key_id' in config:
os.environ['AWS_ACCESS_KEY_ID'] = config[
'aws_template_storage_access_key_id']
if 'aws_template_storage_secret_access_key' in config:
os.environ['AWS_SECRET_ACCESS_KEY'] = config[
'aws_template_storage_secret_access_key']
if 'aws_template_storage_region_name' in config:
os.environ['AWS_DEFAULT_REGION'] = config[
'aws_template_storage_region_name']
gen_config = config.as_gen_format()
extra_sources = [
gen.build_deploy.aws.aws_base_source, aws_advanced_source,
gen.build_deploy.aws.groups['master'][1]
]
sources, targets, _ = gen.get_dcosconfig_source_target_and_templates(
gen_config, [], extra_sources)
targets.append(get_aws_advanced_target())
resolver = gen.internals.resolve_configuration(sources, targets)
# TODO(cmaloney): kill this function and make the API return the structured
# results api as was always intended rather than the flattened / lossy other
# format. This will be an API incompatible change. The messages format was
# specifically so that there wouldn't be this sort of API incompatibility.
messages = normalize_config_validation(resolver.status_dict)
if messages:
print_messages(messages)
return 1
# TODO(cmaloney): This is really hacky but a lot simpler than merging all the config flows into
# one currently.
# Get out the calculated arguments and manually move critical calculated ones to the gen_config
# object.
# NOTE: the copying across, as well as validation is guaranteed to succeed because we've already
# done a validation run.
full_config = {k: v.value for k, v in resolver.arguments.items()}
# TODO(cmaloney): Switch to using the targets
gen_config['bootstrap_url'] = full_config['bootstrap_url']
gen_config['provider'] = full_config['provider']
gen_config['bootstrap_id'] = full_config['bootstrap_id']
gen_config['package_ids'] = full_config['package_ids']
gen_config['cloudformation_s3_url_full'] = full_config[
'cloudformation_s3_url_full']
# Convert the bootstrap_Variant string we have back to a bootstrap_id as used internally by all
# the tooling (never has empty string, uses None to say "no variant")
bootstrap_variant = full_config['bootstrap_variant'] if full_config[
'bootstrap_variant'] else None
artifacts = list()
for built_resource in list(
gen.build_deploy.aws.do_create(
tag='dcos_generate_config.sh --aws-cloudformation',
build_name='Custom',
reproducible_artifact_path=full_config[
'reproducible_artifact_path'],
variant_arguments={bootstrap_variant: gen_config},
commit=full_config['dcos_image_commit'],
all_completes=None)):
artifacts += release.built_resource_to_artifacts(built_resource)
artifacts += list(
release.make_bootstrap_artifacts(
full_config['bootstrap_id'],
json.loads(full_config['package_ids']),
bootstrap_variant,
'artifacts',
))
for package_id in json.loads(full_config['package_ids']):
package_filename = release.make_package_filename(package_id)
artifacts.append({
'reproducible_path': package_filename,
'local_path': 'artifacts/' + package_filename,
})
# Upload all the artifacts to the config-id path and then print out what
# the path that should be used is, as well as saving a local json file for
# easy machine access / processing.
repository = release.Repository(
full_config['aws_template_storage_bucket_path'], None,
'config_id/' + full_config['config_id'])
storage_commands = repository.make_commands({
'core_artifacts': [],
'channel_artifacts': artifacts
})
cf_dir = GENCONF_DIR + '/cloudformation'
log.warning("Writing local copies to {}".format(cf_dir))
storage_provider = release.storage.local.LocalStorageProvider(cf_dir)
release.apply_storage_commands({'local': storage_provider},
storage_commands)
log.warning("Generated templates locally available at %s",
cf_dir + "/" + full_config["reproducible_artifact_path"])
# TODO(cmaloney): Print where the user can find the files locally
if full_config['aws_template_upload'] == 'false':
return 0
storage_provider = release.storage.aws.S3StorageProvider(
bucket=full_config['aws_template_storage_bucket'],
object_prefix=None,
download_url=full_config['cloudformation_s3_url'],
region_name=full_config['aws_template_storage_region_name'],
access_key_id=full_config['aws_template_storage_access_key_id'],
secret_access_key=full_config['aws_template_storage_secret_access_key']
)
log.warning("Uploading to AWS")
release.apply_storage_commands({'aws': storage_provider}, storage_commands)
log.warning("AWS CloudFormation templates now available at: {}".format(
full_config['cloudformation_s3_url']))
# TODO(cmaloney): Print where the user can find the files in AWS
# TODO(cmaloney): Dump out a JSON with machine paths to make scripting easier.
return 0
def do_aws_cf_configure():
"""Returns error code
Generates AWS templates using a custom config.yaml
"""
# TODO(cmaloney): Move to Config class introduced in https://github.com/dcos/dcos/pull/623
config = Config(CONFIG_PATH)
gen_config = config.as_gen_format()
# TODO(cmaloney): this is hacky....
del gen_config['provider']
sources, targets, _ = gen.get_dcosconfig_source_target_and_templates(
gen_config, [])
sources.append(aws_advanced_source)
targets.append(aws_advanced_target)
messages = gen.internals.validate_configuration(sources, targets,
gen_config)
# TODO(cmaloney): kill this function and make the API return the structured
# results api as was always intended rather than the flattened / lossy other
# format. This will be an API incompatible change. The messages format was
# specifically so that there wouldn't be this sort of API incompatibility.
messages = normalize_config_validation(messages)
if messages:
print_messages(messages)
return 1
# TODO(cmaloney): This is really hacky but a lot simpler than merging all the config flows into
# one currently.
# Get out the calculated arguments and manually move critical calculated ones to the gen_config
# object.
# NOTE: the copying across, as well as validation is guaranteed to succeed because we've already
# done a validation run.
full_config = gen.internals.resolve_configuration(sources, targets,
gen_config)
gen_config['bootstrap_url'] = full_config['bootstrap_url']
gen_config['provider'] = full_config['provider']
gen_config['bootstrap_id'] = full_config['bootstrap_id']
gen_config['cloudformation_s3_url'] = full_config['cloudformation_s3_url']
# Convert the bootstrap_Variant string we have back to a bootstrap_id as used internally by all
# the tooling (never has empty string, uses None to say "no variant")
bootstrap_variant = full_config['bootstrap_variant'] if full_config[
'bootstrap_variant'] else None
artifacts = list()
for built_resource in list(
gen.installer.aws.do_create(
tag='dcos_generate_config.sh --aws-cloudformation',
build_name='Custom',
reproducible_artifact_path=full_config[
'reproducible_artifact_path'],
variant_arguments={bootstrap_variant: gen_config},
commit=full_config['dcos_image_commit'],
all_bootstraps=None)):
artifacts += release.built_resource_to_artifacts(built_resource)
artifacts += list(
release.make_bootstrap_artifacts(full_config['bootstrap_id'],
bootstrap_variant, 'artifacts'))
# Upload all the artifacts to the config-id path and then print out what
# the path that should be used is, as well as saving a local json file for
# easy machine access / processing.
repository = release.Repository(
full_config['aws_template_storage_bucket_path'], None,
'config_id/' + full_config['config_id'])
storage_commands = repository.make_commands({
'core_artifacts': [],
'channel_artifacts': artifacts
})
log.warning("Writing local copies to genconf/cloudformation")
storage_provider = release.storage.local.LocalStorageProvider(
'genconf/cloudformation')
release.apply_storage_commands({'local': storage_provider},
storage_commands)
log.warning(
"Generated templates locally available at %s",
"genconf/cloudformation/" + full_config["reproducible_artifact_path"])
# TODO(cmaloney): Print where the user can find the files locally
if full_config['aws_template_upload'] == 'false':
return 0
storage_provider = release.storage.aws.S3StorageProvider(
bucket=full_config['aws_template_storage_bucket'],
object_prefix=None,
download_url=full_config['cloudformation_s3_url'],
region_name=full_config['aws_template_storage_region_name'],
access_key_id=full_config['aws_template_storage_access_key_id'],
secret_access_key=full_config['aws_template_storage_secret_access_key']
)
log.warning("Uploading to AWS")
release.apply_storage_commands({'aws': storage_provider}, storage_commands)
log.warning("AWS CloudFormation templates now available at: {}".format(
full_config['cloudformation_s3_url']))
# TODO(cmaloney): Print where the user can find the files in AWS
# TODO(cmaloney): Dump out a JSON with machine paths to make scripting easier.
return 0
