def send(self, action, install_method, num_errors): """Sends analytics track data to segmentIO. variant: string | open or enterprise action: string | preflight, deploy, or postflight install_method: string | gui, cli or advanced """ analytics.write_key = "51ybGTeFEFU1xo6u10XMDrr6kATFyRyh" # Set customer key here rather than __init__ since we want the most up to date config # and config may change between __init__ and here. config = Config(CONFIG_PATH) customer_key = config.hacky_default_get('customer_key', None) # provider is always onprem when the cli installer is used provider = "onprem" # platform defaults to provider value, if not specified platform = config.hacky_default_get('platform', provider) analytics.track(user_id=customer_key, anonymous_id=self.uuid, event="installer", properties={ "platform": platform, "provider": provider, "source": "installer", "variant": os.environ["BOOTSTRAP_VARIANT"], "install_id": self.uuid, "bootstrap_id": os.environ["BOOTSTRAP_ID"], "install_method": install_method, "action_name": action, "errors": num_errors, "customerKey": customer_key, }) analytics.flush()
def test_configure(client): route = '/api/v1/configure' featured_methods = { 'GET': [200, 'application/json'], # Should return a 400 if validation has errors, # which this POST will return since the ssh_port is not an integer. 'POST': [400, 'application/json', '{"ssh_port": "asdf"}'], 'PUT': [405, 'text/plain'], 'DELETE': [405, 'text/plain'], 'HEAD': [405, 'text/plain'], 'TRACE': [405, 'text/plain'], 'CONNECT': [405, 'text/plain'], } for method, expected in featured_methods.items(): if method == 'POST': res = client.request(route, method=method, body=bytes(expected[2].encode('utf-8')), expect_errors=True) else: res = client.request(route, method=method, expect_errors=True) assert res.status_code == expected[0], '{}: {}'.format( method, expected) assert res.content_type == expected[1], '{}: {}'.format( method, expected) if expected[0] == 200: expected_config = Config('genconf/config.yaml').config # Add ui config parameters which are always set. # TODO(cmaloney): Make this unnecessary expected_config.update({'ssh_key': None, 'ip_detect_script': None}) assert res.json == expected_config
def do_configure(config_path=CONFIG_PATH): """Returns error code :param config_path: path to config.yaml :type config_path: string | CONFIG_PATH (genconf/config.yaml) """ config = Config(config_path) validate_gen = config.do_validate(include_ssh=False) if len(validate_gen) > 0: for key, error in validate_gen.items(): log.error('{}: {}'.format(key, error)) return 1 config_util.do_configure(config) return 0
def generate_node_upgrade_script(installed_cluster_version, config_path=CONFIG_PATH): if installed_cluster_version is None: print('Must provide the version of the cluster upgrading from') return 1 config = Config(config_path) try: gen_out = config_util.onprem_generate(config) except ValidationError as e: validation = normalize_config_validation_exception(e) print_messages(validation) return 1 except ExhibitorTLSBootstrapError as e: log.error('Failed to bootstrap Exhibitor TLS') for i, error in enumerate(e.errors): return log.error("{}: {}".format(i + 1, error)) return 1 config_util.make_serve_dir(gen_out) # generate the upgrade script upgrade.generate_node_upgrade_script(gen_out, installed_cluster_version) return 0
def success(config: Config): """Returns the data for /success/ endpoint. :param config_path: path to config.yaml :type config_path: str | CONFIG_PATH (genconf/config.yaml) """ master_ips = config.hacky_default_get('master_list', []) agent_ips = config.hacky_default_get('agent_list', []) code = 200 msgs = {'success': "", 'master_count': 0, 'agent_count': 0} if not master_ips or not agent_ips: code = 400 return msgs, code msgs['success'] = 'http://{}'.format(master_ips[0]) msgs['master_count'] = len(master_ips) msgs['agent_count'] = len(agent_ips) return msgs, code
def do_validate_config(args): log_warn_only() config = Config(dcos_installer.constants.CONFIG_PATH) validation_errors = config.do_validate(include_ssh=True) if validation_errors: print_validation_errors(validation_errors) return 1 return 0
def success(request): """Return /success :param request: a web requeest object. :type request: request | None """ log.info("Request for success made.") msgs, code = backend.success(Config(CONFIG_PATH)) return web.json_response(msgs, status=code)
def test_set_superuser_password(tmpdir): """Test that --set-superuser-hash works""" with tmpdir.as_cwd(): tmpdir.join('genconf').ensure(dir=True) # TODO(cmaloney): Add tests for the behavior around a non-existent config.yaml # Setting in a non-empty config.yaml which has no password set make_default_config_if_needed('genconf/config.yaml') assert 'superuser_password_hash' not in Config('genconf/config.yaml').config # Set the password subprocess.check_call(['dcos_installer', '--set-superuser-password', 'foo'], cwd=str(tmpdir)) # Check that config.yaml has the password set config = Config('genconf/config.yaml') assert passlib.hash.sha512_crypt.verify('foo', config['superuser_password_hash'])
def test_accept_overrides_for_undefined_config_params(tmpdir): temp_config_path = tmpdir.strpath + '/config.yaml' param = ('fake_test_param_name', 'fake_test_param_value') make_default_config_if_needed(temp_config_path) messages = backend.create_config_from_post(post_data=dict([param]), config_path=temp_config_path) assert not messages, "unexpected validation error: {}".format(messages) assert Config(config_path=temp_config_path)[param[0]] == param[1]
def create_config_from_post(post_data, config_path): """Returns error code and validation messages for only keys POSTed to the UI. :param config_path: path to config.yaml :type config_path: string | CONFIG_PATH (genconf/config.yaml) :param post_data: data from POST to UI :type post_data: dict | {} """ log.info("Updating config with POST data.") # Make sure this isn't passed ssh_key how the web installer used to, the web installer should # take care of it's wrapping / unwrapping. assert 'ssh_key' not in post_data assert 'ip_detect_script' not in post_data # Create a new configuration object, pass it the config.yaml path and POSTed dictionary. # Add in "hidden config" we don't present in the config.yaml, and then create a meta # validation dictionary from gen and ssh validation libs. # We do not use the already built methods for this since those are used to read the # coniguration off disk, here we need to validate the configuration overridees, and # return the key and message for the POSTed parameter. config = Config(config_path) config.update(post_data) validation_messages = config.do_validate(include_ssh=True) # TODO(cmaloney): Return all errors to the UI so it can display / decide how # it wants to log (new parameter might cause an error with an old set key) # Return only the keys the UI POSTed, do not write config to disk if # validation fails. post_validation_errors = { key: validation_messages[key] for key in validation_messages if key in post_data } # If validation is successful, write the data to disk, otherwise, if # they keys POSTed failed, do not write to disk. if post_validation_errors: log.error("POSTed configuration has errors, not writing to disk.") for key, value in post_validation_errors.items(): log.error('{}: {}'.format(key, value)) else: log.debug("Success! POSTed configuration looks good, writing to disk.") config.update(post_data) config.write_config() return post_validation_errors
def success(config: Config): """Returns the data for /success/ endpoint. :param config_path: path to config.yaml :type config_path: str | CONFIG_PATH (genconf/config.yaml) """ master_ips = config.hacky_default_get('master_list', []) agent_ips = config.hacky_default_get('agent_list', []) code = 200 msgs = { 'success': "", 'master_count': 0, 'agent_count': 0 } if not master_ips or not agent_ips: code = 400 return msgs, code msgs['success'] = 'http://{}'.format(master_ips[0]) msgs['master_count'] = len(master_ips) msgs['agent_count'] = len(agent_ips) return msgs, code
def configure_status(request): """Return /configure/status :param request: a web requeest object. :type request: request | None """ log.info("Request for configuration validation made.") code = 200 messages = Config(CONFIG_PATH).do_validate(include_ssh=True) if messages: code = 400 resp = web.json_response(messages, status=code) return resp
def determine_config_type(config_path=CONFIG_PATH): """Returns the configuration type to the UI. One of either 'minimal' or 'advanced'. 'advanced' blocks UI usage. :param config_path: path to config.yaml :type config_path: str | CONFIG_PATH (genconf/config.yaml) """ # TODO(cmaloney): If the config has any arguments not in the set of possible parameters then # the config is always advanced. config = Config(config_path) adv_found = {} advanced_cluster_config = { "bootstrap_url": 'file:///opt/dcos_install_tmp', "docker_remove_delay": None, "exhibitor_storage_backend": 'static', "gc_delay": None, "master_discovery": 'static', "roles": None, "weights": None } for key, value in advanced_cluster_config.items(): # Skip if the key isn't in config if key not in config: continue # None indicates any value means this is advanced config. # A string indicates the value must match. if value is None: log.error('Advanced configuration found in config.yaml: {}: value'.format(key, value)) adv_found[key] = config[key] elif value != config[key]: log.error('Advanced configuration found in config.yaml: {}: value'.format(key, config[key])) adv_found[key] = config[key] if adv_found: message = ( "Advanced configuration detected in {config_path} ({adv_found}).\nPlease backup " "or remove {config_path} to use the UI installer.".format( config_path=CONFIG_PATH, adv_found=adv_found, ) ) config_type = 'advanced' else: message = '' config_type = 'minimal' return { 'message': message, 'type': config_type }
def configure(request): """Return /api/v1/configure :param request: a web requeest object. :type request: request | None """ if request.method == 'POST': new_config = yield from request.json() # Save ssh_key, ip_detect as needed # TODO(cmaloney): make ssh_key derive from ssh_key_path so we can just set ssh_key and skip all this. new_config = extract_external(new_config, 'ssh_key', 'ssh_key_path', SSH_KEY_PATH, 0o600) # TODO(cmaloney): change this to ip_detect_contents removing the need for the remapping. new_config = extract_external(new_config, 'ip_detect_script', 'ip_detect_path', IP_DETECT_PATH, 0o644) log.info('POST to configure: {}'.format(new_config)) messages = backend.create_config_from_post(new_config, CONFIG_PATH) # Map back to DC/OS UI configuration parameters. # TODO(cmaloney): Remove need to remap validation keys. The remapping is making things show up # under the key of the user config chunk that caused them rather than their particular key so # num_masters validation for instance shows up under master_list where the user would expect it. if "ssh_key_path" in messages: messages["ssh_key"] = messages["ssh_key_path"] if "ip_detect_contents" in messages: messages['ip_detect_path'] = messages['ip_detect_contents'] if 'num_masters' in messages: messages['master_list'] = messages['num_masters'] resp = web.json_response({}, status=200) if messages: resp = web.json_response(messages, status=400) return resp elif request.method == 'GET': config = Config(CONFIG_PATH).config # TODO(cmaloney): should exclude the value entirely if the file doesn't exist. config['ssh_key'] = try_read_file(SSH_KEY_PATH) config['ip_detect_script'] = try_read_file(IP_DETECT_PATH) resp = web.json_response(config) resp.headers['Content-Type'] = 'application/json' return resp
def create_config_from_post(post_data, config_path): """Returns error code and validation messages for only keys POSTed to the UI. :param config_path: path to config.yaml :type config_path: string | CONFIG_PATH (genconf/config.yaml) :param post_data: data from POST to UI :type post_data: dict | {} """ log.info("Updating config with POST data.") # Make sure this isn't passed ssh_key how the web installer used to, the web installer should # take care of it's wrapping / unwrapping. assert 'ssh_key' not in post_data assert 'ip_detect_script' not in post_data # Create a new configuration object, pass it the config.yaml path and POSTed dictionary. # Add in "hidden config" we don't present in the config.yaml, and then create a meta # validation dictionary from gen and ssh validation libs. # We do not use the already built methods for this since those are used to read the # coniguration off disk, here we need to validate the configuration overridees, and # return the key and message for the POSTed parameter. config = Config(config_path) config.update(post_data) validation_messages = config.do_validate(include_ssh=True) # TODO(cmaloney): Return all errors to the UI so it can display / decide how # it wants to log (new parameter might cause an error with an old set key) # Return only the keys the UI POSTed, do not write config to disk if # validation fails. post_validation_errors = {key: validation_messages[key] for key in validation_messages if key in post_data} # If validation is successful, write the data to disk, otherwise, if # they keys POSTed failed, do not write to disk. if post_validation_errors: log.error("POSTed configuration has errors, not writing to disk.") for key, value in post_validation_errors.items(): log.error('{}: {}'.format(key, value)) else: log.debug("Success! POSTed configuration looks good, writing to disk.") config.update(post_data) config.write_config() return post_validation_errors
def test_get_config(tmpdir): workspace = tmpdir.strpath temp_config_path = workspace + '/config.yaml' expected_data = { 'cluster_name': 'DC/OS', 'master_discovery': 'static', 'exhibitor_storage_backend': 'static', 'resolvers': ['8.8.8.8', '8.8.4.4'], 'process_timeout': 10000, 'bootstrap_url': 'file:///opt/dcos_install_tmp', } make_default_config_if_needed(temp_config_path) config = Config(temp_config_path) assert expected_data == config.config
def test_do_validate_config(tmpdir): # Create a temp config genconf_dir = tmpdir.join('genconf') genconf_dir.ensure(dir=True) temp_config_path = str(genconf_dir.join('config.yaml')) # Initialize with defautls make_default_config_if_needed(temp_config_path) expected_output = { 'ip_detect_contents': 'ip-detect script `genconf/ip-detect` must exist', 'ssh_user': '******', 'master_list': 'Must set master_list, no way to calculate value.', 'ssh_key_path': 'could not find ssh private key: genconf/ssh_key' } assert Config(config_path=temp_config_path).do_validate(include_ssh=True) == expected_output
def do_configure(config_path=CONFIG_PATH): """Returns error code :param config_path: path to config.yaml :type config_path: string | CONFIG_PATH (genconf/config.yaml) """ config = Config(config_path) validation = validate_gen(config) if not validation: return 1 config_util.do_configure(config) return 0
def do_configure(config_path=CONFIG_PATH): """Returns error code :param config_path: path to config.yaml :type config_path: string | CONFIG_PATH (genconf/config.yaml) """ config = Config(config_path) try: gen_out = config_util.onprem_generate(config) except ValidationError as e: validation = normalize_config_validation_exception(e) print_messages(validation) return 1 config_util.make_serve_dir(gen_out) return 0
def generate_node_upgrade_script(installed_cluster_version, config_path=CONFIG_PATH): if installed_cluster_version is None: print('Must provide the version of the cluster upgrading from') return 1 config = Config(config_path) validation = validate_gen(config) if not validation: return 1 gen_out = config_util.onprem_generate(config) config_util.make_serve_dir(gen_out) # generate the upgrade script upgrade.generate_node_upgrade_script(gen_out, installed_cluster_version) return 0
def test_do_validate_config(tmpdir, monkeypatch): monkeypatch.setenv('BOOTSTRAP_VARIANT', 'test_variant') # Create a temp config genconf_dir = tmpdir.join('genconf') genconf_dir.ensure(dir=True) temp_config_path = str(genconf_dir.join('config.yaml')) # Initialize with defautls make_default_config_if_needed(temp_config_path) create_fake_build_artifacts(tmpdir) expected_output = { 'ip_detect_contents': 'ip-detect script `genconf/ip-detect` must exist', 'master_list': 'Must set master_list, no way to calculate value.', } with tmpdir.as_cwd(): assert Config(config_path='genconf/config.yaml').do_validate() == expected_output
def generate_node_upgrade_script(installed_cluster_version, config_path=CONFIG_PATH): if installed_cluster_version is None: print('Must provide the version of the cluster upgrading from') return 1 config = Config(config_path) try: gen_out = config_util.onprem_generate(config) except ValidationError as e: validation = normalize_config_validation_exception(e) print_messages(validation) return 1 config_util.make_serve_dir(gen_out) # generate the upgrade script upgrade.generate_node_upgrade_script(gen_out, installed_cluster_version) return 0
def do_configure(config_path=CONFIG_PATH): """Returns error code :param config_path: path to config.yaml :type config_path: string | CONFIG_PATH (genconf/config.yaml) """ config = Config(config_path) try: gen_out = config_util.onprem_generate(config) except ValidationError as e: validation = normalize_config_validation_exception(e) print_messages(validation) return 1 except ExhibitorTLSBootstrapError as e: log.error('Failed to bootstrap Exhibitor TLS') for i, error in enumerate(e.errors): return log.error("{}: {}".format(i + 1, error)) return 1 config_util.make_serve_dir(gen_out) return 0
def do_aws_cf_configure(): """Returns error code Generates AWS templates using a custom config.yaml """ # TODO(cmaloney): Move to Config class introduced in https://github.com/dcos/dcos/pull/623 config = Config(CONFIG_PATH) # This process is usually ran from a docker container where default boto3 credential # method may fail and as such, we allow passing these creds explicitly if 'aws_template_storage_access_key_id' in config: os.environ['AWS_ACCESS_KEY_ID'] = config[ 'aws_template_storage_access_key_id'] if 'aws_template_storage_secret_access_key' in config: os.environ['AWS_SECRET_ACCESS_KEY'] = config[ 'aws_template_storage_secret_access_key'] if 'aws_template_storage_region_name' in config: os.environ['AWS_DEFAULT_REGION'] = config[ 'aws_template_storage_region_name'] gen_config = config.as_gen_format() extra_sources = [ gen.build_deploy.aws.aws_base_source, aws_advanced_source, gen.build_deploy.aws.groups['master'][1] ] sources, targets, _ = gen.get_dcosconfig_source_target_and_templates( gen_config, [], extra_sources) targets.append(get_aws_advanced_target()) resolver = gen.internals.resolve_configuration(sources, targets) # TODO(cmaloney): kill this function and make the API return the structured # results api as was always intended rather than the flattened / lossy other # format. This will be an API incompatible change. The messages format was # specifically so that there wouldn't be this sort of API incompatibility. messages = normalize_config_validation(resolver.status_dict) if messages: print_messages(messages) return 1 # TODO(cmaloney): This is really hacky but a lot simpler than merging all the config flows into # one currently. # Get out the calculated arguments and manually move critical calculated ones to the gen_config # object. # NOTE: the copying across, as well as validation is guaranteed to succeed because we've already # done a validation run. full_config = {k: v.value for k, v in resolver.arguments.items()} # Calculate the config ID and values that depend on it. config_id = gen.get_config_id(full_config) reproducible_artifact_path = 'config_id/{}'.format(config_id) cloudformation_s3_url = '{}/config_id/{}'.format( full_config['bootstrap_url'], config_id) cloudformation_s3_url_full = '{}/cloudformation'.format( cloudformation_s3_url) # TODO(cmaloney): Switch to using the targets gen_config['bootstrap_url'] = full_config['bootstrap_url'] gen_config['provider'] = full_config['provider'] gen_config['bootstrap_id'] = full_config['bootstrap_id'] gen_config['package_ids'] = full_config['package_ids'] gen_config['cloudformation_s3_url_full'] = cloudformation_s3_url_full # Convert the bootstrap_Variant string we have back to a bootstrap_id as used internally by all # the tooling (never has empty string, uses None to say "no variant") bootstrap_variant = full_config['bootstrap_variant'] if full_config[ 'bootstrap_variant'] else None artifacts = list() for built_resource in list( gen.build_deploy.aws.do_create( tag='dcos_generate_config.sh --aws-cloudformation', build_name='Custom', reproducible_artifact_path=reproducible_artifact_path, variant_arguments={bootstrap_variant: gen_config}, commit=full_config['dcos_image_commit'], all_completes=None)): artifacts += release.built_resource_to_artifacts(built_resource) artifacts += list( release.make_bootstrap_artifacts( full_config['bootstrap_id'], json.loads(full_config['package_ids']), bootstrap_variant, 'artifacts', )) for package_id in json.loads(full_config['package_ids']): package_filename = release.make_package_filename(package_id) artifacts.append({ 'reproducible_path': package_filename, 'local_path': 'artifacts/' + package_filename, }) # Upload all the artifacts to the config-id path and then print out what # the path that should be used is, as well as saving a local json file for # easy machine access / processing. repository = release.Repository( full_config['aws_template_storage_bucket_path'], None, 'config_id/' + config_id) storage_commands = repository.make_commands({ 'core_artifacts': [], 'channel_artifacts': artifacts }) cf_dir = GENCONF_DIR + '/cloudformation' log.warning("Writing local copies to {}".format(cf_dir)) storage_provider = release.storage.local.LocalStorageProvider(cf_dir) release.apply_storage_commands({'local': storage_provider}, storage_commands) log.warning("Generated templates locally available at %s", cf_dir + "/" + reproducible_artifact_path) # TODO(cmaloney): Print where the user can find the files locally if full_config['aws_template_upload'] == 'false': return 0 storage_provider = release.storage.aws.S3StorageProvider( bucket=full_config['aws_template_storage_bucket'], object_prefix=None, download_url=cloudformation_s3_url, region_name=full_config['aws_template_storage_region_name'], access_key_id=full_config['aws_template_storage_access_key_id'], secret_access_key=full_config['aws_template_storage_secret_access_key'] ) log.warning("Uploading to AWS") release.apply_storage_commands({'aws': storage_provider}, storage_commands) log.warning("AWS CloudFormation templates now available at: {}".format( cloudformation_s3_url)) # TODO(cmaloney): Print where the user can find the files in AWS # TODO(cmaloney): Dump out a JSON with machine paths to make scripting easier. return 0
def action_action_name(request): """Return /action/<action_name> :param request: a web requeest object. :type request: request | None """ global current_action action_name = request.match_info['action_name'] # Update the global action json_state = read_json_state(action_name) current_action = action_name if request.method == 'GET': log.info('GET {}'.format(action_name)) if json_state: return web.json_response(json_state) return web.json_response({}) elif request.method == 'POST': log.info('POST {}'.format(action_name)) action = action_map.get(action_name) # If the action name is preflight, attempt to run configuration # generation. If genconf fails, present the UI with a usable error # for the end-user if action_name == 'preflight': try: log.warning("GENERATING CONFIGURATION") backend.do_configure() except: genconf_failure = { "errors": "Configuration generation failed, please see command line for details" } return web.json_response(genconf_failure, status=400) params = yield from request.post() if json_state: if action_name == 'deploy' and 'retry' in params: if 'hosts' in json_state: failed_hosts = [] for deploy_host, deploy_params in json_state[ 'hosts'].items(): if deploy_params['host_status'] != 'success': failed_hosts.append( Node(deploy_host, tags=deploy_params['tags'], default_port=int( Config(CONFIG_PATH).hacky_default_get( 'ssh_port', 22)))) log.debug('failed hosts: {}'.format(failed_hosts)) if failed_hosts: yield from asyncio. async (action( Config(CONFIG_PATH), state_json_dir=STATE_DIR, hosts=failed_hosts, try_remove_stale_dcos=True, **params)) return web.json_response({ 'status': 'retried', 'details': sorted([ '{}:{}'.format(node.ip, node.port) for node in failed_hosts ]) }) if action_name not in remove_on_done: return web.json_response({ 'status': '{} was already executed, skipping'.format(action_name) }) running = False for host, attributes in json_state['hosts'].items(): if attributes['host_status'].lower() == 'running': running = True log.debug('is action running: {}'.format(running)) if running: return web.json_response( {'status': '{} is running, skipping'.format(action_name)}) else: unlink_state_file(action_name) yield from asyncio. async (action(Config(CONFIG_PATH), state_json_dir=STATE_DIR, options=options, **params)) return web.json_response({'status': '{} started'.format(action_name)})
def do_aws_cf_configure(): """Returns error code Generates AWS templates using a custom config.yaml """ # TODO(cmaloney): Move to Config class introduced in https://github.com/dcos/dcos/pull/623 config = Config(CONFIG_PATH) aws_config_target = gen.ConfigTarget(aws_advanced_parameters) aws_config_target.add_entry(aws_advanced_entry, False) gen_config = config.as_gen_format() # TODO(cmaloney): this is hacky.... del gen_config['provider'] config_targets = [ gen.get_dcosconfig_target_and_templates(gen_config, [])[0], aws_config_target] messages = gen.validate_config_for_targets(config_targets, gen_config) # TODO(cmaloney): kill this function and make the API return the structured # results api as was always intended rather than the flattened / lossy other # format. This will be an API incompatible change. The messages format was # specifically so that there wouldn't be this sort of API incompatibility. messages = normalize_config_validation(messages) if messages: print_messages(messages) return 1 # TODO(cmaloney): This is really hacky but a lot simpler than merging all the config flows into # one currently. # Get out the calculated arguments and manually move critical calculated ones to the gen_config # object. # NOTE: the copying across, as well as validation is guaranteed to succeed because we've already # done a validation run. full_config = gen.calculate_config_for_targets(config_targets, gen_config) gen_config['bootstrap_url'] = full_config['bootstrap_url'] gen_config['provider'] = full_config['provider'] gen_config['bootstrap_id'] = full_config['bootstrap_id'] gen_config['cloudformation_s3_url'] = full_config['cloudformation_s3_url'] # Convert the bootstrap_Variant string we have back to a bootstrap_id as used internally by all # the tooling (never has empty string, uses None to say "no variant") bootstrap_variant = full_config['bootstrap_variant'] if full_config['bootstrap_variant'] else None artifacts = list() for built_resource in list(gen.installer.aws.do_create( tag='dcos_generate_config.sh --aws-cloudformation', build_name='Custom', reproducible_artifact_path=full_config['reproducible_artifact_path'], variant_arguments={bootstrap_variant: gen_config}, commit=full_config['dcos_image_commit'], all_bootstraps=None)): artifacts += release.built_resource_to_artifacts(built_resource) artifacts += list(release.make_bootstrap_artifacts(full_config['bootstrap_id'], bootstrap_variant, 'artifacts')) # Upload all the artifacts to the config-id path and then print out what # the path that should be used is, as well as saving a local json file for # easy machine access / processing. repository = release.Repository( full_config['aws_template_storage_bucket_path'], None, 'config_id/' + full_config['config_id']) storage_commands = repository.make_commands({'core_artifacts': [], 'channel_artifacts': artifacts}) log.warning("Writing local copies to genconf/cloudformation") storage_provider = release.storage.local.LocalStorageProvider('genconf/cloudformation') release.apply_storage_commands({'local': storage_provider}, storage_commands) log.warning( "Generated templates locally available at %s", "genconf/cloudformation/" + full_config["reproducible_artifact_path"]) # TODO(cmaloney): Print where the user can find the files locally if full_config['aws_template_upload'] == 'false': return 0 storage_provider = release.storage.aws.S3StorageProvider( bucket=full_config['aws_template_storage_bucket'], object_prefix=None, download_url=full_config['cloudformation_s3_url'], region_name=full_config['aws_template_storage_region_name'], access_key_id=full_config['aws_template_storage_access_key_id'], secret_access_key=full_config['aws_template_storage_secret_access_key']) log.warning("Uploading to AWS") release.apply_storage_commands({'aws': storage_provider}, storage_commands) log.warning("AWS CloudFormation templates now available at: {}".format( full_config['cloudformation_s3_url'])) # TODO(cmaloney): Print where the user can find the files in AWS # TODO(cmaloney): Dump out a JSON with machine paths to make scripting easier. return 1
def do_aws_cf_configure(): """Returns error code Generates AWS templates using a custom config.yaml """ # TODO(cmaloney): Move to Config class introduced in https://github.com/dcos/dcos/pull/623 config = Config(CONFIG_PATH) gen_config = config.as_gen_format() extra_sources = [ gen.build_deploy.aws.aws_base_source, aws_advanced_source, gen.build_deploy.aws.groups['master'][1]] sources, targets, _ = gen.get_dcosconfig_source_target_and_templates(gen_config, [], extra_sources) targets.append(get_aws_advanced_target()) resolver = gen.internals.resolve_configuration(sources, targets) # TODO(cmaloney): kill this function and make the API return the structured # results api as was always intended rather than the flattened / lossy other # format. This will be an API incompatible change. The messages format was # specifically so that there wouldn't be this sort of API incompatibility. messages = normalize_config_validation(resolver.status_dict) if messages: print_messages(messages) return 1 # TODO(cmaloney): This is really hacky but a lot simpler than merging all the config flows into # one currently. # Get out the calculated arguments and manually move critical calculated ones to the gen_config # object. # NOTE: the copying across, as well as validation is guaranteed to succeed because we've already # done a validation run. full_config = {k: v.value for k, v in resolver.arguments.items()} # TODO(cmaloney): Switch to using the targets gen_config['bootstrap_url'] = full_config['bootstrap_url'] gen_config['provider'] = full_config['provider'] gen_config['bootstrap_id'] = full_config['bootstrap_id'] gen_config['package_ids'] = full_config['package_ids'] gen_config['cloudformation_s3_url_full'] = full_config['cloudformation_s3_url_full'] # Convert the bootstrap_Variant string we have back to a bootstrap_id as used internally by all # the tooling (never has empty string, uses None to say "no variant") bootstrap_variant = full_config['bootstrap_variant'] if full_config['bootstrap_variant'] else None artifacts = list() for built_resource in list(gen.build_deploy.aws.do_create( tag='dcos_generate_config.sh --aws-cloudformation', build_name='Custom', reproducible_artifact_path=full_config['reproducible_artifact_path'], variant_arguments={bootstrap_variant: gen_config}, commit=full_config['dcos_image_commit'], all_completes=None)): artifacts += release.built_resource_to_artifacts(built_resource) artifacts += list(release.make_bootstrap_artifacts( full_config['bootstrap_id'], json.loads(full_config['package_ids']), bootstrap_variant, 'artifacts', )) # Upload all the artifacts to the config-id path and then print out what # the path that should be used is, as well as saving a local json file for # easy machine access / processing. repository = release.Repository( full_config['aws_template_storage_bucket_path'], None, 'config_id/' + full_config['config_id']) storage_commands = repository.make_commands({'core_artifacts': [], 'channel_artifacts': artifacts}) cf_dir = GENCONF_DIR + '/cloudformation' log.warning("Writing local copies to {}".format(cf_dir)) storage_provider = release.storage.local.LocalStorageProvider(cf_dir) release.apply_storage_commands({'local': storage_provider}, storage_commands) log.warning( "Generated templates locally available at %s", cf_dir + "/" + full_config["reproducible_artifact_path"]) # TODO(cmaloney): Print where the user can find the files locally if full_config['aws_template_upload'] == 'false': return 0 storage_provider = release.storage.aws.S3StorageProvider( bucket=full_config['aws_template_storage_bucket'], object_prefix=None, download_url=full_config['cloudformation_s3_url'], region_name=full_config['aws_template_storage_region_name'], access_key_id=full_config['aws_template_storage_access_key_id'], secret_access_key=full_config['aws_template_storage_secret_access_key']) log.warning("Uploading to AWS") release.apply_storage_commands({'aws': storage_provider}, storage_commands) log.warning("AWS CloudFormation templates now available at: {}".format( full_config['cloudformation_s3_url'])) # TODO(cmaloney): Print where the user can find the files in AWS # TODO(cmaloney): Dump out a JSON with machine paths to make scripting easier. return 0