def calculate(self, targets): for target in targets: self._calculate_target(target) if len(self._errors) or len(self._unset): raise ValidationError(self._errors, self._unset) return self._arguments
def validate_and_raise(sources, targets): # TODO(cmaloney): Make it so we only get out the dcosconfig target arguments not all the config target arguments. resolver = gen.internals.resolve_configuration(sources, targets) status = resolver.status_dict if status['status'] == 'errors': raise ValidationError(errors=status['errors'], unset=status['unset']) return resolver
def validate_arguments_strings(arguments: dict): errors = dict() # Validate that all keys and vlaues of arguments are strings for k, v in arguments.items(): if not isinstance(k, str): errors[''] = "All keys in arguments must be strings. '{}' isn't.".format(k) if not isinstance(v, str): errors[k] = ("All values in arguments must be strings. Value for argument {} isn't. " + "Given value: {}").format(k, v) if len(errors): raise ValidationError(errors, set())
def test_normalize_config_validation_exception(): errors = { 'key': { 'message': 'test' }, } validation_error = ValidationError(errors=errors, unset=set(['one', 'two'])) normalized = config.normalize_config_validation_exception(validation_error) expected = { 'key': 'test', 'one': 'Must set one, no way to calculate value.', 'two': 'Must set two, no way to calculate value.', } assert expected == normalized
def validate_all_arguments_match_parameters(parameters, setters, arguments): errors = dict() # Gather all possible parameters from templates as well as setter parameters. all_parameters = flatten_parameters(parameters) for setter_list in setters.values(): for setter in setter_list: all_parameters |= setter.parameters all_parameters.add(setter.name) all_parameters |= {name for name, value in setter.conditions} # Check every argument is in the set of parameters. for argument in arguments: if argument not in all_parameters: errors[argument] = 'Argument {} given but not in possible parameters {}'.format(argument, all_parameters) if len(errors): raise ValidationError(errors, set())
def generate(arguments, extra_templates=list(), extra_sources=list(), extra_targets=list()): # To maintain the old API where we passed arguments rather than the new name. user_arguments = arguments arguments = None sources, targets, templates = get_dcosconfig_source_target_and_templates( user_arguments, extra_templates, extra_sources) # TODO(cmaloney): Make it so we only get out the dcosconfig target arguments not all the config target arguments. resolver = gen.internals.resolve_configuration(sources, targets + extra_targets) status = resolver.status_dict if status['status'] == 'errors': raise ValidationError(errors=status['errors'], unset=status['unset']) # Gather out the late variables. The presence of late variables changes # whether or not a late package is created late_variables = dict() # TODO(branden): Get the late vars and expressions from resolver.late for source in sources: for setter_list in source.setters.values(): for setter in setter_list: if not setter.is_late: continue if setter.name not in resolver.late: continue # Skip late vars that aren't referenced by config. if not resolver.arguments[setter.name].is_finalized: continue # Validate a late variable should only have one source. assert setter.name not in late_variables late_variables[setter.name] = setter.late_expression argument_dict = { k: v.value for k, v in resolver.arguments.items() if v.is_finalized } # expanded_config is a special result which contains all other arguments. It has to come after # the calculation of all the other arguments so it can be filled with everything which was # calculated. Can't be calculated because that would have an infinite recursion problem (the set # of all arguments would want to include itself). # Explicitly / manaully setup so that it'll fit where we want it. # TODO(cmaloney): Make this late-bound by gen.internals argument_dict['expanded_config'] = textwrap.indent( json_prettyprint({ k: v for k, v in argument_dict.items() if not v.startswith(gen.internals.LATE_BIND_PLACEHOLDER_START) }), prefix=' ' * 3, ) log.debug("Final arguments:" + json_prettyprint(argument_dict)) # Fill in the template parameters # TODO(cmaloney): render_templates should ideally take the template targets. rendered_templates = render_templates(templates, argument_dict) # Validate there aren't any unexpected top level directives in any of the files # (likely indicates a misspelling) for name, template in rendered_templates.items(): if name == 'dcos-services.yaml': # yaml list of the service files assert isinstance(template, list) elif name == 'cloud-config.yaml': assert template.keys() <= CLOUDCONFIG_KEYS, template.keys() elif isinstance(template, str): # Not a yaml template pass else: # yaml template file log.debug("validating template file %s", name) assert template.keys() <= PACKAGE_KEYS, template.keys() # Find all files which contain late bind variables and turn them into a "late bind package" # TODO(cmaloney): check there are no late bound variables in cloud-config.yaml late_files, regular_files = extract_files_containing_late_variables( rendered_templates['dcos-config.yaml']['package']) # put the regular files right back rendered_templates['dcos-config.yaml'] = {'package': regular_files} def make_package_filename(package_id, extension): return 'packages/{0}/{1}{2}'.format(package_id.name, repr(package_id), extension) # Render all the cluster packages cluster_package_info = {} # Prepare late binding config, if any. late_package = build_late_package(late_files, argument_dict['config_id'], argument_dict['provider']) if late_variables: # Render the late binding package. This package will be downloaded onto # each cluster node during bootstrap and rendered into the final config # using the values from the late config file. late_package_id = PackageId(late_package['name']) late_package_filename = make_package_filename(late_package_id, '.dcos_config') os.makedirs(os.path.dirname(late_package_filename), mode=0o755) write_yaml(late_package_filename, {'package': late_package['package']}, default_flow_style=False) cluster_package_info[late_package_id.name] = { 'id': late_package['name'], 'filename': late_package_filename } # Add the late config file to cloud config. The expressions in # late_variables will be resolved by the service handling the cloud # config (e.g. Amazon CloudFormation). The rendered late config file # on a cluster node's filesystem will contain the final values. rendered_templates['cloud-config.yaml']['root'].append({ 'path': '/etc/mesosphere/setup-flags/late-config.yaml', 'permissions': '0644', 'owner': 'root', # TODO(cmaloney): don't prettyprint to save bytes. # NOTE: Use yaml here simply to make avoiding painful escaping and # unescaping easier. 'content': render_yaml({ 'late_bound_package_id': late_package['name'], 'bound_values': late_variables }) }) # Render the rest of the packages. for package_id_str in json.loads(argument_dict['cluster_packages']): package_id = PackageId(package_id_str) package_filename = make_package_filename(package_id, '.tar.xz') # Build the package do_gen_package(rendered_templates[package_id.name + '.yaml'], package_filename) cluster_package_info[package_id.name] = { 'id': package_id_str, 'filename': package_filename } # Convert cloud-config to just contain write_files rather than root cc = rendered_templates['cloud-config.yaml'] # Shouldn't contain any packages. Providers should pull what they need to # late bind out of other packages via cc_package_file. assert 'package' not in cc cc_root = cc.pop('root', []) # Make sure write_files exists. assert 'write_files' not in cc cc['write_files'] = [] # Do the transform for item in cc_root: assert item['path'].startswith('/') cc['write_files'].append(item) rendered_templates['cloud-config.yaml'] = cc # Add in the add_services util. Done here instead of the initial # map since we need to bind in parameters def add_services(cloudconfig, cloud_init_implementation): return add_units(cloudconfig, rendered_templates['dcos-services.yaml'], cloud_init_implementation) utils.add_services = add_services return Bunch({ 'arguments': argument_dict, 'cluster_packages': cluster_package_info, 'templates': rendered_templates, 'utils': utils })
def generate( arguments, extra_templates=list(), cc_package_files=list()): # To maintain the old API where we passed arguments rather than the new name. user_arguments = arguments arguments = None sources, targets, templates = get_dcosconfig_source_target_and_templates(user_arguments, extra_templates) # TODO(cmaloney): Make it so we only get out the dcosconfig target arguments not all the config target arguments. resolver = gen.internals.resolve_configuration(sources, targets, user_arguments) status = resolver.status_dict if status['status'] == 'errors': raise ValidationError(errors=status['errors'], unset=status['unset']) argument_dict = {k: v.value for k, v in resolver.arguments.items()} log.debug("Final arguments:" + json_prettyprint(argument_dict)) # expanded_config is a special result which contains all other arguments. It has to come after # the calculation of all the other arguments so it can be filled with everything which was # calculated. Can't be calculated because that would have an infinite recursion problem (the set # of all arguments would want to include itself). # Explicitly / manaully setup so that it'll fit where we want it. # TODO(cmaloney): Make this late-bound by gen.internals argument_dict['expanded_config'] = textwrap.indent(json_prettyprint(argument_dict), prefix=' ' * 3) # Fill in the template parameters # TODO(cmaloney): render_templates should ideally take the template targets. rendered_templates = render_templates(templates, argument_dict) # Validate there aren't any unexpected top level directives in any of the files # (likely indicates a misspelling) for name, template in rendered_templates.items(): if name == 'dcos-services.yaml': # yaml list of the service files assert isinstance(template, list) elif name == 'cloud-config.yaml': assert template.keys() <= CLOUDCONFIG_KEYS, template.keys() elif isinstance(template, str): # Not a yaml template pass else: # yaml template file log.debug("validating template file %s", name) assert template.keys() <= PACKAGE_KEYS, template.keys() # Extract cc_package_files out of the dcos-config template and put them into # the cloud-config package. cc_package_files, dcos_config_files = extract_files_with_path(rendered_templates['dcos-config.yaml']['package'], cc_package_files) rendered_templates['dcos-config.yaml'] = {'package': dcos_config_files} # Add a empty pkginfo.json to the cc_package_files. # Also assert there isn't one already (can only write out a file once). for item in cc_package_files: assert item['path'] != '/pkginfo.json' # If there aren't any files for a cloud-config package don't make one start # existing adding a pkginfo.json if len(cc_package_files) > 0: cc_package_files.append({ "path": "/pkginfo.json", "content": "{}"}) for item in cc_package_files: assert item['path'].startswith('/') item['path'] = '/etc/mesosphere/setup-packages/dcos-provider-{}--setup'.format( argument_dict['provider']) + item['path'] rendered_templates['cloud-config.yaml']['root'].append(item) cluster_package_info = {} # Render all the cluster packages for package_id_str in json.loads(argument_dict['cluster_packages']): package_id = PackageId(package_id_str) package_filename = 'packages/{}/{}.tar.xz'.format( package_id.name, package_id_str) # Build the package do_gen_package(rendered_templates[package_id.name + '.yaml'], package_filename) cluster_package_info[package_id.name] = { 'id': package_id_str, 'filename': package_filename } # Convert cloud-config to just contain write_files rather than root cc = rendered_templates['cloud-config.yaml'] # Shouldn't contain any packages. Providers should pull what they need to # late bind out of other packages via cc_package_file. assert 'package' not in cc cc_root = cc.pop('root', []) # Make sure write_files exists. assert 'write_files' not in cc cc['write_files'] = [] # Do the transform for item in cc_root: assert item['path'].startswith('/') cc['write_files'].append(item) rendered_templates['cloud-config.yaml'] = cc # Add in the add_services util. Done here instead of the initial # map since we need to bind in parameters def add_services(cloudconfig, cloud_init_implementation): return add_units(cloudconfig, rendered_templates['dcos-services.yaml'], cloud_init_implementation) utils.add_services = add_services return Bunch({ 'arguments': argument_dict, 'cluster_packages': cluster_package_info, 'templates': rendered_templates, 'utils': utils })