def get_dcosconfig_source_target_and_templates( user_arguments: Dict[str, str], extra_templates: List[str], extra_sources: List[gen.internals.Source]): log.info("Generating configuration files...") # TODO(cmaloney): Make these all just defined by the base calc.py package_names = ['dcos-config', 'dcos-metadata'] template_filenames = ['dcos-config.yaml', 'cloud-config.yaml', 'dcos-metadata.yaml', 'dcos-services.yaml'] # TODO(cmaloney): Check there are no duplicates between templates and extra_template_files template_filenames += extra_templates # Re-arrange templates to be indexed by common name. Only allow multiple for one key if the key # is yaml (ends in .yaml). templates = dict() for filename in template_filenames: key = os.path.basename(filename) templates.setdefault(key, list()) templates[key].append(filename) if len(templates[key]) > 1 and not key.endswith('.yaml'): raise Exception( "Internal Error: Only know how to merge YAML templates at this point in time. " "Can't merge template {} in template_list {}".format(filename, templates[key])) targets = target_from_templates(templates) base_source = gen.internals.Source(is_user=False) base_source.add_entry(gen.calc.entry, replace_existing=False) # Allow overriding calculators with a `gen_extra/calc.py` if it exists if os.path.exists('gen_extra/calc.py'): mod = importlib.machinery.SourceFileLoader('gen_extra.calc', 'gen_extra/calc.py').load_module() base_source.add_entry(mod.entry, replace_existing=True) def add_builtin(name, value): base_source.add_must(name, json_prettyprint(value)) sources = [base_source] + extra_sources # TODO(cmaloney): Hash the contents of all the templates rather than using the list of filenames # since the filenames might not live in this git repo, or may be locally modified. add_builtin('template_filenames', template_filenames) add_builtin('package_names', list(package_names)) # TODO(cmaloney): user_arguments needs to be a temporary_str since we need to only include used # arguments inside of it. add_builtin('user_arguments', user_arguments) # Add a builtin for expanded_config, so that we won't get unset argument errors. The temporary # value will get replaced with the set of all arguments once calculation is complete temporary_str = 'DO NOT USE THIS AS AN ARGUMENT TO OTHER ARGUMENTS. IT IS TEMPORARY' add_builtin('expanded_config', temporary_str) # Note: must come last so the hash of the "base_source" this is beign added to contains all the # variables but this. add_builtin('sources_id', hash_checkout([hash_checkout(source.make_id()) for source in sources])) return sources, targets, templates
def get_dcosconfig_source_target_and_templates( user_arguments: dict, extra_templates: List[str], extra_sources: List[gen.internals.Source]): log.info("Generating configuration files...") # TODO(cmaloney): Make these all just defined by the base calc.py package_names = ['dcos-config', 'dcos-metadata'] template_filenames = ['dcos-config.yaml', 'cloud-config.yaml', 'dcos-metadata.yaml', 'dcos-services.yaml'] # TODO(cmaloney): Check there are no duplicates between templates and extra_template_files template_filenames += extra_templates # Re-arrange templates to be indexed by common name. Only allow multiple for one key if the key # is yaml (ends in .yaml). templates = dict() for filename in template_filenames: key = os.path.basename(filename) templates.setdefault(key, list()) templates[key].append(filename) if len(templates[key]) > 1 and not key.endswith('.yaml'): raise Exception( "Internal Error: Only know how to merge YAML templates at this point in time. " "Can't merge template {} in template_list {}".format(filename, templates[key])) targets = target_from_templates(templates) base_source = gen.internals.Source(is_user=False) base_source.add_entry(gen.calc.entry, replace_existing=False) # Allow overriding calculators with a `gen_extra/calc.py` if it exists if os.path.exists('gen_extra/calc.py'): mod = importlib.machinery.SourceFileLoader('gen_extra.calc', 'gen_extra/calc.py').load_module() base_source.add_entry(mod.entry, replace_existing=True) def add_builtin(name, value): base_source.add_must(name, json_prettyprint(value)) sources = [base_source, user_arguments_to_source(user_arguments)] + extra_sources # TODO(cmaloney): Hash the contents of all the templates rather than using the list of filenames # since the filenames might not live in this git repo, or may be locally modified. add_builtin('template_filenames', template_filenames) add_builtin('package_names', list(package_names)) # TODO(cmaloney): user_arguments needs to be a temporary_str since we need to only include used # arguments inside of it. add_builtin('user_arguments', user_arguments) # Add a builtin for expanded_config, so that we won't get unset argument errors. The temporary # value will get replaced with the set of all arguments once calculation is complete temporary_str = 'DO NOT USE THIS AS AN ARGUMENT TO OTHER ARGUMENTS. IT IS TEMPORARY' add_builtin('expanded_config', temporary_str) # Note: must come last so the hash of the "base_source" this is beign added to contains all the # variables but this. add_builtin('sources_id', hash_checkout([hash_checkout(source.make_id()) for source in sources])) return sources, targets, templates
def make_id(self): # {key: [hash_checkout(setter.make_id() for setter in setters)] # for key, setters in self.setters.items()} setter_ids = list() for setter_list in self.setters.values(): for setter in setter_list: setter_ids.append(hash_checkout(setter.make_id())) return { 'setters': setter_ids, 'validate': [hash_checkout(function_id(fn)) for fn in self.validate], 'is_user': self.is_user }
def make_id(self): # {key: [hash_checkout(setter.make_id() for setter in setters)] # for key, setters in self.setters.items()} setter_ids = list() for setter_list in self.setters.values(): for setter in setter_list: setter_ids.append(hash_checkout(setter.make_id())) return { 'setters': setter_ids, 'validate': [hash_checkout(function_id(fn)) for fn in self.validate], 'is_user': self.is_user }
def calculate_config_id(dcos_image_commit, user_arguments, template_filenames): return hash_checkout( { "commit": dcos_image_commit, "user_arguments": json.loads(user_arguments), "template_filenames": json.loads(template_filenames), } )
def __init__(self, name: str, value: Union[str, Callable], is_optional: bool, conditions: List[Tuple[str, str]], is_user: bool): self.name = name self.is_optional = is_optional self.conditions = conditions self.is_user = is_user self._value_id = hash_checkout(value_id(value)) def get_value(): return value if isinstance(value, str): self.calc = get_value self.parameters = set() else: assert callable( value), "{} should be a string or callable. Got: {}".format( name, value) self.calc = value self.parameters = get_function_parameters(value)
def __init__( self, name: str, value: Union[str, Callable], is_optional: bool, conditions: List[Tuple[str, str]], is_user: bool): self.name = name self.is_optional = is_optional self.conditions = conditions self.is_user = is_user self._value_id = hash_checkout(value_id(value)) def get_value(): return value if isinstance(value, str): self.calc = get_value self.parameters = set() else: assert callable(value), "{} should be a string or callable. Got: {}".format(name, value) self.calc = value self.parameters = get_function_parameters(value)
def calculate_config_id(dcos_image_commit, user_arguments, template_filenames): return hash_checkout({ "commit": dcos_image_commit, "user_arguments": json.loads(user_arguments), "template_filenames": json.loads(template_filenames)})
def calculate_config_id(dcos_image_commit, template_filenames, sources_id): return hash_checkout({ "commit": dcos_image_commit, "template_filenames": json.loads(template_filenames), "sources_id": sources_id })
def calculate_config_id(dcos_image_commit, template_filenames, sources_id): return hash_checkout({ "commit": dcos_image_commit, "template_filenames": json.loads(template_filenames), "sources_id": sources_id})