def get_dcosconfig_source_target_and_templates(
user_arguments: Dict[str, str],
extra_templates: List[str],
extra_sources: List[gen.internals.Source]):
log.info("Generating configuration files...")
# TODO(cmaloney): Make these all just defined by the base calc.py
package_names = ['dcos-config', 'dcos-metadata']
template_filenames = ['dcos-config.yaml', 'cloud-config.yaml', 'dcos-metadata.yaml', 'dcos-services.yaml']
# TODO(cmaloney): Check there are no duplicates between templates and extra_template_files
template_filenames += extra_templates
# Re-arrange templates to be indexed by common name. Only allow multiple for one key if the key
# is yaml (ends in .yaml).
templates = dict()
for filename in template_filenames:
key = os.path.basename(filename)
templates.setdefault(key, list())
templates[key].append(filename)
if len(templates[key]) > 1 and not key.endswith('.yaml'):
raise Exception(
"Internal Error: Only know how to merge YAML templates at this point in time. "
"Can't merge template {} in template_list {}".format(filename, templates[key]))
targets = target_from_templates(templates)
base_source = gen.internals.Source(is_user=False)
base_source.add_entry(gen.calc.entry, replace_existing=False)
# Allow overriding calculators with a `gen_extra/calc.py` if it exists
if os.path.exists('gen_extra/calc.py'):
mod = importlib.machinery.SourceFileLoader('gen_extra.calc', 'gen_extra/calc.py').load_module()
base_source.add_entry(mod.entry, replace_existing=True)
def add_builtin(name, value):
base_source.add_must(name, json_prettyprint(value))
sources = [base_source] + extra_sources
# TODO(cmaloney): Hash the contents of all the templates rather than using the list of filenames
# since the filenames might not live in this git repo, or may be locally modified.
add_builtin('template_filenames', template_filenames)
add_builtin('package_names', list(package_names))
# TODO(cmaloney): user_arguments needs to be a temporary_str since we need to only include used
# arguments inside of it.
add_builtin('user_arguments', user_arguments)
# Add a builtin for expanded_config, so that we won't get unset argument errors. The temporary
# value will get replaced with the set of all arguments once calculation is complete
temporary_str = 'DO NOT USE THIS AS AN ARGUMENT TO OTHER ARGUMENTS. IT IS TEMPORARY'
add_builtin('expanded_config', temporary_str)
# Note: must come last so the hash of the "base_source" this is beign added to contains all the
# variables but this.
add_builtin('sources_id', hash_checkout([hash_checkout(source.make_id()) for source in sources]))
return sources, targets, templates
def get_dcosconfig_source_target_and_templates(
user_arguments: dict,
extra_templates: List[str],
extra_sources: List[gen.internals.Source]):
log.info("Generating configuration files...")
# TODO(cmaloney): Make these all just defined by the base calc.py
package_names = ['dcos-config', 'dcos-metadata']
template_filenames = ['dcos-config.yaml', 'cloud-config.yaml', 'dcos-metadata.yaml', 'dcos-services.yaml']
# TODO(cmaloney): Check there are no duplicates between templates and extra_template_files
template_filenames += extra_templates
# Re-arrange templates to be indexed by common name. Only allow multiple for one key if the key
# is yaml (ends in .yaml).
templates = dict()
for filename in template_filenames:
key = os.path.basename(filename)
templates.setdefault(key, list())
templates[key].append(filename)
if len(templates[key]) > 1 and not key.endswith('.yaml'):
raise Exception(
"Internal Error: Only know how to merge YAML templates at this point in time. "
"Can't merge template {} in template_list {}".format(filename, templates[key]))
targets = target_from_templates(templates)
base_source = gen.internals.Source(is_user=False)
base_source.add_entry(gen.calc.entry, replace_existing=False)
# Allow overriding calculators with a `gen_extra/calc.py` if it exists
if os.path.exists('gen_extra/calc.py'):
mod = importlib.machinery.SourceFileLoader('gen_extra.calc', 'gen_extra/calc.py').load_module()
base_source.add_entry(mod.entry, replace_existing=True)
def add_builtin(name, value):
base_source.add_must(name, json_prettyprint(value))
sources = [base_source, user_arguments_to_source(user_arguments)] + extra_sources
# TODO(cmaloney): Hash the contents of all the templates rather than using the list of filenames
# since the filenames might not live in this git repo, or may be locally modified.
add_builtin('template_filenames', template_filenames)
add_builtin('package_names', list(package_names))
# TODO(cmaloney): user_arguments needs to be a temporary_str since we need to only include used
# arguments inside of it.
add_builtin('user_arguments', user_arguments)
# Add a builtin for expanded_config, so that we won't get unset argument errors. The temporary
# value will get replaced with the set of all arguments once calculation is complete
temporary_str = 'DO NOT USE THIS AS AN ARGUMENT TO OTHER ARGUMENTS. IT IS TEMPORARY'
add_builtin('expanded_config', temporary_str)
# Note: must come last so the hash of the "base_source" this is beign added to contains all the
# variables but this.
add_builtin('sources_id', hash_checkout([hash_checkout(source.make_id()) for source in sources]))
return sources, targets, templates
def make_id(self):
# {key: [hash_checkout(setter.make_id() for setter in setters)]
# for key, setters in self.setters.items()}
setter_ids = list()
for setter_list in self.setters.values():
for setter in setter_list:
setter_ids.append(hash_checkout(setter.make_id()))
return {
'setters': setter_ids,
'validate': [hash_checkout(function_id(fn)) for fn in self.validate],
'is_user': self.is_user
}
def make_id(self):
# {key: [hash_checkout(setter.make_id() for setter in setters)]
# for key, setters in self.setters.items()}
setter_ids = list()
for setter_list in self.setters.values():
for setter in setter_list:
setter_ids.append(hash_checkout(setter.make_id()))
return {
'setters': setter_ids,
'validate': [hash_checkout(function_id(fn)) for fn in self.validate],
'is_user': self.is_user
}
def calculate_config_id(dcos_image_commit, user_arguments, template_filenames):
return hash_checkout(
{
"commit": dcos_image_commit,
"user_arguments": json.loads(user_arguments),
"template_filenames": json.loads(template_filenames),
}
)
def __init__(self, name: str, value: Union[str,
Callable], is_optional: bool,
conditions: List[Tuple[str, str]], is_user: bool):
self.name = name
self.is_optional = is_optional
self.conditions = conditions
self.is_user = is_user
self._value_id = hash_checkout(value_id(value))
def get_value():
return value
if isinstance(value, str):
self.calc = get_value
self.parameters = set()
else:
assert callable(
value), "{} should be a string or callable. Got: {}".format(
name, value)
self.calc = value
self.parameters = get_function_parameters(value)
def __init__(
self,
name: str,
value: Union[str, Callable],
is_optional: bool,
conditions: List[Tuple[str, str]],
is_user: bool):
self.name = name
self.is_optional = is_optional
self.conditions = conditions
self.is_user = is_user
self._value_id = hash_checkout(value_id(value))
def get_value():
return value
if isinstance(value, str):
self.calc = get_value
self.parameters = set()
else:
assert callable(value), "{} should be a string or callable. Got: {}".format(name, value)
self.calc = value
self.parameters = get_function_parameters(value)
def calculate_config_id(dcos_image_commit, user_arguments, template_filenames):
return hash_checkout({
"commit": dcos_image_commit,
"user_arguments": json.loads(user_arguments),
"template_filenames": json.loads(template_filenames)})
def calculate_config_id(dcos_image_commit, template_filenames, sources_id):
return hash_checkout({
"commit": dcos_image_commit,
"template_filenames": json.loads(template_filenames),
"sources_id": sources_id
})
def calculate_config_id(dcos_image_commit, template_filenames, sources_id):
return hash_checkout({
"commit": dcos_image_commit,
"template_filenames": json.loads(template_filenames),
"sources_id": sources_id})
