def from_dict(cls, d: ty.Mapping) -> _Renderer:
"""Instantiate a new renderer from a dictionary of instructions."""
# raise error if invalid
_validate_renderer(d)
pkg_manager = d["pkg_manager"]
users = d.get("existing_users", None)
# create new renderer object
renderer = cls(pkg_manager=pkg_manager, users=users)
for mapping in d["instructions"]:
method_or_template = mapping["name"]
kwds = mapping["kwds"]
this_instance_method = getattr(renderer, method_or_template, None)
# Method exists and is something like 'copy', 'env', 'run', etc.
if this_instance_method is not None:
try:
this_instance_method(**kwds)
except Exception as e:
raise RendererError(
f"Error on step '{method_or_template}'. Please see the"
" traceback above for details.") from e
# This is actually a template.
else:
try:
renderer.add_registered_template(method_or_template,
**kwds)
except TemplateError as e:
raise RendererError(
f"Error on template '{method_or_template}'. Please see above"
" for more information.") from e
return renderer
def _render_string_from_template(source: str,
template: _BaseInstallationTemplate) -> str:
"""Take a string from a template and render """
# TODO: we could use a while loop or recursive function to render the template until
# there are no jinja-specific things. At this point, we support one level of
# nesting.
n_renders = 0
max_renders = 20
err = (
"A template included in this renderer raised an error. Please check the"
" template definition. A required argument might not be included in the"
" required arguments part of the template. Variables in the template should"
" start with `self.`.")
# Render the string again. This is sometimes necessary because some defaults in the
# template are rendered as {{ self.X }}. These defaults need to be rendered again.
while (_jinja_env.variable_start_string in source
and _jinja_env.variable_end_string in source):
source = source.replace("self.", "template.")
tmpl = _jinja_env.from_string(source)
try:
source = tmpl.render(template=template)
except jinja2.exceptions.UndefinedError as e:
raise RendererError(err) from e
n_renders += 1
if n_renders > max_renders:
raise RendererError(
f"reached maximum rendering iterations ({max_renders}). Templates"
f" should not nest variables more than {max_renders} times.")
return source
def add_registered_template(self,
name: str,
method: installation_methods_type = None,
**kwds) -> _Renderer:
# Template was validated at registration time.
template_dict = _TemplateRegistry.get(name)
# By default, prefer 'binaries', but use 'source' if 'binaries' is not defined.
# TODO: should we require user to provide method?
if method is None:
method = "binaries" if "binaries" in template_dict else "source"
if method not in template_dict:
raise RendererError(
f"Installation method '{method}' not defined for template '{name}'."
" Options are '{}'.".format("', '".join(template_dict.keys())))
binaries_kwds = source_kwds = None
if method == "binaries":
binaries_kwds = kwds
elif method == "source":
source_kwds = kwds
template = Template(template=template_dict,
binaries_kwds=binaries_kwds,
source_kwds=source_kwds)
self.add_template(template=template, method=method)
return self
def __init__(self,
pkg_manager: pkg_managers_type,
users: ty.Optional[ty.Set[str]] = None) -> None:
if pkg_manager not in allowed_pkg_managers:
raise RendererError(
"Unknown package manager '{}'. Allowed package managers are"
" '{}'.".format(pkg_manager,
"', '".join(allowed_pkg_managers)))
self.pkg_manager = pkg_manager
self._users = {"root"} if users is None else users
# This keeps track of the current user. This is useful when saving the JSON
# specification to JSON, because if we are not root, we can change to root,
# write the file, and return to whichever user we were.
self._current_user = "******"
self._instructions: ty.Mapping = {
"pkg_manager": self.pkg_manager,
"existing_users": list(self._users),
"instructions": [],
}
# Strings (comments) that indicate the beginning and end of saving the JSON
# spec to a file. This helps us in testing because sometimes we don't care
# about the JSON but we do care about everything else. We can remove the
# content between these two strings.
self._json_save_start = "# Save specification to JSON."
self._json_save_end = "# End saving to specification to JSON."
def _validate_renderer(d):
"""Validate renderer dictionary against JSON schema. Raise exception if invalid."""
try:
jsonschema.validate(d, schema=_RENDERER_SCHEMA)
except jsonschema.exceptions.ValidationError as e:
raise RendererError(
f"Invalid renderer dictionary: {e.message}.") from e
def _install(pkgs: ty.List[str], pkg_manager: str, opts: str = None) -> str:
if pkg_manager == "apt":
return _apt_install(pkgs, opts)
elif pkg_manager == "yum":
return _yum_install(pkgs, opts)
# TODO: add debs here?
else:
raise RendererError(f"Unknown package manager '{pkg_manager}'.")
def from_(self, base_image: str) -> SingularityRenderer:
if "://" not in base_image:
bootstrap = "docker"
image = base_image
elif base_image.startswith("docker://"):
bootstrap = "docker"
image = base_image[9:]
elif base_image.startswith("library://"):
bootstrap = "library"
image = base_image[10:]
else:
raise RendererError("Unknown singularity bootstrap agent.")
self._header = {"bootstrap": bootstrap, "from_": image}
return self
def _raise_helper(msg: str) -> ty.NoReturn:
raise RendererError(msg)
def add_template(self, template: Template,
method: installation_methods_type) -> _Renderer:
"""Add a template to the renderer.
Parameters
----------
template : Template
The template to add. To reference templates by name, use
`.add_registered_template`.
method : str
The method to use to install the software described in the template.
"""
if not isinstance(template, Template):
raise RendererError(
"template must be an instance of 'Template' but got"
f" '{type(template)}'.")
if method not in allowed_installation_methods:
raise RendererError("method must be '{}' but got '{}'.".format(
"', '".join(sorted(allowed_installation_methods)), method))
template_method: _BaseInstallationTemplate = getattr(template, method)
if template_method is None:
raise RendererError(
f"template does not have entry for: '{method}'")
# Validate kwds passed by user to template, and raise an exception if any are
# invalid.
template_method.validate_kwds()
# TODO: print a message if the template has a nonempty `alert` property.
# If we print to stdout, however, we can cause problems if the user is piping
# the output to a file or directly to a container build command.
# If we keep the `self.VAR` syntax of the template, then we need to pass
# `self=template_method` to the renderer function. But that function is an
# instance method, so passing `self` will override the `self` argument.
# To get around this, we replace `self.` with something that is not an
# argument to the renderer function.
# Add environment (render any jinja templates).
if template_method.env:
d: ty.Mapping[str, str] = {
_render_string_from_template(k, template_method):
_render_string_from_template(v, template_method)
for k, v in template_method.env.items()
}
self.env(**d)
# Patch the `template_method.install_dependencies` instance method so it can be
# used (ie rendered) in a template and have access to the pkg_manager requested.
def install_patch(inner_self: _BaseInstallationTemplate,
pkgs: ty.List[str],
opts: str = None) -> str:
return _install(pkgs=pkgs, pkg_manager=self.pkg_manager)
# mypy complains when we try to patch a class, so we do it behind its back with
# setattr. See https://github.com/python/mypy/issues/2427
setattr(template_method, "install",
types.MethodType(install_patch, template_method))
# Set pkg_manager onto the template.
setattr(template_method, "pkg_manager", self.pkg_manager)
# Patch the `template_method.install_dependencies` instance method so it can be
# used (ie rendered) in a template and have access to the pkg_manager requested.
def install_dependencies_patch(inner_self: _BaseInstallationTemplate,
opts: str = None) -> str:
# TODO: test that template with empty dependencies (apt: []) does not render
# any installation of dependencies.
cmd = ""
pkgs = inner_self.dependencies(pkg_manager=self.pkg_manager)
if pkgs:
cmd += _install(pkgs=pkgs,
pkg_manager=self.pkg_manager,
opts=opts)
if self.pkg_manager == "apt":
debs = inner_self.dependencies("debs")
if debs:
cmd += "\n" + _apt_install_debs(debs)
return cmd
# mypy complains when we try to patch a class, so we do it behind its back with
# setattr. See https://github.com/python/mypy/issues/2427
setattr(
template_method,
"install_dependencies",
types.MethodType(install_dependencies_patch, template_method),
)
# Add installation instructions (render any jinja templates).
if template_method.instructions:
command = _render_string_from_template(
template_method.instructions, template_method)
# TODO: raise exception here or skip the run instruction?
if not command.strip():
raise RendererError(
f"empty rendered instructions in {template.name}")
self.run(command)
return self