def export(self) -> CM:
"""Export configuration template into CommentedMap.
:raises SPSDKError: Error
:return: Configuration template in CM.
"""
loc_schemas = copy.deepcopy(self.schemas)
# 1. Do pre-merge by schema titles
pre_merged: Dict[str, Any] = {}
for schema in loc_schemas:
title = schema.get("title", "General Options")
if title in pre_merged:
deepmerge.always_merger.merge(pre_merged[title], schema)
else:
pre_merged[title] = schema
cfg = CM()
# 2. Add main title of configuration
cfg.yaml_set_start_comment(f"=========== {self.main_title} ===========\n")
# 3. Go through all individual logic blocks
for block in pre_merged.values():
try:
self._fill_up_block(cfg, block)
title = block.get("title", "General Options")
description = block.get("description", "")
cfg.yaml_set_comment_before_after_key(
list(block["properties"].keys())[0], f" \n == {title} == \n {description}"
)
except Exception as exc:
raise SPSDKError(f"Template generation failed: {str(exc)}") from exc
return cfg
def add_new_lines_after_section(recipe_yaml: CommentedMap) -> CommentedMap:
for section in recipe_yaml.keys():
if section == "package":
recipe_yaml.yaml_set_comment_before_after_key(section, "\n\n")
else:
recipe_yaml.yaml_set_comment_before_after_key(section, "\n")
return recipe_yaml
def _add_new_lines_after_section(
self, recipe_yaml: CommentedMap) -> CommentedMap:
for section in recipe_yaml.keys():
if section == "package":
continue
recipe_yaml.yaml_set_comment_before_after_key(section, "\n")
return recipe_yaml
def prepare_document_formatting(
metadata_doc: Mapping,
doc_friendly_label: str = "",
include_source_url: Union[bool, str] = False,
):
"""
Try to format a raw document for readability.
This will change property order, add comments on the type & source url.
"""
def get_property_priority(ordered_properties: List, keyval):
key, val = keyval
if key not in ordered_properties:
return 999
return ordered_properties.index(key)
header_comments = []
if doc_friendly_label:
header_comments.append(doc_friendly_label)
if include_source_url:
if include_source_url is True:
include_source_url = flask.request.url
header_comments.append(f"url: {include_source_url}")
# Give the document the same order as eo-datasets. It's far more readable (ID/names first, sources last etc.)
ordered_metadata = CommentedMap(
sorted(
metadata_doc.items(),
key=functools.partial(get_property_priority,
EODATASETS_PROPERTY_ORDER),
))
# Order any embedded ones too.
if "lineage" in ordered_metadata:
ordered_metadata["lineage"] = dict(
sorted(
ordered_metadata["lineage"].items(),
key=functools.partial(get_property_priority,
EODATASETS_LINEAGE_PROPERTY_ORDER),
))
if "source_datasets" in ordered_metadata["lineage"]:
for type_, source_dataset_doc in ordered_metadata["lineage"][
"source_datasets"].items():
ordered_metadata["lineage"]["source_datasets"][
type_] = prepare_document_formatting(source_dataset_doc)
# Products have an embedded metadata doc (subset of dataset metadata)
if "metadata" in ordered_metadata:
ordered_metadata["metadata"] = prepare_document_formatting(
ordered_metadata["metadata"])
if header_comments:
# Add comments above the first key of the document.
ordered_metadata.yaml_set_comment_before_after_key(
next(iter(metadata_doc.keys())),
before="\n".join(header_comments),
)
return ordered_metadata
def generate_orchestration_playbook(self, url=None, namespace=None, local_images=True, **kwargs):
"""
Generate an Ansible playbook to orchestrate services.
:param url: registry URL where images will be pulled from
:param namespace: registry namespace
:param local_images: bypass pulling images, and use local copies
:return: playbook dict
"""
for service_name in self.services:
image = self.get_latest_image_for_service(service_name)
if local_images:
self.services[service_name]['image'] = image.tags[0]
else:
if namespace is not None:
image_url = urljoin('{}/'.format(urljoin(url, namespace)), image.tags[0])
else:
image_url = urljoin(url, image.tags[0])
self.services[service_name]['image'] = image_url
if kwargs.get('k8s_auth'):
self.k8s_client.set_authorization(kwargs['auth'])
play = CommentedMap()
play['name'] = u'Manage the lifecycle of {} on {}'.format(self.project_name, self.display_name)
play['hosts'] = 'localhost'
play['gather_facts'] = 'no'
play['connection'] = 'local'
play['roles'] = CommentedSeq()
play['tasks'] = CommentedSeq()
role = CommentedMap([
('role', 'kubernetes-modules')
])
play['roles'].append(role)
play.yaml_set_comment_before_after_key(
'roles', before='Include Ansible Kubernetes and OpenShift modules', indent=4)
play.yaml_set_comment_before_after_key('tasks', before='Tasks for setting the application state. '
'Valid tags include: start, stop, restart, destroy', indent=4)
play['tasks'].append(self.deploy.get_namespace_task(state='present', tags=['start']))
play['tasks'].append(self.deploy.get_namespace_task(state='absent', tags=['destroy']))
play['tasks'].extend(self.deploy.get_service_tasks(tags=['start']))
play['tasks'].extend(self.deploy.get_deployment_tasks(engine_state='stop', tags=['stop', 'restart']))
play['tasks'].extend(self.deploy.get_deployment_tasks(tags=['start', 'restart']))
play['tasks'].extend(self.deploy.get_pvc_tasks(tags=['start']))
playbook = CommentedSeq()
playbook.append(play)
logger.debug(u'Created playbook to run project', playbook=playbook)
return playbook
def generate_orchestration_playbook(self, url=None, namespace=None, local_images=True, **kwargs):
"""
Generate an Ansible playbook to orchestrate services.
:param url: registry URL where images will be pulled from
:param namespace: registry namespace
:param local_images: bypass pulling images, and use local copies
:return: playbook dict
"""
for service_name in self.services:
image = self.get_latest_image_for_service(service_name)
if local_images:
self.services[service_name]['image'] = image.tags[0]
else:
self.services[service_name]['image'] = urljoin(urljoin(url, namespace), image.tags[0])
if kwargs.get('k8s_auth'):
self.k8s_client.set_authorization(kwargs['auth'])
play = CommentedMap()
play['name'] = 'Manage the lifecycle of {} on {}'.format(self.project_name, self.display_name)
play['hosts'] = 'localhost'
play['gather_facts'] = 'no'
play['connection'] = 'local'
play['roles'] = CommentedSeq()
play['tasks'] = CommentedSeq()
role = CommentedMap([
('role', 'kubernetes-modules')
])
play['roles'].append(role)
play.yaml_set_comment_before_after_key(
'roles', before='Include Ansible Kubernetes and OpenShift modules', indent=4)
play.yaml_set_comment_before_after_key('tasks', before='Tasks for setting the application state. '
'Valid tags include: start, stop, restart, destroy', indent=4)
play['tasks'].append(self.deploy.get_namespace_task(state='present', tags=['start']))
play['tasks'].append(self.deploy.get_namespace_task(state='absent', tags=['destroy']))
play['tasks'].extend(self.deploy.get_service_tasks(tags=['start']))
play['tasks'].extend(self.deploy.get_deployment_tasks(engine_state='stop', tags=['stop', 'restart']))
play['tasks'].extend(self.deploy.get_deployment_tasks(tags=['start', 'restart']))
play['tasks'].extend(self.deploy.get_pvc_tasks(tags=['start']))
playbook = CommentedSeq()
playbook.append(play)
logger.debug(u'Created playbook to run project', playbook=playbook)
return playbook
def export(self) -> CM:
"""Export configuration template into CommentedMap.
:raises SPSDKError: Error
:return: Configuration template in CM.
"""
loc_schemas = copy.deepcopy(self.schemas)
# 1. Get blocks with their titles and lists of their keys
block_list: Dict[str, Dict[str, List[str]]] = {}
for schema in loc_schemas:
title = schema.get("title", "General Options")
if title in block_list:
block_list[title]["properties"].extend(
self._get_schema_block_keys(schema))
else:
block_list[title] = {}
block_list[title]["properties"] = self._get_schema_block_keys(
schema)
block_list[title]["description"] = schema.get(
"description", "")
# 2. Merge all schemas together to get whole single schema
merged: Dict[str, Any] = {}
for schema in loc_schemas:
deepmerge.always_merger.merge(merged, schema)
cfg = CM()
# 3. Add main title of configuration
cfg.yaml_set_start_comment(
f"=========== {self.main_title} ===========\n")
# 4. Go through all individual logic blocks
for title, info in block_list.items():
try:
self._fill_up_block(cfg, merged, info["properties"])
description = info["description"]
cfg.yaml_set_comment_before_after_key(
info["properties"][0],
f" \n == {title} == \n {description}")
except Exception as exc:
raise SPSDKError(
f"Template generation failed: {str(exc)}") from exc
return cfg
def _recursive_build_dict(self, comment_map: CommentedMap,
source_dict: dict, index: int,
deepness: int) -> int:
# If you find a way to sanely do this without going over it multiple times and not having it be recursive, be
# my guest.
cur_index = index
for key in source_dict:
if not isinstance(key, str) or not key.startswith("_"):
if isinstance(source_dict[key], dict):
new_map = CommentedMap()
comment_map.insert(cur_index, key, new_map)
cur_index += 1
cur_index = self._recursive_build_dict(
new_map, source_dict[key], cur_index, deepness + 1)
else:
comment_map.insert(cur_index, key, source_dict[key])
cur_index += 1
if key_comment := source_dict.get(f"_c_{key}", None):
comment_map.yaml_set_comment_before_after_key(
key, key_comment, deepness * 4)
def prepare_formatting(d: Mapping) -> CommentedMap:
"""
Format an eo3 dataset dict for human-readable yaml serialisation.
This will order fields, add whitespace, comments, etc.
Output is intended for ruamel.yaml.
"""
# Sort properties for readability.
doc = CommentedMap(sorted(d.items(), key=_eo3_key_order))
doc["properties"] = CommentedMap(
sorted(doc["properties"].items(), key=_stac_key_order))
# Whitespace
doc.yaml_set_comment_before_after_key("$schema", before="Dataset")
if "geometry" in doc:
# Set some numeric fields to be compact yaml format.
_use_compact_format(doc["geometry"], "coordinates")
if "grids" in doc:
for grid in doc["grids"].values():
_use_compact_format(grid, "shape", "transform")
_add_space_before(
doc,
"label" if "label" in doc else "id",
"crs",
"properties",
"measurements",
"accessories",
"lineage",
"location",
"locations",
)
p: CommentedMap = doc["properties"]
p.yaml_add_eol_comment("# Ground sample distance (m)", "eo:gsd")
return doc
def _recursive_build_dict(self, comment_map: CommentedMap,
source_dict: dict, index: int,
deepness: int) -> int:
# If you find a way to sanely do this without going over it multiple times and not having it be recursive, be
# my guest.
cur_index = index
for key in source_dict:
if not key.startswith("_"):
if isinstance(source_dict[key], dict):
new_map = CommentedMap()
comment_map.insert(cur_index, key, new_map)
cur_index += 1
cur_index = self._recursive_build_dict(
new_map, source_dict[key], cur_index, deepness + 1)
else:
comment_map.insert(cur_index, key, source_dict[key])
cur_index += 1
# TODO: Change following if statement to use the walrus operator once 3.8+ becomes minimum.
key_comment = source_dict.get(f"_c_{key}", None)
if key_comment:
comment_map.yaml_set_comment_before_after_key(
key, key_comment, deepness * 4)
return cur_index
def r_vars(c, indent: int):
indent += 2
if is_dataclass(c):
cm = CommentedMap(
{x: r_vars(y, indent)
for x, y in vars(c).items()})
if not disable_comments:
[
cm.yaml_set_comment_before_after_key(
k,
after=dedent(v.__doc__).strip(),
after_indent=indent) for k, v in vars(c).items()
if is_dataclass(v) and getattr(v, "__doc__")
]
return cm
elif type(c) == list:
return [r_vars(x, indent) for x in c]
elif type(c) == dict:
return CommentedMap(
{x: r_vars(y, indent)
for x, y in c.items()})
else:
return c
def _add_space_before(d: CommentedMap, *keys):
"""Add an empty line to the document before a section (key)"""
for key in keys:
d.yaml_set_comment_before_after_key(key, before="\n")
def generate_orchestration_playbook(self, url=None, namespace=None, settings=None, repository_prefix=None,
pull_from_url=None, tag=None, vault_files=None, **kwargs):
"""
Generate an Ansible playbook to orchestrate services.
:param url: registry URL where images were pushed.
:param namespace: registry namespace
:param repository_prefix: prefix to use for the image name
:param settings: settings dict from container.yml
:param pull_from_url: if url to pull from is different than url
:return: playbook dict
"""
def _update_service(service_name, service_config):
if url and namespace:
# Reference previously pushed image
image_id = self.get_latest_image_id_for_service(service_name)
if not image_id:
raise exceptions.AnsibleContainerConductorException(
u"Unable to get image ID for service {}. Did you forget to run "
u"`ansible-container build`?".format(service_name)
)
image_tag = tag or self.get_build_stamp_for_image(image_id)
if repository_prefix:
image_name = "{}-{}".format(repository_prefix, service_name)
elif repository_prefix is None:
image_name = "{}-{}".format(self.project_name, service_name)
else:
image_name = service_name
repository = "{}/{}".format(namespace, image_name)
image_name = "{}:{}".format(repository, image_tag)
pull_url = pull_from_url if pull_from_url else url
service_config['image'] = "{}/{}".format(pull_url.rstrip('/'), image_name)
else:
# We're using a local image, so check that the image was built
image = self.get_latest_image_for_service(service_name)
if image is None:
raise exceptions.AnsibleContainerConductorException(
u"No image found for service {}, make sure you've run `ansible-container "
u"build`".format(service_name)
)
service_config['image'] = image.tags[0]
for service_name, service in iteritems(self.services):
# set the image property of each container
if service.get('containers'):
for container in service['containers']:
if container.get('roles'):
container_service_name = "{}-{}".format(service_name, container['container_name'])
_update_service(container_service_name, container)
else:
container['image'] = container['from']
elif service.get('roles'):
_update_service(service_name, service)
else:
service['image'] = service['from']
play = CommentedMap()
play['name'] = u'Manage the lifecycle of {} on {}'.format(self.project_name, self.display_name)
play['hosts'] = 'localhost'
play['gather_facts'] = 'no'
play['connection'] = 'local'
play['roles'] = CommentedSeq()
play['vars_files'] = CommentedSeq()
play['tasks'] = CommentedSeq()
role = CommentedMap([
('role', 'ansible.kubernetes-modules')
])
if vault_files:
play['vars_files'].extend(vault_files)
play['roles'].append(role)
play.yaml_set_comment_before_after_key(
'roles', before='Include Ansible Kubernetes and OpenShift modules', indent=4)
play.yaml_set_comment_before_after_key('tasks', before='Tasks for setting the application state. '
'Valid tags include: start, stop, restart, destroy', indent=4)
play['tasks'].append(self.deploy.get_namespace_task(state='present', tags=['start']))
play['tasks'].append(self.deploy.get_namespace_task(state='absent', tags=['destroy']))
play['tasks'].extend(self.deploy.get_secret_tasks(tags=['start']))
play['tasks'].extend(self.deploy.get_service_tasks(tags=['start']))
play['tasks'].extend(self.deploy.get_deployment_tasks(engine_state='stop', tags=['stop', 'restart']))
play['tasks'].extend(self.deploy.get_deployment_tasks(tags=['start', 'restart']))
play['tasks'].extend(self.deploy.get_pvc_tasks(tags=['start']))
playbook = CommentedSeq()
playbook.append(play)
logger.debug(u'Created playbook to run project', playbook=playbook)
return playbook
def _to_doc(d: DatasetDoc, with_formatting: bool):
if with_formatting:
doc = CommentedMap()
doc.yaml_set_comment_before_after_key("$schema", before="Dataset")
else:
doc = {}
doc["$schema"] = ODC_DATASET_SCHEMA_URL
doc.update(
attr.asdict(
d,
recurse=True,
dict_factory=CommentedMap if with_formatting else dict,
# Exclude fields that are the default.
filter=lambda attr, value: "doc_exclude" not in attr.metadata and
value != attr.default
# Exclude any fields set to None. The distinction should never matter in our docs.
and value is not None,
retain_collection_types=False,
))
# Sort properties for readability.
# PyCharm '19 misunderstands the type of a `sorted(dict.items())`
# noinspection PyTypeChecker
doc["properties"] = CommentedMap(
sorted(doc["properties"].items(), key=_stac_key_order))
if d.geometry is not None:
doc["geometry"] = shapely.geometry.mapping(d.geometry)
doc["id"] = str(d.id)
if with_formatting:
if "geometry" in doc:
# Set some numeric fields to be compact yaml format.
_use_compact_format(doc["geometry"], "coordinates")
if "grids" in doc:
for grid in doc["grids"].values():
_use_compact_format(grid, "shape", "transform")
# Add user-readable names for measurements as a comment if present.
if d.measurements:
for band_name, band_doc in d.measurements.items():
if band_doc.alias and band_name.lower(
) != band_doc.alias.lower():
doc["measurements"].yaml_add_eol_comment(
band_doc.alias, band_name)
_add_space_before(
doc,
"label" if "label" in doc else "id",
"crs",
"properties",
"measurements",
"accessories",
"lineage",
)
p: CommentedMap = doc["properties"]
p.yaml_add_eol_comment("# Ground sample distance (m)", "eo:gsd")
return doc
def add_space_between_main_sections(cwl: CommentedMap):
for k in cwl.keys():
if k in ["inputs", "outputs", "steps", "requirements", "hints", "baseCommand"]:
cwl.yaml_set_comment_before_after_key(key=k, before="\n")
def class_config_yaml(
cls,
outer_cfg, # noqa: C901 # too complex: TODO: FIX Yaml+contextvars
classes=None,
config: trc.Config = None):
"""Get the config section for this Configurable.
:param list classes:
(optional) The list of other classes in the config file,
used to reduce redundant help descriptions.
If given, only params from these classes reported.
:param config:
If given, only what is contained there is included in generated yaml,
with help-descriptions from classes, only where class default-values
differ from the values contained in this dictionary.
"""
from ..utils import yamlutil as yu
from ruamel.yaml.comments import CommentedMap
import textwrap as tw
cfg = CommentedMap()
for name, trait in sorted(cls.class_traits(config=True).items()):
if config is None:
default_value = trait.default()
cfg[name] = yu.preserve_yaml_literals(default_value)
else:
dumpables = _dumpable_trait_value(cls, trait, config)
if dumpables:
value, default_value = dumpables
cfg[name] = yu.preserve_yaml_literals(value)
else:
continue
trait_lines = []
default_repr = yu.ydumps(default_value)
if default_repr and default_repr.count(
'\n') > 1 and default_repr[0] != '\n':
default_repr = tw.indent('\n' + default_repr, ' ' * 9)
if yu._dump_trait_help.get():
if classes:
defining_class = cls._defining_class(trait, classes)
else:
defining_class = cls
if defining_class is cls:
# cls owns the trait, show full help
if trait.help:
trait_lines.append('')
trait_lines.append(_make_comment(trait.help).strip())
if 'Enum' in type(trait).__name__:
# include Enum choices
trait_lines.append('Choices: %s' % trait.info())
trait_lines.append('Default: %s' % default_repr)
else:
# Trait appears multiple times and isn't defined here.
# Truncate help to first line + "See also Original.trait"
if trait.help:
trait_lines.append(
_make_comment(trait.help.split('\n', 1)[0]))
trait_lines.append('See also: %s.%s' %
(defining_class.__name__, name))
cfg.yaml_set_comment_before_after_key(
name, before='\n'.join(trait_lines), indent=2)
if not cfg:
return
outer_cfg[cls.__name__] = cfg
if yu._dump_trait_help.get():
# section header
breaker = '#' * 76
parent_classes = ', '.join(p.__name__ for p in cls.__bases__
if issubclass(p, trc.Configurable))
s = "%s(%s) configuration" % (cls.__name__, parent_classes)
head_lines = ['', '', breaker, s, breaker]
# get the description trait
desc = class_help_description_lines(cls)
if desc:
head_lines.append(_make_comment('\n'.join(desc)))
outer_cfg.yaml_set_comment_before_after_key(cls.__name__,
'\n'.join(head_lines))
class YAMLRoundtripConfig(MutableConfigFile, MutableAbstractItemAccessMixin, MutableAbstractDictFunctionsMixin):
"""
Class for YAML-based (roundtrip) configurations
"""
def __init__(self, owner: Any, manager: "m.StorageManager", path: str, *args: List[Any], **kwargs: Dict[Any, Any]):
self.data = CommentedMap()
super().__init__(owner, manager, path, *args, **kwargs)
def load(self):
with open(self.path, "r") as fh:
self.data = yaml.round_trip_load(fh, version=(1, 2))
def reload(self):
self.unload()
self.load()
def unload(self):
self.data.clear()
def save(self):
if not self.mutable:
raise RuntimeError("You may not modify a defaults file at runtime - check the mutable attribute!")
with open(self.path, "w") as fh:
yaml.round_trip_dump(self.data, fh)
# region: CommentedMap functions
def insert(self, pos, key, value, *, comment=None):
"""
Insert a `key: value` pair at the given position, attaching a comment if provided
Wrapper for `CommentedMap.insert()`
"""
return self.data.insert(pos, key, value, comment)
def add_eol_comment(self, comment, *, key=NoComment, column=30):
"""
Add an end-of-line comment for a key at a particular column (30 by default)
Wrapper for `CommentedMap.yaml_add_eol_comment()`
"""
# Setting the column to None as the API actually defaults to will raise an exception, so we have to
# specify one unfortunately
return self.data.yaml_add_eol_comment(comment, key=key, column=column)
def set_comment_before_key(self, key, comment, *, indent=0):
"""
Set a comment before a given key
Wrapper for `CommentedMap.yaml_set_comment_before_after_key()`
"""
return self.data.yaml_set_comment_before_after_key(
key, before=comment, indent=indent, after=None, after_indent=None
)
def set_start_comment(self, comment, indent=0):
"""
Set the starting comment
Wrapper for `CommentedMap.yaml_set_start_comment()`
"""
return self.data.yaml_set_start_comment(comment, indent=indent)
# endregion
# region: Dict functions
def clear(self):
return self.data.clear()
def copy(self):
return self.data.copy()
def get(self, key, default=None):
return self.data.get(key, default)
def items(self):
return self.data.items()
def keys(self):
return self.data.keys()
def pop(self, key, default=None):
return self.data.pop(key, default)
def popitem(self):
return self.data.popitem()
def setdefault(self, key, default=None):
if key not in self.data:
self.data[key] = default
return default
return self.data[key]
def update(self, other):
return self.data.update(other)
def values(self):
return self.data.values()
# endregion
# Item access functions
def __contains__(self, key):
"""
Wrapper for `dict.__contains__()`
"""
return self.data.__contains__(key)
def __delitem__(self, key):
"""
Wrapper for `dict.__delitem__()`
"""
del self.data[key]
def __getitem__(self, key):
"""
Wrapper for `dict.__getitem__()`
"""
return self.data.__getitem__(key)
def __iter__(self):
"""
Wrapper for `dict.__iter__()`
"""
return self.data.__iter__()
def __len__(self):
"""
Wrapper for `dict.__len__()`
"""
return self.data.__len__()
def __setitem__(self, key, value):
"""
Wrapper for `dict.__getitem__()`
"""
return self.data.__setitem__(key, value)
