def _set_subcomponents(self, match_categories):
"""Set subcomponents for the top component from the match categories."""
data = CommentedMap(self.top_component.as_yaml())
data.yaml_set_start_comment(TOP_LEVEL_COMMENT)
temp_list = CommentedSeq()
# Set the subcomponents and comments
for top_comment, start_index, matches in match_categories:
components = self._matches_to_components(matches)
for subcomponent in components:
try: # Extract inline comment before it's removed
inline_comment = subcomponent.inline_comment
except AttributeError:
inline_comment = None
d2 = CommentedMap(subcomponent.as_yaml())
if inline_comment: # Apply inline comment to data
d2.yaml_add_eol_comment(comment=inline_comment, key='name')
temp_list.append(d2)
temp_list.yaml_set_comment_before_after_key(key=start_index,
before=top_comment,
indent=OFFSET)
data['subcomponents'] = temp_list
return data
def to_dict(self):
ret = CommentedMap()
ret.yaml_set_start_comment('\n' + PLIST_PREFACE_COMMENT + '\n\n')
ret['tagsonomy'] = self.tagsonomy
plist = []
seen_topics = set()
for p in self.project_list:
cur_pdict = p.to_dict()
plist.append(cur_pdict)
# now, determine whether to emit a comment
topic_tags = [
t for t in p._tags if t in self.tag_registry
and self.tag_registry[t].tag_type == 'topic'
]
if not topic_tags:
continue
first_topic = topic_tags[0]
if first_topic in seen_topics:
continue
seen_topics.add(first_topic)
cur_pdict.yaml_set_start_comment('\n' + first_topic.title() +
'\n\n')
ret['projects'] = plist
return ret
def export(self) -> CM:
"""Export configuration template into CommentedMap.
:raises SPSDKError: Error
:return: Configuration template in CM.
"""
loc_schemas = copy.deepcopy(self.schemas)
# 1. Do pre-merge by schema titles
pre_merged: Dict[str, Any] = {}
for schema in loc_schemas:
title = schema.get("title", "General Options")
if title in pre_merged:
deepmerge.always_merger.merge(pre_merged[title], schema)
else:
pre_merged[title] = schema
cfg = CM()
# 2. Add main title of configuration
cfg.yaml_set_start_comment(f"=========== {self.main_title} ===========\n")
# 3. Go through all individual logic blocks
for block in pre_merged.values():
try:
self._fill_up_block(cfg, block)
title = block.get("title", "General Options")
description = block.get("description", "")
cfg.yaml_set_comment_before_after_key(
list(block["properties"].keys())[0], f" \n == {title} == \n {description}"
)
except Exception as exc:
raise SPSDKError(f"Template generation failed: {str(exc)}") from exc
return cfg
def get_yaml_config(self, data: CM, indent: int = 0) -> CM:
"""Return YAML configuration In PfrConfiguration format.
:param data: The registers settings data.
:param indent: YAML start indent.
:return: YAML PFR configuration in commented map(ordered dict).
:raises SPSDKError: When there is no device found
:raises SPSDKError: When there is no type found
"""
if not self.device:
raise SPSDKError("Device not found")
if not self.type:
raise SPSDKError("Type not found")
res_data = CM()
res_data.yaml_set_start_comment(
f"NXP {self.device} PFR {self.type} configuration", indent=indent)
description = CM()
description.insert(1,
"device",
self.device,
comment="The NXP device name.")
description.insert(2,
"revision",
self.revision,
comment="The NXP device revision.")
description.insert(3,
"type",
self.type.upper(),
comment="The PFR type (CMPA, CFPA).")
description.insert(4,
"version",
spsdk_version,
comment="The SPSDK tool version.")
description.insert(5,
"author",
spsdk_author,
comment="The author of the configuration.")
description.insert(6,
"release",
spsdk_release,
comment="The SPSDK release.")
res_data.insert(
1,
"description",
description,
comment=f"The PFR {self.type} configuration description.",
)
res_data.insert(
2,
"settings",
data,
comment=f"The PFR {self.type} registers configuration.")
return res_data
def update_environment_yml():
"""Update conda_dev_env.yml file for conda."""
import re
from ruamel.yaml import YAML
from ruamel.yaml.comments import CommentedMap, CommentedSeq
environment_filename = "conda_dev_env.yml"
cmap = CommentedMap()
cmap.yaml_set_start_comment(
("Usage: conda env create -n myenvname -f {} python=3.6\n"
" conda activate myenvname\n"
" pip install --no-deps -e .".format(environment_filename)))
cmap["name"] = "aiida_crystal17"
cmap["channels"] = CommentedSeq(["conda-forge", "cjs14"])
cmap["channels"].yaml_add_eol_comment("for sqlalchemy-diff and pgtest", 1)
cmap["dependencies"] = dmap = CommentedSeq()
# additional packages
dmap.append("pip")
dmap.append("aiida-core.services")
# fix incompatibilities between conda and pypi
replacements = {"pre-commit": "pre_commit"}
setup_json = get_setup_json()
for base, key in [
(None, "install_requires"),
("extras_require", "testing"),
("extras_require", "code_style"),
("extras_require", "docs"),
]:
requirements = setup_json.get(base, setup_json)[key]
count = 0
for req in sorted(requirements, key=lambda x: x.lower()):
# skip packages required for specific python versions < 3
if re.findall("python_version\\s*\\<\\s*\\'?3", req):
continue
req = req.split(";")[0]
for (regex, replacement) in iter(replacements.items()):
req = re.sub(regex, replacement, req)
count += 1
dmap.append(req.lower())
dmap.yaml_set_comment_before_after_key(len(dmap) - count, before=key)
yaml = YAML(typ="rt")
yaml.default_flow_style = False
yaml.encoding = "utf-8"
yaml.allow_unicode = True
file_path = os.path.join(ROOT_DIR, environment_filename)
with open(file_path, "w") as env_file:
yaml.dump(cmap, env_file)
def test_before_top_map_from_scratch(self):
from ruamel.yaml.comments import CommentedMap
data = CommentedMap()
data['a'] = 1
data['b'] = 2
data.yaml_set_start_comment('Hello\nWorld\n')
# print(data.ca)
# print(data.ca._items)
compare(
data, """
{comment} Hello
{comment} World
a: 1
b: 2
""".format(comment='#'))
def update_environment_yml():
"""
Updates conda_dev_env.yml file for conda.
"""
import re
from ruamel.yaml.comments import CommentedMap, CommentedSeq
from ruamel.yaml import YAML
environment_filename = 'conda_dev_env.yml'
cmap = CommentedMap()
cmap.yaml_set_start_comment(
'Usage: conda env create -n myenvname -f {} python=3.6'.format(
environment_filename))
cmap['name'] = 'aiida_icl'
cmap['channels'] = CommentedSeq(['conda-forge', 'cjs'])
cmap['channels'].yaml_add_eol_comment('for sqlalchemy-diff and pgtest', 1)
cmap['dependencies'] = dmap = CommentedSeq()
# fix incompatibilities between conda and pypi
replacements = {}
setup_json = get_setup_json()
for base, key in [(None, 'install_requires'),
('extras_require', 'testing'),
('extras_require', 'code_style')]:
requirements = setup_json.get(base, setup_json)[key]
count = 0
for req in sorted(requirements, key=lambda x: x.lower()):
# skip packages required for specific python versions < 3
if re.findall("python_version\\s*\\<\\s*\\'?3", req):
continue
req = req.split(';')[0]
for (regex, replacement) in iter(replacements.items()):
req = re.sub(regex, replacement, req)
count += 1
dmap.append(req.lower())
dmap.yaml_set_comment_before_after_key(len(dmap) - count, before=key)
yaml = YAML(typ='rt')
yaml.default_flow_style = False
yaml.encoding = 'utf-8'
yaml.allow_unicode = True
file_path = os.path.join(ROOT_DIR, environment_filename)
with open(file_path, 'w') as env_file:
yaml.dump(cmap, env_file)
def create_yml_config(self, file_name: str, raw: bool = False) -> None:
"""The function creates the configuration YML file.
:param file_name: The file_name (without extension) of stored configuration.
:param raw: Raw output of configuration (including computed fields and anti-pole registers)
"""
antipole_regs = None if raw else list(
self.config.get_antipole_regs(self.device).values())
computed_fields = None if raw else self.config.get_computed_fields(
self.device)
yaml = YAML()
yaml.indent(sequence=SPSDK_YML_INDENT * 2, offset=SPSDK_YML_INDENT)
data = CM()
description = CM()
description.yaml_set_start_comment(
f"NXP {self.device.upper()} Shadow registers configuration",
indent=2)
description.insert(1,
"device",
self.device,
comment="The NXP device name.")
description.insert(2,
"version",
__version__,
comment="The SPSDK Shadow register tool version.")
description.insert(3,
"author",
__author__,
comment="The author of the configuration.")
description.insert(4,
"release",
__release__,
comment="The SPSDK release.")
data["description"] = description
data["registers"] = self.regs.create_yml_config(
exclude_regs=antipole_regs,
exclude_fields=computed_fields,
indent=2)
with open(file_name, "w", encoding="utf8") as out_file:
yaml.dump(data, out_file)
def export(self) -> CM:
"""Export configuration template into CommentedMap.
:raises SPSDKError: Error
:return: Configuration template in CM.
"""
loc_schemas = copy.deepcopy(self.schemas)
# 1. Get blocks with their titles and lists of their keys
block_list: Dict[str, Dict[str, List[str]]] = {}
for schema in loc_schemas:
title = schema.get("title", "General Options")
if title in block_list:
block_list[title]["properties"].extend(
self._get_schema_block_keys(schema))
else:
block_list[title] = {}
block_list[title]["properties"] = self._get_schema_block_keys(
schema)
block_list[title]["description"] = schema.get(
"description", "")
# 2. Merge all schemas together to get whole single schema
merged: Dict[str, Any] = {}
for schema in loc_schemas:
deepmerge.always_merger.merge(merged, schema)
cfg = CM()
# 3. Add main title of configuration
cfg.yaml_set_start_comment(
f"=========== {self.main_title} ===========\n")
# 4. Go through all individual logic blocks
for title, info in block_list.items():
try:
self._fill_up_block(cfg, merged, info["properties"])
description = info["description"]
cfg.yaml_set_comment_before_after_key(
info["properties"][0],
f" \n == {title} == \n {description}")
except Exception as exc:
raise SPSDKError(
f"Template generation failed: {str(exc)}") from exc
return cfg
def generate_config_file_yaml(self, classes=None, config: trc.Config = None):
"""
generate default config file from Configurables
:param config:
If given, only what is contained there is included in generated yaml,
with help-descriptions from classes, only where class default-values
differ from the values contained in this dictionary.
"""
from ruamel.yaml.comments import CommentedMap
import ipython_genutils.text as tw
from ..utils import yamlutil as yu
classes = self.classes if classes is None else classes
config_classes = set(self._classes_with_config_traits(classes))
## Order: subclass at the top.
# Filtering needed because `order_class_hierarchy()` brings full mro().
ordered_classes = [
cls for cls in order_class_hierarchy(classes) if cls in config_classes
]
class_hiearchy = generate_class_hierarchy_text(ordered_classes)
cfg = CommentedMap()
if yu._dump_trait_help.get():
start_comment = tw.dedent("""
############################################################################
Configuration hierarchy for `%s`:
%s
############################################################################
""") % (self.root_object().name, class_hiearchy)
cfg.yaml_set_start_comment(start_comment)
for cls in ordered_classes[::
-1]: # Inverted order from title's class-list, above.
cls.class_config_yaml(cfg, config=config)
return cfg
logging.basicConfig(
level=logging.DEBUG if args.debug else logging.WARNING, )
bear_dirs = [PROJECT_BEAR_DIR]
if args.bear_dirs is not None:
bear_dirs.extend(args.bear_dirs)
all_bears = get_all_bears(bear_dirs)
bear_requirements = get_all_requirements(all_bears)
instance_dict = merge_requirements(bear_requirements)
requirements = CommentedMap()
requirements.yaml_set_start_comment(
'This is an automatically generated file.\n'
'And should not be edited by hand.')
requirements['overrides'] = 'pm-requirements.yaml'
get_bear_requirement_metadata(bear_requirement_sets=instance_dict,
storage=requirements,
old_labels=True)
if args.update or args.check:
input_file_path = os.path.join(PROJECT_DIR, BEAR_REQUIREMENTS_YAML)
try:
input_file = open(input_file_path, 'r')
except FileNotFoundError:
print(YAML_MISSING)
exit(1)
class YAMLRoundtripConfig(MutableConfigFile, MutableAbstractItemAccessMixin, MutableAbstractDictFunctionsMixin):
"""
Class for YAML-based (roundtrip) configurations
"""
def __init__(self, owner: Any, manager: "m.StorageManager", path: str, *args: List[Any], **kwargs: Dict[Any, Any]):
self.data = CommentedMap()
super().__init__(owner, manager, path, *args, **kwargs)
def load(self):
with open(self.path, "r") as fh:
self.data = yaml.round_trip_load(fh, version=(1, 2))
def reload(self):
self.unload()
self.load()
def unload(self):
self.data.clear()
def save(self):
if not self.mutable:
raise RuntimeError("You may not modify a defaults file at runtime - check the mutable attribute!")
with open(self.path, "w") as fh:
yaml.round_trip_dump(self.data, fh)
# region: CommentedMap functions
def insert(self, pos, key, value, *, comment=None):
"""
Insert a `key: value` pair at the given position, attaching a comment if provided
Wrapper for `CommentedMap.insert()`
"""
return self.data.insert(pos, key, value, comment)
def add_eol_comment(self, comment, *, key=NoComment, column=30):
"""
Add an end-of-line comment for a key at a particular column (30 by default)
Wrapper for `CommentedMap.yaml_add_eol_comment()`
"""
# Setting the column to None as the API actually defaults to will raise an exception, so we have to
# specify one unfortunately
return self.data.yaml_add_eol_comment(comment, key=key, column=column)
def set_comment_before_key(self, key, comment, *, indent=0):
"""
Set a comment before a given key
Wrapper for `CommentedMap.yaml_set_comment_before_after_key()`
"""
return self.data.yaml_set_comment_before_after_key(
key, before=comment, indent=indent, after=None, after_indent=None
)
def set_start_comment(self, comment, indent=0):
"""
Set the starting comment
Wrapper for `CommentedMap.yaml_set_start_comment()`
"""
return self.data.yaml_set_start_comment(comment, indent=indent)
# endregion
# region: Dict functions
def clear(self):
return self.data.clear()
def copy(self):
return self.data.copy()
def get(self, key, default=None):
return self.data.get(key, default)
def items(self):
return self.data.items()
def keys(self):
return self.data.keys()
def pop(self, key, default=None):
return self.data.pop(key, default)
def popitem(self):
return self.data.popitem()
def setdefault(self, key, default=None):
if key not in self.data:
self.data[key] = default
return default
return self.data[key]
def update(self, other):
return self.data.update(other)
def values(self):
return self.data.values()
# endregion
# Item access functions
def __contains__(self, key):
"""
Wrapper for `dict.__contains__()`
"""
return self.data.__contains__(key)
def __delitem__(self, key):
"""
Wrapper for `dict.__delitem__()`
"""
del self.data[key]
def __getitem__(self, key):
"""
Wrapper for `dict.__getitem__()`
"""
return self.data.__getitem__(key)
def __iter__(self):
"""
Wrapper for `dict.__iter__()`
"""
return self.data.__iter__()
def __len__(self):
"""
Wrapper for `dict.__len__()`
"""
return self.data.__len__()
def __setitem__(self, key, value):
"""
Wrapper for `dict.__getitem__()`
"""
return self.data.__setitem__(key, value)
def emit_defaults(schema,
include_yaml_comments=False,
yaml_indent=2,
base_cls=None,
*args,
**kwargs):
"""
Emit all default values for the given schema.
Similar to calling ``validate({}, schema, inject_defaults=True)``, except:
1. Ignore schema validation errors and 'required' property errors
2. If no default is given for a property, inject ``"{{NO_DEFAULT}}"``,
even if the property isn't supposed to be a string.
3. If ``include_yaml_comments`` is True, insert ``CommentedMap`` objects instead of ordinary dicts,
and insert a comment above each key, with the contents of the property ``"description"`` in the schema.
Args:
schema:
The schema data to pull defaults from
include_yaml_comments:
Whether or not to return ``ruamel.yaml`` objects so that
comments will be written when the data is dumped to YAML.
yaml_indent:
To ensure correctly indented comments, you must specify the indent
step you plan to use when this data is eventually dumped as yaml.
Returns:
A copy of instance, with default values injected, and comments if specified.
"""
instance = {}
if include_yaml_comments:
instance = CommentedMap(instance)
instance.key_indent = 0 # monkey-patch!
if "description" in schema:
instance.yaml_set_start_comment('\n' + schema["description"] +
'\n\n')
else:
instance = dict(instance)
if base_cls is None:
base_cls = validators.validator_for(schema)
base_cls.check_schema(schema)
def is_object(checker, instance):
return (base_cls.TYPE_CHECKER.is_type(instance, "object")
or isinstance(instance, (ordereddict, CommentedMap)))
def is_array(checker, instance):
return (base_cls.TYPE_CHECKER.is_type(instance, "array")
or isinstance(instance, CommentedSeq))
# By default, jsonschema expects JSON objects to be of type 'dict'.
# We also want to permit ruamel.yaml.comments.CommentedSeq and CommentedMap
type_checker = base_cls.TYPE_CHECKER.redefine_many({
"object": is_object,
"array": is_array
})
cls = validators.extend(base_cls, type_checker=type_checker)
# Add default-injection behavior to the validator
cls = extend_with_default_without_validation(cls, include_yaml_comments,
yaml_indent)
extended_validator = cls(schema, *args, **kwargs)
# Inject defaults.
extended_validator.validate(instance)
return instance
if __name__ == '__main__':
args = get_args()
bear_dirs = [PROJECT_BEAR_DIR]
if args.bear_dirs is not None:
bear_dirs.extend(args.bear_dirs)
all_bears = get_all_bears(bear_dirs)
instance_dict = get_all_requirements(all_bears)
requirements = CommentedMap()
requirements.yaml_set_start_comment(
'This is an automatically generated file.\n'
'And should not be edited by hand.')
requirements['overrides'] = 'coala-build.yaml'
requirements['gem_requirements'] = get_gem_requirements(
instance_dict['GemRequirement'])
requirements['r_script_requirements'] = get_r_requirements(
instance_dict['RscriptRequirement'])
requirements['npm_requirements'] = get_npm_requirements(
instance_dict['NpmRequirement'])
requirements['pip_requirements'] = get_pip_requirements(
instance_dict['PipRequirement'])
requirements['composer_requirements'] = get_composer_requirements(
instance_dict['ComposerRequirement'])
requirements['cabal_requirements'] = get_cabal_requirements(
instance_dict['CabalRequirement'])
