def _set_subcomponents(self, match_categories): """Set subcomponents for the top component from the match categories.""" data = CommentedMap(self.top_component.as_yaml()) data.yaml_set_start_comment(TOP_LEVEL_COMMENT) temp_list = CommentedSeq() # Set the subcomponents and comments for top_comment, start_index, matches in match_categories: components = self._matches_to_components(matches) for subcomponent in components: try: # Extract inline comment before it's removed inline_comment = subcomponent.inline_comment except AttributeError: inline_comment = None d2 = CommentedMap(subcomponent.as_yaml()) if inline_comment: # Apply inline comment to data d2.yaml_add_eol_comment(comment=inline_comment, key='name') temp_list.append(d2) temp_list.yaml_set_comment_before_after_key(key=start_index, before=top_comment, indent=OFFSET) data['subcomponents'] = temp_list return data
def to_dict(self): ret = CommentedMap() ret.yaml_set_start_comment('\n' + PLIST_PREFACE_COMMENT + '\n\n') ret['tagsonomy'] = self.tagsonomy plist = [] seen_topics = set() for p in self.project_list: cur_pdict = p.to_dict() plist.append(cur_pdict) # now, determine whether to emit a comment topic_tags = [ t for t in p._tags if t in self.tag_registry and self.tag_registry[t].tag_type == 'topic' ] if not topic_tags: continue first_topic = topic_tags[0] if first_topic in seen_topics: continue seen_topics.add(first_topic) cur_pdict.yaml_set_start_comment('\n' + first_topic.title() + '\n\n') ret['projects'] = plist return ret
def export(self) -> CM: """Export configuration template into CommentedMap. :raises SPSDKError: Error :return: Configuration template in CM. """ loc_schemas = copy.deepcopy(self.schemas) # 1. Do pre-merge by schema titles pre_merged: Dict[str, Any] = {} for schema in loc_schemas: title = schema.get("title", "General Options") if title in pre_merged: deepmerge.always_merger.merge(pre_merged[title], schema) else: pre_merged[title] = schema cfg = CM() # 2. Add main title of configuration cfg.yaml_set_start_comment(f"=========== {self.main_title} ===========\n") # 3. Go through all individual logic blocks for block in pre_merged.values(): try: self._fill_up_block(cfg, block) title = block.get("title", "General Options") description = block.get("description", "") cfg.yaml_set_comment_before_after_key( list(block["properties"].keys())[0], f" \n == {title} == \n {description}" ) except Exception as exc: raise SPSDKError(f"Template generation failed: {str(exc)}") from exc return cfg
def get_yaml_config(self, data: CM, indent: int = 0) -> CM: """Return YAML configuration In PfrConfiguration format. :param data: The registers settings data. :param indent: YAML start indent. :return: YAML PFR configuration in commented map(ordered dict). :raises SPSDKError: When there is no device found :raises SPSDKError: When there is no type found """ if not self.device: raise SPSDKError("Device not found") if not self.type: raise SPSDKError("Type not found") res_data = CM() res_data.yaml_set_start_comment( f"NXP {self.device} PFR {self.type} configuration", indent=indent) description = CM() description.insert(1, "device", self.device, comment="The NXP device name.") description.insert(2, "revision", self.revision, comment="The NXP device revision.") description.insert(3, "type", self.type.upper(), comment="The PFR type (CMPA, CFPA).") description.insert(4, "version", spsdk_version, comment="The SPSDK tool version.") description.insert(5, "author", spsdk_author, comment="The author of the configuration.") description.insert(6, "release", spsdk_release, comment="The SPSDK release.") res_data.insert( 1, "description", description, comment=f"The PFR {self.type} configuration description.", ) res_data.insert( 2, "settings", data, comment=f"The PFR {self.type} registers configuration.") return res_data
def update_environment_yml(): """Update conda_dev_env.yml file for conda.""" import re from ruamel.yaml import YAML from ruamel.yaml.comments import CommentedMap, CommentedSeq environment_filename = "conda_dev_env.yml" cmap = CommentedMap() cmap.yaml_set_start_comment( ("Usage: conda env create -n myenvname -f {} python=3.6\n" " conda activate myenvname\n" " pip install --no-deps -e .".format(environment_filename))) cmap["name"] = "aiida_crystal17" cmap["channels"] = CommentedSeq(["conda-forge", "cjs14"]) cmap["channels"].yaml_add_eol_comment("for sqlalchemy-diff and pgtest", 1) cmap["dependencies"] = dmap = CommentedSeq() # additional packages dmap.append("pip") dmap.append("aiida-core.services") # fix incompatibilities between conda and pypi replacements = {"pre-commit": "pre_commit"} setup_json = get_setup_json() for base, key in [ (None, "install_requires"), ("extras_require", "testing"), ("extras_require", "code_style"), ("extras_require", "docs"), ]: requirements = setup_json.get(base, setup_json)[key] count = 0 for req in sorted(requirements, key=lambda x: x.lower()): # skip packages required for specific python versions < 3 if re.findall("python_version\\s*\\<\\s*\\'?3", req): continue req = req.split(";")[0] for (regex, replacement) in iter(replacements.items()): req = re.sub(regex, replacement, req) count += 1 dmap.append(req.lower()) dmap.yaml_set_comment_before_after_key(len(dmap) - count, before=key) yaml = YAML(typ="rt") yaml.default_flow_style = False yaml.encoding = "utf-8" yaml.allow_unicode = True file_path = os.path.join(ROOT_DIR, environment_filename) with open(file_path, "w") as env_file: yaml.dump(cmap, env_file)
def test_before_top_map_from_scratch(self): from ruamel.yaml.comments import CommentedMap data = CommentedMap() data['a'] = 1 data['b'] = 2 data.yaml_set_start_comment('Hello\nWorld\n') # print(data.ca) # print(data.ca._items) compare( data, """ {comment} Hello {comment} World a: 1 b: 2 """.format(comment='#'))
def update_environment_yml(): """ Updates conda_dev_env.yml file for conda. """ import re from ruamel.yaml.comments import CommentedMap, CommentedSeq from ruamel.yaml import YAML environment_filename = 'conda_dev_env.yml' cmap = CommentedMap() cmap.yaml_set_start_comment( 'Usage: conda env create -n myenvname -f {} python=3.6'.format( environment_filename)) cmap['name'] = 'aiida_icl' cmap['channels'] = CommentedSeq(['conda-forge', 'cjs']) cmap['channels'].yaml_add_eol_comment('for sqlalchemy-diff and pgtest', 1) cmap['dependencies'] = dmap = CommentedSeq() # fix incompatibilities between conda and pypi replacements = {} setup_json = get_setup_json() for base, key in [(None, 'install_requires'), ('extras_require', 'testing'), ('extras_require', 'code_style')]: requirements = setup_json.get(base, setup_json)[key] count = 0 for req in sorted(requirements, key=lambda x: x.lower()): # skip packages required for specific python versions < 3 if re.findall("python_version\\s*\\<\\s*\\'?3", req): continue req = req.split(';')[0] for (regex, replacement) in iter(replacements.items()): req = re.sub(regex, replacement, req) count += 1 dmap.append(req.lower()) dmap.yaml_set_comment_before_after_key(len(dmap) - count, before=key) yaml = YAML(typ='rt') yaml.default_flow_style = False yaml.encoding = 'utf-8' yaml.allow_unicode = True file_path = os.path.join(ROOT_DIR, environment_filename) with open(file_path, 'w') as env_file: yaml.dump(cmap, env_file)
def create_yml_config(self, file_name: str, raw: bool = False) -> None: """The function creates the configuration YML file. :param file_name: The file_name (without extension) of stored configuration. :param raw: Raw output of configuration (including computed fields and anti-pole registers) """ antipole_regs = None if raw else list( self.config.get_antipole_regs(self.device).values()) computed_fields = None if raw else self.config.get_computed_fields( self.device) yaml = YAML() yaml.indent(sequence=SPSDK_YML_INDENT * 2, offset=SPSDK_YML_INDENT) data = CM() description = CM() description.yaml_set_start_comment( f"NXP {self.device.upper()} Shadow registers configuration", indent=2) description.insert(1, "device", self.device, comment="The NXP device name.") description.insert(2, "version", __version__, comment="The SPSDK Shadow register tool version.") description.insert(3, "author", __author__, comment="The author of the configuration.") description.insert(4, "release", __release__, comment="The SPSDK release.") data["description"] = description data["registers"] = self.regs.create_yml_config( exclude_regs=antipole_regs, exclude_fields=computed_fields, indent=2) with open(file_name, "w", encoding="utf8") as out_file: yaml.dump(data, out_file)
def export(self) -> CM: """Export configuration template into CommentedMap. :raises SPSDKError: Error :return: Configuration template in CM. """ loc_schemas = copy.deepcopy(self.schemas) # 1. Get blocks with their titles and lists of their keys block_list: Dict[str, Dict[str, List[str]]] = {} for schema in loc_schemas: title = schema.get("title", "General Options") if title in block_list: block_list[title]["properties"].extend( self._get_schema_block_keys(schema)) else: block_list[title] = {} block_list[title]["properties"] = self._get_schema_block_keys( schema) block_list[title]["description"] = schema.get( "description", "") # 2. Merge all schemas together to get whole single schema merged: Dict[str, Any] = {} for schema in loc_schemas: deepmerge.always_merger.merge(merged, schema) cfg = CM() # 3. Add main title of configuration cfg.yaml_set_start_comment( f"=========== {self.main_title} ===========\n") # 4. Go through all individual logic blocks for title, info in block_list.items(): try: self._fill_up_block(cfg, merged, info["properties"]) description = info["description"] cfg.yaml_set_comment_before_after_key( info["properties"][0], f" \n == {title} == \n {description}") except Exception as exc: raise SPSDKError( f"Template generation failed: {str(exc)}") from exc return cfg
def generate_config_file_yaml(self, classes=None, config: trc.Config = None): """ generate default config file from Configurables :param config: If given, only what is contained there is included in generated yaml, with help-descriptions from classes, only where class default-values differ from the values contained in this dictionary. """ from ruamel.yaml.comments import CommentedMap import ipython_genutils.text as tw from ..utils import yamlutil as yu classes = self.classes if classes is None else classes config_classes = set(self._classes_with_config_traits(classes)) ## Order: subclass at the top. # Filtering needed because `order_class_hierarchy()` brings full mro(). ordered_classes = [ cls for cls in order_class_hierarchy(classes) if cls in config_classes ] class_hiearchy = generate_class_hierarchy_text(ordered_classes) cfg = CommentedMap() if yu._dump_trait_help.get(): start_comment = tw.dedent(""" ############################################################################ Configuration hierarchy for `%s`: %s ############################################################################ """) % (self.root_object().name, class_hiearchy) cfg.yaml_set_start_comment(start_comment) for cls in ordered_classes[:: -1]: # Inverted order from title's class-list, above. cls.class_config_yaml(cfg, config=config) return cfg
logging.basicConfig( level=logging.DEBUG if args.debug else logging.WARNING, ) bear_dirs = [PROJECT_BEAR_DIR] if args.bear_dirs is not None: bear_dirs.extend(args.bear_dirs) all_bears = get_all_bears(bear_dirs) bear_requirements = get_all_requirements(all_bears) instance_dict = merge_requirements(bear_requirements) requirements = CommentedMap() requirements.yaml_set_start_comment( 'This is an automatically generated file.\n' 'And should not be edited by hand.') requirements['overrides'] = 'pm-requirements.yaml' get_bear_requirement_metadata(bear_requirement_sets=instance_dict, storage=requirements, old_labels=True) if args.update or args.check: input_file_path = os.path.join(PROJECT_DIR, BEAR_REQUIREMENTS_YAML) try: input_file = open(input_file_path, 'r') except FileNotFoundError: print(YAML_MISSING) exit(1)
class YAMLRoundtripConfig(MutableConfigFile, MutableAbstractItemAccessMixin, MutableAbstractDictFunctionsMixin): """ Class for YAML-based (roundtrip) configurations """ def __init__(self, owner: Any, manager: "m.StorageManager", path: str, *args: List[Any], **kwargs: Dict[Any, Any]): self.data = CommentedMap() super().__init__(owner, manager, path, *args, **kwargs) def load(self): with open(self.path, "r") as fh: self.data = yaml.round_trip_load(fh, version=(1, 2)) def reload(self): self.unload() self.load() def unload(self): self.data.clear() def save(self): if not self.mutable: raise RuntimeError("You may not modify a defaults file at runtime - check the mutable attribute!") with open(self.path, "w") as fh: yaml.round_trip_dump(self.data, fh) # region: CommentedMap functions def insert(self, pos, key, value, *, comment=None): """ Insert a `key: value` pair at the given position, attaching a comment if provided Wrapper for `CommentedMap.insert()` """ return self.data.insert(pos, key, value, comment) def add_eol_comment(self, comment, *, key=NoComment, column=30): """ Add an end-of-line comment for a key at a particular column (30 by default) Wrapper for `CommentedMap.yaml_add_eol_comment()` """ # Setting the column to None as the API actually defaults to will raise an exception, so we have to # specify one unfortunately return self.data.yaml_add_eol_comment(comment, key=key, column=column) def set_comment_before_key(self, key, comment, *, indent=0): """ Set a comment before a given key Wrapper for `CommentedMap.yaml_set_comment_before_after_key()` """ return self.data.yaml_set_comment_before_after_key( key, before=comment, indent=indent, after=None, after_indent=None ) def set_start_comment(self, comment, indent=0): """ Set the starting comment Wrapper for `CommentedMap.yaml_set_start_comment()` """ return self.data.yaml_set_start_comment(comment, indent=indent) # endregion # region: Dict functions def clear(self): return self.data.clear() def copy(self): return self.data.copy() def get(self, key, default=None): return self.data.get(key, default) def items(self): return self.data.items() def keys(self): return self.data.keys() def pop(self, key, default=None): return self.data.pop(key, default) def popitem(self): return self.data.popitem() def setdefault(self, key, default=None): if key not in self.data: self.data[key] = default return default return self.data[key] def update(self, other): return self.data.update(other) def values(self): return self.data.values() # endregion # Item access functions def __contains__(self, key): """ Wrapper for `dict.__contains__()` """ return self.data.__contains__(key) def __delitem__(self, key): """ Wrapper for `dict.__delitem__()` """ del self.data[key] def __getitem__(self, key): """ Wrapper for `dict.__getitem__()` """ return self.data.__getitem__(key) def __iter__(self): """ Wrapper for `dict.__iter__()` """ return self.data.__iter__() def __len__(self): """ Wrapper for `dict.__len__()` """ return self.data.__len__() def __setitem__(self, key, value): """ Wrapper for `dict.__getitem__()` """ return self.data.__setitem__(key, value)
def emit_defaults(schema, include_yaml_comments=False, yaml_indent=2, base_cls=None, *args, **kwargs): """ Emit all default values for the given schema. Similar to calling ``validate({}, schema, inject_defaults=True)``, except: 1. Ignore schema validation errors and 'required' property errors 2. If no default is given for a property, inject ``"{{NO_DEFAULT}}"``, even if the property isn't supposed to be a string. 3. If ``include_yaml_comments`` is True, insert ``CommentedMap`` objects instead of ordinary dicts, and insert a comment above each key, with the contents of the property ``"description"`` in the schema. Args: schema: The schema data to pull defaults from include_yaml_comments: Whether or not to return ``ruamel.yaml`` objects so that comments will be written when the data is dumped to YAML. yaml_indent: To ensure correctly indented comments, you must specify the indent step you plan to use when this data is eventually dumped as yaml. Returns: A copy of instance, with default values injected, and comments if specified. """ instance = {} if include_yaml_comments: instance = CommentedMap(instance) instance.key_indent = 0 # monkey-patch! if "description" in schema: instance.yaml_set_start_comment('\n' + schema["description"] + '\n\n') else: instance = dict(instance) if base_cls is None: base_cls = validators.validator_for(schema) base_cls.check_schema(schema) def is_object(checker, instance): return (base_cls.TYPE_CHECKER.is_type(instance, "object") or isinstance(instance, (ordereddict, CommentedMap))) def is_array(checker, instance): return (base_cls.TYPE_CHECKER.is_type(instance, "array") or isinstance(instance, CommentedSeq)) # By default, jsonschema expects JSON objects to be of type 'dict'. # We also want to permit ruamel.yaml.comments.CommentedSeq and CommentedMap type_checker = base_cls.TYPE_CHECKER.redefine_many({ "object": is_object, "array": is_array }) cls = validators.extend(base_cls, type_checker=type_checker) # Add default-injection behavior to the validator cls = extend_with_default_without_validation(cls, include_yaml_comments, yaml_indent) extended_validator = cls(schema, *args, **kwargs) # Inject defaults. extended_validator.validate(instance) return instance
if __name__ == '__main__': args = get_args() bear_dirs = [PROJECT_BEAR_DIR] if args.bear_dirs is not None: bear_dirs.extend(args.bear_dirs) all_bears = get_all_bears(bear_dirs) instance_dict = get_all_requirements(all_bears) requirements = CommentedMap() requirements.yaml_set_start_comment( 'This is an automatically generated file.\n' 'And should not be edited by hand.') requirements['overrides'] = 'coala-build.yaml' requirements['gem_requirements'] = get_gem_requirements( instance_dict['GemRequirement']) requirements['r_script_requirements'] = get_r_requirements( instance_dict['RscriptRequirement']) requirements['npm_requirements'] = get_npm_requirements( instance_dict['NpmRequirement']) requirements['pip_requirements'] = get_pip_requirements( instance_dict['PipRequirement']) requirements['composer_requirements'] = get_composer_requirements( instance_dict['ComposerRequirement']) requirements['cabal_requirements'] = get_cabal_requirements( instance_dict['CabalRequirement'])