def _ValidateEntityContent(self, entity: syaml.YAML) -> None:
"""Validates the contents of a single entity.
The logic will select the appropriate validation schema based on the
config_mode of the container and the defined EntityOperation, if any.
Args:
entity: YAML object for the entityContents
Raises:
KeyError: if self._config_mode is not set to a known value.
"""
if ConfigMode.INITIALIZE == self._config_mode:
schema = syaml.Map(_ENTITY_INIT_SCHEMA)
elif ConfigMode.UPDATE == self._config_mode:
schema = syaml.Map(_ENTITY_UPDATE_SCHEMA)
if ENTITY_OPERATION_KEY in entity:
if entity[ENTITY_OPERATION_KEY] == EntityOperation.ADD.value:
schema = syaml.Map(_ENTITY_ADD_SCHEMA)
elif entity[ENTITY_OPERATION_KEY] == EntityOperation.DELETE.value:
schema = syaml.Map(_ENTITY_DELETE_SCHEMA)
else:
raise KeyError('No valid _config_mode is set')
entity.revalidate(schema)
if TRANSLATION_KEY in entity.data.keys():
if ENTITY_CLOUD_DEVICE_ID_KEY not in entity.data.keys():
raise KeyError('cloud_device_id required when translation is present.')
def _ValidateEntityContent(self, entity: syaml.YAML) -> None:
"""Validates the contents of a single entity.
The logic will select the appropriate validation schema based on the
config_mode of the container and the defined EntityOperation, if any.
Args:
entity: YAML object for the entityContents
Raises:
KeyError: if self._config_mode is not set to a known value.
"""
if ConfigMode.INITIALIZE == self._config_mode:
schema = syaml.Map(_ENTITY_INIT_SCHEMA)
elif ConfigMode.UPDATE == self._config_mode:
schema = syaml.Map(_ENTITY_UPDATE_SCHEMA)
if _ENTITY_MODE_KEY in entity:
if entity[_ENTITY_MODE_KEY] == EntityOperation.ADD.value:
schema = syaml.Map(_ENTITY_ADD_SCHEMA)
elif entity[_ENTITY_MODE_KEY] == EntityOperation.DELETE.value:
schema = syaml.Map(_ENTITY_DELETE_SCHEMA)
else:
raise KeyError('No valid _config_mode is set')
entity.revalidate(schema)
def load_config_from_file(config_file: Path): # pragma: no cover
schema = strictyaml.Map({
'email_address': strictyaml.Email(),
'use_keyring': strictyaml.Bool(),
})
return strictyaml.load(config_file.read_text(), schema).data
def schema(cls):
return syml.Map({
**cls._schema_base(),
"location":
syml.FixedSeq([syml.Float()] * 3),
syml.Optional("frames", 1):
syml.Int(),
})
def _get_match_schema(self):
return strictyaml.Map(
{
"duration": strictyaml.Int(),
"matches": strictyaml.Seq(
strictyaml.Map(
{
"away": strictyaml.Str(),
"date": strictyaml.Str(),
"newdate": strictyaml.Str(),
"our_score": strictyaml.Int(),
"opp_score": strictyaml.Int(),
}
)
),
}
)
def load_config():
schema = strictyaml.Map({
'email_address': strictyaml.Email(),
})
root_dir = path.abspath(path.dirname(__file__))
with open(path.join(root_dir, 'config.yaml')) as config_file:
return strictyaml.load(config_file.read(), schema).data
def _yaml_schema(cls) -> Validator:
return strictyaml.Map({
"project-id":
strictyaml.Str(),
"project-slug":
strictyaml.Str(),
strictyaml.Optional("changelog-file"):
strictyaml.Str(),
})
def schema(cls):
return syml.Map({
**cls._schema_base(),
syml.Optional("interp_order", None):
syml.Enum([str(x) for x in range(6)]),
syml.Optional("extrap_mode", None):
syml.Enum(EXTRAP_MODES),
syml.Optional("extrap_cval", None):
syml.Float() | syml.Int(),
})
def _ProtoDescriptorToSchema(pb_descriptor):
"""Convert a proto Descriptor to a strictyaml schema."""
d = {}
for field in pb_descriptor.fields:
if field.LABEL_REPEATED == field.label:
value_schema = syaml.Seq(_ProtoFieldToSchema(field))
else:
value_schema = _ProtoFieldToSchema(field)
d[syaml.Optional(field.name)] = value_schema
return syaml.Map(d)
def _yaml_schema(cls) -> strictyaml.Validator:
return strictyaml.Map({
"tags":
strictyaml.MapPattern(
strictyaml.Str(),
strictyaml.Str(),
),
"files":
strictyaml.Seq(strictyaml.Str(), ),
})
def schema(cls):
return syml.Map({
**cls._schema_base(),
syml.Optional("roll", 0):
syml.Float(),
syml.Optional("pitch", 0):
syml.Float(),
syml.Optional("yaw", 0):
syml.Float(),
syml.Optional("degrees", True):
syml.Bool(),
syml.Optional("frames", 1):
syml.Int(),
})
def load_config():
schema = strictyaml.Map({
'email_address':
strictyaml.Email(),
'topic_arn':
strictyaml.Str(),
'meals':
strictyaml.Seq(
strictyaml.MapPattern(strictyaml.Str(), strictyaml.Str()))
})
root_dir = path.abspath(path.dirname(__file__))
with open(path.join(root_dir, 'config.yaml')) as config_file:
return strictyaml.load(config_file.read(), schema).data
def schema(cls):
return syml.Map({
**cls._schema_base(),
syml.Optional("forward", 0):
syml.Float(),
syml.Optional("down", 0):
syml.Float(),
syml.Optional("right", 0):
syml.Float(),
syml.Optional("right", 0):
syml.Float(),
syml.Optional("frames", 1):
syml.Int(),
})
def _validate_inputs(self):
"""Validate the input files. Each input type (i.e. image, selavy, noise,
background) may be given as one of the following:
1. A list of files.
2. A glob expression.
3. A list of glob expressions.
4. A mapping of epochs to any of the above.
Each input type is validated individually. Extra input validation steps, e.g. to
ensure each input type has the same number of files, are performed in
`validate()`.
Raises:
PipelineConfigError: The run config inputs fail schema validation.
"""
try:
# first pass validation
self._yaml["inputs"].revalidate(yaml.Map(self._SCHEMA_INPUTS))
for input_type in self._yaml["inputs"]:
input_yaml = self._yaml["inputs"][input_type]
if input_yaml.is_mapping():
# inputs are either epoch-mode, glob expressions, or both
if "glob" in input_yaml:
# validate globs
input_yaml.revalidate(
yaml.Map(self._SCHEMA_GLOB_INPUTS))
else:
# validate epoch mode which may also contain glob expressions
input_yaml.revalidate(
yaml.MapPattern(
yaml.Str(),
yaml.UniqueSeq(yaml.Str())
| yaml.Map(self._SCHEMA_GLOB_INPUTS),
))
except yaml.YAMLValidationError as e:
raise PipelineConfigError(e)
def _yaml_schema(cls) -> strictyaml.Validator:
return strictyaml.Map({
"name":
strictyaml.Str(),
"wow-versions":
strictyaml.Seq(strictyaml.Str(), ),
strictyaml.Optional("curseforge"):
CurseforgeConfig._yaml_schema(),
"addons":
strictyaml.Seq(AddonConfig._yaml_schema()),
strictyaml.Optional("tags"):
strictyaml.MapPattern(
strictyaml.Str(),
strictyaml.Str(),
),
})
def _load_yaml_file(cls, yaml_content):
import strictyaml
schema = strictyaml.Map({
"plugin_name": strictyaml.Str(),
"plugin_version": strictyaml.Str(),
"author": strictyaml.Str(),
"email": strictyaml.Str(),
"shared_lib": strictyaml.Str(),
})
plugin_config_file_content = strictyaml.load(yaml_content, schema).data
if sys.platform == 'win32':
plugin_config_file_content[
'shared_lib_name'] = f"{plugin_config_file_content['shared_lib']}.dll"
else:
plugin_config_file_content[
'shared_lib_name'] = f"lib{plugin_config_file_content['shared_lib']}.so"
return plugin_config_file_content
def _load_yaml_file(cls, yaml_content):
import strictyaml
schema = strictyaml.Map({
"caption": strictyaml.Str(),
"version": strictyaml.Str(),
"author": strictyaml.Str(),
"email": strictyaml.Str(),
"id": strictyaml.Str(),
})
plugin_config_file_content = strictyaml.load(yaml_content, schema).data
if sys.platform == "win32":
plugin_config_file_content[
"shared_lib_name"] = f"{plugin_config_file_content['id']}.dll"
else:
plugin_config_file_content[
"shared_lib_name"] = f"lib{plugin_config_file_content['id']}.so"
return plugin_config_file_content
"""Schema separately parses translation to account for multiple valid formats
github.com/google/digitalbuildings/blob/master/ontology/docs/building_config.md
#defining-translations
"""
_TRANSLATION_SCHEMA = syaml.MapPattern(
syaml.Regex(_FIELD_REGEX),
# Note: This block is somewhat permissive as the logic was difficult to
# implement in syaml. Additional validation occurs in EntityInstance
syaml.Str() | syaml.Map({
PRESENT_VALUE_KEY:
syaml.Str(),
syaml.Optional(STATES_KEY):
syaml.MapPattern(
syaml.Regex(u'^[A-Z][A-Z_]+'),
syaml.Str() | syaml.Seq(syaml.Str())),
syaml.Optional(UNITS_KEY):
syaml.Map({
UNIT_NAME_KEY: syaml.Str(),
UNIT_VALUES_KEY: syaml.MapPattern(syaml.Str(), syaml.Str())
}),
}))
_METADATA_SCHEMA = syaml.Map({
syaml.Optional(_CONFIG_MODE_KEY):
EnumToRegex(ConfigMode, [ConfigMode.EXPORT])
})
_ENTITY_IDS_SCHEMA = {
# this is the Phred primary key, this should not be surfaced/checked
# TODO(b/202278941): remove id reference
"""Schema separately parses translation to account for multiple valid formats
github.com/google/digitalbuildings/blob/master/ontology/docs/building_config.md#defining-translations"""
_TRANSLATION_SCHEMA = syaml.Str() | syaml.Any()
# TODO check valid ontological content in next validation steps
"""strictyaml schema parses a YAML instance from its first level of keys
github.com/google/digitalbuildings/blob/master/ontology/docs/building_config.md#config-format"""
_SCHEMA = syaml.MapPattern(syaml.Str(),
syaml.Map({
'type': syaml.Str(),
'id': syaml.Str(),
syaml.Optional('connections'):
syaml.MapPattern(syaml.Str(),
syaml.Str()) |
syaml.Seq(
syaml.MapPattern(syaml.Str(),
syaml.Str())),
syaml.Optional('links'): syaml.MapPattern(
syaml.Str(),
syaml.MapPattern(syaml.Str(), syaml.Str())),
syaml.Optional('translation'): syaml.Any(),
syaml.Optional('metadata'): syaml.Any()
}))
# TODO add manual check to de-duplicate units/unit_values/states
# TODO add all units/unit_values/states to translation_data_schema
"""Further account for multiple valid translation formats
github.com/google/digitalbuildings/blob/master/ontology/docs/building_config.md#defining-translations"""
_TRANSLATION_DATA_SCHEMA = syaml.Str() | syaml.Map({
'present_value': syaml.Str(),
syaml.Optional('states'): syaml.MapPattern(syaml.Str(), syaml.Str()),
SCHEMA = yaml.Seq(
yaml.Map({
'date':
yaml.Datetime(),
'title':
yaml.Str(),
'event':
yaml.Str(),
yaml.Optional('lang', default='cs'):
yaml.Enum([
'cs',
'en',
]),
yaml.Optional('type', default='talk'):
yaml.Enum([
'talk',
'workshop',
'interview',
'text',
]),
yaml.Optional('url'):
yaml.Url(),
yaml.Optional('resources_type', default='slides'):
yaml.Enum([
'slides',
'text',
]),
yaml.Optional('resources_url'):
yaml.Url(),
}), )
class Categorization:
"""A single categorization system.
A categorization system comprises a set of categories, and their relationships as
well as metadata describing the categorization system itself.
Use the categorization object like a dictionary, where codes can be translated
to their meaning using ``cat[code]`` and all codes are available using
``cat.keys()``. Metadata about the categorization is provided in attributes.
If `pandas` is available, you can access a `pandas.DataFrame` with all
category codes, and their meanings at ``cat.df``.
Attributes
----------
name : str
The unique name/code
references : str
Citable reference(s)
title : str
A short, descriptive title for humans
comment : str
Notes and explanations for humans
institution : str
Where the categorization originates
last_update : datetime.date
The date of the last change
version : str, optional
The version of the Categorization, if there are multiple versions
hierarchical : bool
True if descendants and ancestors are defined
"""
hierarchical: bool = False
_strictyaml_schema = sy.Map({
"name":
sy.Str(),
"title":
sy.Str(),
"comment":
sy.Str(),
"references":
sy.Str(),
"institution":
sy.Str(),
"last_update":
sy.Str(),
"hierarchical":
sy.Bool(),
sy.Optional("version"):
sy.Str(),
"categories":
sy.MapPattern(sy.Str(), Category._strictyaml_schema),
})
def _add_categories(self, categories: typing.Dict[str, typing.Dict]):
for code, spec in categories.items():
cat = Category.from_spec(code=code, spec=spec, categorization=self)
self._primary_code_map[code] = cat
for icode in cat.codes:
self._all_codes_map[icode] = cat
def __init__(
self,
*,
categories: typing.Dict[str, typing.Dict],
name: str,
title: str,
comment: str,
references: str,
institution: str,
last_update: datetime.date,
version: typing.Optional[str] = None,
):
self._primary_code_map = {}
self._all_codes_map = {}
self.name = name
self.references = references
self.title = title
self.comment = comment
self.institution = institution
self.last_update = last_update
self.version = version
self._add_categories(categories)
# is filled in __init__.py to contain all categorizations
self._cats: typing.Dict[str, "Categorization"] = {}
def __hash__(self):
return hash(self.name)
@classmethod
def from_yaml(
cls, filepath: typing.Union[str, pathlib.Path, typing.IO[bytes]]
) -> "Categorization":
"""Read Categorization from a StrictYaml file."""
try:
yaml = sy.load(filepath.read(), schema=cls._strictyaml_schema)
except AttributeError:
with open(filepath) as fd:
yaml = sy.load(fd.read(), schema=cls._strictyaml_schema)
return cls.from_spec(yaml.data)
@classmethod
def from_spec(cls, spec: typing.Dict[str, typing.Any]) -> "Categorization":
"""Create Categorization from a Dictionary specification."""
if spec["hierarchical"] != cls.hierarchical:
raise ValueError(
"Specification is for a hierarchical categorization, use"
"HierarchicalCategorization.from_spec.")
last_update = datetime.date.fromisoformat(spec["last_update"])
return cls(
categories=spec["categories"],
name=spec["name"],
title=spec["title"],
comment=spec["comment"],
references=spec["references"],
institution=spec["institution"],
last_update=last_update,
version=spec.get("version", None),
)
@staticmethod
def from_pickle(
filepath: typing.Union[str, pathlib.Path, typing.IO[bytes]]
) -> "Categorization":
"""De-serialize Categorization from a file written by to_pickle.
Note that this uses the pickle module, which executes arbitrary code in the
provided file. Only load from pickle files that you trust."""
return from_pickle(filepath)
def to_spec(self) -> typing.Dict[str, typing.Any]:
"""Turn this categorization into a specification dictionary ready to be written
to a yaml file.
Returns
-------
spec: dict
Specification dictionary understood by `from_spec`.
"""
spec = {
"name": self.name,
"title": self.title,
"comment": self.comment,
"references": self.references,
"institution": self.institution,
"hierarchical": self.hierarchical,
"last_update": self.last_update.isoformat(),
}
if self.version is not None:
spec["version"] = self.version
spec["categories"] = {}
for cat in self.values():
code, cat_spec = cat.to_spec()
spec["categories"][code] = cat_spec
return spec
def to_yaml(self, filepath: typing.Union[str, pathlib.Path]) -> None:
"""Write to a YAML file."""
spec = self.to_spec()
yaml = YAML()
yaml.default_flow_style = False
with open(filepath, "w") as fd:
yaml.dump(spec, fd)
def to_pickle(self, filepath: typing.Union[str, pathlib.Path]) -> None:
"""Serialize to a file using python's pickle."""
spec = self.to_spec()
with open(filepath, "wb") as fd:
pickle.dump(spec, fd, protocol=4)
def keys(self) -> typing.KeysView[str]:
"""Iterate over the codes for all categories."""
return self._primary_code_map.keys()
def values(self) -> typing.ValuesView[Category]:
"""Iterate over the categories."""
return self._primary_code_map.values()
def items(self) -> typing.ItemsView[str, Category]:
"""Iterate over (primary code, category) pairs."""
return self._primary_code_map.items()
def all_keys(self) -> typing.KeysView[str]:
"""Iterate over all codes for all categories."""
return self._all_codes_map.keys()
def __iter__(self) -> typing.Iterable[str]:
return iter(self._primary_code_map)
def __getitem__(self, code: str) -> Category:
"""Get the category for a code."""
return self._all_codes_map[code]
def __contains__(self, code: str) -> bool:
"""Can the code be mapped to a category?"""
return code in self._all_codes_map
def __len__(self) -> int:
return len(self._primary_code_map)
def __repr__(self) -> str:
return (
f"<Categorization {self.name} {self.title!r} with {len(self)} categories>"
)
def __str__(self) -> str:
return self.name
@property
def df(self) -> "pandas.DataFrame":
"""All category codes as a pandas dataframe."""
titles = []
comments = []
alternative_codes = []
for cat in self.values():
titles.append(cat.title)
comments.append(cat.comment)
alternative_codes.append(cat.codes[1:])
return pandas.DataFrame(
index=self.keys(),
data={
"title": titles,
"comment": comments,
"alternative_codes": alternative_codes,
},
)
def _extend_prepare(
self,
*,
categories: typing.Optional[typing.Dict[str, typing.Dict]] = None,
alternative_codes: typing.Optional[typing.Dict[str, str]] = None,
name: str,
title: typing.Optional[str] = None,
comment: typing.Optional[str] = None,
last_update: typing.Optional[datetime.date] = None,
) -> typing.Dict[str, typing.Any]:
spec = self.to_spec()
spec["name"] = f"{self.name}_{name}"
spec["references"] = ""
spec["institution"] = ""
if title is None:
spec["title"] = f"{self.title} + {name}"
else:
spec["title"] = self.title + title
if comment is None:
spec["comment"] = f"{self.comment} extended by {name}"
else:
spec["comment"] = self.comment + comment
if last_update is None:
spec["last_update"] = datetime.date.today().isoformat()
else:
spec["last_update"] = last_update.isoformat()
if categories is not None:
spec["categories"].update(categories)
if alternative_codes is not None:
for alias, primary in alternative_codes.items():
if "alternative_codes" not in spec["categories"][primary]:
spec["categories"][primary]["alternative_codes"] = []
spec["categories"][primary]["alternative_codes"].append(alias)
return spec
def extend(
self,
*,
categories: typing.Optional[typing.Dict[str, typing.Dict]] = None,
alternative_codes: typing.Optional[typing.Dict[str, str]] = None,
name: str,
title: typing.Optional[str] = None,
comment: typing.Optional[str] = None,
last_update: typing.Optional[datetime.date] = None,
) -> "Categorization":
"""Extend the categorization with additional categories, yielding a new
categorization.
Metadata: the ``name``, ``title``, ``comment``, and ``last_update`` are updated
automatically (see below), the ``institution`` and ``references`` are deleted
and the values for ``version`` and ``hierarchical`` are kept.
You can set more accurate metadata (for example, your institution) on the
returned object if needed.
Parameters
----------
categories: dict, optional
Map of new category codes to their specification. The specification is a
dictionary with the keys "title", optionally "comment", and optionally
"alternative_codes".
alternative_codes: dict, optional
Map of new alternative codes. A dictionary with the new alternative code
as key and existing code as value.
name : str
The name of your extension. The returned Categorization will have a name
of "{old_name}_{name}", indicating that it is an extension of the underlying
Categorization.
title : str, optional
A string to add to the original title. If not provided, " + {name}" will be
used.
comment : str, optional
A string to add to the original comment. If not provided,
" extended by {name}" will be used.
last_update : datetime.date, optional
The date of the last update to this extension. Today will be used if not
provided.
Returns
-------
Extended categorization : Categorization
"""
spec = self._extend_prepare(
name=name,
categories=categories,
title=title,
comment=comment,
last_update=last_update,
alternative_codes=alternative_codes,
)
return Categorization.from_spec(spec)
def __eq__(self, other):
if not isinstance(other, Categorization):
return False
if self.name != other.name:
return False
return self._primary_code_map == other._primary_code_map
def conversion_to(
self, other: typing.Union["Categorization", str]) -> Conversion:
"""Get conversion to other categorization.
If conversion rules for this conversion are not included, raises
NotImplementedError."""
if isinstance(other, str):
other_name = other
else:
other_name = other.name
forward_csv_name = f"conversion.{self.name}.{other_name}.csv"
if importlib.resources.is_resource(data, forward_csv_name):
fd = importlib.resources.open_text(data, forward_csv_name)
return ConversionSpec.from_csv(fd).hydrate(cats=self._cats)
reversed_csv_name = f"conversion.{other_name}.{self.name}.csv"
if importlib.resources.is_resource(data, reversed_csv_name):
fd = importlib.resources.open_text(data, reversed_csv_name)
return ConversionSpec.from_csv(fd).hydrate(
cats=self._cats).reversed()
raise NotImplementedError(
f"Conversion between {self.name} and {other_name} not yet included."
)
DATA_PATH = os.path.join(ROOT_PATH, 'data')
LABS_FILENAME = 'labs.yaml'
PROJECTS_FILENAME = 'projects.yaml'
LABS_SCHEMA = sy.Map({
"labs":
sy.MapPattern(
sy.Str(),
sy.Map({
"name":
sy.Str(),
"prof":
sy.Map({
"name": sy.Seq(sy.Str()),
"email": sy.Email(),
}),
"url":
sy.Url(),
"contacts":
sy.EmptyList() | sy.Seq(
sy.Map({
"name": sy.Str(),
sy.Optional("email"): sy.Email(),
}), ),
sy.Optional("notes"):
sy.Str(),
}))
})
PROJECTS_SCHEMA = sy.Map({
"projects":
sy.MapPattern(
import itertools
import logging
from datetime import datetime, timezone
from pathlib import Path
from operator import itemgetter
from pelican import signals
import feedparser
import strictyaml as yaml
logger = logging.getLogger(__name__)
SCHEMA = yaml.Seq(
yaml.Map({
'title': yaml.Str(),
'lang': yaml.Enum(['en', 'cs']),
'link': yaml.Url(),
'href': yaml.Url(),
}), )
def register():
signals.article_generator_finalized.connect(load_feeds)
def load_feeds(article_generator):
settings = article_generator.settings
articles = article_generator.articles
# prepare blog articles
blog_feed_path = settings.get('FEED_ALL_ATOM_URL',
settings['FEED_ALL_ATOM'])
import itertools
from datetime import datetime, timezone
from pathlib import Path
from operator import attrgetter
from pelican import signals, generators
import feedparser
import strictyaml as yaml
logger = logging.getLogger(__name__)
SCHEMA = yaml.Seq(
yaml.Map({
'href': yaml.Url(),
yaml.Optional('if'): yaml.Str(),
}),
)
def register():
signals.get_generators.connect(get_generators)
def get_generators(pelican_object):
return PocketFeedGenerator
class Item():
def __init__(self, entry):
self.title = entry.title
class HierarchicalCategorization(Categorization):
"""In a hierarchical categorization, descendants and ancestors (parents and
children) are defined for each category.
Attributes
----------
total_sum : bool
If the sum of the values of children equals the value of the parent for
extensive quantities. For example, a Categorization containing the Countries in
the EU and the EU could set `total_sum = True`, because the emissions of all
parts of the EU must equal the emissions of the EU. On the contrary, a
categorization of Industries with categories `Power:Fossil Fuels` and
`Power:Gas` which are both children of `Power` must set `total_sum = False`
to avoid double counting of fossil gas.
canonical_top_level_category : HierarchicalCategory
The level of a category is calculated with respect to the canonical top level
category. Commonly, this will be the world total or a similar category. If the
canonical top level category is not set (i.e. is ``None``), levels are not
defined for categories.
"""
hierarchical = True
_strictyaml_schema = sy.Map({
"name":
sy.Str(),
"title":
sy.Str(),
"comment":
sy.Str(),
"references":
sy.Str(),
"institution":
sy.Str(),
"last_update":
sy.Str(),
"hierarchical":
sy.Bool(),
sy.Optional("version"):
sy.Str(),
"total_sum":
sy.Bool(),
sy.Optional("canonical_top_level_category"):
sy.Str(),
"categories":
sy.MapPattern(sy.Str(), HierarchicalCategory._strictyaml_schema),
})
def _add_categories(self, categories: typing.Dict[str, typing.Dict]):
for code, spec in categories.items():
cat = HierarchicalCategory.from_spec(code=code,
spec=spec,
categorization=self)
self._primary_code_map[code] = cat
self._graph.add_node(cat)
for icode in cat.codes:
self._all_codes_map[icode] = cat
for code, spec in categories.items():
if "children" in spec:
parent = self._all_codes_map[code]
for i, child_set in enumerate(spec["children"]):
for child_code in child_set:
self._graph.add_edge(parent,
self._all_codes_map[child_code],
set=i)
def __init__(
self,
*,
categories: typing.Dict[str, typing.Dict],
name: str,
title: str,
comment: str,
references: str,
institution: str,
last_update: datetime.date,
version: typing.Optional[str] = None,
total_sum: bool,
canonical_top_level_category: typing.Optional[str] = None,
):
self._graph = nx.MultiDiGraph()
Categorization.__init__(
self,
categories=categories,
name=name,
title=title,
comment=comment,
references=references,
institution=institution,
last_update=last_update,
version=version,
)
self.total_sum = total_sum
if canonical_top_level_category is None:
self.canonical_top_level_category: typing.Optional[
HierarchicalCategory] = None
else:
self.canonical_top_level_category = self._all_codes_map[
canonical_top_level_category]
def __getitem__(self, code: str) -> HierarchicalCategory:
"""Get the category for a code."""
return self._all_codes_map[code]
def values(self) -> typing.ValuesView[HierarchicalCategory]:
"""Iterate over the categories."""
return self._primary_code_map.values()
def items(self) -> typing.ItemsView[str, HierarchicalCategory]:
"""Iterate over (primary code, category) pairs."""
return self._primary_code_map.items()
@classmethod
def from_spec(
cls,
spec: typing.Dict[str,
typing.Any]) -> "HierarchicalCategorization":
"""Create Categorization from a Dictionary specification."""
if spec["hierarchical"] != cls.hierarchical:
raise ValueError(
"Specification is for a non-hierarchical categorization, use"
"Categorization.from_spec.")
last_update = datetime.date.fromisoformat(spec["last_update"])
return cls(
categories=spec["categories"],
name=spec["name"],
title=spec["title"],
comment=spec["comment"],
references=spec["references"],
institution=spec["institution"],
last_update=last_update,
version=spec.get("version", None),
total_sum=spec["total_sum"],
canonical_top_level_category=spec.get(
"canonical_top_level_category", None),
)
def to_spec(self) -> typing.Dict[str, typing.Any]:
"""Turn this categorization into a specification dictionary ready to be written
to a yaml file.
Returns
-------
spec: dict
Specification dictionary understood by `from_spec`.
"""
# we can't call Categorization.to_spec here because we need to control ordering
# in the returned dict so that we get nicely ordered yaml files.
spec = {
"name": self.name,
"title": self.title,
"comment": self.comment,
"references": self.references,
"institution": self.institution,
"hierarchical": self.hierarchical,
"last_update": self.last_update.isoformat(),
}
if self.version is not None:
spec["version"] = self.version
spec["total_sum"] = self.total_sum
if self.canonical_top_level_category is not None:
spec[
"canonical_top_level_category"] = self.canonical_top_level_category.codes[
0]
spec["categories"] = {}
for cat in self.values():
code, cat_spec = cat.to_spec()
spec["categories"][code] = cat_spec
return spec
@property
def _canonical_subgraph(self) -> nx.DiGraph:
# TODO: from python 3.8 on, there is functools.cached_property to
# automatically cache this - as soon as we drop python 3.7 support, we can
# easily add it.
return nx.DiGraph(
self._graph.edge_subgraph(
((u, v, 0) for (u, v, s) in self._graph.edges(data="set")
if s == 0)))
def _show_subtree_children(
self,
children: typing.Iterable[HierarchicalCategory],
format_func: typing.Callable,
prefix: str,
maxdepth: typing.Optional[int],
) -> str:
children_sorted = natsort.natsorted(children, key=format_func)
r = "".join(
self._show_subtree(
node=child,
prefix=prefix + "│",
format_func=format_func,
maxdepth=maxdepth,
) for child in children_sorted[:-1])
# Last child needs to be called slightly differently
r += self._show_subtree(
node=children_sorted[-1],
prefix=prefix + " ",
last=True,
format_func=format_func,
maxdepth=maxdepth,
)
return r
@staticmethod
def _render_node(
node: HierarchicalCategory,
last: bool,
prefix: str,
format_func: typing.Callable[[HierarchicalCategory], str],
):
formatted = format_func(node)
if prefix:
if last:
return f"{prefix[:-1]}╰{formatted}\n"
else:
return f"{prefix[:-1]}├{formatted}\n"
else:
return f"{formatted}\n"
def _show_subtree(
self,
*,
node: HierarchicalCategory,
prefix="",
last=False,
format_func: typing.Callable[[HierarchicalCategory], str] = str,
maxdepth: typing.Optional[int],
) -> str:
"""Recursively-called function to show a subtree starting at the given node."""
r = self._render_node(node,
last=last,
prefix=prefix,
format_func=format_func)
if maxdepth is not None:
maxdepth -= 1
if maxdepth == 0: # maxdepth reached, nothing more to do
return r
child_sets = node.children
if len(child_sets) == 1:
children = child_sets[0]
if children:
r += self._show_subtree_children(
children=children,
format_func=format_func,
maxdepth=maxdepth,
prefix=prefix,
)
elif len(child_sets) > 1:
prefix += "║"
i = 1
for children in child_sets:
if children:
if i == 1:
r += (
f"{prefix[:-1]}╠╤══ ('{format_func(node)}'s children,"
f" option 1)\n")
else:
r += (
f"{prefix[:-1]}╠╕ ('{format_func(node)}'s children,"
f" option {i})\n")
r += self._show_subtree_children(
children=children,
format_func=format_func,
maxdepth=maxdepth,
prefix=prefix,
)
i += 1
r += f"{prefix[:-1]}╚═══\n"
return r
def show_as_tree(
self,
*,
format_func: typing.Callable[[HierarchicalCategory], str] = str,
maxdepth: typing.Optional[int] = None,
root: typing.Optional[typing.Union[HierarchicalCategory, str]] = None,
) -> str:
"""Format the hierarchy as a tree.
Starting from the given root, or - if no root is given - the top-level
categories (i.e. categories without parents), the tree of categories that are
transitive children of the root is show, with children connected to their
parents using lines. If a parent category has one set of children, the children
are connected to each other and the parent with a simple line. If a parent
category has multiple sets of children, the sets are connected to parent with
double lines and the children in a set are connected to each other with simple
lines.
Parameters
----------
format_func: callable, optional
Function to call to format categories for display. Each category is
formatted for display using format_func(category), so format_func should
return a string without line breaks, otherwise the tree will look weird.
By default, str() is used, so that the first code and the title of the
category are used.
maxdepth: int, optional
Maximum depth to show in the tree. By default, goes to arbitrary depth.
root: HierarchicalCategory or str, optional
HierarchicalCategory object or code to use as the top-most category.
If not given, the whole tree is shown, starting from all categories without
parents.
Returns
-------
tree_str: str
Representation of the hierarchy as formatted string. print() it for optimal
viewing.
"""
if root is None:
top_level_nodes = (node for node in self.values()
if not node.parents)
else:
if not isinstance(root, HierarchicalCategory):
root = self[root]
top_level_nodes = [root]
return "\n".join((self._show_subtree(
node=top_level_node, format_func=format_func, maxdepth=maxdepth))
for top_level_node in top_level_nodes)
def extend(
self,
*,
categories: typing.Optional[typing.Dict[str, typing.Dict]] = None,
alternative_codes: typing.Optional[typing.Dict[str, str]] = None,
children: typing.Optional[typing.List[tuple]] = None,
name: str,
title: typing.Optional[str] = None,
comment: typing.Optional[str] = None,
last_update: typing.Optional[datetime.date] = None,
) -> "HierarchicalCategorization":
"""Extend the categorization with additional categories and relationships,
yielding a new categorization.
Metadata: the ``name``, ``title``, ``comment``, and ``last_update`` are updated
automatically (see below), the ``institution`` and ``references`` are deleted
and the values for ``version``, ``hierarchical``, ``total_sum``, and
``canonical_top_level_category`` are kept.
You can set more accurate metadata (for example, your institution) on the
returned object if needed.
Parameters
----------
categories: dict, optional
Map of new category codes to their specification. The specification is a
dictionary with the keys "title", optionally "comment", and optionally
"alternative_codes".
alternative_codes: dict, optional
Map of new alternative codes. A dictionary with the new alternative code
as key and existing code as value.
children: list, optional
List of ``(parent, (child1, child2, …))`` pairs. The given relationships will
be inserted in the extended categorization.
name : str
The name of your extension. The returned Categorization will have a name
of "{old_name}_{name}", indicating that it is an extension of the underlying
Categorization.
title : str, optional
A string to add to the original title. If not provided, " + {name}" will be
used.
comment : str, optional
A string to add to the original comment. If not provided,
" extended by {name}" will be used.
last_update : datetime.date, optional
The date of the last update to this extension. Today will be used if not
provided.
Returns
-------
Extended categorization : HierarchicalCategorization
"""
spec = self._extend_prepare(
name=name,
categories=categories,
title=title,
comment=comment,
last_update=last_update,
alternative_codes=alternative_codes,
)
if children is not None:
for parent, child_set in children:
if "children" not in spec["categories"][parent]:
spec["categories"][parent]["children"] = []
spec["categories"][parent]["children"].append(child_set)
return HierarchicalCategorization.from_spec(spec)
@property
def df(self) -> "pandas.DataFrame":
"""All category codes as a pandas dataframe."""
titles = []
comments = []
alternative_codes = []
children = []
for cat in self.values():
titles.append(cat.title)
comments.append(cat.comment)
alternative_codes.append(cat.codes[1:])
children.append(
tuple(
tuple(sorted(c.codes[0] for c in cs))
for cs in cat.children))
return pandas.DataFrame(
index=self.keys(),
data={
"title": titles,
"comment": comments,
"alternative_codes": alternative_codes,
"children": children,
},
)
def level(self, cat: typing.Union[str, HierarchicalCategory]) -> int:
"""The level of the given category.
The canonical top-level category has level 1 and its children have level 2 etc.
To calculate the level, first only the first ("canonical") set of children is
considered. Only if no path from the canonical top-level category to the
given category can be found all other sets of children are considered to
calculate the level.
"""
if not isinstance(cat, HierarchicalCategory):
return self.level(self[cat])
if not isinstance(self.canonical_top_level_category,
HierarchicalCategory):
raise ValueError(
"Can not calculate the level without a canonical_top_level_category."
)
# first use the canonical subgraph for shortest paths
csg = self._canonical_subgraph
try:
sp = nx.shortest_path_length(csg,
self.canonical_top_level_category,
cat)
except (nx.NetworkXNoPath, nx.NodeNotFound):
try:
sp = nx.shortest_path_length(self._graph,
self.canonical_top_level_category,
cat)
except (nx.NetworkXNoPath, nx.NodeNotFound):
raise ValueError(
f"{cat.codes[0]!r} is not a transitive child of the "
f"canonical top level "
f"{self.canonical_top_level_category.codes[0]!r}.")
return sp + 1
def parents(
self, cat: typing.Union[str, HierarchicalCategory]
) -> typing.Set[HierarchicalCategory]:
"""The direct parents of the given category."""
if not isinstance(cat, HierarchicalCategory):
return self.parents(self._all_codes_map[cat])
return set(self._graph.predecessors(cat))
def ancestors(
self, cat: typing.Union[str, HierarchicalCategory]
) -> typing.Set[HierarchicalCategory]:
"""All ancestors of the given category, i.e. the direct parents and their
parents, etc."""
if not isinstance(cat, HierarchicalCategory):
return self.ancestors(self._all_codes_map[cat])
return set(nx.ancestors(self._graph, cat))
def children(
self, cat: typing.Union[str, HierarchicalCategory]
) -> typing.List[typing.Set[HierarchicalCategory]]:
"""The list of sets of direct children of the given category."""
if not isinstance(cat, HierarchicalCategory):
return self.children(self._all_codes_map[cat])
children_dict = {}
for (_, child, setno) in self._graph.edges(cat, "set"):
if setno not in children_dict:
children_dict[setno] = []
children_dict[setno].append(child)
return [set(children_dict[x]) for x in sorted(children_dict.keys())]
def descendants(self, cat: typing.Union[str, HierarchicalCategory]):
"""All descendants of the given category, i.e. the direct children and their
children, etc."""
if not isinstance(cat, HierarchicalCategory):
return self.descendants(self._all_codes_map[cat])
return set(nx.descendants(self._graph, cat))
def _yaml_schema(cls) -> strictyaml.Validator:
return strictyaml.Map({
"path": strictyaml.Str(),
"toc": TocConfig._yaml_schema(),
})
class HierarchicalCategory(Category):
"""A single category from a HierarchicalCategorization."""
_strictyaml_schema = sy.Map({
"title":
sy.Str(),
sy.Optional("comment"):
sy.Str(),
sy.Optional("alternative_codes"):
sy.Seq(sy.Str()),
sy.Optional("info"):
sy.MapPattern(sy.Str(), sy.Any()),
sy.Optional("children"):
sy.Seq(sy.Seq(sy.Str())),
})
def __init__(
self,
codes: typing.Tuple[str],
categorization: "HierarchicalCategorization",
title: str,
comment: typing.Optional[str] = None,
info: typing.Optional[dict] = None,
):
Category.__init__(self, codes, categorization, title, comment, info)
self.categorization = categorization
def to_spec(
self) -> (str, typing.Dict[str, typing.Union[str, dict, list]]):
"""Turn this category into a specification ready to be written to a yaml file.
Returns
-------
(code: str, spec: dict)
Primary code and specification dict
"""
code, spec = Category.to_spec(self)
children = []
for child_set in self.children:
children.append(list(sorted((c.codes[0] for c in child_set))))
if children:
spec["children"] = children
return code, spec
@property
def children(self) -> typing.List[typing.Set["HierarchicalCategory"]]:
"""The sets of subcategories comprising this category.
The first set is canonical, the other sets are alternative.
Only the canonical sets are used to calculate the level of a category."""
return self.categorization.children(self)
@property
def parents(self) -> typing.Set["HierarchicalCategory"]:
"""The super-categories where this category is a member of any set of children.
Note that all possible parents are returned, not "canonical" parents.
"""
return self.categorization.parents(self)
@property
def ancestors(self) -> typing.Set["HierarchicalCategory"]:
"""The super-categories where this category or any of its parents is a member
of any set of children, transitively.
Note that all possible ancestors are returned, not only "canonical" ones.
"""
return self.categorization.ancestors(self)
@property
def descendants(self) -> typing.Set["HierarchicalCategory"]:
"""The sets of subcategories comprising this category directly or indirectly.
Note that all possible descendants are returned, not only "canonical" ones."""
return self.categorization.descendants(self)
@property
def level(self) -> int:
"""The level of the category.
The canonical top-level category has level 1 and its children have level 2 etc.
To calculate the level, only the first ("canonical") set of children is
considered for intermediate categories.
"""
return self.categorization.level(self)
import copy
import json
import os
import re
import sys
import strictyaml
SCHEMA_MANIFEST = strictyaml.Seq(
strictyaml.Map({
"name": strictyaml.Str(),
"description": strictyaml.Str(),
"src": strictyaml.Str(),
}))
SCHEMA_RULE = strictyaml.Seq(
strictyaml.Map({
"process":
strictyaml.Str(),
strictyaml.Optional("action"):
strictyaml.Enum(["allow", "deny", "ask"]),
strictyaml.Optional("codeSignature"):
strictyaml.Enum(["ignore"]),
strictyaml.Optional("direction"):
strictyaml.Enum(["incoming", "outgoing"]),
strictyaml.Optional("disabled"):
strictyaml.Bool(),
strictyaml.Optional("notes"):
strictyaml.Str(),
strictyaml.Optional("ports"):
strictyaml.Regex("^(any|\d+((\s+)?\-(\s+)?\d+)?)$"),
for f in sorted(os.listdir(path)):
if f.endswith(testext):
yield os.path.join(path, f)
options_schema = strictyaml.Map({
strictyaml.Optional('directive'):
strictyaml.Str(),
strictyaml.Optional('directive-arguments'):
strictyaml.Seq(strictyaml.Str()),
strictyaml.Optional('directive-options'):
strictyaml.Map({
strictyaml.Optional('clang'):
strictyaml.Seq(strictyaml.Str()),
strictyaml.Optional('compat'):
strictyaml.Str(),
strictyaml.Optional('file'):
strictyaml.Str(),
strictyaml.Optional('members'):
strictyaml.Seq(strictyaml.Str()) | strictyaml.EmptyList(),
strictyaml.Optional('transform'):
strictyaml.Str(),
}),
strictyaml.Optional('expected-failure'):
strictyaml.Bool(),
})
def get_testcase_options(testcase):
# options are optional
options = {}
class Category:
"""A single category."""
_strictyaml_schema = sy.Map({
"title":
sy.Str(),
sy.Optional("comment"):
sy.Str(),
sy.Optional("alternative_codes"):
sy.Seq(sy.Str()),
sy.Optional("info"):
sy.MapPattern(sy.Str(), sy.Any()),
})
def __init__(
self,
codes: typing.Tuple[str],
categorization: "Categorization",
title: str,
comment: typing.Optional[str] = None,
info: typing.Optional[dict] = None,
):
self.codes = codes
self.title = title
self.comment = comment
self.categorization = categorization
if info is None:
self.info = {}
else:
self.info = info
self._hash = None
@classmethod
def from_spec(cls, code: str, spec: typing.Dict,
categorization: "Categorization"):
codes = [code]
if "alternative_codes" in spec:
codes += spec["alternative_codes"]
del spec["alternative_codes"]
return cls(
codes=tuple(codes),
categorization=categorization,
title=spec["title"],
comment=spec.get("comment", None),
info=spec.get("info", None),
)
def to_spec(
self) -> (str, typing.Dict[str, typing.Union[str, dict, list]]):
"""Turn this category into a specification ready to be written to a yaml file.
Returns
-------
(code: str, spec: dict)
Primary code and specification dict
"""
code = self.codes[0]
spec = {"title": self.title}
if self.comment is not None:
spec["comment"] = self.comment
if len(self.codes) > 1:
spec["alternative_codes"] = list(self.codes[1:])
if self.info:
spec["info"] = self.info
return code, spec
def __str__(self) -> str:
return f"{self.codes[0]} {self.title}"
def __eq__(self, other: "Category"):
if not isinstance(other, Category):
return False
return any((x in other.codes for x in self.codes)) and (
self.categorization is other.categorization or self.categorization.
name.startswith(f"{other.categorization.name}_") or other.
categorization.name.startswith(f"{self.categorization.name}_")
or self.categorization.name == other.categorization.name)
def __repr__(self) -> str:
return f"<{self.categorization.name}: {self.codes[0]!r}>"
def __hash__(self):
if self._hash is None:
self._hash = hash(self.categorization.name + self.codes[0])
return self._hash
def __lt__(self, other):
s = natsort.natsorted((self.codes[0], other.codes[0]))
return s[0] == self.codes[0] and self != other