def generate(self, nsmap):
attrs = OrderedDict()
dt = self.mapped_datatype
if self.used:
if isinstance(self.range, CIMEnum):
var, query_base = self.name_query()
attrs[f"{var}_name"] = Column(String(120),
ForeignKey(CIMEnumValue.name),
name=f"{var}_name")
attrs[f"{var}_namespace"] = Column(
String(120),
ForeignKey(CIMEnumValue.namespace_name),
name=f"{var}_namespace")
attrs[f"{var}_enum_name"] = Column(String(120),
ForeignKey(
CIMEnumValue.enum_name),
name=f"{var}_enum_name")
attrs[f"{var}_enum_namespace"] = Column(
String(120),
ForeignKey(CIMEnumValue.enum_namespace),
name=f"{var}_enum_namespace")
attrs[var] = relationship(
CIMEnumValue,
foreign_keys=(attrs[f"{var}_name"],
attrs[f"{var}_namespace"],
attrs[f"{var}_enum_name"],
attrs[f"{var}_enum_namespace"]))
attrs["__table_args__"] = (ForeignKeyConstraint(
(attrs[f"{var}_name"], attrs[f"{var}_namespace"],
attrs[f"{var}_enum_name"],
attrs[f"{var}_enum_namespace"]),
(CIMEnumValue.name, CIMEnumValue.namespace_name,
CIMEnumValue.enum_name, CIMEnumValue.enum_namespace)), )
self.key = f"{var}"
self.xpath = XPath(query_base + "/@rdf:resource",
namespaces=nsmap)
elif self.range:
self.generate_relationship(nsmap)
elif not self.range:
var, query_base = self.name_query()
log.debug(f"Generating property for {var} on {self.name}")
self.key = var
self.xpath = XPath(query_base + "/text()", namespaces=nsmap)
if dt:
if dt == "String":
attrs[var] = Column(String(50), name=f"{var}")
elif dt in ("Float", "Decimal"):
attrs[var] = Column(Float, name=f"{var}")
elif dt == "Integer":
attrs[var] = Column(Integer, name=f"{var}")
elif dt == "Boolean":
attrs[var] = Column(Boolean, name=f"{var}")
else:
attrs[var] = Column(String(30), name=f"{var}")
else:
# Fallback to parsing as String(50)
attrs[var] = Column(String(50), name=f"{var}")
for attr, attr_value in attrs.items():
setattr(self.cls.class_, attr, attr_value)
def _generateXPathMap(cls):
super()._generateXPathMap()
Map = {
"stereotype": XPath(r"cims:stereotype/text()", namespaces=cls.nsmap),
"datatype": XPath(r"cims:dataType/@rdf:resource", namespaces=cls.nsmap),
"isFixed": XPath(r"cims:isFixed/@rdfs:Literal", namespaces=cls.nsmap)
}
if not cls.XPathMap:
cls.XPathMap = Map
else:
cls.XPathMap = {**cls.XPathMap, **Map}
def _generateXPathMap(cls):
"""
Generator for compiled XPath expressions (those require a namespace_name map to be
present, hence they are runtime-compiled)
:return: None
"""
cls.XPathMap = {
"category":
XPath(r"cims:belongsToCategory/@rdf:resource",
namespaces=cls.nsmap),
"label":
XPath(r"rdfs:label/text()", namespaces=cls.nsmap),
"stereotype_text":
XPath(r"cims:stereotype/text()", namespaces=cls.nsmap)
}
return cls.XPathMap
def _generateXPathMap(cls):
super()._generateXPathMap()
Map = {"type": XPath(r"rdf:type/@rdf:resource", namespaces=cls.nsmap)}
if not cls.XPathMap:
cls.XPathMap = Map
else:
cls.XPathMap = {**cls.XPathMap, **Map}
def _generateXPathMap(cls):
super()._generateXPathMap()
Map = {"category": XPath(r"cims:belongsToCategory/@rdf:resource", namespaces=cls.nsmap)}
if not cls.XPathMap:
cls.XPathMap = Map
else:
cls.XPathMap = {**cls.XPathMap, **Map}
def _generateXPathMap(cls):
"""
Compile XPath Expressions for later use (better performance than tree.xpath(...))
:return: None
"""
super()._generateXPathMap()
Map = {
"parent":
XPath(r"rdfs:subClassOf/@rdf:resource", namespaces=cls.nsmap),
"category":
XPath(r"cims:belongsToCategory/@rdf:resource",
namespaces=cls.nsmap)
}
if not cls.XPathMap:
cls.XPathMap = Map
else:
cls.XPathMap = {**cls.XPathMap, **Map}
def _generateXPathMap(cls):
super()._generateXPathMap()
Map = {
"label":
XPath(r"rdfs:label/text()", namespaces=cls.nsmap),
"association":
XPath(r"cims:AssociationUsed/text()", namespaces=cls.nsmap),
"inverseRoleName":
XPath(r"cims:inverseRoleName/@rdf:resource", namespaces=cls.nsmap),
"datatype":
XPath(r"cims:dataType/@rdf:resource", namespaces=cls.nsmap),
"multiplicity":
XPath(r"cims:multiplicity/@rdf:resource", namespaces=cls.nsmap),
"type":
XPath(r"rdf:type/@rdf:resource", namespaces=cls.nsmap),
"domain":
XPath(r"rdfs:domain/@rdf:resource", namespaces=cls.nsmap),
"range":
XPath(r"rdfs:range/@rdf:resource", namespaces=cls.nsmap)
}
if not cls.XPathMap:
cls.XPathMap = Map
else:
cls.XPathMap = {**cls.XPathMap, **Map}
def generate_relationship(self, nsmap=None):
var, query_base = self.name_query()
attrs = {}
Map = {}
log.debug(f"Generating relationship for {var} on {self.name}")
if self.many_remote:
if self.inverse:
br = self.inverse.name if self.namespace.short == "cim" else \
self.namespace.short + "_" + self.inverse.name
tbl = self.generate_association_table()
self.association_table = tbl
attrs[var] = relationship(self.range.full_name,
secondary=tbl,
backref=br)
else:
tbl = self.generate_association_table()
attrs[var] = relationship(self.range.full_name, secondary=tbl)
else:
attrs[f"{var}_id"] = Column(
String(50),
ForeignKey(f"{self.range.full_name}.id"),
name=f"{var}_id")
if self.inverse:
br = self.inverse.name if self.namespace.short == "cim" else \
self.namespace.short + "_" + self.inverse.name
attrs[var] = relationship(self.range.full_name,
foreign_keys=attrs[f"{var}_id"],
backref=br)
else:
attrs[var] = relationship(self.range.full_name,
foreign_keys=attrs[f"{var}_id"])
self.key = f"{var}_id"
self.xpath = XPath(query_base + "/@rdf:resource", namespaces=nsmap)
class_ = self.cls.class_
for attr, attr_value in attrs.items():
setattr(class_, attr, attr_value)
return Map
def merge_sources(sources, model_schema=None):
"""
Merge different sources of CIM datasets (usually the different profiles, but could also be
multiple instances of the same profile when multiple datasets are merged via boundary datasets)
:param sources: SourceInfo objects of the source files.
:param model_schema: The schema used to deserialize the dataset.
:return: A dictionary of the objects found in the dataset, keyed by classname and object uuid.
"""
uuid2name = dict()
uuid2data = dict()
classname_list = defaultdict(set)
from cimpyorm.auxiliary import XPath
xp = {
"id": XPath("@rdf:ID", namespaces=get_nsmap(sources)),
"about": XPath("@rdf:about", namespaces=get_nsmap(sources))
}
for source in sources:
for element in source.tree.getroot():
try:
uuid = determine_uuid(element, xp)
classname = shorten_namespace(element.tag,
HDict(get_nsmap(sources)))
# Set the classname only when UUID is attribute
try:
uuid = xp["id"](element)[0]
if uuid in uuid2name and uuid2name[uuid] != classname:
# If multiple objects of different class share the same uuid, raise an Error
raise ReferenceError(
f"uuid {uuid}={classname} already defined as {uuid2name[uuid]}"
)
uuid2name[uuid] = classname
except IndexError:
pass
classname_list[uuid] |= {classname}
if uuid not in uuid2data:
uuid2data[uuid] = element
else:
[uuid2data[uuid].append(sub) for sub in element] # pylint: disable=expression-not-assigned
except ValueError:
log.warning(f"Skipped element during merge: {element}.")
# print warning in case uuid references use different classnames
for uuid, name_set in classname_list.items():
if len(name_set) > 1:
log.warning(
f"Ambiguous classnames for {uuid} of type {uuid2name.get(uuid, None)} = {name_set}"
)
# check that the class is the most specific one in the list
if model_schema is not None:
schema_classes = model_schema.get_classes()
for uuid, classname in uuid2name.items():
try:
cls = schema_classes[classname]
except KeyError:
log.info(
f"Class {classname} is not included in schema. Objects of this class are not deserialized."
)
else:
try:
if not all(
issubclass(cls, schema_classes[_cname])
for _cname in classname_list[uuid]):
raise ValueError(
f"Class {classname} is not most specific of {classname_list[uuid]}."
)
except KeyError as ex:
raise ReferenceError(
f"Malformed schema. Class-hierarchy-element is missing: {ex}."
)
# transform the data into output structure
d_ = defaultdict(dict)
for uuid, classname in uuid2name.items():
d_[classname][uuid] = uuid2data[uuid]
return d_
def __init__(self,
dataset=None,
version: str = "16",
rdfs_path=None,
profile_whitelist=None):
"""
Initialize a Schema object, containing information about the schema elements.
"""
self.g = None
if not dataset:
backend = InMemory()
backend.reset()
dataset = backend.ORM
if not rdfs_path:
rdfs_path = find_rdfs_path(version)
if not rdfs_path:
raise FileNotFoundError(
"Failed to find schema file. Please provide one.")
self.rdfs_path = rdfs_path
if profile_whitelist:
profile_whitelist = self.parse_profile_whitelist(profile_whitelist)
self.profiles = profile_whitelist
self.schema_descriptions, profiles = merge_schema_descriptions(
load_schema_descriptions(rdfs_path), profile_whitelist)
log.info(f"Generating Schema backend.")
try:
elements = dataset.query(CIMClass).count()
except OperationalError:
elements = None
if elements:
# A schema is already present, so just load it instead of recreating
self.session = dataset
self.Element_classes = {
c.__name__: c
for c in
[CIMPackage, CIMClass, CIMProp, CIMDT, CIMEnum, CIMEnumValue]
}
self.Elements = {
c.__name__: {
cim_class.name: cim_class
for cim_class in dataset.query(c).all()
}
for c in self.Element_classes.values()
}
else:
self.session = dataset
self.Element_classes = {
c.__name__: c
for c in [
ElementMixin, CIMPackage, CIMClass, CIMProp, CIMDT,
CIMEnum, CIMEnumValue
]
}
self.Elements = {
c.__name__: defaultdict(list)
for c in self.Element_classes.values()
}
_Elements = []
merged_nsmaps = dict(
ChainMap(*(element.nsmap
for element in self.schema_descriptions.values())))
profiles = self._generate_profiles(profiles, merged_nsmaps,
rdfs_path)
self.session.add_all(profiles.values())
xp = {
"type_res":
XPath(f"rdf:type/@rdf:resource", namespaces=merged_nsmaps),
"stype_res":
XPath(f"cims:stereotype/@rdf:resource",
namespaces=merged_nsmaps),
"stype_txt":
XPath(f"cims:stereotype/text()", namespaces=merged_nsmaps)
}
for key, element in self.schema_descriptions.items():
element.extract_types(xp)
element.schema_type = element.get_type(xp)
self._init_parser(merged_nsmaps)
for short, full_uri in merged_nsmaps.items():
_ns = CIMNamespace(short=short, full_name=full_uri)
self.session.add(_ns)
self._generate(profiles)
self.session.commit()
for _, Cat_Elements in self.Elements.items():
self.session.add_all(Cat_Elements.values())
self.session.commit()
log.info(f"Schema generated")
self._generate_ORM(dataset, profiles)
dataset.schema = self