def test_get_identifier(self):
"""Test the get_identifier method."""
self.assertEqual(
OntologyParser.get_parser(YML_FILE).identifier, "parser_test")
self.assertEqual(
OntologyParser.get_parser("parser_test").identifier, "parser_test")
self.assertEqual(OntologyParser.get_parser("emmo").identifier, "emmo")
def test_get_namespace_name(self):
"""Test the get_namespace_name method."""
self.assertItemsEqual(
OntologyParser.get_parser(YML_FILE).namespaces.keys(),
["parser_test"],
)
self.assertItemsEqual(
OntologyParser.get_parser("parser_test").namespaces.keys(),
["parser_test"],
)
self.assertItemsEqual(
OntologyParser.get_parser("emmo").namespaces.keys(),
[
"mereotopology",
"physical",
"top",
"semiotics",
"perceptual",
"reductionistic",
"holistic",
"physicalistic",
"math",
"properties",
"materials",
"metrology",
"models",
"manufacturing",
"isq",
"siunits",
],
)
def test_get_file_path(self):
"""Test the get_file_path method."""
self.assertEqual(
OntologyParser.parse_file_path("test/my_file.yml"),
"test/my_file.yml",
)
self.assertEqual(
OntologyParser.parse_file_path("my_file").lower(),
os.path.abspath(
os.path.relpath(
os.path.join(
os.path.dirname(__file__),
"..",
"osp",
"core",
"ontology",
"docs",
"my_file.yml",
))).lower(),
)
self.assertEqual(
OntologyParser.parse_file_path("emmo").lower(),
os.path.abspath(
os.path.relpath(
os.path.join(
os.path.dirname(__file__),
"..",
"osp",
"core",
"ontology",
"docs",
"emmo.yml",
))).lower(),
)
self.assertEqual(
OntologyParser.parse_file_path("city").lower(),
os.path.abspath(
os.path.relpath(
os.path.join(
os.path.dirname(__file__),
"..",
"osp",
"core",
"ontology",
"docs",
"city.ontology.yml",
))).lower(),
)
def test_parse_guess_format(self):
"""Test the parsing a file without providing the format."""
modified_yml_config_path = Path(YML_FILE)
modified_yml_config_path = str(
modified_yml_config_path.with_name(
modified_yml_config_path.stem + "_mod" +
modified_yml_config_path.suffix))
try:
# Create a copy of YML_FILE and remove the 'format' keyword.
with open(modified_yml_config_path, "w") as modified_yml_config:
with open(YML_FILE, "r") as yml_config:
modified_yml_config.write(
re.sub(
r"^[\s]*format:[\s].*",
"",
yml_config.read(),
flags=re.MULTILINE,
))
parser = OntologyParser.get_parser(modified_yml_config_path)
g1 = rdflib.Graph()
g1.parse(RDF_FILE, format="ttl")
self.assertTrue(parser.graph, g1)
finally:
if os.path.exists(modified_yml_config_path):
os.remove(modified_yml_config_path)
def test_parse_yml_download(self):
"""Test downloading owl ontologies."""
def request_callback(request):
headers = {"request-id": "728d329e-0e86-11e4-a748-0c84dc037c13"}
return 200, headers, "<ns1:a> <ns1:b> <ns1:c> ."
url = "http://my_ontology.com/ontology.owl"
responses.add_callback(
responses.GET,
url,
callback=request_callback,
content_type="text/plain",
)
doc = dict(YML_DOC)
doc["ontology_file"] = url
with tempfile.TemporaryDirectory() as tempdir:
yml_path = os.path.join(tempdir, "parser_test.yml")
with open(yml_path, "w") as file:
yaml.safe_dump(doc, file)
try:
parser = OntologyParser.get_parser(yml_path)
self.assertIn(
(
rdflib.URIRef("ns1:a"),
rdflib.URIRef("ns1:b"),
rdflib.URIRef("ns1:c"),
),
parser.graph,
)
except EmptyOntologyFileError:
pass
def test_parse_rdf(self):
"""Test parsing an rdf file."""
with tempfile.TemporaryDirectory() as tempdir:
new_config = self.config_file
new_config["namespaces"] = {}
new_yml_path = os.path.join(tempdir, os.path.basename(YML_FILE))
new_ttl_path = os.path.join(tempdir, os.path.basename(RDF_FILE))
with open(new_yml_path, "w") as file:
yaml.dump(new_config, file)
shutil.copy(RDF_FILE, new_ttl_path)
self.assertRaises(TypeError, new_yml_path, self.parser.parse)
with tempfile.TemporaryDirectory() as tempdir:
new_config = self.config_file
new_config["identifier"] = "x"
new_yml_path = os.path.join(tempdir, os.path.basename(YML_FILE))
new_ttl_path = os.path.join(tempdir, os.path.basename(RDF_FILE))
with open(new_yml_path, "w") as file:
yaml.dump(new_config, file)
shutil.copy(RDF_FILE, new_ttl_path)
self.assertRaises(TypeError, new_yml_path, self.parser.parse)
with tempfile.TemporaryDirectory() as tempdir:
new_config = self.config_file
new_config["namespaces"] = {}
new_config["identifier"] = "x"
new_config["invalid"] = True
new_yml_path = os.path.join(tempdir, os.path.basename(YML_FILE))
new_ttl_path = os.path.join(tempdir, os.path.basename(RDF_FILE))
with open(new_yml_path, "w") as file:
yaml.dump(new_config, file)
shutil.copy(RDF_FILE, new_ttl_path)
self.assertRaises(TypeError, new_yml_path, self.parser.parse)
with tempfile.TemporaryDirectory() as tempdir:
config = dict(
identifier="parser_test",
ontology_file=RDF_FILE,
namespaces={
"parser_test": "http://www.osp-core.com/parser_test"
},
format="ttl",
file="file.ttl",
)
new_yml_path = os.path.join(tempdir, "file.yml")
new_ttl_path = os.path.join(tempdir, "file.ttl")
with open(new_yml_path, "w") as file:
yaml.dump(config, file)
shutil.copy(RDF_FILE, new_ttl_path)
rdf = rdflib.Graph()
rdf.parse(RDF_FILE, format="ttl")
parser = OntologyParser.get_parser(new_yml_path)
graph = parser.graph
self.assertEqual(len(graph), len(rdf))
self.assertTrue(isomorphic(graph, rdf))
self.assertIn(
rdflib.URIRef("http://www.osp-core.com/parser_test#"),
list(parser.namespaces.values()),
)
def _get_replaced_packages(self, new_packages):
"""Get package paths to install.
Get the package paths that by replacing the package paths of
already installed packages with new packages,
Args:
new_packages (List[str]): Path to new package files.
Returns:
List[str]: Resulting list of package paths.
"""
installed = dict(self.get_installed_packages(return_path=True))
for pkg in new_packages:
installed[OntologyParser.get_parser(pkg).identifier] = pkg
return installed.values()
def test_install(self):
"""Test the store method."""
parser = OntologyParser.get_parser(YML_FILE)
with tempfile.TemporaryDirectory() as destination:
parser.install(destination)
self.assertItemsEqual(os.listdir(destination),
["parser_test.xml", "parser_test.yml"])
with open(os.path.join(destination, "parser_test.yml")) as f:
yml_doc = yaml.safe_load(f)
self.assertEqual(yml_doc["ontology_file"], "parser_test.xml")
yml_doc["ontology_file"] = YML_DOC["ontology_file"]
self.assertEqual(yml_doc["format"], "xml")
copy = YML_DOC.copy()
del yml_doc["format"]
del copy["format"]
self.assertEqual(yml_doc, copy)
g = rdflib.Graph()
g.parse(os.path.join(destination, "parser_test.xml"), format="xml")
self.assertTrue(isomorphic(g, parser.graph))
def _get_new_packages(self, packages):
"""From the given list of packages, return the ones that are new.
Args:
packages (List[str]): Path to ontology file.
Returns:
List[str]: List of package path that are new
"""
result = set(packages)
installed = set(self.get_installed_packages())
for pkg in packages:
identifier = OntologyParser.get_parser(pkg).identifier
if identifier in installed:
logger.info(
"Skipping package %s with identifier %s, "
"because it is already installed." % (pkg, identifier)
)
result.remove(pkg)
return result
from osp.core.cuds import Cuds
from osp.core.session.core_session import CoreSession
from osp.core.utils.general import branch, export_cuds, import_cuds
from .test_transport_session import assertJsonLdEqual
# Load the ontology for the test.
try:
from osp.core.namespaces import test_importexport
except ImportError: # If the ontology is not installed.
from osp.core.ontology.namespace_registry import namespace_registry
from osp.core.ontology.parser.parser import OntologyParser
namespace_registry.load_parser(
OntologyParser.get_parser(
str(Path(__file__).parent / "test_importexport.owl.yml")))
test_importexport = namespace_registry.test_importexport
# Load also the city ontology.
try:
from osp.core.namespaces import city
except ImportError:
from osp.core.ontology import Parser
from osp.core.ontology.namespace_registry import namespace_registry
Parser().parse("city")
city = namespace_registry.city
class TestImportExport(unittest.TestCase):
"""Loads files containing CUDS and checks the correctness of the data.
def _install(self, files, filter_func, clear):
"""Install the ontology.
Args:
files (List[str]): The ontology files to install
filter_func (Callable): Function that takes the list of files
as input and returns a list of files that need to be installed.
clear (bool): Whether it is necessary to clear what is already
installed.
"""
# Determine whether the namespace names need to be unbound from the
# `osp.core.namespaces` and `osp.core` modules manually (Python 3.6)
# or not.
python_36 = (sys.version_info.major, sys.version_info.minor) <= (3, 6)
if python_36 and clear:
from osp.core.ontology.namespace_registry import namespace_registry
unbound_manually = (
True
if self.namespace_registry is namespace_registry
else False
)
else:
unbound_manually = False
os.makedirs(self.path, exist_ok=True)
# Save existing namespace names if namespaces have to be unbound
# manually. Otherwise, just set the variable
# to `None` in order to save computation time.
unbound_manually = (
set(ns for ns in self.namespace_registry)
if unbound_manually
else set()
)
if clear:
self.namespace_registry.clear()
files = self._sort_for_installation(
filter_func(files),
set(self.get_installed_packages()) if not clear else set(),
)
installed_packages = set()
for file in files:
parser = OntologyParser.get_parser(file)
self.namespace_registry.load_parser(parser)
# serialize the result
parser.install(self.path)
installed_packages.add(parser.identifier)
if clear:
all_ontology_files, files_to_keep = set(), set()
for ext in ("yml", "xml"):
all_ontology_files |= set(
os.path.basename(x)
for x in glob.glob(os.path.join(self.path, f"*.{ext}"))
)
files_to_keep |= set(
f"{identifier}.{ext}" for identifier in installed_packages
)
files_to_remove = all_ontology_files - files_to_keep
for file in files_to_remove:
os.remove(os.path.join(self.path, file))
if python_36: # Bound and unbound namespaces manually
from ... import core
from .. import namespaces
if unbound_manually:
unbound_manually = unbound_manually.difference(
ns for ns in self.namespace_registry
) # Remove the namespaces that are kept installed.
self.namespace_registry.update_namespaces(
modules=[core, namespaces], remove=unbound_manually
)
self.namespace_registry.store(self.path)
def _sort_for_installation(self, files, installed):
"""Get the right order to install the files.
Args:
files (List[str]): The list of file paths to sort.
Raises:
RuntimeError: Unsatisfied requirements after installation.
Returns:
List[str]: The sorted list of file paths.
"""
result = list()
files = {OntologyParser.get_parser(f).identifier: f for f in files}
requirements = {
n: OntologyParser.get_parser(f).requirements
for n, f in files.items()
}
# If the requirements for an ontology package are bundled with
# OSP-core, try to install them automatically.
package_and_dependents = dict()
try:
package_and_dependents: Dict[
str, Set[str]
] = self._resolve_dependencies_install(files, requirements, dict())
files.update(
{
OntologyParser.get_parser(f).identifier: f
for f in package_and_dependents
}
)
requirements.update(
{
n: OntologyParser.get_parser(f).requirements
for n, f in files.items()
}
)
except FileNotFoundError:
pass
# order the files
while requirements:
add_to_result = list()
for namespace, req in requirements.items():
req -= installed | set(result)
if not req:
add_to_result.append(namespace)
if not add_to_result:
raise RuntimeError(
"Installation failed. Unsatisfied requirements: \n - %s"
% "\n - ".join(
["%s: %s" % (n, r) for n, r in requirements.items()]
)
)
result += add_to_result
for x in add_to_result:
del requirements[x]
dependencies_to_install = set(package_and_dependents) - set(installed)
if dependencies_to_install:
logger.info(
"Also installing dependencies: %s."
% ", ".join(dependencies_to_install)
)
logger.info("Will install the following namespaces: %s" % result)
return [files[n] for n in result]
def test_parse(self):
"""Test the parsing a file."""
parser = OntologyParser.get_parser(YML_FILE)
g1 = rdflib.Graph()
g1.parse(RDF_FILE, format="ttl")
self.assertTrue(parser.graph, g1)
def test_get_requirements(self):
"""Test the get_requirements() method."""
self.assertEqual(
OntologyParser.get_parser(YML_FILE).requirements, set())
self.assertEqual(
OntologyParser.get_parser("parser_test").requirements, {"city"})
def _resolve_dependencies_install(
self,
files: Dict[str, str],
requirements: Dict[str, Set[str]],
dependents: Dict[str, Set[str]],
) -> Dict[str, Set[str]]:
"""Find and resolve the dependencies of the packages to be installed.
Automatic resolution of dependencies is only feasible if the
dependency is bundled with OSP-core.
Args:
files: The packages that are going to be installed together with
their file path.
requirements: The dependencies for each package that is going to
be installed.
dependents: A dictionary with package names and the packages
that depend on them.
"""
initial_files = files
additional_files: Dict[str, str] = dict()
# The statement below avoids installing the file bundled with
# OSP-core when the user provides a custom file providing the same
# package identifier.
requirements = {
n: {req for req in requirements_set if req not in initial_files}
for n, requirements_set in requirements.items()
}
new_requirements: Dict[str, Set[str]] = dict()
for package, requirements_set in requirements.items():
# Queue the requirements for installation if bundled with
# OSP-core and not already queued.
actually_missing_requirements = {
package
for package in requirements_set
if package not in additional_files
}
for requirement in actually_missing_requirements:
try:
parser = OntologyParser.get_parser(requirement)
additional_files[parser.identifier] = requirement
new_requirements.update(
{parser.identifier: parser.requirements}
)
except FileNotFoundError:
pass
# Store which packages are requiring the requirements that were
# queued for installation to show the information on the logs.
# In addition, the `dependents` dictionary keys are the
# additional packages to be installed.
initially_missing_requirements = {
package
for package in requirements_set
if package not in initial_files
}
for requirement in initially_missing_requirements:
dependents[requirement] = dependents.get(
requirement, set()
) | {package}
files.update(additional_files)
new_requirements_exist = bool(
{req for req_set in new_requirements.values() for req in req_set}
- {req for req_set in requirements.values() for req in req_set}
)
if new_requirements_exist:
requirements.update(new_requirements)
dependents = self._resolve_dependencies_install(
files, requirements, dependents
)
return dependents
def _get_remaining_packages(self, remove_packages):
"""Get package paths to install.
Get list of packages that remain after given list of packages
have been removed.
Args:
remove_packages (List[str]): List of packages to remove.
Raises:
ValueError: Given package to remove is not installed
Returns:
List[str]: The remaining packages.
"""
remove_pkgs = set()
installed_pkgs = dict(self.get_installed_packages(return_path=True))
for pkg in remove_packages:
if pkg.endswith(".yml") and os.path.exists(pkg):
pkg = OntologyParser.get_parser(pkg).identifier
if pkg in installed_pkgs:
remove_pkgs.add(pkg)
else:
raise ValueError(
"Could not uninstall %s. No file nor "
"installed ontology package. "
"Make sure to only specify valid "
"yml files or ontology package names." % pkg
)
remaining_packages = {
k: v for k, v in installed_pkgs.items() if k not in remove_pkgs
}
# Block package removal if another package depends on it.
remaining_packages_requirements = {
name: OntologyParser.get_parser(path).requirements
for name, path in remaining_packages.items()
}
all_conflicts = self._resolve_dependencies_removal(
remaining_packages_requirements, dict(), set(remove_pkgs)
)
if all_conflicts:
"""Raise an exception."""
message = (
"Cannot remove package{plural} {cannot_remove}{comma} "
"because other installed packages depend on {pronoun}: "
"{dependency_list}. "
"Please remove the packages {all_packages_to_remove} "
"all together."
)
cannot_remove = set(
conflict
for conflicts in all_conflicts.values()
for conflict in conflicts
if conflict in remove_pkgs
)
plural = "s" if len(cannot_remove) > 1 else ""
comma = ";" if plural else ","
pronoun = "them" if plural else "it"
cannot_remove = ", ".join(cannot_remove)
one_dependency = next(iter(all_conflicts.values()))
all_dependencies_equal = all(
one_dependency == x for x in all_conflicts.values()
)
if all_dependencies_equal:
dependency_list = ", ".join(all_conflicts)
else:
dependency_list = set()
for package, conflicts in all_conflicts.items():
dependency_list.add(
f"package {package} depends on "
f"{', '.join(conflicts)}"
)
dependency_list = "; ".join(dependency_list)
all_packages_to_remove = set(all_conflicts) | remove_pkgs
last_package = all_packages_to_remove.pop()
all_packages_to_remove = (
", ".join(all_packages_to_remove) + " and " + last_package
)
message = message.format(
plural=plural,
cannot_remove=cannot_remove,
comma=comma,
pronoun=pronoun,
dependency_list=dependency_list,
all_packages_to_remove=all_packages_to_remove,
)
raise RuntimeError(message)
return [v for v in remaining_packages.values()]
"""Test the API of CUDS objects using the foaf ontology."""
import uuid
import rdflib
import unittest2 as unittest
try:
from osp.core.namespaces import foaf
except ImportError:
from osp.core.ontology.namespace_registry import namespace_registry
from osp.core.ontology.parser.parser import OntologyParser
namespace_registry.load_parser(OntologyParser.get_parser("foaf"))
foaf = namespace_registry.foaf
class TestAPIfoaf(unittest.TestCase):
"""Test the API of CUDS objects using the foaf ontology."""
def test_creation(self):
"""Test creation of objectes are possible."""
c = foaf.Person()
self.assertTrue(c.is_a(foaf.Person))
def test_uid(self):
"""Test that the uid variable contains an uid."""
c = foaf.Person()
self.assertIsInstance(c.uid, (uuid.UUID, rdflib.URIRef))
def test_relations(self):
"""Test some relationships."""
a = foaf.Person()