def test_unrelated_imports_excluded(dirs):
imports_ctxid = 'http://example.org/imports'
ctxid_1 = 'http://example.org/ctx1'
ctxid_2 = 'http://example.org/ctx2'
ctxid_3 = 'http://example.org/ctx3'
ctxid_4 = 'http://example.org/ctx4'
# Make a descriptor that includes ctx1 and the imports, but not ctx2
d = Descriptor('test')
d.includes.add(make_include_func(ctxid_1))
d.includes.add(make_include_func(ctxid_2))
# Add some triples so the contexts aren't empty -- we can't save an empty context
g = rdflib.ConjunctiveGraph()
cg_1 = g.get_context(ctxid_1)
cg_2 = g.get_context(ctxid_2)
cg_3 = g.get_context(ctxid_3)
cg_4 = g.get_context(ctxid_4)
cg_imp = g.get_context(imports_ctxid)
with transaction.manager:
cg_1.add((aURI('a'), aURI('b'), aURI('c')))
cg_2.add((aURI('d'), aURI('e'), aURI('f')))
cg_3.add((aURI('g'), aURI('h'), aURI('i')))
cg_4.add((aURI('j'), aURI('k'), aURI('l')))
cg_imp.add((URIRef(ctxid_1), CONTEXT_IMPORTS, URIRef(ctxid_2)))
cg_imp.add((URIRef(ctxid_3), CONTEXT_IMPORTS, URIRef(ctxid_4)))
bi = Installer(*dirs, imports_ctx=imports_ctxid, graph=g)
bi.install(d)
with Bundle(d.id, dirs.bundles_directory) as bnd:
g = bnd.rdf.get_context(bnd.conf[IMPORTS_CONTEXT_KEY])
assert (URIRef(ctxid_3), CONTEXT_IMPORTS, URIRef(ctxid_4)) not in g
def make_bundle(self, ident, fname, version):
desc = Descriptor(ident)
desc.name = 'A Bundle'
desc.version = version
desc.description = 'An example bundle'
desc.includes = (set([
make_include_func('https://example.org/bundles#example'),
make_include_func('https://example.org/imports'),
make_include_func('https://example.org/types'),
make_include_func('http://www.w3.org/2000/01/rdf-schema'),
make_include_func('http://www.w3.org/1999/02/22-rdf-syntax-ns'),
make_include_func(BASE_SCHEMA_URL)
]) | set(make_include_func(x) for x in self.contexts))
rdf = self.conn.conf['rdf.graph']
bi = Installer(self.srcdir,
self.bnddir,
rdf,
imports_ctx='https://example.org/imports')
install_dir = bi.install(desc)
L.info('installed files in bundle %s at %s', ident, install_dir)
with tarfile.open(p(self.srvdir, fname + '.tar.xz'),
mode='w:xz') as ba:
# arcname='.' removes the leading part of the path to the install directory
ba.add(install_dir, arcname='.')
def test_imports_are_included(dirs):
'''
If we have imports and no dependencies, then thrown an exception if we have not
included them in the bundle
'''
imports_ctxid = 'http://example.org/imports'
ctxid_1 = 'http://example.org/ctx1'
ctxid_2 = 'http://example.org/ctx2'
# Make a descriptor that includes ctx1 and the imports, but not ctx2
d = Descriptor('test')
d.includes.add(make_include_func(ctxid_1))
d.includes.add(make_include_func(ctxid_2))
# Add some triples so the contexts aren't empty -- we can't save an empty context
g = rdflib.ConjunctiveGraph()
cg_1 = g.get_context(ctxid_1)
cg_2 = g.get_context(ctxid_2)
cg_imp = g.get_context(imports_ctxid)
with transaction.manager:
cg_1.add((aURI('a'), aURI('b'), aURI('c')))
cg_2.add((aURI('d'), aURI('e'), aURI('f')))
cg_imp.add((URIRef(ctxid_1), CONTEXT_IMPORTS, URIRef(ctxid_2)))
bi = Installer(*dirs, imports_ctx=imports_ctxid, graph=g)
bi.install(d)
with Bundle(d.id, dirs.bundles_directory) as bnd:
g = bnd.rdf.get_context(bnd.conf[IMPORTS_CONTEXT_KEY])
assert (URIRef(ctxid_1), CONTEXT_IMPORTS, URIRef(ctxid_2)) in g
def test_no_dupe(dirs):
'''
Test that if we have two contexts with the same contents that we don't create more
than one file for it.
The index will point to the same file for the two contexts
'''
d = Descriptor('test')
ctxid_1 = 'http://example.org/ctx1'
ctxid_2 = 'http://example.org/ctx2'
d.includes.add(make_include_func(ctxid_1))
d.includes.add(make_include_func(ctxid_2))
g = rdflib.ConjunctiveGraph()
cg = g.get_context(ctxid_1)
with transaction.manager:
cg.add((aURI('a'), aURI('b'), aURI('c')))
cg = g.get_context(ctxid_2)
with transaction.manager:
cg.add((aURI('a'), aURI('b'), aURI('c')))
bi = Installer(*dirs, graph=g)
bi.install(d)
graph_files = [
x for x in listdir(p(dirs.bundles_directory, 'test', '1', 'graphs'))
if x.endswith('.nt')
]
assert len(graph_files) == 1
def test_file_hash(dirs):
d = Descriptor('test')
open(p(dirs[0], 'somefile'), 'w').close()
d.files = FilesDescriptor()
d.files.includes.add('somefile')
g = rdflib.ConjunctiveGraph()
bi = Installer(*dirs, graph=g)
bi.install(d)
assert isfile(p(dirs.bundles_directory, 'test', '1', 'files', 'hashes'))
def test_fail_on_non_empty_target(dirs):
d = Descriptor('test')
g = rdflib.ConjunctiveGraph()
bi = Installer(*dirs, graph=g)
bundles_directory = dirs[1]
sma = p(bundles_directory, 'test', '1', 'blah')
makedirs(sma)
with pytest.raises(TargetIsNotEmpty):
bi.install(d)
def test_file_pattern_copy(dirs):
d = Descriptor('test')
open(p(dirs[0], 'somefile'), 'w').close()
d.files = FilesDescriptor()
d.files.patterns.add('some*')
g = rdflib.ConjunctiveGraph()
bi = Installer(*dirs, graph=g)
bi.install(d)
bfiles = p(dirs.bundles_directory, 'test', '1', 'files')
assert set(listdir(bfiles)) == set(['hashes', 'somefile'])
def test_context_index_file_exists(dirs):
d = Descriptor('test')
ctxid = 'http://example.org/ctx1'
d.includes.add(make_include_func(ctxid))
g = rdflib.ConjunctiveGraph()
cg = g.get_context(ctxid)
cg.add((aURI('a'), aURI('b'), aURI('c')))
bi = Installer(*dirs, graph=g)
bi.install(d)
assert isfile(p(dirs.bundles_directory, 'test', '1', 'graphs', 'index'))
def test_file_hash_content(dirs):
d = Descriptor('test')
open(p(dirs[0], 'somefile'), 'w').close()
d.files = FilesDescriptor()
d.files.includes.add('somefile')
g = rdflib.ConjunctiveGraph()
bi = Installer(*dirs, graph=g)
bi.install(d)
with open(p(dirs.bundles_directory, 'test', '1', 'files', 'hashes'),
'rb') as f:
contents = f.read()
assert b'somefile' in contents
def test_context_index_file_contains_ctxid(dirs):
d = Descriptor('test')
ctxid = 'http://example.org/ctx1'
d.includes.add(make_include_func(ctxid))
g = rdflib.ConjunctiveGraph()
cg = g.get_context(ctxid)
with transaction.manager:
cg.add((aURI('a'), aURI('b'), aURI('c')))
bi = Installer(*dirs, graph=g)
bi.install(d)
with open(p(dirs.bundles_directory, 'test', '1', 'graphs', 'index'),
'rb') as f:
assert f.read().startswith(ctxid.encode('UTF-8'))
def test_class_registry_in_manifest(dirs):
'''
If a class registry context is specified, then include it
'''
cr_ctxid = 'http://example.org/class_registry'
# Make a descriptor that includes ctx1 and the imports, but not ctx2
d = Descriptor('test')
# Add some triples so the contexts aren't empty -- we can't save an empty context
g = rdflib.ConjunctiveGraph()
bi = Installer(*dirs, class_registry_ctx=cr_ctxid, graph=g)
bdir = bi.install(d)
with open(p(bdir, BUNDLE_MANIFEST_FILE_NAME)) as mf:
manifest_data = json.load(mf)
assert manifest_data[CLASS_REGISTRY_CONTEXT_KEY]
def test_class_registry_contents(dirs):
'''
If a class registry context is specified, then include it
'''
cr_ctxid = 'http://example.org/class_registry'
# Make a descriptor that includes ctx1 and the imports, but not ctx2
d = Descriptor('test')
# Add some triples so the contexts aren't empty -- we can't save an empty context
g = rdflib.ConjunctiveGraph()
cg_cr = g.get_context(cr_ctxid)
with transaction.manager:
cg_cr.add((aURI('blah'), aURI('bruh'), aURI('uhhhh')))
bi = Installer(*dirs, class_registry_ctx=cr_ctxid, graph=g)
bi.install(d)
with Bundle(d.id, dirs.bundles_directory) as bnd:
g = bnd.rdf.get_context(bnd.conf[CLASS_REGISTRY_CONTEXT_KEY])
assert (aURI('blah'), aURI('bruh'), aURI('uhhhh')) in g
def test_imports_in_dependencies(dirs):
'''
If we have imports and a dependency includes the context, then we shouldn't have an
error.
Versioned bundles are assumed to be immutable, so we won't re-fetch a bundle already
in the local index
'''
imports_ctxid = 'http://example.org/imports'
ctxid_1 = 'http://example.org/ctx1'
ctxid_2 = 'http://example.org/ctx2'
# Make a descriptor that includes ctx1 and the imports, but not ctx2
d = Descriptor('test')
d.includes.add(make_include_func(ctxid_1))
d.includes.add(make_include_func(imports_ctxid))
d.dependencies.add(DependencyDescriptor('dep'))
dep_d = Descriptor('dep')
dep_d.includes.add(make_include_func(ctxid_2))
# Add some triples so the contexts aren't empty -- we can't save an empty context
g = rdflib.ConjunctiveGraph()
cg_1 = g.get_context(ctxid_1)
cg_2 = g.get_context(ctxid_2)
cg_imp = g.get_context(imports_ctxid)
with transaction.manager:
cg_1.add((aURI('a'), aURI('b'), aURI('c')))
cg_2.add((aURI('d'), aURI('e'), aURI('f')))
cg_imp.add((URIRef(ctxid_1), CONTEXT_IMPORTS, URIRef(ctxid_2)))
bi = Installer(*dirs, imports_ctx=imports_ctxid, graph=g)
bi.install(dep_d)
bi.install(d)
def test_dependency_version_in_manifest_without_spec(dirs):
'''
It is permitted to not specify the version of a bundle dependency in the descriptor,
but we must pin a specific version of the bundle in the manifest.
'''
ctxid_1 = 'http://example.org/ctx1'
ctxid_2 = 'http://example.org/ctx2'
# Make a descriptor that includes ctx1 and the imports, but not ctx2
d = Descriptor('test')
d.includes.add(make_include_func(ctxid_1))
d.dependencies.add(DependencyDescriptor('dep'))
dep_d = Descriptor('dep')
dep_d.includes.add(make_include_func(ctxid_2))
# Add some triples so the contexts aren't empty -- we can't save an empty context
g = rdflib.ConjunctiveGraph()
cg_1 = g.get_context(ctxid_1)
cg_2 = g.get_context(ctxid_2)
cg_1.add((aURI('a'), aURI('b'), aURI('c')))
cg_2.add((aURI('d'), aURI('e'), aURI('f')))
bi = Installer(*dirs, graph=g)
bi.install(dep_d)
bi.install(d)
test_bnd = Bundle('test', bundles_directory=dirs.bundles_directory)
assert test_bnd.manifest_data['dependencies'][0]['version'] == 1
def bundle_helper(descriptor,
graph=None,
bundles_directory=None,
homedir=None,
**kwargs):
'''
Helper for creating bundles for testing.
Uses `~owmeta_core.bundle.Installer` to lay out a bundle
Parameters
----------
descriptor : Descriptor
Describes the bundle
graph : rdflib.graph.ConjunctiveGraph, optional
Graph from which the bundle contexts will be generated. If not provided, a graph
will be created with the triple ``(ex:a, ex:b, ex:c)`` in a context named ``ex:ctx``,
where ``ex:`` expands to ``http://example.org/``
bundles_directory : str, optional
The directory where the bundles should be installed. If not provided, creates a
temporary directory to house the bundles and cleans them up afterwards
homedir : str, optional
Test home directory. If not provided, one will be created based on test directory
'''
res = BundleData()
with tempfile.TemporaryDirectory(prefix=__name__ + '.') as testdir:
res.testdir = testdir
res.test_homedir = homedir or p(res.testdir, 'homedir')
res.bundle_source_directory = p(res.testdir, 'bundle_source')
res.bundles_directory = bundles_directory or p(res.test_homedir,
'.owmeta', 'bundles')
if not homedir:
os.mkdir(res.test_homedir)
os.mkdir(res.bundle_source_directory)
if not bundles_directory:
os.makedirs(res.bundles_directory)
# This is a bit of an integration test since it would be a PITA to maintain the bundle
# format separately from the installer
res.descriptor = descriptor
if graph is None:
graph = ConjunctiveGraph()
ctxg = graph.get_context(URIRef('http://example.org/ctx'))
ctxg.add((URIRef('http://example.org/a'),
URIRef('http://example.org/b'),
URIRef('http://example.org/c')))
res.installer = Installer(res.bundle_source_directory,
res.bundles_directory,
graph=graph,
**kwargs)
res.bundle_directory = res.installer.install(res.descriptor)
yield res
def test_multiple_context_hash(dirs):
d = Descriptor('test')
ctxid_1 = 'http://example.org/ctx1'
ctxid_2 = 'http://example.org/ctx2'
d.includes.add(make_include_func(ctxid_1))
d.includes.add(make_include_func(ctxid_2))
g = rdflib.ConjunctiveGraph()
cg = g.get_context(ctxid_1)
with transaction.manager:
cg.add((aURI('a'), aURI('b'), aURI('c')))
cg = g.get_context(ctxid_2)
with transaction.manager:
cg.add((aURI('a'), aURI('b'), aURI('c')))
bi = Installer(*dirs, graph=g)
bi.install(d)
with open(p(dirs.bundles_directory, 'test', '1', 'graphs', 'hashes'),
'rb') as f:
contents = f.read()
assert ctxid_1.encode('UTF-8') in contents
assert ctxid_2.encode('UTF-8') in contents
def test_bundle_install_directory(dirs):
d = Descriptor('test')
bi = Installer(*dirs, graph=rdflib.ConjunctiveGraph())
bi.install(d)
assert isdir(p(dirs.bundles_directory, 'test', '1'))
def test_imports_in_unfetched_dependencies(dirs):
'''
If we have imports and a dependency includes the context, then we shouldn't have an
error.
Versioned bundles are assumed to be immutable, so we won't re-fetch a bundle already
in the local index
'''
imports_ctxid = 'http://example.org/imports'
ctxid_1 = 'http://example.org/ctx1'
ctxid_2 = 'http://example.org/ctx2'
# Make a descriptor that includes ctx1 and the imports, but not ctx2
d = Descriptor('test')
d.includes.add(make_include_func(ctxid_1))
d.includes.add(make_include_func(imports_ctxid))
d.dependencies.add(DependencyDescriptor('dep'))
dep_d = Descriptor('dep')
dep_d.includes.add(make_include_func(ctxid_2))
# Add some triples so the contexts aren't empty -- we can't save an empty context
g = rdflib.ConjunctiveGraph()
cg_1 = g.get_context(ctxid_1)
cg_2 = g.get_context(ctxid_2)
cg_imp = g.get_context(imports_ctxid)
cg_1.add((URIRef('http://example.com/a'), URIRef('http://example.com/b'),
URIRef('http://example.com/c')))
cg_2.add((URIRef('http://example.com/d'), URIRef('http://example.com/e'),
URIRef('http://example.com/f')))
cg_imp.add((URIRef(ctxid_1), CONTEXT_IMPORTS, URIRef(ctxid_2)))
class loader_class(object):
def __init__(self, *args):
self.bi = None
def can_load(self, *args):
return True
def can_load_from(self, *args):
return True
def bundle_versions(self, *args):
return [1]
def __call__(self, *args):
self.bi.install(dep_d)
loader = loader_class()
class remote_class(Remote):
def generate_loaders(self, *args):
yield loader
bi = Installer(*dirs,
imports_ctx=imports_ctxid,
graph=g,
remotes=[remote_class('remote')])
loader.bi = bi
with patch('owmeta_core.bundle.LOADER_CLASSES', (loader_class, )):
bi.install(d)