def test_dataset_shacl(tmpdir, runner, project, client):
"""Test dataset metadata structure."""
force_dataset_path = Path(
__file__
).parent.parent.parent / 'fixtures' / 'force_dataset_shacl.json'
force_datasetfile_path = Path(
__file__
).parent.parent.parent / 'fixtures' / 'force_datasetfile_shacl.json'
force_datasettag_path = Path(
__file__
).parent.parent.parent / 'fixtures' / 'force_datasettag_shacl.json'
runner.invoke(cli, ['dataset', 'create', 'dataset'])
paths = []
for i in range(3):
new_file = tmpdir.join('file_{0}'.format(i))
new_file.write(str(i))
paths.append(str(new_file))
# add data
runner.invoke(
cli,
['dataset', 'add', 'dataset'] + paths,
catch_exceptions=False,
)
runner.invoke(
cli,
['dataset', 'tag', 'dataset', '1.0'],
catch_exceptions=False,
)
with client.with_dataset('dataset') as dataset:
g = dataset.asjsonld()
rdf = pyld.jsonld.to_rdf(g,
options={
'format': 'application/n-quads',
'produceGeneralizedRdf': True
})
r, _, t = validate_graph(rdf, shacl_path=str(force_dataset_path))
assert r is True, t
r, _, t = validate_graph(rdf, shacl_path=str(force_datasetfile_path))
assert r is True, t
r, _, t = validate_graph(rdf, shacl_path=str(force_datasettag_path))
assert r is True, t
r, _, t = validate_graph(rdf)
assert r is True, t
def rdf(graph, strict=False):
"""Output the graph as RDF."""
xml = _conjunctive_graph(graph).serialize(format="application/rdf+xml")
if strict:
r, _, t = validate_graph(xml, format="xml")
if not r:
raise SHACLValidationError("{}\nCouldn't get log: Invalid Knowledge Graph data".format(t))
click.echo(xml)
def nt(graph, strict=False):
"""Format graph as n-tuples."""
nt = _conjunctive_graph(graph).serialize(format="nt")
if strict:
r, _, t = validate_graph(nt, format="nt")
if not r:
raise SHACLValidationError("{}\nCouldn't get log: Invalid Knowledge Graph data".format(t))
click.echo(nt)
def jsonld(graph, strict=False):
"""Format graph as JSON-LD file."""
ld = _jsonld(graph, "expand")
if strict:
r, _, t = validate_graph(ld, format="json-ld")
if not r:
raise SHACLValidationError("{}\nCouldn't get log: Invalid Knowledge Graph data".format(t))
click.echo(ld)
def check_shacl_structure(path):
"""Validates all metadata aginst the SHACL schema."""
with path.open(mode='r') as fp:
source = yaml.load(fp, Loader=NoDatesSafeLoader) or {}
rdf = pyld.jsonld.to_rdf(source,
options={
'format': 'application/n-quads',
'produceGeneralizedRdf': True
})
return validate_graph(rdf)
def test_project_shacl(project, client):
"""Test project metadata structure."""
from renku.core.models.provenance.agents import Person
path = Path(
__file__
).parent.parent.parent / 'fixtures' / 'force_project_shacl.json'
project = client.project
project.creator = Person(email='*****@*****.**', name='Johnny Doe')
g = project.as_jsonld()
rdf = pyld.jsonld.to_rdf(g,
options={
'format': 'application/n-quads',
'produceGeneralizedRdf': False
})
r, _, t = validate_graph(rdf, shacl_path=str(path))
assert r is True, t
r, _, t = validate_graph(rdf)
assert r is True, t
def test_update(runner, project, run):
"""Test automatic file update."""
from renku.core.utils.shacl import validate_graph
cwd = Path(project)
data = cwd / 'data'
data.mkdir(exist_ok=True, parents=True)
source = cwd / 'source.txt'
output = data / 'result.txt'
repo = git.Repo(project)
update_and_commit('1', source, repo)
assert 0 == run(args=('run', 'wc', '-c'), stdin=source, stdout=output)
with output.open('r') as f:
assert f.read().strip() == '1'
result = runner.invoke(cli, ['status'])
assert 0 == result.exit_code
update_and_commit('12', source, repo)
result = runner.invoke(cli, ['status'])
assert 1 == result.exit_code
assert 0 == run()
result = runner.invoke(cli, ['status'])
assert 0 == result.exit_code
with output.open('r') as f:
assert f.read().strip() == '2'
result = runner.invoke(cli, ['log'], catch_exceptions=False)
assert '(part of' in result.output, result.output
# Source has been updated but output is unchanged.
update_and_commit('34', source, repo)
result = runner.invoke(cli, ['status'])
assert 1 == result.exit_code
assert 0 == run()
result = runner.invoke(cli, ['status'])
assert 0 == result.exit_code
with output.open('r') as f:
assert f.read().strip() == '2'
from renku.cli.log import FORMATS
for output_format in FORMATS:
# Make sure the log contains the original parent.
result = runner.invoke(
cli,
['log', '--format', output_format],
catch_exceptions=False,
)
assert 0 == result.exit_code, result.output
assert source.name in result.output, output_format
if output_format == 'nt':
r, _, t = validate_graph(result.output, format='nt')
assert r is True, t
def test_update(runner, project, renku_cli, no_lfs_warning):
"""Test automatic file update."""
from renku.core.utils.shacl import validate_graph
cwd = Path(project)
data = cwd / DATA_DIR
data.mkdir(exist_ok=True, parents=True)
source = cwd / "source.txt"
output = data / "result.txt"
repo = git.Repo(project)
update_and_commit("1", source, repo)
exit_code, run = renku_cli("run", "wc", "-c", stdin=source, stdout=output)
assert 0 == exit_code
assert 0 == len(run.subprocesses)
previous_run_id = run._id
with output.open("r") as f:
assert f.read().strip() == "1"
result = runner.invoke(cli, ["status"])
assert 0 == result.exit_code
update_and_commit("12", source, repo)
result = runner.invoke(cli, ["status"])
assert 1 == result.exit_code
exit_code, run = renku_cli("update")
assert 0 == exit_code
assert 0 == len(run.subprocesses)
assert previous_run_id == run._id
previous_run_id = run._id
result = runner.invoke(cli, ["status"])
assert 0 == result.exit_code
with output.open("r") as f:
assert f.read().strip() == "2"
# Source has been updated but output is unchanged.
update_and_commit("34", source, repo)
result = runner.invoke(cli, ["status"])
assert 1 == result.exit_code
exit_code, run = renku_cli("update")
assert 0 == exit_code
assert 0 == len(run.subprocesses)
assert previous_run_id == run._id
result = runner.invoke(cli, ["status"])
assert 0 == result.exit_code
with output.open("r") as f:
assert f.read().strip() == "2"
from renku.cli.log import FORMATS
for output_format in FORMATS:
# Make sure the log contains the original parent.
result = runner.invoke(
cli,
["log", "--format", output_format],
catch_exceptions=False,
)
assert 0 == result.exit_code, result.output
assert source.name in result.output, output_format
if output_format == "nt":
r, _, t = validate_graph(result.output, format="nt")
assert r is True, t
