def test_no_zip_in_zip(test_data_dir, tmpdir):
crate_dir = test_data_dir / 'ro-crate-galaxy-sortchangecase'
crate = ROCrate(crate_dir)
zip_name = 'ro_crate_out.crate.zip'
zip_path = crate_dir / zip_name # within the crate dir
crate.write_zip(zip_path)
out_path = tmpdir / 'ro_crate_out'
with zipfile.ZipFile(zip_path, "r") as zf:
zf.extractall(out_path)
assert not (out_path / zip_name).exists()
def test_bad_crate(test_data_dir, tmpdir):
# nonexistent dir
crate_dir = test_data_dir / uuid.uuid4().hex
with pytest.raises(ValueError):
ROCrate(crate_dir)
with pytest.raises(ValueError):
ROCrate(crate_dir, init=True)
# no metadata file
crate_dir = tmpdir / uuid.uuid4().hex
crate_dir.mkdir()
with pytest.raises(ValueError):
ROCrate(crate_dir)
def test_init(test_data_dir, tmpdir, helpers, override):
crate_dir = test_data_dir / "ro-crate-galaxy-sortchangecase"
if not override:
(crate_dir / helpers.METADATA_FILE_NAME).unlink()
crate = ROCrate(crate_dir, init=True)
assert crate.dereference("./") is not None
assert crate.dereference(helpers.METADATA_FILE_NAME) is not None
fpaths = [
"LICENSE", "README.md", "sort-and-change-case.ga",
"test/test1/input.bed", "test/test1/output_exp.bed",
"test/test1/sort-and-change-case-test.yml"
]
dpaths = [
"test/",
"test/test1/",
]
for p in fpaths:
assert isinstance(crate.dereference(p), File)
for p in dpaths:
assert isinstance(crate.dereference(p), Dataset)
out_path = tmpdir / 'ro_crate_out'
out_path.mkdir()
crate.write_crate(out_path)
assert (out_path / helpers.METADATA_FILE_NAME).exists()
json_entities = helpers.read_json_entities(out_path)
data_entity_ids = fpaths + dpaths
helpers.check_crate(json_entities, data_entity_ids=data_entity_ids)
for p in fpaths:
with open(crate_dir / p) as f1, open(out_path / p) as f2:
assert f1.read() == f2.read()
def test_legacy_crate(test_data_dir, tmpdir, helpers):
crate_dir = test_data_dir / 'read_crate'
# Remove the metadata file, leaving only the legacy one
(crate_dir / helpers.METADATA_FILE_NAME).unlink()
crate = ROCrate(crate_dir)
md_prop = crate.metadata.properties()
assert crate.dereference(
helpers.LEGACY_METADATA_FILE_NAME) is crate.metadata
assert md_prop['conformsTo'] == {'@id': helpers.LEGACY_PROFILE}
main_wf = crate.dereference('test_galaxy_wf.ga')
wf_prop = main_wf.properties()
assert set(wf_prop['@type']) == helpers.LEGACY_WORKFLOW_TYPES
def main(args):
wd = pathlib.Path(tempfile.mkdtemp(prefix="life_monitor_"))
crate_dir = wd / pathlib.Path(args.crate_dir).name
shutil.copytree(args.crate_dir, crate_dir)
crate = ROCrate(crate_dir)
check_workflow(crate, crate_dir)
shutil.rmtree(wd)
def test_delete_refs(test_data_dir, tmpdir, helpers):
def_path = "test/test1/sort-and-change-case-test.yml"
crate = ROCrate(test_data_dir / 'ro-crate-galaxy-sortchangecase')
suite = crate.dereference("#test1")
definition = crate.dereference(def_path)
assert suite.definition is definition
crate.delete(definition)
assert suite.definition is not definition # so far, so good
assert suite.definition == str(def_path) # should probably be set to None
crate.write_crate("/tmp/crate_out")
# check json output
out_path = tmpdir / "ro_crate_out"
crate.write_crate(out_path)
json_entities = helpers.read_json_entities(out_path)
assert def_path not in json_entities # good
assert json_entities["#test1"]["definition"]["@id"] == def_path # not good
def test_remote_uri(tmpdir, helpers, fetch_remote):
crate = ROCrate()
url = ('https://raw.githubusercontent.com/ResearchObject/ro-crate-py/'
'master/test/test-data/sample_file.txt')
file_returned = crate.add_file(source=url, fetch_remote=fetch_remote)
if fetch_remote:
assert file_returned.id == 'sample_file.txt'
else:
assert file_returned.id == url
out_path = tmpdir / 'ro_crate_out'
out_path.mkdir()
crate.write_crate(out_path)
metadata_path = out_path / helpers.METADATA_FILE_NAME
assert metadata_path.exists()
file1 = out_path / 'sample_file.txt'
if fetch_remote:
assert file1.exists()
def test_file_stringio(tmpdir, helpers):
crate = ROCrate()
test_file_id = 'test_file.txt'
file_content = 'This will be the content of the file'
file_stringio = io.StringIO(file_content)
file_returned = crate.add_file(file_stringio, test_file_id)
assert file_returned.id == test_file_id
out_path = tmpdir / 'ro_crate_out'
out_path.mkdir()
crate.write_crate(out_path)
metadata_path = out_path / helpers.METADATA_FILE_NAME
assert metadata_path.exists()
file1 = out_path / test_file_id
assert file1.exists()
with open(file1) as f:
assert f.read() == file_content
def test_dataset(test_data_dir, tmpdir):
crate = ROCrate()
path = test_data_dir / "a" / "b"
d1 = crate.add_dataset(str(path))
assert crate.dereference("b") is d1
d2 = crate.add_dataset(str(path), "a/b")
assert crate.dereference("a/b") is d2
out_path = tmpdir / 'ro_crate_out'
out_path.mkdir()
crate.write_crate(out_path)
assert (out_path / "b").is_dir()
assert (out_path / "a" / "b").is_dir()
def test_read(test_data_dir, helpers):
crate_dir = test_data_dir / 'ro-crate-galaxy-sortchangecase'
crate = ROCrate(crate_dir)
wf_id = 'sort-and-change-case.ga'
main_wf = crate.dereference(wf_id)
wf_prop = main_wf.properties()
assert wf_prop['@id'] == wf_id
assert wf_prop['@id'] == main_wf.id
assert set(wf_prop['@type']) == helpers.WORKFLOW_TYPES
test_service = crate.dereference(JENKINS)
assert test_service.id == JENKINS
assert test_service.type == "TestService"
assert test_service.name == "Jenkins"
assert test_service.url == "https://www.jenkins.io"
test_instance = crate.dereference("#test1_1")
assert test_instance.id == "#test1_1"
assert test_instance.type == "TestInstance"
assert test_instance.name == "test1_1"
assert test_instance.url == "http://example.org/jenkins"
assert test_instance.resource == "job/tests/"
assert test_instance.runsOn is test_service
assert test_instance.service is test_service
test_engine = crate.dereference(PLANEMO)
assert test_engine.id == PLANEMO
assert test_engine.type == "SoftwareApplication"
assert test_engine.name == "Planemo"
assert test_engine.url == "https://github.com/galaxyproject/planemo"
def_id = "test/test1/sort-and-change-case-test.yml"
test_definition = crate.dereference(def_id)
assert test_definition.id == def_id
assert set(test_definition.type) == {"File", "TestDefinition"}
assert test_definition.conformsTo is test_engine
assert test_definition.engine is test_engine
assert test_definition.engineVersion == ">=0.70"
test_suite = crate.dereference("#test1")
assert test_suite.id == "#test1"
assert test_suite.type == "TestSuite"
assert test_suite.name == "test1"
assert len(test_suite.instance) == 1
assert test_suite.instance[0] is test_instance
assert test_suite.definition is test_definition
assert test_suite["mainEntity"] is main_wf
assert set(crate.test_suites) == {test_suite}
def test_no_parts(tmpdir):
crate = ROCrate()
out_path = tmpdir / 'ro_crate_out'
out_path.mkdir()
crate.write_crate(out_path)
crate = ROCrate(out_path)
assert not crate.root_dataset["hasPart"]
def test_dereferencing(test_data_dir, helpers):
crate = ROCrate(gen_preview=True)
# verify default entities
root_dataset = crate.dereference('./')
assert crate.root_dataset is root_dataset
metadata_entity = crate.dereference(helpers.METADATA_FILE_NAME)
assert crate.metadata is metadata_entity
preview_entity = crate.dereference(helpers.PREVIEW_FILE_NAME)
assert preview_entity is crate.preview
# dereference added files
sample_file = test_data_dir / 'sample_file.txt'
file_returned = crate.add_file(sample_file)
assert isinstance(file_returned, File)
dereference_file = crate.dereference("sample_file.txt")
assert file_returned is dereference_file
readme_url = f'{RAW_REPO_URL}/master/README.md'
readme_entity = crate.add_file(readme_url)
assert crate.dereference(readme_url) is readme_entity
def test_init_preview(test_data_dir, tmpdir, helpers, gen_preview,
preview_exists):
crate_dir = test_data_dir / "ro-crate-galaxy-sortchangecase"
dummy_prev_content = "foo\nbar\n"
if preview_exists:
with open(crate_dir / helpers.PREVIEW_FILE_NAME, "wt") as f:
f.write(dummy_prev_content)
crate = ROCrate(crate_dir, gen_preview=gen_preview, init=True)
prev = crate.dereference(helpers.PREVIEW_FILE_NAME)
if gen_preview or preview_exists:
assert prev is not None
out_path = tmpdir / 'ro_crate_out'
out_path.mkdir()
crate.write_crate(out_path)
out_prev_path = out_path / helpers.PREVIEW_FILE_NAME
if gen_preview or preview_exists:
assert out_prev_path.is_file()
else:
assert not out_prev_path.exists()
if not gen_preview and preview_exists:
assert out_prev_path.open().read() == dummy_prev_content
def test_add_test_definition(test_data_dir, engine, engine_version):
top_dir = test_data_dir / "ro-crate-galaxy-sortchangecase"
wf_path = top_dir / "sort-and-change-case.ga"
def_path = top_dir / "test" / "test1" / "sort-and-change-case-test.yml"
crate = ROCrate()
wf = crate.add(ComputationalWorkflow(crate, str(wf_path), wf_path.name))
crate.mainEntity = wf
suite = crate.add_test_suite()
assert crate.dereference(PLANEMO) is None
kwargs = {"source": def_path}
if engine:
kwargs["engine"] = engine
if engine_version:
kwargs["engine_version"] = engine_version
d = crate.add_test_definition(suite, **kwargs)
assert crate.dereference(PLANEMO) is d.engine
if engine_version:
assert d.engineVersion == engine_version
def test_dereferencing_equivalent_id(self):
crate = ROCrate()
root_dataset = crate.dereference('./')
self.assertEqual(crate.root_dataset, root_dataset)
root_dataset = crate.dereference('')
self.assertEqual(crate.root_dataset, root_dataset)
metadata_entity = crate.dereference('./ro-crate-metadata.jsonld')
self.assertEqual(metadata_entity, crate.metadata)
metadata_entity = crate.dereference('ro-crate-metadata.jsonld')
self.assertEqual(metadata_entity, crate.metadata)
def test_properties(self):
crate = ROCrate()
new_person = crate.add_person('001', {'name': 'Lee Ritenour'})
crate.creator = new_person
self.assertIsInstance(crate.creator, Person)
self.assertEqual(crate.creator['name'], 'Lee Ritenour')
new_person2 = crate.add_person('002', {'name': 'Lee Ritenour'})
crate.creator = [new_person, new_person2]
self.assertIsInstance(crate.creator, list)
def test_dereferencing_equivalent_id(helpers):
crate = ROCrate()
root_dataset = crate.dereference('./')
assert crate.root_dataset is root_dataset
root_dataset = crate.dereference('')
assert crate.root_dataset is root_dataset
metadata_entity = crate.dereference(helpers.METADATA_FILE_NAME)
assert crate.metadata is metadata_entity
metadata_entity = crate.dereference(f'./{helpers.METADATA_FILE_NAME}')
assert crate.metadata is metadata_entity
def test_missing_source(test_data_dir, fetch_remote, validate_url):
crate = ROCrate()
path = test_data_dir / uuid.uuid4().hex
with pytest.raises(ValueError):
crate.add_file(str(path),
fetch_remote=fetch_remote,
validate_url=validate_url)
with pytest.raises(ValueError):
crate.add_file(str(path),
path.name,
fetch_remote=fetch_remote,
validate_url=validate_url)
def main():
wd = pathlib.Path(tempfile.mkdtemp(prefix="ro_crate_py_"))
crate_dir = wd / RO_CRATE_DIR.name
shutil.copytree(RO_CRATE_DIR, crate_dir)
crate = ROCrate(crate_dir)
print_suites(crate)
main_workflow = crate.root_dataset["mainEntity"]
print("main workflow:", main_workflow.id)
try:
exe = subprocess.check_output("command -v planemo",
shell=True,
universal_newlines=True).strip()
except subprocess.CalledProcessError:
print("planemo executable not found, won't try to run tests")
return
else:
print("planemo executable:", exe)
# run a test suite
suite = crate.test_suites[0]
def_path = crate_dir / suite.definition.id
workflow = suite["mainEntity"]
workflow_path = crate_dir / workflow.id
print("running suite:", suite.id)
print("definition path:", def_path)
print("workflow:", workflow.id)
assert suite.definition.engine.id == PLANEMO
new_workflow_path = str(def_path.parent / workflow_path.name)
# Planemo expects the test definition in the same dir as the workflow file
shutil.copy2(workflow_path, new_workflow_path)
cmd = [
"planemo", "test", "--engine", "docker_galaxy",
"--docker_galaxy_image", GALAXY_IMG, new_workflow_path
]
print("Running Planemo (this may take a while)")
p = subprocess.run(cmd)
p.check_returncode()
print("OK")
shutil.rmtree(wd)
def test_dereferencing(test_data_dir, helpers):
crate = ROCrate()
# verify default entities
root_dataset = crate.dereference('./')
assert crate.root_dataset is root_dataset
metadata_entity = crate.dereference(helpers.METADATA_FILE_NAME)
assert crate.metadata is metadata_entity
preview_entity = crate.dereference(helpers.PREVIEW_FILE_NAME)
assert preview_entity is crate.preview
# dereference added files
sample_file = test_data_dir / 'sample_file.txt'
file_returned = crate.add_file(sample_file)
assert isinstance(file_returned, File)
dereference_file = crate.dereference("sample_file.txt")
assert file_returned is dereference_file
def test_dereferencing(self):
crate = ROCrate()
# verify default entities
root_dataset = crate.dereference('./')
self.assertEqual(crate.root_dataset, root_dataset)
metadata_entity = crate.dereference('./ro-crate-metadata.jsonld')
self.assertEqual(metadata_entity, crate.metadata)
# dereference added files
sample_file = self.test_data_dir / 'sample_file.txt'
file_returned = crate.add_file(sample_file)
self.assertIsInstance(file_returned, File)
dereference_file = crate.dereference("sample_file.txt")
self.assertIsInstance(dereference_file, File)
self.assertEqual(file_returned, dereference_file)
def test_extra_data(test_data_dir, tmpdir, to_zip):
crate_dir = test_data_dir / 'read_extra'
crate = ROCrate(crate_dir)
out_path = tmpdir / 'read_extra_out'
if to_zip:
zip_path = tmpdir / 'ro_crate_out.crate.zip'
crate.write_zip(zip_path)
with zipfile.ZipFile(zip_path, "r") as zf:
zf.extractall(out_path)
else:
out_path.mkdir()
crate.write_crate(out_path)
for rel in {
"listed.txt",
"listed/listed.txt",
"listed/not_listed.txt",
"not_listed.txt",
"not_listed/not_listed.txt",
}:
assert (out_path / rel).is_file()
with open(crate_dir / rel) as f1, open(out_path / rel) as f2:
assert f1.read() == f2.read()
def test_crate_dir_loading(self):
# load crate from directory
crate_dir = self.test_data_dir / 'read_crate'
crate = ROCrate(crate_dir, load_preview=True)
# check loaded entities and properties
main_wf = crate.dereference('test_galaxy_wf.ga')
wf_prop = main_wf.properties()
self.assertEqual(wf_prop['@id'], 'test_galaxy_wf.ga')
wf_author = crate.dereference('#joe')
author_prop = wf_author.properties()
self.assertEqual(author_prop['@type'], 'Person')
self.assertEqual(author_prop['name'], 'Joe Bloggs')
# write the crate in a different directory
out_path = pathlib.Path(tempfile.gettempdir()) / 'crate_read_out'
out_path.mkdir(exist_ok=True)
crate.write_crate(out_path)
metadata_path = out_path / 'ro-crate-metadata.jsonld'
self.assertTrue(metadata_path.exists())
def test_delete(test_data_dir):
crate = ROCrate()
# fundamental entities
with pytest.raises(ValueError):
crate.delete(crate.root_dataset)
with pytest.raises(ValueError):
crate.delete(crate.metadata)
# preview
preview = crate.add(Preview(crate))
assert preview in crate.default_entities
crate.delete(preview)
assert preview not in crate.default_entities
assert crate.preview is None
# data entities
file1 = crate.add_file(test_data_dir / "sample_file.txt")
file2 = crate.add_file(test_data_dir / "sample_cwl_wf.cwl")
for f in file1, file2:
assert f in crate.root_dataset["hasPart"]
assert f in crate.data_entities
crate.delete(file1)
assert file1 not in crate.data_entities
assert file1 not in crate.root_dataset["hasPart"]
crate.delete(file2)
assert file2 not in crate.data_entities
assert crate.root_dataset["hasPart"] is None
crate.delete(file2) # no-op
# contextual entities
john = crate.add(Person(crate, '#john', {'name': 'John Doe'}))
assert john in crate.contextual_entities
crate.delete(john)
assert john not in crate.contextual_entities
crate.delete(john) # no-op
def test_crate_dir_loading(test_data_dir, tmpdir, helpers, load_preview,
from_zip):
# load crate
crate_dir = test_data_dir / 'read_crate'
if from_zip:
zip_source = shutil.make_archive(tmpdir / "read_crate.crate", "zip",
crate_dir)
crate = ROCrate(zip_source, load_preview=load_preview)
else:
crate = ROCrate(crate_dir, load_preview=load_preview)
# check loaded entities and properties
root = crate.dereference('./')
assert crate.root_dataset is root
root_prop = root.properties()
assert root_prop['@id'] == root.id
assert root_prop['@id'] == './'
assert root_prop['@type'] == 'Dataset'
metadata = crate.dereference(helpers.METADATA_FILE_NAME)
assert crate.metadata is metadata
md_prop = metadata.properties()
assert md_prop['@id'] == metadata.id
assert md_prop['@id'] == helpers.METADATA_FILE_NAME
assert md_prop['@type'] == 'CreativeWork'
assert md_prop['about'] == {'@id': './'}
# conformsTo is currently hardcoded in the Metadata class, not read from the crate
assert md_prop['conformsTo'] == {'@id': helpers.PROFILE}
assert metadata.root is root
preview = crate.dereference(helpers.PREVIEW_FILE_NAME)
assert preview == crate.preview
preview_prop = preview.properties()
assert preview_prop['@id'] == helpers.PREVIEW_FILE_NAME
assert preview_prop['@id'] == preview.id
assert preview_prop['@type'] == 'CreativeWork'
assert preview_prop['about'] == {'@id': './'}
if load_preview:
assert Path(preview.source).name == helpers.PREVIEW_FILE_NAME
else:
assert not preview.source
main_wf = crate.dereference('test_galaxy_wf.ga')
wf_prop = main_wf.properties()
assert wf_prop['@id'] == 'test_galaxy_wf.ga'
assert wf_prop['@id'] == main_wf.id
assert set(wf_prop['@type']) == helpers.WORKFLOW_TYPES
assert wf_prop['programmingLanguage'] == {
'@id': 'https://galaxyproject.org'
}
assert wf_prop['subjectOf'] == {'@id': 'abstract_wf.cwl'}
abs_wf = crate.dereference('abstract_wf.cwl')
abs_wf_prop = abs_wf.properties()
assert abs_wf_prop['@id'] == 'abstract_wf.cwl'
assert abs_wf_prop['@id'] == abs_wf.id
assert set(abs_wf_prop['@type']) == helpers.WORKFLOW_TYPES
wf_author = crate.dereference('#joe')
author_prop = wf_author.properties()
assert author_prop['@id'] == '#joe'
assert author_prop['@id'] == wf_author.id
assert author_prop['@type'] == 'Person'
assert author_prop['name'] == 'Joe Bloggs'
test_file = crate.dereference('test_file_galaxy.txt')
test_file_prop = test_file.properties()
assert test_file_prop['@id'] == 'test_file_galaxy.txt'
assert test_file_prop['@id'] == test_file.id
remote_file = crate.dereference(_URL)
remote_file_prop = remote_file.properties()
assert remote_file_prop['@id'] == _URL
assert remote_file_prop['@id'] == remote_file.id
examples_dataset = crate.dereference('examples/')
examples_dataset_prop = examples_dataset.properties()
assert examples_dataset_prop['@id'] == 'examples/'
assert examples_dataset_prop['@id'] == examples_dataset.id
assert crate.examples_dir is examples_dataset
test_dataset = crate.dereference('test/')
test_dataset_prop = test_dataset.properties()
assert test_dataset_prop['@id'] == 'test/'
assert test_dataset_prop['@id'] == test_dataset.id
assert crate.test_dir is test_dataset
assert crate.test_metadata_path == Path("test") / TEST_METADATA_BASENAME
# write the crate in a different directory
out_path = tmpdir / 'crate_read_out'
out_path.mkdir()
crate.write_crate(out_path)
metadata_path = out_path / helpers.METADATA_FILE_NAME
assert metadata_path.exists()
legacy_metadata_path = out_path / helpers.LEGACY_METADATA_FILE_NAME
assert not legacy_metadata_path.exists()
preview_path = out_path / helpers.PREVIEW_FILE_NAME
assert preview_path.exists()
if load_preview:
with open(preview.source) as f1, open(preview_path) as f2:
assert f1.read() == f2.read()
json_entities = helpers.read_json_entities(out_path)
data_entity_ids = [main_wf.id, abs_wf.id, test_file.id, remote_file.id]
helpers.check_crate(json_entities, data_entity_ids=data_entity_ids)
for e in main_wf, abs_wf, test_file:
with open(e.source) as f1, open(out_path / e.id) as f2:
assert f1.read() == f2.read()
