def test_load_with_archive_filepath_modified(self):
# Save an artifact for use in the following test case.
fp = os.path.join(self.test_dir.name, 'artifact.qza')
Artifact._from_view(FourInts, [-1, 42, 0, 43], list).save(fp)
# Load the artifact from a filepath then save a different artifact to
# the same filepath. Assert that both artifacts produce the correct
# views of their data.
#
# `load` used to be lazy, only extracting data when it needed to (e.g.
# when `save` or `view` was called). This was buggy as the filepath
# could have been deleted, or worse, modified to contain a different
# .qza file. Thus, the wrong archive could be extracted on demand, or
# the archive could be missing altogether. There isn't an easy
# cross-platform compatible way to solve this problem, so Artifact.load
# is no longer lazy and always extracts its data immediately. The real
# motivation for lazy loading was for quick inspection of archives
# without extracting/copying data, so that API is now provided through
# Artifact.peek.
artifact1 = Artifact.load(fp)
Artifact._from_view(FourInts, [10, 11, 12, 13], list).save(fp)
artifact2 = Artifact.load(fp)
self.assertEqual(artifact1.view(list), [-1, 42, 0, 43])
self.assertEqual(artifact2.view(list), [10, 11, 12, 13])
def test_call_with_no_parameters(self):
merge_mappings = self.plugin.methods['merge_mappings']
artifact1 = Artifact._from_view({'foo': 'abc', 'bar': 'def'}, Mapping,
None)
artifact2 = Artifact._from_view({'bazz': 'abc'}, Mapping, None)
result = merge_mappings(artifact1, artifact2)
self.assertIsInstance(result, Artifact)
self.assertEqual(result.type, Mapping)
provenance = result.provenance
self.assertIsInstance(provenance, Provenance)
self.assertIsInstance(provenance.execution_uuid, uuid.UUID)
self.assertTrue(
provenance.executor_reference.startswith(merge_mappings.id))
self.assertEqual(provenance.artifact_uuids, {
'mapping1': artifact1.uuid,
'mapping2': artifact2.uuid
})
self.assertEqual(provenance.parameter_references, {})
self.assertIsInstance(result.uuid, uuid.UUID)
self.assertEqual(result.view(dict),
{'foo': 'abc', 'bar': 'def', 'bazz': 'abc'})
def test_call_with_no_parameters(self):
merge_mappings = self.plugin.methods['merge_mappings']
artifact1 = Artifact._from_view(Mapping, {'foo': 'abc', 'bar': 'def'},
dict, None)
artifact2 = Artifact._from_view(Mapping, {'bazz': 'abc'},
dict, None)
result = merge_mappings(artifact1, artifact2)
self.assertIsInstance(result, Artifact)
self.assertEqual(result.type, Mapping)
provenance = result.provenance
self.assertIsInstance(provenance, Provenance)
self.assertIsInstance(provenance.execution_uuid, uuid.UUID)
self.assertTrue(
provenance.executor_reference.startswith(merge_mappings.id))
self.assertEqual(provenance.artifact_uuids, {
'mapping1': artifact1.uuid,
'mapping2': artifact2.uuid
})
self.assertEqual(provenance.parameter_references, {})
self.assertIsInstance(result.uuid, uuid.UUID)
self.assertEqual(result.view(dict),
{'foo': 'abc', 'bar': 'def', 'bazz': 'abc'})
def setUp(self):
# TODO standardize temporary directories created by QIIME
self.test_dir = tempfile.TemporaryDirectory(prefix='qiime2-temp-')
self.dummy_provenance = Provenance(
job_uuid='7e909a23-21e2-44c2-be17-0723fae91dc8',
artifact_uuids={
'input1': 'f16ca3d0-fe83-4b1e-8eea-7e35db3f6b0f',
'input2': '908dece5-db23-4562-ad03-876bb5750145',
},
parameters={
'param1': 'abc',
'param2': 100,
},
workflow_reference=(
'dummy workflow reference, see '
'https://github.com/biocore/qiime2/issues/26'
)
)
self.artifact_with_provenance = Artifact._from_view(
[-1, 42, 0, 43, 43], qiime.core.testing.TestType,
self.dummy_provenance)
self.artifact_without_provenance = Artifact._from_view(
[-1, 42, 0, 43, 43], qiime.core.testing.TestType, None)
def test_load_with_archive_filepath_modified(self):
# Save an artifact for use in the following test case.
fp = os.path.join(self.test_dir.name, 'artifact.qza')
Artifact._from_view([-1, 42, 0, 43], FourInts, None).save(fp)
# Load the artifact from a filepath then save a different artifact to
# the same filepath. Assert that both artifacts produce the correct
# views of their data.
#
# `load` used to be lazy, only extracting data when it needed to (e.g.
# when `save` or `view` was called). This was buggy as the filepath
# could have been deleted, or worse, modified to contain a different
# .qza file. Thus, the wrong archive could be extracted on demand, or
# the archive could be missing altogether. There isn't an easy
# cross-platform compatible way to solve this problem, so Artifact.load
# is no longer lazy and always extracts its data immediately. The real
# motivation for lazy loading was for quick inspection of archives
# without extracting/copying data, so that API is now provided through
# Artifact.peek.
artifact1 = Artifact.load(fp)
Artifact._from_view([10, 11, 12, 13], FourInts, None).save(fp)
artifact2 = Artifact.load(fp)
self.assertEqual(artifact1.view(list), [-1, 42, 0, 43])
self.assertEqual(artifact2.view(list), [10, 11, 12, 13])
def test_call_with_no_parameters(self):
merge_mappings = self.plugin.methods['merge_mappings']
artifact1 = Artifact._from_view(Mapping, {'foo': 'abc', 'bar': 'def'},
dict)
artifact2 = Artifact._from_view(Mapping, {'bazz': 'abc'}, dict)
result = merge_mappings(artifact1, artifact2)
# Test properties of the `Results` object.
self.assertIsInstance(result, tuple)
self.assertIsInstance(result, Results)
self.assertEqual(len(result), 1)
self.assertEqual(result.merged_mapping.view(dict),
{'foo': 'abc', 'bar': 'def', 'bazz': 'abc'})
result = result[0]
self.assertIsInstance(result, Artifact)
self.assertEqual(result.type, Mapping)
self.assertIsInstance(result.uuid, uuid.UUID)
self.assertEqual(result.view(dict),
{'foo': 'abc', 'bar': 'def', 'bazz': 'abc'})
def test_async(self):
concatenate_ints = self.plugin.methods['concatenate_ints']
concatenate_ints_markdown = \
self.plugin.methods['concatenate_ints_markdown']
artifact1 = Artifact._from_view(IntSequence1, [0, 42, 43],
list, None)
artifact2 = Artifact._from_view(IntSequence2, [99, -22],
list, None)
for method in concatenate_ints, concatenate_ints_markdown:
future = method.async(artifact1, artifact1, artifact2, 55, 1)
self.assertIsInstance(future, concurrent.futures.Future)
result = future.result()
self.assertIsInstance(result, Artifact)
self.assertEqual(result.type, IntSequence1)
provenance = result.provenance
self.assertIsInstance(provenance, Provenance)
self.assertIsInstance(provenance.execution_uuid, uuid.UUID)
self.assertTrue(
provenance.executor_reference.startswith(method.id))
self.assertEqual(provenance.artifact_uuids, {
'ints1': artifact1.uuid,
'ints2': artifact1.uuid,
'ints3': artifact2.uuid
})
self.assertEqual(provenance.parameter_references, {
'int1': '55',
'int2': '1'
})
self.assertIsInstance(result.uuid, uuid.UUID)
# Can retrieve multiple views of different type.
exp_list_view = [0, 42, 43, 0, 42, 43, 99, -22, 55, 1]
self.assertEqual(result.view(list), exp_list_view)
self.assertEqual(result.view(list), exp_list_view)
exp_counter_view = collections.Counter(
{0: 2, 42: 2, 43: 2, 99: 1, -22: 1, 55: 1, 1: 1})
self.assertEqual(result.view(collections.Counter),
exp_counter_view)
self.assertEqual(result.view(collections.Counter),
exp_counter_view)
# Accepts IntSequence1 | IntSequence2
artifact3 = Artifact._from_view(IntSequence2, [10, 20], list, None)
future = method.async(artifact3, artifact1, artifact2, 55, 1)
result = future.result()
self.assertEqual(result.type, IntSequence1)
self.assertEqual(result.view(list),
[10, 20, 0, 42, 43, 99, -22, 55, 1])
def test_async(self):
mapping_viz = self.plugin.visualizers['mapping_viz']
artifact1 = Artifact._from_view(Mapping, {'foo': 'abc', 'bar': 'def'},
dict, None)
artifact2 = Artifact._from_view(Mapping, {'baz': 'abc', 'bazz': 'ghi'},
dict, None)
future = mapping_viz.async(artifact1, artifact2, 'Key', 'Value')
self.assertIsInstance(future, concurrent.futures.Future)
result = future.result()
# Test properties of the `Results` object.
self.assertIsInstance(result, tuple)
self.assertIsInstance(result, Results)
self.assertEqual(len(result), 1)
self.assertEqual(result.visualization, result[0])
result = result[0]
self.assertIsInstance(result, Visualization)
self.assertEqual(result.type, qiime.core.type.Visualization)
provenance = result.provenance
self.assertIsInstance(provenance, Provenance)
self.assertIsInstance(provenance.execution_uuid, uuid.UUID)
self.assertTrue(
provenance.executor_reference.startswith(mapping_viz.id))
self.assertEqual(provenance.artifact_uuids, {
'mapping1': artifact1.uuid,
'mapping2': artifact2.uuid
})
self.assertEqual(provenance.parameter_references, {
'key_label': 'Key',
'value_label': 'Value'
})
self.assertIsInstance(result.uuid, uuid.UUID)
# TODO qiime.sdk.Visualization doesn't have an API to access its
# contents yet. For now, save and assert the correct files are present.
filepath = os.path.join(self.test_dir.name, 'visualization.qzv')
result.save(filepath)
with zipfile.ZipFile(filepath, mode='r') as zf:
fps = set(zf.namelist())
expected = {
'visualization/VERSION',
'visualization/metadata.yaml',
'visualization/README.md',
'visualization/data/index.html',
'visualization/data/css/style.css'
}
self.assertEqual(fps, expected)
def test_async(self):
concatenate_ints = self.plugin.methods['concatenate_ints']
concatenate_ints_markdown = \
self.plugin.methods['concatenate_ints_markdown']
artifact1 = Artifact._from_view([0, 42, 43], IntSequence1, None)
artifact2 = Artifact._from_view([99, -22], IntSequence2, None)
for method in concatenate_ints, concatenate_ints_markdown:
future = method.async(artifact1, artifact1, artifact2, 55, 1)
self.assertIsInstance(future, concurrent.futures.Future)
result = future.result()
self.assertIsInstance(result, Artifact)
self.assertEqual(result.type, IntSequence1)
provenance = result.provenance
self.assertIsInstance(provenance, Provenance)
self.assertIsInstance(provenance.execution_uuid, uuid.UUID)
self.assertTrue(
provenance.executor_reference.startswith(method.id))
self.assertEqual(provenance.artifact_uuids, {
'ints1': artifact1.uuid,
'ints2': artifact1.uuid,
'ints3': artifact2.uuid
})
self.assertEqual(provenance.parameter_references, {
'int1': '55',
'int2': '1'
})
self.assertIsInstance(result.uuid, uuid.UUID)
# Can retrieve multiple views of different type.
exp_list_view = [0, 42, 43, 0, 42, 43, 99, -22, 55, 1]
self.assertEqual(result.view(list), exp_list_view)
self.assertEqual(result.view(list), exp_list_view)
exp_counter_view = collections.Counter(
{0: 2, 42: 2, 43: 2, 99: 1, -22: 1, 55: 1, 1: 1})
self.assertEqual(result.view(collections.Counter),
exp_counter_view)
self.assertEqual(result.view(collections.Counter),
exp_counter_view)
# Accepts IntSequence1 | IntSequence2
artifact3 = Artifact._from_view([10, 20], IntSequence2, None)
future = method.async(artifact3, artifact1, artifact2, 55, 1)
result = future.result()
self.assertEqual(result.type, IntSequence1)
self.assertEqual(result.view(list),
[10, 20, 0, 42, 43, 99, -22, 55, 1])
def test_async(self):
concatenate_ints = self.plugin.methods['concatenate_ints']
concatenate_ints_markdown = \
self.plugin.methods['concatenate_ints_markdown']
artifact1 = Artifact._from_view(IntSequence1, [0, 42, 43], list)
artifact2 = Artifact._from_view(IntSequence2, [99, -22], list)
for method in concatenate_ints, concatenate_ints_markdown:
future = method.async(artifact1, artifact1, artifact2, 55, 1)
self.assertIsInstance(future, concurrent.futures.Future)
result = future.result()
# Test properties of the `Results` object.
self.assertIsInstance(result, tuple)
self.assertIsInstance(result, Results)
self.assertEqual(len(result), 1)
self.assertEqual(result.concatenated_ints.view(list),
[0, 42, 43, 0, 42, 43, 99, -22, 55, 1])
result = result[0]
self.assertIsInstance(result, Artifact)
self.assertEqual(result.type, IntSequence1)
self.assertIsInstance(result.uuid, uuid.UUID)
# Can retrieve multiple views of different type.
exp_list_view = [0, 42, 43, 0, 42, 43, 99, -22, 55, 1]
self.assertEqual(result.view(list), exp_list_view)
self.assertEqual(result.view(list), exp_list_view)
exp_counter_view = collections.Counter(
{0: 2, 42: 2, 43: 2, 99: 1, -22: 1, 55: 1, 1: 1})
self.assertEqual(result.view(collections.Counter),
exp_counter_view)
self.assertEqual(result.view(collections.Counter),
exp_counter_view)
# Accepts IntSequence1 | IntSequence2
artifact3 = Artifact._from_view(IntSequence2, [10, 20], list)
future = method.async(artifact3, artifact1, artifact2, 55, 1)
result, = future.result()
self.assertEqual(result.type, IntSequence1)
self.assertEqual(result.view(list),
[10, 20, 0, 42, 43, 99, -22, 55, 1])
def test_call_with_multiple_outputs(self):
split_ints = self.plugin.methods['split_ints']
split_ints_markdown = self.plugin.methods['split_ints_markdown']
artifact = Artifact._from_view([0, 42, -2, 43, 6], IntSequence1, None)
for method in split_ints, split_ints_markdown:
result = method(artifact)
self.assertIsInstance(result, tuple)
self.assertEqual(len(result), 2)
for output_artifact in result:
self.assertEqual(output_artifact.type, IntSequence1)
provenance = output_artifact.provenance
self.assertIsInstance(provenance, Provenance)
self.assertIsInstance(provenance.execution_uuid, uuid.UUID)
self.assertTrue(
provenance.executor_reference.startswith(method.id))
self.assertEqual(provenance.artifact_uuids, {
'ints': artifact.uuid
})
self.assertEqual(provenance.parameter_references, {})
self.assertIsInstance(output_artifact.uuid, uuid.UUID)
# Output artifacts have the same provenance.
self.assertEqual(result[0].provenance, result[1].provenance)
# Output artifacts have different UUIDs.
self.assertNotEqual(result[0].uuid, result[1].uuid)
self.assertEqual(result[0].view(list), [0, 42])
self.assertEqual(result[1].view(list), [-2, 43, 6])
def test_call_with_no_parameters(self):
most_common_viz = self.plugin.visualizers['most_common_viz']
artifact = Artifact._from_view(IntSequence1, [42, 42, 10, 0, 42, 5, 0],
list)
result = most_common_viz(artifact)
# Test properties of the `Results` object.
self.assertIsInstance(result, tuple)
self.assertIsInstance(result, Results)
self.assertEqual(len(result), 1)
self.assertEqual(result.visualization, result[0])
result = result[0]
self.assertIsInstance(result, Visualization)
self.assertEqual(result.type, qiime.core.type.Visualization)
self.assertIsInstance(result.uuid, uuid.UUID)
# TODO qiime.sdk.Visualization doesn't have an API to access its
# contents yet. For now, save and assert the correct files are present.
filepath = os.path.join(self.test_dir.name, 'visualization.qzv')
result.save(filepath)
root_dir = str(result.uuid)
expected = {
'VERSION', 'metadata.yaml', 'data/index.html', 'data/index.tsv'
}
self.assertArchiveMembers(filepath, root_dir, expected)
def test_call_with_no_parameters(self):
most_common_viz = self.plugin.visualizers['most_common_viz']
artifact = Artifact._from_view(IntSequence1, [42, 42, 10, 0, 42, 5, 0],
list, None)
result = most_common_viz(artifact)
self.assertIsInstance(result, Visualization)
self.assertEqual(result.type, qiime.core.type.Visualization)
provenance = result.provenance
self.assertIsInstance(provenance, Provenance)
self.assertIsInstance(provenance.execution_uuid, uuid.UUID)
self.assertTrue(
provenance.executor_reference.startswith(most_common_viz.id))
self.assertEqual(provenance.artifact_uuids, {'ints': artifact.uuid})
self.assertEqual(provenance.parameter_references, {})
self.assertIsInstance(result.uuid, uuid.UUID)
# TODO qiime.sdk.Visualization doesn't have an API to access its
# contents yet. For now, save and assert the correct files are present.
filepath = os.path.join(self.test_dir.name, 'visualization.qzv')
result.save(filepath)
with zipfile.ZipFile(filepath, mode='r') as zf:
fps = set(zf.namelist())
expected = {
'visualization/VERSION', 'visualization/metadata.yaml',
'visualization/README.md', 'visualization/data/index.html',
'visualization/data/index.tsv'
}
self.assertEqual(fps, expected)
def test_visualizer_callable_output(self):
artifact = Artifact._from_view(Mapping, {
'foo': 'abc',
'bar': 'def'
}, dict, None)
# Callable returns a value from `return_vals`
return_vals = (True, False, [], {}, '', 0, 0.0)
for return_val in return_vals:
def func(output_dir: str, foo: dict) -> None:
return return_val
self.plugin.visualizers.register_function(func, {'foo': Mapping},
{}, '', '')
visualizer = self.plugin.visualizers['func']
with self.assertRaisesRegex(TypeError, "should not return"):
visualizer(foo=artifact)
# Callable returns None (default function return)
def func(output_dir: str, foo: dict) -> None:
return None
self.plugin.visualizers.register_function(func, {'foo': Mapping}, {},
'', '')
visualizer = self.plugin.visualizers['func']
# Should not raise an exception
output = visualizer(foo=artifact)
self.assertIsInstance(output, Visualization)
def test_load_different_type_with_multiple_view_types(self):
saved_artifact = Artifact._from_view(IntSequence1,
[42, 42, 43, -999, 42], list)
fp = os.path.join(self.test_dir.name, 'artifact.qza')
saved_artifact.save(fp)
artifact = Artifact.load(fp)
self.assertEqual(artifact.type, IntSequence1)
self.assertEqual(artifact.uuid, saved_artifact.uuid)
self.assertEqual(artifact.view(list), [42, 42, 43, -999, 42])
self.assertEqual(artifact.view(list), [42, 42, 43, -999, 42])
self.assertEqual(artifact.view(collections.Counter),
collections.Counter({
42: 3,
43: 1,
-999: 1
}))
self.assertEqual(artifact.view(collections.Counter),
collections.Counter({
42: 3,
43: 1,
-999: 1
}))
def test_async_with_multiple_outputs(self):
split_ints = self.plugin.methods['split_ints']
split_ints_markdown = self.plugin.methods['split_ints_markdown']
artifact = Artifact._from_view(IntSequence1, [0, 42, -2, 43, 6], list)
for method in split_ints, split_ints_markdown:
future = method.async(artifact)
self.assertIsInstance(future, concurrent.futures.Future)
result = future.result()
self.assertIsInstance(result, tuple)
self.assertEqual(len(result), 2)
for output_artifact in result:
self.assertIsInstance(output_artifact, Artifact)
self.assertEqual(output_artifact.type, IntSequence1)
self.assertIsInstance(output_artifact.uuid, uuid.UUID)
# Output artifacts have different UUIDs.
self.assertNotEqual(result[0].uuid, result[1].uuid)
# Index lookup.
self.assertEqual(result[0].view(list), [0, 42])
self.assertEqual(result[1].view(list), [-2, 43, 6])
# Test properties of the `Results` object.
self.assertIsInstance(result, Results)
self.assertEqual(result.left.view(list), [0, 42])
self.assertEqual(result.right.view(list), [-2, 43, 6])
def test_load_and_save(self):
fp1 = os.path.join(self.test_dir.name, 'artifact1.qza')
fp2 = os.path.join(self.test_dir.name, 'artifact2.qza')
artifact = Artifact._from_view(FourInts, [-1, 42, 0, 43], list)
artifact.save(fp1)
artifact = Artifact.load(fp1)
# Overwriting its source file works.
artifact.save(fp1)
# Saving to a new file works.
artifact.save(fp2)
root_dir = str(artifact.uuid)
expected = {
'VERSION', 'metadata.yaml', 'data/file1.txt', 'data/file2.txt',
'data/nested/file3.txt', 'data/nested/file4.txt'
}
self.assertArchiveMembers(fp1, root_dir, expected)
root_dir = str(artifact.uuid)
expected = {
'VERSION', 'metadata.yaml', 'data/file1.txt', 'data/file2.txt',
'data/nested/file3.txt', 'data/nested/file4.txt'
}
self.assertArchiveMembers(fp2, root_dir, expected)
def test_eq_same_uuid(self):
fp = os.path.join(self.test_dir.name, 'artifact.qza')
artifact1 = Artifact._from_view(FourInts, [-1, 42, 0, 43], list)
artifact1.save(fp)
artifact2 = Artifact.load(fp)
self.assertEqual(artifact1, artifact2)
def test_load_artifact_as_visualization(self):
artifact = Artifact._from_view(FourInts, [0, 0, 42, 1000], list,
self.provenance)
fp = os.path.join(self.test_dir.name, 'artifact.qza')
artifact.save(fp)
with self.assertRaisesRegex(
TypeError, 'Artifact.*Visualization.load.*Artifact.load'):
Visualization.load(fp)
def test_from_view_with_provenance(self):
artifact = Artifact._from_view([-1, 42, 0, 43], FourInts,
self.provenance)
self.assertEqual(artifact.type, FourInts)
self.assertEqual(artifact.provenance, self.provenance)
self.assertIsInstance(artifact.uuid, uuid.UUID)
self.assertEqual(artifact.view(list), [-1, 42, 0, 43])
self.assertEqual(artifact.view(list), [-1, 42, 0, 43])
def test_extract_artifact_as_visualization(self):
artifact = Artifact._from_view(FourInts, [0, 0, 42, 1000], list,
self.provenance)
fp = os.path.join(self.test_dir.name, 'artifact.qza')
artifact.save(fp)
with self.assertRaisesRegex(
TypeError, 'Visualization does not support.*FourInts'):
Visualization.extract(fp, self.test_dir)
def test_from_view(self):
artifact = Artifact._from_view(FourInts, [-1, 42, 0, 43], list)
self.assertEqual(artifact.type, FourInts)
# We don't know what the UUID is because it's generated within
# Artifact._from_view.
self.assertIsInstance(artifact.uuid, uuid.UUID)
self.assertEqual(artifact.view(list), [-1, 42, 0, 43])
# Can produce same view if called again.
self.assertEqual(artifact.view(list), [-1, 42, 0, 43])
def test_save_artifact_warning(self):
artifact = Artifact._from_view(FourInts, [0, 0, 42, 1000], list,
self.provenance)
fp = os.path.join(self.test_dir.name, 'artifact.zip')
with warnings.catch_warnings(record=True) as w:
artifact.save(fp)
self.assertEqual(len(w), 1)
self.assertIsInstance(w[0].message, UserWarning)
self.assertIn(Artifact.extension, str(w[0].message))
def test_from_view(self):
artifact = Artifact._from_view(FourInts, [-1, 42, 0, 43], list,
self.provenance_capture)
self.assertEqual(artifact.type, FourInts)
# We don't know what the UUID is because it's generated within
# Artifact._from_view.
self.assertIsInstance(artifact.uuid, uuid.UUID)
self.assertEqual(artifact.view(list), [-1, 42, 0, 43])
# Can produce same view if called again.
self.assertEqual(artifact.view(list), [-1, 42, 0, 43])
def test_peek(self):
artifact = Artifact._from_view([0, 0, 42, 1000], FourInts, None)
fp = os.path.join(self.test_dir.name, 'artifact.qza')
artifact.save(fp)
metadata = Artifact.peek(fp)
self.assertIsInstance(metadata, ResultMetadata)
self.assertEqual(metadata.type, FourInts)
self.assertIsNone(metadata.provenance)
self.assertEqual(metadata.uuid, artifact.uuid)
def test_peek(self):
artifact = Artifact._from_view(FourInts, [0, 0, 42, 1000], list)
fp = os.path.join(self.test_dir.name, 'artifact.qza')
artifact.save(fp)
metadata = Artifact.peek(fp)
self.assertIsInstance(metadata, ResultMetadata)
self.assertEqual(metadata.type, 'FourInts')
self.assertEqual(metadata.uuid, str(artifact.uuid))
self.assertEqual(metadata.format, 'FourIntsDirectoryFormat')
def test_ne_different_type_same_uuid(self):
artifact = Artifact._from_view(FourInts, [-1, 42, 0, 43], list)
class Faker:
@property
def uuid(self):
return artifact.uuid
faker = Faker()
self.assertNotEqual(artifact, faker)
def test_load(self):
saved_artifact = Artifact._from_view(FourInts, [-1, 42, 0, 43], list)
fp = os.path.join(self.test_dir.name, 'artifact.qza')
saved_artifact.save(fp)
artifact = Artifact.load(fp)
self.assertEqual(artifact.type, FourInts)
self.assertEqual(artifact.uuid, saved_artifact.uuid)
self.assertEqual(artifact.view(list), [-1, 42, 0, 43])
self.assertEqual(artifact.view(list), [-1, 42, 0, 43])
def test_async(self):
mapping_viz = self.plugin.visualizers['mapping_viz']
artifact1 = Artifact._from_view(Mapping, {
'foo': 'abc',
'bar': 'def'
}, dict)
artifact2 = Artifact._from_view(Mapping, {
'baz': 'abc',
'bazz': 'ghi'
}, dict)
future = mapping_viz. async (artifact1, artifact2, 'Key', 'Value')
self.assertIsInstance(future, concurrent.futures.Future)
result = future.result()
# Test properties of the `Results` object.
self.assertIsInstance(result, tuple)
self.assertIsInstance(result, Results)
self.assertEqual(len(result), 1)
self.assertEqual(result.visualization, result[0])
result = result[0]
self.assertIsInstance(result, Visualization)
self.assertEqual(result.type, qiime.core.type.Visualization)
self.assertIsInstance(result.uuid, uuid.UUID)
# TODO qiime.sdk.Visualization doesn't have an API to access its
# contents yet. For now, save and assert the correct files are present.
filepath = os.path.join(self.test_dir.name, 'visualization.qzv')
result.save(filepath)
root_dir = str(result.uuid)
expected = {
'VERSION', 'metadata.yaml', 'data/index.html', 'data/css/style.css'
}
self.assertArchiveMembers(filepath, root_dir, expected)
def test_peek_with_provenance(self):
artifact = Artifact._from_view({'foo': 'bar', 'baz': 'bazz'}, Mapping,
self.provenance)
fp = os.path.join(self.test_dir.name, 'artifact.qza')
artifact.save(fp)
metadata = Artifact.peek(fp)
self.assertIsInstance(metadata, ResultMetadata)
self.assertEqual(metadata.type, Mapping)
self.assertEqual(metadata.provenance, self.provenance)
self.assertEqual(metadata.uuid, artifact.uuid)
def test_load(self):
saved_artifact = Artifact._from_view([-1, 42, 0, 43], FourInts, None)
fp = os.path.join(self.test_dir.name, 'artifact.qza')
saved_artifact.save(fp)
artifact = Artifact.load(fp)
self.assertEqual(artifact.type, FourInts)
self.assertIsNone(artifact.provenance)
self.assertEqual(artifact.uuid, saved_artifact.uuid)
self.assertEqual(artifact.view(list), [-1, 42, 0, 43])
self.assertEqual(artifact.view(list), [-1, 42, 0, 43])
def test_peek_artifact(self):
artifact = Artifact._from_view(FourInts, [0, 0, 42, 1000], list,
self.provenance)
fp = os.path.join(self.test_dir.name, 'artifact.qza')
artifact.save(fp)
metadata = Result.peek(fp)
self.assertIsInstance(metadata, ResultMetadata)
self.assertEqual(metadata.type, FourInts)
self.assertEqual(metadata.provenance, self.provenance)
self.assertEqual(metadata.uuid, artifact.uuid)
def test_writer_transformer(self):
for type in IntSequence1, IntSequence2:
# `Artifact._from_view` invokes transformer that handles `list` ->
# `SingleIntFormat`, because the `SingleIntDirectoryFormat` has
# been registered as the directory format for the semantic type.
# We didn't define a `SingleIntDirectoryFormat` ->
# `SingleIntFormat` tranformer because
# `model.SingleFileDirectoryFormat` handles that transformation for
# us.
artifact = Artifact._from_view(type, [1, 2, 42, -999, 42, 0], list)
# Test that the directory and file format can be read again.
self.assertEqual(artifact.view(list), [1, 2, 42, -999, 42, 0])
def test_load_artifact(self):
saved_artifact = Artifact._from_view(FourInts, [-1, 42, 0, 43], list,
self.provenance)
fp = os.path.join(self.test_dir.name, 'artifact.qza')
saved_artifact.save(fp)
artifact = Result.load(fp)
self.assertIsInstance(artifact, Artifact)
self.assertEqual(artifact.type, FourInts)
self.assertEqual(artifact.provenance, self.provenance)
self.assertEqual(artifact.uuid, saved_artifact.uuid)
self.assertEqual(artifact.view(list), [-1, 42, 0, 43])
def test_writer_transformer(self):
# `Artifact._from_view` invokes transformer that handles `dict` ->
# `MappingFormat`, because the `MappingDirectoryFormat` has
# been registered as the directory format for the semantic type.
# We didn't define a `MappingDirectoryFormat` ->
# `MappingFormat` tranformer because
# `model.SingleFileDirectoryFormat` handles that transformation for
# us.
artifact = Artifact._from_view(Mapping, {
'abc': 'cat',
'def': 'dog'
}, dict)
# Test that the directory and file format can be read again.
self.assertEqual(artifact.view(dict), {'abc': 'cat', 'def': 'dog'})
def test_from_view_and_save(self):
fp = os.path.join(self.test_dir.name, 'artifact.qza')
# Using four-ints data layout because it has multiple files, some of
# which are in a nested directory.
artifact = Artifact._from_view(FourInts, [-1, 42, 0, 43], list)
artifact.save(fp)
root_dir = str(artifact.uuid)
expected = {
'VERSION', 'metadata.yaml', 'data/file1.txt', 'data/file2.txt',
'data/nested/file3.txt', 'data/nested/file4.txt'
}
self.assertArchiveMembers(fp, root_dir, expected)
def test_from_view_different_type_with_multiple_view_types(self):
artifact = Artifact._from_view([42, 42, 43, -999, 42], IntSequence1,
None)
self.assertEqual(artifact.type, IntSequence1)
self.assertIsNone(artifact.provenance)
self.assertIsInstance(artifact.uuid, uuid.UUID)
self.assertEqual(artifact.view(list), [42, 42, 43, -999, 42])
self.assertEqual(artifact.view(list), [42, 42, 43, -999, 42])
self.assertEqual(artifact.view(collections.Counter),
collections.Counter({42: 3, 43: 1, -999: 1}))
self.assertEqual(artifact.view(collections.Counter),
collections.Counter({42: 3, 43: 1, -999: 1}))
def test_roundtrip(self):
fp1 = os.path.join(self.test_dir.name, 'artifact1.qza')
fp2 = os.path.join(self.test_dir.name, 'artifact2.qza')
artifact = Artifact._from_view([-1, 42, 0, 43], FourInts,
self.provenance)
artifact.save(fp1)
artifact1 = Artifact.load(fp1)
artifact1.save(fp2)
artifact2 = Artifact.load(fp2)
self.assertEqual(artifact1.type, artifact2.type)
self.assertEqual(artifact1.provenance, artifact2.provenance)
self.assertEqual(artifact1.uuid, artifact2.uuid)
self.assertEqual(artifact1.view(list), artifact2.view(list))
self.assertEqual(artifact1.view(list), artifact2.view(list))
def test_extract(self):
fp = os.path.join(self.test_dir.name, 'artifact.qza')
artifact = Artifact._from_view(FourInts, [-1, 42, 0, 43], list)
artifact.save(fp)
root_dir = str(artifact.uuid)
output_dir = os.path.join(self.test_dir.name, 'artifact-extract-test')
result_dir = Artifact.extract(fp, output_dir=output_dir)
self.assertEqual(result_dir, os.path.join(output_dir, root_dir))
expected = {
'VERSION', 'metadata.yaml', 'data/file1.txt', 'data/file2.txt',
'data/nested/file3.txt', 'data/nested/file4.txt'
}
self.assertExtractedArchiveMembers(output_dir, root_dir, expected)
def test_from_view_different_type_with_multiple_view_types(self):
artifact = Artifact._from_view(IntSequence1, [42, 42, 43, -999, 42],
list, self.provenance_capture)
self.assertEqual(artifact.type, IntSequence1)
self.assertIsInstance(artifact.uuid, uuid.UUID)
self.assertEqual(artifact.view(list),
[42, 42, 43, -999, 42])
self.assertEqual(artifact.view(list),
[42, 42, 43, -999, 42])
self.assertEqual(artifact.view(collections.Counter),
collections.Counter({42: 3, 43: 1, -999: 1}))
self.assertEqual(artifact.view(collections.Counter),
collections.Counter({42: 3, 43: 1, -999: 1}))
def test_async_with_multiple_outputs(self):
split_ints = self.plugin.methods['split_ints']
split_ints_markdown = self.plugin.methods['split_ints_markdown']
artifact = Artifact._from_view(IntSequence1, [0, 42, -2, 43, 6], list,
None)
for method in split_ints, split_ints_markdown:
future = method.async(artifact)
self.assertIsInstance(future, concurrent.futures.Future)
result = future.result()
self.assertIsInstance(result, tuple)
self.assertEqual(len(result), 2)
for output_artifact in result:
self.assertIsInstance(output_artifact, Artifact)
self.assertEqual(output_artifact.type, IntSequence1)
provenance = output_artifact.provenance
self.assertIsInstance(provenance, Provenance)
self.assertIsInstance(provenance.execution_uuid, uuid.UUID)
self.assertTrue(
provenance.executor_reference.startswith(method.id))
self.assertEqual(provenance.artifact_uuids, {
'ints': artifact.uuid
})
self.assertEqual(provenance.parameter_references, {})
self.assertIsInstance(output_artifact.uuid, uuid.UUID)
# Output artifacts have the same provenance.
self.assertEqual(result[0].provenance, result[1].provenance)
# Output artifacts have different UUIDs.
self.assertNotEqual(result[0].uuid, result[1].uuid)
# Index lookup.
self.assertEqual(result[0].view(list), [0, 42])
self.assertEqual(result[1].view(list), [-2, 43, 6])
# Test properties of the `Results` object.
self.assertIsInstance(result, Results)
self.assertEqual(result.left.view(list), [0, 42])
self.assertEqual(result.right.view(list), [-2, 43, 6])
def test_roundtrip(self):
fp1 = os.path.join(self.test_dir.name, 'artifact1.qza')
fp2 = os.path.join(self.test_dir.name, 'artifact2.qza')
artifact = Artifact._from_view(FourInts, [-1, 42, 0, 43], list)
artifact.save(fp1)
artifact1 = Artifact.load(fp1)
artifact1.save(fp2)
artifact2 = Artifact.load(fp2)
self.assertEqual(artifact1.type, artifact2.type)
self.assertEqual(artifact1.format, artifact2.format)
self.assertEqual(artifact1.uuid, artifact2.uuid)
self.assertEqual(artifact1.view(list), artifact2.view(list))
# double view to make sure multiple views can be taken
self.assertEqual(artifact1.view(list), artifact2.view(list))
def test_load_different_type_with_multiple_view_types(self):
saved_artifact = Artifact._from_view([42, 42, 43, -999, 42],
IntSequence1, None)
fp = os.path.join(self.test_dir.name, 'artifact.qza')
saved_artifact.save(fp)
artifact = Artifact.load(fp)
self.assertEqual(artifact.type, IntSequence1)
self.assertIsNone(artifact.provenance)
self.assertEqual(artifact.uuid, saved_artifact.uuid)
self.assertEqual(artifact.view(list), [42, 42, 43, -999, 42])
self.assertEqual(artifact.view(list), [42, 42, 43, -999, 42])
self.assertEqual(artifact.view(collections.Counter),
collections.Counter({42: 3, 43: 1, -999: 1}))
self.assertEqual(artifact.view(collections.Counter),
collections.Counter({42: 3, 43: 1, -999: 1}))
def test_from_view_and_save(self):
fp = os.path.join(self.test_dir.name, 'artifact.qza')
# Using four-ints data layout because it has multiple files, some of
# which are in a nested directory.
artifact = Artifact._from_view([-1, 42, 0, 43], FourInts,
self.provenance)
artifact.save(fp)
with zipfile.ZipFile(fp, mode='r') as zf:
fps = set(zf.namelist())
expected = {
'artifact/VERSION',
'artifact/metadata.yaml',
'artifact/README.md',
'artifact/data/file1.txt',
'artifact/data/file2.txt',
'artifact/data/nested/file3.txt',
'artifact/data/nested/file4.txt'
}
self.assertEqual(fps, expected)
def test_extract(self):
fp = os.path.join(self.test_dir.name, 'artifact.qza')
artifact = Artifact._from_view([-1, 42, 0, 43], FourInts,
self.provenance)
artifact.save(fp)
output_dir = os.path.join(self.test_dir.name, 'artifact-extract-test')
result_dir = Artifact.extract(fp, output_dir=output_dir)
self.assertEqual(result_dir, output_dir)
contents = [
'artifact/VERSION',
'artifact/metadata.yaml',
'artifact/README.md',
'artifact/data/file1.txt',
'artifact/data/file2.txt',
'artifact/data/nested/file3.txt',
'artifact/data/nested/file4.txt']
for fp in contents:
expected_fp = os.path.join(output_dir, fp)
self.assertTrue(os.path.exists(expected_fp),
'File %s was not extracted.' % fp)
def test_load_and_save(self):
fp1 = os.path.join(self.test_dir.name, 'artifact1.qza')
fp2 = os.path.join(self.test_dir.name, 'artifact2.qza')
artifact = Artifact._from_view([-1, 42, 0, 43], FourInts,
self.provenance)
artifact.save(fp1)
artifact = Artifact.load(fp1)
# Overwriting its source file works.
artifact.save(fp1)
# Saving to a new file works.
artifact.save(fp2)
with zipfile.ZipFile(fp1, mode='r') as zf:
fps = set(zf.namelist())
expected = {
'artifact1/VERSION',
'artifact1/metadata.yaml',
'artifact1/README.md',
'artifact1/data/file1.txt',
'artifact1/data/file2.txt',
'artifact1/data/nested/file3.txt',
'artifact1/data/nested/file4.txt'
}
self.assertEqual(fps, expected)
with zipfile.ZipFile(fp2, mode='r') as zf:
fps = set(zf.namelist())
expected = {
'artifact2/VERSION',
'artifact2/metadata.yaml',
'artifact2/README.md',
'artifact2/data/file1.txt',
'artifact2/data/file2.txt',
'artifact2/data/nested/file3.txt',
'artifact2/data/nested/file4.txt'
}
self.assertEqual(fps, expected)
def test_from_view_and_save(self):
fp = os.path.join(self.test_dir.name, 'artifact.qza')
# Using four-ints data layout because it has multiple files, some of
# which are in a nested directory.
artifact = Artifact._from_view(FourInts, [-1, 42, 0, 43], list,
self.provenance_capture)
artifact.save(fp)
root_dir = str(artifact.uuid)
expected = {
'VERSION',
'metadata.yaml',
'data/file1.txt',
'data/file2.txt',
'data/nested/file3.txt',
'data/nested/file4.txt',
'provenance/metadata.yaml',
'provenance/VERSION',
'provenance/action/action.yaml'
}
self.assertArchiveMembers(fp, root_dir, expected)
def _test_to_script_or_to_markdown(self, to_method, template_lines):
# These methods are so similar that it makes sense to have a helper
# that can test either one instead of duplicating a bunch of code.
workflow = Workflow.from_markdown(self.markdown_fp)
artifact_fp1 = os.path.join(self.test_dir.name, 'artifact1.qza')
artifact = Artifact._from_view(
[-1, 42, 0, 43, 43], qiime.core.testing.TestType, None)
artifact.save(artifact_fp1)
artifact_fp2 = os.path.join(self.test_dir.name, 'artifact2.qza')
artifact = Artifact._from_view(
[1, 2, 100], qiime.core.testing.TestType, None)
artifact.save(artifact_fp2)
artifact_fp3 = os.path.join(self.test_dir.name, 'artifact3.qza')
job = to_method(
workflow,
input_artifact_filepaths={
'input1': artifact_fp1,
'input2': artifact_fp2
},
parameter_references={
'param1': 99,
'param2': -999,
},
output_artifact_filepaths={
'concatenated_inputs': artifact_fp3
}
)
provenance_lines = [
"provenance = Provenance(",
"parameters={",
"'param2': -999"
]
setup_lines = [
"import builtins",
"input1 = Artifact.load(%r).view(builtins.list)" % artifact_fp1,
"input2 = Artifact.load(%r).view(builtins.list)" % artifact_fp2,
"param1 = 99",
"param2 = -999"
]
teardown_lines = [
"artifact = Artifact._from_view(concatenated_inputs, 'TestType', "
"provenance)",
"artifact.save(%r)" % artifact_fp3
]
for expected_lines in (provenance_lines, setup_lines, template_lines,
teardown_lines):
for expected_line in expected_lines:
self.assertIn(expected_line, job.code)
self.assertIsInstance(job.uuid, uuid.UUID)
self.assertEqual(job.uuid.version, 4)
self.assertEqual(
job.input_artifact_filepaths,
{'input1': artifact_fp1, 'input2': artifact_fp2})
self.assertEqual(
job.parameter_references,
{'param1': 99, 'param2': -999})
self.assertEqual(
job.output_artifact_filepaths,
{'concatenated_inputs': artifact_fp3})
