def test_load_with_archive_filepath_modified(self): # Save an artifact for use in the following test case. fp = os.path.join(self.test_dir.name, 'artifact.qza') Artifact._from_view(FourInts, [-1, 42, 0, 43], list).save(fp) # Load the artifact from a filepath then save a different artifact to # the same filepath. Assert that both artifacts produce the correct # views of their data. # # `load` used to be lazy, only extracting data when it needed to (e.g. # when `save` or `view` was called). This was buggy as the filepath # could have been deleted, or worse, modified to contain a different # .qza file. Thus, the wrong archive could be extracted on demand, or # the archive could be missing altogether. There isn't an easy # cross-platform compatible way to solve this problem, so Artifact.load # is no longer lazy and always extracts its data immediately. The real # motivation for lazy loading was for quick inspection of archives # without extracting/copying data, so that API is now provided through # Artifact.peek. artifact1 = Artifact.load(fp) Artifact._from_view(FourInts, [10, 11, 12, 13], list).save(fp) artifact2 = Artifact.load(fp) self.assertEqual(artifact1.view(list), [-1, 42, 0, 43]) self.assertEqual(artifact2.view(list), [10, 11, 12, 13])
def test_call_with_no_parameters(self): merge_mappings = self.plugin.methods['merge_mappings'] artifact1 = Artifact._from_view({'foo': 'abc', 'bar': 'def'}, Mapping, None) artifact2 = Artifact._from_view({'bazz': 'abc'}, Mapping, None) result = merge_mappings(artifact1, artifact2) self.assertIsInstance(result, Artifact) self.assertEqual(result.type, Mapping) provenance = result.provenance self.assertIsInstance(provenance, Provenance) self.assertIsInstance(provenance.execution_uuid, uuid.UUID) self.assertTrue( provenance.executor_reference.startswith(merge_mappings.id)) self.assertEqual(provenance.artifact_uuids, { 'mapping1': artifact1.uuid, 'mapping2': artifact2.uuid }) self.assertEqual(provenance.parameter_references, {}) self.assertIsInstance(result.uuid, uuid.UUID) self.assertEqual(result.view(dict), {'foo': 'abc', 'bar': 'def', 'bazz': 'abc'})
def test_call_with_no_parameters(self): merge_mappings = self.plugin.methods['merge_mappings'] artifact1 = Artifact._from_view(Mapping, {'foo': 'abc', 'bar': 'def'}, dict, None) artifact2 = Artifact._from_view(Mapping, {'bazz': 'abc'}, dict, None) result = merge_mappings(artifact1, artifact2) self.assertIsInstance(result, Artifact) self.assertEqual(result.type, Mapping) provenance = result.provenance self.assertIsInstance(provenance, Provenance) self.assertIsInstance(provenance.execution_uuid, uuid.UUID) self.assertTrue( provenance.executor_reference.startswith(merge_mappings.id)) self.assertEqual(provenance.artifact_uuids, { 'mapping1': artifact1.uuid, 'mapping2': artifact2.uuid }) self.assertEqual(provenance.parameter_references, {}) self.assertIsInstance(result.uuid, uuid.UUID) self.assertEqual(result.view(dict), {'foo': 'abc', 'bar': 'def', 'bazz': 'abc'})
def setUp(self): # TODO standardize temporary directories created by QIIME self.test_dir = tempfile.TemporaryDirectory(prefix='qiime2-temp-') self.dummy_provenance = Provenance( job_uuid='7e909a23-21e2-44c2-be17-0723fae91dc8', artifact_uuids={ 'input1': 'f16ca3d0-fe83-4b1e-8eea-7e35db3f6b0f', 'input2': '908dece5-db23-4562-ad03-876bb5750145', }, parameters={ 'param1': 'abc', 'param2': 100, }, workflow_reference=( 'dummy workflow reference, see ' 'https://github.com/biocore/qiime2/issues/26' ) ) self.artifact_with_provenance = Artifact._from_view( [-1, 42, 0, 43, 43], qiime.core.testing.TestType, self.dummy_provenance) self.artifact_without_provenance = Artifact._from_view( [-1, 42, 0, 43, 43], qiime.core.testing.TestType, None)
def test_load_with_archive_filepath_modified(self): # Save an artifact for use in the following test case. fp = os.path.join(self.test_dir.name, 'artifact.qza') Artifact._from_view([-1, 42, 0, 43], FourInts, None).save(fp) # Load the artifact from a filepath then save a different artifact to # the same filepath. Assert that both artifacts produce the correct # views of their data. # # `load` used to be lazy, only extracting data when it needed to (e.g. # when `save` or `view` was called). This was buggy as the filepath # could have been deleted, or worse, modified to contain a different # .qza file. Thus, the wrong archive could be extracted on demand, or # the archive could be missing altogether. There isn't an easy # cross-platform compatible way to solve this problem, so Artifact.load # is no longer lazy and always extracts its data immediately. The real # motivation for lazy loading was for quick inspection of archives # without extracting/copying data, so that API is now provided through # Artifact.peek. artifact1 = Artifact.load(fp) Artifact._from_view([10, 11, 12, 13], FourInts, None).save(fp) artifact2 = Artifact.load(fp) self.assertEqual(artifact1.view(list), [-1, 42, 0, 43]) self.assertEqual(artifact2.view(list), [10, 11, 12, 13])
def test_call_with_no_parameters(self): merge_mappings = self.plugin.methods['merge_mappings'] artifact1 = Artifact._from_view(Mapping, {'foo': 'abc', 'bar': 'def'}, dict) artifact2 = Artifact._from_view(Mapping, {'bazz': 'abc'}, dict) result = merge_mappings(artifact1, artifact2) # Test properties of the `Results` object. self.assertIsInstance(result, tuple) self.assertIsInstance(result, Results) self.assertEqual(len(result), 1) self.assertEqual(result.merged_mapping.view(dict), {'foo': 'abc', 'bar': 'def', 'bazz': 'abc'}) result = result[0] self.assertIsInstance(result, Artifact) self.assertEqual(result.type, Mapping) self.assertIsInstance(result.uuid, uuid.UUID) self.assertEqual(result.view(dict), {'foo': 'abc', 'bar': 'def', 'bazz': 'abc'})
def test_async(self): concatenate_ints = self.plugin.methods['concatenate_ints'] concatenate_ints_markdown = \ self.plugin.methods['concatenate_ints_markdown'] artifact1 = Artifact._from_view(IntSequence1, [0, 42, 43], list, None) artifact2 = Artifact._from_view(IntSequence2, [99, -22], list, None) for method in concatenate_ints, concatenate_ints_markdown: future = method.async(artifact1, artifact1, artifact2, 55, 1) self.assertIsInstance(future, concurrent.futures.Future) result = future.result() self.assertIsInstance(result, Artifact) self.assertEqual(result.type, IntSequence1) provenance = result.provenance self.assertIsInstance(provenance, Provenance) self.assertIsInstance(provenance.execution_uuid, uuid.UUID) self.assertTrue( provenance.executor_reference.startswith(method.id)) self.assertEqual(provenance.artifact_uuids, { 'ints1': artifact1.uuid, 'ints2': artifact1.uuid, 'ints3': artifact2.uuid }) self.assertEqual(provenance.parameter_references, { 'int1': '55', 'int2': '1' }) self.assertIsInstance(result.uuid, uuid.UUID) # Can retrieve multiple views of different type. exp_list_view = [0, 42, 43, 0, 42, 43, 99, -22, 55, 1] self.assertEqual(result.view(list), exp_list_view) self.assertEqual(result.view(list), exp_list_view) exp_counter_view = collections.Counter( {0: 2, 42: 2, 43: 2, 99: 1, -22: 1, 55: 1, 1: 1}) self.assertEqual(result.view(collections.Counter), exp_counter_view) self.assertEqual(result.view(collections.Counter), exp_counter_view) # Accepts IntSequence1 | IntSequence2 artifact3 = Artifact._from_view(IntSequence2, [10, 20], list, None) future = method.async(artifact3, artifact1, artifact2, 55, 1) result = future.result() self.assertEqual(result.type, IntSequence1) self.assertEqual(result.view(list), [10, 20, 0, 42, 43, 99, -22, 55, 1])
def test_async(self): mapping_viz = self.plugin.visualizers['mapping_viz'] artifact1 = Artifact._from_view(Mapping, {'foo': 'abc', 'bar': 'def'}, dict, None) artifact2 = Artifact._from_view(Mapping, {'baz': 'abc', 'bazz': 'ghi'}, dict, None) future = mapping_viz.async(artifact1, artifact2, 'Key', 'Value') self.assertIsInstance(future, concurrent.futures.Future) result = future.result() # Test properties of the `Results` object. self.assertIsInstance(result, tuple) self.assertIsInstance(result, Results) self.assertEqual(len(result), 1) self.assertEqual(result.visualization, result[0]) result = result[0] self.assertIsInstance(result, Visualization) self.assertEqual(result.type, qiime.core.type.Visualization) provenance = result.provenance self.assertIsInstance(provenance, Provenance) self.assertIsInstance(provenance.execution_uuid, uuid.UUID) self.assertTrue( provenance.executor_reference.startswith(mapping_viz.id)) self.assertEqual(provenance.artifact_uuids, { 'mapping1': artifact1.uuid, 'mapping2': artifact2.uuid }) self.assertEqual(provenance.parameter_references, { 'key_label': 'Key', 'value_label': 'Value' }) self.assertIsInstance(result.uuid, uuid.UUID) # TODO qiime.sdk.Visualization doesn't have an API to access its # contents yet. For now, save and assert the correct files are present. filepath = os.path.join(self.test_dir.name, 'visualization.qzv') result.save(filepath) with zipfile.ZipFile(filepath, mode='r') as zf: fps = set(zf.namelist()) expected = { 'visualization/VERSION', 'visualization/metadata.yaml', 'visualization/README.md', 'visualization/data/index.html', 'visualization/data/css/style.css' } self.assertEqual(fps, expected)
def test_async(self): concatenate_ints = self.plugin.methods['concatenate_ints'] concatenate_ints_markdown = \ self.plugin.methods['concatenate_ints_markdown'] artifact1 = Artifact._from_view([0, 42, 43], IntSequence1, None) artifact2 = Artifact._from_view([99, -22], IntSequence2, None) for method in concatenate_ints, concatenate_ints_markdown: future = method.async(artifact1, artifact1, artifact2, 55, 1) self.assertIsInstance(future, concurrent.futures.Future) result = future.result() self.assertIsInstance(result, Artifact) self.assertEqual(result.type, IntSequence1) provenance = result.provenance self.assertIsInstance(provenance, Provenance) self.assertIsInstance(provenance.execution_uuid, uuid.UUID) self.assertTrue( provenance.executor_reference.startswith(method.id)) self.assertEqual(provenance.artifact_uuids, { 'ints1': artifact1.uuid, 'ints2': artifact1.uuid, 'ints3': artifact2.uuid }) self.assertEqual(provenance.parameter_references, { 'int1': '55', 'int2': '1' }) self.assertIsInstance(result.uuid, uuid.UUID) # Can retrieve multiple views of different type. exp_list_view = [0, 42, 43, 0, 42, 43, 99, -22, 55, 1] self.assertEqual(result.view(list), exp_list_view) self.assertEqual(result.view(list), exp_list_view) exp_counter_view = collections.Counter( {0: 2, 42: 2, 43: 2, 99: 1, -22: 1, 55: 1, 1: 1}) self.assertEqual(result.view(collections.Counter), exp_counter_view) self.assertEqual(result.view(collections.Counter), exp_counter_view) # Accepts IntSequence1 | IntSequence2 artifact3 = Artifact._from_view([10, 20], IntSequence2, None) future = method.async(artifact3, artifact1, artifact2, 55, 1) result = future.result() self.assertEqual(result.type, IntSequence1) self.assertEqual(result.view(list), [10, 20, 0, 42, 43, 99, -22, 55, 1])
def test_async(self): concatenate_ints = self.plugin.methods['concatenate_ints'] concatenate_ints_markdown = \ self.plugin.methods['concatenate_ints_markdown'] artifact1 = Artifact._from_view(IntSequence1, [0, 42, 43], list) artifact2 = Artifact._from_view(IntSequence2, [99, -22], list) for method in concatenate_ints, concatenate_ints_markdown: future = method.async(artifact1, artifact1, artifact2, 55, 1) self.assertIsInstance(future, concurrent.futures.Future) result = future.result() # Test properties of the `Results` object. self.assertIsInstance(result, tuple) self.assertIsInstance(result, Results) self.assertEqual(len(result), 1) self.assertEqual(result.concatenated_ints.view(list), [0, 42, 43, 0, 42, 43, 99, -22, 55, 1]) result = result[0] self.assertIsInstance(result, Artifact) self.assertEqual(result.type, IntSequence1) self.assertIsInstance(result.uuid, uuid.UUID) # Can retrieve multiple views of different type. exp_list_view = [0, 42, 43, 0, 42, 43, 99, -22, 55, 1] self.assertEqual(result.view(list), exp_list_view) self.assertEqual(result.view(list), exp_list_view) exp_counter_view = collections.Counter( {0: 2, 42: 2, 43: 2, 99: 1, -22: 1, 55: 1, 1: 1}) self.assertEqual(result.view(collections.Counter), exp_counter_view) self.assertEqual(result.view(collections.Counter), exp_counter_view) # Accepts IntSequence1 | IntSequence2 artifact3 = Artifact._from_view(IntSequence2, [10, 20], list) future = method.async(artifact3, artifact1, artifact2, 55, 1) result, = future.result() self.assertEqual(result.type, IntSequence1) self.assertEqual(result.view(list), [10, 20, 0, 42, 43, 99, -22, 55, 1])
def test_call_with_multiple_outputs(self): split_ints = self.plugin.methods['split_ints'] split_ints_markdown = self.plugin.methods['split_ints_markdown'] artifact = Artifact._from_view([0, 42, -2, 43, 6], IntSequence1, None) for method in split_ints, split_ints_markdown: result = method(artifact) self.assertIsInstance(result, tuple) self.assertEqual(len(result), 2) for output_artifact in result: self.assertEqual(output_artifact.type, IntSequence1) provenance = output_artifact.provenance self.assertIsInstance(provenance, Provenance) self.assertIsInstance(provenance.execution_uuid, uuid.UUID) self.assertTrue( provenance.executor_reference.startswith(method.id)) self.assertEqual(provenance.artifact_uuids, { 'ints': artifact.uuid }) self.assertEqual(provenance.parameter_references, {}) self.assertIsInstance(output_artifact.uuid, uuid.UUID) # Output artifacts have the same provenance. self.assertEqual(result[0].provenance, result[1].provenance) # Output artifacts have different UUIDs. self.assertNotEqual(result[0].uuid, result[1].uuid) self.assertEqual(result[0].view(list), [0, 42]) self.assertEqual(result[1].view(list), [-2, 43, 6])
def test_call_with_no_parameters(self): most_common_viz = self.plugin.visualizers['most_common_viz'] artifact = Artifact._from_view(IntSequence1, [42, 42, 10, 0, 42, 5, 0], list) result = most_common_viz(artifact) # Test properties of the `Results` object. self.assertIsInstance(result, tuple) self.assertIsInstance(result, Results) self.assertEqual(len(result), 1) self.assertEqual(result.visualization, result[0]) result = result[0] self.assertIsInstance(result, Visualization) self.assertEqual(result.type, qiime.core.type.Visualization) self.assertIsInstance(result.uuid, uuid.UUID) # TODO qiime.sdk.Visualization doesn't have an API to access its # contents yet. For now, save and assert the correct files are present. filepath = os.path.join(self.test_dir.name, 'visualization.qzv') result.save(filepath) root_dir = str(result.uuid) expected = { 'VERSION', 'metadata.yaml', 'data/index.html', 'data/index.tsv' } self.assertArchiveMembers(filepath, root_dir, expected)
def test_call_with_no_parameters(self): most_common_viz = self.plugin.visualizers['most_common_viz'] artifact = Artifact._from_view(IntSequence1, [42, 42, 10, 0, 42, 5, 0], list, None) result = most_common_viz(artifact) self.assertIsInstance(result, Visualization) self.assertEqual(result.type, qiime.core.type.Visualization) provenance = result.provenance self.assertIsInstance(provenance, Provenance) self.assertIsInstance(provenance.execution_uuid, uuid.UUID) self.assertTrue( provenance.executor_reference.startswith(most_common_viz.id)) self.assertEqual(provenance.artifact_uuids, {'ints': artifact.uuid}) self.assertEqual(provenance.parameter_references, {}) self.assertIsInstance(result.uuid, uuid.UUID) # TODO qiime.sdk.Visualization doesn't have an API to access its # contents yet. For now, save and assert the correct files are present. filepath = os.path.join(self.test_dir.name, 'visualization.qzv') result.save(filepath) with zipfile.ZipFile(filepath, mode='r') as zf: fps = set(zf.namelist()) expected = { 'visualization/VERSION', 'visualization/metadata.yaml', 'visualization/README.md', 'visualization/data/index.html', 'visualization/data/index.tsv' } self.assertEqual(fps, expected)
def test_visualizer_callable_output(self): artifact = Artifact._from_view(Mapping, { 'foo': 'abc', 'bar': 'def' }, dict, None) # Callable returns a value from `return_vals` return_vals = (True, False, [], {}, '', 0, 0.0) for return_val in return_vals: def func(output_dir: str, foo: dict) -> None: return return_val self.plugin.visualizers.register_function(func, {'foo': Mapping}, {}, '', '') visualizer = self.plugin.visualizers['func'] with self.assertRaisesRegex(TypeError, "should not return"): visualizer(foo=artifact) # Callable returns None (default function return) def func(output_dir: str, foo: dict) -> None: return None self.plugin.visualizers.register_function(func, {'foo': Mapping}, {}, '', '') visualizer = self.plugin.visualizers['func'] # Should not raise an exception output = visualizer(foo=artifact) self.assertIsInstance(output, Visualization)
def test_load_different_type_with_multiple_view_types(self): saved_artifact = Artifact._from_view(IntSequence1, [42, 42, 43, -999, 42], list) fp = os.path.join(self.test_dir.name, 'artifact.qza') saved_artifact.save(fp) artifact = Artifact.load(fp) self.assertEqual(artifact.type, IntSequence1) self.assertEqual(artifact.uuid, saved_artifact.uuid) self.assertEqual(artifact.view(list), [42, 42, 43, -999, 42]) self.assertEqual(artifact.view(list), [42, 42, 43, -999, 42]) self.assertEqual(artifact.view(collections.Counter), collections.Counter({ 42: 3, 43: 1, -999: 1 })) self.assertEqual(artifact.view(collections.Counter), collections.Counter({ 42: 3, 43: 1, -999: 1 }))
def test_async_with_multiple_outputs(self): split_ints = self.plugin.methods['split_ints'] split_ints_markdown = self.plugin.methods['split_ints_markdown'] artifact = Artifact._from_view(IntSequence1, [0, 42, -2, 43, 6], list) for method in split_ints, split_ints_markdown: future = method.async(artifact) self.assertIsInstance(future, concurrent.futures.Future) result = future.result() self.assertIsInstance(result, tuple) self.assertEqual(len(result), 2) for output_artifact in result: self.assertIsInstance(output_artifact, Artifact) self.assertEqual(output_artifact.type, IntSequence1) self.assertIsInstance(output_artifact.uuid, uuid.UUID) # Output artifacts have different UUIDs. self.assertNotEqual(result[0].uuid, result[1].uuid) # Index lookup. self.assertEqual(result[0].view(list), [0, 42]) self.assertEqual(result[1].view(list), [-2, 43, 6]) # Test properties of the `Results` object. self.assertIsInstance(result, Results) self.assertEqual(result.left.view(list), [0, 42]) self.assertEqual(result.right.view(list), [-2, 43, 6])
def test_load_and_save(self): fp1 = os.path.join(self.test_dir.name, 'artifact1.qza') fp2 = os.path.join(self.test_dir.name, 'artifact2.qza') artifact = Artifact._from_view(FourInts, [-1, 42, 0, 43], list) artifact.save(fp1) artifact = Artifact.load(fp1) # Overwriting its source file works. artifact.save(fp1) # Saving to a new file works. artifact.save(fp2) root_dir = str(artifact.uuid) expected = { 'VERSION', 'metadata.yaml', 'data/file1.txt', 'data/file2.txt', 'data/nested/file3.txt', 'data/nested/file4.txt' } self.assertArchiveMembers(fp1, root_dir, expected) root_dir = str(artifact.uuid) expected = { 'VERSION', 'metadata.yaml', 'data/file1.txt', 'data/file2.txt', 'data/nested/file3.txt', 'data/nested/file4.txt' } self.assertArchiveMembers(fp2, root_dir, expected)
def test_eq_same_uuid(self): fp = os.path.join(self.test_dir.name, 'artifact.qza') artifact1 = Artifact._from_view(FourInts, [-1, 42, 0, 43], list) artifact1.save(fp) artifact2 = Artifact.load(fp) self.assertEqual(artifact1, artifact2)
def test_load_artifact_as_visualization(self): artifact = Artifact._from_view(FourInts, [0, 0, 42, 1000], list, self.provenance) fp = os.path.join(self.test_dir.name, 'artifact.qza') artifact.save(fp) with self.assertRaisesRegex( TypeError, 'Artifact.*Visualization.load.*Artifact.load'): Visualization.load(fp)
def test_from_view_with_provenance(self): artifact = Artifact._from_view([-1, 42, 0, 43], FourInts, self.provenance) self.assertEqual(artifact.type, FourInts) self.assertEqual(artifact.provenance, self.provenance) self.assertIsInstance(artifact.uuid, uuid.UUID) self.assertEqual(artifact.view(list), [-1, 42, 0, 43]) self.assertEqual(artifact.view(list), [-1, 42, 0, 43])
def test_extract_artifact_as_visualization(self): artifact = Artifact._from_view(FourInts, [0, 0, 42, 1000], list, self.provenance) fp = os.path.join(self.test_dir.name, 'artifact.qza') artifact.save(fp) with self.assertRaisesRegex( TypeError, 'Visualization does not support.*FourInts'): Visualization.extract(fp, self.test_dir)
def test_from_view(self): artifact = Artifact._from_view(FourInts, [-1, 42, 0, 43], list) self.assertEqual(artifact.type, FourInts) # We don't know what the UUID is because it's generated within # Artifact._from_view. self.assertIsInstance(artifact.uuid, uuid.UUID) self.assertEqual(artifact.view(list), [-1, 42, 0, 43]) # Can produce same view if called again. self.assertEqual(artifact.view(list), [-1, 42, 0, 43])
def test_save_artifact_warning(self): artifact = Artifact._from_view(FourInts, [0, 0, 42, 1000], list, self.provenance) fp = os.path.join(self.test_dir.name, 'artifact.zip') with warnings.catch_warnings(record=True) as w: artifact.save(fp) self.assertEqual(len(w), 1) self.assertIsInstance(w[0].message, UserWarning) self.assertIn(Artifact.extension, str(w[0].message))
def test_from_view(self): artifact = Artifact._from_view(FourInts, [-1, 42, 0, 43], list, self.provenance_capture) self.assertEqual(artifact.type, FourInts) # We don't know what the UUID is because it's generated within # Artifact._from_view. self.assertIsInstance(artifact.uuid, uuid.UUID) self.assertEqual(artifact.view(list), [-1, 42, 0, 43]) # Can produce same view if called again. self.assertEqual(artifact.view(list), [-1, 42, 0, 43])
def test_peek(self): artifact = Artifact._from_view([0, 0, 42, 1000], FourInts, None) fp = os.path.join(self.test_dir.name, 'artifact.qza') artifact.save(fp) metadata = Artifact.peek(fp) self.assertIsInstance(metadata, ResultMetadata) self.assertEqual(metadata.type, FourInts) self.assertIsNone(metadata.provenance) self.assertEqual(metadata.uuid, artifact.uuid)
def test_peek(self): artifact = Artifact._from_view(FourInts, [0, 0, 42, 1000], list) fp = os.path.join(self.test_dir.name, 'artifact.qza') artifact.save(fp) metadata = Artifact.peek(fp) self.assertIsInstance(metadata, ResultMetadata) self.assertEqual(metadata.type, 'FourInts') self.assertEqual(metadata.uuid, str(artifact.uuid)) self.assertEqual(metadata.format, 'FourIntsDirectoryFormat')
def test_ne_different_type_same_uuid(self): artifact = Artifact._from_view(FourInts, [-1, 42, 0, 43], list) class Faker: @property def uuid(self): return artifact.uuid faker = Faker() self.assertNotEqual(artifact, faker)
def test_load(self): saved_artifact = Artifact._from_view(FourInts, [-1, 42, 0, 43], list) fp = os.path.join(self.test_dir.name, 'artifact.qza') saved_artifact.save(fp) artifact = Artifact.load(fp) self.assertEqual(artifact.type, FourInts) self.assertEqual(artifact.uuid, saved_artifact.uuid) self.assertEqual(artifact.view(list), [-1, 42, 0, 43]) self.assertEqual(artifact.view(list), [-1, 42, 0, 43])
def test_async(self): mapping_viz = self.plugin.visualizers['mapping_viz'] artifact1 = Artifact._from_view(Mapping, { 'foo': 'abc', 'bar': 'def' }, dict) artifact2 = Artifact._from_view(Mapping, { 'baz': 'abc', 'bazz': 'ghi' }, dict) future = mapping_viz. async (artifact1, artifact2, 'Key', 'Value') self.assertIsInstance(future, concurrent.futures.Future) result = future.result() # Test properties of the `Results` object. self.assertIsInstance(result, tuple) self.assertIsInstance(result, Results) self.assertEqual(len(result), 1) self.assertEqual(result.visualization, result[0]) result = result[0] self.assertIsInstance(result, Visualization) self.assertEqual(result.type, qiime.core.type.Visualization) self.assertIsInstance(result.uuid, uuid.UUID) # TODO qiime.sdk.Visualization doesn't have an API to access its # contents yet. For now, save and assert the correct files are present. filepath = os.path.join(self.test_dir.name, 'visualization.qzv') result.save(filepath) root_dir = str(result.uuid) expected = { 'VERSION', 'metadata.yaml', 'data/index.html', 'data/css/style.css' } self.assertArchiveMembers(filepath, root_dir, expected)
def test_peek_with_provenance(self): artifact = Artifact._from_view({'foo': 'bar', 'baz': 'bazz'}, Mapping, self.provenance) fp = os.path.join(self.test_dir.name, 'artifact.qza') artifact.save(fp) metadata = Artifact.peek(fp) self.assertIsInstance(metadata, ResultMetadata) self.assertEqual(metadata.type, Mapping) self.assertEqual(metadata.provenance, self.provenance) self.assertEqual(metadata.uuid, artifact.uuid)
def test_load(self): saved_artifact = Artifact._from_view([-1, 42, 0, 43], FourInts, None) fp = os.path.join(self.test_dir.name, 'artifact.qza') saved_artifact.save(fp) artifact = Artifact.load(fp) self.assertEqual(artifact.type, FourInts) self.assertIsNone(artifact.provenance) self.assertEqual(artifact.uuid, saved_artifact.uuid) self.assertEqual(artifact.view(list), [-1, 42, 0, 43]) self.assertEqual(artifact.view(list), [-1, 42, 0, 43])
def test_peek_artifact(self): artifact = Artifact._from_view(FourInts, [0, 0, 42, 1000], list, self.provenance) fp = os.path.join(self.test_dir.name, 'artifact.qza') artifact.save(fp) metadata = Result.peek(fp) self.assertIsInstance(metadata, ResultMetadata) self.assertEqual(metadata.type, FourInts) self.assertEqual(metadata.provenance, self.provenance) self.assertEqual(metadata.uuid, artifact.uuid)
def test_writer_transformer(self): for type in IntSequence1, IntSequence2: # `Artifact._from_view` invokes transformer that handles `list` -> # `SingleIntFormat`, because the `SingleIntDirectoryFormat` has # been registered as the directory format for the semantic type. # We didn't define a `SingleIntDirectoryFormat` -> # `SingleIntFormat` tranformer because # `model.SingleFileDirectoryFormat` handles that transformation for # us. artifact = Artifact._from_view(type, [1, 2, 42, -999, 42, 0], list) # Test that the directory and file format can be read again. self.assertEqual(artifact.view(list), [1, 2, 42, -999, 42, 0])
def test_load_artifact(self): saved_artifact = Artifact._from_view(FourInts, [-1, 42, 0, 43], list, self.provenance) fp = os.path.join(self.test_dir.name, 'artifact.qza') saved_artifact.save(fp) artifact = Result.load(fp) self.assertIsInstance(artifact, Artifact) self.assertEqual(artifact.type, FourInts) self.assertEqual(artifact.provenance, self.provenance) self.assertEqual(artifact.uuid, saved_artifact.uuid) self.assertEqual(artifact.view(list), [-1, 42, 0, 43])
def test_writer_transformer(self): # `Artifact._from_view` invokes transformer that handles `dict` -> # `MappingFormat`, because the `MappingDirectoryFormat` has # been registered as the directory format for the semantic type. # We didn't define a `MappingDirectoryFormat` -> # `MappingFormat` tranformer because # `model.SingleFileDirectoryFormat` handles that transformation for # us. artifact = Artifact._from_view(Mapping, { 'abc': 'cat', 'def': 'dog' }, dict) # Test that the directory and file format can be read again. self.assertEqual(artifact.view(dict), {'abc': 'cat', 'def': 'dog'})
def test_from_view_and_save(self): fp = os.path.join(self.test_dir.name, 'artifact.qza') # Using four-ints data layout because it has multiple files, some of # which are in a nested directory. artifact = Artifact._from_view(FourInts, [-1, 42, 0, 43], list) artifact.save(fp) root_dir = str(artifact.uuid) expected = { 'VERSION', 'metadata.yaml', 'data/file1.txt', 'data/file2.txt', 'data/nested/file3.txt', 'data/nested/file4.txt' } self.assertArchiveMembers(fp, root_dir, expected)
def test_from_view_different_type_with_multiple_view_types(self): artifact = Artifact._from_view([42, 42, 43, -999, 42], IntSequence1, None) self.assertEqual(artifact.type, IntSequence1) self.assertIsNone(artifact.provenance) self.assertIsInstance(artifact.uuid, uuid.UUID) self.assertEqual(artifact.view(list), [42, 42, 43, -999, 42]) self.assertEqual(artifact.view(list), [42, 42, 43, -999, 42]) self.assertEqual(artifact.view(collections.Counter), collections.Counter({42: 3, 43: 1, -999: 1})) self.assertEqual(artifact.view(collections.Counter), collections.Counter({42: 3, 43: 1, -999: 1}))
def test_roundtrip(self): fp1 = os.path.join(self.test_dir.name, 'artifact1.qza') fp2 = os.path.join(self.test_dir.name, 'artifact2.qza') artifact = Artifact._from_view([-1, 42, 0, 43], FourInts, self.provenance) artifact.save(fp1) artifact1 = Artifact.load(fp1) artifact1.save(fp2) artifact2 = Artifact.load(fp2) self.assertEqual(artifact1.type, artifact2.type) self.assertEqual(artifact1.provenance, artifact2.provenance) self.assertEqual(artifact1.uuid, artifact2.uuid) self.assertEqual(artifact1.view(list), artifact2.view(list)) self.assertEqual(artifact1.view(list), artifact2.view(list))
def test_extract(self): fp = os.path.join(self.test_dir.name, 'artifact.qza') artifact = Artifact._from_view(FourInts, [-1, 42, 0, 43], list) artifact.save(fp) root_dir = str(artifact.uuid) output_dir = os.path.join(self.test_dir.name, 'artifact-extract-test') result_dir = Artifact.extract(fp, output_dir=output_dir) self.assertEqual(result_dir, os.path.join(output_dir, root_dir)) expected = { 'VERSION', 'metadata.yaml', 'data/file1.txt', 'data/file2.txt', 'data/nested/file3.txt', 'data/nested/file4.txt' } self.assertExtractedArchiveMembers(output_dir, root_dir, expected)
def test_from_view_different_type_with_multiple_view_types(self): artifact = Artifact._from_view(IntSequence1, [42, 42, 43, -999, 42], list, self.provenance_capture) self.assertEqual(artifact.type, IntSequence1) self.assertIsInstance(artifact.uuid, uuid.UUID) self.assertEqual(artifact.view(list), [42, 42, 43, -999, 42]) self.assertEqual(artifact.view(list), [42, 42, 43, -999, 42]) self.assertEqual(artifact.view(collections.Counter), collections.Counter({42: 3, 43: 1, -999: 1})) self.assertEqual(artifact.view(collections.Counter), collections.Counter({42: 3, 43: 1, -999: 1}))
def test_async_with_multiple_outputs(self): split_ints = self.plugin.methods['split_ints'] split_ints_markdown = self.plugin.methods['split_ints_markdown'] artifact = Artifact._from_view(IntSequence1, [0, 42, -2, 43, 6], list, None) for method in split_ints, split_ints_markdown: future = method.async(artifact) self.assertIsInstance(future, concurrent.futures.Future) result = future.result() self.assertIsInstance(result, tuple) self.assertEqual(len(result), 2) for output_artifact in result: self.assertIsInstance(output_artifact, Artifact) self.assertEqual(output_artifact.type, IntSequence1) provenance = output_artifact.provenance self.assertIsInstance(provenance, Provenance) self.assertIsInstance(provenance.execution_uuid, uuid.UUID) self.assertTrue( provenance.executor_reference.startswith(method.id)) self.assertEqual(provenance.artifact_uuids, { 'ints': artifact.uuid }) self.assertEqual(provenance.parameter_references, {}) self.assertIsInstance(output_artifact.uuid, uuid.UUID) # Output artifacts have the same provenance. self.assertEqual(result[0].provenance, result[1].provenance) # Output artifacts have different UUIDs. self.assertNotEqual(result[0].uuid, result[1].uuid) # Index lookup. self.assertEqual(result[0].view(list), [0, 42]) self.assertEqual(result[1].view(list), [-2, 43, 6]) # Test properties of the `Results` object. self.assertIsInstance(result, Results) self.assertEqual(result.left.view(list), [0, 42]) self.assertEqual(result.right.view(list), [-2, 43, 6])
def test_roundtrip(self): fp1 = os.path.join(self.test_dir.name, 'artifact1.qza') fp2 = os.path.join(self.test_dir.name, 'artifact2.qza') artifact = Artifact._from_view(FourInts, [-1, 42, 0, 43], list) artifact.save(fp1) artifact1 = Artifact.load(fp1) artifact1.save(fp2) artifact2 = Artifact.load(fp2) self.assertEqual(artifact1.type, artifact2.type) self.assertEqual(artifact1.format, artifact2.format) self.assertEqual(artifact1.uuid, artifact2.uuid) self.assertEqual(artifact1.view(list), artifact2.view(list)) # double view to make sure multiple views can be taken self.assertEqual(artifact1.view(list), artifact2.view(list))
def test_load_different_type_with_multiple_view_types(self): saved_artifact = Artifact._from_view([42, 42, 43, -999, 42], IntSequence1, None) fp = os.path.join(self.test_dir.name, 'artifact.qza') saved_artifact.save(fp) artifact = Artifact.load(fp) self.assertEqual(artifact.type, IntSequence1) self.assertIsNone(artifact.provenance) self.assertEqual(artifact.uuid, saved_artifact.uuid) self.assertEqual(artifact.view(list), [42, 42, 43, -999, 42]) self.assertEqual(artifact.view(list), [42, 42, 43, -999, 42]) self.assertEqual(artifact.view(collections.Counter), collections.Counter({42: 3, 43: 1, -999: 1})) self.assertEqual(artifact.view(collections.Counter), collections.Counter({42: 3, 43: 1, -999: 1}))
def test_from_view_and_save(self): fp = os.path.join(self.test_dir.name, 'artifact.qza') # Using four-ints data layout because it has multiple files, some of # which are in a nested directory. artifact = Artifact._from_view([-1, 42, 0, 43], FourInts, self.provenance) artifact.save(fp) with zipfile.ZipFile(fp, mode='r') as zf: fps = set(zf.namelist()) expected = { 'artifact/VERSION', 'artifact/metadata.yaml', 'artifact/README.md', 'artifact/data/file1.txt', 'artifact/data/file2.txt', 'artifact/data/nested/file3.txt', 'artifact/data/nested/file4.txt' } self.assertEqual(fps, expected)
def test_extract(self): fp = os.path.join(self.test_dir.name, 'artifact.qza') artifact = Artifact._from_view([-1, 42, 0, 43], FourInts, self.provenance) artifact.save(fp) output_dir = os.path.join(self.test_dir.name, 'artifact-extract-test') result_dir = Artifact.extract(fp, output_dir=output_dir) self.assertEqual(result_dir, output_dir) contents = [ 'artifact/VERSION', 'artifact/metadata.yaml', 'artifact/README.md', 'artifact/data/file1.txt', 'artifact/data/file2.txt', 'artifact/data/nested/file3.txt', 'artifact/data/nested/file4.txt'] for fp in contents: expected_fp = os.path.join(output_dir, fp) self.assertTrue(os.path.exists(expected_fp), 'File %s was not extracted.' % fp)
def test_load_and_save(self): fp1 = os.path.join(self.test_dir.name, 'artifact1.qza') fp2 = os.path.join(self.test_dir.name, 'artifact2.qza') artifact = Artifact._from_view([-1, 42, 0, 43], FourInts, self.provenance) artifact.save(fp1) artifact = Artifact.load(fp1) # Overwriting its source file works. artifact.save(fp1) # Saving to a new file works. artifact.save(fp2) with zipfile.ZipFile(fp1, mode='r') as zf: fps = set(zf.namelist()) expected = { 'artifact1/VERSION', 'artifact1/metadata.yaml', 'artifact1/README.md', 'artifact1/data/file1.txt', 'artifact1/data/file2.txt', 'artifact1/data/nested/file3.txt', 'artifact1/data/nested/file4.txt' } self.assertEqual(fps, expected) with zipfile.ZipFile(fp2, mode='r') as zf: fps = set(zf.namelist()) expected = { 'artifact2/VERSION', 'artifact2/metadata.yaml', 'artifact2/README.md', 'artifact2/data/file1.txt', 'artifact2/data/file2.txt', 'artifact2/data/nested/file3.txt', 'artifact2/data/nested/file4.txt' } self.assertEqual(fps, expected)
def test_from_view_and_save(self): fp = os.path.join(self.test_dir.name, 'artifact.qza') # Using four-ints data layout because it has multiple files, some of # which are in a nested directory. artifact = Artifact._from_view(FourInts, [-1, 42, 0, 43], list, self.provenance_capture) artifact.save(fp) root_dir = str(artifact.uuid) expected = { 'VERSION', 'metadata.yaml', 'data/file1.txt', 'data/file2.txt', 'data/nested/file3.txt', 'data/nested/file4.txt', 'provenance/metadata.yaml', 'provenance/VERSION', 'provenance/action/action.yaml' } self.assertArchiveMembers(fp, root_dir, expected)
def _test_to_script_or_to_markdown(self, to_method, template_lines): # These methods are so similar that it makes sense to have a helper # that can test either one instead of duplicating a bunch of code. workflow = Workflow.from_markdown(self.markdown_fp) artifact_fp1 = os.path.join(self.test_dir.name, 'artifact1.qza') artifact = Artifact._from_view( [-1, 42, 0, 43, 43], qiime.core.testing.TestType, None) artifact.save(artifact_fp1) artifact_fp2 = os.path.join(self.test_dir.name, 'artifact2.qza') artifact = Artifact._from_view( [1, 2, 100], qiime.core.testing.TestType, None) artifact.save(artifact_fp2) artifact_fp3 = os.path.join(self.test_dir.name, 'artifact3.qza') job = to_method( workflow, input_artifact_filepaths={ 'input1': artifact_fp1, 'input2': artifact_fp2 }, parameter_references={ 'param1': 99, 'param2': -999, }, output_artifact_filepaths={ 'concatenated_inputs': artifact_fp3 } ) provenance_lines = [ "provenance = Provenance(", "parameters={", "'param2': -999" ] setup_lines = [ "import builtins", "input1 = Artifact.load(%r).view(builtins.list)" % artifact_fp1, "input2 = Artifact.load(%r).view(builtins.list)" % artifact_fp2, "param1 = 99", "param2 = -999" ] teardown_lines = [ "artifact = Artifact._from_view(concatenated_inputs, 'TestType', " "provenance)", "artifact.save(%r)" % artifact_fp3 ] for expected_lines in (provenance_lines, setup_lines, template_lines, teardown_lines): for expected_line in expected_lines: self.assertIn(expected_line, job.code) self.assertIsInstance(job.uuid, uuid.UUID) self.assertEqual(job.uuid.version, 4) self.assertEqual( job.input_artifact_filepaths, {'input1': artifact_fp1, 'input2': artifact_fp2}) self.assertEqual( job.parameter_references, {'param1': 99, 'param2': -999}) self.assertEqual( job.output_artifact_filepaths, {'concatenated_inputs': artifact_fp3})