예제 #1
0
    def test_export_container_files(self, sdk_mock, mocker, base):

        exporter_mock = mocker.patch(
            "container_export.FileExporter.from_client")
        origin = base(files=[])
        side_effect = []
        for i in range(10):
            origin.files.append(flywheel.FileEntry(name=str(i)))
            side_effect.append((str(i) if i % 2 == 0 else None,
                                True if i % 3 == 0 else False))
        exporter_mock.return_value.find_or_create_file_copy.side_effect = side_effect

        found, created, failed = ContainerExporter.export_container_files(
            sdk_mock, origin, "other", None)
        assert failed == ["1", "3", "5", "7", "9"]
        assert created == ["0", "6"]
        assert found == ["2", "4", "8"]
예제 #2
0
def test_get_container_metadata():
    info_dict = {
        'python': {
            'spam': 'eggs'
        },
        'header': {
            'dicom': {
                'PatientID': 'FLYWHEEL'
            }
        }
    }
    file = flywheel.FileEntry(type='file',
                              id='test_id',
                              info=info_dict,
                              modality='MR')
    file._parent = flywheel.Acquisition(parents=flywheel.ContainerParents(
        project='project_id'))
    export_dict = {
        'file': {
            'whitelist':
            ['info.python.spam', 'modality', 'info.header.dicom.PatientID']
        }
    }
    expected_output = {
        'info': {
            'python': {
                'spam': 'eggs'
            },
            'export': {
                'origin_id': util.hash_value('test_id', salt='project_id')
            }
        },
        'modality': 'MR'
    }
    output = get_container_metadata(origin_container=file,
                                    export_dict=export_dict)
    assert output == expected_output
예제 #3
0
    def test_container_files(self):
        fw = self.fw

        acquisition = flywheel.Acquisition(label=self.rand_string(),
                                           session=self.session_id)
        acquisition_id = fw.add_acquisition(acquisition)

        # Upload a file
        poem = 'Turning and turning in the widening gyre'
        fw.upload_file_to_container(acquisition_id,
                                    flywheel.FileSpec('yeats.txt', poem))

        # Check that the file was added to the acquisition
        c_acquisition = fw.get_container(acquisition_id)
        self.assertEqual(len(c_acquisition.files), 1)
        self.assertEqual(c_acquisition.files[0].name, 'yeats.txt')
        self.assertEqual(c_acquisition.files[0].size, 40)
        self.assertEqual(c_acquisition.files[0].mimetype, 'text/plain')

        # Download the file and check content
        self.assertDownloadFileTextEquals(
            fw.download_file_from_container_as_data, acquisition_id,
            'yeats.txt', poem)

        # Test unauthorized download with ticket for the file
        self.assertDownloadFileTextEqualsWithTicket(
            fw.get_container_download_url, acquisition_id, 'yeats.txt', poem)

        # Test file attributes
        self.assertEqual(c_acquisition.files[0].modality, None)
        self.assertEmpty(c_acquisition.files[0].classification)
        self.assertEqual(c_acquisition.files[0].type, 'text')

        resp = fw.modify_container_file(
            acquisition_id, 'yeats.txt',
            flywheel.FileEntry(modality='modality', type='type'))

        # Check that no jobs were triggered, and attrs were modified
        self.assertEqual(resp.jobs_spawned, 0)

        c_acquisition = fw.get_container(acquisition_id)
        self.assertEqual(c_acquisition.files[0].modality, "modality")
        self.assertEmpty(c_acquisition.files[0].classification)
        self.assertEqual(c_acquisition.files[0].type, 'type')

        # Test classifications
        resp = fw.modify_container_file_classification(
            acquisition_id, 'yeats.txt', {
                'modality': 'modality2',
                'replace': {
                    'Custom': ['measurement1', 'measurement2'],
                }
            })
        self.assertEqual(resp.modified, 1)
        self.assertEqual(resp.jobs_spawned, 0)

        c_acquisition = fw.get_container(acquisition_id)
        self.assertEqual(c_acquisition.files[0].modality, 'modality2')
        self.assertEqual(c_acquisition.files[0].classification,
                         {'Custom': ['measurement1', 'measurement2']})

        resp = fw.set_container_file_classification(acquisition_id,
                                                    'yeats.txt',
                                                    {'Custom': ['HelloWorld']})
        self.assertEqual(resp.modified, 1)
        self.assertEqual(resp.jobs_spawned, 0)

        resp = fw.delete_container_file_classification_fields(
            acquisition_id, 'yeats.txt', {'Custom': ['measurement2']})
        self.assertEqual(resp.modified, 1)
        self.assertEqual(resp.jobs_spawned, 0)

        c_acquisition = fw.get_container(acquisition_id)
        self.assertEqual(c_acquisition.files[0].classification, {
            'Custom': ['measurement1', 'HelloWorld'],
        })

        # Test file info
        self.assertEmpty(c_acquisition.files[0].info)
        fw.replace_container_file_info(acquisition_id, 'yeats.txt', {
            'a': 1,
            'b': 2,
            'c': 3,
            'd': 4
        })

        fw.set_container_file_info(acquisition_id, 'yeats.txt', {'c': 5})

        c_acquisition = fw.get_container(acquisition_id)
        self.assertEqual(c_acquisition.files[0].info['a'], 1)
        self.assertEqual(c_acquisition.files[0].info['b'], 2)
        self.assertEqual(c_acquisition.files[0].info['c'], 5)
        self.assertEqual(c_acquisition.files[0].info['d'], 4)

        fw.delete_container_file_info_fields(acquisition_id, 'yeats.txt',
                                             ['c', 'd'])
        c_acquisition = fw.get_container(acquisition_id)
        self.assertEqual(c_acquisition.files[0].info['a'], 1)
        self.assertEqual(c_acquisition.files[0].info['b'], 2)
        self.assertNotIn('c', c_acquisition.files[0].info)
        self.assertNotIn('d', c_acquisition.files[0].info)

        fw.replace_container_file_info(acquisition_id, 'yeats.txt', {})
        c_acquisition = fw.get_container(acquisition_id)
        self.assertEmpty(c_acquisition.files[0].info)

        # Delete file
        fw.delete_container_file(acquisition_id, 'yeats.txt')
        c_acquisition = fw.get_container(acquisition_id)
        self.assertEmpty(c_acquisition.files)

        # Delete acquisition
        fw.delete_container(acquisition_id)
예제 #4
0
    def test_project_files(self):
        fw = self.fw

        project = flywheel.Project(label=self.rand_string(),
                                   group=self.group_id)
        self.project_id = project_id = fw.add_project(project)

        # Upload a file
        poem = 'The ceremony of innocence is drowned;'
        fw.upload_file_to_project(project_id,
                                  flywheel.FileSpec('yeats.txt', poem))

        # Check that the file was added to the project
        r_project = fw.get_project(project_id)
        self.assertEqual(len(r_project.files), 1)
        self.assertEqual(r_project.files[0].name, 'yeats.txt')
        self.assertEqual(r_project.files[0].size, 37)
        self.assertEqual(r_project.files[0].mimetype, 'text/plain')

        # Download the file and check content
        self.assertDownloadFileTextEquals(
            fw.download_file_from_project_as_data, project_id, 'yeats.txt',
            poem)

        # Test unauthorized download with ticket for the file
        self.assertDownloadFileTextEqualsWithTicket(
            fw.get_project_download_url, project_id, 'yeats.txt', poem)

        # Test file attributes
        self.assertEqual(r_project.files[0].modality, None)
        self.assertEmpty(r_project.files[0].classification)
        self.assertEqual(r_project.files[0].type, 'text')

        resp = fw.modify_project_file(
            project_id, 'yeats.txt',
            flywheel.FileEntry(modality='modality', type='type'))

        # Check that no jobs were triggered, and attrs were modified
        self.assertEqual(resp.jobs_spawned, 0)

        r_project = fw.get_project(project_id)
        self.assertEqual(r_project.files[0].modality, "modality")
        self.assertEmpty(r_project.files[0].classification)
        self.assertEqual(r_project.files[0].type, 'type')

        # Test classifications
        resp = fw.modify_project_file_classification(
            project_id, 'yeats.txt',
            {'replace': {
                'Custom': ['measurement1', 'measurement2'],
            }})
        self.assertEqual(resp.modified, 1)
        self.assertEqual(resp.jobs_spawned, 0)

        r_project = fw.get_project(project_id)
        self.assertEqual(r_project.files[0].classification,
                         {'Custom': ['measurement1', 'measurement2']})

        resp = fw.modify_project_file_classification(
            project_id, 'yeats.txt', {
                'add': {
                    'Custom': ['HelloWorld'],
                },
                'delete': {
                    'Custom': ['measurement2']
                }
            })
        self.assertEqual(resp.modified, 1)
        self.assertEqual(resp.jobs_spawned, 0)

        r_project = fw.get_project(project_id)
        self.assertEqual(r_project.files[0].classification, {
            'Custom': ['measurement1', 'HelloWorld'],
        })

        # Test file info
        self.assertEmpty(r_project.files[0].info)
        fw.replace_project_file_info(project_id, 'yeats.txt', {
            'a': 1,
            'b': 2,
            'c': 3,
            'd': 4
        })

        fw.set_project_file_info(project_id, 'yeats.txt', {'c': 5})

        r_project = fw.get_project(project_id)
        self.assertEqual(r_project.files[0].info['a'], 1)
        self.assertEqual(r_project.files[0].info['b'], 2)
        self.assertEqual(r_project.files[0].info['c'], 5)
        self.assertEqual(r_project.files[0].info['d'], 4)

        fw.delete_project_file_info_fields(project_id, 'yeats.txt', ['c', 'd'])
        r_project = fw.get_project(project_id)
        self.assertEqual(r_project.files[0].info['a'], 1)
        self.assertEqual(r_project.files[0].info['b'], 2)
        self.assertNotIn('c', r_project.files[0].info)
        self.assertNotIn('d', r_project.files[0].info)

        fw.replace_project_file_info(project_id, 'yeats.txt', {})
        r_project = fw.get_project(project_id)
        self.assertEmpty(r_project.files[0].info)

        # Delete file
        fw.delete_project_file(project_id, 'yeats.txt')
        r_project = fw.get_project(project_id)
        self.assertEmpty(r_project.files)
예제 #5
0
def test_file_exporter():
    classification_dict = {
        "Custom": ["Spam", "Eggs"],
        "Measurement": "T1",
        "Intent": ["Invalid", "Structural"],
        "InvalidKey": "InvalidValue",
        "Features": [],
    }
    info_dict = {
        "header": {
            "dicom": {
                "InstanceNumber": 1,
                "PatientPosition": "HFS",
                "PatientSex": "O",
                "PatientID": "SPAM_Subject",
                "SeriesDescription": "SPAM_study",
            }
        }
    }
    dicom_map_dict = {
        "PatientID": "Flywheel",
    }
    file_entry = flywheel.FileEntry(
        modality="MR",
        classification=classification_dict,
        name='fi:l*e/p"a?t>h|.t<xt',
        id="test_id",
        info=info_dict,
    )

    def upload_func(container_id, file, metadata):
        return container_id, file, metadata

    file_exporter = FileExporter(file_entry, MR_CLASSIFICATION_SCHEMA,
                                 upload_func, dicom_map_dict)
    # test get_modaility
    assert FileExporter.get_modality(file_entry) == "MR"
    assert (FileExporter.get_modality(
        flywheel.FileEntry(name="test.mriqc.qa.html")) == "MR")
    # test classification
    assert file_exporter.classification == {
        "Custom": ["Spam", "Eggs"],
        "Measurement": ["T1"],
        "Intent": ["Structural"],
    }
    exp_info = {
        "header": {
            "dicom": {
                "InstanceNumber": 1,
                "PatientPosition": "HFS",
                "PatientSex": "O",
                "PatientID": "Flywheel",
                "SeriesDescription": "SPAM_study",
            }
        },
        "export": {
            "origin_id":
            "0fb27832c685c35889ba3653994bae061237518c40ed57d3b41eae17bf923137"
        },
    }
    # test info
    assert file_exporter.info == exp_info
    # test fw_dicom_header
    assert file_exporter.fw_dicom_header == exp_info["header"]["dicom"]
    # test setter
    exp_info["header"]["dicom"].pop("SeriesDescription")
    file_exporter.fw_dicom_header = exp_info["header"]["dicom"]
    assert file_exporter.info == exp_info
    with pytest.raises(ValueError):
        file_exporter.fw_dicom_header = "not a dict"
    # test from_client
    mock_client = MagicMock(spec=dir(flywheel.Client))
    mock_client.get_modality = lambda x: {
        "classification": MR_CLASSIFICATION_SCHEMA
    }
    mock_client.upload_file_to_container = upload_func
    file_exporter = FileExporter.from_client(mock_client, file_entry,
                                             dicom_map_dict)
    # test get_classification_schema
    assert (file_exporter.get_classification_schema(
        mock_client, "MR") == MR_CLASSIFICATION_SCHEMA)

    # Raise with a status that is not 404 and will not get caught with backoff
    def raise_func(*args):
        raise flywheel.ApiException(status=400)

    mock_client.get_modality = raise_func
    with pytest.raises(flywheel.ApiException):
        file_exporter.get_classification_schema(mock_client, "MR")
    # test find_file_copy
    file_entry.info = file_exporter.info.copy()
    export_parent = flywheel.Acquisition(label="SPAM_study",
                                         files=[file_entry],
                                         id="test_id")
    assert file_exporter.find_file_copy(export_parent)
    # test create_file_copy
    setattr(file_entry, "download", lambda x: None)
    assert file_exporter.create_file_copy(export_parent)
    # test find_or_create_file_copy
    fn, created = file_exporter.find_or_create_file_copy(export_parent)
    assert not created
    export_parent.files = list()
    fn, created = file_exporter.find_or_create_file_copy(export_parent)
    assert created
    # test update_dicom
    _, temp_path = tempfile.mkstemp()

    assert not file_exporter.update_dicom(temp_path)
    file_exporter.fw_dicom_header = dict()
    # returns local path if no header is defined
    assert file_exporter.update_dicom(temp_path)
    os.remove(temp_path)