def test_export_container_files(self, sdk_mock, mocker, base):
exporter_mock = mocker.patch(
"container_export.FileExporter.from_client")
origin = base(files=[])
side_effect = []
for i in range(10):
origin.files.append(flywheel.FileEntry(name=str(i)))
side_effect.append((str(i) if i % 2 == 0 else None,
True if i % 3 == 0 else False))
exporter_mock.return_value.find_or_create_file_copy.side_effect = side_effect
found, created, failed = ContainerExporter.export_container_files(
sdk_mock, origin, "other", None)
assert failed == ["1", "3", "5", "7", "9"]
assert created == ["0", "6"]
assert found == ["2", "4", "8"]
def test_get_container_metadata():
info_dict = {
'python': {
'spam': 'eggs'
},
'header': {
'dicom': {
'PatientID': 'FLYWHEEL'
}
}
}
file = flywheel.FileEntry(type='file',
id='test_id',
info=info_dict,
modality='MR')
file._parent = flywheel.Acquisition(parents=flywheel.ContainerParents(
project='project_id'))
export_dict = {
'file': {
'whitelist':
['info.python.spam', 'modality', 'info.header.dicom.PatientID']
}
}
expected_output = {
'info': {
'python': {
'spam': 'eggs'
},
'export': {
'origin_id': util.hash_value('test_id', salt='project_id')
}
},
'modality': 'MR'
}
output = get_container_metadata(origin_container=file,
export_dict=export_dict)
assert output == expected_output
def test_container_files(self):
fw = self.fw
acquisition = flywheel.Acquisition(label=self.rand_string(),
session=self.session_id)
acquisition_id = fw.add_acquisition(acquisition)
# Upload a file
poem = 'Turning and turning in the widening gyre'
fw.upload_file_to_container(acquisition_id,
flywheel.FileSpec('yeats.txt', poem))
# Check that the file was added to the acquisition
c_acquisition = fw.get_container(acquisition_id)
self.assertEqual(len(c_acquisition.files), 1)
self.assertEqual(c_acquisition.files[0].name, 'yeats.txt')
self.assertEqual(c_acquisition.files[0].size, 40)
self.assertEqual(c_acquisition.files[0].mimetype, 'text/plain')
# Download the file and check content
self.assertDownloadFileTextEquals(
fw.download_file_from_container_as_data, acquisition_id,
'yeats.txt', poem)
# Test unauthorized download with ticket for the file
self.assertDownloadFileTextEqualsWithTicket(
fw.get_container_download_url, acquisition_id, 'yeats.txt', poem)
# Test file attributes
self.assertEqual(c_acquisition.files[0].modality, None)
self.assertEmpty(c_acquisition.files[0].classification)
self.assertEqual(c_acquisition.files[0].type, 'text')
resp = fw.modify_container_file(
acquisition_id, 'yeats.txt',
flywheel.FileEntry(modality='modality', type='type'))
# Check that no jobs were triggered, and attrs were modified
self.assertEqual(resp.jobs_spawned, 0)
c_acquisition = fw.get_container(acquisition_id)
self.assertEqual(c_acquisition.files[0].modality, "modality")
self.assertEmpty(c_acquisition.files[0].classification)
self.assertEqual(c_acquisition.files[0].type, 'type')
# Test classifications
resp = fw.modify_container_file_classification(
acquisition_id, 'yeats.txt', {
'modality': 'modality2',
'replace': {
'Custom': ['measurement1', 'measurement2'],
}
})
self.assertEqual(resp.modified, 1)
self.assertEqual(resp.jobs_spawned, 0)
c_acquisition = fw.get_container(acquisition_id)
self.assertEqual(c_acquisition.files[0].modality, 'modality2')
self.assertEqual(c_acquisition.files[0].classification,
{'Custom': ['measurement1', 'measurement2']})
resp = fw.set_container_file_classification(acquisition_id,
'yeats.txt',
{'Custom': ['HelloWorld']})
self.assertEqual(resp.modified, 1)
self.assertEqual(resp.jobs_spawned, 0)
resp = fw.delete_container_file_classification_fields(
acquisition_id, 'yeats.txt', {'Custom': ['measurement2']})
self.assertEqual(resp.modified, 1)
self.assertEqual(resp.jobs_spawned, 0)
c_acquisition = fw.get_container(acquisition_id)
self.assertEqual(c_acquisition.files[0].classification, {
'Custom': ['measurement1', 'HelloWorld'],
})
# Test file info
self.assertEmpty(c_acquisition.files[0].info)
fw.replace_container_file_info(acquisition_id, 'yeats.txt', {
'a': 1,
'b': 2,
'c': 3,
'd': 4
})
fw.set_container_file_info(acquisition_id, 'yeats.txt', {'c': 5})
c_acquisition = fw.get_container(acquisition_id)
self.assertEqual(c_acquisition.files[0].info['a'], 1)
self.assertEqual(c_acquisition.files[0].info['b'], 2)
self.assertEqual(c_acquisition.files[0].info['c'], 5)
self.assertEqual(c_acquisition.files[0].info['d'], 4)
fw.delete_container_file_info_fields(acquisition_id, 'yeats.txt',
['c', 'd'])
c_acquisition = fw.get_container(acquisition_id)
self.assertEqual(c_acquisition.files[0].info['a'], 1)
self.assertEqual(c_acquisition.files[0].info['b'], 2)
self.assertNotIn('c', c_acquisition.files[0].info)
self.assertNotIn('d', c_acquisition.files[0].info)
fw.replace_container_file_info(acquisition_id, 'yeats.txt', {})
c_acquisition = fw.get_container(acquisition_id)
self.assertEmpty(c_acquisition.files[0].info)
# Delete file
fw.delete_container_file(acquisition_id, 'yeats.txt')
c_acquisition = fw.get_container(acquisition_id)
self.assertEmpty(c_acquisition.files)
# Delete acquisition
fw.delete_container(acquisition_id)
def test_project_files(self):
fw = self.fw
project = flywheel.Project(label=self.rand_string(),
group=self.group_id)
self.project_id = project_id = fw.add_project(project)
# Upload a file
poem = 'The ceremony of innocence is drowned;'
fw.upload_file_to_project(project_id,
flywheel.FileSpec('yeats.txt', poem))
# Check that the file was added to the project
r_project = fw.get_project(project_id)
self.assertEqual(len(r_project.files), 1)
self.assertEqual(r_project.files[0].name, 'yeats.txt')
self.assertEqual(r_project.files[0].size, 37)
self.assertEqual(r_project.files[0].mimetype, 'text/plain')
# Download the file and check content
self.assertDownloadFileTextEquals(
fw.download_file_from_project_as_data, project_id, 'yeats.txt',
poem)
# Test unauthorized download with ticket for the file
self.assertDownloadFileTextEqualsWithTicket(
fw.get_project_download_url, project_id, 'yeats.txt', poem)
# Test file attributes
self.assertEqual(r_project.files[0].modality, None)
self.assertEmpty(r_project.files[0].classification)
self.assertEqual(r_project.files[0].type, 'text')
resp = fw.modify_project_file(
project_id, 'yeats.txt',
flywheel.FileEntry(modality='modality', type='type'))
# Check that no jobs were triggered, and attrs were modified
self.assertEqual(resp.jobs_spawned, 0)
r_project = fw.get_project(project_id)
self.assertEqual(r_project.files[0].modality, "modality")
self.assertEmpty(r_project.files[0].classification)
self.assertEqual(r_project.files[0].type, 'type')
# Test classifications
resp = fw.modify_project_file_classification(
project_id, 'yeats.txt',
{'replace': {
'Custom': ['measurement1', 'measurement2'],
}})
self.assertEqual(resp.modified, 1)
self.assertEqual(resp.jobs_spawned, 0)
r_project = fw.get_project(project_id)
self.assertEqual(r_project.files[0].classification,
{'Custom': ['measurement1', 'measurement2']})
resp = fw.modify_project_file_classification(
project_id, 'yeats.txt', {
'add': {
'Custom': ['HelloWorld'],
},
'delete': {
'Custom': ['measurement2']
}
})
self.assertEqual(resp.modified, 1)
self.assertEqual(resp.jobs_spawned, 0)
r_project = fw.get_project(project_id)
self.assertEqual(r_project.files[0].classification, {
'Custom': ['measurement1', 'HelloWorld'],
})
# Test file info
self.assertEmpty(r_project.files[0].info)
fw.replace_project_file_info(project_id, 'yeats.txt', {
'a': 1,
'b': 2,
'c': 3,
'd': 4
})
fw.set_project_file_info(project_id, 'yeats.txt', {'c': 5})
r_project = fw.get_project(project_id)
self.assertEqual(r_project.files[0].info['a'], 1)
self.assertEqual(r_project.files[0].info['b'], 2)
self.assertEqual(r_project.files[0].info['c'], 5)
self.assertEqual(r_project.files[0].info['d'], 4)
fw.delete_project_file_info_fields(project_id, 'yeats.txt', ['c', 'd'])
r_project = fw.get_project(project_id)
self.assertEqual(r_project.files[0].info['a'], 1)
self.assertEqual(r_project.files[0].info['b'], 2)
self.assertNotIn('c', r_project.files[0].info)
self.assertNotIn('d', r_project.files[0].info)
fw.replace_project_file_info(project_id, 'yeats.txt', {})
r_project = fw.get_project(project_id)
self.assertEmpty(r_project.files[0].info)
# Delete file
fw.delete_project_file(project_id, 'yeats.txt')
r_project = fw.get_project(project_id)
self.assertEmpty(r_project.files)
def test_file_exporter():
classification_dict = {
"Custom": ["Spam", "Eggs"],
"Measurement": "T1",
"Intent": ["Invalid", "Structural"],
"InvalidKey": "InvalidValue",
"Features": [],
}
info_dict = {
"header": {
"dicom": {
"InstanceNumber": 1,
"PatientPosition": "HFS",
"PatientSex": "O",
"PatientID": "SPAM_Subject",
"SeriesDescription": "SPAM_study",
}
}
}
dicom_map_dict = {
"PatientID": "Flywheel",
}
file_entry = flywheel.FileEntry(
modality="MR",
classification=classification_dict,
name='fi:l*e/p"a?t>h|.t<xt',
id="test_id",
info=info_dict,
)
def upload_func(container_id, file, metadata):
return container_id, file, metadata
file_exporter = FileExporter(file_entry, MR_CLASSIFICATION_SCHEMA,
upload_func, dicom_map_dict)
# test get_modaility
assert FileExporter.get_modality(file_entry) == "MR"
assert (FileExporter.get_modality(
flywheel.FileEntry(name="test.mriqc.qa.html")) == "MR")
# test classification
assert file_exporter.classification == {
"Custom": ["Spam", "Eggs"],
"Measurement": ["T1"],
"Intent": ["Structural"],
}
exp_info = {
"header": {
"dicom": {
"InstanceNumber": 1,
"PatientPosition": "HFS",
"PatientSex": "O",
"PatientID": "Flywheel",
"SeriesDescription": "SPAM_study",
}
},
"export": {
"origin_id":
"0fb27832c685c35889ba3653994bae061237518c40ed57d3b41eae17bf923137"
},
}
# test info
assert file_exporter.info == exp_info
# test fw_dicom_header
assert file_exporter.fw_dicom_header == exp_info["header"]["dicom"]
# test setter
exp_info["header"]["dicom"].pop("SeriesDescription")
file_exporter.fw_dicom_header = exp_info["header"]["dicom"]
assert file_exporter.info == exp_info
with pytest.raises(ValueError):
file_exporter.fw_dicom_header = "not a dict"
# test from_client
mock_client = MagicMock(spec=dir(flywheel.Client))
mock_client.get_modality = lambda x: {
"classification": MR_CLASSIFICATION_SCHEMA
}
mock_client.upload_file_to_container = upload_func
file_exporter = FileExporter.from_client(mock_client, file_entry,
dicom_map_dict)
# test get_classification_schema
assert (file_exporter.get_classification_schema(
mock_client, "MR") == MR_CLASSIFICATION_SCHEMA)
# Raise with a status that is not 404 and will not get caught with backoff
def raise_func(*args):
raise flywheel.ApiException(status=400)
mock_client.get_modality = raise_func
with pytest.raises(flywheel.ApiException):
file_exporter.get_classification_schema(mock_client, "MR")
# test find_file_copy
file_entry.info = file_exporter.info.copy()
export_parent = flywheel.Acquisition(label="SPAM_study",
files=[file_entry],
id="test_id")
assert file_exporter.find_file_copy(export_parent)
# test create_file_copy
setattr(file_entry, "download", lambda x: None)
assert file_exporter.create_file_copy(export_parent)
# test find_or_create_file_copy
fn, created = file_exporter.find_or_create_file_copy(export_parent)
assert not created
export_parent.files = list()
fn, created = file_exporter.find_or_create_file_copy(export_parent)
assert created
# test update_dicom
_, temp_path = tempfile.mkstemp()
assert not file_exporter.update_dicom(temp_path)
file_exporter.fw_dicom_header = dict()
# returns local path if no header is defined
assert file_exporter.update_dicom(temp_path)
os.remove(temp_path)