class TestGetDestination:
@pytest.mark.parametrize(
"parent,raising",
[
(flywheel.Subject(label="test"), does_not_raise()),
(flywheel.Session(label="test"), does_not_raise()),
(flywheel.Group(label="test"), pytest.raises(ValueError)),
(flywheel.Project(label="test"), pytest.raises(ValueError)),
(flywheel.Acquisition(label="test"), pytest.raises(ValueError)),
],
)
def test_container(self, sdk_mock, parent, raising):
container = flywheel.models.analysis_output.AnalysisOutput(
parent=parent, id="test"
)
sdk_mock.get_analysis.return_value = container
sdk_mock.get.return_value = parent
with raising:
dest = get_destination(sdk_mock, "test")
sdk_mock.get_analysis.assert_called_once_with("test")
# assert dest.__class__ == parent.__class__
assert isinstance(dest, parent.__class__)
def test_analysis_does_not_exist(self, sdk_mock):
container = flywheel.models.analysis_output.AnalysisOutput(
parent=flywheel.Project(), id="test"
)
sdk_mock.get.side_effect = flywheel.rest.ApiException(status=404)
sdk_mock.get_analysis.return_value = container
with pytest.raises(flywheel.rest.ApiException):
dest = get_destination(sdk_mock, "test")
assert isinstance(dest, flywheel.Project)
def test_project_doesnt_exist(self, sdk_mock):
sdk_mock.get_project_rules.side_effect = flywheel.rest.ApiException(status=403)
project = flywheel.Project(
id="test", label="test", parents=flywheel.ContainerParents(group="test")
)
with pytest.raises(flywheel.rest.ApiException):
validate_gear_rules(sdk_mock, project)
def test_get_project_exists(sdk_mock, raises, exists, caplog):
caplog.set_level(logging.DEBUG)
if exists:
# SDK mock instance of unittest.mock.MagicMock
# Mock return value of fw.lookup()
sdk_mock.lookup.return_value = flywheel.Project(label="test_proj")
else:
sdk_mock.lookup.return_value = None
sdk_mock.lookup.side_effect = flywheel.rest.ApiException(status=404)
# test
with raises:
proj = get_project(sdk_mock, "test/test_proj")
# assertions
sdk_mock.lookup.assert_called_once_with("test/test_proj")
if exists:
assert isinstance(proj, flywheel.Project)
assert len(caplog.record_tuples) == 1
assert caplog.records[0].message == "Found Project test_proj, id None"
assert proj.label == "test_proj"
else:
assert proj is None
assert len(caplog.record_tuples) == 1
assert caplog.records[0].message == "Project test/test_proj not found"
def test_create_project_in_root_mode(self):
fw = self.fw
group = fw.get_group(self.group_id)
# Remove permissions from group
user_id = group.permissions[0].id
fw.delete_group_user_permission(self.group_id, user_id)
# Check that permission was removed successfully
group = fw.get_group(self.group_id)
self.assertEmpty(group.permissions)
# Assert that we get a 403 error attempting to create a project without permission
project_name = self.rand_string()
project = flywheel.Project(label=self.rand_string(),
group=self.group_id)
try:
fw.add_project(project)
self.fail('Expected ApiException creating project!')
except flywheel.ApiException as e:
self.assertEqual(e.status, 403)
# We shouldn't get an error in root mode
project_id = self.fw_root.add_project(project)
self.assertNotEmpty(project_id)
try:
# Delete implicit permission from the project
fw.delete_project_user_permission(project_id, user_id)
# Should get a 403 error trying to retrieve the project
try:
fw.get_project(project_id)
self.fail('Expected ApiException retrieving project!')
except flywheel.ApiException as e:
self.assertEqual(e.status, 403)
# Should be able to retrieve as root
r_project = self.fw_root.get_project(project_id)
self.assertEqual(r_project.label, project.label)
# Should be in list retrieved as root
r_project.info = {}
r_project.info_exists = False
r_project.analyses = None
projects = self.fw_root.get_all_projects()
self.assertIn(r_project, projects)
# Should not show up in normal list
projects = fw.get_all_projects()
self.assertNotIn(r_project, projects)
finally:
# Always cleanup project
self.fw_root.delete_project(project_id)
def test_analysis_does_not_exist(self, sdk_mock):
container = flywheel.models.analysis_output.AnalysisOutput(
parent=flywheel.Project(), id="test"
)
sdk_mock.get.side_effect = flywheel.rest.ApiException(status=404)
sdk_mock.get_analysis.return_value = container
with pytest.raises(flywheel.rest.ApiException):
dest = get_destination(sdk_mock, "test")
assert isinstance(dest, flywheel.Project)
def test_gear_rules(self, sdk_mock, rules, returns):
sdk_mock.get_project_rules.return_value = rules
project = flywheel.Project(
id="test", label="test", parents=flywheel.ContainerParents(group="test")
)
errors = []
val = validate_gear_rules(sdk_mock, project)
sdk_mock.get_project_rules.assert_called_once_with(project.id)
assert val == returns
def test_from_gear_context(self, mocker, origin):
gear_context_mock = MagicMock(
spec=dir(flywheel_gear_toolkit.GearToolkitContext))
log_patch = mocker.patch("container_export.ExportLog")
hierarchy_patch = mocker.patch(
"container_export.ContainerExporter.get_hierarchy")
validate_patch = mocker.patch("container_export.validate_context")
export_proj = flywheel.Project(label="export")
archive_proj = (flywheel.Project(label="archive"), )
validate_patch.return_value = [
export_proj,
archive_proj,
origin,
]
exporter = ContainerExporter.from_gear_context(gear_context_mock)
assert exporter.origin_container == origin
log_patch.assert_called_once_with(export_proj, archive_proj)
hierarchy_patch.assert_called_once_with(origin)
def test_project_errors(self):
fw = self.fw
# Try to create project without group id
try:
project = flywheel.Project(label=self.rand_string())
fw.add_project(project)
self.fail('Expected ApiException creating invalid project!')
except flywheel.ApiException as e:
self.assertEqual(e.status, 400)
# Try to get a project that doesn't exist
try:
fw.get_project('DOES_NOT_EXIST')
self.fail('Expected ApiException retrieving invalid project!')
except flywheel.ApiException as e:
self.assertEqual(e.status, 404)
def test_create_project_without_perm(self):
fw = self.fw
group = fw.get_group(self.group_id)
# Remove permissions from group
user_id = group.permissions[0].id
fw.delete_group_user_permission(self.group_id, user_id)
# Check that permission was removed successfully
group = fw.get_group(self.group_id)
self.assertEmpty(group.permissions)
# Assert that we get a 403 error attempting to create a project without permission
project_name = self.rand_string()
project = flywheel.Project(label=self.rand_string(),
group=self.group_id)
project_id = fw.add_project(project)
self.assertNotEmpty(project_id)
try:
# Delete implicit permission from the project
fw.delete_project_user_permission(project_id, user_id)
# retrieve the project
r_project = fw.get_project(project_id)
self.assertEqual(r_project.label, project.label)
r_project.info = {}
r_project.info_exists = False
r_project.analyses = None
# Should be in list retrieved with exhaustive
projects = self.fw.get_all_projects(exhaustive=True)
self.assertIn(r_project, projects)
# Should not show up in normal list
projects = fw.get_all_projects()
self.assertNotIn(r_project, projects)
finally:
# Always cleanup project
self.fw.delete_project(project_id)
def test_project_analysis(self):
fw = self.fw
project = flywheel.Project(group=self.group_id,
label=self.rand_string())
# Add
self.project_id = project_id = fw.add_project(project)
self.assertNotEmpty(project_id)
poem = 'The Second Coming! Hardly are those words out'
fw.upload_file_to_project(project_id,
flywheel.FileSpec('yeats.txt', poem))
file_ref = flywheel.FileReference(id=project_id,
type='project',
name='yeats.txt')
analysis = flywheel.AnalysisInput(label=self.rand_string(),
description=self.rand_string(),
inputs=[file_ref])
# Add
analysis_id = fw.add_project_analysis(project_id, analysis)
self.assertNotEmpty(analysis_id)
# Get the list of analyses in the project
analyses = fw.get_project_analyses(project_id)
self.assertEqual(len(analyses), 1)
r_analysis = analyses[0]
self.assertEqual(r_analysis.id, analysis_id)
self.assertEmpty(r_analysis.job)
self.assertTimestampBeforeNow(r_analysis.created)
self.assertGreaterEqual(r_analysis.modified, r_analysis.created)
self.assertEqual(len(r_analysis.inputs), 1)
self.assertEqual(r_analysis.inputs[0].name, 'yeats.txt')
def test_validate_calls(self, mocker, gear_context, config, call_num, caplog):
caplog.set_level(logging.INFO)
mock_proj = (
flywheel.Project(
label="test",
parents=flywheel.models.container_parents.ContainerParents(
group="test"
),
),
)
gear_context.config = config
get_proj_mock = mocker.patch("validate.get_project")
get_proj_mock.return_value = mock_proj
get_dest_mock = mocker.patch("validate.get_destination")
get_dest_mock.return_value = flywheel.Subject(label="test")
check_exported_mock = mocker.patch("validate.container_needs_export")
check_exported_mock.return_value = True
check_gear_rules_mock = mocker.patch("validate.validate_gear_rules")
check_gear_rules_mock.return_value = True
export, archive, dest = validate_context(gear_context)
assert get_proj_mock.call_count == call_num
get_dest_mock.assert_called_once_with(gear_context.client, "test")
check_exported_mock.assert_called_once_with(
flywheel.Subject(label="test"), config
)
msgs = [rec.message for rec in caplog.records]
if "check_gear_rules" in config:
check_gear_rules_mock.assert_called_once_with(
gear_context.client, mock_proj
)
assert "No enabled rules were found. Moving on..." in msgs
else:
check_gear_rules_mock.assert_not_called()
assert "No enabled rules were found. Moving on..." not in msgs
def test_transfer_log_class():
metadata_path = DATA_ROOT / 'test-transfer-log.xlsx'
config_path = DATA_ROOT / 'test-transfer-log-template.yml'
mock_view_path = DATA_ROOT / 'test-fw-view.csv'
config = transfer_log.load_config_file(config_path)
test_transfer_log = transfer_log.TransferLog(client=None, config=config, transfer_log_path=metadata_path,
project_id=None, case_insensitive=True, match_containers_once=True)
expected_error_dict = {'row_or_id': None, 'subject.label': None, 'session.timestamp': None, 'session.label': None,
'file.modality': None, 'error': None, 'path': None, 'type': None, 'resolved': False,
'label': None}
assert expected_error_dict == test_transfer_log.error_dict
# manually initialize
test_transfer_log.load_metadata_table()
mock_view_df = pd.read_csv(
mock_view_path, dtype={'subject.label': 'object'}
)
mock_view_dict_list = transfer_log.format_flywheel_table(mock_view_df.to_dict(orient='records'))
test_transfer_log.create_flywheel_table(mock_view_dict_list)
project = flywheel.Project(group='test_group', label='test_project')
test_transfer_log.get_path_dict(project)
# Test match_containers_once option
test_transfer_log.match_df_records()
error_df = test_transfer_log.get_error_df()
assert len(error_df) == 4
expected_error_messages = [
'acquisition in flywheel not present in transfer_log',
'1 more records in flywheel than in transfer_log',
'acquisition in transfer_log not present in flywheel'
]
for item in expected_error_messages:
assert item in error_df.values
# Test match_containers_once of False
test_transfer_log.match_containers_once = False
test_transfer_log.match_df_records()
error_df = test_transfer_log.get_error_df()
assert len(error_df) == 5
def test_projects(self):
fw = self.fw
project_name = self.rand_string()
project = flywheel.Project(label=project_name,
group=self.group_id,
description="This is a description",
info={'some-key': 37})
# Add
self.project_id = project_id = fw.add_project(project)
self.assertNotEmpty(project_id)
# Get
r_project = fw.get_project(project_id)
self.assertEqual(r_project.id, project_id)
self.assertEqual(r_project.label, project_name)
self.assertEqual(r_project.description, project.description)
self.assertIn('some-key', r_project.info)
self.assertEqual(r_project.info['some-key'], 37)
self.assertTimestampBeforeNow(r_project.created)
self.assertGreaterEqual(r_project.modified, r_project.created)
# Generic Get is equivalent
self.assertEqual(fw.get(project_id).to_dict(), r_project.to_dict())
# Get All
projects = fw.get_all_projects()
r_project.info = {}
# TODO: Should we be setting this, shouldn't it be coming from api?
r_project.info_exists = True
r_project.analyses = None
self.assertIn(r_project, projects)
# Modify
new_name = self.rand_string()
project_mod = flywheel.Project(label=new_name,
info={'another-key': 52})
fw.modify_project(project_id, project_mod)
changed_project = fw.get_project(project_id)
self.assertEqual(changed_project.label, new_name)
self.assertIn('some-key', changed_project.info)
self.assertIn('another-key', changed_project.info)
self.assertEqual(changed_project.info['another-key'], 52)
self.assertEqual(changed_project.created, r_project.created)
self.assertGreater(changed_project.modified, r_project.modified)
# Notes, Tags
message = 'This is a note'
fw.add_project_note(project_id, message)
tag = 'example-tag'
fw.add_project_tag(project_id, tag)
# Replace Info
fw.replace_project_info(project_id, {'foo': 3, 'bar': 'qaz'})
# Set Info
fw.set_project_info(project_id, {'foo': 42, 'hello': 'world'})
# Check
r_project = fw.get_project(project_id)
self.assertEqual(len(r_project.notes), 1)
self.assertEqual(r_project.notes[0].text, message)
self.assertEqual(len(r_project.tags), 1)
self.assertEqual(r_project.tags[0], tag)
self.assertEqual(r_project.info['foo'], 42)
self.assertEqual(r_project.info['bar'], 'qaz')
self.assertEqual(r_project.info['hello'], 'world')
# Delete info fields
fw.delete_project_info_fields(project_id, ['foo', 'bar'])
r_project = fw.get_project(project_id)
self.assertNotIn('foo', r_project.info)
self.assertNotIn('bar', r_project.info)
self.assertEqual(r_project.info['hello'], 'world')
# Delete
fw.delete_project(project_id)
self.project_id = None
projects = fw.get_all_projects()
self.assertNotIn(r_project, projects)
def test_project_files(self):
fw = self.fw
project = flywheel.Project(label=self.rand_string(),
group=self.group_id)
self.project_id = project_id = fw.add_project(project)
# Upload a file
poem = 'The ceremony of innocence is drowned;'
fw.upload_file_to_project(project_id,
flywheel.FileSpec('yeats.txt', poem))
# Check that the file was added to the project
r_project = fw.get_project(project_id)
self.assertEqual(len(r_project.files), 1)
self.assertEqual(r_project.files[0].name, 'yeats.txt')
self.assertEqual(r_project.files[0].size, 37)
self.assertEqual(r_project.files[0].mimetype, 'text/plain')
# Download the file and check content
self.assertDownloadFileTextEquals(
fw.download_file_from_project_as_data, project_id, 'yeats.txt',
poem)
# Test unauthorized download with ticket for the file
self.assertDownloadFileTextEqualsWithTicket(
fw.get_project_download_url, project_id, 'yeats.txt', poem)
# Test file attributes
self.assertEqual(r_project.files[0].modality, None)
self.assertEmpty(r_project.files[0].classification)
self.assertEqual(r_project.files[0].type, 'text')
resp = fw.modify_project_file(
project_id, 'yeats.txt',
flywheel.FileEntry(modality='modality', type='type'))
# Check that no jobs were triggered, and attrs were modified
self.assertEqual(resp.jobs_spawned, 0)
r_project = fw.get_project(project_id)
self.assertEqual(r_project.files[0].modality, "modality")
self.assertEmpty(r_project.files[0].classification)
self.assertEqual(r_project.files[0].type, 'type')
# Test classifications
resp = fw.modify_project_file_classification(
project_id, 'yeats.txt',
{'replace': {
'Custom': ['measurement1', 'measurement2'],
}})
self.assertEqual(resp.modified, 1)
self.assertEqual(resp.jobs_spawned, 0)
r_project = fw.get_project(project_id)
self.assertEqual(r_project.files[0].classification,
{'Custom': ['measurement1', 'measurement2']})
resp = fw.modify_project_file_classification(
project_id, 'yeats.txt', {
'add': {
'Custom': ['HelloWorld'],
},
'delete': {
'Custom': ['measurement2']
}
})
self.assertEqual(resp.modified, 1)
self.assertEqual(resp.jobs_spawned, 0)
r_project = fw.get_project(project_id)
self.assertEqual(r_project.files[0].classification, {
'Custom': ['measurement1', 'HelloWorld'],
})
# Test file info
self.assertEmpty(r_project.files[0].info)
fw.replace_project_file_info(project_id, 'yeats.txt', {
'a': 1,
'b': 2,
'c': 3,
'd': 4
})
fw.set_project_file_info(project_id, 'yeats.txt', {'c': 5})
r_project = fw.get_project(project_id)
self.assertEqual(r_project.files[0].info['a'], 1)
self.assertEqual(r_project.files[0].info['b'], 2)
self.assertEqual(r_project.files[0].info['c'], 5)
self.assertEqual(r_project.files[0].info['d'], 4)
fw.delete_project_file_info_fields(project_id, 'yeats.txt', ['c', 'd'])
r_project = fw.get_project(project_id)
self.assertEqual(r_project.files[0].info['a'], 1)
self.assertEqual(r_project.files[0].info['b'], 2)
self.assertNotIn('c', r_project.files[0].info)
self.assertNotIn('d', r_project.files[0].info)
fw.replace_project_file_info(project_id, 'yeats.txt', {})
r_project = fw.get_project(project_id)
self.assertEmpty(r_project.files[0].info)
# Delete file
fw.delete_project_file(project_id, 'yeats.txt')
r_project = fw.get_project(project_id)
self.assertEmpty(r_project.files)
class TestValidateContext:
@pytest.mark.parametrize(
"config, call_num",
[
(
{
"export_project": "test1",
"force_export": True,
"check_gear_rules": True,
},
1,
),
(
{
"export_project": "test1",
"force_export": False,
"check_gear_rules": True,
},
1,
),
(
{
"export_project": "test1",
"archive_project": "test2",
"force_export": True,
},
2,
),
(
{
"export_project": "test1",
"archive_project": "test2",
"force_export": False,
},
2,
),
],
)
def test_validate_calls(self, mocker, gear_context, config, call_num, caplog):
caplog.set_level(logging.INFO)
mock_proj = (
flywheel.Project(
label="test",
parents=flywheel.models.container_parents.ContainerParents(
group="test"
),
),
)
gear_context.config = config
get_proj_mock = mocker.patch("validate.get_project")
get_proj_mock.return_value = mock_proj
get_dest_mock = mocker.patch("validate.get_destination")
get_dest_mock.return_value = flywheel.Subject(label="test")
check_exported_mock = mocker.patch("validate.container_needs_export")
check_exported_mock.return_value = True
check_gear_rules_mock = mocker.patch("validate.validate_gear_rules")
check_gear_rules_mock.return_value = True
export, archive, dest = validate_context(gear_context)
assert get_proj_mock.call_count == call_num
get_dest_mock.assert_called_once_with(gear_context.client, "test")
check_exported_mock.assert_called_once_with(
flywheel.Subject(label="test"), config
)
msgs = [rec.message for rec in caplog.records]
if "check_gear_rules" in config:
check_gear_rules_mock.assert_called_once_with(
gear_context.client, mock_proj
)
assert "No enabled rules were found. Moving on..." in msgs
else:
check_gear_rules_mock.assert_not_called()
assert "No enabled rules were found. Moving on..." not in msgs
@pytest.mark.parametrize(
"proj",
[
{"export_project": flywheel.Project(label="export")},
{"archive_project": flywheel.Project(label="archvie")},
],
)
def test_get_proj_errors(self, mocker, sdk_mock, gear_context, caplog, proj):
gear_context.config = {"export_project": "test", "archive_project": "test"}
gear_context.config.update(proj)
def get_proj_side_effect(fw, project):
if not hasattr(project, "label"):
raise flywheel.rest.ApiException(status=600)
get_proj_mock = mocker.patch("validate.get_project")
get_proj_mock.side_effect = get_proj_side_effect
with pytest.raises(SystemExit):
export, archive, dest = validate_context(gear_context)
assert all(
[
rec.levelno == logging.ERROR
for rec in caplog.get_records(when="teardown")
]
)
@pytest.mark.parametrize(
"to_mock,val,to_err,raises,log",
[
(
["get_project"],
[None],
dict(),
True,
"Export project needs to be specified",
),
(
["get_project", "validate_gear_rules"],
[flywheel.Project(label="test"), False],
dict(),
True,
"Aborting Session Export: test has ENABLED GEAR RULES and 'check_gear_rules' == True. If you would like to force the export regardless of enabled gear rules re-run.py the gear with 'check_gear_rules' == False. Warning: Doing so may result in undesired behavior.",
),
(
["get_project", "validate_gear_rules", "get_destination"],
["test", True, None],
{"get_destination": ValueError("test")},
True,
"Could not find destination with id test",
),
(
["get_project", "validate_gear_rules", "get_destination"],
["test", True, None],
{"get_destination": ApiException(status=20)},
True,
"Could not find destination with id test",
),
(
[
"get_project",
"validate_gear_rules",
"get_destination",
"container_needs_export",
],
["test", True, flywheel.Session(label="test"), False],
dict(),
True,
"session test has already been exported and <force_export> = False. Nothing to do!",
),
],
)
def test_errors(
self, mocker, to_mock, val, to_err, raises, log, caplog, gear_context
):
gear_context.config = {
"destination": {"id": "test", "container_type": "session", "label": "test"},
"export_project": "test",
"archive_project": "test",
"check_gear_rules": True,
}
mocks = {}
for mock, val in zip(to_mock, val):
mocks[mock] = mocker.patch(f"validate.{mock}")
mocks[mock].return_value = val
for mock, err in to_err.items():
if mock in mocks:
mocks[mock].side_effect = err
if raises:
my_raise = pytest.raises(SystemExit)
else:
my_raise = does_not_raise()
with my_raise:
validate_context(gear_context)
def test_container_hierarchy():
hierarchy_dict = {
"group":
flywheel.Group(id="test_group", label="Test Group"),
"project":
flywheel.Project(label="test_project"),
"subject":
flywheel.Subject(label="test_subject", sex="other"),
"session":
flywheel.Session(
age=31000000,
label="test_session",
weight=50,
),
}
# test from_dict
test_hierarchy = ContainerHierarchy.from_dict(hierarchy_dict)
# test deepcopy
assert deepcopy(test_hierarchy) != test_hierarchy
# test path
assert test_hierarchy.path == "test_group/test_project/test_subject/test_session"
# test parent
assert test_hierarchy.parent.label == "test_subject"
# test from_container
mock_client = MagicMock(spec=dir(flywheel.Client))
parent_dict = dict()
for item in ("group", "project", "subject"):
value = hierarchy_dict.copy().get(item)
parent_dict[item] = item
setattr(mock_client, f"get_{item}", lambda x: value)
session = flywheel.Session(age=31000000, label="test_session", weight=50)
session.parents = parent_dict
assert (ContainerHierarchy.from_container(
mock_client, session).container_type == "session")
# test _get_container
assert test_hierarchy._get_container(None, None, None) is None
with pytest.raises(ValueError) as exc:
test_hierarchy._get_container(None, "garbage", "garbage_id")
assert str(exc) == "Cannot get a container of type garbage"
mock_client = MagicMock(spec=dir(flywheel.Client))
mock_client.get_session = lambda x: x
assert (test_hierarchy._get_container(mock_client, "session",
"session_id") == "session_id")
# test container_type
assert test_hierarchy.container_type == "session"
# test dicom_map
exp_map = {
"PatientWeight": 50,
"PatientAge": "011M",
"ClinicalTrialTimePointDescription": "test_session",
"PatientSex": "O",
"PatientID": "test_subject",
}
assert exp_map == test_hierarchy.dicom_map
# test get
assert test_hierarchy.get("container_type") == "session"
# test get_patient_sex_from_subject
assert test_hierarchy.get_patientsex_from_subject(flywheel.Subject()) == ""
# test get_patientage_from_session
assert test_hierarchy.get_patientage_from_session(
flywheel.Session()) is None
# test get_child_hierarchy
test_acquisition = flywheel.Acquisition(label="test_acquisition")
acq_hierarchy = test_hierarchy.get_child_hierarchy(test_acquisition)
assert acq_hierarchy.dicom_map[
"SeriesDescription"] == test_acquisition.label
# test get_parent_hierarchy
parent_hierarchy = test_hierarchy.get_parent_hierarchy()
assert parent_hierarchy.container_type == "subject"