class TestGetDestination: @pytest.mark.parametrize( "parent,raising", [ (flywheel.Subject(label="test"), does_not_raise()), (flywheel.Session(label="test"), does_not_raise()), (flywheel.Group(label="test"), pytest.raises(ValueError)), (flywheel.Project(label="test"), pytest.raises(ValueError)), (flywheel.Acquisition(label="test"), pytest.raises(ValueError)), ], ) def test_container(self, sdk_mock, parent, raising): container = flywheel.models.analysis_output.AnalysisOutput( parent=parent, id="test" ) sdk_mock.get_analysis.return_value = container sdk_mock.get.return_value = parent with raising: dest = get_destination(sdk_mock, "test") sdk_mock.get_analysis.assert_called_once_with("test") # assert dest.__class__ == parent.__class__ assert isinstance(dest, parent.__class__) def test_analysis_does_not_exist(self, sdk_mock): container = flywheel.models.analysis_output.AnalysisOutput( parent=flywheel.Project(), id="test" ) sdk_mock.get.side_effect = flywheel.rest.ApiException(status=404) sdk_mock.get_analysis.return_value = container with pytest.raises(flywheel.rest.ApiException): dest = get_destination(sdk_mock, "test") assert isinstance(dest, flywheel.Project)
def test_project_doesnt_exist(self, sdk_mock): sdk_mock.get_project_rules.side_effect = flywheel.rest.ApiException(status=403) project = flywheel.Project( id="test", label="test", parents=flywheel.ContainerParents(group="test") ) with pytest.raises(flywheel.rest.ApiException): validate_gear_rules(sdk_mock, project)
def test_get_project_exists(sdk_mock, raises, exists, caplog): caplog.set_level(logging.DEBUG) if exists: # SDK mock instance of unittest.mock.MagicMock # Mock return value of fw.lookup() sdk_mock.lookup.return_value = flywheel.Project(label="test_proj") else: sdk_mock.lookup.return_value = None sdk_mock.lookup.side_effect = flywheel.rest.ApiException(status=404) # test with raises: proj = get_project(sdk_mock, "test/test_proj") # assertions sdk_mock.lookup.assert_called_once_with("test/test_proj") if exists: assert isinstance(proj, flywheel.Project) assert len(caplog.record_tuples) == 1 assert caplog.records[0].message == "Found Project test_proj, id None" assert proj.label == "test_proj" else: assert proj is None assert len(caplog.record_tuples) == 1 assert caplog.records[0].message == "Project test/test_proj not found"
def test_create_project_in_root_mode(self): fw = self.fw group = fw.get_group(self.group_id) # Remove permissions from group user_id = group.permissions[0].id fw.delete_group_user_permission(self.group_id, user_id) # Check that permission was removed successfully group = fw.get_group(self.group_id) self.assertEmpty(group.permissions) # Assert that we get a 403 error attempting to create a project without permission project_name = self.rand_string() project = flywheel.Project(label=self.rand_string(), group=self.group_id) try: fw.add_project(project) self.fail('Expected ApiException creating project!') except flywheel.ApiException as e: self.assertEqual(e.status, 403) # We shouldn't get an error in root mode project_id = self.fw_root.add_project(project) self.assertNotEmpty(project_id) try: # Delete implicit permission from the project fw.delete_project_user_permission(project_id, user_id) # Should get a 403 error trying to retrieve the project try: fw.get_project(project_id) self.fail('Expected ApiException retrieving project!') except flywheel.ApiException as e: self.assertEqual(e.status, 403) # Should be able to retrieve as root r_project = self.fw_root.get_project(project_id) self.assertEqual(r_project.label, project.label) # Should be in list retrieved as root r_project.info = {} r_project.info_exists = False r_project.analyses = None projects = self.fw_root.get_all_projects() self.assertIn(r_project, projects) # Should not show up in normal list projects = fw.get_all_projects() self.assertNotIn(r_project, projects) finally: # Always cleanup project self.fw_root.delete_project(project_id)
def test_analysis_does_not_exist(self, sdk_mock): container = flywheel.models.analysis_output.AnalysisOutput( parent=flywheel.Project(), id="test" ) sdk_mock.get.side_effect = flywheel.rest.ApiException(status=404) sdk_mock.get_analysis.return_value = container with pytest.raises(flywheel.rest.ApiException): dest = get_destination(sdk_mock, "test") assert isinstance(dest, flywheel.Project)
def test_gear_rules(self, sdk_mock, rules, returns): sdk_mock.get_project_rules.return_value = rules project = flywheel.Project( id="test", label="test", parents=flywheel.ContainerParents(group="test") ) errors = [] val = validate_gear_rules(sdk_mock, project) sdk_mock.get_project_rules.assert_called_once_with(project.id) assert val == returns
def test_from_gear_context(self, mocker, origin): gear_context_mock = MagicMock( spec=dir(flywheel_gear_toolkit.GearToolkitContext)) log_patch = mocker.patch("container_export.ExportLog") hierarchy_patch = mocker.patch( "container_export.ContainerExporter.get_hierarchy") validate_patch = mocker.patch("container_export.validate_context") export_proj = flywheel.Project(label="export") archive_proj = (flywheel.Project(label="archive"), ) validate_patch.return_value = [ export_proj, archive_proj, origin, ] exporter = ContainerExporter.from_gear_context(gear_context_mock) assert exporter.origin_container == origin log_patch.assert_called_once_with(export_proj, archive_proj) hierarchy_patch.assert_called_once_with(origin)
def test_project_errors(self): fw = self.fw # Try to create project without group id try: project = flywheel.Project(label=self.rand_string()) fw.add_project(project) self.fail('Expected ApiException creating invalid project!') except flywheel.ApiException as e: self.assertEqual(e.status, 400) # Try to get a project that doesn't exist try: fw.get_project('DOES_NOT_EXIST') self.fail('Expected ApiException retrieving invalid project!') except flywheel.ApiException as e: self.assertEqual(e.status, 404)
def test_create_project_without_perm(self): fw = self.fw group = fw.get_group(self.group_id) # Remove permissions from group user_id = group.permissions[0].id fw.delete_group_user_permission(self.group_id, user_id) # Check that permission was removed successfully group = fw.get_group(self.group_id) self.assertEmpty(group.permissions) # Assert that we get a 403 error attempting to create a project without permission project_name = self.rand_string() project = flywheel.Project(label=self.rand_string(), group=self.group_id) project_id = fw.add_project(project) self.assertNotEmpty(project_id) try: # Delete implicit permission from the project fw.delete_project_user_permission(project_id, user_id) # retrieve the project r_project = fw.get_project(project_id) self.assertEqual(r_project.label, project.label) r_project.info = {} r_project.info_exists = False r_project.analyses = None # Should be in list retrieved with exhaustive projects = self.fw.get_all_projects(exhaustive=True) self.assertIn(r_project, projects) # Should not show up in normal list projects = fw.get_all_projects() self.assertNotIn(r_project, projects) finally: # Always cleanup project self.fw.delete_project(project_id)
def test_project_analysis(self): fw = self.fw project = flywheel.Project(group=self.group_id, label=self.rand_string()) # Add self.project_id = project_id = fw.add_project(project) self.assertNotEmpty(project_id) poem = 'The Second Coming! Hardly are those words out' fw.upload_file_to_project(project_id, flywheel.FileSpec('yeats.txt', poem)) file_ref = flywheel.FileReference(id=project_id, type='project', name='yeats.txt') analysis = flywheel.AnalysisInput(label=self.rand_string(), description=self.rand_string(), inputs=[file_ref]) # Add analysis_id = fw.add_project_analysis(project_id, analysis) self.assertNotEmpty(analysis_id) # Get the list of analyses in the project analyses = fw.get_project_analyses(project_id) self.assertEqual(len(analyses), 1) r_analysis = analyses[0] self.assertEqual(r_analysis.id, analysis_id) self.assertEmpty(r_analysis.job) self.assertTimestampBeforeNow(r_analysis.created) self.assertGreaterEqual(r_analysis.modified, r_analysis.created) self.assertEqual(len(r_analysis.inputs), 1) self.assertEqual(r_analysis.inputs[0].name, 'yeats.txt')
def test_validate_calls(self, mocker, gear_context, config, call_num, caplog): caplog.set_level(logging.INFO) mock_proj = ( flywheel.Project( label="test", parents=flywheel.models.container_parents.ContainerParents( group="test" ), ), ) gear_context.config = config get_proj_mock = mocker.patch("validate.get_project") get_proj_mock.return_value = mock_proj get_dest_mock = mocker.patch("validate.get_destination") get_dest_mock.return_value = flywheel.Subject(label="test") check_exported_mock = mocker.patch("validate.container_needs_export") check_exported_mock.return_value = True check_gear_rules_mock = mocker.patch("validate.validate_gear_rules") check_gear_rules_mock.return_value = True export, archive, dest = validate_context(gear_context) assert get_proj_mock.call_count == call_num get_dest_mock.assert_called_once_with(gear_context.client, "test") check_exported_mock.assert_called_once_with( flywheel.Subject(label="test"), config ) msgs = [rec.message for rec in caplog.records] if "check_gear_rules" in config: check_gear_rules_mock.assert_called_once_with( gear_context.client, mock_proj ) assert "No enabled rules were found. Moving on..." in msgs else: check_gear_rules_mock.assert_not_called() assert "No enabled rules were found. Moving on..." not in msgs
def test_transfer_log_class(): metadata_path = DATA_ROOT / 'test-transfer-log.xlsx' config_path = DATA_ROOT / 'test-transfer-log-template.yml' mock_view_path = DATA_ROOT / 'test-fw-view.csv' config = transfer_log.load_config_file(config_path) test_transfer_log = transfer_log.TransferLog(client=None, config=config, transfer_log_path=metadata_path, project_id=None, case_insensitive=True, match_containers_once=True) expected_error_dict = {'row_or_id': None, 'subject.label': None, 'session.timestamp': None, 'session.label': None, 'file.modality': None, 'error': None, 'path': None, 'type': None, 'resolved': False, 'label': None} assert expected_error_dict == test_transfer_log.error_dict # manually initialize test_transfer_log.load_metadata_table() mock_view_df = pd.read_csv( mock_view_path, dtype={'subject.label': 'object'} ) mock_view_dict_list = transfer_log.format_flywheel_table(mock_view_df.to_dict(orient='records')) test_transfer_log.create_flywheel_table(mock_view_dict_list) project = flywheel.Project(group='test_group', label='test_project') test_transfer_log.get_path_dict(project) # Test match_containers_once option test_transfer_log.match_df_records() error_df = test_transfer_log.get_error_df() assert len(error_df) == 4 expected_error_messages = [ 'acquisition in flywheel not present in transfer_log', '1 more records in flywheel than in transfer_log', 'acquisition in transfer_log not present in flywheel' ] for item in expected_error_messages: assert item in error_df.values # Test match_containers_once of False test_transfer_log.match_containers_once = False test_transfer_log.match_df_records() error_df = test_transfer_log.get_error_df() assert len(error_df) == 5
def test_projects(self): fw = self.fw project_name = self.rand_string() project = flywheel.Project(label=project_name, group=self.group_id, description="This is a description", info={'some-key': 37}) # Add self.project_id = project_id = fw.add_project(project) self.assertNotEmpty(project_id) # Get r_project = fw.get_project(project_id) self.assertEqual(r_project.id, project_id) self.assertEqual(r_project.label, project_name) self.assertEqual(r_project.description, project.description) self.assertIn('some-key', r_project.info) self.assertEqual(r_project.info['some-key'], 37) self.assertTimestampBeforeNow(r_project.created) self.assertGreaterEqual(r_project.modified, r_project.created) # Generic Get is equivalent self.assertEqual(fw.get(project_id).to_dict(), r_project.to_dict()) # Get All projects = fw.get_all_projects() r_project.info = {} # TODO: Should we be setting this, shouldn't it be coming from api? r_project.info_exists = True r_project.analyses = None self.assertIn(r_project, projects) # Modify new_name = self.rand_string() project_mod = flywheel.Project(label=new_name, info={'another-key': 52}) fw.modify_project(project_id, project_mod) changed_project = fw.get_project(project_id) self.assertEqual(changed_project.label, new_name) self.assertIn('some-key', changed_project.info) self.assertIn('another-key', changed_project.info) self.assertEqual(changed_project.info['another-key'], 52) self.assertEqual(changed_project.created, r_project.created) self.assertGreater(changed_project.modified, r_project.modified) # Notes, Tags message = 'This is a note' fw.add_project_note(project_id, message) tag = 'example-tag' fw.add_project_tag(project_id, tag) # Replace Info fw.replace_project_info(project_id, {'foo': 3, 'bar': 'qaz'}) # Set Info fw.set_project_info(project_id, {'foo': 42, 'hello': 'world'}) # Check r_project = fw.get_project(project_id) self.assertEqual(len(r_project.notes), 1) self.assertEqual(r_project.notes[0].text, message) self.assertEqual(len(r_project.tags), 1) self.assertEqual(r_project.tags[0], tag) self.assertEqual(r_project.info['foo'], 42) self.assertEqual(r_project.info['bar'], 'qaz') self.assertEqual(r_project.info['hello'], 'world') # Delete info fields fw.delete_project_info_fields(project_id, ['foo', 'bar']) r_project = fw.get_project(project_id) self.assertNotIn('foo', r_project.info) self.assertNotIn('bar', r_project.info) self.assertEqual(r_project.info['hello'], 'world') # Delete fw.delete_project(project_id) self.project_id = None projects = fw.get_all_projects() self.assertNotIn(r_project, projects)
def test_project_files(self): fw = self.fw project = flywheel.Project(label=self.rand_string(), group=self.group_id) self.project_id = project_id = fw.add_project(project) # Upload a file poem = 'The ceremony of innocence is drowned;' fw.upload_file_to_project(project_id, flywheel.FileSpec('yeats.txt', poem)) # Check that the file was added to the project r_project = fw.get_project(project_id) self.assertEqual(len(r_project.files), 1) self.assertEqual(r_project.files[0].name, 'yeats.txt') self.assertEqual(r_project.files[0].size, 37) self.assertEqual(r_project.files[0].mimetype, 'text/plain') # Download the file and check content self.assertDownloadFileTextEquals( fw.download_file_from_project_as_data, project_id, 'yeats.txt', poem) # Test unauthorized download with ticket for the file self.assertDownloadFileTextEqualsWithTicket( fw.get_project_download_url, project_id, 'yeats.txt', poem) # Test file attributes self.assertEqual(r_project.files[0].modality, None) self.assertEmpty(r_project.files[0].classification) self.assertEqual(r_project.files[0].type, 'text') resp = fw.modify_project_file( project_id, 'yeats.txt', flywheel.FileEntry(modality='modality', type='type')) # Check that no jobs were triggered, and attrs were modified self.assertEqual(resp.jobs_spawned, 0) r_project = fw.get_project(project_id) self.assertEqual(r_project.files[0].modality, "modality") self.assertEmpty(r_project.files[0].classification) self.assertEqual(r_project.files[0].type, 'type') # Test classifications resp = fw.modify_project_file_classification( project_id, 'yeats.txt', {'replace': { 'Custom': ['measurement1', 'measurement2'], }}) self.assertEqual(resp.modified, 1) self.assertEqual(resp.jobs_spawned, 0) r_project = fw.get_project(project_id) self.assertEqual(r_project.files[0].classification, {'Custom': ['measurement1', 'measurement2']}) resp = fw.modify_project_file_classification( project_id, 'yeats.txt', { 'add': { 'Custom': ['HelloWorld'], }, 'delete': { 'Custom': ['measurement2'] } }) self.assertEqual(resp.modified, 1) self.assertEqual(resp.jobs_spawned, 0) r_project = fw.get_project(project_id) self.assertEqual(r_project.files[0].classification, { 'Custom': ['measurement1', 'HelloWorld'], }) # Test file info self.assertEmpty(r_project.files[0].info) fw.replace_project_file_info(project_id, 'yeats.txt', { 'a': 1, 'b': 2, 'c': 3, 'd': 4 }) fw.set_project_file_info(project_id, 'yeats.txt', {'c': 5}) r_project = fw.get_project(project_id) self.assertEqual(r_project.files[0].info['a'], 1) self.assertEqual(r_project.files[0].info['b'], 2) self.assertEqual(r_project.files[0].info['c'], 5) self.assertEqual(r_project.files[0].info['d'], 4) fw.delete_project_file_info_fields(project_id, 'yeats.txt', ['c', 'd']) r_project = fw.get_project(project_id) self.assertEqual(r_project.files[0].info['a'], 1) self.assertEqual(r_project.files[0].info['b'], 2) self.assertNotIn('c', r_project.files[0].info) self.assertNotIn('d', r_project.files[0].info) fw.replace_project_file_info(project_id, 'yeats.txt', {}) r_project = fw.get_project(project_id) self.assertEmpty(r_project.files[0].info) # Delete file fw.delete_project_file(project_id, 'yeats.txt') r_project = fw.get_project(project_id) self.assertEmpty(r_project.files)
class TestValidateContext: @pytest.mark.parametrize( "config, call_num", [ ( { "export_project": "test1", "force_export": True, "check_gear_rules": True, }, 1, ), ( { "export_project": "test1", "force_export": False, "check_gear_rules": True, }, 1, ), ( { "export_project": "test1", "archive_project": "test2", "force_export": True, }, 2, ), ( { "export_project": "test1", "archive_project": "test2", "force_export": False, }, 2, ), ], ) def test_validate_calls(self, mocker, gear_context, config, call_num, caplog): caplog.set_level(logging.INFO) mock_proj = ( flywheel.Project( label="test", parents=flywheel.models.container_parents.ContainerParents( group="test" ), ), ) gear_context.config = config get_proj_mock = mocker.patch("validate.get_project") get_proj_mock.return_value = mock_proj get_dest_mock = mocker.patch("validate.get_destination") get_dest_mock.return_value = flywheel.Subject(label="test") check_exported_mock = mocker.patch("validate.container_needs_export") check_exported_mock.return_value = True check_gear_rules_mock = mocker.patch("validate.validate_gear_rules") check_gear_rules_mock.return_value = True export, archive, dest = validate_context(gear_context) assert get_proj_mock.call_count == call_num get_dest_mock.assert_called_once_with(gear_context.client, "test") check_exported_mock.assert_called_once_with( flywheel.Subject(label="test"), config ) msgs = [rec.message for rec in caplog.records] if "check_gear_rules" in config: check_gear_rules_mock.assert_called_once_with( gear_context.client, mock_proj ) assert "No enabled rules were found. Moving on..." in msgs else: check_gear_rules_mock.assert_not_called() assert "No enabled rules were found. Moving on..." not in msgs @pytest.mark.parametrize( "proj", [ {"export_project": flywheel.Project(label="export")}, {"archive_project": flywheel.Project(label="archvie")}, ], ) def test_get_proj_errors(self, mocker, sdk_mock, gear_context, caplog, proj): gear_context.config = {"export_project": "test", "archive_project": "test"} gear_context.config.update(proj) def get_proj_side_effect(fw, project): if not hasattr(project, "label"): raise flywheel.rest.ApiException(status=600) get_proj_mock = mocker.patch("validate.get_project") get_proj_mock.side_effect = get_proj_side_effect with pytest.raises(SystemExit): export, archive, dest = validate_context(gear_context) assert all( [ rec.levelno == logging.ERROR for rec in caplog.get_records(when="teardown") ] ) @pytest.mark.parametrize( "to_mock,val,to_err,raises,log", [ ( ["get_project"], [None], dict(), True, "Export project needs to be specified", ), ( ["get_project", "validate_gear_rules"], [flywheel.Project(label="test"), False], dict(), True, "Aborting Session Export: test has ENABLED GEAR RULES and 'check_gear_rules' == True. If you would like to force the export regardless of enabled gear rules re-run.py the gear with 'check_gear_rules' == False. Warning: Doing so may result in undesired behavior.", ), ( ["get_project", "validate_gear_rules", "get_destination"], ["test", True, None], {"get_destination": ValueError("test")}, True, "Could not find destination with id test", ), ( ["get_project", "validate_gear_rules", "get_destination"], ["test", True, None], {"get_destination": ApiException(status=20)}, True, "Could not find destination with id test", ), ( [ "get_project", "validate_gear_rules", "get_destination", "container_needs_export", ], ["test", True, flywheel.Session(label="test"), False], dict(), True, "session test has already been exported and <force_export> = False. Nothing to do!", ), ], ) def test_errors( self, mocker, to_mock, val, to_err, raises, log, caplog, gear_context ): gear_context.config = { "destination": {"id": "test", "container_type": "session", "label": "test"}, "export_project": "test", "archive_project": "test", "check_gear_rules": True, } mocks = {} for mock, val in zip(to_mock, val): mocks[mock] = mocker.patch(f"validate.{mock}") mocks[mock].return_value = val for mock, err in to_err.items(): if mock in mocks: mocks[mock].side_effect = err if raises: my_raise = pytest.raises(SystemExit) else: my_raise = does_not_raise() with my_raise: validate_context(gear_context)
def test_container_hierarchy(): hierarchy_dict = { "group": flywheel.Group(id="test_group", label="Test Group"), "project": flywheel.Project(label="test_project"), "subject": flywheel.Subject(label="test_subject", sex="other"), "session": flywheel.Session( age=31000000, label="test_session", weight=50, ), } # test from_dict test_hierarchy = ContainerHierarchy.from_dict(hierarchy_dict) # test deepcopy assert deepcopy(test_hierarchy) != test_hierarchy # test path assert test_hierarchy.path == "test_group/test_project/test_subject/test_session" # test parent assert test_hierarchy.parent.label == "test_subject" # test from_container mock_client = MagicMock(spec=dir(flywheel.Client)) parent_dict = dict() for item in ("group", "project", "subject"): value = hierarchy_dict.copy().get(item) parent_dict[item] = item setattr(mock_client, f"get_{item}", lambda x: value) session = flywheel.Session(age=31000000, label="test_session", weight=50) session.parents = parent_dict assert (ContainerHierarchy.from_container( mock_client, session).container_type == "session") # test _get_container assert test_hierarchy._get_container(None, None, None) is None with pytest.raises(ValueError) as exc: test_hierarchy._get_container(None, "garbage", "garbage_id") assert str(exc) == "Cannot get a container of type garbage" mock_client = MagicMock(spec=dir(flywheel.Client)) mock_client.get_session = lambda x: x assert (test_hierarchy._get_container(mock_client, "session", "session_id") == "session_id") # test container_type assert test_hierarchy.container_type == "session" # test dicom_map exp_map = { "PatientWeight": 50, "PatientAge": "011M", "ClinicalTrialTimePointDescription": "test_session", "PatientSex": "O", "PatientID": "test_subject", } assert exp_map == test_hierarchy.dicom_map # test get assert test_hierarchy.get("container_type") == "session" # test get_patient_sex_from_subject assert test_hierarchy.get_patientsex_from_subject(flywheel.Subject()) == "" # test get_patientage_from_session assert test_hierarchy.get_patientage_from_session( flywheel.Session()) is None # test get_child_hierarchy test_acquisition = flywheel.Acquisition(label="test_acquisition") acq_hierarchy = test_hierarchy.get_child_hierarchy(test_acquisition) assert acq_hierarchy.dicom_map[ "SeriesDescription"] == test_acquisition.label # test get_parent_hierarchy parent_hierarchy = test_hierarchy.get_parent_hierarchy() assert parent_hierarchy.container_type == "subject"