def test_get_file_model(self, m_gtm, m_requests): # Check static expected values in model. This happens to be actual # return values, however we are mocking out any actual network # communication. expected_m = { '_id': self.EXAMPLE_ITEM_ID, '_modelType': 'file', 'exts': ['png'], 'mimeType': 'image/png', 'name': 'Lenna.png', 'sha512': self.EXAMPLE_SHA512, 'size': 473831 } # as if a successful call m_requests.get().configure_mock(status_code=200) m_requests.get().json.return_value = expected_m e = GirderDataElement(self.EXAMPLE_ITEM_ID, self.EXAMPLE_GIRDER_API_ROOT) m = e.get_file_model() nose.tools.assert_equal(m['_id'], expected_m['_id']) nose.tools.assert_equal(m['_modelType'], expected_m['_modelType']) nose.tools.assert_equal(m['exts'], expected_m['exts']) nose.tools.assert_equal(m['mimeType'], expected_m['mimeType']) nose.tools.assert_equal(m['name'], expected_m['name']) nose.tools.assert_equal(m['sha512'], expected_m['sha512']) nose.tools.assert_equal(m['size'], expected_m['size'])
def test_get_bytes(self, m_getvalue, _m_downloadFile): m_getvalue.return_value = 'foo' e = GirderDataElement('someId') actual_bytes = e.get_bytes() self.assertEqual(actual_bytes, bytes('foo'))
def test_get_file_model(self, m_getFile): # Check static expected values in model. This happens to be actual # return values, however we are mocking out any actual network # communication. expected_m = { '_id': self.EXAMPLE_ITEM_ID, '_modelType': 'file', 'exts': ['png'], 'mimeType': 'image/png', 'name': 'Lenna.png', 'sha512': self.EXAMPLE_SHA512, 'size': 473831 } m_getFile.return_value = expected_m e = GirderDataElement(self.EXAMPLE_ITEM_ID, self.EXAMPLE_GIRDER_API_ROOT) m = e.get_file_model() self.assertEqual(m['_id'], expected_m['_id']) self.assertEqual(m['_modelType'], expected_m['_modelType']) self.assertEqual(m['exts'], expected_m['exts']) self.assertEqual(m['mimeType'], expected_m['mimeType']) self.assertEqual(m['name'], expected_m['name']) self.assertEqual(m['sha512'], expected_m['sha512']) self.assertEqual(m['size'], expected_m['size'])
def test_get_file_model(self, m_getFile): # Check static expected values in model. This happens to be actual # return values, however we are mocking out any actual network # communication. expected_m = { '_id': self.EXAMPLE_ITEM_ID, '_modelType': 'file', 'exts': ['png'], 'mimeType': 'image/png', 'name': 'Lenna.png', 'sha512': self.EXAMPLE_SHA512, 'size': 473831 } m_getFile.return_value = expected_m e = GirderDataElement(self.EXAMPLE_ITEM_ID, self.EXAMPLE_GIRDER_API_ROOT) m = e.get_file_model() self.assertEqual(m['_id'], expected_m['_id']) self.assertEqual(m['_modelType'], expected_m['_modelType']) self.assertEqual(m['exts'], expected_m['exts']) self.assertEqual(m['mimeType'], expected_m['mimeType']) self.assertEqual(m['name'], expected_m['name']) self.assertEqual(m['sha512'], expected_m['sha512']) self.assertEqual(m['size'], expected_m['size'])
def test_get_file_model_item_no_exists(self, m_getFile): def raise_http_error(*_, **__): raise girder_client.HttpError(None, None, None, None) m_getFile.side_effect = raise_http_error e = GirderDataElement('foo', self.EXAMPLE_GIRDER_API_ROOT) m = e.get_file_model() self.assertIsNone(m)
def test_get_file_model_item_no_exists(self, m_gtm, m_requests): # i.e. status 400 returned, should return None m_requests.get().configure_mock(status_code=400) e = GirderDataElement('id') m = e.get_file_model() nose.tools.assert_is_none(m) nose.tools.assert_equal(m_requests.get().raise_for_status.call_count, 0)
def test_get_bytes(self, m_downloadFile): # Simulate actually downloading something bytes. m_downloadFile.side_effect = \ lambda _if, _bytesio: _bytesio.write(b'foo') e = GirderDataElement('someId') actual_bytes = e.get_bytes() self.assertEqual(actual_bytes, b'foo')
def test_get_bytes(self, _m_downloadFile): """ Test that getting bytes is driven by the girder_client downloadFile method. """ _m_downloadFile.side_effect = lambda fid, buf: buf.write(b'foo') e = GirderDataElement('someId') actual_bytes = e.get_bytes() assert actual_bytes == b'foo'
def test_content_type_cached(self, m_gtm, m_requests): expected_mimetype = 'some/type' e = GirderDataElement('id') e._content_type = expected_mimetype actual_type = e.content_type() nose.tools.assert_equal(actual_type, expected_mimetype) nose.tools.assert_equal(m_requests.get.call_count, 0) nose.tools.assert_equal(m_requests.get().json.call_count, 0)
def test_get_bytes(self, m_requests_get, m_gtm): expected_content = 'Some byte content.' expected_response = gen_response(expected_content) # Mock out requests call m_requests_get.return_value = expected_response # Get mocked bytes e = GirderDataElement('someId') actual_bytes = e.get_bytes() nose.tools.assert_equal(expected_content, actual_bytes)
def test_set_bytes_http_errors(self): gde = GirderDataElement('someId') gde.writable = mock.MagicMock(return_value=True) # Test access denied throws ReadOnlyError gde.gc.uploadFileContents = mock.MagicMock( side_effect=girder_client.HttpError(401, '', None, None)) self.assertRaises(ReadOnlyError, gde.set_bytes, b=b'foo') # Test any other error (like a 500) re-raises the HttpError gde.gc.uploadFileContents = mock.MagicMock( side_effect=girder_client.HttpError(500, '', None, None)) self.assertRaises(girder_client.HttpError, gde.set_bytes, b=b'foo')
def test_set_bytes_http_errors(self): gde = GirderDataElement('someId') gde.writable = mock.MagicMock(return_value=True) # Test access denied throws ReadOnlyError gde.gc.uploadFileContents = mock.MagicMock( side_effect=girder_client.HttpError(401, '', None, None)) self.assertRaises(ReadOnlyError, gde.set_bytes, b=b'foo') # Test any other error (like a 500) re-raises the HttpError gde.gc.uploadFileContents = mock.MagicMock( side_effect=girder_client.HttpError(500, '', None, None)) self.assertRaises(girder_client.HttpError, gde.set_bytes, b=b'foo')
def test_writable(self, m_getItem, m_getFolder, _m_get_file_model): m_getItem.return_value = {'folderId': 'someFolderId'} m_getFolder.return_value = {'_accessLevel': 1} self.assertTrue(GirderDataElement('someId').writable()) # Access level 0 should cause it to be unwritable m_getFolder.return_value = {'_accessLevel': 0} self.assertFalse(GirderDataElement('someId').writable()) # A nonexistent file model should make writable return false gde = GirderDataElement('someId') gde.get_file_model = mock.MagicMock(return_value=None) self.assertFalse(gde.writable())
def test_content_type_no_cache(self, m_getFile): # Mocking such that we simulate a valid API root and an existing # item reference # Dummy requests return value expected_mimetype = 'some/type' m_getFile.return_value = {'mimeType': expected_mimetype} e = GirderDataElement('foo') actual_type = e.content_type() nose.tools.assert_equal(actual_type, expected_mimetype) m_getFile.assert_called_once() # Ensure that calling content_type a second time doesn't call getFile again nose.tools.assert_equal(e.content_type(), expected_mimetype) m_getFile.assert_called_once()
def test_configuration_default(self): default_config = GirderDataElement.get_default_config() self.assertEqual(default_config, {"file_id": None, "api_root": self.LOCAL_APIROOT, "api_key": None, "token": None})
def test_content_type_no_cache(self, m_gtm, m_requests): # Mocking such that we simulate a valid API root and an existing # item reference # Dummy requests return value expected_mimetype = 'some/type' m_requests.get('setting json mock return value')\ .json.return_value = { 'mimeType': expected_mimetype } e = GirderDataElement('foo') actual_type = e.content_type() nose.tools.assert_equal(actual_type, expected_mimetype) # once above to set return_value and once it function nose.tools.assert_equal(m_requests.get.call_count, 2) m_requests.get().json.assert_called_once()
def test_new_fileId(self): expected_id = "some id" e = GirderDataElement(expected_id) nose.tools.assert_equal(e.file_id, expected_id) nose.tools.assert_equal(e.api_root, self.LOCAL_APIROOT) nose.tools.assert_is_none(e.token) nose.tools.assert_is_none(e.token_expiration) nose.tools.assert_is_none(e._content_type)
def is_valid(dataElementUri): dfe = GirderDataElement.from_uri(dataElementUri) if is_valid_element(dfe, valid_content_types=valid_content_types, check_image=True): return dfe else: return False
def test_new_fileId(self): expected_id = "some id" e = GirderDataElement(expected_id) nose.tools.assert_equal(e.file_id, expected_id) nose.tools.assert_equal(e.api_root, self.LOCAL_APIROOT) nose.tools.assert_is_none(e.api_key) nose.tools.assert_is_none(e.token) nose.tools.assert_is_none(e._content_type) nose.tools.assert_is_instance(e.gc, girder_client.GirderClient)
def is_valid(dataElementUri): dfe = GirderDataElement.from_uri(dataElementUri) if is_valid_element(dfe, valid_content_types=valid_content_types, check_image=True): return dfe else: return False
def test_new_fileId(self): expected_id = "some id" e = GirderDataElement(expected_id) self.assertEqual(e.file_id, expected_id) self.assertEqual(e.api_root, self.LOCAL_APIROOT) self.assertIsNone(e.api_key) self.assertIsNone(e.token) self.assertIsNone(e._content_type) self.assertIsInstance(e.gc, girder_client.GirderClient)
def test_content_type_no_cache(self, m_getFile): # Mocking such that we simulate a valid API root and an existing # item reference # Dummy requests return value expected_mimetype = 'some/type' m_getFile.return_value = { 'mimeType': expected_mimetype } e = GirderDataElement('foo') actual_type = e.content_type() self.assertEqual(actual_type, expected_mimetype) m_getFile.assert_called_once() # Ensure that calling content_type a second time doesn't call # getFile again. self.assertEqual(e.content_type(), expected_mimetype) m_getFile.assert_called_once()
def compute_hash_codes(task, folderId, **kwargs): """ Celery task for computing hash codes on a given folder (descriptor index). :param task: Celery provided task object. :param folderId: The folder to train ITQ for, note this is only used to infer the descriptor index. """ task.job_manager.updateProgress(message='Computing Hash Codes', forceFlush=True) index = descriptorIndexFromFolderId(task.girder_client, folderId) smqtkFolder = getCreateFolder(task.girder_client, folderId, '.smqtk') meanVecFileId = smqtkFileIdFromName(task.girder_client, smqtkFolder, 'mean_vec.npy') rotationFileId = smqtkFileIdFromName(task.girder_client, smqtkFolder, 'rotation.npy') hash2uuidsFile = initializeItemWithFile( task.girder_client, createOverwriteItem(task.girder_client, smqtkFolder['_id'], 'hash2uuids.pickle')) functor = ItqFunctor( mean_vec_cache=GirderDataElement( meanVecFileId, api_root=task.request.apiUrl, token=task.request.jobInfoSpec['headers']['Girder-Token']), rotation_cache=GirderDataElement( rotationFileId, api_root=task.request.apiUrl, token=task.request.jobInfoSpec['headers']['Girder-Token'])) hash2uuids = compute_functions.compute_hash_codes(index.iterkeys(), index, functor, use_mp=False) data = pickle.dumps(dict((y, x) for (x, y) in hash2uuids)) task.girder_client.uploadFileContents(hash2uuidsFile['_id'], six.BytesIO(data), len(data))
def test_from_config_full_constructor(self, _mock_authenticate): expected_file_id = '34uhki34gh2345ghjk' expected_api_root = 'https://some.other.server/api/v1' expected_api_key = '1234ghk135hlg23435' inst = GirderDataElement(expected_file_id, api_root=expected_api_root, api_key=expected_api_key) for i in configuration_test_helper(inst): # type: GirderDataElement assert i.file_id == expected_file_id assert i.api_root == expected_api_root assert i.api_key == expected_api_key
def test_get_file_model_other_bad_status(self, m_gtm, m_requests): # if some other bad status code is returned an exception is raised def r(*args, **kwds): raise requests.HTTPError('message') m_requests.get().configure_mock(status_code=502) m_requests.get().raise_for_status.side_effect = r e = GirderDataElement('id') nose.tools.assert_raises(requests.HTTPError, e.get_file_model) m_requests.get().raise_for_status.assert_called_once()
def test_get_bytes_bad_response(self, m_requests_get, m_gtm): expected_content = 'Some byte content.' expected_statCode = 501 expected_response = gen_response(expected_content, expected_statCode) # Mock out requests call m_requests_get.return_value = expected_response # Try getting mocked bytes e = GirderDataElement('someId') nose.tools.assert_raises_regexp(requests.HTTPError, "%d" % expected_statCode, e.get_bytes)
def itq(task, folderId, **kwargs): """ Celery task for training ITQ on a given folder. This trains ITQ on all descriptors within the index. Since this is typically called after computing descriptors, it will often only contain what's in the folder. :param task: Celery provided task object. :param folderId: The folder to train ITQ for, note this is only used to infer the descriptor index. """ task.job_manager.updateProgress(message='Training ITQ', forceFlush=True) index = descriptorIndexFromFolderId(task.girder_client, folderId) if not index.count(): # TODO SMQTK should account for this? raise Exception('Descriptor index is empty, cannot train ITQ.') smqtkFolder = getCreateFolder(task.girder_client, folderId, '.smqtk') meanVecFile = initializeItemWithFile( task.girder_client, createOverwriteItem(task.girder_client, smqtkFolder['_id'], 'mean_vec.npy')) rotationFile = initializeItemWithFile( task.girder_client, createOverwriteItem(task.girder_client, smqtkFolder['_id'], 'rotation.npy')) functor = ItqFunctor( mean_vec_cache=GirderDataElement( meanVecFile['_id'], api_root=task.request.apiUrl, token=task.request.jobInfoSpec['headers']['Girder-Token']), rotation_cache=GirderDataElement( rotationFile['_id'], api_root=task.request.apiUrl, token=task.request.jobInfoSpec['headers']['Girder-Token'])) functor.fit(index.iterdescriptors(), use_multiprocessing=False)
def test_from_config_common_partial(self): expected_file_id = '5hjkl1345hjk' expected_api_root = self.LOCAL_APIROOT expected_api_key = None #: :type: GirderDataElement e = GirderDataElement.from_config({'file_id': expected_file_id}) self.assertEqual(e.file_id, expected_file_id) self.assertEqual(e.api_root, expected_api_root) self.assertEqual(e.get_config(), {'file_id': expected_file_id, 'api_root': expected_api_root, 'api_key': expected_api_key, 'token': None})
def test_repr(self): expected_file_id = 'some_file id' expected_api_root = 'https://some.server/api/v1' expected_api_key = 'someKeyHere' e = GirderDataElement(expected_file_id, expected_api_root, expected_api_key) actual_repr = repr(e) expected_repr = "GirderDataElement{id: some_file id, " \ "api_root: https://some.server/api/v1, " \ "api_key: someKeyHere}" nose.tools.assert_equal(actual_repr, expected_repr)
def test_from_config_common_partial(self): expected_file_id = '5hjkl1345hjk' expected_api_root = self.LOCAL_APIROOT expected_api_key = None e = GirderDataElement.from_config({'file_id': expected_file_id}) nose.tools.assert_equal(e.file_id, expected_file_id) nose.tools.assert_equal(e.api_root, expected_api_root) nose.tools.assert_equal(e.token_manager.api_key, expected_api_key) nose.tools.assert_equal( e.get_config(), { 'file_id': expected_file_id, 'api_root': expected_api_root, 'api_key': expected_api_key })
def test_repr(self, mock_requests): expected_file_id = 'some_file id' expected_api_root = 'https://some.server/api/v1' expected_api_key = 'someKeyHere' e = GirderDataElement(expected_file_id, expected_api_root, expected_api_key) actual_repr = repr(e) expected_repr = "GirderDataElement{file_id: %s, " \ "api_root: %s, " \ "api_key: %s, token: }" % ( expected_file_id, expected_api_root, expected_api_key) nose.tools.assert_equal(actual_repr, expected_repr)
def test_from_config_full_constructor(self, mock_authenticate): expected_file_id = '34uhki34gh2345ghjk' expected_api_root = 'https://some.other.server/api/v1' expected_api_key = '1234ghk135hlg23435' new_config = { 'token': None, 'file_id': expected_file_id, 'api_root': expected_api_root, 'api_key': expected_api_key, } e = GirderDataElement.from_config(new_config) nose.tools.assert_equal(e.file_id, expected_file_id) nose.tools.assert_equal(e.api_root, expected_api_root) nose.tools.assert_equal(e.get_config(), new_config)
def test_from_config_full_constructor(self, _mock_authenticate): expected_file_id = '34uhki34gh2345ghjk' expected_api_root = 'https://some.other.server/api/v1' expected_api_key = '1234ghk135hlg23435' new_config = { 'token': None, 'file_id': expected_file_id, 'api_root': expected_api_root, 'api_key': expected_api_key, } #: :type: GirderDataElement e = GirderDataElement.from_config(new_config) self.assertEqual(e.file_id, expected_file_id) self.assertEqual(e.api_root, expected_api_root) self.assertEqual(e.get_config(), new_config)
def test_writable(self, m_getItem, m_getFolder, _m_get_file_model): m_getItem.return_value = {'folderId': 'someFolderId'} m_getFolder.return_value = {'_accessLevel': 1} self.assertTrue(GirderDataElement('someId').writable()) # Access level 0 should cause it to be unwritable m_getFolder.return_value = {'_accessLevel': 0} self.assertFalse(GirderDataElement('someId').writable()) # A nonexistent file model should make writable return false gde = GirderDataElement('someId') gde.get_file_model = mock.MagicMock(return_value=None) self.assertFalse(gde.writable())
def find_girder_files(api_root, folder_ids, item_ids, file_ids, api_key=None, query_batch=None): """ Query girder for file IDs nested under folders and items, and yield GirderDataElement instances for each child file discovered. We first find nested folder and items under given folder IDs, then files under discovered items, finally yielding GirderDataElements for file IDs discovered. Data elements yielded are in order of file IDs given, to files under items given, to files under items discovered in folders in a breadth-first order. :param api_root: Root URL of the girder instance to call. :type api_root: str :param folder_ids: Iterable of Girder folder IDs to recursively collect file elements from. :type folder_ids: collection.Iterable[str] :param item_ids: Iterable of Girder item IDs to collect file elements from. :type item_ids: collection.Iterable[str] :param file_ids: Iterable of Girder file IDs to make elements of. :type file_ids: collection.Iterable[str] :param api_key: Optional user API key to use for authentication when accessing private data. :type api_key: None | str :param query_batch: Number of elements to query for at a time when expanding folder and item contents. A batch size of 0 means all elements are queried for at once. :type query_batch: int :return: Generator yielding GirderDataElement instances. :rtype: __generator[GirderDataElement] """ log = logging.getLogger(__name__) tm = GirderTokenManager(api_root, api_key) # Get the token once before parallel requests tm.get_token() if query_batch is None: query_batch = 0 # Could also do something with multi-threading/processing log.info("Yielding elements from file IDs") file_fifo = collections.deque(file_ids) while file_fifo: # Just file IDs file_id = file_fifo.popleft() log.debug('-f %s', file_id) e = GirderDataElement(file_id, api_root, api_key) e.token_manager = tm # because we already made one yield e log.info("Collecting files from items") item_fifo = collections.deque(item_ids) while item_fifo: item_id = item_fifo.popleft() log.debug('-i %s', item_id) for file_id, ct in get_item_files(api_root, item_id, tm, query_batch): log.debug(' -f %s', file_id) e = GirderDataElement(file_id, api_root, api_key) e._content_type = ct e.token_manager = tm yield e # Collect items from folders, then files from items. log.info("Collecting items from folders") folder_fifo = collections.deque(folder_ids) while folder_fifo: folder_id = folder_fifo.popleft() log.debug("-F %s", folder_id) # Find sub-folders folder_fifo.extend(get_folder_subfolders(api_root, folder_id, tm, query_batch)) for item_id in get_folder_items(api_root, folder_id, tm, query_batch): log.debug(' -i %s', item_id) for file_id, ct in get_item_files(api_root, item_id, tm, query_batch): log.debug(' -f %s (%s)', file_id, ct) e = GirderDataElement(file_id, api_root, api_key) e._content_type = ct e.token_manager = tm yield e
def test_from_uri_full_url(self): e = GirderDataElement.from_uri(self.EXAMPLE_GIRDER_FULL_URI) self.assertEqual(e.file_id, self.EXAMPLE_ITEM_ID)
r = requests.get(DATA_KITWARE_URL) _ = r.content except requests.ConnectionError: internet_available = False def gen_response(content, status_code=200): resp = requests.Response() resp._content = content resp.status_code = status_code resp.headers['content-length'] = len(content) return resp # Only perform these tests if the implementation is usable. if GirderDataElement.is_usable(): class TestGirderDataElement (unittest.TestCase): """ Tests for the GirderDataElement plugin implementation """ LOCAL_APIROOT = "http://localhost:8080/api/v1" EXAMPLE_ITEM_ID = '5820bbeb8d777f10f26efc2f' EXAMPLE_GIRDER_API_ROOT = "%s/api/v1" % DATA_KITWARE_URL EXAMPLE_GIRDER_FULL_URI = ( EXAMPLE_GIRDER_API_ROOT.replace('https', 'girder') + '/file/%s' % EXAMPLE_ITEM_ID ) EXAMPLE_PTH = os.path.join(TEST_DATA_DIR, 'Lenna.png')
def test_is_empty_none_model(self): # Uses model return, empty if no model return (no item in girder by # ID) e = GirderDataElement('someId') e.get_file_model = mock.MagicMock(return_value=None) self.assertTrue(e.is_empty())
def test_is_empty_zero_size(self): # Uses model return size parameter e = GirderDataElement('someId') e.get_file_model = mock.MagicMock(return_value={'size': 0}) self.assertTrue(e.is_empty())
def test_is_empty_nonzero_bytes(self): e = GirderDataElement('someId') e.get_file_model = mock.MagicMock(return_value={'size': 7}) self.assertFalse(e.is_empty())
def test_set_bytes_non_writable(self): gde = GirderDataElement('someId') gde.writable = mock.MagicMock(return_value=False) self.assertRaises(ReadOnlyError, gde.set_bytes, b=None)
def test_from_uri_full_url(self): e = GirderDataElement.from_uri(self.EXAMPLE_GIRDER_FULL_URI) self.assertEqual(e.file_id, self.EXAMPLE_ITEM_ID)
def test_set_bytes_non_writable(self): gde = GirderDataElement('someId') gde.writable = mock.MagicMock(return_value=False) self.assertRaises(ReadOnlyError, gde.set_bytes, b=None)
def test_set_bytes(self, m_uploadFileContents): gde = GirderDataElement('someId') gde.writable = mock.MagicMock(return_value=True) gde.set_bytes(b'foo') m_uploadFileContents.assert_called_once()