def test_get_file_model(self, m_gtm, m_requests):
# Check static expected values in model. This happens to be actual
# return values, however we are mocking out any actual network
# communication.
expected_m = {
'_id': self.EXAMPLE_ITEM_ID,
'_modelType': 'file',
'exts': ['png'],
'mimeType': 'image/png',
'name': 'Lenna.png',
'sha512': self.EXAMPLE_SHA512,
'size': 473831
}
# as if a successful call
m_requests.get().configure_mock(status_code=200)
m_requests.get().json.return_value = expected_m
e = GirderDataElement(self.EXAMPLE_ITEM_ID,
self.EXAMPLE_GIRDER_API_ROOT)
m = e.get_file_model()
nose.tools.assert_equal(m['_id'], expected_m['_id'])
nose.tools.assert_equal(m['_modelType'], expected_m['_modelType'])
nose.tools.assert_equal(m['exts'], expected_m['exts'])
nose.tools.assert_equal(m['mimeType'], expected_m['mimeType'])
nose.tools.assert_equal(m['name'], expected_m['name'])
nose.tools.assert_equal(m['sha512'], expected_m['sha512'])
nose.tools.assert_equal(m['size'], expected_m['size'])
def test_get_bytes(self, m_getvalue, _m_downloadFile):
m_getvalue.return_value = 'foo'
e = GirderDataElement('someId')
actual_bytes = e.get_bytes()
self.assertEqual(actual_bytes, bytes('foo'))
def test_get_file_model(self, m_getFile):
# Check static expected values in model. This happens to be actual
# return values, however we are mocking out any actual network
# communication.
expected_m = {
'_id': self.EXAMPLE_ITEM_ID,
'_modelType': 'file',
'exts': ['png'],
'mimeType': 'image/png',
'name': 'Lenna.png',
'sha512': self.EXAMPLE_SHA512,
'size': 473831
}
m_getFile.return_value = expected_m
e = GirderDataElement(self.EXAMPLE_ITEM_ID,
self.EXAMPLE_GIRDER_API_ROOT)
m = e.get_file_model()
self.assertEqual(m['_id'], expected_m['_id'])
self.assertEqual(m['_modelType'], expected_m['_modelType'])
self.assertEqual(m['exts'], expected_m['exts'])
self.assertEqual(m['mimeType'], expected_m['mimeType'])
self.assertEqual(m['name'], expected_m['name'])
self.assertEqual(m['sha512'], expected_m['sha512'])
self.assertEqual(m['size'], expected_m['size'])
def test_get_file_model(self, m_getFile):
# Check static expected values in model. This happens to be actual
# return values, however we are mocking out any actual network
# communication.
expected_m = {
'_id': self.EXAMPLE_ITEM_ID,
'_modelType': 'file',
'exts': ['png'],
'mimeType': 'image/png',
'name': 'Lenna.png',
'sha512': self.EXAMPLE_SHA512,
'size': 473831
}
m_getFile.return_value = expected_m
e = GirderDataElement(self.EXAMPLE_ITEM_ID,
self.EXAMPLE_GIRDER_API_ROOT)
m = e.get_file_model()
self.assertEqual(m['_id'], expected_m['_id'])
self.assertEqual(m['_modelType'], expected_m['_modelType'])
self.assertEqual(m['exts'], expected_m['exts'])
self.assertEqual(m['mimeType'], expected_m['mimeType'])
self.assertEqual(m['name'], expected_m['name'])
self.assertEqual(m['sha512'], expected_m['sha512'])
self.assertEqual(m['size'], expected_m['size'])
def test_get_file_model_item_no_exists(self, m_getFile):
def raise_http_error(*_, **__):
raise girder_client.HttpError(None, None, None, None)
m_getFile.side_effect = raise_http_error
e = GirderDataElement('foo', self.EXAMPLE_GIRDER_API_ROOT)
m = e.get_file_model()
self.assertIsNone(m)
def test_get_file_model_item_no_exists(self, m_gtm, m_requests):
# i.e. status 400 returned, should return None
m_requests.get().configure_mock(status_code=400)
e = GirderDataElement('id')
m = e.get_file_model()
nose.tools.assert_is_none(m)
nose.tools.assert_equal(m_requests.get().raise_for_status.call_count,
0)
def test_get_bytes(self, m_downloadFile):
# Simulate actually downloading something bytes.
m_downloadFile.side_effect = \
lambda _if, _bytesio: _bytesio.write(b'foo')
e = GirderDataElement('someId')
actual_bytes = e.get_bytes()
self.assertEqual(actual_bytes, b'foo')
def test_get_bytes(self, _m_downloadFile):
""" Test that getting bytes is driven by the girder_client downloadFile
method.
"""
_m_downloadFile.side_effect = lambda fid, buf: buf.write(b'foo')
e = GirderDataElement('someId')
actual_bytes = e.get_bytes()
assert actual_bytes == b'foo'
def test_content_type_cached(self, m_gtm, m_requests):
expected_mimetype = 'some/type'
e = GirderDataElement('id')
e._content_type = expected_mimetype
actual_type = e.content_type()
nose.tools.assert_equal(actual_type, expected_mimetype)
nose.tools.assert_equal(m_requests.get.call_count, 0)
nose.tools.assert_equal(m_requests.get().json.call_count, 0)
def test_get_bytes(self, m_requests_get, m_gtm):
expected_content = 'Some byte content.'
expected_response = gen_response(expected_content)
# Mock out requests call
m_requests_get.return_value = expected_response
# Get mocked bytes
e = GirderDataElement('someId')
actual_bytes = e.get_bytes()
nose.tools.assert_equal(expected_content, actual_bytes)
def test_set_bytes_http_errors(self):
gde = GirderDataElement('someId')
gde.writable = mock.MagicMock(return_value=True)
# Test access denied throws ReadOnlyError
gde.gc.uploadFileContents = mock.MagicMock(
side_effect=girder_client.HttpError(401, '', None, None))
self.assertRaises(ReadOnlyError, gde.set_bytes, b=b'foo')
# Test any other error (like a 500) re-raises the HttpError
gde.gc.uploadFileContents = mock.MagicMock(
side_effect=girder_client.HttpError(500, '', None, None))
self.assertRaises(girder_client.HttpError, gde.set_bytes, b=b'foo')
def test_set_bytes_http_errors(self):
gde = GirderDataElement('someId')
gde.writable = mock.MagicMock(return_value=True)
# Test access denied throws ReadOnlyError
gde.gc.uploadFileContents = mock.MagicMock(
side_effect=girder_client.HttpError(401, '', None, None))
self.assertRaises(ReadOnlyError, gde.set_bytes, b=b'foo')
# Test any other error (like a 500) re-raises the HttpError
gde.gc.uploadFileContents = mock.MagicMock(
side_effect=girder_client.HttpError(500, '', None, None))
self.assertRaises(girder_client.HttpError, gde.set_bytes, b=b'foo')
def test_writable(self, m_getItem, m_getFolder, _m_get_file_model):
m_getItem.return_value = {'folderId': 'someFolderId'}
m_getFolder.return_value = {'_accessLevel': 1}
self.assertTrue(GirderDataElement('someId').writable())
# Access level 0 should cause it to be unwritable
m_getFolder.return_value = {'_accessLevel': 0}
self.assertFalse(GirderDataElement('someId').writable())
# A nonexistent file model should make writable return false
gde = GirderDataElement('someId')
gde.get_file_model = mock.MagicMock(return_value=None)
self.assertFalse(gde.writable())
def test_content_type_no_cache(self, m_getFile):
# Mocking such that we simulate a valid API root and an existing
# item reference
# Dummy requests return value
expected_mimetype = 'some/type'
m_getFile.return_value = {'mimeType': expected_mimetype}
e = GirderDataElement('foo')
actual_type = e.content_type()
nose.tools.assert_equal(actual_type, expected_mimetype)
m_getFile.assert_called_once()
# Ensure that calling content_type a second time doesn't call getFile again
nose.tools.assert_equal(e.content_type(), expected_mimetype)
m_getFile.assert_called_once()
def test_configuration_default(self):
default_config = GirderDataElement.get_default_config()
self.assertEqual(default_config,
{"file_id": None,
"api_root": self.LOCAL_APIROOT,
"api_key": None,
"token": None})
def test_content_type_no_cache(self, m_gtm, m_requests):
# Mocking such that we simulate a valid API root and an existing
# item reference
# Dummy requests return value
expected_mimetype = 'some/type'
m_requests.get('setting json mock return value')\
.json.return_value = {
'mimeType': expected_mimetype
}
e = GirderDataElement('foo')
actual_type = e.content_type()
nose.tools.assert_equal(actual_type, expected_mimetype)
# once above to set return_value and once it function
nose.tools.assert_equal(m_requests.get.call_count, 2)
m_requests.get().json.assert_called_once()
def test_new_fileId(self):
expected_id = "some id"
e = GirderDataElement(expected_id)
nose.tools.assert_equal(e.file_id, expected_id)
nose.tools.assert_equal(e.api_root, self.LOCAL_APIROOT)
nose.tools.assert_is_none(e.token)
nose.tools.assert_is_none(e.token_expiration)
nose.tools.assert_is_none(e._content_type)
def is_valid(dataElementUri):
dfe = GirderDataElement.from_uri(dataElementUri)
if is_valid_element(dfe,
valid_content_types=valid_content_types,
check_image=True):
return dfe
else:
return False
def test_new_fileId(self):
expected_id = "some id"
e = GirderDataElement(expected_id)
nose.tools.assert_equal(e.file_id, expected_id)
nose.tools.assert_equal(e.api_root, self.LOCAL_APIROOT)
nose.tools.assert_is_none(e.api_key)
nose.tools.assert_is_none(e.token)
nose.tools.assert_is_none(e._content_type)
nose.tools.assert_is_instance(e.gc, girder_client.GirderClient)
def is_valid(dataElementUri):
dfe = GirderDataElement.from_uri(dataElementUri)
if is_valid_element(dfe,
valid_content_types=valid_content_types,
check_image=True):
return dfe
else:
return False
def test_new_fileId(self):
expected_id = "some id"
e = GirderDataElement(expected_id)
self.assertEqual(e.file_id, expected_id)
self.assertEqual(e.api_root, self.LOCAL_APIROOT)
self.assertIsNone(e.api_key)
self.assertIsNone(e.token)
self.assertIsNone(e._content_type)
self.assertIsInstance(e.gc, girder_client.GirderClient)
def test_content_type_no_cache(self, m_getFile):
# Mocking such that we simulate a valid API root and an existing
# item reference
# Dummy requests return value
expected_mimetype = 'some/type'
m_getFile.return_value = {
'mimeType': expected_mimetype
}
e = GirderDataElement('foo')
actual_type = e.content_type()
self.assertEqual(actual_type, expected_mimetype)
m_getFile.assert_called_once()
# Ensure that calling content_type a second time doesn't call
# getFile again.
self.assertEqual(e.content_type(), expected_mimetype)
m_getFile.assert_called_once()
def compute_hash_codes(task, folderId, **kwargs):
"""
Celery task for computing hash codes on a given folder (descriptor index).
:param task: Celery provided task object.
:param folderId: The folder to train ITQ for, note this is only used to
infer the descriptor index.
"""
task.job_manager.updateProgress(message='Computing Hash Codes',
forceFlush=True)
index = descriptorIndexFromFolderId(task.girder_client, folderId)
smqtkFolder = getCreateFolder(task.girder_client, folderId, '.smqtk')
meanVecFileId = smqtkFileIdFromName(task.girder_client, smqtkFolder,
'mean_vec.npy')
rotationFileId = smqtkFileIdFromName(task.girder_client, smqtkFolder,
'rotation.npy')
hash2uuidsFile = initializeItemWithFile(
task.girder_client,
createOverwriteItem(task.girder_client, smqtkFolder['_id'],
'hash2uuids.pickle'))
functor = ItqFunctor(
mean_vec_cache=GirderDataElement(
meanVecFileId,
api_root=task.request.apiUrl,
token=task.request.jobInfoSpec['headers']['Girder-Token']),
rotation_cache=GirderDataElement(
rotationFileId,
api_root=task.request.apiUrl,
token=task.request.jobInfoSpec['headers']['Girder-Token']))
hash2uuids = compute_functions.compute_hash_codes(index.iterkeys(),
index,
functor,
use_mp=False)
data = pickle.dumps(dict((y, x) for (x, y) in hash2uuids))
task.girder_client.uploadFileContents(hash2uuidsFile['_id'],
six.BytesIO(data), len(data))
def test_from_config_full_constructor(self, _mock_authenticate):
expected_file_id = '34uhki34gh2345ghjk'
expected_api_root = 'https://some.other.server/api/v1'
expected_api_key = '1234ghk135hlg23435'
inst = GirderDataElement(expected_file_id,
api_root=expected_api_root,
api_key=expected_api_key)
for i in configuration_test_helper(inst): # type: GirderDataElement
assert i.file_id == expected_file_id
assert i.api_root == expected_api_root
assert i.api_key == expected_api_key
def test_get_file_model_other_bad_status(self, m_gtm, m_requests):
# if some other bad status code is returned an exception is raised
def r(*args, **kwds):
raise requests.HTTPError('message')
m_requests.get().configure_mock(status_code=502)
m_requests.get().raise_for_status.side_effect = r
e = GirderDataElement('id')
nose.tools.assert_raises(requests.HTTPError, e.get_file_model)
m_requests.get().raise_for_status.assert_called_once()
def test_get_bytes_bad_response(self, m_requests_get, m_gtm):
expected_content = 'Some byte content.'
expected_statCode = 501
expected_response = gen_response(expected_content, expected_statCode)
# Mock out requests call
m_requests_get.return_value = expected_response
# Try getting mocked bytes
e = GirderDataElement('someId')
nose.tools.assert_raises_regexp(requests.HTTPError,
"%d" % expected_statCode, e.get_bytes)
def itq(task, folderId, **kwargs):
"""
Celery task for training ITQ on a given folder.
This trains ITQ on all descriptors within the index. Since this
is typically called after computing descriptors, it will often
only contain what's in the folder.
:param task: Celery provided task object.
:param folderId: The folder to train ITQ for, note this is only used to
infer the descriptor index.
"""
task.job_manager.updateProgress(message='Training ITQ', forceFlush=True)
index = descriptorIndexFromFolderId(task.girder_client, folderId)
if not index.count():
# TODO SMQTK should account for this?
raise Exception('Descriptor index is empty, cannot train ITQ.')
smqtkFolder = getCreateFolder(task.girder_client, folderId, '.smqtk')
meanVecFile = initializeItemWithFile(
task.girder_client,
createOverwriteItem(task.girder_client, smqtkFolder['_id'],
'mean_vec.npy'))
rotationFile = initializeItemWithFile(
task.girder_client,
createOverwriteItem(task.girder_client, smqtkFolder['_id'],
'rotation.npy'))
functor = ItqFunctor(
mean_vec_cache=GirderDataElement(
meanVecFile['_id'],
api_root=task.request.apiUrl,
token=task.request.jobInfoSpec['headers']['Girder-Token']),
rotation_cache=GirderDataElement(
rotationFile['_id'],
api_root=task.request.apiUrl,
token=task.request.jobInfoSpec['headers']['Girder-Token']))
functor.fit(index.iterdescriptors(), use_multiprocessing=False)
def test_from_config_common_partial(self):
expected_file_id = '5hjkl1345hjk'
expected_api_root = self.LOCAL_APIROOT
expected_api_key = None
#: :type: GirderDataElement
e = GirderDataElement.from_config({'file_id': expected_file_id})
self.assertEqual(e.file_id, expected_file_id)
self.assertEqual(e.api_root, expected_api_root)
self.assertEqual(e.get_config(),
{'file_id': expected_file_id,
'api_root': expected_api_root,
'api_key': expected_api_key,
'token': None})
def test_repr(self):
expected_file_id = 'some_file id'
expected_api_root = 'https://some.server/api/v1'
expected_api_key = 'someKeyHere'
e = GirderDataElement(expected_file_id, expected_api_root,
expected_api_key)
actual_repr = repr(e)
expected_repr = "GirderDataElement{id: some_file id, " \
"api_root: https://some.server/api/v1, " \
"api_key: someKeyHere}"
nose.tools.assert_equal(actual_repr, expected_repr)
def test_from_config_common_partial(self):
expected_file_id = '5hjkl1345hjk'
expected_api_root = self.LOCAL_APIROOT
expected_api_key = None
e = GirderDataElement.from_config({'file_id': expected_file_id})
nose.tools.assert_equal(e.file_id, expected_file_id)
nose.tools.assert_equal(e.api_root, expected_api_root)
nose.tools.assert_equal(e.token_manager.api_key, expected_api_key)
nose.tools.assert_equal(
e.get_config(), {
'file_id': expected_file_id,
'api_root': expected_api_root,
'api_key': expected_api_key
})
def test_repr(self, mock_requests):
expected_file_id = 'some_file id'
expected_api_root = 'https://some.server/api/v1'
expected_api_key = 'someKeyHere'
e = GirderDataElement(expected_file_id, expected_api_root,
expected_api_key)
actual_repr = repr(e)
expected_repr = "GirderDataElement{file_id: %s, " \
"api_root: %s, " \
"api_key: %s, token: }" % (
expected_file_id, expected_api_root, expected_api_key)
nose.tools.assert_equal(actual_repr, expected_repr)
def test_from_config_full_constructor(self, mock_authenticate):
expected_file_id = '34uhki34gh2345ghjk'
expected_api_root = 'https://some.other.server/api/v1'
expected_api_key = '1234ghk135hlg23435'
new_config = {
'token': None,
'file_id': expected_file_id,
'api_root': expected_api_root,
'api_key': expected_api_key,
}
e = GirderDataElement.from_config(new_config)
nose.tools.assert_equal(e.file_id, expected_file_id)
nose.tools.assert_equal(e.api_root, expected_api_root)
nose.tools.assert_equal(e.get_config(), new_config)
def test_from_config_full_constructor(self, _mock_authenticate):
expected_file_id = '34uhki34gh2345ghjk'
expected_api_root = 'https://some.other.server/api/v1'
expected_api_key = '1234ghk135hlg23435'
new_config = {
'token': None,
'file_id': expected_file_id,
'api_root': expected_api_root,
'api_key': expected_api_key,
}
#: :type: GirderDataElement
e = GirderDataElement.from_config(new_config)
self.assertEqual(e.file_id, expected_file_id)
self.assertEqual(e.api_root, expected_api_root)
self.assertEqual(e.get_config(), new_config)
def test_writable(self, m_getItem, m_getFolder, _m_get_file_model):
m_getItem.return_value = {'folderId': 'someFolderId'}
m_getFolder.return_value = {'_accessLevel': 1}
self.assertTrue(GirderDataElement('someId').writable())
# Access level 0 should cause it to be unwritable
m_getFolder.return_value = {'_accessLevel': 0}
self.assertFalse(GirderDataElement('someId').writable())
# A nonexistent file model should make writable return false
gde = GirderDataElement('someId')
gde.get_file_model = mock.MagicMock(return_value=None)
self.assertFalse(gde.writable())
def find_girder_files(api_root, folder_ids, item_ids, file_ids,
api_key=None, query_batch=None):
"""
Query girder for file IDs nested under folders and items, and yield
GirderDataElement instances for each child file discovered.
We first find nested folder and items under given folder IDs, then files
under discovered items, finally yielding GirderDataElements for file IDs
discovered.
Data elements yielded are in order of file IDs given, to files under items
given, to files under items discovered in folders in a breadth-first order.
:param api_root: Root URL of the girder instance to call.
:type api_root: str
:param folder_ids: Iterable of Girder folder IDs to recursively collect file
elements from.
:type folder_ids: collection.Iterable[str]
:param item_ids: Iterable of Girder item IDs to collect file elements from.
:type item_ids: collection.Iterable[str]
:param file_ids: Iterable of Girder file IDs to make elements of.
:type file_ids: collection.Iterable[str]
:param api_key: Optional user API key to use for authentication when
accessing private data.
:type api_key: None | str
:param query_batch: Number of elements to query for at a time when expanding
folder and item contents. A batch size of 0 means all elements are
queried for at once.
:type query_batch: int
:return: Generator yielding GirderDataElement instances.
:rtype: __generator[GirderDataElement]
"""
log = logging.getLogger(__name__)
tm = GirderTokenManager(api_root, api_key)
# Get the token once before parallel requests
tm.get_token()
if query_batch is None:
query_batch = 0
# Could also do something with multi-threading/processing
log.info("Yielding elements from file IDs")
file_fifo = collections.deque(file_ids)
while file_fifo: # Just file IDs
file_id = file_fifo.popleft()
log.debug('-f %s', file_id)
e = GirderDataElement(file_id, api_root, api_key)
e.token_manager = tm # because we already made one
yield e
log.info("Collecting files from items")
item_fifo = collections.deque(item_ids)
while item_fifo:
item_id = item_fifo.popleft()
log.debug('-i %s', item_id)
for file_id, ct in get_item_files(api_root, item_id, tm, query_batch):
log.debug(' -f %s', file_id)
e = GirderDataElement(file_id, api_root, api_key)
e._content_type = ct
e.token_manager = tm
yield e
# Collect items from folders, then files from items.
log.info("Collecting items from folders")
folder_fifo = collections.deque(folder_ids)
while folder_fifo:
folder_id = folder_fifo.popleft()
log.debug("-F %s", folder_id)
# Find sub-folders
folder_fifo.extend(get_folder_subfolders(api_root, folder_id, tm,
query_batch))
for item_id in get_folder_items(api_root, folder_id, tm, query_batch):
log.debug(' -i %s', item_id)
for file_id, ct in get_item_files(api_root, item_id, tm,
query_batch):
log.debug(' -f %s (%s)', file_id, ct)
e = GirderDataElement(file_id, api_root, api_key)
e._content_type = ct
e.token_manager = tm
yield e
def test_from_uri_full_url(self):
e = GirderDataElement.from_uri(self.EXAMPLE_GIRDER_FULL_URI)
self.assertEqual(e.file_id, self.EXAMPLE_ITEM_ID)
r = requests.get(DATA_KITWARE_URL)
_ = r.content
except requests.ConnectionError:
internet_available = False
def gen_response(content, status_code=200):
resp = requests.Response()
resp._content = content
resp.status_code = status_code
resp.headers['content-length'] = len(content)
return resp
# Only perform these tests if the implementation is usable.
if GirderDataElement.is_usable():
class TestGirderDataElement (unittest.TestCase):
"""
Tests for the GirderDataElement plugin implementation
"""
LOCAL_APIROOT = "http://localhost:8080/api/v1"
EXAMPLE_ITEM_ID = '5820bbeb8d777f10f26efc2f'
EXAMPLE_GIRDER_API_ROOT = "%s/api/v1" % DATA_KITWARE_URL
EXAMPLE_GIRDER_FULL_URI = (
EXAMPLE_GIRDER_API_ROOT.replace('https', 'girder')
+ '/file/%s' % EXAMPLE_ITEM_ID
)
EXAMPLE_PTH = os.path.join(TEST_DATA_DIR, 'Lenna.png')
def test_is_empty_none_model(self):
# Uses model return, empty if no model return (no item in girder by
# ID)
e = GirderDataElement('someId')
e.get_file_model = mock.MagicMock(return_value=None)
self.assertTrue(e.is_empty())
def test_is_empty_zero_size(self):
# Uses model return size parameter
e = GirderDataElement('someId')
e.get_file_model = mock.MagicMock(return_value={'size': 0})
self.assertTrue(e.is_empty())
def test_is_empty_nonzero_bytes(self):
e = GirderDataElement('someId')
e.get_file_model = mock.MagicMock(return_value={'size': 7})
self.assertFalse(e.is_empty())
def test_set_bytes_non_writable(self):
gde = GirderDataElement('someId')
gde.writable = mock.MagicMock(return_value=False)
self.assertRaises(ReadOnlyError, gde.set_bytes, b=None)
def test_from_uri_full_url(self):
e = GirderDataElement.from_uri(self.EXAMPLE_GIRDER_FULL_URI)
self.assertEqual(e.file_id, self.EXAMPLE_ITEM_ID)
def test_set_bytes_non_writable(self):
gde = GirderDataElement('someId')
gde.writable = mock.MagicMock(return_value=False)
self.assertRaises(ReadOnlyError, gde.set_bytes, b=None)
def test_set_bytes(self, m_uploadFileContents):
gde = GirderDataElement('someId')
gde.writable = mock.MagicMock(return_value=True)
gde.set_bytes(b'foo')
m_uploadFileContents.assert_called_once()