def test_content_type_cached(self, m_gtm, m_requests):
expected_mimetype = 'some/type'
e = GirderDataElement('id')
e._content_type = expected_mimetype
actual_type = e.content_type()
nose.tools.assert_equal(actual_type, expected_mimetype)
nose.tools.assert_equal(m_requests.get.call_count, 0)
nose.tools.assert_equal(m_requests.get().json.call_count, 0)
def find_girder_files(api_root,
folder_ids,
item_ids,
file_ids,
api_key=None,
query_batch=None):
"""
Query girder for file IDs nested under folders and items, and yield
GirderDataElement instances for each child file discovered.
We first find nested folder and items under given folder IDs, then files
under discovered items, finally yielding GirderDataElements for file IDs
discovered.
Data elements yielded are in order of file IDs given, to files under items
given, to files under items discovered in folders in a breadth-first order.
:param api_root: Root URL of the girder instance to call.
:type api_root: str
:param folder_ids: Iterable of Girder folder IDs to recursively collect file
elements from.
:type folder_ids: collection.Iterable[str]
:param item_ids: Iterable of Girder item IDs to collect file elements from.
:type item_ids: collection.Iterable[str]
:param file_ids: Iterable of Girder file IDs to make elements of.
:type file_ids: collection.Iterable[str]
:param api_key: Optional user API key to use for authentication when
accessing private data.
:type api_key: None | str
:param query_batch: Number of elements to query for at a time when expanding
folder and item contents. A batch size of 0 means all elements are
queried for at once.
:type query_batch: int
:return: Generator yielding GirderDataElement instances.
:rtype: __generator[GirderDataElement]
"""
log = logging.getLogger(__name__)
tm = GirderTokenManager(api_root, api_key)
# Get the token once before parallel requests
tm.get_token()
if query_batch is None:
query_batch = 0
# Could also do something with multi-threading/processing
log.info("Yielding elements from file IDs")
file_fifo = collections.deque(file_ids)
while file_fifo: # Just file IDs
file_id = file_fifo.popleft()
log.debug('-f %s', file_id)
e = GirderDataElement(file_id, api_root, api_key)
e.token_manager = tm # because we already made one
yield e
log.info("Collecting files from items")
item_fifo = collections.deque(item_ids)
while item_fifo:
item_id = item_fifo.popleft()
log.debug('-i %s', item_id)
for file_id, ct in get_item_files(api_root, item_id, tm, query_batch):
log.debug(' -f %s', file_id)
e = GirderDataElement(file_id, api_root, api_key)
e._content_type = ct
e.token_manager = tm
yield e
# Collect items from folders, then files from items.
log.info("Collecting items from folders")
folder_fifo = collections.deque(folder_ids)
while folder_fifo:
folder_id = folder_fifo.popleft()
log.debug("-F %s", folder_id)
# Find sub-folders
folder_fifo.extend(
get_folder_subfolders(api_root, folder_id, tm, query_batch))
for item_id in get_folder_items(api_root, folder_id, tm, query_batch):
log.debug(' -i %s', item_id)
for file_id, ct in get_item_files(api_root, item_id, tm,
query_batch):
log.debug(' -f %s (%s)', file_id, ct)
e = GirderDataElement(file_id, api_root, api_key)
e._content_type = ct
e.token_manager = tm
yield e
def find_girder_files(api_root, folder_ids, item_ids, file_ids,
api_key=None, query_batch=None):
"""
Query girder for file IDs nested under folders and items, and yield
GirderDataElement instances for each child file discovered.
We first find nested folder and items under given folder IDs, then files
under discovered items, finally yielding GirderDataElements for file IDs
discovered.
Data elements yielded are in order of file IDs given, to files under items
given, to files under items discovered in folders in a breadth-first order.
:param api_root: Root URL of the girder instance to call.
:type api_root: str
:param folder_ids: Iterable of Girder folder IDs to recursively collect file
elements from.
:type folder_ids: collection.Iterable[str]
:param item_ids: Iterable of Girder item IDs to collect file elements from.
:type item_ids: collection.Iterable[str]
:param file_ids: Iterable of Girder file IDs to make elements of.
:type file_ids: collection.Iterable[str]
:param api_key: Optional user API key to use for authentication when
accessing private data.
:type api_key: None | str
:param query_batch: Number of elements to query for at a time when expanding
folder and item contents. A batch size of 0 means all elements are
queried for at once.
:type query_batch: int
:return: Generator yielding GirderDataElement instances.
:rtype: __generator[GirderDataElement]
"""
log = logging.getLogger(__name__)
tm = GirderTokenManager(api_root, api_key)
# Get the token once before parallel requests
tm.get_token()
if query_batch is None:
query_batch = 0
# Could also do something with multi-threading/processing
log.info("Yielding elements from file IDs")
file_fifo = collections.deque(file_ids)
while file_fifo: # Just file IDs
file_id = file_fifo.popleft()
log.debug('-f %s', file_id)
e = GirderDataElement(file_id, api_root, api_key)
e.token_manager = tm # because we already made one
yield e
log.info("Collecting files from items")
item_fifo = collections.deque(item_ids)
while item_fifo:
item_id = item_fifo.popleft()
log.debug('-i %s', item_id)
for file_id, ct in get_item_files(api_root, item_id, tm, query_batch):
log.debug(' -f %s', file_id)
e = GirderDataElement(file_id, api_root, api_key)
e._content_type = ct
e.token_manager = tm
yield e
# Collect items from folders, then files from items.
log.info("Collecting items from folders")
folder_fifo = collections.deque(folder_ids)
while folder_fifo:
folder_id = folder_fifo.popleft()
log.debug("-F %s", folder_id)
# Find sub-folders
folder_fifo.extend(get_folder_subfolders(api_root, folder_id, tm,
query_batch))
for item_id in get_folder_items(api_root, folder_id, tm, query_batch):
log.debug(' -i %s', item_id)
for file_id, ct in get_item_files(api_root, item_id, tm,
query_batch):
log.debug(' -f %s (%s)', file_id, ct)
e = GirderDataElement(file_id, api_root, api_key)
e._content_type = ct
e.token_manager = tm
yield e