async def get_blob(self, blob_name: str) -> Blob:
"""Get a blob object by name.
.. code-block:: python
container = storage.get_container('container-name')
picture_blob = container.get_blob('picture.png')
# <Blob picture.png container-name S3>
:param blob_name: The name of the blob to retrieve.
:type blob_name: str
:return: The blob object if it exists.
:rtype: Blob
:raise NotFoundError: If the blob object doesn't exist.
"""
if is_file_url(blob_name):
meta = parse_file_url(blob_name)
if not meta or meta['container'] != self.name or meta[
'store'] != self.driver.alias_name:
raise InvalidFileURLError(messages.FILE_URL_INVALID %
(blob_name, ))
blob_name = meta['blob']
return await self.driver.get_blob(container=self, blob_name=blob_name)
async def download(fileurl,
destfilename: str = 'auto',
destpath: str = None,
**kwargs):
_check_storage_enabled()
parsed = parse_file_url(fileurl)
blob_name = parsed['blob']
container = await _get_container(parsed['container'], parsed['store'],
**kwargs)
blob = await container.get_blob(blob_name)
if not isinstance(destfilename, str) and hasattr(destfilename, 'write'):
if destpath is not None:
raise Exception("destpath is invalid when providing stream")
await blob.download(destfilename)
#dont know the filepath here so return empty string
try:
return str(destfilename.name)
except AttributeError as e:
return ''
if destfilename == 'auto':
#take filename from file url
destfilename = os.path.basename(blob_name)
elif destfilename != 'auto' and destpath is None:
#destfilename may be full filepath or a stream
destpath = os.path.dirname(destfilename)
destfilename = os.path.basename(destfilename)
if destpath:
#ensure directory exists
try:
if not os.path.exists(destpath):
os.makedirs(destpath)
except PermissionError as err:
raise CredentialsError(str(err))
if destpath is not None and destfilename:
download_path = os.path.join(destpath, destfilename)
await blob.download(download_path)
return download_path
else:
#make own temporary file and return it
with tempfile.NamedTemporaryFile(mode='w+b', delete=False) as dfile:
await blob.download(dfile)
return dfile.name
async def bulk_download(filedict: Dict,
destfilename: str = 'auto',
destpath: str = None,
**kwargs):
"""
filedict: a dictionary containing key and file. the returned
dictionary will have the same key along with uploaded
file path
"""
_check_storage_enabled()
if not isinstance(filedict, dict):
raise Exception("Expected dict but got %s" % (type(filedict), ))
if not len(filedict):
return {}
multi_container = kwargs.get('multi_container', True)
if not multi_container:
first_file_url = ''
for key in filedict:
first_file_url = filedict[key]
break
parsed = parse_file_url(first_file_url)
container = await _get_container(parsed['container'], parsed['store'],
**kwargs)
kwargs['container'] = container
tasks = []
keys = []
for key, fileurl in filedict.items():
task = download(fileurl, destfilename, destpath, **kwargs)
tasks.append(asyncio.create_task(task))
keys.append(key)
paths = await asyncio.gather(*tasks)
file_paths = {}
for index, key in enumerate(keys):
file_paths[key] = paths[index]
return file_paths
async def test_download_container_in_url_invalid(binary_blob):
with pytest.raises(NotFoundError) as err:
parsed = parse_file_url(binary_blob.file_url)
invalid_url = "%s://%s/%s" % (parsed['store'], random_container_name(),
parsed['blob'])
filepath = await download(invalid_url)
def test_parse_file_url():
assert parse_file_url('fs://trash/abc.jpg') == {
'store': 'fs',
'container': 'trash',
'blob': 'abc.jpg'
}
assert parse_file_url('fs12://trash/abc/jhfdf$#122.jpg') == {
'store': 'fs12',
'container': 'trash',
'blob': 'abc/jhfdf$#122.jpg'
}
assert parse_file_url('minio://trash/abc/jhfdf$#122.jpg') == {
'store': 'minio',
'container': 'trash',
'blob': 'abc/jhfdf$#122.jpg'
}
assert parse_file_url('MINIO://trash/abc/jhfdf$#122.jpg') == {
'store': 'MINIO',
'container': 'trash',
'blob': 'abc/jhfdf$#122.jpg'
}
assert parse_file_url('S3://trash123/abc/jhfdf$#122.jpg') == {
'store': 'S3',
'container': 'trash123',
'blob': 'abc/jhfdf$#122.jpg'
}
assert parse_file_url('GCS://trash123/abc/jhfdf$#122.jpg') == {
'store': 'GCS',
'container': 'trash123',
'blob': 'abc/jhfdf$#122.jpg'
}
with pytest.raises(InvalidFileURLError) as err:
parse_file_url('HTTP://trash123/abc/jhfdf$#122.jpg')
with pytest.raises(InvalidFileURLError) as err:
parse_file_url('SSh://trash123/abc/jhfdf$#122.jpg')
with pytest.raises(Exception) as err:
parse_file_url('HTTP://trash123/abc/jhfdf$#122.jpg')