def add_image(album_name, username):
gallery_db = connect_to_db()
albums = gallery_db.albums
requested_album = albums.find_one({"name": album_name})
if not requested_album:
return redirect(url_for('albums', album =album_name, message="album not found"))
if not username in requested_album["write"]:
return redirect(url_for('albums', album = album_name, message="permission denied"))
if 'image[]' not in request.files:
return redirect(url_for('albums', album = album_name, message="no file uploaded"))
for req_file in request.files.getlist('image[]'):
file_name = uuid.uuid4().hex
stats_upload_timer = stats_client.timer("upload timer")
stats_upload_timer.start()
blob_service = BlobService(account_name=ACCOUNT_NAME, account_key=ACCOUNT_KEY)
blob_service.put_block_blob_from_file(CONTAINER_NAME, file_name, req_file.stream)
gallery_db.albums.update({'name': album_name}, {'$push': {'images': file_name}})
stats_upload_timer.stop()
# increment the counter of the uploaded images
stats_client.incr("images uploaded", len(request.files.getlist('image[]')))
return redirect(url_for('albums', album = album_name))
class FileService(Component):
def __init__(self):
self.blob_service = None
def generate_blob_service(self):
if self.blob_service is None:
# if storage info doesn't exist in config.py upload file function stop working
self.blob_service = BlobService(account_name=self.util.get_config("storage.azure.account_name"),
account_key=self.util.get_config("storage.azure.account_key"),
host_base=self.util.get_config("storage.azure.blob_service_host_base"))
def create_container_in_storage(self, container_name, access):
"""
create a container if doesn't exist
:param container_name:
:param access:
:return:
"""
self.generate_blob_service()
try:
names = map(lambda x: x.name, self.blob_service.list_containers())
if container_name not in names:
self.blob_service.create_container(container_name, x_ms_blob_public_access=access)
else:
self.log.debug("container already exsit in storage")
return True
except Exception as e:
self.log.error(e)
return False
def upload_file_to_azure(self, stream, container_name, blob_name):
try:
if self.create_container_in_storage(container_name, 'container'):
self.blob_service.put_block_blob_from_file(container_name, blob_name, stream)
return self.blob_service.make_blob_url(container_name, blob_name)
else:
return None
except Exception as e:
self.log.error(e)
return None
def upload_file_to_azure_from_path(self, path, container_name, blob_name):
try:
if self.create_container_in_storage(container_name, 'container'):
self.blob_service.put_block_blob_from_path(container_name, blob_name, path)
return self.blob_service.make_blob_url(container_name, blob_name)
else:
return None
except Exception as e:
self.log.error(e)
return None
def delete_file_from_azure(self, container_name, blob_name):
try:
if self.create_container_in_storage(container_name, 'container'):
self.blob_service.delete_blob(container_name, blob_name)
except Exception as e:
self.log.error(e)
return None
class BlobServiceAdapter(Component):
"""The :class:`BlobServiceAdapter` class is a thin wrapper over azure.storage.BlobService.
All the attributes of the wrapper stream are proxied by the adapter so
it's possible to do ``adapter.create_container()`` instead of the long form
``adapter.blob_service.adapter()``.
"""
def __init__(self):
self.blob_service = BlobService(
account_name=self.util.get_config("storage.azure.account_name"),
account_key=self.util.get_config("storage.azure.account_key"),
host_base=self.util.get_config("storage.azure.blob_service_host_base"),
)
def __getattr__(self, name):
return getattr(self.blob_service, name)
def create_container_in_storage(self, container_name, access="container"):
"""create a container if doesn't exist
:type container_name: str|unicode
:param container_name: Name of container to create.
:type access: str|unicode
:param access: Optional. Possible values include: container, blob
:return:
"""
try:
names = [x.name for x in self.blob_service.list_containers()]
if container_name not in names:
return self.blob_service.create_container(container_name, x_ms_blob_public_access=access)
else:
self.log.debug("container already exists in storage")
return True
except Exception as e:
self.log.error(e)
return False
def upload_file_to_azure(self, container_name, blob_name, stream):
"""
Creates a new block blob from a file/stream, or updates the content of
an existing block blob, with automatic chunking and progress
notifications.
:type container_name: str|unicode
:param container_name: Name of existing container.
:type blob_name: str | unicode
:param blob_name: Name of blob to create or update.
:type stream: file
:param stream: Opened file/stream to upload as the blob content.
"""
try:
if self.create_container_in_storage(container_name, "container"):
self.blob_service.put_block_blob_from_file(container_name, blob_name, stream)
return self.blob_service.make_blob_url(container_name, blob_name)
else:
return None
except Exception as e:
self.log.error(e)
return None
def upload_file_to_azure_from_bytes(self, container_name, blob_name, blob):
"""
Creates a new block blob from an array of bytes, or updates the content
of an existing block blob, with automatic chunking and progress
notifications.
:type container_name: str|unicode
:param container_name: Name of existing container.
:type blob_name: str|unicode
:param blob_name: Name of blob to create or update.
:type blob: bytes
:param blob: Content of blob as an array of bytes.
"""
try:
if self.create_container_in_storage(container_name, "container"):
self.blob_service.put_block_blob_from_bytes(container_name, blob_name, blob)
return self.blob_service.make_blob_url(container_name, blob_name)
else:
return None
except Exception as e:
self.log.error(e)
return None
def upload_file_to_azure_from_text(self, container_name, blob_name, text):
"""
Creates a new block blob from str/unicode, or updates the content of an
existing block blob, with automatic chunking and progress notifications.
:type container_name: str|unicode
:param container_name: Name of existing container.
:type blob_name: str|unicode
:param blob_name: Name of blob to create or update.
:type text: str|unicode
:param text: Text to upload to the blob.
"""
try:
if self.create_container_in_storage(container_name, "container"):
self.blob_service.put_block_blob_from_text(container_name, blob_name, text)
return self.blob_service.make_blob_url(container_name, blob_name)
else:
return None
except Exception as e:
self.log.error(e)
return None
def upload_file_to_azure_from_path(self, container_name, blob_name, path):
"""
Creates a new page blob from a file path, or updates the content of an
existing page blob, with automatic chunking and progress notifications.
:type container_name: str|unicode
:param container_name: Name of existing container.
:type blob_name: str|unicode
:param blob_name: Name of blob to create or update.
:type path: str|unicode
:param path: Path of the file to upload as the blob content.
"""
try:
if self.create_container_in_storage(container_name, "container"):
self.blob_service.put_block_blob_from_path(container_name, blob_name, path)
return self.blob_service.make_blob_url(container_name, blob_name)
else:
return None
except Exception as e:
self.log.error(e)
return None
def delete_file_from_azure(self, container_name, blob_name):
try:
if self.create_container_in_storage(container_name, "container"):
self.blob_service.delete_blob(container_name, blob_name)
except Exception as e:
self.log.error(e)
return None
class Storage(driver.Base):
supports_bytes_range = True
def __init__(self, path=None, config=None):
self._config = config
self._container = self._config.azure_storage_container
protocol = 'https' if self._config.azure_use_https else 'http'
acct_name = self._config.azure_storage_account_name
acct_key = self._config.azure_storage_account_key
self._blob = BlobService(
account_name=acct_name, account_key=acct_key, protocol=protocol)
self._init_container()
logger.debug("Initialized azureblob storage driver")
def _init_container(self):
'''Initializes image container on Azure blob storage if the container
does not exist.
'''
created = self._blob.create_container(
self._container, x_ms_blob_public_access='blob',
fail_on_exist=False)
if created:
logger.info('Created blob container for image registry.')
else:
logger.debug('Registry container already exists.')
return created
@lru.get
def get_content(self, path):
try:
return self._blob.get_blob(self._container, path)
except azure.WindowsAzureMissingResourceError:
raise exceptions.FileNotFoundError('%s is not there' % path)
@lru.set
def put_content(self, path, content):
self._blob.put_blob(self._container, path, content, 'BlockBlob')
return path
def stream_read(self, path, bytes_range=None):
try:
f = io.BytesIO()
self._blob.get_blob_to_file(self._container, path, f)
if bytes_range:
f.seek(bytes_range[0])
total_size = bytes_range[1] - bytes_range[0] + 1
else:
f.seek(0)
while True:
buf = None
if bytes_range:
# Bytes Range is enabled
buf_size = self.buffer_size
if nb_bytes + buf_size > total_size:
# We make sure we don't read out of the range
buf_size = total_size - nb_bytes
if buf_size > 0:
buf = f.read(buf_size)
nb_bytes += len(buf)
else:
# We're at the end of the range
buf = ''
else:
buf = f.read(self.buffer_size)
if not buf:
break
yield buf
except IOError:
raise exceptions.FileNotFoundError('%s is not there' % path)
def stream_write(self, path, fp):
self._blob.put_block_blob_from_file(self._container, path, fp)
def list_directory(self, path=None):
if not path.endswith('/'):
path += '/' # path=a would list a/b.txt as well as 'abc.txt'
blobs = list(self._blob.list_blobs(self._container, path))
if not blobs:
raise exceptions.FileNotFoundError('%s is not there' % path)
return [b.name for b in blobs]
def exists(self, path):
try:
self._blob.get_blob_properties(self._container, path)
return True
except azure.WindowsAzureMissingResourceError:
return False
@lru.remove
def remove(self, path):
is_blob = self.exists(path)
if is_blob:
self._blob.delete_blob(self._container, path)
return
exists = False
blobs = list(self._blob.list_blobs(self._container, path))
if not blobs:
raise exceptions.FileNotFoundError('%s is not there' % path)
for b in blobs:
self._blob.delete_blob(self._container, b.name)
def get_size(self, path):
try:
properties = self._blob.get_blob_properties(self._container, path)
return int(properties['content-length']) # auto-converted to long
except azure.WindowsAzureMissingResourceError:
raise exceptions.FileNotFoundError('%s is not there' % path)
class FileService(Component):
def __init__(self):
self.blob_service = None
def generate_blob_service(self):
if self.blob_service is None:
# if storage info doesn't exist in config.py upload file function stop working
self.blob_service = BlobService(
account_name=self.util.get_config(
"storage.azure.account_name"),
account_key=self.util.get_config("storage.azure.account_key"),
host_base=self.util.get_config(
"storage.azure.blob_service_host_base"))
def create_container_in_storage(self, container_name, access):
"""
create a container if doesn't exist
:param container_name:
:param access:
:return:
"""
self.generate_blob_service()
try:
names = map(lambda x: x.name, self.blob_service.list_containers())
if container_name not in names:
self.blob_service.create_container(
container_name, x_ms_blob_public_access=access)
else:
self.log.debug("container already exsit in storage")
return True
except Exception as e:
self.log.error(e)
return False
def upload_file_to_azure(self, stream, container_name, blob_name):
try:
if self.create_container_in_storage(container_name, 'container'):
self.blob_service.put_block_blob_from_file(
container_name, blob_name, stream)
return self.blob_service.make_blob_url(container_name,
blob_name)
else:
return None
except Exception as e:
self.log.error(e)
return None
def upload_file_to_azure_from_path(self, path, container_name, blob_name):
try:
if self.create_container_in_storage(container_name, 'container'):
self.blob_service.put_block_blob_from_path(
container_name, blob_name, path)
return self.blob_service.make_blob_url(container_name,
blob_name)
else:
return None
except Exception as e:
self.log.error(e)
return None
def delete_file_from_azure(self, container_name, blob_name):
try:
if self.create_container_in_storage(container_name, 'container'):
self.blob_service.delete_blob(container_name, blob_name)
except Exception as e:
self.log.error(e)
return None
class BlobServiceAdapter(Component):
"""The :class:`BlobServiceAdapter` class is a thin wrapper over azure.storage.BlobService.
All the attributes of the wrapper stream are proxied by the adapter so
it's possible to do ``adapter.create_container()`` instead of the long form
``adapter.blob_service.adapter()``.
"""
def __init__(self):
self.blob_service = BlobService(account_name=self.util.get_config("storage.azure.account_name"),
account_key=self.util.get_config("storage.azure.account_key"),
host_base=self.util.get_config("storage.azure.blob_service_host_base"))
def __getattr__(self, name):
return getattr(self.blob_service, name)
def create_container_in_storage(self, container_name, access="container"):
"""create a container if doesn't exist
:type container_name: str|unicode
:param container_name: Name of container to create.
:type access: str|unicode
:param access: Optional. Possible values include: container, blob
:return:
"""
try:
names = [x.name for x in self.blob_service.list_containers()]
if container_name not in names:
return self.blob_service.create_container(container_name, x_ms_blob_public_access=access)
else:
self.log.debug("container already exists in storage")
return True
except Exception as e:
self.log.error(e)
return False
def upload_file_to_azure(self, container_name, blob_name, stream):
"""
Creates a new block blob from a file/stream, or updates the content of
an existing block blob, with automatic chunking and progress
notifications.
:type container_name: str|unicode
:param container_name: Name of existing container.
:type blob_name: str | unicode
:param blob_name: Name of blob to create or update.
:type stream: file
:param stream: Opened file/stream to upload as the blob content.
"""
try:
if self.create_container_in_storage(container_name, 'container'):
self.blob_service.put_block_blob_from_file(container_name, blob_name, stream)
return self.blob_service.make_blob_url(container_name, blob_name)
else:
return None
except Exception as e:
self.log.error(e)
return None
def upload_file_to_azure_from_bytes(self, container_name, blob_name, blob):
"""
Creates a new block blob from an array of bytes, or updates the content
of an existing block blob, with automatic chunking and progress
notifications.
:type container_name: str|unicode
:param container_name: Name of existing container.
:type blob_name: str|unicode
:param blob_name: Name of blob to create or update.
:type blob: bytes
:param blob: Content of blob as an array of bytes.
"""
try:
if self.create_container_in_storage(container_name, 'container'):
self.blob_service.put_block_blob_from_bytes(container_name, blob_name, blob)
return self.blob_service.make_blob_url(container_name, blob_name)
else:
return None
except Exception as e:
self.log.error(e)
return None
def upload_file_to_azure_from_text(self, container_name, blob_name, text):
"""
Creates a new block blob from str/unicode, or updates the content of an
existing block blob, with automatic chunking and progress notifications.
:type container_name: str|unicode
:param container_name: Name of existing container.
:type blob_name: str|unicode
:param blob_name: Name of blob to create or update.
:type text: str|unicode
:param text: Text to upload to the blob.
"""
try:
if self.create_container_in_storage(container_name, 'container'):
self.blob_service.put_block_blob_from_text(container_name, blob_name, text)
return self.blob_service.make_blob_url(container_name, blob_name)
else:
return None
except Exception as e:
self.log.error(e)
return None
def upload_file_to_azure_from_path(self, container_name, blob_name, path):
"""
Creates a new page blob from a file path, or updates the content of an
existing page blob, with automatic chunking and progress notifications.
:type container_name: str|unicode
:param container_name: Name of existing container.
:type blob_name: str|unicode
:param blob_name: Name of blob to create or update.
:type path: str|unicode
:param path: Path of the file to upload as the blob content.
"""
try:
if self.create_container_in_storage(container_name, 'container'):
self.blob_service.put_block_blob_from_path(container_name, blob_name, path)
return self.blob_service.make_blob_url(container_name, blob_name)
else:
return None
except Exception as e:
self.log.error(e)
return None
def delete_file_from_azure(self, container_name, blob_name):
try:
if self.create_container_in_storage(container_name, 'container'):
self.blob_service.delete_blob(container_name, blob_name)
except Exception as e:
self.log.error(e)
return None
class Storage(driver.Base):
supports_bytes_range = True
def __init__(self, path=None, config=None):
self._config = config
self._container = self._config.azure_storage_container
protocol = 'https' if self._config.azure_use_https else 'http'
acct_name = self._config.azure_storage_account_name
acct_key = self._config.azure_storage_account_key
self._blob = BlobService(account_name=acct_name,
account_key=acct_key,
protocol=protocol)
self._init_container()
logger.debug("Initialized azureblob storage driver")
def _init_container(self):
'''Initializes image container on Azure blob storage if the container
does not exist.
'''
created = self._blob.create_container(self._container,
x_ms_blob_public_access='blob',
fail_on_exist=False)
if created:
logger.info('Created blob container for image registry.')
else:
logger.debug('Registry container already exists.')
return created
@lru.get
def get_content(self, path):
try:
return self._blob.get_blob(self._container, path)
except azure.WindowsAzureMissingResourceError:
raise exceptions.FileNotFoundError('%s is not there' % path)
@lru.set
def put_content(self, path, content):
self._blob.put_blob(self._container, path, content, 'BlockBlob')
return path
def stream_read(self, path, bytes_range=None):
try:
f = io.BytesIO()
self._blob.get_blob_to_file(self._container, path, f)
if bytes_range:
f.seek(bytes_range[0])
total_size = bytes_range[1] - bytes_range[0] + 1
else:
f.seek(0)
while True:
buf = None
if bytes_range:
# Bytes Range is enabled
buf_size = self.buffer_size
if nb_bytes + buf_size > total_size:
# We make sure we don't read out of the range
buf_size = total_size - nb_bytes
if buf_size > 0:
buf = f.read(buf_size)
nb_bytes += len(buf)
else:
# We're at the end of the range
buf = ''
else:
buf = f.read(self.buffer_size)
if not buf:
break
yield buf
except IOError:
raise exceptions.FileNotFoundError('%s is not there' % path)
def stream_write(self, path, fp):
self._blob.put_block_blob_from_file(self._container, path, fp)
def list_directory(self, path=None):
if not path.endswith('/'):
path += '/' # path=a would list a/b.txt as well as 'abc.txt'
blobs = list(self._blob.list_blobs(self._container, path))
if not blobs:
raise exceptions.FileNotFoundError('%s is not there' % path)
return [b.name for b in blobs]
def exists(self, path):
try:
self._blob.get_blob_properties(self._container, path)
return True
except azure.WindowsAzureMissingResourceError:
return False
@lru.remove
def remove(self, path):
is_blob = self.exists(path)
if is_blob:
self._blob.delete_blob(self._container, path)
return
exists = False
blobs = list(self._blob.list_blobs(self._container, path))
if not blobs:
raise exceptions.FileNotFoundError('%s is not there' % path)
for b in blobs:
self._blob.delete_blob(self._container, b.name)
def get_size(self, path):
try:
properties = self._blob.get_blob_properties(self._container, path)
return int(properties['content-length']) # auto-converted to long
except azure.WindowsAzureMissingResourceError:
raise exceptions.FileNotFoundError('%s is not there' % path)
blob_service = BlobService(account_name=os.getenv('ACC_NAME'), account_key=os.getenv('ACCESS_KEY'))
# In[32]:
b_path = r"C:\Users\Robin\Downloads\bootstrap-3.3.5-dist\bootstrap-3.3.5-dist"
# In[12]:
for root, subdirs, files in os.walk(b_path):
with open(os.path.join(subdirs,files)) as file:
blob_service.put_block_blob_from_file(
'csvs',
subdirs+files,
file,
x_ms_blob_content_type='zip'
)
# In[34]:
for path, subdirs, files in os.walk(b_path):
for name in files:
full_path = os.path.join(path, name)
path_no_file = os.path.split(full_path)[0]
last_dir = os.path.split(path_no_file)[1]
print last_dir+"/"+name
