def handle_noargs(self, **options):
try:
blob_service = BlobService(AZURE_ACCOUNT_NAME, AZURE_ACCOUNT_KEY)
mixes = Mix.objects.filter(archive_updated=False)
c = len(mixes)
i = 1
for mix in mixes:
try:
blob_name = "%s.%s" % (mix.uid, mix.filetype)
blob = blob_service.get_blob(AZURE_CONTAINER, blob_name)
if blob:
download_name = smart_str('Deep South Sounds - %s.%s' %
(mix.title, mix.filetype))
blob_service.set_blob_properties(
AZURE_CONTAINER,
blob_name,
x_ms_blob_content_type='application/octet-stream',
x_ms_blob_content_disposition='attachment;filename="%s"' % (download_name)
)
print "Processed: %s (%d of %d)" % (mix.uid, i, c)
i = i + 1
mix.archive_updated = True
mix.save()
else:
print "No blob found for: %s" % mix.uid
except WindowsAzureMissingResourceError:
print "No blob found for: %s" % mix.uid
except Exception, ex:
print "Error processing blob %s: %s" % (mix.uid, ex.message)
except Exception, ex:
print "Fatal error, bailing. %s" % (ex.message)
def test_azure_call(request):
import os
try:
from azure.storage import BlobService
bs = BlobService(os.environ["AZURE_STORAGE_ACCOUNT"], os.environ["AZURE_STORAGE_ACCESS_KEY"])
import random
container_name = hex(int(random.random() * 1000000000))
bs.create_container(container_name)
bs.put_blob(container_name, 'testblob', 'hello world\n', 'BlockBlob')
blob = bs.get_blob(container_name, 'testblob')
if blob != 'hello world\n':
return HttpResponse("Failed!", status = '404')
bs.delete_blob(container_name, 'testblob')
bs.delete_container(container_name)
return HttpResponse("Succeeded!")
except:
try:
import traceback
return HttpResponse(traceback.format_exc() + str(os.environ.keys()))
except:
import traceback
return HttpResponse(traceback.format_exc())
def uri_get_file(creds, uri, conn=None):
assert uri.startswith('wabs://')
url_tup = urlparse(uri)
if conn is None:
conn = BlobService(creds.account_name,
creds.account_key,
protocol='https')
# Determin the size of the target blob
props = conn.get_blob_properties(url_tup.netloc, url_tup.path)
blob_size = int(props['content-length'])
ret_size = 0
data = ''
# WABS requires large files to be downloaded in 4MB chunks
while ret_size < blob_size:
ms_range = 'bytes={}-{}'.format(ret_size,
ret_size + WABS_CHUNK_SIZE - 1)
while True:
# Because we're downloading in chunks, catch rate limiting and
# connection errors here instead of letting them bubble up to the
# @retry decorator so that we don't have to start downloading the
# whole file over again.
try:
part = conn.get_blob(url_tup.netloc,
url_tup.path,
x_ms_range=ms_range)
except EnvironmentError as e:
if e.errno in (errno.EBUSY, errno.ECONNRESET):
logger.warning(
msg="retrying after encountering exception",
detail=("Exception traceback:\n{0}".format(
traceback.format_exception(*sys.exc_info()))),
hint="")
gevent.sleep(30)
else:
raise
else:
break
length = len(part)
ret_size += length
data += part
if length > 0 and length < WABS_CHUNK_SIZE:
break
elif length == 0:
break
return data
def uri_get_file(creds, uri, conn=None):
assert uri.startswith('wabs://')
url_tup = urlparse(uri)
if conn is None:
conn = BlobService(creds.account_name, creds.account_key,
protocol='https')
# Determin the size of the target blob
props = conn.get_blob_properties(url_tup.netloc, url_tup.path)
blob_size = int(props['content-length'])
ret_size = 0
data = ''
# WABS requires large files to be downloaded in 4MB chunks
while ret_size < blob_size:
ms_range = 'bytes={0}-{1}'.format(ret_size,
ret_size + WABS_CHUNK_SIZE - 1)
while True:
# Because we're downloading in chunks, catch rate limiting and
# connection errors here instead of letting them bubble up to the
# @retry decorator so that we don't have to start downloading the
# whole file over again.
try:
part = conn.get_blob(url_tup.netloc,
url_tup.path,
x_ms_range=ms_range)
except EnvironmentError as e:
if e.errno in (errno.EBUSY, errno.ECONNRESET):
logger.warning(
msg="retrying after encountering exception",
detail=("Exception traceback:\n{0}".format(
traceback.format_exception(*sys.exc_info()))),
hint="")
gevent.sleep(30)
else:
raise
else:
break
length = len(part)
ret_size += length
data += part
if length > 0 and length < WABS_CHUNK_SIZE:
break
elif length == 0:
break
return data
def process(self):
account_name = self.parameters.azure_account_name
account_key = self.parameters.azure_account_key
blob_service = BlobService(account_name, account_key, protocol="https")
proxy_setting = self.parameters.https_proxy or ""
date_setting = self.parameters.date or ""
date = None
if date_setting:
if date_setting != "yesterday":
date = datetime.datetime.strptime(date_setting, "%Y-%m-%d").date() # for debbuging (probably)
elif date_setting == "yesterday":
date = datetime.date.today() - datetime.timedelta(days=1) # for normal usage
proxy_url = "https://" + proxy_setting if proxy_setting.find("https://") == -1 else proxy_setting
proxy_options = urlparse(proxy_url)
if date:
self.logger.info("Fetching for date: %s (%s)" % (date, date_setting))
else:
self.logger.info("No 'date' was specified, fetching ALL")
if proxy_options.hostname:
self.logger.info("Using https proxy(host=%s, port=%s)" % (proxy_options.hostname, proxy_options.port))
blob_service.set_proxy(host=proxy_options.hostname, port=proxy_options.port)
else:
if proxy_setting:
self.logger.info("Using NO proxy, couldn't use 'https_proxy' it was: %s" % proxy_setting)
else:
self.logger.info("Using NO proxy, 'https_proxy' was empty")
for container in blob_service.list_containers():
container_name = container.name
if container_name == "heartbeat":
continue
if date and (not container_name == "processed-" + str(date)):
self.logger.info("IGNORING container '%s' didn't match date selection" % container_name)
continue
for blob in blob_service.list_blobs(container_name):
self.logger.info("Fetching blob %s in container %s" % (container_name, blob.name))
data = blob_service.get_blob(container_name, blob.name)
cs = StringIO.StringIO(data)
report = gzip.GzipFile(fileobj=cs).read()
self.send_message(report)
# Repeat
while(True):
# get images form *imagesQueue* - it is invoked by CRON
messages = queue_service.get_messages( imagesQueue )
if len(messages) == 0:
sleep(15)
for message in messages:
# get image: image ID
imgBlobName = b64decode( message.message_text )
print( imgBlobName )
tableRowKey = imgBlobName
try:
blob = blob_service.get_blob(blob_container, imgBlobName)
except azure.WindowsAzureMissingResourceError:
queue_service.delete_message( imagesQueue, message.message_id, message.pop_receipt )
continue
image = blobToOpenCV(blob)
# ADDED#2 ########################################
if image is None:
print "GIF attempt in analyse"
queue_service.delete_message( imagesQueue, message.message_id, message.pop_receipt )
blob_service.delete_blob( blob_container, imgBlobName)
table_service.delete_entity( tableName, tablePartitionKey, tableRowKey)
continue
class AzureBlobStorage(Storage):
'''
classdocs
'''
def __init__(self, azure_profile):
'''
Constructor
'''
if not azure_profile:
raise Exception()
else:
container_name = azure_profile['container_name']
account_name = azure_profile['account_name']
account_key = azure_profile['key']
base_url = azure_profile['base_url']
self.blob = BlobService(
account_name=account_name, account_key=account_key)
self.container = container_name
self.base_url = base_url
def delete(self, name):
"""
Delete file.
"""
try:
self.blob.delete_blob(self.container, name)
except WindowsAzureMissingResourceError:
return False
else:
return True
def delete_files(self, files=None):
"""
Delete files in container.
"""
if not files:
files = self.listdir(self.container)[1]
for _file in files:
self.delete(_file)
def exists(self, name, with_properties=False):
"""
Existing check.
"""
result = False
blob_properties = None
try:
blob_properties = self.blob.get_blob_properties(
self.container, name)
except WindowsAzureMissingResourceError:
result = False
else:
result = True
if with_properties:
return result, blob_properties
else:
return result
def get_available_name(self, name):
return super(AzureBlobStorage, self).get_available_name(name.replace('\\', '/'))
def get_valid_name(self, name):
return name
def _list(self, path, prefix, maxresults):
result = []
blobs = self.blob.list_blobs(path, prefix, maxresults)
for _blob in blobs:
result.append(_blob.name)
return result
def listdir(self, path=None, prefix=None, maxresults=None):
"""
Catalog file list.
"""
if not path:
path = self.container
return [], self._list(path, prefix, maxresults)
def size(self, name):
"""
File size.
"""
result, properties = self.exists(name, with_properties=True)
if result:
return int(properties['content-length'])
else:
return 0
def url(self, name, chk_exist=False):
"""
URL for file downloading.
"""
if chk_exist:
if self.exists(name):
return '%s%s/%s' % (self.base_url, self.container, name)
else:
return None
else:
return '%s%s/%s' % (self.base_url, self.container, name)
def _open(self, name, mode='rb'):
"""
Open file.
"""
in_mem_file = StringIO.StringIO(
self.blob.get_blob(self.container, name))
in_mem_file.name = name
in_mem_file.mode = mode
return File(in_mem_file)
def _save(self, name, blob_to_upload, x_ms_blob_type='BlockBlob', content_type=None):
"""
Save file.
"""
if hasattr(blob_to_upload, 'content_type'):
content_type = blob_to_upload.content_type or None
if content_type is None:
content_type = mimetypes.guess_type(name)[0] or None
blob_to_upload.seek(0)
self.blob.put_blob(self.container, name, blob_to_upload,
x_ms_blob_type, x_ms_blob_content_type=content_type)
return name
def modified_time(self, name):
"""
Last modification time.
"""
result, properties = self.exists(name, with_properties=True)
if result:
date_string = properties['last-modified']
modified_dt = parser.parse(date_string)
if timezone.is_naive(modified_dt):
return modified_dt
else:
return timezone.make_naive(modified_dt, timezone.get_current_timezone())
else:
return None
created_time = accessed_time = modified_time
parser = OptionParser()
parser.add_option("-c", "--config", metavar="FILE",
action="store", type="string", dest="conf_filename",
help="name of config file to use")
parser.add_option("-n", "--blob-name",
action="store", type="string", dest="blob_name",
help="name of the blob for Graphite path")
(opts, args) = parser.parse_args()
from azure.storage import BlobService
try:
conf_fh = open(opts.conf_filename)
conf_data = json.load(conf_fh)
# ts = TableService(conf_data["account_name"], conf_data["account_key"])
# entity = ts.get_entity(conf_data["table_name"], conf_data["partition_key"], conf_data["row_key"])
# columns = dir(entity)
# print columns
# print entity.coalitionType
bs = BlobService(conf_data["account_name"], conf_data["account_key"])
start_time = time.time()
ap_blob = bs.get_blob(conf_data["blob_container"], conf_data["blob_id"])
# print ap_blob
complete_time = time.time()
elapsed_time = complete_time - start_time
print "impresys." + opts.blob_name + ".latency\t" + str(elapsed_time) + "\t" + str(int(time.mktime(time.gmtime())))
except:
# print "Azure access failed"
print ""
class AzureFS(LoggingMixIn, Operations):
"""Azure Blob Storage filesystem"""
blobs = None
containers = dict() # <cname, dict(stat:dict,
#files:None|dict<fname, stat>)
fds = dict() # <fd, (path, bytes, dirty)>
fd = 0
def __init__(self, account, key):
self.blobs = BlobService(account, key)
self.rebuild_container_list()
def convert_to_epoch(self, date):
"""Converts Tue, 31 Jul 2012 07:17:34 GMT format to epoch"""
return int(time.mktime(time.strptime(date, TIME_FORMAT)))
def rebuild_container_list(self):
cmap = dict()
cnames = set()
for c in self.blobs.list_containers():
date = c.properties.last_modified
cstat = dict(st_mode=(S_IFDIR | 0755), st_uid=getuid(), st_size=0,
st_mtime=self.convert_to_epoch(date))
cname = c.name
cmap['/' + cname] = dict(stat=cstat, files=None)
cnames.add(cname)
cmap['/'] = dict(files={},
stat=dict(st_mode=(S_IFDIR | 0755),
st_uid=getuid(), st_size=0,
st_mtime=int(time.time())))
self.containers = cmap # destroys fs tree cache resistant to misses
def _parse_path(self, path): # returns </dir, file(=None)>
if path.count('/') > 1: # file
return str(path[:path.rfind('/')]), str(path[path.rfind('/') + 1:])
else: # dir
pos = path.rfind('/', 1)
if pos == -1:
return path, None
else:
return str(path[:pos]), None
def parse_container(self, path):
base_container = path[1:] # /abc/def/g --> abc
if base_container.find('/') > -1:
base_container = base_container[:base_container.find('/')]
return str(base_container)
def _get_dir(self, path, contents_required=False):
if not self.containers:
self.rebuild_container_list()
if path in self.containers and not (contents_required and \
self.containers[path]['files'] is None):
return self.containers[path]
cname = self.parse_container(path)
if '/' + cname not in self.containers:
raise FuseOSError(ENOENT)
else:
if self.containers['/' + cname]['files'] is None:
# fetch contents of container
log.info("------> CONTENTS NOT FOUND: %s" % cname)
blobs = self.blobs.list_blobs(cname)
dirstat = dict(st_mode=(S_IFDIR | 0755), st_size=0,
st_uid=getuid(), st_mtime=time.time())
if self.containers['/' + cname]['files'] is None:
self.containers['/' + cname]['files'] = dict()
for f in blobs:
blob_name = f.name
blob_date = f.properties.last_modified
blob_size = long(f.properties.content_length)
node = dict(st_mode=(S_IFREG | 0644), st_size=blob_size,
st_mtime=self.convert_to_epoch(blob_date),
st_uid=getuid())
if blob_name.find('/') == -1: # file just under container
self.containers['/' + cname]['files'][blob_name] = node
return self.containers['/' + cname]
return None
def _get_file(self, path):
d, f = self._parse_path(path)
dir = self._get_dir(d, True)
if dir is not None and f in dir['files']:
return dir['files'][f]
def getattr(self, path, fh=None):
d, f = self._parse_path(path)
if f is None:
dir = self._get_dir(d)
return dir['stat']
else:
file = self._get_file(path)
if file:
return file
raise FuseOSError(ENOENT)
# FUSE
def mkdir(self, path, mode):
if path.count('/') <= 1: # create on root
name = path[1:]
if not 3 <= len(name) <= 63:
log.error("Container names can be 3 through 63 chars long.")
raise FuseOSError(ENAMETOOLONG)
if name is not name.lower():
log.error("Container names cannot contain uppercase \
characters.")
raise FuseOSError(EACCES)
if name.count('--') > 0:
log.error('Container names cannot contain consecutive \
dashes (-).')
raise FuseOSError(EAGAIN)
#TODO handle all "-"s must be preceded by letter or numbers
#TODO starts with only letter or number, can contain letter, nr,'-'
resp = self.blobs.create_container(name)
if resp:
self.rebuild_container_list()
log.info("CONTAINER %s CREATED" % name)
else:
raise FuseOSError(EACCES)
log.error("Invalid container name or container already \
exists.")
else:
raise FuseOSError(ENOSYS) # TODO support 2nd+ level mkdirs
def rmdir(self, path):
if path.count('/') == 1:
c_name = path[1:]
resp = self.blobs.delete_container(c_name)
if resp:
if path in self.containers:
del self.containers[path]
else:
raise FuseOSError(EACCES)
else:
raise FuseOSError(ENOSYS) # TODO support 2nd+ level mkdirs
def create(self, path, mode):
node = dict(st_mode=(S_IFREG | mode), st_size=0, st_nlink=1,
st_uid=getuid(), st_mtime=time.time())
d, f = self._parse_path(path)
if not f:
log.error("Cannot create files on root level: /")
raise FuseOSError(ENOSYS)
dir = self._get_dir(d, True)
if not dir:
raise FuseOSError(EIO)
dir['files'][f] = node
return self.open(path, data='') # reusing handler provider
def open(self, path, flags=0, data=None):
if data == None: # download contents
c_name = self.parse_container(path)
f_name = path[path.find('/', 1) + 1:]
try:
data = self.blobs.get_blob(c_name, f_name)
except WindowsAzureMissingResourceError:
dir = self._get_dir('/' + c_name, True)
if f_name in dir['files']:
del dir['files'][f_name]
raise FuseOSError(ENOENT)
except WindowsAzureError as e:
log.error("Read blob failed HTTP %d" % e.code)
raise FuseOSError(EAGAIN)
self.fd += 1
self.fds[self.fd] = (path, data, False)
return self.fd
def flush(self, path, fh=None):
if not fh:
raise FuseOSError(EIO)
else:
if fh not in self.fds:
raise FuseOSError(EIO)
path = self.fds[fh][0]
data = self.fds[fh][1]
dirty = self.fds[fh][2]
if not dirty:
return 0 # avoid redundant write
d, f = self._parse_path(path)
c_name = self.parse_container(path)
if data is None:
data = ''
try:
if len(data) < 64 * 1024 * 1024: # 64 mb
self.blobs.put_blob(c_name, f, data, 'BlockBlob')
else:
# divide file by blocks and upload
block_size = 8 * 1024 * 1024
num_blocks = int(math.ceil(len(data) * 1.0 / block_size))
rd = str(random.randint(1, 1e8))
block_ids = list()
for i in range(num_blocks):
part = data[i * block_size:min((i + 1) * block_size,
len(data))]
block_id = base64.encodestring('%s_%s' % (rd,
(8 - len(str(i))) * '0' + str(i)))
self.blobs.put_block(c_name, f, part, block_id)
block_ids.append(block_id)
self.blobs.put_block_list(c_name, f, block_ids)
except WindowsAzureError:
raise FuseOSError(EAGAIN)
dir = self._get_dir(d, True)
if not dir or f not in dir['files']:
raise FuseOSError(EIO)
# update local data
dir['files'][f]['st_size'] = len(data)
dir['files'][f]['st_mtime'] = time.time()
self.fds[fh] = (path, data, False) # mark as not dirty
return 0
def release(self, path, fh=None):
if fh is not None and fh in self.fds:
del self.fds[fh]
def truncate(self, path, length, fh=None):
return 0 # assume done, no need
def write(self, path, data, offset, fh=None):
if not fh or fh not in self.fds:
raise FuseOSError(ENOENT)
else:
d = self.fds[fh][1]
if d is None:
d = ""
self.fds[fh] = (self.fds[fh][0], d[:offset] + data, True)
return len(data)
def unlink(self, path):
c_name = self.parse_container(path)
d, f = self._parse_path(path)
try:
self.blobs.delete_blob(c_name, f)
_dir = self._get_dir(path, True)
if _dir and f in _dir['files']:
del _dir['files'][f]
return 0
except WindowsAzureMissingResourceError:
raise FuseOSError(ENOENT)
except Exception as e:
raise FuseOSError(EAGAIN)
def readdir(self, path, fh):
if path == '/':
return ['.', '..'] + [x[1:] for x in self.containers.keys() \
if x is not '/']
dir = self._get_dir(path, True)
if not dir:
raise FuseOSError(ENOENT)
return ['.', '..'] + dir['files'].keys()
def read(self, path, size, offset, fh):
if not fh or fh not in self.fds:
raise FuseOSError(ENOENT)
f_name = path[path.find('/', 1) + 1:]
c_name = path[1:path.find('/', 1)]
try:
data = self.blobs.get_blob(c_name, f_name)
self.fds[fh] = (self.fds[fh][0], data, False)
return data[offset:offset + size]
except URLError, e:
if e.code == 404:
raise FuseOSError(ENOENT)
elif e.code == 403:
raise FUSEOSError(EPERM)
else:
log.error("Read blob failed HTTP %d" % e.code)
raise FuseOSError(EAGAIN)
data = self.fds[fh][1]
if data is None:
data = ""
return data[offset:offset + size]
sys.stderr.write("Azure key is missing")
sys.exit(1)
if header and not options.output_format:
print '\t'.join(str(h) for h in headers)
blob_service = BlobService(account_name, account_key)
for container in blob_service.list_containers():
c = container.name
if c == "heartbeat": continue
if options.date and not ( c == "processed-"+options.date ): continue
if debug: sys.stderr.write("Processing container: "+str(c)+"\n")
for b in blob_service.list_blobs(c):
if debug: sys.stderr.write("Processing blob: "+str(b.name)+"\n")
data = blob_service.get_blob(c, b.name)
cs = StringIO.StringIO(data)
gzipstream = gzip.GzipFile(fileobj=cs)
if output_format == "txt":
print gzipstream.read()
elif output_format == "json":
d = {}
i = 0
ds = gzipstream.read()
# some DCU entries contains more than 28 values (outside the
# definition of the headers)
for x in ds.strip().split("\t")[:27]:
d[headers[i]] = x
i=i+1
print (json.dumps(d, sort_keys=True))
if options.clear:
class Storage(driver.Base):
supports_bytes_range = True
def __init__(self, path=None, config=None):
self._config = config
self._container = self._config.azure_storage_container
protocol = 'https' if self._config.azure_use_https else 'http'
acct_name = self._config.azure_storage_account_name
acct_key = self._config.azure_storage_account_key
self._blob = BlobService(
account_name=acct_name, account_key=acct_key, protocol=protocol)
self._init_container()
logger.debug("Initialized azureblob storage driver")
def _init_container(self):
'''Initializes image container on Azure blob storage if the container
does not exist.
'''
created = self._blob.create_container(
self._container, x_ms_blob_public_access='blob',
fail_on_exist=False)
if created:
logger.info('Created blob container for image registry.')
else:
logger.debug('Registry container already exists.')
return created
@lru.get
def get_content(self, path):
try:
return self._blob.get_blob(self._container, path)
except azure.WindowsAzureMissingResourceError:
raise exceptions.FileNotFoundError('%s is not there' % path)
@lru.set
def put_content(self, path, content):
self._blob.put_blob(self._container, path, content, 'BlockBlob')
return path
def stream_read(self, path, bytes_range=None):
try:
f = io.BytesIO()
self._blob.get_blob_to_file(self._container, path, f)
if bytes_range:
f.seek(bytes_range[0])
total_size = bytes_range[1] - bytes_range[0] + 1
else:
f.seek(0)
while True:
buf = None
if bytes_range:
# Bytes Range is enabled
buf_size = self.buffer_size
if nb_bytes + buf_size > total_size:
# We make sure we don't read out of the range
buf_size = total_size - nb_bytes
if buf_size > 0:
buf = f.read(buf_size)
nb_bytes += len(buf)
else:
# We're at the end of the range
buf = ''
else:
buf = f.read(self.buffer_size)
if not buf:
break
yield buf
except IOError:
raise exceptions.FileNotFoundError('%s is not there' % path)
def stream_write(self, path, fp):
self._blob.put_block_blob_from_file(self._container, path, fp)
def list_directory(self, path=None):
if not path.endswith('/'):
path += '/' # path=a would list a/b.txt as well as 'abc.txt'
blobs = list(self._blob.list_blobs(self._container, path))
if not blobs:
raise exceptions.FileNotFoundError('%s is not there' % path)
return [b.name for b in blobs]
def exists(self, path):
try:
self._blob.get_blob_properties(self._container, path)
return True
except azure.WindowsAzureMissingResourceError:
return False
@lru.remove
def remove(self, path):
is_blob = self.exists(path)
if is_blob:
self._blob.delete_blob(self._container, path)
return
exists = False
blobs = list(self._blob.list_blobs(self._container, path))
if not blobs:
raise exceptions.FileNotFoundError('%s is not there' % path)
for b in blobs:
self._blob.delete_blob(self._container, b.name)
def get_size(self, path):
try:
properties = self._blob.get_blob_properties(self._container, path)
return int(properties['content-length']) # auto-converted to long
except azure.WindowsAzureMissingResourceError:
raise exceptions.FileNotFoundError('%s is not there' % path)
class AzureBlobStorage(Storage):
def __init__(self, account='nyxstorage', container='pxo'):
self.base_storage_uri = 'http://%s.blob.core.windows.net/%s/' % (
account, container)
self.blob_service = BlobService(
account, get_env_variable('AZURE_BLOB_STORAGE_KEY'))
self.container = container
def _open(self, name, mode='rb'):
data = self.blob_service.get_blob(self.container, name)
return ContentFile(data)
def _save(self, name, content):
_file = content.read()
file_name = content.name[-35:]
self.blob_service.put_blob(
self.container, file_name, _file, x_ms_blob_type='BlockBlob')
return self.base_storage_uri + file_name
def create_container(self, container_name):
result = self.blob_service.create_container(
container_name, x_ms_blob_public_access='container')
return result
def delete(self, name):
self.blob_service.delete_blob(self.container, name)
def exists(self, name):
try:
self.blob_service.get_blob_properties(self.container, name)
except:
return False
else:
return True
def get_available_name(self, name):
return name
def get_blobs(self):
blobs = self.blob_service.list_blobs(self.container)
return blobs
def get_valid_name(self, name):
return name
def modified_time(self, name):
metadata = self.blob_service.get_blob_metadata(self.container, name)
modified_time = float(metadata.get('x-ms-meta-modified_time'))
return datetime.fromtimestamp(modified_time)
def set_public_container(self, container_name):
result = self.blob_service.set_container_acl(
container_name, x_ms_blob_public_access='container')
return result
def size(self, name):
properties = self.blob_service.get_blob_properties(
self.container, name)
return properties.get('content-length')
def url(self, name):
blob = self.blob_service.list_blobs(self.container, prefix=name)
return blob.blobs[0].url
class Storage(driver.Base):
supports_bytes_range = True
def __init__(self, path=None, config=None):
self._config = config
self._container = self._config.azure_storage_container
protocol = 'https' if self._config.azure_use_https else 'http'
acct_name = self._config.azure_storage_account_name
acct_key = self._config.azure_storage_account_key
self._blob = BlobService(account_name=acct_name,
account_key=acct_key,
protocol=protocol)
self._init_container()
logger.debug("Initialized azureblob storage driver")
def _init_container(self):
'''Initializes image container on Azure blob storage if the container
does not exist.
'''
created = self._blob.create_container(self._container,
x_ms_blob_public_access='blob',
fail_on_exist=False)
if created:
logger.info('Created blob container for image registry.')
else:
logger.debug('Registry container already exists.')
return created
@lru.get
def get_content(self, path):
try:
return self._blob.get_blob(self._container, path)
except azure.WindowsAzureMissingResourceError:
raise exceptions.FileNotFoundError('%s is not there' % path)
@lru.set
def put_content(self, path, content):
self._blob.put_blob(self._container, path, content, 'BlockBlob')
return path
def stream_read(self, path, bytes_range=None):
try:
f = io.BytesIO()
self._blob.get_blob_to_file(self._container, path, f)
if bytes_range:
f.seek(bytes_range[0])
total_size = bytes_range[1] - bytes_range[0] + 1
else:
f.seek(0)
while True:
buf = None
if bytes_range:
# Bytes Range is enabled
buf_size = self.buffer_size
if nb_bytes + buf_size > total_size:
# We make sure we don't read out of the range
buf_size = total_size - nb_bytes
if buf_size > 0:
buf = f.read(buf_size)
nb_bytes += len(buf)
else:
# We're at the end of the range
buf = ''
else:
buf = f.read(self.buffer_size)
if not buf:
break
yield buf
except IOError:
raise exceptions.FileNotFoundError('%s is not there' % path)
def stream_write(self, path, fp):
self._blob.put_block_blob_from_file(self._container, path, fp)
def list_directory(self, path=None):
if not path.endswith('/'):
path += '/' # path=a would list a/b.txt as well as 'abc.txt'
blobs = list(self._blob.list_blobs(self._container, path))
if not blobs:
raise exceptions.FileNotFoundError('%s is not there' % path)
return [b.name for b in blobs]
def exists(self, path):
try:
self._blob.get_blob_properties(self._container, path)
return True
except azure.WindowsAzureMissingResourceError:
return False
@lru.remove
def remove(self, path):
is_blob = self.exists(path)
if is_blob:
self._blob.delete_blob(self._container, path)
return
exists = False
blobs = list(self._blob.list_blobs(self._container, path))
if not blobs:
raise exceptions.FileNotFoundError('%s is not there' % path)
for b in blobs:
self._blob.delete_blob(self._container, b.name)
def get_size(self, path):
try:
properties = self._blob.get_blob_properties(self._container, path)
return int(properties['content-length']) # auto-converted to long
except azure.WindowsAzureMissingResourceError:
raise exceptions.FileNotFoundError('%s is not there' % path)
def retrieve(entity, entity_id):
blob_service = BlobService(account_name='shnergledata',
account_key=os.environ['BLOB_KEY'])
name = '/' + entity + '/' + entity_id
blob = blob_service.get_blob('images', name)
return blob
break
else:
pass
# OK
else:
# continue to next message
miniturisation = False
break
if miniturisation == False:
# continue to next message
continue
# Get big image analysis
try:
blobAnalysis = blob_service.get_blob(blob_analysis, imgBlobName)
except azure.WindowsAzureMissingResourceError:
queue_service.delete_message(imagesQueue, message.message_id,
message.pop_receipt)
continue
bigImageAnalysis = blobToArray(blobAnalysis)
# Get big image
try:
blobBigImage = blob_service.get_blob(blob_big, imgBlobName)
except azure.WindowsAzureMissingResourceError:
queue_service.delete_message(imagesQueue, message.message_id,
message.pop_receipt)
continue
bigImage = blobToOpenCV(blobBigImage)
# Resize big image
# Repeat
while (True):
# get images form *imagesQueue* - it is invoked by CRON
messages = queue_service.get_messages(imagesQueue)
if len(messages) == 0:
sleep(15)
for message in messages:
# get image: image ID
imgBlobName = b64decode(message.message_text)
print(imgBlobName)
tableRowKey = imgBlobName
try:
blob = blob_service.get_blob(blob_container, imgBlobName)
except azure.WindowsAzureMissingResourceError:
queue_service.delete_message(imagesQueue, message.message_id,
message.pop_receipt)
continue
image = blobToOpenCV(blob)
# ADDED#2 ########################################
if image is None:
print "GIF attempt in analyse"
queue_service.delete_message(imagesQueue, message.message_id,
message.pop_receipt)
blob_service.delete_blob(blob_container, imgBlobName)
table_service.delete_entity(tableName, tablePartitionKey,
tableRowKey)
