def copyBlobToBlob(sourceUrl, sourceKey, destUrl, destKey):
blobservice = BlobService(destUrl, destkey)
srcblobservice = BlobService(SourceUrl, srckey)
today = datetime.datetime.utcnow()
todayPlusMonth = today + datetime.timedelta(1)
todayPlusMonthISO = todayPlusMonth.replace(microsecond=0).isoformat() + 'Z'
srcSasParam = srcblobservice.generate_shared_access_signature(container,
filename, SharedAccessPolicy(AccessPolicy(None, todayPlusMonthISO, "r"), None))
srcUrl = srcblobservice.make_blob_url(container, filename,
sas_token=srcSasParam)
print srcUrl
blobservice.copy_blob(container, filename, srcUrl)
def __init__(self, config):
tree = ET.parse('SharedConfig.xml')
self.myMachineName = tree.find('.//Instance').get("id")
self.sms = ServiceManagementService(
subscription_id=config.get("azure", "subscription_id"),
cert_file=config.get("azure", "cert_file")
);
self.bus_service = ServiceBusService(
service_namespace=config.get("azure", "bus_namespace"),
shared_access_key_name=config.get("azure", "bus_shared_access_key_name"),
shared_access_key_value=config.get("azure", "bus_shared_access_key_value"))
self.command_queue = config.get("azure", "commandQueuePath")
for tries in range(1,10):
try:
self.bus_service.create_queue(self.command_queue)
break
except:
print "Esperando"
self.status_topic = config.get("azure", "statusTopicPath")
self.bus_service.create_queue(self.status_topic)
self.storage = BlobService(account_name=config.get("azure", "account_name"),
account_key=config.get("azure", "account_key"))
self.algo_storage_name = config.get("azure", "algorithm_storage_name")
self.storage.create_container(self.algo_storage_name, fail_on_exist=False)
self.proj_storage_name = config.get("azure", "project_storage_name")
self.storage.create_container(self.proj_storage_name, fail_on_exist=False)
def submit():
blob_service = BlobService(account_name=ACCOUNT_NAME,
account_key=ACCOUNT_KEY)
# Get a SAS signature (read for 24 hours) for the input container save to a string
inputsig = sasUrl(account=ACCOUNT_NAME,
key=ACCOUNT_KEY,
container=INPUT_CONTAINER,
permission='r')
# Get a SAS signature (write for 24 hours) for the output container save to a string
outputsig = sasUrl(account=ACCOUNT_NAME,
key=ACCOUNT_KEY,
container=OUTPUT_CONTAINER,
permission='rwl')
# List all the blobs and dump the content to a string
blobs = blob_service.list_blobs(INPUT_CONTAINER)
bloblist = []
for blob in blobs:
bloblist.append(blob.name)
os.environ[SLURMDEMO_INPUTSIG] = inputsig
os.environ[SLURMDEMO_OUTPUTSIG] = outputsig
os.environ[SLURMDEMO_BLOBLIST] = json.dumps(bloblist)
os.environ[SLURMDEMO_INPUTCONTAINER] = INPUT_CONTAINER
os.environ[SLURMDEMO_OUTPUTCONTAINER] = OUTPUT_CONTAINER
os.environ[SLURMDEMO_INPUTACCOUNT] = ACCOUNT_NAME
os.environ[SLURMDEMO_OUTPUTACCOUNT] = ACCOUNT_NAME
# Call sbatch
cli = "sbatch --array=0-{nb} slurmdemo.sh".format(nb=len(bloblist))
run(cli, showoutput=True)
def connect(self, creds):
"""Return an azure BlobService instance.
"""
return BlobService(account_name=creds.account_name,
account_key=creds.account_key,
sas_token=creds.access_token,
protocol='https')
def test_no_retry_on_keyboadinterrupt(collect):
"""Ensure that KeyboardInterrupts are forwarded."""
key_name = 'test-key-name'
b = B(name=key_name)
# If vanilla KeyboardInterrupt is used, then sending SIGINT to the
# test can cause it to pass improperly, so use a subtype instead.
class MarkedKeyboardInterrupt(KeyboardInterrupt):
pass
collect.inject(MarkedKeyboardInterrupt('SIGINT, probably'))
d = wabs_deleter.Deleter(BlobService('test', 'ing'), 'test-container')
with pytest.raises(MarkedKeyboardInterrupt):
d.delete(b)
# Exactly when coroutines are scheduled is non-deterministic,
# so spin while yielding to provoke the
# MarkedKeyboardInterrupt being processed within the
# pytest.raises context manager.
while True:
gevent.sleep(0.1)
# Only one key should have been aborted, since the purpose is to
# *not* retry when processing KeyboardInterrupt.
assert collect.aborted_keys == [key_name]
# Turn off fault injection and flush/synchronize with close().
collect.inject(None)
d.close()
# Since there is no retrying, no keys should be deleted.
assert not collect.deleted_keys
def connect(self, host, port, user, password, secure):
'''
Connect to the Azure service with given user and key
@param user - username to use to connect to
@param key - key to use to connect
'''
kwargs = {}
err = None
if not host is None:
kwargs["host_base"] = "." + host
if not user is None:
kwargs["account_name"] = user
elif not self.user is None:
kwargs["account_name"] = self.user
if not password is None:
kwargs["account_key"] = password
elif not self.key is None:
kwargs["account_key"] = self.key
kwargs["protocol"] = "https" if secure else "http"
try:
self.service = BlobService(**kwargs)
except Exception as e:
err = e.message
self.service = None
if self.service is None:
raise OsakaException("Failed to connect to Azure:" +
("" if err is None else err))
def _cleanUpExternalStore(self, containerName):
from toil.jobStores.azureJobStore import _fetchAzureAccountKey
from azure.storage.blob import BlobService
blobService = BlobService(account_key=_fetchAzureAccountKey(
self.accountName),
account_name=self.accountName)
blobService.delete_container(containerName)
def _get_service(self):
if not hasattr(self, '_blob_service'):
self._blob_service = BlobService(account_name=self.account_name,
account_key=self.account_key,
protocol='http')
#print "exiting _get_service"
return self._blob_service
def get_tags():
#TODO: Error checking
global d
d = {}
clarifai_api = ClarifaiApi()
blob_service = BlobService('calhacks', 'mm7EmY+T+MGahePBDSDU5LHpZR5tRXuh4MSco4jFrzHovOPEf06e18c89pxtPIo4NDVhhjSeaQY/FQmKNxjjyA==')
blob_name = request.data
blob_name = blob_name.decode('utf-8')
blob_service.get_blob_to_path('imagestore', blob_name, 'out.png')
print("checkpoint 1")
i = open ('out.png', 'r')
strd = ""
for line in i:
strd += line.strip()
fname = 'img.png'
with open (fname, 'wb') as f:
f.write (base64.b64decode(strd))
f.close()
f = open (fname, 'rb')
result = clarifai_api.tag_images(f)
print(result)
st = result['results'][0]['result']['tag']['classes'][0:6]
for i in ['food', 'nobody', 'still life', 'meal', 'dish', 'plate', 'delicious', 'isolated', 'cutout', 'unhealthy', 'one', 'background']:
while i in st:
st.remove(i)
d = {blob_name: search_terms(st)}
return "success!"
def __init__(self):
super(BlobServiceAdapter, self).__init__(
BlobService(
account_name=self.util.get_config(
"storage.azure.account_name"),
account_key=self.util.get_config("storage.azure.account_key"),
host_base=self.util.get_config(
"storage.azure.blob_service_host_base")))
def enumerate_objects(container):
blob_service = BlobService(AZURE_ACCOUNT_NAME, AZURE_ACCOUNT_KEY)
blobs = blob_service.list_blobs(container)
items = []
for blob in blobs:
items.append(blob.name)
return items
def _get_service(self):
if not hasattr(self, '_blob_service'):
self._blob_service = BlobService(
account_name=self.account_name,
account_key=self.account_key,
protocol='https' if self.use_ssl else 'http')
return self._blob_service
def test_close_error():
"""Ensure that attempts to use a closed Deleter results in an error."""
d = wabs_deleter.Deleter(BlobService('test', 'ing'), 'test-container')
d.close()
with pytest.raises(exception.UserCritical):
d.delete('no value should work')
def __init__(self, account_name, account_key, container_name, maybe_create=False):
self._account_name = account_name
self._container_name = container_name
if account_name not in Container.services:
Container.services[account_name] = BlobService(account_name, account_key)
self._service = Container.services[account_name]
if maybe_create:
self._service.create_container(self._container_name, fail_on_exist=False)
def __get_available_storage_account_and_container(self, hackathon_id):
"""
Get available storage account and container
:param hackathon_id: the id of hackathon
:type hackathon_id: integer
:return: if there is available storage account and container, then return (True, storage
account name, container name). Otherwise, return (False, None, None)
:rtype: 3-element tuple: (bool, str|unicode, str|unicode)
"""
container_name = self.util.safe_get_config(
'dockerhostserver.azure.container', 'dockerhostprivatecontainer')
sms = self.__get_sms_object(hackathon_id)
if sms is None:
self.log.error(
'Something wrong with Azure account of Hackathon:%d' %
hackathon_id)
return False, None, None
storage_accounts = sms.list_storage_accounts()
# check storage account one by one, return True once find a qualified one
for storage in storage_accounts.storage_services:
try:
storage_response = sms.get_storage_account_keys(
storage.service_name)
except Exception as e:
self.log.error(
'Encounter an error when checking storage_account:%s ' %
storage.service_name)
self.log.error(e)
continue
blob_service = BlobService(
account_name=storage.service_name,
account_key=storage_response.storage_service_keys.primary,
host_base=self.util.safe_get_config(
'dockerhostserver.storage.host_base',
'.blob.core.chinacloudapi.cn'))
try:
blob_service.get_container_metadata(container_name)
return True, storage.service_name, container_name
except Exception as e:
if e.message != AzureApiExceptionMessage.CONTAINER_NOT_FOUND:
self.log.error(
'Encounter an error when checking container:%s ' %
container_name)
self.log.error(e)
continue
try:
blob_service.create_container(
container_name=container_name,
x_ms_blob_public_access='container')
return True, storage.service_name, container_name
except Exception as e:
self.log.error(
'Encounter an error when creating container:%s ' %
container_name)
self.log.error(e)
return False, None, None
def open_blob_service(self):
"""
open a blob service
"""
try:
from azure.storage.blob import BlobService
except ImportError:
from azure.storage.blob import BlockBlobService as BlobService
return BlobService(self.account_name, self.account_key)
def open_stream(filename):
"""
Open a file and return a stream to the file.
If filename starts with "http:" or "https:" then file is assumed
to be a URL.
If filename starts with "blob:" then file is assumed to be held
within Azure as a BLOB. This expects the following environment
variables to be set:
* BLOB_SAS_TOKEN
* BLOB_ACCOUNT_NAME
* BLOB_CONTAINER_NAME
Otherwise, the filename is assumed to be held on the file
system.
:param filename: file name or URL
:type filename: str or unicode
:return: open stream
:rtype: cStringIO.StringI (URL or file system) OR io.BytesIO (blob)
"""
assert filename, "Filename must not be ''"
is_url = (filename.lower().startswith(HTTP)
or filename.lower().startswith(HTTPS))
is_blob = (filename.lower().startswith(BLOB))
if is_url:
import requests
from io import StringIO
stream = requests.get(filename, stream=True).raw
stream.decode_content = True
stream = StringIO(stream.read())
elif is_blob:
import io
import os
from azure.storage.blob import BlobService
sas_token = os.environ['BLOB_SAS_TOKEN']
if sas_token[0] == '?':
sas_token = sas_token[1:]
blob_service = BlobService(
account_name=os.environ['BLOB_ACCOUNT_NAME'], sas_token=sas_token)
filename = filename[len(BLOB):]
blob = blob_service.get_blob_to_bytes(
os.environ['BLOB_CONTAINER_NAME'], filename)
stream = io.BytesIO(blob)
else:
stream = open(filename, 'rb')
return stream
def test_processes_one_deletion(collect):
key_name = 'test-key-name'
b = B(name=key_name)
d = wabs_deleter.Deleter(BlobService('test', 'ing'), 'test-container')
d.delete(b)
d.close()
assert collect.deleted_keys == [key_name]
def _createExternalStore(self):
from toil.jobStores.azureJobStore import _fetchAzureAccountKey
from azure.storage.blob import BlobService
blobService = BlobService(account_key=_fetchAzureAccountKey(self.accountName),
account_name=self.accountName)
containerName = 'import-export-test-%s' % uuid.uuid4()
blobService.create_container(containerName)
return containerName
def uploadFile(sourceFile, destUrl, destKey):
storageparts = split_storage_url(destUrl)
blobservice = BlobService(storageparts[0], destKey)
try:
fh=open(sourceFile, "r")
except:
print "No such file", sourceFile
return
log('uploading ' + str(sourceFile), True)
blobservice.put_page_blob_from_file(storageparts[2], sourceFile, fh, getsize(sourceFile))
def upload_file_to_azure(in_file, file_name, container_name=settings.AZURE_CONTAINER):
try:
blob_service = BlobService(AZURE_ACCOUNT_NAME, AZURE_ACCOUNT_KEY)
blob_service.put_block_blob_from_path(
container_name=container_name,
blob_name=file_name,
file_path=in_file,
x_ms_blob_content_type='application/octet-stream'
)
except Exception as ex:
print("Failed to upload blob: {0}".format(ex))
def setUp(self):
self.service = BlobService(ACCOUNT_NAME, ACCOUNT_KEY)
# ensure that there's no log file in the container before each test
containers = [c.name for c in self.service.list_containers()]
for handler in LOGGING['handlers']:
container = self._get_container_name(handler)
if container in containers:
filename = _get_handler_config_value(handler, 'filename')
basename = os.path.basename(filename)
for blob in self.service.list_blobs(container,
prefix=basename):
self.service.delete_blob(container, blob.name)
def _prepareTestFile(self, containerName, size=None):
from toil.jobStores.azureJobStore import _fetchAzureAccountKey
from azure.storage.blob import BlobService
fileName = 'testfile_%s' % uuid.uuid4()
url = 'wasb://%s@%s.blob.core.windows.net/%s' % (containerName, self.accountName, fileName)
if size is None:
return url
blobService = BlobService(account_key=_fetchAzureAccountKey(self.accountName),
account_name=self.accountName)
content = os.urandom(size)
blobService.put_block_blob_from_text(containerName, fileName, content)
return url, hashlib.md5(content).hexdigest()
def main():
service = BlobService(settings.STORAGE_ACCOUNT_NAME,
settings.STORAGE_ACCOUNT_KEY)
service.create_container(CONTAINER_NAME)
process(service,
LOCAL_BLOCK_BLOB_FILES,
CONNECTION_COUNTS,
is_page_blob=False)
process(service,
LOCAL_PAGE_BLOB_FILES,
CONNECTION_COUNTS,
is_page_blob=True)
def __connect(self):
"""
Make sure we have an Azure connection, and set one up if we don't.
"""
if self.connection is None:
RealtimeLogger.debug("Connecting to account {}, using "
"container {} and prefix {}".format(
self.account_name, self.container_name,
self.name_prefix))
# Connect to the blob service where we keep everything
self.connection = BlobService(account_name=self.account_name,
account_key=self.account_key)
def upload_results():
"""
:return: None
"""
logger = logging.getLogger(__name__)
results_fpath = '/data/wsdm_cup/results/results.tsv'
logger.info('Uploading results from {0}'.format(results_fpath))
blob_service = BlobService(account_name='wsdmcupchallenge',
sas_token=Config.SAS_TOKEN)
blob_service.put_block_blob_from_path(container_name='bletchleypark',
blob_name='results.tsv',
file_path=results_fpath)
logger.info('Done uploading')
return
def uri_get_file(creds, uri, conn=None):
assert uri.startswith('wabs://')
url_tup = urlparse(uri)
if conn is None:
conn = BlobService(creds.account_name,
creds.account_key,
sas_token=creds.access_token,
protocol='https')
# Determin the size of the target blob
props = conn.get_blob_properties(url_tup.netloc, url_tup.path.lstrip('/'))
blob_size = int(props['content-length'])
ret_size = 0
data = io.BytesIO()
# WABS requires large files to be downloaded in 4MB chunks
while ret_size < blob_size:
ms_range = 'bytes={0}-{1}'.format(ret_size,
ret_size + WABS_CHUNK_SIZE - 1)
while True:
# Because we're downloading in chunks, catch rate limiting and
# connection errors here instead of letting them bubble up to the
# @retry decorator so that we don't have to start downloading the
# whole file over again.
try:
part = conn.get_blob(url_tup.netloc,
url_tup.path.lstrip('/'),
x_ms_range=ms_range)
except EnvironmentError as e:
if e.errno in (errno.EBUSY, errno.ECONNRESET):
logger.warning(
msg="retrying after encountering exception",
detail=("Exception traceback:\n{0}".format(
traceback.format_exception(*sys.exc_info()))),
hint="")
gevent.sleep(30)
else:
raise
else:
break
length = len(part)
ret_size += length
data.write(part)
if length > 0 and length < WABS_CHUNK_SIZE:
break
elif length == 0:
break
return data.getvalue()
def prepare_storage(settings):
default_storage_account_name = settings["DEFAULT_STORAGE_ACCOUNT_NAME"]
storage_access_key = settings["STORAGE_ACCESS_KEY"]
blob_service = BlobService(default_storage_account_name,
storage_access_key)
blob_service.create_container('bosh')
blob_service.create_container(container_name='stemcell',
x_ms_blob_public_access='blob')
# Prepare the table for storing meta datas of storage account and stemcells
table_service = TableService(default_storage_account_name,
storage_access_key)
table_service.create_table('stemcells')
def getblob(request):
assert isinstance(request, HttpRequest)
blob_service = BlobService(
account_name='araldrift',
account_key=
'otLzzkwQHQD3xFTQxwxy64PCL6eDINWGjSB7x6Ta2XVw3+3ffI5O2MhAEavf/r8qIW4G/dKrZAVg1R64nK7hDQ=='
)
# http://<storage-account-name>.blob.core.windows.net/<container-name>/<blob-name>
name = 'test.txt'
fpath = '{0}\{1}'.format(tempfile.gettempdir(), name)
blob_service.get_blob_to_path('flow', 'NARYN.day', fpath)
response = HttpResponse(content_type='text/plain')
response['Content-Disposition'] = 'attachment; filename=test.txt'
blob.Properties.ContentDisposition = "attachment; filename=" + downloadName
return response
def save_image_to_azure(profile, url):
try:
response = request('GET', url)
response.raise_for_status()
except ConnectionError:
pass
else:
service = BlobService(account_name=storagesettings.AZURE_ACCOUNT_NAME,
account_key=storagesettings.AZURE_ACCOUNT_KEY)
service.put_block_blob_from_bytes(
'avatars',
profile.id,
response.content,
x_ms_blob_content_type=response.headers['content-type'])
