def get_blob_client_by_uri(storage_uri):
container_name, storage_name, key = StorageUtilities.get_storage_from_uri(storage_uri)
blob_service = BlockBlobService(account_name=storage_name, account_key=key)
blob_service.create_container(container_name)
return blob_service, container_name
def index(request):
context = RequestContext(request)
if request.method == 'POST':
formobj = fileform(request.POST,request.FILES)
if formobj.is_valid():
modelobj = formobj.save(commit=False)
modelobj.image=request.FILES['image']
modelobj.save()
mp3= request.FILES['image']
block_blob_service = BlockBlobService(account_name=myaccount, account_key=mykey)
block_blob_service.create_container('harshultest2', public_access=PublicAccess.Container)
block_blob_service.set_container_acl('harshultest2', public_access=PublicAccess.Container)
with open('t.png','wb+') as formfile:
for chunk in mp3.chunks():
formfile.write(chunk)
block_blob_service.create_blob_from_path(
'harshultest2',
str(request.FILES['image']),
't.png',
content_settings=ContentSettings(content_type='image')
)
return HttpResponseRedirect(reverse('list'))
else:
formobj = fileform()
return render_to_response('app/myform.html', {'formobj': formobj},context)
def get_blob_client_by_uri(storage_uri, session):
storage = StorageUtilities.get_storage_from_uri(storage_uri, session)
blob_service = BlockBlobService(
account_name=storage.storage_name,
token_credential=storage.token)
blob_service.create_container(storage.container_name)
return blob_service, storage.container_name, storage.file_prefix
def block_blob_service(self):
from azure.storage.blob import BlockBlobService, PublicAccess
block_blob_service = BlockBlobService(
connection_string=self.conn_string)
if self.create_if_missing:
block_blob_service.create_container(
self.container,
public_access=PublicAccess.Container if self.public else None
)
return block_blob_service
class _BlobStorageFileHandler(object):
def __init__(self,
account_name=None,
account_key=None,
protocol='https',
container='logs',
zip_compression=False,
max_connections=1,
max_retries=5,
retry_wait=1.0,
is_emulated=False):
self.service = BlockBlobService(account_name=account_name,
account_key=account_key,
is_emulated=is_emulated,
protocol=protocol)
self.container_created = False
hostname = gethostname()
self.meta = {'hostname': hostname.replace('_', '-'),
'process': os.getpid()}
self.container = (container % self.meta).lower()
self.meta['hostname'] = hostname
self.zip_compression = zip_compression
self.max_connections = max_connections
self.max_retries = max_retries
self.retry_wait = retry_wait
def put_file_into_storage(self, dirName, fileName):
"""
Ship the outdated log file to the specified blob container.
"""
if not self.container_created:
self.service.create_container(self.container)
self.container_created = True
fd, tmpfile_path = None, ''
try:
file_path = os.path.join(dirName, fileName)
if self.zip_compression:
suffix, content_type = '.zip', 'application/zip'
fd, tmpfile_path = mkstemp(suffix=suffix)
with os.fdopen(fd, 'wb') as f:
with ZipFile(f, 'w', ZIP_DEFLATED) as z:
z.write(file_path, arcname=fileName)
file_path = tmpfile_path
else:
suffix, content_type = '', 'text/plain'
self.service.create_blob_from_path(container_name=self.container,
blob_name=fileName+suffix,
file_path=file_path,
content_settings=ContentSettings(content_type=content_type),
max_connections=self.max_connections
) # max_retries and retry_wait no longer arguments in azure 0.33
finally:
if self.zip_compression and fd:
os.remove(tmpfile_path)
def prepare(self, area):
assert area is not None, 'area is none; should already be validated'
area_config = config.load_area(area)
storage_config = config.load_storage(area_config['storage'])
blob_service = BlockBlobService(account_name=storage_config['name'], account_key=storage_config['key1'])
blob_service.create_container(area_config['container'])
blob_service.set_container_acl(area_config['container'], public_access=PublicAccess.Container)
def initialize_backend():
global _blob_service
global _container
global _timeout
_blob_service = BlockBlobService(
account_name=getenv_required(_ENV_ACCOUNT_NAME),
account_key=getenv_required(_ENV_ACCOUNT_KEY))
_container = getenv(_ENV_CONTAINER, _DEFAULT_CONTAINER)
_timeout = getenv_int(_ENV_TIMEOUT, _DEFAULT_TIMEOUT)
_blob_service.create_container(
_container, fail_on_exist=False, timeout=_timeout)
def run_sample():
try:
# Create the BlockBlockService that is used to call the Blob service for the storage account
block_blob_service = BlockBlobService(account_name='meetpythonstorage', account_key='duOguiKnYb6ZEbJC6BftWqA2lcH67dWkmCSEJj+KxOTOHCNPeV7r4oO6feTw7gSSoFGKHryL4yqSVWlEkm6jWg==')
# Create a container called 'quickstartblobs'.
container_name ='quickstartblobs'
block_blob_service.create_container(container_name)
# Set the permission so the blobs are public.
block_blob_service.set_container_acl(container_name, public_access=PublicAccess.Container)
# Create a file in Documents to test the upload and download.
local_path=os.path.abspath(os.path.curdir)
local_file_name =input("Enter file name to upload : ")
full_path_to_file =os.path.join(local_path, local_file_name)
# Write text to the file.
#file = open(full_path_to_file, 'w')
#file.write("Hello, World!")
#file.close()
print("Temp file = " + full_path_to_file)
print("\nUploading to Blob storage as blob" + local_file_name)
# Upload the created file, use local_file_name for the blob name
block_blob_service.create_blob_from_path(container_name, local_file_name, full_path_to_file)
# List the blobs in the container
print("\nList blobs in the container")
generator = block_blob_service.list_blobs(container_name)
for blob in generator:
print("\t Blob name: " + blob.name)
# Download the blob(s).
# Add '_DOWNLOADED' as prefix to '.txt' so you can see both files in Documents.
full_path_to_file2 = os.path.join(local_path, str.replace(local_file_name ,'.txt', '_DOWNLOADED.txt'))
print("\nDownloading blob to " + full_path_to_file2)
block_blob_service.get_blob_to_path(container_name, local_file_name, full_path_to_file2)
sys.stdout.write("Sample finished running. When you hit <any key>, the sample will be deleted and the sample "
"application will exit.")
sys.stdout.flush()
input()
# Clean up resources. This includes the container and the temp files
block_blob_service.delete_container(container_name)
os.remove(full_path_to_file)
os.remove(full_path_to_file2)
except Exception as e:
print(e)
def upload_assets(self, blob_client: azureblob.BlockBlobService):
"""
Uploads a the file specified in the json parameters file into a storage container that will
delete it's self after 7 days
:param blob_client: A blob service client used for making blob operations.
:type blob_client: `azure.storage.blob.BlockBlobService`
"""
input_container_name = "fgrp-" + self.job_id
output_container_name = "fgrp-" + self.job_id + '-output'
# Create input container
blob_client.create_container(input_container_name, fail_on_exist=False)
logger.info('creating a storage container: {}'.format(input_container_name))
# Create output container
blob_client.create_container(output_container_name, fail_on_exist=False)
logger.info('creating a storage container: {}'.format(output_container_name))
full_sas_url_input = 'https://{}.blob.core.windows.net/{}?{}'.format(
blob_client.account_name,
input_container_name,
utils.get_container_sas_token(
blob_client,
input_container_name,
ContainerPermissions.READ +
ContainerPermissions.LIST))
full_sas_url_output = 'https://{}.blob.core.windows.net/{}?{}'.format(
blob_client.account_name,
output_container_name,
utils.get_container_sas_token(
blob_client,
output_container_name,
ContainerPermissions.READ +
ContainerPermissions.LIST +
ContainerPermissions.WRITE))
# Set the storage info for the container.
self.storage_info = utils.StorageInfo(
input_container_name,
output_container_name,
full_sas_url_input,
full_sas_url_output)
# Upload the asset file that will be rendered and
scenefile = ctm.get_scene_file(self.parameters_file)
for file in os.listdir("Assets"):
if scenefile == file:
file_path = Path("Assets/" + file)
utils.upload_file_to_container(blob_client, input_container_name, file_path)
def block_blob_service(self):
from azure.storage.blob import BlockBlobService, PublicAccess
block_blob_service = BlockBlobService(
connection_string=self.conn_string,
socket_timeout=self.socket_timeout,
)
if self.max_block_size is not None:
block_blob_service.MAX_BLOCK_SIZE = self.max_block_size
if self.max_block_size is not None:
block_blob_service.MAX_SINGLE_PUT_SIZE = self.max_single_put_size
if self.create_if_missing:
block_blob_service.create_container(
self.container,
public_access=PublicAccess.Container if self.public else None
)
return block_blob_service
def blob_service(self):
from azure.storage.blob import BlockBlobService
from azure.common import AzureMissingResourceHttpError
logger.debug("URL {}".format(self.path_info))
logger.debug("Connection string {}".format(self.connection_string))
blob_service = BlockBlobService(
connection_string=self.connection_string
)
logger.debug("Container name {}".format(self.path_info.bucket))
try: # verify that container exists
blob_service.list_blobs(
self.path_info.bucket, delimiter="/", num_results=1
)
except AzureMissingResourceHttpError:
blob_service.create_container(self.path_info.bucket)
return blob_service
def uploadVideo(self):
outputConnectionString = "DefaultEndpointsProtocol=http;BlobEndpoint=http://" + self.privateIPVM + ":11002/" + self.localStorageName + ";AccountName=" + self.localStorageName + ";AccountKey=" + self.localStorageKey + ";"
#print("testrb" + self.privateIPVM + "," + self.localStorageContainer + "," + self.localStorageName + "," + self.localStorageKey + "," + outputConnectionString)
now = datetime.strftime(datetime.now(), "%H%M%S")
inputFileName = self.videoPath[2:-4] + "_" + now + ".MP4"
outputBlob = BlockBlobService(connection_string=outputConnectionString)
print("create container")
outputBlob.create_container(self.localStorageName, fail_on_exist=False)
print("end creating container")
print("start writing to local blob")
outputBlob.create_blob_from_path(self.localStorageName, inputFileName,
self.videoPath)
print("end writing to local blob")
def upload_assets(self, blob_client: azureblob.BlockBlobService):
"""
Uploads a the file specified in the json parameters file into a storage container that will
delete it's self after 7 days
:param blob_client: A blob service client used for making blob operations.
:type blob_client: `azure.storage.blob.BlockBlobService`
"""
input_container_name = "fgrp-" + self.job_id
output_container_name = "fgrp-" + self.job_id + '-output'
# Create input container
blob_client.create_container(input_container_name, fail_on_exist=False)
logger.info(
'creating a storage container: {}'.format(input_container_name))
# Create output container
blob_client.create_container(output_container_name,
fail_on_exist=False)
logger.info(
'creating a storage container: {}'.format(output_container_name))
full_sas_url_input = 'https://{}.blob.core.windows.net/{}?{}'.format(
blob_client.account_name, input_container_name,
utils.get_container_sas_token(
blob_client, input_container_name,
ContainerPermissions.READ + ContainerPermissions.LIST))
full_sas_url_output = 'https://{}.blob.core.windows.net/{}?{}'.format(
blob_client.account_name, output_container_name,
utils.get_container_sas_token(
blob_client, output_container_name, ContainerPermissions.READ +
ContainerPermissions.LIST + ContainerPermissions.WRITE))
# Set the storage info for the container.
self.storage_info = utils.StorageInfo(input_container_name,
output_container_name,
full_sas_url_input,
full_sas_url_output)
# Upload the asset file that will be rendered and
scenefile = ctm.get_scene_file(self.parameters_file)
for file in os.listdir("Assets"):
if scenefile == file:
file_path = Path("Assets/" + file)
utils.upload_file_to_container(blob_client,
input_container_name, file_path)
def run_sample():
try:
block_blob_service = BlockBlobService(account_name='kazen1', account_key='ds7aI/Mo3YuYaWe9a/tgKrPFf0mnXwOEv+I1EFgmcH3dMSGkc9Pbc1Zzt8hPfa+70TlPNJQU/xPg+nbeoCX5Cg==')
container_name ='kazn1'
block_blob_service.create_container(container_name)
block_blob_service.set_container_acl(container_name, public_access=PublicAccess.Container)
local_path=os.path.abspath(os.path.curdir)
local_file_name =input("Enter file name to upload : ")
full_path_to_file =os.path.join(local_path, local_file_name)
print("Temp file = " + full_path_to_file)
print("\nUploading to Blob storage as blob" + local_file_name)
block_blob_service.create_blob_from_path(container_name, local_file_name, full_path_to_file)
print("\nList blobs in the container")
generator = block_blob_service.list_blobs(container_name)
for blob in generator:
print("\t Blob name: " + blob.name)
full_path_to_file2 = os.path.join(local_path, str.replace(local_file_name ,'.txt', '_DOWNLOADED.txt'))
print("\nDownloading blob to " + full_path_to_file2)
block_blob_service.get_blob_to_path(container_name, local_file_name, full_path_to_file2)
sys.stdout.write("Sample finished running. When you hit <any key>, the sample will be deleted and the sample "
"application will exit.")
sys.stdout.flush()
input()
block_blob_service.delete_container(container_name)
os.remove(full_path_to_file)
os.remove(full_path_to_file2)
except Exception as e:
print(e)
def run_sample():
try:
# Create the BlockBlockService that is used to call the Blob service for the storage account
block_blob_service = BlockBlobService(account_name=cred.account_name, account_key=cred.account_key)
# Create a container called 'quickstartblobs'.
container_name ='quickstartblobs'
block_blob_service.create_container(container_name)
# Set the permission so the blobs are public.
block_blob_service.set_container_acl(container_name, public_access=PublicAccess.Container)
# Create a file in Documents to test the upload and download.
local_path=os.path.expanduser("~/Documents")
local_file_name ="QuickStart_" + str(uuid.uuid4()) + ".txt"
full_path_to_file =os.path.join(local_path, local_file_name)
# Write text to the file.
file = open(full_path_to_file, 'w')
file.write("Hello, World!")
file.close()
print("Temp file = " + full_path_to_file)
print("\nUploading to Blob storage as blob" + local_file_name)
# Upload the created file, use local_file_name for the blob name
block_blob_service.create_blob_from_path(container_name, local_file_name, full_path_to_file)
# List the blobs in the container
print("\nList blobs in the container")
generator = block_blob_service.list_blobs(container_name)
for blob in generator:
print("\t Blob name: " + blob.name)
# Download the blob(s).
# Add '_DOWNLOADED' as prefix to '.txt' so you can see both files in Documents.
full_path_to_file2 = os.path.join(local_path, str.replace(local_file_name ,'.txt', '_DOWNLOADED.txt'))
print("\nDownloading blob to " + full_path_to_file2)
block_blob_service.get_blob_to_path(container_name, local_file_name, full_path_to_file2)
# Clean up resources. This includes the container and the temp files
#block_blob_service.delete_container(container_name)
#os.remove(full_path_to_file)
#os.remove(full_path_to_file2)
except Exception as e:
print(e)
def getAzureBlobService(account_name, account_key, container_name):
try:
print('\n initializing azure...')
# Create the BlockBlockService that is used to call the Blob service for the storage account
block_blob_service = BlockBlobService(account_name=account_name,
account_key=account_key)
# Create a container called 'quickstartblobs'.
block_blob_service.create_container(container_name)
# Set the permission so the blobs are public.
block_blob_service.set_container_acl(
container_name, public_access=PublicAccess.Container)
return block_blob_service
except Exception as e:
print(e)
def publish_asset():
# Get parameters for clustering
parameters = request.get_json()
'''
# Uncomment this for OrbitDB hosting (PoC, not Ocean testnet compatible yet)
execute_js('host.js')
with open('host.json', 'r') as infile:
host = json.load(infile)
'''
# Azure storage hosting
azure_account = parameters['azureaccount']
# Unique container name - requires non-collision * under a single Azure account *
# 36^4=1679616 possibilities, Pr[collision] = 1 - ( (36^4-1)/36^4 )^num_datasets_created
container_name = parameters['containername']
# Generate machine-readable download link to hosted dataset
azure_url = 'https://' + azure_account + '.blob.core.windows.net/' + container_name + '/output.json'
try:
# Create service used to call the Blob service for the storage account
block_blob_service = BlockBlobService(
account_name=azure_account, account_key=parameters['azurekey'])
# Create container with name = asset_id
block_blob_service.create_container(container_name)
# Make public
block_blob_service.set_container_acl(
container_name, public_access=PublicAccess.Container)
# Create and upload blob
block_blob_service.create_blob_from_path(container_name, 'output.json',
'output.json')
except Exception as e:
print(e)
# Outlier Ventures' abstraction for easy registration with Keeper and Aquarius
reg.simple_register(parameters['name'], parameters['price'],
parameters['description'], parameters['author'],
azure_url)
return ('', 200)
def main(pretrained_model_type, mmlspark_model_type, config_filename,
output_model_name, sample_frac):
# Load the configuration file
config = ConfigFile(config_filename, pretrained_model_type,
mmlspark_model_type, output_model_name)
write_model_summary_to_blob(config, mmlspark_model_type)
# Log the parameters of the run
run_logger = get_azureml_logger()
run_logger.log('amlrealworld.aerial_image_classification.run_mmlspark','true')
run_logger.log('pretrained_model_type', pretrained_model_type)
run_logger.log('mmlspark_model_type', mmlspark_model_type)
run_logger.log('config_filename', config_filename)
run_logger.log('output_model_name', output_model_name)
run_logger.log('sample_frac', sample_frac)
# Train and save the MMLSpark model
train_df = load_data(config.train_uri, config, sample_frac)
mmlspark_model = mmlspark.TrainClassifier(
model=config.mmlspark_model_type, labelCol='label').fit(train_df)
mmlspark_model.write().overwrite().save(config.output_uri)
# Apply the MMLSpark model to the test set and save the accuracy metric
test_df = load_data(config.test_uri, config, sample_frac)
predictions = mmlspark_model.transform(test_df)
metrics = mmlspark.ComputeModelStatistics(evaluationMetric='accuracy') \
.transform(predictions)
metrics.show()
run_logger.log('accuracy_on_test_set', metrics.first()['accuracy'])
# Save the predictions
tf = mmlspark.IndexToValue().setInputCol('scored_labels') \
.setOutputCol('pred_label')
predictions = tf.transform(predictions).select(
'filepath', 'label', 'pred_label')
output_str = predictions.toPandas().to_csv(index=False)
blob_service = BlockBlobService(config.storage_account_name,
config.storage_account_key)
blob_service.create_container(config.container_prediction_results)
blob_service.create_blob_from_text(
config.container_prediction_results,
config.predictions_filename,
output_str)
return
def vir_typer_upload(request, vir_typer_pk):
vir_typer_project = get_object_or_404(VirTyperProject, pk=vir_typer_pk)
vir_typer_samples = list(
VirTyperRequest.objects.filter(project_name__pk=vir_typer_pk))
sample_names = list()
for sample in vir_typer_samples:
sample_names.append(str(sample.LSTS_ID))
if request.method == 'POST':
seq_files = [
request.FILES.get('file[%d]' % i)
for i in range(0, len(request.FILES))
]
if seq_files:
container_name = VirTyperProject.objects.get(
pk=vir_typer_pk).container_namer()
blob_client = BlockBlobService(
account_name=settings.AZURE_ACCOUNT_NAME,
account_key=settings.AZURE_ACCOUNT_KEY)
blob_client.create_container(container_name)
for item in seq_files:
blob_client.create_blob_from_bytes(
container_name=container_name,
blob_name=item.name,
blob=item.read())
for sample in vir_typer_samples:
# sample_name = '{lsts}_{sn}'.format(lsts=str(sample.LSTS_ID),
# sn=str(sample.sample_name))
for seq_file in seq_files:
if str(sample.LSTS_ID) in str(seq_file):
vir_files = VirTyperFiles(sample_name_id=sample.pk,
sequence_file=seq_file)
vir_files.save()
vir_typer_project.status = 'Processing'
vir_typer_project.save()
run_vir_typer.apply_async(queue='cowbat',
args=(vir_typer_pk, ),
countdown=10)
return redirect('vir_typer:vir_typer_home')
return render(
request, 'vir_typer/vir_typer_upload_sequences.html', {
'vir_typer_project': vir_typer_project,
'vir_typer_samples': vir_typer_samples,
'vir_typer_sample_names': sample_names
})
def storage(host):
"""
Create blob using azurite.
"""
bbs = BlockBlobService(
account_name=USERNAME,
account_key=KEY,
custom_domain=f"http://{host}/devstoreaccount1",
)
bbs.create_container("data", timeout=1)
bbs.create_blob_from_bytes("data", "top_file.txt", data)
bbs.create_blob_from_bytes("data", "root/rfile.txt", data)
bbs.create_blob_from_bytes("data", "root/a/file.txt", data)
bbs.create_blob_from_bytes("data", "root/b/file.txt", data)
bbs.create_blob_from_bytes("data", "root/c/file1.txt", data)
bbs.create_blob_from_bytes("data", "root/c/file2.txt", data)
yield bbs
def __createstorage():
global container_name
global queue_service
global block_blob_service
global queue_service
block_blob_service = BlockBlobService(
account_name=STORAGE_ACCOUNT_NAME,
account_key=STORAGE_ACCOUNT_KEY,
endpoint_suffix=STORAGE_ACCOUNT_SUFFIX)
timestr = time.strftime("%Y%m%d-%H%M%S")
container_name = 'fromcamera' + timestr
block_blob_service.create_container(container_name)
block_blob_service.set_container_acl(container_name,
public_access=PublicAccess.Container)
queue_service = QueueService(account_name=STORAGE_ACCOUNT_NAME,
account_key=STORAGE_ACCOUNT_KEY,
endpoint_suffix=STORAGE_ACCOUNT_SUFFIX)
queue_service.create_queue('fromcamera' + timestr)
def connect(config=False):
import lib.misc as misc
from azure.storage.blob import BlockBlobService as BlobService
global blob_service, container
# Connect to the cloud service.
if not config: config = misc.config['_private']
container = 'streams'
if not 'azure' in config:
logging.debug("no azure config")
return None, None
if not blob_service:
blob_service = BlobService(config['azure']['storage_account_name'], config['azure']['primary_access_key'])
blob_service.create_container(container)
return blob_service, container
def blob():
static_dir_path = "D:\home\site\wwwroot\static"
account_name = 'hanastragetest'
account_key = 'acount_key'
container_name = 'images'
container_url = "https://hanastragetest.blob.core.windows.net/" + container_name
block_blob_service = BlockBlobService(account_name=account_name,
account_key=account_key)
app.logger.info("test message : {}".format(block_blob_service))
# container create
block_blob_service.create_container(container_name)
block_blob_service.set_container_acl(container_name,
public_access=PublicAccess.Container)
#app.logger.info("finish : block_blob_service.set_container_acl")
files = os.listdir(static_dir_path)
for file in files:
# delete
if block_blob_service.exists(container_name, file):
block_blob_service.delete_blob(container_name, file)
# blob write
block_blob_service.create_blob_from_path(
container_name,
file,
static_dir_path + '\\' + file,
content_settings=ContentSettings(content_type='image/png'))
# get container
generator = block_blob_service.list_blobs(container_name)
html = ""
for blob in generator:
#app.logger.info("generator : {}".format(blob.name))
html = "{}<img src='{}/{}'>".format(html, container_url, blob.name)
#app.logger.info("generator_object : {}".format(generator))
result = {
"result": True,
"data": {
"blob_name": [blob.name for blob in generator]
}
}
return make_response(json.dumps(result, ensure_ascii=False) + html)
def run_sample():
try:
# Create the BlockBlockService that is used to call the Blob service for the storage account
block_blob_service = BlockBlobService(account_name='<account_name>', account_key='<account_key>')
# Create a container called 'quickstartblobs'.
container_name ='quickstartblobs'
block_blob_service.create_container(container_name)
# Set the permission so the blobs are public.
block_blob_service.set_container_acl(container_name, public_access=PublicAccess.Container)
# Create upload the fruit photo
local_path=os.getcwd()+'\\data'
local_file_name ='fruit.jpg'
full_path_to_file =os.path.join(local_path, local_file_name)
print("\nUploading to Blob storage as blob" + local_file_name)
# Upload the created file, use local_file_name for the blob name
block_blob_service.create_blob_from_path(container_name, local_file_name, full_path_to_file)
# List the blobs in the container
print("\nList blobs in the container")
generator = block_blob_service.list_blobs(container_name)
for blob in generator:
print("\t Blob name: " + blob.name)
# Download the blob(s).
# Add '_DOWNLOADED' as prefix to '.jpg' so you can see both files in Documents.
full_path_to_file2 = os.path.join(local_path, str.replace(local_file_name ,'.jpg', '_DOWNLOADED.jpg'))
print("\nDownloading blob to " + full_path_to_file2)
block_blob_service.get_blob_to_path(container_name, local_file_name, full_path_to_file2)
sys.stdout.write("Sample finished running. When you hit <any key>, the sample will be deleted and the sample "
"application will exit.")
sys.stdout.flush()
input()
# Clean up resources. This includes the container and the temp files
block_blob_service.delete_container(container_name)
os.remove(full_path_to_file2)
except Exception as e:
print(e)
def generateAzureInputStore(bs_config, stor_name, stor_key, source_dir):
'''
Loads a folder of images with the appropriate filenames into the Azure Blob Storage dir so they are accessible to Input
workers running in the cloud
'''
bs_dir_name = bs_config['data']['blob-storage-con']
storage_uri = ""
block_blob_service = BlockBlobService(account_name=stor_name,
account_key=stor_key)
#, endpoint_suffix="core.usgovcloudapi.net") #addendum for use on gov't
block_blob_service.create_container(bs_dir_name)
logging.debug("Connected to blob service {0}".format(stor_name))
image_count = 0
for dir_path, dir_names, file_names in os.walk(source_dir, topdown=True):
for file_name in file_names:
dir_components = Path(dir_path)
usage = dir_components.parts[len(dir_components.parts) - 1]
entity = dir_components.parts[len(dir_components.parts) - 2]
image_file = open(os.path.join(dir_path, file_name), 'rb').read()
#Calculates image hash, infers the purpose of an image from its folder position and generate a filename
hash = hashlib.md5(image_file).hexdigest()
blob_name = usage + "-" + entity + "-" + str(hash)
#Uploads to the Azure Blob Store
block_blob_service.create_blob_from_path(
bs_dir_name, blob_name, dir_path + "/" + file_name)
logging.info(
"File written to blob container {0} from {1} {2}".format(
bs_dir_name, os.path.join(dir_path, file_name), blob_name))
#Renames the image in place
os.rename(os.path.join(dir_path, file_name),
os.path.join(dir_path, blob_name))
logging.info("Renamed {0} to {1}".format(
os.path.join(dir_path, file_name),
os.path.join(dir_path, blob_name)))
image_count += 1
logging.info("Wrote {0} images total to blob storage".format(image_count))
def StoreExtractedTable(self,images):
block_blob_service = BlockBlobService(account_name='tableextractor',account_key='OA15EYbHBnD3X+p17r30L59gZOVV91Lht5Y0tLidf/xdVexI0UVKoy8Z+/mYX+cFepSVMSElZeIrLYznZ22y2A== ')
container_name = 'extracted-table-images'
block_blob_service.create_container(container_name)
# local_path = '/home/vikas/NeuroNer/NeuroNER-master/src/BlobStorage/'
local_path = "C:\\Users\\M1049308\\Desktop\\tableExtract\\ExtracedTables"
for files in os.listdir(local_path):
block_blob_service.create_blob_from_path(container_name,files,os.path.join(local_path, files))
os.remove(os.path.join(local_path, files))
print('\nList blobs in the container')
generator = block_blob_service.list_blobs(container_name)
print(generator)
ImageNames = []
for blob in generator:
print('\t Blob name: ' + blob.name)
ImageNames.append(blob.name)
return jsonify(BlobNames=ImageNames)
def WriteBlob(self, blob_name, value):
""" 単一 BLOB ファイルを作成しテキストを保存する。 """
szRet = ""
if (LogWriter.DEBUG_MODE):
return ("Debug モードのため書き込みをしません。")
try:
#blob_name = r'sample.txt';
szRet = "BlockBlobService"
blob_service = BlockBlobService(self._name, self._key)
szRet = "create_container"
blob_service.create_container(LogWriter.LOG_CONTAINER_NAME,
public_access=PublicAccess.Blob)
szRet = "create_blob_from_bytes"
#blob_service.create_blob_from_bytes(
# log_container_name,
# log_blob_name,
# b'<center><h1>Hello World!</h1></center>',
# content_settings=ContentSettings('text/html')
#)
if (isinstance(value, str)):
szRet = "create_blob_from_text"
blob_service.create_blob_from_text(
LogWriter.LOG_CONTAINER_NAME, blob_name, value)
else:
szRet = "create_blob_from_stream"
blob_service.create_blob_from_stream(
LogWriter.LOG_CONTAINER_NAME, blob_name, io.BytesIO(value))
#}if
#szRet = "make_blob_url"
#print(blob_service.make_blob_url(log_container_name, log_blob_name))
szRet = "OK"
except:
print(r"Exception.")
#try
return szRet
class AzureStorage(Storage):
def __init__(self, option=None):
self.block_blob_service = \
BlockBlobService(account_name=settings.AZURE_STORAGE_ACCOUNT_NAME,
account_key=settings.AZURE_STORAGE_ACCOUNT_KEY)
self.block_blob_service.create_container(
settings.AZURE_STORAGE_DEFAULT_CONTAINER)
def _save(self, name, content):
content.open()
content_stream = content.read()
self.block_blob_service.create_blob_from_bytes(
'media',
name,
content_stream,
content_settings=(ContentSettings(
content_type=content.file.content_type)))
return name
def _open(self, name, mode='rb'):
extension_index = name.rfind('.')
extension = ''
if extension_index != -1:
extension = name[extension_index:]
tmp_file = tempfile.NamedTemporaryFile(suffix=extension, delete=False)
self.block_blob_service.get_blob_to_stream(
container_name=settings.AZURE_STORAGE_DEFAULT_CONTAINER,
blob_name=name,
stream=tmp_file,
max_connections=2)
tmp_file.seek(0)
return File(tmp_file)
def exists(self, name):
generator = self.block_blob_service.list_blobs('media')
for blob in generator:
if name == blob.name:
return True
return False
def url(self, name):
return self.name
def test_request_callback_signed_header(self):
# Arrange
service = BlockBlobService(self.account_name, self.account_key)
name = self.get_resource_name('cont')
# Act
def callback(request):
if request.method == 'PUT':
request.headers['x-ms-meta-hello'] = 'world'
service.request_callback = callback
# Assert
try:
service.create_container(name)
metadata = service.get_container_metadata(name)
self.assertEqual(metadata, {'hello': 'world'})
finally:
service.delete_container(name)
def temp(params):
block_blob_service = BlockBlobService(
account_name='rvsafeimages',
account_key=
'391TMmlvDdRWu+AsNX+ZMl1i233YQfP5dxo/xhMrPm22KtwWwwMmM9vFAJpJHrGXyBrTW4OoAInjHnby9Couug=='
)
container_name = 'imagescontainer'
#params = params+".jpg"
block_blob_service.create_container(container_name)
block_blob_service.set_container_acl(container_name,
public_access=PublicAccess.Container)
# full_path_to_file2 = os.getcwd()+"/test.jpg"
# block_blob_service.get_blob_to_path(container_name, params, full_path_to_file2)
#import pdb; pdb.set_trace()
data = block_blob_service.get_blob_to_bytes(container_name, params)
# data = open(full_path_to_file2,"rb").read()
#import pdb; pdb.set_trace()
res = Response(data.content, status=200, mimetype="image/jpeg")
return res
def connect(config=False):
import lib.misc as misc
from azure.storage.blob import BlockBlobService as BlobService
global blob_service, container
# Connect to the cloud service.
if not config: config = misc.config['_private']
container = 'streams'
if not 'azure' in config:
logging.debug("no azure config")
return None, None
if not blob_service:
blob_service = BlobService(config['azure']['storage_account_name'],
config['azure']['primary_access_key'])
blob_service.create_container(container)
return blob_service, container
def upload_sequence_data(request, sequencing_run_pk):
sequencing_run = get_object_or_404(SequencingRun, pk=sequencing_run_pk)
check_uploaded_seqids(sequencing_run=sequencing_run)
if request.method == 'POST':
container_name = sequencing_run.run_name.lower().replace('_', '-')
blob_client = BlockBlobService(
account_name=settings.AZURE_ACCOUNT_NAME,
account_key=settings.AZURE_ACCOUNT_KEY)
blob_client.create_container(container_name)
for i in range(0, len(request.FILES)):
item = request.FILES.get('file[%d]' % i)
blob_client.create_blob_from_bytes(container_name=container_name,
blob_name=item.name,
blob=item.read())
# return redirect('cowbat:cowbat_processing', sequencing_run_pk=sequencing_run.pk)
return render(request, 'cowbat/upload_sequence_data.html', {
'sequencing_run': sequencing_run,
})
def main():
args = parsing_options()
az_conf = read_az_conf(args.azureConf)
block_blob_service = BlockBlobService(
account_name=az_conf['storage_account_name'],
account_key=az_conf['storage_account_key'])
block_blob_service.create_container('mycontainer')
block_blob_service.create_blob_from_path(
'mycontainer',
'test.zip',
'/tmp/test.zip',
content_settings=ContentSettings(
content_type='application/zip',
content_md5='M2E5OTI1N2ZmMTRiNDExNzk1ZmFiMDEyZjQ3OGQ3ODIKi'))
generator = block_blob_service.list_blobs('mycontainer')
for blob in generator:
print blob.name
print blob.properties
print blob.metadata
def test_request_callback_signed_header(self):
# Arrange
service = BlockBlobService(self.account_name, self.account_key)
name = self.get_resource_name('cont')
# Act
def callback(request):
if request.method == 'PUT':
request.headers['x-ms-meta-hello'] = 'world'
service.request_callback = callback
# Assert
try:
service.create_container(name)
metadata = service.get_container_metadata(name)
self.assertEqual(metadata, {'hello': 'world'})
finally:
service.delete_container(name)
def copy_to_azure(file_uuid):
try:
CDN_BASE = "https://ucarecdn.com/"
file_url = CDN_BASE + file_uuid + '/'
block_blob_service = BlockBlobService(
account_name='uploadcaretest',
#take out account key to filesystem or database
account_key='')
block_blob_service.create_container('image_container')
block_blob_service.set_container_acl(
'image_container', public_access=PublicAccess.Container)
filename = wget.download(file_url)
block_blob_service.create_blob_from_path(
'image_container',
uuid,
'filename',
content_settings=ContentSettings(content_type='image/png'))
except Exception, e:
print(e.message)
def uploadImg(myblob):
from azure.storage.blob import BlockBlobService
block_blob_service = BlockBlobService(
account_name='eventdetect',
account_key=
'VdqUAaFmd5K8bF5Pp+wt6cDfYUWiAtR2ib7+rKP76sqgJwSo0+friYmuVd+Y5oEWDh6/4oaRa423fXproar3aw=='
)
block_blob_service.create_container('mycontainer')
from azure.storage.blob import PublicAccess
block_blob_service.create_container('mycontainer',
public_access=PublicAccess.Container)
block_blob_service.set_container_acl('mycontainer',
public_access=PublicAccess.Container)
from azure.storage.blob import ContentSettings
block_blob_service.create_blob_from_path(
'mycontainer',
myblob,
'Images\\' + myblob,
content_settings=ContentSettings(content_type='image/jpg'))
def Blob_upload(path = PATH) :
account_name = 'mailsys'
account_key = 'Ql9zTFbmOolovYH7wnVCKqzSV9aQg3owEKviAXCANK1RiRmSp++kWy5HbReqsSPMZv4M8EUtKUGGuwvCubHn6w=='
container_name = 'surgerydata'
# Create the BlockBlockService that is used to call the Blob service for the storage account.
service = BlockBlobService(account_name = account_name, account_key = account_key)
# Create a container called 'surgerydata'.
if not service.exists(container_name) :
print("create container : ", container_name)
service.create_container(container_name)
else :
print("container name : " + container_name)
print("accounnt : " + account_name)
# Set the permission so the blobs are public.
service.set_container_acl(container_name, public_access = PublicAccess.Container)
# get data path and list
abs_path = os.path.join(os.getcwd(), path)
list_video = os.listdir(abs_path)
for name in list_video :
path_tmp = os.path.join(abs_path, name)
print("Temp file : " + path_tmp)
print("\nUploading to Blob storage as blob : " + name)
print("\nUploading...\n")
start_time = time.time()
service.create_blob_from_path(container_name, name, path_tmp)
print("Upload complete!!")
print("Time taken {} sec" .format(time.time() - start_time))
os.remove(path_tmp)
return list_video
def download():
# Create the BlockBlockService that is used to call the Blob service for the storage account
block_blob_service = BlockBlobService(account_name='<account name>',
account_key='<account key>')
# Create a container called 'quickstartblobs'.
container_name = '<container name>'
block_blob_service.create_container(container_name)
# name of the container
generator = block_blob_service.list_blobs(container_name)
# code below lists all the blobs in the container and downloads them one after another
dpath = "C:\\Users\\<user name>\\Desktop\\data\\" #replace <user name> with your user name
for blob in generator:
print(blob.name)
print("{}".format(blob.name))
# check if the path contains a folder structure, create the folder structure
if "/" in "{}".format(blob.name):
print("there is a path in this")
# extract the folder path and check if that folder exists locally, and if not create it
head, tail = os.path.split("{}".format(blob.name))
print(head)
print(tail)
if (os.path.isdir(dpath + "\\" + head)):
# download the files to this directory
print("directory and sub directories exist")
block_blob_service.get_blob_to_path(
'suyash', blob.name, dpath + "\\" + head + "\\" + tail)
else:
# create the diretcory and download the file to it
print("directory doesn't exist, creating it now")
os.makedirs(dpath + "\\" + head, exist_ok=True)
print("directory created, download initiated")
block_blob_service.get_blob_to_path(
'<container name>', blob.name,
dpath + "\\" + head + "\\" + tail)
else:
block_blob_service.get_blob_to_path('<container name>', blob.name,
blob.name)
def uploadFolderCsvToBlob(storage_account, storage_key, container, in_dir, blob_out_dir):
""" Uploads csvs in a folder into blob storage
"""
# Create blob service
block_blob_service = BlockBlobService(account_name=storage_account, account_key=storage_key)
# Create container if not exists
if not any(container in c.name for c in block_blob_service.list_containers()):
block_blob_service.create_container(container, fail_on_exist=False)
# Upload the CSV file to Azure cloud
for root, dirs, filenames in os.walk(in_dir):
for filename in filenames:
if os.path.splitext(filename)[-1] == ".csv":
blob_name = blob_out_dir + '/' + filename
file_path = os.path.join(root, filename)
# Upload the CSV file to Azure cloud
block_blob_service.create_blob_from_path(
container, blob_name, file_path,
content_settings=ContentSettings(content_type='application/CSV'))
def make_public_container(STORAGE_NAME, STORAGE_KEY, NEW_CONTAINER_NAME):
""""create blob service, blob container and set it to public access. return blob service"""
blob_service = BlockBlobService(account_name= STORAGE_NAME, account_key=STORAGE_KEY)
new_container_status = blob_service.create_container(NEW_CONTAINER_NAME)
blob_service.set_container_acl(NEW_CONTAINER_NAME, public_access=PublicAccess.Container)
if new_container_status == True:
print('{} BLOB container has been successfully created: {}'.format(NEW_CONTAINER_NAME, new_container_status))
else:
print('{} something went wrong: check parameters and subscription'.format(NEW_CONTAINER_NAME))
def setup_file_share(container_name: str = os.getenv(
'AZURE_CONTAINER_NAME')) -> 'BlockBlobService':
'''
Create & setup Azure blob as share
Args:
container_name (str, optional): Name of container, will
default to using environment variable if param is not
provided.
Returns:
BlockBlobService: Instance of the Block Blob Service.
'''
blob_service = BlockBlobService(
account_name=os.getenv('STORAGE_ACCOUNT_NAME'),
account_key=os.getenv('STORAGE_ACCOUNT_KEY'))
blob_service.create_container(container_name, fail_on_exist=False)
return blob_service
def save_model_file(self, full_path_to_file, local_file_name):
"""Uploads a local file containg a model to the blob container specifed by self.config.models_container_name
If the container name does not exists it will be created.
Arguments:
full_path_to_file {[type]} -- full path to t the local file
local_file_name {[type]} -- the file name.
"""
assert (self.config.storage_account)
assert (self.config.storage_account_key)
assert (self.config.models_container_name)
block_blob_service = BlockBlobService(
account_name=self.config.storage_account,
account_key=self.config.storage_account_key)
block_blob_service.create_container(self.config.models_container_name,
fail_on_exist=False,
public_access=None)
block_blob_service.create_blob_from_path(
self.config.models_container_name, local_file_name,
full_path_to_file)
def _client(self):
"""Return the Azure Storage Block Blob service.
If this is the first call to the property, the client is created and
the container is created if it doesn't yet exist.
"""
client = BlockBlobService(connection_string=self._connection_string)
created = client.create_container(
container_name=self._container_name, fail_on_exist=False)
if created:
LOGGER.info("Created Azure Blob Storage container %s",
self._container_name)
client.retry = ExponentialRetry(
initial_backoff=self._retry_initial_backoff_sec,
increment_base=self._retry_increment_base,
max_attempts=self._retry_max_attempts).retry
return client
class InstanceStorage(object):
def __init__(self, group_name, location=None, create_if_not_exist=True):
client = arm.instance()
sm = client.storage.storage_accounts
# Check existence of a storage account in the resource group
# TODO : better with rm.list_resources for direct filtering
# but issing doc on Genric filtering format
# So, taking the first result of the iterator : Ouch !
new=True
for sa in sm.list_by_resource_group(group_name):
new=False
self.name = sa.name
self.location=sa.location
logger.debug("Found SA %s" % self.name)
break
if new:
logger.info("Creating storage account...")
#Generating unique name for Azure
unique_name = "%s%s" % (
str(group_name).translate(None, '-_.').lower(),
arm.id_generator()
)
# TODO : Check how to deal with account type...
# Warning : the name of the storageaccount cannot be > 24 chars
self.location=location
result = sm.create(
group_name,
unique_name[:24],
StorageAccountCreateParameters(
location=self.location,
account_type=AccountType.standard_lrs
)
)
# Asysnchronous operation, so wait...
res = result.result()
self.name = res.name
# retrieve the keys and store them in the instance
self.keys = sm.list_keys(group_name, self.name)
logger.debug("Key1 : %s " % repr(self.keys.key1))
logger.debug("Key2 : %s " % repr(self.keys.key2))
# retrieve the blob service
self.blob = BlockBlobService(
account_name=self.name,
account_key=self.keys.key1
)
# Define the storage tree :
# sources for the images imported to create the VM
# vhds for the VM images
self.sources_container= "sources"
self.vhds_container= "vhds"
self.blob.create_container(self.sources_container)
self.blob.create_container(self.vhds_container)
def list_blobs(self):
for blob in self.blob.list_blobs('system'):
print(blob.name)
def copy_source_images_from(self, source_storage, container, filepath):
# Generate a token for 10 minutes read access
token = source_storage.blob.generate_blob_shared_access_signature(
container,
filepath,
BlobPermissions.READ,
datetime.utcnow() + timedelta(minutes=10),
)
# Generate the sour URL of the blob
source_url = source_storage.blob.make_blob_url(
container,
filepath,
sas_token=token
)
# Launch the copy from the distant storage to the current one
self.blob.copy_blob(
self.sources_container,
os.path.basename(filepath),
source_url
)
def file_uploaded(self, botengine, device_object, file_id, filesize_bytes, content_type, file_extension):
"""
A device file has been uploaded
:param botengine: BotEngine environment
:param device_object: Device object that uploaded the file
:param file_id: File ID to reference this file at the server
:param filesize_bytes: The file size in bytes
:param content_type: The content type, for example 'video/mp4'
:param file_extension: The file extension, for example 'mp4'
"""
# We are demonstrating video processing here, so avoid video processing on files that are not videos.
if "video" not in content_type:
botengine.get_logger().info("The uploaded file is not a video, skipping processing ...")
return
# Create full file path
FILE_NAME = "video." + file_extension
FILE_PATH= os.path.dirname(os.path.abspath(__file__))
FULL_FILE_PATH =os.path.join(FILE_PATH, FILE_NAME)
# Download file to local device
botengine.download_file(file_id, FILE_NAME)
# Create Blob service and upload file to Microsoft Blob Storage
block_blob_service = BlockBlobService(account_name='andresdemo', account_key='6kG0iguGEK9h41shJasZOW4v/uPKr1Guu8RFWfLhWf7MnDcJaaw5eFhhYGSqaFk2qpqX8JIpJobe0bY8MmYa+g==')
block_blob_service.create_container(CONTAINER_NAME)
# Set Public Access to container
# A try and except block is used due to a occassional logger exception (Doesn't imapact function)
try:
block_blob_service.set_container_acl(CONTAINER_NAME, public_access=PublicAccess.Container)
except:
pass
# Convert the file into a blob and store it in Microsoft Azure Blob Storage
# A try and except block is used due to a occassional logger exception (Doesn't imapact function)
try:
block_blob_service.create_blob_from_path(CONTAINER_NAME, FILE_NAME, FULL_FILE_PATH)
except:
pass
# Get Video URL
url = "https://" + ACCOUNT_NAME + ".blob.core.windows.net/" + CONTAINER_NAME + "/" + FILE_NAME
# Get Access Token
token = "";
headers = {
'Ocp-Apim-Subscription-Key': API_KEY,
}
params = urllib.urlencode({
'allowEdit': 'True',
})
# HTTP GET request to Video Indexer API to acquire access token
try:
conn = httplib.HTTPSConnection('api.videoindexer.ai')
conn.request("GET", "/auth/" + ACCOUNT_LOCATION + "/Accounts/" + ACCOUNT_ID + "/AccessToken?%s" % params, headers=headers)
response = conn.getresponse()
token = response.read()
token = token[1:len(token)-1]
conn.close()
except Exception as e:
print("[Errno {0}] {1}".format(e.errno, e.strerror))
# Use Access Token to upload Video file
headers = {
'Content-Type': 'multipart/form-data',
}
params = urllib.urlencode({
'Content-Type': 'multipart/form-data',
'videoUrl': url,
'streamingPreset': 'Default',
'privacy': "Public"
})
try:
conn = httplib.HTTPSConnection('api.videoindexer.ai')
conn.request("POST", "/" + ACCOUNT_LOCATION + "/Accounts/" + ACCOUNT_ID + "/Videos?accessToken=" +token + "&name=Sample&%s" % params, headers=headers)
response = conn.getresponse()
data = response.read()
d = json.loads(data)
conn.close()
except Exception as e:
print("[Errno {0}] {1}".format(e.errno, e.strerror))
botengine.get_logger().info('Video Processing..')
# Get Video Index
params = urllib.urlencode({
'accessToken': token,
'language': 'English',
})
conn = httplib.HTTPSConnection('api.videoindexer.ai')
conn.request("GET", "/" + ACCOUNT_LOCATION + "/Accounts/" + ACCOUNT_ID + "/Videos/"+d["id"]+"/Index?%s" % params)
response = conn.getresponse()
data = response.read()
result = json.loads(data)
# Set a timer to queue for results
# Passing in Video ID and Access Token
self.start_timer_s(botengine, 5, [d["id"], token])
return
from datetime import datetime, timedelta
# The name of the new Shared Access policy
policy_name = "readandlistonly"
# The Storage Account Name
storage_account_name = "mystore"
storage_account_key = "mykey"
storage_container_name = "mycontainer"
example_file_path = "..\\sampledata\\sample.log"
policy_name = "mysaspolicy"
# Create the blob service, using the name and key for your Azure Storage account
blob_service = BlockBlobService(storage_account_name, storage_account_key)
# Create the container, if it does not already exist
blob_service.create_container(storage_container_name)
# Upload an example file to the container
blob_service.create_blob_from_path(storage_container_name, "sample.log", example_file_path)
# Create a new policy that expires after a week
access_policy = AccessPolicy(
permission=ContainerPermissions.READ + ContainerPermissions.LIST, expiry=datetime.utcnow() + timedelta(weeks=1)
)
# Get the existing identifiers (policies) for the container
identifiers = blob_service.get_container_acl(storage_container_name)
# And add the new one ot the list
identifiers[policy_name] = access_policy
print(' - Pushing selected files to Azure for ' + Ldir['date_string'])
f_string = 'f' + Ldir['date_string']
# Azure commands
from azure.storage.blob import BlockBlobService
from azure.storage.blob import PublicAccess
ff_string = f_string.replace('.','') # azure does not like dots in container names
# account name and key
azu_dict = Lfun.csv_to_dict(Ldir['data'] + 'accounts/azure_pm_2015.05.25.csv')
account = azu_dict['account']
key = azu_dict['key']
containername = ff_string
# get a handle to the account
blob_service = BlockBlobService(account_name=account, account_key=key)
blob_service.create_container(containername)
blob_service.set_container_acl(containername, public_access=PublicAccess.Container)
# input directory
in_dir = Ldir['roms'] + 'output/' + Ldir['gtagex'] + '/' + f_string + '/'
# output files
out_list = ['ocean_surface.nc', 'low_passed_UBC.nc']
def write_to_azure(out_fn, blob_service, containername, outname):
# write it to Azure
try:
bname = open(out_fn, 'rb')
blob_service.create_blob_from_stream(containername, out_name, bname)
print('done putting ' + out_name)
bname.close()
result = 'success'
class AzureBlobWriter(BaseWriter):
"""
Writes items to azure blob containers.
- account_name (str)
Public acces name of the azure account.
- account_key (str)
Public acces key to the azure account.
- container (str)
Blob container name.
"""
supported_options = {
'account_name': {'type': six.string_types, 'env_fallback': 'EXPORTERS_AZUREWRITER_NAME'},
'account_key': {'type': six.string_types, 'env_fallback': 'EXPORTERS_AZUREWRITER_KEY'},
'container': {'type': six.string_types}
}
hash_algorithm = 'md5'
VALID_CONTAINER_NAME_RE = r'[a-zA-Z0-9-]{3,63}'
def __init__(self, *args, **kw):
from azure.storage.blob import BlockBlobService
super(AzureBlobWriter, self).__init__(*args, **kw)
account_name = self.read_option('account_name')
account_key = self.read_option('account_key')
self.container = self.read_option('container')
if '--' in self.container or not re.match(self.VALID_CONTAINER_NAME_RE, self.container):
help_url = ('https://azure.microsoft.com/en-us/documentation'
'/articles/storage-python-how-to-use-blob-storage/')
warnings.warn("Container name %s doesn't conform with naming rules (see: %s)"
% (self.container, help_url))
self.azure_service = BlockBlobService(account_name, account_key)
self.azure_service.create_container(self.container)
self.logger.info('AzureBlobWriter has been initiated.'
'Writing to container {}'.format(self.container))
self.set_metadata('files_counter', 0)
self.set_metadata('blobs_written', [])
def write(self, dump_path, group_key=None):
self.logger.info('Start uploading {} to {}'.format(dump_path, self.container))
self._write_blob(dump_path)
self.set_metadata('files_counter', self.get_metadata('files_counter') + 1)
@retry_long
def _write_blob(self, dump_path):
blob_name = dump_path.split('/')[-1]
self.azure_service.create_blob_from_path(
self.read_option('container'),
blob_name,
dump_path,
max_connections=5,
)
self.logger.info('Saved {}'.format(blob_name))
self._update_metadata(dump_path, blob_name)
def _update_metadata(self, dump_path, blob_name):
buffer_info = self.write_buffer.metadata[dump_path]
file_info = {
'blob_name': blob_name,
'size': buffer_info['size'],
'hash': b64encode(unhexlify(buffer_info['file_hash'])),
'number_of_records': buffer_info['number_of_records']
}
self.get_metadata('blobs_written').append(file_info)
def _check_write_consistency(self):
from azure.common import AzureMissingResourceHttpError
for blob_info in self.get_metadata('blobs_written'):
try:
blob = self.azure_service.get_blob_properties(
self.read_option('container'), blob_info['blob_name'])
blob_size = blob.properties.content_length
blob_md5 = blob.properties.content_settings.content_md5
if str(blob_size) != str(blob_info['size']):
raise InconsistentWriteState(
'File {} has unexpected size. (expected {} - got {})'.format(
blob_info['blob_name'], blob_info['size'], blob_size
)
)
if str(blob_md5) != str(blob_info['hash']):
raise InconsistentWriteState(
'File {} has unexpected hash. (expected {} - got {})'.format(
blob_info['blob_name'], blob_info['hash'], blob_md5
)
)
except AzureMissingResourceHttpError:
raise InconsistentWriteState('Missing blob {}'.format(blob_info['blob_name']))
self.logger.info('Consistency check passed')
from azure.storage.blob import BlockBlobService
from threading import Thread
import csv
import json
import sys
import time
import smtplib
import string
print 'Connecting to Azure...'
# Azure
name='perfectndsa'
key = '41eoKtqzIXvovIzQhWYi5ZK+JY7PU1/6Tp5fHpnFm9v37z1OPRTCWaJ1hhhQh6rqeF+u3BKXjaUAJ8SgXnhzWQ=='
block_blob = BlockBlobService(account_name=name, account_key=key)
block_blob.create_container('example')
print 'Initializing sensor...'
# Sensor
hat = SenseHat()
# Send data to azure method
def send_data(data):
json_data = json.dumps(data)
block_blob.create_blob_from_text("example", 'data.json', json_data)
# Prepare output csv
with open('data.csv', 'w') as data_file:
writer = csv.writer(data_file, delimiter=',')
writer.writerow(["Timestamp", "Temperature_hum (Cº)", "Temperature_press (Cº)", "Humidity (%rH)", "Pressure (mBars)"])
print("The storage account was taken of a EXISTING storage account.")
'''
Get the keys for create a container where would be the the virtual hard disks
'''
keys = sms.get_storage_account_keys(name_account)
primary_key = keys.storage_service_keys.primary
'''
Create a container
'''
block_blob_service = BlockBlobService(account_name = name_account, account_key = primary_key )
name_container = "azbrcontainer" + str(hash_number)
list_containers = list(block_blob_service.list_containers())
if len(list_containers)==0:
container = block_blob_service.create_container(name_container)
#wait_for_async(container.request_id, 'Creating storage account', 600)
list_containers = list(block_blob_service.list_containers())
container = list_containers[0]
name_container = container.name
block_blob_service.set_container_acl(name_container, public_access=PublicAccess.Container)
''' FINISHING CREAION OF ACCOUNTS OR GET EXISTEN ACCOUNDS'''
'''Create of deplyment and in each deployment a machine'''
class AzureStorage(object):
"""
Storage object representing files on an Azure Storage container.
Usage::
from flask_admin.contrib.fileadmin import BaseFileAdmin
from flask_admin.contrib.fileadmin.azure import AzureStorage
class MyAzureAdmin(BaseFileAdmin):
# Configure your class however you like
pass
fileadmin_view = MyAzureAdmin(storage=AzureStorage(...))
"""
_fakedir = '.dir'
_copy_poll_interval_seconds = 1
_send_file_lookback = timedelta(minutes=15)
_send_file_validity = timedelta(hours=1)
separator = '/'
def __init__(self, container_name, connection_string):
"""
Constructor
:param container_name:
Name of the container that the files are on.
:param connection_string:
Azure Blob Storage Connection String
"""
if not BlockBlobService:
raise ValueError('Could not import Azure Blob Storage SDK. '
'You can install the SDK using '
'pip install azure-storage-blob')
self._container_name = container_name
self._connection_string = connection_string
self.__client = None
@property
def _client(self):
if not self.__client:
self.__client = BlockBlobService(
connection_string=self._connection_string)
self.__client.create_container(
self._container_name, fail_on_exist=False)
return self.__client
@classmethod
def _get_blob_last_modified(cls, blob):
last_modified = blob.properties.last_modified
tzinfo = last_modified.tzinfo
epoch = last_modified - datetime(1970, 1, 1, tzinfo=tzinfo)
return epoch.total_seconds()
@classmethod
def _ensure_blob_path(cls, path):
if path is None:
return None
path_parts = path.split(op.sep)
return cls.separator.join(path_parts).lstrip(cls.separator)
def get_files(self, path, directory):
if directory and path != directory:
path = op.join(path, directory)
path = self._ensure_blob_path(path)
directory = self._ensure_blob_path(directory)
path_parts = path.split(self.separator) if path else []
num_path_parts = len(path_parts)
folders = set()
files = []
for blob in self._client.list_blobs(self._container_name, path):
blob_path_parts = blob.name.split(self.separator)
name = blob_path_parts.pop()
blob_is_file_at_current_level = blob_path_parts == path_parts
blob_is_directory_file = name == self._fakedir
if blob_is_file_at_current_level and not blob_is_directory_file:
rel_path = blob.name
is_dir = False
size = blob.properties.content_length
last_modified = self._get_blob_last_modified(blob)
files.append((name, rel_path, is_dir, size, last_modified))
else:
next_level_folder = blob_path_parts[:num_path_parts + 1]
folder_name = self.separator.join(next_level_folder)
folders.add(folder_name)
folders.discard(directory)
for folder in folders:
name = folder.split(self.separator)[-1]
rel_path = folder
is_dir = True
size = 0
last_modified = 0
files.append((name, rel_path, is_dir, size, last_modified))
return files
def is_dir(self, path):
path = self._ensure_blob_path(path)
num_blobs = 0
for blob in self._client.list_blobs(self._container_name, path):
blob_path_parts = blob.name.split(self.separator)
is_explicit_directory = blob_path_parts[-1] == self._fakedir
if is_explicit_directory:
return True
num_blobs += 1
path_cannot_be_leaf = num_blobs >= 2
if path_cannot_be_leaf:
return True
return False
def path_exists(self, path):
path = self._ensure_blob_path(path)
if path == self.get_base_path():
return True
try:
next(iter(self._client.list_blobs(self._container_name, path)))
except StopIteration:
return False
else:
return True
def get_base_path(self):
return ''
def get_breadcrumbs(self, path):
path = self._ensure_blob_path(path)
accumulator = []
breadcrumbs = []
for folder in path.split(self.separator):
accumulator.append(folder)
breadcrumbs.append((folder, self.separator.join(accumulator)))
return breadcrumbs
def send_file(self, file_path):
file_path = self._ensure_blob_path(file_path)
if not self._client.exists(self._container_name, file_path):
raise ValueError()
now = datetime.utcnow()
url = self._client.make_blob_url(self._container_name, file_path)
sas = self._client.generate_blob_shared_access_signature(
self._container_name, file_path,
BlobPermissions.READ,
expiry=now + self._send_file_validity,
start=now - self._send_file_lookback)
return redirect('%s?%s' % (url, sas))
def read_file(self, path):
path = self._ensure_blob_path(path)
blob = self._client.get_blob_to_bytes(self._container_name, path)
return blob.content
def write_file(self, path, content):
path = self._ensure_blob_path(path)
self._client.create_blob_from_text(self._container_name, path, content)
def save_file(self, path, file_data):
path = self._ensure_blob_path(path)
self._client.create_blob_from_stream(self._container_name, path,
file_data.stream)
def delete_tree(self, directory):
directory = self._ensure_blob_path(directory)
for blob in self._client.list_blobs(self._container_name, directory):
self._client.delete_blob(self._container_name, blob.name)
def delete_file(self, file_path):
file_path = self._ensure_blob_path(file_path)
self._client.delete_blob(self._container_name, file_path)
def make_dir(self, path, directory):
path = self._ensure_blob_path(path)
directory = self._ensure_blob_path(directory)
blob = self.separator.join([path, directory, self._fakedir])
blob = blob.lstrip(self.separator)
self._client.create_blob_from_text(self._container_name, blob, '')
def _copy_blob(self, src, dst):
src_url = self._client.make_blob_url(self._container_name, src)
copy = self._client.copy_blob(self._container_name, dst, src_url)
while copy.status != 'success':
sleep(self._copy_poll_interval_seconds)
copy = self._client.get_blob_properties(
self._container_name, dst).properties.copy
def _rename_file(self, src, dst):
self._copy_blob(src, dst)
self.delete_file(src)
def _rename_directory(self, src, dst):
for blob in self._client.list_blobs(self._container_name, src):
self._rename_file(blob.name, blob.name.replace(src, dst, 1))
def rename_path(self, src, dst):
src = self._ensure_blob_path(src)
dst = self._ensure_blob_path(dst)
if self.is_dir(src):
self._rename_directory(src, dst)
else:
self._rename_file(src, dst)
class AzureIOStore(IOStore):
"""
A class that lets you get input from and send output to Azure Storage.
"""
def __init__(self, account_name, container_name, name_prefix=""):
"""
Make a new AzureIOStore that reads from and writes to the given
container in the given account, adding the given prefix to keys. All
paths will be interpreted as keys or key prefixes.
If the name prefix does not end with a trailing slash, and is not empty,
one will be added automatically.
Account keys are retrieved from the AZURE_ACCOUNT_KEY environment
variable or from the ~/.toilAzureCredentials file, as in Toil itself.
"""
# Make sure azure libraries actually loaded
assert(have_azure)
self.account_name = account_name
self.container_name = container_name
self.name_prefix = name_prefix
if self.name_prefix != "" and not self.name_prefix.endswith("/"):
# Make sure it has the trailing slash required.
self.name_prefix += "/"
# Sneak into Toil and use the same keys it uses
self.account_key = toil.jobStores.azureJobStore._fetchAzureAccountKey(
self.account_name)
# This will hold out Azure blob store connection
self.connection = None
def __getstate__(self):
"""
Return the state to use for pickling. We don't want to try and pickle
an open Azure connection.
"""
return (self.account_name, self.account_key, self.container_name,
self.name_prefix)
def __setstate__(self, state):
"""
Set up after unpickling.
"""
self.account_name = state[0]
self.account_key = state[1]
self.container_name = state[2]
self.name_prefix = state[3]
self.connection = None
def __connect(self):
"""
Make sure we have an Azure connection, and set one up if we don't.
"""
if self.connection is None:
RealTimeLogger.get().debug("Connecting to account {}, using "
"container {} and prefix {}".format(self.account_name,
self.container_name, self.name_prefix))
# Connect to the blob service where we keep everything
self.connection = BlockBlobService(
account_name=self.account_name, account_key=self.account_key)
@backoff
def read_input_file(self, input_path, local_path):
"""
Get input from Azure.
"""
self.__connect()
RealTimeLogger.get().debug("Loading {} from AzureIOStore".format(
input_path))
# Download the blob. This is known to be synchronous, although it can
# call a callback during the process.
self.connection.get_blob_to_path(self.container_name,
self.name_prefix + input_path, local_path)
def list_input_directory(self, input_path, recursive=False,
with_times=False):
"""
Loop over fake /-delimited directories on Azure. The prefix may or may
not not have a trailing slash; if not, one will be added automatically.
Returns the names of files and fake directories in the given input fake
directory, non-recursively.
If with_times is specified, will yield (name, time) pairs including
modification times as datetime objects. Times on directories are None.
"""
self.__connect()
RealTimeLogger.get().info("Enumerating {} from AzureIOStore".format(
input_path))
# Work out what the directory name to list is
fake_directory = self.name_prefix + input_path
if fake_directory != "" and not fake_directory.endswith("/"):
# We have a nonempty prefix, and we need to end it with a slash
fake_directory += "/"
# This will hold the marker that we need to send back to get the next
# page, if there is one. See <http://stackoverflow.com/a/24303682>
marker = None
# This holds the subdirectories we found; we yield each exactly once if
# we aren't recursing.
subdirectories = set()
while True:
# Get the results from Azure. We don't use delimiter since Azure
# doesn't seem to provide the placeholder entries it's supposed to.
result = self.connection.list_blobs(self.container_name,
prefix=fake_directory, marker=marker)
RealTimeLogger.get().info("Found {} files".format(len(result)))
for blob in result:
# Yield each result's blob name, but directory names only once
# Drop the common prefix
relative_path = blob.name[len(fake_directory):]
if (not recursive) and "/" in relative_path:
# We found a file in a subdirectory, and we aren't supposed
# to be recursing.
subdirectory, _ = relative_path.split("/", 1)
if subdirectory not in subdirectories:
# It's a new subdirectory. Yield and remember it
subdirectories.add(subdirectory)
if with_times:
yield subdirectory, None
else:
yield subdirectory
else:
# We found an actual file
if with_times:
mtime = blob.properties.last_modified
if isinstance(mtime, datetime.datetime):
# Make sure we're getting proper localized datetimes
# from the new Azure Storage API.
assert(mtime.tzinfo is not None and
mtime.tzinfo.utcoffset(mtime) is not None)
else:
# Convert mtime from a string as in the old API.
mtime = dateutil.parser.parse(mtime).replace(
tzinfo=dateutil.tz.tzutc())
yield relative_path, mtime
else:
yield relative_path
# Save the marker
marker = result.next_marker
if not marker:
break
@backoff
def write_output_file(self, local_path, output_path):
"""
Write output to Azure. Will create the container if necessary.
"""
self.__connect()
RealTimeLogger.get().debug("Saving {} to AzureIOStore".format(
output_path))
try:
# Make the container
self.connection.create_container(self.container_name)
except azure.WindowsAzureConflictError:
# The container probably already exists
pass
# Upload the blob (synchronously)
# TODO: catch no container error here, make the container, and retry
self.connection.put_block_blob_from_path(self.container_name,
self.name_prefix + output_path, local_path)
@backoff
def exists(self, path):
"""
Returns true if the given input or output file exists in Azure already.
"""
self.__connect()
marker = None
while True:
# Get the results from Azure.
result = self.connection.list_blobs(self.container_name,
prefix=self.name_prefix + path, marker=marker)
for blob in result:
# Look at each blob
if blob.name == self.name_prefix + path:
# Found it
return True
# Save the marker
marker = result.next_marker
if not marker:
break
return False
@backoff
def get_mtime(self, path):
"""
Returns the modification time of the given blob if it exists, or None
otherwise.
"""
self.__connect()
marker = None
while True:
# Get the results from Azure.
result = self.connection.list_blobs(self.container_name,
prefix=self.name_prefix + path, marker=marker)
for blob in result:
# Look at each blob
if blob.name == self.name_prefix + path:
# Found it
mtime = blob.properties.last_modified
if isinstance(mtime, datetime.datetime):
# Make sure we're getting proper localized datetimes
# from the new Azure Storage API.
assert(mtime.tzinfo is not None and
mtime.tzinfo.utcoffset(mtime) is not None)
else:
# Convert mtime from a string as in the old API.
mtime = dateutil.parser.parse(mtime).replace(
tzinfo=dateutil.tz.tzutc())
return mtime
# Save the marker
marker = result.next_marker
if not marker:
break
return None
@backoff
def get_size(self, path):
"""
Returns the size in bytes of the given blob if it exists, or None
otherwise.
"""
self.__connect()
marker = None
while True:
# Get the results from Azure.
result = self.connection.list_blobs(self.container_name,
prefix=self.name_prefix + path, marker=marker)
for blob in result:
# Look at each blob
if blob.name == self.name_prefix + path:
# Found it
size = blob.properties.content_length
return size
# Save the marker
marker = result.next_marker
if not marker:
break
return None
class AzureStorage(plugins.IStoragePlugin):
def configure(self, config):
self.storage = BlockBlobService(account_name=config['account_name'], account_key=config['account_key'])
self.container = config['container']
try:
container = self.storage.get_container_properties(self.container)
log.info("Configuring Azure blob storage %s/%s", self.storage.account_name, self.container)
except AzureMissingResourceHttpError as e:
log.warning("Container '%s' is missing in account '%s', trying to create new", self.container, self.storage.account_name)
try:
self.storage.create_container(self.container)
self.storage.set_container_acl(self.container, public_access=PublicAccess.Container)
except Exception as e:
log.critical("Cannot create new container: %s", e)
raise plugins.PluginInitException("Cannot create new container")
except AzureHttpError as e:
log.critical("Cannot access container '%s' in account '%s': %s", self.container, self.storage.account_name, e)
raise plugins.PluginInitException("Cannot access container")
except Exception as e:
log.critical("Cannot access container '%s' in account '%s': %s", self.container, self.storage.account_name, e)
raise plugins.PluginInitException("Cannot access container")
def delete(self, key):
log.info("Deleting file '%s' from %s/%s", key, self.storage.account_name, self.container)
try:
self.storage.delete_blob(self.container, key)
except AzureMissingResourceHttpError:
log.error("File '%s' was not found in %s/%s", key, self.storage.account_name, self.container)
raise common.NotFound('File not found')
except Exception as e:
log.error("Cannot delete '%s' from %s/%s: %s", key, self.storage.account_name, self.container, e)
raise common.FatalError(e)
def put(self, key, filename=None, file=None):
storage_key = key
try:
if filename:
log.debug("Uploading %s to %s", filename, self.storage.make_blob_url(self.container, storage_key))
self.storage.create_blob_from_path(self.container, storage_key, filename, content_settings=ContentSettings(content_type='application/octet-stream'))
elif file:
old_pos = file.tell()
file.seek(0)
log.debug("Uploading from stream to %s", self.storage.make_blob_url(self.container, storage_key))
self.storage.create_blob_from_stream(self.container, storage_key, file, content_settings=ContentSettings(content_type='application/octet-stream'))
file.seek(old_pos)
except Exception as e:
# TODO: more detailed error inspection
log.critical("Error uploading to %s/%s: %s", self.storage.account_name, self.container, e)
raise common.FatalError(e)
return storage_key
def get(self, key, stream):
# current azure python sdk barely can work with non-seekable streams,
# so we have to implement chunking by our own
# TODO: proper ranging? RFC says server SHOULD return 406 once range is unsatisfiable,
# but Azure is OK with end pos > blob length unless blob is not empty
chunk_size = 4*1024*1024
chunk_start = 0
chunk_end = chunk_size - 1
while True:
try:
chunk = self.storage._get_blob(self.container, key, start_range=chunk_start, end_range=chunk_end)
log.debug("Writing %s bytes from %s", len(chunk.content), chunk_start)
stream.write(chunk.content)
except IOError:
# remote side closed connection
return
except AzureMissingResourceHttpError as e:
raise common.NotFound(e)
except (AzureHttpError, AzureException) as e:
raise common.TemporaryError('Error while downloading {}: {}'.format(key, e))
blob_length = int(chunk.properties.content_range.split('/')[1])
chunk_start, chunk_end, blob_size = map(int, re.match(r'^bytes\s+(\d+)-(\d+)/(\d+)$', chunk.properties.content_range).groups())
if chunk_end == blob_size - 1:
# no more data to stream
break
else:
chunk_start = chunk_end + 1
chunk_end += chunk_size
return 0
def test_job_level_mounting(self, resource_group, location, cluster, storage_account, storage_account_key):
"""Tests if it's possible to mount external file systems for a job."""
job_name = 'job'
# Create file share and container to mount on the job level
if storage_account.name != helpers.FAKE_STORAGE.name:
files = FileService(storage_account.name, storage_account_key)
files.create_share('jobshare', fail_on_exist=False)
blobs = BlockBlobService(storage_account.name, storage_account_key)
blobs.create_container('jobcontainer', fail_on_exist=False)
job = self.client.jobs.create(
resource_group.name,
job_name,
parameters=models.JobCreateParameters(
location=location,
cluster=models.ResourceId(id=cluster.id),
node_count=1,
mount_volumes=models.MountVolumes(
azure_file_shares=[
models.AzureFileShareReference(
account_name=storage_account.name,
azure_file_url='https://{0}.file.core.windows.net/{1}'.format(
storage_account.name, 'jobshare'),
relative_mount_path='job_afs',
credentials=models.AzureStorageCredentialsInfo(
account_key=storage_account_key
),
)
],
azure_blob_file_systems=[
models.AzureBlobFileSystemReference(
account_name=storage_account.name,
container_name='jobcontainer',
relative_mount_path='job_bfs',
credentials=models.AzureStorageCredentialsInfo(
account_key=storage_account_key
),
)
]
),
# Put standard output on cluster level AFS to check that the job has access to it.
std_out_err_path_prefix='$AZ_BATCHAI_MOUNT_ROOT/{0}'.format(helpers.AZURE_FILES_MOUNTING_PATH),
# Create two output directories on job level AFS and blobfuse.
output_directories=[
models.OutputDirectory(id='OUTPUT1', path_prefix='$AZ_BATCHAI_JOB_MOUNT_ROOT/job_afs'),
models.OutputDirectory(id='OUTPUT2', path_prefix='$AZ_BATCHAI_JOB_MOUNT_ROOT/job_bfs')
],
# Check that the job preparation has access to job level file systems.
job_preparation=models.JobPreparation(
command_line='echo afs > $AZ_BATCHAI_OUTPUT_OUTPUT1/prep_afs.txt; '
'echo bfs > $AZ_BATCHAI_OUTPUT_OUTPUT2/prep_bfs.txt; '
'echo done'
),
# Check that the job has access to job
custom_toolkit_settings=models.CustomToolkitSettings(
command_line='echo afs > $AZ_BATCHAI_OUTPUT_OUTPUT1/job_afs.txt; '
'echo bfs > $AZ_BATCHAI_OUTPUT_OUTPUT2/job_bfs.txt; '
'mkdir $AZ_BATCHAI_OUTPUT_OUTPUT1/afs; '
'echo afs > $AZ_BATCHAI_OUTPUT_OUTPUT1/afs/job_afs.txt; '
'mkdir $AZ_BATCHAI_OUTPUT_OUTPUT2/bfs; '
'echo bfs > $AZ_BATCHAI_OUTPUT_OUTPUT2/bfs/job_bfs.txt; '
'echo done'
)
)
).result()
self.assertEqual(
helpers.wait_for_job_completion(self.is_live, self.client, resource_group.name, job.name,
helpers.MINUTE),
models.ExecutionState.succeeded)
job = self.client.jobs.get(resource_group.name, job.name)
# Assert job and job prep standard output is populated on cluster level filesystem
helpers.assert_job_files_are(self, self.client, resource_group.name, job.name,
helpers.STANDARD_OUTPUT_DIRECTORY_ID,
{u'stdout.txt': u'done\n', u'stderr.txt': u'',
u'stdout-job_prep.txt': u'done\n', u'stderr-job_prep.txt': u''})
# Assert files are generated on job level AFS
helpers.assert_job_files_are(self, self.client, resource_group.name, job.name, 'OUTPUT1',
{u'job_afs.txt': u'afs\n', u'prep_afs.txt': u'afs\n', u'afs': None})
# Assert files are generated on job level blobfuse
helpers.assert_job_files_are(self, self.client, resource_group.name, job.name, 'OUTPUT2',
{u'job_bfs.txt': u'bfs\n', u'prep_bfs.txt': u'bfs\n', u'bfs': None})
# Assert subfolders are available via API
helpers.assert_job_files_in_path_are(self, self.client, resource_group.name, job.name, 'OUTPUT1',
'afs', {u'job_afs.txt': u'afs\n'})
helpers.assert_job_files_in_path_are(self, self.client, resource_group.name, job.name, 'OUTPUT2',
'bfs', {u'job_bfs.txt': u'bfs\n'})
# Assert that we can access the output files created on job level mount volumes directly in storage using path
# segment returned by the server.
if storage_account.name != helpers.FAKE_STORAGE.name:
files = FileService(storage_account.name, storage_account_key)
self.assertTrue(
files.exists('jobshare', job.job_output_directory_path_segment +
'/' + helpers.OUTPUT_DIRECTORIES_FOLDER_NAME, 'job_afs.txt'))
blobs = BlockBlobService(storage_account.name, storage_account_key)
self.assertTrue(
blobs.exists('jobcontainer', job.job_output_directory_path_segment +
'/' + helpers.OUTPUT_DIRECTORIES_FOLDER_NAME + '/job_bfs.txt'))
# After the job is done the filesystems should be unmounted automatically, check this by submitting a new job.
checker = self.client.jobs.create(
resource_group.name,
'checker',
parameters=models.JobCreateParameters(
location=location,
cluster=models.ResourceId(id=cluster.id),
node_count=1,
std_out_err_path_prefix='$AZ_BATCHAI_MOUNT_ROOT/{0}'.format(helpers.AZURE_FILES_MOUNTING_PATH),
custom_toolkit_settings=models.CustomToolkitSettings(
command_line='echo job; df | grep -E "job_bfs|job_afs"'
)
)
).result()
# Check the job failed because there are not job level mount volumes anymore
self.assertEqual(
helpers.wait_for_job_completion(self.is_live, self.client, resource_group.name, checker.name,
helpers.MINUTE),
models.ExecutionState.failed)
# Check that the cluster level AFS was still mounted
helpers.assert_job_files_are(self, self.client, resource_group.name, checker.name,
helpers.STANDARD_OUTPUT_DIRECTORY_ID,
{u'stdout.txt': u'job\n', u'stderr.txt': u''})
# -*- coding: utf-8 -*-
import os
from azure.storage.blob import BlockBlobService, ContentSettings
ACCOUNT = os.environ.get('BLOB_ACCOUNT')
KEY = os.environ.get('BLOB_KEY')
CONTAINER = 'beercartography'
block_blob_service = BlockBlobService(account_name=ACCOUNT, account_key=KEY)
block_blob_service.create_container(CONTAINER)
def save_file(stream, filename, mimetype):
block_blob_service.create_blob_from_stream(
CONTAINER,
filename,
stream,
content_settings=ContentSettings(content_type=mimetype)
)
def get_images():
generator = block_blob_service.list_blobs('beercartography')
for blob in generator:
yield blob.name
def get_image(filename):
blob = block_blob_service.get_blob_to_bytes(CONTAINER, filename)
content = blob.content
return content
class AzureTransfer(BaseTransfer):
def __init__(self, account_name, account_key, container_name, prefix=None):
# NOTE: Azure wants all paths to start with a slash
prefix = "/{}".format(prefix.lstrip("/") if prefix else "")
super().__init__(prefix=prefix)
self.account_name = account_name
self.account_key = account_key
self.container_name = container_name
self.conn = BlockBlobService(account_name=self.account_name, account_key=self.account_key)
self.container = self.get_or_create_container(self.container_name)
self.log.debug("AzureTransfer initialized")
# XXX: AzureTransfer isn't actively tested and hasn't its error handling is probably lacking
#self.log.warning("AzureTransfer is experimental and has not been thoroughly tested")
def get_metadata_for_key(self, key):
key = self.format_key_for_backend(key)
return self._list_blobs(key)[0]["metadata"]
def _metadata_for_key(self, key):
return self._list_blobs(key)[0]["metadata"]
def list_path(self, key):
path = self.format_key_for_backend(key, trailing_slash=True)
return self._list_blobs(path)
def _list_blobs(self, path):
self.log.debug("Listing path %r", path)
items = self.conn.list_blobs(self.container_name, prefix=path, delimiter="/", include="metadata")
result = []
for item in items:
result.append({
"last_modified": item.properties.last_modified.isoformat(),
"metadata": item.metadata,
"name": self.format_key_from_backend(item.name),
"size": item.properties.content_length,
})
return result
def delete_key(self, key):
key = self.format_key_for_backend(key)
self.log.debug("Deleting key: %r", key)
return self.conn.delete_blob(self.container_name, key)
def get_contents_to_file(self, key, filepath_to_store_to, *, progress_callback=None):
key = self.format_key_for_backend(key)
self.log.debug("Starting to fetch the contents of: %r to: %r", key, filepath_to_store_to)
meta = self.conn.get_blob_to_path(self.container_name, key, filepath_to_store_to)
if progress_callback:
progress_callback(1, 1)
return meta
def get_contents_to_fileobj(self, key, fileobj_to_store_to, *, progress_callback=None):
key = self.format_key_for_backend(key)
self.log.debug("Starting to fetch the contents of: %r", key)
meta = self.conn.get_blob_to_file(self.container_name, key, fileobj_to_store_to)
if progress_callback:
progress_callback(1, 1)
return meta
def get_contents_to_string(self, key):
key = self.format_key_for_backend(key)
self.log.debug("Starting to fetch the contents of: %r", key)
return self.conn.get_blob_to_bytes(self.container_name, key), self._metadata_for_key(key)
def store_file_from_memory(self, key, memstring, metadata=None):
key = self.format_key_for_backend(key)
self.conn.create_blob_from_bytes(self.container_name, key, memstring,
x_ms_meta_name_values=self.sanitize_metadata(metadata))
def store_file_from_disk(self, key, filepath, metadata=None, multipart=None):
key = self.format_key_for_backend(key)
self.conn.create_blob_from_path(self.container_name, key, filepath,None)
# x_ms_meta_name_values=self.sanitize_metadata(metadata))
def get_or_create_container(self, container_name):
start_time = time.time()
self.conn.create_container(container_name)
self.log.debug("Got/Created container: %r successfully, took: %.3fs", container_name, time.time() - start_time)
return container_name
class AzureTransfer(BaseTransfer):
def __init__(self, account_name, account_key, bucket_name, prefix=None):
prefix = "{}".format(prefix.lstrip("/") if prefix else "")
super().__init__(prefix=prefix)
self.account_name = account_name
self.account_key = account_key
self.container_name = bucket_name
self.conn = BlockBlobService(account_name=self.account_name, account_key=self.account_key)
self.container = self.get_or_create_container(self.container_name)
self.log.debug("AzureTransfer initialized, %r", self.container_name)
def get_metadata_for_key(self, key):
key = self.format_key_for_backend(key, remove_slash_prefix=True, trailing_slash=False)
results = self._list_blobs(key)
if not results:
raise FileNotFoundFromStorageError(key)
return results[0]["metadata"]
def _metadata_for_key(self, key):
return self._list_blobs(key)[0]["metadata"]
def list_path(self, key, trailing_slash=True): # pylint: disable=arguments-differ
# Trailing slash needed when listing directories, without when listing individual files
path = self.format_key_for_backend(key, remove_slash_prefix=True, trailing_slash=trailing_slash)
return self._list_blobs(path)
def _list_blobs(self, path):
self.log.debug("Listing path %r", path)
if path:
items = self.conn.list_blobs(self.container_name, prefix=path, delimiter="/", include="metadata")
else: # If you give Azure an empty path, it gives you an authentication error
items = self.conn.list_blobs(self.container_name, delimiter="/", include="metadata")
results = []
for item in items:
if not isinstance(item, BlobPrefix):
results.append({
"last_modified": item.properties.last_modified,
# Azure Storage cannot handle '-' so we turn them into underscores and back again
"metadata": dict((k.replace("_", "-"), v) for k, v in item.metadata.items()),
"name": self.format_key_from_backend(item.name),
"size": item.properties.content_length,
})
return results
def delete_key(self, key):
key = self.format_key_for_backend(key, remove_slash_prefix=True)
self.log.debug("Deleting key: %r", key)
try:
return self.conn.delete_blob(self.container_name, key)
except azure.common.AzureMissingResourceHttpError as ex:
raise FileNotFoundFromStorageError(key) from ex
def get_contents_to_file(self, key, filepath_to_store_to, *, progress_callback=None):
key = self.format_key_for_backend(key, remove_slash_prefix=True)
self.log.debug("Starting to fetch the contents of: %r to: %r", key, filepath_to_store_to)
try:
self.conn.get_blob_to_path(self.container_name, key, filepath_to_store_to)
except azure.common.AzureMissingResourceHttpError as ex:
raise FileNotFoundFromStorageError(key) from ex
if progress_callback:
progress_callback(1, 1)
return self._metadata_for_key(key)
def get_contents_to_fileobj(self, key, fileobj_to_store_to, *, progress_callback=None):
key = self.format_key_for_backend(key, remove_slash_prefix=True)
self.log.debug("Starting to fetch the contents of: %r", key)
try:
self.conn.get_blob_to_stream(self.container_name, key, fileobj_to_store_to)
except azure.common.AzureMissingResourceHttpError as ex:
raise FileNotFoundFromStorageError(key) from ex
if progress_callback:
progress_callback(1, 1)
return self._metadata_for_key(key)
def get_contents_to_string(self, key):
key = self.format_key_for_backend(key, remove_slash_prefix=True)
self.log.debug("Starting to fetch the contents of: %r", key)
try:
blob = self.conn.get_blob_to_bytes(self.container_name, key)
return blob.content, self._metadata_for_key(key)
except azure.common.AzureMissingResourceHttpError as ex:
raise FileNotFoundFromStorageError(key) from ex
def store_file_from_memory(self, key, memstring, metadata=None):
key = self.format_key_for_backend(key, remove_slash_prefix=True)
self.conn.create_blob_from_bytes(self.container_name, key, memstring,
metadata=self.sanitize_metadata(metadata, replace_hyphen_with="_"))
def store_file_from_disk(self, key, filepath, metadata=None, multipart=None):
key = self.format_key_for_backend(key, remove_slash_prefix=True)
self.conn.create_blob_from_path(self.container_name, key, filepath,
metadata=self.sanitize_metadata(metadata, replace_hyphen_with="_"))
def get_or_create_container(self, container_name):
start_time = time.monotonic()
self.conn.create_container(container_name)
self.log.debug("Got/Created container: %r successfully, took: %.3fs",
container_name, time.monotonic() - start_time)
return container_name
#!/usr/bin/python
import sys,os
from azure.storage.blob import BlockBlobService
from azure.storage.blob import ContentSettings
block_blob_service = BlockBlobService(account_name=str(sys.argv[1]), account_key=str(sys.argv[2]))
block_blob_service.create_container('keys')
block_blob_service.create_blob_from_path(
'keys',
str(sys.argv[3]),
os.path.join(os.getcwd(),str(sys.argv[3])),
content_settings=ContentSettings(content_type='text')
)
