def __init__(self, test_method):
super(TestBatchNCJLive, self).__init__(__file__, test_method)
self.account_name = 'test1'
if not self.playback:
self.account_key = os.environ['AZURE_BATCH_ACCESS_KEY']
else:
self.account_key = 'ZmFrZV9hY29jdW50X2tleQ=='
self.account_endpoint = 'https://test1.westus.batch.azure.com/'
storage_account = 'testaccountforbatch'
if not self.playback:
storage_key = os.environ['AZURE_STORAGE_ACCESS_KEY']
else:
storage_key = '1234'
self.blob_client = CloudStorageAccount(storage_account, storage_key)\
.create_block_blob_service()
credentials = batchauth.SharedKeyCredentials(self.account_name,
self.account_key)
self.batch_client = batch.BatchServiceClient(
credentials, base_url=self.account_endpoint)
self.output_blob_container = 'aaatestcontainer'
sas_token = self.blob_client.generate_container_shared_access_signature(
self.output_blob_container,
permission=BlobPermissions(read=True, write=True),
start=datetime.datetime.utcnow(),
expiry=datetime.datetime.utcnow() + datetime.timedelta(days=1))
self.output_container_sas = 'https://{}.blob.core.windows.net/{}?{}'.format(
storage_account, self.output_blob_container, sas_token)
print('Full container sas: {}'.format(self.output_container_sas))
def get_file_sas_url(
container: str,
name: str,
account_id: Optional[str] = None,
read: bool = False,
add: bool = False,
create: bool = False,
write: bool = False,
delete: bool = False,
list: bool = False,
days: int = 30,
hours: int = 0,
minutes: int = 0,
) -> str:
service = get_blob_service(account_id)
expiry = datetime.datetime.utcnow() + datetime.timedelta(
days=days, hours=hours, minutes=minutes
)
permission = BlobPermissions(read, add, create, write, delete, list)
sas_token = service.generate_blob_shared_access_signature(
container, name, permission=permission, expiry=expiry
)
url = service.make_blob_url(container, name, sas_token=sas_token)
return str(url)
def get_file_sas_url(
container: Container,
name: str,
storage_type: StorageType,
*,
read: bool = False,
add: bool = False,
create: bool = False,
write: bool = False,
delete: bool = False,
list: bool = False,
days: int = 30,
hours: int = 0,
minutes: int = 0,
) -> str:
service = get_service_by_container(container, storage_type)
if not service:
raise Exception("unable to find container: %s - %s" %
(container, storage_type))
expiry = datetime.datetime.utcnow() + datetime.timedelta(
days=days, hours=hours, minutes=minutes)
permission = BlobPermissions(read, add, create, write, delete, list)
sas_token = service.generate_blob_shared_access_signature(
container, name, permission=permission, expiry=expiry)
url = service.make_blob_url(container, name, sas_token=sas_token)
return str(url)
def main(req: func.HttpRequest) -> func.HttpResponse:
logging.info('Python HTTP trigger function processed a request.')
# checking for a POST request.
if (req.method == "POST"):
try:
# check JSON body request that carries the encoded image base64 info
req_body = req.get_json()
logging.info(req_body)
image_name = req_body['name']
image = req_body['image'].replace(' ', '+')
#decoding base64 image from json body
decoded_image = base64.b64decode(image)
logging.info(decoded_image)
# Storage connection string
connect_str = os.getenv('StorageConnStr')
accuntName = os.getenv('StorageName')
accuntKey = os.getenv('StorageConnectionKey')
BlobPermissions(read=False,
add=False,
create=False,
write=False,
delete=False,
_str=None)
logging.info("Successful connection to blob storage.")
#upload to picture to blob storage
block_blob_service = BlockBlobService(account_name=accuntName,
account_key=accuntKey)
container_name = 'machineimages'
blob_name = image_name + '.jpeg'
# Creating the blob
block_blob_service.create_blob_from_bytes(
container_name,
blob_name,
decoded_image,
content_settings=ContentSettings(content_type='image/png'))
logging.info("Successfull blob creating ")
# Returning a succesful post request
return func.HttpResponse(f"successful request")
except ValueError as e:
logging.error("Invalid json format " + str(e))
pass
except Exception as err:
logging.error("Something went wrong decoding json " + str(err))
pass
except AzureException as ae:
logging.error("Something went wrong with azure connection " +
str(ae))
pass
return func.HttpResponse(
"Please pass a name on the query string or in the request body",
status_code=400)
def get_token(self,
container_name,
file_name,
read=True,
create=False,
write=False):
# noinspection PyTypeChecker
return self.blob_service.generate_blob_shared_access_signature(
container_name, file_name,
BlobPermissions(read=read, create=create, write=write),
datetime.utcnow() + timedelta(hours=20))
def get_signed_url(self, sample_url) -> str:
container_name, blob_name = self._parser_url(sample_url)
signature = self.service.generate_blob_shared_access_signature(
container_name,
blob_name,
permission=BlobPermissions(read=True),
expiry=datetime.datetime.utcnow() + datetime.timedelta(minutes=15),
)
url = self.service.make_blob_url(container_name,
blob_name,
protocol='https',
sas_token=signature)
return url
def main(req: func.HttpRequest) -> func.HttpResponse:
logging.info('Python HTTP trigger function processed a request.')
# Check for a DELETE request.
if (req.method == "DELETE"):
try:
# check url parameter for machineID
machineID = req.params.get('MachineID')
logging.info("Getting machine id from url.")
logging.debug(machineID)
# storage connection string
connect_str = os.getenv('StorageConnStr')
accuntName = os.getenv('StorageName')
accuntKey = os.getenv('StorageConnectionKey')
block_blob_service = BlockBlobService(account_name=accuntName,
account_key=accuntKey)
logging.info("Successful connection to blob storage.")
# blob name
blobName = 'machineimages'
BlobPermissions(read=False,
add=False,
create=False,
write=False,
delete=True,
_str=None)
# delete an image from storage
block_blob_service.delete_blob(blobName, machineID)
logging.info('Image deleted successfully')
# clean up
connect_str.close()
logging.info("Machine successfully deleted")
return func.HttpResponse(f"Image successfully deleted")
except ValueError as e:
logging.error("Invalid json format " + str(e))
logging.info("Storage account connection failed")
pass
finally:
logging.info("Image successfuly deleted")
logging.info("Closing connection to the storage...")
# returns a Http 400 status bad request.
return func.HttpResponse(
"Please pass a DELETE request on the query string or in the request body",
status_code=400)
def generate_blob_url_with_sas(blob):
start = timezone.now()
expiry = timezone.now() + timezone.timedelta(seconds=15)
block_blob_service = BlockBlobService(
account_name=settings.AZURE_ACCOUNT_NAME,
account_key=settings.AZURE_ACCOUNT_KEY)
sas_token = block_blob_service.generate_blob_shared_access_signature(
settings.AZURE_CONTAINER,
blob,
permission=BlobPermissions(read=True),
expiry=expiry,
start=start)
final_url = f"https://{settings.AZURE_ACCOUNT_NAME}.blob.core.windows.net/{settings.AZURE_CONTAINER}/{blob}?{sas_token}"
return final_url
def generate_blob_sas_token(self,
container_name,
blob_name,
read=True,
add=True,
create=True,
write=True,
delete=False,
days_to_keep=1):
permission = BlobPermissions(read, add, create, write, delete)
return self.__blob_access_signature.generate_blob(
container_name,
blob_name,
permission=permission,
protocol='https',
expiry=datetime.now() + timedelta(days=days_to_keep))
def post_blob(container_name, blob_name):
try:
account_name, account_key = get_auth(request.authorization)
file_service = BlockBlobService(account_name=account_name,
account_key=account_key)
start_timedelta, expiry_timedelta = get_sas_params(request.args)
if request.headers.get('Transfer-Encoding') == 'chunked':
file_service.create_file_from_stream(container_name,
directory_name,
file_name,
request.stream,
count=4096)
else:
file_service.create_blob_from_bytes(container_name=container_name,
blob_name=blob_name,
blob=request.get_data())
sas_token = file_service.generate_blob_shared_access_signature(
container_name=container_name,
blob_name=blob_name,
permission=BlobPermissions(read=True),
expiry=datetime.now() + expiry_timedelta,
start=start_timedelta,
id=None,
ip=None,
protocol='https',
cache_control=request.headers.get('Cache-Control'),
content_disposition=request.headers.get(
'Content-Disposition: attachment;'),
content_encoding=request.headers.get('Content-Encoding'),
content_language=request.headers.get('Content-Language'),
content_type=request.headers.get('Content-Type'))
url = file_service.make_blob_url(container_name,
blob_name,
protocol='https',
sas_token=sas_token)
return Response(response=url + "",
status=200,
content_type='text/plain')
except Exception as e:
logger.exception(e)
return abort(500, e)
def __init__(self, test_method):
super(TestBatchExtensionsLive, self).__init__(__file__, test_method)
if self.playback:
self.account_name = 'sdktest2'
self.account_endpoint = 'https://sdktest2.westcentralus.batch.azure.com'
self.account_key = 'abc=='
self.subscription_id = "677f962b-9abf-4423-a27b-0c2f4094dcec"
storage_account = 'sdkteststore2'
storage_key = 'abc=='
else:
self.account_name = os.environ.get('AZURE_BATCH_ACCOUNT', 'test1')
self.account_endpoint = os.environ.get(
'AZURE_BATCH_ENDPOINT',
'https://test1.westus.batch.azure.com/')
self.account_key = os.environ['AZURE_BATCH_ACCESS_KEY']
self.subscription_id = os.environ.get(
'AZURE_BATCH_SUBSCRIPTION_ID',
"677f962b-9abf-4423-a27b-0c2f4094dcec")
storage_account = os.environ.get('AZURE_STORAGE_ACCOUNT',
'testaccountforbatch')
storage_key = os.environ.get('AZURE_STORAGE_ACCESS_KEY', 'abc==')
self.data_dir = os.path.join(os.path.dirname(__file__), 'data')
self.blob_client = CloudStorageAccount(storage_account, storage_key)\
.create_block_blob_service()
credentials = batchauth.SharedKeyCredentials(self.account_name,
self.account_key)
self.batch_client = batch.BatchExtensionsClient(
credentials,
batch_url=self.account_endpoint,
subscription_id=self.subscription_id,
batch_account=self.account_name)
self.output_blob_container = 'aaatestcontainer'
sas_token = self.blob_client.generate_container_shared_access_signature(
container_name=self.output_blob_container,
permission=BlobPermissions(read=True, write=True),
start=datetime.datetime.utcnow(),
expiry=datetime.datetime.utcnow() + datetime.timedelta(days=1))
self.output_container_sas = 'https://{}.blob.core.windows.net/{}?{}'.format(
storage_account, self.output_blob_container, sas_token)
print('Full container sas: {}'.format(self.output_container_sas))
def _generate_azure_signed_url(self, obj_name, expiry_timestamp):
""" Generates a signed URL compatible with Azure Blob Storage
"""
# libcloud encodes the access key, so it needs decoding
secret_as_string = base64.b64encode(self.driver.secret).decode()
# azure expiry times need to be ISO8601 (no milliseconds, and with the timezone character)
expires = dt.datetime.utcfromtimestamp(expiry_timestamp).replace(microsecond=0).isoformat() + 'Z'
blob_service = BlockBlobService(self.driver.key, secret_as_string)
signed_url = blob_service.make_blob_url(
self.container.name,
obj_name,
sas_token=blob_service.generate_blob_shared_access_signature(
self.container.name,
obj_name,
permission=BlobPermissions(read=True),
expiry=expires))
return signed_url
def ml_subject_assistant_export_to_microsoft_pt3_create_shareable_azure_blob(
source_filepath,
target_filename,
):
print('[Subject Assistant] Exporting to Microsoft 3/4: create shareable Azure blob')
shareable_file_url = ''
try:
block_blob_service = BlockBlobService(account_name=settings.SUBJECT_ASSISTANT_AZURE_ACCOUNT_NAME, account_key=settings.SUBJECT_ASSISTANT_AZURE_ACCOUNT_KEY)
created_blob = block_blob_service.create_blob_from_path(settings.SUBJECT_ASSISTANT_AZURE_CONTAINER_NAME, target_filename, source_filepath)
blob_permissions = BlobPermissions(read=True)
sas_expiry = (datetime.now() + timedelta(days=30)).strftime('%Y-%m-%d')
generated_sas = block_blob_service.generate_blob_shared_access_signature(
container_name=settings.SUBJECT_ASSISTANT_AZURE_CONTAINER_NAME,
blob_name=target_filename,
permission=blob_permissions,
expiry=sas_expiry
)
shareable_file_url = 'https://{}.blob.core.windows.net/{}/{}?{}'.format(
settings.SUBJECT_ASSISTANT_AZURE_ACCOUNT_NAME,
settings.SUBJECT_ASSISTANT_AZURE_CONTAINER_NAME,
target_filename,
generated_sas
)
except Exception as err:
print('[ERROR] ', err)
raise err
return shareable_file_url
def OFF_test_generate_account(self):
a = self.bs.generate_blob_shared_access_signature(
"tests", "Animal.mp4", BlobPermissions(read=True))
print(a)
# Main method.
if __name__ == '__main__':
account_name = sys.argv[1]
account_key = sys.argv[2]
container_name = sys.argv[3]
blob_name = sys.argv[4]
if sys.argv[5] == 'r':
permission = BlobPermissions.READ
elif sys.argv[5] == 'c':
permission = BlobPermissions.CREATE
elif sys.argv[5] == 'w':
permission = BlobPermissions.WRITE
elif sys.argv[5] == 's':
permission = BlobPermissions(read=True, create=True)
else:
raise Exception('Illegal argument: ' + sys.argv[5])
minutes = int(sys.argv[6])
sas_token, sas_url = generate_blob_sas(
account_name,
account_key,
container_name,
blob_name,
permission,
minutes,
)
print('sas token =', sas_token)
def generate_write_access_key(storage_config):
permission = BlobPermissions(read=True, write=True, add=True, create=True)
expiry = datetime.datetime.utcnow() + datetime.timedelta(hours=1)
return AzureBlobStorageClient.__generate_access_key(permission, expiry, storage_config)
bbs = BlockBlobService(account_name=args.storage_account,
account_key=args.storage_key)
train_client = CustomVisionTrainingClient(args.cv_train_key,
endpoint=args.cv_endpoint)
limit = 63
counter = 0
image_block = []
for blob_name in bbs.list_blob_names(args.storage_container):
if (counter < limit):
print("attempting blob: {}".format(blob_name))
now = datetime.utcnow()
permission = BlobPermissions(read=True)
sas_token = bbs.generate_blob_shared_access_signature(
container_name=args.storage_container,
blob_name=blob_name,
start=now,
expiry=now + timedelta(minutes=15),
permission=permission)
sas_url = bbs.make_blob_url(container_name=args.storage_container,
blob_name=blob_name,
sas_token=sas_token)
print(sas_url)
image_entry = ImageUrlCreateEntry(url=sas_url)
image_block.append(image_entry)
counter += 1
else:
print("attempting upload...")