def validate_azure_connection_read_write(self, bucket_name):
test_file_path = 'diffgram_test_file.txt'
log = regular_log.default()
try:
blob_client = self.connection_client.get_blob_client(container = bucket_name, blob = test_file_path)
my_content_settings = ContentSettings(content_type = 'text/plain')
blob_client.upload_blob('This is a diffgram test file', content_settings = my_content_settings, overwrite=True)
except Exception as e:
log['error']['azure_write_perms'] = 'Error Connecting to Azure: Please check you have write permissions on the Azure container.'
log['error']['details'] = traceback.format_exc()
return False, log
try:
shared_access_signature = BlobSharedAccessSignature(
account_name = self.connection_client.account_name,
account_key = self.connection_client.credential.account_key
)
expiration_offset = 40368000
added_seconds = datetime.timedelta(0, expiration_offset)
expiry_time = datetime.datetime.utcnow() + added_seconds
filename = test_file_path.split("/")[-1]
sas = shared_access_signature.generate_blob(
container_name = bucket_name,
blob_name = test_file_path,
start = datetime.datetime.utcnow(),
expiry = expiry_time,
permission = BlobSasPermissions(read = True),
content_disposition = 'attachment; filename=' + filename,
)
sas_url = 'https://{}.blob.core.windows.net/{}/{}?{}'.format(
self.connection_client.account_name,
bucket_name,
test_file_path,
sas
)
resp = requests.get(sas_url)
if resp.status_code != 200:
raise Exception(
'Error when accessing presigned URL: Status({}). Error: {}'.format(resp.status_code, resp.text))
except:
log['error']['azure_write_perms'] = 'Error Connecting to Azure: Please check you have read permissions on the Azure container.'
log['error']['details'] = traceback.format_exc()
return False, log
return True, log
def get_blob_url_sas(blob_service_client, file_url):
try:
(blob_account_name, blob_container,
blob_name) = parse_blob_information(file_url)
logging.info(
f'source blob: {blob_account_name}/{blob_container}/{blob_name}')
delegation_key = blob_service_client.get_user_delegation_key(
key_start_time=datetime.utcnow(),
key_expiry_time=datetime.utcnow() + timedelta(hours=1))
blob_sas = BlobSharedAccessSignature(
account_name=blob_account_name, user_delegation_key=delegation_key)
sas_token = blob_sas.generate_blob(
container_name=blob_container,
blob_name=blob_name,
permission=AccountSasPermissions(read=True),
expiry=datetime.utcnow() + timedelta(hours=1))
return f'{compose_blob_url(file_url)}?{sas_token}'
except Exception as ex:
logging.error('failed to get SAS for blob: ' + str(ex))
raise
def build_secure_url(self, blob_name, expiration_offset = None, bucket = "web"):
"""
blob_name, string of blob name. don't include CLOUD_STORAGE_BUCKET
expiration_offset, integer, additional time from this moment to allow link
"""
container = None
if bucket == 'web':
container = self.azure_container_name
elif bucket == 'ml':
container = self.azure_container_name_ml
else:
raise Exception('Invalid bucket type provided. Must be either "web" or "ml"')
shared_access_signature = BlobSharedAccessSignature(
account_name = self.azure_service_client.account_name,
account_key = self.azure_service_client.credential.account_key
)
if expiration_offset is None:
expiration_offset = 40368000
added_seconds = datetime.timedelta(0, expiration_offset)
expiry_time = datetime.datetime.utcnow() + added_seconds
filename = blob_name.split("/")[-1]
sas = shared_access_signature.generate_blob(
container_name = container,
blob_name = blob_name,
start = datetime.datetime.utcnow(),
expiry = expiry_time,
permission = BlobSasPermissions(read = True),
content_disposition = 'attachment; filename=' + filename,
)
sas_url = 'https://{}.blob.core.windows.net/{}/{}?{}'.format(
self.azure_service_client.account_name,
container,
blob_name,
sas
)
return sas_url
def __fetch_object(self, opts):
"""
Upload a file to Diffgram from an Azure Blob
:param opts: Dictionary with parameters for object fetching.
:return: file obj if file was uploaded, else False
"""
spec_list = [{'bucket_name': str, 'path': str}]
log = regular_log.default()
log, input = regular_input.input_check_many(untrusted_input = opts,
spec_list = spec_list,
log = log)
if len(log["error"].keys()) >= 1:
return {'log': log}
shared_access_signature = BlobSharedAccessSignature(
account_name = self.connection_client.account_name,
account_key = self.connection_client.credential.account_key
)
expiration_offset = 40368000
blob_name = opts['path']
container = opts['bucket_name']
added_seconds = datetime.timedelta(0, expiration_offset)
expiry_time = datetime.datetime.utcnow() + added_seconds
filename = blob_name.split("/")[-1]
sas = shared_access_signature.generate_blob(
container_name = container,
blob_name = blob_name,
start = datetime.datetime.utcnow(),
expiry = expiry_time,
permission = BlobSasPermissions(read = True),
content_disposition = 'attachment; filename=' + filename,
)
sas_url = 'https://{}.blob.core.windows.net/{}/{}?{}'.format(
self.connection_client.account_name,
container,
blob_name,
sas
)
with sessionMaker.session_scope() as session:
project = Project.get_by_string_id(session, self.config_data.get('project_string_id'))
member = session.query(Member).filter(Member.user_id == opts['event_data']['request_user']).first()
# Deduct Media Type:
extension = Path(opts['path']).suffix
extension = extension.lower()
media_type = None
if extension in images_allowed_file_names:
media_type = 'image'
elif extension in videos_allowed_file_names:
media_type = 'video'
else:
# TODO: Decide, do we want to raise an exception? or just do nothing?
log = regular_log.default()
log['error']['invalid_type'] = 'File must type of: {} {}'.format(str(images_allowed_file_names),
str(videos_allowed_file_names))
log['error']['file_name'] = opts['path']
log['opts'] = opts
Event.new(
session = session,
member_id = opts['event_data']['request_user'],
kind = 'microsoft_azure_new_import_warning',
description = 'Skipped import for {}, invalid file type.'.format(opts['path']),
error_log = log,
project_id = project.id,
member = member,
success = False
)
return None
# metadata = self.connection_client.head_object(Bucket=self.config_data['bucket_name'], Key=path)
created_input = packet.enqueue_packet(self.config_data['project_string_id'],
session = session,
media_url = sas_url,
media_type = media_type,
job_id = opts.get('job_id'),
batch_id = opts.get('batch_id'),
file_name = opts.get('path'),
video_split_duration = opts.get('video_split_duration'),
directory_id = opts.get('directory_id'),
extract_labels_from_batch = True)
log = regular_log.default()
log['opts'] = opts
Event.new(
session = session,
member_id = opts['event_data']['request_user'],
kind = 'microsoft_azure_new_import_success',
description = 'New cloud import for {}'.format(opts['path']),
error_log = opts,
project_id = project.id,
member = member,
success = True
)
return created_input
def validate_azure_connection(self):
connection_string = self.azure_connection_string
bucket_name = self.bucket_name
test_file_path = 'diffgram_test_file.txt'
client = None
bcolors.printcolor('Testing Connection...', bcolors.OKBLUE)
try:
client = BlobServiceClient.from_connection_string(connection_string)
print(bcolors.OKGREEN + '[OK] ' + '\033[0m' + 'Connection To Azure Account')
except Exception as e:
print(bcolors.FAIL + '[ERROR] ' + '\033[0m' + 'Connection To Azure Account')
bcolors.printcolor('Error Connecting to Azure: Please check you entered valid credentials.', bcolors.FAIL)
print('Details: {}'.format(traceback.format_exc()))
bcolors.printcolor('Please update credentials and try again', bcolors.OKBLUE)
return False
time.sleep(0.5)
try:
blob_client = client.get_blob_client(container = bucket_name, blob = test_file_path)
my_content_settings = ContentSettings(content_type = 'text/plain')
blob_client.upload_blob('This is a diffgram test file', content_settings = my_content_settings, overwrite=True)
print(bcolors.OKGREEN + '[OK] ' + '\033[0m' + 'Write Permissions')
except:
print(bcolors.FAIL + '[ERROR] ' + '\033[0m' + 'Write Permissions')
bcolors.printcolor('Error Connecting to Azure: Please check you have write permissions on the Azure container.',
bcolors.FAIL)
print('Details: {}'.format(traceback.format_exc()))
bcolors.printcolor('Please update permissions and try again', bcolors.OKBLUE)
return False
time.sleep(0.5)
try:
shared_access_signature = BlobSharedAccessSignature(
account_name = client.account_name,
account_key = client.credential.account_key
)
expiration_offset = 40368000
added_seconds = datetime.timedelta(0, expiration_offset)
expiry_time = datetime.datetime.utcnow() + added_seconds
filename = test_file_path.split("/")[-1]
sas = shared_access_signature.generate_blob(
container_name = bucket_name,
blob_name = test_file_path,
start = datetime.datetime.utcnow(),
expiry = expiry_time,
permission = BlobSasPermissions(read = True),
content_disposition = 'attachment; filename=' + filename,
)
sas_url = 'https://{}.blob.core.windows.net/{}/{}?{}'.format(
client.account_name,
bucket_name,
test_file_path,
sas
)
resp = requests.get(sas_url)
if resp.status_code != 200:
raise Exception(
'Error when accessing presigned URL: Status({}). Error: {}'.format(resp.status_code, resp.text))
print(bcolors.OKGREEN + '[OK] ' + '\033[0m' + 'Read Permissions')
except:
print(bcolors.FAIL + '[ERROR] ' + '\033[0m' + 'Read Permissions')
bcolors.printcolor('Error Connecting to Azure: Please check you have read permissions on the Azure container.',
bcolors.FAIL)
print('Details: {}'.format(traceback.format_exc()))
bcolors.printcolor('Please update permissions and try again', bcolors.OKBLUE)
return False
time.sleep(0.5)
bcolors.printcolor('Connection to Azure Successful!', bcolors.OKGREEN)
return True