def _save(self, name, content):
"""
Use the Azure Storage service to write ``content`` to a remote file
(called ``name``).
"""
content.open()
content_type = None
if hasattr(content.file, 'content_type'):
content_type = content.file.content_type
else:
content_type = mimetypes.guess_type(name)[0]
cache_control = self.get_cache_control(self.container, name,
content_type)
self._get_service().create_blob_from_stream(
container_name=self.container,
blob_name=name,
stream=content,
content_settings=ContentSettings(
content_type=content_type,
cache_control=cache_control,
),
)
content.close()
return name
def _azure_3_save(self, name, content_type, content_data):
content_type = ContentSettings(content_type=content_type)
self.connection.create_blob_from_bytes(self.azure_container,
name,
content_data,
content_settings=content_type,
timeout=self.azure_timeout)
def pdf_upload(self, debug):
'''
@function: upload pdf file.
@args:
1. debug : debug is "False" the pdf file will be deleted.
debug is "True" the pdf file will not be deleted.
'''
aes_pdfname = aes.encrypt_str(
self.pdf_config_object.pdf_company) + "/" + aes.encrypt_str(
self.pdf_config_object.pdf_property) + "/" + self.pdf_name
self.pdf_url = self.pdf_config_object.pdf_url + aes_pdfname
try:
blob_service = BlockBlobService(
self.pdf_config_object.account_name,
self.pdf_config_object.account_key)
blob_service.create_blob_from_path(
self.pdf_config_object.pdf_upload_path_name,
aes_pdfname,
self.pdf_config_object.pdf_path + "/" + self.pdf_name,
content_settings=ContentSettings(
content_type='application/pdf'))
if debug == "False":
self.pdf_clean()
return self.pdf_url
except Exception as e:
raise e
return None
def put_file_into_storage(self, dirName, fileName):
"""
Ship the outdated log file to the specified blob container.
"""
if not self.container_created:
self.service.create_container(self.container)
self.container_created = True
fd, tmpfile_path = None, ''
try:
file_path = os.path.join(dirName, fileName)
if self.zip_compression:
suffix, content_type = '.zip', 'application/zip'
fd, tmpfile_path = mkstemp(suffix=suffix)
with os.fdopen(fd, 'wb') as f:
with ZipFile(f, 'w', ZIP_DEFLATED) as z:
z.write(file_path, arcname=fileName)
file_path = tmpfile_path
else:
suffix, content_type = '', 'text/plain'
self.service.create_blob_from_path(container_name=self.container,
blob_name=fileName+suffix,
file_path=fileName,
content_settings=ContentSettings(content_type=content_type),
max_connections=self.max_connections
) # max_retries and retry_wait no longer arguments in azure 0.33
finally:
if self.zip_compression and fd:
os.remove(tmpfile_path)
def blob_service_create_blob_from_bytes(account_name, storage_key,
container_name, blob_name, blob):
content_settings = ContentSettings(content_type='image/jpeg')
block_blob_service = get_block_blob_service(account_name, storage_key)
block_blob_service.create_blob_from_bytes(
container_name, blob_name, blob, content_settings=content_settings)
def upload_blob(self):
content_settings = None
if self.content_type or self.content_encoding or self.content_language or self.content_disposition or \
self.cache_control or self.content_md5:
content_settings = ContentSettings(
content_type=self.content_type,
content_encoding=self.content_encoding,
content_language=self.content_language,
content_disposition=self.content_disposition,
cache_control=self.cache_control,
content_md5=self.content_md5)
if not self.check_mode:
try:
self.blob_client.create_blob_from_path(
self.container,
self.blob,
self.src,
metadata=self.tags,
content_settings=content_settings)
except AzureHttpError as exc:
self.fail("Error creating blob {0} - {1}".format(
self.blob, str(exc)))
self.blob_obj = self.get_blob()
self.results['changed'] = True
self.results['actions'].append('created blob {0} from {1}'.format(
self.blob, self.src))
self.results['container'] = self.container_obj
self.results['blob'] = self.blob_obj
def update_blob_content_settings(self):
content_settings = ContentSettings(
content_type=self.content_type,
content_encoding=self.content_encoding,
content_language=self.content_language,
content_disposition=self.content_disposition,
cache_control=self.cache_control,
content_md5=self.content_md5)
if not self.check_mode:
try:
self.blob_client.set_blob_properties(
self.container,
self.blob,
content_settings=content_settings)
except AzureHttpError as exc:
self.fail("Update blob content settings {0}:{1} - {2}".format(
self.container, self.blob, str(exc)))
self.blob_obj = self.get_blob()
self.results['changed'] = True
self.results['actions'].append(
"updated blob {0}:{1} content settings.".format(
self.container, self.blob))
self.results['container'] = self.container_obj
self.results['blob'] = self.blob_obj
def upload(self, id, max_size=10):
"""
Complete the file upload, or clear an existing upload.
:param id: The resource_id.
:param max_size: Ignored.
"""
if self.filename:
if self.can_use_advanced_azure:
from azure.storage import blob as azure_blob
from azure.storage.blob.models import ContentSettings
blob_service = azure_blob.BlockBlobService(
self.driver_options['key'],
self.driver_options['secret']
)
content_settings = None
if self.guess_mimetype:
content_type, _ = mimetypes.guess_type(self.filename)
if content_type:
content_settings = ContentSettings(
content_type=content_type
)
return blob_service.create_blob_from_stream(
container_name=self.container_name,
blob_name=self.path_from_filename(
id,
self.filename
),
stream=self.file_upload,
content_settings=content_settings
)
else:
self.container.upload_object_via_stream(
self.file_upload,
object_name=self.path_from_filename(
id,
self.filename
)
)
elif self._clear and self.old_filename and not self.leave_files:
# This is only set when a previously-uploaded file is replace
# by a link. We want to delete the previously-uploaded file.
try:
self.container.delete_object(
self.container.get_object(
self.path_from_filename(
id,
self.old_filename
)
)
)
except ObjectDoesNotExistError:
# It's possible for the object to have already been deleted, or
# for it to not yet exist in a committed state due to an
# outstanding lease.
return
def _put_file(self, key, file):
from azure.storage.blob.models import ContentSettings
if self.checksum:
content_settings = ContentSettings(content_md5=_file_md5(file))
else:
content_settings = ContentSettings()
with map_azure_exceptions(key=key):
self.block_blob_service.create_blob_from_stream(
container_name=self.container,
blob_name=key,
stream=file,
max_connections=self.max_connections,
content_settings=content_settings,
)
return key
def _put(self, key, data):
from azure.storage.blob.models import ContentSettings
if self.checksum:
content_settings = ContentSettings(content_md5=_byte_buffer_md5(data))
else:
content_settings = ContentSettings()
with map_azure_exceptions(key=key):
self.block_blob_service.create_blob_from_bytes(
container_name=self.container,
blob_name=key,
blob=data,
max_connections=self.max_connections,
content_settings=content_settings,
)
return key
def upload_blob(
self,
container: Container,
filename: FileLike,
blob_name: str = None,
acl: str = None,
meta_data: MetaData = None,
content_type: str = None,
content_disposition: str = None,
cache_control: str = None,
chunk_size: int = 1024,
extra: ExtraOptions = None,
) -> Blob:
if acl:
logger.info(messages.OPTION_NOT_SUPPORTED, "acl")
meta_data = {} if meta_data is None else meta_data
extra = extra if extra is not None else {}
extra_args = self._normalize_parameters(extra, self._PUT_OBJECT_KEYS)
extra_args.setdefault("content_type", content_type)
extra_args.setdefault("content_disposition", content_disposition)
extra_args.setdefault("cache_control", cache_control)
azure_container = self._get_azure_container(container.name)
blob_name = blob_name or validate_file_or_path(filename)
# azure does not set content_md5 on backend
file_hash = file_checksum(filename, hash_type=self.hash_type)
file_digest = file_hash.digest()
checksum = base64.b64encode(file_digest).decode("utf-8").strip()
extra_args.setdefault("content_md5", checksum)
content_settings = ContentSettings(**extra_args)
if isinstance(filename, str):
self.service.create_blob_from_path(
container_name=azure_container.name,
blob_name=blob_name,
file_path=filename,
content_settings=content_settings,
metadata=meta_data,
validate_content=True,
)
else:
self.service.create_blob_from_stream(
container_name=azure_container.name,
blob_name=blob_name,
stream=filename,
content_settings=content_settings,
metadata=meta_data,
validate_content=True,
)
azure_blob = self._get_azure_blob(azure_container.name, blob_name)
return self._convert_azure_blob(container, azure_blob)
def uri_put_file(creds, uri, fp, content_type=None):
assert fp.tell() == 0
assert uri.startswith('wabs://')
def log_upload_failures_on_error(exc_tup, exc_processor_cxt):
def standard_detail_message(prefix=''):
return (prefix + ' There have been {n} attempts to upload '
'file {url} so far.'.format(n=exc_processor_cxt, url=uri))
typ, value, tb = exc_tup
del exc_tup
# Screen for certain kinds of known-errors to retry from
if issubclass(typ, socket.error):
socketmsg = value[1] if isinstance(value, tuple) else value
logger.info(
msg='Retrying upload because of a socket error',
detail=standard_detail_message(
"The socket error's message is '{0}'.".format(socketmsg)))
else:
# For all otherwise untreated exceptions, report them as a
# warning and retry anyway -- all exceptions that can be
# justified should be treated and have error messages
# listed.
logger.warning(
msg='retrying file upload from unexpected exception',
detail=standard_detail_message(
'The exception type is {etype} and its value is '
'{evalue} and its traceback is {etraceback}'.format(
etype=typ,
evalue=value,
etraceback=''.join(traceback.format_tb(tb)))))
# Help Python GC by resolving possible cycles
del tb
url_tup = urlparse(uri)
kwargs = dict(content_settings=ContentSettings(content_type),
validate_content=True)
conn = BlockBlobService(creds.account_name,
creds.account_key,
sas_token=creds.access_token,
protocol='https')
conn.create_blob_from_bytes(url_tup.netloc, url_tup.path.lstrip('/'),
fp.read(), **kwargs)
# To maintain consistency with the S3 version of this function we must
# return an object with a certain set of attributes. Currently, that set
# of attributes consists of only 'size'
return _Key(size=fp.tell())
def __sendFrameForStorage(self, frame):
try:
blobname = time.strftime("%Y%m%d-%H%M%S.jpg")
result = self.block_blob_service.create_blob_from_bytes(
self.imageStorageEndpoint["containername"],
blobname,
frame,
content_settings=ContentSettings('image/jpg'))
if self.verbose:
print("Stored blob name [" + blobname + "]")
except Exception as e:
print("Store blob failed with: " + str(e))
return result
def _guess_content_type(file_path, original):
if original.content_encoding or original.content_type:
return original
mimetypes.add_type('application/json', '.json')
mimetypes.add_type('application/javascript', '.js')
mimetypes.add_type('application/wasm', '.wasm')
content_type, v = mimetypes.guess_type(file_path)
return ContentSettings(
content_type=content_type,
content_disposition=original.content_disposition,
content_language=original.content_language,
content_md5=original.content_md5,
cache_control=original.cache_control)
def uploadImageToBlob(block_blob_service, container_name, local_file_name,
full_path_to_file, image_metadata):
if block_blob_service.exists(container_name) == False:
block_blob_service.create_container(container_name)
# Set the permission so the blobs are public.
block_blob_service.set_container_acl(
container_name, public_access=PublicAccess.Container)
block_blob_service.create_blob_from_path(
container_name,
local_file_name,
full_path_to_file,
content_settings=ContentSettings(content_type='application/png'),
metadata=image_metadata)
def _save(self, name, content):
if hasattr(content.file, 'content_type'):
content_type = content.file.content_type
else:
content_type = mimetypes.guess_type(name)[0]
if hasattr(content, 'chunks'):
content_data = b''.join(chunk for chunk in content.chunks())
else:
content_data = content.read()
self.block_blob_service.create_blob_from_bytes(
self.azure_container,
name,
content_data,
content_settings=ContentSettings(content_type=content_type))
return name
def zzztest_storage_account_creation(self): #!!! need to update
from azure.common.credentials import UserPassCredentials
from azure.mgmt.resource import ResourceManagementClient
from azure.mgmt.storage import StorageManagementClient
from azure.storage import CloudStorageAccount
from azure.storage.blob.models import ContentSettings
import getpass
username, subscription_id = [
s.strip()
for s in open(os.path.expanduser("~") +
"/azurebatch/account.txt").xreadlines()
]
print "Azure password"
password = getpass.getpass()
credentials = UserPassCredentials(username, password)
resource_client = ResourceManagementClient(credentials,
subscription_ids)
storage_client = StorageManagementClient(credentials, subscription_id)
resource_client.resource_groups.create_or_update(
'my_resource_group', {'location': 'westus'})
async_create = storage_client.storage_accounts.create(
'my_resource_group', 'my_storage_account', {
'location': 'westus',
'account_type': 'Standard_LRS'
})
async_create.wait()
storage_keys = storage_client.storage_accounts.list_keys(
'my_resource_group', 'my_storage_account')
storage_keys = {v.key_name: v.value for v in storage_keys.keys}
storage_client = CloudStorageAccount('my_storage_account',
storage_keys['key1'])
blob_service = storage_client.create_block_blob_service()
blob_service.create_container('my_container_name')
blob_service.create_blob_from_bytes(
'my_container_name',
'my_blob_name',
b'<center><h1>Hello World!</h1></center>',
content_settings=ContentSettings('text/html'))
print(blob_service.make_blob_url('my_container_name', 'my_blob_name'))
print('Downloading cloud %s/%s to local %s' %
(container_name, source_audio_name, source_audio_name))
blob_service.get_blob_to_path(container_name, source_audio_name,
source_audio_name)
print('Downloading cloud %s/%s to local %s' %
(container_name, source_video_name, source_video_name))
blob_service.get_blob_to_path(container_name, source_video_name,
source_video_name)
print('smash away here - just copying video file for now')
copyfile(source_video_name, smashed_name)
blob_service.create_blob_from_path(
container_name,
smashed_name,
smashed_name,
content_settings=ContentSettings(content_type=smashed_content_type))
print('URL for public download is https://%s.blob.core.windows.net/%s/%s' %
(storage_account_name, container_name, smashed_name))
print('Uploading local %s to cloud %s/%s' %
(smashed_name, container_name, smashed_name))
smash_video_audio(source_video_name, source_audio_name, smashed_name)
blob_service.create_blob_from_path(
container_name,
smashed_name,
smashed_name,
content_settings=ContentSettings(content_type=smashed_content_type))
print('URL for public download is https://%s.blob.core.windows.net/%s/%s' %
(storage_account_name, container_name, smashed_name))
def saveToBlob(block_blob_service, local_path,local_file_name,container_name):
# Create a file in Documents to test the upload and download.
full_path_to_file =os.path.join(local_path, local_file_name)
print("Temp file = " + full_path_to_file)
print("\nUploading to Blob storage as blob: " + local_file_name)
# Upload the created file, use local_file_name for the blob name
block_blob_service.create_blob_from_path(container_name, local_file_name, full_path_to_file, content_settings=ContentSettings(content_type='image/png'))
def upload_file(data_file, meta, encryption_metadata, max_concurrency):
logger = getLogger(__name__)
try:
azure_metadata = {
u'sfcdigest': meta[SHA256_DIGEST],
}
if (encryption_metadata):
azure_metadata.update({
u'encryptiondata':
json.dumps({
u'EncryptionMode': u'FullBlob',
u'WrappedContentKey': {
u'KeyId': u'symmKey1',
u'EncryptedKey': encryption_metadata.key,
u'Algorithm': u'AES_CBC_256'
},
u'EncryptionAgent': {
u'Protocol': '1.0',
u'EncryptionAlgorithm': u'AES_CBC_128',
},
u'ContentEncryptionIV': encryption_metadata.iv,
u'KeyWrappingMetadata': {
u'EncryptionLibrary': u'Java 5.3.0'
}
}),
u'matdesc':
encryption_metadata.matdesc
})
azure_location = SnowflakeAzureUtil.extract_container_name_and_path(
meta[u'stage_info'][u'location'])
path = azure_location.path + meta[u'dst_file_name'].lstrip('/')
client = meta[u'client']
callback = meta[u'put_callback'](
data_file,
os.path.getsize(data_file),
output_stream=meta[u'put_callback_output_stream']) if \
meta[u'put_callback'] else None
def azure_callback(current, total):
callback(current)
client.create_blob_from_path(
azure_location.container_name,
path,
data_file,
progress_callback=azure_callback
if meta[u'put_callback'] else None,
metadata=azure_metadata,
max_connections=max_concurrency,
content_settings=ContentSettings(
content_type=u'application/octet-stream',
content_encoding=u'utf-8',
))
logger.debug(u'DONE putting a file')
meta[u'dst_file_size'] = meta[u'upload_size']
meta[u'result_status'] = ResultStatus.UPLOADED
except AzureHttpError as err:
if (err.status_code == 403
and "Signature not valid in the specified time frame"
in err.message):
logger.debug(u"AZURE Token expired. Renew and retry")
meta[u'result_status'] = ResultStatus.RENEW_TOKEN
return None
else:
meta[u'last_error'] = err
meta[u'result_status'] = ResultStatus.NEED_RETRY
def main(inputDict: dict) -> str:
## Arguments
fileType = inputDict['fileType']
fileURL = inputDict["fileURL"]
container = inputDict["container"]
selector = inputDict["selector"]
logging.info(f"fileType: {fileType}")
logging.info(f"fileURL: {fileURL}")
logging.info(f"container: {container}")
logging.info(f"selector: {selector}")
## `selector` takes the format "XofY" or "ALL"
_all_ = selector == "ALL"
X = None
Y = None
if not _all_:
X = int(selector.split("of")[0])
Y = int(selector.split("of")[1])
logging.info("changes 12")
## Set file name to be used and get container it came from
fileName = unquote(fileURL.split("/")[-1]) # (contains file extension)
container = fileURL.split("/")[-2]
## Create bbs
bbs = BlockBlobService(
connection_string=os.getenv("fsevideosConnectionString"))
logging.info("bbs created")
## Create SAS URL
sasURL = get_SAS_URL(fileURL=fileURL,
block_blob_service=bbs,
container=container)
logging.info(f"sasURL: {sasURL}")
## Download blob to temporary location
tempClipFilePath = "/tmp/" + fileName
bbs.get_blob_to_path(container_name=container,
blob_name=fileName,
file_path=tempClipFilePath)
logging.info(f'file saved to "{tempClipFilePath}"')
## Set the size of the clips you want (in seconds)
chunk_length_secs = 3600
## Create clip
if fileType == "MP4":
clip = VideoFileClip(tempClipFilePath)
elif fileType == "MP3":
clip = AudioFileClip(tempClipFilePath)
else:
raise ValueError("wrong file type")
## Get number of chunks (files to be created)
logging.info(f"clip.duration: {clip.duration}")
chunk_count = ceil(clip.duration / chunk_length_secs)
## If not _all_, make sure Y is the expected number
if not _all_:
if chunk_count != Y:
vem = (f"Based on the selector passed ({selector}), "
f"there should be {Y} chunks, but instead there "
f"are {chunk_count} chunks.")
raise ValueError(vem)
## Loop through the chunks
for a in range(chunk_count):
A = datetime.now()
## Get prefix to use
subclipPrefix = f"{a+1}of{chunk_count}"
subclipFileName = f"{subclipPrefix}_{fileName}"
logging.info(f"clip: {subclipPrefix}")
## Check how many times this clip has been created already (max 3 atts allowed)
visDF = get_VideoIndexerSplits_rows(subclipFileName=subclipFileName)
## If we're on the last subclip
if a + 1 == chunk_count:
subclipDurationSeconds = clip.duration - (a * chunk_length_secs)
else:
subclipDurationSeconds = chunk_length_secs
startSeconds = chunk_length_secs * a
startHMS = "{:0>8}".format(str(timedelta(seconds=startSeconds)))
endHMS = "{:0>8}".format(
str(timedelta(seconds=startSeconds + chunk_length_secs)))
fileOutPath = "/tmp/" + subclipFileName
# ff = ffmpy.FFmpeg(
# executable="./ffmpeg",
# inputs={
# tempClipFilePath : f"-ss {startSeconds}"
# },
# outputs={
# fileOutPath : f"-t {subclipDurationSeconds} -c copy"
# }
# )
# logging.info(f"ff.cmd: {ff.cmd}")
# ff.run()
if ((_all_) or (X == a + 1)) and (len(visDF) <= 2):
logging.info(f'startSeconds: {startSeconds}')
logging.info(f"tempClipFilePath: {tempClipFilePath}")
logging.info(f"subclipDurationSeconds: {subclipDurationSeconds}")
logging.info(f"fileOutPath: {fileOutPath}")
logging.info(f"startHMS: {startHMS}")
logging.info(f"endHMS: {endHMS}")
# # ffmpegCommand = f'./ffmpeg -ss {startSeconds} -i "{tempClipFilePath}" -t {subclipDurationSeconds} -c copy -bsf:a aac_adtstoasc "{fileOutPath}"'
# ffmpegCommand = f'./ffmpeg -ss {startSeconds} -i "{tempClipFilePath}" -t {subclipDurationSeconds} -bsf:a aac_adtstoasc -acodec copy -vcodec copy "{fileOutPath}"'
ffmpegCommand = f'./ffmpeg -i "{tempClipFilePath}" -ss {startHMS} -to {endHMS} -c copy "{fileOutPath}"'
logging.info(f"ffmpegCommand: {ffmpegCommand}")
# p = subprocess.Popen(ffmpegCommand)
# p.wait()
result = os.popen(ffmpegCommand).read()
logging.info("command run")
logging.info(f"result: {result}")
## Add attempt to SQL
add_VideoIndexerSplits_row(subclipFileName=subclipFileName)
## Create subclip using moviepy
# t_start = startSeconds
# t_end = startSeconds+subclipDurationSeconds
# logging.info(f"t_start: {t_start}")
# logging.info(f"t_end: {t_end}")
# subclip = clip.subclip(
# t_start=t_start,
# t_end=t_end
# )
# logging.info("subclip created")
# # temp_file_path = tempfile.gettempdir() + "/temp-audio.m4a"
# temp_file_path = "/tmp/temp-audio.m4a"
# ## Save to path
# subclip.write_videofile(
# filename=fileOutPath,
# verbose=False,
# logger=None,
# temp_audiofile=temp_file_path,
# remove_temp=True,
# audio_codec="aac"
# )
# logging.info("subclip written to file")
# subclip.close()
# logging.info("subclip closed")
contentType = "video/mp4" if fileType == "MP4" else "audio/mpeg3"
bbs.create_blob_from_path(
container_name=container,
blob_name=subclipFileName,
file_path=fileOutPath,
content_settings=ContentSettings(content_type=contentType))
## Delete created file from temporary storage
os.remove(fileOutPath)
B = datetime.now()
logging.info(f"{subclipPrefix} uploaded, time taken: {B-A}")
else:
logging.info("this ffmpeg run is not happening")
if len(visDF) >= 4:
logging.info("Reason: file has already been created 3 times")
if not _all_:
logging.info(f"Reason: re-do was requested for {X} not {a+1}")
## Delete original file from temporary storage
os.remove(tempClipFilePath)
return f"{chunk_count} files uploaded to the `{container}` container"
def batch_upload(self):
def _glob_files_locally(folder_path):
len_folder_path = len(folder_path) + 1
for root, v, files in os.walk(folder_path):
for f in files:
full_path = os.path.join(root, f)
yield full_path, full_path[len_folder_path:]
def _normalize_blob_file_path(path, name):
path_sep = '/'
if path:
name = path_sep.join((path, name))
return path_sep.join(os.path.normpath(name).split(
os.path.sep)).strip(path_sep)
def _guess_content_type(file_path, original):
if original.content_encoding or original.content_type:
return original
mimetypes.add_type('application/json', '.json')
mimetypes.add_type('application/javascript', '.js')
mimetypes.add_type('application/wasm', '.wasm')
content_type, v = mimetypes.guess_type(file_path)
return ContentSettings(
content_type=content_type,
content_disposition=original.content_disposition,
content_language=original.content_language,
content_md5=original.content_md5,
cache_control=original.cache_control)
if not os.path.exists(self.batch_upload_src):
self.fail(
"batch upload source source directory {0} does not exist".
format(self.batch_upload_src))
if not os.path.isdir(self.batch_upload_src):
self.fail("incorrect usage: {0} is not a directory".format(
self.batch_upload_src))
source_dir = os.path.realpath(self.batch_upload_src)
source_files = list(_glob_files_locally(source_dir))
content_settings = ContentSettings(
content_type=self.content_type,
content_encoding=self.content_encoding,
content_language=self.content_language,
content_disposition=self.content_disposition,
cache_control=self.cache_control,
content_md5=None)
for src, blob_path in source_files:
if self.batch_upload_dst:
blob_path = _normalize_blob_file_path(self.batch_upload_dst,
blob_path)
if not self.check_mode:
try:
self.blob_client.create_blob_from_path(
self.container,
blob_path,
src,
metadata=self.tags,
content_settings=_guess_content_type(
src, content_settings))
except AzureHttpError as exc:
self.fail("Error creating blob {0} - {1}".format(
src, str(exc)))
self.results['actions'].append('created blob from {0}'.format(src))
self.results['changed'] = True
self.results['container'] = self.container_obj
def StoreImage(image_id, image_bytes):
blob_service.create_blob_from_bytes(
IMAGE_STORAGE_CONTAINER_NAME,
image_id,
image_bytes,
content_settings=ContentSettings('image/jpeg'))
CONTAINER_NAME = 'ocean-container'
# log in
storage_client = get_client_from_cli_profile(StorageManagementClient)
# create a public storage container to hold the file
storage_keys = storage_client.storage_accounts.list_keys(RESOURCE_GROUP, STORAGE_ACCOUNT_NAME)
storage_keys = {v.key_name: v.value for v in storage_keys.keys}
storage_client = CloudStorageAccount(STORAGE_ACCOUNT_NAME, storage_keys['key1'])
blob_service = storage_client.create_block_blob_service()
blob_service.create_container(CONTAINER_NAME, public_access=PublicAccess.Container)
#%%
FILE_NAME = 'hello-ocean.html'
blob_service.create_blob_from_bytes(
CONTAINER_NAME,
FILE_NAME,
b'<center><h1> Surf the Ocean again! </h1></center>',
content_settings=ContentSettings('text/html')
)
print(blob_service.make_blob_url(CONTAINER_NAME, FILE_NAME))
#%%
for blob in blob_service.list_blobs(CONTAINER_NAME):
print(blob.name)
def upload_file(data_file, meta, encryption_metadata, max_concurrency):
try:
azure_metadata = {
u'sfcdigest': meta[SHA256_DIGEST],
}
if (encryption_metadata):
azure_metadata.update({
u'encryptiondata':
json.dumps({
u'EncryptionMode': u'FullBlob',
u'WrappedContentKey': {
u'KeyId': u'symmKey1',
u'EncryptedKey': encryption_metadata.key,
u'Algorithm': u'AES_CBC_256'
},
u'EncryptionAgent': {
u'Protocol': '1.0',
u'EncryptionAlgorithm': u'AES_CBC_128',
},
u'ContentEncryptionIV': encryption_metadata.iv,
u'KeyWrappingMetadata': {
u'EncryptionLibrary': u'Java 5.3.0'
}
}),
u'matdesc':
encryption_metadata.matdesc
})
azure_location = SnowflakeAzureUtil.extract_container_name_and_path(
meta[u'stage_info'][u'location'])
path = azure_location.path + meta[u'dst_file_name'].lstrip('/')
client = meta[u'client']
callback = None
if meta[u'put_azure_callback']:
callback = meta[u'put_azure_callback'](
data_file,
os.path.getsize(data_file),
output_stream=meta[u'put_callback_output_stream'],
show_progress_bar=meta[u'show_progress_bar'])
def azure_callback(current, total):
callback(current)
logger.debug(
"data transfer progress from sdk callback. "
"current: %s, total: %s", current, total)
client.create_blob_from_path(
azure_location.container_name,
path,
data_file,
progress_callback=azure_callback
if meta[u'put_azure_callback'] else None,
metadata=azure_metadata,
max_connections=max_concurrency,
content_settings=ContentSettings(
content_type=HTTP_HEADER_VALUE_OCTET_STREAM,
content_encoding=u'utf-8',
))
logger.debug(u'DONE putting a file')
meta[u'dst_file_size'] = meta[u'upload_size']
meta[u'result_status'] = ResultStatus.UPLOADED
except AzureHttpError as err:
logger.debug(
u"Caught exception's status code: {status_code} and message: {ex_representation}"
.format(status_code=err.status_code,
ex_representation=str(err)))
if err.status_code == 403 and SnowflakeAzureUtil._detect_azure_token_expire_error(
err):
logger.debug(u"AZURE Token expired. Renew and retry")
meta[u'result_status'] = ResultStatus.RENEW_TOKEN
return None
else:
meta[u'last_error'] = err
meta[u'result_status'] = ResultStatus.NEED_RETRY
def properties_and_metadata_operations(self, account):
file_blob_name = "HelloWorld.png"
text_blob_name = "Text"
# Create a Block Blob Service object
blockblob_service = account.create_block_blob_service()
container_name = 'blockblobbasicscontainer' + self.random_data.get_random_name(
6)
try:
# Create a new container
print('1. Create a container with name and custom metadata - ' +
container_name)
blockblob_service.create_container(container_name,
{'sample': 'azure-storage'})
# Upload file as a block blob
print(
'2. Uploading BlockBlob from file with properties and custom metadata'
)
#Get full path on drive to file_to_upload by joining the fully qualified directory name and file name on the local drive
full_path_to_file = os.path.join(os.path.dirname(__file__),
file_blob_name)
blockblob_service.create_blob_from_path(
container_name,
file_blob_name,
full_path_to_file,
content_settings=ContentSettings(
content_type='application/png'),
metadata={'category': 'azure-samples'})
blockblob_service.create_blob_from_text(
container_name,
text_blob_name,
'Data',
content_settings=ContentSettings(content_encoding='UTF-8',
content_language='en'),
metadata={
'origin': 'usa',
'title': 'azure-samples'
})
# Get all the container properties
print('3. Get Container metadata')
container = blockblob_service.get_container_properties(
container_name)
print(' Metadata:')
for key in container.metadata:
print(' ' + key + ':' + container.metadata[key])
# Get all the blob properties
print('4. Get Blob properties')
blob = blockblob_service.get_blob_properties(
container_name, file_blob_name)
print(' Metadata:')
for key in blob.metadata:
print(' ' + key + ':' + blob.metadata[key])
print(' Properties:')
print(' Content-Type:' +
blob.properties.content_settings.content_type)
finally:
# Delete the container
print("5. Delete Container")
if blockblob_service.exists(container_name):
blockblob_service.delete_container(container_name)
def main():
logging.basicConfig(level=logging.DEBUG)
with open(TASKDATA) as taskdata_file:
taskdata = json.loads(taskdata_file.read())
github = GithubAdapter(taskdata)
gh_commit = github.get_commit()
config = github.get_config()
blob_service = AppendBlobService(
account_name=taskdata["storage_account_name"],
account_key=taskdata["storage_account_key"])
queue_service = QueueService(
connection_string=taskdata["queue_connection_string"])
loop = asyncio.get_event_loop()
ctx = Context(loop=loop,
config=config,
blob_service=blob_service,
queue_service=queue_service,
taskdata=taskdata)
blob_service.create_container("logs",
fail_on_exist=False,
public_access=PublicAccess.Blob)
blob_service.create_blob("logs",
ctx.pid,
content_settings=ContentSettings(
content_type="text/plain; charset=utf-8"))
gh_commit.create_status(
"pending",
target_url=blob_service.make_blob_url("logs", ctx.pid),
description="Build started",
context=config.get("configuration_name",
"configuration%s" % taskdata["config_num"]))
os.makedirs(REPOSDIR, exist_ok=True)
# Check if we're the only process who updates the git cache on SMB share.
# Otherwise skip updating.
if not os.path.exists(LOCKFILENAME):
lock = open(LOCKFILENAME, "w")
lock.close()
update_git_cache(ctx)
os.unlink(LOCKFILENAME)
if os.path.exists(SRCDIR):
shutil.rmtree(SRCDIR)
os.makedirs(os.path.join(SRCDIR, "build/conf"))
with open(os.path.join(SRCDIR, "build/conf/auto.conf"), "a") as localconf:
localconf.write("\n%s\n" % config.get("localconf", ""))
localconf.write(AUTOCONFIG)
repos = get_repos(config)
repos.append((repodirname(taskdata["gh"]["repository"]["clone_url"]),
taskdata["gh"]["repository"]["clone_url"], None, None))
for reponame, repourl, reporef, _ in repos:
refrepopath = os.path.join(REPOSDIR, reponame)
run(ctx,
["git", "clone", "--reference", refrepopath, repourl, reponame],
cwd=SRCDIR)
if reporef:
LOG.info("Checkout %s to %s" % (reponame, reporef))
run(ctx, ["git", "checkout", "%s" % reporef],
cwd=os.path.join(SRCDIR, reponame))
# Do checkout
if taskdata["gh"]["type"] == "pull_request":
LOG.info("Add remote repo %s" % taskdata["gh"]["clone_url"])
run(ctx, [
"git", "remote", "add", "contributor", taskdata["gh"]["clone_url"]
],
cwd=os.path.join(
SRCDIR,
repodirname(taskdata["gh"]["repository"]["clone_url"])))
LOG.info("Fetch contributor's repo")
run(ctx, ["git", "fetch", "contributor"],
cwd=os.path.join(
SRCDIR,
repodirname(taskdata["gh"]["repository"]["clone_url"])))
LOG.info("Checkout %s to %s" % (repodirname(
taskdata["gh"]["repository"]["clone_url"]), taskdata["gh"]["sha"]))
run(ctx, ["git", "checkout", taskdata["gh"]["sha"]],
cwd=os.path.join(
SRCDIR, repodirname(taskdata["gh"]["repository"]["clone_url"])))
# Fetch sstate if any
if os.path.exists(get_sstate_archive_path(ctx)):
with tarfile.open(name=get_sstate_archive_path(ctx),
mode="r:gz") as sstate_tar:
sstate_tar.extractall(path=SRCDIR)
addlayers = []
for dep in config["dependencies"]:
repodir = repodirname(dep["url"])
layers = dep.get("layers", None)
if layers:
addlayers.extend([
"bitbake-layers add-layer ../%s/%s" % (repodir, layer)
for layer in layers
])
else:
addlayers.append("bitbake-layers add-layer ../%s" % repodir)
addlayers.append("bitbake-layers add-layer ../%s" %
repodirname(taskdata["gh"]["repository"]["clone_url"]))
run_script(ctx,
BUILDSCRIPT % ("\n".join(addlayers), config["bitbake_target"]),
cwd=SRCDIR)
save_sstate(ctx)
# Github auth token has expired by now most probably => renew
github = GithubAdapter(taskdata)
gh_commit = github.get_commit()
gh_commit.create_status(
"success",
target_url=blob_service.make_blob_url("logs", ctx.pid),
description="Target has been built successfully",
context=config.get("configuration_name",
"configuration%s" % taskdata["config_num"]))
loop.close()
# TODO: copy cloud-init log files to share
taskdata["build_result"] = "success"
queue_service.put_message(
"buildresults",
base64.b64encode(json.dumps(taskdata).encode("utf")).decode("utf"))