def CreateLogFile():
""" ログファイルを作成する。WriteLog を呼び出す前に実行すること。 """
szRet = "";
if( DEBUG_MODE ):
return( "Debug モードのためスキップします。" );
try:
szRet = "AppendBlobService";
blob_service = AppendBlobService(
account_name,
account_key
);
szRet = "create_container";
bIsExists = blob_service.exists(
log_container_name
);
if bIsExists:
pass;
else:
blob_service.create_container(
log_container_name,
public_access=PublicAccess.Blob
);
bIsExists = blob_service.exists(
log_container_name,
log_file_name
);
if bIsExists:
szRet = "already blob."
else:
szRet = "create_blob";
blob_service.create_blob(
log_container_name,
log_file_name
);
szRet = "OK";
except:
#szRet = "Log exception";
pass;
return szRet;
def CreateLogFile():
""" ログファイルを作成する。WriteLog を呼び出す前に実行すること。 """
szRet = ""
try:
szRet = "AppendBlobService"
blob_service = AppendBlobService(account_name, account_key)
szRet = "create_container"
bIsExists = blob_service.exists(log_container_name)
if bIsExists:
pass
else:
blob_service.create_container(log_container_name,
public_access=PublicAccess.Blob)
bIsExists = blob_service.exists(log_container_name, log_file_name)
if bIsExists:
szRet = "already blob."
else:
szRet = "create_blob"
blob_service.create_blob(log_container_name, log_file_name)
szRet = "OK"
except:
#szRet = "Log exception";
pass
return szRet
class AzureBlobStore21(implements(StoreInterface)):
def __init__(self, storage_creds, max_retries=10):
self.storage_id = storage_creds["name"]
self.storage_key = storage_creds["key"]
self.bs = BlockBlobService(account_name=self.storage_id,
account_key=self.storage_key)
self.append_bs = AppendBlobService(account_name=self.storage_id,
account_key=self.storage_key)
self.max_retries = max_retries
self.set_retries(max_retries)
# ---- HELPER functions ----
def set_retries(self, count):
old_count = self.max_retries
self.max_retries = count
# bug workaround: standard Retry classes don't retry status=409 (container is being deleted)
#import azure.storage.common.retry as retry
#self.bs.retry = retry.LinearRetry(backoff=5, max_attempts=count).retry
#self.append_bs.retry = retry.LinearRetry(backoff=5, max_attempts=count).retry
self.bs.retry = utils.make_retry_func(count)
self.append_bs.retry = utils.make_retry_func(count)
return old_count
# ---- MISC part of interface ----
def get_service_name(self):
''' return the unique name of the storage service'''
return self.storage_id
def get_retry(self):
return self.bs.retry
def set_retry(self, value):
self.bs.retry = value
# ---- CONTAINER interface ----
def does_container_exist(self, container):
return self.bs.exists(container)
def create_container(self, container):
return self.bs.create_container(container)
def list_containers(self):
containers = self.bs.list_containers()
name_list = [contain.name for contain in containers]
return name_list
def delete_container(self, container):
return self.bs.delete_container(container)
def get_container_properties(self, container):
props = self.bs.get_container_properties(container)
return props
def get_container_metadata(self, container):
md = self.bs.get_container_metadata(container)
return md
# def set_container_metadata(self, container, md_dict):
# return self.bs.set_container_metadata(container, md_dict)
# ---- BLOB interface ----
def does_blob_exist(self, container, blob_path):
return self.bs.exists(container, blob_path)
def create_blob(self, container, blob_path, text, fail_if_exists=False):
ifn = "*" if fail_if_exists else None
return self.bs.create_blob_from_text(container,
blob_path,
text,
if_none_match=ifn)
def create_blob_from_path(self,
container,
blob_path,
source_fn,
progress_callback=None):
result = self.bs.create_blob_from_path(
container,
blob_path,
source_fn,
progress_callback=progress_callback)
return result
def append_blob(self,
container,
blob_path,
text,
append_with_rewrite=False):
# create blob if it doesn't exist
if not append_with_rewrite:
# normal handling
if not self.append_bs.exists(container, blob_path):
self.append_bs.create_blob(container, blob_path)
return self.append_bs.append_blob_from_text(
container, blob_path, text)
'''
Appends text to a normal blob blob by reading and then rewriting the entire blob.
Correctly handles concurrency/race conditions.
Recommended for lots of small items (like 10,000 run names).
Note: we turn off retries on azure CALL-level so that we can retry on
OUR CALL-level.
'''
# experimental local retry loop
old_retry = self.bs.get_retry()
self.bs.set_retry(utils.make_retry_func(0))
succeeded = False
for i in range(20):
try:
if self.bs.does_blob_exist(container, blob_path):
# read prev contents
blob_text = self.bs.get_blob_text(container, blob_path)
# append our text
new_text = blob_text + text
# write blob, ensuring etag matches (no one updated since above read)
self.bs.create_blob(container,
blob_path,
new_text,
if_match=blob.properties.etag)
else:
# if no previous blob, just try to create it
self.bs.create_blob(container, blob_path, text)
except BaseException as ex:
logger.exception(
"Error in _append_blob_with_retries, ex={}".format(ex))
sleep_time = np.random.random() * 4
console.diag(
"XT store received an expected azure exception; will backoff for {:.4f} secs [retry #{}]"
.format(sleep_time, i + 1))
time.sleep(sleep_time)
else:
succeeded = True
break
# restore retry
self.bs.set_retry(old_retry)
if not succeeded:
errors.service_error(
"_append_blob_with_rewrite failed (too many retries)")
def list_blobs(self,
container,
path=None,
return_names=True,
recursive=True):
'''
NOTE: the semantics here a tricky
if recursive:
- return a flat list of all full path names of all files (no directory entries)
else:
- return a flat list of all files and all directory names (add "/" to end of directory names)
if return_names:
- return list of names
else:
- return a list of objects with following properties:
.name (file pathname)
.properties
.content_length (number)
.modified_ns (time in ns)
The delimiter trick: this is when we set the delimiter arg = "/" to tell azure to return only the blobs
in the specified directory - that is, don't return blobs from child directories. In this case, azure
returns the effective child directory name, followed by a "/", but not its contents (which we hope is faster).
'''
delimiter = None if recursive else "/"
# specific Azure path rules for good results
if path:
if path.startswith("/"):
path = path[
1:] # blob API wants this part of path relative to container
# we should only add a "/" if path is a folder path
if path.endswith("*"):
# we just need to block the addition of "/"
path = path[0:-1]
elif not path.endswith("/"):
path += "/" # best if path ends with "/"
blobs = self.bs.list_blobs(container, prefix=path, delimiter=delimiter)
if return_names:
blobs = [blob.name for blob in blobs]
else:
blobs = list(blobs)
return blobs
def delete_blob(self, container, blob_path, snapshot=None):
dss = DeleteSnapshot()
return self.bs.delete_blob(container,
blob_path,
delete_snapshots=dss.Include)
def get_blob_text(self, container, blob_path):
# watch out for 0-length blobs - they trigger an Azure RETRY error
text = ""
# azure storage bug workaround: avoid RETRY errors for 0-length blob
blob = self.bs.get_blob_properties(container, blob_path)
if blob.properties.content_length:
blob = self.bs.get_blob_to_text(container, blob_path)
text = blob.content
return text
def get_blob_to_path(self,
container,
blob_path,
dest_fn,
snapshot=None,
progress_callback=None):
# azure storage bug workaround: avoid RETRY errors for 0-length blob
blob = self.bs.get_blob_properties(container, blob_path)
if blob.properties.content_length:
result = self.bs.get_blob_to_path(
container,
blob_path,
dest_fn,
snapshot=snapshot,
progress_callback=progress_callback)
text = result.content
else:
md = blob.metadata
if "hdi_isfolder" in md and md["hdi_isfolder"]:
# its a directory marker; do NOT create a local file for it
text = ""
else:
# 0-length text file; just write the file outselves
text = ""
with open(dest_fn, "wt") as outfile:
outfile.write(text)
return text
def get_blob_properties(self, container, blob_path):
props = self.bs.get_blob_properties(container, blob_path)
return props
def get_blob_metadata(self, container, blob_path):
return self.bs.get_blob_metadata(container, blob_path)
# def set_blob_metadata(self, container, blob_path, md_dict):
# return self.bs.set_blob_metadata(container, blob_path, md_dict)
def copy_blob(self, source_container, source_blob_path, dest_container,
dest_blob_path):
source_blob_url = self.bs.make_blob_url(source_container,
source_blob_path)
self.bs.copy_blob(dest_container, dest_blob_path, source_blob_url)
def snapshot_blob(self, container, blob_path):
blob = self.bs.snapshot_blob(container, blob_path)
#pd = utils.obj_to_dict(blob)
return blob
class LogWriter(object):
"""description of class"""
LOG_CONTAINER_NAME = r'log-files'
DEBUG_MODE = bool(os.getenv('DEBUG_MODE', False))
# コンストラクタ
def __init__(self, name, key, subFolderName=None):
super(LogWriter, self).__init__()
self._name = name
self._key = key
self.m_szLogFileName = ""
self.m_szSubFolderName = subFolderName
self.m_pBlobService = AppendBlobService(name, key)
#}def __init__
def _CreateLogFile(self):
""" ログファイルを作成する。WriteLog を呼び出す前に実行すること。 """
szRet = ""
if (LogWriter.DEBUG_MODE):
return ("Debug モードのためスキップします。")
try:
if (0 == len(self.m_szLogFileName)):
szRet = "create_container"
bIsExists = self.m_pBlobService.exists(
LogWriter.LOG_CONTAINER_NAME)
if bIsExists:
pass
else:
self.m_pBlobService.create_container(
LogWriter.LOG_CONTAINER_NAME,
public_access=PublicAccess.Blob)
#ログファイル名の決定
#// 後ろに追加しているが len で 0 と調べているため空文字列
if ((self.m_szSubFolderName is not None)
and (0 < len(self.m_szSubFolderName))):
#// サブフォルダー名が指定されているときは追加する
self.m_szLogFileName += self.m_szSubFolderName + "\\"
#}if
self.m_szLogFileName += r"{0:%Y-%m-%dT%H-%M-%S.log}".format(
datetime.datetime.now())
bIsExists = self.m_pBlobService.exists(
LogWriter.LOG_CONTAINER_NAME, self.m_szLogFileName)
if bIsExists:
szRet = "already blob."
else:
szRet = "create_blob"
self.m_pBlobService.create_blob(
LogWriter.LOG_CONTAINER_NAME, self.m_szLogFileName)
szRet = "OK"
else:
szRet = "Already called."
szRet = "OK"
#}if
except Exception as e:
#szRet = "Log exception";
szRet = szRet + "\r\n" + str(e)
pass
return szRet
#}def
def WriteLog(self, txt):
""" ログファイルにテキストを出力する。末尾に改行コードが追加される。 """
szRet = ""
szLogText = r"{0:%Y-%m-%d %H:%M:%S}".format(
datetime.datetime.now()) + r" : " + txt + "\r\n"
if (LogWriter.DEBUG_MODE):
print(szLogText)
return ("Debug モードのためスキップしました。")
try:
#ログファイルの作成
self._CreateLogFile()
szRet = "append_blob_from_text"
self.m_pBlobService.append_blob_from_text(
LogWriter.LOG_CONTAINER_NAME, self.m_szLogFileName, szLogText)
szRet = "OK"
except Exception as e:
#szRet = "Log exception";
szRet = szRet + "\r\n" + str(e)
#try
return szRet
#}def
def WriteBlob(self, blob_name, value):
""" 単一 BLOB ファイルを作成しテキストを保存する。 """
szRet = ""
if (LogWriter.DEBUG_MODE):
return ("Debug モードのため書き込みをしません。")
try:
#blob_name = r'sample.txt';
szRet = "BlockBlobService"
blob_service = BlockBlobService(self._name, self._key)
szRet = "create_container"
blob_service.create_container(LogWriter.LOG_CONTAINER_NAME,
public_access=PublicAccess.Blob)
szRet = "create_blob_from_bytes"
#blob_service.create_blob_from_bytes(
# log_container_name,
# log_blob_name,
# b'<center><h1>Hello World!</h1></center>',
# content_settings=ContentSettings('text/html')
#)
if (isinstance(value, str)):
szRet = "create_blob_from_text"
blob_service.create_blob_from_text(
LogWriter.LOG_CONTAINER_NAME, blob_name, value)
else:
szRet = "create_blob_from_stream"
blob_service.create_blob_from_stream(
LogWriter.LOG_CONTAINER_NAME, blob_name, io.BytesIO(value))
#}if
#szRet = "make_blob_url"
#print(blob_service.make_blob_url(log_container_name, log_blob_name))
szRet = "OK"
except:
print(r"Exception.")
#try
return szRet
#def WriteBlob( blob_name, txt ):
def MakeBlobUri(self, blob_name):
blob_service = BlockBlobService(self._name, self._key)
szRet = blob_service.make_blob_url(LogWriter.LOG_CONTAINER_NAME,
blob_name)
return (szRet)
#}def
#}class
import io
from azure.storage.blob import BlockBlobService
from azure.storage.blob import AppendBlobService
STORAGE_ACCOUNT_NAME = os.environ['STORAGE_ACCOUNT_NAME']
STORAGE_ACCOUNT_KEY = os.environ['STORAGE_ACCOUNT_KEY']
LOGS_CONTAINER_NAME = 'logs'
LOGS_ARCHIVE_CONTAINER_NAME = 'logs-archive'
append_blob_service = AppendBlobService(account_name=STORAGE_ACCOUNT_NAME,
account_key=STORAGE_ACCOUNT_KEY)
block_blob_service = BlockBlobService(account_name=STORAGE_ACCOUNT_NAME,
account_key=STORAGE_ACCOUNT_KEY)
if not append_blob_service.exists(LOGS_CONTAINER_NAME):
exit(0)
if not block_blob_service.exists(LOGS_ARCHIVE_CONTAINER_NAME):
block_blob_service.create_container(LOGS_ARCHIVE_CONTAINER_NAME)
generator = append_blob_service.list_blobs(LOGS_CONTAINER_NAME)
for blob in generator:
with io.BytesIO() as stream:
append_blob_service.get_blob_to_stream(
container_name=LOGS_CONTAINER_NAME,
blob_name=blob.name,
stream=stream,
max_connections=2)
stream.seek(0)
block_blob_service.create_blob_from_stream(
class Results(object):
"""
Handles interacting with encrypted results in blob storage.
"""
def __init__(self, logger, redisHost, redisPort):
"""
Initializes a new instance of the JobStatus class.
:param logger logger: The logger instance to use for logging
:param str redis_host: Redis host where the Redis Q is running
:param int redis_port: Redis port where the Redis Q is running
"""
self.logger = logger
self.config = Config()
self.redis_host = redisHost
self.redis_port = redisPort
# create an instance of AESCipher to use for encryption
aesHelper = AESHelper(self.config)
self.aescipher = aesHelper.create_aescipher_from_config()
if (self.init_storage_services() is False):
raise Exception(
"Errors occurred instantiating results storage service.")
def init_storage_services(self):
"""
Initializes the storage service clients using values from config.py.
:return: True on success. False on failure.
:rtype: boolean
"""
try:
# creates instance of BlockBlobService and AppendBlobService to use for completed results storage
self.storage_service = BlockBlobService(
account_name=self.config.storage_account_name,
sas_token=self.config.results_container_sas_token)
self.append_storage_service = AppendBlobService(
account_name=self.config.storage_account_name,
sas_token=self.config.results_container_sas_token)
self.storage_service.create_container(
self.config.results_container_name)
# creates instances of Azure QueueService
self.job_status_queue_service = QueueService(
account_name=self.config.storage_account_name,
sas_token=self.config.job_status_queue_sas_token)
self.job_status_queue_service.encode_function = models.QueueMessageFormat.noencode
self.results_queue_service = QueueService(
account_name=self.config.storage_account_name,
sas_token=self.config.results_queue_sas_token)
self.results_queue_service.create_queue(
self.config.results_container_name)
self.results_queue_service.encode_function = models.QueueMessageFormat.noencode
# creates instance of Redis client to use for job status storage
pool = redis.ConnectionPool(host=self.redis_host,
port=self.redis_port)
self.storage_service_cache = redis.Redis(connection_pool=pool)
return True
except Exception as ex:
self.log_exception(ex, self.init_storage_services.__name__)
return False
def log_exception(self, exception, functionName):
"""
Logs an exception to the logger instance for this class.
:param Exception exception: The exception thrown.
:param str functionName: Name of the function where the exception occurred.
"""
self.logger.debug("Exception occurred in: " + functionName)
self.logger.debug(type(exception))
self.logger.debug(exception)
def write_result(self, result):
"""
Encrypts and writes result to queue
:param str result: The result to write to queue
:return: True on success. False on failure.
:rtype: boolean
"""
try:
# encrypt the encoded result and then encode it
encryptedResult = base64.b64encode(self.aescipher.encrypt(result))
# put the encoded result into the azure queue for future consolidation
self.results_queue_service.put_message(
self.config.results_queue_name, encryptedResult)
return True
except Exception as ex:
self.log_exception(ex, self.write_result.__name__)
return False
def count_consolidated_results(self):
"""
Returns a count of results that were consolidated.
"return: int count: Total count of results that were consolidated.
"""
try:
consolidatedResults = self.storage_service_cache.get(
self.config.results_consolidated_count_redis_key)
return consolidatedResults
except Exception as ex:
self.log_exception(ex, self.count_consolidated_results.__name__)
return False
except Exception as ex:
self.log_exception(
ex, self.consolidate_results.__name__ +
" - Error consolidating result blob.")
def consolidate_results(self):
"""
Consolidates all individual result files into single result file in storage. Blobs are deleted once they
are added to the consolidated file.
"return: int count: Total count of results consolidated in result file.
"""
try:
# ensure the consolidated append blob exists
if not self.append_storage_service.exists(
self.config.results_container_name,
blob_name=self.config.results_consolidated_file):
self.append_storage_service.create_blob(
self.config.results_container_name,
self.config.results_consolidated_file)
result_messages = []
with io.BytesIO() as consolidated_result:
while len(result_messages
) < self.config.result_consolidation_size:
messages = self.results_queue_service.get_messages(
self.config.results_queue_name,
min(self.config.result_consolidation_size, 32))
# If the queue is empty, stop and consolidate
if not messages:
break
# add the message to the memory stream
for msg in messages:
consolidated_result.write(msg.content + "\n")
result_messages.append(msg)
# append the results to the consolidated file
consolidated_result.seek(0)
self.append_storage_service.append_blob_from_stream(
self.config.results_container_name,
self.config.results_consolidated_file, consolidated_result)
# remove all of the messages from the queue
num_of_consolidated_results = len(result_messages)
for msg in result_messages:
self.results_queue_service.delete_message(
self.config.results_queue_name, msg.id, msg.pop_receipt)
self.storage_service_cache.incrby(
self.config.results_consolidated_count_redis_key,
num_of_consolidated_results)
# write the count of results we consolidated out to queue to provide status
self.job_status_queue_service.put_message(
self.config.job_status_queue_name,
str(num_of_consolidated_results) + " results consolidated.")
return len(result_messages)
except Exception as ex:
self.log_exception(ex, self.consolidate_results.__name__)
return len(result_messages)
def get_total_jobs_consolidated_status(self):
"""
Write out the the current state of the workload; the percentage of jobs that are completed and consolidated
"return: float status: percentage of completed jobs
"""
# log out total job status
total_scheduled_jobs = self.storage_service_cache.get(
self.config.scheduled_jobs_count_redis_key)
total_consolidated_results = self.storage_service_cache.get(
self.config.results_consolidated_count_redis_key)
if total_consolidated_results is None:
total_consolidated_results = "0"
status_message = "Total: " + total_consolidated_results + "/" + total_scheduled_jobs + " jobs have been successfully processed and consolidated."
self.logger.info(status_message)
self.job_status_queue_service.put_message(
self.config.job_status_queue_name, status_message)
return float(total_consolidated_results) / int(total_scheduled_jobs)
