def check_storage_errors(management, storageact_name, storage_type, subtype,
key, warning, critical, verbosity):
"""Check storage errors for the metric given by key
management -- service management object
storageact_name -- storage account name
storage_type -- blob/queue/table
subtype - tx/cap (transactions or capacity)
key - needed only for transaction metric
warning - Nagios warning level
warning - Nagios critical level
"""
primary_key, _ = retrieve_keys(management, storageact_name.lower())
if not primary_key:
return 3, 'Error retrieving storage keys'
storage_account = CloudStorageAccount(storageact_name.lower(), primary_key)
if not storage_account:
return 3, 'Error retrieving storage account'
try:
table_service = storage_account.create_table_service()
except:
return 3, 'System error in creating table service'
if subtype == 'cap':
if storage_type == 'blob':
return check_storagecap_errors(table_service)
else:
return 3, 'Capacity metrics not supported for tables/queues'
else:
if (not key):
return 3, 'Key missing'
return check_storagetx_errors(table_service, storage_type, key.lower(),
warning, critical, verbosity)
def get_blob_client(self,
resource_group_name,
storage_account_name,
storage_blob_type='block'):
keys = dict()
try:
# Get keys from the storage account
self.log('Getting keys')
account_keys = self.storage_client.storage_accounts.list_keys(
resource_group_name, storage_account_name)
except Exception as exc:
self.fail("Error getting keys for account {0} - {1}".format(
storage_account_name, str(exc)))
try:
self.log('Create blob service')
if storage_blob_type == 'page':
return CloudStorageAccount(
storage_account_name,
account_keys.keys[0].value).create_page_blob_service()
elif storage_blob_type == 'block':
return CloudStorageAccount(
storage_account_name,
account_keys.keys[0].value).create_block_blob_service()
else:
raise Exception("Invalid storage blob type defined.")
except Exception as exc:
self.fail(
"Error creating blob service client for storage account {0} - {1}"
.format(storage_account_name, str(exc)))
def check_storage_errors(management, storageact_name, storage_type, subtype,
key, warning, critical, verbosity):
"""Check storage errors for the metric given by key
management -- service management object
storageact_name -- storage account name
storage_type -- blob/queue/table
subtype - tx/cap (transactions or capacity)
key - needed only for transaction metric
warning - Nagios warning level
warning - Nagios critical level
"""
primary_key, _ = retrieve_keys(management, storageact_name.lower())
if not primary_key:
return 3, 'Error retrieving storage keys'
storage_account = CloudStorageAccount(storageact_name.lower(), primary_key)
if not storage_account:
return 3, 'Error retrieving storage account'
try:
table_service = storage_account.create_table_service()
except:
return 3, 'System error in creating table service'
if subtype == 'cap':
if storage_type == 'blob':
return check_storagecap_errors(table_service)
else:
return 3, 'Capacity metrics not supported for tables/queues'
else:
if (not key):
return 3, 'Key missing'
return check_storagetx_errors(table_service, storage_type, key.lower(),
warning, critical, verbosity)
def account_has_blob_containers(self):
'''
If there are blob containers, then there are likely VMs depending on this account and it should
not be deleted.
'''
self.log('Checking for existing blob containers')
keys = dict()
try:
# Get keys from the storage account
account_keys = self.storage_client.storage_accounts.list_keys(
self.resource_group, self.name)
keys['key1'] = account_keys.key1
keys['key2'] = account_keys.key2
except AzureHttpError as e:
self.fail(
"check_for_container:Failed to get account keys: {0}".format(
e))
try:
cloud_storage = CloudStorageAccount(
self.name, keys['key1']).create_page_blob_service()
except Exception as e:
self.fail(
"check_for_container:Error creating blob service: {0}".format(
e))
try:
response = cloud_storage.list_containers()
except AzureMissingResourceHttpError:
# No blob storage available?
return False
if len(response.items) > 0:
return True
return False
def account_has_blob_containers(self):
'''
If there are blob containers, then there are likely VMs depending on this account and it should
not be deleted.
'''
self.log('Checking for existing blob containers')
try:
# Get keys from the storage account
account_keys = self.storage_client.storage_accounts.list_keys(self.resource_group, self.name)
keys = {v.key_name: v.value for v in account_keys.keys}
except AzureHttpError as e:
self.fail("check_for_container:Failed to get account keys: {0}".format(e))
try:
cloud_storage = CloudStorageAccount(self.name, keys['key1']).create_page_blob_service()
except Exception as e:
self.fail("check_for_container:Error creating blob service: {0}".format(e))
try:
response = cloud_storage.list_containers()
except AzureMissingResourceHttpError:
# No blob storage available?
return False
if len(response.items) > 0:
return True
return False
def get_blob_client(self, resource_group_name, storage_account_name):
keys = dict()
try:
# Get keys from the storage account
self.log('Getting keys')
account_keys = self.storage_client.storage_accounts.list_keys(resource_group_name, storage_account_name)
except Exception as exc:
self.fail("Error getting keys for account {0} - {1}".format(storage_account_name, str(exc)))
try:
self.log('Create blob service')
return CloudStorageAccount(storage_account_name, account_keys.keys[0].value).create_block_blob_service()
except Exception as exc:
self.fail("Error creating blob service client for storage account {0} - {1}".format(storage_account_name,
str(exc)))
def __init__(self,
input_folder,
output_folder = 'output',
overwrite_output = False,
remove_html_ext = True,
overwrite_container = False,
account_name = None,
account_key = None,
container_name = DEVSTORE_CONTAINER_NAME):
'''
Constructor function. Creates a new container for blobs under the specified account.
If the container with the same name already exists, delete it if overwrite_container is true.
input_folder: The folder contains all the resources of the static website.
output_folder: The folder contains all the resources uploaded.
overwrite_output: Overwrites the output_folder anyway.
remove_html_ext: Removes the .htm/.html in the url.
overwrite_container: Deletes the existing container.
account_name: Optional. Your storage account name, DEVSTORE_ACCOUNT_NAME is used if None.
account_key: Optional. Your storage account key, DEVSTORE_ACCOUNT_KEY is used if None.
container_name: Optional. Container name, DEVSTORE_CONTAINER_NAME is used if None.
'''
self.input_folder = os.path.abspath(input_folder).lower()
self.output_folder = os.path.abspath(output_folder).lower()
self.overwrite_output = overwrite_output
self.remove_html_ext = remove_html_ext
self.account_name = account_name
self.account_key = account_key
self.container_name = container_name
self.full_path_blob_name_dict = self.list_full_path_with_blob_name()
if not account_name or not account_key:
os.environ['EMULATED'] = 'true'
else:
os.environ['EMULATED'] = 'false'
self.blob_service = CloudStorageAccount(self.account_name, self.account_key).create_blob_service()
if overwrite_container:
self.blob_service.delete_container(container_name)
self.blob_service.create_container(container_name, x_ms_blob_public_access = 'container')
return 3, 'Hosted service {0} not found'.format(cloudservice_name)
errors = []
if not service.deployments:
return 1, 'No deployments found'
# find the production deployment among deployments
production_depl = None
for depl in service.deployments:
if depl.deployment_slot == 'Production':
production_depl = depl
break
error_code_all = 0
if production_depl:
try:
storage_account = CloudStorageAccount(storageacct_name,
primary_key)
table_service = storage_account.create_table_service()
table_name = 'WAD'+ str(production_depl.private_id) + \
'PT'+ '1H' +'R'+'Table'
error_code_all = 0
for role in production_depl.role_list:
role_clause = 'Role eq \'' + role.role_name + '\''
# we use 2 hours earlier since some times no counters
# show up with 1 hour
rngtime = (datetime.utcnow() - datetime.min) - \
timedelta(minutes = 120)
partition_str = '0' + \
str(((rngtime.days * 24 * 60 * 60 + rngtime.seconds)
* 1000 * 1000 + rngtime.microseconds)*10)
partition_clause = 'PartitionKey ge \'' + partition_str + '\''
counter_clause = 'CounterName eq \'' + counter[
return 3, 'Hosted service {0} not found'.format(cloudservice_name)
errors = []
if not service.deployments:
return 1, 'No deployments found'
# find the production deployment among deployments
production_depl = None
for depl in service.deployments:
if depl.deployment_slot == 'Production':
production_depl = depl
break
error_code_all = 0
if production_depl:
try:
storage_account = CloudStorageAccount(storageacct_name, primary_key)
table_service = storage_account.create_table_service()
table_name = 'WAD'+ str(production_depl.private_id) + \
'PT'+ '1H' +'R'+'Table'
error_code_all = 0
for role in production_depl.role_list:
role_clause = 'Role eq \''+ role.role_name + '\''
# we use 2 hours earlier since some times no counters
# show up with 1 hour
rngtime = (datetime.utcnow() - datetime.min) - \
timedelta(minutes = 120)
partition_str = '0' + \
str(((rngtime.days * 24 * 60 * 60 + rngtime.seconds)
* 1000 * 1000 + rngtime.microseconds)*10)
partition_clause = 'PartitionKey ge \''+ partition_str +'\''
counter_clause = 'CounterName eq \''+counter['perf_counter']+'\''
class SitesToAzureBlob:
"""
Class contains functions to upload a static website, which is loacated in a folder,
to Windows Azure Blob storage service.
"""
def __init__(self,
input_folder,
output_folder = 'output',
overwrite_output = False,
remove_html_ext = True,
overwrite_container = False,
account_name = None,
account_key = None,
container_name = DEVSTORE_CONTAINER_NAME):
'''
Constructor function. Creates a new container for blobs under the specified account.
If the container with the same name already exists, delete it if overwrite_container is true.
input_folder: The folder contains all the resources of the static website.
output_folder: The folder contains all the resources uploaded.
overwrite_output: Overwrites the output_folder anyway.
remove_html_ext: Removes the .htm/.html in the url.
overwrite_container: Deletes the existing container.
account_name: Optional. Your storage account name, DEVSTORE_ACCOUNT_NAME is used if None.
account_key: Optional. Your storage account key, DEVSTORE_ACCOUNT_KEY is used if None.
container_name: Optional. Container name, DEVSTORE_CONTAINER_NAME is used if None.
'''
self.input_folder = os.path.abspath(input_folder).lower()
self.output_folder = os.path.abspath(output_folder).lower()
self.overwrite_output = overwrite_output
self.remove_html_ext = remove_html_ext
self.account_name = account_name
self.account_key = account_key
self.container_name = container_name
self.full_path_blob_name_dict = self.list_full_path_with_blob_name()
if not account_name or not account_key:
os.environ['EMULATED'] = 'true'
else:
os.environ['EMULATED'] = 'false'
self.blob_service = CloudStorageAccount(self.account_name, self.account_key).create_blob_service()
if overwrite_container:
self.blob_service.delete_container(container_name)
self.blob_service.create_container(container_name, x_ms_blob_public_access = 'container')
def upload_files_to_blob(self):
'''
Uploads the files to the blob.
full_path_blob_name_dict: A dictionary whose key is the full_path of the file and the value is the blob_name.
'''
#if self.remove_html_ext:
# for blob_name in full_path_blob_name_dict.values():
# file_name, ext = os.path.splitext(blob_name)
# if ext == '.html' or ext == '.htm':
# self.html_blob_name_list.append(blob_name)
curdir = os.getcwd()
for full_path, blob_name in self.full_path_blob_name_dict.iteritems():
output_path = os.path.join(self.output_folder, blob_name)
if not os.path.exists(os.path.dirname(output_path)):
os.makedirs(os.path.dirname(output_path))
if self.overwrite_output is False and os.path.exists(output_path):
if filecmp.cmp(full_path, output_path):
print blob_name + ' skips...'
continue
print blob_name + ' is uploading...'
file_name, ext = os.path.splitext(blob_name)
file_blob = open(full_path, 'rb').read()
content_type = self.fetch_content_type(ext)
if ext == '.htm' or ext == '.html':
if self.remove_html_ext:
blob_name = file_name
os.chdir(os.path.split(full_path)[0])
file_blob = self.adjust_url_links(file_blob)
self.blob_service.put_blob(self.container_name, blob_name, file_blob, x_ms_blob_type = 'BlockBlob', x_ms_blob_content_type = content_type)
shutil.copy(full_path, os.path.dirname(output_path))
os.chdir(curdir)
def list_full_path_with_blob_name(self):
'''
Fetches the full_path as key and blob_name as value into a dictionary.
'''
dict = {}
for root, dirs, files in os.walk(self.input_folder):
for fi in files:
full_path = os.path.abspath(os.path.join(root, fi)).lower()
blob_name = self.list_blob_name(full_path)
dict[full_path] = blob_name.replace('\\', '/') # To replace the Windows backslash \ in the blob_name with /.
return dict
def url_rep(self, matchobj):
'''
This is called for every non-overlapping occurrence of pattern: href|src=[\'"]?([^\'" >]+).
'''
url_blob_name = self.list_blob_name(os.path.abspath(matchobj.group(2))).replace('\\', '/')
if url_blob_name in self.full_path_blob_name_dict.values():
file_name, ext = os.path.splitext(matchobj.group(2))
if self.remove_html_ext and ext == '.html' or ext == '.htm':
return matchobj.group(1) + r'="' + file_name + '"'
else:
return matchobj.group(0)
else:
return matchobj.group(0)
def adjust_url_links(self, file_content):
'''
Adjusts the urls in href and src attributes.
Removes the .html/.htm extension of the linked html files in the file_content if needed.
file_content: the content of the html file
'''
file_content = re.sub(r'(href|src)=[\'"]?([^\'" >]+)', self.url_rep, file_content)
'''
Problem with using BeautifulSoup. It cannot preserve the '<', '>' in the <script type="text/template"...>
'''
#html = soup(file_content)
#for tag in html.findAll('a', {'href': True}):
# href = tag['href']
# if href in html_blob_name_list:
# tag['href'] = os.path.splitext(href)[0]
#return str(html)
return file_content
def list_blob_name(self, full_path):
'''
Gets the file path name in the input_folder for blob storage.
If we uploaded from a subfolder (such as /search), we must rename blobs to have the 'folder/' prefix in their name.
For example, if we uploaded index.html from search subfolder, rename the blob from 'index.html' to 'search/index.html'.
'''
name = full_path
name = name.replace(self.input_folder, '')
if re.match('[A-Za-z0-9_-]', name[0]) is None:
name=name[1:]
return name
def fetch_content_type(self, extension_name):
'''
Fetches the content type from the extension name.
'''
return {
'.png': 'image/png',
'.gif': 'image/gif',
'.jpg':'image/jpg',
'.jpeg':'image/jpeg',
'.mp3':'audio/mp3',
'.jar':'application/java-archive',
'.zip': 'application/zip',
'.htm': 'text/htm',
'.html': 'text/html',
'.js': 'application/javascript',
'.txt': 'text/plain',
'.css': 'text/css',
'.xml':'text/xml',
'.pdf':'application/pdf',
'.json':'application/json'
}.get(extension_name, None) # None is default if extensionName not found
