def delete_vhd_by_name(self, **kwargs):
""" Delete VHD by name """
build_dir = kwargs.get('build_dir')
vhd_url, vhd_name = self.get_vhd_url(build_dir)
logger.info('Got Requests to delete VHD Name:%s' % vhd_name)
try:
block_blob_service = PageBlobService(
account_name=self.storage_account,
account_key=self.storage_account_key)
generator = block_blob_service.list_blobs(self.container_name)
storage_vhds = list()
for blob in generator:
storage_vhds.append(blob.name)
if vhd_name == blob.name:
logger.info('vhd_name: %s Found going to delete it' %
vhd_name)
block_blob_service.delete_blob(self.container_name,
vhd_name)
return True
logger.info('Did not found VHD Name in Storage Blob VHD Name: %s' %
vhd_name)
logger.info('List of VHD Name in Storage Blob VHD Name: %s' %
', '.join(storage_vhds))
except Exception as e:
fail('Error while deleting the VHD name:%s ,exp: %s' %
(vhd_name, e.message))
def delete_oldest_vhd(self, vhd_name):
blob_list = []
vhd = vhd_name.split('-')
try:
block_blob_service = PageBlobService(
account_name=self.storage_account,
account_key=self.storage_account_key)
generator = block_blob_service.list_blobs(self.container_name)
delete_vhd = "%s-%s" % (vhd[0], vhd[1])
for blob in generator:
# get specific version vhd files
if delete_vhd in blob.name:
blob_list.append(blob)
except Exception as e:
fail('Error while getting vhd list: %s' % e.message)
# keep last 4 controller vhd
if len(blob_list) < 4:
return
old_date = blob_list[0].properties.last_modified
oldest = blob_list[0]
for blob in blob_list:
if blob.properties.last_modified < old_date:
old_date = blob.properties.last_modified
oldest = blob
try:
logger.info('Deleting %s' % oldest.name)
block_blob_service.delete_blob(self.container_name, oldest.name)
except Exception as e:
fail('Error while deleting vhd file: %s' % e.message)
def delete_vm_vhd(self, vm_name=None, vhd_name=None):
""" Delete VM VHD """
if not vm_name:
vm_name = self.vm_json.get('name')
if not vhd_name:
vhd_name = 'controller'
ctrl_vhd_name = '%s-%s.vhd' % (vm_name, vhd_name)
# Note: Azure VHD name: smoke-sanity-ctlr-avitest-2-osDisk-smoke-sanity-ctlr-avitest-2-osDisk-controller-18.1.1-16297.vhd
vhd_name = vm_name + '-osDisk-' + vm_name + '-osDisk-'
for i in range(3):
try:
block_blob_service = PageBlobService(
account_name=self.storage_account,
account_key=self.storage_account_key)
# List the blobs in container
blob_list = block_blob_service.list_blobs(self.container_name)
for blob_vhd_name in blob_list:
if vhd_name in blob_vhd_name.name:
logger.info('Going to delete .. vhd name: %s ' %
blob_vhd_name.name)
block_blob_service.delete_blob(self.container_name,
blob_vhd_name.name)
#logger.info("List of blobs : %s" % blob_list.items)
break
except Exception as e:
logger.info('Error while deleting controller vhd: %s' %
e.message)
logger.info('Retrynig again for vhd deletion')
def vhd_exists(self, vhd_name):
page_blob_service = PageBlobService(
account_name=self.storage_account,
account_key=self.storage_account_key)
try:
page_blob_service.get_blob_metadata(self.container_name, vhd_name)
return True
except:
logger.info('%s does not exists' % vhd_name)
return False
def __init__(self, **azure_config):
"""
:param ServiceManagement azure_client: an instance of the azure
serivce managment api client.
:param String service_name: The name of the cloud service
:param
names of Azure volumes to identify cluster
:returns: A ``BlockDeviceVolume``.
"""
self._instance_id = self.compute_instance_id()
creds = ServicePrincipalCredentials(
client_id=azure_config['client_id'],
secret=azure_config['client_secret'],
tenant=azure_config['tenant_id'])
self._resource_client = ResourceManagementClient(
creds,
azure_config['subscription_id'])
self._compute_client = ComputeManagementClient(
creds,
azure_config['subscription_id'])
self._azure_storage_client = PageBlobService(
account_name=azure_config['storage_account_name'],
account_key=azure_config['storage_account_key'])
self._manager = DiskManager(self._resource_client,
self._compute_client,
self._azure_storage_client,
azure_config['storage_account_container'],
azure_config['group_name'],
azure_config['location'])
self._storage_account_name = azure_config['storage_account_name']
self._disk_container_name = azure_config['storage_account_container']
self._resource_group = azure_config['group_name']
def main():
bbs = BlockBlobService(settings.STORAGE_ACCOUNT_NAME, settings.STORAGE_ACCOUNT_KEY)
pbs = PageBlobService(settings.STORAGE_ACCOUNT_NAME, settings.STORAGE_ACCOUNT_KEY)
abs = AppendBlobService(settings.STORAGE_ACCOUNT_NAME, settings.STORAGE_ACCOUNT_KEY)
service.create_container(CONTAINER_NAME)
process(bbs, LOCAL_BLOCK_BLOB_FILES, CONNECTION_COUNTS)
process(pbs, LOCAL_PAGE_BLOB_FILES, CONNECTION_COUNTS)
process(abs, LOCAL_APPEND_BLOB_FILES, CONNECTION_COUNTS)
def check_vm_vhd_exists(self, vm_name=None, vhd_name=None):
""" Check VM VHD Exists ... """
if not vm_name:
vm_name = self.vm_json.get('name')
if not vhd_name:
vhd_name = vm_name + '-osDisk-' + vm_name + '-osDisk-controller'
try:
page_blob_service = PageBlobService(
account_name=self.storage_account,
account_key=self.storage_account_key)
# page_blob_service.get_blob_metadata(self.container_name, vhd_name)
# return True
blob_list = page_blob_service.list_blobs(self.container_name)
for blob_vhd_name in blob_list:
if vhd_name in blob_vhd_name.name:
logger.info('VM VHD exists: %s ' % blob_vhd_name.name)
return True
except:
return False
def setUp(self):
creds = ServicePrincipalCredentials(
client_id=azure_config['client_id'],
secret=azure_config['client_secret'],
tenant=azure_config['tenant_id'])
self._resource_client = ResourceManagementClient(
creds, azure_config['subscription_id'])
self._compute_client = ComputeManagementClient(
creds, azure_config['subscription_id'])
self._page_blob_service = PageBlobService(
account_name=azure_config['storage_account_name'],
account_key=azure_config['storage_account_key'])
self._manager = DiskManager(self._resource_client,
self._compute_client,
self._page_blob_service,
azure_config['storage_account_container'],
azure_config['group_name'],
azure_config['location'])
def _get_client(self):
"""
Azure blob service
Returns:
dict of azure.storage.blob.baseblobservice.BaseBlobService subclass: Service
"""
parameters = self._secured_storage_parameters().copy()
try:
del parameters["blob_type"]
except KeyError:
pass
return {
_BlobTypes.PageBlob: PageBlobService(**parameters),
_BlobTypes.BlockBlob: BlockBlobService(**parameters),
_BlobTypes.AppendBlob: AppendBlobService(**parameters),
}
def _get_client(self):
"""
Azure blob service
Returns:
dict of azure.storage.blob.baseblobservice.BaseBlobService subclass:
Service
"""
parameters = self._secured_storage_parameters().copy()
# Parameter added by pycosio and unsupported by blob services.
try:
del parameters['blob_type']
except KeyError:
pass
return {
_BlobTypes.PageBlob: PageBlobService(**parameters),
_BlobTypes.BlockBlob: BlockBlobService(**parameters),
_BlobTypes.AppendBlob: AppendBlobService(**parameters)
}
def upload_vhd_to_azure(self, build_dir, vhd_name):
""" Upload VHD image to Azure Storage from given build directory """
vhd_file = build_dir + '/controller.vhd'
try:
page_blob_service = PageBlobService(
account_name=self.storage_account,
account_key=self.storage_account_key)
logger.info('%s started uploading....' % vhd_name)
page_blob_service.create_blob_from_path(self.container_name,
vhd_name, vhd_file)
logger.info('%s file uploaded' % vhd_name)
page_blob_service.set_blob_metadata(self.container_name,
vhd_name,
metadata={'status': 'done'})
# Have taken care vai once usage is done going to delete vhd
# self.delete_oldest_vhd(vhd_name)
except Exception as e:
fail('Error while uploading .vhd to container: %s' % str(e))
def stream_thread(ws):
# ws.on_message = lambda m: print(m)
bs = PageBlobService(account_name=ACCOUNT_NAME, account_key=ACCOUNT_KEY)
bbs = PageBlobService(account_name=ACCOUNT_NAME, account_key=ACCOUNT_KEY)
n = -1
chunk = 0
while not ws.closed:
# message = ws.receive()
chunk = (chunk - 1) % 10
properties = bs.set_sequence_number(CONTAINER_NAME,
"buffer",
sequence_number_action="max",
sequence_number=0)
new_n = (properties.sequence_number - 2) % 10
if new_n == n:
continue
n = new_n
data = bbs.get_blob_to_bytes(CONTAINER_NAME,
"buffer",
start_range=n * 512 * 32,
end_range=(n + 1) * 512 * 32)
ws.send(data.content[0:8001 * 2])
from azure.storage.blob import PageBlobService
# Parse command line arguments
parser = argparse.ArgumentParser()
parser.add_argument("account", help="storage account name")
parser.add_argument("key", help="storage account key")
parser.add_argument("container", help="blob container")
parser.add_argument("blob", help="blob name")
args = parser.parse_args()
# Calculate size of name
blob_size_in_bytes = 124 + len(args.blob) * 2
# Calculate size of metadata + data
try:
# Connect to storage account
blob_service = PageBlobService(args.account, args.key)
# Calculate size of metadata
metadata = blob_service.get_blob_metadata(args.container, args.blob)
for key, value in metadata.items():
blob_size_in_bytes += 3 + len(key) + len(value)
# Calculate size of data
ranges = blob_service.get_page_ranges(args.container, args.blob)
for range in ranges:
blob_size_in_bytes += 12 + range.end - range.start
print(blob_size_in_bytes)
except:
sys.exit(1)
else:
sys.exit(0)
f[f < 0] = 0
#fi = np.cumsum(f / fs) / 60
fi = ts * target_speed / 60
N = len(fi)
base = 2 * np.pi * fi
b = np.array([np.sin(base * w) * a for w, a in zip(wk, Ak)])
a = b.sum(axis=0)
a = a #+ np.random.normal(-0.01, 0.01, N)
if s is not None:
a += s
return np.int16(a / np.max(np.abs(a)) * 32767), points[-1]
az_blob_service = PageBlobService(account_name=ACCOUNT_NAME,
account_key=ACCOUNT_KEY)
az_blob_service.create_container(CONTAINER_NAME, fail_on_exist=False)
#if not az_blob_service.exists(CONTAINER_NAME, "buffer"):
properties = az_blob_service.create_blob(CONTAINER_NAME,
"buffer",
512 * 320,
sequence_number=0)
i = 0
t = 0
start = time()
last_speed = 0
def run_example():
"""Virtual Machine management example."""
global mystack_cloud, resource_client, compute_client, storage_client, network_client
arm_url = mystack_cloud.endpoints.resource_manager
storage_endpoint_suffix = arm_url.replace(arm_url.split(".")[0],
"").strip('./')
logging.basicConfig(level=logging.ERROR)
###########
# Prepare #
###########
# Create Resource group
print('\nCreate Resource Group')
resource_client.resource_groups.create_or_update(GROUP_NAME,
{'location': LOCATION})
try:
# Create a storage account
print('\nCreate a storage account')
storage_async_operation = storage_client.storage_accounts.create(
GROUP_NAME, STORAGE_ACCOUNT_NAME, {
'sku': {
'name': 'standard_lrs'
},
'kind': 'storage',
'location': LOCATION
})
storage_async_operation.wait()
# Upload VHD to blob storage
############################
# Get storage account keys
storage_keys = storage_client.storage_accounts.list_keys(
GROUP_NAME, STORAGE_ACCOUNT_NAME)
connection_string = ('DefaultEndpointsProtocol={protocol};'
'AccountName={account_name};'
'AccountKey={account_key};'
'EndpointSuffix={blob_endpoint};').format(
protocol=urlparse(arm_url).scheme,
account_name=STORAGE_ACCOUNT_NAME,
account_key=storage_keys.keys[0].value,
blob_endpoint=storage_endpoint_suffix,
)
# Create a container called 'vmimages'.
# Set the permission so the blobs are public.
# Upload the created file, use vhd_file_name for the blob name.
print("\nUploading to Azure Stack Storage as blob:\n\t" +
vhd_file_name)
blob_client = PageBlobService(connection_string=connection_string)
container_client = blob_client.create_container(
container_name, public_access='container', fail_on_exist=False)
blob_client.create_blob_from_path(container_name, vhd_file_name,
vhd_file_path)
# List the blobs in the container
print("\nListing blobs...")
blob_list = blob_client.list_blobs(container_name)
for blob in blob_list:
print("\t" + blob.name)
# Construct the blob uri so it can be used to create the VM image
blob_uri = urlparse(
arm_url
).scheme + '://' + blob_client.primary_endpoint + '/' + container_name + '/' + vhd_file_name
# Create image from the VHD
async_creation = compute_client.images.create_or_update(
GROUP_NAME, 'UbuntuImage', {
'location': LOCATION,
'storage_profile': {
'os_disk': {
'os_type': 'Linux',
'os_state': "Generalized",
'blob_uri': blob_uri,
'caching': "ReadWrite",
}
}
})
image_resource = async_creation.result()
print("\nVM image resource id: \n\t", image_resource.id)
# Create a NIC
nic = create_nic(network_client)
#############
# VM Sample #
#############
# Create Linux VM
print('\nCreating Linux Virtual Machine')
vm_parameters = create_vm_parameters(nic.id, image_resource.id)
async_vm_creation = compute_client.virtual_machines.create_or_update(
GROUP_NAME, VM_NAME, vm_parameters)
async_vm_creation.wait()
# Tag the VM
print('\nTag Virtual Machine')
async_vm_update = compute_client.virtual_machines.create_or_update(
GROUP_NAME, VM_NAME, {
'location': LOCATION,
'tags': {
'who-rocks': 'python',
'where': 'on azure'
}
})
async_vm_update.wait()
# Create managed data disk
print('\nCreate (empty) managed Data Disk')
async_disk_creation = compute_client.disks.create_or_update(
GROUP_NAME, 'mydatadisk1', {
'location': LOCATION,
'disk_size_gb': 1,
'creation_data': {
'create_option': DiskCreateOption.empty
}
})
data_disk = async_disk_creation.result()
# Get the virtual machine by name
print('\nGet Virtual Machine by Name')
virtual_machine = compute_client.virtual_machines.get(
GROUP_NAME, VM_NAME)
# Attach data disk
print('\nAttach Data Disk')
virtual_machine.storage_profile.data_disks.append({
'lun':
12,
'name':
'mydatadisk1',
'create_option':
DiskCreateOption.attach,
'managed_disk': {
'id': data_disk.id
}
})
async_disk_attach = compute_client.virtual_machines.create_or_update(
GROUP_NAME, virtual_machine.name, virtual_machine)
async_disk_attach.wait()
# Detach data disk
print('\nDetach Data Disk')
data_disks = virtual_machine.storage_profile.data_disks
data_disks[:] = [
disk for disk in data_disks if disk.name != 'mydatadisk1'
]
async_vm_update = compute_client.virtual_machines.create_or_update(
GROUP_NAME, VM_NAME, virtual_machine)
virtual_machine = async_vm_update.result()
# Deallocating the VM (in preparation for a disk resize)
print('\nDeallocating the VM (to prepare for a disk resize)')
async_vm_deallocate = compute_client.virtual_machines.deallocate(
GROUP_NAME, VM_NAME)
async_vm_deallocate.wait()
# Increase OS disk size by 10 GB
print('\nUpdate OS disk size')
os_disk_name = virtual_machine.storage_profile.os_disk.name
os_disk = compute_client.disks.get(GROUP_NAME, os_disk_name)
if not os_disk.disk_size_gb:
print(
"\tServer is not returning the OS disk size, possible bug in the server?"
)
print("\tAssuming that the OS disk size is 30 GB")
os_disk.disk_size_gb = 30
os_disk.disk_size_gb += 10
async_disk_update = compute_client.disks.create_or_update(
GROUP_NAME, os_disk.name, os_disk)
async_disk_update.wait()
# Start the VM
print('\nStart VM')
async_vm_start = compute_client.virtual_machines.start(
GROUP_NAME, VM_NAME)
async_vm_start.wait()
# Restart the VM
print('\nRestart VM')
async_vm_restart = compute_client.virtual_machines.restart(
GROUP_NAME, VM_NAME)
async_vm_restart.wait()
# Stop the VM
print('\nStop VM')
async_vm_stop = compute_client.virtual_machines.power_off(
GROUP_NAME, VM_NAME)
async_vm_stop.wait()
# List VMs in subscription
print('\nList VMs in subscription')
for vm in compute_client.virtual_machines.list_all():
print("\tVM: {}".format(vm.name))
# List VM in resource group
print('\nList VMs in resource group')
for vm in compute_client.virtual_machines.list(GROUP_NAME):
print("\tVM: {}".format(vm.name))
# Delete VM
print('\nDelete VM')
async_vm_delete = compute_client.virtual_machines.delete(
GROUP_NAME, VM_NAME)
async_vm_delete.wait()
# # Create Windows VM
# print('\nCreating Windows Virtual Machine')
# # Recycling NIC of previous VM
# vm_parameters = create_vm_parameters(
# nic.id, VM_REFERENCE['windows'])
# async_vm_creation = compute_client.virtual_machines.create_or_update(
# GROUP_NAME, VM_NAME, vm_parameters)
# async_vm_creation.wait()
except CloudError:
print('A VM operation failed:', traceback.format_exc(), sep='\n')
except Exception as ex:
print("Exception details:\n", ex)
else:
print('All example operations completed successfully!')
finally:
# Delete Resource group and everything in it
print('\nDelete Resource Group')
delete_async_operation = resource_client.resource_groups.delete(
GROUP_NAME)
delete_async_operation.wait()
print("\nDeleted: {}".format(GROUP_NAME))
parser.add_argument("sourceblob", help="source blob name")
parser.add_argument("backupaccount", help="backup storage account name")
parser.add_argument("backupkey", help="backup storage account key")
parser.add_argument("backupcontainer", help="backup blob container")
parser.add_argument("backupblob", help="backup blob name")
parser.add_argument("snapshotcurrent", help="latest page blob snapshot")
parser.add_argument("snapshotprevious", help="previous page blob snapshot")
args = parser.parse_args()
# Define maximum page range (in bytes) for each update operation - 4MB is the maximum supported limit
max_range = 4194304
# Connect to Page Blob Service for Storage Accounts
source_blob_service = PageBlobService(args.sourceaccount, args.sourcekey)
backup_blob_service = PageBlobService(args.backupaccount, args.backupkey)
# Find page differences between snapshots and update backup blob with changes
try:
ranges = source_blob_service.get_page_ranges_diff(args.sourcecontainer, args.sourceblob, args.snapshotprevious, args.snapshotcurrent)
for range in ranges:
if range.is_cleared == True:
print('clearing page range: ({}, {}) '.format(range.start, range.end))
backup_blob_service.clear_page(args.backupcontainer, args.backupblob, range.start, range.end)
else:
byte_offset = 0
last_range = False
while True:
start_range = range.start + byte_offset
# Parse command line arguments
parser = argparse.ArgumentParser()
parser.add_argument("account", help="storage account name")
parser.add_argument("key", help="storage account key")
parser.add_argument("container", help="blob container")
parser.add_argument("blob", help="blob name")
parser.add_argument("-s", "--snapshot", help="create a new snapshot", action="store_true")
parser.add_argument("-d", "--delete", help="delete a snapshot")
parser.add_argument("-r", "--restore", help="restore a snapshot to a new blob")
args = parser.parse_args()
# Connect to Storage Account Blob Service
source_blob_service = PageBlobService(args.account, args.key)
# Perform actions based on arguments
if args.snapshot == True:
print '# Creating a new snapshot...'
source_blob_service.snapshot_blob(args.container, args.blob)
print 'OK.'
if args.delete:
print '# Deleting snapshot...'
source_blob_service.delete_blob(args.container, args.blob, snapshot=args.delete)
print "Deleted", args.delete
if args.restore:
print '# Restoring snapshot to a new blob...'
parser = argparse.ArgumentParser()
parser.add_argument("account", help="storage account name")
parser.add_argument("key", help="storage account key")
parser.add_argument("container", help="blob container")
parser.add_argument("blob", help="blob name")
parser.add_argument("-s",
"--snapshot",
help="create a new snapshot",
action="store_true")
parser.add_argument("-d", "--delete", help="delete a snapshot")
parser.add_argument("-r", "--restore", help="restore a snapshot to a new blob")
args = parser.parse_args()
# Connect to Storage Account Blob Service
source_blob_service = PageBlobService(args.account, args.key)
# Perform actions based on arguments
if args.snapshot == True:
print '# Creating a new snapshot...'
source_blob_service.snapshot_blob(args.container, args.blob)
print 'OK.'
if args.delete:
print '# Deleting snapshot...'
source_blob_service.delete_blob(args.container,
args.blob,
snapshot=args.delete)
print "Deleted", args.delete
parser.add_argument("sourceblob", help="source blob name")
parser.add_argument("backupaccount", help="backup storage account name")
parser.add_argument("backupkey", help="backup storage account key")
parser.add_argument("backupcontainer", help="backup blob container")
parser.add_argument("backupblob", help="backup blob name")
parser.add_argument("snapshotcurrent", help="latest page blob snapshot")
parser.add_argument("snapshotprevious", help="previous page blob snapshot")
args = parser.parse_args()
# Define maximum page range (in bytes) for each update operation - 4MB is the maximum supported limit
max_range = 4194304
# Connect to Page Blob Service for Storage Accounts
source_blob_service = PageBlobService(args.sourceaccount, args.sourcekey)
backup_blob_service = PageBlobService(args.backupaccount, args.backupkey)
# Find page differences between snapshots and update backup blob with changes
try:
ranges = source_blob_service.get_page_ranges_diff(args.sourcecontainer,
args.sourceblob,
args.snapshotprevious,
args.snapshotcurrent)
for range in ranges:
if range.is_cleared == True:
print('clearing page range: ({}, {}) '.format(
range.start, range.end))
backup_blob_service.clear_page(args.backupcontainer,
args.backupblob, range.start,