def store(image, entity, entity_id): blob_service = BlobService(account_name='shnergledata', account_key=os.environ['BLOB_KEY']) myblob = image.read() name = '/' + entity + '/' + entity_id blob_service.put_blob('images', name, myblob, x_ms_blob_type='BlockBlob') return True
def test_azure_call(request): import os try: from azure.storage import BlobService bs = BlobService(os.environ["AZURE_STORAGE_ACCOUNT"], os.environ["AZURE_STORAGE_ACCESS_KEY"]) import random container_name = hex(int(random.random() * 1000000000)) bs.create_container(container_name) bs.put_blob(container_name, 'testblob', 'hello world\n', 'BlockBlob') blob = bs.get_blob(container_name, 'testblob') if blob != 'hello world\n': return HttpResponse("Failed!", status = '404') bs.delete_blob(container_name, 'testblob') bs.delete_container(container_name) return HttpResponse("Succeeded!") except: try: import traceback return HttpResponse(traceback.format_exc() + str(os.environ.keys())) except: import traceback return HttpResponse(traceback.format_exc())
def uri_put_file(access_key, secret_key, uri, fp, content_encoding=None): assert fp.tell() == 0 data = fp.read() assert uri.startswith('wabs://') url_tup = urlparse(uri) check_sum = base64.encodestring(md5(data).digest()) kwargs = dict(x_ms_blob_type='BlockBlob', content_md5=check_sum.strip('\n')) if content_encoding is not None: kwargs['x_ms_blob_content_encoding'] = content_encoding conn = BlobService(access_key, secret_key, protocol='https') conn.put_blob(url_tup.netloc, url_tup.path, data, **kwargs) # To maintain consistency with the S3 version of this function we must # return an object with a certain set of attributes. Currently, that set # of attributes consists of only 'size' return _Key(size=len(data))
class AzureBlobStorage(Storage): ''' classdocs ''' def __init__(self, azure_profile): ''' Constructor ''' if not azure_profile: raise Exception() else: container_name = azure_profile['container_name'] account_name = azure_profile['account_name'] account_key = azure_profile['key'] base_url = azure_profile['base_url'] self.blob = BlobService( account_name=account_name, account_key=account_key) self.container = container_name self.base_url = base_url def delete(self, name): """ Delete file. """ try: self.blob.delete_blob(self.container, name) except WindowsAzureMissingResourceError: return False else: return True def delete_files(self, files=None): """ Delete files in container. """ if not files: files = self.listdir(self.container)[1] for _file in files: self.delete(_file) def exists(self, name, with_properties=False): """ Existing check. """ result = False blob_properties = None try: blob_properties = self.blob.get_blob_properties( self.container, name) except WindowsAzureMissingResourceError: result = False else: result = True if with_properties: return result, blob_properties else: return result def get_available_name(self, name): return super(AzureBlobStorage, self).get_available_name(name.replace('\\', '/')) def get_valid_name(self, name): return name def _list(self, path, prefix, maxresults): result = [] blobs = self.blob.list_blobs(path, prefix, maxresults) for _blob in blobs: result.append(_blob.name) return result def listdir(self, path=None, prefix=None, maxresults=None): """ Catalog file list. """ if not path: path = self.container return [], self._list(path, prefix, maxresults) def size(self, name): """ File size. """ result, properties = self.exists(name, with_properties=True) if result: return int(properties['content-length']) else: return 0 def url(self, name, chk_exist=False): """ URL for file downloading. """ if chk_exist: if self.exists(name): return '%s%s/%s' % (self.base_url, self.container, name) else: return None else: return '%s%s/%s' % (self.base_url, self.container, name) def _open(self, name, mode='rb'): """ Open file. """ in_mem_file = StringIO.StringIO( self.blob.get_blob(self.container, name)) in_mem_file.name = name in_mem_file.mode = mode return File(in_mem_file) def _save(self, name, blob_to_upload, x_ms_blob_type='BlockBlob', content_type=None): """ Save file. """ if hasattr(blob_to_upload, 'content_type'): content_type = blob_to_upload.content_type or None if content_type is None: content_type = mimetypes.guess_type(name)[0] or None blob_to_upload.seek(0) self.blob.put_blob(self.container, name, blob_to_upload, x_ms_blob_type, x_ms_blob_content_type=content_type) return name def modified_time(self, name): """ Last modification time. """ result, properties = self.exists(name, with_properties=True) if result: date_string = properties['last-modified'] modified_dt = parser.parse(date_string) if timezone.is_naive(modified_dt): return modified_dt else: return timezone.make_naive(modified_dt, timezone.get_current_timezone()) else: return None created_time = accessed_time = modified_time
def generate_and_upload(gauge_factory, config, logger): start = datetime.datetime.now() twitter_followers = gauge_factory('twitter.followers') twitter_tweets = gauge_factory('twitter.tweets') fb_friends = gauge_factory('facebook.friends') foursq_checkins = gauge_factory('foursquare.checkins') klout_score = gauge_factory('klout.score') runkeeper_activities = gauge_factory('runkeeper.activities') runkeeper_calories = gauge_factory('runkeeper.calories_burned') runkeeper_weight = gauge_factory('runkeeper.weight') tmp102_celsius = gauge_factory('tmp102.te mperature', gauge_type='hourly') lastfm_listened = gauge_factory('lastfm.listened') jawbone_sleeps = gauge_factory('jawbone.sleeps') jawbone_steps = gauge_factory('jawbone.steps') jawbone_caffeine = gauge_factory('jawbone.caffeine') data = {} data_sources = [ # (output key, gauge, days back, aggregator, postprocessors) ('twitter.followers', twitter_followers, 30, None, [zero_fill_daily, interpolators.linear]), ('twitter.tweets', twitter_tweets, 20, None, [zero_fill_daily]), ('facebook.friends', fb_friends, 180, monthly_max, None), ('foursquare.checkins', foursq_checkins, 14, None, [zero_fill_daily]), ('lastfm.listened', lastfm_listened, 14, None, [zero_fill_daily]), ('klout.score', klout_score, 30, weekly_max, [zero_fill_weekly, interpolators.linear]), ('runkeeper.calories', runkeeper_calories, 60, weekly_sum, [zero_fill_weekly]), ('runkeeper.activities', runkeeper_activities, 60,weekly_sum, [zero_fill_weekly]), ('runkeeper.weight', runkeeper_weight, 180,weekly_min, [zero_fill_weekly, interpolators.linear]), ('sleeps', jawbone_sleeps, 14, None, [zero_fill_daily, interpolators.linear]), ('steps', jawbone_steps, 14, None, [zero_fill_daily, interpolators.linear]), ('caffeine', jawbone_caffeine, 30, None, [zero_fill_daily]), ('tmp102.temperature', tmp102_celsius, 2.5, None, None) ] for ds in data_sources: data[ds[0]] = ds[1].aggregate(today_utc() - timedelta(days=ds[2]), aggregator=ds[3], post_processors=ds[4]) report = { 'generated': str(now_utc()), 'data': data, 'took': (datetime.datetime.now() - start).seconds } report_json = json.dumps(report, indent=4, default=json_date_serializer) report_content = '{0}({1})'.format(JSONP_CALLBACK_NAME, report_json) blob_service = BlobService(config['azure.account'], config['azure.key']) blob_service.create_container(config['azure.blob.container']) blob_service.set_container_acl(config['azure.blob.container'], x_ms_blob_public_access='container') blob_service.put_blob(config['azure.blob.container'], config['azure.blob.name'], report_content, 'BlockBlob') took = (datetime.datetime.now() - start).seconds logger.info('Report generated and uploaded. Took {0} s.'.format(took))
class AzureFS(LoggingMixIn, Operations): """Azure Blob Storage filesystem""" blobs = None containers = dict() # <cname, dict(stat:dict, #files:None|dict<fname, stat>) fds = dict() # <fd, (path, bytes, dirty)> fd = 0 def __init__(self, account, key): self.blobs = BlobService(account, key) self.rebuild_container_list() def convert_to_epoch(self, date): """Converts Tue, 31 Jul 2012 07:17:34 GMT format to epoch""" return int(time.mktime(time.strptime(date, TIME_FORMAT))) def rebuild_container_list(self): cmap = dict() cnames = set() for c in self.blobs.list_containers(): date = c.properties.last_modified cstat = dict(st_mode=(S_IFDIR | 0755), st_uid=getuid(), st_size=0, st_mtime=self.convert_to_epoch(date)) cname = c.name cmap['/' + cname] = dict(stat=cstat, files=None) cnames.add(cname) cmap['/'] = dict(files={}, stat=dict(st_mode=(S_IFDIR | 0755), st_uid=getuid(), st_size=0, st_mtime=int(time.time()))) self.containers = cmap # destroys fs tree cache resistant to misses def _parse_path(self, path): # returns </dir, file(=None)> if path.count('/') > 1: # file return str(path[:path.rfind('/')]), str(path[path.rfind('/') + 1:]) else: # dir pos = path.rfind('/', 1) if pos == -1: return path, None else: return str(path[:pos]), None def parse_container(self, path): base_container = path[1:] # /abc/def/g --> abc if base_container.find('/') > -1: base_container = base_container[:base_container.find('/')] return str(base_container) def _get_dir(self, path, contents_required=False): if not self.containers: self.rebuild_container_list() if path in self.containers and not (contents_required and \ self.containers[path]['files'] is None): return self.containers[path] cname = self.parse_container(path) if '/' + cname not in self.containers: raise FuseOSError(ENOENT) else: if self.containers['/' + cname]['files'] is None: # fetch contents of container log.info("------> CONTENTS NOT FOUND: %s" % cname) blobs = self.blobs.list_blobs(cname) dirstat = dict(st_mode=(S_IFDIR | 0755), st_size=0, st_uid=getuid(), st_mtime=time.time()) if self.containers['/' + cname]['files'] is None: self.containers['/' + cname]['files'] = dict() for f in blobs: blob_name = f.name blob_date = f.properties.last_modified blob_size = long(f.properties.content_length) node = dict(st_mode=(S_IFREG | 0644), st_size=blob_size, st_mtime=self.convert_to_epoch(blob_date), st_uid=getuid()) if blob_name.find('/') == -1: # file just under container self.containers['/' + cname]['files'][blob_name] = node return self.containers['/' + cname] return None def _get_file(self, path): d, f = self._parse_path(path) dir = self._get_dir(d, True) if dir is not None and f in dir['files']: return dir['files'][f] def getattr(self, path, fh=None): d, f = self._parse_path(path) if f is None: dir = self._get_dir(d) return dir['stat'] else: file = self._get_file(path) if file: return file raise FuseOSError(ENOENT) # FUSE def mkdir(self, path, mode): if path.count('/') <= 1: # create on root name = path[1:] if not 3 <= len(name) <= 63: log.error("Container names can be 3 through 63 chars long.") raise FuseOSError(ENAMETOOLONG) if name is not name.lower(): log.error("Container names cannot contain uppercase \ characters.") raise FuseOSError(EACCES) if name.count('--') > 0: log.error('Container names cannot contain consecutive \ dashes (-).') raise FuseOSError(EAGAIN) #TODO handle all "-"s must be preceded by letter or numbers #TODO starts with only letter or number, can contain letter, nr,'-' resp = self.blobs.create_container(name) if resp: self.rebuild_container_list() log.info("CONTAINER %s CREATED" % name) else: raise FuseOSError(EACCES) log.error("Invalid container name or container already \ exists.") else: raise FuseOSError(ENOSYS) # TODO support 2nd+ level mkdirs def rmdir(self, path): if path.count('/') == 1: c_name = path[1:] resp = self.blobs.delete_container(c_name) if resp: if path in self.containers: del self.containers[path] else: raise FuseOSError(EACCES) else: raise FuseOSError(ENOSYS) # TODO support 2nd+ level mkdirs def create(self, path, mode): node = dict(st_mode=(S_IFREG | mode), st_size=0, st_nlink=1, st_uid=getuid(), st_mtime=time.time()) d, f = self._parse_path(path) if not f: log.error("Cannot create files on root level: /") raise FuseOSError(ENOSYS) dir = self._get_dir(d, True) if not dir: raise FuseOSError(EIO) dir['files'][f] = node return self.open(path, data='') # reusing handler provider def open(self, path, flags=0, data=None): if data == None: # download contents c_name = self.parse_container(path) f_name = path[path.find('/', 1) + 1:] try: data = self.blobs.get_blob(c_name, f_name) except WindowsAzureMissingResourceError: dir = self._get_dir('/' + c_name, True) if f_name in dir['files']: del dir['files'][f_name] raise FuseOSError(ENOENT) except WindowsAzureError as e: log.error("Read blob failed HTTP %d" % e.code) raise FuseOSError(EAGAIN) self.fd += 1 self.fds[self.fd] = (path, data, False) return self.fd def flush(self, path, fh=None): if not fh: raise FuseOSError(EIO) else: if fh not in self.fds: raise FuseOSError(EIO) path = self.fds[fh][0] data = self.fds[fh][1] dirty = self.fds[fh][2] if not dirty: return 0 # avoid redundant write d, f = self._parse_path(path) c_name = self.parse_container(path) if data is None: data = '' try: if len(data) < 64 * 1024 * 1024: # 64 mb self.blobs.put_blob(c_name, f, data, 'BlockBlob') else: # divide file by blocks and upload block_size = 8 * 1024 * 1024 num_blocks = int(math.ceil(len(data) * 1.0 / block_size)) rd = str(random.randint(1, 1e8)) block_ids = list() for i in range(num_blocks): part = data[i * block_size:min((i + 1) * block_size, len(data))] block_id = base64.encodestring('%s_%s' % (rd, (8 - len(str(i))) * '0' + str(i))) self.blobs.put_block(c_name, f, part, block_id) block_ids.append(block_id) self.blobs.put_block_list(c_name, f, block_ids) except WindowsAzureError: raise FuseOSError(EAGAIN) dir = self._get_dir(d, True) if not dir or f not in dir['files']: raise FuseOSError(EIO) # update local data dir['files'][f]['st_size'] = len(data) dir['files'][f]['st_mtime'] = time.time() self.fds[fh] = (path, data, False) # mark as not dirty return 0 def release(self, path, fh=None): if fh is not None and fh in self.fds: del self.fds[fh] def truncate(self, path, length, fh=None): return 0 # assume done, no need def write(self, path, data, offset, fh=None): if not fh or fh not in self.fds: raise FuseOSError(ENOENT) else: d = self.fds[fh][1] if d is None: d = "" self.fds[fh] = (self.fds[fh][0], d[:offset] + data, True) return len(data) def unlink(self, path): c_name = self.parse_container(path) d, f = self._parse_path(path) try: self.blobs.delete_blob(c_name, f) _dir = self._get_dir(path, True) if _dir and f in _dir['files']: del _dir['files'][f] return 0 except WindowsAzureMissingResourceError: raise FuseOSError(ENOENT) except Exception as e: raise FuseOSError(EAGAIN) def readdir(self, path, fh): if path == '/': return ['.', '..'] + [x[1:] for x in self.containers.keys() \ if x is not '/'] dir = self._get_dir(path, True) if not dir: raise FuseOSError(ENOENT) return ['.', '..'] + dir['files'].keys() def read(self, path, size, offset, fh): if not fh or fh not in self.fds: raise FuseOSError(ENOENT) f_name = path[path.find('/', 1) + 1:] c_name = path[1:path.find('/', 1)] try: data = self.blobs.get_blob(c_name, f_name) self.fds[fh] = (self.fds[fh][0], data, False) return data[offset:offset + size] except URLError, e: if e.code == 404: raise FuseOSError(ENOENT) elif e.code == 403: raise FUSEOSError(EPERM) else: log.error("Read blob failed HTTP %d" % e.code) raise FuseOSError(EAGAIN) data = self.fds[fh][1] if data is None: data = "" return data[offset:offset + size]
class AzureStorageBlockDeviceAPI(object): """ An ``IBlockDeviceAsyncAPI`` which uses Azure Storage Backed Block Devices Current Support: Azure SMS API """ def __init__(self, **azure_config): """ :param ServiceManagement azure_client: an instance of the azure serivce managment api client. :param String service_name: The name of the cloud service :param names of Azure volumes to identify cluster :returns: A ``BlockDeviceVolume``. """ self._instance_id = self.compute_instance_id() self._azure_service_client = ServiceManagementService( azure_config['subscription_id'], azure_config['management_certificate_path']) self._service_name = azure_config['service_name'] self._azure_storage_client = BlobService( azure_config['storage_account_name'], azure_config['storage_account_key']) self._storage_account_name = azure_config['storage_account_name'] self._disk_container_name = azure_config['disk_container_name'] if azure_config['debug']: to_file(sys.stdout) def allocation_unit(self): """ 1GiB is the minimum allocation unit for azure disks return int: 1 GiB """ return int(GiB(1).to_Byte().value) def compute_instance_id(self): """ Azure Stored a UUID in the SDC kernel module. """ # Node host names should be unique within a vnet return unicode(socket.gethostname()) def create_volume(self, dataset_id, size): """ Create a new volume. :param UUID dataset_id: The Flocker dataset ID of the dataset on this volume. :param int size: The size of the new volume in bytes. :returns: A ``Deferred`` that fires with a ``BlockDeviceVolume`` when the volume has been created. """ size_in_gb = Byte(size).to_GiB().value if size_in_gb % 1 != 0: raise UnsupportedVolumeSize(dataset_id) self._create_volume_blob(size, dataset_id) label = self._disk_label_for_dataset_id(str(dataset_id)) return BlockDeviceVolume( blockdevice_id=unicode(label), size=size, attached_to=None, dataset_id=self._dataset_id_for_disk_label(label)) def destroy_volume(self, blockdevice_id): """ Destroy an existing volume. :param unicode blockdevice_id: The unique identifier for the volume to destroy. :raises UnknownVolume: If the supplied ``blockdevice_id`` does not exist. :return: ``None`` """ log_info('Destorying block device: ' + str(blockdevice_id)) (target_disk, role_name, lun) = \ self._get_disk_vmname_lun(blockdevice_id) if target_disk is None: raise UnknownVolume(blockdevice_id) request = None if lun is not None: request = \ self._azure_service_client.delete_data_disk( service_name=self._service_name, deployment_name=self._service_name, role_name=target_disk.attached_to.role_name, lun=lun, delete_vhd=True) else: if target_disk.__class__.__name__ == 'Blob': # unregistered disk self._azure_storage_client.delete_blob( self._disk_container_name, target_disk.name) else: request = self._azure_service_client.delete_disk( target_disk.name, True) if request is not None: self._wait_for_async(request.request_id, 5000) self._wait_for_detach(blockdevice_id) def attach_volume(self, blockdevice_id, attach_to): """ Attach ``blockdevice_id`` to ``host``. :param unicode blockdevice_id: The unique identifier for the block device being attached. :param unicode attach_to: An identifier like the one returned by the ``compute_instance_id`` method indicating the node to which to attach the volume. :raises UnknownVolume: If the supplied ``blockdevice_id`` does not exist. :raises AlreadyAttachedVolume: If the supplied ``blockdevice_id`` is already attached. :returns: A ``BlockDeviceVolume`` with a ``host`` attribute set to ``host``. """ (target_disk, role_name, lun) = \ self._get_disk_vmname_lun(blockdevice_id) if target_disk is None: raise UnknownVolume(blockdevice_id) if lun is not None: raise AlreadyAttachedVolume(blockdevice_id) log_info('Attempting to attach ' + str(blockdevice_id) + ' to ' + str(attach_to)) disk_size = self._attach_disk(blockdevice_id, target_disk, attach_to) self._wait_for_attach(blockdevice_id) log_info('disk attached') return self._blockdevicevolume_from_azure_volume(blockdevice_id, disk_size, attach_to) def detach_volume(self, blockdevice_id): """ Detach ``blockdevice_id`` from whatever host it is attached to. :param unicode blockdevice_id: The unique identifier for the block device being detached. :raises UnknownVolume: If the supplied ``blockdevice_id`` does not exist. :raises UnattachedVolume: If the supplied ``blockdevice_id`` is not attached to anything. :returns: ``None`` """ (target_disk, role_name, lun) = \ self._get_disk_vmname_lun(blockdevice_id) if target_disk is None: raise UnknownVolume(blockdevice_id) if lun is None: raise UnattachedVolume(blockdevice_id) # contrary to function name it doesn't delete by default, just detachs request = \ self._azure_service_client.delete_data_disk( service_name=self._service_name, deployment_name=self._service_name, role_name=role_name, lun=lun) self._wait_for_async(request.request_id, 5000) self._wait_for_detach(blockdevice_id) def get_device_path(self, blockdevice_id): """ Return the device path that has been allocated to the block device on the host to which it is currently attached. :param unicode blockdevice_id: The unique identifier for the block device. :raises UnknownVolume: If the supplied ``blockdevice_id`` does not exist. :raises UnattachedVolume: If the supplied ``blockdevice_id`` is not attached to a host. :returns: A ``FilePath`` for the device. """ (target_disk_or_blob, role_name, lun) = \ self._get_disk_vmname_lun(blockdevice_id) if target_disk_or_blob is None: raise UnknownVolume(blockdevice_id) if lun is None: raise UnattachedVolume(blockdevice_id) return Lun.get_device_path_for_lun(lun) def list_volumes(self): """ List all the block devices available via the back end API. :returns: A ``list`` of ``BlockDeviceVolume``s. """ media_url_prefix = 'https://' + self._storage_account_name \ + '.blob.core.windows.net/' + self._disk_container_name disks = self._azure_service_client.list_disks() disk_list = [] all_blobs = self._get_flocker_blobs() for d in disks: if media_url_prefix not in d.media_link or \ 'flocker-' not in d.label: continue role_name = None if d.attached_to is not None \ and d.attached_to.role_name is not None: role_name = d.attached_to.role_name disk_list.append(self._blockdevicevolume_from_azure_volume( d.label, self._gibytes_to_bytes(d.logical_disk_size_in_gb), role_name)) if d.label in all_blobs: del all_blobs[d.label] for key in all_blobs: # include unregistered 'disk' blobs disk_list.append(self._blockdevicevolume_from_azure_volume( all_blobs[key].name, all_blobs[key].properties.content_length, None)) return disk_list def _attach_disk( self, blockdevice_id, target_disk, attach_to): """ Attaches disk to specified VM :param string blockdevice_id: The identifier of the disk :param DataVirtualHardDisk/Blob target_disk: The Blob or Disk to be attached :returns int: The size of the attached disk """ lun = Lun.compute_next_lun( self._azure_service_client, self._service_name, str(attach_to)) common_params = { 'service_name': self._service_name, 'deployment_name': self._service_name, 'role_name': attach_to, 'lun': lun } disk_size = None if target_disk.__class__.__name__ == 'Blob': # exclude 512 byte footer disk_size = target_disk.properties.content_length common_params['source_media_link'] = \ 'https://' + self._storage_account_name \ + '.blob.core.windows.net/' + self._disk_container_name \ + '/' + blockdevice_id common_params['disk_label'] = blockdevice_id else: disk_size = self._gibytes_to_bytes( target_disk.logical_disk_size_in_gb) common_params['disk_name'] = target_disk.name request = self._azure_service_client.add_data_disk(**common_params) self._wait_for_async(request.request_id, 5000) return disk_size def _create_volume_blob(self, size, dataset_id): # Create a new page blob as a blank disk self._azure_storage_client.put_blob( container_name=self._disk_container_name, blob_name=self._disk_label_for_dataset_id(dataset_id), blob=None, x_ms_blob_type='PageBlob', x_ms_blob_content_type='application/octet-stream', x_ms_blob_content_length=size) # for disk to be a valid vhd it requires a vhd footer # on the last 512 bytes vhd_footer = Vhd.generate_vhd_footer(size) self._azure_storage_client.put_page( container_name=self._disk_container_name, blob_name=self._disk_label_for_dataset_id(dataset_id), page=vhd_footer, x_ms_page_write='update', x_ms_range='bytes=' + str((size - 512)) + '-' + str(size - 1)) def _disk_label_for_dataset_id(self, dataset_id): """ Returns a disk label for a given Dataset ID :param unicode dataset_id: The identifier of the dataset :returns string: A string representing the disk label """ label = 'flocker-' + str(dataset_id) return label def _dataset_id_for_disk_label(self, disk_label): """ Returns a UUID representing the Dataset ID for the given disk label :param string disk_label: The disk label :returns UUID: The UUID of the dataset """ return UUID(disk_label.replace('flocker-', '')) def _get_disk_vmname_lun(self, blockdevice_id): target_disk = None target_lun = None role_name = None disk_list = self._azure_service_client.list_disks() for d in disk_list: if 'flocker-' not in d.label: continue if d.label == str(blockdevice_id): target_disk = d break if target_disk is None: # check for unregisterd disk blobs = self._get_flocker_blobs() blob = None if str(blockdevice_id) in blobs: blob = blobs[str(blockdevice_id)] return blob, None, None vm_info = None if hasattr(target_disk.attached_to, 'role_name'): vm_info = self._azure_service_client.get_role( self._service_name, self._service_name, target_disk.attached_to.role_name) for d in vm_info.data_virtual_hard_disks: if d.disk_name == target_disk.name: target_lun = d.lun break role_name = target_disk.attached_to.role_name return (target_disk, role_name, target_lun) def _get_flocker_blobs(self): all_blobs = {} blobs = self._azure_storage_client.list_blobs( self._disk_container_name, prefix='flocker-') for b in blobs: # todo - this could be big! all_blobs[b.name] = b return all_blobs def _wait_for_detach(self, blockdevice_id): role_name = '' lun = -1 timeout_count = 0 log_info('waiting for azure to ' + 'report disk as detached...') while role_name is not None or lun is not None: (target_disk, role_name, lun) = \ self._get_disk_vmname_lun(blockdevice_id) time.sleep(1) timeout_count += 1 if timeout_count > 5000: raise AsynchronousTimeout() log_info('Disk Detached') def _wait_for_attach(self, blockdevice_id): timeout_count = 0 lun = None log_info('waiting for azure to report disk as attached...') while lun is None: (target_disk, role_name, lun) = \ self._get_disk_vmname_lun(blockdevice_id) time.sleep(.001) timeout_count += 1 if timeout_count > 5000: raise AsynchronousTimeout() def _wait_for_async(self, request_id, timeout): count = 0 result = self._azure_service_client.get_operation_status(request_id) while result.status == 'InProgress': count = count + 1 if count > timeout: log_error('Timed out waiting for async operation to complete.') raise AsynchronousTimeout() time.sleep(.001) log_info('.') result = self._azure_service_client.get_operation_status( request_id) if result.error: log_error(result.error.code) log_error(str(result.error.message)) log_error(result.status + ' in ' + str(count * 5) + 's') def _gibytes_to_bytes(self, size): return int(GiB(size).to_Byte().value) def _blockdevicevolume_from_azure_volume(self, label, size, attached_to_name): # azure will report the disk size excluding the 512 byte footer # however flocker expects the exact value it requested for disk size # so offset the reported size to flocker by 512 bytes return BlockDeviceVolume( blockdevice_id=unicode(label), size=int(size), attached_to=attached_to_name, dataset_id=self._dataset_id_for_disk_label(label) ) # disk labels are formatted as flocker-<data_set_id>
class Command(BaseCommand): help = "Synchronizes static media to cloud files." option_list = BaseCommand.option_list + ( optparse.make_option('-w', '--wipe', action='store_true', dest='wipe', default=False, help="Wipes out entire contents of container first."), optparse.make_option('-t', '--test-run', action='store_true', dest='test_run', default=False, help="Performs a test run of the sync."), optparse.make_option('-c', '--container', dest='container', help="Override STATIC_CONTAINER."), ) # settings from azurite.settings ACCOUNT_NAME = AZURITE['ACCOUNT_NAME'] ACCOUNT_KEY = AZURITE['ACCOUNT_KEY'] STATIC_CONTAINER = AZURITE['STATIC_CONTAINER'] # paths DIRECTORY = os.path.abspath(settings.STATIC_ROOT) STATIC_URL = settings.STATIC_URL if not DIRECTORY.endswith('/'): DIRECTORY = DIRECTORY + '/' if STATIC_URL.startswith('/'): STATIC_URL = STATIC_URL[1:] local_object_names = [] create_count = 0 upload_count = 0 update_count = 0 skip_count = 0 delete_count = 0 service = None def handle(self, *args, **options): self.wipe = options.get('wipe') self.test_run = options.get('test_run') self.verbosity = int(options.get('verbosity')) if hasattr(options, 'container'): self.STATIC_CONTAINER = options.get('container') self.sync_files() def sync_files(self): self.service = BlobService(account_name=self.ACCOUNT_NAME, account_key=self.ACCOUNT_KEY) try: self.service.get_container_properties(self.STATIC_CONTAINER) except WindowsAzureMissingResourceError: self.service.create_container(self.STATIC_CONTAINER, x_ms_blob_public_access='blob') self.service.set_container_acl(self.STATIC_CONTAINER, x_ms_blob_public_access='blob') # if -w option is provided, wipe out the contents of the container if self.wipe: blob_count = len(self.service.list_blobs(self.STATIC_CONTAINER)) if self.test_run: print "Wipe would delete %d objects." % blob_count else: print "Deleting %d objects..." % blob_count for blob in self.service.list_blobs(self.STATIC_CONTAINER): self.service.delete_blob(self.STATIC_CONTAINER, blob.name) # walk through the directory, creating or updating files on the cloud os.path.walk(self.DIRECTORY, self.upload_files, "foo") # remove any files on remote that don't exist locally self.delete_files() # print out the final tally to the cmd line self.update_count = self.upload_count - self.create_count print if self.test_run: print "Test run complete with the following results:" print "Skipped %d. Created %d. Updated %d. Deleted %d." % ( self.skip_count, self.create_count, self.update_count, self.delete_count) def upload_files(self, arg, dirname, names): # upload or skip items for item in names: file_path = os.path.join(dirname, item) if os.path.isdir(file_path): continue # Don't try to upload directories object_name = self.STATIC_URL + file_path.split(self.DIRECTORY)[1] self.local_object_names.append(object_name) try: properties = self.service.get_blob_properties(self.STATIC_CONTAINER, object_name) except WindowsAzureMissingResourceError: properties = {} self.create_count += 1 cloud_datetime = None if 'last-modified' in properties: cloud_datetime = (properties['last-modified'] and datetime.datetime.strptime( properties['last-modified'], "%a, %d %b %Y %H:%M:%S %Z" ) or None) local_datetime = datetime.datetime.utcfromtimestamp( os.stat(file_path).st_mtime) if cloud_datetime and local_datetime < cloud_datetime: self.skip_count += 1 if self.verbosity > 1: print "Skipped %s: not modified." % object_name continue if not self.test_run: file_contents = open(file_path, 'r').read() content_type, encoding = mimetypes.guess_type(file_path) print "content-type", content_type print "encoding", encoding self.service.put_blob(self.STATIC_CONTAINER, object_name, file_contents, x_ms_blob_type='BlockBlob', x_ms_blob_content_type=content_type, content_encoding=encoding) # sync_headers(cloud_obj) self.upload_count += 1 if self.verbosity > 1: print "Uploaded", object_name def delete_files(self): # remove any objects in the container that don't exist locally for blob in self.service.list_blobs(self.STATIC_CONTAINER): if blob.name not in self.local_object_names: self.delete_count += 1 if self.verbosity > 1: print "Deleted %s" % blob.name if not self.test_run: self.service.delete_blob(self.STATIC_CONTAINER, blob.name)
class Storage(driver.Base): supports_bytes_range = True def __init__(self, path=None, config=None): self._config = config self._container = self._config.azure_storage_container protocol = 'https' if self._config.azure_use_https else 'http' acct_name = self._config.azure_storage_account_name acct_key = self._config.azure_storage_account_key self._blob = BlobService( account_name=acct_name, account_key=acct_key, protocol=protocol) self._init_container() logger.debug("Initialized azureblob storage driver") def _init_container(self): '''Initializes image container on Azure blob storage if the container does not exist. ''' created = self._blob.create_container( self._container, x_ms_blob_public_access='blob', fail_on_exist=False) if created: logger.info('Created blob container for image registry.') else: logger.debug('Registry container already exists.') return created @lru.get def get_content(self, path): try: return self._blob.get_blob(self._container, path) except azure.WindowsAzureMissingResourceError: raise exceptions.FileNotFoundError('%s is not there' % path) @lru.set def put_content(self, path, content): self._blob.put_blob(self._container, path, content, 'BlockBlob') return path def stream_read(self, path, bytes_range=None): try: f = io.BytesIO() self._blob.get_blob_to_file(self._container, path, f) if bytes_range: f.seek(bytes_range[0]) total_size = bytes_range[1] - bytes_range[0] + 1 else: f.seek(0) while True: buf = None if bytes_range: # Bytes Range is enabled buf_size = self.buffer_size if nb_bytes + buf_size > total_size: # We make sure we don't read out of the range buf_size = total_size - nb_bytes if buf_size > 0: buf = f.read(buf_size) nb_bytes += len(buf) else: # We're at the end of the range buf = '' else: buf = f.read(self.buffer_size) if not buf: break yield buf except IOError: raise exceptions.FileNotFoundError('%s is not there' % path) def stream_write(self, path, fp): self._blob.put_block_blob_from_file(self._container, path, fp) def list_directory(self, path=None): if not path.endswith('/'): path += '/' # path=a would list a/b.txt as well as 'abc.txt' blobs = list(self._blob.list_blobs(self._container, path)) if not blobs: raise exceptions.FileNotFoundError('%s is not there' % path) return [b.name for b in blobs] def exists(self, path): try: self._blob.get_blob_properties(self._container, path) return True except azure.WindowsAzureMissingResourceError: return False @lru.remove def remove(self, path): is_blob = self.exists(path) if is_blob: self._blob.delete_blob(self._container, path) return exists = False blobs = list(self._blob.list_blobs(self._container, path)) if not blobs: raise exceptions.FileNotFoundError('%s is not there' % path) for b in blobs: self._blob.delete_blob(self._container, b.name) def get_size(self, path): try: properties = self._blob.get_blob_properties(self._container, path) return int(properties['content-length']) # auto-converted to long except azure.WindowsAzureMissingResourceError: raise exceptions.FileNotFoundError('%s is not there' % path)
blobs. blob_service: Nombre del servicio de gestion de blobs @class BlobService """ for i in blob_service.list_containers().containers: print("Nombre del contenedor: {}".format(i.name)) print("Url del contenedor: {}".format(i.url)) print("##############################") for j in blob_service.list_blobs(i.name).blobs: print("\tNombre del Blob: {}".format(j.name)) print("\tUrl del Blob: {}".format(j.url)) print("\t------------------------------") blob_service = BlobService(credentials.account_name, credentials.account_key) getContainersWithBlobs(blob_service) f_blob = open('Ejercicio10.txt', "w") for i in blob_service.list_containers().containers: f_blob.write("Nombre del contenedor: {}".format(i.name)) f_blob.write("Url del contenedor: {}".format(i.url)) f_blob.write("##############################") for j in blob_service.list_blobs(i.name).blobs: f_blob.write("\tNombre del Blob: {}".format(j.name)) f_blob.write("\tUrl del Blob: {}".format(j.url)) f_blob.write("\t------------------------------") f_blob.close() blob_service.put_blob('code', 'f_blob.txt', f_blob, 'BlockBlob')
class Storage(driver.Base): supports_bytes_range = True def __init__(self, path=None, config=None): self._config = config self._container = self._config.azure_storage_container protocol = 'https' if self._config.azure_use_https else 'http' acct_name = self._config.azure_storage_account_name acct_key = self._config.azure_storage_account_key self._blob = BlobService(account_name=acct_name, account_key=acct_key, protocol=protocol) self._init_container() logger.debug("Initialized azureblob storage driver") def _init_container(self): '''Initializes image container on Azure blob storage if the container does not exist. ''' created = self._blob.create_container(self._container, x_ms_blob_public_access='blob', fail_on_exist=False) if created: logger.info('Created blob container for image registry.') else: logger.debug('Registry container already exists.') return created @lru.get def get_content(self, path): try: return self._blob.get_blob(self._container, path) except azure.WindowsAzureMissingResourceError: raise exceptions.FileNotFoundError('%s is not there' % path) @lru.set def put_content(self, path, content): self._blob.put_blob(self._container, path, content, 'BlockBlob') return path def stream_read(self, path, bytes_range=None): try: f = io.BytesIO() self._blob.get_blob_to_file(self._container, path, f) if bytes_range: f.seek(bytes_range[0]) total_size = bytes_range[1] - bytes_range[0] + 1 else: f.seek(0) while True: buf = None if bytes_range: # Bytes Range is enabled buf_size = self.buffer_size if nb_bytes + buf_size > total_size: # We make sure we don't read out of the range buf_size = total_size - nb_bytes if buf_size > 0: buf = f.read(buf_size) nb_bytes += len(buf) else: # We're at the end of the range buf = '' else: buf = f.read(self.buffer_size) if not buf: break yield buf except IOError: raise exceptions.FileNotFoundError('%s is not there' % path) def stream_write(self, path, fp): self._blob.put_block_blob_from_file(self._container, path, fp) def list_directory(self, path=None): if not path.endswith('/'): path += '/' # path=a would list a/b.txt as well as 'abc.txt' blobs = list(self._blob.list_blobs(self._container, path)) if not blobs: raise exceptions.FileNotFoundError('%s is not there' % path) return [b.name for b in blobs] def exists(self, path): try: self._blob.get_blob_properties(self._container, path) return True except azure.WindowsAzureMissingResourceError: return False @lru.remove def remove(self, path): is_blob = self.exists(path) if is_blob: self._blob.delete_blob(self._container, path) return exists = False blobs = list(self._blob.list_blobs(self._container, path)) if not blobs: raise exceptions.FileNotFoundError('%s is not there' % path) for b in blobs: self._blob.delete_blob(self._container, b.name) def get_size(self, path): try: properties = self._blob.get_blob_properties(self._container, path) return int(properties['content-length']) # auto-converted to long except azure.WindowsAzureMissingResourceError: raise exceptions.FileNotFoundError('%s is not there' % path)
def generate_and_upload(gauge_factory, config, logger): start = datetime.datetime.now() twitter_followers = gauge_factory('twitter.followers') twitter_tweets = gauge_factory('twitter.tweets') fb_friends = gauge_factory('facebook.friends') foursq_checkins = gauge_factory('foursquare.checkins') klout_score = gauge_factory('klout.score') runkeeper_activities = gauge_factory('runkeeper.activities') runkeeper_calories = gauge_factory('runkeeper.calories_burned') runkeeper_weight = gauge_factory('runkeeper.weight') tmp102_celsius = gauge_factory('tmp102.temperature', gauge_type='hourly') lastfm_listened = gauge_factory('lastfm.listened') jawbone_sleeps = gauge_factory('jawbone.sleeps') jawbone_heartrate = gauge_factory('jawbone.resting_heartrate') jawbone_steps = gauge_factory('jawbone.steps') jawbone_caffeine = gauge_factory('jawbone.caffeine') data = {} data_sources = [ # (output key, gauge, days back, aggregator, postprocessors) ('twitter.followers', twitter_followers, 30, None, [zero_fill_daily, interpolators.linear]), ('twitter.tweets', twitter_tweets, 20, None, [zero_fill_daily]), ('facebook.friends', fb_friends, 180, monthly_max, None), ('foursquare.checkins', foursq_checkins, 14, None, [zero_fill_daily]), ('lastfm.listened', lastfm_listened, 14, None, [zero_fill_daily]), ('klout.score', klout_score, 30, weekly_max, [zero_fill_weekly, interpolators.linear]), ('runkeeper.calories', runkeeper_calories, 60, weekly_sum, [zero_fill_weekly]), ('runkeeper.activities', runkeeper_activities, 60, weekly_sum, [zero_fill_weekly]), ('runkeeper.weight', runkeeper_weight, 180, weekly_min, [zero_fill_weekly, interpolators.linear]), ('sleeps', jawbone_sleeps, 14, None, [zero_fill_daily, interpolators.linear]), ('heartrate', jawbone_heartrate, 21, None, [zero_fill_daily, interpolators.linear]), ('steps', jawbone_steps, 14, None, [zero_fill_daily, interpolators.linear]), ('caffeine', jawbone_caffeine, 30, None, [zero_fill_daily]), ('tmp102.temperature', tmp102_celsius, 2.5, None, None) ] for ds in data_sources: data[ds[0]] = ds[1].aggregate(today_utc() - timedelta(days=ds[2]), aggregator=ds[3], post_processors=ds[4]) report = { 'generated': str(now_utc()), 'data': data, 'took': (datetime.datetime.now() - start).seconds } report_json = json.dumps(report, indent=4, default=json_date_serializer) report_content = '{0}({1})'.format(JSONP_CALLBACK_NAME, report_json) blob_service = BlobService(config['azure.account'], config['azure.key']) blob_service.create_container(config['azure.blob.container']) blob_service.set_container_acl(config['azure.blob.container'], x_ms_blob_public_access='container') blob_service.put_blob(config['azure.blob.container'], config['azure.blob.name'], report_content, 'BlockBlob') took = (datetime.datetime.now() - start).seconds logger.info('Report generated and uploaded. Took {0} s.'.format(took))
class AzureStorageBlockDeviceAPI(object): """ An ``IBlockDeviceAsyncAPI`` which uses Azure Storage Backed Block Devices Current Support: Azure SMS API """ def __init__(self, **azure_config): """ :param ServiceManagement azure_client: an instance of the azure serivce managment api client. :param String service_name: The name of the cloud service :param names of Azure volumes to identify cluster :returns: A ``BlockDeviceVolume``. """ self._instance_id = self.compute_instance_id() self._azure_service_client = ServiceManagementService( azure_config['subscription_id'], azure_config['management_certificate_path']) self._service_name = azure_config['service_name'] self._azure_storage_client = BlobService( azure_config['storage_account_name'], azure_config['storage_account_key']) self._storage_account_name = azure_config['storage_account_name'] self._disk_container_name = azure_config['disk_container_name'] if azure_config['debug']: to_file(sys.stdout) def allocation_unit(self): """ 1GiB is the minimum allocation unit for azure disks return int: 1 GiB """ return int(GiB(1).to_Byte().value) def compute_instance_id(self): """ Azure Stored a UUID in the SDC kernel module. """ # Node host names should be unique within a vnet return unicode(socket.gethostname()) def create_volume(self, dataset_id, size): """ Create a new volume. :param UUID dataset_id: The Flocker dataset ID of the dataset on this volume. :param int size: The size of the new volume in bytes. :returns: A ``Deferred`` that fires with a ``BlockDeviceVolume`` when the volume has been created. """ size_in_gb = Byte(size).to_GiB().value if size_in_gb % 1 != 0: raise UnsupportedVolumeSize(dataset_id) self._create_volume_blob(size, dataset_id) label = self._disk_label_for_dataset_id(str(dataset_id)) return BlockDeviceVolume( blockdevice_id=unicode(label), size=size, attached_to=None, dataset_id=self._dataset_id_for_disk_label(label)) def destroy_volume(self, blockdevice_id): """ Destroy an existing volume. :param unicode blockdevice_id: The unique identifier for the volume to destroy. :raises UnknownVolume: If the supplied ``blockdevice_id`` does not exist. :return: ``None`` """ log_info('Destorying block device: ' + str(blockdevice_id)) (target_disk, role_name, lun) = \ self._get_disk_vmname_lun(blockdevice_id) if target_disk is None: raise UnknownVolume(blockdevice_id) request = None if lun is not None: request = \ self._azure_service_client.delete_data_disk( service_name=self._service_name, deployment_name=self._service_name, role_name=target_disk.attached_to.role_name, lun=lun, delete_vhd=True) else: if target_disk.__class__.__name__ == 'Blob': # unregistered disk self._azure_storage_client.delete_blob( self._disk_container_name, target_disk.name) else: request = self._azure_service_client.delete_disk( target_disk.name, True) if request is not None: self._wait_for_async(request.request_id, 5000) self._wait_for_detach(blockdevice_id) def attach_volume(self, blockdevice_id, attach_to): """ Attach ``blockdevice_id`` to ``host``. :param unicode blockdevice_id: The unique identifier for the block device being attached. :param unicode attach_to: An identifier like the one returned by the ``compute_instance_id`` method indicating the node to which to attach the volume. :raises UnknownVolume: If the supplied ``blockdevice_id`` does not exist. :raises AlreadyAttachedVolume: If the supplied ``blockdevice_id`` is already attached. :returns: A ``BlockDeviceVolume`` with a ``host`` attribute set to ``host``. """ (target_disk, role_name, lun) = \ self._get_disk_vmname_lun(blockdevice_id) if target_disk is None: raise UnknownVolume(blockdevice_id) if lun is not None: raise AlreadyAttachedVolume(blockdevice_id) log_info('Attempting to attach ' + str(blockdevice_id) + ' to ' + str(attach_to)) disk_size = self._attach_disk(blockdevice_id, target_disk, attach_to) self._wait_for_attach(blockdevice_id) log_info('disk attached') return self._blockdevicevolume_from_azure_volume( blockdevice_id, disk_size, attach_to) def detach_volume(self, blockdevice_id): """ Detach ``blockdevice_id`` from whatever host it is attached to. :param unicode blockdevice_id: The unique identifier for the block device being detached. :raises UnknownVolume: If the supplied ``blockdevice_id`` does not exist. :raises UnattachedVolume: If the supplied ``blockdevice_id`` is not attached to anything. :returns: ``None`` """ (target_disk, role_name, lun) = \ self._get_disk_vmname_lun(blockdevice_id) if target_disk is None: raise UnknownVolume(blockdevice_id) if lun is None: raise UnattachedVolume(blockdevice_id) # contrary to function name it doesn't delete by default, just detachs request = \ self._azure_service_client.delete_data_disk( service_name=self._service_name, deployment_name=self._service_name, role_name=role_name, lun=lun) self._wait_for_async(request.request_id, 5000) self._wait_for_detach(blockdevice_id) def get_device_path(self, blockdevice_id): """ Return the device path that has been allocated to the block device on the host to which it is currently attached. :param unicode blockdevice_id: The unique identifier for the block device. :raises UnknownVolume: If the supplied ``blockdevice_id`` does not exist. :raises UnattachedVolume: If the supplied ``blockdevice_id`` is not attached to a host. :returns: A ``FilePath`` for the device. """ (target_disk_or_blob, role_name, lun) = \ self._get_disk_vmname_lun(blockdevice_id) if target_disk_or_blob is None: raise UnknownVolume(blockdevice_id) if lun is None: raise UnattachedVolume(blockdevice_id) return Lun.get_device_path_for_lun(lun) def list_volumes(self): """ List all the block devices available via the back end API. :returns: A ``list`` of ``BlockDeviceVolume``s. """ media_url_prefix = 'https://' + self._storage_account_name \ + '.blob.core.windows.net/' + self._disk_container_name disks = self._azure_service_client.list_disks() disk_list = [] all_blobs = self._get_flocker_blobs() for d in disks: if media_url_prefix not in d.media_link or \ 'flocker-' not in d.label: continue role_name = None if d.attached_to is not None \ and d.attached_to.role_name is not None: role_name = d.attached_to.role_name disk_list.append( self._blockdevicevolume_from_azure_volume( d.label, self._gibytes_to_bytes(d.logical_disk_size_in_gb), role_name)) if d.label in all_blobs: del all_blobs[d.label] for key in all_blobs: # include unregistered 'disk' blobs disk_list.append( self._blockdevicevolume_from_azure_volume( all_blobs[key].name, all_blobs[key].properties.content_length, None)) return disk_list def _attach_disk(self, blockdevice_id, target_disk, attach_to): """ Attaches disk to specified VM :param string blockdevice_id: The identifier of the disk :param DataVirtualHardDisk/Blob target_disk: The Blob or Disk to be attached :returns int: The size of the attached disk """ lun = Lun.compute_next_lun(self._azure_service_client, self._service_name, str(attach_to)) common_params = { 'service_name': self._service_name, 'deployment_name': self._service_name, 'role_name': attach_to, 'lun': lun } disk_size = None if target_disk.__class__.__name__ == 'Blob': # exclude 512 byte footer disk_size = target_disk.properties.content_length common_params['source_media_link'] = \ 'https://' + self._storage_account_name \ + '.blob.core.windows.net/' + self._disk_container_name \ + '/' + blockdevice_id common_params['disk_label'] = blockdevice_id else: disk_size = self._gibytes_to_bytes( target_disk.logical_disk_size_in_gb) common_params['disk_name'] = target_disk.name request = self._azure_service_client.add_data_disk(**common_params) self._wait_for_async(request.request_id, 5000) return disk_size def _create_volume_blob(self, size, dataset_id): # Create a new page blob as a blank disk self._azure_storage_client.put_blob( container_name=self._disk_container_name, blob_name=self._disk_label_for_dataset_id(dataset_id), blob=None, x_ms_blob_type='PageBlob', x_ms_blob_content_type='application/octet-stream', x_ms_blob_content_length=size) # for disk to be a valid vhd it requires a vhd footer # on the last 512 bytes vhd_footer = Vhd.generate_vhd_footer(size) self._azure_storage_client.put_page( container_name=self._disk_container_name, blob_name=self._disk_label_for_dataset_id(dataset_id), page=vhd_footer, x_ms_page_write='update', x_ms_range='bytes=' + str((size - 512)) + '-' + str(size - 1)) def _disk_label_for_dataset_id(self, dataset_id): """ Returns a disk label for a given Dataset ID :param unicode dataset_id: The identifier of the dataset :returns string: A string representing the disk label """ label = 'flocker-' + str(dataset_id) return label def _dataset_id_for_disk_label(self, disk_label): """ Returns a UUID representing the Dataset ID for the given disk label :param string disk_label: The disk label :returns UUID: The UUID of the dataset """ return UUID(disk_label.replace('flocker-', '')) def _get_disk_vmname_lun(self, blockdevice_id): target_disk = None target_lun = None role_name = None disk_list = self._azure_service_client.list_disks() for d in disk_list: if 'flocker-' not in d.label: continue if d.label == str(blockdevice_id): target_disk = d break if target_disk is None: # check for unregisterd disk blobs = self._get_flocker_blobs() blob = None if str(blockdevice_id) in blobs: blob = blobs[str(blockdevice_id)] return blob, None, None vm_info = None if hasattr(target_disk.attached_to, 'role_name'): vm_info = self._azure_service_client.get_role( self._service_name, self._service_name, target_disk.attached_to.role_name) for d in vm_info.data_virtual_hard_disks: if d.disk_name == target_disk.name: target_lun = d.lun break role_name = target_disk.attached_to.role_name return (target_disk, role_name, target_lun) def _get_flocker_blobs(self): all_blobs = {} blobs = self._azure_storage_client.list_blobs( self._disk_container_name, prefix='flocker-') for b in blobs: # todo - this could be big! all_blobs[b.name] = b return all_blobs def _wait_for_detach(self, blockdevice_id): role_name = '' lun = -1 timeout_count = 0 log_info('waiting for azure to ' + 'report disk as detached...') while role_name is not None or lun is not None: (target_disk, role_name, lun) = \ self._get_disk_vmname_lun(blockdevice_id) time.sleep(1) timeout_count += 1 if timeout_count > 5000: raise AsynchronousTimeout() log_info('Disk Detached') def _wait_for_attach(self, blockdevice_id): timeout_count = 0 lun = None log_info('waiting for azure to report disk as attached...') while lun is None: (target_disk, role_name, lun) = \ self._get_disk_vmname_lun(blockdevice_id) time.sleep(.001) timeout_count += 1 if timeout_count > 5000: raise AsynchronousTimeout() def _wait_for_async(self, request_id, timeout): count = 0 result = self._azure_service_client.get_operation_status(request_id) while result.status == 'InProgress': count = count + 1 if count > timeout: log_error('Timed out waiting for async operation to complete.') raise AsynchronousTimeout() time.sleep(.001) log_info('.') result = self._azure_service_client.get_operation_status( request_id) if result.error: log_error(result.error.code) log_error(str(result.error.message)) log_error(result.status + ' in ' + str(count * 5) + 's') def _gibytes_to_bytes(self, size): return int(GiB(size).to_Byte().value) def _blockdevicevolume_from_azure_volume(self, label, size, attached_to_name): # azure will report the disk size excluding the 512 byte footer # however flocker expects the exact value it requested for disk size # so offset the reported size to flocker by 512 bytes return BlockDeviceVolume( blockdevice_id=unicode(label), size=int(size), attached_to=attached_to_name, dataset_id=self._dataset_id_for_disk_label( label)) # disk labels are formatted as flocker-<data_set_id>
class Command(BaseCommand): help = "Synchronizes static media to cloud files." option_list = BaseCommand.option_list + ( optparse.make_option( '-w', '--wipe', action='store_true', dest='wipe', default=False, help="Wipes out entire contents of container first."), optparse.make_option('-t', '--test-run', action='store_true', dest='test_run', default=False, help="Performs a test run of the sync."), optparse.make_option('-c', '--container', dest='container', help="Override STATIC_CONTAINER."), ) # settings from azurite.settings ACCOUNT_NAME = AZURITE['ACCOUNT_NAME'] ACCOUNT_KEY = AZURITE['ACCOUNT_KEY'] STATIC_CONTAINER = AZURITE['STATIC_CONTAINER'] # paths DIRECTORY = os.path.abspath(settings.STATIC_ROOT) STATIC_URL = settings.STATIC_URL if not DIRECTORY.endswith('/'): DIRECTORY = DIRECTORY + '/' if STATIC_URL.startswith('/'): STATIC_URL = STATIC_URL[1:] local_object_names = [] create_count = 0 upload_count = 0 update_count = 0 skip_count = 0 delete_count = 0 service = None def handle(self, *args, **options): self.wipe = options.get('wipe') self.test_run = options.get('test_run') self.verbosity = int(options.get('verbosity')) if hasattr(options, 'container'): self.STATIC_CONTAINER = options.get('container') self.sync_files() def sync_files(self): self.service = BlobService(account_name=self.ACCOUNT_NAME, account_key=self.ACCOUNT_KEY) try: self.service.get_container_properties(self.STATIC_CONTAINER) except WindowsAzureMissingResourceError: self.service.create_container(self.STATIC_CONTAINER, x_ms_blob_public_access='blob') self.service.set_container_acl(self.STATIC_CONTAINER, x_ms_blob_public_access='blob') # if -w option is provided, wipe out the contents of the container if self.wipe: blob_count = len(self.service.list_blobs(self.STATIC_CONTAINER)) if self.test_run: print "Wipe would delete %d objects." % blob_count else: print "Deleting %d objects..." % blob_count for blob in self.service.list_blobs(self.STATIC_CONTAINER): self.service.delete_blob(self.STATIC_CONTAINER, blob.name) # walk through the directory, creating or updating files on the cloud os.path.walk(self.DIRECTORY, self.upload_files, "foo") # remove any files on remote that don't exist locally self.delete_files() # print out the final tally to the cmd line self.update_count = self.upload_count - self.create_count print if self.test_run: print "Test run complete with the following results:" print "Skipped %d. Created %d. Updated %d. Deleted %d." % ( self.skip_count, self.create_count, self.update_count, self.delete_count) def upload_files(self, arg, dirname, names): # upload or skip items for item in names: file_path = os.path.join(dirname, item) if os.path.isdir(file_path): continue # Don't try to upload directories object_name = self.STATIC_URL + file_path.split(self.DIRECTORY)[1] self.local_object_names.append(object_name) try: properties = self.service.get_blob_properties( self.STATIC_CONTAINER, object_name) except WindowsAzureMissingResourceError: properties = {} self.create_count += 1 cloud_datetime = None if 'last-modified' in properties: cloud_datetime = ( properties['last-modified'] and datetime.datetime.strptime(properties['last-modified'], "%a, %d %b %Y %H:%M:%S %Z") or None) local_datetime = datetime.datetime.utcfromtimestamp( os.stat(file_path).st_mtime) if cloud_datetime and local_datetime < cloud_datetime: self.skip_count += 1 if self.verbosity > 1: print "Skipped %s: not modified." % object_name continue if not self.test_run: file_contents = open(file_path, 'r').read() content_type, encoding = mimetypes.guess_type(file_path) self.service.put_blob(self.STATIC_CONTAINER, object_name, file_contents, x_ms_blob_type='BlockBlob', x_ms_blob_content_type=content_type, content_encoding=encoding) # sync_headers(cloud_obj) self.upload_count += 1 if self.verbosity > 1: print "Uploaded", object_name def delete_files(self): # remove any objects in the container that don't exist locally for blob in self.service.list_blobs(self.STATIC_CONTAINER): if blob.name not in self.local_object_names: self.delete_count += 1 if self.verbosity > 1: print "Deleted %s" % blob.name if not self.test_run: self.service.delete_blob(self.STATIC_CONTAINER, blob.name)
def uri_put_file(creds, uri, fp, content_encoding=None): assert fp.tell() == 0 assert uri.startswith('wabs://') def log_upload_failures_on_error(exc_tup, exc_processor_cxt): def standard_detail_message(prefix=''): return (prefix + ' There have been {n} attempts to upload ' 'file {url} so far.'.format(n=exc_processor_cxt, url=uri)) typ, value, tb = exc_tup del exc_tup # Screen for certain kinds of known-errors to retry from if issubclass(typ, socket.error): socketmsg = value[1] if isinstance(value, tuple) else value logger.info( msg='Retrying upload because of a socket error', detail=standard_detail_message( "The socket error's message is '{0}'.".format(socketmsg))) else: # For all otherwise untreated exceptions, report them as a # warning and retry anyway -- all exceptions that can be # justified should be treated and have error messages # listed. logger.warning( msg='retrying file upload from unexpected exception', detail=standard_detail_message( 'The exception type is {etype} and its value is ' '{evalue} and its traceback is {etraceback}'.format( etype=typ, evalue=value, etraceback=''.join(traceback.format_tb(tb))))) # Help Python GC by resolving possible cycles del tb # Because we're uploading in chunks, catch rate limiting and # connection errors which occur for each individual chunk instead of # failing the whole file and restarting. @retry(retry_with_count(log_upload_failures_on_error)) def upload_chunk(chunk, block_id): check_sum = base64.encodestring(md5(chunk).digest()).strip('\n') conn.put_block(url_tup.netloc, url_tup.path, chunk, block_id, content_md5=check_sum) url_tup = urlparse(uri) kwargs = dict(x_ms_blob_type='BlockBlob') if content_encoding is not None: kwargs['x_ms_blob_content_encoding'] = content_encoding conn = BlobService(creds.account_name, creds.account_key, protocol='https') conn.put_blob(url_tup.netloc, url_tup.path, '', **kwargs) # WABS requires large files to be uploaded in 4MB chunks block_ids = [] length, index = 0, 0 pool_size = os.getenv('WABS_UPLOAD_POOL_SIZE', 5) p = gevent.pool.Pool(size=pool_size) while True: data = fp.read(WABS_CHUNK_SIZE) if data: length += len(data) block_id = base64.b64encode(str(index)) p.wait_available() p.spawn(upload_chunk, data, block_id) block_ids.append(block_id) index += 1 else: p.join() break conn.put_block_list(url_tup.netloc, url_tup.path, block_ids) # To maintain consistency with the S3 version of this function we must # return an object with a certain set of attributes. Currently, that set # of attributes consists of only 'size' return _Key(size=len(data))
def uri_put_file(creds, uri, fp, content_encoding=None): assert fp.tell() == 0 assert uri.startswith('wabs://') def log_upload_failures_on_error(exc_tup, exc_processor_cxt): def standard_detail_message(prefix=''): return (prefix + ' There have been {n} attempts to upload ' 'file {url} so far.'.format(n=exc_processor_cxt, url=uri)) typ, value, tb = exc_tup del exc_tup # Screen for certain kinds of known-errors to retry from if issubclass(typ, socket.error): socketmsg = value[1] if isinstance(value, tuple) else value logger.info( msg='Retrying upload because of a socket error', detail=standard_detail_message( "The socket error's message is '{0}'." .format(socketmsg))) else: # For all otherwise untreated exceptions, report them as a # warning and retry anyway -- all exceptions that can be # justified should be treated and have error messages # listed. logger.warning( msg='retrying file upload from unexpected exception', detail=standard_detail_message( 'The exception type is {etype} and its value is ' '{evalue} and its traceback is {etraceback}' .format(etype=typ, evalue=value, etraceback=''.join(traceback.format_tb(tb))))) # Help Python GC by resolving possible cycles del tb # Because we're uploading in chunks, catch rate limiting and # connection errors which occur for each individual chunk instead of # failing the whole file and restarting. @retry(retry_with_count(log_upload_failures_on_error)) def upload_chunk(chunk, block_id): check_sum = base64.encodestring(md5(chunk).digest()).strip('\n') conn.put_block(url_tup.netloc, url_tup.path, chunk, block_id, content_md5=check_sum) url_tup = urlparse(uri) kwargs = dict(x_ms_blob_type='BlockBlob') if content_encoding is not None: kwargs['x_ms_blob_content_encoding'] = content_encoding conn = BlobService(creds.account_name, creds.account_key, protocol='https') conn.put_blob(url_tup.netloc, url_tup.path, '', **kwargs) # WABS requires large files to be uploaded in 4MB chunks block_ids = [] length, index = 0, 0 pool_size = os.getenv('WABS_UPLOAD_POOL_SIZE', 5) p = gevent.pool.Pool(size=pool_size) while True: data = fp.read(WABS_CHUNK_SIZE) if data: length += len(data) block_id = base64.b64encode(str(index)) p.wait_available() p.spawn(upload_chunk, data, block_id) block_ids.append(block_id) index += 1 else: p.join() break conn.put_block_list(url_tup.netloc, url_tup.path, block_ids) # To maintain consistency with the S3 version of this function we must # return an object with a certain set of attributes. Currently, that set # of attributes consists of only 'size' return _Key(size=len(data))
class AzureBlobStorage(Storage): def __init__(self, account='nyxstorage', container='pxo'): self.base_storage_uri = 'http://%s.blob.core.windows.net/%s/' % ( account, container) self.blob_service = BlobService( account, get_env_variable('AZURE_BLOB_STORAGE_KEY')) self.container = container def _open(self, name, mode='rb'): data = self.blob_service.get_blob(self.container, name) return ContentFile(data) def _save(self, name, content): _file = content.read() file_name = content.name[-35:] self.blob_service.put_blob( self.container, file_name, _file, x_ms_blob_type='BlockBlob') return self.base_storage_uri + file_name def create_container(self, container_name): result = self.blob_service.create_container( container_name, x_ms_blob_public_access='container') return result def delete(self, name): self.blob_service.delete_blob(self.container, name) def exists(self, name): try: self.blob_service.get_blob_properties(self.container, name) except: return False else: return True def get_available_name(self, name): return name def get_blobs(self): blobs = self.blob_service.list_blobs(self.container) return blobs def get_valid_name(self, name): return name def modified_time(self, name): metadata = self.blob_service.get_blob_metadata(self.container, name) modified_time = float(metadata.get('x-ms-meta-modified_time')) return datetime.fromtimestamp(modified_time) def set_public_container(self, container_name): result = self.blob_service.set_container_acl( container_name, x_ms_blob_public_access='container') return result def size(self, name): properties = self.blob_service.get_blob_properties( self.container, name) return properties.get('content-length') def url(self, name): blob = self.blob_service.list_blobs(self.container, prefix=name) return blob.blobs[0].url