def test_foreign_volume(self): """ Non-Flocker Volumes are not listed. """ cinder_client = self.cinder_client() requested_volume = cinder_client.create( size=int(Byte(self.minimum_allocatable_size).to_GiB().value)) CINDER_VOLUME(id=requested_volume.id).write() self.addCleanup( cinder_client.delete, requested_volume.id, ) wait_for_volume_state( volume_manager=cinder_client, expected_volume=requested_volume, desired_state=u'available', transient_states=(u'creating', ), ) self.assertEqual([], self.api.list_volumes())
def _pod_stats(pod): cpu = sum( map( parse_cpu, ( container["usage"]["cpu"] for container in pod["containers"] ), ), 0, ) mem = sum( map( lambda s: parse_memory(s).amount, ( container["usage"]["memory"] for container in pod["containers"] ), ), Byte(0), ) return (_CPU(cpu), _Memory(mem))
def test_foreign_volume(self): """ ``list_volumes`` lists only those volumes belonging to the current Flocker cluster. """ try: config = get_blockdevice_config(ProviderType.aws) except InvalidConfig as e: raise SkipTest(str(e)) ec2_client = get_ec2_client_for_test(config) requested_volume = ec2_client.connection.create_volume( int(Byte(self.minimum_allocatable_size).to_GiB().value), ec2_client.zone) self.addCleanup(ec2_client.connection.delete_volume, requested_volume.id) _wait_for_volume_state_change(VolumeOperations.CREATE, requested_volume) self.assertEqual(self.api.list_volumes(), [])
def test_parsenum_all_sizes(self, expr, size): """ Send standard size expressions to ``parse_num`` in many sizes, we expect to get correct size results. :param expr str: A string representing the size expression :param size int: A string representing the volume size """ if expr is "KB": expected_size = int(KiB(size).to_Byte()) elif expr is "MB": expected_size = int(MiB(size).to_Byte()) elif expr is "GB": expected_size = int(GiB(size).to_Byte()) elif expr is "TB": expected_size = int(TiB(size).to_Byte()) else: expected_size = int(Byte(size).to_Byte()) return self.assertEqual(expected_size, int(parse_num(str(size) + expr).to_Byte()))
def test_foreign_volume(self): """ Non-Flocker Volumes are not listed. """ try: cls, kwargs = get_blockdeviceapi_args(ProviderType.openstack) except InvalidConfig as e: raise SkipTest(str(e)) cinder_client = kwargs["cinder_client"] requested_volume = cinder_client.volumes.create( size=int(Byte(self.minimum_allocatable_size).to_GiB().value) ) self.addCleanup( cinder_client.volumes.delete, requested_volume.id, ) wait_for_volume( volume_manager=cinder_client.volumes, expected_volume=requested_volume ) self.assertEqual([], self.api.list_volumes())
def test_foreign_volume(self): """ ``list_volumes`` lists only those volumes belonging to the current Flocker cluster. """ try: config = get_blockdevice_config() except InvalidConfig as e: self.skipTest(str(e)) ec2_client = get_ec2_client_for_test(config) meta_client = ec2_client.connection.meta.client requested_volume = meta_client.create_volume( Size=int(Byte(self.minimum_allocatable_size).to_GiB().value), AvailabilityZone=ec2_client.zone) created_volume = ec2_client.connection.Volume( requested_volume['VolumeId']) self.addCleanup(created_volume.delete) _wait_for_volume_state_change(VolumeOperations.CREATE, created_volume) self.assertEqual(self.api.list_volumes(), [])
def create_volume(self, dataset_id, size): blockdevice_id = _dataset_id_to_blockdevice_id(dataset_id) sizeGiB = int(Byte(size).to_GiB()) config = dict( name=blockdevice_id, sizeGb=sizeGiB, description=self._disk_resource_description(), ) # TODO(mewert): Verify timeout and error conditions. self._do_blocking_operation( self._compute.disks().insert, body=config) # TODO(mewert): Test creating a volume in cluster A in this project # with the same UUID as a volume in cluster B in the same project. # make that the logs and errors make this error obvious to the user return BlockDeviceVolume( blockdevice_id=blockdevice_id, size=int(GiB(sizeGiB).to_Byte()), attached_to=None, dataset_id=dataset_id, )
def create_volume(self, dataset_id, size): """ See ``IBlockDeviceAPI.create_volume``. :returns: ``BlockDeviceVolume`` when the volume has been created. """ volume = { 'id': unicode(dataset_id), 'size': int(Byte(size).to_GB().value), 'actual_size': size, 'host': self._generate_host(), 'attach_to': None } provider_location = self._create_vmax_vomume(volume) volume['name'] = provider_location['keybindings']['DeviceID'] volume['provider_location'] = unicode(provider_location) blockdevice_id = self.dbconn.add_volume(volume) return _blockdevicevolume_from_vmax_volume(blockdevice_id, volume)
def test_disk_attachment_fails_with_conflicting_disk(self): """ create_server_volume will raise an exception when Cinder attempts to attach a device to a path that is in use by a non-Cinder volume. """ instance_id = self.blockdevice_api.compute_instance_id() host_device = "/dev/null" tmpdir = FilePath(self.mktemp()) tmpdir.makedirs() virtio = VirtIOClient.using_insecure_tls(instance_id, tmpdir) virtio.attach_disk(host_device, "vdb") self.addCleanup(virtio.detach_disk, host_device) cinder_volume = self.cinder.create( size=int(Byte(get_minimum_allocatable_size()).to_GiB().value)) CINDER_VOLUME(id=cinder_volume.id).write() self.addCleanup(self._cleanup, instance_id, cinder_volume) volume = wait_for_volume_state(volume_manager=self.cinder, expected_volume=cinder_volume, desired_state=u'available', transient_states=(u'creating', )) attached_volume = self.nova.create_server_volume( server_id=instance_id, volume_id=volume.id, device=None, ) with self.assertRaises(UnexpectedStateException) as e: wait_for_volume_state( volume_manager=self.cinder, expected_volume=attached_volume, desired_state=u'in-use', transient_states=( u'available', u'attaching', ), ) self.assertEqual(e.exception.unexpected_state, u'available')
def test_get_device_path_no_attached_disks(self): """ get_device_path returns the most recently attached device """ instance_id = self.blockdevice_api.compute_instance_id() cinder_volume = self.cinder.create( size=int(Byte(get_minimum_allocatable_size()).to_GiB().value)) CINDER_VOLUME(id=cinder_volume.id).write() self.addCleanup(self._cleanup, instance_id, cinder_volume) volume = wait_for_volume_state(volume_manager=self.cinder, expected_volume=cinder_volume, desired_state=u'available', transient_states=(u'creating', )) devices_before = set(FilePath('/dev').children()) attached_volume = self.nova.create_server_volume( server_id=instance_id, volume_id=volume.id, device=None, ) volume = wait_for_volume_state( volume_manager=self.cinder, expected_volume=attached_volume, desired_state=u'in-use', transient_states=( u'available', u'attaching', ), ) devices_after = set(FilePath('/dev').children()) new_devices = devices_after - devices_before [new_device] = new_devices device_path = self.blockdevice_api.get_device_path(volume.id) self.assertEqual(device_path.realpath(), new_device)
def test_get_device_path_virtio_blk_symlink(self): """ ``get_device_path`` on systems using the virtio_blk driver returns the target of a symlink matching ``/dev/disks/by-id/virtio-<volume.id>``. """ instance_id = self.blockdevice_api.compute_instance_id() # Create volume cinder_volume = self.cinder.volumes.create( size=int(Byte(get_minimum_allocatable_size()).to_GiB().value) ) CINDER_VOLUME(id=cinder_volume.id).write() self.addCleanup(self._cleanup, instance_id, cinder_volume) volume = wait_for_volume_state( volume_manager=self.cinder.volumes, expected_volume=cinder_volume, desired_state=u'available', transient_states=(u'creating',)) # Attach volume attached_volume = self.nova.volumes.create_server_volume( server_id=instance_id, volume_id=volume.id, device=None, ) volume = wait_for_volume_state( volume_manager=self.cinder.volumes, expected_volume=attached_volume, desired_state=u'in-use', transient_states=(u'available', u'attaching',), ) self.assertEqual( FilePath( '/dev/disk/by-id/virtio-{}'.format(volume.id[:20]) ).realpath(), self.blockdevice_api.get_device_path( volume.id ) )
def create_volume(self, dataset_id, size): """ Create a volume on EBS. Store Flocker-specific {metadata version, cluster id, dataset id} for the volume as volume tag data. Open issues: https://clusterhq.atlassian.net/browse/FLOC-1792 """ requested_volume = self.connection.create_volume(size=int( Byte(size).to_GiB().value), zone=self.zone) message_type = BOTO_LOG_RESULT + u':created_volume' Message.new(message_type=message_type, volume_id=unicode(requested_volume.id), dataset_id=unicode(dataset_id), size=unicode(size)).write() # Stamp created volume with Flocker-specific tags. metadata = { METADATA_VERSION_LABEL: '1', CLUSTER_ID_LABEL: unicode(self.cluster_id), DATASET_ID_LABEL: unicode(dataset_id), # EC2 convention for naming objects, e.g. as used in EC2 web # console (http://stackoverflow.com/a/12798180). "Name": u"flocker-{}".format(dataset_id), } self.connection.create_tags([requested_volume.id], metadata) message_type = BOTO_LOG_RESULT + u':created_tags' Message.new(message_type=message_type, requested_volume=requested_volume.id, tags=metadata).write() # Wait for created volume to reach 'available' state. _wait_for_volume_state_change(VolumeOperations.CREATE, requested_volume) # Return created volume in BlockDeviceVolume format. return _blockdevicevolume_from_ebs_volume(requested_volume)
def create_volume(self, dataset_id, size): """ Create a new volume. :param UUID dataset_id: The Flocker dataset ID of the dataset on this volume. :param int size: The size of the new volume in bytes. :returns: A ``Deferred`` that fires with a ``BlockDeviceVolume`` when the volume has been created. """ size_in_gb = Byte(size).to_GiB().value if size_in_gb % 1 != 0: raise UnsupportedVolumeSize(dataset_id) disk_label = self._disk_label_for_dataset_id(dataset_id) self._manager.create_disk(disk_label, size_in_gb) return BlockDeviceVolume( blockdevice_id=unicode(disk_label), size=size, attached_to=None, dataset_id=dataset_id)
def test_foreign_volume(self): """ Test that ``list_volumes`` lists only those volumes belonging to the current Flocker cluster. """ try: cls, kwargs = get_blockdeviceapi_args(ProviderType.aws) except InvalidConfig as e: raise SkipTest(str(e)) ec2_client = kwargs["ec2_client"] requested_volume = ec2_client.connection.create_volume( int(Byte(self.minimum_allocatable_size).to_GiB().value), ec2_client.zone) self.addCleanup(ec2_client.connection.delete_volume, requested_volume.id) _wait_for_volume(requested_volume, u'', u'creating', u'available') self.assertEqual(self.api.list_volumes(), [])
def create_volume_with_profile(self, dataset_id, size, profile_name): """ Create a new volume with the specified profile. When called by ``IDeployer``, the supplied size will be rounded up to the nearest ``IBlockDeviceAPI.allocation_unit()``. :param UUID dataset_id: The Flocker dataset ID of the dataset on this volume. :param int size: The size of the new volume in bytes. :param unicode profile_name: The name of the storage profile for this volume. :returns: A ``BlockDeviceVolume`` of the newly created volume. """ if profile_name not in self.get_profile_list(): LOG.error(u'Ignoring unknown profile name ' + unicode(profile_name)) profile_name = self._get_default_profile() blockdevice_id = unicode(dataset_id) volume = { 'id': self._blockdevice_id_to_volume_id(blockdevice_id), 'size': int(Byte(size).to_GB().value), 'actual_size': size, 'PROFILE': profile_name, 'host': self._generate_host(profile=profile_name), 'attach_to': None } provider_location = self._create_vmax_volume(volume) volume['name'] = provider_location['keybindings']['DeviceID'] volume['provider_location'] = unicode(provider_location) return _blockdevicevolume_from_vmax_volume(blockdevice_id, volume)
def parse_num(expression): """ Parse a string of a dataset size 10g, 100kib etc into a usable integer. If user doesn't submit a correct size, give back the default size. :param expression: the dataset expression to parse. """ if not expression: return DEFAULT_SIZE if type(expression) is unicode: expression = str(expression) def _match(exp, search=re.compile( r'^(\d+){1}([KMGTkmgt][IiBb]){0,1}([Bb]){0,1}').search): return bool(search(exp)) if _match(expression): unit = expression.translate(None, "1234567890.") num = int(expression.replace(unit, "")) unit = unit.lower() if unit == 'tb' or unit == 't' or unit == 'tib': return TiB(num) elif unit == 'gb' or unit == 'g' or unit == 'gib': return GiB(num) elif unit == 'mb' or unit == 'm' or unit == 'mib': return MiB(num) elif unit == 'kb' or unit == 'k' or unit == 'kib': return KiB(num) elif unit == '': return Byte(num) else: return DEFAULT_SIZE else: return DEFAULT_SIZE
def test_foreign_volume(self): """ Non-Flocker Volumes are not listed. """ try: config = get_blockdevice_config(ProviderType.openstack) except InvalidConfig as e: raise SkipTest(str(e)) session = get_keystone_session(**config) region = get_openstack_region_for_test() cinder_client = get_cinder_v1_client(session, region) requested_volume = cinder_client.volumes.create( size=int(Byte(self.minimum_allocatable_size).to_GiB().value)) self.addCleanup( cinder_client.volumes.delete, requested_volume.id, ) wait_for_volume_state( volume_manager=cinder_client.volumes, expected_volume=requested_volume, desired_state=u'available', transient_states=(u'creating', ), ) self.assertEqual([], self.api.list_volumes())
def test_exbibytes(self): self.assertEqual( " 100.00 EiB", _Memory(Byte(2**60 * 100)).render("8.2"), )
def _attach_vmdk(self, vm, vsphere_volume): vm_device = vm.config.hardware.device vm_uuid = vm.config.instanceUuid vm_name = vm.name logging.debug('vSphere volume {} will be attached to {} ({})'.format(vsphere_volume, vm_uuid, vm_name)) controller_to_use = None available_units = [0] # setting up a baseline list base_unit_range = range(0, 16) # disks per controller base_unit_range.remove(7) # unit_number 7 reserved for scsi controller # get relationship controller - disk devices = [] for dev in vm_device: if isinstance(dev, vim.vm.device.VirtualSCSIController): devices.append({'controller': dev, 'devices': [d.unitNumber for d in vm_device for k in dev.device if k == d.key]}) # getting any available slots in existing controllers for controller in devices: available = set(base_unit_range) - set(controller['devices']) if available: controller_to_use = controller['controller'] available_units = list(available) break # Creates controller if none available if not controller_to_use: # Either there are no available slots or # does not have SCSI controller self._create_new_scsi_controller(self._si, vm) vm = self._si.content.searchIndex.FindByUuid(None, vm_uuid, True, True) controller_to_use = [dev for dev in vm.config.hardware.device if isinstance(dev, vim.vm.device.VirtualSCSIController)][0] logging.debug('New SCSI controller created: {}'.format(controller_to_use.deviceInfo.label)) logging.debug('SCSI controller to use: {}'.format(controller_to_use.deviceInfo.label)) new_disk_kb = int(Byte(vsphere_volume.blockDeviceVolume.size).to_KiB().value) disk_spec = vim.vm.device.VirtualDeviceSpec() disk_spec.operation = vim.vm.device.VirtualDeviceSpec.Operation.add # device config disk_spec.device = vim.vm.device.VirtualDisk() disk_spec.device.unitNumber = available_units[0] disk_spec.device.capacityInKB = new_disk_kb disk_spec.device.controllerKey = controller_to_use.key # device backing info disk_spec.device.backing = vim.vm.device.VirtualDisk.FlatVer2BackingInfo() disk_spec.device.backing.thinProvisioned = True disk_spec.device.backing.diskMode = vim.vm.device.VirtualDiskOption.DiskMode.persistent disk_spec.device.backing.fileName = vsphere_volume.path # Submitting config spec spec = vim.vm.ConfigSpec() spec.deviceChange = [disk_spec] tasks = [vm.ReconfigVM_Task(spec=spec)] self._wait_for_tasks(tasks, self._si) # vsphere volume volume = vsphere_volume.blockDeviceVolume attached_volume = volume.set('attached_to', unicode(vm.config.instanceUuid)) logging.debug('vSphere volume {} attached to {} ({}) successfully'.format(vsphere_volume, vm.config.instanceUuid, vm.name)) return attached_volume
def test_gibibytes(self): self.assertEqual( " 1.05 GiB", _Memory(Byte(2**30 + 2**30 / 20)).render("8.2"), )
def test_pebibytes(self): self.assertEqual( " 100.00 PiB", _Memory(Byte(2**50 * 100)).render("8.2"), )
def test_kibibytes(self): self.assertEqual( " 12.50 KiB", _Memory(Byte(1024 * 12 + 512)).render("8.2"), )
def test_mebibytes(self): self.assertEqual( " 123.25 MiB", _Memory(Byte(2**20 * 123 + 2**20 / 4)).render("8.2"), )
def bytes_to_mbytes(size): """ :param bytes size: byte size of the volume :return size in megabytes """ return int(Byte(size).to_MiB().value)
def test_bytes(self): self.assertEqual( " 123.00 Byte", _Memory(Byte(123)).render("8.2"), )
def create_volume_with_profile(self, dataset_id, size, profile_name): """ Create a volume on EBS. Store Flocker-specific {metadata version, cluster id, dataset id} for the volume as volume tag data. Open issues: https://clusterhq.atlassian.net/browse/FLOC-1792 """ requested_size = int(Byte(size).to_GiB().value) try: volume_type, iops = _volume_type_and_iops_for_profile_name( profile_name, requested_size) requested_volume = self._create_ebs_volume( size=requested_size, zone=self.zone, volume_type=volume_type, iops=iops) except ClientError as e: # If we failed to create a volume with attributes complying # with requested profile, make a second attempt at volume # creation with default profile. # Compliance violation of the volume's requested profile # will be detected and remediated in a future release (FLOC-3275). if e.response['Error']['Code'] != INVALID_PARAMETER_VALUE: raise e CREATE_VOLUME_FAILURE( dataset_id=unicode(dataset_id), aws_code=e.response['Error']['Code'], aws_message=unicode(e.response['Error']['Message']) ).write() volume_type, iops = _volume_type_and_iops_for_profile_name( MandatoryProfiles.DEFAULT.value, requested_size) requested_volume = self._create_ebs_volume( size=requested_size, zone=self.zone, volume_type=volume_type, iops=iops) message_type = BOTO_LOG_RESULT + u':created_volume' Message.new( message_type=message_type, volume_id=unicode(requested_volume.id), dataset_id=unicode(dataset_id), size=unicode(size) ).write() # Stamp created volume with Flocker-specific tags. metadata = { METADATA_VERSION_LABEL: '1', CLUSTER_ID_LABEL: unicode(self.cluster_id), DATASET_ID_LABEL: unicode(dataset_id), # EC2 convention for naming objects, e.g. as used in EC2 web # console (http://stackoverflow.com/a/12798180). "Name": u"flocker-{}".format(dataset_id), } tags_list = [] for key, value in metadata.items(): tags_list.append(dict(Key=key, Value=value)) requested_volume.create_tags(Tags=tags_list) message_type = BOTO_LOG_RESULT + u':created_tags' Message.new( message_type=message_type, requested_volume=requested_volume.id, tags=metadata ).write() # Wait for created volume to reach 'available' state. _wait_for_volume_state_change(VolumeOperations.CREATE, requested_volume) # Return created volume in BlockDeviceVolume format. return _blockdevicevolume_from_ebs_volume(requested_volume)
def mptcp_compute_throughput(rawdf, mptcpstreamid: MpTcpStreamId, destination: ConnectionRoles, merged_df: bool) -> MpTcpUnidirectionalStats: """ Very raw computation: substract highest dsn from lowest by the elapsed time Args: merged_df: True if merged_df Returns: a tuple (True/false, dict) """ assert isinstance(destination, ConnectionRoles), "destination is %r" % destination con = rawdf.mptcp.connection(mptcpstreamid) q = con.generate_direction_query(destination) df = unidirectional_df = rawdf.query(q, engine="python") # -1 because of syn dsn_range, dsn_max, dsn_min = transmitted_seq_range(df, "dsn") msg = "dsn_range ({}) = {} (dsn_max) - {} (dsn_min) - 1" log.debug(msg.format(dsn_range, dsn_max, dsn_min)) _col = _sender if merged_df else lambda x: x # print("test _sender %s" % _col("toto")) # Could groupby destination as well groups = df.groupby(_col('tcpstream')) subflow_stats: List[TcpUnidirectionalStats] = [] for tcpstream, subdf in groups: # subdf.iloc[0, subdf.columns.get_loc(_second('abstime'))] # debug_dataframe(subdf, "subdf for stream %d" % tcpstream) dest = subdf.iloc[0, subdf.columns.get_loc(_col('tcpdest'))] sf_stats = tcp_get_stats( subdf, tcpstream, # work around pandas issue (since for now it's a float ConnectionRoles(dest), True) fields = ["tcpdest", "mptcpdest", "dss_dsn", "dss_length"] # debug_dataframe(subdf, "Debugging", usecols=[fields]) # DSNs can be discontinuous, so we have to look at each packet # we drop duplicates transmitted_dsn_df = subdf.drop_duplicates(subset="dsn") sf_stats.mptcp_application_bytes = transmitted_dsn_df["tcplen"].sum() # + 1 to deal with syn oddity assert sf_stats.mptcp_application_bytes <= sf_stats.tcp_byte_range + 1, sf_stats log.log(mp.TRACE, "Adding subflow stats %r", sf_stats) subflow_stats.append(sf_stats) times = df["abstime"] duration = times.iloc[-1] - times.iloc[0] total_tput = sum(map(lambda x: x.throughput_bytes, subflow_stats)) for sf in subflow_stats: # can be > 1 in case of redundant packets if total_tput > 0: sf.throughput_contribution = sf.throughput_bytes.bytes / total_tput else: sf.throughput_contribution = 0 log.warn("Total Throughput <= 0. Something fishy possibly ?") """ If it's a merged df, then we can classify reinjections and give more results on the goodput """ if merged_df: df = classify_reinjections(unidirectional_df) debug_dataframe(df, "after reinjections have been analyzed") # mptcp_application_bytes = df.loc[df.redundant == False, "tcplen"].sum() for sf in subflow_stats: log.debug("for tcpstream %d" % sf.tcpstreamid) # columns.get_loc(_first('abstime'))] df_sf = df[df.tcpstream == sf.tcpstreamid] non_redundant_pkts = df_sf.loc[df_sf.redundant == False, "tcplen"] # print("non_redundant_pkts") # print(non_redundant_pkts) sf.mptcp_application_bytes = non_redundant_pkts.sum() # print("sf.mptcp_application_bytes" , sf.mptcp_application_bytes) sf.goodput_contribution = sf.mptcp_application_bytes / dsn_range return MpTcpUnidirectionalStats( mptcpstreamid=mptcpstreamid, mptcp_application_bytes=Byte(dsn_range), mptcp_duration=duration, subflow_stats=subflow_stats, )
def mptcp_throughput_bytes(self) -> Byte: ''' sum of total bytes transferred ''' return Byte(sum(map(lambda x: x.throughput_bytes, self.subflow_stats)))
def is_allowed_file_size(f): """ check if given file size is less or equal as max allowed in config """ max_size = get_storage_max_size() return Byte(get_file_size(f)) <= MiB(float(max_size))
def parse_memory(s): return _Memory(Byte(parse_k8s_resource(s, default_scale=1)))