def test_ibm_storage_block_device_api__get_blockdevicevolume_by_vol(
self, multipathing_mock):
multipathing_mock.return_value = True
vol_info = VolInfo('f_{}_{}'.format(UUID1_STR, UUID1_SLUG),
int(GiB(16).to_Byte().value),
1111,
UUID1_STR)
blockdevicevolume = BlockDeviceVolume(
blockdevice_id=unicode(UUID1_STR),
size=int(GiB(16).to_Byte().value),
attached_to=u'fake_host',
dataset_id=UUID(UUID1_STR),
)
self.mock_client.get_vol_mapping = MagicMock(return_value='fake_host')
driver_obj = driver.IBMStorageBlockDeviceAPI(
UUID(FAKE_CLUSTER_ID), self.mock_client, DRIVER_BASIC_CONF)
driver_obj._is_cluster_volume = MagicMock(return_value=False)
with self.assertRaises(UnknownVolume):
driver_obj._get_blockdevicevolume_by_vol(vol_info)
driver_obj._is_cluster_volume = MagicMock(return_value=True)
self.assertEqual(blockdevicevolume,
driver_obj._get_blockdevicevolume_by_vol(vol_info))
def allocation_unit(self):
"""
Return allocation unit
"""
logging.debug("vSphere allocation unit: " +
str(int(GiB(4).to_Byte().value)))
return int(GiB(4).to_Byte().value)
def configured_cluster_for_nodes(reactor, certificates, nodes, dataset_backend,
dataset_backend_configuration):
"""
Get a ``Cluster`` with Flocker services running on the right nodes.
:param reactor: The reactor.
:param Certificates certificates: The certificates to install on the
cluster.
:param nodes: The ``ManagedNode``s on which to operate.
:param NamedConstant dataset_backend: The ``DatasetBackend`` constant
representing the dataset backend that the nodes will be configured to
use when they are "started".
:param dict dataset_backend_configuration: The backend-specific
configuration the nodes will be given for their dataset backend.
:returns: A ``Deferred`` which fires with ``Cluster`` when it is
configured.
"""
default_volume_size = GiB(1)
if dataset_backend_configuration.get('auth_plugin') == 'rackspace':
default_volume_size = GiB(100)
cluster = Cluster(all_nodes=pvector(nodes),
control_node=nodes[0],
agent_nodes=nodes,
dataset_backend=dataset_backend,
default_volume_size=int(
default_volume_size.to_Byte().value),
certificates=certificates)
configuring = perform(
make_dispatcher(reactor),
configure_cluster(cluster, dataset_backend_configuration))
configuring.addCallback(lambda ignored: cluster)
return configuring
class HedvigBlockDeviceAPIInterfaceTests(
make_iblockdeviceapi_tests(
blockdevice_api_factory=(
lambda test_case: GetTestHedvigStorage(test_case)),
minimum_allocatable_size=int(GiB(8).to_Byte().value),
device_allocation_unit=int(GiB(8).to_Byte().value),
unknown_blockdevice_id_factory=lambda test: unicode(uuid4()))):
"""
def allocation_unit(self):
"""
Return allocation unit
"""
logger.debug('Setting minimum allocation unit to Multiplies of 1GiB')
logger.debug("Reduxio System allocation unit: " +
str(int(GiB(1).to_Byte().value)))
return int(GiB(1).to_Byte().value)
def configured_cluster_for_nodes(
reactor, certificates, nodes, dataset_backend,
dataset_backend_configuration, dataset_backend_config_file
):
"""
Get a ``Cluster`` with Flocker services running on the right nodes.
:param reactor: The reactor.
:param Certificates certificates: The certificates to install on the
cluster.
:param nodes: The ``ManagedNode``s on which to operate.
:param NamedConstant dataset_backend: The ``DatasetBackend`` constant
representing the dataset backend that the nodes will be configured to
use when they are "started".
:param dict dataset_backend_configuration: The backend-specific
configuration the nodes will be given for their dataset backend.
:param FilePath dataset_backend_config_file: A FilePath that has the
dataset_backend info stored.
:returns: A ``Deferred`` which fires with ``Cluster`` when it is
configured.
"""
# XXX: There is duplication between the values here and those in
# f.node.agents.test.blockdevicefactory.MINIMUM_ALLOCATABLE_SIZES. We want
# the default volume size to be greater than or equal to the minimum
# allocatable size.
#
# Ideally, the minimum allocatable size (and perhaps the default volume
# size) would be something known by an object that represents the dataset
# backend. Unfortunately:
# 1. There is no such object
# 2. There is existing confusion in the code around 'openstack' and
# 'rackspace'
#
# Here, we special-case Rackspace (presumably) because it has a minimum
# allocatable size that is different from other Openstack backends.
#
# FLOC-2584 also discusses this.
default_volume_size = GiB(1)
if dataset_backend_configuration.get('auth_plugin') == 'rackspace':
default_volume_size = GiB(100)
cluster = Cluster(
all_nodes=pvector(nodes),
control_node=nodes[0],
agent_nodes=nodes,
dataset_backend=dataset_backend,
default_volume_size=int(default_volume_size.to_Byte().value),
certificates=certificates,
dataset_backend_config_file=dataset_backend_config_file
)
configuring = perform(
make_dispatcher(reactor),
configure_cluster(cluster, dataset_backend_configuration)
)
configuring.addCallback(lambda ignored: cluster)
return configuring
class EMCScaleIOBlockDeviceAPIInterfaceTests(
make_iblockdeviceapi_tests(
blockdevice_api_factory=(
lambda test_case: emcsioblockdeviceapi_for_test(
uuid4(), test_case)),
minimum_allocatable_size=int(GiB(8).to_Byte().value),
device_allocation_unit=int(GiB(8).to_Byte().value),
unknown_blockdevice_id_factory=lambda test: unicode(uuid4()))):
"""
class AzureStorageBlockDeviceAPIInterfaceTests(
make_iblockdeviceapi_tests(
blockdevice_api_factory=(
lambda test_case: azureblockdeviceasyncapi_for_test(test_case)
),
minimum_allocatable_size=int(GiB(1).to_Byte().value),
device_allocation_unit=int(GiB(1).to_Byte().value),
unknown_blockdevice_id_factory=lambda test: unicode(uuid4()))):
"""
class EMCUnityBlockAPIInterfaceTests(
test_blockdevice.make_iblockdeviceapi_tests(
blockdevice_api_factory=(
lambda test_case: emcblockdeviceapi_for_test(
# XXX A hack to work around the LUN name length limit. We
# need a better way to store the cluster_id.
unicode(uuid4()).split('-')[0],
test_case)),
minimum_allocatable_size=int(GiB(8).to_Byte().value),
device_allocation_unit=int(GiB(8).to_Byte().value),
unknown_blockdevice_id_factory=lambda test: unicode(uuid4()))):
"""
def test_ibm_storage_block_device_api__create_volume(
self, multipathing_mock):
size = int(GiB(16).to_Byte().value)
multipathing_mock.return_value = True
mock_vol_obj = MagicMock
mock_vol_obj.id = 'vol-id-11111'
mock_vol_obj.size = size
mock_vol_obj.name = 'volfack'
mock_vol_obj.wwn = '11111111111'
self.mock_client.list_volumes = MagicMock(
return_value=[mock_vol_obj])
self.mock_client.create_volume = MagicMock
self.mock_client.handle_default_pool = MagicMock
self.mock_client.handle_default_profile = MagicMock(
return_value='fakepool')
self.mock_client.con_info = CONF_INFO_MOCK
driver_obj = driver.IBMStorageBlockDeviceAPI(
UUID1, self.mock_client, DRIVER_BASIC_CONF)
bdv = driver_obj.create_volume(UUID(UUID1_STR), size)
expacted_blockdevicevolume = BlockDeviceVolume(
blockdevice_id=unicode(mock_vol_obj.wwn),
size=size,
attached_to=None,
dataset_id=UUID(UUID1_STR),
)
self.assertEqual(bdv, expacted_blockdevicevolume)
def allocation_unit(self):
"""
1GiB is the minimum allocation unit for azure disks
return int: 1 GiB
"""
return int(GiB(1).to_Byte().value)
def create_volume_with_profile(self, dataset_id, size, profile_name):
blockdevice_id = _dataset_id_to_blockdevice_id(dataset_id)
size = Byte(size)
profile_type = MandatoryProfiles.lookupByValue(profile_name).name
gce_disk_type = GCEStorageProfiles.lookupByName(profile_type).value
try:
self._operations.create_disk(
name=blockdevice_id,
size=size,
description=self._disk_resource_description(),
gce_disk_type=gce_disk_type
)
except HttpError as e:
if e.resp.status == 409:
msg = ("A dataset named {} already exists in this GCE "
"project.".format(dataset_id))
raise GCEVolumeException(msg)
else:
raise
disk = self._operations.get_disk_details(blockdevice_id)
return BlockDeviceVolume(
blockdevice_id=blockdevice_id,
size=int(GiB(int(disk['sizeGb'])).to_Byte()),
attached_to=_extract_attached_to(disk),
dataset_id=_blockdevice_id_to_dataset_id(blockdevice_id),
)
def from_options(cls, reactor, options):
"""
Create a cluster container deployment object from the
options given through command line.
:param reactor: reactor
:param options: ``ContainerOptions`` container the parsed
options given to the script.
"""
try:
image = DockerImage(repository=options['image'])
max_size = int(GiB(options['max-size']).to_Byte().value)
mountpoint = unicode(options['mountpoint'])
control_node_address = options['control-node']
timeout = options['wait']
except Exception as e:
sys.stderr.write("%s: %s\n" % ("Missing or wrong arguments", e))
sys.stderr.write(e.args[0])
sys.stderr.write('\n\n')
sys.stderr.write(options.getSynopsis())
sys.stderr.write('\n')
sys.stderr.write(options.getUsage())
raise SystemExit(1)
certificates_path = FilePath(options['cert-directory'])
cluster_cert = certificates_path.child(b"cluster.crt")
user_cert = certificates_path.child(b"user.crt")
user_key = certificates_path.child(b"user.key")
# Initialise client
client = FlockerClient(reactor, control_node_address, REST_API_PORT,
cluster_cert, user_cert, user_key)
return cls(reactor, image, max_size, mountpoint, control_node_address,
timeout, cluster_cert, user_cert, user_key, client)
def test_ibm_storage_block_device_api__get_device_path(
self, ospathexist, multipathing_mock, check_output_mock):
multipathing_mock.return_value = True
ospathexist.return_value = True
check_output_mock.return_value = test_host_actions.MULTIPATH_OUTPUT2
blockdevicevolume = BlockDeviceVolume(
blockdevice_id=unicode(test_host_actions.MULTIPATH_OUTPUT_WWN2),
size=int(GiB(16).to_Byte().value),
attached_to=u'fakehost',
dataset_id=UUID(UUID1_STR),
)
driver_obj = driver.IBMStorageBlockDeviceAPI(
UUID1, self.mock_client, DRIVER_BASIC_CONF)
class VolInfoFake(object):
name = 'fake_volname'
driver_obj._get_volume_object = MagicMock(return_value=VolInfoFake())
driver_obj._get_blockdevicevolume_by_vol = MagicMock(
return_value=blockdevicevolume)
self.mock_client.backend_type = 'XIV'
self.mock_client.list_volumes = MagicMock(return_value=[FAKERList])
dpath = driver_obj.get_device_path(blockdevicevolume.blockdevice_id)
self.assertEqual(
dpath,
FilePath('{}/{}'.format(
PREFIX_DEVICE_PATH,
test_host_actions.WWN_PREFIX +
test_host_actions.MULTIPATH_OUTPUT_WWN2,
)))
def allocation_unit(self):
"""
8GiB is the minimum allocation unit described by the ScaelIO Guide
https://community.emc.com/docs/DOC-43968 for ScaleIO v1.31
return int: 8 GiB
"""
return int(GiB(8).to_Byte().value)
def _test_sized_vol_container(self, cluster, node):
"""
Create a volume with a size, then create a
container that can use that volume and run lsbk to test
its size. An http server inside the container uses lsblk to
return the size of the volume when path is not "/is_http".
:param cluster: flocker cluster
:param node: node the contianer and volume will be on.
"""
client = get_docker_client(cluster, node.public_address)
volume_name = random_name(self)
size = integers(min_value=75, max_value=100).example()
expression = volume_expression.example()
size_opt = "".join(str(size)) + expression
size_bytes = int(GiB(size).to_Byte().value)
self._create_volume(client,
volume_name,
driver_opts={'size': size_opt})
http_port = 8080
container_identifier = self.run_python_container(
cluster,
node.public_address, {
"host_config":
client.create_host_config(
binds=["{}:/sizedvoldata".format(volume_name)],
port_bindings={http_port: 0},
restart_policy={"Name": "always"},
privileged=True),
"ports": [http_port]
},
SCRIPTS.child(b"lsblkhttp.py"), [u"/sizedvoldata"],
client=client)
host_port = extract_external_port(
client,
container_identifier,
http_port,
)
d = assert_http_server(self,
node.public_address,
host_port,
expected_response=str(size_bytes))
def _get_datasets(unused_arg):
return cluster.client.list_datasets_configuration()
d.addCallback(_get_datasets)
def _verify_volume_metadata_size(datasets):
dataset = next(d for d in datasets
if d.metadata.get(u'name') == volume_name)
self.assertEqual(int(dataset.metadata.get(u'maximum_size')),
size_bytes)
d.addCallback(_verify_volume_metadata_size)
return d
def vmax_round_allocation(in_bytes):
"""
Returns a size in bytes that is on a clean cylinder boundary.
:param in_bytes: input size in bytes
:return:
"""
if in_bytes < int(GiB(1).to_Byte().value):
in_bytes = int(GiB(1).to_Byte().value)
boundary = int(KiB(1920).to_Byte().value)
remainder = in_bytes % boundary
if remainder == 0:
round_up = in_bytes
else:
round_up = in_bytes + boundary - remainder
return round_up
class EBSProfiledBlockDeviceAPIInterfaceTests(
make_iprofiledblockdeviceapi_tests(
profiled_blockdevice_api_factory=ebsblockdeviceapi_for_test,
dataset_size=GiB(4).to_Byte().value)):
"""
Interface adherence tests for ``IProfiledBlockDeviceAPI``.
"""
pass
def allocation_unit(self):
"""
Can only allocate PDs in GiB units.
Documentation claims `GB` but experimentally this was determined to
actually be `GiB`.
"""
return int(GiB(1).to_Byte().value)
def test_sbatch_arguments_unsupported_provided():
params = AllocationParameters(nodes=1,
cores=2,
walltime=Walltime(minutes=10),
memory_per_node=GiB(1))
params.all['Provided Unsupported Param'] = 12
with pytest.raises(ValueError):
SbatchArguments(params=params)
def setUp(self):
super(AttachVolumeAndWaitTests, self).setUp()
self.volume = BlockDeviceVolume(
dataset_id=uuid4(),
blockdevice_id=u"opaque storage backend id",
size=int(GiB(100).to_Byte()),
)
self.compute_id = u"opaque compute backend id"
def vmax_allocation_unit(size_in_gb):
"""
Round size to next greatest allocation unit
:param size_in_gb: size in GigaBytes
:return:
Size in bytes
"""
return EMCVmaxBlockDeviceAPI.vmax_round_allocation(
int(GiB(size_in_gb).to_Byte().value))
class EMCUnityProfiledBlockAPIInterfaceTests(
test_blockdevice.make_iprofiledblockdeviceapi_tests(
profiled_blockdevice_api_factory=(
lambda test_case: emcblockdeviceapi_for_test(
# XXX A hack to work around the LUN name length limit. We
# need a better way to store the cluster_id.
unicode(uuid4()).split('-')[0],
test_case)),
dataset_size=int(GiB(8).to_Byte().value))):
"""
def test_parsenum_size(self, expr, size):
"""
Send different forms of size expressions
to ``parse_num``, we expect G(Gigabyte) size results.
:param expr str: A string representing the size expression
:param size int: A string representing the volume size
"""
expected_size = int(GiB(size).to_Byte())
return self.assertEqual(expected_size,
int(parse_num(str(size) + expr).to_Byte()))
def attach_volume(self, blockdevice_id, attach_to):
"""
Attach ``blockdevice_id`` to ``host``.
:param unicode blockdevice_id: The unique identifier for the block
device being attached.
:param unicode attach_to: An identifier like the one returned by the
``compute_instance_id`` method indicating the node to which to
attach the volume.
:raises UnknownVolume: If the supplied ``blockdevice_id`` does not
exist.
:raises AlreadyAttachedVolume: If the supplied ``blockdevice_id`` is
already attached.
:returns: A ``BlockDeviceVolume`` with a ``host`` attribute set to
``host``.
"""
log_info('Attempting to attach ' + str(blockdevice_id)
+ ' to ' + str(attach_to))
_vmstate_lock.acquire()
try:
# Make sure disk is present. Also, need the disk size is needed.
disks = self._manager.list_disks()
target_disk = None
for disk in disks:
if disk.name == blockdevice_id:
target_disk = disk
break
if target_disk is None:
raise UnknownVolume(blockdevice_id)
(disk, vmname, lun) = self._get_disk_vmname_lun(blockdevice_id)
if vmname is not None:
if unicode(vmname) != self.compute_instance_id():
raise AlreadyAttachedVolume(blockdevice_id)
else:
return self._blockdevicevolume_from_azure_volume(
blockdevice_id,
target_disk.properties.content_length,
attach_to)
self._manager.attach_disk(
str(attach_to),
target_disk.name,
int(GiB(bytes=target_disk.properties.content_length)))
finally:
_vmstate_lock.release()
log_info('disk attached')
return self._blockdevicevolume_from_azure_volume(
blockdevice_id,
target_disk.properties.content_length,
attach_to)
def allocation_unit(self):
"""
1GiB is the minimum allocation unit described by the OpenStack
Cinder API documentation.
* http://developer.openstack.org/api-ref-blockstorage-v2.html#createVolume # noqa
Some Cinder storage drivers may actually allocate more than
this, but as long as the requested size is a multiple of this
unit, the Cinder API will always report the size that was
requested.
"""
return int(GiB(1).to_Byte().value)
def get_minimum_allocatable_size():
"""
Return the minimum supported volume size, in bytes, defined as an
environment variable or 1GiB if the variable is not set.
:returns: An ``int`` minimum_allocatable_size in bytes.
"""
return int(
environ.get(
'FLOCKER_FUNCTIONAL_TEST_MINIMUM_ALLOCATABLE_SIZE',
GiB(1).to_Byte().value,
))
def test_sbatch_arguments_create():
params = AllocationParameters(nodes=1,
cores=2,
memory_per_node=GiB(1),
walltime=Walltime(minutes=10))
sbatch_args = SbatchArguments(params=params)
assert sbatch_args.native_args == {}
assert sbatch_args.args == {'--nodes': '1',
'--cpus-per-task': '2',
'--mem': '1048576K',
'--time': '0-00:10:00'}
def test_allocation_parameters_create():
"""Tests construction of allocation parameters."""
params = AllocationParameters(nodes=1,
cores=2,
memory_per_node=GiB(1),
walltime=Walltime(minutes=10),
native_args={
'--abc': None,
'--def': '80'
})
assert params.all == {
'nodes': 1,
'cores': 2,
'memory_per_node': GiB(1),
'walltime': Walltime(minutes=10)
}
assert params.nodes == 1
assert params.cores == 2
assert params.memory_per_node == GiB(1)
assert params.walltime == Walltime(minutes=10)
assert params.native_args == {'--abc': None, '--def': '80'}
def _blockdevicevolume_from_ebs_volume(ebs_volume):
"""
Helper function to convert Volume information from
EBS format to Flocker block device format.
:param boto.ec2.volume ebs_volume: Volume in EC2 format.
:return: Input volume in BlockDeviceVolume format.
"""
return BlockDeviceVolume(blockdevice_id=unicode(ebs_volume.id),
size=int(GiB(ebs_volume.size).to_Byte().value),
attached_to=ebs_volume.attach_data.instance_id,
dataset_id=UUID(
ebs_volume.tags[DATASET_ID_LABEL]))
