def _create_partition(container, disk):
"""
Models a partition
"""
partition = DiskPartition()
partition.disk = disk
DiskController._update_partition(partition, container)
def _create_partition(container, disk):
"""
Models a partition
"""
partition = DiskPartition()
partition.disk = disk
DiskController._update_partition(partition, container)
def _model_partition(cls, generic_partition_model, disk):
# type: (GenericPartition, Disk) -> DiskPartition
"""
Models a partition
:param generic_partition_model: The generic modeled partition (returned by Disktools)
:type generic_partition_model: GenericPartition
:param disk: The modeled disk
:type disk: Disk
:return: The newly modeled partition
:rtype: DiskPartition
"""
partition = DiskPartition()
partition.disk = disk
cls._update_partition(partition, generic_partition_model)
return partition
def validate_local_cache_size(self, requested_proxies):
"""
Validate whether the requested amount of proxies can be deployed on local StorageRouter partitions having the WRITE role
:param requested_proxies: Amount of proxies that have been requested for deployment
:type requested_proxies: int
:return: None
:rtype: NoneType
"""
if not 1 <= requested_proxies <= 16:
raise RuntimeError(
'The requested amount of proxies to deploy should be a value between 1 and 16'
)
if len(
self.write_partitions
) == 0 or self.global_write_buffer_requested_size is None or self.global_write_buffer_available_size is None:
raise RuntimeError(
'Global write buffer calculation has not been done yet')
# Calculate available write cache size
largest_ssd_size = 0
largest_sata_size = 0
largest_ssd_write_partition = None
largest_sata_write_partition = None
for info in self.write_partitions:
if info['ssd'] is True and info['available'] > largest_ssd_size:
largest_ssd_size = info['available']
largest_ssd_write_partition = info['guid']
elif info['ssd'] is False and info['available'] > largest_sata_size:
largest_sata_size = info['available']
largest_sata_write_partition = info['guid']
if largest_ssd_write_partition is None and largest_sata_write_partition is None:
raise RuntimeError(
'No {0} partition found to put the local caches on'.format(
DiskPartition.ROLES.WRITE))
self.requested_proxies = requested_proxies
self.largest_write_partition = DiskPartition(
largest_ssd_write_partition or largest_sata_write_partition)
if self.sd_installer.block_cache_local is True:
self.requested_local_proxies += requested_proxies
if self.sd_installer.fragment_cache_local is True:
self.requested_local_proxies += requested_proxies
if self.requested_local_proxies > 0:
proportion = float(
largest_ssd_size
or largest_sata_size) / self.global_write_buffer_available_size
available_size = proportion * self.global_write_buffer_requested_size * 0.10 # Only 10% is used on the largest WRITE partition for fragment caching
available_size_gib = available_size / 1024.0**3
if available_size / self.requested_local_proxies < 1024**3:
raise RuntimeError(
'Not enough space available ({0}GiB) on largest local WRITE partition to deploy {1} prox{2}'
.format(available_size_gib, requested_proxies,
'y' if requested_proxies == 1 else 'ies'))
self._logger.debug(
'Local cache size validated. Requested vpool proxies: {0}, requested cache proxies: {1}, Largest write partition: {1}'
.format(self.requested_proxies, self.requested_local_proxies,
str(self.largest_write_partition)))
def test_happypath(self):
"""
Validates the happy path; Hourly snapshots are taken with a few manual consistent
every now an then. The delete policy is executed every day
"""
# Setup
# There are 2 machines; one with two disks, one with one disk and a stand-alone additional disk
failure_domain = FailureDomain()
failure_domain.name = 'Test'
failure_domain.save()
backend_type = BackendType()
backend_type.name = 'BackendType'
backend_type.code = 'BT'
backend_type.save()
vpool = VPool()
vpool.name = 'vpool'
vpool.status = 'RUNNING'
vpool.backend_type = backend_type
vpool.save()
pmachine = PMachine()
pmachine.name = 'PMachine'
pmachine.username = '******'
pmachine.ip = '127.0.0.1'
pmachine.hvtype = 'VMWARE'
pmachine.save()
storage_router = StorageRouter()
storage_router.name = 'storage_router'
storage_router.ip = '127.0.0.1'
storage_router.pmachine = pmachine
storage_router.machine_id = System.get_my_machine_id()
storage_router.rdma_capable = False
storage_router.primary_failure_domain = failure_domain
storage_router.save()
disk = Disk()
disk.name = 'physical_disk_1'
disk.path = '/dev/non-existent'
disk.size = 500 * 1024 ** 3
disk.state = 'OK'
disk.is_ssd = True
disk.storagerouter = storage_router
disk.save()
disk_partition = DiskPartition()
disk_partition.id = 'disk_partition_id'
disk_partition.disk = disk
disk_partition.path = '/dev/disk/non-existent'
disk_partition.size = 400 * 1024 ** 3
disk_partition.state = 'OK'
disk_partition.offset = 1024
disk_partition.roles = [DiskPartition.ROLES.SCRUB]
disk_partition.mountpoint = '/var/tmp'
disk_partition.save()
vmachine_1 = VMachine()
vmachine_1.name = 'vmachine_1'
vmachine_1.devicename = 'dummy'
vmachine_1.pmachine = pmachine
vmachine_1.save()
vdisk_1_1 = VDisk()
vdisk_1_1.name = 'vdisk_1_1'
vdisk_1_1.volume_id = 'vdisk_1_1'
vdisk_1_1.vmachine = vmachine_1
vdisk_1_1.vpool = vpool
vdisk_1_1.devicename = 'dummy'
vdisk_1_1.size = 0
vdisk_1_1.save()
vdisk_1_1.reload_client()
vdisk_1_2 = VDisk()
vdisk_1_2.name = 'vdisk_1_2'
vdisk_1_2.volume_id = 'vdisk_1_2'
vdisk_1_2.vmachine = vmachine_1
vdisk_1_2.vpool = vpool
vdisk_1_2.devicename = 'dummy'
vdisk_1_2.size = 0
vdisk_1_2.save()
vdisk_1_2.reload_client()
vmachine_2 = VMachine()
vmachine_2.name = 'vmachine_2'
vmachine_2.devicename = 'dummy'
vmachine_2.pmachine = pmachine
vmachine_2.save()
vdisk_2_1 = VDisk()
vdisk_2_1.name = 'vdisk_2_1'
vdisk_2_1.volume_id = 'vdisk_2_1'
vdisk_2_1.vmachine = vmachine_2
vdisk_2_1.vpool = vpool
vdisk_2_1.devicename = 'dummy'
vdisk_2_1.size = 0
vdisk_2_1.save()
vdisk_2_1.reload_client()
vdisk_3 = VDisk()
vdisk_3.name = 'vdisk_3'
vdisk_3.volume_id = 'vdisk_3'
vdisk_3.vpool = vpool
vdisk_3.devicename = 'dummy'
vdisk_3.size = 0
vdisk_3.save()
vdisk_3.reload_client()
for disk in [vdisk_1_1, vdisk_1_2, vdisk_2_1, vdisk_3]:
[dynamic for dynamic in disk._dynamics if dynamic.name == 'snapshots'][0].timeout = 0
# Run the testing scenario
travis = 'TRAVIS' in os.environ and os.environ['TRAVIS'] == 'true'
if travis is True:
print 'Running in Travis, reducing output.'
debug = not travis
amount_of_days = 50
base = datetime.datetime.now().date()
day = datetime.timedelta(1)
minute = 60
hour = minute * 60
for d in xrange(0, amount_of_days):
base_timestamp = self._make_timestamp(base, day * d)
print ''
print 'Day cycle: {0}: {1}'.format(d, datetime.datetime.fromtimestamp(base_timestamp).strftime('%Y-%m-%d'))
# At the start of the day, delete snapshot policy runs at 00:30
print '- Deleting snapshots'
ScheduledTaskController.delete_snapshots(timestamp=base_timestamp + (minute * 30))
# Validate snapshots
print '- Validating snapshots'
for vdisk in [vdisk_1_1, vdisk_1_2, vdisk_2_1, vdisk_3]:
self._validate(vdisk, d, base, amount_of_days, debug)
# During the day, snapshots are taken
# - Create non consistent snapshot every hour, between 2:00 and 22:00
# - Create consistent snapshot at 6:30, 12:30, 18:30
print '- Creating snapshots'
for h in xrange(2, 23):
timestamp = base_timestamp + (hour * h)
for vm in [vmachine_1, vmachine_2]:
VMachineController.snapshot(machineguid=vm.guid,
label='ss_i_{0}:00'.format(str(h)),
is_consistent=False,
timestamp=timestamp)
if h in [6, 12, 18]:
ts = (timestamp + (minute * 30))
VMachineController.snapshot(machineguid=vm.guid,
label='ss_c_{0}:30'.format(str(h)),
is_consistent=True,
timestamp=ts)
VDiskController.create_snapshot(diskguid=vdisk_3.guid,
metadata={'label': 'ss_i_{0}:00'.format(str(h)),
'is_consistent': False,
'timestamp': str(timestamp),
'machineguid': None})
if h in [6, 12, 18]:
ts = (timestamp + (minute * 30))
VDiskController.create_snapshot(diskguid=vdisk_3.guid,
metadata={'label': 'ss_c_{0}:30'.format(str(h)),
'is_consistent': True,
'timestamp': str(ts),
'machineguid': None})
def build_dal_structure(structure, previous_structure=None):
"""
Builds a model structure
Example:
structure = DalHelper.build_service_structure(
{'vpools': [1],
'domains': [],
'storagerouters': [1],
'storagedrivers': [(1, 1, 1)], # (<id>, <vpool_id>, <storagerouter_id>)
'mds_services': [(1, 1)], # (<id>, <storagedriver_id>)
'storagerouter_domains': []} # (<id>, <storagerouter_id>, <domain_id>)
)
"""
Configuration.set(key=Configuration.EDITION_KEY,
value=PackageFactory.EDITION_ENTERPRISE)
if previous_structure is None:
previous_structure = {}
vdisks = previous_structure.get('vdisks', {})
vpools = previous_structure.get('vpools', {})
domains = previous_structure.get('domains', {})
services = previous_structure.get('services', {})
mds_services = previous_structure.get('mds_services', {})
storagerouters = previous_structure.get('storagerouters', {})
storagedrivers = previous_structure.get('storagedrivers', {})
storagerouter_domains = previous_structure.get('storagerouter_domains',
{})
service_types = {}
for service_type_name in ServiceType.SERVICE_TYPES.values():
service_type = ServiceTypeList.get_by_name(service_type_name)
if service_type is None:
service_type = ServiceType()
service_type.name = service_type_name
service_type.save()
service_types[service_type_name] = service_type
srclients = {}
for domain_id in structure.get('domains', []):
if domain_id not in domains:
domain = Domain()
domain.name = 'domain_{0}'.format(domain_id)
domain.save()
domains[domain_id] = domain
for vpool_id in structure.get('vpools', []):
if vpool_id not in vpools:
vpool = VPool()
vpool.name = str(vpool_id)
vpool.status = 'RUNNING'
vpool.metadata = {'backend': {}, 'caching_info': {}}
vpool.metadata_store_bits = 5
vpool.save()
vpools[vpool_id] = vpool
else:
vpool = vpools[vpool_id]
srclients[vpool_id] = StorageRouterClient(vpool.guid, None)
Configuration.set(
'/ovs/vpools/{0}/mds_config|mds_tlogs'.format(vpool.guid), 100)
Configuration.set(
'/ovs/vpools/{0}/mds_config|mds_safety'.format(vpool.guid), 2)
Configuration.set(
'/ovs/vpools/{0}/mds_config|mds_maxload'.format(vpool.guid),
75)
Configuration.set(
'/ovs/vpools/{0}/proxies/scrub/generic_scrub'.format(
vpool.guid),
json.dumps({}, indent=4),
raw=True)
for sr_id in structure.get('storagerouters', []):
if sr_id not in storagerouters:
storagerouter = StorageRouter()
storagerouter.name = str(sr_id)
storagerouter.ip = '10.0.0.{0}'.format(sr_id)
storagerouter.rdma_capable = False
storagerouter.node_type = 'MASTER'
storagerouter.machine_id = str(sr_id)
storagerouter.save()
storagerouters[sr_id] = storagerouter
disk = Disk()
disk.storagerouter = storagerouter
disk.state = 'OK'
disk.name = '/dev/uda'
disk.size = 1 * 1024**4
disk.is_ssd = True
disk.aliases = ['/dev/uda']
disk.save()
partition = DiskPartition()
partition.offset = 0
partition.size = disk.size
partition.aliases = ['/dev/uda-1']
partition.state = 'OK'
partition.mountpoint = '/tmp/unittest/sr_{0}/disk_1/partition_1'.format(
sr_id)
partition.disk = disk
partition.roles = [
DiskPartition.ROLES.DB, DiskPartition.ROLES.SCRUB
]
partition.save()
else:
storagerouter = storagerouters[sr_id]
# noinspection PyProtectedMember
System._machine_id[storagerouter.ip] = str(sr_id)
mds_start = 10000 + 100 * (sr_id - 1)
mds_end = 10000 + 100 * sr_id - 1
arakoon_start = 20000 + 100 * (sr_id - 1)
storagedriver_start = 30000 + 100 * (sr_id - 1)
storagedriver_end = 30000 + 100 * sr_id - 1
Configuration.initialize_host(
host_id=sr_id,
port_info={
'mds': [mds_start, mds_end],
'arakoon': arakoon_start,
'storagedriver': [storagedriver_start, storagedriver_end]
})
for sd_id, vpool_id, sr_id in structure.get('storagedrivers', ()):
if sd_id not in storagedrivers:
storagedriver = StorageDriver()
storagedriver.vpool = vpools[vpool_id]
storagedriver.storagerouter = storagerouters[sr_id]
storagedriver.name = str(sd_id)
storagedriver.mountpoint = '/'
storagedriver.cluster_ip = storagerouters[sr_id].ip
storagedriver.storage_ip = '10.0.1.{0}'.format(sr_id)
storagedriver.storagedriver_id = str(sd_id)
storagedriver.ports = {
'management': 1,
'xmlrpc': 2,
'dtl': 3,
'edge': 4
}
storagedriver.save()
storagedrivers[sd_id] = storagedriver
DalHelper.set_vpool_storage_driver_configuration(
vpool=vpools[vpool_id], storagedriver=storagedriver)
for mds_id, sd_id in structure.get('mds_services', ()):
if mds_id not in mds_services:
sd = storagedrivers[sd_id]
s_id = '{0}-{1}'.format(sd.storagerouter.name, mds_id)
service = Service()
service.name = s_id
service.storagerouter = sd.storagerouter
service.ports = [mds_id]
service.type = service_types['MetadataServer']
service.save()
services[s_id] = service
mds_service = MDSService()
mds_service.service = service
mds_service.number = 0
mds_service.capacity = 10
mds_service.vpool = sd.vpool
mds_service.save()
mds_services[mds_id] = mds_service
StorageDriverController.add_storagedriverpartition(
sd, {
'size': None,
'role': DiskPartition.ROLES.DB,
'sub_role': StorageDriverPartition.SUBROLE.MDS,
'partition': sd.storagerouter.disks[0].partitions[0],
'mds_service': mds_service
})
for vdisk_id, storage_driver_id, vpool_id, mds_id in structure.get(
'vdisks', ()):
if vdisk_id not in vdisks:
vpool = vpools[vpool_id]
devicename = 'vdisk_{0}'.format(vdisk_id)
mds_backend_config = DalHelper.generate_mds_metadata_backend_config(
[] if mds_id is None else [mds_services[mds_id]])
volume_id = srclients[vpool_id].create_volume(
devicename, mds_backend_config, 0, str(storage_driver_id))
vdisk = VDisk()
vdisk.name = str(vdisk_id)
vdisk.devicename = devicename
vdisk.volume_id = volume_id
vdisk.vpool = vpool
vdisk.size = 0
vdisk.save()
vdisk.reload_client('storagedriver')
vdisks[vdisk_id] = vdisk
for srd_id, sr_id, domain_id, backup in structure.get(
'storagerouter_domains', ()):
if srd_id not in storagerouter_domains:
sr_domain = StorageRouterDomain()
sr_domain.backup = backup
sr_domain.domain = domains[domain_id]
sr_domain.storagerouter = storagerouters[sr_id]
sr_domain.save()
storagerouter_domains[srd_id] = sr_domain
return {
'vdisks': vdisks,
'vpools': vpools,
'domains': domains,
'services': services,
'mds_services': mds_services,
'service_types': service_types,
'storagerouters': storagerouters,
'storagedrivers': storagedrivers,
'storagerouter_domains': storagerouter_domains
}
def setUpClass(cls):
"""
Sets up the unittest, mocking a certain set of 3rd party libraries and extensions.
This makes sure the unittests can be executed without those libraries installed
"""
# Load dummy stores
PersistentFactory.store = DummyPersistentStore()
VolatileFactory.store = DummyVolatileStore()
# Replace mocked classes
sys.modules[
'ovs.extensions.storageserver.storagedriver'] = StorageDriverModule
# Import required modules/classes after mocking is done
from ovs.dal.hybrids.backendtype import BackendType
from ovs.dal.hybrids.disk import Disk
from ovs.dal.hybrids.diskpartition import DiskPartition
from ovs.dal.hybrids.failuredomain import FailureDomain
from ovs.dal.hybrids.pmachine import PMachine
from ovs.dal.hybrids.storagerouter import StorageRouter
from ovs.dal.hybrids.vdisk import VDisk
from ovs.dal.hybrids.vmachine import VMachine
from ovs.dal.hybrids.vpool import VPool
from ovs.extensions.generic.volatilemutex import VolatileMutex
from ovs.lib.vmachine import VMachineController
from ovs.lib.vdisk import VDiskController
from ovs.lib.scheduledtask import ScheduledTaskController
# Globalize mocked classes
global Disk
global VDisk
global VMachine
global PMachine
global VPool
global BackendType
global DiskPartition
global FailureDomain
global StorageRouter
global VolatileMutex
global VMachineController
global VDiskController
global ScheduledTaskController
_ = VDisk(), VolatileMutex('dummy'), VMachine(), PMachine(), VPool(), BackendType(), FailureDomain(), \
VMachineController, VDiskController, ScheduledTaskController, StorageRouter(), Disk(), DiskPartition()
# Cleaning storage
VolatileFactory.store.clean()
PersistentFactory.store.clean()
def test_clone_snapshot(self):
"""
Validates that a snapshot that has clones will not be deleted while other snapshots will be deleted
"""
# Setup
# There are 2 disks, second one cloned from a snapshot of the first
vpool = VPool()
vpool.name = 'vpool'
vpool.status = 'RUNNING'
vpool.save()
storage_router = StorageRouter()
storage_router.name = 'storage_router'
storage_router.ip = '127.0.0.1'
storage_router.machine_id = System.get_my_machine_id()
storage_router.rdma_capable = False
storage_router.save()
disk = Disk()
disk.name = 'physical_disk_1'
disk.aliases = ['/dev/non-existent']
disk.size = 500 * 1024 ** 3
disk.state = 'OK'
disk.is_ssd = True
disk.storagerouter = storage_router
disk.save()
disk_partition = DiskPartition()
disk_partition.disk = disk
disk_partition.aliases = ['/dev/disk/non-existent']
disk_partition.size = 400 * 1024 ** 3
disk_partition.state = 'OK'
disk_partition.offset = 1024
disk_partition.roles = [DiskPartition.ROLES.SCRUB]
disk_partition.mountpoint = '/var/tmp'
disk_partition.save()
storage_driver = StorageDriver()
storage_driver.vpool = vpool
storage_driver.storagerouter = storage_router
storage_driver.name = 'storage_driver_1'
storage_driver.mountpoint = '/'
storage_driver.cluster_ip = storage_router.ip
storage_driver.storage_ip = '127.0.0.1'
storage_driver.storagedriver_id = 'storage_driver_1'
storage_driver.ports = {'management': 1,
'xmlrpc': 2,
'dtl': 3,
'edge': 4}
storage_driver.save()
service_type = ServiceType()
service_type.name = 'MetadataServer'
service_type.save()
service = Service()
service.name = 'service_1'
service.storagerouter = storage_driver.storagerouter
service.ports = [1]
service.type = service_type
service.save()
mds_service = MDSService()
mds_service.service = service
mds_service.number = 0
mds_service.capacity = 10
mds_service.vpool = storage_driver.vpool
mds_service.save()
vdisk_1_1 = VDisk()
vdisk_1_1.name = 'vdisk_1_1'
vdisk_1_1.volume_id = 'vdisk_1_1'
vdisk_1_1.vpool = vpool
vdisk_1_1.devicename = 'dummy'
vdisk_1_1.size = 0
vdisk_1_1.save()
vdisk_1_1.reload_client('storagedriver')
[dynamic for dynamic in vdisk_1_1._dynamics if dynamic.name == 'snapshots'][0].timeout = 0
travis = 'TRAVIS' in os.environ and os.environ['TRAVIS'] == 'true'
if travis is True:
print 'Running in Travis, reducing output.'
base = datetime.datetime.now().date()
day = datetime.timedelta(1)
base_timestamp = self._make_timestamp(base, day)
minute = 60
hour = minute * 60
for h in [6, 12, 18]:
timestamp = base_timestamp + (hour * h)
VDiskController.create_snapshot(vdisk_guid=vdisk_1_1.guid,
metadata={'label': 'snapshot_{0}:30'.format(str(h)),
'is_consistent': True,
'timestamp': str(timestamp),
'machineguid': None})
base_snapshot_guid = vdisk_1_1.snapshots[0]['guid'] # Oldest
clone_vdisk = VDisk()
clone_vdisk.name = 'clone_vdisk'
clone_vdisk.volume_id = 'clone_vdisk'
clone_vdisk.vpool = vpool
clone_vdisk.devicename = 'dummy'
clone_vdisk.parentsnapshot = base_snapshot_guid
clone_vdisk.size = 0
clone_vdisk.save()
clone_vdisk.reload_client('storagedriver')
for h in [6, 12, 18]:
timestamp = base_timestamp + (hour * h)
VDiskController.create_snapshot(vdisk_guid=clone_vdisk.guid,
metadata={'label': 'snapshot_{0}:30'.format(str(h)),
'is_consistent': True,
'timestamp': str(timestamp),
'machineguid': None})
base_timestamp = self._make_timestamp(base, day * 2)
ScheduledTaskController.delete_snapshots(timestamp=base_timestamp + (minute * 30))
self.assertIn(base_snapshot_guid, [snap['guid'] for snap in vdisk_1_1.snapshots], 'Snapshot was deleted while there are still clones of it')
def _voldrv_arakoon_checkup(create_cluster):
def add_service(service_storagerouter, arakoon_result):
"""
Add a service to the storage router
:param service_storagerouter: Storage Router to add the service to
:type service_storagerouter: StorageRouter
:param arakoon_result: Port information
:type arakoon_result: Dictionary
:return: The newly created and added service
"""
new_service = Service()
new_service.name = service_name
new_service.type = service_type
new_service.ports = [
arakoon_result['client_port'], arakoon_result['messaging_port']
]
new_service.storagerouter = service_storagerouter
new_service.save()
return new_service
cluster_name = 'voldrv'
service_name = 'arakoon-voldrv'
service_type = ServiceTypeList.get_by_name('Arakoon')
current_services = []
current_ips = []
for service in service_type.services:
if service.name == service_name:
current_services.append(service)
current_ips.append(service.storagerouter.ip)
all_sr_ips = [
storagerouter.ip
for storagerouter in StorageRouterList.get_slaves()
]
available_storagerouters = {}
for storagerouter in StorageRouterList.get_masters():
storagerouter.invalidate_dynamics(['partition_config'])
if len(storagerouter.partition_config[DiskPartition.ROLES.DB]) > 0:
available_storagerouters[storagerouter] = DiskPartition(
storagerouter.partition_config[DiskPartition.ROLES.DB][0])
all_sr_ips.append(storagerouter.ip)
if create_cluster is True and len(
current_services) == 0 and len(available_storagerouters) > 0:
storagerouter, partition = available_storagerouters.items()[0]
result = ArakoonInstaller.create_cluster(cluster_name=cluster_name,
ip=storagerouter.ip,
base_dir=partition.folder)
current_services.append(add_service(storagerouter, result))
ArakoonInstaller.restart_cluster_add(cluster_name, current_ips,
storagerouter.ip)
current_ips.append(storagerouter.ip)
StorageDriverController._configure_arakoon_to_volumedriver()
if 0 < len(current_services) < len(available_storagerouters):
for storagerouter, partition in available_storagerouters.iteritems(
):
if storagerouter.ip in current_ips:
continue
result = ArakoonInstaller.extend_cluster(
current_services[0].storagerouter.ip, storagerouter.ip,
cluster_name, partition.folder)
add_service(storagerouter, result)
current_ips.append(storagerouter.ip)
ArakoonInstaller.restart_cluster_add(cluster_name, current_ips,
storagerouter.ip)
StorageDriverController._configure_arakoon_to_volumedriver()
def _voldrv_arakoon_checkup(create_cluster):
def add_service(service_storagerouter, arakoon_ports):
"""
Add a service to the storage router
:param service_storagerouter: Storage Router to add the service to
:type service_storagerouter: StorageRouter
:param arakoon_ports: Port information
:type arakoon_ports: list
:return: The newly created and added service
:rtype: Service
"""
new_service = Service()
new_service.name = service_name
new_service.type = service_type
new_service.ports = arakoon_ports
new_service.storagerouter = service_storagerouter
new_service.save()
return new_service
service_name = 'arakoon-voldrv'
service_type = ServiceTypeList.get_by_name(ServiceType.SERVICE_TYPES.ARAKOON)
current_ips = []
current_services = []
for service in service_type.services:
if service.name == service_name:
current_services.append(service)
if service.is_internal is True:
current_ips.append(service.storagerouter.ip)
all_sr_ips = [storagerouter.ip for storagerouter in StorageRouterList.get_slaves()]
available_storagerouters = {}
for storagerouter in StorageRouterList.get_masters():
storagerouter.invalidate_dynamics(['partition_config'])
if len(storagerouter.partition_config[DiskPartition.ROLES.DB]) > 0:
available_storagerouters[storagerouter] = DiskPartition(storagerouter.partition_config[DiskPartition.ROLES.DB][0])
all_sr_ips.append(storagerouter.ip)
if create_cluster is True and len(current_services) == 0: # Create new cluster
metadata = ArakoonInstaller.get_unused_arakoon_metadata_and_claim(cluster_type=ServiceType.ARAKOON_CLUSTER_TYPES.SD)
if metadata is None: # No externally managed cluster found, we create 1 ourselves
if not available_storagerouters:
raise RuntimeError('Could not find any Storage Router with a DB role')
storagerouter, partition = available_storagerouters.items()[0]
result = ArakoonInstaller.create_cluster(cluster_name='voldrv',
cluster_type=ServiceType.ARAKOON_CLUSTER_TYPES.SD,
ip=storagerouter.ip,
base_dir=partition.folder,
filesystem=False)
ports = [result['client_port'], result['messaging_port']]
metadata = result['metadata']
ArakoonInstaller.restart_cluster_add(cluster_name='voldrv',
current_ips=current_ips,
new_ip=storagerouter.ip,
filesystem=False)
ArakoonInstaller.claim_cluster(cluster_name='voldrv',
master_ip=storagerouter.ip,
filesystem=False,
metadata=metadata)
current_ips.append(storagerouter.ip)
else:
ports = []
storagerouter = None
cluster_name = metadata['cluster_name']
Configuration.set('/ovs/framework/arakoon_clusters|voldrv', cluster_name)
StorageDriverController._logger.info('Claiming {0} managed arakoon cluster: {1}'.format('externally' if storagerouter is None else 'internally', cluster_name))
StorageDriverController._configure_arakoon_to_volumedriver(cluster_name=cluster_name)
current_services.append(add_service(service_storagerouter=storagerouter, arakoon_ports=ports))
cluster_name = Configuration.get('/ovs/framework/arakoon_clusters').get('voldrv')
if cluster_name is None:
return
metadata = ArakoonInstaller.get_arakoon_metadata_by_cluster_name(cluster_name=cluster_name)
if 0 < len(current_services) < len(available_storagerouters) and metadata['internal'] is True:
for storagerouter, partition in available_storagerouters.iteritems():
if storagerouter.ip in current_ips:
continue
result = ArakoonInstaller.extend_cluster(master_ip=current_services[0].storagerouter.ip,
new_ip=storagerouter.ip,
cluster_name=cluster_name,
base_dir=partition.folder)
add_service(storagerouter, [result['client_port'], result['messaging_port']])
current_ips.append(storagerouter.ip)
ArakoonInstaller.restart_cluster_add(cluster_name=cluster_name,
current_ips=current_ips,
new_ip=storagerouter.ip,
filesystem=False)
StorageDriverController._configure_arakoon_to_volumedriver(cluster_name=cluster_name)
def execute_scrub(vpool_guids=None,
vdisk_guids=None,
storagerouter_guid=None):
"""
Divide the scrub work among all StorageRouters with a SCRUB partition
:param vpool_guids: Guids of the vPools that need to be scrubbed completely
:type vpool_guids: list
:param vdisk_guids: Guids of the vDisks that need to be scrubbed
:type vdisk_guids: list
:param storagerouter_guid: Guid of the StorageRouter to execute the scrub work on
:type storagerouter_guid: str
:return: None
:rtype: NoneType
"""
if vpool_guids is not None and not isinstance(vpool_guids, list):
raise ValueError('vpool_guids should be a list')
if vdisk_guids is not None and not isinstance(vdisk_guids, list):
raise ValueError('vdisk_guids should be a list')
if storagerouter_guid is not None and not isinstance(
storagerouter_guid, basestring):
raise ValueError('storagerouter_guid should be a str')
GenericController._logger.info('Scrubber - Started')
scrub_locations = []
storagerouters = StorageRouterList.get_storagerouters(
) if storagerouter_guid is None else [
StorageRouter(storagerouter_guid)
]
for storage_router in storagerouters:
scrub_partitions = storage_router.partition_config.get(
DiskPartition.ROLES.SCRUB, [])
if len(scrub_partitions) == 0:
continue
try:
SSHClient(endpoint=storage_router, username='******')
for partition_guid in scrub_partitions:
partition = DiskPartition(partition_guid)
GenericController._logger.info(
'Scrubber - Storage Router {0} has {1} partition at {2}'
.format(storage_router.ip, DiskPartition.ROLES.SCRUB,
partition.folder))
scrub_locations.append({
'scrub_path': str(partition.folder),
'partition_guid': partition.guid,
'storage_router': storage_router
})
except UnableToConnectException:
GenericController._logger.warning(
'Scrubber - Storage Router {0} is not reachable'.format(
storage_router.ip))
if len(scrub_locations) == 0:
raise ValueError('No scrub locations found, cannot scrub')
vpool_vdisk_map = {}
if vpool_guids is None and vdisk_guids is None:
vpool_vdisk_map = dict((vpool, list(vpool.vdisks))
for vpool in VPoolList.get_vpools())
else:
if vpool_guids is not None:
for vpool_guid in set(vpool_guids):
vpool = VPool(vpool_guid)
vpool_vdisk_map[vpool] = list(vpool.vdisks)
if vdisk_guids is not None:
for vdisk_guid in set(vdisk_guids):
vdisk = VDisk(vdisk_guid)
if vdisk.vpool not in vpool_vdisk_map:
vpool_vdisk_map[vdisk.vpool] = []
if vdisk not in vpool_vdisk_map[vdisk.vpool]:
vpool_vdisk_map[vdisk.vpool].append(vdisk)
number_of_vpools = len(vpool_vdisk_map)
if number_of_vpools >= 6:
max_stacks_per_vpool = 1
elif number_of_vpools >= 3:
max_stacks_per_vpool = 2
else:
max_stacks_per_vpool = 5
threads = []
counter = 0
error_messages = []
for vp, vdisks in vpool_vdisk_map.iteritems():
# Verify amount of vDisks on vPool
GenericController._logger.info(
'Scrubber - vPool {0} - Checking scrub work'.format(vp.name))
if len(vdisks) == 0:
GenericController._logger.info(
'Scrubber - vPool {0} - No scrub work'.format(vp.name))
continue
# Fill queue with all vDisks for current vPool
vpool_queue = Queue()
for vd in vdisks:
if vd.is_vtemplate is True:
GenericController._logger.info(
'Scrubber - vPool {0} - vDisk {1} {2} - Is a template, not scrubbing'
.format(vp.name, vd.guid, vd.name))
continue
vd.invalidate_dynamics('storagedriver_id')
if not vd.storagedriver_id:
GenericController._logger.warning(
'Scrubber - vPool {0} - vDisk {1} {2} - No StorageDriver ID found'
.format(vp.name, vd.guid, vd.name))
continue
vpool_queue.put(vd.guid)
stacks_to_spawn = min(max_stacks_per_vpool, len(scrub_locations))
GenericController._logger.info(
'Scrubber - vPool {0} - Spawning {1} stack{2}'.format(
vp.name, stacks_to_spawn,
'' if stacks_to_spawn == 1 else 's'))
for _ in xrange(stacks_to_spawn):
scrub_target = scrub_locations[counter % len(scrub_locations)]
stack = Thread(
target=GenericController._deploy_stack_and_scrub,
args=(vpool_queue, vp, scrub_target, error_messages))
stack.start()
threads.append(stack)
counter += 1
for thread in threads:
thread.join()
if len(error_messages) > 0:
raise Exception('Errors occurred while scrubbing:\n - {0}'.format(
'\n - '.join(error_messages)))
def create_partitions(self):
"""
Configure all partitions for a StorageDriver (junctions between a StorageDriver and a DiskPartition)
:raises: ValueError: - When calculating the cache sizes went wrong
:return: Dict with information about the created items
:rtype: dict
"""
if self.storagedriver is None:
raise RuntimeError('A StorageDriver needs to be created first')
if self.sr_installer is None:
raise RuntimeError('No StorageRouterInstaller instance found')
# Assign WRITE / Fragment cache
for writecache_info in self.sr_installer.write_partitions:
available = writecache_info['available']
partition = DiskPartition(writecache_info['guid'])
proportion = available * 100.0 / self.sr_installer.global_write_buffer_available_size
size_to_be_used = proportion * self.sr_installer.global_write_buffer_requested_size / 100
write_cache_percentage = 0.98
if self.sr_installer.requested_local_proxies > 0 and partition == self.sr_installer.largest_write_partition: # At least 1 local proxy has been requested either for fragment or block cache
self.cache_size_local = int(size_to_be_used * 0.10) # Bytes
write_cache_percentage = 0.88
for _ in xrange(self.sr_installer.requested_proxies):
storagedriver_partition_cache = StorageDriverController.add_storagedriverpartition(storagedriver=self.storagedriver,
partition_info={'size': None,
'role': DiskPartition.ROLES.WRITE,
'sub_role': StorageDriverPartition.SUBROLE.FCACHE,
'partition': partition})
self.sr_installer.created_dirs.append(storagedriver_partition_cache.path)
if self.block_cache_local is True:
self.sr_installer.created_dirs.append('{0}/bc'.format(storagedriver_partition_cache.path))
if self.fragment_cache_local is True:
self.sr_installer.created_dirs.append('{0}/fc'.format(storagedriver_partition_cache.path))
self.storagedriver_partitions_caches.append(storagedriver_partition_cache)
w_size = int(size_to_be_used * write_cache_percentage / 1024 / 4096) * 4096
storagedriver_partition_write = StorageDriverController.add_storagedriverpartition(storagedriver=self.storagedriver,
partition_info={'size': long(size_to_be_used),
'role': DiskPartition.ROLES.WRITE,
'sub_role': StorageDriverPartition.SUBROLE.SCO,
'partition': partition})
self.write_caches.append({'path': storagedriver_partition_write.path,
'size': '{0}KiB'.format(w_size)})
self.sr_installer.created_dirs.append(storagedriver_partition_write.path)
if self.sr_installer.smallest_write_partition_size in [0, None] or (w_size * 1024) < self.sr_installer.smallest_write_partition_size:
self.sr_installer.smallest_write_partition_size = w_size * 1024
# Verify cache size
if self.cache_size_local is None and (self.block_cache_local is True or self.fragment_cache_local is True):
raise ValueError('Something went wrong trying to calculate the cache sizes')
# Assign FD partition
self.storagedriver_partition_file_driver = StorageDriverController.add_storagedriverpartition(storagedriver=self.storagedriver,
partition_info={'size': None,
'role': DiskPartition.ROLES.WRITE,
'sub_role': StorageDriverPartition.SUBROLE.FD,
'partition': self.sr_installer.largest_write_partition})
self.sr_installer.created_dirs.append(self.storagedriver_partition_file_driver.path)
# Assign DB partition
db_info = self.sr_installer.partition_info[DiskPartition.ROLES.DB][0]
self.storagedriver_partition_tlogs = StorageDriverController.add_storagedriverpartition(storagedriver=self.storagedriver,
partition_info={'size': None,
'role': DiskPartition.ROLES.DB,
'sub_role': StorageDriverPartition.SUBROLE.TLOG,
'partition': DiskPartition(db_info['guid'])})
self.storagedriver_partition_metadata = StorageDriverController.add_storagedriverpartition(storagedriver=self.storagedriver,
partition_info={'size': None,
'role': DiskPartition.ROLES.DB,
'sub_role': StorageDriverPartition.SUBROLE.MD,
'partition': DiskPartition(db_info['guid'])})
self.sr_installer.created_dirs.append(self.storagedriver_partition_tlogs.path)
self.sr_installer.created_dirs.append(self.storagedriver_partition_metadata.path)
# Assign DTL
dtl_info = self.sr_installer.partition_info[DiskPartition.ROLES.DTL][0]
self.storagedriver_partition_dtl = StorageDriverController.add_storagedriverpartition(storagedriver=self.storagedriver,
partition_info={'size': None,
'role': DiskPartition.ROLES.DTL,
'partition': DiskPartition(dtl_info['guid'])})
self.sr_installer.created_dirs.append(self.storagedriver_partition_dtl.path)
self.sr_installer.created_dirs.append(self.storagedriver.mountpoint)
# Create the directories
self.sr_installer.root_client.dir_create(directories=self.sr_installer.created_dirs)
def build_service_structure(structure, previous_structure=None):
"""
Builds an MDS service structure
Example:
structure = Helper.build_service_structure(
{'vpools': [1],
'domains': [],
'storagerouters': [1],
'storagedrivers': [(1, 1, 1)], # (<id>, <vpool_id>, <storagerouter_id>)
'mds_services': [(1, 1)], # (<id>, <storagedriver_id>)
'storagerouter_domains': []} # (<id>, <storagerouter_id>, <domain_id>)
)
"""
if previous_structure is None:
previous_structure = {}
vdisks = previous_structure.get("vdisks", {})
vpools = previous_structure.get("vpools", {})
domains = previous_structure.get("domains", {})
services = previous_structure.get("services", {})
mds_services = previous_structure.get("mds_services", {})
storagerouters = previous_structure.get("storagerouters", {})
storagedrivers = previous_structure.get("storagedrivers", {})
storagerouter_domains = previous_structure.get("storagerouter_domains", {})
service_type = ServiceTypeList.get_by_name("MetadataServer")
if service_type is None:
service_type = ServiceType()
service_type.name = "MetadataServer"
service_type.save()
srclients = {}
for domain_id in structure.get("domains", []):
if domain_id not in domains:
domain = Domain()
domain.name = "domain_{0}".format(domain_id)
domain.save()
domains[domain_id] = domain
for vpool_id in structure.get("vpools", []):
if vpool_id not in vpools:
vpool = VPool()
vpool.name = str(vpool_id)
vpool.status = "RUNNING"
vpool.save()
vpools[vpool_id] = vpool
else:
vpool = vpools[vpool_id]
srclients[vpool_id] = StorageRouterClient(vpool.guid, None)
for sr_id in structure.get("storagerouters", []):
if sr_id not in storagerouters:
storagerouter = StorageRouter()
storagerouter.name = str(sr_id)
storagerouter.ip = "10.0.0.{0}".format(sr_id)
storagerouter.rdma_capable = False
storagerouter.node_type = "MASTER"
storagerouter.machine_id = str(sr_id)
storagerouter.save()
storagerouters[sr_id] = storagerouter
disk = Disk()
disk.storagerouter = storagerouter
disk.state = "OK"
disk.name = "/dev/uda"
disk.size = 1 * 1024 ** 4
disk.is_ssd = True
disk.aliases = ["/dev/uda"]
disk.save()
partition = DiskPartition()
partition.offset = 0
partition.size = disk.size
partition.aliases = ["/dev/uda-1"]
partition.state = "OK"
partition.mountpoint = "/tmp/unittest/sr_{0}/disk_1/partition_1".format(sr_id)
partition.disk = disk
partition.roles = [DiskPartition.ROLES.DB, DiskPartition.ROLES.SCRUB]
partition.save()
for sd_id, vpool_id, sr_id in structure.get("storagedrivers", ()):
if sd_id not in storagedrivers:
storagedriver = StorageDriver()
storagedriver.vpool = vpools[vpool_id]
storagedriver.storagerouter = storagerouters[sr_id]
storagedriver.name = str(sd_id)
storagedriver.mountpoint = "/"
storagedriver.cluster_ip = storagerouters[sr_id].ip
storagedriver.storage_ip = "10.0.1.{0}".format(sr_id)
storagedriver.storagedriver_id = str(sd_id)
storagedriver.ports = {"management": 1, "xmlrpc": 2, "dtl": 3, "edge": 4}
storagedriver.save()
storagedrivers[sd_id] = storagedriver
Helper._set_vpool_storage_driver_configuration(vpool=vpools[vpool_id], storagedriver=storagedriver)
for mds_id, sd_id in structure.get("mds_services", ()):
if mds_id not in mds_services:
sd = storagedrivers[sd_id]
s_id = "{0}-{1}".format(sd.storagerouter.name, mds_id)
service = Service()
service.name = s_id
service.storagerouter = sd.storagerouter
service.ports = [mds_id]
service.type = service_type
service.save()
services[s_id] = service
mds_service = MDSService()
mds_service.service = service
mds_service.number = 0
mds_service.capacity = 10
mds_service.vpool = sd.vpool
mds_service.save()
mds_services[mds_id] = mds_service
StorageDriverController.add_storagedriverpartition(
sd,
{
"size": None,
"role": DiskPartition.ROLES.DB,
"sub_role": StorageDriverPartition.SUBROLE.MDS,
"partition": sd.storagerouter.disks[0].partitions[0],
"mds_service": mds_service,
},
)
for vdisk_id, storage_driver_id, vpool_id, mds_id in structure.get("vdisks", ()):
if vdisk_id not in vdisks:
vpool = vpools[vpool_id]
devicename = "vdisk_{0}".format(vdisk_id)
mds_backend_config = Helper._generate_mdsmetadatabackendconfig(
[] if mds_id is None else [mds_services[mds_id]]
)
volume_id = srclients[vpool_id].create_volume(devicename, mds_backend_config, 0, str(storage_driver_id))
vdisk = VDisk()
vdisk.name = str(vdisk_id)
vdisk.devicename = devicename
vdisk.volume_id = volume_id
vdisk.vpool = vpool
vdisk.size = 0
vdisk.save()
vdisk.reload_client("storagedriver")
vdisks[vdisk_id] = vdisk
for srd_id, sr_id, domain_id, backup in structure.get("storagerouter_domains", ()):
if srd_id not in storagerouter_domains:
sr_domain = StorageRouterDomain()
sr_domain.backup = backup
sr_domain.domain = domains[domain_id]
sr_domain.storagerouter = storagerouters[sr_id]
sr_domain.save()
storagerouter_domains[srd_id] = sr_domain
return {
"vdisks": vdisks,
"vpools": vpools,
"domains": domains,
"services": services,
"service_type": service_type,
"mds_services": mds_services,
"storagerouters": storagerouters,
"storagedrivers": storagedrivers,
"storagerouter_domains": storagerouter_domains,
}
def test_clone_snapshot(self):
"""
Validates that a snapshot that has clones will not be deleted while other snapshots will be deleted
"""
# Setup
# There are 2 disks, second one cloned from a snapshot of the first
vpool = VPool()
vpool.name = 'vpool'
vpool.status = 'RUNNING'
vpool.save()
storage_router = StorageRouter()
storage_router.name = 'storage_router'
storage_router.ip = '127.0.0.1'
storage_router.machine_id = System.get_my_machine_id()
storage_router.rdma_capable = False
storage_router.save()
disk = Disk()
disk.name = 'physical_disk_1'
disk.aliases = ['/dev/non-existent']
disk.size = 500 * 1024**3
disk.state = 'OK'
disk.is_ssd = True
disk.storagerouter = storage_router
disk.save()
disk_partition = DiskPartition()
disk_partition.disk = disk
disk_partition.aliases = ['/dev/disk/non-existent']
disk_partition.size = 400 * 1024**3
disk_partition.state = 'OK'
disk_partition.offset = 1024
disk_partition.roles = [DiskPartition.ROLES.SCRUB]
disk_partition.mountpoint = '/var/tmp'
disk_partition.save()
storage_driver = StorageDriver()
storage_driver.vpool = vpool
storage_driver.storagerouter = storage_router
storage_driver.name = 'storage_driver_1'
storage_driver.mountpoint = '/'
storage_driver.cluster_ip = storage_router.ip
storage_driver.storage_ip = '127.0.0.1'
storage_driver.storagedriver_id = 'storage_driver_1'
storage_driver.ports = {
'management': 1,
'xmlrpc': 2,
'dtl': 3,
'edge': 4
}
storage_driver.save()
service_type = ServiceType()
service_type.name = 'MetadataServer'
service_type.save()
service = Service()
service.name = 'service_1'
service.storagerouter = storage_driver.storagerouter
service.ports = [1]
service.type = service_type
service.save()
mds_service = MDSService()
mds_service.service = service
mds_service.number = 0
mds_service.capacity = 10
mds_service.vpool = storage_driver.vpool
mds_service.save()
vdisk_1_1 = VDisk()
vdisk_1_1.name = 'vdisk_1_1'
vdisk_1_1.volume_id = 'vdisk_1_1'
vdisk_1_1.vpool = vpool
vdisk_1_1.devicename = 'dummy'
vdisk_1_1.size = 0
vdisk_1_1.save()
vdisk_1_1.reload_client('storagedriver')
[
dynamic for dynamic in vdisk_1_1._dynamics
if dynamic.name == 'snapshots'
][0].timeout = 0
travis = 'TRAVIS' in os.environ and os.environ['TRAVIS'] == 'true'
if travis is True:
print 'Running in Travis, reducing output.'
base = datetime.datetime.now().date()
day = datetime.timedelta(1)
base_timestamp = self._make_timestamp(base, day)
minute = 60
hour = minute * 60
for h in [6, 12, 18]:
timestamp = base_timestamp + (hour * h)
VDiskController.create_snapshot(vdisk_guid=vdisk_1_1.guid,
metadata={
'label':
'snapshot_{0}:30'.format(
str(h)),
'is_consistent':
True,
'timestamp':
str(timestamp),
'machineguid':
None
})
base_snapshot_guid = vdisk_1_1.snapshots[0]['guid'] # Oldest
clone_vdisk = VDisk()
clone_vdisk.name = 'clone_vdisk'
clone_vdisk.volume_id = 'clone_vdisk'
clone_vdisk.vpool = vpool
clone_vdisk.devicename = 'dummy'
clone_vdisk.parentsnapshot = base_snapshot_guid
clone_vdisk.size = 0
clone_vdisk.save()
clone_vdisk.reload_client('storagedriver')
for h in [6, 12, 18]:
timestamp = base_timestamp + (hour * h)
VDiskController.create_snapshot(vdisk_guid=clone_vdisk.guid,
metadata={
'label':
'snapshot_{0}:30'.format(
str(h)),
'is_consistent':
True,
'timestamp':
str(timestamp),
'machineguid':
None
})
base_timestamp = self._make_timestamp(base, day * 2)
ScheduledTaskController.delete_snapshots(timestamp=base_timestamp +
(minute * 30))
self.assertIn(
base_snapshot_guid, [snap['guid'] for snap in vdisk_1_1.snapshots],
'Snapshot was deleted while there are still clones of it')
def test_happypath(self):
"""
Validates the happy path; Hourly snapshots are taken with a few manual consistent
every now and then. The delete policy is executed every day
"""
vpool = VPool()
vpool.name = 'vpool'
vpool.status = 'RUNNING'
vpool.save()
storage_router = StorageRouter()
storage_router.name = 'storage_router'
storage_router.ip = '127.0.0.1'
storage_router.machine_id = System.get_my_machine_id()
storage_router.rdma_capable = False
storage_router.save()
disk = Disk()
disk.name = 'physical_disk_1'
disk.aliases = ['/dev/non-existent']
disk.size = 500 * 1024**3
disk.state = 'OK'
disk.is_ssd = True
disk.storagerouter = storage_router
disk.save()
disk_partition = DiskPartition()
disk_partition.disk = disk
disk_partition.aliases = ['/dev/disk/non-existent']
disk_partition.size = 400 * 1024**3
disk_partition.state = 'OK'
disk_partition.offset = 1024
disk_partition.roles = [DiskPartition.ROLES.SCRUB]
disk_partition.mountpoint = '/var/tmp'
disk_partition.save()
vdisk_1 = VDisk()
vdisk_1.name = 'vdisk_1'
vdisk_1.volume_id = 'vdisk_1'
vdisk_1.vpool = vpool
vdisk_1.devicename = 'dummy'
vdisk_1.size = 0
vdisk_1.save()
vdisk_1.reload_client('storagedriver')
[
dynamic for dynamic in vdisk_1._dynamics
if dynamic.name == 'snapshots'
][0].timeout = 0
# Run the testing scenario
travis = 'TRAVIS' in os.environ and os.environ['TRAVIS'] == 'true'
if travis is True:
self._print_message('Running in Travis, reducing output.')
debug = not travis
amount_of_days = 50
base = datetime.datetime.now().date()
day = datetime.timedelta(1)
minute = 60
hour = minute * 60
for d in xrange(0, amount_of_days):
base_timestamp = self._make_timestamp(base, day * d)
self._print_message('')
self._print_message('Day cycle: {0}: {1}'.format(
d,
datetime.datetime.fromtimestamp(base_timestamp).strftime(
'%Y-%m-%d')))
# At the start of the day, delete snapshot policy runs at 00:30
self._print_message('- Deleting snapshots')
ScheduledTaskController.delete_snapshots(timestamp=base_timestamp +
(minute * 30))
# Validate snapshots
self._print_message('- Validating snapshots')
self._validate(vdisk_1, d, base, amount_of_days, debug)
# During the day, snapshots are taken
# - Create non consistent snapshot every hour, between 2:00 and 22:00
# - Create consistent snapshot at 6:30, 12:30, 18:30
self._print_message('- Creating snapshots')
for h in xrange(2, 23):
timestamp = base_timestamp + (hour * h)
VDiskController.create_snapshot(vdisk_guid=vdisk_1.guid,
metadata={
'label':
'ss_i_{0}:00'.format(
str(h)),
'is_consistent':
False,
'timestamp':
str(timestamp),
'machineguid':
None
})
if h in [6, 12, 18]:
ts = (timestamp + (minute * 30))
VDiskController.create_snapshot(vdisk_guid=vdisk_1.guid,
metadata={
'label':
'ss_c_{0}:30'.format(
str(h)),
'is_consistent':
True,
'timestamp':
str(ts),
'machineguid':
None
})
def test_happypath(self):
"""
Validates the happy path; Hourly snapshots are taken with a few manual consistent
every now an then. The delete policy is executed every day
"""
# Setup
# There are 2 machines; one with two disks, one with one disk and an additional disk
failure_domain = FailureDomain()
failure_domain.name = "Test"
failure_domain.save()
backend_type = BackendType()
backend_type.name = "BackendType"
backend_type.code = "BT"
backend_type.save()
vpool = VPool()
vpool.name = "vpool"
vpool.backend_type = backend_type
vpool.save()
pmachine = PMachine()
pmachine.name = "PMachine"
pmachine.username = "******"
pmachine.ip = "127.0.0.1"
pmachine.hvtype = "VMWARE"
pmachine.save()
storage_router = StorageRouter()
storage_router.name = "storage_router"
storage_router.ip = "127.0.0.1"
storage_router.pmachine = pmachine
storage_router.machine_id = System.get_my_machine_id()
storage_router.rdma_capable = False
storage_router.primary_failure_domain = failure_domain
storage_router.save()
disk = Disk()
disk.name = "physical_disk_1"
disk.path = "/dev/non-existent"
disk.size = 500 * 1024 ** 3
disk.state = "OK"
disk.is_ssd = True
disk.storagerouter = storage_router
disk.save()
disk_partition = DiskPartition()
disk_partition.id = "disk_partition_id"
disk_partition.disk = disk
disk_partition.path = "/dev/disk/non-existent"
disk_partition.size = 400 * 1024 ** 3
disk_partition.state = "OK"
disk_partition.offset = 1024
disk_partition.roles = [DiskPartition.ROLES.SCRUB]
disk_partition.mountpoint = "/var/tmp"
disk_partition.save()
vmachine_1 = VMachine()
vmachine_1.name = "vmachine_1"
vmachine_1.devicename = "dummy"
vmachine_1.pmachine = pmachine
vmachine_1.save()
vdisk_1_1 = VDisk()
vdisk_1_1.name = "vdisk_1_1"
vdisk_1_1.volume_id = "vdisk_1_1"
vdisk_1_1.vmachine = vmachine_1
vdisk_1_1.vpool = vpool
vdisk_1_1.devicename = "dummy"
vdisk_1_1.size = 0
vdisk_1_1.save()
vdisk_1_1.reload_client()
vdisk_1_2 = VDisk()
vdisk_1_2.name = "vdisk_1_2"
vdisk_1_2.volume_id = "vdisk_1_2"
vdisk_1_2.vmachine = vmachine_1
vdisk_1_2.vpool = vpool
vdisk_1_2.devicename = "dummy"
vdisk_1_2.size = 0
vdisk_1_2.save()
vdisk_1_2.reload_client()
vmachine_2 = VMachine()
vmachine_2.name = "vmachine_2"
vmachine_2.devicename = "dummy"
vmachine_2.pmachine = pmachine
vmachine_2.save()
vdisk_2_1 = VDisk()
vdisk_2_1.name = "vdisk_2_1"
vdisk_2_1.volume_id = "vdisk_2_1"
vdisk_2_1.vmachine = vmachine_2
vdisk_2_1.vpool = vpool
vdisk_2_1.devicename = "dummy"
vdisk_2_1.size = 0
vdisk_2_1.save()
vdisk_2_1.reload_client()
vdisk_3 = VDisk()
vdisk_3.name = "vdisk_3"
vdisk_3.volume_id = "vdisk_3"
vdisk_3.vpool = vpool
vdisk_3.devicename = "dummy"
vdisk_3.size = 0
vdisk_3.save()
vdisk_3.reload_client()
for disk in [vdisk_1_1, vdisk_1_2, vdisk_2_1, vdisk_3]:
[dynamic for dynamic in disk._dynamics if dynamic.name == "snapshots"][0].timeout = 0
# Run the testing scenario
debug = True
amount_of_days = 50
base = datetime.now().date()
day = timedelta(1)
minute = 60
hour = minute * 60
for d in xrange(0, amount_of_days):
base_timestamp = DeleteSnapshots._make_timestamp(base, day * d)
print ""
print "Day cycle: {0}: {1}".format(d, datetime.fromtimestamp(base_timestamp).strftime("%Y-%m-%d"))
# At the start of the day, delete snapshot policy runs at 00:30
print "- Deleting snapshots"
ScheduledTaskController.delete_snapshots(timestamp=base_timestamp + (minute * 30))
# Validate snapshots
print "- Validating snapshots"
for vdisk in [vdisk_1_1, vdisk_1_2, vdisk_2_1, vdisk_3]:
self._validate(vdisk, d, base, amount_of_days, debug)
# During the day, snapshots are taken
# - Create non consistent snapshot every hour, between 2:00 and 22:00
# - Create consistent snapshot at 6:30, 12:30, 18:30
print "- Creating snapshots"
for h in xrange(2, 23):
timestamp = base_timestamp + (hour * h)
for vm in [vmachine_1, vmachine_2]:
VMachineController.snapshot(
machineguid=vm.guid,
label="ss_i_{0}:00".format(str(h)),
is_consistent=False,
timestamp=timestamp,
)
if h in [6, 12, 18]:
ts = timestamp + (minute * 30)
VMachineController.snapshot(
machineguid=vm.guid, label="ss_c_{0}:30".format(str(h)), is_consistent=True, timestamp=ts
)
VDiskController.create_snapshot(
diskguid=vdisk_3.guid,
metadata={
"label": "ss_i_{0}:00".format(str(h)),
"is_consistent": False,
"timestamp": str(timestamp),
"machineguid": None,
},
)
if h in [6, 12, 18]:
ts = timestamp + (minute * 30)
VDiskController.create_snapshot(
diskguid=vdisk_3.guid,
metadata={
"label": "ss_c_{0}:30".format(str(h)),
"is_consistent": True,
"timestamp": str(ts),
"machineguid": None,
},
)
def build_service_structure(structure, previous_structure=None):
"""
Builds an MDS service structure
Example:
structure = Helper.build_service_structure(
{'vpools': [1],
'domains': [],
'storagerouters': [1],
'storagedrivers': [(1, 1, 1)], # (<id>, <vpool_id>, <storagerouter_id>)
'mds_services': [(1, 1)], # (<id>, <storagedriver_id>)
'storagerouter_domains': []} # (<id>, <storagerouter_id>, <domain_id>)
)
"""
if previous_structure is None:
previous_structure = {}
vdisks = previous_structure.get('vdisks', {})
vpools = previous_structure.get('vpools', {})
domains = previous_structure.get('domains', {})
services = previous_structure.get('services', {})
mds_services = previous_structure.get('mds_services', {})
storagerouters = previous_structure.get('storagerouters', {})
storagedrivers = previous_structure.get('storagedrivers', {})
storagerouter_domains = previous_structure.get('storagerouter_domains',
{})
service_type = ServiceTypeList.get_by_name('MetadataServer')
if service_type is None:
service_type = ServiceType()
service_type.name = 'MetadataServer'
service_type.save()
srclients = {}
for domain_id in structure.get('domains', []):
if domain_id not in domains:
domain = Domain()
domain.name = 'domain_{0}'.format(domain_id)
domain.save()
domains[domain_id] = domain
for vpool_id in structure.get('vpools', []):
if vpool_id not in vpools:
vpool = VPool()
vpool.name = str(vpool_id)
vpool.status = 'RUNNING'
vpool.save()
vpools[vpool_id] = vpool
else:
vpool = vpools[vpool_id]
srclients[vpool_id] = StorageRouterClient(vpool.guid, None)
for sr_id in structure.get('storagerouters', []):
if sr_id not in storagerouters:
storagerouter = StorageRouter()
storagerouter.name = str(sr_id)
storagerouter.ip = '10.0.0.{0}'.format(sr_id)
storagerouter.rdma_capable = False
storagerouter.node_type = 'MASTER'
storagerouter.machine_id = str(sr_id)
storagerouter.save()
storagerouters[sr_id] = storagerouter
disk = Disk()
disk.storagerouter = storagerouter
disk.state = 'OK'
disk.name = '/dev/uda'
disk.size = 1 * 1024**4
disk.is_ssd = True
disk.aliases = ['/dev/uda']
disk.save()
partition = DiskPartition()
partition.offset = 0
partition.size = disk.size
partition.aliases = ['/dev/uda-1']
partition.state = 'OK'
partition.mountpoint = '/tmp/unittest/sr_{0}/disk_1/partition_1'.format(
sr_id)
partition.disk = disk
partition.roles = [
DiskPartition.ROLES.DB, DiskPartition.ROLES.SCRUB
]
partition.save()
for sd_id, vpool_id, sr_id in structure.get('storagedrivers', ()):
if sd_id not in storagedrivers:
storagedriver = StorageDriver()
storagedriver.vpool = vpools[vpool_id]
storagedriver.storagerouter = storagerouters[sr_id]
storagedriver.name = str(sd_id)
storagedriver.mountpoint = '/'
storagedriver.cluster_ip = storagerouters[sr_id].ip
storagedriver.storage_ip = '10.0.1.{0}'.format(sr_id)
storagedriver.storagedriver_id = str(sd_id)
storagedriver.ports = {
'management': 1,
'xmlrpc': 2,
'dtl': 3,
'edge': 4
}
storagedriver.save()
storagedrivers[sd_id] = storagedriver
Helper._set_vpool_storage_driver_configuration(
vpool=vpools[vpool_id], storagedriver=storagedriver)
for mds_id, sd_id in structure.get('mds_services', ()):
if mds_id not in mds_services:
sd = storagedrivers[sd_id]
s_id = '{0}-{1}'.format(sd.storagerouter.name, mds_id)
service = Service()
service.name = s_id
service.storagerouter = sd.storagerouter
service.ports = [mds_id]
service.type = service_type
service.save()
services[s_id] = service
mds_service = MDSService()
mds_service.service = service
mds_service.number = 0
mds_service.capacity = 10
mds_service.vpool = sd.vpool
mds_service.save()
mds_services[mds_id] = mds_service
StorageDriverController.add_storagedriverpartition(
sd, {
'size': None,
'role': DiskPartition.ROLES.DB,
'sub_role': StorageDriverPartition.SUBROLE.MDS,
'partition': sd.storagerouter.disks[0].partitions[0],
'mds_service': mds_service
})
for vdisk_id, storage_driver_id, vpool_id, mds_id in structure.get(
'vdisks', ()):
if vdisk_id not in vdisks:
vpool = vpools[vpool_id]
devicename = 'vdisk_{0}'.format(vdisk_id)
mds_backend_config = Helper._generate_mdsmetadatabackendconfig(
[] if mds_id is None else [mds_services[mds_id]])
volume_id = srclients[vpool_id].create_volume(
devicename, mds_backend_config, 0, str(storage_driver_id))
vdisk = VDisk()
vdisk.name = str(vdisk_id)
vdisk.devicename = devicename
vdisk.volume_id = volume_id
vdisk.vpool = vpool
vdisk.size = 0
vdisk.save()
vdisk.reload_client('storagedriver')
vdisks[vdisk_id] = vdisk
for srd_id, sr_id, domain_id, backup in structure.get(
'storagerouter_domains', ()):
if srd_id not in storagerouter_domains:
sr_domain = StorageRouterDomain()
sr_domain.backup = backup
sr_domain.domain = domains[domain_id]
sr_domain.storagerouter = storagerouters[sr_id]
sr_domain.save()
storagerouter_domains[srd_id] = sr_domain
return {
'vdisks': vdisks,
'vpools': vpools,
'domains': domains,
'services': services,
'service_type': service_type,
'mds_services': mds_services,
'storagerouters': storagerouters,
'storagedrivers': storagedrivers,
'storagerouter_domains': storagerouter_domains
}
def _voldrv_arakoon_checkup(create_cluster):
def _add_service(service_storagerouter, arakoon_ports, service_name):
""" Add a service to the storage router """
new_service = Service()
new_service.name = service_name
new_service.type = service_type
new_service.ports = arakoon_ports
new_service.storagerouter = service_storagerouter
new_service.save()
return new_service
current_ips = []
current_services = []
service_type = ServiceTypeList.get_by_name(
ServiceType.SERVICE_TYPES.ARAKOON)
cluster_name = Configuration.get(
'/ovs/framework/arakoon_clusters').get('voldrv')
if cluster_name is not None:
arakoon_service_name = ArakoonInstaller.get_service_name_for_cluster(
cluster_name=cluster_name)
for service in service_type.services:
if service.name == arakoon_service_name:
current_services.append(service)
if service.is_internal is True:
current_ips.append(service.storagerouter.ip)
all_sr_ips = [
storagerouter.ip
for storagerouter in StorageRouterList.get_slaves()
]
available_storagerouters = {}
for storagerouter in StorageRouterList.get_masters():
storagerouter.invalidate_dynamics(['partition_config'])
if len(storagerouter.partition_config[DiskPartition.ROLES.DB]) > 0:
available_storagerouters[storagerouter] = DiskPartition(
storagerouter.partition_config[DiskPartition.ROLES.DB][0])
all_sr_ips.append(storagerouter.ip)
if create_cluster is True and len(
current_services) == 0: # Create new cluster
metadata = ArakoonInstaller.get_unused_arakoon_metadata_and_claim(
cluster_type=ServiceType.ARAKOON_CLUSTER_TYPES.SD)
if metadata is None: # No externally managed cluster found, we create 1 ourselves
if not available_storagerouters:
raise RuntimeError(
'Could not find any Storage Router with a DB role')
storagerouter, partition = available_storagerouters.items()[0]
arakoon_voldrv_cluster = 'voldrv'
arakoon_installer = ArakoonInstaller(
cluster_name=arakoon_voldrv_cluster)
arakoon_installer.create_cluster(
cluster_type=ServiceType.ARAKOON_CLUSTER_TYPES.SD,
ip=storagerouter.ip,
base_dir=partition.folder,
log_sinks=LogHandler.get_sink_path(
'arakoon-server_{0}'.format(arakoon_voldrv_cluster)),
crash_log_sinks=LogHandler.get_sink_path(
'arakoon-server-crash_{0}'.format(
arakoon_voldrv_cluster)))
arakoon_installer.start_cluster()
ports = arakoon_installer.ports[storagerouter.ip]
metadata = arakoon_installer.metadata
current_ips.append(storagerouter.ip)
else:
ports = []
storagerouter = None
cluster_name = metadata['cluster_name']
Configuration.set('/ovs/framework/arakoon_clusters|voldrv',
cluster_name)
StorageDriverController._logger.info(
'Claiming {0} managed arakoon cluster: {1}'.format(
'externally' if storagerouter is None else 'internally',
cluster_name))
StorageDriverController._configure_arakoon_to_volumedriver(
cluster_name=cluster_name)
current_services.append(
_add_service(
service_storagerouter=storagerouter,
arakoon_ports=ports,
service_name=ArakoonInstaller.get_service_name_for_cluster(
cluster_name=cluster_name)))
cluster_name = Configuration.get(
'/ovs/framework/arakoon_clusters').get('voldrv')
if cluster_name is None:
return
metadata = ArakoonInstaller.get_arakoon_metadata_by_cluster_name(
cluster_name=cluster_name)
if 0 < len(current_services) < len(
available_storagerouters) and metadata['internal'] is True:
for storagerouter, partition in available_storagerouters.iteritems(
):
if storagerouter.ip in current_ips:
continue
arakoon_installer = ArakoonInstaller(cluster_name=cluster_name)
arakoon_installer.load()
arakoon_installer.extend_cluster(
new_ip=storagerouter.ip,
base_dir=partition.folder,
log_sinks=LogHandler.get_sink_path(
'arakoon-server_{0}'.format(cluster_name)),
crash_log_sinks=LogHandler.get_sink_path(
'arakoon-server-crash_{0}'.format(cluster_name)))
_add_service(
service_storagerouter=storagerouter,
arakoon_ports=arakoon_installer.ports[storagerouter.ip],
service_name=ArakoonInstaller.get_service_name_for_cluster(
cluster_name=cluster_name))
current_ips.append(storagerouter.ip)
arakoon_installer.restart_cluster_after_extending(
new_ip=storagerouter.ip)
StorageDriverController._configure_arakoon_to_volumedriver(
cluster_name=cluster_name)
def configure_disk(storagerouter_guid, disk_guid, partition_guid, offset,
size, roles):
"""
Configures a partition
:param storagerouter_guid: Guid of the StorageRouter to configure a disk on
:type storagerouter_guid: str
:param disk_guid: Guid of the disk to configure
:type disk_guid: str
:param partition_guid: Guid of the partition on the disk
:type partition_guid: str
:param offset: Offset for the partition
:type offset: int
:param size: Size of the partition
:type size: int
:param roles: Roles assigned to the partition
:type roles: list
:return: None
:rtype: NoneType
"""
# Validations
storagerouter = StorageRouter(storagerouter_guid)
for role in roles:
if role not in DiskPartition.ROLES or role == DiskPartition.ROLES.BACKEND:
raise RuntimeError('Invalid role specified: {0}'.format(role))
disk = Disk(disk_guid)
if disk.storagerouter_guid != storagerouter_guid:
raise RuntimeError(
'The given Disk is not on the given StorageRouter')
for partition in disk.partitions:
if DiskPartition.ROLES.BACKEND in partition.roles:
raise RuntimeError('The given Disk is in use by a Backend')
if len({DiskPartition.ROLES.DB, DiskPartition.ROLES.DTL}.intersection(
set(roles))) > 0:
roles_on_sr = StorageRouterController._get_roles_on_storagerouter(
storagerouter.ip)
for role in [DiskPartition.ROLES.DB, DiskPartition.ROLES.DTL]:
if role in roles_on_sr and role in roles and roles_on_sr[role][
0] != disk.name: # DB and DTL roles still have to be unassignable
raise RoleDuplicationException(
'Disk {0} cannot have the {1} role due to presence on disk {2}'
.format(disk.name, role, roles_on_sr[role][0]))
# Create partition
if partition_guid is None:
StorageRouterController._logger.debug(
'Creating new partition - Offset: {0} bytes - Size: {1} bytes - Roles: {2}'
.format(offset, size, roles))
with remote(storagerouter.ip, [DiskTools], username='******') as rem:
if len(disk.aliases) == 0:
raise ValueError(
'Disk {0} does not have any aliases'.format(disk.name))
rem.DiskTools.create_partition(disk_alias=disk.aliases[0],
disk_size=disk.size,
partition_start=offset,
partition_size=size)
DiskController.sync_with_reality(storagerouter_guid)
disk = Disk(disk_guid)
end_point = offset + size
partition = None
for part in disk.partitions:
if offset < part.offset + part.size and end_point > part.offset:
partition = part
break
if partition is None:
raise RuntimeError(
'No new partition detected on disk {0} after having created 1'
.format(disk.name))
StorageRouterController._logger.debug('Partition created')
else:
StorageRouterController._logger.debug('Using existing partition')
partition = DiskPartition(partition_guid)
if partition.disk_guid != disk_guid:
raise RuntimeError(
'The given DiskPartition is not on the given Disk')
if partition.filesystem in [
'swap', 'linux_raid_member', 'LVM2_member'
]:
raise RuntimeError(
"It is not allowed to assign roles on partitions of type: ['swap', 'linux_raid_member', 'LVM2_member']"
)
metadata = StorageRouterController.get_metadata(storagerouter_guid)
partition_info = metadata['partitions']
removed_roles = set(partition.roles) - set(roles)
used_roles = []
for role in removed_roles:
for info in partition_info[role]:
if info['in_use'] and info['guid'] == partition.guid:
used_roles.append(role)
if len(used_roles) > 0:
raise RuntimeError(
'Roles in use cannot be removed. Used roles: {0}'.format(
', '.join(used_roles)))
# Add filesystem
if partition.filesystem is None or partition_guid is None:
StorageRouterController._logger.debug('Creating filesystem')
if len(partition.aliases) == 0:
raise ValueError(
'Partition with offset {0} does not have any aliases'.
format(partition.offset))
with remote(storagerouter.ip, [DiskTools], username='******') as rem:
rem.DiskTools.make_fs(partition_alias=partition.aliases[0])
DiskController.sync_with_reality(storagerouter_guid)
partition = DiskPartition(partition.guid)
if partition.filesystem not in ['ext4', 'xfs']:
raise RuntimeError('Unexpected filesystem')
StorageRouterController._logger.debug('Filesystem created')
# Mount the partition and add to FSTab
if partition.mountpoint is None:
StorageRouterController._logger.debug('Configuring mount point')
with remote(storagerouter.ip, [DiskTools], username='******') as rem:
counter = 1
mountpoint = '/mnt/{0}{1}'.format(
'ssd' if disk.is_ssd else 'hdd', counter)
while True:
if not rem.DiskTools.mountpoint_exists(mountpoint):
break
counter += 1
mountpoint = '/mnt/{0}{1}'.format(
'ssd' if disk.is_ssd else 'hdd', counter)
StorageRouterController._logger.debug(
'Found mount point: {0}'.format(mountpoint))
rem.DiskTools.add_fstab(partition_aliases=partition.aliases,
mountpoint=mountpoint,
filesystem=partition.filesystem)
rem.DiskTools.mount(mountpoint)
DiskController.sync_with_reality(storagerouter_guid)
partition = DiskPartition(partition.guid)
if partition.mountpoint != mountpoint:
raise RuntimeError('Unexpected mount point')
StorageRouterController._logger.debug('Mount point configured')
partition.roles = roles
partition.save()
StorageRouterController._logger.debug('Partition configured')
def test_happypath(self):
"""
Validates the happy path; Hourly snapshots are taken with a few manual consistent
every now and then. The delete policy is executed every day
"""
vpool = VPool()
vpool.name = 'vpool'
vpool.status = 'RUNNING'
vpool.save()
storage_router = StorageRouter()
storage_router.name = 'storage_router'
storage_router.ip = '127.0.0.1'
storage_router.machine_id = System.get_my_machine_id()
storage_router.rdma_capable = False
storage_router.save()
disk = Disk()
disk.name = 'physical_disk_1'
disk.aliases = ['/dev/non-existent']
disk.size = 500 * 1024 ** 3
disk.state = 'OK'
disk.is_ssd = True
disk.storagerouter = storage_router
disk.save()
disk_partition = DiskPartition()
disk_partition.disk = disk
disk_partition.aliases = ['/dev/disk/non-existent']
disk_partition.size = 400 * 1024 ** 3
disk_partition.state = 'OK'
disk_partition.offset = 1024
disk_partition.roles = [DiskPartition.ROLES.SCRUB]
disk_partition.mountpoint = '/var/tmp'
disk_partition.save()
vdisk_1 = VDisk()
vdisk_1.name = 'vdisk_1'
vdisk_1.volume_id = 'vdisk_1'
vdisk_1.vpool = vpool
vdisk_1.devicename = 'dummy'
vdisk_1.size = 0
vdisk_1.save()
vdisk_1.reload_client('storagedriver')
[dynamic for dynamic in vdisk_1._dynamics if dynamic.name == 'snapshots'][0].timeout = 0
# Run the testing scenario
travis = 'TRAVIS' in os.environ and os.environ['TRAVIS'] == 'true'
if travis is True:
self._print_message('Running in Travis, reducing output.')
debug = not travis
amount_of_days = 50
base = datetime.datetime.now().date()
day = datetime.timedelta(1)
minute = 60
hour = minute * 60
for d in xrange(0, amount_of_days):
base_timestamp = self._make_timestamp(base, day * d)
self._print_message('')
self._print_message('Day cycle: {0}: {1}'.format(d, datetime.datetime.fromtimestamp(base_timestamp).strftime('%Y-%m-%d')))
# At the start of the day, delete snapshot policy runs at 00:30
self._print_message('- Deleting snapshots')
ScheduledTaskController.delete_snapshots(timestamp=base_timestamp + (minute * 30))
# Validate snapshots
self._print_message('- Validating snapshots')
self._validate(vdisk_1, d, base, amount_of_days, debug)
# During the day, snapshots are taken
# - Create non consistent snapshot every hour, between 2:00 and 22:00
# - Create consistent snapshot at 6:30, 12:30, 18:30
self._print_message('- Creating snapshots')
for h in xrange(2, 23):
timestamp = base_timestamp + (hour * h)
VDiskController.create_snapshot(vdisk_guid=vdisk_1.guid,
metadata={'label': 'ss_i_{0}:00'.format(str(h)),
'is_consistent': False,
'timestamp': str(timestamp),
'machineguid': None})
if h in [6, 12, 18]:
ts = (timestamp + (minute * 30))
VDiskController.create_snapshot(vdisk_guid=vdisk_1.guid,
metadata={'label': 'ss_c_{0}:30'.format(str(h)),
'is_consistent': True,
'timestamp': str(ts),
'machineguid': None})
def test_happypath(self):
"""
Validates the happy path; Hourly snapshots are taken with a few manual consistent
every now an then. The delete policy is executed every day
"""
# Setup
# There are 2 machines; one with two disks, one with one disk and a stand-alone additional disk
failure_domain = FailureDomain()
failure_domain.name = 'Test'
failure_domain.save()
backend_type = BackendType()
backend_type.name = 'BackendType'
backend_type.code = 'BT'
backend_type.save()
vpool = VPool()
vpool.name = 'vpool'
vpool.status = 'RUNNING'
vpool.backend_type = backend_type
vpool.save()
pmachine = PMachine()
pmachine.name = 'PMachine'
pmachine.username = '******'
pmachine.ip = '127.0.0.1'
pmachine.hvtype = 'VMWARE'
pmachine.save()
storage_router = StorageRouter()
storage_router.name = 'storage_router'
storage_router.ip = '127.0.0.1'
storage_router.pmachine = pmachine
storage_router.machine_id = System.get_my_machine_id()
storage_router.rdma_capable = False
storage_router.primary_failure_domain = failure_domain
storage_router.save()
disk = Disk()
disk.name = 'physical_disk_1'
disk.path = '/dev/non-existent'
disk.size = 500 * 1024**3
disk.state = 'OK'
disk.is_ssd = True
disk.storagerouter = storage_router
disk.save()
disk_partition = DiskPartition()
disk_partition.id = 'disk_partition_id'
disk_partition.disk = disk
disk_partition.path = '/dev/disk/non-existent'
disk_partition.size = 400 * 1024**3
disk_partition.state = 'OK'
disk_partition.offset = 1024
disk_partition.roles = [DiskPartition.ROLES.SCRUB]
disk_partition.mountpoint = '/var/tmp'
disk_partition.save()
vmachine_1 = VMachine()
vmachine_1.name = 'vmachine_1'
vmachine_1.devicename = 'dummy'
vmachine_1.pmachine = pmachine
vmachine_1.save()
vdisk_1_1 = VDisk()
vdisk_1_1.name = 'vdisk_1_1'
vdisk_1_1.volume_id = 'vdisk_1_1'
vdisk_1_1.vmachine = vmachine_1
vdisk_1_1.vpool = vpool
vdisk_1_1.devicename = 'dummy'
vdisk_1_1.size = 0
vdisk_1_1.save()
vdisk_1_1.reload_client()
vdisk_1_2 = VDisk()
vdisk_1_2.name = 'vdisk_1_2'
vdisk_1_2.volume_id = 'vdisk_1_2'
vdisk_1_2.vmachine = vmachine_1
vdisk_1_2.vpool = vpool
vdisk_1_2.devicename = 'dummy'
vdisk_1_2.size = 0
vdisk_1_2.save()
vdisk_1_2.reload_client()
vmachine_2 = VMachine()
vmachine_2.name = 'vmachine_2'
vmachine_2.devicename = 'dummy'
vmachine_2.pmachine = pmachine
vmachine_2.save()
vdisk_2_1 = VDisk()
vdisk_2_1.name = 'vdisk_2_1'
vdisk_2_1.volume_id = 'vdisk_2_1'
vdisk_2_1.vmachine = vmachine_2
vdisk_2_1.vpool = vpool
vdisk_2_1.devicename = 'dummy'
vdisk_2_1.size = 0
vdisk_2_1.save()
vdisk_2_1.reload_client()
vdisk_3 = VDisk()
vdisk_3.name = 'vdisk_3'
vdisk_3.volume_id = 'vdisk_3'
vdisk_3.vpool = vpool
vdisk_3.devicename = 'dummy'
vdisk_3.size = 0
vdisk_3.save()
vdisk_3.reload_client()
for disk in [vdisk_1_1, vdisk_1_2, vdisk_2_1, vdisk_3]:
[
dynamic for dynamic in disk._dynamics
if dynamic.name == 'snapshots'
][0].timeout = 0
# Run the testing scenario
travis = 'TRAVIS' in os.environ and os.environ['TRAVIS'] == 'true'
if travis is True:
print 'Running in Travis, reducing output.'
debug = not travis
amount_of_days = 50
base = datetime.datetime.now().date()
day = datetime.timedelta(1)
minute = 60
hour = minute * 60
for d in xrange(0, amount_of_days):
base_timestamp = self._make_timestamp(base, day * d)
print ''
print 'Day cycle: {0}: {1}'.format(
d,
datetime.datetime.fromtimestamp(base_timestamp).strftime(
'%Y-%m-%d'))
# At the start of the day, delete snapshot policy runs at 00:30
print '- Deleting snapshots'
ScheduledTaskController.delete_snapshots(timestamp=base_timestamp +
(minute * 30))
# Validate snapshots
print '- Validating snapshots'
for vdisk in [vdisk_1_1, vdisk_1_2, vdisk_2_1, vdisk_3]:
self._validate(vdisk, d, base, amount_of_days, debug)
# During the day, snapshots are taken
# - Create non consistent snapshot every hour, between 2:00 and 22:00
# - Create consistent snapshot at 6:30, 12:30, 18:30
print '- Creating snapshots'
for h in xrange(2, 23):
timestamp = base_timestamp + (hour * h)
for vm in [vmachine_1, vmachine_2]:
VMachineController.snapshot(machineguid=vm.guid,
label='ss_i_{0}:00'.format(
str(h)),
is_consistent=False,
timestamp=timestamp)
if h in [6, 12, 18]:
ts = (timestamp + (minute * 30))
VMachineController.snapshot(machineguid=vm.guid,
label='ss_c_{0}:30'.format(
str(h)),
is_consistent=True,
timestamp=ts)
VDiskController.create_snapshot(diskguid=vdisk_3.guid,
metadata={
'label':
'ss_i_{0}:00'.format(
str(h)),
'is_consistent':
False,
'timestamp':
str(timestamp),
'machineguid':
None
})
if h in [6, 12, 18]:
ts = (timestamp + (minute * 30))
VDiskController.create_snapshot(diskguid=vdisk_3.guid,
metadata={
'label':
'ss_c_{0}:30'.format(
str(h)),
'is_consistent':
True,
'timestamp':
str(ts),
'machineguid':
None
})
def execute_scrub():
"""
Retrieve and execute scrub work
:return: None
"""
ScheduledTaskController._logger.info('Scrubber - Started')
vpools = VPoolList.get_vpools()
error_messages = []
scrub_locations = []
for storage_router in StorageRouterList.get_storagerouters():
scrub_partitions = storage_router.partition_config.get(
DiskPartition.ROLES.SCRUB, [])
if len(scrub_partitions) == 0:
continue
if len(scrub_partitions) > 1:
raise RuntimeError(
'Multiple {0} partitions defined for StorageRouter {1}'.
format(DiskPartition.ROLES.SCRUB, storage_router.name))
partition = DiskPartition(scrub_partitions[0])
ScheduledTaskController._logger.info(
'Scrubber - Storage Router {0:<15} has {1} partition at {2}'.
format(storage_router.ip, DiskPartition.ROLES.SCRUB,
partition.folder))
try:
SSHClient(storage_router, 'root')
scrub_locations.append({
'scrub_path': str(partition.folder),
'storage_router': storage_router
})
except UnableToConnectException:
ScheduledTaskController._logger.warning(
'Scrubber - Storage Router {0:<15} is not reachable'.
format(storage_router.ip))
number_of_vpools = len(vpools)
if number_of_vpools >= 6:
max_threads_per_vpool = 1
elif number_of_vpools >= 3:
max_threads_per_vpool = 2
else:
max_threads_per_vpool = 5
threads = []
counter = 0
for vp in vpools:
# Verify amount of vDisks on vPool
ScheduledTaskController._logger.info(
'Scrubber - vPool {0} - Checking scrub work'.format(vp.name))
if len(vp.vdisks) == 0:
ScheduledTaskController._logger.info(
'Scrubber - vPool {0} - No scrub work'.format(vp.name))
continue
# Fill queue with all vDisks for current vPool
vpool_queue = Queue()
for vd in vp.vdisks:
if vd.is_vtemplate is True:
continue
vd.invalidate_dynamics('storagedriver_id')
if not vd.storagedriver_id:
ScheduledTaskController._logger.warning(
'Scrubber - vPool {0} - vDisk {1} {2} - No StorageDriver ID found'
.format(vp.name, vd.guid, vd.name))
continue
vpool_queue.put(vd.guid)
# Copy backend connection manager information in separate key
threads_to_spawn = min(max_threads_per_vpool, len(scrub_locations))
ScheduledTaskController._logger.info(
'Scrubber - vPool {0} - Spawning {1} thread{2}'.format(
vp.name, threads_to_spawn,
'' if threads_to_spawn == 1 else 's'))
for _ in range(threads_to_spawn):
scrub_target = scrub_locations[counter % len(scrub_locations)]
thread = Thread(
target=ScheduledTaskController.execute_scrub_work,
name='scrub_{0}_{1}'.format(
vp.guid, scrub_target['storage_router'].guid),
args=(vpool_queue, vp, scrub_target, error_messages))
thread.start()
threads.append(thread)
counter += 1
for thread in threads:
thread.join()
if len(error_messages) > 0:
raise Exception('Errors occurred while scrubbing:\n - {0}'.format(
'\n - '.join(error_messages)))