def _model_disk(cls, generic_disk_model, storagerouter):
# type: (GenericDisk, StorageRouter) -> Disk
"""
Models a disk
:param generic_disk_model: The generic modeled disk (returned by Disktools)
:type generic_disk_model: GenericDisk
:param storagerouter: Storagerouter to which this disk belongs
:type storagerouter: StorageRouter
:return: The newly modeled disk
:rtype: Disk
"""
DiskController._logger.info('Disk {0} - Creating disk - {1}'.format(generic_disk_model.name, generic_disk_model.__dict__))
disk = Disk()
disk.storagerouter = storagerouter
disk.name = generic_disk_model.name
DiskController._update_disk(disk, generic_disk_model)
for partition in generic_disk_model.partitions: # type: GenericPartition
cls._model_partition(partition, disk)
return disk
def test_happypath(self):
"""
Validates the happy path; Hourly snapshots are taken with a few manual consistent
every now an then. The delete policy is executed every day
"""
# Setup
# There are 2 machines; one with two disks, one with one disk and a stand-alone additional disk
failure_domain = FailureDomain()
failure_domain.name = 'Test'
failure_domain.save()
backend_type = BackendType()
backend_type.name = 'BackendType'
backend_type.code = 'BT'
backend_type.save()
vpool = VPool()
vpool.name = 'vpool'
vpool.status = 'RUNNING'
vpool.backend_type = backend_type
vpool.save()
pmachine = PMachine()
pmachine.name = 'PMachine'
pmachine.username = '******'
pmachine.ip = '127.0.0.1'
pmachine.hvtype = 'VMWARE'
pmachine.save()
storage_router = StorageRouter()
storage_router.name = 'storage_router'
storage_router.ip = '127.0.0.1'
storage_router.pmachine = pmachine
storage_router.machine_id = System.get_my_machine_id()
storage_router.rdma_capable = False
storage_router.primary_failure_domain = failure_domain
storage_router.save()
disk = Disk()
disk.name = 'physical_disk_1'
disk.path = '/dev/non-existent'
disk.size = 500 * 1024**3
disk.state = 'OK'
disk.is_ssd = True
disk.storagerouter = storage_router
disk.save()
disk_partition = DiskPartition()
disk_partition.id = 'disk_partition_id'
disk_partition.disk = disk
disk_partition.path = '/dev/disk/non-existent'
disk_partition.size = 400 * 1024**3
disk_partition.state = 'OK'
disk_partition.offset = 1024
disk_partition.roles = [DiskPartition.ROLES.SCRUB]
disk_partition.mountpoint = '/var/tmp'
disk_partition.save()
vmachine_1 = VMachine()
vmachine_1.name = 'vmachine_1'
vmachine_1.devicename = 'dummy'
vmachine_1.pmachine = pmachine
vmachine_1.save()
vdisk_1_1 = VDisk()
vdisk_1_1.name = 'vdisk_1_1'
vdisk_1_1.volume_id = 'vdisk_1_1'
vdisk_1_1.vmachine = vmachine_1
vdisk_1_1.vpool = vpool
vdisk_1_1.devicename = 'dummy'
vdisk_1_1.size = 0
vdisk_1_1.save()
vdisk_1_1.reload_client()
vdisk_1_2 = VDisk()
vdisk_1_2.name = 'vdisk_1_2'
vdisk_1_2.volume_id = 'vdisk_1_2'
vdisk_1_2.vmachine = vmachine_1
vdisk_1_2.vpool = vpool
vdisk_1_2.devicename = 'dummy'
vdisk_1_2.size = 0
vdisk_1_2.save()
vdisk_1_2.reload_client()
vmachine_2 = VMachine()
vmachine_2.name = 'vmachine_2'
vmachine_2.devicename = 'dummy'
vmachine_2.pmachine = pmachine
vmachine_2.save()
vdisk_2_1 = VDisk()
vdisk_2_1.name = 'vdisk_2_1'
vdisk_2_1.volume_id = 'vdisk_2_1'
vdisk_2_1.vmachine = vmachine_2
vdisk_2_1.vpool = vpool
vdisk_2_1.devicename = 'dummy'
vdisk_2_1.size = 0
vdisk_2_1.save()
vdisk_2_1.reload_client()
vdisk_3 = VDisk()
vdisk_3.name = 'vdisk_3'
vdisk_3.volume_id = 'vdisk_3'
vdisk_3.vpool = vpool
vdisk_3.devicename = 'dummy'
vdisk_3.size = 0
vdisk_3.save()
vdisk_3.reload_client()
for disk in [vdisk_1_1, vdisk_1_2, vdisk_2_1, vdisk_3]:
[
dynamic for dynamic in disk._dynamics
if dynamic.name == 'snapshots'
][0].timeout = 0
# Run the testing scenario
travis = 'TRAVIS' in os.environ and os.environ['TRAVIS'] == 'true'
if travis is True:
print 'Running in Travis, reducing output.'
debug = not travis
amount_of_days = 50
base = datetime.datetime.now().date()
day = datetime.timedelta(1)
minute = 60
hour = minute * 60
for d in xrange(0, amount_of_days):
base_timestamp = self._make_timestamp(base, day * d)
print ''
print 'Day cycle: {0}: {1}'.format(
d,
datetime.datetime.fromtimestamp(base_timestamp).strftime(
'%Y-%m-%d'))
# At the start of the day, delete snapshot policy runs at 00:30
print '- Deleting snapshots'
ScheduledTaskController.delete_snapshots(timestamp=base_timestamp +
(minute * 30))
# Validate snapshots
print '- Validating snapshots'
for vdisk in [vdisk_1_1, vdisk_1_2, vdisk_2_1, vdisk_3]:
self._validate(vdisk, d, base, amount_of_days, debug)
# During the day, snapshots are taken
# - Create non consistent snapshot every hour, between 2:00 and 22:00
# - Create consistent snapshot at 6:30, 12:30, 18:30
print '- Creating snapshots'
for h in xrange(2, 23):
timestamp = base_timestamp + (hour * h)
for vm in [vmachine_1, vmachine_2]:
VMachineController.snapshot(machineguid=vm.guid,
label='ss_i_{0}:00'.format(
str(h)),
is_consistent=False,
timestamp=timestamp)
if h in [6, 12, 18]:
ts = (timestamp + (minute * 30))
VMachineController.snapshot(machineguid=vm.guid,
label='ss_c_{0}:30'.format(
str(h)),
is_consistent=True,
timestamp=ts)
VDiskController.create_snapshot(diskguid=vdisk_3.guid,
metadata={
'label':
'ss_i_{0}:00'.format(
str(h)),
'is_consistent':
False,
'timestamp':
str(timestamp),
'machineguid':
None
})
if h in [6, 12, 18]:
ts = (timestamp + (minute * 30))
VDiskController.create_snapshot(diskguid=vdisk_3.guid,
metadata={
'label':
'ss_c_{0}:30'.format(
str(h)),
'is_consistent':
True,
'timestamp':
str(ts),
'machineguid':
None
})
def sync_with_reality(storagerouter_guid=None):
"""
Syncs the Disks from all StorageRouters with the reality.
:param storagerouter_guid: Guid of the Storage Router to synchronize
"""
storagerouters = []
if storagerouter_guid is not None:
storagerouters.append(StorageRouter(storagerouter_guid))
else:
storagerouters = StorageRouterList.get_storagerouters()
for storagerouter in storagerouters:
try:
client = SSHClient(storagerouter, username='******')
except UnableToConnectException:
DiskController._logger.info(
'Could not connect to StorageRouter {0}, skipping'.format(
storagerouter.ip))
continue
configuration = {}
# Gather mount data
mount_mapping = {}
mount_data = client.run('mount')
for mount in mount_data.splitlines():
mount = mount.strip()
match = re.search('(/dev/(.+?)) on (/.*?) type.*', mount)
if match is not None:
dev_name = match.groups()[0]
uuid = client.run(
'blkid -o value -s UUID {0}'.format(dev_name))
if uuid:
mount_mapping[uuid] = match.groups()[2]
else:
mount_mapping[match.groups()[1]] = match.groups()[2]
# Gather raid information
try:
md_information = client.run('mdadm --detail /dev/md*',
suppress_logging=True)
except CalledProcessError:
md_information = ''
raid_members = []
for member in re.findall('(?: +[0-9]+){4} +[^/]+/dev/([a-z0-9]+)',
md_information):
raid_members.append(member)
# Gather disk information
with remote(storagerouter.ip, [Context, os]) as rem:
context = rem.Context()
devices = [
device
for device in context.list_devices(subsystem='block')
if ('ID_TYPE' in device and device['ID_TYPE'] == 'disk') or
('DEVNAME' in device and
('loop' in device['DEVNAME'] or 'nvme' in
device['DEVNAME'] or 'md' in device['DEVNAME']))
]
for device in devices:
is_partition = device['DEVTYPE'] == 'partition'
device_path = device['DEVNAME']
device_name = device_path.split('/')[-1]
partition_id = None
partition_name = None
extended_partition_info = None
if is_partition is True:
partition_name = device[
'ID_FS_UUID'] if 'ID_FS_UUID' in device else device_name
if 'ID_PART_ENTRY_NUMBER' in device:
extended_partition_info = True
partition_id = device['ID_PART_ENTRY_NUMBER']
if device_name.startswith(
'nvme') or device_name.startswith('loop'):
device_name = device_name[:0 -
int(len(partition_id)
) - 1]
elif device_name.startswith('md'):
device_name = device_name[:device_name.
index('p')]
else:
device_name = device_name[:0 -
int(len(partition_id)
)]
else:
DiskController._logger.debug(
'Partition {0} has no partition metadata'.
format(device_path))
extended_partition_info = False
match = re.match('^(\D+?)(\d+)$', device_name)
if match is None:
DiskController._logger.debug(
'Could not handle disk/partition {0}'.
format(device_path))
continue # Unable to handle this disk/partition
partition_id = match.groups()[1]
device_name = match.groups()[0]
sectors = int(
client.run(
'cat /sys/block/{0}/size'.format(device_name)))
sector_size = int(
client.run(
'cat /sys/block/{0}/queue/hw_sector_size'.format(
device_name)))
rotational = int(
client.run(
'cat /sys/block/{0}/queue/rotational'.format(
device_name)))
if sectors == 0:
continue
if device_name in raid_members:
continue
if device_name not in configuration:
configuration[device_name] = {'partitions': {}}
path = None
for path_type in ['by-id', 'by-uuid']:
if path is not None:
break
if 'DEVLINKS' in device:
for item in device['DEVLINKS'].split(' '):
if path_type in item:
path = item
if path is None:
path = device_path
if is_partition is True:
if 'ID_PART_ENTRY_TYPE' in device and device[
'ID_PART_ENTRY_TYPE'] == '0x5':
continue # This is an extended partition, let's skip that one
if extended_partition_info is True:
offset = int(
device['ID_PART_ENTRY_OFFSET']) * sector_size
size = int(
device['ID_PART_ENTRY_SIZE']) * sector_size
else:
match = re.match('^(\D+?)(\d+)$', device_path)
if match is None:
DiskController._logger.debug(
'Could not handle disk/partition {0}'.
format(device_path))
continue # Unable to handle this disk/partition
partitions_info = DiskTools.get_partitions_info(
match.groups()[0])
if device_path in partitions_info:
partition_info = partitions_info[device_path]
offset = int(partition_info['start'])
size = int(partition_info['size'])
else:
DiskController._logger.warning(
'Could not retrieve partition info for disk/partition {0}'
.format(device_path))
continue
configuration[device_name]['partitions'][
partition_id] = {
'offset': offset,
'size': size,
'path': path,
'state': 'OK'
}
partition_data = configuration[device_name][
'partitions'][partition_id]
if partition_name in mount_mapping:
mountpoint = mount_mapping[partition_name]
partition_data['mountpoint'] = mountpoint
partition_data['inode'] = rem.os.stat(
mountpoint).st_dev
del mount_mapping[partition_name]
try:
client.run('touch {0}/{1}; rm {0}/{1}'.format(
mountpoint, str(time.time())))
except CalledProcessError:
partition_data['state'] = 'FAILURE'
pass
if 'ID_FS_TYPE' in device:
partition_data['filesystem'] = device['ID_FS_TYPE']
else:
configuration[device_name].update({
'name':
device_name,
'path':
path,
'vendor':
device['ID_VENDOR']
if 'ID_VENDOR' in device else None,
'model':
device['ID_MODEL']
if 'ID_MODEL' in device else None,
'size':
sector_size * sectors,
'is_ssd':
rotational == 0,
'state':
'OK'
})
for partition_name in mount_mapping:
device_name = partition_name.split('/')[-1]
match = re.search('^(\D+?)(\d+)$', device_name)
if match is not None:
device_name = match.groups()[0]
partition_id = match.groups()[1]
if device_name not in configuration:
configuration[device_name] = {
'partitions': {},
'state': 'MISSING'
}
configuration[device_name]['partitions'][
partition_id] = {
'mountpoint':
mount_mapping[partition_name],
'state': 'MISSING'
}
# Sync the model
disk_names = []
for disk in storagerouter.disks:
if disk.name not in configuration:
for partition in disk.partitions:
partition.delete()
disk.delete()
else:
disk_names.append(disk.name)
DiskController._update_disk(disk, configuration[disk.name])
partitions = []
partition_info = configuration[disk.name]['partitions']
for partition in disk.partitions:
if partition.id not in partition_info:
partition.delete()
else:
partitions.append(partition.id)
DiskController._update_partition(
partition, partition_info[partition.id])
for partition_id in partition_info:
if partition_id not in partitions:
DiskController._create_partition(
partition_id, partition_info[partition_id],
disk)
for disk_name in configuration:
if disk_name not in disk_names and configuration[disk_name][
'state'] not in ['MISSING']:
disk = Disk()
disk.storagerouter = storagerouter
disk.name = disk_name
DiskController._update_disk(disk, configuration[disk_name])
partition_info = configuration[disk_name]['partitions']
for partition_id in partition_info:
if partition_info[partition_id]['state'] not in [
'MISSING'
]:
DiskController._create_partition(
partition_id, partition_info[partition_id],
disk)
def sync_with_reality(storagerouter_guid):
"""
Syncs the Disks from all StorageRouters with the reality.
CHANGES MADE TO THIS CODE SHOULD BE REFLECTED IN THE ASD-MANAGER list_disks CALL TOO!!!!!!!!!!!!!!!!!!!!
:param storagerouter_guid: Guid of the Storage Router to synchronize
:type storagerouter_guid: str
:return: None
"""
storagerouter = StorageRouter(storagerouter_guid)
try:
client = SSHClient(storagerouter, username='******')
except UnableToConnectException:
DiskController._logger.exception('Could not connect to StorageRouter {0}'.format(storagerouter.ip))
raise
# Retrieve all symlinks for all devices
# Example of name_alias_mapping:
# {'/dev/md0': ['/dev/disk/by-id/md-uuid-ad2de634:26d97253:5eda0a23:96986b76', '/dev/disk/by-id/md-name-OVS-1:0'],
# '/dev/sda': ['/dev/disk/by-path/pci-0000:03:00.0-sas-0x5000c295fe2ff771-lun-0'],
# '/dev/sda1': ['/dev/disk/by-uuid/e3e0bc62-4edc-4c6b-a6ce-1f39e8f27e41', '/dev/disk/by-path/pci-0000:03:00.0-sas-0x5000c295fe2ff771-lun-0-part1']}
name_alias_mapping = {}
alias_name_mapping = {}
for path_type in client.dir_list(directory='/dev/disk'):
if path_type in ['by-uuid', 'by-partuuid']: # UUIDs can change after creating a filesystem on a partition
continue
directory = '/dev/disk/{0}'.format(path_type)
for symlink in client.dir_list(directory=directory):
symlink_path = '{0}/{1}'.format(directory, symlink)
link = client.file_read_link(symlink_path)
if link not in name_alias_mapping:
name_alias_mapping[link] = []
name_alias_mapping[link].append(symlink_path)
alias_name_mapping[symlink_path] = link
# Parse 'lsblk' output
# --exclude 1 for RAM devices, 2 for floppy devices, 11 for CD-ROM devices (See https://www.kernel.org/doc/Documentation/devices.txt)
command = ['lsblk', '--pairs', '--bytes', '--noheadings', '--exclude', '1,2,11']
output = '--output=KNAME,SIZE,MODEL,STATE,MAJ:MIN,FSTYPE,TYPE,ROTA,MOUNTPOINT,LOG-SEC{0}'
regex = '^KNAME="(?P<name>.*)" SIZE="(?P<size>\d*)" MODEL="(?P<model>.*)" STATE="(?P<state>.*)" MAJ:MIN="(?P<dev_nr>.*)" FSTYPE="(?P<fstype>.*)" TYPE="(?P<type>.*)" ROTA="(?P<rota>[0,1])" MOUNTPOINT="(?P<mtpt>.*)" LOG-SEC="(?P<sector_size>\d*)"( SERIAL="(?P<serial>.*)")?$'
try:
devices = client.run(command + [output.format(',SERIAL')]).splitlines()
except:
devices = client.run(command + [output.format('')]).splitlines()
device_regex = re.compile(regex)
configuration = {}
parsed_devices = []
for device in devices:
match = re.match(device_regex, device)
if match is None:
DiskController._logger.error('Device regex did not match for {0}. Please investigate'.format(device))
raise Exception('Failed to parse \'lsblk\' output')
groupdict = match.groupdict()
name = groupdict['name'].strip()
size = groupdict['size'].strip()
model = groupdict['model'].strip()
state = groupdict['state'].strip()
dev_nr = groupdict['dev_nr'].strip()
serial = (groupdict['serial'] or '').strip()
fs_type = groupdict['fstype'].strip()
dev_type = groupdict['type'].strip()
rotational = groupdict['rota'].strip()
mount_point = groupdict['mtpt'].strip()
sector_size = groupdict['sector_size'].strip()
if dev_type == 'rom':
continue
link = client.file_read_link('/sys/block/{0}'.format(name))
device_state = None
friendly_path = '/dev/{0}'.format(name)
system_aliases = name_alias_mapping.get(friendly_path, [friendly_path])
device_is_also_partition = False
if link is not None: # If this returns, it means its a device and not a partition
device_is_also_partition = mount_point != '' # LVM, RAID1, ... have the tendency to be a device with a partition on it, but the partition is not reported by 'lsblk'
device_state = Disk.STATES.OK if state == 'running' or dev_nr.split(':')[0] != '8' else Disk.STATES.FAILURE
parsed_devices.append({'name': name,
'state': device_state})
configuration[name] = {'name': name,
'size': int(size),
'model': model if model != '' else None,
'serial': serial if serial != '' else None,
'state': device_state,
'is_ssd': rotational == '0',
'aliases': system_aliases,
'partitions': {}}
if link is None or device_is_also_partition is True:
current_device = None
current_device_state = None
if device_is_also_partition is True:
offset = 0
current_device = name
current_device_state = device_state
else:
offset = 0
for device_info in reversed(parsed_devices):
try:
current_device = device_info['name']
current_device_state = device_info['state']
offset = int(client.file_read('/sys/block/{0}/{1}/start'.format(current_device, name))) * int(sector_size)
break
except Exception:
pass
if current_device is None:
raise RuntimeError('Failed to retrieve the device information for current partition')
mount_point = mount_point if mount_point != '' else None
partition_state = Disk.STATES.OK if current_device_state == Disk.STATES.OK else Disk.STATES.FAILURE
if mount_point is not None and fs_type != 'swap':
try:
filename = '{0}/{1}'.format(mount_point, str(time.time()))
client.run(['touch', filename])
client.run(['rm', filename])
except Exception:
partition_state = Disk.STATES.FAILURE
configuration[current_device]['partitions'][offset] = {'size': int(size),
'state': partition_state,
'offset': offset,
'aliases': system_aliases,
'filesystem': fs_type if fs_type != '' else None,
'mountpoint': mount_point}
# Sync the model
for disk in storagerouter.disks:
disk_info = None
for alias in disk.aliases:
if alias in alias_name_mapping:
name = alias_name_mapping[alias].replace('/dev/', '')
if name in configuration:
disk_info = configuration.pop(name)
break
if disk_info is None and disk.name in configuration and (disk.name.startswith('fio') or
disk.name.startswith('loop') or
disk.name.startswith('nvme')): # Partitioned loop, nvme devices no longer show up in alias_name_mapping
disk_info = configuration.pop(disk.name)
# Remove disk / partitions if not reported by 'lsblk'
if disk_info is None:
DiskController._logger.info('Disk {0} - No longer found'.format(disk.name))
delete = True
for partition in disk.partitions:
if len(partition.roles) > 0:
delete = False
DiskController._logger.warning('Disk {0} - Partition with offset {1} - Has roles, will not delete'.format(disk.name, partition.offset))
break
if delete is True:
for partition in disk.partitions:
partition.delete()
disk.delete()
DiskController._logger.info('Disk {0} - Deleted'.format(disk.name))
else:
for partition in disk.partitions:
DiskController._update_partition(partition, {'state': 'MISSING'})
DiskController._logger.warning('Disk {0} - Partition with offset {1} - Updated status to MISSING'.format(disk.name, partition.offset))
DiskController._update_disk(disk, {'state': 'MISSING'})
DiskController._logger.warning('Disk {0} - Updated status to MISSING'.format(disk.name))
else: # Update existing disks and their partitions
DiskController._logger.info('Disk {0} - Found, updating'.format(disk.name))
DiskController._update_disk(disk, disk_info)
partition_info = disk_info['partitions']
for partition in disk.partitions:
if partition.offset not in partition_info:
DiskController._logger.info('Disk {0} - Partition with offset {1} - No longer found'.format(disk.name, partition.offset))
if len(partition.roles) > 0:
DiskController._logger.warning('Disk {0} - Partition with offset {1} - Update status to MISSING'.format(disk.name, partition.offset))
DiskController._update_partition(partition, {'state': 'MISSING'})
else:
DiskController._logger.info('Disk {0} - Partition with offset {1} - Deleting'.format(disk.name, partition.offset))
partition.delete()
else:
DiskController._update_partition(partition, partition_info.pop(partition.offset))
for partition_offset in partition_info:
DiskController._logger.info('Disk {0} - Creating partition - {1}'.format(disk.name, partition_info[partition_offset]))
DiskController._create_partition(partition_info[partition_offset], disk)
# Create all disks and their partitions not yet modeled
for disk_name in configuration:
DiskController._logger.info('Disk {0} - Creating disk - {1}'.format(disk_name, configuration[disk_name]))
disk = Disk()
disk.storagerouter = storagerouter
disk.name = disk_name
DiskController._update_disk(disk, configuration[disk_name])
partition_info = configuration[disk_name]['partitions']
for partition_offset in partition_info:
DiskController._create_partition(partition_info[partition_offset], disk)
def sync_with_reality(storagerouter_guid):
"""
Syncs the Disks from all StorageRouters with the reality.
CHANGES MADE TO THIS CODE SHOULD BE REFLECTED IN THE ASD-MANAGER list_disks CALL TOO!!!!!!!!!!!!!!!!!!!!
:param storagerouter_guid: Guid of the Storage Router to synchronize
:type storagerouter_guid: str
:return: None
"""
storagerouter = StorageRouter(storagerouter_guid)
try:
client = SSHClient(storagerouter, username='******')
except UnableToConnectException:
DiskController._logger.exception(
'Could not connect to StorageRouter {0}'.format(
storagerouter.ip))
raise
# Retrieve all symlinks for all devices
# Example of name_alias_mapping:
# {'/dev/md0': ['/dev/disk/by-id/md-uuid-ad2de634:26d97253:5eda0a23:96986b76', '/dev/disk/by-id/md-name-OVS-1:0'],
# '/dev/sda': ['/dev/disk/by-path/pci-0000:03:00.0-sas-0x5000c295fe2ff771-lun-0'],
# '/dev/sda1': ['/dev/disk/by-uuid/e3e0bc62-4edc-4c6b-a6ce-1f39e8f27e41', '/dev/disk/by-path/pci-0000:03:00.0-sas-0x5000c295fe2ff771-lun-0-part1']}
name_alias_mapping = {}
alias_name_mapping = {}
for path_type in client.dir_list(directory='/dev/disk'):
if path_type in [
'by-uuid', 'by-partuuid'
]: # UUIDs can change after creating a filesystem on a partition
continue
directory = '/dev/disk/{0}'.format(path_type)
for symlink in client.dir_list(directory=directory):
symlink_path = '{0}/{1}'.format(directory, symlink)
link = client.file_read_link(symlink_path)
if link not in name_alias_mapping:
name_alias_mapping[link] = []
name_alias_mapping[link].append(symlink_path)
alias_name_mapping[symlink_path] = link
# Parse 'lsblk' output
# --exclude 1 for RAM devices, 2 for floppy devices, 11 for CD-ROM devices (See https://www.kernel.org/doc/Documentation/devices.txt)
devices = client.run([
'lsblk', '--pairs', '--bytes', '--noheadings', '--exclude',
'1,2,11',
'--output=KNAME,SIZE,MODEL,STATE,MAJ:MIN,FSTYPE,TYPE,ROTA,MOUNTPOINT,LOG-SEC'
]).splitlines()
device_regex = re.compile(
'^KNAME="(?P<name>.*)" SIZE="(?P<size>\d*)" MODEL="(?P<model>.*)" STATE="(?P<state>.*)" MAJ:MIN="(?P<dev_nr>.*)" FSTYPE="(?P<fstype>.*)" TYPE="(?P<type>.*)" ROTA="(?P<rota>[0,1])" MOUNTPOINT="(?P<mtpt>.*)" LOG-SEC="(?P<sector_size>\d*)"$'
)
configuration = {}
parsed_devices = []
for device in devices:
match = re.match(device_regex, device)
if match is None:
DiskController._logger.error(
'Device regex did not match for {0}. Please investigate'.
format(device))
raise Exception('Failed to parse \'lsblk\' output')
groupdict = match.groupdict()
name = groupdict['name'].strip()
size = groupdict['size'].strip()
model = groupdict['model'].strip()
state = groupdict['state'].strip()
dev_nr = groupdict['dev_nr'].strip()
fs_type = groupdict['fstype'].strip()
dev_type = groupdict['type'].strip()
rotational = groupdict['rota'].strip()
mount_point = groupdict['mtpt'].strip()
sector_size = groupdict['sector_size'].strip()
if dev_type == 'rom':
continue
link = client.file_read_link('/sys/block/{0}'.format(name))
device_state = None
friendly_path = '/dev/{0}'.format(name)
system_aliases = name_alias_mapping.get(friendly_path,
[friendly_path])
device_is_also_partition = False
if link is not None: # If this returns, it means its a device and not a partition
device_is_also_partition = mount_point != '' # LVM, RAID1, ... have the tendency to be a device with a partition on it, but the partition is not reported by 'lsblk'
device_state = Disk.STATES.OK if state == 'running' or dev_nr.split(
':')[0] != '8' else Disk.STATES.FAILURE
parsed_devices.append({'name': name, 'state': device_state})
configuration[name] = {
'name': name,
'size': int(size),
'model': model if model != '' else None,
'state': device_state,
'is_ssd': rotational == '0',
'aliases': system_aliases,
'partitions': {}
}
if link is None or device_is_also_partition is True:
current_device = None
current_device_state = None
if device_is_also_partition is True:
offset = 0
current_device = name
current_device_state = device_state
else:
offset = 0
for device_info in reversed(parsed_devices):
try:
current_device = device_info['name']
current_device_state = device_info['state']
offset = int(
client.file_read(
'/sys/block/{0}/{1}/start'.format(
current_device,
name))) * int(sector_size)
break
except Exception:
pass
if current_device is None:
raise RuntimeError(
'Failed to retrieve the device information for current partition'
)
mount_point = mount_point if mount_point != '' else None
partition_state = Disk.STATES.OK if current_device_state == Disk.STATES.OK else Disk.STATES.FAILURE
if mount_point is not None and fs_type != 'swap':
try:
filename = '{0}/{1}'.format(mount_point,
str(time.time()))
client.run(['touch', filename])
client.run(['rm', filename])
except Exception:
partition_state = Disk.STATES.FAILURE
configuration[current_device]['partitions'][offset] = {
'size': int(size),
'state': partition_state,
'offset': offset,
'aliases': system_aliases,
'filesystem': fs_type if fs_type != '' else None,
'mountpoint': mount_point
}
# Sync the model
for disk in storagerouter.disks:
disk_info = None
for alias in disk.aliases:
if alias in alias_name_mapping:
name = alias_name_mapping[alias].replace('/dev/', '')
if name in configuration:
disk_info = configuration.pop(name)
break
if disk_info is None and disk.name in configuration and (
disk.name.startswith('fio') or disk.name.startswith('loop')
or disk.name.startswith('nvme')
): # Partitioned loop, nvme devices no longer show up in alias_name_mapping
disk_info = configuration.pop(disk.name)
# Remove disk / partitions if not reported by 'lsblk'
if disk_info is None:
DiskController._logger.info(
'Disk {0} - No longer found'.format(disk.name))
delete = True
for partition in disk.partitions:
if len(partition.roles) > 0:
delete = False
DiskController._logger.warning(
'Disk {0} - Partition with offset {1} - Has roles, will not delete'
.format(disk.name, partition.offset))
break
if delete is True:
for partition in disk.partitions:
partition.delete()
disk.delete()
DiskController._logger.info('Disk {0} - Deleted'.format(
disk.name))
else:
for partition in disk.partitions:
DiskController._update_partition(
partition, {'state': 'MISSING'})
DiskController._logger.warning(
'Disk {0} - Partition with offset {1} - Updated status to MISSING'
.format(disk.name, partition.offset))
DiskController._update_disk(disk, {'state': 'MISSING'})
DiskController._logger.warning(
'Disk {0} - Updated status to MISSING'.format(
disk.name))
else: # Update existing disks and their partitions
DiskController._logger.info(
'Disk {0} - Found, updating'.format(disk.name))
DiskController._update_disk(disk, disk_info)
partition_info = disk_info['partitions']
for partition in disk.partitions:
if partition.offset not in partition_info:
DiskController._logger.info(
'Disk {0} - Partition with offset {1} - No longer found'
.format(disk.name, partition.offset))
if len(partition.roles) > 0:
DiskController._logger.warning(
'Disk {0} - Partition with offset {1} - Update status to MISSING'
.format(disk.name, partition.offset))
DiskController._update_partition(
partition, {'state': 'MISSING'})
else:
DiskController._logger.info(
'Disk {0} - Partition with offset {1} - Deleting'
.format(disk.name, partition.offset))
partition.delete()
else:
DiskController._update_partition(
partition, partition_info.pop(partition.offset))
for partition_offset in partition_info:
DiskController._logger.info(
'Disk {0} - Creating partition - {1}'.format(
disk.name, partition_info[partition_offset]))
DiskController._create_partition(
partition_info[partition_offset], disk)
# Create all disks and their partitions not yet modeled
for disk_name in configuration:
DiskController._logger.info(
'Disk {0} - Creating disk - {1}'.format(
disk_name, configuration[disk_name]))
disk = Disk()
disk.storagerouter = storagerouter
disk.name = disk_name
DiskController._update_disk(disk, configuration[disk_name])
partition_info = configuration[disk_name]['partitions']
for partition_offset in partition_info:
DiskController._create_partition(
partition_info[partition_offset], disk)
def test_happypath(self):
"""
Validates the happy path; Hourly snapshots are taken with a few manual consistent
every now an then. The delete policy is executed every day
"""
# Setup
# There are 2 machines; one with two disks, one with one disk and an additional disk
failure_domain = FailureDomain()
failure_domain.name = "Test"
failure_domain.save()
backend_type = BackendType()
backend_type.name = "BackendType"
backend_type.code = "BT"
backend_type.save()
vpool = VPool()
vpool.name = "vpool"
vpool.backend_type = backend_type
vpool.save()
pmachine = PMachine()
pmachine.name = "PMachine"
pmachine.username = "******"
pmachine.ip = "127.0.0.1"
pmachine.hvtype = "VMWARE"
pmachine.save()
storage_router = StorageRouter()
storage_router.name = "storage_router"
storage_router.ip = "127.0.0.1"
storage_router.pmachine = pmachine
storage_router.machine_id = System.get_my_machine_id()
storage_router.rdma_capable = False
storage_router.primary_failure_domain = failure_domain
storage_router.save()
disk = Disk()
disk.name = "physical_disk_1"
disk.path = "/dev/non-existent"
disk.size = 500 * 1024 ** 3
disk.state = "OK"
disk.is_ssd = True
disk.storagerouter = storage_router
disk.save()
disk_partition = DiskPartition()
disk_partition.id = "disk_partition_id"
disk_partition.disk = disk
disk_partition.path = "/dev/disk/non-existent"
disk_partition.size = 400 * 1024 ** 3
disk_partition.state = "OK"
disk_partition.offset = 1024
disk_partition.roles = [DiskPartition.ROLES.SCRUB]
disk_partition.mountpoint = "/var/tmp"
disk_partition.save()
vmachine_1 = VMachine()
vmachine_1.name = "vmachine_1"
vmachine_1.devicename = "dummy"
vmachine_1.pmachine = pmachine
vmachine_1.save()
vdisk_1_1 = VDisk()
vdisk_1_1.name = "vdisk_1_1"
vdisk_1_1.volume_id = "vdisk_1_1"
vdisk_1_1.vmachine = vmachine_1
vdisk_1_1.vpool = vpool
vdisk_1_1.devicename = "dummy"
vdisk_1_1.size = 0
vdisk_1_1.save()
vdisk_1_1.reload_client()
vdisk_1_2 = VDisk()
vdisk_1_2.name = "vdisk_1_2"
vdisk_1_2.volume_id = "vdisk_1_2"
vdisk_1_2.vmachine = vmachine_1
vdisk_1_2.vpool = vpool
vdisk_1_2.devicename = "dummy"
vdisk_1_2.size = 0
vdisk_1_2.save()
vdisk_1_2.reload_client()
vmachine_2 = VMachine()
vmachine_2.name = "vmachine_2"
vmachine_2.devicename = "dummy"
vmachine_2.pmachine = pmachine
vmachine_2.save()
vdisk_2_1 = VDisk()
vdisk_2_1.name = "vdisk_2_1"
vdisk_2_1.volume_id = "vdisk_2_1"
vdisk_2_1.vmachine = vmachine_2
vdisk_2_1.vpool = vpool
vdisk_2_1.devicename = "dummy"
vdisk_2_1.size = 0
vdisk_2_1.save()
vdisk_2_1.reload_client()
vdisk_3 = VDisk()
vdisk_3.name = "vdisk_3"
vdisk_3.volume_id = "vdisk_3"
vdisk_3.vpool = vpool
vdisk_3.devicename = "dummy"
vdisk_3.size = 0
vdisk_3.save()
vdisk_3.reload_client()
for disk in [vdisk_1_1, vdisk_1_2, vdisk_2_1, vdisk_3]:
[dynamic for dynamic in disk._dynamics if dynamic.name == "snapshots"][0].timeout = 0
# Run the testing scenario
debug = True
amount_of_days = 50
base = datetime.now().date()
day = timedelta(1)
minute = 60
hour = minute * 60
for d in xrange(0, amount_of_days):
base_timestamp = DeleteSnapshots._make_timestamp(base, day * d)
print ""
print "Day cycle: {0}: {1}".format(d, datetime.fromtimestamp(base_timestamp).strftime("%Y-%m-%d"))
# At the start of the day, delete snapshot policy runs at 00:30
print "- Deleting snapshots"
ScheduledTaskController.delete_snapshots(timestamp=base_timestamp + (minute * 30))
# Validate snapshots
print "- Validating snapshots"
for vdisk in [vdisk_1_1, vdisk_1_2, vdisk_2_1, vdisk_3]:
self._validate(vdisk, d, base, amount_of_days, debug)
# During the day, snapshots are taken
# - Create non consistent snapshot every hour, between 2:00 and 22:00
# - Create consistent snapshot at 6:30, 12:30, 18:30
print "- Creating snapshots"
for h in xrange(2, 23):
timestamp = base_timestamp + (hour * h)
for vm in [vmachine_1, vmachine_2]:
VMachineController.snapshot(
machineguid=vm.guid,
label="ss_i_{0}:00".format(str(h)),
is_consistent=False,
timestamp=timestamp,
)
if h in [6, 12, 18]:
ts = timestamp + (minute * 30)
VMachineController.snapshot(
machineguid=vm.guid, label="ss_c_{0}:30".format(str(h)), is_consistent=True, timestamp=ts
)
VDiskController.create_snapshot(
diskguid=vdisk_3.guid,
metadata={
"label": "ss_i_{0}:00".format(str(h)),
"is_consistent": False,
"timestamp": str(timestamp),
"machineguid": None,
},
)
if h in [6, 12, 18]:
ts = timestamp + (minute * 30)
VDiskController.create_snapshot(
diskguid=vdisk_3.guid,
metadata={
"label": "ss_c_{0}:30".format(str(h)),
"is_consistent": True,
"timestamp": str(ts),
"machineguid": None,
},
)
def test_happypath(self):
"""
Validates the happy path; Hourly snapshots are taken with a few manual consistent
every now an then. The delete policy is executed every day
"""
# Setup
# There are 2 machines; one with two disks, one with one disk and a stand-alone additional disk
failure_domain = FailureDomain()
failure_domain.name = 'Test'
failure_domain.save()
backend_type = BackendType()
backend_type.name = 'BackendType'
backend_type.code = 'BT'
backend_type.save()
vpool = VPool()
vpool.name = 'vpool'
vpool.status = 'RUNNING'
vpool.backend_type = backend_type
vpool.save()
pmachine = PMachine()
pmachine.name = 'PMachine'
pmachine.username = '******'
pmachine.ip = '127.0.0.1'
pmachine.hvtype = 'VMWARE'
pmachine.save()
storage_router = StorageRouter()
storage_router.name = 'storage_router'
storage_router.ip = '127.0.0.1'
storage_router.pmachine = pmachine
storage_router.machine_id = System.get_my_machine_id()
storage_router.rdma_capable = False
storage_router.primary_failure_domain = failure_domain
storage_router.save()
disk = Disk()
disk.name = 'physical_disk_1'
disk.path = '/dev/non-existent'
disk.size = 500 * 1024 ** 3
disk.state = 'OK'
disk.is_ssd = True
disk.storagerouter = storage_router
disk.save()
disk_partition = DiskPartition()
disk_partition.id = 'disk_partition_id'
disk_partition.disk = disk
disk_partition.path = '/dev/disk/non-existent'
disk_partition.size = 400 * 1024 ** 3
disk_partition.state = 'OK'
disk_partition.offset = 1024
disk_partition.roles = [DiskPartition.ROLES.SCRUB]
disk_partition.mountpoint = '/var/tmp'
disk_partition.save()
vmachine_1 = VMachine()
vmachine_1.name = 'vmachine_1'
vmachine_1.devicename = 'dummy'
vmachine_1.pmachine = pmachine
vmachine_1.save()
vdisk_1_1 = VDisk()
vdisk_1_1.name = 'vdisk_1_1'
vdisk_1_1.volume_id = 'vdisk_1_1'
vdisk_1_1.vmachine = vmachine_1
vdisk_1_1.vpool = vpool
vdisk_1_1.devicename = 'dummy'
vdisk_1_1.size = 0
vdisk_1_1.save()
vdisk_1_1.reload_client()
vdisk_1_2 = VDisk()
vdisk_1_2.name = 'vdisk_1_2'
vdisk_1_2.volume_id = 'vdisk_1_2'
vdisk_1_2.vmachine = vmachine_1
vdisk_1_2.vpool = vpool
vdisk_1_2.devicename = 'dummy'
vdisk_1_2.size = 0
vdisk_1_2.save()
vdisk_1_2.reload_client()
vmachine_2 = VMachine()
vmachine_2.name = 'vmachine_2'
vmachine_2.devicename = 'dummy'
vmachine_2.pmachine = pmachine
vmachine_2.save()
vdisk_2_1 = VDisk()
vdisk_2_1.name = 'vdisk_2_1'
vdisk_2_1.volume_id = 'vdisk_2_1'
vdisk_2_1.vmachine = vmachine_2
vdisk_2_1.vpool = vpool
vdisk_2_1.devicename = 'dummy'
vdisk_2_1.size = 0
vdisk_2_1.save()
vdisk_2_1.reload_client()
vdisk_3 = VDisk()
vdisk_3.name = 'vdisk_3'
vdisk_3.volume_id = 'vdisk_3'
vdisk_3.vpool = vpool
vdisk_3.devicename = 'dummy'
vdisk_3.size = 0
vdisk_3.save()
vdisk_3.reload_client()
for disk in [vdisk_1_1, vdisk_1_2, vdisk_2_1, vdisk_3]:
[dynamic for dynamic in disk._dynamics if dynamic.name == 'snapshots'][0].timeout = 0
# Run the testing scenario
travis = 'TRAVIS' in os.environ and os.environ['TRAVIS'] == 'true'
if travis is True:
print 'Running in Travis, reducing output.'
debug = not travis
amount_of_days = 50
base = datetime.datetime.now().date()
day = datetime.timedelta(1)
minute = 60
hour = minute * 60
for d in xrange(0, amount_of_days):
base_timestamp = self._make_timestamp(base, day * d)
print ''
print 'Day cycle: {0}: {1}'.format(d, datetime.datetime.fromtimestamp(base_timestamp).strftime('%Y-%m-%d'))
# At the start of the day, delete snapshot policy runs at 00:30
print '- Deleting snapshots'
ScheduledTaskController.delete_snapshots(timestamp=base_timestamp + (minute * 30))
# Validate snapshots
print '- Validating snapshots'
for vdisk in [vdisk_1_1, vdisk_1_2, vdisk_2_1, vdisk_3]:
self._validate(vdisk, d, base, amount_of_days, debug)
# During the day, snapshots are taken
# - Create non consistent snapshot every hour, between 2:00 and 22:00
# - Create consistent snapshot at 6:30, 12:30, 18:30
print '- Creating snapshots'
for h in xrange(2, 23):
timestamp = base_timestamp + (hour * h)
for vm in [vmachine_1, vmachine_2]:
VMachineController.snapshot(machineguid=vm.guid,
label='ss_i_{0}:00'.format(str(h)),
is_consistent=False,
timestamp=timestamp)
if h in [6, 12, 18]:
ts = (timestamp + (minute * 30))
VMachineController.snapshot(machineguid=vm.guid,
label='ss_c_{0}:30'.format(str(h)),
is_consistent=True,
timestamp=ts)
VDiskController.create_snapshot(diskguid=vdisk_3.guid,
metadata={'label': 'ss_i_{0}:00'.format(str(h)),
'is_consistent': False,
'timestamp': str(timestamp),
'machineguid': None})
if h in [6, 12, 18]:
ts = (timestamp + (minute * 30))
VDiskController.create_snapshot(diskguid=vdisk_3.guid,
metadata={'label': 'ss_c_{0}:30'.format(str(h)),
'is_consistent': True,
'timestamp': str(ts),
'machineguid': None})
def test_happypath(self):
"""
Validates the happy path; Hourly snapshots are taken with a few manual consistent
every now and then. The delete policy is executed every day
"""
vpool = VPool()
vpool.name = 'vpool'
vpool.status = 'RUNNING'
vpool.save()
storage_router = StorageRouter()
storage_router.name = 'storage_router'
storage_router.ip = '127.0.0.1'
storage_router.machine_id = System.get_my_machine_id()
storage_router.rdma_capable = False
storage_router.save()
disk = Disk()
disk.name = 'physical_disk_1'
disk.aliases = ['/dev/non-existent']
disk.size = 500 * 1024**3
disk.state = 'OK'
disk.is_ssd = True
disk.storagerouter = storage_router
disk.save()
disk_partition = DiskPartition()
disk_partition.disk = disk
disk_partition.aliases = ['/dev/disk/non-existent']
disk_partition.size = 400 * 1024**3
disk_partition.state = 'OK'
disk_partition.offset = 1024
disk_partition.roles = [DiskPartition.ROLES.SCRUB]
disk_partition.mountpoint = '/var/tmp'
disk_partition.save()
vdisk_1 = VDisk()
vdisk_1.name = 'vdisk_1'
vdisk_1.volume_id = 'vdisk_1'
vdisk_1.vpool = vpool
vdisk_1.devicename = 'dummy'
vdisk_1.size = 0
vdisk_1.save()
vdisk_1.reload_client('storagedriver')
[
dynamic for dynamic in vdisk_1._dynamics
if dynamic.name == 'snapshots'
][0].timeout = 0
# Run the testing scenario
travis = 'TRAVIS' in os.environ and os.environ['TRAVIS'] == 'true'
if travis is True:
self._print_message('Running in Travis, reducing output.')
debug = not travis
amount_of_days = 50
base = datetime.datetime.now().date()
day = datetime.timedelta(1)
minute = 60
hour = minute * 60
for d in xrange(0, amount_of_days):
base_timestamp = self._make_timestamp(base, day * d)
self._print_message('')
self._print_message('Day cycle: {0}: {1}'.format(
d,
datetime.datetime.fromtimestamp(base_timestamp).strftime(
'%Y-%m-%d')))
# At the start of the day, delete snapshot policy runs at 00:30
self._print_message('- Deleting snapshots')
ScheduledTaskController.delete_snapshots(timestamp=base_timestamp +
(minute * 30))
# Validate snapshots
self._print_message('- Validating snapshots')
self._validate(vdisk_1, d, base, amount_of_days, debug)
# During the day, snapshots are taken
# - Create non consistent snapshot every hour, between 2:00 and 22:00
# - Create consistent snapshot at 6:30, 12:30, 18:30
self._print_message('- Creating snapshots')
for h in xrange(2, 23):
timestamp = base_timestamp + (hour * h)
VDiskController.create_snapshot(vdisk_guid=vdisk_1.guid,
metadata={
'label':
'ss_i_{0}:00'.format(
str(h)),
'is_consistent':
False,
'timestamp':
str(timestamp),
'machineguid':
None
})
if h in [6, 12, 18]:
ts = (timestamp + (minute * 30))
VDiskController.create_snapshot(vdisk_guid=vdisk_1.guid,
metadata={
'label':
'ss_c_{0}:30'.format(
str(h)),
'is_consistent':
True,
'timestamp':
str(ts),
'machineguid':
None
})
def test_clone_snapshot(self):
"""
Validates that a snapshot that has clones will not be deleted while other snapshots will be deleted
"""
# Setup
# There are 2 disks, second one cloned from a snapshot of the first
vpool = VPool()
vpool.name = 'vpool'
vpool.status = 'RUNNING'
vpool.save()
storage_router = StorageRouter()
storage_router.name = 'storage_router'
storage_router.ip = '127.0.0.1'
storage_router.machine_id = System.get_my_machine_id()
storage_router.rdma_capable = False
storage_router.save()
disk = Disk()
disk.name = 'physical_disk_1'
disk.aliases = ['/dev/non-existent']
disk.size = 500 * 1024 ** 3
disk.state = 'OK'
disk.is_ssd = True
disk.storagerouter = storage_router
disk.save()
disk_partition = DiskPartition()
disk_partition.disk = disk
disk_partition.aliases = ['/dev/disk/non-existent']
disk_partition.size = 400 * 1024 ** 3
disk_partition.state = 'OK'
disk_partition.offset = 1024
disk_partition.roles = [DiskPartition.ROLES.SCRUB]
disk_partition.mountpoint = '/var/tmp'
disk_partition.save()
storage_driver = StorageDriver()
storage_driver.vpool = vpool
storage_driver.storagerouter = storage_router
storage_driver.name = 'storage_driver_1'
storage_driver.mountpoint = '/'
storage_driver.cluster_ip = storage_router.ip
storage_driver.storage_ip = '127.0.0.1'
storage_driver.storagedriver_id = 'storage_driver_1'
storage_driver.ports = {'management': 1,
'xmlrpc': 2,
'dtl': 3,
'edge': 4}
storage_driver.save()
service_type = ServiceType()
service_type.name = 'MetadataServer'
service_type.save()
service = Service()
service.name = 'service_1'
service.storagerouter = storage_driver.storagerouter
service.ports = [1]
service.type = service_type
service.save()
mds_service = MDSService()
mds_service.service = service
mds_service.number = 0
mds_service.capacity = 10
mds_service.vpool = storage_driver.vpool
mds_service.save()
vdisk_1_1 = VDisk()
vdisk_1_1.name = 'vdisk_1_1'
vdisk_1_1.volume_id = 'vdisk_1_1'
vdisk_1_1.vpool = vpool
vdisk_1_1.devicename = 'dummy'
vdisk_1_1.size = 0
vdisk_1_1.save()
vdisk_1_1.reload_client('storagedriver')
[dynamic for dynamic in vdisk_1_1._dynamics if dynamic.name == 'snapshots'][0].timeout = 0
travis = 'TRAVIS' in os.environ and os.environ['TRAVIS'] == 'true'
if travis is True:
print 'Running in Travis, reducing output.'
base = datetime.datetime.now().date()
day = datetime.timedelta(1)
base_timestamp = self._make_timestamp(base, day)
minute = 60
hour = minute * 60
for h in [6, 12, 18]:
timestamp = base_timestamp + (hour * h)
VDiskController.create_snapshot(vdisk_guid=vdisk_1_1.guid,
metadata={'label': 'snapshot_{0}:30'.format(str(h)),
'is_consistent': True,
'timestamp': str(timestamp),
'machineguid': None})
base_snapshot_guid = vdisk_1_1.snapshots[0]['guid'] # Oldest
clone_vdisk = VDisk()
clone_vdisk.name = 'clone_vdisk'
clone_vdisk.volume_id = 'clone_vdisk'
clone_vdisk.vpool = vpool
clone_vdisk.devicename = 'dummy'
clone_vdisk.parentsnapshot = base_snapshot_guid
clone_vdisk.size = 0
clone_vdisk.save()
clone_vdisk.reload_client('storagedriver')
for h in [6, 12, 18]:
timestamp = base_timestamp + (hour * h)
VDiskController.create_snapshot(vdisk_guid=clone_vdisk.guid,
metadata={'label': 'snapshot_{0}:30'.format(str(h)),
'is_consistent': True,
'timestamp': str(timestamp),
'machineguid': None})
base_timestamp = self._make_timestamp(base, day * 2)
ScheduledTaskController.delete_snapshots(timestamp=base_timestamp + (minute * 30))
self.assertIn(base_snapshot_guid, [snap['guid'] for snap in vdisk_1_1.snapshots], 'Snapshot was deleted while there are still clones of it')
def test_happypath(self):
"""
Validates the happy path; Hourly snapshots are taken with a few manual consistent
every now and then. The delete policy is executed every day
"""
vpool = VPool()
vpool.name = 'vpool'
vpool.status = 'RUNNING'
vpool.save()
storage_router = StorageRouter()
storage_router.name = 'storage_router'
storage_router.ip = '127.0.0.1'
storage_router.machine_id = System.get_my_machine_id()
storage_router.rdma_capable = False
storage_router.save()
disk = Disk()
disk.name = 'physical_disk_1'
disk.aliases = ['/dev/non-existent']
disk.size = 500 * 1024 ** 3
disk.state = 'OK'
disk.is_ssd = True
disk.storagerouter = storage_router
disk.save()
disk_partition = DiskPartition()
disk_partition.disk = disk
disk_partition.aliases = ['/dev/disk/non-existent']
disk_partition.size = 400 * 1024 ** 3
disk_partition.state = 'OK'
disk_partition.offset = 1024
disk_partition.roles = [DiskPartition.ROLES.SCRUB]
disk_partition.mountpoint = '/var/tmp'
disk_partition.save()
vdisk_1 = VDisk()
vdisk_1.name = 'vdisk_1'
vdisk_1.volume_id = 'vdisk_1'
vdisk_1.vpool = vpool
vdisk_1.devicename = 'dummy'
vdisk_1.size = 0
vdisk_1.save()
vdisk_1.reload_client('storagedriver')
[dynamic for dynamic in vdisk_1._dynamics if dynamic.name == 'snapshots'][0].timeout = 0
# Run the testing scenario
travis = 'TRAVIS' in os.environ and os.environ['TRAVIS'] == 'true'
if travis is True:
self._print_message('Running in Travis, reducing output.')
debug = not travis
amount_of_days = 50
base = datetime.datetime.now().date()
day = datetime.timedelta(1)
minute = 60
hour = minute * 60
for d in xrange(0, amount_of_days):
base_timestamp = self._make_timestamp(base, day * d)
self._print_message('')
self._print_message('Day cycle: {0}: {1}'.format(d, datetime.datetime.fromtimestamp(base_timestamp).strftime('%Y-%m-%d')))
# At the start of the day, delete snapshot policy runs at 00:30
self._print_message('- Deleting snapshots')
ScheduledTaskController.delete_snapshots(timestamp=base_timestamp + (minute * 30))
# Validate snapshots
self._print_message('- Validating snapshots')
self._validate(vdisk_1, d, base, amount_of_days, debug)
# During the day, snapshots are taken
# - Create non consistent snapshot every hour, between 2:00 and 22:00
# - Create consistent snapshot at 6:30, 12:30, 18:30
self._print_message('- Creating snapshots')
for h in xrange(2, 23):
timestamp = base_timestamp + (hour * h)
VDiskController.create_snapshot(vdisk_guid=vdisk_1.guid,
metadata={'label': 'ss_i_{0}:00'.format(str(h)),
'is_consistent': False,
'timestamp': str(timestamp),
'machineguid': None})
if h in [6, 12, 18]:
ts = (timestamp + (minute * 30))
VDiskController.create_snapshot(vdisk_guid=vdisk_1.guid,
metadata={'label': 'ss_c_{0}:30'.format(str(h)),
'is_consistent': True,
'timestamp': str(ts),
'machineguid': None})
def sync_with_reality(storagerouter_guid=None):
"""
Syncs the Disks from all StorageRouters with the reality.
"""
storagerouters = []
if storagerouter_guid is not None:
storagerouters.append(StorageRouter(storagerouter_guid))
else:
storagerouters = StorageRouterList.get_storagerouters()
for storagerouter in storagerouters:
try:
client = SSHClient(storagerouter, username='******')
except UnableToConnectException:
logger.info(
'Could not connect to StorageRouter {0}, skipping'.format(
storagerouter.ip))
continue
configuration = {}
# Gather mount data
mount_mapping = {}
mount_data = client.run('mount')
for mount in mount_data.splitlines():
mount = mount.strip()
match = re.search('/dev/(.+?) on (/.*?) type.*', mount)
if match is not None:
mount_mapping[match.groups()[0]] = match.groups()[1]
# Gather disk information
with Remote(storagerouter.ip, [Context, os]) as remote:
context = remote.Context()
devices = [
device
for device in context.list_devices(subsystem='block')
if 'ID_TYPE' in device and device['ID_TYPE'] == 'disk'
]
for device in devices:
is_partition = device['DEVTYPE'] == 'partition'
device_path = device['DEVNAME']
device_name = device_path.split('/')[-1]
partition_id = None
partition_name = None
extended_parition_info = None
if is_partition is True:
if 'ID_PART_ENTRY_NUMBER' in device:
extended_parition_info = True
partition_id = device['ID_PART_ENTRY_NUMBER']
partition_name = device_name
device_name = device_name[:0 -
int(len(partition_id))]
else:
logger.debug(
'Partition {0} has no partition metadata'.
format(device_path))
extended_parition_info = False
match = re.match('^(\D+?)(\d+)$', device_name)
if match is None:
logger.debug(
'Could not handle disk/partition {0}'.
format(device_path))
continue # Unable to handle this disk/partition
partition_name = device_name
partition_id = match.groups()[1]
device_name = match.groups()[0]
if device_name not in configuration:
configuration[device_name] = {'partitions': {}}
path = None
for path_type in ['by-id', 'by-uuid']:
if path is not None:
break
for item in device['DEVLINKS'].split(' '):
if path_type in item:
path = item
if path is None:
path = device_path
sectors = int(
client.run(
'cat /sys/block/{0}/size'.format(device_name)))
sector_size = int(
client.run(
'cat /sys/block/{0}/queue/hw_sector_size'.format(
device_name)))
rotational = int(
client.run(
'cat /sys/block/{0}/queue/rotational'.format(
device_name)))
if is_partition is True:
if 'ID_PART_ENTRY_TYPE' in device and device[
'ID_PART_ENTRY_TYPE'] == '0x5':
continue # This is an extended partition, let's skip that one
if extended_parition_info is True:
offset = int(
device['ID_PART_ENTRY_OFFSET']) * sector_size
size = int(
device['ID_PART_ENTRY_SIZE']) * sector_size
else:
match = re.match('^(\D+?)(\d+)$', device_path)
if match is None:
logger.debug(
'Could not handle disk/partition {0}'.
format(device_path))
continue # Unable to handle this disk/partition
fdisk_info = client.run(
'fdisk -l {0} | grep {1}'.format(
match.groups()[0], device_path)).strip()
fdisk_data = filter(None, fdisk_info.split(' '))
offset = int(fdisk_data[1]) * sector_size
size = (int(fdisk_data[2]) -
int(fdisk_data[1])) * sector_size
configuration[device_name]['partitions'][
partition_id] = {
'offset': offset,
'size': size,
'path': path,
'state': 'OK'
}
partition_data = configuration[device_name][
'partitions'][partition_id]
if partition_name in mount_mapping:
mountpoint = mount_mapping[partition_name]
partition_data['mountpoint'] = mountpoint
partition_data['inode'] = remote.os.stat(
mountpoint).st_dev
del mount_mapping[partition_name]
try:
client.run('touch {0}/{1}; rm {0}/{1}'.format(
mountpoint, str(time.time())))
except CalledProcessError:
partition_data['state'] = 'ERROR'
pass
if 'ID_FS_TYPE' in device:
partition_data['filesystem'] = device['ID_FS_TYPE']
else:
configuration[device_name].update({
'name':
device_name,
'path':
path,
'vendor':
device['ID_VENDOR']
if 'ID_VENDOR' in device else None,
'model':
device['ID_MODEL']
if 'ID_MODEL' in device else None,
'size':
sector_size * sectors,
'is_ssd':
rotational == 0,
'state':
'OK'
})
for partition_name in mount_mapping:
device_name = partition_name.split('/')[-1]
match = re.search('^(\D+?)(\d+)$', device_name)
if match is not None:
device_name = match.groups()[0]
partition_id = match.groups()[1]
if device_name not in configuration:
configuration[device_name] = {
'partitions': {},
'state': 'MISSING'
}
configuration[device_name]['partitions'][
partition_id] = {
'mountpoint':
mount_mapping[partition_name],
'state': 'MISSING'
}
# Sync the model
disk_names = []
for disk in storagerouter.disks:
if disk.name not in configuration:
for partition in disk.partitions:
partition.delete()
disk.delete()
else:
disk_names.append(disk.name)
DiskController._update_disk(disk, configuration[disk.name])
partitions = []
partition_info = configuration[disk.name]['partitions']
for partition in disk.partitions:
if partition.id not in partition_info:
partition.delete()
else:
partitions.append(partition.id)
DiskController._update_partition(
partition, partition_info[partition.id])
for partition_id in partition_info:
if partition_id not in partitions:
DiskController._create_partition(
partition_id, partition_info[partition_id],
disk)
for disk_name in configuration:
if disk_name not in disk_names and configuration[disk_name][
'state'] not in ['MISSING']:
disk = Disk()
disk.storagerouter = storagerouter
disk.name = disk_name
DiskController._update_disk(disk, configuration[disk_name])
partition_info = configuration[disk_name]['partitions']
for partition_id in partition_info:
if partition_info[partition_id]['state'] not in [
'MISSING'
]:
DiskController._create_partition(
partition_id, partition_info[partition_id],
disk)
def build_service_structure(structure, previous_structure=None):
"""
Builds an MDS service structure
Example:
structure = Helper.build_service_structure(
{'vpools': [1],
'domains': [],
'storagerouters': [1],
'storagedrivers': [(1, 1, 1)], # (<id>, <vpool_id>, <storagerouter_id>)
'mds_services': [(1, 1)], # (<id>, <storagedriver_id>)
'storagerouter_domains': []} # (<id>, <storagerouter_id>, <domain_id>)
)
"""
if previous_structure is None:
previous_structure = {}
vdisks = previous_structure.get('vdisks', {})
vpools = previous_structure.get('vpools', {})
domains = previous_structure.get('domains', {})
services = previous_structure.get('services', {})
mds_services = previous_structure.get('mds_services', {})
storagerouters = previous_structure.get('storagerouters', {})
storagedrivers = previous_structure.get('storagedrivers', {})
storagerouter_domains = previous_structure.get('storagerouter_domains',
{})
service_type = ServiceTypeList.get_by_name('MetadataServer')
if service_type is None:
service_type = ServiceType()
service_type.name = 'MetadataServer'
service_type.save()
srclients = {}
for domain_id in structure.get('domains', []):
if domain_id not in domains:
domain = Domain()
domain.name = 'domain_{0}'.format(domain_id)
domain.save()
domains[domain_id] = domain
for vpool_id in structure.get('vpools', []):
if vpool_id not in vpools:
vpool = VPool()
vpool.name = str(vpool_id)
vpool.status = 'RUNNING'
vpool.save()
vpools[vpool_id] = vpool
else:
vpool = vpools[vpool_id]
srclients[vpool_id] = StorageRouterClient(vpool.guid, None)
for sr_id in structure.get('storagerouters', []):
if sr_id not in storagerouters:
storagerouter = StorageRouter()
storagerouter.name = str(sr_id)
storagerouter.ip = '10.0.0.{0}'.format(sr_id)
storagerouter.rdma_capable = False
storagerouter.node_type = 'MASTER'
storagerouter.machine_id = str(sr_id)
storagerouter.save()
storagerouters[sr_id] = storagerouter
disk = Disk()
disk.storagerouter = storagerouter
disk.state = 'OK'
disk.name = '/dev/uda'
disk.size = 1 * 1024**4
disk.is_ssd = True
disk.aliases = ['/dev/uda']
disk.save()
partition = DiskPartition()
partition.offset = 0
partition.size = disk.size
partition.aliases = ['/dev/uda-1']
partition.state = 'OK'
partition.mountpoint = '/tmp/unittest/sr_{0}/disk_1/partition_1'.format(
sr_id)
partition.disk = disk
partition.roles = [
DiskPartition.ROLES.DB, DiskPartition.ROLES.SCRUB
]
partition.save()
for sd_id, vpool_id, sr_id in structure.get('storagedrivers', ()):
if sd_id not in storagedrivers:
storagedriver = StorageDriver()
storagedriver.vpool = vpools[vpool_id]
storagedriver.storagerouter = storagerouters[sr_id]
storagedriver.name = str(sd_id)
storagedriver.mountpoint = '/'
storagedriver.cluster_ip = storagerouters[sr_id].ip
storagedriver.storage_ip = '10.0.1.{0}'.format(sr_id)
storagedriver.storagedriver_id = str(sd_id)
storagedriver.ports = {
'management': 1,
'xmlrpc': 2,
'dtl': 3,
'edge': 4
}
storagedriver.save()
storagedrivers[sd_id] = storagedriver
Helper._set_vpool_storage_driver_configuration(
vpool=vpools[vpool_id], storagedriver=storagedriver)
for mds_id, sd_id in structure.get('mds_services', ()):
if mds_id not in mds_services:
sd = storagedrivers[sd_id]
s_id = '{0}-{1}'.format(sd.storagerouter.name, mds_id)
service = Service()
service.name = s_id
service.storagerouter = sd.storagerouter
service.ports = [mds_id]
service.type = service_type
service.save()
services[s_id] = service
mds_service = MDSService()
mds_service.service = service
mds_service.number = 0
mds_service.capacity = 10
mds_service.vpool = sd.vpool
mds_service.save()
mds_services[mds_id] = mds_service
StorageDriverController.add_storagedriverpartition(
sd, {
'size': None,
'role': DiskPartition.ROLES.DB,
'sub_role': StorageDriverPartition.SUBROLE.MDS,
'partition': sd.storagerouter.disks[0].partitions[0],
'mds_service': mds_service
})
for vdisk_id, storage_driver_id, vpool_id, mds_id in structure.get(
'vdisks', ()):
if vdisk_id not in vdisks:
vpool = vpools[vpool_id]
devicename = 'vdisk_{0}'.format(vdisk_id)
mds_backend_config = Helper._generate_mdsmetadatabackendconfig(
[] if mds_id is None else [mds_services[mds_id]])
volume_id = srclients[vpool_id].create_volume(
devicename, mds_backend_config, 0, str(storage_driver_id))
vdisk = VDisk()
vdisk.name = str(vdisk_id)
vdisk.devicename = devicename
vdisk.volume_id = volume_id
vdisk.vpool = vpool
vdisk.size = 0
vdisk.save()
vdisk.reload_client('storagedriver')
vdisks[vdisk_id] = vdisk
for srd_id, sr_id, domain_id, backup in structure.get(
'storagerouter_domains', ()):
if srd_id not in storagerouter_domains:
sr_domain = StorageRouterDomain()
sr_domain.backup = backup
sr_domain.domain = domains[domain_id]
sr_domain.storagerouter = storagerouters[sr_id]
sr_domain.save()
storagerouter_domains[srd_id] = sr_domain
return {
'vdisks': vdisks,
'vpools': vpools,
'domains': domains,
'services': services,
'service_type': service_type,
'mds_services': mds_services,
'storagerouters': storagerouters,
'storagedrivers': storagedrivers,
'storagerouter_domains': storagerouter_domains
}
def build_service_structure(structure, previous_structure=None):
"""
Builds an MDS service structure
Example:
structure = Helper.build_service_structure(
{'vpools': [1],
'domains': [],
'storagerouters': [1],
'storagedrivers': [(1, 1, 1)], # (<id>, <vpool_id>, <storagerouter_id>)
'mds_services': [(1, 1)], # (<id>, <storagedriver_id>)
'storagerouter_domains': []} # (<id>, <storagerouter_id>, <domain_id>)
)
"""
if previous_structure is None:
previous_structure = {}
vdisks = previous_structure.get("vdisks", {})
vpools = previous_structure.get("vpools", {})
domains = previous_structure.get("domains", {})
services = previous_structure.get("services", {})
mds_services = previous_structure.get("mds_services", {})
storagerouters = previous_structure.get("storagerouters", {})
storagedrivers = previous_structure.get("storagedrivers", {})
storagerouter_domains = previous_structure.get("storagerouter_domains", {})
service_type = ServiceTypeList.get_by_name("MetadataServer")
if service_type is None:
service_type = ServiceType()
service_type.name = "MetadataServer"
service_type.save()
srclients = {}
for domain_id in structure.get("domains", []):
if domain_id not in domains:
domain = Domain()
domain.name = "domain_{0}".format(domain_id)
domain.save()
domains[domain_id] = domain
for vpool_id in structure.get("vpools", []):
if vpool_id not in vpools:
vpool = VPool()
vpool.name = str(vpool_id)
vpool.status = "RUNNING"
vpool.save()
vpools[vpool_id] = vpool
else:
vpool = vpools[vpool_id]
srclients[vpool_id] = StorageRouterClient(vpool.guid, None)
for sr_id in structure.get("storagerouters", []):
if sr_id not in storagerouters:
storagerouter = StorageRouter()
storagerouter.name = str(sr_id)
storagerouter.ip = "10.0.0.{0}".format(sr_id)
storagerouter.rdma_capable = False
storagerouter.node_type = "MASTER"
storagerouter.machine_id = str(sr_id)
storagerouter.save()
storagerouters[sr_id] = storagerouter
disk = Disk()
disk.storagerouter = storagerouter
disk.state = "OK"
disk.name = "/dev/uda"
disk.size = 1 * 1024 ** 4
disk.is_ssd = True
disk.aliases = ["/dev/uda"]
disk.save()
partition = DiskPartition()
partition.offset = 0
partition.size = disk.size
partition.aliases = ["/dev/uda-1"]
partition.state = "OK"
partition.mountpoint = "/tmp/unittest/sr_{0}/disk_1/partition_1".format(sr_id)
partition.disk = disk
partition.roles = [DiskPartition.ROLES.DB, DiskPartition.ROLES.SCRUB]
partition.save()
for sd_id, vpool_id, sr_id in structure.get("storagedrivers", ()):
if sd_id not in storagedrivers:
storagedriver = StorageDriver()
storagedriver.vpool = vpools[vpool_id]
storagedriver.storagerouter = storagerouters[sr_id]
storagedriver.name = str(sd_id)
storagedriver.mountpoint = "/"
storagedriver.cluster_ip = storagerouters[sr_id].ip
storagedriver.storage_ip = "10.0.1.{0}".format(sr_id)
storagedriver.storagedriver_id = str(sd_id)
storagedriver.ports = {"management": 1, "xmlrpc": 2, "dtl": 3, "edge": 4}
storagedriver.save()
storagedrivers[sd_id] = storagedriver
Helper._set_vpool_storage_driver_configuration(vpool=vpools[vpool_id], storagedriver=storagedriver)
for mds_id, sd_id in structure.get("mds_services", ()):
if mds_id not in mds_services:
sd = storagedrivers[sd_id]
s_id = "{0}-{1}".format(sd.storagerouter.name, mds_id)
service = Service()
service.name = s_id
service.storagerouter = sd.storagerouter
service.ports = [mds_id]
service.type = service_type
service.save()
services[s_id] = service
mds_service = MDSService()
mds_service.service = service
mds_service.number = 0
mds_service.capacity = 10
mds_service.vpool = sd.vpool
mds_service.save()
mds_services[mds_id] = mds_service
StorageDriverController.add_storagedriverpartition(
sd,
{
"size": None,
"role": DiskPartition.ROLES.DB,
"sub_role": StorageDriverPartition.SUBROLE.MDS,
"partition": sd.storagerouter.disks[0].partitions[0],
"mds_service": mds_service,
},
)
for vdisk_id, storage_driver_id, vpool_id, mds_id in structure.get("vdisks", ()):
if vdisk_id not in vdisks:
vpool = vpools[vpool_id]
devicename = "vdisk_{0}".format(vdisk_id)
mds_backend_config = Helper._generate_mdsmetadatabackendconfig(
[] if mds_id is None else [mds_services[mds_id]]
)
volume_id = srclients[vpool_id].create_volume(devicename, mds_backend_config, 0, str(storage_driver_id))
vdisk = VDisk()
vdisk.name = str(vdisk_id)
vdisk.devicename = devicename
vdisk.volume_id = volume_id
vdisk.vpool = vpool
vdisk.size = 0
vdisk.save()
vdisk.reload_client("storagedriver")
vdisks[vdisk_id] = vdisk
for srd_id, sr_id, domain_id, backup in structure.get("storagerouter_domains", ()):
if srd_id not in storagerouter_domains:
sr_domain = StorageRouterDomain()
sr_domain.backup = backup
sr_domain.domain = domains[domain_id]
sr_domain.storagerouter = storagerouters[sr_id]
sr_domain.save()
storagerouter_domains[srd_id] = sr_domain
return {
"vdisks": vdisks,
"vpools": vpools,
"domains": domains,
"services": services,
"service_type": service_type,
"mds_services": mds_services,
"storagerouters": storagerouters,
"storagedrivers": storagedrivers,
"storagerouter_domains": storagerouter_domains,
}
def test_clone_snapshot(self):
"""
Validates that a snapshot that has clones will not be deleted while other snapshots will be deleted
"""
# Setup
# There are 2 disks, second one cloned from a snapshot of the first
vpool = VPool()
vpool.name = 'vpool'
vpool.status = 'RUNNING'
vpool.save()
storage_router = StorageRouter()
storage_router.name = 'storage_router'
storage_router.ip = '127.0.0.1'
storage_router.machine_id = System.get_my_machine_id()
storage_router.rdma_capable = False
storage_router.save()
disk = Disk()
disk.name = 'physical_disk_1'
disk.aliases = ['/dev/non-existent']
disk.size = 500 * 1024**3
disk.state = 'OK'
disk.is_ssd = True
disk.storagerouter = storage_router
disk.save()
disk_partition = DiskPartition()
disk_partition.disk = disk
disk_partition.aliases = ['/dev/disk/non-existent']
disk_partition.size = 400 * 1024**3
disk_partition.state = 'OK'
disk_partition.offset = 1024
disk_partition.roles = [DiskPartition.ROLES.SCRUB]
disk_partition.mountpoint = '/var/tmp'
disk_partition.save()
storage_driver = StorageDriver()
storage_driver.vpool = vpool
storage_driver.storagerouter = storage_router
storage_driver.name = 'storage_driver_1'
storage_driver.mountpoint = '/'
storage_driver.cluster_ip = storage_router.ip
storage_driver.storage_ip = '127.0.0.1'
storage_driver.storagedriver_id = 'storage_driver_1'
storage_driver.ports = {
'management': 1,
'xmlrpc': 2,
'dtl': 3,
'edge': 4
}
storage_driver.save()
service_type = ServiceType()
service_type.name = 'MetadataServer'
service_type.save()
service = Service()
service.name = 'service_1'
service.storagerouter = storage_driver.storagerouter
service.ports = [1]
service.type = service_type
service.save()
mds_service = MDSService()
mds_service.service = service
mds_service.number = 0
mds_service.capacity = 10
mds_service.vpool = storage_driver.vpool
mds_service.save()
vdisk_1_1 = VDisk()
vdisk_1_1.name = 'vdisk_1_1'
vdisk_1_1.volume_id = 'vdisk_1_1'
vdisk_1_1.vpool = vpool
vdisk_1_1.devicename = 'dummy'
vdisk_1_1.size = 0
vdisk_1_1.save()
vdisk_1_1.reload_client('storagedriver')
[
dynamic for dynamic in vdisk_1_1._dynamics
if dynamic.name == 'snapshots'
][0].timeout = 0
travis = 'TRAVIS' in os.environ and os.environ['TRAVIS'] == 'true'
if travis is True:
print 'Running in Travis, reducing output.'
base = datetime.datetime.now().date()
day = datetime.timedelta(1)
base_timestamp = self._make_timestamp(base, day)
minute = 60
hour = minute * 60
for h in [6, 12, 18]:
timestamp = base_timestamp + (hour * h)
VDiskController.create_snapshot(vdisk_guid=vdisk_1_1.guid,
metadata={
'label':
'snapshot_{0}:30'.format(
str(h)),
'is_consistent':
True,
'timestamp':
str(timestamp),
'machineguid':
None
})
base_snapshot_guid = vdisk_1_1.snapshots[0]['guid'] # Oldest
clone_vdisk = VDisk()
clone_vdisk.name = 'clone_vdisk'
clone_vdisk.volume_id = 'clone_vdisk'
clone_vdisk.vpool = vpool
clone_vdisk.devicename = 'dummy'
clone_vdisk.parentsnapshot = base_snapshot_guid
clone_vdisk.size = 0
clone_vdisk.save()
clone_vdisk.reload_client('storagedriver')
for h in [6, 12, 18]:
timestamp = base_timestamp + (hour * h)
VDiskController.create_snapshot(vdisk_guid=clone_vdisk.guid,
metadata={
'label':
'snapshot_{0}:30'.format(
str(h)),
'is_consistent':
True,
'timestamp':
str(timestamp),
'machineguid':
None
})
base_timestamp = self._make_timestamp(base, day * 2)
ScheduledTaskController.delete_snapshots(timestamp=base_timestamp +
(minute * 30))
self.assertIn(
base_snapshot_guid, [snap['guid'] for snap in vdisk_1_1.snapshots],
'Snapshot was deleted while there are still clones of it')
def sync_with_reality(storagerouter_guid=None):
"""
Syncs the Disks from all StorageRouters with the reality.
"""
storagerouters = []
if storagerouter_guid is not None:
storagerouters.append(StorageRouter(storagerouter_guid))
else:
storagerouters = StorageRouterList.get_storagerouters()
for storagerouter in storagerouters:
try:
client = SSHClient(storagerouter, username='******')
except UnableToConnectException:
logger.info('Could not connect to StorageRouter {0}, skipping'.format(storagerouter.ip))
continue
configuration = {}
# Gather mount data
mount_mapping = {}
mount_data = client.run('mount')
for mount in mount_data.splitlines():
mount = mount.strip()
match = re.search('/dev/(.+?) on (/.*?) type.*', mount)
if match is not None:
mount_mapping[match.groups()[0]] = match.groups()[1]
# Gather disk information
with Remote(storagerouter.ip, [Context, os]) as remote:
context = remote.Context()
devices = [device for device in context.list_devices(subsystem='block')
if 'ID_TYPE' in device and device['ID_TYPE'] == 'disk']
for device in devices:
is_partition = device['DEVTYPE'] == 'partition'
device_path = device['DEVNAME']
device_name = device_path.split('/')[-1]
partition_id = None
partition_name = None
extended_parition_info = None
if is_partition is True:
if 'ID_PART_ENTRY_NUMBER' in device:
extended_parition_info = True
partition_id = device['ID_PART_ENTRY_NUMBER']
partition_name = device_name
device_name = device_name[:0 - int(len(partition_id))]
else:
logger.debug('Partition {0} has no partition metadata'.format(device_path))
extended_parition_info = False
match = re.match('^(\D+?)(\d+)$', device_name)
if match is None:
logger.debug('Could not handle disk/partition {0}'.format(device_path))
continue # Unable to handle this disk/partition
partition_name = device_name
partition_id = match.groups()[1]
device_name = match.groups()[0]
if device_name not in configuration:
configuration[device_name] = {'partitions': {}}
path = None
for path_type in ['by-id', 'by-uuid']:
if path is not None:
break
for item in device['DEVLINKS'].split(' '):
if path_type in item:
path = item
if path is None:
path = device_path
sectors = int(client.run('cat /sys/block/{0}/size'.format(device_name)))
sector_size = int(client.run('cat /sys/block/{0}/queue/hw_sector_size'.format(device_name)))
rotational = int(client.run('cat /sys/block/{0}/queue/rotational'.format(device_name)))
if is_partition is True:
if 'ID_PART_ENTRY_TYPE' in device and device['ID_PART_ENTRY_TYPE'] == '0x5':
continue # This is an extended partition, let's skip that one
if extended_parition_info is True:
offset = int(device['ID_PART_ENTRY_OFFSET']) * sector_size
size = int(device['ID_PART_ENTRY_SIZE']) * sector_size
else:
match = re.match('^(\D+?)(\d+)$', device_path)
if match is None:
logger.debug('Could not handle disk/partition {0}'.format(device_path))
continue # Unable to handle this disk/partition
fdisk_info = client.run('fdisk -l {0} | grep {1}'.format(match.groups()[0], device_path)).strip()
fdisk_data = filter(None, fdisk_info.split(' '))
offset = int(fdisk_data[1]) * sector_size
size = (int(fdisk_data[2]) - int(fdisk_data[1])) * sector_size
configuration[device_name]['partitions'][partition_id] = {'offset': offset,
'size': size,
'path': path,
'state': 'OK'}
partition_data = configuration[device_name]['partitions'][partition_id]
if partition_name in mount_mapping:
mountpoint = mount_mapping[partition_name]
partition_data['mountpoint'] = mountpoint
partition_data['inode'] = remote.os.stat(mountpoint).st_dev
del mount_mapping[partition_name]
try:
client.run('touch {0}/{1}; rm {0}/{1}'.format(mountpoint, str(time.time())))
except CalledProcessError:
partition_data['state'] = 'FAILURE'
pass
if 'ID_FS_TYPE' in device:
partition_data['filesystem'] = device['ID_FS_TYPE']
else:
configuration[device_name].update({'name': device_name,
'path': path,
'vendor': device['ID_VENDOR'] if 'ID_VENDOR' in device else None,
'model': device['ID_MODEL'] if 'ID_MODEL' in device else None,
'size': sector_size * sectors,
'is_ssd': rotational == 0,
'state': 'OK'})
for partition_name in mount_mapping:
device_name = partition_name.split('/')[-1]
match = re.search('^(\D+?)(\d+)$', device_name)
if match is not None:
device_name = match.groups()[0]
partition_id = match.groups()[1]
if device_name not in configuration:
configuration[device_name] = {'partitions': {},
'state': 'MISSING'}
configuration[device_name]['partitions'][partition_id] = {'mountpoint': mount_mapping[partition_name],
'state': 'MISSING'}
# Sync the model
disk_names = []
for disk in storagerouter.disks:
if disk.name not in configuration:
for partition in disk.partitions:
partition.delete()
disk.delete()
else:
disk_names.append(disk.name)
DiskController._update_disk(disk, configuration[disk.name])
partitions = []
partition_info = configuration[disk.name]['partitions']
for partition in disk.partitions:
if partition.id not in partition_info:
partition.delete()
else:
partitions.append(partition.id)
DiskController._update_partition(partition, partition_info[partition.id])
for partition_id in partition_info:
if partition_id not in partitions:
DiskController._create_partition(partition_id, partition_info[partition_id], disk)
for disk_name in configuration:
if disk_name not in disk_names and configuration[disk_name]['state'] not in ['MISSING']:
disk = Disk()
disk.storagerouter = storagerouter
disk.name = disk_name
DiskController._update_disk(disk, configuration[disk_name])
partition_info = configuration[disk_name]['partitions']
for partition_id in partition_info:
if partition_info[partition_id]['state'] not in ['MISSING']:
DiskController._create_partition(partition_id, partition_info[partition_id], disk)
def sync_with_reality(storagerouter_guid=None):
"""
Syncs the Disks from all StorageRouters with the reality.
:param storagerouter_guid: Guid of the Storage Router to synchronize
"""
storagerouters = []
if storagerouter_guid is not None:
storagerouters.append(StorageRouter(storagerouter_guid))
else:
storagerouters = StorageRouterList.get_storagerouters()
for storagerouter in storagerouters:
try:
client = SSHClient(storagerouter, username='******')
except UnableToConnectException:
DiskController._logger.info('Could not connect to StorageRouter {0}, skipping'.format(storagerouter.ip))
continue
configuration = {}
# Gather mount data
mount_mapping = {}
mount_data = client.run('mount')
for mount in mount_data.splitlines():
mount = mount.strip()
match = re.search('(/dev/(.+?)) on (/.*?) type.*', mount)
if match is not None:
dev_name = match.groups()[0]
uuid = client.run('blkid -o value -s UUID {0}'.format(dev_name))
if uuid:
mount_mapping[uuid] = match.groups()[2]
else:
mount_mapping[match.groups()[1]] = match.groups()[2]
# Gather raid information
try:
md_information = client.run('mdadm --detail /dev/md*', suppress_logging=True)
except CalledProcessError:
md_information = ''
raid_members = []
for member in re.findall('(?: +[0-9]+){4} +[^/]+/dev/([a-z0-9]+)', md_information):
raid_members.append(member)
# Gather disk information
with remote(storagerouter.ip, [Context, os]) as rem:
context = rem.Context()
devices = [device for device in context.list_devices(subsystem='block')
if ('ID_TYPE' in device and device['ID_TYPE'] == 'disk') or
('DEVNAME' in device and ('loop' in device['DEVNAME'] or 'nvme' in device['DEVNAME'] or 'md' in device['DEVNAME']))]
for device in devices:
is_partition = device['DEVTYPE'] == 'partition'
device_path = device['DEVNAME']
device_name = device_path.split('/')[-1]
partition_id = None
partition_name = None
extended_partition_info = None
if is_partition is True:
partition_name = device['ID_FS_UUID'] if 'ID_FS_UUID' in device else device_name
if 'ID_PART_ENTRY_NUMBER' in device:
extended_partition_info = True
partition_id = device['ID_PART_ENTRY_NUMBER']
if device_name.startswith('nvme') or device_name.startswith('loop'):
device_name = device_name[:0 - int(len(partition_id)) - 1]
elif device_name.startswith('md'):
device_name = device_name[:device_name.index('p')]
else:
device_name = device_name[:0 - int(len(partition_id))]
else:
DiskController._logger.debug('Partition {0} has no partition metadata'.format(device_path))
extended_partition_info = False
match = re.match('^(\D+?)(\d+)$', device_name)
if match is None:
DiskController._logger.debug('Could not handle disk/partition {0}'.format(device_path))
continue # Unable to handle this disk/partition
partition_id = match.groups()[1]
device_name = match.groups()[0]
sectors = int(client.run('cat /sys/block/{0}/size'.format(device_name)))
sector_size = int(client.run('cat /sys/block/{0}/queue/hw_sector_size'.format(device_name)))
rotational = int(client.run('cat /sys/block/{0}/queue/rotational'.format(device_name)))
if sectors == 0:
continue
if device_name in raid_members:
continue
if device_name not in configuration:
configuration[device_name] = {'partitions': {}}
path = None
for path_type in ['by-id', 'by-uuid']:
if path is not None:
break
if 'DEVLINKS' in device:
for item in device['DEVLINKS'].split(' '):
if path_type in item:
path = item
if path is None:
path = device_path
if is_partition is True:
if 'ID_PART_ENTRY_TYPE' in device and device['ID_PART_ENTRY_TYPE'] == '0x5':
continue # This is an extended partition, let's skip that one
if extended_partition_info is True:
offset = int(device['ID_PART_ENTRY_OFFSET']) * sector_size
size = int(device['ID_PART_ENTRY_SIZE']) * sector_size
else:
match = re.match('^(\D+?)(\d+)$', device_path)
if match is None:
DiskController._logger.debug('Could not handle disk/partition {0}'.format(device_path))
continue # Unable to handle this disk/partition
partitions_info = DiskTools.get_partitions_info(match.groups()[0])
if device_path in partitions_info:
partition_info = partitions_info[device_path]
offset = int(partition_info['start'])
size = int(partition_info['size'])
else:
DiskController._logger.warning('Could not retrieve partition info for disk/partition {0}'.format(device_path))
continue
configuration[device_name]['partitions'][partition_id] = {'offset': offset,
'size': size,
'path': path,
'state': 'OK'}
partition_data = configuration[device_name]['partitions'][partition_id]
if partition_name in mount_mapping:
mountpoint = mount_mapping[partition_name]
partition_data['mountpoint'] = mountpoint
partition_data['inode'] = rem.os.stat(mountpoint).st_dev
del mount_mapping[partition_name]
try:
client.run('touch {0}/{1}; rm {0}/{1}'.format(mountpoint, str(time.time())))
except CalledProcessError:
partition_data['state'] = 'FAILURE'
pass
if 'ID_FS_TYPE' in device:
partition_data['filesystem'] = device['ID_FS_TYPE']
else:
configuration[device_name].update({'name': device_name,
'path': path,
'vendor': device['ID_VENDOR'] if 'ID_VENDOR' in device else None,
'model': device['ID_MODEL'] if 'ID_MODEL' in device else None,
'size': sector_size * sectors,
'is_ssd': rotational == 0,
'state': 'OK'})
for partition_name in mount_mapping:
device_name = partition_name.split('/')[-1]
match = re.search('^(\D+?)(\d+)$', device_name)
if match is not None:
device_name = match.groups()[0]
partition_id = match.groups()[1]
if device_name not in configuration:
configuration[device_name] = {'partitions': {},
'state': 'MISSING'}
configuration[device_name]['partitions'][partition_id] = {'mountpoint': mount_mapping[partition_name],
'state': 'MISSING'}
# Sync the model
disk_names = []
for disk in storagerouter.disks:
if disk.name not in configuration:
for partition in disk.partitions:
partition.delete()
disk.delete()
else:
disk_names.append(disk.name)
DiskController._update_disk(disk, configuration[disk.name])
partitions = []
partition_info = configuration[disk.name]['partitions']
for partition in disk.partitions:
if partition.id not in partition_info:
partition.delete()
else:
partitions.append(partition.id)
DiskController._update_partition(partition, partition_info[partition.id])
for partition_id in partition_info:
if partition_id not in partitions:
DiskController._create_partition(partition_id, partition_info[partition_id], disk)
for disk_name in configuration:
if disk_name not in disk_names and configuration[disk_name]['state'] not in ['MISSING']:
disk = Disk()
disk.storagerouter = storagerouter
disk.name = disk_name
DiskController._update_disk(disk, configuration[disk_name])
partition_info = configuration[disk_name]['partitions']
for partition_id in partition_info:
if partition_info[partition_id]['state'] not in ['MISSING']:
DiskController._create_partition(partition_id, partition_info[partition_id], disk)
def build_dal_structure(structure, previous_structure=None):
"""
Builds a model structure
Example:
structure = DalHelper.build_service_structure(
{'vpools': [1],
'domains': [],
'storagerouters': [1],
'storagedrivers': [(1, 1, 1)], # (<id>, <vpool_id>, <storagerouter_id>)
'mds_services': [(1, 1)], # (<id>, <storagedriver_id>)
'storagerouter_domains': []} # (<id>, <storagerouter_id>, <domain_id>)
)
"""
Configuration.set(key=Configuration.EDITION_KEY,
value=PackageFactory.EDITION_ENTERPRISE)
if previous_structure is None:
previous_structure = {}
vdisks = previous_structure.get('vdisks', {})
vpools = previous_structure.get('vpools', {})
domains = previous_structure.get('domains', {})
services = previous_structure.get('services', {})
mds_services = previous_structure.get('mds_services', {})
storagerouters = previous_structure.get('storagerouters', {})
storagedrivers = previous_structure.get('storagedrivers', {})
storagerouter_domains = previous_structure.get('storagerouter_domains',
{})
service_types = {}
for service_type_name in ServiceType.SERVICE_TYPES.values():
service_type = ServiceTypeList.get_by_name(service_type_name)
if service_type is None:
service_type = ServiceType()
service_type.name = service_type_name
service_type.save()
service_types[service_type_name] = service_type
srclients = {}
for domain_id in structure.get('domains', []):
if domain_id not in domains:
domain = Domain()
domain.name = 'domain_{0}'.format(domain_id)
domain.save()
domains[domain_id] = domain
for vpool_id in structure.get('vpools', []):
if vpool_id not in vpools:
vpool = VPool()
vpool.name = str(vpool_id)
vpool.status = 'RUNNING'
vpool.metadata = {'backend': {}, 'caching_info': {}}
vpool.metadata_store_bits = 5
vpool.save()
vpools[vpool_id] = vpool
else:
vpool = vpools[vpool_id]
srclients[vpool_id] = StorageRouterClient(vpool.guid, None)
Configuration.set(
'/ovs/vpools/{0}/mds_config|mds_tlogs'.format(vpool.guid), 100)
Configuration.set(
'/ovs/vpools/{0}/mds_config|mds_safety'.format(vpool.guid), 2)
Configuration.set(
'/ovs/vpools/{0}/mds_config|mds_maxload'.format(vpool.guid),
75)
Configuration.set(
'/ovs/vpools/{0}/proxies/scrub/generic_scrub'.format(
vpool.guid),
json.dumps({}, indent=4),
raw=True)
for sr_id in structure.get('storagerouters', []):
if sr_id not in storagerouters:
storagerouter = StorageRouter()
storagerouter.name = str(sr_id)
storagerouter.ip = '10.0.0.{0}'.format(sr_id)
storagerouter.rdma_capable = False
storagerouter.node_type = 'MASTER'
storagerouter.machine_id = str(sr_id)
storagerouter.save()
storagerouters[sr_id] = storagerouter
disk = Disk()
disk.storagerouter = storagerouter
disk.state = 'OK'
disk.name = '/dev/uda'
disk.size = 1 * 1024**4
disk.is_ssd = True
disk.aliases = ['/dev/uda']
disk.save()
partition = DiskPartition()
partition.offset = 0
partition.size = disk.size
partition.aliases = ['/dev/uda-1']
partition.state = 'OK'
partition.mountpoint = '/tmp/unittest/sr_{0}/disk_1/partition_1'.format(
sr_id)
partition.disk = disk
partition.roles = [
DiskPartition.ROLES.DB, DiskPartition.ROLES.SCRUB
]
partition.save()
else:
storagerouter = storagerouters[sr_id]
# noinspection PyProtectedMember
System._machine_id[storagerouter.ip] = str(sr_id)
mds_start = 10000 + 100 * (sr_id - 1)
mds_end = 10000 + 100 * sr_id - 1
arakoon_start = 20000 + 100 * (sr_id - 1)
storagedriver_start = 30000 + 100 * (sr_id - 1)
storagedriver_end = 30000 + 100 * sr_id - 1
Configuration.initialize_host(
host_id=sr_id,
port_info={
'mds': [mds_start, mds_end],
'arakoon': arakoon_start,
'storagedriver': [storagedriver_start, storagedriver_end]
})
for sd_id, vpool_id, sr_id in structure.get('storagedrivers', ()):
if sd_id not in storagedrivers:
storagedriver = StorageDriver()
storagedriver.vpool = vpools[vpool_id]
storagedriver.storagerouter = storagerouters[sr_id]
storagedriver.name = str(sd_id)
storagedriver.mountpoint = '/'
storagedriver.cluster_ip = storagerouters[sr_id].ip
storagedriver.storage_ip = '10.0.1.{0}'.format(sr_id)
storagedriver.storagedriver_id = str(sd_id)
storagedriver.ports = {
'management': 1,
'xmlrpc': 2,
'dtl': 3,
'edge': 4
}
storagedriver.save()
storagedrivers[sd_id] = storagedriver
DalHelper.set_vpool_storage_driver_configuration(
vpool=vpools[vpool_id], storagedriver=storagedriver)
for mds_id, sd_id in structure.get('mds_services', ()):
if mds_id not in mds_services:
sd = storagedrivers[sd_id]
s_id = '{0}-{1}'.format(sd.storagerouter.name, mds_id)
service = Service()
service.name = s_id
service.storagerouter = sd.storagerouter
service.ports = [mds_id]
service.type = service_types['MetadataServer']
service.save()
services[s_id] = service
mds_service = MDSService()
mds_service.service = service
mds_service.number = 0
mds_service.capacity = 10
mds_service.vpool = sd.vpool
mds_service.save()
mds_services[mds_id] = mds_service
StorageDriverController.add_storagedriverpartition(
sd, {
'size': None,
'role': DiskPartition.ROLES.DB,
'sub_role': StorageDriverPartition.SUBROLE.MDS,
'partition': sd.storagerouter.disks[0].partitions[0],
'mds_service': mds_service
})
for vdisk_id, storage_driver_id, vpool_id, mds_id in structure.get(
'vdisks', ()):
if vdisk_id not in vdisks:
vpool = vpools[vpool_id]
devicename = 'vdisk_{0}'.format(vdisk_id)
mds_backend_config = DalHelper.generate_mds_metadata_backend_config(
[] if mds_id is None else [mds_services[mds_id]])
volume_id = srclients[vpool_id].create_volume(
devicename, mds_backend_config, 0, str(storage_driver_id))
vdisk = VDisk()
vdisk.name = str(vdisk_id)
vdisk.devicename = devicename
vdisk.volume_id = volume_id
vdisk.vpool = vpool
vdisk.size = 0
vdisk.save()
vdisk.reload_client('storagedriver')
vdisks[vdisk_id] = vdisk
for srd_id, sr_id, domain_id, backup in structure.get(
'storagerouter_domains', ()):
if srd_id not in storagerouter_domains:
sr_domain = StorageRouterDomain()
sr_domain.backup = backup
sr_domain.domain = domains[domain_id]
sr_domain.storagerouter = storagerouters[sr_id]
sr_domain.save()
storagerouter_domains[srd_id] = sr_domain
return {
'vdisks': vdisks,
'vpools': vpools,
'domains': domains,
'services': services,
'mds_services': mds_services,
'service_types': service_types,
'storagerouters': storagerouters,
'storagedrivers': storagedrivers,
'storagerouter_domains': storagerouter_domains
}