def snapshot(self, volume, destination, name, snap_size=None):
# python 3 prints sometimes too many decimals in case of numbers
# like 123.00000001, so cut it down to one, like python 2 did
if snap_size:
snap_size = "%.1f" % snap_size
misc.run([self.f, self.v, self.y, 'vol snapshot', volume, destination,
name, str(snap_size)])
def _create_filesystem(self, pool, name, devs, size=None, raid=None):
if not devs:
raise Exception("To create btrfs volume, some devices must be " +
"provided")
self._binary = misc.check_binary('mkfs.btrfs')
if not self._binary:
self.problem.check(self.problem.TOOL_MISSING, 'mkfs.btrfs')
command = ['mkfs.btrfs', '-L', name]
if raid:
if raid['level'] == '0':
command.extend(['-m', 'raid0', '-d', 'raid0'])
elif raid['level'] == '1':
command.extend(['-m', 'raid1', '-d', 'raid1'])
elif raid['level'] == '10':
command.extend(['-m', 'raid10', '-d', 'raid10'])
else:
raise Exception("Btrfs backed currently does not support " +
"RAID level {0}".format(raid['level']))
if size:
command.extend(['-b', "{0}".format(int(float(size) * 1024))])
# This might seem weird, but btrfs is mostly broken when it comes to
# checking existing signatures because it will for example check for
# backup superblocks as well, which is wrong. Also we have check for
# existing file system signatures in the ssm itself. Other things
# than file system should be covered by the backend and we should
# have tried to remove the device from the respective pool already.
# So at this point there should not be any useful signatures to
# speak of. However as I mentioned btrfs is broken, so force it.
command.extend(['-f'])
command.extend(devs)
misc.run(command, stdout=True)
misc.send_udev_event(devs[0], "change")
return name
def run_cryptsetup(self, command, stdout=True, password=None):
if not misc.check_binary('cryptsetup'):
self.problem.check(self.problem.TOOL_MISSING, 'cryptsetup')
command.insert(0, "cryptsetup")
if password != None:
return misc.run(command, stdout=stdout, stdin_data=password)
else:
return misc.run(command, stdout=stdout)
def run_lvm(self, command, noforce=False):
if not self.binary:
self.problem.check(self.problem.TOOL_MISSING, 'lvm')
if self.options.force and not noforce:
command.insert(1, "-f")
if self.options.verbose:
command.insert(1, "-v")
command.insert(0, "lvm")
misc.run(command, stdout=True)
def create_thin_volume(parent_pool, thin_pool, virtsize, lvname):
pool_volume = parent_pool + '/' + thin_pool
# Ignore options for non existing thin volumes somehow
command = [
'lvcreate', '-n', lvname, '-T', pool_volume, '-V',
str(virtsize) + 'K'
]
command.insert(0, "lvm")
misc.run(command, stdout=True)
return "{0}/{1}/{2}".format(DM_DEV_DIR, parent_pool, lvname)
def _list_subvolumes(self, mount, list_snapshots=False):
command = ['btrfs', 'subvolume', 'list']
if self.modified_list_version:
command.append('-a')
if list_snapshots:
command.append('-s')
ret, output = misc.run(command + [mount], stdout=False, can_fail=True)
if ret:
command = ['btrfs', 'subvolume', 'list']
if list_snapshots:
command.append('-s')
output = misc.run(command + [mount], stdout=False)[1]
self.modified_list_version = False
return output
def get_cryptsetup_version():
try:
output = misc.run(['cryptsetup', '--version'], can_fail=True)[1]
version = list(map(int, output.strip().split()[-1].split('.', 3)))
except (OSError, AttributeError):
version = [0, 0, 0]
return version
def create(self, pool, size='', name='', devs='',
options=None):
options = options or {}
if type(devs) is not list:
devices = [devs]
if 'raid' in options:
stripes = options['stripes']
stripesize = options['stripesize']
level = options['raid']
else:
stripes = stripesize = level = ""
misc.run([self.f, self.v, self.y, 'pool create', pool, size, name,
level, stripes, stripesize, " ".join(devs)])
if not name:
name = "lvol001"
return "/dev/{0}/{1}".format(pool, name)
def get_volume_data(self, volname):
data = {}
data['dev_name'] = self.get_real_device(volname)
data['hide'] = False
command = [MP, '-ll', volname]
try:
output = misc.run(command, stderr=False)[1].split("\n")
except OSError:
# probably multipath not installed
output = []
if len(output) > 0:
match = re.search(r"\(([^)]+)\)", output[0])
data['wwid'] = match.group(1)
data['dev_size'] = misc.get_device_size(data['dev_name'])
data['nodes'] = []
data['total_nodes'] = 0
for entry in zip(output[2::2], output[3::2]):
""" Some string operations to remove the tree path symbols
from the output. """
dev = list(
filter(
None, entry[1][re.search(r"[a-zA-Z0-9]", entry[1]).
start():].split(" ")))
data['nodes'].append("/dev/" + self.get_real_device(dev[1]))
data['total_nodes'] += 1
return data
def __init__(self, *args, **kwargs):
super(DmCryptVolume, self).__init__(*args, **kwargs)
command = ['dmsetup', 'table']
self.output = misc.run(command, stderr=False)[1]
for line in self.output.split("\n"):
if not line or line == "No devices found":
break
dm = {}
array = line.split()
if len(array) == 1:
continue
dm['type'] = array[3]
if dm['type'] != 'crypt':
continue
dm['vol_size'] = str(int(array[2]) / 2.0)
devname = re.sub(":$", "",
"{0}/mapper/{1}".format(DM_DEV_DIR, array[0]))
dm['dm_name'] = devname
dm['pool_name'] = self.default_pool_name
dm['dev_name'] = devname
dm['real_dev'] = misc.get_real_device(devname)
if dm['real_dev'] in self.mounts:
dm['mount'] = self.mounts[dm['real_dev']]['mp']
# Check if the device really exists in the system. In some cases
# (tests) DM_DEV_DIR can lie to us, if that is the case, simple
# ignore the device.
if not os.path.exists(devname):
continue
command = ['cryptsetup', 'status', devname]
self._parse_cryptsetup(command, dm)
self.data[dm['dev_name']] = dm
def _parse_data(self, command):
if not self.binary:
return
ret, self.output, err = misc.run(command, stderr=False, can_fail=True)
# A workaround for LVM behaviour:
# lvm lvs' exit code is 5 on exported volumes, even if everything
# is ok. So, if the code is 5, command was 'lvm lvs ...'
# and error message says that a volume was exported, ignore the
# error
if ret != 0:
err_msg = "ERROR exit code {0} for running command: \"{1}\"".format(
ret, " ".join(command))
if ret != 5 or command[0:2] != ['lvm', 'lvs'] or \
not str(err).endswith('is exported\n'):
if err is not None:
print(err)
raise problem.CommandFailed(err_msg, exitcode=ret)
for line in self.output.split("\n"):
if not line:
break
array = line.split("|")
row = dict([(self.attrs[index], array[index].lstrip())
for index in range(len(array))])
if self._skip_data(row):
continue
self._fill_additional_info(row)
self.data[self._data_index(row)] = row
def get_volume_data(self, devname):
data = {}
data['dev_name'] = devname
data['real_dev'] = devname
data['pool_name'] = SSM_DM_DEFAULT_POOL
if data['dev_name'] in self.mounts:
data['mount'] = self.mounts[data['dev_name']]['mp']
else:
for swap in self.swaps:
if swap[0] == data['dev_name']:
data['mount'] = "SWAP"
break
command = [MDADM, '--detail', devname]
for line in misc.run(command, stderr=False)[1].split("\n"):
array = line.split(":")
if len(array) < 2:
continue
item = array[0].strip()
value = array[1].strip()
if item == 'Raid Level':
data['type'] = value
elif item == 'Array Size':
data['vol_size'] = value.split()[0]
elif item == 'Total Devices':
data['total_devices'] = value
return data
def get_btrfs_version():
try:
output = misc.run(['btrfs', '--version'], can_fail=True)[1]
output = output.strip().split("\n")[-1]
version = re.search(r'(?<=v)\d+\.\d+', output).group(0)
except (OSError, AttributeError):
version = "0.0"
return float(version)
def _parse_cryptsetup(self, cmd, dm):
self.output = misc.run(cmd, stderr=False)[1]
for line in self.output.split("\n"):
if not line:
break
array = line.split()
if array[0].strip() == 'cipher:':
dm['cipher'] = array[1]
elif array[0].strip() == 'keysize:':
dm['keysize'] = array[1]
elif array[0].strip() == 'device:':
dm['crypt_device'] = array[1]
def _can_btrfs_force(self, command):
"""
This is just ridiculous. Unfortunately btrfs tools usually change
behaviour and options without bumping version number. So we have
to check whether btrfs allows to 'force' file system creation.
"""
output = misc.run(command + ['--force'], can_fail=True)[1]
found = re.search('invalid option', output)
if found:
return False
else:
return True
def run_bash_tests(names):
cur = os.getcwd()
os.chdir('./tests/bashtests')
command = ['ls', '-m']
if os.access('.coverage', os.R_OK):
os.remove('.coverage')
failed = []
passed = []
count = 0
misc.run('./set.sh', stdout=False)
output = misc.run(command, stdout=False)[1]
t0 = time.time()
for script in output.split(","):
script = script.strip()
if not re.match("^\d\d\d-.*\.sh$", script):
continue
if names and script not in names:
continue
count += 1
sys.stdout.write("{0:<29}".format(script) + " ")
sys.stdout.flush()
bad_file = re.sub("\.sh$",".bad", script)
if os.access(bad_file, os.R_OK):
os.remove(bad_file)
ret, out = misc.run(['./' + script], stdout=False, can_fail=True)
if ret:
print("\033[91m[FAILED]\033[0m")
failed.append(script)
with open(bad_file, 'w') as f:
f.write(out)
elif re.search("Traceback", out):
# There should be no tracebacks in the output
out += "\nWARNING: Traceback in the output!\n"
print("\033[93m[WARNING]\033[0m")
with open(bad_file, 'w') as f:
f.write(out)
else:
print("\033[92m[PASSED]\033[0m")
passed.append(script)
if count == 0 and names:
print("[+] No bash test matches the name(s)")
return 0
t1 = time.time() - t0
print("Ran {0} tests in {1} seconds.".format(count, round(t1, 2)))
print("{0} tests PASSED: {1}".format(len(passed), ", ".join(passed)))
ret = 0
if len(failed) > 0:
print("{0} tests FAILED: {1}".format(len(failed), ", ".join(failed)))
print("See files with \"bad\" extension for output")
ret = 1
# Show coverage report output if possible
if misc.check_binary('coverage'):
print("[+] Coverage")
misc.run(['coverage', 'report'], stdout=True, can_fail=True)
os.chdir(cur)
return ret
def _get_snap_name_list(self, mount):
snap = []
if BTRFS_VERSION < 0.20:
return snap
command = ['btrfs', 'subvolume', 'list', '-s', mount]
output = misc.run(command, stdout=False)[1]
for line in output.strip().split("\n"):
if not line:
continue
path = re.search('(?<=path ).*$', line).group(0)
snap.append(path)
return snap
def _parse_data(self, command):
if not self.binary:
return
self.output = misc.run(command, stderr=False)[1]
for line in self.output.split("\n"):
if not line:
break
array = line.split("|")
row = dict([(self.attrs[index], array[index].lstrip())
for index in range(len(array))])
if self._skip_data(row):
continue
self._fill_aditional_info(row)
self.data[self._data_index(row)] = row
def get_lvm_version():
try:
output = misc.run(['lvm', 'version'], can_fail=True)[1]
output = output.strip().split("\n")
pattern = re.compile("LVM version:")
version = [0, 0, 0]
for line in output:
if pattern.match(line.strip()):
match = " ".join(line.split())
tmp = re.search(r'(?<=LVM version: )\d+\.\d+\.\d+',
match).group(0)
version = list(map(int, tmp.split(".", 3)))
except (OSError, AttributeError):
version = [0, 0, 0]
return version
def get_device_data(self, devname, devsize):
data = {}
data['dev_name'] = devname
data['hide'] = False
command = [MDADM, '--examine', devname]
output = misc.run(command, stderr=False)[1].split("\n")
for line in output:
array = line.split(":")
if len(array) < 2:
continue
item = array[0].strip()
if item == "Name":
data['pool_name'] = SSM_DM_DEFAULT_POOL
data['dev_used'] = data['dev_size'] = devsize
data['dev_free'] = 0
return data
def get_mp_devices(self):
""" Find all multipath devices (but not their nodes). """
devices = []
command = [MP, '-ll']
try:
output = misc.run(command, stderr=False, can_fail=True)[1].split("\n")
pattern = re.compile(r"^([a-z0-9]+) \([^)]+\)")
except (problem.CommandFailed, OSError):
# probably multipath not installed
output = []
for line in output:
match = pattern.match(line)
if match:
devices.append(match.group(1))
return devices
def __init__(self, options, data=None):
self.type = 'crypt'
self.data = data or {}
self.output = None
self.options = options
self.mounts = misc.get_mounts('{0}/mapper'.format(DM_DEV_DIR))
self.default_pool_name = SSM_CRYPT_DEFAULT_POOL
self.problem = problem.ProblemSet(options)
if not misc.check_binary('dmsetup') or \
not misc.check_binary('cryptsetup'):
return
command = ['dmsetup', 'table']
self.output = misc.run(command, stderr=False)[1]
for line in self.output.split("\n"):
if not line or line == "No devices found":
break
dm = {}
array = line.split()
dm['type'] = array[3]
if dm['type'] != 'crypt':
continue
dm['vol_size'] = str(int(array[2]) / 2.0)
devname = re.sub(":$", "",
"{0}/mapper/{1}".format(DM_DEV_DIR, array[0]))
dm['dm_name'] = devname
dm['pool_name'] = 'dm-crypt'
dm['dev_name'] = misc.get_real_device(devname)
dm['real_dev'] = dm['dev_name']
if dm['real_dev'] in self.mounts:
dm['mount'] = self.mounts[dm['real_dev']]['mp']
# Check if the device really exists in the system. In some cases
# (tests) DM_DEV_DIR can lie to us, if that is the case, simple
# ignore the device.
if not os.path.exists(devname):
continue
command = ['cryptsetup', 'status', devname]
self._parse_cryptsetup(command, dm)
self.data[dm['dev_name']] = dm
def run_mdadm(self, command):
if not self._binary:
self.problem.check(self.problem.TOOL_MISSING, MDADM)
command.insert(0, MDADM)
return misc.run(command, stdout=True)
def __init__(self, *args, **kwargs):
super(Btrfs, self).__init__(*args, **kwargs)
self.type = 'btrfs'
self.default_pool_name = SSM_BTRFS_DEFAULT_POOL
self._vol = {}
self._pool = {}
self._dev = {}
self._snap = {}
self._subvolumes = {}
self._binary = misc.check_binary('btrfs')
self.modified_list_version = True
if not self._binary:
return
self.mounts = misc.get_mounts('btrfs')
command = ['btrfs', 'filesystem', 'show']
self.output = misc.run(command, stderr=False)[1]
vol = {}
pool = {}
dev = {}
partitions = {}
fs_size = pool_size = fs_used = 0
pool_name = ''
for line in misc.get_partitions():
partitions[line[3]] = line
for line in self.output.strip().split("\n"):
if not line:
continue
array = line.split()
if array[0] == 'Label:':
if len(vol) > 0:
self._store_data(vol, pool, fs_used, fs_size, pool_size,
pool_name)
vol = {}
pool = {}
fs_size = pool_size = 0
pool_name = ''
label = array[1].strip("'")
uuid = array[3]
pool['uuid'] = vol['uuid'] = uuid
try:
vol['real_dev'] = misc.get_device_by_uuid(uuid)
if vol['real_dev'] in self.mounts:
pool['mount'] = self.mounts[vol['real_dev']]['mp']
vol['mount'] = self.mounts[vol['real_dev']]['mp']
else:
for dev_i in self.mounts:
found = re.findall(
r'{0}:/.*'.format(vol['real_dev']), dev_i)
if found:
pool['mount'] = self.mounts[found[0]]['mp']
break
except OSError:
# udev is "hard-to-work-with" sometimes so this is fallback
vol['real_dev'] = ""
if label != 'none':
vol['label'] = label
vol['ID'] = 0
elif array[0] == 'Total':
pool['dev_count'] = array[2]
fs_used = float(misc.get_real_size(array[6]))
elif array[0] == 'devid':
# This is ugly hack to fix a problem with test suite and btrfs
# where ?sometimes? btrfs prints out device name in the path
# of the test suite rather than path in the real '/dev/'
# directory. This should cover that without any impact on
# real usage
if not os.path.islink(array[7]):
array[7] = re.sub(r'.*/dev/', '/dev/', array[7])
dev['dev_name'] = misc.get_real_device(array[7])
if not pool_name:
pool_name = self._find_uniq_pool_name(label, array[7])
dev['pool_name'] = pool_name
# Fallback in case we could not find real_dev by uuid
if 'mount' not in pool:
if dev['dev_name'] in self.mounts:
pool['mount'] = self.mounts[dev['dev_name']]['mp']
vol['real_dev'] = dev['dev_name']
if 'root' in self.mounts[dev['dev_name']]:
if self.mounts[dev['dev_name']]['root'] == '/':
vol['mount'] = self.mounts[
dev['dev_name']]['mp']
else:
for dev_i in self.mounts:
found = re.findall(
r'{0}:/.*'.format(dev['dev_name']), dev_i)
if found:
pool['mount'] = self.mounts[found[0]]['mp']
vol['real_dev'] = found[0].split(':')[0]
break
dev_used = float(misc.get_real_size(array[5]))
dev['dev_used'] = str(dev_used)
fs_size += float(misc.get_real_size(array[3]))
dev_size = \
int(partitions[dev['dev_name']][2])
pool_size += dev_size
dev['dev_free'] = dev_size - dev_used
dev['hide'] = False
self._dev[dev['dev_name']] = dev
dev = {}
if len(vol) > 0:
self._store_data(vol, pool, fs_used, fs_size, pool_size, pool_name)
def run_btrfs(self, command):
if not self._binary:
self.problem.check(self.problem.TOOL_MISSING, 'btrfs')
command.insert(0, "btrfs")
return misc.run(command, stdout=True)
def remove(self, pool):
misc.run([self.f, self.v, self.y, 'pool remove', pool])
def extend(self, pool, devices):
if type(devices) is not list:
devices = [devices]
cmd = [self.f, self.v, self.y, 'pool extend', pool]
cmd.extend(devices)
misc.run(cmd)
def __init__(self, *args, **kwargs):
super(VolumeInfo, self).__init__(*args, **kwargs)
self.data.update(misc.run(['volumedata']))
def remove(self, volume):
misc.run([self.f, self.v, self.y, 'vol remove', volume])
def resize(self, lv, size, resize_fs=True):
misc.run([
self.f, self.v, self.y, 'vol resize', lv,
str(size),
str(resize_fs)
])