def unmount_target(uuid):
# This is called by the Target RA from corosync
# only unmount targets that are controlled by chroma:Target
try:
result = cibxpath("query", "//primitive")
except OSError as err:
if err.rc == errno.ENOENT:
exit(-1)
raise err
dom = ET.fromstring(result.stdout)
# Searches for <nvpair name="target" value=uuid> in
# <primitive provider="chroma" type="Target"> in dom
if (next(
(ops for res in dom.findall(".//primitive")
if res.get("provider") == "chroma" and res.get("type") == "Target"
for ops in res.findall(".//nvpair")
if ops.get("name") == "target" and ops.get("value") == uuid),
None,
) is not None):
return
dom.unlink()
info = _get_target_config(uuid)
filesystem = FileSystem(info["backfstype"], info["bdev"])
filesystem.umount()
if agent_result_is_error(export_target(info["device_type"], info["bdev"])):
exit(-1)
def mount_target(uuid, pacemaker_ha_operation):
# This is called by the Target RA from corosync
info = _get_target_config(uuid)
import_retries = 60
succeeded = False
for i in xrange(import_retries):
# This loop is needed due pools not being immediately importable during
# STONITH operations. Track: https://github.com/zfsonlinux/zfs/issues/6727
result = import_target(info['device_type'], info['bdev'],
pacemaker_ha_operation)
succeeded = agent_result_is_ok(result)
if succeeded:
break
elif (not pacemaker_ha_operation) or (info['device_type'] != 'zfs'):
exit(-1)
time.sleep(1)
if succeeded is False:
exit(-1)
filesystem = FileSystem(info['backfstype'], info['bdev'])
filesystem.mount(info['mntpt'])
def mount_target(uuid, pacemaker_ha_operation):
# This is called by the Target RA from corosync
info = _get_target_config(uuid)
import_retries = 60
succeeded = False
while import_retries > 0:
# This loop is needed due pools not being immediately importable during
# STONITH operations. Track: https://github.com/zfsonlinux/zfs/issues/6727
result = import_target(info['device_type'], info['bdev'],
pacemaker_ha_operation)
succeeded = agent_result_is_ok(result)
if succeeded:
break
elif (not pacemaker_ha_operation) or (info['device_type'] != 'zfs'):
exit(-1)
time.sleep(1)
import_retries -= 1
if succeeded is False:
exit(-1)
filesystem = FileSystem(info['backfstype'], info['bdev'])
try:
filesystem.mount(info['mntpt'])
except RuntimeError, err:
# Make sure we export any pools when a mount fails
export_target(info['device_type'], info['bdev'])
raise err
def target_running(uuid):
# This is called by the Target RA from corosync
from os import _exit
try:
info = _get_target_config(uuid)
except (KeyError, TypeError) as err:
# it can't possibly be running here if the config entry for
# it doesn't even exist, or if the store doesn't even exist!
console_log.warning("Exception getting target config: %s", err)
_exit(1)
filesystem = FileSystem(info["backfstype"], info["bdev"])
for devices, mntpnt, _ in get_local_mounts():
if (mntpnt == info["mntpt"]) and next(
(True for device in devices
if filesystem.devices_match(device, info["bdev"], uuid)),
False,
):
_exit(0)
console_log.warning(
"Did not find mount with matching mntpt and device for %s", uuid)
_exit(1)
def unmount_target(uuid):
# This is called by the Target RA from corosync
info = _get_target_config(uuid)
filesystem = FileSystem(info['backfstype'], info['bdev'])
filesystem.umount()
if agent_result_is_error(export_target(info['device_type'], info['bdev'])):
exit(-1)
def mount_target(uuid, pacemaker_ha_operation):
# This is called by the Target RA from corosync
info = _get_target_config(uuid)
if agent_result_is_error(
import_target(info['device_type'], info['bdev'],
pacemaker_ha_operation)):
exit(-1)
filesystem = FileSystem(info['backfstype'], info['bdev'])
filesystem.mount(info['mntpt'])
def _get_zpool_zvols(self, pool_name, zpool_uuid, drives, block_devices):
zpool_vols = {}
for zvol_path in glob.glob("/dev/%s/*" % pool_name):
major_minor = block_devices.path_to_major_minor(zvol_path)
if major_minor is None:
continue
uuid = zvol_path
zpool_vols[uuid] = {
"name": zvol_path,
"path": zvol_path,
"block_device": major_minor,
"uuid": uuid,
"size": block_devices.block_device_nodes[major_minor]["size"],
"drives": drives
}
# Do this to cache the device, type see blockdevice and filesystem for info.
BlockDevice('zfs', zvol_path)
FileSystem('zfs', zvol_path)
return zpool_vols
def _get_zpool_zvols(pool_name, drives, block_devices):
"""
Each zfs pool may have zvol entries in it. This will parse those zvols and create
device entries for them
"""
zpool_vols = {}
for zvol_path in glob.glob("/dev/%s/*" % pool_name):
major_minor = block_devices.path_to_major_minor(zvol_path)
if major_minor is None:
continue
uuid = zvol_path
zpool_vols[uuid] = {
"name": zvol_path,
"path": zvol_path,
"block_device": major_minor,
"uuid": uuid,
"size": block_devices.block_device_nodes[major_minor]["size"],
"drives": drives
}
# Do this to cache the device, type see blockdevice and filesystem for info.
BlockDevice('zfs', zvol_path)
FileSystem('zfs', zvol_path)
return zpool_vols
def _scan_mounts(self):
mounts = {}
for device, mntpnt, fstype in Mounts().all():
if fstype != 'lustre':
continue
# Assume that while a filesystem is mounted, its UUID and LABEL don't change.
# Therefore we can avoid repeated blkid calls with a little caching.
if device in self._mount_cache:
fs_uuid = self._mount_cache[device]['fs_uuid']
fs_label = self._mount_cache[device]['fs_label']
else:
# Sending none as the type means BlockDevice will use it's local cache to work the type.
# This is not a good method, and we should work on a way of not storing such state but for the
# present it is the best we have.
try:
fs_uuid = BlockDevice(None, device).uuid
fs_label = FileSystem(None, device).label
# If we have scanned the devices then it is safe to cache the values.
if LinuxDevicePlugin.devices_scanned:
self._mount_cache[device]['fs_uuid'] = fs_uuid
self._mount_cache[device]['fs_label'] = fs_label
except AgentShell.CommandExecutionError:
continue
dev_normalized = ndp.normalized_device_path(device)
recovery_status = {}
try:
recovery_file = glob.glob(
"/proc/fs/lustre/*/%s/recovery_status" % fs_label)[0]
recovery_status_text = open(recovery_file).read()
for line in recovery_status_text.split("\n"):
tokens = line.split(":")
if len(tokens) != 2:
continue
k = tokens[0].strip()
v = tokens[1].strip()
recovery_status[k] = v
except IndexError:
# If the recovery_status file doesn't exist,
# we will return an empty dict for recovery info
pass
mounts[device] = {
'device': dev_normalized,
'fs_uuid': fs_uuid,
'mount_point': mntpnt,
'recovery_status': recovery_status
}
# Drop cached info about anything that is no longer mounted
for k in self._mount_cache.keys():
if not k in mounts:
del self._mount_cache[k]
return mounts.values()
def _update_pool_or_datasets(self, block_devices, pool, datasets, zvols):
if (datasets == {}) and (zvols == {}):
name = pool['name']
block_devices.block_device_nodes[pool['block_device']] = {
'major_minor': pool['block_device'],
'path': name,
'paths': [name],
'serial_80': None,
'serial_83': None,
'size': pool['size'],
'filesystem_type': None,
'parent': None
}
# Do this to cache the device, type see blockdevice and filesystem for info.
BlockDevice('zfs', name)
FileSystem('zfs', name)
self._zpools[pool['uuid']] = pool
if datasets != {}:
for info in datasets.itervalues():
major_minor = info['block_device']
name = info['name']
block_devices.block_device_nodes[major_minor] = {
'major_minor': major_minor,
'path': name,
'paths': [name],
'serial_80': None,
'serial_83': None,
'size': info['size'],
'filesystem_type': 'zfs',
'parent': None
}
# Do this to cache the device, type see blockdevice and filesystem for info.
BlockDevice('zfs', name)
FileSystem('zfs', name)
self._datasets.update(datasets)
if zvols != {}:
self._zvols.update(zvols)
def target_running(uuid):
from os import _exit
try:
info = _get_target_config(uuid)
except (KeyError, TypeError) as e:
# it can't possibly be running here if the config entry for
# it doesn't even exist, or if the store doesn't even exist!
console_log.warning("Exception getting target config: '%s'" % e)
_exit(1)
filesystem = FileSystem(info['backfstype'], info['bdev'])
for device, mntpnt, fstype in get_local_mounts():
if (mntpnt == info['mntpt']) and filesystem.devices_match(
device, info['bdev'], uuid):
_exit(0)
console_log.warning(
"Did not find mount with matching mntpt and device for %s" % uuid)
_exit(1)
def _add_zfs_pool(self, line, block_devices):
pool, size_str, uuid, health = line.split()
if health in self.acceptable_health:
size = util.human_to_bytes(size_str)
drive_mms = block_devices.paths_to_major_minors(
self._get_all_zpool_devices(pool))
if drive_mms is None:
daemon_log.warn("Could not find major minors for zpool '%s'" %
pool)
return
# This will need discussion, but for now fabricate a major:minor. Do we ever use them as numbers?
block_device = "zfspool:%s" % pool
datasets = self._get_zpool_datasets(pool, uuid, drive_mms,
block_devices)
zvols = self._get_zpool_zvols(pool, drive_mms, uuid, block_devices)
if (datasets == {}) and (zvols == {}):
block_devices.block_device_nodes[block_device] = {
'major_minor': block_device,
'path': pool,
'serial_80': None,
'serial_83': None,
'size': size,
'filesystem_type': None,
'parent': None
}
# Do this to cache the device, type see blockdevice and filesystem for info.
BlockDevice('zfs', pool)
FileSystem('zfs', pool)
self._zpools[uuid] = {
"name": pool,
"path": pool,
"block_device": block_device,
"uuid": uuid,
"size": size,
"drives": drive_mms,
}
if datasets != {}:
self._datasets.update(datasets)
if zvols != {}:
self._zvols.update(zvols)
def register_target(device_path, mount_point, backfstype):
filesystem = FileSystem(backfstype, device_path)
_mkdir_p_concurrent(mount_point)
filesystem.mount(mount_point)
filesystem.umount()
return {'label': filesystem.label}
def register_target(device_path, mount_point, backfstype):
"""
Mount/unmount target so it registers with MGS
"""
filesystem = FileSystem(backfstype, device_path)
_mkdir_p_concurrent(mount_point)
filesystem.mount(mount_point)
filesystem.umount()
return {"label": filesystem.label}
def _get_zpool_datasets(self, pool_name, zpool_uuid, drives,
block_devices):
out = AgentShell.try_run(
['zfs', 'list', '-H', '-o', 'name,avail,guid'])
zpool_datasets = {}
if out.strip() != "no datasets available":
for line in filter(None, out.split('\n')):
name, size_str, uuid = line.split()
size = util.human_to_bytes(size_str)
if name.startswith("%s/" % pool_name):
# This will need discussion, but for now fabricate a major:minor. Do we ever use them as numbers?
major_minor = "zfsset:%s" % (len(self.datasets) + 1)
block_devices.block_device_nodes[major_minor] = {
'major_minor': major_minor,
'path': name,
'serial_80': None,
'serial_83': None,
'size': size,
'filesystem_type': 'zfs',
'parent': None
}
# Do this to cache the device, type see blockdevice and filesystem for info.
BlockDevice('zfs', name)
FileSystem('zfs', name)
zpool_datasets[uuid] = {
"name": name,
"path": name,
"block_device": major_minor,
"uuid": uuid,
"size": size,
"drives": drives
}
daemon_log.debug("zfs mount '%s'" % name)
return zpool_datasets
# Searches for <nvpair name="target" value=uuid> in
# <primitive provider="chroma" type="Target"> in dom
if not next(
(ops
for res in dom.getElementsByTagName('primitive') if res.getAttribute(
"provider") == "chroma" and res.getAttribute("type") == "Target"
for ops in res.getElementsByTagName('nvpair')
if ops.getAttribute("name") == "target"
and ops.getAttribute("value") == uuid), False):
return
dom.unlink()
info = _get_target_config(uuid)
filesystem = FileSystem(info['backfstype'], info['bdev'])
filesystem.umount()
if agent_result_is_error(export_target(info['device_type'], info['bdev'])):
exit(-1)
def import_target(device_type, path, pacemaker_ha_operation):
"""
Passed a device type and a path import the device if such an operation make sense.
For example a jbod scsi disk does not have the concept of import whilst zfs does.
:param device_type: the type of device to import
:param path: path of device to import
:param pacemaker_ha_operation: This import is at the request of pacemaker. In HA operations the
device may often have not have been cleanly exported because the previous mounted node failed
def _scan_mounts(self):
mounts = {}
data = scanner_cmd("Stream")
local_mounts = parse_local_mounts(data["localMounts"])
zfs_mounts = [(d, m, f) for d, m, f in local_mounts if f == "zfs"]
lustre_mounts = [(d, m, f) for d, m, f in local_mounts
if f == "lustre"]
for device, mntpnt, _ in lustre_mounts:
fs_label, fs_uuid = process_zfs_mount(device, data, zfs_mounts)
if not fs_label:
fs_label, fs_uuid = process_lvm_mount(device, data)
if not fs_label:
# todo: derive information directly from device-scanner output for ldiskfs
# Assume that while a filesystem is mounted, its UUID and LABEL don't change.
# Therefore we can avoid repeated blkid calls with a little caching.
if device in self._mount_cache:
fs_uuid = self._mount_cache[device]["fs_uuid"]
fs_label = self._mount_cache[device]["fs_label"]
else:
# Sending none as the type means BlockDevice will use it's local
# cache to work the type. This is not a good method, and we
# should work on a way of not storing such state but for the
# present it is the best we have.
try:
fs_uuid = BlockDevice(None, device).uuid
fs_label = FileSystem(None, device).label
# If we have scanned the devices then it is safe to cache the values.
if LinuxDevicePlugin.devices_scanned:
self._mount_cache[device]["fs_uuid"] = fs_uuid
self._mount_cache[device][
"fs_label"] = fs_label
except AgentShell.CommandExecutionError:
continue
recovery_status = {}
try:
lines = AgentShell.try_run([
"lctl", "get_param", "-n",
"*.%s.recovery_status" % fs_label
])
for line in lines.split("\n"):
tokens = line.split(":")
if len(tokens) != 2:
continue
k = tokens[0].strip()
v = tokens[1].strip()
recovery_status[k] = v
except Exception:
# If the recovery_status file doesn't exist,
# we will return an empty dict for recovery info
pass
mounts[device] = {
"fs_uuid": fs_uuid,
"mount_point": mntpnt,
"recovery_status": recovery_status,
}
# Drop cached info about anything that is no longer mounted
for k in self._mount_cache.keys():
if k not in mounts:
del self._mount_cache[k]
return mounts.values()
def format_target(device_type,
target_name,
device,
backfstype,
target_types=(),
mgsnode=(),
fsname=None,
failnode=(),
servicenode=(),
param=None,
index=None,
comment=None,
mountfsoptions=None,
network=(),
device_size=None,
mkfsoptions=None,
reformat=False,
stripe_count_hint=None,
iam_dir=False,
dryrun=False,
verbose=False,
quiet=False):
"""Perform a mkfs.lustre operation on a target device.
Device may be a number of devices, block"""
# freeze a view of the namespace before we start messing with it
args = dict(locals())
options = []
# Now remove the locals that are not parameters for mkfs.
del args['device_type']
del args['target_name']
single = "--{}".format
double = "--{}={}".format
tuple_options = [
"target_types", "mgsnode", "failnode", "servicenode", "network"
]
for name in tuple_options:
arg = args[name]
# ensure that our tuple arguments are always tuples, and not strings
if not hasattr(arg, "__iter__"):
arg = (arg, )
if name == "target_types":
options += [single(target) for target in arg]
elif name == 'mgsnode':
options += [double(name, ",".join(mgs_nids)) for mgs_nids in arg]
elif len(arg) > 0:
options.append(double(name, ",".join(arg)))
flag_options = {
'dryrun': '--dryrun',
'reformat': '--reformat',
'iam_dir': '--iam-dir',
'verbose': '--verbose',
'quiet': '--quiet',
}
options += [flag_options[arg] for arg in flag_options if args[arg]]
dict_options = ["param"]
for name in dict_options:
arg = args[name]
if arg:
options += [
x for key in arg if arg[key] is not None
for x in [single(name), "{}={}".format(key, arg[key])]
]
# everything else
handled = set(flag_options.keys() + tuple_options + dict_options)
options += [
double(name, args[name]) for name in set(args.keys()) - handled
if name != "device" and args[name] is not None
]
# cache BlockDevice to store knowledge of the device_type at this path
BlockDevice(device_type, device)
filesystem = FileSystem(backfstype, device)
return filesystem.mkfs(target_name, options)
def format_target(device_type,
target_name,
device,
backfstype,
target_types=(),
mgsnode=(),
fsname=None,
failnode=(),
servicenode=(),
param={},
index=None,
comment=None,
mountfsoptions=None,
network=(),
device_size=None,
mkfsoptions=None,
reformat=False,
stripe_count_hint=None,
iam_dir=False,
dryrun=False,
verbose=False,
quiet=False):
"""Perform a mkfs.lustre operation on a target device.
Device may be a number of devices, block"""
# freeze a view of the namespace before we start messing with it
args = dict(locals())
options = []
# Now remove the locals that are not parameters for mkfs.
del args['device_type']
del args['target_name']
tuple_options = [
"target_types", "mgsnode", "failnode", "servicenode", "network"
]
for name in tuple_options:
arg = args[name]
# ensure that our tuple arguments are always tuples, and not strings
if not hasattr(arg, "__iter__"):
arg = (arg, )
if name == "target_types":
for type in arg:
options.append("--%s" % type)
elif name == 'mgsnode':
for mgs_nids in arg:
options.append("--%s=%s" % (name, ",".join(mgs_nids)))
else:
if len(arg) > 0:
options.append("--%s=%s" % (name, ",".join(arg)))
flag_options = {
'dryrun': '--dryrun',
'reformat': '--reformat',
'iam_dir': '--iam-dir',
'verbose': '--verbose',
'quiet': '--quiet',
}
for arg in flag_options:
if args[arg]:
options.append("%s" % flag_options[arg])
dict_options = ["param"]
for name in dict_options:
for key, value in args[name].items():
if value is not None:
options.extend(["--%s" % name, "%s=%s" % (key, value)])
# everything else
handled = set(flag_options.keys() + tuple_options + dict_options)
for name in set(args.keys()) - handled:
if name == "device":
continue
value = args[name]
if value is not None:
options.append("--%s=%s" % (name, value))
# cache BlockDevice to store knowledge of the device_type at this path
BlockDevice(device_type, device)
filesystem = FileSystem(backfstype, device)
return filesystem.mkfs(target_name, options)