def do_check(self):
cmd = ['mpathadm', 'list', 'LU']
out, err, ret = justcall(cmd)
if ret != 0:
return self.undef
lines = out.split('\n')
if len(lines) < 4:
return self.undef
r = []
dev = None
wwid = ""
for line in lines:
if "/dev/" in line:
# new mpath
# - remember current dev
# - remember current wwid
# - reset path counter
dev = line.strip()
wwid = line[line.index('t') + 1:line.rindex('d')]
n = 0
elif '/disk@g' in line:
# unmapped dev
# - remember current dev
# - remember current wwid
# - reset path counter
dev = line.strip()
wwid = '_' + line[line.index('@g') + 2:]
n = 0
if "Total Path Count:" in line:
continue
if "Operational Path Count:" in line:
# - store current dev if valid
# - then:
# - reset path counter
# - reset dev
n = int(line.split(':')[-1].strip())
if dev is not None:
r.append({
"instance": wwid,
"value": str(n),
"path": self.find_svc(dev),
})
dev = None
n = 0
return r
def get_lvs_attr():
cmd = [
Env.syspaths.lvs, '-o', 'vg_name,lv_name,lv_attr', '--noheadings',
'--separator=;'
]
out, err, ret = justcall(cmd)
data = {}
for line in out.splitlines():
l = line.split(";")
if len(l) != 3:
continue
vgname = l[0].strip()
lvname = l[1].strip()
attr = l[2].strip()
if vgname not in data:
data[vgname] = {}
data[vgname][lvname] = attr
return data
def dev_to_paths(dev, log=None):
dev = os.path.realpath(dev)
if dev.startswith("/dev/sd"):
return [dev]
if dev.startswith("/dev/vx"):
from utilities.subsystems.veritas import vx_dev_to_paths
return vx_dev_to_paths(dev)
if not dev.startswith("/dev/dm-"):
return []
name = os.path.basename(dev)
cmd = ["dmsetup", "table", "-j", str(major("device-mapper")), "-m", dev[8:]]
out, err, ret = justcall(cmd)
if ret != 0:
raise ex.Error(err)
if "multipath" not in out:
return []
paths = ["/dev/"+os.path.basename(_name) for _name in glob.glob("/sys/block/%s/slaves/*" % name)]
return paths
def abort_start(self):
if self.svc.topology != "failover":
return
if self.is_standby:
return
try:
for node in self.svc.nodes:
if node == Env.nodename:
continue
cmd = Env.rsh.split() + [node, "test", "-f", self.flag_f]
out, err, ret = justcall(cmd)
if ret == 0:
self.log.error("already up on %s", node)
return True
return False
except Exception as exc:
self.log.exception(exc)
return True
def cciss_id(self, dev):
if dev in self.disk_ids:
return self.disk_ids[dev]
if which('cciss_id'):
cciss_id = 'cciss_id'
else:
return ""
cmd = [cciss_id, dev]
out, err, ret = justcall(cmd)
if ret == 0:
id = out.split('\n')[0]
if id.startswith('3'):
id = id[1:]
else:
id = self.prefix_local(id)
self.disk_ids[dev] = id
return id
return ""
def parse_mounts(self):
mounts = []
out, err, ret = justcall(['mount'])
for l in out.split('\n'):
words = l.split()
if len(words) < 4:
break
dev = words[0]
mnt = words[2]
opts = ' '.join(words[3:]).strip('(').strip(')').split(', ')
type = opts[0]
if len(opts) < 3:
mnt_opt = ''
else:
mnt_opt = ','.join(opts[2:])
m = Mount(dev, mnt, type, mnt_opt)
mounts.append(m)
return mounts
def _get_dev_stats(self, mntpt, data):
cmd = ['btrfs', 'dev', 'stats', mntpt]
out, err, ret = justcall(cmd)
if ret != 0:
return data
for line in out.split('\n'):
l = line.split()
if len(l) != 2:
continue
key, val = l
l = key.split('.')
if len(l) != 2:
continue
dev, err_type = l
dev = dev.lstrip('[').rstrip(']')
if dev not in data:
data[dev] = {}
data[dev][err_type] = val
return data
def parse_memory(self):
self.banks = 0
self.slots = 0
cmd = ['cprop', '-summary', '-c', 'Memory']
out, err, ret = justcall(cmd)
if ret != 0:
return '0'
in_banks = True
for line in out.split('\n'):
if 'Empty Slots' in line:
in_banks = False
elif 'Instance' in line:
self.slots += 1
if in_banks:
self.banks += 1
def scan(self, dev):
cmd = [
"scsimgr", "-p", "get_attr", "-D",
self.dev2char(dev), "-a", "wwid", "-a", "device_file", "-a", "vid",
"-a", "pid", "-a", "capacity"
]
out, err, ret = justcall(cmd)
if ret != 0:
self.h[dev] = dict(wwid="", vid="", pid="", size=0)
return
(wwid, foo, vid, pid, size) = out.split(':')
wwid = wwid.replace('0x', '')
if len(size) != 0:
size = int(size) / 2048
else:
size = 0
vid = vid.strip('" ')
pid = pid.strip('" ')
self.h[dev] = dict(wwid=wwid, vid=vid, pid=pid, size=size)
def naviseccli(cmd,
scope=None,
spa=None,
spb=None,
username=None,
password=None):
if which('/opt/Navisphere/bin/naviseccli') is None:
raise ex.Error(
'can not find Navicli programs in usual /opt/Navisphere/bin')
_cmd = ['/opt/Navisphere/bin/naviseccli', '-h', spa]
_cmd += cmd
out, err, ret = justcall(_cmd)
if "Security file not found" in out:
print(_cmd)
print(out)
raise ex.Error("naviseccli command execution error")
return out, err
def network_route_add(self, dst=None, gw=None, dev=None, local_ip=None, brdev=None, brip=None, table=None, tunnel="auto", **kwargs):
if dst is None:
return
if tunnel == "auto":
if gw is not None:
cmd = ["ip", "route", "replace", dst, "via", gw, "table", table]
elif dev is not None:
cmd = ["ip", "route", "replace", dst, "dev", dev, "table", table]
out, err, ret = justcall(cmd)
else:
err = ""
if tunnel == "always" or "invalid gateway" in err or "is unreachable" in err:
tun = self.network_tunnel_ipip_add(local_ip, gw)
cmd = ["ip", "route", "replace", dst, "dev", tun["dev"], "src", brip.split("/")[0], "table", table]
self.vcall(cmd)
else:
self.log.info(" ".join(cmd))
for line in out.splitlines():
self.log.info(line)
def pool_status(self):
from utilities.converters import convert_size
if not os.path.exists(self.path):
os.makedirs(self.path)
data = {
"type": self.type,
"name": self.name,
"capabilities": self.capabilities,
"head": self.path,
}
cmd = ["df", "-P", self.path]
out, err, ret = justcall(cmd)
if ret != 0:
return data
l = out.splitlines()[-1].split()
data["free"] = int(l[3])
data["used"] = int(l[2])
data["size"] = int(l[1])
return data
def state_changing_action(self, cmd, timeout=10):
"""
State changing action can be denied by a peer node during
commits. This method implement a retry loop waiting for
the action to be not-denied.
"""
self.log.info(" ".join(cmd))
for i in range(timeout):
out, err, ret = justcall(cmd)
if ret == 11:
# cluster-wide drbd state change in-progress
time.sleep(1)
continue
elif ret != 0:
call_log(buff=err, log=self.log, level="error")
raise ex.Error()
call_log(buff=out, log=self.log, level="info")
return out, err, ret
raise ex.Error("timeout waiting for action non-denied by peer")
def action(self, nodename, thr=None, **kwargs):
if "node.x.drbdadm" not in capabilities:
raise ex.Error("this node is not drbd capable")
out, err, ret = justcall(["drbdadm", "dump"])
if ret:
raise ex.HTTP(500, err)
minors = set()
ports = set()
for line in out.splitlines():
m = re.match(RE_MINOR, line)
if m is not None:
minors.add(int(m.group(1)))
m = re.match(RE_PORT, line)
if m is not None:
ports.add(int(m.group(1)))
return {
"minors": sorted(list(minors)),
"ports": sorted(list(ports)),
}
def _get_os_release(self):
r = self._get_os_release_os_release()
if r and r not in (
"/Linux",
"Linux 7 (Core)" # centos7 poor pretty_name
):
return r
files = ['/etc/debian_version',
'/etc/vmware-release',
'/etc/oracle-release',
'/etc/redhat-release',
'/etc/gentoo-release']
if os.path.exists('/etc/SuSE-release'):
v = []
with open('/etc/SuSE-release') as f:
for line in f.readlines():
if 'VERSION' in line:
v += [line.split('=')[-1].replace('\n','').strip('" ')]
if 'PATCHLEVEL' in line:
v += [line.split('=')[-1].replace('\n','').strip('" ')]
return '.'.join(v)
if os.path.exists('/etc/alpine-release'):
with open('/etc/alpine-release') as f:
return f.read().strip()
r = self._get_os_release_lsb()
if r:
return r
r = self._get_os_release_debian_version()
if r:
return r
if os.path.exists('/etc/oracle-release') and \
os.path.exists('/etc/redhat-release'):
with open('/etc/oracle-release') as f1:
if " VM " in f1.read():
with open('/etc/redhat-release') as f2:
return f2.read().split('\n')[0].replace(self._get_os_vendor(), '').strip()
for f in files:
if os.path.exists(f):
(out, err, ret) = justcall(['cat', f])
if ret != 0:
return 'Unknown'
return out.split('\n')[0].replace(self._get_os_vendor(), '').replace("GNU/Linux", "").replace("Linux", "").replace("release", "").strip()
return 'Unknown'
def vxprint(self):
cmd = ["vxprint", "-g", self.vg, "-v"]
out, err, ret = justcall(cmd)
if ret == 11:
# no lv
return {}
if ret != 0:
raise ex.Error(err)
data = {}
for line in out.splitlines():
words = line.split()
if len(words) < 7:
continue
if words[0] == "TY":
headers = list(words)
continue
lv = namedtuple("lv", headers)._make(words)
data[lv.NAME] = lv
return data
def _get_cpu_model(self):
(out, err, ret) = justcall(['/usr/sbin/psrinfo', '-pv'])
if ret != 0:
return 'Unknown'
lines = out.split('\n')
lines = [line for line in lines if len(line) > 0]
if len(lines) == 0:
return 'Unknown'
model = lines[-1].strip()
if model.startswith('The '):
model = model.replace('The ', '')
known_garbage = [' (chipid', ' (portid', ' physical proc']
for s in known_garbage:
try:
i = model.index(s)
model = model[:i]
except ValueError:
continue
return model
def netdev(self, d, day, start, end):
f = self.sarfile(day)
cols = [
'date', 'dev', 'rxpckps', 'txpckps', 'rxkBps', 'txkBps', 'nodename'
]
if f is None:
return [], []
cmd = ['sar', '-t', '-n', 'DEV', '-f', f, '-s', start, '-e', end]
(buff, err, ret) = justcall(cmd)
if "%ifutil" in buff:
n = 10
else:
n = 9
lines = []
div = 1
for line in buff.split('\n'):
l = line.split()
if len(l) != n:
continue
if l[1] in ['IFACE', 'lo']:
if 'rxbyt/s' in l:
div = 1024
continue
if 'dummy' in l[1] or 'vnet' in l[1] or 'veth' in l[1] or \
'pan' in l[1] or 'sit' in l[1]:
continue
if l[0] == 'Average:':
continue
m = []
m.append('%s %s' % (d, l[0]))
m.append(l[1])
m.append(str(float(l[4]) / div))
m.append(str(float(l[5]) / div))
m.append(l[2])
m.append(l[3])
m.append(self.nodename)
lines.append(m)
return cols, lines
def snapdestroykey(self, s):
if protected_mount(self.snaps[s]['snap_mnt']):
self.log.error("the snapshot is no longer mounted in %s. panic." %
self.snaps[s]['snap_mnt'])
raise ex.Error
cmd = ['fuser', '-kmv', self.snaps[s]['snap_mnt']]
(ret, out, err) = self.vcall(cmd, err_to_info=True)
cmd = [Env.syspaths.umount, self.snaps[s]['snap_mnt']]
(ret, out, err) = self.vcall(cmd)
utilities.devices.linux.udevadm_settle()
cmd = ['lvremove', '-A', 'n', '-f', self.snaps[s]['snap_dev']]
self.log.info(' '.join(cmd))
for i in range(1, 30):
out, err, ret = justcall(cmd)
if ret == 0:
break
err_l1 = err.split('\n')
err_l2 = []
out_l = out.split('\n')
for e in err_l1:
if 'This metadata update is NOT backed up' in e:
pass
elif 'Falling back to direct link removal.' in e:
out_l.append(e)
elif 'Falling back to direct node removal.' in e:
out_l.append(e)
else:
err_l2.append(e)
err = '\n'.join(err_l2)
out = '\n'.join(out_l)
if len(out) > 0:
self.log.info(out)
if len(err) > 0:
self.log.error(err)
if ret != 0:
self.log.error("failed to remove snapshot %s (attempts: %d)" %
(self.snaps[s]['snap_dev'], i))
elif i > 1:
self.log.info("successfully removed snapshot %s (attempts: %d)" %
(self.snaps[s]['snap_dev'], i))
del (self.snaps[s])
def fs_u_t(t):
if not which('df'):
return []
cmd = ['df', '-F', t, '-k']
(out, err, ret) = justcall(cmd)
if ret != 0:
return []
lines = out.split('\n')
if len(lines) < 2:
return []
vals = []
for line in lines[1:]:
l = line.split()
if len(l) == 5:
l = [''] + l
elif len(l) != 6:
continue
vals.append(
[now, node.nodename, l[5], l[1], l[4].replace('%', '')])
return vals
def showmapped(self, refresh=False):
if not refresh:
try:
return getattr(self, "mapped_data")
except AttributeError:
pass
self.modprobe()
cmd = ["rbd", "showmapped", "--format", "json"]
out, err, ret = justcall(cmd)
if ret != 0:
raise ex.Error("rbd showmapped failed: " + err)
try:
_data = json.loads(out)
except Exception as e:
raise ex.Error(str(e))
data = {}
for id, img_data in _data.items():
data[img_data["pool"] + "/" + img_data["name"]] = img_data
self.mapped_data = data
return data
def rcp(self, src, dst):
if self.guestos == "windows":
""" Windows has no sshd.
"""
raise ex.NotSupported("remote copy not supported on Windows")
self.getaddr()
if self.addr is None:
raise ex.Error('no usable public ip to send files to')
timeout = 5
cmd = [
'scp', '-o', 'StrictHostKeyChecking=no', '-o',
'ConnectTimeout=' + str(timeout), '-i',
self.keyfile(), src, self.addr + ':' + dst
]
out, err, ret = justcall(cmd)
if ret != 0:
raise ex.Error("'%s' execution error:\n%s" % (' '.join(cmd), err))
return out, err, ret
def _get_fc_hbas(self):
hbas = []
if not which('fcinfo'):
return []
cmd = ['fcinfo']
out, err, ret = justcall(cmd)
if ret != 0:
return []
for line in out.split('\n'):
if 'PortWWN' not in line:
continue
l = line.split()
i = l.index('PortWWN:')
if len(l) < i+2:
continue
index = l[0].split('-')[-1].strip(':')
portwwn = l[i+1].replace(':', '')
host = int(l[-1].split('Scsi')[-1].strip(':'))
hbas.append((index, portwwn, host))
return hbas
def vxdisk_list(self):
if not which("vxdisk"):
return {}
cmd = ["vxdisk", "list"]
out, err, ret = justcall(cmd)
if ret != 0:
raise ex.Error(err)
data = {}
for line in out.splitlines():
words = line.split(None, 4)
if len(words) < 5:
continue
if words[0] == "DEVICE":
headers = list(words)
continue
dev = namedtuple("dev", headers)._make(words)
if dev.GROUP != self.name and dev.GROUP != "(%s)" % self.name:
continue
data[dev.DEVICE] = dev
return data
def _findphys(self, netif):
res = ""
cmd = [
'/usr/sbin/dladm', 'show-link', '-p', '-o', 'link,class,over',
netif
]
out, err, ret = justcall(cmd)
if ret != 0:
return ""
for line in out.splitlines():
if len(line) == 0:
break
v = line.split(':')
if v[1] == 'phys':
self.l[self.topif].add(v[0])
else:
ifs = v[2].split(' ')
for i in ifs:
res = self._findphys(i)
return "OK"
def has_pv(self, pv):
cmd = ["vxdisk", "list", pv]
out, err, ret = justcall(cmd)
if ret != 0:
return False
for line in out.splitlines():
if line.startswith("group:"):
if "name=%s " % self.name in line:
self.log.info("pv %s is already a member of vg %s", pv,
self.name)
return True
elif "name= " in line:
# pv already initialized but not in a dg
return True
else:
vg = line.split("name=", 1)[0].split()[0]
raise ex.Error("pv %s in use by vg %s" % (pv, vg))
if line.startswith("flags:") and "invalid" in line:
return False
return False
def __init__(self):
"""scan node to get informations about its zones
"""
self.zone_list = []
self.zonename2zone = dict()
self.zonepath2zone = dict()
cmd = [ZONEADM, 'list', '-cip']
(out, err, status) = justcall(cmd)
if status == 0:
#zoneid:zonename:state:zonepath:uuid:brand:ip-type
for zoneadm_line in out.split('\n'):
v = zoneadm_line.split(':')
if len(v) != 7:
continue
zone = Zone(zoneid=v[0], zonename=v[1], state=v[2],
zonepath=v[3], uuid=v[4], brand=v[5], ip_type=v[6])
self.zonename2zone[zone.zonename] = zone
self.zonepath2zone[zone.zonepath] = zone
self.zone_list.append(zone)
def destroy(self, options=None):
"""
Destroy the dataset.
"""
if options is None:
options = []
if not self.exists():
return True
cmd = [Env.syspaths.zfs, 'destroy'] + options + [self.name]
if self.log:
self.log.info(" ".join(cmd))
_, err, ret = justcall(cmd)
if ret == 0:
return True
elif "could not find any snapshot" in err:
return True
elif "dataset does not exist" in err:
return True
else:
return False
def sign_csr(log=None, **data):
days = data["validity"]
if days < 1:
days = 1
cmd = ["openssl", "x509", "-req",
"-in", data["csr"],
"-CA", data["cacrt"],
"-CAkey", data["cakey"],
"-CAcreateserial",
"-out", data["crt"],
"-days", str(days),
"-sha256"]
if data.get("alt_names"):
write_openssl_cnf(data)
cmd += ["-extfile", data["cnf"], "-extensions", "SAN"]
if log:
log.info(" ".join(cmd))
out, err, ret = justcall(cmd)
if ret != 0:
raise ex.Error(out+err)
def boot_and_wait_reboot(self):
"""
Boot freshly installed zones, then wait for automatic zone reboot
boot zone
ensure for zone init process is running
wait for 2 'system boot' (this is only usable on freshly installed zones)
wait for zone running
wait for zone operational
We wait for 2 'system boot', this is only usable on freshly installed zones
"""
def wait_boot_count(count, max_retries=240):
retries = 0
self.log.info('wait for %s boot count is %s (max retries %s)',
self.name, count, max_retries)
cmd = [ZLOGIN, self.name, 'last', 'reboot']
while retries < max_retries:
out, err, st = justcall(cmd)
if st == 0:
reboots = len([
line for line in out.split('\n')
if 'system boot' in line
])
self.log.info('%s boot count: %s', self.name, reboots)
if reboots >= count:
return True
time.sleep(1)
retries += 1
self.log.info("wait for zone %s boot and reboot...", self.name)
self.zone_boot()
if self.is_running is False:
raise ex.Error("zone is not running")
cmd = [PGREP, "-z", self.name, "-f", INIT]
out, err, st = justcall(cmd)
if st != 0:
raise ex.Error("fail to detect zone init process")
wait_boot_count(2)
self.wait_for_fn(self.is_up, self.start_timeout, 2)
self.log.info("wait for zone operational")
self.wait_for_fn(self.operational, self.start_timeout, 2)