def listpkg_ips():
"""
Return a list of ips packages installed.
"""
#
# PKGINST: SUNWzoneu
# NAME: Solaris Zones (Usr)
# CATEGORY: system
# ARCH: i386
# VERSION: 11.11,REV=2009.04.08.17.26
# VENDOR: Sun Microsystems, Inc.
# DESC: Solaris Zones Configuration and Administration
# HOTLINE: Please contact your local service provider
# STATUS: completely installed
#
if which('uname') is None:
return []
cmd = ['uname', '-p']
out, _, _ = justcall(cmd)
arc = out.split('\n')[0]
if which('pkg') is None:
return []
cmd = ['pkg', 'list', '-H']
out, _, _ = justcall(cmd)
lines = []
for line in out.split('\n'):
elems = line.split()
if len(elems) != 3:
continue
data = [rcEnv.nodename, elems[0], elems[1], arc, "ips", ""]
lines.append(data)
return lines
def operational(self):
if not resContainer.Container.operational(self):
return False
cmd = self.runmethod + ['test', '-f', '/bin/systemctl']
out, _, ret = justcall(cmd)
if ret == 1:
# not a systemd container. no more checking.
self.log.debug("/bin/systemctl not found in container")
return True
# systemd on-demand loading will let us start the encap service before
# the network is fully initialized, causing start issues with nfs mounts
# and listening apps.
# => wait for systemd default target to become active
cmd = self.runmethod + ['systemctl', 'is-active', 'default.target']
out, _, ret = justcall(cmd)
if ret == 1:
# if systemctl is-active fails, retry later
self.log.debug("systemctl is-active failed")
return False
if out.strip() == "active":
self.log.debug("systemctl is-active succeeded")
return True
# ok, wait some more
self.log.debug("waiting for lxc to come up")
return False
def _scsi_id(self, dev, args=[]):
wwid = self.mpath_id(dev)
if wwid is not None:
return wwid
if dev in self.disk_ids:
return self.disk_ids[dev]
if which('scsi_id'):
scsi_id = 'scsi_id'
elif which('/lib/udev/scsi_id'):
scsi_id = '/lib/udev/scsi_id'
else:
return ""
cmd = [scsi_id, '-g', '-u'] + args + ['-d', dev]
out, err, ret = justcall(cmd)
if ret == 0:
id = out.split('\n')[0]
if id.startswith('3') or id.startswith('2') or id.startswith('5'):
id = id[1:]
else:
id = self.prefix_local(id)
self.disk_ids[dev] = id
return id
sdev = dev.replace("/dev/", "/block/")
cmd = [scsi_id, '-g', '-u'] + args + ['-s', sdev]
out, err, ret = justcall(cmd)
if ret == 0:
id = out.split('\n')[0]
if id.startswith('3') or id.startswith('2') or id.startswith('5'):
id = id[1:]
else:
id = self.prefix_local(id)
self.disk_ids[dev] = id
return id
return ""
def swap(self, d, day, start, end):
f = self.sarfile(day)
cols = [
'date', 'kbswpfree', 'kbswpused', 'pct_swpused', 'kbswpcad',
'pct_swpcad', 'nodename'
]
if f is None:
return [], []
cmd = ['sar', '-t', '-S', '-f', f, '-s', start, '-e', end]
(buff, err, ret) = justcall(cmd)
if ret != 0:
""" redhat 5
"""
cmd = ['sar', '-t', '-r', '-f', f, '-s', start, '-e', end]
(buff, err, ret) = justcall(cmd)
lines = []
for line in buff.split('\n'):
l = line.split()
if len(l) == 10:
""" redhat 5
"""
l = [l[0]] + l[6:] + ['0']
if len(l) != 6:
continue
if 'kbswpfree' in l:
continue
if l[0] == 'Average:':
continue
l.append(self.nodename)
l[0] = '%s %s' % (d, l[0])
lines.append(l)
return cols, lines
def hpacucli(self, cmd):
cmd = ['hpacucli'] + cmd
try:
out, err, ret = justcall(cmd)
except OSError:
cmd = [os.environ['SHELL']] + cmd
out, err, ret = justcall(cmd)
return out, err, ret
def get_mcast(self):
if which('netstat'):
cmd = ['netstat', '-gn']
out, _, _ = justcall(cmd)
return self.parse_mcast_netstat(out)
elif which(rcEnv.syspaths.ip):
cmd = [rcEnv.syspaths.ip, 'maddr']
out, _, _ = justcall(cmd)
return self.parse_mcast_ip(out)
def _get_targets(self):
import glob
# fc / fcoe
l = []
hbas = self._get_hba()
for hba in hbas:
if not hba["hba_type"].startswith('fc'):
continue
targets = glob.glob(
'/sys/class/fc_transport/target%s:*/port_name' % hba["host"])
targets += glob.glob(
'/sys/class/fc_remote_ports/rport-%s:*/port_name' %
hba["host"])
for target in targets:
with open(target, 'r') as f:
tgt_id = f.read().strip('0x').strip('\n')
if (hba["hba_id"], tgt_id) not in l:
l.append((hba["hba_id"], tgt_id))
# iscsi
hba_id = self.get_iscsi_hba_id()
if hba_id is not None:
cmd = ['iscsiadm', '-m', 'session']
out, err, ret = justcall(cmd)
if ret == 0:
"""
tcp: [1] 192.168.231.141:3260,1 iqn.2000-08.com.datacore:sds1-1
tcp: [2] 192.168.231.142:3260,1 iqn.2000-08.com.datacore:sds2-1 (non-flash)
"""
for line in out.splitlines():
if len(line) == 0:
continue
line = line.replace(" (non-flash)", "")
l.append((hba_id, line.split()[-1]))
# gce
if self._get_model() == "Google":
try:
cmd = [
"gcloud", "compute", "regions", "list", "-q", "--format",
"json"
]
out, err, ret = justcall(cmd)
import json
from rcGlobalEnv import rcEnv
data = json.loads(out)
hba_id = rcEnv.nodename
for region in data:
i = region["selfLink"].index("/projects")
tgt_id = region["selfLink"][i:].replace("/projects",
"").replace(
"/regions", "")
l.append((hba_id, tgt_id))
except:
pass
return [{"hba_id": e[0], "tgt_id": e[1]} for e in l]
def cpu(self, d, day, start, end):
f = self.sarfile(day)
if f is None:
return [], []
cmd = [
'sar', '-t', '-u', 'ALL', '-P', 'ALL', '-f', f, '-s', start, '-e',
end
]
(buff, err, ret) = justcall(cmd)
if ret != 0:
cmd = [
'sar', '-t', '-u', '-P', 'ALL', '-f', f, '-s', start, '-e', end
]
(buff, err, ret) = justcall(cmd)
cols = []
lines = []
for line in buff.split('\n'):
l = line.split()
if 'Linux' in l:
continue
if len(l) == 7:
""" redhat 4
18:50:01 CPU %user %nice %system %iowait %idle
"""
cols = [
'date', 'cpu', 'usr', 'nice', 'sys', 'iowait', 'idle',
'nodename'
]
elif len(l) == 8:
""" redhat 5
05:20:01 CPU %user %nice %system %iowait %steal %idle
"""
cols = [
'date', 'cpu', 'usr', 'nice', 'sys', 'iowait', 'steal',
'idle', 'nodename'
]
elif len(l) == 11:
cols = [
'date', 'cpu', 'usr', 'nice', 'sys', 'iowait', 'steal',
'irq', 'soft', 'guest', 'idle', 'nodename'
]
elif len(l) == 12:
cols = [
'date', 'cpu', 'usr', 'nice', 'sys', 'iowait', 'steal',
'irq', 'soft', 'guest', 'gnice', 'idle', 'nodename'
]
else:
continue
if l[1] == 'CPU':
continue
if l[0] == 'Average:':
continue
l.append(self.nodename)
l[0] = '%s %s' % (d, l[0])
lines.append(l)
return cols, lines
def mem_u():
basedir = os.path.join(rcEnv.paths.pathvar, 'stats')
if not os.path.exists(basedir):
os.makedirs(basedir)
fname = os.path.join(basedir, 'mem_u%0.2d' % now.day)
if not os.path.exists(fname):
try:
f = open(fname, 'w')
except:
return
else:
mtime = os.stat(fname).st_mtime
if datetime.datetime.fromtimestamp(
mtime) < now - datetime.timedelta(days=1):
os.unlink(fname)
try:
f = open(fname, 'w')
except:
return
else:
try:
f = open(fname, 'a')
except:
return
cmd = ['/usr/sbin/sysctl', '-n', 'hw.pagesize']
(out, err, ret) = justcall(cmd)
if ret != 0:
return
pagesize = int(out.split()[0])
cmd = ['vm_stat']
(out, err, ret) = justcall(cmd)
if ret != 0:
return
h = {}
for line in out.split('\n'):
l = line.split(':')
if len(l) != 2:
continue
key = l[0]
try:
val = int(l[1].strip(' .'))
except:
continue
h[key] = val
f.write(' '.join((now.strftime('%H:%M:%S'),
str(h['Pages free'] * pagesize / 1024),
str(h['Pages active'] * pagesize / 1024),
str(h['Pages inactive'] * pagesize / 1024),
str(h['Pages speculative'] * pagesize / 1024),
str(h['Pages wired down'] * pagesize / 1024))) +
'\n')
def do_check(self):
if not which("lanscan"):
return []
cmd = ["lanscan", "-q"]
out, err, ret = justcall(cmd)
if ret != 0:
return self.undef
r = []
self.lag = {}
for line in out.split("\n"):
if len(line) == 0:
continue
l = line.split()
n = len(l)
if n < 2:
# not apa
continue
if self.has_inet(l[0]):
self.lag[l[0]] = l[1:]
cmd = ["lanscan", "-v"]
out, err, ret = justcall(cmd)
if ret != 0:
return self.undef
self.intf_status = {}
for line in out.split("\n"):
if 'ETHER' not in line or line.startswith('0x'):
continue
l = line.split()
n = len(l)
if n < 5 or not l[3].startswith('lan'):
continue
intf = l[3].replace('lan', '')
status = l[2]
self.intf_status[intf] = status
for intf, slaves in self.lag.items():
i = 0
for slave in slaves:
if slave in self.intf_status and self.intf_status[slave] == 'UP':
i += 1
inst = "lan" + intf + ".paths"
val = str(i)
r.append({
"instance": inst,
"value": val,
"path": '',
})
return r
def stats_meminfo(self):
"""
Memory sizes are store in MB.
Avails are percentages.
"""
cmd = [
"sysctl",
"hw.realmem",
"hw.pagesize",
"vm.stats.vm.v_page_count",
"vm.stats.vm.v_wire_count",
"vm.stats.vm.v_active_count",
"vm.stats.vm.v_inactive_count",
"vm.stats.vm.v_cache_count",
"vm.stats.vm.v_free_count",
"vm.swap_total",
]
raw_data = {}
data = {}
out, err, ret = justcall(cmd)
for line in out.splitlines():
key, val = line.split(":", 1)
val = val.strip()
raw_data[key] = int(val)
data["mem_total"] = raw_data["hw.realmem"] // 1024 // 1024
data["mem_avail"] = (raw_data["vm.stats.vm.v_inactive_count"] +
raw_data["vm.stats.vm.v_cache_count"] +
raw_data["vm.stats.vm.v_free_count"]
) * raw_data["hw.pagesize"] // 1024 // 1024
if data["mem_total"]:
data["mem_avail"] = 100 * data["mem_avail"] // data["mem_total"]
else:
data["mem_avail"] = 0
data["swap_total"] = raw_data["vm.swap_total"] // 1024 // 1024
cmd = ["pstat", "-T"]
out, err, ret = justcall(cmd)
for line in out.splitlines():
if "swap" in line:
swap_used = int(line.split("M/")[0])
break
if data["swap_total"]:
swap_avail = data["swap_total"] - swap_used
data["swap_avail"] = 100 * swap_avail // data["swap_total"]
else:
data["swap_avail"] = 0
return data
def docker_login(self, ref):
if "/" not in ref:
return
reg = ref.split("/")[0]
if reg == "docker.io":
return
try:
cmd = self.docker_cmd + ["login", reg
] + self.login_as_service_args()
except Exception:
self.svc.log.debug(
"skip registry login as service: node not registered")
return
justcall(cmd)
def __init__(self, node=None):
rcAsset.Asset.__init__(self, node)
self.osver = 0.
self.zone = is_zone()
(out, err, ret) = justcall(['prtdiag'])
if ret != 0 and len(out) < 4:
self.prtdiag = []
else:
self.prtdiag = out.split('\n')
(out, err, ret) = justcall(['prtconf'])
if ret != 0 and len(out) < 4:
self.prtconf = []
else:
self.prtconf = out.split('\n')
def has_pv(self, pv):
cmd = [rcEnv.syspaths.pvscan, "--cache", pv]
justcall(cmd)
cmd = [rcEnv.syspaths.pvs, "-o", "vg_name", "--noheadings", pv]
out, err, ret = justcall(cmd)
if ret != 0:
return False
out = out.strip()
if out == self.r.name:
self.r.log.info("pv %s is already a member of vg %s", pv,
self.r.name)
return True
if out != "":
raise ex.excError("pv %s in use by vg %s" % (pv, out))
return True
def do_check(self):
cmd = ["lanscan", "-q"]
out, err, ret = justcall(cmd)
if ret != 0:
return self.undef
r = []
intf = set()
for line in out.split("\n"):
if len(line) == 0:
continue
l = line.split()
n = len(l)
if n == 1:
# add interfaces with an inet config
if self.has_inet(l[0]):
intf.add(l[0])
elif n > 1:
# add slaves for apa with an inet config
if self.has_inet(l[0]):
for w in l[1:]:
intf.add(w)
else:
continue
for i in intf:
r += self.do_check_intf(i)
return r
def load_mpath_native(self):
cmd = [rcEnv.syspaths.multipath, '-l']
out, err, ret = justcall(cmd)
if ret != 0:
return
lines = out.split('\n')
if len(lines) == 0:
return
self.mpath_h = {}
regex = re.compile('[(]*[0-9a-f]*[)]*')
for line in lines:
if len(line) > 0 and \
line[0] not in (' ', '\\', '[', '`', '|'):
l = line.split()
if l[0].startswith("size="):
continue
wwid = None
for w in l:
w = w.strip("()")
if len(w) not in [17, 33]:
continue
if regex.match(w) is None:
continue
if w[0] in ("2,", "3", "5"):
wwid = w[1:]
elif " sd" in line:
l = line.split()
for i, w in enumerate(l):
if w.startswith('sd'):
dev = "/dev/" + w
self.mpath_h[dev] = wwid
def check_ping(addr, timeout=5, count=1):
ping = 'ping.exe'
cmd = [ping, '-n', repr(count), '-w', repr(timeout), addr]
out, err, ret = justcall(cmd)
if ret == 0:
return True
return False
def scan(self, lname):
vid = 'unknown'
pid = 'unknown'
wwid = 'unknown'
size = 'unknown'
cmd = ['lscfg', '-vpl', lname]
(ret, out, err) = call(cmd)
for f in out.split('\n'):
if "Manufacturer" in f:
vid = f.split('.')[-1]
if "Machine Type and Model" in f:
pid = f.split('.')[-1]
cmd = ['bootinfo', '-s', lname]
out, err, ret = justcall(cmd)
if ret == 0:
size = int(out.strip())
else:
size = 0
wwid = self.odmget(lname, 'ww_name').replace('0x', '')
if wwid == 'unknown':
wwid = self.get_vscsi_id(lname)
self.h[lname] = dict(vid=vid, pid=pid, wwid=wwid, size=size)
def gce_instance_data(self):
cmd = [
"gcloud", "compute", "instances", "describe", "-q", "--format",
"json", rcEnv.nodename
]
out, err, ret = justcall(cmd)
return json.loads(out)
def get_status(self, nodename=None):
"""
NAME TYPE STATE SUBTYPE ROOTPATH
iisdevs1 system maintenance private /var/hpsrp/iisdevs1
"""
cmd = ['srp', '-status', self.name]
if nodename is not None:
cmd = rcEnv.rsh.split() + [nodename] + cmd
out, err, ret = justcall(cmd)
if ret != 0:
raise ex.excError("srp -status returned %d:\n%s" % (ret, err))
lines = out.split('\n')
if len(lines) < 2:
raise ex.excError("srp -status output too short:\n%s" % out)
l = lines[1].split()
if l[0] != self.name:
raise ex.excError(
"srp -status second line, first entry does not match container name"
)
if len(l) != 5:
raise ex.excError("unexpected number of entries in %s" % str(l))
_type, _state, _subtype, _rootpath = l[1:]
return {
'type': l[1],
'state': l[2],
'subtype': l[3],
'rootpath': l[4],
}
def do_check(self):
cmd = [
rcEnv.syspaths.vgs, '--units', 'b', '--noheadings', '-o',
'vg_name,vg_size,vg_free'
]
(out, err, ret) = justcall(cmd)
if ret != 0:
return self.undef
lines = out.split('\n')
if len(lines) < 1:
return self.undef
r = []
for line in lines:
l = line.split()
if len(l) != 3:
continue
size = int(l[1].replace('B', ''))
free = int(l[2].replace('B', ''))
val = int(100 * (size - free) / size)
r.append({
"instance": l[0],
"value": str(val),
"path": self.find_svc(l[0]),
})
return r
def provisioner(self):
if not which('vgdisplay'):
self.r.log.error("vgdisplay command not found")
raise ex.excError
if not which('lvcreate'):
self.r.log.error("lvcreate command not found")
raise ex.excError
self.size = self.r.oget("size")
self.size = convert_size(self.size, _to="m")
self.vg = self.r.oget("vg")
cmd = ['vgdisplay', self.vg]
out, err, ret = justcall(cmd)
if ret != 0:
self.r.log.error("volume group %s does not exist" % self.vg)
raise ex.excError
dev = os.path.basename(self.r.device)
# create the logical volume
cmd = ['lvcreate', '-n', dev, '-L', str(self.size) + 'M', self.vg]
ret, out, err = self.r.vcall(cmd)
if ret != 0:
raise ex.excError
self.r.svc.node.unset_lazy("devtree")
def cleanup_link(self, link):
cmd = ["ip", "link", "del", "dev", link]
out, err, ret = justcall(cmd)
if ret == 0:
self.log.info(" ".join(cmd))
else:
self.log.debug(" ".join(cmd) + out + err)
def do_check(self):
cmd = ['df', '-lP']
(out, err, ret) = justcall(cmd)
if ret != 0:
return self.undef
lines = out.split('\n')
if len(lines) < 2:
return self.undef
r = []
for line in lines[1:]:
l = line.split()
if len(l) != 6:
continue
# discard bind mounts: we get metric from the source anyway
if l[0].startswith('/') and not l[0].startswith(
'/dev') and not l[0].startswith('//'):
continue
if l[5].startswith('/Volumes'):
continue
if l[5].startswith('/run'):
continue
if l[5].startswith('/sys/'):
continue
if l[5] == "/dev/shm":
continue
if "osvc_sync_" in l[0]:
# do not report osvc sync snapshots fs usage
continue
r.append({
"instance": l[5],
"value": l[4],
"path": self.find_svc(l[5]),
})
return r
def _cmd(self, cmd, target, info=False):
if target == "local":
filer = self.local()
elif target == "master":
filer = self.master()
elif target == "slave":
filer = self.slave()
elif target in self.filers.values():
filer = target
else:
raise ex.excError("unable to find the %s filer" % target)
_cmd = rcEnv.rsh.split() + [self.user + '@' + filer] + cmd
if info:
self.log.info(' '.join(_cmd))
out, err, ret = justcall(rcEnv.rsh.split() +
[self.user + '@' + filer] + cmd)
if info:
if len(out) > 0:
self.log.info(out)
if len(err) > 0:
self.log.error(err)
return ret, out, err
def loop_to_file(f):
"""
Given a loop dev, returns the loop file associated. For example,
/dev/loop0 => /path/to/file
"""
data = losetup_data()
if data:
for _data in data:
if _data["name"] == f:
return _data["back-file"]
return
out, err, ret = justcall([rcEnv.syspaths.losetup, f])
if len(out) == 0:
return
for line in out.split('\n'):
l = line.split('(')
if len(l) == 0:
continue
fpath = l[-1].rstrip(")")
if len(fpath) == 0:
continue
if not os.path.exists(fpath):
continue
return fpath
def file_to_loop(f):
"""
Given a file path, returns the loop device associated. For example,
/path/to/file => /dev/loop0
"""
data = losetup_data()
if data:
return [_data["name"] for _data in data if _data["back-file"] == f]
out, err, ret = justcall([rcEnv.syspaths.losetup, '-j', f])
if len(out) == 0:
return []
# It's possible multiple loopdev are associated with the same file
devs = []
for line in out.split('\n'):
l = line.split(':')
if len(l) == 0:
continue
if len(l[0]) == 0:
continue
if not os.path.exists(l[0]):
continue
devs.append(l[0])
return devs
def losetup_data():
cmd = ["losetup", "-J"]
out, err, ret = justcall(cmd)
try:
return json.loads(out)["loopdevices"]
except ValueError:
return
def get_verbose_list(self):
"""
Name: iisdevs1 Template: system Service: provision ID: 1
----------------------------------------------------------------------
autostart=0
srp_name=iisdevs1
...
"""
cmd = ['srp', '-list', self.name, '-v']
out, err, ret = justcall(cmd)
if ret != 0:
raise ex.excError("srp -list returned %d:\n%s" % (ret, err))
data = {}
words = out.split()
for i, w in enumerate(words):
if w == "Service:":
service = words[i + 1]
if '=' in w:
key = service + '.' + w[:w.index('=')]
val = w[w.index('=') + 1:]
data[key] = val
return data
def fs_u_zfs():
if not which(rcEnv.syspaths.zfs):
return []
cmd = [
rcEnv.syspaths.zfs, 'list', '-o', 'name,used,avail,mountpoint',
'-H'
]
(out, err, ret) = justcall(cmd)
if ret != 0:
return []
lines = out.split('\n')
if len(lines) == 0:
return []
vals = []
for line in lines:
l = line.split()
if len(l) != 4:
continue
if "@" in l[0]:
# do not report clone usage
continue
if "osvc_sync_" in l[0]:
# do not report osvc sync snapshots fs usage
continue
used = convert_size(l[1], _to="KB")
if l[2] == '0':
l[2] = '0K'
avail = convert_size(l[2], _to="KB")
total = used + avail
pct = used / total * 100
vals.append([now, node.nodename, l[0], str(total), str(pct)])
return vals
