def collect_message(destroot):
"""Get messages on the host."""
dmesg = subproc.check_output([_DMESG])
with io.open('%s/dmesg' % destroot, 'wb') as f:
f.write(dmesg.encode(encoding='utf8', errors='replace'))
messages = subproc.check_output([_TAIL, '-n', '100', '/var/log/messages'])
dest_messages = '%s/var/log/messages' % destroot
fs.mkdir_safe(os.path.dirname(dest_messages))
with io.open(dest_messages, 'wb') as f:
f.write(messages.encode(encoding='utf8', errors='replace'))
def collect_message(destroot):
"""Get messages on the host."""
dmesg = subproc.check_output([_DMESG])
with open('%s/dmesg' % destroot, 'w') as f:
f.write(dmesg)
messages = subproc.check_output([_TAIL, '-n', '100', '/var/log/messages'])
dest_messages = '%s/var/log/messages' % destroot
if not os.path.exists(os.path.dirname(dest_messages)):
os.makedirs(os.path.dirname(dest_messages))
with open(dest_messages, 'w') as f:
f.write(messages)
def collect_localdisk(approot, destroot):
"""Get host local disk information."""
src = '%s/localdisk_svc' % approot
dest = '%s%s' % (destroot, src)
try:
shutil.copytree(src, dest)
except (shutil.Error, OSError):
_LOGGER.warning('skip %s => %s', src, dest)
vg_info = subproc.check_output([_LVM, _VGDISPLAY, 'treadmill'])
lv_info = subproc.check_output([_LVM, _LVDISPLAY, 'treadmill'])
with io.open('%s/lvm' % destroot, 'w') as f:
f.write('%s\n%s' % (vg_info, lv_info))
def add_raw_rule(table, chain, rule, safe=False):
"""Adds rule to a fiven table/chain.
:param ``str`` table:
Name of the table where the chain resides.
:param ``str`` chain:
Name of the chain where to insert the rule.
:param ``str`` rule:
Raw iptables rule in the same format as "iptables -S"
:param ``bool`` safe:
Query iptables prior to adding to prevent duplicates
"""
add_cmd = ['iptables', '-t', table, '-A', chain] + rule.split()
_LOGGER.info('%s', add_cmd)
if safe:
# Check if the rule already exists, and if it is, do nothing.
list_cmd = ['iptables', '-t', table, '-S', chain]
_LOGGER.info('%s', list_cmd)
lines = [
line.strip()
for line in subproc.check_output(list_cmd).splitlines()
]
match = '-A %s %s' % (chain, rule)
if match in lines:
return
subproc.check_call(add_cmd)
def blk_fs_info(block_dev):
"""Returns blocks group information for the filesystem present on
block_dev.
:param block_dev:
Block device for the filesystem info to query.
:type block_dev:
``str``
:returns:
Blocks group information.
:rtype:
``dict``
"""
res = {}
# TODO: it might worth to convert the appropriate values to int, date etc.
# in the result.
try:
output = subproc.check_output(['dumpe2fs', '-h', block_dev])
except subproc.CalledProcessError:
return res
for line in output.split(os.linesep):
if not line.strip():
continue
key, val = line.split(':', 1)
res[key.lower()] = val.strip()
return res
def _get_current_snat_rules(chain):
"""Extract all SNAT rules in chain from iptables.
:param ``str`` chain:
Iptables chain to process.
:returns:
``set([SNATRule])`` -- Set of rules.
"""
if chain is None:
chain = POSTROUTING_SNAT
rules = set()
iptables_cmd = ['iptables', '-t', 'nat', '-S', chain]
for line in subproc.check_output(iptables_cmd).splitlines():
snat_match = _SNAT_RULE_RE.match(line.strip())
if snat_match:
data = snat_match.groupdict()
rule = firewall.SNATRule(proto=data['proto'],
src_ip=data['src_ip'],
src_port=data['src_port'],
dst_ip=data['dst_ip'],
dst_port=data['dst_port'],
new_ip=data['new_ip'],
new_port=data['new_port'])
rules.add(rule)
continue
return rules
def last(rrdfile, rrdtool=RRDTOOL, rrd_socket=SOCKET):
"""
Returns the UNIX timestamp of the most recent update of the RRD.
"""
epoch = subproc.check_output(
[rrdtool, 'last', '--daemon',
'unix:%s' % rrd_socket, rrdfile])
return epoch.strip()
def first(rrdfile, rrdtool=RRDTOOL, rrd_socket=SOCKET):
"""
Returns the UNIX timestamp of the first data sample entered into the RRD.
"""
epoch = subproc.check_output(
[rrdtool, 'first', '--daemon',
'unix:%s' % rrd_socket, rrdfile])
return epoch.strip()
def blk_uuid(block_dev):
"""Get device uuid
"""
output = subproc.check_output(['blkid', block_dev])
match_obj = _UUID_RE.match(output)
if match_obj is None:
raise ValueError('Invalid device: %s' % block_dev)
else:
return match_obj.group(1)
def get_state(svcroot):
"""Checks the state of services under given root."""
services = glob.glob(os.path.join(svcroot, '*'))
services_state = {}
for service in services:
state = subproc.check_output(['s6_svstat', service])
services_state[os.path.basename(service)] = _parse_state(state)
return services_state
def last(rrdfile, rrdtool=RRDTOOL, rrd_socket=SOCKET, exec_on_node=True):
"""
Returns the UNIX timestamp of the most recent update of the RRD.
"""
if exec_on_node:
epoch = subproc.check_output([rrdtool, 'last', '--daemon',
'unix:%s' % rrd_socket, rrdfile])
else:
epoch = subprocess.check_output([rrdtool, 'last', rrdfile])
return epoch.decode().strip()
def collect_network(approot, destroot):
"""Get host network information."""
src = '%s/network_svc' % approot
dest = '%s%s' % (destroot, src)
try:
shutil.copytree(src, dest)
except (shutil.Error, OSError):
_LOGGER.exception('fail to copy %s => %s', src, dest)
ifconfig = subproc.check_output([_IFCONFIG])
with open('%s/ifconfig' % destroot, 'w') as f:
f.write(ifconfig)
def collect_network(approot, destroot):
"""Get host network information."""
src = '%s/network_svc' % approot
dest = '%s%s' % (destroot, src)
try:
shutil.copytree(src, dest)
except (shutil.Error, OSError):
_LOGGER.warning('skip %s => %s', src, dest)
ifconfig = subproc.check_output([_IFCONFIG])
with io.open('%s/ifconfig' % destroot, 'wb') as f:
f.write(ifconfig.encode(encoding='utf8', errors='replace'))
def _run(command):
"""Run command."""
success = False
try:
output = subproc.check_output(command, )
_LOGGER.info('Success.')
success = True
except subproc.CalledProcessError as err:
output = str(err)
_LOGGER.error('Failure.')
cli.out(output)
return success
def _read_mounted_cgroups():
"""Read all the currently mounted cgroups and their mount points."""
try:
res = subproc.check_output(['lssubsys', '-M'])
except subprocess.CalledProcessError as err:
if err.returncode == 255:
_LOGGER.warning('old version of lssubsys.')
res = subproc.check_output(['lssubsys', '-m'])
else:
raise
if res is not None:
res = res.strip()
cgroups_info = {}
for cginfo in res.split('\n'):
cgname, cgmount = cginfo.split(' ', 1)
if ',' in cgname:
for tied_cg in cgname.split(','):
cgroups_info.setdefault(tied_cg, []).append(cgmount)
else:
cgroups_info.setdefault(cgname, []).append(cgmount)
return cgroups_info
def blk_uuid(block_dev):
"""Get device uuid.
:returns:
``str | None`` - Device UUID or None if it does not have one.
"""
try:
output = subproc.check_output(['blkid', block_dev])
except subproc.CalledProcessError:
_LOGGER.warning('Device %r does not have a UUID', block_dev)
return None
match_obj = _UUID_RE.search(output)
if match_obj is None:
raise ValueError('Invalid device: %s' % block_dev)
return match_obj.group(1)
def _add_output(archive, command):
"""Add output of command to the archive."""
tarinfo = tarfile.TarInfo(os.path.join('diag', '#'.join(command)))
tarinfo.mtime = time.time()
try:
output = subproc.check_output(command).encode(encoding='utf8',
errors='replace')
stream = six.BytesIO(output)
tarinfo.size = len(output)
except subprocess.CalledProcessError as exc:
output = str(exc).encode(encoding='utf8', errors='replace')
stream = six.BytesIO(output)
tarinfo.size = len(output)
stream.seek(0)
archive.addfile(tarinfo, fileobj=stream)
def _port_range_windows():
"""Returns local port range."""
cmd = 'netsh.exe int ipv4 show dynamicport tcp'
output = subproc.check_output(cmd).split('\r\n')
low = 0
ports = 0
for line in output:
if line.lower().startswith('start port'):
low = int((line).split(':')[1])
elif line.lower().startswith('number of ports'):
ports = int((line).split(':')[1])
high = ports - low + 1
return low, high
def get_json_metrics(rrdfile, timeframe, rrdtool=RRDTOOL, rrd_socket=SOCKET):
"""Return the metrics in the rrd file as a json string."""
_LOGGER.info('Get the metrics in JSON for %s', rrdfile)
# the command sends a FLUSH to the rrdcached implicitly...
cmd = [
rrdtool,
'graph',
'-',
'--daemon',
'unix:%s' % rrd_socket,
'--imgformat',
'JSONTIME',
'--start=%s' % first(rrdfile, timeframe),
'--end=%s' % last(rrdfile),
# mem usage
'DEF:memory_usage=%s:memory_usage:MAX' % rrdfile,
'LINE:memory_usage:memory usage',
# mem hardlimit
'DEF:memory_hardlimit=%s:memory_hardlimit:MAX' % rrdfile,
'LINE:memory_hardlimit:memory limit',
# cpu usage
'DEF:cpu_usage=%s:cpu_usage:AVERAGE' % rrdfile,
'LINE:cpu_usage:cpu usage',
# cpu ratio
'DEF:cpu_ratio=%s:cpu_ratio:AVERAGE' % rrdfile,
'LINE:cpu_ratio:cpu ratio',
# blk read
'DEF:blk_read_iops=%s:blk_read_iops:MAX' % rrdfile,
'LINE:blk_read_iops:read iops',
# blk write
'DEF:blk_write_iops=%s:blk_write_iops:MAX' % rrdfile,
'LINE:blk_write_iops:write iops',
# blk read
'DEF:blk_read_bps=%s:blk_read_bps:MAX' % rrdfile,
'LINE:blk_read_bps:read bps',
# blk write
'DEF:blk_write_bps=%s:blk_write_bps:MAX' % rrdfile,
'LINE:blk_write_bps:write bps',
# fs_used_bytes
'DEF:fs_used_bytes=%s:fs_used_bytes:MAX' % rrdfile,
'LINE:fs_used_bytes:used bytes'
]
return subproc.check_output(cmd)
def first(rrdfile, timeframe, rrdtool=RRDTOOL, rrd_socket=SOCKET,
exec_on_node=True):
"""
Returns the UNIX timestamp of the first data sample entered into the RRD.
"""
try:
rra_idx = TIMEFRAME_TO_RRA_IDX[timeframe]
except KeyError:
rra_idx = TIMEFRAME_TO_RRA_IDX['short']
if exec_on_node:
epoch = subproc.check_output([rrdtool, 'first', rrdfile,
'--daemon', 'unix:%s' % rrd_socket,
'--rraindex', rra_idx])
else:
epoch = subprocess.check_output([rrdtool, 'first', rrdfile,
'--rraindex', rra_idx])
return epoch.decode().strip()
def _iptables_output(table, action, chain):
"""Invoke iptables with the given table, action and chain and capture its
output.
"""
cmd = ['iptables', '-t', table, action, chain]
_LOGGER.debug('%r', cmd)
while True:
try:
res = subproc.check_output(cmd)
break
except subproc.CalledProcessError as err:
if err.returncode == _IPTABLES_EXIT_LOCKED:
_LOGGER.debug('xtable locked, retrying.')
# table locked, spin and try again
time.sleep(random.uniform(0, 1))
else:
raise
return res
def _publish_ticket(tkt_file):
"""Publish ticket details."""
if tkt_file.startswith('.'):
return
if not any([tkt_file.endswith(realm) for realm in realms]):
_LOGGER.info('Ignore tkt_file: %s', tkt_file)
return
try:
tkt_details = subproc.check_output(
['klist', '-5', '-e', '-f', tkt_file])
tkt_node = z.path.tickets(os.path.basename(tkt_file),
self.hostname)
zkutils.put(self.zkclient,
tkt_node,
tkt_details,
ephemeral=True)
except subproc.CalledProcessError:
_LOGGER.warning('Unable to get tickets details.')
def lastupdate(rrdfile, rrdtool=RRDTOOL, rrd_socket=SOCKET):
"""Get lastupdate metric"""
last_udpate = subproc.check_output([rrdtool, 'lastupdate', '--daemon',
'unix:%s' % rrd_socket, rrdfile])
[titles, _empty, data_str] = last_udpate.strip().split('\n')
(timestamp, value_str) = data_str.split(':')
values = value_str.strip().split(' ')
result = {'timestamp': int(timestamp)}
for idx, title in enumerate(titles.strip().split(' ')):
try:
result[title] = int(values[idx])
except ValueError:
# can not be convert to int
try:
result[title] = float(values[idx])
except ValueError:
# it is possible value is 'U'
result[title] = 0
return result
def _get_current_passthrough_rules(chain):
"""Extract all PassThrough rules from iptables.
:param ``str`` chain:
Iptables chain to process.
:returns:
``set([PassThroughRule])`` -- Set of rules.
"""
rules = set()
if chain is None:
chain = PREROUTING_PASSTHROUGH
iptables_cmd = ['iptables', '-t', 'nat', '-S', chain]
for line in subproc.check_output(iptables_cmd).splitlines():
match = _PASSTHROUGH_RULE_RE.match(line.strip())
if match:
data = match.groupdict()
rule = firewall.PassThroughRule(src_ip=data['src_ip'],
dst_ip=data['dst_ip'])
rules.add(rule)
return rules
def loop_dev_for(filename):
"""Lookup the loop device associated with a given filename.
:param filename:
Name of the file
:type filename:
``str``
:returns:
Name of the loop device or None if not found
:raises:
subproc.CalledProcessError if the file doesn't exist
"""
filename = os.path.realpath(filename)
loop_dev = subproc.check_output(['losetup', '-j', filename])
loop_dev = loop_dev.strip()
match = re.match(
r'^(?P<loop_dev>[^:]+):.*\({fname}\)'.format(fname=filename), loop_dev)
if match is not None:
loop_dev = match.groupdict()['loop_dev']
else:
loop_dev = None
return loop_dev
def collect_sysctl(destroot):
"""Get host sysctl (related to kernel)."""
sysctl = subproc.check_output([_SYSCTL, '-a'])
with io.open('%s/sysctl' % destroot, 'wb') as f:
f.write(sysctl.encode(encoding='utf8', errors='replace'))
def get_pid(app_root, service):
"""Returns pid of the service or None if the service is not running."""
output = subproc.check_output(
['s6_svstat', os.path.join(app_root, service)])
return _parse_state(output).get('pid', None)
def collect_sysctl(destroot):
"""Get host sysctl (related to kernel)."""
sysctl = subproc.check_output([_SYSCTL, '-a'])
with open('%s/sysctl' % destroot, 'w+') as f:
f.write(sysctl)