def __init__(self, ps_data):
"""Construct the Posix thread object.
:param ps_data: The data with which to initialise this object, as generated by execute_ps().
:type ps_data: dict
:return: None
:rtype: None
"""
super(PosixProcess, self).__init__(ps_data)
self.threads = dict() # A process is basically a container of threads
if 'threads' in ps_data:
try:
for children in ps_data['threads']:
try:
if children['pid'] == children['lwp']:
thread = PosixProcess(children)
else:
thread = PosixThread(children)
self.threads[thread.lwp] = thread
except KeyError:
logger.warning('{0} is not valid thread/process data'.format(children))
except Exception as e:
logger.warning('Invalid thread information {0}, ignored.'.format(ps_data['threads']))
def execute_ps():
"""Execute the Linux 'ps' command and return the parsed result.
:return: The parsed result from the ps command, basically a list of thread/process stats objects,
PosixThread or PosixProcess.
:rtype: [PosixThread]
"""
global _ps_args
ps_proc = subprocess.Popen(_ps_args, stdout=subprocess.PIPE, close_fds=True)
try:
ps_out = ps_proc.communicate()
ps_stdout = str(ps_out[0]) # Using str() to convert the strings '\n' to actual new lines
matches = _ps_pattern.finditer(ps_stdout)
if matches:
ret = []
x = dict()
for match in matches:
raw_data = match.groups()
ps_data = dict(zip(['user', 'pid', 'ppid', 'lwp', 'pcpu', 'pmem', 'vsz', 'rss', 'comm'], raw_data))
if ps_data['pid'] == ps_data['lwp']:
ret.append(PosixProcess(ps_data))
else:
ret.append(PosixThread(ps_data))
return ret
else:
logger.warning("Didn't find any matches for the ps output {0}".format(ps_stdout))
finally:
ps_proc.wait() # Preventing zombies
return None
def format_process(level, proc, format):
"""Create a printable string representing a Posix process/thread object, recursively.
The resulting string conforms to the JSON format.
:param level: The indentation level
:type level: int
:param proc: The process object to be represented
:type proc: com.mitel.pyrate.resmon_util.PosixThread
:param format: The output format, must be one of 'json', 'pretty', or 'xml'
:type format: str
:return: The printable string representing the given process object
:rtype: str
"""
if format == 'json':
return json.dumps(proc.export_data(), indent=2)
elif format == 'xml':
if isinstance(proc, PosixThread):
top = proc.export_xml()
return ElementTree.tostring(top)
logger.warning('{0} is neither a process nor thread'.format(proc))
return None
else: # Compact pretty print
tabs = ''
if level:
tabs += (' ' * level)
ret = [tabs, '{']
is_thread = False
if proc.lwp != proc.pid:
is_thread = True
ret.append('lwp:')
ret.append(repr(proc.lwp))
else:
ret.append('user:'******', pid:')
ret.append(repr(proc.pid))
ret.append(', %cpu:{0:6.2f}, %mem:{1:6.2f}, rss:{2}, comm:{3}'.format(proc.pcpu, proc.pmem, repr(proc.rss),
repr(proc.cmd)))
if isinstance(proc, PosixProcess):
if proc.threads:
ret.append(', threads:[\n')
for pid, subproc in proc.threads.items():
ret.append(format_process(level + 1, subproc, format) + ',\n')
ret.append(tabs)
ret.append(']')
ret.append('}')
return ''.join(ret)
def __call__(self):
"""The run entry for the worker thread."""
try:
self.run() # Sub-classes must implement this.
except IOError as e:
logger.warning('{0} encountered {1} at {2}'.format(repr(self), e, traceback.format_tb(sys.exc_info()[2])))
except BaseException as e:
logger.error("{0} encountered unexpected {1} at {2}".format(repr(self), e, traceback.format_tb(sys.exc_info()[2])))
finally:
# Make sure everything is properly reset
with self._lock:
self._runFlag = False
self.stopWaiting.set()
def build_process_tree(pthreads):
"""Build a process tree from a given list of threads, recursively.
:param pthreads: The list of threads to be processed, the kind generated by execute_ps()
:type pthreads: list
:return: The dictionary of processes, keyed by PIDs, created from the given thread list.
:rtype: dict
"""
processes = dict()
# Start by building the process dictionary ...
for pthread in pthreads:
if isinstance(pthread, PosixProcess):
processes[pthread.pid] = pthread
# ... Then add the threads into the processes ...
for pthread in pthreads:
if pthread.pid == pthread.lwp: # Skip processes
continue
try:
process = processes[pthread.pid]
process.add_thread(pthread)
except KeyError as e:
logger.warning('Cannot find process {0} for thread {1}'.format(pthread.pid, pthread))
# ... Finally build the process trees and the return data
ret = dict() # The return is a dictionary of the root processes, which, in turn, contain all the other processes.
for pid, proc in processes.items():
if proc.ppid == 0: # Root process, add it to the return set
ret[proc.pid] = proc
continue
# If we are here then the process is not the root, and must therefore have a parent.
try:
parent = processes[proc.ppid]
parent.add_thread(proc)
except KeyError:
logger.warning('Cannot find parent process {0} for {1}'.format(proc.ppid, proc))
return ret
def main(args):
"""Main entry to program.
:param args: Parsed command-line args.
:return: Exit code.
"""
try:
start_ts = (int(time.time()) / _step) * _step # TS should be aligned with the period
# Step 1: Start the monitoring
if (not os.path.exists(_base_name + 'cpu.rrd')) or (not os.path.exists(_base_name + 'mem.rrd')):
fill_in_the_rrds(_base_name + 'cpu.rrd', _base_name + 'mem.rrd', start_ts - (24 * 3600), start_ts, _step)
else:
fill_in_the_rrds(_base_name + 'cpu.rrd', _base_name + 'mem.rrd',
int(rrdutil.get_last_update(_base_name + 'cpu.rrd')) + _step, start_ts, _step)
monitor = resmon_util.start_monitor(_base_name, _step)
try:
prev_ts = start_ts
for loop in range(0, 360): # Run for an hour (360 loops of 10 seconds).
# Step 2: Wait for some time (10s) for the monitor to gather data
logger.info('Waiting for data, loop #{0}'.format(loop))
time.sleep(10)
# Step 3: Observe the result
last_cpu_data = rrdutil.fetch_last_data(monitor.get_cpu_rrd(), start_ts=prev_ts, resolution=1)
last_mem_data = rrdutil.fetch_last_data(monitor.get_mem_rrd(), start_ts=prev_ts)
# Print out the result
print('\nLAST CPU data {0}, curr time = {1}'.format(last_cpu_data[0], int(time.time())))
print(rrdutil.format_cpu_stats(last_cpu_data))
print('\nLAST Memory data {0}'.format(last_mem_data[0]))
print(rrdutil.format_mem_stats(last_mem_data))
prev_ts = (int(time.time()) / _step) * _step # TS should be aligned with the period
avg_data = rrdutil.fetch_avg_data(monitor.get_cpu_rrd(), start_ts=start_ts, resolution=10)
print('\nAVG CPU data {0}'.format(avg_data[0]))
print(rrdutil.format_cpu_stats(avg_data))
avg_data = rrdutil.fetch_avg_data(monitor.get_mem_rrd(), start_ts=start_ts, resolution=10)
print('\nAVG Memory data {0}'.format(avg_data[0]))
print(rrdutil.format_mem_stats(avg_data))
finally:
# Don't forget to stop the monitor when finish
logger.info('Stopping {0}'.format(monitor))
monitor.stop()
# Generate a graph
end_graf = time.time()
start_graf = end_graf - (2 * SECONDS_IN_HOUR)
gen_graph('cpu.png', rrd_name=monitor.get_cpu_rrd(), title='CPU Utilisation', cf='AVERAGE',
start_ts=start_graf, end_ts=end_graf,
dataset=[('idle', '%Idle'), ('wait', '%Wait'), ('load', 'Load avg')])
gen_graph('mem.png', rrd_name=monitor.get_mem_rrd(), title='Memory Utilisation', cf='AVERAGE',
start_ts=start_graf, end_ts=end_graf,
dataset=[('memtotal', 'Total'), ('memfree', 'Free'), ('buffers', 'Buffers'),
('cached', 'Cached')])
except (TypeError, ValueError) as e:
logger.warning('Encountered invalid monitor period {0}, at {1}'.format(e, format_tb(sys.exc_info()[2])))
except KeyboardInterrupt:
logger.warning('User interruption at {0}'.format(format_tb(sys.exc_info()[2])))
except BaseException as e:
logger.error('Encountered unexpected exception {0} at {1}'.format(repr(e), format_tb(sys.exc_info()[2])))
def main(args):
"""Main entry to the gen_graph command.
:param args: Command line argument list. A list of strings.
:type args: list
:return: The return code for the command, 0 is success, anything else for failure
:rtype: int
"""
exit_code = 0 # Default to success
parser = None
try:
# Parse the given argument list
parser = argparse.ArgumentParser(prog='fetch', description='Fetch some resource utilisation data from a resmon DB')
parser.add_argument('-s', metavar='start_time', help='The start time, default to 24h before the end time',
type=int)
parser.add_argument('-e', metavar='end_time', help='The end time, default to now', type=int)
parser.add_argument('-c', help='Consolidation function: ave | last. Default to ave', type=str,
choices=['ave', 'last'], default='ave')
parser.add_argument('-r', help='Resolution of the requested data, e.g. 1s, 30s, 1m, ...', type=str, required=True)
parser.add_argument('-p', metavar='period/timeframe', help='If a start time is not specified then this will '
'specify the period of time against which the graph will cover. E.g. 23s, 5m, 6h, 1d, 2w',
type=str, default='1d')
parser.add_argument('-f', metavar='RRD_file', help='Name of the RRD file', type=str, required=True)
arg = parser.parse_args(args=args)
# Calculating the end timestamp
end_ts = int(time.time())
if arg.e: # End time
end_ts = int(arg.e)
# Calculating the start timestamp
start_ts = end_ts - (24 * 3600) # Default start time is one day old data
if arg.s: # Start time
start_ts = int(arg.s)
elif arg.p: # Period is meaningful only when start time is not specified
start_ts = end_ts - parse_timespec(arg.p)
ret = []
# Calculating the consolidation function
if arg.c == 'last':
ret += fetch_last_data(arg.f, start_ts=start_ts, end_ts=end_ts, resolution=parse_timespec(arg.r))
else:
ret += fetch_avg_data(arg.f, start_ts=start_ts, end_ts=end_ts, resolution=parse_timespec(arg.r))
if len(ret) > 2:
curr_ts = ret[0][0]
inc = int(ret[0][2])
print(rrd_fields_to_csv('ts', ret[1]))
for line in ret[2]:
print(rrd_fields_to_csv(curr_ts, line))
curr_ts += inc
except KeyboardInterrupt:
tb = sys.exc_info()[2]
logger.warning('User interruption at {0}'.format(format_tb(tb)))
exit_code = errno.EINTR
except ValueError as e:
tb = sys.exc_info()[2]
logger.warning('Encountered {0} at {1}'.format(e, format_tb(tb)))
exit_code = errno.EINVAL
except Exception as e:
tb = sys.exc_info()[2]
logger.warning('Encountered unexpected {0} at {1}'.format(e, format_tb(tb)))
exit_code = errno.EINVAL
return exit_code
def main(args):
"""Main entry to the gen_graph command.
:param args: Command line argument list. A list of strings.
:type args: list
:return: The return code for the command, 0 is success, anything else for failure
:rtype: int
"""
exit_code = 0 # Default to success
parser = None
try:
# Parse the given argument list
parser = argparse.ArgumentParser(prog="gengraph.py", description="Generate resource utilisation graphs")
parser.add_argument("image", metavar="img_file", help="Name of the image file", type=str)
parser.add_argument(
"-d",
metavar="data_set",
help="Space delimited dataset to be included in the graph",
type=str,
required=True,
)
parser.add_argument(
"-s", metavar="start_time", help="The start time, default to 24h before the end time", type=int
)
parser.add_argument("-e", metavar="end_time", help="The end time, default to now", type=int)
parser.add_argument(
"-c",
help="Consolidation function: ave | last. Default to ave",
type=str,
choices=["ave", "last"],
default="ave",
)
parser.add_argument(
"-p",
metavar="period/timeframe",
help="If a start time is not specified then this will "
"specify the period of time against which the graph will cover. E.g. 23s, 5m, 6h, 1d, 2w",
type=str,
default="1d",
)
parser.add_argument("-f", metavar="RRD_file", help="Name of the RRD file", type=str, required=True)
parser.add_argument(
"-x",
metavar="width",
help="The width of the graph, in number of pixels (default=640)",
type=int,
default=640,
)
parser.add_argument(
"-y",
metavar="height",
help="The height of the graph, in number of pixels (default=480)",
type=int,
default=480,
)
arg = parser.parse_args(args=args)
# Calculating the end timestamp
end_graf = int(time.time())
if arg.e: # End time
end_graf = arg.e
# Calculating the start timestamp
start_graf = end_graf - (24 * 3600) # Default start time is one day old data
if arg.s: # Start time
start_graf = arg.s
elif arg.p: # Period is meaningful only when start time is not specified
start_graf = end_graf - parse_timespec(arg.p)
# Calculating the consolidation function
cf = "AVERAGE"
if arg.c == "last":
cf = "LAST"
execute_graph_request(
rrd_name=arg.f,
data_set=arg.d.split(),
image_file=arg.image,
start_ts=start_graf,
end_ts=end_graf,
consolidation_func=cf,
width=arg.x,
height=arg.y,
)
except KeyboardInterrupt:
tb = sys.exc_info()[2]
logger.warning("User interruption at {0}".format(format_tb(tb)))
exit_code = errno.EINTR
except ValueError as e:
tb = sys.exc_info()[2]
logger.warning("Encountered {0} at {1}".format(e, format_tb(tb)))
exit_code = errno.EINVAL
except Exception as e:
tb = sys.exc_info()[2]
logger.warning("Encountered unexpected {0} at {1}".format(e, format_tb(tb)))
exit_code = errno.EINVAL
return exit_code