def click(x, notify=False, pid=None, pids=None, webdriver=None, window_name=None, debug=False):
if debug:
print("[beryl] starting click")
print("\tpid: " + str(pid))
print("\twebdriver: " + str(webdriver))
print("\twindow_name: " + str(window_name))
print("\tstr(type(webdriver)): " + str(type(webdriver)))
type_as_string = str(type(x))
webdriver_type_as_string = str(type(webdriver))
if webdriver_type_as_string == "<class 'selenium.webdriver.firefox.webdriver.WebDriver'>":
pids = [webdriver.binary.process.pid]
elif webdriver_type_as_string == "<class 'selenium.webdriver.chrome.webdriver.WebDriver'>":
process = Process(webdriver.service.process.pid)
if hasattr(process, "children"):
pids = [p.pid for p in process.children()]
elif hasattr(process, "get_children"):
pids = [p.pid for p in process.get_children()]
if isinstance(x, str) or isinstance(x, unicode):
if x.endswith(".png") or x.endswith(".jpg"):
click_image(x, notify=notify)
else:
click_text(x, notify=notify, pids=pids, window_name=window_name, debug=debug)
elif isinstance(x, PngImageFile):
click_image(x,notify=notify)
elif isinstance(x, tuple):
click_location(x,notify=notify)
def end_broker_process(self):
try:
broker_process = Process(self.broker.pid)
except NoSuchProcess:
return # was killed
# get stdout and stderr
select_config = [self.broker.stdout, self.broker.stderr], [], [], 0.1
stdout, stderr = [], []
result = select.select(*select_config)
while any(result):
if result[0]:
stdout.append(result[0][0].readline())
if result[1]:
stderr.append(result[1][0].readline())
result = select.select(*select_config)
if stdout and DEBUG_STDOUT:
_print_debug('STDOUT', ''.join(stdout))
if stderr and DEBUG_STDERR:
_print_debug('STDERR', ''.join(stderr))
# kill main process and its children
children = [process.pid for process in broker_process.get_children()]
_kill(self.broker.pid, timeout=TIMEOUT / 1000.0)
for child_pid in children:
_kill(child_pid, timeout=TIMEOUT / 1000.0)
def end_broker_process(self):
try:
broker_process = Process(self.broker.pid)
except NoSuchProcess:
return # was killed
# get stdout and stderr
select_config = [self.broker.stdout, self.broker.stderr], [], [], 0.1
stdout, stderr = [], []
result = select.select(*select_config)
while any(result):
if result[0]:
stdout.append(result[0][0].readline())
if result[1]:
stderr.append(result[1][0].readline())
result = select.select(*select_config)
if stdout and DEBUG_STDOUT:
_print_debug('STDOUT', ''.join(stdout))
if stderr and DEBUG_STDERR:
_print_debug('STDERR', ''.join(stderr))
# kill main process and its children
children = [process.pid for process in broker_process.get_children()]
_kill(self.broker.pid, timeout=TIMEOUT / 1000.0)
for child_pid in children:
_kill(child_pid, timeout=TIMEOUT / 1000.0)
def main(argv):
p = Popen(argv[1:])
try:
client = ExitCodeClient()
try:
proc = Process(pid=p.pid)
while p.poll() is None:
total_cpu_percent = proc.get_cpu_percent(interval=0)
for child_proc in proc.get_children(recursive=True):
total_cpu_percent += child_proc.get_cpu_percent(interval=0)
client.send_status(total_cpu_percent)
time.sleep(0.1) # recommended waiting period from psutil docs
except NoSuchProcess:
pass
except KeyboardInterrupt:
try:
p.terminate()
except OSError:
pass
client.send_status(LEDMode.Success if p.returncode == 0 else LEDMode.Error)
time.sleep(0.5) # Give server time to read value before connection closes
client.shutdown()
return p.returncode
def done(self):
try:
myself = Process(os.getpid())
for child in myself.get_children():
child.kill()
except Exception as e:
sys.stderr.write(str(e) + '\n')
def done(self):
try:
myself = Process(os.getpid())
for child in myself.get_children():
child.kill()
except Exception as e:
sys.stderr.write(str(e) + '\n')
def _psutil_kill_pid(pid):
"""
http://stackoverflow.com/questions/1230669/subprocess-deleting-child-processes-in-windows
"""
try:
parent = Process(pid)
for child in parent.get_children(recursive=True):
child.kill()
parent.kill()
except NoSuchProcess:
return
def get_time_used_by_tree(proc = this_proc):
"""
Gets the CPU time used by a process and all its children (user+sys). If the process is not
provided, this process is used. The argument must be a pid or a psutils.Process object.
Return values is in seconds.
"""
if isinstance(proc, int): proc = Process(proc) # was given a PID
time = sum(proc.get_cpu_times())
for p in proc.get_children(True):
try:
if p.is_running():
time += sum(p.get_cpu_times())
except: pass
return time
def get_time_used_by_tree(proc=this_proc):
"""
Gets the CPU time used by a process and all its children (user+sys). If the process is not
provided, this process is used. The argument must be a pid or a psutils.Process object.
Return values is in seconds.
"""
if isinstance(proc, (int, long)): proc = Process(proc) # was given a PID
time = sum(proc.get_cpu_times())
for p in proc.get_children(True):
try:
if p.is_running():
time += sum(p.get_cpu_times())
except:
pass
return time
def wait_output(settings, popen):
"""Returns `True` if we can get output of the command in the
`wait_command` time.
Command will be killed if it wasn't finished in the time.
"""
proc = Process(popen.pid)
try:
proc.wait(settings.wait_command)
return True
except TimeoutExpired:
for child in proc.get_children(recursive=True):
child.kill()
proc.kill()
return False
def wait_output(settings, popen):
"""Returns `True` if we can get output of the command in the
`wait_command` time.
Command will be killed if it wasn't finished in the time.
"""
proc = Process(popen.pid)
try:
proc.wait(settings.wait_command)
return True
except TimeoutExpired:
for child in proc.get_children(recursive=True):
child.kill()
proc.kill()
return False
def get_mem_used_by_tree(proc = this_proc):
"""
Gets the memory used by a process and all its children (RSS). If the process is not
provided, this process is used. The argument must be a pid or a psutils.Process object.
Return value is in bytes.
"""
# This would be nice, but it turns out it crashes the whole program if the process finished between creating the list of children and getting the memory usage
# Adding "if p.is_running()" would help but still have a window for the process to finish before getting the memory usage
#return sum((p.get_memory_info()[0] for p in proc.get_children(True)), proc.get_memory_info()[0])
if isinstance(proc, int): proc = Process(proc) # was given a PID
mem = proc.get_memory_info()[0]
for p in proc.get_children(True):
try:
if p.is_running():
mem += p.get_memory_info()[0]
except: pass
return mem
def get_mem_used_by_tree(proc=this_proc):
"""
Gets the memory used by a process and all its children (RSS). If the process is not
provided, this process is used. The argument must be a pid or a psutils.Process object.
Return value is in bytes.
"""
# This would be nice, but it turns out it crashes the whole program if the process finished between creating the list of children and getting the memory usage
# Adding "if p.is_running()" would help but still have a window for the process to finish before getting the memory usage
#return sum((p.get_memory_info()[0] for p in proc.get_children(True)), proc.get_memory_info()[0])
if isinstance(proc, (int, long)): proc = Process(proc) # was given a PID
mem = proc.get_memory_info()[0]
for p in proc.get_children(True):
try:
if p.is_running():
mem += p.get_memory_info()[0]
except:
pass
return mem
def end_pipeliner_process(self):
try:
pipeliner_process = Process(self.pipeliner.pid)
except NoSuchProcess:
return # was killed
# kill main process and its children
children = [process.pid for process in pipeliner_process.get_children()]
_kill(self.pipeliner.pid, timeout=TIMEOUT / 1000.0)
for child_pid in children:
_kill(child_pid, timeout=TIMEOUT / 1000.0)
# get stdout and stderr
stdout = self.pipeliner.stdout.read()
stderr = self.pipeliner.stderr.read()
if stdout and DEBUG_STDOUT:
_print_debug('STDOUT', ''.join(stdout))
if stderr and DEBUG_STDERR:
_print_debug('STDERR', ''.join(stderr))
def end_pipeliner_process(self):
try:
pipeliner_process = Process(self.pipeliner.pid)
except NoSuchProcess:
return # was killed
# kill main process and its children
children = [
process.pid for process in pipeliner_process.get_children()
]
_kill(self.pipeliner.pid, timeout=TIMEOUT / 1000.0)
for child_pid in children:
_kill(child_pid, timeout=TIMEOUT / 1000.0)
# get stdout and stderr
stdout = self.pipeliner.stdout.read()
stderr = self.pipeliner.stderr.read()
if stdout and DEBUG_STDOUT:
_print_debug('STDOUT', ''.join(stdout))
if stderr and DEBUG_STDERR:
_print_debug('STDERR', ''.join(stderr))
def _on_server(self, args):
os.chdir(self.original_dir)
abs_args_path = os.path.abspath(args.path)
component_details = open(os.path.join(abs_args_path, ZATO_INFO_FILE)).read()
out = {
'component_details': component_details,
'component_full_path': abs_args_path,
'component_host': current_host(),
'component_running': False,
'current_time': datetime.now().isoformat(),
'current_time_utc': datetime.utcnow().isoformat(),
'master_proc_connections': None,
'master_proc_pid': None,
'master_proc_name': None,
'master_proc_create_time': None,
'master_proc_create_time_utc': None,
'master_proc_username': None,
'master_proc_workers_no': None,
'master_proc_workers_pids': None,
}
master_proc_pid = self._zdaemon_command('status')
master_proc_pid = master_proc_pid.values()
if master_proc_pid and master_proc_pid[0]:
out['component_running'] = True
master_proc_pid = int(master_proc_pid[0])
master_proc = Process(master_proc_pid)
workers_pids = sorted(elem.pid for elem in master_proc.get_children())
out['master_proc_connections'] = master_proc.get_connections()
out['master_proc_pid'] = master_proc.pid
out['master_proc_create_time'] = datetime.fromtimestamp(master_proc.create_time).isoformat()
out['master_proc_create_time_utc'] = datetime.fromtimestamp(master_proc.create_time, UTC).isoformat()
out['master_proc_username'] = master_proc.username
out['master_proc_name'] = master_proc.name
out['master_proc_workers_no'] = len(workers_pids)
out['master_proc_workers_pids'] = workers_pids
for pid in workers_pids:
worker = Process(pid)
out['worker_{}_create_time'.format(pid)] = datetime.fromtimestamp(worker.create_time).isoformat()
out['worker_{}_create_time_utc'.format(pid)] = datetime.fromtimestamp(worker.create_time, UTC).isoformat()
out['worker_{}_connections'.format(pid)] = worker.get_connections()
if getattr(args, 'json', False):
out['component_details'] = loads(out['component_details'])
self.logger.info(dumps(out))
else:
cols_width = args.cols_width if args.cols_width else DEFAULT_COLS_WIDTH
cols_width = (elem.strip() for elem in cols_width.split(','))
cols_width = [int(elem) for elem in cols_width]
table = Texttable()
table.set_cols_width(cols_width)
# Use text ('t') instead of auto so that boolean values don't get converted into ints
table.set_cols_dtype(['t', 't'])
rows = [['Key', 'Value']]
rows.extend(sorted(out.items()))
table.add_rows(rows)
self.logger.info(table.draw())
def _on_server(self, args):
os.chdir(self.original_dir)
abs_args_path = os.path.abspath(args.path)
component_details = open(os.path.join(abs_args_path, ZATO_INFO_FILE)).read()
out = {
"component_details": component_details,
"component_full_path": abs_args_path,
"component_host": current_host(),
"component_running": False,
"current_time": datetime.now().isoformat(),
"current_time_utc": datetime.utcnow().isoformat(),
"master_proc_connections": None,
"master_proc_pid": None,
"master_proc_name": None,
"master_proc_create_time": None,
"master_proc_create_time_utc": None,
"master_proc_username": None,
"master_proc_workers_no": None,
"master_proc_workers_pids": None,
}
master_proc_pid = self._zdaemon_command("status")
master_proc_pid = master_proc_pid.values()
if master_proc_pid and master_proc_pid[0]:
out["component_running"] = True
master_proc_pid = int(master_proc_pid[0])
master_proc = Process(master_proc_pid)
workers_pids = sorted(elem.pid for elem in master_proc.get_children())
out["master_proc_connections"] = master_proc.get_connections()
out["master_proc_pid"] = master_proc.pid
out["master_proc_create_time"] = datetime.fromtimestamp(master_proc.create_time).isoformat()
out["master_proc_create_time_utc"] = datetime.fromtimestamp(master_proc.create_time, UTC).isoformat()
out["master_proc_username"] = master_proc.username
out["master_proc_name"] = master_proc.name
out["master_proc_workers_no"] = len(workers_pids)
out["master_proc_workers_pids"] = workers_pids
for pid in workers_pids:
worker = Process(pid)
worker_memory_percent = worker.get_memory_percent()
out["worker_{}_create_time".format(pid)] = datetime.fromtimestamp(worker.create_time).isoformat()
out["worker_{}_create_time_utc".format(pid)] = datetime.fromtimestamp(
worker.create_time, UTC
).isoformat()
out["worker_{}_connections".format(pid)] = worker.get_connections()
if getattr(args, "json", False):
out["component_details"] = loads(out["component_details"])
self.logger.info(dumps(out))
else:
cols_width = args.cols_width if args.cols_width else DEFAULT_COLS_WIDTH
cols_width = (elem.strip() for elem in cols_width.split(","))
cols_width = [int(elem) for elem in cols_width]
table = Texttable()
table.set_cols_width(cols_width)
# Use text ('t') instead of auto so that boolean values don't get converted into ints
table.set_cols_dtype(["t", "t"])
rows = [["Key", "Value"]]
rows.extend(sorted(out.items()))
table.add_rows(rows)
self.logger.info(table.draw())
def _on_server(self, args):
os.chdir(self.original_dir)
abs_args_path = os.path.abspath(args.path)
component_details = open(os.path.join(abs_args_path,
ZATO_INFO_FILE)).read()
out = {
'component_details': component_details,
'component_full_path': abs_args_path,
'component_host': current_host(),
'component_running': False,
'current_time': datetime.now().isoformat(),
'current_time_utc': datetime.utcnow().isoformat(),
'master_proc_connections': None,
'master_proc_pid': None,
'master_proc_name': None,
'master_proc_create_time': None,
'master_proc_create_time_utc': None,
'master_proc_username': None,
'master_proc_workers_no': None,
'master_proc_workers_pids': None,
}
master_proc_pid = self._zdaemon_command('status')
master_proc_pid = master_proc_pid.values()
if master_proc_pid and master_proc_pid[0]:
out['component_running'] = True
master_proc_pid = int(master_proc_pid[0])
master_proc = Process(master_proc_pid)
workers_pids = sorted(elem.pid
for elem in master_proc.get_children())
out['master_proc_connections'] = master_proc.get_connections()
out['master_proc_pid'] = master_proc.pid
out['master_proc_create_time'] = datetime.fromtimestamp(
master_proc.create_time).isoformat()
out['master_proc_create_time_utc'] = datetime.fromtimestamp(
master_proc.create_time, UTC).isoformat()
out['master_proc_username'] = master_proc.username
out['master_proc_name'] = master_proc.name
out['master_proc_workers_no'] = len(workers_pids)
out['master_proc_workers_pids'] = workers_pids
for pid in workers_pids:
worker = Process(pid)
worker_memory_percent = worker.get_memory_percent()
out['worker_{}_create_time'.format(
pid)] = datetime.fromtimestamp(
worker.create_time).isoformat()
out['worker_{}_create_time_utc'.format(
pid)] = datetime.fromtimestamp(worker.create_time,
UTC).isoformat()
out['worker_{}_connections'.format(
pid)] = worker.get_connections()
if getattr(args, 'json', False):
out['component_details'] = loads(out['component_details'])
self.logger.info(dumps(out))
else:
cols_width = args.cols_width if args.cols_width else DEFAULT_COLS_WIDTH
cols_width = (elem.strip() for elem in cols_width.split(','))
cols_width = [int(elem) for elem in cols_width]
table = Texttable()
table.set_cols_width(cols_width)
# Use text ('t') instead of auto so that boolean values don't get converted into ints
table.set_cols_dtype(['t', 't'])
rows = [['Key', 'Value']]
rows.extend(sorted(out.items()))
table.add_rows(rows)
self.logger.info(table.draw())
class ProcessMonitor(object):
def __init__(self, commands, timeout, interval, output_dir=None, monitor_dir=None, network=False):
self.start_time = time()
self.timed_out = False
self.end_time = self.start_time + timeout if timeout else 0 # Do not time out if time_limit is 0.
self._interval = interval
self._rm = ResourceMonitor(output_dir, commands)
self.monitor_file = None
self.network_monitor_file = None
self.fd_file = None
self.psutil_process = Process()
if monitor_dir:
self.monitor_file = open(monitor_dir + "/resource_usage.log", "w", (1024 ** 2) * 10) # Set the file's buffering to 10MB
self.fd_file = open(monitor_dir + "/fd_usage.log", "w", (1024 ** 2) * 10) # Set the file's buffering to 10MB
self.fd_file.write("time pid num_fds\n")
# We read the jiffie -> second conversion rate from the os, by dividing the utime
# and stime values by this conversion rate we will get the actual cpu seconds spend during this second.
try:
sc_clk_tck = float(sysconf(sysconf_names['SC_CLK_TCK']))
except AttributeError:
sc_clk_tck = 100.0
try:
import resource
pagesize = resource.getpagesize()
except:
pagesize = 4 * 1024
self.monitor_file.write(json.dumps({"sc_clk_tck": sc_clk_tck, 'pagesize': pagesize}) + "\n")
# If monitoring network, open a separate file.
if network:
self.network_monitor_file = open(monitor_dir + "/network_usage.log", "w", (1024 ** 2) * 10) # Set the file's buffering to 10MB
# Capture SIGTERM to kill all the child processes before dying
self.stopping = False
signal(SIGTERM, self._termTrap)
def stop(self):
self.stopping = True
if self.monitor_file:
self.monitor_file.close()
if self.fd_file:
self.fd_file.close()
# Check if any process exited with an error code before killing the remaining ones
failed = self._rm.get_failed_commands()
self._rm.terminate()
if failed:
print "Some processes failed:"
for pid, (command, exit_code) in failed.iteritems():
print " %s (%d) exited value: %d" % (command, pid, exit_code)
print "Process guard exiting with error"
return COMMANDS_FAILED_EXIT_CODE
else:
return OK_EXIT_CODE
def _termTrap(self, *argv):
print "Captured TERM signal"
if not self.stopping:
self.stop()
def monitoring_loop(self):
check_for_new_processes = 60
time_start = time()
sleep_time = self._interval
last_subprocess_update = time_start
while not self.stopping:
timestamp = time()
r_timestamp = ceil(timestamp / self._interval) * self._interval # rounding timestamp to nearest interval to try to overlap multiple nodes
self._rm.prune_pid_list()
# Look for new subprocesses only once a second and only during the first "check_for_new_processes" seconds
if (timestamp < time_start + check_for_new_processes) and (timestamp - last_subprocess_update >= 1):
self._rm.update_pid_tree()
last_subprocess_update = timestamp
if self._rm.is_everyone_dead():
print "All child processes have died, exiting"
return self.stop()
elif self.monitor_file:
next_wake = timestamp + self._interval
for line in self._rm.get_raw_stats():
self.monitor_file.write("%.1f %s\n" % (r_timestamp, line))
sleep_time = next_wake - timestamp
if sleep_time < 0:
print "Can't keep up with this interval, try a higher value!", sleep_time
return self.stop()
if self.network_monitor_file:
for line in self._rm.get_network_stats():
self.network_monitor_file.write("%.1f %s\n" % (r_timestamp, line))
if hasattr(self.psutil_process, 'children'):
p_children = self.psutil_process.children(recursive=True)
else:
p_children = self.psutil_process.get_children(recursive=True)
if self.fd_file:
for child_process in p_children:
try:
if hasattr(self.psutil_process, 'num_fds'):
self.fd_file.write("%.1f %s %d\n" %
(r_timestamp, child_process.pid, child_process.num_fds()))
else:
self.fd_file.write("%.1f %s %d\n" %
(r_timestamp, child_process.pid, child_process.get_num_fds()))
except (AccessDenied, NoSuchProcess):
pass # Just ignore the file descriptors of this invalid process
if self.end_time and timestamp > self.end_time: # if self.end_time == 0 the time out is disabled.
print "Time out, killing monitored processes."
self.timed_out = True
return self.stop()
sleep(sleep_time)
def get_info(process=None, interval=0):
"""Return information about a process.
If process is None, will return the information about the current process
"""
if process is None:
process = Process(os.getpid())
info = {}
try:
mem_info = process.get_memory_info()
info['mem_info1'] = bytes2human(mem_info[0])
info['mem_info2'] = bytes2human(mem_info[1])
except AccessDenied:
info['mem_info1'] = info['mem_info2'] = "N/A"
try:
info['cpu'] = process.get_cpu_percent(interval=interval)
except AccessDenied:
info['cpu'] = "N/A"
try:
info['mem'] = round(process.get_memory_percent(), 1)
except AccessDenied:
info['mem'] = "N/A"
try:
cpu_times = process.get_cpu_times()
ctime = timedelta(seconds=sum(cpu_times))
ctime = "%s:%s.%s" % (ctime.seconds // 60 % 60,
str((ctime.seconds % 60)).zfill(2),
str(ctime.microseconds)[:2])
except AccessDenied:
ctime = "N/A"
info['ctime'] = ctime
try:
info['pid'] = process.pid
except AccessDenied:
info['pid'] = 'N/A'
try:
info['username'] = process.username
except AccessDenied:
info['username'] = '******'
try:
info['nice'] = process.nice
except AccessDenied:
info['nice'] = 'N/A'
except NoSuchProcess:
info['nice'] = 'Zombie'
try:
cmdline = os.path.basename(shlex.split(process.cmdline[0])[0])
except (AccessDenied, IndexError):
cmdline = "N/A"
info['cmdline'] = cmdline
info['children'] = []
for child in process.get_children():
info['children'].append(get_info(child, interval=interval))
return info
