def click(x, notify=False, pid=None, pids=None, webdriver=None, window_name=None, debug=False): if debug: print("[beryl] starting click") print("\tpid: " + str(pid)) print("\twebdriver: " + str(webdriver)) print("\twindow_name: " + str(window_name)) print("\tstr(type(webdriver)): " + str(type(webdriver))) type_as_string = str(type(x)) webdriver_type_as_string = str(type(webdriver)) if webdriver_type_as_string == "<class 'selenium.webdriver.firefox.webdriver.WebDriver'>": pids = [webdriver.binary.process.pid] elif webdriver_type_as_string == "<class 'selenium.webdriver.chrome.webdriver.WebDriver'>": process = Process(webdriver.service.process.pid) if hasattr(process, "children"): pids = [p.pid for p in process.children()] elif hasattr(process, "get_children"): pids = [p.pid for p in process.get_children()] if isinstance(x, str) or isinstance(x, unicode): if x.endswith(".png") or x.endswith(".jpg"): click_image(x, notify=notify) else: click_text(x, notify=notify, pids=pids, window_name=window_name, debug=debug) elif isinstance(x, PngImageFile): click_image(x,notify=notify) elif isinstance(x, tuple): click_location(x,notify=notify)
def end_broker_process(self): try: broker_process = Process(self.broker.pid) except NoSuchProcess: return # was killed # get stdout and stderr select_config = [self.broker.stdout, self.broker.stderr], [], [], 0.1 stdout, stderr = [], [] result = select.select(*select_config) while any(result): if result[0]: stdout.append(result[0][0].readline()) if result[1]: stderr.append(result[1][0].readline()) result = select.select(*select_config) if stdout and DEBUG_STDOUT: _print_debug('STDOUT', ''.join(stdout)) if stderr and DEBUG_STDERR: _print_debug('STDERR', ''.join(stderr)) # kill main process and its children children = [process.pid for process in broker_process.get_children()] _kill(self.broker.pid, timeout=TIMEOUT / 1000.0) for child_pid in children: _kill(child_pid, timeout=TIMEOUT / 1000.0)
def main(argv): p = Popen(argv[1:]) try: client = ExitCodeClient() try: proc = Process(pid=p.pid) while p.poll() is None: total_cpu_percent = proc.get_cpu_percent(interval=0) for child_proc in proc.get_children(recursive=True): total_cpu_percent += child_proc.get_cpu_percent(interval=0) client.send_status(total_cpu_percent) time.sleep(0.1) # recommended waiting period from psutil docs except NoSuchProcess: pass except KeyboardInterrupt: try: p.terminate() except OSError: pass client.send_status(LEDMode.Success if p.returncode == 0 else LEDMode.Error) time.sleep(0.5) # Give server time to read value before connection closes client.shutdown() return p.returncode
def done(self): try: myself = Process(os.getpid()) for child in myself.get_children(): child.kill() except Exception as e: sys.stderr.write(str(e) + '\n')
def _psutil_kill_pid(pid): """ http://stackoverflow.com/questions/1230669/subprocess-deleting-child-processes-in-windows """ try: parent = Process(pid) for child in parent.get_children(recursive=True): child.kill() parent.kill() except NoSuchProcess: return
def get_time_used_by_tree(proc = this_proc): """ Gets the CPU time used by a process and all its children (user+sys). If the process is not provided, this process is used. The argument must be a pid or a psutils.Process object. Return values is in seconds. """ if isinstance(proc, int): proc = Process(proc) # was given a PID time = sum(proc.get_cpu_times()) for p in proc.get_children(True): try: if p.is_running(): time += sum(p.get_cpu_times()) except: pass return time
def get_time_used_by_tree(proc=this_proc): """ Gets the CPU time used by a process and all its children (user+sys). If the process is not provided, this process is used. The argument must be a pid or a psutils.Process object. Return values is in seconds. """ if isinstance(proc, (int, long)): proc = Process(proc) # was given a PID time = sum(proc.get_cpu_times()) for p in proc.get_children(True): try: if p.is_running(): time += sum(p.get_cpu_times()) except: pass return time
def wait_output(settings, popen): """Returns `True` if we can get output of the command in the `wait_command` time. Command will be killed if it wasn't finished in the time. """ proc = Process(popen.pid) try: proc.wait(settings.wait_command) return True except TimeoutExpired: for child in proc.get_children(recursive=True): child.kill() proc.kill() return False
def get_mem_used_by_tree(proc = this_proc): """ Gets the memory used by a process and all its children (RSS). If the process is not provided, this process is used. The argument must be a pid or a psutils.Process object. Return value is in bytes. """ # This would be nice, but it turns out it crashes the whole program if the process finished between creating the list of children and getting the memory usage # Adding "if p.is_running()" would help but still have a window for the process to finish before getting the memory usage #return sum((p.get_memory_info()[0] for p in proc.get_children(True)), proc.get_memory_info()[0]) if isinstance(proc, int): proc = Process(proc) # was given a PID mem = proc.get_memory_info()[0] for p in proc.get_children(True): try: if p.is_running(): mem += p.get_memory_info()[0] except: pass return mem
def get_mem_used_by_tree(proc=this_proc): """ Gets the memory used by a process and all its children (RSS). If the process is not provided, this process is used. The argument must be a pid or a psutils.Process object. Return value is in bytes. """ # This would be nice, but it turns out it crashes the whole program if the process finished between creating the list of children and getting the memory usage # Adding "if p.is_running()" would help but still have a window for the process to finish before getting the memory usage #return sum((p.get_memory_info()[0] for p in proc.get_children(True)), proc.get_memory_info()[0]) if isinstance(proc, (int, long)): proc = Process(proc) # was given a PID mem = proc.get_memory_info()[0] for p in proc.get_children(True): try: if p.is_running(): mem += p.get_memory_info()[0] except: pass return mem
def end_pipeliner_process(self): try: pipeliner_process = Process(self.pipeliner.pid) except NoSuchProcess: return # was killed # kill main process and its children children = [process.pid for process in pipeliner_process.get_children()] _kill(self.pipeliner.pid, timeout=TIMEOUT / 1000.0) for child_pid in children: _kill(child_pid, timeout=TIMEOUT / 1000.0) # get stdout and stderr stdout = self.pipeliner.stdout.read() stderr = self.pipeliner.stderr.read() if stdout and DEBUG_STDOUT: _print_debug('STDOUT', ''.join(stdout)) if stderr and DEBUG_STDERR: _print_debug('STDERR', ''.join(stderr))
def end_pipeliner_process(self): try: pipeliner_process = Process(self.pipeliner.pid) except NoSuchProcess: return # was killed # kill main process and its children children = [ process.pid for process in pipeliner_process.get_children() ] _kill(self.pipeliner.pid, timeout=TIMEOUT / 1000.0) for child_pid in children: _kill(child_pid, timeout=TIMEOUT / 1000.0) # get stdout and stderr stdout = self.pipeliner.stdout.read() stderr = self.pipeliner.stderr.read() if stdout and DEBUG_STDOUT: _print_debug('STDOUT', ''.join(stdout)) if stderr and DEBUG_STDERR: _print_debug('STDERR', ''.join(stderr))
def _on_server(self, args): os.chdir(self.original_dir) abs_args_path = os.path.abspath(args.path) component_details = open(os.path.join(abs_args_path, ZATO_INFO_FILE)).read() out = { 'component_details': component_details, 'component_full_path': abs_args_path, 'component_host': current_host(), 'component_running': False, 'current_time': datetime.now().isoformat(), 'current_time_utc': datetime.utcnow().isoformat(), 'master_proc_connections': None, 'master_proc_pid': None, 'master_proc_name': None, 'master_proc_create_time': None, 'master_proc_create_time_utc': None, 'master_proc_username': None, 'master_proc_workers_no': None, 'master_proc_workers_pids': None, } master_proc_pid = self._zdaemon_command('status') master_proc_pid = master_proc_pid.values() if master_proc_pid and master_proc_pid[0]: out['component_running'] = True master_proc_pid = int(master_proc_pid[0]) master_proc = Process(master_proc_pid) workers_pids = sorted(elem.pid for elem in master_proc.get_children()) out['master_proc_connections'] = master_proc.get_connections() out['master_proc_pid'] = master_proc.pid out['master_proc_create_time'] = datetime.fromtimestamp(master_proc.create_time).isoformat() out['master_proc_create_time_utc'] = datetime.fromtimestamp(master_proc.create_time, UTC).isoformat() out['master_proc_username'] = master_proc.username out['master_proc_name'] = master_proc.name out['master_proc_workers_no'] = len(workers_pids) out['master_proc_workers_pids'] = workers_pids for pid in workers_pids: worker = Process(pid) out['worker_{}_create_time'.format(pid)] = datetime.fromtimestamp(worker.create_time).isoformat() out['worker_{}_create_time_utc'.format(pid)] = datetime.fromtimestamp(worker.create_time, UTC).isoformat() out['worker_{}_connections'.format(pid)] = worker.get_connections() if getattr(args, 'json', False): out['component_details'] = loads(out['component_details']) self.logger.info(dumps(out)) else: cols_width = args.cols_width if args.cols_width else DEFAULT_COLS_WIDTH cols_width = (elem.strip() for elem in cols_width.split(',')) cols_width = [int(elem) for elem in cols_width] table = Texttable() table.set_cols_width(cols_width) # Use text ('t') instead of auto so that boolean values don't get converted into ints table.set_cols_dtype(['t', 't']) rows = [['Key', 'Value']] rows.extend(sorted(out.items())) table.add_rows(rows) self.logger.info(table.draw())
def _on_server(self, args): os.chdir(self.original_dir) abs_args_path = os.path.abspath(args.path) component_details = open(os.path.join(abs_args_path, ZATO_INFO_FILE)).read() out = { "component_details": component_details, "component_full_path": abs_args_path, "component_host": current_host(), "component_running": False, "current_time": datetime.now().isoformat(), "current_time_utc": datetime.utcnow().isoformat(), "master_proc_connections": None, "master_proc_pid": None, "master_proc_name": None, "master_proc_create_time": None, "master_proc_create_time_utc": None, "master_proc_username": None, "master_proc_workers_no": None, "master_proc_workers_pids": None, } master_proc_pid = self._zdaemon_command("status") master_proc_pid = master_proc_pid.values() if master_proc_pid and master_proc_pid[0]: out["component_running"] = True master_proc_pid = int(master_proc_pid[0]) master_proc = Process(master_proc_pid) workers_pids = sorted(elem.pid for elem in master_proc.get_children()) out["master_proc_connections"] = master_proc.get_connections() out["master_proc_pid"] = master_proc.pid out["master_proc_create_time"] = datetime.fromtimestamp(master_proc.create_time).isoformat() out["master_proc_create_time_utc"] = datetime.fromtimestamp(master_proc.create_time, UTC).isoformat() out["master_proc_username"] = master_proc.username out["master_proc_name"] = master_proc.name out["master_proc_workers_no"] = len(workers_pids) out["master_proc_workers_pids"] = workers_pids for pid in workers_pids: worker = Process(pid) worker_memory_percent = worker.get_memory_percent() out["worker_{}_create_time".format(pid)] = datetime.fromtimestamp(worker.create_time).isoformat() out["worker_{}_create_time_utc".format(pid)] = datetime.fromtimestamp( worker.create_time, UTC ).isoformat() out["worker_{}_connections".format(pid)] = worker.get_connections() if getattr(args, "json", False): out["component_details"] = loads(out["component_details"]) self.logger.info(dumps(out)) else: cols_width = args.cols_width if args.cols_width else DEFAULT_COLS_WIDTH cols_width = (elem.strip() for elem in cols_width.split(",")) cols_width = [int(elem) for elem in cols_width] table = Texttable() table.set_cols_width(cols_width) # Use text ('t') instead of auto so that boolean values don't get converted into ints table.set_cols_dtype(["t", "t"]) rows = [["Key", "Value"]] rows.extend(sorted(out.items())) table.add_rows(rows) self.logger.info(table.draw())
def _on_server(self, args): os.chdir(self.original_dir) abs_args_path = os.path.abspath(args.path) component_details = open(os.path.join(abs_args_path, ZATO_INFO_FILE)).read() out = { 'component_details': component_details, 'component_full_path': abs_args_path, 'component_host': current_host(), 'component_running': False, 'current_time': datetime.now().isoformat(), 'current_time_utc': datetime.utcnow().isoformat(), 'master_proc_connections': None, 'master_proc_pid': None, 'master_proc_name': None, 'master_proc_create_time': None, 'master_proc_create_time_utc': None, 'master_proc_username': None, 'master_proc_workers_no': None, 'master_proc_workers_pids': None, } master_proc_pid = self._zdaemon_command('status') master_proc_pid = master_proc_pid.values() if master_proc_pid and master_proc_pid[0]: out['component_running'] = True master_proc_pid = int(master_proc_pid[0]) master_proc = Process(master_proc_pid) workers_pids = sorted(elem.pid for elem in master_proc.get_children()) out['master_proc_connections'] = master_proc.get_connections() out['master_proc_pid'] = master_proc.pid out['master_proc_create_time'] = datetime.fromtimestamp( master_proc.create_time).isoformat() out['master_proc_create_time_utc'] = datetime.fromtimestamp( master_proc.create_time, UTC).isoformat() out['master_proc_username'] = master_proc.username out['master_proc_name'] = master_proc.name out['master_proc_workers_no'] = len(workers_pids) out['master_proc_workers_pids'] = workers_pids for pid in workers_pids: worker = Process(pid) worker_memory_percent = worker.get_memory_percent() out['worker_{}_create_time'.format( pid)] = datetime.fromtimestamp( worker.create_time).isoformat() out['worker_{}_create_time_utc'.format( pid)] = datetime.fromtimestamp(worker.create_time, UTC).isoformat() out['worker_{}_connections'.format( pid)] = worker.get_connections() if getattr(args, 'json', False): out['component_details'] = loads(out['component_details']) self.logger.info(dumps(out)) else: cols_width = args.cols_width if args.cols_width else DEFAULT_COLS_WIDTH cols_width = (elem.strip() for elem in cols_width.split(',')) cols_width = [int(elem) for elem in cols_width] table = Texttable() table.set_cols_width(cols_width) # Use text ('t') instead of auto so that boolean values don't get converted into ints table.set_cols_dtype(['t', 't']) rows = [['Key', 'Value']] rows.extend(sorted(out.items())) table.add_rows(rows) self.logger.info(table.draw())
class ProcessMonitor(object): def __init__(self, commands, timeout, interval, output_dir=None, monitor_dir=None, network=False): self.start_time = time() self.timed_out = False self.end_time = self.start_time + timeout if timeout else 0 # Do not time out if time_limit is 0. self._interval = interval self._rm = ResourceMonitor(output_dir, commands) self.monitor_file = None self.network_monitor_file = None self.fd_file = None self.psutil_process = Process() if monitor_dir: self.monitor_file = open(monitor_dir + "/resource_usage.log", "w", (1024 ** 2) * 10) # Set the file's buffering to 10MB self.fd_file = open(monitor_dir + "/fd_usage.log", "w", (1024 ** 2) * 10) # Set the file's buffering to 10MB self.fd_file.write("time pid num_fds\n") # We read the jiffie -> second conversion rate from the os, by dividing the utime # and stime values by this conversion rate we will get the actual cpu seconds spend during this second. try: sc_clk_tck = float(sysconf(sysconf_names['SC_CLK_TCK'])) except AttributeError: sc_clk_tck = 100.0 try: import resource pagesize = resource.getpagesize() except: pagesize = 4 * 1024 self.monitor_file.write(json.dumps({"sc_clk_tck": sc_clk_tck, 'pagesize': pagesize}) + "\n") # If monitoring network, open a separate file. if network: self.network_monitor_file = open(monitor_dir + "/network_usage.log", "w", (1024 ** 2) * 10) # Set the file's buffering to 10MB # Capture SIGTERM to kill all the child processes before dying self.stopping = False signal(SIGTERM, self._termTrap) def stop(self): self.stopping = True if self.monitor_file: self.monitor_file.close() if self.fd_file: self.fd_file.close() # Check if any process exited with an error code before killing the remaining ones failed = self._rm.get_failed_commands() self._rm.terminate() if failed: print "Some processes failed:" for pid, (command, exit_code) in failed.iteritems(): print " %s (%d) exited value: %d" % (command, pid, exit_code) print "Process guard exiting with error" return COMMANDS_FAILED_EXIT_CODE else: return OK_EXIT_CODE def _termTrap(self, *argv): print "Captured TERM signal" if not self.stopping: self.stop() def monitoring_loop(self): check_for_new_processes = 60 time_start = time() sleep_time = self._interval last_subprocess_update = time_start while not self.stopping: timestamp = time() r_timestamp = ceil(timestamp / self._interval) * self._interval # rounding timestamp to nearest interval to try to overlap multiple nodes self._rm.prune_pid_list() # Look for new subprocesses only once a second and only during the first "check_for_new_processes" seconds if (timestamp < time_start + check_for_new_processes) and (timestamp - last_subprocess_update >= 1): self._rm.update_pid_tree() last_subprocess_update = timestamp if self._rm.is_everyone_dead(): print "All child processes have died, exiting" return self.stop() elif self.monitor_file: next_wake = timestamp + self._interval for line in self._rm.get_raw_stats(): self.monitor_file.write("%.1f %s\n" % (r_timestamp, line)) sleep_time = next_wake - timestamp if sleep_time < 0: print "Can't keep up with this interval, try a higher value!", sleep_time return self.stop() if self.network_monitor_file: for line in self._rm.get_network_stats(): self.network_monitor_file.write("%.1f %s\n" % (r_timestamp, line)) if hasattr(self.psutil_process, 'children'): p_children = self.psutil_process.children(recursive=True) else: p_children = self.psutil_process.get_children(recursive=True) if self.fd_file: for child_process in p_children: try: if hasattr(self.psutil_process, 'num_fds'): self.fd_file.write("%.1f %s %d\n" % (r_timestamp, child_process.pid, child_process.num_fds())) else: self.fd_file.write("%.1f %s %d\n" % (r_timestamp, child_process.pid, child_process.get_num_fds())) except (AccessDenied, NoSuchProcess): pass # Just ignore the file descriptors of this invalid process if self.end_time and timestamp > self.end_time: # if self.end_time == 0 the time out is disabled. print "Time out, killing monitored processes." self.timed_out = True return self.stop() sleep(sleep_time)
def get_info(process=None, interval=0): """Return information about a process. If process is None, will return the information about the current process """ if process is None: process = Process(os.getpid()) info = {} try: mem_info = process.get_memory_info() info['mem_info1'] = bytes2human(mem_info[0]) info['mem_info2'] = bytes2human(mem_info[1]) except AccessDenied: info['mem_info1'] = info['mem_info2'] = "N/A" try: info['cpu'] = process.get_cpu_percent(interval=interval) except AccessDenied: info['cpu'] = "N/A" try: info['mem'] = round(process.get_memory_percent(), 1) except AccessDenied: info['mem'] = "N/A" try: cpu_times = process.get_cpu_times() ctime = timedelta(seconds=sum(cpu_times)) ctime = "%s:%s.%s" % (ctime.seconds // 60 % 60, str((ctime.seconds % 60)).zfill(2), str(ctime.microseconds)[:2]) except AccessDenied: ctime = "N/A" info['ctime'] = ctime try: info['pid'] = process.pid except AccessDenied: info['pid'] = 'N/A' try: info['username'] = process.username except AccessDenied: info['username'] = '******' try: info['nice'] = process.nice except AccessDenied: info['nice'] = 'N/A' except NoSuchProcess: info['nice'] = 'Zombie' try: cmdline = os.path.basename(shlex.split(process.cmdline[0])[0]) except (AccessDenied, IndexError): cmdline = "N/A" info['cmdline'] = cmdline info['children'] = [] for child in process.get_children(): info['children'].append(get_info(child, interval=interval)) return info