def run(self): logger.info("Started App usage tracker thread") file_loc = os.path.join(os.path.dirname(__file__), "apps-usage", "apps-usage.exe") try: hJob = win32job.CreateJobObject(None, "") extended_info = win32job.QueryInformationJobObject( hJob, win32job.JobObjectExtendedLimitInformation) extended_info['BasicLimitInformation'][ 'LimitFlags'] = win32job.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE win32job.SetInformationJobObject( hJob, win32job.JobObjectExtendedLimitInformation, extended_info) p = subprocess.Popen(f"{file_loc} \"{ROOT_DIR}\"", shell=False) perms = win32con.PROCESS_TERMINATE | win32con.PROCESS_SET_QUOTA hProcess = win32api.OpenProcess(perms, False, p.pid) win32job.AssignProcessToJobObject(hJob, hProcess) while True and not self.kill: time.sleep(0.5) p.terminate() p.kill() except Exception as e: logger.error(f"Error Starting Apps-Usage. {e} ") logger.warning("Stopped app usage tracking thread")
def start(self): """ Starts the process and the logger pipes for its stdout and stderr. """ creation_flags = 0 if sys.platform == "win32" and _JOB_OBJECT is not None: creation_flags |= win32process.CREATE_BREAKAWAY_FROM_JOB with _POPEN_LOCK: self._process = subprocess.Popen(self.args, env=self.env, creationflags=creation_flags, stdout=subprocess.PIPE, stderr=subprocess.PIPE) self.pid = self._process.pid self._stdout_pipe = pipe.LoggerPipe(self.logger, logging.INFO, self._process.stdout) self._stderr_pipe = pipe.LoggerPipe(self.logger, logging.ERROR, self._process.stderr) self._stdout_pipe.wait_until_started() self._stderr_pipe.wait_until_started() if sys.platform == "win32" and _JOB_OBJECT is not None: try: win32job.AssignProcessToJobObject(_JOB_OBJECT, self._process._handle) except win32job.error as err: # ERROR_ACCESS_DENIED (winerror=5) is received when the process has already died. if err.winerror != winerror.ERROR_ACCESS_DENIED: raise return_code = win32process.GetExitCodeProcess(self._process._handle) if return_code == win32con.STILL_ACTIVE: raise
def start(self): """Start the process and the logger pipes for its stdout and stderr.""" creation_flags = 0 if sys.platform == "win32" and _JOB_OBJECT is not None: creation_flags |= win32process.CREATE_BREAKAWAY_FROM_JOB # Tests fail if a process takes too long to startup and listen to a socket. Use buffered # I/O pipes to give the process some leeway. buffer_size = 1024 * 1024 # Close file descriptors in the child process before executing the program. This prevents # file descriptors that were inherited due to multiple calls to fork() -- either within one # thread, or concurrently from multiple threads -- from causing another subprocess to wait # for the completion of the newly spawned child process. Closing other file descriptors # isn't supported on Windows when stdout and stderr are redirected. close_fds = (sys.platform != "win32") with _POPEN_LOCK: self._process = subprocess.Popen(self.args, bufsize=buffer_size, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=close_fds, env=self.env, creationflags=creation_flags, cwd=self._cwd) self.pid = self._process.pid if _config.UNDO_RECORDER_PATH is not None and ( "mongod" in self.args[0] or "mongos" in self.args[0]): recorder_args = [ _config.UNDO_RECORDER_PATH, "--thread-fuzzing", "-p", str(self.pid) ] self._recorder = subprocess.Popen(recorder_args, bufsize=buffer_size, env=self.env, creationflags=creation_flags) self._stdout_pipe = pipe.LoggerPipe(self.logger, logging.INFO, self._process.stdout) self._stderr_pipe = pipe.LoggerPipe(self.logger, logging.ERROR, self._process.stderr) self._stdout_pipe.wait_until_started() self._stderr_pipe.wait_until_started() if sys.platform == "win32" and _JOB_OBJECT is not None: try: win32job.AssignProcessToJobObject(_JOB_OBJECT, self._process._handle) except win32job.error as err: # ERROR_ACCESS_DENIED (winerror=5) is received when the process has already died. if err.winerror != winerror.ERROR_ACCESS_DENIED: raise return_code = win32process.GetExitCodeProcess( self._process._handle) if return_code == win32con.STILL_ACTIVE: raise
def _start(self, argv): """In most cases, just call subprocess.Popen(). On windows, add the started process to a new Job Object, so that any child processes of this process can be killed with a single call to TerminateJobObject (see self.stop()). """ proc = Popen(argv) if os.sys.platform == "win32": # Create a job object with the "kill on job close" # flag; this is inherited by child processes (ie # the mongod started on our behalf by buildlogger) # and lets us terminate the whole tree of processes # rather than orphaning the mongod. import win32job self.job_object = win32job.CreateJobObject(None, '') job_info = win32job.QueryInformationJobObject( self.job_object, win32job.JobObjectExtendedLimitInformation) job_info['BasicLimitInformation'][ 'LimitFlags'] |= win32job.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE win32job.SetInformationJobObject( self.job_object, win32job.JobObjectExtendedLimitInformation, job_info) win32job.AssignProcessToJobObject(self.job_object, proc._handle) return proc
def _add_to_job_object(self): global _global_process_job_handle if _global_process_job_handle is not None: #This means that we are creating another process family - we'll all be in the same job return already_in_job = win32job.IsProcessInJob(win32api.GetCurrentProcess(), None) #Create a new job and put us in it before we create any children logger.debug("Creating job object and adding parent process to it") security_attrs = win32security.SECURITY_ATTRIBUTES() security_attrs.bInheritHandle = 0 _global_process_job_handle = win32job.CreateJobObject(security_attrs, self.get_job_object_name()) extended_info = win32job.QueryInformationJobObject(_global_process_job_handle, win32job.JobObjectExtendedLimitInformation) extended_info['BasicLimitInformation']['LimitFlags'] = win32job.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE win32job.SetInformationJobObject(_global_process_job_handle, win32job.JobObjectExtendedLimitInformation, extended_info) try: win32job.AssignProcessToJobObject(_global_process_job_handle, win32api.GetCurrentProcess()) except Exception as e: winv = sys.getwindowsversion() logger.error("Error raised during assignment of the current process to a new job object. " +\ "The process %s already in a job. " +\ "The windows version is %d.%d.\nError: %s", "is" if already_in_job else "is not", winv.major, winv.minor, _exception_str()) if already_in_job and not (winv.major >= 6 and winv.minor >= 2): raise JobObjectAssignError("On Windows versions older than Windows 8 / Windows Server 2012, ProcessFamily relies on the parent process NOT being in a job already", e, already_in_job) raise JobObjectAssignError("Error raised during assignment of the current process to a new job object.", e, already_in_job) logger.debug("Added to job object")
def SvcDoRun(self): if hasattr(sys, "frozen"): this_dir = os.path.dirname(win32api.GetModuleFileName(None)) else: this_dir = os.path.dirname(os.path.abspath(__file__)) # TODO: maybe it is better to run this in a job object too with open(os.path.join(this_dir, 'npm.log'), 'w') as npm_log: subprocess.check_call('npm install', cwd=this_dir, shell=True, stdin=None, stdout=npm_log, stderr=subprocess.STDOUT) security_attributes = win32security.SECURITY_ATTRIBUTES() security_attributes.bInheritHandle = True startup = win32process.STARTUPINFO() startup.dwFlags |= win32process.STARTF_USESTDHANDLES startup.hStdInput = None startup.hStdOutput = win32file.CreateFile( os.path.join(this_dir, "service_stderr.log"), win32file.GENERIC_WRITE, win32file.FILE_SHARE_READ, security_attributes, win32file.CREATE_ALWAYS, 0, None) startup.hStdError = win32file.CreateFile( os.path.join(this_dir, "service_stdout.log"), win32file.GENERIC_WRITE, win32file.FILE_SHARE_READ, security_attributes, win32file.CREATE_ALWAYS, 0, None) (hProcess, hThread, processId, threadId) = win32process.CreateProcess( None, r'"C:\Program Files\nodejs\node.exe" node_worker.js', None, None, True, win32process.CREATE_SUSPENDED | win32process.CREATE_BREAKAWAY_FROM_JOB, None, this_dir, startup) assert not win32job.IsProcessInJob(hProcess, None) self.hJob = win32job.CreateJobObject(None, "") extended_info = win32job.QueryInformationJobObject( self.hJob, win32job.JobObjectExtendedLimitInformation) extended_info['BasicLimitInformation'][ 'LimitFlags'] = win32job.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE | win32job.JOB_OBJECT_LIMIT_BREAKAWAY_OK win32job.SetInformationJobObject( self.hJob, win32job.JobObjectExtendedLimitInformation, extended_info) win32job.AssignProcessToJobObject(self.hJob, hProcess) win32process.ResumeThread(hThread) win32api.CloseHandle(hThread) signalled = win32event.WaitForMultipleObjects( [self.hWaitStop, hProcess], False, win32event.INFINITE) if signalled == win32event.WAIT_OBJECT_0 + 1 and win32process.GetExitCodeProcess( hProcess) != 0: servicemanager.LogErrorMsg( self._svc_name_ + " process exited with non-zero status " + str(win32process.GetExitCodeProcess(hProcess))) win32api.CloseHandle(hProcess) win32api.CloseHandle(self.hJob) win32api.CloseHandle(self.hWaitStop) win32api.CloseHandle(startup.hStdOutput) win32api.CloseHandle(startup.hStdError)
def create_job(hProcess): hJob = win32job.CreateJobObject(None, "") extended_info = win32job.QueryInformationJobObject(hJob, win32job.JobObjectExtendedLimitInformation) extended_info['BasicLimitInformation']['LimitFlags'] = win32job.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE win32job.SetInformationJobObject(hJob, win32job.JobObjectExtendedLimitInformation, extended_info) win32job.AssignProcessToJobObject(hJob, hProcess) return hJob
def attach_queue_to_stdout(self): start_ts = time.time() while time.time() - start_ts < self.START_SECONDS_DEFAULT: if self.is_suprocess_started: hJob = win32job.CreateJobObject(None, "") extended_info = win32job.QueryInformationJobObject( hJob, win32job.JobObjectExtendedLimitInformation) extended_info['BasicLimitInformation'][ 'LimitFlags'] = win32job.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE win32job.SetInformationJobObject( hJob, win32job.JobObjectExtendedLimitInformation, extended_info) perms = win32con.PROCESS_TERMINATE | win32con.PROCESS_SET_QUOTA hProcess = win32api.OpenProcess(perms, False, self._pipe.pid) win32job.AssignProcessToJobObject(hJob, hProcess) self.log.debug("attaching queue to stdout") while True: try: if self._stopped: break if self._start_failed: break gc.disable() nextline = self._pipe.stdout.readline() if nextline == '': # and self._pipe.poll() is not None: time.sleep(0.05) continue self.log.debug("got from stdout: %s" % nextline.strip()) try: self._stdout_queue.put(nextline.strip()) except Exception as e: self.log.exception( "could not put result to stdout queue, reason: %s" % e.message) gc.enable() except AttributeError: self.log.exception("stdout queue broken") break finally: gc.enable() #if self._pipe: # self._pipe.stdout.close() else: if not self._stopped: self.log.warning( "pipe is None; can't attach queue to stdout") time.sleep(0.2)
def assign_job(hjob): global g_hjob hprocess = win32api.GetCurrentProcess() try: win32job.AssignProcessToJobObject(hjob, hprocess) g_hjob = hjob except win32job.error as e: if (e.winerror != winerror.ERROR_ACCESS_DENIED or sys.getwindowsversion() >= (6, 2) or not win32job.IsProcessInJob(hprocess, None)): raise warnings.warn('The process is already in a job. Nested jobs are not ' 'supported prior to Windows 8.')
def start(self): """ Starts the process and the logger pipes for its stdout and stderr. """ creation_flags = 0 if sys.platform == "win32" and _JOB_OBJECT is not None: creation_flags |= win32process.CREATE_BREAKAWAY_FROM_JOB # Use unbuffered I/O pipes to avoid adding delay between when the subprocess writes output # and when the LoggerPipe thread reads it. buffer_size = 0 # Close file descriptors in the child process before executing the program. This prevents # file descriptors that were inherited due to multiple calls to fork() -- either within one # thread, or concurrently from multiple threads -- from causing another subprocess to wait # for the completion of the newly spawned child process. Closing other file descriptors # isn't supported on Windows when stdout and stderr are redirected. close_fds = (sys.platform != "win32") with _POPEN_LOCK: self._process = subprocess.Popen(self.args, bufsize=buffer_size, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=close_fds, env=self.env, creationflags=creation_flags) self.pid = self._process.pid self._stdout_pipe = pipe.LoggerPipe(self.logger, logging.INFO, self._process.stdout) self._stderr_pipe = pipe.LoggerPipe(self.logger, logging.ERROR, self._process.stderr) self._stdout_pipe.wait_until_started() self._stderr_pipe.wait_until_started() if sys.platform == "win32" and _JOB_OBJECT is not None: try: win32job.AssignProcessToJobObject(_JOB_OBJECT, self._process._handle) except win32job.error as err: # ERROR_ACCESS_DENIED (winerror=5) is received when the process has already died. if err.winerror != winerror.ERROR_ACCESS_DENIED: raise return_code = win32process.GetExitCodeProcess( self._process._handle) if return_code == win32con.STILL_ACTIVE: raise
def start(self): argv, env = [self.executable] + self.arguments, self.env if self.env_vars: if not env: env = os.environ.copy() env.update(self.env_vars) creation_flags = 0 if os.sys.platform == "win32": # Magic number needed to allow job reassignment in Windows 7 # see: MSDN - Process Creation Flags - ms684863 CREATE_BREAKAWAY_FROM_JOB = 0x01000000 creation_flags = CREATE_BREAKAWAY_FROM_JOB stdout = sys.stdout if not self.logger else subprocess.PIPE stderr = sys.stderr if not self.logger else subprocess.PIPE self.subprocess = subprocess.Popen(argv, env=env, creationflags=creation_flags, stdout=stdout, stderr=stderr) if stdout == subprocess.PIPE: self.stdout_logger = LoggerPipe(self.logger, logging.INFO, self.subprocess.stdout) self.stdout_logger.wait_until_started() if stderr == subprocess.PIPE: self.stderr_logger = LoggerPipe(self.logger, logging.ERROR, self.subprocess.stderr) self.stderr_logger.wait_until_started() if os.sys.platform == "win32": # Create a job object with the "kill on job close" flag # This is inherited by child processes (i.e. the mongod started on our behalf by # buildlogger) and lets us terminate the whole tree of processes rather than # orphaning the mongod. import win32job job_object = win32job.CreateJobObject(None, '') job_info = win32job.QueryInformationJobObject( job_object, win32job.JobObjectExtendedLimitInformation) job_info['BasicLimitInformation']['LimitFlags'] |= \ win32job.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE win32job.SetInformationJobObject(job_object, win32job.JobObjectExtendedLimitInformation, job_info) win32job.AssignProcessToJobObject(job_object, proc._handle) self.subprocess_job_object = job_object
def _launch_worker(self, wait=False): worker = Worker(self._push_server_address, self._pull_server_address, self._ipc_authkey) worker.start() if sys.platform == 'win32': permissions = win32con.PROCESS_TERMINATE | win32con.PROCESS_SET_QUOTA handle = win32api.OpenProcess(permissions, False, worker.pid) try: win32job.AssignProcessToJobObject(self._job_obj, handle) finally: win32api.CloseHandle(handle) if wait: worker.wait_start() self.workers.append(worker) return worker
def job(self) -> WIN32JOB: """Get the job associated with this process. Caches between calls for safety. Returns ------- WIN32JOB A Windows job which consists of one or more processes. In this case we just have the one process """ if not hasattr(self, "_job"): # First try to import stuff, an easy exception to catch and give good # information about try: import win32api import win32job import winerror except ModuleNotFoundError as e: raise ModuleNotFoundError(_win32_import_error_msg) from e # Here, we assign it to a windows "Job", whatever that is # If the process is already assigned to a job, then we have # to check if it's less than Windows 8 because apparently # nested jobs aren't supported there try: job = win32job.CreateJobObject(None, "") process = win32api.GetCurrentProcess() win32job.AssignProcessToJobObject(job, process) self._job = job except win32job.error as e: if (e.winerror != winerror.ERROR_ACCESS_DENIED or sys.getwindowsversion() >= (6, 2) # type: ignore or not win32job.IsProcessInJob(process, None)): raise e msg = ("The process is already in a job." " Nested jobs are not supported prior to Windows 8.") raise RuntimeError(msg) from e return self._job
def __init__(self, *args, **kwargs): kwargs = dict(_Popen_defaults + kwargs.items()) if 'creationflagsmerge' in kwargs: kwargs['creationflags'] = (kwargs.get('creationflags', 0) | kwargs['creationflagsmerge']) del kwargs['creationflagsmerge'] for f in ['stdout', 'stderr']: if kwargs[f] is SINK: kwargs[f] = create_sink() # super() does some magic that makes **kwargs not work, so just call # our super-constructor directly subprocess.Popen.__init__(self, *args, **kwargs) _CHILD_PROCS.append(self) if mswindows and _kill_children_on_death: handle = windll.kernel32.OpenProcess( win32con.SYNCHRONIZE | win32con.PROCESS_SET_QUOTA | win32con.PROCESS_TERMINATE, 0, self.pid) if win32job.AssignProcessToJobObject(_chJob, handle) == 0: raise WinError()
def create_job(hProcess): ''' create_job(hProcess) creates win32job with correct flags: Note on JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE: https://msdn.microsoft.com/en-us/library/windows/desktop/ms684161(v=vs.85).aspx However, if the job has the JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE flag specified, closing the last job object handle terminates all associated processes and then destroys the job object itself. ''' hJob = win32job.CreateJobObject(None, "") extended_info = win32job.QueryInformationJobObject( hJob, win32job.JobObjectExtendedLimitInformation) extended_info['BasicLimitInformation'][ 'LimitFlags'] = win32job.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE win32job.SetInformationJobObject( hJob, win32job.JobObjectExtendedLimitInformation, extended_info) win32job.AssignProcessToJobObject(hJob, hProcess) return hJob
def __register_job_object(self): if self.is_windows8_or_above_flag or not self.is_cloud_mode_flag: self.child = None self.hJob = None self.hProcess = None self.hJob = win32job.CreateJobObject(None, self.win32_job_name) extended_info = win32job.QueryInformationJobObject( self.hJob, win32job.JobObjectExtendedLimitInformation) extended_info['BasicLimitInformation'][ 'LimitFlags'] = win32job.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE win32job.SetInformationJobObject( self.hJob, win32job.JobObjectExtendedLimitInformation, extended_info) # Convert process id to process handle: perms = win32con.PROCESS_TERMINATE | win32con.PROCESS_SET_QUOTA self.hProcess = win32api.OpenProcess(perms, False, self.__p.pid) try: win32job.AssignProcessToJobObject(self.hJob, self.hProcess) self.__user_job_object_flag = True except Exception as e: self.__user_job_object_flag = False else: self.__user_job_object_flag = False
def SvcDoRun(self): import servicemanager servicemanager.LogInfoMsg(self._svc_name_ + " Start Requested") try: hJob = win32job.CreateJobObject(None, "") extended_info = win32job.QueryInformationJobObject(hJob, win32job.JobObjectExtendedLimitInformation) extended_info['BasicLimitInformation']['LimitFlags'] = win32job.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE win32job.SetInformationJobObject(hJob, win32job.JobObjectExtendedLimitInformation, extended_info) command = "resilient-circuits.exe run " + self._resilient_args_ command_args = shlex.split(command) self.process_handle = subprocess.Popen(command_args) # Convert process id to process handle: perms = win32con.PROCESS_TERMINATE | win32con.PROCESS_SET_QUOTA hProcess = win32api.OpenProcess(perms, False, self.process_handle.pid) win32job.AssignProcessToJobObject(hJob, hProcess) except: servicemanager.LogErrorMsg(self._svc_name_ + " failed to launch resilient-circuits.exe") raise servicemanager.LogInfoMsg(self._svc_name_ + " Started") while self.isAlive: if self.process_handle.poll() != None: self.SvcStop() win32api.SleepEx(10000, True)
def _start(self, argv): """In most cases, just call subprocess.Popen(). On windows, add the started process to a new Job Object, so that any child processes of this process can be killed with a single call to TerminateJobObject (see self.stop()). """ if valgrind: argv = [ 'buildscripts/valgrind.bash', '--show-reachable=yes', '--leak-check=full', '--suppressions=valgrind.suppressions' ] + argv elif drd: argv = ['buildscripts/valgrind.bash', '--tool=drd'] + argv proc = Popen(argv, stdout=self.outfile) if os.sys.platform == "win32": # Create a job object with the "kill on job close" # flag; this is inherited by child processes (ie # the mongod started on our behalf by buildlogger) # and lets us terminate the whole tree of processes # rather than orphaning the mongod. import win32job self.job_object = win32job.CreateJobObject(None, '') job_info = win32job.QueryInformationJobObject( self.job_object, win32job.JobObjectExtendedLimitInformation) job_info['BasicLimitInformation'][ 'LimitFlags'] |= win32job.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE win32job.SetInformationJobObject( self.job_object, win32job.JobObjectExtendedLimitInformation, job_info) win32job.AssignProcessToJobObject(self.job_object, proc._handle) return proc
def add_child(self, pid): """Adds the child process to the job""" # Convert process id to process handle perms = win32con.PROCESS_TERMINATE | win32con.PROCESS_SET_QUOTA hProcess = win32api.OpenProcess(perms, False, pid) win32job.AssignProcessToJobObject(self.hJob, hProcess)
# allows python child process to print unicode characters to console without UnicodeEncodeError exception my_env = os.environ my_env['PYTHONIOENCODING'] = 'utf-8' # TODO: find more portable solution without setting enviroment variable. # Entering command in cmd.exe shell + Enter button does allow to print unicode characters # PYTHONIOENCODING enviroment variable is supposed to work only with python children processes if DEF_universal_newlines_True[0]: proc=subprocess.Popen(sppd_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, creationflags = subprocess.CREATE_NO_WINDOW, env = my_env, universal_newlines=True) else: proc=subprocess.Popen(sppd_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, creationflags = subprocess.CREATE_NO_WINDOW, env = my_env) real_sppd_cmd = psutil.Process(proc.pid).cmdline() runner_hProcess = win32api.OpenProcess(perms, False, proc.pid) win32job.AssignProcessToJobObject(hJob, runner_hProcess) print("--- runtsharksppdpipe.py configured to terminate if wrapper.py dies ---", flush=True) ## print('proc.stdout.fileno() = ' + repr(proc.stdout.fileno()), flush = True) tmpe = None try: if DEF_universal_newlines_True[0]: while proc.poll() is None: line = proc.stdout.readline() if line: print(line, flush = True, end='') else: while proc.poll() is None: bytes_variable = os.read(proc.stdout.fileno(), 2 << 16) if bytes_variable:
def proc(args, time_limit, mem_size, time_elapsed, return_val, _stdin_name, _stdout_name, _stderr_name): """测试的 runner 函数 Args: args: 待测程序及其参数的列表 time_limit: float,秒为单位的运行时间限制 mem_size: int,byte 为单位的运行内存限制,零值为不限制 time_elapsed: double 类型的 multiprocessing 中的 shared_ctypes 对象 用以存储并返回实际运行时间 return_val: int 类型的 multiprocessing 中的 shared_ctypes 对象 用以存储并返回待测程序的返回值 _stdin_name, _stdout_name, _stderr_name: 为文件路径 分别为待测程序的标准输入、输出、错误输出的重定向目标 """ if mem_size: try: if _mswindows: import win32api import win32job job = win32job.CreateJobObject(None, "judge_mem_limiter") win32job.SetInformationJobObject( job, win32job.JobObjectExtendedLimitInformation, { "BasicLimitInformation": { "PerProcessUserTimeLimit": 0, "PerJobUserTimeLimit": 0, "LimitFlags": win32job.JOB_OBJECT_LIMIT_PROCESS_MEMORY, "MinimumWorkingSetSize": 0, "MaximumWorkingSetSize": 0, "ActiveProcessLimit": 0, "Affinity": 0, "PriorityClass": 0, "SchedulingClass": 0 }, "IoInfo": { "ReadOperationCount": 0, "WriteOperationCount": 0, "OtherOperationCount": 0, "ReadTransferCount": 0, "WriteTransferCount": 0, "OtherTransferCount": 0, }, "JobMemoryLimit": 0, "PeakProcessMemoryUsed": 0, "PeakJobMemoryUsed": 0, "ProcessMemoryLimit": mem_size }) win32job.AssignProcessToJobObject(job, win32api.GetCurrentProcess()) else: import resource resource.setrlimit(resource.RLIMIT_DATA, (mem_size, mem_size)) except: _logger.error("unable to set memory limit under win32: %s", traceback.format_exc()) with open(_stdin_name, "rb") as stdin,\ open(_stdout_name, "wb") as stdout,\ open(_stderr_name, "wb") as stderr: proc = subprocess.Popen(args, stdin=stdin, stdout=stdout, stderr=stderr) t_start = time.perf_counter() try: proc.wait(timeout=time_limit + 1.) except subprocess.TimeoutExpired: proc.kill() proc.returncode = -1 t_end = time.perf_counter() time_elapsed.value = t_end - t_start return_val.value = proc.returncode
def startBackgroundProcess(self): """Method to start a process running in the background. """ with process_lock: # security attributes for pipes sAttrs = win32security.SECURITY_ATTRIBUTES() sAttrs.bInheritHandle = 1 # create pipes for the process to write to hStdin_r, hStdin = win32pipe.CreatePipe(sAttrs, 0) hStdout = win32file.CreateFile( self.stdout, win32file.GENERIC_WRITE | win32file.GENERIC_READ, win32file.FILE_SHARE_DELETE | win32file.FILE_SHARE_READ | win32file.FILE_SHARE_WRITE, sAttrs, win32file.CREATE_ALWAYS, win32file.FILE_ATTRIBUTE_NORMAL, None) hStderr = win32file.CreateFile( self.stderr, win32file.GENERIC_WRITE | win32file.GENERIC_READ, win32file.FILE_SHARE_DELETE | win32file.FILE_SHARE_READ | win32file.FILE_SHARE_WRITE, sAttrs, win32file.CREATE_ALWAYS, win32file.FILE_ATTRIBUTE_NORMAL, None) try: # set the info structure for the new process. StartupInfo = win32process.STARTUPINFO() StartupInfo.hStdInput = hStdin_r StartupInfo.hStdOutput = hStdout StartupInfo.hStdError = hStderr StartupInfo.dwFlags = win32process.STARTF_USESTDHANDLES # Create new handles for the thread ends of the pipes. The duplicated handles will # have their inheritence properties set to false so that any children inheriting these # handles will not have non-closeable handles to the pipes pid = win32api.GetCurrentProcess() tmp = win32api.DuplicateHandle(pid, hStdin, pid, 0, 0, win32con.DUPLICATE_SAME_ACCESS) win32file.CloseHandle(hStdin) hStdin = tmp # start the process, and close down the copies of the process handles # we have open after the process creation (no longer needed here) old_command = command = self.__quotePath(self.command) for arg in self.arguments: command = '%s %s' % (command, self.__quotePath(arg)) # Windows CreateProcess maximum lpCommandLine length is 32,768 # http://msdn.microsoft.com/en-us/library/ms682425%28VS.85%29.aspx if len(command) >= 32768: # pragma: no cover raise ValueError( "Command line length exceeded 32768 characters: %s..." % command[:1000]) dwCreationFlags = 0 if IS_PRE_WINDOWS_8: # pragma: no cover # In case PySys is itself running in a job, might need to explicitly breakaway from it so we can give # it its own, but only for old pre-windows 8/2012, which support nested jobs dwCreationFlags = dwCreationFlags | win32process.CREATE_BREAKAWAY_FROM_JOB if self.command.lower().endswith(('.bat', '.cmd')): # If we don't start suspended there's a slight race condition but due to some issues with # initially-suspended processes hanging (seen many years ago), to be safe, only bother to close the # race condition for shell scripts (which is the main use case for this anyway) dwCreationFlags = dwCreationFlags | win32con.CREATE_SUSPENDED self.__job = self._createParentJob() try: self.__hProcess, self.__hThread, self.pid, self.__tid = win32process.CreateProcess( None, command, None, None, 1, dwCreationFlags, self.environs, os.path.normpath(self.workingDir), StartupInfo) except pywintypes.error as e: raise ProcessError("Error creating process %s: %s" % (old_command, e)) try: if not self.disableKillingChildProcesses: win32job.AssignProcessToJobObject( self.__job, self.__hProcess) else: self.__job = None # pragma: no cover except Exception as e: # pragma: no cover # Shouldn't fail unless process already terminated (which can happen since # if we didn't use SUSPENDED there's an inherent race here) if win32process.GetExitCodeProcess( self.__hProcess) == win32con.STILL_ACTIVE: log.warning( 'Failed to associate process %s with new job: %s (this may prevent automatic cleanup of child processes)' % (self, e)) # force use of TerminateProcess not TerminateJobObject if this failed self.__job = None if (dwCreationFlags & win32con.CREATE_SUSPENDED) != 0: win32process.ResumeThread(self.__hThread) finally: win32file.CloseHandle(hStdin_r) win32file.CloseHandle(hStdout) win32file.CloseHandle(hStderr) # set the handle to the stdin of the process self.__stdin = hStdin
def __init__(self, script: str = "", ahk_path: Path = None, execute_from: Path = None) -> None: self.file = None self.script = script if ahk_path is None: ahk_path = PACKAGE_PATH / r'lib\AutoHotkey\AutoHotkey.exe' assert ahk_path and ahk_path.is_file() # Windows notification area relies on consistent exe path if execute_from is not None: execute_from_dir = Path(execute_from) assert execute_from_dir.is_dir() ahk_into_folder = execute_from_dir / ahk_path.name try: if os.path.getmtime(ahk_into_folder) != os.path.getmtime( ahk_path): os.remove(ahk_into_folder) except FileNotFoundError: pass try: os.link(ahk_path, ahk_into_folder) except FileExistsError: pass except OSError as ex: # 5: "Access is denied" # 17: "The system cannot move the file to a different disk drive" if ex.winerror in (5, 17): shutil.copyfile(ahk_path, ahk_into_folder) ahk_path = ahk_into_folder self.pid = os.getpid() # if we exit, exit AutoHotkey atexit.register(self.exit) # if we terminate, terminate AutoHotkey self.job = win32job.CreateJobObject(None, "") extended_info = win32job.QueryInformationJobObject( self.job, win32job.JobObjectExtendedLimitInformation) extended_info['BasicLimitInformation'][ 'LimitFlags'] = win32job.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE win32job.SetInformationJobObject( self.job, win32job.JobObjectExtendedLimitInformation, extended_info) # add ourselves and subprocess will inherit job membership handle = win32api.OpenProcess( win32con.PROCESS_TERMINATE | win32con.PROCESS_SET_QUOTA, False, self.pid) win32job.AssignProcessToJobObject(self.job, handle) win32api.CloseHandle(handle) # user script exceptions are already caught and sent to stderr, so /ErrorStdOut would only affect debugging CORE # self.cmd = [str(ahk_path), "/ErrorStdOut=utf-16-raw", "/CP65001", "*"] self.cmd = [str(ahk_path), "/CP65001", "*"] # must pipe all three within a PyInstaller bundled exe self.popen = subprocess.Popen(self.cmd, bufsize=Script.BUFFER_SIZE, executable=str(ahk_path), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # keep grandchild processes from inheriting job membership extended_info['BasicLimitInformation'][ 'LimitFlags'] |= win32job.JOB_OBJECT_LIMIT_SILENT_BREAKAWAY_OK win32job.SetInformationJobObject( self.job, win32job.JobObjectExtendedLimitInformation, extended_info) self.popen.stdin.write(Script.CORE.encode('utf-8')) self.popen.stdin.write(self.script.encode('utf-8')) self.popen.stdin.close() self.hwnd = int(self._read_response(), 16) assert self._read_response() == "Initialized"
sys.stdout.flush() if is_win: # Create a job object and assign ourselves to it, so that # all remaining test subprocesses get killed off on completion. # This prevents AppVeyor from waiting an hour # https://stackoverflow.com/a/23587108 (and its first comment) import win32api, win32con, win32job # noqa hJob = win32job.CreateJobObject(None, "") extended_info = win32job.QueryInformationJobObject(hJob, win32job.JobObjectExtendedLimitInformation) extended_info['BasicLimitInformation']['LimitFlags'] = win32job.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE win32job.SetInformationJobObject(hJob, win32job.JobObjectExtendedLimitInformation, extended_info) perms = win32con.PROCESS_TERMINATE | win32con.PROCESS_SET_QUOTA hProcess = win32api.OpenProcess(perms, False, os.getpid()) win32job.AssignProcessToJobObject(hJob, hProcess) def find_test_keys(): if os.environ.get('CONDA_BUILD'): # The current version of conda build manually adds the activation # directories to the PATH---and then calls the standard conda # activation script, which does it again. This frustrates conda's # ability to deactivate this environment. Most package builds are # not affected by this, but we are, because our tests need to do # environment activation and deactivation. To fix this, we remove # the duplicate PATH entries conda-build added. print('BEFORE: {}'.format(os.environ['PATH'])) path_list = os.environ['PATH'].split(os.pathsep) path_dups = set() path_list = [p for p in path_list
def CreateProcess(self, *args, **kwargs): hp, ht, pid, tid = self._oldapi.CreateProcess(*args, **kwargs) win32job.AssignProcessToJobObject(self._job, hp) win32process.ResumeThread(ht) return hp, ht, pid, tid
def main(): # escape list of arguments command = _win32_arglist_to_string(sys.argv[1:]) # create job hJob = win32job.CreateJobObject(None, '') extended_info = win32job.QueryInformationJobObject( hJob, win32job.JobObjectExtendedLimitInformation) extended_info['BasicLimitInformation'][ 'LimitFlags'] = win32job.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE win32job.SetInformationJobObject( hJob, win32job.JobObjectExtendedLimitInformation, extended_info) # associate job with completion port hIoPort = win32file.CreateIoCompletionPort(win32file.INVALID_HANDLE_VALUE, None, 0, 1) # pywin32 is missing support for JOBOBJECT_ASSOCIATE_COMPLETION_PORT, therefore # we call it through ctypes port = JOBOBJECT_ASSOCIATE_COMPLETION_PORT() port.CompletionKey = hJob.handle port.CompletionPort = hIoPort.handle assert bool( ctypes.windll.kernel32.SetInformationJobObject( ctypes.wintypes.HANDLE(hJob.handle), ctypes.c_int(JobObjectAssociateCompletionPortInformation), ctypes.byref(port), ctypes.sizeof(JOBOBJECT_ASSOCIATE_COMPLETION_PORT))) # create process suspended si = win32process.STARTUPINFO() hProcess, hThread, processId, threadId = win32process.CreateProcess( None, command, None, None, True, win32process.CREATE_BREAKAWAY_FROM_JOB | win32process.CREATE_SUSPENDED, None, None, si) # add process to job win32job.AssignProcessToJobObject(hJob, hProcess) # resume process win32process.ResumeThread(hThread) win32api.CloseHandle(hThread) win32api.CloseHandle(hProcess) # wait for job termination numberOfBytes = ctypes.wintypes.DWORD(0) completionKey = ctypes.wintypes.HANDLE(0) overlapped = OVERLAPPED() while True: # calling this through pywin32 crashes the program, therefore we call it through ctypes res = bool( ctypes.windll.kernel32.GetQueuedCompletionStatus( ctypes.wintypes.HANDLE(hIoPort.handle), ctypes.byref(numberOfBytes), ctypes.byref(completionKey), ctypes.byref(overlapped), ctypes.wintypes.DWORD(win32event.INFINITE))) if not res or (bytes(completionKey) == bytes( ctypes.c_void_p(hJob.handle)) and bytes(numberOfBytes) == bytes( ctypes.c_ulong( win32job.JOB_OBJECT_MSG_ACTIVE_PROCESS_ZERO))): break
def start(self): """Start the process and the logger pipes for its stdout and stderr.""" creation_flags = 0 if sys.platform == "win32" and _JOB_OBJECT is not None: creation_flags |= win32process.CREATE_BREAKAWAY_FROM_JOB # Tests fail if a process takes too long to startup and listen to a socket. Use buffered # I/O pipes to give the process some leeway. buffer_size = 1024 * 1024 # Close file descriptors in the child process before executing the program. This prevents # file descriptors that were inherited due to multiple calls to fork() -- either within one # thread, or concurrently from multiple threads -- from causing another subprocess to wait # for the completion of the newly spawned child process. Closing other file descriptors # isn't supported on Windows when stdout and stderr are redirected. close_fds = (sys.platform != "win32") with _POPEN_LOCK: # Record unittests directly since resmoke doesn't not interact with them and they can finish # too quickly for the recorder to have a chance at attaching. recorder_args = [] if _config.UNDO_RECORDER_PATH is not None and self.args[ 0].endswith("_test"): now_str = datetime.now().strftime("%Y-%m-%dT%H-%M-%S") # Only use the process name since we have to be able to correlate the recording name # with the binary name easily. recorder_output_file = "{process}-{t}.undo".format( process=os.path.basename(self.args[0]), t=now_str) recorder_args = [ _config.UNDO_RECORDER_PATH, "-o", recorder_output_file ] self._process = subprocess.Popen(recorder_args + self.args, bufsize=buffer_size, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=close_fds, env=self.env, creationflags=creation_flags, cwd=self._cwd) self.pid = self._process.pid if _config.UNDO_RECORDER_PATH is not None and ( not self.args[0].endswith("_test")) and ( "mongod" in self.args[0] or "mongos" in self.args[0]): now_str = datetime.now().strftime("%Y-%m-%dT%H-%M-%S") recorder_output_file = "{logger}-{process}-{pid}-{t}.undo".format( logger=self.logger.name.replace('/', '-'), process=os.path.basename(self.args[0]), pid=self.pid, t=now_str) recorder_args = [ _config.UNDO_RECORDER_PATH, "-p", str(self.pid), "-o", recorder_output_file ] self._recorder = subprocess.Popen(recorder_args, bufsize=buffer_size, env=self.env, creationflags=creation_flags) self._stdout_pipe = pipe.LoggerPipe(self.logger, logging.INFO, self._process.stdout) self._stderr_pipe = pipe.LoggerPipe(self.logger, logging.ERROR, self._process.stderr) self._stdout_pipe.wait_until_started() self._stderr_pipe.wait_until_started() if sys.platform == "win32" and _JOB_OBJECT is not None: try: win32job.AssignProcessToJobObject(_JOB_OBJECT, self._process._handle) except win32job.error as err: # ERROR_ACCESS_DENIED (winerror=5) is received when the process has already died. if err.winerror != winerror.ERROR_ACCESS_DENIED: raise return_code = win32process.GetExitCodeProcess( self._process._handle) if return_code == win32con.STILL_ACTIVE: raise