def run(self):
logger.info("Started App usage tracker thread")
file_loc = os.path.join(os.path.dirname(__file__), "apps-usage",
"apps-usage.exe")
try:
hJob = win32job.CreateJobObject(None, "")
extended_info = win32job.QueryInformationJobObject(
hJob, win32job.JobObjectExtendedLimitInformation)
extended_info['BasicLimitInformation'][
'LimitFlags'] = win32job.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
win32job.SetInformationJobObject(
hJob, win32job.JobObjectExtendedLimitInformation,
extended_info)
p = subprocess.Popen(f"{file_loc} \"{ROOT_DIR}\"", shell=False)
perms = win32con.PROCESS_TERMINATE | win32con.PROCESS_SET_QUOTA
hProcess = win32api.OpenProcess(perms, False, p.pid)
win32job.AssignProcessToJobObject(hJob, hProcess)
while True and not self.kill:
time.sleep(0.5)
p.terminate()
p.kill()
except Exception as e:
logger.error(f"Error Starting Apps-Usage. {e} ")
logger.warning("Stopped app usage tracking thread")
def start(self):
"""
Starts the process and the logger pipes for its stdout and
stderr.
"""
creation_flags = 0
if sys.platform == "win32" and _JOB_OBJECT is not None:
creation_flags |= win32process.CREATE_BREAKAWAY_FROM_JOB
with _POPEN_LOCK:
self._process = subprocess.Popen(self.args,
env=self.env,
creationflags=creation_flags,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
self.pid = self._process.pid
self._stdout_pipe = pipe.LoggerPipe(self.logger, logging.INFO, self._process.stdout)
self._stderr_pipe = pipe.LoggerPipe(self.logger, logging.ERROR, self._process.stderr)
self._stdout_pipe.wait_until_started()
self._stderr_pipe.wait_until_started()
if sys.platform == "win32" and _JOB_OBJECT is not None:
try:
win32job.AssignProcessToJobObject(_JOB_OBJECT, self._process._handle)
except win32job.error as err:
# ERROR_ACCESS_DENIED (winerror=5) is received when the process has already died.
if err.winerror != winerror.ERROR_ACCESS_DENIED:
raise
return_code = win32process.GetExitCodeProcess(self._process._handle)
if return_code == win32con.STILL_ACTIVE:
raise
def start(self):
"""Start the process and the logger pipes for its stdout and stderr."""
creation_flags = 0
if sys.platform == "win32" and _JOB_OBJECT is not None:
creation_flags |= win32process.CREATE_BREAKAWAY_FROM_JOB
# Tests fail if a process takes too long to startup and listen to a socket. Use buffered
# I/O pipes to give the process some leeway.
buffer_size = 1024 * 1024
# Close file descriptors in the child process before executing the program. This prevents
# file descriptors that were inherited due to multiple calls to fork() -- either within one
# thread, or concurrently from multiple threads -- from causing another subprocess to wait
# for the completion of the newly spawned child process. Closing other file descriptors
# isn't supported on Windows when stdout and stderr are redirected.
close_fds = (sys.platform != "win32")
with _POPEN_LOCK:
self._process = subprocess.Popen(self.args,
bufsize=buffer_size,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=close_fds,
env=self.env,
creationflags=creation_flags,
cwd=self._cwd)
self.pid = self._process.pid
if _config.UNDO_RECORDER_PATH is not None and (
"mongod" in self.args[0] or "mongos" in self.args[0]):
recorder_args = [
_config.UNDO_RECORDER_PATH, "--thread-fuzzing", "-p",
str(self.pid)
]
self._recorder = subprocess.Popen(recorder_args,
bufsize=buffer_size,
env=self.env,
creationflags=creation_flags)
self._stdout_pipe = pipe.LoggerPipe(self.logger, logging.INFO,
self._process.stdout)
self._stderr_pipe = pipe.LoggerPipe(self.logger, logging.ERROR,
self._process.stderr)
self._stdout_pipe.wait_until_started()
self._stderr_pipe.wait_until_started()
if sys.platform == "win32" and _JOB_OBJECT is not None:
try:
win32job.AssignProcessToJobObject(_JOB_OBJECT,
self._process._handle)
except win32job.error as err:
# ERROR_ACCESS_DENIED (winerror=5) is received when the process has already died.
if err.winerror != winerror.ERROR_ACCESS_DENIED:
raise
return_code = win32process.GetExitCodeProcess(
self._process._handle)
if return_code == win32con.STILL_ACTIVE:
raise
def _start(self, argv):
"""In most cases, just call subprocess.Popen(). On windows,
add the started process to a new Job Object, so that any
child processes of this process can be killed with a single
call to TerminateJobObject (see self.stop()).
"""
proc = Popen(argv)
if os.sys.platform == "win32":
# Create a job object with the "kill on job close"
# flag; this is inherited by child processes (ie
# the mongod started on our behalf by buildlogger)
# and lets us terminate the whole tree of processes
# rather than orphaning the mongod.
import win32job
self.job_object = win32job.CreateJobObject(None, '')
job_info = win32job.QueryInformationJobObject(
self.job_object, win32job.JobObjectExtendedLimitInformation)
job_info['BasicLimitInformation'][
'LimitFlags'] |= win32job.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
win32job.SetInformationJobObject(
self.job_object, win32job.JobObjectExtendedLimitInformation,
job_info)
win32job.AssignProcessToJobObject(self.job_object, proc._handle)
return proc
def _add_to_job_object(self):
global _global_process_job_handle
if _global_process_job_handle is not None:
#This means that we are creating another process family - we'll all be in the same job
return
already_in_job = win32job.IsProcessInJob(win32api.GetCurrentProcess(), None)
#Create a new job and put us in it before we create any children
logger.debug("Creating job object and adding parent process to it")
security_attrs = win32security.SECURITY_ATTRIBUTES()
security_attrs.bInheritHandle = 0
_global_process_job_handle = win32job.CreateJobObject(security_attrs, self.get_job_object_name())
extended_info = win32job.QueryInformationJobObject(_global_process_job_handle, win32job.JobObjectExtendedLimitInformation)
extended_info['BasicLimitInformation']['LimitFlags'] = win32job.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
win32job.SetInformationJobObject(_global_process_job_handle, win32job.JobObjectExtendedLimitInformation, extended_info)
try:
win32job.AssignProcessToJobObject(_global_process_job_handle, win32api.GetCurrentProcess())
except Exception as e:
winv = sys.getwindowsversion()
logger.error("Error raised during assignment of the current process to a new job object. " +\
"The process %s already in a job. " +\
"The windows version is %d.%d.\nError: %s",
"is" if already_in_job else "is not",
winv.major,
winv.minor,
_exception_str())
if already_in_job and not (winv.major >= 6 and winv.minor >= 2):
raise JobObjectAssignError("On Windows versions older than Windows 8 / Windows Server 2012, ProcessFamily relies on the parent process NOT being in a job already", e, already_in_job)
raise JobObjectAssignError("Error raised during assignment of the current process to a new job object.", e, already_in_job)
logger.debug("Added to job object")
def SvcDoRun(self):
if hasattr(sys, "frozen"):
this_dir = os.path.dirname(win32api.GetModuleFileName(None))
else:
this_dir = os.path.dirname(os.path.abspath(__file__))
# TODO: maybe it is better to run this in a job object too
with open(os.path.join(this_dir, 'npm.log'), 'w') as npm_log:
subprocess.check_call('npm install',
cwd=this_dir,
shell=True,
stdin=None,
stdout=npm_log,
stderr=subprocess.STDOUT)
security_attributes = win32security.SECURITY_ATTRIBUTES()
security_attributes.bInheritHandle = True
startup = win32process.STARTUPINFO()
startup.dwFlags |= win32process.STARTF_USESTDHANDLES
startup.hStdInput = None
startup.hStdOutput = win32file.CreateFile(
os.path.join(this_dir, "service_stderr.log"),
win32file.GENERIC_WRITE, win32file.FILE_SHARE_READ,
security_attributes, win32file.CREATE_ALWAYS, 0, None)
startup.hStdError = win32file.CreateFile(
os.path.join(this_dir, "service_stdout.log"),
win32file.GENERIC_WRITE, win32file.FILE_SHARE_READ,
security_attributes, win32file.CREATE_ALWAYS, 0, None)
(hProcess, hThread, processId, threadId) = win32process.CreateProcess(
None, r'"C:\Program Files\nodejs\node.exe" node_worker.js', None,
None, True, win32process.CREATE_SUSPENDED
| win32process.CREATE_BREAKAWAY_FROM_JOB, None, this_dir, startup)
assert not win32job.IsProcessInJob(hProcess, None)
self.hJob = win32job.CreateJobObject(None, "")
extended_info = win32job.QueryInformationJobObject(
self.hJob, win32job.JobObjectExtendedLimitInformation)
extended_info['BasicLimitInformation'][
'LimitFlags'] = win32job.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE | win32job.JOB_OBJECT_LIMIT_BREAKAWAY_OK
win32job.SetInformationJobObject(
self.hJob, win32job.JobObjectExtendedLimitInformation,
extended_info)
win32job.AssignProcessToJobObject(self.hJob, hProcess)
win32process.ResumeThread(hThread)
win32api.CloseHandle(hThread)
signalled = win32event.WaitForMultipleObjects(
[self.hWaitStop, hProcess], False, win32event.INFINITE)
if signalled == win32event.WAIT_OBJECT_0 + 1 and win32process.GetExitCodeProcess(
hProcess) != 0:
servicemanager.LogErrorMsg(
self._svc_name_ + " process exited with non-zero status " +
str(win32process.GetExitCodeProcess(hProcess)))
win32api.CloseHandle(hProcess)
win32api.CloseHandle(self.hJob)
win32api.CloseHandle(self.hWaitStop)
win32api.CloseHandle(startup.hStdOutput)
win32api.CloseHandle(startup.hStdError)
def create_job(hProcess):
hJob = win32job.CreateJobObject(None, "")
extended_info = win32job.QueryInformationJobObject(hJob, win32job.JobObjectExtendedLimitInformation)
extended_info['BasicLimitInformation']['LimitFlags'] = win32job.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
win32job.SetInformationJobObject(hJob, win32job.JobObjectExtendedLimitInformation, extended_info)
win32job.AssignProcessToJobObject(hJob, hProcess)
return hJob
def attach_queue_to_stdout(self):
start_ts = time.time()
while time.time() - start_ts < self.START_SECONDS_DEFAULT:
if self.is_suprocess_started:
hJob = win32job.CreateJobObject(None, "")
extended_info = win32job.QueryInformationJobObject(
hJob, win32job.JobObjectExtendedLimitInformation)
extended_info['BasicLimitInformation'][
'LimitFlags'] = win32job.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
win32job.SetInformationJobObject(
hJob, win32job.JobObjectExtendedLimitInformation,
extended_info)
perms = win32con.PROCESS_TERMINATE | win32con.PROCESS_SET_QUOTA
hProcess = win32api.OpenProcess(perms, False, self._pipe.pid)
win32job.AssignProcessToJobObject(hJob, hProcess)
self.log.debug("attaching queue to stdout")
while True:
try:
if self._stopped:
break
if self._start_failed:
break
gc.disable()
nextline = self._pipe.stdout.readline()
if nextline == '': # and self._pipe.poll() is not None:
time.sleep(0.05)
continue
self.log.debug("got from stdout: %s" %
nextline.strip())
try:
self._stdout_queue.put(nextline.strip())
except Exception as e:
self.log.exception(
"could not put result to stdout queue, reason: %s"
% e.message)
gc.enable()
except AttributeError:
self.log.exception("stdout queue broken")
break
finally:
gc.enable()
#if self._pipe:
# self._pipe.stdout.close()
else:
if not self._stopped:
self.log.warning(
"pipe is None; can't attach queue to stdout")
time.sleep(0.2)
def assign_job(hjob):
global g_hjob
hprocess = win32api.GetCurrentProcess()
try:
win32job.AssignProcessToJobObject(hjob, hprocess)
g_hjob = hjob
except win32job.error as e:
if (e.winerror != winerror.ERROR_ACCESS_DENIED
or sys.getwindowsversion() >= (6, 2)
or not win32job.IsProcessInJob(hprocess, None)):
raise
warnings.warn('The process is already in a job. Nested jobs are not '
'supported prior to Windows 8.')
def start(self):
"""
Starts the process and the logger pipes for its stdout and
stderr.
"""
creation_flags = 0
if sys.platform == "win32" and _JOB_OBJECT is not None:
creation_flags |= win32process.CREATE_BREAKAWAY_FROM_JOB
# Use unbuffered I/O pipes to avoid adding delay between when the subprocess writes output
# and when the LoggerPipe thread reads it.
buffer_size = 0
# Close file descriptors in the child process before executing the program. This prevents
# file descriptors that were inherited due to multiple calls to fork() -- either within one
# thread, or concurrently from multiple threads -- from causing another subprocess to wait
# for the completion of the newly spawned child process. Closing other file descriptors
# isn't supported on Windows when stdout and stderr are redirected.
close_fds = (sys.platform != "win32")
with _POPEN_LOCK:
self._process = subprocess.Popen(self.args,
bufsize=buffer_size,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=close_fds,
env=self.env,
creationflags=creation_flags)
self.pid = self._process.pid
self._stdout_pipe = pipe.LoggerPipe(self.logger, logging.INFO,
self._process.stdout)
self._stderr_pipe = pipe.LoggerPipe(self.logger, logging.ERROR,
self._process.stderr)
self._stdout_pipe.wait_until_started()
self._stderr_pipe.wait_until_started()
if sys.platform == "win32" and _JOB_OBJECT is not None:
try:
win32job.AssignProcessToJobObject(_JOB_OBJECT,
self._process._handle)
except win32job.error as err:
# ERROR_ACCESS_DENIED (winerror=5) is received when the process has already died.
if err.winerror != winerror.ERROR_ACCESS_DENIED:
raise
return_code = win32process.GetExitCodeProcess(
self._process._handle)
if return_code == win32con.STILL_ACTIVE:
raise
def start(self):
argv, env = [self.executable] + self.arguments, self.env
if self.env_vars:
if not env:
env = os.environ.copy()
env.update(self.env_vars)
creation_flags = 0
if os.sys.platform == "win32":
# Magic number needed to allow job reassignment in Windows 7
# see: MSDN - Process Creation Flags - ms684863
CREATE_BREAKAWAY_FROM_JOB = 0x01000000
creation_flags = CREATE_BREAKAWAY_FROM_JOB
stdout = sys.stdout if not self.logger else subprocess.PIPE
stderr = sys.stderr if not self.logger else subprocess.PIPE
self.subprocess = subprocess.Popen(argv, env=env, creationflags=creation_flags,
stdout=stdout, stderr=stderr)
if stdout == subprocess.PIPE:
self.stdout_logger = LoggerPipe(self.logger, logging.INFO, self.subprocess.stdout)
self.stdout_logger.wait_until_started()
if stderr == subprocess.PIPE:
self.stderr_logger = LoggerPipe(self.logger, logging.ERROR, self.subprocess.stderr)
self.stderr_logger.wait_until_started()
if os.sys.platform == "win32":
# Create a job object with the "kill on job close" flag
# This is inherited by child processes (i.e. the mongod started on our behalf by
# buildlogger) and lets us terminate the whole tree of processes rather than
# orphaning the mongod.
import win32job
job_object = win32job.CreateJobObject(None, '')
job_info = win32job.QueryInformationJobObject(
job_object,
win32job.JobObjectExtendedLimitInformation)
job_info['BasicLimitInformation']['LimitFlags'] |= \
win32job.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
win32job.SetInformationJobObject(job_object,
win32job.JobObjectExtendedLimitInformation,
job_info)
win32job.AssignProcessToJobObject(job_object, proc._handle)
self.subprocess_job_object = job_object
def _launch_worker(self, wait=False):
worker = Worker(self._push_server_address, self._pull_server_address, self._ipc_authkey)
worker.start()
if sys.platform == 'win32':
permissions = win32con.PROCESS_TERMINATE | win32con.PROCESS_SET_QUOTA
handle = win32api.OpenProcess(permissions, False, worker.pid)
try:
win32job.AssignProcessToJobObject(self._job_obj, handle)
finally:
win32api.CloseHandle(handle)
if wait:
worker.wait_start()
self.workers.append(worker)
return worker
def job(self) -> WIN32JOB:
"""Get the job associated with this process.
Caches between calls for safety.
Returns
-------
WIN32JOB
A Windows job which consists of one or more processes. In this case
we just have the one process
"""
if not hasattr(self, "_job"):
# First try to import stuff, an easy exception to catch and give good
# information about
try:
import win32api
import win32job
import winerror
except ModuleNotFoundError as e:
raise ModuleNotFoundError(_win32_import_error_msg) from e
# Here, we assign it to a windows "Job", whatever that is
# If the process is already assigned to a job, then we have
# to check if it's less than Windows 8 because apparently
# nested jobs aren't supported there
try:
job = win32job.CreateJobObject(None, "")
process = win32api.GetCurrentProcess()
win32job.AssignProcessToJobObject(job, process)
self._job = job
except win32job.error as e:
if (e.winerror != winerror.ERROR_ACCESS_DENIED
or sys.getwindowsversion() >= (6, 2) # type: ignore
or not win32job.IsProcessInJob(process, None)):
raise e
msg = ("The process is already in a job."
" Nested jobs are not supported prior to Windows 8.")
raise RuntimeError(msg) from e
return self._job
def __init__(self, *args, **kwargs):
kwargs = dict(_Popen_defaults + kwargs.items())
if 'creationflagsmerge' in kwargs:
kwargs['creationflags'] = (kwargs.get('creationflags', 0)
| kwargs['creationflagsmerge'])
del kwargs['creationflagsmerge']
for f in ['stdout', 'stderr']:
if kwargs[f] is SINK:
kwargs[f] = create_sink()
# super() does some magic that makes **kwargs not work, so just call
# our super-constructor directly
subprocess.Popen.__init__(self, *args, **kwargs)
_CHILD_PROCS.append(self)
if mswindows and _kill_children_on_death:
handle = windll.kernel32.OpenProcess(
win32con.SYNCHRONIZE | win32con.PROCESS_SET_QUOTA
| win32con.PROCESS_TERMINATE, 0, self.pid)
if win32job.AssignProcessToJobObject(_chJob, handle) == 0:
raise WinError()
def create_job(hProcess):
'''
create_job(hProcess) creates win32job with correct flags:
Note on JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE:
https://msdn.microsoft.com/en-us/library/windows/desktop/ms684161(v=vs.85).aspx
However, if the job has the
JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE flag specified,
closing the last job object handle terminates all
associated processes and then destroys the job
object itself.
'''
hJob = win32job.CreateJobObject(None, "")
extended_info = win32job.QueryInformationJobObject(
hJob, win32job.JobObjectExtendedLimitInformation)
extended_info['BasicLimitInformation'][
'LimitFlags'] = win32job.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
win32job.SetInformationJobObject(
hJob, win32job.JobObjectExtendedLimitInformation, extended_info)
win32job.AssignProcessToJobObject(hJob, hProcess)
return hJob
def __register_job_object(self):
if self.is_windows8_or_above_flag or not self.is_cloud_mode_flag:
self.child = None
self.hJob = None
self.hProcess = None
self.hJob = win32job.CreateJobObject(None, self.win32_job_name)
extended_info = win32job.QueryInformationJobObject(
self.hJob, win32job.JobObjectExtendedLimitInformation)
extended_info['BasicLimitInformation'][
'LimitFlags'] = win32job.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
win32job.SetInformationJobObject(
self.hJob, win32job.JobObjectExtendedLimitInformation,
extended_info)
# Convert process id to process handle:
perms = win32con.PROCESS_TERMINATE | win32con.PROCESS_SET_QUOTA
self.hProcess = win32api.OpenProcess(perms, False, self.__p.pid)
try:
win32job.AssignProcessToJobObject(self.hJob, self.hProcess)
self.__user_job_object_flag = True
except Exception as e:
self.__user_job_object_flag = False
else:
self.__user_job_object_flag = False
def SvcDoRun(self):
import servicemanager
servicemanager.LogInfoMsg(self._svc_name_ + " Start Requested")
try:
hJob = win32job.CreateJobObject(None, "")
extended_info = win32job.QueryInformationJobObject(hJob, win32job.JobObjectExtendedLimitInformation)
extended_info['BasicLimitInformation']['LimitFlags'] = win32job.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
win32job.SetInformationJobObject(hJob, win32job.JobObjectExtendedLimitInformation, extended_info)
command = "resilient-circuits.exe run " + self._resilient_args_
command_args = shlex.split(command)
self.process_handle = subprocess.Popen(command_args)
# Convert process id to process handle:
perms = win32con.PROCESS_TERMINATE | win32con.PROCESS_SET_QUOTA
hProcess = win32api.OpenProcess(perms, False, self.process_handle.pid)
win32job.AssignProcessToJobObject(hJob, hProcess)
except:
servicemanager.LogErrorMsg(self._svc_name_ + " failed to launch resilient-circuits.exe")
raise
servicemanager.LogInfoMsg(self._svc_name_ + " Started")
while self.isAlive:
if self.process_handle.poll() != None:
self.SvcStop()
win32api.SleepEx(10000, True)
def _start(self, argv):
"""In most cases, just call subprocess.Popen(). On windows,
add the started process to a new Job Object, so that any
child processes of this process can be killed with a single
call to TerminateJobObject (see self.stop()).
"""
if valgrind:
argv = [
'buildscripts/valgrind.bash', '--show-reachable=yes',
'--leak-check=full', '--suppressions=valgrind.suppressions'
] + argv
elif drd:
argv = ['buildscripts/valgrind.bash', '--tool=drd'] + argv
proc = Popen(argv, stdout=self.outfile)
if os.sys.platform == "win32":
# Create a job object with the "kill on job close"
# flag; this is inherited by child processes (ie
# the mongod started on our behalf by buildlogger)
# and lets us terminate the whole tree of processes
# rather than orphaning the mongod.
import win32job
self.job_object = win32job.CreateJobObject(None, '')
job_info = win32job.QueryInformationJobObject(
self.job_object, win32job.JobObjectExtendedLimitInformation)
job_info['BasicLimitInformation'][
'LimitFlags'] |= win32job.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
win32job.SetInformationJobObject(
self.job_object, win32job.JobObjectExtendedLimitInformation,
job_info)
win32job.AssignProcessToJobObject(self.job_object, proc._handle)
return proc
def add_child(self, pid):
"""Adds the child process to the job"""
# Convert process id to process handle
perms = win32con.PROCESS_TERMINATE | win32con.PROCESS_SET_QUOTA
hProcess = win32api.OpenProcess(perms, False, pid)
win32job.AssignProcessToJobObject(self.hJob, hProcess)
# allows python child process to print unicode characters to console without UnicodeEncodeError exception
my_env = os.environ
my_env['PYTHONIOENCODING'] = 'utf-8'
# TODO: find more portable solution without setting enviroment variable.
# Entering command in cmd.exe shell + Enter button does allow to print unicode characters
# PYTHONIOENCODING enviroment variable is supposed to work only with python children processes
if DEF_universal_newlines_True[0]:
proc=subprocess.Popen(sppd_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, creationflags = subprocess.CREATE_NO_WINDOW, env = my_env, universal_newlines=True)
else:
proc=subprocess.Popen(sppd_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, creationflags = subprocess.CREATE_NO_WINDOW, env = my_env)
real_sppd_cmd = psutil.Process(proc.pid).cmdline()
runner_hProcess = win32api.OpenProcess(perms, False, proc.pid)
win32job.AssignProcessToJobObject(hJob, runner_hProcess)
print("--- runtsharksppdpipe.py configured to terminate if wrapper.py dies ---", flush=True)
## print('proc.stdout.fileno() = ' + repr(proc.stdout.fileno()), flush = True)
tmpe = None
try:
if DEF_universal_newlines_True[0]:
while proc.poll() is None:
line = proc.stdout.readline()
if line:
print(line, flush = True, end='')
else:
while proc.poll() is None:
bytes_variable = os.read(proc.stdout.fileno(), 2 << 16)
if bytes_variable:
def proc(args, time_limit, mem_size, time_elapsed, return_val, _stdin_name,
_stdout_name, _stderr_name):
"""测试的 runner 函数
Args:
args: 待测程序及其参数的列表
time_limit: float，秒为单位的运行时间限制
mem_size: int，byte 为单位的运行内存限制，零值为不限制
time_elapsed: double 类型的 multiprocessing 中的 shared_ctypes 对象
用以存储并返回实际运行时间
return_val: int 类型的 multiprocessing 中的 shared_ctypes 对象
用以存储并返回待测程序的返回值
_stdin_name, _stdout_name, _stderr_name: 为文件路径
分别为待测程序的标准输入、输出、错误输出的重定向目标
"""
if mem_size:
try:
if _mswindows:
import win32api
import win32job
job = win32job.CreateJobObject(None, "judge_mem_limiter")
win32job.SetInformationJobObject(
job, win32job.JobObjectExtendedLimitInformation, {
"BasicLimitInformation": {
"PerProcessUserTimeLimit": 0,
"PerJobUserTimeLimit": 0,
"LimitFlags":
win32job.JOB_OBJECT_LIMIT_PROCESS_MEMORY,
"MinimumWorkingSetSize": 0,
"MaximumWorkingSetSize": 0,
"ActiveProcessLimit": 0,
"Affinity": 0,
"PriorityClass": 0,
"SchedulingClass": 0
},
"IoInfo": {
"ReadOperationCount": 0,
"WriteOperationCount": 0,
"OtherOperationCount": 0,
"ReadTransferCount": 0,
"WriteTransferCount": 0,
"OtherTransferCount": 0,
},
"JobMemoryLimit": 0,
"PeakProcessMemoryUsed": 0,
"PeakJobMemoryUsed": 0,
"ProcessMemoryLimit": mem_size
})
win32job.AssignProcessToJobObject(job,
win32api.GetCurrentProcess())
else:
import resource
resource.setrlimit(resource.RLIMIT_DATA, (mem_size, mem_size))
except:
_logger.error("unable to set memory limit under win32: %s",
traceback.format_exc())
with open(_stdin_name, "rb") as stdin,\
open(_stdout_name, "wb") as stdout,\
open(_stderr_name, "wb") as stderr:
proc = subprocess.Popen(args,
stdin=stdin,
stdout=stdout,
stderr=stderr)
t_start = time.perf_counter()
try:
proc.wait(timeout=time_limit + 1.)
except subprocess.TimeoutExpired:
proc.kill()
proc.returncode = -1
t_end = time.perf_counter()
time_elapsed.value = t_end - t_start
return_val.value = proc.returncode
def startBackgroundProcess(self):
"""Method to start a process running in the background.
"""
with process_lock:
# security attributes for pipes
sAttrs = win32security.SECURITY_ATTRIBUTES()
sAttrs.bInheritHandle = 1
# create pipes for the process to write to
hStdin_r, hStdin = win32pipe.CreatePipe(sAttrs, 0)
hStdout = win32file.CreateFile(
self.stdout, win32file.GENERIC_WRITE | win32file.GENERIC_READ,
win32file.FILE_SHARE_DELETE | win32file.FILE_SHARE_READ
| win32file.FILE_SHARE_WRITE, sAttrs, win32file.CREATE_ALWAYS,
win32file.FILE_ATTRIBUTE_NORMAL, None)
hStderr = win32file.CreateFile(
self.stderr, win32file.GENERIC_WRITE | win32file.GENERIC_READ,
win32file.FILE_SHARE_DELETE | win32file.FILE_SHARE_READ
| win32file.FILE_SHARE_WRITE, sAttrs, win32file.CREATE_ALWAYS,
win32file.FILE_ATTRIBUTE_NORMAL, None)
try:
# set the info structure for the new process.
StartupInfo = win32process.STARTUPINFO()
StartupInfo.hStdInput = hStdin_r
StartupInfo.hStdOutput = hStdout
StartupInfo.hStdError = hStderr
StartupInfo.dwFlags = win32process.STARTF_USESTDHANDLES
# Create new handles for the thread ends of the pipes. The duplicated handles will
# have their inheritence properties set to false so that any children inheriting these
# handles will not have non-closeable handles to the pipes
pid = win32api.GetCurrentProcess()
tmp = win32api.DuplicateHandle(pid, hStdin, pid, 0, 0,
win32con.DUPLICATE_SAME_ACCESS)
win32file.CloseHandle(hStdin)
hStdin = tmp
# start the process, and close down the copies of the process handles
# we have open after the process creation (no longer needed here)
old_command = command = self.__quotePath(self.command)
for arg in self.arguments:
command = '%s %s' % (command, self.__quotePath(arg))
# Windows CreateProcess maximum lpCommandLine length is 32,768
# http://msdn.microsoft.com/en-us/library/ms682425%28VS.85%29.aspx
if len(command) >= 32768: # pragma: no cover
raise ValueError(
"Command line length exceeded 32768 characters: %s..."
% command[:1000])
dwCreationFlags = 0
if IS_PRE_WINDOWS_8: # pragma: no cover
# In case PySys is itself running in a job, might need to explicitly breakaway from it so we can give
# it its own, but only for old pre-windows 8/2012, which support nested jobs
dwCreationFlags = dwCreationFlags | win32process.CREATE_BREAKAWAY_FROM_JOB
if self.command.lower().endswith(('.bat', '.cmd')):
# If we don't start suspended there's a slight race condition but due to some issues with
# initially-suspended processes hanging (seen many years ago), to be safe, only bother to close the
# race condition for shell scripts (which is the main use case for this anyway)
dwCreationFlags = dwCreationFlags | win32con.CREATE_SUSPENDED
self.__job = self._createParentJob()
try:
self.__hProcess, self.__hThread, self.pid, self.__tid = win32process.CreateProcess(
None, command, None, None, 1,
dwCreationFlags, self.environs,
os.path.normpath(self.workingDir), StartupInfo)
except pywintypes.error as e:
raise ProcessError("Error creating process %s: %s" %
(old_command, e))
try:
if not self.disableKillingChildProcesses:
win32job.AssignProcessToJobObject(
self.__job, self.__hProcess)
else:
self.__job = None # pragma: no cover
except Exception as e: # pragma: no cover
# Shouldn't fail unless process already terminated (which can happen since
# if we didn't use SUSPENDED there's an inherent race here)
if win32process.GetExitCodeProcess(
self.__hProcess) == win32con.STILL_ACTIVE:
log.warning(
'Failed to associate process %s with new job: %s (this may prevent automatic cleanup of child processes)'
% (self, e))
# force use of TerminateProcess not TerminateJobObject if this failed
self.__job = None
if (dwCreationFlags & win32con.CREATE_SUSPENDED) != 0:
win32process.ResumeThread(self.__hThread)
finally:
win32file.CloseHandle(hStdin_r)
win32file.CloseHandle(hStdout)
win32file.CloseHandle(hStderr)
# set the handle to the stdin of the process
self.__stdin = hStdin
def __init__(self,
script: str = "",
ahk_path: Path = None,
execute_from: Path = None) -> None:
self.file = None
self.script = script
if ahk_path is None:
ahk_path = PACKAGE_PATH / r'lib\AutoHotkey\AutoHotkey.exe'
assert ahk_path and ahk_path.is_file()
# Windows notification area relies on consistent exe path
if execute_from is not None:
execute_from_dir = Path(execute_from)
assert execute_from_dir.is_dir()
ahk_into_folder = execute_from_dir / ahk_path.name
try:
if os.path.getmtime(ahk_into_folder) != os.path.getmtime(
ahk_path):
os.remove(ahk_into_folder)
except FileNotFoundError:
pass
try:
os.link(ahk_path, ahk_into_folder)
except FileExistsError:
pass
except OSError as ex:
# 5: "Access is denied"
# 17: "The system cannot move the file to a different disk drive"
if ex.winerror in (5, 17):
shutil.copyfile(ahk_path, ahk_into_folder)
ahk_path = ahk_into_folder
self.pid = os.getpid()
# if we exit, exit AutoHotkey
atexit.register(self.exit)
# if we terminate, terminate AutoHotkey
self.job = win32job.CreateJobObject(None, "")
extended_info = win32job.QueryInformationJobObject(
self.job, win32job.JobObjectExtendedLimitInformation)
extended_info['BasicLimitInformation'][
'LimitFlags'] = win32job.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
win32job.SetInformationJobObject(
self.job, win32job.JobObjectExtendedLimitInformation,
extended_info)
# add ourselves and subprocess will inherit job membership
handle = win32api.OpenProcess(
win32con.PROCESS_TERMINATE | win32con.PROCESS_SET_QUOTA, False,
self.pid)
win32job.AssignProcessToJobObject(self.job, handle)
win32api.CloseHandle(handle)
# user script exceptions are already caught and sent to stderr, so /ErrorStdOut would only affect debugging CORE
# self.cmd = [str(ahk_path), "/ErrorStdOut=utf-16-raw", "/CP65001", "*"]
self.cmd = [str(ahk_path), "/CP65001", "*"]
# must pipe all three within a PyInstaller bundled exe
self.popen = subprocess.Popen(self.cmd,
bufsize=Script.BUFFER_SIZE,
executable=str(ahk_path),
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
# keep grandchild processes from inheriting job membership
extended_info['BasicLimitInformation'][
'LimitFlags'] |= win32job.JOB_OBJECT_LIMIT_SILENT_BREAKAWAY_OK
win32job.SetInformationJobObject(
self.job, win32job.JobObjectExtendedLimitInformation,
extended_info)
self.popen.stdin.write(Script.CORE.encode('utf-8'))
self.popen.stdin.write(self.script.encode('utf-8'))
self.popen.stdin.close()
self.hwnd = int(self._read_response(), 16)
assert self._read_response() == "Initialized"
sys.stdout.flush()
if is_win:
# Create a job object and assign ourselves to it, so that
# all remaining test subprocesses get killed off on completion.
# This prevents AppVeyor from waiting an hour
# https://stackoverflow.com/a/23587108 (and its first comment)
import win32api, win32con, win32job # noqa
hJob = win32job.CreateJobObject(None, "")
extended_info = win32job.QueryInformationJobObject(hJob, win32job.JobObjectExtendedLimitInformation)
extended_info['BasicLimitInformation']['LimitFlags'] = win32job.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
win32job.SetInformationJobObject(hJob, win32job.JobObjectExtendedLimitInformation, extended_info)
perms = win32con.PROCESS_TERMINATE | win32con.PROCESS_SET_QUOTA
hProcess = win32api.OpenProcess(perms, False, os.getpid())
win32job.AssignProcessToJobObject(hJob, hProcess)
def find_test_keys():
if os.environ.get('CONDA_BUILD'):
# The current version of conda build manually adds the activation
# directories to the PATH---and then calls the standard conda
# activation script, which does it again. This frustrates conda's
# ability to deactivate this environment. Most package builds are
# not affected by this, but we are, because our tests need to do
# environment activation and deactivation. To fix this, we remove
# the duplicate PATH entries conda-build added.
print('BEFORE: {}'.format(os.environ['PATH']))
path_list = os.environ['PATH'].split(os.pathsep)
path_dups = set()
path_list = [p for p in path_list
def CreateProcess(self, *args, **kwargs):
hp, ht, pid, tid = self._oldapi.CreateProcess(*args, **kwargs)
win32job.AssignProcessToJobObject(self._job, hp)
win32process.ResumeThread(ht)
return hp, ht, pid, tid
def main():
# escape list of arguments
command = _win32_arglist_to_string(sys.argv[1:])
# create job
hJob = win32job.CreateJobObject(None, '')
extended_info = win32job.QueryInformationJobObject(
hJob, win32job.JobObjectExtendedLimitInformation)
extended_info['BasicLimitInformation'][
'LimitFlags'] = win32job.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
win32job.SetInformationJobObject(
hJob, win32job.JobObjectExtendedLimitInformation, extended_info)
# associate job with completion port
hIoPort = win32file.CreateIoCompletionPort(win32file.INVALID_HANDLE_VALUE,
None, 0, 1)
# pywin32 is missing support for JOBOBJECT_ASSOCIATE_COMPLETION_PORT, therefore
# we call it through ctypes
port = JOBOBJECT_ASSOCIATE_COMPLETION_PORT()
port.CompletionKey = hJob.handle
port.CompletionPort = hIoPort.handle
assert bool(
ctypes.windll.kernel32.SetInformationJobObject(
ctypes.wintypes.HANDLE(hJob.handle),
ctypes.c_int(JobObjectAssociateCompletionPortInformation),
ctypes.byref(port),
ctypes.sizeof(JOBOBJECT_ASSOCIATE_COMPLETION_PORT)))
# create process suspended
si = win32process.STARTUPINFO()
hProcess, hThread, processId, threadId = win32process.CreateProcess(
None, command, None, None, True,
win32process.CREATE_BREAKAWAY_FROM_JOB | win32process.CREATE_SUSPENDED,
None, None, si)
# add process to job
win32job.AssignProcessToJobObject(hJob, hProcess)
# resume process
win32process.ResumeThread(hThread)
win32api.CloseHandle(hThread)
win32api.CloseHandle(hProcess)
# wait for job termination
numberOfBytes = ctypes.wintypes.DWORD(0)
completionKey = ctypes.wintypes.HANDLE(0)
overlapped = OVERLAPPED()
while True:
# calling this through pywin32 crashes the program, therefore we call it through ctypes
res = bool(
ctypes.windll.kernel32.GetQueuedCompletionStatus(
ctypes.wintypes.HANDLE(hIoPort.handle),
ctypes.byref(numberOfBytes), ctypes.byref(completionKey),
ctypes.byref(overlapped),
ctypes.wintypes.DWORD(win32event.INFINITE)))
if not res or (bytes(completionKey) == bytes(
ctypes.c_void_p(hJob.handle))
and bytes(numberOfBytes) == bytes(
ctypes.c_ulong(
win32job.JOB_OBJECT_MSG_ACTIVE_PROCESS_ZERO))):
break
def start(self):
"""Start the process and the logger pipes for its stdout and stderr."""
creation_flags = 0
if sys.platform == "win32" and _JOB_OBJECT is not None:
creation_flags |= win32process.CREATE_BREAKAWAY_FROM_JOB
# Tests fail if a process takes too long to startup and listen to a socket. Use buffered
# I/O pipes to give the process some leeway.
buffer_size = 1024 * 1024
# Close file descriptors in the child process before executing the program. This prevents
# file descriptors that were inherited due to multiple calls to fork() -- either within one
# thread, or concurrently from multiple threads -- from causing another subprocess to wait
# for the completion of the newly spawned child process. Closing other file descriptors
# isn't supported on Windows when stdout and stderr are redirected.
close_fds = (sys.platform != "win32")
with _POPEN_LOCK:
# Record unittests directly since resmoke doesn't not interact with them and they can finish
# too quickly for the recorder to have a chance at attaching.
recorder_args = []
if _config.UNDO_RECORDER_PATH is not None and self.args[
0].endswith("_test"):
now_str = datetime.now().strftime("%Y-%m-%dT%H-%M-%S")
# Only use the process name since we have to be able to correlate the recording name
# with the binary name easily.
recorder_output_file = "{process}-{t}.undo".format(
process=os.path.basename(self.args[0]), t=now_str)
recorder_args = [
_config.UNDO_RECORDER_PATH, "-o", recorder_output_file
]
self._process = subprocess.Popen(recorder_args + self.args,
bufsize=buffer_size,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=close_fds,
env=self.env,
creationflags=creation_flags,
cwd=self._cwd)
self.pid = self._process.pid
if _config.UNDO_RECORDER_PATH is not None and (
not self.args[0].endswith("_test")) and (
"mongod" in self.args[0] or "mongos" in self.args[0]):
now_str = datetime.now().strftime("%Y-%m-%dT%H-%M-%S")
recorder_output_file = "{logger}-{process}-{pid}-{t}.undo".format(
logger=self.logger.name.replace('/', '-'),
process=os.path.basename(self.args[0]),
pid=self.pid,
t=now_str)
recorder_args = [
_config.UNDO_RECORDER_PATH, "-p",
str(self.pid), "-o", recorder_output_file
]
self._recorder = subprocess.Popen(recorder_args,
bufsize=buffer_size,
env=self.env,
creationflags=creation_flags)
self._stdout_pipe = pipe.LoggerPipe(self.logger, logging.INFO,
self._process.stdout)
self._stderr_pipe = pipe.LoggerPipe(self.logger, logging.ERROR,
self._process.stderr)
self._stdout_pipe.wait_until_started()
self._stderr_pipe.wait_until_started()
if sys.platform == "win32" and _JOB_OBJECT is not None:
try:
win32job.AssignProcessToJobObject(_JOB_OBJECT,
self._process._handle)
except win32job.error as err:
# ERROR_ACCESS_DENIED (winerror=5) is received when the process has already died.
if err.winerror != winerror.ERROR_ACCESS_DENIED:
raise
return_code = win32process.GetExitCodeProcess(
self._process._handle)
if return_code == win32con.STILL_ACTIVE:
raise
