def run(self):
"""Set up input/output streams and execute the child function in a new
thread. This is part of the `threading.Thread` interface and should
not be called directly.
"""
if self.f is None:
return
if self.stdin is not None:
sp_stdin = io.TextIOWrapper(self.stdin)
else:
sp_stdin = io.StringIO("")
if ON_WINDOWS:
if self.c2pwrite != -1:
self.c2pwrite = msvcrt.open_osfhandle(self.c2pwrite.Detach(), 0)
if self.errwrite != -1:
self.errwrite = msvcrt.open_osfhandle(self.errwrite.Detach(), 0)
if self.c2pwrite != -1:
sp_stdout = io.TextIOWrapper(io.open(self.c2pwrite, 'wb', -1))
else:
sp_stdout = sys.stdout
if self.errwrite == self.c2pwrite:
sp_stderr = sp_stdout
elif self.errwrite != -1:
sp_stderr = io.TextIOWrapper(io.open(self.errwrite, 'wb', -1))
else:
sp_stderr = sys.stderr
r = self.f(self.args, sp_stdin, sp_stdout, sp_stderr)
self.returncode = 0 if r is None else r
def __init__(self, args, stdin=None, stdout=None, stderr=None, **kwds):
stdin_rfd = stdout_wfd = stderr_wfd = None
stdin_wh = stdout_rh = stderr_rh = None
if stdin == PIPE:
stdin_rh, stdin_wh = pipe(overlapped=(False, True))
stdin_rfd = msvcrt.open_osfhandle(stdin_rh, os.O_RDONLY)
if stdout == PIPE:
stdout_rh, stdout_wh = pipe(overlapped=(True, False))
stdout_wfd = msvcrt.open_osfhandle(stdout_wh, 0)
if stderr == PIPE:
stderr_rh, stderr_wh = pipe(overlapped=(True, False))
stderr_wfd = msvcrt.open_osfhandle(stderr_wh, 0)
try:
super().__init__(args, bufsize=0, universal_newlines=False,
stdin=stdin_rfd, stdout=stdout_wfd,
stderr=stderr_wfd, **kwds)
except:
for h in (stdin_wh, stdout_rh, stderr_rh):
_winapi.CloseHandle(h)
raise
else:
if stdin_wh is not None:
self.stdin = PipeHandle(stdin_wh)
if stdout_rh is not None:
self.stdout = PipeHandle(stdout_rh)
if stderr_rh is not None:
self.stderr = PipeHandle(stderr_rh)
finally:
if stdin == PIPE:
os.close(stdin_rfd)
if stdout == PIPE:
os.close(stdout_wfd)
if stderr == PIPE:
os.close(stderr_wfd)
def override_io():
old_out = sys.stdout
old_err = sys.stderr
fd_out = int(os.environ['TEST_WRITE_OUT'])
fd_err = int(os.environ['TEST_WRITE_ERR'])
if sys.platform == 'win32':
import msvcrt
fd_out = msvcrt.open_osfhandle(fd_out, 0)
fd_err = msvcrt.open_osfhandle(fd_err, 0)
api_out = os.fdopen(fd_out, 'w')
api_err = os.fdopen(fd_err, 'w')
class Intercept:
def __init__(self, api, old):
self.api = api
self.old = old
def write(self, data):
import threading
if threading.current_thread().name.startswith('APIThread'):
self.api.write(data)
else:
self.old.write(data)
def flush(self):
self.api.flush()
self.old.flush()
sys.stdout = Intercept(api_out, old_out)
sys.stderr = Intercept(api_err, old_err)
def _get_handles(self, stdin, stdout, stderr):
"""Construct and return tupel with IO objects:
p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
"""
if stdin == None and stdout == None and stderr == None:
return (None, None, None, None, None, None)
p2cread, p2cwrite = None, None
c2pread, c2pwrite = None, None
errread, errwrite = None, None
if stdin == None:
p2cread = GetStdHandle(STD_INPUT_HANDLE)
elif stdin == PIPE:
p2cread, p2cwrite = CreatePipe(None, 0)
# Detach and turn into fd
p2cwrite = p2cwrite.Detach()
p2cwrite = msvcrt.open_osfhandle(p2cwrite, 0)
elif type(stdin) == int:
p2cread = msvcrt.get_osfhandle(stdin)
else:
# Assuming file-like object
p2cread = msvcrt.get_osfhandle(stdin.fileno())
p2cread = self._make_inheritable(p2cread)
if stdout == None:
c2pwrite = GetStdHandle(STD_OUTPUT_HANDLE)
elif stdout == PIPE:
c2pread, c2pwrite = CreatePipe(None, 0)
# Detach and turn into fd
c2pread = c2pread.Detach()
c2pread = msvcrt.open_osfhandle(c2pread, 0)
elif type(stdout) == int:
c2pwrite = msvcrt.get_osfhandle(stdout)
else:
# Assuming file-like object
c2pwrite = msvcrt.get_osfhandle(stdout.fileno())
c2pwrite = self._make_inheritable(c2pwrite)
if stderr == None:
errwrite = GetStdHandle(STD_ERROR_HANDLE)
elif stderr == PIPE:
errread, errwrite = CreatePipe(None, 0)
# Detach and turn into fd
errread = errread.Detach()
errread = msvcrt.open_osfhandle(errread, 0)
elif stderr == STDOUT:
errwrite = c2pwrite
elif type(stderr) == int:
errwrite = msvcrt.get_osfhandle(stderr)
else:
# Assuming file-like object
errwrite = msvcrt.get_osfhandle(stderr.fileno())
errwrite = self._make_inheritable(errwrite)
return (p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite)
def __init__(self, args, bufsize = 0, executable = None, stdin = None, stdout = None, stderr = None, preexec_fn = None, close_fds = False, shell = False, cwd = None, env = None, universal_newlines = False, startupinfo = None, creationflags = 0):
"""Create new Popen instance."""
_cleanup()
if not isinstance(bufsize, (int, long)):
raise TypeError('bufsize must be an integer')
if mswindows:
if preexec_fn is not None:
raise ValueError('preexec_fn is not supported on Windows platforms')
if close_fds and (stdin is not None or stdout is not None or stderr is not None):
raise ValueError('close_fds is not supported on Windows platforms if you redirect stdin/stdout/stderr')
else:
if startupinfo is not None:
raise ValueError('startupinfo is only supported on Windows platforms')
if creationflags != 0:
raise ValueError('creationflags is only supported on Windows platforms')
self.stdin = None
self.stdout = None
self.stderr = None
self.pid = None
self.returncode = None
self.universal_newlines = universal_newlines
(p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite), to_close = self._get_handles(stdin, stdout, stderr)
try:
self._execute_child(args, executable, preexec_fn, close_fds, cwd, env, universal_newlines, startupinfo, creationflags, shell, to_close, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite)
except Exception:
exc_type, exc_value, exc_trace = sys.exc_info()
for fd in to_close:
try:
if mswindows:
fd.Close()
else:
os.close(fd)
except EnvironmentError:
pass
raise exc_type, exc_value, exc_trace
if mswindows:
if p2cwrite is not None:
p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0)
if c2pread is not None:
c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0)
if errread is not None:
errread = msvcrt.open_osfhandle(errread.Detach(), 0)
if p2cwrite is not None:
self.stdin = os.fdopen(p2cwrite, 'wb', bufsize)
if c2pread is not None:
if universal_newlines:
self.stdout = os.fdopen(c2pread, 'rU', bufsize)
else:
self.stdout = os.fdopen(c2pread, 'rb', bufsize)
if errread is not None:
if universal_newlines:
self.stderr = os.fdopen(errread, 'rU', bufsize)
else:
self.stderr = os.fdopen(errread, 'rb', bufsize)
return
def wtrf():
if sys.platform != "win32":
wt = int(os.environ['MYHDL_TO_PIPE'])
rf = int(os.environ['MYHDL_FROM_PIPE'])
else:
wt = msvcrt.open_osfhandle(int(os.environ['MYHDL_TO_PIPE']), os.O_APPEND | os.O_TEXT)
rf = msvcrt.open_osfhandle(int(os.environ['MYHDL_FROM_PIPE']), os.O_RDONLY | os.O_TEXT)
return wt, rf
def __init__(self, args, bufsize = 0, executable = None, stdin = None, stdout = None, stderr = None, preexec_fn = None, close_fds = _PLATFORM_DEFAULT_CLOSE_FDS, shell = False, cwd = None, env = None, universal_newlines = False, startupinfo = None, creationflags = 0, restore_signals = True, start_new_session = False, pass_fds = ()):
_cleanup()
self._child_created = False
self._input = None
self._communication_started = False
if not isinstance(bufsize, (int, long)):
raise TypeError('bufsize must be an integer')
if mswindows:
if preexec_fn is not None:
raise ValueError('preexec_fn is not supported on Windows platforms')
any_stdio_set = stdin is not None or stdout is not None or stderr is not None
if close_fds is _PLATFORM_DEFAULT_CLOSE_FDS:
if any_stdio_set:
close_fds = False
else:
close_fds = True
elif close_fds and any_stdio_set:
raise ValueError('close_fds is not supported on Windows platforms if you redirect stdin/stdout/stderr')
else:
if close_fds is _PLATFORM_DEFAULT_CLOSE_FDS:
close_fds = True
if pass_fds and not close_fds:
warnings.warn('pass_fds overriding close_fds.', RuntimeWarning)
close_fds = True
if startupinfo is not None:
raise ValueError('startupinfo is only supported on Windows platforms')
if creationflags != 0:
raise ValueError('creationflags is only supported on Windows platforms')
self.args = args
self.stdin = None
self.stdout = None
self.stderr = None
self.pid = None
self.returncode = None
self.universal_newlines = universal_newlines
p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite = self._get_handles(stdin, stdout, stderr)
self._execute_child(args, executable, preexec_fn, close_fds, pass_fds, cwd, env, universal_newlines, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, restore_signals, start_new_session)
if mswindows:
if p2cwrite != -1:
p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0)
if c2pread != -1:
c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0)
if errread != -1:
errread = msvcrt.open_osfhandle(errread.Detach(), 0)
if p2cwrite != -1:
self.stdin = os.fdopen(p2cwrite, 'wb', bufsize)
if c2pread != -1:
if universal_newlines:
self.stdout = os.fdopen(c2pread, 'rU', bufsize)
else:
self.stdout = os.fdopen(c2pread, 'rb', bufsize)
if errread != -1:
if universal_newlines:
self.stderr = os.fdopen(errread, 'rU', bufsize)
else:
self.stderr = os.fdopen(errread, 'rb', bufsize)
def main():
import sys
fd_r = int(sys.argv[1])
fd_w = int(sys.argv[2])
if platform.system() == 'Windows':
import msvcrt
fd_r = msvcrt.open_osfhandle(fd_r, 0)
fd_w = msvcrt.open_osfhandle(fd_w, 0)
serve(fd_r, fd_w)
def __init__(self, args, stdin=None, stdout=None, stderr=None, **kwargs):
self.stdin = self.stdout = self.stderr = None
stdin_rh = stdin_wh = stdout_rh = stdout_wh = stderr_rh = stderr_wh = None
if stdin == subprocess.PIPE:
stdin_rh, stdin_wh = pipe()
stdin_rfd = msvcrt.open_osfhandle(stdin_rh.Detach(), os.O_RDONLY)
self.stdin_rh = stdin_rh
else:
stdin_rfd = stdin
self.stdin_rh = None
if stdout == subprocess.PIPE:
stdout_rh, stdout_wh = pipe()
stdout_wfd = msvcrt.open_osfhandle(stdout_wh, 0)
else:
stdout_wfd = stdout
if stderr == subprocess.PIPE:
stderr_rh, stderr_wh = pipe()
stderr_wfd = msvcrt.open_osfhandle(stderr_wh, 0)
elif stderr == subprocess.STDOUT:
stderr_wfd = stdout_wfd
else:
stderr_wfd = stderr
try:
super(Popen, self).__init__(args, stdin=stdin_rfd, stdout=stdout_wfd,
stderr=stderr_wfd, **kwargs)
except:
for handle in (stdin_rh, stdin_wh, stdout_rh, stdout_wh, stderr_rh, stderr_wh):
if handle is not None:
win32file.CloseHandle(handle)
raise
else:
if stdin_wh is not None:
self.stdin = AsyncFile(stdin_wh, mode='w')
if stdout_rh is not None:
self.stdout = AsyncFile(stdout_rh, mode='r')
if stderr_rh is not None:
self.stderr = AsyncFile(stderr_rh, mode='r')
finally:
if stdin == subprocess.PIPE:
os.close(stdin_rfd)
if stdout == subprocess.PIPE:
os.close(stdout_wfd)
if stderr == subprocess.PIPE:
os.close(stderr_wfd)
def from_subprocess(cls, arg):
arg = int(arg)
if MS_WINDOWS:
fd = msvcrt.open_osfhandle(arg, os.O_WRONLY)
else:
fd = arg
return cls(fd)
def __init__(self, process_obj):
os.environ["MULTIPROCESSING_FORKING_DISABLE"] = "1"
spawn._Django_old_layout_hack__save()
prep_data = spawn.get_preparation_data(process_obj._name)
# read end of pipe will be "stolen" by the child process
# -- see spawn_main() in spawn.py.
rhandle, whandle = _winapi.CreatePipe(None, 0)
wfd = msvcrt.open_osfhandle(whandle, 0)
cmd = spawn.get_command_line(parent_pid=os.getpid(), pipe_handle=rhandle)
cmd = " ".join('"%s"' % x for x in cmd)
with io.open(wfd, "wb", closefd=True) as to_child:
# start process
try:
hp, ht, pid, tid = CreateProcess(spawn.get_executable(), cmd, None, None, False, 0, None, None, None)
_winapi.CloseHandle(ht)
except:
_winapi.CloseHandle(rhandle)
raise
# set attributes of self
self.pid = pid
self.returncode = None
self._handle = hp
self.sentinel = int(hp)
# send information to child
context.set_spawning_popen(self)
try:
reduction.dump(prep_data, to_child)
reduction.dump(process_obj, to_child)
finally:
context.set_spawning_popen(None)
def spawn_main(pipe_handle, parent_pid=None, tracker_fd=None):
'''
Run code specified by data received over pipe
'''
assert is_forking(sys.argv), "Not forking"
if sys.platform == 'win32':
import msvcrt
import _winapi
if parent_pid is not None:
source_process = _winapi.OpenProcess(
_winapi.PROCESS_DUP_HANDLE, False, parent_pid)
else:
source_process = None
try:
new_handle = reduction.duplicate(pipe_handle,
source_process=source_process)
finally:
if source_process is not None:
_winapi.CloseHandle(source_process)
fd = msvcrt.open_osfhandle(new_handle, os.O_RDONLY)
else:
from . import semaphore_tracker
semaphore_tracker._semaphore_tracker._fd = tracker_fd
fd = pipe_handle
exitcode = _main(fd)
sys.exit(exitcode)
def _open(handle):
"""
Open a file descriptor for the specified HANDLE (int -> int).
Closing the file descriptor will also close the handle.
"""
return msvcrt.open_osfhandle(handle, 0)
def __init__(self, args, stdin=None, stdout=None, stderr=None, **kwds):
assert not kwds.get('universal_newlines')
assert kwds.get('bufsize', 0) == 0
stdin_rfd = stdout_wfd = stderr_wfd = None
stdin_wh = stdout_rh = stderr_rh = None
if stdin == PIPE:
stdin_rh, stdin_wh = pipe(overlapped=(False, True), duplex=True)
stdin_rfd = msvcrt.open_osfhandle(stdin_rh, os.O_RDONLY)
else:
stdin_rfd = stdin
if stdout == PIPE:
stdout_rh, stdout_wh = pipe(overlapped=(True, False))
stdout_wfd = msvcrt.open_osfhandle(stdout_wh, 0)
else:
stdout_wfd = stdout
if stderr == PIPE:
stderr_rh, stderr_wh = pipe(overlapped=(True, False))
stderr_wfd = msvcrt.open_osfhandle(stderr_wh, 0)
elif stderr == STDOUT:
stderr_wfd = stdout_wfd
else:
stderr_wfd = stderr
try:
super(Popen, self).__init__(args,
stdin=stdin_rfd,
stdout=stdout_wfd,
stderr=stderr_wfd,
**kwds)
except:
for h in (stdin_wh, stdout_rh, stderr_rh):
if h is not None:
_winapi.CloseHandle(h)
raise
else:
if stdin_wh is not None:
self.stdin = PipeHandle(stdin_wh)
if stdout_rh is not None:
self.stdout = PipeHandle(stdout_rh)
if stderr_rh is not None:
self.stderr = PipeHandle(stderr_rh)
finally:
if stdin == PIPE:
os.close(stdin_rfd)
if stdout == PIPE:
os.close(stdout_wfd)
if stderr == PIPE:
os.close(stderr_wfd)
def _open(self):
"""Opens the current file without handle inheritance."""
if self.encoding is None:
with open(self.baseFilename, self.mode) as stream:
newosf = _duplicate(msvcrt.get_osfhandle(stream.fileno()))
new_fd = msvcrt.open_osfhandle(newosf, os.O_APPEND)
return os.fdopen(new_fd, self.mode)
return codecs.open(self.baseFilename, self.mode, self.encoding)
def _recv_item(connection, item):
"""receive one item. If it's a FileObject, unwrap it."""
if isinstance(item, _FileObject):
handle = recv_handle(connection)
if sys.platform == 'win32':
handle = msvcrt.open_osfhandle(handle, os.O_RDONLY)
return os.fdopen(handle, 'rb')
return item
def getOsFileHandle(pipe, flags): # Get file descriptor from argument pipe = int(pipe) if platform == "win32": # windows pipeoutfd = open_osfhandle(pipe, flags) else: # linux pipeoutfd = pipe return pipeoutfd
def detach_fd(self):
"""
Open a file descriptor for the HANDLE and release ownership.
Closing the file descriptor will also close the handle.
"""
fd = msvcrt.open_osfhandle(self.handle, 0)
self.handle = None
return fd
def open_handle(handle, mode):
flags = 0
if 'w' not in mode and '+' not in mode:
flags |= os.O_RDONLY
if 'b' not in mode:
flags |= os.O_TEXT
if 'a' in mode:
flags |= os.O_APPEND
return msvcrt.open_osfhandle(handle, flags)
def __init__(self, fd, mode):
assert mode in ('r', 'w')
self._mode = mode
flags = os.O_APPEND | os.O_BINARY
flags |= os.O_RDONLY if mode == 'r' else os.O_WRONLY
handle = open_osfhandle(fd, flags)
self._f = os.fdopen(handle, mode + 'b')
self._h = handle
self._fd = fd
def main():
'''
Run code specifed by data received over pipe
'''
global _forking_is_enabled
_Django_old_layout_hack__load()
assert is_forking(sys.argv)
_forking_is_enabled = False
handle = int(sys.argv[-1])
if sys.platform == 'win32':
fd = msvcrt.open_osfhandle(handle, os.O_RDONLY)
else:
fd = handle
from_parent = os.fdopen(fd, 'rb')
process.current_process()._inheriting = True
preparation_data = load(from_parent)
prepare(preparation_data)
# Huge hack to make logging before Process.run work.
try:
os.environ["MP_MAIN_FILE"] = sys.modules["__main__"].__file__
except KeyError:
pass
except AttributeError:
pass
loglevel = os.environ.get("_MP_FORK_LOGLEVEL_")
logfile = os.environ.get("_MP_FORK_LOGFILE_") or None
format = os.environ.get("_MP_FORK_LOGFORMAT_")
if loglevel:
from billiard import util
import logging
logger = util.get_logger()
logger.setLevel(int(loglevel))
if not logger.handlers:
logger._rudimentary_setup = True
logfile = logfile or sys.__stderr__
if hasattr(logfile, "write"):
handler = logging.StreamHandler(logfile)
else:
handler = logging.FileHandler(logfile)
formatter = logging.Formatter(
format or util.DEFAULT_LOGGING_FORMAT,
)
handler.setFormatter(formatter)
logger.addHandler(handler)
self = load(from_parent)
process.current_process()._inheriting = False
from_parent.close()
exitcode = self._bootstrap()
exit(exitcode)
def _getThumbnailSync(self, user):
import win32security
import win32api
import win32process
import win32con
import win32event
import win32file
from tempfile import mkstemp
from msvcrt import open_osfhandle
sAttrs = win32security.SECURITY_ATTRIBUTES()
sAttrs.bInheritHandle = True
tmpFd, tmpFname = mkstemp()
os.close(tmpFd)
hWrite = win32file.CreateFile(tmpFname, win32file.GENERIC_WRITE,
win32file.FILE_SHARE_READ,
sAttrs,
win32file.TRUNCATE_EXISTING,
win32file.FILE_ATTRIBUTE_TEMPORARY, 0)
hRead = win32file.CreateFile(tmpFname, win32file.GENERIC_READ,
win32file.FILE_SHARE_WRITE,
None,
win32file.OPEN_EXISTING,
0, 0)
startupInfo = win32process.STARTUPINFO()
startupInfo.dwFlags = win32process.STARTF_USESTDHANDLES | win32process.STARTF_USESHOWWINDOW
startupInfo.lpDesktop = r'winsta0\default'
startupInfo.hStdOutput = hWrite
startupInfo.hStdError = hWrite
hUser = self.__getUserToken(user)
try:
hProcess, hThread, dwPid, dwTid = \
win32process.CreateProcessAsUser(hUser, r'c:\sepiida\screenshot.exe',
'screenshot.exe - 320x240 50',
None, None, True, 0, None, None, startupInfo)
finally:
win32api.CloseHandle(hUser)
win32event.WaitForSingleObject(hProcess, win32event.INFINITE)
exitCode = win32process.GetExitCodeProcess(hProcess)
win32api.CloseHandle(hProcess)
win32api.CloseHandle(hThread)
win32api.CloseHandle(hWrite)
fd = open_osfhandle(hRead, os.O_RDONLY)
f = os.fdopen(fd, 'rb')
data = f.read()
f.close()
os.unlink(tmpFname)
return data
def redirectIOToConsole():
class CONSOLE_SCREEN_BUFFER_INFO(Structure):
_fields_ = [("dwSize", COORD),
("dwCursorPosition", COORD),
("wAttributes", WORD),
("srWindow", SMALL_RECT),
("dwMaximumWindowSize", DWORD)]
coninfo = CONSOLE_SCREEN_BUFFER_INFO()
# allocate console
if(windll.kernel32.GetStdHandle(STD_OUTPUT_HANDLE) != INVALID_HANDLE_VALUE):
windll.kernel32.AllocConsole()
# set the screen buffer to be big enough to let us scroll text
windll.kernel32.GetConsoleScreenBufferInfo(windll.kernel32.GetStdHandle(STD_OUTPUT_HANDLE), byref(coninfo))
coninfo.dwSize.Y = MAX_LINES
windll.kernel32.SetConsoleScreenBufferSize(windll.kernel32.GetStdHandle(STD_OUTPUT_HANDLE), coninfo.dwSize)
#redirect unbuffered STDOUT to the console
lStdHandle = windll.kernel32.GetStdHandle(STD_OUTPUT_HANDLE)
hConHandle = msvcrt.open_osfhandle(lStdHandle, os.O_TEXT)
fp = os.fdopen( hConHandle, "w" )
sys.stdout = fp
setvbuf( stdout, NULL, _IONBF, 0 )
# redirect unbuffered STDIN to the console
lStdHandle = windll.kernel32.GetStdHandle(STD_INPUT_HANDLE)
hConHandle = msvcrt.open_osfhandle(lStdHandle, os.O_TEXT)
fp = os.fdopen( hConHandle, "r" )
sys.stdin = fp
setvbuf( stdin, NULL, _IONBF, 0 )
#redirect unbuffered STDERR to the console
lStdHandle = windll.kernel32.GetStdHandle(STD_ERROR_HANDLE)
hConHandle = msvcrt.open_osfhandle(lStdHandle, os.O_TEXT)
fp = os.fdopen( hConHandle, "w" )
sys.stderr = fp
setvbuf( stderr, NULL, _IONBF, 0 )
# make cout, wcout, cin, wcin, wcerr, cerr, wclog and clog
# point to console as well
ios::sync_with_stdio()
def File2FileObject(pipe, mode):
"""Make a C stdio file object out of a win32 file handle"""
if string.find(mode, 'r') >= 0:
wmode = os.O_RDONLY
elif string.find(mode, 'w') >= 0:
wmode = os.O_WRONLY
if string.find(mode, 'b') >= 0:
wmode = wmode | os.O_BINARY
if string.find(mode, 't') >= 0:
wmode = wmode | os.O_TEXT
return os.fdopen(msvcrt.open_osfhandle(pipe.Detach(),wmode),mode)
def __init__(self, process_obj):
prep_data = spawn.get_preparation_data(
process_obj._name, process_obj.init_main_module)
# read end of pipe will be "stolen" by the child process
# -- see spawn_main() in spawn.py.
rhandle, wfd = _winapi.CreatePipe(None, 0)
if sys.version_info[:2] > (3, 3):
wfd = msvcrt.open_osfhandle(wfd, 0)
cmd = spawn.get_command_line(parent_pid=os.getpid(),
pipe_handle=rhandle)
cmd = ' '.join('"%s"' % x for x in cmd)
try:
with open(wfd, 'wb') as to_child:
# start process
try:
inherit = sys.version_info[:2] < (3, 4)
hp, ht, pid, tid = _winapi.CreateProcess(
spawn.get_executable(), cmd,
None, None, inherit, 0,
None, None, None)
_winapi.CloseHandle(ht)
except:
_winapi.CloseHandle(rhandle)
raise
# set attributes of self
self.pid = pid
self.returncode = None
self._handle = hp
self.sentinel = int(hp)
util.Finalize(self, _winapi.CloseHandle, (self.sentinel,))
# send information to child
set_spawning_popen(self)
if sys.version_info[:2] < (3, 4):
Popen._tls.process_handle = int(hp)
try:
reduction.dump(prep_data, to_child)
reduction.dump(process_obj, to_child)
finally:
set_spawning_popen(None)
if sys.version_info[:2] < (3, 4):
del Popen._tls.process_handle
except IOError as exc:
# IOError 22 happens when the launched subprocess terminated before
# wfd.close is called. Thus we can safely ignore it.
if exc.errno != 22:
raise
util.debug("While starting {}, ignored a IOError 22"
.format(process_obj._name))
def get_fd(fileobj):
""" Gets the file descriptor for a given fileobject
On Unix systems this returns the result of fileno()
On Windows systems, fileno() returns a HANDLE. This will open
that HANDLE and return a CRT file descriptor
"""
if IS_WINDOWS:
import msvcrt
return msvcrt.open_osfhandle(fileobj.fileno(), os.O_TEXT)
return fileobj.fileno()
def __init__(self, process_obj):
prep_data = spawn.get_preparation_data(process_obj._name)
# read end of pipe will be duplicated by the child process
# -- see spawn_main() in spawn.py.
#
# bpo-33929: Previously, the read end of pipe was "stolen" by the child
# process, but it leaked a handle if the child process had been
# terminated before it could steal the handle from the parent process.
rhandle, whandle = _winapi.CreatePipe(None, 0)
wfd = msvcrt.open_osfhandle(whandle, 0)
cmd = spawn.get_command_line(parent_pid=os.getpid(),
pipe_handle=rhandle)
cmd = ' '.join('"%s"' % x for x in cmd)
python_exe = spawn.get_executable()
# bpo-35797: When running in a venv, we bypass the redirect
# executor and launch our base Python.
if WINENV and _path_eq(python_exe, sys.executable):
python_exe = sys._base_executable
env = os.environ.copy()
env["__PYVENV_LAUNCHER__"] = sys.executable
else:
env = None
with open(wfd, 'wb', closefd=True) as to_child:
# start process
try:
hp, ht, pid, tid = _winapi.CreateProcess(
python_exe, cmd,
env, None, False, 0, None, None, None)
_winapi.CloseHandle(ht)
except:
_winapi.CloseHandle(rhandle)
raise
# set attributes of self
self.pid = pid
self.returncode = None
self._handle = hp
self.sentinel = int(hp)
self.finalizer = util.Finalize(self, _close_handles,
(self.sentinel, int(rhandle)))
# send information to child
set_spawning_popen(self)
try:
reduction.dump(prep_data, to_child)
reduction.dump(process_obj, to_child)
finally:
set_spawning_popen(None)
def _open_parent_file_handle(parent_process_handle, parent_file_handle, mode='r'):
if mode not in ['r', 'w']:
raise ValueError("mode must be 'r' or 'w'")
my_file_handle = win32api.DuplicateHandle(
parent_process_handle,
parent_file_handle,
win32api.GetCurrentProcess(),
0, #desiredAccess ignored because of DUPLICATE_SAME_ACCESS
0, #Inheritable
win32con.DUPLICATE_SAME_ACCESS | win32con.DUPLICATE_CLOSE_SOURCE)
infd = msvcrt.open_osfhandle(int(my_file_handle), os.O_RDONLY if mode == 'r' else os.O_WRONLY)
f = _ParentPassedFile(os.fdopen(infd, mode), my_file_handle)
return f
def _NamedPipe():
#TODO check that created handles are properly destroyed
import multiprocessing.connection as mpc
import serial, ctypes, msvcrt
kernel32 = ctypes.windll.kernel32
address = mpc.arbitrary_address('AF_PIPE')
h1 = kernel32.CreateNamedPipeA(address,
mpc.win32.PIPE_ACCESS_INBOUND | 0x40000000, # FILE_FLAG_OVERLAPPED
0, 1, 0,0, mpc.win32.NMPWAIT_WAIT_FOREVER, None)
h2 = kernel32.CreateFileA(address, mpc.win32.GENERIC_WRITE, 0,
None, mpc.win32.OPEN_EXISTING, 0, None)
overlapped = serial.win32.OVERLAPPED()
overlapped.hEvent = serial.win32.CreateEvent(None, 1, 0, None)
try:
err = kernel32.ConnectNamedPipe(h1, ctypes.byref(overlapped))
if err == 0 and serial.win32.GetLastError() == serial.win32.ERROR_IO_PENDING:
kernel32.WaitForSingleObject(overlapped.hEvent, -1)
finally:
kernel32.CloseHandle(overlapped.hEvent)
fdr = os.fdopen(msvcrt.open_osfhandle(h1, 0), 'rb')
fdw = os.fdopen(msvcrt.open_osfhandle(h2, 0), 'wb')
return fdr, fdw
def _open(self):
"""
Open the current base file with the (original) mode and encoding.
"""
if self.encoding is None:
stream = open(self.baseFilename, self.mode)
newosf = duplicate(msvcrt.get_osfhandle(stream.fileno()), inheritable=False)
newFD = msvcrt.open_osfhandle(newosf,os.O_APPEND)
newstream = os.fdopen(newFD,self.mode)
stream.close()
return newstream
else:
stream = codecs.open(self.baseFilename, self.mode, self.encoding)
return stream
def main():
'''
Run code specified by data received over pipe
'''
assert is_forking(sys.argv)
handle = int(sys.argv[-1])
fd = msvcrt.open_osfhandle(handle, os.O_RDONLY)
from_parent = os.fdopen(fd, 'rb')
process.current_process()._inheriting = True
preparation_data = load(from_parent)
spawn.prepare(preparation_data)
self = load(from_parent)
process.current_process()._inheriting = False
from_parent.close()
exitcode = self._bootstrap()
sys.exit(exitcode)
def loadFile(filename):
"""
This method is important for win32 to avoid locking the ConanSanbox.log file while it is opened by a running Conan server (which will be the usual).
In other platforms it is not necessary
"""
# get a handle using win32 API, specifyng the SHARED access!
handle = win32file.CreateFile(filename,win32file.GENERIC_READ,
win32file.FILE_SHARE_DELETE|win32file.FILE_SHARE_READ|win32file.FILE_SHARE_WRITE,
None,
win32file.OPEN_EXISTING,
0,
None)
# detach the handle
detached_handle = handle.Detach()
# get a file descriptor associated to the handle
file_descriptor = msvcrt.open_osfhandle(detached_handle, os.O_RDONLY)
return open(file_descriptor,"r",encoding="cp437", errors='ignore')
def __init__(self, process_obj):
prep_data = spawn.get_preparation_data(process_obj._name)
# read end of pipe will be duplicated by the child process
# -- see spawn_main() in spawn.py.
#
# bpo-33929: Previously, the read end of pipe was "stolen" by the child
# process, but it leaked a handle if the child process had been
# terminated before it could steal the handle from the parent process.
rhandle, whandle = _winapi.CreatePipe(None, 0)
wfd = msvcrt.open_osfhandle(whandle, 0)
cmd = spawn.get_command_line(parent_pid=os.getpid(),
pipe_handle=rhandle)
cmd = ' '.join('"%s"' % x for x in cmd)
with open(wfd, 'wb', closefd=True) as to_child:
# start process
try:
hp, ht, pid, tid = _winapi.CreateProcess(
spawn.get_executable(), cmd,
None, None, False, 0, None, None, None)
_winapi.CloseHandle(ht)
except:
_winapi.CloseHandle(rhandle)
raise
# set attributes of self
self.pid = pid
self.returncode = None
self._handle = hp
self.sentinel = int(hp)
self.finalizer = util.Finalize(self, _close_handles,
(self.sentinel, int(rhandle)))
# send information to child
set_spawning_popen(self)
try:
reduction.dump(prep_data, to_child)
reduction.dump(process_obj, to_child)
finally:
set_spawning_popen(None)
def os_open(path, flags, mode=0o777, share_flags=FILE_SHARE_VALID_FLAGS):
'''
Replacement for os.open() allowing moving or unlinking before closing
'''
if not isinstance(flags, Integral):
raise TypeError('flags must be an integer')
if not isinstance(mode, Integral):
raise TypeError('mode must be an integer')
if share_flags & ~FILE_SHARE_VALID_FLAGS:
raise ValueError('bad share_flags: %r' % share_flags)
access_flags = _ACCESS_MAP[flags & _ACCESS_MASK]
create_flags = _CREATE_MAP[flags & _CREATE_MASK]
attrib_flags = FILE_ATTRIBUTE_NORMAL
if flags & os.O_CREAT and mode & ~0o444 == 0:
attrib_flags = FILE_ATTRIBUTE_READONLY
if flags & os.O_TEMPORARY:
share_flags |= FILE_SHARE_DELETE
attrib_flags |= FILE_FLAG_DELETE_ON_CLOSE
access_flags |= DELETE
if flags & os.O_SHORT_LIVED:
attrib_flags |= FILE_ATTRIBUTE_TEMPORARY
if flags & os.O_SEQUENTIAL:
attrib_flags |= FILE_FLAG_SEQUENTIAL_SCAN
if flags & os.O_RANDOM:
attrib_flags |= FILE_FLAG_RANDOM_ACCESS
try:
h = win32file.CreateFileW(
path, access_flags, share_flags, None, create_flags, attrib_flags, None)
except pywintypes.error as e:
raise_winerror(e)
ans = msvcrt.open_osfhandle(h, flags | os.O_NOINHERIT)
h.Detach() # We dont want the handle to be automatically closed when h is deleted
return ans
def os_open(path,
flags,
mode=0o777,
share_flags=winutil.FILE_SHARE_VALID_FLAGS):
'''
Replacement for os.open() allowing moving or unlinking before closing
'''
if not isinstance(flags, Integral):
raise TypeError('flags must be an integer')
if not isinstance(mode, Integral):
raise TypeError('mode must be an integer')
if share_flags & ~winutil.FILE_SHARE_VALID_FLAGS:
raise ValueError('bad share_flags: %r' % share_flags)
access_flags = _ACCESS_MAP[flags & _ACCESS_MASK]
create_flags = _CREATE_MAP[flags & _CREATE_MASK]
attrib_flags = winutil.FILE_ATTRIBUTE_NORMAL
if flags & os.O_CREAT and mode & ~0o444 == 0:
attrib_flags = winutil.FILE_ATTRIBUTE_READONLY
if flags & os.O_TEMPORARY:
share_flags |= winutil.FILE_SHARE_DELETE
attrib_flags |= winutil.FILE_FLAG_DELETE_ON_CLOSE
access_flags |= winutil.DELETE
if flags & os.O_SHORT_LIVED:
attrib_flags |= winutil.FILE_ATTRIBUTE_TEMPORARY
if flags & os.O_SEQUENTIAL:
attrib_flags |= winutil.FILE_FLAG_SEQUENTIAL_SCAN
if flags & os.O_RANDOM:
attrib_flags |= winutil.FILE_FLAG_RANDOM_ACCESS
h = winutil.create_file(path, access_flags, share_flags, create_flags,
attrib_flags)
ans = msvcrt.open_osfhandle(int(h), flags | os.O_NOINHERIT)
h.detach()
return ans
def __init__(self,
name,
pipe_type='server' or 'client',
*,
open_mode=win32pipe.PIPE_ACCESS_DUPLEX
| win32file.FILE_FLAG_OVERLAPPED,
pipe_mode=win32pipe.PIPE_TYPE_BYTE | win32pipe.PIPE_NOWAIT,
maxinstances=255,
out_buffer_size=1000000,
in_buffer_size=1000000,
default_timeout=50,
security_attrib=None):
"""An implementation of a file-like python object pipe
https://msdn.microsoft.com/en-us/library/windows/desktop/aa365150(v=vs.85).aspx"""
self.pipe_type = pipe_type
self.name = name
self.open_mode = open_mode
self.pipe_mode = pipe_mode
if pipe_type == 'server':
self.handle = win32pipe.CreateNamedPipe(
PIPE_ROOT + name,
open_mode, # default PIPE_ACCESS_DUPLEX|FILE_FLAG_OVERLAPPED
pipe_mode, # default PIPE_TYPE_BYTE|PIPE_NOWAIT
maxinstances, # default 255
out_buffer_size, # default 1000000
in_buffer_size, # default 1000000
default_timeout, # default 50
security_attrib # default None
)
elif pipe_type == 'client':
# it doesn't matter what type of pipe the server is so long as we know the name
self.handle = win32file.CreateFile(
PIPE_ROOT + name,
win32file.GENERIC_READ | win32file.GENERIC_WRITE, 0, None,
win32file.OPEN_EXISTING, 0, None)
else:
raise ValueError('pipe_type', ('server', 'client'))
self.fd = msvcrt.open_osfhandle(self.handle, 0)
self.is_connected = False
self.flags, self.out_buffer_size, self.in_buffer_size, self.maxinstances = win32pipe.GetNamedPipeInfo(
self.handle)
def __init__(self, output=sys.stdout):
self._close_output = False
# If output is integer, assume it is file descriptor and open it
if isinstance(output, int):
self._close_output = True
if sys.platform == 'win32':
output = msvcrt.open_osfhandle(output, 0)
output = open(output, 'wb')
# Get underlying buffered file object
try:
self.output = output.buffer
except AttributeError:
self.output = output
# Use only one writer thread to preserve sequence of written frequencies
self._executor = threadpool.ThreadPoolExecutor(
max_workers=1,
max_queue_size=100,
thread_name_prefix='Writer_thread')
def ToFileDescriptor(self) -> int:
"""Converts the value to a file descriptor."""
if self._file_descriptor is not None:
return self._file_descriptor
if platform.system() == "Windows":
import msvcrt # pylint: disable=g-import-not-at-top
if self._mode == Mode.READ:
mode = os.O_RDONLY
elif self._mode == Mode.WRITE:
mode = os.O_APPEND
else:
raise ValueError(f"Invalid mode {self._mode}")
if self._handle is None:
raise ValueError("Handle is required.")
# pytype doesn't see the functions in msvcrt
self._file_descriptor = msvcrt.open_osfhandle(self._handle, mode) # pytype: disable=module-attr
# The file descriptor takes ownership of the handle.
self._handle = None
return self._file_descriptor
else:
raise ValueError("File descriptor is required.")
def open_log_file() -> Tuple[TextIO, str]:
log_filename = get_newest_log_filename()
logger.debug(f"Opening log file {log_filename}")
# source:
# https://www.thepythoncorner.com/2016/10/python-how-to-open-a-file-on-windows-without-locking-it/
# get a handle using win32 API, specifying SHARED access!
handle = win32file.CreateFile(
log_filename, win32file.GENERIC_READ, win32file.FILE_SHARE_DELETE
| win32file.FILE_SHARE_READ | win32file.FILE_SHARE_WRITE, None,
win32file.OPEN_EXISTING, 0, None)
# detach the handle
detached_handle = handle.Detach()
# get a file descriptor associated to the handle
file_descriptor = msvcrt.open_osfhandle(detached_handle, os.O_RDONLY)
# open the file descriptor
f = open(file_descriptor, encoding="UTF-8")
# seek to end
# f.seek(0, os.SEEK_END)
logger.debug(f"Opened log file {log_filename}")
return f, log_filename
def __init__(self, process_obj):
os.environ["MULTIPROCESSING_FORKING_DISABLE"] = "1"
spawn._Django_old_layout_hack__save()
prep_data = spawn.get_preparation_data(process_obj._name)
# read end of pipe will be "stolen" by the child process
# -- see spawn_main() in spawn.py.
rhandle, whandle = _winapi.CreatePipe(None, 0)
wfd = msvcrt.open_osfhandle(whandle, 0)
cmd = spawn.get_command_line(parent_pid=os.getpid(),
pipe_handle=rhandle)
cmd = ' '.join('"%s"' % x for x in cmd)
with io.open(wfd, 'wb', closefd=True) as to_child:
# start process
try:
hp, ht, pid, tid = _winapi.CreateProcess(
spawn.get_executable(), cmd,
None, None, False, 0, None, None, None)
_winapi.CloseHandle(ht)
except:
_winapi.CloseHandle(rhandle)
raise
# set attributes of self
self.pid = pid
self.returncode = None
self._handle = hp
self.sentinel = int(hp)
util.Finalize(self, _winapi.CloseHandle, (self.sentinel,))
# send information to child
context.set_spawning_popen(self)
try:
reduction.dump(prep_data, to_child)
reduction.dump(process_obj, to_child)
finally:
context.set_spawning_popen(None)
def share_open(path, *args, **kargs):
# does need all three file share flags.
handle = win32file.CreateFile(
path, win32file.GENERIC_READ, win32file.FILE_SHARE_DELETE
| win32file.FILE_SHARE_READ | win32file.FILE_SHARE_WRITE, None,
win32file.OPEN_EXISTING, 0, None)
# detach the handle
detached_handle = handle.Detach()
# get a file descriptor associated to the handle
file_descriptor = msvcrt.open_osfhandle(detached_handle,
os.O_RDONLY)
## these can be called *before* the with open read, so
## clearly they don't matter to that and I don't want to
## risk leaking file descriptors.
handle.close()
win32file.CloseHandle(handle)
# open the file descriptor
# this will still allow for "with share_open():"
return open(file_descriptor, *args, **kargs)
def spawn_main(pipe_handle, parent_pid=None, tracker_fd=None):
'''
Run code specified by data received over pipe
'''
assert is_forking(sys.argv), "Not forking"
if sys.platform == 'win32':
import msvcrt
import _winapi
if parent_pid is not None:
source_process = _winapi.OpenProcess(
_winapi.PROCESS_DUP_HANDLE, False, parent_pid)
else:
source_process = None
new_handle = reduction.duplicate(pipe_handle,
source_process=source_process)
fd = msvcrt.open_osfhandle(new_handle, os.O_RDONLY)
else:
from . import semaphore_tracker
semaphore_tracker._semaphore_tracker._fd = tracker_fd
fd = pipe_handle
exitcode = _main(fd)
sys.exit(exitcode)
def _open_log_file(self):
self.logger.debug("(Re-)Opening server log file...")
# source:
# https://www.thepythoncorner.com/2016/10/python-how-to-open-a-file-on-windows-without-locking-it/
# get a handle using win32 API, specifying the SHARED access!
handle = win32file.CreateFile(
self.log_filename, win32file.GENERIC_READ,
win32file.FILE_SHARE_DELETE | win32file.FILE_SHARE_READ
| win32file.FILE_SHARE_WRITE, None, win32file.OPEN_EXISTING, 0,
None)
# detach the handle
detached_handle = handle.Detach()
# get a file descriptor associated to the handle
file_descriptor = msvcrt.open_osfhandle(detached_handle, os.O_RDONLY)
# open the file descriptor
f = open(file_descriptor, encoding="UTF-8")
# seek to end
f.seek(0, os.SEEK_END)
size = os.fstat(f.fileno()).st_size
self.logger.debug(f"Opened server log file. Size: {size}")
return f, size
def winpopen4(orig, cmd, env=None, newlines=False, bufsize=-1):
"""Same as util.popen4, but manually creates an input pipe with a
larger than default buffer"""
import msvcrt
import _subprocess
handles = _subprocess.CreatePipe(None, pipei_bufsize)
rfd, wfd = [msvcrt.open_osfhandle(h, 0) for h in handles]
handles[0].Detach()
handles[1].Detach()
p = subprocess.Popen(
cmd,
shell=True,
bufsize=bufsize,
close_fds=False,
stdin=rfd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=newlines,
env=env,
)
p.stdin = util.fdopen(wfd, "wb", bufsize)
return p.stdin, p.stdout, p.stderr, p
def open_file_nonblocking(filename, access):
# Removes the b for binary access.
internal_access = access.replace("b", "")
access_mode = ACCESS_MODES[internal_access]
open_mode = OPEN_MODES[internal_access]
handle = wintypes.HANDLE(
ctypes.windll.kernel32.CreateFileW(
wintypes.LPWSTR(filename),
wintypes.DWORD(access_mode),
wintypes.DWORD(2 | 1), # File share read and write
ctypes.c_void_p(0),
wintypes.DWORD(open_mode),
wintypes.DWORD(0),
wintypes.HANDLE(0)))
try:
fd = msvcrt.open_osfhandle(handle.value, 0)
except OverflowError as exc:
# Python 3.X
raise OSError("Failed to open file.") from None
# Python 2
# raise OSError("Failed to open file.")
return os.fdopen(fd, access)
def start(self):
# Create a named pipe which allows for asynchronous IO in Windows
pipe_name = r'\\.\pipe\typeperf_output_' + str(uuid.uuid4())
open_mode = _winapi.PIPE_ACCESS_INBOUND
open_mode |= _winapi.FILE_FLAG_FIRST_PIPE_INSTANCE
open_mode |= _winapi.FILE_FLAG_OVERLAPPED
# This is the read end of the pipe, where we will be grabbing output
self.pipe = _winapi.CreateNamedPipe(pipe_name, open_mode,
_winapi.PIPE_WAIT, 1, BUFSIZE,
BUFSIZE,
_winapi.NMPWAIT_WAIT_FOREVER,
_winapi.NULL)
# The write end of the pipe which is passed to the created process
pipe_write_end = _winapi.CreateFile(pipe_name, _winapi.GENERIC_WRITE,
0, _winapi.NULL,
_winapi.OPEN_EXISTING, 0,
_winapi.NULL)
# Open up the handle as a python file object so we can pass it to
# subprocess
command_stdout = msvcrt.open_osfhandle(pipe_write_end, 0)
# Connect to the read end of the pipe in overlap/async mode
overlap = _winapi.ConnectNamedPipe(self.pipe, overlapped=True)
overlap.GetOverlappedResult(True)
# Spawn off the load monitor
counter_name = self._get_counter_name()
command = ['typeperf', counter_name, '-si', str(SAMPLING_INTERVAL)]
self._popen = subprocess.Popen(' '.join(command),
stdout=command_stdout,
cwd=support.SAVEDCWD)
# Close our copy of the write end of the pipe
os.close(command_stdout)
def _DoParent(self):
super(RedirectedExecutable, self)._DoParent()
# Process the various redirected streams until none of the
# streams remain open.
if sys.platform != "win32":
while 1:
# Prepare the lists of interesting descriptors.
read_fds = []
write_fds = []
if self._stdout_pipe:
read_fds.append(self._stdout_pipe[0])
if self._stderr_pipe:
read_fds.append(self._stderr_pipe[0])
if self._stdin_pipe:
write_fds.append(self._stdin_pipe[1])
# If there are no longer any interesting descriptors, we are
# done.
if not read_fds and not write_fds:
return
# See which descriptors are ready for processing.
read_ready, write_ready \
= select.select(read_fds, write_fds, [])[:2]
# Process them.
if self._stdout_pipe and self._stdout_pipe[0] in read_ready:
self._ReadStdout()
if self._stderr_pipe and self._stderr_pipe[0] in read_ready:
self._ReadStderr()
if self._stdin_pipe and self._stdin_pipe[1] in write_ready:
self._WriteStdin()
else:
# Under Windows, neither select, nor
# WaitForMultipleObjects, works on pipes. The only
# approach that is reliable under all versions of Windows
# is to use a separate thread for each handle. By
# converting the pipe ends from OS handles to file
# descriptors at this point, _ReadStdout, _ReadStderr, and
# _WriteStdin can use the same implementations under
# Windows that they do under UNIX.
if self._stdin_pipe:
h = self._stdin_pipe[1]
self._stdin_pipe[1] = msvcrt.open_osfhandle(h, 0)
h.Detach()
stdin_thread = Thread(target=self.__CallUntilNone,
args=(self._WriteStdin, "_stdin_pipe"))
else:
stdin_thread = None
if self._stdout_pipe:
h = self._stdout_pipe[0]
self._stdout_pipe[0] = msvcrt.open_osfhandle(h, 0)
h.Detach()
stdout_thread = Thread(target=self.__CallUntilNone,
args=(self._ReadStdout, "_stdout_pipe"))
else:
stdout_thread = None
if self._stderr_pipe:
h = self._stderr_pipe[0]
self._stderr_pipe[0] = msvcrt.open_osfhandle(h, 0)
h.Detach()
stderr_thread = Thread(target=self.__CallUntilNone,
args=(self._ReadStderr, "_stderr_pipe"))
else:
stderr_thread = None
# Start the threads.
for t in stdin_thread, stdout_thread, stderr_thread:
if t:
t.start()
# Wait for them to finish.
for t in stdin_thread, stdout_thread, stderr_thread:
if t:
t.join()
def fdopen(handle, mode):
# type: (int, str) -> IO[Any]
os_mode = os.O_WRONLY if mode == "wb" else os.O_RDONLY
fileno = open_osfhandle(handle, os_mode)
return util.fdopen(fileno, mode)
def __init__(self, path_handle, mode='r', share=None):
"""If 'path_handle' is a string, opens that file for
asynchronous I/O; if it is a handle (pipe client / server,
for example), sets up for asynchronous I/O. 'mode' is as
per 'open' Python function, although limited to
basic/common modes.
"""
self._overlap = pywintypes.OVERLAPPED()
if isinstance(path_handle, str):
self._path = path_handle
if mode.startswith('r'):
access = win32file.GENERIC_READ
if share is None:
share = win32con.FILE_SHARE_READ | win32con.FILE_SHARE_WRITE
create = win32file.OPEN_EXISTING
if '+' in mode:
access |= win32file.GENERIC_WRITE
elif mode.startswith('w'):
access = win32file.GENERIC_WRITE
if share is None:
share = win32con.FILE_SHARE_READ | win32con.FILE_SHARE_WRITE
create = win32file.CREATE_ALWAYS
if '+' in mode:
access |= win32file.GENERIC_READ
elif mode.startswith('a'):
access = win32file.GENERIC_WRITE
if share is None:
share = win32con.FILE_SHARE_READ | win32con.FILE_SHARE_WRITE
create = win32file.OPEN_ALWAYS
if '+' in mode:
access |= win32file.GENERIC_READ
# TODO: if reading, offset should be 0?
sb = os.stat(path_handle)
self._overlap.Offset = sb.st_size
else:
self._overlap = None
raise ValueError('invalid mode "%s"' % mode)
flags = win32file.FILE_FLAG_OVERLAPPED
try:
self._handle = win32file.CreateFile(path_handle, access, share, None, create,
flags, None)
except Exception:
self._overlap = None
raise
if mode.startswith('r'):
flags = os.O_RDONLY
elif mode.startswith('a'):
flags = os.O_APPEND
else:
flags = 0
self._fileno = msvcrt.open_osfhandle(self._handle, flags)
else:
self._handle = path_handle
# pipe mode should be either 'r' or 'w'
flags = os.O_RDONLY if mode.startswith('r') else 0
self._fileno = msvcrt.open_osfhandle(self._handle, flags)
self._buflist = []
self._read_result = None
self._write_result = None
self._timeout = None
self._timeout_id = None
self._pycos = Pycos.scheduler()
if self._pycos:
self._notifier = self._pycos._notifier
self._notifier.register(self._handle)
else:
self._notifier = None
self._event = None
def connect(self) -> BufferedIOBase:
_winapi.ConnectNamedPipe(self._handle, _winapi.NULL)
fd = msvcrt.open_osfhandle(self._handle, 0)
self.file = os.fdopen(fd, 'w+b')
return self.file
def __init__(self,
f,
args,
stdin=None,
stdout=None,
stderr=None,
universal_newlines=False):
"""Parameters
----------
f : function
The function to be executed.
args : list
A (possibly empty) list containing the arguments that were given on
the command line
stdin : file-like, optional
A file-like object representing stdin (input can be read from
here). If `stdin` is not provided or if it is explicitly set to
`None`, then an instance of `io.StringIO` representing an empty
file is used.
stdout : file-like, optional
A file-like object representing stdout (normal output can be
written here). If `stdout` is not provided or if it is explicitly
set to `None`, then `sys.stdout` is used.
stderr : file-like, optional
A file-like object representing stderr (error output can be
written here). If `stderr` is not provided or if it is explicitly
set to `None`, then `sys.stderr` is used.
"""
self.f = f
"""
The function to be executed. It should be a function of four
arguments, described below.
Parameters
----------
args : list
A (possibly empty) list containing the arguments that were given on
the command line
stdin : file-like
A file-like object representing stdin (input can be read from
here).
stdout : file-like
A file-like object representing stdout (normal output can be
written here).
stderr : file-like
A file-like object representing stderr (error output can be
written here).
"""
self.args = args
self.pid = None
self.returncode = None
self.wait = self.join
handles = self._get_handles(stdin, stdout, stderr)
(self.p2cread, self.p2cwrite, self.c2pread, self.c2pwrite,
self.errread, self.errwrite) = handles
# default values
self.stdin = stdin
self.stdout = None
self.stderr = None
if ON_WINDOWS:
if self.p2cwrite != -1:
self.p2cwrite = msvcrt.open_osfhandle(self.p2cwrite.Detach(),
0)
if self.c2pread != -1:
self.c2pread = msvcrt.open_osfhandle(self.c2pread.Detach(), 0)
if self.errread != -1:
self.errread = msvcrt.open_osfhandle(self.errread.Detach(), 0)
if self.p2cwrite != -1:
self.stdin = io.open(self.p2cwrite, 'wb', -1)
if universal_newlines:
self.stdin = io.TextIOWrapper(self.stdin,
write_through=True,
line_buffering=False)
if self.c2pread != -1:
self.stdout = io.open(self.c2pread, 'rb', -1)
if universal_newlines:
self.stdout = io.TextIOWrapper(self.stdout)
if self.errread != -1:
self.stderr = io.open(self.errread, 'rb', -1)
if universal_newlines:
self.stderr = io.TextIOWrapper(self.stderr)
Thread.__init__(self)
self.start()
def create_pipe_to_child_stdin(): # noqa: F811
# for stdin, we want the write end (our end) to use overlapped I/O
rh, wh = windows_pipe(overlapped=(False, True))
return PipeSendStream(wh), msvcrt.open_osfhandle(rh, os.O_RDONLY)
def __init__(self, args, bufsize=0, executable=None,
stdin=None, stdout=None, stderr=None,
preexec_fn=None, close_fds=False, shell=False,
cwd=None, env=None, universal_newlines=False,
startupinfo=None, creationflags=0):
"""Create new Popen instance."""
_cleanup()
if not isinstance(bufsize, (int, long)):
raise TypeError("bufsize must be an integer")
if mswindows:
if preexec_fn is not None:
raise ValueError("preexec_fn is not supported on Windows "
"platforms")
if close_fds and (stdin is not None or stdout is not None or
stderr is not None):
raise ValueError("close_fds is not supported on Windows "
"platforms if you redirect stdin/stdout/stderr")
else:
# POSIX
if startupinfo is not None:
raise ValueError("startupinfo is only supported on Windows "
"platforms")
if creationflags != 0:
raise ValueError("creationflags is only supported on Windows "
"platforms")
self.stdin = None
self.stdout = None
self.stderr = None
self.pid = None
self.returncode = None
self.universal_newlines = universal_newlines
# Input and output objects. The general principle is like
# this:
#
# Parent Child
# ------ -----
# p2cwrite ---stdin---> p2cread
# c2pread <--stdout--- c2pwrite
# errread <--stderr--- errwrite
#
# On POSIX, the child objects are file descriptors. On
# Windows, these are Windows file handles. The parent objects
# are file descriptors on both platforms. The parent objects
# are None when not using PIPEs. The child objects are None
# when not redirecting.
(p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite), to_close = self._get_handles(stdin, stdout, stderr)
try:
self._execute_child(args, executable, preexec_fn, close_fds,
cwd, env, universal_newlines,
startupinfo, creationflags, shell, to_close,
p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite)
except Exception:
# Preserve original exception in case os.close raises.
exc_type, exc_value, exc_trace = sys.exc_info()
for fd in to_close:
try:
if mswindows:
fd.Close()
else:
os.close(fd)
except EnvironmentError:
pass
raise exc_type, exc_value, exc_trace
if mswindows:
if p2cwrite is not None:
p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0)
if c2pread is not None:
c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0)
if errread is not None:
errread = msvcrt.open_osfhandle(errread.Detach(), 0)
if p2cwrite is not None:
self.stdin = os.fdopen(p2cwrite, 'wb', bufsize)
if c2pread is not None:
if universal_newlines:
self.stdout = os.fdopen(c2pread, 'rU', bufsize)
else:
self.stdout = os.fdopen(c2pread, 'rb', bufsize)
if errread is not None:
if universal_newlines:
self.stderr = os.fdopen(errread, 'rU', bufsize)
else:
self.stderr = os.fdopen(errread, 'rb', bufsize)
import os
import win32file
import msvcrt
filename = "prova.log"
# get an handle using win32 API, specifyng the SHARED access!
handle = win32file.CreateFile(filename,
win32file.GENERIC_READ,
win32file.FILE_SHARE_DELETE |
win32file.FILE_SHARE_READ |
win32file.FILE_SHARE_WRITE,
None,
win32file.OPEN_EXISTING,
0,
None)
# detach the handle
detached_handle = handle.Detach()
# get a file descriptor associated to the handle\
file_descriptor = msvcrt.open_osfhandle(
detached_handle, os.O_RDONLY)
# open the file descriptor
file = open(file_descriptor)
for line in file:
print(line)
def main(self, token):
connection = Connection('localhost', port=5672)
connection.open()
session = connection.session(str(uuid4()))
receiver = session.receiver('amq.topic')
local_ip = socket.gethostbyname(socket.gethostname())
localhost_name = platform.uname()[1]
def make_inheritable(token):
"""Return a duplicate of handle, which is inheritable"""
return win32api.DuplicateHandle(win32api.GetCurrentProcess(),
token,
win32api.GetCurrentProcess(), 0, 1,
win32con.DUPLICATE_SAME_ACCESS)
while True:
message = receiver.fetch()
session.acknowledge()
sender = session.sender(message.reply_to)
command = base64.b64decode(message.content)
if command.startswith('winrs' or 'winrm') != True or command.find(
'-r:'
) == -1 or command.find('localhost') != -1 or command.find(
localhost_name) != -1 or command.find(local_ip) != -1:
sender.send(
Message(
base64.b64encode(
'Commands against the proxy are not accepted')))
else:
#Start the process:
# First let's create the communication pipes used by the process
# we need to have the pipes inherit the rights from token
stdin_read, stdin_write = win32pipe.CreatePipe(None, 0)
stdin_read = make_inheritable(stdin_read)
stdout_read, stdout_write = win32pipe.CreatePipe(None, 0)
stdout_write = make_inheritable(stdout_write)
stderr_read, stderr_write = win32pipe.CreatePipe(None, 0)
stderr_write = make_inheritable(stderr_write)
# Set start-up parameters the process will use.
#Here we specify the pipes for input, output and error.
si = win32process.STARTUPINFO()
si.dwFlags = win32con.STARTF_USESTDHANDLES
si.hStdInput = stdin_read
si.hStdOutput = stdout_write
si.hStdError = stderr_write
procArgs = (
None, # appName
command, # commandLine
None, # processAttributes
None, # threadAttributes
1, # bInheritHandles
0, # dwCreationFlags
None, # newEnvironment
None, # currentDirectory
si) # startupinfo
# CreateProcessAsUser takes the first parameter the token,
# this way the process will impersonate a user
try:
hProcess, hThread, PId, TId = win32process.CreateProcessAsUser(
token, *procArgs)
hThread.Close()
if stdin_read is not None:
stdin_read.Close()
if stdout_write is not None:
stdout_write.Close()
if stderr_write is not None:
stderr_write.Close()
stdin_write = msvcrt.open_osfhandle(
stdin_write.Detach(), 0)
stdout_read = msvcrt.open_osfhandle(
stdout_read.Detach(), 0)
stderr_read = msvcrt.open_osfhandle(
stderr_read.Detach(), 0)
stdin_file = os.fdopen(stdin_write, 'wb', 0)
stdout_file = os.fdopen(stdout_read, 'rU', 0)
stderr_file = os.fdopen(stderr_read, 'rU', 0)
def readerthread(fh, buffer):
buffer.append(fh.read())
def translate_newlines(data):
data = data.replace("\r\n", "\n")
data = data.replace("\r", "\n")
return data
def wait():
"""Wait for child process to terminate. Returns returncode
attribute."""
win32event.WaitForSingleObject(hProcess,
win32event.INFINITE)
returncode = win32process.GetExitCodeProcess(hProcess)
return returncode
def communicate():
if stdout_file:
stdout = []
stdout_thread = threading.Thread(
target=readerthread,
args=(stdout_file, stdout))
stdout_thread.setDaemon(True)
stdout_thread.start()
if stderr_file:
stderr = []
stderr_thread = threading.Thread(
target=readerthread,
args=(stderr_file, stderr))
stderr_thread.setDaemon(True)
stderr_thread.start()
stdin_file.close()
if stdout_file:
stdout_thread.join()
if stderr_file:
stderr_thread.join()
if stdout is not None:
stdout = stdout[0]
if stderr is not None:
stderr = stderr[0]
if stdout:
stdout = translate_newlines(stdout)
if stderr:
stderr = translate_newlines(stderr)
return_code = wait()
return (stdout, stderr, return_code)
ret_stdout, ret_stderr, retcode = communicate()
result = Message(base64.b64encode(str(ret_stdout)))
result.properties["retcode"] = base64.b64encode(
str(retcode))
if ret_stderr:
result.properties["stderr"] = base64.b64encode(
str(ret_stderr))
else:
result.properties["stderr"] = base64.b64encode('')
sender.send(result)
except Exception as exception_message:
result = Message(base64.b64encode(''))
result.properties["retcode"] = base64.b64encode(
str(exception_message[0]))
result.properties["stderr"] = base64.b64encode(
str(exception_message[2]))
sender.send(result)
def runas(cmdLine, username, password=None, cwd=None):
"""
Run a command as another user. If the process is running as an admin or
system account this method does not require a password. Other non
privileged accounts need to provide a password for the user to runas.
Commands are run in with the highest level privileges possible for the
account provided.
"""
# Validate the domain and sid exist for the username
username, domain = split_username(username)
try:
_, domain, _ = win32security.LookupAccountName(domain, username)
except pywintypes.error as exc:
message = win32api.FormatMessage(exc.winerror).rstrip("\n")
raise CommandExecutionError(message)
# Elevate the token from the current process
access = win32security.TOKEN_QUERY | win32security.TOKEN_ADJUST_PRIVILEGES
th = win32security.OpenProcessToken(win32api.GetCurrentProcess(), access)
salt.platform.win.elevate_token(th)
# Try to impersonate the SYSTEM user. This process needs to be running as a
# user who as been granted the SeImpersonatePrivilege, Administrator
# accounts have this permission by default.
try:
impersonation_token = salt.platform.win.impersonate_sid(
salt.platform.win.SYSTEM_SID,
session_id=0,
privs=["SeTcbPrivilege"],
)
except OSError: # pylint: disable=undefined-variable
log.debug("Unable to impersonate SYSTEM user")
impersonation_token = None
win32api.CloseHandle(th)
# Impersonation of the SYSTEM user failed. Fallback to an un-privileged
# runas.
if not impersonation_token:
log.debug("No impersonation token, using unprivileged runas")
return runas_unpriv(cmdLine, username, password, cwd)
if domain == "NT AUTHORITY":
# Logon as a system level account, SYSTEM, LOCAL SERVICE, or NETWORK
# SERVICE.
user_token = win32security.LogonUser(
username,
domain,
"",
win32con.LOGON32_LOGON_SERVICE,
win32con.LOGON32_PROVIDER_DEFAULT,
)
elif password:
# Login with a password.
user_token = win32security.LogonUser(
username,
domain,
password,
win32con.LOGON32_LOGON_INTERACTIVE,
win32con.LOGON32_PROVIDER_DEFAULT,
)
else:
# Login without a password. This always returns an elevated token.
user_token = salt.platform.win.logon_msv1_s4u(username).Token
# Get a linked user token to elevate if needed
elevation_type = win32security.GetTokenInformation(
user_token, win32security.TokenElevationType)
if elevation_type > 1:
user_token = win32security.GetTokenInformation(
user_token, win32security.TokenLinkedToken)
# Elevate the user token
salt.platform.win.elevate_token(user_token)
# Make sure the user's token has access to a windows station and desktop
salt.platform.win.grant_winsta_and_desktop(user_token)
# Create pipes for standard in, out and error streams
security_attributes = win32security.SECURITY_ATTRIBUTES()
security_attributes.bInheritHandle = 1
stdin_read, stdin_write = win32pipe.CreatePipe(security_attributes, 0)
stdin_read = salt.platform.win.make_inheritable(stdin_read)
stdout_read, stdout_write = win32pipe.CreatePipe(security_attributes, 0)
stdout_write = salt.platform.win.make_inheritable(stdout_write)
stderr_read, stderr_write = win32pipe.CreatePipe(security_attributes, 0)
stderr_write = salt.platform.win.make_inheritable(stderr_write)
# Run the process without showing a window.
creationflags = (win32process.CREATE_NO_WINDOW
| win32process.CREATE_NEW_CONSOLE
| win32process.CREATE_SUSPENDED)
startup_info = salt.platform.win.STARTUPINFO(
dwFlags=win32con.STARTF_USESTDHANDLES,
hStdInput=stdin_read.handle,
hStdOutput=stdout_write.handle,
hStdError=stderr_write.handle,
)
# Create the environment for the user
env = create_env(user_token, False)
hProcess = None
try:
# Start the process in a suspended state.
process_info = salt.platform.win.CreateProcessWithTokenW(
int(user_token),
logonflags=1,
applicationname=None,
commandline=cmdLine,
currentdirectory=cwd,
creationflags=creationflags,
startupinfo=startup_info,
environment=env,
)
hProcess = process_info.hProcess
hThread = process_info.hThread
dwProcessId = process_info.dwProcessId
dwThreadId = process_info.dwThreadId
# We don't use these so let's close the handle
salt.platform.win.kernel32.CloseHandle(stdin_write.handle)
salt.platform.win.kernel32.CloseHandle(stdout_write.handle)
salt.platform.win.kernel32.CloseHandle(stderr_write.handle)
ret = {"pid": dwProcessId}
# Resume the process
psutil.Process(dwProcessId).resume()
# Wait for the process to exit and get its return code.
if (win32event.WaitForSingleObject(
hProcess, win32event.INFINITE) == win32con.WAIT_OBJECT_0):
exitcode = win32process.GetExitCodeProcess(hProcess)
ret["retcode"] = exitcode
# Read standard out
fd_out = msvcrt.open_osfhandle(stdout_read.handle,
os.O_RDONLY | os.O_TEXT)
with os.fdopen(fd_out, "r") as f_out:
stdout = f_out.read()
ret["stdout"] = stdout
# Read standard error
fd_err = msvcrt.open_osfhandle(stderr_read.handle,
os.O_RDONLY | os.O_TEXT)
with os.fdopen(fd_err, "r") as f_err:
stderr = f_err.read()
ret["stderr"] = stderr
finally:
if hProcess is not None:
salt.platform.win.kernel32.CloseHandle(hProcess)
win32api.CloseHandle(th)
win32api.CloseHandle(user_token)
if impersonation_token:
win32security.RevertToSelf()
win32api.CloseHandle(impersonation_token)
return ret
def create_pipe_from_child_output(): # noqa: F811
# for stdout/err, it's the read end that's overlapped
rh, wh = windows_pipe(overlapped=(True, False))
return PipeReceiveStream(rh), msvcrt.open_osfhandle(wh, 0)
import interface
import win32pipe
import win32file
import msvcrt
import os
#mode='npceditor' will fetch answers only from npceditor.
#mode='classifier' will fetch answers only from classifier.
#mode='ensemble' will fetch answers from both classifier and ensemble and decide the best
bi = interface.BackendInterface(mode='ensemble')
# open server pipe from Unity
pipe = win32file.CreateFile("\\\\.\\pipe\\pipe_unity",
win32pipe.PIPE_ACCESS_DUPLEX, 0, None,
win32file.OPEN_EXISTING, 0, None)
read_fd = msvcrt.open_osfhandle(pipe, os.O_RDONLY)
reader = open(read_fd, "r")
input = reader.readline()
write_fd = msvcrt.open_osfhandle(pipe, os.O_WRONLY)
writer = open(write_fd, "w")
# get the list of topics
topics = bi.get_topics()
end_flag = False
while end_flag == False:
input = input.replace('\r', '')
input = input.replace('\n', '')
print(input)
# let Unity know the pipes are ready for communication
import argparse
parser = argparse.ArgumentParser('Command line parser')
parser.add_argument('--pipe', type=int, default=None,
help='File handle numbers for the pipe')
parser.add_argument('--semaphore', type=int, default=None,
help='File handle name for the semaphore tracker')
parser.add_argument('--strat', type=str, default='buff',
help='Strategy for communication dump')
args = parser.parse_args()
info = dict()
r = args.pipe
if sys.platform == 'win32':
import msvcrt
r = msvcrt.open_osfhandle(r, os.O_RDONLY)
else:
semaphore_tracker._semaphore_tracker._fd = args.semaphore
if sys.version_info[:2] > (3, 3):
from multiprocessing import context
from .process import LokyContext
context._concrete_contexts['loky'] = LokyContext()
exitcode = 1
try:
from_parent = os.fdopen(r, 'rb')
prep_data = pickle.load(from_parent)
spawn.prepare(prep_data)
process_obj = pickle.load(from_parent)
from_parent.close()
def __init__(self, handle=None):
if not handle:
handle = WSASocket(socket.AF_INET, socket.SOCK_STREAM,
socket.IPPROTO_TCP)
self._file_handle = handle
self._file_no = open_osfhandle(self._file_handle, 0)
