def read_bytes(self, buffer_size=4096):
out = self.popen.stdout
if POSIX:
while True:
i, _, _ = select.select([out], [], [])
if i:
return out.read(buffer_size)
else:
import ctypes
import msvcrt
kernel32 = ctypes.windll.kernel32
buffer_size = 1
bytes_read = bytearray()
#wait for some output synchronously, to not cause infinite loop
bytes_read.extend(out.read(buffer_size))
#read until end of current output
kernel32.SetNamedPipeHandleState(ctypes.c_void_p(msvcrt.get_osfhandle(out.fileno())), ctypes.byref(ctypes.c_int(1)), None, None)
#'Invalid Argument' means that there are no more bytes left to read
while True:
try:
cur_bytes_read=out.read(buffer_size)
if not cur_bytes_read:
break
bytes_read.extend(cur_bytes_read)
except (OSError, IOError):
break
kernel32.SetNamedPipeHandleState(ctypes.c_void_p(msvcrt.get_osfhandle(out.fileno())), ctypes.byref(ctypes.c_int(0)), None, None)
# f'in HACK, for \r\n -> \n translation on windows
# I tried universal_endlines but it was pain and misery! :'(
return bytes_read.replace(b'\r\n', b'\n')
def _mk_inheritable(fd):
if version >= (3, 3):
if sys.platform == 'win32':
# Change to Windwos file handle
import msvcrt
fdh = msvcrt.get_osfhandle(fd)
os.set_handle_inheritable(fdh, True)
return fdh
else:
os.set_inheritable(fd, True)
return fd
elif sys.platform == 'win32':
# TODO: find a hack??
# Not yet working
import msvcrt
import _subprocess
curproc = _subprocess.GetCurrentProcess()
fdh = msvcrt.get_osfhandle(fd)
fdh = _subprocess.DuplicateHandle(
curproc, fdh, curproc, 0,
True, # set inheritable FLAG
_subprocess.DUPLICATE_SAME_ACCESS)
return fdh
else:
return fd
def _create_tmp_files():
out_file = tempfile.TemporaryFile(mode="r+b")
err_file = tempfile.TemporaryFile(mode="r+b")
return (msvcrt.get_osfhandle(out_file.fileno()),
msvcrt.get_osfhandle(err_file.fileno()),
out_file,
err_file)
def __init__( self, *args, **kwargs ):
"""
Both output streams stdout and stderr can be set to any object
that has a write function. Leaving stdout or stderr out will
redirect both streams to their system equivalent (sys.stdout and
sys.stderr).
"""
self._bypass_stdout = False
self._bypass_stderr = False
if 'stdout' in kwargs and not kwargs['stdout'] is subprocess.PIPE:
self._stdout_file = kwargs['stdout']
kwargs['stdout'] = subprocess.PIPE
self._bypass_stdout = True
elif not 'stdout' in kwargs:
self._stdout_file = sys.stdout
kwargs['stdout'] = subprocess.PIPE
self._bypass_stdout = True
if 'stderr' in kwargs and not kwargs['stderr'] is subprocess.PIPE:
self._stderr_file = kwargs['stderr']
kwargs['stderr'] = subprocess.PIPE
self._bypass_stderr = True
elif not 'stderr' in kwargs:
self._stderr_file = sys.stderr
kwargs['stderr'] = subprocess.PIPE
self._bypass_stderr = True
subprocess.Popen.__init__( self, *args, **kwargs )
if self._bypass_stdout:
self._stdout_hdl = msvcrt.get_osfhandle( self.stdout.fileno() )
if self._bypass_stderr:
self._stderr_hdl = msvcrt.get_osfhandle( self.stderr.fileno() )
def _checkFileObjInheritable(cls, fileobj, handle_name):
"""Check if a given file-like object (or whatever else subprocess.Popen
takes as a handle/stream) can be correctly inherited by a child process.
This just duplicates the code in subprocess.Popen._get_handles to make
sure we go down the correct code path; this to catch some non-standard
corner cases."""
import _subprocess
import ctypes
import msvcrt
new_handle = None
try:
if fileobj is None:
handle = _subprocess.GetStdHandle(getattr(_subprocess,
handle_name))
if handle is None:
return True # No need to check things we create
elif fileobj == subprocess.PIPE:
return True # No need to check things we create
elif isinstance(fileobj, int):
handle = msvcrt.get_osfhandle(fileobj)
else:
# Assuming file-like object
handle = msvcrt.get_osfhandle(fileobj.fileno())
new_handle = self._make_inheritable(handle)
return True
except:
return False
finally:
CloseHandle = ctypes.windll.kernel32.CloseHandle
if new_handle is not None:
CloseHandle(new_handle)
def _create_tmp_files(env=None):
dirname = None
if env is None:
env = os.environ
for env_var_name in 'TMPDIR', 'TEMP', 'TMP':
if env.has_key(env_var_name):
dirname = env[env_var_name]
if dirname and os.path.exists(dirname):
break
if dirname is None:
for dirname2 in r'c:\temp', r'c:\tmp', r'\temp', r'\tmp':
try:
os.makedirs(dirname2)
dirname = dirname2
break
except:
pass
if dirname is None:
raise Exception('Unable to create temp dir. Insufficient access rights.')
out_file = tempfile.TemporaryFile(mode="r+b", dir=dirname)
err_file = tempfile.TemporaryFile(mode="r+b", dir=dirname)
return (msvcrt.get_osfhandle(out_file.fileno()),
msvcrt.get_osfhandle(err_file.fileno()),
out_file,
err_file)
def __init__ (self, executable, args, env, logfile=None):
self.logfile = logfile
self.executable = find_executable(executable)
self.args = args
self.exitstatus = None
self.running = 0
tempfile.template = None
if logfile:
fd = os.open(logfile.filename,
O_WRONLY | O_CREAT | O_APPEND, 0664)
# Get the NT handle for fd
hStdout = msvcrt.get_osfhandle(fd)
# Equivalent of os.dup() except for Win32 HANDLE's
hStderr = win32api.DuplicateHandle(hStdout)
else:
# Grab the HANDLE's for current stdout & stderr
hStdout = msvcrt.get_osfhandle(sys.stdout)
hStderr = msvcrt.get_osfhandle(sys.stderr)
# Grab the HANDLE for stdin
hStdin = msvcrt.get_osfhandle(sys.stdin)
for key in env.keys():
os.environ[key] = env[key]
if logfile:
output = fd
else:
output = sys.stderr
output.write("executable is " +
str(self.executable) +
", args are " +
str(args) + ", env = " +
str(os.environ) + "\n")
# Create the process
# All of this footwork, should allow this sucker to run from a console, or GUI app.
sCmdLine = self.executable
for arg in args:
sCmdLine = sCmdLine + " " + arg
StartupInfo = win32process.STARTUPINFO()
StartupInfo.dwFlags = win32con.STARTF_USESTDHANDLES | win32con.STARTF_USESHOWWINDOW
StartupInfo.hStdInput = hStdin
StartupInfo.hStdOutput = hStdout
StartupInfo.hStdError = hStderr
StartupInfo.wShowWindow = win32con.SW_HIDE
hProcess, hThread, dwPid, dwTid = win32api.CreateProcess(self.executable,
sCmdLine,
None,
None,
0,
win32process.CREATE_NEW_CONSOLE,
None,
None,
StartupInfo)
win32api.CloseHandle(hThread)
self.pid = dwPid
self.hProcess = hProcess
self.running = 1
add_exit_function(lambda x=self: x.kill())
def __init__(self, incoming, outgoing):
import msvcrt
self._keepalive = (incoming, outgoing)
if hasattr(incoming, "fileno"):
self._fileno = incoming.fileno()
incoming = msvcrt.get_osfhandle(incoming.fileno())
if hasattr(outgoing, "fileno"):
outgoing = msvcrt.get_osfhandle(outgoing.fileno())
self.incoming = incoming
self.outgoing = outgoing
def _isFileObjInheritable(cls, fileobj, stream_name):
"""Check if a given file-like object (or whatever else subprocess.Popen
takes as a handle/stream) can be correctly inherited by a child process.
This just duplicates the code in subprocess.Popen._get_handles to make
sure we go down the correct code path; this to catch some non-standard
corner cases.
@param fileobj The object being used as a fd/handle/whatever
@param stream_name The name of the stream, "stdin", "stdout", or "stderr"
"""
import _subprocess
import ctypes
import msvcrt
new_handle = None
# Things that we know how to reset (i.e. not custom fds)
valid_list = {
"stdin": (sys.stdin, sys.__stdin__, 0),
"stdout": (sys.stdout, sys.__stdout__, 1),
"stderr": (sys.stderr, sys.__stderr__, 2),
}[stream_name]
try:
if fileobj is None:
std_handle = {
"stdin": _subprocess.STD_INPUT_HANDLE,
"stdout": _subprocess.STD_OUTPUT_HANDLE,
"stderr": _subprocess.STD_ERROR_HANDLE,
}[stream_name]
handle = _subprocess.GetStdHandle(std_handle)
if handle is None:
# subprocess.Popen._get_handles creates a new pipe here
# we don't have to worry about things we create
return True
elif fileobj == subprocess.PIPE:
# We're creating a new pipe; newly created things are inheritable
return True
elif fileobj not in valid_list:
# We are trying to use a custom fd; don't do anything fancy here,
# we don't want to actually use subprocess.PIPE
return True
elif isinstance(fileobj, int):
handle = msvcrt.get_osfhandle(fileobj)
else:
# Assuming file-like object
handle = msvcrt.get_osfhandle(fileobj.fileno())
new_handle = self._make_inheritable(handle)
return True
except:
return False
finally:
CloseHandle = ctypes.windll.kernel32.CloseHandle
if new_handle is not None:
CloseHandle(new_handle)
def _get_handles(self, stdin, stdout, stderr):
if stdin is None and stdout is None and stderr is None:
return (-1, -1, -1, -1, -1, -1)
(p2cread, p2cwrite) = (-1, -1)
(c2pread, c2pwrite) = (-1, -1)
(errread, errwrite) = (-1, -1)
if stdin is None:
p2cread = _winapi.GetStdHandle(_winapi.STD_INPUT_HANDLE)
(p2cread, _) = _winapi.CreatePipe(None, 0)
p2cread = Handle(p2cread)
_winapi.CloseHandle(_)
elif stdin == PIPE:
(p2cread, p2cwrite) = _winapi.CreatePipe(None, 0)
(p2cread, p2cwrite) = (Handle(p2cread), Handle(p2cwrite))
elif stdin == DEVNULL:
p2cread = msvcrt.get_osfhandle(self._get_devnull())
elif isinstance(stdin, int):
p2cread = msvcrt.get_osfhandle(stdin)
else:
p2cread = msvcrt.get_osfhandle(stdin.fileno())
p2cread = self._make_inheritable(p2cread)
if stdout is None:
c2pwrite = _winapi.GetStdHandle(_winapi.STD_OUTPUT_HANDLE)
(_, c2pwrite) = _winapi.CreatePipe(None, 0)
c2pwrite = Handle(c2pwrite)
_winapi.CloseHandle(_)
elif stdout == PIPE:
(c2pread, c2pwrite) = _winapi.CreatePipe(None, 0)
(c2pread, c2pwrite) = (Handle(c2pread), Handle(c2pwrite))
elif stdout == DEVNULL:
c2pwrite = msvcrt.get_osfhandle(self._get_devnull())
elif isinstance(stdout, int):
c2pwrite = msvcrt.get_osfhandle(stdout)
else:
c2pwrite = msvcrt.get_osfhandle(stdout.fileno())
c2pwrite = self._make_inheritable(c2pwrite)
if stderr is None:
errwrite = _winapi.GetStdHandle(_winapi.STD_ERROR_HANDLE)
(_, errwrite) = _winapi.CreatePipe(None, 0)
errwrite = Handle(errwrite)
_winapi.CloseHandle(_)
elif stderr == PIPE:
(errread, errwrite) = _winapi.CreatePipe(None, 0)
(errread, errwrite) = (Handle(errread), Handle(errwrite))
elif stderr == STDOUT:
errwrite = c2pwrite
elif stderr == DEVNULL:
errwrite = msvcrt.get_osfhandle(self._get_devnull())
elif isinstance(stderr, int):
errwrite = msvcrt.get_osfhandle(stderr)
else:
errwrite = msvcrt.get_osfhandle(stderr.fileno())
errwrite = self._make_inheritable(errwrite)
return (p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite)
def _stdin_raw_nonblock(self):
"""Use the raw Win32 handle of sys.stdin to do non-blocking reads"""
# WARNING: This is experimental, and produces inconsistent results.
# It's possible for the handle not to be appropriate for use
# with WaitForSingleObject, among other things.
handle = msvcrt.get_osfhandle(sys.stdin.fileno())
result = WaitForSingleObject(handle, 100)
if result == WAIT_FAILED:
raise ctypes.WinError()
elif result == WAIT_TIMEOUT:
print(".", end='')
return None
else:
data = ctypes.create_string_buffer(256)
bytesRead = DWORD(0)
print('?', end='')
if not ReadFile(handle, data, 256,
ctypes.byref(bytesRead), None):
raise ctypes.WinError()
# This ensures the non-blocking works with an actual console
# Not checking the error, so the processing will still work with
# other handle types
FlushConsoleInputBuffer(handle)
data = data.value
data = data.replace('\r\n', '\n')
data = data.replace('\r', '\n')
print(repr(data) + " ", end='')
return data
def __init__(self, stream):
self.stream = stream or sys.stdout
self.isatty = getattr(self.stream, 'isatty', lambda : False)()
force_ansi = 'CALIBRE_FORCE_ANSI' in os.environ
if not self.isatty and force_ansi:
self.isatty = True
self.isansi = force_ansi or not iswindows
self.set_console = self.write_console = None
self.is_console = False
if not self.isansi:
try:
import msvcrt
self.msvcrt = msvcrt
self.file_handle = msvcrt.get_osfhandle(self.stream.fileno())
from ctypes import windll, wintypes, byref, POINTER, WinDLL
mode = wintypes.DWORD(0)
f = windll.kernel32.GetConsoleMode
f.argtypes, f.restype = [wintypes.HANDLE, POINTER(wintypes.DWORD)], wintypes.BOOL
if f(self.file_handle, byref(mode)):
# Stream is a console
self.set_console = windll.kernel32.SetConsoleTextAttribute
self.default_console_text_attributes = WCOLORS['white']
kernel32 = WinDLL(native_string_type('kernel32'), use_last_error=True)
self.write_console = kernel32.WriteConsoleW
self.write_console.argtypes = [wintypes.HANDLE, wintypes.c_wchar_p, wintypes.DWORD, POINTER(wintypes.DWORD), wintypes.LPVOID]
self.write_console.restype = wintypes.BOOL
kernel32.GetConsoleScreenBufferInfo.argtypes = [wintypes.HANDLE, ctypes.POINTER(CONSOLE_SCREEN_BUFFER_INFO)]
kernel32.GetConsoleScreenBufferInfo.restype = wintypes.BOOL
csbi = CONSOLE_SCREEN_BUFFER_INFO()
if kernel32.GetConsoleScreenBufferInfo(self.file_handle, byref(csbi)):
self.default_console_text_attributes = csbi.wAttributes
self.is_console = True
except:
pass
def __init__(self, process_obj):
# create pipe for communication with child
rfd, wfd = os.pipe()
# get handle for read end of the pipe and make it inheritable
rhandle = duplicate(msvcrt.get_osfhandle(rfd), inheritable=True)
os.close(rfd)
# start process
cmd = get_command_line() + [rhandle]
cmd = ' '.join('"%s"' % x for x in cmd)
hp, ht, pid, tid = _subprocess.CreateProcess(
_python_exe, cmd, None, None, 1, 0, None, None, None
)
ht.Close()
close(rhandle)
# set attributes of self
self.pid = pid
self.returncode = None
self._handle = hp
# send information to child
prep_data = get_preparation_data(process_obj._name)
to_child = os.fdopen(wfd, 'wb')
Popen._tls.process_handle = int(hp)
try:
dump(prep_data, to_child, HIGHEST_PROTOCOL)
dump(process_obj, to_child, HIGHEST_PROTOCOL)
finally:
del Popen._tls.process_handle
to_child.close()
def __DoOneRead(self):
"""Read one line of output and post results.
@return: bool (True if more), (False if not)
"""
if subprocess.mswindows:
# Windows nonblocking pipe read implementation
read = u''
try:
handle = msvcrt.get_osfhandle(self._proc.stdout.fileno())
avail = ctypes.c_long()
ctypes.windll.kernel32.PeekNamedPipe(handle, None, 0, 0,
ctypes.byref(avail), None)
if avail.value > 0:
read = self._proc.stdout.read(avail.value)
if read.endswith(os.linesep):
read = read[:-1 * len(os.linesep)]
else:
if self._proc.poll() is None:
time.sleep(1)
return True
else:
# Process has Exited
return False
except ValueError:
return False
except (subprocess.pywintypes.error, Exception), msg:
if msg[0] in (109, errno.ESHUTDOWN):
return False
def _get_named_pipe_from_fileno(fileno):
import msvcrt
from ctypes import WinError
result = msvcrt.get_osfhandle(fileno)
if result == INVALID_HANDLE_VALUE:
raise WinError(result)
return result
def __init__(self):
# Create a pipe so we can write to stdin of the loader process.
pipeReadOrig, self._pipeWrite = winKernel.CreatePipe(None, 0)
# Make the read end of the pipe inheritable.
pipeRead = self._duplicateAsInheritable(pipeReadOrig)
winKernel.closeHandle(pipeReadOrig)
# stdout/stderr of the loader process should go to nul.
with file("nul", "w") as nul:
nulHandle = self._duplicateAsInheritable(msvcrt.get_osfhandle(nul.fileno()))
# Set the process to start with the appropriate std* handles.
si = winKernel.STARTUPINFO(dwFlags=winKernel.STARTF_USESTDHANDLES, hSTDInput=pipeRead, hSTDOutput=nulHandle, hSTDError=nulHandle)
pi = winKernel.PROCESS_INFORMATION()
# Even if we have uiAccess privileges, they will not be inherited by default.
# Therefore, explicitly specify our own process token, which causes them to be inherited.
token = winKernel.OpenProcessToken(winKernel.GetCurrentProcess(), winKernel.MAXIMUM_ALLOWED)
try:
winKernel.CreateProcessAsUser(token, None, os.path.join(versionedLib64Path,u"nvdaHelperRemoteLoader.exe"), None, None, True, None, None, None, si, pi)
# We don't need the thread handle.
winKernel.closeHandle(pi.hThread)
self._process = pi.hProcess
except:
winKernel.closeHandle(self._pipeWrite)
raise
finally:
winKernel.closeHandle(pipeRead)
winKernel.closeHandle(token)
def _recv(self, which, maxsize):
conn, maxsize = self.get_conn_maxsize(which, maxsize)
if conn is None:
return None
try:
x = msvcrt.get_osfhandle(conn.fileno())
(read, nAvail, nMessage) = PeekNamedPipe(x, 0)
if maxsize < nAvail:
nAvail = maxsize
if nAvail > 0:
(errCode, read) = ReadFile(x, nAvail, None)
except ValueError:
return self._close(which)
except (subprocess.pywintypes.error, Exception) as why:
if why.args[0] in (109, errno.ESHUTDOWN):
return self._close(which)
raise
getattr(self, '{0}_buff'.format(which)).write(read)
getattr(self, '_{0}_logger'.format(which)).debug(read.rstrip())
if self.stream_stds:
getattr(sys, which).write(read)
if self.universal_newlines:
read = self._translate_newlines(read)
return read
def _connect_stdio(self):
if os.name == 'nt':
pipe = PipeHandle(msvcrt.get_osfhandle(sys.stdin.fileno()))
else:
pipe = sys.stdin
coroutine = self._loop.connect_read_pipe(self._fact, pipe)
self._loop.run_until_complete(coroutine)
debug("native stdin connection successful")
if os.name == 'nt':
pipe = PipeHandle(msvcrt.get_osfhandle(sys.stdout.fileno()))
else:
pipe = sys.stdout
coroutine = self._loop.connect_write_pipe(self._fact, pipe)
self._loop.run_until_complete(coroutine)
debug("native stdout connection successful")
def __init__(self, process_obj):
cmd = ' '.join('"%s"' % x for x in get_command_line())
prep_data = get_preparation_data(process_obj._name)
# create pipe for communication with child
rfd, wfd = os.pipe()
# get handle for read end of the pipe and make it inheritable
rhandle = duplicate(msvcrt.get_osfhandle(rfd), inheritable=True)
os.close(rfd)
with open(wfd, 'wb', closefd=True) as to_child:
# start process
try:
hp, ht, pid, tid = _winapi.CreateProcess(
_python_exe, cmd + (' %s' % rhandle),
None, None, 1, 0, None, None, None
)
_winapi.CloseHandle(ht)
finally:
close(rhandle)
# set attributes of self
self.pid = pid
self.returncode = None
self._handle = hp
self.sentinel = int(hp)
# send information to child
Popen._tls.process_handle = int(hp)
try:
dump(prep_data, to_child, HIGHEST_PROTOCOL)
dump(process_obj, to_child, HIGHEST_PROTOCOL)
finally:
del Popen._tls.process_handle
def _crashHandler(exceptionInfo):
threadId = ctypes.windll.kernel32.GetCurrentThreadId()
# An exception might have been set for this thread.
# Clear it so that it doesn't get raised in this function.
ctypes.pythonapi.PyThreadState_SetAsyncExc(threadId, None)
# Write a minidump.
dumpPath = os.path.abspath(os.path.join(globalVars.appArgs.logFileName, "..", "nvda_crash.dmp"))
try:
with file(dumpPath, "w") as mdf:
mdExc = MINIDUMP_EXCEPTION_INFORMATION(ThreadId=threadId,
ExceptionPointers=exceptionInfo, ClientPointers=False)
if not ctypes.windll.DbgHelp.MiniDumpWriteDump(
ctypes.windll.kernel32.GetCurrentProcess(),
os.getpid(),
msvcrt.get_osfhandle(mdf.fileno()),
0, # MiniDumpNormal
ctypes.byref(mdExc),
None,
None
):
raise ctypes.WinError()
except:
log.critical("NVDA crashed! Error writing minidump", exc_info=True)
else:
log.critical("NVDA crashed! Minidump written to %s" % dumpPath)
log.info("Restarting due to crash")
core.restart()
return 1 # EXCEPTION_EXECUTE_HANDLER
def recv_multi_impl(conns, maxsize, timeout):
"""Reads from the first available pipe.
If timeout is None, it's blocking. If timeout is 0, it is not blocking.
"""
# TODO(maruel): Use WaitForMultipleObjects(). Python creates anonymous pipes
# for proc.stdout and proc.stderr but they are implemented as named pipes on
# Windows. Since named pipes are not waitable object, they can't be passed
# as-is to WFMO(). So this means N times CreateEvent(), N times ReadFile()
# and finally WFMO(). This requires caching the events handles in the Popen
# object and remembering the pending ReadFile() calls. This will require
# some re-architecture to store the relevant event handle and OVERLAPPEDIO
# object in Popen or the file object.
maxsize = max(maxsize or 16384, 1)
if timeout:
start = time.time()
handles = [
(i, msvcrt.get_osfhandle(c.fileno())) for i, c in enumerate(conns)
]
while handles:
for index, handle in handles:
try:
avail = min(PeekNamedPipe(handle), maxsize)
if avail:
return index, ReadFile(handle, avail)[1]
if (timeout and (time.time() - start) >= timeout) or timeout == 0:
return None, None
# Polling rocks.
time.sleep(0.001)
except OSError:
handles.remove((index, handle))
break
# Nothing to wait for.
return None, None
def __call__(self, code=""):
tmpFile = os.tmpfile()
tmpFile.write(code)
tmpFile.seek(0)
self.plugin.msgThread.dll.MceIrPlaybackFromFile(
get_osfhandle(tmpFile.fileno())
)
def _start_remote_console(self):
"""
Starts remote console support for this VM.
"""
# starts the Telnet to pipe thread
pipe_name = self._get_pipe_name()
if sys.platform.startswith("win"):
try:
self._serial_pipe = open(pipe_name, "a+b")
except OSError as e:
raise VirtualBoxError("Could not open the pipe {}: {}".format(pipe_name, e))
try:
self._telnet_server_thread = TelnetServer(self._vmname, msvcrt.get_osfhandle(self._serial_pipe.fileno()), self._manager.port_manager.console_host, self._console)
except OSError as e:
raise VirtualBoxError("Unable to create Telnet server: {}".format(e))
self._telnet_server_thread.start()
else:
try:
self._serial_pipe = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
self._serial_pipe.connect(pipe_name)
except OSError as e:
raise VirtualBoxError("Could not connect to the pipe {}: {}".format(pipe_name, e))
try:
self._telnet_server_thread = TelnetServer(self._vmname, self._serial_pipe, self._manager.port_manager.console_host, self._console)
except OSError as e:
raise VirtualBoxError("Unable to create Telnet server: {}".format(e))
self._telnet_server_thread.start()
def unlock(file):
hfile = msvcrt.get_osfhandle(file.fileno())
overlapped = OVERLAPPED()
if UnlockFileEx(hfile, 0, 0, 0xFFFF0000, ctypes.byref(overlapped)):
return True
else:
return False
def set_inheritable(fd, inheritable):
# This implementation of set_inheritable is based on a code sample in
# [PEP 0446](https://www.python.org/dev/peps/pep-0446/) and on the
# CPython implementation of that proposal which can be browsed [here]
# (hg.python.org/releasing/3.4/file/8671f89107c8/Modules/posixmodule.c#l11130)
if sys.platform == "win32":
import msvcrt
import ctypes.windll.kernel32 as kernel32
HANDLE_FLAG_INHERIT = 1
if (
kernel32.SetHandleInformation(msvcrt.get_osfhandle(fd), HANDLE_FLAG_INHERIT, 1 if inheritable else 0)
== 0
):
raise IOError("Failed on HANDLE_FLAG_INHERIT")
else:
import fcntl
fd_flags = fcntl.fcntl(fd, fcntl.F_GETFD)
if inheritable:
fd_flags &= ~fcntl.FD_CLOEXEC
else:
fd_flags |= fcntl.FD_CLOEXEC
fcntl.fcntl(fd, fcntl.F_SETFD, fd_flags)
def run(self):
start = time.time()
p = subprocess.Popen(self.cmd, shell=True, bufsize=0,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
outputHandle = msvcrt.get_osfhandle(p.stdout.fileno())
while p.poll() is None and not self.exit:
try:
avail = c_ulong(0)
ctypes.windll.kernel32.PeekNamedPipe(outputHandle, None, 0, None, \
ctypes.byref(avail), None)
#~ print "%%", avail.value
if avail.value:
self.send(p.stdout.read(avail.value))
time.sleep(0.0)
else:
# Nothing happening so sleep on this thread so UI is responsive
time.sleep(0.01)
except:
# Print out here otherwise gets munged inside trasfer to C++
traceback.print_exc()
raise
data = p.stdout.read()
#~ print "End of data", len(data), data
self.send(data)
if p.returncode is not None:
end = time.time()
self.send("> Exit code %d %f\n" % (p.returncode, (end-start)))
def __init__(self, process_obj):
# create pipe for communication with child
r, w = os.pipe()
# get handle for read end of the pipe and make it inheritable
rhandle = msvcrt.get_osfhandle(r)
win32.SetHandleInformation(rhandle, win32.HANDLE_FLAG_INHERIT, win32.HANDLE_FLAG_INHERIT)
# start process
cmd = getCommandLine() + [rhandle]
cmd = " ".join('"%s"' % x for x in cmd)
hp, ht, pid, tid = _subprocess.CreateProcess(sys.executable, cmd, None, None, 1, 0, None, None, None)
os.close(r)
ht.Close()
# set attributes of self
self.pid = pid
self.returncode = None
self._handle = hp
# send information to child
prep_data = getPreparationData(process_obj._name)
to_child = os.fdopen(w, "wb")
tls.is_spawning = True
try:
dump(prep_data, to_child, HIGHEST_PROTOCOL)
dump(process_obj, to_child, HIGHEST_PROTOCOL)
finally:
tls.is_spawning = False
to_child.close()
def readWithTimeout(self, f, timeout):
"""
Try to read a line of output from the file object |f|. |f| must be a
pipe, like the |stdout| member of a subprocess.Popen object created
with stdout=PIPE. Returns a tuple (line, did_timeout), where |did_timeout|
is True if the read timed out, and False otherwise. If no output is
received within |timeout| seconds, returns a blank line.
"""
if timeout is None:
timeout = 0
x = msvcrt.get_osfhandle(f.fileno())
l = ctypes.c_long()
done = time.time() + timeout
buffer = ""
while timeout == 0 or time.time() < done:
if self.PeekNamedPipe(x, None, 0, None, ctypes.byref(l), None) == 0:
err = self.GetLastError()
if err == 38 or err == 109: # ERROR_HANDLE_EOF || ERROR_BROKEN_PIPE
return ("", False)
else:
self.log.error("readWithTimeout got error: %d", err)
# read a character at a time, checking for eol. Return once we get there.
index = 0
while index < l.value:
char = f.read(1)
buffer += char
if char == "\n":
return (buffer, False)
index = index + 1
time.sleep(0.01)
return (buffer, True)
def _check_ready(self, conn, maxsize=1024):
if maxsize < 1:
maxsize = 1
if conn is None:
return
try:
x = msvcrt.get_osfhandle(conn.fileno())
(read, nAvail, nMessage) = PeekNamedPipe(x, 0)
if maxsize < nAvail:
nAvail = maxsize
if nAvail > 0:
if conn is self.stdout:
self.emit('stdout-ready')
elif conn is self.stderr:
self.emit('stderr-ready')
except ValueError:
return conn.close()
except (WindowsError, Exception) as ex:
if ex[0] in (109, errno.ESHUTDOWN):
return conn.close()
raise
return True
def readpipeNONBLK(self, cmdpipe):
#
# Don't block waiting for output to appear, just grab
# whatever is available now on the pipe. fstat on
# bsd unix pipes gives the available-to-read count.
# But doesn't work on linux, or on win32 CreatePipe
# anonymous pipes (purportedly works on win32 posix pipes).
#
bytes = ""
try:
# anything available to read right now?
nAvail = 0
x = cmdpipe.fileno()
if subprocess.mswindows:
h = msvcrt.get_osfhandle(x)
dword = ctypes.c_ulong()
rc = ctypes.windll.kernel32.PeekNamedPipe(
h, None, 0, None, ctypes.byref(dword), None
)
if rc:
nAvail = int(dword.value)
else:
#unix
# unfortunately, the os.fstat trick doesn't work on linux
# nAvail = os.fstat(x).st_size
# whereas the select "poll" seems to work on all *nix
if select.select([cmdpipe], [], [], 0)[0]:
nAvail = 262144
if nAvail > 0:
bytes = os.read(x,nAvail)
except Exception:
pass
return bytes
def __init__(self, stream=None):
super(WindowsColorStreamHandler, self).__init__(stream)
# get file handle for the stream
import msvcrt
self._outhdl = msvcrt.get_osfhandle(self.stream.fileno())
def __init__(self, fd):
self._fd = fd
self._file = None
if MS_WINDOWS:
self._handle = msvcrt.get_osfhandle(fd)
def execute(self, source):
"""
Get the source code to execute (a unicode string).
Compile it. If there was a syntax error, return
(False, (msg, line, col)).
If compilation was successful, return (True, None), then run the code
and then send (is_success, res_no, res_str, exception_string, rem_stdin).
is_success - True if there was no exception.
res_no - number of the result in the history count, or None if there
was no result or there's no history.
res_str - a string representation of the result.
exception_string - description of the exception, or None if is_success.
rem_stdin - data that was sent into stdin and wasn't consumed.
"""
# pause_idle was called before execute, disable it.
self.idle_paused = False
if ast:
success, r = self.compile_ast(source)
else:
success, r = self.compile_no_ast(source)
if not success:
yield False, r
return
else:
yield True, None
codeobs = r
self.last_res = None
try:
unmask_sigint()
try:
# Execute
for codeob in codeobs:
exec codeob in self.locs
# Work around http://bugs.python.org/issue8213 - stdout buffered
# in Python 3.
if not sys.stdout.closed:
sys.stdout.flush()
if not sys.stderr.closed:
sys.stderr.flush()
# Convert the result to a string. This is here because exceptions
# may be raised here.
if self.last_res is not None:
if self.is_pprint:
res_str = self.safe_pformat(self.last_res)
else:
res_str = unicode(repr(self.last_res))
if len(res_str) > MAX_RES_STR_LEN:
res_str = (res_str[:MAX_RES_STR_LEN] +
'\n[%d chars truncated]' %
(len(res_str) - MAX_RES_STR_LEN))
else:
res_str = None
finally:
mask_sigint()
except:
if not sys.stdout.closed:
sys.stdout.flush()
excinfo = sys.exc_info()
sys.last_type, sys.last_value, sys.last_traceback = excinfo
exception_string = trunc_traceback(excinfo, __file__)
is_success = False
res_no = None
res_str = None
else:
is_success = True
exception_string = None
if self.last_res is not None:
res_no = self.store_in_reshist(self.last_res)
else:
res_no = None
# Discard the reference to the result
self.last_res = None
# Send back any data left on stdin.
rem_stdin = []
if sys.platform == 'linux2':
while select([sys.stdin], [], [], 0)[0]:
r = os.read(sys.stdin.fileno(), 8192)
if not r:
# File may be in error state
break
rem_stdin.append(unicodify(r))
elif sys.platform == 'win32':
fd = sys.stdin.fileno()
handle = get_osfhandle(fd)
avail = c_ulong(0)
PeekNamedPipe(handle, None, 0, None, byref(avail), None)
nAvail = avail.value
if nAvail > 0:
rem_stdin.append(os.read(fd, nAvail))
else:
# I don't know how to do this in Jython.
pass
rem_stdin = u''.join(rem_stdin)
# Check if matplotlib in non-interactive mode was imported
self.check_matplotlib_ia()
yield is_success, res_no, res_str, exception_string, rem_stdin
def assert_not_inheritable(f):
if win32api.GetHandleInformation(msvcrt.get_osfhandle(
f.fileno())) & 0b1:
raise SystemExit('File handle is inheritable!')
def from_fd(cls, fd):
return cls(get_osfhandle(fd), doclose=False)
def _get_handles(self, stdin, stdout, stderr):
"""Construct and return tupel with IO objects:
p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
"""
if stdin is None and stdout is None and stderr is None:
return (None, None, None, None, None, None)
p2cread, p2cwrite = None, None
c2pread, c2pwrite = None, None
errread, errwrite = None, None
if stdin is None:
p2cread = GetStdHandle(STD_INPUT_HANDLE)
if p2cread is not None:
pass
elif stdin is None or stdin == PIPE:
p2cread, p2cwrite = CreatePipe(None, 0)
# Detach and turn into fd
p2cwrite = p2cwrite.Detach()
p2cwrite = msvcrt.open_osfhandle(p2cwrite, 0)
elif isinstance(stdin, int):
p2cread = msvcrt.get_osfhandle(stdin)
else:
# Assuming file-like object
p2cread = msvcrt.get_osfhandle(stdin.fileno())
p2cread = self._make_inheritable(p2cread)
if stdout is None:
c2pwrite = GetStdHandle(STD_OUTPUT_HANDLE)
if c2pwrite is not None:
pass
elif stdout is None or stdout == PIPE:
c2pread, c2pwrite = CreatePipe(None, 0)
# Detach and turn into fd
c2pread = c2pread.Detach()
c2pread = msvcrt.open_osfhandle(c2pread, 0)
elif isinstance(stdout, int):
c2pwrite = msvcrt.get_osfhandle(stdout)
else:
# Assuming file-like object
c2pwrite = msvcrt.get_osfhandle(stdout.fileno())
c2pwrite = self._make_inheritable(c2pwrite)
if stderr is None:
errwrite = GetStdHandle(STD_ERROR_HANDLE)
if errwrite is not None:
pass
elif stderr is None or stderr == PIPE:
errread, errwrite = CreatePipe(None, 0)
# Detach and turn into fd
errread = errread.Detach()
errread = msvcrt.open_osfhandle(errread, 0)
elif stderr == STDOUT:
errwrite = c2pwrite
elif isinstance(stderr, int):
errwrite = msvcrt.get_osfhandle(stderr)
else:
# Assuming file-like object
errwrite = msvcrt.get_osfhandle(stderr.fileno())
errwrite = self._make_inheritable(errwrite)
return (p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite)
def __init__(self, *args, **kw):
_builtin_file.__init__(self, *args, **kw)
win32api.SetHandleInformation(msvcrt.get_osfhandle(self.fileno()),
win32con.HANDLE_FLAG_INHERIT, 0)
def __init__(self, fd):
assert isatty(
fd
), "file descriptor must refer to a console (note that on Windows, NUL satisfies isatty(), but is not a console)"
self.fd = fd
self.handle = msvcrt.get_osfhandle(fd)
def get_handle_from_file(f):
"""Get the ``Windows`` HANDLE of a python :class:`file`"""
return msvcrt.get_osfhandle(f.fileno())
def _unlock_file(file_):
_WinAPI_UnlockFile(msvcrt.get_osfhandle(file_.fileno()), 0, 0, 1,
0)
def unlock(file):
hfile = msvcrt.get_osfhandle(file.fileno())
overlapped = OVERLAPPED()
UnlockFileEx(hfile, 0, 0, 0xFFFF0000, ctypes.byref(overlapped))
def _scons_open(*args, **kw):
fp = _builtin_open(*args, **kw)
win32api.SetHandleInformation(msvcrt.get_osfhandle(fp.fileno()),
win32con.HANDLE_FLAG_INHERIT, 0)
return fp
def run(challenges, timeout, seed, logfunc):
""" Challenge launcher for replay services
This will setup fds for all challenges according to:
https://github.com/CyberGrandChallenge/cgc-release-documentation/blob/master/newsletter/ipc.md
Args:
challenges (list): List of absolute paths to all challenges to launch
timeout (int): Maximum time in seconds a challenge is allowed to run for
seed (str): Hex encoded seed for libcgc random
logfunc ((str) -> None): Replayer log function used for reporting results
Returns:
(list): all processes that were started
"""
cb_env = {'seed': seed} # Environment variables for all challenges
# This is the first fd after all of the challenges
last_fd = 2 * len(challenges) + 3
# Create all challenge fds
if len(challenges) > 1:
# Close fds where the pipes will be placed
os.closerange(3, last_fd)
new_fd = 3 # stderr + 1
for i in xrange(len(challenges)):
# Create a pipe for every running binary
rpipe, wpipe = os.pipe()
# The write end of the pipe needs to be at the lower fd, so it *may* get dup'd over the read end
# Preemptively dup the read fd here to avoid the issue
rpipe_tmp = os.dup(rpipe)
pipe_fds = [wpipe, rpipe_tmp]
# Duplicate the pipe ends to the correct fds if needed
for fd in pipe_fds:
if fd != new_fd:
os.dup2(fd, new_fd)
new_fd += 1
# Done with the temporary dup
os.close(rpipe_tmp)
# None of the above file descriptors will actually be inherited on Windows
# Prepare the environment so libcgc can regenerate this setup
# with the inherited HANDLEs
if IS_WINDOWS:
import msvcrt
# Store the number of pipes that need to be set up
numpipes = len(challenges) * 2 # Pipe pair for each
cb_env['PIPE_COUNT'] = str(numpipes)
# Store the HANDLE for each of the pipes
for i in xrange(len(challenges) * 2):
cb_env['PIPE_{}'.format(i)] = str(msvcrt.get_osfhandle(3 + i)) # First pipe is at 3
# Start all challenges
# Launch the main binary first
mainchal, otherchals = challenges[0], challenges[1:]
procs = [sp.Popen(mainchal, env=cb_env, stdin=sp.PIPE,
stdout=sp.PIPE, stderr=sp.PIPE)]
# Any others should be launched with the same std i/o pipes
# as the main binary
if len(otherchals) > 0:
main = procs[0]
procs += [sp.Popen(c, env=cb_env, stdin=main.stdin,
stdout=main.stdout, stderr=main.stderr) for c in otherchals]
# Start a watcher to report results when the challenges exit
watcher = threading.Thread(target=chal_watcher, args=(challenges, procs, timeout, logfunc))
watcher.setDaemon(True)
watcher.start()
return procs, watcher
def _unlock_file(f):
assert f._lock_file_overlapped_p
handle = msvcrt.get_osfhandle(f.fileno())
if not UnlockFileEx(handle, 0, whole_low, whole_high,
f._lock_file_overlapped_p):
raise OSError('Unlocking file failed: %r' % ctypes.FormatError())
def remove_reader(self, fd):
" Stop watching the file descriptor for read availability. "
h = msvcrt.get_osfhandle(fd)
self.remove_win32_handle(h)
def unlock(f):
hfile = msvcrt.get_osfhandle(_fd(f))
overlapped = OVERLAPPED()
ret = UnlockFileEx(hfile, 0, 0, 0xFFFF0000, byref(overlapped))
return bool(ret)
def _winapi_childhandle_prepare_transfer(self):
"""Prepare file descriptor for transfer to child process on Windows.
What follows now is an overview for the process of transferring a
Windows pipe handle to a child process, for different Python versions
(explanation / background can be found below):
Python versions < 3.4:
1) In the parent, get WinAPI handle from C file descriptor via
msvcrt.get_osfhandle().
2) WinAPI call DuplicateHandle(... ,bInheritHandle=True) in
parent. Close old handle, let inheritable duplicate live on.
2.5) multiprocessing internals invoke WinAPI call CreateProcess(...,
InheritHandles=True).
3) Close the duplicate in the parent.
4) Use msvcrt.open_osfhandle() in child for converting the Windows
pipe handle to a C file descriptor, and for setting the
(read/write)-only access flag. This file descriptor will be
used by user code.
Python versions >= 3.4:
1) Same as above.
2) Store handle and process ID. Both are integers that will be
pickled to and used by the child.
2.5) multiprocessing internals invoke WinAPI call
CreateProcess(..., InheritHandles=False).
3) Steal the Windows pipe handle from the parent process: in the
child use the parent's process ID for getting a WinAPI handle
to the parent process via WinAPI call OpenProcess(). Use option
PROCESS_DUP_HANDLE as desired access right. Invoke
DuplicateHandle() in the child and use the handle to the parent
process as source process handle. Use the
DUPLICATE_CLOSE_SOURCE and the DUPLICATE_SAME_ACCESS flags. The
result is a Windows pipe handle, "stolen" from the parent.
4) Same as above.
Background:
By default, file descriptors are not inherited by child processes on
Windows. However, they can be made inheritable via calling the system
function `DuplicateHandle` while setting `bInheritHandle` to True.
From MSDN:
bInheritHandle:
A variable that indicates whether the handle is inheritable.
If TRUE, the duplicate handle can be inherited by new processes
created by the target process. If FALSE, the new handle cannot
be inherited.
The internals of Python's `subprocess` and `multiprocessing` make use of
this. There is, however, no officially exposed Python API. Nevertheless,
the function `multiprocessing.forking.duplicate` (in Python versions
smaller than 3.4) and `multiprocessing.reduction.duplicate` (>= 3.4)
seems to be safely usable. In all versions, `duplicate` is part of
`multiprocessing.reduction`. As of 2015-07-20, the reduction module is
part of multiprocessing in all Python versions from 2.6 to 3.5.
The just outlined approach (DuplicateHandle() in parent, automatically
inherit it in child) only works for Python versions smaller than 3.4:
from Python 3.4 on, the child process is created with CreateProcess()'s
`InheritHandles` attribute set to False (this was explicitly set to True
in older Python versions). A different method needs to be used, referred
to as "stealing" the handle: DuplicateHandle can be called in the child
for retrieving the handle from the parent while using the
_winapi.DUPLICATE_CLOSE_SOURCE flag, which automatically closes the
handle in the parent. This method is used by
`multiprocessing.popen_spawn_win32` and implemented in
`multiprocessing.reduction.steal_handle`.
Refs:
https://msdn.microsoft.com/en-us/library/windows/desktop/ms684880.aspx
https://msdn.microsoft.com/en-us/library/windows/desktop/ms684320.aspx
https://msdn.microsoft.com/en-us/library/ks2530z6.aspx
https://msdn.microsoft.com/en-us/library/bdts1c9x.aspx
"""
if WINAPI_HANDLE_TRANSFER_STEAL:
self._parent_winapihandle = msvcrt.get_osfhandle(self._fd)
self._parent_pid = os.getpid()
return
# Get Windows file handle from C file descriptor.
winapihandle = msvcrt.get_osfhandle(self._fd)
# Duplicate file handle, rendering the duplicate inheritable by
# processes created by the current process.
self._inheritable_winapihandle = multiprocessing.reduction.duplicate(
handle=winapihandle, inheritable=True)
# Close "old" (in-inheritable) file descriptor.
os.close(self._fd)
# Mark file descriptor as "already closed".
self._fd = None
def file_handle(f):
return msvcrt.get_osfhandle(f.fileno())
def add_reader(self, fd, callback):
" Start watching the file descriptor for read availability. "
h = msvcrt.get_osfhandle(fd)
self._read_fds[h] = callback
def remove_reader(self, fd):
" Stop watching the file descriptor for read availability. "
h = msvcrt.get_osfhandle(fd)
if h in self._read_fds:
del self._read_fds[h]
def acquire_lock(cls, mode):
def condition():
if mode == "r":
return not cls._writer
else:
return not cls._writer and cls._readers == 0
if mode not in ("r", "w"):
raise ValueError("Invalid lock mode: %s" % mode)
# Use a primitive lock which only works within one process as a
# precondition for inter-process file-based locking
with cls._lock:
if cls._waiters or not condition():
# use FIFO for access requests
waiter = threading.Condition(lock=cls._lock)
cls._waiters.append(waiter)
while True:
waiter.wait()
if condition():
break
cls._waiters.pop(0)
if mode == "r":
cls._readers += 1
# notify additional potential readers
if cls._waiters:
cls._waiters[0].notify()
else:
cls._writer = True
if not cls._lock_file:
folder = os.path.expanduser(
cls.configuration.get("storage", "filesystem_folder"))
if not os.path.exists(folder):
os.makedirs(folder, exist_ok=True)
lock_path = os.path.join(folder, ".Radicale.lock")
cls._lock_file = open(lock_path, "w+")
# set access rights to a necessary minimum to prevent locking
# by arbitrary users
try:
os.chmod(lock_path, stat.S_IWUSR | stat.S_IRUSR)
except OSError:
cls.logger.debug("Failed to set permissions on lock file")
if not cls._lock_file_locked:
if os.name == "nt":
handle = msvcrt.get_osfhandle(cls._lock_file.fileno())
flags = LOCKFILE_EXCLUSIVE_LOCK if mode == "w" else 0
overlapped = Overlapped()
if not lock_file_ex(handle, flags, 0, 1, 0, overlapped):
cls.logger.debug("Locking not supported")
elif os.name == "posix":
_cmd = fcntl.LOCK_EX if mode == "w" else fcntl.LOCK_SH
try:
fcntl.lockf(cls._lock_file.fileno(), _cmd)
except OSError:
cls.logger.debug("Locking not supported")
cls._lock_file_locked = True
try:
yield
finally:
with cls._lock:
if mode == "r":
cls._readers -= 1
else:
cls._writer = False
if cls._readers == 0:
if os.name == "nt":
handle = msvcrt.get_osfhandle(cls._lock_file.fileno())
overlapped = Overlapped()
if not unlock_file_ex(handle, 0, 1, 0, overlapped):
cls.logger.debug("Unlocking not supported")
elif os.name == "posix":
try:
fcntl.lockf(cls._lock_file.fileno(), fcntl.LOCK_UN)
except OSError:
cls.logger.debug("Unlocking not supported")
cls._lock_file_locked = False
if cls._waiters:
cls._waiters[0].notify()
def session(file_names, codes=None):
"""Create a new iverilog session by compile the file.
Parameters
----------
file_names : str or list of str
The name of the file
codes : str or list of str
The code in str.
Returns
-------
sess : VPISession
The created session.
"""
if isinstance(file_names, string_types):
file_names = [file_names]
path = util.tempdir()
if codes:
if isinstance(codes, (list, tuple)):
codes = '\n'.join(codes)
fcode = path.relpath("temp_code.v")
with open(fcode, "w") as out_file:
out_file.write(codes)
file_names.append(fcode)
for name in file_names:
if not os.path.exists(name):
raise ValueError("Cannot find file %s" % name)
target = path.relpath(os.path.basename(file_names[0].rsplit(".", 1)[0]))
compile_file(file_names, target)
vpi_path = _find_vpi_path()
cmd = ["vvp"]
cmd += ["-M", vpi_path]
cmd += ["-m", "tvm_vpi"]
cmd += [target]
env = os.environ.copy()
read_device, write_host = os.pipe()
read_host, write_device = os.pipe()
if sys.platform == "win32":
import msvcrt
env['TVM_DREAD_PIPE'] = str(msvcrt.get_osfhandle(read_device))
env['TVM_DWRITE_PIPE'] = str(msvcrt.get_osfhandle(write_device))
read_host = msvcrt.get_osfhandle(read_host)
write_host = msvcrt.get_osfhandle(write_host)
else:
env['TVM_DREAD_PIPE'] = str(read_device)
env['TVM_DWRITE_PIPE'] = str(write_device)
env['TVM_HREAD_PIPE'] = str(read_host)
env['TVM_HWRITE_PIPE'] = str(write_host)
try:
# close_fds does not work well for all python3
# Use pass_fds instead.
# pylint: disable=unexpected-keyword-arg
pass_fds = (read_device, write_device, read_host, write_host)
proc = subprocess.Popen(cmd, pass_fds=pass_fds, env=env)
except TypeError:
# This is effective for python2
proc = subprocess.Popen(cmd, close_fds=False, env=env)
# close device side pipe
os.close(read_device)
os.close(write_device)
sess = _api_internal._vpi_SessMake(read_host, write_host)
sess.proc = proc
sess.execpath = path
return sess
def add_reader(self, fd, callback):
" Start watching the file descriptor for read availability. "
callback = wrap_in_current_context(callback)
h = msvcrt.get_osfhandle(fd)
self.add_win32_handle(h, callback)
def _get_handles(self, stdin, stdout, stderr):
"""Construct and return tuple with IO objects:
p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
"""
if stdin is None and stdout is None and stderr is None:
return (-1, -1, -1, -1, -1, -1)
p2cread, p2cwrite = -1, -1
c2pread, c2pwrite = -1, -1
errread, errwrite = -1, -1
if stdin is None:
p2cread = _winapi.GetStdHandle(_winapi.STD_INPUT_HANDLE)
if p2cread is None:
p2cread, _ = _winapi.CreatePipe(None, 0)
p2cread = Handle(p2cread)
_winapi.CloseHandle(_)
elif stdin == PIPE:
p2cread, p2cwrite = _winapi.CreatePipe(None, 0)
p2cread, p2cwrite = Handle(p2cread), Handle(p2cwrite)
elif stdin == DEVNULL:
p2cread = msvcrt.get_osfhandle(self._get_devnull())
elif isinstance(stdin, int):
p2cread = msvcrt.get_osfhandle(stdin)
else:
# Assuming file-like object
p2cread = msvcrt.get_osfhandle(stdin.fileno())
p2cread = self._make_inheritable(p2cread)
if stdout is None:
c2pwrite = _winapi.GetStdHandle(_winapi.STD_OUTPUT_HANDLE)
if c2pwrite is None:
_, c2pwrite = _winapi.CreatePipe(None, 0)
c2pwrite = Handle(c2pwrite)
_winapi.CloseHandle(_)
elif stdout == PIPE:
c2pread, c2pwrite = _winapi.CreatePipe(None, 0)
c2pread, c2pwrite = Handle(c2pread), Handle(c2pwrite)
elif stdout == DEVNULL:
c2pwrite = msvcrt.get_osfhandle(self._get_devnull())
elif isinstance(stdout, int):
c2pwrite = msvcrt.get_osfhandle(stdout)
else:
# Assuming file-like object
c2pwrite = msvcrt.get_osfhandle(stdout.fileno())
c2pwrite = self._make_inheritable(c2pwrite)
if stderr is None:
errwrite = _winapi.GetStdHandle(_winapi.STD_ERROR_HANDLE)
if errwrite is None:
_, errwrite = _winapi.CreatePipe(None, 0)
errwrite = Handle(errwrite)
_winapi.CloseHandle(_)
elif stderr == PIPE:
errread, errwrite = _winapi.CreatePipe(None, 0)
errread, errwrite = Handle(errread), Handle(errwrite)
elif stderr == STDOUT:
errwrite = c2pwrite
elif stderr == DEVNULL:
errwrite = msvcrt.get_osfhandle(self._get_devnull())
elif isinstance(stderr, int):
errwrite = msvcrt.get_osfhandle(stderr)
else:
# Assuming file-like object
errwrite = msvcrt.get_osfhandle(stderr.fileno())
errwrite = self._make_inheritable(errwrite)
return (p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite)
def ensure_running(self):
'''Make sure that resource tracker process is running.
This can be run from any process. Usually a child process will use
the resource created by its parent.'''
with self._lock:
if self._fd is not None:
# resource tracker was launched before, is it still running?
if self._check_alive():
# => still alive
return
# => dead, launch it again
os.close(self._fd)
if os.name == "posix":
try:
# At this point, the resource_tracker process has been
# killed or crashed. Let's remove the process entry
# from the process table to avoid zombie processes.
os.waitpid(self._pid, 0)
except OSError:
# The process was terminated or is a child from an
# ancestor of the current process.
pass
self._fd = None
self._pid = None
warnings.warn('resource_tracker: process died unexpectedly, '
'relaunching. Some folders/sempahores might '
'leak.')
fds_to_pass = []
try:
fds_to_pass.append(sys.stderr.fileno())
except Exception:
pass
r, w = os.pipe()
if sys.platform == "win32":
_r = duplicate(msvcrt.get_osfhandle(r), inheritable=True)
os.close(r)
r = _r
cmd = 'from {} import main; main({}, {})'.format(
main.__module__, r, VERBOSE)
try:
fds_to_pass.append(r)
# process will out live us, so no need to wait on pid
exe = spawn.get_executable()
args = [exe] + util._args_from_interpreter_flags()
# In python 3.3, there is a bug which put `-RRRRR..` instead of
# `-R` in args. Replace it to get the correct flags.
# See https://github.com/python/cpython/blob/3.3/Lib/subprocess.py#L488
if sys.version_info[:2] <= (3, 3):
import re
for i in range(1, len(args)):
args[i] = re.sub("-R+", "-R", args[i])
args += ['-c', cmd]
util.debug("launching resource tracker: {}".format(args))
# bpo-33613: Register a signal mask that will block the
# signals. This signal mask will be inherited by the child
# that is going to be spawned and will protect the child from a
# race condition that can make the child die before it
# registers signal handlers for SIGINT and SIGTERM. The mask is
# unregistered after spawning the child.
try:
if _HAVE_SIGMASK:
signal.pthread_sigmask(signal.SIG_BLOCK,
_IGNORED_SIGNALS)
pid = spawnv_passfds(exe, args, fds_to_pass)
finally:
if _HAVE_SIGMASK:
signal.pthread_sigmask(signal.SIG_UNBLOCK,
_IGNORED_SIGNALS)
except BaseException:
os.close(w)
raise
else:
self._fd = w
self._pid = pid
finally:
if sys.platform == "win32":
_winapi.CloseHandle(r)
else:
os.close(r)
def __call__(self, code=""):
tmpFile = os.tmpfile()
tmpFile.write(code)
tmpFile.seek(0)
self.plugin.msgThread.dll.MceIrPlaybackFromFile(
get_osfhandle(tmpFile.fileno()))
def unlock(f):
hfile = msvcrt.get_osfhandle(_fd(f))
def init():
# should_wrap instructs colorama to wrap stdout/stderr with an ASNI colorcode
# interpreter that converts them to SetConsoleTextAttribute calls. This only
# should be True in cases where we're connected to cmd.exe's console. Setting
# this to True on non-windows systems has no effect.
should_wrap = False
global IS_TTY, OUT_TYPE
IS_TTY = sys.stdout.isatty()
if IS_TTY:
# Yay! We detected a console in the normal way. It doesn't really matter
# if it's windows or not, we win.
OUT_TYPE = 'console'
should_wrap = True
elif sys.platform.startswith('win'):
# assume this is some sort of file
OUT_TYPE = 'file (win)'
import msvcrt
h = msvcrt.get_osfhandle(sys.stdout.fileno())
# h is the win32 HANDLE for stdout.
ftype = ctypes.windll.kernel32.GetFileType(h)
if ftype == 2: # FILE_TYPE_CHAR
# This is a normal cmd console, but we'll only get here if we're running
# inside a `git command` which is actually git->bash->command. Not sure
# why isatty doesn't detect this case.
OUT_TYPE = 'console (cmd via msys)'
IS_TTY = True
should_wrap = True
elif ftype == 3: # FILE_TYPE_PIPE
OUT_TYPE = 'pipe (win)'
# This is some kind of pipe on windows. This could either be a real pipe
# or this could be msys using a pipe to emulate a pty. We use the same
# algorithm that msys-git uses to determine if it's connected to a pty or
# not.
# This function and the structures are defined in the MSDN documentation
# using the same names.
def NT_SUCCESS(status):
# The first two bits of status are the severity. The success
# severities are 0 and 1, and the !success severities are 2 and 3.
# Therefore since ctypes interprets the default restype of the call
# to be an 'C int' (which is guaranteed to be signed 32 bits), All
# success codes are positive, and all !success codes are negative.
return status >= 0
class UNICODE_STRING(ctypes.Structure):
_fields_ = [('Length', ctypes.c_ushort),
('MaximumLength', ctypes.c_ushort),
('Buffer', ctypes.c_wchar_p)]
class OBJECT_NAME_INFORMATION(ctypes.Structure):
_fields_ = [('Name', UNICODE_STRING),
('NameBuffer', ctypes.c_wchar_p)]
buf = ctypes.create_string_buffer(1024)
# Ask NT what the name of the object our stdout HANDLE is. It would be
# possible to use GetFileInformationByHandleEx, but it's only available
# on Vista+. If you're reading this in 2017 or later, feel free to
# refactor this out.
#
# The '1' here is ObjectNameInformation
if NT_SUCCESS(
ctypes.windll.ntdll.NtQueryObject(h, 1, buf,
len(buf) - 2, None)):
out = OBJECT_NAME_INFORMATION.from_buffer(buf)
name = out.Name.Buffer.split('\\')[-1]
IS_TTY = name.startswith('msys-') and '-pty' in name
if IS_TTY:
OUT_TYPE = 'bash (msys)'
else:
# A normal file, or an unknown file type.
pass
else:
# This is non-windows, so we trust isatty.
OUT_TYPE = 'pipe or file'
# Enable native ANSI color codes on Windows 10.
if IS_TTY and platform.release() == '10':
if enable_native_ansi():
should_wrap = False
colorama.init(wrap=should_wrap)
def __open_inheritance_hack(*args, **kwargs):
result = __builtin__open(*args, **kwargs)
handle = msvcrt.get_osfhandle(result.fileno())
windll.kernel32.SetHandleInformation(handle, 1, 0)
return result
def lock(f, flags):
hfile = msvcrt.get_osfhandle(_fd(f))
