def adb_server():
"""Context manager for an ADB server.
This creates an ADB server and returns the port it's listening on.
"""
port = find_open_port()
read_pipe, write_pipe = os.pipe()
if sys.platform == "win32":
import msvcrt
write_handle = msvcrt.get_osfhandle(write_pipe)
os.set_handle_inheritable(write_handle, True)
reply_fd = str(write_handle)
else:
os.set_inheritable(write_pipe, True)
reply_fd = str(write_pipe)
proc = subprocess.Popen(["adb", "-L", "tcp:localhost:{}".format(port),
"fork-server", "server",
"--reply-fd", reply_fd], close_fds=False)
try:
os.close(write_pipe)
greeting = os.read(read_pipe, 1024)
assert greeting == b"OK\n", repr(greeting)
yield port
finally:
proc.terminate()
proc.wait()
def _start(self):
"""Start a new subprocess if nothing is available"""
if self._proc is not None:
return
# connect subprocess with a pair of pipes
main_read, worker_write = os.pipe()
worker_read, main_write = os.pipe()
cmd = [sys.executable, "-m", "tvm.exec.popen_worker"]
if sys.platform == "win32":
# pylint: disable=import-outside-toplevel
import msvcrt
worker_read_handle = msvcrt.get_osfhandle(worker_read)
worker_write_handle = msvcrt.get_osfhandle(worker_write)
os.set_handle_inheritable(worker_read_handle, True)
os.set_handle_inheritable(worker_write_handle, True)
cmd += [str(worker_read_handle), str(worker_write_handle)]
self._proc = subprocess.Popen(cmd, close_fds=False)
else:
cmd += [str(worker_read), str(worker_write)]
self._proc = subprocess.Popen(cmd,
pass_fds=(worker_read, worker_write))
# close worker side of the pipe
os.close(worker_read)
os.close(worker_write)
self._reader = os.fdopen(main_read, "rb")
self._writer = os.fdopen(main_write, "wb")
def _mk_inheritable(fd):
if sys.version_info[:2] > (3, 3):
if sys.platform == 'win32':
# Change to Windwos file handle
import msvcrt
fdh = msvcrt.get_osfhandle(fd)
os.set_handle_inheritable(fdh, True)
return fdh
else:
os.set_inheritable(fd, True)
return fd
elif sys.platform == 'win32':
# TODO: find a hack??
# Not yet working
import msvcrt
import _subprocess
curproc = _subprocess.GetCurrentProcess()
fdh = msvcrt.get_osfhandle(fd)
fdh = _subprocess.DuplicateHandle(
curproc,
fdh,
curproc,
0,
True, # set inheritable FLAG
_subprocess.DUPLICATE_SAME_ACCESS)
return fdh
else:
return fd
def process_data(data_path: Path, open_flags: int, shm_tag: Optional[str],
num_procs: int, executor: Executor,
prog_config: Tuple[int, int]) -> int:
# Use os.open instead of the built-in open() to avoid any unnecessary overhead in the creation
# of a file object.
fd = os.open(data_path, open_flags)
# Use a memory map to reduce the number of I/O operations. API differs betwees OSes:
if WINDOWS:
#assert shm_tag
# Obtain underlying file handle from file descriptor
os.set_handle_inheritable(get_osfhandle(fd), True)
mm = mmap(fd, 0, shm_tag, ACCESS_READ)
else:
os.set_inheritable(fd, True)
mm = mmap(fd, 0, MAP_SHARED, PROT_READ)
# Disregard the CSV file's header row by skipping past the first newline.
file_start = mm.find(b"\n") + 1
file_end = mm.size()
# If n = the number of CPU cores, create n processes that each work on 1/n of the total file.
num_procs = min(num_procs, line_count :=
count_lines(mm, file_start, file_end))
boundaries = get_boundaries(mm, file_start, file_end, num_procs)
if DEBUG:
print("<DEBUG>Boundaries:", boundaries)
# Create other variables for the child processes.
print_lock = LockFactory()
progress_bar = ProgressBar(line_count, *prog_config)
# We avoid the actual Pool class so we can get some more control over what gets sent where.
pool = tuple(
Process(target=proc_exec,
args=(str(i + 1), mm.size() if WINDOWS else fd, shm_tag,
boundaries[i], boundaries[i + 1], print_lock, executor,
progress_bar),
daemon=True) for i in range(num_procs))
# START PARSING
progress_bar() # Initial print
for p in pool:
p.start()
try: # Wait for completion of all or failure of one.
sentinel_map = {p.sentinel: p for p in pool}
sentinels = sentinel_map.keys() # Live view: updates with the dict
while num_procs != 0:
for sentinel in wait(sentinels): # wait() is a blocking call
if sentinel_map.pop(sentinel).exitcode == 0:
num_procs -= 1
else:
exit(1)
except BaseException as e:
# This section is reached when there's a keyboard interruption or something
print()
print(type(e).__name__, file=stderr)
exit(2)
print()
mm.close()
os.close(fd)
return line_count
def _mk_inheritable(fd):
if version >= (3, 3):
if sys.platform == 'win32':
# Change to Windwos file handle
import msvcrt
fdh = msvcrt.get_osfhandle(fd)
os.set_handle_inheritable(fdh, True)
return fdh
else:
os.set_inheritable(fd, True)
return fd
elif sys.platform == 'win32':
# TODO: find a hack??
# Not yet working
import msvcrt
import _subprocess
curproc = _subprocess.GetCurrentProcess()
fdh = msvcrt.get_osfhandle(fd)
fdh = _subprocess.DuplicateHandle(
curproc, fdh, curproc, 0,
True, # set inheritable FLAG
_subprocess.DUPLICATE_SAME_ACCESS)
return fdh
else:
return fd
def adb_server():
"""Context manager for an ADB server.
This creates an ADB server and returns the port it's listening on.
"""
port = 5038
# Kill any existing server on this non-default port.
subprocess.check_output(["adb", "-P", str(port), "kill-server"],
stderr=subprocess.STDOUT)
read_pipe, write_pipe = os.pipe()
if sys.platform == "win32":
import msvcrt
write_handle = msvcrt.get_osfhandle(write_pipe)
os.set_handle_inheritable(write_handle, True)
reply_fd = str(write_handle)
else:
os.set_inheritable(write_pipe, True)
reply_fd = str(write_pipe)
proc = subprocess.Popen(["adb", "-L", "tcp:localhost:{}".format(port),
"fork-server", "server",
"--reply-fd", reply_fd], close_fds=False)
try:
os.close(write_pipe)
greeting = os.read(read_pipe, 1024)
assert greeting == b"OK\n", repr(greeting)
yield port
finally:
proc.terminate()
proc.wait()
def to_subprocess(self):
if MS_WINDOWS:
os.set_handle_inheritable(self._handle, True)
arg = self._handle
else:
os.set_inheritable(self._fd, True)
arg = self._fd
return str(arg)
def _mk_inheritable(fd):
version = sys.version_info[:2]
if version >= (3, 3):
if sys.platform == 'win32':
# Change to Windwos file handle
import msvcrt
fdh = msvcrt.get_osfhandle(fd)
os.set_handle_inheritable(fdh, True)
return fdh
else:
os.set_inheritable(fd, True)
return fd
elif sys.platform == 'win32':
# TODO: find a hack??
# Not yet working
import msvcrt
fdh = msvcrt.get_osfhandle(fd)
return fdh
else:
return fd
def start_pipe_worker(command, env=None, priority='normal', **process_args):
import subprocess
w = Worker(env or {})
args = {
'stdout': subprocess.PIPE,
'stdin': subprocess.PIPE,
'env': w.env,
'close_fds': True
}
args.update(process_args)
pass_fds = None
try:
if iswindows:
priority = {
'high': subprocess.HIGH_PRIORITY_CLASS,
'normal': subprocess.NORMAL_PRIORITY_CLASS,
'low': subprocess.IDLE_PRIORITY_CLASS
}[priority]
args['creationflags'] = subprocess.CREATE_NO_WINDOW | priority
pass_fds = args.pop('pass_fds', None)
if pass_fds:
for fd in pass_fds:
os.set_handle_inheritable(fd, True)
args['startupinfo'] = subprocess.STARTUPINFO(
lpAttributeList={'handle_list': pass_fds})
else:
niceness = {'normal': 0, 'low': 10, 'high': 20}[priority]
args['env']['CALIBRE_WORKER_NICENESS'] = str(niceness)
exe = w.executable
cmd = [exe] if isinstance(exe, string_or_bytes) else exe
p = subprocess.Popen(cmd + ['--pipe-worker', command], **args)
finally:
if iswindows and pass_fds:
for fd in pass_fds:
os.set_handle_inheritable(fd, False)
return p
def set_inheritable(self, inheritable):
os.set_handle_inheritable(self.fileno(), inheritable)
def set_inheritable(self, inheritable):
os.set_handle_inheritable(self.fileno(), inheritable)
def __init__(self, args: List[str], extra_fds: Optional[List[int]] = None):
"""Constructor.
Args:
args: Command line to run, in argv format.
extra_fds: Optional list of extra file descriptors to share with the
subprocess.
Raises:
Error: if a win32 call fails.
"""
size = SIZE_T()
InitializeProcThreadAttributeList(None, 1, 0, ctypes.byref(size))
attr_list = ctypes.create_string_buffer(size.value)
res = InitializeProcThreadAttributeList(attr_list, 1, 0, ctypes.byref(size))
if not res:
raise Error("InitializeProcThreadAttributeList failed.")
extra_handles = [msvcrt.get_osfhandle(fd) for fd in extra_fds]
for extra_handle in extra_handles:
os.set_handle_inheritable(extra_handle, True)
handle_list_size = len(extra_handles)
handle_list = (HANDLE * handle_list_size)(
*[HANDLE(handle) for handle in extra_handles])
res = UpdateProcThreadAttribute(attr_list, 0,
PROC_THREAD_ATTRIBUTE_HANDLE_LIST,
handle_list, ctypes.sizeof(handle_list),
None, None)
if not res:
raise Error("UpdateProcThreadAttribute failed.")
DeleteProcThreadAttributeList(attr_list)
siex = STARTUPINFOEXW()
si = siex.StartupInfo
si.cb = ctypes.sizeof(siex)
si.wShowWindow = False
siex.lpAttributeList = ctypes.cast(attr_list, LPPROC_THREAD_ATTRIBUTE_LIST)
pi = PROCESS_INFORMATION()
command_line = subprocess.list2cmdline(args)
res = CreateProcessW(
None,
command_line,
None,
None,
True,
EXTENDED_STARTUPINFO_PRESENT,
None,
None,
ctypes.byref(si),
ctypes.byref(pi),
)
if not res:
raise Error("CreateProcessW failed.")
self._handle = pi.hProcess
win32api.CloseHandle(pi.hThread)
def __call__(self, redirect_output=True, cwd=None, priority=None, pass_fds=()):
'''
If redirect_output is True, output from the child is redirected
to a file on disk and this method returns the path to that file.
'''
exe = self.gui_executable if self.gui else self.executable
env = self.env
try:
origwd = cwd or os.path.abspath(getcwd())
except EnvironmentError:
# cwd no longer exists
origwd = cwd or os.path.expanduser('~')
env[native_string_type('ORIGWD')] = environ_item(as_hex_unicode(msgpack_dumps(origwd)))
_cwd = cwd
if priority is None:
priority = prefs['worker_process_priority']
cmd = [exe] if isinstance(exe, string_or_bytes) else exe
args = {
'env' : env,
'cwd' : _cwd,
}
if iswindows:
priority = {
'high' : subprocess.HIGH_PRIORITY_CLASS,
'normal' : subprocess.NORMAL_PRIORITY_CLASS,
'low' : subprocess.IDLE_PRIORITY_CLASS}[priority]
args['creationflags'] = subprocess.CREATE_NO_WINDOW|priority
else:
niceness = {
'normal' : 0,
'low' : 10,
'high' : 20,
}[priority]
args['env']['CALIBRE_WORKER_NICENESS'] = str(niceness)
ret = None
if redirect_output:
self._file = PersistentTemporaryFile('_worker_redirect.log')
args['stdout'] = self._file._fd
args['stderr'] = subprocess.STDOUT
if iswindows:
args['stdin'] = subprocess.PIPE
ret = self._file.name
if iswindows and 'stdin' not in args:
# On windows when using the pythonw interpreter,
# stdout, stderr and stdin may not be valid
args['stdin'] = subprocess.PIPE
args['stdout'] = windows_null_file
args['stderr'] = subprocess.STDOUT
args['close_fds'] = True
try:
if pass_fds:
if iswindows:
for fd in pass_fds:
os.set_handle_inheritable(fd, True)
args['startupinfo'] = subprocess.STARTUPINFO(lpAttributeList={'handle_list':pass_fds})
else:
args['pass_fds'] = pass_fds
self.child = subprocess.Popen(cmd, **args)
finally:
if iswindows and pass_fds:
for fd in pass_fds:
os.set_handle_inheritable(fd, False)
if 'stdin' in args:
self.child.stdin.close()
self.log_path = ret
return ret
def __init__(self,
args: List[str],
extra_handles: Optional[List[int]] = None):
"""Constructor.
Args:
args: Command line to run, in argv format.
extra_handles: Optional list of extra handles to share with the
subprocess.
Raises:
Error: if a win32 call fails.
"""
# Stack for resources which are needed by this instance.
self._exit_stack = contextlib.ExitStack()
# Stack for resources which are needed only during this method.
with contextlib.ExitStack() as stack:
sandbox_obj = self._exit_stack.enter_context(
sandbox.CreateSandbox())
size = SIZE_T()
InitializeProcThreadAttributeList(None, 2, 0, ctypes.byref(size))
attr_list = ctypes.create_string_buffer(size.value)
res = InitializeProcThreadAttributeList(attr_list, 2, 0,
ctypes.byref(size))
if not res:
raise Error("InitializeProcThreadAttributeList failed.")
stack.callback(DeleteProcThreadAttributeList, attr_list)
if extra_handles is None:
extra_handles = []
for extra_handle in extra_handles:
os.set_handle_inheritable(extra_handle, True)
handle_list_size = len(extra_handles)
handle_list = (HANDLE * handle_list_size)(
*[HANDLE(handle) for handle in extra_handles])
if handle_list:
res = UpdateProcThreadAttribute(
attr_list, 0, PROC_THREAD_ATTRIBUTE_HANDLE_LIST,
handle_list, ctypes.sizeof(handle_list), None, None)
if not res:
raise Error("UpdateProcThreadAttribute failed.")
if sandbox_obj.sid_string is not None:
psid = PSID()
if not ConvertStringSidToSidW(sandbox_obj.sid_string,
ctypes.byref(psid)):
raise Error("ConvertStringSidToSidW")
stack.callback(FreeSid, psid)
security_capabilities = SECURITY_CAPABILITIES()
security_capabilities.AppContainerSid = psid
res = UpdateProcThreadAttribute(
attr_list, 0, PROC_THREAD_ATTRIBUTE_SECURITY_CAPABILITIES,
ctypes.byref(security_capabilities),
ctypes.sizeof(security_capabilities), None, None)
if not res:
raise Error("UpdateProcThreadAttribute failed.")
siex = STARTUPINFOEXW()
si = siex.StartupInfo
si.cb = ctypes.sizeof(siex)
si.wShowWindow = False
siex.lpAttributeList = ctypes.cast(attr_list,
LPPROC_THREAD_ATTRIBUTE_LIST)
if sandbox_obj.desktop_name is not None:
si.lpDesktop = sandbox_obj.desktop_name
pi = PROCESS_INFORMATION()
command_line = subprocess.list2cmdline(args)
res = CreateProcessW(
None,
command_line,
None,
None,
True,
EXTENDED_STARTUPINFO_PRESENT,
None,
None,
ctypes.byref(si),
ctypes.byref(pi),
)
if not res:
raise Error("CreateProcessW failed.")
self._handle = pi.hProcess
self._exit_stack.callback(win32api.CloseHandle, pi.hProcess)
win32api.CloseHandle(pi.hThread)
self.pid = GetProcessId(self._handle)
if self.pid == 0:
raise Error("GetProcessId failed.")
def run_simple(hostname, port, application, use_reloader=False,
use_debugger=False, use_evalex=True,
extra_files=None, reloader_interval=1,
reloader_type='auto', threaded=False,
processes=1, request_handler=None, static_files=None,
passthrough_errors=False, ssl_context=None):
"""Start a WSGI application. Optional features include a reloader,
multithreading and fork support.
This function has a command-line interface too::
python -m werkzeug.serving --help
.. versionadded:: 0.5
`static_files` was added to simplify serving of static files as well
as `passthrough_errors`.
.. versionadded:: 0.6
support for SSL was added.
.. versionadded:: 0.8
Added support for automatically loading a SSL context from certificate
file and private key.
.. versionadded:: 0.9
Added command-line interface.
.. versionadded:: 0.10
Improved the reloader and added support for changing the backend
through the `reloader_type` parameter. See :ref:`reloader`
for more information.
:param hostname: The host for the application. eg: ``'localhost'``
:param port: The port for the server. eg: ``8080``
:param application: the WSGI application to execute
:param use_reloader: should the server automatically restart the python
process if modules were changed?
:param use_debugger: should the werkzeug debugging system be used?
:param use_evalex: should the exception evaluation feature be enabled?
:param extra_files: a list of files the reloader should watch
additionally to the modules. For example configuration
files.
:param reloader_interval: the interval for the reloader in seconds.
:param reloader_type: the type of reloader to use. The default is
auto detection. Valid values are ``'stat'`` and
``'watchdog'``. See :ref:`reloader` for more
information.
:param threaded: should the process handle each request in a separate
thread?
:param processes: if greater than 1 then handle each request in a new process
up to this maximum number of concurrent processes.
:param request_handler: optional parameter that can be used to replace
the default one. You can use this to replace it
with a different
:class:`~BaseHTTPServer.BaseHTTPRequestHandler`
subclass.
:param static_files: a dict of paths for static files. This works exactly
like :class:`SharedDataMiddleware`, it's actually
just wrapping the application in that middleware before
serving.
:param passthrough_errors: set this to `True` to disable the error catching.
This means that the server will die on errors but
it can be useful to hook debuggers in (pdb etc.)
:param ssl_context: an SSL context for the connection. Either an
:class:`ssl.SSLContext`, a tuple in the form
``(cert_file, pkey_file)``, the string ``'adhoc'`` if
the server should automatically create one, or ``None``
to disable SSL (which is the default).
"""
if use_debugger:
from werkzeug.debug import DebuggedApplication
application = DebuggedApplication(application, use_evalex)
if static_files:
from werkzeug.wsgi import SharedDataMiddleware
application = SharedDataMiddleware(application, static_files)
def inner():
try:
fd = int(os.environ['WERKZEUG_SERVER_FD'])
except (LookupError, ValueError):
fd = None
make_server(hostname, port, application, threaded,
processes, request_handler,
passthrough_errors, ssl_context,
fd=fd).serve_forever()
if use_reloader:
# If we're not running already in the subprocess that is the
# reloader we want to open up a socket early to make sure the
# port is actually available.
if os.environ.get('WERKZEUG_RUN_MAIN') != 'true':
if port == 0 and not can_open_by_fd:
raise ValueError('Cannot bind to a random port with enabled '
'reloader if the Python interpreter does '
'not support socket opening by fd.')
# Create and destroy a socket so that any exceptions are
# raised before we spawn a separate Python interpreter and
# lose this ability.
address_family = select_ip_version(hostname, port)
s = socket.socket(address_family, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind((hostname, port))
if hasattr(os, 'set_inheritable'):
if platform.system() == 'Windows':
os.set_handle_inheritable(s.fileno(), True)
else:
os.set_inheritable(s.fileno(), True)
# If we can open the socket by file descriptor, then we can just
# reuse this one and our socket will survive the restarts.
if can_open_by_fd:
os.environ['WERKZEUG_SERVER_FD'] = str(s.fileno())
s.listen(LISTEN_QUEUE)
else:
s.close()
from ._reloader import run_with_reloader
run_with_reloader(inner, extra_files, reloader_interval,
reloader_type)
else:
inner()
