def adb_server(): """Context manager for an ADB server. This creates an ADB server and returns the port it's listening on. """ port = find_open_port() read_pipe, write_pipe = os.pipe() if sys.platform == "win32": import msvcrt write_handle = msvcrt.get_osfhandle(write_pipe) os.set_handle_inheritable(write_handle, True) reply_fd = str(write_handle) else: os.set_inheritable(write_pipe, True) reply_fd = str(write_pipe) proc = subprocess.Popen(["adb", "-L", "tcp:localhost:{}".format(port), "fork-server", "server", "--reply-fd", reply_fd], close_fds=False) try: os.close(write_pipe) greeting = os.read(read_pipe, 1024) assert greeting == b"OK\n", repr(greeting) yield port finally: proc.terminate() proc.wait()
def _start(self): """Start a new subprocess if nothing is available""" if self._proc is not None: return # connect subprocess with a pair of pipes main_read, worker_write = os.pipe() worker_read, main_write = os.pipe() cmd = [sys.executable, "-m", "tvm.exec.popen_worker"] if sys.platform == "win32": # pylint: disable=import-outside-toplevel import msvcrt worker_read_handle = msvcrt.get_osfhandle(worker_read) worker_write_handle = msvcrt.get_osfhandle(worker_write) os.set_handle_inheritable(worker_read_handle, True) os.set_handle_inheritable(worker_write_handle, True) cmd += [str(worker_read_handle), str(worker_write_handle)] self._proc = subprocess.Popen(cmd, close_fds=False) else: cmd += [str(worker_read), str(worker_write)] self._proc = subprocess.Popen(cmd, pass_fds=(worker_read, worker_write)) # close worker side of the pipe os.close(worker_read) os.close(worker_write) self._reader = os.fdopen(main_read, "rb") self._writer = os.fdopen(main_write, "wb")
def _mk_inheritable(fd): if sys.version_info[:2] > (3, 3): if sys.platform == 'win32': # Change to Windwos file handle import msvcrt fdh = msvcrt.get_osfhandle(fd) os.set_handle_inheritable(fdh, True) return fdh else: os.set_inheritable(fd, True) return fd elif sys.platform == 'win32': # TODO: find a hack?? # Not yet working import msvcrt import _subprocess curproc = _subprocess.GetCurrentProcess() fdh = msvcrt.get_osfhandle(fd) fdh = _subprocess.DuplicateHandle( curproc, fdh, curproc, 0, True, # set inheritable FLAG _subprocess.DUPLICATE_SAME_ACCESS) return fdh else: return fd
def process_data(data_path: Path, open_flags: int, shm_tag: Optional[str], num_procs: int, executor: Executor, prog_config: Tuple[int, int]) -> int: # Use os.open instead of the built-in open() to avoid any unnecessary overhead in the creation # of a file object. fd = os.open(data_path, open_flags) # Use a memory map to reduce the number of I/O operations. API differs betwees OSes: if WINDOWS: #assert shm_tag # Obtain underlying file handle from file descriptor os.set_handle_inheritable(get_osfhandle(fd), True) mm = mmap(fd, 0, shm_tag, ACCESS_READ) else: os.set_inheritable(fd, True) mm = mmap(fd, 0, MAP_SHARED, PROT_READ) # Disregard the CSV file's header row by skipping past the first newline. file_start = mm.find(b"\n") + 1 file_end = mm.size() # If n = the number of CPU cores, create n processes that each work on 1/n of the total file. num_procs = min(num_procs, line_count := count_lines(mm, file_start, file_end)) boundaries = get_boundaries(mm, file_start, file_end, num_procs) if DEBUG: print("<DEBUG>Boundaries:", boundaries) # Create other variables for the child processes. print_lock = LockFactory() progress_bar = ProgressBar(line_count, *prog_config) # We avoid the actual Pool class so we can get some more control over what gets sent where. pool = tuple( Process(target=proc_exec, args=(str(i + 1), mm.size() if WINDOWS else fd, shm_tag, boundaries[i], boundaries[i + 1], print_lock, executor, progress_bar), daemon=True) for i in range(num_procs)) # START PARSING progress_bar() # Initial print for p in pool: p.start() try: # Wait for completion of all or failure of one. sentinel_map = {p.sentinel: p for p in pool} sentinels = sentinel_map.keys() # Live view: updates with the dict while num_procs != 0: for sentinel in wait(sentinels): # wait() is a blocking call if sentinel_map.pop(sentinel).exitcode == 0: num_procs -= 1 else: exit(1) except BaseException as e: # This section is reached when there's a keyboard interruption or something print() print(type(e).__name__, file=stderr) exit(2) print() mm.close() os.close(fd) return line_count
def _mk_inheritable(fd): if version >= (3, 3): if sys.platform == 'win32': # Change to Windwos file handle import msvcrt fdh = msvcrt.get_osfhandle(fd) os.set_handle_inheritable(fdh, True) return fdh else: os.set_inheritable(fd, True) return fd elif sys.platform == 'win32': # TODO: find a hack?? # Not yet working import msvcrt import _subprocess curproc = _subprocess.GetCurrentProcess() fdh = msvcrt.get_osfhandle(fd) fdh = _subprocess.DuplicateHandle( curproc, fdh, curproc, 0, True, # set inheritable FLAG _subprocess.DUPLICATE_SAME_ACCESS) return fdh else: return fd
def adb_server(): """Context manager for an ADB server. This creates an ADB server and returns the port it's listening on. """ port = 5038 # Kill any existing server on this non-default port. subprocess.check_output(["adb", "-P", str(port), "kill-server"], stderr=subprocess.STDOUT) read_pipe, write_pipe = os.pipe() if sys.platform == "win32": import msvcrt write_handle = msvcrt.get_osfhandle(write_pipe) os.set_handle_inheritable(write_handle, True) reply_fd = str(write_handle) else: os.set_inheritable(write_pipe, True) reply_fd = str(write_pipe) proc = subprocess.Popen(["adb", "-L", "tcp:localhost:{}".format(port), "fork-server", "server", "--reply-fd", reply_fd], close_fds=False) try: os.close(write_pipe) greeting = os.read(read_pipe, 1024) assert greeting == b"OK\n", repr(greeting) yield port finally: proc.terminate() proc.wait()
def to_subprocess(self): if MS_WINDOWS: os.set_handle_inheritable(self._handle, True) arg = self._handle else: os.set_inheritable(self._fd, True) arg = self._fd return str(arg)
def _mk_inheritable(fd): version = sys.version_info[:2] if version >= (3, 3): if sys.platform == 'win32': # Change to Windwos file handle import msvcrt fdh = msvcrt.get_osfhandle(fd) os.set_handle_inheritable(fdh, True) return fdh else: os.set_inheritable(fd, True) return fd elif sys.platform == 'win32': # TODO: find a hack?? # Not yet working import msvcrt fdh = msvcrt.get_osfhandle(fd) return fdh else: return fd
def start_pipe_worker(command, env=None, priority='normal', **process_args): import subprocess w = Worker(env or {}) args = { 'stdout': subprocess.PIPE, 'stdin': subprocess.PIPE, 'env': w.env, 'close_fds': True } args.update(process_args) pass_fds = None try: if iswindows: priority = { 'high': subprocess.HIGH_PRIORITY_CLASS, 'normal': subprocess.NORMAL_PRIORITY_CLASS, 'low': subprocess.IDLE_PRIORITY_CLASS }[priority] args['creationflags'] = subprocess.CREATE_NO_WINDOW | priority pass_fds = args.pop('pass_fds', None) if pass_fds: for fd in pass_fds: os.set_handle_inheritable(fd, True) args['startupinfo'] = subprocess.STARTUPINFO( lpAttributeList={'handle_list': pass_fds}) else: niceness = {'normal': 0, 'low': 10, 'high': 20}[priority] args['env']['CALIBRE_WORKER_NICENESS'] = str(niceness) exe = w.executable cmd = [exe] if isinstance(exe, string_or_bytes) else exe p = subprocess.Popen(cmd + ['--pipe-worker', command], **args) finally: if iswindows and pass_fds: for fd in pass_fds: os.set_handle_inheritable(fd, False) return p
def set_inheritable(self, inheritable): os.set_handle_inheritable(self.fileno(), inheritable)
def __init__(self, args: List[str], extra_fds: Optional[List[int]] = None): """Constructor. Args: args: Command line to run, in argv format. extra_fds: Optional list of extra file descriptors to share with the subprocess. Raises: Error: if a win32 call fails. """ size = SIZE_T() InitializeProcThreadAttributeList(None, 1, 0, ctypes.byref(size)) attr_list = ctypes.create_string_buffer(size.value) res = InitializeProcThreadAttributeList(attr_list, 1, 0, ctypes.byref(size)) if not res: raise Error("InitializeProcThreadAttributeList failed.") extra_handles = [msvcrt.get_osfhandle(fd) for fd in extra_fds] for extra_handle in extra_handles: os.set_handle_inheritable(extra_handle, True) handle_list_size = len(extra_handles) handle_list = (HANDLE * handle_list_size)( *[HANDLE(handle) for handle in extra_handles]) res = UpdateProcThreadAttribute(attr_list, 0, PROC_THREAD_ATTRIBUTE_HANDLE_LIST, handle_list, ctypes.sizeof(handle_list), None, None) if not res: raise Error("UpdateProcThreadAttribute failed.") DeleteProcThreadAttributeList(attr_list) siex = STARTUPINFOEXW() si = siex.StartupInfo si.cb = ctypes.sizeof(siex) si.wShowWindow = False siex.lpAttributeList = ctypes.cast(attr_list, LPPROC_THREAD_ATTRIBUTE_LIST) pi = PROCESS_INFORMATION() command_line = subprocess.list2cmdline(args) res = CreateProcessW( None, command_line, None, None, True, EXTENDED_STARTUPINFO_PRESENT, None, None, ctypes.byref(si), ctypes.byref(pi), ) if not res: raise Error("CreateProcessW failed.") self._handle = pi.hProcess win32api.CloseHandle(pi.hThread)
def __call__(self, redirect_output=True, cwd=None, priority=None, pass_fds=()): ''' If redirect_output is True, output from the child is redirected to a file on disk and this method returns the path to that file. ''' exe = self.gui_executable if self.gui else self.executable env = self.env try: origwd = cwd or os.path.abspath(getcwd()) except EnvironmentError: # cwd no longer exists origwd = cwd or os.path.expanduser('~') env[native_string_type('ORIGWD')] = environ_item(as_hex_unicode(msgpack_dumps(origwd))) _cwd = cwd if priority is None: priority = prefs['worker_process_priority'] cmd = [exe] if isinstance(exe, string_or_bytes) else exe args = { 'env' : env, 'cwd' : _cwd, } if iswindows: priority = { 'high' : subprocess.HIGH_PRIORITY_CLASS, 'normal' : subprocess.NORMAL_PRIORITY_CLASS, 'low' : subprocess.IDLE_PRIORITY_CLASS}[priority] args['creationflags'] = subprocess.CREATE_NO_WINDOW|priority else: niceness = { 'normal' : 0, 'low' : 10, 'high' : 20, }[priority] args['env']['CALIBRE_WORKER_NICENESS'] = str(niceness) ret = None if redirect_output: self._file = PersistentTemporaryFile('_worker_redirect.log') args['stdout'] = self._file._fd args['stderr'] = subprocess.STDOUT if iswindows: args['stdin'] = subprocess.PIPE ret = self._file.name if iswindows and 'stdin' not in args: # On windows when using the pythonw interpreter, # stdout, stderr and stdin may not be valid args['stdin'] = subprocess.PIPE args['stdout'] = windows_null_file args['stderr'] = subprocess.STDOUT args['close_fds'] = True try: if pass_fds: if iswindows: for fd in pass_fds: os.set_handle_inheritable(fd, True) args['startupinfo'] = subprocess.STARTUPINFO(lpAttributeList={'handle_list':pass_fds}) else: args['pass_fds'] = pass_fds self.child = subprocess.Popen(cmd, **args) finally: if iswindows and pass_fds: for fd in pass_fds: os.set_handle_inheritable(fd, False) if 'stdin' in args: self.child.stdin.close() self.log_path = ret return ret
def __init__(self, args: List[str], extra_handles: Optional[List[int]] = None): """Constructor. Args: args: Command line to run, in argv format. extra_handles: Optional list of extra handles to share with the subprocess. Raises: Error: if a win32 call fails. """ # Stack for resources which are needed by this instance. self._exit_stack = contextlib.ExitStack() # Stack for resources which are needed only during this method. with contextlib.ExitStack() as stack: sandbox_obj = self._exit_stack.enter_context( sandbox.CreateSandbox()) size = SIZE_T() InitializeProcThreadAttributeList(None, 2, 0, ctypes.byref(size)) attr_list = ctypes.create_string_buffer(size.value) res = InitializeProcThreadAttributeList(attr_list, 2, 0, ctypes.byref(size)) if not res: raise Error("InitializeProcThreadAttributeList failed.") stack.callback(DeleteProcThreadAttributeList, attr_list) if extra_handles is None: extra_handles = [] for extra_handle in extra_handles: os.set_handle_inheritable(extra_handle, True) handle_list_size = len(extra_handles) handle_list = (HANDLE * handle_list_size)( *[HANDLE(handle) for handle in extra_handles]) if handle_list: res = UpdateProcThreadAttribute( attr_list, 0, PROC_THREAD_ATTRIBUTE_HANDLE_LIST, handle_list, ctypes.sizeof(handle_list), None, None) if not res: raise Error("UpdateProcThreadAttribute failed.") if sandbox_obj.sid_string is not None: psid = PSID() if not ConvertStringSidToSidW(sandbox_obj.sid_string, ctypes.byref(psid)): raise Error("ConvertStringSidToSidW") stack.callback(FreeSid, psid) security_capabilities = SECURITY_CAPABILITIES() security_capabilities.AppContainerSid = psid res = UpdateProcThreadAttribute( attr_list, 0, PROC_THREAD_ATTRIBUTE_SECURITY_CAPABILITIES, ctypes.byref(security_capabilities), ctypes.sizeof(security_capabilities), None, None) if not res: raise Error("UpdateProcThreadAttribute failed.") siex = STARTUPINFOEXW() si = siex.StartupInfo si.cb = ctypes.sizeof(siex) si.wShowWindow = False siex.lpAttributeList = ctypes.cast(attr_list, LPPROC_THREAD_ATTRIBUTE_LIST) if sandbox_obj.desktop_name is not None: si.lpDesktop = sandbox_obj.desktop_name pi = PROCESS_INFORMATION() command_line = subprocess.list2cmdline(args) res = CreateProcessW( None, command_line, None, None, True, EXTENDED_STARTUPINFO_PRESENT, None, None, ctypes.byref(si), ctypes.byref(pi), ) if not res: raise Error("CreateProcessW failed.") self._handle = pi.hProcess self._exit_stack.callback(win32api.CloseHandle, pi.hProcess) win32api.CloseHandle(pi.hThread) self.pid = GetProcessId(self._handle) if self.pid == 0: raise Error("GetProcessId failed.")
def run_simple(hostname, port, application, use_reloader=False, use_debugger=False, use_evalex=True, extra_files=None, reloader_interval=1, reloader_type='auto', threaded=False, processes=1, request_handler=None, static_files=None, passthrough_errors=False, ssl_context=None): """Start a WSGI application. Optional features include a reloader, multithreading and fork support. This function has a command-line interface too:: python -m werkzeug.serving --help .. versionadded:: 0.5 `static_files` was added to simplify serving of static files as well as `passthrough_errors`. .. versionadded:: 0.6 support for SSL was added. .. versionadded:: 0.8 Added support for automatically loading a SSL context from certificate file and private key. .. versionadded:: 0.9 Added command-line interface. .. versionadded:: 0.10 Improved the reloader and added support for changing the backend through the `reloader_type` parameter. See :ref:`reloader` for more information. :param hostname: The host for the application. eg: ``'localhost'`` :param port: The port for the server. eg: ``8080`` :param application: the WSGI application to execute :param use_reloader: should the server automatically restart the python process if modules were changed? :param use_debugger: should the werkzeug debugging system be used? :param use_evalex: should the exception evaluation feature be enabled? :param extra_files: a list of files the reloader should watch additionally to the modules. For example configuration files. :param reloader_interval: the interval for the reloader in seconds. :param reloader_type: the type of reloader to use. The default is auto detection. Valid values are ``'stat'`` and ``'watchdog'``. See :ref:`reloader` for more information. :param threaded: should the process handle each request in a separate thread? :param processes: if greater than 1 then handle each request in a new process up to this maximum number of concurrent processes. :param request_handler: optional parameter that can be used to replace the default one. You can use this to replace it with a different :class:`~BaseHTTPServer.BaseHTTPRequestHandler` subclass. :param static_files: a dict of paths for static files. This works exactly like :class:`SharedDataMiddleware`, it's actually just wrapping the application in that middleware before serving. :param passthrough_errors: set this to `True` to disable the error catching. This means that the server will die on errors but it can be useful to hook debuggers in (pdb etc.) :param ssl_context: an SSL context for the connection. Either an :class:`ssl.SSLContext`, a tuple in the form ``(cert_file, pkey_file)``, the string ``'adhoc'`` if the server should automatically create one, or ``None`` to disable SSL (which is the default). """ if use_debugger: from werkzeug.debug import DebuggedApplication application = DebuggedApplication(application, use_evalex) if static_files: from werkzeug.wsgi import SharedDataMiddleware application = SharedDataMiddleware(application, static_files) def inner(): try: fd = int(os.environ['WERKZEUG_SERVER_FD']) except (LookupError, ValueError): fd = None make_server(hostname, port, application, threaded, processes, request_handler, passthrough_errors, ssl_context, fd=fd).serve_forever() if use_reloader: # If we're not running already in the subprocess that is the # reloader we want to open up a socket early to make sure the # port is actually available. if os.environ.get('WERKZEUG_RUN_MAIN') != 'true': if port == 0 and not can_open_by_fd: raise ValueError('Cannot bind to a random port with enabled ' 'reloader if the Python interpreter does ' 'not support socket opening by fd.') # Create and destroy a socket so that any exceptions are # raised before we spawn a separate Python interpreter and # lose this ability. address_family = select_ip_version(hostname, port) s = socket.socket(address_family, socket.SOCK_STREAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) s.bind((hostname, port)) if hasattr(os, 'set_inheritable'): if platform.system() == 'Windows': os.set_handle_inheritable(s.fileno(), True) else: os.set_inheritable(s.fileno(), True) # If we can open the socket by file descriptor, then we can just # reuse this one and our socket will survive the restarts. if can_open_by_fd: os.environ['WERKZEUG_SERVER_FD'] = str(s.fileno()) s.listen(LISTEN_QUEUE) else: s.close() from ._reloader import run_with_reloader run_with_reloader(inner, extra_files, reloader_interval, reloader_type) else: inner()