def __init__(self, name: str, timeout: Optional[float] = None) -> None: if sys.platform == 'win32': name = r'\\.\pipe\{}-{}.pipe'.format( name, base64.urlsafe_b64encode(os.urandom(6)).decode()) else: name = f'{name}.sock' super().__init__(name, timeout) if sys.platform == 'win32': self.connection = _winapi.CreateNamedPipe( self.name, _winapi.PIPE_ACCESS_DUPLEX | _winapi.FILE_FLAG_FIRST_PIPE_INSTANCE | _winapi.FILE_FLAG_OVERLAPPED, _winapi.PIPE_READMODE_MESSAGE | _winapi.PIPE_TYPE_MESSAGE | _winapi.PIPE_WAIT | 0x8, # PIPE_REJECT_REMOTE_CLIENTS 1, # one instance self.BUFFER_SIZE, self.BUFFER_SIZE, _winapi.NMPWAIT_WAIT_FOREVER, 0, # Use default security descriptor ) if self.connection == -1: # INVALID_HANDLE_VALUE err = _winapi.GetLastError() raise IPCException(f'Invalid handle to pipe: {err}') else: self.sock_directory = tempfile.mkdtemp() sockfile = os.path.join(self.sock_directory, self.name) self.sock = socket.socket(socket.AF_UNIX) self.sock.bind(sockfile) self.sock.listen(1) if timeout is not None: self.sock.settimeout(timeout)
def __init__(self, size): self.size = size for i in range(100): name = 'pym-%d-%s' % (os.getpid(), next(self._rand)) buf = mmap.mmap(-1, size, tagname=name) if _winapi.GetLastError() == 0: break buf.close() else: raise FileExistsError('Cannot find name for new mmap') self.name = name self.buffer = buf self._state = self.size, self.name
def cmdl(ctx, components, config, n_environments, exploit, show_ui): """Run RELAAX components. \b COMPONENTS: all - run environments and servers (default) environment - run environment servers - run rlx-server, parameter-server, metrics-server and wsproxy (if specified in config yaml) rlx-server - run rlx-server parameter-server - run parameter-server metrics-server - run metrics-server wsproxy - run websockets proxy \b For example: - run environment, rlx-server, parameter-server, metrics-server, and wsproxy $relaax run all - run rlx-server, parameter-server, metrics-server, and wsproxy $relaax run servers """ ctx.setup_logger(format='%(asctime)s %(name)s\t\t | %(message)s') # Disable TF warnings os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' os.environ["COMSPEC"] = "powershell.exe" # Execute command if sys.platform == 'win32': honcho.manager.KILL_WAIT = 120 honcho.process.Popen = PopenPatched import _winapi import ctypes firstRun = False mutex = ctypes.windll.kernel32.CreateMutexA(None, False, "RELAAX_WINDOWS_MUTEX") if _winapi.GetLastError() == 0: firstRun = True if firstRun: os.system("start powershell " + ' '.join(sys.argv)) time.sleep(5) _winapi.CloseHandle(mutex) else: _winapi.CloseHandle(mutex) honcho.process.Popen = PopenPatched CmdlRun(ctx, set(components), config.name, n_environments, exploit, show_ui).run_components() else: CmdlRun(ctx, set(components), config.name, n_environments, exploit, show_ui).run_components()
def __init__(self, name=None, create=False, size=0): if not size >= 0: raise ValueError("'size' must be a positive integer") if create: self._flags = _O_CREX | os.O_RDWR if name is None and not self._flags & os.O_EXCL: raise ValueError("'name' can only be None if create=True") # Windows Named Shared Memory if create: while True: temp_name = _make_filename() if name is None else name # Create and reserve shared memory block with this name # until it can be attached to by mmap. h_map = _winapi.CreateFileMapping( _winapi.INVALID_HANDLE_VALUE, _winapi.NULL, _winapi.PAGE_READWRITE, (size >> 32) & 0xFFFFFFFF, size & 0xFFFFFFFF, temp_name) try: last_error_code = _winapi.GetLastError() if last_error_code == _winapi.ERROR_ALREADY_EXISTS: if name is not None: raise FileExistsError( errno.EEXIST, os.strerror(errno.EEXIST), name, _winapi.ERROR_ALREADY_EXISTS) else: continue self._mmap = mmap.mmap(-1, size, tagname=temp_name) finally: _winapi.CloseHandle(h_map) self._name = temp_name break else: self._name = name # Dynamically determine the existing named shared memory # block's size which is likely a multiple of mmap.PAGESIZE. h_map = _winapi.OpenFileMapping(_winapi.FILE_MAP_READ, False, name) try: p_buf = _winapi.MapViewOfFile(h_map, _winapi.FILE_MAP_READ, 0, 0, 0) finally: _winapi.CloseHandle(h_map) size = _winapi.VirtualQuerySize(p_buf) self._mmap = mmap.mmap(-1, size, tagname=name) self._size = size self._buf = memoryview(self._mmap)
def __setstate__(self, state): self.size, self.name = self._state = state self.buffer = mmap.mmap(-1, self.size, tagname=self.name) assert _winapi.GetLastError() == _winapi.ERROR_ALREADY_EXISTS
def __init__(self, name=None, create=False, size=0): if not size >= 0: raise ValueError("'size' must be a positive integer") if create: self._flags = _O_CREX | os.O_RDWR if size == 0: raise ValueError("'size' must be a positive number different from zero") if name is None and not self._flags & os.O_EXCL: raise ValueError("'name' can only be None if create=True") if _USE_POSIX: # POSIX Shared Memory if name is None: while True: name = _make_filename() try: self._fd = _posixshmem.shm_open( name, self._flags, mode=self._mode ) except FileExistsError: continue self._name = name break else: name = "/" + name if self._prepend_leading_slash else name self._fd = _posixshmem.shm_open( name, self._flags, mode=self._mode ) self._name = name try: if create and size: os.ftruncate(self._fd, size) stats = os.fstat(self._fd) size = stats.st_size self._mmap = mmap.mmap(self._fd, size) except OSError: self.unlink() raise from .resource_tracker import register register(self._name, "shared_memory") else: # Windows Named Shared Memory if create: while True: temp_name = _make_filename() if name is None else name # Create and reserve shared memory block with this name # until it can be attached to by mmap. h_map = _winapi.CreateFileMapping( _winapi.INVALID_HANDLE_VALUE, _winapi.NULL, _winapi.PAGE_READWRITE, (size >> 32) & 0xFFFFFFFF, size & 0xFFFFFFFF, temp_name ) try: last_error_code = _winapi.GetLastError() if last_error_code == _winapi.ERROR_ALREADY_EXISTS: if name is not None: raise FileExistsError( errno.EEXIST, os.strerror(errno.EEXIST), name, _winapi.ERROR_ALREADY_EXISTS ) else: continue self._mmap = mmap.mmap(-1, size, tagname=temp_name) finally: _winapi.CloseHandle(h_map) self._name = temp_name break else: self._name = name # Dynamically determine the existing named shared memory # block's size which is likely a multiple of mmap.PAGESIZE. h_map = _winapi.OpenFileMapping( _winapi.FILE_MAP_READ, False, name ) try: p_buf = _winapi.MapViewOfFile( h_map, _winapi.FILE_MAP_READ, 0, 0, 0 ) finally: _winapi.CloseHandle(h_map) size = _winapi.VirtualQuerySize(p_buf) self._mmap = mmap.mmap(-1, size, tagname=name) self._size = size self._buf = memoryview(self._mmap)
def __init__(self, map_id, size_nbytes): """ Create a segment of shared memory with an automatically generated name. Or open an already created segment with the provided name. :param map_id: The name of the shared memory. Should be None if a new segment is to be created. :param size_nbytes: The size, in bytes, of the segment. """ # This flag is used to signal if the shared memory has been unlinked by the owning process. self._flag = None self._closed = False try: # We actually need one extra byte for the signal flag. alloc_nbytes = size_nbytes + 1 # The master process is the process that created the memory. # The master handles the lifetime of the memory, and unlinks it when it is no longer needed. master = (map_id is None) if _USE_INTERNAL: if master: self._raw_mem = multiprocessing.shared_memory.SharedMemory( create=True, size=alloc_nbytes) map_id = self._raw_mem.name else: self._raw_mem = multiprocessing.shared_memory.SharedMemory( name=map_id, size=alloc_nbytes) def unlink(): self._raw_mem.unlink() def close(): self._raw_mem.close() self._buf = self._raw_mem.buf def release_buf(): pass # Releasing the buffer is handled by multiprocessing. else: if _USE_POSIX: if master: #master = True while True: # Generate a new ID. map_id = _make_map_id() # Attempt to create the buffer with this ID, this command will fail if a buffer already exists with that name. try: self._fd = _shm_open(map_id, os.O_CREAT | os.O_EXCL | os.O_RDWR, mode=0o600) except OSError as e: # If a buffer with that name already exists, try again. if e.errno == errno.EEXIST: continue else: raise break else: self._fd = _shm_open(map_id, os.O_RDWR, mode=0o600) def unlink(): if self._fd is not None: _shm_unlink(map_id) try: if master: # If the segment has just been created, resize it to the appropriate size. os.ftruncate(self._fd, alloc_nbytes) # Now map it into memory. self._mmap = mmap.mmap(self._fd, alloc_nbytes) except OSError: unlink() raise def close(): if self._mmap is not None: self._mmap.close() self._mmap = None if self._fd is not None: os.close(self._fd) self._fd = None else: ## WINDOWS if master: while True: map_id = _make_map_id() # It might be better here to open a mmap with length 0, catch the exception, then reopen it with the correct size self._mmap = mmap.mmap( -1, alloc_nbytes, tagname=map_id.decode("utf-8")) if _winapi.GetLastError() == 0: # If mapping the segment suceeded, then stop now, otherwise try again. break self._mmap.close() else: self._mmap = mmap.mmap(-1, alloc_nbytes, tagname=map_id.decode("utf-8")) def unlink(): pass def close(): self._mmap.close() if _PYTHON3: self._buf = memoryview(self._mmap) else: # Python 2.7 requires passing the mapped memory through ctypes first. self._buf = memoryview( (ctypes.c_byte * alloc_nbytes).from_buffer(self._mmap)) def release_buf(): if self._buf is not None: if _PYTHON3: self._buf.release() self._buf = None # The flag is the first byte of the memory self._flag = self._buf[:1] # The actual exposed buffer is the rest of the memory. self._ary = self._buf[1:] def release(): # Release these pointers when the buffer is closed. if self._flag is not None: if _PYTHON3: self._flag.release() self._flag = None if self._ary is not None: if _PYTHON3: self._ary.release() self._ary = None release_buf() self._flag[:] = b'\x00' # Flag set to initially available. self.name = map_id self.size_nbytes = size_nbytes self._lock = threading.RLock() except: # If something went wrong, make sure these functions are defined. if 'release' not in locals(): def release(): pass if 'close' not in locals(): def close(): pass if 'unlink' not in locals(): def unlink(): pass raise finally: def cleanup(): self._cleanup = lambda: None if master and self._flag is not None: self._flag[:] = b'\x01' # Set the flag to signal this buffer has been unlinked. release() close() if master: # Unlinking has to happen last, but the flag needs to be set before the pointers are released. unlink() self._cleanup = cleanup
def __init__(self, size): self.size = size self.name = 'pym-%d-%d' % (os.getpid(), next(Arena._counter)) self.buffer = mmap.mmap(-1, self.size, tagname=self.name) assert _winapi.GetLastError() == 0, 'tagname already in use' self._state = (self.size, self.name)
NULL = DWORD(0) OPEN_EXISTING = DWORD(3) FILE_SHARE_WRITE = DWORD(2) input_path = ctypes.create_unicode_buffer( u'D:\\Downloads\\linuxmint-19-cinnamon-64bit-v2.iso') input_file = ctypes.windll.kernel32.CreateFileW( input_path, GENERIC_READ, DWORD(0), # dwShareMode NULL, # lpSecurityAttributes OPEN_EXISTING, # dwCreationDisposition _file_flags, # Not overlapped. # dwFlagsAndAttributes NULL, # hTemplateFile ) assert input_file > 0, (input_file, _winapi.GetLastError()) # output_path = ctypes.create_unicode_buffer(u'\\\\.\\f:') output_path = ctypes.create_unicode_buffer(u'\\\\.\\PHYSICALDRIVE3') output_file = ctypes.windll.kernel32.CreateFileW( output_path, _generic_rw, FILE_SHARE_WRITE, # dwShareMode NULL, # lpSecurityAttributes OPEN_EXISTING, # dwCreationDisposition _file_flags, # Not overlapped. # dwFlagsAndAttributes NULL, # hTemplateFile ) # assert output_file > 0, (output_file, _winapi.GetLastError()) buf_size = DWORD(1024 * 1024 * 16)
def __init__(self, name: str = '', size: int = 0, readonly: bool = False, mode: int = stat.S_IREAD | stat.S_IWRITE, prefix: str = 'calibre-'): if size < 0: raise TypeError("'size' must be a non-negative integer") if size and name: raise TypeError('Cannot specify both name and size') if not name: flags = os.O_CREAT | os.O_EXCL if not size: raise TypeError("'size' must be > 0") else: flags = 0 flags |= os.O_RDONLY if readonly else os.O_RDWR access = mmap.ACCESS_READ if readonly else mmap.ACCESS_WRITE create = not name tries = 30 while not name and tries > 0: tries -= 1 q = make_filename(prefix) if iswindows: h_map = _winapi.CreateFileMapping( _winapi.INVALID_HANDLE_VALUE, _winapi.NULL, _winapi.PAGE_READONLY if readonly else _winapi.PAGE_READWRITE, (size >> 32) & 0xFFFFFFFF, size & 0xFFFFFFFF, q) try: last_error_code = _winapi.GetLastError() if last_error_code == _winapi.ERROR_ALREADY_EXISTS: continue self._mmap = mmap.mmap(-1, size, tagname=q, access=access) name = q finally: _winapi.CloseHandle(h_map) else: try: self._fd = _posixshmem.shm_open(q, flags, mode=mode) name = q except FileExistsError: continue if tries <= 0: raise OSError( f'Failed to create a uniquely named SHM file, try shortening the prefix from: {prefix}' ) self._name = name if not create and iswindows: h_map = _winapi.OpenFileMapping(_winapi.FILE_MAP_READ, False, name) try: p_buf = _winapi.MapViewOfFile(h_map, _winapi.FILE_MAP_READ, 0, 0, 0) finally: _winapi.CloseHandle(h_map) size = _winapi.VirtualQuerySize(p_buf) self._mmap = mmap.mmap(-1, size, tagname=name) if not iswindows: if not create: self._fd = _posixshmem.shm_open(name, flags, mode) try: if flags & os.O_CREAT and size: os.ftruncate(self._fd, size) self.stats = os.fstat(self._fd) size = self.stats.st_size self._mmap = mmap.mmap(self._fd, size, access=access) except OSError: self.unlink() raise self._size = size