def __init__(self, name: str, timeout: Optional[float] = None) -> None:
if sys.platform == 'win32':
name = r'\\.\pipe\{}-{}.pipe'.format(
name,
base64.urlsafe_b64encode(os.urandom(6)).decode())
else:
name = f'{name}.sock'
super().__init__(name, timeout)
if sys.platform == 'win32':
self.connection = _winapi.CreateNamedPipe(
self.name,
_winapi.PIPE_ACCESS_DUPLEX
| _winapi.FILE_FLAG_FIRST_PIPE_INSTANCE
| _winapi.FILE_FLAG_OVERLAPPED,
_winapi.PIPE_READMODE_MESSAGE
| _winapi.PIPE_TYPE_MESSAGE
| _winapi.PIPE_WAIT
| 0x8, # PIPE_REJECT_REMOTE_CLIENTS
1, # one instance
self.BUFFER_SIZE,
self.BUFFER_SIZE,
_winapi.NMPWAIT_WAIT_FOREVER,
0, # Use default security descriptor
)
if self.connection == -1: # INVALID_HANDLE_VALUE
err = _winapi.GetLastError()
raise IPCException(f'Invalid handle to pipe: {err}')
else:
self.sock_directory = tempfile.mkdtemp()
sockfile = os.path.join(self.sock_directory, self.name)
self.sock = socket.socket(socket.AF_UNIX)
self.sock.bind(sockfile)
self.sock.listen(1)
if timeout is not None:
self.sock.settimeout(timeout)
def __init__(self, size):
self.size = size
for i in range(100):
name = 'pym-%d-%s' % (os.getpid(), next(self._rand))
buf = mmap.mmap(-1, size, tagname=name)
if _winapi.GetLastError() == 0:
break
buf.close()
else:
raise FileExistsError('Cannot find name for new mmap')
self.name = name
self.buffer = buf
self._state = self.size, self.name
def cmdl(ctx, components, config, n_environments, exploit, show_ui):
"""Run RELAAX components.
\b
COMPONENTS:
all - run environments and servers (default)
environment - run environment
servers - run rlx-server, parameter-server, metrics-server and wsproxy
(if specified in config yaml)
rlx-server - run rlx-server
parameter-server - run parameter-server
metrics-server - run metrics-server
wsproxy - run websockets proxy
\b
For example:
- run environment, rlx-server, parameter-server, metrics-server, and wsproxy
$relaax run all
- run rlx-server, parameter-server, metrics-server, and wsproxy
$relaax run servers
"""
ctx.setup_logger(format='%(asctime)s %(name)s\t\t | %(message)s')
# Disable TF warnings
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
os.environ["COMSPEC"] = "powershell.exe"
# Execute command
if sys.platform == 'win32':
honcho.manager.KILL_WAIT = 120
honcho.process.Popen = PopenPatched
import _winapi
import ctypes
firstRun = False
mutex = ctypes.windll.kernel32.CreateMutexA(None, False,
"RELAAX_WINDOWS_MUTEX")
if _winapi.GetLastError() == 0:
firstRun = True
if firstRun:
os.system("start powershell " + ' '.join(sys.argv))
time.sleep(5)
_winapi.CloseHandle(mutex)
else:
_winapi.CloseHandle(mutex)
honcho.process.Popen = PopenPatched
CmdlRun(ctx, set(components), config.name, n_environments, exploit,
show_ui).run_components()
else:
CmdlRun(ctx, set(components), config.name, n_environments, exploit,
show_ui).run_components()
def __init__(self, name=None, create=False, size=0):
if not size >= 0:
raise ValueError("'size' must be a positive integer")
if create:
self._flags = _O_CREX | os.O_RDWR
if name is None and not self._flags & os.O_EXCL:
raise ValueError("'name' can only be None if create=True")
# Windows Named Shared Memory
if create:
while True:
temp_name = _make_filename() if name is None else name
# Create and reserve shared memory block with this name
# until it can be attached to by mmap.
h_map = _winapi.CreateFileMapping(
_winapi.INVALID_HANDLE_VALUE, _winapi.NULL,
_winapi.PAGE_READWRITE, (size >> 32) & 0xFFFFFFFF,
size & 0xFFFFFFFF, temp_name)
try:
last_error_code = _winapi.GetLastError()
if last_error_code == _winapi.ERROR_ALREADY_EXISTS:
if name is not None:
raise FileExistsError(
errno.EEXIST,
os.strerror(errno.EEXIST), name,
_winapi.ERROR_ALREADY_EXISTS)
else:
continue
self._mmap = mmap.mmap(-1, size, tagname=temp_name)
finally:
_winapi.CloseHandle(h_map)
self._name = temp_name
break
else:
self._name = name
# Dynamically determine the existing named shared memory
# block's size which is likely a multiple of mmap.PAGESIZE.
h_map = _winapi.OpenFileMapping(_winapi.FILE_MAP_READ,
False, name)
try:
p_buf = _winapi.MapViewOfFile(h_map,
_winapi.FILE_MAP_READ, 0,
0, 0)
finally:
_winapi.CloseHandle(h_map)
size = _winapi.VirtualQuerySize(p_buf)
self._mmap = mmap.mmap(-1, size, tagname=name)
self._size = size
self._buf = memoryview(self._mmap)
def __setstate__(self, state):
self.size, self.name = self._state = state
self.buffer = mmap.mmap(-1, self.size, tagname=self.name)
assert _winapi.GetLastError() == _winapi.ERROR_ALREADY_EXISTS
def __init__(self, name=None, create=False, size=0):
if not size >= 0:
raise ValueError("'size' must be a positive integer")
if create:
self._flags = _O_CREX | os.O_RDWR
if size == 0:
raise ValueError("'size' must be a positive number different from zero")
if name is None and not self._flags & os.O_EXCL:
raise ValueError("'name' can only be None if create=True")
if _USE_POSIX:
# POSIX Shared Memory
if name is None:
while True:
name = _make_filename()
try:
self._fd = _posixshmem.shm_open(
name,
self._flags,
mode=self._mode
)
except FileExistsError:
continue
self._name = name
break
else:
name = "/" + name if self._prepend_leading_slash else name
self._fd = _posixshmem.shm_open(
name,
self._flags,
mode=self._mode
)
self._name = name
try:
if create and size:
os.ftruncate(self._fd, size)
stats = os.fstat(self._fd)
size = stats.st_size
self._mmap = mmap.mmap(self._fd, size)
except OSError:
self.unlink()
raise
from .resource_tracker import register
register(self._name, "shared_memory")
else:
# Windows Named Shared Memory
if create:
while True:
temp_name = _make_filename() if name is None else name
# Create and reserve shared memory block with this name
# until it can be attached to by mmap.
h_map = _winapi.CreateFileMapping(
_winapi.INVALID_HANDLE_VALUE,
_winapi.NULL,
_winapi.PAGE_READWRITE,
(size >> 32) & 0xFFFFFFFF,
size & 0xFFFFFFFF,
temp_name
)
try:
last_error_code = _winapi.GetLastError()
if last_error_code == _winapi.ERROR_ALREADY_EXISTS:
if name is not None:
raise FileExistsError(
errno.EEXIST,
os.strerror(errno.EEXIST),
name,
_winapi.ERROR_ALREADY_EXISTS
)
else:
continue
self._mmap = mmap.mmap(-1, size, tagname=temp_name)
finally:
_winapi.CloseHandle(h_map)
self._name = temp_name
break
else:
self._name = name
# Dynamically determine the existing named shared memory
# block's size which is likely a multiple of mmap.PAGESIZE.
h_map = _winapi.OpenFileMapping(
_winapi.FILE_MAP_READ,
False,
name
)
try:
p_buf = _winapi.MapViewOfFile(
h_map,
_winapi.FILE_MAP_READ,
0,
0,
0
)
finally:
_winapi.CloseHandle(h_map)
size = _winapi.VirtualQuerySize(p_buf)
self._mmap = mmap.mmap(-1, size, tagname=name)
self._size = size
self._buf = memoryview(self._mmap)
def __init__(self, map_id, size_nbytes):
"""
Create a segment of shared memory with an automatically generated name.
Or open an already created segment with the provided name.
:param map_id: The name of the shared memory. Should be None if a new segment is to be created.
:param size_nbytes: The size, in bytes, of the segment.
"""
# This flag is used to signal if the shared memory has been unlinked by the owning process.
self._flag = None
self._closed = False
try:
# We actually need one extra byte for the signal flag.
alloc_nbytes = size_nbytes + 1
# The master process is the process that created the memory.
# The master handles the lifetime of the memory, and unlinks it when it is no longer needed.
master = (map_id is None)
if _USE_INTERNAL:
if master:
self._raw_mem = multiprocessing.shared_memory.SharedMemory(
create=True, size=alloc_nbytes)
map_id = self._raw_mem.name
else:
self._raw_mem = multiprocessing.shared_memory.SharedMemory(
name=map_id, size=alloc_nbytes)
def unlink():
self._raw_mem.unlink()
def close():
self._raw_mem.close()
self._buf = self._raw_mem.buf
def release_buf():
pass # Releasing the buffer is handled by multiprocessing.
else:
if _USE_POSIX:
if master:
#master = True
while True:
# Generate a new ID.
map_id = _make_map_id()
# Attempt to create the buffer with this ID, this command will fail if a buffer already exists with that name.
try:
self._fd = _shm_open(map_id,
os.O_CREAT | os.O_EXCL
| os.O_RDWR,
mode=0o600)
except OSError as e:
# If a buffer with that name already exists, try again.
if e.errno == errno.EEXIST:
continue
else:
raise
break
else:
self._fd = _shm_open(map_id, os.O_RDWR, mode=0o600)
def unlink():
if self._fd is not None:
_shm_unlink(map_id)
try:
if master:
# If the segment has just been created, resize it to the appropriate size.
os.ftruncate(self._fd, alloc_nbytes)
# Now map it into memory.
self._mmap = mmap.mmap(self._fd, alloc_nbytes)
except OSError:
unlink()
raise
def close():
if self._mmap is not None:
self._mmap.close()
self._mmap = None
if self._fd is not None:
os.close(self._fd)
self._fd = None
else: ## WINDOWS
if master:
while True:
map_id = _make_map_id()
# It might be better here to open a mmap with length 0, catch the exception, then reopen it with the correct size
self._mmap = mmap.mmap(
-1,
alloc_nbytes,
tagname=map_id.decode("utf-8"))
if _winapi.GetLastError() == 0:
# If mapping the segment suceeded, then stop now, otherwise try again.
break
self._mmap.close()
else:
self._mmap = mmap.mmap(-1,
alloc_nbytes,
tagname=map_id.decode("utf-8"))
def unlink():
pass
def close():
self._mmap.close()
if _PYTHON3:
self._buf = memoryview(self._mmap)
else:
# Python 2.7 requires passing the mapped memory through ctypes first.
self._buf = memoryview(
(ctypes.c_byte * alloc_nbytes).from_buffer(self._mmap))
def release_buf():
if self._buf is not None:
if _PYTHON3:
self._buf.release()
self._buf = None
# The flag is the first byte of the memory
self._flag = self._buf[:1]
# The actual exposed buffer is the rest of the memory.
self._ary = self._buf[1:]
def release():
# Release these pointers when the buffer is closed.
if self._flag is not None:
if _PYTHON3:
self._flag.release()
self._flag = None
if self._ary is not None:
if _PYTHON3:
self._ary.release()
self._ary = None
release_buf()
self._flag[:] = b'\x00' # Flag set to initially available.
self.name = map_id
self.size_nbytes = size_nbytes
self._lock = threading.RLock()
except:
# If something went wrong, make sure these functions are defined.
if 'release' not in locals():
def release():
pass
if 'close' not in locals():
def close():
pass
if 'unlink' not in locals():
def unlink():
pass
raise
finally:
def cleanup():
self._cleanup = lambda: None
if master and self._flag is not None:
self._flag[:] = b'\x01' # Set the flag to signal this buffer has been unlinked.
release()
close()
if master:
# Unlinking has to happen last, but the flag needs to be set before the pointers are released.
unlink()
self._cleanup = cleanup
def __init__(self, size):
self.size = size
self.name = 'pym-%d-%d' % (os.getpid(), next(Arena._counter))
self.buffer = mmap.mmap(-1, self.size, tagname=self.name)
assert _winapi.GetLastError() == 0, 'tagname already in use'
self._state = (self.size, self.name)
NULL = DWORD(0)
OPEN_EXISTING = DWORD(3)
FILE_SHARE_WRITE = DWORD(2)
input_path = ctypes.create_unicode_buffer(
u'D:\\Downloads\\linuxmint-19-cinnamon-64bit-v2.iso')
input_file = ctypes.windll.kernel32.CreateFileW(
input_path,
GENERIC_READ,
DWORD(0), # dwShareMode
NULL, # lpSecurityAttributes
OPEN_EXISTING, # dwCreationDisposition
_file_flags, # Not overlapped. # dwFlagsAndAttributes
NULL, # hTemplateFile
)
assert input_file > 0, (input_file, _winapi.GetLastError())
# output_path = ctypes.create_unicode_buffer(u'\\\\.\\f:')
output_path = ctypes.create_unicode_buffer(u'\\\\.\\PHYSICALDRIVE3')
output_file = ctypes.windll.kernel32.CreateFileW(
output_path,
_generic_rw,
FILE_SHARE_WRITE, # dwShareMode
NULL, # lpSecurityAttributes
OPEN_EXISTING, # dwCreationDisposition
_file_flags, # Not overlapped. # dwFlagsAndAttributes
NULL, # hTemplateFile
)
# assert output_file > 0, (output_file, _winapi.GetLastError())
buf_size = DWORD(1024 * 1024 * 16)
def __init__(self,
name: str = '',
size: int = 0,
readonly: bool = False,
mode: int = stat.S_IREAD | stat.S_IWRITE,
prefix: str = 'calibre-'):
if size < 0:
raise TypeError("'size' must be a non-negative integer")
if size and name:
raise TypeError('Cannot specify both name and size')
if not name:
flags = os.O_CREAT | os.O_EXCL
if not size:
raise TypeError("'size' must be > 0")
else:
flags = 0
flags |= os.O_RDONLY if readonly else os.O_RDWR
access = mmap.ACCESS_READ if readonly else mmap.ACCESS_WRITE
create = not name
tries = 30
while not name and tries > 0:
tries -= 1
q = make_filename(prefix)
if iswindows:
h_map = _winapi.CreateFileMapping(
_winapi.INVALID_HANDLE_VALUE, _winapi.NULL,
_winapi.PAGE_READONLY
if readonly else _winapi.PAGE_READWRITE,
(size >> 32) & 0xFFFFFFFF, size & 0xFFFFFFFF, q)
try:
last_error_code = _winapi.GetLastError()
if last_error_code == _winapi.ERROR_ALREADY_EXISTS:
continue
self._mmap = mmap.mmap(-1, size, tagname=q, access=access)
name = q
finally:
_winapi.CloseHandle(h_map)
else:
try:
self._fd = _posixshmem.shm_open(q, flags, mode=mode)
name = q
except FileExistsError:
continue
if tries <= 0:
raise OSError(
f'Failed to create a uniquely named SHM file, try shortening the prefix from: {prefix}'
)
self._name = name
if not create and iswindows:
h_map = _winapi.OpenFileMapping(_winapi.FILE_MAP_READ, False, name)
try:
p_buf = _winapi.MapViewOfFile(h_map, _winapi.FILE_MAP_READ, 0,
0, 0)
finally:
_winapi.CloseHandle(h_map)
size = _winapi.VirtualQuerySize(p_buf)
self._mmap = mmap.mmap(-1, size, tagname=name)
if not iswindows:
if not create:
self._fd = _posixshmem.shm_open(name, flags, mode)
try:
if flags & os.O_CREAT and size:
os.ftruncate(self._fd, size)
self.stats = os.fstat(self._fd)
size = self.stats.st_size
self._mmap = mmap.mmap(self._fd, size, access=access)
except OSError:
self.unlink()
raise
self._size = size
