class SharedMemoryFrameWriter:
def __init__(self, name):
self.shm_region = None
logging.info("Writer launched")
self.md_region = SharedMemory(name + '-meta', O_CREAT, size=sizeof(MD))
self.md_buf = mmap.mmap(self.md_region.fd, self.md_region.size)
self.md_region.close_fd()
self.shm_buf = None
self.shm_name = name
self.count = 0
try:
self.sem = Semaphore(name, O_CREX)
except ExistentialError:
sem = Semaphore(name, O_CREAT)
sem.unlink()
self.sem = Semaphore(name, O_CREX)
self.sem.release()
def add(self, frame: np.ndarray):
byte_size = frame.nbytes
if not self.shm_region:
self.shm_region = SharedMemory(self.shm_name,
O_CREAT,
size=byte_size)
self.shm_buf = mmap.mmap(self.shm_region.fd, byte_size)
self.shm_region.close_fd()
self.count += 1
md = MD(frame.shape[0], frame.shape[1], frame.shape[2], byte_size,
self.count)
self.sem.acquire()
memmove(md_buf, addressof(md), sizeof(md))
self.md_buf[:] = bytes(md_buf)
self.shm_buf[:] = frame.tobytes()
self.sem.release()
def release(self):
self.sem.acquire()
self.md_buf.close()
unlink_shared_memory(self.shm_name + '-meta')
self.shm_buf.close()
unlink_shared_memory(self.shm_name)
self.sem.release()
self.sem.close()
logging.info("Writer terminated")
class ShmRead:
def __init__(self, name):
self.shm_buf = None
self.md_buf = None
while not self.md_buf:
try:
# print("Waiting for MetaData shared memory is available.")
md_region = SharedMemory(name + "-meta")
self.md_buf = mmap.mmap(md_region.fd, sizeof(MD))
md_region.close_fd()
# sleep(1)
except ExistentialError:
pass
# sleep(1)
self.shm_name = name
self.sem = Semaphore(name, 0)
def get(self):
md = MD()
self.sem.acquire()
md_buf[:] = self.md_buf
memmove(addressof(md), md_buf, sizeof(md))
self.sem.release()
while not self.shm_buf:
try:
# print("Waiting for Data shared memory is available.")
shm_region = SharedMemory(name=self.shm_name)
self.shm_buf = mmap.mmap(shm_region.fd, md.size)
shm_region.close_fd()
# sleep(1)
except ExistentialError:
pass
# sleep(1)
self.sem.acquire()
f = np.ndarray(
shape=(md.shape_0, md.shape_1, md.shape_2),
dtype="uint8",
buffer=self.shm_buf,
)
self.sem.release()
return f
def release(self):
self.md_buf.close()
self.shm_buf.close()
class SharedMemoryFrameReader:
def __init__(self, name):
self.shm_buf = None
self.md_buf = None
logging.info("Reader launched")
while not self.md_buf:
try:
logging.warning(
"Waiting for MetaData shared memory is available.")
md_region = SharedMemory(name + '-meta')
self.md_buf = mmap.mmap(md_region.fd, sizeof(MD))
md_region.close_fd()
sleep(1)
except ExistentialError:
sleep(1)
self.shm_name = name
self.sem = Semaphore(name, 0)
def get(self):
md = MD()
self.sem.acquire()
md_buf[:] = self.md_buf
memmove(addressof(md), md_buf, sizeof(md))
self.sem.release()
while not self.shm_buf:
try:
logging.warning("Waiting for Data shared memory is available.")
shm_region = SharedMemory(name=self.shm_name)
self.shm_buf = mmap.mmap(shm_region.fd, md.size)
shm_region.close_fd()
sleep(1)
except ExistentialError:
sleep(1)
self.sem.acquire()
f = np.ndarray(shape=(md.shape_0, md.shape_1, md.shape_2),
dtype='uint8',
buffer=self.shm_buf)
self.sem.release()
return f
def release(self):
self.md_buf.close()
self.shm_buf.close()
logging.info("Reader terminated")
class Wire_Read:
def __init__(self, name):
self.shm_buf = None
self.md_buf = None
while not self.md_buf:
try:
print("Waiting for Block Input...")
md_region = SharedMemory(name + '-meta')
self.md_buf = mmap.mmap(md_region.fd, sizeof(MD))
md_region.close_fd()
sleep(1)
except ExistentialError:
sleep(1)
self.shm_name = name
self.sem = Semaphore(name, 0)
def get(self):
md = MD()
self.sem.acquire()
md_buf[:] = self.md_buf
memmove(addressof(md), md_buf, sizeof(md))
self.sem.release()
while not self.shm_buf:
try:
print("Waiting for Data...")
shm_region = SharedMemory(name=self.shm_name)
self.shm_buf = mmap.mmap(shm_region.fd, md.size)
shm_region.close_fd()
sleep(1)
except ExistentialError:
sleep(1)
self.sem.acquire()
f = np.ndarray(shape=(md.shape_0, md.shape_1, md.shape_2),
dtype='uint8',
buffer=self.shm_buf)
self.sem.release()
return f
def release(self):
self.md_buf.close()
self.shm_buf.close()
class ValueEvent(object):
"""Provides behavior similar to threading.Event. However, this
allows associating a value when "setting" (or notifying) the Event object.
The ValueEvent object is used to communicate between two threads when one
thread needs a value that is computed by another. The reading thread waits
on the value (but does not set it), and the writing thread sets the value
then triggers the event. In other words, there is exactly one write to the
value from one thread, and one read from a different thread, and the
ValueEvent object guarantees they don't overlap so long as the get and set
methods are used by the reading and writing thread, respectively.
Additionally, Python's threading.Event object provides a timeout feature
like ours but introduces unacceptable delays. For that reason we use
posix_ipc.Semaphore to mimic threading.Event without the performance
penalty.
http://stackoverflow.com/questions/21779183/python-eventwait-with-timeout-gives-delay
"""
def __init__(self, name=None):
self._value = None
self._semaphore = Semaphore(name, flags=O_CREX)
def __del__(self):
# If unlink isn't explicitly called the OS will *not* release the
# semaphore object, even if the program crashes. We may want to spawn
# a new process to manage them or give the semaphores known names when
# creating them so they can be reclaimed on restart.
self._semaphore.unlink()
def get(self, timeout):
try:
self._semaphore.acquire(timeout)
except BusyError:
raise ValueEventTimeout(timeout)
return self._value
def set(self, value):
self._value = value
self._semaphore.release()
def pi(n):
pids = []
unit = n / 10
sem_lock = Semaphore('/pi_sem_lock', flags=posix_ipc.O_CREX, initial_value=1)
memory = Memory('/pi_rw', size=8, flags=posix_ipc.O_CREX)
os.lseek(memory.fd, 0, os.SEEK_SET)
os.write(memory.fd, struct.pack('d', 0.0))
for i in range(10):
mink = unit * i
maxk = mink + unit
pid = os.fork()
if pid > 0:
pids.append(pid)
else:
s = calc_slice(mink, maxk)
sem_lock.acquire()
try:
os.lseek(memory.fd, 0, os.SEEK_SET)
bs = os.read(memory.fd, 8)
cur_val, = struct.unpack('d', bs)
cur_val += s
bs = struct.pack('d', cur_val) # 序列化
os.lseek(memory.fd, 0, os.SEEK_SET)
os.write(memory.fd, bs)
memory.close_fd()
finally:
sem_lock.release()
sys.exit(0)
sums = []
for pid in pids:
os.waitpid(pid, 0)
os.lseek(memory.fd, 0, os.SEEK_SET)
bs = os.read(memory.fd, 8)
sums, = struct.unpack('d', bs)
memory.close_fd()
memory.unlink()
sem_lock.unlink()
return math.sqrt(sums * 8)
class ValueEvent(object):
"""Provides behavior similar to threading.Event. However, this
allows associating a value when "setting" (or notifying) the Event object.
The ValueEvent object is used to communicate between two threads when one
thread needs a value that is computed by another. The reading thread waits
on the value (but does not set it), and the writing thread sets the value
then triggers the event. In other words, there is exactly one write to the
value from one thread, and one read from a different thread, and the
ValueEvent object guarantees they don't overlap so long as the get and set
methods are used by the reading and writing thread, respectively.
Additionally, Python's threading.Event object provides a timeout feature
like ours but introduces unacceptable delays. For that reason we use
posix_ipc.Semaphore to mimic threading.Event without the performance
penalty.
http://stackoverflow.com/questions/21779183/python-eventwait-with-timeout-gives-delay
"""
def __init__(self, name=None):
self._value = None
self._semaphore = Semaphore(name, flags=O_CREX)
def __del__(self):
# If unlink isn't explicitly called the OS will *not* release the
# semaphore object, even if the program crashes. We may want to spawn
# a new process to manage them or give the semaphores known names when
# creating them so they can be reclaimed on restart.
self._semaphore.unlink()
def get(self, timeout):
try:
self._semaphore.acquire(timeout)
except BusyError:
raise ValueEventTimeout(timeout)
return self._value
def set(self, value):
self._value = value
self._semaphore.release()
class SharedValue:
def __init__(self, name):
# Initialize varaibles for memory regions and buffers and Semaphore
self.shm_buf = None
self.shm_region = None
self.value_lock = None
self.shm_name = name
self.value_lock_name = name
# Initialize shared memory buffer
try:
self.shm_region = SharedMemory(self.shm_name)
self.shm_buf = mmap.mmap(self.shm_region.fd, sizeof(c_float))
self.shm_region.close_fd()
except ExistentialError:
self.shm_region = SharedMemory(self.shm_name,
O_CREAT,
size=sizeof(c_float))
self.shm_buf = mmap.mmap(self.shm_region.fd, self.shm_region.size)
self.shm_region.close_fd()
# Initialize or retreive Semaphore
try:
self.value_lock = Semaphore(self.value_lock_name, O_CREX)
except ExistentialError:
value_lock = Semaphore(self.value_lock_name, O_CREAT)
value_lock.unlink()
self.value_lock = Semaphore(self.value_lock_name, O_CREX)
self.value_lock.release()
# Get the shared value
def get(self):
# Retreive the data from buffer
self.value_lock.acquire()
value = struct.unpack('f', self.shm_buf)[0]
self.value_lock.release()
return value
# Add the shared value
def add(self, value):
# Send the data to shared regions
self.value_lock.acquire()
self.shm_buf[:] = struct.pack('f', value)
self.value_lock.release()
# Destructor function to unlink and disconnect
def close(self):
self.value_lock.acquire()
self.shm_buf.close()
try:
unlink_shared_memory(self.shm_name)
except ExistentialError:
pass
self.value_lock.release()
self.value_lock.close()
from z3 import *
from ast import *
from utils import *
from posix_ipc import Semaphore, O_CREAT
import os
import sys
import atexit
N_CPUS = 4
## Use semaphore to limit the number of concurrent processes,
## we allow N_CPUS processes running simultaneously.
sem = Semaphore("/fork_sem", O_CREAT, 0o644, N_CPUS)
sem.unlink()
sem.acquire()
def on_exit():
sem.release()
## reap all zombies.
try:
while True:
os.waitpid(0)
except:
pass
log("exit")
atexit.register(on_exit)
class SharedImage:
def __init__(self, name):
# Initialize variables for memory regions and buffers and Semaphore
self.shm_buf = None
self.shm_region = None
self.md_buf = None
self.md_region = None
self.image_lock = None
self.shm_name = name
self.md_name = name + "-meta"
self.image_lock_name = name
# Initialize or retreive metadata memory region
try:
self.md_region = SharedMemory(self.md_name)
self.md_buf = mmap.mmap(self.md_region.fd, sizeof(MD))
self.md_region.close_fd()
except ExistentialError:
self.md_region = SharedMemory(self.md_name,
O_CREAT,
size=sizeof(MD))
self.md_buf = mmap.mmap(self.md_region.fd, self.md_region.size)
self.md_region.close_fd()
# Initialize or retreive Semaphore
try:
self.image_lock = Semaphore(self.image_lock_name, O_CREX)
except ExistentialError:
image_lock = Semaphore(self.image_lock_name, O_CREAT)
image_lock.unlink()
self.image_lock = Semaphore(self.image_lock_name, O_CREX)
self.image_lock.release()
# Get the shared image
def get(self):
# Define metadata
metadata = MD()
# Get metadata from the shared region
self.image_lock.acquire()
md_buf[:] = self.md_buf
memmove(addressof(metadata), md_buf, sizeof(metadata))
self.image_lock.release()
# Try to retreive the image from shm_buffer
# Otherwise return a zero image
try:
self.shm_region = SharedMemory(self.shm_name)
self.shm_buf = mmap.mmap(self.shm_region.fd, metadata.size)
self.shm_region.close_fd()
self.image_lock.acquire()
image = np.ndarray(shape=(metadata.shape_0, metadata.shape_1,
metadata.shape_2),
dtype='uint8',
buffer=self.shm_buf)
self.image_lock.release()
# Check for a None image
if (image.size == 0):
image = np.zeros((3, 3, 3), np.uint8)
except ExistentialError:
image = np.zeros((3, 3, 3), np.uint8)
return image
# Add the shared image
def add(self, image):
# Get byte size of the image
byte_size = image.nbytes
# Get the shared memory buffer to read from
if not self.shm_region:
self.shm_region = SharedMemory(self.shm_name,
O_CREAT,
size=byte_size)
self.shm_buf = mmap.mmap(self.shm_region.fd, byte_size)
self.shm_region.close_fd()
# Generate meta data
metadata = MD(image.shape[0], image.shape[1], image.shape[2],
byte_size)
# Send the meta data and image to shared regions
self.image_lock.acquire()
memmove(md_buf, addressof(metadata), sizeof(metadata))
self.md_buf[:] = md_buf[:]
self.shm_buf[:] = image.tobytes()
self.image_lock.release()
# Destructor function to unlink and disconnect
def close(self):
self.image_lock.acquire()
self.md_buf.close()
try:
unlink_shared_memory(self.md_name)
unlink_shared_memory(self.shm_name)
except ExistentialError:
pass
self.image_lock.release()
self.image_lock.close()
