class SharedMemoryFrameWriter: def __init__(self, name): self.shm_region = None logging.info("Writer launched") self.md_region = SharedMemory(name + '-meta', O_CREAT, size=sizeof(MD)) self.md_buf = mmap.mmap(self.md_region.fd, self.md_region.size) self.md_region.close_fd() self.shm_buf = None self.shm_name = name self.count = 0 try: self.sem = Semaphore(name, O_CREX) except ExistentialError: sem = Semaphore(name, O_CREAT) sem.unlink() self.sem = Semaphore(name, O_CREX) self.sem.release() def add(self, frame: np.ndarray): byte_size = frame.nbytes if not self.shm_region: self.shm_region = SharedMemory(self.shm_name, O_CREAT, size=byte_size) self.shm_buf = mmap.mmap(self.shm_region.fd, byte_size) self.shm_region.close_fd() self.count += 1 md = MD(frame.shape[0], frame.shape[1], frame.shape[2], byte_size, self.count) self.sem.acquire() memmove(md_buf, addressof(md), sizeof(md)) self.md_buf[:] = bytes(md_buf) self.shm_buf[:] = frame.tobytes() self.sem.release() def release(self): self.sem.acquire() self.md_buf.close() unlink_shared_memory(self.shm_name + '-meta') self.shm_buf.close() unlink_shared_memory(self.shm_name) self.sem.release() self.sem.close() logging.info("Writer terminated")
class ShmRead: def __init__(self, name): self.shm_buf = None self.md_buf = None while not self.md_buf: try: # print("Waiting for MetaData shared memory is available.") md_region = SharedMemory(name + "-meta") self.md_buf = mmap.mmap(md_region.fd, sizeof(MD)) md_region.close_fd() # sleep(1) except ExistentialError: pass # sleep(1) self.shm_name = name self.sem = Semaphore(name, 0) def get(self): md = MD() self.sem.acquire() md_buf[:] = self.md_buf memmove(addressof(md), md_buf, sizeof(md)) self.sem.release() while not self.shm_buf: try: # print("Waiting for Data shared memory is available.") shm_region = SharedMemory(name=self.shm_name) self.shm_buf = mmap.mmap(shm_region.fd, md.size) shm_region.close_fd() # sleep(1) except ExistentialError: pass # sleep(1) self.sem.acquire() f = np.ndarray( shape=(md.shape_0, md.shape_1, md.shape_2), dtype="uint8", buffer=self.shm_buf, ) self.sem.release() return f def release(self): self.md_buf.close() self.shm_buf.close()
class SharedMemoryFrameReader: def __init__(self, name): self.shm_buf = None self.md_buf = None logging.info("Reader launched") while not self.md_buf: try: logging.warning( "Waiting for MetaData shared memory is available.") md_region = SharedMemory(name + '-meta') self.md_buf = mmap.mmap(md_region.fd, sizeof(MD)) md_region.close_fd() sleep(1) except ExistentialError: sleep(1) self.shm_name = name self.sem = Semaphore(name, 0) def get(self): md = MD() self.sem.acquire() md_buf[:] = self.md_buf memmove(addressof(md), md_buf, sizeof(md)) self.sem.release() while not self.shm_buf: try: logging.warning("Waiting for Data shared memory is available.") shm_region = SharedMemory(name=self.shm_name) self.shm_buf = mmap.mmap(shm_region.fd, md.size) shm_region.close_fd() sleep(1) except ExistentialError: sleep(1) self.sem.acquire() f = np.ndarray(shape=(md.shape_0, md.shape_1, md.shape_2), dtype='uint8', buffer=self.shm_buf) self.sem.release() return f def release(self): self.md_buf.close() self.shm_buf.close() logging.info("Reader terminated")
class Wire_Read: def __init__(self, name): self.shm_buf = None self.md_buf = None while not self.md_buf: try: print("Waiting for Block Input...") md_region = SharedMemory(name + '-meta') self.md_buf = mmap.mmap(md_region.fd, sizeof(MD)) md_region.close_fd() sleep(1) except ExistentialError: sleep(1) self.shm_name = name self.sem = Semaphore(name, 0) def get(self): md = MD() self.sem.acquire() md_buf[:] = self.md_buf memmove(addressof(md), md_buf, sizeof(md)) self.sem.release() while not self.shm_buf: try: print("Waiting for Data...") shm_region = SharedMemory(name=self.shm_name) self.shm_buf = mmap.mmap(shm_region.fd, md.size) shm_region.close_fd() sleep(1) except ExistentialError: sleep(1) self.sem.acquire() f = np.ndarray(shape=(md.shape_0, md.shape_1, md.shape_2), dtype='uint8', buffer=self.shm_buf) self.sem.release() return f def release(self): self.md_buf.close() self.shm_buf.close()
class ValueEvent(object): """Provides behavior similar to threading.Event. However, this allows associating a value when "setting" (or notifying) the Event object. The ValueEvent object is used to communicate between two threads when one thread needs a value that is computed by another. The reading thread waits on the value (but does not set it), and the writing thread sets the value then triggers the event. In other words, there is exactly one write to the value from one thread, and one read from a different thread, and the ValueEvent object guarantees they don't overlap so long as the get and set methods are used by the reading and writing thread, respectively. Additionally, Python's threading.Event object provides a timeout feature like ours but introduces unacceptable delays. For that reason we use posix_ipc.Semaphore to mimic threading.Event without the performance penalty. http://stackoverflow.com/questions/21779183/python-eventwait-with-timeout-gives-delay """ def __init__(self, name=None): self._value = None self._semaphore = Semaphore(name, flags=O_CREX) def __del__(self): # If unlink isn't explicitly called the OS will *not* release the # semaphore object, even if the program crashes. We may want to spawn # a new process to manage them or give the semaphores known names when # creating them so they can be reclaimed on restart. self._semaphore.unlink() def get(self, timeout): try: self._semaphore.acquire(timeout) except BusyError: raise ValueEventTimeout(timeout) return self._value def set(self, value): self._value = value self._semaphore.release()
def pi(n): pids = [] unit = n / 10 sem_lock = Semaphore('/pi_sem_lock', flags=posix_ipc.O_CREX, initial_value=1) memory = Memory('/pi_rw', size=8, flags=posix_ipc.O_CREX) os.lseek(memory.fd, 0, os.SEEK_SET) os.write(memory.fd, struct.pack('d', 0.0)) for i in range(10): mink = unit * i maxk = mink + unit pid = os.fork() if pid > 0: pids.append(pid) else: s = calc_slice(mink, maxk) sem_lock.acquire() try: os.lseek(memory.fd, 0, os.SEEK_SET) bs = os.read(memory.fd, 8) cur_val, = struct.unpack('d', bs) cur_val += s bs = struct.pack('d', cur_val) # 序列化 os.lseek(memory.fd, 0, os.SEEK_SET) os.write(memory.fd, bs) memory.close_fd() finally: sem_lock.release() sys.exit(0) sums = [] for pid in pids: os.waitpid(pid, 0) os.lseek(memory.fd, 0, os.SEEK_SET) bs = os.read(memory.fd, 8) sums, = struct.unpack('d', bs) memory.close_fd() memory.unlink() sem_lock.unlink() return math.sqrt(sums * 8)
class ValueEvent(object): """Provides behavior similar to threading.Event. However, this allows associating a value when "setting" (or notifying) the Event object. The ValueEvent object is used to communicate between two threads when one thread needs a value that is computed by another. The reading thread waits on the value (but does not set it), and the writing thread sets the value then triggers the event. In other words, there is exactly one write to the value from one thread, and one read from a different thread, and the ValueEvent object guarantees they don't overlap so long as the get and set methods are used by the reading and writing thread, respectively. Additionally, Python's threading.Event object provides a timeout feature like ours but introduces unacceptable delays. For that reason we use posix_ipc.Semaphore to mimic threading.Event without the performance penalty. http://stackoverflow.com/questions/21779183/python-eventwait-with-timeout-gives-delay """ def __init__(self, name=None): self._value = None self._semaphore = Semaphore(name, flags=O_CREX) def __del__(self): # If unlink isn't explicitly called the OS will *not* release the # semaphore object, even if the program crashes. We may want to spawn # a new process to manage them or give the semaphores known names when # creating them so they can be reclaimed on restart. self._semaphore.unlink() def get(self, timeout): try: self._semaphore.acquire(timeout) except BusyError: raise ValueEventTimeout(timeout) return self._value def set(self, value): self._value = value self._semaphore.release()
class SharedValue: def __init__(self, name): # Initialize varaibles for memory regions and buffers and Semaphore self.shm_buf = None self.shm_region = None self.value_lock = None self.shm_name = name self.value_lock_name = name # Initialize shared memory buffer try: self.shm_region = SharedMemory(self.shm_name) self.shm_buf = mmap.mmap(self.shm_region.fd, sizeof(c_float)) self.shm_region.close_fd() except ExistentialError: self.shm_region = SharedMemory(self.shm_name, O_CREAT, size=sizeof(c_float)) self.shm_buf = mmap.mmap(self.shm_region.fd, self.shm_region.size) self.shm_region.close_fd() # Initialize or retreive Semaphore try: self.value_lock = Semaphore(self.value_lock_name, O_CREX) except ExistentialError: value_lock = Semaphore(self.value_lock_name, O_CREAT) value_lock.unlink() self.value_lock = Semaphore(self.value_lock_name, O_CREX) self.value_lock.release() # Get the shared value def get(self): # Retreive the data from buffer self.value_lock.acquire() value = struct.unpack('f', self.shm_buf)[0] self.value_lock.release() return value # Add the shared value def add(self, value): # Send the data to shared regions self.value_lock.acquire() self.shm_buf[:] = struct.pack('f', value) self.value_lock.release() # Destructor function to unlink and disconnect def close(self): self.value_lock.acquire() self.shm_buf.close() try: unlink_shared_memory(self.shm_name) except ExistentialError: pass self.value_lock.release() self.value_lock.close()
from z3 import * from ast import * from utils import * from posix_ipc import Semaphore, O_CREAT import os import sys import atexit N_CPUS = 4 ## Use semaphore to limit the number of concurrent processes, ## we allow N_CPUS processes running simultaneously. sem = Semaphore("/fork_sem", O_CREAT, 0o644, N_CPUS) sem.unlink() sem.acquire() def on_exit(): sem.release() ## reap all zombies. try: while True: os.waitpid(0) except: pass log("exit") atexit.register(on_exit)
class SharedImage: def __init__(self, name): # Initialize variables for memory regions and buffers and Semaphore self.shm_buf = None self.shm_region = None self.md_buf = None self.md_region = None self.image_lock = None self.shm_name = name self.md_name = name + "-meta" self.image_lock_name = name # Initialize or retreive metadata memory region try: self.md_region = SharedMemory(self.md_name) self.md_buf = mmap.mmap(self.md_region.fd, sizeof(MD)) self.md_region.close_fd() except ExistentialError: self.md_region = SharedMemory(self.md_name, O_CREAT, size=sizeof(MD)) self.md_buf = mmap.mmap(self.md_region.fd, self.md_region.size) self.md_region.close_fd() # Initialize or retreive Semaphore try: self.image_lock = Semaphore(self.image_lock_name, O_CREX) except ExistentialError: image_lock = Semaphore(self.image_lock_name, O_CREAT) image_lock.unlink() self.image_lock = Semaphore(self.image_lock_name, O_CREX) self.image_lock.release() # Get the shared image def get(self): # Define metadata metadata = MD() # Get metadata from the shared region self.image_lock.acquire() md_buf[:] = self.md_buf memmove(addressof(metadata), md_buf, sizeof(metadata)) self.image_lock.release() # Try to retreive the image from shm_buffer # Otherwise return a zero image try: self.shm_region = SharedMemory(self.shm_name) self.shm_buf = mmap.mmap(self.shm_region.fd, metadata.size) self.shm_region.close_fd() self.image_lock.acquire() image = np.ndarray(shape=(metadata.shape_0, metadata.shape_1, metadata.shape_2), dtype='uint8', buffer=self.shm_buf) self.image_lock.release() # Check for a None image if (image.size == 0): image = np.zeros((3, 3, 3), np.uint8) except ExistentialError: image = np.zeros((3, 3, 3), np.uint8) return image # Add the shared image def add(self, image): # Get byte size of the image byte_size = image.nbytes # Get the shared memory buffer to read from if not self.shm_region: self.shm_region = SharedMemory(self.shm_name, O_CREAT, size=byte_size) self.shm_buf = mmap.mmap(self.shm_region.fd, byte_size) self.shm_region.close_fd() # Generate meta data metadata = MD(image.shape[0], image.shape[1], image.shape[2], byte_size) # Send the meta data and image to shared regions self.image_lock.acquire() memmove(md_buf, addressof(metadata), sizeof(metadata)) self.md_buf[:] = md_buf[:] self.shm_buf[:] = image.tobytes() self.image_lock.release() # Destructor function to unlink and disconnect def close(self): self.image_lock.acquire() self.md_buf.close() try: unlink_shared_memory(self.md_name) unlink_shared_memory(self.shm_name) except ExistentialError: pass self.image_lock.release() self.image_lock.close()