def __init__(self, name): # Initialize variables for memory regions and buffers and Semaphore self.shm_buf = None self.shm_region = None self.md_buf = None self.md_region = None self.image_lock = None self.shm_name = name self.md_name = name + "-meta" self.image_lock_name = name # Initialize or retreive metadata memory region try: self.md_region = SharedMemory(self.md_name) self.md_buf = mmap.mmap(self.md_region.fd, sizeof(MD)) self.md_region.close_fd() except ExistentialError: self.md_region = SharedMemory(self.md_name, O_CREAT, size=sizeof(MD)) self.md_buf = mmap.mmap(self.md_region.fd, self.md_region.size) self.md_region.close_fd() # Initialize or retreive Semaphore try: self.image_lock = Semaphore(self.image_lock_name, O_CREX) except ExistentialError: image_lock = Semaphore(self.image_lock_name, O_CREAT) image_lock.unlink() self.image_lock = Semaphore(self.image_lock_name, O_CREX) self.image_lock.release()
def __init__(self, name): # Initialize varaibles for memory regions and buffers and Semaphore self.shm_buf = None self.shm_region = None self.value_lock = None self.shm_name = name self.value_lock_name = name # Initialize shared memory buffer try: self.shm_region = SharedMemory(self.shm_name) self.shm_buf = mmap.mmap(self.shm_region.fd, sizeof(c_float)) self.shm_region.close_fd() except ExistentialError: self.shm_region = SharedMemory(self.shm_name, O_CREAT, size=sizeof(c_float)) self.shm_buf = mmap.mmap(self.shm_region.fd, self.shm_region.size) self.shm_region.close_fd() # Initialize or retreive Semaphore try: self.value_lock = Semaphore(self.value_lock_name, O_CREX) except ExistentialError: value_lock = Semaphore(self.value_lock_name, O_CREAT) value_lock.unlink() self.value_lock = Semaphore(self.value_lock_name, O_CREX) self.value_lock.release()
def _grab(self): def offset(channel): return channel * self.buf_info.size * self.FLOAT_SIZE if self.buf_info is None: return None if self.shm_obj is None: self.shm_obj = SharedMemory(self.buf_info.buf_id) self.data = [] try: for c in range(self.buf_info.channels): os.lseek(self.shm_obj.fd, offset(c), os.SEEK_SET) slc = os.read(self.shm_obj.fd, int(self.buf_info.size * self.FLOAT_SIZE)) self.data.append( list(numpy.fromstring(slc, dtype=numpy.float32))) self.set_bounds(0, None, len(self.data[0]) * 1000 / self.samplerate, None) except Exception, e: log.debug("scopeplot: error grabbing data", e) import traceback traceback.print_exc() return None
def fetch_framebuffer(mq): """ When the frame_buffer message queue is ready for read, this function is called to read the data and cast it to a frame object. Args: mq: The message queue containing the frame. Returns:frames.frame_buffer """ frame = frames.frame_message(mq.receive()[0]) if frame.valid is False: logger.info(' -- Received invalid message: ignoring') return None try: #logger.info(' -- Connecting to ' + frame.shm_location) shm_frame = SharedMemory(frame.shm_location, O_RDONLY, size=frame.frame_length) except StandardError as e: logger.info(' -- Error connecting to', frame.shm_location, '::', e.__repr__()) return None mapped = mmap(shm_frame.fd, shm_frame.size) shm_frame.close_fd() frame_buffer = frames.frame_buffer(mapped.read(frame.frame_length)) mapped.close() if frame_buffer.valid is False: logger.info(' -- Error Parsing Frame') return None return frame_buffer
def get(self): md = MD() self.sem.acquire() md_buf[:] = self.md_buf memmove(addressof(md), md_buf, sizeof(md)) self.sem.release() while not self.shm_buf: try: # print("Waiting for Data shared memory is available.") shm_region = SharedMemory(name=self.shm_name) self.shm_buf = mmap.mmap(shm_region.fd, md.size) shm_region.close_fd() # sleep(1) except ExistentialError: pass # sleep(1) self.sem.acquire() f = np.ndarray( shape=(md.shape_0, md.shape_1, md.shape_2), dtype="uint8", buffer=self.shm_buf, ) self.sem.release() return f
def clean_shared_memory(pid=None): ''' This function cleans up shared memory segments created by the GUI or a buggy Open MPI or CUDA. ''' # this only works for linux that has /dev/shm if not sys.platform.startswith('linux'): print("This function works only under Linux. Stop.", file=sys.stderr) return assert os.path.isdir('/dev/shm/') from posix_ipc import SharedMemory shm_list = os.listdir('/dev/shm/') user = getpass.getuser() for shm in shm_list: if (shm.startswith('ptycho') \ or shm.startswith('vader') \ or shm.startswith('cuda')) \ and user == find_owner('/dev/shm/'+shm): if (pid is None) or (pid is not None and pid in shm): s = SharedMemory("/" + shm) s.close_fd() s.unlink() print("Done.")
def get(self): # Define metadata metadata = MD() # Get metadata from the shared region self.image_lock.acquire() md_buf[:] = self.md_buf memmove(addressof(metadata), md_buf, sizeof(metadata)) self.image_lock.release() # Try to retreive the image from shm_buffer # Otherwise return a zero image try: self.shm_region = SharedMemory(self.shm_name) self.shm_buf = mmap.mmap(self.shm_region.fd, metadata.size) self.shm_region.close_fd() self.image_lock.acquire() image = np.ndarray(shape=(metadata.shape_0, metadata.shape_1, metadata.shape_2), dtype='uint8', buffer=self.shm_buf) self.image_lock.release() # Check for a None image if (image.size == 0): image = np.zeros((3, 3, 3), np.uint8) except ExistentialError: image = np.zeros((3, 3, 3), np.uint8) return image
def __init__(self, name, offset, size, flags=O_CREAT): self.shm = SharedMemory(name, flags=flags) self.offset = offset + INDY_SHM_MGR_OFFSET self.size = size + ROBOT_SN_LENGHT_MAX self.fmt = 'bbbbb%ss' % ROBOT_SN_LENGHT_MAX lseek(self.shm_fd, self.offset, SEEK_SET) self.write(0, 0, 0, 0, 0, 'INIT12345678'.encode('utf-8'))
def __init__(self, logger=None): if logger is None: logger = logging.getLogger('shared_memory.api') self.log = logger self.log.debug('try to connect to shared memory %s', self.MEM_SOUND_LEVEL) self.alsa_mem = SharedMemory(self.MEM_SOUND_LEVEL) self.alsa_mem_mmap = mmap(self.alsa_mem.fd, self.alsa_mem.size) self.alsa_mem_size = self.alsa_mem.size
def _connect(self): ''' attempt to connect to the shared memory ''' LOG.info('shmem %s: attempting connect', self._name) shmem = SharedMemory(self._name) self._mmap = mmap.mmap(shmem.fd, shmem.size) shmem.close_fd() self._lock = Semaphore('/pyspace_%s_lock' % self._name) LOG.info('shmem %s: connect succeeded', self._name)
def add(self, frame: np.ndarray): byte_size = frame.nbytes if not self.shm_region: self.shm_region = SharedMemory(self.shm_name, O_CREAT, size=byte_size) self.shm_buf = mmap.mmap(self.shm_region.fd, byte_size) self.shm_region.close_fd() self.count += 1 md = MD(frame.shape[0], frame.shape[1], frame.shape[2], byte_size, self.count) self.sem.acquire() memmove(md_buf, addressof(md), sizeof(md)) self.md_buf[:] = bytes(md_buf) self.shm_buf[:] = frame.tobytes() self.sem.release()
def __init__(self, name): self.shm_buf = None self.md_buf = None while not self.md_buf: try: print("Waiting for Block Input...") md_region = SharedMemory(name + '-meta') self.md_buf = mmap.mmap(md_region.fd, sizeof(MD)) md_region.close_fd() sleep(1) except ExistentialError: sleep(1) self.shm_name = name self.sem = Semaphore(name, 0)
def __init__(self, name): self.shm_buf = None self.md_buf = None logging.info("Reader launched") while not self.md_buf: try: logging.warning( "Waiting for MetaData shared memory is available.") md_region = SharedMemory(name + '-meta') self.md_buf = mmap.mmap(md_region.fd, sizeof(MD)) md_region.close_fd() sleep(1) except ExistentialError: sleep(1) self.shm_name = name self.sem = Semaphore(name, 0)
def __init__(self, name): self.shm_buf = None self.md_buf = None while not self.md_buf: try: # print("Waiting for MetaData shared memory is available.") md_region = SharedMemory(name + "-meta") self.md_buf = mmap.mmap(md_region.fd, sizeof(MD)) md_region.close_fd() # sleep(1) except ExistentialError: pass # sleep(1) self.shm_name = name self.sem = Semaphore(name, 0)
def __init__(self, name): self.shm_region = None logging.info("Writer launched") self.md_region = SharedMemory(name + '-meta', O_CREAT, size=sizeof(MD)) self.md_buf = mmap.mmap(self.md_region.fd, self.md_region.size) self.md_region.close_fd() self.shm_buf = None self.shm_name = name self.count = 0 try: self.sem = Semaphore(name, O_CREX) except ExistentialError: sem = Semaphore(name, O_CREAT) sem.unlink() self.sem = Semaphore(name, O_CREX) self.sem.release()
def _create(self): ''' attempt to create and initialize the shared memory ''' LOG.info('shmem %s: attempting create shmem', self._name) shmem = SharedMemory(self._name, size=SHMEM_SIZE, flags=O_CREX) LOG.info('shmem %s: attempting create mmap', self._name) self._mmap = mmap.mmap(shmem.fd, shmem.size) shmem.close_fd() LOG.info('shmem %s: attempting create semaphore', self._name) self._lock = Semaphore('/pyspace_%s_lock' % self._name, flags=O_CREX) LOG.info('shmem %s: create succeeded', self._name) try: self._initialize() self._lock.release() except: LOG.exception('shmem %s: initialize failed; attempting unlink', self._name) shmem.unlink() self._lock.unlink() raise
def get(self): md = MD() self.sem.acquire() md_buf[:] = self.md_buf memmove(addressof(md), md_buf, sizeof(md)) self.sem.release() while not self.shm_buf: try: print("Waiting for Data...") shm_region = SharedMemory(name=self.shm_name) self.shm_buf = mmap.mmap(shm_region.fd, md.size) shm_region.close_fd() sleep(1) except ExistentialError: sleep(1) self.sem.acquire() f = np.ndarray(shape=(md.shape_0), dtype='<U64', buffer=self.shm_buf) self.sem.release() return f
def add(self, image): # Get byte size of the image byte_size = image.nbytes # Get the shared memory buffer to read from if not self.shm_region: self.shm_region = SharedMemory(self.shm_name, O_CREAT, size=byte_size) self.shm_buf = mmap.mmap(self.shm_region.fd, byte_size) self.shm_region.close_fd() # Generate meta data metadata = MD(image.shape[0], image.shape[1], image.shape[2], byte_size) # Send the meta data and image to shared regions self.image_lock.acquire() memmove(md_buf, addressof(metadata), sizeof(metadata)) self.md_buf[:] = md_buf[:] self.shm_buf[:] = image.tobytes() self.image_lock.release()
def slice(self, start, end, channel=0): if self.shm_obj is None: self.shm_obj = SharedMemory(self.buf_id) if start < 0: start = 0 if start >= self.size: start = self.size - 1 if end < 0: end = 0 if end >= self.size: end = self.size - 1 try: os.lseek(self.shm_obj.fd, self.offset(channel, start), os.SEEK_SET) slc = os.read(self.shm_obj.fd, (end - start) * self.FLOAT_SIZE) self.outlets[1] = list(numpy.fromstring(slc, dtype=numpy.float32)) except Exception as e: import traceback tb = traceback.format_exc() log.debug("buffer~: slice error '%s" % e) self.error(tb) return None
def __init__(self, name, offset, size, flags=O_CREAT): # flag = 0 self.shm = SharedMemory(name, flags=flags) self.offset = offset + INDY_SHM_MGR_OFFSET self.size = size
def __init__(self, target_path, args=[]): global _lock global _process global _target_path global _shm global _mm global _ping_sem global _pong_sem global _clients global _next_client_id if _shm is None: launch_afl_forkserver = True else: launch_afl_forkserver = False with _lock: self.client_id = _next_client_id _next_client_id += 1 if launch_afl_forkserver: if _process is None: env = os.environ.copy() if 'EXTERNAL_AFL_FORKSERVER' not in env: print("Starting afl-forkserver...") fd, afl_out_file = tempfile.mkstemp( suffix='afl_out_file') os.close(fd) FNULL = open(os.devnull, 'w') cmd = [ gym_fuzz1ng.afl_forkserver_path(), '-f', afl_out_file, '--', target_path, ] cmd += args cmd += ['@@'] _process = subprocess.Popen( cmd, env=env, stdout=FNULL, stderr=subprocess.STDOUT, ) _target_path = target_path time.sleep(1) _shm = SharedMemory(SHARED_MEM_NAME) _mm = mmap.mmap(_shm.fd, 0) _ping_sem = Semaphore(SEM_PING_SIGNAL_NAME) _pong_sem = Semaphore(SEM_PONG_SIGNAL_NAME) else: if target_path != _target_path: raise Exception( "Concurrent targets is not supported: {} {}". format( target_path, _target_path, ), ) else: print("Skipping afl-forkserver start.") _clients[self.client_id] = True
def run(self): # overrides threading.Thread.run() """ Overrides threading.Thread.run() Creates initial connection to expected message queues; echometrics, frames, and tracks. It polls each one waiting for the read-ready flag to be set and consumes the data. It reformats the data into an object and writes it out to clients who are registered. Returns ------- None """ logger.info('Creating message_queues: ' + self.display_q_name) display_q = MessageQueue(self.display_q_name, O_CREAT) em_q = MessageQueue(self.em_q_name, O_CREAT) # TODO: READ THE NIMS FRAMEBUFFER NAME FROM THE YAML try: # connect to nims ingestor to get backscatter data nims_framebuffer = '/' + self.config['FRAMEBUFFER_NAME'] logger.info('Connecting to ' + nims_framebuffer) framebuffer_q = MessageQueue(nims_framebuffer, O_RDONLY) framebuffer_q.send(self.display_q_name) logger.info(" - sent queue: " + self.display_q_name) except ExistentialError as e: logger.info(' - Could not connect to ' + nims_framebuffer + '::' + e.__repr__()) try: # connect to nims tracker to get track data (probably out of sync) tracker_name = '/' + self.config['TRACKER_NAME'] logger.info('Connecting to ' + tracker_name) trackbuffer_q = MessageQueue(tracker_name, O_RDONLY) except ExistentialError as e: logger.info(' - Could not connect to ' + tracker_name + '::' + e.__repr__()) try: # connect to the echometrics queue for periodic em data em_queue_name = self.config['ECHOMETRICS']['queue_name'] logger.info('Connecting to ' + em_queue_name) em_request_q = MessageQueue(em_queue_name, O_RDONLY) em_request_q.send(self.em_q_name) except: logger.info(' - Could not connect to ' + em_queue_name) poller = select.poll() poller.register(em_q.mqd, select.POLLIN) poller.register(display_q.mqd, select.POLLIN) poller.register(trackbuffer_q.mqd, select.POLLIN) time.sleep( 1 ) # apparently necessary to create this latency for the frame_buffer app? while True: frame_buffer = None track_buffer = None em_buffer = None mq_state = poller.poll() for state in mq_state: mqd = state[0] if mqd == em_q.mqd: buf = em_q.receive()[0] em_buffer = ast.literal_eval(buf) elif mqd == display_q.mqd: frame = frames.frame_message(display_q.receive()[0]) if frame.valid is False: logger.info('Received invalid message: ignoring') continue try: #logger.info(' -- Connecting to ' + frame.shm_location) shm_frame = SharedMemory(frame.shm_location, O_RDONLY, size=frame.frame_length) except StandardError as e: logger.info(' -- Error connecting to', frame.shm_location, '::', e.__repr__()) continue mapped = mmap(shm_frame.fd, shm_frame.size) shm_frame.close_fd() frame_buffer = frames.frame_buffer( mapped.read(frame.frame_length)) mapped.close() if frame_buffer.valid is False: logger.info(' -- Error Parsing Frame') continue image = np.array(frame_buffer.image) image = image.reshape((frame_buffer.num_samples[0], frame_buffer.num_beams[0])) image = image[0:-1:4, ::] frame_buffer.image = image.flatten().tolist() frame_buffer.num_samples = (frame_buffer.num_samples[0] / 4, 0) elif mqd == trackbuffer_q.mqd: track_buffer = frames.track_message( trackbuffer_q.receive()[0]) clients = copy.copy(self.clients) for client in clients: try: if frame_buffer: client.send_image(frame_buffer) if track_buffer: client.send_tracks(track_buffer) if em_buffer: client.send_metrics(em_buffer) except StandardError as e: logger.info("Error sending data to client") print sys.exc_info() continue return
# * Unless required by applicable law or agreed to in writing, software # * distributed under the License is distributed on an "AS IS" BASIS, # * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # * See the License for the specific language governing permissions and # * limitations under the License. # * # */ import mmap from posix_ipc import SharedMemory from ctypes import sizeof import ctypes import time import sys rc_region = SharedMemory("db_rc_values_t") #typedef struct { # uint16_t ch[NUM_CHANNELS]; # uint16_t rssi; # struct timeval last_update; #} __attribute__((packed)) db_rc_values_t; class timeval(ctypes.Structure): _fields_ = [("tv_sec", ctypes.c_long), ("tv_usec", ctypes.c_long)] class stru(ctypes.Structure): _fields_ = [("ch1", ctypes.c_uint16), ("ch2", ctypes.c_uint16), ("ch3", ctypes.c_uint16), ("ch4", ctypes.c_uint16),
def __init__(self, name, size): self.shmem = None self.shmem = SharedMemory(name, flags=O_CREAT, mode=0o666, size=size) self.file = mmap(self.shmem.fd, size) self.__init_file_methods() self.__shmem_refs = [1]