def clean_up(SHARED_MEMORY_NAME, SEMAPHORE_NAME): try: posix_ipc.unlink_shared_memory(SHARED_MEMORY_NAME) posix_ipc.unlink_semaphore(SEMAPHORE_NAME) except: print ("memory or semaphore doesn't need cleanup") print ("\nAll clean!")
def test_unlink_shared_memory(self): """Exercise unlink_shared_memory""" mem = posix_ipc.SharedMemory(None, posix_ipc.O_CREX, size=1024) mem.close_fd() posix_ipc.unlink_shared_memory(mem.name) self.assertRaises(posix_ipc.ExistentialError, posix_ipc.SharedMemory, mem.name)
def remove(addr): """ Remove shared memory bucket """ try: with get_lock(addr): posix_ipc.unlink_shared_memory(addr) except posix_ipc.ExistentialError: pass
def close(self): """Method to close the shared space """ if not self.getAvailability(): if self.__log is not None: self.__writeLog(0, "Client already stopped.") else: print("INFO: Client already stopped.") return if self.__mapfile is not None: self.__mapfile.seek(0) self.__mapfile.write_byte(_BEGIN) self.__mapfile.write_byte(_CLOSED) self.__mapfile.write_byte(_END) self.__mapfile.close() self.__mapfile = None if self.__memory is not None: try: posix_ipc.unlink_shared_memory(self.__name_memory) self.__semaphore.release() self.__semaphore.unlink() except: if self.__log: self.__writeLog(0, "SharedMemory already closed.") self.__memory = None self.__semaphore = None
def INT_handler(sig_num, arg): if mapped_memory != None: mapped_memory.close() posix_ipc.unlink_shared_memory("test") #desaloca a o semaforo sem.close() sys.exit(0)
def close(self): self.ShMem.close() posix_ipc.unlink_shared_memory('/hoplite') self.ShLock.release() self.ShLock.unlink() super(Web, self).close()
def ndarray_shm(shape, dtype, location, readonly=False, order='F', **kwargs): """Create a shared memory numpy array. Requires /dev/shm to exist.""" import posix_ipc from posix_ipc import O_CREAT import psutil nbytes = Vec(*shape).rectVolume() * np.dtype(dtype).itemsize available = psutil.virtual_memory().available preexisting = 0 # This might only work on Ubuntu shmloc = os.path.join(SHM_DIRECTORY, location) if os.path.exists(shmloc): preexisting = os.path.getsize(shmloc) elif readonly: raise SharedMemoryReadError(shmloc + " has not been allocated. Requested " + str(nbytes) + " bytes.") if readonly and preexisting != nbytes: raise SharedMemoryReadError("{} exists, but the allocation size ({} bytes) does not match the request ({} bytes).".format( shmloc, preexisting, nbytes )) if (nbytes - preexisting) > available: overallocated = nbytes - preexisting - available overpercent = (100 * overallocated / (preexisting + available)) raise SharedMemoryAllocationError(""" Requested more memory than is available. Shared Memory Location: {} Shape: {} Requested Bytes: {} Available Bytes: {} Preexisting Bytes*: {} Overallocated Bytes*: {} (+{:.2f}%) * Preexisting is only correct on linux systems that support /dev/shm/""" \ .format(location, shape, nbytes, available, preexisting, overallocated, overpercent)) # This might seem like we're being "extra safe" but consider # a threading condition where the condition of the shared memory # was adjusted between the check above and now. Better to make sure # that we don't accidently change anything if readonly is set. flags = 0 if readonly else O_CREAT size = 0 if readonly else int(nbytes) try: shared = posix_ipc.SharedMemory(location, flags=flags, size=size) array_like = mmap.mmap(shared.fd, shared.size) os.close(shared.fd) renderbuffer = np.ndarray(buffer=array_like, dtype=dtype, shape=shape, order=order, **kwargs) except OSError as err: if err.errno == errno.ENOMEM: # Out of Memory posix_ipc.unlink_shared_memory(location) raise renderbuffer.setflags(write=(not readonly)) return array_like, renderbuffer
def unlink_shm(location): import posix_ipc try: posix_ipc.unlink_shared_memory(location) except posix_ipc.ExistentialError: return False return True
def init_shared_params(self, job_name, params, param_sync_rule, cleanup=False): """ Intialize shared memory parameters. This must be called before accessing the params attribute and/or calling :meth:`sync_params`, :meth:`lock_params` or :meth:`unlock_params`. Paramters --------- job_name : str An identifier. This must be the same across all Workers that share paramters. params : shared variables Theano shared variables representing the weights of your model. param_sync_rule : ParamSyncRule Update rule for the parameters cleanup : bool Whether to cleanup a previous run with the same identifier. Will also copy the current values of `params` to the shared memory. This is required on certain platform due to system restrictions. """ self.update_fn = param_sync_rule.make_update_function(params) if cleanup: try: posix_ipc.unlink_semaphore(job_name + 'lock') except posix_ipc.ExistentialError: pass self.lock = posix_ipc.Semaphore(job_name + 'lock', posix_ipc.O_CREAT, initial_value=1) params_descr = [(numpy.dtype(p.dtype), p.get_value(borrow=True).shape) for p in params] params_size = sum(descr_size(*d) for d in params_descr) if cleanup: try: posix_ipc.unlink_shared_memory(job_name + 'params') except posix_ipc.ExistentialError: pass self._shmref = posix_ipc.SharedMemory(job_name + 'params', posix_ipc.O_CREAT, size=params_size) self._shmref = posix_ipc.SharedMemory(job_name + 'params') self._shm = _mmap(fd=self._shmref.fd, length=params_size) self._shmref.close_fd() self.shared_params = [] off = 0 for dtype, shape in params_descr: self.shared_params.append( numpy.ndarray(shape, dtype=dtype, buffer=self._shm, offset=off)) off += descr_size(dtype, shape)
def pyspace_shmem(shmem_name): nothrow(ExistentialError, unlink_shared_memory, (shmem_name, )) nothrow(ExistentialError, unlink_semaphore, ('/pyspace_%s_lock' % shmem_name, )) with pyspaces.PySpaceShMem(shmem_name) as space: yield space unlink_shared_memory(shmem_name) unlink_semaphore('/pyspace_%s_lock' % shmem_name)
def init_shared_params(self, params, param_sync_rule): """ Initialize shared memory parameters. This must be called before accessing the params attribute and/or calling :meth:`sync_params`, :meth:`lock_params` or :meth:`unlock_params`. Parameters ---------- params : shared variables Theano shared variables representing the weights of your model. param_sync_rule : ParamSyncRule Update rule for the parameters cleanup : bool Whether to cleanup a previous run with the same identifier. Will also copy the current values of `params` to the shared memory. This is required on certain platform due to system restrictions. """ self.update_fn = param_sync_rule.make_update_function(params) self.local_params = params params_descr = [(numpy.dtype(p.dtype), p.get_value(borrow=True).shape) for p in params] params_size = sum(self._get_descr_size(*d) for d in params_descr) shared_mem_name = "{}_params".format(self._job_uid) # Acquire lock to decide who will init the shared memory self.lock_params() need_init = self.send_req("platoon-need_init") if need_init: # The ExistentialError is apparently the only way to verify if the shared_memory exists. try: posix_ipc.unlink_shared_memory(shared_mem_name) except posix_ipc.ExistentialError: pass self._shmref = posix_ipc.SharedMemory(shared_mem_name, posix_ipc.O_CREAT, size=params_size) else: self._shmref = posix_ipc.SharedMemory(shared_mem_name) self._shm = self._mmap(fd=self._shmref.fd, length=params_size) self._shmref.close_fd() self.shared_params = [] off = 0 for dtype, shape in params_descr: self.shared_params.append(numpy.ndarray(shape, dtype=dtype, buffer=self._shm, offset=off)) off += self._get_descr_size(dtype, shape) if need_init: self.copy_to_global(synchronous=False) self.unlock_params()
def create_shm(name, size): try: memory = posix_ipc.SharedMemory(name, posix_ipc.O_CREX, 0600, size) except posix_ipc.ExistentialError: posix_ipc.unlink_shared_memory(name) memory = posix_ipc.SharedMemory(name, posix_ipc.O_CREX, 0600, size) mapfile = mmap.mmap(memory.fd, memory.size) memory.close_fd() return mapfile
def signal_handler(signum, frame): global map_memory print('File ', __file__, ': Signal handler called with signal', signum) if map_memory: map_memory.close() posix_ipc.unlink_shared_memory(SHM_KEY) sys.exit(0)
def init_shared_params(self, params, param_sync_rule): """ Initialize shared memory parameters. This must be called before accessing the params attribute and/or calling :meth:`sync_params`. Parameters ---------- params : list of :ref:`theano.compile.SharedVariable` Theano shared variables representing the weights of your model. param_sync_rule : :class:`param_sync.ParamSyncRule` Update rule for the parameters """ self.update_fn = param_sync_rule.make_update_function(params) self.local_params = params params_descr = [(numpy.dtype(p.dtype), p.get_value(borrow=True).shape) for p in params] params_size = sum(self._get_descr_size(*d) for d in params_descr) shared_mem_name = "{}_params".format(self._job_uid) # Acquire lock to decide who will init the shared memory self.lock() need_init = self.send_req("platoon-need_init") if need_init: # The ExistentialError is apparently the only way to verify # if the shared_memory exists. try: posix_ipc.unlink_shared_memory(shared_mem_name) except posix_ipc.ExistentialError: pass self._shmref = posix_ipc.SharedMemory(shared_mem_name, posix_ipc.O_CREAT, size=params_size) else: self._shmref = posix_ipc.SharedMemory(shared_mem_name) self._shm = mmap(fd=self._shmref.fd, length=params_size) self._shmref.close_fd() self.shared_params = [] off = 0 for dtype, shape in params_descr: self.shared_params.append( numpy.ndarray(shape, dtype=dtype, buffer=self._shm, offset=off)) off += self._get_descr_size(dtype, shape) if need_init: self.copy_to_global(synchronous=False) self.unlock()
def init_shared_params(self, params, param_sync_rule): """ Initialize shared memory parameters. This must be called before accessing the params attribute and/or calling :meth:`sync_params`. Parameters ---------- params : list of :ref:`theano.compile.SharedVariable` Theano shared variables representing the weights of your model. param_sync_rule : :class:`param_sync.ParamSyncRule` Update rule for the parameters """ self.update_fn = param_sync_rule.make_update_function(params) self.local_params = params params_descr = [(numpy.dtype(p.dtype), p.get_value(borrow=True).shape) for p in params] params_size = sum(self._get_descr_size(*d) for d in params_descr) shared_mem_name = "{}_params".format(self._job_uid) # Acquire lock to decide who will init the shared memory self.lock() need_init = self.send_req("platoon-need_init") if need_init: # The ExistentialError is apparently the only way to verify # if the shared_memory exists. try: posix_ipc.unlink_shared_memory(shared_mem_name) except posix_ipc.ExistentialError: pass self._shmref = posix_ipc.SharedMemory(shared_mem_name, posix_ipc.O_CREAT, size=params_size) else: self._shmref = posix_ipc.SharedMemory(shared_mem_name) self._shm = mmap(fd=self._shmref.fd, length=params_size) self._shmref.close_fd() self.shared_params = [] off = 0 for dtype, shape in params_descr: self.shared_params.append(numpy.ndarray(shape, dtype=dtype, buffer=self._shm, offset=off)) off += self._get_descr_size(dtype, shape) if need_init: self.copy_to_global(synchronous=False) self.unlock()
def init_shared_params(self, job_name, params, param_sync_rule, cleanup=False): """ Intialize shared memory parameters. This must be called before accessing the params attribute and/or calling :meth:`sync_params`, :meth:`lock_params` or :meth:`unlock_params`. Paramters --------- job_name : str An identifier. This must be the same across all soldiers that share paramters. params : shared variables Theano shared variables representing the weights of your model. param_sync_rule : ParamSyncRule Update rule for the parameters cleanup : bool Whether to cleanup a previous run with the same identifier. Will also copy the current values of `params` to the shared memory. This is required on certain platform due to system restrictions. """ self.local_params = params self.param_sync_rule = param_sync_rule if cleanup: try: posix_ipc.unlink_semaphore(job_name+'lock') except posix_ipc.ExistentialError: pass self.lock = posix_ipc.Semaphore(job_name+'lock', posix_ipc.O_CREAT, initial_value=1) params_descr = [(numpy.dtype(p.dtype), p.get_value(borrow=True).shape) for p in params] params_size = sum(descr_size(*d) for d in params_descr) if cleanup: try: posix_ipc.unlink_shared_memory(job_name+'params') except posix_ipc.ExistentialError: pass self._shmref = posix_ipc.SharedMemory(job_name+'params', posix_ipc.O_CREAT, size=params_size) self._shmref = posix_ipc.SharedMemory(job_name+'params') self._shm = _mmap(fd=self._shmref.fd, length=params_size) self._shmref.close_fd() self.shared_params = [] off = 0 for dtype, shape in params_descr: self.shared_params.append(numpy.ndarray(shape, dtype=dtype, buffer=self._shm, offset=off)) off += descr_size(dtype, shape)
def cleanup(self): self.semaphore.release() time.sleep(2) self.semaphore.acquire() self.mapfile.close() posix_ipc.unlink_shared_memory(self.memory_name) self.semaphore.release() self.semaphore.unlink()
def __del__(self): try: self.mmap.close() self.queue.close() posix_ipc.unlink_shared_memory("/ipcomm" + self.name + "mmap") self.sem.release() self.sem.unlink() self.queue.unlink() except Exception as e: print("Error shutting down IPComm %s: %s" % (self.name, e))
def _shared_array_unlink(X) : if isinstance(X,str): name = X else : name = X._shared_fname try: posix_ipc.unlink_shared_memory(name) except (posix_ipc.ExistentialError, OSError) : pass
def _shared_array_unlink(X): if isinstance(X, str): name = X else: name = X._shared_fname try: posix_ipc.unlink_shared_memory(name) except (posix_ipc.ExistentialError, OSError): pass
def _init_new_shmem(self, req_info): """ Initiates a POSIX shared memory buffer on a :class:`Worker`'s request which will be accesible to all :class:`Worker`s in a host. Parameters ---------- req_info : dict Request info from individual :class:`Worker`. Returns ------- response : str Response to a :class:`Worker`'s request. Contains common reference name to POSIX shared memory. req_info -------- * *size* : int Shared memory's size in bytes. .. versionadded:: 0.6.0 """ first = self._is_worker_first( self._init_new_shmem_count) # See :meth:`_is_worker_first` if first: self._last_shmem_name = "platoon-{0}_{1}_buffer".format( self._job_uid, len(self.shared_buffers)) try: posix_ipc.unlink_shared_memory(self._last_shmem_name) except posix_ipc.ExistentialError: pass size = req_info['size'] try: shmref = posix_ipc.SharedMemory(self._last_shmem_name, posix_ipc.O_CREAT, size=size) shm = mmap(fd=shmref.fd, length=size) shmref.close_fd() except Exception as exc: try: shmref.unlink() except (NameError, posix_ipc.ExistentialError): pass raise PlatoonError("Failed to initialize new shared buffer.", exc) # We want every worker to get the same shared memory name that is # was declared in the first call of a mass request to this # controller for initializing a new shared memory. self._shmrefs[self._last_shmem_name] = shmref # Keep for unlinking when closing self.shared_buffers[self._last_shmem_name] = shm return self._last_shmem_name
def release(self): self.map_md.close() ipc.unlink_shared_memory(self.SHM_NAME_MD) self.map_fr.close() ipc.unlink_shared_memory(self.SHM_NAME_FR) self.sem_w.close() self.sem_r.close() logging.info("Writer terminated")
def tearDown(self): for shm in shm_list: posix_ipc.unlink_shared_memory(shm) for sem in sem_list: sem.release() sem.unlink() #stop_usrpserver(self.serversock, self.pid) for sock in self.serversock: sock.close()
def release(self): self.sem.acquire() self.md_buf.close() unlink_shared_memory(self.shm_name + "-meta") self.shm_buf.close() unlink_shared_memory(self.shm_name) self.sem.release() self.sem.close()
def close(self): self.value_lock.acquire() self.shm_buf.close() try: unlink_shared_memory(self.shm_name) except ExistentialError: pass self.value_lock.release() self.value_lock.close()
def destroy(self): """ Order operating system to destroy allocated shared memory. Important you should always destroy shared_memory after use. Otherwise you will have memory leak. This should be called by master process, otherwise child processes can destroy it without waiting for other processes to read the information. @return: None @rtype: None """ posix_ipc.unlink_shared_memory(self.shared_memory.name)
def __del__(self): ''' If this variable is deleted, we should manage it appropriately ''' try: self.mapFile.close() posix_ipc.unlink_shared_memory(self.SHARED_MEMORY_NAME) print("Memory unlinked!") except Exception as e: print(str(e)) print("Could not unlink shared memory for some reason") return
def exit_cleanup(): """Clean up any shared memory that has not yet been freed. In theory this should not be required, but it is here as a safety net.""" global _all_shared_arrays for fname in _all_shared_arrays: try: posix_ipc.unlink_shared_memory(fname) except (posix_ipc.ExistentialError, OSError): pass
def fini(self): try: self.wsproc.terminate() print("WebSockets terminated") self.memmap.close() self.queue.close() posix_ipc.unlink_shared_memory("flyermmap") self.sem.release() self.sem.unlink() self.queue.unlink() except: print("Error shutting down Flyer")
def release(self): self.sem.acquire() self.md_buf.close() unlink_shared_memory(self.shm_name + '-meta') self.shm_buf.close() unlink_shared_memory(self.shm_name) self.sem.release() self.sem.close() logging.info("Writer terminated")
def _init_new_shmem(self, req_info): """ Initiates a POSIX shared memory buffer on a :class:`Worker`'s request which will be accesible to all :class:`Worker`s in a host. Parameters ---------- req_info : dict Request info from individual :class:`Worker`. Returns ------- response : str Response to a :class:`Worker`'s request. Contains common reference name to POSIX shared memory. req_info -------- * *size* : int Shared memory's size in bytes. .. versionadded:: 0.6.0 """ first = self._is_worker_first(self._init_new_shmem_count) # See :meth:`_is_worker_first` if first: self._last_shmem_name = "platoon-{0}_{1}_buffer".format(self._job_uid, len(self.shared_buffers)) try: posix_ipc.unlink_shared_memory(self._last_shmem_name) except posix_ipc.ExistentialError: pass size = req_info['size'] try: shmref = posix_ipc.SharedMemory(self._last_shmem_name, posix_ipc.O_CREAT, size=size) shm = mmap(fd=shmref.fd, length=size) shmref.close_fd() except Exception as exc: try: shmref.unlink() except (NameError, posix_ipc.ExistentialError): pass raise PlatoonError("Failed to initialize new shared buffer.", exc) # We want every worker to get the same shared memory name that is # was declared in the first call of a mass request to this # controller for initializing a new shared memory. self._shmrefs[self._last_shmem_name] = shmref # Keep for unlinking when closing self.shared_buffers[self._last_shmem_name] = shm return self._last_shmem_name
def destroy_shm_semaphore(semaphore, mapfile): params = utils.read_params() utils.say("Destroying semaphore and shared memory.") mapfile.close() # I could call memory.unlink() here but in order to demonstrate # unlinking at the module level I'll do it that way. posix_ipc.unlink_shared_memory(params["SHARED_MEMORY_NAME"]) semaphore.release() # I could also unlink the semaphore by calling # posix_ipc.unlink_semaphore() but I'll do it this way instead. semaphore.unlink()
def fini(self): try: self.memmap.close() self.server.close() self.queue.close() self.loop.stop() self.loop.close() print("WebSockets closed") posix_ipc.unlink_shared_memory("flyermmap") self.sem.release() self.sem.unlink() self.queue.unlink() except: print("Error closing WebSockets")
def cleanup(self, signum=None, frame=None): self.save_config(self.config, self.config_file) for hx in self.hx_handles: try: hx.power_down() except RuntimeError: # GPIO already cleaned up pass GPIO.cleanup() try: self.ShMem.close() posix_ipc.unlink_shared_memory('/hoplite') self.ShLock.release() self.ShLock.unlink() except posix_ipc.ExistentialError: # shmem already cleaned up pass
async def run_command(self, *args, timeout=None): print("run controller loop.") self.process = await asyncio.create_subprocess_exec(*args, stdout=PIPE, stdin=PIPE, stderr=STDOUT) await self.prompt_avdeccctl_netdev() self.mq.send("ready") print("have sent ready") while(True): # read notification # check mqueue print("waiting for msg") try: msg, = self.mq.receive(10) msg = msg.decode() if "discover" in msg: print("received discover cmd") await self.command_avdeccctl_list("list") self.mq.send("ack") elif "connect" in msg: pass elif "view" in msg: pass elif "quit" in msg: self.writeStdin("quit") break except (posix_ipc.BusyError, ValueError): await self.command_avdeccctl_list("list") self.process.kill() # I could call memory.unlink() here but in order to demonstrate # unlinking at the module level I'll do it that way. posix_ipc.unlink_shared_memory(self.params["SHARED_MEMORY_NAME"]) self.semaphore.release() self.semaphore_mq_gui.release() self.semaphore_mq_wrapper.release() self.semaphore.unlink() self.semaphore_mq_gui.unlink() self.semaphore_mq_wrapper.unlink() self.mapfile.close() self.mq.close() return await self.process.wait() # wait for the child process to exit
def unlink_all(self): """ @brief Unlink (remove) all posix shared memory sections and semaphores. """ log.debug("Unlinking all relevant posix shared memory segments and semaphores") try: posix_ipc.unlink_semaphore(self.counting_semaphore_key) except posix_ipc.ExistentialError: pass try: posix_ipc.unlink_semaphore(self.mutex_semaphore_key) except posix_ipc.ExistentialError: pass try: posix_ipc.unlink_shared_memory(self.shared_buffer_key) except posix_ipc.ExistentialError: pass
mem.unlink() print_mem_after() say("Running memory create/destroy test 2...") print_mem_before() for i in the_range: name = "/" + ''.join(random.sample(NAME_CHARACTERS, NAME_LENGTH)) mem = posix_ipc.SharedMemory(name, posix_ipc.O_CREX, size = 4096) os.close(mem.fd) posix_ipc.unlink_shared_memory(name) print_mem_after() say("Running memory create/destroy test 3...") print_mem_before() for i in the_range: mem = posix_ipc.SharedMemory(None, posix_ipc.O_CREX, size = 4096) os.close(mem.fd) mem.unlink() print_mem_after()
def close_shm(name, mapfile, unlink=False): mapfile.close() if unlink: posix_ipc.unlink_shared_memory(name)
def _shared_array_unlink(X) : # os.unlink(X._shared_fname) posix_ipc.unlink_shared_memory(X._shared_fname)
import posix_ipc import utils params = utils.read_params() try: posix_ipc.unlink_shared_memory(params["SHARED_MEMORY_NAME"]) s = "memory segment %s removed" % params["SHARED_MEMORY_NAME"] print (s) except: print ("memory doesn't need cleanup") try: posix_ipc.unlink_semaphore(params["SEMAPHORE_NAME"]) s = "semaphore %s removed" % params["SEMAPHORE_NAME"] print (s) except: print ("semaphore doesn't need cleanup") print ("\nAll clean!")
def _shared_array_unlink(X) : # os.unlink(X._shared_fname) try: posix_ipc.unlink_shared_memory(X._shared_fname) except (posix_ipc.ExistentialError, OSError) : pass
import posix_ipc # clean up the mess in case sender/receiver was terminated unexpectedly try: posix_ipc.unlink_shared_memory("/mysharedmem") except: print "failed to clean up shared memory" try: posix_ipc.unlink_semaphore("/mysemaphore") except: print "failed to clean up semaphore"