def _collect_names_then_unlink(r): # protect the process from ^C and "killall python" etc signal.signal(signal.SIGINT, signal.SIG_IGN) signal.signal(signal.SIGTERM, signal.SIG_IGN) # close all fds except r try: MAXFD = os.sysconf("SC_OPEN_MAX") except: MAXFD = 256 closerange(0, r) closerange(r + 1, MAXFD) # collect data written to pipe data = [] while 1: try: s = os.read(r, 512) except: # XXX IO lock might be held at fork, so don't try # printing unexpected exception - see issue 6721 pass else: if not s: break data.append(s) # attempt to unlink each collected name for name in bytes('', 'ascii').join(data).split(bytes('\0', 'ascii')): try: sem_unlink(name.decode('ascii')) except: # XXX IO lock might be held at fork, so don't try # printing unexpected exception - see issue 6721 pass
def test_sem_unlink(self): from _multiprocessing import sem_unlink import errno try: sem_unlink("non-existent") except OSError as e: assert e.errno in (errno.ENOENT, errno.EINVAL) else: assert 0, "should have raised"
def main(fd): '''Run semaphore tracker.''' # protect the process from ^C and "killall python" etc signal.signal(signal.SIGINT, signal.SIG_IGN) signal.signal(signal.SIGTERM, signal.SIG_IGN) if _HAVE_SIGMASK: signal.pthread_sigmask(signal.SIG_UNBLOCK, _IGNORED_SIGNALS) for f in (sys.stdin, sys.stdout): try: f.close() except Exception: pass cache = set() try: # keep track of registered/unregistered semaphores with open(fd, 'rb') as f: for line in f: try: cmd, name = line.strip().split(b':') if cmd == b'REGISTER': cache.add(name) elif cmd == b'UNREGISTER': cache.remove(name) elif cmd == b'PROBE': pass else: raise RuntimeError('unrecognized command %r' % cmd) except Exception: try: sys.excepthook(*sys.exc_info()) except: pass finally: # all processes have terminated; cleanup any remaining semaphores if cache: try: warnings.warn('semaphore_tracker: There appear to be %d ' 'leaked semaphores to clean up at shutdown' % len(cache)) except Exception: pass for name in cache: # For some reason the process which created and registered this # semaphore has failed to unregister it. Presumably it has died. # We therefore unlink it. try: name = name.decode('ascii') try: _multiprocessing.sem_unlink(name) except Exception as e: warnings.warn('semaphore_tracker: %r: %s' % (name, e)) finally: pass
def resource_exists(name, rtype): if rtype in ["folder", "file"]: return os.path.exists(name) elif rtype == "semlock": try: _SemLock(1, 1, 1, name, False) sem_unlink(name) return False except OSError: return True else: raise ValueError(f"Resource type {rtype} not understood")
def main(fd): '''Run semaphore tracker.''' # protect the process from ^C and "killall python" etc signal.signal(signal.SIGINT, signal.SIG_IGN) signal.signal(signal.SIGTERM, signal.SIG_IGN) if _HAVE_SIGMASK: signal.pthread_sigmask(signal.SIG_UNBLOCK, _IGNORED_SIGNALS) for f in (sys.stdin, sys.stdout): try: f.close() except Exception: pass cache = set() try: # keep track of registered/unregistered semaphores with open(fd, 'rb') as f: for line in f: try: cmd, name = line.strip().split(b':') if cmd == b'REGISTER': cache.add(name) elif cmd == b'UNREGISTER': cache.remove(name) else: raise RuntimeError('unrecognized command %r' % cmd) except Exception: try: sys.excepthook(*sys.exc_info()) except: pass finally: # all processes have terminated; cleanup any remaining semaphores if cache: try: warnings.warn('semaphore_tracker: There appear to be %d ' 'leaked semaphores to clean up at shutdown' % len(cache)) except Exception: pass for name in cache: # For some reason the process which created and registered this # semaphore has failed to unregister it. Presumably it has died. # We therefore unlink it. try: name = name.decode('ascii') try: _multiprocessing.sem_unlink(name) except Exception as e: warnings.warn('semaphore_tracker: %r: %s' % (name, e)) finally: pass
def main(fd): """Run semaphore tracker.""" signal.signal(signal.SIGINT, signal.SIG_IGN) signal.signal(signal.SIGTERM, signal.SIG_IGN) for f in (sys.stdin, sys.stdout): try: f.close() except Exception: pass cache = set() try: with open(fd, 'rb') as f: for line in f: try: cmd, name = line.strip().split(b':') if cmd == b'REGISTER': cache.add(name) elif cmd == b'UNREGISTER': cache.remove(name) else: raise RuntimeError('unrecognized command %r' % cmd) except Exception: try: sys.excepthook(*sys.exc_info()) except: pass finally: if cache: try: warnings.warn( 'semaphore_tracker: There appear to be %d leaked semaphores to clean up at shutdown' % len(cache)) except Exception: pass for name in cache: try: name = name.decode('ascii') try: _multiprocessing.sem_unlink(name) except Exception as e: warnings.warn('semaphore_tracker: %r: %s' % (name, e)) finally: pass
def test_semaphore_rebuild(self): import sys if sys.platform == 'win32': from _multiprocessing import SemLock def sem_unlink(*args): pass else: from _multiprocessing import SemLock, sem_unlink kind = self.SEMAPHORE value = 1 maxvalue = 1 sem = SemLock(kind, value, maxvalue, "4.2", unlink=False) try: sem2 = SemLock._rebuild(-1, kind, value, "4.2") #assert sem.handle != sem2.handle---even though they come # from different calls to sem_open(), on Linux at least, # they are the same pointer sem2 = SemLock._rebuild(sem.handle, kind, value, None) assert sem.handle == sem2.handle finally: sem_unlink("4.2")
def run(self): cache = set() try: with open(self._read_fd, 'rb') as f: for line in f: try: cmd, name = line.strip().split(b':') if cmd == b'REGISTER': cache.add(name) elif cmd == b'UNREGISTER': cache.remove(name) else: raise RuntimeError( '[{}] Unrecognized command {}'.format( self.__class__.__name__, cmd)) except Exception: try: sys.excepthook(*sys.exc_info()) except: pass finally: if cache: try: warnings.warn( "[{}] There appear to be {} leaked semaphores".format( self.__class__.__name__, len(cache))) except Exception: pass for name in cache: name = name.decode('ascii') try: _multiprocessing.sem_unlink(name) except Exception as e: warnings.warn('[{}] {}: {}'.format( self.__class__.__name__, name, e))
def _cleanup(name): sem_unlink(name) resource_tracker.unregister(name, "semlock")
def main(fd): '''Run semaphore tracker.''' # protect the process from ^C and "killall python" etc signal.signal(signal.SIGINT, signal.SIG_IGN) signal.signal(signal.SIGTERM, signal.SIG_IGN) for f in (sys.stdin, sys.stdout): try: f.close() except Exception: pass if VERBOSE: # pragma: no cover sys.stderr.write("Main semaphore tracker is running\n") sys.stderr.flush() cache = set() try: # keep track of registered/unregistered semaphores with os.fdopen(fd, 'rb') as f: for line in f: try: cmd, name = line.strip().split(b':') if cmd == b'REGISTER': name = name.decode('ascii') cache.add(name) if VERBOSE: # pragma: no cover sys.stderr.write( "[SemaphoreTracker] register {}\n".format( name)) sys.stderr.flush() elif cmd == b'UNREGISTER': name = name.decode('ascii') cache.remove(name) if VERBOSE: # pragma: no cover sys.stderr.write("[SemaphoreTracker] unregister {}" ": cache({})\n".format( name, len(cache))) sys.stderr.flush() elif cmd == b'PROBE': pass else: raise RuntimeError('unrecognized command %r' % cmd) except BaseException: try: sys.excepthook(*sys.exc_info()) except BaseException: pass finally: # all processes have terminated; cleanup any remaining semaphores if cache: try: warnings.warn('semaphore_tracker: There appear to be %d ' 'leaked semaphores to clean up at shutdown' % len(cache)) except Exception: pass for name in cache: # For some reason the process which created and registered this # semaphore has failed to unregister it. Presumably it has died. # We therefore unlink it. try: try: sem_unlink(name) if VERBOSE: # pragma: no cover sys.stderr.write( "[SemaphoreTracker] unlink {}\n".format(name)) sys.stderr.flush() except Exception as e: warnings.warn('semaphore_tracker: %r: %r' % (name, e)) finally: pass if VERBOSE: # pragma: no cover sys.stderr.write("semaphore tracker shut down\n") sys.stderr.flush()
def _cleanup(name): sem_unlink(name) semaphore_tracker.unregister(name)
def _cleanup(name): from .resource_tracker import unregister sem_unlink(name) unregister(name, "semaphore")
def _cleanup(name): from .semaphore_tracker import unregister sem_unlink(name) unregister(name)
def main(fd): '''Run semaphore tracker.''' # protect the process from ^C and "killall python" etc signal.signal(signal.SIGINT, signal.SIG_IGN) signal.signal(signal.SIGTERM, signal.SIG_IGN) for f in (sys.stdin, sys.stdout): try: f.close() except Exception: pass if VERBOSE: # pragma: no cover sys.stderr.write("Main semaphore tracker is running\n") sys.stderr.flush() cache = set() try: # keep track of registered/unregistered semaphores with os.fdopen(fd, 'rb') as f: for line in f: try: cmd, name = line.strip().split(b':') if cmd == b'REGISTER': name = name.decode('ascii') cache.add(name) if VERBOSE: # pragma: no cover sys.stderr.write("[SemaphoreTracker] register {}\n" .format(name)) sys.stderr.flush() elif cmd == b'UNREGISTER': name = name.decode('ascii') cache.remove(name) if VERBOSE: # pragma: no cover sys.stderr.write("[SemaphoreTracker] unregister {}" ": cache({})\n" .format(name, len(cache))) sys.stderr.flush() elif cmd == b'PROBE': pass else: raise RuntimeError('unrecognized command %r' % cmd) except BaseException: try: sys.excepthook(*sys.exc_info()) except BaseException: pass finally: # all processes have terminated; cleanup any remaining semaphores if cache: try: warnings.warn('semaphore_tracker: There appear to be %d ' 'leaked semaphores to clean up at shutdown' % len(cache)) except Exception: pass for name in cache: # For some reason the process which created and registered this # semaphore has failed to unregister it. Presumably it has died. # We therefore unlink it. try: try: sem_unlink(name) if VERBOSE: # pragma: no cover sys.stderr.write("[SemaphoreTracker] unlink {}\n" .format(name)) sys.stderr.flush() except Exception as e: warnings.warn('semaphore_tracker: %r: %r' % (name, e)) finally: pass if VERBOSE: # pragma: no cover sys.stderr.write("semaphore tracker shut down\n") sys.stderr.flush()