def _collect_names_then_unlink(r):
# protect the process from ^C and "killall python" etc
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, signal.SIG_IGN)
# close all fds except r
try:
MAXFD = os.sysconf("SC_OPEN_MAX")
except:
MAXFD = 256
closerange(0, r)
closerange(r + 1, MAXFD)
# collect data written to pipe
data = []
while 1:
try:
s = os.read(r, 512)
except:
# XXX IO lock might be held at fork, so don't try
# printing unexpected exception - see issue 6721
pass
else:
if not s:
break
data.append(s)
# attempt to unlink each collected name
for name in bytes('', 'ascii').join(data).split(bytes('\0', 'ascii')):
try:
sem_unlink(name.decode('ascii'))
except:
# XXX IO lock might be held at fork, so don't try
# printing unexpected exception - see issue 6721
pass
def _collect_names_then_unlink(r):
# protect the process from ^C and "killall python" etc
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, signal.SIG_IGN)
# close all fds except r
try:
MAXFD = os.sysconf("SC_OPEN_MAX")
except:
MAXFD = 256
closerange(0, r)
closerange(r + 1, MAXFD)
# collect data written to pipe
data = []
while 1:
try:
s = os.read(r, 512)
except:
# XXX IO lock might be held at fork, so don't try
# printing unexpected exception - see issue 6721
pass
else:
if not s:
break
data.append(s)
# attempt to unlink each collected name
for name in bytes('', 'ascii').join(data).split(bytes('\0', 'ascii')):
try:
sem_unlink(name.decode('ascii'))
except:
# XXX IO lock might be held at fork, so don't try
# printing unexpected exception - see issue 6721
pass
def test_sem_unlink(self):
from _multiprocessing import sem_unlink
import errno
try:
sem_unlink("non-existent")
except OSError as e:
assert e.errno in (errno.ENOENT, errno.EINVAL)
else:
assert 0, "should have raised"
def main(fd):
'''Run semaphore tracker.'''
# protect the process from ^C and "killall python" etc
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, signal.SIG_IGN)
if _HAVE_SIGMASK:
signal.pthread_sigmask(signal.SIG_UNBLOCK, _IGNORED_SIGNALS)
for f in (sys.stdin, sys.stdout):
try:
f.close()
except Exception:
pass
cache = set()
try:
# keep track of registered/unregistered semaphores
with open(fd, 'rb') as f:
for line in f:
try:
cmd, name = line.strip().split(b':')
if cmd == b'REGISTER':
cache.add(name)
elif cmd == b'UNREGISTER':
cache.remove(name)
elif cmd == b'PROBE':
pass
else:
raise RuntimeError('unrecognized command %r' % cmd)
except Exception:
try:
sys.excepthook(*sys.exc_info())
except:
pass
finally:
# all processes have terminated; cleanup any remaining semaphores
if cache:
try:
warnings.warn('semaphore_tracker: There appear to be %d '
'leaked semaphores to clean up at shutdown' %
len(cache))
except Exception:
pass
for name in cache:
# For some reason the process which created and registered this
# semaphore has failed to unregister it. Presumably it has died.
# We therefore unlink it.
try:
name = name.decode('ascii')
try:
_multiprocessing.sem_unlink(name)
except Exception as e:
warnings.warn('semaphore_tracker: %r: %s' % (name, e))
finally:
pass
def resource_exists(name, rtype):
if rtype in ["folder", "file"]:
return os.path.exists(name)
elif rtype == "semlock":
try:
_SemLock(1, 1, 1, name, False)
sem_unlink(name)
return False
except OSError:
return True
else:
raise ValueError(f"Resource type {rtype} not understood")
def main(fd):
'''Run semaphore tracker.'''
# protect the process from ^C and "killall python" etc
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, signal.SIG_IGN)
if _HAVE_SIGMASK:
signal.pthread_sigmask(signal.SIG_UNBLOCK, _IGNORED_SIGNALS)
for f in (sys.stdin, sys.stdout):
try:
f.close()
except Exception:
pass
cache = set()
try:
# keep track of registered/unregistered semaphores
with open(fd, 'rb') as f:
for line in f:
try:
cmd, name = line.strip().split(b':')
if cmd == b'REGISTER':
cache.add(name)
elif cmd == b'UNREGISTER':
cache.remove(name)
else:
raise RuntimeError('unrecognized command %r' % cmd)
except Exception:
try:
sys.excepthook(*sys.exc_info())
except:
pass
finally:
# all processes have terminated; cleanup any remaining semaphores
if cache:
try:
warnings.warn('semaphore_tracker: There appear to be %d '
'leaked semaphores to clean up at shutdown' %
len(cache))
except Exception:
pass
for name in cache:
# For some reason the process which created and registered this
# semaphore has failed to unregister it. Presumably it has died.
# We therefore unlink it.
try:
name = name.decode('ascii')
try:
_multiprocessing.sem_unlink(name)
except Exception as e:
warnings.warn('semaphore_tracker: %r: %s' % (name, e))
finally:
pass
def main(fd):
"""Run semaphore tracker."""
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, signal.SIG_IGN)
for f in (sys.stdin, sys.stdout):
try:
f.close()
except Exception:
pass
cache = set()
try:
with open(fd, 'rb') as f:
for line in f:
try:
cmd, name = line.strip().split(b':')
if cmd == b'REGISTER':
cache.add(name)
elif cmd == b'UNREGISTER':
cache.remove(name)
else:
raise RuntimeError('unrecognized command %r' % cmd)
except Exception:
try:
sys.excepthook(*sys.exc_info())
except:
pass
finally:
if cache:
try:
warnings.warn(
'semaphore_tracker: There appear to be %d leaked semaphores to clean up at shutdown'
% len(cache))
except Exception:
pass
for name in cache:
try:
name = name.decode('ascii')
try:
_multiprocessing.sem_unlink(name)
except Exception as e:
warnings.warn('semaphore_tracker: %r: %s' % (name, e))
finally:
pass
def test_semaphore_rebuild(self):
import sys
if sys.platform == 'win32':
from _multiprocessing import SemLock
def sem_unlink(*args):
pass
else:
from _multiprocessing import SemLock, sem_unlink
kind = self.SEMAPHORE
value = 1
maxvalue = 1
sem = SemLock(kind, value, maxvalue, "4.2", unlink=False)
try:
sem2 = SemLock._rebuild(-1, kind, value, "4.2")
#assert sem.handle != sem2.handle---even though they come
# from different calls to sem_open(), on Linux at least,
# they are the same pointer
sem2 = SemLock._rebuild(sem.handle, kind, value, None)
assert sem.handle == sem2.handle
finally:
sem_unlink("4.2")
def run(self):
cache = set()
try:
with open(self._read_fd, 'rb') as f:
for line in f:
try:
cmd, name = line.strip().split(b':')
if cmd == b'REGISTER':
cache.add(name)
elif cmd == b'UNREGISTER':
cache.remove(name)
else:
raise RuntimeError(
'[{}] Unrecognized command {}'.format(
self.__class__.__name__, cmd))
except Exception:
try:
sys.excepthook(*sys.exc_info())
except:
pass
finally:
if cache:
try:
warnings.warn(
"[{}] There appear to be {} leaked semaphores".format(
self.__class__.__name__, len(cache)))
except Exception:
pass
for name in cache:
name = name.decode('ascii')
try:
_multiprocessing.sem_unlink(name)
except Exception as e:
warnings.warn('[{}] {}: {}'.format(
self.__class__.__name__, name, e))
def _cleanup(name):
sem_unlink(name)
resource_tracker.unregister(name, "semlock")
def main(fd):
'''Run semaphore tracker.'''
# protect the process from ^C and "killall python" etc
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, signal.SIG_IGN)
for f in (sys.stdin, sys.stdout):
try:
f.close()
except Exception:
pass
if VERBOSE: # pragma: no cover
sys.stderr.write("Main semaphore tracker is running\n")
sys.stderr.flush()
cache = set()
try:
# keep track of registered/unregistered semaphores
with os.fdopen(fd, 'rb') as f:
for line in f:
try:
cmd, name = line.strip().split(b':')
if cmd == b'REGISTER':
name = name.decode('ascii')
cache.add(name)
if VERBOSE: # pragma: no cover
sys.stderr.write(
"[SemaphoreTracker] register {}\n".format(
name))
sys.stderr.flush()
elif cmd == b'UNREGISTER':
name = name.decode('ascii')
cache.remove(name)
if VERBOSE: # pragma: no cover
sys.stderr.write("[SemaphoreTracker] unregister {}"
": cache({})\n".format(
name, len(cache)))
sys.stderr.flush()
elif cmd == b'PROBE':
pass
else:
raise RuntimeError('unrecognized command %r' % cmd)
except BaseException:
try:
sys.excepthook(*sys.exc_info())
except BaseException:
pass
finally:
# all processes have terminated; cleanup any remaining semaphores
if cache:
try:
warnings.warn('semaphore_tracker: There appear to be %d '
'leaked semaphores to clean up at shutdown' %
len(cache))
except Exception:
pass
for name in cache:
# For some reason the process which created and registered this
# semaphore has failed to unregister it. Presumably it has died.
# We therefore unlink it.
try:
try:
sem_unlink(name)
if VERBOSE: # pragma: no cover
sys.stderr.write(
"[SemaphoreTracker] unlink {}\n".format(name))
sys.stderr.flush()
except Exception as e:
warnings.warn('semaphore_tracker: %r: %r' % (name, e))
finally:
pass
if VERBOSE: # pragma: no cover
sys.stderr.write("semaphore tracker shut down\n")
sys.stderr.flush()
def _cleanup(name):
sem_unlink(name)
semaphore_tracker.unregister(name)
def _cleanup(name):
from .resource_tracker import unregister
sem_unlink(name)
unregister(name, "semaphore")
def _cleanup(name):
sem_unlink(name)
semaphore_tracker.unregister(name)
def _cleanup(name):
from .semaphore_tracker import unregister
sem_unlink(name)
unregister(name)
def _cleanup(name):
from .semaphore_tracker import unregister
sem_unlink(name)
unregister(name)
def main(fd):
'''Run semaphore tracker.'''
# protect the process from ^C and "killall python" etc
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, signal.SIG_IGN)
for f in (sys.stdin, sys.stdout):
try:
f.close()
except Exception:
pass
if VERBOSE: # pragma: no cover
sys.stderr.write("Main semaphore tracker is running\n")
sys.stderr.flush()
cache = set()
try:
# keep track of registered/unregistered semaphores
with os.fdopen(fd, 'rb') as f:
for line in f:
try:
cmd, name = line.strip().split(b':')
if cmd == b'REGISTER':
name = name.decode('ascii')
cache.add(name)
if VERBOSE: # pragma: no cover
sys.stderr.write("[SemaphoreTracker] register {}\n"
.format(name))
sys.stderr.flush()
elif cmd == b'UNREGISTER':
name = name.decode('ascii')
cache.remove(name)
if VERBOSE: # pragma: no cover
sys.stderr.write("[SemaphoreTracker] unregister {}"
": cache({})\n"
.format(name, len(cache)))
sys.stderr.flush()
elif cmd == b'PROBE':
pass
else:
raise RuntimeError('unrecognized command %r' % cmd)
except BaseException:
try:
sys.excepthook(*sys.exc_info())
except BaseException:
pass
finally:
# all processes have terminated; cleanup any remaining semaphores
if cache:
try:
warnings.warn('semaphore_tracker: There appear to be %d '
'leaked semaphores to clean up at shutdown' %
len(cache))
except Exception:
pass
for name in cache:
# For some reason the process which created and registered this
# semaphore has failed to unregister it. Presumably it has died.
# We therefore unlink it.
try:
try:
sem_unlink(name)
if VERBOSE: # pragma: no cover
sys.stderr.write("[SemaphoreTracker] unlink {}\n"
.format(name))
sys.stderr.flush()
except Exception as e:
warnings.warn('semaphore_tracker: %r: %r' % (name, e))
finally:
pass
if VERBOSE: # pragma: no cover
sys.stderr.write("semaphore tracker shut down\n")
sys.stderr.flush()
