def give_terminal_to(pgid):
if pgid is None:
return False
oldmask = signal.pthread_sigmask(signal.SIG_BLOCK,
_block_when_giving)
try:
os.tcsetpgrp(FD_STDERR, pgid)
return True
except ProcessLookupError:
# when the process finished before giving terminal to it,
# see issue #2288
return False
except OSError as e:
if e.errno == 22: # [Errno 22] Invalid argument
# there are cases that all the processes of pgid have
# finished, then we don't need to do anything here, see
# issue #2220
return False
elif e.errno == 25: # [Errno 25] Inappropriate ioctl for device
# There are also cases where we are not connected to a
# real TTY, even though we may be run in interactive
# mode. See issue #2267 for an example with emacs
return False
else:
raise
finally:
signal.pthread_sigmask(signal.SIG_SETMASK, oldmask)
def _run(self):
if hasattr(signal, 'pthread_sigmask'):
# this thread should not handle any signal
mask = range(1, signal.NSIG)
signal.pthread_sigmask(signal.SIG_BLOCK, mask)
self.schedule()
while self.stop_lock.acquire(0):
self.stop_lock.release()
delay = self.once()
if delay is not None:
assert delay > 0.0
with self.sleep_lock:
interrupted = self.sleep_lock.wait(timeout=delay)
if interrupted:
break
continue
task = self._task_ref()
try:
task.call()
except Exception as err:
# the task is not rescheduled on error
exc_type, exc_value, exc_tb = sys.exc_info()
# FIXME: log the traceback
print(("%s: %s" % (exc_type, exc_value)), file=sys.stderr)
break
if self.ncall is not None:
self.ncall -= 1
if self.ncall <= 0:
break
self.schedule()
def _give_terminal_to(pgid):
st = _shell_tty()
if st is not None and os.isatty(st):
oldmask = signal.pthread_sigmask(signal.SIG_BLOCK,
_block_when_giving)
os.tcsetpgrp(st, pgid)
signal.pthread_sigmask(signal.SIG_SETMASK, oldmask)
def _serve(self):
if hasattr(signal, "pthread_sigmask"):
signal.pthread_sigmask(signal.SIG_BLOCK, range(1, signal.NSIG))
while 1:
try:
conn = self._listener.accept()
msg = conn.recv()
if msg is None:
break
key, destination_pid = msg
send, close = self._cache.pop(key)
send(conn, destination_pid)
close()
conn.close()
except:
if not is_exiting():
import traceback
sub_warning(
"thread for sharing handles raised exception :\n"
+ "-" * 79
+ "\n"
+ traceback.format_exc()
+ "-" * 79
)
def ensure_running(self):
'''Make sure that semaphore tracker process is running.
This can be run from any process. Usually a child process will use
the semaphore created by its parent.'''
with self._lock:
if self._pid is not None:
# semaphore tracker was launched before, is it still running?
try:
pid, _ = os.waitpid(self._pid, os.WNOHANG)
except ChildProcessError:
# The process terminated
pass
else:
if not pid:
# => still alive
return
# => dead, launch it again
os.close(self._fd)
self._fd = None
self._pid = None
warnings.warn('semaphore_tracker: process died unexpectedly, '
'relaunching. Some semaphores might leak.')
fds_to_pass = []
try:
fds_to_pass.append(sys.stderr.fileno())
except Exception:
pass
cmd = 'from multiprocessing.semaphore_tracker import main;main(%d)'
r, w = os.pipe()
try:
fds_to_pass.append(r)
# process will out live us, so no need to wait on pid
exe = spawn.get_executable()
args = [exe] + util._args_from_interpreter_flags()
args += ['-c', cmd % r]
# bpo-33613: Register a signal mask that will block the signals.
# This signal mask will be inherited by the child that is going
# to be spawned and will protect the child from a race condition
# that can make the child die before it registers signal handlers
# for SIGINT and SIGTERM. The mask is unregistered after spawning
# the child.
try:
if _HAVE_SIGMASK:
signal.pthread_sigmask(signal.SIG_BLOCK, _IGNORED_SIGNALS)
pid = util.spawnv_passfds(exe, args, fds_to_pass)
finally:
if _HAVE_SIGMASK:
signal.pthread_sigmask(signal.SIG_UNBLOCK, _IGNORED_SIGNALS)
except:
os.close(w)
raise
else:
self._fd = w
self._pid = pid
finally:
os.close(r)
def _do_exit(self, signo):
if signo == 0 or signo == signal.SIGINT:
return
curses.endwin()
signal.pthread_sigmask(signal.SIG_UNBLOCK, [signo])
signal.signal(signo, signal.SIG_DFL)
os.kill(self._pid, signo)
def _give_terminal_to(pgid):
# over-simplified version of:
# give_terminal_to from bash 4.3 source, jobs.c, line 4030
# this will give the terminal to the process group pgid
if _shell_tty is not None and os.isatty(_shell_tty):
oldmask = signal.pthread_sigmask(signal.SIG_BLOCK, _block_when_giving)
os.tcsetpgrp(_shell_tty, pgid)
signal.pthread_sigmask(signal.SIG_SETMASK, oldmask)
def run(self):
self.take_snapshot()
if hasattr(signal, 'pthread_sigmask'):
# Available on UNIX with Python 3.3+
signal.pthread_sigmask(signal.SIG_BLOCK, range(1, signal.NSIG))
time.sleep(init_delay)
while True:
self.take_snapshot()
time.sleep(snapshot_delay)
def halt(self):
oldh = signal.signal(signal.SIGINT, lambda i,f: None)
signal.pthread_sigmask(signal.SIG_BLOCK, {signal.SIGINT})
for thr in self.threads:
thr.request.halt_loop()
signal.pthread_sigmask(signal.SIG_UNBLOCK, {signal.SIGINT})
signal.signal(signal.SIGINT, oldh)
def init_signal_handling(self):
# In Python 3.5 system calls are no longer interrupted by signals.
# Thus, we can no longer use time.sleep() or otherwise the processing
# of signals would be delayed until the sleep has finished.
# So, we now use signal.sigtimedwait() instead. Due to the lack of
# documentation (and my unwillingness to spend more time on this than
# necessary) I'm not quite sure, if I'm doing this completely right.
# In the following, we set all signals that we're interested in as
# blocked, so that they queue up. In TimeProvider.sleep() they're taken
# again from the queue by signal.sigtimedwait().
signal.pthread_sigmask(signal.SIG_BLOCK, SIGNALS)
def run(host, port, Payload):
"""Select loop of scheduler.
:param host: A string with the host to bind to.
:param port: An integer with the port to bind to.
:param Payload: A class that follows the interface of ``types.Payload``.
"""
scheduler = Scheduler()
sock = init_socket('127.0.0.1', 8000)
selector = selectors.DefaultSelector()
callback = partial(handle_request, klass=Payload)
selector.register(sock, selectors.EVENT_READ, callback)
sigint_fd = linuxfd.signalfd(
signalset={signal.SIGINT, signal.SIGTERM}, nonBlocking=True
)
selector.register(sigint_fd, selectors.EVENT_READ, True)
sighup_fd = linuxfd.signalfd(signalset={signal.SIGHUP}, nonBlocking=True)
selector.register(sighup_fd, selectors.EVENT_READ, scheduler.report)
signal.pthread_sigmask(
signal.SIG_BLOCK, {signal.SIGINT, signal.SIGHUP, signal.SIGTERM}
)
timestamp = None
should_exit = False
while True:
if should_exit:
break
if timestamp is None:
timeout = timestamp
else:
timeout = timestamp - time.time()
assert timeout >= 0
logger.debug('Selecting on timeout {0}'.format(timeout))
events = selector.select(timeout)
if not events:
item = scheduler.pop()
item.execute()
timestamp = getattr(scheduler.top(), 'timestamp', None)
for key, mask in events:
callback = key.data
if not callable(callback):
should_exit = True
elif key.fileobj == sock:
item = callback(key.fileobj)
scheduler.push(item)
timestamp = scheduler.top().timestamp
else:
key.fileobj.read()
callback()
close_socket(sock)
def main(fd):
'''Run semaphore tracker.'''
# protect the process from ^C and "killall python" etc
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGTERM, signal.SIG_IGN)
if _HAVE_SIGMASK:
signal.pthread_sigmask(signal.SIG_UNBLOCK, _IGNORED_SIGNALS)
for f in (sys.stdin, sys.stdout):
try:
f.close()
except Exception:
pass
cache = set()
try:
# keep track of registered/unregistered semaphores
with open(fd, 'rb') as f:
for line in f:
try:
cmd, name = line.strip().split(b':')
if cmd == b'REGISTER':
cache.add(name)
elif cmd == b'UNREGISTER':
cache.remove(name)
else:
raise RuntimeError('unrecognized command %r' % cmd)
except Exception:
try:
sys.excepthook(*sys.exc_info())
except:
pass
finally:
# all processes have terminated; cleanup any remaining semaphores
if cache:
try:
warnings.warn('semaphore_tracker: There appear to be %d '
'leaked semaphores to clean up at shutdown' %
len(cache))
except Exception:
pass
for name in cache:
# For some reason the process which created and registered this
# semaphore has failed to unregister it. Presumably it has died.
# We therefore unlink it.
try:
name = name.decode('ascii')
try:
_multiprocessing.sem_unlink(name)
except Exception as e:
warnings.warn('semaphore_tracker: %r: %s' % (name, e))
finally:
pass
def run(self):
if hasattr(signal, 'pthread_sigmask'):
# Available on UNIX with Python 3.3+
signal.pthread_sigmask(signal.SIG_BLOCK, range(1, signal.NSIG))
while True:
if self.interval and self.snapshot_q.empty():
time.sleep(self.interval)
item = self.snapshot_q.get()
logger.info('Sending {0} snapshot file'.format(item))
logger.info('{0} items pending to send.'.format(
self.snapshot_q.qsize())
)
snapshot = tracemalloc.Snapshot.load(item)
self.client.send(snapshot)
def check_sigwait(self, wait_func):
signum = signal.SIGUSR1
pid = os.getpid()
old_handler = signal.signal(signum, lambda *args: None)
self.addCleanup(signal.signal, signum, old_handler)
code = '\n'.join((
'import os, time',
'pid = %s' % os.getpid(),
'signum = %s' % int(signum),
'sleep_time = %r' % self.sleep_time,
'time.sleep(sleep_time)',
'os.kill(pid, signum)',
))
old_mask = signal.pthread_sigmask(signal.SIG_BLOCK, [signum])
self.addCleanup(signal.pthread_sigmask, signal.SIG_UNBLOCK, [signum])
t0 = time.monotonic()
proc = self.subprocess(code)
with kill_on_error(proc):
wait_func(signum)
dt = time.monotonic() - t0
self.assertEqual(proc.wait(), 0)
def _serve(self):
if hasattr(signal, 'pthread_sigmask'):
signal.pthread_sigmask(signal.SIG_BLOCK, range(1, signal.NSIG))
while 1:
try:
with self._listener.accept() as conn:
msg = conn.recv()
if msg is None:
break
key, destination_pid = msg
send, close = self._cache.pop(key)
try:
send(conn, destination_pid)
finally:
close()
except:
if not util.is_exiting():
sys.excepthook(*sys.exc_info())
def run(self):
if hasattr(signal, 'pthread_sigmask'):
# Available on UNIX with Python 3.3+
signal.pthread_sigmask(signal.SIG_BLOCK, range(1, signal.NSIG))
while True:
logger.debug('Sleeping {0} secongs...'.format(self.interval))
time.sleep(self.interval)
filename = ("/tmp/tracemalloc-%d-%04d.dump"
% (os.getpid(), self.counter))
logger.info("Write snapshot into %s..." % filename)
gc.collect()
snapshot = tracemalloc.take_snapshot()
snapshot.dump(filename )
self.snapshot_q.put(filename)
logger.debug('Queue size: {0}'.format(self.snapshot_q.qsize()))
snapshot = None
logger.info("Snapshot written into %s" % filename)
self.counter += 1
def run(self):
with no_daemon_context(self.working_dir, self.lock_file, self.signals):
signal_map = {
signal.SIGHUP: self._handle_reconfigure,
signal.SIGINT: self._handle_shutdown,
signal.SIGTERM: self._handle_shutdown,
signal.SIGQUIT: self._handle_shutdown,
signal.SIGUSR1: self._handle_debug,
}
signal.pthread_sigmask(signal.SIG_BLOCK, signal_map.keys())
self._run_mcp()
self._run_www_api()
self._run_manhole()
self._run_reactor()
while True:
signum = signal.sigwait(list(signal_map.keys()))
if signum in signal_map:
logging.info(f"Got signal {str(signum)}")
signal_map[signum](signum, None)
def action_toggle(self, from_, chan, msg, parts):
t = time.time()
if t - self.last_toggle >= 5:
self.last_toggle = t
try:
signal.pthread_sigmask(signal.SIG_BLOCK, {signal.SIGUSR1})
subprocess.call(
["/home/simark/avr/serieViaUSB/serieViaUSB", "-e", "-f", "/home/simark/avr/serieViaUSB/fichier"])
self.irc.privmsg(chan, "Your wish is my command")
# time.sleep(1)
if get_plafond_status():
self.irc.privmsg(chan, "Light is now on")
else:
self.irc.privmsg(chan, "Light is now off")
except:
raise
finally:
signal.pthread_sigmask(signal.SIG_UNBLOCK, {signal.SIGUSR1})
else:
self.irc.privmsg(
chan, "You have to wait 5 seconds between two toggles.")
def test_pthread_sigmask_arguments(self):
self.assertRaises(TypeError, signal.pthread_sigmask)
self.assertRaises(TypeError, signal.pthread_sigmask, 1)
self.assertRaises(TypeError, signal.pthread_sigmask, 1, 2, 3)
self.assertRaises(OSError, signal.pthread_sigmask, 1700, [])
with self.assertRaises(ValueError):
signal.pthread_sigmask(signal.SIG_BLOCK, [signal.NSIG])
with self.assertRaises(ValueError):
signal.pthread_sigmask(signal.SIG_BLOCK, [0])
with self.assertRaises(ValueError):
signal.pthread_sigmask(signal.SIG_BLOCK, [1<<1000])
def test_pthread_sigmask_valid_signals(self):
s = signal.pthread_sigmask(signal.SIG_BLOCK, signal.valid_signals())
self.addCleanup(signal.pthread_sigmask, signal.SIG_SETMASK, s)
# Get current blocked set
s = signal.pthread_sigmask(signal.SIG_UNBLOCK, signal.valid_signals())
self.assertLessEqual(s, signal.valid_signals())
efd = linuxfd.eventfd(initval=0,semaphore=True,nonBlocking=True)
for i in range(0,3):
print(" writing to sempahore")
efd.write()
try:
while True:
value = efd.read()
print(" read '{}' from semaphore".format(value))
except BlockingIOError:
print(" semaphore exhausted")
#
# test signalfd
#
sfd = linuxfd.signalfd(signalset={signal.SIGALRM})
signal.pthread_sigmask(signal.SIG_SETMASK,{signal.SIGALRM})
print("\ntesting signalfd (fd={}) with SIGALRM".format(sfd.fileno()))
print("guarded signals = {}".format(sfd.signals()))
print("starting alarm timer (3 seconds)")
signal.alarm(3)
value = sfd.read()
print("received SIGALRM, signalfd.read() returned:")
pprint.pprint(value)
signal.alarm(0)
#
# test timerfd
#
tfd = linuxfd.timerfd(rtc=True)
print("\ntesting timerfd (fd={})".format(sfd.fileno()))
print(" {:.2f}: setting timer (value=3,interval=1)".format(time.time()))
import event
import event.log
import signal, os
signal.pthread_sigmask(signal.SIG_BLOCK, [signal.SIGCHLD])
e_loop=event.Loop(log=event.log.Stderr())
def c(cld, status):
print('Done', status)
@e_loop.main_callback
def stuff():
pid=event.Child._raw_spawn(
b'/usr/bin/sleep', (b'sleep', b'.5'), (), fd=(0,0,1,1,2,2)
)
cld=e_loop.new(event.Child, pid, callback_dead=c)
e_loop.run()
def block_all_signals():
"""Block asynchronous delivery of all signals to this process."""
if _HAS_SIGWAIT:
signal.pthread_sigmask(signal.SIG_BLOCK, _ALL_SIGNALS)
def signal_handler(signum, _):
signal.pthread_sigmask(signal.SIG_BLOCK, {signum})
logging.warning("Signal %d received, cleaning up...", signum)
cleanup()
logging.warning("done. Exiting with error.")
sys.exit(1)
def container_run(platform: str,
nvidia_runtime: bool,
docker_registry: str,
shared_memory_size: str,
local_ccache_dir: str,
command: List[str],
cleanup: Cleanup,
dry_run: bool = False) -> int:
"""Run command in a container"""
container_wait_s = 600
#
# Environment setup
#
environment = {
'CCACHE_MAXSIZE': '500G',
'CCACHE_TEMPDIR': '/tmp/ccache', # temp dir should be local and not shared
'CCACHE_DIR': '/work/ccache', # this path is inside the container as /work/ccache is
# mounted
'CCACHE_LOGFILE': '/tmp/ccache.log', # a container-scoped log, useful for ccache
# verification.
}
# These variables are passed to the container to the process tree killer can find runaway
# process inside the container
# https://wiki.jenkins.io/display/JENKINS/ProcessTreeKiller
# https://github.com/jenkinsci/jenkins/blob/578d6bacb33a5e99f149de504c80275796f0b231/core/src/main/java/hudson/model/Run.java#L2393
#
jenkins_env_vars = ['BUILD_NUMBER', 'BUILD_ID', 'BUILD_TAG']
environment.update({k: os.environ[k] for k in jenkins_env_vars if k in os.environ})
environment.update({k: os.environ[k] for k in ['CCACHE_MAXSIZE'] if k in os.environ})
tag = get_docker_tag(platform=platform, registry=docker_registry)
mx_root = get_mxnet_root()
local_build_folder = buildir()
# We need to create it first, otherwise it will be created by the docker daemon with root only permissions
os.makedirs(local_build_folder, exist_ok=True)
os.makedirs(local_ccache_dir, exist_ok=True)
logging.info("Using ccache directory: %s", local_ccache_dir)
docker_client = docker.from_env()
# Equivalent command
docker_cmd_list = [
get_docker_binary(nvidia_runtime),
'run',
"--cap-add",
"SYS_PTRACE", # Required by ASAN
'--rm',
'--shm-size={}'.format(shared_memory_size),
# mount mxnet root
'-v', "{}:/work/mxnet".format(mx_root),
# mount mxnet/build for storing build
'-v', "{}:/work/build".format(local_build_folder),
'-v', "{}:/work/ccache".format(local_ccache_dir),
'-u', '{}:{}'.format(os.getuid(), os.getgid()),
'-e', 'CCACHE_MAXSIZE={}'.format(environment['CCACHE_MAXSIZE']),
# temp dir should be local and not shared
'-e', 'CCACHE_TEMPDIR={}'.format(environment['CCACHE_TEMPDIR']),
# this path is inside the container as /work/ccache is mounted
'-e', "CCACHE_DIR={}".format(environment['CCACHE_DIR']),
# a container-scoped log, useful for ccache verification.
'-e', "CCACHE_LOGFILE={}".format(environment['CCACHE_LOGFILE']),
'-ti',
tag]
docker_cmd_list.extend(command)
docker_cmd = ' \\\n\t'.join(docker_cmd_list)
logging.info("Running %s in container %s", command, tag)
logging.info("Executing the equivalent of:\n%s\n", docker_cmd)
# return code of the command inside docker
ret = 0
if not dry_run:
#############################
#
signal.pthread_sigmask(signal.SIG_BLOCK, {signal.SIGINT, signal.SIGTERM})
# noinspection PyShadowingNames
runtime = None
if nvidia_runtime:
# noinspection PyShadowingNames
# runc is default (docker info | grep -i runtime)
runtime = 'nvidia'
container = docker_client.containers.run(
tag,
runtime=runtime,
detach=True,
command=command,
shm_size=shared_memory_size,
user='******'.format(os.getuid(), os.getgid()),
cap_add='SYS_PTRACE',
volumes={
mx_root:
{'bind': '/work/mxnet', 'mode': 'rw'},
local_build_folder:
{'bind': '/work/build', 'mode': 'rw'},
local_ccache_dir:
{'bind': '/work/ccache', 'mode': 'rw'},
},
environment=environment)
try:
logging.info("Started container: %s", trim_container_id(container.id))
# Race condition:
# If the previous call is interrupted then it's possible that the container is not cleaned up
# We avoid by masking the signals temporarily
cleanup.add_container(container)
signal.pthread_sigmask(signal.SIG_UNBLOCK, {signal.SIGINT, signal.SIGTERM})
#
#############################
stream = container.logs(stream=True, stdout=True, stderr=True)
sys.stdout.flush()
for chunk in stream:
sys.stdout.buffer.write(chunk)
sys.stdout.buffer.flush()
sys.stdout.flush()
stream.close()
try:
logging.info("Waiting for status of container %s for %d s.",
trim_container_id(container.id),
container_wait_s)
wait_result = container.wait(timeout=container_wait_s)
logging.info("Container exit status: %s", wait_result)
ret = wait_result.get('StatusCode', 200)
except Exception as e:
logging.exception(e)
ret = 150
# Stop
try:
logging.info("Stopping container: %s", trim_container_id(container.id))
container.stop()
except Exception as e:
logging.exception(e)
ret = 151
# Remove
try:
logging.info("Removing container: %s", trim_container_id(container.id))
container.remove()
except Exception as e:
logging.exception(e)
ret = 152
cleanup.remove_container(container)
containers = docker_client.containers.list()
if containers:
logging.info("Other running containers: %s", [trim_container_id(x.id) for x in containers])
except docker.errors.NotFound as e:
logging.info("Container was stopped before cleanup started: %s", e)
return ret
#!/usr/bin/python3
import signal
import sys
import time
def signal_handler(signal, frame):
print('Received sigquit, doing graceful shutdown')
time.sleep(0.5)
sys.exit(0)
print('Running {}'.format(str(sys.argv)))
if sys.argv[1] == '-v':
print('Asked for version')
sys.exit(0)
# The parent golang process blocks SIGQUIT in subprocesses, for some reason.
# So we unblock it manually - same as what nginx does.
signal.pthread_sigmask(signal.SIG_UNBLOCK, {signal.SIGQUIT})
signal.signal(signal.SIGQUIT, signal_handler)
signal.pause
time.sleep(5)
print('Should have handled SIGQUIT')
sys.exit(-1)
def ignore_signals(signals):
signal.pthread_sigmask(signal.SIG_BLOCK, signals)
yield
signal.pthread_sigmask(signal.SIG_UNBLOCK, signals)
def reset_signal_handling():
if _HAS_SIGWAIT:
signal.pthread_sigmask(signal.SIG_SETMASK, {})
def unblock_signals():
signal.pthread_sigmask(signal.SIG_UNBLOCK, [signal.SIGTERM, signal.SIGINT])
def run(self):
if hasattr(signal, 'pthread_sigmask'):
signal.pthread_sigmask(signal.SIG_BLOCK, range(1, signal.NSIG))
super(WebSocketClient, self).run()
