def daemonize():
"""Standard daemonization of a process.
"""
# guard to prevent daemonization with gevent loaded
for module in sys.modules.keys():
if module.startswith('gevent'):
raise ValueError('Cannot daemonize if gevent is loaded')
if hasattr(os, 'fork'):
child_pid = os.fork()
else:
raise ValueError("Daemonizing is not available on this platform.")
if child_pid != 0:
# we're in the parent
os._exit(0)
# child process
os.setsid()
subchild = os.fork()
if subchild:
os._exit(0)
# subchild
maxfd = get_maxfd()
closerange(0, maxfd)
os.open(REDIRECT_TO, os.O_RDWR)
os.dup2(0, 1)
os.dup2(0, 2)
def preexec_fn():
fd = socket.fileno()
os.environ['LISTEN_FDS'] = '1'
os.environ['LISTEN_PID'] = str(os.getpid())
if fd != 3:
os.dup2(fd, 3)
os.closerange(4, 100)
def forkexec(*argv):
"""Fork a child process."""
child_pid = os.fork()
if child_pid == 0:
os.closerange(3, MAXFD)
os.execv(argv[0], argv)
return child_pid
def start_daemon(mainfunc, pidfile, termfunc=__default_term_handler):
if not pidfile is None:
atexit.register(__remove_pid_file, pidfile)
if not termfunc is None:
signal.signal(signal.SIGTERM, termfunc)
try:
pid = os.fork()
except OSError as e:
raise Exception("%s [%d]" % (e.strerror, e.errno))
if pid == 0:
os.closerange(0,3)
os.setsid()
try:
pid = os.fork()
except OSError as e:
raise Exception("%s [%d]" % (e.strerror, e.errno))
if pid == 0:
mainfunc()
else:
if not pidfile is None:
with open(pidfile, "w") as pf:
pf.write(str(pid))
os._exit(0)
else:
os._exit(0)
def daemonize(logger, stdin='/dev/null', stdout='/dev/null', stderr='/dev/null'):
# do the UNIX double-fork magic, see Stevens' "Advanced Programming in the UNIX Environment" for details
logger.info('daemonize(stdin={0:s}, stdout={1:s}, stderr={2:s}) enter'.format(stdin, stdout, stderr))
(soft_flimit, hard_flimit) = resource.getrlimit(resource.RLIMIT_NOFILE)
_fork(1, logger)
# decouple from parent environment
os.chdir("/")
os.setsid()
os.umask(0)
# ignore SIGHUP
_ignore_signal(signal.SIGHUP, logger)
# do second fork
_fork(2, logger)
sys.stdout.flush()
sys.stderr.flush()
# close all file descriptors
os.closerange(0, hard_flimit)
# redirect standard file descriptors
si = open(stdin, 'r')
so = open(stdout, 'a+')
se = open(stderr, 'a+', 0)
os.dup2(si.fileno(), sys.stdin.fileno())
os.dup2(so.fileno(), sys.stdout.fileno())
os.dup2(se.fileno(), sys.stderr.fileno())
logger.info('daemonize(stdin={0:s}, stdout={1:s}, stderr={2:s}) exit'.format(stdin, stdout, stderr))
def exit():
"""
Causes python to exit without garbage-collecting any objects, and thus avoids
calling object destructor methods. This is a sledgehammer workaround for
a variety of bugs in PyQt and Pyside that cause crashes on exit.
This function does the following in an attempt to 'safely' terminate
the process:
* Invoke atexit callbacks
* Close all open file handles
* os._exit()
Note: there is some potential for causing damage with this function if you
are using objects that _require_ their destructors to be called (for example,
to properly terminate log files, disconnect from devices, etc). Situations
like this are probably quite rare, but use at your own risk.
"""
## first disable our own cleanup function; won't be needing it.
setConfigOptions(exitCleanup=False)
## invoke atexit callbacks
atexit._run_exitfuncs()
## close file handles
if sys.platform == 'darwin':
for fd in range(3, 4096):
if fd not in [7]: # trying to close 7 produces an illegal instruction on the Mac.
os.close(fd)
else:
os.closerange(3, 4096) ## just guessing on the maximum descriptor count..
os._exit(0)
def popen(cmd, stdin=None, stdout=None, wait=False):
if stdin is None:
_stdin = stdin = open('/dev/null', 'rb')
if not isinstance(stdin, int):
stdin = stdin.fileno()
if stdout is None:
_stdout = stdout = open('/dev/null', 'wb')
if not isinstance(stdout, int):
stdout = stdout.fileno()
# exec
if isinstance(cmd, basestring):
cmd = [cmd]
else:
cmd = list(cmd)
cmd = ["/bin/sh", "-c"]+cmd
gc_was_enabled = gc.isenabled()
gc.disable()
try:
pid = os.fork()
except:
if gc_was_enabled:
gc.enable()
raise
if pid == 0:
if stdout == 0:
stdout = os.dup(stdout)
os.dup2(stdin, 0)
os.dup2(stdout, 1)
if hasattr(os, 'closerange'):
os.closerange(3, MAXFD)
else:
for i in xrange(3, MAXFD):
try:
os.close(i)
except:
pass
os.execvp(cmd[0], cmd)
os._exit(255)
# parent
if gc_was_enabled:
gc.enable()
if stdin is not None and stdin != 0:
os.close(stdin)
if stdout is not None and stdout != 1:
os.close(stdout)
if wait:
pid, sts = os.waitpid(pid, 0)
if os.WIFSIGNALED(sts):
return os.WTERMSIG(sts)
elif os.WIFEXITED(sts):
return os.WEXITSTATUS(sts)
return pid
def restart(self, signum=None, frame=None):
LicornEvent('daemon_is_restarting').emit(priorities.HIGH)
assert ltrace(TRACE_DAEMON, '| restart(signum={0}, frame={1})',
(ST_UGID, signum), frame)
if signum:
logging.notice(_(u'{0}: SIGUSR1 received, preparing our '
u'restart.').format(self))
else:
logging.notice(_(u'{0}: restart needed, shutting '
u'everything down.').format(self))
if hasattr(self, 'daemon_shutdown'):
self.daemon_shutdown()
# close every file descriptor (except stdin/out/err, used for logging and
# on the console). This is needed for Pyro thread to release its socket,
# else it's done too late and on restart the port can't be rebinded on.
os.closerange(3, 32)
# unlink the pid_file, else the new exec'ed process will try to kill
# itself in the check_or_replace() method.
self.unlink_pid_file()
self.restart_command()
def _close_fds(self, keep):
# `keep` is a set of fds, so we
# use os.closerange from 3 to min(keep)
# and then from max(keep + 1) to MAXFD and
# loop through filling in the gaps.
# Under new python versions, we need to explicitly set
# passed fds to be inheritable or they will go away on exec
if hasattr(os, 'set_inheritable'):
set_inheritable = os.set_inheritable
else:
set_inheritable = lambda i, v: True
if hasattr(os, 'closerange'):
keep = sorted(keep)
min_keep = min(keep)
max_keep = max(keep)
os.closerange(3, min_keep)
os.closerange(max_keep + 1, MAXFD)
for i in xrange(min_keep, max_keep):
if i in keep:
set_inheritable(i, True)
continue
try:
os.close(i)
except:
pass
else:
for i in xrange(3, MAXFD):
if i in keep:
set_inheritable(i, True)
continue
try:
os.close(i)
except:
pass
def add_nested_seat(self, *xephyr_args):
'Start a nested display'
def xephyr_signal_handler(sig, frame):
# Fugly, nonlocal (Py3K+) would make this prettier
xephyr_signal_handler.was_called = True
def setup_xephyr_handler():
xephyr_signal_handler.original_handler = signal.getsignal(signal.SIGUSR1)
xephyr_signal_handler.was_called = False
signal.signal(signal.SIGUSR1, xephyr_signal_handler)
def wait_for_xephyr(pid):
try:
os.waitpid(pid, 0)
except: # On purpose
pass
signal.signal(signal.SIGUSR1, xephyr_signal_handler.original_handler)
return xephyr_signal_handler.was_called
xephyr_argv = ['Xephyr']
# Determine the display number to use for Xephyr
for arg in xephyr_args:
if arg.startswith(':'):
try:
xephyr_display_number = int(arg.lstrip(':'))
break
except ValueError:
continue
else:
xephyr_display_number = get_free_display_number()
xephyr_argv += ':{0}'.format(xephyr_display_number)
xephyr_argv.extend(xephyr_args)
# Wait for signal from Xephyr when it is ready
setup_xephyr_handler()
# Spawn Xephyr
xephyr_pid = os.fork()
if xephyr_pid == 0:
# In child
os.closerange(0, 1023)
# This makes X(ephyr) SIGUSR1 its parent when ready.
signal.signal(signal.SIGUSR1, signal.SIG_IGN)
try:
os.execvp(xephyr_argv[0], xephyr_argv)
except OSError as e:
sys.exit(e.errno)
# Wait for Xephyr to signal us
if wait_for_xephyr(xephyr_pid):
try:
return self._dbus_call('AddLocalXSeat', xephyr_display_number)
except Exception as e:
os.kill(xephyr_pid, signal.SIGQUIT)
raise Exception('Unable to add seat: {0}'.format(e))
else:
raise Exception('Xephyr launch failed')
def start_stud(exe, front_port, back_port, hmac_key, capath, keycertfile):
ppid = os.getpid()
pid = os.fork()
if pid > 0:
return pid
try:
MAXFD = os.sysconf("SC_OPEN_MAX")
except:
MAXFD = 256
os.closerange(0, MAXFD)
cmd = [exe, "--backend", "[127.0.0.1]:%d" % back_port,
"--frontend", "[*]:%d" % front_port,
"--verify-depth", "10", "--verify-proxy", "--verify-require",
"--inject-chain", "--ca-path", capath,
"--hmac-key", hmac_key.encode("hex"),
keycertfile]
try:
os.execvp(cmd[0], cmd)
raise RuntimeError("execvp failed for cmd: " + repr(cmd))
except Exception, exc:
sys.stderr = open("/dev/tty", "w")
sys.stderr.write("Failed to start stud: " + str(exc)+"\n")
sys.stderr.write("Command line was:\n")
sys.stderr.write(" ".join(cmd) + "\n")
sys.stderr.flush()
os.kill(ppid, 15)
sys.exit(1)
def startDaemon(mainFunc, pidfile=None, termFunc=__defaultTermHandler, closeStds=True):
if closeStds:
os.closerange(0,3)
if not pidfile is None:
atexit.register(__removePidfile, pidfile)
if not termFunc is None:
signal.signal(signal.SIGTERM, termFunc)
try:
pid = os.fork()
except OSError as e:
raise Exception, "%s [%d]" % (e.strerror, e.errno)
if pid == 0:
os.setsid()
try:
pid = os.fork()
except OSError as e:
raise Exception, "%s [%d]" % (e.strerror, e.errno)
if pid == 0:
mainFunc()
else:
if not pidfile is None:
with open(pidfile, "w") as pf:
pf.write(str(pid))
os._exit(0)
else:
os._exit(0)
def daemonize():
"""Daemonize the current process."""
# fork the parent process
pid = os.fork()
if pid < 0:
print("failed to fork parent process\n", file=sys.stderr)
sys.exit(1)
elif pid > 0:
# we are the parent process so we exit. the child process continues
# running under the init process and independent of any shell the
# parent was invoked in
sys.exit(0)
# start a new session and disconnect from parent's controlling terminal
try:
os.setsid()
except OSError as e:
print(e)
syslog(LOG_ERR, "failed to create a new session")
sys.exit(1)
# fork the parent process again so any terminal devices we open will not
# become the controlling terminal
pid = os.fork()
if pid < 0:
syslog(LOG_ERR, "failed to fork parent process for the second time")
sys.exit(1)
elif pid > 0:
# we are the parent process, so quit
sys.exit(0)
# clear the process umask so any files or directories we create will have
# the requested permissions
os.umask(0)
# change the working directory to the root directory to avoid locking any
# non-root filesystem thereby preventing it from being unmounted
os.chdir("/")
# close all open file descriptors to remove any references to our original
# terminal and to avoid locking any non-root filesystems
open_fd = os.listdir("/proc/{}/fd".format(os.getpid()))
fd_low = int(open_fd[0])
fd_high = int(open_fd[len(open_fd) - 1])
os.closerange(fd_low, fd_high + 1)
# close the standard output stream and re-open it using /dev/null
# note that because we closed all open file descriptors, the next time
# we open a file the descriptor should start with 0 which is STDIN_FILENO
sys.stdin.close()
nullfd = os.open(os.devnull, os.O_RDWR)
if nullfd != STDIN_FILENO:
syslog(LOG_ERR, "failed to open /dev/null as stdin")
sys.exit(1)
# duplicate the stdin file descriptor for stdout and stderr
os.dup2(STDIN_FILENO, STDOUT_FILENO)
os.dup2(STDIN_FILENO, STDERR_FILENO)
def _clean_fork(self, output=None):
pipe_r, pipe_w = os.pipe()
pid = os.fork()
if pid:
self.children.append(pid)
os.close(pipe_w)
return pipe_r, False
else:
try:
self.children = []
self.toplevel = False
os.chdir(self.working_dir)
sys.stdin.close()
# Move the pipe fd to position 3, close everything above it
os.dup2(pipe_w, 3)
os.closerange(4, MAXFD)
pipe_w = 3
if output is not None:
with open(os.path.join(self.working_dir, output), "w") as fp:
os.dup2(fp.fileno(), 1)
os.dup2(fp.fileno(), 2)
else:
sys.stdout.close()
sys.stderr.close()
except:
try:
traceback.print_exc()
finally:
os._exit(1)
return pipe_w, True
def main():
if iswindows:
if '--multiprocessing-fork' in sys.argv:
# We are using the multiprocessing module on windows to launch a
# worker process
from multiprocessing import freeze_support
freeze_support()
return 0
# Close open file descriptors inherited from parent
# On Unix this is done by the subprocess module
os.closerange(3, 256)
if isosx and 'CALIBRE_WORKER_ADDRESS' not in os.environ and 'CALIBRE_SIMPLE_WORKER' not in os.environ and '--pipe-worker' not in sys.argv:
# On some OS X computers launchd apparently tries to
# launch the last run process from the bundle
# so launch the gui as usual
from calibre.gui2.main import main as gui_main
return gui_main(['calibre'])
csw = os.environ.get('CALIBRE_SIMPLE_WORKER', None)
if csw:
mod, _, func = csw.partition(':')
mod = importlib.import_module(mod)
func = getattr(mod, func)
func()
return
if '--pipe-worker' in sys.argv:
try:
exec (sys.argv[-1])
except Exception:
print('Failed to run pipe worker with command:', sys.argv[-1])
raise
return
address = cPickle.loads(unhexlify(os.environ['CALIBRE_WORKER_ADDRESS']))
key = unhexlify(os.environ['CALIBRE_WORKER_KEY'])
resultf = unhexlify(os.environ['CALIBRE_WORKER_RESULT']).decode('utf-8')
with closing(Client(address, authkey=key)) as conn:
name, args, kwargs, desc = eintr_retry_call(conn.recv)
if desc:
prints(desc)
sys.stdout.flush()
func, notification = get_func(name)
notifier = Progress(conn)
if notification:
kwargs[notification] = notifier
notifier.start()
result = func(*args, **kwargs)
if result is not None and os.path.exists(os.path.dirname(resultf)):
cPickle.dump(result, open(resultf, 'wb'), -1)
notifier.queue.put(None)
try:
sys.stdout.flush()
except EnvironmentError:
pass # Happens sometimes on OS X for GUI processes (EPIPE)
try:
sys.stderr.flush()
except EnvironmentError:
pass # Happens sometimes on OS X for GUI processes (EPIPE)
return 0
def run(self):
self.pid = os.fork()
if not self.pid:
os.close(self.splerr)
stdin = os.open('/dev/zero', os.O_RDONLY)
os.dup2(stdin, 0)
os.close(stdin)
os.dup2(self.stdoutIN, 1)
os.dup2(self.stderrIN, 2)
os.dup2(self.splerrIN, 255)
os.closerange(3, 255)
try:
_cmd = shlex.split(self.cmd)
except ValueError, E:
os.write(255, E.args[0])
os._exit(255)
try:
os.execvp(_cmd[0], _cmd)
except OSError, E:
os.write(255, E.args[1])
os._exit(E.errno)
def reopen_std_streams(self):
# stdin/stdout on a forked process should have assured
# desinations instead of a possibly vanishing controlling TTY.
# Closure is an acceptable solution, but redirection to the
# null device provides a stable destination and thus ensures
# that subsequent write operations do not raise exceptions. Note:
#
# * If stdin/stderr remain open/unredirected, SSH sessions
# will block when an interactive shell is exited because
# SSH is waiting for forthcoming data. The presence of an
# open/unredirected stderr does not have this effect.
#
# * Merely redirecting sys.std* is inadequate because copies
# of the original FDs are preserved in sys.__std*__ and
# remain open unless explicitely closed.
if self.full_closure:
try:
max_fd = os.sysconf("SC_OPEN_MAX")
except:
max_fd = 1024
os.closerange(0, max_fd)
with open(self.stdin, 'r') as stdin:
os.dup2(stdin.fileno(), sys.stdin.fileno())
os.dup2(stdin.fileno(), sys.__stdin__.fileno())
with open(self.stdout, 'a+') as stdout:
os.dup2(stdout.fileno(), sys.stdout.fileno())
os.dup2(stdout.fileno(), sys.__stdout__.fileno())
if self.stderr:
with open(self.stderr, 'a+', 0) as stderr:
os.dup2(stderr.fileno(), sys.stderr.fileno())
os.dup2(stderr.fileno(), sys.__stderr__.fileno())
def forkexec_pty(*argv):
"""Fork a child process attached to a pty."""
child_pid, child_fd = pty.fork()
if child_pid == 0:
os.closerange(3, MAXFD)
os.execv(argv[0], argv)
return child_pid, child_fd
def close_all_fds(except_fds=None):
"""
Closes all fds outside of stdout,stderr for this process/subprocess.
:param except_fds: list of files, or fds to 'not' close
"""
except_filenos = [1, 2]
if except_fds is not None:
for except_fd in except_fds:
if except_fd is None:
pass
elif isinstance(except_fd, int):
except_filenos.append(except_fd)
elif hasattr(except_fd, "fileno"):
except_filenos.append(except_fd.fileno())
else:
raise ValueError("{0} must be an int or have a fileno method".format(repr(except_fd)))
fileno_ranges = []
next_range_min = 0
for except_fileno in sorted(except_filenos):
if except_fileno > next_range_min:
fileno_ranges.append((next_range_min, except_fileno))
next_range_min = max(next_range_min, except_fileno + 1)
fileno_ranges.append((next_range_min, 1024))
for fileno_range in fileno_ranges:
os.closerange(fileno_range[0], fileno_range[1])
def daemonize():
"""Standard daemonization of a process.
"""
# guard to prevent daemonization with gevent loaded
for module in sys.modules.keys():
if module.startswith('gevent'):
raise ValueError('Cannot daemonize if gevent is loaded')
child_pid = os.fork()
if child_pid != 0:
# we're in the parent
os._exit(0)
# child process
os.setsid()
subchild = os.fork()
if subchild:
os._exit(0)
# subchild
os.umask(0)
maxfd = get_maxfd()
closerange(0, maxfd)
os.open(REDIRECT_TO, os.O_RDWR)
os.dup2(0, 1)
os.dup2(0, 2)
def _exec_gpg(self, stdin, stdout, stderr, statusfd):
try:
if stdin != 0:
os.dup2(stdin, 0)
if stdout != 1:
os.dup2(stdout, 1)
if stderr != 2:
os.dup2(stderr, 2)
if statusfd != 3:
os.dup2(statusfd, 3)
for fd in range(4):
old = fcntl.fcntl(fd, fcntl.F_GETFD)
fcntl.fcntl(fd, fcntl.F_SETFD, old & ~fcntl.FD_CLOEXEC)
os.closerange(4, _MAXFD)
args = [self.gpg,
"--status-fd=3",
"--no-default-keyring",
"--batch",
"--no-tty",
"--trust-model", "always",
"--fixed-list-mode"]
for k in self.keyrings:
args.append("--keyring=%s" % k)
args.extend(["--decrypt", "-"])
os.execvp(self.gpg, args)
finally:
os._exit(1)
def spawn(prog, args):
p2cread, p2cwrite = os.pipe()
c2pread, c2pwrite = os.pipe()
pid = os.fork()
if pid == 0:
# Child
for i in 0, 1, 2:
try:
os.close(i)
except os.error:
pass
if os.dup(p2cread) != 0:
sys.stderr.write('popen2: bad read dup\n')
if os.dup(c2pwrite) != 1:
sys.stderr.write('popen2: bad write dup\n')
if os.dup(c2pwrite) != 2:
sys.stderr.write('popen2: bad write dup\n')
os.closerange(3, MAXFD)
try:
os.execvp(prog, args)
finally:
sys.stderr.write('execvp failed\n')
os._exit(1)
os.close(p2cread)
os.close(c2pwrite)
return pid, c2pread, p2cwrite
def _close_fds(pass_fds):
pass_fds = sorted(pass_fds)
previous = 3
for fd in pass_fds:
os.closerange(previous, fd)
previous = fd + 1
os.closerange(previous, MAXFD)
def _close_fds(self, fds_to_keep):
start_fd = 3
for fd in sorted(fds_to_keep):
while fd >= start_fd:
os.closerange(start_fd, fd)
start_fd = fd + 1
if start_fd <= MAXFD:
os.closerange(start_fd, MAXFD)
def _run_child(self, cmd):
if isinstance(cmd, basestring):
cmd = ['/bin/sh', '-c', cmd]
os.closerange(3, MAXFD)
try:
os.execvp(cmd[0], cmd)
finally:
os._exit(1)
def close_fds(self):
logger.debug("Closing open file descriptors...")
maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
if maxfd == resource.RLIM_INFINITY:
maxfd = 1024
# Do not close stdin, stdout and stderr (0, 1, 2)
os.closerange(3, maxfd)
def close_open_fds(keep=None):
# must make sure this is 0-inclusive (Issue #1882)
keep = list(uniq(sorted(f for f in map(maybe_fileno, keep or []) if f is not None)))
maxfd = get_fdmax(default=2048)
kL, kH = iter([-1] + keep), iter(keep + [maxfd])
for low, high in zip_longest(kL, kH):
if low + 1 != high:
os.closerange(low + 1, high)
def command_serve(self):
"""
Run both HTTP and WS servers for development purpouse
"""
def http_process():
print('Staring HTTP server process...')
from admin import App as HTTPApp
from admin import Cfg as HTTPCfg
from wsgiref.simple_server import make_server
cfg = HTTPCfg()
cfg.update_from_py()
app = HTTPApp(cfg)
server = make_server(cfg.HTTP_SERVER_HOST, cfg.HTTP_SERVER_PORT, app)
server.serve_forever()
def ws_process():
print('Starting WS server process...')
from ws_admin import App as WSApp
from ws_admin import Cfg as WSCfg
from ikcms.ws_servers.websockets import WS_Server
cfg = WSCfg()
cfg.update_from_py()
app = WSApp(cfg)
server = WS_Server(cfg.WS_SERVER_HOST, cfg.WS_SERVER_PORT, app)
server.serve_forever()
p1 = Process(target=http_process)
p2 = Process(target=ws_process)
p1.start()
p2.start()
try:
wait_for_code_change()
p1.terminate()
p1.join()
p2.terminate()
p2.join()
flush_fds()
pid = os.fork()
if pid:
os.closerange(3, MAXFD)
os.waitpid(pid, 0)
os.execvp(sys.executable, [sys.executable] + sys.argv)
else:
sys.exit()
except KeyboardInterrupt:
print('Terminating HTTP and WS servers...')
p1.terminate()
p2.terminate()
sys.exit()
def create(self, rows=24, cols=80, env=None):
"""
Creates a new virtual terminal (tty) and executes self.cmd within it.
Also sets up our read/write callback and attaches them to Tornado's
IOLoop.
*cols*
The number of columns to emulate on the virtual terminal (width)
*rows*
The number of rows to emulate (height).
*env*
A dictionary of environment variables to set when executing self.cmd.
"""
logging.debug("create(rows=%s, cols=%s, env=%s)" % (rows, cols, repr(env)))
pid, fd = pty.fork()
if pid == 0: # We're inside the child process
# Close all file descriptors other than stdin, stdout, and stderr (0, 1, 2)
try:
os.closerange(3, 256)
except OSError:
pass
if not env:
env = {}
stdin = 0
stdout = 1
stderr = 2
#env["COLUMNS"] = str(cols)
#env["LINES"] = str(rows)
env["TERM"] = "xterm" # TODO: This needs to be configurable on-the-fly
env["PATH"] = os.environ['PATH']
#env["LANG"] = os.environ['LANG']
# Setup stdout to be more Gate One friendly
# Set non-blocking so we don't wait forever for a read()
fcntl.fcntl(stdout, fcntl.F_SETFL, os.O_NONBLOCK)
p = Popen(
self.cmd,
env=env,
shell=True,
stdin=stdin,
stdout=stdout,
stderr=stderr)
p.wait()
# This exit() ensures IOLoop doesn't freak out about a missing fd
sys.exit(0)
else: # We're inside this Python script
self.fd = fd
self.pid = pid
self.time = time.time()
self.term = self.terminal_emulator(rows=rows, cols=cols)
# Tell our IOLoop instance to start watching the child
self.io_loop.add_handler(
fd, self.proc_read, self.io_loop.READ)
self.prev_output = [None for a in xrange(rows-1)]
# Set non-blocking so we don't wait forever for a read()
fcntl.fcntl(self.fd, fcntl.F_SETFL, os.O_NONBLOCK)
# Set the size of the terminal
self.resize(rows, cols)
return fd
def close_open_fds(keep=None):
keep = list(uniq(sorted(filter(None, (
maybe_fileno(f) for f in keep or []
)))))
maxfd = get_fdmax(default=2048)
kL, kH = iter([-1] + keep), iter(keep + [maxfd])
for low, high in zip_longest(kL, kH):
if low + 1 != high:
os.closerange(low + 1, high)
def _daemonize():
if not 'fork' in os.__dict__:
return
thispid = os.fork()
if thispid > 0:
os.waitpid(thispid, 0)
os._exit(0)
os.setsid()
thispid = os.fork()
if thispid > 0:
print('confluent server starting as pid {0}'.format(thispid))
os._exit(0)
os.closerange(0, 2)
os.open(os.devnull, os.O_RDWR)
os.dup2(0, 1)
os.dup2(0, 2)
log.daemonized = True
def closefrom(lowfd):
# type: (int) -> None
"""Closes all file descriptors starting with "lowfd", ignoring errors
Deletes all open file descriptors greater than or equal to lowfd from the
per-process object reference table. Any errors encountered while closing
file descriptors are ignored.
Difference to os.closerange() is that this automatically determines the
highest fd number to close.
"""
try:
highfd = os.sysconf("SC_OPEN_MAX")
except ValueError:
highfd = 1024
os.closerange(lowfd, highfd)
def test_socket_activation(self):
'Test socket activation'
os.closerange(3, 4) # Ensure the socket gets fileno == 3
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
s.bind(('localhost', 0))
port = s.getsockname()[1]
self.ae(s.fileno(), 3)
os.environ['LISTEN_PID'] = unicode_type(os.getpid())
os.environ['LISTEN_FDS'] = '1'
with TestServer(lambda data: (data.path[0] + data.read()),
allow_socket_preallocation=True) as server:
conn = server.connect()
conn.request('GET', '/test', 'body')
r = conn.getresponse()
self.ae(r.status, http_client.OK)
self.ae(r.read(), b'testbody')
self.ae(server.loop.bound_address[1], port)
def __call__(self):
"""Run the app."""
# ensure no weird file descriptors are open.
os.closerange(3, MAXFD)
# config guarantees this is set
command = self.pgconf['command']
# argparse guarantees this is an attribute
command = getattr(self, command)
try:
result = command()
except PgctlUserError as error:
# we don't need or want a stack trace for user errors
result = str(error)
if isinstance(result, basestring):
return CHANNEL + ' ERROR: ' + result
else:
return result
def test_closerange(self):
os = self.posix
if not hasattr(os, 'closerange'):
skip("missing os.closerange()")
fds = [
os.open(self.path + str(i), os.O_CREAT | os.O_WRONLY, 0o777)
for i in range(15)
]
fds.sort()
start = fds.pop()
stop = start + 1
while len(fds) > 3 and fds[-1] == start - 1:
start = fds.pop()
os.closerange(start, stop)
for fd in fds:
os.close(fd) # should not have been closed
for fd in range(start, stop):
raises(OSError, os.fstat, fd) # should have been closed
def audioProcess(argv):
global AUDIO_PROCESS
debug('Run audioProcess: %s\n' % str(argv))
try:
audioKill()
os.system("sync && echo 3 > /proc/sys/vm/drop_caches && echo 1 > /proc/sys/vm/overcommit_memory")
AUDIO_PROCESS = os.fork()
if not AUDIO_PROCESS:
debug('Child audioProcess() argv: %s\n' % str(argv))
os.closerange(3,1024)
os.setsid()
debug('Child audioProcess() execv(): %s\n' % str(argv))
os.execv(argv[0], argv)
debug('Parent audioProcess() fork(): %s\n' % str(AUDIO_PROCESS))
except:
debug('Parent audioProcess() forking exception: %s\n' % str(argv))
debug(traceback.format_exc())
os.system("sync && echo 3 > /proc/sys/vm/drop_caches && echo 0 > /proc/sys/vm/overcommit_memory")
def daemonize():
"""Standard daemonization of a process.
"""
#if not 'CIRCUS_PID' in os.environ:
if os.fork():
os._exit(0)
os.setsid()
if os.fork():
os._exit(0)
os.umask(0)
maxfd = get_maxfd()
closerange(0, maxfd)
os.open(REDIRECT_TO, os.O_RDWR)
os.dup2(0, 1)
os.dup2(0, 2)
def chal_watcher(paths, procs, timeout, log):
# Continue until any of the processes die
# Wait until any process exits
start = time()
while time() - start < timeout \
and all(proc.poll() is None for proc in procs):
sleep(0.1)
# Give the others a chance to exit
while time() - start < timeout \
and any(proc.poll() is None for proc in procs):
sleep(0.1)
# Kill any remaining processes
for proc in procs:
if proc.poll() is None:
proc.terminate()
proc.wait()
# Close all of the ipc pipes
if len(procs) > 1:
last_fd = 2 * len(procs) + 3
os.closerange(3, last_fd)
# If any of the processes crashed, print out crash info
for path, proc in zip(paths, procs):
pid, sig = proc.pid, abs(proc.returncode)
if sig not in [None, 0, signal.SIGTERM]:
log('[DEBUG] pid: {}, sig: {}'.format(pid, sig))
# Attempt to get register values
regs = get_core_dump_regs(path, pid, log)
if regs is not None:
# If a core dump was generated, report this as a crash
# log('Process generated signal (pid: {}, signal: {}) - {}\n'.format(pid, sig, testpath))
log('Process generated signal (pid: {}, signal: {})'.format(pid, sig))
# Report the register states
reg_str = ' '.join(['{}:{}'.format(reg, val) for reg, val in regs.iteritems()])
log('register states - {}'.format(reg_str))
# Final cleanup
clean_cores(paths, procs)
def daemonize(enable_stdio_inheritance=False):
"""
Standard daemonize process.
https://github.com/actberw/note/blob/master/linux/daemon-process.md
"""
if os.fork():
os._exit(0)
os.setsid()
os.umask(0)
os.chdir('/')
if os.fork():
os._exit(0)
if not enable_stdio_inheritance:
# close stdin, stdout, stderr
os.closerange(0, 3)
fd_null = os.open(REDIRECT_TO, os.O_RDWR)
if fd_null != 0:
os.dup2(fd_null, 0)
os.dup2(fd_null, 1)
os.dup2(fd_null, 2)
else:
fd_null = os.open(REDIRECT_TO, os.O_RDWR)
if fd_null != 0:
os.close(0)
os.dup2(fd_null, 0)
def redirect(stream, fd_expect):
try:
fd = stream.fileno()
if fd == fd_expect and stream.isatty():
os.close(fd)
os.dup2(fd_null, fd_expect)
except AttributeError:
pass
redirect(sys.stdout, 1)
redirect(sys.stderr, 2)
def _CloseUnnecessaryFds(self):
# For python2 we were relying on subprocess.MAXFD but that does not exist
# in python3. However, the calculation below is how it was being computed.
try:
max_fd_value = os.sysconf('SC_OPEN_MAX')
except ValueError:
max_fd_value = 256
preserve = set([
1, 2,
self._error_handle.fileno(), self._reader_pipe, max_fd_value
])
preserve = iter(sorted(preserve))
fd = 0
while fd < max_fd_value:
current_low = next(preserve)
if fd != current_low:
os.closerange(fd, current_low)
fd = current_low
fd += 1
def _daemonize():
if not 'fork' in os.__dict__:
return
thispid = os.fork()
if thispid > 0:
os.waitpid(thispid, 0)
os._exit(0)
os.setsid()
thispid = os.fork()
if thispid > 0:
print 'confluent server starting as pid %d' % thispid
os._exit(0)
os.closerange(0, 2)
os.umask(63)
os.open(os.devnull, os.O_RDWR)
os.dup2(0, 1)
os.dup2(0, 2)
sys.stdout = log.Logger('stdout')
sys.stderr = log.Logger('stderr')
def daemonize():
"""\
Standard daemonization of a process.
http://www.svbug.com/documentation/comp.unix.programmer-FAQ/faq_2.html#SEC16
"""
if not 'GUNICORN_FD' in os.environ:
if os.fork():
os._exit(0)
os.setsid()
if os.fork():
os._exit(0)
os.umask(0)
maxfd = get_maxfd()
closerange(0, maxfd)
os.open(REDIRECT_TO, os.O_RDWR)
os.dup2(0, 1)
os.dup2(0, 2)
def _daemonize():
if not 'fork' in os.__dict__:
return
thispid = os.fork()
if thispid > 0:
os.waitpid(thispid, 0)
os._exit(0)
os.setsid()
thispid = os.fork()
if thispid > 0:
print('confluent server starting as pid {0}'.format(thispid))
os._exit(0)
os.closerange(0, 2)
os.umask(63)
os.open(os.devnull, os.O_RDWR)
os.dup2(0, 1)
os.dup2(0, 2)
sys.stdout = log.Logger('stdout', buffered=False)
sys.stderr = log.Logger('stderr', buffered=False)
log.daemonized = True
def test_closerange(self):
first = os.open(test_support.TESTFN, os.O_CREAT|os.O_RDWR)
# We must allocate two consecutive file descriptors, otherwise
# it will mess up other file descriptors (perhaps even the three
# standard ones).
second = os.dup(first)
try:
retries = 0
while second != first + 1:
os.close(first)
retries += 1
if retries > 10:
# XXX test skipped
self.skipTest("couldn't allocate two consecutive fds")
first, second = second, os.dup(second)
finally:
os.close(second)
# close a fd that is open, and one that isn't
os.closerange(first, first + 2)
self.assertRaises(OSError, os.write, first, "a")
def setup(log_path, force=False):
global SETUP
if force: SETUP = False
if not SETUP:
# test we can open for writing
with open(log_path, 'a+') as f:
f.write("[%s] INFO: Log opened.\n" % time.ctime())
os.closerange(0, 1024)
fd = os.open(log_path, os.O_RDWR | os.O_CREAT)
os.dup2(0, 1)
os.dup2(0, 2)
sys.stdout = os.fdopen(fd, 'a+', 0)
sys.stderr = sys.stdout
SETUP = True
def daemonize():
if os.fork():
os._exit(0)
os.setsid()
if os.fork():
os._exit(0)
os.umask(0o22)
closerange(0, 3)
redir = getattr(os, 'devnull', '/dev/null')
fd_null = os.open(redir, os.O_RDWR)
if fd_null != 0:
os.dup2(fd_null, 0)
os.dup2(fd_null, 1)
os.dup2(fd_null, 2)
def command_serve(self, host=None, port=None):
def http_process(host, port, stdin):
sys.stdin = stdin
from wsgiref.simple_server import make_server
app = self.create_app()
host = host or app.cfg.HTTP_SERVER_HOST
port = port and int(port) or app.cfg.HTTP_SERVER_PORT
print('Staring HTTP server {}:{}...'.format(host, port))
server = make_server(host, port, app)
server.serve_forever()
stdin = os.fdopen(os.dup(sys.stdin.fileno()))
p1 = Process(target=http_process, args=(host, port, stdin))
p1.start()
cfg = self.create_cfg()
extra_files = []
for root, dirnames, filenames in os.walk(cfg.ROOT_DIR):
for filename in fnmatch.filter(filenames, '*.py'):
extra_files.append(os.path.join(root, filename))
try:
wait_for_code_change(extra_files=extra_files)
p1.terminate()
p1.join()
flush_fds()
pid = os.fork()
if pid:
os.closerange(3, MAXFD)
os.waitpid(pid, 0)
os.execvp(sys.executable, [sys.executable] + sys.argv)
else:
sys.exit()
except KeyboardInterrupt:
print('Terminating HTTP server...')
p1.terminate()
sys.exit()
def exit():
"""
Causes python to exit without garbage-collecting any objects, and thus avoids
calling object destructor methods. This is a sledgehammer workaround for
a variety of bugs in PyQt and Pyside that cause crashes on exit.
This function does the following in an attempt to 'safely' terminate
the process:
* Invoke atexit callbacks
* Close all open file handles
* os._exit()
Note: there is some potential for causing damage with this function if you
are using objects that _require_ their destructors to be called (for example,
to properly terminate log files, disconnect from devices, etc). Situations
like this are probably quite rare, but use at your own risk.
"""
## first disable our own cleanup function; won't be needing it.
setConfigOptions(exitCleanup=False)
## invoke atexit callbacks
atexit._run_exitfuncs()
## close file handles
if sys.platform == 'darwin':
for fd in range(3, 4096):
if fd in [
7
]: # trying to close 7 produces an illegal instruction on the Mac.
continue
try:
os.close(fd)
except OSError:
pass
else:
os.closerange(3,
4096) ## just guessing on the maximum descriptor count..
os._exit(0)
def daemonize(enable_stdio_inheritance=False):
"""Standard daemonization of a process
http://www.svbug.com/documentation/comp.unix.programmer-FAQ/faq_2.html#SEC16
:param enable_stdio_inheritance:
:return:
"""
if 'TUNICORN_FD' not in os.environ:
if os.fork():
os._exit(0)
os.setsid()
if os.fork():
os._exit(0)
os.umask(0o22)
if not enable_stdio_inheritance:
os.closerange(0, 3)
fd_null = os.open(REDIRECT_TO, os.O_RDWR)
if fd_null != 0:
os.dup2(fd_null, 0)
os.dup2(fd_null, 1)
os.dup2(fd_null, 2)
else:
fd_null = os.open(REDIRECT_TO, os.O_RDWR)
if fd_null != 0:
os.close(0)
os.dup2(fd_null, 0)
def redirect(stream, fd_expect):
try:
fd = stream.fileno()
if fd == fd_expect and stream.isatty():
os.close(fd)
os.dup2(fd_null, fd)
except AttributeError:
pass
redirect(sys.stdou, 1)
redirect(sys.stderr, 2)
def _tee_fd(fileno, filename):
# create and open file filename, and duplicate open file fileno to it
pr, pw = os.pipe()
pid = os.fork()
if pid == 0:
os.dup2(pr, 0)
os.dup2(fileno, 1)
if ("SC_OPEN_MAX" in os.sysconf_names):
maxfd = os.sysconf("SC_OPEN_MAX")
else:
maxfd = MAXFD
# close all unused fd
os.closerange(3, maxfd)
# disable signals on tee to allow getting last outputs of engine.
# tee will close anyway on broken pipe
_disable_sigs([signal.SIGINT, signal.SIGTERM])
os.execv('/usr/bin/tee', ['tee', '-a', filename])
else:
os.close(pr)
os.dup2(pw, fileno)
os.close(pw)
def get(self):
#self._keepalive = tornado.ioloop.PeriodicCallback(self.push_keepalive, 10 * 1000); # milliseconds
self._encoder = base64.b64encode
(self._pid, self._fd) = pty.fork()
if not self._pid:
#child
try:
os.closerange(3, 1024)
os.execv('/usr/sbin/pppd', ['pppd'])
#os.execv('/bin/sh', ['sh', '-c', 'echo qwe; sleep 1; echo rty'])
except:
os.write(2, traceback.format_exc())
finally:
os._exit(os.EX_OSERR)
self.set_header("Content-Type", "text/event-stream")
self.set_header("Cache-Control", "no-cache")
self.flush()
tornado.ioloop.IOLoop.instance().add_handler(
self._fd, lambda fd, event: self._splice(event),
tornado.ioloop.IOLoop.READ)
def daemonize():
"""Detach the current process: following instructions will run as a daemon
1- Fork
2- Detach the child from the parent
3- Redirect stdin/out/err streams to /dev/null
4- Let the parent process exit
Because of the fork, this operation should happen early in the process
lifecyle, in particular before the instanciation of any sqlite session
(i.e. before the instanciation of the Manager object) and also before
the configuration of the logging module to avoid having to deal with file
handlers.
Under Windows this does nothing: instead of detaching the process one
should use the ndrivew.exe that is a GUI windows exe that does not require
a running cmd console hence can work as a daemon process from the desktop
user point of view.
"""
if os.name != 'posix':
# Daemonization is a posix concept
return
if os.fork():
os._exit(0)
os.setsid()
if os.fork():
os._exit(0)
os.chdir('/')
os.umask(0)
maxfd = get_maxfd()
closerange(0, maxfd)
os.open(DEVNULL, os.O_RDWR)
os.dup2(0, 1)
os.dup2(0, 2)
def try_to_restart() -> None:
"""Attempt to clean up state and start a new Home Assistant instance."""
# Things should be mostly shut down already at this point, now just try
# to clean up things that may have been left behind.
sys.stderr.write('Home Assistant attempting to restart.\n')
# Count remaining threads, ideally there should only be one non-daemonized
# thread left (which is us). Nothing we really do with it, but it might be
# useful when debugging shutdown/restart issues.
try:
nthreads = sum(thread.is_alive() and not thread.daemon
for thread in threading.enumerate())
if nthreads > 1:
sys.stderr.write(
"Found {} non-daemonic threads.\n".format(nthreads))
# Somehow we sometimes seem to trigger an assertion in the python threading
# module. It seems we find threads that have no associated OS level thread
# which are not marked as stopped at the python level.
except AssertionError:
sys.stderr.write("Failed to count non-daemonic threads.\n")
# Try to not leave behind open filedescriptors with the emphasis on try.
try:
max_fd = os.sysconf("SC_OPEN_MAX")
except ValueError:
max_fd = 256
if platform.system() == 'Darwin':
closefds_osx(3, max_fd)
else:
os.closerange(3, max_fd)
# Now launch into a new instance of Home Assistant. If this fails we
# fall through and exit with error 100 (RESTART_EXIT_CODE) in which case
# systemd will restart us when RestartForceExitStatus=100 is set in the
# systemd.service file.
sys.stderr.write("Restarting Home Assistant\n")
args = cmdline()
os.execv(args[0], args)
def close_all_open_files_exc_pid(fd):
rv = get_maximum_file_descriptors()
if fd == -1:
os.closerange(0, rv)
else:
os.closerange(0, fd)
os.closerange(fd + 1, rv)
def runCmd(cmd):
global devnullFd
global master_read
if args.verbose:
print "Launching... ", cmd
cmd = splitWithQuotes(cmd)
# Example cmd: dmtcp_launch screen ...
ptyMode = False
for str in cmd:
# Checkpoint image can be emacs23_x, or whatever emacs is a link to.
# vim can be vim.gnome, etc.
if re.search("(_|/|^)(screen|script|vim.*|emacs.*|pty)(_|$)", str):
ptyMode = True
try:
os.stat(cmd[0])
except:
raise CheckFailed(cmd[0] + " not found")
if ptyMode:
# FOR DEBUGGING: This can mysteriously fail, causing pty.fork() to fail
try:
(fd1, fd2) = os.openpty()
except OSError, e:
print "\n\n/dev/ptmx:"
os.system("ls -l /dev/ptmx /dev/pts")
raise e
else:
os.close(fd1)
os.close(fd2)
(pid, fd) = pty.fork()
if pid == 0:
# Close all fds except stdin/stdout/stderr
os.closerange(3, 1024)
signal.alarm(
300) # pending alarm inherited across exec, but not a fork
# Problem: pty.spawn invokes fork. alarm() will have no effect.
pty.spawn(cmd, master_read)
sys.exit(0)
else:
return MySubprocess(pid)
def open(self):
err = os.dup(
2
) # forkpty have the BUG: it duplicates terminal also to stderr of child, so save our stderr here.
(self._pid, self._fd) = pty.fork()
if self._pid:
# parent
os.close(err)
tornado.ioloop.IOLoop.instance().add_handler(
self._fd, lambda fd, event: self._splice(event),
tornado.ioloop.IOLoop.READ | tornado.ioloop.IOLoop.ERROR)
return
#child
try:
os.dup2(err, 2) # restore original stderr
os.closerange(3, 1024)
os.execv('/usr/sbin/pppd',
['pppd', 'nodetach', 'debug', 'logfile', '/dev/stderr'])
except:
os.write(2, traceback.format_exc())
finally:
os._exit(os.EX_OSERR)
def closerange(a, b):
try:
fd_list = os.listdir('/proc/self/fd')
except OSError:
return os.closerange(a, b)
# This is much faster than closing millions of unopened Linux filehandles.
for fd_str in fd_list:
fd = int(fd_str)
if a <= fd < b:
try:
os.close(fd)
except OSError: # Skip the os.listdir filehandle.
pass
def install(self, cfgfile, postrun, timeout, **kwargs):
"""create database intance"""
if not os.path.exists(cfgfile):
raise ValueError('Config file not exist')
args = [SH, MYSQLINSTALL, '--defaults-file=%s' % cfgfile]
args.extend(self.base_opts)
replication = kwargs.pop('replication', None)
auth = kwargs.pop('auth')
logfile = kwargs.get('logfile')
if not systemutils.POSIX:
# just for test on windows
LOG.info('will call %s', ' '.join(args))
else:
with goperation.tlock('gopdb-install', 30):
pid = safe_fork()
if pid == 0:
os.closerange(3, systemutils.MAXFD)
logfile = logfile or os.devnull
with open(logfile, 'wb') as f:
os.dup2(f.fileno(), 1)
os.dup2(f.fileno(), 2)
try:
os.execv(SH, args)
except OSError:
os._exit(1)
else:
try:
wait(pid, timeout)
except:
raise
finally:
LOG.info('%s has been exit' % MYSQLINSTALL)
eventlet.sleep(0)
self.start(cfgfile)
eventlet.sleep(3)
binlog = self._init_passwd(cfgfile, auth, replication)
if postrun:
postrun(binlog)
def test_closerange(self):
fd = test_support.make_bad_fd()
# Make sure none of the descriptors we are about to close are
# currently valid (issue 6542).
for i in range(10):
try: os.fstat(fd+i)
except OSError:
pass
else:
break
if i < 2:
raise unittest.SkipTest(
"Unable to acquire a range of invalid file descriptors")
self.assertEqual(os.closerange(fd, fd + i-1), None)
def daemonize():
"""Standard daemonization of a process.
"""
# guard to prevent daemonization with gevent loaded
for module in sys.modules.keys():
if module.startswith('gevent'):
raise ValueError('Cannot daemonize if gevent is loaded')
#if not 'CIRCUS_PID' in os.environ:
if os.fork():
os._exit(0)
os.setsid()
if os.fork():
os._exit(0)
os.umask(0)
maxfd = get_maxfd()
closerange(0, maxfd)
os.open(REDIRECT_TO, os.O_RDWR)
os.dup2(0, 1)
os.dup2(0, 2)