def popen_sync(command,
env=None,
stdout=True,
stderr=True,
retcode=True,
cd=None,
su=None):
"""This function implements a subset of the functionality provided
by the subprocess.Popen class. The subprocess module in Python 2.4
and 2.5 have some problems dealing with processes termination on
multi-thread environments (593800, 1731717)."""
stdin_r, stdin_w = os.pipe()
stdout_r, stdout_w = os.pipe()
stderr_r, stderr_w = os.pipe()
pid = os.fork()
if pid == 0:
# Close parent's pipe ends
os.close(stdin_w)
os.close(stdout_r)
os.close(stderr_r)
# Dup fds for child
os.dup2(stdin_r, 0)
os.dup2(stdout_w, 1)
os.dup2(stderr_w, 2)
# Close fds
for i in range(3, MAXFD):
try:
os.close(i)
except:
pass
# Change directory
if cd:
try:
os.chdir(cd)
except Except as e:
print("Could not change directory to: %s" % (cd),
file=sys.stderr)
print(traceback.format_exc(), file=sys.stderr)
# Change user
if su:
try:
os.setuid(su)
except Except as e:
print("Could not set user: %s" % (su), file=sys.stderr)
print(traceback.format_exc(), file=sys.stderr)
# Pass control to the executable
if not env:
os.execv('/bin/sh', ['sh', '-c', command])
else:
os.execve('/bin/sh', ['sh', '-c', command], env)
# Poll on child's process outputs
buf_stderr = ''
buf_stdout = ''
set_non_blocking(stdout_r)
set_non_blocking(stderr_r)
# Read stdout and stderr
while True:
rlist, wlist, xlist = select.select([stdout_r, stderr_r], [],
[stdout_r, stderr_r],
POLLING_LAPSE)
if stderr_r in rlist:
data = ''
try:
data = os.read(stderr_r, READ_CHUNK_SIZE)
except OSError as e:
if e[0] in (errno.EAGAIN, ):
raise
if data:
buf_stderr += data
if stdout_r in rlist:
data = ''
try:
data = os.read(stdout_r, READ_CHUNK_SIZE)
except OSError as e:
if e[0] in (errno.EAGAIN, ):
raise
if data:
buf_stdout += data
# Has it finished?
try:
pid_ret, sts = os.waitpid(pid, os.WNOHANG)
if pid_ret == pid:
if os.WIFSIGNALED(sts):
returncode = -os.WTERMSIG(sts)
elif os.WIFEXITED(sts):
returncode = os.WEXITSTATUS(sts)
break
except OSError as e:
returncode = None
break
# Clean up
os.close(stdin_w)
os.close(stderr_r)
os.close(stdout_r)
os.close(stdin_r)
os.close(stderr_w)
os.close(stdout_w)
# Return information
ret = {
'command': command,
'stdout': buf_stdout,
'stderr': buf_stderr,
'retcode': returncode
}
return ret
def run_rnasubopt(ifname, deltaenergy, number=None):
errs = []
rna_args = ['nice', args.rnasubopt, '-e', str(deltaenergy), '-i', ifname]
safety_args = ['nice', args.trivialsafety]
if number != None:
safety_args.append('-num')
safety_args.append(str(number))
#print(' '.join(rna_args), '|', ' '.join(safety_args))
rna = psutil.Popen(rna_args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
safety = psutil.Popen(safety_args,
stdin=rna.stdout,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
rna.stdout.close()
if args.timeout > 0:
resource.prlimit(
rna.pid, resource.RLIMIT_CPU,
(int(args.timeout * 60 * 60), int(args.timeout * 60 * 60)))
folddata = None
try:
folddata = json.loads(safety.stdout.read().decode('utf-8'))
except json.decoder.JSONDecodeError as e:
errs.append('{}: Failed to decode trivial safety output:'.format(
ifname, e))
rnaerr = rna.stderr.read()
if rnaerr != None and len(rnaerr) > 0:
errs.append('RNAsubopt for {} returned errors:\n{}'.format(
ifname, rnaerr))
safetyerr = safety.stderr.read()
if safetyerr != None and len(safetyerr) > 0:
errs.append('Trivial safety for {} returned errors:\n{}'.format(
ifname, safetyerr))
rpid, rstatus, rres = os.wait4(rna.pid, 0)
#print('RNA: Status: {}, user (s): {:5.1f}, sys (s): {:5.1f}, maxrss (kB): {:6d}'.format(
# rstatus, rres.ru_utime, rres.ru_stime, rres.ru_maxrss))
spid, sstatus, sres = os.wait4(safety.pid, 0)
#print('Safety: Status: {}, user (s): {:5.1f}, sys (s): {:5.1f}, maxrss (kB): {:6d}'.format(
# sstatus, sres.ru_utime, sres.ru_stime, sres.ru_maxrss))
if os.WIFSIGNALED(rstatus) and not (number != None and os.WTERMSIG(rstatus)
== signal.SIGPIPE):
errs.append('{}: RNAsubopt was terminated with signal {}'.format(
ifname, os.WTERMSIG(rstatus)))
return (None, errs)
if os.WIFSIGNALED(sstatus):
errs.append('{}: Trivialsafety was terminated with signal {}'.format(
ifname, os.WTERMSIG(sstatus)))
return (None, errs)
if folddata == None:
errs.append('{}: folddata was None for unknown reason'.format(ifname))
return (None, errs)
folddata['Command'] = ' '.join(rna_args), '|', ' '.join(safety_args)
folddata['Resources'] = {
'RNAsuboptUser': rres.ru_utime,
'RNAsuboptSys': rres.ru_stime,
'RNAsuboptRSS': rres.ru_maxrss,
'TrivialSafetyUser': sres.ru_utime,
'TrivialSafetySys': sres.ru_stime,
'TrivialSafetyRSS': sres.ru_maxrss,
}
return (folddata, errs)
def __init__(self,
command,
working_dir=None,
capture_stderr=True,
env=None):
"""Changes into a specified directory, if provided, and executes a command.
Restores the old directory afterwards.
Args:
command: The command to run, in the form of sys.argv.
working_dir: The directory to change into.
capture_stderr: Determines whether to capture stderr in the output member
or to discard it.
env: Dictionary with environment to pass to the subprocess.
Returns:
An object that represents outcome of the executed process. It has the
following attributes:
terminated_by_signal True iff the child process has been terminated
by a signal.
signal Sygnal that terminated the child process.
exited True iff the child process exited normally.
exit_code The code with which the child process exited.
output Child process's stdout and stderr output
combined in a string.
"""
# The subprocess module is the preferrable way of running programs
# since it is available and behaves consistently on all platforms,
# including Windows. But it is only available starting in python 2.4.
# In earlier python versions, we revert to the popen2 module, which is
# available in python 2.0 and later but doesn't provide required
# functionality (Popen4) under Windows. This allows us to support Mac
# OS X 10.4 Tiger, which has python 2.3 installed.
if _SUBPROCESS_MODULE_AVAILABLE:
if capture_stderr:
stderr = subprocess.STDOUT
else:
stderr = subprocess.PIPE
p = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=stderr,
cwd=working_dir,
universal_newlines=True,
env=env)
# communicate returns a tuple with the file object for the child's
# output.
self.output = p.communicate()[0]
self._return_code = p.returncode
else:
old_dir = os.getcwd()
def _ReplaceEnvDict(dest, src):
# Changes made by os.environ.clear are not inheritable by child
# processes until Python 2.6. To produce inheritable changes we have
# to delete environment items with the del statement.
for key in dest.keys():
del dest[key]
dest.update(src)
# When 'env' is not None, backup the environment variables and replace
# them with the passed 'env'. When 'env' is None, we simply use the
# current 'os.environ' for compatibility with the subprocess.Popen
# semantics used above.
if env is not None:
old_environ = os.environ.copy()
_ReplaceEnvDict(os.environ, env)
try:
if working_dir is not None:
os.chdir(working_dir)
if capture_stderr:
p = popen2.Popen4(command)
else:
p = popen2.Popen3(command)
p.tochild.close()
self.output = p.fromchild.read()
ret_code = p.wait()
finally:
os.chdir(old_dir)
# Restore the old environment variables
# if they were replaced.
if env is not None:
_ReplaceEnvDict(os.environ, old_environ)
# Converts ret_code to match the semantics of
# subprocess.Popen.returncode.
if os.WIFSIGNALED(ret_code):
self._return_code = -os.WTERMSIG(ret_code)
else: # os.WIFEXITED(ret_code) should return True here.
self._return_code = os.WEXITSTATUS(ret_code)
if self._return_code < 0:
self.terminated_by_signal = True
self.exited = False
self.signal = -self._return_code
else:
self.terminated_by_signal = False
self.exited = True
self.exit_code = self._return_code
def _spawn_posix(cmd, search_path=1, verbose=0, dry_run=0):
global _cfg_target
global _cfg_target_split
log.info(' '.join(cmd))
if dry_run:
return
else:
executable = cmd[0]
exec_fn = search_path and os.execvp or os.execv
env = None
if sys.platform == 'darwin':
if _cfg_target is None:
_cfg_target = sysconfig.get_config_var(
'MACOSX_DEPLOYMENT_TARGET') or ''
if _cfg_target:
_cfg_target_split = [
int(x) for x in _cfg_target.split('.')
]
if _cfg_target:
cur_target = os.environ.get('MACOSX_DEPLOYMENT_TARGET',
_cfg_target)
if _cfg_target_split > [int(x) for x in cur_target.split('.')]:
my_msg = '$MACOSX_DEPLOYMENT_TARGET mismatch: now "%s" but "%s" during configure' % (
cur_target, _cfg_target)
raise DistutilsPlatformError(my_msg)
env = dict(os.environ, MACOSX_DEPLOYMENT_TARGET=cur_target)
exec_fn = search_path and os.execvpe or os.execve
pid = os.fork()
if pid == 0:
try:
if env is None:
exec_fn(executable, cmd)
else:
exec_fn(executable, cmd, env)
except OSError as e:
if not DEBUG:
cmd = executable
sys.stderr.write('unable to execute %r: %s\n' %
(cmd, e.strerror))
os._exit(1)
if not DEBUG:
cmd = executable
sys.stderr.write('unable to execute %r for unknown reasons' % cmd)
os._exit(1)
else:
while 1:
try:
pid, status = os.waitpid(pid, 0)
except OSError as exc:
import errno
if exc.errno == errno.EINTR:
continue
if not DEBUG:
cmd = executable
raise DistutilsExecError, 'command %r failed: %s' % (
cmd, exc[-1])
if os.WIFSIGNALED(status):
if not DEBUG:
cmd = executable
raise DistutilsExecError, 'command %r terminated by signal %d' % (
cmd, os.WTERMSIG(status))
elif os.WIFEXITED(status):
exit_status = os.WEXITSTATUS(status)
if exit_status == 0:
return
if not DEBUG:
cmd = executable
raise DistutilsExecError, 'command %r failed with exit status %d' % (
cmd, exit_status)
elif os.WIFSTOPPED(status):
continue
else:
if not DEBUG:
cmd = executable
raise DistutilsExecError, 'unknown error executing %r: termination status %d' % (
cmd, status)
return
import os
import popen2
import sys
ok = []
new = []
fail = []
fault = []
for m in glob.glob('[A-Za-z0-9]*.l1'):
print m,
sys.stdout.flush()
cmd = '../l1 -6 -b -w ' + m
p = popen2.Popen4(cmd)
rv = p.wait()
if os.WIFSIGNALED(rv):
print 'fault'
fault.append(m)
continue
out = p.fromchild.read()
del p
fn = m + '.out'
if os.path.exists(fn):
f = open(fn)
s = f.read()
f.close()
if s == out:
print 'ok'
ok.append(m)
else:
f = open(m + '.failed', 'w')
def fork_processes(num_processes, max_restarts=100):
"""Starts multiple worker processes.
If ``num_processes`` is None or <= 0, we detect the number of cores
available on this machine and fork that number of child
processes. If ``num_processes`` is given and > 0, we fork that
specific number of sub-processes.
Since we use processes and not threads, there is no shared memory
between any server code.
Note that multiple processes are not compatible with the autoreload
module (or the ``autoreload=True`` option to `tornado.web.Application`
which defaults to True when ``debug=True``).
When using multiple processes, no IOLoops can be created or
referenced until after the call to ``fork_processes``.
In each child process, ``fork_processes`` returns its *task id*, a
number between 0 and ``num_processes``. Processes that exit
abnormally (due to a signal or non-zero exit status) are restarted
with the same id (up to ``max_restarts`` times). In the parent
process, ``fork_processes`` returns None if all child processes
have exited normally, but will otherwise only exit by throwing an
exception.
"""
global _task_id
assert _task_id is None
if num_processes is None or num_processes <= 0:
num_processes = cpu_count()
if ioloop.IOLoop.initialized():
raise RuntimeError(
"Cannot run in multiple processes: IOLoop instance "
"has already been initialized. You cannot call "
"IOLoop.instance() before calling start_processes()")
logger.info("Starting %d processes", num_processes)
children = {}
def start_child(i):
pid = os.fork()
if pid == 0:
# child process
_reseed_random()
global _task_id
_task_id = i
return i
else:
children[pid] = i
return None
for i in range(num_processes):
id = start_child(i)
if id is not None:
return id
global exiting
exiting = False
def receive_signal(sig, frame):
logger.debug('Received signal')
global exiting
exiting = True
for pid, taskid in children.items():
os.kill(pid, signal.SIGTERM)
signal.signal(signal.SIGTERM, receive_signal)
signal.signal(signal.SIGINT, receive_signal)
num_restarts = 0
while children and not exiting:
logger.debug('Exiting : %s' % exiting)
try:
pid, status = os.wait()
except OSError as e:
if errno_from_exception(e) == errno.EINTR:
continue
raise
if pid not in children:
continue
id = children.pop(pid)
if os.WIFSIGNALED(status):
logger.warning("child %d (pid %d) killed by signal %d, restarting",
id, pid, os.WTERMSIG(status))
elif os.WEXITSTATUS(status) != 0:
logger.warning(
"child %d (pid %d) exited with status %d, restarting", id, pid,
os.WEXITSTATUS(status))
else:
logger.info("child %d (pid %d) exited normally", id, pid)
continue
num_restarts += 1
if num_restarts > max_restarts:
raise RuntimeError("Too many child restarts, giving up")
new_id = start_child(id)
if new_id is not None:
return new_id
# All child processes exited cleanly, so exit the master process
# instead of just returning to right after the call to
# fork_processes (which will probably just start up another IOLoop
# unless the caller checks the return value).
sys.exit(0)
def _run_in_child(
self,
*,
chroot: Path,
chroot_paths: List[Path],
compiled_module: CompiledModule,
timeout: float,
result: Any,
function: str,
args: List[Any],
) -> None:
"""
Fork a child process to run `function` with `args`.
`args` must be Thrift data types. `result` must also be a Thrift type --
its `.read()` function will be called, which may produce an error if
the child process has a bug. (EOFError is very likely.)
Raise ModuleExitedError if the child process did not behave as expected.
Raise ModuleTimeoutError if it did not exit after a delay -- or if it
closed its file descriptors long before it exited.
"""
limit_time = time.time() + timeout
module_process = self._forkserver.spawn_module(
process_name=compiled_module.module_slug,
chroot_dir=chroot,
chroot_provide_paths=[(p, p) for p in chroot_paths],
args=[compiled_module, function, args],
)
# stdout is Thrift package; stderr is logs
output_reader = ChildReader(module_process.stdout.fileno(),
OUTPUT_BUFFER_MAX_BYTES)
log_reader = ChildReader(module_process.stderr.fileno(),
LOG_BUFFER_MAX_BYTES)
# Read until the child closes its stdout and stderr
with selectors.DefaultSelector() as selector:
selector.register(output_reader.fileno, selectors.EVENT_READ)
selector.register(log_reader.fileno, selectors.EVENT_READ)
timed_out = False
while selector.get_map():
remaining = limit_time - time.time()
if remaining <= 0:
if not timed_out:
timed_out = True
module_process.kill(
) # untrusted code could ignore SIGTERM
timeout = None # wait as long as it takes for everything to die
# Fall through. After SIGKILL the child will close each fd,
# sending EOF to us. That means the selector _must_ return.
else:
timeout = remaining # wait until we reach our timeout
events = selector.select(timeout=timeout)
ready = frozenset(key.fd for key, _ in events)
for reader in (output_reader, log_reader):
if reader.fileno in ready:
reader.ingest()
if reader.eof:
selector.unregister(reader.fileno)
# The child closed its fds, so it should die soon. If it doesn't, that's
# a bug -- so kill -9 it!
#
# os.wait() has no timeout option, and asyncio messes with signals so
# we won't use those. Spin until the process dies, and force-kill if we
# spin too long.
for _ in range(DEAD_PROCESS_N_WAITS):
pid, exit_status = module_process.wait(os.WNOHANG)
if pid != 0: # pid==0 means process is still running
break
time.sleep(DEAD_PROCESS_WAIT_POLL_INTERVAL)
else:
# we waited and waited. No luck. Dead module. Kill it.
timed_out = True
module_process.kill()
_, exit_status = module_process.wait(0)
if os.WIFEXITED(exit_status):
exit_code = os.WEXITSTATUS(exit_status)
elif os.WIFSIGNALED(exit_status):
exit_code = -os.WTERMSIG(exit_status)
else:
raise RuntimeError("Unhandled wait() status: %r" % exit_status)
if timed_out:
raise ModuleTimeoutError
if exit_code != 0:
raise ModuleExitedError(exit_code, log_reader.to_str())
transport = thrift.transport.TTransport.TMemoryBuffer(
output_reader.buffer)
protocol = thrift.protocol.TBinaryProtocol.TBinaryProtocol(transport)
try:
result.read(protocol)
except EOFError: # TODO handle other errors Thrift may throw
raise ModuleExitedError(exit_code, log_reader.to_str()) from None
# We should be at the end of the output now. If we aren't, that means
# the child wrote too much.
if transport.read(1) != b"":
raise ModuleExitedError(exit_code, log_reader.to_str())
if log_reader.buffer:
logger.info("Output from module process: %s", log_reader.to_str())
return result
class BaseProcess(object):
def __init__(self, command = None, args = [], environment = None, terminal_preferred = True, terminal_required = False, check_timeout = 0.00):
self.has_terminal = False
#self._echo = True
self._flush_buffer = True
self._buffer = ""
self._seeker = 0
self._maxread = 2000
self._maxwrite = 500
self._canread_timeout = check_timeout
self._canwrite_timeout = check_timeout
self._pipe = None
self._terminal_preferred = terminal_preferred
self._terminal_required = terminal_required
self._ready = False
if environment is None:
environment = create_environment(TERM = 'xterm',TERM_PROGRAM= 'Apple_Terminal', TERM_PROGRAM_VERSION='273.1')
if not command is None:
self.spawn(command, args,environment)
@property
def buffer(self):
"""
This property holds the buffer of the session, i.e. a log over
all output from the terminal.
"""
self._updateBuffer()
return self._buffer
def terminate(self):
"""
Terminates the process.
"""
self._ready = False
if self._pipe is None:
return
self._pipe.terminate()
def kill(self):
"""
Kills the process.
"""
self._ready = False
if self._pipe is None:
return
self._pipe.kill()
def isalive(self):
"""
Checks whether the process is alive or not.
"""
if not self._pipe is None:
return self._pipe.poll() is None
if not self._ready:
return False
### For this part I owe a great deal of acknowledgement to
### the pexpect project, since the following is basically
### copy and paste
pid = None
status = None
i = 0
while pid == 0 and i<2: # TODO: Fix this part and test it properly
try:
pid, status = os.waitpid(self.pid, 0) # TODO: either os.WNOHANG or 0
except OSError, e: # No child processes
if e[0] == errno.ECHILD:
raise ExceptionPexpect ('isalive() encountered condition where "terminated" is 0, but there was no child process. Did someone else call waitpid() on our process?')
else:
raise
i+=1
if pid == 0:
return True
if not status is None and ( os.WIFEXITED (status) or os.WIFSIGNALED (status) ):
self._ready = False
return False
return True
def ikos_analyzer(db_path, pp_path, opt):
# Fix huge slow down when ikos-analyzer uses DROP TABLE on an existing db
if os.path.isfile(db_path):
os.remove(db_path)
cmd = [settings.ikos_analyzer()]
# analysis options
cmd += ['-a=%s' % ','.join(opt.analyses),
'-d=%s' % opt.domain,
'-entry-points=%s' % ','.join(opt.entry_points),
'-globals-init=%s' % opt.globals_init,
'-prec=%s' % opt.precision_level,
'-proc=%s' % opt.procedural]
if opt.no_init_globals:
cmd.append('-no-init-globals=%s' % ','.join(opt.no_init_globals))
if opt.no_liveness:
cmd.append('-no-liveness')
if opt.no_pointer:
cmd.append('-no-pointer')
if opt.no_fixpoint_profiles:
cmd.append('-no-fixpoint-profiles')
if opt.hardware_addresses:
cmd.append('-hardware-addresses=%s' % ','.join(opt.hardware_addresses))
if opt.hardware_addresses_file:
cmd.append('-hardware-addresses-file=%s' % opt.hardware_addresses_file)
if opt.argc is not None:
cmd.append('-argc=%d' % opt.argc)
# import options
if opt.no_libc:
cmd.append('-no-libc')
if opt.no_libcpp:
cmd.append('-no-libcpp')
if opt.no_libikos:
cmd.append('-no-libikos')
# add -allow-dbg-mismatch if necessary
if opt.opt_level in ('basic', 'aggressive'):
cmd.append('-allow-dbg-mismatch')
# AR passes options
if opt.disable_type_check:
cmd.append('-disable-type-check')
if opt.no_simplify_cfg:
cmd.append('-no-simplify-cfg')
if opt.no_simplify_upcast_comparison:
cmd.append('-no-simplify-upcast-comparison')
if 'gauge' in opt.domain:
cmd.append('-add-loop-counters')
# debug options
cmd += ['-display-checks=%s' % opt.display_checks,
'-display-inv=%s' % opt.display_inv]
if opt.display_ar:
cmd.append('-display-ar')
if opt.display_liveness:
cmd.append('-display-liveness')
if opt.display_function_pointer:
cmd.append('-display-function-pointer')
if opt.display_pointer:
cmd.append('-display-pointer')
if opt.display_fixpoint_profiles:
cmd.append('-display-fixpoint-profiles')
if opt.generate_dot:
cmd += ['-generate-dot', '-generate-dot-dir', opt.generate_dot_dir]
# add -name-values if necessary
if (opt.display_checks in ('all', 'fail') or
opt.display_inv in ('all', 'fail') or
opt.display_liveness or
opt.display_fixpoint_profiles or
opt.display_function_pointer or
opt.display_pointer or
opt.display_raw_checks):
cmd.append('-name-values')
# misc. options
cmd += ['-color=%s' % opt.color,
'-log=%s' % opt.log_level]
# input/output
cmd += [pp_path, '-o', db_path]
# set resource limit, if requested
if opt.mem > 0:
import resource # fails on Windows
def set_limits():
mem_bytes = opt.mem * 1024 * 1024
resource.setrlimit(resource.RLIMIT_AS, [mem_bytes, mem_bytes])
else:
set_limits = None
# called after timeout
def kill(p):
try:
log.error('Timeout')
p.send_signal(signal.SIGALRM)
except OSError:
pass
log.info('Running ikos analyzer')
log.debug('Running %s' % command_string(cmd))
p = subprocess.Popen(cmd, preexec_fn=set_limits)
timer = threading.Timer(opt.cpu, kill, [p])
if opt.cpu > 0:
timer.start()
try:
if sys.platform.startswith('win'):
return_status = p.wait()
else:
_, return_status = os.waitpid(p.pid, 0)
finally:
# kill the timer if the process has terminated already
if timer.isAlive():
timer.cancel()
# special case for Windows, since it does not define WIFEXITED & co.
if sys.platform.startswith('win'):
if return_status != 0:
raise AnalyzerError('a run-time error occured', cmd, return_status)
else:
return
# if it did not terminate properly, propagate this error code
if os.WIFEXITED(return_status) and os.WEXITSTATUS(return_status) != 0:
exit_status = os.WEXITSTATUS(return_status)
raise AnalyzerError('a run-time error occured', cmd, exit_status)
if os.WIFSIGNALED(return_status):
signum = os.WTERMSIG(return_status)
raise AnalyzerError('exited with signal %s' % signal_name(signum),
cmd,
signum)
if os.WIFSTOPPED(return_status):
signum = os.WSTOPSIG(return_status)
raise AnalyzerError('exited with signal %d' % signal_name(signum),
cmd,
signum)
def build_functions_py(self):
(apr_prefix, apr_include_dir, cpp, ldflags, flags,
library_path) = self.get_build_config()
cwd = os.getcwd()
if self.svn_include_dir[-18:] == "subversion/include":
includes = ('subversion/include/svn_*.h '
'%s/ap[ru]_*.h' % apr_include_dir)
cmd = [
"%s %s --cpp '%s %s' %s "
"%s -o subversion/bindings/ctypes-python/svn_all.py "
"--no-macro-warnings --strip-build-path=%s" %
(sys.executable, self.ctypesgen_py, cpp, flags, ldflags,
includes, self.svn_include_dir[:-19])
]
os.chdir(self.svn_include_dir[:-19])
else:
includes = ('%s/svn_*.h '
'%s/ap[ru]_*.h' %
(self.svn_include_dir, apr_include_dir))
cmd = [
"%s %s --cpp '%s %s' %s "
"%s -o svn_all.py --no-macro-warnings" %
(sys.executable, self.ctypesgen_py, cpp, flags, ldflags,
includes)
]
if self.lib_dirs:
cmd.extend('-R ' + x for x in self.lib_dirs.split(":"))
cmd = ' '.join(cmd)
if self.save_preprocessed_headers:
cmd += " --save-preprocessed-headers=%s" % \
os.path.abspath(self.save_preprocessed_headers)
if self.verbose or self.dry_run:
status = self.execute(os.system, (cmd, ), cmd)
else:
f = os.popen(cmd, 'r')
f.read() # Required to avoid the 'Broken pipe' error.
status = f.close() # None is returned for the usual 0 return code
os.chdir(cwd)
if os.name == "posix" and status and status != 0:
if os.WIFEXITED(status):
status = os.WEXITSTATUS(status)
if status != 0:
sys.exit(status)
elif os.WIFSIGNALED(status):
log.error("ctypesgen.py killed with signal %d" %
os.WTERMSIG(status))
sys.exit(2)
elif os.WIFSTOPPED(status):
log.error("ctypesgen.py stopped with signal %d" %
os.WSTOPSIG(status))
sys.exit(2)
else:
log.error("ctypesgen.py exited with invalid status %d",
status)
sys.exit(2)
if not self.dry_run:
r = re.compile(r"(\s+\w+)\.restype = POINTER\(svn_error_t\)")
out = open("svn_all2.py", "w")
for line in open("svn_all.py"):
line = r.sub(
"\\1.restype = POINTER(svn_error_t)\n\\1.errcheck = _svn_errcheck",
line)
if not line.startswith("FILE ="):
out.write(line)
out.close()
cmd = "cat csvn/core/functions.py.in svn_all2.py > csvn/core/functions.py"
self.execute(os.system, (cmd, ), cmd)
log.info("Generated csvn/core/functions.py successfully")
# - the environment had to be passed as a sequence of strings
# ("FOO=1", "BAR=2") (GNOME bug 583078)
# - directory keyword could not be set to None (GNOME bug 583129)
# The bugs have been fixed, but for compatibility reasons the old
# compatibility code is still in place.
self.child_pid = self.terminal.fork_command(command=command[0],
argv=command,
**kws)
while self.vte_fork_running:
gtk.main_iteration()
if self.quit:
raise ExitRequestedException()
self.child_pid = None
if os.WIFEXITED(self.vte_child_exit_status):
rc = os.WEXITSTATUS(self.vte_child_exit_status)
elif os.WIFSIGNALED(self.vte_child_exit_status):
raise CommandError(
_('%(command)s died with signal %(rc)s') % {
'command': short_command,
'rc': os.WTERMSIG(self.vte_child_exit_status)
})
if rc:
raise CommandError(
_('%(command)s returned with an error code (%(rc)s)') % {
'command': short_command,
'rc': rc
})
def on_vte_child_exit_cb(self, terminal):
self.vte_fork_running = False
def run_wsgi(conf_path, app_section, *args, **kwargs):
"""
Runs the server according to some strategy. The default strategy runs a
specified number of workers in pre-fork model. The object-server (only)
may use a servers-per-port strategy if its config has a servers_per_port
setting with a value greater than zero.
:param conf_path: Path to paste.deploy style configuration file/directory
:param app_section: App name from conf file to load config from
:returns: 0 if successful, nonzero otherwise
"""
# Load configuration, Set logger and Load request processor
try:
(conf, logger, log_name) = \
_initrp(conf_path, app_section, *args, **kwargs)
except ConfigFileError as e:
print(e)
return 1
# optional nice/ionice priority scheduling
utils.modify_priority(conf, logger)
servers_per_port = int(conf.get('servers_per_port', '0') or 0)
# NOTE: for now servers_per_port is object-server-only; future work could
# be done to test and allow it to be used for account and container
# servers, but that has not been done yet.
if servers_per_port and app_section == 'object-server':
strategy = ServersPerPortStrategy(conf,
logger,
servers_per_port=servers_per_port)
else:
strategy = WorkersStrategy(conf, logger)
# patch event before loadapp
utils.eventlet_monkey_patch()
# Ensure the configuration and application can be loaded before proceeding.
global_conf = {'log_name': log_name}
if 'global_conf_callback' in kwargs:
kwargs['global_conf_callback'](conf, global_conf)
loadapp(conf_path, global_conf=global_conf)
# set utils.FALLOCATE_RESERVE if desired
utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
utils.config_fallocate_value(conf.get('fallocate_reserve', '1%'))
# Start listening on bind_addr/port
error_msg = strategy.do_bind_ports()
if error_msg:
logger.error(error_msg)
print(error_msg)
return 1
# Redirect errors to logger and close stdio. Do this *after* binding ports;
# we use this to signal that the service is ready to accept connections.
capture_stdio(logger)
no_fork_sock = strategy.no_fork_sock()
if no_fork_sock:
run_server(conf, logger, no_fork_sock, global_conf=global_conf)
return 0
def stop_with_signal(signum, *args):
"""Set running flag to False and capture the signum"""
running_context[0] = False
running_context[1] = signum
# context to hold boolean running state and stop signum
running_context = [True, None]
signal.signal(signal.SIGTERM, stop_with_signal)
signal.signal(signal.SIGHUP, stop_with_signal)
while running_context[0]:
for sock, sock_info in strategy.new_worker_socks():
pid = os.fork()
if pid == 0:
signal.signal(signal.SIGHUP, signal.SIG_DFL)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
strategy.post_fork_hook()
run_server(conf, logger, sock)
strategy.log_sock_exit(sock, sock_info)
return 0
else:
strategy.register_worker_start(sock, sock_info, pid)
# The strategy may need to pay attention to something in addition to
# child process exits (like new ports showing up in a ring).
#
# NOTE: a timeout value of None will just instantiate the Timeout
# object and not actually schedule it, which is equivalent to no
# timeout for the green_os.wait().
loop_timeout = strategy.loop_timeout()
with Timeout(loop_timeout, exception=False):
try:
try:
pid, status = green_os.wait()
if os.WIFEXITED(status) or os.WIFSIGNALED(status):
strategy.register_worker_exit(pid)
except OSError as err:
if err.errno not in (errno.EINTR, errno.ECHILD):
raise
if err.errno == errno.ECHILD:
# If there are no children at all (ECHILD), then
# there's nothing to actually wait on. We sleep
# for a little bit to avoid a tight CPU spin
# and still are able to catch any KeyboardInterrupt
# events that happen. The value of 0.01 matches the
# value in eventlet's waitpid().
sleep(0.01)
except KeyboardInterrupt:
logger.notice('User quit')
running_context[0] = False
break
if running_context[1] is not None:
try:
signame = SIGNUM_TO_NAME[running_context[1]]
except KeyError:
logger.error('Stopping with unexpected signal %r' %
running_context[1])
else:
logger.error('%s received', signame)
if running_context[1] == signal.SIGTERM:
os.killpg(0, signal.SIGTERM)
strategy.shutdown_sockets()
signal.signal(signal.SIGTERM, signal.SIG_IGN)
logger.notice('Exited (%s)', os.getpid())
return 0
dmtcp_tmpdir()) and os.path.isdir(ckptDir):
if subprocess.call(("cp -pr " + ckptDir + ' ' +
dmtcp_tmpdir()).split()) == 0:
print "\n***** Copied checkpoint images to " + dmtcp_tmpdir() \
+ "/" + ckptDir
raise e
else:
printFixed("FAILED ")
(oldpid,
oldstatus) = os.waitpid(procs[-1].pid, os.WNOHANG)
if oldpid == procs[-1].pid:
if os.WIFEXITED(oldstatus):
printFixed(
"(first process exited: oldstatus " +
str(os.WEXITSTATUS(oldstatus)) + ")")
if os.WIFSIGNALED(oldstatus):
printFixed("(first process rec'd signal " +
str(os.WTERMSIG(oldstatus)) + ")")
if os.WCOREDUMP(oldstatus):
coredump = "core." + str(oldpid)
if os.path.isdir(dmtcp_tmpdir(
)) and os.path.isfile(coredump):
if subprocess.call(
("cp -pr " + coredump + ' ' +
dmtcp_tmpdir()).split()) == 0:
printFixed(" (" + coredump +
" copied to DMTCP_TMPDIR:" +
dmtcp_tmpdir() + "/)")
else:
printFixed("(first process didn't die)")
printFixed(" retry:")
def run_wsgi(conf_path, app_section, *args, **kwargs):
"""
Runs the server according to some strategy. The default strategy runs a
specified number of workers in pre-fork model. The object-server (only)
may use a servers-per-port strategy if its config has a servers_per_port
setting with a value greater than zero.
:param conf_path: Path to paste.deploy style configuration file/directory
:param app_section: App name from conf file to load config from
:returns: 0 if successful, nonzero otherwise
"""
# Load configuration, Set logger and Load request processor
try:
(conf, logger, log_name) = \
_initrp(conf_path, app_section, *args, **kwargs)
except ConfigFileError as e:
print(e)
return 1
servers_per_port = int(conf.get('servers_per_port', '0') or 0)
# NOTE: for now servers_per_port is object-server-only; future work could
# be done to test and allow it to be used for account and container
# servers, but that has not been done yet.
if servers_per_port and app_section == 'object-server':
strategy = ServersPerPortStrategy(
conf, logger, servers_per_port=servers_per_port)
else:
strategy = WorkersStrategy(conf, logger)
error_msg = strategy.bind_ports()
if error_msg:
logger.error(error_msg)
print(error_msg)
return 1
# Ensure the configuration and application can be loaded before proceeding.
global_conf = {'log_name': log_name}
if 'global_conf_callback' in kwargs:
kwargs['global_conf_callback'](conf, global_conf)
loadapp(conf_path, global_conf=global_conf)
# set utils.FALLOCATE_RESERVE if desired
reserve = int(conf.get('fallocate_reserve', 0))
if reserve > 0:
utils.FALLOCATE_RESERVE = reserve
# redirect errors to logger and close stdio
capture_stdio(logger)
no_fork_sock = strategy.no_fork_sock()
if no_fork_sock:
run_server(conf, logger, no_fork_sock, global_conf=global_conf)
return 0
def kill_children(*args):
"""Kills the entire process group."""
logger.error('SIGTERM received')
signal.signal(signal.SIGTERM, signal.SIG_IGN)
running[0] = False
os.killpg(0, signal.SIGTERM)
def hup(*args):
"""Shuts down the server, but allows running requests to complete"""
logger.error('SIGHUP received')
signal.signal(signal.SIGHUP, signal.SIG_IGN)
running[0] = False
running = [True]
signal.signal(signal.SIGTERM, kill_children)
signal.signal(signal.SIGHUP, hup)
while running[0]:
for sock, sock_info in strategy.new_worker_socks():
pid = os.fork()
if pid == 0:
signal.signal(signal.SIGHUP, signal.SIG_DFL)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
strategy.post_fork_hook()
run_server(conf, logger, sock)
strategy.log_sock_exit(sock, sock_info)
return 0
else:
strategy.register_worker_start(sock, sock_info, pid)
# The strategy may need to pay attention to something in addition to
# child process exits (like new ports showing up in a ring).
#
# NOTE: a timeout value of None will just instantiate the Timeout
# object and not actually schedule it, which is equivalent to no
# timeout for the green_os.wait().
loop_timeout = strategy.loop_timeout()
with Timeout(loop_timeout, exception=False):
try:
pid, status = green_os.wait()
if os.WIFEXITED(status) or os.WIFSIGNALED(status):
strategy.register_worker_exit(pid)
except OSError as err:
if err.errno not in (errno.EINTR, errno.ECHILD):
raise
except KeyboardInterrupt:
logger.notice('User quit')
running[0] = False
break
strategy.shutdown_sockets()
logger.notice('Exited')
return 0
def test_terminate_sigkill(self):
self._terminate_with_signal(signal.SIGKILL)
status = self._reap_test()
self.assertTrue(os.WIFSIGNALED(status))
self.assertEqual(os.WTERMSIG(status), signal.SIGKILL)
class BackgroundProcess:
"""
Python threads don't work so well. We fork a new process to execute
runnable.Run(). If an alarm is set, we fork another process to
perform the timeout. Join() returns the unpickled return value of
runnable.Run(), or raises the unpickled exception of runnable.Run(),
as appropriate.
"""
def __init__(self, runnable, *args, **kwargs):
"""
Start to execute runnable.Run() asynchronously. If the "alarm"
keyword is set, the process will be killed and Join() will return
gexcept.TimeoutException after the specified number of seconds.
"stdin", "stdout", and "stderr" keywords may be specified as
follows: default is to close the stream; if set to a false value,
the stream is simply a copy of the parent's stream; other values
are reserved for future use.
"""
assert not args # TODO: args exists only to placate a buggy pychecker
assert not kwargs.get('stdin', None) # reserve for future use
assert not kwargs.get('stdout', None) # reserve for future use
assert not kwargs.get('stderr', None) # reserve for future use
# the following apparently corrects a Python bug: if stdout is
# redirected to a file and the subprocess prints, buffered data
# is printed more than once.
sys.stdout.flush()
(readval, writeval) = os.pipe() # return value is transferred here
pid = os.fork()
if pid == 0:
# then we are the child
if not kwargs.has_key('stdin'): os.close(0)
if not kwargs.has_key('stdout'): os.close(1)
if not kwargs.has_key('stderr'): os.close(2)
# one-way communication from child to parent
self._CloseFileDescriptors(3, butnot=writeval)
valfile = os.fdopen(writeval, 'w')
try:
result = runnable.Run()
try:
valfile.write('R')
pickle.dump(result, valfile)
except:
os._exit(99)
# end try
except:
valfile.write('E')
(cl, exc, traceobj) = sys.exc_info()
fil = StringIO.StringIO()
traceback.print_exc(None, fil)
pickle.dump((cl, exc, fil.getvalue()), valfile)
# end try
# this valfile.close() could be writing to a dead socket,
# so ignore any failure: if the parent isn't reading then
# the error isn't very important.
try:
valfile.close()
except:
pass
# end try
os._exit(0)
else: # we're the original process
os.close(writeval)
self.readval = readval
fcntl.fcntl(self.readval, fcntl.F_SETFL, os.O_NONBLOCK)
self.pid = pid
self.apid = 0 # no alarm yet
alarm = kwargs.get('alarm', 0)
if alarm:
apid = os.fork()
if 0 == apid:
self._CloseFileDescriptors(0)
start = time.time()
while (time.time() - start) < alarm:
time.sleep(1 + alarm - (time.time() - start))
# end while
self.Stop()
os._exit(0)
else:
self.apid = apid
# end if
# end if
# end if
return
def Stop(self, signal=15):
"""Stop the runnable by sending it a signal."""
if self.pid < 0:
raise Exception, 'already stopped'
# end if
os.kill(self.pid, signal)
if self.apid:
try:
os.kill(self.apid, signal)
except OSError:
pass
# end try
# end if
time.sleep(1)
try:
os.kill(self.pid, 9)
except OSError:
pass
# end try
return
def Join(self, timeout=-1):
"""
Wait for the runnable to complete. Throws gexcept.TimeoutException
if timeout was reached before the runnable completed. If timeout is
negative, we wait forever.
"""
if self.pid < 0:
raise Exception, 'already stopped'
# end if
if timeout < 0:
(readers, writers, errs) = select.select([self.readval], [], [])
assert readers
else:
(readers, writers, errs) = select.select([self.readval], [], [],
timeout)
if not readers:
raise gexcept.TimeoutException
# end if
# end if
buffer = ''
while 1:
try:
thistime = os.read(self.readval, 8192)
if not thistime:
break
# end if
buffer += thistime
except OSError, (err, m):
if errno.EAGAIN != err:
raise
# end if
# end except
# end while
os.close(self.readval)
(pid, status) = os.waitpid(self.pid, 0)
assert pid == self.pid
if self.apid: # then kill the alarm and wait for it
try:
os.kill(self.apid, 15)
except OSError:
pass
# end try
(apid, astatus) = os.waitpid(self.apid, 0)
assert apid == self.apid
# end if
if os.WIFSIGNALED(status):
raise gexcept.TimeoutException
else:
assert os.WIFEXITED(status)
# end if
self.exitstat = os.WEXITSTATUS(status)
if not buffer:
raise ExitException(self.exitstat)
# end if
obj = pickle.loads(buffer[1:])
typ = buffer[0]
if 'E' == typ:
raise gexcept.NestedException(obj)
elif 'R' == typ:
return obj
else:
raise Exception, 'unknown return %s' % buffer
time.sleep(1)
ret = subprocess.Popen(
["../../criu/criu", "dump", "-t",
str(p.pid), "-v4", "--external", ttyid]).wait()
if ret:
sys.exit(ret)
p.wait()
new_master, slave = pty.openpty() # get another pty pair
os.close(master)
ttyid = "fd[%d]:tty[%x:%x]" % (slave, st.st_rdev, st.st_dev)
ret = subprocess.Popen([
"../../criu/criu", "restore", "-v4", "--inherit-fd", ttyid,
"--restore-sibling", "--restore-detach"
]).wait()
if ret:
sys.exit(ret)
os.close(slave)
os.waitpid(-1, os.WNOHANG) # is the process alive
os.close(new_master)
_, status = os.wait()
if not os.WIFSIGNALED(status) or not os.WTERMSIG(status):
print status
sys.exit(1)
print "PASS"
def main():
EPILOG = """This script is not intended for use by end-users. To configure
Chrome Remote Desktop, please install the app from the Chrome
Web Store: https://chrome.google.com/remotedesktop"""
parser = optparse.OptionParser(
usage="Usage: %prog [options] [ -- [ X server options ] ]",
epilog=EPILOG)
parser.add_option(
"-s",
"--size",
dest="size",
action="append",
help="Dimensions of virtual desktop. This can be specified "
"multiple times to make multiple screen resolutions "
"available (if the Xvfb server supports this).")
parser.add_option("-f",
"--foreground",
dest="foreground",
default=False,
action="store_true",
help="Don't run as a background daemon.")
parser.add_option("",
"--start",
dest="start",
default=False,
action="store_true",
help="Start the host.")
parser.add_option("-k",
"--stop",
dest="stop",
default=False,
action="store_true",
help="Stop the daemon currently running.")
parser.add_option(
"",
"--check-running",
dest="check_running",
default=False,
action="store_true",
help="Return 0 if the daemon is running, or 1 otherwise.")
parser.add_option("",
"--config",
dest="config",
action="store",
help="Use the specified configuration file.")
parser.add_option(
"",
"--reload",
dest="reload",
default=False,
action="store_true",
help="Signal currently running host to reload the config.")
parser.add_option(
"",
"--add-user",
dest="add_user",
default=False,
action="store_true",
help="Add current user to the chrome-remote-desktop group.")
parser.add_option("",
"--host-version",
dest="host_version",
default=False,
action="store_true",
help="Prints version of the host.")
(options, args) = parser.parse_args()
# Determine the filename of the host configuration and PID files.
if not options.config:
options.config = os.path.join(CONFIG_DIR, "host#%s.json" % g_host_hash)
# Check for a modal command-line option (start, stop, etc.)
if options.check_running:
pid = get_daemon_pid()
return 0 if pid != 0 else 1
if options.stop:
pid = get_daemon_pid()
if pid == 0:
print "The daemon is not currently running"
else:
print "Killing process %s" % pid
os.kill(pid, signal.SIGTERM)
return 0
if options.reload:
pid = get_daemon_pid()
if pid == 0:
return 1
os.kill(pid, signal.SIGHUP)
return 0
if options.add_user:
if os.getenv("DISPLAY"):
sudo_command = "gksudo --description \"Chrome Remote Desktop\""
else:
sudo_command = "sudo"
command = (
"sudo -k && exec %(sudo)s -- sh -c "
"\"groupadd -f %(group)s && gpasswd --add %(user)s %(group)s\"" % {
'group': CHROME_REMOTING_GROUP_NAME,
'user': getpass.getuser(),
'sudo': sudo_command
})
os.execv("/bin/sh", ["/bin/sh", "-c", command])
return 1
if options.host_version:
# TODO(sergeyu): Also check RPM package version once we add RPM package.
return os.system(locate_executable(HOST_BINARY_NAME) +
" --version") >> 8
if not options.start:
# If no modal command-line options specified, print an error and exit.
print >> sys.stderr, EPILOG
return 1
# Collate the list of sizes that XRANDR should support.
if not options.size:
default_sizes = DEFAULT_SIZES
if os.environ.has_key(DEFAULT_SIZES_ENV_VAR):
default_sizes = os.environ[DEFAULT_SIZES_ENV_VAR]
options.size = default_sizes.split(",")
sizes = []
for size in options.size:
size_components = size.split("x")
if len(size_components) != 2:
parser.error("Incorrect size format '%s', should be WIDTHxHEIGHT" %
size)
try:
width = int(size_components[0])
height = int(size_components[1])
# Enforce minimum desktop size, as a sanity-check. The limit of 100 will
# detect typos of 2 instead of 3 digits.
if width < 100 or height < 100:
raise ValueError
except ValueError:
parser.error("Width and height should be 100 pixels or greater")
sizes.append((width, height))
# Register an exit handler to clean up session process and the PID file.
atexit.register(cleanup)
# Load the initial host configuration.
host_config = Config(options.config)
try:
host_config.load()
except (IOError, ValueError) as e:
print >> sys.stderr, "Failed to load config: " + str(e)
return 1
# Register handler to re-load the configuration in response to signals.
for s in [signal.SIGHUP, signal.SIGINT, signal.SIGTERM]:
signal.signal(s, SignalHandler(host_config))
# Verify that the initial host configuration has the necessary fields.
auth = Authentication()
auth_config_valid = auth.copy_from(host_config)
host = Host()
host_config_valid = host.copy_from(host_config)
if not host_config_valid or not auth_config_valid:
logging.error("Failed to load host configuration.")
return 1
# Determine whether a desktop is already active for the specified host
# host configuration.
pid = get_daemon_pid()
if pid != 0:
# Debian policy requires that services should "start" cleanly and return 0
# if they are already running.
print "Service already running."
return 0
# Detach a separate "daemon" process to run the session, unless specifically
# requested to run in the foreground.
if not options.foreground:
daemonize()
logging.info("Using host_id: " + host.host_id)
desktop = Desktop(sizes)
# Keep track of the number of consecutive failures of any child process to
# run for longer than a set period of time. The script will exit after a
# threshold is exceeded.
# There is no point in tracking the X session process separately, since it is
# launched at (roughly) the same time as the X server, and the termination of
# one of these triggers the termination of the other.
x_server_inhibitor = RelaunchInhibitor("X server")
host_inhibitor = RelaunchInhibitor("host")
all_inhibitors = [x_server_inhibitor, host_inhibitor]
# Don't allow relaunching the script on the first loop iteration.
allow_relaunch_self = False
while True:
# Set the backoff interval and exit if a process failed too many times.
backoff_time = SHORT_BACKOFF_TIME
for inhibitor in all_inhibitors:
if inhibitor.failures >= MAX_LAUNCH_FAILURES:
logging.error("Too many launch failures of '%s', exiting." %
inhibitor.label)
return 1
elif inhibitor.failures >= SHORT_BACKOFF_THRESHOLD:
backoff_time = LONG_BACKOFF_TIME
relaunch_times = []
# If the session process or X server stops running (e.g. because the user
# logged out), kill the other. This will trigger the next conditional block
# as soon as os.waitpid() reaps its exit-code.
if desktop.session_proc is None and desktop.x_proc is not None:
logging.info("Terminating X server")
desktop.x_proc.terminate()
elif desktop.x_proc is None and desktop.session_proc is not None:
logging.info("Terminating X session")
desktop.session_proc.terminate()
elif desktop.x_proc is None and desktop.session_proc is None:
# Both processes have terminated.
if (allow_relaunch_self and x_server_inhibitor.failures == 0
and host_inhibitor.failures == 0):
# Since the user's desktop is already gone at this point, there's no
# state to lose and now is a good time to pick up any updates to this
# script that might have been installed.
logging.info("Relaunching self")
relaunch_self()
else:
# If there is a non-zero |failures| count, restarting the whole script
# would lose this information, so just launch the session as normal.
if x_server_inhibitor.is_inhibited():
logging.info("Waiting before launching X server")
relaunch_times.append(
x_server_inhibitor.earliest_relaunch_time)
else:
logging.info("Launching X server and X session.")
desktop.launch_session(args)
x_server_inhibitor.record_started(MINIMUM_PROCESS_LIFETIME,
backoff_time)
allow_relaunch_self = True
if desktop.host_proc is None:
if host_inhibitor.is_inhibited():
logging.info("Waiting before launching host process")
relaunch_times.append(host_inhibitor.earliest_relaunch_time)
else:
logging.info("Launching host process")
desktop.launch_host(host_config)
host_inhibitor.record_started(MINIMUM_PROCESS_LIFETIME,
backoff_time)
deadline = min(relaunch_times) if relaunch_times else 0
pid, status = waitpid_handle_exceptions(-1, deadline)
if pid == 0:
continue
logging.info("wait() returned (%s,%s)" % (pid, status))
# When a process has terminated, and we've reaped its exit-code, any Popen
# instance for that process is no longer valid. Reset any affected instance
# to None.
if desktop.x_proc is not None and pid == desktop.x_proc.pid:
logging.info("X server process terminated")
desktop.x_proc = None
x_server_inhibitor.record_stopped()
if desktop.session_proc is not None and pid == desktop.session_proc.pid:
logging.info("Session process terminated")
desktop.session_proc = None
if desktop.host_proc is not None and pid == desktop.host_proc.pid:
logging.info("Host process terminated")
desktop.host_proc = None
desktop.host_ready = False
host_inhibitor.record_stopped()
# These exit-codes must match the ones used by the host.
# See remoting/host/host_error_codes.h.
# Delete the host or auth configuration depending on the returned error
# code, so the next time this script is run, a new configuration
# will be created and registered.
if os.WIFEXITED(status):
if os.WEXITSTATUS(status) == 100:
logging.info("Host configuration is invalid - exiting.")
host_config.clear_auth()
host_config.clear_host_info()
host_config.save_and_log_errors()
return 0
elif os.WEXITSTATUS(status) == 101:
logging.info("Host ID has been deleted - exiting.")
host_config.clear_host_info()
host_config.save_and_log_errors()
return 0
elif os.WEXITSTATUS(status) == 102:
logging.info("OAuth credentials are invalid - exiting.")
host_config.clear_auth()
host_config.save_and_log_errors()
return 0
elif os.WEXITSTATUS(status) == 103:
logging.info("Host domain is blocked by policy - exiting.")
host_config.clear_auth()
host_config.clear_host_info()
host_config.save_and_log_errors()
return 0
# Nothing to do for Mac-only status 104 (login screen unsupported)
elif os.WEXITSTATUS(status) == 105:
logging.info("Username is blocked by policy - exiting.")
host_config.clear_auth()
host_config.clear_host_info()
host_config.save_and_log_errors()
return 0
else:
logging.info("Host exited with status %s." %
os.WEXITSTATUS(status))
elif os.WIFSIGNALED(status):
logging.info("Host terminated by signal %s." %
os.WTERMSIG(status))
def _failed(self, status=0):
if self.status is None:
return None
if os.WIFSIGNALED(status):
return None
return _status(self) != status
def get_output(
self,
body,
headers=None,
include_dirs=None,
libraries=None,
library_dirs=None,
lang="c",
use_tee=None,
):
"""Try to compile, link to an executable, and run a program
built from 'body' and 'headers'. Returns the exit status code
of the program and its output.
"""
# 2008-11-16, RemoveMe
warnings.warn(
"\n+++++++++++++++++++++++++++++++++++++++++++++++++\n"
"Usage of get_output is deprecated: please do not \n"
"use it anymore, and avoid configuration checks \n"
"involving running executable on the target machine.\n"
"+++++++++++++++++++++++++++++++++++++++++++++++++\n",
DeprecationWarning,
stacklevel=2,
)
self._check_compiler()
exitcode, output = 255, ""
try:
grabber = GrabStdout()
try:
src, obj, exe = self._link(body, headers, include_dirs,
libraries, library_dirs, lang)
grabber.restore()
except Exception:
output = grabber.data
grabber.restore()
raise
exe = os.path.join(".", exe)
try:
# specify cwd arg for consistency with
# historic usage pattern of exec_command()
# also, note that exe appears to be a string,
# which exec_command() handled, but we now
# use a list for check_output() -- this assumes
# that exe is always a single command
output = subprocess.check_output([exe], cwd=".")
except subprocess.CalledProcessError as exc:
exitstatus = exc.returncode
output = ""
except OSError:
# preserve the EnvironmentError exit status
# used historically in exec_command()
exitstatus = 127
output = ""
else:
output = filepath_from_subprocess_output(output)
if hasattr(os, "WEXITSTATUS"):
exitcode = os.WEXITSTATUS(exitstatus)
if os.WIFSIGNALED(exitstatus):
sig = os.WTERMSIG(exitstatus)
log.error("subprocess exited with signal %d" % (sig, ))
if sig == signal.SIGINT:
# control-C
raise KeyboardInterrupt
else:
exitcode = exitstatus
log.info("success!")
except (CompileError, LinkError):
log.info("failure.")
self._clean()
return exitcode, output
from __future__ import print_function
import os
import signal
import sys
import time
for num_fils in range(0, 3):
if (os.fork() == 0):
if (num_fils == 0):
sys.exit(1)
elif (num_fils == 1):
sys.exit(2)
else:
os.kill(0, signal.SIGUSR1)
else:
while True:
try:
(pid, status) = os.wait()
if (os.WIFEXITED(status)):
print("Le processus", pid,
"s'est termine normalement avec le code",
os.WEXITSTATUS(status))
elif (os.WIFSIGNALED(status)):
print("Le processus", pid, "a ete tue par le signal",
os.WTERMSIG(status))
if (os.WCOREDUMP(status)):
print("Fichier core cree")
except:
break
def inhibit(self, *a):
"""inhibit a gluster filesystem
Mount glusterfs over a temporary mountpoint,
change into the mount, and lazy unmount the
filesystem.
"""
mpi, mpo = os.pipe()
mh = Popen.fork()
if mh:
os.close(mpi)
fcntl.fcntl(mpo, fcntl.F_SETFD, fcntl.FD_CLOEXEC)
d = None
margv = self.make_mount_argv(*a)
if self.mntpt:
# mntpt is determined pre-mount
d = self.mntpt
os.write(mpo, d + '\0')
po = Popen(margv, **self.mountkw)
self.handle_mounter(po)
po.terminate_geterr()
logging.debug('auxiliary glusterfs mount in place')
if not d:
# mntpt is determined during mount
d = self.mntpt
os.write(mpo, d + '\0')
os.write(mpo, 'M')
t = syncdutils.Thread(target=lambda: os.chdir(d))
t.start()
tlim = gconf.starttime + int(gconf.connection_timeout)
while True:
if not t.isAlive():
break
if time.time() >= tlim:
syncdutils.finalize(exval=1)
time.sleep(1)
os.close(mpo)
_, rv = syncdutils.waitpid(mh, 0)
if rv:
rv = (os.WIFEXITED(rv) and os.WEXITSTATUS(rv) or 0) - \
(os.WIFSIGNALED(rv) and os.WTERMSIG(rv) or 0)
logging.warn('stale mount possibly left behind on ' + d)
raise GsyncdError("cleaning up temp mountpoint %s failed with status %d" % \
(d, rv))
else:
rv = 0
try:
os.setsid()
os.close(mpo)
mntdata = ''
while True:
c = os.read(mpi, 1)
if not c:
break
mntdata += c
if mntdata:
mounted = False
if mntdata[-1] == 'M':
mntdata = mntdata[:-1]
assert (mntdata)
mounted = True
assert (mntdata[-1] == '\0')
mntpt = mntdata[:-1]
assert (mntpt)
if mounted:
po = self.umount_l(mntpt)
po.terminate_geterr(fail_on_err=False)
if po.returncode != 0:
po.errlog()
rv = po.returncode
self.cleanup_mntpt(mntpt)
except:
logging.exception('mount cleanup failure:')
rv = 200
os._exit(rv)
logging.debug('auxiliary glusterfs mount prepared')
def main(listener_fd, alive_r, preload, main_path=None, sys_path=None):
'''Run forkserver.'''
if preload:
if '__main__' in preload and main_path is not None:
process.current_process()._inheriting = True
try:
spawn.import_main_path(main_path)
finally:
del process.current_process()._inheriting
for modname in preload:
try:
__import__(modname)
except ImportError:
pass
util._close_stdin()
sig_r, sig_w = os.pipe()
os.set_blocking(sig_r, False)
os.set_blocking(sig_w, False)
def sigchld_handler(*_unused):
# Dummy signal handler, doesn't do anything
pass
# letting SIGINT through avoids KeyboardInterrupt tracebacks
# unblocking SIGCHLD allows the wakeup fd to notify our event loop
handlers = {
signal.SIGCHLD: sigchld_handler,
signal.SIGINT: signal.SIG_DFL,
}
old_handlers = {
sig: signal.signal(sig, val)
for (sig, val) in handlers.items()
}
# calling os.write() in the Python signal handler is racy
signal.set_wakeup_fd(sig_w)
# map child pids to client fds
pid_to_fd = {}
with socket.socket(socket.AF_UNIX, fileno=listener_fd) as listener, \
selectors.DefaultSelector() as selector:
_forkserver._forkserver_address = listener.getsockname()
selector.register(listener, selectors.EVENT_READ)
selector.register(alive_r, selectors.EVENT_READ)
selector.register(sig_r, selectors.EVENT_READ)
while True:
try:
while True:
rfds = [key.fileobj for (key, events) in selector.select()]
if rfds:
break
if alive_r in rfds:
# EOF because no more client processes left
assert os.read(alive_r, 1) == b'', "Not at EOF?"
raise SystemExit
if sig_r in rfds:
# Got SIGCHLD
os.read(sig_r, 65536) # exhaust
while True:
# Scan for child processes
try:
pid, sts = os.waitpid(-1, os.WNOHANG)
except ChildProcessError:
break
if pid == 0:
break
child_w = pid_to_fd.pop(pid, None)
if child_w is not None:
if os.WIFSIGNALED(sts):
returncode = -os.WTERMSIG(sts)
else:
if not os.WIFEXITED(sts):
raise AssertionError(
"Child {0:n} status is {1:n}".format(
pid, sts))
returncode = os.WEXITSTATUS(sts)
# Send exit code to client process
try:
write_signed(child_w, returncode)
except BrokenPipeError:
# client vanished
pass
os.close(child_w)
else:
# This shouldn't happen really
warnings.warn('forkserver: waitpid returned '
'unexpected pid %d' % pid)
if listener in rfds:
# Incoming fork request
with listener.accept()[0] as s:
# Receive fds from client
fds = reduction.recvfds(s, MAXFDS_TO_SEND + 1)
if len(fds) > MAXFDS_TO_SEND:
raise RuntimeError(
"Too many ({0:n}) fds to send".format(
len(fds)))
child_r, child_w, *fds = fds
s.close()
pid = os.fork()
if pid == 0:
# Child
code = 1
try:
listener.close()
selector.close()
unused_fds = [alive_r, child_w, sig_r, sig_w]
unused_fds.extend(pid_to_fd.values())
code = _serve_one(child_r, fds, unused_fds,
old_handlers)
except Exception:
sys.excepthook(*sys.exc_info())
sys.stderr.flush()
finally:
os._exit(code)
else:
# Send pid to client process
try:
write_signed(child_w, pid)
except BrokenPipeError:
# client vanished
pass
pid_to_fd[pid] = child_w
os.close(child_r)
for fd in fds:
os.close(fd)
except OSError as e:
if e.errno != errno.ECONNABORTED:
raise
def cmd(self,
cmd,
soutfile=None,
allowed_exit=None,
capture_stderr=False,
timeout=None,
mention_outputfile_on_errors=True,
python=False):
"Execute an OS command and captures the stderr and stdout which are returned in a file"
if allowed_exit is None:
allowed_exit = [0]
if not soutfile:
soutfile = tempfile.NamedTemporaryFile(mode='w+t',
suffix='.out',
delete=False).name
logger.debug('Running shell command: %s' % cmd)
try:
t0 = time.time()
already_killed = False
timeout0 = timeout
launcher = ['/bin/sh', '-c']
args = ['%s > %s 2>&1' % (cmd, soutfile)]
command = launcher
for i in args:
command.append(i)
this_cwd = os.path.abspath(os.getcwd())
if not os.path.exists(this_cwd):
this_cwd = os.path.abspath(tempfile.gettempdir())
logger.debug("Using CWD: %s" % this_cwd)
#import traceback
# traceback.print_stack()
process = subprocess.Popen(command, env=self.env, cwd=this_cwd)
pid = process.pid
while True:
wpid, sts = os.waitpid(pid, os.WNOHANG)
if wpid != 0:
if os.WIFSIGNALED(sts):
rc = -os.WTERMSIG(sts)
break
elif os.WIFEXITED(sts):
rc = os.WEXITSTATUS(sts)
break
if timeout and time.time() - t0 > timeout:
logger.warning(
'Command interrupted - timeout %ss reached: %s',
timeout0, cmd)
if already_killed:
sig = signal.SIGKILL
else:
sig = signal.SIGTERM
logger.debug('killing process %d with signal %d', pid, sig)
os.kill(pid, sig)
t0 = time.time()
# wait just 5 seconds before killing with SIGKILL
timeout = 5
already_killed = True
time.sleep(0.05)
except OSError as e:
if e.errno == 10:
rc = process.returncode
logger.debug(
"Process has already exitted which will throw a 10")
logger.debug("Exit status is: %s" % rc)
else:
logger.warning('Problem with shell command: %s, %s', e.errno,
e.strerror)
rc = 255
BYTES = 4096
if rc not in allowed_exit:
logger.warning('exit status [%d] of command %s', rc, cmd)
if mention_outputfile_on_errors:
logger.warning('full output is in file: %s', soutfile)
with open(soutfile) as sout_file:
logger.warning('<first %d bytes of output>\n%s', BYTES,
sout_file.read(BYTES))
logger.warning('<end of first %d bytes of output>', BYTES)
# FIXME /bin/sh might have also other error messages
m = None
if rc != 0:
with open(soutfile) as sout_file:
m = re.search('command not found\n', sout_file.read())
if m:
logger.warning('command %s not found', cmd)
return rc, soutfile, m is None
def exit_signalled(s):
""" child teminated due to receipt of SIGUSR1 """
return (os.WIFSIGNALED(s) and (os.WTERMSIG(s) == signal.SIGUSR1))
def _spawn_posix(cmd, search_path=1, verbose=0, dry_run=0):
log.info(' '.join(cmd))
if dry_run:
return
executable = cmd[0]
exec_fn = search_path and os.execvp or os.execv
env = None
if sys.platform == 'darwin':
global _cfg_target, _cfg_target_split
if _cfg_target is None:
_cfg_target = sysconfig.get_config_var(
'MACOSX_DEPLOYMENT_TARGET') or ''
if _cfg_target:
_cfg_target_split = [int(x) for x in _cfg_target.split('.')]
if _cfg_target:
# ensure that the deployment target of build process is not less
# than that used when the interpreter was built. This ensures
# extension modules are built with correct compatibility values
cur_target = os.environ.get('MACOSX_DEPLOYMENT_TARGET',
_cfg_target)
if _cfg_target_split > [int(x) for x in cur_target.split('.')]:
my_msg = ('$MACOSX_DEPLOYMENT_TARGET mismatch: '
'now "%s" but "%s" during configure' %
(cur_target, _cfg_target))
raise DistutilsPlatformError(my_msg)
env = dict(os.environ, MACOSX_DEPLOYMENT_TARGET=cur_target)
exec_fn = search_path and os.execvpe or os.execve
pid = os.fork()
if pid == 0: # in the child
try:
if env is None:
exec_fn(executable, cmd)
else:
exec_fn(executable, cmd, env)
except OSError as e:
if not DEBUG:
cmd = executable
sys.stderr.write("unable to execute %r: %s\n" % (cmd, e.strerror))
os._exit(1)
if not DEBUG:
cmd = executable
sys.stderr.write("unable to execute %r for unknown reasons" % cmd)
os._exit(1)
else: # in the parent
# Loop until the child either exits or is terminated by a signal
# (ie. keep waiting if it's merely stopped)
while True:
try:
pid, status = os.waitpid(pid, 0)
except OSError as exc:
if not DEBUG:
cmd = executable
raise DistutilsExecError("command %r failed: %s" %
(cmd, exc.args[-1]))
if os.WIFSIGNALED(status):
if not DEBUG:
cmd = executable
raise DistutilsExecError("command %r terminated by signal %d" %
(cmd, os.WTERMSIG(status)))
elif os.WIFEXITED(status):
exit_status = os.WEXITSTATUS(status)
if exit_status == 0:
return # hey, it succeeded!
else:
if not DEBUG:
cmd = executable
raise DistutilsExecError(
"command %r failed with exit status %d" %
(cmd, exit_status))
elif os.WIFSTOPPED(status):
continue
else:
if not DEBUG:
cmd = executable
raise DistutilsExecError(
"unknown error executing %r: termination status %d" %
(cmd, status))
children = []
while running[0]:
while len(children) < worker_count:
pid = os.fork()
if pid == 0:
signal.signal(signal.SIGHUP, signal.SIG_DFL)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
run_server()
logger.notice('Child %d exiting normally' % os.getpid())
return
else:
logger.notice('Started child %s' % pid)
children.append(pid)
try:
pid, status = os.wait()
if os.WIFEXITED(status) or os.WIFSIGNALED(status):
logger.error('Removing dead child %s' % pid)
children.remove(pid)
except OSError, err:
if err.errno not in (errno.EINTR, errno.ECHILD):
raise
except KeyboardInterrupt:
logger.notice('User quit')
break
greenio.shutdown_safe(sock)
sock.close()
logger.notice('Exited')
class ConfigFileError(Exception):
pass
out_limit = max(2**15, os.stat(file_o).st_size * 4)
result = os.read(r, mmap.PAGESIZE)
while len(result) < out_limit:
buf = os.read(r, mmap.PAGESIZE)
if len(buf) == 0:
break
result += buf
os.kill(submission, signal.SIGKILL)
os.close(r)
_, return_status, rusage = os.wait4(submission, 0)
# TODO: Other processes randomly sending signals can break this
if len(result) > out_limit:
verdict = Verdict.OLE
elif (os.WIFSIGNALED(return_status) and
os.WTERMSIG(return_status) == signal.SIGKILL or rusage.ru_utime > self.time_limit):
verdict = Verdict.TLE
elif os.WIFSIGNALED(return_status):
verdict = Verdict.RTE
elif os.WEXITSTATUS(return_status) != 0:
verdict = Verdict.RE
else:
with open(file_o, 'rb') as out:
answer = out.read()
verdict = Verdict.AC if self.check(answer, result) else Verdict.WA
return Case(case, verdict, rusage.ru_utime, rusage.ru_maxrss, return_status)
def file_sort(self, text):
convert = (lambda c: int(c) if c.isdigit() else c)
return list(map(convert, re.split(r'(\d+)', text)))
debug_print('#################################')
debug_print('Performing C to VHDL stage...')
debug_print('#################################')
# Execute gcc2suif
suif_filename = c_filename_prefix + '.suif'
cmd_convert_c_to_suif = GCC2SUIF + SPACE + c_filename
debug_print('EXECUTE: ' + cmd_convert_c_to_suif)
retval = os.system(cmd_convert_c_to_suif)
if os.WEXITSTATUS(retval) != 0:
error_print(
ERROR_GCC2SUIF,
'GCC2SUIF COMPILATION ERROR(retval=%d)' % os.WEXITSTATUS(retval))
elif os.WIFSIGNALED(retval):
# USER SIGINT SIGNALED
if os.WTERMSIG(retval) == 2:
error_print(
ERROR_SIGINT_KILL, "TERMINATED BY USER (SIGNAL = %d) = %s" %
(os.WTERMSIG(retval), sig_dict[os.WTERMSIG(retval)]))
# possible segfault
else:
error_print(
ERROR_GCC2SUIF, "TERMINATED BY SIGNAL(%d) = %s" %
(os.WTERMSIG(retval), sig_dict[os.WTERMSIG(retval)]))
for pass_list in CONST_NEW_SUIFDRIVER_PASSES_LIST:
# Execute all optimization stages
cmd_suifdriver_execute = SUIFDRIVER + ' -e ' + DOUBLEQUOTE + pass_list + DOUBLEQUOTE
os.system('echo "' + pass_list + '" > SUIFDRIVER_PASSES.LST')
def wait_pid(pid,
timeout=None,
proc_name=None,
_waitpid=os.waitpid,
_timer=getattr(time, 'monotonic', time.time),
_min=min,
_sleep=time.sleep,
_pid_exists=pid_exists):
"""Wait for a process PID to terminate.
If the process terminated normally by calling exit(3) or _exit(2),
or by returning from main(), the return value is the positive integer
passed to *exit().
If it was terminated by a signal it returns the negated value of the
signal which caused the termination (e.g. -SIGTERM).
If PID is not a children of os.getpid() (current process) just
wait until the process disappears and return None.
If PID does not exist at all return None immediately.
If *timeout* != None and process is still alive raise TimeoutExpired.
timeout=0 is also possible (either return immediately or raise).
"""
if pid <= 0:
raise ValueError("can't wait for PID 0") # see "man waitpid"
interval = 0.0001
flags = 0
if timeout is not None:
flags |= os.WNOHANG
stop_at = _timer() + timeout
def sleep(interval):
# Sleep for some time and return a new increased interval.
if timeout is not None:
if _timer() >= stop_at:
raise TimeoutExpired(timeout, pid=pid, name=proc_name)
_sleep(interval)
return _min(interval * 2, 0.04)
# See: https://linux.die.net/man/2/waitpid
while True:
try:
retpid, status = os.waitpid(pid, flags)
except InterruptedError:
interval = sleep(interval)
except ChildProcessError:
# This has two meanings:
# - PID is not a child of os.getpid() in which case
# we keep polling until it's gone
# - PID never existed in the first place
# In both cases we'll eventually return None as we
# can't determine its exit status code.
while _pid_exists(pid):
interval = sleep(interval)
return
else:
if retpid == 0:
# WNOHANG flag was used and PID is still running.
interval = sleep(interval)
continue
elif os.WIFEXITED(status):
# Process terminated normally by calling exit(3) or _exit(2),
# or by returning from main(). The return value is the
# positive integer passed to *exit().
return os.WEXITSTATUS(status)
elif os.WIFSIGNALED(status):
# Process exited due to a signal. Return the negative value
# of that signal.
return negsig_to_enum(-os.WTERMSIG(status))
# elif os.WIFSTOPPED(status):
# # Process was stopped via SIGSTOP or is being traced, and
# # waitpid() was called with WUNTRACED flag. PID is still
# # alive. From now on waitpid() will keep returning (0, 0)
# # until the process state doesn't change.
# # It may make sense to catch/enable this since stopped PIDs
# # ignore SIGTERM.
# interval = sleep(interval)
# continue
# elif os.WIFCONTINUED(status):
# # Process was resumed via SIGCONT and waitpid() was called
# # with WCONTINUED flag.
# interval = sleep(interval)
# continue
else:
# Should never happen.
raise ValueError("unknown process exit status %r" % status)
