def _process_worker(call_queue, result_queue):
"""
Evaluates calls from call_queue and places the results in result_queue.
This worker is run in a separate process.
Args:
call_queue: A multiprocessing.Queue of _CallItems that will be read and
evaluated by the worker.
result_queue: A multiprocessing.Queue of _ResultItems that will written
to by the worker.
shutdown: A multiprocessing.Event that will be set as a signal to the
worker that it should exit when call_queue is empty.
"""
prctl.set_pdeathsig(signal.SIGKILL)
while True:
call_item = call_queue.get(block=True)
if call_item is None:
# Wake up queue management thread
result_queue.put(None)
return
try:
r = call_item.fn(*call_item.args, **call_item.kwargs)
except BaseException:
e = sys.exc_info()[1]
result_queue.put(_ResultItem(call_item.work_id,
exception=e))
else:
result_queue.put(_ResultItem(call_item.work_id,
result=r))
def _process_configure() -> None:
if os.getuid() == 0:
os.setgid(shutil._get_gid('postgres')) # type: ignore
os.setuid(shutil._get_uid('postgres')) # type: ignore
# Setting "pdeathsig" will make the process be killed if our process dies for any reason.
prctl.set_pdeathsig(signal.SIGTERM)
def _process_worker(call_queue, result_queue):
"""
Evaluates calls from call_queue and places the results in result_queue.
This worker is run in a separate process.
Args:
call_queue: A multiprocessing.Queue of _CallItems that will be read and
evaluated by the worker.
result_queue: A multiprocessing.Queue of _ResultItems that will written
to by the worker.
shutdown: A multiprocessing.Event that will be set as a signal to the
worker that it should exit when call_queue is empty.
"""
prctl.set_pdeathsig(signal.SIGKILL)
while True:
call_item = call_queue.get(block=True)
if call_item is None:
# Wake up queue management thread
result_queue.put(None)
return
try:
r = call_item.fn(*call_item.args, **call_item.kwargs)
except BaseException:
e = sys.exc_info()[1]
result_queue.put(_ResultItem(call_item.work_id, exception=e))
else:
result_queue.put(_ResultItem(call_item.work_id, result=r))
def pre_exec_fn(self, i):
#This is called after fork(), but before exec()
#Assign this new process to a new group
if self.NEW_PROCESS_GROUP:
os.setpgrp()
if self.LINUX_USE_PDEATHSIG:
prctl.set_pdeathsig(self.get_pdeath_sig())
def exit_when_parent_or_child_dies(sig):
# 当前进程组的ID
gid = os.getpgrp()
signal(SIGCHLD, make_child_die_signal_handler(gid))
# print("gid_c:",gid)
try:
import prctl
signal(SIGHUP, make_quit_signal_handler(gid))
# give me SIGHUP if my parent dies
# 如果父进程挂掉了，给自己SIGHUP信号？
prctl.set_pdeathsig(SIGHUP)
while True:
pause()
except ImportError:
# fallback to polling status of parent
while True:
# print("os.getppid()", os.getppid())
# print("os.getpid",os.getpid())
if os.getppid() == 1:
# parent died, suicide
signal(SIGTERM, SIG_DFL)
os.killpg(gid, sig)
sys.exit()
time.sleep(20)
def preexec():
# TODO investigate what happens when this limit is really low
# Note: this limit is broken on Mac, check it on Linux
resource.setrlimit(
resource.RLIMIT_AS,
(params.memory_limit_bytes, params.memory_limit_bytes))
if prctl_enabled:
prctl.set_pdeathsig(signal.SIGKILL)
def preexec_function_escdaq():
try:
dem = demote.demote('es-cdaq')
dem()
except:
pass
try:
prctl.set_pdeathsig(signal.SIGKILL) #is this necessary?
except:pass
def preexec_function_elasticsearch():
try:
dem = demote.demote('elasticsearch')
dem()
except:
pass
try:
prctl.set_pdeathsig(signal.SIGKILL) #is this necessary?
except:pass
def spawn_child(work):
pid = os.fork()
if (pid == 0):
prctl.set_pdeathsig(signal.SIGHUP)
try:
work()
except KeyboardInterrupt:
print "{0:d}: exiting".format(os.getpid())
sys.exit()
return pid
def spawn_child(work):
pid = os.fork()
if (pid == 0):
prctl.set_pdeathsig(signal.SIGHUP)
try:
work()
except KeyboardInterrupt:
print "{0:d}: exiting".format(os.getpid())
sys.exit()
return pid
def _wakeup(sec):
"""Waiting functions, used to wake up the process pool"""
try:
import prctl
except ImportError:
pass
else:
# if the parent dies, the children die
prctl.set_pdeathsig(signal.SIGKILL)
time.sleep(sec)
return os.getpid()
def _wakeup(sec):
"""Waiting functions, used to wake up the process pool"""
try:
import prctl
except ImportError:
pass
else:
# if the parent dies, the children die
prctl.set_pdeathsig(signal.SIGKILL)
time.sleep(sec)
return os.getpid()
def main():
(opts, args) = parse_arguments(sys.argv[1:])
out = opts.output
pid = os.fork()
if pid == 0:
# In child process:
# Make sure we die with the parent and are not left behind
# WARNING: This uses the prctl(2) call and is Linux-specific.
prctl.set_pdeathsig(signal.SIGHUP)
# exec command specified in arguments,
# searching the $PATH, keeping all environment
os.execvpe(args[0], args, os.environ)
sys.stderr.write("execvpe failed, exiting with non-zero status")
os.exit(1)
# In parent process:
iofname = "/proc/%d/io" % pid
iof = open(iofname, "r", 0) # 0: unbuffered open
sys.stderr.write("%s: created child PID = %d, monitoring file %s\n" %
(sys.argv[0], pid, iofname))
message = {}
message['type'] = MSG_TYPE
message['total'] = opts.read_bytes
while True:
# check if the child process is still alive
(wpid, status) = os.waitpid(pid, os.WNOHANG)
if wpid == pid:
report_wait_status(pid, status)
if (os.WIFEXITED(status) or os.WIFSIGNALED(status)):
if not (os.WIFEXITED(status) and os.WEXITSTATUS(status) == 0):
return 1
else:
message['position'] = message['total']
message['progress'] = float(100)
send_message(out, message)
return 0
iof.seek(0)
for l in iof.readlines():
if l.startswith("rchar:"):
message['position'] = int(l.split(': ')[1])
message['progress'] = float(0) if opts.read_bytes == 0 \
else float("%2.2f" % (
message['position'] * 100.0 / message['total']))
send_message(out, message)
break
# Sleep for a while
time.sleep(3)
def make_driver():
port = 8123
sys.path.append(os.path.dirname(os.path.realpath(__file__)))
script_dir = os.path.dirname(os.path.realpath(__file__))
# Start an HTTP server in the "unpacked" folder
http_server_proc = Popen(
["python", "-m", "SimpleHTTPServer",
str(port)],
cwd=script_dir + "/../../../../",
preexec_fn=lambda: prctl.set_pdeathsig(signal.SIGKILL))
# Start selenium
selenium_env = os.environ.copy()
selenium_env["PATH"] = script_dir + ":" + selenium_env["PATH"]
print("Selenium env: ", selenium_env)
selenium_proc = Popen(
[
"java", "-jar",
script_dir + "/binaries/selenium-server-standalone-2.53.1.jar"
],
env=selenium_env,
preexec_fn=lambda: prctl.set_pdeathsig(signal.SIGKILL))
global driver
driver = webdriver.Chrome(executable_path=script_dir +
"/binaries/chromedriver")
# Navigate to page
driver.get("http://*****:*****@id="MathJax-Element-1-Frame"]/*[name()="svg"]/*[name()="g"]/*[name()="g"]'
)
svg.click()
# Delete the single "x" we start with
ActionChains(driver).key_down(Keys.SHIFT).send_keys(
Keys.ARROW_RIGHT).send_keys(Keys.BACKSPACE).key_up(
Keys.SHIFT).perform()
return driver
def enable_death_signal():
"""
Set the "death signal" of the current process, so that
the current process will be cleaned with guarantee
in case the parent dies accidentally.
"""
try:
import prctl
except ImportError:
return
else:
# is SIGHUP a good choice?
prctl.set_pdeathsig(signal.SIGHUP)
def enable_death_signal():
"""
Set the "death signal" of the current process, so that
the current process will be cleaned with guarantee
in case the parent dies accidentally.
"""
try:
import prctl
except ImportError:
return
else:
# is SIGHUP a good choice?
prctl.set_pdeathsig(signal.SIGHUP)
def init_workers():
"""Waiting function, used to wake up the process pool"""
setproctitle('oq-worker')
# unregister raiseMasterKilled in oq-workers to avoid deadlock
# since processes are terminated via pool.terminate()
signal.signal(signal.SIGTERM, signal.SIG_DFL)
# prctl is still useful (on Linux) to terminate all spawned processes
# when master is killed via SIGKILL
try:
import prctl
except ImportError:
pass
else:
# if the parent dies, the children die
prctl.set_pdeathsig(signal.SIGKILL)
def enable_death_signal():
"""
Set the "death signal" of the current process, so that
the current process will be cleaned with guarantee
in case the parent dies accidentally.
"""
try:
import prctl # pip install prctl-python
except ImportError:
return
else:
assert hasattr(prctl, 'set_pdeathsig'), \
"prctl.set_pdeathsig does not exist! Note that you need to install 'prctl-python' instead of 'prctl'."
# is SIGHUP a good choice?
prctl.set_pdeathsig(signal.SIGHUP)
def enable_death_signal():
"""
Set the "death signal" of the current process, so that
the current process will be cleaned with guarantee
in case the parent dies accidentally.
"""
try:
import prctl # pip install prctl-python
except ImportError:
return
else:
assert hasattr(prctl, 'set_pdeathsig'), \
"prctl.set_pdeathsig does not exist! Note that you need to install 'prctl-python' instead of 'prctl'."
# is SIGHUP a good choice?
prctl.set_pdeathsig(signal.SIGHUP)
def runcmd(cmd, raise_err=False, log=None):
"""Run given command.
Return tuple of (RC, stdout, stderr)."""
execprocess = Popen(
[cmd],
stdout=PIPE,
stderr=PIPE,
# if parent process dies, child procs do also
preexec_fn=lambda: prctl.set_pdeathsig(signal.SIGKILL),
shell=True)
(stdout, stderr) = execprocess.communicate()
returncode = execprocess.returncode
stdout = ANSI_ESCAPE.sub('', stdout.decode('utf-8'))
stderr = ANSI_ESCAPE.sub('', stderr.decode('utf-8'))
# Start from new line for easy reading of the log output:
msg = ("\nCOMMAND: %s\nSTDOUT: %s\nSTDERR: %s\nRETURNCODE=%d" %
(cmd, stdout or "", stderr or "", returncode)).replace(
SSH_CMD, "ssh")
if raise_err and execprocess.returncode:
err = "Command failed: {}".format(msg)
raise Exception(err)
if log:
log.debug(msg)
return (returncode, stdout, stderr)
def run_process(target, *args, auto_quit=True, **kwargs):
try:
pid = os.fork()
except OSError:
print("unable to fork")
sys.exit(-1)
if pid:
return
else:
if auto_quit and prctl:
prctl.set_pdeathsig(signal.SIGTERM)
try:
target(*args, **kwargs)
except KeyboardInterrupt:
sys.stderr.write("process exiting...\n")
sys.stderr.flush()
sys.exit()
def run(self, fn):
"""
Runs some function from within a dummy child process in its own pgroup.
This allows for calls us to send signals to kill the entire pgroup,
and also allows for send a deathsig in case the subprocess resets
its pgroup. This should handle if a subprocess also calls to
CleanChildProcesses.
"""
self.is_stopped = False
"""
When running out of remote shell, SIGHUP is only sent to the session
leader normally, the remote shell, so we need to make sure we are sent
SIGHUP. This also allows us not to kill ourselves with SIGKILL.
- A process group is called orphaned when the parent of every member is
either in the process group or outside the session. In particular,
the process group of the session leader is always orphaned.
- If termination of a process causes a process group to become orphaned,
and some member is stopped, then all are sent first SIGHUP and then
SIGCONT.
"""
self.childpid = os.fork() # return 0 in the child branch, and the childpid in the parent branch
if self.childpid == 0:
try:
os.setpgrp() # create new process group, become its leader
"""
Using prctl means that if a subprocesses run from within this scope
also uses a CleanProcessGroup, that group of processes will still get
stopped.
Note: I am not guranteeing this because prctl isn't in the std lib,
and I previousely had some issues downloading it. Can't pip3 it used
"sudo apt-get install python3-prctl".
"""
prctl.set_pdeathsig(signal.SIGINT)
return fn()
os.kill(os.getpid(), signal.SIGSTOP) # child fork stops itself
finally:
os._exit(0) # shut down without going to __exit__
os.waitpid(self.childpid, os.WUNTRACED) # wait until child stopped after it created the process group
os.setpgid(0, self.childpid) # join child's group so processes spawned under this classes scope will be within the child's group.
self.is_foreground = self._maybe_become_foreground()
self.exit_signals = {s: signal.signal(s, self._signal_hdlr)
for s in self.SIGNALS}
def enable_death_signal():
"""
Set the "death signal" of the current process, so that
the current process will be cleaned with guarantee
in case the parent dies accidentally.
"""
if platform.system() != 'Linux':
return
try:
import prctl # pip install python-prctl
except ImportError:
log_once('Install python-prctl so that processes can be cleaned with guarantee.', 'warn')
return
else:
assert hasattr(prctl, 'set_pdeathsig'), \
"prctl.set_pdeathsig does not exist! Note that you need to install 'python-prctl' instead of 'prctl'."
# is SIGHUP a good choice?
prctl.set_pdeathsig(signal.SIGHUP)
def enable_death_signal():
"""
Set the "death signal" of the current process, so that
the current process will be cleaned with guarantee
in case the parent dies accidentally.
"""
if platform.system() != 'Linux':
return
try:
import prctl # pip install python-prctl
except ImportError:
log_once('Install python-prctl so that processes can be cleaned with guarantee.', 'warn')
return
else:
assert hasattr(prctl, 'set_pdeathsig'), \
"prctl.set_pdeathsig does not exist! Note that you need to install 'python-prctl' instead of 'prctl'."
# is SIGHUP a good choice?
prctl.set_pdeathsig(signal.SIGHUP)
def exit_when_parent_or_child_dies(sig):
gid = os.getpgrp()
signal(SIGCHLD, make_child_die_signal_handler(gid))
try:
import prctl
signal(SIGHUP, make_quit_signal_handler(gid))
# give me SIGHUP if my parent dies
prctl.set_pdeathsig(SIGHUP)
pause()
except ImportError:
# fallback to polling status of parent
while True:
if os.getppid() == 1:
# parent died, suicide
signal(SIGTERM, SIG_DFL)
os.killpg(gid, sig)
sys.exit()
time.sleep(5)
def make_driver():
port = 8123
sys.path.append(os.path.dirname(os.path.realpath(__file__)))
script_dir = os.path.dirname(os.path.realpath(__file__))
# Start an HTTP server in the "unpacked" folder
http_server_proc = Popen(["python", "-m", "SimpleHTTPServer", str(port)],
cwd=script_dir + "/../../../../",
preexec_fn=lambda: prctl.set_pdeathsig(signal.SIGKILL))
# Start selenium
selenium_env = os.environ.copy()
selenium_env["PATH"] = script_dir + ":" + selenium_env["PATH"]
print("Selenium env: ", selenium_env)
selenium_proc = Popen(["java", "-jar", script_dir + "/binaries/selenium-server-standalone-2.53.1.jar"],
env=selenium_env,
preexec_fn=lambda: prctl.set_pdeathsig(signal.SIGKILL))
global driver
driver = webdriver.Chrome(executable_path=script_dir + "/binaries/chromedriver")
# Navigate to page
driver.get("http://*****:*****@id="MathJax-Element-1-Frame"]/*[name()="svg"]/*[name()="g"]/*[name()="g"]')
svg.click()
# Delete the single "x" we start with
ActionChains(driver).key_down(Keys.SHIFT).send_keys(Keys.ARROW_RIGHT).send_keys(Keys.BACKSPACE).key_up(Keys.SHIFT).perform()
return driver
def _run_server(bind_address):
"""Start a server in a subprocess."""
prctl.set_pdeathsig(signal.SIGTERM)
logging.info('Starting new server.')
options = (
('grpc.so_reuseport', 1),
('grpc.max_send_message_length', 128 * 1024 * 1024),
('grpc.max_receive_message_length', 128 * 1024 * 1024),
)
# WARNING: This example takes advantage of SO_REUSEPORT. Due to the
# limitations of manylinux1, none of our precompiled Linux wheels currently
# support this option. (https://github.com/grpc/grpc/issues/18210). To take
# advantage of this feature, install from source with
# `pip install grpcio --no-binary grpcio`.
server = grpc.server(futures.ThreadPoolExecutor(max_workers=4),
options=options)
aggregator_pb2_grpc.add_AggregatorServicer_to_server(Aggregator(), server)
server.add_insecure_port(bind_address)
server.start()
_wait_forever(server)
def run(max_allowed, args):
global process
temp = get_temperature_nvidia()
if (temp > max_allowed):
print(
"ERROR: Temperature above max limit, not staring (temp, max: %d, %d)"
% (temp, max_allowed))
return
try:
# Prctl ensures the subprocess gets KILL signal when super exits
process = subprocess.Popen(
args, preexec_fn=lambda: prctl.set_pdeathsig(signal.SIGKILL))
while True:
temp = get_temperature_nvidia()
if (temp > max_allowed):
print(
"ERROR: Temperature above max limit, killing process(temp, max: %d, %d)"
% (temp, max_allowed))
kill(process)
try:
process.wait(timeout=1.0)
return process.returncode
except subprocess.TimeoutExpired as e:
pass
except KeyboardInterrupt as ki:
kill(process, True)
except Exception as e:
print("ERROR: Got exception from main loop, killing process")
kill(process)
raise (e)
def _listener_thread(self):
listener = os.path.join(UTILS_DIR, b"intercom_listener/dist/Debug/GNU-Linux-x86/intercom_listener")
while True:
try:
process = subprocess.Popen([listener, self.args["audio_input"]], stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
preexec_fn=lambda: prctl.set_pdeathsig(signal.SIGKILL))
fd = process.stdout.fileno()
fl = fcntl.fcntl(fd, fcntl.F_GETFL)
fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)
self.set_error(None)
while True:
process.poll()
if process.returncode is not None:
raise Exception("Listener process returned code %d" % process.returncode)
try:
input = process.stdout.readline()
except IOError as e:
if e.errno != errno.EWOULDBLOCK:
raise
else:
if input:
self.emit_signal("listener_input", input=input.strip())
while not self.tell_listener_queue.empty():
process.stdin.write(self.tell_listener_queue.get() + "\n")
time.sleep(0.01)
except:
self.set_error({"listener_thread": sys.exc_info()})
time.sleep(1)
import logging
import signal
import sys
import os
from django.core.management.base import BaseCommand, CommandError
from grading.models import GradingAttempt, attempt_grading
prctl_enabled = False
if os.getenv('prctl_disabled', None) != "1":
prctl_enabled = True
import prctl
prctl.set_pdeathsig(signal.SIGKILL)
logger = logging.getLogger(__name__)
def exit_on_signal(signum, frame):
logger.info('Grading terminated by signal %s', signum)
sys.exit(1)
class Command(BaseCommand):
help = 'Grades single attempt (internal)'
def add_arguments(self, parser):
parser.add_argument('attempt_id', type=int)
def handle(self, *args, **options):
signal.signal(signal.SIGINT, exit_on_signal)
sys.path.append(os.path.join(os.environ['dipole_topdir'], "src"))
import libdipole
import FusekiConnection
@libdipole.exportclass
class FusekiDatasets:
def __init__(self, dataset_url):
self.dataset_url = dataset_url
def get_dataset_url(self):
return self.dataset_url
if __name__ == "__main__":
# https://github.com/seveas/python-prctl -- prctl wrapper module
# more on pdeathsignal: https://stackoverflow.com/questions/284325/how-to-make-child-process-die-after-parent-exits
prctl.set_pdeathsig(signal.SIGTERM) # if parent dies this child will get SIGTERM
dpl_server = libdipole.DipoleServer()
dpl_event_handler = libdipole.BackendEventHandler(libdipole.port_assignment_handler, sys.argv[1])
dpl_server.set_event_handler(dpl_event_handler)
dispatcher = libdipole.Dispatcher(dpl_server)
dpl_event_handler.dispatcher = dispatcher
dataset_url = 'http://localhost:3030/testdb'
print "adding object datasets"
dispatcher.add_object("datasets", FusekiDatasets(dataset_url))
print "adding object shacl_editor"
dispatcher.add_object("shacl_editor", FusekiConnection.FusekiConnection())
dpl_server.run_listener(port = 0)
def connect(self):
signal.signal(signal.SIGCHLD, signal.SIG_IGN)
server = ServerTCP.server
while True:
conn, addr = server.accept()
print('Accepted connection from {} and port {}'.format(
addr[0], addr[1]))
process_id = os.fork()
if process_id == 0:
server.close()
while True:
prctl.set_pdeathsig(
signal.SIGKILL
) #send SIGKILL to child if parent was killed (this is the signal that the calling process will get when its parent die)
data = conn.recv(buffer_size)
if len(data) > 0: #if client pass sth
if data[:2].decode(
"utf-8"
) == 'cd': #cd doesen't work in subprocess.Popen
try: #try-except prevents executing incorrect command
os.chdir(data[3:].decode("utf-8"))
cmd = subprocess.Popen(data[:],
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE)
output_bytes = cmd.stdout.read()
output_str = str(output_bytes, "utf-8")
conn.send(
str.encode(output_str + str(os.getcwd()) +
'$'))
except:
conn.send(
str.encode('Wrong folder!\n' +
str(os.getcwd()) + '$'))
else: #executing command and sending results
cmd = subprocess.Popen(data[:],
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE)
output_bytes = cmd.stdout.read()
output_str = str(output_bytes, "utf-8")
conn.send(
str.encode(output_str + str(os.getcwd()) +
'$'))
if data[:4].decode(
"utf-8"
) == 'quit': #if client send "quit" conection will end
child_pid = os.getpid()
print('Child procees with PID {} has been terminated'.
format(child_pid))
os.kill(child_pid, signal.SIGKILL)
break
if process_id > 0:
conn.close()
conn.close()
def pre_exec():
# new process group - all our child will be in that group
prctl.set_pdeathsig(signal.SIGTERM)
os.setpgid(0, 0)
def SigQUIT(SIG, FRM):
reactor.stop()
args = sys.argv[1:]
if len(sys.argv[1:]) != 2:
exit(1)
access_list = []
blocked_ua_list = []
tor_exits_list = []
ports = []
rpc_factory = pb.PBClientFactory()
reactor.connectUNIX(os.path.join("/var/run/tor2web/rpc.socket"), rpc_factory)
signal.signal(signal.SIGUSR1, SigQUIT)
signal.signal(signal.SIGTERM, SigQUIT)
signal.signal(signal.SIGINT, SigQUIT)
prctl.set_pdeathsig(signal.SIGINT)
prctl.set_proctitle("tor2web-worker")
start()
reactor.run()
exit(0)
def pre_exec():
# new process group - all our child will be in that group
prctl.set_pdeathsig(signal.SIGTERM)
os.setpgid(0, 0)
def preexec_function():
dem = demote.demote(conf.user)
dem()
prctl.set_pdeathsig(SIGKILL)
def test_pdeathsig(self):
"""Test manipulation of the pdeathsig setting"""
self.assertRaises(ValueError, prctl.set_pdeathsig, 999)
self.assertEqual(prctl.get_pdeathsig(), 0)
prctl.set_pdeathsig(signal.SIGINT)
self.assertEqual(prctl.get_pdeathsig(), signal.SIGINT)
def test_pdeathsig(self):
"""Test manipulation of the pdeathsig setting"""
self.assertRaises(ValueError, prctl.set_pdeathsig, 999)
self.assertEqual(prctl.get_pdeathsig(), 0)
prctl.set_pdeathsig(signal.SIGINT)
self.assertEqual(prctl.get_pdeathsig(), signal.SIGINT)
def run_qemu(args):
log("Starting QEMU")
qemu_default_args = ['-no-reboot']
# Setup KVM and required guest hardware features
qemu_default_args += ['-enable-kvm']
qemu_default_args += ['-cpu', 'host']
# 'host,migratable=no,+invtsc,+tsc,+x2apic,+fsgsbase']
# Use serial communication
# '-nographic',
# qemu_default_args += ['-display', 'none', '-serial', 'stdio']
qemu_default_args += [
'-nographic', '-chardev',
'stdio,id=char0,mux=on,logfile=sv6_serial.log,signal=off',
'-serial', 'chardev:char0', '-mon', 'chardev=char0'
]
# Enable networking with outside world
qemu_default_args += [
'-net',
'user',
'-net',
'nic,model=e1000',
# '-redir', 'tcp:2323::23',
# '-redir', 'tcp:8080::80'
]
qemu_default_args += [
'-device', 'ahci,id=ahci0', '-drive',
'if=none,file=./o.qemu/fs.img,format=raw,id=drive-sata0-0-0',
'-device', 'ide-hd,bus=ahci0.0,drive=drive-sata0-0-0,id=sata0-0-0'
]
qemu_default_args += ['-kernel', './o.qemu/kernel.elf']
def query_host_numa():
online = cat["/sys/devices/system/node/online"]()
if "-" in online:
nlow, nmax = online.split('-')
assert int(nlow) == 0
return int(nmax)
else:
return int(online.strip())
host_numa_nodes = query_host_numa()
if args.qemu_nodes and args.qemu_nodes > 0 and args.qemu_cores > 1:
for node in range(0, args.qemu_nodes):
mem_per_node = int(args.qemu_memory) / args.qemu_nodes
qemu_default_args += [
'-object',
'memory-backend-ram,id=nmem{},merge=off,dump=on,prealloc=off,size={}M,host-nodes={},policy=bind'
.format(
node, int(mem_per_node),
0 if host_numa_nodes == 0 else node % host_numa_nodes)
]
qemu_default_args += [
'-numa', "node,memdev=nmem{},nodeid={}".format(node, node)
]
qemu_default_args += [
"-numa", "cpu,node-id={},socket-id={}".format(node, node)
]
if args.qemu_cores and args.qemu_cores > 1 and args.qemu_nodes:
qemu_default_args += [
"-smp",
"{},sockets={},maxcpus={}".format(args.qemu_cores,
args.qemu_nodes,
args.qemu_cores)
]
else:
qemu_default_args += [
"-smp", "{},sockets=1".format(args.qemu_cores)
]
if args.qemu_memory:
qemu_default_args += ['-m', str(args.qemu_memory)]
if args.qemu_debug_cpu:
qemu_default_args += ['-d', 'int,cpu_reset']
if args.qemu_monitor:
qemu_default_args += [
'-monitor', 'telnet:127.0.0.1:55555,server,nowait'
]
# Name threads on host for `qemu_affinity.py` to find it
qemu_default_args += ['-name', 'sv6,debug-threads=on']
qemu_args = ['qemu-system-x86_64'] + qemu_default_args.copy()
if args.qemu_settings:
qemu_args += args.qemu_settings.split()
# Create a tap interface to communicate with guest and give it an IP
# user = (whoami)().strip()
# group = (local['id']['-gn'])().strip()
# TODO: Could probably avoid 'sudo' here by doing
# sudo setcap cap_net_admin .../run.py
# in the setup.sh script
# sudo[tunctl[['-t', QEMU_TAP_NAME, '-u', user, '-g', group]]]()
# sudo[ifconfig[QEMU_TAP_NAME, QEMU_TAP_ZONE]]()
# Run a QEMU instance
cmd = qemu_args
if args.verbose:
print(' '.join(cmd))
# Spawn qemu first, then set the guest CPU affinities
# The `preexec_fn` ensures that qemu dies if run.py exits
execution = subprocess.Popen(
cmd,
stderr=None,
stdout=None,
env=os.environ.copy(),
preexec_fn=lambda: prctl.set_pdeathsig(signal.SIGKILL))
from plumbum.machines import LocalCommand
LocalCommand.QUOTE_LEVEL = 3
if args.qemu_cores and args.qemu_affinity:
affinity_list = str(corealloc['-c',
str(args.qemu_cores), '-t',
'interleave']()).strip()
# For big machines it can take a while to spawn all threads in qemu
# if but if the threads are not spawned qemu_affinity.py fails, so we sleep
sleep(10.00)
if args.verbose:
log("QEMU affinity {}".format(affinity_list))
sudo[python3['./qemu_affinity.py', '-k',
affinity_list.split(' '), '--',
str(execution.pid)]]()
# Wait until qemu exits
execution.wait()
sv6_exit_code = execution.returncode >> 1
if SV6_EXIT_CODES.get(sv6_exit_code):
print(SV6_EXIT_CODES[sv6_exit_code])
else:
print(
"[FAIL] Kernel exited with unknown error status {}... Update the script!"
.format(sv6_exit_code))
if sv6_exit_code != 0:
log("Invocation was: {}".format(cmd))
if execution.stderr:
print("STDERR: {}".format(execution.stderr.decode('utf-8')))
return sv6_exit_code
"-an",
"-c:v",
"copy",
"-flags",
"global_header",
"-bsf",
"dump_extra",
f"rtp://localhost:{janus_rtp_start_port + camera['sequence']}",
]
# start the process and make stdout unblocking
p = subprocess.Popen(
cmd_args,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
preexec_fn=lambda: prctl.set_pdeathsig(signal.SIGKILL),
)
fd = p.stdout.fileno()
fl = fcntl.fcntl(fd, fcntl.F_GETFL)
fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)
g_shutdown = False
def signal_handler(sig, frame):
if sig == signal.SIGINT or sig == signal.SIGTERM:
p.stdin.writelines(["q".encode()])
global g_shutdown
g_shutdown = True
def preexec_fn():
prctl.set_pdeathsig(signal.SIGKILL)
def die_with_parent():
prctl.set_pdeathsig(signal.SIGKILL)
def dataflowProcess(p: Process):
"""Wrapper for running user requested function.
This wraper auto-initializes all inbound and outbound pipes
for this process so the underlying function doesn't need to
worry about any setup. All pipes will be good to go.
All arguments are optional because this can form either:
- an inbound consumer
- connects to external service then forwards to outbound
using 'forward' function
- a processing layer
- reads from 'inbound', processes using 'forward' function,
and writes to 'outbound'
- a reply layer
- reads from 'inbound' and writes to server connections
"""
setproctitle.setproctitle(f"{sys.argv[0]} [{p.mode}:{p.id}]")
if isLinux:
import prctl # type: ignore
import signal
# tell the kernel to kill this process when the parent exits
prctl.set_pdeathsig(signal.SIGKILL)
async def runProcessing():
# Setup all AsyncPIPEs passed as kwargs
# (either 'inbound' or 'outbound' (or both at once)
# For the first layer of consumers and the last layer of repliers,
# 'inbound' and 'outbound' will be just lists of AsyncPIPE.
# For the middle layer of processors, 'inbound' and 'outbound' are
# lists of tuples.
# logger.info(f"Inbound: {p.inbound}")
# logger.info(f"Outbound: {p.outbound}")
# Double convert lists here because sometimes inputs are tuple vs. list
for pipe in list(p.inbound) + list(p.outbound):
pipe.pipe.setup()
assert not (
p.server and (p.inbound and p.outbound)
), "Server layers can't have inbound and outbound pipes at the same time!"
# If this is a server level, we run the server directly and the
# server spawns new async tasks for each client based on a
# new client callback.
if p.server:
# Run a network connection (client or server) with (optional) processing callback
runServer = p.server(p, p.forwardFromClient)
asyncio.create_task(runServer())
# now configure if this is a processing level or a forwarding level.
# (if this is the first consume level, 'inbound' will not be populated,
# so the rest of this function is a noop)
# run on-start setup if needed / if exists
if p.startup:
if asyncio.iscoroutinefunction(p.startup):
await p.startup(p)
else:
p.startup(p)
if p.inbound:
assert p.forwardFromInternal, "No forward for level?"
fl = ForeverLoop()
# add all inbound pipes to our processing loop
# if reader function has the data argument as type binary, return
# the data instead of converting it first!
aspec = inspect.getfullargspec(p.forwardFromInternal)
annos = aspec.annotations
secondArgName = aspec.args[1]
typeOfSecond = annos.get(secondArgName)
if typeOfSecond == bytes:
# requested binary input so return the data bytes directly
for i in p.inbound:
fl.addRunner(i.pipe.readActual)
else:
# else, decode python to objects!
for i in p.inbound:
fl.addRunner(i.pipe.readPickle)
# now dispatch input from inbound pipes to callback, forever.
# (we factor out into two functions so we aren't having to
# evaluate the "iscoroutinefunction()" condition on each
# new inbound received data)
if asyncio.iscoroutinefunction(p.forwardFromInternal):
async for got in fl.runForever():
# Run with internal server callback (async)
got = await got
try:
await p.forwardFromInternal(p, got)
except Exception as e:
logger.exception(
f"Error while iterating async worker? {p}, {got}"
)
else:
async for got in fl.runForever():
# Run with internal server callback (sync)
got = await got
try:
p.forwardFromInternal(p, got)
except Exception as e:
logger.exception(f"Error while iterating worker? {p}, {got}")
async def startProcessing():
# This is just a simple way of catching all exceptions in the forked
# process without wrapping the entire processing function in an
# extra level of 'try/except' indentation.
try:
with logger.contextualize(task_id=p.id):
await runProcessing()
except KeyboardInterrupt:
# ignore control-c
pass
except asyncio.CancelledError:
# also control-c
pass
except:
logger.exception("huh what?")
safeLoop(startProcessing())
# we do not want all workers to die in the same moment
requests_countdown = config['requests_per_process'] / random.randint(3, 5)
sys.excepthook = MailException
def SigQUIT(SIG, FRM):
reactor.stop()
args = sys.argv[1:]
if len(sys.argv[1:]) != 2:
exit(1)
ports = []
requests_countdown = 10000 / random.randint(3, 5)
rpc_factory = pb.PBClientFactory()
reactor.connectTCP("127.0.0.1", 8789, rpc_factory)
signal.signal(signal.SIGUSR1, SigQUIT)
signal.signal(signal.SIGTERM, SigQUIT)
signal.signal(signal.SIGINT, SigQUIT)
prctl.set_pdeathsig(signal.SIGINT)
prctl.set_proctitle("tor2web-worker")
start()
reactor.run()
exit(0)
# Background color of selected even tabs.
# Type: QtColor
c.colors.tabs.selected.even.bg = blue
# Foreground color of selected even tabs.
# Type: QtColor
c.colors.tabs.selected.even.fg = foreground
# Background color of selected odd tabs.
# Type: QtColor
c.colors.tabs.selected.odd.bg = blue
# Foreground color of selected odd tabs.
# Type: QtColor
c.colors.tabs.selected.odd.fg = foreground
# Background color for webpages if unset (or empty to use the theme's
# color)
# Type: QtColor
c.colors.webpage.bg = background
if qutewal_dynamic_loading or bool(os.getenv('QUTEWAL_DYNAMIC_LOADING')):
import signal
import subprocess
import prctl
# start iqutefy to refresh colors on the fly
qutewald = subprocess.Popen(
[daemon_absolute, colors_absolute],
preexec_fn=lambda: prctl.set_pdeathsig(signal.SIGTERM))
def run(self):
# Start IPC to receive remote commands
self.ipc = ws_ipc.IpcServer(self.ipc_port)
self.ipc.start()
# Get gpu info
for device, ds in self.device_settings.items():
ds.uuid = nvidia_smi.device(device)["uuid"]
# Homie setup
if self.mqtt_host:
self.homie_setup()
# Start excavator
if self.run_excavator:
self.excavator_proc = subprocess.Popen(['./temperature_guard.py', '80', 'excavator'], preexec_fn=lambda: prctl.set_pdeathsig(signal.SIGKILL))
logging.info('connecting to excavator')
while not self.excavator.is_alive():
sleep(5)
#self.excavator.subscribe(self.region, self.wallet, self.name)
self.device_monitor.start()
last_nh_update = 0.0
last_speed_update = 0.0
self.state = Driver.State.RUNNING
while True:
now = time.time()
# Read device events
try:
event = self.device_monitor.get_event(block=False)
#event = self.device_monitor.get_event(block=True, timeout=0.1)
#logging.debug("Device event: "+str(event))
if(event["type"] == "xidEvent"):
if(event["value"] == 43):
logging.error("Gpu %i: crashed! Waiting for signal 45 (xid: %i)" % (event["id"], event["value"]))
self.state = Driver.State.CRASHING
self.cleanup()
elif(event["value"] == 45):
logging.info("Gpu %i: recovered after crash (xid: %i)" % (event["id"], event["value"]))
self.state = Driver.State.RESTARTING
else:
logging.error("Gpu %i: unhandled error (xid: %i)" % (event["id"], event["value"]))
elif event["type"] == "temp":
self.device_settings[event["id"]].temperature = event["value"]
except self.device_monitor.Empty:
pass
# Read IPC events
try:
event = self.ipc.get_event(block=False)
response = None
logging.debug("IPC event: "+str(event))
d = event.data
if d["cmd"] == "device.enable":
self.device_settings[d["device_id"]].enabled = d["enable"]
elif d["cmd"] == "publish.state":
for device, ds in self.device_settings.items():
self.ipc.publish({
"type": "device.algo",
"device_id": device,
"device_uuid": ds.uuid,
"algo": ds.current_algo,
"speed": ds.current_speed,
"paying": ds.paying
})
event.respond(response)
except ws_ipc.IpcServer.Empty:
pass
except Exception as e:
logging.error("Error from IPC Server: "+str(e))
import traceback
print(traceback.format_exc(e))
sleep(0.1)
# Update device speeds
if self.state == Driver.State.RUNNING and now > last_speed_update + SPEED_INTERVAL:
last_speed_update = now
for device, ds in self.device_settings.items():
speeds = self.excavator.device_speeds(device)
ds.current_speed = speeds[ds.current_algo] if ds.current_algo in speeds else 0.0
if ds.oc_session:
ds.oc_session.loop(ds.current_speed)
ds.paying = self.nicehash_mbtc_algo_per_day(ds.current_algo, ds.current_speed)
self.ipc.publish({
"type": "device.algo",
"device_id": device,
"device_uuid": ds.uuid,
"algo": ds.current_algo,
"speed": ds.current_speed,
"paying": ds.paying
})
self.publish_devices()
# Algorithm switching
if self.state == Driver.State.RUNNING and now > last_nh_update + UPDATE_INTERVAL:
last_nh_update = now
if not self.excavator.is_alive():
logging.error("Excavator is not alive, exiting")
self.cleanup()
return
try:
self.paying_current = nicehash_api.multialgo_info()
except urllib.error.HTTPError as err:
logging.warning('server error retrieving NiceHash stats: %s %s' % (err.code, err.reason))
except urllib.error.URLError as err:
logging.warning('failed to retrieve NiceHash stats: %s' % err.reason)
except socket.timeout:
logging.warning('failed to retrieve NiceHash stats: timed out')
except (json.decoder.JSONDecodeError, KeyError):
logging.warning('failed to parse NiceHash stats')
else:
for device in self.devices:
payrates = self.nicehash_mbtc_per_day(device, self.paying_current)
best_algo = max(payrates.keys(), key=lambda algo: payrates[algo])
current_algo = self.device_settings[device].current_algo
best_pay = payrates[best_algo]
current_pay = payrates[current_algo] if current_algo else 0.0
if best_pay > current_pay * (1.0 + self.switching_threshold):
self.device_settings[device].best_algo = best_algo
# Apply device settings
for device in self.devices:
ds = self.device_settings[device]
if ds.enabled:
if ds.best_algo:
if ds.current_algo != ds.best_algo:
payrates = self.nicehash_mbtc_per_day(device, self.paying_current)
logging.info('Switching device %s to %s (%.2f mBTC/day)' % (device, ds.best_algo, payrates[ds.best_algo]))
ds.current_speed = 0.0
ds.current_pay = 0.0
self.ipc.publish({
"type": "device.algo",
"device_id": device,
"device_uuid": ds.uuid,
"algo": best_algo,
"speed": ds.current_speed,
"paying": ds.paying
})
self.publish_device(device)
if ds.running:
self.free_device(device)
self.dispatch_device(device, best_algo)
else:
if ds.running:
logging.info("Disabling device %i" % (device))
ds.current_speed = 0.0
ds.current_pay = 0.0
self.ipc.publish({
"type": "device.algo",
"device_id": device,
"device_uuid": ds.uuid,
"algo": None,
"speed": ds.current_speed,
"paying": ds.paying
})
self.publish_device(device)
self.free_device(device)
#sleep(UPDATE_INTERVAL)
sleep(0.1)