def do_send_with_send_error(self, raise_on_send, send_exception,
expect_answer=True,
expect_send_exception=None):
"""
Sets up two connected sockets, wraps the sender socket into a BadSocket
class, then performs a do_send() test.
Parameters:
raise_on_send: the byte at which send_exception should be raised
(see BadSocket)
send_exception: the exception to raise (see BadSocket)
expect_answer: whether the send is expected to complete (and hence
the read socket should get the message)
expect_send_exception: the exception msgq is expected to raise when
send_exception is raised by BadSocket.
"""
(write, read) = socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM)
(control_write, control_read) = socket.socketpair(socket.AF_UNIX,
socket.SOCK_STREAM)
badwrite = BadSocket(write, raise_on_send, send_exception)
self.do_send(badwrite, read, control_write, control_read,
expect_answer, expect_send_exception)
write.close()
read.close()
control_write.close()
control_read.close()
def test_pause_pipe_writer(self):
"""Test pausing and resuming writing to a socketpair"""
data = 4*1024*1024*'*'
rsock1, wsock1 = socket.socketpair()
rsock2, wsock2 = socket.socketpair()
reader1, writer1 = yield from asyncio.open_unix_connection(sock=rsock1)
reader2, writer2 = yield from asyncio.open_unix_connection(sock=rsock2)
with (yield from self.connect()) as conn:
process = yield from conn.create_process(input=data)
yield from asyncio.sleep(1)
yield from process.redirect_stdout(wsock1)
yield from process.redirect_stderr(wsock2)
stdout_data, stderr_data = \
yield from asyncio.gather(reader1.read(), reader2.read())
writer1.close()
writer2.close()
yield from process.wait()
self.assertEqual(stdout_data.decode(), data)
self.assertEqual(stderr_data.decode(), data)
def testWritePrio(self):
reactor = Reactor()
local0, remote0 = socketpair()
local1, remote1 = socketpair()
local1.setblocking(0)
def remoteHandler0():
remote0.send('ape')
def remoteHandler1():
remote1.send('nut')
reactor.addWriter(remote0, remoteHandler0, 0)
reactor.addWriter(remote1, remoteHandler1, 2)
reactor.step() #0
self.assertEquals('ape', local0.recv(999))
try:
local1.recv(999)
self.fail('must be no data on the socket yet')
except error:
pass
reactor.step() #1
try:
local1.recv(999)
self.fail('must be no data on the socket yet')
except error:
pass
reactor.step() #2
self.assertEquals('nut', local1.recv(999))
def __init__ (self, ircprox, localid, remotid): """Start a new PrivateChannel. The server can be reached over ircprox.send_to_server(), which will ensure proper locking. The communication is between the given localid and remotid (both in nick form, see nick2nai(), and anything passing over the PrivateChannel will be sent as a PRIVMSG targeted at remotid, followed by TLS and one line of base64-encoded TLS binary data. Large TLS data portions may be split over multiple lines, but each is separately base64-encoded, meaning, each line has its own "=" sign trailer. """ assert (remotid [:1] != '#') self.ircprox = ircprox self.localid = localid self.remotid = remotid self.localid_nai = nick2nai (localid) self.remotid_nai = nick2nai (remotid) self.localhs = None self.remoths = None self.plaindownbuf = '' self.poolcrypt, self.chancrypt = socket.socketpair () self.poolplain, self.chanplain = socket.socketpair () print 'PrivateChannel crypt @pool =', self.poolcrypt, '@chan =', self.chancrypt print 'PrivateChannel plain @pool =', self.poolplain, '@chan =', self.chanplain self.insecure = 0 #TODO# Following should not return before done # self.start () self.initiate_starttls_handshake ()
def __init__(self, pcapfile, connection, ssfile): self.ssfile = ssfile self.pcapfile = pcapfile self.connection = connection self.inbound=dict() self.outbound=dict() self.in_rsocket,self.in_wsocket = socket.socketpair() self.out_rsocket,self.out_wsocket = socket.socketpair()
def __init__(self):
self._coros = []
self.selector = selectors.DefaultSelector()
self._executor = ThreadPoolExecutor()
self._rsock, self._wsock = socket.socketpair()
self._rsock.setblocking(False)
self.selector.register(self._rsock, selectors.EVENT_READ)
def makeSockets(cls):
"""Make socket(s) to communicate with remote process.
Returns string to send to remote process
"""
if ( hasattr(socket, 'AF_UNIX') and hasattr(socket, 'socketpair') ):
# convenient interface
cls.sockfamily = socket.AF_UNIX
sock, socket2 = socket.socketpair(cls.sockfamily,
socket.SOCK_STREAM)
# socket is closed on popen in Python 3.4+ without this (PEP 446)
try:
os.set_inheritable(socket2.fileno(), True)
except AttributeError:
pass
sendtext = 'unix %i\n' % socket2.fileno()
cls.socket2 = socket2 # prevent socket being destroyed
waitaccept = False
else:
# otherwise mess around with internet sockets
# * This is required for windows, which doesn't have AF_UNIX
# * It is required where socketpair is not supported
cls.sockfamily = socket.AF_INET
sock = socket.socket(cls.sockfamily, socket.SOCK_STREAM)
sock.bind( ('localhost', 0) )
interface, port = sock.getsockname()
sock.listen(1)
sendtext = 'internet %s %i\n' % (interface, port)
waitaccept = True
return (sock, sendtext.encode('ascii'), waitaccept)
def live_profile(script, timer, interval, pickle_protocol, mono):
"""Profile a Python script continuously."""
filename, code, globals_ = script
sys.argv[:] = [filename]
parent_sock, child_sock = socket.socketpair()
pid = os.fork()
if pid == 0:
# child
devnull = os.open(os.devnull, os.O_RDWR)
for f in [sys.stdin, sys.stdout, sys.stderr]:
os.dup2(devnull, f.fileno())
frame = sys._getframe()
profiler = BackgroundProfiler(timer, frame, code)
profiler.prepare()
server_args = (noop, interval, pickle_protocol)
server = SelectProfilingServer(None, profiler, *server_args)
server.clients.add(child_sock)
spawn_thread(server.connected, child_sock)
try:
exec_(code, globals_)
finally:
child_sock.close()
else:
# parent
viewer, loop = make_viewer(mono)
title = get_title(filename)
client = ProfilingClient(viewer, loop.event_loop, parent_sock, title)
client.start()
try:
loop.run()
except KeyboardInterrupt:
pass
finally:
parent_sock.close()
os.kill(pid, signal.SIGINT)
def create_sock_pair(port=0):
'''Create socket pair. Works also on windows by using an ephemeral TCP port.'''
if hasattr(socket, 'socketpair'):
client_sock, srv_sock = socket.socketpair()
return client_sock, srv_sock
# Create a non-blocking temporary server socket
temp_srv_sock = socket.socket()
temp_srv_sock.setblocking(False)
temp_srv_sock.bind(('localhost', port))
port = temp_srv_sock.getsockname()[1]
temp_srv_sock.listen(1)
with closing(temp_srv_sock):
# Create non-blocking client socket
client_sock = socket.socket()
client_sock.setblocking(False)
while True:
try:
client_sock.connect(('localhost', port))
except socket.error as err:
# EWOULDBLOCK is not an error, as the socket is non-blocking
if err.errno not in socket_errors_nonblocking:
raise
# Use select to wait for connect() to succeed.
timeout = 1
readable = select.select([temp_srv_sock], [], [], timeout)[0]
if temp_srv_sock not in readable:
raise Exception('Client socket not connected in {} second(s)'.format(timeout))
srv_sock = temp_srv_sock.accept()[0]
client_sock.setblocking(True)
return client_sock, srv_sock
def _start_worker(self):
env = dict(os.environ)
env["ABUSEHELPER_SUBPROCESS"] = ""
# Find out the full package & module name. Don't refer to the
# variable __loader__ directly to keep flake8 (version 2.5.0)
# linter happy.
fullname = globals()["__loader__"].fullname
own_conn, other_conn = native_socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM)
try:
process = subprocess.Popen(
[sys.executable, "-m", fullname],
preexec_fn=os.setpgrp,
stdin=other_conn.fileno(),
close_fds=True,
env=env,
)
try:
conn = socket.fromfd(own_conn.fileno(), socket.AF_UNIX, socket.SOCK_STREAM)
except:
process.terminate()
process.wait()
raise
finally:
own_conn.close()
other_conn.close()
return process, conn
def rpc_run(self, pcode, user, visitor):
uid = 61018
# localtime = time.asctime(time.localtime(time.time()))
# userdir = '/tmp/' + (localtime[8]+localtime[17]+localtime[18]+localtime[14]+localtime[15])
userdir = "/tmp" + "/" + user.replace("/", "9")
if not os.path.exists(userdir):
os.mkdir(userdir)
os.chmod(userdir, 0770)
# print "Directory created " + userdir
db = zoodb.cred_setup()
person = db.query(zoodb.Cred).get(user)
if not person:
return None
token = person.token
(sa, sb) = socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM, 0)
pid = os.fork()
if pid == 0:
if os.fork() <= 0:
sa.close()
ProfileAPIServer(user, visitor, token).run_sock(sb)
sys.exit(0)
else:
sys.exit(0)
sb.close()
os.waitpid(pid, 0)
sandbox = sandboxlib.Sandbox(userdir, uid, "/profilesvc/lockfile")
with rpclib.RpcClient(sa) as profile_api_client:
return sandbox.run(lambda: run_profile(pcode, profile_api_client))
def test_handle_error_failures(self):
class EvilDispatcher(Echo):
def handle_read(self):
Echo.handle_read(self)
raise Exception("Expected error")
def handle_error(self):
# This is a very big anti-pattern for dispatchers,
# asyncore.poll2 will raise errors raised from handle_error.
raise Exception("Evil error")
def pinger(sock):
msg = b"ping"
osutils.uninterruptible(sock.send, msg)
osutils.uninterruptible(sock.recv, len(msg))
sock.close()
self.loop.call_soon_threadsafe(self.loop.stop)
sock1, sock2 = socket.socketpair()
# The dispatcher and pinger owns the sockets
self.loop.create_dispatcher(EvilDispatcher, sock2)
t = concurrent.thread(pinger, args=(sock1,))
t.start()
try:
# Correct error handling willl allow this test to complete without
# errors. This used to abort the event loop with the error raised
# in handle_error.
self.loop.run_forever()
finally:
t.join()
def test_echo(self, concurrency):
msg = b"ping"
sockets = []
try:
for i in range(concurrency):
sock1, sock2 = socket.socketpair()
self.loop.create_dispatcher(Echo, sock2)
sockets.append(sock1)
t = concurrent.thread(self.loop.run_forever)
t.start()
try:
start = time.time()
for sock in sockets:
osutils.uninterruptible(sock.send, msg)
for sock in sockets:
data = osutils.uninterruptible(sock.recv, len(msg))
self.assertEqual(data, msg)
elapsed = time.time() - start
print("%7d echos: %f seconds" % (concurrency, elapsed))
finally:
self.loop.call_soon_threadsafe(self.loop.stop)
t.join()
finally:
for sock in sockets:
sock.close()
def _test_recv(self, recv_func):
rd, wr = socket.socketpair()
self.addCleanup(rd.close)
# wr closed explicitly by parent
# single-byte payload guard us against partial recv
datas = [b"x", b"y", b"z"]
code = '\n'.join((
'import os, socket, sys, time',
'',
'fd = int(sys.argv[1])',
'family = %s' % int(wr.family),
'sock_type = %s' % int(wr.type),
'datas = %r' % datas,
'sleep_time = %r' % self.sleep_time,
'',
'wr = socket.fromfd(fd, family, sock_type)',
'os.close(fd)',
'',
'with wr:',
' for data in datas:',
' # let the parent block on recv()',
' time.sleep(sleep_time)',
' wr.sendall(data)',
))
fd = wr.fileno()
proc = self.subprocess(code, str(fd), pass_fds=[fd])
with kill_on_error(proc):
wr.close()
for data in datas:
self.assertEqual(data, recv_func(rd, len(data)))
self.assertEqual(proc.wait(), 0)
def __init__(self, fd):
(self._rsock, wsock) = socket.socketpair(socket.AF_UNIX, socket.SOCK_DGRAM)
child = os.fork()
if child != 0:
wsock.close()
self._child = child
return
self._rsock.close()
# This is the read loop in the forked process and it won't quit until either the process
# gets killed or there is a read error.
try:
while True:
packet = handleEAgain(os.read, fd, MAX_PACKET_SIZE)
handleEAgain(wsock.send, pickle.dumps((0, packet)))
if len(packet) == 0:
break
except KeyboardInterrupt:
pass
except EnvironmentError as e:
handleEAgain(wsock.send, pickle.dumps((e.errno, '')))
finally:
os.close(fd)
wsock.close()
os._exit(os.EX_OK)
def _create_socketpair():
"""
Create a socketpair that will be used for signaling to select() during
the initialisation procedure (and binding on MS Windows).
"""
if hasattr(socket, "socketpair"):
return socket.socketpair()
# Backward compatibility on Windows from Python 3.5.
# Origin: https://gist.github.com/4325783, by Geert Jansen. Public domain.
def socketpair(family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0):
import errno
# We create a connected TCP socket. Note the trick with setblocking(0)
# that prevents us from having to create a thread.
lsock = socket.socket(family, type, proto)
lsock.bind(('localhost', 0))
lsock.listen(1)
addr, port = lsock.getsockname()
csock = socket.socket(family, type, proto)
csock.setblocking(0)
try:
csock.connect((addr, port))
except socket.error as serr:
if serr.errno != errno.WSAEWOULDBLOCK:
raise
ssock, addr = lsock.accept()
csock.setblocking(1)
lsock.close()
return (ssock, csock)
return socketpair()
def test_pack_ping(self):
config = kademlia.utils.load_config("config.json")
loop = asyncio.get_event_loop()
loop.set_debug(config['debug']['asyncio']['enabled'])
service = kademlia.Service(config, loop)
echo = kademlia.utils.get_echo_bytes()
rsock, wsock = socketpair()
reader, writer = loop.run_until_complete(
asyncio.open_connection(sock=rsock, loop=loop)
)
wsock.send(
service.tcpService.rpc.pack_ping(service.tcpService.node, echo)
)
_command, _echo, _remoteNode, _data = loop.run_until_complete(
asyncio.ensure_future(
service.tcpService.rpc.read_command(reader)
)
)
writer.close()
wsock.close()
self.assertEqual(_command, kademlia.const.kad.command.PING)
self.assertEqual(echo, _echo)
def _init_loopback(self):
# Initialize the loopback socket and launch the kernel task if needed
self._notify_sock, self._wait_sock = socket.socketpair()
self._wait_sock.setblocking(False)
self._notify_sock.setblocking(False)
task = self._new_task(self._kernel_task(), daemon=True)
self._kernel_task_id = task.id
def _pair(self):
a, b = socket.socketpair()
a.setblocking(False)
b.setblocking(False)
c, d = ScheduledFile.fromSocket(a), ScheduledFile.fromSocket(b)
c.autoflush = d.autoflush = True
return c, d
def pair(cls, name):
parent, child = socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM)
# We don't set non-blocking (right now), it shouldn't be needed because
# communication tends to be modal for a given pipe, and we wait until
# reads would not block to begin reading.
return SocketChannel(name + 'Parent', parent), SocketChannel(name + 'Child', child)
def test_select1(self):
import select
import socket
s1, s2 = socket.socketpair()
with hub.Timeout(1, MyException):
select.select([s2.fileno()], [], [])
def test_remove_handler_from_handler(self):
# Create two sockets with simultaneous read events.
client, server = socket.socketpair()
try:
client.send(b'abc')
server.send(b'abc')
# After reading from one fd, remove the other from the IOLoop.
chunks = []
def handle_read(fd, events):
chunks.append(fd.recv(1024))
if fd is client:
self.io_loop.remove_handler(server)
else:
self.io_loop.remove_handler(client)
self.io_loop.add_handler(client, handle_read, self.io_loop.READ)
self.io_loop.add_handler(server, handle_read, self.io_loop.READ)
self.io_loop.call_later(0.1, self.stop)
self.wait()
# Only one fd was read; the other was cleanly removed.
self.assertEqual(chunks, [b'abc'])
finally:
client.close()
server.close()
def test_timeout(self):
A, B = socket.socketpair()
# specific user timeout
A.settimeout(0.1)
# some games so that the test won't hang if recv() never times out
def op(sock):
try:
sock.recv(10)
assert False, "Missed expected Timeout"
except socket.error:
return sock
opA = cothread.Spawn(op, A, raise_on_wait=True)
try:
V = opA.Wait(1.0)
except:
opA.AbortWait()
raise
self.assertTrue(V is A)
opB = cothread.Spawn(op, B, raise_on_wait=True)
try:
V = opB.Wait(2.0)
except:
opB.AbortWait()
raise
self.assertTrue(V is B)
A.close()
B.close()
def test_create_unix_connection_5(self):
s1, s2 = socket.socketpair(socket.AF_UNIX)
excs = []
class Proto(asyncio.Protocol):
def connection_lost(self, exc):
excs.append(exc)
proto = Proto()
async def client():
t, _ = await self.loop.create_unix_connection(
lambda: proto,
None,
sock=s2)
t.write(b'AAAAA')
s1.close()
t.write(b'AAAAA')
await asyncio.sleep(0.1, loop=self.loop)
self.loop.run_until_complete(client())
self.assertEqual(len(excs), 1)
self.assertIn(excs[0].__class__,
(BrokenPipeError, ConnectionResetError))
def setUp(self):
(self.c, self.s) = socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM)
self.client = TransPort(self.c)
self.server = threading.Thread(target=_server, args=(self.s,))
self.server.start()
def SocketPair(family=None, type=socket.SOCK_STREAM, proto=0):
""" Creats a connected pair of sockets. Works on win32 by creating
a listening soket on localhost and then connecting to it. The
family arg defaults to AF_INET on win32 and AF_UNIX elsewhere. """
if sys.platform != 'win32':
if family is None:
family = socket.AF_UNIX
return socket.socketpair(family, type, proto)
if family is None:
family = socket.AF_INET
accepted = None
conn = None
listener = socket.socket(family=family, type=type, proto=proto)
try:
listener.bind(('127.0.0.1', 0))
listener.listen(16)
conn = socket.socket(family=family, type=type, proto=proto)
conn.connect(listener.getsockname())
while True:
accepted, addr = listener.accept()
if addr == conn.getsockname():
return accepted, conn
except Exception:
if accepted:
accepted.close()
if conn:
conn.close()
raise
finally:
listener.close()
def __init__(self, bootstrap, timeout=60, debug=True, token=None):
self.debug = debug
self.timeout = timeout
self.domainname = None
self.token = token
self.socket = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
self.socket.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1)
self.socket.setsockopt(socket.IPPROTO_TCP,socket.TCP_NODELAY,1)
self.bootstraplist = self.discoverbootstrap(bootstrap)
if len(self.bootstraplist) == 0:
raise ConnectionError("No bootstrap found")
if not self.connecttobootstrap():
raise ConnectionError("Cannot connect to any bootstrap")
myaddr = findOwnIP()
myport = self.socket.getsockname()[1]
self.me = Peer(myaddr,myport,NODE_CLIENT)
self.commandnumber = random.randint(1, sys.maxint)
# synchronization
self.lock = RLock()
self.ctrlsockets, self.ctrlsocketr = socket.socketpair()
self.reqlist = [] # requests we have received from client threads
self.pendingops = {} # pending requests indexed by command number
# spawn thread, invoke comm_loop
comm_thread = Thread(target=self.comm_loop, name='CommunicationThread')
comm_thread.start()
def test_parse_from_real_socket():
# would fail on python2.6 before the recv_into hack
sock, sink = socket.socketpair()
sink.send(complete_request)
reader = SocketReader(sock)
stream = HttpStream(reader)
assert stream.headers()
def _start_child(nonetns):
# Create socket pair to communicate
(s0, s1) = socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM, 0)
# Spawn a child that will run in a loop
pid = os.fork()
if pid:
s1.close()
return (s0, pid)
# FIXME: clean up signal handers, atexit functions, etc.
try: # pragma: no cover
# coverage doesn't seem to understand fork
s0.close()
srv = nemu.protocol.Server(s1, s1)
if not nonetns:
# create new name space
unshare.unshare(unshare.CLONE_NEWNET)
# Enable packet forwarding
execute([SYSCTL_PATH, '-w', 'net.ipv4.ip_forward=1'])
execute([SYSCTL_PATH, '-w', 'net.ipv6.conf.default.forwarding=1'])
srv.run()
except BaseException, e:
s = "Slave node aborting: %s\n" % str(e)
sep = "=" * 70 + "\n"
sys.stderr.write(s + sep)
traceback.print_exc(file=sys.stdout)
sys.stderr.write(sep)
try:
# try to pass the error to parent, if possible
s1.send("500 " + s)
except:
pass
os._exit(1)
def test_buildProtocolReturnsNone(self):
"""
{IReactorSocket.adoptStreamConnection} returns None if the given
factory's buildProtocol returns None.
"""
# Build reactor before anything else: allow self.buildReactor()
# to skip the test if any of the self.requiredInterfaces isn't
# provided by the reactor (example: Windows), preventing later
# failures unrelated to the test itself.
reactor = self.buildReactor()
from socket import socketpair
class NoneFactory(ServerFactory):
def buildProtocol(self, address):
return None
s1, s2 = socketpair(AF_UNIX, SOCK_STREAM)
s1.setblocking(False)
self.addCleanup(s1.close)
self.addCleanup(s2.close)
s1FD = s1.fileno()
factory = NoneFactory()
result = reactor.adoptStreamConnection(s1FD, AF_UNIX, factory)
self.assertIsNone(result)
def _init_loopback(self):
self._notify_sock, self._wait_sock = socket.socketpair()
self._wait_sock.setblocking(False)
self._notify_sock.setblocking(False)
return self._wait_sock.fileno()
def _make_self_pipe(self):
self._ssock, self._csock = socket.socketpair()
self._ssock.setblocking(False)
self._csock.setblocking(False)
self.add_reader(self._ssock.fileno(), self._read_from_self)
def __init__(self, record_interval):
super().__init__(name="recorder")
self._record_interval = record_interval
self._packets = []
self._packets_lock = threading.Lock()
self._socketpair = socket.socketpair()
def create_fds():
(sk1, sk2) = socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM)
(sk3, sk4) = socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM)
return [(sk1.makefile("wb"), sk2.makefile("rb")),
(sk3.makefile("wb"), sk4.makefile("rb"))]
def live_profile(script, argv, profiler_factory, interval, spawn, signum,
pickle_protocol, mono):
"""Profile a Python script continuously."""
filename, code, globals_ = script
sys.argv[:] = [filename] + list(argv)
parent_sock, child_sock = socket.socketpair()
stderr_r_fd, stderr_w_fd = os.pipe()
pid = os.fork()
if pid:
# parent
os.close(stderr_w_fd)
viewer, loop = make_viewer(mono)
# loop.screen._term_output_file = open(os.devnull, 'w')
title = get_title(filename)
client = ProfilingClient(viewer, loop.event_loop, parent_sock, title)
client.start()
try:
loop.run()
except KeyboardInterrupt:
os.kill(pid, signal.SIGINT)
except:
# unexpected profiler error.
os.kill(pid, signal.SIGTERM)
raise
finally:
parent_sock.close()
# get exit code of child.
w_pid, status = os.waitpid(pid, os.WNOHANG)
if w_pid == 0:
os.kill(pid, signal.SIGTERM)
exit_code = os.WEXITSTATUS(status)
# print stderr of child.
with os.fdopen(stderr_r_fd, 'r') as f:
child_stderr = f.read()
if child_stderr:
sys.stdout.flush()
sys.stderr.write(child_stderr)
# exit with exit code of child.
sys.exit(exit_code)
else:
# child
os.close(stderr_r_fd)
# mute stdin, stdout.
devnull = os.open(os.devnull, os.O_RDWR)
for f in [sys.stdin, sys.stdout]:
os.dup2(devnull, f.fileno())
# redirect stderr to parent.
os.dup2(stderr_w_fd, sys.stderr.fileno())
frame = sys._getframe()
profiler = profiler_factory(top_frame=frame, top_code=code)
profiler_trigger = BackgroundProfiler(profiler, signum)
profiler_trigger.prepare()
server_args = (interval, noop, pickle_protocol)
server = SelectProfilingServer(None, profiler_trigger, *server_args)
server.clients.add(child_sock)
spawn(server.connected, child_sock)
try:
exec_(code, globals_)
finally:
os.close(stderr_w_fd)
child_sock.shutdown(socket.SHUT_WR)
def setUp(self):
self.serv, self.cli = socket.socketpair()
def __init__(self, picard_args, unparsed_args, localedir, autoupdate):
super().__init__(sys.argv)
self.__class__.__instance = self
config._setup(self, picard_args.config_file)
theme.setup(self)
self._cmdline_files = picard_args.FILE
self.autoupdate_enabled = autoupdate
self._no_restore = picard_args.no_restore
self._no_plugins = picard_args.no_plugins
self.set_log_level(config.setting['log_verbosity'])
if picard_args.debug or "PICARD_DEBUG" in os.environ:
self.set_log_level(logging.DEBUG)
# FIXME: Figure out what's wrong with QThreadPool.globalInstance().
# It's a valid reference, but its start() method doesn't work.
self.thread_pool = QtCore.QThreadPool(self)
# Provide a separate thread pool for operations that should not be
# delayed by longer background processing tasks, e.g. because the user
# expects instant feedback instead of waiting for a long list of
# operations to finish.
self.priority_thread_pool = QtCore.QThreadPool(self)
self.priority_thread_pool.setMaxThreadCount(1)
# Use a separate thread pool for file saving, with a thread count of 1,
# to avoid race conditions in File._save_and_rename.
self.save_thread_pool = QtCore.QThreadPool(self)
self.save_thread_pool.setMaxThreadCount(1)
if not IS_WIN:
# Set up signal handling
# It's not possible to call all available functions from signal
# handlers, therefore we need to set up a QSocketNotifier to listen
# on a socket. Sending data through a socket can be done in a
# signal handler, so we use the socket to notify the application of
# the signal.
# This code is adopted from
# https://qt-project.org/doc/qt-4.8/unix-signals.html
# To not make the socket module a requirement for the Windows
# installer, import it here and not globally
import socket
self.signalfd = socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM, 0)
self.signalnotifier = QtCore.QSocketNotifier(self.signalfd[1].fileno(),
QtCore.QSocketNotifier.Read, self)
self.signalnotifier.activated.connect(self.sighandler)
signal.signal(signal.SIGHUP, self.signal)
signal.signal(signal.SIGINT, self.signal)
signal.signal(signal.SIGTERM, self.signal)
if IS_MACOS:
# On macOS it is not common that the global menu shows icons
self.setAttribute(QtCore.Qt.AA_DontShowIconsInMenus)
# Setup logging
log.debug("Starting Picard from %r", os.path.abspath(__file__))
log.debug("Platform: %s %s %s", platform.platform(),
platform.python_implementation(), platform.python_version())
log.debug("Versions: %s", versions.as_string())
log.debug("Configuration file path: %r", config.config.fileName())
log.debug("User directory: %r", os.path.abspath(USER_DIR))
# for compatibility with pre-1.3 plugins
QtCore.QObject.tagger = self
QtCore.QObject.config = config
QtCore.QObject.log = log
check_io_encoding()
# Must be before config upgrade because upgrade dialogs need to be
# translated
setup_gettext(localedir, config.setting["ui_language"], log.debug)
upgrade_config(config.config)
self.webservice = WebService()
self.mb_api = MBAPIHelper(self.webservice)
load_user_collections()
# Initialize fingerprinting
acoustid_api = AcoustIdAPIHelper(self.webservice)
self._acoustid = acoustid.AcoustIDClient(acoustid_api)
self._acoustid.init()
self.acoustidmanager = AcoustIDManager(acoustid_api)
# Load plugins
self.pluginmanager = PluginManager()
if not self._no_plugins:
if IS_FROZEN:
self.pluginmanager.load_plugins_from_directory(os.path.join(os.path.dirname(sys.argv[0]), "plugins"))
else:
mydir = os.path.dirname(os.path.abspath(__file__))
self.pluginmanager.load_plugins_from_directory(os.path.join(mydir, "plugins"))
if not os.path.exists(USER_PLUGIN_DIR):
os.makedirs(USER_PLUGIN_DIR)
self.pluginmanager.load_plugins_from_directory(USER_PLUGIN_DIR)
self.browser_integration = BrowserIntegration()
self._pending_files_count = 0
self.files = {}
self.clusters = ClusterList()
self.albums = {}
self.release_groups = {}
self.mbid_redirects = {}
self.unclustered_files = UnclusteredFiles()
self.nats = None
self.window = MainWindow(disable_player=picard_args.no_player)
self.exit_cleanup = []
self.stopping = False
# Load release version information
if self.autoupdate_enabled:
self.updatecheckmanager = UpdateCheckManager(parent=self.window)
def test_warn_set_wakeup_fd_overwrite():
assert signal.set_wakeup_fd(-1) == -1
async def trio_main(in_host):
return "ok"
a, b = socket.socketpair()
with a, b:
a.setblocking(False)
# Warn if there's already a wakeup fd
signal.set_wakeup_fd(a.fileno())
try:
with pytest.warns(RuntimeWarning,
match="signal handling code.*collided"):
assert trivial_guest_run(trio_main) == "ok"
finally:
assert signal.set_wakeup_fd(-1) == a.fileno()
signal.set_wakeup_fd(a.fileno())
try:
with pytest.warns(RuntimeWarning,
match="signal handling code.*collided"):
assert (trivial_guest_run(
trio_main, host_uses_signal_set_wakeup_fd=False) == "ok")
finally:
assert signal.set_wakeup_fd(-1) == a.fileno()
# Don't warn if there isn't already a wakeup fd
with pytest.warns(None) as record:
assert trivial_guest_run(trio_main) == "ok"
# Apparently this is how you assert 'there were no RuntimeWarnings'
with pytest.raises(AssertionError):
record.pop(RuntimeWarning)
with pytest.warns(None) as record:
assert (trivial_guest_run(
trio_main, host_uses_signal_set_wakeup_fd=True) == "ok")
with pytest.raises(AssertionError):
record.pop(RuntimeWarning)
# If there's already a wakeup fd, but we've been told to trust it,
# then it's left alone and there's no warning
signal.set_wakeup_fd(a.fileno())
try:
async def trio_check_wakeup_fd_unaltered(in_host):
fd = signal.set_wakeup_fd(-1)
assert fd == a.fileno()
signal.set_wakeup_fd(fd)
return "ok"
with pytest.warns(None) as record:
assert (trivial_guest_run(
trio_check_wakeup_fd_unaltered,
host_uses_signal_set_wakeup_fd=True,
) == "ok")
with pytest.raises(AssertionError):
record.pop(RuntimeWarning)
finally:
assert signal.set_wakeup_fd(-1) == a.fileno()
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Parent/child communication through a socket pair.
"""
import socket
import os
parent, child = socket.socketpair()
pid = os.fork()
if pid:
print 'in parent, sending message'
child.close()
parent.sendall('ping')
response = parent.recv(1024)
print 'response from child:', response
parent.close()
else:
print 'in child, waiting for message'
parent.close()
message = child.recv(1024)
print 'message from parent:', message
child.sendall('pong')
child.close()
def __init__(self, udp_port, recorder_thread,_live_storage):
super().__init__(name="receiver")
self._udp_port = udp_port
self._recorder_thread = recorder_thread
self._socketpair = socket.socketpair()
self.live_storage = _live_storage
#a['ZipInfoType'] = _zip.getinfo('x')
a['TarFileType'] = tarfile.open(fileobj=_fileW, mode='w')
# file formats (CH 13)
a['DialectType'] = csv.get_dialect('excel')
a['PackerType'] = xdrlib.Packer()
# optional operating system services (CH 16)
a['LockType'] = threading.Lock()
a['RLockType'] = threading.RLock()
# generic operating system services (CH 15) # also closed/open and r/w/etc...
a['NamedLoggerType'] = _logger = logging.getLogger(
__name__) #FIXME: fail >= 3.2 and <= 2.6
#a['FrozenModuleType'] = __hello__ #FIXME: prints "Hello world..."
# interprocess communication (CH 17)
if PY3:
a['SocketType'] = _socket = socket.socket() #FIXME: fail >= 3.3
a['SocketPairType'] = socket.socketpair()[0] #FIXME: fail >= 3.3
else:
a['SocketType'] = _socket = socket.socket()
a['SocketPairType'] = _socket._sock
# python runtime services (CH 27)
if PY3:
a['GeneratorContextManagerType'] = contextlib.contextmanager(max)([1])
else:
a['GeneratorContextManagerType'] = contextlib.GeneratorContextManager(max)
try: # ipython
__IPYTHON__ is True # is ipython
except NameError:
# built-in constants (CH 4)
a['QuitterType'] = quit
d['ExitType'] = a['QuitterType']
def init_stop_fd(self):
sks = socket.socketpair()
self._stop_fd = sks[0]
return sks[1]
def __init__(self, *args, **kwargs):
super(FdQueue, self).__init__(*args, **kwargs)
self._fd_reader, self._fd_writer = socket.socketpair(socket.AF_UNIX)
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
import asyncio
try:
from socket import socketpair
except ImportError:
print('On windows: from asyncio.windows_utils import socketpair')
from asyncio.windows_utils import socketpair
# Create a pair of connected file descriptors
rsock, wsock = socketpair()
loop = asyncio.get_event_loop()
def reader():
data = rsock.recv(100)
print("Received:", data.decode())
# We are done: unregister the file descriptor
loop.remove_reader(rsock)
# Stop the event loop
loop.stop()
# Register the file descriptor for read event
loop.add_reader(rsock, reader)
# Simulate the reception of data from the network
loop.call_soon(wsock.send, 'abc'.encode())
def _MultiprocessingPipe():
"""multiprocess.Pipe reimplementation that uses MpConnection wrapper"""
s1, s2 = socket.socketpair()
return (_MpConnection(s1), _MpConnection(s2))
import socket
import asyncio
def reader():
buf = r.recv(1)
print('successs')
loop.stop()
r, w = socket.socketpair()
loop = asyncio.get_event_loop()
loop.add_reader(r, reader)
loop.call_soon(w.send, b'x')
loop.run_forever()
def test_read_from_socket(self):
a, b = socket.socketpair()
a.send(b'test')
stream = io.Stream(b)
expect(stream.read(32)).to(equal(b'test'))
def test_write_to_socket(self):
a, b = socket.socketpair()
stream = io.Stream(a)
stream.write(b'test')
expect(b.recv(32)).to(equal(b'test'))
import pickle
from socket import socketpair
from pickle_basics import move0, move1, move2, move3
# Defime sockets for writing and reading
ws, rs = socketpair()
w, r = ws.makefile('wb'), rs.makefile('rb')
# Serialize
pickle.dump(move0, w)
pickle.dump(move1, w)
pickle.dump(move2, w)
pickle.dump(move3, w)
w.flush() # send to another socket
# De-serialize
for _ in range(4):
move = pickle.load(r)
print(f'{move.limb} {move.what}')
async def test_SocketType_non_blocking_paths():
a, b = stdlib_socket.socketpair()
with a, b:
ta = tsocket.from_stdlib_socket(a)
b.setblocking(False)
# cancel before even calling
b.send(b"1")
with _core.open_cancel_scope() as cscope:
cscope.cancel()
with assert_checkpoints():
with pytest.raises(_core.Cancelled):
await ta.recv(10)
# immedate success (also checks that the previous attempt didn't
# actually read anything)
with assert_checkpoints():
await ta.recv(10) == b"1"
# immediate failure
with assert_checkpoints():
with pytest.raises(TypeError):
await ta.recv("haha")
# block then succeed
async def do_successful_blocking_recv():
with assert_checkpoints():
assert await ta.recv(10) == b"2"
async with _core.open_nursery() as nursery:
nursery.start_soon(do_successful_blocking_recv)
await wait_all_tasks_blocked()
b.send(b"2")
# block then cancelled
async def do_cancelled_blocking_recv():
with assert_checkpoints():
with pytest.raises(_core.Cancelled):
await ta.recv(10)
async with _core.open_nursery() as nursery:
nursery.start_soon(do_cancelled_blocking_recv)
await wait_all_tasks_blocked()
nursery.cancel_scope.cancel()
# Okay, here's the trickiest one: we want to exercise the path where
# the task is signaled to wake, goes to recv, but then the recv fails,
# so it has to go back to sleep and try again. Strategy: have two
# tasks waiting on two sockets (to work around the rule against having
# two tasks waiting on the same socket), wake them both up at the same
# time, and whichever one runs first "steals" the data from the
# other:
tb = tsocket.from_stdlib_socket(b)
async def t1():
with assert_checkpoints():
assert await ta.recv(1) == b"a"
with assert_checkpoints():
assert await tb.recv(1) == b"b"
async def t2():
with assert_checkpoints():
assert await tb.recv(1) == b"b"
with assert_checkpoints():
assert await ta.recv(1) == b"a"
async with _core.open_nursery() as nursery:
nursery.start_soon(t1)
nursery.start_soon(t2)
await wait_all_tasks_blocked()
a.send(b"b")
b.send(b"a")
await wait_all_tasks_blocked()
a.send(b"b")
b.send(b"a")
def setUp(self):
"""
Create a pair of UNIX sockets.
"""
self.input, self.output = socketpair(AF_UNIX)
def __init__(self, picard_args, unparsed_args, localedir, autoupdate):
# Set the WM_CLASS to 'MusicBrainz-Picard' so desktop environments
# can use it to look up the app
QtWidgets.QApplication.__init__(self,
['MusicBrainz-Picard'] + unparsed_args)
self.__class__.__instance = self
config._setup(self, picard_args.config_file)
# Allow High DPI Support
self.setAttribute(QtCore.Qt.AA_UseHighDpiPixmaps)
self.setAttribute(QtCore.Qt.AA_EnableHighDpiScaling)
self._cmdline_files = picard_args.FILE
self._autoupdate = autoupdate
self._debug = False
# FIXME: Figure out what's wrong with QThreadPool.globalInstance().
# It's a valid reference, but its start() method doesn't work.
self.thread_pool = QtCore.QThreadPool(self)
# Use a separate thread pool for file saving, with a thread count of 1,
# to avoid race conditions in File._save_and_rename.
self.save_thread_pool = QtCore.QThreadPool(self)
self.save_thread_pool.setMaxThreadCount(1)
if not sys.platform == "win32":
# Set up signal handling
# It's not possible to call all available functions from signal
# handlers, therefore we need to set up a QSocketNotifier to listen
# on a socket. Sending data through a socket can be done in a
# signal handler, so we use the socket to notify the application of
# the signal.
# This code is adopted from
# https://qt-project.org/doc/qt-4.8/unix-signals.html
# To not make the socket module a requirement for the Windows
# installer, import it here and not globally
import socket
self.signalfd = socket.socketpair(socket.AF_UNIX,
socket.SOCK_STREAM, 0)
self.signalnotifier = QtCore.QSocketNotifier(
self.signalfd[1].fileno(), QtCore.QSocketNotifier.Read, self)
self.signalnotifier.activated.connect(self.sighandler)
signal.signal(signal.SIGHUP, self.signal)
signal.signal(signal.SIGINT, self.signal)
signal.signal(signal.SIGTERM, self.signal)
# Setup logging
self.debug(picard_args.debug or "PICARD_DEBUG" in os.environ)
log.debug("Starting Picard from %r", os.path.abspath(__file__))
log.debug("Platform: %s %s %s", platform.platform(),
platform.python_implementation(), platform.python_version())
log.debug("Versions: %s", versions.as_string())
log.debug("Configuration file path: %r", config.config.fileName())
# TODO remove this before the final release
if sys.platform == "win32":
olduserdir = "~\\Local Settings\\Application Data\\MusicBrainz Picard"
else:
olduserdir = "~/.picard"
olduserdir = os.path.expanduser(olduserdir)
if os.path.isdir(olduserdir):
log.info("Moving %r to %r", olduserdir, USER_DIR)
try:
shutil.move(olduserdir, USER_DIR)
except:
pass
log.debug("User directory: %r", os.path.abspath(USER_DIR))
# for compatibility with pre-1.3 plugins
QtCore.QObject.tagger = self
QtCore.QObject.config = config
QtCore.QObject.log = log
check_io_encoding()
# Must be before config upgrade because upgrade dialogs need to be
# translated
setup_gettext(localedir, config.setting["ui_language"], log.debug)
upgrade_config()
self.xmlws = XmlWebService()
load_user_collections()
# Initialize fingerprinting
self._acoustid = acoustid.AcoustIDClient()
self._acoustid.init()
# Load plugins
self.pluginmanager = PluginManager()
if hasattr(sys, "frozen"):
self.pluginmanager.load_plugindir(
os.path.join(os.path.dirname(sys.argv[0]), "plugins"))
else:
mydir = os.path.dirname(os.path.abspath(__file__))
self.pluginmanager.load_plugindir(os.path.join(mydir, "plugins"))
self.pluginmanager.load_plugindir(
os.path.join(mydir, os.pardir, "contrib", "plugins"))
if not os.path.exists(USER_PLUGIN_DIR):
os.makedirs(USER_PLUGIN_DIR)
self.pluginmanager.load_plugindir(USER_PLUGIN_DIR)
self.pluginmanager.query_available_plugins()
self.acoustidmanager = AcoustIDManager()
self.browser_integration = BrowserIntegration()
self.files = {}
self.clusters = ClusterList()
self.albums = {}
self.release_groups = {}
self.mbid_redirects = {}
self.unmatched_files = UnmatchedFiles()
self.nats = None
self.window = MainWindow()
self.exit_cleanup = []
self.stopping = False
async def setUp(self):
self._l, self._r = socket.socketpair()
def __init__(self):
self._local_sock, self._remote_sock = socket.socketpair()
self.__raise_on_recv = False # see set_raise_on_recv()
def _socketpair(self):
return socket.socketpair()
def _make_self_pipe(self):
# A self-socket, really. :-)
self._ssock, self._csock = socket.socketpair()
self._ssock.setblocking(False)
self._csock.setblocking(False)
self._internal_fds += 1
os.environ["LOGNAME"] = os.environ["USER"] = u.pw_name
os.environ["PWD"] = os.getcwd()
os.environ["HOME"] = u.pw_dir
os.environ["SHELL"] = u.pw_shell
os.setregid(u.pw_gid, u.pw_gid)
try:
os.initgroups(user, u.pw_gid)
except OverflowError:
pass # FIXME???
os.setreuid(u.pw_uid, u.pw_uid)
assert os.name == "posix"
assert os.getuid() == 0, "graderutils must be run as root for uid switching"
control = socketpair()
stdin, stdout, stderr = os.pipe(), os.pipe(), os.pipe()
pid = os.fork()
if pid == 0: # Child process: rpyc server running student code
status = 250
try:
become(student_user)
# Pipe I/O on fd level
os.close(stdin[1])
os.close(stdout[0])
os.close(stderr[0])
os.dup2(stdin[0], 0)
os.dup2(stdout[1], 1)
os.dup2(stderr[1], 2)
control[0].close()
rsock.close()
sys.exit()
#print("Sending: ", cmd)
cltSkt.sendall(cmd + nlBytes)
lstnSkt = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
lstnSkt.bind((bindAddress, localport))
lstnSkt.listen(10)
print("[*] Listening on %s:%d." % (bindAddress, localport))
cltSkt, clientAddr = lstnSkt.accept()
cltIp = clientAddr[0]
print("[*] Incoming connection from %s." % cltIp)
print("[i] Type %chonker-exit at any prompt to quit Chonker.")
rsock, ssock = socket.socketpair()
t1 = threading.Thread(target=listenerThrd, args=(rsock, cltSkt))
t1.start()
# Free listening socket
lstnSkt.close()
# Prompt settings
promptSession = PromptSession(history=FileHistory(historyFile))
style = Style.from_dict({
'': '#ff0066', # User input (default text).
'prompt': '#aa22ff', # Prompt.
})
promptMessage = [
('class:prompt', promptStr),
if args.vnc_password and not args.vnc_port:
logger.error("--vnc-password can only be used with --vnc-port")
sys.exit(1)
if args.display == 'text':
from mcu.screen_text import TextScreen as Screen
elif args.display == 'headless':
from mcu.headless import Headless as Screen
else:
from mcu.screen import QtScreen as Screen
automation_path = None
if args.automation:
automation_path = automation.Automation(args.automation)
s1, s2 = socket.socketpair()
run_qemu(s1, s2, getattr(args, 'app.elf'), args.library, args.seed,
args.debug, args.trace, args.deterministic_rng, args.model,
args.rampage, args.sdk)
s1.close()
apdu = apdu_server.ApduServer(host="0.0.0.0", port=args.apdu_port)
seph = seproxyhal.SeProxyHal(s2, automation=automation_path)
button_tcp = None
if args.button_port:
button_tcp = FakeButton(args.button_port)
finger_tcp = None
if args.finger_port:
def main():
"""Run the aminer main program."""
# Extract program name, but only when sure to contain no problematic characters.
warnings.filterwarnings('ignore', category=ImportWarning)
program_name = sys.argv[0].split('/')[-1]
if (program_name == '.') or (program_name == '..') or (re.match('^[a-zA-Z0-9._-]+$', program_name) is None):
print('Invalid program name, check your execution args', file=sys.stderr)
sys.exit(1)
# We will not read stdin from here on, so get rid of it immediately, thus aberrant child cannot manipulate caller's stdin using it.
stdin_fd = os.open('/dev/null', os.O_RDONLY)
os.dup2(stdin_fd, 0)
os.close(stdin_fd)
help_message = 'aminer - logdata-anomaly-miner\n'
if supports_color():
help_message += colflame
else:
help_message += flame
parser = argparse.ArgumentParser(description=help_message, formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('-v', '--version', action='version', version=__version_string__)
parser.add_argument('-u', '--check-updates', action='store_true', help='check if updates for the aminer are available.')
parser.add_argument('-c', '--config', default='/etc/aminer/config.yml', type=str, help='path to the config-file')
parser.add_argument('-D', '--daemon', action='store_false', help='run as a daemon process')
parser.add_argument('-s', '--stat', choices=["0", "1", "2"], type=str,
help='set the stat level. Possible stat-levels are 0 for no statistics, 1 for normal statistic level and 2 for '
'verbose statistics.')
parser.add_argument('-d', '--debug', choices=["0", "1", "2"], type=str,
help='set the debug level. Possible debug-levels are 0 for no debugging, 1 for normal output (INFO and above), 2 '
'for printing all debug information.')
parser.add_argument('--run-analysis', action='store_true', help='enable/disable analysis')
parser.add_argument('-C', '--clear', action='store_true', help='removes all persistence directories')
parser.add_argument('-r', '--remove', action='append', type=str, help='removes a specific persistence directory')
parser.add_argument('-R', '--restore', type=str, help='restore a persistence backup')
parser.add_argument('-f', '--from-begin', action='store_true', help='removes RepositioningData before starting the aminer')
parser.add_argument('-o', '--offline-mode', action='store_true', help='stop the aminer after all logs have been processed.')
parser.add_argument("--config-properties", metavar="KEY=VALUE", nargs='+',
help="Set a number of config_properties by using key-value pairs (do not put spaces before or after the = sign). "
"If a value contains spaces, you should define it with double quotes: 'foo=\"this is a sentence\". Note that "
"values are always treated as strings. If values are already defined in the config_properties, the input "
"types are converted to the ones already existing.")
args = parser.parse_args()
if args.check_updates:
import urllib3
url = 'https://raw.githubusercontent.com/ait-aecid/logdata-anomaly-miner/main/source/root/usr/lib/logdata-anomaly-miner/metadata.py'
http = urllib3.PoolManager()
r = http.request('GET', url, preload_content=True)
metadata = r.data.decode()
http.clear()
lines = metadata.split('\n')
curr_version = None
for line in lines:
if '__version__ = ' in line:
curr_version = line.split('__version__ = ')[1].strip('"')
break
if __version__ == curr_version:
print("The current aminer version %s is installed." % curr_version)
else:
print("A new aminer version exists (%s). Currently version %s is installed." % (curr_version, __version__))
print("Use git pull to update the aminer version.")
sys.exit(0)
config_file_name = args.config
run_in_foreground_flag = args.daemon
run_analysis_child_flag = args.run_analysis
clear_persistence_flag = args.clear
remove_persistence_dirs = args.remove
from_begin_flag = args.from_begin
global offline_mode # skipcq: PYL-W0603
offline_mode = args.offline_mode
if args.restore is not None and ('.' in args.restore or '/' in args.restore):
parser.error('The restore path %s must not contain any . or /' % args.restore)
if args.remove is not None:
for remove in args.remove:
if '.' in remove or '/' in remove:
parser.error('The remove path %s must not contain any . or /' % remove)
restore_relative_persistence_path = args.restore
stat_level = 1
debug_level = 1
stat_level_console_flag = False
debug_level_console_flag = False
if args.stat is not None:
stat_level = int(args.stat)
stat_level_console_flag = True
if args.debug is not None:
debug_level = int(args.debug)
debug_level_console_flag = True
# Load the main configuration file.
if not os.path.exists(config_file_name):
print('%s: config "%s" not (yet) available!' % (program_name, config_file_name), file=sys.stderr)
sys.exit(1)
# using the solution here to override config_properties:
# https://stackoverflow.com/questions/27146262/create-variable-key-value-pairs-with-argparse-python
use_temp_config = False
config_properties = parse_vars(args.config_properties)
if args.config_properties and "LearnMode" in config_properties:
ymlext = [".YAML", ".YML", ".yaml", ".yml"]
extension = os.path.splitext(config_file_name)[1]
if extension in ymlext:
use_temp_config = True
fd, temp_config = tempfile.mkstemp(suffix=".yml")
with open(config_file_name) as f:
for line in f:
if "LearnMode" in line:
line = "LearnMode: %s" % config_properties["LearnMode"]
os.write(fd, line.encode())
config_file_name = temp_config
os.close(fd)
else:
msg = "The LearnMode parameter does not exist in .py configs!"
print(msg, sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
sys.exit(1)
# Minimal import to avoid loading too much within the privileged process.
try:
aminer_config = AminerConfig.load_config(config_file_name)
if use_temp_config:
os.remove(config_file_name)
config_file_name = args.config
except ValueError as e:
sys.exit(1)
for config_property in config_properties:
if config_property == "LearnMode":
continue
old_value = aminer_config.config_properties.get(config_property)
value = config_properties[config_property]
if old_value is not None:
try:
if isinstance(old_value, bool):
if value == "True":
value = True
elif value == "False":
value = False
else:
msg = "The %s parameter must be of type %s!" % (config_property, type(old_value))
print(msg, sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
sys.exit(1)
elif isinstance(old_value, int):
value = int(value)
elif isinstance(old_value, float):
value = float(value)
elif isinstance(old_value, list):
value = ast.literal_eval(value)
except ValueError:
msg = "The %s parameter must be of type %s!" % (config_property, type(old_value))
print(msg, sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
sys.exit(1)
else:
msg = "The %s parameter is not set in the config. It will be treated as a string!" % config_property
print("WARNING: " + msg, sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).warning(msg)
aminer_config.config_properties[config_property] = value
persistence_dir = aminer_config.config_properties.get(AminerConfig.KEY_PERSISTENCE_DIR, AminerConfig.DEFAULT_PERSISTENCE_DIR)
child_user_name = aminer_config.config_properties.get(AminerConfig.KEY_AMINER_USER)
child_group_name = aminer_config.config_properties.get(AminerConfig.KEY_AMINER_GROUP)
child_user_id = -1
child_group_id = -1
try:
if child_user_name is not None:
child_user_id = getpwnam(child_user_name).pw_uid
if child_group_name is not None:
child_group_id = getgrnam(child_group_name).gr_gid
except: # skipcq: FLK-E722
print('Failed to resolve %s or %s' % (AminerConfig.KEY_AMINER_USER, AminerConfig.KEY_AMINER_GROUP), file=sys.stderr)
sys.exit(1)
initialize_loggers(aminer_config, child_user_id, child_group_id)
if restore_relative_persistence_path is not None and (clear_persistence_flag or remove_persistence_dirs):
msg = 'The --restore parameter removes all persistence files. Do not use this parameter with --Clear or --Remove!'
print(msg, sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
sys.exit(1)
if not stat_level_console_flag and AminerConfig.KEY_LOG_STAT_LEVEL in aminer_config.config_properties:
stat_level = aminer_config.config_properties[AminerConfig.KEY_LOG_STAT_LEVEL]
if not debug_level_console_flag and AminerConfig.KEY_LOG_DEBUG_LEVEL in aminer_config.config_properties:
debug_level = aminer_config.config_properties[AminerConfig.KEY_LOG_DEBUG_LEVEL]
if AminerConfig.CONFIG_KEY_ENCODING in aminer_config.config_properties:
AminerConfig.ENCODING = aminer_config.config_properties[AminerConfig.CONFIG_KEY_ENCODING]
AminerConfig.STAT_LEVEL = stat_level
AminerConfig.DEBUG_LEVEL = debug_level
if clear_persistence_flag:
if remove_persistence_dirs:
msg = 'The --clear and --remove arguments must not be used together!'
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
sys.exit(1)
clear_persistence(persistence_dir)
if remove_persistence_dirs:
persistence_dir_name = aminer_config.config_properties.get(AminerConfig.KEY_PERSISTENCE_DIR, AminerConfig.DEFAULT_PERSISTENCE_DIR)
for filename in os.listdir(persistence_dir_name):
file_path = os.path.join(persistence_dir_name, filename)
try:
if not os.path.isdir(file_path):
msg = 'The aminer persistence directory should not contain any files.'
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).warning(msg)
continue
shutil.rmtree(file_path)
except OSError as e:
msg = 'Failed to delete %s. Reason: %s' % (file_path, e)
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
for filename in remove_persistence_dirs:
file_path = os.path.join(persistence_dir, filename)
try:
if not os.path.exists(file_path):
continue
if not os.path.isdir(file_path):
msg = 'The aminer persistence directory should not contain any files.'
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).warning(msg)
continue
shutil.rmtree(file_path)
except OSError as e:
msg = 'Failed to delete %s. Reason: %s' % (file_path, e)
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
if restore_relative_persistence_path is not None:
absolute_persistence_path = os.path.join(persistence_dir, 'backup', restore_relative_persistence_path)
if not os.path.exists(absolute_persistence_path):
msg = '%s does not exist. Continuing without restoring persistence.' % absolute_persistence_path
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).warning(msg)
else:
clear_persistence(persistence_dir)
copytree(absolute_persistence_path, persistence_dir)
persistence_dir_fd = SecureOSFunctions.secure_open_base_directory(persistence_dir, os.O_RDONLY | os.O_DIRECTORY | os.O_PATH)
for dirpath, _dirnames, filenames in os.walk(persistence_dir):
os.chown(dirpath, child_user_id, child_group_id, dir_fd=persistence_dir_fd, follow_symlinks=False)
for filename in filenames:
os.chown(os.path.join(dirpath, filename), child_user_id, child_user_id, dir_fd=persistence_dir_fd,
follow_symlinks=False)
if from_begin_flag:
repositioning_data_path = os.path.join(aminer_config.config_properties.get(
AminerConfig.KEY_PERSISTENCE_DIR, AminerConfig.DEFAULT_PERSISTENCE_DIR), 'AnalysisChild', 'RepositioningData')
if os.path.exists(repositioning_data_path):
os.remove(repositioning_data_path)
if run_analysis_child_flag:
# Call analysis process, this function will never return.
run_analysis_child(aminer_config, program_name)
# Start importing of aminer specific components after reading of "config.py" to allow replacement of components via sys.path
# from within configuration.
log_sources_list = aminer_config.config_properties.get(AminerConfig.KEY_LOG_SOURCES_LIST)
if (log_sources_list is None) or not log_sources_list:
msg = '%s: %s not defined' % (program_name, AminerConfig.KEY_LOG_SOURCES_LIST)
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
sys.exit(1)
# Now create the management entries for each logfile.
log_data_resource_dict = {}
for log_resource_name in log_sources_list:
# From here on log_resource_name is a byte array.
log_resource_name = decode_string_as_byte_string(log_resource_name)
log_resource = None
if log_resource_name.startswith(b'file://'):
log_resource = FileLogDataResource(log_resource_name, -1)
elif log_resource_name.startswith(b'unix://'):
log_resource = UnixSocketLogDataResource(log_resource_name, -1)
else:
msg = 'Unsupported schema in %s: %s' % (AminerConfig.KEY_LOG_SOURCES_LIST, repr(log_resource_name))
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
sys.exit(1)
if not os.path.exists(log_resource_name[7:].decode()):
msg = "WARNING: file or socket '%s' does not exist (yet)!" % log_resource_name[7:].decode()
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).warning(msg)
try:
log_resource.open()
except OSError as open_os_error:
if open_os_error.errno == errno.EACCES:
msg = '%s: no permission to access %s' % (program_name, repr(log_resource_name))
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
sys.exit(1)
else:
msg = '%s: unexpected error opening %s: %d (%s)' % (
program_name, repr(log_resource_name), open_os_error.errno, os.strerror(open_os_error.errno))
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
sys.exit(1)
log_data_resource_dict[log_resource_name] = log_resource
# Create the remote control socket, if any. Do this in privileged mode to allow binding it at arbitrary locations and support restricted
# permissions of any type for current (privileged) uid.
remote_control_socket_name = aminer_config.config_properties.get(AminerConfig.KEY_REMOTE_CONTROL_SOCKET_PATH, None)
remote_control_socket = None
if remote_control_socket_name is not None:
if os.path.exists(remote_control_socket_name):
try:
os.unlink(remote_control_socket_name)
except OSError:
msg = 'Failed to clean up old remote control socket at %s' % remote_control_socket_name
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
sys.exit(1)
# Create the local socket: there is no easy way to create it with correct permissions, hence a fork is needed, setting umask,
# bind the socket. It is also recommended to create the socket in a directory having the correct permissions already.
remote_control_socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
remote_control_socket.setblocking(False)
bind_child_pid = os.fork()
if bind_child_pid == 0:
os.umask(0o177)
remote_control_socket.bind(remote_control_socket_name)
# Do not perform any cleanup, flushing of streams. Use _exit(0) to avoid interference with fork.
os._exit(0) # skipcq: PYL-W0212
os.waitpid(bind_child_pid, 0)
remote_control_socket.listen(4)
# Now have checked all we can get from the configuration in the privileged process. Detach from the TTY when in daemon mode.
if not run_in_foreground_flag:
child_pid = 0
try:
# Fork a child to make sure, we are not the process group leader already.
child_pid = os.fork()
except Exception as fork_exception: # skipcq: PYL-W0703
msg = 'Failed to daemonize: %s' % fork_exception
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
sys.exit(1)
if child_pid != 0:
# This is the parent.
os._exit(0) # skipcq: PYL-W0212
# This is the child. Create a new session and become process group leader. Here we get rid of the controlling tty.
os.setsid()
# Fork again to become an orphaned process not being session leader, hence not able to get a controlling tty again.
try:
child_pid = os.fork()
except Exception as fork_exception: # skipcq: PYL-W0703
msg = 'Failed to daemonize: %s' % fork_exception
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
sys.exit(1)
if child_pid != 0:
# This is the parent.
os._exit(0) # skipcq: PYL-W0212
# Move to root directory to avoid lingering in some cwd someone else might want to unmount.
os.chdir('/')
# Change the umask here to clean all group/other mask bits so that accidentially created files are not accessible by other.
os.umask(0o77)
# Install a signal handler catching common stop signals and relaying it to all children for sure.
# skipcq: PYL-W0603
global child_termination_triggered_flag
child_termination_triggered_flag = False
def graceful_shutdown_handler(_signo, _stackFrame):
"""React on typical shutdown signals."""
msg = '%s: caught signal, shutting down' % program_name
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).info(msg)
# Just set the flag. It is likely, that child received same signal also so avoid multiple signaling, which could interrupt the
# shutdown procedure again.
# skipcq: PYL-W0603
global child_termination_triggered_flag
child_termination_triggered_flag = True
import signal
signal.signal(signal.SIGHUP, graceful_shutdown_handler)
signal.signal(signal.SIGINT, graceful_shutdown_handler)
signal.signal(signal.SIGTERM, graceful_shutdown_handler)
# Now create the socket to connect the analysis child.
(parent_socket, child_socket) = socket.socketpair(socket.AF_UNIX, socket.SOCK_DGRAM, 0)
# Have it nonblocking from here on.
parent_socket.setblocking(False)
child_socket.setblocking(False)
# Use normal fork, we should have been detached from TTY already. Flush stderr to avoid duplication of output if both child and
# parent want to write something.
sys.stderr.flush()
child_pid = os.fork()
if child_pid == 0:
# Relocate the child socket fd to 3 if needed
if child_socket.fileno() != 3:
os.dup2(child_socket.fileno(), 3)
child_socket.close()
# Clear the supplementary groups before dropping privileges. This makes only sense when changing the uid or gid.
if os.getuid() == 0:
if ((child_user_id != -1) and (child_user_id != os.getuid())) or ((child_group_id != -1) and (child_group_id != os.getgid())):
os.setgroups([])
# Drop privileges before executing child. setuid/gid will raise an exception when call has failed.
if child_group_id != -1:
os.setgid(child_group_id)
if child_user_id != -1:
os.setuid(child_user_id)
else:
msg = 'INFO: No privilege separation when started as unprivileged user'
print(msg, file=sys.stderr)
tmp_username = aminer_config.config_properties.get(AminerConfig.KEY_AMINER_USER)
tmp_group = aminer_config.config_properties.get(AminerConfig.KEY_AMINER_GROUP)
aminer_user_id = -1
aminer_group_id = -1
try:
if tmp_username is not None:
aminer_user_id = getpwnam(tmp_username).pw_uid
if tmp_group is not None:
aminer_group_id = getgrnam(tmp_group).gr_gid
except: # skipcq: FLK-E722
print('Failed to resolve %s or %s' % (AminerConfig.KEY_AMINER_USER, AminerConfig.KEY_AMINER_GROUP), file=sys.stderr)
sys.exit(1)
initialize_loggers(aminer_config, aminer_user_id, aminer_group_id)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).info(msg)
# Now resolve the specific analysis configuration file (if any).
analysis_config_file_name = aminer_config.config_properties.get(AminerConfig.KEY_ANALYSIS_CONFIG_FILE, None)
if analysis_config_file_name is None:
analysis_config_file_name = config_file_name
elif not os.path.isabs(analysis_config_file_name):
analysis_config_file_name = os.path.join(os.path.dirname(config_file_name), analysis_config_file_name)
# This is the child. Close all parent file descriptors, we do not need. Perhaps this could be done more elegantly.
for close_fd in range(4, 1 << 16):
try:
os.close(close_fd)
except OSError as open_os_error:
if open_os_error.errno == errno.EBADF:
continue
msg = '%s: unexpected exception closing file descriptors: %s' % (program_name, open_os_error)
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
# Flush stderr before exit without any cleanup.
sys.stderr.flush()
os._exit(1) # skipcq: PYL-W0212
# Now execute the very same program again, but user might have moved or renamed it meanwhile. This would be problematic with
# SUID-binaries (which we do not yet support). Do NOT just fork but also exec to avoid child circumventing
# parent's ALSR due to cloned kernel VMA.
exec_args = ['aminerChild', '--run-analysis', '--config', analysis_config_file_name, '--stat', str(stat_level), '--debug',
str(debug_level)]
if offline_mode:
exec_args.append("--offline-mode")
if args.config_properties:
exec_args.append("--config-properties")
for config_property in args.config_properties:
exec_args.append(config_property)
os.execv(sys.argv[0], exec_args) # skipcq: BAN-B606
msg = 'Failed to execute child process'
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
sys.stderr.flush()
os._exit(1) # skipcq: PYL-W0212
child_socket.close()
# Send all log resource information currently available to child process.
for log_resource_name, log_resource in log_data_resource_dict.items():
if (log_resource is not None) and (log_resource.get_file_descriptor() >= 0):
SecureOSFunctions.send_logstream_descriptor(parent_socket, log_resource.get_file_descriptor(), log_resource_name)
log_resource.close()
# Send the remote control server socket, if any and close it afterwards. It is not needed any more on parent side.
if remote_control_socket is not None:
SecureOSFunctions.send_annotated_file_descriptor(parent_socket, remote_control_socket.fileno(), 'remotecontrol', '')
remote_control_socket.close()
exit_status = 0
child_termination_triggered_count = 0
while True:
if child_termination_triggered_flag:
if child_termination_triggered_count == 0:
time.sleep(1)
elif child_termination_triggered_count < 5:
os.kill(child_pid, signal.SIGTERM)
else:
os.kill(0, signal.SIGKILL)
child_termination_triggered_count += 1
(sig_child_pid, sig_status) = os.waitpid(-1, os.WNOHANG)
if sig_child_pid != 0:
if sig_child_pid == child_pid:
if child_termination_triggered_flag or offline_mode:
# This was expected, just terminate.
break
msg = '%s: Analysis child process %d terminated unexpectedly with signal 0x%x' % (program_name, sig_child_pid, sig_status)
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
exit_status = 1
break
# So the child has been cloned, the clone has terminated. This should not happen either.
msg = '%s: untracked child %d terminated with with signal 0x%x' % (program_name, sig_child_pid, sig_status)
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
exit_status = 1
# Child information handled, scan for rotated logfiles or other resources, where reopening might make sense.
for log_resouce_name, log_data_resource in log_data_resource_dict.items():
try:
if not log_data_resource.open(reopen_flag=True):
continue
except OSError as open_os_error:
if open_os_error.errno == errno.EACCES:
msg = '%s: no permission to access %s' % (program_name, log_resouce_name)
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
else:
msg = '%s: unexpected error reopening %s: %d (%s)' % (
program_name, log_resouce_name, open_os_error.errno, os.strerror(open_os_error.errno))
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
exit_status = 2
continue
SecureOSFunctions.send_logstream_descriptor(parent_socket, log_data_resource.get_file_descriptor(), log_resouce_name)
log_data_resource.close()
time.sleep(1)
parent_socket.close()
SecureOSFunctions.close_base_directory()
SecureOSFunctions.close_log_directory()
sys.exit(exit_status)