def get_events(fd, timeout=None):
(rlist, _, _) = select.select([fd], [], [], timeout)
if not rlist:
return []
events = []
while True:
buf = os.read(fd, _BUF_LEN)
i = 0
while i < len(buf):
(wd, mask, cookie, len_) = struct.unpack_from(_EVENT_FMT, buf, i)
name = None
if len_ > 0:
start = i + _EVENT_SIZE
end = start + len_
# remove \0 terminator and padding
name = buf[start:end].rstrip('\0')
events.append(InotifyEvent(wd, mask, cookie, name))
i += _EVENT_SIZE + len_
(rlist, _, _) = select.select([fd], [], [], 0)
if not rlist:
break
return events
def recv(handle):
tl = handle.recv(4)
if not tl:
return None
while len(tl) < 4:
ndata = handle.recv(4 - len(tl))
if not ndata:
raise Exception("Error reading data")
tl += ndata
if len(tl) == 0:
return None
tl = struct.unpack("!I", tl)[0]
if tl & 0b10000000000000000000000000000000:
raise Exception("Protocol Violation, reserved bit set")
# 4 byte tlv
dlen = tl & 16777215 # grab lower 24 bits
datatype = (tl & 2130706432) >> 24 # grab 7 bits from near beginning
if dlen == 0:
return None
if datatype == tlv.Types.filehandle:
filehandles = array.array('i')
rawbuffer = bytearray(2048)
pkttype = ctypes.c_ubyte * 2048
data = pkttype.from_buffer(rawbuffer)
cmsgsize = CMSG_SPACE(ctypes.sizeof(ctypes.c_int)).value
cmsgarr = bytearray(cmsgsize)
cmtype = ctypes.c_ubyte * cmsgsize
cmsg = cmtype.from_buffer(cmsgarr)
cmsg.cmsg_level = socket.SOL_SOCKET
cmsg.cmsg_type = SCM_RIGHTS
cmsg.cmsg_len = CMSG_LEN(ctypes.sizeof(ctypes.c_int))
iov = iovec()
iov.iov_base = ctypes.addressof(data)
iov.iov_len = 2048
msg = msghdr()
msg.msg_iov = ctypes.pointer(iov)
msg.msg_iovlen = 1
msg.msg_control = ctypes.addressof(cmsg)
msg.msg_controllen = ctypes.sizeof(cmsg)
select.select([handle], [], [])
i = recvmsg(handle.fileno(), ctypes.pointer(msg), 0)
cdata = cmsgarr[CMSG_LEN(0).value:]
data = rawbuffer[:i]
if cmsg.cmsg_level == socket.SOL_SOCKET and cmsg.cmsg_type == SCM_RIGHTS:
filehandles.fromstring(bytes(
cdata[:len(cdata) - len(cdata) % filehandles.itemsize]))
data = json.loads(bytes(data))
return ClientFile(data['filename'], data['mode'], filehandles[0])
else:
data = handle.recv(dlen)
while len(data) < dlen:
ndata = handle.recv(dlen - len(data))
if not ndata:
raise Exception("Error reading data")
data += ndata
if datatype == tlv.Types.text:
return data
elif datatype == tlv.Types.json:
return json.loads(data)
def raw_input(self, prompt):
sys.stdout.write(prompt)
sys.stdout.flush()
select.select([sys.stdin], [], [])
s = sys.stdin.readline()
if not s:
raise EOFError()
return s.strip()
def raw_input(self, prompt):
sys.stdout.write(prompt)
sys.stdout.flush()
select.select([sys.stdin],[],[])
s = sys.stdin.readline()
if not s:
raise EOFError()
return s.strip()
def _grab_rsps(socks, rsps, interval, xidmap):
r, _, _ = select.select(socks, (), (), interval)
while r:
for s in r:
(rsp, peer) = s.recvfrom(9000)
neighutil.refresh_neigh()
_parse_slp_packet(rsp, peer, rsps, xidmap)
r, _, _ = select.select(socks, (), (), interval)
def watch_for_cert(self):
libc = ctypes.CDLL(ctypes.util.find_library('c'))
watcher = libc.inotify_init()
if libc.inotify_add_watch(watcher, '/etc/confluent/', 0x100) > -1:
while True:
select.select((watcher, ), (), (), 86400)
if self.should_run_remoteapi():
os.close(watcher)
self.start_remoteapi()
break
def _recv_offload():
upacker = msgpack.Unpacker(encoding='utf8')
instream = _offloader.stdout.fileno()
while True:
select.select([_offloader.stdout], [], [])
upacker.feed(os.read(instream, 128))
for result in upacker:
if result[0] not in _offloadevts:
print("Uh oh, unexpected event id... " + repr(result))
continue
_offloadevts[result[0]].send(result[1:])
eventlet.sleep(0)
def _bro_thread(self):
while True:
try:
self.bc = Connection('127.0.0.1:47758')
fd = bro_conn_get_fd(self.bc.bc)
while True:
select.select([fd, ], [], [])
self.bc.processInput()
print('bro process input')
except Exception,e:
print(e.message)
time.sleep(1)
def watch_for_cert(self):
watcher = libc.inotify_init1(os.O_NONBLOCK)
if libc.inotify_add_watch(watcher, b'/etc/confluent/', 0x100) > -1:
while True:
select.select((watcher, ), (), (), 86400)
try:
os.read(watcher, 1024)
except Exception:
pass
if self.should_run_remoteapi():
os.close(watcher)
self.start_remoteapi()
break
def _grab_rsps(socks, rsps, interval, xidmap):
r = None
res = select.select(socks, (), (), interval)
if res:
r = res[0]
while r:
for s in r:
(rsp, peer) = s.recvfrom(9000)
neighutil.refresh_neigh()
_parse_slp_packet(rsp, peer, rsps, xidmap)
res = select.select(socks, (), (), interval)
if not res:
r = None
else:
r = res[0]
def _find_service(service, target):
net4 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
net6 = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
net6.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)
if target:
addrs = socket.getaddrinfo(target, 1900, 0, socket.SOCK_DGRAM)
for addr in addrs:
host = addr[4][0]
if addr[0] == socket.AF_INET:
net4.sendto(smsg.format(host, service), addr[4])
elif addr[0] == socket.AF_INET6:
host = '[{0}]'.format(host)
net6.sendto(smsg.format(host, service), addr[4])
else:
net4.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
for idx in util.list_interface_indexes():
net6.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_IF, idx)
try:
net6.sendto(smsg.format('[{0}]'.format(mcastv6addr), service),
(mcastv6addr, 1900, 0, 0))
except socket.error:
# ignore interfaces without ipv6 multicast causing error
pass
for i4 in util.list_ips():
if 'broadcast' not in i4:
continue
addr = i4['addr']
bcast = i4['broadcast']
net4.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_IF,
socket.inet_aton(addr))
net4.sendto(smsg.format(mcastv4addr, service), (mcastv4addr, 1900))
net4.sendto(smsg.format(bcast, service), (bcast, 1900))
# SSDP by spec encourages responses to spread out over a 3 second interval
# hence we must be a bit more patient
deadline = util.monotonic_time() + 4
r, _, _ = select.select((net4, net6), (), (), 4)
peerdata = {}
while r:
for s in r:
(rsp, peer) = s.recvfrom(9000)
neighutil.refresh_neigh()
_parse_ssdp(peer, rsp, peerdata)
timeout = deadline - util.monotonic_time()
if timeout < 0:
timeout = 0
r, _, _ = select.select((net4, net6), (), (), timeout)
for nid in peerdata:
yield peerdata[nid]
def _sendall(sock, data):
"""
Writes data to a socket and does not return until all the data is sent.
"""
length = len(data)
while length:
try:
sent = sock.send(data)
except socket.error, e:
if e[0] == errno.EAGAIN:
select.select([], [sock], [])
continue
else:
raise
data = data[sent:]
length -= sent
def relaydata(self):
while self.subproc is not None:
rdylist, _, _ = select.select(
(self._master, self.subproc.stderr), (), (),
3600 + (random.random() * 120))
if self._master in rdylist:
try:
somedata = os.read(self._master, 128)
while somedata:
self._datacallback(somedata)
eventlet.sleep(0)
somedata = os.read(self._master, 128)
except OSError as e:
if e.errno == 5:
self._datacallback(conapi.ConsoleEvent.Disconnect)
self.subproc = None
return
if e.errno != 11:
raise
if self.subproc.stderr in rdylist:
try:
somedata = self.subproc.stderr.read()
while somedata:
self._datacallback(somedata)
eventlet.sleep(0)
somedata = self.subproc.stderr.read()
except IOError as e:
if e.errno != 11:
raise
childstate = self.subproc.poll()
if childstate is not None:
self._datacallback(conapi.ConsoleEvent.Disconnect)
self.subproc = None
def _write_child_when_able(self):
"""
Select on the child's input file descriptor becoming writable, and then write commands to it.
If not successful in writing all the commands, spawn a new greenlet to retry.
"""
LOG.debug("write_child_when_able")
buffer_contents = self._read_from_write_buffer()
if not buffer_contents:
return
try:
r, w, x = select.select([],[self._fd],[])
except Exception, e:
# The next 9 lines are taken from Facebook's Tornado project, which is open-sourced under
# the Apache license.
# Depending on python version and poll implementation,
# different exception types may be thrown and there are
# two ways EINTR might be signaled:
# * e.errno == errno.EINTR
# * e.args is like (errno.EINTR, 'Interrupted system call')
if (getattr(e, 'errno') == errno.EINTR or
(isinstance(getattr(e, 'args'), tuple) and
len(e.args) == 2 and e.args[0] == errno.EINTR)):
LOG.warning("Interrupted system call", exc_info=1)
eventlet.spawn_n(self._write_child_when_able)
else:
LOG.error("Unexpected error on select")
self.mark_for_cleanup()
return
def test_nonblocking_accept_mark_as_reopened(self):
evt_hub = get_hub()
with mock.patch.object(evt_hub,
"mark_as_reopened") as patched_mark_as_reopened:
def connect_once(listener):
# delete/overwrite the original conn
# object, only keeping the file object around
# closing the file object should close everything
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client.connect(('127.0.0.1', listener.getsockname()[1]))
client.close()
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server.bind(('127.0.0.1', 0))
server.listen(50)
server.setblocking(False)
acceptlet = eventlet.spawn(connect_once, server)
out = select.select([server], [], [])
conn, addr = server.accept()
conn.sendall(b'hello\n')
connfileno = conn.fileno()
conn.close()
assert patched_mark_as_reopened.called
assert patched_mark_as_reopened.call_count == 3, "3 fds were opened, but the hub was " \
"only notified {call_count} times" \
.format(call_count=patched_mark_as_reopened.call_count)
args, kwargs = patched_mark_as_reopened.call_args
assert args == (connfileno,), "Expected mark_as_reopened to be called " \
"with {expected_fileno}, but it was called " \
"with {fileno}".format(expected_fileno=connfileno,
fileno=args[0])
server.close()
def test_nonblocking_accept_mark_as_reopened(self):
evt_hub = get_hub()
with mock.patch.object(evt_hub, "mark_as_reopened") as patched_mark_as_reopened:
def connect_once(listener):
# delete/overwrite the original conn
# object, only keeping the file object around
# closing the file object should close everything
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client.connect(('127.0.0.1', listener.getsockname()[1]))
client.close()
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server.bind(('127.0.0.1', 0))
server.listen(50)
server.setblocking(False)
acceptlet = eventlet.spawn(connect_once, server)
out = select.select([server], [], [])
conn, addr = server.accept()
conn.sendall(b'hello\n')
connfileno = conn.fileno()
conn.close()
assert patched_mark_as_reopened.called
assert patched_mark_as_reopened.call_count == 3, "3 fds were opened, but the hub was " \
"only notified {call_count} times" \
.format(call_count=patched_mark_as_reopened.call_count)
args, kwargs = patched_mark_as_reopened.call_args
assert args == (connfileno,), "Expected mark_as_reopened to be called " \
"with {expected_fileno}, but it was called " \
"with {fileno}".format(expected_fileno=connfileno,
fileno=args[0])
server.close()
def relaydata(self):
while self.subproc is not None:
rdylist, _, _ = select.select((self._master, self.subproc.stderr),
(), (),
3600 + (random.random() * 120))
if self._master in rdylist:
try:
somedata = os.read(self._master, 128)
while somedata:
self._datacallback(somedata)
eventlet.sleep(0)
somedata = os.read(self._master, 128)
except OSError as e:
if e.errno == 5:
self._datacallback(conapi.ConsoleEvent.Disconnect)
self.subproc = None
return
if e.errno != 11:
raise
if self.subproc.stderr in rdylist:
try:
somedata = self.subproc.stderr.read()
while somedata:
self._datacallback(somedata)
eventlet.sleep(0)
somedata = self.subproc.stderr.read()
except IOError as e:
if e.errno != 11:
raise
childstate = self.subproc.poll()
if childstate is not None:
self._datacallback(conapi.ConsoleEvent.Disconnect)
self.subproc = None
def _write_child_when_able(self):
"""
Select on the child's input file descriptor becoming writable, and then write commands to it.
If not successful in writing all the commands, spawn a new greenlet to retry.
"""
LOG.debug("write_child_when_able")
buffer_contents = self._read_from_write_buffer()
if not buffer_contents:
return
try:
r, w, x = select.select([],[self._fd],[])
except Exception, e:
# The next 9 lines are taken from Facebook's Tornado project, which is open-sourced under
# the Apache license.
# Depending on python version and poll implementation,
# different exception types may be thrown and there are
# two ways EINTR might be signaled:
# * e.errno == errno.EINTR
# * e.args is like (errno.EINTR, 'Interrupted system call')
if (getattr(e, 'errno') == errno.EINTR or
(isinstance(getattr(e, 'args'), tuple) and
len(e.args) == 2 and e.args[0] == errno.EINTR)):
LOG.warning("Interrupted system call", exc_info=1)
eventlet.spawn_n(self._write_child_when_able)
else:
LOG.error("Unexpected error on select")
self.mark_for_cleanup()
return
def testClose(self):
conn, addr = self.serv.accept()
conn.close()
sd = self.cli
read, write, err = select.select([sd], [], [], 1.0)
self.assertEqual(read, [sd])
self.assertEqual(sd.recv(1), '')
def testClose(self):
conn, addr = self.serv.accept()
conn.close()
sd = self.cli
read, write, err = select.select([sd], [], [], 1.0)
self.assertEqual(read, [sd])
self.assertEqual(sd.recv(1), '')
def proxydhcp():
net4011 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
net4011.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
net4011.setsockopt(socket.IPPROTO_IP, IP_PKTINFO, 1)
net4011.bind(('', 4011))
cfg = cfm.ConfigManager(None)
while True:
ready = select.select([net4011], [], [], None)
if not ready or not ready[0]:
continue
rq = bytearray(1024)
rqv = memoryview(rq)
nb, client = net4011.recvfrom_into(rq)
if nb < 240:
continue
rp = bytearray(1024)
rpv = memoryview(rp)
try:
optidx = rq.index(b'\x63\x82\x53\x63') + 4
except ValueError:
continue
opts, disco = opts_to_dict(rq, optidx, 3)
disco['uuid']
node = None
if disco.get('hwaddr', None) in macmap:
node = macmap[disco['hwaddr']]
elif disco.get('uuid', None) in uuidmap:
node = uuidmap[disco['uuid']]
if not node:
continue
hwlen = rq[2]
myipn = myipbypeer.get(rqv[28:28 + hwlen].tobytes(), None)
if not myipn:
continue
if opts.get(77, None) == b'iPXE':
cfd = cfg.get_node_attributes(node, ('deployment.*'))
profile = cfd.get(node, {}).get('deployment.pendingprofile',
{}).get('value', None)
if not profile:
continue
myip = socket.inet_ntoa(myipn)
bootfile = 'http://{0}/confluent-public/os/{1}/boot.ipxe'.format(
myip, profile).encode('utf8')
elif disco['arch'] == 'uefi-x64':
bootfile = b'confluent/x86_64/ipxe.efi'
elif disco['arch'] == 'bios-x86':
bootfile = b'confluent/x86_64/ipxe.kkpxe'
rpv[:240] = rqv[:240].tobytes()
rpv[0:1] = b'\x02'
rpv[108:108 + len(bootfile)] = bootfile
rpv[240:243] = b'\x35\x01\x05'
rpv[243:249] = b'\x36\x04' + myipn
rpv[20:24] = myipn
rpv[249:268] = b'\x61\x11' + opts[97]
rpv[268:280] = b'\x3c\x09PXEClient\xff'
net4011.sendto(rpv[:281], client)
def _process_files(self):
while True:
try:
ready = select.select([f for f in self._files if not f.active and not f.closed], [], [])[0]
except RestartSelect:
continue
else:
for file in ready:
file.active = True
self._command_channel.send(Command('process_results', files=[f for f in ready if not f.closed]))
def handle_connection(incoming, outgoing):
while True:
r, _, _ = select.select((incoming, outgoing), (), (), 60)
for mysock in r:
data = mysock.recv(32768)
if not data:
return
if mysock == incoming:
outgoing.sendall(data)
elif mysock == outgoing:
incoming.sendall(data)
def forward(conn_a, conn_b): """Forward data both ways between connections until one closes.""" conns = conn_a, conn_b while True: # Get connections that are ready to read from for conn in select.select(conns, [], [])[0]: data = conn.recv(32768) if len(data) == 0: return # Write data to the other connection conns[1-conns.index(conn)].sendall(data)
def get_buffer_output(nodename):
out = _bufferdaemon.stdin
instream = _bufferdaemon.stdout
if not isinstance(nodename, bytes):
nodename = nodename.encode('utf8')
outdata = bytearray()
with _bufferlock:
out.write(struct.pack('I', len(nodename)))
out.write(nodename)
out.flush()
select.select((instream, ), (), (), 30)
while not outdata or outdata[-1]:
try:
chunk = os.read(instream.fileno(), 128)
except IOError:
chunk = None
if chunk:
outdata.extend(chunk)
else:
select.select((instream, ), (), (), 0)
return bytes(outdata[:-1])
def _recvall(sock, length):
"""
Attempts to receive length bytes from a socket, blocking if necessary.
(Socket may be blocking or non-blocking.)
"""
dataList = []
recvLen = 0
while length:
try:
data = sock.recv(length)
except socket.error, e:
if e[0] == errno.EAGAIN:
select.select([sock], [], [])
continue
else:
raise
if not data: # EOF
break
dataList.append(data)
dataLen = len(data)
recvLen += dataLen
length -= dataLen
def testAccept(self):
# Testing non-blocking accept
self.serv.setblocking(0)
try:
conn, addr = self.serv.accept()
except socket.error:
pass
else:
self.fail("Error trying to do non-blocking accept.")
read, write, err = select.select([self.serv], [], [])
if self.serv in read:
conn, addr = self.serv.accept()
else:
self.fail("Error trying to do accept after select.")
def _process_files(self):
while True:
try:
ready = select.select(
[f for f in self._files if not f.active and not f.closed],
[], [])[0]
except RestartSelect:
continue
else:
for file in ready:
file.active = True
self._command_channel.send(
Command('process_results',
files=[f for f in ready if not f.closed]))
def testAccept(self):
# Testing non-blocking accept
self.serv.setblocking(0)
try:
conn, addr = self.serv.accept()
except socket.error:
pass
else:
self.fail("Error trying to do non-blocking accept.")
read, write, err = select.select([self.serv], [], [])
if self.serv in read:
conn, addr = self.serv.accept()
else:
self.fail("Error trying to do accept after select.")
def testRecv(self):
# Testing non-blocking recv
conn, addr = self.serv.accept()
conn.setblocking(0)
try:
msg = conn.recv(len(MSG))
except socket.error:
pass
else:
self.fail("Error trying to do non-blocking recv.")
read, write, err = select.select([conn], [], [])
if conn in read:
msg = conn.recv(len(MSG))
self.assertEqual(msg, MSG)
else:
self.fail("Error during select call to non-blocking socket.")
def testRecv(self):
# Testing non-blocking recv
conn, addr = self.serv.accept()
conn.setblocking(0)
try:
msg = conn.recv(len(MSG))
except socket.error:
pass
else:
self.fail("Error trying to do non-blocking recv.")
read, write, err = select.select([conn], [], [])
if conn in read:
msg = conn.recv(len(MSG))
self.assertEqual(msg, MSG)
else:
self.fail("Error during select call to non-blocking socket.")
def check_events(self, timeout=None):
while True:
try:
# blocks up to 'timeout' milliseconds
if timeout is None:
timeout = self._timeout
ret = select.select([self._fd], [self._fd], [self._fd])
except select.error as err:
if six.PY2 and err[0] == errno.EINTR:
break
elif six.PY3 and err.errno == errno.EINTR:
break # interrupted, retry
else:
raise
else:
break
# only one fd is polled
return bool(ret[0])
def check_events(self, timeout=None):
while True:
try:
# blocks up to 'timeout' milliseconds
if timeout is None:
timeout = self._timeout
ret = select.select([self._fd], [self._fd], [self._fd])
except select.error as err:
if six.PY2 and err[0] == errno.EINTR:
break
elif six.PY3 and err.errno == errno.EINTR:
break # interrupted, retry
else:
raise
else:
break
# only one fd is polled
return bool(ret[0])
def snoop(handler, protocol=None):
"""Watch for SLP activity
handler will be called with a dictionary of relevant attributes
:param handler:
:return:
"""
tracelog = log.Logger('trace')
try:
active_scan(handler, protocol)
except Exception as e:
tracelog.log(traceback.format_exc(),
ltype=log.DataTypes.event,
event=log.Events.stacktrace)
net = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
net.setsockopt(IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)
slpg = socket.inet_pton(socket.AF_INET6, 'ff01::123')
slpg2 = socket.inet_pton(socket.AF_INET6, 'ff02::123')
for i6idx in util.list_interface_indexes():
mreq = slpg + struct.pack('=I', i6idx)
net.setsockopt(IPPROTO_IPV6, socket.IPV6_JOIN_GROUP, mreq)
mreq = slpg2 + struct.pack('=I', i6idx)
net.setsockopt(IPPROTO_IPV6, socket.IPV6_JOIN_GROUP, mreq)
net4 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
net.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
net4.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
for i4 in util.list_ips():
if 'broadcast' not in i4:
continue
slpmcast = socket.inet_aton('239.255.255.253') + \
socket.inet_aton(i4['addr'])
try:
net4.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP,
slpmcast)
except socket.error as e:
if e.errno != 98:
raise
# socket in use can occur when aliased ipv4 are encountered
net.bind(('', 427))
net4.bind(('', 427))
while True:
try:
newmacs = set([])
r, _, _ = select.select((net, net4), (), (), 60)
# clear known_peers and peerbymacaddress
# to avoid stale info getting in...
# rely upon the select(0.2) to catch rapid fire and aggregate ip
# addresses that come close together
# calling code needs to understand deeper context, as snoop
# will now yield dupe info over time
known_peers = set([])
peerbymacaddress = {}
while r:
for s in r:
(rsp, peer) = s.recvfrom(9000)
ip = peer[0].partition('%')[0]
if peer in known_peers:
continue
if ip not in neighutil.neightable:
neighutil.update_neigh()
if ip not in neighutil.neightable:
continue
known_peers.add(peer)
mac = neighutil.neightable[ip]
if mac in peerbymacaddress:
peerbymacaddress[mac]['addresses'].append(peer)
else:
q = query_srvtypes(peer)
if not q or not q[0]:
# SLP might have started and not ready yet
# ignore for now
known_peers.discard(peer)
continue
# we want to prioritize the very well known services
svcs = []
for svc in q:
if svc in _slp_services:
svcs.insert(0, svc)
else:
svcs.append(svc)
peerbymacaddress[mac] = {
'services': svcs,
'addresses': [peer],
}
newmacs.add(mac)
r, _, _ = select.select((net, net4), (), (), 0.2)
for mac in newmacs:
peerbymacaddress[mac]['xid'] = 1
_add_attributes(peerbymacaddress[mac])
peerbymacaddress[mac]['hwaddr'] = mac
peerbymacaddress[mac]['protocol'] = protocol
for srvurl in peerbymacaddress[mac].get('urls', ()):
if len(srvurl) > 4:
srvurl = srvurl[:-3]
if srvurl.endswith('://Athena:'):
continue
if 'service:ipmi' in peerbymacaddress[mac]['services']:
continue
if 'service:lightttpd' in peerbymacaddress[mac]['services']:
currinf = peerbymacaddress[mac]
curratt = currinf.get('attributes', {})
if curratt.get('System-Manufacturing',
[None])[0] == 'Lenovo' and curratt.get(
'type',
[None])[0] == 'LenovoThinkServer':
peerbymacaddress[mac]['services'] = [
'service:lenovo-tsm'
]
else:
continue
handler(peerbymacaddress[mac])
except Exception as e:
tracelog.log(traceback.format_exc(),
ltype=log.DataTypes.event,
event=log.Events.stacktrace)
def test_socketpair_select(self):
# https://github.com/eventlet/eventlet/pull/25
s1, s2 = socket.socketpair()
assert select.select([], [s1], [], 0) == ([], [s1], [])
assert select.select([], [s1], [], 0) == ([], [s1], [])
def test_socketpair_select(self):
# https://github.com/eventlet/eventlet/pull/25
s1, s2 = socket.socketpair()
assert select.select([], [s1], [], 0) == ([], [s1], [])
assert select.select([], [s1], [], 0) == ([], [s1], [])
def receive(sock, n, timeout=5):
r, w, x = select.select([sock], [], [], timeout)
if sock in r:
return sock.recv(n)
else:
raise RuntimeError, "timed out on %r" % (sock,)
def select(self, *args, **kwargs):
with _yield_before_after():
return green_select.select(*args, **kwargs)
def snoop(handler, byehandler=None):
"""Watch for SSDP notify messages
The handler shall be called on any service coming online.
byehandler is called whenever a system advertises that it is departing.
If no byehandler is specified, byebye messages are ignored. The handler is
given (as possible), the mac address, a list of viable sockaddrs to reference
the peer, and the notification type (e.g.
'urn:dmtf-org:service:redfish-rest:1'
:param handler: A handler for online notifications from network
:param byehandler: Optional handler for devices going off the network
"""
# Normally, I like using v6/v4 agnostic socket. However, since we are
# dabbling in multicast wizardry here, such sockets can cause big problems,
# so we will have two distinct sockets
known_peers = set([])
net6 = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
net6.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)
for ifidx in util.list_interface_indexes():
v6grp = ssdp6mcast + struct.pack('=I', ifidx)
net6.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_JOIN_GROUP, v6grp)
net6.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
net4 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
for i4 in util.list_ips():
ssdp4mcast = socket.inet_pton(socket.AF_INET, mcastv4addr) + \
socket.inet_aton(i4['addr'])
net4.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP,
ssdp4mcast)
net4.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
net4.bind(('', 1900))
net6.bind(('', 1900))
peerbymacaddress = {}
while True:
newmacs = set([])
machandlers = {}
r, _, _ = select.select((net4, net6), (), (), 60)
neighutil.update_neigh()
while r:
for s in r:
(rsp, peer) = s.recvfrom(9000)
rsp = rsp.split('\r\n')
method, _, _ = rsp[0].split(' ', 2)
if method == 'NOTIFY':
ip = peer[0].partition('%')[0]
if ip not in neighutil.neightable:
continue
if peer in known_peers:
continue
mac = neighutil.neightable[ip]
known_peers.add(peer)
newmacs.add(mac)
if mac in peerbymacaddress:
peerbymacaddress[mac]['peers'].append(peer)
else:
peerbymacaddress[mac] = {
'hwaddr': mac,
'peers': [peer],
}
peerdata = peerbymacaddress[mac]
for headline in rsp[1:]:
if not headline:
continue
header, _, value = headline.partition(':')
header = header.strip()
value = value.strip()
if header == 'NT':
peerdata['service'] = value
elif header == 'NTS':
if value == 'ssdp:byebye':
machandlers[mac] = byehandler
elif value == 'ssdp:alive':
machandlers[mac] = handler
r, _, _ = select.select((net4, net6), (), (), 0.1)
for mac in newmacs:
thehandler = machandlers.get(mac, None)
if thehandler:
thehandler(peerbymacaddress[mac])
def retrieve_output(self, username, hue_instance_id, shell_pairs):
"""
Called when an output request is received from the client. Sends the request to the appropriate
shell instances.
"""
time_received = time.time()
current_greenlet = eventlet.getcurrent()
self._greenlets_by_hid[hue_instance_id] = current_greenlet
shell_pairs = set(shell_pairs)
# Update the time stamps on all shells
self._update_access_time(username,
time_received,
[ p[0] for p in shell_pairs ])
result = None
# The main long-polling loop
while (time.time() - time_received) < constants.BROWSER_REQUEST_TIMEOUT:
# If we have cached output, find that and return immediately
cached_output = self._retrieve_cached_output(username, shell_pairs)
if len(cached_output) != 0:
return cached_output
fds_to_listen_for = []
shell_instances_for_listened_fds = {}
#
# Figure out which shell we should select on.
#
# Note that only one greenlet (request handler) may select on a given
# shell. So we build a registration mechanism with _hids_by_pid.
# If somebody else is already doing a select, then we add ourselves to
# _greenlets_to_notify.
#
# Each hid is generated by the frontend uniquely. It safely maps to a
# unique greenlet.
#
for shell_id, _ in shell_pairs:
shell_instance = self._shells.get((username, shell_id))
# Here we can assume shell_instance exists because if it didn't, we would have broken out of
# the while loop above and we wouldn't be executing this code.
listening_hid = self._hids_by_pid.get(shell_instance.pid)
if listening_hid is not None and listening_hid != hue_instance_id:
self._greenlets_to_notify.setdefault(shell_instance.pid, set()).add(current_greenlet)
else:
fds_to_listen_for.append(shell_instance._fd)
shell_instances_for_listened_fds[shell_instance._fd] = shell_instance
self._hids_by_pid[shell_instance.pid] = hue_instance_id
try:
time_remaining = constants.BROWSER_REQUEST_TIMEOUT - (time.time() - time_received)
self._greenlet_interruptable[current_greenlet] = True
readable, writable, exception_occurred = select.select(fds_to_listen_for, [], [], time_remaining)
self._greenlet_interruptable[current_greenlet] = False
except NewShellInterrupt, nsi:
self._greenlet_interruptable[current_greenlet] = False
# Here, I'm assuming that we won't have a situation where one of the (shell_id, offset)
# tuples in nsi.new_shell_pairs has the same shell_id as an item in shell_pairs, but
# an offset with a different (has to be higher) number.
shell_pairs.update(nsi.new_shell_pairs)
else:
if not readable:
result = { constants.PERIODIC_RESPONSE: True }
else:
result = {}
for fd in readable:
shell_instance = shell_instances_for_listened_fds[fd]
if shell_instance.destroyed:
result[shell_instance.shell_id] = { constants.SHELL_KILLED : True }
else:
result[shell_instance.shell_id] = self._read_helper(shell_instance)
eventlet.spawn_n(self._interrupt_with_output, readable)
break
def direct_transfer_select(fd_in, off_in, fd_out, off_out, length):
LOG.debug("Transfering %s bytes from %s to %s" % (length, fd_in, fd_out))
(pipe_r, pipe_w) = os.pipe()
pipe_r = os.fdopen(pipe_r)
pipe_w = os.fdopen(pipe_w,'w')
fd_pipe_r = pipe_r.fileno()
fd_pipe_w = pipe_w.fileno()
LOG.debug(" source(%s) -> pipe(%s, %s) -> dest(%s)" % (fd_in, fd_pipe_w, fd_pipe_r, fd_out))
set_nonblock(fd_in)
set_nonblock(fd_out)
set_nonblock(fd_pipe_r)
set_nonblock(fd_pipe_w)
flags = SPLICE_F_MOVE | SPLICE_F_MORE | SPLICE_F_NONBLOCK
epoll = select.epoll() # TODO: find a proper way to check if the fd's are files
fd_in_is_file = False
fd_out_is_file = False
try:
epoll.register(fd_in, select.EPOLLIN)
except:
fd_in_is_file = True
try:
epoll.register(fd_out, select.EPOLLOUT)
except:
fd_out_is_file = True
if fd_in_is_file:
readers = [fd_pipe_r]
else:
readers = [fd_in, fd_pipe_r]
if fd_out_is_file:
writers = [fd_pipe_w]
else:
writers = [fd_pipe_w, fd_out]
try:
towrite0 = length
towrite1 = length
ready = []
while towrite0 > 0 or towrite1 > 0:
readable_set, writable_set, _ = green_select.select(readers, writers, [])
for x in readable_set:
ready.append(x)
readers.remove(x)
for x in writable_set:
ready.append(x)
writers.remove(x)
# two transfer options
if (fd_in_is_file or fd_in in ready) and (fd_pipe_w in ready):
# transfer from source to pipe
try:
done = splice.splice(fd_in, None, fd_pipe_w, None, towrite0, flags)
LOG.debug('> Source -> pipe %s bytes' % done)
towrite0 -= done
except IOError as ioex:
if ioex.errno in [errno.EAGAIN, errno.EWOULDBLOCK]: continue
else: raise
# reset
if not fd_in_is_file:
ready.remove(fd_in)
readers.append(fd_in)
ready.remove(fd_pipe_w)
writers.append(fd_pipe_w)
if (fd_pipe_r in ready) and (fd_out_is_file or fd_out in ready):
# transfer from pipe to destination
try:
done = splice.splice(fd_pipe_r, None, fd_out, None, towrite1, flags)
LOG.debug('> pipe -> dest %s bytes' % done)
towrite1 -= done
except IOError as ioex:
if ioex.errno in [errno.EAGAIN, errno.EWOULDBLOCK]: continue
else: raise
# reset
if not fd_out_is_file:
ready.remove(fd_out)
writers.append(fd_out)
ready.remove(fd_pipe_r)
readers.append(fd_pipe_r)
except Exception as ex:
traceback.print_exc()
raise
def run(self):
self._evt.wait()
while True:
r, w, x = select.select(self.servers, [], [])
for s in r:
s.recv(0)
def once():
try:
select.select([-1], [], [])
assert False, 'Expected ValueError'
except ValueError:
pass
def snoop(handler, protocol=None):
#TODO(jjohnson2): ipv6 socket and multicast for DHCPv6, should that be
#prominent
#TODO(jjohnson2): enable unicast replies. This would suggest either
# injection into the neigh table before OFFER or using SOCK_RAW.
global attribwatcher
cfg = cfm.ConfigManager(None)
remap_nodes(cfg.list_nodes(), cfg)
attribwatcher = cfg.watch_attributes(cfg.list_nodes(),
('id.uuid', 'net.*hwaddr'),
remap_nodes)
cfg.watch_nodecollection(new_nodes)
net4 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
net4.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
net4.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
net4.setsockopt(socket.IPPROTO_IP, IP_PKTINFO, 1)
net4.bind(('', 67))
while True:
# Just need some delay, picked a prime number so that overlap with other
# timers might be reduced, though it really is probably nothing
ready = select.select([net4], [], [], None)
if not ready or not ready[0]:
continue
clientaddr = sockaddr_in()
rawbuffer = bytearray(2048)
data = pkttype.from_buffer(rawbuffer)
msg = msghdr()
cmsgarr = bytearray(cmsgsize)
cmsg = cmsgtype.from_buffer(cmsgarr)
iov = iovec()
iov.iov_base = ctypes.addressof(data)
iov.iov_len = 2048
msg.msg_iov = ctypes.pointer(iov)
msg.msg_iovlen = 1
msg.msg_control = ctypes.addressof(cmsg)
msg.msg_controllen = ctypes.sizeof(cmsg)
msg.msg_name = ctypes.addressof(clientaddr)
msg.msg_namelen = ctypes.sizeof(clientaddr)
# We'll leave name and namelen blank for now
i = recvmsg(net4.fileno(), ctypes.pointer(msg), 0)
# if we have a small packet, just skip, it can't possible hold enough
# data and avoids some downstream IndexErrors that would be messy
# with try/except
if i < 64:
continue
#peer = ipfromint(clientaddr.sin_addr.s_addr)
# We don't need peer yet, generally it's 0.0.0.0
_, level, typ = struct.unpack('QII', cmsgarr[:16])
if level == socket.IPPROTO_IP and typ == IP_PKTINFO:
idx, recv, targ = struct.unpack('III', cmsgarr[16:28])
recv = ipfromint(recv)
targ = ipfromint(targ)
# peer is the source ip (in dhcpdiscover, 0.0.0.0)
# recv is the 'ip' that recevied the packet, regardless of target
# targ is the ip in the destination ip of the header.
# idx is the ip link number of the receiving nic
# For example, a DHCPDISCOVER will probably have:
# peer of 0.0.0.0
# targ of 255.255.255.255
# recv of <actual ip address that could reply>
# idx correlated to the nic
rqv = memoryview(rawbuffer)
rq = bytearray(rqv[:i])
if rq[0] == 1: # Boot request
addrlen = rq[2]
if addrlen > 16 or addrlen == 0:
continue
rawnetaddr = rq[28:28 + addrlen]
netaddr = ':'.join(['{0:02x}'.format(x) for x in rawnetaddr])
optidx = 0
try:
optidx = rq.index(b'\x63\x82\x53\x63') + 4
except ValueError:
continue
txid = rq[4:8] # struct.unpack('!I', rq[4:8])[0]
rqinfo, disco = opts_to_dict(rq, optidx)
vivso = disco.get('vivso', None)
if vivso:
# info['modelnumber'] = info['attributes']['enclosure-machinetype-model'][0]
info = {
'hwaddr': netaddr,
'uuid': disco['uuid'],
'architecture': vivso.get('arch', ''),
'services': (vivso['service-type'], ),
'netinfo': {
'ifidx': idx,
'recvip': recv,
'txid': txid
},
'attributes': {
'enclosure-machinetype-model':
[vivso.get('machine', '')]
}
}
handler(info)
#consider_discover(info, rqinfo, net4, cfg, rqv)
continue
# We will fill out service to have something to byte into,
# but the nature of the beast is that we do not have peers,
# so that will not be present for a pxe snoop
info = {
'hwaddr': netaddr,
'uuid': disco['uuid'],
'architecture': disco['arch'],
'netinfo': {
'ifidx': idx,
'recvip': recv,
'txid': txid
},
'services': ('pxe-client', )
}
if disco['uuid']: #TODO(jjohnson2): need to explictly check for
# discover, so that the parser can go ahead and
# parse the options including uuid to enable
# ACK
handler(info)
consider_discover(info, rqinfo, net4, cfg, rqv)
def select(self, *args, **kwargs):
with _yield_before_after():
return green_select.select(*args, **kwargs)
def _find_service(service, target):
net4 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
net6 = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
net6.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)
if target:
addrs = socket.getaddrinfo(target, 1900, 0, socket.SOCK_DGRAM)
for addr in addrs:
host = addr[4][0]
if addr[0] == socket.AF_INET:
msg = smsg.format(host, service)
if not isinstance(msg, bytes):
msg = msg.encode('utf8')
net4.sendto(msg, addr[4])
elif addr[0] == socket.AF_INET6:
host = '[{0}]'.format(host)
msg = smsg.format(host, service)
if not isinstance(msg, bytes):
msg = msg.encode('utf8')
net6.sendto(msg, addr[4])
else:
net4.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
for idx in util.list_interface_indexes():
net6.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_IF, idx)
try:
msg = smsg.format('[{0}]'.format(mcastv6addr), service)
if not isinstance(msg, bytes):
msg = msg.encode('utf8')
net6.sendto(msg, (mcastv6addr, 1900, 0, 0))
except socket.error:
# ignore interfaces without ipv6 multicast causing error
pass
for i4 in util.list_ips():
if 'broadcast' not in i4:
continue
addr = i4['addr']
bcast = i4['broadcast']
net4.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_IF,
socket.inet_aton(addr))
msg = smsg.format(mcastv4addr, service)
if not isinstance(msg, bytes):
msg = msg.encode('utf8')
net4.sendto(msg, (mcastv4addr, 1900))
msg = smsg.format(bcast, service)
if not isinstance(msg, bytes):
msg = msg.encode('utf8')
net4.sendto(msg, (bcast, 1900))
# SSDP by spec encourages responses to spread out over a 3 second interval
# hence we must be a bit more patient
deadline = util.monotonic_time() + 4
r, _, _ = select.select((net4, net6), (), (), 4)
peerdata = {}
while r:
for s in r:
(rsp, peer) = s.recvfrom(9000)
neighutil.refresh_neigh()
_parse_ssdp(peer, rsp, peerdata)
timeout = deadline - util.monotonic_time()
if timeout < 0:
timeout = 0
r, _, _ = select.select((net4, net6), (), (), timeout)
querypool = gp.GreenPool()
pooltargs = []
for nid in peerdata:
for url in peerdata[nid].get('urls', ()):
if url.endswith('/desc.tmpl'):
pooltargs.append((url, peerdata[nid]))
for pi in querypool.imap(check_cpstorage, pooltargs):
if pi is not None:
yield pi
def once():
try:
select.select([-1], [], [])
assert False, 'Expected ValueError'
except ValueError:
pass
def is_connected(skt):
try:
fno = skt.fileno()
except socket.error as e:
if e[0] == errno.EBADF:
return False
raise
try:
if hasattr(select, "epoll"):
ep = select.epoll()
ep.register(fno, select.EPOLLOUT | select.EPOLLIN)
events = ep.poll(0)
for fd, ev in events:
if fno == fd and \
(ev & select.EPOLLOUT or ev & select.EPOLLIN):
ep.unregister(fno)
return True
ep.unregister(fno)
elif hasattr(select, "poll"):
p = select.poll()
p.register(fno, select.POLLOUT | select.POLLIN)
events = p.poll(0)
for fd, ev in events:
if fno == fd and \
(ev & select.POLLOUT or ev & select.POLLIN):
p.unregister(fno)
return True
p.unregister(fno)
elif can_use_kqueue():
kq = select.kqueue()
events = [
select.kevent(fno, select.KQ_FILTER_READ, select.KQ_EV_ADD),
select.kevent(fno, select.KQ_FILTER_WRITE, select.KQ_EV_ADD)
]
kq.control(events, 0)
kevents = kq.control(None, 4, 0)
for ev in kevents:
if ev.ident == fno:
if ev.flags & select.KQ_EV_ERROR:
return False
else:
return True
# delete
events = [
select.kevent(fno, select.KQ_FILTER_READ, select.KQ_EV_DELETE),
select.kevent(fno, select.KQ_FILTER_WRITE, select.KQ_EV_DELETE)
]
kq.control(events, 0)
kq.close()
return True
else:
r, _, _ = select.select([fno], [], [], 0)
if not r:
return True
except IOError:
pass
except (ValueError, select.error,) as e:
pass
return False
def retrieve_output(self, username, hue_instance_id, shell_pairs):
"""
Called when an output request is received from the client. Sends the request to the appropriate
shell instances.
"""
time_received = time.time()
current_greenlet = eventlet.getcurrent()
self._greenlets_by_hid[hue_instance_id] = current_greenlet
shell_pairs = set(shell_pairs)
# Update the time stamps on all shells
self._update_access_time(username,
time_received,
[ p[0] for p in shell_pairs ])
result = None
# The main long-polling loop
while (time.time() - time_received) < constants.BROWSER_REQUEST_TIMEOUT:
# If we have cached output, find that and return immediately
cached_output = self._retrieve_cached_output(username, shell_pairs)
if len(cached_output) != 0:
return cached_output
fds_to_listen_for = []
shell_instances_for_listened_fds = {}
#
# Figure out which shell we should select on.
#
# Note that only one greenlet (request handler) may select on a given
# shell. So we build a registration mechanism with _hids_by_pid.
# If somebody else is already doing a select, then we add ourselves to
# _greenlets_to_notify.
#
# Each hid is generated by the frontend uniquely. It safely maps to a
# unique greenlet.
#
for shell_id, _ in shell_pairs:
shell_instance = self._shells.get((username, shell_id))
# Here we can assume shell_instance exists because if it didn't, we would have broken out of
# the while loop above and we wouldn't be executing this code.
listening_hid = self._hids_by_pid.get(shell_instance.pid)
if listening_hid is not None and listening_hid != hue_instance_id:
self._greenlets_to_notify.setdefault(shell_instance.pid, set()).add(current_greenlet)
else:
fds_to_listen_for.append(shell_instance._fd)
shell_instances_for_listened_fds[shell_instance._fd] = shell_instance
self._hids_by_pid[shell_instance.pid] = hue_instance_id
try:
time_remaining = constants.BROWSER_REQUEST_TIMEOUT - (time.time() - time_received)
self._greenlet_interruptable[current_greenlet] = True
readable, writable, exception_occurred = select.select(fds_to_listen_for, [], [], time_remaining)
self._greenlet_interruptable[current_greenlet] = False
except NewShellInterrupt, nsi:
self._greenlet_interruptable[current_greenlet] = False
# Here, I'm assuming that we won't have a situation where one of the (shell_id, offset)
# tuples in nsi.new_shell_pairs has the same shell_id as an item in shell_pairs, but
# an offset with a different (has to be higher) number.
shell_pairs.update(nsi.new_shell_pairs)
else:
if not readable:
result = { constants.PERIODIC_RESPONSE: True }
else:
result = {}
for fd in readable:
shell_instance = shell_instances_for_listened_fds[fd]
if shell_instance.destroyed:
result[shell_instance.shell_id] = { constants.SHELL_KILLED : True }
else:
result[shell_instance.shell_id] = self._read_helper(shell_instance)
eventlet.spawn_n(self._interrupt_with_output, readable)
break
def run_cgi(self):
"""Execute a CGI script."""
dir, rest = self.cgi_info
path = dir + '/' + rest
i = path.find('/', len(dir)+1)
while i >= 0:
nextdir = path[:i]
nextrest = path[i+1:]
scriptdir = self.translate_path(nextdir)
if os.path.isdir(scriptdir):
dir, rest = nextdir, nextrest
i = path.find('/', len(dir)+1)
else:
break
# find an explicit query string, if present.
rest, _, query = rest.partition('?')
# dissect the part after the directory name into a script name &
# a possible additional path, to be stored in PATH_INFO.
i = rest.find('/')
if i >= 0:
script, rest = rest[:i], rest[i:]
else:
script, rest = rest, ''
scriptname = dir + '/' + script
scriptfile = self.translate_path(scriptname)
if not os.path.exists(scriptfile):
self.send_error(
HTTPStatus.NOT_FOUND,
"No such CGI script (%r)" % scriptname)
return
if not os.path.isfile(scriptfile):
self.send_error(
HTTPStatus.FORBIDDEN,
"CGI script is not a plain file (%r)" % scriptname)
return
ispy = self.is_python(scriptname)
if self.have_fork or not ispy:
if not self.is_executable(scriptfile):
self.send_error(
HTTPStatus.FORBIDDEN,
"CGI script is not executable (%r)" % scriptname)
return
# Reference: http://hoohoo.ncsa.uiuc.edu/cgi/env.html
# XXX Much of the following could be prepared ahead of time!
env = copy.deepcopy(os.environ)
env['SERVER_SOFTWARE'] = self.version_string()
env['SERVER_NAME'] = self.server.server_name
env['GATEWAY_INTERFACE'] = 'CGI/1.1'
env['SERVER_PROTOCOL'] = self.protocol_version
env['SERVER_PORT'] = str(self.server.server_port)
env['REQUEST_METHOD'] = self.command
uqrest = urllib.parse.unquote(rest)
env['PATH_INFO'] = uqrest
env['PATH_TRANSLATED'] = self.translate_path(uqrest)
env['SCRIPT_NAME'] = scriptname
if query:
env['QUERY_STRING'] = query
env['REMOTE_ADDR'] = self.client_address[0]
authorization = self.headers.get("authorization")
if authorization:
authorization = authorization.split()
if len(authorization) == 2:
import base64, binascii
env['AUTH_TYPE'] = authorization[0]
if authorization[0].lower() == "basic":
try:
authorization = authorization[1].encode('ascii')
authorization = base64.decodebytes(authorization).\
decode('ascii')
except (binascii.Error, UnicodeError):
pass
else:
authorization = authorization.split(':')
if len(authorization) == 2:
env['REMOTE_USER'] = authorization[0]
# XXX REMOTE_IDENT
if self.headers.get('content-type') is None:
env['CONTENT_TYPE'] = self.headers.get_content_type()
else:
env['CONTENT_TYPE'] = self.headers['content-type']
length = self.headers.get('content-length')
if length:
env['CONTENT_LENGTH'] = length
referer = self.headers.get('referer')
if referer:
env['HTTP_REFERER'] = referer
accept = []
for line in self.headers.getallmatchingheaders('accept'):
if line[:1] in "\t\n\r ":
accept.append(line.strip())
else:
accept = accept + line[7:].split(',')
env['HTTP_ACCEPT'] = ','.join(accept)
ua = self.headers.get('user-agent')
if ua:
env['HTTP_USER_AGENT'] = ua
co = filter(None, self.headers.get_all('cookie', []))
cookie_str = ', '.join(co)
if cookie_str:
env['HTTP_COOKIE'] = cookie_str
# XXX Other HTTP_* headers
# Since we're setting the env in the parent, provide empty
# values to override previously set values
for k in ('QUERY_STRING', 'REMOTE_HOST', 'CONTENT_LENGTH',
'HTTP_USER_AGENT', 'HTTP_COOKIE', 'HTTP_REFERER'):
env.setdefault(k, "")
self.send_response(HTTPStatus.OK, "Script output follows")
self.flush_headers()
decoded_query = query.replace('+', ' ')
if self.have_fork:
# Unix -- fork as we should
args = [script]
if '=' not in decoded_query:
args.append(decoded_query)
nobody = nobody_uid()
self.wfile.flush() # Always flush before forking
pid = os.fork()
if pid != 0:
# Parent
pid, sts = os.waitpid(pid, 0)
# throw away additional data [see bug #427345]
while select.select([self.rfile], [], [], 0)[0]:
if not self.rfile.read(1):
break
if sts:
self.log_error("CGI script exit status %#x", sts)
return
# Child
try:
try:
os.setuid(nobody)
except OSError:
pass
os.dup2(self.rfile.fileno(), 0)
os.dup2(self.wfile.fileno(), 1)
os.execve(scriptfile, args, env)
except:
self.server.handle_error(self.request, self.client_address)
os._exit(127)
else:
# Non-Unix -- use subprocess
cmdline = [scriptfile]
if self.is_python(scriptfile):
interp = sys.executable
if interp.lower().endswith("w.exe"):
# On Windows, use python.exe, not pythonw.exe
interp = interp[:-5] + interp[-4:]
cmdline = [interp, '-u'] + cmdline
if '=' not in query:
cmdline.append(query)
self.log_message("command: %s", subprocess.list2cmdline(cmdline))
try:
nbytes = int(length)
except (TypeError, ValueError):
nbytes = 0
p = subprocess.Popen(cmdline,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env = env
)
if self.command.lower() == "post" and nbytes > 0:
data = self.rfile.read(nbytes)
else:
data = None
# throw away additional data [see bug #427345]
while select.select([self.rfile._sock], [], [], 0)[0]:
if not self.rfile._sock.recv(1):
break
stdout, stderr = p.communicate(data)
self.wfile.write(stdout)
if stderr:
self.log_error('%s', stderr)
p.stderr.close()
p.stdout.close()
status = p.returncode
if status:
self.log_error("CGI script exit status %#x", status)
else:
self.log_message("CGI script exited OK")
def run_cgi(self):
"""Execute a CGI script."""
dir, rest = self.cgi_info
path = dir + '/' + rest
i = path.find('/', len(dir) + 1)
while i >= 0:
nextdir = path[:i]
nextrest = path[i + 1:]
scriptdir = self.translate_path(nextdir)
if os.path.isdir(scriptdir):
dir, rest = nextdir, nextrest
i = path.find('/', len(dir) + 1)
else:
break
# find an explicit query string, if present.
rest, _, query = rest.partition('?')
# dissect the part after the directory name into a script name &
# a possible additional path, to be stored in PATH_INFO.
i = rest.find('/')
if i >= 0:
script, rest = rest[:i], rest[i:]
else:
script, rest = rest, ''
scriptname = dir + '/' + script
scriptfile = self.translate_path(scriptname)
if not os.path.exists(scriptfile):
self.send_error(HTTPStatus.NOT_FOUND,
"No such CGI script (%r)" % scriptname)
return
if not os.path.isfile(scriptfile):
self.send_error(HTTPStatus.FORBIDDEN,
"CGI script is not a plain file (%r)" % scriptname)
return
ispy = self.is_python(scriptname)
if self.have_fork or not ispy:
if not self.is_executable(scriptfile):
self.send_error(
HTTPStatus.FORBIDDEN,
"CGI script is not executable (%r)" % scriptname)
return
# Reference: http://hoohoo.ncsa.uiuc.edu/cgi/env.html
# XXX Much of the following could be prepared ahead of time!
env = copy.deepcopy(os.environ)
env['SERVER_SOFTWARE'] = self.version_string()
env['SERVER_NAME'] = self.server.server_name
env['GATEWAY_INTERFACE'] = 'CGI/1.1'
env['SERVER_PROTOCOL'] = self.protocol_version
env['SERVER_PORT'] = str(self.server.server_port)
env['REQUEST_METHOD'] = self.command
uqrest = urllib.parse.unquote(rest)
env['PATH_INFO'] = uqrest
env['PATH_TRANSLATED'] = self.translate_path(uqrest)
env['SCRIPT_NAME'] = scriptname
if query:
env['QUERY_STRING'] = query
env['REMOTE_ADDR'] = self.client_address[0]
authorization = self.headers.get("authorization")
if authorization:
authorization = authorization.split()
if len(authorization) == 2:
import base64, binascii
env['AUTH_TYPE'] = authorization[0]
if authorization[0].lower() == "basic":
try:
authorization = authorization[1].encode('ascii')
authorization = base64.decodebytes(authorization).\
decode('ascii')
except (binascii.Error, UnicodeError):
pass
else:
authorization = authorization.split(':')
if len(authorization) == 2:
env['REMOTE_USER'] = authorization[0]
# XXX REMOTE_IDENT
if self.headers.get('content-type') is None:
env['CONTENT_TYPE'] = self.headers.get_content_type()
else:
env['CONTENT_TYPE'] = self.headers['content-type']
length = self.headers.get('content-length')
if length:
env['CONTENT_LENGTH'] = length
referer = self.headers.get('referer')
if referer:
env['HTTP_REFERER'] = referer
accept = []
for line in self.headers.getallmatchingheaders('accept'):
if line[:1] in "\t\n\r ":
accept.append(line.strip())
else:
accept = accept + line[7:].split(',')
env['HTTP_ACCEPT'] = ','.join(accept)
ua = self.headers.get('user-agent')
if ua:
env['HTTP_USER_AGENT'] = ua
co = filter(None, self.headers.get_all('cookie', []))
cookie_str = ', '.join(co)
if cookie_str:
env['HTTP_COOKIE'] = cookie_str
# XXX Other HTTP_* headers
# Since we're setting the env in the parent, provide empty
# values to override previously set values
for k in ('QUERY_STRING', 'REMOTE_HOST', 'CONTENT_LENGTH',
'HTTP_USER_AGENT', 'HTTP_COOKIE', 'HTTP_REFERER'):
env.setdefault(k, "")
self.send_response(HTTPStatus.OK, "Script output follows")
self.flush_headers()
decoded_query = query.replace('+', ' ')
if self.have_fork:
# Unix -- fork as we should
args = [script]
if '=' not in decoded_query:
args.append(decoded_query)
nobody = nobody_uid()
self.wfile.flush() # Always flush before forking
pid = os.fork()
if pid != 0:
# Parent
pid, sts = os.waitpid(pid, 0)
# throw away additional data [see bug #427345]
while select.select([self.rfile], [], [], 0)[0]:
if not self.rfile.read(1):
break
if sts:
self.log_error("CGI script exit status %#x", sts)
return
# Child
try:
try:
os.setuid(nobody)
except OSError:
pass
os.dup2(self.rfile.fileno(), 0)
os.dup2(self.wfile.fileno(), 1)
os.execve(scriptfile, args, env)
except:
self.server.handle_error(self.request, self.client_address)
os._exit(127)
else:
# Non-Unix -- use subprocess
cmdline = [scriptfile]
if self.is_python(scriptfile):
interp = sys.executable
if interp.lower().endswith("w.exe"):
# On Windows, use python.exe, not pythonw.exe
interp = interp[:-5] + interp[-4:]
cmdline = [interp, '-u'] + cmdline
if '=' not in query:
cmdline.append(query)
self.log_message("command: %s", subprocess.list2cmdline(cmdline))
try:
nbytes = int(length)
except (TypeError, ValueError):
nbytes = 0
p = subprocess.Popen(cmdline,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env)
if self.command.lower() == "post" and nbytes > 0:
data = self.rfile.read(nbytes)
else:
data = None
# throw away additional data [see bug #427345]
while select.select([self.rfile._sock], [], [], 0)[0]:
if not self.rfile._sock.recv(1):
break
stdout, stderr = p.communicate(data)
self.wfile.write(stdout)
if stderr:
self.log_error('%s', stderr)
p.stderr.close()
p.stdout.close()
status = p.returncode
if status:
self.log_error("CGI script exit status %#x", status)
else:
self.log_message("CGI script exited OK")
def snoop(handler, byehandler=None, protocol=None, uuidlookup=None):
"""Watch for SSDP notify messages
The handler shall be called on any service coming online.
byehandler is called whenever a system advertises that it is departing.
If no byehandler is specified, byebye messages are ignored. The handler is
given (as possible), the mac address, a list of viable sockaddrs to reference
the peer, and the notification type (e.g.
'urn:dmtf-org:service:redfish-rest:1'
:param handler: A handler for online notifications from network
:param byehandler: Optional handler for devices going off the network
"""
# Normally, I like using v6/v4 agnostic socket. However, since we are
# dabbling in multicast wizardry here, such sockets can cause big problems,
# so we will have two distinct sockets
tracelog = log.Logger('trace')
known_peers = set([])
net6 = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
net6.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)
for ifidx in util.list_interface_indexes():
v6grp = ssdp6mcast + struct.pack('=I', ifidx)
net6.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_JOIN_GROUP, v6grp)
net6.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
net4 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
for i4 in util.list_ips():
ssdp4mcast = socket.inet_pton(socket.AF_INET, mcastv4addr) + \
socket.inet_aton(i4['addr'])
try:
net4.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP,
ssdp4mcast)
except socket.error as e:
if e.errno != 98:
# errno 98 can happen if aliased, skip for now
raise
net4.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
net4.bind(('', 1900))
net6.bind(('', 1900))
peerbymacaddress = {}
while True:
try:
newmacs = set([])
machandlers = {}
r, _, _ = select.select((net4, net6), (), (), 60)
while r:
for s in r:
(rsp, peer) = s.recvfrom(9000)
if rsp[:4] == b'PING':
continue
rsp = rsp.split(b'\r\n')
method, _, _ = rsp[0].split(b' ', 2)
if method == b'NOTIFY':
ip = peer[0].partition('%')[0]
if peer in known_peers:
continue
if ip not in neighutil.neightable:
neighutil.update_neigh()
if ip not in neighutil.neightable:
continue
mac = neighutil.neightable[ip]
known_peers.add(peer)
newmacs.add(mac)
if mac in peerbymacaddress:
peerbymacaddress[mac]['addresses'].append(peer)
else:
peerbymacaddress[mac] = {
'hwaddr': mac,
'addresses': [peer],
}
peerdata = peerbymacaddress[mac]
for headline in rsp[1:]:
if not headline:
continue
headline = util.stringify(headline)
header, _, value = headline.partition(':')
header = header.strip()
value = value.strip()
if header == 'NT':
peerdata['service'] = value
elif header == 'NTS':
if value == 'ssdp:byebye':
machandlers[mac] = byehandler
elif value == 'ssdp:alive':
machandlers[mac] = None # handler
elif method == b'M-SEARCH':
if not uuidlookup:
continue
#ip = peer[0].partition('%')[0]
for headline in rsp[1:]:
if not headline:
continue
headline = util.stringify(headline)
headline = headline.partition(':')
if len(headline) < 3:
continue
if headline[0] == 'ST' and headline[-1].startswith(
' urn:xcat.org:service:confluent:'):
try:
cfm.check_quorum()
except Exception:
continue
for query in headline[-1].split('/'):
if query.startswith('uuid='):
curruuid = query.split('=',
1)[1].lower()
node = uuidlookup(curruuid)
if not node:
break
# Do not bother replying to a node that
# we have no deployment activity
# planned for
cfg = cfm.ConfigManager(None)
cfd = cfg.get_node_attributes(
node, [
'deployment.pendingprofile',
'collective.managercandidates'
])
if not cfd.get(node, {}).get(
'deployment.pendingprofile',
{}).get('value', None):
break
candmgrs = cfd.get(node, {}).get(
'collective.managercandidates',
{}).get('value', None)
if candmgrs:
candmgrs = noderange.NodeRange(
candmgrs, cfg).nodes
if collective.get_myname(
) not in candmgrs:
break
currtime = time.time()
seconds = int(currtime)
msecs = int(currtime * 1000 % 1000)
reply = 'HTTP/1.1 200 OK\r\nNODENAME: {0}\r\nCURRTIME: {1}\r\nCURRMSECS: {2}\r\n'.format(
node, seconds, msecs)
if '%' in peer[0]:
iface = peer[0].split('%', 1)[1]
reply += 'MGTIFACE: {0}\r\n'.format(
peer[0].split('%', 1)[1])
ncfg = netutil.get_nic_config(
cfg, node, ifidx=iface)
if ncfg.get(
'matchesnodename', None):
reply += 'DEFAULTNET: 1\r\n'
elif not netutil.address_is_local(
peer[0]):
continue
if not isinstance(reply, bytes):
reply = reply.encode('utf8')
s.sendto(reply, peer)
r, _, _ = select.select((net4, net6), (), (), 0.2)
for mac in newmacs:
thehandler = machandlers.get(mac, None)
if thehandler:
thehandler(peerbymacaddress[mac])
except Exception:
tracelog.log(traceback.format_exc(),
ltype=log.DataTypes.event,
event=log.Events.stacktrace)