def start_monitor(dirs):
last_run = time.time()
files_stats = []
paths = []
current_dir = os.getcwd()
kq = kqueue()
source_events = []
for dir_name in dirs:
dir_path = current_dir + '/' + dir_name
paths.append(dir_path)
fd = os.open(dir_path, os.O_RDONLY)
event = kevent(fd, filter=select.KQ_FILTER_VNODE,
flags=select.KQ_EV_ADD | select.KQ_EV_CLEAR,
fflags=select.KQ_NOTE_WRITE)
source_events.append(event)
while True:
events = kq.control(source_events, len(source_events), 2000)
if any(map(lambda e: e.fflags & select.KQ_NOTE_WRITE, events)):
if (time.time() - last_run) < LIMIT:
continue
if check_modifications(current_dir, paths):
try:
async_test(["make", "test"])
os.system('clear')
subprocess.Popen("neurotic")
except NeuroticError as ne:
os.system('clear')
if b"ERROR" in ne.content[0]:
print(ne.content[0])
else:
subprocess.Popen("neurotic")
last_run = time.time()
def test_create_queue(self):
kq = select.kqueue()
self.assertTrue(kq.fileno() > 0, kq.fileno())
self.assertTrue(not kq.closed)
kq.close()
self.assertTrue(kq.closed)
self.assertRaises(ValueError, kq.fileno)
def _main():
# Bootstrap by adding the root SId
scope_add(ROOT_SID, ROOT_SID, ROOT_SID)
# Open the events file and get its descriptor
# (Note: This file is only read by scoped, not normal appications.)
pubs_file = open("/pubsub/pubs")
fd = pubs_file.fileno()
# Register to events
kq = select.kqueue()
evf = select.kevent(fd, # ident
select.KQ_FILTER_VNODE, # filter
select.KQ_EV_ADD
| select.KQ_EV_CLEAR, # flags
NOTE_PUBLISH, # fflags
0, # data
0) # udata
kq.control([evf], 0, None) # changelist, max_events, timeout
# Listen to events
while True:
evl = kq.control(None, 1, None)
if evl is not None:
# Read events (SId/RId pairs)
pub_evs = pep_get_sidrids(pubs_file)
# Add scopes and/or publications
for sid, rid in pub_evs:
scope_add(ROOT_SID, sid, rid)
def do_work_kqueue(start, end, port, get_hostname):
print "kqueue scanning "+int2ip(start)+" to "+int2ip(end)
connected = []
total = end-start
while (start <= end):
kq = select.kqueue()
fd_map = {}
kevents = []
while (start <= end):
try:
s = open_adb_socket(int2ip(start),port)
except:
break
else:
fd_map[s.fileno()] = (start,s)
start = start+1
kevents.append(select.kevent(s,filter=select.KQ_FILTER_WRITE))
kq.control(kevents,0,0)
time.sleep(0.2)
for k in kq.control([],len(kevents),0.1):
w = fd_map[k.ident][1]
addr = fd_map[w.fileno()][0]
if w.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) == 0:
print 'ADDCON2', k.ident, w.fileno(), int2ip(addr), fd_map[w.fileno()]
connected.append(int2ip(addr))
for fd,t in fd_map.iteritems():
t[1].close()
sys.stdout.write("\r%d/%d" % (total-(end-start),total))
sys.stdout.flush()
print
if get_hostname:
for c in connected:
connect_with_adb(c,port)
def watch_parent():
"""
Thread to watch for the parent pid.
If this process has been orphaned it means middlewared process has crashed
and there is nothing left to do here other than commit suicide!
"""
kqueue = select.kqueue()
try:
kqueue.control([
select.kevent(
os.getppid(),
filter=select.KQ_FILTER_PROC,
flags=select.KQ_EV_ADD,
fflags=select.KQ_NOTE_EXIT,
)
], 0, 0)
except ProcessLookupError:
os._exit(1)
while True:
ppid = os.getppid()
if ppid == 1:
break
kqueue.control(None, 1)
os._exit(1)
def watch_files(filenames):
def _watch_file(kq, filename, flags = select.KQ_EV_ADD | select.KQ_EV_ENABLE | select.KQ_EV_ONESHOT, fflags = select.KQ_NOTE_WRITE | select.KQ_NOTE_DELETE | select.KQ_NOTE_EXTEND | select.KQ_NOTE_RENAME):
fd = os.open(filename, os.O_RDONLY)
event = [select.kevent(fd, filter=select.KQ_FILTER_VNODE, flags=flags, fflags=fflags)]
kq.control(event, 0, 0)
return fd
kq = select.kqueue()
# filedescriptors -> filename
fds = {}
for filename in filenames:
# expand out '~/' nonsense if its their
filename = os.path.expanduser(filename)
# get absolute path if its relative
filename = os.path.abspath(filename)
fds[_watch_file(kq, filename)] = filename
try:
events = kq.control([], 1, None)
finally:
kq.close()
for fd in fds:
os.close(fd)
changed_files = set()
for event in events:
changed_files.add(fds[event.ident])
return changed_files
def __watch(self):
kq = select.kqueue()
passwd_fd = os.open(self.passwd_filename, os.O_RDONLY)
group_fd = os.open(self.group_filename, os.O_RDONLY)
ev = [
select.kevent(
passwd_fd,
filter=select.KQ_FILTER_VNODE, flags=select.KQ_EV_ADD | select.KQ_EV_ENABLE,
fflags=select.KQ_NOTE_WRITE | select.KQ_NOTE_EXTEND | select.KQ_NOTE_RENAME
),
select.kevent(
group_fd,
filter=select.KQ_FILTER_VNODE, flags=select.KQ_EV_ADD | select.KQ_EV_ENABLE,
fflags=select.KQ_NOTE_WRITE | select.KQ_NOTE_EXTEND | select.KQ_NOTE_RENAME
)
]
kq.control(ev, 0)
while True:
event, = kq.control(None, 1)
name = self.passwd_filename if event.ident == passwd_fd else self.group_filename
logger.warning('{0} was modified, reloading'.format(name))
self.__load()
def loop_kqueue(timeout=0.0, map=None):
if map is None:
map = socket_map
if timeout is not None:
# timeout is in milliseconds
timeout = int(timeout*1000)
kq = select.kqueue()
if map:
evs = []
for fd, obj in map.items():
flags = select.KQ_EV_ADD | select.KQ_EV_CLEAR
if obj.readable():
evs.append(select.kevent(fd, filter=select.KQ_FILTER_READ, flags=flags))
if obj.writable():
evs.append(select.kevent(fd, filter=select.KQ_FILTER_WRITE, flags=flags))
r_evs = kq.control(evs, 100, timeout)
for ev in r_evs:
obj = map.get(ev.ident)
if obj is None:
continue
try:
if ev.filter == select.KQ_FILTER_READ:
obj.handle_read_event()
if ev.filter == select.KQ_FILTER_WRITE:
obj.handle_write_event()
except socket.error, e:
if e.args[0] not in (EBADF, ECONNRESET, ENOTCONN, ESHUTDOWN, ECONNABORTED):
obj.handle_error()
else:
obj.handle_close()
except _reraised_exceptions:
raise
except:
async def worker_kill(self, t_worker):
# If connection has been closed lets make sure shell is killed
if t_worker.shell_pid:
try:
kqueue = select.kqueue()
kevent = select.kevent(t_worker.shell_pid, select.KQ_FILTER_PROC, select.KQ_EV_ADD | select.KQ_EV_ENABLE, select.KQ_NOTE_EXIT)
kqueue.control([kevent], 0)
os.kill(t_worker.shell_pid, signal.SIGTERM)
# If process has not died in 2 seconds, try the big gun
events = await self.middleware.run_in_thread(kqueue.control, None, 1, 2)
if not events:
os.kill(t_worker.shell_pid, signal.SIGKILL)
# If process has not died even with the big gun
# There is nothing else we can do, leave it be and
# release the worker thread
events = await self.middleware.run_in_thread(kqueue.control, None, 1, 2)
if not events:
t_worker.die()
except ProcessLookupError:
pass
# Wait thread join in yet another thread to avoid event loop blockage
# There may be a simpler/better way to do this?
await self.middleware.run_in_thread(t_worker.join)
def kqueue_poller(timeout=0.0, map=None):
"""A poller which uses kqueue(), BSD specific."""
if map is None:
map = socket_map
if map:
kqueue = select.kqueue()
flags = select.KQ_EV_ADD | select.KQ_EV_ENABLE
selectables = 0
for fd, obj in map.items():
filter = 0
if obj.readable():
filter |= select.KQ_FILTER_READ
if obj.writable():
filter |= select.KQ_FILTER_WRITE
if filter:
ev = select.kevent(fd, filter=filter, flags=flags)
kqueue.control([ev], 0)
selectables += 1
events = kqueue.control(None, selectables, timeout)
for event in events:
fd = event.ident
obj = map.get(fd)
if obj is None:
continue
if event.filter == select.KQ_FILTER_READ:
read(obj)
if event.filter == select.KQ_FILTER_WRITE:
write(obj)
kqueue.close()
def __init__(self):
self._fd_map = {}
self._handler_map = {}
self._event_map = {}
self.kq = select.kqueue()
self.klist = []
self._stop = False
def wait_for_file_to_exist(self, path, timeout=0):
path_dir = os.path.abspath(os.path.realpath(os.path.dirname(path)))
filename = os.path.basename(path)
fullpath = os.path.join(path_dir, filename)
exists = False
fd = os.open(path_dir, os.O_RDONLY)
kq = select.kqueue()
events = [select.kevent(
fd, filter=select.KQ_FILTER_VNODE,
flags=select.KQ_EV_ADD | select.KQ_EV_ENABLE | select.KQ_EV_CLEAR,
fflags=select.KQ_NOTE_WRITE | select.KQ_NOTE_EXTEND
)]
events = kq.control(events, 0, 0)
while (not exists):
proc_events = kq.control([], 1024)
for event in proc_events:
if ((event.fflags & select.KQ_NOTE_WRITE) or
(event.fflags & select.KQ_NOTE_EXTEND)):
if os.access(fullpath, os.F_OK):
exists = True
if exists is True:
break
kq.close()
os.close(fd)
return exists
def wait_for_file_change(self, path, timeout=0):
changed = False
with open(path, "r") as f:
fd = f.fileno()
kq = select.kqueue()
events = [select.kevent(
fd, filter=select.KQ_FILTER_VNODE,
flags=select.KQ_EV_ADD | select.KQ_EV_ENABLE | select.KQ_EV_CLEAR,
fflags=select.KQ_NOTE_WRITE | select.KQ_NOTE_EXTEND
)]
events = kq.control(events, 0, 0)
while (not changed):
proc_events = kq.control([], 1024)
for event in proc_events:
if ((event.fflags & select.KQ_NOTE_WRITE) or
(event.fflags & select.KQ_NOTE_EXTEND)):
if os.access(path, os.F_OK):
changed = True
if changed is True:
break
kq.close()
return changed
def __init__(self):
Base.__init__(self)
Base._poller = self
if hasattr(select, 'epoll'):
self._epoll = select.epoll()
elif hasattr(select, 'kqueue'):
self._kqueue = select.kqueue()
def __init__(self, *args):
self._event_types = {
select.KQ_FILTER_READ: 'fd_readable',
select.KQ_FILTER_WRITE: 'fd_writable'}
LoopBase.__init__(self, *args)
self.max_ev = 50 # maximum number of events to pull from the queue
self._kq = select.kqueue()
def worker(self):
self.logger.debug("Opened tail stream on file {0} ({1} lines)".format(self.path, self.backlog))
with io.open(self.fd, "wb") as fd:
with open(self.path, "rb") as f:
kq = select.kqueue()
try:
ev = [
select.kevent(
fd.fileno(), filter=select.KQ_FILTER_READ, flags=select.KQ_EV_ADD | select.KQ_EV_ENABLE
),
select.kevent(
f.fileno(), filter=select.KQ_FILTER_READ, flags=select.KQ_EV_ADD | select.KQ_EV_ENABLE
),
]
fd.write(self.tail(f, self.backlog))
fd.flush()
kq.control(ev, 0)
f.seek(0, os.SEEK_END)
while True:
event, = kq.control(None, 1)
self.logger.debug("kqueue event {0}".format(event))
if event.ident == fd.fileno():
if event.flags & select.KQ_EV_EOF or event.flags & select.KQ_EV_ERROR:
break
if event.ident == f.fileno():
fd.write(f.read())
fd.flush()
finally:
kq.close()
def testKqueue(self):
k = select.kqueue()
fd = FD(k)
try:
self.checkSubprocessFDs([fd], close=False)
finally:
del k
def connect(self, connectionInfo, elementListener):
"""
Connect according to the info in connectionInfo, and use
elementListener.
:param UdpTransport.ConnectionInfo connectionInfo: A
UdpTransport.ConnectionInfo.
:param elementListener: The elementListener must remain valid during the
life of this object.
:type elementListener: An object with onReceivedData
"""
self.close()
# Save the _address to use in sendto.
self._address = (connectionInfo.getHost(), connectionInfo.getPort())
self._socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
if hasattr(select, "poll"):
# Set up _poll. (Ubuntu, etc.)
self._poll = select.poll()
self._poll.register(self._socket.fileno(), select.POLLIN)
elif hasattr(select, "kqueue"):
## Set up _kqueue. (BSD and OS X)
self._kqueue = select.kqueue()
self._kevents = [select.kevent(
self._socket.fileno(), filter = select.KQ_FILTER_READ,
flags = select.KQ_EV_ADD | select.KQ_EV_ENABLE |
select.KQ_EV_CLEAR)]
elif not hasattr(select, "select"):
# Most Python implementations have this fallback, so we
# don't expect this error.
raise RuntimeError("Cannot find a polling utility for sockets")
self._elementReader = ElementReader(elementListener)
def main():
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server_socket.bind(('0.0.0.0', 5555))
server_socket.listen(5)
kq = select.kqueue()
kq.control([select.kevent(server_socket.fileno(), select.KQ_FILTER_READ, select.KQ_EV_ADD)], 0, 0)
connections = {}
while True:
events = kq.control([], 10000)
for event in events:
if server_socket.fileno() == event.ident:
(client_socket, client_address) = server_socket.accept()
print 'got connection from', client_address
kq.control([select.kevent(client_socket.fileno(), select.KQ_FILTER_READ, select.KQ_EV_ADD)], 0, 0)
connections[client_socket.fileno()] = client_socket
elif event.filter == select.KQ_FILTER_READ:
client_socket = connections[event.ident]
data = client_socket.recv(4096)
if data:
client_socket.send(data)
else:
kq.control(
[select.kevent(client_socket.fileno(), select.KQ_FILTER_READ, select.KQ_EV_DELETE)], 0, 0)
client_socket.close()
connections.pop(event.ident)
def run(self, *args, **kwargs):
super(pbid, self).run(*args, **kwargs)
"""
pbid --refresh is async and will remove /var/db/pbi/index/*.time,
once the refresh is done the file will exist again.
Wait for the file using kqueue for the sync operation
"""
if self.__sync is True and self.flags & PBID_FLAGS_REFRESH:
fd = os.open(PBID_INDEXDIR, os.O_RDONLY)
evts = [
select.kevent(fd,
filter=select.KQ_FILTER_VNODE,
flags=select.KQ_EV_ADD | select.KQ_EV_CLEAR,
fflags=select.KQ_NOTE_WRITE | select.KQ_NOTE_EXTEND,
)
]
kq = select.kqueue()
kq.control(evts, 0, 0)
timeout = True
for i in xrange(30):
kq.control(None, 2, 1)
if len(glob.glob('%s/*.time' % PBID_INDEXDIR)) > 0:
timeout = False
break
if timeout:
log.debug("pbid.run: sync refresh timed out")
def __init__(self, AutoBot, listenhost, listenport):
Thread.__init__(self)
self.setDaemon(1)
self.AutoBot = AutoBot
self.listenport = listenport
self.accept_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.accept_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
#self.accept_socket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
self.accept_socket.bind((listenhost, listenport))
self.accept_socket.listen(10)
self.accept_socket.setblocking(False)
#self.accept_socket.settimeout(None)
#for bsd
self.kq = select.kqueue()
self.kevent = [
select.kevent(self.accept_socket.fileno(),
filter=select.KQ_FILTER_READ,
flags=select.KQ_EV_ADD | select.KQ_EV_ENABLE)
]
#for linux
#self.epoll = select.epoll()
#self.epoll.register(self.accept_socket.fileno(), select.EPOLLIN)
self.stuff = {}
def start(self):
try:
self._kq = select.kqueue()
kevents = []
for f in self._files:
self._files[f] = fd = os.open(f, os.O_RDONLY)
ev = select.kevent(fd,
filter=select.KQ_FILTER_VNODE,
flags=select.KQ_EV_ADD |
select.KQ_EV_ENABLE |
select.KQ_EV_CLEAR,
fflags=select.KQ_NOTE_WRITE |
select.KQ_NOTE_EXTEND)
kevents.append(ev)
events = self._kq.control(kevents, 0)
while True:
events = self._kq.control(kevents, 1)
fds = [e.ident for e in events]
modified = [k for k, v in self._files.iteritems()
if v in fds]
self.on_change(modified)
finally:
self.close()
def __init__(self, fileno, handler, events):
SelectPoller.__init__(self, fileno, handler, events)
# Make our events 0 by default for first run of update_handler
self.events = 0
# Create our KQueue object
self._kqueue = select.kqueue()
# KQueue needs us to register each event individually
self.update_handler(fileno, events)
def __init__(self, from_kq=None):
self._readable = util.Event()
self._writable = util.Event()
if from_kq:
self._kqueue = from_kq
else:
self._kqueue = select.kqueue()
scheduler._register_fd(self._kqueue.fileno(), self._on_readable, self._on_writable)
def __init__(self):
self.q = select.kqueue()
self.to_ = {
select.KQ_FILTER_READ: POLLIN,
select.KQ_FILTER_WRITE: POLLOUT, }
self.from_ = dict((v, k) for k, v in self.to_.iteritems())
def _is_closed_kqueue(f):
kqueue = select.kqueue()
event = select.kevent(
f.fileno(), filter=select.KQ_FILTER_READ,
flags=select.KQ_EV_ADD | select.KQ_EV_ENABLE)
for event in kqueue.control([event], 128, 0.0):
if event.flags & select.KQ_EV_EOF:
return True
return False
def __init__(self):
self._driver = self
# Saves observed fd, event_mask in `Dict`.
self._events = {}
# OS dependent `kqueue` implementation.
self._kq = select.kqueue()
self._kevent = Kevent()
# Max number of events that will be returned from `kqueue.control`
self._max_events = 200
def test_create(self):
import select
kq = select.kqueue()
assert kq.fileno() > 0
assert not kq.closed
kq.close()
assert kq.closed
raises(ValueError, kq.fileno)
def run(self):
poller = kqueue()
fds = {}
events = []
while True:
while not self.inputqueue.empty():
# loop through the queue and gather new pipes to add the
# kernel queue
buff, pipe, mode = self.inputqueue.get()
if 'r' in mode:
events.append(kevent(pipe, KQ_FILTER_READ, KQ_EV_ADD))
elif 'w' in mode:
events.append(kevent(pipe, KQ_FILTER_WRITE, KQ_EV_ADD))
else:
continue
fds[pipe.fileno()] = (weakref.ref(buff), pipe)
if len(events) == 0:
events = None
events = poller.control(events, 16, 0.1)
for i in range(len(events)):
# loop through response and handle events
event = events.pop()
buff, pipe = fds[event.ident]
if buff() is None:
# buffer object has closed out from underneath us
# pipe will be automatically removed from kqueue
pipe.close()
del fds[event.ident]
continue
if (abs(event.filter) & abs(KQ_FILTER_READ)) and event.data:
# new data has come in, push into the buffer
buff().write(pipe.read(event.data))
if (abs(event.filter) & abs(KQ_FILTER_WRITE)) and event.data:
# space is available to write data
pipe.write(buff().read(\
min(buff()._nbytes, event.data, 2**16)))
if abs(event.flags) & abs(KQ_EV_EOF):
# pipe has been closed and all IO has been processed
# pipe will be automatically removed from kqueue
buff().close()
pipe.close()
del fds[event.ident]
if len(fds) == 0:
# no pipes referenced
if self.idletime + 20 < time():
# idle timeout reached, terminate
break
sleep(0.1)
else:
self.idletime = time()
def test_kqueue(self):
kqueue = select.kqueue()
self.addCleanup(kqueue.close)
t0 = time.monotonic()
kqueue.control(None, 1, self.sleep_time)
dt = time.monotonic() - t0
self.stop_alarm()
self.assertGreaterEqual(dt, self.sleep_time)
def start(self):
self._kq = select.kqueue()
kevents = []
for f in self.filemap:
self.filemap[f] = fd = os.open(f, os.O_RDONLY)
ev = select.kevent(fd,
filter=select.KQ_FILTER_VNODE,
flags=select.KQ_EV_ADD |
select.KQ_EV_ENABLE |
select.KQ_EV_CLEAR,
fflags=select.KQ_NOTE_WRITE |
select.KQ_NOTE_EXTEND)
kevents.append(ev)
events = self._kq.control(kevents, 0)
while not self.shutdown_event.is_set():
events = self._kq.control(kevents, 1)
fds = [e.ident for e in events]
modified = [k for k, v in self.filemap.iteritems()
if v in fds]
self.on_change(modified)
def wait_for_start_line(cmd, confirmed_line, watch='stdout', timeout=5):
if isinstance(confirmed_line, str):
confirmed_line = confirmed_line.encode('utf-8')
process = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
pipe = getattr(process, watch)
# wait until we get the "server started" message, otherwise tests
# may fail due to redis not having actually started yet
start_time = time.time()
if hasattr(select, 'epoll'):
poll = select.epoll()
poll.register(pipe, select.POLLIN)
while time.time() - timeout < start_time:
if poll.poll(0):
line = pipe.readline()
if confirmed_line in line:
poll.unregister(pipe)
return process
poll.unregister(pipe)
elif hasattr(select, 'kqueue'):
kq = select.kqueue()
kq.control([
select.kevent(pipe, select.KQ_FILTER_READ,
select.KQ_EV_ENABLE | select.KQ_EV_ADD)
], 0, 0)
while time.time() - timeout < start_time:
if kq.control(None, 1, 1.):
line = pipe.readline()
if confirmed_line in line:
kq.close()
return process
kq.close()
else:
process.terminate()
raise Exception("No polling available")
process.terminate()
raise Exception("process took too long to start...")
def start(self):
try:
self._kq = select.kqueue()
kevents = []
for f in self._files:
self._files[f] = fd = os.open(f, os.O_RDONLY)
ev = select.kevent(fd,
filter=select.KQ_FILTER_VNODE,
flags=select.KQ_EV_ADD | select.KQ_EV_ENABLE
| select.KQ_EV_CLEAR,
fflags=select.KQ_NOTE_WRITE
| select.KQ_NOTE_EXTEND)
kevents.append(ev)
events = self._kq.control(kevents, 0)
while True:
events = self._kq.control(kevents, 1)
fds = [e.ident for e in events]
modified = [k for k, v in self._files.iteritems() if v in fds]
self.on_change(modified)
finally:
self.close()
def __init__(self, sock):
self._socket = sock
self._poll = None
self._kqueue = None
self._kevents = None
if hasattr(select, "poll"):
# Set up _poll. (Ubuntu, etc.)
#pylint: disable=E1103
self._poll = select.poll()
self._poll.register(sock.fileno(), select.POLLIN)
#pylint: enable=E1103
elif hasattr(select, "kqueue"):
## Set up _kqueue. (BSD and OS X)
self._kqueue = select.kqueue()
self._kevents = [select.kevent(
sock.fileno(), filter = select.KQ_FILTER_READ,
flags = select.KQ_EV_ADD | select.KQ_EV_ENABLE |
select.KQ_EV_CLEAR)]
elif not hasattr(select, "select"):
# Most Python implementations have this fallback, so we
# don't expect this error.
raise RuntimeError("Cannot find a polling utility for sockets")
def test_issue30058(self):
# changelist must be an iterable
kq = select.kqueue()
a, b = socket.socketpair()
ev = select.kevent(a, select.KQ_FILTER_READ, select.KQ_EV_ADD | select.KQ_EV_ENABLE)
kq.control([ev], 0)
# not a list
kq.control((ev,), 0)
# __len__ is not consistent with __iter__
class BadList:
def __len__(self):
return 0
def __iter__(self):
for i in range(100):
yield ev
kq.control(BadList(), 0)
# doesn't have __len__
kq.control(iter([ev]), 0)
a.close()
b.close()
kq.close()
def serve_forever_bsd(self, max_events=1000):
kqueue = select.kqueue()
kevents = [select.kevent(self.socket.fileno(), filter=select.KQ_FILTER_READ, flags=select.KQ_EV_ADD)]
index = 0
request_addr = dict()
while True:
events = kqueue.control(kevents, max_events)
for event in events:
# socket进来，将event加入监听
if event.ident == self.socket.fileno():
request, client_address = self.socket.accept()
index += 1
request_addr[index] = (request, client_address)
kevents.append(select.kevent(request.fileno(), filter=select.KQ_FILTER_READ, flags=select.KQ_EV_ADD, udata=index))
elif event.filter == select.KQ_FILTER_READ and event.udata > 0 and event.flags == select.KQ_EV_ADD:
try:
kevents.remove(
select.kevent(request_addr[event.udata][0].fileno(),
filter=select.KQ_FILTER_READ, flags=select.KQ_EV_ADD, udata=event.udata))
except Exception as e:
pass
# t = Thread(target=self._handle_request_noblock_bsd, args=request_addr[event.udata])
self._handle_request_noblock_bsd(*request_addr[event.udata])
def __init__(self, announce, host, port):
threading.Thread.__init__(self)
self.announce = announce
self.host = host
self.port = port
self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
self._socket.settimeout(None)
self._socket.setblocking(0)
self._socket.bind((self.host, self.port))
self._socket.listen(5)
self.kq = select.kqueue()
self.kevent = [
select.kevent(self._socket.fileno(),
filter=select.KQ_FILTER_READ,
flags=select.KQ_EV_ADD | select.KQ_EV_ENABLE)
]
self.connections = {}
def __init__(self, force_select=False):
platform = sys.platform
if force_select:
self.__async_mode = "select"
self.__iowait_func = self.__select_iowait
return
if platform.find("win32") > -1 or platform.find("cygwin") > -1:
self.__async_mode = "select"
self.__iowait_func = self.__select_iowait
if platform.find("darwin") > -1 or platform.find("freebsd") > -1:
self.__async_mode = "kqueue"
self.__kqueue_object = select.kqueue()
self.__iowait_func = self.__kqueue_iowait
if platform.find("linux") > -1:
self.__async_mode = "epoll"
self.__epoll_object = select.epoll()
self.__iowait_func = self.__epoll_iowait
return
def main():
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server_socket.bind(('0.0.0.0', 5555))
server_socket.listen(5)
kq = select.kqueue()
kq.control([
select.kevent(server_socket.fileno(), select.KQ_FILTER_READ,
select.KQ_EV_ADD)
], 0, 0)
connections = {}
while True:
events = kq.control([], 10000)
for event in events:
if server_socket.fileno() == event.ident:
(client_socket, client_address) = server_socket.accept()
print 'got connection from', client_address
kq.control([
select.kevent(client_socket.fileno(),
select.KQ_FILTER_READ, select.KQ_EV_ADD)
], 0, 0)
connections[client_socket.fileno()] = client_socket
elif event.filter == select.KQ_FILTER_READ:
client_socket = connections[event.ident]
data = client_socket.recv(4096)
if data:
client_socket.send(data)
else:
kq.control([
select.kevent(client_socket.fileno(),
select.KQ_FILTER_READ,
select.KQ_EV_DELETE)
], 0, 0)
client_socket.close()
connections.pop(event.ident)
def kqueueMode():
print('kqueue Mode')
running = True
kq = select.kqueue()
kq.control([select.kevent(sock.fileno(), select.KQ_FILTER_READ, select.KQ_EV_ADD)], 0)
kq.control([select.kevent(sock.fileno(), select.KQ_FILTER_WRITE, select.KQ_EV_ADD)], 0)
kq.control([select.kevent(sys.stdin.fileno(), select.KQ_FILTER_READ, select.KQ_EV_ADD)], 0)
while running:
try:
events = kq.control(None, 1024)
except Exception as e:
print(e)
for each in events:
if each.ident == sock.fileno():
if each.flags & select.KQ_EV_ADD and each.filter == select.KQ_FILTER_READ:
data, addr = sock.recvfrom(1024)
print(f'receive from {addr} data {data}')
writeCache.append(addr)
writeCache.append(data)
elif each.flags & select.KQ_EV_ADD and each.filter == select.KQ_FILTER_WRITE:
if writeCache:
sock.sendto(writeCache.pop(), writeCache.pop())
else:
print(f'unkown sock event {each.flags} {each.filter}')
elif each.ident == sys.stdin.fileno():
if each.flags & select.KQ_EV_ADD and each.filter == select.KQ_FILTER_READ:
junk = sys.stdin.readline()
running = False
kq.control([select.kevent(sock.fileno(), select.KQ_FILTER_READ, select.KQ_EV_DELETE)],0)
kq.control([select.kevent(sock.fileno(), select.KQ_FILTER_WRITE, select.KQ_EV_DELETE)],0)
kq.control([select.kevent(sys.stdin.fileno(), select.KQ_FILTER_READ, select.KQ_EV_DELETE)], 0)
break
else:
print(f'unkown stdin {each.ident} {each.flags} {each.filter}')
else:
print(f'unkown each {each.ident} {each.flags} {each.filter}')
sock.close()
def run(self):
set_thread_name('mdnsd_monitor')
while True:
"""
If the system has not completely booted yet we need to way at least
for DNS to be configured.
In case middlewared is started after boot, system.ready will be set after this plugin
is loaded, hence the dns_sync timeout.
"""
if not self.middleware.call_sync('system.ready'):
if not self.dns_sync.wait(timeout=2):
continue
pid = self.is_alive()
if not pid:
self.start_mdnsd()
time.sleep(2)
continue
kqueue = select.kqueue()
try:
kqueue.control([
select.kevent(
pid,
filter=select.KQ_FILTER_PROC,
flags=select.KQ_EV_ADD,
fflags=select.KQ_NOTE_EXIT,
)
], 0, 0)
except ProcessLookupError:
continue
self.mdnsd_running.set()
self.middleware.call_sync('mdnsadvertise.restart')
kqueue.control(None, 1)
self.mdnsd_running.clear()
kqueue.close()
def kqueue_code_changed():
"""
Checks for changed code using kqueue. After being called
it blocks until a change event has been fired.
"""
# Maximum number of open file descriptors is typically too low (256).
filenames = list(gen_filenames())
resource.setrlimit(resource.RLIMIT_NOFILE,
(NOFILES_SOFT + len(filenames), NOFILES_HARD))
kqueue = select.kqueue()
fds = [open(filename) for filename in filenames]
_filter = select.KQ_FILTER_VNODE
flags = select.KQ_EV_ADD
fflags = select.KQ_NOTE_DELETE | select.KQ_NOTE_WRITE | select.KQ_NOTE_RENAME
kevents = [select.kevent(fd, _filter, flags, fflags) for fd in fds]
kqueue.control(kevents, 1)
for fd in fds:
fd.close()
kqueue.close()
return True
def main():
# 1.买个手机（创建套接字）
tcp_server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
tcp_server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# 2.插上电话卡（绑定本地地址）
tcp_server_socket.bind(('', 12345))
# 3.将手机设置为正常的响铃模式（让主动套接字变为被动套接字）
tcp_server_socket.listen(128)
tcp_server_socket.setblocking(False) # 将套接字改为非堵塞
# 创建epoll对象
epl = select.kqueue()
#
client_socket_list = list()
while True:
try:
new_client, client_addr = tcp_server_socket.accept()
except Exception as ret:
pass
else:
new_client.setblocking(False)
client_socket_list.append(new_client)
for client_socket in client_socket_list:
try:
recv_data = client_socket.recv(1024).decode('utf-8')
except Exception as ret:
pass
else:
if recv_data:
service_client(client_socket, recv_data)
else:
client_socket.close()
client_socket_list.remove(client_socket)
def start_monitor(dirs):
last_run = time.time()
files_stats = []
paths = []
current_dir = os.getcwd()
kq = kqueue()
source_events = []
for dir_name in dirs:
dir_path = current_dir + '/' + dir_name
paths.append(dir_path)
fd = os.open(dir_path, os.O_RDONLY)
event = kevent(fd,
filter=select.KQ_FILTER_VNODE,
flags=select.KQ_EV_ADD | select.KQ_EV_CLEAR,
fflags=select.KQ_NOTE_WRITE)
source_events.append(event)
while True:
events = kq.control(source_events, len(source_events), 2000)
if any(map(lambda e: e.fflags & select.KQ_NOTE_WRITE, events)):
if (time.time() - last_run) < LIMIT:
continue
if check_modifications(current_dir, paths):
try:
async_test(["make", "test"])
os.system('clear')
subprocess.Popen("neurotic")
except NeuroticError as ne:
os.system('clear')
if b"ERROR" in ne.content[0]:
print(ne.content[0])
else:
subprocess.Popen("neurotic")
last_run = time.time()
def worker(self):
self.logger.debug('Opened tail stream on file {0} ({1} lines)'.format(
self.path, self.backlog))
with io.open(self.fd, 'wb') as fd:
with open(self.path, 'rb') as f:
kq = select.kqueue()
try:
ev = [
select.kevent(fd.fileno(),
filter=select.KQ_FILTER_READ,
flags=select.KQ_EV_ADD
| select.KQ_EV_ENABLE),
select.kevent(f.fileno(),
filter=select.KQ_FILTER_READ,
flags=select.KQ_EV_ADD
| select.KQ_EV_ENABLE)
]
fd.write(self.tail(f, self.backlog))
fd.flush()
kq.control(ev, 0)
f.seek(0, os.SEEK_END)
while True:
event, = kq.control(None, 1)
self.logger.debug('kqueue event {0}'.format(event))
if event.ident == fd.fileno():
if event.flags & select.KQ_EV_EOF or event.flags & select.KQ_EV_ERROR:
break
if event.ident == f.fileno():
fd.write(f.read())
fd.flush()
finally:
kq.close()
def __init__(self):
"""
调用select.kqueue获取IO复用接口kqueue，并初始化socket文件描述符字典fds
"""
self._kqueue = select.kqueue()
self._fds = {}
def test_queue_event(self):
serverSocket = socket.socket()
serverSocket.bind(('127.0.0.1', 0))
serverSocket.listen(1)
client = socket.socket()
client.setblocking(False)
try:
client.connect(('127.0.0.1', serverSocket.getsockname()[1]))
except socket.error, e:
self.assertEqual(e.args[0], errno.EINPROGRESS)
else:
#raise AssertionError("Connect should have raised EINPROGRESS")
pass # FreeBSD doesn't raise an exception here
server, addr = serverSocket.accept()
kq = select.kqueue()
kq2 = select.kqueue.fromfd(kq.fileno())
ev = select.kevent(server.fileno(),
select.KQ_FILTER_WRITE,
select.KQ_EV_ADD | select.KQ_EV_ENABLE)
kq.control([ev], 0)
ev = select.kevent(server.fileno(),
select.KQ_FILTER_READ,
select.KQ_EV_ADD | select.KQ_EV_ENABLE)
kq.control([ev], 0)
ev = select.kevent(client.fileno(),
select.KQ_FILTER_WRITE,
select.KQ_EV_ADD | select.KQ_EV_ENABLE)
kq2.control([ev], 0)
ev = select.kevent(client.fileno(),
def after_daemonize(self):
self._kqueue = select.kqueue()
for fd in self.readables:
self.register_readable(fd)
for fd in self.writables:
self.register_writable(fd)
def initialize(self):
self._kqueue = select.kqueue()
self.readables = set()
self.writables = set()
def __init__(self):
super(KqueueSelector, self).__init__()
self._kqueue = select.kqueue()
def __init__(self):
_baseServer.__init__(self)
self.kq = select.kqueue()
def __init__(self):
self._kqueue = select.kqueue()
self._fds = {}
def __init__(self):
super().__init__()
self._kqueue = select.kqueue()
def __init__(self, socket):
super(KQueueSelector, self).__init__(socket)
self._queue = select.kqueue()
self._events = [
select.kevent(self._socket.fileno(), filter=select.KQ_FILTER_READ)
]
def _init_poller(self):
"""Notify the implementation to allocate the poller resource"""
assert self._kqueue is None
self._kqueue = select.kqueue()
def __init__(self):
self._kqueue = select.kqueue()
self._active = {}
def is_connected(skt):
try:
fno = skt.fileno()
except socket.error as e:
if e[0] == errno.EBADF:
return False
raise
try:
if hasattr(select, "epoll"):
ep = select.epoll()
ep.register(fno, select.EPOLLOUT | select.EPOLLIN)
events = ep.poll(0)
for fd, ev in events:
if fno == fd and \
(ev & select.EPOLLOUT or ev & select.EPOLLIN):
ep.unregister(fno)
return True
ep.unregister(fno)
elif hasattr(select, "poll"):
p = select.poll()
p.register(fno, select.POLLOUT | select.POLLIN)
events = p.poll(0)
for fd, ev in events:
if fno == fd and \
(ev & select.POLLOUT or ev & select.POLLIN):
p.unregister(fno)
return True
p.unregister(fno)
elif can_use_kqueue():
kq = select.kqueue()
events = [
select.kevent(fno, select.KQ_FILTER_READ, select.KQ_EV_ADD),
select.kevent(fno, select.KQ_FILTER_WRITE, select.KQ_EV_ADD)
]
kq.control(events, 0)
kevents = kq.control(None, 4, 0)
for ev in kevents:
if ev.ident == fno:
if ev.flags & select.KQ_EV_ERROR:
return False
else:
return True
# delete
events = [
select.kevent(fno, select.KQ_FILTER_READ, select.KQ_EV_DELETE),
select.kevent(fno, select.KQ_FILTER_WRITE, select.KQ_EV_DELETE)
]
kq.control(events, 0)
kq.close()
return True
else:
r, _, _ = select.select([fno], [], [], 0)
if not r:
return True
except IOError:
pass
except (ValueError, select.error,) as e:
pass
return False
def __init__(self):
super(KQueueIOLoop, self).__init__(impl=select.kqueue())
def test_fd_non_inheritable(self):
kqueue = select.kqueue()
self.addCleanup(kqueue.close)
self.assertEqual(os.get_inheritable(kqueue.fileno()), False)
def test_queue_event(self):
serverSocket = socket.socket()
serverSocket.bind(('127.0.0.1', 0))
serverSocket.listen()
client = socket.socket()
client.setblocking(False)
try:
client.connect(('127.0.0.1', serverSocket.getsockname()[1]))
except OSError as e:
self.assertEqual(e.args[0], errno.EINPROGRESS)
else:
#raise AssertionError("Connect should have raised EINPROGRESS")
pass # FreeBSD doesn't raise an exception here
server, addr = serverSocket.accept()
kq = select.kqueue()
kq2 = select.kqueue.fromfd(kq.fileno())
ev = select.kevent(server.fileno(), select.KQ_FILTER_WRITE,
select.KQ_EV_ADD | select.KQ_EV_ENABLE)
kq.control([ev], 0)
ev = select.kevent(server.fileno(), select.KQ_FILTER_READ,
select.KQ_EV_ADD | select.KQ_EV_ENABLE)
kq.control([ev], 0)
ev = select.kevent(client.fileno(), select.KQ_FILTER_WRITE,
select.KQ_EV_ADD | select.KQ_EV_ENABLE)
kq2.control([ev], 0)
ev = select.kevent(client.fileno(), select.KQ_FILTER_READ,
select.KQ_EV_ADD | select.KQ_EV_ENABLE)
kq2.control([ev], 0)
events = kq.control(None, 4, 1)
events = set((e.ident, e.filter) for e in events)
self.assertEqual(
events,
set([(client.fileno(), select.KQ_FILTER_WRITE),
(server.fileno(), select.KQ_FILTER_WRITE)]))
client.send(b"Hello!")
server.send(b"world!!!")
# We may need to call it several times
for i in range(10):
events = kq.control(None, 4, 1)
if len(events) == 4:
break
time.sleep(1.0)
else:
self.fail('timeout waiting for event notifications')
events = set((e.ident, e.filter) for e in events)
self.assertEqual(
events,
set([(client.fileno(), select.KQ_FILTER_WRITE),
(client.fileno(), select.KQ_FILTER_READ),
(server.fileno(), select.KQ_FILTER_WRITE),
(server.fileno(), select.KQ_FILTER_READ)]))
# Remove completely client, and server read part
ev = select.kevent(client.fileno(), select.KQ_FILTER_WRITE,
select.KQ_EV_DELETE)
kq.control([ev], 0)
ev = select.kevent(client.fileno(), select.KQ_FILTER_READ,
select.KQ_EV_DELETE)
kq.control([ev], 0)
ev = select.kevent(server.fileno(), select.KQ_FILTER_READ,
select.KQ_EV_DELETE)
kq.control([ev], 0, 0)
events = kq.control([], 4, 0.99)
events = set((e.ident, e.filter) for e in events)
self.assertEqual(events,
set([(server.fileno(), select.KQ_FILTER_WRITE)]))
client.close()
server.close()
serverSocket.close()
