def __init__(self, readCallback, writeCallback, acceptCallback,
errorCallback):
self.__log = logging.getLogger("root.litesockets.SelectSelector")
self.__log.info("Creating basic select selector for: {}".format(
platform.system()))
self.__readCallback = readCallback
self.__writeCallback = writeCallback
self.__acceptCallback = acceptCallback
self.__errorCallback = errorCallback
self.__readClients = set()
self.__writeClients = set()
self.__acceptServers = set()
self.__nb_readClients = set()
self.__nb_writeClients = set()
self.__nb_acceptServers = set()
self.__writeLock = threading.Condition()
self.__readLock = threading.Condition()
self.__acceptLock = threading.Condition()
self.__nbLock = threading.Condition()
self.__localExecuter = Scheduler(
5) #need 5 thread, all can be blocked at once
self.__running = True
self.__localExecuter.execute(self.__doReads)
self.__localExecuter.execute(self.__doWrites)
self.__localExecuter.execute(self.__doAcceptor)
class TestSESelect(TestSE):
def setUp(self):
self.SCH = Scheduler(10)
self.SE = litesockets.SocketExecuter(scheduler=self.SCH, forcePlatform="win")
def tearDown(self):
self.SE.stop()
self.SCH.shutdown_now()
class TestSESelect(TestSE):
def setUp(self):
self.SCH = Scheduler(10)
self.SE = litesockets.SocketExecuter(scheduler=self.SCH,
forcePlatform="win")
def tearDown(self):
self.SE.stop()
self.SCH.shutdown_now()
def __init__(self, threads=5, scheduler=None, forcePlatform=None):
"""
Constructs a new SocketExecuter
`threads` used to set the number of threads used when creating a Scheduler when no Scheduler is provided.
`scheduler` this scheduler will be used with the SocketExecuters client/server callbacks.
`forcePlatform` this sets the detected platform, this can be used to switch the selector object made.
"""
self.__log = logging.getLogger(
"root.litesockets.SocketExecuter:{}".format(id(self)))
self.__clients = dict()
self.__servers = dict()
self.__internalExec = None
if scheduler == None:
self.__executor = Scheduler(threads)
self.__internalExec = self.__executor
else:
self.__executor = scheduler
self.__stats = Stats()
if forcePlatform == None:
forcePlatform = platform.system()
if forcePlatform.lower().find("linux") > -1:
self.__selector = EpollSelector(self.__clientRead,
self.__clientWrite,
self.__serverAccept,
self.__socketerrors)
else:
self.__selector = SelectSelector(self.__clientRead,
self.__clientWrite,
self.__serverAccept,
self.__socketerrors)
self.__running = True
def __init__(self, readCallback, writeCallback, acceptCallback, errorCallback):
self.__log = logging.getLogger("root.litesockets.SelectSelector")
self.__log.info("Creating basic select selector for: {}".format(platform.system()))
self.__readCallback = readCallback
self.__writeCallback = writeCallback
self.__acceptCallback = acceptCallback
self.__errorCallback = errorCallback
self.__readClients = set()
self.__writeClients = set()
self.__acceptServers = set()
self.__nb_readClients = set()
self.__nb_writeClients = set()
self.__nb_acceptServers = set()
self.__writeLock = threading.Condition()
self.__readLock = threading.Condition()
self.__acceptLock = threading.Condition()
self.__nbLock = threading.Condition()
self.__localExecuter = Scheduler(5) #need 5 thread, all can be blocked at once
self.__running = True
self.__localExecuter.execute(self.__doReads)
self.__localExecuter.execute(self.__doWrites)
self.__localExecuter.execute(self.__doAcceptor)
def __init__(self, readCallback, writeCallback, acceptCallback,
errorCallback):
self.__log = logging.getLogger("root.litesockets.EpollSelector")
self.__log.info("Creating epoll selector for: {}".format(
platform.system()))
self.__DEFAULT_READ_POLLS = select.EPOLLIN | select.EPOLLRDHUP | select.EPOLLHUP | select.EPOLLERR
self.__DEFAULT_ACCEPT_POLLS = select.EPOLLIN | select.EPOLLRDHUP | select.EPOLLHUP | select.EPOLLERR
self.__readCallback = readCallback
self.__writeCallback = writeCallback
self.__acceptCallback = acceptCallback
self.__errorCallback = errorCallback
self.__ReadSelector = select.epoll()
self.__WriteSelector = select.epoll()
self.__AcceptorSelector = select.epoll()
self.__running = True
self.__localExecuter = Scheduler(3)
self.__localExecuter.execute(self.__doReads)
self.__localExecuter.execute(self.__doWrites)
self.__localExecuter.execute(self.__doAcceptor)
def __init__(self, readCallback, writeCallback, acceptCallback, errorCallback):
self.__log = logging.getLogger("root.litesockets.EpollSelector")
self.__log.info("Creating epoll selector for: {}".format(platform.system()))
self.__DEFAULT_READ_POLLS = select.EPOLLIN|select.EPOLLRDHUP|select.EPOLLHUP|select.EPOLLERR
self.__DEFAULT_ACCEPT_POLLS = select.EPOLLIN|select.EPOLLRDHUP|select.EPOLLHUP|select.EPOLLERR
self.__readCallback = readCallback
self.__writeCallback = writeCallback
self.__acceptCallback = acceptCallback
self.__errorCallback = errorCallback
self.__ReadSelector = select.epoll()
self.__WriteSelector = select.epoll()
self.__AcceptorSelector = select.epoll()
self.__running = True
self.__localExecuter = Scheduler(3)
self.__localExecuter.execute(self.__doReads)
self.__localExecuter.execute(self.__doWrites)
self.__localExecuter.execute(self.__doAcceptor)
def setUp(self): self.SCH = Scheduler(10) self.SE = litesockets.SocketExecuter(scheduler=self.SCH)
def setUp(self): self.SCH = Scheduler(10) self.SE = litesockets.SocketExecuter(scheduler=self.SCH, forcePlatform="win")
class TestSE(unittest.TestCase):
def setUp(self):
self.SCH = Scheduler(10)
self.SE = litesockets.SocketExecuter(scheduler=self.SCH)
def tearDown(self):
self.SE.stop()
self.SCH.shutdown_now()
def test_SE_ServerStartMany(self):
SE1 = litesockets.SocketExecuter()
self.assertTrue(self.SE.isRunning(), "SE not running")
self.assertTrue(SE1.isRunning(), "SE1 not running")
self.SE.stop()
SE1.stop()
self.assertFalse(self.SE.isRunning(), "SE not running")
self.assertFalse(SE1.isRunning(), "SE1 not running")
self.SCH.shutdown_now()
def test_SE_ClientAddRemove(self):
CLIENT_NUM = 1
ta = testClass(self.SE)
server = self.SE.createTCPServer("localhost", 0)
server.setOnClient(ta.accept)
PORT = server.getSocket().getsockname()[1]
server.start()
clients = list()
for i in xrange(CLIENT_NUM):
client = self.SE.createTCPClient("localhost", PORT)
client.connect()
for i in xrange(500):
if (len(self.SE.getClients()) == (len(clients) * 2) + 2):
break
else:
time.sleep(.01)
self.assertEqual(len(self.SE.getClients()), (len(clients) * 2) + 2)
clients.append(client)
self.assertEqual(len(self.SE.getClients()), len(clients) * 2)
# for i in xrange(CLIENT_NUM):
# clients[i].write("TEST")
for i in xrange(CLIENT_NUM):
clients[i].close()
for Q in xrange(500):
if len(self.SE.getClients()) == (CLIENT_NUM * 2) - (
(i + 1) * 2):
break
else:
time.sleep(.1)
self.assertEqual(len(self.SE.getClients()),
(CLIENT_NUM * 2) - ((i + 1) * 2))
self.assertEqual(len(self.SE.getClients()), 0)
server.stop()
server.close()
self.SE.stop()
def test_SE_ServerAddRemove(self):
SERVER_NUM = 5
testA = list()
servers = list()
for i in xrange(SERVER_NUM):
ta = testClass(self.SE)
server = self.SE.createTCPServer("localhost", 0)
server.setOnClient(ta.accept)
server.start()
testA.append(ta)
servers.append(server)
self.assertEqual(len(self.SE.getServers()), i + 1)
for i in xrange(SERVER_NUM):
servers[i].close()
c = 0
while len(
self.SE.getServers()) > len(servers) - (i + 1) or c > 500:
time.sleep(.01)
c += 1
self.assertEqual(len(self.SE.getServers()), len(servers) - (i + 1))
c = 0
while len(self.SE.getServers()) > 0 or c > 500:
time.sleep(.01)
c += 1
self.assertEqual(len(self.SE.getServers()), 0)
self.SE.startServer("TEST")
self.assertEqual(len(self.SE.getServers()), 0)
self.SE.stop()
def test_SE_Stats(self):
ta = testClass(self.SE)
server = self.SE.createTCPServer("localhost", 0)
server.setOnClient(ta.accept)
server.start()
PORT = server.getSocket().getsockname()[1]
client = self.SE.createTCPClient("localhost", PORT)
cta = testClass(self.SE)
client.setReader(cta.read)
client.connect()
waitTill(lambda X: len(ta.clients) < X, 1, 500)
client.write("X" * 1000)
waitTill(lambda X: ta.read_len < X, 1000, 500)
self.assertEqual(1000, ta.read_len)
self.assertEqual(1000, self.SE.getStats().getTotalRead())
self.assertEqual(1000, self.SE.getStats().getTotalWrite())
self.assertTrue(1000, self.SE.getStats().getReadRate() > 0.0)
self.assertTrue(1000, self.SE.getStats().getWriteRate() > 0.0)
server.close()
client.close()
def test_SE_ClientMaxReads(self):
ta = testClass(self.SE)
server = self.SE.createTCPServer("localhost", 0)
server.setOnClient(ta.accept)
server.start()
PORT = server.getSocket().getsockname()[1]
client = self.SE.createTCPClient("localhost", PORT)
client.MAXBUFFER = 1
cta = testClass(self.SE)
client.setReader(cta.read)
client.connect()
c = 0
while len(ta.clients) <= 0 and c < 500:
time.sleep(.01)
c += 1
self.assertEqual(len(ta.clients), 1)
ta.clients[0].MAXBUFFER = 20
for i in xrange(20):
ta.clients[0].write("T" * 10)
c = 0
while cta.read_len < 200 and c < 500:
time.sleep(.01)
c += 1
self.assertEqual(cta.read_len, 200)
self.SE.stop()
class TestSE(unittest.TestCase):
def setUp(self):
self.SCH = Scheduler(10)
self.SE = litesockets.SocketExecuter(scheduler=self.SCH)
def tearDown(self):
self.SE.stop()
self.SCH.shutdown_now()
def test_SE_ServerStartMany(self):
SE1 = litesockets.SocketExecuter()
self.assertTrue(self.SE.isRunning(), "SE not running")
self.assertTrue(SE1.isRunning(), "SE1 not running")
self.SE.stop()
SE1.stop()
self.assertFalse(self.SE.isRunning(), "SE not running")
self.assertFalse(SE1.isRunning(), "SE1 not running")
self.SCH.shutdown_now()
def test_SE_ClientAddRemove(self):
CLIENT_NUM = 1
ta = testClass(self.SE)
server = self.SE.createTCPServer("localhost", 0)
server.setOnClient(ta.accept)
PORT = server.getSocket().getsockname()[1]
server.start()
clients = list()
for i in xrange(CLIENT_NUM):
client = self.SE.createTCPClient("localhost", PORT)
client.connect()
for i in xrange(500):
if(len(self.SE.getClients()) == (len(clients)*2)+2):
break
else:
time.sleep(.01)
self.assertEqual(len(self.SE.getClients()), (len(clients)*2)+2)
clients.append(client)
self.assertEqual(len(self.SE.getClients()), len(clients)*2)
# for i in xrange(CLIENT_NUM):
# clients[i].write("TEST")
for i in xrange(CLIENT_NUM):
clients[i].close()
for Q in xrange(500):
if len(self.SE.getClients()) == (CLIENT_NUM*2)-((i+1)*2):
break
else:
time.sleep(.1)
self.assertEqual(len(self.SE.getClients()), (CLIENT_NUM*2)-((i+1)*2))
self.assertEqual(len(self.SE.getClients()), 0)
server.stop()
server.close()
self.SE.stop()
def test_SE_ServerAddRemove(self):
SERVER_NUM = 5
testA = list()
servers = list()
for i in xrange(SERVER_NUM):
ta = testClass(self.SE)
server = self.SE.createTCPServer("localhost", 0)
server.setOnClient(ta.accept)
server.start()
testA.append(ta)
servers.append(server)
self.assertEqual(len(self.SE.getServers()), i+1)
for i in xrange(SERVER_NUM):
servers[i].close()
c = 0
while len(self.SE.getServers()) > len(servers)-(i+1) or c > 500:
time.sleep(.01)
c+=1
self.assertEqual(len(self.SE.getServers()), len(servers)-(i+1))
c = 0
while len(self.SE.getServers()) > 0 or c > 500:
time.sleep(.01)
c+=1
self.assertEqual(len(self.SE.getServers()), 0)
self.SE.startServer("TEST")
self.assertEqual(len(self.SE.getServers()), 0)
self.SE.stop()
def test_SE_Stats(self):
ta = testClass(self.SE)
server = self.SE.createTCPServer("localhost", 0)
server.setOnClient(ta.accept)
server.start()
PORT = server.getSocket().getsockname()[1]
client = self.SE.createTCPClient("localhost", PORT)
cta = testClass(self.SE)
client.setReader(cta.read)
client.connect()
waitTill(lambda X: len(ta.clients) < X, 1, 500)
client.write("X"*1000)
waitTill(lambda X: ta.read_len < X, 1000, 500)
self.assertEqual(1000, ta.read_len)
self.assertEqual(1000, self.SE.getStats().getTotalRead())
self.assertEqual(1000, self.SE.getStats().getTotalWrite())
self.assertTrue(1000, self.SE.getStats().getReadRate() > 0.0)
self.assertTrue(1000, self.SE.getStats().getWriteRate() > 0.0)
server.close()
client.close()
def test_SE_ClientMaxReads(self):
ta = testClass(self.SE)
server = self.SE.createTCPServer("localhost", 0)
server.setOnClient(ta.accept)
server.start()
PORT = server.getSocket().getsockname()[1]
client = self.SE.createTCPClient("localhost", PORT)
client.MAXBUFFER = 1
cta = testClass(self.SE)
client.setReader(cta.read)
client.connect()
c = 0
while len(ta.clients) <= 0 and c < 500:
time.sleep(.01)
c+=1
self.assertEqual(len(ta.clients), 1)
ta.clients[0].MAXBUFFER=20
for i in xrange(20):
ta.clients[0].write("T"*10)
c = 0
while cta.read_len < 200 and c < 500:
time.sleep(.01)
c+=1
self.assertEqual(cta.read_len, 200)
self.SE.stop()
class EpollSelector():
def __init__(self, readCallback, writeCallback, acceptCallback, errorCallback):
self.__log = logging.getLogger("root.litesockets.EpollSelector")
self.__log.info("Creating epoll selector for: {}".format(platform.system()))
self.__DEFAULT_READ_POLLS = select.EPOLLIN|select.EPOLLRDHUP|select.EPOLLHUP|select.EPOLLERR
self.__DEFAULT_ACCEPT_POLLS = select.EPOLLIN|select.EPOLLRDHUP|select.EPOLLHUP|select.EPOLLERR
self.__readCallback = readCallback
self.__writeCallback = writeCallback
self.__acceptCallback = acceptCallback
self.__errorCallback = errorCallback
self.__ReadSelector = select.epoll()
self.__WriteSelector = select.epoll()
self.__AcceptorSelector = select.epoll()
self.__running = True
self.__localExecuter = Scheduler(3)
self.__localExecuter.execute(self.__doReads)
self.__localExecuter.execute(self.__doWrites)
self.__localExecuter.execute(self.__doAcceptor)
def stop(self):
self.__running = False
self.__ReadSelector.close()
self.__WriteSelector.close()
self.__AcceptorSelector.close()
self.__localExecuter.shutdown_now()
def addServer(self, fileno):
try:
self.__AcceptorSelector.register(fileno, self.__DEFAULT_ACCEPT_POLLS)
except:
noExcept(self.__AcceptorSelector.modify, fileno, self.__DEFAULT_ACCEPT_POLLS)
def removeServer(self, fileno):
noExcept(self.__AcceptorSelector.unregister, fileno)
def addReader(self, fileno):
try:
self.__ReadSelector.register(fileno, self.__DEFAULT_READ_POLLS)
except:
noExcept(self.__ReadSelector.modify, fileno, self.__DEFAULT_READ_POLLS)
def removeReader(self, fileno):
noExcept(self.__ReadSelector.unregister, fileno)
def addWriter(self, fileno):
try:
self.__WriteSelector.register(fileno, select.EPOLLOUT)
except:
noExcept(self.__WriteSelector.modify, fileno, select.EPOLLOUT)
def removeWriter(self, fileno):
noExcept(self.__WriteSelector.unregister, fileno)
def __doThread(self, t):
while self.__running:
try:
t()
except Exception as e:
self.__log.error("GP Socket Exception: %s: %s"%(t, sys.exc_info()[0]))
self.__log.error(e)
def __doReads(self):
events = self.__ReadSelector.poll(100)
for fileno, event in events:
try:
if event & select.EPOLLIN:
self.__readCallback(fileno)
if (event & select.EPOLLRDHUP or event & select.EPOLLHUP or event & select.EPOLLERR):
self.__errorCallback(fileno)
noExcept(self.__ReadSelector.unregister, fileno)
except Exception as e:
self.__log.debug("Read Error: %s"%(sys.exc_info()[0]))
self.__log.debug(e)
if self.__running:
self.__localExecuter.execute(self.__doReads)
def __doWrites(self):
events = self.__WriteSelector.poll(100)
for fileno, event in events:
try:
if event & select.EPOLLOUT:
self.__writeCallback(fileno)
else:
self.__errorCallback(fileno)
noExcept(self.__WriteSelector.unregister, fileno)
except Exception as e:
self.__log.debug("Write Error: %s"%(sys.exc_info()[0]))
self.__log.debug(e)
if self.__running:
self.__localExecuter.execute(self.__doWrites)
def __doAcceptor(self):
events = self.__AcceptorSelector.poll(100)
for fileno, event in events:
try:
if event & select.EPOLLIN:
self.__acceptCallback(fileno)
else:
self.__errorCallback(fileno)
noExcept(self.__WriteSelector.unregister, fileno)
except Exception as e:
self.__log.debug("Accept Error: %s"%(sys.exc_info()[0]))
self.__log.debug(e)
logging.exception("")
if self.__running:
self.__localExecuter.execute(self.__doAcceptor)
class SelectSelector():
def __init__(self, readCallback, writeCallback, acceptCallback, errorCallback):
self.__log = logging.getLogger("root.litesockets.SelectSelector")
self.__log.info("Creating basic select selector for: {}".format(platform.system()))
self.__readCallback = readCallback
self.__writeCallback = writeCallback
self.__acceptCallback = acceptCallback
self.__errorCallback = errorCallback
self.__readClients = set()
self.__writeClients = set()
self.__acceptServers = set()
self.__nb_readClients = set()
self.__nb_writeClients = set()
self.__nb_acceptServers = set()
self.__writeLock = threading.Condition()
self.__readLock = threading.Condition()
self.__acceptLock = threading.Condition()
self.__nbLock = threading.Condition()
self.__localExecuter = Scheduler(5) #need 5 thread, all can be blocked at once
self.__running = True
self.__localExecuter.execute(self.__doReads)
self.__localExecuter.execute(self.__doWrites)
self.__localExecuter.execute(self.__doAcceptor)
def stop(self):
self.__running = False
self.__localExecuter.shutdown_now()
def addServer(self, fileno):
self.__acceptLock.acquire()
self.__acceptServers.add(FileNoWrapper(fileno))
self.__acceptLock.release()
def removeServer(self, fileno):
now = FileNoWrapper(fileno)
if now in self.__acceptServers:
self.__acceptServers.remove(now)
def addReader(self, fileno):
now = FileNoWrapper(fileno)
if now in self.__readClients or now in self.__nb_readClients:
return
if self.__readLock.acquire(blocking=False):
self.__readClients.add(now)
self.__readLock.release()
else:
self.__nb_readClients.add(now)
self.__localExecuter.schedule(self.__tmpClientSelect, delay=0, recurring=False, key="SimpleKey")
self.__localExecuter.schedule(self.__update_from_nb_selector, key="UpdateTask")
def removeReader(self, fileno):
now = FileNoWrapper(fileno)
if now in self.__readClients:
self.__readClients.remove(now)
if now in self.__nb_readClients:
self.__nb_readClients.remove(now)
def addWriter(self, fileno):
now = FileNoWrapper(fileno)
if now in self.__writeClients or now in self.__nb_writeClients:
return
if self.__writeLock.acquire(blocking=False):
self.__writeClients.add(now)
self.__writeLock.release()
else:
self.__nb_writeClients.add(now)
self.__localExecuter.schedule(self.__tmpClientSelect, key="SimpleKey")
self.__localExecuter.schedule(self.__update_from_nb_selector, key="UpdateTask")
def removeWriter(self, fileno):
now = FileNoWrapper(fileno)
if now in self.__writeClients:
self.__writeClients.remove(now)
if now in self.__nb_writeClients:
self.__nb_writeClients.remove(now)
def __doThread(self, t):
while self.__running:
try:
t()
except Exception as e:
self.__log.error("GP Socket Exception: %s: %s"%(t, sys.exc_info()[0]))
self.__log.error(e)
def __update_from_nb_selector(self):
if len(self.__nb_readClients) + len(self.__nb_writeClients) == 0:
return
else:
self.__readLock.acquire()
self.__nbLock.acquire()
for r in self.__nb_readClients:
self.__readClients.add(r)
self.__nb_readClients.clear()
self.__nbLock.release()
self.__readLock.release()
self.__writeLock.acquire()
self.__nbLock.acquire()
for r in self.__nb_writeClients:
self.__writeClients.add(r)
self.__nb_writeClients.clear()
self.__nbLock.release()
self.__writeLock.release()
def __tmpClientSelect(self):
if len(self.__nb_readClients) + len(self.__nb_writeClients) == 0:
return
rlist = []
wlist = []
xlist = []
self.__nbLock.acquire()
try:
rlist, wlist, xlist = select.select(self.__nb_readClients, self.__nb_writeClients, self.__readClients, 0.001)
except:
#We sometimes throw here when a client is removed from the set during the loop
pass
for rdy in rlist:
try:
self.__readCallback(rdy.fileno())
except Exception as e:
self.__log.debug("nbRead Error: %s"%(sys.exc_info()[0]))
self.__log.debug(e)
for rdy in wlist:
try:
self.__writeCallback(rdy.fileno())
except Exception as e:
self.__log.debug("nbWrite Error: %s"%(sys.exc_info()[0]))
self.__log.debug(e)
for bad in xlist:
try:
self.__errorCallback(bad.fileno())
except:
self.__log.debug("nberrorCB Error: %s"%(sys.exc_info()[0]))
self.__log.debug(e)
self.__nbLock.release()
if len(self.__nb_readClients) + len(self.__nb_writeClients) > 0:
self.__localExecuter.schedule(self.__tmpClientSelect, key="SimpleKey")
def __doReads(self):
rlist = []
wlist = []
xlist = []
self.__readLock.acquire()
if len(self.__readClients) > 0:
try:
rlist, wlist, xlist = select.select(self.__readClients, [], self.__readClients, .1)
except Exception as e:
#We sometimes throw here when a client is removed from the set during the loop
pass
else:
time.sleep(.1)
self.__readLock.release()
for rdy in rlist:
try:
if rdy in self.__readClients:
self.__readCallback(rdy.fileno())
except IOError as e:
if e.errno != errno.EBADF:
self.__log.error("Unknown error in Selector Read")
self.__log.error(e, exc_info=True)
except Exception as e:
self.__log.debug("Read Error: %s"%(sys.exc_info()[0]))
self.__log.debug(e)
for bad in xlist:
try:
self.__errorCallback(bad.fileno())
except:
self.__log.debug("errorCB Error: %s"%(sys.exc_info()[0]))
self.__log.debug(e)
if self.__running:
self.__localExecuter.execute(self.__doReads)
def __doWrites(self):
rlist = []
wlist = []
xlist = []
self.__writeLock.acquire()
if len(self.__writeClients) > 0:
try:
rlist, wlist, xlist = select.select([], self.__writeClients,[] , .1)
except Exception as e:
#We sometimes throw here when a client is removed from the set during the loop
pass
else:
time.sleep(.1)
self.__writeLock.release()
for rdy in wlist:
try:
if rdy in self.__writeClients:
self.__writeCallback(rdy.fileno())
except Exception as e:
self.__log.debug("Write Error: %s"%(sys.exc_info()[0]))
self.__log.debug(e)
self.__writeClients.remove(rdy)
if self.__running:
self.__localExecuter.execute(self.__doWrites)
def __doAcceptor(self):
rlist = []
wlist = []
xlist = []
self.__acceptLock.acquire()
if len(self.__acceptServers) > 0:
try:
rlist, wlist, xlist = select.select(self.__acceptServers, [], self.__acceptServers, .1)
except:
#We sometimes throw here when a server is removed from the set during the loop
pass
else:
time.sleep(.1)
self.__acceptLock.release()
for bad in xlist:
try:
self.__errorCallback(bad.fileno())
self.__writeClients.remove(bad)
except Exception as e:
self.__log.debug("errorCB Error: %s"%(sys.exc_info()[0]))
self.__log.debug(e)
for rdy in rlist:
try:
if rdy in self.__acceptServers:
self.__acceptCallback(rdy.fileno())
except Exception as e:
self.__log.debug("Accept Error: %s"%(sys.exc_info()[0]))
self.__log.debug(e)
logging.exception("")
self.__writeClients.remove(rdy)
if self.__running:
self.__localExecuter.execute(self.__doAcceptor)
def setUp(self):
self.SCH = Scheduler(10)
self.SE = litesockets.SocketExecuter(scheduler=self.SCH,
forcePlatform="win")
class EpollSelector():
def __init__(self, readCallback, writeCallback, acceptCallback,
errorCallback):
self.__log = logging.getLogger("root.litesockets.EpollSelector")
self.__log.info("Creating epoll selector for: {}".format(
platform.system()))
self.__DEFAULT_READ_POLLS = select.EPOLLIN | select.EPOLLRDHUP | select.EPOLLHUP | select.EPOLLERR
self.__DEFAULT_ACCEPT_POLLS = select.EPOLLIN | select.EPOLLRDHUP | select.EPOLLHUP | select.EPOLLERR
self.__readCallback = readCallback
self.__writeCallback = writeCallback
self.__acceptCallback = acceptCallback
self.__errorCallback = errorCallback
self.__ReadSelector = select.epoll()
self.__WriteSelector = select.epoll()
self.__AcceptorSelector = select.epoll()
self.__running = True
self.__localExecuter = Scheduler(3)
self.__localExecuter.execute(self.__doReads)
self.__localExecuter.execute(self.__doWrites)
self.__localExecuter.execute(self.__doAcceptor)
def stop(self):
self.__running = False
self.__ReadSelector.close()
self.__WriteSelector.close()
self.__AcceptorSelector.close()
self.__localExecuter.shutdown_now()
def addServer(self, fileno):
try:
self.__AcceptorSelector.register(fileno,
self.__DEFAULT_ACCEPT_POLLS)
except:
noExcept(self.__AcceptorSelector.modify, fileno,
self.__DEFAULT_ACCEPT_POLLS)
def removeServer(self, fileno):
noExcept(self.__AcceptorSelector.unregister, fileno)
def addReader(self, fileno):
try:
self.__ReadSelector.register(fileno, self.__DEFAULT_READ_POLLS)
except:
noExcept(self.__ReadSelector.modify, fileno,
self.__DEFAULT_READ_POLLS)
def removeReader(self, fileno):
noExcept(self.__ReadSelector.unregister, fileno)
def addWriter(self, fileno):
try:
self.__WriteSelector.register(fileno, select.EPOLLOUT)
except:
noExcept(self.__WriteSelector.modify, fileno, select.EPOLLOUT)
def removeWriter(self, fileno):
noExcept(self.__WriteSelector.unregister, fileno)
def __doThread(self, t):
while self.__running:
try:
t()
except Exception as e:
self.__log.error("GP Socket Exception: %s: %s" %
(t, sys.exc_info()[0]))
self.__log.error(e)
def __doReads(self):
events = self.__ReadSelector.poll(100)
for fileno, event in events:
try:
if event & select.EPOLLIN:
self.__readCallback(fileno)
if (event & select.EPOLLRDHUP or event & select.EPOLLHUP
or event & select.EPOLLERR):
self.__errorCallback(fileno)
noExcept(self.__ReadSelector.unregister, fileno)
except Exception as e:
self.__log.debug("Read Error: %s" % (sys.exc_info()[0]))
self.__log.debug(e)
if self.__running:
self.__localExecuter.execute(self.__doReads)
def __doWrites(self):
events = self.__WriteSelector.poll(100)
for fileno, event in events:
try:
if event & select.EPOLLOUT:
self.__writeCallback(fileno)
else:
self.__errorCallback(fileno)
noExcept(self.__WriteSelector.unregister, fileno)
except Exception as e:
self.__log.debug("Write Error: %s" % (sys.exc_info()[0]))
self.__log.debug(e)
if self.__running:
self.__localExecuter.execute(self.__doWrites)
def __doAcceptor(self):
events = self.__AcceptorSelector.poll(100)
for fileno, event in events:
try:
if event & select.EPOLLIN:
self.__acceptCallback(fileno)
else:
self.__errorCallback(fileno)
noExcept(self.__WriteSelector.unregister, fileno)
except Exception as e:
self.__log.debug("Accept Error: %s" % (sys.exc_info()[0]))
self.__log.debug(e)
logging.exception("")
if self.__running:
self.__localExecuter.execute(self.__doAcceptor)
def setUp(self):
self.SCH = Scheduler(10)
self.SE = litesockets.SocketExecuter(scheduler=self.SCH)
class SelectSelector():
def __init__(self, readCallback, writeCallback, acceptCallback,
errorCallback):
self.__log = logging.getLogger("root.litesockets.SelectSelector")
self.__log.info("Creating basic select selector for: {}".format(
platform.system()))
self.__readCallback = readCallback
self.__writeCallback = writeCallback
self.__acceptCallback = acceptCallback
self.__errorCallback = errorCallback
self.__readClients = set()
self.__writeClients = set()
self.__acceptServers = set()
self.__nb_readClients = set()
self.__nb_writeClients = set()
self.__nb_acceptServers = set()
self.__writeLock = threading.Condition()
self.__readLock = threading.Condition()
self.__acceptLock = threading.Condition()
self.__nbLock = threading.Condition()
self.__localExecuter = Scheduler(
5) #need 5 thread, all can be blocked at once
self.__running = True
self.__localExecuter.execute(self.__doReads)
self.__localExecuter.execute(self.__doWrites)
self.__localExecuter.execute(self.__doAcceptor)
def stop(self):
self.__running = False
self.__localExecuter.shutdown_now()
def addServer(self, fileno):
self.__acceptLock.acquire()
self.__acceptServers.add(FileNoWrapper(fileno))
self.__acceptLock.release()
def removeServer(self, fileno):
now = FileNoWrapper(fileno)
if now in self.__acceptServers:
self.__acceptServers.remove(now)
def addReader(self, fileno):
now = FileNoWrapper(fileno)
if now in self.__readClients or now in self.__nb_readClients:
return
if self.__readLock.acquire(blocking=False):
self.__readClients.add(now)
self.__readLock.release()
else:
self.__nb_readClients.add(now)
self.__localExecuter.schedule(self.__tmpClientSelect,
delay=0,
recurring=False,
key="SimpleKey")
self.__localExecuter.schedule(self.__update_from_nb_selector,
key="UpdateTask")
def removeReader(self, fileno):
now = FileNoWrapper(fileno)
if now in self.__readClients:
self.__readClients.remove(now)
if now in self.__nb_readClients:
self.__nb_readClients.remove(now)
def addWriter(self, fileno):
now = FileNoWrapper(fileno)
if now in self.__writeClients or now in self.__nb_writeClients:
return
if self.__writeLock.acquire(blocking=False):
self.__writeClients.add(now)
self.__writeLock.release()
else:
self.__nb_writeClients.add(now)
self.__localExecuter.schedule(self.__tmpClientSelect,
key="SimpleKey")
self.__localExecuter.schedule(self.__update_from_nb_selector,
key="UpdateTask")
def removeWriter(self, fileno):
now = FileNoWrapper(fileno)
if now in self.__writeClients:
self.__writeClients.remove(now)
if now in self.__nb_writeClients:
self.__nb_writeClients.remove(now)
def __doThread(self, t):
while self.__running:
try:
t()
except Exception as e:
self.__log.error("GP Socket Exception: %s: %s" %
(t, sys.exc_info()[0]))
self.__log.error(e)
def __update_from_nb_selector(self):
if len(self.__nb_readClients) + len(self.__nb_writeClients) == 0:
return
else:
self.__readLock.acquire()
self.__nbLock.acquire()
for r in self.__nb_readClients:
self.__readClients.add(r)
self.__nb_readClients.clear()
self.__nbLock.release()
self.__readLock.release()
self.__writeLock.acquire()
self.__nbLock.acquire()
for r in self.__nb_writeClients:
self.__writeClients.add(r)
self.__nb_writeClients.clear()
self.__nbLock.release()
self.__writeLock.release()
def __tmpClientSelect(self):
if len(self.__nb_readClients) + len(self.__nb_writeClients) == 0:
return
rlist = []
wlist = []
xlist = []
self.__nbLock.acquire()
try:
rlist, wlist, xlist = select.select(self.__nb_readClients,
self.__nb_writeClients,
self.__readClients, 0.001)
except:
#We sometimes throw here when a client is removed from the set during the loop
pass
for rdy in rlist:
try:
self.__readCallback(rdy.fileno())
except Exception as e:
self.__log.debug("nbRead Error: %s" % (sys.exc_info()[0]))
self.__log.debug(e)
for rdy in wlist:
try:
self.__writeCallback(rdy.fileno())
except Exception as e:
self.__log.debug("nbWrite Error: %s" % (sys.exc_info()[0]))
self.__log.debug(e)
for bad in xlist:
try:
self.__errorCallback(bad.fileno())
except:
self.__log.debug("nberrorCB Error: %s" % (sys.exc_info()[0]))
self.__log.debug(e)
self.__nbLock.release()
if len(self.__nb_readClients) + len(self.__nb_writeClients) > 0:
self.__localExecuter.schedule(self.__tmpClientSelect,
key="SimpleKey")
def __doReads(self):
rlist = []
wlist = []
xlist = []
self.__readLock.acquire()
if len(self.__readClients) > 0:
try:
rlist, wlist, xlist = select.select(self.__readClients, [],
self.__readClients, .1)
except Exception as e:
#We sometimes throw here when a client is removed from the set during the loop
pass
else:
time.sleep(.1)
self.__readLock.release()
for rdy in rlist:
try:
if rdy in self.__readClients:
self.__readCallback(rdy.fileno())
except IOError as e:
if e.errno != errno.EBADF:
self.__log.error("Unknown error in Selector Read")
self.__log.error(e, exc_info=True)
except Exception as e:
self.__log.debug("Read Error: %s" % (sys.exc_info()[0]))
self.__log.debug(e)
for bad in xlist:
try:
self.__errorCallback(bad.fileno())
except:
self.__log.debug("errorCB Error: %s" % (sys.exc_info()[0]))
self.__log.debug(e)
if self.__running:
self.__localExecuter.execute(self.__doReads)
def __doWrites(self):
rlist = []
wlist = []
xlist = []
self.__writeLock.acquire()
if len(self.__writeClients) > 0:
try:
rlist, wlist, xlist = select.select([], self.__writeClients,
[], .1)
except Exception as e:
#We sometimes throw here when a client is removed from the set during the loop
pass
else:
time.sleep(.1)
self.__writeLock.release()
for rdy in wlist:
try:
if rdy in self.__writeClients:
self.__writeCallback(rdy.fileno())
except Exception as e:
self.__log.debug("Write Error: %s" % (sys.exc_info()[0]))
self.__log.debug(e)
self.__writeClients.remove(rdy)
if self.__running:
self.__localExecuter.execute(self.__doWrites)
def __doAcceptor(self):
rlist = []
wlist = []
xlist = []
self.__acceptLock.acquire()
if len(self.__acceptServers) > 0:
try:
rlist, wlist, xlist = select.select(self.__acceptServers, [],
self.__acceptServers, .1)
except:
#We sometimes throw here when a server is removed from the set during the loop
pass
else:
time.sleep(.1)
self.__acceptLock.release()
for bad in xlist:
try:
self.__errorCallback(bad.fileno())
self.__writeClients.remove(bad)
except Exception as e:
self.__log.debug("errorCB Error: %s" % (sys.exc_info()[0]))
self.__log.debug(e)
for rdy in rlist:
try:
if rdy in self.__acceptServers:
self.__acceptCallback(rdy.fileno())
except Exception as e:
self.__log.debug("Accept Error: %s" % (sys.exc_info()[0]))
self.__log.debug(e)
logging.exception("")
self.__writeClients.remove(rdy)
if self.__running:
self.__localExecuter.execute(self.__doAcceptor)