def prepareServer(RequestHandlerClass, pipe, threads, timeout):
'''
Prepare in a process the request handling.
'''
def process(request, address):
RequestHandlerClass(request, address, None)
try:
request.shutdown(socket.SHUT_WR)
except socket.error:
pass # some platforms may raise ENOTCONN here
request.close()
pool = ThreadPoolExecutor(threads)
while True:
if not pipe.poll(timeout): break
else:
data = pipe.recv()
if data is None: break
elif data is True: continue
requestfd, address = data
request = socket.fromfd(rebuild_handle(requestfd), socket.AF_INET,
socket.SOCK_STREAM)
pool.submit(process, request, address)
pool.shutdown(False)
def in_q_cb(self, watcher, revents):
try:
val = self.in_q.get()
#val = self.in_q.get(True,interval)
logging.debug("ServerWorker[{0}:{1}]: Received inQ event!".format(os.getpid(),self.name))
if type(val) == type((1,)):
# Construct a proper socket object from the socket FD
client_socket_handle,client_address = val
client_fd = rebuild_handle(client_socket_handle)
client_socket = socket.fromfd(client_fd, socket.AF_INET, socket.SOCK_STREAM)
logging.debug("ServerWorker[{0}:{1}]: Adding connection [{2}] from [{3}].".format(os.getpid(),self.name,self.client_count,client_address))
self.client_count += 1
self.cnxns[client_address] = Connection(client_socket, client_address, self.loop, self.client_count, self)
self.reset(pyev.EV_READ)
elif type(val) == type("") and val == "quit":
logging.info("ServerWorker[{0}:{1}]: Received quit message!".format(os.getpid(),self.name))
self.stop()
except Queue.Empty:
# Timed-out, carry on
pass
def accept_handler(fd, events):
try:
#n.write("looking into queue\n")
#n.flush()
#io_loop.add_callback(queue_callback)
n = open("/tmp/tTest2", 'a')
n.write("try reading from queue\n")
n.flush()
queue_read = queue.get_nowait()
#n.write(str(qr)+"\n")
n.flush()
connection = rebuild_handle(queue_read[0])
n.write("read from queue\n")
n.flush()
n.write("conn: "+str(connection)+"\n")
n.write("meep"+"\n")
n.flush()
sock = socket.fromfd(connection,socket.AF_INET, socket.SOCK_STREAM)
n.write("queue: "+str(fd)+"\n")
n.write("queue reader: "+str(queue._reader.fileno())+"\n")
n.write("queue writer : "+str(queue._writer.fileno())+"\n")
n.flush()
callback(sock, queue_read[1])
#io_loop.add_callback(accept_handler, sock)
n.write("meep2\n")
n.flush()
except:
pass
def in_q_cb(self, watcher, revents):
try:
val = self.in_q.get()
#val = self.in_q.get(True,interval)
logging.debug("ServerWorker[{0}:{1}]: Received inQ event!".format(os.getpid(),self.name))
if type(val) == type((1,)):
# Construct a proper socket object from the socket FD
client_socket_handle,client_address = val
client_fd = rebuild_handle(client_socket_handle)
client_socket = socket.fromfd(client_fd, socket.AF_INET, socket.SOCK_STREAM)
logging.debug("ServerWorker[{0}:{1}]: Adding connection [{2}] from [{3}].".format(os.getpid(),self.name,self.client_count,client_address))
self.client_count += 1
self.cnxns[client_address] = Connection(client_socket, client_address, self.loop, self.client_count, self)
self.reset(pyev.EV_READ)
elif type(val) == type("") and val == "quit":
if (debug):
logging.info("ServerWorker[{0}:{1}]: Received quit message!".format(os.getpid(),self.name))
self.stop()
except Queue.Empty:
# Timed-out, carry on
pass
def quit(signum, frame):
for each in users:
fd = rebuild_handle(users[each])
quitSock = fromfd(fd, AF_INET, SOCK_STREAM)
quitSock.send("Server terminated.")
quitSock.close()
del users[each]
exit(0)
def run(self):
while not self.kill_received:
try:
h = self.socket_queue.get_nowait()
fd = rebuild_handle(h)
client_socket = socket.fromfd(fd, socket.AF_INET, socket.SOCK_STREAM)
received = client_socket.recv(1024)
print "Recieved on client: ", received
client_socket.close()
except Queue.Empty:
pass
time.sleep(self.SLEEP_INTERVAL)
def start_conn_handling(self):
log.debug('Waiting for connection....')
serialized_socket = self.conn_queue.get()
fd = rebuild_handle(serialized_socket)
connection = socket.fromfd(fd, socket.AF_INET, socket.SOCK_STREAM)
log.info('Handling connection: %r' % connection)
if self.request_handler:
self.request_handler(connection)
else:
self.echo_handler(connection)
log.info('Handler: Finished successfully.')
def start_conn_handling(self):
log.debug('Waiting for connection....')
serialized_socket = self.conn_queue.get()
fd = rebuild_handle(serialized_socket)
connection = socket.fromfd(fd, socket.AF_INET, socket.SOCK_STREAM)
log.info('Handling connection: %r' % connection)
if self.request_handler:
self.request_handler(connection)
else:
self.echo_handler(connection)
log.info('Handler: Finished successfully.')
def __init__(self, config, clientSocket):
super(Engine, self).__init__()
self.clientSocket = rebuild_handle(clientSocket)
self.clientSocket = socket.fromfd(self.clientSocket,
socket.AF_INET,
socket.SOCK_STREAM)
self.clientSocket.setsockopt(socket.IPPROTO_TCP,
socket.TCP_NODELAY,
1)
self.clientSocket.setsockopt(socket.SOL_SOCKET,
socket.SO_KEEPALIVE,
1)
self.clientSocket.settimeout(None)
self.config = config
self.kill_received = False
self.logger = logging.getLogger('pymp.engine')
if len(self.logger.handlers) == 0:
streamHandler = logging.StreamHandler(sys.stdout)
if int(config['log']['verbose']) > 2:
self.logger.setLevel(logging.DEBUG)
formatter = EngineLogFormat('[%(asctime)s %(process)d '
+ '%(filename)s:%(lineno)d] '
+ '%(message)s')
elif int(config['log']['verbose']) > 1:
self.logger.setLevel(logging.INFO)
formatter = EngineLogFormat('[%(asctime)s %(process)d] '
+ '%(message)s')
else:
self.logger.setLevel(logging.WARNING)
formatter = logging.Formatter('[%(asctime)s %(process)d] '
+ '%(filename)s:%(lineno)d '
+ '%(message)s')
streamHandler.setFormatter(formatter)
streamHandler.setLevel(logging.DEBUG)
self.logger.addHandler(streamHandler)
self.logger.info('Accepted connection')
if 'Proxy' in config['plugins']['enabled']:
from plugins.proxy import Proxy
self.plugins['Proxy'] = Proxy()
def run(self):
while not self.kill_received:
try:
h = self.socket_queue.get()
fd = rebuild_handle(h)
client_socket = socket.fromfd(fd, socket.AF_INET,
socket.SOCK_STREAM)
print self.recvall(
client_socket), "on", client_socket.getsockname()[1]
client_socket.close()
except Queue.Empty:
print "WHAT1?"
pass
except Exception as e:
print "WHAT2?"
pass
def __init__(self, config, clientSocket):
super(Engine, self).__init__()
self.clientSocket = rebuild_handle(clientSocket)
self.clientSocket = socket.fromfd(self.clientSocket, socket.AF_INET,
socket.SOCK_STREAM)
self.clientSocket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
self.clientSocket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
self.clientSocket.settimeout(None)
self.config = config
self.kill_received = False
self.logger = logging.getLogger('pymp.engine')
if len(self.logger.handlers) == 0:
streamHandler = logging.StreamHandler(sys.stdout)
if int(config['log']['verbose']) > 2:
self.logger.setLevel(logging.DEBUG)
formatter = EngineLogFormat('[%(asctime)s %(process)d ' +
'%(filename)s:%(lineno)d] ' +
'%(message)s')
elif int(config['log']['verbose']) > 1:
self.logger.setLevel(logging.INFO)
formatter = EngineLogFormat('[%(asctime)s %(process)d] ' +
'%(message)s')
else:
self.logger.setLevel(logging.WARNING)
formatter = logging.Formatter('[%(asctime)s %(process)d] ' +
'%(filename)s:%(lineno)d ' +
'%(message)s')
streamHandler.setFormatter(formatter)
streamHandler.setLevel(logging.DEBUG)
self.logger.addHandler(streamHandler)
self.logger.info('Accepted connection')
if 'Proxy' in config['plugins']['enabled']:
from plugins.proxy import Proxy
self.plugins['Proxy'] = Proxy()
def worker_routine(self, reduced_socket):
fd = rebuild_handle(reduced_socket)
sock = socket.fromfd(fd, socket.AF_INET, socket.SOCK_STREAM)
mp.forking.close(fd)
if self._key_storage:
sock = Connection(self._ssl_context, sock)
sock.setup_ssl()
sock.set_accept_state()
sock.accept_ssl()
socket_proc = SocketProcessor(sock)
try:
self.process(socket_proc)
socket_proc.close_socket()
except Exception, err:
socket_proc.close_socket(force=True)
raise err
def handle_users(u):
users = u
signal(SIGTERM, quit)
signal(SIGINT, quit)
sock = socket(AF_INET, SOCK_STREAM)
sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
sock.bind((host, port))
sock.listen(1)
while True:
clntSock, clntAddr = sock.accept()
clntFile = clntSock.makefile("r", 0)
message = clntFile.readline().strip()
action, username = message.split()
if (action == "ENTER"):
pickled_socket = reduce_handle(clntSock.fileno())
users[username] = pickled_socket
print "{"
for keys, values in users.items():
print keys + ":", values
print "}"
elif (action == "EXIT"):
fd = rebuild_handle(users[username])
quitSock = fromfd(fd, AF_INET, SOCK_STREAM)
quitSock.close()
del users[username]
clntSock.close()
print "{"
for keys, values in users.items():
print keys + ": ", values
print "}"
clntFile.close()
def run(self):
while not self.kill_received:
try:
#If you used pipe, then recieve as below
#h=pipe.recv()
#else dequeue
h = self.socket_queue.get_nowait()
fd = rebuild_handle(h)
client_socket = socket.fromfd(fd, socket.AF_INET,
socket.SOCK_STREAM)
#client_socket.send("hellofromtheworkerprocess\r\n")
received = client_socket.recv(1024)
print "Recieved on client: ", received
client_socket.close()
except Queue.Empty:
pass
#Dummy timer
time.sleep(self.SLEEP_INTERVAL)
def prepareServer(RequestHandlerClass, pipe, threads, timeout):
'''
Prepare in a process the request handling.
'''
def process(request, address):
RequestHandlerClass(request, address, None)
try: request.shutdown(socket.SHUT_WR)
except socket.error: pass # some platforms may raise ENOTCONN here
request.close()
pool = ThreadPoolExecutor(threads)
while True:
if not pipe.poll(timeout): break
else:
data = pipe.recv()
if data is None: break
elif data is True: continue
requestfd, address = data
request = socket.fromfd(rebuild_handle(requestfd), socket.AF_INET, socket.SOCK_STREAM)
pool.submit(process, request, address)
pool.shutdown(False)
def run(self):
try:
while True:
#new client
pipeNum = 0
while self.childPipe.poll():
clientHandler = self.childPipe.recv()
clientFileno = reduction.rebuild_handle(clientHandler)
clientSocket = socket.fromfd(clientFileno, socket.AF_INET,
socket.SOCK_STREAM)
clientSocket.setblocking(False)
self.workerEpoll.register(clientSocket.fileno(),
select.EPOLLIN)
self.connections[clientSocket.fileno()] = clientSocket
#control increasing client
pipeNum += 1
if pipeNum > 10:
break
#new events
events = self.workerEpoll.poll(0.001)
if not events:
continue
for fd, event in events:
if event & select.EPOLLIN:
#recvd data from client
if self.requests.has_key(fd):
request = self.requests[fd]
else:
request = Request()
self.requests[fd] = request
data = self.connections[fd].recv(1024)
if not data:
self.workerEpoll.modify(fd, 0)
self.connections[fd].shutdown(socket.SHUT_RDWR)
continue
request.rawInStr += data
flag = parseHttpHeader(request)
if flag == 0:
handle = self.reqHandler()
handle.handle(request)
self.workerEpoll.modify(fd, select.EPOLLOUT)
elif flag == 1:
continue
else:
self.workerEpoll.modify(fd, 0)
self.connections[fd].shutdown(socket.SHUT_RDWR)
elif event & select.EPOLLOUT:
#write data to client
if self.requests.has_key(fd):
request = self.requests[fd]
while len(request.rawOutStr) > 0:
sendBytes = self.connections[fd].send(
request.rawOutStr)
request.rawOutStr = request.rawOutStr[
sendBytes:]
self.workerEpoll.modify(fd, 0)
self.connections[fd].shutdown(socket.SHUT_RDWR)
elif event & select.EPOLLHUP:
#client close connection
self.close_client(fd)
except Exception, e:
print "stop worker..."
self.childPipe.close()
for fd in self.connections.keys():
self.workerEpoll.unregister(fd)
self.connections[fd].close()
del self.connections[fd]
print "end clean..."
def run(idx, client_queue, read_queue, write_queue, debug):
logger = Logger(debug)
logger.log("Start handler {0}".format(idx))
inputs = []
outputs = []
server_connections = {}
connection_ids = {}
try:
while True:
try:
if len(server_connections) == 0:
h = client_queue.get()
else:
h = client_queue.get(False)
except queue.Empty:
pass
except Exception as ex:
logger.log(ex)
else:
fd = rebuild_handle(h)
client = socket.fromfd(fd, socket.AF_INET, socket.SOCK_STREAM)
logger.log('Handler {0} got new client {1}'.format(
idx, client.getpeername()))
inputs.append(client)
sc = Connection(client)
server_connections[client] = sc
connection_ids[sc.id] = sc
try:
messages = write_queue.get(False)
except queue.Empty:
pass
else:
if len(messages) > 0:
logger.log(
'Handler {0}: add {1} messages to write buffer'.format(
idx, messages))
for message in messages:
connection = connection_ids[message["id"]]
add_messages_to_write_buffer(idx, connection, message,
logger)
#if we have message to sent to server, add to outputs
if not connection.connection in outputs:
logger.log(
'Handler {0}: add client {1} to write select list'.
format(idx, connection.connection))
outputs.append(client)
if connection.connection in inputs:
inputs.remove(connection.connection)
if not inputs and not outputs:
continue
#logger.log('Handler {0}: inputs={1}, outputs={2}'.format(idx, len(inputs), len(outputs)))
readable, writable, exceptional = select.select(
inputs, outputs, inputs)
#logger.log('Handler {0}: readable {1}, writable {2}, exceptions {3}'.format(idx, len(readable), len(writable), len(exceptional)))
read_messages = []
for client in readable:
try:
logger.log('Handler {0} read data from client {1}'.format(
idx, client.getpeername()))
messages = read(idx, server_connections[client], logger)
logger.log('Handler {0}: processed {1} messages'.format(
idx, len(messages)))
#add ID for client in message dictionary
for message in messages:
message["id"] = server_connections[client].id
if len(messages) > 0:
for message in messages:
read_messages.append(message)
if not client in outputs:
outputs.append(client)
logger.log('Handler {0}: exit read for client {1}'.format(
idx, client.getpeername()))
except socket.error as error:
logger.log("Handler {0}: {1}".format(idx, repr(error)))
if client in inputs:
inputs.remove(client)
if client in outputs:
outputs.remove(client)
del connection_ids[server_connections[client].id]
del server_connections[client]
except ValueError as error:
logger.log('Handler {0}: client {1} disconnected'.format(
idx, client.getpeername()))
if client in outputs:
outputs.remove(client)
if client in inputs:
inputs.remove(client)
del connection_ids[server_connections[client].id]
del server_connections[client]
client.close()
if len(read_messages) > 0:
read_queue.put(read_messages)
for client in writable:
try:
if client in server_connections and len(
server_connections[client].write_buffer) > 0:
logger.log(
'Handler {0} write data to client {1}'.format(
idx, client.getpeername()))
write(idx, server_connections[client], logger)
if len(server_connections[client].write_buffer) == 0:
logger.log(
'Handler {0}: write buffer for client {1} is empty'
.format(idx, client))
#if buffer is empty go to reading
if client in outputs:
outputs.remove(client)
#if we wrote whole buffer begin reading
if not client in inputs:
inputs.append(client)
else:
logger.log(
'Handler {0}: looks like nothing was sent for client {1}'
.format(idx, client))
#else:
#print('Connection write buffer empty')
except socket.error as error:
logger.log("Handler {0}: {1}".format(idx, repr(error)))
if client in inputs:
inputs.remove(client)
if client in outputs:
outputs.remove(client)
del connection_ids[server_connections[client].id]
del server_connections[client]
except ValueError as error:
logger.log('Handler {0}: client {1} disconnected'.format(
idx, client.getpeername()))
if client in outputs:
outputs.remove(client)
if client in inputs:
inputs.remove(client)
del connection_ids[server_connections[client].id]
del server_connections[client]
client.close()
# Handle "exceptional conditions"
for client in exceptional:
logger.log('Handling exceptional condition for'.format(
client.getpeername()))
# Stop listening for input on the connection
inputs.remove(client)
outputs.remove(client)
client.close()
logger.log('Handler {0}: exit loop'.format(idx))
except Exception as ex:
logger.log('Handler {0} exception: {1}'.format(idx, ex))
def _rebuild_socket(reduced_handle, family, type_, proto):
handle = rebuild_handle(reduced_handle)
s = fromfd(handle, family, type_, proto)
close(handle)
return s
def run(idx, client_queue, debug):
logger = Logger(debug)
logger.log("Start dispatcher {0}".format(idx))
inputs = []
outputs = []
server_connections = {}
while True:
#add new client to input list
start_ = datetime.datetime.now()
try:
h = client_queue.get_nowait()
fd = rebuild_handle(h)
client = socket.fromfd(fd, socket.AF_INET, socket.SOCK_STREAM)
except queue.Empty:
pass
except Exception as ex:
logger.log(ex)
else:
logger.log('Dispatcher {0} got new client {1}'.format(
idx, client.getpeername()))
inputs.append(client)
sc = ServerConnection(client)
server_connections[client] = sc
if not inputs:
continue
logger.log('Dispatcher {0}: inputs={1}, outputs={2}'.format(
idx, len(inputs), len(outputs)))
readable, writable, exceptional = select.select(
inputs, outputs, inputs)
logger.log(
'Dispatcher {0}: readable {1}, writable {2}, exceptions {3}'.
format(idx, len(readable), len(writable), len(exceptional)))
for client in readable:
try:
logger.log('Dispatcher {0} read data from client {1}'.format(
idx, client.getpeername()))
messages = read(idx, server_connections[client], logger)
logger.log('Dispatcher{0}: processed {1} messages'.format(
idx, len(messages)))
messages = process_messages(messages)
add_messages_to_write_buffer(idx, server_connections[client],
messages, logger)
dummy_messages(server_connections[client])
if not client in outputs:
logger.log(
'Dispatcher {0}: add client {1} to write select list'.
format(idx, client))
outputs.append(client)
logger.log('Dispatcher {0}: exit read for client {1}'.format(
idx, client.getpeername()))
except socket.error as error:
logger.log(repr(error))
inputs.remove(client)
# TODO handle message queue
except ValueError as error:
logger.log('Client {0} disconnected'.format(
client.getpeername()))
if client in outputs:
outputs.remove(client)
if client in inputs:
inputs.remove(client)
client.close()
del server_connections[client]
else:
if client not in outputs:
outputs.append(client)
for client in writable:
try:
if client in server_connections:
write(idx, server_connections[client], logger)
if len(server_connections[client].write_buffer) == 0:
logger.log(
'Dispatcher {0}: write buffer for client {1} is empty'
.format(idx, client))
outputs.remove(client)
except socket.error as error:
logger.log(repr(error))
inputs.remove(client)
# TODO handle message queue
except ValueError as error:
logger.log('Client {0} disconnected'.format(
client.getpeername()))
if client in outputs:
outputs.remove(client)
if client in inputs:
inputs.remove(client)
client.close()
del server_connections[client]
# Handle "exceptional conditions"
for client in exceptional:
logger.log('Handling exceptional condition for'.format(
client.getpeername()))
# Stop listening for input on the connection
inputs.remove(client)
outputs.remove(client)
client.close()
end_ = datetime.datetime.now()
logger.log('Dispatcher {0}: loop took {1} ms'.format(
idx, (end_ - start_).microseconds / 1000))
def deserialise_socket(serialised):
fd = reduction.rebuild_handle(serialised)
socket_object = socket.fromfd(fd, socket.AF_INET, socket.SOCK_STREAM)
if six.PY3:
return socket_object
return socket.socket(_sock=socket_object)
def _rebuild_socket(reduced_handle, family, type_, proto):
handle = rebuild_handle(reduced_handle)
s = fromfd(handle, family, type_, proto)
close(handle)
return s
def run(self):
try:
while True:
#new client
pipeNum = 0
while self.childPipe.poll():
clientHandler = self.childPipe.recv();
clientFileno = reduction.rebuild_handle(clientHandler);
clientSocket = socket.fromfd(clientFileno, socket.AF_INET, socket.SOCK_STREAM);
clientSocket.setblocking(False);
self.workerEpoll.register(clientSocket.fileno(), select.EPOLLIN);
self.connections[clientSocket.fileno()] = clientSocket;
#control increasing client
pipeNum += 1;
if pipeNum > 10:
break;
#new events
events = self.workerEpoll.poll(0.001);
if not events:
continue;
for fd, event in events:
if event & select.EPOLLIN:
#recvd data from client
if self.requests.has_key(fd):
request = self.requests[fd];
else:
request = Request();
self.requests[fd] = request;
data = self.connections[fd].recv(1024);
if not data:
self.workerEpoll.modify(fd, 0);
self.connections[fd].shutdown(socket.SHUT_RDWR);
continue;
request.rawInStr += data;
flag = parseHttpHeader(request)
if flag == 0:
handle = self.reqHandler();
handle.handle(request);
self.workerEpoll.modify(fd, select.EPOLLOUT);
elif flag == 1:
continue;
else:
self.workerEpoll.modify(fd, 0);
self.connections[fd].shutdown(socket.SHUT_RDWR);
elif event & select.EPOLLOUT:
#write data to client
if self.requests.has_key(fd):
request = self.requests[fd];
while len(request.rawOutStr) > 0:
sendBytes = self.connections[fd].send(request.rawOutStr);
request.rawOutStr = request.rawOutStr[sendBytes:];
self.workerEpoll.modify(fd, 0);
self.connections[fd].shutdown(socket.SHUT_RDWR);
elif event & select.EPOLLHUP:
#client close connection
self.close_client(fd);
except Exception, e:
print "stop worker...";
self.childPipe.close();
for fd in self.connections.keys():
self.workerEpoll.unregister(fd);
self.connections[fd].close();
del self.connections[fd];
print "end clean..."
