def prepareServer(RequestHandlerClass, pipe, threads, timeout): ''' Prepare in a process the request handling. ''' def process(request, address): RequestHandlerClass(request, address, None) try: request.shutdown(socket.SHUT_WR) except socket.error: pass # some platforms may raise ENOTCONN here request.close() pool = ThreadPoolExecutor(threads) while True: if not pipe.poll(timeout): break else: data = pipe.recv() if data is None: break elif data is True: continue requestfd, address = data request = socket.fromfd(rebuild_handle(requestfd), socket.AF_INET, socket.SOCK_STREAM) pool.submit(process, request, address) pool.shutdown(False)
def in_q_cb(self, watcher, revents): try: val = self.in_q.get() #val = self.in_q.get(True,interval) logging.debug("ServerWorker[{0}:{1}]: Received inQ event!".format(os.getpid(),self.name)) if type(val) == type((1,)): # Construct a proper socket object from the socket FD client_socket_handle,client_address = val client_fd = rebuild_handle(client_socket_handle) client_socket = socket.fromfd(client_fd, socket.AF_INET, socket.SOCK_STREAM) logging.debug("ServerWorker[{0}:{1}]: Adding connection [{2}] from [{3}].".format(os.getpid(),self.name,self.client_count,client_address)) self.client_count += 1 self.cnxns[client_address] = Connection(client_socket, client_address, self.loop, self.client_count, self) self.reset(pyev.EV_READ) elif type(val) == type("") and val == "quit": logging.info("ServerWorker[{0}:{1}]: Received quit message!".format(os.getpid(),self.name)) self.stop() except Queue.Empty: # Timed-out, carry on pass
def accept_handler(fd, events): try: #n.write("looking into queue\n") #n.flush() #io_loop.add_callback(queue_callback) n = open("/tmp/tTest2", 'a') n.write("try reading from queue\n") n.flush() queue_read = queue.get_nowait() #n.write(str(qr)+"\n") n.flush() connection = rebuild_handle(queue_read[0]) n.write("read from queue\n") n.flush() n.write("conn: "+str(connection)+"\n") n.write("meep"+"\n") n.flush() sock = socket.fromfd(connection,socket.AF_INET, socket.SOCK_STREAM) n.write("queue: "+str(fd)+"\n") n.write("queue reader: "+str(queue._reader.fileno())+"\n") n.write("queue writer : "+str(queue._writer.fileno())+"\n") n.flush() callback(sock, queue_read[1]) #io_loop.add_callback(accept_handler, sock) n.write("meep2\n") n.flush() except: pass
def in_q_cb(self, watcher, revents): try: val = self.in_q.get() #val = self.in_q.get(True,interval) logging.debug("ServerWorker[{0}:{1}]: Received inQ event!".format(os.getpid(),self.name)) if type(val) == type((1,)): # Construct a proper socket object from the socket FD client_socket_handle,client_address = val client_fd = rebuild_handle(client_socket_handle) client_socket = socket.fromfd(client_fd, socket.AF_INET, socket.SOCK_STREAM) logging.debug("ServerWorker[{0}:{1}]: Adding connection [{2}] from [{3}].".format(os.getpid(),self.name,self.client_count,client_address)) self.client_count += 1 self.cnxns[client_address] = Connection(client_socket, client_address, self.loop, self.client_count, self) self.reset(pyev.EV_READ) elif type(val) == type("") and val == "quit": if (debug): logging.info("ServerWorker[{0}:{1}]: Received quit message!".format(os.getpid(),self.name)) self.stop() except Queue.Empty: # Timed-out, carry on pass
def quit(signum, frame): for each in users: fd = rebuild_handle(users[each]) quitSock = fromfd(fd, AF_INET, SOCK_STREAM) quitSock.send("Server terminated.") quitSock.close() del users[each] exit(0)
def run(self): while not self.kill_received: try: h = self.socket_queue.get_nowait() fd = rebuild_handle(h) client_socket = socket.fromfd(fd, socket.AF_INET, socket.SOCK_STREAM) received = client_socket.recv(1024) print "Recieved on client: ", received client_socket.close() except Queue.Empty: pass time.sleep(self.SLEEP_INTERVAL)
def start_conn_handling(self): log.debug('Waiting for connection....') serialized_socket = self.conn_queue.get() fd = rebuild_handle(serialized_socket) connection = socket.fromfd(fd, socket.AF_INET, socket.SOCK_STREAM) log.info('Handling connection: %r' % connection) if self.request_handler: self.request_handler(connection) else: self.echo_handler(connection) log.info('Handler: Finished successfully.')
def __init__(self, config, clientSocket): super(Engine, self).__init__() self.clientSocket = rebuild_handle(clientSocket) self.clientSocket = socket.fromfd(self.clientSocket, socket.AF_INET, socket.SOCK_STREAM) self.clientSocket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) self.clientSocket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) self.clientSocket.settimeout(None) self.config = config self.kill_received = False self.logger = logging.getLogger('pymp.engine') if len(self.logger.handlers) == 0: streamHandler = logging.StreamHandler(sys.stdout) if int(config['log']['verbose']) > 2: self.logger.setLevel(logging.DEBUG) formatter = EngineLogFormat('[%(asctime)s %(process)d ' + '%(filename)s:%(lineno)d] ' + '%(message)s') elif int(config['log']['verbose']) > 1: self.logger.setLevel(logging.INFO) formatter = EngineLogFormat('[%(asctime)s %(process)d] ' + '%(message)s') else: self.logger.setLevel(logging.WARNING) formatter = logging.Formatter('[%(asctime)s %(process)d] ' + '%(filename)s:%(lineno)d ' + '%(message)s') streamHandler.setFormatter(formatter) streamHandler.setLevel(logging.DEBUG) self.logger.addHandler(streamHandler) self.logger.info('Accepted connection') if 'Proxy' in config['plugins']['enabled']: from plugins.proxy import Proxy self.plugins['Proxy'] = Proxy()
def run(self): while not self.kill_received: try: h = self.socket_queue.get() fd = rebuild_handle(h) client_socket = socket.fromfd(fd, socket.AF_INET, socket.SOCK_STREAM) print self.recvall( client_socket), "on", client_socket.getsockname()[1] client_socket.close() except Queue.Empty: print "WHAT1?" pass except Exception as e: print "WHAT2?" pass
def worker_routine(self, reduced_socket): fd = rebuild_handle(reduced_socket) sock = socket.fromfd(fd, socket.AF_INET, socket.SOCK_STREAM) mp.forking.close(fd) if self._key_storage: sock = Connection(self._ssl_context, sock) sock.setup_ssl() sock.set_accept_state() sock.accept_ssl() socket_proc = SocketProcessor(sock) try: self.process(socket_proc) socket_proc.close_socket() except Exception, err: socket_proc.close_socket(force=True) raise err
def handle_users(u): users = u signal(SIGTERM, quit) signal(SIGINT, quit) sock = socket(AF_INET, SOCK_STREAM) sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1) sock.bind((host, port)) sock.listen(1) while True: clntSock, clntAddr = sock.accept() clntFile = clntSock.makefile("r", 0) message = clntFile.readline().strip() action, username = message.split() if (action == "ENTER"): pickled_socket = reduce_handle(clntSock.fileno()) users[username] = pickled_socket print "{" for keys, values in users.items(): print keys + ":", values print "}" elif (action == "EXIT"): fd = rebuild_handle(users[username]) quitSock = fromfd(fd, AF_INET, SOCK_STREAM) quitSock.close() del users[username] clntSock.close() print "{" for keys, values in users.items(): print keys + ": ", values print "}" clntFile.close()
def run(self): while not self.kill_received: try: #If you used pipe, then recieve as below #h=pipe.recv() #else dequeue h = self.socket_queue.get_nowait() fd = rebuild_handle(h) client_socket = socket.fromfd(fd, socket.AF_INET, socket.SOCK_STREAM) #client_socket.send("hellofromtheworkerprocess\r\n") received = client_socket.recv(1024) print "Recieved on client: ", received client_socket.close() except Queue.Empty: pass #Dummy timer time.sleep(self.SLEEP_INTERVAL)
def run(self): try: while True: #new client pipeNum = 0 while self.childPipe.poll(): clientHandler = self.childPipe.recv() clientFileno = reduction.rebuild_handle(clientHandler) clientSocket = socket.fromfd(clientFileno, socket.AF_INET, socket.SOCK_STREAM) clientSocket.setblocking(False) self.workerEpoll.register(clientSocket.fileno(), select.EPOLLIN) self.connections[clientSocket.fileno()] = clientSocket #control increasing client pipeNum += 1 if pipeNum > 10: break #new events events = self.workerEpoll.poll(0.001) if not events: continue for fd, event in events: if event & select.EPOLLIN: #recvd data from client if self.requests.has_key(fd): request = self.requests[fd] else: request = Request() self.requests[fd] = request data = self.connections[fd].recv(1024) if not data: self.workerEpoll.modify(fd, 0) self.connections[fd].shutdown(socket.SHUT_RDWR) continue request.rawInStr += data flag = parseHttpHeader(request) if flag == 0: handle = self.reqHandler() handle.handle(request) self.workerEpoll.modify(fd, select.EPOLLOUT) elif flag == 1: continue else: self.workerEpoll.modify(fd, 0) self.connections[fd].shutdown(socket.SHUT_RDWR) elif event & select.EPOLLOUT: #write data to client if self.requests.has_key(fd): request = self.requests[fd] while len(request.rawOutStr) > 0: sendBytes = self.connections[fd].send( request.rawOutStr) request.rawOutStr = request.rawOutStr[ sendBytes:] self.workerEpoll.modify(fd, 0) self.connections[fd].shutdown(socket.SHUT_RDWR) elif event & select.EPOLLHUP: #client close connection self.close_client(fd) except Exception, e: print "stop worker..." self.childPipe.close() for fd in self.connections.keys(): self.workerEpoll.unregister(fd) self.connections[fd].close() del self.connections[fd] print "end clean..."
def run(idx, client_queue, read_queue, write_queue, debug): logger = Logger(debug) logger.log("Start handler {0}".format(idx)) inputs = [] outputs = [] server_connections = {} connection_ids = {} try: while True: try: if len(server_connections) == 0: h = client_queue.get() else: h = client_queue.get(False) except queue.Empty: pass except Exception as ex: logger.log(ex) else: fd = rebuild_handle(h) client = socket.fromfd(fd, socket.AF_INET, socket.SOCK_STREAM) logger.log('Handler {0} got new client {1}'.format( idx, client.getpeername())) inputs.append(client) sc = Connection(client) server_connections[client] = sc connection_ids[sc.id] = sc try: messages = write_queue.get(False) except queue.Empty: pass else: if len(messages) > 0: logger.log( 'Handler {0}: add {1} messages to write buffer'.format( idx, messages)) for message in messages: connection = connection_ids[message["id"]] add_messages_to_write_buffer(idx, connection, message, logger) #if we have message to sent to server, add to outputs if not connection.connection in outputs: logger.log( 'Handler {0}: add client {1} to write select list'. format(idx, connection.connection)) outputs.append(client) if connection.connection in inputs: inputs.remove(connection.connection) if not inputs and not outputs: continue #logger.log('Handler {0}: inputs={1}, outputs={2}'.format(idx, len(inputs), len(outputs))) readable, writable, exceptional = select.select( inputs, outputs, inputs) #logger.log('Handler {0}: readable {1}, writable {2}, exceptions {3}'.format(idx, len(readable), len(writable), len(exceptional))) read_messages = [] for client in readable: try: logger.log('Handler {0} read data from client {1}'.format( idx, client.getpeername())) messages = read(idx, server_connections[client], logger) logger.log('Handler {0}: processed {1} messages'.format( idx, len(messages))) #add ID for client in message dictionary for message in messages: message["id"] = server_connections[client].id if len(messages) > 0: for message in messages: read_messages.append(message) if not client in outputs: outputs.append(client) logger.log('Handler {0}: exit read for client {1}'.format( idx, client.getpeername())) except socket.error as error: logger.log("Handler {0}: {1}".format(idx, repr(error))) if client in inputs: inputs.remove(client) if client in outputs: outputs.remove(client) del connection_ids[server_connections[client].id] del server_connections[client] except ValueError as error: logger.log('Handler {0}: client {1} disconnected'.format( idx, client.getpeername())) if client in outputs: outputs.remove(client) if client in inputs: inputs.remove(client) del connection_ids[server_connections[client].id] del server_connections[client] client.close() if len(read_messages) > 0: read_queue.put(read_messages) for client in writable: try: if client in server_connections and len( server_connections[client].write_buffer) > 0: logger.log( 'Handler {0} write data to client {1}'.format( idx, client.getpeername())) write(idx, server_connections[client], logger) if len(server_connections[client].write_buffer) == 0: logger.log( 'Handler {0}: write buffer for client {1} is empty' .format(idx, client)) #if buffer is empty go to reading if client in outputs: outputs.remove(client) #if we wrote whole buffer begin reading if not client in inputs: inputs.append(client) else: logger.log( 'Handler {0}: looks like nothing was sent for client {1}' .format(idx, client)) #else: #print('Connection write buffer empty') except socket.error as error: logger.log("Handler {0}: {1}".format(idx, repr(error))) if client in inputs: inputs.remove(client) if client in outputs: outputs.remove(client) del connection_ids[server_connections[client].id] del server_connections[client] except ValueError as error: logger.log('Handler {0}: client {1} disconnected'.format( idx, client.getpeername())) if client in outputs: outputs.remove(client) if client in inputs: inputs.remove(client) del connection_ids[server_connections[client].id] del server_connections[client] client.close() # Handle "exceptional conditions" for client in exceptional: logger.log('Handling exceptional condition for'.format( client.getpeername())) # Stop listening for input on the connection inputs.remove(client) outputs.remove(client) client.close() logger.log('Handler {0}: exit loop'.format(idx)) except Exception as ex: logger.log('Handler {0} exception: {1}'.format(idx, ex))
def _rebuild_socket(reduced_handle, family, type_, proto): handle = rebuild_handle(reduced_handle) s = fromfd(handle, family, type_, proto) close(handle) return s
def run(idx, client_queue, debug): logger = Logger(debug) logger.log("Start dispatcher {0}".format(idx)) inputs = [] outputs = [] server_connections = {} while True: #add new client to input list start_ = datetime.datetime.now() try: h = client_queue.get_nowait() fd = rebuild_handle(h) client = socket.fromfd(fd, socket.AF_INET, socket.SOCK_STREAM) except queue.Empty: pass except Exception as ex: logger.log(ex) else: logger.log('Dispatcher {0} got new client {1}'.format( idx, client.getpeername())) inputs.append(client) sc = ServerConnection(client) server_connections[client] = sc if not inputs: continue logger.log('Dispatcher {0}: inputs={1}, outputs={2}'.format( idx, len(inputs), len(outputs))) readable, writable, exceptional = select.select( inputs, outputs, inputs) logger.log( 'Dispatcher {0}: readable {1}, writable {2}, exceptions {3}'. format(idx, len(readable), len(writable), len(exceptional))) for client in readable: try: logger.log('Dispatcher {0} read data from client {1}'.format( idx, client.getpeername())) messages = read(idx, server_connections[client], logger) logger.log('Dispatcher{0}: processed {1} messages'.format( idx, len(messages))) messages = process_messages(messages) add_messages_to_write_buffer(idx, server_connections[client], messages, logger) dummy_messages(server_connections[client]) if not client in outputs: logger.log( 'Dispatcher {0}: add client {1} to write select list'. format(idx, client)) outputs.append(client) logger.log('Dispatcher {0}: exit read for client {1}'.format( idx, client.getpeername())) except socket.error as error: logger.log(repr(error)) inputs.remove(client) # TODO handle message queue except ValueError as error: logger.log('Client {0} disconnected'.format( client.getpeername())) if client in outputs: outputs.remove(client) if client in inputs: inputs.remove(client) client.close() del server_connections[client] else: if client not in outputs: outputs.append(client) for client in writable: try: if client in server_connections: write(idx, server_connections[client], logger) if len(server_connections[client].write_buffer) == 0: logger.log( 'Dispatcher {0}: write buffer for client {1} is empty' .format(idx, client)) outputs.remove(client) except socket.error as error: logger.log(repr(error)) inputs.remove(client) # TODO handle message queue except ValueError as error: logger.log('Client {0} disconnected'.format( client.getpeername())) if client in outputs: outputs.remove(client) if client in inputs: inputs.remove(client) client.close() del server_connections[client] # Handle "exceptional conditions" for client in exceptional: logger.log('Handling exceptional condition for'.format( client.getpeername())) # Stop listening for input on the connection inputs.remove(client) outputs.remove(client) client.close() end_ = datetime.datetime.now() logger.log('Dispatcher {0}: loop took {1} ms'.format( idx, (end_ - start_).microseconds / 1000))
def deserialise_socket(serialised): fd = reduction.rebuild_handle(serialised) socket_object = socket.fromfd(fd, socket.AF_INET, socket.SOCK_STREAM) if six.PY3: return socket_object return socket.socket(_sock=socket_object)
def run(self): try: while True: #new client pipeNum = 0 while self.childPipe.poll(): clientHandler = self.childPipe.recv(); clientFileno = reduction.rebuild_handle(clientHandler); clientSocket = socket.fromfd(clientFileno, socket.AF_INET, socket.SOCK_STREAM); clientSocket.setblocking(False); self.workerEpoll.register(clientSocket.fileno(), select.EPOLLIN); self.connections[clientSocket.fileno()] = clientSocket; #control increasing client pipeNum += 1; if pipeNum > 10: break; #new events events = self.workerEpoll.poll(0.001); if not events: continue; for fd, event in events: if event & select.EPOLLIN: #recvd data from client if self.requests.has_key(fd): request = self.requests[fd]; else: request = Request(); self.requests[fd] = request; data = self.connections[fd].recv(1024); if not data: self.workerEpoll.modify(fd, 0); self.connections[fd].shutdown(socket.SHUT_RDWR); continue; request.rawInStr += data; flag = parseHttpHeader(request) if flag == 0: handle = self.reqHandler(); handle.handle(request); self.workerEpoll.modify(fd, select.EPOLLOUT); elif flag == 1: continue; else: self.workerEpoll.modify(fd, 0); self.connections[fd].shutdown(socket.SHUT_RDWR); elif event & select.EPOLLOUT: #write data to client if self.requests.has_key(fd): request = self.requests[fd]; while len(request.rawOutStr) > 0: sendBytes = self.connections[fd].send(request.rawOutStr); request.rawOutStr = request.rawOutStr[sendBytes:]; self.workerEpoll.modify(fd, 0); self.connections[fd].shutdown(socket.SHUT_RDWR); elif event & select.EPOLLHUP: #client close connection self.close_client(fd); except Exception, e: print "stop worker..."; self.childPipe.close(); for fd in self.connections.keys(): self.workerEpoll.unregister(fd); self.connections[fd].close(); del self.connections[fd]; print "end clean..."