def __init__(self, name, sock, peer, logger, max_buffer): self.name = name self.ipv4 = isipv4(sock.getsockname()[0]) self.sock = sock self.peer = peer self.reader = self._read(sock, max_buffer) self.writer = self._write(sock) self.w_buffer = '' self.icap_parser = ICAPParser(configuration={}) self.log = logger # start the _read coroutine self.reader.next()
def __init__(self, name, sock, peer, logger, max_buffer): self.name = name self.ipv4 = isipv4(sock.getsockname()[0]) self.sock = sock self.peer = peer self.reader = self._read(sock,max_buffer) self.writer = self._write(sock) self.w_buffer = '' self.icap_parser = ICAPParser(configuration={}) self.log = logger # start the _read coroutine self.reader.next()
class ICAPClient(object): eor = ['\r\n\r\n', '\n\n'] __slots__ = [ 'name', 'ipv4', 'sock', 'peer', 'reader', 'writer', 'w_buffer', 'icap_parser', 'log' ] def __init__(self, name, sock, peer, logger, max_buffer): self.name = name self.ipv4 = isipv4(sock.getsockname()[0]) self.sock = sock self.peer = peer self.reader = self._read(sock, max_buffer) self.writer = self._write(sock) self.w_buffer = '' self.icap_parser = ICAPParser(configuration={}) self.log = logger # start the _read coroutine self.reader.next() def checkRequest(self, r_buffer, size, seek=0): for eor in self.eor: pos = r_buffer[seek:].find(eor) if pos == -1: continue buff = r_buffer[:seek + pos] if not buff: continue if not count_quotes(buff) % 2: # we have matching pairs return buff + eor, r_buffer[seek + pos + len(eor):], seek seek += pos + len(eor) if size and len(r_buffer) > size: return None, None, None return '', r_buffer, seek def checkChunkSize(self, r_buffer): # return a tuple : bool, length # * the bool is : is there more chunk to come # * the len contains the size of the chunk(s) extracted # a size of None means that we could not decode as it is invalid total_len = 0 while r_buffer: if not '\n' in r_buffer: if len(r_buffer) > 6: # len('FFFF') + len(\r\n) return True, None return True, 0 header, r_buffer = r_buffer.split('\n', 1) len_header = len(header) + 1 if header.endswith('\r'): header = header[:-1] len_eol = 2 else: len_eol = 1 if ';' in header: header = header.split(';', 1)[0] if not ishex(header): return True, None len_chunk = int(header, 16) # 0xFFFF is not enough - coad is complaining :p if len_chunk > 0x100000: return True, None if len_chunk == 0: total_len += len_header return False, total_len else: total = len_chunk + len_eol total_len += total + len_header r_buffer = r_buffer[total:] return True, total_len def _read(self, sock, max_buffer, read_size=64 * 1024): """Coroutine managing data read from the client""" # yield request, content # request is the text that form the request header # content any text which is related to the current request after the headers yield '' r_buffer = '' nb_to_send = 0 content_length = 0 seek = 0 processing = False # mode can be one of : request, chunk, extension, relay # icap : we are reading the icap headers # request : we are reading the request (read all you can until a separator) # extra-headers : we are reading data until a separator # chunked : we are reading chunk-encoded darta # transfer : we are reading as much as requested in remaining # passthrough : read as much as can to be relayed mode = 'icap' while True: try: while True: if not processing: data = sock.recv(read_size) if not data: break # read failed so we abort self.log.debug("<< [%s]" % data.replace('\t', '\\t').replace( '\r', '\\r').replace('\n', '\\n')) r_buffer += data else: processing = False if mode == 'passthrough': r_buffer, tmp = '', [r_buffer] yield [''], [''], tmp continue if nb_to_send: if mode == 'transfer': r_len = len(r_buffer) length = min(r_len, nb_to_send) # do not yield yet if we are chunked since the end of the chunk may # very well be in the rest of the data we just read r_buffer, tmp = r_buffer[length:], [ r_buffer[:length] ] _, extra_size = yield [''], [''], tmp nb_to_send = nb_to_send - length + extra_size # we still have data to read before we can send more. if nb_to_send != 0: continue mode = 'icap' if mode == 'chunked': r_len = len(r_buffer) length = min(r_len, nb_to_send) # do not yield yet if we are chunked since the end of the chunk may # very well be in the rest of the data we just read if r_len <= nb_to_send: r_buffer, tmp = r_buffer[length:], [ r_buffer[:length] ] _, extra_size = yield '', '', tmp nb_to_send = nb_to_send - length + extra_size # we still have data to read before we can send more. if nb_to_send != 0: continue if mode == 'chunked': # sum of the sizes of all chunks in our buffer chunked, new_to_send = self.checkChunkSize( r_buffer[nb_to_send:]) if new_to_send is None: # could not read any chunk (data is invalid) break nb_to_send += new_to_send if chunked: continue mode = 'end-chunk' # seek is only set if we already passed once and found we needed more data to check if mode == 'end-chunk': if r_buffer[nb_to_send:].startswith('\r\n'): nb_to_send += 2 processing = True mode = 'transfer' continue elif r_buffer[nb_to_send:].startswith('\n'): nb_to_send += 1 processing = True mode = 'transfer' continue if not r_buffer[nb_to_send:]: yield [''], [''], [''] continue mode = 'extra-headers' seek = nb_to_send if mode == 'extra-headers': # seek is up to where we know there is no double CRLF related, r_buffer, seek = self.checkRequest( r_buffer, max_buffer, seek) if related is None: # most likely could not find an header break if related: related, tmp = '', [related] yield [''], [''], tmp else: yield [''], [''], [''] continue seek = 0 mode = 'transfer' if mode not in ('icap', 'request'): self.log.error( 'The programmers are monkeys - please give them bananas ..' ) self.log.error('the mode was spelled : [%s]' % mode) self.log.error( '.. if it works, we are lucky - but it may work.') mode = 'icap' if mode == 'icap': # ignore EOL r_buffer = r_buffer.lstrip('\r\n') # check to see if we have read an entire request request, r_buffer, seek = self.checkRequest( r_buffer, max_buffer, seek) if request is None: # most likely could not find a header break if request: icap_request = request parsed_request = self.icap_parser.parseRequest( self.peer, icap_request, '') content_length = parsed_request.content_length if parsed_request is None or parsed_request.contains_body: # we do not (yet) support having content sent to us break if not parsed_request.contains_headers: # we need at least an HTTP header break # no reason to keep this around in memory longer than we need it parsed_request = None request, seek = '', 0 mode = 'request' else: yield [''], [''], [''] continue if mode == 'request': r_len = len(r_buffer) if r_len < content_length: yield [''], [''], [''] continue request, r_buffer = r_buffer[:content_length], r_buffer[ content_length:] seek = 0 processing = True # nb_to_send is how much we expect to need to get the rest of the request icap_request, tmp_icap = '', [icap_request] request, tmp = '', [request] mode, nb_to_send = yield tmp_icap, tmp, [''] # break out of the outer loop as soon as we leave the inner loop # through normal execution break except socket.error, e: if e.args[0] in errno_block: yield [''], [''], [''] else: break yield [None], [None], [None]
class ICAPClient (object): eor = ['\r\n\r\n', '\n\n'] __slots__ = ['name', 'ipv4', 'sock', 'peer', 'reader', 'writer', 'w_buffer', 'icap_parser', 'log'] def __init__(self, name, sock, peer, logger, max_buffer): self.name = name self.ipv4 = isipv4(sock.getsockname()[0]) self.sock = sock self.peer = peer self.reader = self._read(sock,max_buffer) self.writer = self._write(sock) self.w_buffer = '' self.icap_parser = ICAPParser(configuration={}) self.log = logger # start the _read coroutine self.reader.next() def checkRequest (self, r_buffer, size, seek=0): for eor in self.eor: pos = r_buffer[seek:].find(eor) if pos == -1: continue buff = r_buffer[:seek+pos] if not buff: continue if not count_quotes(buff) % 2: # we have matching pairs return buff + eor, r_buffer[seek+pos+len(eor):], seek seek += pos + len(eor) if size and len(r_buffer) > size: return None,None,None return '', r_buffer, seek def checkChunkSize (self, r_buffer): # return a tuple : bool, length # * the bool is : is there more chunk to come # * the len contains the size of the chunk(s) extracted # a size of None means that we could not decode as it is invalid total_len = 0 while r_buffer: if not '\n' in r_buffer: if len(r_buffer) > 6: # len('FFFF') + len(\r\n) return True, None return True, 0 header,r_buffer = r_buffer.split('\n', 1) len_header = len(header) + 1 if header.endswith('\r'): header = header[:-1] len_eol = 2 else: len_eol = 1 if ';' in header: header = header.split(';',1)[0] if not ishex(header): return True,None len_chunk = int(header, 16) # 0xFFFF is not enough - coad is complaining :p if len_chunk > 0x100000: return True,None if len_chunk == 0: total_len += len_header return False, total_len else: total = len_chunk + len_eol total_len += total + len_header r_buffer = r_buffer[total:] return True,total_len def _read (self, sock, max_buffer, read_size=64*1024): """Coroutine managing data read from the client""" # yield request, content # request is the text that form the request header # content any text which is related to the current request after the headers yield '' r_buffer = '' nb_to_send = 0 content_length = 0 seek = 0 processing = False # mode can be one of : request, chunk, extension, relay # icap : we are reading the icap headers # request : we are reading the request (read all you can until a separator) # extra-headers : we are reading data until a separator # chunked : we are reading chunk-encoded darta # transfer : we are reading as much as requested in remaining # passthrough : read as much as can to be relayed mode = 'icap' while True: try: while True: if not processing: data = sock.recv(read_size) if not data: break # read failed so we abort self.log.debug("<< [%s]" % data.replace('\t','\\t').replace('\r','\\r').replace('\n','\\n')) r_buffer += data else: processing = False if mode == 'passthrough': r_buffer, tmp = '', [r_buffer] yield [''], [''], tmp continue if nb_to_send: if mode == 'transfer' : r_len = len(r_buffer) length = min(r_len, nb_to_send) # do not yield yet if we are chunked since the end of the chunk may # very well be in the rest of the data we just read r_buffer, tmp = r_buffer[length:], [r_buffer[:length]] _, extra_size = yield [''], [''], tmp nb_to_send = nb_to_send - length + extra_size # we still have data to read before we can send more. if nb_to_send != 0: continue mode = 'icap' if mode == 'chunked': r_len = len(r_buffer) length = min(r_len, nb_to_send) # do not yield yet if we are chunked since the end of the chunk may # very well be in the rest of the data we just read if r_len <= nb_to_send: r_buffer, tmp = r_buffer[length:], [r_buffer[:length]] _, extra_size = yield '', '', tmp nb_to_send = nb_to_send - length + extra_size # we still have data to read before we can send more. if nb_to_send != 0: continue if mode == 'chunked': # sum of the sizes of all chunks in our buffer chunked, new_to_send = self.checkChunkSize(r_buffer[nb_to_send:]) if new_to_send is None: # could not read any chunk (data is invalid) break nb_to_send += new_to_send if chunked: continue mode = 'end-chunk' # seek is only set if we already passed once and found we needed more data to check if mode == 'end-chunk': if r_buffer[nb_to_send:].startswith('\r\n'): nb_to_send += 2 processing = True mode = 'transfer' continue elif r_buffer[nb_to_send:].startswith('\n'): nb_to_send += 1 processing = True mode = 'transfer' continue if not r_buffer[nb_to_send:]: yield [''], [''], [''] continue mode = 'extra-headers' seek = nb_to_send if mode == 'extra-headers': # seek is up to where we know there is no double CRLF related, r_buffer, seek = self.checkRequest(r_buffer,max_buffer,seek) if related is None: # most likely could not find an header break if related: related, tmp = '', [related] yield [''], [''], tmp else: yield [''], [''], [''] continue seek = 0 mode = 'transfer' if mode not in ('icap', 'request'): self.log.error('The programmers are monkeys - please give them bananas ..') self.log.error('the mode was spelled : [%s]' % mode) self.log.error('.. if it works, we are lucky - but it may work.') mode = 'icap' if mode == 'icap': # ignore EOL r_buffer = r_buffer.lstrip('\r\n') # check to see if we have read an entire request request, r_buffer, seek = self.checkRequest(r_buffer, max_buffer, seek) if request is None: # most likely could not find a header break if request: icap_request = request parsed_request = self.icap_parser.parseRequest(self.peer, icap_request, '') content_length = parsed_request.content_length if parsed_request is None or parsed_request.contains_body: # we do not (yet) support having content sent to us break if not parsed_request.contains_headers: # we need at least an HTTP header break # no reason to keep this around in memory longer than we need it parsed_request = None request, seek = '', 0 mode = 'request' else: yield [''], [''], [''] continue if mode == 'request': r_len = len(r_buffer) if r_len < content_length: yield [''], [''], [''] continue request, r_buffer = r_buffer[:content_length], r_buffer[content_length:] seek = 0 processing = True # nb_to_send is how much we expect to need to get the rest of the request icap_request, tmp_icap = '', [icap_request] request, tmp = '', [request] mode, nb_to_send = yield tmp_icap, tmp, [''] # break out of the outer loop as soon as we leave the inner loop # through normal execution break except socket.error, e: if e.args[0] in errno_block: yield [''], [''], [''] else: break yield [None], [None], [None]