示例#1
0
    def __init__(self, name, sock, peer, logger, max_buffer):
        self.name = name
        self.ipv4 = isipv4(sock.getsockname()[0])
        self.sock = sock
        self.peer = peer
        self.reader = self._read(sock, max_buffer)
        self.writer = self._write(sock)
        self.w_buffer = ''

        self.icap_parser = ICAPParser(configuration={})
        self.log = logger

        # start the _read coroutine
        self.reader.next()
示例#2
0
文件: icap.py 项目: Safe3/exaproxy
	def __init__(self, name, sock, peer, logger, max_buffer):
		self.name = name
		self.ipv4 = isipv4(sock.getsockname()[0])
		self.sock = sock
		self.peer = peer
		self.reader = self._read(sock,max_buffer)
		self.writer = self._write(sock)
		self.w_buffer = ''

		self.icap_parser = ICAPParser(configuration={})
		self.log = logger

		# start the _read coroutine
		self.reader.next()
示例#3
0
class ICAPClient(object):
    eor = ['\r\n\r\n', '\n\n']

    __slots__ = [
        'name', 'ipv4', 'sock', 'peer', 'reader', 'writer', 'w_buffer',
        'icap_parser', 'log'
    ]

    def __init__(self, name, sock, peer, logger, max_buffer):
        self.name = name
        self.ipv4 = isipv4(sock.getsockname()[0])
        self.sock = sock
        self.peer = peer
        self.reader = self._read(sock, max_buffer)
        self.writer = self._write(sock)
        self.w_buffer = ''

        self.icap_parser = ICAPParser(configuration={})
        self.log = logger

        # start the _read coroutine
        self.reader.next()

    def checkRequest(self, r_buffer, size, seek=0):
        for eor in self.eor:
            pos = r_buffer[seek:].find(eor)
            if pos == -1: continue

            buff = r_buffer[:seek + pos]
            if not buff: continue

            if not count_quotes(buff) % 2:  # we have matching pairs
                return buff + eor, r_buffer[seek + pos + len(eor):], seek

            seek += pos + len(eor)

        if size and len(r_buffer) > size:
            return None, None, None

        return '', r_buffer, seek

    def checkChunkSize(self, r_buffer):
        # return a tuple : bool, length
        # * the bool is : is there more chunk to come
        # * the len contains the size of the chunk(s) extracted
        #   a size of None means that we could not decode as it is invalid

        total_len = 0

        while r_buffer:
            if not '\n' in r_buffer:
                if len(r_buffer) > 6:  # len('FFFF') + len(\r\n)
                    return True, None
                return True, 0

            header, r_buffer = r_buffer.split('\n', 1)
            len_header = len(header) + 1

            if header.endswith('\r'):
                header = header[:-1]
                len_eol = 2
            else:
                len_eol = 1

            if ';' in header:
                header = header.split(';', 1)[0]

            if not ishex(header):
                return True, None

            len_chunk = int(header, 16)

            # 0xFFFF is not enough - coad is complaining :p
            if len_chunk > 0x100000:
                return True, None

            if len_chunk == 0:
                total_len += len_header
                return False, total_len
            else:
                total = len_chunk + len_eol
                total_len += total + len_header
                r_buffer = r_buffer[total:]

        return True, total_len

    def _read(self, sock, max_buffer, read_size=64 * 1024):
        """Coroutine managing data read from the client"""
        # yield request, content
        # request is the text that form the request header
        # content any text which is related to the current request after the headers

        yield ''

        r_buffer = ''
        nb_to_send = 0
        content_length = 0
        seek = 0
        processing = False

        # mode can be one of : request, chunk, extension, relay
        # icap : we are reading the icap headers
        # request : we are reading the request (read all you can until a separator)
        # extra-headers : we are reading data until a separator
        # chunked : we are reading chunk-encoded darta
        # transfer : we are reading as much as requested in remaining
        # passthrough : read as much as can to be relayed

        mode = 'icap'

        while True:
            try:
                while True:
                    if not processing:
                        data = sock.recv(read_size)
                        if not data:
                            break  # read failed so we abort
                        self.log.debug("<< [%s]" %
                                       data.replace('\t', '\\t').replace(
                                           '\r', '\\r').replace('\n', '\\n'))
                        r_buffer += data
                    else:
                        processing = False

                    if mode == 'passthrough':
                        r_buffer, tmp = '', [r_buffer]
                        yield [''], [''], tmp
                        continue

                    if nb_to_send:
                        if mode == 'transfer':
                            r_len = len(r_buffer)
                            length = min(r_len, nb_to_send)

                            # do not yield yet if we are chunked since the end of the chunk may
                            # very well be in the rest of the data we just read
                            r_buffer, tmp = r_buffer[length:], [
                                r_buffer[:length]
                            ]
                            _, extra_size = yield [''], [''], tmp

                            nb_to_send = nb_to_send - length + extra_size

                            # we still have data to read before we can send more.
                            if nb_to_send != 0:
                                continue
                            mode = 'icap'

                        if mode == 'chunked':
                            r_len = len(r_buffer)
                            length = min(r_len, nb_to_send)

                            # do not yield yet if we are chunked since the end of the chunk may
                            # very well be in the rest of the data we just read
                            if r_len <= nb_to_send:
                                r_buffer, tmp = r_buffer[length:], [
                                    r_buffer[:length]
                                ]
                                _, extra_size = yield '', '', tmp

                                nb_to_send = nb_to_send - length + extra_size

                                # we still have data to read before we can send more.
                                if nb_to_send != 0:
                                    continue

                    if mode == 'chunked':
                        # sum of the sizes of all chunks in our buffer
                        chunked, new_to_send = self.checkChunkSize(
                            r_buffer[nb_to_send:])

                        if new_to_send is None:
                            # could not read any chunk (data is invalid)
                            break

                        nb_to_send += new_to_send
                        if chunked:
                            continue

                        mode = 'end-chunk'

                    # seek is only set if we already passed once and found we needed more data to check
                    if mode == 'end-chunk':
                        if r_buffer[nb_to_send:].startswith('\r\n'):
                            nb_to_send += 2
                            processing = True
                            mode = 'transfer'
                            continue
                        elif r_buffer[nb_to_send:].startswith('\n'):
                            nb_to_send += 1
                            processing = True
                            mode = 'transfer'
                            continue

                        if not r_buffer[nb_to_send:]:
                            yield [''], [''], ['']
                            continue

                        mode = 'extra-headers'
                        seek = nb_to_send

                    if mode == 'extra-headers':
                        # seek is up to where we know there is no double CRLF
                        related, r_buffer, seek = self.checkRequest(
                            r_buffer, max_buffer, seek)

                        if related is None:
                            # most likely could not find an header
                            break

                        if related:
                            related, tmp = '', [related]
                            yield [''], [''], tmp

                        else:
                            yield [''], [''], ['']
                            continue

                        seek = 0
                        mode = 'transfer'

                    if mode not in ('icap', 'request'):
                        self.log.error(
                            'The programmers are monkeys - please give them bananas ..'
                        )
                        self.log.error('the mode was spelled : [%s]' % mode)
                        self.log.error(
                            '.. if it works, we are lucky - but it may work.')
                        mode = 'icap'

                    if mode == 'icap':
                        # ignore EOL
                        r_buffer = r_buffer.lstrip('\r\n')

                        # check to see if we have read an entire request
                        request, r_buffer, seek = self.checkRequest(
                            r_buffer, max_buffer, seek)

                        if request is None:
                            # most likely could not find a header
                            break

                        if request:
                            icap_request = request

                            parsed_request = self.icap_parser.parseRequest(
                                self.peer, icap_request, '')
                            content_length = parsed_request.content_length

                            if parsed_request is None or parsed_request.contains_body:
                                # we do not (yet) support having content sent to us
                                break

                            if not parsed_request.contains_headers:
                                # we need at least an HTTP header
                                break

                            # no reason to keep this around in memory longer than we need it
                            parsed_request = None

                            request, seek = '', 0
                            mode = 'request'

                        else:
                            yield [''], [''], ['']
                            continue

                    if mode == 'request':
                        r_len = len(r_buffer)

                        if r_len < content_length:
                            yield [''], [''], ['']
                            continue

                    request, r_buffer = r_buffer[:content_length], r_buffer[
                        content_length:]

                    seek = 0
                    processing = True

                    # nb_to_send is how much we expect to need to get the rest of the request
                    icap_request, tmp_icap = '', [icap_request]
                    request, tmp = '', [request]

                    mode, nb_to_send = yield tmp_icap, tmp, ['']

                # break out of the outer loop as soon as we leave the inner loop
                # through normal execution
                break

            except socket.error, e:
                if e.args[0] in errno_block:
                    yield [''], [''], ['']
                else:
                    break

        yield [None], [None], [None]
示例#4
0
文件: icap.py 项目: Safe3/exaproxy
class ICAPClient (object):
	eor = ['\r\n\r\n', '\n\n']

	__slots__ = ['name', 'ipv4', 'sock', 'peer', 'reader', 'writer', 'w_buffer', 'icap_parser', 'log']

	def __init__(self, name, sock, peer, logger, max_buffer):
		self.name = name
		self.ipv4 = isipv4(sock.getsockname()[0])
		self.sock = sock
		self.peer = peer
		self.reader = self._read(sock,max_buffer)
		self.writer = self._write(sock)
		self.w_buffer = ''

		self.icap_parser = ICAPParser(configuration={})
		self.log = logger

		# start the _read coroutine
		self.reader.next()

	def checkRequest (self, r_buffer, size, seek=0):
		for eor in self.eor:
			pos = r_buffer[seek:].find(eor)
			if pos == -1: continue

			buff = r_buffer[:seek+pos]
			if not buff: continue

			if not count_quotes(buff) % 2:  # we have matching pairs
				return buff + eor, r_buffer[seek+pos+len(eor):], seek

			seek += pos + len(eor)

		if size and len(r_buffer) > size:
			return None,None,None

		return '', r_buffer, seek


	def checkChunkSize (self, r_buffer):
		# return a tuple : bool, length
		# * the bool is : is there more chunk to come
		# * the len contains the size of the chunk(s) extracted
		#   a size of None means that we could not decode as it is invalid

		total_len = 0

		while r_buffer:
			if not '\n' in r_buffer:
				if len(r_buffer) > 6:  # len('FFFF') + len(\r\n)
					return True, None
				return True, 0

			header,r_buffer = r_buffer.split('\n', 1)
			len_header = len(header) + 1

			if header.endswith('\r'):
				header = header[:-1]
				len_eol = 2
			else:
				len_eol = 1

			if ';' in header:
				header = header.split(';',1)[0]

			if not ishex(header):
				return True,None

			len_chunk = int(header, 16)

			# 0xFFFF is not enough - coad is complaining :p
			if len_chunk > 0x100000:
				return True,None

			if len_chunk == 0:
				total_len += len_header
				return False, total_len
			else:
				total = len_chunk + len_eol
				total_len += total + len_header
				r_buffer = r_buffer[total:]

		return True,total_len


	def _read (self, sock, max_buffer, read_size=64*1024):
		"""Coroutine managing data read from the client"""
		# yield request, content
		# request is the text that form the request header
		# content any text which is related to the current request after the headers

		yield ''

		r_buffer = ''
		nb_to_send = 0
		content_length = 0
		seek = 0
		processing = False

		# mode can be one of : request, chunk, extension, relay
		# icap : we are reading the icap headers
		# request : we are reading the request (read all you can until a separator)
		# extra-headers : we are reading data until a separator
		# chunked : we are reading chunk-encoded darta
		# transfer : we are reading as much as requested in remaining
		# passthrough : read as much as can to be relayed

		mode = 'icap'

		while True:
			try:
				while True:
					if not processing:
						data = sock.recv(read_size)
						if not data:
							break  # read failed so we abort
						self.log.debug("<< [%s]" % data.replace('\t','\\t').replace('\r','\\r').replace('\n','\\n'))
						r_buffer += data
					else:
						processing = False

					if mode == 'passthrough':
						r_buffer, tmp = '', [r_buffer]
						yield [''], [''], tmp
						continue

					if nb_to_send:
						if mode == 'transfer' :
							r_len = len(r_buffer)
							length = min(r_len, nb_to_send)

							# do not yield yet if we are chunked since the end of the chunk may
							# very well be in the rest of the data we just read
							r_buffer, tmp = r_buffer[length:], [r_buffer[:length]]
							_, extra_size = yield [''], [''], tmp

							nb_to_send = nb_to_send - length + extra_size

							# we still have data to read before we can send more.
							if nb_to_send != 0:
								continue
							mode = 'icap'

						if mode == 'chunked':
							r_len = len(r_buffer)
							length = min(r_len, nb_to_send)

							# do not yield yet if we are chunked since the end of the chunk may
							# very well be in the rest of the data we just read
							if r_len <= nb_to_send:
								r_buffer, tmp = r_buffer[length:], [r_buffer[:length]]
								_, extra_size = yield '', '', tmp

								nb_to_send = nb_to_send - length + extra_size

								# we still have data to read before we can send more.
								if nb_to_send != 0:
									continue

					if mode == 'chunked':
						# sum of the sizes of all chunks in our buffer
						chunked, new_to_send = self.checkChunkSize(r_buffer[nb_to_send:])

						if new_to_send is None:
							# could not read any chunk (data is invalid)
							break

						nb_to_send += new_to_send
						if chunked:
							continue

						mode = 'end-chunk'

					# seek is only set if we already passed once and found we needed more data to check
					if mode == 'end-chunk':
						if r_buffer[nb_to_send:].startswith('\r\n'):
							nb_to_send += 2
							processing = True
							mode = 'transfer'
							continue
						elif r_buffer[nb_to_send:].startswith('\n'):
							nb_to_send += 1
							processing = True
							mode = 'transfer'
							continue

						if not r_buffer[nb_to_send:]:
							yield [''], [''], ['']
							continue

						mode = 'extra-headers'
						seek = nb_to_send

					if mode == 'extra-headers':
						# seek is up to where we know there is no double CRLF
						related, r_buffer, seek = self.checkRequest(r_buffer,max_buffer,seek)

						if related is None:
							# most likely could not find an header
							break

						if related:
							related, tmp = '', [related]
							yield [''], [''], tmp

						else:
							yield [''], [''], ['']
							continue

						seek = 0
						mode = 'transfer'

					if mode not in ('icap', 'request'):
						self.log.error('The programmers are monkeys - please give them bananas ..')
						self.log.error('the mode was spelled : [%s]' % mode)
						self.log.error('.. if it works, we are lucky - but it may work.')
						mode = 'icap'


					if mode == 'icap':
						# ignore EOL
						r_buffer = r_buffer.lstrip('\r\n')

						# check to see if we have read an entire request
						request, r_buffer, seek = self.checkRequest(r_buffer, max_buffer, seek)

						if request is None:
							# most likely could not find a header
							break

						if request:
							icap_request = request

							parsed_request = self.icap_parser.parseRequest(self.peer, icap_request, '')
							content_length = parsed_request.content_length

							if parsed_request is None or parsed_request.contains_body:
								# we do not (yet) support having content sent to us
								break

							if not parsed_request.contains_headers:
								# we need at least an HTTP header
								break

							# no reason to keep this around in memory longer than we need it
							parsed_request = None

							request, seek = '', 0
							mode = 'request'

						else:
							yield [''], [''], ['']
							continue

					if mode == 'request':
						r_len = len(r_buffer)

						if r_len < content_length:
							yield [''], [''], ['']
							continue

					request, r_buffer = r_buffer[:content_length], r_buffer[content_length:]

					seek = 0
					processing = True

					# nb_to_send is how much we expect to need to get the rest of the request
					icap_request, tmp_icap = '', [icap_request]
					request, tmp = '', [request]

					mode, nb_to_send = yield tmp_icap, tmp, ['']

				# break out of the outer loop as soon as we leave the inner loop
				# through normal execution
				break

			except socket.error, e:
				if e.args[0] in errno_block:
					yield [''], [''], ['']
				else:
					break

		yield [None], [None], [None]