Exemplo n.º 1
0
	def feed(self, data):
		"""	Encode some data and write the encoded data 
			into the internal buffer.
		"""
		if not self._feedable:
			raise IOError("Encoding already terminated")

		encoded, self._crc, self._column = _yenc.encode_string(data, self._crc, self._column)
		self._encoded = self._encoded + len(encoded)
		self._buffer.write(encoded)
		return len(encoded)
Exemplo n.º 2
0
	def feed(self, data):
		"""	Encode some data and write the encoded data 
			into the internal buffer
		"""
		if self._terminated:
			raise IOError("Encoding already terminated")
		encoded, self._crc, self._column = _yenc.encode_string(data, 
							self._crc, self._column)
		self._encoded = self._encoded + len(encoded)
		self._buffer.write(encoded)
		return len(encoded)
Exemplo n.º 3
0
def worker_thread(ready_event):
    global conn_workers
    logging.info("Starting new NNTP thread")
    tries = 0
    while True:
        try:
            n = UpstreamNNTP(server, username, password, port, use_ssl)
            n.connect()
            logging.info("NNTP thread %s started" % n)
            ready_event.set(True)
        except Exception, e:
            logging.info("NNTP thread failed! %s" % e.args[-1])
            if tries == 3:
                ready_event.set(False)
                return False, e.args[-1]
            tries += 1
            continue
        while True:
            try:
                messageid, data, name, customheaders, yencdata, finish = river_queue.get(True, 60)
                ypart, ytotal, ybegin, yend = yencdata
                headers = []
                ydata, ycrc, ylen = encode_string(data)
                mcrc = "%08x" % (binascii.crc32(data) & 2 ** 32L - 1)
                headers.append("From: %s" % poster)
                headers.append("Newsgroups: %s" % group)
                headers.append("Subject: %s" % subject.format(name=name))
                headers.append("Message-ID: <%s>" % messageid)
                headers += customheaders
                outdata = "\r\n".join(headers) + "\r\n\r\n"
                outdata += "=ybegin part=%d total=%d line=128 size=%d name=%s\r\n" % (ypart, ytotal, ylen, name)
                outdata += "=ypart begin=%d end=%d\r\n" % (ybegin, yend)
                outdata += ydata.replace("\r\n.", "\r\n..")
                outdata += "\r\n=yend size=%d pcrc32=%s" % (ylen, mcrc)
                result = n.post(outdata)
                finish.set()
            except gevent.queue.Empty:
                try:
                    n.date()
                except:
                    logging.info("NNTP thread %s timed out" % n)
                    break
Exemplo n.º 4
0
 def testEncode(self):
     e, c, z = _yenc.encode_string(b'Hello world!')
     self.assertEqual(e, b'r\x8f\x96\x96\x99J\xa1\x99\x9c\x96\x8eK')
     self.assertEqual(c, 3833259626)
Exemplo n.º 5
0
    def encode(self, content, mem_buf=DEFAULT_BUFFER_SIZE):
        """
        Encodes an NNTPContent object passed in
        """

        if isinstance(content, NNTPContent):
            # Create our ascii instance
            _encoded = NNTPAsciiContent(
                filepath=content.filename,
                part=content.part,
                total_parts=content.total_parts,
                sort_no=content.sort_no,
                work_dir=self.work_dir,
                # We want to ensure we're working with a unique attached file
                unique=True,
            )

        else:
            # If we reach here, we presume our content is a filename

            # Create our ascii instance
            _encoded = NNTPAsciiContent(
                filepath=content,
                work_dir=self.work_dir,
                # We want to ensure we're working with a unique attached file
                unique=True,
            )

            # Convert our content object into an NNTPContent object
            content = NNTPContent(
                filepath=content,
                work_dir=self.work_dir,
            )

        # yEnc (v1.3) begin
        fmt_ybegin = '=ybegin part=%d total=%d line=%d size=%d name=%s' % (
            content.part,
            content.total_parts,
            self.linelen,
            len(content),
            content.filename,
        )

        # yEnc part
        fmt_ypart = '=ypart begin=%d end=%d' % (
            content.begin() + 1,
            content.end(),
        )

        if isinstance(content._parent, NNTPContent):
            # yEnc end
            fmt_yend = '=yend size=%d part=%d pcrc32=%s crc32=%s' % (
                len(content),
                content.part,
                content.crc32(),
                content._parent.crc32(),
            )

        else:
            # yEnc end
            fmt_yend = '=yend size=%d part=%d pcrc32=%s' % (
                len(content),
                content.part,
                content.crc32(),
            )

        # Write =ybegin line
        _encoded.write(fmt_ybegin + EOL)
        # Write =ypart line
        _encoded.write(fmt_ypart + EOL)

        if not content.open():
            return None

        # Prepare our result set
        results = ""

        # Column is used for decoding
        column = 0
        crc = BIN_MASK

        # We need to parse the content until we either reach
        # the end of the file or get to an 'end' tag
        while True:
            # Read in our data
            data = content.stream.read(mem_buf)
            if not data:
                # We're done
                break

            if FAST_YENC_SUPPORT:
                try:
                    _results, crc, column = encode_string(data, crc, column)
                    # Append our parsed content onto our ongoing buffer
                    results += _results

                except YencError as e:
                    logger.error("Failed to encode Yenc for %s." % content)
                    logger.debug('Yenc exception: %s' % (str(e)))
                    return None

            else:
                # The slow and painful way, the below looks complicated
                # but it really isn't at the the end of the day; yEnc is
                # pretty basic;
                #  - first we translate the all of the characters by adding
                #    42 to their value with the exception of a few special
                #    characters that are explicitly reserved for the yEnc
                #    language (and conflict with the NNTP Server language).
                #
                #  - next, we need to apply our ENCODE_SPECIAL_MAP to be
                #    sure to handle the characters that are reserved as
                #    special keywords used by both NNTP Servers and the yEnc
                #    protocol itself.
                #
                #  - finally we want to prevent our string from going on for
                #    to many characters (horizontally).  So we need to split
                #    our content up
                #

                idx = 0
                while idx < len(data):
                    _byte = (ord(data[idx]) + 42) & 0xff
                    if _byte in YENC_ENCODE_ESCAPED_CHARACTERS:
                        _byte = (_byte + 64) & 0xff
                        # Esape Sequence
                        results += '='

                    # Store our character
                    results += chr(_byte)

                    # Increment Index
                    idx += 1

            # Our offset
            offset = 0

            while offset < (len(results) - self.linelen + 1):
                eol = offset + self.linelen
                if results[offset:eol][-1] == '=':
                    # Lines can't end with the escape sequence (=). If we get
                    # here then this one did. We just adjust our end-of-line
                    # by 1 and keep moving
                    eol -= 1

                _encoded.write(results[offset:eol] + EOL)
                offset = eol

            if offset < len(results):
                results = results[-(len(results) - offset):]

            else:
                # reset string
                results = ''

        # We're done reading our data
        content.close()

        if len(results):
            # We still have content left in our buffer
            _encoded.write(results + EOL)

        # Write footer
        _encoded.write(fmt_yend + EOL)

        if _encoded:
            # close article when complete
            _encoded.close()

        # Return our encoded object
        return _encoded
Exemplo n.º 6
0
    def encode(self, content, mem_buf=DEFAULT_BUFFER_SIZE):
        """
        Encodes an NNTPContent object passed in
        """

        if isinstance(content, NNTPContent):
            # Create our ascii instance
            _encoded = NNTPAsciiContent(
                filepath=content.filename,
                part=content.part,
                total_parts=content.total_parts,
                sort_no=content.sort_no,
                work_dir=self.work_dir,
                # We want to ensure we're working with a unique attached file
                unique=True,
            )

        else:
            # If we reach here, we presume our content is a filename

            # Create our ascii instance
            _encoded = NNTPAsciiContent(
                filepath=content,
                work_dir=self.work_dir,
                # We want to ensure we're working with a unique attached file
                unique=True,
            )

            # Convert our content object into an NNTPContent object
            content = NNTPContent(
                filepath=content,
                work_dir=self.work_dir,
            )

        # yEnc (v1.3) begin
        fmt_ybegin = '=ybegin part=%d total=%d line=%d size=%d name=%s' % (
            content.part, content.total_parts, self.linelen,
            len(content), content.filename,
        )

        # yEnc part
        fmt_ypart = '=ypart begin=%d end=%d' % (
            content.begin() + 1,
            content.end(),
        )

        if isinstance(content._parent, NNTPContent):
            # yEnc end
            fmt_yend = '=yend size=%d part=%d pcrc32=%s crc32=%s' % (
                len(content), content.part,
                content.crc32(), content._parent.crc32(),
            )

        else:
            # yEnc end
            fmt_yend = '=yend size=%d part=%d pcrc32=%s' % (
                len(content), content.part, content.crc32(),
            )

        # Write =ybegin line
        _encoded.write(fmt_ybegin + EOL)
        # Write =ypart line
        _encoded.write(fmt_ypart + EOL)

        if not content.open():
            return None

        # Prepare our result set
        results = ""

        # Column is used for decoding
        column = 0
        crc = BIN_MASK

        # We need to parse the content until we either reach
        # the end of the file or get to an 'end' tag
        while True:
            # Read in our data
            data = content.stream.read(mem_buf)
            if not data:
                # We're done
                break

            if FAST_YENC_SUPPORT:
                try:
                    _results, crc, column = encode_string(data, crc, column)
                    # Append our parsed content onto our ongoing buffer
                    results += _results

                except YencError as e:
                    logger.error("Failed to encode Yenc for %s." % content)
                    logger.debug('Yenc exception: %s' % (str(e)))
                    return None

            else:
                # The slow and painful way, the below looks complicated
                # but it really isn't at the the end of the day; yEnc is
                # pretty basic;
                #  - first we translate the all of the characters by adding
                #    42 to their value with the exception of a few special
                #    characters that are explicitly reserved for the yEnc
                #    language (and conflict with the NNTP Server language).
                #
                #  - next, we need to apply our ENCODE_SPECIAL_MAP to be
                #    sure to handle the characters that are reserved as
                #    special keywords used by both NNTP Servers and the yEnc
                #    protocol itself.
                #
                #  - finally we want to prevent our string from going on for
                #    to many characters (horizontally).  So we need to split
                #    our content up
                #

                idx = 0
                while idx < len(data):
                    _byte = (ord(data[idx]) + 42) & 0xff
                    if _byte in YENC_ENCODE_ESCAPED_CHARACTERS:
                        _byte = (_byte + 64) & 0xff
                        # Esape Sequence
                        results += '='

                    # Store our character
                    results += chr(_byte)

                    # Increment Index
                    idx += 1

            # Our offset
            offset = 0

            while offset < (len(results)-self.linelen+1):
                eol = offset+self.linelen
                if results[offset:eol][-1] == '=':
                    # Lines can't end with the escape sequence (=). If we get
                    # here then this one did. We just adjust our end-of-line
                    # by 1 and keep moving
                    eol -= 1

                _encoded.write(results[offset:eol] + EOL)
                offset = eol

            if offset < len(results):
                results = results[-(len(results) - offset):]

            else:
                # reset string
                results = ''

        # We're done reading our data
        content.close()

        if len(results):
            # We still have content left in our buffer
            _encoded.write(results + EOL)

        # Write footer
        _encoded.write(fmt_yend + EOL)

        if _encoded:
            # close article when complete
            _encoded.close()

        # Return our encoded object
        return _encoded