def feed(self, data): """ Encode some data and write the encoded data into the internal buffer. """ if not self._feedable: raise IOError("Encoding already terminated") encoded, self._crc, self._column = _yenc.encode_string(data, self._crc, self._column) self._encoded = self._encoded + len(encoded) self._buffer.write(encoded) return len(encoded)
def feed(self, data): """ Encode some data and write the encoded data into the internal buffer """ if self._terminated: raise IOError("Encoding already terminated") encoded, self._crc, self._column = _yenc.encode_string(data, self._crc, self._column) self._encoded = self._encoded + len(encoded) self._buffer.write(encoded) return len(encoded)
def worker_thread(ready_event): global conn_workers logging.info("Starting new NNTP thread") tries = 0 while True: try: n = UpstreamNNTP(server, username, password, port, use_ssl) n.connect() logging.info("NNTP thread %s started" % n) ready_event.set(True) except Exception, e: logging.info("NNTP thread failed! %s" % e.args[-1]) if tries == 3: ready_event.set(False) return False, e.args[-1] tries += 1 continue while True: try: messageid, data, name, customheaders, yencdata, finish = river_queue.get(True, 60) ypart, ytotal, ybegin, yend = yencdata headers = [] ydata, ycrc, ylen = encode_string(data) mcrc = "%08x" % (binascii.crc32(data) & 2 ** 32L - 1) headers.append("From: %s" % poster) headers.append("Newsgroups: %s" % group) headers.append("Subject: %s" % subject.format(name=name)) headers.append("Message-ID: <%s>" % messageid) headers += customheaders outdata = "\r\n".join(headers) + "\r\n\r\n" outdata += "=ybegin part=%d total=%d line=128 size=%d name=%s\r\n" % (ypart, ytotal, ylen, name) outdata += "=ypart begin=%d end=%d\r\n" % (ybegin, yend) outdata += ydata.replace("\r\n.", "\r\n..") outdata += "\r\n=yend size=%d pcrc32=%s" % (ylen, mcrc) result = n.post(outdata) finish.set() except gevent.queue.Empty: try: n.date() except: logging.info("NNTP thread %s timed out" % n) break
def testEncode(self): e, c, z = _yenc.encode_string(b'Hello world!') self.assertEqual(e, b'r\x8f\x96\x96\x99J\xa1\x99\x9c\x96\x8eK') self.assertEqual(c, 3833259626)
def encode(self, content, mem_buf=DEFAULT_BUFFER_SIZE): """ Encodes an NNTPContent object passed in """ if isinstance(content, NNTPContent): # Create our ascii instance _encoded = NNTPAsciiContent( filepath=content.filename, part=content.part, total_parts=content.total_parts, sort_no=content.sort_no, work_dir=self.work_dir, # We want to ensure we're working with a unique attached file unique=True, ) else: # If we reach here, we presume our content is a filename # Create our ascii instance _encoded = NNTPAsciiContent( filepath=content, work_dir=self.work_dir, # We want to ensure we're working with a unique attached file unique=True, ) # Convert our content object into an NNTPContent object content = NNTPContent( filepath=content, work_dir=self.work_dir, ) # yEnc (v1.3) begin fmt_ybegin = '=ybegin part=%d total=%d line=%d size=%d name=%s' % ( content.part, content.total_parts, self.linelen, len(content), content.filename, ) # yEnc part fmt_ypart = '=ypart begin=%d end=%d' % ( content.begin() + 1, content.end(), ) if isinstance(content._parent, NNTPContent): # yEnc end fmt_yend = '=yend size=%d part=%d pcrc32=%s crc32=%s' % ( len(content), content.part, content.crc32(), content._parent.crc32(), ) else: # yEnc end fmt_yend = '=yend size=%d part=%d pcrc32=%s' % ( len(content), content.part, content.crc32(), ) # Write =ybegin line _encoded.write(fmt_ybegin + EOL) # Write =ypart line _encoded.write(fmt_ypart + EOL) if not content.open(): return None # Prepare our result set results = "" # Column is used for decoding column = 0 crc = BIN_MASK # We need to parse the content until we either reach # the end of the file or get to an 'end' tag while True: # Read in our data data = content.stream.read(mem_buf) if not data: # We're done break if FAST_YENC_SUPPORT: try: _results, crc, column = encode_string(data, crc, column) # Append our parsed content onto our ongoing buffer results += _results except YencError as e: logger.error("Failed to encode Yenc for %s." % content) logger.debug('Yenc exception: %s' % (str(e))) return None else: # The slow and painful way, the below looks complicated # but it really isn't at the the end of the day; yEnc is # pretty basic; # - first we translate the all of the characters by adding # 42 to their value with the exception of a few special # characters that are explicitly reserved for the yEnc # language (and conflict with the NNTP Server language). # # - next, we need to apply our ENCODE_SPECIAL_MAP to be # sure to handle the characters that are reserved as # special keywords used by both NNTP Servers and the yEnc # protocol itself. # # - finally we want to prevent our string from going on for # to many characters (horizontally). So we need to split # our content up # idx = 0 while idx < len(data): _byte = (ord(data[idx]) + 42) & 0xff if _byte in YENC_ENCODE_ESCAPED_CHARACTERS: _byte = (_byte + 64) & 0xff # Esape Sequence results += '=' # Store our character results += chr(_byte) # Increment Index idx += 1 # Our offset offset = 0 while offset < (len(results) - self.linelen + 1): eol = offset + self.linelen if results[offset:eol][-1] == '=': # Lines can't end with the escape sequence (=). If we get # here then this one did. We just adjust our end-of-line # by 1 and keep moving eol -= 1 _encoded.write(results[offset:eol] + EOL) offset = eol if offset < len(results): results = results[-(len(results) - offset):] else: # reset string results = '' # We're done reading our data content.close() if len(results): # We still have content left in our buffer _encoded.write(results + EOL) # Write footer _encoded.write(fmt_yend + EOL) if _encoded: # close article when complete _encoded.close() # Return our encoded object return _encoded
def encode(self, content, mem_buf=DEFAULT_BUFFER_SIZE): """ Encodes an NNTPContent object passed in """ if isinstance(content, NNTPContent): # Create our ascii instance _encoded = NNTPAsciiContent( filepath=content.filename, part=content.part, total_parts=content.total_parts, sort_no=content.sort_no, work_dir=self.work_dir, # We want to ensure we're working with a unique attached file unique=True, ) else: # If we reach here, we presume our content is a filename # Create our ascii instance _encoded = NNTPAsciiContent( filepath=content, work_dir=self.work_dir, # We want to ensure we're working with a unique attached file unique=True, ) # Convert our content object into an NNTPContent object content = NNTPContent( filepath=content, work_dir=self.work_dir, ) # yEnc (v1.3) begin fmt_ybegin = '=ybegin part=%d total=%d line=%d size=%d name=%s' % ( content.part, content.total_parts, self.linelen, len(content), content.filename, ) # yEnc part fmt_ypart = '=ypart begin=%d end=%d' % ( content.begin() + 1, content.end(), ) if isinstance(content._parent, NNTPContent): # yEnc end fmt_yend = '=yend size=%d part=%d pcrc32=%s crc32=%s' % ( len(content), content.part, content.crc32(), content._parent.crc32(), ) else: # yEnc end fmt_yend = '=yend size=%d part=%d pcrc32=%s' % ( len(content), content.part, content.crc32(), ) # Write =ybegin line _encoded.write(fmt_ybegin + EOL) # Write =ypart line _encoded.write(fmt_ypart + EOL) if not content.open(): return None # Prepare our result set results = "" # Column is used for decoding column = 0 crc = BIN_MASK # We need to parse the content until we either reach # the end of the file or get to an 'end' tag while True: # Read in our data data = content.stream.read(mem_buf) if not data: # We're done break if FAST_YENC_SUPPORT: try: _results, crc, column = encode_string(data, crc, column) # Append our parsed content onto our ongoing buffer results += _results except YencError as e: logger.error("Failed to encode Yenc for %s." % content) logger.debug('Yenc exception: %s' % (str(e))) return None else: # The slow and painful way, the below looks complicated # but it really isn't at the the end of the day; yEnc is # pretty basic; # - first we translate the all of the characters by adding # 42 to their value with the exception of a few special # characters that are explicitly reserved for the yEnc # language (and conflict with the NNTP Server language). # # - next, we need to apply our ENCODE_SPECIAL_MAP to be # sure to handle the characters that are reserved as # special keywords used by both NNTP Servers and the yEnc # protocol itself. # # - finally we want to prevent our string from going on for # to many characters (horizontally). So we need to split # our content up # idx = 0 while idx < len(data): _byte = (ord(data[idx]) + 42) & 0xff if _byte in YENC_ENCODE_ESCAPED_CHARACTERS: _byte = (_byte + 64) & 0xff # Esape Sequence results += '=' # Store our character results += chr(_byte) # Increment Index idx += 1 # Our offset offset = 0 while offset < (len(results)-self.linelen+1): eol = offset+self.linelen if results[offset:eol][-1] == '=': # Lines can't end with the escape sequence (=). If we get # here then this one did. We just adjust our end-of-line # by 1 and keep moving eol -= 1 _encoded.write(results[offset:eol] + EOL) offset = eol if offset < len(results): results = results[-(len(results) - offset):] else: # reset string results = '' # We're done reading our data content.close() if len(results): # We still have content left in our buffer _encoded.write(results + EOL) # Write footer _encoded.write(fmt_yend + EOL) if _encoded: # close article when complete _encoded.close() # Return our encoded object return _encoded