def test_decoding_yenc_multi_part(self): """ Test decoding of a yEnc multi-part This test was generated after visiting http://www.yenc.org and finding the examples they provide on their site. Downloaded the following zip file: http://www.yenc.org/yenc2.zip Then extracting it revealed 3 files: - 00000020.ntx This is the yEnc file as it would have been seen after being downloaded from the NNTP server (part 1 of 2) - 00000021.ntx This is the yEnc file as it would have been seen after being downloaded from the NNTP server (part 2 of 2) - joystick.jpg This is what the contents of the file should look like after being decoded (and assembled). This is what we use to test the file against. """ # A simple test for ensuring that the yEnc # library exists; otherwise we want this test # to fail; the below line will handle this for # us; we'll let the test fail on an import error import yenc # Input File encoded_filepath_1 = join(self.var_dir, '00000020.ntx') encoded_filepath_2 = join(self.var_dir, '00000021.ntx') assert isfile(encoded_filepath_1) assert isfile(encoded_filepath_2) # Compare File decoded_filepath = join(self.var_dir, 'joystick.jpg') assert isfile(decoded_filepath) # Python Solution fd1_py = BytesIO() fd2_py = BytesIO() # C Solution fd1_c = BytesIO() fd2_c = BytesIO() # Initialize Codec decoder = CodecYenc(work_dir=self.test_dir) contents_py = [] contents_c = [] # Force to operate in python (manual/slow) mode CodecYenc.FAST_YENC_SUPPORT = False with open(encoded_filepath_1, 'r') as fd_in: contents_py.append(decoder.decode(fd_in)) with open(encoded_filepath_2, 'r') as fd_in: contents_py.append(decoder.decode(fd_in)) for x in contents_py: # Verify our data is good assert x.is_valid() is True # Force to operate with the C extension yEnc # This require the extensions to be installed # on the system CodecYenc.FAST_YENC_SUPPORT = True with open(encoded_filepath_1, 'r') as fd_in: contents_c.append(decoder.decode(fd_in)) with open(encoded_filepath_2, 'r') as fd_in: contents_c.append(decoder.decode(fd_in)) for x in contents_c: # Verify our data is good assert x.is_valid() is True # Confirm that our output from our python implimentation # matches that of our yEnc C version. assert fd1_py.tell() == fd1_c.tell() assert fd2_py.tell() == fd2_c.tell() with open(decoded_filepath, 'r') as fd_in: decoded = fd_in.read() # Assemble (TODO) contents_py.sort() contents_c.sort() content_py = NNTPBinaryContent( filepath=contents_py[0].filename, save_dir=self.out_dir, ) content_c = NNTPBinaryContent( filepath=contents_c[0].filename, save_dir=self.out_dir, ) # append() takes a list or another NNTPContent # and appends it's content to the end of the content content_py.append(contents_py) content_c.append(contents_py) assert len(content_py) == len(decoded) assert len(content_c) == len(decoded) # Compare our processed content with the expected results assert content_py.getvalue() == decoded assert content_c.getvalue() == decoded
def test_yenc_multi_message(self): """ Tests the handling of a yenc multi-message """ # Create a non-secure connection sock = NNTPConnection( host=self.nttp_ipaddr, port=self.nntp_portno, username='******', password='******', secure=False, join_group=True, ) assert sock.connect() is True assert sock._iostream == NNTPIOStream.RFC3977_GZIP articles = sortedset(key=lambda x: x.key()) # We intententionally fetch the content out of order # ideally we'd want 20 followed by 21 articles.add( sock.get(id='21', work_dir=self.tmp_dir, group=self.common_group)) assert sock.group_name == self.common_group articles.add(sock.get(id='20', work_dir=self.tmp_dir)) assert sock.group_name == self.common_group newfile = NNTPBinaryContent( # This looks rough; # we're basically looking at the first article stored (since our # set is sorted, and then we're looking at the first content entry # TODO: update the article function so it's much easier to get # an iterator to decoded list filepath=iter(iter(articles).next().decoded).next().filename, work_dir=self.tmp_dir, ) for article in articles: assert isinstance(article, NNTPArticle) is True assert len(article.decoded) == 1 assert isinstance(iter(article.decoded).next(), NNTPBinaryContent) assert iter(article.decoded).next().is_valid() is True # Build on new file newfile.append(iter(article.decoded).next()) # keep open file count low iter(article.decoded).next().close() # Compare File decoded_filepath = join(self.var_dir, 'joystick.jpg') assert isfile(decoded_filepath) with open(decoded_filepath, 'r') as fd_in: decoded = fd_in.read() assert isfile(newfile.filepath) is True old_filepath = newfile.filepath newfile.save() new_filepath = newfile.filepath assert old_filepath != new_filepath assert isfile(old_filepath) is False assert isfile(new_filepath) is True assert decoded == newfile.getvalue() # Close up our socket sock.close() while len(articles): article = articles.pop() # length hasn't changed assert len(article.decoded) == 1 old_filepath = iter(article.decoded).next().filepath assert isfile(old_filepath) is True # If we remove the article, we automatically destroy # all associated decoded with it (that aren't detached) del article # Since there is only 1 attachment per article in this test # we can see that the file is now gone assert isfile(old_filepath) is False # Remove the file del newfile # We called save() so the file has been detached and will still exist! assert isfile(new_filepath) is True # cleanup our file unlink(new_filepath)
def test_yenc_multi_message(self): """ Tests the handling of a yenc multi-message """ # Create a non-secure connection sock = NNTPConnection( host=self.nttp_ipaddr, port=self.nntp_portno, username='******', password='******', secure=False, join_group=True, ) assert sock.connect() is True assert sock._iostream == NNTPIOStream.RFC3977_GZIP articles = sortedset(key=lambda x: x.key()) # We intententionally fetch the content out of order # ideally we'd want 20 followed by 21 articles.add(sock.get(id='21', work_dir=self.tmp_dir, group=self.common_group)) assert sock.group_name == self.common_group articles.add(sock.get(id='20', work_dir=self.tmp_dir)) assert sock.group_name == self.common_group newfile = NNTPBinaryContent( # This looks rough; # we're basically looking at the first article stored (since our # set is sorted, and then we're looking at the first content entry # TODO: update the article function so it's much easier to get # an iterator to decoded list filepath=iter(iter(articles).next().decoded).next().filename, work_dir=self.tmp_dir, ) for article in articles: assert isinstance(article, NNTPArticle) is True assert len(article.decoded) == 1 assert isinstance(iter(article.decoded).next(), NNTPBinaryContent) assert iter(article.decoded).next().is_valid() is True # Build on new file newfile.append(iter(article.decoded).next()) # keep open file count low iter(article.decoded).next().close() # Compare File decoded_filepath = join(self.var_dir, 'joystick.jpg') assert isfile(decoded_filepath) with open(decoded_filepath, 'r') as fd_in: decoded = fd_in.read() assert isfile(newfile.filepath) is True old_filepath = newfile.filepath newfile.save() new_filepath = newfile.filepath assert old_filepath != new_filepath assert isfile(old_filepath) is False assert isfile(new_filepath) is True assert decoded == newfile.getvalue() # Close up our socket sock.close() while len(articles): article = articles.pop() # length hasn't changed assert len(article.decoded) == 1 old_filepath = iter(article.decoded).next().filepath assert isfile(old_filepath) is True # If we remove the article, we automatically destroy # all associated decoded with it (that aren't detached) del article # Since there is only 1 attachment per article in this test # we can see that the file is now gone assert isfile(old_filepath) is False # Remove the file del newfile # We called save() so the file has been detached and will still exist! assert isfile(new_filepath) is True # cleanup our file unlink(new_filepath)
def test_decoding_yenc_multi_part(self): """ Test decoding of a yEnc multi-part This test was generated after visiting http://www.yenc.org and finding the examples they provide on their site. Downloaded the following zip file: http://www.yenc.org/yenc2.zip Then extracting it revealed 3 files: - 00000020.ntx This is the yEnc file as it would have been seen after being downloaded from the NNTP server (part 1 of 2) - 00000021.ntx This is the yEnc file as it would have been seen after being downloaded from the NNTP server (part 2 of 2) - joystick.jpg This is what the contents of the file should look like after being decoded (and assembled). This is what we use to test the file against. """ # A simple test for ensuring that the yEnc # library exists; otherwise we want this test # to fail; the below line will handle this for # us; we'll let the test fail on an import error import yenc # Input File encoded_filepath_1 = join(self.var_dir, '00000020.ntx') encoded_filepath_2 = join(self.var_dir, '00000021.ntx') assert isfile(encoded_filepath_1) assert isfile(encoded_filepath_2) # Compare File decoded_filepath = join(self.var_dir, 'joystick.jpg') assert isfile(decoded_filepath) # Python Solution fd1_py = BytesIO() fd2_py = BytesIO() # C Solution fd1_c = BytesIO() fd2_c = BytesIO() # Initialize Codec decoder = CodecYenc(work_dir=self.test_dir) contents_py = [] contents_c = [] # Force to operate in python (manual/slow) mode CodecYenc.FAST_YENC_SUPPORT = False with open(encoded_filepath_1, 'r') as fd_in: contents_py.append(decoder.decode(fd_in)) with open(encoded_filepath_2, 'r') as fd_in: contents_py.append(decoder.decode(fd_in)) for x in contents_py: # Verify our data is good assert x.is_valid() is True # Force to operate with the C extension yEnc # This require the extensions to be installed # on the system CodecYenc.FAST_YENC_SUPPORT = True with open(encoded_filepath_1, 'r') as fd_in: contents_c.append(decoder.decode(fd_in)) with open(encoded_filepath_2, 'r') as fd_in: contents_c.append(decoder.decode(fd_in)) for x in contents_c: # Verify our data is good assert x.is_valid() is True # Confirm that our output from our python implimentation # matches that of our yEnc C version. assert fd1_py.tell() == fd1_c.tell() assert fd2_py.tell() == fd2_c.tell() with open(decoded_filepath, 'r') as fd_in: decoded = fd_in.read() # Assemble (TODO) contents_py.sort() contents_c.sort() content_py = NNTPBinaryContent( filepath=contents_py[0].filename, save_dir=self.out_dir, ) content_c = NNTPBinaryContent( filepath=contents_c[0].filename, save_dir=self.out_dir, ) # append() takes a list or another NNTPContent # and appends it's content to the end of the content content_py.append(contents_py) content_c.append(contents_py) assert len(content_py) == len(decoded) assert len(content_c) == len(decoded) # Compare our processed content with the expected results assert content_py.getvalue() == decoded assert content_c.getvalue() == decoded