def test_decoding_yenc_multi_part(self):
"""
Test decoding of a yEnc multi-part
This test was generated after visiting http://www.yenc.org and finding
the examples they provide on their site.
Downloaded the following zip file:
http://www.yenc.org/yenc2.zip
Then extracting it revealed 3 files:
- 00000020.ntx
This is the yEnc file as it would have been seen after
being downloaded from the NNTP server (part 1 of 2)
- 00000021.ntx
This is the yEnc file as it would have been seen after
being downloaded from the NNTP server (part 2 of 2)
- joystick.jpg
This is what the contents of the file should look like
after being decoded (and assembled). This is what we use
to test the file against.
"""
# A simple test for ensuring that the yEnc
# library exists; otherwise we want this test
# to fail; the below line will handle this for
# us; we'll let the test fail on an import error
import yenc
# Input File
encoded_filepath_1 = join(self.var_dir, '00000020.ntx')
encoded_filepath_2 = join(self.var_dir, '00000021.ntx')
assert isfile(encoded_filepath_1)
assert isfile(encoded_filepath_2)
# Compare File
decoded_filepath = join(self.var_dir, 'joystick.jpg')
assert isfile(decoded_filepath)
# Python Solution
fd1_py = BytesIO()
fd2_py = BytesIO()
# C Solution
fd1_c = BytesIO()
fd2_c = BytesIO()
# Initialize Codec
decoder = CodecYenc(work_dir=self.test_dir)
contents_py = []
contents_c = []
# Force to operate in python (manual/slow) mode
CodecYenc.FAST_YENC_SUPPORT = False
with open(encoded_filepath_1, 'r') as fd_in:
contents_py.append(decoder.decode(fd_in))
with open(encoded_filepath_2, 'r') as fd_in:
contents_py.append(decoder.decode(fd_in))
for x in contents_py:
# Verify our data is good
assert x.is_valid() is True
# Force to operate with the C extension yEnc
# This require the extensions to be installed
# on the system
CodecYenc.FAST_YENC_SUPPORT = True
with open(encoded_filepath_1, 'r') as fd_in:
contents_c.append(decoder.decode(fd_in))
with open(encoded_filepath_2, 'r') as fd_in:
contents_c.append(decoder.decode(fd_in))
for x in contents_c:
# Verify our data is good
assert x.is_valid() is True
# Confirm that our output from our python implimentation
# matches that of our yEnc C version.
assert fd1_py.tell() == fd1_c.tell()
assert fd2_py.tell() == fd2_c.tell()
with open(decoded_filepath, 'r') as fd_in:
decoded = fd_in.read()
# Assemble (TODO)
contents_py.sort()
contents_c.sort()
content_py = NNTPBinaryContent(
filepath=contents_py[0].filename,
save_dir=self.out_dir,
)
content_c = NNTPBinaryContent(
filepath=contents_c[0].filename,
save_dir=self.out_dir,
)
# append() takes a list or another NNTPContent
# and appends it's content to the end of the content
content_py.append(contents_py)
content_c.append(contents_py)
assert len(content_py) == len(decoded)
assert len(content_c) == len(decoded)
# Compare our processed content with the expected results
assert content_py.getvalue() == decoded
assert content_c.getvalue() == decoded
def test_yenc_multi_message(self):
"""
Tests the handling of a yenc multi-message
"""
# Create a non-secure connection
sock = NNTPConnection(
host=self.nttp_ipaddr,
port=self.nntp_portno,
username='******',
password='******',
secure=False,
join_group=True,
)
assert sock.connect() is True
assert sock._iostream == NNTPIOStream.RFC3977_GZIP
articles = sortedset(key=lambda x: x.key())
# We intententionally fetch the content out of order
# ideally we'd want 20 followed by 21
articles.add(
sock.get(id='21', work_dir=self.tmp_dir, group=self.common_group))
assert sock.group_name == self.common_group
articles.add(sock.get(id='20', work_dir=self.tmp_dir))
assert sock.group_name == self.common_group
newfile = NNTPBinaryContent(
# This looks rough;
# we're basically looking at the first article stored (since our
# set is sorted, and then we're looking at the first content entry
# TODO: update the article function so it's much easier to get
# an iterator to decoded list
filepath=iter(iter(articles).next().decoded).next().filename,
work_dir=self.tmp_dir,
)
for article in articles:
assert isinstance(article, NNTPArticle) is True
assert len(article.decoded) == 1
assert isinstance(iter(article.decoded).next(), NNTPBinaryContent)
assert iter(article.decoded).next().is_valid() is True
# Build on new file
newfile.append(iter(article.decoded).next())
# keep open file count low
iter(article.decoded).next().close()
# Compare File
decoded_filepath = join(self.var_dir, 'joystick.jpg')
assert isfile(decoded_filepath)
with open(decoded_filepath, 'r') as fd_in:
decoded = fd_in.read()
assert isfile(newfile.filepath) is True
old_filepath = newfile.filepath
newfile.save()
new_filepath = newfile.filepath
assert old_filepath != new_filepath
assert isfile(old_filepath) is False
assert isfile(new_filepath) is True
assert decoded == newfile.getvalue()
# Close up our socket
sock.close()
while len(articles):
article = articles.pop()
# length hasn't changed
assert len(article.decoded) == 1
old_filepath = iter(article.decoded).next().filepath
assert isfile(old_filepath) is True
# If we remove the article, we automatically destroy
# all associated decoded with it (that aren't detached)
del article
# Since there is only 1 attachment per article in this test
# we can see that the file is now gone
assert isfile(old_filepath) is False
# Remove the file
del newfile
# We called save() so the file has been detached and will still exist!
assert isfile(new_filepath) is True
# cleanup our file
unlink(new_filepath)
def test_yenc_multi_message(self):
"""
Tests the handling of a yenc multi-message
"""
# Create a non-secure connection
sock = NNTPConnection(
host=self.nttp_ipaddr,
port=self.nntp_portno,
username='******',
password='******',
secure=False,
join_group=True,
)
assert sock.connect() is True
assert sock._iostream == NNTPIOStream.RFC3977_GZIP
articles = sortedset(key=lambda x: x.key())
# We intententionally fetch the content out of order
# ideally we'd want 20 followed by 21
articles.add(sock.get(id='21', work_dir=self.tmp_dir, group=self.common_group))
assert sock.group_name == self.common_group
articles.add(sock.get(id='20', work_dir=self.tmp_dir))
assert sock.group_name == self.common_group
newfile = NNTPBinaryContent(
# This looks rough;
# we're basically looking at the first article stored (since our
# set is sorted, and then we're looking at the first content entry
# TODO: update the article function so it's much easier to get
# an iterator to decoded list
filepath=iter(iter(articles).next().decoded).next().filename,
work_dir=self.tmp_dir,
)
for article in articles:
assert isinstance(article, NNTPArticle) is True
assert len(article.decoded) == 1
assert isinstance(iter(article.decoded).next(), NNTPBinaryContent)
assert iter(article.decoded).next().is_valid() is True
# Build on new file
newfile.append(iter(article.decoded).next())
# keep open file count low
iter(article.decoded).next().close()
# Compare File
decoded_filepath = join(self.var_dir, 'joystick.jpg')
assert isfile(decoded_filepath)
with open(decoded_filepath, 'r') as fd_in:
decoded = fd_in.read()
assert isfile(newfile.filepath) is True
old_filepath = newfile.filepath
newfile.save()
new_filepath = newfile.filepath
assert old_filepath != new_filepath
assert isfile(old_filepath) is False
assert isfile(new_filepath) is True
assert decoded == newfile.getvalue()
# Close up our socket
sock.close()
while len(articles):
article = articles.pop()
# length hasn't changed
assert len(article.decoded) == 1
old_filepath = iter(article.decoded).next().filepath
assert isfile(old_filepath) is True
# If we remove the article, we automatically destroy
# all associated decoded with it (that aren't detached)
del article
# Since there is only 1 attachment per article in this test
# we can see that the file is now gone
assert isfile(old_filepath) is False
# Remove the file
del newfile
# We called save() so the file has been detached and will still exist!
assert isfile(new_filepath) is True
# cleanup our file
unlink(new_filepath)
def test_decoding_yenc_multi_part(self):
"""
Test decoding of a yEnc multi-part
This test was generated after visiting http://www.yenc.org and finding
the examples they provide on their site.
Downloaded the following zip file:
http://www.yenc.org/yenc2.zip
Then extracting it revealed 3 files:
- 00000020.ntx
This is the yEnc file as it would have been seen after
being downloaded from the NNTP server (part 1 of 2)
- 00000021.ntx
This is the yEnc file as it would have been seen after
being downloaded from the NNTP server (part 2 of 2)
- joystick.jpg
This is what the contents of the file should look like
after being decoded (and assembled). This is what we use
to test the file against.
"""
# A simple test for ensuring that the yEnc
# library exists; otherwise we want this test
# to fail; the below line will handle this for
# us; we'll let the test fail on an import error
import yenc
# Input File
encoded_filepath_1 = join(self.var_dir, '00000020.ntx')
encoded_filepath_2 = join(self.var_dir, '00000021.ntx')
assert isfile(encoded_filepath_1)
assert isfile(encoded_filepath_2)
# Compare File
decoded_filepath = join(self.var_dir, 'joystick.jpg')
assert isfile(decoded_filepath)
# Python Solution
fd1_py = BytesIO()
fd2_py = BytesIO()
# C Solution
fd1_c = BytesIO()
fd2_c = BytesIO()
# Initialize Codec
decoder = CodecYenc(work_dir=self.test_dir)
contents_py = []
contents_c = []
# Force to operate in python (manual/slow) mode
CodecYenc.FAST_YENC_SUPPORT = False
with open(encoded_filepath_1, 'r') as fd_in:
contents_py.append(decoder.decode(fd_in))
with open(encoded_filepath_2, 'r') as fd_in:
contents_py.append(decoder.decode(fd_in))
for x in contents_py:
# Verify our data is good
assert x.is_valid() is True
# Force to operate with the C extension yEnc
# This require the extensions to be installed
# on the system
CodecYenc.FAST_YENC_SUPPORT = True
with open(encoded_filepath_1, 'r') as fd_in:
contents_c.append(decoder.decode(fd_in))
with open(encoded_filepath_2, 'r') as fd_in:
contents_c.append(decoder.decode(fd_in))
for x in contents_c:
# Verify our data is good
assert x.is_valid() is True
# Confirm that our output from our python implimentation
# matches that of our yEnc C version.
assert fd1_py.tell() == fd1_c.tell()
assert fd2_py.tell() == fd2_c.tell()
with open(decoded_filepath, 'r') as fd_in:
decoded = fd_in.read()
# Assemble (TODO)
contents_py.sort()
contents_c.sort()
content_py = NNTPBinaryContent(
filepath=contents_py[0].filename,
save_dir=self.out_dir,
)
content_c = NNTPBinaryContent(
filepath=contents_c[0].filename,
save_dir=self.out_dir,
)
# append() takes a list or another NNTPContent
# and appends it's content to the end of the content
content_py.append(contents_py)
content_c.append(contents_py)
assert len(content_py) == len(decoded)
assert len(content_c) == len(decoded)
# Compare our processed content with the expected results
assert content_py.getvalue() == decoded
assert content_c.getvalue() == decoded
