def test_copy02(self):
"""
Test the copy() function some more
"""
# First we create a 1MB file
tmp_file = join(self.tmp_dir, 'NNTPContent_Test.copy', '1MB.rar')
# File should not already exist
assert(isfile(tmp_file) is False)
# Create our file
assert(self.touch(tmp_file, size='1MB', random=True) is True)
# File should exist now
assert(isfile(tmp_file) is True)
# Now we want to load it into a NNTPContent object
content_a = NNTPContent(filepath=tmp_file, work_dir=self.tmp_dir)
# We want to make a copy of it
content_b = content_a.copy()
# Our content should be the same
assert(len(content_a) == len(content_b))
assert(content_a.md5() == content_b.md5())
# Because we made a copy, we're working with a different file however
# we share the exact same content and filename identifier.
assert(content_a == content_b)
# Our paths are not the same though; this is what makes us a official
# copy
assert(content_a.path() != content_b.path())
def test_writes(self):
"""
More overhead then a normal write() but none the less, using the
write() in this class keeps things simple since the file is
automatically opened if it was otherwise closed
"""
# First we create a 1MB file
tmp_file = join(self.tmp_dir, 'NNTPContent_Test.write', 'tmp.file')
# File should not already exist
assert (isfile(tmp_file) is False)
# Now we want to create our NNTPContent() object surrouding this
# file that does not exist.
content = NNTPContent(filepath=tmp_file, work_dir=dirname(tmp_file))
# It's worth noting that this file will 'still' not exist
assert (isfile(tmp_file) is False)
# we'll write data
data = 'hello\r\n'
content.write(data)
# It's worth noting that this file will ''STILL'' not exist
assert (isfile(tmp_file) is False)
# Save content
assert (content.save() is True)
# Now the file 'will' exist
assert (isfile(tmp_file) is True)
# Open our file and verify it is the data we saved.
with open(tmp_file) as f:
data_read = f.read()
assert (data == data_read)
def test_copy02(self):
"""
Test the copy() function some more
"""
# First we create a 1MB file
tmp_file = join(self.tmp_dir, 'NNTPContent_Test.copy', '1MB.rar')
# File should not already exist
assert (isfile(tmp_file) is False)
# Create our file
assert (self.touch(tmp_file, size='1MB', random=True) is True)
# File should exist now
assert (isfile(tmp_file) is True)
# Now we want to load it into a NNTPContent object
content_a = NNTPContent(filepath=tmp_file, work_dir=self.tmp_dir)
# We want to make a copy of it
content_b = content_a.copy()
# Our content should be the same
assert (len(content_a) == len(content_b))
assert (content_a.md5() == content_b.md5())
# Because we made a copy, we're working with a different file however
# we share the exact same content and filename identifier.
assert (content_a == content_b)
# Our paths are not the same though; this is what makes us a official
# copy
assert (content_a.path() != content_b.path())
def test_with(self):
"""
Test the use of the with clause
"""
# First we create a 1MB file
tmp_file = join(self.tmp_dir, 'NNTPContent_Test.with', '1MB.rar')
# File should not already exist
assert (isfile(tmp_file) is False)
# Create our file
assert (self.touch(tmp_file, size='1MB', random=True) is True)
# File should exist now
assert (isfile(tmp_file) is True)
# Now we want to load it into a NNTPContent object
content_a = NNTPContent(filepath=tmp_file, work_dir=self.tmp_dir)
content_a.attach()
# Create a second object
content_b = NNTPContent(work_dir=self.tmp_dir)
with content_a as fp:
while 1:
# Read our data
buf = fp.read(1024)
if not buf:
break
# Write our content to disk
content_b.write(buf)
# At this point we should have a duplicate of our original object
assert (len(content_a) == len(content_b))
assert (content_a.md5() == content_b.md5())
def test_writes(self):
"""
More overhead then a normal write() but none the less, using the
write() in this class keeps things simple since the file is
automatically opened if it was otherwise closed
"""
# First we create a 1MB file
tmp_file = join(self.tmp_dir, 'NNTPContent_Test.write', 'tmp.file')
# File should not already exist
assert(isfile(tmp_file) is False)
# Now we want to create our NNTPContent() object surrouding this
# file that does not exist.
content = NNTPContent(filepath=tmp_file, work_dir=dirname(tmp_file))
# It's worth noting that this file will 'still' not exist
assert(isfile(tmp_file) is False)
# we'll write data
data = 'hello\r\n'
content.write(data)
# It's worth noting that this file will ''STILL'' not exist
assert(isfile(tmp_file) is False)
# Save content
assert(content.save() is True)
# Now the file 'will' exist
assert(isfile(tmp_file) is True)
# Open our file and verify it is the data we saved.
with open(tmp_file) as f:
data_read = f.read()
assert(data == data_read)
def test_from_bestguess(self):
"""
test from_bestguess()
bestguess() does the best of both worlds: from_file() and
from_filename(). It never returns None unless you give it
bad data.
"""
# Initialize our mime object
m = Mime()
# Empty content just gives us an empty response
assert(m.from_bestguess(None) is None)
assert(m.from_bestguess("") is None)
assert(m.from_bestguess(u"") is None)
# First we take a binary file
image = join(self.var_dir, 'joystick.jpg')
c = NNTPContent(image, work_dir=self.tmp_dir)
copy = c.copy()
# since we have a filename, we can pick it up from that
assert(m.from_bestguess(copy.filename).type() == 'image/jpeg')
# We can also get it from_file() because even though our temporary
# file does not have an extension at all, we can still
assert(m.from_bestguess(copy.path()).type() == 'image/jpeg')
def test_with(self):
"""
Test the use of the with clause
"""
# First we create a 1MB file
tmp_file = join(self.tmp_dir, 'NNTPContent_Test.with', '1MB.rar')
# File should not already exist
assert(isfile(tmp_file) is False)
# Create our file
assert(self.touch(tmp_file, size='1MB', random=True) is True)
# File should exist now
assert(isfile(tmp_file) is True)
# Now we want to load it into a NNTPContent object
content_a = NNTPContent(filepath=tmp_file, work_dir=self.tmp_dir)
content_a.attach()
# Create a second object
content_b = NNTPContent(work_dir=self.tmp_dir)
with content_a as fp:
while 1:
# Read our data
buf = fp.read(1024)
if not buf:
break
# Write our content to disk
content_b.write(buf)
# At this point we should have a duplicate of our original object
assert(len(content_a) == len(content_b))
assert(content_a.md5() == content_b.md5())
def test_checksum(self):
"""Test the assorted checksums supported
"""
# First we create a 1MB file
tmp_file = join(
self.tmp_dir, 'NNTPContent_Test.checksum', 'tmpa.tmp')
# File should not already exist
assert(isfile(tmp_file) is False)
# Create a random file
assert(self.touch(tmp_file, size='1MB', random=True) is True)
# File should exist now
assert(isfile(tmp_file) is True)
# Now we want to load it into a NNTPContent object
content = NNTPContent(filepath=tmp_file, work_dir=self.tmp_dir)
md5 = content.md5()
sha1 = content.sha1()
sha256 = content.sha256()
assert(md5 is not None)
assert(sha1 is not None)
assert(sha256 is not None)
tmp_file_2 = join(
self.tmp_dir, 'NNTPContent_Test.checksum', 'tmp2.rar')
# File should not already exist
assert(isfile(tmp_file_2) is False)
# We'll create a copy of our file
assert(content.save(filepath=tmp_file_2, copy=True) is True)
# Now it should
assert(isfile(tmp_file_2) is True)
# Now we'll open the new file we created
content_2 = NNTPContent(filepath=tmp_file_2, work_dir=self.tmp_dir)
md5_2 = content_2.md5()
sha1_2 = content_2.sha1()
sha256_2 = content_2.sha256()
assert(md5_2 is not None)
assert(sha1_2 is not None)
assert(sha256_2 is not None)
# files should be the same
assert(md5 == md5_2)
assert(sha1 == sha1_2)
assert(sha256 == sha256_2)
def test_password_protection(self):
"""
Tests the un-raring of content
"""
# Generate temporary folder to work with
work_dir = join(self.tmp_dir, 'CodecRar_Test.rar.pwd', 'tmp')
# Now we want to prepare a work folder
source_dir = join(self.tmp_dir, 'CodecRar_Test.rar.pwd', 'source')
# Initialize Codec
cr = CodecRar(work_dir=work_dir, password='******')
# create some dummy file entries
file_list = []
for i in range(0, 10):
# Create some temporary files to work with in our source
# directory
tmp_file = join(source_dir, 'DSC_IMG%.3d.jpeg' % i)
self.touch(tmp_file, size='100K', random=True)
# Add our file to the encoding process
file_list.append(NNTPContent(tmp_file))
# Now we want to compress this content
content = cr.encode(name="mystuff", content=file_list)
assert isinstance(content, sortedset)
assert len(content) == 1
# Our content should be password protected at this point; so if i
# create another CodecRar item (without a password), i should fail to
# extract the content.
tmp_cr = CodecRar(work_dir=work_dir)
decoded = tmp_cr.decode(content)
# Bad Password means no results
assert decoded is None
# We could have saved ourselves time in determining this because
# testing doesn't have as much overhead and allows us to check for any
# passwords associated with the data
assert tmp_cr.test(content) is False
# But simply applying a password would have done wonders
assert tmp_cr.test(content, password='******') is True
# We can extract the contents by passing in the password
content = tmp_cr.decode(content, password='******')
assert isinstance(content, sortedset) is True
assert len(content) == 1
assert isinstance(content[0], NNTPBinaryContent) is True
# Content is always attached
assert content[0].is_attached() is True
def test_directory_support(self):
"""
NNTPContent objects can wrap directories too
"""
my_dir = join(self.tmp_dir, 'NNTPContent', 'my_dir')
assert (isdir(my_dir) is False)
assert (mkdir(my_dir) is True)
assert (isdir(my_dir) is True)
# Now create our NNTPContent Object against our directory
obj = NNTPContent(
filepath=my_dir,
work_dir=self.tmp_dir,
)
# Directories are never attached by default
assert (obj.is_attached() is False)
# Deleting the object means the directory remains behind
del obj
assert (isdir(my_dir) is True)
# However
obj = NNTPContent(
filepath=my_dir,
work_dir=self.tmp_dir,
)
assert (obj.is_attached() is False)
obj.attach()
assert (obj.is_attached() is True)
# Now we've attached the NNTPContent to the object so deleting the
# object destroys the directory
del obj
assert (exists(my_dir) is False)
assert (isdir(my_dir) is False)
def test_directory_support(self):
"""
NNTPContent objects can wrap directories too
"""
my_dir = join(self.tmp_dir, 'NNTPContent', 'my_dir')
assert(isdir(my_dir) is False)
assert(mkdir(my_dir) is True)
assert(isdir(my_dir) is True)
# Now create our NNTPContent Object against our directory
obj = NNTPContent(
filepath=my_dir,
work_dir=self.tmp_dir,
)
# Directories are never attached by default
assert(obj.is_attached() is False)
# Deleting the object means the directory remains behind
del obj
assert(isdir(my_dir) is True)
# However
obj = NNTPContent(
filepath=my_dir,
work_dir=self.tmp_dir,
)
assert(obj.is_attached() is False)
obj.attach()
assert(obj.is_attached() is True)
# Now we've attached the NNTPContent to the object so deleting the
# object destroys the directory
del obj
assert(exists(my_dir) is False)
assert(isdir(my_dir) is False)
def encode(self, content, mem_buf=DEFAULT_BUFFER_SIZE):
"""
Encodes an NNTPContent object passed in
"""
if isinstance(content, NNTPContent):
# Create our ascii instance
_encoded = NNTPAsciiContent(
filepath=content.filename,
part=content.part,
total_parts=content.total_parts,
sort_no=content.sort_no,
work_dir=self.work_dir,
# We want to ensure we're working with a unique attached file
unique=True,
)
else:
# If we reach here, we presume our content is a filename
# Create our ascii instance
_encoded = NNTPAsciiContent(
filepath=content,
work_dir=self.work_dir,
# We want to ensure we're working with a unique attached file
unique=True,
)
# Convert our content object into an NNTPContent object
content = NNTPContent(
filepath=content,
work_dir=self.work_dir,
)
# yEnc (v1.3) begin
fmt_ybegin = '=ybegin part=%d total=%d line=%d size=%d name=%s' % (
content.part,
content.total_parts,
self.linelen,
len(content),
content.filename,
)
# yEnc part
fmt_ypart = '=ypart begin=%d end=%d' % (
content.begin() + 1,
content.end(),
)
if isinstance(content._parent, NNTPContent):
# yEnc end
fmt_yend = '=yend size=%d part=%d pcrc32=%s crc32=%s' % (
len(content),
content.part,
content.crc32(),
content._parent.crc32(),
)
else:
# yEnc end
fmt_yend = '=yend size=%d part=%d pcrc32=%s' % (
len(content),
content.part,
content.crc32(),
)
# Write =ybegin line
_encoded.write(fmt_ybegin + EOL)
# Write =ypart line
_encoded.write(fmt_ypart + EOL)
if not content.open():
return None
# Prepare our result set
results = ""
# Column is used for decoding
column = 0
crc = BIN_MASK
# We need to parse the content until we either reach
# the end of the file or get to an 'end' tag
while True:
# Read in our data
data = content.stream.read(mem_buf)
if not data:
# We're done
break
if FAST_YENC_SUPPORT:
try:
_results, crc, column = encode_string(data, crc, column)
# Append our parsed content onto our ongoing buffer
results += _results
except YencError as e:
logger.error("Failed to encode Yenc for %s." % content)
logger.debug('Yenc exception: %s' % (str(e)))
return None
else:
# The slow and painful way, the below looks complicated
# but it really isn't at the the end of the day; yEnc is
# pretty basic;
# - first we translate the all of the characters by adding
# 42 to their value with the exception of a few special
# characters that are explicitly reserved for the yEnc
# language (and conflict with the NNTP Server language).
#
# - next, we need to apply our ENCODE_SPECIAL_MAP to be
# sure to handle the characters that are reserved as
# special keywords used by both NNTP Servers and the yEnc
# protocol itself.
#
# - finally we want to prevent our string from going on for
# to many characters (horizontally). So we need to split
# our content up
#
idx = 0
while idx < len(data):
_byte = (ord(data[idx]) + 42) & 0xff
if _byte in YENC_ENCODE_ESCAPED_CHARACTERS:
_byte = (_byte + 64) & 0xff
# Esape Sequence
results += '='
# Store our character
results += chr(_byte)
# Increment Index
idx += 1
# Our offset
offset = 0
while offset < (len(results) - self.linelen + 1):
eol = offset + self.linelen
if results[offset:eol][-1] == '=':
# Lines can't end with the escape sequence (=). If we get
# here then this one did. We just adjust our end-of-line
# by 1 and keep moving
eol -= 1
_encoded.write(results[offset:eol] + EOL)
offset = eol
if offset < len(results):
results = results[-(len(results) - offset):]
else:
# reset string
results = ''
# We're done reading our data
content.close()
if len(results):
# We still have content left in our buffer
_encoded.write(results + EOL)
# Write footer
_encoded.write(fmt_yend + EOL)
if _encoded:
# close article when complete
_encoded.close()
# Return our encoded object
return _encoded
def test_append(self):
"""
Test the split() and then append() function
"""
# First we create a 1MB file
tmp_file = join(self.tmp_dir, 'NNTPContent_Test.append', '1MB.rar')
# File should not already exist
assert (isfile(tmp_file) is False)
# Create our file
assert (self.touch(tmp_file, size='1MB', random=True) is True)
# File should exist now
assert (isfile(tmp_file) is True)
# Now we want to load it into a NNTPContent object
content_a = NNTPContent(filepath=tmp_file, work_dir=self.tmp_dir)
# Attach our content
assert (content_a.is_attached() is False)
content_a.attach()
# We'll split it in 4
results = content_a.split(strsize_to_bytes('512K'))
# Tests that our results are expected
assert (isinstance(results, sortedset) is True)
assert (len(results) == 2)
# Create a new content object
content_b = NNTPContent(work_dir=self.tmp_dir)
# Our content should be attached
assert (content_a.is_attached() is True)
assert (content_b.is_attached() is True)
# For each of our disassembled items, re-assemble them
for content in results:
assert (content_b.append(content) is True)
# Test that our content is the same again
assert (len(content_a) == len(content_b))
assert (content_a.md5() == content_b.md5())
def test_invalid_split_cases(self):
"""
Test errors that are generated out of the split function
"""
work_dir = join(self.tmp_dir, 'NNTPContent_Test.chunk')
# Now we want to load it into a NNTPContent object
content = NNTPContent(work_dir=work_dir)
# Nothing to split gives an error
assert(content.split() is None)
tmp_file = join(self.tmp_dir, 'NNTPContent_Test.chunk', '5K.rar')
assert(isfile(tmp_file) is False)
assert(self.touch(tmp_file, size='1MB', random=True) is True)
assert(isfile(tmp_file) is True)
# Now we want to load it into a NNTPContent object
content = NNTPContent(filepath=tmp_file, work_dir=self.tmp_dir)
# No size to split on gives an error
assert(content.split(size=0) is None)
assert(content.split(size=-1) is None)
assert(content.split(size=None) is None)
assert(content.split(size='bad_string') is None)
# Invalid Memory Limit
assert(content.split(mem_buf=0) is None)
assert(content.split(mem_buf=-1) is None)
assert(content.split(mem_buf=None) is None)
assert(content.split(mem_buf='bad_string') is None)
def test_copy01(self):
"""
Test our copy function
The copy function allows us to duplicate an existing NNTPContent
object without obstructing the original. Copied content is
always attached; so if the object falls out of scope; so does
the file.
"""
my_dir = join(self.tmp_dir, 'NNTPContent', 'test_copy')
assert (isdir(my_dir) is False)
assert (mkdir(my_dir) is True)
assert (isdir(my_dir) is True)
# Now create our NNTPContent object witin our directory
obj = NNTPContent(
filepath='myfile',
work_dir=my_dir,
)
# Content is attached by default
assert (obj.is_attached() is True)
obj.detach()
assert (obj.is_attached() is False)
new_dir = join(my_dir, 'copy')
assert (isdir(new_dir) is False)
# Create a copy of our object
obj_copy = obj.copy()
# Successfully loaded files are never attached reguardless
# of the original copy
assert (obj_copy.is_attached() is True)
# Reattach the original so it dies when this test is over
obj.attach()
assert (obj.is_attached() is True)
# Create a copy of our copy
obj_copy2 = obj_copy.copy()
assert (obj_copy2.is_attached() is True)
assert (isfile(obj_copy2.path()))
_path = obj_copy2.path()
del obj_copy2
assert (isfile(_path) is False)
assert (isfile(obj_copy.path()) is True)
_path = obj_copy.path()
del obj_copy
assert (isfile(_path) is False)
assert (isfile(obj.path()) is True)
_path = obj.path()
del obj
assert (isfile(_path) is False)
# now lets do a few more tests but with actual files this time
tmp_file = join(my_dir, '2MB.zip')
assert (self.touch(tmp_file, size='2MB', random=True) is True)
# Now create our NNTPContent object witin our directory
obj = NNTPContent(
filepath=tmp_file,
work_dir=my_dir,
)
assert (isfile(obj.path()) is True)
obj_copy = obj.copy()
assert (isfile(obj_copy.path()) is True)
stats = stat(obj_copy.path())
assert (bytes_to_strsize(stats['size']) == "2.00MB")
# Compare that our content is the same
assert (compare(obj_copy.path(), obj.path()) is True)
# note that the filenames are NOT the same so we are dealing with 2
# distinct copies here
assert (isfile(obj_copy.path()) is True)
assert (isfile(obj.path()) is True)
assert (obj.path() != obj_copy.path())
def test_split(self):
"""
Test the split() function
"""
# First we create a 1MB file
tmp_file = join(self.tmp_dir, 'NNTPContent_Test.chunk', '1MB.rar')
# File should not already exist
assert(isfile(tmp_file) is False)
# Create our file
assert(self.touch(tmp_file, size='1MB', random=True) is True)
# File should exist now
assert(isfile(tmp_file) is True)
# Now we want to load it into a NNTPContent object
content = NNTPContent(filepath=tmp_file, work_dir=self.tmp_dir)
# Loaded files are always detached; The following is loaded
# because the path exists.
assert(content.is_attached() is False)
# We'll split it in 2
results = content.split(strsize_to_bytes('512K'))
# Tests that our results are expected
assert(isinstance(results, sortedset) is True)
assert(len(results) == 2)
# We support passing the string format directly in too
results = content.split('512K')
# Tests that our results are expected
assert(isinstance(results, sortedset) is True)
assert(len(results) == 2)
# Now lets merge them into one again
content = NNTPContent(work_dir=self.tmp_dir)
assert(content.load(results) is True)
# NNTPContent() sets as well as individual objects passed into
# load are always attached by default
assert(content.is_attached() is True)
# Our combined file should be the correct filesize
assert(len(content) == strsize_to_bytes('1M'))
# Once we save our object, it is no longer attached
assert(content.save(filepath=tmp_file) is True)
# Now our content is no longer attached
assert(content.is_attached() is False)
# we'll re-attach it
content.attach()
# Our file will be gone now if we try to delete it
assert(content.is_attached() is True)
assert(isfile(tmp_file) is True)
del content
assert(isfile(tmp_file) is False)
def test_checksum(self):
"""Test the assorted checksums supported
"""
# First we create a 1MB file
tmp_file = join(self.tmp_dir, 'NNTPContent_Test.checksum', 'tmpa.tmp')
# File should not already exist
assert (isfile(tmp_file) is False)
# Create a random file
assert (self.touch(tmp_file, size='1MB', random=True) is True)
# File should exist now
assert (isfile(tmp_file) is True)
# Now we want to load it into a NNTPContent object
content = NNTPContent(filepath=tmp_file, work_dir=self.tmp_dir)
md5 = content.md5()
sha1 = content.sha1()
sha256 = content.sha256()
assert (md5 is not None)
assert (sha1 is not None)
assert (sha256 is not None)
tmp_file_2 = join(self.tmp_dir, 'NNTPContent_Test.checksum',
'tmp2.rar')
# File should not already exist
assert (isfile(tmp_file_2) is False)
# We'll create a copy of our file
assert (content.save(filepath=tmp_file_2, copy=True) is True)
# Now it should
assert (isfile(tmp_file_2) is True)
# Now we'll open the new file we created
content_2 = NNTPContent(filepath=tmp_file_2, work_dir=self.tmp_dir)
md5_2 = content_2.md5()
sha1_2 = content_2.sha1()
sha256_2 = content_2.sha256()
assert (md5_2 is not None)
assert (sha1_2 is not None)
assert (sha256_2 is not None)
# files should be the same
assert (md5 == md5_2)
assert (sha1 == sha1_2)
assert (sha256 == sha256_2)
def test_split(self):
"""
Test the split() function
"""
# First we create a 1MB file
tmp_file = join(self.tmp_dir, 'NNTPContent_Test.chunk', '1MB.rar')
# File should not already exist
assert (isfile(tmp_file) is False)
# Create our file
assert (self.touch(tmp_file, size='1MB', random=True) is True)
# File should exist now
assert (isfile(tmp_file) is True)
# Now we want to load it into a NNTPContent object
content = NNTPContent(filepath=tmp_file, work_dir=self.tmp_dir)
# Loaded files are always detached; The following is loaded
# because the path exists.
assert (content.is_attached() is False)
# We'll split it in 2
results = content.split(strsize_to_bytes('512K'))
# Tests that our results are expected
assert (isinstance(results, sortedset) is True)
assert (len(results) == 2)
# We support passing the string format directly in too
results = content.split('512K')
# Tests that our results are expected
assert (isinstance(results, sortedset) is True)
assert (len(results) == 2)
# Now lets merge them into one again
content = NNTPContent(work_dir=self.tmp_dir)
assert (content.load(results) is True)
# NNTPContent() sets as well as individual objects passed into
# load are always attached by default
assert (content.is_attached() is True)
# Our combined file should be the correct filesize
assert (len(content) == strsize_to_bytes('1M'))
# Once we save our object, it is no longer attached
assert (content.save(filepath=tmp_file) is True)
# Now our content is no longer attached
assert (content.is_attached() is False)
# we'll re-attach it
content.attach()
# Our file will be gone now if we try to delete it
assert (content.is_attached() is True)
assert (isfile(tmp_file) is True)
del content
assert (isfile(tmp_file) is False)
def encode(self, content, mem_buf=DEFAULT_BUFFER_SIZE):
"""
Encodes an NNTPContent object passed in
"""
if isinstance(content, NNTPContent):
# Create our ascii instance
_encoded = NNTPAsciiContent(
filepath=content.filename,
part=content.part,
total_parts=content.total_parts,
sort_no=content.sort_no,
work_dir=self.work_dir,
# We want to ensure we're working with a unique attached file
unique=True,
)
else:
# If we reach here, we presume our content is a filename
# Create our ascii instance
_encoded = NNTPAsciiContent(
filepath=content,
work_dir=self.work_dir,
# We want to ensure we're working with a unique attached file
unique=True,
)
# Convert our content object into an NNTPContent object
content = NNTPContent(
filepath=content,
work_dir=self.work_dir,
)
# yEnc (v1.3) begin
fmt_ybegin = '=ybegin part=%d total=%d line=%d size=%d name=%s' % (
content.part, content.total_parts, self.linelen,
len(content), content.filename,
)
# yEnc part
fmt_ypart = '=ypart begin=%d end=%d' % (
content.begin() + 1,
content.end(),
)
if isinstance(content._parent, NNTPContent):
# yEnc end
fmt_yend = '=yend size=%d part=%d pcrc32=%s crc32=%s' % (
len(content), content.part,
content.crc32(), content._parent.crc32(),
)
else:
# yEnc end
fmt_yend = '=yend size=%d part=%d pcrc32=%s' % (
len(content), content.part, content.crc32(),
)
# Write =ybegin line
_encoded.write(fmt_ybegin + EOL)
# Write =ypart line
_encoded.write(fmt_ypart + EOL)
if not content.open():
return None
# Prepare our result set
results = ""
# Column is used for decoding
column = 0
crc = BIN_MASK
# We need to parse the content until we either reach
# the end of the file or get to an 'end' tag
while True:
# Read in our data
data = content.stream.read(mem_buf)
if not data:
# We're done
break
if FAST_YENC_SUPPORT:
try:
_results, crc, column = encode_string(data, crc, column)
# Append our parsed content onto our ongoing buffer
results += _results
except YencError as e:
logger.error("Failed to encode Yenc for %s." % content)
logger.debug('Yenc exception: %s' % (str(e)))
return None
else:
# The slow and painful way, the below looks complicated
# but it really isn't at the the end of the day; yEnc is
# pretty basic;
# - first we translate the all of the characters by adding
# 42 to their value with the exception of a few special
# characters that are explicitly reserved for the yEnc
# language (and conflict with the NNTP Server language).
#
# - next, we need to apply our ENCODE_SPECIAL_MAP to be
# sure to handle the characters that are reserved as
# special keywords used by both NNTP Servers and the yEnc
# protocol itself.
#
# - finally we want to prevent our string from going on for
# to many characters (horizontally). So we need to split
# our content up
#
idx = 0
while idx < len(data):
_byte = (ord(data[idx]) + 42) & 0xff
if _byte in YENC_ENCODE_ESCAPED_CHARACTERS:
_byte = (_byte + 64) & 0xff
# Esape Sequence
results += '='
# Store our character
results += chr(_byte)
# Increment Index
idx += 1
# Our offset
offset = 0
while offset < (len(results)-self.linelen+1):
eol = offset+self.linelen
if results[offset:eol][-1] == '=':
# Lines can't end with the escape sequence (=). If we get
# here then this one did. We just adjust our end-of-line
# by 1 and keep moving
eol -= 1
_encoded.write(results[offset:eol] + EOL)
offset = eol
if offset < len(results):
results = results[-(len(results) - offset):]
else:
# reset string
results = ''
# We're done reading our data
content.close()
if len(results):
# We still have content left in our buffer
_encoded.write(results + EOL)
# Write footer
_encoded.write(fmt_yend + EOL)
if _encoded:
# close article when complete
_encoded.close()
# Return our encoded object
return _encoded
def test_rar_errors(self):
"""
Test that we fail under certain conditions
"""
# Generate temporary folder to work with
work_dir = join(self.tmp_dir, 'CodecRar_Test.rar.fail', 'work')
# Now we want to prepare a folder filled with temporary content
# Note: this directory is horrible because it's 'within' our work_dir
# as a result, adding content should not succeed
source_dir = join(work_dir, 'test')
# Initialize Codec (without volume_size disables it)
cr = CodecRar(work_dir=work_dir)
# No files
assert len(cr) == 0
tmp_file = join(source_dir, 'temp_file_non-existant')
assert isfile(tmp_file) is False
# We can't add content that does not exist
assert cr.add(tmp_file) is False
# Still No files
assert len(cr) == 0
# However directories can not cross into our work directory
tmp_dir = dirname(work_dir)
# We intentionally pick a directory that has the work_dir
# as a child within it
assert isdir(tmp_dir)
# Denied adding the file because it would include the work_dir
# if we did
assert cr.add(tmp_file) is False
# Temporary file (within directory denied in previous check)
tmp_file = join(tmp_dir, 'temp_file')
assert isfile(tmp_file) is False
# Create our temporary file now
self.touch(tmp_file, size='120K', random=True)
assert isfile(tmp_file) is True
# This file is within our work_dir but we're still okay because
# it's accessing the file explicitly and the fact it's a file
# and not a directory
assert cr.add(tmp_file) is True
# Now we'll have 1 entry in our list
assert len(cr) == 1
# You can't add duplicates
assert cr.add(tmp_file) is False
# We still have 1 entry
assert len(cr) == 1
# Empty NNTPContent() can not be added
content = NNTPContent(unique=True, work_dir=self.tmp_dir)
# Can't do it
assert cr.add(content) is False
# Store some data
content.write('some data\r\n')
# Now we can add it because it has data in it
assert cr.add(content) is True
# We now have 2 entries
assert len(cr) == 2
# We can't add duplicates
assert cr.add(content) is False
# We still have 2 entries
assert len(cr) == 2
# Empty NNTPArticle() can not be added
article = NNTPArticle(work_dir=self.tmp_dir)
# Can't do it
assert cr.add(article) is False
# If we add content that's already been added, nothing
# new will happen either
assert article.add(content) is True
# Still can't do (only because it was already added)
assert cr.add(article) is False
# We still have 2 entries
assert len(cr) == 2
# New Empty NNTPContent() can not be added
content = NNTPContent(unique=True, work_dir=self.tmp_dir)
# We'll add our new content to our article
assert article.add(content) is True
# Our new content has no data associated with it, so this should
# still fail
assert cr.add(article) is False
# We still have 2 entries
assert len(cr) == 2
# Store some new data
content.write('some new data\r\n')
# Our new content within our article now has data so this will work
assert cr.add(article) is True
# We now have 3 entries
assert len(cr) == 3
def test_append(self):
"""
Test the split() and then append() function
"""
# First we create a 1MB file
tmp_file = join(self.tmp_dir, 'NNTPContent_Test.append', '1MB.rar')
# File should not already exist
assert(isfile(tmp_file) is False)
# Create our file
assert(self.touch(tmp_file, size='1MB', random=True) is True)
# File should exist now
assert(isfile(tmp_file) is True)
# Now we want to load it into a NNTPContent object
content_a = NNTPContent(filepath=tmp_file, work_dir=self.tmp_dir)
# Attach our content
assert(content_a.is_attached() is False)
content_a.attach()
# We'll split it in 4
results = content_a.split(strsize_to_bytes('512K'))
# Tests that our results are expected
assert(isinstance(results, sortedset) is True)
assert(len(results) == 2)
# Create a new content object
content_b = NNTPContent(work_dir=self.tmp_dir)
# Our content should be attached
assert(content_a.is_attached() is True)
assert(content_b.is_attached() is True)
# For each of our disassembled items, re-assemble them
for content in results:
assert(content_b.append(content) is True)
# Test that our content is the same again
assert(len(content_a) == len(content_b))
assert(content_a.md5() == content_b.md5())
def test_copy01(self):
"""
Test our copy function
The copy function allows us to duplicate an existing NNTPContent
object without obstructing the original. Copied content is
always attached; so if the object falls out of scope; so does
the file.
"""
my_dir = join(self.tmp_dir, 'NNTPContent', 'test_copy')
assert(isdir(my_dir) is False)
assert(mkdir(my_dir) is True)
assert(isdir(my_dir) is True)
# Now create our NNTPContent object witin our directory
obj = NNTPContent(
filepath='myfile',
work_dir=my_dir,
)
# Content is attached by default
assert(obj.is_attached() is True)
obj.detach()
assert(obj.is_attached() is False)
new_dir = join(my_dir, 'copy')
assert(isdir(new_dir) is False)
# Create a copy of our object
obj_copy = obj.copy()
# Successfully loaded files are never attached reguardless
# of the original copy
assert(obj_copy.is_attached() is True)
# Reattach the original so it dies when this test is over
obj.attach()
assert(obj.is_attached() is True)
# Create a copy of our copy
obj_copy2 = obj_copy.copy()
assert(obj_copy2.is_attached() is True)
assert(isfile(obj_copy2.path()))
_path = obj_copy2.path()
del obj_copy2
assert(isfile(_path) is False)
assert(isfile(obj_copy.path()) is True)
_path = obj_copy.path()
del obj_copy
assert(isfile(_path) is False)
assert(isfile(obj.path()) is True)
_path = obj.path()
del obj
assert(isfile(_path) is False)
# now lets do a few more tests but with actual files this time
tmp_file = join(my_dir, '2MB.zip')
assert(self.touch(tmp_file, size='2MB', random=True) is True)
# Now create our NNTPContent object witin our directory
obj = NNTPContent(
filepath=tmp_file,
work_dir=my_dir,
)
assert(isfile(obj.path()) is True)
obj_copy = obj.copy()
assert(isfile(obj_copy.path()) is True)
stats = stat(obj_copy.path())
assert(bytes_to_strsize(stats['size']) == "2.00MB")
# Compare that our content is the same
assert(compare(obj_copy.path(), obj.path()) is True)
# note that the filenames are NOT the same so we are dealing with 2
# distinct copies here
assert(isfile(obj_copy.path()) is True)
assert(isfile(obj.path()) is True)
assert(obj.path() != obj_copy.path())
def test_7z_errors(self):
"""
Test that we fail under certain conditions
"""
# Generate temporary folder to work with
work_dir = join(self.tmp_dir, 'Codec7Zip_Test.7z.fail', 'work')
# Now we want to prepare a folder filled with temporary content
# Note: this directory is horrible because it's 'within' our work_dir
# as a result, adding content should not succeed
source_dir = join(work_dir, 'test')
# Initialize Codec (without volume_size disables it)
cr = Codec7Zip(work_dir=work_dir)
# No files
assert len(cr) == 0
tmp_file = join(source_dir, 'temp_file_non-existant')
assert isfile(tmp_file) is False
# We can't add content that does not exist
assert cr.add(tmp_file) is False
# Still No files
assert len(cr) == 0
# However directories can not cross into our work directory
tmp_dir = dirname(work_dir)
# We intentionally pick a directory that has the work_dir
# as a child within it
assert isdir(tmp_dir)
# Denied adding the file because it would include the work_dir
# if we did
assert cr.add(tmp_file) is False
# Temporary file (within directory denied in previous check)
tmp_file = join(tmp_dir, 'temp_file')
assert isfile(tmp_file) is False
# Create our temporary file now
self.touch(tmp_file, size='120K', random=True)
assert isfile(tmp_file) is True
# This file is within our work_dir but we're still okay because
# it's accessing the file explicitly and the fact it's a file
# and not a directory
assert cr.add(tmp_file) is True
# Now we'll have 1 entry in our list
assert len(cr) == 1
# You can't add duplicates
assert cr.add(tmp_file) is False
# We still have 1 entry
assert len(cr) == 1
# Empty NNTPContent() can not be added
content = NNTPContent(unique=True, work_dir=self.tmp_dir)
# Can't do it
assert cr.add(content) is False
# Store some data
content.write('some data\r\n')
# Now we can add it because it has data in it
assert cr.add(content) is True
# We now have 2 entries
assert len(cr) == 2
# We can't add duplicates
assert cr.add(content) is False
# We still have 2 entries
assert len(cr) == 2
# Empty NNTPArticle() can not be added
article = NNTPArticle(work_dir=self.tmp_dir)
# Can't do it
assert cr.add(article) is False
# If we add content that's already been added, nothing
# new will happen either
assert article.add(content) is True
# Still can't do (only because it was already added)
assert cr.add(article) is False
# We still have 2 entries
assert len(cr) == 2
# New Empty NNTPContent() can not be added
content = NNTPContent(unique=True, work_dir=self.tmp_dir)
# We'll add our new content to our article
assert article.add(content) is True
# Our new content has no data associated with it, so this should
# still fail
assert cr.add(article) is False
# We still have 2 entries
assert len(cr) == 2
# Store some new data
content.write('some new data\r\n')
# Our new content within our article now has data so this will work
assert cr.add(article) is True
# We now have 3 entries
assert len(cr) == 3
def test_saves(self):
"""
Saving allows for a variety of inputs, test that they all
check out okay
"""
# First we create a 1MB file
tmp_file = join(self.tmp_dir, 'NNTPContent_Test.save', 'testfile.tmp')
# File should not already exist
assert (isfile(tmp_file) is False)
# Create a random file
assert (self.touch(tmp_file, size='5MB', random=True) is True)
# File should exist now
assert (isfile(tmp_file) is True)
# Now we want to load it into a NNTPContent object
content = NNTPContent(filepath=tmp_file, work_dir=self.tmp_dir)
# Test our file exists
assert (len(content) == strsize_to_bytes('5M'))
# By default our load makes it so our file is NOT attached
assert (content.is_attached() is False)
# as we can't make a copy of our current file on top of the old
_filepath = content.path()
assert (content.save(copy=True) is True)
assert (content.path() == _filepath)
assert (content.path() == tmp_file)
# File should still be detached
assert (content.is_attached() is False)
# Filepath shoul still not have changed
# Let's attach it
content.attach()
assert (content.is_attached() is True)
# If we call save() a copy parameter, there should be no change
assert (content.save(copy=True) is True)
# Still no change
assert (content.is_attached() is True)
# Now lets actually copy it to a new location
tmp_file_copy = join(self.tmp_dir, 'NNTPContent_Test.save',
'testfile.copy.tmp')
# File should not already exist
assert (isfile(tmp_file_copy) is False)
# call save using our copy variable and new filename
assert (content.save(tmp_file_copy, copy=True) is True)
# File should exist now
assert (isfile(tmp_file_copy) is True)
# Old File should still exist too
assert (isfile(tmp_file) is True)
# Path should still be the old path and not the new
assert (content.path() == tmp_file)
# Still no change in attachment
assert (content.is_attached() is True)
# Create a new file now
tmp_file_copy2 = join(self.tmp_dir, 'NNTPContent_Test.save',
'testfile.copy2.tmp')
assert (isfile(tmp_file_copy2) is False)
# call save with copy set to false; This performs an official
# move (adjusting our internal records.
assert (content.save(tmp_file_copy2, copy=False) is True)
# Old File should no longer exist
assert (isfile(tmp_file) is False)
# Content should be detached
assert (content.is_attached() is False)
# New file should exist
assert (isfile(tmp_file_copy2) is True)
assert (content.path() != _filepath)
assert (content.path() == tmp_file_copy2)
def test_saves(self):
"""
Saving allows for a variety of inputs, test that they all
check out okay
"""
# First we create a 1MB file
tmp_file = join(
self.tmp_dir, 'NNTPContent_Test.save', 'testfile.tmp')
# File should not already exist
assert(isfile(tmp_file) is False)
# Create a random file
assert(self.touch(tmp_file, size='5MB', random=True) is True)
# File should exist now
assert(isfile(tmp_file) is True)
# Now we want to load it into a NNTPContent object
content = NNTPContent(filepath=tmp_file, work_dir=self.tmp_dir)
# Test our file exists
assert(len(content) == strsize_to_bytes('5M'))
# By default our load makes it so our file is NOT attached
assert(content.is_attached() is False)
# as we can't make a copy of our current file on top of the old
_filepath = content.path()
assert(content.save(copy=True) is True)
assert(content.path() == _filepath)
assert(content.path() == tmp_file)
# File should still be detached
assert(content.is_attached() is False)
# Filepath shoul still not have changed
# Let's attach it
content.attach()
assert(content.is_attached() is True)
# If we call save() a copy parameter, there should be no change
assert(content.save(copy=True) is True)
# Still no change
assert(content.is_attached() is True)
# Now lets actually copy it to a new location
tmp_file_copy = join(
self.tmp_dir, 'NNTPContent_Test.save', 'testfile.copy.tmp')
# File should not already exist
assert(isfile(tmp_file_copy) is False)
# call save using our copy variable and new filename
assert(content.save(tmp_file_copy, copy=True) is True)
# File should exist now
assert(isfile(tmp_file_copy) is True)
# Old File should still exist too
assert(isfile(tmp_file) is True)
# Path should still be the old path and not the new
assert(content.path() == tmp_file)
# Still no change in attachment
assert(content.is_attached() is True)
# Create a new file now
tmp_file_copy2 = join(
self.tmp_dir, 'NNTPContent_Test.save', 'testfile.copy2.tmp')
assert(isfile(tmp_file_copy2) is False)
# call save with copy set to false; This performs an official
# move (adjusting our internal records.
assert(content.save(tmp_file_copy2, copy=False) is True)
# Old File should no longer exist
assert(isfile(tmp_file) is False)
# Content should be detached
assert(content.is_attached() is False)
# New file should exist
assert(isfile(tmp_file_copy2) is True)
assert(content.path() != _filepath)
assert(content.path() == tmp_file_copy2)
def test_get_paths(self):
"""
Test that we fail under certain conditions
"""
# Generate temporary folder to work with
work_dir = join(self.tmp_dir, 'CodecFile_Test', 'work')
# Initialize Codec (without volume_size disables it)
cr = CodecFile(work_dir=work_dir)
# create some dummy file entries
tmp_files = set()
for i in range(0, 10):
# Create some temporary files to work with in our source
# directory
tmp_file = join(work_dir, 'DSC_IMG%.3d.jpeg' % i)
self.touch(tmp_file, size='120K', random=True)
# Add a file to our tmp_files list
tmp_files.add(tmp_file)
# Non-existant file reference
invalid_file = join(self.tmp_dir, 'non-existant-file')
assert isfile(invalid_file) is False
content = NNTPContent(
join(work_dir, 'testfile'),
work_dir=self.tmp_dir,
)
content.write('test data')
# Empty NNTPArticle() can not be added
article = NNTPArticle(work_dir=self.tmp_dir)
# New Empty NNTPContent() can not be added
article_content = NNTPContent(
join(work_dir, 'testfile2'),
work_dir=self.tmp_dir,
)
# Store some new data
article_content.write('some more test data')
# We'll add our new content to our article
assert article.add(content) is True
# save path
sub_dir = join(work_dir, 'subdir')
assert mkdir(sub_dir) is True
assert isdir(sub_dir) is True
# string work
assert len(cr.get_paths(self.tmp_dir)) == 1
assert cr.get_paths(self.tmp_dir).pop() == self.tmp_dir
# Sub-directories that exist within a root directory also included are
# removed
assert len(cr.get_paths([self.tmp_dir, sub_dir])) == 1
assert cr.get_paths([self.tmp_dir, sub_dir]).pop() == self.tmp_dir
# Invalid files/dirs are not found
assert len(cr.get_paths(invalid_file)) == 0
# Create a list of many assorted type of items
__set = set([
work_dir,
sub_dir,
article_content,
content,
invalid_file,
]) | set(tmp_files)
# At the end of the day, the work_dir includes all of the sub-content
# and the invalid_file is simply just tossed. However because our
# NNTPContent() and NNTPArticle() files are stored outside of our
# work_dir, they will also be included in the results
results = cr.get_paths(__set)
assert len(results) == 3
assert work_dir in results
assert content.filepath in results
assert article_content.filepath in results
# Now if we did the same test but without the work_dir directory, then
# we'd have a much larger list; we'll work with lists this time to
# show that we support them too
__list = [
sub_dir,
article_content,
content,
invalid_file,
]
__list.extend(tmp_files)
results = cr.get_paths(__list)
# +1 for content
# +1 for sub_dir
assert len(results) == (len(tmp_files) + len(article) + 2)
for f in tmp_files:
# Each file in our tmp_files will be in our results
assert f in results
assert content.filepath in results
assert article_content.filepath in results
assert sub_dir in results
def test_invalid_split_cases(self):
"""
Test errors that are generated out of the split function
"""
work_dir = join(self.tmp_dir, 'NNTPContent_Test.chunk')
# Now we want to load it into a NNTPContent object
content = NNTPContent(work_dir=work_dir)
# Nothing to split gives an error
assert (content.split() is None)
tmp_file = join(self.tmp_dir, 'NNTPContent_Test.chunk', '5K.rar')
assert (isfile(tmp_file) is False)
assert (self.touch(tmp_file, size='1MB', random=True) is True)
assert (isfile(tmp_file) is True)
# Now we want to load it into a NNTPContent object
content = NNTPContent(filepath=tmp_file, work_dir=self.tmp_dir)
# No size to split on gives an error
assert (content.split(size=0) is None)
assert (content.split(size=-1) is None)
assert (content.split(size=None) is None)
assert (content.split(size='bad_string') is None)
# Invalid Memory Limit
assert (content.split(mem_buf=0) is None)
assert (content.split(mem_buf=-1) is None)
assert (content.split(mem_buf=None) is None)
assert (content.split(mem_buf='bad_string') is None)
