def test_split(self):
"""
Test the split() function
"""
# First we create a 1MB file
tmp_file = join(self.tmp_dir, 'NNTPContent_Test.chunk', '1MB.rar')
# File should not already exist
assert(isfile(tmp_file) is False)
# Create our file
assert(self.touch(tmp_file, size='1MB', random=True) is True)
# File should exist now
assert(isfile(tmp_file) is True)
# Now we want to load it into a NNTPContent object
content = NNTPContent(filepath=tmp_file, work_dir=self.tmp_dir)
# Loaded files are always detached; The following is loaded
# because the path exists.
assert(content.is_attached() is False)
# We'll split it in 2
results = content.split(strsize_to_bytes('512K'))
# Tests that our results are expected
assert(isinstance(results, sortedset) is True)
assert(len(results) == 2)
# We support passing the string format directly in too
results = content.split('512K')
# Tests that our results are expected
assert(isinstance(results, sortedset) is True)
assert(len(results) == 2)
# Now lets merge them into one again
content = NNTPContent(work_dir=self.tmp_dir)
assert(content.load(results) is True)
# NNTPContent() sets as well as individual objects passed into
# load are always attached by default
assert(content.is_attached() is True)
# Our combined file should be the correct filesize
assert(len(content) == strsize_to_bytes('1M'))
# Once we save our object, it is no longer attached
assert(content.save(filepath=tmp_file) is True)
# Now our content is no longer attached
assert(content.is_attached() is False)
# we'll re-attach it
content.attach()
# Our file will be gone now if we try to delete it
assert(content.is_attached() is True)
assert(isfile(tmp_file) is True)
del content
assert(isfile(tmp_file) is False)
def test_split(self):
"""
Test the split() function
"""
# First we create a 1MB file
tmp_file = join(self.tmp_dir, 'NNTPContent_Test.chunk', '1MB.rar')
# File should not already exist
assert (isfile(tmp_file) is False)
# Create our file
assert (self.touch(tmp_file, size='1MB', random=True) is True)
# File should exist now
assert (isfile(tmp_file) is True)
# Now we want to load it into a NNTPContent object
content = NNTPContent(filepath=tmp_file, work_dir=self.tmp_dir)
# Loaded files are always detached; The following is loaded
# because the path exists.
assert (content.is_attached() is False)
# We'll split it in 2
results = content.split(strsize_to_bytes('512K'))
# Tests that our results are expected
assert (isinstance(results, sortedset) is True)
assert (len(results) == 2)
# We support passing the string format directly in too
results = content.split('512K')
# Tests that our results are expected
assert (isinstance(results, sortedset) is True)
assert (len(results) == 2)
# Now lets merge them into one again
content = NNTPContent(work_dir=self.tmp_dir)
assert (content.load(results) is True)
# NNTPContent() sets as well as individual objects passed into
# load are always attached by default
assert (content.is_attached() is True)
# Our combined file should be the correct filesize
assert (len(content) == strsize_to_bytes('1M'))
# Once we save our object, it is no longer attached
assert (content.save(filepath=tmp_file) is True)
# Now our content is no longer attached
assert (content.is_attached() is False)
# we'll re-attach it
content.attach()
# Our file will be gone now if we try to delete it
assert (content.is_attached() is True)
assert (isfile(tmp_file) is True)
del content
assert (isfile(tmp_file) is False)
def __init__(
self,
work_dir=None,
bin_path=DEFAULT_PATH,
# The volume size identifies how to split up the 7-Zip file
# (into multi-parts). If set to False then no splitting is
# done. If it's set to True, then the Default is used.
volume_size=False,
# Don't overwrite existing files if one of the same name is
# found in the archive. Set this to True if you want to
# overwrite matched content.
overwrite=False,
*args,
**kwargs):
"""
Initialize the Codec
"""
super(Codec7Zip, self).__init__(work_dir=work_dir, *args, **kwargs)
# Initalize Paths
self._bin = bin_path
# +++++++++++++++++
# Encoding Settings
# +++++++++++++++++
# Exclude the base directory when creating an archive
self.volume_size = volume_size
if isinstance(self.volume_size, basestring):
self.volume_size = int(strsize_to_bytes(self.volume_size))
if self.volume_size is True:
self.volume_size = DEFAULT_SPLIT_SIZE
self.overwrite = overwrite
def test_dirsize(self):
"""
tests dirsize()
"""
# No problem if the directory doesn't exist; it's sizeless
assert(dirsize('non/existant/directory') == 0)
work_dir = join(self.tmp_dir, 'Utils_Test.dirsize', 'dirA')
# The directory should not exist
assert isdir(work_dir) is False
# mkdir() should be successful
assert mkdir(work_dir) is True
# The directory should exist now
assert isdir(work_dir) is True
# Directory should be zero-bytes
assert(dirsize(work_dir) == 0)
tmp_file01 = join(work_dir, 'test01_1MB')
assert self.touch(tmp_file01, size='1MB')
# Directory should be 1MB in size
assert(dirsize(work_dir) == strsize_to_bytes('1MB'))
tmp_file02 = join(work_dir, 'test02_1MB')
assert self.touch(tmp_file02, size='1MB')
# Now we should have double the storage
assert(dirsize(work_dir) == strsize_to_bytes('2MB'))
# Lets make the directory inaccessible
chmod(work_dir, 0000)
# Since we can't officially calculate the total size we abort
# internally, but instead of returning zero, we return None
# to signify our failure
assert(dirsize(work_dir) is None)
# restore our accessibility
chmod(work_dir, 0755)
# Back to normal
assert(dirsize(work_dir) == strsize_to_bytes('2MB'))
def test_dirsize(self):
"""
tests dirsize()
"""
# No problem if the directory doesn't exist; it's sizeless
assert (dirsize('non/existant/directory') == 0)
work_dir = join(self.tmp_dir, 'Utils_Test.dirsize', 'dirA')
# The directory should not exist
assert isdir(work_dir) is False
# mkdir() should be successful
assert mkdir(work_dir) is True
# The directory should exist now
assert isdir(work_dir) is True
# Directory should be zero-bytes
assert (dirsize(work_dir) == 0)
tmp_file01 = join(work_dir, 'test01_1MB')
assert self.touch(tmp_file01, size='1MB')
# Directory should be 1MB in size
assert (dirsize(work_dir) == strsize_to_bytes('1MB'))
tmp_file02 = join(work_dir, 'test02_1MB')
assert self.touch(tmp_file02, size='1MB')
# Now we should have double the storage
assert (dirsize(work_dir) == strsize_to_bytes('2MB'))
# Lets make the directory inaccessible
chmod(work_dir, 0000)
# Since we can't officially calculate the total size we abort
# internally, but instead of returning zero, we return None
# to signify our failure
assert (dirsize(work_dir) is None)
# restore our accessibility
chmod(work_dir, 0755)
# Back to normal
assert (dirsize(work_dir) == strsize_to_bytes('2MB'))
def test_article_splitting(self):
"""
Tests that articles can split
"""
# Duplicates groups are are removed automatically
article = NNTPArticle(
work_dir=self.tmp_dir,
subject='split-test',
poster='<*****@*****.**>',
groups='alt.binaries.l2g',
)
# Nothing to split gives an error
assert(article.split() is None)
tmp_file = join(self.tmp_dir, 'NNTPArticle_Test.chunk', '1MB.rar')
# The file doesn't exist at first
assert(isfile(tmp_file) is False)
# Create it
assert(self.touch(tmp_file, size='1MB', random=True) is True)
# Now it does
assert(isfile(tmp_file) is True)
# Now we want to load it into a NNTPContent object
content = NNTPBinaryContent(filepath=tmp_file, work_dir=self.tmp_dir)
# Add our object to our article
assert(article.add(content) is True)
# No size to split on gives an error
assert(article.split(size=0) is None)
assert(article.split(size=-1) is None)
assert(article.split(size=None) is None)
assert(article.split(size='bad_string') is None)
# Invalid Memory Limit
assert(article.split(mem_buf=0) is None)
assert(article.split(mem_buf=-1) is None)
assert(article.split(mem_buf=None) is None)
assert(article.split(mem_buf='bad_string') is None)
# We'll split it in 2
results = article.split(strsize_to_bytes('512K'))
# Tests that our results are expected
assert(isinstance(results, sortedset) is True)
assert(len(results) == 2)
# Test that the parts were assigned correctly
for i, article in enumerate(results):
# We should only have one content object
assert(isinstance(article, NNTPArticle) is True)
assert(len(article) == 1)
# Our content object should correctly have the part and
# total part contents populated correctly
assert(article[0].part == (i+1))
assert(article[0].total_parts == len(results))
def __init__(
self,
work_dir=None,
rar_path=DEFAULT_RAR_PATH,
unrar_path=DEFAULT_UNRAR_PATH,
# Recovery Record Percentage (default at 5%)
recovery_record='5p',
# The volume size identifies how to split up the RAR file
# (into multi-parts). If set to False then no splitting is
# done. If it's set to True, then the Default is used.
volume_size=False,
# Even if the RAR is damaged, keep going if you can; Set this
# to True if you actually want this to happen. Broken files
# are written to as much as they can and extraction continues
# as long as it can.
keep_broken=False,
# Don't overwrite existing files if one of the same name is
# found in the archive. Set this to True if you want to
# overwrite matched content.
overwrite=False,
# Update the dates associated with the files extracted to be
# current.
freshen=False,
*args,
**kwargs):
"""
Initialize the Codec
"""
super(CodecRar, self).__init__(work_dir=work_dir, *args, **kwargs)
# Initalize Paths
self._rar = rar_path
self._unrar = unrar_path
# +++++++++++++++++
# Encoding Settings
# +++++++++++++++++
# Recovery record
# For RAR 4.x
# The parameter can be either the number of recovery sectors
# (n=1… 524288) or percent of archive size if '%' or 'p'
self.recovery_record = recovery_record
if isinstance(self.recovery_record, int):
self.recovery_record = str(self.recovery_record)
# Exclude the base directory when creating an archive
self.volume_size = volume_size
if isinstance(self.volume_size, basestring):
self.volume_size = int(strsize_to_bytes(self.volume_size))
if self.volume_size is True:
self.volume_size = DEFAULT_SPLIT_SIZE
self.keep_broken = keep_broken
self.overwrite = overwrite
self.freshen = freshen
def __init__(self, work_dir=None,
rar_path=DEFAULT_RAR_PATH,
unrar_path=DEFAULT_UNRAR_PATH,
# Recovery Record Percentage (default at 5%)
recovery_record='5p',
# The volume size identifies how to split up the RAR file
# (into multi-parts). If set to False then no splitting is
# done. If it's set to True, then the Default is used.
volume_size=False,
# Even if the RAR is damaged, keep going if you can; Set this
# to True if you actually want this to happen. Broken files
# are written to as much as they can and extraction continues
# as long as it can.
keep_broken=False,
# Don't overwrite existing files if one of the same name is
# found in the archive. Set this to True if you want to
# overwrite matched content.
overwrite=False,
# Update the dates associated with the files extracted to be
# current.
freshen=False,
*args, **kwargs):
"""
Initialize the Codec
"""
super(CodecRar, self).__init__(work_dir=work_dir, *args, **kwargs)
# Initalize Paths
self._rar = rar_path
self._unrar = unrar_path
# +++++++++++++++++
# Encoding Settings
# +++++++++++++++++
# Recovery record
# For RAR 4.x
# The parameter can be either the number of recovery sectors
# (n=1… 524288) or percent of archive size if '%' or 'p'
self.recovery_record = recovery_record
if isinstance(self.recovery_record, int):
self.recovery_record = str(self.recovery_record)
# Exclude the base directory when creating an archive
self.volume_size = volume_size
if isinstance(self.volume_size, basestring):
self.volume_size = int(strsize_to_bytes(self.volume_size))
if self.volume_size is True:
self.volume_size = DEFAULT_SPLIT_SIZE
self.keep_broken = keep_broken
self.overwrite = overwrite
self.freshen = freshen
def __init__(self, work_dir=None,
par_path=DEFAULT_PAR2_PATH,
par_version=ParVersion.Two,
# Recovery Record Percentage (default at 5%)
recovery_percent=DEFAULT_RECOVERY_PERCENT,
# The block size should be set to the size of the article
# being posted on usenet. For example, if each article is
# 300K, then that is what the block_size should be set to.
# If set to False then no splitting is done. If it's set to
# True, then the Default is used.
block_size=True,
*args, **kwargs):
"""
Initialize the Codec
"""
super(CodecPar, self).__init__(work_dir=work_dir, *args, **kwargs)
# Initalize Paths
self._par = par_path
self._par_version = par_version
if self._par_version not in PAR_VERSIONS:
raise AttributeError(
'CodecPar: Invalid PAR Version specified %s' %
str(self._par_version))
# +++++++++++++++++
# Encoding Settings
# +++++++++++++++++
self.block_size = block_size
if isinstance(self.block_size, basestring):
self.block_size = int(strsize_to_bytes(self.block_size))
if self.block_size is True:
self.block_size = DEFAULT_BLOCK_SIZE
self.recovery_percent = recovery_percent
if self.recovery_percent:
if self.recovery_percent < 0 or self.recovery_percent > 100:
raise AttributeError(
'CodecPar: Invalid recovery record percent %s%%.' %
self.recovery_percent)
def setUp(self):
"""
Grab a few more things from the config
"""
super(NNTPSimpleFilter_Test, self).setUp()
# Create a template entry we can clone from to make
# thing easier to manipulate in each individual test
self.template_entry = {
'id': 'testuniqueid',
'article_no': 1,
'poster': 'Unknown Poster <*****@*****.**>',
'date': datetime(2000, 1, 1, 0, 0, 0, tzinfo=pytz.UTC),
'subject': 'What.A.Great.Show (1/1) "what.a.great.show.mkv" Yenc (1/1)',
'size': strsize_to_bytes('25M'),
'lines': 3000,
'group': 'alt.binaries.test',
'xgroups': { 'alt.binaries.ohwell': 2, 'alt.binaries.ohwell2': 3, },
}
def test_append(self):
"""
Test the split() and then append() function
"""
# First we create a 1MB file
tmp_file = join(self.tmp_dir, 'NNTPContent_Test.append', '1MB.rar')
# File should not already exist
assert(isfile(tmp_file) is False)
# Create our file
assert(self.touch(tmp_file, size='1MB', random=True) is True)
# File should exist now
assert(isfile(tmp_file) is True)
# Now we want to load it into a NNTPContent object
content_a = NNTPContent(filepath=tmp_file, work_dir=self.tmp_dir)
# Attach our content
assert(content_a.is_attached() is False)
content_a.attach()
# We'll split it in 4
results = content_a.split(strsize_to_bytes('512K'))
# Tests that our results are expected
assert(isinstance(results, sortedset) is True)
assert(len(results) == 2)
# Create a new content object
content_b = NNTPContent(work_dir=self.tmp_dir)
# Our content should be attached
assert(content_a.is_attached() is True)
assert(content_b.is_attached() is True)
# For each of our disassembled items, re-assemble them
for content in results:
assert(content_b.append(content) is True)
# Test that our content is the same again
assert(len(content_a) == len(content_b))
assert(content_a.md5() == content_b.md5())
def test_append(self):
"""
Test the split() and then append() function
"""
# First we create a 1MB file
tmp_file = join(self.tmp_dir, 'NNTPContent_Test.append', '1MB.rar')
# File should not already exist
assert (isfile(tmp_file) is False)
# Create our file
assert (self.touch(tmp_file, size='1MB', random=True) is True)
# File should exist now
assert (isfile(tmp_file) is True)
# Now we want to load it into a NNTPContent object
content_a = NNTPContent(filepath=tmp_file, work_dir=self.tmp_dir)
# Attach our content
assert (content_a.is_attached() is False)
content_a.attach()
# We'll split it in 4
results = content_a.split(strsize_to_bytes('512K'))
# Tests that our results are expected
assert (isinstance(results, sortedset) is True)
assert (len(results) == 2)
# Create a new content object
content_b = NNTPContent(work_dir=self.tmp_dir)
# Our content should be attached
assert (content_a.is_attached() is True)
assert (content_b.is_attached() is True)
# For each of our disassembled items, re-assemble them
for content in results:
assert (content_b.append(content) is True)
# Test that our content is the same again
assert (len(content_a) == len(content_b))
assert (content_a.md5() == content_b.md5())
def touch(self, path, size=None, random=False, perm=None, time=None):
"""Simplify the dynamic creation of files or the updating of their
modified time. If a size is specified, then a file of that size
will be created on the disk. If the file already exists, then the
size= attribute is ignored (for safey reasons).
if random is set to true, then the file created is actually
created using tons of randomly generated content. This is MUCH
slower but nessisary for certain tests.
"""
path = abspath(path)
if not isdir(dirname(path)):
mkdir(dirname(path), 0700)
if not exists(path):
size = strsize_to_bytes(size)
if not random:
f = open(path, "wb")
if isinstance(size, int) and size > 0:
f.seek(size-1)
f.write("\0")
f.close()
else: # fill our file with randomly generaed content
with open(path, 'wb') as f:
# Fill our file with garbage
f.write(urandom(size))
# Update our path
utime(path, time)
if perm is not None:
# Adjust permissions
chmod(path, perm)
# Return True
return True
def touch(self, path, size=None, random=False, perm=None, time=None):
"""Simplify the dynamic creation of files or the updating of their
modified time. If a size is specified, then a file of that size
will be created on the disk. If the file already exists, then the
size= attribute is ignored (for safey reasons).
if random is set to true, then the file created is actually
created using tons of randomly generated content. This is MUCH
slower but nessisary for certain tests.
"""
path = abspath(path)
if not isdir(dirname(path)):
mkdir(dirname(path), 0700)
if not exists(path):
size = strsize_to_bytes(size)
if not random:
f = open(path, "wb")
if isinstance(size, int) and size > 0:
f.seek(size - 1)
f.write("\0")
f.close()
else: # fill our file with randomly generaed content
with open(path, 'wb') as f:
# Fill our file with garbage
f.write(urandom(size))
# Update our path
utime(path, time)
if perm is not None:
# Adjust permissions
chmod(path, perm)
# Return True
return True
def test_saves(self):
"""
Saving allows for a variety of inputs, test that they all
check out okay
"""
# First we create a 1MB file
tmp_file = join(
self.tmp_dir, 'NNTPContent_Test.save', 'testfile.tmp')
# File should not already exist
assert(isfile(tmp_file) is False)
# Create a random file
assert(self.touch(tmp_file, size='5MB', random=True) is True)
# File should exist now
assert(isfile(tmp_file) is True)
# Now we want to load it into a NNTPContent object
content = NNTPContent(filepath=tmp_file, work_dir=self.tmp_dir)
# Test our file exists
assert(len(content) == strsize_to_bytes('5M'))
# By default our load makes it so our file is NOT attached
assert(content.is_attached() is False)
# as we can't make a copy of our current file on top of the old
_filepath = content.path()
assert(content.save(copy=True) is True)
assert(content.path() == _filepath)
assert(content.path() == tmp_file)
# File should still be detached
assert(content.is_attached() is False)
# Filepath shoul still not have changed
# Let's attach it
content.attach()
assert(content.is_attached() is True)
# If we call save() a copy parameter, there should be no change
assert(content.save(copy=True) is True)
# Still no change
assert(content.is_attached() is True)
# Now lets actually copy it to a new location
tmp_file_copy = join(
self.tmp_dir, 'NNTPContent_Test.save', 'testfile.copy.tmp')
# File should not already exist
assert(isfile(tmp_file_copy) is False)
# call save using our copy variable and new filename
assert(content.save(tmp_file_copy, copy=True) is True)
# File should exist now
assert(isfile(tmp_file_copy) is True)
# Old File should still exist too
assert(isfile(tmp_file) is True)
# Path should still be the old path and not the new
assert(content.path() == tmp_file)
# Still no change in attachment
assert(content.is_attached() is True)
# Create a new file now
tmp_file_copy2 = join(
self.tmp_dir, 'NNTPContent_Test.save', 'testfile.copy2.tmp')
assert(isfile(tmp_file_copy2) is False)
# call save with copy set to false; This performs an official
# move (adjusting our internal records.
assert(content.save(tmp_file_copy2, copy=False) is True)
# Old File should no longer exist
assert(isfile(tmp_file) is False)
# Content should be detached
assert(content.is_attached() is False)
# New file should exist
assert(isfile(tmp_file_copy2) is True)
assert(content.path() != _filepath)
assert(content.path() == tmp_file_copy2)
def test_strsize_n_bytes(self):
"""
A formatting tool to make bytes more readable for an end user
"""
# Garbage Entry
assert strsize_to_bytes(None) is None
assert strsize_to_bytes("0J") is None
assert strsize_to_bytes("") is None
assert strsize_to_bytes("totalgarbage") is None
# Allow integers
assert strsize_to_bytes(0) == 0
assert strsize_to_bytes(1024) == 1024
# Good Entries
assert strsize_to_bytes("0B") == 0
assert strsize_to_bytes("0") == 0
assert strsize_to_bytes("10") == 10
assert strsize_to_bytes("1K") == 1024
assert strsize_to_bytes("1M") == 1024 * 1024
assert strsize_to_bytes("1G") == 1024 * 1024 * 1024
assert strsize_to_bytes("1T") == 1024 * 1024 * 1024 * 1024
# Spaces between units and value are fine too
assert strsize_to_bytes(" 0 B ") == 0
assert strsize_to_bytes(" 1 K ") == 1024
assert strsize_to_bytes(" 1 M ") == 1024 * 1024
assert strsize_to_bytes(" 1 G ") == 1024 * 1024 * 1024
assert strsize_to_bytes(" 1 T ") == 1024 * 1024 * 1024 * 1024
# Support Byte character
assert strsize_to_bytes("1KB") == 1024
assert strsize_to_bytes("1MB") == 1024 * 1024
assert strsize_to_bytes("1GB") == 1024 * 1024 * 1024
assert strsize_to_bytes("1TB") == 1024 * 1024 * 1024 * 1024
# Support bit character
assert strsize_to_bytes("1Kb") == 1000
assert strsize_to_bytes("1Mb") == 1000 * 1000
assert strsize_to_bytes("1Gb") == 1000 * 1000 * 1000
assert strsize_to_bytes("1Tb") == 1000 * 1000 * 1000 * 1000
# Garbage Entry
assert bytes_to_strsize(None) is None
assert bytes_to_strsize('') is None
assert bytes_to_strsize('GARBAGE') is None
# Good Entries
assert bytes_to_strsize(0) == "0.00B"
assert bytes_to_strsize(1) == "1.00B"
assert bytes_to_strsize(1024) == "1.00KB"
assert bytes_to_strsize(1024 * 1024) == "1.00MB"
assert bytes_to_strsize(1024 * 1024 * 1024) == "1.00GB"
assert bytes_to_strsize(1024 * 1024 * 1024 * 1024) == "1.00TB"
# Support strings too
assert bytes_to_strsize("0") == "0.00B"
assert bytes_to_strsize("1024") == "1.00KB"
def test_detect_split_size(self):
"""
Test detect_split_size()
"""
cfg_file = join(self.tmp_dir, 'NNTPManager.config.yaml')
server = {
'username': '******',
'password': '******',
'host': 'localhost',
'port': 119,
'secure': 'False',
'compress': 'False',
'priority': '1',
'join_group': 'False',
}
processing = {
# Our test server only supports one connection at this
# time
'threads': 1,
'header_batch_size': 8000,
}
# Create a yaml configuration entry we can test with
# The output is invalid (formatting)
with open(cfg_file, 'w') as fp:
fp.write('%s:\n' % PROCESSING_KEY)
fp.write(' %s' % (' '.join(
['%s: %s\n' % (k, v) for (k, v) in processing.items()])))
fp.write('%s:\n' % SERVER_LIST_KEY)
fp.write(' - %s' % (' '.join(
['%s: %s\n' % (k, v) for (k, v) in server.items()])))
# Settings Object
setting = NNTPSettings(cfg_file=cfg_file)
# Create our NNTP Manager Instance
mgr = NNTPManager(setting)
pf = NNTPPostFactory(connection=mgr)
assert(pf.detect_split_size(None) is False)
assert(pf.detect_split_size('') is False)
assert(pf.detect_split_size('garbage') is False)
# 0-100MB -> 5MB/archive
assert(pf.detect_split_size(0) == strsize_to_bytes('5MB'))
assert(pf.detect_split_size(strsize_to_bytes('5MB') - 1) ==
strsize_to_bytes('5MB'))
# 100MB-1GB -> 15MB/archive
assert(pf.detect_split_size('100M') == strsize_to_bytes('15MB'))
assert(pf.detect_split_size(strsize_to_bytes('1GB') - 1) ==
strsize_to_bytes('15MB'))
# 1GB-5GB -> 50MB/archive
assert(pf.detect_split_size('1G') == strsize_to_bytes('50MB'))
assert(pf.detect_split_size(strsize_to_bytes('5GB') - 1) ==
strsize_to_bytes('50MB'))
# 5GB-15GB -> 100MB/archive
assert(pf.detect_split_size('5G') == strsize_to_bytes('100MB'))
assert(pf.detect_split_size(strsize_to_bytes('15GB') - 1) ==
strsize_to_bytes('100MB'))
# 15GB-25GB -> 200MB/archive
assert(pf.detect_split_size('15G') == strsize_to_bytes('200MB'))
assert(pf.detect_split_size(strsize_to_bytes('25GB') - 1) ==
strsize_to_bytes('200MB'))
# 25GB+ -> 400MB/archive
assert(pf.detect_split_size('25G') == strsize_to_bytes('400MB'))
assert(pf.detect_split_size('50G') == strsize_to_bytes('400MB'))
assert(pf.detect_split_size('100G') == strsize_to_bytes('400MB'))
def test_saves(self):
"""
Saving allows for a variety of inputs, test that they all
check out okay
"""
# First we create a 1MB file
tmp_file = join(self.tmp_dir, 'NNTPContent_Test.save', 'testfile.tmp')
# File should not already exist
assert (isfile(tmp_file) is False)
# Create a random file
assert (self.touch(tmp_file, size='5MB', random=True) is True)
# File should exist now
assert (isfile(tmp_file) is True)
# Now we want to load it into a NNTPContent object
content = NNTPContent(filepath=tmp_file, work_dir=self.tmp_dir)
# Test our file exists
assert (len(content) == strsize_to_bytes('5M'))
# By default our load makes it so our file is NOT attached
assert (content.is_attached() is False)
# as we can't make a copy of our current file on top of the old
_filepath = content.path()
assert (content.save(copy=True) is True)
assert (content.path() == _filepath)
assert (content.path() == tmp_file)
# File should still be detached
assert (content.is_attached() is False)
# Filepath shoul still not have changed
# Let's attach it
content.attach()
assert (content.is_attached() is True)
# If we call save() a copy parameter, there should be no change
assert (content.save(copy=True) is True)
# Still no change
assert (content.is_attached() is True)
# Now lets actually copy it to a new location
tmp_file_copy = join(self.tmp_dir, 'NNTPContent_Test.save',
'testfile.copy.tmp')
# File should not already exist
assert (isfile(tmp_file_copy) is False)
# call save using our copy variable and new filename
assert (content.save(tmp_file_copy, copy=True) is True)
# File should exist now
assert (isfile(tmp_file_copy) is True)
# Old File should still exist too
assert (isfile(tmp_file) is True)
# Path should still be the old path and not the new
assert (content.path() == tmp_file)
# Still no change in attachment
assert (content.is_attached() is True)
# Create a new file now
tmp_file_copy2 = join(self.tmp_dir, 'NNTPContent_Test.save',
'testfile.copy2.tmp')
assert (isfile(tmp_file_copy2) is False)
# call save with copy set to false; This performs an official
# move (adjusting our internal records.
assert (content.save(tmp_file_copy2, copy=False) is True)
# Old File should no longer exist
assert (isfile(tmp_file) is False)
# Content should be detached
assert (content.is_attached() is False)
# New file should exist
assert (isfile(tmp_file_copy2) is True)
assert (content.path() != _filepath)
assert (content.path() == tmp_file_copy2)
# Fedora, CentOS, and RedHat
'/usr/bin/par2',
# Ubuntu/Debian
'/usr/local/bin/par2',
) if exists(c)), None)
# Used to detect the par part #
# - supports .vol07+08.par2 (the data block)
# - supports .par2 (The index file)
PAR_PART_RE = re.compile(
'^.*?\.(vol(?P<index>\d+)\+(?P<count>\d+)\.)?par2?$',
re.IGNORECASE,
)
# Size to default spliting to if not otherwise specified
DEFAULT_BLOCK_SIZE = strsize_to_bytes('300K')
# By default we create recovery records for up to 5% of the total data
DEFAULT_RECOVERY_PERCENT = 5
class ParReturnCode(object):
"""
Converts the return values of the par2 command line to interpretable
response types.
"""
# This is the return type if there simply isn't any damage to the contents
NoParFiles = -1
# This is the return type if there simply isn't any damage to the contents
def test_strsize_n_bytes(self):
"""
A formatting tool to make bytes more readable for an end user
"""
# Garbage Entry
assert strsize_to_bytes(None) is None
assert strsize_to_bytes("0J") is None
assert strsize_to_bytes("") is None
assert strsize_to_bytes("totalgarbage") is None
# Allow integers
assert strsize_to_bytes(0) == 0
assert strsize_to_bytes(1024) == 1024
# Good Entries
assert strsize_to_bytes("0B") == 0
assert strsize_to_bytes("0") == 0
assert strsize_to_bytes("10") == 10
assert strsize_to_bytes("1K") == 1024
assert strsize_to_bytes("1M") == 1024*1024
assert strsize_to_bytes("1G") == 1024*1024*1024
assert strsize_to_bytes("1T") == 1024*1024*1024*1024
# Spaces between units and value are fine too
assert strsize_to_bytes(" 0 B ") == 0
assert strsize_to_bytes(" 1 K ") == 1024
assert strsize_to_bytes(" 1 M ") == 1024*1024
assert strsize_to_bytes(" 1 G ") == 1024*1024*1024
assert strsize_to_bytes(" 1 T ") == 1024*1024*1024*1024
# Support Byte character
assert strsize_to_bytes("1KB") == 1024
assert strsize_to_bytes("1MB") == 1024*1024
assert strsize_to_bytes("1GB") == 1024*1024*1024
assert strsize_to_bytes("1TB") == 1024*1024*1024*1024
# Support bit character
assert strsize_to_bytes("1Kb") == 1000
assert strsize_to_bytes("1Mb") == 1000*1000
assert strsize_to_bytes("1Gb") == 1000*1000*1000
assert strsize_to_bytes("1Tb") == 1000*1000*1000*1000
# Garbage Entry
assert bytes_to_strsize(None) is None
assert bytes_to_strsize('') is None
assert bytes_to_strsize('GARBAGE') is None
# Good Entries
assert bytes_to_strsize(0) == "0.00B"
assert bytes_to_strsize(1) == "1.00B"
assert bytes_to_strsize(1024) == "1.00KB"
assert bytes_to_strsize(1024*1024) == "1.00MB"
assert bytes_to_strsize(1024*1024*1024) == "1.00GB"
assert bytes_to_strsize(1024*1024*1024*1024) == "1.00TB"
# Support strings too
assert bytes_to_strsize("0") == "0.00B"
assert bytes_to_strsize("1024") == "1.00KB"
def test_article_append(self):
"""
Test article append()
Appending effectively takes another's article and appends it's
content to the end of the article doing the appending.
Consider:
- test.rar.000 (ArticleA)
- test.rar.001 (ArticleB)
- test.rar.002 (Articlec)
# The following would assemble the entire article
ArticleA.append(ArticleB)
ArticleA.append(ArticleC)
"""
# Create a temporary file we can use
tmp_file = join(self.tmp_dir, 'NNTPArticle_Test.append', '1MB.rar')
# The file doesn't exist at first
assert(not isfile(tmp_file))
# Create it
assert(self.touch(tmp_file, size='1MB', random=True))
# Now it does
assert(isfile(tmp_file))
# Duplicates groups are are removed automatically
article_a = NNTPArticle(
work_dir=self.tmp_dir,
subject='split-test-a',
poster='<*****@*****.**>',
groups='alt.binaries.l2g',
)
# No size at this point
assert(article_a.size() == 0)
# Add our file to our article
assert(article_a.add(tmp_file) is True)
# We should be equal to the size we created our content with
assert(article_a.size() == strsize_to_bytes('1M'))
# We'll split it in 2
results = article_a.split(strsize_to_bytes('512K'))
# Size doesn't change even if we're split
assert(article_a.size() == strsize_to_bytes('1M'))
# Tests that our results are expected
assert(isinstance(results, sortedset) is True)
assert(len(results) == 2)
# We'll create another article
article_b = NNTPArticle(
subject='split-test-b',
poster='<*****@*****.**>',
groups='alt.binaries.l2g',
work_dir=self.tmp_dir,
)
# Now we'll join the contents using append
assert(article_b.size() == 0)
for article in results:
assert(isinstance(article, NNTPArticle) is True)
assert(article_b.append(article) is True)
assert(article_b.size() == article_a.size())
assert(article_b[0].md5() == article_a[0].md5())
from newsreap.SubProcess import SubProcess
from newsreap.Utils import random_str
from newsreap.Utils import strsize_to_bytes
from newsreap.Utils import pushd
from newsreap.Utils import rm
# Logging
import logging
from newsreap.Logging import NEWSREAP_CODEC
logger = logging.getLogger(NEWSREAP_CODEC)
# 7-Zip Binary Path
DEFAULT_PATH = '/usr/bin/7za'
# Size to default spliting to if not otherwise specified
DEFAULT_SPLIT_SIZE = strsize_to_bytes('25M')
# Used to detect the 7z part #
# - supports .7z, 7za
# - supports .7z0, .7z1, .7z2, etc
# - supports .part00.7z, .part01.7z, etc
# - supports .7z.000, .7z.001, etc
# - supports .7za.000, .7za.001, etc
SEVEN_ZIP_PART_RE = re.compile(
'^.*?\.(part(?P<part>[0-9]+)\.7z|7za?\.?(?P<part0>[0-9]+)?)$',
re.IGNORECASE,
)
# Return Codes
ERROR_CODE_NORMAL = 0
ERROR_CODE_WARNING = 1
def test_detect_split_size(self):
"""
Test detect_split_size()
"""
cfg_file = join(self.tmp_dir, 'NNTPManager.config.yaml')
server = {
'username': '******',
'password': '******',
'host': 'localhost',
'port': 119,
'secure': 'False',
'compress': 'False',
'priority': '1',
'join_group': 'False',
}
processing = {
# Our test server only supports one connection at this
# time
'threads': 1,
'header_batch_size': 8000,
}
# Create a yaml configuration entry we can test with
# The output is invalid (formatting)
with open(cfg_file, 'w') as fp:
fp.write('%s:\n' % PROCESSING_KEY)
fp.write(' %s' % (' '.join(
['%s: %s\n' % (k, v) for (k, v) in processing.items()])))
fp.write('%s:\n' % SERVER_LIST_KEY)
fp.write(
' - %s' %
(' '.join(['%s: %s\n' % (k, v)
for (k, v) in server.items()])))
# Settings Object
setting = NNTPSettings(cfg_file=cfg_file)
# Create our NNTP Manager Instance
mgr = NNTPManager(setting)
pf = NNTPPostFactory(connection=mgr)
assert (pf.detect_split_size(None) is False)
assert (pf.detect_split_size('') is False)
assert (pf.detect_split_size('garbage') is False)
# 0-100MB -> 5MB/archive
assert (pf.detect_split_size(0) == strsize_to_bytes('5MB'))
assert (pf.detect_split_size(strsize_to_bytes('5MB') -
1) == strsize_to_bytes('5MB'))
# 100MB-1GB -> 15MB/archive
assert (pf.detect_split_size('100M') == strsize_to_bytes('15MB'))
assert (pf.detect_split_size(strsize_to_bytes('1GB') -
1) == strsize_to_bytes('15MB'))
# 1GB-5GB -> 50MB/archive
assert (pf.detect_split_size('1G') == strsize_to_bytes('50MB'))
assert (pf.detect_split_size(strsize_to_bytes('5GB') -
1) == strsize_to_bytes('50MB'))
# 5GB-15GB -> 100MB/archive
assert (pf.detect_split_size('5G') == strsize_to_bytes('100MB'))
assert (pf.detect_split_size(strsize_to_bytes('15GB') -
1) == strsize_to_bytes('100MB'))
# 15GB-25GB -> 200MB/archive
assert (pf.detect_split_size('15G') == strsize_to_bytes('200MB'))
assert (pf.detect_split_size(strsize_to_bytes('25GB') -
1) == strsize_to_bytes('200MB'))
# 25GB+ -> 400MB/archive
assert (pf.detect_split_size('25G') == strsize_to_bytes('400MB'))
assert (pf.detect_split_size('50G') == strsize_to_bytes('400MB'))
assert (pf.detect_split_size('100G') == strsize_to_bytes('400MB'))
# Fedora, CentOS, and RedHat
'/usr/bin/rar',
# Ubuntu/Debian
'/usr/local/bin/rar',
) if exists(c)), None)
# UnRar Path
DEFAULT_UNRAR_PATH = next((c for c in (
# Fedora, CentOS, and RedHat
'/usr/bin/unrar',
# Ubuntu/Debian
'/usr/local/bin/unrar',
) if exists(c)), None)
# Size to default spliting to if not otherwise specified
DEFAULT_SPLIT_SIZE = strsize_to_bytes('25M')
# Used to detect the rar part #
# - supports .rar
# - supports .r00, .r01, .r02, etc
# - supports .part00.rar, .part01.rar, etc
RAR_PART_RE = re.compile(
'^.*?\.((part|r)(?P<part>[0-9]+)(\.rar)?|rar)$',
re.IGNORECASE,
)
class CodecRar(CodecFile):
"""
CodecRar is a wrapper to the systems rar binary since there is no
open source code that will otherwise allow us to extract rar