def test_pushd_popd(self):
"""
Just a simple wrapper to makedirs, but tries a few times before
completely aborting.
"""
# Temporary directory to work with
work_dir = join(self.tmp_dir, 'Utils_Test.pushd', 'newdir')
# Ensure it doesn't already exist
assert isdir(work_dir) is False
# Store our current directory
cur_dir = getcwd()
try:
with pushd(work_dir):
# We should throw an exeption here and never make it to the
# assert call below; but just incase ...
assert False
except OSError, e:
# Directory doesn't exist
assert e[0] is errno.ENOENT
assert getcwd() == cur_dir
def test_pushd_popd(self):
"""
Just a simple wrapper to makedirs, but tries a few times before
completely aborting.
"""
# Temporary directory to work with
work_dir = join(self.tmp_dir, 'Utils_Test.pushd', 'newdir')
# Ensure it doesn't already exist
assert isdir(work_dir) is False
# Store our current directory
cur_dir = getcwd()
try:
with pushd(work_dir):
# We should throw an exeption here and never make it to the
# assert call below; but just incase ...
assert False
except OSError, e:
# Directory doesn't exist
assert e[0] is errno.ENOENT
assert getcwd() == cur_dir
def database_status(ctx):
"""
displays details on the current database store
"""
db_path = join(ctx['NNTPSettings'].work_dir, 'cache', 'search')
logger.debug('Scanning %s for databases...' % db_path)
with pushd(db_path, create_if_missing=True):
results = find(
db_path,
suffix_filter=SQLITE_DATABASE_EXTENSION,
fsinfo=True,
max_depth=1,
)
# Use our Database first if it exists
session = ctx['NNTPSettings'].session()
if not session:
logger.error('Could not acquire a database connection.')
exit(1)
# PEP8 E712 does not allow us to make a comparison to a boolean value
# using the == instead of the keyword 'in'. However SQLAlchemy
# requires us to do just because that's how the amazing tool works.
# so to get around the pep8 error, we'll just define a variable equal
# to True and then we can compare to it
pep8_e712 = True
try:
# Get a list of watched groups
groups = dict(
session.query(Group.name,
Group.id).filter(Group.watch == pep8_e712).all())
except OperationalError:
# Get a list of watched groups
logger.warning('The database does not appear to be initialized.')
logger.info('Try running: "nr db init" first.')
exit(0)
if not len(results):
logger.info('There are no groups configured to be watched.')
exit(0)
for _, meta in results.iteritems():
# Open up the database
flags = ''
if meta['filename'] in groups:
flags += 'W'
print('%-65s %-10s %s' % (
meta['filename'],
bytes_to_strsize(meta['size']),
flags,
))
def database_status(ctx):
"""
displays details on the current database store
"""
db_path = join(ctx['NNTPSettings'].work_dir, 'cache', 'search')
logger.debug('Scanning %s for databases...' % db_path)
with pushd(db_path, create_if_missing=True):
results = find(
db_path,
suffix_filter=SQLITE_DATABASE_EXTENSION,
fsinfo=True,
max_depth=1,
)
# Use our Database first if it exists
session = ctx['NNTPSettings'].session()
if not session:
logger.error('Could not acquire a database connection.')
exit(1)
# PEP8 E712 does not allow us to make a comparison to a boolean value
# using the == instead of the keyword 'in'. However SQLAlchemy
# requires us to do just because that's how the amazing tool works.
# so to get around the pep8 error, we'll just define a variable equal
# to True and then we can compare to it
pep8_e712 = True
try:
# Get a list of watched groups
groups = dict(session.query(Group.name, Group.id)
.filter(Group.watch == pep8_e712).all())
except OperationalError:
# Get a list of watched groups
logger.warning('The database does not appear to be initialized.')
logger.info('Try running: "nr db init" first.')
exit(0)
if not len(results):
logger.info('There are no groups configured to be watched.')
exit(0)
for _, meta in results.iteritems():
# Open up the database
flags = ''
if meta['filename'] in groups:
flags += 'W'
print('%-65s %-10s %s' % (
meta['filename'],
bytes_to_strsize(meta['size']),
flags,
))
def database_reset(ctx):
"""
Reset's the database based on the current configuration
"""
logger.info('Resetting database ...')
ctx['NNTPSettings'].open(reset=True)
__db_prep(ctx)
db_path = join(ctx['NNTPSettings'].base_dir, 'cache', 'search')
logger.debug('Scanning %s for databases...' % db_path)
with pushd(db_path, create_if_missing=True):
for entry in listdir(db_path):
db_file = join(db_path, entry)
if not isfile(db_file):
continue
try:
unlink(db_file)
logger.info('Removed %s ...' % entry)
except:
logger.warning('Failed to remove %s ...' % entry)
def database_reset(ctx):
"""
Reset's the database based on the current configuration
"""
logger.info('Resetting database ...')
ctx['NNTPSettings'].open(reset=True)
__db_prep(ctx)
db_path = join(ctx['NNTPSettings'].base_dir, 'cache', 'search')
logger.debug('Scanning %s for databases...' % db_path)
with pushd(db_path, create_if_missing=True):
for entry in listdir(db_path):
db_file = join(db_path, entry)
if not isfile(db_file):
continue
try:
unlink(db_file)
logger.info('Removed %s ...' % entry)
except:
logger.warning('Failed to remove %s ...' % entry)
class Utils_Test(TestBase):
"""
Testing the Utils class
"""
def test_strsize_n_bytes(self):
"""
A formatting tool to make bytes more readable for an end user
"""
# Garbage Entry
assert strsize_to_bytes(None) is None
assert strsize_to_bytes("0J") is None
assert strsize_to_bytes("") is None
assert strsize_to_bytes("totalgarbage") is None
# Allow integers
assert strsize_to_bytes(0) == 0
assert strsize_to_bytes(1024) == 1024
# Good Entries
assert strsize_to_bytes("0B") == 0
assert strsize_to_bytes("0") == 0
assert strsize_to_bytes("10") == 10
assert strsize_to_bytes("1K") == 1024
assert strsize_to_bytes("1M") == 1024 * 1024
assert strsize_to_bytes("1G") == 1024 * 1024 * 1024
assert strsize_to_bytes("1T") == 1024 * 1024 * 1024 * 1024
# Spaces between units and value are fine too
assert strsize_to_bytes(" 0 B ") == 0
assert strsize_to_bytes(" 1 K ") == 1024
assert strsize_to_bytes(" 1 M ") == 1024 * 1024
assert strsize_to_bytes(" 1 G ") == 1024 * 1024 * 1024
assert strsize_to_bytes(" 1 T ") == 1024 * 1024 * 1024 * 1024
# Support Byte character
assert strsize_to_bytes("1KB") == 1024
assert strsize_to_bytes("1MB") == 1024 * 1024
assert strsize_to_bytes("1GB") == 1024 * 1024 * 1024
assert strsize_to_bytes("1TB") == 1024 * 1024 * 1024 * 1024
# Support bit character
assert strsize_to_bytes("1Kb") == 1000
assert strsize_to_bytes("1Mb") == 1000 * 1000
assert strsize_to_bytes("1Gb") == 1000 * 1000 * 1000
assert strsize_to_bytes("1Tb") == 1000 * 1000 * 1000 * 1000
# Garbage Entry
assert bytes_to_strsize(None) is None
assert bytes_to_strsize('') is None
assert bytes_to_strsize('GARBAGE') is None
# Good Entries
assert bytes_to_strsize(0) == "0.00B"
assert bytes_to_strsize(1) == "1.00B"
assert bytes_to_strsize(1024) == "1.00KB"
assert bytes_to_strsize(1024 * 1024) == "1.00MB"
assert bytes_to_strsize(1024 * 1024 * 1024) == "1.00GB"
assert bytes_to_strsize(1024 * 1024 * 1024 * 1024) == "1.00TB"
# Support strings too
assert bytes_to_strsize("0") == "0.00B"
assert bytes_to_strsize("1024") == "1.00KB"
def test_stat(self):
"""
Stat makes it easier to disect the file extension, filesystem info
and mime information.
"""
general_keys = ('extension', 'basename', 'filename', 'dirname')
filesys_keys = ('created', 'modified', 'accessed', 'size')
mime_keys = ('mime', )
# Test a file that doesn't exist
tmp_file = join(self.tmp_dir, 'Utils_Test.stat', 'missing_file')
stats = stat(tmp_file)
assert stats is None
stats = stat(tmp_file, fsinfo=False)
assert stats is None
stats = stat(tmp_file, fsinfo=False, mime=False)
assert stats is None
# Create Temporary file 1MB in size
tmp_file = join(self.tmp_dir, 'Utils_Test.stat', '1MB.rar')
assert self.touch(tmp_file, size='1MB')
stats = stat(tmp_file)
# This check basically makes sure all of the expected keys
# are in place and that there aren't more or less
k_iter = chain(mime_keys, filesys_keys, general_keys)
k_len = len(mime_keys) + len(filesys_keys) + len(general_keys)
assert isinstance(stats, dict) is True
assert len([k for k in k_iter if k not in stats.keys()]) == 0
assert k_len == len(stats)
# Filesize should actually match what we set it as
assert bytes_to_strsize(stats['size']) == "1.00MB"
# different OS's and variations of python can yield different
# results. We're trying to just make sure that we find the
# rar keyword in the mime type
assert (stats['mime'] == 'application/x-rar-compressed')
# Create Temporary file 1MB in size
tmp_file = join(self.tmp_dir, 'Utils_Test.stat', '2MB.zip')
assert self.touch(tmp_file, size='2MB')
stats = stat(tmp_file)
# This check basically makes sure all of the expected keys
# are in place and that there aren't more or less
k_iter = chain(mime_keys, filesys_keys, general_keys)
k_len = len(mime_keys) + len(filesys_keys) + len(general_keys)
assert isinstance(stats, dict) is True
assert len([k for k in k_iter if k not in stats.keys()]) == 0
assert k_len == len(stats)
# Filesize should actually match what we set it as
assert bytes_to_strsize(stats['size']) == "2.00MB"
assert re.search(
'application/.*zip.*',
stats['mime'],
re.IGNORECASE,
) is not None
# Test different variations
stats = stat(tmp_file, mime=False)
# This check basically makes sure all of the expected keys
# are in place and that there aren't more or less
k_iter = chain(filesys_keys, general_keys)
k_len = len(filesys_keys) + len(general_keys)
assert isinstance(stats, dict) is True
assert len([k for k in k_iter if k not in stats.keys()]) == 0
assert k_len == len(stats)
# Test different variations
stats = stat(tmp_file, fsinfo=False, mime=True)
# This check basically makes sure all of the expected keys
# are in place and that there aren't more or less
k_iter = chain(mime_keys, general_keys)
k_len = len(mime_keys) + len(general_keys)
assert isinstance(stats, dict) is True
assert len([k for k in k_iter if k not in stats.keys()]) == 0
assert k_len == len(stats)
# Test different variations
stats = stat(tmp_file, fsinfo=False, mime=False)
# This check basically makes sure all of the expected keys
# are in place and that there aren't more or less
k_iter = chain(general_keys)
k_len = len(general_keys)
assert isinstance(stats, dict) is True
assert len([k for k in k_iter if k not in stats.keys()]) == 0
assert k_len == len(stats)
def test_mkdir(self):
"""
Just a simple wrapper to makedirs, but tries a few times before
completely aborting.
"""
work_dir = join(self.tmp_dir, 'Utils_Test.mkdir', 'dirA')
# The directory should not exist
assert isdir(work_dir) is False
# mkdir() should be successful
assert mkdir(work_dir) is True
# The directory should exist now
assert isdir(work_dir) is True
# mkdir() gracefully handles 2 calls to the same location
assert mkdir(work_dir) is True
# Create Temporary file 1KB in size
tmp_file = join(self.tmp_dir, 'Utils_Test.mkdir', '2KB')
assert self.touch(tmp_file, size='2KB')
# Now the isdir() will still return False because there is a file
# there now, not a directory
assert isdir(tmp_file) is False
# mkdir() will fail to create a directory in place of file
assert mkdir(tmp_file) is False
# And reference a new directory (not created yet) within
new_work_dir = join(work_dir, 'subdir')
# Confirm our directory doesn't exist
assert isdir(new_work_dir) is False
# Now we'll protect our original directory
chmod(work_dir, 0000)
# mkdir() will fail because of permissions, but incase it doesn't work
# as planned, just store the result in a variable. We'll flip our
# permission back first
result = mkdir(new_work_dir)
# reset the permission
chmod(work_dir, 0700)
# Our result should yield a failed result
assert result is False
# Confirm that the directory was never created:
assert isdir(new_work_dir) is False
def test_pushd_popd(self):
"""
Just a simple wrapper to makedirs, but tries a few times before
completely aborting.
"""
# Temporary directory to work with
work_dir = join(self.tmp_dir, 'Utils_Test.pushd', 'newdir')
# Ensure it doesn't already exist
assert isdir(work_dir) is False
# Store our current directory
cur_dir = getcwd()
try:
with pushd(work_dir):
# We should throw an exeption here and never make it to the
# assert call below; but just incase ...
assert False
except OSError, e:
# Directory doesn't exist
assert e[0] is errno.ENOENT
assert getcwd() == cur_dir
# Now we'll make the directory
with pushd(work_dir, create_if_missing=True):
# We're in a new directory
assert getcwd() == work_dir
# We're back to where we were
assert getcwd() == cur_dir
try:
with pushd(work_dir, create_if_missing=True):
# We're in a new directory
assert getcwd() == work_dir
# Throw an exception
raise Exception
except Exception:
# We're back to where we were
assert getcwd() == cur_dir
def decode(self, content=None, name=None, password=None, *args, **kwargs):
"""
content must be pointing to a directory containing rar files that can
be easily sorted on. Alternatively, path can be of type NNTPContent()
or a set/list of.
If no password is specified, then the password configuration loaded
into the class is used instead.
An NNTPBinaryContent() object containing the contents of the package
within a sortedset() object. All decoded() functions have to return
a resultset() to be consistent with one another.
"""
if content is not None:
self.add(content)
# Some simple error checking to save from doing to much here
if len(self) == 0:
return None
if not self.can_exe(self._unrar):
return None
if not password:
password = self.password
# Initialize our command
execute = [
# Our Executable RAR Application
self._unrar,
# Use Add Flag
'x',
# Assume Yes
'-y',
]
# Password Protection
if password is not None:
execute.append('-p%s' % password)
else:
# Do not prompt for password
execute.append('-p-')
if self.keep_broken:
# Keep Broken Flag
execute.append('-kb')
if self.overwrite:
# Overwrite files
execute.append('-o+')
else:
# Don't overwrite files
execute.append('-o-')
if self.freshen:
# Freshen files
execute.append('-f')
# Stop Switch Parsing
execute.append('--')
if not name:
name = self.name
if not name:
name = random_str()
for _path in self:
# Temporary Path
tmp_path, _ = self.mkstemp(content=name)
with pushd(tmp_path):
# Create our SubProcess Instance
sp = SubProcess(list(execute) + [_path])
# Start our execution now
sp.start()
found_set = None
while not sp.is_complete(timeout=1.5):
found_set = self.watch_dir(
tmp_path,
ignore=found_set,
)
# Handle remaining content
found_set = self.watch_dir(
tmp_path,
ignore=found_set,
seconds=-1,
)
# Let the caller know our status
if not sp.successful():
# Cleanup Temporary Path
rm(tmp_path)
return None
if not len(found_set):
logger.warning(
'RAR archive (%s) contained no content.' %
basename(_path),
)
# Clean our are list of objects to archive
self.clear()
# Return path containing unrar'ed content
results = NNTPBinaryContent(tmp_path)
# We intentionally attach it's content
results.attach()
# Create a sortedset to return
_resultset = sortedset(key=lambda x: x.key())
_resultset.add(results)
# Return our content
return _resultset
def decode(self, content=None, name=None, password=None, *args, **kwargs):
"""
content must be pointing to a directory containing 7-Zip files that can
be easily sorted on. Alternatively, path can be of type NNTPContent()
or a set/list of.
If no password is specified, then the password configuration loaded
into the class is used instead.
An NNTPBinaryContent() object containing the contents of the package
within a sortedset() object. All decoded() functions have to return
a resultset() to be consistent with one another.
"""
if content is not None:
self.add(content)
# Some simple error checking to save from doing to much here
if len(self) == 0:
return None
if not self.can_exe(self._bin):
return None
if not password:
password = self.password
# Initialize our command
execute = [
# Our Executable 7-Zip Application
self._bin,
# Use Add Flag
'x',
# Assume Yes
'-y',
]
# Password Protection
if password is not None:
execute.append('-p%s' % password)
else:
# Do not prompt for password
execute.append('-p-')
if self.overwrite:
# Overwrite files
execute.append('-aoa')
else:
# Don't overwrite files
execute.append('-aos')
# Stop Switch Parsing
execute.append('--')
if not name:
name = self.name
if not name:
name = random_str()
for _path in self:
# Temporary Path
tmp_path, _ = self.mkstemp(content=name)
with pushd(tmp_path):
# Create our SubProcess Instance
sp = SubProcess(list(execute) + [_path])
# Start our execution now
sp.start()
found_set = None
while not sp.is_complete(timeout=1.5):
found_set = self.watch_dir(
tmp_path,
ignore=found_set,
)
# Handle remaining content
found_set = self.watch_dir(
tmp_path,
ignore=found_set,
seconds=-1,
)
# Let the caller know our status
if not sp.successful():
# Cleanup Temporary Path
rm(tmp_path)
return None
if not len(found_set):
logger.warning(
'7Z archive (%s) contained no content.' %
basename(_path), )
# Clean our are list of objects to archive
self.clear()
# Return path containing unrar'ed content
results = NNTPBinaryContent(tmp_path)
# We intentionally attach it's content
results.attach()
# Create a sortedset to return
_resultset = sortedset(key=lambda x: x.key())
_resultset.add(results)
# Return our content
return _resultset
def test(self, content=None):
"""
content must be pointing to a directory containing par files that can
be easily sorted on. Alternatively, path can be of type NNTPContent()
or a set/list of.
This function just tests an archive to see if it can be properly
prepared (it is effectively a wrapper to verify)
If anything but True is returned then there was a problem verifying
the results and a code identified in ParReturnCode() is returned
instead.
"""
if content is not None:
paths = self.get_paths(content)
elif len(self.archive):
# Get first item in archive
paths = iter(self.archive)
else:
raise AttributeError("CodecPar: No par file detected.")
if not self.can_exe(self._par):
return None
# filter our results by indexes
indexes = self.__filter_pars(paths, indexes=True, volumes=False)
if not len(indexes):
logger.warning('Archive contained no PAR files.')
return ParReturnCode.NoParFiles
# Initialize our command
execute = [
# Our Executable PAR Application
self._par,
# Use Test Flag
'verify',
]
if self.cpu_cores is not None and self.cpu_cores > 1:
# to checksum concurrently - uses multiple threads
execute.append('-t+')
# Stop Switch Parsing
execute.append('--')
for _path in indexes:
# Get the directory the par file resides in
par_path = dirname(_path)
with pushd(par_path):
# Create our SubProcess Instance
sp = SubProcess(list(execute) + [basename(_path)])
# Start our execution now
sp.start()
sp.join()
# Let the caller know our status
if sp.response_code() is not ParReturnCode.NoRepairRequired:
return sp.response_code()
return True
def decode(self, content=None, *args, **kwargs):
"""
content must be pointing to a directory containing par files that can
be easily retrieved. Alternatively, path can be of type NNTPContent()
or a set/list of.
An sortedset of NNTPBinaryContent() objects are returned containing
any new content that was generated as a result of the par2 call
If an error occurs then None is returned.
"""
if content is not None:
self.add(content)
# Some simple error checking to save from doing to much here
if len(self) == 0:
return None
if not self.can_exe(self._par):
return None
# filter our results by indexes
indexes = self.__filter_pars(self.archive, indexes=True, volumes=False)
# Initialize our command
execute = [
# Our Executable PAR Application
self._par,
# Use Repair
'repair',
]
if self.cpu_cores is not None and self.cpu_cores > 1:
# to repair concurrently - uses multiple threads
execute.append('-t+')
# Stop Switch Parsing
execute.append('--')
results = sortedset(key=lambda x: x.key())
for _path in indexes:
# Get the directory the par file resides in
par_path = dirname(_path)
with pushd(par_path):
# create a before snapshot
before_snapshot = self.watch_dir(
par_path,
seconds=-1,
)
# Create our SubProcess Instance
sp = SubProcess(list(execute) + [basename(_path)])
# Start our execution now
sp.start()
# Track files after
after_snapshot = sortedset()
while not sp.is_complete(timeout=1.5):
after_snapshot = self.watch_dir(
par_path,
ignore=after_snapshot,
)
# Handle remaining content
after_snapshot = self.watch_dir(
par_path,
ignore=after_snapshot,
seconds=-1,
)
# Add any new files detected to our result set otherwise we
# just return an empty set
total_parts = after_snapshot - before_snapshot
for no, path in enumerate(total_parts):
content = NNTPBinaryContent(
path,
part=no+1,
total_parts=len(total_parts),
)
# Loaded data is by default detached; we want to attach it
content.attach()
# Add our attached content to our results
results.add(content)
# Let the caller know our status
if not sp.successful():
return None
# Clean our are list of objects to archive
self.clear()
return results
def encode(self, content=None, *args, **kwargs):
"""
Takes a specified path (and or file) and creates par2 files based on
it. If this function is successful, it returns a set of
NNTPBinaryContent() objects identifying the PAR2 files generated
based on the passed in content.
The function returns None if it fails in any way.
"""
if content is not None:
self.add(content)
# Some simple error checking to save from doing to much here
if len(self) == 0:
return None
if not self.can_exe(self._par):
return None
for target in self.archive:
# Base entry on first file in the list
name = basename(target)
target_dir = dirname(target)
# tmp_path, tmp_file = self.mkstemp(content=name, suffix='.par2')
# Initialize our command
execute = [
# Our Executable PAR Application
self._par,
# Use Create Flag
'create',
]
# Handle PAR Block Size
if self.block_size:
execute.append('-s%s' % self.block_size)
if self.recovery_percent:
execute.append('-r%d' % self.recovery_percent)
if self.cpu_cores is not None and self.cpu_cores > 1:
# to repair concurrently - uses multiple threads
execute.append('-t+')
# Stop Switch Parsing
execute.append('--')
# Now add our target (we can only do one at a time which i why we
# loop) and run our setups
execute.append(target)
found_set = sortedset()
with pushd(target_dir):
# Create our SubProcess Instance
sp = SubProcess(execute)
# Start our execution now
sp.start()
while not sp.is_complete(timeout=1.5):
found_set = self.watch_dir(
target_dir,
prefix=name,
regex=PAR_PART_RE,
ignore=found_set,
)
# Handle remaining content
found_set = self.watch_dir(
target_dir,
prefix=name,
regex=PAR_PART_RE,
ignore=found_set,
seconds=-1,
)
# Let the caller know our status
if not sp.successful():
# We're done; we failed
return None
if not len(found_set):
# We're done; we failed
return None
# Create a resultset
results = sortedset(key=lambda x: x.key())
part = 0
# iterate through our found_set and create NNTPBinaryContent()
# objects from them.
for path in found_set:
# Iterate over our found files and determine their part
# information
part += 1
content = NNTPBinaryContent(
path,
part=part,
total_parts=len(found_set),
)
# Loaded data is by default detached; we want to attach it
content.attach()
# Add our attached content to our results
results.add(content)
# Clean our are list of objects to archive
self.clear()
# Return our
return results