def __repr__(self):
"""
Return a printable version of the file being read
"""
if self.part is not None:
return '<NNTPBinaryContent sort=%d filename="%s" part=%d/%d len=%s />' % (
self.sort_no,
self.filename,
self.part,
self.total_parts,
bytes_to_strsize(len(self)),
)
else:
return '<NNTPBinaryContent sort=%d filename="%s" len=%s />' % (
self.sort_no,
self.filename,
bytes_to_strsize(len(self)),
)
def __repr__(self):
"""
Return a printable version of the file being read
"""
if self.part is not None:
return '<NNTPBinaryContent sort=%d filename="%s" part=%d/%d len=%s />' % (
self.sort_no,
self.filename,
self.part,
self.total_parts,
bytes_to_strsize(len(self)),
)
else:
return '<NNTPBinaryContent sort=%d filename="%s" len=%s />' % (
self.sort_no,
self.filename,
bytes_to_strsize(len(self)),
)
def database_status(ctx):
"""
displays details on the current database store
"""
db_path = join(ctx['NNTPSettings'].work_dir, 'cache', 'search')
logger.debug('Scanning %s for databases...' % db_path)
with pushd(db_path, create_if_missing=True):
results = find(
db_path,
suffix_filter=SQLITE_DATABASE_EXTENSION,
fsinfo=True,
max_depth=1,
)
# Use our Database first if it exists
session = ctx['NNTPSettings'].session()
if not session:
logger.error('Could not acquire a database connection.')
exit(1)
# PEP8 E712 does not allow us to make a comparison to a boolean value
# using the == instead of the keyword 'in'. However SQLAlchemy
# requires us to do just because that's how the amazing tool works.
# so to get around the pep8 error, we'll just define a variable equal
# to True and then we can compare to it
pep8_e712 = True
try:
# Get a list of watched groups
groups = dict(
session.query(Group.name,
Group.id).filter(Group.watch == pep8_e712).all())
except OperationalError:
# Get a list of watched groups
logger.warning('The database does not appear to be initialized.')
logger.info('Try running: "nr db init" first.')
exit(0)
if not len(results):
logger.info('There are no groups configured to be watched.')
exit(0)
for _, meta in results.iteritems():
# Open up the database
flags = ''
if meta['filename'] in groups:
flags += 'W'
print('%-65s %-10s %s' % (
meta['filename'],
bytes_to_strsize(meta['size']),
flags,
))
def database_status(ctx):
"""
displays details on the current database store
"""
db_path = join(ctx['NNTPSettings'].work_dir, 'cache', 'search')
logger.debug('Scanning %s for databases...' % db_path)
with pushd(db_path, create_if_missing=True):
results = find(
db_path,
suffix_filter=SQLITE_DATABASE_EXTENSION,
fsinfo=True,
max_depth=1,
)
# Use our Database first if it exists
session = ctx['NNTPSettings'].session()
if not session:
logger.error('Could not acquire a database connection.')
exit(1)
# PEP8 E712 does not allow us to make a comparison to a boolean value
# using the == instead of the keyword 'in'. However SQLAlchemy
# requires us to do just because that's how the amazing tool works.
# so to get around the pep8 error, we'll just define a variable equal
# to True and then we can compare to it
pep8_e712 = True
try:
# Get a list of watched groups
groups = dict(session.query(Group.name, Group.id)
.filter(Group.watch == pep8_e712).all())
except OperationalError:
# Get a list of watched groups
logger.warning('The database does not appear to be initialized.')
logger.info('Try running: "nr db init" first.')
exit(0)
if not len(results):
logger.info('There are no groups configured to be watched.')
exit(0)
for _, meta in results.iteritems():
# Open up the database
flags = ''
if meta['filename'] in groups:
flags += 'W'
print('%-65s %-10s %s' % (
meta['filename'],
bytes_to_strsize(meta['size']),
flags,
))
def watch_dir(self,
path,
regex=None,
prefix=None,
suffix=None,
ignore=None,
case_sensitive=True,
seconds=15):
"""Monitors a directory for files that have been added/changed
path: is the path to monitor
ignore: is a sortedset of files already parsed
seconds: is how long it takes a file to go untouched for before
we presume it has been completely written to disk.
"""
if ignore is None:
ignore = sortedset()
findings = find(
path,
fsinfo=True,
regex_filter=regex,
prefix_filter=prefix,
suffix_filter=suffix,
case_sensitive=case_sensitive,
)
findings = [
(p, f['size'], f['created'], f['modified'])
for p, f in findings.items()
if (f['modified'] - f['created']).total_seconds() >= seconds
and f['basename'] not in ignore
]
# Sort list by created date
findings.sort(key=lambda x: x[3])
for f in findings:
logger.info('Created %s (size=%s)' % (
f,
bytes_to_strsize(f[1]),
))
# Add to our filter list
ignore.add(f[0])
# Return our ignore list (which is acutally also a found list)
return ignore
def watch_dir(self, path, regex=None, prefix=None, suffix=None,
ignore=None, case_sensitive=True, seconds=15):
"""Monitors a directory for files that have been added/changed
path: is the path to monitor
ignore: is a sortedset of files already parsed
seconds: is how long it takes a file to go untouched for before
we presume it has been completely written to disk.
"""
if ignore is None:
ignore = sortedset()
findings = find(
path, fsinfo=True,
regex_filter=regex,
prefix_filter=prefix,
suffix_filter=suffix,
case_sensitive=case_sensitive,
)
findings = [
(p, f['size'], f['created'], f['modified'])
for p, f in findings.items()
if (f['modified'] - f['created']).total_seconds() >= seconds
and f['basename'] not in ignore
]
# Sort list by created date
findings.sort(key=lambda x: x[3])
for f in findings:
logger.info('Created %s (size=%s)' % (
f, bytes_to_strsize(f[1]),
))
# Add to our filter list
ignore.add(f[0])
# Return our ignore list (which is acutally also a found list)
return ignore
def test_strsize_n_bytes(self):
"""
A formatting tool to make bytes more readable for an end user
"""
# Garbage Entry
assert strsize_to_bytes(None) is None
assert strsize_to_bytes("0J") is None
assert strsize_to_bytes("") is None
assert strsize_to_bytes("totalgarbage") is None
# Allow integers
assert strsize_to_bytes(0) == 0
assert strsize_to_bytes(1024) == 1024
# Good Entries
assert strsize_to_bytes("0B") == 0
assert strsize_to_bytes("0") == 0
assert strsize_to_bytes("10") == 10
assert strsize_to_bytes("1K") == 1024
assert strsize_to_bytes("1M") == 1024 * 1024
assert strsize_to_bytes("1G") == 1024 * 1024 * 1024
assert strsize_to_bytes("1T") == 1024 * 1024 * 1024 * 1024
# Spaces between units and value are fine too
assert strsize_to_bytes(" 0 B ") == 0
assert strsize_to_bytes(" 1 K ") == 1024
assert strsize_to_bytes(" 1 M ") == 1024 * 1024
assert strsize_to_bytes(" 1 G ") == 1024 * 1024 * 1024
assert strsize_to_bytes(" 1 T ") == 1024 * 1024 * 1024 * 1024
# Support Byte character
assert strsize_to_bytes("1KB") == 1024
assert strsize_to_bytes("1MB") == 1024 * 1024
assert strsize_to_bytes("1GB") == 1024 * 1024 * 1024
assert strsize_to_bytes("1TB") == 1024 * 1024 * 1024 * 1024
# Support bit character
assert strsize_to_bytes("1Kb") == 1000
assert strsize_to_bytes("1Mb") == 1000 * 1000
assert strsize_to_bytes("1Gb") == 1000 * 1000 * 1000
assert strsize_to_bytes("1Tb") == 1000 * 1000 * 1000 * 1000
# Garbage Entry
assert bytes_to_strsize(None) is None
assert bytes_to_strsize('') is None
assert bytes_to_strsize('GARBAGE') is None
# Good Entries
assert bytes_to_strsize(0) == "0.00B"
assert bytes_to_strsize(1) == "1.00B"
assert bytes_to_strsize(1024) == "1.00KB"
assert bytes_to_strsize(1024 * 1024) == "1.00MB"
assert bytes_to_strsize(1024 * 1024 * 1024) == "1.00GB"
assert bytes_to_strsize(1024 * 1024 * 1024 * 1024) == "1.00TB"
# Support strings too
assert bytes_to_strsize("0") == "0.00B"
assert bytes_to_strsize("1024") == "1.00KB"
def test_stat(self):
"""
Stat makes it easier to disect the file extension, filesystem info
and mime information.
"""
general_keys = ('extension', 'basename', 'filename', 'dirname')
filesys_keys = ('created', 'modified', 'accessed', 'size')
mime_keys = ('mime', )
# Test a file that doesn't exist
tmp_file = join(self.tmp_dir, 'Utils_Test.stat', 'missing_file')
stats = stat(tmp_file)
assert stats is None
stats = stat(tmp_file, fsinfo=False)
assert stats is None
stats = stat(tmp_file, fsinfo=False, mime=False)
assert stats is None
# Create Temporary file 1MB in size
tmp_file = join(self.tmp_dir, 'Utils_Test.stat', '1MB.rar')
assert self.touch(tmp_file, size='1MB')
stats = stat(tmp_file)
# This check basically makes sure all of the expected keys
# are in place and that there aren't more or less
k_iter = chain(mime_keys, filesys_keys, general_keys)
k_len = len(mime_keys) + len(filesys_keys) + len(general_keys)
assert isinstance(stats, dict) is True
assert len([k for k in k_iter if k not in stats.keys()]) == 0
assert k_len == len(stats)
# Filesize should actually match what we set it as
assert bytes_to_strsize(stats['size']) == "1.00MB"
# different OS's and variations of python can yield different
# results. We're trying to just make sure that we find the
# rar keyword in the mime type
assert (stats['mime'] == 'application/x-rar-compressed')
# Create Temporary file 1MB in size
tmp_file = join(self.tmp_dir, 'Utils_Test.stat', '2MB.zip')
assert self.touch(tmp_file, size='2MB')
stats = stat(tmp_file)
# This check basically makes sure all of the expected keys
# are in place and that there aren't more or less
k_iter = chain(mime_keys, filesys_keys, general_keys)
k_len = len(mime_keys) + len(filesys_keys) + len(general_keys)
assert isinstance(stats, dict) is True
assert len([k for k in k_iter if k not in stats.keys()]) == 0
assert k_len == len(stats)
# Filesize should actually match what we set it as
assert bytes_to_strsize(stats['size']) == "2.00MB"
assert re.search(
'application/.*zip.*',
stats['mime'],
re.IGNORECASE,
) is not None
# Test different variations
stats = stat(tmp_file, mime=False)
# This check basically makes sure all of the expected keys
# are in place and that there aren't more or less
k_iter = chain(filesys_keys, general_keys)
k_len = len(filesys_keys) + len(general_keys)
assert isinstance(stats, dict) is True
assert len([k for k in k_iter if k not in stats.keys()]) == 0
assert k_len == len(stats)
# Test different variations
stats = stat(tmp_file, fsinfo=False, mime=True)
# This check basically makes sure all of the expected keys
# are in place and that there aren't more or less
k_iter = chain(mime_keys, general_keys)
k_len = len(mime_keys) + len(general_keys)
assert isinstance(stats, dict) is True
assert len([k for k in k_iter if k not in stats.keys()]) == 0
assert k_len == len(stats)
# Test different variations
stats = stat(tmp_file, fsinfo=False, mime=False)
# This check basically makes sure all of the expected keys
# are in place and that there aren't more or less
k_iter = chain(general_keys)
k_len = len(general_keys)
assert isinstance(stats, dict) is True
assert len([k for k in k_iter if k not in stats.keys()]) == 0
assert k_len == len(stats)
def test_copy01(self):
"""
Test our copy function
The copy function allows us to duplicate an existing NNTPContent
object without obstructing the original. Copied content is
always attached; so if the object falls out of scope; so does
the file.
"""
my_dir = join(self.tmp_dir, 'NNTPContent', 'test_copy')
assert (isdir(my_dir) is False)
assert (mkdir(my_dir) is True)
assert (isdir(my_dir) is True)
# Now create our NNTPContent object witin our directory
obj = NNTPContent(
filepath='myfile',
work_dir=my_dir,
)
# Content is attached by default
assert (obj.is_attached() is True)
obj.detach()
assert (obj.is_attached() is False)
new_dir = join(my_dir, 'copy')
assert (isdir(new_dir) is False)
# Create a copy of our object
obj_copy = obj.copy()
# Successfully loaded files are never attached reguardless
# of the original copy
assert (obj_copy.is_attached() is True)
# Reattach the original so it dies when this test is over
obj.attach()
assert (obj.is_attached() is True)
# Create a copy of our copy
obj_copy2 = obj_copy.copy()
assert (obj_copy2.is_attached() is True)
assert (isfile(obj_copy2.path()))
_path = obj_copy2.path()
del obj_copy2
assert (isfile(_path) is False)
assert (isfile(obj_copy.path()) is True)
_path = obj_copy.path()
del obj_copy
assert (isfile(_path) is False)
assert (isfile(obj.path()) is True)
_path = obj.path()
del obj
assert (isfile(_path) is False)
# now lets do a few more tests but with actual files this time
tmp_file = join(my_dir, '2MB.zip')
assert (self.touch(tmp_file, size='2MB', random=True) is True)
# Now create our NNTPContent object witin our directory
obj = NNTPContent(
filepath=tmp_file,
work_dir=my_dir,
)
assert (isfile(obj.path()) is True)
obj_copy = obj.copy()
assert (isfile(obj_copy.path()) is True)
stats = stat(obj_copy.path())
assert (bytes_to_strsize(stats['size']) == "2.00MB")
# Compare that our content is the same
assert (compare(obj_copy.path(), obj.path()) is True)
# note that the filenames are NOT the same so we are dealing with 2
# distinct copies here
assert (isfile(obj_copy.path()) is True)
assert (isfile(obj.path()) is True)
assert (obj.path() != obj_copy.path())
def test_copy01(self):
"""
Test our copy function
The copy function allows us to duplicate an existing NNTPContent
object without obstructing the original. Copied content is
always attached; so if the object falls out of scope; so does
the file.
"""
my_dir = join(self.tmp_dir, 'NNTPContent', 'test_copy')
assert(isdir(my_dir) is False)
assert(mkdir(my_dir) is True)
assert(isdir(my_dir) is True)
# Now create our NNTPContent object witin our directory
obj = NNTPContent(
filepath='myfile',
work_dir=my_dir,
)
# Content is attached by default
assert(obj.is_attached() is True)
obj.detach()
assert(obj.is_attached() is False)
new_dir = join(my_dir, 'copy')
assert(isdir(new_dir) is False)
# Create a copy of our object
obj_copy = obj.copy()
# Successfully loaded files are never attached reguardless
# of the original copy
assert(obj_copy.is_attached() is True)
# Reattach the original so it dies when this test is over
obj.attach()
assert(obj.is_attached() is True)
# Create a copy of our copy
obj_copy2 = obj_copy.copy()
assert(obj_copy2.is_attached() is True)
assert(isfile(obj_copy2.path()))
_path = obj_copy2.path()
del obj_copy2
assert(isfile(_path) is False)
assert(isfile(obj_copy.path()) is True)
_path = obj_copy.path()
del obj_copy
assert(isfile(_path) is False)
assert(isfile(obj.path()) is True)
_path = obj.path()
del obj
assert(isfile(_path) is False)
# now lets do a few more tests but with actual files this time
tmp_file = join(my_dir, '2MB.zip')
assert(self.touch(tmp_file, size='2MB', random=True) is True)
# Now create our NNTPContent object witin our directory
obj = NNTPContent(
filepath=tmp_file,
work_dir=my_dir,
)
assert(isfile(obj.path()) is True)
obj_copy = obj.copy()
assert(isfile(obj_copy.path()) is True)
stats = stat(obj_copy.path())
assert(bytes_to_strsize(stats['size']) == "2.00MB")
# Compare that our content is the same
assert(compare(obj_copy.path(), obj.path()) is True)
# note that the filenames are NOT the same so we are dealing with 2
# distinct copies here
assert(isfile(obj_copy.path()) is True)
assert(isfile(obj.path()) is True)
assert(obj.path() != obj_copy.path())
def test_strsize_n_bytes(self):
"""
A formatting tool to make bytes more readable for an end user
"""
# Garbage Entry
assert strsize_to_bytes(None) is None
assert strsize_to_bytes("0J") is None
assert strsize_to_bytes("") is None
assert strsize_to_bytes("totalgarbage") is None
# Allow integers
assert strsize_to_bytes(0) == 0
assert strsize_to_bytes(1024) == 1024
# Good Entries
assert strsize_to_bytes("0B") == 0
assert strsize_to_bytes("0") == 0
assert strsize_to_bytes("10") == 10
assert strsize_to_bytes("1K") == 1024
assert strsize_to_bytes("1M") == 1024*1024
assert strsize_to_bytes("1G") == 1024*1024*1024
assert strsize_to_bytes("1T") == 1024*1024*1024*1024
# Spaces between units and value are fine too
assert strsize_to_bytes(" 0 B ") == 0
assert strsize_to_bytes(" 1 K ") == 1024
assert strsize_to_bytes(" 1 M ") == 1024*1024
assert strsize_to_bytes(" 1 G ") == 1024*1024*1024
assert strsize_to_bytes(" 1 T ") == 1024*1024*1024*1024
# Support Byte character
assert strsize_to_bytes("1KB") == 1024
assert strsize_to_bytes("1MB") == 1024*1024
assert strsize_to_bytes("1GB") == 1024*1024*1024
assert strsize_to_bytes("1TB") == 1024*1024*1024*1024
# Support bit character
assert strsize_to_bytes("1Kb") == 1000
assert strsize_to_bytes("1Mb") == 1000*1000
assert strsize_to_bytes("1Gb") == 1000*1000*1000
assert strsize_to_bytes("1Tb") == 1000*1000*1000*1000
# Garbage Entry
assert bytes_to_strsize(None) is None
assert bytes_to_strsize('') is None
assert bytes_to_strsize('GARBAGE') is None
# Good Entries
assert bytes_to_strsize(0) == "0.00B"
assert bytes_to_strsize(1) == "1.00B"
assert bytes_to_strsize(1024) == "1.00KB"
assert bytes_to_strsize(1024*1024) == "1.00MB"
assert bytes_to_strsize(1024*1024*1024) == "1.00GB"
assert bytes_to_strsize(1024*1024*1024*1024) == "1.00TB"
# Support strings too
assert bytes_to_strsize("0") == "0.00B"
assert bytes_to_strsize("1024") == "1.00KB"
def test_stat(self):
"""
Stat makes it easier to disect the file extension, filesystem info
and mime information.
"""
general_keys = ('extension', 'basename', 'filename', 'dirname')
filesys_keys = ('created', 'modified', 'accessed', 'size')
mime_keys = ('mime',)
# Test a file that doesn't exist
tmp_file = join(self.tmp_dir, 'Utils_Test.stat', 'missing_file')
stats = stat(tmp_file)
assert stats is None
stats = stat(tmp_file, fsinfo=False)
assert stats is None
stats = stat(tmp_file, fsinfo=False, mime=False)
assert stats is None
# Create Temporary file 1MB in size
tmp_file = join(self.tmp_dir, 'Utils_Test.stat', '1MB.rar')
assert self.touch(tmp_file, size='1MB')
stats = stat(tmp_file)
# This check basically makes sure all of the expected keys
# are in place and that there aren't more or less
k_iter = chain(mime_keys, filesys_keys, general_keys)
k_len = len(mime_keys) + len(filesys_keys) + len(general_keys)
assert isinstance(stats, dict) is True
assert len([k for k in k_iter if k not in stats.keys()]) == 0
assert k_len == len(stats)
# Filesize should actually match what we set it as
assert bytes_to_strsize(stats['size']) == "1.00MB"
# different OS's and variations of python can yield different
# results. We're trying to just make sure that we find the
# rar keyword in the mime type
assert(stats['mime'] == 'application/x-rar-compressed')
# Create Temporary file 1MB in size
tmp_file = join(self.tmp_dir, 'Utils_Test.stat', '2MB.zip')
assert self.touch(tmp_file, size='2MB')
stats = stat(tmp_file)
# This check basically makes sure all of the expected keys
# are in place and that there aren't more or less
k_iter = chain(mime_keys, filesys_keys, general_keys)
k_len = len(mime_keys) + len(filesys_keys) + len(general_keys)
assert isinstance(stats, dict) is True
assert len([k for k in k_iter if k not in stats.keys()]) == 0
assert k_len == len(stats)
# Filesize should actually match what we set it as
assert bytes_to_strsize(stats['size']) == "2.00MB"
assert re.search(
'application/.*zip.*',
stats['mime'],
re.IGNORECASE,
) is not None
# Test different variations
stats = stat(tmp_file, mime=False)
# This check basically makes sure all of the expected keys
# are in place and that there aren't more or less
k_iter = chain(filesys_keys, general_keys)
k_len = len(filesys_keys) + len(general_keys)
assert isinstance(stats, dict) is True
assert len([k for k in k_iter if k not in stats.keys()]) == 0
assert k_len == len(stats)
# Test different variations
stats = stat(tmp_file, fsinfo=False, mime=True)
# This check basically makes sure all of the expected keys
# are in place and that there aren't more or less
k_iter = chain(mime_keys, general_keys)
k_len = len(mime_keys) + len(general_keys)
assert isinstance(stats, dict) is True
assert len([k for k in k_iter if k not in stats.keys()]) == 0
assert k_len == len(stats)
# Test different variations
stats = stat(tmp_file, fsinfo=False, mime=False)
# This check basically makes sure all of the expected keys
# are in place and that there aren't more or less
k_iter = chain(general_keys)
k_len = len(general_keys)
assert isinstance(stats, dict) is True
assert len([k for k in k_iter if k not in stats.keys()]) == 0
assert k_len == len(stats)
