def __repr__(self): """ Return a printable version of the file being read """ if self.part is not None: return '<NNTPBinaryContent sort=%d filename="%s" part=%d/%d len=%s />' % ( self.sort_no, self.filename, self.part, self.total_parts, bytes_to_strsize(len(self)), ) else: return '<NNTPBinaryContent sort=%d filename="%s" len=%s />' % ( self.sort_no, self.filename, bytes_to_strsize(len(self)), )
def __repr__(self): """ Return a printable version of the file being read """ if self.part is not None: return '<NNTPBinaryContent sort=%d filename="%s" part=%d/%d len=%s />' % ( self.sort_no, self.filename, self.part, self.total_parts, bytes_to_strsize(len(self)), ) else: return '<NNTPBinaryContent sort=%d filename="%s" len=%s />' % ( self.sort_no, self.filename, bytes_to_strsize(len(self)), )
def database_status(ctx): """ displays details on the current database store """ db_path = join(ctx['NNTPSettings'].work_dir, 'cache', 'search') logger.debug('Scanning %s for databases...' % db_path) with pushd(db_path, create_if_missing=True): results = find( db_path, suffix_filter=SQLITE_DATABASE_EXTENSION, fsinfo=True, max_depth=1, ) # Use our Database first if it exists session = ctx['NNTPSettings'].session() if not session: logger.error('Could not acquire a database connection.') exit(1) # PEP8 E712 does not allow us to make a comparison to a boolean value # using the == instead of the keyword 'in'. However SQLAlchemy # requires us to do just because that's how the amazing tool works. # so to get around the pep8 error, we'll just define a variable equal # to True and then we can compare to it pep8_e712 = True try: # Get a list of watched groups groups = dict( session.query(Group.name, Group.id).filter(Group.watch == pep8_e712).all()) except OperationalError: # Get a list of watched groups logger.warning('The database does not appear to be initialized.') logger.info('Try running: "nr db init" first.') exit(0) if not len(results): logger.info('There are no groups configured to be watched.') exit(0) for _, meta in results.iteritems(): # Open up the database flags = '' if meta['filename'] in groups: flags += 'W' print('%-65s %-10s %s' % ( meta['filename'], bytes_to_strsize(meta['size']), flags, ))
def database_status(ctx): """ displays details on the current database store """ db_path = join(ctx['NNTPSettings'].work_dir, 'cache', 'search') logger.debug('Scanning %s for databases...' % db_path) with pushd(db_path, create_if_missing=True): results = find( db_path, suffix_filter=SQLITE_DATABASE_EXTENSION, fsinfo=True, max_depth=1, ) # Use our Database first if it exists session = ctx['NNTPSettings'].session() if not session: logger.error('Could not acquire a database connection.') exit(1) # PEP8 E712 does not allow us to make a comparison to a boolean value # using the == instead of the keyword 'in'. However SQLAlchemy # requires us to do just because that's how the amazing tool works. # so to get around the pep8 error, we'll just define a variable equal # to True and then we can compare to it pep8_e712 = True try: # Get a list of watched groups groups = dict(session.query(Group.name, Group.id) .filter(Group.watch == pep8_e712).all()) except OperationalError: # Get a list of watched groups logger.warning('The database does not appear to be initialized.') logger.info('Try running: "nr db init" first.') exit(0) if not len(results): logger.info('There are no groups configured to be watched.') exit(0) for _, meta in results.iteritems(): # Open up the database flags = '' if meta['filename'] in groups: flags += 'W' print('%-65s %-10s %s' % ( meta['filename'], bytes_to_strsize(meta['size']), flags, ))
def watch_dir(self, path, regex=None, prefix=None, suffix=None, ignore=None, case_sensitive=True, seconds=15): """Monitors a directory for files that have been added/changed path: is the path to monitor ignore: is a sortedset of files already parsed seconds: is how long it takes a file to go untouched for before we presume it has been completely written to disk. """ if ignore is None: ignore = sortedset() findings = find( path, fsinfo=True, regex_filter=regex, prefix_filter=prefix, suffix_filter=suffix, case_sensitive=case_sensitive, ) findings = [ (p, f['size'], f['created'], f['modified']) for p, f in findings.items() if (f['modified'] - f['created']).total_seconds() >= seconds and f['basename'] not in ignore ] # Sort list by created date findings.sort(key=lambda x: x[3]) for f in findings: logger.info('Created %s (size=%s)' % ( f, bytes_to_strsize(f[1]), )) # Add to our filter list ignore.add(f[0]) # Return our ignore list (which is acutally also a found list) return ignore
def watch_dir(self, path, regex=None, prefix=None, suffix=None, ignore=None, case_sensitive=True, seconds=15): """Monitors a directory for files that have been added/changed path: is the path to monitor ignore: is a sortedset of files already parsed seconds: is how long it takes a file to go untouched for before we presume it has been completely written to disk. """ if ignore is None: ignore = sortedset() findings = find( path, fsinfo=True, regex_filter=regex, prefix_filter=prefix, suffix_filter=suffix, case_sensitive=case_sensitive, ) findings = [ (p, f['size'], f['created'], f['modified']) for p, f in findings.items() if (f['modified'] - f['created']).total_seconds() >= seconds and f['basename'] not in ignore ] # Sort list by created date findings.sort(key=lambda x: x[3]) for f in findings: logger.info('Created %s (size=%s)' % ( f, bytes_to_strsize(f[1]), )) # Add to our filter list ignore.add(f[0]) # Return our ignore list (which is acutally also a found list) return ignore
def test_strsize_n_bytes(self): """ A formatting tool to make bytes more readable for an end user """ # Garbage Entry assert strsize_to_bytes(None) is None assert strsize_to_bytes("0J") is None assert strsize_to_bytes("") is None assert strsize_to_bytes("totalgarbage") is None # Allow integers assert strsize_to_bytes(0) == 0 assert strsize_to_bytes(1024) == 1024 # Good Entries assert strsize_to_bytes("0B") == 0 assert strsize_to_bytes("0") == 0 assert strsize_to_bytes("10") == 10 assert strsize_to_bytes("1K") == 1024 assert strsize_to_bytes("1M") == 1024 * 1024 assert strsize_to_bytes("1G") == 1024 * 1024 * 1024 assert strsize_to_bytes("1T") == 1024 * 1024 * 1024 * 1024 # Spaces between units and value are fine too assert strsize_to_bytes(" 0 B ") == 0 assert strsize_to_bytes(" 1 K ") == 1024 assert strsize_to_bytes(" 1 M ") == 1024 * 1024 assert strsize_to_bytes(" 1 G ") == 1024 * 1024 * 1024 assert strsize_to_bytes(" 1 T ") == 1024 * 1024 * 1024 * 1024 # Support Byte character assert strsize_to_bytes("1KB") == 1024 assert strsize_to_bytes("1MB") == 1024 * 1024 assert strsize_to_bytes("1GB") == 1024 * 1024 * 1024 assert strsize_to_bytes("1TB") == 1024 * 1024 * 1024 * 1024 # Support bit character assert strsize_to_bytes("1Kb") == 1000 assert strsize_to_bytes("1Mb") == 1000 * 1000 assert strsize_to_bytes("1Gb") == 1000 * 1000 * 1000 assert strsize_to_bytes("1Tb") == 1000 * 1000 * 1000 * 1000 # Garbage Entry assert bytes_to_strsize(None) is None assert bytes_to_strsize('') is None assert bytes_to_strsize('GARBAGE') is None # Good Entries assert bytes_to_strsize(0) == "0.00B" assert bytes_to_strsize(1) == "1.00B" assert bytes_to_strsize(1024) == "1.00KB" assert bytes_to_strsize(1024 * 1024) == "1.00MB" assert bytes_to_strsize(1024 * 1024 * 1024) == "1.00GB" assert bytes_to_strsize(1024 * 1024 * 1024 * 1024) == "1.00TB" # Support strings too assert bytes_to_strsize("0") == "0.00B" assert bytes_to_strsize("1024") == "1.00KB"
def test_stat(self): """ Stat makes it easier to disect the file extension, filesystem info and mime information. """ general_keys = ('extension', 'basename', 'filename', 'dirname') filesys_keys = ('created', 'modified', 'accessed', 'size') mime_keys = ('mime', ) # Test a file that doesn't exist tmp_file = join(self.tmp_dir, 'Utils_Test.stat', 'missing_file') stats = stat(tmp_file) assert stats is None stats = stat(tmp_file, fsinfo=False) assert stats is None stats = stat(tmp_file, fsinfo=False, mime=False) assert stats is None # Create Temporary file 1MB in size tmp_file = join(self.tmp_dir, 'Utils_Test.stat', '1MB.rar') assert self.touch(tmp_file, size='1MB') stats = stat(tmp_file) # This check basically makes sure all of the expected keys # are in place and that there aren't more or less k_iter = chain(mime_keys, filesys_keys, general_keys) k_len = len(mime_keys) + len(filesys_keys) + len(general_keys) assert isinstance(stats, dict) is True assert len([k for k in k_iter if k not in stats.keys()]) == 0 assert k_len == len(stats) # Filesize should actually match what we set it as assert bytes_to_strsize(stats['size']) == "1.00MB" # different OS's and variations of python can yield different # results. We're trying to just make sure that we find the # rar keyword in the mime type assert (stats['mime'] == 'application/x-rar-compressed') # Create Temporary file 1MB in size tmp_file = join(self.tmp_dir, 'Utils_Test.stat', '2MB.zip') assert self.touch(tmp_file, size='2MB') stats = stat(tmp_file) # This check basically makes sure all of the expected keys # are in place and that there aren't more or less k_iter = chain(mime_keys, filesys_keys, general_keys) k_len = len(mime_keys) + len(filesys_keys) + len(general_keys) assert isinstance(stats, dict) is True assert len([k for k in k_iter if k not in stats.keys()]) == 0 assert k_len == len(stats) # Filesize should actually match what we set it as assert bytes_to_strsize(stats['size']) == "2.00MB" assert re.search( 'application/.*zip.*', stats['mime'], re.IGNORECASE, ) is not None # Test different variations stats = stat(tmp_file, mime=False) # This check basically makes sure all of the expected keys # are in place and that there aren't more or less k_iter = chain(filesys_keys, general_keys) k_len = len(filesys_keys) + len(general_keys) assert isinstance(stats, dict) is True assert len([k for k in k_iter if k not in stats.keys()]) == 0 assert k_len == len(stats) # Test different variations stats = stat(tmp_file, fsinfo=False, mime=True) # This check basically makes sure all of the expected keys # are in place and that there aren't more or less k_iter = chain(mime_keys, general_keys) k_len = len(mime_keys) + len(general_keys) assert isinstance(stats, dict) is True assert len([k for k in k_iter if k not in stats.keys()]) == 0 assert k_len == len(stats) # Test different variations stats = stat(tmp_file, fsinfo=False, mime=False) # This check basically makes sure all of the expected keys # are in place and that there aren't more or less k_iter = chain(general_keys) k_len = len(general_keys) assert isinstance(stats, dict) is True assert len([k for k in k_iter if k not in stats.keys()]) == 0 assert k_len == len(stats)
def test_copy01(self): """ Test our copy function The copy function allows us to duplicate an existing NNTPContent object without obstructing the original. Copied content is always attached; so if the object falls out of scope; so does the file. """ my_dir = join(self.tmp_dir, 'NNTPContent', 'test_copy') assert (isdir(my_dir) is False) assert (mkdir(my_dir) is True) assert (isdir(my_dir) is True) # Now create our NNTPContent object witin our directory obj = NNTPContent( filepath='myfile', work_dir=my_dir, ) # Content is attached by default assert (obj.is_attached() is True) obj.detach() assert (obj.is_attached() is False) new_dir = join(my_dir, 'copy') assert (isdir(new_dir) is False) # Create a copy of our object obj_copy = obj.copy() # Successfully loaded files are never attached reguardless # of the original copy assert (obj_copy.is_attached() is True) # Reattach the original so it dies when this test is over obj.attach() assert (obj.is_attached() is True) # Create a copy of our copy obj_copy2 = obj_copy.copy() assert (obj_copy2.is_attached() is True) assert (isfile(obj_copy2.path())) _path = obj_copy2.path() del obj_copy2 assert (isfile(_path) is False) assert (isfile(obj_copy.path()) is True) _path = obj_copy.path() del obj_copy assert (isfile(_path) is False) assert (isfile(obj.path()) is True) _path = obj.path() del obj assert (isfile(_path) is False) # now lets do a few more tests but with actual files this time tmp_file = join(my_dir, '2MB.zip') assert (self.touch(tmp_file, size='2MB', random=True) is True) # Now create our NNTPContent object witin our directory obj = NNTPContent( filepath=tmp_file, work_dir=my_dir, ) assert (isfile(obj.path()) is True) obj_copy = obj.copy() assert (isfile(obj_copy.path()) is True) stats = stat(obj_copy.path()) assert (bytes_to_strsize(stats['size']) == "2.00MB") # Compare that our content is the same assert (compare(obj_copy.path(), obj.path()) is True) # note that the filenames are NOT the same so we are dealing with 2 # distinct copies here assert (isfile(obj_copy.path()) is True) assert (isfile(obj.path()) is True) assert (obj.path() != obj_copy.path())
def test_copy01(self): """ Test our copy function The copy function allows us to duplicate an existing NNTPContent object without obstructing the original. Copied content is always attached; so if the object falls out of scope; so does the file. """ my_dir = join(self.tmp_dir, 'NNTPContent', 'test_copy') assert(isdir(my_dir) is False) assert(mkdir(my_dir) is True) assert(isdir(my_dir) is True) # Now create our NNTPContent object witin our directory obj = NNTPContent( filepath='myfile', work_dir=my_dir, ) # Content is attached by default assert(obj.is_attached() is True) obj.detach() assert(obj.is_attached() is False) new_dir = join(my_dir, 'copy') assert(isdir(new_dir) is False) # Create a copy of our object obj_copy = obj.copy() # Successfully loaded files are never attached reguardless # of the original copy assert(obj_copy.is_attached() is True) # Reattach the original so it dies when this test is over obj.attach() assert(obj.is_attached() is True) # Create a copy of our copy obj_copy2 = obj_copy.copy() assert(obj_copy2.is_attached() is True) assert(isfile(obj_copy2.path())) _path = obj_copy2.path() del obj_copy2 assert(isfile(_path) is False) assert(isfile(obj_copy.path()) is True) _path = obj_copy.path() del obj_copy assert(isfile(_path) is False) assert(isfile(obj.path()) is True) _path = obj.path() del obj assert(isfile(_path) is False) # now lets do a few more tests but with actual files this time tmp_file = join(my_dir, '2MB.zip') assert(self.touch(tmp_file, size='2MB', random=True) is True) # Now create our NNTPContent object witin our directory obj = NNTPContent( filepath=tmp_file, work_dir=my_dir, ) assert(isfile(obj.path()) is True) obj_copy = obj.copy() assert(isfile(obj_copy.path()) is True) stats = stat(obj_copy.path()) assert(bytes_to_strsize(stats['size']) == "2.00MB") # Compare that our content is the same assert(compare(obj_copy.path(), obj.path()) is True) # note that the filenames are NOT the same so we are dealing with 2 # distinct copies here assert(isfile(obj_copy.path()) is True) assert(isfile(obj.path()) is True) assert(obj.path() != obj_copy.path())
def test_strsize_n_bytes(self): """ A formatting tool to make bytes more readable for an end user """ # Garbage Entry assert strsize_to_bytes(None) is None assert strsize_to_bytes("0J") is None assert strsize_to_bytes("") is None assert strsize_to_bytes("totalgarbage") is None # Allow integers assert strsize_to_bytes(0) == 0 assert strsize_to_bytes(1024) == 1024 # Good Entries assert strsize_to_bytes("0B") == 0 assert strsize_to_bytes("0") == 0 assert strsize_to_bytes("10") == 10 assert strsize_to_bytes("1K") == 1024 assert strsize_to_bytes("1M") == 1024*1024 assert strsize_to_bytes("1G") == 1024*1024*1024 assert strsize_to_bytes("1T") == 1024*1024*1024*1024 # Spaces between units and value are fine too assert strsize_to_bytes(" 0 B ") == 0 assert strsize_to_bytes(" 1 K ") == 1024 assert strsize_to_bytes(" 1 M ") == 1024*1024 assert strsize_to_bytes(" 1 G ") == 1024*1024*1024 assert strsize_to_bytes(" 1 T ") == 1024*1024*1024*1024 # Support Byte character assert strsize_to_bytes("1KB") == 1024 assert strsize_to_bytes("1MB") == 1024*1024 assert strsize_to_bytes("1GB") == 1024*1024*1024 assert strsize_to_bytes("1TB") == 1024*1024*1024*1024 # Support bit character assert strsize_to_bytes("1Kb") == 1000 assert strsize_to_bytes("1Mb") == 1000*1000 assert strsize_to_bytes("1Gb") == 1000*1000*1000 assert strsize_to_bytes("1Tb") == 1000*1000*1000*1000 # Garbage Entry assert bytes_to_strsize(None) is None assert bytes_to_strsize('') is None assert bytes_to_strsize('GARBAGE') is None # Good Entries assert bytes_to_strsize(0) == "0.00B" assert bytes_to_strsize(1) == "1.00B" assert bytes_to_strsize(1024) == "1.00KB" assert bytes_to_strsize(1024*1024) == "1.00MB" assert bytes_to_strsize(1024*1024*1024) == "1.00GB" assert bytes_to_strsize(1024*1024*1024*1024) == "1.00TB" # Support strings too assert bytes_to_strsize("0") == "0.00B" assert bytes_to_strsize("1024") == "1.00KB"
def test_stat(self): """ Stat makes it easier to disect the file extension, filesystem info and mime information. """ general_keys = ('extension', 'basename', 'filename', 'dirname') filesys_keys = ('created', 'modified', 'accessed', 'size') mime_keys = ('mime',) # Test a file that doesn't exist tmp_file = join(self.tmp_dir, 'Utils_Test.stat', 'missing_file') stats = stat(tmp_file) assert stats is None stats = stat(tmp_file, fsinfo=False) assert stats is None stats = stat(tmp_file, fsinfo=False, mime=False) assert stats is None # Create Temporary file 1MB in size tmp_file = join(self.tmp_dir, 'Utils_Test.stat', '1MB.rar') assert self.touch(tmp_file, size='1MB') stats = stat(tmp_file) # This check basically makes sure all of the expected keys # are in place and that there aren't more or less k_iter = chain(mime_keys, filesys_keys, general_keys) k_len = len(mime_keys) + len(filesys_keys) + len(general_keys) assert isinstance(stats, dict) is True assert len([k for k in k_iter if k not in stats.keys()]) == 0 assert k_len == len(stats) # Filesize should actually match what we set it as assert bytes_to_strsize(stats['size']) == "1.00MB" # different OS's and variations of python can yield different # results. We're trying to just make sure that we find the # rar keyword in the mime type assert(stats['mime'] == 'application/x-rar-compressed') # Create Temporary file 1MB in size tmp_file = join(self.tmp_dir, 'Utils_Test.stat', '2MB.zip') assert self.touch(tmp_file, size='2MB') stats = stat(tmp_file) # This check basically makes sure all of the expected keys # are in place and that there aren't more or less k_iter = chain(mime_keys, filesys_keys, general_keys) k_len = len(mime_keys) + len(filesys_keys) + len(general_keys) assert isinstance(stats, dict) is True assert len([k for k in k_iter if k not in stats.keys()]) == 0 assert k_len == len(stats) # Filesize should actually match what we set it as assert bytes_to_strsize(stats['size']) == "2.00MB" assert re.search( 'application/.*zip.*', stats['mime'], re.IGNORECASE, ) is not None # Test different variations stats = stat(tmp_file, mime=False) # This check basically makes sure all of the expected keys # are in place and that there aren't more or less k_iter = chain(filesys_keys, general_keys) k_len = len(filesys_keys) + len(general_keys) assert isinstance(stats, dict) is True assert len([k for k in k_iter if k not in stats.keys()]) == 0 assert k_len == len(stats) # Test different variations stats = stat(tmp_file, fsinfo=False, mime=True) # This check basically makes sure all of the expected keys # are in place and that there aren't more or less k_iter = chain(mime_keys, general_keys) k_len = len(mime_keys) + len(general_keys) assert isinstance(stats, dict) is True assert len([k for k in k_iter if k not in stats.keys()]) == 0 assert k_len == len(stats) # Test different variations stats = stat(tmp_file, fsinfo=False, mime=False) # This check basically makes sure all of the expected keys # are in place and that there aren't more or less k_iter = chain(general_keys) k_len = len(general_keys) assert isinstance(stats, dict) is True assert len([k for k in k_iter if k not in stats.keys()]) == 0 assert k_len == len(stats)