def do_deletion(self, delete_function):
# Pass a function that does the deletion
to_delete = [
'delete/not/protected', 'dir/to/delete', 'make/a/dir/to/delete',
'hello/delete'
]
touched_new = ['touch/this']
too_new = ['hello/new', 'new']
all_dirs = to_delete + too_new + touched_new + \
listdeletable.config.DIRS_TO_AVOID + protected_list
start_time = time.time()
for next_dir in all_dirs:
if next_dir not in too_new:
self.tmpdir.write(os.path.join(next_dir, 'test_file.root'),
bytearray(os.urandom(1024)))
print('Waiting for some time.')
time.sleep(5)
cutoff_time = int(time.time())
time.sleep(5)
for next_dir in too_new:
self.tmpdir.write(os.path.join(next_dir, 'test_file.root'),
bytearray(os.urandom(1024)))
for next_dir in touched_new:
os.utime(
self.tmpdir.getpath(os.path.join(next_dir, 'test_file.root')),
None)
listdeletable.config.MIN_AGE = int(time.time() - cutoff_time)
listdeletable.NOW = int(time.time())
listdeletable.main()
delete_function() # Function that does deletion is done here
for dir in all_dirs:
check_file = self.tmpdir.getpath(
os.path.join(dir, 'test_file.root'))
self.assertEqual(os.path.exists(check_file), dir not in to_delete,
'File status is unexpected: %s' % check_file)
def test_directory_reduction(self):
test_to_delete = 'dir/to/delete/test_file.root'
self.tmpdir.write(test_to_delete,
bytearray(os.urandom(1024)))
print 'Waiting for some time.'
time.sleep(5)
cutoff_time = int(time.time())
time.sleep(5)
listdeletable.config.WHICH_LIST = 'directories'
listdeletable.config.MIN_AGE = int(time.time() - cutoff_time)
listdeletable.NOW = int(time.time())
listdeletable.main()
listdeletable.do_delete()
self.assertFalse(os.path.exists(self.tmpdir.getpath(test_to_delete)))
self.assertFalse(os.path.exists(self.tmpdir.getpath(os.path.dirname(test_to_delete))))
self.assertTrue(os.path.exists(self.tmpdir.getpath('dir')))
exit(0)
# Then let's do no timing info:
from cmstoolbox.unmergedcleaner import listdeletable
listdeletable.set_config('/home/dabercro/dev_python/consistency_config.json',
'ListDeletable')
listdeletable.PROTECTED_LIST = listdeletable.get_protected()
listdeletable.PROTECTED_LIST.sort()
deletion_file = '/home/dabercro/dev_python/dynamo-consistency/notime.txt'
listdeletable.config.DELETION_FILE = deletion_file
listdeletable.get_unmerged_files = lambda: unmerged.get_files(0)
listdeletable.main()
if unmerged.get_node('unmerged/logs', make_new=False):
with open(deletion_file, 'a') as d_file:
d_file.write('\n' + '\n'.join(
unmerged.get_node('unmerged/logs').get_files(
min_age=0, path='/store/unmerged')))
with open(deletion_file, 'r') as deleted:
files = [line.strip() for line in deleted if line.strip()]
print 'No time limit'
if len(files):
# Print the sum of the file sizes
print 'To delete: %s' % sum(
def clean_unmerged(site): # pylint: disable=too-complex, too-many-branches
"""
Lists the /store/unmerged area of a site, and then uses
:py:mod:`cmstoolbox.unmergedcleaner.listdeletable`
to list files to delete and adds them to the registry.
.. Warning::
This function has a number of side effects to various module configurations.
Definitely call this after running the main site consistency.
:param str site: The site to run the check over
:returns: The number of files entered into the register and the number that are log files
:rtype: int, int
"""
# First, we do a bunch of hacky configuration changes for /store/unmerged
listdeletable.set_config(config.LOCATION, 'ListDeletable')
# Set the directory list to unmerged only
config.DIRECTORYLIST = ['unmerged']
logsdirs = ['unmerged/logs']
if opts.MORELOGS:
morelogs = config.CONFIG.get('AdditionalLogDeletions',
{}).get(site, [])
LOG.debug('Adding more log directories: %s', morelogs)
config.DIRECTORYLIST.extend(morelogs)
logsdirs.extend(morelogs)
# Set the IgnoreAge for directories to match the listdeletable config
datatypes.DirectoryInfo.ignore_age = listdeletable.config.MIN_AGE / (24 *
3600)
# Get the list of protected directories
try:
listdeletable.PROTECTED_LIST = listdeletable.get_protected()
except listdeletable.SuspiciousConditions:
LOG.error('Problem with protected list, returning nothing')
return 0, 0
# If list is empty, exit because something is wrong
if not listdeletable.PROTECTED_LIST:
LOG.error(
'No protected list was found... Not cleaning /store/unmerged')
return 0, 0
listdeletable.PROTECTED_LIST.sort()
with open(
os.path.join(config.vardir('cache/%s' % site),
'protectedLFNs.txt'), 'w') as protected:
protected.write('\n'.join(listdeletable.PROTECTED_LIST) + '\n')
# Create a tree structure that will hold the protected directories
protected_tree = datatypes.DirectoryInfo()
for directory in listdeletable.PROTECTED_LIST:
protected_tree.get_node(directory)
def check_protected(path):
"""
Determine if the path should be protected or not
:param str path: full path of directory
:returns: If the path should be protected
:rtype: bool
"""
# If the directory is explicitly protected, of course don't delete it
if bool(protected_tree.get_node(path, make_new=False)):
LOG.debug('%s is protected', path)
return True
for protected in listdeletable.PROTECTED_LIST:
# If a subdirectory, don't delete
if path.startswith(protected):
LOG.debug('%s is protected', path)
return True
# We sorted the protected list, so we don't have to check all of them
if path < protected:
break
LOG.debug('%s is NOT protected', path)
return False
# Turn on the filtering of ignored directories
listers.FILTER_IGNORED = True
# Overwrite the RootPath, if needed
new_root = config.CONFIG['ListDeletable'].get('AlternateRoot',
{}).get(site)
if new_root:
config.CONFIG['RootPath'] = new_root
listdeletable.config.UNMERGED_DIR_LOCATION = os.path.join(
new_root, 'unmerged')
# And do a listing of unmerged
site_tree = remotelister.listing( # pylint: disable=unexpected-keyword-arg
site,
cache='unmerged', # cache keyword comes from cache decorator
callback=EmptyRemover(site, check_protected))
# Setup the config a bit more
deletion_file = os.path.join(config.vardir('work'),
'%s_unmerged.txt' % site)
listdeletable.config.DELETION_FILE = deletion_file
# Reset the protected list in case the listing took a long time
try:
listdeletable.PROTECTED_LIST = listdeletable.get_protected()
except listdeletable.SuspiciousConditions:
LOG.error('Problem with protected list, returning nothing')
return 0, 0
listdeletable.PROTECTED_LIST.sort()
# Only consider things older than four weeks
# Only look inside unmerged for the cleaning
rootpath = config.CONFIG['RootPath']
listdeletable.get_unmerged_files = lambda: [
os.path.join(rootpath, fil) for fil in site_tree.get_node('unmerged').
get_files(listdeletable.config.MIN_AGE)
]
# Do the cleaning
listdeletable.main()
config_dict = config.config_dict()
# Delete the contents of the deletion file and the contents of the log directory that are old
with open(deletion_file, 'a') as d_file:
for logs in logsdirs:
node = site_tree.get_node(logs, make_new=False)
if node:
d_file.write('\n' + '\n'.join(
node.get_files(
min_age=(int(config_dict['UnmergedLogsAge']) * 24 *
3600),
path=os.path.join(rootpath, os.path.dirname(logs)))))
to_delete = set()
with open(deletion_file, 'r') as d_file:
to_delete.update([l.strip() for l in d_file if l.strip()])
report_contents(site_tree.timestamp, site,
[f for f in site_tree.get_files() if f not in to_delete])
history.report_unmerged([(name, site_tree.get_file(name))
for name in to_delete])
return registry.delete(site, to_delete), len(
[f for f in to_delete if f.strip().endswith('.tar.gz')])