def do_deletion(self, delete_function): # Pass a function that does the deletion to_delete = [ 'delete/not/protected', 'dir/to/delete', 'make/a/dir/to/delete', 'hello/delete' ] touched_new = ['touch/this'] too_new = ['hello/new', 'new'] all_dirs = to_delete + too_new + touched_new + \ listdeletable.config.DIRS_TO_AVOID + protected_list start_time = time.time() for next_dir in all_dirs: if next_dir not in too_new: self.tmpdir.write(os.path.join(next_dir, 'test_file.root'), bytearray(os.urandom(1024))) print('Waiting for some time.') time.sleep(5) cutoff_time = int(time.time()) time.sleep(5) for next_dir in too_new: self.tmpdir.write(os.path.join(next_dir, 'test_file.root'), bytearray(os.urandom(1024))) for next_dir in touched_new: os.utime( self.tmpdir.getpath(os.path.join(next_dir, 'test_file.root')), None) listdeletable.config.MIN_AGE = int(time.time() - cutoff_time) listdeletable.NOW = int(time.time()) listdeletable.main() delete_function() # Function that does deletion is done here for dir in all_dirs: check_file = self.tmpdir.getpath( os.path.join(dir, 'test_file.root')) self.assertEqual(os.path.exists(check_file), dir not in to_delete, 'File status is unexpected: %s' % check_file)
def test_directory_reduction(self): test_to_delete = 'dir/to/delete/test_file.root' self.tmpdir.write(test_to_delete, bytearray(os.urandom(1024))) print 'Waiting for some time.' time.sleep(5) cutoff_time = int(time.time()) time.sleep(5) listdeletable.config.WHICH_LIST = 'directories' listdeletable.config.MIN_AGE = int(time.time() - cutoff_time) listdeletable.NOW = int(time.time()) listdeletable.main() listdeletable.do_delete() self.assertFalse(os.path.exists(self.tmpdir.getpath(test_to_delete))) self.assertFalse(os.path.exists(self.tmpdir.getpath(os.path.dirname(test_to_delete)))) self.assertTrue(os.path.exists(self.tmpdir.getpath('dir')))
exit(0) # Then let's do no timing info: from cmstoolbox.unmergedcleaner import listdeletable listdeletable.set_config('/home/dabercro/dev_python/consistency_config.json', 'ListDeletable') listdeletable.PROTECTED_LIST = listdeletable.get_protected() listdeletable.PROTECTED_LIST.sort() deletion_file = '/home/dabercro/dev_python/dynamo-consistency/notime.txt' listdeletable.config.DELETION_FILE = deletion_file listdeletable.get_unmerged_files = lambda: unmerged.get_files(0) listdeletable.main() if unmerged.get_node('unmerged/logs', make_new=False): with open(deletion_file, 'a') as d_file: d_file.write('\n' + '\n'.join( unmerged.get_node('unmerged/logs').get_files( min_age=0, path='/store/unmerged'))) with open(deletion_file, 'r') as deleted: files = [line.strip() for line in deleted if line.strip()] print 'No time limit' if len(files): # Print the sum of the file sizes print 'To delete: %s' % sum(
def clean_unmerged(site): # pylint: disable=too-complex, too-many-branches """ Lists the /store/unmerged area of a site, and then uses :py:mod:`cmstoolbox.unmergedcleaner.listdeletable` to list files to delete and adds them to the registry. .. Warning:: This function has a number of side effects to various module configurations. Definitely call this after running the main site consistency. :param str site: The site to run the check over :returns: The number of files entered into the register and the number that are log files :rtype: int, int """ # First, we do a bunch of hacky configuration changes for /store/unmerged listdeletable.set_config(config.LOCATION, 'ListDeletable') # Set the directory list to unmerged only config.DIRECTORYLIST = ['unmerged'] logsdirs = ['unmerged/logs'] if opts.MORELOGS: morelogs = config.CONFIG.get('AdditionalLogDeletions', {}).get(site, []) LOG.debug('Adding more log directories: %s', morelogs) config.DIRECTORYLIST.extend(morelogs) logsdirs.extend(morelogs) # Set the IgnoreAge for directories to match the listdeletable config datatypes.DirectoryInfo.ignore_age = listdeletable.config.MIN_AGE / (24 * 3600) # Get the list of protected directories try: listdeletable.PROTECTED_LIST = listdeletable.get_protected() except listdeletable.SuspiciousConditions: LOG.error('Problem with protected list, returning nothing') return 0, 0 # If list is empty, exit because something is wrong if not listdeletable.PROTECTED_LIST: LOG.error( 'No protected list was found... Not cleaning /store/unmerged') return 0, 0 listdeletable.PROTECTED_LIST.sort() with open( os.path.join(config.vardir('cache/%s' % site), 'protectedLFNs.txt'), 'w') as protected: protected.write('\n'.join(listdeletable.PROTECTED_LIST) + '\n') # Create a tree structure that will hold the protected directories protected_tree = datatypes.DirectoryInfo() for directory in listdeletable.PROTECTED_LIST: protected_tree.get_node(directory) def check_protected(path): """ Determine if the path should be protected or not :param str path: full path of directory :returns: If the path should be protected :rtype: bool """ # If the directory is explicitly protected, of course don't delete it if bool(protected_tree.get_node(path, make_new=False)): LOG.debug('%s is protected', path) return True for protected in listdeletable.PROTECTED_LIST: # If a subdirectory, don't delete if path.startswith(protected): LOG.debug('%s is protected', path) return True # We sorted the protected list, so we don't have to check all of them if path < protected: break LOG.debug('%s is NOT protected', path) return False # Turn on the filtering of ignored directories listers.FILTER_IGNORED = True # Overwrite the RootPath, if needed new_root = config.CONFIG['ListDeletable'].get('AlternateRoot', {}).get(site) if new_root: config.CONFIG['RootPath'] = new_root listdeletable.config.UNMERGED_DIR_LOCATION = os.path.join( new_root, 'unmerged') # And do a listing of unmerged site_tree = remotelister.listing( # pylint: disable=unexpected-keyword-arg site, cache='unmerged', # cache keyword comes from cache decorator callback=EmptyRemover(site, check_protected)) # Setup the config a bit more deletion_file = os.path.join(config.vardir('work'), '%s_unmerged.txt' % site) listdeletable.config.DELETION_FILE = deletion_file # Reset the protected list in case the listing took a long time try: listdeletable.PROTECTED_LIST = listdeletable.get_protected() except listdeletable.SuspiciousConditions: LOG.error('Problem with protected list, returning nothing') return 0, 0 listdeletable.PROTECTED_LIST.sort() # Only consider things older than four weeks # Only look inside unmerged for the cleaning rootpath = config.CONFIG['RootPath'] listdeletable.get_unmerged_files = lambda: [ os.path.join(rootpath, fil) for fil in site_tree.get_node('unmerged'). get_files(listdeletable.config.MIN_AGE) ] # Do the cleaning listdeletable.main() config_dict = config.config_dict() # Delete the contents of the deletion file and the contents of the log directory that are old with open(deletion_file, 'a') as d_file: for logs in logsdirs: node = site_tree.get_node(logs, make_new=False) if node: d_file.write('\n' + '\n'.join( node.get_files( min_age=(int(config_dict['UnmergedLogsAge']) * 24 * 3600), path=os.path.join(rootpath, os.path.dirname(logs))))) to_delete = set() with open(deletion_file, 'r') as d_file: to_delete.update([l.strip() for l in d_file if l.strip()]) report_contents(site_tree.timestamp, site, [f for f in site_tree.get_files() if f not in to_delete]) history.report_unmerged([(name, site_tree.get_file(name)) for name in to_delete]) return registry.delete(site, to_delete), len( [f for f in to_delete if f.strip().endswith('.tar.gz')])