def purge(self, maxtemp=300, maxlock=600): """Purge the queue by removing unused intermediate directories, removing too old temporary elements and unlocking too old locked elements (aka staled locks); note: this can take a long time on queues with many elements. maxtemp - maximum time for a temporary element (in seconds, default 300); if set to 0, temporary elements will not be removed maxlock - maximum time for a locked element (in seconds, default 600); if set to 0, locked elements will not be unlocked """ # get list of intermediate directories dirs = [] self.__get_list_of_interm_dirs(dirs) # remove all but old temporary or locked elements oldtemp = maxtemp != 0 and time.time() - maxtemp or 0 oldlock = maxlock != 0 and time.time() - maxlock or 0 if oldtemp or oldlock: for _dir in dirs: path = '%s/%s' % (self.path, _dir) tmp_lock_elems = [x for x in os.listdir(path) if re.search('(%s|%s)$' % (TEMPORARY_SUFFIX, LOCKED_SUFFIX), x)] for old in tmp_lock_elems: try: stat = os.stat('%s/%s' % (path, old)) except OSError: error = sys.exc_info()[1] if error.errno == errno.ENOENT: continue raise error if (old.endswith(TEMPORARY_SUFFIX) and stat.st_mtime >= oldtemp): continue if old.endswith(LOCKED_SUFFIX) and \ stat.st_mtime >= oldlock: continue _warn("WARNING: removing too old volatile file: %s/%s" % (self.path, old)) try: os.unlink('%s/%s' % (path, old)) except OSError: error = sys.exc_info()[1] if error.errno != errno.ENOENT: raise error # try to purge all but the last intermediate directory if len(dirs) > 1: dirs.sort() dirs.pop() for _dir in dirs: path = '%s/%s' % (self.path, _dir) if len(os.listdir(path)) == 0: _special_rmdir(path)
def purge(self, maxtemp=300, maxlock=600): """Purge the queue: * delete unused intermediate directories * delete too old temporary directories * unlock too old locked directories Arguments: maxtemp - maximum time for a temporary element. If 0, temporary elements will not be removed. maxlock - maximum time for a locked element. If 0, locked elements will not be unlocked. Raise: OSError - problem deleting element from disk Note: this uses first()/next() to iterate so this will reset the cursor """ # get the list of intermediate directories _list = [] for name in _directory_contents(self.path): if _DIRECTORY_REGEXP.match(name): _list.append(name) _list.sort() # try to purge all but last one if len(_list) > 1: _list.pop() for name in _list: path = "%s/%s" % (self.path, name) if _subdirs_num(path): continue _special_rmdir(path) # remove the volatile directories which are too old if maxtemp: oldtime = time.time() - maxtemp for name in self._volatile(): path = "%s/%s" % (self.path, name) if _older(path, oldtime): _warn("* removing too old volatile element: %s" % name) for file_name in _directory_contents(path, True): if file_name == LOCKED_DIRECTORY: continue fpath = "%s/%s" % (path, file_name) try: os.unlink(fpath) except Exception: error = sys.exc_info()[1] if error.errno != errno.ENOENT: raise OSError("cannot unlink(%s): %s" % (fpath, error)) _special_rmdir("%s/%s" % (path, LOCKED_DIRECTORY)) _special_rmdir(path) # iterate to find abandoned locked entries if maxlock: oldtime = time.time() - maxlock name = self.first() while name: if self._is_locked(name, oldtime): _warn("* removing too old locked element: %s" % name) self.unlock(name, True) name = self.next()
def purge(self, maxtemp=300, maxlock=600): """Purge the queue: * delete unused intermediate directories * delete too old temporary directories * unlock too old locked directories Arguments: maxtemp - maximum time for a temporary element. If 0, temporary elements will not be removed. maxlock - maximum time for a locked element. If 0, locked elements will not be unlocked. Raise: OSError - problem deleting element from disk Note: this uses first()/next() to iterate so this will reset the cursor """ # get the list of intermediate directories _list = [] for name in _directory_contents(self.path): if _DirectoryRegexp.match(name): _list.append(name) _list.sort() # try to purge all but last one if len(_list) > 1: _list.pop() for name in _list: path = '%s/%s' % (self.path, name) if _subdirs_num(path): continue _special_rmdir(path) # remove the volatile directories which are too old if maxtemp: oldtime = time.time() - maxtemp for name in self._volatile(): path = '%s/%s' % (self.path, name) if _older(path, oldtime): _warn("* removing too old volatile element: %s" % name) for file_name in _directory_contents(path, True): if file_name == LOCKED_DIRECTORY: continue fpath = '%s/%s' % (path, file_name) try: os.unlink(fpath) except Exception: error = sys.exc_info()[1] if error.errno != errno.ENOENT: raise OSError("cannot unlink(%s): %s" % (fpath, error)) _special_rmdir('%s/%s' % (path, LOCKED_DIRECTORY)) _special_rmdir(path) # iterate to find abandoned locked entries if maxlock: oldtime = time.time() - maxlock name = self.first() while name: if self._is_locked(name, oldtime): _warn("* removing too old locked element: %s" % name) self.unlock(name, True) name = self.next()