def purge(self, maxtemp=300, maxlock=600): """Purge the queue: * delete unused intermediate directories * delete too old temporary directories * unlock too old locked directories Arguments: maxtemp - maximum time for a temporary element. If 0, temporary elements will not be removed. maxlock - maximum time for a locked element. If 0, locked elements will not be unlocked. Raise: OSError - problem deleting element from disk Note: this uses first()/next() to iterate so this will reset the cursor """ # get the list of intermediate directories _list = [] for name in _directory_contents(self.path): if _DIRECTORY_REGEXP.match(name): _list.append(name) _list.sort() # try to purge all but last one if len(_list) > 1: _list.pop() for name in _list: path = "%s/%s" % (self.path, name) if _subdirs_num(path): continue _special_rmdir(path) # remove the volatile directories which are too old if maxtemp: oldtime = time.time() - maxtemp for name in self._volatile(): path = "%s/%s" % (self.path, name) if _older(path, oldtime): _warn("* removing too old volatile element: %s" % name) for file_name in _directory_contents(path, True): if file_name == LOCKED_DIRECTORY: continue fpath = "%s/%s" % (path, file_name) try: os.unlink(fpath) except Exception: error = sys.exc_info()[1] if error.errno != errno.ENOENT: raise OSError("cannot unlink(%s): %s" % (fpath, error)) _special_rmdir("%s/%s" % (path, LOCKED_DIRECTORY)) _special_rmdir(path) # iterate to find abandoned locked entries if maxlock: oldtime = time.time() - maxlock name = self.first() while name: if self._is_locked(name, oldtime): _warn("* removing too old locked element: %s" % name) self.unlock(name, True) name = self.next()
def _volatile(self): """ Return the list of volatile (i.e. temporary or obsolete) directories. """ _list = [] for name in _directory_contents("%s/%s" % (self.path, TEMPORARY_DIRECTORY), True): if _ELEMENT_REGEXP.match(name): _list.append("%s/%s" % (TEMPORARY_DIRECTORY, name)) for name in _directory_contents("%s/%s" % (self.path, OBSOLETE_DIRECTORY), True): if _ELEMENT_REGEXP.match(name): _list.append("%s/%s" % (OBSOLETE_DIRECTORY, name)) return _list
def _volatile(self): """ Return the list of volatile (i.e. temporary or obsolete) directories. """ _list = [] for name in _directory_contents( '%s/%s' % (self.path, TEMPORARY_DIRECTORY), True): if _ElementRegexp.match(name): _list.append('%s/%s' % (TEMPORARY_DIRECTORY, name)) for name in _directory_contents( '%s/%s' % (self.path, OBSOLETE_DIRECTORY), True): if _ElementRegexp.match(name): _list.append('%s/%s' % (OBSOLETE_DIRECTORY, name)) return _list
def __subdirs_num_count(path): """Count the number of sub-directories in the given directory: - return 0 if the directory does not exist (anymore) For systems where we cannot rely on the number of links, so we simply count the number of sub-directories. """ return len(_directory_contents(path, missingok=True))
def remove(self, ename): """Remove locked element from the queue. Arguments: ename - name of an element Raise: QueueError - invalid element name; element not locked; unexpected file in the element directory OSError - can't rename/remove a file/directory Note: doesn't return anything explicitly (i.e. returns NoneType) or fails """ _check_element(ename) if not self._is_locked(ename): raise QueueError("cannot remove %s: not locked" % ename) # move the element out of its intermediate directory path = "%s/%s" % (self.path, ename) while True: temp = "%s/%s/%s" % (self.path, OBSOLETE_DIRECTORY, _name(self.rndhex)) try: os.rename(path, temp) break except Exception: error = sys.exc_info()[1] if error.errno != errno.ENOTEMPTY and error.errno != errno.EEXIST: raise OSError("cannot rename(%s, %s): %s" % (ename, temp, error)) # RACE: the target directory was already present... # remove the data files for name in _directory_contents(temp): if name == LOCKED_DIRECTORY: continue if not _FILE_REGEXP.match(name): raise QueueError("unexpected file in %s: %s" % (temp, name)) path = "%s/%s" % (temp, name) try: os.unlink(path) except Exception: error = sys.exc_info()[1] raise OSError("cannot unlink(%s): %s" % (path, error)) # remove the locked directory path = "%s/%s" % (temp, LOCKED_DIRECTORY) while True: try: os.rmdir(path) except Exception: error = sys.exc_info()[1] raise OSError("cannot rmdir(%s): %s" % (path, error)) try: os.rmdir(temp) return except Exception: error = sys.exc_info()[1] if error.errno != errno.ENOTEMPTY and error.errno != errno.EEXIST: raise OSError("cannot rmdir(%s): %s" % (temp, error))
def _count(path): """Return the number of elements in the queue, regardless of their state. Raise: OSError - can't list/stat element directories """ count = 0 for name in [x for x in _directory_contents(path)]: subdirs = _subdirs_num("%s/%s" % (path, name)) if subdirs: count += subdirs return count
def _count(path): """Return the number of elements in the queue, regardless of their state. Raise: OSError - can't list/stat element directories """ count = 0 for name in [x for x in _directory_contents(path)]: subdirs = _subdirs_num('%s/%s' % (path, name)) if subdirs: count += subdirs return count
def _build_elements(self): """Build list of elements. Raise: OSError - can't list element directories """ while self.dirs: directory = self.dirs.pop(0) _list = [] for name in _directory_contents("%s/%s" % (self.path, directory), True): if _ELEMENT_REGEXP.match(name): _list.append(name) if not _list: continue self.elts = ["%s/%s" % (directory, x) for x in sorted(_list)] return True return False
def _build_elements(self): """Build list of elements. Raise: OSError - can't list element directories """ while self.dirs: directory = self.dirs.pop(0) _list = [] for name in _directory_contents('%s/%s' % (self.path, directory), True): if _ElementRegexp.match(name): _list.append(name) if not _list: continue self.elts = ['%s/%s' % (directory, x) for x in sorted(_list)] return True return False
def _insertion_directory(self): """Return the name of the intermediate directory that can be used for insertion: * if there is none, an initial one will be created * if it is full, a new one will be created * in any case the name will match $_DIRECTORY_REGEXP Raise: OSError - can't list/make element directories """ _list = [] # get the list of existing directories for name in _directory_contents(self.path): if _DIRECTORY_REGEXP.match(name): _list.append(name) # handle the case with no directories yet if not _list: name = "%08x" % 0 _special_mkdir("%s/%s" % (self.path, name), self.umask) return name # check the last directory _list.sort() name = _list[-1] subdirs = _subdirs_num("%s/%s" % (self.path, name)) if subdirs: if subdirs < self.maxelts: return name else: # RACE: at this point, the directory does not exist anymore, # so it must have been purged after we listed the directory # contents. We do not try to do more and simply create a new # directory pass # we need a new directory name = "%08x" % (int(name, 16) + 1) _special_mkdir("%s/%s" % (self.path, name), self.umask) return name
def _insertion_directory(self): """Return the name of the intermediate directory that can be used for insertion: * if there is none, an initial one will be created * if it is full, a new one will be created * in any case the name will match $_DirectoryRegexp Raise: OSError - can't list/make element directories """ _list = [] # get the list of existing directories for name in _directory_contents(self.path): if _DirectoryRegexp.match(name): _list.append(name) # handle the case with no directories yet if not _list: name = '%08x' % 0 _special_mkdir('%s/%s' % (self.path, name), self.umask) return name # check the last directory _list.sort() name = _list[-1] subdirs = _subdirs_num('%s/%s' % (self.path, name)) if subdirs: if subdirs < self.maxelts: return name else: # RACE: at this point, the directory does not exist anymore, # so it must have been purged after we listed the directory # contents. We do not try to do more and simply create a new # directory pass # we need a new directory name = '%08x' % (int(name, 16) + 1) _special_mkdir('%s/%s' % (self.path, name), self.umask) return name
def purge(self, maxtemp=300, maxlock=600): """Purge the queue: * delete unused intermediate directories * delete too old temporary directories * unlock too old locked directories Arguments: maxtemp - maximum time for a temporary element. If 0, temporary elements will not be removed. maxlock - maximum time for a locked element. If 0, locked elements will not be unlocked. Raise: OSError - problem deleting element from disk Note: this uses first()/next() to iterate so this will reset the cursor """ # get the list of intermediate directories _list = [] for name in _directory_contents(self.path): if _DirectoryRegexp.match(name): _list.append(name) _list.sort() # try to purge all but last one if len(_list) > 1: _list.pop() for name in _list: path = '%s/%s' % (self.path, name) if _subdirs_num(path): continue _special_rmdir(path) # remove the volatile directories which are too old if maxtemp: oldtime = time.time() - maxtemp for name in self._volatile(): path = '%s/%s' % (self.path, name) if _older(path, oldtime): _warn("* removing too old volatile element: %s" % name) for file_name in _directory_contents(path, True): if file_name == LOCKED_DIRECTORY: continue fpath = '%s/%s' % (path, file_name) try: os.unlink(fpath) except Exception: error = sys.exc_info()[1] if error.errno != errno.ENOENT: raise OSError("cannot unlink(%s): %s" % (fpath, error)) _special_rmdir('%s/%s' % (path, LOCKED_DIRECTORY)) _special_rmdir(path) # iterate to find abandoned locked entries if maxlock: oldtime = time.time() - maxlock name = self.first() while name: if self._is_locked(name, oldtime): _warn("* removing too old locked element: %s" % name) self.unlock(name, True) name = self.next()
def remove(self, ename): """Remove locked element from the queue. Arguments: ename - name of an element Raise: QueueError - invalid element name; element not locked; unexpected file in the element directory OSError - can't rename/remove a file/directory Note: doesn't return anything explicitly (i.e. returns NoneType) or fails """ _check_element(ename) if not self._is_locked(ename): raise QueueError("cannot remove %s: not locked" % ename) # move the element out of its intermediate directory path = '%s/%s' % (self.path, ename) while True: temp = '%s/%s/%s' % (self.path, OBSOLETE_DIRECTORY, _name(self.rndhex)) try: os.rename(path, temp) break except Exception: error = sys.exc_info()[1] if error.errno != errno.ENOTEMPTY and \ error.errno != errno.EEXIST: raise OSError("cannot rename(%s, %s): %s" % (ename, temp, error)) # RACE: the target directory was already present... # remove the data files for name in _directory_contents(temp): if name == LOCKED_DIRECTORY: continue if not _FileRegexp.match(name): raise QueueError("unexpected file in %s: %s" % (temp, name)) path = '%s/%s' % (temp, name) try: os.unlink(path) except Exception: error = sys.exc_info()[1] raise OSError("cannot unlink(%s): %s" % (path, error)) # remove the locked directory path = '%s/%s' % (temp, LOCKED_DIRECTORY) while True: try: os.rmdir(path) except Exception: error = sys.exc_info()[1] raise OSError("cannot rmdir(%s): %s" % (path, error)) try: os.rmdir(temp) return except Exception: error = sys.exc_info()[1] if error.errno != errno.ENOTEMPTY and \ error.errno != errno.EEXIST: raise OSError("cannot rmdir(%s): %s" % (temp, error))