def purge(self, maxtemp=300, maxlock=600): """Purge the queue: * delete unused intermediate directories * delete too old temporary directories * unlock too old locked directories Arguments: maxtemp - maximum time for a temporary element. If 0, temporary elements will not be removed. maxlock - maximum time for a locked element. If 0, locked elements will not be unlocked. Raise: OSError - problem deleting element from disk Note: this uses first()/next() to iterate so this will reset the cursor """ # get the list of intermediate directories _list = [] for name in _directory_contents(self.path): if _DIRECTORY_REGEXP.match(name): _list.append(name) _list.sort() # try to purge all but last one if len(_list) > 1: _list.pop() for name in _list: path = "%s/%s" % (self.path, name) if _subdirs_num(path): continue _special_rmdir(path) # remove the volatile directories which are too old if maxtemp: oldtime = time.time() - maxtemp for name in self._volatile(): path = "%s/%s" % (self.path, name) if _older(path, oldtime): _warn("* removing too old volatile element: %s" % name) for file_name in _directory_contents(path, True): if file_name == LOCKED_DIRECTORY: continue fpath = "%s/%s" % (path, file_name) try: os.unlink(fpath) except Exception: error = sys.exc_info()[1] if error.errno != errno.ENOENT: raise OSError("cannot unlink(%s): %s" % (fpath, error)) _special_rmdir("%s/%s" % (path, LOCKED_DIRECTORY)) _special_rmdir(path) # iterate to find abandoned locked entries if maxlock: oldtime = time.time() - maxlock name = self.first() while name: if self._is_locked(name, oldtime): _warn("* removing too old locked element: %s" % name) self.unlock(name, True) name = self.next()
def _insertion_directory(self): """Return the name of the intermediate directory that can be used for insertion: * if there is none, an initial one will be created * if it is full, a new one will be created * in any case the name will match $_DIRECTORY_REGEXP Raise: OSError - can't list/make element directories """ _list = [] # get the list of existing directories for name in _directory_contents(self.path): if _DIRECTORY_REGEXP.match(name): _list.append(name) # handle the case with no directories yet if not _list: name = "%08x" % 0 _special_mkdir("%s/%s" % (self.path, name), self.umask) return name # check the last directory _list.sort() name = _list[-1] subdirs = _subdirs_num("%s/%s" % (self.path, name)) if subdirs: if subdirs < self.maxelts: return name else: # RACE: at this point, the directory does not exist anymore, # so it must have been purged after we listed the directory # contents. We do not try to do more and simply create a new # directory pass # we need a new directory name = "%08x" % (int(name, 16) + 1) _special_mkdir("%s/%s" % (self.path, name), self.umask) return name
def _insertion_directory(self): """Return the name of the intermediate directory that can be used for insertion: * if there is none, an initial one will be created * if it is full, a new one will be created * in any case the name will match $_DIRECTORY_REGEXP Raise: OSError - can't list/make element directories """ _list = [] # get the list of existing directories for name in _directory_contents(self.path): if _DIRECTORY_REGEXP.match(name): _list.append(name) # handle the case with no directories yet if not _list: name = '%08x' % 0 _special_mkdir('%s/%s' % (self.path, name), self.umask) return name # check the last directory _list.sort() name = _list[-1] subdirs = _subdirs_num('%s/%s' % (self.path, name)) if subdirs: if subdirs < self.maxelts: return name else: # RACE: at this point, the directory does not exist anymore, # so it must have been purged after we listed the directory # contents. We do not try to do more and simply create a new # directory pass # we need a new directory name = '%08x' % (int(name, 16) + 1) _special_mkdir('%s/%s' % (self.path, name), self.umask) return name
def purge(self, maxtemp=300, maxlock=600): """Purge the queue: * delete unused intermediate directories * delete too old temporary directories * unlock too old locked directories Arguments: maxtemp - maximum time for a temporary element. If 0, temporary elements will not be removed. maxlock - maximum time for a locked element. If 0, locked elements will not be unlocked. Raise: OSError - problem deleting element from disk Note: this uses first()/next() to iterate so this will reset the cursor """ # get the list of intermediate directories _list = [] for name in _directory_contents(self.path): if _DIRECTORY_REGEXP.match(name): _list.append(name) _list.sort() # try to purge all but last one if len(_list) > 1: _list.pop() for name in _list: path = '%s/%s' % (self.path, name) if _subdirs_num(path): continue _special_rmdir(path) # remove the volatile directories which are too old if maxtemp: oldtime = time.time() - maxtemp for name in self._volatile(): path = '%s/%s' % (self.path, name) if _older(path, oldtime): _warn("* removing too old volatile element: %s" % name) for file_name in _directory_contents(path, True): if file_name == LOCKED_DIRECTORY: continue fpath = '%s/%s' % (path, file_name) try: os.unlink(fpath) except Exception: error = sys.exc_info()[1] if error.errno != errno.ENOENT: raise OSError("cannot unlink(%s): %s" % (fpath, error)) _special_rmdir('%s/%s' % (path, LOCKED_DIRECTORY)) _special_rmdir(path) # iterate to find abandoned locked entries if maxlock: oldtime = time.time() - maxlock name = self.first() while name: if self._is_locked(name, oldtime): _warn("* removing too old locked element: %s" % name) self.unlock(name, True) name = self.next()
def __get_list_of_interm_dirs(self, dirs): """Fill out provided list with names of intermediate directories. """ for name in os.listdir(self.path): if _DIRECTORY_REGEXP.match(name): dirs.append(name)