def purge(self, maxtemp=300, maxlock=600):
"""Purge the queue by removing unused intermediate directories,
removing too old temporary elements and unlocking too old locked
elements (aka staled locks); note: this can take a long time on
queues with many elements.
maxtemp - maximum time for a temporary element
(in seconds, default 300);
if set to 0, temporary elements will not be removed
maxlock - maximum time for a locked element
(in seconds, default 600);
if set to 0, locked elements will not be unlocked
"""
# get list of intermediate directories
dirs = []
self.__get_list_of_interm_dirs(dirs)
# remove all but old temporary or locked elements
oldtemp = maxtemp != 0 and time.time() - maxtemp or 0
oldlock = maxlock != 0 and time.time() - maxlock or 0
if oldtemp or oldlock:
for _dir in dirs:
path = '%s/%s' % (self.path, _dir)
tmp_lock_elems = [x for x in os.listdir(path)
if re.search('(%s|%s)$' %
(TEMPORARY_SUFFIX,
LOCKED_SUFFIX), x)]
for old in tmp_lock_elems:
try:
stat = os.stat('%s/%s' % (path, old))
except OSError:
error = sys.exc_info()[1]
if error.errno == errno.ENOENT:
continue
raise error
if (old.endswith(TEMPORARY_SUFFIX) and
stat.st_mtime >= oldtemp):
continue
if old.endswith(LOCKED_SUFFIX) and \
stat.st_mtime >= oldlock:
continue
_warn("WARNING: removing too old volatile file: %s/%s" %
(self.path, old))
try:
os.unlink('%s/%s' % (path, old))
except OSError:
error = sys.exc_info()[1]
if error.errno != errno.ENOENT:
raise error
# try to purge all but the last intermediate directory
if len(dirs) > 1:
dirs.sort()
dirs.pop()
for _dir in dirs:
path = '%s/%s' % (self.path, _dir)
if len(os.listdir(path)) == 0:
_special_rmdir(path)
def purge(self, maxtemp=300, maxlock=600):
"""Purge the queue:
* delete unused intermediate directories
* delete too old temporary directories
* unlock too old locked directories
Arguments:
maxtemp - maximum time for a temporary element. If 0, temporary
elements will not be removed.
maxlock - maximum time for a locked element. If 0, locked
elements will not be unlocked.
Raise:
OSError - problem deleting element from disk
Note:
this uses first()/next() to iterate so this will reset the cursor
"""
# get the list of intermediate directories
_list = []
for name in _directory_contents(self.path):
if _DIRECTORY_REGEXP.match(name):
_list.append(name)
_list.sort()
# try to purge all but last one
if len(_list) > 1:
_list.pop()
for name in _list:
path = "%s/%s" % (self.path, name)
if _subdirs_num(path):
continue
_special_rmdir(path)
# remove the volatile directories which are too old
if maxtemp:
oldtime = time.time() - maxtemp
for name in self._volatile():
path = "%s/%s" % (self.path, name)
if _older(path, oldtime):
_warn("* removing too old volatile element: %s" % name)
for file_name in _directory_contents(path, True):
if file_name == LOCKED_DIRECTORY:
continue
fpath = "%s/%s" % (path, file_name)
try:
os.unlink(fpath)
except Exception:
error = sys.exc_info()[1]
if error.errno != errno.ENOENT:
raise OSError("cannot unlink(%s): %s" % (fpath, error))
_special_rmdir("%s/%s" % (path, LOCKED_DIRECTORY))
_special_rmdir(path)
# iterate to find abandoned locked entries
if maxlock:
oldtime = time.time() - maxlock
name = self.first()
while name:
if self._is_locked(name, oldtime):
_warn("* removing too old locked element: %s" % name)
self.unlock(name, True)
name = self.next()
def purge(self, maxtemp=300, maxlock=600):
"""Purge the queue:
* delete unused intermediate directories
* delete too old temporary directories
* unlock too old locked directories
Arguments:
maxtemp - maximum time for a temporary element. If 0, temporary
elements will not be removed.
maxlock - maximum time for a locked element. If 0, locked
elements will not be unlocked.
Raise:
OSError - problem deleting element from disk
Note:
this uses first()/next() to iterate so this will reset the cursor
"""
# get the list of intermediate directories
_list = []
for name in _directory_contents(self.path):
if _DirectoryRegexp.match(name):
_list.append(name)
_list.sort()
# try to purge all but last one
if len(_list) > 1:
_list.pop()
for name in _list:
path = '%s/%s' % (self.path, name)
if _subdirs_num(path):
continue
_special_rmdir(path)
# remove the volatile directories which are too old
if maxtemp:
oldtime = time.time() - maxtemp
for name in self._volatile():
path = '%s/%s' % (self.path, name)
if _older(path, oldtime):
_warn("* removing too old volatile element: %s" % name)
for file_name in _directory_contents(path, True):
if file_name == LOCKED_DIRECTORY:
continue
fpath = '%s/%s' % (path, file_name)
try:
os.unlink(fpath)
except Exception:
error = sys.exc_info()[1]
if error.errno != errno.ENOENT:
raise OSError("cannot unlink(%s): %s" %
(fpath, error))
_special_rmdir('%s/%s' % (path, LOCKED_DIRECTORY))
_special_rmdir(path)
# iterate to find abandoned locked entries
if maxlock:
oldtime = time.time() - maxlock
name = self.first()
while name:
if self._is_locked(name, oldtime):
_warn("* removing too old locked element: %s" % name)
self.unlock(name, True)
name = self.next()
