def purge(self, maxtemp=300, maxlock=600):
"""Purge the queue:
* delete unused intermediate directories
* delete too old temporary directories
* unlock too old locked directories
Arguments:
maxtemp - maximum time for a temporary element. If 0, temporary
elements will not be removed.
maxlock - maximum time for a locked element. If 0, locked
elements will not be unlocked.
Raise:
OSError - problem deleting element from disk
Note:
this uses first()/next() to iterate so this will reset the cursor
"""
# get the list of intermediate directories
_list = []
for name in _directory_contents(self.path):
if _DIRECTORY_REGEXP.match(name):
_list.append(name)
_list.sort()
# try to purge all but last one
if len(_list) > 1:
_list.pop()
for name in _list:
path = "%s/%s" % (self.path, name)
if _subdirs_num(path):
continue
_special_rmdir(path)
# remove the volatile directories which are too old
if maxtemp:
oldtime = time.time() - maxtemp
for name in self._volatile():
path = "%s/%s" % (self.path, name)
if _older(path, oldtime):
_warn("* removing too old volatile element: %s" % name)
for file_name in _directory_contents(path, True):
if file_name == LOCKED_DIRECTORY:
continue
fpath = "%s/%s" % (path, file_name)
try:
os.unlink(fpath)
except Exception:
error = sys.exc_info()[1]
if error.errno != errno.ENOENT:
raise OSError("cannot unlink(%s): %s" % (fpath, error))
_special_rmdir("%s/%s" % (path, LOCKED_DIRECTORY))
_special_rmdir(path)
# iterate to find abandoned locked entries
if maxlock:
oldtime = time.time() - maxlock
name = self.first()
while name:
if self._is_locked(name, oldtime):
_warn("* removing too old locked element: %s" % name)
self.unlock(name, True)
name = self.next()
def _insertion_directory(self):
"""Return the name of the intermediate directory that can be used for
insertion:
* if there is none, an initial one will be created
* if it is full, a new one will be created
* in any case the name will match $_DIRECTORY_REGEXP
Raise:
OSError - can't list/make element directories
"""
_list = []
# get the list of existing directories
for name in _directory_contents(self.path):
if _DIRECTORY_REGEXP.match(name):
_list.append(name)
# handle the case with no directories yet
if not _list:
name = "%08x" % 0
_special_mkdir("%s/%s" % (self.path, name), self.umask)
return name
# check the last directory
_list.sort()
name = _list[-1]
subdirs = _subdirs_num("%s/%s" % (self.path, name))
if subdirs:
if subdirs < self.maxelts:
return name
else:
# RACE: at this point, the directory does not exist anymore,
# so it must have been purged after we listed the directory
# contents. We do not try to do more and simply create a new
# directory
pass
# we need a new directory
name = "%08x" % (int(name, 16) + 1)
_special_mkdir("%s/%s" % (self.path, name), self.umask)
return name
def _insertion_directory(self):
"""Return the name of the intermediate directory that can be used for
insertion:
* if there is none, an initial one will be created
* if it is full, a new one will be created
* in any case the name will match $_DIRECTORY_REGEXP
Raise:
OSError - can't list/make element directories
"""
_list = []
# get the list of existing directories
for name in _directory_contents(self.path):
if _DIRECTORY_REGEXP.match(name):
_list.append(name)
# handle the case with no directories yet
if not _list:
name = '%08x' % 0
_special_mkdir('%s/%s' % (self.path, name), self.umask)
return name
# check the last directory
_list.sort()
name = _list[-1]
subdirs = _subdirs_num('%s/%s' % (self.path, name))
if subdirs:
if subdirs < self.maxelts:
return name
else:
# RACE: at this point, the directory does not exist anymore,
# so it must have been purged after we listed the directory
# contents. We do not try to do more and simply create a new
# directory
pass
# we need a new directory
name = '%08x' % (int(name, 16) + 1)
_special_mkdir('%s/%s' % (self.path, name), self.umask)
return name
def purge(self, maxtemp=300, maxlock=600):
"""Purge the queue:
* delete unused intermediate directories
* delete too old temporary directories
* unlock too old locked directories
Arguments:
maxtemp - maximum time for a temporary element. If 0, temporary
elements will not be removed.
maxlock - maximum time for a locked element. If 0, locked
elements will not be unlocked.
Raise:
OSError - problem deleting element from disk
Note:
this uses first()/next() to iterate so this will reset the cursor
"""
# get the list of intermediate directories
_list = []
for name in _directory_contents(self.path):
if _DIRECTORY_REGEXP.match(name):
_list.append(name)
_list.sort()
# try to purge all but last one
if len(_list) > 1:
_list.pop()
for name in _list:
path = '%s/%s' % (self.path, name)
if _subdirs_num(path):
continue
_special_rmdir(path)
# remove the volatile directories which are too old
if maxtemp:
oldtime = time.time() - maxtemp
for name in self._volatile():
path = '%s/%s' % (self.path, name)
if _older(path, oldtime):
_warn("* removing too old volatile element: %s" % name)
for file_name in _directory_contents(path, True):
if file_name == LOCKED_DIRECTORY:
continue
fpath = '%s/%s' % (path, file_name)
try:
os.unlink(fpath)
except Exception:
error = sys.exc_info()[1]
if error.errno != errno.ENOENT:
raise OSError("cannot unlink(%s): %s" %
(fpath, error))
_special_rmdir('%s/%s' % (path, LOCKED_DIRECTORY))
_special_rmdir(path)
# iterate to find abandoned locked entries
if maxlock:
oldtime = time.time() - maxlock
name = self.first()
while name:
if self._is_locked(name, oldtime):
_warn("* removing too old locked element: %s" % name)
self.unlock(name, True)
name = self.next()
def __get_list_of_interm_dirs(self, dirs):
"""Fill out provided list with names of intermediate directories.
"""
for name in os.listdir(self.path):
if _DIRECTORY_REGEXP.match(name):
dirs.append(name)
