def purge(self, maxtemp=300, maxlock=600):
"""Purge the queue:
* delete unused intermediate directories
* delete too old temporary directories
* unlock too old locked directories
Arguments:
maxtemp - maximum time for a temporary element. If 0, temporary
elements will not be removed.
maxlock - maximum time for a locked element. If 0, locked
elements will not be unlocked.
Raise:
OSError - problem deleting element from disk
Note:
this uses first()/next() to iterate so this will reset the cursor
"""
# get the list of intermediate directories
_list = []
for name in _directory_contents(self.path):
if _DIRECTORY_REGEXP.match(name):
_list.append(name)
_list.sort()
# try to purge all but last one
if len(_list) > 1:
_list.pop()
for name in _list:
path = "%s/%s" % (self.path, name)
if _subdirs_num(path):
continue
_special_rmdir(path)
# remove the volatile directories which are too old
if maxtemp:
oldtime = time.time() - maxtemp
for name in self._volatile():
path = "%s/%s" % (self.path, name)
if _older(path, oldtime):
_warn("* removing too old volatile element: %s" % name)
for file_name in _directory_contents(path, True):
if file_name == LOCKED_DIRECTORY:
continue
fpath = "%s/%s" % (path, file_name)
try:
os.unlink(fpath)
except Exception:
error = sys.exc_info()[1]
if error.errno != errno.ENOENT:
raise OSError("cannot unlink(%s): %s" % (fpath, error))
_special_rmdir("%s/%s" % (path, LOCKED_DIRECTORY))
_special_rmdir(path)
# iterate to find abandoned locked entries
if maxlock:
oldtime = time.time() - maxlock
name = self.first()
while name:
if self._is_locked(name, oldtime):
_warn("* removing too old locked element: %s" % name)
self.unlock(name, True)
name = self.next()
def _volatile(self):
"""
Return the list of volatile (i.e. temporary or obsolete) directories.
"""
_list = []
for name in _directory_contents("%s/%s" % (self.path, TEMPORARY_DIRECTORY), True):
if _ELEMENT_REGEXP.match(name):
_list.append("%s/%s" % (TEMPORARY_DIRECTORY, name))
for name in _directory_contents("%s/%s" % (self.path, OBSOLETE_DIRECTORY), True):
if _ELEMENT_REGEXP.match(name):
_list.append("%s/%s" % (OBSOLETE_DIRECTORY, name))
return _list
def _volatile(self):
"""
Return the list of volatile (i.e. temporary or obsolete) directories.
"""
_list = []
for name in _directory_contents(
'%s/%s' % (self.path, TEMPORARY_DIRECTORY), True):
if _ElementRegexp.match(name):
_list.append('%s/%s' % (TEMPORARY_DIRECTORY, name))
for name in _directory_contents(
'%s/%s' % (self.path, OBSOLETE_DIRECTORY), True):
if _ElementRegexp.match(name):
_list.append('%s/%s' % (OBSOLETE_DIRECTORY, name))
return _list
def __subdirs_num_count(path):
"""Count the number of sub-directories in the given directory:
- return 0 if the directory does not exist (anymore)
For systems where we cannot rely on the number of links, so we simply count
the number of sub-directories.
"""
return len(_directory_contents(path, missingok=True))
def __subdirs_num_count(path):
"""Count the number of sub-directories in the given directory:
- return 0 if the directory does not exist (anymore)
For systems where we cannot rely on the number of links, so we simply count
the number of sub-directories.
"""
return len(_directory_contents(path, missingok=True))
def remove(self, ename):
"""Remove locked element from the queue.
Arguments:
ename - name of an element
Raise:
QueueError - invalid element name; element not locked;
unexpected file in the element directory
OSError - can't rename/remove a file/directory
Note:
doesn't return anything explicitly (i.e. returns NoneType)
or fails
"""
_check_element(ename)
if not self._is_locked(ename):
raise QueueError("cannot remove %s: not locked" % ename)
# move the element out of its intermediate directory
path = "%s/%s" % (self.path, ename)
while True:
temp = "%s/%s/%s" % (self.path, OBSOLETE_DIRECTORY, _name(self.rndhex))
try:
os.rename(path, temp)
break
except Exception:
error = sys.exc_info()[1]
if error.errno != errno.ENOTEMPTY and error.errno != errno.EEXIST:
raise OSError("cannot rename(%s, %s): %s" % (ename, temp, error))
# RACE: the target directory was already present...
# remove the data files
for name in _directory_contents(temp):
if name == LOCKED_DIRECTORY:
continue
if not _FILE_REGEXP.match(name):
raise QueueError("unexpected file in %s: %s" % (temp, name))
path = "%s/%s" % (temp, name)
try:
os.unlink(path)
except Exception:
error = sys.exc_info()[1]
raise OSError("cannot unlink(%s): %s" % (path, error))
# remove the locked directory
path = "%s/%s" % (temp, LOCKED_DIRECTORY)
while True:
try:
os.rmdir(path)
except Exception:
error = sys.exc_info()[1]
raise OSError("cannot rmdir(%s): %s" % (path, error))
try:
os.rmdir(temp)
return
except Exception:
error = sys.exc_info()[1]
if error.errno != errno.ENOTEMPTY and error.errno != errno.EEXIST:
raise OSError("cannot rmdir(%s): %s" % (temp, error))
def _count(path):
"""Return the number of elements in the queue, regardless of
their state.
Raise:
OSError - can't list/stat element directories
"""
count = 0
for name in [x for x in _directory_contents(path)]:
subdirs = _subdirs_num("%s/%s" % (path, name))
if subdirs:
count += subdirs
return count
def _count(path):
"""Return the number of elements in the queue, regardless of
their state.
Raise:
OSError - can't list/stat element directories
"""
count = 0
for name in [x for x in _directory_contents(path)]:
subdirs = _subdirs_num('%s/%s' % (path, name))
if subdirs:
count += subdirs
return count
def _build_elements(self):
"""Build list of elements.
Raise:
OSError - can't list element directories
"""
while self.dirs:
directory = self.dirs.pop(0)
_list = []
for name in _directory_contents("%s/%s" % (self.path, directory), True):
if _ELEMENT_REGEXP.match(name):
_list.append(name)
if not _list:
continue
self.elts = ["%s/%s" % (directory, x) for x in sorted(_list)]
return True
return False
def _build_elements(self):
"""Build list of elements.
Raise:
OSError - can't list element directories
"""
while self.dirs:
directory = self.dirs.pop(0)
_list = []
for name in _directory_contents('%s/%s' % (self.path, directory),
True):
if _ElementRegexp.match(name):
_list.append(name)
if not _list:
continue
self.elts = ['%s/%s' % (directory, x) for x in sorted(_list)]
return True
return False
def _insertion_directory(self):
"""Return the name of the intermediate directory that can be used for
insertion:
* if there is none, an initial one will be created
* if it is full, a new one will be created
* in any case the name will match $_DIRECTORY_REGEXP
Raise:
OSError - can't list/make element directories
"""
_list = []
# get the list of existing directories
for name in _directory_contents(self.path):
if _DIRECTORY_REGEXP.match(name):
_list.append(name)
# handle the case with no directories yet
if not _list:
name = "%08x" % 0
_special_mkdir("%s/%s" % (self.path, name), self.umask)
return name
# check the last directory
_list.sort()
name = _list[-1]
subdirs = _subdirs_num("%s/%s" % (self.path, name))
if subdirs:
if subdirs < self.maxelts:
return name
else:
# RACE: at this point, the directory does not exist anymore,
# so it must have been purged after we listed the directory
# contents. We do not try to do more and simply create a new
# directory
pass
# we need a new directory
name = "%08x" % (int(name, 16) + 1)
_special_mkdir("%s/%s" % (self.path, name), self.umask)
return name
def _insertion_directory(self):
"""Return the name of the intermediate directory that can be used for
insertion:
* if there is none, an initial one will be created
* if it is full, a new one will be created
* in any case the name will match $_DirectoryRegexp
Raise:
OSError - can't list/make element directories
"""
_list = []
# get the list of existing directories
for name in _directory_contents(self.path):
if _DirectoryRegexp.match(name):
_list.append(name)
# handle the case with no directories yet
if not _list:
name = '%08x' % 0
_special_mkdir('%s/%s' % (self.path, name), self.umask)
return name
# check the last directory
_list.sort()
name = _list[-1]
subdirs = _subdirs_num('%s/%s' % (self.path, name))
if subdirs:
if subdirs < self.maxelts:
return name
else:
# RACE: at this point, the directory does not exist anymore,
# so it must have been purged after we listed the directory
# contents. We do not try to do more and simply create a new
# directory
pass
# we need a new directory
name = '%08x' % (int(name, 16) + 1)
_special_mkdir('%s/%s' % (self.path, name), self.umask)
return name
def purge(self, maxtemp=300, maxlock=600):
"""Purge the queue:
* delete unused intermediate directories
* delete too old temporary directories
* unlock too old locked directories
Arguments:
maxtemp - maximum time for a temporary element. If 0, temporary
elements will not be removed.
maxlock - maximum time for a locked element. If 0, locked
elements will not be unlocked.
Raise:
OSError - problem deleting element from disk
Note:
this uses first()/next() to iterate so this will reset the cursor
"""
# get the list of intermediate directories
_list = []
for name in _directory_contents(self.path):
if _DirectoryRegexp.match(name):
_list.append(name)
_list.sort()
# try to purge all but last one
if len(_list) > 1:
_list.pop()
for name in _list:
path = '%s/%s' % (self.path, name)
if _subdirs_num(path):
continue
_special_rmdir(path)
# remove the volatile directories which are too old
if maxtemp:
oldtime = time.time() - maxtemp
for name in self._volatile():
path = '%s/%s' % (self.path, name)
if _older(path, oldtime):
_warn("* removing too old volatile element: %s" % name)
for file_name in _directory_contents(path, True):
if file_name == LOCKED_DIRECTORY:
continue
fpath = '%s/%s' % (path, file_name)
try:
os.unlink(fpath)
except Exception:
error = sys.exc_info()[1]
if error.errno != errno.ENOENT:
raise OSError("cannot unlink(%s): %s" %
(fpath, error))
_special_rmdir('%s/%s' % (path, LOCKED_DIRECTORY))
_special_rmdir(path)
# iterate to find abandoned locked entries
if maxlock:
oldtime = time.time() - maxlock
name = self.first()
while name:
if self._is_locked(name, oldtime):
_warn("* removing too old locked element: %s" % name)
self.unlock(name, True)
name = self.next()
def remove(self, ename):
"""Remove locked element from the queue.
Arguments:
ename - name of an element
Raise:
QueueError - invalid element name; element not locked;
unexpected file in the element directory
OSError - can't rename/remove a file/directory
Note:
doesn't return anything explicitly (i.e. returns NoneType)
or fails
"""
_check_element(ename)
if not self._is_locked(ename):
raise QueueError("cannot remove %s: not locked" % ename)
# move the element out of its intermediate directory
path = '%s/%s' % (self.path, ename)
while True:
temp = '%s/%s/%s' % (self.path, OBSOLETE_DIRECTORY,
_name(self.rndhex))
try:
os.rename(path, temp)
break
except Exception:
error = sys.exc_info()[1]
if error.errno != errno.ENOTEMPTY and \
error.errno != errno.EEXIST:
raise OSError("cannot rename(%s, %s): %s" %
(ename, temp, error))
# RACE: the target directory was already present...
# remove the data files
for name in _directory_contents(temp):
if name == LOCKED_DIRECTORY:
continue
if not _FileRegexp.match(name):
raise QueueError("unexpected file in %s: %s" % (temp, name))
path = '%s/%s' % (temp, name)
try:
os.unlink(path)
except Exception:
error = sys.exc_info()[1]
raise OSError("cannot unlink(%s): %s" % (path, error))
# remove the locked directory
path = '%s/%s' % (temp, LOCKED_DIRECTORY)
while True:
try:
os.rmdir(path)
except Exception:
error = sys.exc_info()[1]
raise OSError("cannot rmdir(%s): %s" % (path, error))
try:
os.rmdir(temp)
return
except Exception:
error = sys.exc_info()[1]
if error.errno != errno.ENOTEMPTY and \
error.errno != errno.EEXIST:
raise OSError("cannot rmdir(%s): %s" % (temp, error))
