def add(self, data):
"""Add a new element to the queue and return its name.
Arguments:
data - element as a dictionary (should conform to the schema)
Raise:
QueueError - problem with schema definition or data
OSError - problem putting element on disk
Note:
the destination directory must _not_ be created beforehand as
it would be seen as a valid (but empty) element directory by
another process, we therefore use rename() from a temporary
directory
"""
if not self.type:
raise QueueError("unknown schema")
while True:
temp = "%s/%s/%s" % (self.path, TEMPORARY_DIRECTORY, _name(self.rndhex))
if _special_mkdir(temp, self.umask):
break
for name in data.keys():
if name not in self.type:
raise QueueError("unexpected data: %s" % name)
if self.type[name] == "binary":
if type(data[name]) not in VALID_STR_TYPES:
raise QueueError("unexpected binary data in %s: %r" % (name, data[name]))
_file_write("%s/%s" % (temp, name), 0, self.umask, data[name])
elif self.type[name] == "string":
if type(data[name]) not in VALID_STR_TYPES:
raise QueueError("unexpected string data in %s: %r" % (name, data[name]))
_file_write("%s/%s" % (temp, name), 1, self.umask, data[name])
elif self.type[name] == "table":
if not isinstance(data[name], dict):
raise QueueError("unexpected table data in %s: %r" % (name, data[name]))
_file_write("%s/%s" % (temp, name), 1, self.umask, _hash2string(data[name]))
else:
raise QueueError("unexpected data type in %s: %r" % (name, self.type[name]))
for name in self.mandatory.keys():
if name not in data:
raise QueueError("missing mandatory data: %s" % name)
while True:
name = "%s/%s" % (self._insertion_directory(), _name(self.rndhex))
path = "%s/%s" % (self.path, name)
try:
os.rename(temp, path)
return name
except Exception:
error = sys.exc_info()[1]
if error.errno != errno.ENOTEMPTY and error.errno != errno.EEXIST:
raise OSError("cannot rename(%s, %s): %s" % (temp, path, error))
def _add_data(self, data):
"""Write 'data' to a file.
Return: (tuple) directory name where the file was written, full path to
the temporary file.
"""
_dir = self._add_dir()
while 1:
tmp = '%s/%s/%s%s' % (self.path, _dir, _name(self.rndhex),
TEMPORARY_SUFFIX)
try:
if is_bytes(data):
new_file = _file_create(tmp, umask=self.umask, utf8=False)
else:
new_file = _file_create(tmp, umask=self.umask, utf8=True)
except EnvironmentError:
error = sys.exc_info()[1]
if error.errno == errno.ENOENT:
_special_mkdir('%s/%s' % (self.path, _dir), self.umask)
continue
else:
if new_file:
break
new_file.write(data)
new_file.close()
return _dir, tmp
def remove(self, ename):
"""Remove locked element from the queue.
Arguments:
ename - name of an element
Raise:
QueueError - invalid element name; element not locked;
unexpected file in the element directory
OSError - can't rename/remove a file/directory
Note:
doesn't return anything explicitly (i.e. returns NoneType)
or fails
"""
_check_element(ename)
if not self._is_locked(ename):
raise QueueError("cannot remove %s: not locked" % ename)
# move the element out of its intermediate directory
path = "%s/%s" % (self.path, ename)
while True:
temp = "%s/%s/%s" % (self.path, OBSOLETE_DIRECTORY, _name(self.rndhex))
try:
os.rename(path, temp)
break
except Exception:
error = sys.exc_info()[1]
if error.errno != errno.ENOTEMPTY and error.errno != errno.EEXIST:
raise OSError("cannot rename(%s, %s): %s" % (ename, temp, error))
# RACE: the target directory was already present...
# remove the data files
for name in _directory_contents(temp):
if name == LOCKED_DIRECTORY:
continue
if not _FILE_REGEXP.match(name):
raise QueueError("unexpected file in %s: %s" % (temp, name))
path = "%s/%s" % (temp, name)
try:
os.unlink(path)
except Exception:
error = sys.exc_info()[1]
raise OSError("cannot unlink(%s): %s" % (path, error))
# remove the locked directory
path = "%s/%s" % (temp, LOCKED_DIRECTORY)
while True:
try:
os.rmdir(path)
except Exception:
error = sys.exc_info()[1]
raise OSError("cannot rmdir(%s): %s" % (path, error))
try:
os.rmdir(temp)
return
except Exception:
error = sys.exc_info()[1]
if error.errno != errno.ENOTEMPTY and error.errno != errno.EEXIST:
raise OSError("cannot rmdir(%s): %s" % (temp, error))
def _add_path(self, tmp, _dir):
"""Given temporary file and directory where it resides: create a hard
link to that file and remove initial one.
Return: element name (<directory name>/<file name>).
"""
while 1:
name = _name(self.rndhex)
new = '%s/%s/%s' % (self.path, _dir, name)
try:
os.link(tmp, new)
except OSError:
error = sys.exc_info()[1]
if error.errno != errno.EEXIST:
raise error
else:
continue
os.unlink(tmp)
return '%s/%s' % (_dir, name)
def add(self, data):
"""Add a new element to the queue and return its name.
Arguments:
data - element as a dictionary (should conform to the schema)
Raise:
QueueError - problem with schema definition or data
OSError - problem putting element on disk
Note:
the destination directory must _not_ be created beforehand as
it would be seen as a valid (but empty) element directory by
another process, we therefore use rename() from a temporary
directory
"""
if not self.type:
raise QueueError("unknown schema")
while True:
temp = '%s/%s/%s' % (self.path, TEMPORARY_DIRECTORY,
_name(self.rndhex))
if _special_mkdir(temp, self.umask):
break
for name in data.keys():
if name not in self.type:
raise QueueError("unexpected data: %s" % name)
if self.type[name] == 'binary':
if type(data[name]) not in VALID_STR_TYPES:
raise QueueError("unexpected binary data in %s: %r" %
(name, data[name]))
_file_write('%s/%s' % (temp, name), 0, self.umask, data[name])
elif self.type[name] == 'string':
if type(data[name]) not in VALID_STR_TYPES:
raise QueueError("unexpected string data in %s: %r" %
(name, data[name]))
_file_write('%s/%s' % (temp, name), 1, self.umask, data[name])
elif self.type[name] == 'table':
if not isinstance(data[name], dict):
raise QueueError("unexpected table data in %s: %r" %
(name, data[name]))
_file_write('%s/%s' % (temp, name), 1, self.umask,
_hash2string(data[name]))
else:
raise QueueError("unexpected data type in %s: %r" %
(name, self.type[name]))
for name in self.mandatory.keys():
if name not in data:
raise QueueError("missing mandatory data: %s" % name)
while True:
name = '%s/%s' % (self._insertion_directory(), _name(self.rndhex))
path = '%s/%s' % (self.path, name)
try:
os.rename(temp, path)
return name
except Exception:
error = sys.exc_info()[1]
if error.errno != errno.ENOTEMPTY and \
error.errno != errno.EEXIST:
raise OSError("cannot rename(%s, %s): %s" %
(temp, path, error))
def remove(self, ename):
"""Remove locked element from the queue.
Arguments:
ename - name of an element
Raise:
QueueError - invalid element name; element not locked;
unexpected file in the element directory
OSError - can't rename/remove a file/directory
Note:
doesn't return anything explicitly (i.e. returns NoneType)
or fails
"""
_check_element(ename)
if not self._is_locked(ename):
raise QueueError("cannot remove %s: not locked" % ename)
# move the element out of its intermediate directory
path = '%s/%s' % (self.path, ename)
while True:
temp = '%s/%s/%s' % (self.path, OBSOLETE_DIRECTORY,
_name(self.rndhex))
try:
os.rename(path, temp)
break
except Exception:
error = sys.exc_info()[1]
if error.errno != errno.ENOTEMPTY and \
error.errno != errno.EEXIST:
raise OSError("cannot rename(%s, %s): %s" %
(ename, temp, error))
# RACE: the target directory was already present...
# remove the data files
for name in _directory_contents(temp):
if name == LOCKED_DIRECTORY:
continue
if not _FileRegexp.match(name):
raise QueueError("unexpected file in %s: %s" % (temp, name))
path = '%s/%s' % (temp, name)
try:
os.unlink(path)
except Exception:
error = sys.exc_info()[1]
raise OSError("cannot unlink(%s): %s" % (path, error))
# remove the locked directory
path = '%s/%s' % (temp, LOCKED_DIRECTORY)
while True:
try:
os.rmdir(path)
except Exception:
error = sys.exc_info()[1]
raise OSError("cannot rmdir(%s): %s" % (path, error))
try:
os.rmdir(temp)
return
except Exception:
error = sys.exc_info()[1]
if error.errno != errno.ENOTEMPTY and \
error.errno != errno.EEXIST:
raise OSError("cannot rmdir(%s): %s" % (temp, error))