def __init__(self,
path,
umask=None,
rndhex=None,
maxelts=16000,
schema=dict()):
"""Check and set schema. Build the queue directory structure.
Arguments:
path
the queue toplevel directory
umask
the umask to use when creating files and directories
(default: use the running process' umask)
rndhex
the hexadecimal digit to use in names
(default: randomly chosen)
maxelts
the maximum number of elements that an intermediate
directory can hold (default: 16,000)
schema
the schema defining how to interpret user supplied
data (mandatory if elements are added or read)
Raise:
TypeError - wrong input data types provided
QueueError - problems with the queue schema definition
OSError - can't create directory structure
"""
super(Queue, self).__init__(path, umask=umask, rndhex=rndhex)
if type(maxelts) in VALID_INT_TYPES:
self.maxelts = maxelts
else:
raise TypeError("'maxelts' should be int or long")
# check schema
self.type = {}
self.mandatory = {}
if schema:
if not isinstance(schema, dict):
raise QueueError("invalid schema: %r" % schema)
for name in schema.keys():
if not _FileRegexp.match(name):
raise QueueError("invalid schema name: %r" % name)
if not isinstance(schema[name], str):
raise QueueError("invalid data type for schema " +
"specification: %r" % type(schema[name]))
match = re.match('(binary|string|table)([\\?\\*]{0,2})?$',
schema[name])
if not match:
raise QueueError("invalid schema data type: %r" %
schema[name])
self.type[name] = match.group(1)
if not re.search('\\?', match.group(2)):
self.mandatory[name] = True
if not self.mandatory:
raise QueueError("invalid schema: no mandatory data")
# create directories
for directory in (TEMPORARY_DIRECTORY, OBSOLETE_DIRECTORY):
_special_mkdir('%s/%s' % (self.path, directory), self.umask)
def _add(self, *queues):
"""Add lists of queues to existing ones. Copies of the object
instances are used.
Arguments:
*queues - add([q1,..]/(q1,..)) or add(q1,..)
Raise:
QueueError - queue already in the set
TypeError - wrong queue object type provided
"""
type_queue = False
for queue in queues:
if type(queue) in [list, tuple] and not type_queue:
for _queue in queue:
if isinstance(_queue, QueueBase):
if _queue.id in [x.id for x in self.qset]:
raise QueueError("queue already in the set: %s" %
_queue.path)
self.qset.append(_queue.copy())
else:
raise TypeError("QueueBase objects expected.")
break
elif isinstance(queue, QueueBase):
type_queue = True
self.qset.append(queue.copy())
else:
raise TypeError("expected QueueBase object(s) or list/tuple "
"of QueueBase objects")
def _check_element(name):
"""Check the given string to make sure it represents a valid element name.
Raise:
QueueError - given element is invalid
"""
if not _DirElemRegexp.match(name):
raise QueueError("invalid element name: %s" % name)
def get(self, ename):
"""Get an element data from a locked element.
Arguments:
ename - name of an element
Return:
dictionary representing an element
Raise:
QueueError - schema is unknown; unexpected data type in
the schema specification; missing mandatory
file of the element
OSError - problems opening/closing file
IOError - file read error
"""
if not self.type:
raise QueueError("unknown schema")
_check_element(ename)
if not self._is_locked(ename):
raise QueueError("cannot get %s: not locked" % ename)
data = {}
for dname in self.type.keys():
path = '%s/%s/%s' % (self.path, ename, dname)
try:
os.lstat(path)
except Exception:
error = sys.exc_info()[1]
if error.errno != errno.ENOENT:
raise OSError("cannot lstat(%s): %s" % (path, error))
if dname in self.mandatory:
raise QueueError("missing data file: %s" % path)
else:
continue
if self.type[dname] == 'binary':
data[dname] = _file_read(path, 0)
elif self.type[dname] == 'string':
data[dname] = _file_read(path, 1)
elif self.type[dname] == 'table':
data[dname] = _string2hash(_file_read(path, 1))
else:
raise QueueError("unexpected data type: %s" % self.type[dname])
return data
def _hash2string(data):
"""Transform a hash of strings into a string.
Raise:
QueueError - invalid type of a value in hash
(allowed string or unicode)
Note:
the keys are sorted so that identical hashes yield to identical strings
"""
string = ''
for key in sorted(data.keys()):
val = data[key]
if type(val) not in VALID_STR_TYPES:
raise QueueError("invalid hash value type: %r" % val)
key = _H2SRegexp.sub(lambda m: _Byte2Esc[m.group(1)], key)
val = _H2SRegexp.sub(lambda m: _Byte2Esc[m.group(1)], val)
string = '%s%s' % (string, '%s\x09%s\x0a' % (key, val))
return string
def _string2hash(given):
"""Transform a string into a hash of strings.
Raise:
QueueError - unexpected hash line
Note:
duplicate keys are not checked (the last one wins)
"""
_hash = dict()
if not given:
return _hash
for line in given.strip('\n').split('\x0a'):
match = _KeyValRegexp.match(line)
if not match:
raise QueueError("unexpected hash line: %s" % line)
key = _S2HRegexp.sub(lambda m: _Esc2Byte[str(m.group(1))],
match.group(1))
val = _S2HRegexp.sub(lambda m: _Esc2Byte[str(m.group(1))],
match.group(2))
_hash[key] = val
return _hash
def add(self, data):
"""Add a new element to the queue and return its name.
Arguments:
data - element as a dictionary (should conform to the schema)
Raise:
QueueError - problem with schema definition or data
OSError - problem putting element on disk
Note:
the destination directory must _not_ be created beforehand as
it would be seen as a valid (but empty) element directory by
another process, we therefore use rename() from a temporary
directory
"""
if not self.type:
raise QueueError("unknown schema")
while True:
temp = '%s/%s/%s' % (self.path, TEMPORARY_DIRECTORY,
_name(self.rndhex))
if _special_mkdir(temp, self.umask):
break
for name in data.keys():
if name not in self.type:
raise QueueError("unexpected data: %s" % name)
if self.type[name] == 'binary':
if type(data[name]) not in VALID_STR_TYPES:
raise QueueError("unexpected binary data in %s: %r" %
(name, data[name]))
_file_write('%s/%s' % (temp, name), 0, self.umask, data[name])
elif self.type[name] == 'string':
if type(data[name]) not in VALID_STR_TYPES:
raise QueueError("unexpected string data in %s: %r" %
(name, data[name]))
_file_write('%s/%s' % (temp, name), 1, self.umask, data[name])
elif self.type[name] == 'table':
if not isinstance(data[name], dict):
raise QueueError("unexpected table data in %s: %r" %
(name, data[name]))
_file_write('%s/%s' % (temp, name), 1, self.umask,
_hash2string(data[name]))
else:
raise QueueError("unexpected data type in %s: %r" %
(name, self.type[name]))
for name in self.mandatory.keys():
if name not in data:
raise QueueError("missing mandatory data: %s" % name)
while True:
name = '%s/%s' % (self._insertion_directory(), _name(self.rndhex))
path = '%s/%s' % (self.path, name)
try:
os.rename(temp, path)
return name
except Exception:
error = sys.exc_info()[1]
if error.errno != errno.ENOTEMPTY and \
error.errno != errno.EEXIST:
raise OSError("cannot rename(%s, %s): %s" %
(temp, path, error))
def remove(self, ename):
"""Remove locked element from the queue.
Arguments:
ename - name of an element
Raise:
QueueError - invalid element name; element not locked;
unexpected file in the element directory
OSError - can't rename/remove a file/directory
Note:
doesn't return anything explicitly (i.e. returns NoneType)
or fails
"""
_check_element(ename)
if not self._is_locked(ename):
raise QueueError("cannot remove %s: not locked" % ename)
# move the element out of its intermediate directory
path = '%s/%s' % (self.path, ename)
while True:
temp = '%s/%s/%s' % (self.path, OBSOLETE_DIRECTORY,
_name(self.rndhex))
try:
os.rename(path, temp)
break
except Exception:
error = sys.exc_info()[1]
if error.errno != errno.ENOTEMPTY and \
error.errno != errno.EEXIST:
raise OSError("cannot rename(%s, %s): %s" %
(ename, temp, error))
# RACE: the target directory was already present...
# remove the data files
for name in _directory_contents(temp):
if name == LOCKED_DIRECTORY:
continue
if not _FileRegexp.match(name):
raise QueueError("unexpected file in %s: %s" % (temp, name))
path = '%s/%s' % (temp, name)
try:
os.unlink(path)
except Exception:
error = sys.exc_info()[1]
raise OSError("cannot unlink(%s): %s" % (path, error))
# remove the locked directory
path = '%s/%s' % (temp, LOCKED_DIRECTORY)
while True:
try:
os.rmdir(path)
except Exception:
error = sys.exc_info()[1]
raise OSError("cannot rmdir(%s): %s" % (path, error))
try:
os.rmdir(temp)
return
except Exception:
error = sys.exc_info()[1]
if error.errno != errno.ENOTEMPTY and \
error.errno != errno.EEXIST:
raise OSError("cannot rmdir(%s): %s" % (temp, error))