def _add_data(self, data):
"""Write 'data' to a file.
Return: (tuple) directory name where the file was written, full path to
the temporary file.
"""
_dir = self._add_dir()
while 1:
tmp = '%s/%s/%s%s' % (self.path, _dir, _name(self.rndhex),
TEMPORARY_SUFFIX)
try:
if is_bytes(data):
new_file = _file_create(tmp, umask=self.umask, utf8=False)
else:
new_file = _file_create(tmp, umask=self.umask, utf8=True)
except EnvironmentError:
error = sys.exc_info()[1]
if error.errno == errno.ENOENT:
_special_mkdir('%s/%s' % (self.path, _dir), self.umask)
continue
else:
if new_file:
break
new_file.write(data)
new_file.close()
return _dir, tmp
def __init__(self,
path,
umask=None,
rndhex=None,
maxelts=16000,
schema=dict()):
"""Check and set schema. Build the queue directory structure.
Arguments:
path
the queue toplevel directory
umask
the umask to use when creating files and directories
(default: use the running process' umask)
rndhex
the hexadecimal digit to use in names
(default: randomly chosen)
maxelts
the maximum number of elements that an intermediate
directory can hold (default: 16,000)
schema
the schema defining how to interpret user supplied
data (mandatory if elements are added or read)
Raise:
TypeError - wrong input data types provided
QueueError - problems with the queue schema definition
OSError - can't create directory structure
"""
super(Queue, self).__init__(path, umask=umask, rndhex=rndhex)
if type(maxelts) in VALID_INT_TYPES:
self.maxelts = maxelts
else:
raise TypeError("'maxelts' should be int or long")
# check schema
self.type = {}
self.mandatory = {}
if schema:
if not isinstance(schema, dict):
raise QueueError("invalid schema: %r" % schema)
for name in schema.keys():
if not _FileRegexp.match(name):
raise QueueError("invalid schema name: %r" % name)
if not isinstance(schema[name], str):
raise QueueError("invalid data type for schema " +
"specification: %r" % type(schema[name]))
match = re.match('(binary|string|table)([\\?\\*]{0,2})?$',
schema[name])
if not match:
raise QueueError("invalid schema data type: %r" %
schema[name])
self.type[name] = match.group(1)
if not re.search('\\?', match.group(2)):
self.mandatory[name] = True
if not self.mandatory:
raise QueueError("invalid schema: no mandatory data")
# create directories
for directory in (TEMPORARY_DIRECTORY, OBSOLETE_DIRECTORY):
_special_mkdir('%s/%s' % (self.path, directory), self.umask)
def add_path(self, path):
"""Add the given file (identified by its path) to the queue and return
the corresponding element name, the file must be on the same
filesystem and will be moved to the queue
"""
_dir = self._add_dir()
_special_mkdir('%s/%s' % (self.path, _dir), self.umask)
return self._add_path(path, _dir)
def __init__(self, path, umask=None, rndhex=None, maxelts=16000,
schema=dict()):
"""Check and set schema. Build the queue directory structure.
Arguments:
path
the queue toplevel directory
umask
the umask to use when creating files and directories
(default: use the running process' umask)
rndhex
the hexadecimal digit to use in names
(default: randomly chosen)
maxelts
the maximum number of elements that an intermediate
directory can hold (default: 16,000)
schema
the schema defining how to interpret user supplied
data (mandatory if elements are added or read)
Raise:
TypeError - wrong input data types provided
QueueError - problems with the queue schema definition
OSError - can't create directory structure
"""
super(Queue, self).__init__(path, umask=umask, rndhex=rndhex)
if type(maxelts) in VALID_INT_TYPES:
self.maxelts = maxelts
else:
raise TypeError("'maxelts' should be int or long")
# check schema
self.type = {}
self.mandatory = {}
if schema:
if not isinstance(schema, dict):
raise QueueError("invalid schema: %r" % schema)
for name in schema.keys():
if not _FileRegexp.match(name):
raise QueueError("invalid schema name: %r" % name)
if not isinstance(schema[name], str):
raise QueueError("invalid data type for schema " +
"specification: %r" % type(schema[name]))
match = re.match('(binary|string|table)([\\?\\*]{0,2})?$',
schema[name])
if not match:
raise QueueError("invalid schema data type: %r" %
schema[name])
self.type[name] = match.group(1)
if not re.search('\\?', match.group(2)):
self.mandatory[name] = True
if not self.mandatory:
raise QueueError("invalid schema: no mandatory data")
# create directories
for directory in (TEMPORARY_DIRECTORY, OBSOLETE_DIRECTORY):
_special_mkdir('%s/%s' % (self.path, directory), self.umask)
def add(self, data):
"""Add a new element to the queue and return its name.
Arguments:
data - element as a dictionary (should conform to the schema)
Raise:
QueueError - problem with schema definition or data
OSError - problem putting element on disk
Note:
the destination directory must _not_ be created beforehand as
it would be seen as a valid (but empty) element directory by
another process, we therefore use rename() from a temporary
directory
"""
if not self.type:
raise QueueError("unknown schema")
while True:
temp = "%s/%s/%s" % (self.path, TEMPORARY_DIRECTORY, _name(self.rndhex))
if _special_mkdir(temp, self.umask):
break
for name in data.keys():
if name not in self.type:
raise QueueError("unexpected data: %s" % name)
if self.type[name] == "binary":
if type(data[name]) not in VALID_STR_TYPES:
raise QueueError("unexpected binary data in %s: %r" % (name, data[name]))
_file_write("%s/%s" % (temp, name), 0, self.umask, data[name])
elif self.type[name] == "string":
if type(data[name]) not in VALID_STR_TYPES:
raise QueueError("unexpected string data in %s: %r" % (name, data[name]))
_file_write("%s/%s" % (temp, name), 1, self.umask, data[name])
elif self.type[name] == "table":
if not isinstance(data[name], dict):
raise QueueError("unexpected table data in %s: %r" % (name, data[name]))
_file_write("%s/%s" % (temp, name), 1, self.umask, _hash2string(data[name]))
else:
raise QueueError("unexpected data type in %s: %r" % (name, self.type[name]))
for name in self.mandatory.keys():
if name not in data:
raise QueueError("missing mandatory data: %s" % name)
while True:
name = "%s/%s" % (self._insertion_directory(), _name(self.rndhex))
path = "%s/%s" % (self.path, name)
try:
os.rename(temp, path)
return name
except Exception:
error = sys.exc_info()[1]
if error.errno != errno.ENOTEMPTY and error.errno != errno.EEXIST:
raise OSError("cannot rename(%s, %s): %s" % (temp, path, error))
def _insertion_directory(self):
"""Return the name of the intermediate directory that can be used for
insertion:
* if there is none, an initial one will be created
* if it is full, a new one will be created
* in any case the name will match $_DIRECTORY_REGEXP
Raise:
OSError - can't list/make element directories
"""
_list = []
# get the list of existing directories
for name in _directory_contents(self.path):
if _DIRECTORY_REGEXP.match(name):
_list.append(name)
# handle the case with no directories yet
if not _list:
name = "%08x" % 0
_special_mkdir("%s/%s" % (self.path, name), self.umask)
return name
# check the last directory
_list.sort()
name = _list[-1]
subdirs = _subdirs_num("%s/%s" % (self.path, name))
if subdirs:
if subdirs < self.maxelts:
return name
else:
# RACE: at this point, the directory does not exist anymore,
# so it must have been purged after we listed the directory
# contents. We do not try to do more and simply create a new
# directory
pass
# we need a new directory
name = "%08x" % (int(name, 16) + 1)
_special_mkdir("%s/%s" % (self.path, name), self.umask)
return name
def _insertion_directory(self):
"""Return the name of the intermediate directory that can be used for
insertion:
* if there is none, an initial one will be created
* if it is full, a new one will be created
* in any case the name will match $_DirectoryRegexp
Raise:
OSError - can't list/make element directories
"""
_list = []
# get the list of existing directories
for name in _directory_contents(self.path):
if _DirectoryRegexp.match(name):
_list.append(name)
# handle the case with no directories yet
if not _list:
name = '%08x' % 0
_special_mkdir('%s/%s' % (self.path, name), self.umask)
return name
# check the last directory
_list.sort()
name = _list[-1]
subdirs = _subdirs_num('%s/%s' % (self.path, name))
if subdirs:
if subdirs < self.maxelts:
return name
else:
# RACE: at this point, the directory does not exist anymore,
# so it must have been purged after we listed the directory
# contents. We do not try to do more and simply create a new
# directory
pass
# we need a new directory
name = '%08x' % (int(name, 16) + 1)
_special_mkdir('%s/%s' % (self.path, name), self.umask)
return name
def add(self, data):
"""Add a new element to the queue and return its name.
Arguments:
data - element as a dictionary (should conform to the schema)
Raise:
QueueError - problem with schema definition or data
OSError - problem putting element on disk
Note:
the destination directory must _not_ be created beforehand as
it would be seen as a valid (but empty) element directory by
another process, we therefore use rename() from a temporary
directory
"""
if not self.type:
raise QueueError("unknown schema")
while True:
temp = '%s/%s/%s' % (self.path, TEMPORARY_DIRECTORY,
_name(self.rndhex))
if _special_mkdir(temp, self.umask):
break
for name in data.keys():
if name not in self.type:
raise QueueError("unexpected data: %s" % name)
if self.type[name] == 'binary':
if type(data[name]) not in VALID_STR_TYPES:
raise QueueError("unexpected binary data in %s: %r" %
(name, data[name]))
_file_write('%s/%s' % (temp, name), 0, self.umask, data[name])
elif self.type[name] == 'string':
if type(data[name]) not in VALID_STR_TYPES:
raise QueueError("unexpected string data in %s: %r" %
(name, data[name]))
_file_write('%s/%s' % (temp, name), 1, self.umask, data[name])
elif self.type[name] == 'table':
if not isinstance(data[name], dict):
raise QueueError("unexpected table data in %s: %r" %
(name, data[name]))
_file_write('%s/%s' % (temp, name), 1, self.umask,
_hash2string(data[name]))
else:
raise QueueError("unexpected data type in %s: %r" %
(name, self.type[name]))
for name in self.mandatory.keys():
if name not in data:
raise QueueError("missing mandatory data: %s" % name)
while True:
name = '%s/%s' % (self._insertion_directory(), _name(self.rndhex))
path = '%s/%s' % (self.path, name)
try:
os.rename(temp, path)
return name
except Exception:
error = sys.exc_info()[1]
if error.errno != errno.ENOTEMPTY and \
error.errno != errno.EEXIST:
raise OSError("cannot rename(%s, %s): %s" %
(temp, path, error))
