def _add_data(self, data): """Write 'data' to a file. Return: (tuple) directory name where the file was written, full path to the temporary file. """ _dir = self._add_dir() while 1: tmp = '%s/%s/%s%s' % (self.path, _dir, _name(self.rndhex), TEMPORARY_SUFFIX) try: if is_bytes(data): new_file = _file_create(tmp, umask=self.umask, utf8=False) else: new_file = _file_create(tmp, umask=self.umask, utf8=True) except EnvironmentError: error = sys.exc_info()[1] if error.errno == errno.ENOENT: _special_mkdir('%s/%s' % (self.path, _dir), self.umask) continue else: if new_file: break new_file.write(data) new_file.close() return _dir, tmp
def __init__(self, path, umask=None, rndhex=None, maxelts=16000, schema=dict()): """Check and set schema. Build the queue directory structure. Arguments: path the queue toplevel directory umask the umask to use when creating files and directories (default: use the running process' umask) rndhex the hexadecimal digit to use in names (default: randomly chosen) maxelts the maximum number of elements that an intermediate directory can hold (default: 16,000) schema the schema defining how to interpret user supplied data (mandatory if elements are added or read) Raise: TypeError - wrong input data types provided QueueError - problems with the queue schema definition OSError - can't create directory structure """ super(Queue, self).__init__(path, umask=umask, rndhex=rndhex) if type(maxelts) in VALID_INT_TYPES: self.maxelts = maxelts else: raise TypeError("'maxelts' should be int or long") # check schema self.type = {} self.mandatory = {} if schema: if not isinstance(schema, dict): raise QueueError("invalid schema: %r" % schema) for name in schema.keys(): if not _FileRegexp.match(name): raise QueueError("invalid schema name: %r" % name) if not isinstance(schema[name], str): raise QueueError("invalid data type for schema " + "specification: %r" % type(schema[name])) match = re.match('(binary|string|table)([\\?\\*]{0,2})?$', schema[name]) if not match: raise QueueError("invalid schema data type: %r" % schema[name]) self.type[name] = match.group(1) if not re.search('\\?', match.group(2)): self.mandatory[name] = True if not self.mandatory: raise QueueError("invalid schema: no mandatory data") # create directories for directory in (TEMPORARY_DIRECTORY, OBSOLETE_DIRECTORY): _special_mkdir('%s/%s' % (self.path, directory), self.umask)
def add_path(self, path): """Add the given file (identified by its path) to the queue and return the corresponding element name, the file must be on the same filesystem and will be moved to the queue """ _dir = self._add_dir() _special_mkdir('%s/%s' % (self.path, _dir), self.umask) return self._add_path(path, _dir)
def add(self, data): """Add a new element to the queue and return its name. Arguments: data - element as a dictionary (should conform to the schema) Raise: QueueError - problem with schema definition or data OSError - problem putting element on disk Note: the destination directory must _not_ be created beforehand as it would be seen as a valid (but empty) element directory by another process, we therefore use rename() from a temporary directory """ if not self.type: raise QueueError("unknown schema") while True: temp = "%s/%s/%s" % (self.path, TEMPORARY_DIRECTORY, _name(self.rndhex)) if _special_mkdir(temp, self.umask): break for name in data.keys(): if name not in self.type: raise QueueError("unexpected data: %s" % name) if self.type[name] == "binary": if type(data[name]) not in VALID_STR_TYPES: raise QueueError("unexpected binary data in %s: %r" % (name, data[name])) _file_write("%s/%s" % (temp, name), 0, self.umask, data[name]) elif self.type[name] == "string": if type(data[name]) not in VALID_STR_TYPES: raise QueueError("unexpected string data in %s: %r" % (name, data[name])) _file_write("%s/%s" % (temp, name), 1, self.umask, data[name]) elif self.type[name] == "table": if not isinstance(data[name], dict): raise QueueError("unexpected table data in %s: %r" % (name, data[name])) _file_write("%s/%s" % (temp, name), 1, self.umask, _hash2string(data[name])) else: raise QueueError("unexpected data type in %s: %r" % (name, self.type[name])) for name in self.mandatory.keys(): if name not in data: raise QueueError("missing mandatory data: %s" % name) while True: name = "%s/%s" % (self._insertion_directory(), _name(self.rndhex)) path = "%s/%s" % (self.path, name) try: os.rename(temp, path) return name except Exception: error = sys.exc_info()[1] if error.errno != errno.ENOTEMPTY and error.errno != errno.EEXIST: raise OSError("cannot rename(%s, %s): %s" % (temp, path, error))
def _insertion_directory(self): """Return the name of the intermediate directory that can be used for insertion: * if there is none, an initial one will be created * if it is full, a new one will be created * in any case the name will match $_DIRECTORY_REGEXP Raise: OSError - can't list/make element directories """ _list = [] # get the list of existing directories for name in _directory_contents(self.path): if _DIRECTORY_REGEXP.match(name): _list.append(name) # handle the case with no directories yet if not _list: name = "%08x" % 0 _special_mkdir("%s/%s" % (self.path, name), self.umask) return name # check the last directory _list.sort() name = _list[-1] subdirs = _subdirs_num("%s/%s" % (self.path, name)) if subdirs: if subdirs < self.maxelts: return name else: # RACE: at this point, the directory does not exist anymore, # so it must have been purged after we listed the directory # contents. We do not try to do more and simply create a new # directory pass # we need a new directory name = "%08x" % (int(name, 16) + 1) _special_mkdir("%s/%s" % (self.path, name), self.umask) return name
def _insertion_directory(self): """Return the name of the intermediate directory that can be used for insertion: * if there is none, an initial one will be created * if it is full, a new one will be created * in any case the name will match $_DirectoryRegexp Raise: OSError - can't list/make element directories """ _list = [] # get the list of existing directories for name in _directory_contents(self.path): if _DirectoryRegexp.match(name): _list.append(name) # handle the case with no directories yet if not _list: name = '%08x' % 0 _special_mkdir('%s/%s' % (self.path, name), self.umask) return name # check the last directory _list.sort() name = _list[-1] subdirs = _subdirs_num('%s/%s' % (self.path, name)) if subdirs: if subdirs < self.maxelts: return name else: # RACE: at this point, the directory does not exist anymore, # so it must have been purged after we listed the directory # contents. We do not try to do more and simply create a new # directory pass # we need a new directory name = '%08x' % (int(name, 16) + 1) _special_mkdir('%s/%s' % (self.path, name), self.umask) return name
def add(self, data): """Add a new element to the queue and return its name. Arguments: data - element as a dictionary (should conform to the schema) Raise: QueueError - problem with schema definition or data OSError - problem putting element on disk Note: the destination directory must _not_ be created beforehand as it would be seen as a valid (but empty) element directory by another process, we therefore use rename() from a temporary directory """ if not self.type: raise QueueError("unknown schema") while True: temp = '%s/%s/%s' % (self.path, TEMPORARY_DIRECTORY, _name(self.rndhex)) if _special_mkdir(temp, self.umask): break for name in data.keys(): if name not in self.type: raise QueueError("unexpected data: %s" % name) if self.type[name] == 'binary': if type(data[name]) not in VALID_STR_TYPES: raise QueueError("unexpected binary data in %s: %r" % (name, data[name])) _file_write('%s/%s' % (temp, name), 0, self.umask, data[name]) elif self.type[name] == 'string': if type(data[name]) not in VALID_STR_TYPES: raise QueueError("unexpected string data in %s: %r" % (name, data[name])) _file_write('%s/%s' % (temp, name), 1, self.umask, data[name]) elif self.type[name] == 'table': if not isinstance(data[name], dict): raise QueueError("unexpected table data in %s: %r" % (name, data[name])) _file_write('%s/%s' % (temp, name), 1, self.umask, _hash2string(data[name])) else: raise QueueError("unexpected data type in %s: %r" % (name, self.type[name])) for name in self.mandatory.keys(): if name not in data: raise QueueError("missing mandatory data: %s" % name) while True: name = '%s/%s' % (self._insertion_directory(), _name(self.rndhex)) path = '%s/%s' % (self.path, name) try: os.rename(temp, path) return name except Exception: error = sys.exc_info()[1] if error.errno != errno.ENOTEMPTY and \ error.errno != errno.EEXIST: raise OSError("cannot rename(%s, %s): %s" % (temp, path, error))