class Ini(SortedDict): def __init__(self, inifile='', commentchar=None, encoding=None, env=None, convertors=None, lazy=False, writable=False, raw=False, import_env=True, basepath='.', pre_variables=None): """ lazy is used to parse first but not deal at time, and only when the user invoke finish() function, it'll parse the data. import_env will import all environment variables if inifile is dict, then automatically add to ini object """ super(Ini, self).__init__() if isinstance(inifile, dict): self._inifile = '' data = inifile else: self._inifile = inifile data = None self._basepath = basepath self._commentchar = commentchar or __default_env__.get( 'commentchar', '#') self._encoding = encoding or __default_env__.get('encoding', 'utf-8') self._env = __default_env__.get('env', {}).copy() self._env.update(env or {}) self._env['set'] = set self.update(self._env) self._globals = SortedDict() self._pre_variables = pre_variables or {} self._import_env = import_env if self._import_env: self._globals.update(os.environ) self._convertors = __default_env__.get('convertors', {}).copy() self._convertors.update(convertors or {}) self._lazy = lazy self._writable = writable self._raw = raw if lazy: self._globals.update(self._env.copy()) if self._inifile: self.read(self._inifile) if data: for k, v in data.items(): s = self.add(k) for _k, _v in v.items(): s[_k] = _v def set_filename(self, filename): self._inifile = filename def get_filename(self): return self._inifile def set_basepath(self, basepath): self._basepath = basepath def set_pre_variables(self, v): self._pre_variables = v or {} filename = property(get_filename, set_filename) def _pre_var(self, value): """ replace predefined variables, the format is #{name} """ def sub_(m): return self._pre_variables.get(m.group()[2:-1].strip(), '') return r_pre_var.sub(sub_, value) def read(self, fobj, filename=''): encoding = None if isinstance(fobj, (str, unicode)): f = open(fobj, 'rb') text = f.read() f.close() else: text = fobj.read() text = text + '\n' begin = 0 if text.startswith(codecs.BOM_UTF8): begin = 3 encoding = 'UTF-8' elif text.startswith(codecs.BOM_UTF16): begin = 2 encoding = 'UTF-16' if not encoding: try: unicode(text, 'UTF-8') encoding = 'UTF-8' except: encoding = defaultencoding self._encoding = encoding f = StringIO.StringIO(text) f.seek(begin) lineno = 0 comments = [] status = 'c' section = None while 1: lastpos = f.tell() line = f.readline() lineno += 1 if not line: break line = line.strip() if line: if line.startswith(self._commentchar): if lineno == 1: #first comment line b = r_encoding.search(line[1:]) if b: self._encoding = b.groups()[0] continue comments.append(line) elif line.startswith('[') and line.endswith(']'): sec_name = line[1:-1].strip() #process include notation if sec_name.startswith('include:'): _filename = sec_name[8:].strip() _file = os.path.join(self._basepath, _filename) if os.path.exists(_file): old_encoding = self._encoding old_filename = self.filename self.set_filename(_file) self.read(_file, filename=_file) self.set_filename(old_filename) self._encoding = old_encoding else: import warnings warnings.warn(Warning( "Can't find the file [%s], so just skip it" % _filename), stacklevel=2) continue info = RawValue(self._inifile, lineno, sec_name) section = self.add(sec_name, comments, info=info) comments = [] elif '=' in line: if section is None: raise Exception( "No section found, please define it first in %s file" % self.filename) #if find <=, then it'll replace the old value for mutable variables #because the default behavior will merge list and dict pos = line.find('<=') if pos != -1: begin, end = pos, pos + 2 replace_flag = True else: pos = line.find('=') begin, end = pos, pos + 1 replace_flag = False keyname = line[:begin].strip() #check keyname if keyname in self._env: raise KeyError( "Settings key %s is alread defined in env, please change it's name" % keyname) rest = line[end:].strip() #if key= then value will be set '' if rest == '': value = 'None' else: f.seek(lastpos + end) try: value, iden_existed = self.__read_line(f) #add pre variables process value = self._pre_var(value) except Exception as e: print_exc() raise Exception( "Parsing ini file error in %s:%d:%s" % (filename or self._inifile, lineno, line)) if self._lazy: if iden_existed: v = EvalValue(value, filename or self._inifile, lineno, line) else: v = value else: if self._raw: v = RawValue(self._inifile, lineno, value, replace_flag) else: try: v = eval_value(value, self.env(), self[sec_name], self._encoding, self._import_env) except Exception as e: print_exc() raise Exception( "Converting value (%s) error in %s:%d:%s" % (value, filename or self._inifile, lineno, line)) section.add(keyname, v, comments, replace=replace_flag) comments = [] else: comments.append(line) def save(self, filename=None): if not filename: filename = self.filename if not filename: filename = sys.stdout if isinstance(filename, (str, unicode)): f = open(filename, 'wb') need_close = True else: f = filename need_close = False print >> f, '#coding=%s' % self._encoding for s in self.keys(): if s in self._env: continue section = self[s] section.dumps(f, convertors=self._convertors) if need_close: f.close() def __read_line(self, f): """ Get logic line according the syntax not the physical line It'll return the line text and if there is identifier existed return line, bool """ g = tokenize.generate_tokens(f.readline) buf = [] time = 0 iden_existed = False while 1: v = g.next() tokentype, t, start, end, line = v if tokentype == 54: continue if tokentype in (token.INDENT, token.DEDENT, tokenize.COMMENT): continue if tokentype == token.NAME: iden_existed = True if tokentype == token.NEWLINE: return ''.join(buf), iden_existed else: if t == '=' and time == 0: time += 1 continue buf.append(t) def __setitem__(self, key, value): if key not in self: super(Ini, self).__setitem__(key, value) def update(self, value): for k, v in value.items(): self.set_var(k, v) def add(self, sec_name, comments=None, info=None): if sec_name in self: section = self[sec_name] else: section = Section(sec_name, comments, self._encoding, root=self, info=info) self[sec_name] = section return section def __str__(self): buf = StringIO.StringIO() self.save(buf) return buf.getvalue() def get_var(self, key, default=None): obj = self for i in key.split('/', 1): obj = obj.get(i) if obj is None: break if obj is None: return default return obj def set_var(self, key, value): s = key.split('/', 1) obj = self for i in s[:-1]: obj = obj.add(i) obj[s[-1]] = value return True def del_var(self, key): s = key.split('/', 1) obj = self for i in s[:-1]: obj = obj.get(i) if obj is None: return False if s[-1] in obj: del obj[s[-1]] flag = True else: flag = False return flag def items(self): return ((k, self[k]) for k in self.keys() if not k in self._env) def env(self): if self._import_env: d = {} d.update(os.environ.copy()) d.update(dict(self)) return d return self def freeze(self): """ Process all EvalValue to real value """ self._lazy = False for k, v in self.items(): if k in self._env: continue for _k, _v in v.items(): if isinstance(_v, Lazy): if self.writable: _v.get() else: try: v.__setitem__(_k, _v.get(), replace=True) except: print "Error ini key:", _k raise del _v self._globals = SortedDict()
class Ini(SortedDict): def __init__(self, inifile='', commentchar=None, encoding=None, env=None, convertors=None, lazy=False, writable=False, raw=False, import_env=True): """ lazy is used to parse first but not deal at time, and only when the user invoke finish() function, it'll parse the data. import_env will import all environment variables """ super(Ini, self).__init__() self._inifile = inifile self._commentchar = commentchar or __default_env__.get( 'commentchar', '#') self._encoding = encoding or __default_env__.get('encoding', 'utf-8') self._env = __default_env__.get('env', {}).copy() self._env.update(env or {}) self._env['set'] = set self.update(self._env) self._globals = SortedDict() self._import_env = import_env if self._import_env: self._globals.update(os.environ) self._convertors = __default_env__.get('convertors', {}).copy() self._convertors.update(convertors or {}) self._lazy = lazy self._writable = writable self._raw = raw if lazy: self._globals.update(self._env.copy()) if self._inifile: self.read(self._inifile) def set_filename(self, filename): self._inifile = filename def get_filename(self): return self._inifile filename = property(get_filename, set_filename) def read(self, fobj, filename=''): encoding = None if isinstance(fobj, (str, unicode)): f = open(fobj, 'rb') text = f.read() f.close() else: text = fobj.read() text = text + '\n' begin = 0 if text.startswith(codecs.BOM_UTF8): begin = 3 encoding = 'UTF-8' elif text.startswith(codecs.BOM_UTF16): begin = 2 encoding = 'UTF-16' if not encoding: try: unicode(text, 'UTF-8') encoding = 'UTF-8' except: encoding = defaultencoding self._encoding = encoding f = StringIO.StringIO(text) f.seek(begin) lineno = 0 comments = [] status = 'c' section = None while 1: lastpos = f.tell() line = f.readline() lineno += 1 if not line: break line = line.strip() if line: if line.startswith(self._commentchar): if lineno == 1: #first comment line b = r_encoding.search(line[1:]) if b: self._encoding = b.groups()[0] continue comments.append(line) elif line.startswith('[') and line.endswith(']'): sec_name = line[1:-1].strip() #process include notation if sec_name.startswith('include:'): _filename = sec_name[8:].strip() _filename = os.path.abspath(_filename) if os.path.exists(_filename): old_encoding = self._encoding self.read(_filename) self._encoding = old_encoding else: import warnings warnings.warn(Warning( "Can't find the file [%s], so just skip it" % _filename), stacklevel=2) continue info = RawValue(self._inifile, lineno, sec_name) section = self.add(sec_name, comments, info=info) comments = [] elif '=' in line: if section is None: raise Exception, "No section found, please define it first in %s file" % self.filename #if find <=, then it'll replace the old value for mutable variables #because the default behavior will merge list and dict pos = line.find('<=') if pos != -1: begin, end = pos, pos + 2 replace_flag = True else: pos = line.find('=') begin, end = pos, pos + 1 replace_flag = False keyname = line[:begin].strip() #check keyname if keyname in self._env: raise KeyError( "Settings key %s is alread defined in env, please change it's name" % keyname) rest = line[end:].strip() #if key= then value will be set '' if rest == '': value = 'None' else: f.seek(lastpos + end) try: value, iden_existed = self.__read_line(f) except Exception, e: print_exc() raise Exception, "Parsing ini file error in %s:%d:%s" % ( filename or self._inifile, lineno, line) if self._lazy: if iden_existed: v = EvalValue(value, filename or self._inifile, lineno, line) else: v = value else: if self._raw: v = RawValue(self._inifile, lineno, value, replace_flag) else: try: v = eval_value(value, self.env(), self[sec_name], self._encoding, self._import_env) except Exception as e: print_exc() print dict(self) raise Exception( "Converting value (%s) error in %s:%d:%s" % (value, filename or self._inifile, lineno, line)) section.add(keyname, v, comments, replace=replace_flag) comments = [] else: comments.append(line)
class Ini(SortedDict): def __init__(self, inifile='', commentchar=None, encoding=None, env=None, convertors=None, lazy=False, writable=False, raw=False, import_env=True): """ lazy is used to parse first but not deal at time, and only when the user invoke finish() function, it'll parse the data. import_env will import all environment variables """ super(Ini, self).__init__() self._inifile = inifile self._commentchar = commentchar or __default_env__.get('commentchar', '#') self._encoding = encoding or __default_env__.get('encoding', 'utf-8') self._env = __default_env__.get('env', {}).copy() self._env.update(env or {}) self._env['set'] = set self.update(self._env) self._globals = SortedDict() self._import_env = import_env if self._import_env: self._globals.update(os.environ) self._convertors = __default_env__.get('convertors', {}).copy() self._convertors.update(convertors or {}) self._lazy = lazy self._writable = writable self._raw = raw if lazy: self._globals.update(self._env.copy()) if self._inifile: self.read(self._inifile) def set_filename(self, filename): self._inifile = filename def get_filename(self): return self._inifile filename = property(get_filename, set_filename) def read(self, fobj, filename=''): encoding = None if isinstance(fobj, (str, unicode)): f = open(fobj, 'rb') text = f.read() f.close() else: text = fobj.read() text = text + '\n' begin = 0 if text.startswith(codecs.BOM_UTF8): begin = 3 encoding = 'UTF-8' elif text.startswith(codecs.BOM_UTF16): begin = 2 encoding = 'UTF-16' if not encoding: try: unicode(text, 'UTF-8') encoding = 'UTF-8' except: encoding = defaultencoding self._encoding = encoding f = StringIO.StringIO(text) f.seek(begin) lineno = 0 comments = [] status = 'c' section = None while 1: lastpos = f.tell() line = f.readline() lineno += 1 if not line: break line = line.strip() if line: if line.startswith(self._commentchar): if lineno == 1: #first comment line b = r_encoding.search(line[1:]) if b: self._encoding = b.groups()[0] continue comments.append(line) elif line.startswith('[') and line.endswith(']'): sec_name = line[1:-1].strip() #process include notation if sec_name.startswith('include:'): _filename = sec_name[8:].strip() _filename = os.path.abspath(_filename) if os.path.exists(_filename): old_encoding = self._encoding self.read(_filename) self._encoding = old_encoding else: import warnings warnings.warn(Warning("Can't find the file [%s], so just skip it" % _filename), stacklevel=2) continue info = RawValue(self._inifile, lineno, sec_name) section = self.add(sec_name, comments, info=info) comments = [] elif '=' in line: if section is None: raise Exception, "No section found, please define it first in %s file" % self.filename #if find <=, then it'll replace the old value for mutable variables #because the default behavior will merge list and dict pos = line.find('<=') if pos != -1: begin, end = pos, pos+2 replace_flag = True else: pos = line.find('=') begin, end = pos, pos+1 replace_flag = False keyname = line[:begin].strip() #check keyname if keyname in self._env: raise KeyError("Settings key %s is alread defined in env, please change it's name" % keyname) rest = line[end:].strip() #if key= then value will be set '' if rest == '': v = None else: f.seek(lastpos+end) try: value, iden_existed = self.__read_line(f) except Exception, e: print_exc() raise Exception, "Parsing ini file error in %s:%d:%s" % (filename or self._inifile, lineno, line) if self._lazy: if iden_existed: v = EvalValue(value, filename or self._inifile, lineno, line) else: v = value else: if self._raw: v = RawValue(self._inifile, lineno, value, replace_flag) else: try: v = eval_value(value, self.env(), self[sec_name], self._encoding) except Exception as e: print_exc() print dict(self) raise Exception("Converting value (%s) error in %s:%d:%s" % (value, filename or self._inifile, lineno, line)) section.add(keyname, v, comments, replace=replace_flag) comments = [] else: comments.append(line)
class Ini(SortedDict): def __init__(self, inifile='', commentchar=None, encoding=None, env=None, convertors=None, lazy=False, writable=False, raw=False, import_env=True, basepath='.', pre_variables=None): """ lazy is used to parse first but not deal at time, and only when the user invoke finish() function, it'll parse the data. import_env will import all environment variables if inifile is dict, then automatically add to ini object """ super(Ini, self).__init__() if isinstance(inifile, dict): self._inifile = '' data = inifile else: self._inifile = inifile data = None self._basepath = basepath self._commentchar = commentchar or __default_env__.get('commentchar', '#') self._encoding = encoding or __default_env__.get('encoding', 'utf-8') self._env = __default_env__.get('env', {}).copy() self._env.update(env or {}) self._env['set'] = set self.update(self._env) self._globals = SortedDict() self._pre_variables = pre_variables or {} self._import_env = import_env if self._import_env: self._globals.update(os.environ) self._convertors = __default_env__.get('convertors', {}).copy() self._convertors.update(convertors or {}) self._lazy = lazy self._writable = writable self._raw = raw if lazy: self._globals.update(self._env.copy()) if self._inifile: self.read(self._inifile) if data: for k, v in data.items(): s = self.add(k) for _k, _v in v.items(): s[_k] = _v def set_filename(self, filename): self._inifile = filename def get_filename(self): return self._inifile def set_basepath(self, basepath): self._basepath = basepath def set_pre_variables(self, v): self._pre_variables = v or {} filename = property(get_filename, set_filename) def _pre_var(self, value): """ replace predefined variables, the format is #{name} """ def sub_(m): return self._pre_variables.get(m.group()[2:-1].strip(), '') return r_pre_var.sub(sub_, value) def read(self, fobj, filename=''): encoding = None if isinstance(fobj, (str, unicode)): f = open(fobj, 'rb') text = f.read() f.close() else: text = fobj.read() text = text + '\n' begin = 0 if text.startswith(codecs.BOM_UTF8): begin = 3 encoding = 'UTF-8' elif text.startswith(codecs.BOM_UTF16): begin = 2 encoding = 'UTF-16' if not encoding: try: unicode(text, 'UTF-8') encoding = 'UTF-8' except: encoding = defaultencoding self._encoding = encoding f = StringIO.StringIO(text) f.seek(begin) lineno = 0 comments = [] status = 'c' section = None while 1: lastpos = f.tell() line = f.readline() lineno += 1 if not line: break line = line.strip() if line: if line.startswith(self._commentchar): if lineno == 1: #first comment line b = r_encoding.search(line[1:]) if b: self._encoding = b.groups()[0] continue comments.append(line) elif line.startswith('[') and line.endswith(']'): sec_name = line[1:-1].strip() #process include notation if sec_name.startswith('include:'): _filename = sec_name[8:].strip() _file = os.path.join(self._basepath, _filename) if os.path.exists(_file): old_encoding = self._encoding old_filename = self.filename self.set_filename(_file) self.read(_file, filename=_file) self.set_filename(old_filename) self._encoding = old_encoding else: import warnings warnings.warn(Warning("Can't find the file [%s], so just skip it" % _filename), stacklevel=2) continue info = RawValue(self._inifile, lineno, sec_name) section = self.add(sec_name, comments, info=info) comments = [] elif '=' in line: if section is None: raise Exception("No section found, please define it first in %s file" % self.filename) #if find <=, then it'll replace the old value for mutable variables #because the default behavior will merge list and dict pos = line.find('<=') if pos != -1: begin, end = pos, pos+2 replace_flag = True else: pos = line.find('=') begin, end = pos, pos+1 replace_flag = False keyname = line[:begin].strip() #check keyname if keyname in self._env: raise KeyError("Settings key %s is alread defined in env, please change it's name" % keyname) rest = line[end:].strip() #if key= then value will be set '' if rest == '': value = 'None' else: f.seek(lastpos+end) try: value, iden_existed = self.__read_line(f) #add pre variables process value = self._pre_var(value) except Exception as e: print_exc() raise Exception("Parsing ini file error in %s:%d:%s" % (filename or self._inifile, lineno, line)) if self._lazy: if iden_existed: v = EvalValue(value, filename or self._inifile, lineno, line) else: v = value else: if self._raw: v = RawValue(self._inifile, lineno, value, replace_flag) else: try: v = eval_value(value, self.env(), self[sec_name], self._encoding, self._import_env) except Exception as e: print_exc() raise Exception("Converting value (%s) error in %s:%d:%s" % (value, filename or self._inifile, lineno, line)) section.add(keyname, v, comments, replace=replace_flag) comments = [] else: comments.append(line) def save(self, filename=None): if not filename: filename = self.filename if not filename: filename = sys.stdout if isinstance(filename, (str, unicode)): f = open(filename, 'wb') need_close = True else: f = filename need_close = False print >> f, '#coding=%s' % self._encoding for s in self.keys(): if s in self._env: continue section = self[s] section.dumps(f, convertors=self._convertors) if need_close: f.close() def __read_line(self, f): """ Get logic line according the syntax not the physical line It'll return the line text and if there is identifier existed return line, bool """ g = tokenize.generate_tokens(f.readline) buf = [] time = 0 iden_existed = False while 1: v = g.next() tokentype, t, start, end, line = v if tokentype == 54: continue if tokentype in (token.INDENT, token.DEDENT, tokenize.COMMENT): continue if tokentype == token.NAME: iden_existed = True if tokentype == token.NEWLINE: return ''.join(buf), iden_existed else: if t == '=' and time == 0: time += 1 continue buf.append(t) def __setitem__(self, key, value): if key not in self: super(Ini, self).__setitem__(key, value) def update(self, value): for k, v in value.items(): self.set_var(k, v) def add(self, sec_name, comments=None, info=None): if sec_name in self: section = self[sec_name] else: section = Section(sec_name, comments, self._encoding, root=self, info=info) self[sec_name] = section return section def __str__(self): buf = StringIO.StringIO() self.save(buf) return buf.getvalue() def get_var(self, key, default=None): obj = self for i in key.split('/', 1): obj = obj.get(i) if obj is None: break if obj is None: return default return obj def set_var(self, key, value): s = key.split('/', 1) obj = self for i in s[:-1]: obj = obj.add(i) obj[s[-1]] = value return True def del_var(self, key): s = key.split('/', 1) obj = self for i in s[:-1]: obj = obj.get(i) if obj is None: return False if s[-1] in obj: del obj[s[-1]] flag = True else: flag = False return flag def items(self): return ((k, self[k]) for k in self.keys() if not k in self._env) def env(self): if self._import_env: d = {} d.update(os.environ.copy()) d.update(dict(self)) return d return self def freeze(self): """ Process all EvalValue to real value """ self._lazy = False for k, v in self.items(): if k in self._env: continue for _k, _v in v.items(): if isinstance(_v, Lazy): if self.writable: _v.get() else: v.__setitem__(_k, _v.get(), replace=True) del _v self._globals = SortedDict()
def __new__(cls, name, bases, attrs): metaclass = attrs.get("__metaclass__") super_new = super(DocumentMetaclass, cls).__new__ if metaclass and issubclass(metaclass, DocumentMetaclass): return super_new(cls, name, bases, attrs) doc_fields = SortedDict() class_name = [name] superclasses = {} simple_class = True for base in bases: # Include all fields present in superclasses if hasattr(base, "_fields"): doc_fields.update(base._fields) class_name.append(base._class_name) # Get superclasses from superclass superclasses[base._class_name] = base superclasses.update(base._superclasses) if hasattr(base, "_meta"): # Ensure that the Document class may be subclassed - # inheritance may be disabled to remove dependency on # additional fields _cls and _types if base._meta.get("allow_inheritance", True) == False: raise ValueError("Document %s may not be subclassed" % base.__name__) else: simple_class = False meta = attrs.get("_meta", attrs.get("meta", {})) if "allow_inheritance" not in meta: meta["allow_inheritance"] = True # Only simple classes - direct subclasses of Document - may set # allow_inheritance to False if not simple_class and not meta["allow_inheritance"]: raise ValueError("Only direct subclasses of Document may set " '"allow_inheritance" to False') attrs["_meta"] = meta attrs["_class_name"] = ".".join(reversed(class_name)) attrs["_superclasses"] = superclasses # Add the document's fields to the _fields attribute declared_fields = {} for attr_name, attr_value in attrs.items(): if hasattr(attr_value, "__class__") and issubclass(attr_value.__class__, BaseField): attr_value.name = attr_name if not attr_value.db_field: attr_value.db_field = attr_name declared_fields[attr_name] = attr_value # sort fields based on creation_counter by_value = lambda x, y: cmp(x[1], y[1]) doc_fields.update(SortedDict(sorted(declared_fields.items(), by_value))) attrs["_fields"] = doc_fields new_class = super_new(cls, name, bases, attrs) for field in new_class._fields.values(): field.owner_document = new_class module = attrs.get("__module__") base_excs = tuple(base.DoesNotExist for base in bases if hasattr(base, "DoesNotExist")) or (DoesNotExist,) exc = subclass_exception("DoesNotExist", base_excs, module) new_class.add_to_class("DoesNotExist", exc) base_excs = tuple(base.MultipleObjectsReturned for base in bases if hasattr(base, "MultipleObjectsReturned")) base_excs = base_excs or (MultipleObjectsReturned,) exc = subclass_exception("MultipleObjectsReturned", base_excs, module) new_class.add_to_class("MultipleObjectsReturned", exc) global _document_registry _document_registry[name] = new_class return new_class
def test_update(self): a = SortedDict({'a': 1}) expected = SortedDict({'a': 1, 'b': 2}) a.update({'b': 2}) self.assertEqual(a, expected)