def iterentries_from_text(text, encoding='utf-8'): py2_decode = identity if hasattr(text, 'read'): text = text.read(-1) if not isinstance(text, str): text = text.decode(encoding) for entrytype, (bibkey, fields) in LowLevelParser(text): fields = { py2_decode(name).lower(): whitespace_re.sub(' ', py2_decode(''.join(values)).strip()) for name, values in fields} yield py2_decode(bibkey), (py2_decode(entrytype), fields)
def iterentries(filename, encoding=None): encoding = encoding or 'utf8' with memorymapped(filename) as source: try: for entrytype, (bibkey, fields) in BibTeXEntryIterator(source): fields = { name.decode(encoding).lower(): whitespace_re.sub(' ', ''.join(values).decode(encoding).strip()) for name, values in fields} yield bibkey.decode(encoding), (entrytype.decode(encoding), fields) except PybtexSyntaxError as e: # pragma: no cover debug_pybtex(source, e)
def iterentries(filename, encoding=None): encoding = encoding or 'utf8' with memorymapped(filename) as source: try: for entrytype, (bibkey, fields) in BibTeXEntryIterator(source): fields = { name.decode(encoding).lower(): whitespace_re.sub(' ', ''.join(values).decode(encoding).strip()) for name, values in fields } yield bibkey.decode(encoding), (entrytype.decode(encoding), fields) except PybtexSyntaxError as e: # pragma: no cover debug_pybtex(source, e)
def iterentries_from_text(text, encoding='utf-8'): if PY2: py2_decode = lambda text: text.decode(encoding) else: # pragma: no cover py2_decode = lambda text: text if hasattr(text, 'read'): text = text.read() if not isinstance(text, text_type): text = text.decode(encoding) for entrytype, (bibkey, fields) in BibTeXEntryIterator(text): fields = { py2_decode(name).lower(): whitespace_re.sub(' ', py2_decode(''.join(values)).strip()) for name, values in fields} yield py2_decode(bibkey), (py2_decode(entrytype), fields)
def iterentries(filename, encoding=None, use_pybtex=True): if not use_pybtex: # legacy code path for conversion/comparison if encoding not in (None, 'ascii'): raise NotImplementedError import _libmonster with memorymapped(filename) as source: for bibkey, entrytype, fields in _libmonster.pitems(source): yield bibkey, (entrytype, fields) elif encoding is None: raise NotImplementedError else: with memorymapped(filename) as source: try: for entrytype, (bibkey, fields) in BibTeXEntryIterator(source): fields = {name.decode(encoding).lower(): whitespace_re.sub(' ', ''.join(values).decode(encoding).strip()) for name, values in fields} yield bibkey.decode(encoding), (entrytype.decode(encoding), fields) except PybtexSyntaxError as e: debug_pybtex(source, e)