def process_txt(self, directory, name):
if name.startswith("pep-"):
publisher = "PEPs"
else:
publisher = ".txt"
settings = self.get_settings(publisher, directory)
errout = ErrorOutput(encoding=settings.error_encoding)
pub_struct = self.publishers[publisher]
settings._source = os.path.normpath(os.path.join(directory, name))
settings._destination = settings._source[:-4] + ".html"
if not self.initial_settings.silent:
errout.write(" ::: Processing: %s\n" % name)
sys.stderr.flush()
try:
if not settings.dry_run:
core.publish_file(
source_path=settings._source,
destination_path=settings._destination,
reader_name=pub_struct.reader_name,
parser_name="restructuredtext",
writer_name=pub_struct.writer_name,
settings=settings,
)
except ApplicationError:
error = sys.exc_info()[1] # get exception in Python <2.6 and 3.x
errout.write(" %s\n" % ErrorString(error))
def __init__(self, *args, **kwargs):
CP.RawConfigParser.__init__(self, *args, **kwargs)
self._files = []
"""List of paths of configuration files read."""
self._stderr = ErrorOutput()
"""Wrapper around sys.stderr catching en-/decoding errors"""
def __init__(
self,
reader=None,
parser=None,
writer=None,
source=None,
source_class=io.FileInput,
destination=None,
destination_class=io.FileOutput,
settings=None,
):
"""
Initial setup. If any of `reader`, `parser`, or `writer` are not
specified, the corresponding ``set_...`` method should be called with
a component name (`set_reader` sets the parser as well).
"""
self.document = None
"""The document tree (`docutils.nodes` objects)."""
self.reader = reader
"""A `docutils.readers.Reader` instance."""
self.parser = parser
"""A `docutils.parsers.Parser` instance."""
self.writer = writer
"""A `docutils.writers.Writer` instance."""
for component in "reader", "parser", "writer":
assert not isinstance(getattr(self, component), str), (
'passed string "%s" as "%s" parameter; pass an instance, '
'or use the "%s_name" parameter instead (in '
"docutils.core.publish_* convenience functions)." %
(getattr(self, component), component, component))
self.source = source
"""The source of input data, a `docutils.io.Input` instance."""
self.source_class = source_class
"""The class for dynamically created source objects."""
self.destination = destination
"""The destination for docutils output, a `docutils.io.Output`
instance."""
self.destination_class = destination_class
"""The class for dynamically created destination objects."""
self.settings = settings
"""An object containing Docutils settings as instance attributes.
Set by `self.process_command_line()` or `self.get_settings()`."""
self._stderr = ErrorOutput()
def test_ubuf(self):
buf = UBuf() # buffer only accepting unicode string
# decode of binary strings
e = ErrorOutput(buf, encoding='ascii')
e.write(b('b\xfc'))
self.assertEqual(buf.getvalue(), 'b\ufffd') # use REPLACEMENT CHARACTER
# write Unicode string and Exceptions with Unicode args
e.write(' u\xfc')
self.assertEqual(buf.getvalue(), 'b\ufffd u\xfc')
e.write(AttributeError(' e\xfc'))
self.assertEqual(buf.getvalue(), 'b\ufffd u\xfc e\xfc')
# decode with `encoding` attribute
e.encoding = 'latin1'
e.write(b(' b\xfc'))
self.assertEqual(buf.getvalue(), 'b\ufffd u\xfc e\xfc b\xfc')
def __init__(self, source=None, source_path=None,
encoding=None, error_handler='strict',
autoclose=True, mode='rU'):
"""
:Parameters:
- `source`: either a file-like object (which is read directly), or
`None` (which implies `sys.stdin` if no `source_path` given).
- `source_path`: a path to a file, which is opened and then read.
- `encoding`: the expected text encoding of the input file.
- `error_handler`: the encoding error handler to use.
- `autoclose`: close automatically after read (except when
`sys.stdin` is the source).
- `mode`: how the file is to be opened (see standard function
`open`). The default 'rU' provides universal newline support
for text files.
"""
Input.__init__(self, source, source_path, encoding, error_handler)
self.autoclose = autoclose
self._stderr = ErrorOutput()
if source is None:
if source_path:
# Specify encoding in Python 3
if sys.version_info >= (3,0):
kwargs = {'encoding': self.encoding,
'errors': self.error_handler}
else:
kwargs = {}
try:
self.source = open(source_path, mode, **kwargs)
except IOError, error:
raise InputError(error.errno, error.strerror, source_path)
else:
self.source = sys.stdin
def test_bbuf(self):
buf = BBuf() # buffer storing byte string
e = ErrorOutput(buf, encoding='ascii')
# write byte-string as-is
e.write(b('b\xfc'))
self.assertEqual(buf.getvalue(), b('b\xfc'))
# encode unicode data with backslashescape fallback replacement:
e.write(' u\xfc')
self.assertEqual(buf.getvalue(), b('b\xfc u\\xfc'))
# handle Exceptions with Unicode string args
# unicode(Exception(u'e\xfc')) # fails in Python < 2.6
e.write(AttributeError(' e\xfc'))
self.assertEqual(buf.getvalue(), b('b\xfc u\\xfc e\\xfc'))
# encode with `encoding` attribute
e.encoding = 'utf8'
e.write(' u\xfc')
self.assertEqual(buf.getvalue(), b('b\xfc u\\xfc e\\xfc u\xc3\xbc'))
def __init__(self, *args, **kwargs):
CP.RawConfigParser.__init__(self, *args, **kwargs)
self._files = []
"""List of paths of configuration files read."""
self._stderr = ErrorOutput()
"""Wrapper around sys.stderr catching en-/decoding errors"""
def __init__(
self,
source,
report_level,
halt_level,
stream=None,
debug=False,
encoding=None,
error_handler="backslashreplace",
):
"""
:Parameters:
- `source`: The path to or description of the source data.
- `report_level`: The level at or above which warning output will
be sent to `stream`.
- `halt_level`: The level at or above which `SystemMessage`
exceptions will be raised, halting execution.
- `debug`: Show debug (level=0) system messages?
- `stream`: Where warning output is sent. Can be file-like (has a
``.write`` method), a string (file name, opened for writing),
'' (empty string) or `False` (for discarding all stream messages)
or `None` (implies `sys.stderr`; default).
- `encoding`: The output encoding.
- `error_handler`: The error handler for stderr output encoding.
"""
self.source = source
"""The path to or description of the source data."""
self.error_handler = error_handler
"""The character encoding error handler."""
self.debug_flag = debug
"""Show debug (level=0) system messages?"""
self.report_level = report_level
"""The level at or above which warning output will be sent
to `self.stream`."""
self.halt_level = halt_level
"""The level at or above which `SystemMessage` exceptions
will be raised, halting execution."""
if not isinstance(stream, ErrorOutput):
stream = ErrorOutput(stream, encoding, error_handler)
self.stream = stream
"""Where warning output is sent."""
self.encoding = encoding or getattr(stream, "encoding", "ascii")
"""The output character encoding."""
self.observers = []
"""List of bound methods or functions to call with each system_message
created."""
self.max_level = -1
"""The highest level system message generated so far."""
def __init__(self, source=None, source_path=None,
encoding=None, error_handler='strict',
autoclose=True,
mode='r' if sys.version_info >= (3, 0) else 'rU', **kwargs):
"""
:Parameters:
- `source`: either a file-like object (which is read directly), or
`None` (which implies `sys.stdin` if no `source_path` given).
- `source_path`: a path to a file, which is opened and then read.
- `encoding`: the expected text encoding of the input file.
- `error_handler`: the encoding error handler to use.
- `autoclose`: close automatically after read (except when
`sys.stdin` is the source).
- `mode`: how the file is to be opened (see standard function
`open`). The default 'rU' provides universal newline support
for text files on Python < 3.4.
"""
Input.__init__(self, source, source_path, encoding, error_handler)
self.autoclose = autoclose
self._stderr = ErrorOutput()
# deprecation warning
for key in kwargs:
if key == 'handle_io_errors':
sys.stderr.write('deprecation warning: '
'io.FileInput() argument `handle_io_errors` '
'is ignored since Docutils 0.10 (2012-12-16) '
'and will soon be removed.')
else:
raise TypeError('__init__() got an unexpected keyword '
"argument '%s'" % key)
if source is None:
if source_path:
# Specify encoding in Python 3
if sys.version_info >= (3, 0):
kwargs = {'encoding': self.encoding,
'errors': self.error_handler}
else:
kwargs = {}
try:
self.source = open(source_path, mode, **kwargs)
except IOError as error:
raise InputError(error.errno, error.strerror, source_path)
else:
self.source = sys.stdin
elif (sys.version_info >= (3, 0) and
check_encoding(self.source, self.encoding) is False):
# TODO: re-open, warn or raise error?
raise UnicodeError('Encoding clash: encoding given is "%s" '
'but source is opened with encoding "%s".' %
(self.encoding, self.source.encoding))
if not source_path:
try:
self.source_path = self.source.name
except AttributeError:
pass
def set_conditions(self, category, report_level, halt_level,
stream=None, debug=False):
warnings.warn('docutils.utils.Reporter.set_conditions deprecated; '
'set attributes via configuration settings or directly',
DeprecationWarning, stacklevel=2)
self.report_level = report_level
self.halt_level = halt_level
if not isinstance(stream, ErrorOutput):
stream = ErrorOutput(stream, self.encoding, self.error_handler)
self.stream = stream
self.debug_flag = debug
def __init__(
self,
reader=None,
parser=None,
writer=None,
source=None,
source_class=io.FileInput,
destination=None,
destination_class=io.FileOutput,
settings=None,
):
"""
Initial setup. If any of `reader`, `parser`, or `writer` are not
specified, the corresponding ``set_...`` method should be called with
a component name (`set_reader` sets the parser as well).
"""
self.document = None
"""The document tree (`docutils.nodes` objects)."""
self.reader = reader
"""A `docutils.readers.Reader` instance."""
self.parser = parser
"""A `docutils.parsers.Parser` instance."""
self.writer = writer
"""A `docutils.writers.Writer` instance."""
for component in "reader", "parser", "writer":
assert not isinstance(getattr(self, component), str), (
'passed string "%s" as "%s" parameter; pass an instance, '
'or use the "%s_name" parameter instead (in '
"docutils.core.publish_* convenience functions)." % (getattr(self, component), component, component)
)
self.source = source
"""The source of input data, a `docutils.io.Input` instance."""
self.source_class = source_class
"""The class for dynamically created source objects."""
self.destination = destination
"""The destination for docutils output, a `docutils.io.Output`
instance."""
self.destination_class = destination_class
"""The class for dynamically created destination objects."""
self.settings = settings
"""An object containing Docutils settings as instance attributes.
Set by `self.process_command_line()` or `self.get_settings()`."""
self._stderr = ErrorOutput()
def __init__(self,
destination=None,
destination_path=None,
encoding=None,
error_handler='strict',
autoclose=True,
handle_io_errors=None,
mode=None):
"""
:Parameters:
- `destination`: either a file-like object (which is written
directly) or `None` (which implies `sys.stdout` if no
`destination_path` given).
- `destination_path`: a path to a file, which is opened and then
written.
- `encoding`: the text encoding of the output file.
- `error_handler`: the encoding error handler to use.
- `autoclose`: close automatically after write (except when
`sys.stdout` or `sys.stderr` is the destination).
- `handle_io_errors`: ignored, deprecated, will be removed.
- `mode`: how the file is to be opened (see standard function
`open`). The default is 'w', providing universal newline
support for text files.
"""
Output.__init__(self, destination, destination_path, encoding,
error_handler)
self.opened = True
self.autoclose = autoclose
if handle_io_errors is not None:
warnings.warn(
'io.FileOutput: initialization argument '
'"handle_io_errors" is ignored and will be removed in '
'Docutils 1.2.',
DeprecationWarning,
stacklevel=2)
if mode is not None:
self.mode = mode
self._stderr = ErrorOutput()
if destination is None:
if destination_path:
self.opened = False
else:
self.destination = sys.stdout
elif ( # destination is file-type object -> check mode:
mode and hasattr(self.destination, 'mode')
and mode != self.destination.mode):
print('Warning: Destination mode "%s" differs from specified '
'mode "%s"' % (self.destination.mode, mode),
file=self._stderr)
if not destination_path:
try:
self.destination_path = self.destination.name
except AttributeError:
pass
def visit(self, directory, names, subdirectories):
settings = self.get_settings("", directory)
errout = ErrorOutput(encoding=settings.error_encoding)
if settings.prune and (os.path.abspath(directory) in settings.prune):
errout.write("/// ...Skipping directory (pruned): %s\n" % directory)
sys.stderr.flush()
del subdirectories[:]
return
if not self.initial_settings.silent:
errout.write("/// Processing directory: %s" % directory)
sys.stderr.flush()
# settings.ignore grows many duplicate entries as we recurse
# if we add patterns in config files or on the command line.
for pattern in utils.uniq(settings.ignore):
for i in range(len(names) - 1, -1, -1):
if fnmatch(names[i], pattern):
# Modify in place!
del names[i]
for name in names:
if name.endswith(".txt"):
self.process_txt(directory, name)
def __init__(
self,
source=None,
source_path=None,
encoding=None,
error_handler="strict",
autoclose=True,
mode="r" if sys.version_info >= (3, 0) else "rU",
):
"""
:Parameters:
- `source`: either a file-like object (which is read directly), or
`None` (which implies `sys.stdin` if no `source_path` given).
- `source_path`: a path to a file, which is opened and then read.
- `encoding`: the expected text encoding of the input file.
- `error_handler`: the encoding error handler to use.
- `autoclose`: close automatically after read (except when
`sys.stdin` is the source).
- `mode`: how the file is to be opened (see standard function
`open`). The default 'rU' provides universal newline support
for text files with Python 2.x.
"""
Input.__init__(self, source, source_path, encoding, error_handler)
self.autoclose = autoclose
self._stderr = ErrorOutput()
if source is None:
if source_path:
# Specify encoding in Python 3
if sys.version_info >= (3, 0):
kwargs = {
"encoding": self.encoding,
"errors": self.error_handler
}
else:
kwargs = {}
try:
self.source = open(source_path, mode, **kwargs)
except IOError as error:
raise InputError(error.errno, error.strerror, source_path)
else:
self.source = sys.stdin
elif (sys.version_info >= (3, 0)
and check_encoding(self.source, self.encoding) is False):
# TODO: re-open, warn or raise error?
raise UnicodeError('Encoding clash: encoding given is "%s" '
'but source is opened with encoding "%s".' %
(self.encoding, self.source.encoding))
if not source_path:
try:
self.source_path = self.source.name
except AttributeError:
pass
def __init__(self,
destination=None,
destination_path=None,
encoding=None,
error_handler='strict',
autoclose=True,
mode=None):
"""
:Parameters:
- `destination`: either a file-like object (which is written
directly) or `None` (which implies `sys.stdout` if no
`destination_path` given).
- `destination_path`: a path to a file, which is opened and then
written.
- `encoding`: the text encoding of the output file.
- `error_handler`: the encoding error handler to use.
- `autoclose`: close automatically after write (except when
`sys.stdout` or `sys.stderr` is the destination).
- `mode`: how the file is to be opened (see standard function
`open`). The default is 'w', providing universal newline
support for text files.
"""
Output.__init__(self, destination, destination_path, encoding,
error_handler)
self.opened = True
self.autoclose = autoclose
if mode is not None:
self.mode = mode
self._stderr = ErrorOutput()
if destination is None:
if destination_path:
self.opened = False
else:
self.destination = sys.stdout
elif ( # destination is file-type object -> check mode:
mode and hasattr(self.destination, 'mode')
and mode != self.destination.mode):
print >> self._stderr, ('Warning: Destination mode "%s" '
'differs from specified mode "%s"' %
(self.destination.mode, mode))
if not destination_path:
try:
self.destination_path = self.destination.name
except AttributeError:
pass
def visit(self, directory, names, subdirectories):
settings = self.get_settings('', directory)
errout = ErrorOutput(encoding=settings.error_encoding)
if settings.prune and (os.path.abspath(directory) in settings.prune):
errout.write('/// ...Skipping directory (pruned): %s\n' %
directory)
sys.stderr.flush()
del subdirectories[:]
return
if not self.initial_settings.silent:
errout.write('/// Processing directory: %s\n' % directory)
sys.stderr.flush()
# settings.ignore grows many duplicate entries as we recurse
# if we add patterns in config files or on the command line.
for pattern in utils.uniq(settings.ignore):
for i in range(len(names) - 1, -1, -1):
if fnmatch(names[i], pattern):
# Modify in place!
del names[i]
for name in names:
if name.endswith('.txt'):
self.process_txt(directory, name)
def process_txt(self, directory, name):
if name.startswith('pep-'):
publisher = 'PEPs'
else:
publisher = '.txt'
settings = self.get_settings(publisher, directory)
errout = ErrorOutput(encoding=settings.error_encoding)
pub_struct = self.publishers[publisher]
settings._source = os.path.normpath(os.path.join(directory, name))
settings._destination = settings._source[:-4] + '.html'
if not self.initial_settings.silent:
errout.write(' ::: Processing: %s\n' % name)
sys.stderr.flush()
try:
if not settings.dry_run:
core.publish_file(source_path=settings._source,
destination_path=settings._destination,
reader_name=pub_struct.reader_name,
parser_name='restructuredtext',
writer_name=pub_struct.writer_name,
settings=settings)
except ApplicationError:
error = sys.exc_info()[1] # get exception in Python <2.6 and 3.x
errout.write(' %s\n' % ErrorString(error))
class Publisher:
"""
A facade encapsulating the high-level logic of a Docutils system.
"""
def __init__(self, reader=None, parser=None, writer=None,
source=None, source_class=io.FileInput,
destination=None, destination_class=io.FileOutput,
settings=None):
"""
Initial setup. If any of `reader`, `parser`, or `writer` are not
specified, the corresponding ``set_...`` method should be called with
a component name (`set_reader` sets the parser as well).
"""
self.document = None
"""The document tree (`docutils.nodes` objects)."""
self.reader = reader
"""A `docutils.readers.Reader` instance."""
self.parser = parser
"""A `docutils.parsers.Parser` instance."""
self.writer = writer
"""A `docutils.writers.Writer` instance."""
for component in 'reader', 'parser', 'writer':
assert not isinstance(getattr(self, component), str), (
'passed string "%s" as "%s" parameter; pass an instance, '
'or use the "%s_name" parameter instead (in '
'docutils.core.publish_* convenience functions).'
% (getattr(self, component), component, component))
self.source = source
"""The source of input data, a `docutils.io.Input` instance."""
self.source_class = source_class
"""The class for dynamically created source objects."""
self.destination = destination
"""The destination for docutils output, a `docutils.io.Output`
instance."""
self.destination_class = destination_class
"""The class for dynamically created destination objects."""
self.settings = settings
"""An object containing Docutils settings as instance attributes.
Set by `self.process_command_line()` or `self.get_settings()`."""
self._stderr = ErrorOutput()
def set_reader(self, reader_name, parser, parser_name):
"""Set `self.reader` by name."""
reader_class = readers.get_reader_class(reader_name)
self.reader = reader_class(parser, parser_name)
self.parser = self.reader.parser
def set_writer(self, writer_name):
"""Set `self.writer` by name."""
writer_class = writers.get_writer_class(writer_name)
self.writer = writer_class()
def set_components(self, reader_name, parser_name, writer_name):
if self.reader is None:
self.set_reader(reader_name, self.parser, parser_name)
if self.parser is None:
if self.reader.parser is None:
self.reader.set_parser(parser_name)
self.parser = self.reader.parser
if self.writer is None:
self.set_writer(writer_name)
def setup_option_parser(self, usage=None, description=None,
settings_spec=None, config_section=None,
**defaults):
if config_section:
if not settings_spec:
settings_spec = SettingsSpec()
settings_spec.config_section = config_section
parts = config_section.split()
if len(parts) > 1 and parts[-1] == 'application':
settings_spec.config_section_dependencies = ['applications']
#@@@ Add self.source & self.destination to components in future?
option_parser = OptionParser(
components=(self.parser, self.reader, self.writer, settings_spec),
defaults=defaults, read_config_files=True,
usage=usage, description=description)
return option_parser
def get_settings(self, usage=None, description=None,
settings_spec=None, config_section=None, **defaults):
"""
Set and return default settings (overrides in `defaults` dict).
Set components first (`self.set_reader` & `self.set_writer`).
Explicitly setting `self.settings` disables command line option
processing from `self.publish()`.
"""
option_parser = self.setup_option_parser(
usage, description, settings_spec, config_section, **defaults)
self.settings = option_parser.get_default_values()
return self.settings
def process_programmatic_settings(self, settings_spec,
settings_overrides,
config_section):
if self.settings is None:
defaults = (settings_overrides or {}).copy()
# Propagate exceptions by default when used programmatically:
defaults.setdefault('traceback', True)
self.get_settings(settings_spec=settings_spec,
config_section=config_section,
**defaults)
def process_command_line(self, argv=None, usage=None, description=None,
settings_spec=None, config_section=None,
**defaults):
"""
Pass an empty list to `argv` to avoid reading `sys.argv` (the
default).
Set components first (`self.set_reader` & `self.set_writer`).
"""
option_parser = self.setup_option_parser(
usage, description, settings_spec, config_section, **defaults)
if argv is None:
argv = sys.argv[1:]
# converting to Unicode (Python 3 does this automatically):
if sys.version_info < (3,0):
# TODO: make this failsafe and reversible?
argv_encoding = (frontend.locale_encoding or 'ascii')
argv = [a.decode(argv_encoding) for a in argv]
self.settings = option_parser.parse_args(argv)
def set_io(self, source_path=None, destination_path=None):
if self.source is None:
self.set_source(source_path=source_path)
if self.destination is None:
self.set_destination(destination_path=destination_path)
def set_source(self, source=None, source_path=None):
if source_path is None:
source_path = self.settings._source
else:
self.settings._source = source_path
# Raise IOError instead of system exit with `tracback == True`
# TODO: change io.FileInput's default behaviour and remove this hack
try:
self.source = self.source_class(
source=source, source_path=source_path,
encoding=self.settings.input_encoding)
except TypeError:
self.source = self.source_class(
source=source, source_path=source_path,
encoding=self.settings.input_encoding)
def set_destination(self, destination=None, destination_path=None):
if destination_path is None:
destination_path = self.settings._destination
else:
self.settings._destination = destination_path
self.destination = self.destination_class(
destination=destination, destination_path=destination_path,
encoding=self.settings.output_encoding,
error_handler=self.settings.output_encoding_error_handler)
def apply_transforms(self):
self.document.transformer.populate_from_components(
(self.source, self.reader, self.reader.parser, self.writer,
self.destination))
self.document.transformer.apply_transforms()
def publish(self, argv=None, usage=None, description=None,
settings_spec=None, settings_overrides=None,
config_section=None, enable_exit_status=False):
"""
Process command line options and arguments (if `self.settings` not
already set), run `self.reader` and then `self.writer`. Return
`self.writer`'s output.
"""
exit = None
try:
if self.settings is None:
self.process_command_line(
argv, usage, description, settings_spec, config_section,
**(settings_overrides or {}))
self.set_io()
self.document = self.reader.read(self.source, self.parser,
self.settings)
self.apply_transforms()
output = self.writer.write(self.document, self.destination)
self.writer.assemble_parts()
except SystemExit as error:
exit = 1
exit_status = error.code
except Exception as error:
if not self.settings: # exception too early to report nicely
raise
if self.settings.traceback: # Propagate exceptions?
self.debugging_dumps()
raise
self.report_Exception(error)
exit = True
exit_status = 1
self.debugging_dumps()
if (enable_exit_status and self.document
and (self.document.reporter.max_level
>= self.settings.exit_status_level)):
sys.exit(self.document.reporter.max_level + 10)
elif exit:
sys.exit(exit_status)
return output
def debugging_dumps(self):
if not self.document:
return
if self.settings.dump_settings:
print >>self._stderr, '\n::: Runtime settings:'
print >>self._stderr, pprint.pformat(self.settings.__dict__)
if self.settings.dump_internals:
print >>self._stderr, '\n::: Document internals:'
print >>self._stderr, pprint.pformat(self.document.__dict__)
if self.settings.dump_transforms:
print >>self._stderr, '\n::: Transforms applied:'
print >>self._stderr, (' (priority, transform class, '
'pending node details, keyword args)')
print >>self._stderr, pprint.pformat(
[(priority, '%s.%s' % (xclass.__module__, xclass.__name__),
pending and pending.details, kwargs)
for priority, xclass, pending, kwargs
in self.document.transformer.applied])
if self.settings.dump_pseudo_xml:
print >>self._stderr, '\n::: Pseudo-XML:'
print >>self._stderr, self.document.pformat().encode(
'raw_unicode_escape')
def report_Exception(self, error):
if isinstance(error, utils.SystemMessage):
self.report_SystemMessage(error)
elif isinstance(error, UnicodeEncodeError):
self.report_UnicodeError(error)
elif isinstance(error, io.InputError):
self._stderr.write(u'Unable to open source file for reading:\n'
u' %s\n' % ErrorString(error))
elif isinstance(error, io.OutputError):
self._stderr.write(
u'Unable to open destination file for writing:\n'
u' %s\n' % ErrorString(error))
else:
print >>self._stderr, u'%s' % ErrorString(error)
print >>self._stderr, ("""\
Exiting due to error. Use "--traceback" to diagnose.
Please report errors to <*****@*****.**>.
Include "--traceback" output, Docutils version (%s%s),
Python version (%s), your OS type & version, and the
command line used.""" % (__version__,
docutils.__version_details__ and
' [%s]'%docutils.__version_details__ or '',
sys.version.split()[0]))
def report_SystemMessage(self, error):
print >>self._stderr, ('Exiting due to level-%s (%s) system message.'
% (error.level,
utils.Reporter.levels[error.level]))
def report_UnicodeError(self, error):
data = error.object[error.start:error.end]
self._stderr.write(
'%s\n'
'\n'
'The specified output encoding (%s) cannot\n'
'handle all of the output.\n'
'Try setting "--output-encoding-error-handler" to\n'
'\n'
'* "xmlcharrefreplace" (for HTML & XML output);\n'
' the output will contain "%s" and should be usable.\n'
'* "backslashreplace" (for other output formats);\n'
' look for "%s" in the output.\n'
'* "replace"; look for "?" in the output.\n'
'\n'
'"--output-encoding-error-handler" is currently set to "%s".\n'
'\n'
'Exiting due to error. Use "--traceback" to diagnose.\n'
'If the advice above doesn\'t eliminate the error,\n'
'please report it to <*****@*****.**>.\n'
'Include "--traceback" output, Docutils version (%s),\n'
'Python version (%s), your OS type & version, and the\n'
'command line used.\n'
% (ErrorString(error),
self.settings.output_encoding,
data.encode('ascii', 'xmlcharrefreplace'),
data.encode('ascii', 'backslashreplace'),
self.settings.output_encoding_error_handler,
__version__, sys.version.split()[0]))
class ConfigParser(CP.RawConfigParser):
old_settings = {
'pep_stylesheet': ('pep_html writer', 'stylesheet'),
'pep_stylesheet_path': ('pep_html writer', 'stylesheet_path'),
'pep_template': ('pep_html writer', 'template')}
"""{old setting: (new section, new setting)} mapping, used by
`handle_old_config`, to convert settings from the old [options] section."""
old_warning = """
The "[option]" section is deprecated. Support for old-format configuration
files may be removed in a future Docutils release. Please revise your
configuration files. See <http://docutils.sf.net/docs/user/config.html>,
section "Old-Format Configuration Files".
"""
not_utf8_error = """\
Unable to read configuration file "%s": content not encoded as UTF-8.
Skipping "%s" configuration file.
"""
def __init__(self, *args, **kwargs):
CP.RawConfigParser.__init__(self, *args, **kwargs)
self._files = []
"""List of paths of configuration files read."""
self._stderr = ErrorOutput()
"""Wrapper around sys.stderr catching en-/decoding errors"""
def read(self, filenames, option_parser):
if type(filenames) in (str, str):
filenames = [filenames]
for filename in filenames:
try:
# Config files must be UTF-8-encoded:
fp = codecs.open(filename, 'r', 'utf-8')
except IOError:
continue
try:
if sys.version_info < (3,2):
CP.RawConfigParser.readfp(self, fp, filename)
else:
CP.RawConfigParser.read_file(self, fp, filename)
except UnicodeDecodeError:
self._stderr.write(self.not_utf8_error % (filename, filename))
fp.close()
continue
fp.close()
self._files.append(filename)
if self.has_section('options'):
self.handle_old_config(filename)
self.validate_settings(filename, option_parser)
def handle_old_config(self, filename):
warnings.warn_explicit(self.old_warning, ConfigDeprecationWarning,
filename, 0)
options = self.get_section('options')
if not self.has_section('general'):
self.add_section('general')
for key, value in list(options.items()):
if key in self.old_settings:
section, setting = self.old_settings[key]
if not self.has_section(section):
self.add_section(section)
else:
section = 'general'
setting = key
if not self.has_option(section, setting):
self.set(section, setting, value)
self.remove_section('options')
def validate_settings(self, filename, option_parser):
"""
Call the validator function and implement overrides on all applicable
settings.
"""
for section in self.sections():
for setting in self.options(section):
try:
option = option_parser.get_option_by_dest(setting)
except KeyError:
continue
if option.validator:
value = self.get(section, setting)
try:
new_value = option.validator(
setting, value, option_parser,
config_parser=self, config_section=section)
except Exception as error:
raise ValueError(
'Error in config file "%s", section "[%s]":\n'
' %s\n'
' %s = %s'
% (filename, section, ErrorString(error),
setting, value))
self.set(section, setting, new_value)
if option.overrides:
self.set(section, option.overrides, None)
def optionxform(self, optionstr):
"""
Transform '-' to '_' so the cmdline form of option names can be used.
"""
return optionstr.lower().replace('-', '_')
def get_section(self, section):
"""
Return a given section as a dictionary (empty if the section
doesn't exist).
"""
section_dict = {}
if self.has_section(section):
for option in self.options(section):
section_dict[option] = self.get(section, option)
return section_dict
class Publisher(object):
"""
A facade encapsulating the high-level logic of a Docutils system.
"""
def __init__(self,
reader=None,
parser=None,
writer=None,
source=None,
source_class=io.FileInput,
destination=None,
destination_class=io.FileOutput,
settings=None):
"""
Initial setup. If any of `reader`, `parser`, or `writer` are not
specified, the corresponding ``set_...`` method should be called with
a component name (`set_reader` sets the parser as well).
"""
self.document = None
"""The document tree (`docutils.nodes` objects)."""
self.reader = reader
"""A `docutils.readers.Reader` instance."""
self.parser = parser
"""A `docutils.parsers.Parser` instance."""
self.writer = writer
"""A `docutils.writers.Writer` instance."""
for component in 'reader', 'parser', 'writer':
assert not isinstance(getattr(self, component), str), (
'passed string "%s" as "%s" parameter; pass an instance, '
'or use the "%s_name" parameter instead (in '
'docutils.core.publish_* convenience functions).' %
(getattr(self, component), component, component))
self.source = source
"""The source of input data, a `docutils.io.Input` instance."""
self.source_class = source_class
"""The class for dynamically created source objects."""
self.destination = destination
"""The destination for docutils output, a `docutils.io.Output`
instance."""
self.destination_class = destination_class
"""The class for dynamically created destination objects."""
self.settings = settings
"""An object containing Docutils settings as instance attributes.
Set by `self.process_command_line()` or `self.get_settings()`."""
self._stderr = ErrorOutput()
def set_reader(self, reader_name, parser, parser_name):
"""Set `self.reader` by name."""
reader_class = readers.get_reader_class(reader_name)
self.reader = reader_class(parser, parser_name)
self.parser = self.reader.parser
def set_writer(self, writer_name):
"""Set `self.writer` by name."""
writer_class = writers.get_writer_class(writer_name)
self.writer = writer_class()
def set_components(self, reader_name, parser_name, writer_name):
if self.reader is None:
self.set_reader(reader_name, self.parser, parser_name)
if self.parser is None:
if self.reader.parser is None:
self.reader.set_parser(parser_name)
self.parser = self.reader.parser
if self.writer is None:
self.set_writer(writer_name)
def setup_option_parser(self,
usage=None,
description=None,
settings_spec=None,
config_section=None,
**defaults):
if config_section:
if not settings_spec:
settings_spec = SettingsSpec()
settings_spec.config_section = config_section
parts = config_section.split()
if len(parts) > 1 and parts[-1] == 'application':
settings_spec.config_section_dependencies = ['applications']
# @@@ Add self.source & self.destination to components in future?
option_parser = OptionParser(components=(self.parser, self.reader,
self.writer, settings_spec),
defaults=defaults,
read_config_files=True,
usage=usage,
description=description)
return option_parser
def get_settings(self,
usage=None,
description=None,
settings_spec=None,
config_section=None,
**defaults):
"""
Set and return default settings (overrides in `defaults` dict).
Set components first (`self.set_reader` & `self.set_writer`).
Explicitly setting `self.settings` disables command line option
processing from `self.publish()`.
"""
option_parser = self.setup_option_parser(usage, description,
settings_spec, config_section,
**defaults)
self.settings = option_parser.get_default_values()
return self.settings
def process_programmatic_settings(self, settings_spec, settings_overrides,
config_section):
if self.settings is None:
defaults = (settings_overrides or {}).copy()
# Propagate exceptions by default when used programmatically:
defaults.setdefault('traceback', True)
self.get_settings(settings_spec=settings_spec,
config_section=config_section,
**defaults)
def process_command_line(self,
argv=None,
usage=None,
description=None,
settings_spec=None,
config_section=None,
**defaults):
"""
Pass an empty list to `argv` to avoid reading `sys.argv` (the
default).
Set components first (`self.set_reader` & `self.set_writer`).
"""
option_parser = self.setup_option_parser(usage, description,
settings_spec, config_section,
**defaults)
if argv is None:
argv = sys.argv[1:]
# converting to Unicode (Python 3 does this automatically):
if sys.version_info < (3, 0):
# TODO: make this failsafe and reversible?
argv_encoding = (frontend.locale_encoding or 'ascii')
argv = [a.decode(argv_encoding) for a in argv]
self.settings = option_parser.parse_args(argv)
def set_io(self, source_path=None, destination_path=None):
if self.source is None:
self.set_source(source_path=source_path)
if self.destination is None:
self.set_destination(destination_path=destination_path)
def set_source(self, source=None, source_path=None):
if source_path is None:
source_path = self.settings._source
else:
self.settings._source = source_path
# Raise IOError instead of system exit with `tracback == True`
# TODO: change io.FileInput's default behaviour and remove this hack
try:
self.source = self.source_class(
source=source,
source_path=source_path,
encoding=self.settings.input_encoding)
except TypeError:
self.source = self.source_class(
source=source,
source_path=source_path,
encoding=self.settings.input_encoding)
def set_destination(self, destination=None, destination_path=None):
if destination_path is None:
destination_path = self.settings._destination
else:
self.settings._destination = destination_path
self.destination = self.destination_class(
destination=destination,
destination_path=destination_path,
encoding=self.settings.output_encoding,
error_handler=self.settings.output_encoding_error_handler)
def apply_transforms(self):
self.document.transformer.populate_from_components(
(self.source, self.reader, self.reader.parser, self.writer,
self.destination))
self.document.transformer.apply_transforms()
def publish(self,
argv=None,
usage=None,
description=None,
settings_spec=None,
settings_overrides=None,
config_section=None,
enable_exit_status=False):
"""
Process command line options and arguments (if `self.settings` not
already set), run `self.reader` and then `self.writer`. Return
`self.writer`'s output.
"""
exit = None
try:
if self.settings is None:
self.process_command_line(argv, usage, description,
settings_spec, config_section,
**(settings_overrides or {}))
self.set_io()
self.document = self.reader.read(self.source, self.parser,
self.settings)
self.apply_transforms()
output = self.writer.write(self.document, self.destination)
self.writer.assemble_parts()
except SystemExit as error:
exit = 1
exit_status = error.code
except Exception as error:
if not self.settings: # exception too early to report nicely
raise
if self.settings.traceback: # Propagate exceptions?
self.debugging_dumps()
raise
self.report_Exception(error)
exit = True
exit_status = 1
self.debugging_dumps()
if (enable_exit_status and self.document
and (self.document.reporter.max_level >=
self.settings.exit_status_level)):
sys.exit(self.document.reporter.max_level + 10)
elif exit:
sys.exit(exit_status)
return output
def debugging_dumps(self):
if not self.document:
return
if self.settings.dump_settings:
print('\n::: Runtime settings:', file=self._stderr)
print(pprint.pformat(self.settings.__dict__), file=self._stderr)
if self.settings.dump_internals:
print('\n::: Document internals:', file=self._stderr)
print(pprint.pformat(self.document.__dict__), file=self._stderr)
if self.settings.dump_transforms:
print('\n::: Transforms applied:', file=self._stderr)
print(
' (priority, transform class, pending node details, '
'keyword args)',
file=self._stderr)
print(pprint.pformat([
(priority, '%s.%s' % (xclass.__module__, xclass.__name__),
pending and pending.details, kwargs) for priority, xclass,
pending, kwargs in self.document.transformer.applied
]),
file=self._stderr)
if self.settings.dump_pseudo_xml:
print('\n::: Pseudo-XML:', file=self._stderr)
print(self.document.pformat().encode('raw_unicode_escape'),
file=self._stderr)
def report_Exception(self, error):
if isinstance(error, utils.SystemMessage):
self.report_SystemMessage(error)
elif isinstance(error, UnicodeEncodeError):
self.report_UnicodeError(error)
elif isinstance(error, io.InputError):
self._stderr.write(u'Unable to open source file for reading:\n'
u' %s\n' % ErrorString(error))
elif isinstance(error, io.OutputError):
self._stderr.write(
u'Unable to open destination file for writing:\n'
u' %s\n' % ErrorString(error))
else:
print(u'%s' % ErrorString(error), file=self._stderr)
print(
("""\
Exiting due to error. Use "--traceback" to diagnose.
Please report errors to <*****@*****.**>.
Include "--traceback" output, Docutils version (%s%s),
Python version (%s), your OS type & version, and the
command line used.""" %
(__version__, docutils.__version_details__ and ' [%s]' %
docutils.__version_details__ or '', sys.version.split()[0])),
file=self._stderr)
def report_SystemMessage(self, error):
print('Exiting due to level-%s (%s) system message.' %
(error.level, utils.Reporter.levels[error.level]),
file=self._stderr)
def report_UnicodeError(self, error):
data = error.object[error.start:error.end]
self._stderr.write(
'%s\n'
'\n'
'The specified output encoding (%s) cannot\n'
'handle all of the output.\n'
'Try setting "--output-encoding-error-handler" to\n'
'\n'
'* "xmlcharrefreplace" (for HTML & XML output);\n'
' the output will contain "%s" and should be usable.\n'
'* "backslashreplace" (for other output formats);\n'
' look for "%s" in the output.\n'
'* "replace"; look for "?" in the output.\n'
'\n'
'"--output-encoding-error-handler" is currently set to "%s".\n'
'\n'
'Exiting due to error. Use "--traceback" to diagnose.\n'
'If the advice above doesn\'t eliminate the error,\n'
'please report it to <*****@*****.**>.\n'
'Include "--traceback" output, Docutils version (%s),\n'
'Python version (%s), your OS type & version, and the\n'
'command line used.\n' %
(ErrorString(error), self.settings.output_encoding,
data.encode('ascii', 'xmlcharrefreplace'),
data.encode('ascii', 'backslashreplace'),
self.settings.output_encoding_error_handler, __version__,
sys.version.split()[0]))
class ConfigParser(CP.RawConfigParser):
old_settings = {
"pep_stylesheet": ("pep_html writer", "stylesheet"),
"pep_stylesheet_path": ("pep_html writer", "stylesheet_path"),
"pep_template": ("pep_html writer", "template"),
}
"""{old setting: (new section, new setting)} mapping, used by
`handle_old_config`, to convert settings from the old [options] section."""
old_warning = """
The "[option]" section is deprecated. Support for old-format configuration
files may be removed in a future Docutils release. Please revise your
configuration files. See <http://docutils.sf.net/docs/user/config.html>,
section "Old-Format Configuration Files".
"""
not_utf8_error = """\
Unable to read configuration file "%s": content not encoded as UTF-8.
Skipping "%s" configuration file.
"""
def __init__(self, *args, **kwargs):
CP.RawConfigParser.__init__(self, *args, **kwargs)
self._files = []
"""List of paths of configuration files read."""
self._stderr = ErrorOutput()
"""Wrapper around sys.stderr catching en-/decoding errors"""
def read(self, filenames, option_parser):
if type(filenames) in (str, str):
filenames = [filenames]
for filename in filenames:
try:
# Config files must be UTF-8-encoded:
fp = codecs.open(filename, "r", "utf-8")
except IOError:
continue
try:
if sys.version_info < (3, 2):
CP.RawConfigParser.readfp(self, fp, filename)
else:
CP.RawConfigParser.read_file(self, fp, filename)
except UnicodeDecodeError:
self._stderr.write(self.not_utf8_error % (filename, filename))
fp.close()
continue
fp.close()
self._files.append(filename)
if self.has_section("options"):
self.handle_old_config(filename)
self.validate_settings(filename, option_parser)
def handle_old_config(self, filename):
warnings.warn_explicit(self.old_warning, ConfigDeprecationWarning, filename, 0)
options = self.get_section("options")
if not self.has_section("general"):
self.add_section("general")
for key, value in list(options.items()):
if key in self.old_settings:
section, setting = self.old_settings[key]
if not self.has_section(section):
self.add_section(section)
else:
section = "general"
setting = key
if not self.has_option(section, setting):
self.set(section, setting, value)
self.remove_section("options")
def validate_settings(self, filename, option_parser):
"""
Call the validator function and implement overrides on all applicable
settings.
"""
for section in self.sections():
for setting in self.options(section):
try:
option = option_parser.get_option_by_dest(setting)
except KeyError:
continue
if option.validator:
value = self.get(section, setting)
try:
new_value = option.validator(
setting, value, option_parser, config_parser=self, config_section=section
)
except Exception as error:
raise ValueError(
'Error in config file "%s", section "[%s]":\n'
" %s\n"
" %s = %s" % (filename, section, ErrorString(error), setting, value)
)
self.set(section, setting, new_value)
if option.overrides:
self.set(section, option.overrides, None)
def optionxform(self, optionstr):
"""
Transform '-' to '_' so the cmdline form of option names can be used.
"""
return optionstr.lower().replace("-", "_")
def get_section(self, section):
"""
Return a given section as a dictionary (empty if the section
doesn't exist).
"""
section_dict = {}
if self.has_section(section):
for option in self.options(section):
section_dict[option] = self.get(section, option)
return section_dict
class ConfigParser(CP.RawConfigParser):
old_settings = {
'pep_stylesheet': ('pep_html writer', 'stylesheet'),
'pep_stylesheet_path': ('pep_html writer', 'stylesheet_path'),
'pep_template': ('pep_html writer', 'template')}
"""{old setting: (new section, new setting)} mapping, used by
`handle_old_config`, to convert settings from the old [options] section."""
old_warning = """
The "[option]" section is deprecated. Support for old-format configuration
files may be removed in a future Docutils release. Please revise your
configuration files. See <http://docutils.sf.net/docs/user/config.html>,
section "Old-Format Configuration Files".
"""
not_utf8_error = """\
Unable to read configuration file "%s": content not encoded as UTF-8.
Skipping "%s" configuration file.
"""
def __init__(self, *args, **kwargs):
CP.RawConfigParser.__init__(self, *args, **kwargs)
self._files = []
"""List of paths of configuration files read."""
self._stderr = ErrorOutput()
"""Wrapper around sys.stderr catching en-/decoding errors"""
def read(self, filenames, option_parser):
if type(filenames) in (str, unicode):
filenames = [filenames]
for filename in filenames:
try:
# Config files must be UTF-8-encoded:
fp = codecs.open(filename, 'r', 'utf-8')
except IOError:
continue
try:
if sys.version_info < (3,2):
CP.RawConfigParser.readfp(self, fp, filename)
else:
CP.RawConfigParser.read_file(self, fp, filename)
except UnicodeDecodeError:
self._stderr.write(self.not_utf8_error % (filename, filename))
fp.close()
continue
fp.close()
self._files.append(filename)
if self.has_section('options'):
self.handle_old_config(filename)
self.validate_settings(filename, option_parser)
def handle_old_config(self, filename):
warnings.warn_explicit(self.old_warning, ConfigDeprecationWarning,
filename, 0)
options = self.get_section('options')
if not self.has_section('general'):
self.add_section('general')
for key, value in options.items():
if key in self.old_settings:
section, setting = self.old_settings[key]
if not self.has_section(section):
self.add_section(section)
else:
section = 'general'
setting = key
if not self.has_option(section, setting):
self.set(section, setting, value)
self.remove_section('options')
def validate_settings(self, filename, option_parser):
"""
Call the validator function and implement overrides on all applicable
settings.
"""
for section in self.sections():
for setting in self.options(section):
try:
option = option_parser.get_option_by_dest(setting)
except KeyError:
continue
if option.validator:
value = self.get(section, setting)
try:
new_value = option.validator(
setting, value, option_parser,
config_parser=self, config_section=section)
except Exception, error:
raise (ValueError(
'Error in config file "%s", section "[%s]":\n'
' %s\n'
' %s = %s'
% (filename, section, ErrorString(error),
setting, value)), None, sys.exc_info()[2])
self.set(section, setting, new_value)
if option.overrides:
self.set(section, option.overrides, None)
def test_ubuf(self):
buf = UBuf() # buffer only accepting unicode string
# decode of binary strings
e = ErrorOutput(buf, encoding='ascii')
e.write(b'b\xfc')
self.assertEqual(buf.getvalue(), u'b\ufffd') # use REPLACEMENT CHARACTER
# write Unicode string and Exceptions with Unicode args
e.write(u' u\xfc')
self.assertEqual(buf.getvalue(), u'b\ufffd u\xfc')
e.write(AttributeError(u' e\xfc'))
self.assertEqual(buf.getvalue(), u'b\ufffd u\xfc e\xfc')
# decode with `encoding` attribute
e.encoding = 'latin1'
e.write(b' b\xfc')
self.assertEqual(buf.getvalue(), u'b\ufffd u\xfc e\xfc b\xfc')
def test_bbuf(self):
buf = BBuf() # buffer storing byte string
e = ErrorOutput(buf, encoding='ascii')
# write byte-string as-is
e.write(b'b\xfc')
self.assertEqual(buf.getvalue(), b'b\xfc')
# encode unicode data with backslashescape fallback replacement:
e.write(u' u\xfc')
self.assertEqual(buf.getvalue(), b'b\xfc u\\xfc')
# handle Exceptions with Unicode string args
# unicode(Exception(u'e\xfc')) # fails in Python < 2.6
e.write(AttributeError(u' e\xfc'))
self.assertEqual(buf.getvalue(), b'b\xfc u\\xfc e\\xfc')
# encode with `encoding` attribute
e.encoding = 'utf8'
e.write(u' u\xfc')
self.assertEqual(buf.getvalue(), b'b\xfc u\\xfc e\\xfc u\xc3\xbc')
def test_defaults(self):
e = ErrorOutput()
self.assertEqual(e.stream, sys.stderr)
