def process_txt(self, directory, name): if name.startswith("pep-"): publisher = "PEPs" else: publisher = ".txt" settings = self.get_settings(publisher, directory) errout = ErrorOutput(encoding=settings.error_encoding) pub_struct = self.publishers[publisher] settings._source = os.path.normpath(os.path.join(directory, name)) settings._destination = settings._source[:-4] + ".html" if not self.initial_settings.silent: errout.write(" ::: Processing: %s\n" % name) sys.stderr.flush() try: if not settings.dry_run: core.publish_file( source_path=settings._source, destination_path=settings._destination, reader_name=pub_struct.reader_name, parser_name="restructuredtext", writer_name=pub_struct.writer_name, settings=settings, ) except ApplicationError: error = sys.exc_info()[1] # get exception in Python <2.6 and 3.x errout.write(" %s\n" % ErrorString(error))
def __init__(self, *args, **kwargs): CP.RawConfigParser.__init__(self, *args, **kwargs) self._files = [] """List of paths of configuration files read.""" self._stderr = ErrorOutput() """Wrapper around sys.stderr catching en-/decoding errors"""
def __init__( self, reader=None, parser=None, writer=None, source=None, source_class=io.FileInput, destination=None, destination_class=io.FileOutput, settings=None, ): """ Initial setup. If any of `reader`, `parser`, or `writer` are not specified, the corresponding ``set_...`` method should be called with a component name (`set_reader` sets the parser as well). """ self.document = None """The document tree (`docutils.nodes` objects).""" self.reader = reader """A `docutils.readers.Reader` instance.""" self.parser = parser """A `docutils.parsers.Parser` instance.""" self.writer = writer """A `docutils.writers.Writer` instance.""" for component in "reader", "parser", "writer": assert not isinstance(getattr(self, component), str), ( 'passed string "%s" as "%s" parameter; pass an instance, ' 'or use the "%s_name" parameter instead (in ' "docutils.core.publish_* convenience functions)." % (getattr(self, component), component, component)) self.source = source """The source of input data, a `docutils.io.Input` instance.""" self.source_class = source_class """The class for dynamically created source objects.""" self.destination = destination """The destination for docutils output, a `docutils.io.Output` instance.""" self.destination_class = destination_class """The class for dynamically created destination objects.""" self.settings = settings """An object containing Docutils settings as instance attributes. Set by `self.process_command_line()` or `self.get_settings()`.""" self._stderr = ErrorOutput()
def test_ubuf(self): buf = UBuf() # buffer only accepting unicode string # decode of binary strings e = ErrorOutput(buf, encoding='ascii') e.write(b('b\xfc')) self.assertEqual(buf.getvalue(), 'b\ufffd') # use REPLACEMENT CHARACTER # write Unicode string and Exceptions with Unicode args e.write(' u\xfc') self.assertEqual(buf.getvalue(), 'b\ufffd u\xfc') e.write(AttributeError(' e\xfc')) self.assertEqual(buf.getvalue(), 'b\ufffd u\xfc e\xfc') # decode with `encoding` attribute e.encoding = 'latin1' e.write(b(' b\xfc')) self.assertEqual(buf.getvalue(), 'b\ufffd u\xfc e\xfc b\xfc')
def __init__(self, source=None, source_path=None, encoding=None, error_handler='strict', autoclose=True, mode='rU'): """ :Parameters: - `source`: either a file-like object (which is read directly), or `None` (which implies `sys.stdin` if no `source_path` given). - `source_path`: a path to a file, which is opened and then read. - `encoding`: the expected text encoding of the input file. - `error_handler`: the encoding error handler to use. - `autoclose`: close automatically after read (except when `sys.stdin` is the source). - `mode`: how the file is to be opened (see standard function `open`). The default 'rU' provides universal newline support for text files. """ Input.__init__(self, source, source_path, encoding, error_handler) self.autoclose = autoclose self._stderr = ErrorOutput() if source is None: if source_path: # Specify encoding in Python 3 if sys.version_info >= (3,0): kwargs = {'encoding': self.encoding, 'errors': self.error_handler} else: kwargs = {} try: self.source = open(source_path, mode, **kwargs) except IOError, error: raise InputError(error.errno, error.strerror, source_path) else: self.source = sys.stdin
def test_bbuf(self): buf = BBuf() # buffer storing byte string e = ErrorOutput(buf, encoding='ascii') # write byte-string as-is e.write(b('b\xfc')) self.assertEqual(buf.getvalue(), b('b\xfc')) # encode unicode data with backslashescape fallback replacement: e.write(' u\xfc') self.assertEqual(buf.getvalue(), b('b\xfc u\\xfc')) # handle Exceptions with Unicode string args # unicode(Exception(u'e\xfc')) # fails in Python < 2.6 e.write(AttributeError(' e\xfc')) self.assertEqual(buf.getvalue(), b('b\xfc u\\xfc e\\xfc')) # encode with `encoding` attribute e.encoding = 'utf8' e.write(' u\xfc') self.assertEqual(buf.getvalue(), b('b\xfc u\\xfc e\\xfc u\xc3\xbc'))
def __init__( self, source, report_level, halt_level, stream=None, debug=False, encoding=None, error_handler="backslashreplace", ): """ :Parameters: - `source`: The path to or description of the source data. - `report_level`: The level at or above which warning output will be sent to `stream`. - `halt_level`: The level at or above which `SystemMessage` exceptions will be raised, halting execution. - `debug`: Show debug (level=0) system messages? - `stream`: Where warning output is sent. Can be file-like (has a ``.write`` method), a string (file name, opened for writing), '' (empty string) or `False` (for discarding all stream messages) or `None` (implies `sys.stderr`; default). - `encoding`: The output encoding. - `error_handler`: The error handler for stderr output encoding. """ self.source = source """The path to or description of the source data.""" self.error_handler = error_handler """The character encoding error handler.""" self.debug_flag = debug """Show debug (level=0) system messages?""" self.report_level = report_level """The level at or above which warning output will be sent to `self.stream`.""" self.halt_level = halt_level """The level at or above which `SystemMessage` exceptions will be raised, halting execution.""" if not isinstance(stream, ErrorOutput): stream = ErrorOutput(stream, encoding, error_handler) self.stream = stream """Where warning output is sent.""" self.encoding = encoding or getattr(stream, "encoding", "ascii") """The output character encoding.""" self.observers = [] """List of bound methods or functions to call with each system_message created.""" self.max_level = -1 """The highest level system message generated so far."""
def __init__(self, source=None, source_path=None, encoding=None, error_handler='strict', autoclose=True, mode='r' if sys.version_info >= (3, 0) else 'rU', **kwargs): """ :Parameters: - `source`: either a file-like object (which is read directly), or `None` (which implies `sys.stdin` if no `source_path` given). - `source_path`: a path to a file, which is opened and then read. - `encoding`: the expected text encoding of the input file. - `error_handler`: the encoding error handler to use. - `autoclose`: close automatically after read (except when `sys.stdin` is the source). - `mode`: how the file is to be opened (see standard function `open`). The default 'rU' provides universal newline support for text files on Python < 3.4. """ Input.__init__(self, source, source_path, encoding, error_handler) self.autoclose = autoclose self._stderr = ErrorOutput() # deprecation warning for key in kwargs: if key == 'handle_io_errors': sys.stderr.write('deprecation warning: ' 'io.FileInput() argument `handle_io_errors` ' 'is ignored since Docutils 0.10 (2012-12-16) ' 'and will soon be removed.') else: raise TypeError('__init__() got an unexpected keyword ' "argument '%s'" % key) if source is None: if source_path: # Specify encoding in Python 3 if sys.version_info >= (3, 0): kwargs = {'encoding': self.encoding, 'errors': self.error_handler} else: kwargs = {} try: self.source = open(source_path, mode, **kwargs) except IOError as error: raise InputError(error.errno, error.strerror, source_path) else: self.source = sys.stdin elif (sys.version_info >= (3, 0) and check_encoding(self.source, self.encoding) is False): # TODO: re-open, warn or raise error? raise UnicodeError('Encoding clash: encoding given is "%s" ' 'but source is opened with encoding "%s".' % (self.encoding, self.source.encoding)) if not source_path: try: self.source_path = self.source.name except AttributeError: pass
def set_conditions(self, category, report_level, halt_level, stream=None, debug=False): warnings.warn('docutils.utils.Reporter.set_conditions deprecated; ' 'set attributes via configuration settings or directly', DeprecationWarning, stacklevel=2) self.report_level = report_level self.halt_level = halt_level if not isinstance(stream, ErrorOutput): stream = ErrorOutput(stream, self.encoding, self.error_handler) self.stream = stream self.debug_flag = debug
def __init__( self, reader=None, parser=None, writer=None, source=None, source_class=io.FileInput, destination=None, destination_class=io.FileOutput, settings=None, ): """ Initial setup. If any of `reader`, `parser`, or `writer` are not specified, the corresponding ``set_...`` method should be called with a component name (`set_reader` sets the parser as well). """ self.document = None """The document tree (`docutils.nodes` objects).""" self.reader = reader """A `docutils.readers.Reader` instance.""" self.parser = parser """A `docutils.parsers.Parser` instance.""" self.writer = writer """A `docutils.writers.Writer` instance.""" for component in "reader", "parser", "writer": assert not isinstance(getattr(self, component), str), ( 'passed string "%s" as "%s" parameter; pass an instance, ' 'or use the "%s_name" parameter instead (in ' "docutils.core.publish_* convenience functions)." % (getattr(self, component), component, component) ) self.source = source """The source of input data, a `docutils.io.Input` instance.""" self.source_class = source_class """The class for dynamically created source objects.""" self.destination = destination """The destination for docutils output, a `docutils.io.Output` instance.""" self.destination_class = destination_class """The class for dynamically created destination objects.""" self.settings = settings """An object containing Docutils settings as instance attributes. Set by `self.process_command_line()` or `self.get_settings()`.""" self._stderr = ErrorOutput()
def __init__(self, destination=None, destination_path=None, encoding=None, error_handler='strict', autoclose=True, handle_io_errors=None, mode=None): """ :Parameters: - `destination`: either a file-like object (which is written directly) or `None` (which implies `sys.stdout` if no `destination_path` given). - `destination_path`: a path to a file, which is opened and then written. - `encoding`: the text encoding of the output file. - `error_handler`: the encoding error handler to use. - `autoclose`: close automatically after write (except when `sys.stdout` or `sys.stderr` is the destination). - `handle_io_errors`: ignored, deprecated, will be removed. - `mode`: how the file is to be opened (see standard function `open`). The default is 'w', providing universal newline support for text files. """ Output.__init__(self, destination, destination_path, encoding, error_handler) self.opened = True self.autoclose = autoclose if handle_io_errors is not None: warnings.warn( 'io.FileOutput: initialization argument ' '"handle_io_errors" is ignored and will be removed in ' 'Docutils 1.2.', DeprecationWarning, stacklevel=2) if mode is not None: self.mode = mode self._stderr = ErrorOutput() if destination is None: if destination_path: self.opened = False else: self.destination = sys.stdout elif ( # destination is file-type object -> check mode: mode and hasattr(self.destination, 'mode') and mode != self.destination.mode): print('Warning: Destination mode "%s" differs from specified ' 'mode "%s"' % (self.destination.mode, mode), file=self._stderr) if not destination_path: try: self.destination_path = self.destination.name except AttributeError: pass
def visit(self, directory, names, subdirectories): settings = self.get_settings("", directory) errout = ErrorOutput(encoding=settings.error_encoding) if settings.prune and (os.path.abspath(directory) in settings.prune): errout.write("/// ...Skipping directory (pruned): %s\n" % directory) sys.stderr.flush() del subdirectories[:] return if not self.initial_settings.silent: errout.write("/// Processing directory: %s" % directory) sys.stderr.flush() # settings.ignore grows many duplicate entries as we recurse # if we add patterns in config files or on the command line. for pattern in utils.uniq(settings.ignore): for i in range(len(names) - 1, -1, -1): if fnmatch(names[i], pattern): # Modify in place! del names[i] for name in names: if name.endswith(".txt"): self.process_txt(directory, name)
def __init__( self, source=None, source_path=None, encoding=None, error_handler="strict", autoclose=True, mode="r" if sys.version_info >= (3, 0) else "rU", ): """ :Parameters: - `source`: either a file-like object (which is read directly), or `None` (which implies `sys.stdin` if no `source_path` given). - `source_path`: a path to a file, which is opened and then read. - `encoding`: the expected text encoding of the input file. - `error_handler`: the encoding error handler to use. - `autoclose`: close automatically after read (except when `sys.stdin` is the source). - `mode`: how the file is to be opened (see standard function `open`). The default 'rU' provides universal newline support for text files with Python 2.x. """ Input.__init__(self, source, source_path, encoding, error_handler) self.autoclose = autoclose self._stderr = ErrorOutput() if source is None: if source_path: # Specify encoding in Python 3 if sys.version_info >= (3, 0): kwargs = { "encoding": self.encoding, "errors": self.error_handler } else: kwargs = {} try: self.source = open(source_path, mode, **kwargs) except IOError as error: raise InputError(error.errno, error.strerror, source_path) else: self.source = sys.stdin elif (sys.version_info >= (3, 0) and check_encoding(self.source, self.encoding) is False): # TODO: re-open, warn or raise error? raise UnicodeError('Encoding clash: encoding given is "%s" ' 'but source is opened with encoding "%s".' % (self.encoding, self.source.encoding)) if not source_path: try: self.source_path = self.source.name except AttributeError: pass
def __init__(self, destination=None, destination_path=None, encoding=None, error_handler='strict', autoclose=True, mode=None): """ :Parameters: - `destination`: either a file-like object (which is written directly) or `None` (which implies `sys.stdout` if no `destination_path` given). - `destination_path`: a path to a file, which is opened and then written. - `encoding`: the text encoding of the output file. - `error_handler`: the encoding error handler to use. - `autoclose`: close automatically after write (except when `sys.stdout` or `sys.stderr` is the destination). - `mode`: how the file is to be opened (see standard function `open`). The default is 'w', providing universal newline support for text files. """ Output.__init__(self, destination, destination_path, encoding, error_handler) self.opened = True self.autoclose = autoclose if mode is not None: self.mode = mode self._stderr = ErrorOutput() if destination is None: if destination_path: self.opened = False else: self.destination = sys.stdout elif ( # destination is file-type object -> check mode: mode and hasattr(self.destination, 'mode') and mode != self.destination.mode): print >> self._stderr, ('Warning: Destination mode "%s" ' 'differs from specified mode "%s"' % (self.destination.mode, mode)) if not destination_path: try: self.destination_path = self.destination.name except AttributeError: pass
def visit(self, directory, names, subdirectories): settings = self.get_settings('', directory) errout = ErrorOutput(encoding=settings.error_encoding) if settings.prune and (os.path.abspath(directory) in settings.prune): errout.write('/// ...Skipping directory (pruned): %s\n' % directory) sys.stderr.flush() del subdirectories[:] return if not self.initial_settings.silent: errout.write('/// Processing directory: %s\n' % directory) sys.stderr.flush() # settings.ignore grows many duplicate entries as we recurse # if we add patterns in config files or on the command line. for pattern in utils.uniq(settings.ignore): for i in range(len(names) - 1, -1, -1): if fnmatch(names[i], pattern): # Modify in place! del names[i] for name in names: if name.endswith('.txt'): self.process_txt(directory, name)
def process_txt(self, directory, name): if name.startswith('pep-'): publisher = 'PEPs' else: publisher = '.txt' settings = self.get_settings(publisher, directory) errout = ErrorOutput(encoding=settings.error_encoding) pub_struct = self.publishers[publisher] settings._source = os.path.normpath(os.path.join(directory, name)) settings._destination = settings._source[:-4] + '.html' if not self.initial_settings.silent: errout.write(' ::: Processing: %s\n' % name) sys.stderr.flush() try: if not settings.dry_run: core.publish_file(source_path=settings._source, destination_path=settings._destination, reader_name=pub_struct.reader_name, parser_name='restructuredtext', writer_name=pub_struct.writer_name, settings=settings) except ApplicationError: error = sys.exc_info()[1] # get exception in Python <2.6 and 3.x errout.write(' %s\n' % ErrorString(error))
class Publisher: """ A facade encapsulating the high-level logic of a Docutils system. """ def __init__(self, reader=None, parser=None, writer=None, source=None, source_class=io.FileInput, destination=None, destination_class=io.FileOutput, settings=None): """ Initial setup. If any of `reader`, `parser`, or `writer` are not specified, the corresponding ``set_...`` method should be called with a component name (`set_reader` sets the parser as well). """ self.document = None """The document tree (`docutils.nodes` objects).""" self.reader = reader """A `docutils.readers.Reader` instance.""" self.parser = parser """A `docutils.parsers.Parser` instance.""" self.writer = writer """A `docutils.writers.Writer` instance.""" for component in 'reader', 'parser', 'writer': assert not isinstance(getattr(self, component), str), ( 'passed string "%s" as "%s" parameter; pass an instance, ' 'or use the "%s_name" parameter instead (in ' 'docutils.core.publish_* convenience functions).' % (getattr(self, component), component, component)) self.source = source """The source of input data, a `docutils.io.Input` instance.""" self.source_class = source_class """The class for dynamically created source objects.""" self.destination = destination """The destination for docutils output, a `docutils.io.Output` instance.""" self.destination_class = destination_class """The class for dynamically created destination objects.""" self.settings = settings """An object containing Docutils settings as instance attributes. Set by `self.process_command_line()` or `self.get_settings()`.""" self._stderr = ErrorOutput() def set_reader(self, reader_name, parser, parser_name): """Set `self.reader` by name.""" reader_class = readers.get_reader_class(reader_name) self.reader = reader_class(parser, parser_name) self.parser = self.reader.parser def set_writer(self, writer_name): """Set `self.writer` by name.""" writer_class = writers.get_writer_class(writer_name) self.writer = writer_class() def set_components(self, reader_name, parser_name, writer_name): if self.reader is None: self.set_reader(reader_name, self.parser, parser_name) if self.parser is None: if self.reader.parser is None: self.reader.set_parser(parser_name) self.parser = self.reader.parser if self.writer is None: self.set_writer(writer_name) def setup_option_parser(self, usage=None, description=None, settings_spec=None, config_section=None, **defaults): if config_section: if not settings_spec: settings_spec = SettingsSpec() settings_spec.config_section = config_section parts = config_section.split() if len(parts) > 1 and parts[-1] == 'application': settings_spec.config_section_dependencies = ['applications'] #@@@ Add self.source & self.destination to components in future? option_parser = OptionParser( components=(self.parser, self.reader, self.writer, settings_spec), defaults=defaults, read_config_files=True, usage=usage, description=description) return option_parser def get_settings(self, usage=None, description=None, settings_spec=None, config_section=None, **defaults): """ Set and return default settings (overrides in `defaults` dict). Set components first (`self.set_reader` & `self.set_writer`). Explicitly setting `self.settings` disables command line option processing from `self.publish()`. """ option_parser = self.setup_option_parser( usage, description, settings_spec, config_section, **defaults) self.settings = option_parser.get_default_values() return self.settings def process_programmatic_settings(self, settings_spec, settings_overrides, config_section): if self.settings is None: defaults = (settings_overrides or {}).copy() # Propagate exceptions by default when used programmatically: defaults.setdefault('traceback', True) self.get_settings(settings_spec=settings_spec, config_section=config_section, **defaults) def process_command_line(self, argv=None, usage=None, description=None, settings_spec=None, config_section=None, **defaults): """ Pass an empty list to `argv` to avoid reading `sys.argv` (the default). Set components first (`self.set_reader` & `self.set_writer`). """ option_parser = self.setup_option_parser( usage, description, settings_spec, config_section, **defaults) if argv is None: argv = sys.argv[1:] # converting to Unicode (Python 3 does this automatically): if sys.version_info < (3,0): # TODO: make this failsafe and reversible? argv_encoding = (frontend.locale_encoding or 'ascii') argv = [a.decode(argv_encoding) for a in argv] self.settings = option_parser.parse_args(argv) def set_io(self, source_path=None, destination_path=None): if self.source is None: self.set_source(source_path=source_path) if self.destination is None: self.set_destination(destination_path=destination_path) def set_source(self, source=None, source_path=None): if source_path is None: source_path = self.settings._source else: self.settings._source = source_path # Raise IOError instead of system exit with `tracback == True` # TODO: change io.FileInput's default behaviour and remove this hack try: self.source = self.source_class( source=source, source_path=source_path, encoding=self.settings.input_encoding) except TypeError: self.source = self.source_class( source=source, source_path=source_path, encoding=self.settings.input_encoding) def set_destination(self, destination=None, destination_path=None): if destination_path is None: destination_path = self.settings._destination else: self.settings._destination = destination_path self.destination = self.destination_class( destination=destination, destination_path=destination_path, encoding=self.settings.output_encoding, error_handler=self.settings.output_encoding_error_handler) def apply_transforms(self): self.document.transformer.populate_from_components( (self.source, self.reader, self.reader.parser, self.writer, self.destination)) self.document.transformer.apply_transforms() def publish(self, argv=None, usage=None, description=None, settings_spec=None, settings_overrides=None, config_section=None, enable_exit_status=False): """ Process command line options and arguments (if `self.settings` not already set), run `self.reader` and then `self.writer`. Return `self.writer`'s output. """ exit = None try: if self.settings is None: self.process_command_line( argv, usage, description, settings_spec, config_section, **(settings_overrides or {})) self.set_io() self.document = self.reader.read(self.source, self.parser, self.settings) self.apply_transforms() output = self.writer.write(self.document, self.destination) self.writer.assemble_parts() except SystemExit as error: exit = 1 exit_status = error.code except Exception as error: if not self.settings: # exception too early to report nicely raise if self.settings.traceback: # Propagate exceptions? self.debugging_dumps() raise self.report_Exception(error) exit = True exit_status = 1 self.debugging_dumps() if (enable_exit_status and self.document and (self.document.reporter.max_level >= self.settings.exit_status_level)): sys.exit(self.document.reporter.max_level + 10) elif exit: sys.exit(exit_status) return output def debugging_dumps(self): if not self.document: return if self.settings.dump_settings: print >>self._stderr, '\n::: Runtime settings:' print >>self._stderr, pprint.pformat(self.settings.__dict__) if self.settings.dump_internals: print >>self._stderr, '\n::: Document internals:' print >>self._stderr, pprint.pformat(self.document.__dict__) if self.settings.dump_transforms: print >>self._stderr, '\n::: Transforms applied:' print >>self._stderr, (' (priority, transform class, ' 'pending node details, keyword args)') print >>self._stderr, pprint.pformat( [(priority, '%s.%s' % (xclass.__module__, xclass.__name__), pending and pending.details, kwargs) for priority, xclass, pending, kwargs in self.document.transformer.applied]) if self.settings.dump_pseudo_xml: print >>self._stderr, '\n::: Pseudo-XML:' print >>self._stderr, self.document.pformat().encode( 'raw_unicode_escape') def report_Exception(self, error): if isinstance(error, utils.SystemMessage): self.report_SystemMessage(error) elif isinstance(error, UnicodeEncodeError): self.report_UnicodeError(error) elif isinstance(error, io.InputError): self._stderr.write(u'Unable to open source file for reading:\n' u' %s\n' % ErrorString(error)) elif isinstance(error, io.OutputError): self._stderr.write( u'Unable to open destination file for writing:\n' u' %s\n' % ErrorString(error)) else: print >>self._stderr, u'%s' % ErrorString(error) print >>self._stderr, ("""\ Exiting due to error. Use "--traceback" to diagnose. Please report errors to <*****@*****.**>. Include "--traceback" output, Docutils version (%s%s), Python version (%s), your OS type & version, and the command line used.""" % (__version__, docutils.__version_details__ and ' [%s]'%docutils.__version_details__ or '', sys.version.split()[0])) def report_SystemMessage(self, error): print >>self._stderr, ('Exiting due to level-%s (%s) system message.' % (error.level, utils.Reporter.levels[error.level])) def report_UnicodeError(self, error): data = error.object[error.start:error.end] self._stderr.write( '%s\n' '\n' 'The specified output encoding (%s) cannot\n' 'handle all of the output.\n' 'Try setting "--output-encoding-error-handler" to\n' '\n' '* "xmlcharrefreplace" (for HTML & XML output);\n' ' the output will contain "%s" and should be usable.\n' '* "backslashreplace" (for other output formats);\n' ' look for "%s" in the output.\n' '* "replace"; look for "?" in the output.\n' '\n' '"--output-encoding-error-handler" is currently set to "%s".\n' '\n' 'Exiting due to error. Use "--traceback" to diagnose.\n' 'If the advice above doesn\'t eliminate the error,\n' 'please report it to <*****@*****.**>.\n' 'Include "--traceback" output, Docutils version (%s),\n' 'Python version (%s), your OS type & version, and the\n' 'command line used.\n' % (ErrorString(error), self.settings.output_encoding, data.encode('ascii', 'xmlcharrefreplace'), data.encode('ascii', 'backslashreplace'), self.settings.output_encoding_error_handler, __version__, sys.version.split()[0]))
class ConfigParser(CP.RawConfigParser): old_settings = { 'pep_stylesheet': ('pep_html writer', 'stylesheet'), 'pep_stylesheet_path': ('pep_html writer', 'stylesheet_path'), 'pep_template': ('pep_html writer', 'template')} """{old setting: (new section, new setting)} mapping, used by `handle_old_config`, to convert settings from the old [options] section.""" old_warning = """ The "[option]" section is deprecated. Support for old-format configuration files may be removed in a future Docutils release. Please revise your configuration files. See <http://docutils.sf.net/docs/user/config.html>, section "Old-Format Configuration Files". """ not_utf8_error = """\ Unable to read configuration file "%s": content not encoded as UTF-8. Skipping "%s" configuration file. """ def __init__(self, *args, **kwargs): CP.RawConfigParser.__init__(self, *args, **kwargs) self._files = [] """List of paths of configuration files read.""" self._stderr = ErrorOutput() """Wrapper around sys.stderr catching en-/decoding errors""" def read(self, filenames, option_parser): if type(filenames) in (str, str): filenames = [filenames] for filename in filenames: try: # Config files must be UTF-8-encoded: fp = codecs.open(filename, 'r', 'utf-8') except IOError: continue try: if sys.version_info < (3,2): CP.RawConfigParser.readfp(self, fp, filename) else: CP.RawConfigParser.read_file(self, fp, filename) except UnicodeDecodeError: self._stderr.write(self.not_utf8_error % (filename, filename)) fp.close() continue fp.close() self._files.append(filename) if self.has_section('options'): self.handle_old_config(filename) self.validate_settings(filename, option_parser) def handle_old_config(self, filename): warnings.warn_explicit(self.old_warning, ConfigDeprecationWarning, filename, 0) options = self.get_section('options') if not self.has_section('general'): self.add_section('general') for key, value in list(options.items()): if key in self.old_settings: section, setting = self.old_settings[key] if not self.has_section(section): self.add_section(section) else: section = 'general' setting = key if not self.has_option(section, setting): self.set(section, setting, value) self.remove_section('options') def validate_settings(self, filename, option_parser): """ Call the validator function and implement overrides on all applicable settings. """ for section in self.sections(): for setting in self.options(section): try: option = option_parser.get_option_by_dest(setting) except KeyError: continue if option.validator: value = self.get(section, setting) try: new_value = option.validator( setting, value, option_parser, config_parser=self, config_section=section) except Exception as error: raise ValueError( 'Error in config file "%s", section "[%s]":\n' ' %s\n' ' %s = %s' % (filename, section, ErrorString(error), setting, value)) self.set(section, setting, new_value) if option.overrides: self.set(section, option.overrides, None) def optionxform(self, optionstr): """ Transform '-' to '_' so the cmdline form of option names can be used. """ return optionstr.lower().replace('-', '_') def get_section(self, section): """ Return a given section as a dictionary (empty if the section doesn't exist). """ section_dict = {} if self.has_section(section): for option in self.options(section): section_dict[option] = self.get(section, option) return section_dict
class Publisher(object): """ A facade encapsulating the high-level logic of a Docutils system. """ def __init__(self, reader=None, parser=None, writer=None, source=None, source_class=io.FileInput, destination=None, destination_class=io.FileOutput, settings=None): """ Initial setup. If any of `reader`, `parser`, or `writer` are not specified, the corresponding ``set_...`` method should be called with a component name (`set_reader` sets the parser as well). """ self.document = None """The document tree (`docutils.nodes` objects).""" self.reader = reader """A `docutils.readers.Reader` instance.""" self.parser = parser """A `docutils.parsers.Parser` instance.""" self.writer = writer """A `docutils.writers.Writer` instance.""" for component in 'reader', 'parser', 'writer': assert not isinstance(getattr(self, component), str), ( 'passed string "%s" as "%s" parameter; pass an instance, ' 'or use the "%s_name" parameter instead (in ' 'docutils.core.publish_* convenience functions).' % (getattr(self, component), component, component)) self.source = source """The source of input data, a `docutils.io.Input` instance.""" self.source_class = source_class """The class for dynamically created source objects.""" self.destination = destination """The destination for docutils output, a `docutils.io.Output` instance.""" self.destination_class = destination_class """The class for dynamically created destination objects.""" self.settings = settings """An object containing Docutils settings as instance attributes. Set by `self.process_command_line()` or `self.get_settings()`.""" self._stderr = ErrorOutput() def set_reader(self, reader_name, parser, parser_name): """Set `self.reader` by name.""" reader_class = readers.get_reader_class(reader_name) self.reader = reader_class(parser, parser_name) self.parser = self.reader.parser def set_writer(self, writer_name): """Set `self.writer` by name.""" writer_class = writers.get_writer_class(writer_name) self.writer = writer_class() def set_components(self, reader_name, parser_name, writer_name): if self.reader is None: self.set_reader(reader_name, self.parser, parser_name) if self.parser is None: if self.reader.parser is None: self.reader.set_parser(parser_name) self.parser = self.reader.parser if self.writer is None: self.set_writer(writer_name) def setup_option_parser(self, usage=None, description=None, settings_spec=None, config_section=None, **defaults): if config_section: if not settings_spec: settings_spec = SettingsSpec() settings_spec.config_section = config_section parts = config_section.split() if len(parts) > 1 and parts[-1] == 'application': settings_spec.config_section_dependencies = ['applications'] # @@@ Add self.source & self.destination to components in future? option_parser = OptionParser(components=(self.parser, self.reader, self.writer, settings_spec), defaults=defaults, read_config_files=True, usage=usage, description=description) return option_parser def get_settings(self, usage=None, description=None, settings_spec=None, config_section=None, **defaults): """ Set and return default settings (overrides in `defaults` dict). Set components first (`self.set_reader` & `self.set_writer`). Explicitly setting `self.settings` disables command line option processing from `self.publish()`. """ option_parser = self.setup_option_parser(usage, description, settings_spec, config_section, **defaults) self.settings = option_parser.get_default_values() return self.settings def process_programmatic_settings(self, settings_spec, settings_overrides, config_section): if self.settings is None: defaults = (settings_overrides or {}).copy() # Propagate exceptions by default when used programmatically: defaults.setdefault('traceback', True) self.get_settings(settings_spec=settings_spec, config_section=config_section, **defaults) def process_command_line(self, argv=None, usage=None, description=None, settings_spec=None, config_section=None, **defaults): """ Pass an empty list to `argv` to avoid reading `sys.argv` (the default). Set components first (`self.set_reader` & `self.set_writer`). """ option_parser = self.setup_option_parser(usage, description, settings_spec, config_section, **defaults) if argv is None: argv = sys.argv[1:] # converting to Unicode (Python 3 does this automatically): if sys.version_info < (3, 0): # TODO: make this failsafe and reversible? argv_encoding = (frontend.locale_encoding or 'ascii') argv = [a.decode(argv_encoding) for a in argv] self.settings = option_parser.parse_args(argv) def set_io(self, source_path=None, destination_path=None): if self.source is None: self.set_source(source_path=source_path) if self.destination is None: self.set_destination(destination_path=destination_path) def set_source(self, source=None, source_path=None): if source_path is None: source_path = self.settings._source else: self.settings._source = source_path # Raise IOError instead of system exit with `tracback == True` # TODO: change io.FileInput's default behaviour and remove this hack try: self.source = self.source_class( source=source, source_path=source_path, encoding=self.settings.input_encoding) except TypeError: self.source = self.source_class( source=source, source_path=source_path, encoding=self.settings.input_encoding) def set_destination(self, destination=None, destination_path=None): if destination_path is None: destination_path = self.settings._destination else: self.settings._destination = destination_path self.destination = self.destination_class( destination=destination, destination_path=destination_path, encoding=self.settings.output_encoding, error_handler=self.settings.output_encoding_error_handler) def apply_transforms(self): self.document.transformer.populate_from_components( (self.source, self.reader, self.reader.parser, self.writer, self.destination)) self.document.transformer.apply_transforms() def publish(self, argv=None, usage=None, description=None, settings_spec=None, settings_overrides=None, config_section=None, enable_exit_status=False): """ Process command line options and arguments (if `self.settings` not already set), run `self.reader` and then `self.writer`. Return `self.writer`'s output. """ exit = None try: if self.settings is None: self.process_command_line(argv, usage, description, settings_spec, config_section, **(settings_overrides or {})) self.set_io() self.document = self.reader.read(self.source, self.parser, self.settings) self.apply_transforms() output = self.writer.write(self.document, self.destination) self.writer.assemble_parts() except SystemExit as error: exit = 1 exit_status = error.code except Exception as error: if not self.settings: # exception too early to report nicely raise if self.settings.traceback: # Propagate exceptions? self.debugging_dumps() raise self.report_Exception(error) exit = True exit_status = 1 self.debugging_dumps() if (enable_exit_status and self.document and (self.document.reporter.max_level >= self.settings.exit_status_level)): sys.exit(self.document.reporter.max_level + 10) elif exit: sys.exit(exit_status) return output def debugging_dumps(self): if not self.document: return if self.settings.dump_settings: print('\n::: Runtime settings:', file=self._stderr) print(pprint.pformat(self.settings.__dict__), file=self._stderr) if self.settings.dump_internals: print('\n::: Document internals:', file=self._stderr) print(pprint.pformat(self.document.__dict__), file=self._stderr) if self.settings.dump_transforms: print('\n::: Transforms applied:', file=self._stderr) print( ' (priority, transform class, pending node details, ' 'keyword args)', file=self._stderr) print(pprint.pformat([ (priority, '%s.%s' % (xclass.__module__, xclass.__name__), pending and pending.details, kwargs) for priority, xclass, pending, kwargs in self.document.transformer.applied ]), file=self._stderr) if self.settings.dump_pseudo_xml: print('\n::: Pseudo-XML:', file=self._stderr) print(self.document.pformat().encode('raw_unicode_escape'), file=self._stderr) def report_Exception(self, error): if isinstance(error, utils.SystemMessage): self.report_SystemMessage(error) elif isinstance(error, UnicodeEncodeError): self.report_UnicodeError(error) elif isinstance(error, io.InputError): self._stderr.write(u'Unable to open source file for reading:\n' u' %s\n' % ErrorString(error)) elif isinstance(error, io.OutputError): self._stderr.write( u'Unable to open destination file for writing:\n' u' %s\n' % ErrorString(error)) else: print(u'%s' % ErrorString(error), file=self._stderr) print( ("""\ Exiting due to error. Use "--traceback" to diagnose. Please report errors to <*****@*****.**>. Include "--traceback" output, Docutils version (%s%s), Python version (%s), your OS type & version, and the command line used.""" % (__version__, docutils.__version_details__ and ' [%s]' % docutils.__version_details__ or '', sys.version.split()[0])), file=self._stderr) def report_SystemMessage(self, error): print('Exiting due to level-%s (%s) system message.' % (error.level, utils.Reporter.levels[error.level]), file=self._stderr) def report_UnicodeError(self, error): data = error.object[error.start:error.end] self._stderr.write( '%s\n' '\n' 'The specified output encoding (%s) cannot\n' 'handle all of the output.\n' 'Try setting "--output-encoding-error-handler" to\n' '\n' '* "xmlcharrefreplace" (for HTML & XML output);\n' ' the output will contain "%s" and should be usable.\n' '* "backslashreplace" (for other output formats);\n' ' look for "%s" in the output.\n' '* "replace"; look for "?" in the output.\n' '\n' '"--output-encoding-error-handler" is currently set to "%s".\n' '\n' 'Exiting due to error. Use "--traceback" to diagnose.\n' 'If the advice above doesn\'t eliminate the error,\n' 'please report it to <*****@*****.**>.\n' 'Include "--traceback" output, Docutils version (%s),\n' 'Python version (%s), your OS type & version, and the\n' 'command line used.\n' % (ErrorString(error), self.settings.output_encoding, data.encode('ascii', 'xmlcharrefreplace'), data.encode('ascii', 'backslashreplace'), self.settings.output_encoding_error_handler, __version__, sys.version.split()[0]))
class ConfigParser(CP.RawConfigParser): old_settings = { "pep_stylesheet": ("pep_html writer", "stylesheet"), "pep_stylesheet_path": ("pep_html writer", "stylesheet_path"), "pep_template": ("pep_html writer", "template"), } """{old setting: (new section, new setting)} mapping, used by `handle_old_config`, to convert settings from the old [options] section.""" old_warning = """ The "[option]" section is deprecated. Support for old-format configuration files may be removed in a future Docutils release. Please revise your configuration files. See <http://docutils.sf.net/docs/user/config.html>, section "Old-Format Configuration Files". """ not_utf8_error = """\ Unable to read configuration file "%s": content not encoded as UTF-8. Skipping "%s" configuration file. """ def __init__(self, *args, **kwargs): CP.RawConfigParser.__init__(self, *args, **kwargs) self._files = [] """List of paths of configuration files read.""" self._stderr = ErrorOutput() """Wrapper around sys.stderr catching en-/decoding errors""" def read(self, filenames, option_parser): if type(filenames) in (str, str): filenames = [filenames] for filename in filenames: try: # Config files must be UTF-8-encoded: fp = codecs.open(filename, "r", "utf-8") except IOError: continue try: if sys.version_info < (3, 2): CP.RawConfigParser.readfp(self, fp, filename) else: CP.RawConfigParser.read_file(self, fp, filename) except UnicodeDecodeError: self._stderr.write(self.not_utf8_error % (filename, filename)) fp.close() continue fp.close() self._files.append(filename) if self.has_section("options"): self.handle_old_config(filename) self.validate_settings(filename, option_parser) def handle_old_config(self, filename): warnings.warn_explicit(self.old_warning, ConfigDeprecationWarning, filename, 0) options = self.get_section("options") if not self.has_section("general"): self.add_section("general") for key, value in list(options.items()): if key in self.old_settings: section, setting = self.old_settings[key] if not self.has_section(section): self.add_section(section) else: section = "general" setting = key if not self.has_option(section, setting): self.set(section, setting, value) self.remove_section("options") def validate_settings(self, filename, option_parser): """ Call the validator function and implement overrides on all applicable settings. """ for section in self.sections(): for setting in self.options(section): try: option = option_parser.get_option_by_dest(setting) except KeyError: continue if option.validator: value = self.get(section, setting) try: new_value = option.validator( setting, value, option_parser, config_parser=self, config_section=section ) except Exception as error: raise ValueError( 'Error in config file "%s", section "[%s]":\n' " %s\n" " %s = %s" % (filename, section, ErrorString(error), setting, value) ) self.set(section, setting, new_value) if option.overrides: self.set(section, option.overrides, None) def optionxform(self, optionstr): """ Transform '-' to '_' so the cmdline form of option names can be used. """ return optionstr.lower().replace("-", "_") def get_section(self, section): """ Return a given section as a dictionary (empty if the section doesn't exist). """ section_dict = {} if self.has_section(section): for option in self.options(section): section_dict[option] = self.get(section, option) return section_dict
class ConfigParser(CP.RawConfigParser): old_settings = { 'pep_stylesheet': ('pep_html writer', 'stylesheet'), 'pep_stylesheet_path': ('pep_html writer', 'stylesheet_path'), 'pep_template': ('pep_html writer', 'template')} """{old setting: (new section, new setting)} mapping, used by `handle_old_config`, to convert settings from the old [options] section.""" old_warning = """ The "[option]" section is deprecated. Support for old-format configuration files may be removed in a future Docutils release. Please revise your configuration files. See <http://docutils.sf.net/docs/user/config.html>, section "Old-Format Configuration Files". """ not_utf8_error = """\ Unable to read configuration file "%s": content not encoded as UTF-8. Skipping "%s" configuration file. """ def __init__(self, *args, **kwargs): CP.RawConfigParser.__init__(self, *args, **kwargs) self._files = [] """List of paths of configuration files read.""" self._stderr = ErrorOutput() """Wrapper around sys.stderr catching en-/decoding errors""" def read(self, filenames, option_parser): if type(filenames) in (str, unicode): filenames = [filenames] for filename in filenames: try: # Config files must be UTF-8-encoded: fp = codecs.open(filename, 'r', 'utf-8') except IOError: continue try: if sys.version_info < (3,2): CP.RawConfigParser.readfp(self, fp, filename) else: CP.RawConfigParser.read_file(self, fp, filename) except UnicodeDecodeError: self._stderr.write(self.not_utf8_error % (filename, filename)) fp.close() continue fp.close() self._files.append(filename) if self.has_section('options'): self.handle_old_config(filename) self.validate_settings(filename, option_parser) def handle_old_config(self, filename): warnings.warn_explicit(self.old_warning, ConfigDeprecationWarning, filename, 0) options = self.get_section('options') if not self.has_section('general'): self.add_section('general') for key, value in options.items(): if key in self.old_settings: section, setting = self.old_settings[key] if not self.has_section(section): self.add_section(section) else: section = 'general' setting = key if not self.has_option(section, setting): self.set(section, setting, value) self.remove_section('options') def validate_settings(self, filename, option_parser): """ Call the validator function and implement overrides on all applicable settings. """ for section in self.sections(): for setting in self.options(section): try: option = option_parser.get_option_by_dest(setting) except KeyError: continue if option.validator: value = self.get(section, setting) try: new_value = option.validator( setting, value, option_parser, config_parser=self, config_section=section) except Exception, error: raise (ValueError( 'Error in config file "%s", section "[%s]":\n' ' %s\n' ' %s = %s' % (filename, section, ErrorString(error), setting, value)), None, sys.exc_info()[2]) self.set(section, setting, new_value) if option.overrides: self.set(section, option.overrides, None)
def test_ubuf(self): buf = UBuf() # buffer only accepting unicode string # decode of binary strings e = ErrorOutput(buf, encoding='ascii') e.write(b'b\xfc') self.assertEqual(buf.getvalue(), u'b\ufffd') # use REPLACEMENT CHARACTER # write Unicode string and Exceptions with Unicode args e.write(u' u\xfc') self.assertEqual(buf.getvalue(), u'b\ufffd u\xfc') e.write(AttributeError(u' e\xfc')) self.assertEqual(buf.getvalue(), u'b\ufffd u\xfc e\xfc') # decode with `encoding` attribute e.encoding = 'latin1' e.write(b' b\xfc') self.assertEqual(buf.getvalue(), u'b\ufffd u\xfc e\xfc b\xfc')
def test_bbuf(self): buf = BBuf() # buffer storing byte string e = ErrorOutput(buf, encoding='ascii') # write byte-string as-is e.write(b'b\xfc') self.assertEqual(buf.getvalue(), b'b\xfc') # encode unicode data with backslashescape fallback replacement: e.write(u' u\xfc') self.assertEqual(buf.getvalue(), b'b\xfc u\\xfc') # handle Exceptions with Unicode string args # unicode(Exception(u'e\xfc')) # fails in Python < 2.6 e.write(AttributeError(u' e\xfc')) self.assertEqual(buf.getvalue(), b'b\xfc u\\xfc e\\xfc') # encode with `encoding` attribute e.encoding = 'utf8' e.write(u' u\xfc') self.assertEqual(buf.getvalue(), b'b\xfc u\\xfc e\\xfc u\xc3\xbc')
def test_defaults(self): e = ErrorOutput() self.assertEqual(e.stream, sys.stderr)