def visit(self, directory, names, subdirectories): settings = self.get_settings('', directory) errout = ErrorOutput(encoding=settings.error_encoding) if settings.prune and (os.path.abspath(directory) in settings.prune): errout.write('/// ...Skipping directory (pruned): %s\n' % directory) sys.stderr.flush() del subdirectories[:] return if not self.initial_settings.silent: errout.write('/// Processing directory: %s' % directory) sys.stderr.flush() # settings.ignore grows many duplicate entries as we recurse # if we add patterns in config files or on the command line. for pattern in utils.uniq(settings.ignore): for i in range(len(names) - 1, -1, -1): if fnmatch(names[i], pattern): # Modify in place! del names[i] for name in names: if name.endswith('.txt'): self.process_txt(directory, name)
def test_ubuf(self): buf = UBuf() # buffer only accepting unicode string # decode of binary strings e = ErrorOutput(buf, encoding='ascii') e.write(b('b\xfc')) self.assertEqual(buf.getvalue(), u'b\ufffd') # use REPLACEMENT CHARACTER # write Unicode string and Exceptions with Unicode args e.write(u' u\xfc') self.assertEqual(buf.getvalue(), u'b\ufffd u\xfc') e.write(AttributeError(u' e\xfc')) self.assertEqual(buf.getvalue(), u'b\ufffd u\xfc e\xfc') # decode with `encoding` attribute e.encoding = 'latin1' e.write(b(' b\xfc')) self.assertEqual(buf.getvalue(), u'b\ufffd u\xfc e\xfc b\xfc')
def test_ubuf(self): buf = UBuf() # buffer only accepting unicode string # decode of binary strings e = ErrorOutput(buf, encoding='ascii') e.write(b('b\xfc')) self.assertEquals(buf.getvalue(), u'b\ufffd') # use REPLACEMENT CHARACTER # write Unicode string and Exceptions with Unicode args e.write(u' u\xfc') self.assertEquals(buf.getvalue(), u'b\ufffd u\xfc') e.write(AttributeError(u' e\xfc')) self.assertEquals(buf.getvalue(), u'b\ufffd u\xfc e\xfc') # decode with `encoding` attribute e.encoding = 'latin1' e.write(b(' b\xfc')) self.assertEquals(buf.getvalue(), u'b\ufffd u\xfc e\xfc b\xfc')
def test_bbuf(self): buf = BBuf() # buffer storing byte string e = ErrorOutput(buf, encoding='ascii') # write byte-string as-is e.write(b('b\xfc')) self.assertEqual(buf.getvalue(), b('b\xfc')) # encode unicode data with backslashescape fallback replacement: e.write(u' u\xfc') self.assertEqual(buf.getvalue(), b('b\xfc u\\xfc')) # handle Exceptions with Unicode string args # unicode(Exception(u'e\xfc')) # fails in Python < 2.6 e.write(AttributeError(u' e\xfc')) self.assertEqual(buf.getvalue(), b('b\xfc u\\xfc e\\xfc')) # encode with `encoding` attribute e.encoding = 'utf8' e.write(u' u\xfc') self.assertEqual(buf.getvalue(), b('b\xfc u\\xfc e\\xfc u\xc3\xbc'))
def process_txt(self, directory, name): if name.startswith('pep-'): publisher = 'PEPs' else: publisher = '.txt' settings = self.get_settings(publisher, directory) errout = ErrorOutput(encoding=settings.error_encoding) pub_struct = self.publishers[publisher] settings._source = os.path.normpath(os.path.join(directory, name)) settings._destination = settings._source[:-4]+'.html' if not self.initial_settings.silent: errout.write(' ::: Processing: %s\n' % name) sys.stderr.flush() try: if not settings.dry_run: core.publish_file(source_path=settings._source, destination_path=settings._destination, reader_name=pub_struct.reader_name, parser_name='restructuredtext', writer_name=pub_struct.writer_name, settings=settings) except ApplicationError: error = sys.exc_info()[1] # get exception in Python <2.6 and 3.x errout.write(' %s\n' % ErrorString(error))
def test_bbuf(self): buf = BBuf() # buffer storing byte string e = ErrorOutput(buf, encoding='ascii') # write byte-string as-is e.write(b('b\xfc')) self.assertEquals(buf.getvalue(), b('b\xfc')) # encode unicode data with backslashescape fallback replacement: e.write(u' u\xfc') self.assertEquals(buf.getvalue(), b('b\xfc u\\xfc')) # handle Exceptions with Unicode string args # unicode(Exception(u'e\xfc')) # fails in Python < 2.6 e.write(AttributeError(u' e\xfc')) self.assertEquals(buf.getvalue(), b('b\xfc u\\xfc e\\xfc')) # encode with `encoding` attribute e.encoding = 'utf8' e.write(u' u\xfc') self.assertEquals(buf.getvalue(), b('b\xfc u\\xfc e\\xfc u\xc3\xbc'))
class ConfigParser(CP.RawConfigParser): old_settings = { 'pep_stylesheet': ('pep_html writer', 'stylesheet'), 'pep_stylesheet_path': ('pep_html writer', 'stylesheet_path'), 'pep_template': ('pep_html writer', 'template')} """{old setting: (new section, new setting)} mapping, used by `handle_old_config`, to convert settings from the old [options] section.""" old_warning = """ The "[option]" section is deprecated. Support for old-format configuration files may be removed in a future Docutils release. Please revise your configuration files. See <http://docutils.sf.net/docs/user/config.html>, section "Old-Format Configuration Files". """ not_utf8_error = """\ Unable to read configuration file "%s": content not encoded as UTF-8. Skipping "%s" configuration file. """ def __init__(self, *args, **kwargs): CP.RawConfigParser.__init__(self, *args, **kwargs) self._files = [] """List of paths of configuration files read.""" self._stderr = ErrorOutput() """Wrapper around sys.stderr catching en-/decoding errors""" def read(self, filenames, option_parser): if type(filenames) in (str, unicode): filenames = [filenames] for filename in filenames: try: # Config files must be UTF-8-encoded: fp = codecs.open(filename, 'r', 'utf-8') except IOError: continue try: if sys.version_info < (3,2): CP.RawConfigParser.readfp(self, fp, filename) else: CP.RawConfigParser.read_file(self, fp, filename) except UnicodeDecodeError: self._stderr.write(self.not_utf8_error % (filename, filename)) fp.close() continue fp.close() self._files.append(filename) if self.has_section('options'): self.handle_old_config(filename) self.validate_settings(filename, option_parser) def handle_old_config(self, filename): warnings.warn_explicit(self.old_warning, ConfigDeprecationWarning, filename, 0) options = self.get_section('options') if not self.has_section('general'): self.add_section('general') for key, value in options.items(): if key in self.old_settings: section, setting = self.old_settings[key] if not self.has_section(section): self.add_section(section) else: section = 'general' setting = key if not self.has_option(section, setting): self.set(section, setting, value) self.remove_section('options') def validate_settings(self, filename, option_parser): """ Call the validator function and implement overrides on all applicable settings. """ for section in self.sections(): for setting in self.options(section): try: option = option_parser.get_option_by_dest(setting) except KeyError: continue if option.validator: value = self.get(section, setting) try: new_value = option.validator( setting, value, option_parser, config_parser=self, config_section=section) except Exception, error: raise (ValueError( 'Error in config file "%s", section "[%s]":\n' ' %s\n' ' %s = %s' % (filename, section, ErrorString(error), setting, value)), None, sys.exc_info()[2]) self.set(section, setting, new_value) if option.overrides: self.set(section, option.overrides, None)
class ConfigParser(CP.RawConfigParser): old_settings = { 'pep_stylesheet': ('pep_html writer', 'stylesheet'), 'pep_stylesheet_path': ('pep_html writer', 'stylesheet_path'), 'pep_template': ('pep_html writer', 'template') } """{old setting: (new section, new setting)} mapping, used by `handle_old_config`, to convert settings from the old [options] section.""" old_warning = """ The "[option]" section is deprecated. Support for old-format configuration files may be removed in a future Docutils release. Please revise your configuration files. See <http://docutils.sf.net/docs/user/config.html>, section "Old-Format Configuration Files". """ not_utf8_error = """\ Unable to read configuration file "%s": content not encoded as UTF-8. Skipping "%s" configuration file. """ def __init__(self, *args, **kwargs): CP.RawConfigParser.__init__(self, *args, **kwargs) self._files = [] """List of paths of configuration files read.""" self._stderr = ErrorOutput() """Wrapper around sys.stderr catching en-/decoding errors""" def read(self, filenames, option_parser): if type(filenames) in (str, unicode): filenames = [filenames] for filename in filenames: try: # Config files must be UTF-8-encoded: fp = codecs.open(filename, 'r', 'utf-8') except IOError: continue try: if sys.version_info < (3, 2): CP.RawConfigParser.readfp(self, fp, filename) else: CP.RawConfigParser.read_file(self, fp, filename) except UnicodeDecodeError: self._stderr.write(self.not_utf8_error % (filename, filename)) fp.close() continue fp.close() self._files.append(filename) if self.has_section('options'): self.handle_old_config(filename) self.validate_settings(filename, option_parser) def handle_old_config(self, filename): warnings.warn_explicit(self.old_warning, ConfigDeprecationWarning, filename, 0) options = self.get_section('options') if not self.has_section('general'): self.add_section('general') for key, value in options.items(): if key in self.old_settings: section, setting = self.old_settings[key] if not self.has_section(section): self.add_section(section) else: section = 'general' setting = key if not self.has_option(section, setting): self.set(section, setting, value) self.remove_section('options') def validate_settings(self, filename, option_parser): """ Call the validator function and implement overrides on all applicable settings. """ for section in self.sections(): for setting in self.options(section): try: option = option_parser.get_option_by_dest(setting) except KeyError: continue if option.validator: value = self.get(section, setting) try: new_value = option.validator(setting, value, option_parser, config_parser=self, config_section=section) except Exception, error: raise (ValueError( 'Error in config file "%s", section "[%s]":\n' ' %s\n' ' %s = %s' % (filename, section, ErrorString(error), setting, value)), None, sys.exc_info()[2]) self.set(section, setting, new_value) if option.overrides: self.set(section, option.overrides, None)
class Publisher: """ A facade encapsulating the high-level logic of a Docutils system. """ def __init__(self, reader=None, parser=None, writer=None, source=None, source_class=io.FileInput, destination=None, destination_class=io.FileOutput, settings=None): """ Initial setup. If any of `reader`, `parser`, or `writer` are not specified, the corresponding ``set_...`` method should be called with a component name (`set_reader` sets the parser as well). """ self.document = None """The document tree (`docutils.nodes` objects).""" self.reader = reader """A `docutils.readers.Reader` instance.""" self.parser = parser """A `docutils.parsers.Parser` instance.""" self.writer = writer """A `docutils.writers.Writer` instance.""" for component in 'reader', 'parser', 'writer': assert not isinstance(getattr(self, component), str), ( 'passed string "%s" as "%s" parameter; pass an instance, ' 'or use the "%s_name" parameter instead (in ' 'docutils.core.publish_* convenience functions).' % (getattr(self, component), component, component)) self.source = source """The source of input data, a `docutils.io.Input` instance.""" self.source_class = source_class """The class for dynamically created source objects.""" self.destination = destination """The destination for docutils output, a `docutils.io.Output` instance.""" self.destination_class = destination_class """The class for dynamically created destination objects.""" self.settings = settings """An object containing Docutils settings as instance attributes. Set by `self.process_command_line()` or `self.get_settings()`.""" self._stderr = ErrorOutput() def set_reader(self, reader_name, parser, parser_name): """Set `self.reader` by name.""" reader_class = readers.get_reader_class(reader_name) self.reader = reader_class(parser, parser_name) self.parser = self.reader.parser def set_writer(self, writer_name): """Set `self.writer` by name.""" writer_class = writers.get_writer_class(writer_name) self.writer = writer_class() def set_components(self, reader_name, parser_name, writer_name): if self.reader is None: self.set_reader(reader_name, self.parser, parser_name) if self.parser is None: if self.reader.parser is None: self.reader.set_parser(parser_name) self.parser = self.reader.parser if self.writer is None: self.set_writer(writer_name) def setup_option_parser(self, usage=None, description=None, settings_spec=None, config_section=None, **defaults): if config_section: if not settings_spec: settings_spec = SettingsSpec() settings_spec.config_section = config_section parts = config_section.split() if len(parts) > 1 and parts[-1] == 'application': settings_spec.config_section_dependencies = ['applications'] #@@@ Add self.source & self.destination to components in future? option_parser = OptionParser( components=(self.parser, self.reader, self.writer, settings_spec), defaults=defaults, read_config_files=True, usage=usage, description=description) return option_parser def get_settings(self, usage=None, description=None, settings_spec=None, config_section=None, **defaults): """ Set and return default settings (overrides in `defaults` dict). Set components first (`self.set_reader` & `self.set_writer`). Explicitly setting `self.settings` disables command line option processing from `self.publish()`. """ option_parser = self.setup_option_parser( usage, description, settings_spec, config_section, **defaults) self.settings = option_parser.get_default_values() return self.settings def process_programmatic_settings(self, settings_spec, settings_overrides, config_section): if self.settings is None: defaults = (settings_overrides or {}).copy() # Propagate exceptions by default when used programmatically: defaults.setdefault('traceback', True) self.get_settings(settings_spec=settings_spec, config_section=config_section, **defaults) def process_command_line(self, argv=None, usage=None, description=None, settings_spec=None, config_section=None, **defaults): """ Pass an empty list to `argv` to avoid reading `sys.argv` (the default). Set components first (`self.set_reader` & `self.set_writer`). """ option_parser = self.setup_option_parser( usage, description, settings_spec, config_section, **defaults) if argv is None: argv = sys.argv[1:] # converting to Unicode (Python 3 does this automatically): if sys.version_info < (3,0): # TODO: make this failsafe and reversible? argv_encoding = (frontend.locale_encoding or 'ascii') argv = [a.decode(argv_encoding) for a in argv] self.settings = option_parser.parse_args(argv) def set_io(self, source_path=None, destination_path=None): if self.source is None: self.set_source(source_path=source_path) if self.destination is None: self.set_destination(destination_path=destination_path) def set_source(self, source=None, source_path=None): if source_path is None: source_path = self.settings._source else: self.settings._source = source_path # Raise IOError instead of system exit with `tracback == True` # TODO: change io.FileInput's default behaviour and remove this hack try: self.source = self.source_class( source=source, source_path=source_path, encoding=self.settings.input_encoding, handle_io_errors=False) except TypeError: self.source = self.source_class( source=source, source_path=source_path, encoding=self.settings.input_encoding) def set_destination(self, destination=None, destination_path=None): if destination_path is None: destination_path = self.settings._destination else: self.settings._destination = destination_path self.destination = self.destination_class( destination=destination, destination_path=destination_path, encoding=self.settings.output_encoding, error_handler=self.settings.output_encoding_error_handler) # Raise IOError instead of system exit with `tracback == True` # TODO: change io.FileInput's default behaviour and remove this hack self.destination.handle_io_errors=False def apply_transforms(self): self.document.transformer.populate_from_components( (self.source, self.reader, self.reader.parser, self.writer, self.destination)) self.document.transformer.apply_transforms() def publish(self, argv=None, usage=None, description=None, settings_spec=None, settings_overrides=None, config_section=None, enable_exit_status=False): """ Process command line options and arguments (if `self.settings` not already set), run `self.reader` and then `self.writer`. Return `self.writer`'s output. """ exit = None try: if self.settings is None: self.process_command_line( argv, usage, description, settings_spec, config_section, **(settings_overrides or {})) self.set_io() self.document = self.reader.read(self.source, self.parser, self.settings) self.apply_transforms() output = self.writer.write(self.document, self.destination) self.writer.assemble_parts() except SystemExit as error: exit = 1 exit_status = error.code except Exception as error: if not self.settings: # exception too early to report nicely raise if self.settings.traceback: # Propagate exceptions? self.debugging_dumps() raise self.report_Exception(error) exit = True exit_status = 1 self.debugging_dumps() if (enable_exit_status and self.document and (self.document.reporter.max_level >= self.settings.exit_status_level)): sys.exit(self.document.reporter.max_level + 10) elif exit: sys.exit(exit_status) return output def debugging_dumps(self): if not self.document: return if self.settings.dump_settings: print('\n::: Runtime settings:', file=self._stderr) print(pprint.pformat(self.settings.__dict__), file=self._stderr) if self.settings.dump_internals: print('\n::: Document internals:', file=self._stderr) print(pprint.pformat(self.document.__dict__), file=self._stderr) if self.settings.dump_transforms: print('\n::: Transforms applied:', file=self._stderr) print((' (priority, transform class, ' 'pending node details, keyword args)'), file=self._stderr) print(pprint.pformat( [(priority, '%s.%s' % (xclass.__module__, xclass.__name__), pending and pending.details, kwargs) for priority, xclass, pending, kwargs in self.document.transformer.applied]), file=self._stderr) if self.settings.dump_pseudo_xml: print('\n::: Pseudo-XML:', file=self._stderr) print(self.document.pformat().encode( 'raw_unicode_escape'), file=self._stderr) def report_Exception(self, error): if isinstance(error, utils.SystemMessage): self.report_SystemMessage(error) elif isinstance(error, UnicodeEncodeError): self.report_UnicodeError(error) elif isinstance(error, io.InputError): self._stderr.write('Unable to open source file for reading:\n' ' %s\n' % ErrorString(error)) elif isinstance(error, io.OutputError): self._stderr.write( 'Unable to open destination file for writing:\n' ' %s\n' % ErrorString(error)) else: print('%s' % ErrorString(error), file=self._stderr) print(("""\ Exiting due to error. Use "--traceback" to diagnose. Please report errors to <*****@*****.**>. Include "--traceback" output, Docutils version (%s [%s]), Python version (%s), your OS type & version, and the command line used.""" % (__version__, __version_details__, sys.version.split()[0])), file=self._stderr) def report_SystemMessage(self, error): print(('Exiting due to level-%s (%s) system message.' % (error.level, utils.Reporter.levels[error.level])), file=self._stderr) def report_UnicodeError(self, error): data = error.object[error.start:error.end] self._stderr.write( '%s\n' '\n' 'The specified output encoding (%s) cannot\n' 'handle all of the output.\n' 'Try setting "--output-encoding-error-handler" to\n' '\n' '* "xmlcharrefreplace" (for HTML & XML output);\n' ' the output will contain "%s" and should be usable.\n' '* "backslashreplace" (for other output formats);\n' ' look for "%s" in the output.\n' '* "replace"; look for "?" in the output.\n' '\n' '"--output-encoding-error-handler" is currently set to "%s".\n' '\n' 'Exiting due to error. Use "--traceback" to diagnose.\n' 'If the advice above doesn\'t eliminate the error,\n' 'please report it to <*****@*****.**>.\n' 'Include "--traceback" output, Docutils version (%s),\n' 'Python version (%s), your OS type & version, and the\n' 'command line used.\n' % (ErrorString(error), self.settings.output_encoding, data.encode('ascii', 'xmlcharrefreplace'), data.encode('ascii', 'backslashreplace'), self.settings.output_encoding_error_handler, __version__, sys.version.split()[0]))