class SafeStringTests(unittest.TestCase):
# the error message in EnvironmentError instances comes from the OS
# and in some locales (e.g. ru_RU), contains high bit chars.
# -> see the test in test_error_reporting.py
# test data:
bs = b'\xfc' # unicode(bs) fails, str(bs) in Python 3 return repr()
us = u'\xfc' # bytes(us) fails; str(us) fails in Python 2
be = Exception(bs) # unicode(be) fails
ue = Exception(us) # bytes(ue) fails, str(ue) fails in Python 2;
# unicode(ue) fails in Python < 2.6 (issue2517_)
# .. _issue2517: http://bugs.python.org/issue2517
# wrapped test data:
wbs = SafeString(bs)
wus = SafeString(us)
wbe = SafeString(be)
wue = SafeString(ue)
def test_7bit(self):
# wrapping (not required with 7-bit chars) must not change the
# result of conversions:
bs7 = b'foo'
us7 = u'foo'
be7 = Exception(bs7)
ue7 = Exception(us7)
self.assertEqual(str(42), str(SafeString(42)))
self.assertEqual(str(bs7), str(SafeString(bs7)))
self.assertEqual(str(us7), str(SafeString(us7)))
self.assertEqual(str(be7), str(SafeString(be7)))
self.assertEqual(str(ue7), str(SafeString(ue7)))
self.assertEqual(unicode(7), unicode(SafeString(7)))
self.assertEqual(unicode(bs7), unicode(SafeString(bs7)))
self.assertEqual(unicode(us7), unicode(SafeString(us7)))
self.assertEqual(unicode(be7), unicode(SafeString(be7)))
self.assertEqual(unicode(ue7), unicode(SafeString(ue7)))
def test_ustr(self):
"""Test conversion to a unicode-string."""
# unicode(self.bs) fails
self.assertEqual(unicode, type(unicode(self.wbs)))
self.assertEqual(unicode(self.us), unicode(self.wus))
# unicode(self.be) fails
self.assertEqual(unicode, type(unicode(self.wbe)))
# unicode(ue) fails in Python < 2.6 (issue2517_)
self.assertEqual(unicode, type(unicode(self.wue)))
self.assertEqual(self.us, unicode(self.wue))
def test_str(self):
"""Test conversion to a string (bytes in Python 2, unicode in Python 3)."""
self.assertEqual(str(self.bs), str(self.wbs))
self.assertEqual(str(self.be), str(self.be))
# str(us) fails in Python 2
self.assertEqual(str, type(str(self.wus)))
# str(ue) fails in Python 2
self.assertEqual(str, type(str(self.wue)))
def run(self):
# To maximize code reuse, we just write the output in a temporary
# file and call the base class. Otherwise we'd have to copy & paste
# all the code to handle start-line, end-line etc options.
source = self.state_machine.input_lines.source(
self.lineno - self.state_machine.input_offset - 1)
source_dir = os.path.dirname(os.path.abspath(source))
command = self.arguments[0]
command_list = shlex.split(command)
if command_list[0] == 'python':
command_list[0] = sys.executable
with tempfile.NamedTemporaryFile() as fh:
exitcode = subprocess.call(command_list, stdout=fh, cwd=source_dir)
if exitcode != 0:
raise self.severe('Problems with "%s" directive:\n'
'Command %s returned with exit code %d' %
(self.name, SafeString(command), exitcode))
fh.flush()
self.arguments[0] = fh.name
return super().run()
def test_str(self):
self.assertEqual('Exception: spam',
str(ErrorString(Exception('spam'))))
self.assertEqual('IndexError: '+str(self.bs),
str(ErrorString(IndexError(self.bs))))
self.assertEqual('ImportError: %s' % SafeString(self.us),
str(ErrorString(ImportError(self.us))))
def test_unicode(self):
self.assertEqual(u'Exception: spam',
unicode(ErrorString(Exception(u'spam'))))
self.assertEqual(u'IndexError: '+self.us,
unicode(ErrorString(IndexError(self.us))))
self.assertEqual(u'ImportError: %s' % SafeString(self.bs),
unicode(ErrorString(ImportError(self.bs))))
def run(self):
"""Include a file as part of the content of this reST file."""
if not self.state.document.settings.file_insertion_enabled:
raise self.warning('"%s" directive disabled.' % self.name)
source = self.state_machine.input_lines.source(
self.lineno - self.state_machine.input_offset - 1)
source_dir = os.path.dirname(os.path.abspath(source))
path = directives.path(self.arguments[0])
if path.startswith('<') and path.endswith('>'):
path = os.path.join(self.standard_include_path, path[1:-1])
path = os.path.normpath(os.path.join(source_dir, path))
path = utils.relative_path(None, path)
path = nodes.reprunicode(path)
encoding = self.options.get(
'encoding', self.state.document.settings.input_encoding)
e_handler = self.state.document.settings.input_encoding_error_handler
tab_width = self.options.get('tab-width',
self.state.document.settings.tab_width)
try:
self.state.document.settings.record_dependencies.add(path)
include_file = io.FileInput(source_path=path,
encoding=encoding,
error_handler=e_handler)
except UnicodeEncodeError, error:
raise self.severe(u'Problems with "%s" directive path:\n'
'Cannot encode input file path "%s" '
'(wrong locale?).' %
(self.name, SafeString(path)))
def __init__(self, components=(), defaults=None, read_config_files=None,
*args, **kwargs):
"""
`components` is a list of Docutils components each containing a
``.settings_spec`` attribute. `defaults` is a mapping of setting
default overrides.
"""
self.lists = {}
"""Set of list-type settings."""
self.config_files = []
"""List of paths of applied configuration files."""
optparse.OptionParser.__init__(
self, option_class=Option, add_help_option=None,
formatter=optparse.TitledHelpFormatter(width=78),
*args, **kwargs)
if not self.version:
self.version = self.version_template
# Make an instance copy (it will be modified):
self.relative_path_settings = list(self.relative_path_settings)
self.components = (self,) + tuple(components)
self.populate_from_components(self.components)
self.set_defaults_from_dict(defaults or {})
if read_config_files and not self.defaults['_disable_config']:
try:
config_settings = self.get_standard_config_settings()
except ValueError as error:
self.error(SafeString(error))
self.set_defaults_from_dict(config_settings.__dict__)
class Role(Directive):
has_content = True
argument_pattern = re.compile(r'(%s)\s*(\(\s*(%s)\s*\)\s*)?$' %
((states.Inliner.simplename, ) * 2))
def run(self):
"""Dynamically create and register a custom interpreted text role."""
if self.content_offset > self.lineno or not self.content:
raise self.error('"%s" directive requires arguments on the first '
'line.' % self.name)
args = self.content[0]
match = self.argument_pattern.match(args)
if not match:
raise self.error('"%s" directive arguments not valid role names: '
'"%s".' % (self.name, args))
new_role_name = match.group(1)
base_role_name = match.group(3)
messages = []
if base_role_name:
base_role, messages = roles.role(base_role_name,
self.state_machine.language,
self.lineno, self.state.reporter)
if base_role is None:
error = self.state.reporter.error(
'Unknown interpreted text role "%s".' % base_role_name,
nodes.literal_block(self.block_text, self.block_text),
line=self.lineno)
return messages + [error]
else:
base_role = roles.generic_custom_role
assert not hasattr(base_role, 'arguments'), (
'Supplemental directive arguments for "%s" directive not '
'supported (specified by "%r" role).' % (self.name, base_role))
try:
converted_role = convert_directive_function(base_role)
(arguments, options, content,
content_offset) = (self.state.parse_directive_block(
self.content[1:],
self.content_offset,
converted_role,
option_presets={}))
except states.MarkupError, detail:
error = self.state_machine.reporter.error(
'Error in "%s" directive:\n%s.' % (self.name, detail),
nodes.literal_block(self.block_text, self.block_text),
line=self.lineno)
return messages + [error]
if 'class' not in options:
try:
options['class'] = directives.class_option(new_role_name)
except ValueError, detail:
error = self.state_machine.reporter.error(
u'Invalid argument for "%s" directive:\n%s.' %
(self.name, SafeString(detail)),
nodes.literal_block(self.block_text, self.block_text),
line=self.lineno)
return messages + [error]
def run(self):
settings = self.state.document.settings
if not settings.file_insertion_enabled:
raise self.warning('"%s" directive disabled.' % self.name)
env = self.state.document.settings.env
rel_path, path = env.relfn2path(self.arguments[0])
path = os.path.normpath(path)
encoding = self.options.get('encoding', settings.input_encoding)
e_handler = settings.input_encoding_error_handler
try:
settings.record_dependencies.add(path)
f = io.FileInput(source_path=path, encoding=encoding,
error_handler=e_handler)
except UnicodeEncodeError as error:
raise self.severe('Problems with "%s" directive path:\n'
'Cannot encode input file path "%s" '
'(wrong locale?).' %
(self.name, SafeString(path)))
except IOError as error:
raise self.severe('Problems with "%s" directive path:\n%s.' %
(self.name, ErrorString(error)))
raw_lines = f.read().splitlines()
f.close()
rst = None
lines = []
for line in raw_lines:
if rst is not None and rst != '#':
# Bracket mode: check for end bracket
pos = line.find(rst)
if pos >= 0:
if line[0] == '#':
line = ''
else:
line = line[0:pos]
rst = None
else:
# Line mode: check for .rst start (bracket or line)
m = self.re_start.match(line)
if m:
rst = ']%s]' % m.group('eq')
line = ''
elif line == '#.rst:':
rst = '#'
line = ''
elif rst == '#':
if line == '#' or line[:2] == '# ':
line = line[2:]
else:
rst = None
line = ''
elif rst is None:
line = ''
lines.append(line)
if rst is not None and rst != '#':
raise self.warning('"%s" found unclosed bracket "#[%s[.rst:" in %s'
% (self.name, rst[1:-1], path))
self.state_machine.insert_input(lines, path)
return []
def test_7bit(self):
# wrapping (not required with 7-bit chars) must not change the
# result of conversions:
bs7 = b'foo'
us7 = u'foo'
be7 = Exception(bs7)
ue7 = Exception(us7)
self.assertEqual(str(42), str(SafeString(42)))
self.assertEqual(str(bs7), str(SafeString(bs7)))
self.assertEqual(str(us7), str(SafeString(us7)))
self.assertEqual(str(be7), str(SafeString(be7)))
self.assertEqual(str(ue7), str(SafeString(ue7)))
self.assertEqual(unicode(7), unicode(SafeString(7)))
self.assertEqual(unicode(bs7), unicode(SafeString(bs7)))
self.assertEqual(unicode(us7), unicode(SafeString(us7)))
self.assertEqual(unicode(be7), unicode(SafeString(be7)))
self.assertEqual(unicode(ue7), unicode(SafeString(ue7)))
def get_json_data(self):
"""
Get JSON data from the directive content, from an external
file, or from a URL reference.
"""
if self.arguments:
filename, pointer = self._splitpointer(self.arguments[0])
else:
filename = None
pointer = ''
if self.content:
schema, source = self.from_content(filename)
elif filename and filename.startswith('http'):
schema, source = self.from_url(filename)
elif filename:
schema, source = self.from_file(filename)
else:
raise self.error(
'"%s" directive has no content or a reference to an external file.'
% self.name)
try:
schema = self.ordered_load(schema)
except Exception as error:
error = self.state_machine.reporter.error(
'"%s" directive encountered a the following error while parsing the data.\n %s'
% (self.name,
SafeString("".join(format_exception_only(
type(error), error)))),
nodes.literal_block(schema, schema),
line=self.lineno)
raise SystemMessagePropagation(error)
if pointer:
try:
schema = resolve_pointer(schema, pointer)
except KeyError:
error = self.state_machine.reporter.error(
'"%s" directive encountered a KeyError when trying to resolve the pointer'
' in schema: %s' % (self.name, SafeString(pointer)),
nodes.literal_block(schema, schema),
line=self.lineno)
raise SystemMessagePropagation(error)
return schema, source, pointer
def parse(self, inputstring, document):
"""Parse the nblink file.
Adds the linked file as a dependency, read the file, and
pass the content to the nbshpinx.NotebookParser.
"""
link = json.loads(inputstring)
env = document.settings.env
source_dir = os.path.dirname(env.doc2path(env.docname))
abs_path = os.path.normpath(os.path.join(source_dir, link['path']))
path = utils.relative_path(None, abs_path)
path = nodes.reprunicode(path)
extra_media = link.get('extra-media', None)
if extra_media:
source_file = env.doc2path(env.docname)
collect_extra_media(extra_media, source_file, path, document)
register_dependency(path, document)
target_root = env.config.nbsphinx_link_target_root
target = utils.relative_path(target_root, abs_path)
target = nodes.reprunicode(target).replace(os.path.sep, '/')
env.metadata[env.docname]['nbsphinx-link-target'] = target
# Copy parser from nbsphinx for our cutom format
try:
formats = env.config.nbsphinx_custom_formats
except AttributeError:
pass
else:
formats.setdefault(
'.nblink',
lambda s: nbformat.reads(s, as_version=_ipynbversion))
try:
include_file = io.FileInput(source_path=path, encoding='utf8')
except UnicodeEncodeError as error:
raise NotebookError(u'Problems with linked notebook "%s" path:\n'
'Cannot encode input file path "%s" '
'(wrong locale?).' %
(env.docname, SafeString(path)))
except IOError as error:
raise NotebookError(
u'Problems with linked notebook "%s" path:\n%s.' %
(env.docname, ErrorString(error)))
try:
rawtext = include_file.read()
except UnicodeError as error:
raise NotebookError(u'Problem with linked notebook "%s":\n%s' %
(env.docname, ErrorString(error)))
return super(LinkedNotebookParser, self).parse(rawtext, document)
def __parser_helper(self, link, source_dir, document, env):
"""Helper method for adding a single notebook as a linked dependency.
Adds the linked file as a dependency, read the file, and pass the content to
the nbshpinx.NotebookParser.
"""
abs_path = os.path.normpath(os.path.join(source_dir, link["path"]))
path = utils.relative_path(None, abs_path)
path = nodes.reprunicode(path)
extra_media = link.get("extra-media", None)
if extra_media:
source_file = env.doc2path(env.docname)
collect_extra_media(extra_media, source_file, path, document)
register_dependency(path, document)
target_root = env.config.nbsphinx_link_target_root
target = utils.relative_path(target_root, abs_path)
target = nodes.reprunicode(target).replace(os.path.sep, "/")
env.metadata[env.docname]["nbsphinx-link-target"] = target
# Copy parser from nbsphinx for our cutom format
try:
formats = env.config.nbsphinx_custom_formats
except AttributeError:
pass
else:
formats.setdefault(
".nblink",
lambda s: nbformat.reads(s, as_version=_ipynbversion))
try:
include_file = io.FileInput(source_path=path, encoding="utf8")
except UnicodeEncodeError as error:
raise NotebookError(u'Problems with linked notebook "%s" path:\n'
'Cannot encode input file path "%s" '
"(wrong locale?)." %
(env.docname, SafeString(path)))
except IOError as error:
raise NotebookError(
u'Problems with linked notebook "%s" path:\n%s.' %
(env.docname, ErrorString(error)))
try:
rawtext = include_file.read()
except UnicodeError as error:
raise NotebookError(u'Problem with linked notebook "%s":\n%s' %
(env.docname, ErrorString(error)))
super(LinkedNotebookParser, self).parse(rawtext, document)
def system_message(self, level, message, *children, **kwargs):
"""
Return a system_message object.
Raise an exception or generate a warning if appropriate.
"""
# `message` can be a `string`, `unicode`, or `Exception` instance.
if isinstance(message, Exception):
message = SafeString(message)
attributes = kwargs.copy()
if 'base_node' in kwargs:
source, line = get_source_line(kwargs['base_node'])
del attributes['base_node']
if source is not None:
attributes.setdefault('source', source)
if line is not None:
attributes.setdefault('line', line)
# assert source is not None, "node has line- but no source-argument"
if not 'source' in attributes: # 'line' is absolute line number
try: # look up (source, line-in-source)
source, line = self.get_source_and_line(attributes.get('line'))
# print "locator lookup", kwargs.get('line'), "->", source, line
except AttributeError:
source, line = None, None
if source is not None:
attributes['source'] = source
if line is not None:
attributes['line'] = line
# assert attributes['line'] is not None, (message, kwargs)
# assert attributes['source'] is not None, (message, kwargs)
attributes.setdefault('source', self.source)
msg = nodes.system_message(message,
level=level,
type=self.levels[level],
*children,
**attributes)
if self.stream and (level >= self.report_level
or self.debug_flag and level == self.DEBUG_LEVEL
or level >= self.halt_level):
self.stream.write(msg.astext() + '\n')
if level >= self.halt_level:
raise SystemMessage(msg, level)
if level > self.DEBUG_LEVEL or self.debug_flag:
self.notify_observers(msg)
self.max_level = max(level, self.max_level)
return msg
def get_csv_data(self):
"""
Get CSV data from the directive content, from an external
file, or from a URL reference.
"""
encoding = self.options.get(
'encoding', self.state.document.settings.input_encoding)
error_handler = self.state.document.settings.input_encoding_error_handler
if self.content:
# CSV data is from directive content.
if 'file' in self.options or 'url' in self.options:
error = self.state_machine.reporter.error(
'"%s" directive may not both specify an external file and'
' have content.' % self.name,
nodes.literal_block(self.block_text, self.block_text),
line=self.lineno)
raise SystemMessagePropagation(error)
source = self.content.source(0)
csv_data = self.content
elif 'file' in self.options:
# CSV data is from an external file.
if 'url' in self.options:
error = self.state_machine.reporter.error(
'The "file" and "url" options may not be simultaneously'
' specified for the "%s" directive.' % self.name,
nodes.literal_block(self.block_text, self.block_text),
line=self.lineno)
raise SystemMessagePropagation(error)
source_dir = os.path.dirname(
os.path.abspath(self.state.document.current_source))
source = os.path.normpath(
os.path.join(source_dir, self.options['file']))
source = utils.relative_path(None, source)
try:
self.state.document.settings.record_dependencies.add(source)
csv_file = io.FileInput(source_path=source,
encoding=encoding,
error_handler=error_handler)
csv_data = csv_file.read().splitlines()
except IOError, error:
severe = self.state_machine.reporter.severe(
u'Problems with "%s" directive path:\n%s.' %
(self.name, SafeString(error)),
nodes.literal_block(self.block_text, self.block_text),
line=self.lineno)
raise SystemMessagePropagation(severe)
def run(self):
settings = self.state.document.settings
if not settings.file_insertion_enabled:
raise self.warning('"%s" directive disabled.' % self.name)
env = self.state.document.settings.env
rel_path, path = env.relfn2path(self.arguments[0])
path = os.path.normpath(path)
encoding = self.options.get('encoding', settings.input_encoding)
e_handler = settings.input_encoding_error_handler
try:
settings.record_dependencies.add(path)
f = io.FileInput(source_path=path,
encoding=encoding,
error_handler=e_handler)
except UnicodeEncodeError, error:
raise self.severe('Problems with "%s" directive path:\n'
'Cannot encode input file path "%s" '
'(wrong locale?).' %
(self.name, SafeString(path)))
def run(self):
"""Include a file as part of the content of this reST file."""
if not self.state.document.settings.file_insertion_enabled:
raise self.warning('"%s" directive disabled.' % self.name)
source = self.state_machine.input_lines.source(
self.lineno - self.state_machine.input_offset - 1)
source_dir = os.path.dirname(os.path.abspath(source))
path = directives.path(self.arguments[0])
path = os.path.normpath(os.path.join(source_dir, path))
path = utils.relative_path(None, path)
path = nodes.reprunicode(path)
encoding = self.options.get(
"encoding", self.state.document.settings.input_encoding)
e_handler = self.state.document.settings.input_encoding_error_handler
tab_width = self.options.get("tab-width",
self.state.document.settings.tab_width)
try:
self.state.document.settings.record_dependencies.add(path)
include_file = io.FileInput(source_path=path,
encoding=encoding,
error_handler=e_handler)
except UnicodeEncodeError as error:
raise self.severe('Problems with "%s" directive path:\n'
'Cannot encode input file path "%s" '
"(wrong locale?)." %
(self.name, SafeString(path)))
except IOError as error:
raise self.severe('Problems with "%s" directive path:\n%s.' %
(self.name, ErrorString(error)))
startline = self.options.get("start-line", None)
endline = self.options.get("end-line", None)
try:
if startline or (endline is not None):
lines = include_file.readlines()
rawtext = "".join(lines[startline:endline])
else:
rawtext = include_file.read()
except UnicodeError as error:
raise self.severe('Problem with "%s" directive:\n%s' %
(self.name, ErrorString(error)))
include_lines = statemachine.string2lines(rawtext,
tab_width,
convert_whitespace=True)
# default lexer to 'text'
lexer = self.options.get("lexer", "text")
self.options["source"] = path
codeblock = Pygments(
self.name,
[lexer], # arguments
{}, # no options for this directive
include_lines, # content
self.lineno,
self.content_offset,
self.block_text,
self.state,
self.state_machine,
)
return codeblock.run()
uioe = e
try:
os.chdir(b'\xfc')
except OSError, e:
bose = e
try:
os.chdir(u'\xfc')
except OSError, e:
uose = e
except UnicodeEncodeError:
try:
os.chdir(u'\xfc'.encode(sys.getfilesystemencoding(), 'replace'))
except OSError, e:
uose = e
# wrapped test data:
wbioe = SafeString(bioe)
wuioe = SafeString(uioe)
wbose = SafeString(bose)
wuose = SafeString(uose)
# reset locale
if testlocale:
locale.setlocale(locale.LC_ALL, oldlocale)
def test_ustr(self):
"""Test conversion to a unicode-string."""
# unicode(bioe) fails with e.g. 'ru_RU.utf8' locale
self.assertEqual(unicode, type(unicode(self.wbioe)))
self.assertEqual(unicode, type(unicode(self.wuioe)))
self.assertEqual(unicode, type(unicode(self.wbose)))
self.assertEqual(unicode, type(unicode(self.wuose)))
def run(self):
"""Include a file as part of the content of this reST file."""
if not self.state.document.settings.file_insertion_enabled:
raise self.warning('"%s" directive disabled.' % self.name)
source = self.state_machine.input_lines.source(
self.lineno - self.state_machine.input_offset - 1)
source_dir = os.path.dirname(os.path.abspath(source))
path = directives.path(self.arguments[0])
if path.startswith('<') and path.endswith('>'):
path = os.path.join(self.standard_include_path, path[1:-1])
path = os.path.normpath(os.path.join(source_dir, path))
path = utils.relative_path(None, path)
path = nodes.reprunicode(path)
encoding = self.options.get(
'encoding', self.state.document.settings.input_encoding)
e_handler = self.state.document.settings.input_encoding_error_handler
tab_width = self.options.get('tab-width',
self.state.document.settings.tab_width)
try:
self.state.document.settings.record_dependencies.add(path)
include_file = io.FileInput(source_path=path,
encoding=encoding,
error_handler=e_handler)
except UnicodeEncodeError as error:
raise self.severe('Problems with "%s" directive path:\n'
'Cannot encode input file path "%s" '
'(wrong locale?).' %
(self.name, SafeString(path)))
except IOError as error:
raise self.severe('Problems with "%s" directive path:\n%s.' %
(self.name, ErrorString(error)))
startline = self.options.get('start-line', None)
endline = self.options.get('end-line', None)
try:
if startline or (endline is not None):
lines = include_file.readlines()
rawtext = ''.join(lines[startline:endline])
else:
rawtext = include_file.read()
except UnicodeError as error:
raise self.severe('Problem with "%s" directive:\n%s' %
(self.name, ErrorString(error)))
# start-after/end-before: no restrictions on newlines in match-text,
# and no restrictions on matching inside lines vs. line boundaries
after_text = self.options.get('start-after', None)
if after_text:
# skip content in rawtext before *and incl.* a matching text
after_index = rawtext.find(after_text)
if after_index < 0:
raise self.severe('Problem with "start-after" option of "%s" '
'directive:\nText not found.' % self.name)
rawtext = rawtext[after_index + len(after_text):]
before_text = self.options.get('end-before', None)
if before_text:
# skip content in rawtext after *and incl.* a matching text
before_index = rawtext.find(before_text)
if before_index < 0:
raise self.severe('Problem with "end-before" option of "%s" '
'directive:\nText not found.' % self.name)
rawtext = rawtext[:before_index]
# **Added code** from here...
##---------------------------
# Only Sphinx has the ``env`` attribute.
env = getattr(self.state.document.settings, 'env', None)
# If the lexer is specified, include it.
code_to_rest_options = {}
lexer_alias = self.options.get('lexer')
if lexer_alias:
code_to_rest_options['alias'] = lexer_alias
elif env:
# If Sphinx is running, try getting a user-specified lexer from the Sphinx configuration.
lfg = env.app.config.CodeChat_lexer_for_glob
for glob, lexer_alias in lfg.items():
if Path(path).match(glob):
code_to_rest_options['alias'] = lexer_alias
# Translate the source code to reST.
lexer = get_lexer(filename=path, code=rawtext, **code_to_rest_options)
rawtext = code_to_rest_string(rawtext, lexer=lexer)
# If Sphinx is running, insert the appropriate highlight directive.
if env:
rawtext = add_highlight_language(rawtext, lexer)
##------------
# ... to here.
include_lines = statemachine.string2lines(rawtext,
tab_width,
convert_whitespace=True)
# **Deleted code**: Options for ``literal`` and ``code`` don't apply.
self.state_machine.insert_input(include_lines, path)
return []
def run(self):
# For code or literal include blocks we run the normal include
if 'literal' in self.options or 'code' in self.options:
return super(FormatedInclude, self).run()
"""Include a file as part of the content of this reST file."""
if not self.state.document.settings.file_insertion_enabled:
raise self.warning('"%s" directive disabled.' % self.name)
source = self.state_machine.input_lines.source(
self.lineno - self.state_machine.input_offset - 1)
source_dir = os.path.dirname(os.path.abspath(source))
rel_filename, filename = self.env.relfn2path(self.arguments[0])
self.arguments[0] = filename
self.env.note_included(filename)
path = directives.path(self.arguments[0])
if path.startswith('<') and path.endswith('>'):
path = os.path.join(self.standard_include_path, path[1:-1])
path = os.path.normpath(os.path.join(source_dir, path))
path = utils.relative_path(None, path)
path = nodes.reprunicode(path)
encoding = self.options.get(
'encoding', self.state.document.settings.input_encoding)
e_handler = self.state.document.settings.input_encoding_error_handler
tab_width = self.options.get('tab-width',
self.state.document.settings.tab_width)
try:
self.state.document.settings.record_dependencies.add(path)
include_file = io.FileInput(source_path=path,
encoding=encoding,
error_handler=e_handler)
except UnicodeEncodeError:
raise self.severe(u'Problems with "%s" directive path:\n'
'Cannot encode input file path "%s" '
'(wrong locale?).' %
(self.name, SafeString(path)))
except IOError as error:
raise self.severe(u'Problems with "%s" directive path:\n%s.' %
(self.name, ErrorString(error)))
startline = self.options.get('start-line', None)
endline = self.options.get('end-line', None)
try:
if startline or (endline is not None):
lines = include_file.readlines()
rawtext = ''.join(lines[startline:endline])
else:
rawtext = include_file.read()
except UnicodeError as error:
raise self.severe(u'Problem with "%s" directive:\n%s' %
(self.name, ErrorString(error)))
# Format input
sub = StringSubstituter()
config = self.state.document.settings.env.config
sub.init_sub_strings(config)
rawtext = sub.substitute(rawtext)
# start-after/end-before: no restrictions on newlines in match-text,
# and no restrictions on matching inside lines vs. line boundaries
after_text = self.options.get('start-after', None)
if after_text:
# skip content in rawtext before *and incl.* a matching text
after_index = rawtext.find(after_text)
if after_index < 0:
raise self.severe('Problem with "start-after" option of "%s" '
'directive:\nText not found.' % self.name)
rawtext = rawtext[after_index + len(after_text):]
before_text = self.options.get('end-before', None)
if before_text:
# skip content in rawtext after *and incl.* a matching text
before_index = rawtext.find(before_text)
if before_index < 0:
raise self.severe('Problem with "end-before" option of "%s" '
'directive:\nText not found.' % self.name)
rawtext = rawtext[:before_index]
include_lines = statemachine.string2lines(rawtext,
tab_width,
convert_whitespace=True)
self.state_machine.insert_input(include_lines, path)
return []
def run(self):
"""
Verbatim copy of docutils.parsers.rst.directives.misc.Include.run()
that just calls to our Code instead of builtin CodeBlock but otherwise
just passes it back to the parent implementation.
"""
if not 'code' in self.options:
return docutils.parsers.rst.directives.misc.Include.run(self)
source = self.state_machine.input_lines.source(
self.lineno - self.state_machine.input_offset - 1)
source_dir = os.path.dirname(os.path.abspath(source))
path = directives.path(self.arguments[0])
if path.startswith('<') and path.endswith('>'):
path = os.path.join(self.standard_include_path, path[1:-1])
path = os.path.normpath(os.path.join(source_dir, path))
path = utils.relative_path(None, path)
path = nodes.reprunicode(path)
encoding = self.options.get(
'encoding', self.state.document.settings.input_encoding)
e_handler = self.state.document.settings.input_encoding_error_handler
tab_width = self.options.get('tab-width',
self.state.document.settings.tab_width)
try:
self.state.document.settings.record_dependencies.add(path)
include_file = io.FileInput(source_path=path,
encoding=encoding,
error_handler=e_handler)
except UnicodeEncodeError as error:
raise self.severe('Problems with "%s" directive path:\n'
'Cannot encode input file path "%s" '
'(wrong locale?).' %
(self.name, SafeString(path)))
except IOError as error:
raise self.severe('Problems with "%s" directive path:\n%s.' %
(self.name, ErrorString(error)))
startline = self.options.get('start-line', None)
endline = self.options.get('end-line', None)
try:
if startline or (endline is not None):
lines = include_file.readlines()
rawtext = ''.join(lines[startline:endline])
else:
rawtext = include_file.read()
except UnicodeError as error:
raise self.severe('Problem with "%s" directive:\n%s' %
(self.name, ErrorString(error)))
# start-after/end-before: no restrictions on newlines in match-text,
# and no restrictions on matching inside lines vs. line boundaries
after_text = self.options.get('start-after', None)
if after_text:
# skip content in rawtext before *and incl.* a matching text
after_index = rawtext.find(after_text)
if after_index < 0:
raise self.severe('Problem with "start-after" option of "%s" '
'directive:\nText not found.' % self.name)
rawtext = rawtext[after_index + len(after_text):]
before_text = self.options.get('end-before', None)
if before_text:
# skip content in rawtext after *and incl.* a matching text
before_index = rawtext.find(before_text)
if before_index < 0:
raise self.severe('Problem with "end-before" option of "%s" '
'directive:\nText not found.' % self.name)
rawtext = rawtext[:before_index]
include_lines = statemachine.string2lines(rawtext,
tab_width,
convert_whitespace=True)
self.options['source'] = path
codeblock = Code(
self.name,
[self.options.pop('code')], # arguments
self.options,
include_lines, # content
self.lineno,
self.content_offset,
self.block_text,
self.state,
self.state_machine)
return codeblock.run()
def run(self):
"""Include a file as part of the content of this reST file."""
if not self.state.document.settings.file_insertion_enabled:
raise self.warning('"%s" directive disabled.' % self.name)
source = self.state_machine.input_lines.source(
self.lineno - self.state_machine.input_offset - 1)
source_dir = os.path.dirname(os.path.abspath(source))
startline = self.options.get('start-line', None)
endline = self.options.get('end-line', None)
encoding = self.options.get(
'encoding', self.state.document.settings.input_encoding)
e_handler = self.state.document.settings.input_encoding_error_handler
tab_width = self.options.get(
'tab-width', self.state.document.settings.tab_width)
if re.match('^https*://', self.arguments[0]):
rawtext = self.fetch_url(self.arguments[0])
if startline or (endline is not None):
rawtext = self.start_end_lines(rawtext, startline, endline)
else:
path = directives.path(self.arguments[0])
path = os.path.normpath(os.path.join(source_dir, path))
path = utils.relative_path(None, path)
path = nodes.reprunicode(path)
try:
self.state.document.settings.record_dependencies.add(path)
include_file = io.FileInput(source_path=path,
encoding=encoding,
error_handler=e_handler)
except UnicodeEncodeError as error:
raise self.severe('Problems with "%s" directive path:\n'
'Cannot encode input file path "%s" '
'(wrong locale?).' %
(self.name, SafeString(path)))
except IOError as error:
raise self.severe('Problems with "%s" directive '
'path:\n%s.' %
(self.name, ErrorString(error)))
self.options['source'] = path
if startline or (endline is not None):
rawtext = include_file.read()
rawtext = self.start_end_lines(rawtext, startline, endline)
include_lines = statemachine.string2lines(rawtext, tab_width,
convert_whitespace=True)
# default lexer to 'text'
lexer = self.options.get('lexer', 'text')
linenos = self.options.get('linenos', 'none')
linenostart = self.options.get('linenostart', 1)
codeblock = Pygments(self.name,
[lexer], # arguments
{ 'linenos': linenos, 'linenostart': linenostart }, # no options for this directive
include_lines, # content
self.lineno,
self.content_offset,
self.block_text,
self.state,
self.state_machine)
return codeblock.run()
def run(self):
"""Include a file as part of the content of this reST file."""
if not self.state.document.settings.file_insertion_enabled:
raise self.warning('"%s" directive disabled.' % self.name)
source = self.state_machine.input_lines.source(
self.lineno - self.state_machine.input_offset - 1)
source_dir = os.path.dirname(os.path.abspath(source))
path = directives.path(self.arguments[0])
if path.startswith('<') and path.endswith('>'):
path = os.path.join(self.standard_include_path, path[1:-1])
path = os.path.normpath(os.path.join(source_dir, path))
path = utils.relative_path(None, path)
path = nodes.reprunicode(path)
encoding = self.options.get(
'encoding', self.state.document.settings.input_encoding)
e_handler = self.state.document.settings.input_encoding_error_handler
tab_width = self.options.get('tab-width',
self.state.document.settings.tab_width)
try:
self.state.document.settings.record_dependencies.add(path)
include_file = io.FileInput(source_path=path,
encoding=encoding,
error_handler=e_handler)
except UnicodeEncodeError as error:
raise self.severe('Problems with "%s" directive path:\n'
'Cannot encode input file path "%s" '
'(wrong locale?).' %
(self.name, SafeString(path)))
except IOError as error:
raise self.severe('Problems with "%s" directive path:\n%s.' %
(self.name, ErrorString(error)))
startline = self.options.get('start-line', None)
endline = self.options.get('end-line', None)
try:
if startline or (endline is not None):
lines = include_file.readlines()
rawtext = ''.join(lines[startline:endline])
else:
rawtext = include_file.read()
except UnicodeError as error:
raise self.severe('Problem with "%s" directive:\n%s' %
(self.name, ErrorString(error)))
# start-after/end-before: no restrictions on newlines in match-text,
# and no restrictions on matching inside lines vs. line boundaries
after_text = self.options.get('start-after', None)
if after_text:
# skip content in rawtext before *and incl.* a matching text
after_index = rawtext.find(after_text)
if after_index < 0:
raise self.severe('Problem with "start-after" option of "%s" '
'directive:\nText not found.' % self.name)
rawtext = rawtext[after_index + len(after_text):]
before_text = self.options.get('end-before', None)
if before_text:
# skip content in rawtext after *and incl.* a matching text
before_index = rawtext.find(before_text)
if before_index < 0:
raise self.severe('Problem with "end-before" option of "%s" '
'directive:\nText not found.' % self.name)
rawtext = rawtext[:before_index]
include_lines = statemachine.string2lines(rawtext,
tab_width,
convert_whitespace=True)
if 'literal' in self.options:
# Convert tabs to spaces, if `tab_width` is positive.
if tab_width >= 0:
text = rawtext.expandtabs(tab_width)
else:
text = rawtext
literal_block = nodes.literal_block(rawtext,
source=path,
classes=self.options.get(
'class', []))
literal_block.line = 1
self.add_name(literal_block)
if 'number-lines' in self.options:
try:
startline = int(self.options['number-lines'] or 1)
except ValueError:
raise self.error(':number-lines: with non-integer '
'start value')
endline = startline + len(include_lines)
if text.endswith('\n'):
text = text[:-1]
tokens = NumberLines([([], text)], startline, endline)
for classes, value in tokens:
if classes:
literal_block += nodes.inline(value,
value,
classes=classes)
else:
literal_block += nodes.Text(value, value)
else:
literal_block += nodes.Text(text, text)
return [literal_block]
if 'code' in self.options:
self.options['source'] = path
codeblock = CodeBlock(
self.name,
[self.options.pop('code')], # arguments
self.options,
include_lines, # content
self.lineno,
self.content_offset,
self.block_text,
self.state,
self.state_machine)
return codeblock.run()
self.state_machine.insert_input(include_lines, path)
return []
class SafeStringTests_locale(unittest.TestCase):
"""
Test docutils.SafeString with 'problematic' locales.
The error message in `EnvironmentError` instances comes from the OS
and in some locales (e.g. ru_RU), contains high bit chars.
"""
if testlocale:
locale.setlocale(locale.LC_ALL, testlocale)
# test data:
bs = b'\xfc'
us = u'\xfc'
try:
open(b'\xfc')
except IOError as e: # in Python 3 the name for the exception instance
bioe = e # is local to the except clause
try:
open(u'\xfc')
except IOError as e:
uioe = e
except UnicodeEncodeError:
try:
open(u'\xfc'.encode(sys.getfilesystemencoding(), 'replace'))
except IOError as e:
uioe = e
try:
os.chdir(b'\xfc')
except OSError as e:
bose = e
try:
os.chdir(u'\xfc')
except OSError as e:
uose = e
except UnicodeEncodeError:
try:
os.chdir(u'\xfc'.encode(sys.getfilesystemencoding(), 'replace'))
except OSError as e:
uose = e
# wrapped test data:
wbioe = SafeString(bioe)
wuioe = SafeString(uioe)
wbose = SafeString(bose)
wuose = SafeString(uose)
# reset locale
if testlocale:
locale.setlocale(locale.LC_ALL, oldlocale)
def test_ustr(self):
"""Test conversion to a unicode-string."""
# unicode(bioe) fails with e.g. 'ru_RU.utf8' locale
self.assertEqual(unicode, type(unicode(self.wbioe)))
self.assertEqual(unicode, type(unicode(self.wuioe)))
self.assertEqual(unicode, type(unicode(self.wbose)))
self.assertEqual(unicode, type(unicode(self.wuose)))
def test_str(self):
"""Test conversion to a string (bytes in Python 2, unicode in Python 3)."""
self.assertEqual(str(self.bioe), str(self.wbioe))
self.assertEqual(str(self.uioe), str(self.wuioe))
self.assertEqual(str(self.bose), str(self.wbose))
self.assertEqual(str(self.uose), str(self.wuose))
def run(self):
"""
Verbatim copy of docutils.parsers.rst.directives.misc.Include.run()
that just calls to our Code instead of builtin CodeBlock, is without
the rarely useful :encoding:, :literal: and :name: options and adds
support for :start-on:, empty :end-before: and :strip-prefix:.
"""
source = self.state_machine.input_lines.source(
self.lineno - self.state_machine.input_offset - 1)
source_dir = os.path.dirname(os.path.abspath(source))
path = directives.path(self.arguments[0])
if path.startswith('<') and path.endswith('>'):
path = os.path.join(self.standard_include_path, path[1:-1])
path = os.path.normpath(os.path.join(source_dir, path))
path = utils.relative_path(None, path)
path = nodes.reprunicode(path)
e_handler = self.state.document.settings.input_encoding_error_handler
tab_width = self.options.get('tab-width',
self.state.document.settings.tab_width)
try:
self.state.document.settings.record_dependencies.add(path)
include_file = io.FileInput(source_path=path,
error_handler=e_handler)
except UnicodeEncodeError as error:
raise self.severe('Problems with "%s" directive path:\n'
'Cannot encode input file path "%s" '
'(wrong locale?).' %
(self.name, SafeString(path)))
except IOError as error:
raise self.severe('Problems with "%s" directive path:\n%s.' %
(self.name, ErrorString(error)))
startline = self.options.get('start-line', None)
endline = self.options.get('end-line', None)
try:
if startline or (endline is not None):
lines = include_file.readlines()
rawtext = ''.join(lines[startline:endline])
else:
rawtext = include_file.read()
except UnicodeError as error:
raise self.severe('Problem with "%s" directive:\n%s' %
(self.name, ErrorString(error)))
# start-after/end-before: no restrictions on newlines in match-text,
# and no restrictions on matching inside lines vs. line boundaries
after_text = self.options.get('start-after', None)
if after_text:
# skip content in rawtext before *and incl.* a matching text
after_index = rawtext.find(after_text)
if after_index < 0:
raise self.severe('Problem with "start-after" option of "%s" '
'directive:\nText not found.' % self.name)
rawtext = rawtext[after_index + len(after_text):]
# Compared to start-after, this includes the matched line
on_text = self.options.get('start-on', None)
if on_text:
on_index = rawtext.find('\n' + on_text)
if on_index < 0:
raise self.severe('Problem with "start-on" option of "%s" '
'directive:\nText not found.' % self.name)
rawtext = rawtext[on_index:]
# Compared to builtin include directive, the end-before can be empty,
# in which case it simply matches the first empty line (which is
# usually end of the code block)
before_text = self.options.get('end-before', None)
if before_text is not None:
# skip content in rawtext after *and incl.* a matching text
if before_text == '':
before_index = rawtext.find('\n\n')
else:
before_index = rawtext.find(before_text)
if before_index < 0:
raise self.severe('Problem with "end-before" option of "%s" '
'directive:\nText not found.' % self.name)
rawtext = rawtext[:before_index]
include_lines = statemachine.string2lines(rawtext,
tab_width,
convert_whitespace=True)
# Strip a common prefix from all lines. Useful for example when
# including a reST snippet that's embedded in a comment, or cutting
# away excessive indentation. Can be wrapped in quotes in order to
# avoid trailing whitespace in reST markup.
if 'strip-prefix' in self.options and self.options['strip-prefix']:
prefix = self.options['strip-prefix']
if prefix[0] == prefix[-1] and prefix[0] in ['\'', '"']:
prefix = prefix[1:-1]
for i, line in enumerate(include_lines):
if line.startswith(prefix):
include_lines[i] = line[len(prefix):]
# Strip the prefix also if the line is just the prefix alone,
# with trailing whitespace removed
elif line.rstrip() == prefix.rstrip():
include_lines[i] = ''
if 'code' in self.options:
self.options['source'] = path
# Don't convert tabs to spaces, if `tab_width` is negative:
if tab_width < 0:
include_lines = rawtext.splitlines()
codeblock = Code(
self.name,
[self.options.pop('code')], # arguments
self.options,
include_lines, # content
self.lineno,
self.content_offset,
self.block_text,
self.state,
self.state_machine)
return codeblock.run()
self.state_machine.insert_input(include_lines, path)
return []
def run(self):
"""Include a file as part of the content of this reST file."""
if not self.state.document.settings.file_insertion_enabled:
raise self.warning('"%s" directive disabled.' % self.name)
source = self.state_machine.input_lines.source(
self.lineno - self.state_machine.input_offset - 1)
source_dir = os.path.dirname(os.path.abspath(source))
path = directives.path(self.arguments[0])
if path.startswith('<') and path.endswith('>'):
path = os.path.join(self.standard_include_path, path[1:-1])
path = os.path.normpath(os.path.join(source_dir, path))
path = utils.relative_path(None, path)
path = nodes.reprunicode(path)
encoding = self.options.get(
'encoding', self.state.document.settings.input_encoding)
e_handler = self.state.document.settings.input_encoding_error_handler
tab_width = self.options.get('tab-width',
self.state.document.settings.tab_width)
try:
self.state.document.settings.record_dependencies.add(path)
include_file = io.FileInput(source_path=path,
encoding=encoding,
error_handler=e_handler)
except UnicodeEncodeError as error:
raise self.severe(u'Problems with "%s" directive path:\n'
'Cannot encode input file path "%s" '
'(wrong locale?).' %
(self.name, SafeString(path)))
except IOError as error:
raise self.severe(u'Problems with "%s" directive path:\n%s.' %
(self.name, ErrorString(error)))
# Get to-be-included content
startline = self.options.get('start-line', None)
endline = self.options.get('end-line', None)
try:
if startline or (endline is not None):
lines = include_file.readlines()
rawtext = ''.join(lines[startline:endline])
else:
rawtext = include_file.read()
except UnicodeError as error:
raise self.severe(u'Problem with "%s" directive:\n%s' %
(self.name, ErrorString(error)))
# start-after/end-before: no restrictions on newlines in match-text,
# and no restrictions on matching inside lines vs. line boundaries
after_text = self.options.get('start-after', None)
if after_text:
# skip content in rawtext before *and incl.* a matching text
after_index = rawtext.find(after_text)
if after_index < 0:
raise self.severe('Problem with "start-after" option of "%s" '
'directive:\nText not found.' % self.name)
rawtext = rawtext[after_index + len(after_text):]
before_text = self.options.get('end-before', None)
if before_text:
# skip content in rawtext after *and incl.* a matching text
before_index = rawtext.find(before_text)
if before_index < 0:
raise self.severe('Problem with "end-before" option of "%s" '
'directive:\nText not found.' % self.name)
rawtext = rawtext[:before_index]
include_lines = statemachine.string2lines(rawtext,
tab_width,
convert_whitespace=True)
for i, line in enumerate(include_lines):
if len(line) > self.state.document.settings.line_length_limit:
raise self.warning('"%s": line %d exceeds the'
' line-length-limit.' % (path, i + 1))
if 'literal' in self.options:
# Don't convert tabs to spaces, if `tab_width` is negative.
if tab_width >= 0:
text = rawtext.expandtabs(tab_width)
else:
text = rawtext
literal_block = nodes.literal_block(rawtext,
source=path,
classes=self.options.get(
'class', []))
literal_block.line = 1
self.add_name(literal_block)
if 'number-lines' in self.options:
try:
startline = int(self.options['number-lines'] or 1)
except ValueError:
raise self.error(':number-lines: with non-integer '
'start value')
endline = startline + len(include_lines)
if text.endswith('\n'):
text = text[:-1]
tokens = NumberLines([([], text)], startline, endline)
for classes, value in tokens:
if classes:
literal_block += nodes.inline(value,
value,
classes=classes)
else:
literal_block += nodes.Text(value)
else:
literal_block += nodes.Text(text)
return [literal_block]
if 'code' in self.options:
self.options['source'] = path
# Don't convert tabs to spaces, if `tab_width` is negative:
if tab_width < 0:
include_lines = rawtext.splitlines()
codeblock = CodeBlock(
self.name,
[self.options.pop('code')], # arguments
self.options,
include_lines, # content
self.lineno,
self.content_offset,
self.block_text,
self.state,
self.state_machine)
return codeblock.run()
if 'parser' in self.options:
parser = self.options['parser']()
# parse into a new (dummy) document
document = utils.new_document(path, self.state.document.settings)
parser.parse('\n'.join(include_lines), document)
return document.children
# include as rST source
#
# Prevent circular inclusion:
source = utils.relative_path(None, source)
clip_options = (startline, endline, before_text, after_text)
include_log = self.state.document.include_log
if not include_log: # new document:
# log entries: (<source>, <clip-options>, <insertion end index>)
include_log = [(source, (None, None, None, None), sys.maxsize / 2)]
# cleanup: we may have passed the last inclusion(s):
include_log = [
entry for entry in include_log if entry[2] >= self.lineno
]
if (path, clip_options) in [(pth, opt)
for (pth, opt, e) in include_log]:
raise self.warning(
'circular inclusion in "%s" directive: %s' %
(self.name,
' < '.join([path] +
[pth for (pth, opt, e) in include_log[::-1]])))
# include as input
self.state_machine.insert_input(include_lines, path)
# update include-log
include_log.append((path, clip_options, self.lineno))
self.state.document.include_log = [(pth, opt,
e + len(include_lines) + 2)
for (pth, opt, e) in include_log]
return []
def run(self):
"""Include a file as part of the content of this reST file."""
# copied from docutils.parsers.rst.directives.misc.Include
if not self.state.document.settings.file_insertion_enabled:
raise self.warning('"%s" directive disabled.' % self.name)
source = self.state_machine.input_lines.source(
self.lineno - self.state_machine.input_offset - 1)
source_dir = os.path.dirname(os.path.abspath(source))
path = directives.path(self.arguments[0])
if path.startswith('<') and path.endswith('>'):
path = os.path.join(self.standard_include_path, path[1:-1])
path = os.path.normpath(os.path.join(source_dir, path))
path = utils.relative_path(None, path)
path = nodes.reprunicode(path)
encoding = self.options.get(
'encoding', self.state.document.settings.input_encoding)
e_handler = self.state.document.settings.input_encoding_error_handler
try:
self.state.document.settings.record_dependencies.add(path)
include_file = io.FileInput(source_path=path,
encoding=encoding,
error_handler=e_handler)
except UnicodeEncodeError as error:
raise self.severe(u'Problems with "%s" directive path:\n'
'Cannot encode input file path "%s" '
'(wrong locale?).' %
(self.name, SafeString(path)))
except IOError as error:
raise self.severe(u'Problems with "%s" directive path:\n%s.' %
(self.name, ErrorString(error)))
startline = self.options.get('start-line', None)
endline = self.options.get('end-line', None)
try:
if startline or (endline is not None):
lines = include_file.readlines()
rawtext = ''.join(lines[startline:endline])
else:
rawtext = include_file.read()
except UnicodeError as error:
raise self.severe(u'Problem with "%s" directive:\n%s' %
(self.name, ErrorString(error)))
# start-after/end-before: no restrictions on newlines in match-text,
# and no restrictions on matching inside lines vs. line boundaries
after_text = self.options.get('start-after', None)
if after_text:
# skip content in rawtext before *and incl.* a matching text
after_index = rawtext.find(after_text)
if after_index < 0:
raise self.severe('Problem with "start-after" option of "%s" '
'directive:\nText not found.' % self.name)
rawtext = rawtext[after_index + len(after_text):]
before_text = self.options.get('end-before', None)
if before_text:
# skip content in rawtext after *and incl.* a matching text
before_index = rawtext.find(before_text)
if before_index < 0:
raise self.severe('Problem with "end-before" option of "%s" '
'directive:\nText not found.' % self.name)
rawtext = rawtext[:before_index]
# copied code ends
parser = CommonMarkParser()
md_document = utils.new_document(path, self.state.document.settings)
parser.parse(rawtext, md_document)
return md_document.children
def get_csv_data(self):
"""
Get CSV data from the directive content, from an external
file, or from a URL reference.
"""
encoding = self.options.get(
'encoding', self.state.document.settings.input_encoding)
error_handler = self.state.document.settings.input_encoding_error_handler
if self.content:
# CSV data is from directive content.
if 'file' in self.options or 'url' in self.options:
error = self.state_machine.reporter.error(
'"%s" directive may not both specify an external file and'
' have content.' % self.name,
nodes.literal_block(self.block_text, self.block_text),
line=self.lineno)
raise SystemMessagePropagation(error)
source = self.content.source(0)
csv_data = self.content
elif 'file' in self.options:
# CSV data is from an external file.
if 'url' in self.options:
error = self.state_machine.reporter.error(
'The "file" and "url" options may not be simultaneously'
' specified for the "%s" directive.' % self.name,
nodes.literal_block(self.block_text, self.block_text),
line=self.lineno)
raise SystemMessagePropagation(error)
source_dir = os.path.dirname(
os.path.abspath(self.state.document.current_source))
source = os.path.normpath(
os.path.join(source_dir, self.options['file']))
source = utils.relative_path(None, source)
try:
self.state.document.settings.record_dependencies.add(source)
csv_file = io.FileInput(source_path=source,
encoding=encoding,
error_handler=error_handler)
csv_data = csv_file.read().splitlines()
except IOError as error:
severe = self.state_machine.reporter.severe(
'Problems with "%s" directive path:\n%s.' %
(self.name, SafeString(error)),
nodes.literal_block(self.block_text, self.block_text),
line=self.lineno)
raise SystemMessagePropagation(severe)
elif 'url' in self.options:
# CSV data is from a URL.
# Do not import urllib2 at the top of the module because
# it may fail due to broken SSL dependencies, and it takes
# about 0.15 seconds to load.
import urllib.request, urllib.error, urllib.parse
source = self.options['url']
try:
csv_text = urllib.request.urlopen(source).read()
except (urllib.error.URLError, IOError, OSError,
ValueError) as error:
severe = self.state_machine.reporter.severe(
'Problems with "%s" directive URL "%s":\n%s.' %
(self.name, self.options['url'], SafeString(error)),
nodes.literal_block(self.block_text, self.block_text),
line=self.lineno)
raise SystemMessagePropagation(severe)
csv_file = io.StringInput(
source=csv_text, source_path=source, encoding=encoding,
error_handler=(self.state.document.settings.\
input_encoding_error_handler))
csv_data = csv_file.read().splitlines()
else:
error = self.state_machine.reporter.warning(
'The "%s" directive requires content; none supplied.' %
self.name,
nodes.literal_block(self.block_text, self.block_text),
line=self.lineno)
raise SystemMessagePropagation(error)
return csv_data, source
nodes.literal_block(self.block_text, self.block_text),
line=self.lineno)
raise SystemMessagePropagation(severe)
elif 'url' in self.options:
# CSV data is from a URL.
# Do not import urllib2 at the top of the module because
# it may fail due to broken SSL dependencies, and it takes
# about 0.15 seconds to load.
import urllib2
source = self.options['url']
try:
csv_text = urllib2.urlopen(source).read()
except (urllib2.URLError, IOError, OSError, ValueError), error:
severe = self.state_machine.reporter.severe(
'Problems with "%s" directive URL "%s":\n%s.' %
(self.name, self.options['url'], SafeString(error)),
nodes.literal_block(self.block_text, self.block_text),
line=self.lineno)
raise SystemMessagePropagation(severe)
csv_file = io.StringInput(
source=csv_text, source_path=source, encoding=encoding,
error_handler=(self.state.document.settings.\
input_encoding_error_handler))
csv_data = csv_file.read().splitlines()
else:
error = self.state_machine.reporter.warning(
'The "%s" directive requires content; none supplied.' %
self.name,
nodes.literal_block(self.block_text, self.block_text),
line=self.lineno)
raise SystemMessagePropagation(error)
def run(self):
"""Include a file as part of the content of this reST file."""
# Direct copy from Include with changes noted.
if not self.state.document.settings.file_insertion_enabled:
raise self.warning('"%s" directive disabled.' % self.name)
source = self.state_machine.input_lines.source(
self.lineno - self.state_machine.input_offset - 1)
source_dir = os.path.dirname(os.path.abspath(source))
# CHANGE: for some reason the arg to the include directive is expanded to
# the full path for the docutils Include directive, but for ours, it
# remains unchanged from the value supplied in the source, so we have to
# expand it ourselves
env = self.state.document.settings.env
if not os.path.exists(self.arguments[0]):
self.arguments[0] = env.srcdir + self.arguments[0]
path = directives.path(self.arguments[0])
if path.startswith('<') and path.endswith('>'):
path = os.path.join(self.standard_include_path, path[1:-1])
path = os.path.normpath(os.path.join(source_dir, path))
path = utils.relative_path(None, path)
path = nodes.reprunicode(path)
encoding = self.options.get(
'encoding', self.state.document.settings.input_encoding)
e_handler = self.state.document.settings.input_encoding_error_handler
tab_width = self.options.get('tab-width',
self.state.document.settings.tab_width)
try:
self.state.document.settings.record_dependencies.add(path)
include_file = io.FileInput(source_path=path,
encoding=encoding,
error_handler=e_handler)
except UnicodeEncodeError as error:
raise self.severe('Problems with "%s" directive path:\n'
'Cannot encode input file path "%s" '
'(wrong locale?).' %
(self.name, SafeString(path)))
except IOError as error:
raise self.severe('Problems with "%s" directive path:\n%s.' %
(self.name, ErrorString(error)))
startline = self.options.get('start-line', None)
endline = self.options.get('end-line', None)
try:
if startline or (endline is not None):
lines = include_file.readlines()
rawtext = ''.join(lines[startline:endline])
else:
rawtext = include_file.read()
except UnicodeError as error:
raise self.severe('Problem with "%s" directive:\n%s' %
(self.name, ErrorString(error)))
# start-after/end-before: no restrictions on newlines in match-text,
# and no restrictions on matching inside lines vs. line boundaries
after_text = self.options.get('start-after', None)
if after_text:
# skip content in rawtext before *and incl.* a matching text
after_index = rawtext.find(after_text)
if after_index < 0:
raise self.severe('Problem with "start-after" option of "%s" '
'directive:\nText not found.' % self.name)
rawtext = rawtext[after_index + len(after_text):]
before_text = self.options.get('end-before', None)
if before_text:
# skip content in rawtext after *and incl.* a matching text
before_index = rawtext.find(before_text)
if before_index < 0:
raise self.severe('Problem with "end-before" option of "%s" '
'directive:\nText not found.' % self.name)
rawtext = rawtext[:before_index]
include_lines = statemachine.string2lines(rawtext,
tab_width,
convert_whitespace=True)
if 'literal' in self.options:
# Convert tabs to spaces, if `tab_width` is positive.
if tab_width >= 0:
text = rawtext.expandtabs(tab_width)
else:
text = rawtext
literal_block = nodes.literal_block(rawtext,
source=path,
classes=self.options.get(
'class', []))
literal_block.line = 1
self.add_name(literal_block)
if 'number-lines' in self.options:
try:
startline = int(self.options['number-lines'] or 1)
except ValueError:
raise self.error(':number-lines: with non-integer '
'start value')
endline = startline + len(include_lines)
if text.endswith('\n'):
text = text[:-1]
tokens = NumberLines([([], text)], startline, endline)
for classes, value in tokens:
if classes:
literal_block += nodes.inline(value,
value,
classes=classes)
else:
literal_block += nodes.Text(value, value)
else:
literal_block += nodes.Text(text, text)
return [literal_block]
if 'code' in self.options:
self.options['source'] = path
codeblock = CodeBlock(
self.name,
[self.options.pop('code')], # arguments
self.options,
include_lines, # content
self.lineno,
self.content_offset,
self.block_text,
self.state,
self.state_machine)
return codeblock.run()
# CHANGE: add the suffixes to all the references (only do it for vimdocs)
if isinstance(env.app.builder,
VimdocBuilder) and 'ref-suffix' in self.options:
suffix = self.options['ref-suffix']
for i, line in enumerate(include_lines):
# relying on a regex is gross, but it's easy and we just have to worry
# about the eclim docs, so it'll do.
match = self.REF.match(line)
if match:
include_lines[i] = '%s_%s%s' % (match.group(1), suffix,
match.group(2))
self.state_machine.insert_input(include_lines, path)
return []
def run(self):
if not isinstance(self.config.ncs_include_mapping, dict):
raise self.severe('Invalid "ncs_include_mapping" config')
docset = self.options.get('docset', None)
if docset is not None and \
docset not in self.config.ncs_include_mapping:
raise self.severe('The supplied "docset" was not found: "%s"' %
docset)
indent = self.options.get('indent', 0)
dedent = self.options.get('dedent', 0)
auto_dedent = 'auto-dedent' in self.options
if indent and (dedent or auto_dedent):
raise self.severe('Both "indent" and one of the "dedent" options '
'are set, please use only one')
if dedent and auto_dedent:
raise self.severe('Choose one of "dedent" and "auto-dedent" only')
if not self.state.document.settings.file_insertion_enabled:
raise self.warning('"%s" directive disabled.' % self.name)
if docset is None:
# without a docset fallback to Sphinx style include
_, path = self.env.relfn2path(self.arguments[0])
else:
path = os.path.join(self.config.ncs_include_mapping[docset],
self.arguments[0])
encoding = self.options.get(
'encoding', self.state.document.settings.input_encoding)
e_handler = self.state.document.settings.input_encoding_error_handler
tab_width = self.options.get('tab-width',
self.state.document.settings.tab_width)
try:
self.state.document.settings.record_dependencies.add(path)
include_file = io.FileInput(source_path=path,
encoding=encoding,
error_handler=e_handler)
except UnicodeEncodeError:
raise self.severe('Problems with "%s" directive path:\n'
'Cannot encode input file path "%s" '
'(wrong locale?).' %
(self.name, SafeString(path)))
except IOError as error:
raise self.severe('Problems with "%s" directive path:\n%s.' %
(self.name, ErrorString(error)))
# Get to-be-included content
startline = self.options.get('start-line', None)
endline = self.options.get('end-line', None)
try:
if startline or (endline is not None):
lines = include_file.readlines()
rawtext = ''.join(lines[startline:endline])
else:
rawtext = include_file.read()
except UnicodeError as error:
raise self.severe(u'Problem with "%s" directive:\n%s' %
(self.name, ErrorString(error)))
# start-after/end-before: no restrictions on newlines in match-text,
# and no restrictions on matching inside lines vs. line boundaries
after_text = self.options.get('start-after', None)
if after_text:
# skip content in rawtext before *and incl.* a matching text
after_index = rawtext.find(after_text)
if after_index < 0:
raise self.severe('Problem with "start-after" option of "%s" '
'directive:\nText not found.' % self.name)
rawtext = rawtext[after_index + len(after_text):]
before_text = self.options.get('end-before', None)
if before_text:
# skip content in rawtext after *and incl.* a matching text
before_index = rawtext.find(before_text)
if before_index < 0:
raise self.severe('Problem with "end-before" option of "%s" '
'directive:\nText not found.' % self.name)
rawtext = rawtext[:before_index]
include_lines = statemachine.string2lines(rawtext,
tab_width,
convert_whitespace=True)
def is_blank(line):
# XXX might need to check for a stripped line
return len(line) == 0
if auto_dedent:
min_spaces = None
for i, line in enumerate(include_lines):
if is_blank(line):
continue
spaces = len(line) - len(line.lstrip(' '))
if min_spaces is None:
min_spaces = spaces
else:
min_spaces = min(spaces, min_spaces)
print(min_spaces, line)
# it can't get smaller, so leave early
if min_spaces == 0:
break
amount = min_spaces
for i, line in enumerate(include_lines):
if is_blank(line):
continue
include_lines[i] = line[amount:]
if dedent:
for i, line in enumerate(include_lines):
if is_blank(line):
continue
spaces = len(line) - len(line.lstrip())
amount = min(spaces, dedent)
include_lines[i] = line[amount:]
if indent:
spaces = ' ' * indent
for i, line in enumerate(include_lines):
if is_blank(line):
continue
include_lines[i] = spaces + line
self.state_machine.insert_input(include_lines, path)
return []
