class SafeStringTests(unittest.TestCase):
# the error message in EnvironmentError instances comes from the OS
# and in some locales (e.g. ru_RU), contains high bit chars.
# -> see the test in test_error_reporting.py
# test data:
bs = b('\xfc') # unicode(bs) fails, str(bs) in Python 3 return repr()
us = u'\xfc' # bytes(us) fails; str(us) fails in Python 2
be = Exception(bs) # unicode(be) fails
ue = Exception(us) # bytes(ue) fails, str(ue) fails in Python 2;
# unicode(ue) fails in Python < 2.6 (issue2517_)
# .. _issue2517: http://bugs.python.org/issue2517
# wrapped test data:
wbs = SafeString(bs)
wus = SafeString(us)
wbe = SafeString(be)
wue = SafeString(ue)
def test_7bit(self):
# wrapping (not required with 7-bit chars) must not change the
# result of conversions:
bs7 = b('foo')
us7 = u'foo'
be7 = Exception(bs7)
ue7 = Exception(us7)
self.assertEqual(str(42), str(SafeString(42)))
self.assertEqual(str(bs7), str(SafeString(bs7)))
self.assertEqual(str(us7), str(SafeString(us7)))
self.assertEqual(str(be7), str(SafeString(be7)))
self.assertEqual(str(ue7), str(SafeString(ue7)))
self.assertEqual(unicode(7), unicode(SafeString(7)))
self.assertEqual(unicode(bs7), unicode(SafeString(bs7)))
self.assertEqual(unicode(us7), unicode(SafeString(us7)))
self.assertEqual(unicode(be7), unicode(SafeString(be7)))
self.assertEqual(unicode(ue7), unicode(SafeString(ue7)))
def test_ustr(self):
"""Test conversion to a unicode-string."""
# unicode(self.bs) fails
self.assertEqual(unicode, type(unicode(self.wbs)))
self.assertEqual(unicode(self.us), unicode(self.wus))
# unicode(self.be) fails
self.assertEqual(unicode, type(unicode(self.wbe)))
# unicode(ue) fails in Python < 2.6 (issue2517_)
self.assertEqual(unicode, type(unicode(self.wue)))
self.assertEqual(self.us, unicode(self.wue))
def test_str(self):
"""Test conversion to a string (bytes in Python 2, unicode in Python 3)."""
self.assertEqual(str(self.bs), str(self.wbs))
self.assertEqual(str(self.be), str(self.be))
# str(us) fails in Python 2
self.assertEqual(str, type(str(self.wus)))
# str(ue) fails in Python 2
self.assertEqual(str, type(str(self.wue)))
def run(self):
"""Include a file as part of the content of this reST file."""
if not self.state.document.settings.file_insertion_enabled:
raise self.warning('"%s" directive disabled.' % self.name)
source = self.state_machine.input_lines.source(
self.lineno - self.state_machine.input_offset - 1)
source_dir = os.path.dirname(os.path.abspath(source))
path = directives.path(self.arguments[0])
if path.startswith('<') and path.endswith('>'):
path = os.path.join(self.standard_include_path, path[1:-1])
path = os.path.normpath(os.path.join(source_dir, path))
path = utils.relative_path(None, path)
path = nodes.reprunicode(path)
encoding = self.options.get(
'encoding', self.state.document.settings.input_encoding)
tab_width = self.options.get('tab-width',
self.state.document.settings.tab_width)
try:
self.state.document.settings.record_dependencies.add(path)
include_file = io.FileInput(
source_path=path, encoding=encoding,
error_handler=(self.state.document.settings.\
input_encoding_error_handler),
handle_io_errors=None)
except UnicodeEncodeError, error:
raise self.severe(u'Problems with "%s" directive path:\n'
'Cannot encode input file path "%s" '
'(wrong locale?).' %
(self.name, SafeString(path)))
def test_unicode(self):
self.assertEqual(u'Exception: spam',
unicode(ErrorString(Exception(u'spam'))))
self.assertEqual(u'IndexError: '+self.us,
unicode(ErrorString(IndexError(self.us))))
self.assertEqual(u'ImportError: %s' % SafeString(self.bs),
unicode(ErrorString(ImportError(self.bs))))
def test_str(self):
self.assertEqual('Exception: spam',
str(ErrorString(Exception('spam'))))
self.assertEqual('IndexError: '+str(self.bs),
str(ErrorString(IndexError(self.bs))))
self.assertEqual('ImportError: %s' % SafeString(self.us),
str(ErrorString(ImportError(self.us))))
class Role(Directive):
has_content = True
argument_pattern = re.compile(r'(%s)\s*(\(\s*(%s)\s*\)\s*)?$' %
((states.Inliner.simplename, ) * 2))
def run(self):
"""Dynamically create and register a custom interpreted text role."""
if self.content_offset > self.lineno or not self.content:
raise self.error('"%s" directive requires arguments on the first '
'line.' % self.name)
args = self.content[0]
match = self.argument_pattern.match(args)
if not match:
raise self.error('"%s" directive arguments not valid role names: '
'"%s".' % (self.name, args))
new_role_name = match.group(1)
base_role_name = match.group(3)
messages = []
if base_role_name:
base_role, messages = roles.role(base_role_name,
self.state_machine.language,
self.lineno, self.state.reporter)
if base_role is None:
error = self.state.reporter.error(
'Unknown interpreted text role "%s".' % base_role_name,
nodes.literal_block(self.block_text, self.block_text),
line=self.lineno)
return messages + [error]
else:
base_role = roles.generic_custom_role
assert not hasattr(base_role, 'arguments'), (
'Supplemental directive arguments for "%s" directive not '
'supported (specified by "%r" role).' % (self.name, base_role))
try:
converted_role = convert_directive_function(base_role)
(arguments, options, content,
content_offset) = (self.state.parse_directive_block(
self.content[1:],
self.content_offset,
converted_role,
option_presets={}))
except states.MarkupError, detail:
error = self.state_machine.reporter.error(
'Error in "%s" directive:\n%s.' % (self.name, detail),
nodes.literal_block(self.block_text, self.block_text),
line=self.lineno)
return messages + [error]
if 'class' not in options:
try:
options['class'] = directives.class_option(new_role_name)
except ValueError, detail:
error = self.state_machine.reporter.error(
u'Invalid argument for "%s" directive:\n%s.' %
(self.name, SafeString(detail)),
nodes.literal_block(self.block_text, self.block_text),
line=self.lineno)
return messages + [error]
def test_7bit(self):
# wrapping (not required with 7-bit chars) must not change the
# result of conversions:
bs7 = b('foo')
us7 = u'foo'
be7 = Exception(bs7)
ue7 = Exception(us7)
self.assertEqual(str(42), str(SafeString(42)))
self.assertEqual(str(bs7), str(SafeString(bs7)))
self.assertEqual(str(us7), str(SafeString(us7)))
self.assertEqual(str(be7), str(SafeString(be7)))
self.assertEqual(str(ue7), str(SafeString(ue7)))
self.assertEqual(unicode(7), unicode(SafeString(7)))
self.assertEqual(unicode(bs7), unicode(SafeString(bs7)))
self.assertEqual(unicode(us7), unicode(SafeString(us7)))
self.assertEqual(unicode(be7), unicode(SafeString(be7)))
self.assertEqual(unicode(ue7), unicode(SafeString(ue7)))
def system_message(self, level, message, *children, **kwargs):
"""
Return a system_message object.
Raise an exception or generate a warning if appropriate.
"""
# `message` can be a `string`, `unicode`, or `Exception` instance.
if isinstance(message, Exception):
message = SafeString(message)
attributes = kwargs.copy()
if 'base_node' in kwargs:
source, line = get_source_line(kwargs['base_node'])
del attributes['base_node']
if source is not None:
attributes.setdefault('source', source)
if line is not None:
attributes.setdefault('line', line)
# assert source is not None, "node has line- but no source-argument"
if not 'source' in attributes: # 'line' is absolute line number
try: # look up (source, line-in-source)
source, line = self.locator(attributes.get('line'))
# print "locator lookup", kwargs.get('line'), "->", source, line
except AttributeError:
source, line = None, None
if source is not None:
attributes['source'] = source
if line is not None:
attributes['line'] = line
# assert attributes['line'] is not None, (message, kwargs)
# assert attributes['source'] is not None, (message, kwargs)
attributes.setdefault('source', self.source)
msg = nodes.system_message(message,
level=level,
type=self.levels[level],
*children,
**attributes)
if self.stream and (level >= self.report_level
or self.debug_flag and level == self.DEBUG_LEVEL
or level >= self.halt_level):
self.stream.write(msg.astext() + '\n')
if level >= self.halt_level:
raise SystemMessage(msg, level)
if level > self.DEBUG_LEVEL or self.debug_flag:
self.notify_observers(msg)
self.max_level = max(level, self.max_level)
return msg
def get_csv_data(self):
"""
Get CSV data from the directive content, from an external
file, or from a URL reference.
"""
encoding = self.options.get(
'encoding', self.state.document.settings.input_encoding)
if self.content:
# CSV data is from directive content.
if 'file' in self.options or 'url' in self.options:
error = self.state_machine.reporter.error(
'"%s" directive may not both specify an external file and'
' have content.' % self.name,
nodes.literal_block(self.block_text, self.block_text),
line=self.lineno)
raise SystemMessagePropagation(error)
source = self.content.source(0)
csv_data = self.content
elif 'file' in self.options:
# CSV data is from an external file.
if 'url' in self.options:
error = self.state_machine.reporter.error(
'The "file" and "url" options may not be simultaneously'
' specified for the "%s" directive.' % self.name,
nodes.literal_block(self.block_text, self.block_text),
line=self.lineno)
raise SystemMessagePropagation(error)
source_dir = os.path.dirname(
os.path.abspath(self.state.document.current_source))
source = os.path.normpath(
os.path.join(source_dir, self.options['file']))
source = utils.relative_path(None, source)
try:
self.state.document.settings.record_dependencies.add(source)
csv_file = io.FileInput(
source_path=source, encoding=encoding,
error_handler=(self.state.document.settings.\
input_encoding_error_handler),
handle_io_errors=None)
csv_data = csv_file.read().splitlines()
except IOError, error:
severe = self.state_machine.reporter.severe(
u'Problems with "%s" directive path:\n%s.' %
(self.name, SafeString(error)),
nodes.literal_block(self.block_text, self.block_text),
line=self.lineno)
raise SystemMessagePropagation(severe)
nodes.literal_block(self.block_text, self.block_text),
line=self.lineno)
raise SystemMessagePropagation(severe)
elif 'url' in self.options:
# CSV data is from a URL.
# Do not import urllib2 at the top of the module because
# it may fail due to broken SSL dependencies, and it takes
# about 0.15 seconds to load.
import urllib2
source = self.options['url']
try:
csv_text = urllib2.urlopen(source).read()
except (urllib2.URLError, IOError, OSError, ValueError), error:
severe = self.state_machine.reporter.severe(
'Problems with "%s" directive URL "%s":\n%s.' %
(self.name, self.options['url'], SafeString(error)),
nodes.literal_block(self.block_text, self.block_text),
line=self.lineno)
raise SystemMessagePropagation(severe)
csv_file = io.StringInput(
source=csv_text, source_path=source, encoding=encoding,
error_handler=(self.state.document.settings.\
input_encoding_error_handler))
csv_data = csv_file.read().splitlines()
else:
error = self.state_machine.reporter.warning(
'The "%s" directive requires content; none supplied.' %
self.name,
nodes.literal_block(self.block_text, self.block_text),
line=self.lineno)
raise SystemMessagePropagation(error)
def run(self):
"""Include a file as part of the content of this reST file."""
if not self.state.document.settings.file_insertion_enabled:
raise self.warning('"%s" directive disabled.' % self.name)
source = self.state_machine.input_lines.source(
self.lineno - self.state_machine.input_offset - 1)
source_dir = os.path.dirname(os.path.abspath(source))
path = directives.path(self.arguments[0])
if path.startswith('<') and path.endswith('>'):
path = os.path.join(self.standard_include_path, path[1:-1])
path = os.path.normpath(os.path.join(source_dir, path))
path = utils.relative_path(None, path)
path = nodes.reprunicode(path)
encoding = self.options.get(
'encoding', self.state.document.settings.input_encoding)
tab_width = self.options.get(
'tab-width', self.state.document.settings.tab_width)
try:
self.state.document.settings.record_dependencies.add(path)
include_file = io.FileInput(
source_path=path, encoding=encoding,
error_handler=(self.state.document.settings.\
input_encoding_error_handler),
handle_io_errors=None)
except UnicodeEncodeError as error:
raise self.severe('Problems with "%s" directive path:\n'
'Cannot encode input file path "%s" '
'(wrong locale?).' %
(self.name, SafeString(path)))
except IOError as error:
raise self.severe('Problems with "%s" directive path:\n%s.' %
(self.name, ErrorString(error)))
startline = self.options.get('start-line', None)
endline = self.options.get('end-line', None)
try:
if startline or (endline is not None):
lines = include_file.readlines()
rawtext = ''.join(lines[startline:endline])
else:
rawtext = include_file.read()
except UnicodeError as error:
raise self.severe('Problem with "%s" directive:\n%s' %
(self.name, ErrorString(error)))
# start-after/end-before: no restrictions on newlines in match-text,
# and no restrictions on matching inside lines vs. line boundaries
after_text = self.options.get('start-after', None)
if after_text:
# skip content in rawtext before *and incl.* a matching text
after_index = rawtext.find(after_text)
if after_index < 0:
raise self.severe('Problem with "start-after" option of "%s" '
'directive:\nText not found.' % self.name)
rawtext = rawtext[after_index + len(after_text):]
before_text = self.options.get('end-before', None)
if before_text:
# skip content in rawtext after *and incl.* a matching text
before_index = rawtext.find(before_text)
if before_index < 0:
raise self.severe('Problem with "end-before" option of "%s" '
'directive:\nText not found.' % self.name)
rawtext = rawtext[:before_index]
include_lines = statemachine.string2lines(rawtext, tab_width,
convert_whitespace=True)
if 'literal' in self.options:
# Convert tabs to spaces, if `tab_width` is positive.
if tab_width >= 0:
text = rawtext.expandtabs(tab_width)
else:
text = rawtext
literal_block = nodes.literal_block(rawtext, source=path,
classes=self.options.get('class', []))
literal_block.line = 1
self.add_name(literal_block)
if 'number-lines' in self.options:
try:
startline = int(self.options['number-lines'] or 1)
except ValueError:
raise self.error(':number-lines: with non-integer '
'start value')
endline = startline + len(include_lines)
if text.endswith('\n'):
text = text[:-1]
tokens = NumberLines([([], text)], startline, endline)
for classes, value in tokens:
if classes:
literal_block += nodes.inline(value, value,
classes=classes)
else:
literal_block += nodes.Text(value, value)
else:
literal_block += nodes.Text(text, text)
return [literal_block]
if 'code' in self.options:
self.options['source'] = path
codeblock = CodeBlock(self.name,
[self.options.pop('code')], # arguments
self.options,
include_lines, # content
self.lineno,
self.content_offset,
self.block_text,
self.state,
self.state_machine)
return codeblock.run()
self.state_machine.insert_input(include_lines, path)
return []
def get_csv_data(self):
"""
Get CSV data from the directive content, from an external
file, or from a URL reference.
"""
encoding = self.options.get(
'encoding', self.state.document.settings.input_encoding)
if self.content:
# CSV data is from directive content.
if 'file' in self.options or 'url' in self.options:
error = self.state_machine.reporter.error(
'"%s" directive may not both specify an external file and'
' have content.' % self.name,
nodes.literal_block(self.block_text, self.block_text),
line=self.lineno)
raise SystemMessagePropagation(error)
source = self.content.source(0)
csv_data = self.content
elif 'file' in self.options:
# CSV data is from an external file.
if 'url' in self.options:
error = self.state_machine.reporter.error(
'The "file" and "url" options may not be simultaneously'
' specified for the "%s" directive.' % self.name,
nodes.literal_block(self.block_text, self.block_text),
line=self.lineno)
raise SystemMessagePropagation(error)
source_dir = os.path.dirname(
os.path.abspath(self.state.document.current_source))
source = os.path.normpath(
os.path.join(source_dir, self.options['file']))
source = utils.relative_path(None, source)
try:
self.state.document.settings.record_dependencies.add(source)
csv_file = io.FileInput(
source_path=source, encoding=encoding,
error_handler=(self.state.document.settings.\
input_encoding_error_handler),
handle_io_errors=None)
csv_data = csv_file.read().splitlines()
except IOError as error:
severe = self.state_machine.reporter.severe(
'Problems with "%s" directive path:\n%s.' %
(self.name, SafeString(error)),
nodes.literal_block(self.block_text, self.block_text),
line=self.lineno)
raise SystemMessagePropagation(severe)
elif 'url' in self.options:
# CSV data is from a URL.
# Do not import urllib2 at the top of the module because
# it may fail due to broken SSL dependencies, and it takes
# about 0.15 seconds to load.
import urllib.request, urllib.error, urllib.parse
source = self.options['url']
try:
csv_text = urllib.request.urlopen(source).read()
except (urllib.error.URLError, IOError, OSError,
ValueError) as error:
severe = self.state_machine.reporter.severe(
'Problems with "%s" directive URL "%s":\n%s.' %
(self.name, self.options['url'], SafeString(error)),
nodes.literal_block(self.block_text, self.block_text),
line=self.lineno)
raise SystemMessagePropagation(severe)
csv_file = io.StringInput(
source=csv_text, source_path=source, encoding=encoding,
error_handler=(self.state.document.settings.\
input_encoding_error_handler))
csv_data = csv_file.read().splitlines()
else:
error = self.state_machine.reporter.warning(
'The "%s" directive requires content; none supplied.' %
self.name,
nodes.literal_block(self.block_text, self.block_text),
line=self.lineno)
raise SystemMessagePropagation(error)
return csv_data, source
uioe = e
try:
os.chdir(b('\xfc'))
except OSError, e:
bose = e
try:
os.chdir(u'\xfc')
except OSError, e:
uose = e
except UnicodeEncodeError:
try:
os.chdir(u'\xfc'.encode(sys.getfilesystemencoding(), 'replace'))
except OSError, e:
uose = e
# wrapped test data:
wbioe = SafeString(bioe)
wuioe = SafeString(uioe)
wbose = SafeString(bose)
wuose = SafeString(uose)
# reset locale
if testlocale:
locale.setlocale(locale.LC_ALL, oldlocale)
def test_ustr(self):
"""Test conversion to a unicode-string."""
# unicode(bioe) fails with e.g. 'ru_RU.utf8' locale
self.assertEqual(unicode, type(unicode(self.wbioe)))
self.assertEqual(unicode, type(unicode(self.wuioe)))
self.assertEqual(unicode, type(unicode(self.wbose)))
self.assertEqual(unicode, type(unicode(self.wuose)))