def _test_parse_string_literal(text, flags):
r"""
Attempt to parse ``text``. If it parses cleanly to a single string
literal, return its value. Otherwise return ``None``.
>>> _test_parse_string_literal(r'"foo\n" r"\nbar"', 0)
'foo\n\\nbar'
"""
text = FileText(text)
if PY2:
try:
text.joined.encode('ascii')
except UnicodeError:
text = FileText(u'# encoding: utf-8\n' + unicode(text),
filename=text.filename)
try:
module_node = _parse_ast_nodes(text, flags, False, "eval")
except SyntaxError:
return None
body = module_node.body
if not isinstance(body, (ast.Str, Bytes)):
return None
return body.s
def test_FileText_eqne_1():
text1a = FileText("hello\n")
text1b = FileText("hello\n")
text2 = FileText("hello\nhello\n")
assert (text1a == text1b)
assert not (text1a != text1b)
assert (text1a != text2 )
assert not (text1a == text2 )
def test_FileText_eqne_lineno_1():
text1a = FileText("hello\n", startpos=(100,1))
text1b = FileText("hello\n", startpos=(100,1))
text2 = FileText("hello\n", startpos=(101,1))
assert (text1a == text1b)
assert not (text1a != text1b)
assert (text1a != text2 )
assert not (text1a == text2 )
def test_FileText_eqne_filename_1():
text1a = FileText("hello\n", filename='/foo')
text1b = FileText("hello\n", filename='/foo')
text2 = FileText("hello\n", filename='/bar')
assert (text1a == text1b)
assert not (text1a != text1b)
assert (text1a != text2 )
assert not (text1a == text2 )
def test_PythonBlock_FileText_1():
text = FileText(dedent('''
foo()
bar()
''').lstrip(), filename="/foo/test_PythonBlock_1.py", startpos=(101,55))
block = PythonBlock(text)
assert text is block.text
assert text is FileText(block)
assert block.filename == Filename("/foo/test_PythonBlock_1.py")
assert block.startpos == FilePos(101, 55)
def from_text(cls,
text,
filename=None,
startpos=None,
flags=None,
auto_flags=False):
"""
@type text:
L{FileText} or convertible
@type filename:
C{Filename}
@param filename:
Filename, if not already given by C{text}.
@type startpos:
C{FilePos}
@param startpos:
Starting position, if not already given by C{text}.
@type flags:
C{CompilerFlags}
@param flags:
Input compiler flags.
@param auto_flags:
Whether to try other flags if C{flags} fails.
@rtype:
L{PythonBlock}
"""
text = FileText(text, filename=filename, startpos=startpos)
self = object.__new__(cls)
self.text = text
self._input_flags = CompilerFlags(flags)
self._auto_flags = auto_flags
return self
def from_text(cls,
text,
filename=None,
startpos=None,
flags=None,
auto_flags=False):
"""
:type text:
`FileText` or convertible
:type filename:
``Filename``
:param filename:
Filename, if not already given by ``text``.
:type startpos:
``FilePos``
:param startpos:
Starting position, if not already given by ``text``.
:type flags:
``CompilerFlags``
:param flags:
Input compiler flags.
:param auto_flags:
Whether to try other flags if ``flags`` fails.
:rtype:
`PythonBlock`
"""
text = FileText(text, filename=filename, startpos=startpos)
self = object.__new__(cls)
self.text = text
self._input_flags = CompilerFlags(flags)
self._auto_flags = auto_flags
return self
def _parse_ast_nodes(text, flags, auto_flags, mode):
"""
Parse a block of lines into an AST.
Also annotate ``input_flags``, ``source_flags``, and ``flags`` on the
resulting ast node.
:type text:
``FileText``
:type flags:
``CompilerFlags``
:type auto_flags:
``bool``
:param auto_flags:
Whether to guess different flags if ``text`` can't be parsed with
``flags``.
:param mode:
Compilation mode: "exec", "single", or "eval".
:rtype:
``ast.Module``
"""
text = FileText(text)
flags = CompilerFlags(flags)
filename = str(text.filename) if text.filename else "<unknown>"
source = text.joined
source = dedent(source)
if PY2 and isinstance(source, unicode):
source = source.encode('utf-8')
if not source.endswith("\n"):
# Ensure that the last line ends with a newline (``ast`` barfs
# otherwise).
source += "\n"
exp = None
for flags in _flags_to_try(source, flags, auto_flags, mode):
cflags = ast.PyCF_ONLY_AST | int(flags)
try:
result = compile(source,
filename,
mode,
flags=cflags,
dont_inherit=1)
except SyntaxError as e:
exp = e
pass
else:
# Attach flags to the result.
result.input_flags = flags
result.source_flags = CompilerFlags.from_ast(result)
result.flags = result.input_flags | result.source_flags
result.text = text
return result
raise exp # SyntaxError
def _parse_ast_nodes(text, flags, auto_flags, mode):
"""
Parse a block of lines into an AST.
Also annotate C{input_flags}, C{source_flags}, and C{flags} on the
resulting ast node.
@type text:
C{FileText}
@type flags:
C{CompilerFlags}
@type auto_flags:
C{bool}
@param auto_flags:
Whether to guess different flags if C{text} can't be parsed with
C{flags}.
@param mode:
Compilation mode: "exec", "single", or "eval".
@rtype:
C{ast.Module}
"""
text = FileText(text)
flags = CompilerFlags(flags)
filename = str(text.filename) if text.filename else "<unknown>"
source = text.joined
source = dedent(source)
if not source.endswith("\n"):
# Ensure that the last line ends with a newline (C{ast} barfs
# otherwise).
source += "\n"
exp = None
for flags in _flags_to_try(source, flags, auto_flags, mode):
cflags = ast.PyCF_ONLY_AST | int(flags)
try:
result = compile(source,
filename,
mode,
flags=cflags,
dont_inherit=1)
except SyntaxError as e:
exp = e
pass
else:
# Attach flags to the result.
result.input_flags = flags
result.source_flags = CompilerFlags.from_ast(result)
result.flags = result.input_flags | result.source_flags
result.text = text
return result
raise exp # SyntaxError
def get_doctests(self):
r"""
Return doctests in this code.
>>> PythonBlock("x\n'''\n >>> foo(bar\n ... + baz)\n'''\n").get_doctests()
[PythonBlock('foo(bar\n + baz)\n', startpos=(3,2))]
@rtype:
C{list} of L{PythonStatement}s
"""
import doctest
parser = doctest.DocTestParser()
doctest_blocks = []
filename = self.filename
flags = self.flags
for ast_node in self._get_docstring_nodes():
try:
examples = parser.get_examples(ast_node.s)
except Exception:
blob = ast_node.s
if len(blob) > 60:
blob = blob[:60] + '...'
# TODO: let caller decide how to handle
logger.warning("Can't parse docstring; ignoring: %r", blob)
continue
for example in examples:
lineno = ast_node.startpos.lineno + example.lineno
colno = ast_node.startpos.colno + example.indent # dubious
text = FileText(example.source,
filename=filename,
startpos=(lineno, colno))
try:
block = PythonBlock(text, flags=flags)
block.ast_node # make sure we can parse
except Exception:
blob = text.joined
if len(blob) > 60:
blob = blob[:60] + '...'
logger.warning("Can't parse doctest; ignoring: %r", blob)
continue
doctest_blocks.append(block)
return doctest_blocks
def concatenate(cls, blocks, assume_contiguous=False):
"""
Concatenate a bunch of blocks into one block.
@type blocks:
sequence of L{PythonBlock}s and/or L{PythonStatement}s
@param assume_contiguous:
Whether to assume, without checking, that the input blocks were
originally all contiguous. This must be set to True to indicate the
caller understands the assumption; False is not implemented.
"""
if not assume_contiguous:
raise NotImplementedError
blocks = [PythonBlock(b) for b in blocks]
if len(blocks) == 1:
return blocks[0]
assert blocks
text = FileText.concatenate([b.text for b in blocks])
# The contiguous assumption is important here because C{ast_node}
# contains line information that would otherwise be wrong.
ast_nodes = [n for b in blocks for n in b.annotated_ast_node.body]
flags = blocks[0].flags
return cls.__construct_from_annotated_ast(ast_nodes, text, flags)
def pretty_print(self, params=None):
params = ImportFormatParams(params)
result = [block.pretty_print(params=params) for block in self.blocks]
return FileText.concatenate(result)
def text(self):
return FileText(self.filename)
def output_content(self):
return FileText(self.modifier(self.input_content), filename=self.filename)
def test_FileText_one_partial_line_offset_1():
text = FileText("foo", startpos=(101,55))
assert text.endpos == FilePos(101,58)
def test_FileText_slice_col_almost_eof_1():
text = FileText("two4567\nthree6789\nfour\n", startpos=(102,101))
result = text[ (102,103) : (104,5) ]
expected = FileText("o4567\nthree6789\nfour", startpos=(102,103))
assert result == expected
def test_FileText_ne_other_1():
text = FileText("hello\n")
assert (text != object())
assert not (text == object())
assert (text != "hello\n")
assert not (text == "hello\n")
def test_FileText_slice_offset_1():
text = FileText("a\nb\nc\nd", startpos=(101,5))
assert text[102:104] == FileText('b\nc\n', startpos=(102,1))
def test_FileText_endpos_offset_1():
text = FileText("foo\nbar\n", startpos=(101,55))
assert text.endpos == FilePos(103,1)
def test_FileText_slice_out_of_range_empty_1():
text = FileText("two4567\nthree6789\nfour\n", startpos=(102,101))
with pytest.raises(IndexError):
text[ (102,200) : (102,200) ]
def test_FileText_getitem_out_of_range_1():
text = FileText("a\nb\nc\nd")
with pytest.raises(IndexError):
text[0]
def test_FileText_slice_col_1():
text = FileText("one\ntwo4567\nthree6789\nfour\n", startpos=(101,55))
result = text[ (102,3) : (103,8) ]
expected = FileText("o4567\nthree67", startpos=(102,3))
assert result == expected
def test_FileText_slice_col_out_of_range_end_colno_2():
text = FileText("two4567\nthree6789\nfour\n", startpos=(102,101))
with pytest.raises(IndexError):
text[ (102,103) : (104,6) ]
def test_FileText_slice_col_out_of_range_start_lineno_1():
text = FileText("two4567\nthree6789\nfour\n", startpos=(102,101))
with pytest.raises(IndexError):
text[ (101,1) : (103,1) ]
def test_FileText_one_full_line_offset_1():
text = FileText("foo\n", startpos=(101,55))
assert text.endpos == FilePos(102,1)
def test_FileText_slice_1():
assert FileText("a\nb\nc\nd")[2:4] == FileText('b\nc\n', startpos=(2,1))
def test_FileText_empty_1():
text = FileText("", startpos=(5,5))
assert text.lines == ("",)
assert text.joined == ""
assert text.startpos == text.endpos == FilePos(5,5)
def test_FileText_getitem_1():
assert FileText("a\nb\nc\nd")[2] == 'b'
def test_FileText_slice_empty_1():
text = FileText("two4567\nthree6789\nfour\n", startpos=(102,101))
result = text[ (102,103) : (102,103) ]
expected = FileText("", startpos=(102,103))
assert result == expected
def test_FileText_slice_idempotent_2():
text = FileText("two4567\nthree6789\nfour", startpos=(102,101))
result = text[ (102,101) : (104,5) ]
assert result is text
