def _test_parse_string_literal(text, flags): r""" Attempt to parse ``text``. If it parses cleanly to a single string literal, return its value. Otherwise return ``None``. >>> _test_parse_string_literal(r'"foo\n" r"\nbar"', 0) 'foo\n\\nbar' """ text = FileText(text) if PY2: try: text.joined.encode('ascii') except UnicodeError: text = FileText(u'# encoding: utf-8\n' + unicode(text), filename=text.filename) try: module_node = _parse_ast_nodes(text, flags, False, "eval") except SyntaxError: return None body = module_node.body if not isinstance(body, (ast.Str, Bytes)): return None return body.s
def test_FileText_eqne_1(): text1a = FileText("hello\n") text1b = FileText("hello\n") text2 = FileText("hello\nhello\n") assert (text1a == text1b) assert not (text1a != text1b) assert (text1a != text2 ) assert not (text1a == text2 )
def test_FileText_eqne_lineno_1(): text1a = FileText("hello\n", startpos=(100,1)) text1b = FileText("hello\n", startpos=(100,1)) text2 = FileText("hello\n", startpos=(101,1)) assert (text1a == text1b) assert not (text1a != text1b) assert (text1a != text2 ) assert not (text1a == text2 )
def test_FileText_eqne_filename_1(): text1a = FileText("hello\n", filename='/foo') text1b = FileText("hello\n", filename='/foo') text2 = FileText("hello\n", filename='/bar') assert (text1a == text1b) assert not (text1a != text1b) assert (text1a != text2 ) assert not (text1a == text2 )
def test_PythonBlock_FileText_1(): text = FileText(dedent(''' foo() bar() ''').lstrip(), filename="/foo/test_PythonBlock_1.py", startpos=(101,55)) block = PythonBlock(text) assert text is block.text assert text is FileText(block) assert block.filename == Filename("/foo/test_PythonBlock_1.py") assert block.startpos == FilePos(101, 55)
def from_text(cls, text, filename=None, startpos=None, flags=None, auto_flags=False): """ @type text: L{FileText} or convertible @type filename: C{Filename} @param filename: Filename, if not already given by C{text}. @type startpos: C{FilePos} @param startpos: Starting position, if not already given by C{text}. @type flags: C{CompilerFlags} @param flags: Input compiler flags. @param auto_flags: Whether to try other flags if C{flags} fails. @rtype: L{PythonBlock} """ text = FileText(text, filename=filename, startpos=startpos) self = object.__new__(cls) self.text = text self._input_flags = CompilerFlags(flags) self._auto_flags = auto_flags return self
def from_text(cls, text, filename=None, startpos=None, flags=None, auto_flags=False): """ :type text: `FileText` or convertible :type filename: ``Filename`` :param filename: Filename, if not already given by ``text``. :type startpos: ``FilePos`` :param startpos: Starting position, if not already given by ``text``. :type flags: ``CompilerFlags`` :param flags: Input compiler flags. :param auto_flags: Whether to try other flags if ``flags`` fails. :rtype: `PythonBlock` """ text = FileText(text, filename=filename, startpos=startpos) self = object.__new__(cls) self.text = text self._input_flags = CompilerFlags(flags) self._auto_flags = auto_flags return self
def _parse_ast_nodes(text, flags, auto_flags, mode): """ Parse a block of lines into an AST. Also annotate ``input_flags``, ``source_flags``, and ``flags`` on the resulting ast node. :type text: ``FileText`` :type flags: ``CompilerFlags`` :type auto_flags: ``bool`` :param auto_flags: Whether to guess different flags if ``text`` can't be parsed with ``flags``. :param mode: Compilation mode: "exec", "single", or "eval". :rtype: ``ast.Module`` """ text = FileText(text) flags = CompilerFlags(flags) filename = str(text.filename) if text.filename else "<unknown>" source = text.joined source = dedent(source) if PY2 and isinstance(source, unicode): source = source.encode('utf-8') if not source.endswith("\n"): # Ensure that the last line ends with a newline (``ast`` barfs # otherwise). source += "\n" exp = None for flags in _flags_to_try(source, flags, auto_flags, mode): cflags = ast.PyCF_ONLY_AST | int(flags) try: result = compile(source, filename, mode, flags=cflags, dont_inherit=1) except SyntaxError as e: exp = e pass else: # Attach flags to the result. result.input_flags = flags result.source_flags = CompilerFlags.from_ast(result) result.flags = result.input_flags | result.source_flags result.text = text return result raise exp # SyntaxError
def _parse_ast_nodes(text, flags, auto_flags, mode): """ Parse a block of lines into an AST. Also annotate C{input_flags}, C{source_flags}, and C{flags} on the resulting ast node. @type text: C{FileText} @type flags: C{CompilerFlags} @type auto_flags: C{bool} @param auto_flags: Whether to guess different flags if C{text} can't be parsed with C{flags}. @param mode: Compilation mode: "exec", "single", or "eval". @rtype: C{ast.Module} """ text = FileText(text) flags = CompilerFlags(flags) filename = str(text.filename) if text.filename else "<unknown>" source = text.joined source = dedent(source) if not source.endswith("\n"): # Ensure that the last line ends with a newline (C{ast} barfs # otherwise). source += "\n" exp = None for flags in _flags_to_try(source, flags, auto_flags, mode): cflags = ast.PyCF_ONLY_AST | int(flags) try: result = compile(source, filename, mode, flags=cflags, dont_inherit=1) except SyntaxError as e: exp = e pass else: # Attach flags to the result. result.input_flags = flags result.source_flags = CompilerFlags.from_ast(result) result.flags = result.input_flags | result.source_flags result.text = text return result raise exp # SyntaxError
def get_doctests(self): r""" Return doctests in this code. >>> PythonBlock("x\n'''\n >>> foo(bar\n ... + baz)\n'''\n").get_doctests() [PythonBlock('foo(bar\n + baz)\n', startpos=(3,2))] @rtype: C{list} of L{PythonStatement}s """ import doctest parser = doctest.DocTestParser() doctest_blocks = [] filename = self.filename flags = self.flags for ast_node in self._get_docstring_nodes(): try: examples = parser.get_examples(ast_node.s) except Exception: blob = ast_node.s if len(blob) > 60: blob = blob[:60] + '...' # TODO: let caller decide how to handle logger.warning("Can't parse docstring; ignoring: %r", blob) continue for example in examples: lineno = ast_node.startpos.lineno + example.lineno colno = ast_node.startpos.colno + example.indent # dubious text = FileText(example.source, filename=filename, startpos=(lineno, colno)) try: block = PythonBlock(text, flags=flags) block.ast_node # make sure we can parse except Exception: blob = text.joined if len(blob) > 60: blob = blob[:60] + '...' logger.warning("Can't parse doctest; ignoring: %r", blob) continue doctest_blocks.append(block) return doctest_blocks
def concatenate(cls, blocks, assume_contiguous=False): """ Concatenate a bunch of blocks into one block. @type blocks: sequence of L{PythonBlock}s and/or L{PythonStatement}s @param assume_contiguous: Whether to assume, without checking, that the input blocks were originally all contiguous. This must be set to True to indicate the caller understands the assumption; False is not implemented. """ if not assume_contiguous: raise NotImplementedError blocks = [PythonBlock(b) for b in blocks] if len(blocks) == 1: return blocks[0] assert blocks text = FileText.concatenate([b.text for b in blocks]) # The contiguous assumption is important here because C{ast_node} # contains line information that would otherwise be wrong. ast_nodes = [n for b in blocks for n in b.annotated_ast_node.body] flags = blocks[0].flags return cls.__construct_from_annotated_ast(ast_nodes, text, flags)
def pretty_print(self, params=None): params = ImportFormatParams(params) result = [block.pretty_print(params=params) for block in self.blocks] return FileText.concatenate(result)
def text(self): return FileText(self.filename)
def output_content(self): return FileText(self.modifier(self.input_content), filename=self.filename)
def test_FileText_one_partial_line_offset_1(): text = FileText("foo", startpos=(101,55)) assert text.endpos == FilePos(101,58)
def test_FileText_slice_col_almost_eof_1(): text = FileText("two4567\nthree6789\nfour\n", startpos=(102,101)) result = text[ (102,103) : (104,5) ] expected = FileText("o4567\nthree6789\nfour", startpos=(102,103)) assert result == expected
def test_FileText_ne_other_1(): text = FileText("hello\n") assert (text != object()) assert not (text == object()) assert (text != "hello\n") assert not (text == "hello\n")
def test_FileText_slice_offset_1(): text = FileText("a\nb\nc\nd", startpos=(101,5)) assert text[102:104] == FileText('b\nc\n', startpos=(102,1))
def test_FileText_endpos_offset_1(): text = FileText("foo\nbar\n", startpos=(101,55)) assert text.endpos == FilePos(103,1)
def test_FileText_slice_out_of_range_empty_1(): text = FileText("two4567\nthree6789\nfour\n", startpos=(102,101)) with pytest.raises(IndexError): text[ (102,200) : (102,200) ]
def test_FileText_getitem_out_of_range_1(): text = FileText("a\nb\nc\nd") with pytest.raises(IndexError): text[0]
def test_FileText_slice_col_1(): text = FileText("one\ntwo4567\nthree6789\nfour\n", startpos=(101,55)) result = text[ (102,3) : (103,8) ] expected = FileText("o4567\nthree67", startpos=(102,3)) assert result == expected
def test_FileText_slice_col_out_of_range_end_colno_2(): text = FileText("two4567\nthree6789\nfour\n", startpos=(102,101)) with pytest.raises(IndexError): text[ (102,103) : (104,6) ]
def test_FileText_slice_col_out_of_range_start_lineno_1(): text = FileText("two4567\nthree6789\nfour\n", startpos=(102,101)) with pytest.raises(IndexError): text[ (101,1) : (103,1) ]
def test_FileText_one_full_line_offset_1(): text = FileText("foo\n", startpos=(101,55)) assert text.endpos == FilePos(102,1)
def test_FileText_slice_1(): assert FileText("a\nb\nc\nd")[2:4] == FileText('b\nc\n', startpos=(2,1))
def test_FileText_empty_1(): text = FileText("", startpos=(5,5)) assert text.lines == ("",) assert text.joined == "" assert text.startpos == text.endpos == FilePos(5,5)
def test_FileText_getitem_1(): assert FileText("a\nb\nc\nd")[2] == 'b'
def test_FileText_slice_empty_1(): text = FileText("two4567\nthree6789\nfour\n", startpos=(102,101)) result = text[ (102,103) : (102,103) ] expected = FileText("", startpos=(102,103)) assert result == expected
def test_FileText_slice_idempotent_2(): text = FileText("two4567\nthree6789\nfour", startpos=(102,101)) result = text[ (102,101) : (104,5) ] assert result is text