def _replace_magics(source: Sequence[str],
magic_substitutions: List[MagicHandler],
command: str) -> Iterator[str]:
"""
Replace IPython line magics with valid python code.
Parameters
----------
source
Source from notebook cell.
magic_substitutions
List to store all the ipython magics substitutions
Yields
------
str
Line from cell, with line magics replaced with python code
"""
source_itr = enumerate(source)
for line_no, line in source_itr:
if MagicHandler.is_ipython_magic(line):
# always pass the source starting from the current line
ipython_magic = _extract_ipython_magic(line, source_itr)
magic_handler = MagicHandler.get_magic_handler(
ipython_magic, command)
magic_substitutions.append(magic_handler)
line = _handle_magic_indentation(source[:line_no], ipython_magic,
magic_handler.replace_magic())
yield line
def _replace_magics(
source: List[str],
magic_substitutions: List[MagicSubstitution]) -> Iterator[str]:
"""
Replace IPython line magics with valid python code.
Parameters
----------
source
Source from notebook cell.
magic_substitutions
List to store all the ipython magics substitutions
Yields
------
str
Line from cell, with line magics replaced with python code
"""
for line_no, line in enumerate(source):
trimmed_line: str = line.strip()
if MagicHandler.is_ipython_magic(trimmed_line):
magic_handler = MagicHandler.get_magic_handler(trimmed_line)
magic_substitution = magic_handler.replace_magic(trimmed_line)
magic_substitutions.append(magic_substitution)
line = _handle_magic_indentation(source[:line_no], line,
magic_substitution)
yield line
def _process_source(
source: str,
command: str,
magic_substitutions: List[MagicHandler],
*,
skip_bad_cells: bool,
) -> str:
"""Temporarily replace ipython magics - don't process if can't."""
try:
ast.parse(source)
except SyntaxError:
pass
else:
# Source has no IPython magic, return it directly
return source
body = TransformerManager().transform_cell(source)
if len(body.splitlines()) != len(source.splitlines()):
handler = MagicHandler(source, command, magic_type=None)
magic_substitutions.append(handler)
return handler.replacement
try:
tree = ast.parse(body)
except SyntaxError:
if skip_bad_cells:
handler = MagicHandler(source, command, magic_type=None)
magic_substitutions.append(handler)
return handler.replacement
return source
system_assigns_finder = SystemAssignsFinder()
system_assigns_finder.visit(tree)
visitor = Visitor(system_assigns_finder.system_assigns)
visitor.visit(tree)
new_src = []
for i, line in enumerate(body.splitlines(), start=1):
if i in visitor.magics:
col_offset, src, magic_type = visitor.magics[i][0]
if src is None or len(visitor.magics[i]) > 1:
# unusual case - skip cell completely for now
handler = MagicHandler(source, command, magic_type=magic_type)
magic_substitutions.append(handler)
return handler.replacement
handler = MagicHandler(
src,
command,
magic_type=magic_type,
)
magic_substitutions.append(handler)
line = line[:col_offset] + handler.replacement
new_src.append(line)
return "\n" * (len(source) - len(source.lstrip("\n"))) + "\n".join(new_src)
def _should_ignore_code_cell(source: Sequence[str],
process_cells: Sequence[str]) -> bool:
"""
Return True if the current cell should be ignored from processing.
Parameters
----------
source
Source from the notebook cell
process_cells
Extra cells which nbqa should process.
Returns
-------
bool
True if the cell should ignored else False
"""
if not source:
return True
process = MAGIC + [i.strip() for i in process_cells]
magic_type = MagicHandler.get_ipython_magic_type(source[0])
if magic_type != IPythonMagicType.CELL:
return False
first_line = source[0].lstrip()
return first_line.split()[0] not in {f"%%{magic}" for magic in process}
def _extract_ipython_magic(magic: str,
cell_source: Iterator[Tuple[int, str]]) -> str:
r"""Extract the ipython magic from the notebook cell source.
To extract ipython magic, we use `nbconvert.get_lines` because it can extract
the ipython magic statement that can span multiple lines.
.. code:: python
# example of ipython magic spanning multiple lines
%time result_pymc3 = eval_lda(\
transform_pymc3, beta_pymc3, docs_te.toarray(), np.arange(100)\
)
`nbconvert.filters.get_lines` has the capability to parse such line magics and
return the result as a single line with trailing backslash removed. `get_lines`
would return `%time result_pymc3 = eval_lda( transform_pymc3, beta_pymc3,
docs_te.toarray(), np.arange(100) )`
If the magic was spanning multiple lines, then we need to remove all trailing
backslashes introduced by previous runs of nbqa. Also we do need to preserve the
newline characters, otherwise tools like black will format the code again. Linters
like flake8, pylint will complain about line length. After we extract the code, we
should have string like below
.. code:: python
%time result_pymc3 =eval_lda(
transform_pymc3, beta_pymc3, docs_te.toarray(), np.arange(100)
)
Parameters
----------
magic : str
First line of the ipython magic
cell_source: Iterator[Tuple[int, str]]
Iterator to the notebook cell source
Returns
-------
str
IPython line magic statement
"""
magic_type = MagicHandler.get_ipython_magic_type(magic)
if magic_type and magic_type not in [
IPythonMagicType.CELL, IPythonMagicType.HELP
]:
# Here we look for ipython magics spanning across multiple lines.
while INPUT_SPLITTER.check_complete(magic)[0] == "incomplete":
try:
magic += next(cell_source)[1]
except StopIteration:
# This scenario is a syntax error where a line magic
# ends with a backslash and does not have a next line.
break
return magic
def _replace_magics(source: Sequence[str],
magic_substitutions: List[NewMagicHandler],
command: str) -> Iterator[str]:
"""
Replace IPython line magics with valid python code.
Parameters
----------
source
Source from notebook cell.
magic_substitutions
List to store all the ipython magics substitutions
Yields
------
str
Line from cell, with line magics replaced with python code
"""
def _process_source(source: str) -> str:
"""Temporarily replace ipython magics - don't process if can't."""
try:
ast.parse(source)
except SyntaxError:
pass
else:
# Source has no IPython magic, return it directly
return source
body = TransformerManager().transform_cell(source)
try:
tree = ast.parse(body)
except SyntaxError:
return source
visitor = Visitor()
visitor.visit(tree)
new_src = []
for i, line in enumerate(body.splitlines(), start=1):
if i in visitor.magics:
col_offset, src, magic_type = visitor.magics[i][0]
if (src is None or line[:col_offset].strip()
or len(visitor.magics[i]) > 1):
# unusual case - skip cell completely for now
handler = NewMagicHandler(source,
command,
magic_type=magic_type)
magic_substitutions.append(handler)
return handler.replacement
handler = NewMagicHandler(
src,
command,
magic_type=magic_type,
)
magic_substitutions.append(handler)
line = line[:col_offset] + handler.replacement
new_src.append(line)
leading_newlines = len(source) - len(source.lstrip("\n"))
return "\n" * leading_newlines + "\n".join(new_src)
# if first line is cell magic, process it separately
if MagicHandler.get_ipython_magic_type(source[0]) == IPythonMagicType.CELL:
header = _process_source(source[0])
cell = _process_source("".join(source[1:]))
yield "\n".join([header, cell])
else:
yield _process_source("".join(source))
def _replace_magics(
source: Sequence[str],
magic_substitutions: List[MagicHandler],
command: str,
*,
skip_bad_cells: bool,
) -> str:
"""
Replace IPython line magics with valid python code.
Parameters
----------
source
Source from notebook cell.
magic_substitutions
List to store all the ipython magics substitutions
Returns
-------
str
Line from cell, with line magics replaced with python code
"""
try:
ast.parse("".join(source))
except SyntaxError:
pass
else:
# Source has no IPython magic, return it directly
return "".join(source)
cell_magic_finder = CellMagicFinder()
body = TransformerManager().transform_cell("".join(source))
try:
tree = ast.parse(body)
except SyntaxError:
if skip_bad_cells:
handler = MagicHandler("".join(source), command, magic_type=None)
magic_substitutions.append(handler)
return handler.replacement
return "".join(source)
cell_magic_finder.visit(tree)
# if first line is cell magic, process it separately
if cell_magic_finder.header is not None:
assert cell_magic_finder.body is not None
header = _process_source(
cell_magic_finder.header,
command,
magic_substitutions,
skip_bad_cells=skip_bad_cells,
)
cell = _process_source(
cell_magic_finder.body,
command,
magic_substitutions,
skip_bad_cells=skip_bad_cells,
)
return "\n".join([header, cell])
return _process_source(
"".join(source), command, magic_substitutions, skip_bad_cells=skip_bad_cells
)