def detect_in_dependencies(source_code: str, pipeline_parameters: dict = None):
"""Detect missing names from one pipeline step source code.
Args:
source_code: Multiline Python source code
pipeline_parameters: Pipeline parameters dict
"""
commented_source_code = utils.comment_magic_commands(source_code)
ins = pyflakes_report(code=commented_source_code)
# Pipeline parameters will be part of the names that are missing,
# but of course we don't want to marshal them in as they will be
# present as parameters
relevant_parameters = set()
if pipeline_parameters:
# Not all pipeline parameters are needed in every pipeline step,
# these are the parameters that are actually needed by this step.
relevant_parameters = ins.intersection(pipeline_parameters.keys())
ins.difference_update(relevant_parameters)
step_params = {k: pipeline_parameters[k] for k in relevant_parameters}
return ins, step_params
def test_comment_magic_commands():
"""Test the magic common properly comments a multiline code block."""
code = '''
%%a magic cell command
some code
%matplotlib inline
%consecutive command
some other code
some other code
%another command
some other code
'''
target = '''
#%%a magic cell command
some code
#%matplotlib inline
#%consecutive command
some other code
some other code
#%another command
some other code
'''
assert utils.comment_magic_commands(code) == target.strip()
def get_marshal_candidates(code):
"""Get all the names that could be selected as objects to be marshalled.
This function is used by a descendant node onto its ancestors to resolve
its missing dependencies. Example:
+---+ +---+ +---+
| A | --> | B | --> | C |
+---+ +---+ +---+
When C runs the PyFlakes report on its source code, `x` is detected as a
missing variable. Then C runs `get_marshal_candidates` on its ancestors,
in order, starting from B. All the marshal candidates found in B that match
any of the missing C's names, as set as B's `outs`.
Ast nodes that become candidates:
- ast.Name
- ast.Import
- ast.ImportFrom
- ast.Tuple
Ast nodes that create local contexts must be excluded from the search, as
they can crete local variables that alias global ones. These nodes include
ast.FunctionDef, ast.ClassDef, context manager names and list and dict
comprehensions variables.
Args:
code (str): multiple string representing Python code
Returns (list(str)): a list of names
"""
names = set()
# Comment IPython magic commands.
# Note #1: This is needed to correctly parse the code using AST, as it does
# not understand IPython magic commands.
# Note #2: This will comment out both in-line magics and cell magics. This
# can lead to potential errors in case a cell magic like `%%capture out`
# is used. In that case, Kale would detect as missing the `out` variable
# declared by the magic command and will try to marshal it in at the
# beginning of the pipeline step. These cases should be very rare, and
# will be handled case by case as specific issues arise.
# Note #3: Magic commands are preserved in the resulting Python executable,
# they are commented just here in order to make AST run.
commented_code = utils.comment_magic_commands(code)
# need to exclude all the nodes that mights *define* variables in a local
# scope. For example, a function may define a variable x that is aliasing
# a global variable x, and we don't want to marshal it in that step, but
# from the step that defines the global x.
# TODO: Search for all possible python nodes that define local vars.
# List comprehensions ([i for i in list])
# Dict comprehensions
# Exception handling?
# Decorators?
# Context manager (just the alias)
contexts = (
ast.FunctionDef,
ast.ClassDef,
)
tree = ast.parse(commented_code)
for block in tree.body:
for node in walk(block, stop_at=contexts):
if isinstance(node, contexts):
names.add(node.name)
if isinstance(node, (ast.Name, )):
names.add(node.id)
if isinstance(node, (
ast.Import,
ast.ImportFrom,
)):
for _n in node.names:
if _n.asname is None:
names.add(_n.name)
else:
names.add(_n.asname)
if isinstance(node, (ast.Tuple, ast.List)):
names.update(get_list_tuple_names(node))
return names