def get_code_bits(code):
atok = ASTTokens(code, parse=1)
for node in ast.walk(atok.tree):
if isinstance(node, ast.Name):
if not atok.get_text(node):
continue
node_text = node.id
assert atok.get_text(node) == node_text
if node_text in builtins.__dict__ or len(node_text) == 1:
continue
elif isinstance(node,
(ast.FunctionDef, ast.ClassDef, ast.AsyncFunctionDef)):
for arg in node.args.args:
if len(arg.arg) == 1:
continue
yield arg, arg.arg
node_text = node.name
elif isinstance(node, (ast.Str, ast.JoinedStr)):
node_text = atok.get_text(node)
if not re.search(r"[a-zA-Z]", node_text) or re.match(
r"""^['"][a-zA-Z]['"]""", node_text):
continue
else:
continue
yield node, node_text
def mark_text_ranges(node, source: bytes):
"""
Node is an AST, source is corresponding source as string.
Function adds recursively attributes end_lineno and end_col_offset to each node
which has attributes lineno and col_offset.
"""
try:
from asttokens import ASTTokens
except ImportError:
# asttokens may be missing in some configurations
from thonny.ast_utils_old_range_marker import old_mark_text_ranges
logger = logging.getLogger("thonny")
if not getattr(logger, "old_astutils_warned", False):
logger.warn("asttokens not found, using deprecated alternative")
logger.old_astutils_warned = True
return old_mark_text_ranges(node, source)
ASTTokens(source.decode('utf8'), tree=node)
for child in ast.walk(node):
if hasattr(child, 'last_token'):
child.end_lineno, child.end_col_offset = child.last_token.end
if hasattr(child, 'lineno'):
# Fixes problems with some nodes like binop
child.lineno, child.col_offset = child.first_token.start
def clean_program(program, cls):
func = program
if isinstance(program, FunctionType):
source = dedent(inspect.getsource(program))
lines = source.splitlines()
if lines[-1].strip().startswith("return "):
func = NoMethodWrapper(program(None))
assert lines[0] == "def solution(self):"
assert lines[-1] == f" return {func.__name__}"
source = dedent("\n".join(lines[1:-1]))
program = clean_solution_function(func, source)
else:
atok = ASTTokens(source, parse=True)
func_node = atok.tree.body[0]
lines = lines[func_node.body[0].first_token.start[0] - 1:]
if hasattr(cls, "test_values"):
inputs = list(cls.test_values())[0][0]
else:
inputs = {}
inputs = inputs_string(inputs)
program = inputs + '\n' + dedent('\n'.join(lines))
compile(program, "<program>", "exec") # check validity
if not any(isinstance(node, ast.Return) for node in ast.walk(ast.parse(source))):
func = returns_stdout(func)
no_weird_whitespace(program)
return program.strip(), func
def asttokens(self) -> ASTTokens:
"""
Returns an ASTTokens object for getting the source of specific AST nodes.
See http://asttokens.readthedocs.io/en/latest/api-index.html
"""
return ASTTokens(self.source, tree=self.tree, filename=self.path)
def clean_program(program, inputs=None):
if callable(program):
inputs = inputs_string(inputs or {})
source = dedent(inspect.getsource(program))
atok = ASTTokens(source, parse=True)
func = atok.tree.body[0]
lines = source.splitlines()[func.body[0].first_token.start[0] - 1:]
program = inputs + '\n' + dedent('\n'.join(lines))
compile(program, "<program>", "exec") # check validity
no_weird_whitespace(program)
return program.strip()
def asttokens(self):
"""
Returns an ASTTokens object for getting the source of specific AST nodes.
See http://asttokens.readthedocs.io/en/latest/api-index.html
"""
from asttokens import ASTTokens # must be installed separately
return ASTTokens(
self.text,
tree=self.tree,
filename=self.filename,
)
def process(
positions: List[Position],
content: str,
filename: str,
) -> List[Insertion]:
try:
asttokens = ASTTokens(content, parse=True, filename=filename)
except SyntaxError as e:
# Catch syntax errors within the file we were asked to parse. We trust
# that the error is not within asttokens itself.
raise TargetSyntaxError(e) from e
insertions = merge_insertions(
determine_insertions(asttokens, position) for position in positions)
return insertions
def try_to_extract(file):
if not os.path.isfile(file):
return None
with open(file) as fin:
try:
atok = ASTTokens(fin.read().encode('utf-8'),
parse=True,
filename=file)
except TokenError:
try:
atok = ASTTokens(fin.read().encode("cp1252"),
parse=True,
filename=file)
except TokenError:
print("Unable to extract from {}".format(file))
return None
except SyntaxError:
try:
atok = ASTTokens(fin.read().encode("cp1252"),
parse=True,
filename=file)
except SyntaxError:
print("Unable to extract from {}".format(file))
return None
except ValueError:
try:
atok = ASTTokens(fin.read().encode("cp1252"),
parse=True,
filename=file)
except ValueError:
print("Unable to extract from {}".format(file))
return None
except IndexError:
try:
atok = ASTTokens(fin.read().encode("cp1252"),
parse=True,
filename=file)
except IndexError:
print("Unable to extract from {}".format(file))
return None
except TypeError:
try:
atok = ASTTokens(fin.read().encode("cp1252"),
parse=True,
filename=file)
except TypeError:
print("Unable to extract from {}".format(file))
return None
return atok.tree
def create_asttokens(source):
builder = astroid.builder.AstroidBuilder()
tree = builder.string_build(source)
return ASTTokens(source, tree=tree)
def compile(self, source, filename, flags=0):
traced_file = super(BirdsEye, self).compile(source, filename, flags)
traced_file.tokens = ASTTokens(source, tree=traced_file.root)
return traced_file
def get_tokenised(filepath):
with open(filepath, "r") as f:
source = f.read()
tokenised = ASTTokens(source, parse=True)
return tokenised
def create_asttokens(source):
return ASTTokens(source, parse=True)
def get_all_objects(line, frame):
"""Given a (partial) line of code and a frame,
obtains a dict containing all the relevant information about objects
found on that line so that they can be formatted as part of the
answer to "where()" or they can be used during the analysis
of the cause of the exception.
The dict returned has five keys.
The first three, 'locals', 'globals', 'builtins',
each containing a list of tuples, each tuple being of the form
(name, repr(obj), obj) where name --> obj.
The fourth key, 'expressions', contains a list of tuples of the form
('name', obj). It is only occasionally used in helping to make
suggestions regarding the cause of some exception.
"""
objects = {
"locals": [],
"globals": [],
"builtins": [],
"expressions": [],
"name, obj": [],
}
scopes = (
("locals", frame.f_locals), # always have locals before globals
("globals", frame.f_globals),
)
names = set()
tokens = token_utils.get_significant_tokens(line)
for tok in tokens:
if tok.is_identifier():
name = tok.string
if name in names:
continue
for scope, scope_dict in scopes:
if name in scope_dict:
names.add(name)
obj = scope_dict[name]
objects[scope].append((name, repr(obj), obj))
objects["name, obj"].append((name, obj))
break
else:
if name in dir(builtins):
names.add(name)
obj = getattr(builtins, name)
objects["builtins"].append((name, repr(obj), obj))
objects["name, obj"].append((name, obj))
try:
atok = ASTTokens(line, parse=True)
except SyntaxError: # this should not happen
return objects
if atok is not None:
evaluator = Evaluator.from_frame(frame)
for nodes, obj in group_expressions(
pair for pair in evaluator.find_expressions(atok.tree)):
name = atok.get_text(nodes[0])
if name in names:
continue
names.add(name)
objects["name, obj"].append((name, obj))
try:
# We're not interested in showing literals in the list of variables
ast.literal_eval(name)
except Exception: # noqa
objects["expressions"].append((name, obj))
return objects
def get_all_objects(line, frame):
"""Given a (partial) line of code and a frame,
obtains a dict containing all the relevant information about objects
found on that line so that they can be formatted as part of the
answer to "where()" or they can be used during the analysis
of the cause of the exception.
The dict returned has four keys.
The first three, 'locals', 'globals', 'nonlocals',
each containing a list of tuples, each tuple being of the form
(name, repr(obj), obj) where name --> obj.
The fourth key, 'literals', contains a list of tuples of the form
('name', obj). It is only occasionally used in helping to make
suggestions regarding the cause of some exception.
"""
objects = {
"locals": [],
"globals": [],
"literals": [],
"builtins": [],
"name, obj": [],
}
scopes = (
("locals", frame.f_locals), # always have locals before globals
("globals", frame.f_globals),
)
names = set([])
try:
atok = ASTTokens(line, parse=True)
except SyntaxError: # this should not happen
atok = None
if atok is not None:
for scope, scope_dict in scopes:
for nodes, obj in Evaluator(
scope_dict).interesting_expressions_grouped(atok.tree):
name = atok.get_text(nodes[0])
if name in names:
continue
names.add(name)
objects[scope].append((name, repr(obj), obj))
objects["name, obj"].append((name, obj))
Evaluator.literal_expressions_grouped = literal_expressions_grouped
for nodes, obj in Evaluator({}).literal_expressions_grouped(
atok.tree): # noqa
name = atok.get_text(nodes[0])
objects["literals"].append((name, obj))
objects["name, obj"].append((name, obj))
tokens = token_utils.get_significant_tokens(line)
for tok in tokens:
if tok.is_identifier():
name = tok.string
if name in names:
continue
for scope, scope_dict in scopes:
if name in scope_dict:
names.add(name)
obj = scope_dict[name]
objects[scope].append((name, repr(obj), obj))
objects["name, obj"].append((name, obj))
break
else:
if name in dir(builtins):
obj = getattr(builtins, name)
objects["builtins"].append((name, repr(obj), obj))
objects["name, obj"].append((name, obj))
dotted_names = get_dotted_names(line)
for name in dotted_names:
for scope, scope_dict in scopes:
if name not in scope_dict:
continue
obj = scope_dict[name]
if (name, obj) not in objects["name, obj"]:
objects[scope].append((name, repr(obj), obj))
objects["name, obj"].append((name, obj))
# TODO: check to see if this is still needed
objects["nonlocals"] = get_nonlocal_objects(frame)
return objects
def multilinify_import_stmt_str(import_stmt_str,
indent_spaces=4,
trailing_comma=True):
"""
Takes a single line import statement and turns it into a multiline string.
Will raise a `ValueError` if given a multiline string (a newline at the end
of the string is permitted).
This function is written in expectation of the output of `get_import_stmt_str`,
and is not intended to process all potential ways of writing an import statement.
import_stmt_str: String of Python code carrying out an import statement.
indent_spaces: Number of spaces to indent by in multiline format.
trailing_comma: Whether to add a trailing comma to the final alias in a
multiline list of import aliases (default: True)
"""
import_stmt_str = import_stmt_str.rstrip(nl)
n_nl = import_stmt_str.count(nl)
if n_nl > 0:
raise ValueError(f"{import_stmt_str} is not a single line string")
imp_ast = ast.parse(import_stmt_str)
assert type(imp_ast.body[0]) in [IType,
IFType], "Not a valid import statement"
tko = ASTTokens(import_stmt_str)
first_tok = tko.tokens[0]
import_tok = tko.find_token(first_tok, tok_type=1, tok_str="import")
assert import_tok.type > 0, f"Unable to find import token in the given string"
imp_preamble_str = import_stmt_str[:import_tok.endpos]
post_import_tok = tko.tokens[import_tok.index + 1]
imp_names_str = import_stmt_str[post_import_tok.startpos:]
aliases = [(x.name, x.asname) for x in imp_ast.body[0].names]
seen_comma_tok = None
multiline_import_stmt_str = imp_preamble_str
multiline_import_stmt_str += " (" + nl
for al_i, (a_n, a_as) in enumerate(aliases):
is_final_alias = al_i + 1 == len(aliases)
if seen_comma_tok is None:
# Get start of alias by either full name or first part of .-separated name
al_n_tok = tko.find_token(import_tok, 1, tok_str=a_n.split(".")[0])
assert al_n_tok.type > 0, f"Unable to find the token for {a_n}"
else:
al_n_tok = tko.find_token(seen_comma_tok,
1,
tok_str=a_n.split(".")[0])
assert al_n_tok.type > 0, f"Unable to find the token for {a_n}"
al_startpos = al_n_tok.startpos
if a_as is None:
if is_final_alias:
# There won't be a comma after this (it is the last import name token)
al_endpos = al_n_tok.endpos
else:
comma_tok = tko.find_token(al_n_tok, tok_type=53, tok_str=",")
if comma_tok.type == 0:
# Due to an error in asttokens, sometimes tok_type is given as 54
# although this should be an error (the failure tok_type is 0)
comma_tok = tko.find_token(al_n_tok,
tok_type=54,
tok_str=",")
assert comma_tok.type > 0, f"Unable to find comma token"
al_endpos = comma_tok.endpos
else:
al_as_tok = tko.find_token(al_n_tok, tok_type=1, tok_str=a_as)
assert al_as_tok.type > 0, f"Unable to find the token for {a_as}"
if is_final_alias:
# There won't be a comma after this (it's the last import asname token)
al_endpos = al_as_tok.endpos
else:
comma_tok = tko.find_token(al_as_tok, tok_type=53, tok_str=",")
if comma_tok.type == 0:
# Due to an error in asttokens, sometimes tok_type is given as 54
# although this should be an error (the failure tok_type is 0)
comma_tok = tko.find_token(al_n_tok,
tok_type=54,
tok_str=",")
assert comma_tok.type > 0, f"Unable to find comma token"
al_endpos = comma_tok.endpos
alias_chunk = import_stmt_str[al_startpos:al_endpos]
if is_final_alias:
if trailing_comma:
alias_chunk += ","
else:
seen_comma_tok = comma_tok
multiline_import_stmt_str += (" " * indent_spaces) + alias_chunk + nl
# Finally, verify that the end of the tokenised string was reached
assert al_endpos == tko.tokens[
-1].endpos, "Did not tokenise to the end of string"
# No need to further slice the input string, return the final result
multiline_import_stmt_str += ")" + nl
return multiline_import_stmt_str
def colour_imp_stmt(imp_stmt, lines):
"""
Summary: get a string which when printed will show the separate parts of an
import statement in different colours (preamble in blue, alias names in red,
alias asnames in purple, the word "as" itself in yellow, commas between import
aliases in light green, and post-matter (a bracket) in light red.
For an import statement within an asttokens-annotated AST, which comes with
all subnodes annotated with first and last token start/end positional information,
access all the tokens corresponding to the import statement name(s) and asname(s).
Do this using a list of lines (i.e. a list of strings, each of which is a line),
the subset of which corresponding to the import statement `imp_stmt` are given
by its `first_token.start` and `last_token.end` attributes (in each case, the
attribute is a tuple of `(line, column)` numbers, and it is conventional to store
these as a 1-based index, so to cross-reference to a 0-based index of the list
of lines we decrement this value and store as `imp_startln` and `imp_endln`).
The subset of lines corresponding to `imp_stmt` is then assigned as `nodelines`,
and we join this into a single string as `nodestring`.
Then a new ASTTokens object, `tko`, can be made by parsing `nodestring`, on which
the `find_tokens` method provides access to each name/asname one at a time, when
matched to the name/asname string. These name/asname strings are available
within the `imp_stmt` object via its `names` attribute, which is a list of
`ast.alias` class instances, each of which has both a `name` and `asname` attribute
(the latter of which is `None` if no asname is given for the import name).
`find_tokens` returns a token with attribute `type` of value `1` for a name (1 is
the index of "NAME" in the `token.tok_name` dictionary), and `startpos`/`endpos`
attributes (integers which indicate the string offsets within `nodestring`).
These `startpos` integers are an efficient way to store this list of tokens
(the "NAME" tokens corresponding to import statement alias names and asnames),
and so even though it would be possible to store all tokens, I choose to simply
re-access them with the `tko.get_token_from_offset(startpos)` method.
At the moment, I only re-access these tokens to retrieve their `endpos` (end
position offset), which is also an integer and could also be stored easily
without much problem, however for the sake of clarity I prefer to re-access
the entire token and not have to construct an arbitrary data structure for
storing the start and end positions (which could get confusing).
Lastly, I construct a colourful string representation of the import statement
by using these start positions and re-retrieved end positions to pull out
and modify (using the `mvdef.colours`та╢`colour_str` function) the names and asnames
(names are coloured red, asnames are coloured purple), and use string slicing
to swap the ranges that the names and asnames were in in the original
`nodestring` for these colourful replacements.
The end result, `modified_nodestring` is then returned, which will then
display in colour on Linux and OSX (I don't think Windows supports ANSI codes,
so I made `colour_str` only apply on these platforms).
"""
assert "first_token" in imp_stmt.__dir__(
), "Not an asttokens-annotated AST node"
assert type(imp_stmt) in [IType, IFType], "Not an import statement"
is_from = type(imp_stmt) is IFType
imp_startln = imp_stmt.first_token.start[0] - 1 # Use 0-based line index
imp_endln = imp_stmt.last_token.end[0] - 1 # to match list of lines
nodelines = lines[imp_startln:(imp_endln + 1)]
n_implines = len(nodelines)
nodestring = "".join(nodelines)
tko = ASTTokens(nodestring)
new_nodelines = [list() for _ in range(n_implines)]
# Subtract the import statement start position from the name or asname
# token start position to get the offset, then use the offset to extract
# a range of text from the re-parsed ASTTokens object for the nodestring
# corresponding to the import name or asname in question.
imp_startpos = imp_stmt.first_token.startpos
alias_starts = []
for alias in imp_stmt.names:
al_n, al_as = alias.name, alias.asname
# 1 is the key for "NAME" in Python's tokens.tok_name
s = [tko.find_token(tko.tokens[0], 1, tok_str=al_n).startpos]
if al_as is not None:
s.append(tko.find_token(tko.tokens[0], 1, tok_str=al_as).startpos)
alias_starts.append(s)
assert len(alias_starts) > 0, "An import statement cannot import no names!"
assert alias_starts[0][
0] > 0, "An import statement cannot begin with a name!"
modified_nodestring = ""
# -------------------------------------------------------------------------
# Now set up colour definitions for the modified import statement string
name_colour, asname_colour = ["red", "purple"]
pre_colour, post_colour = ["light_blue", "light_red"]
as_string_colour = "yellow"
comma_colour = "light_green"
# -------------------------------------------------------------------------
first_import_name_startpos = alias_starts[0][0]
pre_str = nodestring[:first_import_name_startpos]
modified_nodestring += colour(pre_colour, pre_str)
seen_endpos = first_import_name_startpos
# (Could add a try/except here to verify colours are in colour dict if modifiable)
for al_i, alias_start_list in enumerate(alias_starts):
for al_j, al_start in enumerate(alias_start_list):
if seen_endpos < al_start:
# There is an intervening string, append it to modified_nodestring
intervening_str = nodestring[seen_endpos:al_start]
if al_j > 0:
# This is the word "as", which comes between a name and an asname
modified_nodestring += colour(as_string_colour,
intervening_str)
else:
if al_i > 0:
assert "," in intervening_str, "Import aliases not comma-sep.?"
modified_nodestring += colour(comma_colour,
intervening_str)
else:
modified_nodestring += intervening_str
# Possible here to distinguish between names and asnames by al_j if needed
is_asname = bool(al_j) # al_j is 0 if name, 1 if asname
name_tok = tko.get_token_from_offset(al_start)
assert name_tok.type > 0, f"No import name at {al_start} in {nodestring}"
al_endpos = name_tok.endpos
imp_name = nodestring[al_start:al_endpos]
cstr_colour = [name_colour, asname_colour][al_j]
cstr = colour(cstr_colour, imp_name)
modified_nodestring += cstr
seen_endpos = al_endpos
end_str = nodestring[seen_endpos:]
modified_nodestring += colour(post_colour, end_str)
return modified_nodestring
