def leave_AnnAssign(
self, original_node: cst.AnnAssign, updated_node: cst.AnnAssign
) -> Union[cst.BaseSmallStatement, cst.RemovalSentinel]:
# It handles a special case where a type-annotated variable has not initialized, e.g. foo: str
# This case will be converted to foo = ... so that nodes traversal won't encounter exceptions later on
if match.matches(
original_node,
match.AnnAssign(
target=match.Name(value=match.DoNotCare()),
annotation=match.Annotation(annotation=match.DoNotCare()),
value=None)):
updated_node = cst.Assign(
targets=[cst.AssignTarget(target=original_node.target)],
value=cst.Ellipsis())
# Handles type-annotated class attributes that has not been initialized, e.g. self.foo: str
elif match.matches(
original_node,
match.AnnAssign(
target=match.Attribute(value=match.DoNotCare()),
annotation=match.Annotation(annotation=match.DoNotCare()),
value=None)):
updated_node = cst.Assign(
targets=[cst.AssignTarget(target=original_node.target)],
value=cst.Ellipsis())
else:
updated_node = cst.Assign(
targets=[cst.AssignTarget(target=original_node.target)],
value=original_node.value)
return updated_node
def __extract_names_multi_assign(self, elements):
# Add self vars. in tuple assignments, e.g. self.x, self.y = 1, 2
# Adds variables in tuple(s) in multiple assignments, e.g. a, (b, c) = 1, (2, 3)
names: List[cst.Name] = []
i = 0
while i < len(elements):
if match.matches(
elements[i],
match.Element(value=match.Name(value=match.DoNotCare()))):
names.append(elements[i].value)
elif match.matches(
elements[i],
match.Element(value=match.Attribute(attr=match.Name(
value=match.DoNotCare())))):
names.append(elements[i].value)
elif match.matches(
elements[i],
match.Element(value=match.Tuple(
elements=match.DoNotCare()))):
elements.extend(
match.findall(
elements[i].value,
match.Element(value=match.OneOf(
match.Attribute(attr=match.Name(
value=match.DoNotCare())),
match.Name(value=match.DoNotCare())))))
i += 1
return names
def test_extract_sequence_element(self) -> None:
# Verify true behavior
expression = cst.parse_expression("a + b[c], d(e, f * g, h.i.j)")
nodes = m.extract(
expression,
m.Tuple(elements=[
m.DoNotCare(),
m.Element(
m.Call(args=[m.SaveMatchedNode(m.ZeroOrMore(), "args")])),
]),
)
extracted_seq = cst.ensure_type(
cst.ensure_type(expression, cst.Tuple).elements[1].value,
cst.Call).args
self.assertEqual(nodes, {"args": extracted_seq})
# Verify false behavior
nodes = m.extract(
expression,
m.Tuple(elements=[
m.DoNotCare(),
m.Element(
m.Call(args=[
m.SaveMatchedNode(m.ZeroOrMore(m.Arg(m.Subscript())),
"args")
])),
]),
)
self.assertIsNone(nodes)
def __name2annotation(self, type_name: str):
ext_annot = lambda t: match.extract(
cst.parse_module("x: %s=None" % t).body[0].body[0],
match.AnnAssign(target=match.Name(value=match.DoNotCare()),
annotation=match.SaveMatchedNode(
match.DoNotCare(), "type")))['type']
try:
return ext_annot(type_name)
except cst._exceptions.ParserSyntaxError:
return None
def __get_var_names_counter(self, node, scope):
vars_name = match.extractall(
node,
match.OneOf(
match.AssignTarget(target=match.SaveMatchedNode(
match.Name(value=match.DoNotCare()), "name")),
match.AnnAssign(target=match.SaveMatchedNode(
match.Name(value=match.DoNotCare()), "name"))))
return Counter([
n['name'].value for n in vars_name if isinstance(
self.get_metadata(cst.metadata.ScopeProvider, n['name']),
scope)
])
def leave_SimpleStatementLine(self, original_node: cst.SimpleStatementLine,
updated_node: cst.SimpleStatementLine):
if match.matches(
original_node,
match.SimpleStatementLine(body=[
match.Assign(targets=[
match.AssignTarget(target=match.Name(
value=match.DoNotCare()))
])
])):
t = self.__get_var_type_assign_t(
original_node.body[0].targets[0].target.value)
if t is not None:
t_annot_node_resolved = self.resolve_type_alias(t)
t_annot_node = self.__name2annotation(t_annot_node_resolved)
if t_annot_node is not None:
self.all_applied_types.add(
(t_annot_node_resolved, t_annot_node))
return updated_node.with_changes(body=[
cst.AnnAssign(
target=original_node.body[0].targets[0].target,
value=original_node.body[0].value,
annotation=t_annot_node,
equal=cst.AssignEqual(
whitespace_after=original_node.body[0].
targets[0].whitespace_after_equal,
whitespace_before=original_node.body[0].
targets[0].whitespace_before_equal))
])
elif match.matches(
original_node,
match.SimpleStatementLine(body=[
match.AnnAssign(target=match.Name(value=match.DoNotCare()))
])):
t = self.__get_var_type_an_assign(
original_node.body[0].target.value)
if t is not None:
t_annot_node_resolved = self.resolve_type_alias(t)
t_annot_node = self.__name2annotation(t_annot_node_resolved)
if t_annot_node is not None:
self.all_applied_types.add(
(t_annot_node_resolved, t_annot_node))
return updated_node.with_changes(body=[
cst.AnnAssign(target=original_node.body[0].target,
value=original_node.body[0].value,
annotation=t_annot_node,
equal=original_node.body[0].equal)
])
return original_node
def find_required_modules(all_types):
req_mod = set()
for _, a_node in all_types:
m = match.findall(
a_node.annotation,
match.Attribute(value=match.DoNotCare(),
attr=match.DoNotCare()))
if len(m) != 0:
for i in m:
req_mod.add([
n.value for n in match.findall(
i, match.Name(value=match.DoNotCare()))
][0])
return req_mod
def visit_Call(self, node: cst.Call) -> None:
if m.matches(
node,
m.Call(
func=m.Name("list") | m.Name("set") | m.Name("dict"),
args=[m.Arg(value=m.GeneratorExp() | m.ListComp())],
),
):
call_name = cst.ensure_type(node.func, cst.Name).value
if m.matches(node.args[0].value, m.GeneratorExp()):
exp = cst.ensure_type(node.args[0].value, cst.GeneratorExp)
message_formatter = UNNECESSARY_GENERATOR
else:
exp = cst.ensure_type(node.args[0].value, cst.ListComp)
message_formatter = UNNECESSARY_LIST_COMPREHENSION
replacement = None
if call_name == "list":
replacement = node.deep_replace(
node, cst.ListComp(elt=exp.elt, for_in=exp.for_in))
elif call_name == "set":
replacement = node.deep_replace(
node, cst.SetComp(elt=exp.elt, for_in=exp.for_in))
elif call_name == "dict":
elt = exp.elt
key = None
value = None
if m.matches(elt, m.Tuple(m.DoNotCare(), m.DoNotCare())):
elt = cst.ensure_type(elt, cst.Tuple)
key = elt.elements[0].value
value = elt.elements[1].value
elif m.matches(elt, m.List(m.DoNotCare(), m.DoNotCare())):
elt = cst.ensure_type(elt, cst.List)
key = elt.elements[0].value
value = elt.elements[1].value
else:
# Unrecoginized form
return
replacement = node.deep_replace(
node,
# pyre-fixme[6]: Expected `BaseAssignTargetExpression` for 1st
# param but got `BaseExpression`.
cst.DictComp(key=key, value=value, for_in=exp.for_in),
)
self.report(node,
message_formatter.format(func=call_name),
replacement=replacement)
def __extract_variable_name_type(self, node: cst.AnnAssign):
"""
Extracts a variable's identifier name and its type annotation
"""
return match.extract(
node,
match.AnnAssign( # Annotated Assignment
target=match.OneOf(
match.Name( # Variable name of assignment (only one)
value=match.SaveMatchedNode( # Save result
match.MatchRegex(
r'(.)+'), # Match any string literal
"name")),
# This extracts variables inside __init__ which typically starts with self (e.g. self.x:int=2)
match.Attribute(
value=match.Name(value=match.SaveMatchedNode(
match.MatchRegex(r'(.)+'),
"obj_name" # Object name
)),
attr=match.Name(
match.SaveMatchedNode(match.MatchRegex(r'(.)+'),
"name")),
)),
annotation=match.SaveMatchedNode( # Save result
match.DoNotCare(), # Match any string literal
"type")))
def handle_any_string(
self, node: Union[cst.SimpleString,
cst.ConcatenatedString]) -> None:
value = node.evaluated_value
if value is None:
return
mod = cst.parse_module(value)
extracted_nodes = m.extractall(
mod,
m.Name(
value=m.SaveMatchedNode(m.DoNotCare(), "name"),
metadata=m.MatchMetadataIfTrue(
cst.metadata.ParentNodeProvider,
lambda parent: not isinstance(parent, cst.Attribute),
),
)
| m.SaveMatchedNode(m.Attribute(), "attribute"),
metadata_resolver=MetadataWrapper(mod, unsafe_skip_copy=True),
)
names = {
cast(str, values["name"])
for values in extracted_nodes if "name" in values
} | {
name
for values in extracted_nodes if "attribute" in values
for name, _ in cst.metadata.scope_provider._gen_dotted_names(
cast(cst.Attribute, values["attribute"]))
}
self.names.update(names)
def on_visit(self, node):
name = type(node).__name__
if name == 'Name' and node.value == '__':
return m.DoNotCare()
kwargs = {}
for field in dataclasses.fields(node):
if field.name in ['semicolon', 'comma', 'equal']:
continue
if 'whitespace' in field.name:
continue
value = getattr(node, field.name)
if value is None:
continue
if isinstance(value, str):
pass
elif isinstance(value, Iterable):
value = [self.on_visit(e) for e in value]
else:
value = self.on_visit(value)
kwargs[field.name] = value
return getattr(m, name)(**kwargs)
def collect_targets(
self, stack: Tuple[cst.BaseExpression, ...]
) -> Tuple[
List[cst.BaseExpression], Dict[cst.BaseExpression, List[cst.BaseExpression]]
]:
targets = {}
operands = []
for operand in stack:
if m.matches(
operand, m.Call(func=m.DoNotCare(), args=[m.Arg(), m.Arg(~m.Tuple())])
):
call = cst.ensure_type(operand, cst.Call)
if not QualifiedNameProvider.has_name(self, call, _ISINSTANCE):
operands.append(operand)
continue
target, match = call.args[0].value, call.args[1].value
for possible_target in targets:
if target.deep_equals(possible_target):
targets[possible_target].append(match)
break
else:
operands.append(target)
targets[target] = [match]
else:
operands.append(operand)
return operands, targets
def __process_extracted_assign_names(self, extracted_names,
name_node: cst.Name):
"""
Auxiliary function to process the output of a matcher extraction
for extracting assign targets.
The given extracted names variable is expected to be a dictionary
with either 'name' holding a single string corresponding to
the variable name, or 'names' which is expected to be a list of
match.Element entries, corresponding to tuple assignment.
The function processes the extracted names for both cases, and
adds these definitions to the function.
"""
if "name" in extracted_names:
# Single name extracted
extracted_name = extracted_names["name"]
# Add the variable to function
self.__add_variable_to_function(extracted_name,
extracted_names['type'][0],
name_node)
self.module_all_annotations[(
self.cls_stack[-1].name if len(self.cls_stack) > 0 else None,
self.stack[-1].name, extracted_name)] = extracted_names['type']
elif "names" in extracted_names:
# Iterate through all target names
for name in extracted_names["names"]:
name_type = self.__get_type_from_metadata(name)
if match.matches(name, match.Name(value=match.DoNotCare())):
self.__add_variable_to_function(name.value, name_type,
name)
name = name.value
elif match.matches(
name,
match.Attribute(attr=match.Name(
value=match.DoNotCare()))):
self.__add_variable_to_function(name.attr.value, name_type,
name.attr)
name = name.attr.value
self.module_all_annotations[(self.cls_stack[-1].name if len(self.cls_stack) > 0 else None,
self.stack[-1].name, name)] = \
(name_type, INF_TYPE_ANNOT if name_type else UNK_TYPE_ANNOT)
def test_extract_precedence_sequence_wildcard(self) -> None:
expression = cst.parse_expression("a + b[c], d(e, f * g)")
nodes = m.extract(
expression,
m.Tuple(elements=[
m.DoNotCare(),
m.Element(
m.Call(args=[
m.ZeroOrMore(
m.Arg(m.SaveMatchedNode(m.DoNotCare(), "arg")))
])),
]),
)
extracted_node = (cst.ensure_type(
cst.ensure_type(expression, cst.Tuple).elements[1].value,
cst.Call).args[1].value)
self.assertEqual(nodes, {"arg": extracted_node})
def extract_names_from_type_annot(type_annot: str):
"""
Extracts all the names/identifiers from a type annotation
"""
return [
n.value for n in match.findall(cst.parse_expression(type_annot),
match.Name(value=match.DoNotCare()))
]
def visit_With(self, node: cst.With):
with_names = [
n.value for n in match.findall(
match.extract(
node,
match.With(items=match.SaveMatchedNode(
match.DoNotCare(), 'with_items')))['with_items'][0],
match.Name(
value=match.SaveMatchedNode(match.DoNotCare(), 'name')))
]
if len(self.stack) > 0:
self.fn_may_args_var_use.append(with_names)
if len(self.cls_stack) > 0:
if self.cls_stack[0].name in with_names:
self.cls_may_vars_use.append(with_names)
self.__find_module_vars_use(with_names)
class FlexGridSizerCommand(VisitorBasedCodemodCommand):
DESCRIPTION: str = "Updates wx.FlexGridSize constructor's calls"
matcher = matchers.Call(
func=matchers.Attribute(value=matchers.Name(value="wx"),
attr=matchers.Name(value="FlexGridSizer")),
args=[matchers.DoNotCare(), matchers.DoNotCare()],
)
def leave_Call(self, original_node: cst.Call,
updated_node: cst.Call) -> cst.Call:
if matchers.matches(updated_node, self.matcher):
return updated_node.with_changes(args=[
*updated_node.args,
cst.Arg(value=cst.Integer(value="0"))
])
return updated_node
def test_extract_precedence_parent(self) -> None:
expression = cst.parse_expression("a + b[c], d(e, f * g)")
nodes = m.extract(
expression,
m.Tuple(elements=[
m.DoNotCare(),
m.Element(
m.SaveMatchedNode(
m.Call(args=[
m.Arg(m.SaveMatchedNode(m.Name(), "name")),
m.DoNotCare(),
]),
"name",
)),
]),
)
extracted_node = cst.ensure_type(expression,
cst.Tuple).elements[1].value
self.assertEqual(nodes, {"name": extracted_node})
def test_extract_optional_wildcard_present(self) -> None:
expression = cst.parse_expression("a + b[c], d(e, f * g, h.i.j)")
nodes = m.extract(
expression,
m.Tuple(elements=[
m.DoNotCare(),
m.Element(
m.Call(args=[
m.DoNotCare(),
m.DoNotCare(),
m.ZeroOrOne(
m.Arg(m.SaveMatchedNode(m.Attribute(), "arg"))),
])),
]),
)
extracted_node = (cst.ensure_type(
cst.ensure_type(expression, cst.Tuple).elements[1].value,
cst.Call).args[2].value)
self.assertEqual(nodes, {"arg": extracted_node})
def __get_fn_params(self, fn_params: cst.Parameters):
p_names: List[str] = []
kwarg = [fn_params.star_kwarg
] if fn_params.star_kwarg is not None else []
stararg = [fn_params.star_arg] if match.matches(
fn_params.star_arg,
match.Param(name=match.Name(value=match.DoNotCare()))) else []
for p in list(fn_params.params) + list(fn_params.kwonly_params) + list(
fn_params.posonly_params) + stararg + kwarg:
p_names.append(self.nlp_p(p.name.value))
return p_names
def visit_AssignTarget(self, node: cst.AssignTarget):
if match.matches(
node,
match.AssignTarget(target=match.Name(
value=match.DoNotCare()))):
if self.last_visited_assign_t_name == node.target.value:
self.last_visited_assign_t_count += 1
elif self.last_visited_assign_t_count == 0:
self.last_visited_assign_t_count = 1
else:
self.last_visited_assign_t_count = 1
self.last_visited_assign_t_name = node.target.value
def __name2annotation(self, type_name: str):
"""
Converts Name nodes to valid annotation nodes
"""
try:
return match.extract(
cst.parse_module("x: %s=None" % type_name).body[0].body[0],
match.AnnAssign(target=match.Name(value=match.DoNotCare()),
annotation=match.SaveMatchedNode(
match.DoNotCare(), "type")))['type']
except cst._exceptions.ParserSyntaxError:
# To handle a bug in LibCST's scope provider where a local name shadows a type annotation with the same name
if (self.last_visited_name.value,
cst.metadata.QualifiedNameSource.IMPORT
) in self.q_names_cache:
return match.extract(
cst.parse_module(
"x: %s=None" %
self.q_names_cache[(self.last_visited_name.value,
cst.metadata.QualifiedNameSource.
IMPORT)]).body[0].body[0],
match.AnnAssign(target=match.Name(value=match.DoNotCare()),
annotation=match.SaveMatchedNode(
match.DoNotCare(), "type")))['type']
else:
return match.extract(
cst.parse_module(
"x: %s=None" %
self.last_visited_name.value).body[0].body[0],
match.AnnAssign(target=match.Name(value=match.DoNotCare()),
annotation=match.SaveMatchedNode(
match.DoNotCare(), "type")))['type']
def test_extract_sequence(self) -> None:
expression = cst.parse_expression("a + b[c], d(e, f * g, h.i.j)")
nodes = m.extract(
expression,
m.Tuple(elements=[
m.DoNotCare(),
m.Element(
m.Call(args=m.SaveMatchedNode([m.ZeroOrMore()], "args"))),
]),
)
extracted_seq = cst.ensure_type(
cst.ensure_type(expression, cst.Tuple).elements[1].value,
cst.Call).args
self.assertEqual(nodes, {"args": extracted_seq})
def test_extract_optional_wildcard(self) -> None:
expression = cst.parse_expression("a + b[c], d(e, f * g)")
nodes = m.extract(
expression,
m.Tuple(elements=[
m.DoNotCare(),
m.Element(
m.Call(args=[
m.ZeroOrMore(),
m.ZeroOrOne(
m.Arg(m.SaveMatchedNode(m.Attribute(), "arg"))),
])),
]),
)
self.assertEqual(nodes, {})
def visit_If(self, node: cst.If):
if_names = [
n.value for n in match.findall(
node.test,
match.Name(
value=match.SaveMatchedNode(match.DoNotCare(), 'name')))
]
if len(self.cls_stack) > 0:
if self.cls_stack[0].name in if_names:
self.cls_may_vars_use.append(if_names)
if len(self.stack) > 0:
self.fn_may_args_var_use.append(if_names)
self.__find_module_vars_use(if_names)
def visit_SimpleStatementLine(self, node: cst.SimpleStatementLine):
smt_names = [
n.value for n in match.findall(
node,
match.Name(
value=match.SaveMatchedNode(match.DoNotCare(), 'name')))
]
if len(self.stack) > 0:
self.fn_may_args_var_use.append(smt_names)
if len(self.cls_stack) > 0:
if self.cls_stack[0].name in smt_names:
self.cls_may_vars_use.append(smt_names)
self.__find_module_vars_use(smt_names)
def visit_Call(self, node: cst.Call) -> None:
# print(node)
d = m.extract(
node,
m.Call(
func=m.OneOf(m.Name("Extension"), m.Name("addMacExtension")),
args=(
m.Arg(value=m.SaveMatchedNode(m.SimpleString(),
"extension_name")),
m.ZeroOrMore(m.DoNotCare()),
),
),
)
if d:
assert isinstance(d["extension_name"], cst.SimpleString)
self.extension_names.append(d["extension_name"].evaluated_value)
def leave_SubscriptElement(self, original_node: cst.SubscriptElement,
updated_node: cst.SubscriptElement):
if self.type_annot_visited and self.parametric_type_annot_visited:
if match.matches(
original_node,
match.SubscriptElement(slice=match.Index(
value=match.Subscript()))):
q_name, _ = self.__get_qualified_name(
original_node.slice.value.value)
if q_name is not None:
return updated_node.with_changes(slice=cst.Index(
value=cst.Subscript(
value=self.__name2annotation(q_name).annotation,
slice=updated_node.slice.value.slice)))
elif match.matches(
original_node,
match.SubscriptElement(slice=match.Index(
value=match.Ellipsis()))):
# TODO: Should the original node be returned?!
return updated_node.with_changes(slice=cst.Index(
value=cst.Ellipsis()))
elif match.matches(
original_node,
match.SubscriptElement(slice=match.Index(
value=match.SimpleString(value=match.DoNotCare())))):
return updated_node.with_changes(slice=cst.Index(
value=updated_node.slice.value))
elif match.matches(
original_node,
match.SubscriptElement(slice=match.Index(value=match.Name(
value='None')))):
return original_node
elif match.matches(
original_node,
match.SubscriptElement(slice=match.Index(
value=match.List()))):
return updated_node.with_changes(slice=cst.Index(
value=updated_node.slice.value))
else:
q_name, _ = self.__get_qualified_name(
original_node.slice.value)
if q_name is not None:
return updated_node.with_changes(slice=cst.Index(
value=self.__name2annotation(q_name).annotation))
return original_node
class MenuAppendCommand(VisitorBasedCodemodCommand):
DESCRIPTION: str = "Migrate to wx.MenuAppend() method and update keywords"
args_map = {"help": "helpString", "text": "item"}
args_matchers_map = {
matchers.Arg(keyword=matchers.Name(value=value)): renamed
for value, renamed in args_map.items()
}
call_matcher = matchers.Call(
func=matchers.Attribute(attr=matchers.Name(value="Append")),
args=matchers.MatchIfTrue(lambda args: bool(
set(arg.keyword.value for arg in args if arg and arg.keyword).
intersection(MenuAppendCommand.args_map.keys()))),
)
deprecated_call_matcher = matchers.Call(
func=matchers.Attribute(attr=matchers.Name(value="AppendItem")),
args=[matchers.DoNotCare()],
)
def leave_Call(self, original_node: cst.Call,
updated_node: cst.Call) -> cst.Call:
# Migrate form deprecated method AppendItem()
if matchers.matches(updated_node, self.deprecated_call_matcher):
updated_node = updated_node.with_changes(
func=updated_node.func.with_changes(attr=cst.Name(
value="Append")))
# Update keywords
if matchers.matches(updated_node, self.call_matcher):
updated_node_args = list(updated_node.args)
for arg_matcher, renamed in self.args_matchers_map.items():
for i, node_arg in enumerate(updated_node.args):
if matchers.matches(node_arg, arg_matcher):
updated_node_args[i] = node_arg.with_changes(
keyword=cst.Name(value=renamed))
updated_node = updated_node.with_changes(
args=updated_node_args)
return updated_node
def visit_ImportAlias(self, node: cst.ImportAlias):
"""
Extracts imports.
Even if an Import has no alias, the Import node will still have an ImportAlias
with it's real name.
"""
# Extract information from the Import Alias node.
# Both the (real) import name and the alias are extracted.
# Result is returned as a dictionary with a name:node KV pair,
# and an alias:name KV pair if an alias is present.
import_info = match.extract(
node,
match.ImportAlias(
asname=match.AsName( # Attempt to match alias
name=match.Name( # Check for alias name
value=match.SaveMatchedNode( # Save the alias name
match.MatchRegex(
r'(.)+'), # Match any string literal
"alias"))) | ~match.AsName(),
# If there is no AsName, we should still match. We negate AsName to and OR to form a tautology.
name=match.SaveMatchedNode( # Match & save name of import
match.DoNotCare(), "name")))
# Add import if a name could be extracted.
if "name" in import_info:
# Append import name to imports list.
# We convert the ImportAlias node to the code representation for simplified conversion
# of multi-level imports (e.g. import.x.y.z)
# TODO: This might be un-needed after implementing import type extraction change.
# TODO: So far, no differentiation between import and from imports.
import_name = self.__convert_node_to_code(import_info["name"])
import_name = self.__clean_string_whitespace(import_name)
self.imports.append(import_name)
if "alias" in import_info:
import_name = self.__clean_string_whitespace(import_info["alias"])
self.imports.append(import_name)
# No need to traverse children, as we already performed the matching.
return False