class UnhashableListTransformer(NoqaAwareTransformer):
@m.call_if_inside(
m.Call(
func=m.OneOf(m.Name(value="set"), m.Name(value="frozenset")),
args=[m.Arg(value=m.OneOf(m.List(), m.Tuple(), m.Set()))],
)
| m.Set() # noqa: W503
)
@m.leave(m.List() | m.Set() | m.Tuple())
def convert_list_arg(
self, _, updated_node: Union[cst.Set, cst.List,
cst.Tuple]) -> cst.BaseExpression:
modified_elements = convert_lists_to_tuples(updated_node.elements)
return updated_node.with_changes(elements=modified_elements)
def visit_Annotation(self, node: cst.Annotation):
if not match.matches(
node,
match.Annotation(
annotation=match.OneOf(match.Name(
value='None'), match.List(elements=[])))):
self.type_annot_visited = True
def test_extract_optional_wildcard_tail(self) -> None:
expression = cst.parse_expression("[3]")
nodes = m.extract(
expression,
m.List(elements=[
m.Element(value=m.Integer(value="3")),
m.SaveMatchedNode(m.ZeroOrMore(), "tail1"),
m.SaveMatchedNode(m.ZeroOrMore(), "tail2"),
]),
)
self.assertEqual(nodes, {"tail1": (), "tail2": ()})
def visit_Assign(self, node: cst.Assign) -> None:
d = m.extract(
node,
m.Assign(
targets=(m.AssignTarget(target=m.Name("CARBON_EXTS")), ),
value=m.SaveMatchedNode(m.List(), "list"),
),
)
if d:
assert isinstance(d["list"], cst.List)
for item in d["list"].elements:
if isinstance(item.value, cst.SimpleString):
self.extension_names.append(item.value.evaluated_value)
def visit_Call(self, node: cst.Call) -> None:
if m.matches(
node,
m.Call(
func=m.Name("list") | m.Name("set") | m.Name("dict"),
args=[m.Arg(value=m.GeneratorExp() | m.ListComp())],
),
):
call_name = cst.ensure_type(node.func, cst.Name).value
if m.matches(node.args[0].value, m.GeneratorExp()):
exp = cst.ensure_type(node.args[0].value, cst.GeneratorExp)
message_formatter = UNNECESSARY_GENERATOR
else:
exp = cst.ensure_type(node.args[0].value, cst.ListComp)
message_formatter = UNNECESSARY_LIST_COMPREHENSION
replacement = None
if call_name == "list":
replacement = node.deep_replace(
node, cst.ListComp(elt=exp.elt, for_in=exp.for_in))
elif call_name == "set":
replacement = node.deep_replace(
node, cst.SetComp(elt=exp.elt, for_in=exp.for_in))
elif call_name == "dict":
elt = exp.elt
key = None
value = None
if m.matches(elt, m.Tuple(m.DoNotCare(), m.DoNotCare())):
elt = cst.ensure_type(elt, cst.Tuple)
key = elt.elements[0].value
value = elt.elements[1].value
elif m.matches(elt, m.List(m.DoNotCare(), m.DoNotCare())):
elt = cst.ensure_type(elt, cst.List)
key = elt.elements[0].value
value = elt.elements[1].value
else:
# Unrecoginized form
return
replacement = node.deep_replace(
node,
# pyre-fixme[6]: Expected `BaseAssignTargetExpression` for 1st
# param but got `BaseExpression`.
cst.DictComp(key=key, value=value, for_in=exp.for_in),
)
self.report(node,
message_formatter.format(func=call_name),
replacement=replacement)
def visit_AugAssign(self, node: cst.AugAssign) -> bool:
if m.matches(
node,
m.AugAssign(
target=m.Name("__all__"),
operator=m.AddAssign(),
value=m.List() | m.Tuple(),
),
):
value = node.value
if isinstance(value, (cst.List, cst.Tuple)):
self._is_assigned_export.add(value)
return True
return False
def convert_lists_to_tuples(
elements: Sequence[cst.BaseElement], ) -> List[cst.BaseElement]:
result: List[cst.BaseElement] = []
for element in elements:
if m.matches(element, m.Element(value=m.List())):
unhashable_list: cst.List = cst.ensure_type(
element.value, cst.List)
result.append(
element.with_changes(value=cst.Tuple(
elements=convert_lists_to_tuples(
unhashable_list.elements))))
else:
result.append(element)
return result
def leave_SubscriptElement(self, original_node: cst.SubscriptElement,
updated_node: cst.SubscriptElement):
if self.type_annot_visited and self.parametric_type_annot_visited:
if match.matches(
original_node,
match.SubscriptElement(slice=match.Index(
value=match.Subscript()))):
q_name, _ = self.__get_qualified_name(
original_node.slice.value.value)
if q_name is not None:
return updated_node.with_changes(slice=cst.Index(
value=cst.Subscript(
value=self.__name2annotation(q_name).annotation,
slice=updated_node.slice.value.slice)))
elif match.matches(
original_node,
match.SubscriptElement(slice=match.Index(
value=match.Ellipsis()))):
# TODO: Should the original node be returned?!
return updated_node.with_changes(slice=cst.Index(
value=cst.Ellipsis()))
elif match.matches(
original_node,
match.SubscriptElement(slice=match.Index(
value=match.SimpleString(value=match.DoNotCare())))):
return updated_node.with_changes(slice=cst.Index(
value=updated_node.slice.value))
elif match.matches(
original_node,
match.SubscriptElement(slice=match.Index(value=match.Name(
value='None')))):
return original_node
elif match.matches(
original_node,
match.SubscriptElement(slice=match.Index(
value=match.List()))):
return updated_node.with_changes(slice=cst.Index(
value=updated_node.slice.value))
else:
q_name, _ = self.__get_qualified_name(
original_node.slice.value)
if q_name is not None:
return updated_node.with_changes(slice=cst.Index(
value=self.__name2annotation(q_name).annotation))
return original_node
def visit_Call(self, node: cst.Call) -> None:
if m.matches(
node,
m.Call(
func=m.Name("tuple") | m.Name("list") | m.Name("set")
| m.Name("dict"),
args=[m.Arg(value=m.List() | m.Tuple())],
),
) or m.matches(
node,
m.Call(func=m.Name("tuple") | m.Name("list") | m.Name("dict"),
args=[]),
):
pairs_matcher = m.ZeroOrMore(
m.Element(m.Tuple(
elements=[m.DoNotCare(), m.DoNotCare()]))
| m.Element(m.List(
elements=[m.DoNotCare(), m.DoNotCare()])))
exp = cst.ensure_type(node, cst.Call)
call_name = cst.ensure_type(exp.func, cst.Name).value
# If this is a empty call, it's an Unnecessary Call where we rewrite the call
# to literal, except set().
if not exp.args:
elements = []
message_formatter = UNNCESSARY_CALL
else:
arg = exp.args[0].value
elements = cst.ensure_type(
arg, cst.List
if isinstance(arg, cst.List) else cst.Tuple).elements
message_formatter = UNNECESSARY_LITERAL
if call_name == "tuple":
new_node = cst.Tuple(elements=elements)
elif call_name == "list":
new_node = cst.List(elements=elements)
elif call_name == "set":
# set() doesn't have an equivelant literal call. If it was
# matched here, it's an unnecessary literal suggestion.
if len(elements) == 0:
self.report(
node,
UNNECESSARY_LITERAL.format(func=call_name),
replacement=node.deep_replace(
node, cst.Call(func=cst.Name("set"))),
)
return
new_node = cst.Set(elements=elements)
elif len(elements) == 0 or m.matches(
exp.args[0].value,
m.Tuple(elements=[pairs_matcher])
| m.List(elements=[pairs_matcher]),
):
new_node = cst.Dict(elements=[(
lambda val: cst.DictElement(val.elements[
0].value, val.elements[1].value))(cst.ensure_type(
ele.value,
cst.Tuple if isinstance(ele.value, cst.Tuple
) else cst.List,
)) for ele in elements])
else:
# Unrecoginized form
return
self.report(
node,
message_formatter.format(func=call_name),
replacement=node.deep_replace(node, new_node),
)
def obf_universal(self, node: cst.CSTNode, *types):
if m.matches(node, m.Name()):
types = ('a', 'ca', 'v', 'cv') if not types else types
node = cst.ensure_type(node, cst.Name)
if self.can_rename(node.value, *types):
node = self.get_new_cst_name(node)
elif m.matches(node, m.NameItem()):
node = cst.ensure_type(node, cst.NameItem)
node = node.with_changes(name=self.obf_universal(node.name))
elif m.matches(node, m.Call()):
node = cst.ensure_type(node, cst.Call)
if self.change_methods or self.change_functions:
node = self.new_obf_function_name(node)
if self.change_arguments or self.change_method_arguments:
node = self.obf_function_args(node)
elif m.matches(node, m.Attribute()):
node = cst.ensure_type(node, cst.Attribute)
value = node.value
attr = node.attr
self.obf_universal(value)
self.obf_universal(attr)
elif m.matches(node, m.AssignTarget()):
node = cst.ensure_type(node, cst.AssignTarget)
node = node.with_changes(target=self.obf_universal(node.target))
elif m.matches(node, m.List() | m.Tuple()):
node = cst.ensure_type(node, cst.List) if m.matches(
node, m.List()) else cst.ensure_type(node, cst.Tuple)
new_elements = []
for el in node.elements:
new_elements.append(self.obf_universal(el))
node = node.with_changes(elements=new_elements)
elif m.matches(node, m.Subscript()):
node = cst.ensure_type(node, cst.Subscript)
new_slice = []
for el in node.slice:
new_slice.append(
el.with_changes(slice=self.obf_slice(el.slice)))
node = node.with_changes(slice=new_slice)
node = node.with_changes(value=self.obf_universal(node.value))
elif m.matches(node, m.Element()):
node = cst.ensure_type(node, cst.Element)
node = node.with_changes(value=self.obf_universal(node.value))
elif m.matches(node, m.Dict()):
node = cst.ensure_type(node, cst.Dict)
new_elements = []
for el in node.elements:
new_elements.append(self.obf_universal(el))
node = node.with_changes(elements=new_elements)
elif m.matches(node, m.DictElement()):
node = cst.ensure_type(node, cst.DictElement)
new_key = self.obf_universal(node.key)
new_val = self.obf_universal(node.value)
node = node.with_changes(key=new_key, value=new_val)
elif m.matches(node, m.StarredDictElement()):
node = cst.ensure_type(node, cst.StarredDictElement)
node = node.with_changes(value=self.obf_universal(node.value))
elif m.matches(node, m.If() | m.While()):
node = cst.ensure_type(node, cst.IfExp) if m.matches(
node, cst.If
| cst.IfExp) else cst.ensure_type(node, cst.While)
node = node.with_changes(test=self.obf_universal(node.test))
elif m.matches(node, m.IfExp()):
node = cst.ensure_type(node, cst.IfExp)
node = node.with_changes(body=self.obf_universal(node.body))
node = node.with_changes(test=self.obf_universal(node.test))
node = node.with_changes(orelse=self.obf_universal(node.orelse))
elif m.matches(node, m.Comparison()):
node = cst.ensure_type(node, cst.Comparison)
new_compars = []
for target in node.comparisons:
new_compars.append(self.obf_universal(target))
node = node.with_changes(left=self.obf_universal(node.left))
node = node.with_changes(comparisons=new_compars)
elif m.matches(node, m.ComparisonTarget()):
node = cst.ensure_type(node, cst.ComparisonTarget)
node = node.with_changes(
comparator=self.obf_universal(node.comparator))
elif m.matches(node, m.FormattedString()):
node = cst.ensure_type(node, cst.FormattedString)
new_parts = []
for part in node.parts:
new_parts.append(self.obf_universal(part))
node = node.with_changes(parts=new_parts)
elif m.matches(node, m.FormattedStringExpression()):
node = cst.ensure_type(node, cst.FormattedStringExpression)
node = node.with_changes(
expression=self.obf_universal(node.expression))
elif m.matches(node, m.BinaryOperation() | m.BooleanOperation()):
node = cst.ensure_type(node, cst.BinaryOperation) if m.matches(
node, m.BinaryOperation()) else cst.ensure_type(
node, cst.BooleanOperation)
node = node.with_changes(left=self.obf_universal(node.left),
right=self.obf_universal(node.right))
elif m.matches(node, m.UnaryOperation()):
node = cst.ensure_type(node, cst.UnaryOperation)
node = node.with_changes(
expression=self.obf_universal(node.expression))
elif m.matches(node, m.ListComp()):
node = cst.ensure_type(node, cst.ListComp)
node = node.with_changes(elt=self.obf_universal(node.elt))
node = node.with_changes(for_in=self.obf_universal(node.for_in))
elif m.matches(node, m.DictComp()):
node = cst.ensure_type(node, cst.DictComp)
node = node.with_changes(key=self.obf_universal(node.key))
node = node.with_changes(value=self.obf_universal(node.value))
node = node.with_changes(for_in=self.obf_universal(node.for_in))
elif m.matches(node, m.CompFor()):
node = cst.ensure_type(node, cst.CompFor)
new_ifs = []
node = node.with_changes(target=self.obf_universal(node.target))
node = node.with_changes(iter=self.obf_universal(node.iter))
for el in node.ifs:
new_ifs.append(self.obf_universal(el))
node = node.with_changes(ifs=new_ifs)
elif m.matches(node, m.CompIf()):
node = cst.ensure_type(node, cst.CompIf)
node = node.with_changes(test=self.obf_universal(node.test))
elif m.matches(node, m.Integer() | m.Float() | m.SimpleString()):
pass
else:
pass
# print(node)
return node
def leave_FunctionDef(
self, original_node: cst.FunctionDef, updated_node: cst.FunctionDef
) -> Union[cst.BaseStatement, cst.RemovalSentinel]:
modified_defaults: List = []
mutable_args: List[Tuple[cst.Name, Union[cst.List, cst.Dict]]] = []
for param in updated_node.params.params:
if not m.matches(param,
m.Param(default=m.OneOf(m.List(), m.Dict()))):
modified_defaults.append(param)
continue
# This line here is just for type checkers peace of mind,
# since it cannot reason about variables from matchers result.
if not isinstance(param.default, (cst.List, cst.Dict)):
continue
mutable_args.append((param.name, param.default))
modified_defaults.append(
param.with_changes(default=cst.Name("None"), ))
if not mutable_args:
return original_node
modified_params: cst.Parameters = updated_node.params.with_changes(
params=modified_defaults)
initializations: List[Union[
cst.SimpleStatementLine, cst.BaseCompoundStatement]] = [
# We use generation by template here since construction of the
# resulting 'if' can be burdensome due to many nested objects
# involved. Additional line is attached so that we may control
# exact spacing between generated statements.
parse_template_statement(
DEFAULT_INIT_TEMPLATE,
config=self.module_config,
arg=arg,
init=init).with_changes(leading_lines=[EMPTY_LINE])
for arg, init in mutable_args
]
# Docstring should always go right after the function definition,
# so we take special care to insert our initializations after the
# last docstring found.
docstrings = takewhile(is_docstring, updated_node.body.body)
function_code = dropwhile(is_docstring, updated_node.body.body)
# It is not possible to insert empty line after the statement line,
# because whitespace is owned by the next statement after it.
stmt_with_empty_line = next(function_code).with_changes(
leading_lines=[EMPTY_LINE])
modified_body = (
*docstrings,
*initializations,
stmt_with_empty_line,
*function_code,
)
return updated_node.with_changes(
params=modified_params,
body=updated_node.body.with_changes(body=modified_body),
)
def visit_Call(self, node: cst.Call) -> None:
result = m.extract(
node,
m.Call(
func=m.Attribute(value=m.Name("self"),
attr=m.Name("assertTrue")),
args=[
m.DoNotCare(),
m.Arg(value=m.SaveMatchedNode(
m.OneOf(
m.Integer(),
m.Float(),
m.Imaginary(),
m.Tuple(),
m.List(),
m.Set(),
m.Dict(),
m.Name("None"),
m.Name("True"),
m.Name("False"),
),
"second",
)),
],
),
)
if result:
second_arg = result["second"]
if isinstance(second_arg, Sequence):
second_arg = second_arg[0]
if m.matches(second_arg, m.Name("True")):
new_call = node.with_changes(args=[
node.args[0].with_changes(comma=cst.MaybeSentinel.DEFAULT)
], )
elif m.matches(second_arg, m.Name("None")):
new_call = node.with_changes(
func=node.func.with_deep_changes(
old_node=cst.ensure_type(node.func,
cst.Attribute).attr,
value="assertIsNone",
),
args=[
node.args[0].with_changes(
comma=cst.MaybeSentinel.DEFAULT)
],
)
elif m.matches(second_arg, m.Name("False")):
new_call = node.with_changes(
func=node.func.with_deep_changes(
old_node=cst.ensure_type(node.func,
cst.Attribute).attr,
value="assertFalse",
),
args=[
node.args[0].with_changes(
comma=cst.MaybeSentinel.DEFAULT)
],
)
else:
new_call = node.with_deep_changes(
old_node=cst.ensure_type(node.func, cst.Attribute).attr,
value="assertEqual",
)
self.report(node, replacement=new_call)
