class UnhashableListTransformer(NoqaAwareTransformer): @m.call_if_inside( m.Call( func=m.OneOf(m.Name(value="set"), m.Name(value="frozenset")), args=[m.Arg(value=m.OneOf(m.List(), m.Tuple(), m.Set()))], ) | m.Set() # noqa: W503 ) @m.leave(m.List() | m.Set() | m.Tuple()) def convert_list_arg( self, _, updated_node: Union[cst.Set, cst.List, cst.Tuple]) -> cst.BaseExpression: modified_elements = convert_lists_to_tuples(updated_node.elements) return updated_node.with_changes(elements=modified_elements)
def visit_Annotation(self, node: cst.Annotation): if not match.matches( node, match.Annotation( annotation=match.OneOf(match.Name( value='None'), match.List(elements=[])))): self.type_annot_visited = True
def test_extract_optional_wildcard_tail(self) -> None: expression = cst.parse_expression("[3]") nodes = m.extract( expression, m.List(elements=[ m.Element(value=m.Integer(value="3")), m.SaveMatchedNode(m.ZeroOrMore(), "tail1"), m.SaveMatchedNode(m.ZeroOrMore(), "tail2"), ]), ) self.assertEqual(nodes, {"tail1": (), "tail2": ()})
def visit_Assign(self, node: cst.Assign) -> None: d = m.extract( node, m.Assign( targets=(m.AssignTarget(target=m.Name("CARBON_EXTS")), ), value=m.SaveMatchedNode(m.List(), "list"), ), ) if d: assert isinstance(d["list"], cst.List) for item in d["list"].elements: if isinstance(item.value, cst.SimpleString): self.extension_names.append(item.value.evaluated_value)
def visit_Call(self, node: cst.Call) -> None: if m.matches( node, m.Call( func=m.Name("list") | m.Name("set") | m.Name("dict"), args=[m.Arg(value=m.GeneratorExp() | m.ListComp())], ), ): call_name = cst.ensure_type(node.func, cst.Name).value if m.matches(node.args[0].value, m.GeneratorExp()): exp = cst.ensure_type(node.args[0].value, cst.GeneratorExp) message_formatter = UNNECESSARY_GENERATOR else: exp = cst.ensure_type(node.args[0].value, cst.ListComp) message_formatter = UNNECESSARY_LIST_COMPREHENSION replacement = None if call_name == "list": replacement = node.deep_replace( node, cst.ListComp(elt=exp.elt, for_in=exp.for_in)) elif call_name == "set": replacement = node.deep_replace( node, cst.SetComp(elt=exp.elt, for_in=exp.for_in)) elif call_name == "dict": elt = exp.elt key = None value = None if m.matches(elt, m.Tuple(m.DoNotCare(), m.DoNotCare())): elt = cst.ensure_type(elt, cst.Tuple) key = elt.elements[0].value value = elt.elements[1].value elif m.matches(elt, m.List(m.DoNotCare(), m.DoNotCare())): elt = cst.ensure_type(elt, cst.List) key = elt.elements[0].value value = elt.elements[1].value else: # Unrecoginized form return replacement = node.deep_replace( node, # pyre-fixme[6]: Expected `BaseAssignTargetExpression` for 1st # param but got `BaseExpression`. cst.DictComp(key=key, value=value, for_in=exp.for_in), ) self.report(node, message_formatter.format(func=call_name), replacement=replacement)
def visit_AugAssign(self, node: cst.AugAssign) -> bool: if m.matches( node, m.AugAssign( target=m.Name("__all__"), operator=m.AddAssign(), value=m.List() | m.Tuple(), ), ): value = node.value if isinstance(value, (cst.List, cst.Tuple)): self._is_assigned_export.add(value) return True return False
def convert_lists_to_tuples( elements: Sequence[cst.BaseElement], ) -> List[cst.BaseElement]: result: List[cst.BaseElement] = [] for element in elements: if m.matches(element, m.Element(value=m.List())): unhashable_list: cst.List = cst.ensure_type( element.value, cst.List) result.append( element.with_changes(value=cst.Tuple( elements=convert_lists_to_tuples( unhashable_list.elements)))) else: result.append(element) return result
def leave_SubscriptElement(self, original_node: cst.SubscriptElement, updated_node: cst.SubscriptElement): if self.type_annot_visited and self.parametric_type_annot_visited: if match.matches( original_node, match.SubscriptElement(slice=match.Index( value=match.Subscript()))): q_name, _ = self.__get_qualified_name( original_node.slice.value.value) if q_name is not None: return updated_node.with_changes(slice=cst.Index( value=cst.Subscript( value=self.__name2annotation(q_name).annotation, slice=updated_node.slice.value.slice))) elif match.matches( original_node, match.SubscriptElement(slice=match.Index( value=match.Ellipsis()))): # TODO: Should the original node be returned?! return updated_node.with_changes(slice=cst.Index( value=cst.Ellipsis())) elif match.matches( original_node, match.SubscriptElement(slice=match.Index( value=match.SimpleString(value=match.DoNotCare())))): return updated_node.with_changes(slice=cst.Index( value=updated_node.slice.value)) elif match.matches( original_node, match.SubscriptElement(slice=match.Index(value=match.Name( value='None')))): return original_node elif match.matches( original_node, match.SubscriptElement(slice=match.Index( value=match.List()))): return updated_node.with_changes(slice=cst.Index( value=updated_node.slice.value)) else: q_name, _ = self.__get_qualified_name( original_node.slice.value) if q_name is not None: return updated_node.with_changes(slice=cst.Index( value=self.__name2annotation(q_name).annotation)) return original_node
def visit_Call(self, node: cst.Call) -> None: if m.matches( node, m.Call( func=m.Name("tuple") | m.Name("list") | m.Name("set") | m.Name("dict"), args=[m.Arg(value=m.List() | m.Tuple())], ), ) or m.matches( node, m.Call(func=m.Name("tuple") | m.Name("list") | m.Name("dict"), args=[]), ): pairs_matcher = m.ZeroOrMore( m.Element(m.Tuple( elements=[m.DoNotCare(), m.DoNotCare()])) | m.Element(m.List( elements=[m.DoNotCare(), m.DoNotCare()]))) exp = cst.ensure_type(node, cst.Call) call_name = cst.ensure_type(exp.func, cst.Name).value # If this is a empty call, it's an Unnecessary Call where we rewrite the call # to literal, except set(). if not exp.args: elements = [] message_formatter = UNNCESSARY_CALL else: arg = exp.args[0].value elements = cst.ensure_type( arg, cst.List if isinstance(arg, cst.List) else cst.Tuple).elements message_formatter = UNNECESSARY_LITERAL if call_name == "tuple": new_node = cst.Tuple(elements=elements) elif call_name == "list": new_node = cst.List(elements=elements) elif call_name == "set": # set() doesn't have an equivelant literal call. If it was # matched here, it's an unnecessary literal suggestion. if len(elements) == 0: self.report( node, UNNECESSARY_LITERAL.format(func=call_name), replacement=node.deep_replace( node, cst.Call(func=cst.Name("set"))), ) return new_node = cst.Set(elements=elements) elif len(elements) == 0 or m.matches( exp.args[0].value, m.Tuple(elements=[pairs_matcher]) | m.List(elements=[pairs_matcher]), ): new_node = cst.Dict(elements=[( lambda val: cst.DictElement(val.elements[ 0].value, val.elements[1].value))(cst.ensure_type( ele.value, cst.Tuple if isinstance(ele.value, cst.Tuple ) else cst.List, )) for ele in elements]) else: # Unrecoginized form return self.report( node, message_formatter.format(func=call_name), replacement=node.deep_replace(node, new_node), )
def obf_universal(self, node: cst.CSTNode, *types): if m.matches(node, m.Name()): types = ('a', 'ca', 'v', 'cv') if not types else types node = cst.ensure_type(node, cst.Name) if self.can_rename(node.value, *types): node = self.get_new_cst_name(node) elif m.matches(node, m.NameItem()): node = cst.ensure_type(node, cst.NameItem) node = node.with_changes(name=self.obf_universal(node.name)) elif m.matches(node, m.Call()): node = cst.ensure_type(node, cst.Call) if self.change_methods or self.change_functions: node = self.new_obf_function_name(node) if self.change_arguments or self.change_method_arguments: node = self.obf_function_args(node) elif m.matches(node, m.Attribute()): node = cst.ensure_type(node, cst.Attribute) value = node.value attr = node.attr self.obf_universal(value) self.obf_universal(attr) elif m.matches(node, m.AssignTarget()): node = cst.ensure_type(node, cst.AssignTarget) node = node.with_changes(target=self.obf_universal(node.target)) elif m.matches(node, m.List() | m.Tuple()): node = cst.ensure_type(node, cst.List) if m.matches( node, m.List()) else cst.ensure_type(node, cst.Tuple) new_elements = [] for el in node.elements: new_elements.append(self.obf_universal(el)) node = node.with_changes(elements=new_elements) elif m.matches(node, m.Subscript()): node = cst.ensure_type(node, cst.Subscript) new_slice = [] for el in node.slice: new_slice.append( el.with_changes(slice=self.obf_slice(el.slice))) node = node.with_changes(slice=new_slice) node = node.with_changes(value=self.obf_universal(node.value)) elif m.matches(node, m.Element()): node = cst.ensure_type(node, cst.Element) node = node.with_changes(value=self.obf_universal(node.value)) elif m.matches(node, m.Dict()): node = cst.ensure_type(node, cst.Dict) new_elements = [] for el in node.elements: new_elements.append(self.obf_universal(el)) node = node.with_changes(elements=new_elements) elif m.matches(node, m.DictElement()): node = cst.ensure_type(node, cst.DictElement) new_key = self.obf_universal(node.key) new_val = self.obf_universal(node.value) node = node.with_changes(key=new_key, value=new_val) elif m.matches(node, m.StarredDictElement()): node = cst.ensure_type(node, cst.StarredDictElement) node = node.with_changes(value=self.obf_universal(node.value)) elif m.matches(node, m.If() | m.While()): node = cst.ensure_type(node, cst.IfExp) if m.matches( node, cst.If | cst.IfExp) else cst.ensure_type(node, cst.While) node = node.with_changes(test=self.obf_universal(node.test)) elif m.matches(node, m.IfExp()): node = cst.ensure_type(node, cst.IfExp) node = node.with_changes(body=self.obf_universal(node.body)) node = node.with_changes(test=self.obf_universal(node.test)) node = node.with_changes(orelse=self.obf_universal(node.orelse)) elif m.matches(node, m.Comparison()): node = cst.ensure_type(node, cst.Comparison) new_compars = [] for target in node.comparisons: new_compars.append(self.obf_universal(target)) node = node.with_changes(left=self.obf_universal(node.left)) node = node.with_changes(comparisons=new_compars) elif m.matches(node, m.ComparisonTarget()): node = cst.ensure_type(node, cst.ComparisonTarget) node = node.with_changes( comparator=self.obf_universal(node.comparator)) elif m.matches(node, m.FormattedString()): node = cst.ensure_type(node, cst.FormattedString) new_parts = [] for part in node.parts: new_parts.append(self.obf_universal(part)) node = node.with_changes(parts=new_parts) elif m.matches(node, m.FormattedStringExpression()): node = cst.ensure_type(node, cst.FormattedStringExpression) node = node.with_changes( expression=self.obf_universal(node.expression)) elif m.matches(node, m.BinaryOperation() | m.BooleanOperation()): node = cst.ensure_type(node, cst.BinaryOperation) if m.matches( node, m.BinaryOperation()) else cst.ensure_type( node, cst.BooleanOperation) node = node.with_changes(left=self.obf_universal(node.left), right=self.obf_universal(node.right)) elif m.matches(node, m.UnaryOperation()): node = cst.ensure_type(node, cst.UnaryOperation) node = node.with_changes( expression=self.obf_universal(node.expression)) elif m.matches(node, m.ListComp()): node = cst.ensure_type(node, cst.ListComp) node = node.with_changes(elt=self.obf_universal(node.elt)) node = node.with_changes(for_in=self.obf_universal(node.for_in)) elif m.matches(node, m.DictComp()): node = cst.ensure_type(node, cst.DictComp) node = node.with_changes(key=self.obf_universal(node.key)) node = node.with_changes(value=self.obf_universal(node.value)) node = node.with_changes(for_in=self.obf_universal(node.for_in)) elif m.matches(node, m.CompFor()): node = cst.ensure_type(node, cst.CompFor) new_ifs = [] node = node.with_changes(target=self.obf_universal(node.target)) node = node.with_changes(iter=self.obf_universal(node.iter)) for el in node.ifs: new_ifs.append(self.obf_universal(el)) node = node.with_changes(ifs=new_ifs) elif m.matches(node, m.CompIf()): node = cst.ensure_type(node, cst.CompIf) node = node.with_changes(test=self.obf_universal(node.test)) elif m.matches(node, m.Integer() | m.Float() | m.SimpleString()): pass else: pass # print(node) return node
def leave_FunctionDef( self, original_node: cst.FunctionDef, updated_node: cst.FunctionDef ) -> Union[cst.BaseStatement, cst.RemovalSentinel]: modified_defaults: List = [] mutable_args: List[Tuple[cst.Name, Union[cst.List, cst.Dict]]] = [] for param in updated_node.params.params: if not m.matches(param, m.Param(default=m.OneOf(m.List(), m.Dict()))): modified_defaults.append(param) continue # This line here is just for type checkers peace of mind, # since it cannot reason about variables from matchers result. if not isinstance(param.default, (cst.List, cst.Dict)): continue mutable_args.append((param.name, param.default)) modified_defaults.append( param.with_changes(default=cst.Name("None"), )) if not mutable_args: return original_node modified_params: cst.Parameters = updated_node.params.with_changes( params=modified_defaults) initializations: List[Union[ cst.SimpleStatementLine, cst.BaseCompoundStatement]] = [ # We use generation by template here since construction of the # resulting 'if' can be burdensome due to many nested objects # involved. Additional line is attached so that we may control # exact spacing between generated statements. parse_template_statement( DEFAULT_INIT_TEMPLATE, config=self.module_config, arg=arg, init=init).with_changes(leading_lines=[EMPTY_LINE]) for arg, init in mutable_args ] # Docstring should always go right after the function definition, # so we take special care to insert our initializations after the # last docstring found. docstrings = takewhile(is_docstring, updated_node.body.body) function_code = dropwhile(is_docstring, updated_node.body.body) # It is not possible to insert empty line after the statement line, # because whitespace is owned by the next statement after it. stmt_with_empty_line = next(function_code).with_changes( leading_lines=[EMPTY_LINE]) modified_body = ( *docstrings, *initializations, stmt_with_empty_line, *function_code, ) return updated_node.with_changes( params=modified_params, body=updated_node.body.with_changes(body=modified_body), )
def visit_Call(self, node: cst.Call) -> None: result = m.extract( node, m.Call( func=m.Attribute(value=m.Name("self"), attr=m.Name("assertTrue")), args=[ m.DoNotCare(), m.Arg(value=m.SaveMatchedNode( m.OneOf( m.Integer(), m.Float(), m.Imaginary(), m.Tuple(), m.List(), m.Set(), m.Dict(), m.Name("None"), m.Name("True"), m.Name("False"), ), "second", )), ], ), ) if result: second_arg = result["second"] if isinstance(second_arg, Sequence): second_arg = second_arg[0] if m.matches(second_arg, m.Name("True")): new_call = node.with_changes(args=[ node.args[0].with_changes(comma=cst.MaybeSentinel.DEFAULT) ], ) elif m.matches(second_arg, m.Name("None")): new_call = node.with_changes( func=node.func.with_deep_changes( old_node=cst.ensure_type(node.func, cst.Attribute).attr, value="assertIsNone", ), args=[ node.args[0].with_changes( comma=cst.MaybeSentinel.DEFAULT) ], ) elif m.matches(second_arg, m.Name("False")): new_call = node.with_changes( func=node.func.with_deep_changes( old_node=cst.ensure_type(node.func, cst.Attribute).attr, value="assertFalse", ), args=[ node.args[0].with_changes( comma=cst.MaybeSentinel.DEFAULT) ], ) else: new_call = node.with_deep_changes( old_node=cst.ensure_type(node.func, cst.Attribute).attr, value="assertEqual", ) self.report(node, replacement=new_call)