class TestVisitor(MatcherDecoratableVisitor): @leave(m.SimpleString() | m.Pass()) def _string_leave(self, node: cst.BaseExpression) -> None: pass
class TestVisitor(MatcherDecoratableTransformer): @leave(m.SimpleString()) def _string_leave( self, original_node: cst.SimpleString, updated_node: cst.SimpleString ) -> cst.BaseExpression: return updated_node
class TestVisitor(MatcherDecoratableTransformer): @leave(m.SimpleString()) def _string_visit( self, original_node: cst.SimpleString, updated_node: cst.SimpleString ) -> Union[cst.SimpleString, cst.RemovalSentinel]: return updated_node
class TestVisitor(MatcherDecoratableVisitor): @leave(m.SimpleString() | m.Name()) def _string_leave(self, node: cst.SimpleString) -> None: pass
def is_docstring(node): return m.matches( node, m.SimpleStatementLine(body=[m.Expr(value=m.SimpleString())]))
def check_not_simple_string(self, first_arg: Arg): """Translated patterns are not supported.""" if not m.matches(first_arg, m.Arg(value=m.SimpleString())): raise PatternNotSupported()
class TestVisitor(MatcherDecoratableVisitor): @leave(m.SimpleString()) def _string_leave( self, original_node: cst.SimpleString, updated_node: cst.SimpleString ) -> None: pass
def leave_Call( # noqa: C901 self, original_node: cst.Call, updated_node: cst.Call) -> cst.BaseExpression: # Lets figure out if this is a "".format() call extraction = self.extract( updated_node, m.Call(func=m.Attribute( value=m.SaveMatchedNode(m.SimpleString(), "string"), attr=m.Name("format"), )), ) if extraction is not None: fstring: List[cst.BaseFormattedStringContent] = [] inserted_sequence: int = 0 stringnode = cst.ensure_type(extraction["string"], cst.SimpleString) tokens = _get_tokens(stringnode.raw_value) for (literal_text, field_name, format_spec, conversion) in tokens: if literal_text: fstring.append(cst.FormattedStringText(literal_text)) if field_name is None: # This is not a format-specification continue # Auto-insert field sequence if it is empty if field_name == "": field_name = str(inserted_sequence) inserted_sequence += 1 # Now, if there is a valid format spec, parse it as a f-string # as well, since it allows for insertion of parameters just # like regular f-strings. format_spec_parts: List[cst.BaseFormattedStringContent] = [] if format_spec is not None and len(format_spec) > 0: # Parse the format spec out as a series of tokens as well. format_spec_tokens = _get_tokens(format_spec) for ( spec_literal_text, spec_field_name, spec_format_spec, spec_conversion, ) in format_spec_tokens: if spec_format_spec: # This shouldn't be possible, we don't allow it in the spec! raise Exception("Logic error!") if spec_literal_text: format_spec_parts.append( cst.FormattedStringText(spec_literal_text)) if spec_field_name is None: # This is not a format-specification continue # Auto-insert field sequence if it is empty if spec_field_name == "": spec_field_name = str(inserted_sequence) inserted_sequence += 1 # Now, convert the spec expression itself. fstring_expression = self._convert_token_to_fstring_expression( spec_field_name, spec_conversion, updated_node.args, stringnode, ) if fstring_expression is None: return updated_node format_spec_parts.append(fstring_expression) # Finally, output the converted value. fstring_expression = self._convert_token_to_fstring_expression( field_name, conversion, updated_node.args, stringnode) if fstring_expression is None: return updated_node # Technically its valid to add the parts even if it is empty, but # it results in an empty format spec being added which is ugly. if format_spec_parts: fstring_expression = fstring_expression.with_changes( format_spec=format_spec_parts) fstring.append(fstring_expression) # We converted each part, so lets bang together the f-string itself. return cst.FormattedString( parts=fstring, start=f"f{stringnode.prefix}{stringnode.quote}", end=stringnode.quote, ) return updated_node
class TestVisitor(MatcherDecoratableTransformer): @visit(m.SimpleString()) def _string_visit( self, original_node: cst.SimpleString, updated_node: cst.SimpleString ) -> None: pass
class TestVisitor(MatcherDecoratableTransformer): @leave(m.SimpleString()) def _string_leave( self, original_node: cst.SimpleString ) -> cst.SimpleString: return original_node
class TestVisitor(MatcherDecoratableTransformer): @visit(m.SimpleString()) def _string_visit(self, node: cst.SimpleString) -> bool: return False
class TestVisitor(MatcherDecoratableTransformer): @visit(m.SimpleString()) def visit_SimpleString(self, node: cst.SimpleString) -> None: pass
def leave_Call( # noqa: C901 self, original_node: cst.Call, updated_node: cst.Call) -> cst.BaseExpression: # Lets figure out if this is a "".format() call if self.matches( updated_node, m.Call(func=m.Attribute(value=m.SimpleString(), attr=m.Name("format"))), ): fstring: List[cst.BaseFormattedStringContent] = [] inserted_sequence: int = 0 # TODO: Use `extract` when it becomes available. stringvalue = cst.ensure_type( cst.ensure_type(updated_node.func, cst.Attribute).value, cst.SimpleString, ).value prefix, quote, innards = _string_prefix_and_quotes(stringvalue) tokens = _get_tokens(innards) for (literal_text, field_name, format_spec, conversion) in tokens: if literal_text: fstring.append(cst.FormattedStringText(literal_text)) if field_name is None: # This is not a format-specification continue if format_spec is not None and len(format_spec) > 0: # TODO: This is supportable since format specs are compatible # with f-string format specs, but it would require matching # format specifier expansions. self.warn( f"Unsupported format_spec {format_spec} in format() call" ) return updated_node # Auto-insert field sequence if it is empty if field_name == "": field_name = str(inserted_sequence) inserted_sequence += 1 expr = _find_expr_from_field_name(field_name, updated_node.args) if expr is None: # Most likely they used * expansion in a format. self.warn( f"Unsupported field_name {field_name} in format() call" ) return updated_node # Verify that we don't have any comments or newlines. Comments aren't # allowed in f-strings, and newlines need parenthesization. We can # have formattedstrings inside other formattedstrings, but I chose not # to doeal with that for now. if self.findall(expr, m.Comment()): # We could strip comments, but this is a formatting change so # we choose not to for now. self.warn(f"Unsupported comment in format() call") return updated_node if self.findall(expr, m.FormattedString()): self.warn(f"Unsupported f-string in format() call") return updated_node if self.findall(expr, m.Await()): # This is fixed in 3.7 but we don't currently have a flag # to enable/disable it. self.warn(f"Unsupported await in format() call") return updated_node # Stripping newlines is effectively a format-only change. expr = cst.ensure_type( expr.visit(StripNewlinesTransformer(self.context)), cst.BaseExpression, ) # Try our best to swap quotes on any strings that won't fit expr = cst.ensure_type( expr.visit( SwitchStringQuotesTransformer(self.context, quote[0])), cst.BaseExpression, ) # Verify that the resulting expression doesn't have a backslash # in it. raw_expr_string = self.module.code_for_node(expr) if "\\" in raw_expr_string: self.warn(f"Unsupported backslash in format expression") return updated_node # For safety sake, if this is a dict/set or dict/set comprehension, # wrap it in parens so that it doesn't accidentally create an # escape. if (raw_expr_string.startswith("{") or raw_expr_string.endswith("}")) and (not expr.lpar or not expr.rpar): expr = expr.with_changes(lpar=[cst.LeftParen()], rpar=[cst.RightParen()]) # Verify that any strings we insert don't have the same quote quote_gatherer = StringQuoteGatherer(self.context) expr.visit(quote_gatherer) for stringend in quote_gatherer.stringends: if stringend in quote: self.warn( f"Cannot embed string with same quote from format() call" ) return updated_node fstring.append( cst.FormattedStringExpression(expression=expr, conversion=conversion)) if quote not in ['"', '"""', "'", "'''"]: raise Exception(f"Invalid f-string quote {quote}") return cst.FormattedString( parts=fstring, start=f"f{prefix}{quote}", # pyre-ignore I know what I'm doing with end, so no Literal[str] # here. We get the string start/end from the original SimpleString # so we know its correct. end=quote, ) return updated_node
class TestVisitor(MatcherDecoratableTransformer): @visit(m.SimpleString() | m.Name()) def _string_visit(self, node: Union[cst.SimpleString, cst.Name]) -> None: pass
class TestVisitor(MatcherDecoratableVisitor): @leave(m.SimpleString()) def _string_leave(self, original_node: cst.SimpleString) -> bool: return False
class TestVisitor(MatcherDecoratableTransformer): @visit(m.SimpleString() | m.Name()) def _string_visit(self, node: cst.BaseExpression) -> None: pass
class TestVisitor(MatcherDecoratableTransformer): @leave(m.SimpleString()) def _string_visit( self, original_node: cst.SimpleString, updated_node: cst.SimpleString ) -> cst.BaseParenthesizableWhitespace: return cst.SimpleWhitespace("")
def visit_BinaryOperation(self, node: cst.BinaryOperation) -> None: expr_key = "expr" extracts = m.extract( node, m.BinaryOperation( left=m.MatchIfTrue(_match_simple_string), operator=m.Modulo(), right=m.SaveMatchedNode( m.MatchIfTrue( _gen_match_simple_expression( self.context.wrapper.module)), expr_key, ), ), ) if extracts: expr = extracts[expr_key] parts = [] simple_string = cst.ensure_type(node.left, cst.SimpleString) innards = simple_string.raw_value.replace("{", "{{").replace("}", "}}") tokens = innards.split("%s") token = tokens[0] if len(token) > 0: parts.append(cst.FormattedStringText(value=token)) expressions = ([elm.value for elm in expr.elements] if isinstance( expr, cst.Tuple) else [expr]) escape_transformer = EscapeStringQuote(simple_string.quote) i = 1 while i < len(tokens): if i - 1 >= len(expressions): # Only generate warning for cases where %-string not comes with same number of elements in tuple self.report(node) return try: parts.append( cst.FormattedStringExpression(expression=cast( cst.BaseExpression, expressions[i - 1].visit(escape_transformer), ))) except Exception: self.report(node) return token = tokens[i] if len(token) > 0: parts.append(cst.FormattedStringText(value=token)) i += 1 start = f"f{simple_string.prefix}{simple_string.quote}" replacement = cst.FormattedString(parts=parts, start=start, end=simple_string.quote) self.report(node, replacement=replacement) elif m.matches( node, m.BinaryOperation( left=m.SimpleString(), operator=m.Modulo())) and isinstance( cst.ensure_type( node.left, cst.SimpleString).evaluated_value, str): self.report(node)
class TestVisitor(MatcherDecoratableTransformer): @leave(m.SimpleString()) def _string_visit( self, original_node: cst.SimpleString, updated_node: cst.SimpleString ) -> cst.Pass: return cst.Pass()
def obf_universal(self, node: cst.CSTNode, *types): if m.matches(node, m.Name()): types = ('a', 'ca', 'v', 'cv') if not types else types node = cst.ensure_type(node, cst.Name) if self.can_rename(node.value, *types): node = self.get_new_cst_name(node) elif m.matches(node, m.NameItem()): node = cst.ensure_type(node, cst.NameItem) node = node.with_changes(name=self.obf_universal(node.name)) elif m.matches(node, m.Call()): node = cst.ensure_type(node, cst.Call) if self.change_methods or self.change_functions: node = self.new_obf_function_name(node) if self.change_arguments or self.change_method_arguments: node = self.obf_function_args(node) elif m.matches(node, m.Attribute()): node = cst.ensure_type(node, cst.Attribute) value = node.value attr = node.attr self.obf_universal(value) self.obf_universal(attr) elif m.matches(node, m.AssignTarget()): node = cst.ensure_type(node, cst.AssignTarget) node = node.with_changes(target=self.obf_universal(node.target)) elif m.matches(node, m.List() | m.Tuple()): node = cst.ensure_type(node, cst.List) if m.matches( node, m.List()) else cst.ensure_type(node, cst.Tuple) new_elements = [] for el in node.elements: new_elements.append(self.obf_universal(el)) node = node.with_changes(elements=new_elements) elif m.matches(node, m.Subscript()): node = cst.ensure_type(node, cst.Subscript) new_slice = [] for el in node.slice: new_slice.append( el.with_changes(slice=self.obf_slice(el.slice))) node = node.with_changes(slice=new_slice) node = node.with_changes(value=self.obf_universal(node.value)) elif m.matches(node, m.Element()): node = cst.ensure_type(node, cst.Element) node = node.with_changes(value=self.obf_universal(node.value)) elif m.matches(node, m.Dict()): node = cst.ensure_type(node, cst.Dict) new_elements = [] for el in node.elements: new_elements.append(self.obf_universal(el)) node = node.with_changes(elements=new_elements) elif m.matches(node, m.DictElement()): node = cst.ensure_type(node, cst.DictElement) new_key = self.obf_universal(node.key) new_val = self.obf_universal(node.value) node = node.with_changes(key=new_key, value=new_val) elif m.matches(node, m.StarredDictElement()): node = cst.ensure_type(node, cst.StarredDictElement) node = node.with_changes(value=self.obf_universal(node.value)) elif m.matches(node, m.If() | m.While()): node = cst.ensure_type(node, cst.IfExp) if m.matches( node, cst.If | cst.IfExp) else cst.ensure_type(node, cst.While) node = node.with_changes(test=self.obf_universal(node.test)) elif m.matches(node, m.IfExp()): node = cst.ensure_type(node, cst.IfExp) node = node.with_changes(body=self.obf_universal(node.body)) node = node.with_changes(test=self.obf_universal(node.test)) node = node.with_changes(orelse=self.obf_universal(node.orelse)) elif m.matches(node, m.Comparison()): node = cst.ensure_type(node, cst.Comparison) new_compars = [] for target in node.comparisons: new_compars.append(self.obf_universal(target)) node = node.with_changes(left=self.obf_universal(node.left)) node = node.with_changes(comparisons=new_compars) elif m.matches(node, m.ComparisonTarget()): node = cst.ensure_type(node, cst.ComparisonTarget) node = node.with_changes( comparator=self.obf_universal(node.comparator)) elif m.matches(node, m.FormattedString()): node = cst.ensure_type(node, cst.FormattedString) new_parts = [] for part in node.parts: new_parts.append(self.obf_universal(part)) node = node.with_changes(parts=new_parts) elif m.matches(node, m.FormattedStringExpression()): node = cst.ensure_type(node, cst.FormattedStringExpression) node = node.with_changes( expression=self.obf_universal(node.expression)) elif m.matches(node, m.BinaryOperation() | m.BooleanOperation()): node = cst.ensure_type(node, cst.BinaryOperation) if m.matches( node, m.BinaryOperation()) else cst.ensure_type( node, cst.BooleanOperation) node = node.with_changes(left=self.obf_universal(node.left), right=self.obf_universal(node.right)) elif m.matches(node, m.UnaryOperation()): node = cst.ensure_type(node, cst.UnaryOperation) node = node.with_changes( expression=self.obf_universal(node.expression)) elif m.matches(node, m.ListComp()): node = cst.ensure_type(node, cst.ListComp) node = node.with_changes(elt=self.obf_universal(node.elt)) node = node.with_changes(for_in=self.obf_universal(node.for_in)) elif m.matches(node, m.DictComp()): node = cst.ensure_type(node, cst.DictComp) node = node.with_changes(key=self.obf_universal(node.key)) node = node.with_changes(value=self.obf_universal(node.value)) node = node.with_changes(for_in=self.obf_universal(node.for_in)) elif m.matches(node, m.CompFor()): node = cst.ensure_type(node, cst.CompFor) new_ifs = [] node = node.with_changes(target=self.obf_universal(node.target)) node = node.with_changes(iter=self.obf_universal(node.iter)) for el in node.ifs: new_ifs.append(self.obf_universal(el)) node = node.with_changes(ifs=new_ifs) elif m.matches(node, m.CompIf()): node = cst.ensure_type(node, cst.CompIf) node = node.with_changes(test=self.obf_universal(node.test)) elif m.matches(node, m.Integer() | m.Float() | m.SimpleString()): pass else: pass # print(node) return node
def leave_Expr(self, old_node, new_node): if m.matches(old_node, m.Expr(m.SimpleString())): return cst.RemovalSentinel.REMOVE return new_node