class TestVisitor(MatcherDecoratableVisitor):
@leave(m.SimpleString() | m.Pass())
def _string_leave(self, node: cst.BaseExpression) -> None:
pass
class TestVisitor(MatcherDecoratableTransformer):
@leave(m.SimpleString())
def _string_leave(
self, original_node: cst.SimpleString, updated_node: cst.SimpleString
) -> cst.BaseExpression:
return updated_node
class TestVisitor(MatcherDecoratableTransformer):
@leave(m.SimpleString())
def _string_visit(
self, original_node: cst.SimpleString, updated_node: cst.SimpleString
) -> Union[cst.SimpleString, cst.RemovalSentinel]:
return updated_node
class TestVisitor(MatcherDecoratableVisitor):
@leave(m.SimpleString() | m.Name())
def _string_leave(self, node: cst.SimpleString) -> None:
pass
def is_docstring(node):
return m.matches(
node, m.SimpleStatementLine(body=[m.Expr(value=m.SimpleString())]))
def check_not_simple_string(self, first_arg: Arg):
"""Translated patterns are not supported."""
if not m.matches(first_arg, m.Arg(value=m.SimpleString())):
raise PatternNotSupported()
class TestVisitor(MatcherDecoratableVisitor):
@leave(m.SimpleString())
def _string_leave(
self, original_node: cst.SimpleString, updated_node: cst.SimpleString
) -> None:
pass
def leave_Call( # noqa: C901
self, original_node: cst.Call,
updated_node: cst.Call) -> cst.BaseExpression:
# Lets figure out if this is a "".format() call
extraction = self.extract(
updated_node,
m.Call(func=m.Attribute(
value=m.SaveMatchedNode(m.SimpleString(), "string"),
attr=m.Name("format"),
)),
)
if extraction is not None:
fstring: List[cst.BaseFormattedStringContent] = []
inserted_sequence: int = 0
stringnode = cst.ensure_type(extraction["string"],
cst.SimpleString)
tokens = _get_tokens(stringnode.raw_value)
for (literal_text, field_name, format_spec, conversion) in tokens:
if literal_text:
fstring.append(cst.FormattedStringText(literal_text))
if field_name is None:
# This is not a format-specification
continue
# Auto-insert field sequence if it is empty
if field_name == "":
field_name = str(inserted_sequence)
inserted_sequence += 1
# Now, if there is a valid format spec, parse it as a f-string
# as well, since it allows for insertion of parameters just
# like regular f-strings.
format_spec_parts: List[cst.BaseFormattedStringContent] = []
if format_spec is not None and len(format_spec) > 0:
# Parse the format spec out as a series of tokens as well.
format_spec_tokens = _get_tokens(format_spec)
for (
spec_literal_text,
spec_field_name,
spec_format_spec,
spec_conversion,
) in format_spec_tokens:
if spec_format_spec:
# This shouldn't be possible, we don't allow it in the spec!
raise Exception("Logic error!")
if spec_literal_text:
format_spec_parts.append(
cst.FormattedStringText(spec_literal_text))
if spec_field_name is None:
# This is not a format-specification
continue
# Auto-insert field sequence if it is empty
if spec_field_name == "":
spec_field_name = str(inserted_sequence)
inserted_sequence += 1
# Now, convert the spec expression itself.
fstring_expression = self._convert_token_to_fstring_expression(
spec_field_name,
spec_conversion,
updated_node.args,
stringnode,
)
if fstring_expression is None:
return updated_node
format_spec_parts.append(fstring_expression)
# Finally, output the converted value.
fstring_expression = self._convert_token_to_fstring_expression(
field_name, conversion, updated_node.args, stringnode)
if fstring_expression is None:
return updated_node
# Technically its valid to add the parts even if it is empty, but
# it results in an empty format spec being added which is ugly.
if format_spec_parts:
fstring_expression = fstring_expression.with_changes(
format_spec=format_spec_parts)
fstring.append(fstring_expression)
# We converted each part, so lets bang together the f-string itself.
return cst.FormattedString(
parts=fstring,
start=f"f{stringnode.prefix}{stringnode.quote}",
end=stringnode.quote,
)
return updated_node
class TestVisitor(MatcherDecoratableTransformer):
@visit(m.SimpleString())
def _string_visit(
self, original_node: cst.SimpleString, updated_node: cst.SimpleString
) -> None:
pass
class TestVisitor(MatcherDecoratableTransformer):
@leave(m.SimpleString())
def _string_leave(
self, original_node: cst.SimpleString
) -> cst.SimpleString:
return original_node
class TestVisitor(MatcherDecoratableTransformer):
@visit(m.SimpleString())
def _string_visit(self, node: cst.SimpleString) -> bool:
return False
class TestVisitor(MatcherDecoratableTransformer):
@visit(m.SimpleString())
def visit_SimpleString(self, node: cst.SimpleString) -> None:
pass
def leave_Call( # noqa: C901
self, original_node: cst.Call,
updated_node: cst.Call) -> cst.BaseExpression:
# Lets figure out if this is a "".format() call
if self.matches(
updated_node,
m.Call(func=m.Attribute(value=m.SimpleString(),
attr=m.Name("format"))),
):
fstring: List[cst.BaseFormattedStringContent] = []
inserted_sequence: int = 0
# TODO: Use `extract` when it becomes available.
stringvalue = cst.ensure_type(
cst.ensure_type(updated_node.func, cst.Attribute).value,
cst.SimpleString,
).value
prefix, quote, innards = _string_prefix_and_quotes(stringvalue)
tokens = _get_tokens(innards)
for (literal_text, field_name, format_spec, conversion) in tokens:
if literal_text:
fstring.append(cst.FormattedStringText(literal_text))
if field_name is None:
# This is not a format-specification
continue
if format_spec is not None and len(format_spec) > 0:
# TODO: This is supportable since format specs are compatible
# with f-string format specs, but it would require matching
# format specifier expansions.
self.warn(
f"Unsupported format_spec {format_spec} in format() call"
)
return updated_node
# Auto-insert field sequence if it is empty
if field_name == "":
field_name = str(inserted_sequence)
inserted_sequence += 1
expr = _find_expr_from_field_name(field_name,
updated_node.args)
if expr is None:
# Most likely they used * expansion in a format.
self.warn(
f"Unsupported field_name {field_name} in format() call"
)
return updated_node
# Verify that we don't have any comments or newlines. Comments aren't
# allowed in f-strings, and newlines need parenthesization. We can
# have formattedstrings inside other formattedstrings, but I chose not
# to doeal with that for now.
if self.findall(expr, m.Comment()):
# We could strip comments, but this is a formatting change so
# we choose not to for now.
self.warn(f"Unsupported comment in format() call")
return updated_node
if self.findall(expr, m.FormattedString()):
self.warn(f"Unsupported f-string in format() call")
return updated_node
if self.findall(expr, m.Await()):
# This is fixed in 3.7 but we don't currently have a flag
# to enable/disable it.
self.warn(f"Unsupported await in format() call")
return updated_node
# Stripping newlines is effectively a format-only change.
expr = cst.ensure_type(
expr.visit(StripNewlinesTransformer(self.context)),
cst.BaseExpression,
)
# Try our best to swap quotes on any strings that won't fit
expr = cst.ensure_type(
expr.visit(
SwitchStringQuotesTransformer(self.context, quote[0])),
cst.BaseExpression,
)
# Verify that the resulting expression doesn't have a backslash
# in it.
raw_expr_string = self.module.code_for_node(expr)
if "\\" in raw_expr_string:
self.warn(f"Unsupported backslash in format expression")
return updated_node
# For safety sake, if this is a dict/set or dict/set comprehension,
# wrap it in parens so that it doesn't accidentally create an
# escape.
if (raw_expr_string.startswith("{")
or raw_expr_string.endswith("}")) and (not expr.lpar or
not expr.rpar):
expr = expr.with_changes(lpar=[cst.LeftParen()],
rpar=[cst.RightParen()])
# Verify that any strings we insert don't have the same quote
quote_gatherer = StringQuoteGatherer(self.context)
expr.visit(quote_gatherer)
for stringend in quote_gatherer.stringends:
if stringend in quote:
self.warn(
f"Cannot embed string with same quote from format() call"
)
return updated_node
fstring.append(
cst.FormattedStringExpression(expression=expr,
conversion=conversion))
if quote not in ['"', '"""', "'", "'''"]:
raise Exception(f"Invalid f-string quote {quote}")
return cst.FormattedString(
parts=fstring,
start=f"f{prefix}{quote}",
# pyre-ignore I know what I'm doing with end, so no Literal[str]
# here. We get the string start/end from the original SimpleString
# so we know its correct.
end=quote,
)
return updated_node
class TestVisitor(MatcherDecoratableTransformer):
@visit(m.SimpleString() | m.Name())
def _string_visit(self, node: Union[cst.SimpleString, cst.Name]) -> None:
pass
class TestVisitor(MatcherDecoratableVisitor):
@leave(m.SimpleString())
def _string_leave(self, original_node: cst.SimpleString) -> bool:
return False
class TestVisitor(MatcherDecoratableTransformer):
@visit(m.SimpleString() | m.Name())
def _string_visit(self, node: cst.BaseExpression) -> None:
pass
class TestVisitor(MatcherDecoratableTransformer):
@leave(m.SimpleString())
def _string_visit(
self, original_node: cst.SimpleString, updated_node: cst.SimpleString
) -> cst.BaseParenthesizableWhitespace:
return cst.SimpleWhitespace("")
def visit_BinaryOperation(self, node: cst.BinaryOperation) -> None:
expr_key = "expr"
extracts = m.extract(
node,
m.BinaryOperation(
left=m.MatchIfTrue(_match_simple_string),
operator=m.Modulo(),
right=m.SaveMatchedNode(
m.MatchIfTrue(
_gen_match_simple_expression(
self.context.wrapper.module)),
expr_key,
),
),
)
if extracts:
expr = extracts[expr_key]
parts = []
simple_string = cst.ensure_type(node.left, cst.SimpleString)
innards = simple_string.raw_value.replace("{",
"{{").replace("}", "}}")
tokens = innards.split("%s")
token = tokens[0]
if len(token) > 0:
parts.append(cst.FormattedStringText(value=token))
expressions = ([elm.value for elm in expr.elements] if isinstance(
expr, cst.Tuple) else [expr])
escape_transformer = EscapeStringQuote(simple_string.quote)
i = 1
while i < len(tokens):
if i - 1 >= len(expressions):
# Only generate warning for cases where %-string not comes with same number of elements in tuple
self.report(node)
return
try:
parts.append(
cst.FormattedStringExpression(expression=cast(
cst.BaseExpression,
expressions[i - 1].visit(escape_transformer),
)))
except Exception:
self.report(node)
return
token = tokens[i]
if len(token) > 0:
parts.append(cst.FormattedStringText(value=token))
i += 1
start = f"f{simple_string.prefix}{simple_string.quote}"
replacement = cst.FormattedString(parts=parts,
start=start,
end=simple_string.quote)
self.report(node, replacement=replacement)
elif m.matches(
node,
m.BinaryOperation(
left=m.SimpleString(),
operator=m.Modulo())) and isinstance(
cst.ensure_type(
node.left, cst.SimpleString).evaluated_value, str):
self.report(node)
class TestVisitor(MatcherDecoratableTransformer):
@leave(m.SimpleString())
def _string_visit(
self, original_node: cst.SimpleString, updated_node: cst.SimpleString
) -> cst.Pass:
return cst.Pass()
def obf_universal(self, node: cst.CSTNode, *types):
if m.matches(node, m.Name()):
types = ('a', 'ca', 'v', 'cv') if not types else types
node = cst.ensure_type(node, cst.Name)
if self.can_rename(node.value, *types):
node = self.get_new_cst_name(node)
elif m.matches(node, m.NameItem()):
node = cst.ensure_type(node, cst.NameItem)
node = node.with_changes(name=self.obf_universal(node.name))
elif m.matches(node, m.Call()):
node = cst.ensure_type(node, cst.Call)
if self.change_methods or self.change_functions:
node = self.new_obf_function_name(node)
if self.change_arguments or self.change_method_arguments:
node = self.obf_function_args(node)
elif m.matches(node, m.Attribute()):
node = cst.ensure_type(node, cst.Attribute)
value = node.value
attr = node.attr
self.obf_universal(value)
self.obf_universal(attr)
elif m.matches(node, m.AssignTarget()):
node = cst.ensure_type(node, cst.AssignTarget)
node = node.with_changes(target=self.obf_universal(node.target))
elif m.matches(node, m.List() | m.Tuple()):
node = cst.ensure_type(node, cst.List) if m.matches(
node, m.List()) else cst.ensure_type(node, cst.Tuple)
new_elements = []
for el in node.elements:
new_elements.append(self.obf_universal(el))
node = node.with_changes(elements=new_elements)
elif m.matches(node, m.Subscript()):
node = cst.ensure_type(node, cst.Subscript)
new_slice = []
for el in node.slice:
new_slice.append(
el.with_changes(slice=self.obf_slice(el.slice)))
node = node.with_changes(slice=new_slice)
node = node.with_changes(value=self.obf_universal(node.value))
elif m.matches(node, m.Element()):
node = cst.ensure_type(node, cst.Element)
node = node.with_changes(value=self.obf_universal(node.value))
elif m.matches(node, m.Dict()):
node = cst.ensure_type(node, cst.Dict)
new_elements = []
for el in node.elements:
new_elements.append(self.obf_universal(el))
node = node.with_changes(elements=new_elements)
elif m.matches(node, m.DictElement()):
node = cst.ensure_type(node, cst.DictElement)
new_key = self.obf_universal(node.key)
new_val = self.obf_universal(node.value)
node = node.with_changes(key=new_key, value=new_val)
elif m.matches(node, m.StarredDictElement()):
node = cst.ensure_type(node, cst.StarredDictElement)
node = node.with_changes(value=self.obf_universal(node.value))
elif m.matches(node, m.If() | m.While()):
node = cst.ensure_type(node, cst.IfExp) if m.matches(
node, cst.If
| cst.IfExp) else cst.ensure_type(node, cst.While)
node = node.with_changes(test=self.obf_universal(node.test))
elif m.matches(node, m.IfExp()):
node = cst.ensure_type(node, cst.IfExp)
node = node.with_changes(body=self.obf_universal(node.body))
node = node.with_changes(test=self.obf_universal(node.test))
node = node.with_changes(orelse=self.obf_universal(node.orelse))
elif m.matches(node, m.Comparison()):
node = cst.ensure_type(node, cst.Comparison)
new_compars = []
for target in node.comparisons:
new_compars.append(self.obf_universal(target))
node = node.with_changes(left=self.obf_universal(node.left))
node = node.with_changes(comparisons=new_compars)
elif m.matches(node, m.ComparisonTarget()):
node = cst.ensure_type(node, cst.ComparisonTarget)
node = node.with_changes(
comparator=self.obf_universal(node.comparator))
elif m.matches(node, m.FormattedString()):
node = cst.ensure_type(node, cst.FormattedString)
new_parts = []
for part in node.parts:
new_parts.append(self.obf_universal(part))
node = node.with_changes(parts=new_parts)
elif m.matches(node, m.FormattedStringExpression()):
node = cst.ensure_type(node, cst.FormattedStringExpression)
node = node.with_changes(
expression=self.obf_universal(node.expression))
elif m.matches(node, m.BinaryOperation() | m.BooleanOperation()):
node = cst.ensure_type(node, cst.BinaryOperation) if m.matches(
node, m.BinaryOperation()) else cst.ensure_type(
node, cst.BooleanOperation)
node = node.with_changes(left=self.obf_universal(node.left),
right=self.obf_universal(node.right))
elif m.matches(node, m.UnaryOperation()):
node = cst.ensure_type(node, cst.UnaryOperation)
node = node.with_changes(
expression=self.obf_universal(node.expression))
elif m.matches(node, m.ListComp()):
node = cst.ensure_type(node, cst.ListComp)
node = node.with_changes(elt=self.obf_universal(node.elt))
node = node.with_changes(for_in=self.obf_universal(node.for_in))
elif m.matches(node, m.DictComp()):
node = cst.ensure_type(node, cst.DictComp)
node = node.with_changes(key=self.obf_universal(node.key))
node = node.with_changes(value=self.obf_universal(node.value))
node = node.with_changes(for_in=self.obf_universal(node.for_in))
elif m.matches(node, m.CompFor()):
node = cst.ensure_type(node, cst.CompFor)
new_ifs = []
node = node.with_changes(target=self.obf_universal(node.target))
node = node.with_changes(iter=self.obf_universal(node.iter))
for el in node.ifs:
new_ifs.append(self.obf_universal(el))
node = node.with_changes(ifs=new_ifs)
elif m.matches(node, m.CompIf()):
node = cst.ensure_type(node, cst.CompIf)
node = node.with_changes(test=self.obf_universal(node.test))
elif m.matches(node, m.Integer() | m.Float() | m.SimpleString()):
pass
else:
pass
# print(node)
return node
def leave_Expr(self, old_node, new_node):
if m.matches(old_node, m.Expr(m.SimpleString())):
return cst.RemovalSentinel.REMOVE
return new_node
