def visit_Assign(self, node) -> None:
if (m.matches(node, m.Assign(targets=[m.AssignTarget(m.Name())]))
and self.toplevel == 0):
name = node.targets[0].target
self.imprts[name.value] = cst.ImportFrom(
module=parse_expr(self.mod),
names=[cst.ImportAlias(name=name, asname=None)])
class TestVisitor(MatcherDecoratableTransformer):
@leave(m.AssignTarget())
def _string_visit(
self, original_node: cst.AssignTarget,
updated_node: cst.AssignTarget
) -> Union[cst.AssignTarget, cst.RemovalSentinel]:
return updated_node
def visit_AssignTarget(self, node: cst.AssignTarget) -> Optional[bool]:
if m.matches(node, m.AssignTarget(target=m.Name())):
target = cst.ensure_type(node.target, cst.Name)
if target.value == "__all__":
self.already_exists = True
else:
self.process_node(node.target, target.value)
return None
def visit_AssignTarget(self, node: cst.AssignTarget):
if match.matches(
node,
match.AssignTarget(target=match.Name(
value=match.DoNotCare()))):
if self.last_visited_assign_t_name == node.target.value:
self.last_visited_assign_t_count += 1
elif self.last_visited_assign_t_count == 0:
self.last_visited_assign_t_count = 1
else:
self.last_visited_assign_t_count = 1
self.last_visited_assign_t_name = node.target.value
def __get_var_names_counter(self, node, scope):
vars_name = match.extractall(
node,
match.OneOf(
match.AssignTarget(target=match.SaveMatchedNode(
match.Name(value=match.DoNotCare()), "name")),
match.AnnAssign(target=match.SaveMatchedNode(
match.Name(value=match.DoNotCare()), "name"))))
return Counter([
n['name'].value for n in vars_name if isinstance(
self.get_metadata(cst.metadata.ScopeProvider, n['name']),
scope)
])
def visit_Assign(self, node: cst.Assign) -> None:
d = m.extract(
node,
m.Assign(
targets=(m.AssignTarget(target=m.Name("CARBON_EXTS")), ),
value=m.SaveMatchedNode(m.List(), "list"),
),
)
if d:
assert isinstance(d["list"], cst.List)
for item in d["list"].elements:
if isinstance(item.value, cst.SimpleString):
self.extension_names.append(item.value.evaluated_value)
def leave_SimpleStatementLine(self, original_node: cst.SimpleStatementLine,
updated_node: cst.SimpleStatementLine):
if match.matches(
original_node,
match.SimpleStatementLine(body=[
match.Assign(targets=[
match.AssignTarget(target=match.Name(
value=match.DoNotCare()))
])
])):
t = self.__get_var_type_assign_t(
original_node.body[0].targets[0].target.value)
if t is not None:
t_annot_node_resolved = self.resolve_type_alias(t)
t_annot_node = self.__name2annotation(t_annot_node_resolved)
if t_annot_node is not None:
self.all_applied_types.add(
(t_annot_node_resolved, t_annot_node))
return updated_node.with_changes(body=[
cst.AnnAssign(
target=original_node.body[0].targets[0].target,
value=original_node.body[0].value,
annotation=t_annot_node,
equal=cst.AssignEqual(
whitespace_after=original_node.body[0].
targets[0].whitespace_after_equal,
whitespace_before=original_node.body[0].
targets[0].whitespace_before_equal))
])
elif match.matches(
original_node,
match.SimpleStatementLine(body=[
match.AnnAssign(target=match.Name(value=match.DoNotCare()))
])):
t = self.__get_var_type_an_assign(
original_node.body[0].target.value)
if t is not None:
t_annot_node_resolved = self.resolve_type_alias(t)
t_annot_node = self.__name2annotation(t_annot_node_resolved)
if t_annot_node is not None:
self.all_applied_types.add(
(t_annot_node_resolved, t_annot_node))
return updated_node.with_changes(body=[
cst.AnnAssign(target=original_node.body[0].target,
value=original_node.body[0].value,
annotation=t_annot_node,
equal=original_node.body[0].equal)
])
return original_node
def _split_module(
self, orig_module: libcst.Module, updated_module: libcst.Module
) -> Tuple[List[Union[libcst.SimpleStatementLine,
libcst.BaseCompoundStatement]],
List[Union[libcst.SimpleStatementLine,
libcst.BaseCompoundStatement]], List[Union[
libcst.SimpleStatementLine,
libcst.BaseCompoundStatement]], ]:
statement_before_import_location = 0
import_add_location = 0
# never insert an import before initial __strict__ flag
if m.matches(
orig_module,
m.Module(body=[
m.SimpleStatementLine(body=[
m.Assign(targets=[
m.AssignTarget(target=m.Name("__strict__"))
])
]),
m.ZeroOrMore(),
]),
):
statement_before_import_location = import_add_location = 1
# This works under the principle that while we might modify node contents,
# we have yet to modify the number of statements. So we can match on the
# original tree but break up the statements of the modified tree. If we
# change this assumption in this visitor, we will have to change this code.
for i, statement in enumerate(orig_module.body):
if m.matches(
statement,
m.SimpleStatementLine(
body=[m.Expr(value=m.SimpleString())])):
statement_before_import_location = import_add_location = 1
elif isinstance(statement, libcst.SimpleStatementLine):
for possible_import in statement.body:
for last_import in self.all_imports:
if possible_import is last_import:
import_add_location = i + 1
break
return (
list(updated_module.body[:statement_before_import_location]),
list(updated_module.
body[statement_before_import_location:import_add_location]),
list(updated_module.body[import_add_location:]),
)
def visit_Assign(self, node: cst.Assign) -> None:
metadata: Optional[Collection[QualifiedName]] = self.get_metadata(
QualifiedNameProvider, node.value, None)
if metadata is not None:
for qualname in metadata:
# If the assignment is done with some objects from the typing or
# collections module, then we will skip the check as the assignment
# could be a type alias or the variable could be a class made using
# ``collections.namedtuple``.
if qualname.name.startswith(("typing", "collections")):
return None
for target_node in node.targets:
if m.matches(target_node, m.AssignTarget(target=m.Name())):
nodename = cst.ensure_type(target_node.target, cst.Name).value
self._validate_nodename(node, nodename,
NamingConvention.SNAKE_CASE)
def leave_Assign(self, original_node, updated_node):
if any([
m.matches(
updated_node,
m.Assign(targets=[m.AssignTarget(m.Name())],
value=pattern)) for pattern in self.rhs_patterns
]):
var = original_node.targets[0].target
scope = self.get_metadata(ScopeProvider, var)
children = self._scope_children[scope]
if len(scope.assignments[var]) == 1:
valid_scopes = [scope] + [
child
for child in children if len(child.assignments[var]) == 0
]
self.propagate(valid_scopes, var, updated_node.value)
return cst.RemoveFromParent()
return updated_node
class VersionTransformer(m.MatcherDecoratableTransformer):
new_version: Union[None, Version] = None
def __init__(self, version_mod: Callable[[Version], Version]):
super().__init__()
self.version_mod = version_mod
@m.call_if_inside(
m.Assign(
targets=[m.AssignTarget(target=m.Name("__version__"))],
value=m.SimpleString(),
))
@m.leave(m.SimpleString())
def update_version(self, original_node: cst.SimpleString,
updated_node: cst.SimpleString) -> cst.SimpleString:
if self.new_version:
raise Exception("Multiple versions found.")
old_version = Version(updated_node.evaluated_value)
self.new_version = self.version_mod(old_version)
return updated_node.with_changes(value=f'"{self.new_version}"')
def __extract_variable_name(self, node: cst.AssignTarget):
extracted_var_names = match.extract(
node,
match.AssignTarget( # Assignment operator
target=match.OneOf( # Two cases exist
match.Name( # Single target
value=match.SaveMatchedNode( # Save result
match.MatchRegex(
r'(.)+'), # Match any string literal
"name")),
match.Tuple( # Multi-target
elements=match.SaveMatchedNode( # Save result
match.DoNotCare(), # Type of list
"names")),
# This extracts variables inside __init__ without type annotation (e.g. self.x=2)
match.Attribute(
value=match.Name(value=match.SaveMatchedNode(
match.MatchRegex(r'(.)+'),
"obj_name" # Object name
)),
attr=match.Name(
match.SaveMatchedNode(match.MatchRegex(r'(.)+'),
"name")),
))))
if extracted_var_names is not None:
if "name" in extracted_var_names:
t = self.__get_type_from_metadata(node.target)
extracted_var_names['type'] = (t, INF_TYPE_ANNOT
if t else UNK_TYPE_ANNOT)
return extracted_var_names
elif "names" in extracted_var_names:
return {
'names':
self.__extract_names_multi_assign(
list(extracted_var_names['names']))
}
else:
return extracted_var_names
def __extract_assign_newtype(self, node: cst.Assign):
"""
Attempts extracting a NewType declaration from the provided Assign node.
If the Assign node corresponds to a NewType assignment, the NewType name is
added to the class definitions of the Visitor.
"""
# Define matcher to extract NewType assignment
matcher_newtype = match.Assign(
targets=[ # Check the assign targets
match.AssignTarget( # There should only be one target
target=match.Name( # Check target name
value=match.SaveMatchedNode( # Save target name
match.MatchRegex(
r'(.)+'), # Match any string literal
"type")))
],
value=match.Call( # We are examining a function call
func=match.Name( # Function must have a name
value="NewType" # Name must be 'NewType'
),
args=[
match.Arg( # Check first argument
value=match.SimpleString(
) # First argument must be the name for the type
),
match.ZeroOrMore(
) # We allow any number of arguments after by def. of NewType
]))
extracted_type = match.extract(node, matcher_newtype)
if extracted_type is not None:
# Append the additional type to the list
# TODO: Either rename class defs, or create new list for additional types
self.class_defs.append(extracted_type["type"].strip("\'"))
def visit_ClassDef(self, node: cst.ClassDef) -> None:
doc_string = node.get_docstring()
if not doc_string or "@sorted-attributes" not in doc_string:
return
found_any_assign: bool = False
pre_assign_lines: List[LineType] = []
assign_lines: List[LineType] = []
post_assign_lines: List[LineType] = []
def _add_unmatched_line(line: LineType) -> None:
post_assign_lines.append(
line) if found_any_assign else pre_assign_lines.append(line)
for line in node.body.body:
if m.matches(
line,
m.SimpleStatementLine(
body=[m.Assign(targets=[m.AssignTarget()])])):
found_any_assign = True
assign_lines.append(line)
else:
_add_unmatched_line(line)
continue
sorted_assign_lines = sorted(
assign_lines,
key=lambda line: line.body[0].targets[0].target.value)
if sorted_assign_lines == assign_lines:
return
self.report(
node,
replacement=node.with_changes(body=node.body.with_changes(
body=pre_assign_lines + sorted_assign_lines +
post_assign_lines)),
)
def _is_property_fset(self, assgn):
return m.matches(assgn,
m.Assign(targets=[m.AssignTarget(m.Attribute())]))
def obf_universal(self, node: cst.CSTNode, *types):
if m.matches(node, m.Name()):
types = ('a', 'ca', 'v', 'cv') if not types else types
node = cst.ensure_type(node, cst.Name)
if self.can_rename(node.value, *types):
node = self.get_new_cst_name(node)
elif m.matches(node, m.NameItem()):
node = cst.ensure_type(node, cst.NameItem)
node = node.with_changes(name=self.obf_universal(node.name))
elif m.matches(node, m.Call()):
node = cst.ensure_type(node, cst.Call)
if self.change_methods or self.change_functions:
node = self.new_obf_function_name(node)
if self.change_arguments or self.change_method_arguments:
node = self.obf_function_args(node)
elif m.matches(node, m.Attribute()):
node = cst.ensure_type(node, cst.Attribute)
value = node.value
attr = node.attr
self.obf_universal(value)
self.obf_universal(attr)
elif m.matches(node, m.AssignTarget()):
node = cst.ensure_type(node, cst.AssignTarget)
node = node.with_changes(target=self.obf_universal(node.target))
elif m.matches(node, m.List() | m.Tuple()):
node = cst.ensure_type(node, cst.List) if m.matches(
node, m.List()) else cst.ensure_type(node, cst.Tuple)
new_elements = []
for el in node.elements:
new_elements.append(self.obf_universal(el))
node = node.with_changes(elements=new_elements)
elif m.matches(node, m.Subscript()):
node = cst.ensure_type(node, cst.Subscript)
new_slice = []
for el in node.slice:
new_slice.append(
el.with_changes(slice=self.obf_slice(el.slice)))
node = node.with_changes(slice=new_slice)
node = node.with_changes(value=self.obf_universal(node.value))
elif m.matches(node, m.Element()):
node = cst.ensure_type(node, cst.Element)
node = node.with_changes(value=self.obf_universal(node.value))
elif m.matches(node, m.Dict()):
node = cst.ensure_type(node, cst.Dict)
new_elements = []
for el in node.elements:
new_elements.append(self.obf_universal(el))
node = node.with_changes(elements=new_elements)
elif m.matches(node, m.DictElement()):
node = cst.ensure_type(node, cst.DictElement)
new_key = self.obf_universal(node.key)
new_val = self.obf_universal(node.value)
node = node.with_changes(key=new_key, value=new_val)
elif m.matches(node, m.StarredDictElement()):
node = cst.ensure_type(node, cst.StarredDictElement)
node = node.with_changes(value=self.obf_universal(node.value))
elif m.matches(node, m.If() | m.While()):
node = cst.ensure_type(node, cst.IfExp) if m.matches(
node, cst.If
| cst.IfExp) else cst.ensure_type(node, cst.While)
node = node.with_changes(test=self.obf_universal(node.test))
elif m.matches(node, m.IfExp()):
node = cst.ensure_type(node, cst.IfExp)
node = node.with_changes(body=self.obf_universal(node.body))
node = node.with_changes(test=self.obf_universal(node.test))
node = node.with_changes(orelse=self.obf_universal(node.orelse))
elif m.matches(node, m.Comparison()):
node = cst.ensure_type(node, cst.Comparison)
new_compars = []
for target in node.comparisons:
new_compars.append(self.obf_universal(target))
node = node.with_changes(left=self.obf_universal(node.left))
node = node.with_changes(comparisons=new_compars)
elif m.matches(node, m.ComparisonTarget()):
node = cst.ensure_type(node, cst.ComparisonTarget)
node = node.with_changes(
comparator=self.obf_universal(node.comparator))
elif m.matches(node, m.FormattedString()):
node = cst.ensure_type(node, cst.FormattedString)
new_parts = []
for part in node.parts:
new_parts.append(self.obf_universal(part))
node = node.with_changes(parts=new_parts)
elif m.matches(node, m.FormattedStringExpression()):
node = cst.ensure_type(node, cst.FormattedStringExpression)
node = node.with_changes(
expression=self.obf_universal(node.expression))
elif m.matches(node, m.BinaryOperation() | m.BooleanOperation()):
node = cst.ensure_type(node, cst.BinaryOperation) if m.matches(
node, m.BinaryOperation()) else cst.ensure_type(
node, cst.BooleanOperation)
node = node.with_changes(left=self.obf_universal(node.left),
right=self.obf_universal(node.right))
elif m.matches(node, m.UnaryOperation()):
node = cst.ensure_type(node, cst.UnaryOperation)
node = node.with_changes(
expression=self.obf_universal(node.expression))
elif m.matches(node, m.ListComp()):
node = cst.ensure_type(node, cst.ListComp)
node = node.with_changes(elt=self.obf_universal(node.elt))
node = node.with_changes(for_in=self.obf_universal(node.for_in))
elif m.matches(node, m.DictComp()):
node = cst.ensure_type(node, cst.DictComp)
node = node.with_changes(key=self.obf_universal(node.key))
node = node.with_changes(value=self.obf_universal(node.value))
node = node.with_changes(for_in=self.obf_universal(node.for_in))
elif m.matches(node, m.CompFor()):
node = cst.ensure_type(node, cst.CompFor)
new_ifs = []
node = node.with_changes(target=self.obf_universal(node.target))
node = node.with_changes(iter=self.obf_universal(node.iter))
for el in node.ifs:
new_ifs.append(self.obf_universal(el))
node = node.with_changes(ifs=new_ifs)
elif m.matches(node, m.CompIf()):
node = cst.ensure_type(node, cst.CompIf)
node = node.with_changes(test=self.obf_universal(node.test))
elif m.matches(node, m.Integer() | m.Float() | m.SimpleString()):
pass
else:
pass
# print(node)
return node
import libcst as cst
import libcst.matchers as m
from libcst.metadata import ScopeProvider, ParentNodeProvider
import inspect
from ..tracer import TracerArgs
from ..common import SEP, parse_expr, a2s, EvalException
from .base_pass import BasePass
obj_new_pattern = m.Assign(
targets=[m.AssignTarget(m.Name())],
value=m.Call(func=m.Attribute(value=m.Name(), attr=m.Name("__new__"))))
class FindSafeObjsToConvert(cst.CSTVisitor):
METADATA_DEPENDENCIES = (ScopeProvider, ParentNodeProvider)
def __init__(self, pass_):
self.pass_ = pass_
self.whitelist = set()
self.blacklist = set()
def visit_Assign(self, node):
if m.matches(node, obj_new_pattern):
name = node.targets[0].target.value
if name in self.pass_.globls:
obj = self.pass_.globls[name]
scope = self.get_metadata(ScopeProvider, node)
for access in scope.accesses[name]:
parent = self.get_metadata(ParentNodeProvider, access.node)
def leave_Assign(self, original_node, updated_node):
if m.matches(original_node,
m.Assign(targets=[m.AssignTarget(m.Name())])):
if self.unused_vars[original_node] and is_pure(updated_node.value):
return cst.RemoveFromParent()
return updated_node
