def visit_Assign(self, node) -> None: if (m.matches(node, m.Assign(targets=[m.AssignTarget(m.Name())])) and self.toplevel == 0): name = node.targets[0].target self.imprts[name.value] = cst.ImportFrom( module=parse_expr(self.mod), names=[cst.ImportAlias(name=name, asname=None)])
class TestVisitor(MatcherDecoratableTransformer): @leave(m.AssignTarget()) def _string_visit( self, original_node: cst.AssignTarget, updated_node: cst.AssignTarget ) -> Union[cst.AssignTarget, cst.RemovalSentinel]: return updated_node
def visit_AssignTarget(self, node: cst.AssignTarget) -> Optional[bool]: if m.matches(node, m.AssignTarget(target=m.Name())): target = cst.ensure_type(node.target, cst.Name) if target.value == "__all__": self.already_exists = True else: self.process_node(node.target, target.value) return None
def visit_AssignTarget(self, node: cst.AssignTarget): if match.matches( node, match.AssignTarget(target=match.Name( value=match.DoNotCare()))): if self.last_visited_assign_t_name == node.target.value: self.last_visited_assign_t_count += 1 elif self.last_visited_assign_t_count == 0: self.last_visited_assign_t_count = 1 else: self.last_visited_assign_t_count = 1 self.last_visited_assign_t_name = node.target.value
def __get_var_names_counter(self, node, scope): vars_name = match.extractall( node, match.OneOf( match.AssignTarget(target=match.SaveMatchedNode( match.Name(value=match.DoNotCare()), "name")), match.AnnAssign(target=match.SaveMatchedNode( match.Name(value=match.DoNotCare()), "name")))) return Counter([ n['name'].value for n in vars_name if isinstance( self.get_metadata(cst.metadata.ScopeProvider, n['name']), scope) ])
def visit_Assign(self, node: cst.Assign) -> None: d = m.extract( node, m.Assign( targets=(m.AssignTarget(target=m.Name("CARBON_EXTS")), ), value=m.SaveMatchedNode(m.List(), "list"), ), ) if d: assert isinstance(d["list"], cst.List) for item in d["list"].elements: if isinstance(item.value, cst.SimpleString): self.extension_names.append(item.value.evaluated_value)
def leave_SimpleStatementLine(self, original_node: cst.SimpleStatementLine, updated_node: cst.SimpleStatementLine): if match.matches( original_node, match.SimpleStatementLine(body=[ match.Assign(targets=[ match.AssignTarget(target=match.Name( value=match.DoNotCare())) ]) ])): t = self.__get_var_type_assign_t( original_node.body[0].targets[0].target.value) if t is not None: t_annot_node_resolved = self.resolve_type_alias(t) t_annot_node = self.__name2annotation(t_annot_node_resolved) if t_annot_node is not None: self.all_applied_types.add( (t_annot_node_resolved, t_annot_node)) return updated_node.with_changes(body=[ cst.AnnAssign( target=original_node.body[0].targets[0].target, value=original_node.body[0].value, annotation=t_annot_node, equal=cst.AssignEqual( whitespace_after=original_node.body[0]. targets[0].whitespace_after_equal, whitespace_before=original_node.body[0]. targets[0].whitespace_before_equal)) ]) elif match.matches( original_node, match.SimpleStatementLine(body=[ match.AnnAssign(target=match.Name(value=match.DoNotCare())) ])): t = self.__get_var_type_an_assign( original_node.body[0].target.value) if t is not None: t_annot_node_resolved = self.resolve_type_alias(t) t_annot_node = self.__name2annotation(t_annot_node_resolved) if t_annot_node is not None: self.all_applied_types.add( (t_annot_node_resolved, t_annot_node)) return updated_node.with_changes(body=[ cst.AnnAssign(target=original_node.body[0].target, value=original_node.body[0].value, annotation=t_annot_node, equal=original_node.body[0].equal) ]) return original_node
def _split_module( self, orig_module: libcst.Module, updated_module: libcst.Module ) -> Tuple[List[Union[libcst.SimpleStatementLine, libcst.BaseCompoundStatement]], List[Union[libcst.SimpleStatementLine, libcst.BaseCompoundStatement]], List[Union[ libcst.SimpleStatementLine, libcst.BaseCompoundStatement]], ]: statement_before_import_location = 0 import_add_location = 0 # never insert an import before initial __strict__ flag if m.matches( orig_module, m.Module(body=[ m.SimpleStatementLine(body=[ m.Assign(targets=[ m.AssignTarget(target=m.Name("__strict__")) ]) ]), m.ZeroOrMore(), ]), ): statement_before_import_location = import_add_location = 1 # This works under the principle that while we might modify node contents, # we have yet to modify the number of statements. So we can match on the # original tree but break up the statements of the modified tree. If we # change this assumption in this visitor, we will have to change this code. for i, statement in enumerate(orig_module.body): if m.matches( statement, m.SimpleStatementLine( body=[m.Expr(value=m.SimpleString())])): statement_before_import_location = import_add_location = 1 elif isinstance(statement, libcst.SimpleStatementLine): for possible_import in statement.body: for last_import in self.all_imports: if possible_import is last_import: import_add_location = i + 1 break return ( list(updated_module.body[:statement_before_import_location]), list(updated_module. body[statement_before_import_location:import_add_location]), list(updated_module.body[import_add_location:]), )
def visit_Assign(self, node: cst.Assign) -> None: metadata: Optional[Collection[QualifiedName]] = self.get_metadata( QualifiedNameProvider, node.value, None) if metadata is not None: for qualname in metadata: # If the assignment is done with some objects from the typing or # collections module, then we will skip the check as the assignment # could be a type alias or the variable could be a class made using # ``collections.namedtuple``. if qualname.name.startswith(("typing", "collections")): return None for target_node in node.targets: if m.matches(target_node, m.AssignTarget(target=m.Name())): nodename = cst.ensure_type(target_node.target, cst.Name).value self._validate_nodename(node, nodename, NamingConvention.SNAKE_CASE)
def leave_Assign(self, original_node, updated_node): if any([ m.matches( updated_node, m.Assign(targets=[m.AssignTarget(m.Name())], value=pattern)) for pattern in self.rhs_patterns ]): var = original_node.targets[0].target scope = self.get_metadata(ScopeProvider, var) children = self._scope_children[scope] if len(scope.assignments[var]) == 1: valid_scopes = [scope] + [ child for child in children if len(child.assignments[var]) == 0 ] self.propagate(valid_scopes, var, updated_node.value) return cst.RemoveFromParent() return updated_node
class VersionTransformer(m.MatcherDecoratableTransformer): new_version: Union[None, Version] = None def __init__(self, version_mod: Callable[[Version], Version]): super().__init__() self.version_mod = version_mod @m.call_if_inside( m.Assign( targets=[m.AssignTarget(target=m.Name("__version__"))], value=m.SimpleString(), )) @m.leave(m.SimpleString()) def update_version(self, original_node: cst.SimpleString, updated_node: cst.SimpleString) -> cst.SimpleString: if self.new_version: raise Exception("Multiple versions found.") old_version = Version(updated_node.evaluated_value) self.new_version = self.version_mod(old_version) return updated_node.with_changes(value=f'"{self.new_version}"')
def __extract_variable_name(self, node: cst.AssignTarget): extracted_var_names = match.extract( node, match.AssignTarget( # Assignment operator target=match.OneOf( # Two cases exist match.Name( # Single target value=match.SaveMatchedNode( # Save result match.MatchRegex( r'(.)+'), # Match any string literal "name")), match.Tuple( # Multi-target elements=match.SaveMatchedNode( # Save result match.DoNotCare(), # Type of list "names")), # This extracts variables inside __init__ without type annotation (e.g. self.x=2) match.Attribute( value=match.Name(value=match.SaveMatchedNode( match.MatchRegex(r'(.)+'), "obj_name" # Object name )), attr=match.Name( match.SaveMatchedNode(match.MatchRegex(r'(.)+'), "name")), )))) if extracted_var_names is not None: if "name" in extracted_var_names: t = self.__get_type_from_metadata(node.target) extracted_var_names['type'] = (t, INF_TYPE_ANNOT if t else UNK_TYPE_ANNOT) return extracted_var_names elif "names" in extracted_var_names: return { 'names': self.__extract_names_multi_assign( list(extracted_var_names['names'])) } else: return extracted_var_names
def __extract_assign_newtype(self, node: cst.Assign): """ Attempts extracting a NewType declaration from the provided Assign node. If the Assign node corresponds to a NewType assignment, the NewType name is added to the class definitions of the Visitor. """ # Define matcher to extract NewType assignment matcher_newtype = match.Assign( targets=[ # Check the assign targets match.AssignTarget( # There should only be one target target=match.Name( # Check target name value=match.SaveMatchedNode( # Save target name match.MatchRegex( r'(.)+'), # Match any string literal "type"))) ], value=match.Call( # We are examining a function call func=match.Name( # Function must have a name value="NewType" # Name must be 'NewType' ), args=[ match.Arg( # Check first argument value=match.SimpleString( ) # First argument must be the name for the type ), match.ZeroOrMore( ) # We allow any number of arguments after by def. of NewType ])) extracted_type = match.extract(node, matcher_newtype) if extracted_type is not None: # Append the additional type to the list # TODO: Either rename class defs, or create new list for additional types self.class_defs.append(extracted_type["type"].strip("\'"))
def visit_ClassDef(self, node: cst.ClassDef) -> None: doc_string = node.get_docstring() if not doc_string or "@sorted-attributes" not in doc_string: return found_any_assign: bool = False pre_assign_lines: List[LineType] = [] assign_lines: List[LineType] = [] post_assign_lines: List[LineType] = [] def _add_unmatched_line(line: LineType) -> None: post_assign_lines.append( line) if found_any_assign else pre_assign_lines.append(line) for line in node.body.body: if m.matches( line, m.SimpleStatementLine( body=[m.Assign(targets=[m.AssignTarget()])])): found_any_assign = True assign_lines.append(line) else: _add_unmatched_line(line) continue sorted_assign_lines = sorted( assign_lines, key=lambda line: line.body[0].targets[0].target.value) if sorted_assign_lines == assign_lines: return self.report( node, replacement=node.with_changes(body=node.body.with_changes( body=pre_assign_lines + sorted_assign_lines + post_assign_lines)), )
def _is_property_fset(self, assgn): return m.matches(assgn, m.Assign(targets=[m.AssignTarget(m.Attribute())]))
def obf_universal(self, node: cst.CSTNode, *types): if m.matches(node, m.Name()): types = ('a', 'ca', 'v', 'cv') if not types else types node = cst.ensure_type(node, cst.Name) if self.can_rename(node.value, *types): node = self.get_new_cst_name(node) elif m.matches(node, m.NameItem()): node = cst.ensure_type(node, cst.NameItem) node = node.with_changes(name=self.obf_universal(node.name)) elif m.matches(node, m.Call()): node = cst.ensure_type(node, cst.Call) if self.change_methods or self.change_functions: node = self.new_obf_function_name(node) if self.change_arguments or self.change_method_arguments: node = self.obf_function_args(node) elif m.matches(node, m.Attribute()): node = cst.ensure_type(node, cst.Attribute) value = node.value attr = node.attr self.obf_universal(value) self.obf_universal(attr) elif m.matches(node, m.AssignTarget()): node = cst.ensure_type(node, cst.AssignTarget) node = node.with_changes(target=self.obf_universal(node.target)) elif m.matches(node, m.List() | m.Tuple()): node = cst.ensure_type(node, cst.List) if m.matches( node, m.List()) else cst.ensure_type(node, cst.Tuple) new_elements = [] for el in node.elements: new_elements.append(self.obf_universal(el)) node = node.with_changes(elements=new_elements) elif m.matches(node, m.Subscript()): node = cst.ensure_type(node, cst.Subscript) new_slice = [] for el in node.slice: new_slice.append( el.with_changes(slice=self.obf_slice(el.slice))) node = node.with_changes(slice=new_slice) node = node.with_changes(value=self.obf_universal(node.value)) elif m.matches(node, m.Element()): node = cst.ensure_type(node, cst.Element) node = node.with_changes(value=self.obf_universal(node.value)) elif m.matches(node, m.Dict()): node = cst.ensure_type(node, cst.Dict) new_elements = [] for el in node.elements: new_elements.append(self.obf_universal(el)) node = node.with_changes(elements=new_elements) elif m.matches(node, m.DictElement()): node = cst.ensure_type(node, cst.DictElement) new_key = self.obf_universal(node.key) new_val = self.obf_universal(node.value) node = node.with_changes(key=new_key, value=new_val) elif m.matches(node, m.StarredDictElement()): node = cst.ensure_type(node, cst.StarredDictElement) node = node.with_changes(value=self.obf_universal(node.value)) elif m.matches(node, m.If() | m.While()): node = cst.ensure_type(node, cst.IfExp) if m.matches( node, cst.If | cst.IfExp) else cst.ensure_type(node, cst.While) node = node.with_changes(test=self.obf_universal(node.test)) elif m.matches(node, m.IfExp()): node = cst.ensure_type(node, cst.IfExp) node = node.with_changes(body=self.obf_universal(node.body)) node = node.with_changes(test=self.obf_universal(node.test)) node = node.with_changes(orelse=self.obf_universal(node.orelse)) elif m.matches(node, m.Comparison()): node = cst.ensure_type(node, cst.Comparison) new_compars = [] for target in node.comparisons: new_compars.append(self.obf_universal(target)) node = node.with_changes(left=self.obf_universal(node.left)) node = node.with_changes(comparisons=new_compars) elif m.matches(node, m.ComparisonTarget()): node = cst.ensure_type(node, cst.ComparisonTarget) node = node.with_changes( comparator=self.obf_universal(node.comparator)) elif m.matches(node, m.FormattedString()): node = cst.ensure_type(node, cst.FormattedString) new_parts = [] for part in node.parts: new_parts.append(self.obf_universal(part)) node = node.with_changes(parts=new_parts) elif m.matches(node, m.FormattedStringExpression()): node = cst.ensure_type(node, cst.FormattedStringExpression) node = node.with_changes( expression=self.obf_universal(node.expression)) elif m.matches(node, m.BinaryOperation() | m.BooleanOperation()): node = cst.ensure_type(node, cst.BinaryOperation) if m.matches( node, m.BinaryOperation()) else cst.ensure_type( node, cst.BooleanOperation) node = node.with_changes(left=self.obf_universal(node.left), right=self.obf_universal(node.right)) elif m.matches(node, m.UnaryOperation()): node = cst.ensure_type(node, cst.UnaryOperation) node = node.with_changes( expression=self.obf_universal(node.expression)) elif m.matches(node, m.ListComp()): node = cst.ensure_type(node, cst.ListComp) node = node.with_changes(elt=self.obf_universal(node.elt)) node = node.with_changes(for_in=self.obf_universal(node.for_in)) elif m.matches(node, m.DictComp()): node = cst.ensure_type(node, cst.DictComp) node = node.with_changes(key=self.obf_universal(node.key)) node = node.with_changes(value=self.obf_universal(node.value)) node = node.with_changes(for_in=self.obf_universal(node.for_in)) elif m.matches(node, m.CompFor()): node = cst.ensure_type(node, cst.CompFor) new_ifs = [] node = node.with_changes(target=self.obf_universal(node.target)) node = node.with_changes(iter=self.obf_universal(node.iter)) for el in node.ifs: new_ifs.append(self.obf_universal(el)) node = node.with_changes(ifs=new_ifs) elif m.matches(node, m.CompIf()): node = cst.ensure_type(node, cst.CompIf) node = node.with_changes(test=self.obf_universal(node.test)) elif m.matches(node, m.Integer() | m.Float() | m.SimpleString()): pass else: pass # print(node) return node
import libcst as cst import libcst.matchers as m from libcst.metadata import ScopeProvider, ParentNodeProvider import inspect from ..tracer import TracerArgs from ..common import SEP, parse_expr, a2s, EvalException from .base_pass import BasePass obj_new_pattern = m.Assign( targets=[m.AssignTarget(m.Name())], value=m.Call(func=m.Attribute(value=m.Name(), attr=m.Name("__new__")))) class FindSafeObjsToConvert(cst.CSTVisitor): METADATA_DEPENDENCIES = (ScopeProvider, ParentNodeProvider) def __init__(self, pass_): self.pass_ = pass_ self.whitelist = set() self.blacklist = set() def visit_Assign(self, node): if m.matches(node, obj_new_pattern): name = node.targets[0].target.value if name in self.pass_.globls: obj = self.pass_.globls[name] scope = self.get_metadata(ScopeProvider, node) for access in scope.accesses[name]: parent = self.get_metadata(ParentNodeProvider, access.node)
def leave_Assign(self, original_node, updated_node): if m.matches(original_node, m.Assign(targets=[m.AssignTarget(m.Name())])): if self.unused_vars[original_node] and is_pure(updated_node.value): return cst.RemoveFromParent() return updated_node