def GetCodeTree(code): ''' Obtain the lib2to3 node from the given code (unicode). * Handles the necessary of EOL at the end of the code. :param unicode code: Python code (multiline) :return: Parsed lib2to3 syntax tree. ''' from terraformer import TerraFormer code += '\n' # Append EOL so ParseString works try: result = TerraFormer._Parse(code) except Exception, e: Reraise(e, 'While parsing code::\n%s' % code)
def testSymbolVisitor(): def PrintScopes(scopes): result = [] for i in sorted(scopes, key=lambda x:x.name): result.append('- %s' % i) for j in sorted(i.uses): result.append(' - %s' % j) return '\n'.join(result) source_code = Dedent( ''' from alpha import Alpha import coilib50 class Zulu(Alpha): """ Zulu class docs. """ def __init__(self, name): """ Zulu.__init__ docs. """ self._name = name alpha = bravo coilib50.Charlie() f = coilib50.Delta(echo, foxtrot) ''' ) from terraformer._visitor import ASTVisitor code = TerraFormer._Parse(source_code) visitor = ASTVisitor() visitor.Visit(code) assert visitor._module.AsString() == Dedent( ''' module (1, 0) module IMPORT-BLOCK (1, 0) import-block #0 IMPORT-FROM (0, 0) alpha IMPORT (1, 0) alpha.Alpha IMPORT (2, 0) coilib50 USE (3, 0) Alpha class (0, 0) Zulu def (8, 4) __init__ ARG (9, 17) self ARG (9, 23) name DEF (13, 8) self._name USE (13, 21) name DEF (14, 8) alpha USE (14, 16) bravo USE (15, 8) coilib50.Charlie DEF (16, 8) f USE (16, 12) coilib50.Delta USE (16, 27) echo USE (16, 33) foxtrot ''' ) # Compares the results with the one given by compiler.symbols.SymbolVisitor, our inspiration. from compiler.symbols import SymbolVisitor from compiler.transformer import parse from compiler.visitor import walk code = parse(source_code) symbol_visitor = SymbolVisitor() walk(code, symbol_visitor) assert PrintScopes(symbol_visitor.scopes.values()) == Dedent( ''' - <ClassScope: Zulu> - <FunctionScope: __init__> - bravo - coilib50 - echo - foxtrot - name - self - <ModuleScope: global> - Alpha ''' )
def testParse(): from lib2to3.pgen2.parse import ParseError with pytest.raises(ParseError): TerraFormer._Parse('class Class:\n')
def _ConvertToPytestImpl(self, source_code, refactor={}): from ._lib2to3 import MyRefactoringTool from lib2to3.fixer_base import BaseFix from terraformer import TerraFormer class ConvertPyTestFix(BaseFix): refactorings = [ dict( find='self.assertEqual(*)', replace='assert $1 == $2', ), dict( find='self.assertTrue(*)', replace='assert $1 == True', ), dict( find='self.assertRaises(*)', replace='with pytest.raises($1):\n $2($3+)', ), dict( find='self.GetDataFilename(*)', replace='embed_data[$1]', func_params=['embed_data'], pytest_plugin=['coilib50._pytest.fixtures'], ), dict( find='self.GetDataDirectory(*)', replace='embed_data.GetDataDirectory($0)', func_params=['embed_data'], pytest_plugin=['coilib50._pytest.fixtures'], ), ] CHANGES = { 'self.assertEqual' : dict(assert_='=='), 'self.assertEquals' : dict(assert_='=='), 'self.assertNotEqual' : dict(assert_='!='), 'self.assertSetEqual' : dict(assert_='==', wrapper='set(%s)'), 'self.assertIsSame' : dict(assert_='is'), 'self.assertIn' : dict(assert_='in'), 'self.assertTrue' : dict(assert_=None, wrapper='%s == True'), 'self.assertFalse' : dict(assert_=None, wrapper='%s == False'), 'self.assert_' : dict(assert_=None, wrapper='%s'), 'self.assertRaises' : dict(replace='with pytest.raises(arg1):\n arg2(args3)'), 'self.ExecuteCommandLineTestsFromFile' : dict(replace='command_line_executer.ExecuteCommandLineTestsFromFile(args1)'), 'self.GetDataDirectory' : dict(replace='embed_data.GetDataDirectory()', fixtures=['embed_data']), 'self.GetDataFilename' : dict(replace='embed_data[arg1]', fixtures=['embed_data']), } def GetArgNodes(self, nodes): for i_node in nodes: if i_node.type == 12 and i_node.value in (',',): continue yield i_node def start_tree(self, tree, filename): self.fixtures = set() return BaseFix.start_tree(self, tree, filename) def match(self, node): # Identify the Test class if node.type == self.syms.classdef and node.children[1].value == 'Test': return self.syms.classdef # Identify the test function if node.type == self.syms.funcdef and node.children[1].value.startswith("test"): return self.syms.funcdef # Identify "self.XXX" calls... # For now, "self.XXX" is is the only kind of replacements we do if node.type == self.syms.power and node.children[0].value == 'self': funccall = '%s.%s' % (node.children[0].value, node.children[1].children[1]) changes = self.CHANGES.get(funccall) if changes is None: return False args_node = node.children[2].children[1] args_node2 = deepcopy(node.children[2].children[1]) args_node2.remove() if len(node.children[2].children) < 3: args = [] elif args_node.type == self.syms.arglist: args = [i for i in self.GetArgNodes(args_node.children)] else: args = [args_node] for i in args: i.remove() for i in changes.get('fixtures', []): self.fixtures.add(i) return dict( args_node=args_node2, args=args, changes=changes, ) def transform(self, node, results): def FindPrefix(node): def FindIndentation(text): text = text.lstrip('\n') count = len(text) - len(text.lstrip(' ')) return ' ' * count result = node.prefix if result == '': # Handles the indentation for the first statement of a function, that is not # in the first statement node prefix, but in a leaf node of type "5" # parent: Finds the 'suite' parent, a function definition. parent = node.parent while parent is not None: if parent.type == self.syms.suite: break parent = parent.parent else: parent = None if parent: # indent_leaf: finds the leaf containing the indentation. for indent_leaf in parent.children: if indent_leaf.type == 5: break else: indent_leaf = None # result: Finally, obtain the indentation from type-5 leaf value. if indent_leaf: result = indent_leaf.value return FindIndentation(result) # Handle Test class declaration: # * Removes any derived class if results == self.syms.classdef: process = False for i_child in node.children[:]: if process: if i_child.type < 256 and i_child.value == ')': i_child.remove() break i_child.remove() continue if i_child.type < 256 and i_child.value == '(': i_child.remove() process = True continue return # Handle test functions # * Adds any fixtures requested by pytest conversion. # * Note that funcdef is matched AFTER all internal code is matched, so this is # called AFTER all method calls were processed. if results == self.syms.funcdef: # TODO: BEN-74: [refactor.to_pytest] Handle the addition of needed fixtures # args = [i.value for i in WalkLeafs(node.children[2])] return next_line_prefix = FindPrefix(node) # dd: Create a replacement dict from the original information from the original # code. dd = {} dd['args_node'] = results['args_node'] dd.update(results['changes']) wrapper = results['changes'].get('wrapper', '%s') multiline = False for i, i_arg in enumerate(results['args']): multiline = multiline or '\n' in i_arg.prefix i_arg.prefix = '' dd['arg%d' % (i + 1)] = i_arg dd['p%d' % (i + 1)] = wrapper % i_arg nodes = deepcopy(results['args_node']) nodes.children = nodes.children[i * 2:] if nodes.children: nodes.children[0].prefix = '' dd['args%d' % (i + 1)] = nodes # new_code: A string containing the new replacement code. if 'assert_' in dd: if dd['assert_'] is None: new_code = 'assert %(p1)s' % dd else: new_code = 'assert %(p1)s %(assert_)s %(p2)s' % dd if multiline: new_code = 'assert (\n %(p1)s\n %(assert_)s %(p2)s\n)' % dd elif 'wrapper' in dd: if 'p1' in dd: new_code = dd['p1'] else: new_code = dd['wrapper'] % '' elif 'replace' in dd: new_code = dd['replace'] # new_node: A new lib2to3 node representing the new-code new_node = TerraForming.GetCodeTree(new_code) if 'replace' in dd: for i_leaf in TerraFormer.WalkLeafs(new_node): if i_leaf.value in ('arg1', 'arg2', 'arg3'): node_ = copy(dd[i_leaf.value]) node_.prefix = i_leaf.prefix i_leaf.replace(node_) if i_leaf.value in ('args1', 'args2', 'args3'): new_value = dd.get(i_leaf.value) if new_value is None: # If we don't have ARGS we delete the parent, which olds "()" for # the function call. # CASE: self.assertRaises(E, F) # note that there is no third+ parameter i_leaf.parent.remove() else: node_ = copy(new_value) i_leaf.replace(node_) TerraForming.FixIndentation(new_node, node.prefix, next_lines_prefix=next_line_prefix) if False: print '=' * 80 print unicode(node) print '-' * 80 print "1-PREFIX: '%s'" % node.prefix print "N-PREFIX: '%s'" % next_line_prefix print '-' * 80 print unicode(new_code) print '-' * 80 print unicode(new_node) node.replace(new_node) try: tree = TerraFormer._Parse(source_code) except Exception as exception: Reraise(exception, 'While processing source::\n%s\n---\n' % source_code) options = {'isymbols' : {}} rt = MyRefactoringTool([ConvertPyTestFix], options=options) rt.refactor_tree(tree, 'ConvertPyTestFix') return unicode(tree)