def test_attribute(self): attr = ast.Attribute(ast.Name("x", ast.Store()), "y", ast.Load()) self.expr(attr, "must have Load context")
def helper(self, name: str, *args: ast.expr) -> ast.expr: """Call a helper in this module.""" py_name = ast.Name("@pytest_ar", ast.Load()) attr = ast.Attribute(py_name, name, ast.Load()) return ast.Call(attr, list(args), [])
def _parse_body(self, body, argument_names): """Recursive function transforming a @body to a block expression Return: - AST to append to body (real python statements) - a list of blocks, ie list of affblock, ie list of ExprAff (AST)""" # Init ## Real instructions real_body = [] ## Final blocks blocks = [[[]]] for statement in body: if isinstance(statement, ast.Assign): src = self.transformer.visit(statement.value) dst = self.transformer.visit(statement.targets[0]) if (isinstance(dst, ast.Name) and dst.id not in argument_names and dst.id not in self._ctx and dst.id not in self._local_ctx): # Real variable declaration statement.value = src real_body.append(statement) self._local_ctx[dst.id] = src continue dst.ctx = ast.Load() res = ast.Call(func=ast.Name(id='ExprAff', ctx=ast.Load()), args=[dst, src], keywords=[], starargs=None, kwargs=None) blocks[-1][-1].append(res) elif (isinstance(statement, ast.Expr) and isinstance(statement.value, ast.Str)): # String (docstring, comment, ...) -> keep it real_body.append(statement) elif isinstance(statement, ast.If): # Create jumps : ir.IRDst = loc_if if cond else loc_end # if .. else .. are also handled cond = statement.test real_body += self._create_labels(loc_else=True) loc_end = ast.Name(id='loc_end_expr', ctx=ast.Load()) loc_if = ast.Name(id='loc_if_expr', ctx=ast.Load()) loc_else = ast.Name(id='loc_else_expr', ctx=ast.Load()) \ if statement.orelse else loc_end dst = ast.Call(func=ast.Name(id='ExprCond', ctx=ast.Load()), args=[cond, loc_if, loc_else], keywords=[], starargs=None, kwargs=None) if (isinstance(cond, ast.UnaryOp) and isinstance(cond.op, ast.Not)): ## if not cond -> switch exprCond dst.args[1:] = dst.args[1:][::-1] dst.args[0] = cond.operand IRDst = ast.Attribute(value=ast.Name(id='ir', ctx=ast.Load()), attr='IRDst', ctx=ast.Load()) blocks[-1][-1].append( ast.Call(func=ast.Name(id='ExprAff', ctx=ast.Load()), args=[IRDst, dst], keywords=[], starargs=None, kwargs=None)) # Create the new blocks elements = [(statement.body, 'loc_if')] if statement.orelse: elements.append((statement.orelse, 'loc_else')) for content, loc_name in elements: sub_blocks, sub_body = self._parse_body( content, argument_names) if len(sub_blocks) > 1: raise RuntimeError("Imbricated if unimplemented") ## Close the last block jmp_end = ast.Call(func=ast.Name(id='ExprAff', ctx=ast.Load()), args=[IRDst, loc_end], keywords=[], starargs=None, kwargs=None) sub_blocks[-1][-1].append(jmp_end) instr = ast.Name(id='instr', ctx=ast.Load()) effects = ast.List(elts=sub_blocks[-1][-1], ctx=ast.Load()) assignblk = ast.Call(func=ast.Name(id='AssignBlock', ctx=ast.Load()), args=[effects, instr], keywords=[], starargs=None, kwargs=None) ## Replace the block with a call to 'IRBlock' loc_if_name = ast.Name(id=loc_name, ctx=ast.Load()) assignblks = ast.List(elts=[assignblk], ctx=ast.Load()) sub_blocks[-1] = ast.Call(func=ast.Name(id='IRBlock', ctx=ast.Load()), args=[loc_if_name, assignblks], keywords=[], starargs=None, kwargs=None) blocks += sub_blocks real_body += sub_body # Prepare a new block for following statement blocks.append([[]]) else: # TODO: real var, +=, /=, -=, <<=, >>=, if/else, ... raise RuntimeError("Unimplemented %s" % statement) return blocks, real_body
def rewrite_append(node): return ast.Call( func=ast.Attribute(value=node.func.value, attr='push', ctx=ast.Load()), args=node.args, keywords=node.keywords, )
def _replace(node, b): return ast.Call(ast.Attribute(value=node.args[0], attr=b), node.args[1:], node.keywords)
def visit_Attribute(self, node): 'Make sure to make a new version of the Attribute so it does not get reused' return ast.Attribute(value=self.visit(node.value), attr=node.attr, ctx=ast.Load())
value=ast.Name(id='console', ctx=ast.Load()), attr='log', ctx=ast.Load(), ), args=node.args, keywords=node.keywords, ) @rewrite.register(ast.Call(func=ast.Name(id='len'))) def rewrite_len(node): assert len(node.args) == 1 return ast.Attribute(value=node.args[0], attr='length', ctx=ast.Load()) @rewrite.register(ast.Call(func=ast.Attribute(attr='append'))) def rewrite_append(node): return ast.Call( func=ast.Attribute(value=node.func.value, attr='push', ctx=ast.Load()), args=node.args, keywords=node.keywords, ) @rewrite.register( ast.Call(func=ast.Attribute(value=ast.Name(id='Array'), attr='from_'))) def rewrite_array_from(node): return ast.Call( func=ast.Attribute(value=node.func.value, attr='from'), args=node.args, keywords=node.keywords,
def _get_attr_node(names): """Builds an Attribute node, or a Name node if names has just one entry.""" node = ast.Name(id=names[0], ctx=ast.Load()) for name in names[1:]: node = ast.Attribute(value=node, attr=name, ctx=ast.Load()) return node
def test_ast_not_there(empty_db): db = FuncADLDBAccess(empty_db) r = db.lookup_results(ast.Attribute()) assert r is None
async def meval(code, globs, **kwargs): # Note to self: please don't set globals here as they will be lost. # Don't clutter locals locs = {} # Restore globals later globs = globs.copy() # This code saves __name__ and __package into a kwarg passed to the func. # It is set before the users code runs to make sure relative imports work global_args = "_globs" while global_args in globs.keys(): # Make sure there's no name collision, just keep prepending _s global_args = "_" + global_args kwargs[global_args] = {} for glob in ["__name__", "__package__"]: # Copy data to args we are sending kwargs[global_args][glob] = globs[glob] root = ast.parse(code, "exec") code = root.body ret_name = "_ret" ok = False while True: if ret_name in globs.keys(): ret_name = "_" + ret_name continue for node in ast.walk(root): if isinstance(node, ast.Name) and node.id == ret_name: ret_name = "_" + ret_name break ok = True if ok: break if not code: return None if not any(isinstance(node, ast.Return) for node in code): for i in range(len(code)): if isinstance(code[i], ast.Expr): if (i == len(code) - 1 or not isinstance(code[i].value, ast.Call)): code[i] = ast.copy_location( ast.Expr( ast.Call(func=ast.Attribute(value=ast.Name( id=ret_name, ctx=ast.Load()), attr="append", ctx=ast.Load()), args=[code[i].value], keywords=[])), code[-1]) else: for node in code: if isinstance(node, ast.Return): node.value = ast.List(elts=[node.value], ctx=ast.Load()) code.append( ast.copy_location( ast.Return(value=ast.Name(id=ret_name, ctx=ast.Load())), code[-1])) # globals().update(**<global_args>) glob_copy = ast.Expr( ast.Call( func=ast.Attribute(value=ast.Call(func=ast.Name(id="globals", ctx=ast.Load()), args=[], keywords=[]), attr="update", ctx=ast.Load()), args=[], keywords=[ ast.keyword(arg=None, value=ast.Name(id=global_args, ctx=ast.Load())) ])) ast.fix_missing_locations(glob_copy) code.insert(0, glob_copy) ret_decl = ast.Assign(targets=[ast.Name(id=ret_name, ctx=ast.Store())], value=ast.List(elts=[], ctx=ast.Load())) ast.fix_missing_locations(ret_decl) code.insert(1, ret_decl) args = [] for a in list(map(lambda x: ast.arg(x, None), kwargs.keys())): ast.fix_missing_locations(a) args += [a] args = ast.arguments(args=[], vararg=None, kwonlyargs=args, kwarg=None, defaults=[], kw_defaults=[None for i in range(len(args))]) args.posonlyargs = [] fun = ast.AsyncFunctionDef(name="tmp", args=args, body=code, decorator_list=[], returns=None) ast.fix_missing_locations(fun) mod = ast.parse("") mod.body = [fun] comp = compile(mod, "<string>", "exec") exec(comp, {}, locs) r = await locs["tmp"](**kwargs) for i in range(len(r)): if hasattr(r[i], "__await__"): r[i] = await r[i] # workaround for 3.5 i = 0 while i < len(r) - 1: if r[i] is None: del r[i] else: i += 1 if len(r) == 1: [r] = r elif not r: r = None return r
def visit_FunctionDef(self, node): node.parent = self.fundef self.fundef = node if len( list( filter( lambda n: isinstance(n, ast.Name) and n.id is 'rep_fun', node.decorator_list))) > 0: self.recs.append(node.name) self.generic_visit(node) r_args = {} for arg in node.args.args: arg_name = arg.arg try: if self.fundef.locals[arg_name] > 1: r_args[arg_name] = self.freshName('x') # self.fundef.locals[arg_name] += 1 except KeyError as e: pass # generate code to pre-initialize staged vars # we stage all vars that are written to more than once inits = [ast.Assign(targets=[ast.Name(id=id, ctx=ast.Store())], value=ast.Call( func=ast.Name(id='_var', ctx=ast.Load()), args=[], keywords=[])) \ for id in node.locals if node.locals[id] > 1] a_nodes = [ast.Expr( ast.Call( func=ast.Name(id='_assign', ctx=ast.Load()), args=[ast.Name(id=arg,ctx=ast.Load()), ast.Name(id=r_args[arg],ctx=ast.Load())], keywords=[])) \ for arg in r_args] new_node = ast.FunctionDef( name=node.name, args=ast.arguments(args=[ ast.arg(arg=r_args[arg.arg], annotation=None) if arg.arg in r_args else arg for arg in node.args.args ], vararg=None, kwonlyargs=[], kwarg=None, defaults=[], kw_defaults=[]), # node.args, body=[ ast.Try(body=inits + a_nodes + node.body, handlers=[ ast.ExceptHandler( type=ast.Name(id='NonLocalReturnValue', ctx=ast.Load()), name='r', body=[ ast.Return(value=ast.Attribute( value=ast.Name(id='r', ctx=ast.Load()), attr='value', ctx=ast.Load())) ]) ], orelse=[], finalbody=[]) ], decorator_list=list( filter( lambda n: isinstance(n, ast.Name) and n.id != 'lms' and n. id != 'rep_fun', node.decorator_list))) ast.copy_location(new_node, node) ast.fix_missing_locations(new_node) self.fundef = node.parent return new_node
def to_attribute(value, attr, ctx=None): """ Create an Attribute AST node. """ return ast.Attribute(value, attr, ctx or ast.Load())
def visit_Attribute(self, node): return ast.copy_location( ast.Attribute(value=self.visit(node.value), attr=node.attr.lower()), node)
def visit_Attribute(self, node): return py_ast.Attribute( value=self.visit(node.value), attr=node.attribute, )
def _compile_directive_call_assets(self, el, options): """ This special 't-call' tag can be used in order to aggregate/minify javascript and css assets""" if len(el): raise SyntaxError("t-call-assets cannot contain children nodes") # nodes = self._get_asset(xmlid, options, css=css, js=js, debug=values.get('debug'), async=async, values=values) # # for index, (tagName, t_attrs, content) in enumerate(nodes): # if index: # append('\n ') # append('<') # append(tagName) # # self._post_processing_att(tagName, t_attrs, options) # for name, value in t_attrs.items(): # if value or isinstance(value, string_types)): # append(u' ') # append(name) # append(u'="') # append(escape(pycompat.to_text((value))) # append(u'"') # # if not content and tagName in self._void_elements: # append('/>') # else: # append('>') # if content: # append(content) # append('</') # append(tagName) # append('>') # space = el.getprevious() is not None and el.getprevious( ).tail or el.getparent().text sep = u'\n' + space.rsplit('\n').pop() return [ ast.Assign( targets=[ast.Name(id='nodes', ctx=ast.Store())], value=ast.Call( func=ast.Attribute(value=ast.Name(id='self', ctx=ast.Load()), attr='_get_asset', ctx=ast.Load()), args=[ ast.Str(el.get('t-call-assets')), ast.Name(id='options', ctx=ast.Load()), ], keywords=[ ast.keyword('css', self._get_attr_bool(el.get('t-css', True))), ast.keyword('js', self._get_attr_bool(el.get('t-js', True))), ast.keyword( 'debug', ast.Call(func=ast.Attribute(value=ast.Name( id='values', ctx=ast.Load()), attr='get', ctx=ast.Load()), args=[ast.Str('debug')], keywords=[], starargs=None, kwargs=None)), ast.keyword( 'async', self._get_attr_bool(el.get('async', False))), ast.keyword('values', ast.Name(id='values', ctx=ast.Load())), ], starargs=None, kwargs=None)), ast.For( target=ast.Tuple(elts=[ ast.Name(id='index', ctx=ast.Store()), ast.Tuple(elts=[ ast.Name(id='tagName', ctx=ast.Store()), ast.Name(id='t_attrs', ctx=ast.Store()), ast.Name(id='content', ctx=ast.Store()) ], ctx=ast.Store()) ], ctx=ast.Store()), iter=ast.Call(func=ast.Name(id='enumerate', ctx=ast.Load()), args=[ast.Name(id='nodes', ctx=ast.Load())], keywords=[], starargs=None, kwargs=None), body=[ ast.If(test=ast.Name(id='index', ctx=ast.Load()), body=[self._append(ast.Str(sep))], orelse=[]), self._append(ast.Str(u'<')), self._append(ast.Name(id='tagName', ctx=ast.Load())), ] + self._append_attributes() + [ ast.If(test=ast.BoolOp( op=ast.And(), values=[ ast.UnaryOp(ast.Not(), ast.Name(id='content', ctx=ast.Load()), lineno=0, col_offset=0), ast.Compare( left=ast.Name(id='tagName', ctx=ast.Load()), ops=[ast.In()], comparators=[ ast.Attribute(value=ast.Name( id='self', ctx=ast.Load()), attr='_void_elements', ctx=ast.Load()) ]), ]), body=[self._append(ast.Str(u'/>'))], orelse=[ self._append(ast.Str(u'>')), ast.If(test=ast.Name(id='content', ctx=ast.Load()), body=[ self._append( ast.Name(id='content', ctx=ast.Load())) ], orelse=[]), self._append(ast.Str(u'</')), self._append( ast.Name(id='tagName', ctx=ast.Load())), self._append(ast.Str(u'>')), ]) ], orelse=[]) ]
def visitAttribute(self, n, *args): return ast.Attribute(value=self.dispatch(n.value, *args), attr=n.attr, ctx=n.ctx)
def visit_ref_read(self, read: Expr): ref, defs = self.visit(read.ref) return (ast.Attribute(ref, 'value', Load()), defs)
def mathTransform(self, attribute, args): # operations from the math python module or builtin operations that exist in JavaScript Math: if attribute in self.directMathOperations: func = ast.Attribute( value=ast.Name(id='Math', ctx=ast.Load()), attr=attribute, ctx=ast.Load() ) return ast.Call( func=func, args=args, keywords=[] ) # substitutable operations, e.g. a = sum(b,c) => a = [b,c].reduce( function(x,y) { return x+y: }) elif attribute in self.subtitutableOperations: # a = sum(b,c) => a = [b,c].reduce( function(x,y) { return x+y: }) if attribute == 'sum': func = ast.Attribute( value=ast.List( elts=args, ctx=ast.Load() ), attr='reduce', ctx=ast.Load() ) args = [ ast.Call( func=ast.Name(id='JS', ctx=ast.Load()), args=[ast.Str(s=' function(x,y) { return x+y; }')], keywords=[] ) ] return ast.Call( func=func, args=args, keywords=[] ) # randint(a,b) => Math.floor(Math.random() * (b - a + 1)) + a elif attribute == 'randint': left = ast.Call( func=ast.Attribute( value=ast.Name(id='Math', ctx=ast.Load()), attr='floor', ctx=ast.Load() ), args=[ ast.BinOp( left=ast.Call( func=ast.Attribute( value=ast.Name(id='Math', ctx=ast.Load()), attr='random', ctx=ast.Load() ), args=[], keywords=[] ), op=ast.Mult(), right=ast.BinOp( left=ast.BinOp( left=args[1], op=ast.Sub(), right=args[0] ), op=ast.Add(), right=ast.Num(n=1))) ], keywords=[] ) right = args[0] return ast.BinOp( left=left, op=ast.Add(), right=right ) else: return None
def attr(left, right): return ast.Attribute(value=name(left), attr=right, lineno=1, col_offset=0, ctx=ast.Load())
def visit_FormattedValue(self, node): # unformatted f-strings: if not node.format_spec: return node # formatted f-strings: if isinstance(node.format_spec, ast.JoinedStr) and len(node.format_spec.values) > 0 and isinstance( node.format_spec.values[0], ast.Str): # split the format: format = node.format_spec.values[0].s match = re.search(r"([0-9]*).([0-9]+)(f|i)", format) if not match: raise Exception(format + ' format is not currently supported') matchGroups = match.groups() width = matchGroups[0] width = int(width) if width != '' else 1 precision = matchGroups[1] precision = int(precision) if precision != '' else 1 conversion = matchGroups[2] value = node.value # prepare the conversion: conversionFunc = ast.Call( func=ast.Attribute( value=ast.Name(id='Number', ctx=ast.Load()), attr='parseFloat' if conversion == 'f' else 'parseInt', ctx=ast.Load() ), args=[value], keywords=[] ) # deal with precision: precisionCall = ast.Call( func=ast.Attribute( value=conversionFunc, attr='toPrecision', ctx=ast.Load() ), args=[ast.Num(n=precision)], keywords=[] ) # deal with width: widthCall = ast.Call( func=ast.Name(id='pad', ctx=ast.Load()), args=[precisionCall, ast.Num(n=width)], keywords=[] ) # return the node: node.value = widthCall node.conversion = -1 node.format_spec = None return node raise Exception('formatted f-string are not all supported at the moment')
def rewrite_len(node): assert len(node.args) == 1 return ast.Attribute(value=node.args[0], attr='length', ctx=ast.Load())
def ast_call(node): convert_loss_scale_api(node) if _call_name_match(node.func, "set_experimental_options"): log_msg( getattr(node, 'lineno', 'None'), 'change set_experimental_options(*) to set_experimental_options(experimental_options)' ) node.args = [ast.Name(id='experimental_options', ctx=ast.Load())] node.keywords = [] util_global.set_value('need_conver', True) if isinstance(node.func, ast.Name) and node.func.id == 'check_available_gpus': log_msg(getattr(node, 'lineno', 'None'), "change check_available_gpus() to ['/device:CPU:0']") util_global.set_value('need_conver', True) return ast.List(elts=[ast.Str(s="/device:CPU:0")], ctx=ast.Load()) if (isinstance(node.func, ast.Name) and node.func.id == 'ConfigProto') or \ (isinstance(node.func, ast.Attribute) and node.func.attr == 'ConfigProto'): log_success_report(getattr(node, 'lineno', 'None'), 'ConfigProto()') src = copy.deepcopy(node) node.func = ast.Name(id='npu_config_proto', ctx=ast.Load()) node.args = [] node.keywords = [] node.keywords.append(ast.keyword(arg='config_proto', value=src)) util_global.set_value('need_conver', True) return node if (isinstance(node.func, ast.Name) and node.func.id == 'GraphOptions') or \ (isinstance(node.func, ast.Attribute) and node.func.attr == 'GraphOptions'): log_success_report(getattr(node, 'lineno', 'None'), 'GraphOptions()') src = copy.deepcopy(node) node.func = ast.Name(id='npu_graph_options', ctx=ast.Load()) node.args = [] node.keywords = [] node.keywords.append(ast.keyword(arg='graph_options', value=src)) util_global.set_value('need_conver', True) return node if (isinstance(node.func, ast.Name) and node.func.id == 'OptimizerOptions') or \ (isinstance(node.func, ast.Attribute) and node.func.attr == 'OptimizerOptions'): log_success_report(getattr(node, 'lineno', 'None'), 'OptimizerOptions()') src = copy.deepcopy(node) node.func = ast.Name(id='npu_optimizer_options', ctx=ast.Load()) node.args = [] node.keywords = [] node.keywords.append(ast.keyword(arg='optimizer_options', value=src)) util_global.set_value('need_conver', True) return node if (isinstance(node.func, ast.Name) and node.func.id == 'Session') or \ (isinstance(node.func, ast.Attribute) and node.func.attr == 'Session'): log_success_report(getattr(node, 'lineno', 'None'), 'Session()') config = None for index, _ in enumerate(node.args): if index == 2: config = node.args.pop(2) break for keyword in node.keywords: if keyword.arg == 'config': config = keyword break if config: if isinstance(config, ast.keyword): config.value = ast.Call(func=ast.Name(id='npu_config_proto', ctx=ast.Load()), args=[], keywords=[ ast.keyword(arg='config_proto', value=config.value) ]) else: node.keywords.append( ast.keyword(arg='config', value=ast.Call( func=ast.Name(id='npu_config_proto', ctx=ast.Load()), args=[], keywords=[ ast.keyword(arg='config_proto', value=config) ]))) else: node.keywords.append( ast.keyword(arg='config', value=ast.Call(func=ast.Name(id='npu_config_proto', ctx=ast.Load()), args=[], keywords=[]))) util_global.set_value('need_conver', True) return node if isinstance(node.func, ast.Attribute ) and node.func.attr == "BroadcastGlobalVariablesHook": log_success_report(getattr(node, "lineno", "None"), 'BroadcastGlobalVariablesHook') node.func = ast.Name(id="NpuEmptyHook", ctx=ast.Load()) node.args = [] node.keywords = [] util_global.set_value('need_conver', True) return node if isinstance(node.func, ast.Attribute) and node.func.attr == "DistributedOptimizer": log_success_report(getattr(node, "lineno", "None"), 'DistributedOptimizer') return node.args[0] if isinstance(node.func, ast.Attribute) and node.func.attr == 'shard': log_success_report(getattr(node, "lineno", "None"), 'shard') node.args = [ ast.Call(func=ast.Name(id='get_rank_size', ctx=ast.Load()), args=[], keywords=[]), ast.Call(func=ast.Name(id='get_rank_id', ctx=ast.Load()), args=[], keywords=[]) ] util_global.set_value("has_hccl_api", True) util_global.set_value('need_conver', True) if isinstance(node.func, ast.Attribute) and node.func.attr == 'dropout': if isinstance(node.func.value, ast.Attribute) and node.func.value.attr == 'nn': log_success_report(getattr(node, "lineno", "None"), 'dropout') node.func = ast.Attribute(value=ast.Name(id='npu_ops', ctx=ast.Load()), attr='dropout', ctx=ast.Load()) keywords_new = [] for keyword in node.keywords: if keyword.arg != 'rate': keywords_new.append(keyword) node.keywords = keywords_new util_global.set_value('need_conver', True) if isinstance(node.func, ast.Attribute) and ((node.func.attr == 'map_and_batch') or (node.func.attr == 'batch' \ and (not isinstance(node.func.value, ast.Attribute) or (isinstance(node.func.value, ast.Attribute) and node.func.value.attr != 'train')))): exist = False for keyword in node.keywords: if keyword.arg == 'drop_remainder': exist = True if ((isinstance(keyword.value, ast.NameConstant) and keyword.value.value != True) or (not isinstance(keyword.value, ast.NameConstant))): log_success_report(getattr(node, "lineno", "None"), node.func.attr) keyword.value = pasta.parse('True') util_global.set_value('need_conver', True) if not exist: log_success_report(getattr(node, "lineno", "None"), node.func.attr) keyword = ast.keyword(arg='drop_remainder', value=pasta.parse('True')) node.keywords.insert(0, keyword) util_global.set_value('need_conver', True) if (isinstance(node.func, ast.Attribute) and isinstance(node.func.value, ast.Name) and node.func.value.id == 'tf' and node.func.attr == 'device'): log_success_report(getattr(node, "lineno", "None"), node.func.attr) node.args = [ast.Str(s='/cpu:0')] util_global.set_value('need_conver', True) if isinstance(node.func, ast.Attribute) and ( node.func.attr == "get_distribution_strategy" or node.func.attr == "MirroredStrategy" or node.func.attr == "MultiWorkerMirroredStrategy"): log_success_report(getattr(node, "lineno", "None"), node.func.attr) new_func = ast.Attribute(value=ast.Name(id="npu_strategy", ctx=ast.Load()), attr="NPUStrategy", ctx=ast.Load()) ast.copy_location(new_func, node.func) node.func = new_func node.keywords = [] node.args = [] util_global.set_value('need_conver', True) if isinstance(node.func, ast.Attribute) and (node.func.attr == 'AdviceProto'): log_success_report(getattr(node, 'lineno', 'None'), 'tf.profiler.AdviceProto') util_global.set_value('need_conver', True) node = ast.NameConstant(value=None) return node if isinstance(node.func, ast.Attribute) and (node.func.attr == 'Checker'): log_success_report(getattr(node, 'lineno', 'None'), 'tf.profiler.AdviceProto.Checker') util_global.set_value('need_conver', True) node = ast.NameConstant(value=None) return node if isinstance(node.func, ast.Attribute) and (node.func.attr == 'CheckersEntry'): log_success_report(getattr(node, 'lineno', 'None'), 'tf.profiler.AdviceProto.CheckersEntry') util_global.set_value('need_conver', True) node = ast.NameConstant(value=None) return node if isinstance(node.func, ast.Attribute) and (node.func.attr == 'GraphNodeProto'): log_success_report(getattr(node, 'lineno', 'None'), 'tf.profiler.GraphNodeProto') util_global.set_value('need_conver', True) node = ast.NameConstant(value=None) return node if isinstance(node.func, ast.Attribute) and (node.func.attr == 'InputShapesEntry'): log_success_report(getattr(node, 'lineno', 'None'), 'tf.profiler.GraphNodeProto.InputShapesEntry') util_global.set_value('need_conver', True) node = ast.NameConstant(value=None) return node if isinstance(node.func, ast.Attribute) and (node.func.attr == 'MultiGraphNodeProto'): log_success_report(getattr(node, 'lineno', 'None'), 'tf.profiler.MultiGraphNodeProto') util_global.set_value('need_conver', True) node = ast.NameConstant(value=None) return node if isinstance(node.func, ast.Attribute) and (node.func.attr == 'OpLogProto'): log_success_report(getattr(node, 'lineno', 'None'), 'tf.profiler.OpLogProto') util_global.set_value('need_conver', True) node = ast.NameConstant(value=None) return node if isinstance(node.func, ast.Attribute) and (node.func.attr == 'IdToStringEntry'): log_success_report(getattr(node, 'lineno', 'None'), 'tf.profiler.OpLogProto.IdToStringEntry') util_global.set_value('need_conver', True) node = ast.NameConstant(value=None) return node if isinstance(node.func, ast.Attribute) and (node.func.attr == 'ProfileOptionBuilder'): log_success_report(getattr(node, 'lineno', 'None'), 'tf.profiler.ProfileOptionBuilder') util_global.set_value('need_conver', True) node = ast.NameConstant(value=None) return node if isinstance(node.func, ast.Attribute) and (node.func.attr == 'advise'): log_success_report(getattr(node, 'lineno', 'None'), 'tf.profiler.advise') util_global.set_value('need_conver', True) node = ast.NameConstant(value=None) return node if isinstance(node.func, ast.Attribute) and (node.func.attr == 'profile'): log_success_report(getattr(node, 'lineno', 'None'), 'tf.profiler.profile') util_global.set_value('need_conver', True) node = ast.NameConstant(value=None) return node if isinstance(node.func, ast.Attribute) and (node.func.attr == 'write_op_log'): log_success_report(getattr(node, 'lineno', 'None'), 'tf.profiler.write_op_log') util_global.set_value('need_conver', True) node = ast.NameConstant(value=None) return node if isinstance(node.func, ast.Attribute) and (node.func.attr == 'TPUEstimator') and \ ((isinstance(node.func.value, ast.Attribute) and (node.func.value.attr == 'tpu')) or \ (isinstance(node.func.value, ast.Name) and (node.func.value.id == 'tpu'))): add_eval_on_tpu = True add_use_tpu = True add_export_to_tpu = True for keyword in node.keywords: if (keyword.arg == 'eval_on_tpu') or ( keyword.arg == 'use_tpu') or (keyword.arg == 'export_to_tpu'): if (not isinstance(keyword.value, ast.NameConstant)) or ( isinstance(keyword.value, ast.NameConstant) and (keyword.value.value != False)): log_success_report(getattr(node, 'lineno', 'None'), 'TPUEstimator(' + keyword.arg + '=*)') keyword.value = pasta.parse('False') util_global.set_value('need_conver', True) if add_eval_on_tpu and (keyword.arg == 'eval_on_tpu'): add_eval_on_tpu = False if add_use_tpu and (keyword.arg == 'use_tpu'): add_use_tpu = False if add_export_to_tpu and (keyword.arg == 'export_to_tpu'): add_export_to_tpu = False if add_eval_on_tpu: log_success_report(getattr(node, 'lineno', 'None'), 'TPUEstimator(eval_on_tpu=*)') node.keywords.append( ast.keyword(arg='eval_on_tpu', value=pasta.parse('False'))) util_global.set_value('need_conver', True) if add_use_tpu: log_success_report(getattr(node, 'lineno', 'None'), 'TPUEstimator(use_tpu=*)') node.keywords.append( ast.keyword(arg='use_tpu', value=pasta.parse('False'))) util_global.set_value('need_conver', True) if add_export_to_tpu: log_success_report(getattr(node, 'lineno', 'None'), 'TPUEstimator(export_to_tpu=*)') node.keywords.append( ast.keyword(arg='export_to_tpu', value=pasta.parse('False'))) util_global.set_value('need_conver', True) if isinstance(node.func, ast.Attribute) and (node.func.attr == 'VirtualDeviceConfiguration'): log_success_report(getattr(node, 'lineno', 'None'), 'VirtualDeviceConfiguration') util_global.set_value('need_conver', True) memory_limit = None for keyword in node.keywords: if keyword.arg == 'memory_limit': memory_limit = keyword break if memory_limit: memory_limit.value = ast.NameConstant(value=None) else: node.keywords.append( ast.keyword(arg='memory_limit', value=ast.NameConstant(value=None))) return node if isinstance(node.func, ast.Attribute) and (node.func.attr == 'set_soft_device_placement'): log_success_report(getattr(node, 'lineno', 'None'), 'set_soft_device_placement') util_global.set_value('need_conver', True) node.args = [] node.keywords = [ ast.keyword(arg='enabled', value=ast.NameConstant(value=True)) ] return node if isinstance(node.func, ast.Attribute) and (node.func.attr == 'set_memory_growth'): log_success_report(getattr(node, 'lineno', 'None'), 'set_memory_growth') util_global.set_value('need_conver', True) node = ast.NameConstant(value=None) return node if isinstance(node.func, ast.Attribute) and (node.func.attr == 'set_virtual_device_configuration'): log_success_report(getattr(node, 'lineno', 'None'), 'set_virtual_device_configuration') util_global.set_value('need_conver', True) node = ast.NameConstant(value=None) return node if isinstance(node.func, ast.Attribute) and (node.func.attr == 'jit_scope'): if isinstance(node.func.value, ast.Attribute) and (node.func.value.attr == 'experimental'): if isinstance(node.func.value.value, ast.Attribute) and (node.func.value.value.attr == 'xla'): log_success_report(getattr(node, 'lineno', 'None'), '*.xla.experimental.jit_scope') util_global.set_value('need_conver', True) compile_ops = None for keyword in node.keywords: if keyword.arg == 'compile_ops': compile_ops = keyword break if compile_ops: compile_ops.value = pasta.parse('False') else: node.keywords.append( ast.keyword(arg='compile_ops', value=pasta.parse('False'))) return node for estimator in util_global.get_value('Estimators', []): if (isinstance(node.func, ast.Attribute) and (node.func.attr == estimator)) \ or (isinstance(node.func, ast.Name) and (node.func.id == estimator)): log_msg(getattr(node, 'lineno'), estimator + '() add config=npu_run_config_init()') config = None for keyword in node.keywords: if keyword.arg == 'config': config = keyword break if config: new_value = ast.Call(func=ast.Name(id='npu_run_config_init', ctx=ast.Load()), args=[], keywords=[ ast.keyword(arg='run_config', value=config.value) ]) ast.copy_location(new_value, config.value) config.value = new_value else: node.keywords.append( ast.keyword(arg='config', value=pasta.parse('npu_run_config_init()'))) util_global.set_value('need_conver', True) return node for estimator_func in util_global.get_value('EstimatorFunc', []): if isinstance(node.func, ast.Attribute) and (node.func.attr == estimator_func): if isinstance( node.func.value, ast.Attribute) and node.func.value.attr == "learning": return node input_fn = None hooks = None for index, _ in enumerate(node.args): if index == 0: input_fn = node.args[0] elif index == 1: hooks = node.args.pop(1) for keyword in node.keywords: if keyword.arg == 'input_fn': input_fn = keyword elif keyword.arg == 'hooks': hooks = keyword if not input_fn: break if not hooks: node.keywords.append( ast.keyword(arg='hooks', value=pasta.parse('npu_hooks_append()'))) elif isinstance(hooks, ast.keyword): new_value = ast.Call(func=ast.Name(id='npu_hooks_append', ctx=ast.Load()), args=[], keywords=[ ast.keyword(arg='hooks_list', value=hooks.value) ]) ast.copy_location(new_value, hooks.value) hooks.value = new_value else: node.keywords.append( ast.keyword(arg='hooks', value=ast.Call( func=ast.Name(id='npu_hooks_append', ctx=ast.Load()), args=[], keywords=[ ast.keyword(arg='hooks_list', value=hooks) ]))) util_global.set_value('need_conver', True) return node if isinstance(node.func, ast.Attribute) and (node.func.attr == 'compile'): opt_map = { "adadelta": "tf.keras.optimizers.Adadelta()", "adagrad": "tf.keras.optimizers.Adagrad()", "adam": "tf.keras.optimizers.Adam()", "adamax": "tf.keras.optimizers.Adamax()", "ftrl": "tf.keras.optimizers.Ftrl()", "nadam": "tf.keras.optimizers.Nadam()", "rmsprop": "tf.keras.optimizers.RMSprop()", "sgd": "tf.keras.optimizers.SGD()" } for keyword in node.keywords: if keyword.arg == "optimizer": log_success_report(getattr(node, 'lineno', 'None'), 'KerasDistributeOptimizer') if isinstance(keyword.value, ast.Str): keras_opt = opt_map[keyword.value.s] npu_keras_opt = "npu_keras_optimizer(" + keras_opt + ")" keyword.value = pasta.parse(npu_keras_opt) util_global.set_value('need_conver', True) return node if isinstance(node.func, ast.Attribute) and isinstance( node.func.value, ast.Attribute): if (node.func.attr.find("Optimizer") != -1) and (node.func.attr != 'ScipyOptimizerInterface'): log_msg(getattr(node, "lineno", "None"), "add NPUDistributedOptimizer()") new_node = ast.Call(func=ast.Name(id="npu_tf_optimizer", ctx=ast.Load()), args=[node], keywords=[]) ast.copy_location(new_node, node) util_global.set_value('need_conver', True) return new_node if isinstance(node.func, ast.Attribute): opt_list = [ "Adadelta", "Adagrad", "Adam", "Adamax", "Ftrl", "Nadam", "RMSprop", "SGD" ] if node.func.attr in opt_list: log_success_report(getattr(node, "lineno", "None"), "KerasDistributeOptimizer") new_node = ast.Call(func=ast.Name(id="npu_keras_optimizer", ctx=ast.Load()), args=[node], keywords=[]) ast.copy_location(new_node, node) util_global.set_value('need_conver', True) return new_node if (isinstance(node.func, ast.Attribute) and (node.func.attr == 'MonitoredTrainingSession')) or \ (isinstance(node.func, ast.Name) and (node.func.id == 'MonitoredTrainingSession')): log_success_report(getattr(node, "lineno", "None"), 'MonitoredTrainingSession') hooks = None for index, _ in enumerate(node.args): if index == 4: hooks = node.args.pop(4) break for keyword in node.keywords: if keyword.arg == 'hooks': hooks = keyword break if not hooks: node.keywords.append( ast.keyword(arg='hooks', value=pasta.parse('npu_hooks_append()'))) elif isinstance(hooks, ast.keyword): new_value = ast.Call( func=ast.Name(id='npu_hooks_append', ctx=ast.Load()), args=[], keywords=[ast.keyword(arg='hooks_list', value=hooks.value)]) ast.copy_location(new_value, hooks.value) hooks.value = new_value else: node.keywords.append( ast.keyword(arg='hooks', value=ast.Call(func=ast.Name(id='npu_hooks_append', ctx=ast.Load()), args=[], keywords=[ ast.keyword(arg='hooks_list', value=hooks) ]))) util_global.set_value('need_conver', True) return node specs = {'TrainSpec': 2, 'EvalSpec': 3} for spec, hooks_index in specs.items(): if _call_name_match(node.func, spec): log_success_report(getattr(node, "lineno", "None"), spec) hooks = None for index, _ in enumerate(node.args): if index == hooks_index: hooks = node.args.pop(hooks_index) break for keyword in node.keywords: if keyword.arg == 'hooks': hooks = keyword break if not hooks: node.keywords.append( ast.keyword(arg='hooks', value=pasta.parse('npu_hooks_append()'))) elif isinstance(hooks, ast.keyword): new_value = ast.Call(func=ast.Name(id='npu_hooks_append', ctx=ast.Load()), args=[], keywords=[ ast.keyword(arg='hooks_list', value=hooks.value) ]) ast.copy_location(new_value, hooks.value) hooks.value = new_value else: node.keywords.append( ast.keyword(arg='hooks', value=ast.Call( func=ast.Name(id='npu_hooks_append', ctx=ast.Load()), args=[], keywords=[ ast.keyword(arg='hooks_list', value=hooks) ]))) util_global.set_value('need_conver', True) return node
def rewrite_array_from(node): return ast.Call( func=ast.Attribute(value=node.func.value, attr='from'), args=node.args, keywords=node.keywords, )
def _makeAttrAssign(name, attr, value, lineno=1, indent=""): name = ast.Name(id=name, lineno=lineno, col_offset=len(indent), ctx=ast.Load()) attr = ast.Attribute(attr=attr, value=name, lineno=lineno, col_offset = len(f"{indent}{name}."), ctx=ast.Store()) value = ast.Name(id=value, lineno=lineno, col_offset=len(f"{indent}{name}.{attr} = "), ctx=ast.Load()) assign = ast.Assign(targets=[attr], value=value, lineno=lineno, col_offset=len(f"{indent}{name}.{attr} ")) return assign
def helper(self, name, *args): """Call a helper in this module.""" py_name = ast.Name("@pytest_ar", ast.Load()) attr = ast.Attribute(py_name, "_" + name, ast.Load()) return ast_Call(attr, list(args), [])
def build_ast_class(): ''' ClassDef(name='TestWat', bases=[Name(id='object')], keywords=[], starargs=None, kwargs=None, body=[ FunctionDef(name='__init__', args=arguments(args=[arg(arg='self', annotation=None)], vararg=arg(arg='args', annotation=None), kwonlyargs=[], kw_defaults=[], kwarg=arg(arg='kwargs', annotation=None), defaults=[]), body=[ Expr( value=Call( func=Attribute( value=Call(func=Name(id='super'), args=[], keywords=[], starargs=None, kwargs=None), attr='__init__'), args=[Name(id='self')], keywords=[], starargs=Name(id='args'), kwargs=Name(id='kwargs')))], decorator_list=[], returns=None)], decorator_list=[]), ''' init_call = ast.Call(func=ast.Name(id='super', ctx=ast.Load()), args=[], keywords=[]) super_func = ast.Call( func=ast.Attribute(value=init_call, attr='__init__', ctx=ast.Load()), args=[ast.Name(id='self', ctx=ast.Load())], starargs=ast.Name(id='args', ctx=ast.Load()), kwargs=ast.Name(id='kwargs', ctx=ast.Load()), keywords=[], ) super_init = ast.Expr( value=super_func, lineno=3, col_offset=0, ) body = [super_init] # body = [ast.Pass()] sig = ast.arguments(args=[ast.arg('self', None)], vararg=ast.arg(arg='args', annotation=None), kwarg=ast.arg(arg='kwargs', annotation=None), varargannotation=None, kwonlyargs=[], kwargannotation=None, defaults=[], kw_defaults=[]) init_func = ast.FunctionDef( name="__init__", args=sig, body=body, decorator_list=[], lineno=2, col_offset=0, ) body = [ast.Expr(value=ast.Str(s='\n\n\t')), init_func] # print(body) interface_class = ast.ClassDef( name="Test-Class", bases=[], body=body, keywords=[], decorator_list=[], starargs=None, kwargs=None, lineno=1, col_offset=0, ) print("Interface class:", interface_class) return interface_class
def builtin(self, name: str) -> ast.Attribute: """Return the builtin called *name*.""" builtin_name = ast.Name("@py_builtins", ast.Load()) return ast.Attribute(builtin_name, name, ast.Load())
def augment_ast(root): mode = os.environ.get("PGZERO_MODE", "False") assert mode != "False" warning_prelude = "WARNING: Pygame Zero mode is turned on (Run → Pygame Zero mode)" try: import pgzero # @UnusedImport except ImportError: if mode == "True": print(warning_prelude + ",\nbut pgzero module is not found. Running program in regular mode.\n", file=sys.stderr) else: assert mode == "auto" return # Check if draw is defined for stmt in root.body: if isinstance(stmt, ast.FunctionDef) and stmt.name == "draw": break else: if mode == "auto": return else: print(warning_prelude + ",\nbut your program doesn't look like usual Pygame Zero program\n" + "(draw function is missing).\n", file=sys.stderr) # need more checks in auto mode if mode == "auto": # check that draw method is not called in the code for node in ast.walk(root): if (isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == "draw"): return # prepend "import pgzrun as __pgzrun" imp = ast.Import([ast.alias("pgzrun", "__pgzrun")]) imp.lineno = 0 imp.col_offset = 0 ast.fix_missing_locations(imp) imp.tags = {"ignore"} root.body.insert(0, imp) # append "__pgzrun.go()" go = ast.Expr( ast.Call( ast.Attribute( ast.Name("__pgzrun", ast.Load()), "go", ast.Load() ), [], [] ) ) go.lineno = 1000000 go.col_offset = 0 ast.fix_missing_locations(go) go.tags = {"ignore"} root.body.append(go)
def visit_ClassDef(self, node: ast.ClassDef) -> Any: # only expand contract methods if node.name != "Contract": return node # if there is a `deploy` method, do nothing for body_node in node.body: if type(body_node) == ast.FunctionDef: if body_node.name == "deploy": return node # Factor body ast.AnnAssign into dataclass storage_keys_spec = {} new_node_body = [] for i, body_node in enumerate(node.body): if type(body_node) == ast.AnnAssign: storage_keys_spec[body_node.target.id] = body_node.annotation else: new_node_body.append(body_node) node.body = new_node_body self.storage_dataclass = make_dataclass('Storage', storage_keys_spec) # For all methods, update `self.<storage_key>` into `self.storage.<key>` # and add return `self.storage` new_body = [] for body_node in node.body: new_body_node = ExpandStorageInEntrypoints().visit(body_node) if type(body_node) == ast.FunctionDef: if not body_node.returns: body_node.returns = ast.Name(id='Storage', ctx=ast.Load()) return_storage_node = ast.Return(value=ast.Attribute( value=ast.Name(id='self', ctx=ast.Load()), attr='storage', ctx=ast.Load())) new_body_node.body.append(return_storage_node) new_body.append(new_body_node) node.body = new_body # Create deploy function deploy_function_node = ast.FunctionDef( name='deploy', args=ast.arguments(posonlyargs=[], args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[ ast.Return( value=ast.Call(func=ast.Name(id='Storage', ctx=ast.Load()), args=[], keywords=[])) ], decorator_list=[], returns=None, type_comment=None, ) node.body = [deploy_function_node] + node.body return node
def visitClose_statement(self, ctx: PlSqlParser.Close_statementContext): ret = self.visitChildren(ctx) cursor = ret[0] return ast.Call(func=ast.Attribute(value=cursor, attr="CLOSE"), args=[], keywords=[])