def test_attribute(self):
attr = ast.Attribute(ast.Name("x", ast.Store()), "y", ast.Load())
self.expr(attr, "must have Load context")
def helper(self, name: str, *args: ast.expr) -> ast.expr:
"""Call a helper in this module."""
py_name = ast.Name("@pytest_ar", ast.Load())
attr = ast.Attribute(py_name, name, ast.Load())
return ast.Call(attr, list(args), [])
def _parse_body(self, body, argument_names):
"""Recursive function transforming a @body to a block expression
Return:
- AST to append to body (real python statements)
- a list of blocks, ie list of affblock, ie list of ExprAff (AST)"""
# Init
## Real instructions
real_body = []
## Final blocks
blocks = [[[]]]
for statement in body:
if isinstance(statement, ast.Assign):
src = self.transformer.visit(statement.value)
dst = self.transformer.visit(statement.targets[0])
if (isinstance(dst, ast.Name) and dst.id not in argument_names
and dst.id not in self._ctx
and dst.id not in self._local_ctx):
# Real variable declaration
statement.value = src
real_body.append(statement)
self._local_ctx[dst.id] = src
continue
dst.ctx = ast.Load()
res = ast.Call(func=ast.Name(id='ExprAff', ctx=ast.Load()),
args=[dst, src],
keywords=[],
starargs=None,
kwargs=None)
blocks[-1][-1].append(res)
elif (isinstance(statement, ast.Expr)
and isinstance(statement.value, ast.Str)):
# String (docstring, comment, ...) -> keep it
real_body.append(statement)
elif isinstance(statement, ast.If):
# Create jumps : ir.IRDst = loc_if if cond else loc_end
# if .. else .. are also handled
cond = statement.test
real_body += self._create_labels(loc_else=True)
loc_end = ast.Name(id='loc_end_expr', ctx=ast.Load())
loc_if = ast.Name(id='loc_if_expr', ctx=ast.Load())
loc_else = ast.Name(id='loc_else_expr', ctx=ast.Load()) \
if statement.orelse else loc_end
dst = ast.Call(func=ast.Name(id='ExprCond', ctx=ast.Load()),
args=[cond, loc_if, loc_else],
keywords=[],
starargs=None,
kwargs=None)
if (isinstance(cond, ast.UnaryOp)
and isinstance(cond.op, ast.Not)):
## if not cond -> switch exprCond
dst.args[1:] = dst.args[1:][::-1]
dst.args[0] = cond.operand
IRDst = ast.Attribute(value=ast.Name(id='ir', ctx=ast.Load()),
attr='IRDst',
ctx=ast.Load())
blocks[-1][-1].append(
ast.Call(func=ast.Name(id='ExprAff', ctx=ast.Load()),
args=[IRDst, dst],
keywords=[],
starargs=None,
kwargs=None))
# Create the new blocks
elements = [(statement.body, 'loc_if')]
if statement.orelse:
elements.append((statement.orelse, 'loc_else'))
for content, loc_name in elements:
sub_blocks, sub_body = self._parse_body(
content, argument_names)
if len(sub_blocks) > 1:
raise RuntimeError("Imbricated if unimplemented")
## Close the last block
jmp_end = ast.Call(func=ast.Name(id='ExprAff',
ctx=ast.Load()),
args=[IRDst, loc_end],
keywords=[],
starargs=None,
kwargs=None)
sub_blocks[-1][-1].append(jmp_end)
instr = ast.Name(id='instr', ctx=ast.Load())
effects = ast.List(elts=sub_blocks[-1][-1], ctx=ast.Load())
assignblk = ast.Call(func=ast.Name(id='AssignBlock',
ctx=ast.Load()),
args=[effects, instr],
keywords=[],
starargs=None,
kwargs=None)
## Replace the block with a call to 'IRBlock'
loc_if_name = ast.Name(id=loc_name, ctx=ast.Load())
assignblks = ast.List(elts=[assignblk], ctx=ast.Load())
sub_blocks[-1] = ast.Call(func=ast.Name(id='IRBlock',
ctx=ast.Load()),
args=[loc_if_name, assignblks],
keywords=[],
starargs=None,
kwargs=None)
blocks += sub_blocks
real_body += sub_body
# Prepare a new block for following statement
blocks.append([[]])
else:
# TODO: real var, +=, /=, -=, <<=, >>=, if/else, ...
raise RuntimeError("Unimplemented %s" % statement)
return blocks, real_body
def rewrite_append(node):
return ast.Call(
func=ast.Attribute(value=node.func.value, attr='push', ctx=ast.Load()),
args=node.args,
keywords=node.keywords,
)
def _replace(node, b):
return ast.Call(ast.Attribute(value=node.args[0], attr=b),
node.args[1:], node.keywords)
def visit_Attribute(self, node):
'Make sure to make a new version of the Attribute so it does not get reused'
return ast.Attribute(value=self.visit(node.value),
attr=node.attr,
ctx=ast.Load())
value=ast.Name(id='console', ctx=ast.Load()),
attr='log',
ctx=ast.Load(),
),
args=node.args,
keywords=node.keywords,
)
@rewrite.register(ast.Call(func=ast.Name(id='len')))
def rewrite_len(node):
assert len(node.args) == 1
return ast.Attribute(value=node.args[0], attr='length', ctx=ast.Load())
@rewrite.register(ast.Call(func=ast.Attribute(attr='append')))
def rewrite_append(node):
return ast.Call(
func=ast.Attribute(value=node.func.value, attr='push', ctx=ast.Load()),
args=node.args,
keywords=node.keywords,
)
@rewrite.register(
ast.Call(func=ast.Attribute(value=ast.Name(id='Array'), attr='from_')))
def rewrite_array_from(node):
return ast.Call(
func=ast.Attribute(value=node.func.value, attr='from'),
args=node.args,
keywords=node.keywords,
def _get_attr_node(names):
"""Builds an Attribute node, or a Name node if names has just one entry."""
node = ast.Name(id=names[0], ctx=ast.Load())
for name in names[1:]:
node = ast.Attribute(value=node, attr=name, ctx=ast.Load())
return node
def test_ast_not_there(empty_db):
db = FuncADLDBAccess(empty_db)
r = db.lookup_results(ast.Attribute())
assert r is None
async def meval(code, globs, **kwargs):
# Note to self: please don't set globals here as they will be lost.
# Don't clutter locals
locs = {}
# Restore globals later
globs = globs.copy()
# This code saves __name__ and __package into a kwarg passed to the func.
# It is set before the users code runs to make sure relative imports work
global_args = "_globs"
while global_args in globs.keys():
# Make sure there's no name collision, just keep prepending _s
global_args = "_" + global_args
kwargs[global_args] = {}
for glob in ["__name__", "__package__"]:
# Copy data to args we are sending
kwargs[global_args][glob] = globs[glob]
root = ast.parse(code, "exec")
code = root.body
ret_name = "_ret"
ok = False
while True:
if ret_name in globs.keys():
ret_name = "_" + ret_name
continue
for node in ast.walk(root):
if isinstance(node, ast.Name) and node.id == ret_name:
ret_name = "_" + ret_name
break
ok = True
if ok:
break
if not code:
return None
if not any(isinstance(node, ast.Return) for node in code):
for i in range(len(code)):
if isinstance(code[i], ast.Expr):
if (i == len(code) - 1
or not isinstance(code[i].value, ast.Call)):
code[i] = ast.copy_location(
ast.Expr(
ast.Call(func=ast.Attribute(value=ast.Name(
id=ret_name, ctx=ast.Load()),
attr="append",
ctx=ast.Load()),
args=[code[i].value],
keywords=[])), code[-1])
else:
for node in code:
if isinstance(node, ast.Return):
node.value = ast.List(elts=[node.value], ctx=ast.Load())
code.append(
ast.copy_location(
ast.Return(value=ast.Name(id=ret_name, ctx=ast.Load())), code[-1]))
# globals().update(**<global_args>)
glob_copy = ast.Expr(
ast.Call(
func=ast.Attribute(value=ast.Call(func=ast.Name(id="globals",
ctx=ast.Load()),
args=[],
keywords=[]),
attr="update",
ctx=ast.Load()),
args=[],
keywords=[
ast.keyword(arg=None,
value=ast.Name(id=global_args, ctx=ast.Load()))
]))
ast.fix_missing_locations(glob_copy)
code.insert(0, glob_copy)
ret_decl = ast.Assign(targets=[ast.Name(id=ret_name, ctx=ast.Store())],
value=ast.List(elts=[], ctx=ast.Load()))
ast.fix_missing_locations(ret_decl)
code.insert(1, ret_decl)
args = []
for a in list(map(lambda x: ast.arg(x, None), kwargs.keys())):
ast.fix_missing_locations(a)
args += [a]
args = ast.arguments(args=[],
vararg=None,
kwonlyargs=args,
kwarg=None,
defaults=[],
kw_defaults=[None for i in range(len(args))])
args.posonlyargs = []
fun = ast.AsyncFunctionDef(name="tmp",
args=args,
body=code,
decorator_list=[],
returns=None)
ast.fix_missing_locations(fun)
mod = ast.parse("")
mod.body = [fun]
comp = compile(mod, "<string>", "exec")
exec(comp, {}, locs)
r = await locs["tmp"](**kwargs)
for i in range(len(r)):
if hasattr(r[i], "__await__"):
r[i] = await r[i] # workaround for 3.5
i = 0
while i < len(r) - 1:
if r[i] is None:
del r[i]
else:
i += 1
if len(r) == 1:
[r] = r
elif not r:
r = None
return r
def visit_FunctionDef(self, node):
node.parent = self.fundef
self.fundef = node
if len(
list(
filter(
lambda n: isinstance(n, ast.Name) and n.id is
'rep_fun', node.decorator_list))) > 0:
self.recs.append(node.name)
self.generic_visit(node)
r_args = {}
for arg in node.args.args:
arg_name = arg.arg
try:
if self.fundef.locals[arg_name] > 1:
r_args[arg_name] = self.freshName('x')
# self.fundef.locals[arg_name] += 1
except KeyError as e:
pass
# generate code to pre-initialize staged vars
# we stage all vars that are written to more than once
inits = [ast.Assign(targets=[ast.Name(id=id, ctx=ast.Store())],
value=ast.Call(
func=ast.Name(id='_var', ctx=ast.Load()),
args=[],
keywords=[])) \
for id in node.locals if node.locals[id] > 1]
a_nodes = [ast.Expr(
ast.Call(
func=ast.Name(id='_assign', ctx=ast.Load()),
args=[ast.Name(id=arg,ctx=ast.Load()), ast.Name(id=r_args[arg],ctx=ast.Load())],
keywords=[])) \
for arg in r_args]
new_node = ast.FunctionDef(
name=node.name,
args=ast.arguments(args=[
ast.arg(arg=r_args[arg.arg], annotation=None)
if arg.arg in r_args else arg for arg in node.args.args
],
vararg=None,
kwonlyargs=[],
kwarg=None,
defaults=[],
kw_defaults=[]), # node.args,
body=[
ast.Try(body=inits + a_nodes + node.body,
handlers=[
ast.ExceptHandler(
type=ast.Name(id='NonLocalReturnValue',
ctx=ast.Load()),
name='r',
body=[
ast.Return(value=ast.Attribute(
value=ast.Name(id='r', ctx=ast.Load()),
attr='value',
ctx=ast.Load()))
])
],
orelse=[],
finalbody=[])
],
decorator_list=list(
filter(
lambda n: isinstance(n, ast.Name) and n.id != 'lms' and n.
id != 'rep_fun', node.decorator_list)))
ast.copy_location(new_node, node)
ast.fix_missing_locations(new_node)
self.fundef = node.parent
return new_node
def to_attribute(value, attr, ctx=None):
""" Create an Attribute AST node.
"""
return ast.Attribute(value, attr, ctx or ast.Load())
def visit_Attribute(self, node):
return ast.copy_location(
ast.Attribute(value=self.visit(node.value),
attr=node.attr.lower()), node)
def visit_Attribute(self, node):
return py_ast.Attribute(
value=self.visit(node.value),
attr=node.attribute,
)
def _compile_directive_call_assets(self, el, options):
""" This special 't-call' tag can be used in order to aggregate/minify javascript and css assets"""
if len(el):
raise SyntaxError("t-call-assets cannot contain children nodes")
# nodes = self._get_asset(xmlid, options, css=css, js=js, debug=values.get('debug'), async=async, values=values)
#
# for index, (tagName, t_attrs, content) in enumerate(nodes):
# if index:
# append('\n ')
# append('<')
# append(tagName)
#
# self._post_processing_att(tagName, t_attrs, options)
# for name, value in t_attrs.items():
# if value or isinstance(value, string_types)):
# append(u' ')
# append(name)
# append(u'="')
# append(escape(pycompat.to_text((value)))
# append(u'"')
#
# if not content and tagName in self._void_elements:
# append('/>')
# else:
# append('>')
# if content:
# append(content)
# append('</')
# append(tagName)
# append('>')
#
space = el.getprevious() is not None and el.getprevious(
).tail or el.getparent().text
sep = u'\n' + space.rsplit('\n').pop()
return [
ast.Assign(
targets=[ast.Name(id='nodes', ctx=ast.Store())],
value=ast.Call(
func=ast.Attribute(value=ast.Name(id='self',
ctx=ast.Load()),
attr='_get_asset',
ctx=ast.Load()),
args=[
ast.Str(el.get('t-call-assets')),
ast.Name(id='options', ctx=ast.Load()),
],
keywords=[
ast.keyword('css',
self._get_attr_bool(el.get('t-css',
True))),
ast.keyword('js',
self._get_attr_bool(el.get('t-js', True))),
ast.keyword(
'debug',
ast.Call(func=ast.Attribute(value=ast.Name(
id='values', ctx=ast.Load()),
attr='get',
ctx=ast.Load()),
args=[ast.Str('debug')],
keywords=[],
starargs=None,
kwargs=None)),
ast.keyword(
'async',
self._get_attr_bool(el.get('async', False))),
ast.keyword('values',
ast.Name(id='values', ctx=ast.Load())),
],
starargs=None,
kwargs=None)),
ast.For(
target=ast.Tuple(elts=[
ast.Name(id='index', ctx=ast.Store()),
ast.Tuple(elts=[
ast.Name(id='tagName', ctx=ast.Store()),
ast.Name(id='t_attrs', ctx=ast.Store()),
ast.Name(id='content', ctx=ast.Store())
],
ctx=ast.Store())
],
ctx=ast.Store()),
iter=ast.Call(func=ast.Name(id='enumerate', ctx=ast.Load()),
args=[ast.Name(id='nodes', ctx=ast.Load())],
keywords=[],
starargs=None,
kwargs=None),
body=[
ast.If(test=ast.Name(id='index', ctx=ast.Load()),
body=[self._append(ast.Str(sep))],
orelse=[]),
self._append(ast.Str(u'<')),
self._append(ast.Name(id='tagName', ctx=ast.Load())),
] + self._append_attributes() + [
ast.If(test=ast.BoolOp(
op=ast.And(),
values=[
ast.UnaryOp(ast.Not(),
ast.Name(id='content', ctx=ast.Load()),
lineno=0,
col_offset=0),
ast.Compare(
left=ast.Name(id='tagName', ctx=ast.Load()),
ops=[ast.In()],
comparators=[
ast.Attribute(value=ast.Name(
id='self', ctx=ast.Load()),
attr='_void_elements',
ctx=ast.Load())
]),
]),
body=[self._append(ast.Str(u'/>'))],
orelse=[
self._append(ast.Str(u'>')),
ast.If(test=ast.Name(id='content',
ctx=ast.Load()),
body=[
self._append(
ast.Name(id='content',
ctx=ast.Load()))
],
orelse=[]),
self._append(ast.Str(u'</')),
self._append(
ast.Name(id='tagName', ctx=ast.Load())),
self._append(ast.Str(u'>')),
])
],
orelse=[])
]
def visitAttribute(self, n, *args):
return ast.Attribute(value=self.dispatch(n.value, *args),
attr=n.attr,
ctx=n.ctx)
def visit_ref_read(self, read: Expr):
ref, defs = self.visit(read.ref)
return (ast.Attribute(ref, 'value', Load()), defs)
def mathTransform(self, attribute, args):
# operations from the math python module or builtin operations that exist in JavaScript Math:
if attribute in self.directMathOperations:
func = ast.Attribute(
value=ast.Name(id='Math', ctx=ast.Load()),
attr=attribute,
ctx=ast.Load()
)
return ast.Call(
func=func,
args=args,
keywords=[]
)
# substitutable operations, e.g. a = sum(b,c) => a = [b,c].reduce( function(x,y) { return x+y: })
elif attribute in self.subtitutableOperations:
# a = sum(b,c) => a = [b,c].reduce( function(x,y) { return x+y: })
if attribute == 'sum':
func = ast.Attribute(
value=ast.List(
elts=args,
ctx=ast.Load()
),
attr='reduce',
ctx=ast.Load()
)
args = [
ast.Call(
func=ast.Name(id='JS', ctx=ast.Load()),
args=[ast.Str(s=' function(x,y) { return x+y; }')],
keywords=[]
)
]
return ast.Call(
func=func,
args=args,
keywords=[]
)
# randint(a,b) => Math.floor(Math.random() * (b - a + 1)) + a
elif attribute == 'randint':
left = ast.Call(
func=ast.Attribute(
value=ast.Name(id='Math', ctx=ast.Load()),
attr='floor',
ctx=ast.Load()
),
args=[
ast.BinOp(
left=ast.Call(
func=ast.Attribute(
value=ast.Name(id='Math', ctx=ast.Load()),
attr='random',
ctx=ast.Load()
),
args=[],
keywords=[]
),
op=ast.Mult(),
right=ast.BinOp(
left=ast.BinOp(
left=args[1],
op=ast.Sub(),
right=args[0]
),
op=ast.Add(),
right=ast.Num(n=1)))
],
keywords=[]
)
right = args[0]
return ast.BinOp(
left=left,
op=ast.Add(),
right=right
)
else:
return None
def attr(left, right):
return ast.Attribute(value=name(left), attr=right, lineno=1, col_offset=0, ctx=ast.Load())
def visit_FormattedValue(self, node):
# unformatted f-strings:
if not node.format_spec:
return node
# formatted f-strings:
if isinstance(node.format_spec, ast.JoinedStr) and len(node.format_spec.values) > 0 and isinstance(
node.format_spec.values[0], ast.Str):
# split the format:
format = node.format_spec.values[0].s
match = re.search(r"([0-9]*).([0-9]+)(f|i)", format)
if not match:
raise Exception(format + ' format is not currently supported')
matchGroups = match.groups()
width = matchGroups[0]
width = int(width) if width != '' else 1
precision = matchGroups[1]
precision = int(precision) if precision != '' else 1
conversion = matchGroups[2]
value = node.value
# prepare the conversion:
conversionFunc = ast.Call(
func=ast.Attribute(
value=ast.Name(id='Number', ctx=ast.Load()),
attr='parseFloat' if conversion == 'f' else 'parseInt',
ctx=ast.Load()
),
args=[value],
keywords=[]
)
# deal with precision:
precisionCall = ast.Call(
func=ast.Attribute(
value=conversionFunc,
attr='toPrecision',
ctx=ast.Load()
),
args=[ast.Num(n=precision)],
keywords=[]
)
# deal with width:
widthCall = ast.Call(
func=ast.Name(id='pad', ctx=ast.Load()),
args=[precisionCall, ast.Num(n=width)],
keywords=[]
)
# return the node:
node.value = widthCall
node.conversion = -1
node.format_spec = None
return node
raise Exception('formatted f-string are not all supported at the moment')
def rewrite_len(node):
assert len(node.args) == 1
return ast.Attribute(value=node.args[0], attr='length', ctx=ast.Load())
def ast_call(node):
convert_loss_scale_api(node)
if _call_name_match(node.func, "set_experimental_options"):
log_msg(
getattr(node, 'lineno', 'None'),
'change set_experimental_options(*) to set_experimental_options(experimental_options)'
)
node.args = [ast.Name(id='experimental_options', ctx=ast.Load())]
node.keywords = []
util_global.set_value('need_conver', True)
if isinstance(node.func,
ast.Name) and node.func.id == 'check_available_gpus':
log_msg(getattr(node, 'lineno', 'None'),
"change check_available_gpus() to ['/device:CPU:0']")
util_global.set_value('need_conver', True)
return ast.List(elts=[ast.Str(s="/device:CPU:0")], ctx=ast.Load())
if (isinstance(node.func, ast.Name) and node.func.id == 'ConfigProto') or \
(isinstance(node.func, ast.Attribute) and node.func.attr == 'ConfigProto'):
log_success_report(getattr(node, 'lineno', 'None'), 'ConfigProto()')
src = copy.deepcopy(node)
node.func = ast.Name(id='npu_config_proto', ctx=ast.Load())
node.args = []
node.keywords = []
node.keywords.append(ast.keyword(arg='config_proto', value=src))
util_global.set_value('need_conver', True)
return node
if (isinstance(node.func, ast.Name) and node.func.id == 'GraphOptions') or \
(isinstance(node.func, ast.Attribute) and node.func.attr == 'GraphOptions'):
log_success_report(getattr(node, 'lineno', 'None'), 'GraphOptions()')
src = copy.deepcopy(node)
node.func = ast.Name(id='npu_graph_options', ctx=ast.Load())
node.args = []
node.keywords = []
node.keywords.append(ast.keyword(arg='graph_options', value=src))
util_global.set_value('need_conver', True)
return node
if (isinstance(node.func, ast.Name) and node.func.id == 'OptimizerOptions') or \
(isinstance(node.func, ast.Attribute) and node.func.attr == 'OptimizerOptions'):
log_success_report(getattr(node, 'lineno', 'None'),
'OptimizerOptions()')
src = copy.deepcopy(node)
node.func = ast.Name(id='npu_optimizer_options', ctx=ast.Load())
node.args = []
node.keywords = []
node.keywords.append(ast.keyword(arg='optimizer_options', value=src))
util_global.set_value('need_conver', True)
return node
if (isinstance(node.func, ast.Name) and node.func.id == 'Session') or \
(isinstance(node.func, ast.Attribute) and node.func.attr == 'Session'):
log_success_report(getattr(node, 'lineno', 'None'), 'Session()')
config = None
for index, _ in enumerate(node.args):
if index == 2:
config = node.args.pop(2)
break
for keyword in node.keywords:
if keyword.arg == 'config':
config = keyword
break
if config:
if isinstance(config, ast.keyword):
config.value = ast.Call(func=ast.Name(id='npu_config_proto',
ctx=ast.Load()),
args=[],
keywords=[
ast.keyword(arg='config_proto',
value=config.value)
])
else:
node.keywords.append(
ast.keyword(arg='config',
value=ast.Call(
func=ast.Name(id='npu_config_proto',
ctx=ast.Load()),
args=[],
keywords=[
ast.keyword(arg='config_proto',
value=config)
])))
else:
node.keywords.append(
ast.keyword(arg='config',
value=ast.Call(func=ast.Name(id='npu_config_proto',
ctx=ast.Load()),
args=[],
keywords=[])))
util_global.set_value('need_conver', True)
return node
if isinstance(node.func, ast.Attribute
) and node.func.attr == "BroadcastGlobalVariablesHook":
log_success_report(getattr(node, "lineno", "None"),
'BroadcastGlobalVariablesHook')
node.func = ast.Name(id="NpuEmptyHook", ctx=ast.Load())
node.args = []
node.keywords = []
util_global.set_value('need_conver', True)
return node
if isinstance(node.func,
ast.Attribute) and node.func.attr == "DistributedOptimizer":
log_success_report(getattr(node, "lineno", "None"),
'DistributedOptimizer')
return node.args[0]
if isinstance(node.func, ast.Attribute) and node.func.attr == 'shard':
log_success_report(getattr(node, "lineno", "None"), 'shard')
node.args = [
ast.Call(func=ast.Name(id='get_rank_size', ctx=ast.Load()),
args=[],
keywords=[]),
ast.Call(func=ast.Name(id='get_rank_id', ctx=ast.Load()),
args=[],
keywords=[])
]
util_global.set_value("has_hccl_api", True)
util_global.set_value('need_conver', True)
if isinstance(node.func, ast.Attribute) and node.func.attr == 'dropout':
if isinstance(node.func.value,
ast.Attribute) and node.func.value.attr == 'nn':
log_success_report(getattr(node, "lineno", "None"), 'dropout')
node.func = ast.Attribute(value=ast.Name(id='npu_ops',
ctx=ast.Load()),
attr='dropout',
ctx=ast.Load())
keywords_new = []
for keyword in node.keywords:
if keyword.arg != 'rate':
keywords_new.append(keyword)
node.keywords = keywords_new
util_global.set_value('need_conver', True)
if isinstance(node.func, ast.Attribute) and ((node.func.attr == 'map_and_batch') or (node.func.attr == 'batch' \
and (not isinstance(node.func.value, ast.Attribute) or (isinstance(node.func.value, ast.Attribute) and node.func.value.attr != 'train')))):
exist = False
for keyword in node.keywords:
if keyword.arg == 'drop_remainder':
exist = True
if ((isinstance(keyword.value, ast.NameConstant)
and keyword.value.value != True)
or (not isinstance(keyword.value, ast.NameConstant))):
log_success_report(getattr(node, "lineno", "None"),
node.func.attr)
keyword.value = pasta.parse('True')
util_global.set_value('need_conver', True)
if not exist:
log_success_report(getattr(node, "lineno", "None"), node.func.attr)
keyword = ast.keyword(arg='drop_remainder',
value=pasta.parse('True'))
node.keywords.insert(0, keyword)
util_global.set_value('need_conver', True)
if (isinstance(node.func, ast.Attribute)
and isinstance(node.func.value, ast.Name)
and node.func.value.id == 'tf' and node.func.attr == 'device'):
log_success_report(getattr(node, "lineno", "None"), node.func.attr)
node.args = [ast.Str(s='/cpu:0')]
util_global.set_value('need_conver', True)
if isinstance(node.func, ast.Attribute) and (
node.func.attr == "get_distribution_strategy"
or node.func.attr == "MirroredStrategy"
or node.func.attr == "MultiWorkerMirroredStrategy"):
log_success_report(getattr(node, "lineno", "None"), node.func.attr)
new_func = ast.Attribute(value=ast.Name(id="npu_strategy",
ctx=ast.Load()),
attr="NPUStrategy",
ctx=ast.Load())
ast.copy_location(new_func, node.func)
node.func = new_func
node.keywords = []
node.args = []
util_global.set_value('need_conver', True)
if isinstance(node.func, ast.Attribute) and (node.func.attr
== 'AdviceProto'):
log_success_report(getattr(node, 'lineno', 'None'),
'tf.profiler.AdviceProto')
util_global.set_value('need_conver', True)
node = ast.NameConstant(value=None)
return node
if isinstance(node.func, ast.Attribute) and (node.func.attr == 'Checker'):
log_success_report(getattr(node, 'lineno', 'None'),
'tf.profiler.AdviceProto.Checker')
util_global.set_value('need_conver', True)
node = ast.NameConstant(value=None)
return node
if isinstance(node.func, ast.Attribute) and (node.func.attr
== 'CheckersEntry'):
log_success_report(getattr(node, 'lineno', 'None'),
'tf.profiler.AdviceProto.CheckersEntry')
util_global.set_value('need_conver', True)
node = ast.NameConstant(value=None)
return node
if isinstance(node.func, ast.Attribute) and (node.func.attr
== 'GraphNodeProto'):
log_success_report(getattr(node, 'lineno', 'None'),
'tf.profiler.GraphNodeProto')
util_global.set_value('need_conver', True)
node = ast.NameConstant(value=None)
return node
if isinstance(node.func, ast.Attribute) and (node.func.attr
== 'InputShapesEntry'):
log_success_report(getattr(node, 'lineno', 'None'),
'tf.profiler.GraphNodeProto.InputShapesEntry')
util_global.set_value('need_conver', True)
node = ast.NameConstant(value=None)
return node
if isinstance(node.func, ast.Attribute) and (node.func.attr
== 'MultiGraphNodeProto'):
log_success_report(getattr(node, 'lineno', 'None'),
'tf.profiler.MultiGraphNodeProto')
util_global.set_value('need_conver', True)
node = ast.NameConstant(value=None)
return node
if isinstance(node.func, ast.Attribute) and (node.func.attr
== 'OpLogProto'):
log_success_report(getattr(node, 'lineno', 'None'),
'tf.profiler.OpLogProto')
util_global.set_value('need_conver', True)
node = ast.NameConstant(value=None)
return node
if isinstance(node.func, ast.Attribute) and (node.func.attr
== 'IdToStringEntry'):
log_success_report(getattr(node, 'lineno', 'None'),
'tf.profiler.OpLogProto.IdToStringEntry')
util_global.set_value('need_conver', True)
node = ast.NameConstant(value=None)
return node
if isinstance(node.func, ast.Attribute) and (node.func.attr
== 'ProfileOptionBuilder'):
log_success_report(getattr(node, 'lineno', 'None'),
'tf.profiler.ProfileOptionBuilder')
util_global.set_value('need_conver', True)
node = ast.NameConstant(value=None)
return node
if isinstance(node.func, ast.Attribute) and (node.func.attr == 'advise'):
log_success_report(getattr(node, 'lineno', 'None'),
'tf.profiler.advise')
util_global.set_value('need_conver', True)
node = ast.NameConstant(value=None)
return node
if isinstance(node.func, ast.Attribute) and (node.func.attr == 'profile'):
log_success_report(getattr(node, 'lineno', 'None'),
'tf.profiler.profile')
util_global.set_value('need_conver', True)
node = ast.NameConstant(value=None)
return node
if isinstance(node.func, ast.Attribute) and (node.func.attr
== 'write_op_log'):
log_success_report(getattr(node, 'lineno', 'None'),
'tf.profiler.write_op_log')
util_global.set_value('need_conver', True)
node = ast.NameConstant(value=None)
return node
if isinstance(node.func, ast.Attribute) and (node.func.attr == 'TPUEstimator') and \
((isinstance(node.func.value, ast.Attribute) and (node.func.value.attr == 'tpu')) or \
(isinstance(node.func.value, ast.Name) and (node.func.value.id == 'tpu'))):
add_eval_on_tpu = True
add_use_tpu = True
add_export_to_tpu = True
for keyword in node.keywords:
if (keyword.arg == 'eval_on_tpu') or (
keyword.arg == 'use_tpu') or (keyword.arg
== 'export_to_tpu'):
if (not isinstance(keyword.value, ast.NameConstant)) or (
isinstance(keyword.value, ast.NameConstant) and
(keyword.value.value != False)):
log_success_report(getattr(node, 'lineno', 'None'),
'TPUEstimator(' + keyword.arg + '=*)')
keyword.value = pasta.parse('False')
util_global.set_value('need_conver', True)
if add_eval_on_tpu and (keyword.arg == 'eval_on_tpu'):
add_eval_on_tpu = False
if add_use_tpu and (keyword.arg == 'use_tpu'):
add_use_tpu = False
if add_export_to_tpu and (keyword.arg == 'export_to_tpu'):
add_export_to_tpu = False
if add_eval_on_tpu:
log_success_report(getattr(node, 'lineno', 'None'),
'TPUEstimator(eval_on_tpu=*)')
node.keywords.append(
ast.keyword(arg='eval_on_tpu', value=pasta.parse('False')))
util_global.set_value('need_conver', True)
if add_use_tpu:
log_success_report(getattr(node, 'lineno', 'None'),
'TPUEstimator(use_tpu=*)')
node.keywords.append(
ast.keyword(arg='use_tpu', value=pasta.parse('False')))
util_global.set_value('need_conver', True)
if add_export_to_tpu:
log_success_report(getattr(node, 'lineno', 'None'),
'TPUEstimator(export_to_tpu=*)')
node.keywords.append(
ast.keyword(arg='export_to_tpu', value=pasta.parse('False')))
util_global.set_value('need_conver', True)
if isinstance(node.func,
ast.Attribute) and (node.func.attr
== 'VirtualDeviceConfiguration'):
log_success_report(getattr(node, 'lineno', 'None'),
'VirtualDeviceConfiguration')
util_global.set_value('need_conver', True)
memory_limit = None
for keyword in node.keywords:
if keyword.arg == 'memory_limit':
memory_limit = keyword
break
if memory_limit:
memory_limit.value = ast.NameConstant(value=None)
else:
node.keywords.append(
ast.keyword(arg='memory_limit',
value=ast.NameConstant(value=None)))
return node
if isinstance(node.func,
ast.Attribute) and (node.func.attr
== 'set_soft_device_placement'):
log_success_report(getattr(node, 'lineno', 'None'),
'set_soft_device_placement')
util_global.set_value('need_conver', True)
node.args = []
node.keywords = [
ast.keyword(arg='enabled', value=ast.NameConstant(value=True))
]
return node
if isinstance(node.func, ast.Attribute) and (node.func.attr
== 'set_memory_growth'):
log_success_report(getattr(node, 'lineno', 'None'),
'set_memory_growth')
util_global.set_value('need_conver', True)
node = ast.NameConstant(value=None)
return node
if isinstance(node.func,
ast.Attribute) and (node.func.attr
== 'set_virtual_device_configuration'):
log_success_report(getattr(node, 'lineno', 'None'),
'set_virtual_device_configuration')
util_global.set_value('need_conver', True)
node = ast.NameConstant(value=None)
return node
if isinstance(node.func, ast.Attribute) and (node.func.attr
== 'jit_scope'):
if isinstance(node.func.value, ast.Attribute) and (node.func.value.attr
== 'experimental'):
if isinstance(node.func.value.value,
ast.Attribute) and (node.func.value.value.attr
== 'xla'):
log_success_report(getattr(node, 'lineno', 'None'),
'*.xla.experimental.jit_scope')
util_global.set_value('need_conver', True)
compile_ops = None
for keyword in node.keywords:
if keyword.arg == 'compile_ops':
compile_ops = keyword
break
if compile_ops:
compile_ops.value = pasta.parse('False')
else:
node.keywords.append(
ast.keyword(arg='compile_ops',
value=pasta.parse('False')))
return node
for estimator in util_global.get_value('Estimators', []):
if (isinstance(node.func, ast.Attribute) and (node.func.attr == estimator)) \
or (isinstance(node.func, ast.Name) and (node.func.id == estimator)):
log_msg(getattr(node, 'lineno'),
estimator + '() add config=npu_run_config_init()')
config = None
for keyword in node.keywords:
if keyword.arg == 'config':
config = keyword
break
if config:
new_value = ast.Call(func=ast.Name(id='npu_run_config_init',
ctx=ast.Load()),
args=[],
keywords=[
ast.keyword(arg='run_config',
value=config.value)
])
ast.copy_location(new_value, config.value)
config.value = new_value
else:
node.keywords.append(
ast.keyword(arg='config',
value=pasta.parse('npu_run_config_init()')))
util_global.set_value('need_conver', True)
return node
for estimator_func in util_global.get_value('EstimatorFunc', []):
if isinstance(node.func, ast.Attribute) and (node.func.attr
== estimator_func):
if isinstance(
node.func.value,
ast.Attribute) and node.func.value.attr == "learning":
return node
input_fn = None
hooks = None
for index, _ in enumerate(node.args):
if index == 0:
input_fn = node.args[0]
elif index == 1:
hooks = node.args.pop(1)
for keyword in node.keywords:
if keyword.arg == 'input_fn':
input_fn = keyword
elif keyword.arg == 'hooks':
hooks = keyword
if not input_fn:
break
if not hooks:
node.keywords.append(
ast.keyword(arg='hooks',
value=pasta.parse('npu_hooks_append()')))
elif isinstance(hooks, ast.keyword):
new_value = ast.Call(func=ast.Name(id='npu_hooks_append',
ctx=ast.Load()),
args=[],
keywords=[
ast.keyword(arg='hooks_list',
value=hooks.value)
])
ast.copy_location(new_value, hooks.value)
hooks.value = new_value
else:
node.keywords.append(
ast.keyword(arg='hooks',
value=ast.Call(
func=ast.Name(id='npu_hooks_append',
ctx=ast.Load()),
args=[],
keywords=[
ast.keyword(arg='hooks_list',
value=hooks)
])))
util_global.set_value('need_conver', True)
return node
if isinstance(node.func, ast.Attribute) and (node.func.attr == 'compile'):
opt_map = {
"adadelta": "tf.keras.optimizers.Adadelta()",
"adagrad": "tf.keras.optimizers.Adagrad()",
"adam": "tf.keras.optimizers.Adam()",
"adamax": "tf.keras.optimizers.Adamax()",
"ftrl": "tf.keras.optimizers.Ftrl()",
"nadam": "tf.keras.optimizers.Nadam()",
"rmsprop": "tf.keras.optimizers.RMSprop()",
"sgd": "tf.keras.optimizers.SGD()"
}
for keyword in node.keywords:
if keyword.arg == "optimizer":
log_success_report(getattr(node, 'lineno', 'None'),
'KerasDistributeOptimizer')
if isinstance(keyword.value, ast.Str):
keras_opt = opt_map[keyword.value.s]
npu_keras_opt = "npu_keras_optimizer(" + keras_opt + ")"
keyword.value = pasta.parse(npu_keras_opt)
util_global.set_value('need_conver', True)
return node
if isinstance(node.func, ast.Attribute) and isinstance(
node.func.value, ast.Attribute):
if (node.func.attr.find("Optimizer") !=
-1) and (node.func.attr != 'ScipyOptimizerInterface'):
log_msg(getattr(node, "lineno", "None"),
"add NPUDistributedOptimizer()")
new_node = ast.Call(func=ast.Name(id="npu_tf_optimizer",
ctx=ast.Load()),
args=[node],
keywords=[])
ast.copy_location(new_node, node)
util_global.set_value('need_conver', True)
return new_node
if isinstance(node.func, ast.Attribute):
opt_list = [
"Adadelta", "Adagrad", "Adam", "Adamax", "Ftrl", "Nadam",
"RMSprop", "SGD"
]
if node.func.attr in opt_list:
log_success_report(getattr(node, "lineno", "None"),
"KerasDistributeOptimizer")
new_node = ast.Call(func=ast.Name(id="npu_keras_optimizer",
ctx=ast.Load()),
args=[node],
keywords=[])
ast.copy_location(new_node, node)
util_global.set_value('need_conver', True)
return new_node
if (isinstance(node.func, ast.Attribute) and (node.func.attr == 'MonitoredTrainingSession')) or \
(isinstance(node.func, ast.Name) and (node.func.id == 'MonitoredTrainingSession')):
log_success_report(getattr(node, "lineno", "None"),
'MonitoredTrainingSession')
hooks = None
for index, _ in enumerate(node.args):
if index == 4:
hooks = node.args.pop(4)
break
for keyword in node.keywords:
if keyword.arg == 'hooks':
hooks = keyword
break
if not hooks:
node.keywords.append(
ast.keyword(arg='hooks',
value=pasta.parse('npu_hooks_append()')))
elif isinstance(hooks, ast.keyword):
new_value = ast.Call(
func=ast.Name(id='npu_hooks_append', ctx=ast.Load()),
args=[],
keywords=[ast.keyword(arg='hooks_list', value=hooks.value)])
ast.copy_location(new_value, hooks.value)
hooks.value = new_value
else:
node.keywords.append(
ast.keyword(arg='hooks',
value=ast.Call(func=ast.Name(id='npu_hooks_append',
ctx=ast.Load()),
args=[],
keywords=[
ast.keyword(arg='hooks_list',
value=hooks)
])))
util_global.set_value('need_conver', True)
return node
specs = {'TrainSpec': 2, 'EvalSpec': 3}
for spec, hooks_index in specs.items():
if _call_name_match(node.func, spec):
log_success_report(getattr(node, "lineno", "None"), spec)
hooks = None
for index, _ in enumerate(node.args):
if index == hooks_index:
hooks = node.args.pop(hooks_index)
break
for keyword in node.keywords:
if keyword.arg == 'hooks':
hooks = keyword
break
if not hooks:
node.keywords.append(
ast.keyword(arg='hooks',
value=pasta.parse('npu_hooks_append()')))
elif isinstance(hooks, ast.keyword):
new_value = ast.Call(func=ast.Name(id='npu_hooks_append',
ctx=ast.Load()),
args=[],
keywords=[
ast.keyword(arg='hooks_list',
value=hooks.value)
])
ast.copy_location(new_value, hooks.value)
hooks.value = new_value
else:
node.keywords.append(
ast.keyword(arg='hooks',
value=ast.Call(
func=ast.Name(id='npu_hooks_append',
ctx=ast.Load()),
args=[],
keywords=[
ast.keyword(arg='hooks_list',
value=hooks)
])))
util_global.set_value('need_conver', True)
return node
def rewrite_array_from(node):
return ast.Call(
func=ast.Attribute(value=node.func.value, attr='from'),
args=node.args,
keywords=node.keywords,
)
def _makeAttrAssign(name, attr, value, lineno=1, indent=""):
name = ast.Name(id=name, lineno=lineno, col_offset=len(indent), ctx=ast.Load())
attr = ast.Attribute(attr=attr, value=name, lineno=lineno, col_offset = len(f"{indent}{name}."), ctx=ast.Store())
value = ast.Name(id=value, lineno=lineno, col_offset=len(f"{indent}{name}.{attr} = "), ctx=ast.Load())
assign = ast.Assign(targets=[attr], value=value, lineno=lineno, col_offset=len(f"{indent}{name}.{attr} "))
return assign
def helper(self, name, *args):
"""Call a helper in this module."""
py_name = ast.Name("@pytest_ar", ast.Load())
attr = ast.Attribute(py_name, "_" + name, ast.Load())
return ast_Call(attr, list(args), [])
def build_ast_class():
'''
ClassDef(name='TestWat',
bases=[Name(id='object')],
keywords=[],
starargs=None,
kwargs=None,
body=[
FunctionDef(name='__init__',
args=arguments(args=[arg(arg='self', annotation=None)],
vararg=arg(arg='args', annotation=None),
kwonlyargs=[],
kw_defaults=[],
kwarg=arg(arg='kwargs', annotation=None),
defaults=[]),
body=[
Expr(
value=Call(
func=Attribute(
value=Call(func=Name(id='super'), args=[], keywords=[], starargs=None, kwargs=None),
attr='__init__'),
args=[Name(id='self')],
keywords=[],
starargs=Name(id='args'),
kwargs=Name(id='kwargs')))],
decorator_list=[],
returns=None)],
decorator_list=[]),
'''
init_call = ast.Call(func=ast.Name(id='super', ctx=ast.Load()),
args=[],
keywords=[])
super_func = ast.Call(
func=ast.Attribute(value=init_call, attr='__init__', ctx=ast.Load()),
args=[ast.Name(id='self', ctx=ast.Load())],
starargs=ast.Name(id='args', ctx=ast.Load()),
kwargs=ast.Name(id='kwargs', ctx=ast.Load()),
keywords=[],
)
super_init = ast.Expr(
value=super_func,
lineno=3,
col_offset=0,
)
body = [super_init]
# body = [ast.Pass()]
sig = ast.arguments(args=[ast.arg('self', None)],
vararg=ast.arg(arg='args', annotation=None),
kwarg=ast.arg(arg='kwargs', annotation=None),
varargannotation=None,
kwonlyargs=[],
kwargannotation=None,
defaults=[],
kw_defaults=[])
init_func = ast.FunctionDef(
name="__init__",
args=sig,
body=body,
decorator_list=[],
lineno=2,
col_offset=0,
)
body = [ast.Expr(value=ast.Str(s='\n\n\t')), init_func]
# print(body)
interface_class = ast.ClassDef(
name="Test-Class",
bases=[],
body=body,
keywords=[],
decorator_list=[],
starargs=None,
kwargs=None,
lineno=1,
col_offset=0,
)
print("Interface class:", interface_class)
return interface_class
def builtin(self, name: str) -> ast.Attribute:
"""Return the builtin called *name*."""
builtin_name = ast.Name("@py_builtins", ast.Load())
return ast.Attribute(builtin_name, name, ast.Load())
def augment_ast(root):
mode = os.environ.get("PGZERO_MODE", "False")
assert mode != "False"
warning_prelude = "WARNING: Pygame Zero mode is turned on (Run → Pygame Zero mode)"
try:
import pgzero # @UnusedImport
except ImportError:
if mode == "True":
print(warning_prelude
+ ",\nbut pgzero module is not found. Running program in regular mode.\n",
file=sys.stderr)
else:
assert mode == "auto"
return
# Check if draw is defined
for stmt in root.body:
if isinstance(stmt, ast.FunctionDef) and stmt.name == "draw":
break
else:
if mode == "auto":
return
else:
print(warning_prelude
+ ",\nbut your program doesn't look like usual Pygame Zero program\n"
+ "(draw function is missing).\n",
file=sys.stderr)
# need more checks in auto mode
if mode == "auto":
# check that draw method is not called in the code
for node in ast.walk(root):
if (isinstance(node, ast.Call) and isinstance(node.func, ast.Name)
and node.func.id == "draw"):
return
# prepend "import pgzrun as __pgzrun"
imp = ast.Import([ast.alias("pgzrun", "__pgzrun")])
imp.lineno = 0
imp.col_offset = 0
ast.fix_missing_locations(imp)
imp.tags = {"ignore"}
root.body.insert(0, imp)
# append "__pgzrun.go()"
go = ast.Expr(
ast.Call(
ast.Attribute(
ast.Name("__pgzrun", ast.Load()),
"go",
ast.Load()
),
[],
[]
)
)
go.lineno = 1000000
go.col_offset = 0
ast.fix_missing_locations(go)
go.tags = {"ignore"}
root.body.append(go)
def visit_ClassDef(self, node: ast.ClassDef) -> Any:
# only expand contract methods
if node.name != "Contract":
return node
# if there is a `deploy` method, do nothing
for body_node in node.body:
if type(body_node) == ast.FunctionDef:
if body_node.name == "deploy":
return node
# Factor body ast.AnnAssign into dataclass
storage_keys_spec = {}
new_node_body = []
for i, body_node in enumerate(node.body):
if type(body_node) == ast.AnnAssign:
storage_keys_spec[body_node.target.id] = body_node.annotation
else:
new_node_body.append(body_node)
node.body = new_node_body
self.storage_dataclass = make_dataclass('Storage', storage_keys_spec)
# For all methods, update `self.<storage_key>` into `self.storage.<key>`
# and add return `self.storage`
new_body = []
for body_node in node.body:
new_body_node = ExpandStorageInEntrypoints().visit(body_node)
if type(body_node) == ast.FunctionDef:
if not body_node.returns:
body_node.returns = ast.Name(id='Storage', ctx=ast.Load())
return_storage_node = ast.Return(value=ast.Attribute(
value=ast.Name(id='self', ctx=ast.Load()),
attr='storage',
ctx=ast.Load()))
new_body_node.body.append(return_storage_node)
new_body.append(new_body_node)
node.body = new_body
# Create deploy function
deploy_function_node = ast.FunctionDef(
name='deploy',
args=ast.arguments(posonlyargs=[],
args=[],
vararg=None,
kwonlyargs=[],
kw_defaults=[],
kwarg=None,
defaults=[]),
body=[
ast.Return(
value=ast.Call(func=ast.Name(id='Storage', ctx=ast.Load()),
args=[],
keywords=[]))
],
decorator_list=[],
returns=None,
type_comment=None,
)
node.body = [deploy_function_node] + node.body
return node
def visitClose_statement(self, ctx: PlSqlParser.Close_statementContext):
ret = self.visitChildren(ctx)
cursor = ret[0]
return ast.Call(func=ast.Attribute(value=cursor, attr="CLOSE"),
args=[],
keywords=[])
