def parse_layer_from_yaml(ast_state, yaml_data, layer_name, output_fn,
fn_name_fn):
matchers = []
for yaml_datum in yaml_data['filters']:
matcher = create_matcher(yaml_datum, output_fn)
matchers.append(matcher)
stmts = []
for matcher in matchers:
# columns in the query should be those needed by the rule, union those
# needed by the output, minus any which are synthetic and local to the
# query function.
stmts.append(matcher.as_ast(ast_state))
prepend_statements = []
if ast_state.has_way_area:
way_area_stmt = make_way_area_assignment()
prepend_statements.append(way_area_stmt)
if ast_state.has_volume:
volume_stmt = make_volume_assignment()
prepend_statements.append(volume_stmt)
if ast_state.has_zoom:
zoom_stmt = make_zoom_assignment()
prepend_statements.append(zoom_stmt)
stmts = prepend_statements + stmts
func = ast.FunctionDef(
fn_name_fn(layer_name),
ast.arguments([
ast.Name('shape', ast.Param()),
ast.Name('props', ast.Param()),
ast.Name('fid', ast.Param()),
ast.Name('meta', ast.Param()),
], None, None, []), stmts, [])
return func
def visit_arguments(self, node):
"""Get surrogate name nodes from function arguments.
This method only handles the *args and **kwargs variable names. In PY2
the rest of the children for this node names. In PY3+ they are args
and handled by another visitor.
"""
# PY2 vararg and kwarg are raw strings which represent the tokens of
# the variable name chosen. These have become arg nodes in PY33.
if pycompat.PY2 or (pycompat.PY3 and pycompat.VERSION.minor < 4):
if node.vararg is not None:
yield references.copy_location(
ast.Name(
id=node.vararg,
ctx=ast.Param(),
),
node,
)
if node.kwarg is not None:
yield references.copy_location(
ast.Name(
id=node.kwarg,
ctx=ast.Param(),
),
node,
)
def python_ast(self):
decorators = []
iseq = []
args = []
if self.method:
if self.name == "Thavron":
self.name = "__init__"
args.append(
ast.Name("self",
ast.Param(),
lineno=self.line,
col_offset=self.column))
if self.bound:
if self.method:
decorators.append(ast.Name('classmethod', ast.Load()))
else:
pass #TODO: Raise Semantic exception!
if self.instruction_sequence:
iseq = self.instruction_sequence.python_ast()
return ast.FunctionDef(self.name,
ast.arguments(
args + [
ast.Name(name,
ast.Param(),
lineno=self.line,
col_offset=self.column)
for name in self.arguments
], None, None, []),
iseq,
decorators,
lineno=self.line,
col_offset=self.column)
def p_params(p):
'''params : type ID
| params COMMA type ID'''
if len(p) == 3:
p[0] = [ast.Param(p[1], p[2]).at(loc(p, 2))]
else:
p[0] = p[1] + [ast.Param(p[3], p[4]).at(loc(p, 4))]
def visit_arguments(self, node):
# missing locations for vararg and kwarg set at function level
if node.vararg:
vararg = ast.Name(node.vararg, ast.Param())
else:
vararg = None
if node.kwarg:
kwarg = ast.Name(node.kwarg, ast.Param())
else:
kwarg = None
if node.vararg:
vararg = ast.Name(node.vararg, ast.Param())
else:
vararg = None
new_node = gast.arguments(
self._visit(node.args),
[], # posonlyargs
self._visit(vararg),
[], # kwonlyargs
[], # kw_defaults
self._visit(kwarg),
self._visit(node.defaults),
)
return new_node
def visit_FunctionDef(self, node):
self.IN_INIT = False
if self.CONVERT and node.name == '__init__':
new_args = (node.args.args[:2] + [
ast.Name(id='new1', ctx=ast.Param()),
ast.Name(id='new2', ctx=ast.Param())
])
node.args.args = new_args
self.IN_INIT = True
self.generic_visit(node)
return node
def parse(self, func):
"""Function decorator, returning a correct method from a pseudo-Python
one"""
# Get the function AST
parsed = ast.parse(inspect.getsource(func))
fc_ast = parsed.body[0]
argument_names = [get_arg_name(name) for name in fc_ast.args.args]
# Init local cache
self._local_ctx = {}
# Translate (blocks[0][0] is the current instr)
blocks, body = self._parse_body(fc_ast.body, argument_names)
# Build the new function
fc_ast.args.args[0:0] = [
gen_arg('ir', ast.Param()),
gen_arg('instr', ast.Param())
]
cur_instr = blocks[0][0]
if len(blocks[-1][0]) == 0:
## Last block can be empty
blocks.pop()
other_blocks = blocks[1:]
body.append(
ast.Return(value=ast.Tuple(elts=[
ast.List(elts=cur_instr, ctx=ast.Load()),
ast.List(elts=other_blocks, ctx=ast.Load())
],
ctx=ast.Load())))
ret = ast.parse('')
ret.body = [
ast.FunctionDef(name=fc_ast.name,
args=fc_ast.args,
body=body,
decorator_list=[])
]
# To display the generated function, use codegen.to_source
# codegen: https://github.com/andreif/codegen
# Compile according to the context
fixed = ast.fix_missing_locations(ret)
codeobj = compile(fixed, '<string>', 'exec')
ctx = self._ctx.copy()
eval(codeobj, ctx)
# Get the function back
self._functions[fc_ast.name] = ctx[fc_ast.name]
return ctx[fc_ast.name]
def _transform_function_arguments(left):
if type(left) is ast.Name:
names = [left]
else:
names = left.elts
# Python3
if hasattr(_ast, 'arg'):
args = [
_ast.arg(annotation=None,
arg=name.id,
col_offset=name.col_offset,
lineno=name.lineno) for name in names
]
return ast.arguments(args=args,
defaults=[],
kwonlyargs=[],
kw_defaults=[])
# Python 2
arguments = ast.arguments(args=names, defaults=[])
for argument in arguments.args:
argument.ctx = ast.Param()
return arguments
def formal_parameters(self):
""" formal_parameters : ID (COMMA ID)* COLON type_spec
procedure's params, just like Foo(a, b : int)
current process is a, b : int
"""
param_nodes = []
param_tokens = [self.current_token]
self.eat(TokenType.ID)
# while contain ','
while self.current_token.type == TokenType.COMMA:
self.eat(TokenType.COMMA)
# add all ID
param_tokens.append(self.current_token)
self.eat(TokenType.ID)
# eat ':'
self.eat(TokenType.COLON)
# start params type
type_node = self.type_spec()
for param_token in param_tokens:
var = ast.Var(param_token)
param_node = ast.Param(var, type_node)
param_nodes.append(param_node)
return param_nodes
def visit_GeneratorExp(self, node):
if node in self.optimizable_comprehension:
self.generic_visit(node)
iters = [self.make_Iterator(gen) for gen in node.generators]
variables = [ast.Name(gen.target.id, ast.Param())
for gen in node.generators]
# If dim = 1, product is useless
if len(iters) == 1:
iterAST = iters[0]
varAST = ast.arguments([variables[0]], None, None, [])
else:
prodName = ast.Attribute(
value=ast.Name(id='itertools', ctx=ast.Load()),
attr='product', ctx=ast.Load())
iterAST = ast.Call(prodName, iters, [], None, None)
varAST = ast.arguments([ast.Tuple(variables, ast.Store())],
None, None, [])
imapName = ast.Attribute(
value=ast.Name(id='itertools', ctx=ast.Load()),
attr='imap', ctx=ast.Load())
ldBodyimap = node.elt
ldimap = ast.Lambda(varAST, ldBodyimap)
return ast.Call(imapName, [ldimap, iterAST], [], None, None)
else:
return self.generic_visit(node)
def visit_GeneratorExp(self, node):
node.elt = self.visit(node.elt)
name = "generator_expression{0}".format(self.count)
self.count += 1
args = self.passmanager.gather(ImportedIds, node, self.ctx)
self.count_iter = 0
body = reduce(self.nest_reducer,
reversed(node.generators),
ast.Expr(ast.Yield(node.elt))
)
sargs = sorted(ast.Name(arg, ast.Param()) for arg in args)
fd = ast.FunctionDef(name,
ast.arguments(sargs, None, None, []),
[body],
[])
self.ctx.module.body.append(fd)
return ast.Call(
ast.Name(name, ast.Load()),
[ast.Name(arg.id, ast.Load()) for arg in sargs],
[],
None,
None
) # no sharing !
def wrap_callable(self, node):
return ast.Lambda(
args=ast.arguments(
args=[ast.Name(id=CVAR_NAME, ctx=ast.Param())], vararg=None, kwarg=None, defaults=[]
),
body=node,
)
def resolveNameRef(self, visitor, callinfo):
# assert False
# visitor.logger.debug('LibFuncMatch.resolveNameRef', self, visitor, callinfo)
callinfo.getOwnerFunc().info.dispatched = True
argcount = len(callinfo.args)
assert argcount == 2
caseblock = callinfo.args[-1]
assert isinstance(caseblock, ast.CaseBlock)
if argcount == 2:
caseblock.matchVar = callinfo.args[0]
for entry in caseblock.entries:
# visitor.logger.debug('match resolve entry', callinfo.caller, caseblock.matchVar, entry.pattern, callinfo.getOwnerFunc())
assert hasattr(entry.pattern,
'name'), ('match resolve entry',
callinfo.caller,
caseblock.matchVar, entry.pattern,
callinfo.getOwnerFunc())
assert hasattr(caseblock.matchVar,
'name'), ('match resolve matchVar',
callinfo.caller,
caseblock.matchVar, entry.pattern,
callinfo.getOwnerFunc())
casevar = ast.CaseVarSpec(
ast.Param(caseblock.matchVar.name,
ast.UserType([entry.pattern.name])))
entry.addSymbol(casevar.variable)
casevar.setOwner(entry)
visitor.visitNewItem(casevar)
def visit_Lambda(self, node):
if modules['functools'] not in self.global_declarations.values():
import_ = ast.Import([ast.alias('functools', None)])
self.imports.append(import_)
self.global_declarations['functools'] = modules['functools']
self.generic_visit(node)
forged_name = "{0}_lambda{1}".format(self.prefix,
len(self.lambda_functions))
ii = self.passmanager.gather(ImportedIds, node, self.ctx)
ii.difference_update(self.lambda_functions) # remove current lambdas
binded_args = [ast.Name(iin, ast.Load()) for iin in sorted(ii)]
former_nbargs = len(node.args.args)
node.args.args = ([ast.Name(iin, ast.Param())
for iin in sorted(ii)] + node.args.args)
forged_fdef = ast.FunctionDef(forged_name, copy(node.args),
[ast.Return(node.body)], [])
self.lambda_functions.append(forged_fdef)
proxy_call = ast.Name(forged_name, ast.Load())
if binded_args:
return ast.Call(
ast.Attribute(ast.Name('functools', ast.Load()), "partial",
ast.Load()), [proxy_call] + binded_args, [],
None, None)
else:
return proxy_call
def _translateFuncToPyAst(self, func):
assert isinstance(func, CFunc)
base = FuncEnv(globalScope=self.globalScope)
assert func.name is not None
base.func = func
base.astNode.name = func.name
base.pushScope(base.astNode.body)
for arg in func.args:
name = base.registerNewVar(arg.name, arg)
assert name is not None
base.astNode.args.args.append(ast.Name(id=name, ctx=ast.Param()))
if func.body is None:
# TODO: search in other C files
# Hack for now: ignore :)
print "XXX:", func.name, "is not loaded yet"
else:
cCodeToPyAstList(base, func.body)
base.popScope()
if isSameType(self._cStateWrapper, func.type, CVoidType()):
returnValueAst = NoneAstNode
else:
returnTypeAst = getAstNodeForVarType(self, func.type)
returnValueAst = makeAstNodeCall(returnTypeAst)
base.astNode.body.append(ast.Return(value=returnValueAst))
return base
def __call__(self, selection):
if not self.is_initialized:
self._initialize()
try:
parse_result = self.expression.parseString(selection, parseAll=True)
except ParseException as e:
msg = str(e)
lines = ["%s: %s" % (msg, selection),
" " * (12 + len("%s: " % msg) + (e.loc)) + "^^^"]
raise ValueError('\n'.join(lines))
# Change __ATOM__ in function bodies. It must bind to the arg
# name specified below (i.e. 'atom')
astnode = self.transformer.visit(deepcopy(parse_result[0].ast()))
if PY2:
args = [ast.Name(id='atom', ctx=ast.Param())]
signature = ast.arguments(args=args, vararg=None, kwarg=None,
defaults=[])
else:
args = [ast.arg(arg='atom', annotation=None)]
signature = ast.arguments(args=args, vararg=None, kwarg=None,
kwonlyargs=[], defaults=[],
kw_defaults=[])
func = ast.Expression(body=ast.Lambda(signature, astnode))
source = codegen.to_source(astnode)
expr = eval(
compile(ast.fix_missing_locations(func), '<string>', mode='eval'),
SELECTION_GLOBALS)
return _ParsedSelection(expr, source, astnode)
def _make_fn(name, chain_fn, args, defaults):
args_with_self = ['_self'] + list(args)
arguments = [_ast.Name(id=arg, ctx=_ast.Load()) for arg in args_with_self]
defs = [_ast.Name(id='_def{0}'.format(idx), ctx=_ast.Load()) for idx, _ in enumerate(defaults)]
if _PY2:
parameters = _ast.arguments(args=[_ast.Name(id=arg, ctx=_ast.Param()) for arg in args_with_self],
defaults=defs)
else:
parameters = _ast.arguments(args=[_ast.arg(arg=arg) for arg in args_with_self],
kwonlyargs=[],
defaults=defs,
kw_defaults=[])
module_node = _ast.Module(body=[_ast.FunctionDef(name=name,
args=parameters,
body=[_ast.Return(value=_ast.Call(func=_ast.Name(id='_chain', ctx=_ast.Load()),
args=arguments,
keywords=[]))],
decorator_list=[])])
module_node = _ast.fix_missing_locations(module_node)
# compile the ast
code = compile(module_node, '<string>', 'exec')
# and eval it in the right context
globals_ = {'_chain': chain_fn}
locals_ = dict(('_def{0}'.format(idx), value) for idx, value in enumerate(defaults))
eval(code, globals_, locals_)
# extract our function from the newly created module
return locals_[name]
def visit_AnyComp(self, node, comp_type, *path):
node.elt = self.visit(node.elt)
name = "{0}_comprehension{1}".format(comp_type, self.count)
self.count += 1
args = self.passmanager.gather(ImportedIds, node, self.ctx)
self.count_iter = 0
starget = "__target"
body = reduce(
self.nest_reducer, reversed(node.generators),
ast.Expr(
ast.Call(
reduce(lambda x, y: ast.Attribute(x, y, ast.Load()),
path[1:], ast.Name(path[0], ast.Load())),
[ast.Name(starget, ast.Load()), node.elt], [], None,
None)))
# add extra metadata to this node
metadata.add(body, metadata.Comprehension(starget))
init = ast.Assign([ast.Name(starget, ast.Store())],
ast.Call(
ast.Attribute(
ast.Name('__builtin__', ast.Load()),
comp_type, ast.Load()), [], [], None, None))
result = ast.Return(ast.Name(starget, ast.Load()))
sargs = sorted(ast.Name(arg, ast.Param()) for arg in args)
fd = ast.FunctionDef(name, ast.arguments(sargs, None, None, []),
[init, body, result], [])
self.ctx.module.body.append(fd)
return ast.Call(ast.Name(name, ast.Load()),
[ast.Name(arg.id, ast.Load()) for arg in sargs], [],
None, None) # no sharing !
def visitName(self, node, ctype=None):
if isinstance(node.ctx, ast.Param):
if node.id not in self.argtypes:
raise CTypeError(
node.id, 'function %s() requires argument %r' %
(self.func_name, node.id))
ctype = self.argtypes[node.id]
return cast.CName(node.id, ast.Param(), ctype, **n(node))
elif isinstance(node.ctx, ast.Load):
try:
ctype = self.scope(node.id)
except NameError as err:
raise cast.CError(node, NameError, err.args[0])
return cast.CName(node.id, ast.Load(), ctype, **n(node))
elif isinstance(node.ctx, ast.Store):
assert type is not None
if node.id in self.locls:
ectype = self.locls[node.id]
try:
greatest_common_type(ctype, ectype)
except: # Raise a custom exception if the types are not compatible
raise
ctype = ectype
self.locls[node.id] = ctype
return cast.CName(node.id, ast.Store(), ctype, **n(node))
else:
assert False
def new_class_def():
node = ast.ClassDef(
name='Conf',
bases=[ast.Name(id='object', ctx=ast.Load())],
body=[
ast.FunctionDef(
name='__init__',
args=ast.arguments(args=[ast.Name(id='self', ctx=ast.Param())],
vararg=None,
kwarg=None,
defaults=[]),
body=[
ast.Assign(targets=[
ast.Attribute(value=ast.Name(id='self',
ctx=ast.Load()),
attr='AA',
ctx=ast.Store())
],
value=ast.List(elts=[], ctx=ast.Load())),
ast.Assign(targets=[
ast.Attribute(value=ast.Name(id='self',
ctx=ast.Load()),
attr='BB',
ctx=ast.Store())
],
value=ast.Num(n=2))
],
decorator_list=[])
],
decorator_list=[])
return node
def compile_func(arg_names, statements, name='_the_func', debug=False):
"""Compile a list of statements as the body of a function and return
the resulting Python function. If `debug`, then print out the
bytecode of the compiled function.
"""
func_def = ast.FunctionDef(
name.encode('utf8'),
ast.arguments(
[ast.Name(n, ast.Param()) for n in arg_names],
None,
None,
[ex_literal(None) for _ in arg_names],
),
statements,
[],
)
mod = ast.Module([func_def])
ast.fix_missing_locations(mod)
prog = compile(mod, b'<generated>', b'exec')
# Debug: show bytecode.
if debug:
dis.dis(prog)
for const in prog.co_consts:
if isinstance(const, types.CodeType):
dis.dis(const)
the_locals = {}
exec prog in {}, the_locals
return the_locals[name]
def assign_namespace(self, root):
setup_def = [
func for func in root.body
if isinstance(func, ast.FunctionDef) and func.name == 'setup'
]
setup_def = None if len(setup_def) == 0 else setup_def[0]
if setup_def is None:
body = [self.get_assign_node(self.namespace)]
if self.app_metadata:
body.append(
self.get_call_append_metadata_node(self.app_metadata))
setup_node = ast.FunctionDef(
name='setup',
args=ast.arguments(args=[
ast.arg(arg='self', annotation=None)
if six.PY3 else ast.Name(id='self', ctx=ast.Param())
],
vararg=None,
kwarg=None,
defaults=[]),
body=body,
decorator_list=[])
root.body.append(setup_node)
else:
assign_node = [
assign for assign in setup_def.body
if isinstance(assign, ast.Assign)
and self.has_target(assign.targets, "__pd_gateway_namespace__")
]
if len(assign_node) == 0:
setup_def.body.append(self.get_assign_node(self.namespace))
if self.app_metadata:
setup_def.body.append(
self.get_call_append_metadata_node(self.app_metadata))
def _build_arg(name):
try:
# Python 3
return ast.arg(arg=name)
except AttributeError:
# Python 2
return ast.Name(id=name, ctx=ast.Param())
def visit_Expr(self, node):
if type(node.value) is ast.Call:
call = node.value
if self.is_concurrent_call(call):
self.encounter_call(call)
return node
elif any([self.is_concurrent_call(arg) for arg in call.args]):
conc_args = [(i, arg) for i, arg in enumerate(call.args) if self.is_concurrent_call(arg)]
if len(conc_args) > 1:
raise self.not_implemented_error(call, "Functions with multiple @concurrent parameters are unsupported")
conc_call = conc_args[0][1]
if isinstance(call.func, ast.Attribute):
self.arguments.add(SchedulerRewriter.top_level_name(call.func.value))
self.encounter_call(conc_call)
call.args[conc_args[0][0]] = ast.Name("__value__", ast.Load())
if sys.version_info >= (3, 0):
args = [ast.arg("__value__", None)]
else:
args = [ast.Name("__value__", ast.Param())]
call_lambda = ast.Lambda(ast.arguments(args = args, defaults = [], kwonlyargs = [], kw_defaults = []), call)
copy_location_kwargs = {
"func": ast.Attribute(conc_call.func, 'call', ast.Load()),
"args": [call_lambda] + conc_call.args,
"keywords": conc_call.keywords
}
if(sys.version_info < (3, 0)):
copy_location_kwargs["kwargs"] = conc_call.kwargs
return copy_location(ast.Expr(ast.Call(**copy_location_kwargs)), node)
return self.generic_visit(node)
def gen_unpack_wrapper(self, node, target, ctx='store'):
"""Helper function to protect tuple unpacks.
node: used to copy the locations for the new nodes.
target: is the tuple which must be protected.
ctx: Defines the context of the returned temporary node.
It returns a tuple with two element.
Element 1: Is a temporary name node which must be used to
replace the target.
The context (store, param) is defined
by the 'ctx' parameter..
Element 2: Is a try .. finally where the body performs the
protected tuple unpack of the temporary variable
into the original target.
"""
# Generate a tmp name to replace the tuple with.
tnam = self.tmpName
# Generates an expressions which protects the unpack.
# converter looks like 'wrapper(tnam)'.
# 'wrapper' takes care to protect sequence unpacking with __rl_getiter__.
converter = self.protect_unpack_sequence(target,
ast.Name(tnam, ast.Load()))
# Assign the expression to the original names.
# Cleanup the temporary variable.
# Generates:
# try:
# # converter is 'wrapper(tnam)'
# arg = converter
# finally:
# del tmp_arg
try_body = [ast.Assign(targets=[target], value=converter)]
finalbody = [self.gen_del_stmt(tnam)]
cleanup = ast.Try(body=try_body,
finalbody=finalbody,
handlers=[],
orelse=[])
if ctx == 'store':
ctx = ast.Store()
elif ctx == 'param':
ctx = ast.Param()
else: # pragma: no cover
# Only store and param are defined ctx.
raise NotImplementedError('bad ctx "%s"' % type(ctx))
# This node is used to catch the tuple in a tmp variable.
tmp_target = ast.Name(tnam, ctx)
copy_locations(tmp_target, node)
copy_locations(cleanup, node)
return (tmp_target, cleanup)
def visitLambda(self, node):
body = self.visit(node.body)
args = ast.arguments(args=[], vararg=None, kwarg=None, defaults=[])
for var_id in sorted(self.locls.keys()):
args.args.append(ast.Name(var_id, ast.Param(), **n(node)))
return ast.Lambda(args, body, **n(node))
def cont_as_function(self, name, k, ck):
formals = ast.arguments()
if k.name and k.name != '_':
#formals.args = [ast.Name (k.name, ast.Param())]
formals.args = [ast.arg(k.name, ast.Param())]
else:
formals.args = []
return FunctionDef(name, True, formals, [], k.exp, dead_cont(ck))
def build_class():
"""
Constructs a :class:`ast.ClassDef` node that wraps the entire template
file. The class will have an entry function ``root`` with:
.. function:: root(context)
Starts the template parsing with the given context.
:returns: Returns a generator of strings that can be joined to the
rendered template.
:returns: a 2-tuple with the class and the entry function
"""
args = {}
if PY3:
args.update({
'args': [
ast.arg(arg='self', annotation=None),
ast.arg(arg='context', annotation=None),
],
'kwonlyargs': [],
'kw_defaults': [],
})
else:
args['args'] = [
ast.Name(id='self', ctx=ast.Param()),
ast.Name(id='context', ctx=ast.Param())
]
root_func = ast.FunctionDef(
name='root',
args=ast.arguments(vararg=None, kwarg=None, defaults=[], **args),
body=[
# we add an empty string to guarantee for a string and generator on
# root level
build_yield(ast.Str(s=''))
],
decorator_list=[])
klass = ast.ClassDef(name='Template',
bases=[ast.Name(id='object', ctx=ast.Load())],
keywords=[],
starargs=None,
kwargs=None,
body=[root_func],
decorator_list=[])
return klass, root_func
def _arg_name(name):
"""
:param str name:
:return:
"""
if sys.version_info[0] == 2:
return ast.Name(id=name, ctx=ast.Param())
return ast.arg(arg=name, annotation=None)
def new_param(self):
id = self.expect('id')
name = stdast.Name()
name.lineno = self.i + 1
name.id = id
name.ctx = stdast.Param()
name.ctx.lineno = self.i + 1
return name
