def make_arg(name):
"""
Shim for differing Python 2 + 3 representations of function args
"""
if arg is not None:
return arg(arg=name, annotation=None)
return Name(id=name, ctx=Param())
def rewrite_with_to_binds(body, monad):
new_body = []
# Construct a transformer for this specific monad's mreturn
rdb = RewriteDoBody(monad)
# This is the body of the lambda we're about to construct
last_part = body[-1].value
# Rewrite mreturn
rdb.visit(last_part)
# Iterate in reverse, making each line the into a lambda whose body is the
# rest of the lines (which are each lambdas), and whose names are the
# bind assignments.
for b in reversed(body[:-1]):
rdb.visit(b)
if isinstance(b, Assign):
name = b.targets[0].id
value = b.value
else:
# If there was no assignment to the bind, just use a random name, eek
name = '__DO_NOT_NAME_A_VARIABLE_THIS_STRING__'
value = b.value
# last part = value.bind(lambda name: last_part)
last_part = Call(func=Attribute(value=value, attr='bind', ctx=Load()),
args=[
Lambda(args=arguments(args=[
Name(id=name, ctx=Param()),
],
vararg=None,
kwarg=None,
defaults=[]),
body=last_part),
],
keywords=[],
starargs=None,
kwargs=None)
return last_part
def compile_tree(tree):
tree_node = analyze_tree(tree, None)
node = Interactive(body=[
FunctionDef(name='run_rules',
args=arguments(args=[Name(id='t', ctx=Param())],
vararg=None,
kwarg=None,
defaults=[]),
body=[tree_node],
decorator_list=[]),
])
fixed = fix_missing_locations(node)
code = compile(fixed, '<luca rule compiler>', 'single')
globals_dict = {'datetime': datetime, 'search': re.search}
eval(code, globals_dict)
run_rules = globals_dict['run_rules']
return run_rules
def pack(self):
compiler = AstCompiler()
requirements = set()
for scriptlet in self.scriptlets:
if type(scriptlet.dependencies) == dict:
for dependency in scriptlet.dependencies.get('all', []):
requirements.add(dependency)
for dependency in scriptlet.dependencies.get(self.os, []):
requirements.add(dependency)
else:
for dependency in scriptlet.dependencies:
requirements.add(dependency)
if requirements:
compiler.add_ast(
parse('\n'.join([
'import pupyimporter',
dependencies.importer(requirements, os=self.os)
]) + '\n'))
for scriptlet, kwargs in self.scriptlets.iteritems():
template = WRAPPING_TEMPLATE.format(scriptlet=scriptlet.name)
# Select part with proper OS if any
# Should be top-level if statement if string test
while True:
os_selection_idx = None
for idx, item in enumerate(scriptlet.ast.body):
if not (type(item) == If and type(item.test) == Str and \
item.test.s.startswith('__os:') and item.test.s.endswith('__')):
continue
os_selection_idx = idx
break
if os_selection_idx is None:
break
new_body = select_body_by_os(
scriptlet.ast.body[os_selection_idx], self.os)
scriptlet.ast.body = \
scriptlet.ast.body[:os_selection_idx] + \
new_body + scriptlet.ast.body[os_selection_idx+1:]
# Bind args
# There should be top level function main
main_found = False
shadow_kwargs = {'logger', 'pupy'}
for item in scriptlet.ast.body:
if not (type(item) == FunctionDef and item.name == 'main'):
continue
main_found = True
lineno = 0
col_offset = 0
item.name = scriptlet.name + '_main'
for idx, (arg, value) in enumerate(
zip(item.args.args, item.args.defaults)):
lineno = value.lineno
col_offset = value.col_offset
vtype = type(value)
if arg.id in shadow_kwargs:
shadow_kwargs.remove(arg.id)
elif arg.id in kwargs:
default = kwargs[arg.id]
if vtype == Num:
if type(default) not in (int, long):
default = str_to_int(default)
value.n = default
elif vtype == Str:
if type(default) not in (str, unicode):
default = str(default)
value.s = default
elif vtype == Name:
if value.id in ('True', 'False'):
if default.lower() in ('true', 'yes', 'on',
'1'):
value.id = 'True'
elif default.lower() in ('false', 'no', 'off',
'0'):
value.id = 'False'
else:
raise ValueError(
'Expect True/False value for {}'.
format(arg.id))
else:
new_value = None
try:
new_value = Num(str_to_int(default))
except ValueError:
new_value = Str(default)
new_value.lineno = value.lineno
new_value.col_offset = value.col_offset
item.args.defaults[idx] = new_value
elif vtype == Str and value.s.startswith(
'__global:') and value.s.endswith('__'):
global_name = value.s[9:-2]
global_ref = Name(global_name, Load())
global_ref.lineno = value.lineno
global_ref.col_offset = value.col_offset
item.args.defaults[idx] = global_ref
for idx, shadow_kwarg in enumerate(shadow_kwargs):
shadow_name = Name(shadow_kwarg, Param())
shadow_name.lineno = lineno
shadow_name.col_offset = col_offset + (idx * 16)
item.args.args.append(shadow_name)
shadow_value = Name('None', Load())
shadow_value.lineno = lineno
shadow_value.col_offset = col_offset + (idx * 16) + 7
item.args.defaults.append(shadow_value)
break
if not main_found:
raise ValueError('Scriptlet {} - Invalid source code. '
'"def main():" not found'.format(
scriptlet.name))
placeholder_idx = None
# Wrap in try/except, and other things
template_ast = parse(template)
for item in template_ast.body:
if not (type(item) == FunctionDef and item.name
== '__{}_closure__'.format(scriptlet.name)):
continue
assert (len(item.body) == 1
and type(item.body[0]) == TryExcept)
closure = item.body[0]
for idx, payload in enumerate(closure.body):
if type(payload) is not Expr:
continue
if type(payload.value
) is Str and payload.value.s == 'PLACEHOLDER':
placeholder_idx = idx
break
assert (placeholder_idx is not None)
closure.body = closure.body[:placeholder_idx] + scriptlet.ast.body + \
closure.body[placeholder_idx+1:]
break
if placeholder_idx is None:
raise ValueError(
'Template placeholder not found. Fill the bug report')
compiler.add_ast(template_ast)
return compiler.compile('sbundle', raw=True)
def compile_function_ast(expressions, symbols, arg_names, output_names=None, funname='anonymous', return_ast=False, print_code=False, definitions=None, vectorize=True, use_file=False):
'''
expressions: list of equations as string
'''
from collections import OrderedDict
table = OrderedDict()
aa = arg_names
if output_names is not None:
aa = arg_names + [output_names]
for a in aa:
symbol_group = a[0]
date = a[1]
an = a[2]
for b in symbols[symbol_group]:
index = symbols[symbol_group].index(b)
table[(b, date)] = (an, index)
table_symbols = {k: (std_date_symbol(*k)) for k in table.keys()}
# standard assignment: i.e. k = s[0]
index = lambda x: Index(Num(x))
# declare symbols
aux_short_names = [e[2] for e in arg_names if e[0]=='auxiliaries']
preamble = []
for k in table: # order it
# k : var, date
arg, pos = table[k]
if not (arg in aux_short_names):
std_name = table_symbols[k]
val = Subscript(value=Name(id=arg, ctx=Load()), slice=index(pos), ctx=Load())
line = Assign(targets=[Name(id=std_name, ctx=Store())], value=val)
if arg != 'out':
preamble.append(line)
body = []
std_dates = StandardizeDates(symbols, aa)
if definitions is not None:
for k,v in definitions.items():
if isinstance(k, str):
lhs = ast.parse(k).body[0].value
if isinstance(v, str):
rhs = ast.parse(v).body[0].value
else:
rhs = v
lhs = std_dates.visit(lhs)
rhs = std_dates.visit(rhs)
vname = lhs.id
line = Assign(targets=[Name(id=vname, ctx=Store())], value=rhs)
preamble.append(line)
outs = []
for i, expr in enumerate(expressions):
expr = ast.parse(expr).body[0].value
# if definitions is not None:
# expr = ReplaceName(defs).visit(expr)
rexpr = std_dates.visit(expr)
rhs = rexpr
if output_names is not None:
varname = symbols[output_names[0]][i]
date = output_names[1]
out_name = table_symbols[(varname, date)]
else:
out_name = 'out_{}'.format(i)
line = Assign(targets=[Name(id=out_name, ctx=Store())], value=rhs)
body.append(line)
line = Assign(targets=[Subscript(value=Name(id='out', ctx=Load()),
slice=index(i), ctx=Store())], value=Name(id=out_name, ctx=Load()))
body.append(line)
arg_names = [e for e in arg_names if e[0]!="auxiliaries"]
args = [e[2] for e in arg_names] + ['out']
if is_python_3:
from ast import arg
f = FunctionDef(name=funname, args=arguments(args=[arg(arg=a) for a in args], vararg=None, kwarg=None, kwonlyargs=[], kw_defaults=[], defaults=[]),
body=preamble + body, decorator_list=[])
else:
f = FunctionDef(name=funname, args=arguments(args=[Name(id=a, ctx=Param()) for a in args], vararg=None, kwarg=None, kwonlyargs=[], kw_defaults=[], defaults=[]),
body=preamble + body, decorator_list=[])
mod = Module(body=[f])
mod = ast.fix_missing_locations(mod)
if print_code:
s = "Function {}".format(mod.body[0].name)
print("-" * len(s))
print(s)
print("-" * len(s))
print(to_source(mod))
if vectorize:
from numba import float64, void
coredims = [len(symbols[an[0]]) for an in arg_names]
signature = str.join(',', ['(n_{})'.format(d) for d in coredims])
n_out = len(expressions)
if n_out in coredims:
signature += '->(n_{})'.format(n_out)
# ftylist = float64[:](*([float64[:]] * len(coredims)))
fty = "void(*[float64[:]]*{})".format(len(coredims)+1)
else:
signature += ',(n_{})'.format(n_out)
fty = "void(*[float64[:]]*{})".format(len(coredims)+1)
ftylist = [fty]
else:
signature=None
ftylist=None
if use_file:
fun = eval_ast_with_file(mod, print_code=True)
else:
fun = eval_ast(mod)
jitted = njit(fun)
if vectorize:
gufun = guvectorize([fty], signature, target='parallel', nopython=True)(fun)
return jitted, gufun
else:
return jitted
def visit_FunctionDef(self, node):
function_name = node.name
INTERNAL_CONTEXT_NAME = '___INJECT_CONTEXT_INTERNAL'
INTERNAL_RESOURCES_NAME = '___INJECT_CONTEXT_INTERNAL_RESOURCES'
last_existing_arg = node.args.args[
-1] # non-empty because we inject somewhere
new_args = node.args.args + [
copy_location(Name(id=INTERNAL_CONTEXT_NAME, ctx=Param()),
last_existing_arg)
]
last_existing_default = node.args.defaults[-1]
new_defaults = node.args.defaults + [
copy_location(Name(id=self.default_argument_name, ctx=Load()),
last_existing_default)
]
# Format: (Name(id='a', ctx=Param()), Name(id='foo', ctx=Load()))
def generate_default_nodes():
for (argument, default_value) in zip(
node.args.args[-len(node.args.defaults):],
node.args.defaults):
if argument.id in self.injected_arguments_set:
yield (argument.id, default_value)
default_nodes_mapping = dict(generate_default_nodes())
def _generate_assignment((arg_name, arg_resource_handle)):
# type: (Tuple[basestring, int]) -> If
"""
We have a function that looks like:
def do_something(param, model_=INJECTED):
<...>
We insert into its beginning a statement like
___INJECT_CONTEXT_INTERNAL_RESOURCES = ___INJECT_CONTEXT_INTERNAL.resources
if model_ is INJECTED:
model_ = ___INJECT_CONTEXT_INTERNAL_RESOURCES[3]
if model is ___INJECT_CONTEXT_INTERNAL: # means that no resource is available
___INJECT_CONTEXT_INTERNAL_RESOURCES.flag_missing('model_')
Code outside of this function sets a global variable _INJECTED__model to point at the right thing.
"""
target_attribute = Subscript(
value=Name(id=INTERNAL_RESOURCES_NAME, ctx=Load()),
slice=Index(value=Num(n=arg_resource_handle)),
ctx=Load())
consequence = [
Assign(targets=[Name(id=arg_name, ctx=Store())],
value=target_attribute),
If(test=Compare(
left=Name(id=arg_name, ctx=Load()),
ops=[Is()],
comparators=[Name(id=INTERNAL_CONTEXT_NAME, ctx=Load())]),
body=[
Expr(value=Call(func=Attribute(value=Name(
id=INTERNAL_CONTEXT_NAME, ctx=Load()),
attr='flag_missing',
ctx=Load()),
keywords=[],
starargs=None,
kwargs=None,
args=[Str(s=arg_name)]))
],
orelse=[])
] # type: List[Union[Assign, If]
return If(test=Compare(
left=Name(id=arg_name, ctx=Load()),
ops=[Is()],
comparators=[default_nodes_mapping[arg_name]]),
body=consequence,
orelse=[])
first_body_element = node.body[0]
align_with_first_body_element = partial(copy_location,
old_node=first_body_element)
new_body = [
align_with_first_body_element(
Assign(targets=[Name(id=INTERNAL_RESOURCES_NAME, ctx=Store())],
value=Attribute(value=Name(id=INTERNAL_CONTEXT_NAME,
ctx=Load()),
attr='resources',
ctx=Load())))
]
new_body.extend(
map(align_with_first_body_element,
map(_generate_assignment, self.parameters)))
new_body.extend(node.body)
result = copy_location(
FunctionDef(
name=function_name,
args=arguments(args=new_args,
vararg=node.args.vararg,
kwarg=node.args.kwarg,
defaults=new_defaults),
body=new_body,
decorator_list=[]
), # Note that no decorators are applied, as @inject has to be the first
node)
return result
def compile_function_ast(expressions,
symbols,
arg_names,
output_names=None,
funname='anonymous',
data_order='columns',
use_numexpr=False,
return_ast=False,
print_code=False,
definitions=None):
'''
expressions: list of equations as string
'''
vectorization_type = 'ellipsis'
data_order = 'columns'
from collections import OrderedDict
table = OrderedDict()
aa = arg_names
if output_names is not None:
aa = arg_names + [output_names]
for a in aa:
symbol_group = a[0]
date = a[1]
an = a[2]
for b in symbols[symbol_group]:
index = symbols[symbol_group].index(b)
table[(b, date)] = (an, index)
table_symbols = {k: (std_date_symbol(*k)) for k in table.keys()}
if data_order is None:
# standard assignment: i.e. k = s[0]
index = lambda x: Index(Num(x))
elif vectorization_type == 'ellipsis':
el = Ellipsis()
if data_order == 'columns':
# column broadcasting: i.e. k = s[...,0]
index = lambda x: ExtSlice(dims=[el, Index(value=Num(n=x))])
else:
# rows broadcasting: i.e. k = s[0,...]
index = lambda x: ExtSlice(dims=[Index(value=Num(n=x)), el])
# declare symbols
preamble = []
for k in table: # order it
# k : var, date
arg, pos = table[k]
std_name = table_symbols[k]
val = Subscript(value=Name(id=arg, ctx=Load()),
slice=index(pos),
ctx=Load())
line = Assign(targets=[Name(id=std_name, ctx=Store())], value=val)
preamble.append(line)
if use_numexpr:
for i in range(len(expressions)):
# k : var, date
val = Subscript(value=Name(id='out', ctx=Load()),
slice=index(i),
ctx=Load())
line = Assign(targets=[Name(id='out_{}'.format(i), ctx=Store())],
value=val)
preamble.append(line)
body = []
std_dates = StandardizeDates(symbols, aa)
if definitions is not None:
defs = {
e: ast.parse(definitions[e]).body[0].value
for e in definitions
}
for i, expr in enumerate(expressions):
expr = ast.parse(expr).body[0].value
if definitions is not None:
expr = ReplaceName(defs).visit(expr)
rexpr = std_dates.visit(expr)
if not use_numexpr:
rhs = rexpr
else:
import codegen
src = codegen.to_source(rexpr)
rhs = Call(func=Name(id='evaluate', ctx=Load()),
args=[Str(s=src)],
keywords=[
keyword(arg='out',
value=Name(id='out_{}'.format(i),
ctx=Load()))
],
starargs=None,
kwargs=None)
if not use_numexpr:
val = Subscript(value=Name(id='out', ctx=Load()),
slice=index(i),
ctx=Store())
line = Assign(targets=[val], value=rhs)
else:
line = Expr(
value=rhs
) #Assign(targets=[Name(id='out_{}'.format(i), ctx=Load())], value=rhs )
body.append(line)
if output_names is not None:
varname = symbols[output_names[0]][i]
date = output_names[1]
out_name = table_symbols[(varname, date)]
line = Assign(targets=[Name(id=out_name.format(i), ctx=Store())],
value=Name(id='out_{}'.format(i), ctx=Store()))
# body.append(line)
args = [e[2] for e in arg_names] + ['out']
f = FunctionDef(name=funname,
args=arguments(
args=[Name(id=a, ctx=Param()) for a in args],
vararg=None,
kwarg=None,
defaults=[]),
body=preamble + body,
decorator_list=[])
mod = Module(body=[f])
mod = ast.fix_missing_locations(mod)
# print_code=True
if print_code:
s = "Function {}".format(mod.body[0].name)
print("-" * len(s))
print(s)
print("-" * len(s))
import codegen
print(codegen.to_source(mod))
if return_ast:
return mod
else:
fun = eval_ast(mod)
return fun
def p_formal_param(self, p):
'''formal_param : type stars id_d'''
p[0] = Param(p[3].value, p[1], len(p[2]))
# Copyright (c) 2011, James Hanlon, All rights reserved
# This software is freely distributable under a derivative of the
# University of Illinois/NCSA Open Source License posted in
# LICENSE.txt and at <http://github.xcore.com/>
# This module defines processes and functions which are built-in to the
# language. Those marked as mobile will be added to the jump table and will be
# executable remotely.
from ast import ProcDef, Param
from typedefs import *
SVAL_PARAM = Param('v', T_VAL_SINGLE, None)
SREF_PARAM = Param('v', T_REF_SINGLE, None)
AVAL_PARAM = Param('v', T_REF_ARRAY, None)
CHANEND_PARAM = Param('v', T_CHANEND_SINGLE, None)
class Builtin(object):
"""
A class to represent a builtin and its mobility.
"""
def __init__(self, definition, mobile):
self.definition = definition
self.mobile = mobile
# Create a process declaration (prototype).
def proc_decl(name, params, mobile=False):
return Builtin(ProcDef(name, T_PROC, params, None, None), mobile)
def generate_param_name(name):
return Name(id=name, ctx=Param())
def create_param_name(id):
"creates a named parameter"
return Name(id=id, ctx=Param())