def SoS_exec(script: str,
_dict: dict = None,
return_result: bool = True) -> None:
"""Execute a statement."""
if _dict is None:
_dict = env.sos_dict.dict()
if not return_result:
if env.verbosity == 0:
with contextlib.redirect_stdout(None):
exec(
compile(script,
filename=stmtHash.hash(script),
mode="exec"), _dict)
else:
exec(compile(script, filename=stmtHash.hash(script), mode="exec"),
_dict)
return None
try:
stmts = list(ast.iter_child_nodes(ast.parse(script)))
if not stmts:
return
if isinstance(stmts[-1], ast.Expr):
# the last one is an expression and we will try to return the results
# so we first execute the previous statements
if len(stmts) > 1:
if env.verbosity == 0:
with contextlib.redirect_stdout(None):
exec(
compile(
ast.Module(body=stmts[:-1], type_ignores=[]),
filename=stmtHash.hash(script),
mode="exec",
),
_dict,
)
else:
exec(
compile(
ast.Module(body=stmts[:-1], type_ignores=[]),
filename=stmtHash.hash(script),
mode="exec",
),
_dict,
)
# then we eval the last one
if env.verbosity == 0:
with contextlib.redirect_stdout(None):
res = eval(
compile(
ast.Expression(body=stmts[-1].value),
filename=stmtHash.hash(script),
mode="eval",
),
_dict,
)
else:
res = eval(
compile(
ast.Expression(body=stmts[-1].value),
filename=stmtHash.hash(script),
mode="eval",
),
_dict,
)
else:
# otherwise we just execute the entire code
if env.verbosity == 0:
with contextlib.redirect_stdout(None):
exec(
compile(script,
filename=stmtHash.hash(script),
mode="exec"),
_dict,
)
else:
exec(
compile(script,
filename=stmtHash.hash(script),
mode="exec"), _dict)
res = None
except SyntaxError as e:
raise SyntaxError(f"Invalid code {script}: {e}")
# if check_readonly:
# env.sos_dict.check_readonly_vars()
return res
def test_module(self):
body = [ast.Num(42)]
x = ast.Module(body)
self.assertEqual(x.body, body)
def stmt(self, stmt, msg=None):
mod = ast.Module([stmt])
self.mod(mod, msg)
def test_invalid_string(self):
m = ast.Module([ast.Expr(ast.Str(42))])
ast.fix_missing_locations(m)
with self.assertRaises(TypeError) as cm:
compile(m, "<test>", "exec")
self.assertIn("string must be of type str", str(cm.exception))
def make_module(*stmts):
m = ast.Module(list(stmts))
return ast.copy_location(m, stmts[0]) if stmts else m
def test_add_single_name_from_import(self):
tree = ast.Module(body=[])
self.assertEqual(
'foo', import_utils.add_import(tree, 'foo', from_import=True))
self.assertEqual('import foo\n', pasta.dump(tree))
def test_invalid_sum(self):
pos = dict(lineno=2, col_offset=3)
m = ast.Module([ast.Expr(ast.expr(**pos), **pos)])
with self.assertRaises(TypeError) as cm:
compile(m, "<test>", "exec")
self.assertIn("but got <_ast.expr", str(cm.exception))
def compile_func(gen: 'Generator',
func: Callable,
strategy: Strategy,
with_hooks: bool = False) -> Callable:
"""
The compilation basically assigns functionality to each of the operator calls as
governed by the semantics (strategy). Memoization is done with the keys as the `func`,
the class of the `strategy` and the `with_hooks` argument.
Args:
gen (Generator): The generator object containing the function to compile
func (Callable): The function to compile
strategy (Strategy): The strategy governing the behavior of the operators
with_hooks (bool): Whether support for hooks is required
Returns:
The compiled function
"""
if isinstance(strategy, PartialReplayStrategy):
strategy = strategy.backup_strategy
if with_hooks:
cache = CompilationCache.WITH_HOOKS[strategy.__class__]
else:
cache = CompilationCache.WITHOUT_HOOKS[strategy.__class__]
if func in cache:
return cache[func]
cache[func] = None
source_code, start_lineno = inspect.getsourcelines(func)
source_code = ''.join(source_code)
f_ast = astutils.parse(textwrap.dedent(source_code))
# This matches up line numbers with original file and is thus super useful for debugging
ast.increment_lineno(f_ast, start_lineno - 1)
# Remove the ``@generator`` decorator to avoid recursive compilation
f_ast.decorator_list = [
d for d in f_ast.decorator_list
if (not isinstance(d, ast.Name) or d.id != 'generator') and (
not isinstance(d, ast.Attribute) or d.attr != 'generator') and (
not (isinstance(d, ast.Call) and isinstance(d.func, ast.Name))
or d.func.id != 'generator')
]
# Get all the external dependencies of this function.
# We rely on a modified closure function adopted from the ``inspect`` library.
closure_vars = getclosurevars_recursive(func, f_ast)
g = {**closure_vars.nonlocals.copy(), **closure_vars.globals.copy()}
known_ops: Set[str] = strategy.get_known_ops()
known_methods: Set[str] = strategy.get_known_methods()
op_info_constructor = OpInfoConstructor()
delayed_compilations: List[Tuple[Generator, str]] = []
ops = {}
handlers = {}
op_infos = {}
op_idx: int = 0
composition_cnt: int = 0
for n in astutils.preorder_traversal(f_ast):
if isinstance(n, ast.Call) and isinstance(
n.func, ast.Name) and n.func.id in known_ops:
# Rename the function call, and assign a new function to be called during execution.
# This new function is determined by the semantics (strategy) being used for compilation.
# Also determine if there any eligible hooks for this operator call.
op_idx += 1
handler_idx = len(handlers)
op_info: OpInfo = op_info_constructor.get(n, gen.name, gen.group)
n.keywords.append(
ast.keyword(arg='model',
value=ast.Name(_GEN_MODEL_VAR, ast.Load())))
n.keywords.append(
ast.keyword(arg='op_info',
value=ast.Name(f"_op_info_{op_idx}", ast.Load())))
op_infos[f"_op_info_{op_idx}"] = op_info
n.keywords.append(
ast.keyword(arg='handler',
value=ast.Name(f"_handler_{handler_idx}",
ast.Load())))
handler = strategy.get_op_handler(op_info)
handlers[f"_handler_{handler_idx}"] = handler
if not with_hooks:
n.func = astutils.parse(
f"{_GEN_STRATEGY_VAR}.generic_op").value
else:
n.keywords.append(
ast.keyword(arg=_GEN_HOOK_VAR,
value=ast.Name(_GEN_HOOK_VAR, ctx=ast.Load())))
n.keywords.append(
ast.keyword(arg=_GEN_STRATEGY_VAR,
value=ast.Name(_GEN_STRATEGY_VAR,
ctx=ast.Load())))
n.func.id = _GEN_HOOK_WRAPPER
ops[_GEN_HOOK_WRAPPER] = hook_wrapper
if returns_lambda(handler):
n.func = ast.Call(func=n.func,
args=n.args[:],
keywords=n.keywords[:])
n.keywords = []
n.args = [n.args[0]]
ast.fix_missing_locations(n)
elif isinstance(n, ast.Call) and isinstance(
n.func, ast.Name) and n.func.id in known_methods:
# Similar in spirit to the known_ops case, just much less fancy stuff to do.
# Only need to get the right handler which we will achieve by simply making this
# a method call instead of a regular call.
n.func = ast.Attribute(value=ast.Name(_GEN_STRATEGY_VAR,
ctx=ast.Load()),
attr=n.func.id,
ctx=ast.Load())
ast.fix_missing_locations(n)
elif isinstance(n, ast.Call):
# Try to check if it is a call to a Generator
# TODO : Can we be more sophisticated in our static analysis here
try:
function = eval(astunparse.unparse(n.func), g)
except:
continue
if isinstance(function, Generator):
call_id = f"{_GEN_COMPOSITION_ID}_{composition_cnt}"
composition_cnt += 1
n.func.id = call_id
n.keywords.append(
ast.keyword(arg=_GEN_EXEC_ENV_VAR,
value=ast.Name(_GEN_EXEC_ENV_VAR, ast.Load())))
n.keywords.append(
ast.keyword(arg=_GEN_STRATEGY_VAR,
value=ast.Name(_GEN_STRATEGY_VAR, ast.Load())))
n.keywords.append(
ast.keyword(arg=_GEN_MODEL_VAR,
value=ast.Name(_GEN_MODEL_VAR, ast.Load())))
n.keywords.append(
ast.keyword(arg=_GEN_HOOK_VAR,
value=ast.Name(_GEN_HOOK_VAR, ast.Load())))
ast.fix_missing_locations(n)
# We delay compilation to handle mutually recursive generators
delayed_compilations.append((function, call_id))
elif function is CallGenerator:
wrapped_func = n.args[0]
n.func = wrapped_func.func
n.args = wrapped_func.args[:]
n.keywords = wrapped_func.keywords[:]
n.keywords.append(
ast.keyword(arg=_GEN_EXEC_ENV_VAR,
value=ast.Name(_GEN_EXEC_ENV_VAR, ast.Load())))
ast.fix_missing_locations(n)
# Add the execution environment argument to the function
f_ast.args.kwonlyargs.append(
ast.arg(arg=_GEN_EXEC_ENV_VAR, annotation=None))
f_ast.args.kw_defaults.append(ast.NameConstant(value=None))
# Add the strategy argument to the function
f_ast.args.kwonlyargs.append(
ast.arg(arg=_GEN_STRATEGY_VAR, annotation=None))
f_ast.args.kw_defaults.append(ast.NameConstant(value=None))
# Add the strategy argument to the function
f_ast.args.kwonlyargs.append(ast.arg(arg=_GEN_MODEL_VAR, annotation=None))
f_ast.args.kw_defaults.append(ast.NameConstant(value=None))
# Add the hook argument to the function
f_ast.args.kwonlyargs.append(ast.arg(arg=_GEN_HOOK_VAR, annotation=None))
f_ast.args.kw_defaults.append(ast.NameConstant(value=None))
ast.fix_missing_locations(f_ast)
# New name so it doesn't clash with original
func_name = f"{_GEN_COMPILED_TARGET_ID}_{len(cache)}"
g.update({k: v for k, v in ops.items()})
g.update({k: v for k, v in handlers.items()})
g.update({k: v for k, v in op_infos.items()})
module = ast.Module()
module.body = [f_ast]
# Passing ``g`` to exec allows us to execute all the new functions
# we assigned to every operator call in the previous AST walk
filename = inspect.getabsfile(func)
exec(compile(module, filename=filename, mode="exec"), g)
result = g[func.__name__]
g["__name__"] = filename
if inspect.ismethod(func):
result = result.__get__(func.__self__, func.__self__.__class__)
# Restore the correct namespace so that tracebacks contain actual function names
g[gen.name] = gen
g[func_name] = result
cache[func] = result
# Handle the delayed compilations now that we have populated the cache
for gen, call_id in delayed_compilations:
compiled_func = compile_func(gen, gen.func, strategy, with_hooks)
if gen.caching and isinstance(strategy, DfsStrategy):
# Add instructions for using cached result if any
g[call_id] = cache_wrapper(compiled_func)
else:
g[call_id] = compiled_func
return result
def test_empty_init(self):
# Jython 2.5.0 did not allow empty constructors for many ast node types
# but CPython ast nodes do allow this. For the moment, I don't see a
# reason to allow construction of the super types (like ast.AST and
# ast.stmt) as well as the op types that are implemented as enums in
# Jython (like boolop), but I've left them in but commented out for
# now. We may need them in the future since CPython allows this, but
# it may fall under implementation detail.
#ast.AST()
ast.Add()
ast.And()
ast.Assert()
ast.Assign()
ast.Attribute()
ast.AugAssign()
ast.AugLoad()
ast.AugStore()
ast.BinOp()
ast.BitAnd()
ast.BitOr()
ast.BitXor()
ast.BoolOp()
ast.Break()
ast.Call()
ast.ClassDef()
ast.Compare()
ast.Continue()
ast.Del()
ast.Delete()
ast.Dict()
ast.Div()
ast.Ellipsis()
ast.Eq()
ast.Exec()
ast.Expr()
ast.Expression()
ast.ExtSlice()
ast.FloorDiv()
ast.For()
ast.FunctionDef()
ast.GeneratorExp()
ast.Global()
ast.Gt()
ast.GtE()
ast.If()
ast.IfExp()
ast.Import()
ast.ImportFrom()
ast.In()
ast.Index()
ast.Interactive()
ast.Invert()
ast.Is()
ast.IsNot()
ast.LShift()
ast.Lambda()
ast.List()
ast.ListComp()
ast.Load()
ast.Lt()
ast.LtE()
ast.Mod()
ast.Module()
ast.Mult()
ast.Name()
ast.Not()
ast.NotEq()
ast.NotIn()
ast.Num()
ast.Or()
ast.Param()
ast.Pass()
ast.Pow()
ast.Print()
ast.RShift()
ast.Raise()
ast.Repr()
ast.Return()
ast.Slice()
ast.Store()
ast.Str()
ast.Sub()
ast.Subscript()
ast.Suite()
ast.TryExcept()
ast.TryFinally()
ast.Tuple()
ast.UAdd()
ast.USub()
ast.UnaryOp()
ast.While()
ast.With()
ast.Yield()
ast.alias()
ast.arguments()
#ast.boolop()
#ast.cmpop()
ast.comprehension()
#ast.excepthandler()
#ast.expr()
#ast.expr_context()
ast.keyword()
def __init__(self):
"""Instantiate translator."""
import ast
self.c_hello = GreeterTranslator(ast.Module())
def sample_from_root(memsize_k, memsize_i, callback):
root = MyAST(parent=None, node=ast.Module())
root._sample(memsize_k, memsize_i, callback)
return root
jconf = entry_point.getSparkConf()
jsc = entry_point.getJavaSparkContext()
conf = SparkConf(_jvm=gateway.jvm, _jconf=jconf)
sc = SparkContext(jsc=jsc, gateway=gateway, conf=conf)
spark = SparkSession(sc, entry_point.getSparkSession())
sqlc = spark._wrapped
ama_context = AmaContext(sc, sqlc)
while True:
actionData = queue.getNext()
resultQueue = entry_point.getResultQueue(actionData._2())
actionSource = actionData._1()
tree = ast.parse(actionSource)
for node in tree.body:
wrapper = ast.Module(body=[node])
try:
co = compile(wrapper, "<ast>", 'exec')
exec(co)
resultQueue.put('success', actionData._2(),
codegen.to_source(node), '')
except:
resultQueue.put('error', actionData._2(), codegen.to_source(node),
str(sys.exc_info()[1]))
resultQueue.put('completion', '', '', '')
def run_cell(droplet_client, code_str, namespace):
import ast
import sys
import time
import cloudpickle as cp
import collections.abc
# https://docs.python.org/3/library/collections.abc.html
import anna.lattices
"""
CachedDict provides a caching wrapper around any object
that exposes a dict interface.
Does not ensure freshness of get().
"""
class CachedDict(collections.abc.MutableMapping):
def __init__(self, dict_to_wrap):
# dict_to_wrap: the dictionary object to wrap.
self.dict_to_wrap = dict_to_wrap
self.dict_cache = dict()
def __getitem__(self, key, refresh=False):
# If we are asking for a refresh,
# or item is not cached, fetch it.
# Then, serves the item from cache.
# Note: This does not deal with staleness.
if refresh or (key not in self.dict_cache):
self.dict_cache[key] = self.dict_to_wrap[key]
# If this causes a KeyError from dict_to_wrap,
# we want it to propagate upwards, so we leave it uncaught.
return self.dict_cache[key]
def __setitem__(self, key, value):
# Write through to backing dict.
self.dict_to_wrap[key] = value
# Write to cache too.
self.dict_cache[key] = value
def __delitem__(self, key):
del self.dict_to_wrap[key]
del self.dict_cache[key]
def __iter__(self):
# Currently: just bypass the cache entirely.
return self.dict_to_wrap.__iter__()
def __len__(self):
# Currently: just bypass the cache entirely.
return self.dict_to_wrap.__len__()
"""
HydroBackedDict is a dict whose contents are backed by
the Hydro KVS instead of residing in process memory.
"""
class HydroBackedDict(collections.abc.MutableMapping):
# namespace is the permanent global namespace
# that will be used to store this dict in Hydro.
def __init__(self, droplet_client, namespace):
self.droplet_client = droplet_client
self.namespace = namespace
def __getitem__(self, key):
# Convert the key into a global key.
global_key = self._get_global_key(key)
# Use this key to get the raw bytes from the KVS.
item = self.droplet_client.get(global_key)
if item is None: raise KeyError(key)
# Deserialize not necessary?.
return item
# key: str
# value: any Python value
def __setitem__(self, key, value):
# Convert the key to a global key.
key = self._get_global_key(key)
# Serialize the value.
value = cp.dumps(value)
# Store it into the kvs.
self.droplet_client.put(key, value)
def __delitem__(self, key):
raise NotImplementedError
def __iter__(self):
raise NotImplementedError
def __len__(self):
raise NotImplementedError
# Helper that converts a key for this dict
# to a global key "<namespace>:key:<key>".
def _get_global_key(self, key):
return "{}:key:{}".format(self.namespace, key)
# Capture stdout and stderr.
output = []
class logger:
def __init__(self, tag):
self.tag = tag
def write(self, data):
output.append((self.tag, data))
sys.stdout = logger('stdout')
sys.stderr = logger('stderr')
# Initialize the program state dictionary.
hydro_user_ns = HydroBackedDict(droplet_client, namespace)
# user_ns = CachedDict(hydro_user_ns)
user_ns = hydro_user_ns # Disable app-level caching for now.
# Parse the code to execute.
nodelist = ast.parse(code_str).body
# The semantics of cell execution are that we run all statements,
# but only return the result of the last one.
nodes_exec = nodelist[:-1]
nodes_interactive = nodelist[-1:]
bytecodes = []
for node in nodes_exec:
node = ast.Module([node])
bytecode = compile(node, '<string>', 'exec')
bytecodes.append(bytecode)
for node in nodes_interactive:
node = ast.Interactive([node])
bytecode = compile(node, '<string>', 'single')
bytecodes.append(bytecode)
try:
for bytecode in bytecodes:
exec(bytecode, globals(), user_ns)
except:
sys.stderr.write(str(sys.exc_info()[1]))
# XXX DROPLET HACK
# Droplet can't handle empty lists, so we append None.
# output.append(None)
# XXX DROPLET HACK
# Droplet sorts lists, so we number the list.
output = list(enumerate(output))
# Also, get the execution count.
exc_count = droplet_client.get('{}:exc_count'.format(namespace))
if exc_count is None: exc_count = 0
exc_count += 1
droplet_client.put('{}:exc_count'.format(namespace), exc_count)
return (exc_count, output)
def test_execute_function(self) -> None:
module = ast.Module(body=[])
concat.execute.execute('<test>', module, {})
def test_add_normal_import(self):
tree = ast.Module(body=[])
self.assertEqual(
'a.b.c', import_utils.add_import(tree, 'a.b.c', from_import=False))
self.assertEqual('import a.b.c\n', pasta.dump(tree))
def _process_function(self, function, namespace, arguments=[]):
"""
Processes the arguments and body of a user-defined function. If the
function is recursive, the recursive calls are not processed (otherwise
this would throw `Saplings` into an infinite loop). If the function
returns a closure, that function is added to the list of functions in
the current scope.
Parameters
----------
function : Function
function that's being called
namespace : dict
namespace within which the function should be processed
arguments : list, optional
arguments passed into the function when called (as a list of
ArgTokens)
Returns
-------
{ObjectNode, Function, Class, ClassInstance, None}
namespace entity corresponding to the return value of the function;
None if the function has no return value or returns something we
don't care about (i.e. something that isn't tracked in the
namespace)
"""
parameters = function.def_node.args
pos_params = [a.arg for a in parameters.args]
kw_params = [a.arg for a in parameters.kwonlyargs]
# Namespace entities corresponding to default values
default_entities, null_defaults = self._process_default_args(
pos_params, parameters.defaults, namespace)
kw_default_entities, null_kw_defaults = self._process_default_args(
kw_params, parameters.kw_defaults, namespace)
# Update namespace with default values
namespace = {**namespace, **default_entities, **kw_default_entities}
for null_arg_name in null_defaults + null_kw_defaults:
if null_arg_name in namespace:
del namespace[null_arg_name]
utils.delete_sub_aliases(null_arg_name, namespace)
for index, argument in enumerate(arguments):
if argument.arg_name == '': # Positional argument
if index < len(pos_params):
arg_name = pos_params[index]
else: # *arg
self._process_attribute_chain(argument.arg_val)
continue
elif argument.arg_name is not None: # Keyword argument
arg_name = argument.arg_name
if arg_name not in pos_params + kw_params: # **kwargs
self._process_attribute_chain(argument.arg_val)
continue
else: # **kwargs
self._process_attribute_chain(argument.arg_val)
continue
arg_entity, _ = self._process_attribute_chain(argument.arg_val)
if not arg_entity:
if arg_name in namespace:
del namespace[arg_name]
utils.delete_sub_aliases(arg_name, namespace)
continue
utils.delete_sub_aliases(arg_name, namespace)
namespace[arg_name] = arg_entity
# TODO (V2): Handle star args and **kwargs (blocked by data structure
# handling)
if parameters.vararg and parameters.vararg.arg in namespace:
del namespace[parameters.vararg.arg]
utils.delete_sub_aliases(parameters.vararg.arg, namespace)
if parameters.kwarg and parameters.kwarg.arg in namespace:
del namespace[parameters.kwarg.arg]
utils.delete_sub_aliases(parameters.kwarg.arg, namespace)
# Handles recursive functions by deleting all names of the function node
for name, node in list(namespace.items()):
if node == function:
del namespace[name]
# Processes function body
func_saplings = self._process_subtree_in_new_scope(
ast.Module(body=function.def_node.body), namespace)
function.called = True
return_value = func_saplings._return_value
# If the function returns a closure then treat it like a function
# defined in the current scope by adding it to self._functions
if isinstance(return_value, Function):
return_value.is_closure = True
# TODO (V1): What if the function is defined in an outer scope but
# returned in this scope? Then it's not a closure. Handle these.
if not return_value.called:
self._functions.add(return_value)
elif isinstance(return_value, Class):
for name, entity in return_value.init_instance_namespace.items():
if isinstance(entity, Function):
entity.is_closure = True
if not entity.called:
self._functions.add(entity)
return return_value, func_saplings
def test_add_from_import(self):
tree = ast.Module(body=[])
self.assertEqual(
'c', import_utils.add_import(tree, 'a.b.c', from_import=True))
self.assertEqual('from a.b import c\n', pasta.dump(tree))
def visit_ClassDef(self, node):
"""
TODO
"""
for base_node in node.bases: # TODO (V2): Handle inheritance
self.visit(ast.Call(func=base_node, args=[], keywords=[]))
# TODO (V2): Handle metaclasses
methods, nested_classes, static_variables = [], [], []
stripped_body = [] # ;)
for n in node.body:
if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)):
for decorator in n.decorator_list:
if not hasattr(decorator, "id"):
continue
if decorator.id == "staticmethod":
n.method_type = "static"
break
elif decorator.id == "classmethod":
n.method_type = "class"
break
else:
n.method_type = "instance"
methods.append(n)
continue
elif isinstance(n, ast.ClassDef):
nested_classes.append(n)
continue
elif isinstance(n, (ast.Assign, ast.AnnAssign, ast.AugAssign)):
# BUG: Static variables can be defined without an assignment.
# For example:
# class foo(object):
# for x in range(10):
# pass
# foo.x is valid.
targets = [n.target
] if not isinstance(n, ast.Assign) else n.targets
for target in targets:
if isinstance(target, ast.Tuple):
for element in target.elts:
static_variables.append(
utils.stringify_node(element))
else:
static_variables.append(utils.stringify_node(target))
stripped_body.append(n)
class_level_namespace = self._process_subtree_in_new_scope(
ast.Module(body=stripped_body), self._namespace.copy())._namespace
class_entity = Class(node, self._namespace.copy())
self._namespace[node.name] = class_entity
static_variable_map = {}
for name, n in class_level_namespace.items():
if name in static_variables:
self._namespace['.'.join((node.name, name))] = n
static_variable_map[name] = n
def create_callable_attribute_map(callables):
callable_map = {}
for callable in callables:
callable_name = callable.name
# Handles callables that are accessed by the enclosing class
adjusted_name = '.'.join((node.name, callable_name))
callable.name = adjusted_name
if isinstance(callable,
(ast.FunctionDef, ast.AsyncFunctionDef)):
entity = self.visit_FunctionDef(callable)
elif isinstance(callable, ast.ClassDef):
entity = self.visit_ClassDef(callable)
callable.name = callable_name
if isinstance(entity, Function):
entity.method_type = callable.method_type
entity.containing_class = class_entity
callable_map[callable_name] = entity
return callable_map
method_map = create_callable_attribute_map(methods)
nested_class_map = create_callable_attribute_map(nested_classes)
# Everything here is an attribute of `self`
class_entity.init_instance_namespace = {
**static_variable_map,
**method_map,
**nested_class_map
}
return class_entity
def test_add_import_after_docstring(self):
tree = ast.Module(body=[ast.Expr(value=ast.Str(s='Docstring.'))])
self.assertEqual('a', import_utils.add_import(tree, 'a'))
self.assertEqual('\'Docstring.\'\nimport a\n', pasta.dump(tree))
if final_code:
# use exec mode to compile the statements except the last statement,
# so that the last statement's evaluation will be printed to stdout
code = compile('\n'.join(final_code), '<stdin>', 'exec',
ast.PyCF_ONLY_AST, 1)
to_run_hooks = []
if (nhooks > 0):
to_run_hooks = code.body[-nhooks:]
to_run_exec, to_run_single = (code.body[:-(nhooks + 1)],
[code.body[-(nhooks + 1)]])
try:
for node in to_run_exec:
mod = ast.Module([node])
code = compile(mod, '<stdin>', 'exec')
exec(code, _zcUserQueryNameSpace)
for node in to_run_single:
mod = ast.Interactive([node])
code = compile(mod, '<stdin>', 'single')
exec(code, _zcUserQueryNameSpace)
for node in to_run_hooks:
mod = ast.Module([node])
code = compile(mod, '<stdin>', 'exec')
exec(code, _zcUserQueryNameSpace)
except:
raise Exception(traceback.format_exc())
def test_invalid_identitifer(self):
m = ast.Module([ast.Expr(ast.Name(42, ast.Load()))])
ast.fix_missing_locations(m)
with self.assertRaises(TypeError) as cm:
compile(m, "<test>", "exec")
self.assertIn("identifier must be of type str", str(cm.exception))
def visit_Module(self, node):
self.generic_visit(node)
importIt = ast.Import(names=[ast.alias(name='itertools', asname=None)])
return ast.Module(body=([importIt] + node.body))
def _debug_scope(func):
tree = ast.Module(body=get_body_ast(func))
fill_scopes(tree)
print tree.scope # Only shown then test fails
return tree
def decorator(f):
tree = ast.parse("with 0:\n" + inspect.getsource(f))
func_decl = tree.body[0].body[0]
#print(ast.dump(func_decl))
#print()
#print()
name = f.__name__
start_block, init = ParsePass().parse_func(f, func_decl)
start_block = SimplifyPass().simplify(start_block)
firstline = f.__code__.co_firstlineno
states = MergePass().generate_states(start_block, firstline)
ast_stmts = []
for state in states:
ast_stmts.append(AstGenerator(wildcard_import).generate_ast(state))
# print([state.dump() for state in states])
fsm_name = None
if wildcard_import:
fsm_name = ast.Name(id="FSM", ctx=ast.Load(), lineno=firstline, col_offset=0)
else:
fsm_name = ast.Attribute(
value=ast.Name(id="migen", ctx=ast.Load(), lineno=firstline, col_offset=0),
attr="FSM",
ctx=ast.Load(),
lineno=firstline,
col_offset=0)
prologue = [
ast.Assign(
targets=[ast.Name(id="fsm", ctx=ast.Store(), lineno=firstline, col_offset=0)],
value=ast.Call(
func=fsm_name,
args=[],
keywords=[
ast.keyword(
arg="reset_state",
value=ast.Str(states[0].name, lineno=firstline, col_offset=0),
lineno=firstline,
col_offset=0)
],
lineno=firstline,
col_offset=0),
lineno=firstline,
col_offset=0)
]
epilogue = [
ast.Return(
value=ast.Name(id="fsm", ctx=ast.Load(), lineno=firstline, col_offset=0),
lineno=firstline,
col_offset=0)
]
new_body = prologue + init + ast_stmts + epilogue
new_func_decl = ast.Module(body=[
ast.FunctionDef(
name=func_decl.name,
args=func_decl.args,
body=new_body,
decorator_list=func_decl.decorator_list[:-1], # remove this decorator from the list
returns=func_decl.returns,
lineno=firstline,
col_offset=0)
])
# print(ast.dump(new_func_decl))
if astor is not None:
f.fsmgen_source = astor.to_source(new_func_decl)
p = compile(new_func_decl, "<ast>", mode="exec")
exec(p)
f.__code__ = locals()[func_decl.name].__code__
return f
def run_ast_nodes(self,
nodelist,
cell_name,
interactivity='last_expr',
compiler=compile,
result=None):
"""Run a sequence of AST nodes. The execution mode depends on the
interactivity parameter.
Parameters
----------
nodelist : list
A sequence of AST nodes to run.
cell_name : str
Will be passed to the compiler as the filename of the cell. Typically
the value returned by ip.compile.cache(cell).
interactivity : str
'all', 'last', 'last_expr' or 'none', specifying which nodes should be
run interactively (displaying output from expressions). 'last_expr'
will run the last node interactively only if it is an expression (i.e.
expressions in loops or other blocks are not displayed. Other values
for this parameter will raise a ValueError.
compiler : callable
A function with the same interface as the built-in compile(), to turn
the AST nodes into code objects. Default is the built-in compile().
result : ExecutionResult, optional
An object to store exceptions that occur during execution.
Returns
-------
True if an exception occurred while running code, False if it finished
running.
"""
if not nodelist:
return
if interactivity == 'last_expr':
if isinstance(nodelist[-1], ast.Expr):
interactivity = "last"
else:
interactivity = "none"
if interactivity == 'none':
to_run_exec, to_run_interactive = nodelist, []
elif interactivity == 'last':
to_run_exec, to_run_interactive = nodelist[:-1], nodelist[-1:]
elif interactivity == 'all':
to_run_exec, to_run_interactive = [], nodelist
else:
raise ValueError("Interactivity was %r" % interactivity)
try:
for i, node in enumerate(to_run_exec):
mod = ast.Module([node])
code = compiler(mod, cell_name, "exec")
if self.run_code(code, result):
return True
for i, node in enumerate(to_run_interactive):
mod = ast.Interactive([node])
code = compiler(mod, cell_name, "single")
if self.run_code(code, result):
return True
# Flush softspace
if softspace(sys.stdout, 0):
print()
except:
# It's possible to have exceptions raised here, typically by
# compilation of odd code (such as a naked 'return' outside a
# function) that did parse but isn't valid. Typically the exception
# is a SyntaxError, but it's safest just to catch anything and show
# the user a traceback.
# We do only one try/except outside the loop to minimize the impact
# on runtime, and also because if any node in the node list is
# broken, we should stop execution completely.
if result:
result.error_before_exec = sys.exc_info()[1]
self.showtraceback()
return True
return False
def gen_test(self, fn_template, fol, verdict_var, bindings=None):
if isinstance(fol, str):
return self.str_to_ast('%(verdict_var)s = %(constant)s' % {
'verdict_var': verdict_var,
'constant': fol
})
(k, v), = fol.items()
if k == 'policy':
policy_name, object_name = v
policy_fn = fn_template % policy_name
call_str = """
if obj.%(object_name)s:
%(verdict_var)s = %(policy_fn)s(obj.%(object_name)s, ctx)
else:
# Everybody has access to null objects
%(verdict_var)s = True
""" % {
'verdict_var': verdict_var,
'policy_fn': policy_fn,
'object_name': object_name
}
call_ast = self.str_to_ast(call_str)
return call_ast
if k == 'python':
try:
expr_ast = self.str_to_ast(v)
except SyntaxError:
raise PolicyException('Syntax error in %s' % v)
if not isinstance(expr_ast, ast.Expr):
raise PolicyException('%s is not an expression' % expr_ast)
assignment_str = """
%(verdict_var)s = (%(escape_expr)s)
""" % {
'verdict_var': verdict_var,
'escape_expr': v
}
assignment_ast = self.str_to_ast(assignment_str)
return assignment_ast
elif k == 'not':
top_vvar = verdict_var
self.verdict_next()
sub_vvar = self.verdict_variable
block = self.gen_test(fn_template, v, sub_vvar)
assignment_str = """
%(verdict_var)s = not (%(subvar)s)
""" % {
'verdict_var': top_vvar,
'subvar': sub_vvar
}
assignment_ast = self.str_to_ast(assignment_str)
return ast.Module(body=[block, assignment_ast])
elif k in ['=', 'in']:
# This is the simplest case, we don't recurse further
# To use terms that are not simple variables, use
# the Python escape, e.g. {{ slice.creator is not None }}
lhs, rhs = v
assignments = []
try:
for t in lhs, rhs:
py_expr = t['python']
self.verdict_next()
vv = self.verdict_variable
try:
expr_ast = self.str_to_ast(py_expr)
except SyntaxError:
raise PolicyException('Syntax error in %s' % v)
if not isinstance(expr_ast, ast.Expr):
raise PolicyException('%s is not an expression' %
expr_ast)
assignment_str = """
%(verdict_var)s = (%(escape_expr)s)
""" % {
'verdict_var': vv,
'escape_expr': py_expr
}
if t == lhs:
lhs = vv
else:
rhs = vv
assignment_ast = self.str_to_ast(assignment_str)
assignments.append(assignment_ast)
except TypeError:
pass
if k == '=':
operator = '=='
elif k == 'in':
operator = 'in'
comparison_str = """
%(verdict_var)s = (%(lhs)s %(operator)s %(rhs)s)
""" % {
'verdict_var': verdict_var,
'lhs': lhs,
'rhs': rhs,
'operator': operator
}
comparison_ast = self.str_to_ast(comparison_str)
combined_ast = ast.Module(body=assignments + [comparison_ast])
return combined_ast
elif k in BINOPS:
lhs, rhs = v
top_vvar = verdict_var
self.verdict_next()
lvar = self.verdict_variable
self.verdict_next()
rvar = self.verdict_variable
lblock = self.gen_test(fn_template, lhs, lvar)
rblock = self.gen_test(fn_template, rhs, rvar)
invert = ''
if k == '&':
binop = 'and'
elif k == '|':
binop = 'or'
elif k == '->':
binop = 'or'
invert = 'not'
binop_str = """
%(verdict_var)s = %(invert)s %(lvar)s %(binop)s %(rvar)s
""" % {
'verdict_var': top_vvar,
'invert': invert,
'lvar': lvar,
'binop': binop,
'rvar': rvar
}
binop_ast = self.str_to_ast(binop_str)
combined_ast = ast.Module(body=[lblock, rblock, binop_ast])
return combined_ast
elif k == 'exists':
# If the variable starts with a capital letter,
# we assume that it is a model. If it starts with
# a small letter, we assume it is an enumerable
#
# We do not support nested exists yet. FIXME.
var, expr = v
if var.istitle():
f = self.fol_to_python_filter(var, expr, django=True)
entry = f.pop()
python_str = """
%(verdict_var)s = not not %(model)s.objects.filter(%(query)s)
""" % {
'verdict_var': verdict_var,
'model': var,
'query': entry
}
python_ast = ast.parse(python_str)
else:
f = self.fol_to_python_filter(var, expr, django=False)
entry = f.pop()
python_str = """
%(verdict_var)s = filter(lambda __elt:%(query)s, %(model)s)
""" % {
'verdict_var': verdict_var,
'model': var,
'query': entry
}
python_ast = ast.parse(python_str)
return python_ast
elif k == 'forall':
var, expr = v
if var.istitle():
f = self.fol_to_python_filter(var,
expr,
django=True,
negate=True)
entry = f.pop()
self.verdict_next()
vvar = self.verdict_variable
python_str = """
%(verdict_var)s = not not %(model)s.objects.filter(%(query)s)
""" % {
'verdict_var': vvar,
'model': var,
'query': entry
}
python_ast = ast.parse(python_str)
else:
f = self.fol_to_python_filter(var,
expr,
django=False,
negate=True)
entry = f.pop()
python_str = """
%(verdict_var)s = next(elt for elt in %(model)s if %(query)s)
""" % {
'verdict_var': vvar,
'model': var,
'query': entry
}
python_ast = ast.parse(python_str)
negate_str = """
%(verdict_var)s = not %(vvar)s
""" % {
'verdict_var': verdict_var,
'vvar': vvar
}
negate_ast = ast.parse(negate_str)
return ast.Module(body=[python_ast, negate_ast])
def expr(self, node, msg=None, *, exc=ValueError):
mod = ast.Module([ast.Expr(node)])
self.mod(mod, msg, exc=exc)
def make_module(function_asts, main_ast):
module_body = function_asts
module_body.append(ast.Expr(value=main_ast))
module_ast = ast.Module(body=module_body)
ast.fix_missing_locations(module_ast)
return module_ast
if mapdef['types']['left'] == 'list':
sub_expr = [eval_node(sub_expr, ctx)]
else:
sub_expr = eval_node(sub_expr, ctx)
elif isinstance(sub_expr, list):
list_expr = sub_expr
sub_expr = []
for list_node in list_expr:
if not isinstance(list_node, Node):
if isinstance(list_node, str):
sub_expr.append(ast.Str(s=list_node, lineno=0,
col_offset=0))
else:
sub_expr.append(eval_node(list_node, ctx))
elif isinstance(sub_expr, int):
sub_expr = ast.Num(n=sub_expr, lineno=0, col_offset=0)
setattr(instance, key, sub_expr)
return instance
for node in output:
body.append(eval_node(node))
expr = ast.Module(body, lineno=0, col_offset=0)
eval(compile(ast.parse(expr), '<string>', mode='exec'))
def visit_Module(self, node):
new_node = ast.Module(self._visit(node.body))
return new_node
