def execute(self, src, output_contains=None):
src = utils.unindent(src)
expected = self._run(src)
node = ast.parse(src)
node = saneitizer.Saneitizer().process(node)
naming.MakeIdsValid().visit(node)
transformed_code = astor.to_source(node)
pydron_builtins = "from pydron.translation.builtins import *"
transformed_code = pydron_builtins + "\n\n" + transformed_code
try:
# just to see if it compiles
compile(node, "[string]", 'exec')
# we actually use the source code to run
actual = self._run(transformed_code)
self.assertEqual(actual, expected)
if output_contains:
self.assertIn(output_contains, actual)
except:
sys.stderr.write(transformed_code)
sys.stderr.write("\n\n")
raise
def translate_function(function, scheduler, saneitize=True):
"""
Translates a function into a :class:`tasks.ScheduledCallable`.
"""
def main_workaround(module_name):
"""
If the function is inside __main__ we have problem
since the workers will have a different module called
__main__.
So we make a best-effort attemt to find if __main__
is also reachable as a module.
"""
if module_name != "__main__":
return module_name # we are fine.
# See if we can find a sys.path that matches the location of __main__
main_file = getattr(sys.modules["__main__"], "__file__", "")
candidates = {path for path in sys.path if main_file.startswith(path)}
candidates = sorted(candidates, key=len)
for candidate in candidates:
# Try to create the absolute module name from the filename only.
module_name = main_file[len(candidate):]
if module_name.endswith(".py"):
module_name = module_name[:-3]
if module_name.endswith(".pyc"):
module_name = module_name[:-4]
module_name = module_name.replace("/", ".")
module_name = module_name.replace("\\", ".")
while module_name.startswith("."):
module_name = module_name[1:]
# Check if it actually works.
try:
module = importlib.import_module(module_name)
return module.__name__
except ImportError:
pass
# we were unlucky.
raise ValueError(
"The functions in the __main__ module cannot be translated.")
source = inspect.getsourcelines(function)
source = "".join(source[0])
logger.info("Translating: \n%s" % source)
node = ast.parse(utils.unindent(source))
# Remove decorators
# TODO handle decorators properly
assert len(node.body) == 1
funcdef = node.body[0]
assert isinstance(funcdef, ast.FunctionDef)
funcdef.decorator_list = []
if len(funcdef.args.defaults) != 0:
# TODO add support
raise ValueError(
"Cannot translate %f: @schedule does not support functions with default arguments"
)
id_factory = naming.UniqueIdentifierFactory()
if saneitize:
makesane = saneitizer.Saneitizer()
node = makesane.process(node, id_factory)
module_name = getattr(function, "__module__", None)
if not module_name:
raise ValueError(
"Cannot translate %f: The module in which it is defined is unknown."
)
module_name = main_workaround(module_name)
import astor
logger.info("Preprocessed source:\n%s" % astor.to_source(node))
translator = Translator(id_factory, scheduler, module_name)
graph = translator.visit(node)
def find_FunctionDefTask(graph):
for tick in graph.get_all_ticks():
task = graph.get_task(tick)
if isinstance(task, tasks.FunctionDefTask):
return task
raise ValueError("No function was translated.")
funcdeftask = find_FunctionDefTask(graph)
defaults = function.__defaults__
if not defaults:
defaults = tuple()
if funcdeftask.num_defaults != len(defaults):
raise ValueError("Number of default arguments doesn't match.")
if function.__closure__:
raise ValueError("Translating closures currently not supported.")
inputs = {"default_%s" % i: v for i, v in enumerate(defaults)}
scheduled_callable = funcdeftask.evaluate(inputs)['function']
return scheduled_callable
def _check(self, src):
node = ast.parse(utils.unindent(src))
utils.EncodeNames().visit(node)
scoping.ScopeAssigner().visit(node)
scoping.ExtendedScopeAssigner().visit(node)
return self.check(node)
def translate_function(function, scheduler, saneitize=True):
"""
Translates a function into a :class:`tasks.ScheduledCallable`.
"""
def main_workaround(module_name):
"""
If the function is inside __main__ we have problem
since the workers will have a different module called
__main__.
So we make a best-effort attemt to find if __main__
is also reachable as a module.
"""
if module_name != "__main__":
return module_name # we are fine.
# See if we can find a sys.path that matches the location of __main__
main_file = getattr(sys.modules["__main__"], "__file__", "")
candidates = {path for path in sys.path if main_file.startswith(path)}
candidates = sorted(candidates, key=len)
for candidate in candidates:
# Try to create the absolute module name from the filename only.
module_name = main_file[len(candidate):]
if module_name.endswith(".py"):
module_name = module_name[:-3]
if module_name.endswith(".pyc"):
module_name = module_name[:-4]
module_name = module_name.replace("/", ".")
module_name = module_name.replace("\\", ".")
while module_name.startswith("."):
module_name = module_name[1:]
# Check if it actually works.
try:
module = importlib.import_module(module_name)
return module.__name__
except ImportError:
pass
# we were unlucky.
raise ValueError("The functions in the __main__ module cannot be translated.")
source = inspect.getsourcelines(function)
source = "".join(source[0])
logger.info("Translating: \n%s" % source)
node = ast.parse(utils.unindent(source))
# Remove decorators
# TODO handle decorators properly
assert len(node.body) == 1
funcdef = node.body[0]
assert isinstance(funcdef, ast.FunctionDef)
funcdef.decorator_list = []
if len(funcdef.args.defaults) != 0:
# TODO add support
raise ValueError("Cannot translate %f: @schedule does not support functions with default arguments")
id_factory = naming.UniqueIdentifierFactory()
if saneitize:
makesane = saneitizer.Saneitizer()
node = makesane.process(node, id_factory)
module_name = getattr(function, "__module__", None)
if not module_name:
raise ValueError("Cannot translate %f: The module in which it is defined is unknown.")
module_name = main_workaround(module_name)
import astor
logger.info("Preprocessed source:\n%s" % astor.to_source(node))
translator = Translator(id_factory, scheduler, module_name)
graph = translator.visit(node)
def find_FunctionDefTask(graph):
for tick in graph.get_all_ticks():
task = graph.get_task(tick)
if isinstance(task, tasks.FunctionDefTask):
return task
raise ValueError("No function was translated.")
funcdeftask = find_FunctionDefTask(graph)
defaults = function.__defaults__
if not defaults:
defaults = tuple()
if funcdeftask.num_defaults != len(defaults):
raise ValueError("Number of default arguments doesn't match.")
if function.__closure__:
raise ValueError("Translating closures currently not supported.")
inputs = {"default_%s"%i:v for i, v in enumerate(defaults)}
scheduled_callable = funcdeftask.evaluate(inputs)['function']
return scheduled_callable
