def _set_outputs(self, fn, args):
sig = self._sig
if type(sig.return_annotation) == tuple:
for index, ret_type in enumerate(sig.return_annotation):
self.outputs[str(
index)] = Backend.get_current_backend().get_port(
get_type(ret_type), str(index), index, self)
else:
ret_type = sig.return_annotation
type_obj = None
if ret_type == sig.empty:
# [FIXME] Code debt - Currently we have two dynamic types. One
# for builtins and one for files. Here I just assume if we
# an untyped return it is a file type. This needs fixing if we
# want to return any untyped builtins as well.
ret_type = 'anyfile'
type_obj = get_type(ret_type)
elif type(ret_type) == str and ret_type.startswith('@args'):
# Get the actual type from the task args input
# arg index follows '@args' prefix. Need to make it zero indexed
arg_index = int(ret_type[5]) - 1
# arg accessor follows the arg_index
arg_accessor = ret_type[7:]
arg = args[arg_index]
ret_type = getattr(arg, arg_accessor)
type_obj = get_type(ret_type)
else:
type_obj = get_type(ret_type)
self.outputs[str(0)] = Backend.get_current_backend().get_port(
type_obj, str(0), 0, self)
def receive(self):
is_one_sided_receive = False
for name, inport in self.inputs.items():
if not inport.is_immediate:
# result = inport.receive()
# print("{} : {}".format(inport.name, result))
is_one_sided_receive |= inport.is_one_sided_receive
inport.receive()
# If communication is one sided i.e: task is not actively waiting for
# inputs we shouldn't block this thread since other in-ports needs to
# run on this thread and push the rest of the inputs to the task
if self._latch.value and is_one_sided_receive:
return
# Latch is triggered and task run when we get all the inputs
# That will be the case if the in-ports are blocking at receive()
# or communication is one-sided (in which case we make sure we get to
# here only after getting all the inputs as per the conditional above).
# If the in-ports are non blocking we wait on the `triggered` monitor
while self._latch.value:
with self.triggered:
self.triggered.wait()
Backend.get_current_backend().run_task(self)
def run(self, graph, ctx):
Pass.run(self, graph, ctx)
for tid, task in graph.tasks.items():
# Infer if the task is a source
is_source = True
for name, inport in task.inputs.items():
if not inport.is_immediate:
is_source = False
break
if is_source:
graph.set_source(task)
# Infer if the task is a sink and generate sink out-ports
if not task.edges:
task.is_sink = True
# If there no out edges it means this is a sink and all outputs
# are sinks. Make them so...
current_backend = Backend.get_current_backend()
for name, outport in task.outputs.items():
# If current out port is not a local port make it so
if not current_backend.name == 'LOCAL_NON_THREADED':
outport = Backend.get_backend(
BackendConfig(BackendType.LOCAL_NON_THREADED,
'Local Non Threaded')).get_port(
outport.type, outport.name,
outport.index, outport.task_ref)
task.edges.append(Edge(outport, Sink(outport)))
return PassResult.CONTINUE
def transplant(edge):
source = edge.source
current_backend = Backend.get_current_backend()
if not current_backend.name == 'LOCAL_NON_THREADED':
source = Backend.get_backend(
BackendConfig(BackendType.LOCAL_NON_THREADED,
'Local Non Threaded')).get_port(
source.type, source.name, source.index,
source.task_ref)
# Update the task out-port to be a local port
source.task_ref.outputs[source.name] = source
# Update the edge
edge.source = source
dest = edge.dest
if not current_backend.name == 'LOCAL_NON_THREADED':
dest = Backend.get_backend(
BackendConfig(BackendType.LOCAL_NON_THREADED,
'Local Non Threaded')).get_port(
dest.type, dest.name, dest.index,
dest.task_ref)
# Update the task in-port to be a local port
dest.task_ref.inputs[source.name] = dest
# Update the edge
edge.dest = dest
return edge
def run(self, ctx):
logger = Backend.get_current_backend().logger
log_str = logger.fmt(
"[Runner] {} output : ".format(ctx.get('__name__')),
LogColor.GREEN)
log_str += logger.fmt("{}".format(ctx.ret), LogColor.BLUE)
logger.log(log_str)
logger.log("========================================")
logger.log(" \/ \n")
'''
def run(self, ctx):
timer = ctx.get('__timer__')
timer.stop()
logger = Backend.get_current_backend().logger
log_str = logger.fmt(
"[Runner] {} took {}".format(ctx.get('__name__'), timer.elapsed()),
LogColor.GREEN)
logger.log(log_str)
'''
def run(self, ctx):
logger = Backend.get_current_backend().logger
log_str = logger.fmt("[Runner] Running {}".format(ctx.get('__name__')),
LogColor.GREEN)
logger.log(log_str)
log_str = logger.fmt(
"[Runner] {} inputs : ".format(ctx.get('__name__')),
LogColor.GREEN)
log_str += logger.fmt("{}".format(ctx.args), LogColor.BLUE)
logger.log(log_str)
log_str = logger.fmt(" . ", LogColor.GREEN)
logger.log(log_str)
logger.log(log_str)
'''
def __init__(self, app, backend="local"):
self.app = app
if backend == "slurm":
config = BackendConfig(BackendType.SLURM, "Slurm")
elif backend == "local":
config = BackendConfig(BackendType.LOCAL, "Local Threaded")
elif backend == "serial":
config = BackendConfig(BackendType.LOCAL_NON_THREADED, "Serial")
else:
raise Exception("Unknown backend {}".format(backend))
# Threaded backend has issues on OS X due to
# https://bugs.python.org/issue30385. Fall back to serial backend
if platform.system().startswith("Darwin") and \
config.backend_type == BackendType.LOCAL:
config = BackendConfig(BackendType.LOCAL_NON_THREADED,
"Local Non Threaded")
Backend.set_current_backend(config)
self.backend = Backend.get_current_backend()
print("[KISSERU] Using '{}' backend\n".format(self.backend.name))
def _set_inputs(self, fn, args, kwargs):
sig = self._sig
params = sig.parameters
# print(params)
ba = sig.bind(*args, **kwargs)
ba.apply_defaults()
arguments = ba.arguments
# print(arguments)
if len(params) != len(arguments):
raise Exception(
"{} accepts {} arguments. But {} were given".format(
self.name, len(params), len(arguments)))
for pname, param in params.items():
value = arguments[pname]
py_type = param.annotation
param_type = None
if py_type == param.empty:
if isinstance(value, Task):
parent = value
# Get the only out-port of the parent task
outport = next(iter(parent.outputs.values()))
param_type = outport.type
elif isinstance(value, Tasklet):
parent = value.parent
outport = parent.outputs[value.out_slot_in_parent]
param_type = outport.type
else:
py_type = type(value)
param_type = get_type(py_type)
else:
param_type = get_type(py_type)
self._args[pname] = value
inport = Backend.get_current_backend().get_port(
param_type, pname, -1,
self) #LocalPort(param_type, pname, -1, self)
self.inputs[pname] = inport
if isinstance(value, Task):
inport.is_immediate = False
parent = value
# Get the only out-port of the parent task
outport = next(iter(parent.outputs.values()))
edge = Edge(outport, inport)
parent.edges.append(edge)
# Unsynchronized access here since we know graph generation is
# single threaded
self._latch.value += 1
elif isinstance(value, Tasklet):
inport.is_immediate = False
parent = value.parent
outport = parent.outputs[value.out_slot_in_parent]
edge = Edge(outport, inport)
parent.edges.append(edge)
# Unsynchronized access here since we know graph generation is
# single threaded
self._latch.value += 1
def receive(self, value, from_port):
""" Receives the result and logs it """
logger = Backend.get_current_backend().logger
log_str = logger.fmt(
"[KISSERU] Pipeline output : {}".format(str(value)), LogColor.BLUE)
logger.log(log_str)
