def _set_outputs(self, fn, args): sig = self._sig if type(sig.return_annotation) == tuple: for index, ret_type in enumerate(sig.return_annotation): self.outputs[str( index)] = Backend.get_current_backend().get_port( get_type(ret_type), str(index), index, self) else: ret_type = sig.return_annotation type_obj = None if ret_type == sig.empty: # [FIXME] Code debt - Currently we have two dynamic types. One # for builtins and one for files. Here I just assume if we # an untyped return it is a file type. This needs fixing if we # want to return any untyped builtins as well. ret_type = 'anyfile' type_obj = get_type(ret_type) elif type(ret_type) == str and ret_type.startswith('@args'): # Get the actual type from the task args input # arg index follows '@args' prefix. Need to make it zero indexed arg_index = int(ret_type[5]) - 1 # arg accessor follows the arg_index arg_accessor = ret_type[7:] arg = args[arg_index] ret_type = getattr(arg, arg_accessor) type_obj = get_type(ret_type) else: type_obj = get_type(ret_type) self.outputs[str(0)] = Backend.get_current_backend().get_port( type_obj, str(0), 0, self)
def receive(self): is_one_sided_receive = False for name, inport in self.inputs.items(): if not inport.is_immediate: # result = inport.receive() # print("{} : {}".format(inport.name, result)) is_one_sided_receive |= inport.is_one_sided_receive inport.receive() # If communication is one sided i.e: task is not actively waiting for # inputs we shouldn't block this thread since other in-ports needs to # run on this thread and push the rest of the inputs to the task if self._latch.value and is_one_sided_receive: return # Latch is triggered and task run when we get all the inputs # That will be the case if the in-ports are blocking at receive() # or communication is one-sided (in which case we make sure we get to # here only after getting all the inputs as per the conditional above). # If the in-ports are non blocking we wait on the `triggered` monitor while self._latch.value: with self.triggered: self.triggered.wait() Backend.get_current_backend().run_task(self)
def run(self, graph, ctx): Pass.run(self, graph, ctx) for tid, task in graph.tasks.items(): # Infer if the task is a source is_source = True for name, inport in task.inputs.items(): if not inport.is_immediate: is_source = False break if is_source: graph.set_source(task) # Infer if the task is a sink and generate sink out-ports if not task.edges: task.is_sink = True # If there no out edges it means this is a sink and all outputs # are sinks. Make them so... current_backend = Backend.get_current_backend() for name, outport in task.outputs.items(): # If current out port is not a local port make it so if not current_backend.name == 'LOCAL_NON_THREADED': outport = Backend.get_backend( BackendConfig(BackendType.LOCAL_NON_THREADED, 'Local Non Threaded')).get_port( outport.type, outport.name, outport.index, outport.task_ref) task.edges.append(Edge(outport, Sink(outport))) return PassResult.CONTINUE
def transplant(edge): source = edge.source current_backend = Backend.get_current_backend() if not current_backend.name == 'LOCAL_NON_THREADED': source = Backend.get_backend( BackendConfig(BackendType.LOCAL_NON_THREADED, 'Local Non Threaded')).get_port( source.type, source.name, source.index, source.task_ref) # Update the task out-port to be a local port source.task_ref.outputs[source.name] = source # Update the edge edge.source = source dest = edge.dest if not current_backend.name == 'LOCAL_NON_THREADED': dest = Backend.get_backend( BackendConfig(BackendType.LOCAL_NON_THREADED, 'Local Non Threaded')).get_port( dest.type, dest.name, dest.index, dest.task_ref) # Update the task in-port to be a local port dest.task_ref.inputs[source.name] = dest # Update the edge edge.dest = dest return edge
def run(self, ctx): logger = Backend.get_current_backend().logger log_str = logger.fmt( "[Runner] {} output : ".format(ctx.get('__name__')), LogColor.GREEN) log_str += logger.fmt("{}".format(ctx.ret), LogColor.BLUE) logger.log(log_str) logger.log("========================================") logger.log(" \/ \n") '''
def run(self, ctx): timer = ctx.get('__timer__') timer.stop() logger = Backend.get_current_backend().logger log_str = logger.fmt( "[Runner] {} took {}".format(ctx.get('__name__'), timer.elapsed()), LogColor.GREEN) logger.log(log_str) '''
def run(self, ctx): logger = Backend.get_current_backend().logger log_str = logger.fmt("[Runner] Running {}".format(ctx.get('__name__')), LogColor.GREEN) logger.log(log_str) log_str = logger.fmt( "[Runner] {} inputs : ".format(ctx.get('__name__')), LogColor.GREEN) log_str += logger.fmt("{}".format(ctx.args), LogColor.BLUE) logger.log(log_str) log_str = logger.fmt(" . ", LogColor.GREEN) logger.log(log_str) logger.log(log_str) '''
def __init__(self, app, backend="local"): self.app = app if backend == "slurm": config = BackendConfig(BackendType.SLURM, "Slurm") elif backend == "local": config = BackendConfig(BackendType.LOCAL, "Local Threaded") elif backend == "serial": config = BackendConfig(BackendType.LOCAL_NON_THREADED, "Serial") else: raise Exception("Unknown backend {}".format(backend)) # Threaded backend has issues on OS X due to # https://bugs.python.org/issue30385. Fall back to serial backend if platform.system().startswith("Darwin") and \ config.backend_type == BackendType.LOCAL: config = BackendConfig(BackendType.LOCAL_NON_THREADED, "Local Non Threaded") Backend.set_current_backend(config) self.backend = Backend.get_current_backend() print("[KISSERU] Using '{}' backend\n".format(self.backend.name))
def _set_inputs(self, fn, args, kwargs): sig = self._sig params = sig.parameters # print(params) ba = sig.bind(*args, **kwargs) ba.apply_defaults() arguments = ba.arguments # print(arguments) if len(params) != len(arguments): raise Exception( "{} accepts {} arguments. But {} were given".format( self.name, len(params), len(arguments))) for pname, param in params.items(): value = arguments[pname] py_type = param.annotation param_type = None if py_type == param.empty: if isinstance(value, Task): parent = value # Get the only out-port of the parent task outport = next(iter(parent.outputs.values())) param_type = outport.type elif isinstance(value, Tasklet): parent = value.parent outport = parent.outputs[value.out_slot_in_parent] param_type = outport.type else: py_type = type(value) param_type = get_type(py_type) else: param_type = get_type(py_type) self._args[pname] = value inport = Backend.get_current_backend().get_port( param_type, pname, -1, self) #LocalPort(param_type, pname, -1, self) self.inputs[pname] = inport if isinstance(value, Task): inport.is_immediate = False parent = value # Get the only out-port of the parent task outport = next(iter(parent.outputs.values())) edge = Edge(outport, inport) parent.edges.append(edge) # Unsynchronized access here since we know graph generation is # single threaded self._latch.value += 1 elif isinstance(value, Tasklet): inport.is_immediate = False parent = value.parent outport = parent.outputs[value.out_slot_in_parent] edge = Edge(outport, inport) parent.edges.append(edge) # Unsynchronized access here since we know graph generation is # single threaded self._latch.value += 1
def receive(self, value, from_port): """ Receives the result and logs it """ logger = Backend.get_current_backend().logger log_str = logger.fmt( "[KISSERU] Pipeline output : {}".format(str(value)), LogColor.BLUE) logger.log(log_str)