Esempio n. 1
0
    def _set_outputs(self, fn, args):
        sig = self._sig
        if type(sig.return_annotation) == tuple:
            for index, ret_type in enumerate(sig.return_annotation):
                self.outputs[str(
                    index)] = Backend.get_current_backend().get_port(
                        get_type(ret_type), str(index), index, self)
        else:
            ret_type = sig.return_annotation
            type_obj = None
            if ret_type == sig.empty:
                # [FIXME] Code debt - Currently we have two dynamic types. One
                # for builtins and one for files. Here I just assume if we
                # an untyped return it is a file type. This needs fixing if we
                # want to return any untyped builtins as well.
                ret_type = 'anyfile'
                type_obj = get_type(ret_type)
            elif type(ret_type) == str and ret_type.startswith('@args'):
                # Get the actual type from the task args input
                # arg index follows '@args' prefix. Need to make it zero indexed
                arg_index = int(ret_type[5]) - 1
                # arg accessor follows the arg_index
                arg_accessor = ret_type[7:]

                arg = args[arg_index]
                ret_type = getattr(arg, arg_accessor)
                type_obj = get_type(ret_type)
            else:
                type_obj = get_type(ret_type)

            self.outputs[str(0)] = Backend.get_current_backend().get_port(
                type_obj, str(0), 0, self)
Esempio n. 2
0
    def receive(self):
        is_one_sided_receive = False
        for name, inport in self.inputs.items():
            if not inport.is_immediate:
                # result = inport.receive()
                # print("{} : {}".format(inport.name, result))
                is_one_sided_receive |= inport.is_one_sided_receive
                inport.receive()

        # If communication is one sided i.e: task is not actively waiting for
        # inputs we shouldn't block this thread since other in-ports needs to
        # run on this thread and push the rest of the inputs to the task
        if self._latch.value and is_one_sided_receive:
            return

        # Latch is triggered and task run when we get all the inputs
        # That will be the case if the in-ports are blocking at receive()
        # or communication is one-sided (in which case we make sure we get to
        # here only after getting all the inputs as per the conditional above).
        # If the in-ports are non blocking we wait on the `triggered` monitor
        while self._latch.value:
            with self.triggered:
                self.triggered.wait()

        Backend.get_current_backend().run_task(self)
Esempio n. 3
0
    def run(self, graph, ctx):
        Pass.run(self, graph, ctx)
        for tid, task in graph.tasks.items():
            # Infer if the task is a source
            is_source = True
            for name, inport in task.inputs.items():
                if not inport.is_immediate:
                    is_source = False
                    break
            if is_source:
                graph.set_source(task)

            # Infer if the task is a sink and generate sink out-ports
            if not task.edges:
                task.is_sink = True
                # If there no out edges it means this is a sink and all outputs
                # are sinks. Make them so...
                current_backend = Backend.get_current_backend()
                for name, outport in task.outputs.items():
                    # If current out port is not a local port make it so
                    if not current_backend.name == 'LOCAL_NON_THREADED':
                        outport = Backend.get_backend(
                            BackendConfig(BackendType.LOCAL_NON_THREADED,
                                          'Local Non Threaded')).get_port(
                                              outport.type, outport.name,
                                              outport.index, outport.task_ref)
                        task.edges.append(Edge(outport, Sink(outport)))
        return PassResult.CONTINUE
Esempio n. 4
0
        def transplant(edge):
            source = edge.source
            current_backend = Backend.get_current_backend()
            if not current_backend.name == 'LOCAL_NON_THREADED':
                source = Backend.get_backend(
                    BackendConfig(BackendType.LOCAL_NON_THREADED,
                                  'Local Non Threaded')).get_port(
                                      source.type, source.name, source.index,
                                      source.task_ref)
                # Update the task out-port to be a local port
                source.task_ref.outputs[source.name] = source
                # Update the edge
                edge.source = source

            dest = edge.dest
            if not current_backend.name == 'LOCAL_NON_THREADED':
                dest = Backend.get_backend(
                    BackendConfig(BackendType.LOCAL_NON_THREADED,
                                  'Local Non Threaded')).get_port(
                                      dest.type, dest.name, dest.index,
                                      dest.task_ref)
                # Update the task in-port to be a local port
                dest.task_ref.inputs[source.name] = dest
                # Update the edge
                edge.dest = dest
            return edge
Esempio n. 5
0
    def run(self, ctx):
        logger = Backend.get_current_backend().logger

        log_str = logger.fmt(
            "[Runner] {} output : ".format(ctx.get('__name__')),
            LogColor.GREEN)
        log_str += logger.fmt("{}".format(ctx.ret), LogColor.BLUE)
        logger.log(log_str)
        logger.log("========================================")
        logger.log("                  \/                    \n")
        '''
Esempio n. 6
0
    def run(self, ctx):
        timer = ctx.get('__timer__')
        timer.stop()

        logger = Backend.get_current_backend().logger

        log_str = logger.fmt(
            "[Runner] {} took {}".format(ctx.get('__name__'), timer.elapsed()),
            LogColor.GREEN)
        logger.log(log_str)
        '''
Esempio n. 7
0
    def run(self, ctx):
        logger = Backend.get_current_backend().logger

        log_str = logger.fmt("[Runner] Running {}".format(ctx.get('__name__')),
                             LogColor.GREEN)
        logger.log(log_str)

        log_str = logger.fmt(
            "[Runner] {} inputs : ".format(ctx.get('__name__')),
            LogColor.GREEN)
        log_str += logger.fmt("{}".format(ctx.args), LogColor.BLUE)
        logger.log(log_str)

        log_str = logger.fmt("            .             ", LogColor.GREEN)
        logger.log(log_str)
        logger.log(log_str)
        '''
Esempio n. 8
0
    def __init__(self, app, backend="local"):
        self.app = app

        if backend == "slurm":
            config = BackendConfig(BackendType.SLURM, "Slurm")
        elif backend == "local":
            config = BackendConfig(BackendType.LOCAL, "Local Threaded")
        elif backend == "serial":
            config = BackendConfig(BackendType.LOCAL_NON_THREADED, "Serial")
        else:
            raise Exception("Unknown backend {}".format(backend))

        # Threaded backend has issues on OS X due to
        # https://bugs.python.org/issue30385. Fall back to serial backend
        if platform.system().startswith("Darwin") and \
                config.backend_type == BackendType.LOCAL:
            config = BackendConfig(BackendType.LOCAL_NON_THREADED,
                                   "Local Non Threaded")

        Backend.set_current_backend(config)
        self.backend = Backend.get_current_backend()
        print("[KISSERU] Using '{}' backend\n".format(self.backend.name))
Esempio n. 9
0
    def _set_inputs(self, fn, args, kwargs):
        sig = self._sig
        params = sig.parameters
        # print(params)

        ba = sig.bind(*args, **kwargs)
        ba.apply_defaults()
        arguments = ba.arguments
        # print(arguments)

        if len(params) != len(arguments):
            raise Exception(
                "{} accepts {} arguments. But {} were given".format(
                    self.name, len(params), len(arguments)))

        for pname, param in params.items():
            value = arguments[pname]
            py_type = param.annotation

            param_type = None
            if py_type == param.empty:
                if isinstance(value, Task):
                    parent = value
                    # Get the only out-port of the parent task
                    outport = next(iter(parent.outputs.values()))
                    param_type = outport.type
                elif isinstance(value, Tasklet):
                    parent = value.parent
                    outport = parent.outputs[value.out_slot_in_parent]
                    param_type = outport.type
                else:
                    py_type = type(value)
                    param_type = get_type(py_type)
            else:
                param_type = get_type(py_type)

            self._args[pname] = value
            inport = Backend.get_current_backend().get_port(
                param_type, pname, -1,
                self)  #LocalPort(param_type, pname, -1, self)
            self.inputs[pname] = inport

            if isinstance(value, Task):
                inport.is_immediate = False
                parent = value
                # Get the only out-port of the parent task
                outport = next(iter(parent.outputs.values()))

                edge = Edge(outport, inport)
                parent.edges.append(edge)
                # Unsynchronized access here since we know graph generation is
                # single threaded
                self._latch.value += 1
            elif isinstance(value, Tasklet):
                inport.is_immediate = False
                parent = value.parent
                outport = parent.outputs[value.out_slot_in_parent]

                edge = Edge(outport, inport)
                parent.edges.append(edge)
                # Unsynchronized access here since we know graph generation is
                # single threaded
                self._latch.value += 1
Esempio n. 10
0
 def receive(self, value, from_port):
     """ Receives the result and logs it """
     logger = Backend.get_current_backend().logger
     log_str = logger.fmt(
         "[KISSERU] Pipeline output : {}".format(str(value)), LogColor.BLUE)
     logger.log(log_str)