Example #1
0
    def __call__(self, iterable):
        n_jobs = self.n_jobs
        if n_jobs is None or multiprocessing is None or n_jobs == 1:
            n_jobs = 1
            from __builtin__ import apply
        else:
            if n_jobs == -1:
                n_jobs = multiprocessing.cpu_count()
            pool = multiprocessing.Pool(n_jobs)
            apply = pool.apply_async

        output = list()
        try:
            #jobs = create_jobs( apply, iterable )
            output = list()
            for function, args, kwargs in iterable:
                output.append(apply(function, args, kwargs))

            if n_jobs > 1:
                output = [job.get() for job in output]


        finally:
            if n_jobs > 1:
                pool.close()
                pool.join()
        return output
Example #2
0
    def __call__(self, iterable):
        n_jobs = self.n_jobs
        if n_jobs == -1:
            if multiprocessing is None:
                 n_jobs = 1
            else:
                n_jobs = multiprocessing.cpu_count()

        if n_jobs is None or multiprocessing is None or n_jobs == 1:
            n_jobs = 1
            from __builtin__ import apply
        else:
            pool = multiprocessing.Pool(n_jobs)
            apply = pool.apply_async

        output = list()
        start_time = time.time()
        try:
            for index, (function, args, kwargs) in enumerate(iterable):
                if n_jobs > 1:
                    function = SafeFunction(function)
                output.append(apply(function, args, kwargs))
                if self.verbose and n_jobs == 1:
                    print '[%s]: Done job %3i | elapsed: %s' % (
                            self, index, 
                            short_format_time(time.time() - start_time)
                        )
            if n_jobs > 1:
                start_time = time.time()
                jobs = output
                output = list()
                for index, job in enumerate(jobs):
                    try:
                        output.append(job.get())
                        if self.verbose:
                            print_progress(self, index, len(jobs), start_time,
                                           n_jobs=n_jobs)
                    except JoblibException, exception:
                        # Capture exception to add information on 
                        # the local stack in addition to the distant
                        # stack
                        this_report = format_outer_frames(
                                                context=10,
                                                stack_start=1,
                                                )
                        report = """Multiprocessing exception:
%s
---------------------------------------------------------------------------
Sub-process traceback: 
---------------------------------------------------------------------------
%s""" % (
                                    this_report,
                                    exception.message,
                                )
                        raise JoblibException(report)
        finally:
            if n_jobs > 1:
                pool.close()
                pool.join()
        return output
Example #3
0
File: tools.py Project: xNUTs/PTVS
def apply(func, *rangeIn):
    """Excel equivalent to the built-in apply().

    Ranges as well as Python iterables are accepted. Ranges
    are converted to lists of Python values (with Range.get()).
    
    The value returned by `func` is then passed to xl.view"""
    import __builtin__
    xs = (_to_value(r) for r in rangeIn)
    name = getattr(func, '__name__', "<callable>")
    y = __builtin__.apply(func, xs)
    r = _dest_for_source_ranges(rangeIn)
    return view(y, name, to=r)
Example #4
0
def apply(func, *rangeIn):
    """Excel equivalent to the built-in apply().

    Ranges as well as Python iterables are accepted. Ranges
    are converted to lists of Python values (with Range.get()).
    
    The value returned by `func` is then passed to xl.view"""
    import __builtin__
    xs = (_to_value(r) for r in rangeIn)    
    name = getattr(func, '__name__', "<callable>")
    y = __builtin__.apply(func, xs)
    r = _dest_for_source_ranges(rangeIn)
    return view(y, name, to=r)
Example #5
0
    def __call__(self, iterable):
        n_jobs = self.n_jobs
        if n_jobs is None or multiprocessing is None or n_jobs == 1:
            n_jobs = 1
            from __builtin__ import apply
        else:
            if n_jobs == -1:
                n_jobs = multiprocessing.cpu_count()
            pool = multiprocessing.Pool(n_jobs)
            apply = pool.apply_async

        output = list()
        try:
            for function, args, kwargs in iterable:
                output.append(apply(function, args, kwargs))

            if n_jobs > 1:
                output = [job.get() for job in output]
        finally:
            if n_jobs > 1:
                pool.close()
                pool.join()
        return output