Exemplo n.º 1
0
 def test_count(self):
     self.assertEqual(0, iter_utils.count([]))
     self.assertEqual(1, iter_utils.count(['a']))
     self.assertEqual(10, iter_utils.count(compat_range(0, 10)))
     self.assertEqual(1000, iter_utils.count(compat_range(0, 1000)))
     self.assertEqual(0, iter_utils.count(compat_range(0)))
     self.assertEqual(0, iter_utils.count(compat_range(-1)))
Exemplo n.º 2
0
    def execute(self, image_config, mandelbrot_config, chunk):
        """Returns the number of iterations before the computation "escapes".

        Given the real and imaginary parts of a complex number, determine if it
        is a candidate for membership in the mandelbrot set given a fixed
        number of iterations.
        """

        # Parts borrowed from (credit to mark harris and benoît mandelbrot).
        #
        # http://nbviewer.ipython.org/gist/harrism/f5707335f40af9463c43
        def mandelbrot(x, y, max_iters):
            c = complex(x, y)
            z = 0.0j
            for i in compat_range(max_iters):
                z = z * z + c
                if (z.real * z.real + z.imag * z.imag) >= 4:
                    return i
            return max_iters

        min_x, max_x, min_y, max_y, max_iters = mandelbrot_config
        height, width = image_config['size']
        pixel_size_x = (max_x - min_x) / width
        pixel_size_y = (max_y - min_y) / height
        block = []
        for y in compat_range(chunk[0], chunk[1]):
            row = []
            imag = min_y + y * pixel_size_y
            for x in compat_range(0, width):
                real = min_x + x * pixel_size_x
                row.append(mandelbrot(real, imag, max_iters))
            block.append(row)
        return block
def main():
    if six.PY3:
        # TODO(harlowja): Hack to make eventlet work right, remove when the
        # following is fixed: https://github.com/eventlet/eventlet/issues/230
        from taskflow.utils import eventlet_utils as _eu  # noqa
        try:
            import eventlet as _eventlet  # noqa
        except ImportError:
            pass
    with contextlib.closing(fake_client.FakeClient()) as c:
        created = []
        for i in compat_range(0, PRODUCERS):
            p = threading_utils.daemon_thread(producer, i + 1, c)
            created.append(p)
            p.start()
        consumed = collections.deque()
        for i in compat_range(0, WORKERS):
            w = threading_utils.daemon_thread(worker, i + 1, c, consumed)
            created.append(w)
            w.start()
        while created:
            t = created.pop()
            t.join()
        # At the end there should be nothing leftover, let's verify that.
        board = backends.fetch('verifier', SHARED_CONF.copy(), client=c)
        board.connect()
        with contextlib.closing(board):
            if board.job_count != 0 or len(consumed) != EXPECTED_UNITS:
                return 1
            return 0
def main():
    if len(sys.argv) == 2:
        tbl = []
        with open(sys.argv[1], 'rb') as fh:
            reader = csv.reader(fh)
            for row in reader:
                tbl.append([float(r) if r else 0.0 for r in row])
    else:
        # Make some random table out of thin air...
        tbl = []
        cols = random.randint(1, 100)
        rows = random.randint(1, 100)
        for _i in compat_range(0, rows):
            row = []
            for _j in compat_range(0, cols):
                row.append(random.random())
            tbl.append(row)

    # Generate the work to be done.
    f = make_flow(tbl)

    # Now run it (using the specified executor)...
    try:
        executor = futurist.GreenThreadPoolExecutor(max_workers=5)
    except RuntimeError:
        # No eventlet currently active, use real threads instead.
        executor = futurist.ThreadPoolExecutor(max_workers=5)
    try:
        e = engines.load(f, engine='parallel', executor=executor)
        for st in e.run_iter():
            print(st)
    finally:
        executor.shutdown()

    # Find the old rows and put them into place...
    #
    # TODO(harlowja): probably easier just to sort instead of search...
    computed_tbl = []
    for i in compat_range(0, len(tbl)):
        for t in f:
            if t.index == i:
                computed_tbl.append(e.storage.get(t.name))

    # Do some basic validation (which causes the return code of this process
    # to be different if things were not as expected...)
    if len(computed_tbl) != len(tbl):
        return 1
    else:
        return 0
Exemplo n.º 5
0
 def mandelbrot(x, y, max_iters):
     c = complex(x, y)
     z = 0.0j
     for i in compat_range(max_iters):
         z = z * z + c
         if (z.real * z.real + z.imag * z.imag) >= 4:
             return i
     return max_iters
Exemplo n.º 6
0
def calculate(engine_conf):
    # Subdivide the work into X pieces, then request each worker to calculate
    # one of those chunks and then later we will write these chunks out to
    # an image bitmap file.

    # And unordered flow is used here since the mandelbrot calculation is an
    # example of a embarrassingly parallel computation that we can scatter
    # across as many workers as possible.
    flow = uf.Flow("mandelbrot")

    # These symbols will be automatically given to tasks as input to there
    # execute method, in this case these are constants used in the mandelbrot
    # calculation.
    store = {
        'mandelbrot_config': [-2.0, 1.0, -1.0, 1.0, MAX_ITERATIONS],
        'image_config': {
            'size': IMAGE_SIZE,
        }
    }

    # We need the task names to be in the right order so that we can extract
    # the final results in the right order (we don't care about the order when
    # executing).
    task_names = []

    # Compose our workflow.
    height, _width = IMAGE_SIZE
    chunk_size = int(math.ceil(height / float(CHUNK_COUNT)))
    for i in compat_range(0, CHUNK_COUNT):
        chunk_name = 'chunk_%s' % i
        task_name = "calculation_%s" % i
        # Break the calculation up into chunk size pieces.
        rows = [i * chunk_size, i * chunk_size + chunk_size]
        flow.add(
            MandelCalculator(task_name,
                             # This ensures the storage symbol with name
                             # 'chunk_name' is sent into the tasks local
                             # symbol 'chunk'. This is how we give each
                             # calculator its own correct sequence of rows
                             # to work on.
                             rebind={'chunk': chunk_name}))
        store[chunk_name] = rows
        task_names.append(task_name)

    # Now execute it.
    eng = engines.load(flow, store=store, engine_conf=engine_conf)
    eng.run()

    # Gather all the results and order them for further processing.
    gather = []
    for name in task_names:
        gather.extend(eng.storage.get(name))
    points = []
    for y, row in enumerate(gather):
        for x, color in enumerate(row):
            points.append(((x, y), color))
    return points
Exemplo n.º 7
0
 def wrapper(in_self, timeout=None, prefetch_size=1):
     incomings = []
     watch = timeutils.StopWatch(duration=timeout)
     with watch:
         for __ in compat_range(prefetch_size):
             msg = func(in_self, timeout=watch.leftover(return_none=True))
             if msg is not None:
                 incomings.append(msg)
             else:
                 # timeout reached or listener stopped
                 break
     return incomings
Exemplo n.º 8
0
def iter_forever(limit):
    """Yields values from iterator until a limit is reached.

    if limit is negative, we iterate forever.
    """
    if limit < 0:
        i = itertools.count()
        while True:
            yield next(i)
    else:
        for i in compat_range(0, limit):
            yield i
Exemplo n.º 9
0
def create_fractal():
    logging.basicConfig(level=logging.ERROR)

    # Setup our transport configuration and merge it into the worker and
    # engine configuration so that both of those use it correctly.
    shared_conf = dict(BASE_SHARED_CONF)
    shared_conf.update({
        'transport': 'memory',
        'transport_options': {
            'polling_interval': 0.1,
        },
    })

    if len(sys.argv) >= 2:
        output_filename = sys.argv[1]
    else:
        output_filename = None

    worker_conf = dict(WORKER_CONF)
    worker_conf.update(shared_conf)
    engine_conf = dict(ENGINE_CONF)
    engine_conf.update(shared_conf)
    workers = []
    worker_topics = []

    print('Calculating your mandelbrot fractal of size %sx%s.' % IMAGE_SIZE)
    try:
        # Create a set of workers to simulate actual remote workers.
        print('Running %s workers.' % (WORKERS))
        for i in compat_range(0, WORKERS):
            worker_conf['topic'] = 'calculator_%s' % (i + 1)
            worker_topics.append(worker_conf['topic'])
            w = worker.Worker(**worker_conf)
            runner = threading.Thread(target=w.run)
            runner.daemon = True
            runner.start()
            w.wait()
            workers.append((runner, w.stop))

        # Now use those workers to do something.
        engine_conf['topics'] = worker_topics
        results = calculate(engine_conf)
        print('Execution finished.')
    finally:
        # And cleanup.
        print('Stopping workers.')
        while workers:
            r, stopper = workers.pop()
            stopper()
            r.join()
    print("Writing image...")
    write_image(results, output_filename=output_filename)
def main():
    with contextlib.closing(fake_client.FakeClient()) as c:
        created = []
        for i in compat_range(0, PRODUCERS):
            p = threading_utils.daemon_thread(producer, i + 1, c)
            created.append(p)
            p.start()
        consumed = collections.deque()
        for i in compat_range(0, WORKERS):
            w = threading_utils.daemon_thread(worker, i + 1, c, consumed)
            created.append(w)
            w.start()
        while created:
            t = created.pop()
            t.join()
        # At the end there should be nothing leftover, let's verify that.
        board = backends.fetch('verifier', SHARED_CONF.copy(), client=c)
        board.connect()
        with contextlib.closing(board):
            if board.job_count != 0 or len(consumed) != EXPECTED_UNITS:
                return 1
            return 0
Exemplo n.º 11
0
    def join(self, key_piece, *more_key_pieces):
        """Create and return a namespaced key from many segments.

        NOTE(harlowja): all pieces that are text/unicode are converted into
        their binary equivalent (if they are already binary no conversion
        takes place) before being joined (as redis expects binary keys and not
        unicode/text ones).
        """
        namespace_pieces = []
        if self._namespace is not None:
            namespace_pieces = [self._namespace, self.NAMESPACE_SEP]
        else:
            namespace_pieces = []
        key_pieces = [key_piece]
        if more_key_pieces:
            key_pieces.extend(more_key_pieces)
        for i in compat_range(0, len(namespace_pieces)):
            namespace_pieces[i] = misc.binary_encode(namespace_pieces[i])
        for i in compat_range(0, len(key_pieces)):
            key_pieces[i] = misc.binary_encode(key_pieces[i])
        namespace = b"".join(namespace_pieces)
        key = self.KEY_PIECE_SEP.join(key_pieces)
        return namespace + key
Exemplo n.º 12
0
Arquivo: async.py Projeto: jzako/anvil
 def run(self, funcs):
     if self._workers:
         raise RuntimeError("Can not start another `run` with %s" " existing workers" % (len(self._workers)))
     self._queue = compat_queue.Queue()
     self._death.clear()
     futs = []
     for i in compat_range(0, self._max_workers):
         w = threading.Thread(target=_chained_worker, args=(i + 1, self._death, self._queue, futs))
         w.daemon = True
         w.start()
         self._workers.append(w)
     for func in funcs:
         fut = futures.Future()
         futs.append(fut)
         self._queue.put((func, fut))
     return futs
def producer(ident, client):
    # Create a personal board (using the same client so that it works in
    # the same process) and start posting jobs on the board that we want
    # some entity to perform.
    name = "P-%s" % (ident)
    safe_print(name, "started")
    with backends.backend(name, SHARED_CONF.copy(), client=client) as board:
        for i in compat_range(0, PRODUCER_UNITS):
            job_name = "%s-%s" % (name, i)
            details = {
                'color': random.choice(['red', 'blue']),
            }
            job = board.post(job_name, book=None, details=details)
            safe_print(name, "'%s' [posted]" % (job))
            time.sleep(PRODUCER_DELAY)
    safe_print(name, "finished", prefix=">>>")
Exemplo n.º 14
0
    def __call__(self, value):
        value = str(value)
        num = "0|-?[1-9][0-9]*"
        m = re.match("^(%s)(?:-(%s))?$" % (num, num), value)
        if not m:
            raise ValueError('Invalid Range: %s' % value)
        left = int(m.group(1))
        right = int(left if m.group(2) is None else m.group(2))

        if left < right:
            left = Integer(min=self.min)(left)
            right = Integer(max=self.max)(right)
            step = 1
        else:
            left = Integer(max=self.max)(left)
            right = Integer(min=self.min)(right)
            step = -1
        if self.inclusive:
            right += step
        return compat_range(left, right, step)
Exemplo n.º 15
0
def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--profile', "-p",
                        dest='profile', action='store_true',
                        default=False,
                        help='profile instead of gather timing'
                             ' (default: False)')
    parser.add_argument('--dummies', "-d",
                        dest='dummies', action='store', type=int,
                        default=100, metavar="<number>",
                        help='how many dummy/no-op tasks to inject'
                             ' (default: 100)')
    parser.add_argument('--limit', '-l',
                        dest='limit', action='store', type=float,
                        default=100.0, metavar="<number>",
                        help='percentage of profiling output to show'
                             ' (default: 100%%)')
    args = parser.parse_args()
    if args.profile:
        ctx_manager = ProfileIt
    else:
        ctx_manager = TimeIt
    dummy_am = max(0, args.dummies)
    with ctx_manager("Building linear flow with %s tasks" % dummy_am, args):
        f = lf.Flow("root")
        for i in compat_range(0, dummy_am):
            f.add(DummyTask(name="dummy_%s" % i))
    with ctx_manager("Loading", args):
        e = engines.load(f)
    with ctx_manager("Compiling", args):
        e.compile()
    with ctx_manager("Preparing", args):
        e.prepare()
    with ctx_manager("Validating", args):
        e.validate()
    with ctx_manager("Running", args):
        e.run()
Exemplo n.º 16
0
 def __iter__(self):
     if self.count <= 0:
         raise StopIteration()
     for i in compat_range(0, self.count):
         yield min(self.exponent ** i, self.max_backoff)
Exemplo n.º 17
0
 def assertRange(self, s, r1, r2, step=1):
     self.assertEqual(list(compat_range(r1, r2, step)),
                      list(self.type_instance(s)))
 def test_fill_many_empty(self):
     result = list(iter_utils.fill(compat_range(0, 50), 500))
     self.assertEqual(450, sum(1 for x in result if x is None))
     self.assertEqual(50, sum(1 for x in result if x is not None))
Exemplo n.º 19
0
    engine_options = dict(BASE_SHARED_CONF)
    workers = []

    # These topics will be used to request worker information on; those
    # workers will respond with there capabilities which the executing engine
    # will use to match pending tasks to a matched worker, this will cause
    # the task to be sent for execution, and the engine will wait until it
    # is finished (a response is recieved) and then the engine will either
    # continue with other tasks, do some retry/failure resolution logic or
    # stop (and potentially re-raise the remote workers failure)...
    worker_topics = []

    try:
        # Create a set of worker threads to simulate actual remote workers...
        print('Running %s workers.' % (MEMORY_WORKERS))
        for i in compat_range(0, MEMORY_WORKERS):
            # Give each one its own unique topic name so that they can
            # correctly communicate with the engine (they will all share the
            # same exchange).
            worker_conf['topic'] = 'worker-%s' % (i + 1)
            worker_topics.append(worker_conf['topic'])
            w = worker.Worker(**worker_conf)
            runner = threading_utils.daemon_thread(w.run)
            runner.start()
            w.wait()
            workers.append((runner, w.stop))

        # Now use those workers to do something.
        print('Executing some work.')
        engine_options['topics'] = worker_topics
        result = run(engine_options)