コード例 #1
0
def task_events(interval, deque, times, index, rectangles, workers, last_seen):
    i = 0
    with log_errors():
        stream = yield eventstream('%(host)s:%(tcp-port)d' % options, 0.100)
        while True:
            try:
                msgs = yield read(stream)
            except StreamClosedError:
                break
            else:
                if not msgs:
                    continue

                last_seen[0] = time()
                for msg in msgs:
                    if 'compute_start' in msg:
                        deque.append(msg)
                        times.append(msg['compute_start'])
                        index.append(i)
                        i += 1
                        if msg.get('transfer_start') is not None:
                            index.append(i)
                            i += 1
                        if msg.get('disk_load_start') is not None:
                            index.append(i)
                            i += 1
                        task_stream_append(rectangles, msg, workers)
コード例 #2
0
def task_events(interval, deque, times, index, rectangles, workers, last_seen):
    i = 0
    try:
        stream = yield eventstream('%(host)s:%(tcp-port)d' % options, 0.100)
        while True:
            msgs = yield read(stream)
            if not msgs:
                continue

            last_seen[0] = time()
            for msg in msgs:
                if 'compute_start' in msg:
                    deque.append(msg)
                    times.append(msg['compute_start'])
                    index.append(i)
                    i += 1
                    if msg.get('transfer_start') is not None:
                        index.append(i)
                        i += 1
                    if msg.get('disk_load_start') is not None:
                        index.append(i)
                        i += 1
                    task_stream_append(rectangles, msg, workers)
    except StreamClosedError:
        pass  # don't log StreamClosedErrors
    except Exception as e:
        logger.exception(e)
    finally:
        try:
            sys.exit(0)
        except:
            pass
コード例 #3
0
def task_events(interval, deque, times, index, rectangles, workers, last_seen):
    i = 0
    try:
        stream = yield eventstream('%(host)s:%(tcp-port)d' % options, 0.100)
        while True:
            msgs = yield read(stream)
            if not msgs:
                continue

            last_seen[0] = time()
            for msg in msgs:
                if 'startstops' in msg:
                    deque.append(msg)
                    count = task_stream_append(rectangles, msg, workers)
                    for _ in range(count):
                        index.append(i)
                        i += 1

    except StreamClosedError:
        pass  # don't log StreamClosedErrors
    except Exception as e:
        logger.exception(e)
    finally:
        try:
            sys.exit(0)
        except:
            pass
コード例 #4
0
ファイル: server_lifecycle.py プロジェクト: dask/distributed
def task_events(interval, deque, times, index, rectangles, workers, last_seen):
    i = 0
    try:
        stream = yield eventstream("%(host)s:%(tcp-port)d" % options, 0.100)
        while True:
            msgs = yield read(stream)
            if not msgs:
                continue

            last_seen[0] = time()
            for msg in msgs:
                if "compute_start" in msg:
                    deque.append(msg)
                    times.append(msg["compute_start"])
                    index.append(i)
                    i += 1
                    if msg.get("transfer_start") is not None:
                        index.append(i)
                        i += 1
                    if msg.get("disk_load_start") is not None:
                        index.append(i)
                        i += 1
                    task_stream_append(rectangles, msg, workers)
    except StreamClosedError:
        pass  # don't log StreamClosedErrors
    except Exception as e:
        logger.exception(e)
    finally:
        try:
            sys.exit(0)
        except:
            pass
コード例 #5
0
def task_events(interval, deque, times, index, rectangles, workers, last_seen):
    i = 0
    with log_errors():
        stream = yield eventstream('localhost:8786', 0.100)
        while True:
            try:
                msgs = yield read(stream)
            except StreamClosedError:
                break
            else:
                if not msgs:
                    continue

                last_seen[0] = time()
                for msg in msgs:
                    if 'compute-start' in msg:
                        deque.append(msg)
                        times.append(msg['compute-start'])
                        index.append(i); i += 1
                        if 'transfer-start' in msg:
                            index.append(i); i += 1
                        task_stream_append(rectangles, msg, workers)
コード例 #6
0
def task_events(interval, deque, times, index, rectangles, workers, last_seen):
    i = 0
    with log_errors():
        stream = yield eventstream('%(host)s:%(tcp-port)d' % options, 0.100)
        while True:
            try:
                msgs = yield read(stream)
            except StreamClosedError:
                break
            else:
                if not msgs:
                    continue

                last_seen[0] = time()
                for msg in msgs:
                    if 'compute_start' in msg:
                        deque.append(msg)
                        times.append(msg['compute_start'])
                        index.append(i); i += 1
                        if msg.get('transfer_start') is not None:
                            index.append(i); i += 1
                        task_stream_append(rectangles, msg, workers)
コード例 #7
0
def test_eventstream_remote(e, s, a, b):
    stream = yield eventstream(s.address, interval=0.010)

    start = time()
    while not s.plugins:
        yield gen.sleep(0.01)
        assert time() < start + 5

    futures = e.map(div, [1] * 10, range(10))

    start = time()
    total = []
    while len(total) < 10:
        msgs = yield read(stream)
        assert isinstance(msgs, list)
        total.extend(msgs)
        assert time() < start + 5

    stream.close()
    start = time()
    while s.plugins:
        yield gen.sleep(0.01)
        assert time() < start + 5
コード例 #8
0
def task_events(interval, deque, times, index, rectangles, workers, last_seen):
    i = 0
    try:
        addr = options['scheduler-address']
        comm = yield eventstream(addr, 0.100)
        while True:
            msgs = yield comm.read()
            if not msgs:
                continue

            last_seen[0] = time()
            for msg in msgs:
                if 'startstops' in msg:
                    deque.append(msg)
                    count = task_stream_append(rectangles, msg, workers)
                    for _ in range(count):
                        index.append(i)
                        i += 1

    except CommClosedError:
        pass  # don't log CommClosedErrors
    except Exception as e:
        logger.exception(e)
コード例 #9
0
def test_eventstream_remote(c, s, a, b):
    base_plugins = len(s.plugins)
    comm = yield eventstream(s.address, interval=0.010)

    start = time()
    while len(s.plugins) == base_plugins:
        yield gen.sleep(0.01)
        assert time() < start + 5

    futures = c.map(div, [1] * 10, range(10))

    start = time()
    total = []
    while len(total) < 10:
        msgs = yield comm.read()
        assert isinstance(msgs, list)
        total.extend(msgs)
        assert time() < start + 5

    yield comm.close()
    start = time()
    while len(s.plugins) > base_plugins:
        yield gen.sleep(0.01)
        assert time() < start + 5
コード例 #10
0
def task_events(interval, deque, times, index, rectangles, workers, last_seen):
    i = 0
    with log_errors():
        stream = yield eventstream('localhost:8786', 0.100)
        while True:
            try:
                msgs = yield read(stream)
            except StreamClosedError:
                break
            else:
                if not msgs:
                    continue

                last_seen[0] = time()
                for msg in msgs:
                    if 'compute-start' in msg:
                        deque.append(msg)
                        times.append(msg['compute-start'])
                        index.append(i)
                        i += 1
                        if 'transfer-start' in msg:
                            index.append(i)
                            i += 1
                        task_stream_append(rectangles, msg, workers)
コード例 #11
0
def test_eventstream_remote(c, s, a, b):
    base_plugins = len(s.plugins)
    comm = yield eventstream(s.address, interval=0.010)

    start = time()
    while len(s.plugins) == base_plugins:
        yield gen.sleep(0.01)
        assert time() < start + 5

    futures = c.map(div, [1] * 10, range(10))

    start = time()
    total = []
    while len(total) < 10:
        msgs = yield comm.read()
        assert isinstance(msgs, tuple)
        total.extend(msgs)
        assert time() < start + 5

    yield comm.close()
    start = time()
    while len(s.plugins) > base_plugins:
        yield gen.sleep(0.01)
        assert time() < start + 5