def test_task_stream_append():
    msgs = [{'status': 'OK', 'compute_start': 10, 'compute_stop': 20,
             'key':'inc-1', 'thread': 5855, 'worker':'127.0.0.1:9999'},
            {'status': 'OK', 'compute_start': 15, 'compute_stop': 25,
             'key':'inc-2', 'thread': 6000, 'worker':'127.0.0.1:9999'},
            {'status': 'error', 'compute_start': 10, 'compute_stop': 14,
             'key':'inc-3', 'thread': 6000, 'worker':'127.0.0.1:9999'},
            {'status': 'OK', 'compute_start': 10, 'compute_stop': 30,
             'transfer_start': 8, 'transfer_stop': 10,
             'key':'add-1', 'thread': 4000, 'worker':'127.0.0.2:9999'}]

    lists = {name: [] for name in
            'start duration key name color worker worker_thread y alpha'.split()}
    workers = {'127.0.0.1:9999-5855': 0}

    for msg in msgs:
        task_stream_append(lists, msg, workers)
    assert len(lists['start']) == len(msgs) + 1  # +1 for a transfer
    assert len(workers) == 3
    assert set(workers) == set(lists['worker_thread'])
    assert set(workers.values()) == set(range(len(workers)))
    assert lists['color'][-1] == 'red'
    L = lists['color']
    assert L[0] == L[1]
    assert L[2] == 'black'
    assert L[3] != L[0]
Exemplo n.º 2
0
def task_events(interval, deque, times, index, rectangles, workers, last_seen):
    i = 0
    with log_errors():
        stream = yield eventstream('%(host)s:%(tcp-port)d' % options, 0.100)
        while True:
            try:
                msgs = yield read(stream)
            except StreamClosedError:
                break
            else:
                if not msgs:
                    continue

                last_seen[0] = time()
                for msg in msgs:
                    if 'compute_start' in msg:
                        deque.append(msg)
                        times.append(msg['compute_start'])
                        index.append(i)
                        i += 1
                        if msg.get('transfer_start') is not None:
                            index.append(i)
                            i += 1
                        if msg.get('disk_load_start') is not None:
                            index.append(i)
                            i += 1
                        task_stream_append(rectangles, msg, workers)
Exemplo n.º 3
0
def task_events(interval, deque, times, index, rectangles, workers, last_seen):
    i = 0
    try:
        stream = yield eventstream('%(host)s:%(tcp-port)d' % options, 0.100)
        while True:
            msgs = yield read(stream)
            if not msgs:
                continue

            last_seen[0] = time()
            for msg in msgs:
                if 'compute_start' in msg:
                    deque.append(msg)
                    times.append(msg['compute_start'])
                    index.append(i)
                    i += 1
                    if msg.get('transfer_start') is not None:
                        index.append(i)
                        i += 1
                    if msg.get('disk_load_start') is not None:
                        index.append(i)
                        i += 1
                    task_stream_append(rectangles, msg, workers)
    except StreamClosedError:
        pass  # don't log StreamClosedErrors
    except Exception as e:
        logger.exception(e)
    finally:
        try:
            sys.exit(0)
        except:
            pass
Exemplo n.º 4
0
def task_events(interval, deque, times, index, rectangles, workers, last_seen):
    i = 0
    try:
        stream = yield eventstream('%(host)s:%(tcp-port)d' % options, 0.100)
        while True:
            msgs = yield read(stream)
            if not msgs:
                continue

            last_seen[0] = time()
            for msg in msgs:
                if 'compute_start' in msg:
                    deque.append(msg)
                    times.append(msg['compute_start'])
                    index.append(i)
                    i += 1
                    if msg.get('transfer_start') is not None:
                        index.append(i)
                        i += 1
                    if msg.get('disk_load_start') is not None:
                        index.append(i)
                        i += 1
                    task_stream_append(rectangles, msg, workers)
    except StreamClosedError:
        pass  # don't log StreamClosedErrors
    except Exception as e:
        logger.exception(e)
    finally:
        try:
            sys.exit(0)
        except:
            pass
Exemplo n.º 5
0
def task_events(interval, deque, times, index, rectangles, workers, last_seen):
    i = 0
    with log_errors():
        stream = yield eventstream('localhost:8786', 0.100)
        while True:
            try:
                msgs = yield read(stream)
            except StreamClosedError:
                break
            else:
                if not msgs:
                    continue

                last_seen[0] = time()
                for msg in msgs:
                    if 'compute-start' in msg:
                        deque.append(msg)
                        times.append(msg['compute-start'])
                        index.append(i); i += 1
                        if 'transfer-start' in msg:
                            index.append(i); i += 1
                        task_stream_append(rectangles, msg, workers)
def task_events(interval, deque, times, index, rectangles, workers, last_seen):
    i = 0
    with log_errors():
        stream = yield eventstream('%(host)s:%(tcp-port)d' % options, 0.100)
        while True:
            try:
                msgs = yield read(stream)
            except StreamClosedError:
                break
            else:
                if not msgs:
                    continue

                last_seen[0] = time()
                for msg in msgs:
                    if 'compute_start' in msg:
                        deque.append(msg)
                        times.append(msg['compute_start'])
                        index.append(i); i += 1
                        if msg.get('transfer_start') is not None:
                            index.append(i); i += 1
                        task_stream_append(rectangles, msg, workers)
Exemplo n.º 7
0
def task_events(interval, deque, times, index, rectangles, workers, last_seen):
    i = 0
    with log_errors():
        stream = yield eventstream('localhost:8786', 0.100)
        while True:
            try:
                msgs = yield read(stream)
            except StreamClosedError:
                break
            else:
                if not msgs:
                    continue

                last_seen[0] = time()
                for msg in msgs:
                    if 'compute-start' in msg:
                        deque.append(msg)
                        times.append(msg['compute-start'])
                        index.append(i)
                        i += 1
                        if 'transfer-start' in msg:
                            index.append(i)
                            i += 1
                        task_stream_append(rectangles, msg, workers)