def test_task_stream_append(): msgs = [{'status': 'OK', 'compute_start': 10, 'compute_stop': 20, 'key':'inc-1', 'thread': 5855, 'worker':'127.0.0.1:9999'}, {'status': 'OK', 'compute_start': 15, 'compute_stop': 25, 'key':'inc-2', 'thread': 6000, 'worker':'127.0.0.1:9999'}, {'status': 'error', 'compute_start': 10, 'compute_stop': 14, 'key':'inc-3', 'thread': 6000, 'worker':'127.0.0.1:9999'}, {'status': 'OK', 'compute_start': 10, 'compute_stop': 30, 'transfer_start': 8, 'transfer_stop': 10, 'key':'add-1', 'thread': 4000, 'worker':'127.0.0.2:9999'}] lists = {name: [] for name in 'start duration key name color worker worker_thread y alpha'.split()} workers = {'127.0.0.1:9999-5855': 0} for msg in msgs: task_stream_append(lists, msg, workers) assert len(lists['start']) == len(msgs) + 1 # +1 for a transfer assert len(workers) == 3 assert set(workers) == set(lists['worker_thread']) assert set(workers.values()) == set(range(len(workers))) assert lists['color'][-1] == 'red' L = lists['color'] assert L[0] == L[1] assert L[2] == 'black' assert L[3] != L[0]
def task_events(interval, deque, times, index, rectangles, workers, last_seen): i = 0 with log_errors(): stream = yield eventstream('%(host)s:%(tcp-port)d' % options, 0.100) while True: try: msgs = yield read(stream) except StreamClosedError: break else: if not msgs: continue last_seen[0] = time() for msg in msgs: if 'compute_start' in msg: deque.append(msg) times.append(msg['compute_start']) index.append(i) i += 1 if msg.get('transfer_start') is not None: index.append(i) i += 1 if msg.get('disk_load_start') is not None: index.append(i) i += 1 task_stream_append(rectangles, msg, workers)
def task_events(interval, deque, times, index, rectangles, workers, last_seen): i = 0 try: stream = yield eventstream('%(host)s:%(tcp-port)d' % options, 0.100) while True: msgs = yield read(stream) if not msgs: continue last_seen[0] = time() for msg in msgs: if 'compute_start' in msg: deque.append(msg) times.append(msg['compute_start']) index.append(i) i += 1 if msg.get('transfer_start') is not None: index.append(i) i += 1 if msg.get('disk_load_start') is not None: index.append(i) i += 1 task_stream_append(rectangles, msg, workers) except StreamClosedError: pass # don't log StreamClosedErrors except Exception as e: logger.exception(e) finally: try: sys.exit(0) except: pass
def task_events(interval, deque, times, index, rectangles, workers, last_seen): i = 0 with log_errors(): stream = yield eventstream('localhost:8786', 0.100) while True: try: msgs = yield read(stream) except StreamClosedError: break else: if not msgs: continue last_seen[0] = time() for msg in msgs: if 'compute-start' in msg: deque.append(msg) times.append(msg['compute-start']) index.append(i); i += 1 if 'transfer-start' in msg: index.append(i); i += 1 task_stream_append(rectangles, msg, workers)
def task_events(interval, deque, times, index, rectangles, workers, last_seen): i = 0 with log_errors(): stream = yield eventstream('%(host)s:%(tcp-port)d' % options, 0.100) while True: try: msgs = yield read(stream) except StreamClosedError: break else: if not msgs: continue last_seen[0] = time() for msg in msgs: if 'compute_start' in msg: deque.append(msg) times.append(msg['compute_start']) index.append(i); i += 1 if msg.get('transfer_start') is not None: index.append(i); i += 1 task_stream_append(rectangles, msg, workers)
def task_events(interval, deque, times, index, rectangles, workers, last_seen): i = 0 with log_errors(): stream = yield eventstream('localhost:8786', 0.100) while True: try: msgs = yield read(stream) except StreamClosedError: break else: if not msgs: continue last_seen[0] = time() for msg in msgs: if 'compute-start' in msg: deque.append(msg) times.append(msg['compute-start']) index.append(i) i += 1 if 'transfer-start' in msg: index.append(i) i += 1 task_stream_append(rectangles, msg, workers)