def test_task_stream_append():
msgs = [{'status': 'OK', 'compute_start': 10, 'compute_stop': 20,
'key':'inc-1', 'thread': 5855, 'worker':'127.0.0.1:9999'},
{'status': 'OK', 'compute_start': 15, 'compute_stop': 25,
'key':'inc-2', 'thread': 6000, 'worker':'127.0.0.1:9999'},
{'status': 'error', 'compute_start': 10, 'compute_stop': 14,
'key':'inc-3', 'thread': 6000, 'worker':'127.0.0.1:9999'},
{'status': 'OK', 'compute_start': 10, 'compute_stop': 30,
'transfer_start': 8, 'transfer_stop': 10,
'key':'add-1', 'thread': 4000, 'worker':'127.0.0.2:9999'}]
lists = {name: [] for name in
'start duration key name color worker worker_thread y alpha'.split()}
workers = {'127.0.0.1:9999-5855': 0}
for msg in msgs:
task_stream_append(lists, msg, workers)
assert len(lists['start']) == len(msgs) + 1 # +1 for a transfer
assert len(workers) == 3
assert set(workers) == set(lists['worker_thread'])
assert set(workers.values()) == set(range(len(workers)))
assert lists['color'][-1] == 'red'
L = lists['color']
assert L[0] == L[1]
assert L[2] == 'black'
assert L[3] != L[0]
def task_events(interval, deque, times, index, rectangles, workers, last_seen):
i = 0
with log_errors():
stream = yield eventstream('%(host)s:%(tcp-port)d' % options, 0.100)
while True:
try:
msgs = yield read(stream)
except StreamClosedError:
break
else:
if not msgs:
continue
last_seen[0] = time()
for msg in msgs:
if 'compute_start' in msg:
deque.append(msg)
times.append(msg['compute_start'])
index.append(i)
i += 1
if msg.get('transfer_start') is not None:
index.append(i)
i += 1
if msg.get('disk_load_start') is not None:
index.append(i)
i += 1
task_stream_append(rectangles, msg, workers)
def task_events(interval, deque, times, index, rectangles, workers, last_seen):
i = 0
try:
stream = yield eventstream('%(host)s:%(tcp-port)d' % options, 0.100)
while True:
msgs = yield read(stream)
if not msgs:
continue
last_seen[0] = time()
for msg in msgs:
if 'compute_start' in msg:
deque.append(msg)
times.append(msg['compute_start'])
index.append(i)
i += 1
if msg.get('transfer_start') is not None:
index.append(i)
i += 1
if msg.get('disk_load_start') is not None:
index.append(i)
i += 1
task_stream_append(rectangles, msg, workers)
except StreamClosedError:
pass # don't log StreamClosedErrors
except Exception as e:
logger.exception(e)
finally:
try:
sys.exit(0)
except:
pass
def task_events(interval, deque, times, index, rectangles, workers, last_seen):
i = 0
try:
stream = yield eventstream('%(host)s:%(tcp-port)d' % options, 0.100)
while True:
msgs = yield read(stream)
if not msgs:
continue
last_seen[0] = time()
for msg in msgs:
if 'compute_start' in msg:
deque.append(msg)
times.append(msg['compute_start'])
index.append(i)
i += 1
if msg.get('transfer_start') is not None:
index.append(i)
i += 1
if msg.get('disk_load_start') is not None:
index.append(i)
i += 1
task_stream_append(rectangles, msg, workers)
except StreamClosedError:
pass # don't log StreamClosedErrors
except Exception as e:
logger.exception(e)
finally:
try:
sys.exit(0)
except:
pass
def task_events(interval, deque, times, index, rectangles, workers, last_seen):
i = 0
with log_errors():
stream = yield eventstream('localhost:8786', 0.100)
while True:
try:
msgs = yield read(stream)
except StreamClosedError:
break
else:
if not msgs:
continue
last_seen[0] = time()
for msg in msgs:
if 'compute-start' in msg:
deque.append(msg)
times.append(msg['compute-start'])
index.append(i); i += 1
if 'transfer-start' in msg:
index.append(i); i += 1
task_stream_append(rectangles, msg, workers)
def task_events(interval, deque, times, index, rectangles, workers, last_seen):
i = 0
with log_errors():
stream = yield eventstream('%(host)s:%(tcp-port)d' % options, 0.100)
while True:
try:
msgs = yield read(stream)
except StreamClosedError:
break
else:
if not msgs:
continue
last_seen[0] = time()
for msg in msgs:
if 'compute_start' in msg:
deque.append(msg)
times.append(msg['compute_start'])
index.append(i); i += 1
if msg.get('transfer_start') is not None:
index.append(i); i += 1
task_stream_append(rectangles, msg, workers)
def task_events(interval, deque, times, index, rectangles, workers, last_seen):
i = 0
with log_errors():
stream = yield eventstream('localhost:8786', 0.100)
while True:
try:
msgs = yield read(stream)
except StreamClosedError:
break
else:
if not msgs:
continue
last_seen[0] = time()
for msg in msgs:
if 'compute-start' in msg:
deque.append(msg)
times.append(msg['compute-start'])
index.append(i)
i += 1
if 'transfer-start' in msg:
index.append(i)
i += 1
task_stream_append(rectangles, msg, workers)