def show_connection_pool_wait(scan):
scan.seek(0)
connection_pool_waits = []
connection_pool_timestamps = []
for line in scan:
match = CONNECTION_POOL_WAIT.search(line)
if match:
connection_pool_waits.append(float(match.group(1)))
connection_pool_timestamps.append(get_line_epoch(line))
first_timestamp = get_first_timestamp(scan)
last_timestamp = get_last_timestamp(scan)
spent_time_epoch = last_timestamp - first_timestamp
connection_pool_timestamps = [
ts - first_timestamp for ts in connection_pool_timestamps
]
if not connection_pool_waits:
print('No connection pool wait data found')
return
print('Time waited for worker threads for an available TCP/IP connection')
print(' Total: %.2f sec' % sum(connection_pool_waits))
print('')
fig = plotille.Figure()
fig.width = 90
fig.height = 20
fig.register_label_formatter(float, _num_formatter)
fig.register_label_formatter(int, _num_formatter)
fig.y_label = 'Waited time'
fig.x_label = 'Time'
fig.color_mode = 'byte'
fig.set_x_limits(min_=0, max_=spent_time_epoch)
fig.set_y_limits(min_=0, max_=None)
fig.plot(connection_pool_timestamps, connection_pool_waits)
print(fig.show())
print('')
print('')
fig = plotille.Figure()
fig.width = 90
fig.height = 20
fig.register_label_formatter(float, _num_formatter)
fig.register_label_formatter(int, _num_formatter)
fig.y_label = 'Count'
fig.x_label = 'Time waiting for available TCP/IP connection'
fig.set_x_limits(min_=0)
fig.set_y_limits(min_=0)
fig.color_mode = 'byte'
print('Time waiting for available TCP/IP connection')
print('')
print(plotille.hist(connection_pool_waits, bins=25))
print('')
print('')
def __init__(self, term, args):
self.args = args
self.distance_figure = pt.Figure()
self.distance_figure.width = 60
self.distance_figure.height = 20
self.distance_figure.x_label = '# sequences'
self.distance_figure.y_label = 'Δdistance'
self.distance_figure.set_x_limits(min_=0)
self.distance_figure.set_y_limits(min_=0)
self.distance_figure.color_mode = 'byte'
self.hist_figure = pt.Figure()
self.hist_figure.width = 30
self.hist_figure.height = 20
self.hist_figure.x_label = 'Δdistance'
self.hist_figure.y_label = 'counts'
self.hist_figure.color_mode = 'byte'
self.term = term
name_block = Figlet(
font='doom').renderText('draff').rstrip().split('\n')
name_block[-1] = name_block[-1] + ' stream, v0.1'
self.name_block = TextBlock('\n'.join(name_block))
param_block = term.normal + term.underline + (
' ' * 40) + term.normal + '\n\n'
param_block += '{term.bold}distance window size: {term.normal}{distance_window}\n'\
'{term.bold}distance metric: {term.normal}{distance_metric}\n'\
'{term.bold}distance stdev cutoff: {term.normal}{stdev_cutoff}'.format(term = self.term,
distance_window = args.distance_window,
distance_metric = args.distance_metric,
stdev_cutoff = args.stdev_cutoff)
self.param_block = TextBlock(param_block)
metric_text = term.normal + term.underline + (' ' *
40) + term.normal + '\n'
metric_text += '{term.move_down}{term.bold}'
metric_text += '# sequences: '.ljust(20)
metric_text += '{term.normal}{n_reads:,}\n'
metric_text += '{term.bold}'
metric_text += 'Δdistance: '.ljust(20)
metric_text += '{term.normal}{prev_d:.20f}\n'
metric_text += '{term.bold}'
metric_text += 'σ(Δdistance): '.ljust(20)
metric_text += '{term.normal}{stdev:.20f}\n'
metric_text += '{term.underline}' + (' ' * 40) + '{term.normal}'
self.metric_text = metric_text
#message_text = term.normal + term.underline + (' ' * 40) + term.normal + '\n'
message_text = '{messages}'
self.message_text = message_text
def draw_scene(mouse_list, cat_list):
fig = plotille.Figure()
fig.width = 50
fig.height = 20
listx = []
listy = []
names = []
for mouse_index, mouse in enumerate(mouse_list):
listx.append(mouse[0])
listy.append(mouse[1])
names.append('MOUSE %d' % (mouse_index))
fig.scatter(listy, listx, lc='red', label='MOUSE', marker='o', text=names)
listx = []
listy = []
for cat_index, cat in enumerate(cat_list):
listx.append(cat[0])
listy.append(cat[1])
fig.scatter(listy, listx, lc='green', label='Cat', marker='x', text='Cat')
print(fig.show(legend=True))
def draw_timeout(scan_log_filename, scan):
scan.seek(0)
timeouts, timeout_timestamps = get_timeout_data(scan_log_filename, scan)
first_timestamp = get_first_timestamp(scan)
last_timestamp = get_last_timestamp(scan)
spent_time_epoch = last_timestamp - first_timestamp
timeout_timestamps = [ts - first_timestamp for ts in timeout_timestamps]
if not timeouts:
print('No socket timeout data found')
return
print('Socket timeout over time')
print('')
fig = plotille.Figure()
fig.width = 90
fig.height = 20
fig.register_label_formatter(float, num_formatter)
fig.register_label_formatter(int, num_formatter)
fig.y_label = 'Socket timeout'
fig.x_label = 'Time'
fig.color_mode = 'byte'
fig.set_x_limits(min_=0, max_=spent_time_epoch)
fig.set_y_limits(min_=0, max_=None)
fig.plot(timeout_timestamps,
timeouts,
label='Timeout')
print(fig.show())
print('')
def show_rtt_histo(scan):
scan.seek(0)
rtts = []
for line in scan:
match = RTT_RE.search(line)
if match:
rtts.append(float(match.group(1)))
fig = plotille.Figure()
fig.width = 90
fig.height = 20
fig.register_label_formatter(float, _num_formatter)
fig.register_label_formatter(int, _num_formatter)
fig.y_label = 'Count'
fig.x_label = 'RTT'
fig.set_x_limits(min_=0)
fig.set_y_limits(min_=0)
fig.color_mode = 'byte'
print('RTT Histogram')
print('')
print(plotille.hist(rtts, bins=25))
print('')
print('')
def test_overlay():
fig = plotille.Figure()
fig.with_colors = False
fig.width = 20
fig.height = 20
fig.scatter([4], [5], marker='x', text='Test')
c = fig.show(legend=False)
expected = """ (Y) ^
5.50000000 |
5.45000000 | ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
5.40000000 | ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
5.35000000 | ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
5.30000000 | ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
5.25000000 | ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
5.20000000 | ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
5.15000000 | ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
5.10000000 | ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
5.05000000 | ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
5 | ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀x⠀Test⠀⠀⠀⠀
4.95000000 | ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
4.90000000 | ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
4.85000000 | ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
4.80000000 | ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
4.75000000 | ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
4.70000000 | ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
4.65000000 | ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
4.60000000 | ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
4.55000000 | ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
4.50000000 | ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
-----------|-|---------|---------|-> (X)
| 3.6000000 4 4.4000000"""
assert expected == c
print(c)
def draw_rtt(scan_log_filename, scan):
scan.seek(0)
rtt, rtt_timestamps = get_rtt_data(scan_log_filename, scan)
first_timestamp = get_first_timestamp(scan)
last_timestamp = get_last_timestamp(scan)
spent_time_epoch = last_timestamp - first_timestamp
rtt_timestamps = [ts - first_timestamp for ts in rtt_timestamps]
if not rtt:
print('No RTT data found')
return
print('RTT over time')
print('')
fig = plotille.Figure()
fig.width = 90
fig.height = 20
fig.register_label_formatter(float, num_formatter)
fig.register_label_formatter(int, num_formatter)
fig.y_label = 'RTT'
fig.x_label = 'Time'
fig.color_mode = 'byte'
fig.set_x_limits(min_=0, max_=spent_time_epoch)
fig.set_y_limits(min_=0, max_=None)
fig.plot(rtt_timestamps,
rtt,
label='RTT')
print(fig.show())
print('')
def draw_queue_size_grep(scan_log_filename, scan):
grep_queue_sizes, grep_queue_timestamps = get_queue_size_grep_data(
scan_log_filename, scan)
# Get the last timestamp to use as max in the graphs
first_timestamp = get_first_timestamp(scan)
last_timestamp = get_last_timestamp(scan)
spent_time_epoch = last_timestamp - first_timestamp
grep_queue_timestamps = [
ts - first_timestamp for ts in grep_queue_timestamps
]
if not grep_queue_sizes:
print('No grep consumer queue size data found')
return
print('Grep consumer queue size')
print(' Latest queue size value: %s' % grep_queue_sizes[-1])
print('')
fig = plotille.Figure()
fig.width = 90
fig.height = 20
fig.register_label_formatter(float, num_formatter)
fig.register_label_formatter(int, num_formatter)
fig.y_label = 'Items in Grep queue'
fig.x_label = 'Time'
fig.color_mode = 'byte'
fig.set_x_limits(min_=0, max_=spent_time_epoch)
fig.set_y_limits(min_=0, max_=None)
fig.plot(grep_queue_timestamps, grep_queue_sizes, label='Grep')
print(fig.show())
print('')
def test_circle():
fig = plotille.Figure()
fig.with_colors = False
fig.width = 40
fig.height = 20
fig.circle(xcenter=0, ycenter=0, radius=0.5)
c = fig.show(legend=False)
expected = """ (Y) ^
0.60000000 |
0.54000000 | ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⡇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
0.48000000 | ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⣀⡠⠤⠤⠤⡧⠤⠤⢄⣀⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
0.42000000 | ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⡠⠔⠊⠁⠀⠀⠀⠀⠀⡇⠀⠀⠀⠀⠈⠑⠢⢄⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
0.36000000 | ⠀⠀⠀⠀⠀⠀⠀⠀⢀⠔⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⡇⠀⠀⠀⠀⠀⠀⠀⠀⠈⠑⢄⠀⠀⠀⠀⠀⠀⠀⠀
0.30000000 | ⠀⠀⠀⠀⠀⠀⢀⠔⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⡇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠑⢄⠀⠀⠀⠀⠀⠀
0.24000000 | ⠀⠀⠀⠀⠀⢀⠎⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⡇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢣⠀⠀⠀⠀⠀
0.18000000 | ⠀⠀⠀⠀⢀⠎⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⡇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢣⠀⠀⠀⠀
0.12000000 | ⠀⠀⠀⠀⡎⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⡇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⡇⠀⠀⠀
0.06000000 | ⠀⠀⠀⢰⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⡇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢸⠀⠀⠀
0 | ⣀⣀⣀⣸⣀⣀⣀⣀⣀⣀⣀⣀⣀⣀⣀⣀⣀⣀⣀⣀⣇⣀⣀⣀⣀⣀⣀⣀⣀⣀⣀⣀⣀⣀⣀⣀⣸⣀⣀⣀
-0.0600000 | ⠀⠀⠀⢸⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⡇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢰⠀⠀⠀
-0.1200000 | ⠀⠀⠀⠸⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⡇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⡜⠀⠀⠀
-0.1800000 | ⠀⠀⠀⠀⢇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⡇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢠⠃⠀⠀⠀
-0.2400000 | ⠀⠀⠀⠀⠈⢆⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⡇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢠⠃⠀⠀⠀⠀
-0.3000000 | ⠀⠀⠀⠀⠀⠈⢆⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⡇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢠⠃⠀⠀⠀⠀⠀
-0.3600000 | ⠀⠀⠀⠀⠀⠀⠀⠑⢄⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⡇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⠔⠁⠀⠀⠀⠀⠀⠀
-0.4200000 | ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠑⠤⣀⠀⠀⠀⠀⠀⠀⠀⠀⡇⠀⠀⠀⠀⠀⠀⠀⠀⣀⠔⠁⠀⠀⠀⠀⠀⠀⠀⠀
-0.4800000 | ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠉⠒⠤⣀⣀⠀⠀⠀⡇⠀⠀⠀⣀⣀⠤⠒⠉⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
-0.5400000 | ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠉⠉⠉⡏⠉⠉⠉⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
-0.6000000 | ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⡇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
-----------|-|---------|---------|---------|---------|-> (X)
| -0.600000 -0.300000 0 0.3000000 0.6000000""" # noqa: Q000
assert expected == c
print(c)
def draw_active_threads(scan_log_filename, scan):
active_threads, active_threads_timestamps = get_active_threads_data(
scan_log_filename, scan)
first_timestamp = get_first_timestamp(scan)
last_timestamp = get_last_timestamp(scan)
spent_epoch = last_timestamp - first_timestamp
active_threads_timestamps = [
ts - first_timestamp for ts in active_threads_timestamps
]
if not active_threads:
print('No active thread data found')
return
print('Active thread count over time')
print('')
fig = plotille.Figure()
fig.width = 90
fig.height = 20
fig.register_label_formatter(float, num_formatter)
fig.register_label_formatter(int, num_formatter)
fig.y_label = 'Thread count'
fig.x_label = 'Time'
fig.color_mode = 'byte'
fig.set_x_limits(min_=0, max_=spent_epoch)
fig.set_y_limits(min_=0, max_=None)
fig.plot(active_threads_timestamps, active_threads)
print(fig.show())
print('')
def main():
parser = argparse.ArgumentParser(
description='Plots an experiment in terminal')
parser.add_argument('glob', help='directory of experiment to plot')
parser.add_argument('--window',
help='smoothing window',
type=int,
default=100)
parser.add_argument('--width', help='smoothing window', type=int)
parser.add_argument('--height', help='smoothing window', type=int)
parser.add_argument('-x', help='x axis', default='frames')
parser.add_argument('-y', help='y axis', default='mean_win_rate')
args = parser.parse_args()
exps_and_logs = Experiment.discover_logs(args.glob, JSONLogger())
print('loaded {} experiments'.format(len(exps_and_logs)))
fig = plotille.Figure()
kwargs = {}
for exp, log in exps_and_logs:
if log:
df = pd.DataFrame(log)
fig.plot(X=df[args.x],
Y=df[args.y].rolling(args.window, min_periods=1).mean(),
label=exp.name,
**kwargs)
fig.x_label = args.x
fig.y_label = args.y
if args.height:
fig.height = args.height
if args.width:
fig.width = args.width
print(fig.show(legend=True))
def draw_extended_urllib_error_rate(scan_log_filename, scan):
error_rate, error_rate_timestamps = get_error_rate_data(
scan_log_filename, scan)
first_timestamp = get_first_timestamp(scan)
last_timestamp = get_last_timestamp(scan)
spent_epoch = last_timestamp - first_timestamp
error_rate_timestamps = [
ts - first_timestamp for ts in error_rate_timestamps
]
if not error_rate:
print('No error rate information found')
print('')
return
fig = plotille.Figure()
fig.width = 90
fig.height = 20
fig.register_label_formatter(float, num_formatter)
fig.register_label_formatter(int, num_formatter)
fig.y_label = 'Error rate'
fig.x_label = 'Time'
fig.color_mode = 'byte'
fig.set_x_limits(min_=0, max_=spent_epoch)
fig.set_y_limits(min_=0, max_=max(error_rate) * 1.1)
fig.plot(error_rate_timestamps, error_rate, label='Error rate')
print(fig.show())
print('')
def test_ellipse():
fig = plotille.Figure()
fig.with_colors = False
fig.width = 20
fig.height = 20
fig.ellipse(xcenter=0, ycenter=0, xamplitude=0.5, yamplitude=0.5)
c = fig.show(legend=False)
expected = """ (Y) ^
0.59974412 |
0.53976971 | ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⡇⠀⠀⠀⠀⠀⠀⠀⠀⠀
0.47979530 | ⠀⠀⠀⠀⠀⠀⠀⢀⠤⠤⡧⠤⢄⠀⠀⠀⠀⠀⠀⠀
0.41982088 | ⠀⠀⠀⠀⠀⢀⠎⠁⠀⠀⡇⠀⠀⠉⢢⠀⠀⠀⠀⠀
0.35984647 | ⠀⠀⠀⠀⡠⠊⠀⠀⠀⠀⡇⠀⠀⠀⠀⠣⡀⠀⠀⠀
0.29987206 | ⠀⠀⠀⢀⠇⠀⠀⠀⠀⠀⡇⠀⠀⠀⠀⠀⢸⠀⠀⠀
0.23989765 | ⠀⠀⠀⡇⠀⠀⠀⠀⠀⠀⡇⠀⠀⠀⠀⠀⠀⢇⠀⠀
0.17992324 | ⠀⠀⢸⠀⠀⠀⠀⠀⠀⠀⡇⠀⠀⠀⠀⠀⠀⠸⡀⠀
0.11994882 | ⠀⠀⡇⠀⠀⠀⠀⠀⠀⠀⡇⠀⠀⠀⠀⠀⠀⠀⡇⠀
0.05997441 | ⠀⠀⡇⠀⠀⠀⠀⠀⠀⠀⡇⠀⠀⠀⠀⠀⠀⠀⢇⠀
0 | ⣀⣸⣀⣀⣀⣀⣀⣀⣀⣀⣇⣀⣀⣀⣀⣀⣀⣀⣸⣀
-0.0599744 | ⠀⢸⠀⠀⠀⠀⠀⠀⠀⠀⡇⠀⠀⠀⠀⠀⠀⠀⡸⠀
-0.1199488 | ⠀⠀⡇⠀⠀⠀⠀⠀⠀⠀⡇⠀⠀⠀⠀⠀⠀⠀⡇⠀
-0.1799232 | ⠀⠀⡇⠀⠀⠀⠀⠀⠀⠀⡇⠀⠀⠀⠀⠀⠀⢀⠇⠀
-0.2398976 | ⠀⠀⢸⠀⠀⠀⠀⠀⠀⠀⡇⠀⠀⠀⠀⠀⠀⡸⠀⠀
-0.2998721 | ⠀⠀⠀⢇⠀⠀⠀⠀⠀⠀⡇⠀⠀⠀⠀⠀⢀⠇⠀⠀
-0.3598465 | ⠀⠀⠀⠸⡀⠀⠀⠀⠀⠀⡇⠀⠀⠀⠀⠀⡇⠀⠀⠀
-0.4198209 | ⠀⠀⠀⠀⠈⢆⠀⠀⠀⠀⡇⠀⠀⠀⢠⠊⠀⠀⠀⠀
-0.4797953 | ⠀⠀⠀⠀⠀⠈⠢⢄⠀⠀⡇⠀⢀⠤⠃⠀⠀⠀⠀⠀
-0.5397697 | ⠀⠀⠀⠀⠀⠀⠀⠀⠉⠉⡏⠉⠁⠀⠀⠀⠀⠀⠀⠀
-0.5997441 | ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⡇⠀⠀⠀⠀⠀⠀⠀⠀⠀
-----------|-|---------|---------|-> (X)
| -0.599744 0 0.5997441"""
assert expected == c
print(c)
def _plot_azimuth_and_elevation_from_time(date_series: Sequence[datetime.datetime],
azimuth_series: Sequence[float], elevation_series: Sequence[float],
width=50, height=20, scale_elevation=True, axis_in_local_time=True):
'''
Plot azimuth/elevation from time chart.
Series colors are compatible with gPredict.
'''
# We scale elevation, because plotille library allow draw only one Y axis.
if scale_elevation:
elevation_series = [el *4 for el in elevation_series]
if axis_in_local_time:
target_tz = tz.tzlocal()
date_series = [d.astimezone(target_tz) for d in date_series]
if len(date_series) < 2:
print("No plot. Not enough data.")
return
fig = plotille.Figure()
fig.width = width
fig.height = height
fig.color_mode = 'byte'
fig.set_y_limits(0, 360)
fig.plot(date_series, azimuth_series, label="Azimuth", lc=COLOR_BLUE)
elevation_label = "Elevation"
if scale_elevation:
elevation_label += " * 4"
else:
fig.plot(date_series, [90,] * len(date_series), interp=None, label="Y=90")
fig.plot(date_series, elevation_series, label=elevation_label, lc=COLOR_RED)
fig.y_label = "Degrees"
x_label = "Time (%s)" % ("local" if axis_in_local_time else "UTC",)
fig.x_label = x_label
print(fig.show(legend=True))
def display_total_deaths(
self, country_total_deaths: Country, x_axis_days_data: list,
y_axis_cumulative_deaths: list) -> None:
from screens import print_centred
import replit
import time
# instantiate total deaths plotille graph
figure_total_deaths = plotille.Figure()
# set heigh and width of the plotille graph
figure_total_deaths.width = 130
figure_total_deaths.height = 25
# set color mode of plotille graph to names
figure_total_deaths.color_mode = "names"
# set x axis label to days
figure_total_deaths.x_label = "days"
# set y axis label to deaths
figure_total_deaths.y_label = "deaths"
# set graph so x=0 and y=0
figure_total_deaths.set_x_limits(min_=0)
figure_total_deaths.set_y_limits(min_=0)
# command to not print the dotted x and y origins
figure_total_deaths.origin = False
# for loop to animate the total deaths graph - update x
# and y axis list data, clear screen & plotille graph, add
# data to plotille graph, print new plotille graph, rinse and repeat
for index in range(0, len(country_total_deaths.data)):
# update y_axis_cumulative_deaths list with next index data
y_axis_cumulative_deaths.append(
country_total_deaths.data[index]["Deaths"])
# update x_axis_days_data list with next index
x_axis_days_data.append(index)
# clear the screen
replit.clear()
# remove all plots from the total deaths plotille graph
figure_total_deaths.clear()
# add data to existing total deaths plotille graph
figure_total_deaths.scatter(
x_axis_days_data, y_axis_cumulative_deaths, lc="magenta",
label=f"{country_total_deaths.graph_name}")
# print graph header
print_centred("Cumulative Deaths")
# print new total deaths plotille graph with updated data
print(figure_total_deaths.show(legend=True))
# suspend execution for 0.1 seconds
time.sleep(.1)
def draw_worker_pool_size(scan_log_filename, scan):
worker_pool_sizes, worker_pool_timestamps = get_worker_pool_size_data(scan_log_filename, scan)
first_timestamp = get_first_timestamp(scan)
last_timestamp = get_last_timestamp(scan)
spent_time_epoch = last_timestamp - first_timestamp
worker_pool_timestamps = [ts - first_timestamp for ts in worker_pool_timestamps]
if not worker_pool_sizes:
print('No worker pool size data found')
return
print('Worker pool size over time')
print('')
fig = plotille.Figure()
fig.width = 90
fig.height = 20
fig.register_label_formatter(float, num_formatter)
fig.register_label_formatter(int, num_formatter)
fig.y_label = 'Worker pool size'
fig.x_label = 'Time'
fig.color_mode = 'byte'
fig.set_x_limits(min_=0, max_=spent_time_epoch)
fig.set_y_limits(min_=0, max_=None)
fig.plot(worker_pool_timestamps,
worker_pool_sizes,
label='Workers')
print(fig.show())
print('')
def draw_parser_process_memory_limit(scan_log_filename, scan):
memory_limit, memory_limit_timestamps = get_parser_process_memory_limit_data(scan_log_filename, scan)
first_timestamp = get_first_timestamp(scan)
last_timestamp = get_last_timestamp(scan)
spent_epoch = last_timestamp - first_timestamp
memory_limit_timestamps = [ts - first_timestamp for ts in memory_limit_timestamps]
if not memory_limit:
print('No parser process memory limit information found')
return
fig = plotille.Figure()
fig.width = 90
fig.height = 20
fig.register_label_formatter(float, num_formatter)
fig.register_label_formatter(int, num_formatter)
fig.y_label = 'Parser memory limit (MB)'
fig.x_label = 'Time'
fig.color_mode = 'byte'
fig.set_x_limits(min_=0, max_=spent_epoch)
fig.set_y_limits(min_=0, max_=max(memory_limit) * 1.1)
fig.plot(memory_limit_timestamps,
memory_limit,
label='Memory limit')
print(fig.show())
print('')
def draw_should_grep(scan_log_filename, scan):
should_grep_data, should_grep_timestamps = get_should_grep_data(
scan_log_filename, scan)
# Get the last timestamp to use as max in the graphs
first_timestamp = get_first_timestamp(scan)
last_timestamp = get_last_timestamp(scan)
spent_time_epoch = last_timestamp - first_timestamp
should_grep_timestamps = [
ts - first_timestamp for ts in should_grep_timestamps
]
if not should_grep_data:
print('No should_grep data found')
return
last_data = should_grep_data[-1]
print('should_grep() stats')
print(' Latest should_grep() count: %r' % last_data)
# Calculate %
last_data = should_grep_data[-1]
total = sum(v for k, v in last_data.iteritems())
total = float(total)
data_percent = dict(
(k, round((v / total) * 100)) for k, v in last_data.iteritems())
print(' Latest should_grep() percentages: %r' % data_percent)
print('')
fig = plotille.Figure()
fig.width = 90
fig.height = 20
fig.register_label_formatter(float, num_formatter)
fig.register_label_formatter(int, num_formatter)
fig.y_label = 'Percentage of rejected and accepted HTTP request and response grep tasks'
fig.x_label = 'Time'
fig.color_mode = 'byte'
fig.set_x_limits(min_=0, max_=spent_time_epoch)
fig.set_y_limits(min_=0, max_=None)
for key in should_grep_data[-1].keys():
key_slice = []
for data_point in should_grep_data:
total = sum(v for k, v in data_point.iteritems())
total = float(total)
if total == 0:
key_slice.append(0)
continue
data_percent = dict(
(k, (v / total) * 100) for k, v in data_point.iteritems())
key_slice.append(data_percent[key])
fig.plot(should_grep_timestamps, key_slice, label=key)
print(fig.show(legend=True))
print('')
def show_parser_errors(scan):
scan.seek(0)
timeout_count = 0
timeout_errors = []
timeout_errors_timestamps = []
memory_count = 0
memory_errors = []
memory_errors_timestamps = []
for line in scan:
if PARSER_TIMEOUT in line:
timeout_count += 1
timeout_errors.append(timeout_count)
timeout_errors_timestamps.append(get_line_epoch(line))
if PARSER_MEMORY_LIMIT in line:
memory_count += 1
memory_errors.append(memory_count)
memory_errors_timestamps.append(get_line_epoch(line))
first_timestamp = get_first_timestamp(scan)
last_timestamp = get_last_timestamp(scan)
spent_epoch = last_timestamp - first_timestamp
timeout_errors_timestamps = [ts - first_timestamp for ts in timeout_errors_timestamps]
memory_errors_timestamps = [ts - first_timestamp for ts in memory_errors_timestamps]
if not memory_errors and not timeout_errors:
print('No parser errors found')
return
print('Parser errors')
print(' Timeout errors: %s' % timeout_count)
print(' Memory errors: %s' % memory_count)
print('')
fig = plotille.Figure()
fig.width = 90
fig.height = 20
fig.y_label = 'Parser errors'
fig.x_label = 'Time'
fig.color_mode = 'byte'
#fig.set_x_limits(min_=0, max_=spent_epoch)
fig.set_y_limits(min_=0, max_=max(memory_count, timeout_count))
fig.plot(timeout_errors,
timeout_errors_timestamps,
label='Timeout errors',
lc=50)
fig.plot(memory_errors,
memory_errors_timestamps,
label='Memory errors',
lc=200)
print(fig.show(legend=True))
print('')
print('')
def plot_graph(data):
fig = plotille.Figure()
fig.color_mode = "byte"
fig.height = 15
fig.width = 130
fig.plot([x[2] for x in data], [x[1] for x in data], lc=42)
fig.show()
print(fig.show())
def plot(data, args, label):
"""Generate the plots as classical or scatter plots.
plot() is generating the classical or scatter plots according to the arguments
`args`.
Parameters
----------
data : float-array
2D-Numpy-array with `dtype=float`.
args : dict
Dictionary of the keywords and values from the parser.
label : str
The label of the scatter-plot(s) is the current filename.
"""
fig = plt.Figure()
fig.widht = args["size"][0]
fig.height = args["size"][1]
try:
if args["x_limits"]:
fig.set_x_limits(min_=args["x_limits"][0],
max_=args["x_limits"][1])
else:
fig.set_x_limits(min_=np.min(data[:, 0]), max_=np.max(data[:, 0]))
if args["y_limits"]:
fig.set_y_limits(min_=args["y_limits"][0],
max_=args["y_limits"][1])
else:
try:
fig.set_y_limits(min_=np.min(data[:, 1:]),
max_=np.max(data[:, 1:]))
except ValueError:
min_max = np.mean(data[:, 1:])
fig.set_y_limits(min_=min_max - min_max * 0.1,
max_=min_max + min_max * 0.1)
if args["color"]:
fig.color_mode = "rgb"
x = data[:, 0]
y = data[:, 1:]
if args["scatter"]:
fig = plot_scatter(fig, x=x, Y=y, label=label)
else:
fig = plot_plot(fig, x=x, Y=y, label=label)
if args["legend"]:
log(fig.show(legend=True))
else:
log(fig.show(legend=False))
except IndexError:
log(f"corrupted data in {label}", mode=1)
sys.exit(1)
def plot_it(xs, ys, names):
fig = plot.Figure()
fig.width=60 #(min(args.y)+max(args.x)) * 10
fig.height=20 #(min(args.y)+max(args.y)) * 10
fig.set_x_limits(min_= min(args.x), max_= max(args.x))
fig.set_y_limits(min_= min(args.y), max_= max(args.y))
fig.color_mode="byte"
#fig.plot([0, 10], [-1, 10], lc=25, label='First line')
for i in range(len(names)):
#print(len(xs), len(ys))
fig.plot(xs, ys[i], lc=50*(i+1), label=names[i])
print(fig.show(legend=-True))
def print_graph(stock_data, fx_data=pd.DataFrame()):
if (stock_data.empty):
print("No data found for this stock. Please try with correct symbol",
file=sys.stderr)
else:
fig = plotille.Figure()
fig.width = 60
fig.height = 30
fig.color_mode = 'byte'
x = stock_data['c']
y = stock_data['t']
fig.plot(y, x, lc=200, label='plot')
print(fig.show(legend=True))
def picture_oneplot(title, xlabel, ylabel, data, cols):
for i in cols:
fig = plotille.Figure()
fig.color_mode = 'byte'
fig.set_x_limits(min_=0)
fig.width = 60
fig.height = 20
#fig.background = 0
fig.plot(data[0][1:], data[i][1:], label=data[i][0])
print()
print(fig.show(legend=True))
print("title = ", title)
print("ylabel = ", ylabel)
print("xlabel = ", xlabel)
print()
def plot_drink_events(event, users):
global y_previous
drink_events = event['drink_events']
if not drink_events:
print('No drink events so far...')
return
usernames = {}
for user in users:
usernames[user['id']] = user['username']
user_drink_events = defaultdict(list)
for drink_event in drink_events:
drink_event['datetime'] = datetime.datetime.fromisoformat(
drink_event['datetime'].rstrip('Z'))
user_drink_events[drink_event['user']].append(drink_event)
first_de = min(drink_events, key=lambda de: de['datetime'])
last_de = max(drink_events, key=lambda de: de['datetime'])
w, h = terminal_size()
margin_w, margin_h = [20] * 2
fig = plotille.Figure()
fig.width = w - margin_w
fig.height = h - margin_h
fig.register_label_formatter(float, int_formatter)
fig.set_x_limits(min_=first_de['datetime'].timestamp(),
max_=last_de['datetime'].timestamp())
fig.set_y_limits(min_=0,
max_=max(len(evs) for evs in user_drink_events.values()))
fig.color_mode = 'byte'
user_des = sorted(user_drink_events.items(), key=lambda des: -len(des[1]))
for user_id, evs in user_des:
xs = [e['datetime'] for e in evs]
ys = list(range(1, len(evs) + 1))
fig.plot(xs,
ys,
lc=user_id,
label=f'{usernames[user_id]} ({len(evs)})')
y_previous = set()
print(fig.show(legend=True))
def __init__(
self, stocks: list, width: int, height: int, colors: list, *args, **kwargs
):
self.stocks = stocks
self.graph = ""
self.colors = colors
self.plot = plotille.Figure()
self.plot.width = width
self.plot.height = height
self.plot.color_mode = "rgb"
self.plot.X_label = "Time"
self.plot.Y_label = "Value"
if "timezone" in kwargs.keys():
self.timezone = pytz.timezone(kwargs["timezone"])
else:
self.timezone = pytz.utc
if "starttime" in kwargs.keys():
self.start = (
kwargs["startend"].replace(tzinfo=pytz.utc).astimezone(self.timezone)
)
else:
self.start = (
datetime.now()
.replace(hour=14, minute=30, second=0)
.replace(tzinfo=pytz.utc)
.astimezone(self.timezone)
)
if "endtime" in kwargs.keys():
self.end = (
kwargs["endtime"].replace(tzinfo=pytz.utc).astimezone(self.timezone)
)
else:
self.end = (
datetime.now()
.replace(hour=21, minute=0, second=0)
.replace(tzinfo=pytz.utc)
.astimezone(self.timezone)
)
self.plot.set_x_limits(min_=self.start, max_=self.end)
return
def show_queue_size_audit(scan):
scan.seek(0)
auditor_queue_sizes = []
auditor_queue_timestamps = []
for line in scan:
match = AUDITOR_DISK_DICT.search(line)
if match:
auditor_queue_sizes.append(int(match.group(1)))
auditor_queue_timestamps.append(get_line_epoch(line))
# Get the last timestamp to use as max in the graphs
first_timestamp = get_first_timestamp(scan)
last_timestamp = get_last_timestamp(scan)
spent_time_epoch = last_timestamp - first_timestamp
auditor_queue_timestamps = [
ts - first_timestamp for ts in auditor_queue_timestamps
]
if not auditor_queue_sizes:
print('No audit consumer queue size data found')
print('')
return
print('Audit consumer queue size')
print(' Latest queue size value: %s' % auditor_queue_sizes[-1])
print('')
fig = plotille.Figure()
fig.width = 90
fig.height = 20
fig.register_label_formatter(float, _num_formatter)
fig.register_label_formatter(int, _num_formatter)
fig.y_label = 'Items in Audit queue'
fig.x_label = 'Time'
fig.color_mode = 'byte'
fig.set_x_limits(min_=0, max_=spent_time_epoch)
fig.set_y_limits(min_=0, max_=None)
fig.plot(auditor_queue_timestamps, auditor_queue_sizes, label='Audit')
print(fig.show())
print('')
print('')
def generate_rgb_histogram(im: np.ndarray, show_axes: bool = False) -> str:
"""Generate histogram for terminal.
Parameters
----------
im
Image to evaluate
show_axes
Show x and y axis labels
Returns
-------
str:
Histogram to print
"""
img = np.asarray(im)
hist_width = 50
hist_height = 10
hist_bins = 256
# set up graph
fig = plotille.Figure()
fig.width = hist_width
fig.height = hist_height
fig.origin = False # Don't draw 0 lines
fig.set_x_limits(min_=0, max_=hist_bins - 1)
fig.set_y_limits(min_=0)
fig.color_mode = "names"
img_h, img_w, img_c = img.shape
colors = ["red", "green", "blue"]
chans = cv2.split(img)
# for i in range(img_c):
for (chan, color) in zip(chans, colors):
hist_data, bins = np.histogram(chan, bins=hist_bins, range=(0, 255))
fig.plot(bins[:hist_bins], hist_data, lc=color)
if not show_axes:
graph = ("\n".join([
"".join(ln.split("|")[1]) for ln in fig.show().splitlines()[1:-2]
]) + "\n")
else:
graph = fig.show()
return graph
def draw_rtt_histogram(scan_log_filename, scan):
rtts = get_rtt_histogram_data(scan_log_filename, scan)
fig = plotille.Figure()
fig.width = 90
fig.height = 20
fig.register_label_formatter(float, num_formatter)
fig.register_label_formatter(int, num_formatter)
fig.y_label = 'Count'
fig.x_label = 'RTT'
fig.set_x_limits(min_=0)
fig.set_y_limits(min_=0)
fig.color_mode = 'byte'
print('[rtt_histogram]')
print('')
print(plotille.hist(rtts, bins=25))
print('')
def show_queue_size_crawl(scan):
scan.seek(0)
crawl_queue_sizes = []
crawl_queue_timestamps = []
for line in scan:
match = CRAWLINFRA_DISK_DICT.search(line)
if match:
crawl_queue_sizes.append(int(match.group(1)))
crawl_queue_timestamps.append(get_line_epoch(line))
# Get the last timestamp to use as max in the graphs
first_timestamp = get_first_timestamp(scan)
last_timestamp = get_last_timestamp(scan)
spent_time_epoch = last_timestamp - first_timestamp
crawl_queue_timestamps = [
ts - first_timestamp for ts in crawl_queue_timestamps
]
if not crawl_queue_sizes:
print('No crawl consumer queue size data found')
return
print('Crawl consumer queue size')
print(' Latest queue size value: %s' % crawl_queue_sizes[-1])
print('')
fig = plotille.Figure()
fig.width = 90
fig.height = 20
fig.register_label_formatter(float, _num_formatter)
fig.register_label_formatter(int, _num_formatter)
fig.y_label = 'Items in CrawlInfra queue'
fig.x_label = 'Time'
fig.color_mode = 'byte'
fig.set_x_limits(min_=0, max_=spent_time_epoch)
fig.set_y_limits(min_=0, max_=None)
fig.plot(crawl_queue_timestamps, crawl_queue_sizes, label='Crawl')
print(fig.show())
print('')
print('')
