def plot_horn_action(self):
'''
plot the inductance curve from the horn switch
'''
now_str = dt.datetime.utcnow().strftime(self.date_fmt)
if self.header['IS_GOOD'] == 1:
goodbad = 'good'
else:
goodbad = 'bad'
if self.header['HORN_ID'] is not None:
hornid = self.header['HORN_ID']
else:
hornid = 'unknown'
if self.header['CHANNEL'] is not None:
channel = self.header['CHANNEL']
else:
channel = 'unknown'
infotxt = 'Horn %s is %s (measured on channel %s)' % (hornid, goodbad,
channel)
msg = '%s: %s' % (now_str, infotxt)
if self.plot_type == 'ascii':
gp.plot(self.dat,
terminal="dumb",
_with="points pointtype '+'",
unset="grid",
title=msg)
return
self.ax.plot(self.dat, label=msg)
self.ax.legend()
plt.pause(0.01)
return
def test_compl_sumatoria_constante(self):
print()
best, fitted = big_o.big_o(sumatoria.sumatoria_constante,
big_o.datagen.n_,
min_n=1,
max_n=10000,
n_measures=10000,
n_repeats=3,
n_timings=5,
verbose=False,
classes=[cmpl.Constant, cmpl.Quadratic],
return_raw_data=True)
xs = fitted['measures']
ys = fitted['times']
gp.plot(xs,
ys,
_with='lines',
terminal='dumb 60,30',
unset='grid',
title='sumatoria_constante',
xlabel='n',
ylabel='T(n)')
for k, v in fitted.items():
if isinstance(k, big_o.complexities.ComplexityClass):
residual = v
r2 = 1 - residual / (ys.size * ys.var())
print(k, f' (r={residual}, r^2={r2})')
if not isinstance(best, big_o.complexities.Constant):
warnings.warn(
f'Complejidad esperada Constante, complejidad estimada {best}')
def plot_adapt_scenario():
nb_iter = 300
nb_clients = {100: 200, 200: 400}
rtt_sla = 10
infra = Infra(1000, 100, 100)
monit = InfraMonitor(infra)
adapt = AdaptationEngine(infra, monit)
slaviol = []
instances = []
for i in range(nb_iter):
if i in nb_clients.keys():
infra.scale_clients(nb_clients[i], 200)
instances.append(monit.report_nb_instances())
slaviol.append(monit.report_sla_violations(rtt_sla))
adapt.adapt_scaling_log(rtt_sla)
x = np.arange(nb_iter)
gp.plot((x, np.array(slaviol), {
'legend': '# SLA violations',
'with': 'filledcurve x1 fill transparent solid 0.5'
}), (x, np.array(instances), {
'legend': '# instances',
'with': 'linespoints'
}),
xlabel="iterations",
ylabel="Count",
hardcopy="./plots/adapt_scenario.png")
def plot_history(history, dists, gnuplot):
if args.plot_history:
plt.switch_backend('agg')
fig = plt.figure(figsize=(4, 1.5), dpi=200)
plt.rcParams.update({'font.size': 7})
ax1 = fig.add_axes([0, 0, 0.4, 1])
ax1.plot(history.history['val_loss'][3:], "-", color="black", lw=0.5)
ax1.set_xlabel("Validation Loss")
#ax1.set_yscale("log")
#
ax2 = fig.add_axes([0.55, 0, 0.4, 1])
ax2.plot(history.history['loss'][3:], "-", color="black", lw=0.5)
ax2.set_xlabel("Training Loss")
#ax2.set_yscale("log")
#
fig.savefig(args.out + "_fitplot.pdf", bbox_inches='tight')
#sys.tracebacklimit = 0 #gp.plot throws an error when printing to stdout from command line
if gnuplot:
gp.plot(
np.array(history.history['val_loss'][3:]),
unset='grid',
terminal='dumb 60 20',
#set= 'logscale y',
title='Validation Loss by Epoch')
gp.plot((np.array(dists),
dict(histogram='freq', binwidth=np.std(dists) / 5)),
unset='grid',
terminal='dumb 60 20',
title='Test Error')
def show_chart(self):
config = get_config()
span = config['Chart span']
interval = config['Chart interval']
historicals = [[[
float(d['low_price']) * float(ac.info['quantity']),
float(d['open_price']) * float(ac.info['quantity']),
float(d['close_price']) * float(ac.info['quantity']),
float(d['high_price']) * float(ac.info['quantity'])
] for d in rs.get_stock_historicals(
ac.ticker, span=span, interval=interval)] for ac in self.holdings]
x = np.arange(len(historicals[0]))
low_price = np.array(
[sum([t[0] for t in date]) for date in zip(*historicals)])
open_price = np.array(
[sum([t[1] for t in date]) for date in zip(*historicals)])
high_price = np.array(
[sum([t[3] for t in date]) for date in zip(*historicals)])
close_price = np.array(
[sum([t[2] for t in date]) for date in zip(*historicals)])
try:
gp.plot(x, open_price, low_price, high_price, close_price,
terminal='dumb',
title=f'Past {span} | Low: ${round(min(low_price),2)} - ' \
f'High: ${round(max(high_price), 2)}',
_with='candlesticks', tuplesize=5,
unset=['grid','border','xtics','ytics']
)
except:
click.echo('Install gnuplot to view chart')
def plot_single_ascii(target, width, height, window=-1, xrange=None, yrange=None):
"""
Plot the target file using the specified width and height.
If window > 0, use it to specify the window size for rolling averages.
xrange and yrange are used to specify the zoom level of the plot.
"""
print("plotting %s" % str(target))
df = pd.read_csv(target, sep="\t")
steps = np.array(df["# Step"])
if window < 0:
window = len(steps) // width + 1
window = df["mean_episode_return"].rolling(window=window, min_periods=0)
returns = np.array(window.mean())
stderrs = np.array(window.std())
plot_options = {}
plot_options["with"] = "yerrorbars"
plot_options["terminal"] = "dumb %d %d ansi" % (width, height)
plot_options["tuplesize"] = 3
plot_options["title"] = "averaged episode return"
plot_options["xlabel"] = "steps"
if xrange is not None:
plot_options["xrange"] = xrange
if yrange is not None:
plot_options["yrange"] = yrange
gp.plot(steps, returns, stderrs, **plot_options)
def _comparar_curvas(fitted_a, fitted_b):
xs_a = fitted_a['measures']
ys_a = fitted_a['times']
xs_b = fitted_b['measures']
ys_b = fitted_b['times']
gp.plot((xs_a, ys_a), (xs_b, ys_b), _with='lines', terminal='dumb 60,30',
unset='grid', title='Comparación', xlabel='n', ylabel='tiempo')
def plot_log_adaptation():
nb_iter = 100
rtt_sla = 10
infra = Infra(1000, 100, 100)
monit = InfraMonitor(infra)
adapt = AdaptationEngine(infra, monit)
slaviol = []
instances = []
for i in range(nb_iter):
instances.append(monit.report_nb_instances())
slaviol.append(monit.report_sla_violations(rtt_sla))
adapt.adapt_scaling_log(rtt_sla)
x = np.arange(nb_iter)
gp.plot((x, np.array(slaviol), {
'legend': '# SLA violations',
'with': 'filledcurve x1 fill transparent solid 0.5'
}), (x, np.array(instances), {
'legend': '# instances',
'with': 'linespoints linewidth 2'
}),
xlabel="iterations",
ylabel="Count",
hardcopy="./plots/adapt_log.png")
def track( cur ):
global curr_loc_
global static_features_img_
global trajectory_
# Apply a good bilinear filter. This will smoothen the image but preserve
# the edges.
cur = cv2.bilateralFilter( cur, 5, 50, 50 )
p0 = cv2.goodFeaturesToTrack( cur, 200, 0.1, 5 )
insert_int_corners( p0 )
draw_point( cur, p0, 1 )
res = update_mouse_location( p0, cur )
p1 = res[ 'contour' ]
ellipse = res[ 'ellipse' ]
# if p1 is not None:
# for p in p1:
# (x, y) = p.ravel()
# cv2.circle( cur, (x,y), 10, 20, 2 )
# if ellipse is not None:
# cv2.drawContours( cur, [p1], 0, 255, 2 )
# cv2.ellipse( cur, ellipse, 1 )
# pass
display_frame( cur, 1 )
# Plot the trajectory
# toPlot = zip(*trajectory_[-100:])
if len( trajectory_ ) % 20 == 0:
y, x = zip( *trajectory_ )
# Smooth them
cols, rows = [ smooth( a, 20 ) for a in [y,x] ]
gpl.plot( cols, rows
, terminal = 'x11', _with = 'line'
# To make sure the origin is located at top-left.
, cmds = [ 'set yrange [:] reverse' ]
)
return
def plot_terminal(data, title, xtitle):
"""
Plot data to the terminal using gnuplotlib
"""
import gnuplotlib as gp
x = data.index.to_numpy()
y = data[title].to_numpy()
gp.plot(x, y, _with = 'lines', terminal = 'dumb 160,40', unset = 'grid', title = title, xlabel = xtitle)
def plot_saved_event(self, files):
'''
plot the inductance curved of a saved event
'''
if files is None: return
self.setup_horn_plot()
for f in files:
h = fits.open(f)
if len(h)!=2\
or 'INSTRUME' not in h[1].header.keys()\
or h[1].header['INSTRUME']!='QUBIC'\
or 'EXTNAME' not in h[1].header.keys()\
or h[1].header['EXTNAME']!='HORNSWITCH':
print('not a horn switch fits file: %s' % f)
h.close()
continue
dat = h[1].data.field(0)
obsdate = str2dt(h[1].header['DATE-OBS'])
obsdate_str = obsdate.strftime(self.date_fmt)
if 'IS_GOOD' in h[1].header.keys():
if h[1].header['IS_GOOD'] == 1:
goodbad = 'good'
else:
goodbad = 'bad'
else:
goodbad = 'unknown'
if 'HORN_ID' in h[1].header.keys():
hornid = h[1].header['HORN_ID']
else:
hornid = 'unknown'
if 'CHANNEL' in h[1].header.keys():
channel = h[1].header['CHANNEL']
else:
channel = 'unknown'
infotxt = 'Horn %s is %s (measured on channel %s)' % (
hornid, goodbad, channel)
msg = '%s: %s' % (obsdate_str, infotxt)
h.close()
if self.plot_type == 'ascii':
gp.plot(self.dat,
terminal="dumb",
_with="points pointtype '+'",
unset="grid",
title=msg)
continue
self.ax.plot(dat, label=msg)
self.ax.legend()
return
def pngplot(*l,**d):
out = os.environ["HOME"] + "/workspace/tmp/sway" + str(int(10000*r())) + ".pdf"
if os.path.exists(out):
os.remove(out)
gnuplotlib.plot(*l,
unset='grid',
output=out,
terminal = 'pdf solid color font ",10" size 3.5in,3.5in',
**d)
return out
def _comparar_curvas(fitted_a, fitted_b):
xs_a = fitted_a['measures']
ys_a = fitted_a['times']
xs_b = fitted_b['measures']
ys_b = fitted_b['times']
if importlib.util.find_spec('gnuplotlib') is not None:
import gnuplotlib as gp
gp.plot((xs_a, ys_a), (xs_b, ys_b), _with='lines', terminal='dumb 60,30',
unset='grid', title='Comparación', xlabel='n', ylabel='tiempo')
def plot_electricity(self):
x = np.array([x + 1 for x in range(12)])
sum_square_error = 0
for i in range(12):
sum_square_error += (self.actual[i] - self.e_plus[i]) ** 2
r_m_s_e = math.sqrt(sum_square_error)
gp.plot(
(x, np.array(self.actual), {'with': 'lines', 'legend': 'Actual'}),
(x, np.array(self.e_plus), {'with': 'lines', 'legend': 'EnergyPlus'}),
terminal='dumb 120,25', unset='grid', title='Comparing E+ to Actual (RMSE = %s)' % r_m_s_e
)
def pngplot(*l, **d):
out = os.environ["HOME"] + "/workspace/tmp/sway" + str(int(
10000 * r())) + ".pdf"
if os.path.exists(out):
os.remove(out)
gnuplotlib.plot(*l,
unset='grid',
output=out,
terminal='pdf solid color font ",10" size 3.5in,3.5in',
**d)
return out
def _graficar(fitted, titulo):
xs = fitted['measures']
ys = fitted['times']
gp.plot(xs, ys, _with='lines', terminal='dumb 60,30',
unset='grid', title=titulo, xlabel='n', ylabel='tiempo')
for k, v in fitted.items():
if isinstance(k, big_o.complexities.ComplexityClass):
residual = v
r2 = 1 - residual / (ys.size * ys.var())
print(k, f' (r={residual}, r^2={r2})')
def _plot_gnuplotlib(self, channel="0", title=None, **kwargs):
import gnuplotlib as gp
if title is None:
title = "Channel {} Tsys (K) vs Offset in {}".format(
channel, self.axis)
offset_data = self.channels[channel].offset_data
fit_x = np.linspace(offset_data[0], offset_data[-1], 200)
gp.plot((offset_data, self.channels[channel].tsys_data),
(fit_x,
DRbu.gauss_function(fit_x, *self.channels[channel]["popt"])),
_with='lines',
terminal='dumb 80,40',
unset='grid')
def plot_slaviol_instances(nb_run):
max_instances = 200 # Max number of instances
curves = {5: None, 10: None, 25: None, 50: None, 100: None, 200: None}
# Run for nb_run times
for j in range(nb_run):
c = {}
# Init arrays for this run
for rtt_sla in curves.keys():
c[rtt_sla] = []
# Init and monitor
infra = Infra(1000, 100, 100)
monit = InfraMonitor(infra)
# Increase (or decrease) number of instances
for i in range(max_instances):
#for i in range(max_instances,0,-1):
# Scale instances and reset usage
infra.scale_instances(i)
infra.reset_usage()
infra.select_instances()
# Evaluate nb of sla violation for each sla
for rtt_sla in curves.keys():
nb_slaviol = monit.report_sla_violations(rtt_sla)
c[rtt_sla].append(nb_slaviol)
# Add results for this run in the table
for rtt_sla in curves.keys():
if curves[rtt_sla] == None:
curves[rtt_sla] = np.atleast_2d(c[rtt_sla])
else:
curves[rtt_sla] = np.concatenate(
(curves[rtt_sla], np.atleast_2d(c[rtt_sla])), axis=0)
# Plot the results after average calculation on each column
nx = np.arange(max_instances)
gp.plot((nx, np.average(curves[5], axis=0), {
'legend': '5%'
}), (nx, np.average(curves[10], axis=0), {
'legend': '10%'
}), (nx, np.average(curves[25], axis=0), {
'legend': '25%'
}), (nx, np.average(curves[50], axis=0), {
'legend': '50%'
}), (nx, np.average(curves[100], axis=0), {
'legend': '100%'
}), (nx, np.average(curves[200], axis=0), {
'legend': '200%'
}),
hardcopy="./plots/slaviol_instances.png",
title="Variation of SLA violations with service deployment",
xlabel="# services instances",
ylabel="# SLA violations")
def plot(self,terminal=True, xpad=[-1,1], ypad=[-0.2,1.2]):
""" Plots the fuzzy_member to ether the terminal or gui
terminal=True plots the fuzzy_member to terminal else gui.
xpad = [-1,1] adds x padding to the plots so that it prints niser
ypad = [-0.2, 1.2] adds y padding to the plot so it prints niser
return None
"""
x = np.arange(self.points[0,0]+xpad[0],self.points[-1,0]+xpad[1],0.01)
y = np.array(list(map(lambda x: float(self.fire(x)[0]), x)))
print(f'shapes x={np.shape(x)} y={np.shape(y)}')
if terminal:
gp.plot(x,y, _yrange = ypad, terminal="dumb 80,20", unset='grid')
else:
gp.plot(x,y, _yrange = ypad, unset='grid')
def animate(i):
global data_
# global time_text_
global box_
global tvec_, y1_, y2_
global cap_
# global fig_ax_
t = float(i) / fps_
ret, img = cap_.read()
img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
(x0, y0), (x1, y1) = box_
try:
frame = img[y0:y1, x0:x1]
except Exception as e:
print('[WARN] Frame %s dropped' % i)
return lines_.values(), time_text_
if save_video_:
fig_ax_.imshow(frame[::2, ::2], interpolation='nearest')
else:
#cv2.imshow('image', frame)
cv2.waitKey(1)
inI, outI, edge, pixal = webcam.process_frame(frame)
cv2.imshow('Eye', np.concatenate((frame, outI)))
tvec_.append(t)
y1_.append(edge)
y2_.append(pixal)
update_axis_limits(axes_['raw'], t, edge)
update_axis_limits(axes_['raw_twin'], t, pixal)
tA, bA = [0], [0]
if i % int(fps_ * 0.1) == 0:
data_ = np.array((tvec_, y1_)).T
try:
tA, bA = extract.find_blinks_using_edge(data_[:, :])
except Exception as e:
print(
'[WARN] Failed to detect blink data using egdes in frame %s' %
i)
tA, bA = [0], [0]
gplt.plot(
(np.array(tvec_[-1000:]), np.array(y1_[-1000:]))
# , ( np.array(tA), np.array(bA), {} )
,
title='Blink detection. Running window 1000 frames',
terminal='x11')
def plot_terminal(data, title, xlabel, yrange):
"""
Plot data to the terminal using gnuplot
"""
import gnuplotlib as gp
x = data.index.to_numpy()
y = data[title].to_numpy()
gp.plot(x,
y,
_with='lines',
terminal='dumb 160,40',
unset='grid',
set=f"yrange [{yrange[0]}:{yrange[1]}]",
title=title,
xlabel=xlabel)
def plot_rtt_distribution():
data = []
MED_RTT = 100
for i in range(1, 500):
l = Link(MED_RTT)
data.append(l.rtt)
h, b = np.histogram(data, 100)
h2 = np.cumsum(h)
b2 = b[1:]
gp.plot(b2,
h2,
hardcopy="./plots/rtt_dist.png",
title="Cumulative distribution of RTT",
xlabel="RTT (ms)",
ylabel="# nodes (cumul)")
def animate(i):
global data_
# global time_text_
global box_
global tvec_, y1_, y2_
global cap_
# global fig_ax_
t = float(i) / fps_
ret, img = cap_.read()
img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
(x0, y0), (x1, y1) = box_
try:
frame = img[y0:y1,x0:x1]
except Exception as e:
print('[WARN] Frame %s dropped' % i)
return lines_.values(), time_text_
if save_video_:
fig_ax_.imshow(frame[::2,::2], interpolation='nearest')
else:
#cv2.imshow('image', frame)
cv2.waitKey(1)
inI, outI, edge, pixal = webcam.process_frame(frame)
cv2.imshow('Eye', np.concatenate((frame, outI)))
tvec_.append(t); y1_.append(edge); y2_.append(pixal)
update_axis_limits(axes_['raw'], t, edge)
update_axis_limits(axes_['raw_twin'], t, pixal)
tA, bA = [0], [0]
if i % int(fps_*0.1) == 0:
data_ = np.array((tvec_, y1_)).T
try:
tA, bA = extract.find_blinks_using_edge(data_[:,:])
except Exception as e:
print('[WARN] Failed to detect blink data using egdes in frame %s' % i)
tA, bA = [0], [0]
gplt.plot( ( np.array(tvec_[-1000:]), np.array( y1_[-1000:] ))
# , ( np.array(tA), np.array(bA), {} )
, title = 'Blink detection. Running window 1000 frames'
, terminal = 'x11'
)
def optim(objf, wolf_count, iters, plot):
print(f'optimizing {objf["name"]}:')
best_result, convergence = GWO(objf, wolf_count, iters)
print(f' range: <{objf["low"]},{objf["high"]}>')
print(f' dimension: {objf["dim"]}')
print(f' optimum: {objf["opt"]}')
print(f' wolf count: {wolf_count}; iters: {iters}')
print(
f' result: {best_result} (distance: {np.abs(objf["opt"] - best_result)})'
)
if plot:
import gnuplotlib as gp
print(' convergence plot:')
gp.plot(convergence,
terminal='dumb 80,40',
unset='grid',
_with='lines')
def ram():
free = [0] * 100
y = [i for i in range(0, 100)]
while (True):
os.system("clear")
free.append(getram())
free.pop(0)
ui_design.color(3)
gb.plot(np.array(y),
np.array(free),
terminal="dumb",
_with='lines',
unset='grid')
ui_design.reset()
print(free[len(free) - 1])
os.system("clear")
time.sleep(5)
def battery():
batt = [get_battery()] * 1000
y = [i for i in range(0, 100)]
while (True):
os.system("clear")
batt.append(get_battery())
batt.pop(0)
if (batt[len(batt) - 1] > 60):
ui_design.color(2)
elif (batt[len(batt) - 1] > 30):
ui_design.color(4)
else:
ui_design.color(1)
gb.plot(np.array(batt), terminal="dumb", _with='lines', unset='grid')
ui_design.reset()
print(batt[len(batt) - 1])
os.system("clear")
time.sleep(1)
def reader(self):
"""
Chart screen
Note : because of the raw_input only
a threaded function can refresh x time (it seems)
Also the thread is surrounded by 2 os.system('clear')
otherwise they have no effect
"""
os.system('clear')
threading.Timer(self.refresh_time, self.reader).start()
os.system('clear')
y_price, x_time = self.requestData()
y = np.array(y_price)
x = np.array(x_time)
my_title = '* %s/%s (%s) *' % (self.from_cur, self.to_cur, self.time_s)
gp.plot(
x,
y,
xlabel='\\nTime (GMT)',
ylabel=self.to_cur,
set=[
'lmargin 12',
'ylabel',
'xlabel',
'xdata time',
'format x "%s"' % self.timeform
],
unset='grid',
_with='lines',
terminal='dumb %s %s' % (self.width, self.height),
title=my_title+'\\n'+'*'*(len(my_title))
)
print(self.colorize('[Enter] menu', 'grey'))
choice = raw_input('')
self.exec_menu(choice)
def run_test(conf):
"""Test connectivity to TriOS RAMSES radiometer sensors using the routines used in the main application"""
ports = list_ports.comports()
for p in ports:
print(p)
config = conf['RADIOMETERS']
rad, Rad_manager = rad_init(config, ports)
ports = [config['port1'], config['port2'], config['port3']]
print("Starting radiometry manager")
radiometry_manager = Rad_manager(rad)
time.sleep(1)
print(radiometry_manager)
trig_id, specs, sids, itimes = radiometry_manager.sample_all(
datetime.datetime.now())
for i, s in zip(sids, specs):
print("Received spectrum from {0}: {1}".format(i, s))
if plot:
gp.plot(np.array(s), terminal='dumb 80,40', unset='grid')
time.sleep(0.5)
print("Stopping radiometry manager threads")
if radiometry_manager is not None:
radiometry_manager.stop()
# switch off gpio
if rad['use_gpio_control']:
print("Switch off GPIO control")
pin = int(rad['gpio1'])
GPIO.setmode(GPIO.BCM)
GPIO.output(pin, GPIO.LOW)
GPIO.cleanup()
sys.exit(0)
args = parser.parse_args()
segment_boundaries = np.loadtxt(args.segments_filename, usecols=(2, 3))
segment_lengths = segment_boundaries[:, 1] - segment_boundaries[:, 0]
count = len(segment_lengths)
mean = np.mean(segment_lengths)
median = np.median(segment_lengths)
print("num segments read: {:d}".format(count))
print("total time (h): {:.2f}".format(np.sum(segment_lengths) / 3600))
print("mean (s): {:.2f}".format(mean))
print("median (s): {:.2f}".format(median))
print("skew: {:.2f}".format(st.skew(segment_lengths, bias=True)))
print("skew [corrected]: {:.2f}".format(st.skew(segment_lengths, bias=False)))
print("skewtest: {}".format(st.skewtest(segment_lengths)))
print("kurtosis: {:.2f}".format(st.kurtosis(segment_lengths)))
# Figure out how many segments would fill the desired number of hours,
# then round up to the nearest 10k.
possible_num_hours_segmentations = (100, 300, 500, 1000, 1500, 3000)
print("=== from mean ===")
for num_hours in possible_num_hours_segmentations:
num_segments = int(num_hours * 3600 / mean)
print("{:d} h: {:d} ({:d}) segments".format(num_hours, round(num_segments, -4), num_segments))
print("=== from median ===")
for num_hours in possible_num_hours_segmentations:
num_segments = int(num_hours * 3600 / median)
print("{:d} h: {:d} ({:d}) segments".format(num_hours, round(num_segments, -4), num_segments))
gp.plot((segment_lengths, {"histogram": "freq", "binwidth": 1}))
def textplot(*l,**d):
assert len(l[0]) > 0, "no data found"
gnuplotlib.plot(*l,unset='grid',
terminal='dumb 80 30',**d)
def two_line_test_func(results, plots, two_line_threshold=0.5):
"""
Perform a double linear regression on intersecting subsets of the data in results to determine whether to terminate
and how many dimensions to return in the RC during two_line_test.
Can only be called with len(results) >= 5.
Parameters
----------
results : list
List of dictionary objects indexed by step of two_line_test, each possessing attribute 'fun' giving the optimization
score for that step
plots : bool
If True, plot lines using gnuplot
two_line_threshold : float
Ratio of second slope to first slope (as a fraction) below which the two-line test can pass
Returns
-------
out : int
Index of selected 'best' RC from two-line test; or, -1 if no best RC could be determined
"""
if len(results) < 5:
raise RuntimeError(
'two_line_test can only be called with at least 5 optimized models'
)
best_closest = [
] # result for which the intersection is closest to the shared point
for test_index in range(
len(results) -
2): # - 2 to account for minimum of two points in each line
first_segment = range(1, 3 + test_index)
second_segment = range(first_segment[-1], len(results) + 1)
opt1 = stats.linregress(first_segment,
[results[i - 1].fun for i in first_segment])
opt2 = stats.linregress(second_segment,
[results[i - 1].fun for i in second_segment])
# Now evaluate closest point in results to the intersection of the two lines
x_intersect = (opt1.intercept - opt2.intercept) / (opt2.slope -
opt1.slope)
y_intersect = (opt1.slope * x_intersect) + opt1.intercept
x_val = 0 # initialize index for keeping track of x values
min_diff = -1 # initialize smallest distance between intersection and point
closest = 0 # initialize index of closest point to intersection
for result in results:
y_val = result.fun
x_val += 1
y_diff = y_val - y_intersect
x_diff = x_val - x_intersect
diff = numpy.sqrt(y_diff**2 + x_diff**2)
if min_diff < 0:
min_diff = diff
closest = [x_val, diff]
elif diff < min_diff:
min_diff = diff
closest = [x_val, diff]
# if the closest point to the intersection is the shared point of the lines;
if closest[0] == test_index + 2:
if not best_closest: # for the first time
best_closest = [closest, opt1, opt2]
elif closest[1] < best_closest[0][1]: # update the closest yet
best_closest = [closest, opt1, opt2]
if gnuplot and plots:
if len(results[0].x) + 2 == len(
results[1].x
): # if this is True, results include rate-of-change terms
min_dims = (
len(results[0].x) - 1
) / 2 # smallest model dimensionality to be plotted (-1 for constant)
else: # no rate-of-change terms
min_dims = len(results[0].x) - 1
points1 = [[i + min_dims for i in range(len(results))],
[
best_closest[1].slope * (i + 1) +
best_closest[1].intercept for i in range(len(results))
]]
points2 = [[i + min_dims for i in range(len(results))],
[
best_closest[2].slope * (i + 1) +
best_closest[2].intercept for i in range(len(results))
]]
gnuplotlib.plot(
(numpy.asarray([item + min_dims for item in range(len(results))]),
numpy.asarray([result.fun for result in results])),
(numpy.asarray(points1[0]), numpy.asarray(points1[1]), {
'legend': '1st slope: ' + '%.3f' % best_closest[1].slope
}), (numpy.asarray(points2[0]), numpy.asarray(points2[1]), {
'legend': '2nd slope: ' + '%.3f' % best_closest[2].slope
}),
_with='lines',
terminal='dumb 80,40',
unset='grid')
if plots:
print('Two_line_test plot data:')
print(' Model scores: ' +
str(numpy.asarray([result.fun for result in results])))
print(' First line values: ' + str(points1[1]))
print(' Second line values: ' + str(points2[1]))
if not best_closest: # no pairs of lines whose intersection was closest to their shared point
print(
'Two line test: found no suitable model, performing an additional optimization step and retrying'
)
return -1
slope_fract = best_closest[2].slope / best_closest[1].slope
if slope_fract > two_line_threshold: # best point does not meet threshold for relative difference in slopes
print('Two line test: best model has ratio of slopes ' +
str(slope_fract) + ', which does not meet threshold ' +
str(two_line_threshold) +
'; performing an additional optimization step and retrying')
return -1
else: # DOES meet threshold; return the index of the passing result
return best_closest[0][
0] - 1 # - 1 because of different indexing standards
y_3d = (np.cos(ph) * np.sin(th)) .ravel()
z_3d = (np.sin(ph) * np.ones( th.shape )) .ravel()
sleep_interval = 1
#################################
# Now the demos!
#################################
# first, some very basic stuff. Testing implicit domains, multiple curves in
# arguments, packed broadcastable data, etc
gp.plot(x**2)
time.sleep(sleep_interval)
gp.plot(-x, x**3)
time.sleep(sleep_interval)
gp.plot((x**2))
time.sleep(sleep_interval)
gp.plot((-x, x**3, {'with': 'lines'}), (x**2,))
time.sleep(sleep_interval)
gp.plot( x, np.vstack((x**3, x**2)) )
time.sleep(sleep_interval)
gp.plot( np.vstack((-x**3, x**2)), _with='lines' )
time.sleep(sleep_interval)
gp.plot( (np.vstack((x**3, -x**2)), {'with': 'points'} ))
time.sleep(sleep_interval)
#################################
# Other metrics
trainingSummary = lrModel.summary
trainingSummary.roc.show()
print("areaUnderROC: " + str(trainingSummary.areaUnderROC))
#### Bonus: Visualisation example
# Simple visualisations
$ sudo yum install gnuplot
$ sudo /usr/local/bin/pip install gnuplotlib
$ sudo /usr/local/bin/pip install pandas
# Python Data Analysis Library
# http://pandas.pydata.org/
import pandas as pd
import gnuplotlib as gp
roc = trainingSummary.roc.toPandas()
gp.plot( (roc.FPR, roc.TPR),
_with = 'lines',
terminal = 'dumb 80,40',
unset = 'grid')
# See also: https://github.com/justmarkham/DAT8/blob/master/notebooks/12_titanic_confusion.ipynb
# and: https://github.com/justmarkham/DAT8#class-12-logistic-regression
def plot_multiple_ascii(
target,
width,
height,
window=-1,
xrange=None,
yrange=None,
no_legend=False,
shuffle=False,
):
"""
Plot files under the target path using the specified width and height.
If window > 0, use it to specify the window size for rolling averages.
xrange and yrange are used to specify the zoom level of the plot.
Set no_legend to true to save the visual space for the plot.
shuffle randomizes the order of the plot (does NOT preserve auto-assigned curve
labels), which can help to see a curve which otherwise is overwritten.
"""
dfs = collect_logs(target)
if window < 0:
max_size = max(len(df["# Step"]) for name, df in dfs)
window = 2 * max_size // width + 1
datasets = []
for name, df in dfs:
steps = np.array(df["# Step"])
if window > 1:
roll = df["mean_episode_return"].rolling(window=window, min_periods=0)
try:
rewards = np.array(roll.mean())
except pd.core.base.DataError:
print("Error reading file at %s" % name)
continue
else:
rewards = np.array(df["mean_episode_return"])
if no_legend:
datasets.append((steps, rewards))
else:
datasets.append((steps, rewards, dict(legend=" " + name + ":")))
errs = len(dfs) - len(datasets)
if errs > 0:
print(
"Skipped %d runs (%f) due to errors reading data" % (errs, errs / len(dfs))
)
print(
"Plotting %d runs with window_size %d from %s" % (len(datasets), window, target)
)
plot_options = {}
plot_options["terminal"] = "dumb %d %d ansi" % (width, height)
plot_options["tuplesize"] = 2
plot_options["title"] = "averaged episode return"
plot_options["xlabel"] = "steps"
plot_options["set"] = "key outside below"
if xrange is not None:
plot_options["xrange"] = xrange
if yrange is not None:
plot_options["yrange"] = yrange
if shuffle:
random.shuffle(datasets)
gp.plot(*datasets, **plot_options)
import numpy as np
import gnuplotlib as gp
x = np.arange(101) - 50
gp.plot(x**2)
#[ basic parabola plot pops up ]
g1 = gp.gnuplotlib(title='Parabola with error bars', _with='xyerrorbars')
g1.plot(x**2 * 10,
np.abs(x) / 10,
np.abs(x) * 5,
legend='Parabola',
tuplesize=4)
#[ parabola with x,y errobars pops up in a new window ]
x, y = np.ogrid[-10:11, -10:11]
gp.plot(x**2 + y**2,
title='Heat map',
unset='grid',
cmds='set view map',
_with='image',
tuplesize=3)
#[ Heat map pops up where first parabola used to be ]
theta = np.linspace(0, 6 * np.pi, 200)
z = np.linspace(0, 5, 200)
g2 = gp.gnuplotlib(_3d=True)
g2.plot(np.cos(theta), np.vstack((np.sin(theta), -np.sin(theta))), z)
#[ Two 3D spirals together in a new window ]
x = np.arange(1000)
def makeplots(dohardcopy, processoptions_base):
processoptions = copy.deepcopy(processoptions_base)
gp.add_plot_option(processoptions, 'set', ('xtics 0.01', 'ytics 0.01'))
# The key should use smaller text than the rest of the plot, if possible
if 'terminal' in processoptions:
m = re.search('font ",([0-9]+)"', processoptions['terminal'])
if m is not None:
s = int(m.group(1))
gp.add_plot_option(
processoptions, 'set',
('xtics 0.01', 'ytics 0.01', f'key font ",{int(s*0.8)}"'))
if dohardcopy:
processoptions['hardcopy'] = \
f'{args.make_documentation_plots_path}--ellipses.{extension}'
processoptions['terminal'] = shorter_terminal(
processoptions['terminal'])
# Hardcopies do things a little differently, to be nicer for docs
gp.plot(*subplots,
multiplot=f'layout 1,2',
unset='grid',
**processoptions)
else:
# Interactive plotting, so no multiplots. Interactive plots
for p in subplots:
gp.plot(*p[:-1], **p[-1], **processoptions)
processoptions = copy.deepcopy(processoptions_base)
if dohardcopy:
processoptions['hardcopy'] = \
f'{args.make_documentation_plots_path}--p0-p1-magnitude-covariance.{extension}'
processoptions['title'] = title_covariance
gp.plotimage(
np.abs(Var_p_joint.reshape(Npoints * 3, Npoints * 3)),
square=True,
xlabel='Variable index (left point x,y,z; right point x,y,z)',
ylabel='Variable index (left point x,y,z; right point x,y,z)',
**processoptions)
processoptions = copy.deepcopy(processoptions_base)
binwidth = np.sqrt(Var_ranges_joint[0]) / 4.
equation_range0_observed_gaussian = \
mrcal.fitted_gaussian_equation(x = ranges_sampled[0],
binwidth = binwidth,
legend = "Idealized gaussian fit to data")
equation_range0_predicted_joint_gaussian = \
mrcal.fitted_gaussian_equation(mean = ranges[0],
sigma = np.sqrt(Var_ranges_joint[0]),
N = len(ranges_sampled[0]),
binwidth = binwidth,
legend = "Predicted-joint")
equation_range0_predicted_calibration_gaussian = \
mrcal.fitted_gaussian_equation(mean = ranges[0],
sigma = np.sqrt(Var_ranges_calibration[0]),
N = len(ranges_sampled[0]),
binwidth = binwidth,
legend = "Predicted-calibration")
equation_range0_predicted_observations_gaussian = \
mrcal.fitted_gaussian_equation(mean = ranges[0],
sigma = np.sqrt(Var_ranges_observations[0]),
N = len(ranges_sampled[0]),
binwidth = binwidth,
legend = "Predicted-observations")
if dohardcopy:
processoptions['hardcopy'] = \
f'{args.make_documentation_plots_path}--range-to-p0.{extension}'
processoptions['title'] = title_range0
gp.add_plot_option(processoptions, 'set', 'samples 1000')
gp.plot(
ranges_sampled[0],
histogram=True,
binwidth=binwidth,
equation_above=(equation_range0_predicted_joint_gaussian,
equation_range0_predicted_calibration_gaussian,
equation_range0_predicted_observations_gaussian,
equation_range0_observed_gaussian),
xlabel="Range to the left triangulated point (m)",
ylabel="Frequency",
**processoptions)
processoptions = copy.deepcopy(processoptions_base)
binwidth = np.sqrt(Var_distance) / 4.
equation_distance_observed_gaussian = \
mrcal.fitted_gaussian_equation(x = distance_sampled,
binwidth = binwidth,
legend = "Idealized gaussian fit to data")
equation_distance_predicted_gaussian = \
mrcal.fitted_gaussian_equation(mean = distance,
sigma = np.sqrt(Var_distance),
N = len(distance_sampled),
binwidth = binwidth,
legend = "Predicted")
if dohardcopy:
processoptions['hardcopy'] = \
f'{args.make_documentation_plots_path}--distance-p1-p0.{extension}'
processoptions['title'] = title_distance
gp.add_plot_option(processoptions, 'set', 'samples 1000')
gp.plot(distance_sampled,
histogram=True,
binwidth=binwidth,
equation_above=(equation_distance_predicted_gaussian,
equation_distance_observed_gaussian),
xlabel="Distance between triangulated points (m)",
ylabel="Frequency",
**processoptions)
