def extrapolation(what, **kwargs):
what = 'extrap_'+what
fname = utils.filename(what, **kwargs)
if path.exists(fname):
return utils.load(fname)
else:
import pairfield_correlations, density_correlations, density, energy
if what == 'extrap_pairfield':
return utils.load_or_evaluate(what, pairfield_correlations.evaluate_extrap_at, **kwargs)
elif what == 'extrap_densdens':
return utils.load_or_evaluate(what, density_correlations.evaluate_extrap_at, **kwargs)
elif what == 'extrap_density':
return utils.load_or_evaluate(what, density.evaluate_extrap_at, **kwargs)
elif what == 'extrap_energy':
return utils.load_or_evaluate(what, energy.evaluate_extrap, **kwargs)
else:
raise Exception('`%s` is not a valid measurement.' % what)
def result(what, **kwargs):
fname = utils.filename(what, **kwargs)
if path.exists(fname):
return utils.load(fname)
else:
try:
import pyalps_dset
import pairfield_correlations, density_correlations, density, amplitudes
if what == 'pairfield':
return utils.load_or_evaluate(what, pairfield_correlations.evaluate, **kwargs)
elif what == 'densdens':
return utils.load_or_evaluate(what, density_correlations.evaluate, **kwargs)
elif what == 'density':
return utils.load_or_evaluate(what, density.evaluate, **kwargs)
elif what == 'density_fit':
return utils.load_or_evaluate(what, density.evaluate_fit, **kwargs)
elif what == 'density_amplitudes':
return utils.load_or_evaluate(what, amplitudes.evaluate, **kwargs)
else:
raise Exception('`%s` is not a valid measurement.' % what)
except ImportError, e:
print 'To execute new evaluations you need the ALPS.Python library.'
print e
#!/usr/bin/env python3
from utils import filename, save2pdf, setup
from utils.plots_helper import lof
from matplotlib.backends.backend_pdf import PdfPages
make, fname = filename("lof-plots.pdf")
dfile = "../datasets/real/intel/sensors-1000-dirty.txt"
pdf = PdfPages(fname)
for k in [2, 10]:
title = "Local Outlier Factor\n$k=" + str(k) + "$"
ofile = "../results/k" + str(k) + "data01.out"
setup()
lof(title, dfile, ofile)
save2pdf(pdf)
pdf.close()
x = linspace(-10, 10, num = 200)
y = gaussian(x, 0, 1)
NDEV = 1.5
nlo = bisect_left(x, -NDEV)
nhi = bisect_left(x, +NDEV)
xlo, ylo = x[:nlo], y[:nlo]
xmi, ymi = x[nlo-1:nhi+1], y[nlo-1:nhi+1]
xhi, yhi = x[nhi:], y[nhi:]
ax = pyplot.axes(frameon = False)
ax.set_xlim((-5,5))
pyplot.plot(xmi, ymi, color = TANGO["green"][2])
pyplot.fill_between(xmi, 0, ymi, color = TANGO["green"][1])
for (xx, yy) in ((xlo, ylo), (xhi, yhi)):
pyplot.plot(xx, yy, color = TANGO["red"][1])
pyplot.fill_between(xx, 0, yy, color = TANGO["red"][0])
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
pyplot.tight_layout()
batch_mode, fname = filename("gaussians-preview.pdf")
if batch_mode:
gaussian_plt()
pyplot.savefig(fname, transparent = True)
def __init__(self, path, url_base=None, url_map=dict(), scss_root=None):
self.path = path
self.url_base = url_base if url_base != None\
else (os.path.dirname(path) + os.sep\
if os.path.dirname(path) != '' else '')
self.url_map = url_map
self.scss_root = scss_root
self.filename, self.basename, self.extension = filename(path)
self.dead = False
self.frozen = False
self.bootstrap = False
self.placeholder = False
self.stale_age = self.STALE_AGE
self.noversion = False
self.nomap = False
self._map = ''
self._version = -1
self._uglyversion = self._base36encode(int(time.time() * 1000)) + \
str(random.randrange(0, 9999))
self._file_depend = []
self._tempfile = []
# scan cfile attribute
with open(path) as lines:
for line in lines:
matchobj = re.search(self.rdead, line)
if matchobj:
self.dead = True
return
matchobj = re.search(self.rbootstrap, line)
if matchobj:
self.bootstrap = True
matchobj = re.search(self.rplaceholder, line)
if matchobj:
self.placeholder = True
matchobj = re.search(self.rmap, line)
if matchobj:
self._map = matchobj.group(1)
matchobj = re.search(self.rnomap, line)
if matchobj:
self.nomap = True
matchobj = re.search(self.rversion, line)
if matchobj:
self._version = int(matchobj.group(1))
matchobj = re.search(self.rnoversion, line)
if matchobj:
self.noversion = True
matchobj = re.search(self.rdepend, line)
if matchobj:
dependstr = matchobj.group(1)
self._file_depend.extend(dependstr.split(','))
if self.version < 0:
self.update_version()
pyplot.rcParams['xtick.major.pad'] = '2'
pyplot.figure(figsize = (.75, .4))
ax = pyplot.axes(frameon = True)
ax.set_xlim(0, 5)
for side in ('top', 'left', 'right'):
ax.spines[side].set_visible(False)
ax.yaxis.set_visible(False)
pyplot.xticks([x + W for x in xs], labels, rotation=45, ha='right')
ax.xaxis.set_ticklabels([str(l) for l in labels])
ax.tick_params(top=False, bottom=False)
pyplot.bar(xs, ys, width = W, color = colors, edgecolor = edgecolors, linewidth=0.5)
batch, fname = filename("table-histograms.pdf")
if batch:
pdf = PdfPages(fname)
YS = [[1,3,3],[2,1.01,2,2],[1,6],[1,6]]
LABELS = [["1970","2014","2015"], ["Wed.","Thu.","Fri.","Sun."], ["8", "12"], [r"nnn", r"nn"]]
for ys, labels in zip(YS, LABELS):
make_hist(ys, labels)
if batch:
pyplot.savefig(pdf, pad_inches=1, format = 'pdf')
else:
pyplot.show()
matplotlib.rcParams['lines.linewidth'] = 2
def histogram_plt():
y = [0.1, 1, 5, 6, 0.2, 1, 10, 8, 0]
x = linspace(0, 8, num=9)
T = 0.5
FILLS = (TANGO["red"][0], TANGO["green"][1])
STROKES = (TANGO["red"][1], TANGO["green"][2])
colors = [FILLS[yy > T] for yy in y]
edgecolors = [STROKES[yy > T] for yy in y]
pyplot.clf()
pyplot.bar(x, y, width=1)
ax = pyplot.axes(frameon=False)
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
ax.set_xlim(-1, 10)
pyplot.bar(x, y, width=1, color=colors, edgecolor=edgecolors, linewidth=2)
pyplot.tight_layout()
batch_mode, fname = filename("models-plots.pdf")
if batch_mode:
histogram_plt()
pyplot.savefig(fname, transparent=True)
#!/usr/bin/env python3
from utils import filename
from utils.plots_helper import sensors3D
from matplotlib.backends.backend_pdf import PdfPages
from matplotlib import pyplot
batch, fname = filename("3d-plots.pdf")
dfile = "../datasets/real/intel/sensors-1000-dirty.txt"
MIX_OUTFILE = "../results/sensors_dirty_stat{}_mixture{}_{}.out"
GAUSS_OUTFILE = "../results/sensors_dirty_stat{}_gaussian{}.out"
LOF_OUTFILE = "../results/sensors_dirty_lof{}.out"
sources = [(GAUSS_OUTFILE, 1, 1.5), # from gauss-plots
# (MIX_OUTFILE, 0, 1, 0.005), # from sensor-plots (p3)
(MIX_OUTFILE, 0.7, 1, 0.1), # from sensor-plots (p1 and p2)
(MIX_OUTFILE, 0.7, 2, 0.05), # from sensor-mix-plots (p1 and p2)
(LOF_OUTFILE, 2)] # from lof-plots
if batch:
pdf = PdfPages(fname)
for out_template, *args in sources:
ofile = out_template.format(*args)
sensors3D(dfile,ofile)
if batch:
pyplot.savefig(pdf, pad_inches=1, format = 'pdf')
else:
pyplot.show()
peaked(PartitionedHistogram))
axis.set_xlabel(label)
di_x_start, dd_x_start = bars[di_first_hi].get_x(
), bars[dd_first_hi].get_x()
x_end = bars[-1].get_x() + BAR_WIDTH
y_di, y_dd = 1.1, 1.2
markers(axis, di_x_start, x_end, y_di, GREEN, True)
markers(axis, dd_x_start, x_end, y_dd, ORANGE, False)
legend = fig.legend(handles=[
Line2D((0, 50), (0, 0), dashes=DASHES[True], color=GREEN),
Line2D((0, 50), (0, 0), dashes=DASHES[False], color=ORANGE)
],
labels=[
"top bins ($D$-independent model)",
"top bins ($D$-dependent model)"
],
loc=8,
ncol=2,
bbox_to_anchor=(0.5, -0.15),
bbox_transform=fig.transFigure)
fig.tight_layout()
batch_mode, fname = filename("peakiness.pdf")
if batch_mode:
fig.savefig(fname, transparent=True)
else:
pyplot.show()
from numpy import linspace
matplotlib.rcParams['lines.linewidth'] = 2
def histogram_plt():
y = [0.1, 1, 5, 6, 0.2, 1, 10, 8, 0]
x = linspace(0, 8, num = 9)
T = 0.5
FILLS = (TANGO["red"][0], TANGO["green"][1])
STROKES = (TANGO["red"][1], TANGO["green"][2])
colors = [FILLS[yy > T] for yy in y]
edgecolors = [STROKES[yy > T] for yy in y]
pyplot.clf()
pyplot.bar(x, y, width=1)
ax = pyplot.axes(frameon = False)
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
ax.set_xlim(-1, 10)
pyplot.bar(x, y, width = 1, color = colors, edgecolor = edgecolors, linewidth = 2)
pyplot.tight_layout()
batch_mode, fname = filename("models-plots.pdf")
if batch_mode:
histogram_plt()
pyplot.savefig(fname, transparent = True)
#!/usr/bin/env python3
from utils import filename, save2pdf, setup, rcparams, to_inches
from utils.plots_helper import sensors
import matplotlib
from matplotlib import pyplot
from matplotlib.backends.backend_pdf import PdfPages
import itertools
matplotlib.rcParams['text.latex.preamble'] = [r"\usepackage{siunitx}"]
make,fname = filename("scalability.pdf")
INTEL_TOTAL = 2313153
# labels: vary train size + algo type
# x: vary test size
# y: runtime in s
trs = [1000,100000]#,2313153]
tes = [INTEL_TOTAL]
tes_h = 2000000
_trs = ["1K","100K"]#,2313153]
_tes = [5000000,1000000,15000000,20000000]
#_tes = [0.100,1.000,10.000,100.000,1000.000,2313.153]
#es = ["1_gaussian1.5","0.7_mixture1_0.075","0.7_mixture2_0.075"]
es = [
[1,"gaussian",1.5],
[0.7,"mixture1",0.1],
[0.7,"mixture2",0.05],
[0.7,"histogram"]
]
# build data
#!/usr/bin/env python3
from utils import filename, save2pdf, setup
from utils.plots_helper import sensors
import matplotlib
from matplotlib import pyplot
from matplotlib.backends.backend_pdf import PdfPages
make, fname = filename("sensor-gaus-plots.pdf")
dfile = "../datasets/real/intel/sensors-1000-dirty.txt"
# e, s, y, x
args = [(1, 1.5, 0, 1), (1, 1.5, 2, 3)]
pdf = PdfPages(fname)
for (e, s, y, x) in args:
title = "Outliers in Sensor Data\nGaussian, stddev=" + str(s)
ofile = "../results/sensors_dirty_stat" + str(e) + "_gaussian" + str(
s) + ".out"
setup()
sensors(title, x, y, dfile, ofile)
save2pdf(pdf)
pdf.close()
#! /usr/bin/env python3
from matplotlib import pyplot
from utils import TANGO, rcparams, to_inches, filename
import numpy as np
batch, fname = filename("csail-stats.pdf")
real_outliers = {} # from the annotated set
bad_outliers = {}
good_outliers = {}
dubious_outliers = {}
ghost_outliers = {}
total = {}
bad = 0
good = 1
dubious = 2
ghost = 3
dictionaries = [bad_outliers, good_outliers, dubious_outliers, ghost_outliers]
# Real annotations
with open('../datasets/real/csail-stats-annotated.txt', 'r') as f_true:
for line in f_true:
info = line.strip().split('\t')
linum = int(info[0])
cat = int(info[len(info) - 1])
real_outliers[linum] = cat
#!/usr/bin/env python3
from utils import filename, save2pdf, setup
from utils.plots_helper import sensors
import matplotlib
from matplotlib import pyplot
from matplotlib.backends.backend_pdf import PdfPages
make,fname = filename("sensor-gaus-plots.pdf")
dfile = "../datasets/real/intel/sensors-1000-dirty.txt"
# e, s, y, x
args = [
(1,1.5,0,1),
(1,1.5,2,3)]
pdf = PdfPages(fname)
for (e,s,y,x) in args:
title = "Outliers in Sensor Data\nGaussian, stddev=" + str(s)
ofile = "../results/sensors_dirty_stat" + str(e) + "_gaussian" + str(s) + ".out"
setup()
sensors(title,x,y,dfile,ofile)
save2pdf(pdf)
pdf.close()
#!/usr/bin/env python3
from utils import filename, save2pdf, setup
from utils.plots_helper import lof
from matplotlib.backends.backend_pdf import PdfPages
make,fname = filename("lof-plots.pdf")
dfile = "../datasets/real/intel/sensors-1000-dirty.txt"
pdf = PdfPages(fname)
for k in [2,10]:
title = "Local Outlier Factor\n$k=" + str(k) + "$"
ofile = "../results/k" + str(k) + "data01.out"
setup()
lof(title,dfile,ofile)
save2pdf(pdf)
pdf.close()
for side in ('top', 'left', 'right'):
ax.spines[side].set_visible(False)
ax.yaxis.set_visible(False)
pyplot.xticks([x + W for x in xs], labels, rotation=45, ha='right')
ax.xaxis.set_ticklabels([str(l) for l in labels])
ax.tick_params(top=False, bottom=False)
pyplot.bar(xs,
ys,
width=W,
color=colors,
edgecolor=edgecolors,
linewidth=0.5)
batch, fname = filename("table-histograms.pdf")
if batch:
pdf = PdfPages(fname)
YS = [[1, 3, 3], [2, 1.01, 2, 2], [1, 6], [1, 6]]
LABELS = [["1970", "2014", "2015"], ["Wed.", "Thu.", "Fri.", "Sun."],
["8", "12"], [r"nnn", r"nn"]]
for ys, labels in zip(YS, LABELS):
make_hist(ys, labels)
if batch:
pyplot.savefig(pdf, pad_inches=1, format='pdf')
else:
#!/usr/bin/env python3
from utils import filename, save2pdf, setup
from utils.plots_helper import sensors
from matplotlib.backends.backend_pdf import PdfPages
make, fname = filename("sensor-mix-plots.pdf")
dfile = "../datasets/real/intel/sensors-1000-dirty.txt"
# e, p, t, y, x
args = [
(0.7,2,0.05,0,1),
(0.7,2,0.05,0,3)]
pdf = PdfPages(fname)
for (e,p,t,y,x) in args:
title = "Outliers in Sensor Data\n"+str(p)+" Gaussians,$\\theta$=" + str(t)
ofile = "../results/sensors_dirty_stat" + str(e) + "_mixture" + str(p) + "_" + str(t) + ".out"
setup()
sensors(title,x,y,dfile,ofile)
save2pdf(pdf)
pdf.close()
#!/usr/bin/env python3
from utils import filename
from utils.plots_helper import sensors3D
from matplotlib.backends.backend_pdf import PdfPages
from matplotlib import pyplot
batch, fname = filename("3d-plots.pdf")
dfile = "../datasets/real/intel/sensors-1000-dirty.txt"
MIX_OUTFILE = "../results/sensors_dirty_stat{}_mixture{}_{}.out"
GAUSS_OUTFILE = "../results/sensors_dirty_stat{}_gaussian{}.out"
LOF_OUTFILE = "../results/sensors_dirty_lof{}.out"
sources = [
(GAUSS_OUTFILE, 1, 1.5), # from gauss-plots
# (MIX_OUTFILE, 0, 1, 0.005), # from sensor-plots (p3)
(MIX_OUTFILE, 0.7, 1, 0.1), # from sensor-plots (p1 and p2)
(MIX_OUTFILE, 0.7, 2, 0.05), # from sensor-mix-plots (p1 and p2)
(LOF_OUTFILE, 2)
] # from lof-plots
if batch:
pdf = PdfPages(fname)
for out_template, *args in sources:
ofile = out_template.format(*args)
sensors3D(dfile, ofile)
if batch:
pyplot.savefig(pdf, pad_inches=1, format='pdf')
else:
