def setUpClass(cls):
CUnitTest.setUpClass()
# We now prepare all the urls and path required to mock requests
# via gitlab API to https://gitlab.com/secml/secml-zoo repository
# Fake models definitions
cls.test_models_def = \
fm.join(fm.abspath(__file__), 'models_dict_test.json')
# Test model's definition
cls.test_model_id = '_test_model'
cls.test_model = \
fm.join(fm.abspath(__file__), '_test_model_clf.py')
cls.test_model_state = \
fm.join(fm.abspath(__file__), '_test_model-clf.gz')
# Url for mocking requests to the model zoo repository
repo = parse.quote('secml/secml-zoo', safe='')
file_model = parse.quote('models/_test/_test_model_clf.py', safe='')
file_state = parse.quote('models/_test/_test_model-clf.gz', safe='')
file_defs = parse.quote('models_dict.json', safe='')
vers = 'v' + re.search(r'^\d+.\d+', secml.__version__).group(0)
api_url = 'https://gitlab.com/api/v4/projects/' \
'{:}/repository/files/{:}/raw?ref={:}'
# One url for master branch, one for current library version
# One for model file, one for state file
cls.api_url_model_master = api_url.format(repo, file_model, 'master')
cls.api_url_model_vers = api_url.format(repo, file_model, vers)
cls.api_url_state_master = api_url.format(repo, file_state, 'master')
cls.api_url_state_vers = api_url.format(repo, file_state, vers)
cls.api_url_defs_master = api_url.format(repo, file_defs, 'master')
cls.api_url_defs_vers = api_url.format(repo, file_defs, vers)
cls.api_model_headers = {
'Content-Disposition': r'inline; filename="_test_model_clf.py"'
}
cls.api_state_headers = {
'Content-Disposition': r'inline; filename="_test_model-clf.gz"'
}
cls.api_defs_headers = {
'Content-Disposition': r'inline; filename="models_dict.json"'
}
# Set the debug level of models loader to debug
_logger.set_level('DEBUG')
def test_load_paths(self):
"""Testing img dataset path loading."""
dl = CDataLoaderImgClients()
self.logger.info("Testing loading paths of clients dataset...")
ds_path = fm.join(fm.abspath(__file__), "ds_clients")
ds = dl.load(ds_path=ds_path, img_format='jpeg', load_data=False)
self.logger.info(
"Loaded {:} images of {:} features, {:} classes".format(
ds.num_samples, ds.num_features, ds.num_classes))
# TODO: USE 'U' AFTER TRANSITION TO PYTHON 3
self.assertIn(ds.X.dtype.char, ('S', 'U'))
# Checking correct label-img association
self.assertEqual(ds.Y[0].item(),
fm.split(ds.X[0, :].item())[1].replace('.jpeg', ''))
self.assertEqual(ds.Y[1].item(),
fm.split(ds.X[1, :].item())[1].replace('.jpeg', ''))
# Checking behavior of `get_labels_ovr`
ovr = ds.get_labels_ovr(pos_label='tiger') # Y : ['coyote', 'tiger']
self.assert_array_equal(ovr, CArray([0, 1]))
def test_plot(self):
""" Compare the classifiers graphically"""
ds = CDLRandomBlobs(n_samples=100, centers=3, n_features=2,
random_state=1).load()
fig = self._test_plot(self.rnd_forest, ds, levels=[0.5])
fig.savefig(fm.join(fm.abspath(__file__), 'figs',
'test_c_classifier_random_forest.pdf'))
def test_plot(self):
""" Compare the classifiers graphically"""
ds = CDLRandomBlobs(n_samples=100, centers=3, n_features=2,
random_state=1).load()
fig = self._test_plot(self.nc, ds, [-10])
fig.savefig(fm.join(fm.abspath(__file__), 'figs',
'test_c_classifier_nearest_centroid.pdf'))
def test_plot(self):
""" Compare the classifiers graphically"""
ds = CDLRandom(n_features=2, n_redundant=0, n_informative=2,
n_clusters_per_class=1, random_state=0).load()
ds.X = CNormalizerMinMax().fit_transform(ds.X)
fig = self._test_plot(self.ridges[0], ds)
fig.savefig(fm.join(fm.abspath(__file__), 'figs',
'test_c_classifier_ridge.pdf'))
def test_plot(self):
ds = CDLRandomBlobs(n_samples=100,
centers=3,
n_features=2,
random_state=1).load()
fig = self._test_plot(self.knn, ds, levels=[0.5])
fig.savefig(
fm.join(fm.abspath(__file__), 'figs', 'test_c_classifier_knn.pdf'))
def _save_fig(self):
"""Visualizing the function being optimized with line search."""
x_range = CArray.arange(-5, 20, 0.5, )
score_range = x_range.T.apply_along_axis(self.fun.fun, axis=1)
ref_line = CArray.zeros(x_range.size)
fig = CFigure(height=6, width=12)
fig.sp.plot(x_range, score_range, color='b')
fig.sp.plot(x_range, ref_line, color='k')
filename = fm.join(fm.abspath(__file__), 'test_line_search_bisect.pdf')
fig.savefig(filename)
def test_save_and_load_svmlight_file(self):
"""Testing libsvm dataset loading and saving."""
self.logger.info("Testing libsvm dataset loading and saving...")
test_file = fm.join(fm.abspath(__file__), "myfile.libsvm")
# Cleaning test file
try:
fm.remove_file(test_file)
except (OSError, IOError) as e:
if e.errno != 2:
raise e
self.logger.info("Patterns saved:\n{:}".format(self.patterns))
self.logger.info("Labels saved:\n{:}".format(self.labels))
CDataLoaderSvmLight.dump(CDataset(self.patterns, self.labels),
test_file)
new_dataset = CDataLoaderSvmLight().load(test_file)
self.assertFalse((new_dataset.X != self.patterns).any())
self.assertFalse((new_dataset.Y != self.labels).any())
# load data but now remove all zero features (colums)
new_dataset = CDataLoaderSvmLight().load(test_file,
remove_all_zero=True)
self.logger.info("Patterns loaded:\n{:}".format(new_dataset.X))
self.logger.info("Labels loaded:\n{:}".format(new_dataset.Y))
self.logger.info("Mapping back:\n{:}".format(
new_dataset.header.idx_mapping))
self.assertTrue(new_dataset.X.issparse)
self.assertTrue(new_dataset.Y.isdense)
self.assertTrue(new_dataset.header.idx_mapping.isdense)
# non-zero elements should be unchanged
self.assertEqual(self.patterns.nnz, new_dataset.X.nnz)
new_nnz_data = new_dataset.X.nnz_data
self.assertFalse((self.patterns.nnz_data != new_nnz_data.sort()).any())
# With idx_mapping we should be able to reconstruct original data
original = CArray.zeros(self.patterns.shape, sparse=True)
original[:, new_dataset.header.idx_mapping] = new_dataset.X
self.assertFalse((self.patterns != original).any())
# Cleaning test file
try:
fm.remove_file(test_file)
except (OSError, IOError) as e:
if e.errno != 2:
raise e
def test_ps_kmedians(self):
rule = 'k-medians'
self.logger.info("Testing: " + rule + " selector.")
ps = CPrototypesSelector.create(rule)
ps.verbose = 2
ds_reduced = ps.select(self.dataset, n_prototypes=20, random_state=0)
if self.plots is True:
self.draw_selection(ds_reduced, rule)
# k_means in sklearn > 0.24 returns a different result
import sklearn
from pkg_resources import parse_version
if parse_version(sklearn.__version__) < parse_version("0.24"):
idx_path = fm.join(fm.abspath(__file__), "idx_{:}.gz".format(rule))
else:
idx_path = fm.join(fm.abspath(__file__),
"idx_{:}_sk0-24.gz".format(rule))
self.assert_array_equal(ps.sel_idx,
CArray.load(idx_path, dtype=int).ravel())
def _load_mnist():
"""Load MNIST 4971 dataset."""
digits = [4, 9, 7, 1]
digits_str = "".join(['{:}-'.format(i) for i in digits[:-1]])
digits_str += '{:}'.format(digits[-1])
# FIXME: REMOVE THIS AFTER CDATALOADERS AUTOMATICALLY STORE DS
tr_file = fm.join(fm.abspath(__file__),
'mnist_tr_{:}.gz'.format(digits_str))
if not fm.file_exist(tr_file):
loader = CDataLoaderMNIST()
tr = loader.load('training', digits=digits)
pickle_utils.save(tr_file, tr)
else:
tr = pickle_utils.load(tr_file, encoding='latin1')
ts_file = fm.join(fm.abspath(__file__),
'mnist_ts_{:}.gz'.format(digits_str))
if not fm.file_exist(ts_file):
loader = CDataLoaderMNIST()
ts = loader.load('testing', digits=digits)
pickle_utils.save(ts_file, ts)
else:
ts = pickle_utils.load(ts_file, encoding='latin1')
idx = CArray.arange(tr.num_samples)
val_dts_idx = CArray.randsample(idx, 200, random_state=0)
val_dts = tr[val_dts_idx, :]
tr_dts_idx = CArray.randsample(idx, 200, random_state=0)
tr = tr[tr_dts_idx, :]
idx = CArray.arange(0, ts.num_samples)
ts_dts_idx = CArray.randsample(idx, 200, random_state=0)
ts = ts[ts_dts_idx, :]
tr.X /= 255.0
ts.X /= 255.0
return tr, val_dts, ts, digits, tr.header.img_w, tr.header.img_h
def test_load_img(self):
"""Testing img dataset loading."""
dl = CDataLoaderImgFolders()
self.logger.info("Testing loading rgb dataset...")
ds_rgb_path = fm.join(fm.abspath(__file__), "ds_rgb")
ds = dl.load(ds_path=ds_rgb_path, img_format='jpeg')
self.logger.info(
"Loaded {:} images of {:} features, {:} classes".format(
ds.num_samples, ds.num_features, ds.num_classes))
self.assertEqual((2, 151875), ds.X.shape)
self.assertEqual(2, ds.num_classes)
self.assertTrue((ds.header.img_w == 225).all())
self.assertTrue((ds.header.img_h == 225).all())
self.assertTrue((ds.header.img_c == 3).all())
self.logger.info("Testing loading grayscale dataset...")
ds_gray_path = fm.join(fm.abspath(__file__), "ds_gray")
ds = dl.load(ds_path=ds_gray_path, img_format='jpeg')
self.logger.info(
"Loaded {:} images of {:} features, {:} classes".format(
ds.num_samples, ds.num_features, ds.num_classes))
self.assertEqual((2, 50625), ds.X.shape)
self.assertEqual(2, ds.num_classes)
self.assertTrue((ds.header.img_w == 225).all())
self.assertTrue((ds.header.img_h == 225).all())
self.assertTrue((ds.header.img_c == 1).all())
def _test_rule(self, rule, n_prototypes=20, random_state=None):
"""Generic test case for prototype selectors."""
self.logger.info("Testing: " + rule + " selector.")
ps = CPrototypesSelector.create(rule)
ps.verbose = 2
if random_state is None:
ds_reduced = ps.select(self.dataset, n_prototypes=n_prototypes)
else:
ds_reduced = ps.select(self.dataset,
n_prototypes=n_prototypes,
random_state=random_state)
idx_path = fm.join(fm.abspath(__file__), "idx_{:}.gz".format(rule))
self.assert_array_equal(ps.sel_idx,
CArray.load(idx_path, dtype=int).ravel())
if self.plots is True:
self.draw_selection(ds_reduced, rule)
def test_ps_kmedians(self):
rule = 'k-medians'
self.logger.info("Testing: " + rule + " selector.")
ps = CPrototypesSelector.create(rule)
ps.verbose = 2
ds_reduced = ps.select(self.dataset, n_prototypes=20, random_state=0)
# this test will fail with sklearn < 0.22, because of an issue in
# random_state setting inside the k-means algorithm
import sklearn
from pkg_resources import parse_version
if not parse_version(sklearn.__version__) < parse_version("0.22"):
idx_path = fm.join(fm.abspath(__file__), "idx_{:}.gz".format(rule))
self.assert_array_equal(ps.sel_idx,
CArray.load(idx_path, dtype=int).ravel())
if self.plots is True:
self.draw_selection(ds_reduced, rule)
def test_save_load(self):
"""Test save/load of sparse arrays"""
self.logger.info("UNITTEST - CSparse - save/load")
test_file = fm.join(fm.abspath(__file__), 'test.txt')
# Cleaning test file
try:
fm.remove_file(test_file)
except (OSError, IOError) as e:
if e.errno != 2:
raise e
self.logger.info(
"UNITTEST - CSparse - Testing save/load for sparse matrix")
self.sparse_matrix.save(test_file)
self.logger.info(
"Saving again with overwrite=False... IOError should be raised.")
with self.assertRaises(IOError) as e:
self.sparse_matrix.save(test_file)
self.logger.info(e.exception)
loaded_sparse_matrix = CSparse.load(test_file, dtype=int)
self.assertFalse((loaded_sparse_matrix != self.sparse_matrix).any(),
"Saved and loaded arrays (matrices) are not equal!")
self.logger.info(
"UNITTEST - CSparse - Testing save/load for sparse vector")
self.sparse_vector.save(test_file, overwrite=True)
loaded_sparse_vector = CSparse.load(test_file, dtype=int)
self.assertFalse((loaded_sparse_vector != self.sparse_vector).any(),
"Saved and loaded arrays (vectors) are not equal!")
# Cleaning test file
try:
fm.remove_file(test_file)
except (OSError, IOError) as e:
if e.errno != 2:
raise e
def test_load_paths(self):
"""Testing img dataset path loading."""
dl = CDataLoaderImgFolders()
self.logger.info("Testing loading paths of rgb dataset...")
ds_rgb_path = fm.join(fm.abspath(__file__), "ds_rgb")
ds = dl.load(ds_path=ds_rgb_path, img_format='jpeg', load_data=False)
self.logger.info(
"Loaded {:} images of {:} features, {:} classes".format(
ds.num_samples, ds.num_features, ds.num_classes))
# TODO: USE 'U' AFTER TRANSITION TO PYTHON 3
self.assertIn(ds.X.dtype.char, ('S', 'U'))
# Checking behavior of `get_labels_ovr`
ovr = ds.get_labels_ovr(pos_label='tiger') # Y : ['coyote', 'tiger']
self.assert_array_equal(ovr, CArray([0, 1]))
def test_margin(self):
self.logger.info("Testing margin separation of SGD...")
# we create 50 separable points
dataset = CDLRandomBlobs(n_samples=50,
centers=2,
random_state=0,
cluster_std=0.60).load()
# fit the model
clf = CClassifierSGD(loss=CLossHinge(),
regularizer=CRegularizerL2(),
alpha=0.01,
max_iter=200,
random_state=0)
clf.fit(dataset.X, dataset.Y)
# plot the line, the points, and the nearest vectors to the plane
xx = CArray.linspace(-1, 5, 10)
yy = CArray.linspace(-1, 5, 10)
X1, X2 = np.meshgrid(xx.tondarray(), yy.tondarray())
Z = CArray.empty(X1.shape)
for (i, j), val in np.ndenumerate(X1):
x1 = val
x2 = X2[i, j]
Z[i, j] = clf.decision_function(CArray([x1, x2]), y=1)
levels = [-1.0, 0.0, 1.0]
linestyles = ['dashed', 'solid', 'dashed']
colors = 'k'
fig = CFigure(linewidth=1)
fig.sp.contour(X1, X2, Z, levels, colors=colors, linestyles=linestyles)
fig.sp.scatter(dataset.X[:, 0].ravel(),
dataset.X[:, 1].ravel(),
c=dataset.Y,
s=40)
fig.savefig(
fm.join(fm.abspath(__file__), 'figs',
'test_c_classifier_sgd2.pdf'))
def test_margin(self):
self.logger.info("Testing margin separation of SVM...")
import numpy as np
# we create 40 separable points
rng = np.random.RandomState(0)
n_samples_1 = 1000
n_samples_2 = 100
X = np.r_[1.5 * rng.randn(n_samples_1, 2),
0.5 * rng.randn(n_samples_2, 2) + [2, 2]]
y = [0] * (n_samples_1) + [1] * (n_samples_2)
dataset = CDataset(X, y)
# fit the model
clf = CClassifierSVM()
clf.fit(dataset.X, dataset.Y)
w = clf.w
a = -w[0] / w[1]
xx = CArray.linspace(-5, 5)
yy = a * xx - clf.b / w[1]
wclf = CClassifierSVM(class_weight={0: 1, 1: 10})
wclf.fit(dataset.X, dataset.Y)
ww = wclf.w
wa = -ww[0] / ww[1]
wyy = wa * xx - wclf.b / ww[1]
fig = CFigure(linewidth=1)
fig.sp.plot(xx, yy.ravel(), 'k-', label='no weights')
fig.sp.plot(xx, wyy.ravel(), 'k--', label='with weights')
fig.sp.scatter(X[:, 0].ravel(), X[:, 1].ravel(), c=y)
fig.sp.legend()
fig.savefig(
fm.join(fm.abspath(__file__), 'figs', 'test_c_classifier_svm.pdf'))
def test_draw(self):
""" Compare the classifiers graphically"""
self.logger.info("Testing classifiers graphically")
# generate 2D synthetic data
dataset = CDLRandom(n_features=2,
n_redundant=1,
n_informative=1,
n_clusters_per_class=1).load()
dataset.X = CNormalizerMinMax().fit_transform(dataset.X)
self.sgds[0].fit(dataset.X, dataset.Y)
svm = CClassifierSVM()
svm.fit(dataset.X, dataset.Y)
fig = CFigure(width=10, markersize=8)
fig.subplot(2, 1, 1)
# Plot dataset points
fig.sp.plot_ds(dataset)
# Plot objective function
fig.sp.plot_fun(svm.decision_function,
grid_limits=dataset.get_bounds(),
y=1)
fig.sp.title('SVM')
fig.subplot(2, 1, 2)
# Plot dataset points
fig.sp.plot_ds(dataset)
# Plot objective function
fig.sp.plot_fun(self.sgds[0].decision_function,
grid_limits=dataset.get_bounds(),
y=1)
fig.sp.title('SGD Classifier')
fig.savefig(
fm.join(fm.abspath(__file__), 'figs',
'test_c_classifier_sgd1.pdf'))
def _test_2D(self,
fun,
grid_limits=None,
levels=None,
vmin=None,
vmax=None,
fun_args=()):
"""2D plot of the function.
Parameters
----------
fun : CFunction
grid_limits : list of tuple or None, optional
levels : list or None, optional
vmin, vmax : scalar or None, optional
fun_args : tuple
"""
fun_name = fun.__class__.__name__
self.logger.info("Plotting 2D of {:}".format(fun_name))
fig = CFigure(width=7)
fig.sp.plot_fun(func=fun.fun,
plot_levels=True,
grid_limits=grid_limits,
levels=levels,
n_grid_points=50,
n_colors=200,
vmin=vmin,
vmax=vmax,
func_args=fun_args)
fig.sp.title(fun_name)
fig.savefig(
fm.join(fm.abspath(__file__),
'test_function_{:}.pdf'.format(fun.class_type)))
def _test_plot(self, c, *points):
"""Visualize the constraint.
Parameters
----------
c : CConstraint
*points : CArray
A series of point to plot. Each point will be plotted before
and after cosntraint projection.
"""
self.logger.info("Plotting constrain {:}".format(c.class_type))
grid_limits = [(-1, 2.5), (-1, 2.5)]
fig = CFigure(height=6, width=6)
fig.sp.plot_fun(func=c.constraint,
grid_limits=grid_limits,
n_grid_points=40,
levels=[0],
levels_linewidth=1.5)
colors = ['g', 'r', 'c', 'm', 'y', 'k', 'w']
for p_i, p in enumerate(points):
self.logger.info("Plotting point (color {:}): {:}".format(
colors[p_i], p))
fig.sp.scatter(*p, c=colors[p_i], zorder=10)
p_proj = c.projection(p)
self.logger.info("Plotting point (color {:}): {:}".format(
colors[p_i], p_proj))
fig.sp.scatter(*p_proj, c=colors[p_i], zorder=10)
filename = "test_constraint_{:}.pdf".format(c.class_type)
fig.savefig(fm.join(fm.abspath(__file__), filename))
def save(file_path, obj):
"""Save object to file using cPickle.
This functions stores a generic python object into
a compressed gzip file (`*.gz`).
Saved objects can be loaded using `.load`.
Parameters
----------
file_path : str
Path to destination file.
obj : object
Any python object to save.
Returns
-------
obj_path : str
Full path to the stored object.
Notes
-----
Objects are stored using **protocol 4** data stream format.
For more information see
https://docs.python.org/3/library/pickle.html#data-stream-format
"""
# Adding extension to destination file if user forgot about it...
file_ext = fm.splitext(file_path)[1]
file_path = file_path + '.gz' if file_ext != '.gz' else file_path
# open the reference to target file
with gzip.open(file_path, 'wb') as f_ref:
pickle.dump(obj, f_ref, protocol=4)
return fm.join(fm.abspath(file_path), fm.split(file_path)[1])
def setUp(self):
super(TestCArraySaveLoad, self).setUp()
self.test_file = fm.join(fm.abspath(__file__), 'test.txt')
self.test_file_2 = fm.join(fm.abspath(__file__), 'test2.txt')
def test_save_load(self):
self.logger.info("UNITTEST - CDense - save/load matrix")
test_file = fm.join(fm.abspath(__file__), 'test.txt')
# Cleaning test file
try:
fm.remove_file(test_file)
except (OSError, IOError) as e:
if e.errno != 2:
raise e
a = CDense().zeros((1000, 1000))
with self.timer():
a.save(test_file)
with self.timer():
b = CDense().load(test_file,
startrow=100,
cols=CDense(np.arange(0, 100)))
self.assertFalse((a[100:, 0:100] != b).any())
self.logger.info("UNITTEST - CDense - save/load vector")
a = CDense().zeros(1000, dtype=int)
with self.timer():
a.save(test_file, overwrite=True)
with self.timer():
b = CDense().load(test_file,
cols=list(range(100, 1000)),
dtype=int).ravel()
self.assertFalse((a[0, 100] != b).any())
if np.__version__ < '1.18':
with self.assertRaises(IndexError) as e:
CDense().load(test_file, startrow=10)
self.logger.info("Expected error: {:}".format(e.exception))
else:
with self.logger.catch_warnings():
self.logger.filterwarnings(
"ignore", message="genfromtxt: Empty input file")
a = CDense().load(test_file, startrow=10)
self.assertEqual(a.size, 0)
self.logger.info("UNITTEST - CDense - save/load row vector")
a = CDense().zeros((1, 1000))
with self.timer():
a.save(test_file, overwrite=True)
with self.timer():
b = CDense().load(test_file, cols=CDense.arange(100, 1000))
self.assertFalse((a[:, 100:] != b).any())
# For some reasons np.genfromtxt does not close the file here
# Let's handle the resource warning about unclosed file
with self.logger.catch_warnings():
self.logger.filterwarnings("ignore", message="unclosed file")
if np.__version__ < '1.18':
with self.assertRaises(IndexError) as e:
CDense().load(test_file, startrow=10)
self.logger.info("Expected error: {:}".format(e.exception))
else:
self.logger.filterwarnings(
"ignore", message="genfromtxt: Empty input file")
a = CDense().load(test_file, startrow=10)
self.assertEqual(a.size, 0)
self.logger.info("UNITTEST - CDense - save/load negative vector")
a = -CDense().zeros(1000)
a.save(test_file, overwrite=True)
with open(test_file, mode='at+') as fhandle:
with self.timer():
a.save(fhandle, overwrite=True)
b = CDense().load(test_file)
# Simulating double save \w append
a = a.atleast_2d().append(a.atleast_2d(), axis=0)
self.assertFalse((a != b).any())
a = CDense(['a', 'b'])
with self.timer():
a.save(test_file, overwrite=True)
b = CDense().load(test_file, dtype=str).ravel()
self.assertFalse((a != b).any())
# Cleaning test file
try:
fm.remove_file(test_file)
except (OSError, IOError) as e:
if e.errno != 2:
raise e
def _plot_optimization(self,
solver,
x_0,
g_min,
grid_limits,
method=None,
vmin=None,
vmax=None,
label=None):
"""Plots the optimization problem.
Parameters
----------
solver : COptimizer
x_0 : CArray
Starting point.
g_min : CArray
Final point (after optimization).
grid_limits : list of tuple
vmin, vmax : int or None, optional
label : str or None, optional
"""
fig = CFigure(markersize=12)
# Plot objective function
fig.sp.plot_fun(func=CArray.apply_along_axis,
plot_background=True,
n_grid_points=30,
n_colors=25,
grid_limits=grid_limits,
levels=[0.5],
levels_color='gray',
levels_style='--',
colorbar=True,
func_args=(
solver.f.fun,
1,
),
vmin=vmin,
vmax=vmax)
if solver.bounds is not None: # Plot box constraint
fig.sp.plot_fun(func=lambda x: solver.bounds.constraint(x),
plot_background=False,
n_grid_points=20,
grid_limits=grid_limits,
levels=[0],
colorbar=False)
if solver.constr is not None: # Plot distance constraint
fig.sp.plot_fun(func=lambda x: solver.constr.constraint(x),
plot_background=False,
n_grid_points=20,
grid_limits=grid_limits,
levels=[0],
colorbar=False)
# Plot optimization trace
if solver.x_seq is not None:
fig.sp.plot_path(solver.x_seq)
else:
fig.sp.plot_path(x_0.append(g_min, axis=0))
fig.sp.title("{:}(fun={:}) - Glob Min @ {:}".format(
solver.class_type, solver.f.class_type,
solver.f.global_min_x().round(2).tolist()))
test_img_fold_name = 'test_images'
test_img_fold_path = fm.join(fm.abspath(__file__), test_img_fold_name)
if not fm.folder_exist(test_img_fold_path):
fm.make_folder(test_img_fold_path)
if method is None:
filename = fm.join(test_img_fold_path,
solver.class_type + '-' + solver.f.class_type)
else:
filename = fm.join(
test_img_fold_path,
solver.class_type + '-' + method + '-' + solver.f.class_type)
filename += '-' + label if label is not None else ''
test_img_fold_name = 'test_images'
if not fm.folder_exist(test_img_fold_name):
fm.make_folder(test_img_fold_name)
fig.savefig('{:}.pdf'.format(filename))
def test_plot(self):
""" Compare the classifiers graphically"""
fig = self._test_plot(self.log, self.dataset)
fig.savefig(fm.join(fm.abspath(__file__), 'figs',
'test_c_classifier_logistic.pdf'))
def load_model(model_id):
"""Load a pre-trained classifier.
Returns a pre-trained SecML classifier given the id of the model.
Check https://gitlab.com/secml/secml-zoo for the list of available models.
Parameters
----------
model_id : str
Identifier of the pre-trained model to load.
Returns
-------
CClassifier
Desired pre-trained model.
"""
model_info = _get_models_dict()[model_id]
model_path = fm.join(SECML_MODELS_DIR, model_info['model'] + '.py')
# Download (if needed) model's script, check md5 and extract it
if not fm.file_exist(model_path) or \
model_info['model_md5'] != md5(model_path):
model_url_parts = ('models', model_info['model'] + '.py')
model_url = '/'.join(s.strip('/') for s in model_url_parts)
out_dir = fm.abspath(model_path)
# Download requested model from current version's branch first,
# then from master branch
_dl_data_versioned(model_url, out_dir, model_info['model_md5'])
# Check if file has been correctly downloaded
if not fm.file_exist(model_path):
raise RuntimeError('Something wrong happened while '
'downloading the model. Please try again.')
state_path = fm.join(SECML_MODELS_DIR, model_info['state'] + '.gz')
# Download (if needed) state, check md5 and extract it
if not fm.file_exist(state_path) or \
model_info['state_md5'] != md5(state_path):
state_url_parts = ('models', model_info['state'] + '.gz')
state_url = '/'.join(s.strip('/') for s in state_url_parts)
out_dir = fm.abspath(state_path)
# Download requested model state from current version's branch first,
# then from master branch
_dl_data_versioned(state_url, out_dir, model_info['state_md5'])
# Check if file has been correctly downloaded
if not fm.file_exist(state_path):
raise RuntimeError('Something wrong happened while '
'downloading the model. Please try again.')
def import_module(full_name, path):
"""Import a python module from a path."""
from importlib import util
spec = util.spec_from_file_location(full_name, path)
mod = util.module_from_spec(spec)
spec.loader.exec_module(mod)
return mod
# Name of the function returning the model
model_name = model_info["model"].split('/')[-1]
# Import the python module containing the function returning the model
model_module = import_module(model_name, model_path)
# Run the function returning the model
model = getattr(model_module, model_name)()
# Restore the state of the model from file
model.load_state(state_path)
return model
from secml.testing import CUnitTest
import os
import numpy as np
from secml.array import CArray
from secml.data.loader import CDLRandomBlobs
from secml.optim.constraints import \
CConstraintBox, CConstraintL1, CConstraintL2
from secml.ml.features.normalization import CNormalizerMinMax
from secml.ml.classifiers import CClassifierSVM, CClassifierDecisionTree
from secml.core.type_utils import is_list, is_float
from secml.figure import CFigure
from secml.utils import fm
IMAGES_FOLDER = fm.join(fm.abspath(__file__), 'test_images')
if not fm.folder_exist(IMAGES_FOLDER):
fm.make_folder(IMAGES_FOLDER)
class CAttackEvasionTestCases(CUnitTest):
"""Unittests interface for CAttackEvasion."""
images_folder = IMAGES_FOLDER
make_figures = os.getenv('MAKE_FIGURES', False) # True to produce figures
def _load_blobs(self, n_feats, n_clusters, sparse=False, seed=None):
"""Load Random Blobs dataset.
- n_samples = 50
- center_box = (-0.5, 0.5)
- cluster_std = 0.5
