def setUpClass(cls): CUnitTest.setUpClass() # We now prepare all the urls and path required to mock requests # via gitlab API to https://gitlab.com/secml/secml-zoo repository # Fake models definitions cls.test_models_def = \ fm.join(fm.abspath(__file__), 'models_dict_test.json') # Test model's definition cls.test_model_id = '_test_model' cls.test_model = \ fm.join(fm.abspath(__file__), '_test_model_clf.py') cls.test_model_state = \ fm.join(fm.abspath(__file__), '_test_model-clf.gz') # Url for mocking requests to the model zoo repository repo = parse.quote('secml/secml-zoo', safe='') file_model = parse.quote('models/_test/_test_model_clf.py', safe='') file_state = parse.quote('models/_test/_test_model-clf.gz', safe='') file_defs = parse.quote('models_dict.json', safe='') vers = 'v' + re.search(r'^\d+.\d+', secml.__version__).group(0) api_url = 'https://gitlab.com/api/v4/projects/' \ '{:}/repository/files/{:}/raw?ref={:}' # One url for master branch, one for current library version # One for model file, one for state file cls.api_url_model_master = api_url.format(repo, file_model, 'master') cls.api_url_model_vers = api_url.format(repo, file_model, vers) cls.api_url_state_master = api_url.format(repo, file_state, 'master') cls.api_url_state_vers = api_url.format(repo, file_state, vers) cls.api_url_defs_master = api_url.format(repo, file_defs, 'master') cls.api_url_defs_vers = api_url.format(repo, file_defs, vers) cls.api_model_headers = { 'Content-Disposition': r'inline; filename="_test_model_clf.py"' } cls.api_state_headers = { 'Content-Disposition': r'inline; filename="_test_model-clf.gz"' } cls.api_defs_headers = { 'Content-Disposition': r'inline; filename="models_dict.json"' } # Set the debug level of models loader to debug _logger.set_level('DEBUG')
def test_load_paths(self): """Testing img dataset path loading.""" dl = CDataLoaderImgClients() self.logger.info("Testing loading paths of clients dataset...") ds_path = fm.join(fm.abspath(__file__), "ds_clients") ds = dl.load(ds_path=ds_path, img_format='jpeg', load_data=False) self.logger.info( "Loaded {:} images of {:} features, {:} classes".format( ds.num_samples, ds.num_features, ds.num_classes)) # TODO: USE 'U' AFTER TRANSITION TO PYTHON 3 self.assertIn(ds.X.dtype.char, ('S', 'U')) # Checking correct label-img association self.assertEqual(ds.Y[0].item(), fm.split(ds.X[0, :].item())[1].replace('.jpeg', '')) self.assertEqual(ds.Y[1].item(), fm.split(ds.X[1, :].item())[1].replace('.jpeg', '')) # Checking behavior of `get_labels_ovr` ovr = ds.get_labels_ovr(pos_label='tiger') # Y : ['coyote', 'tiger'] self.assert_array_equal(ovr, CArray([0, 1]))
def test_plot(self): """ Compare the classifiers graphically""" ds = CDLRandomBlobs(n_samples=100, centers=3, n_features=2, random_state=1).load() fig = self._test_plot(self.rnd_forest, ds, levels=[0.5]) fig.savefig(fm.join(fm.abspath(__file__), 'figs', 'test_c_classifier_random_forest.pdf'))
def test_plot(self): """ Compare the classifiers graphically""" ds = CDLRandomBlobs(n_samples=100, centers=3, n_features=2, random_state=1).load() fig = self._test_plot(self.nc, ds, [-10]) fig.savefig(fm.join(fm.abspath(__file__), 'figs', 'test_c_classifier_nearest_centroid.pdf'))
def test_plot(self): """ Compare the classifiers graphically""" ds = CDLRandom(n_features=2, n_redundant=0, n_informative=2, n_clusters_per_class=1, random_state=0).load() ds.X = CNormalizerMinMax().fit_transform(ds.X) fig = self._test_plot(self.ridges[0], ds) fig.savefig(fm.join(fm.abspath(__file__), 'figs', 'test_c_classifier_ridge.pdf'))
def test_plot(self): ds = CDLRandomBlobs(n_samples=100, centers=3, n_features=2, random_state=1).load() fig = self._test_plot(self.knn, ds, levels=[0.5]) fig.savefig( fm.join(fm.abspath(__file__), 'figs', 'test_c_classifier_knn.pdf'))
def _save_fig(self): """Visualizing the function being optimized with line search.""" x_range = CArray.arange(-5, 20, 0.5, ) score_range = x_range.T.apply_along_axis(self.fun.fun, axis=1) ref_line = CArray.zeros(x_range.size) fig = CFigure(height=6, width=12) fig.sp.plot(x_range, score_range, color='b') fig.sp.plot(x_range, ref_line, color='k') filename = fm.join(fm.abspath(__file__), 'test_line_search_bisect.pdf') fig.savefig(filename)
def test_save_and_load_svmlight_file(self): """Testing libsvm dataset loading and saving.""" self.logger.info("Testing libsvm dataset loading and saving...") test_file = fm.join(fm.abspath(__file__), "myfile.libsvm") # Cleaning test file try: fm.remove_file(test_file) except (OSError, IOError) as e: if e.errno != 2: raise e self.logger.info("Patterns saved:\n{:}".format(self.patterns)) self.logger.info("Labels saved:\n{:}".format(self.labels)) CDataLoaderSvmLight.dump(CDataset(self.patterns, self.labels), test_file) new_dataset = CDataLoaderSvmLight().load(test_file) self.assertFalse((new_dataset.X != self.patterns).any()) self.assertFalse((new_dataset.Y != self.labels).any()) # load data but now remove all zero features (colums) new_dataset = CDataLoaderSvmLight().load(test_file, remove_all_zero=True) self.logger.info("Patterns loaded:\n{:}".format(new_dataset.X)) self.logger.info("Labels loaded:\n{:}".format(new_dataset.Y)) self.logger.info("Mapping back:\n{:}".format( new_dataset.header.idx_mapping)) self.assertTrue(new_dataset.X.issparse) self.assertTrue(new_dataset.Y.isdense) self.assertTrue(new_dataset.header.idx_mapping.isdense) # non-zero elements should be unchanged self.assertEqual(self.patterns.nnz, new_dataset.X.nnz) new_nnz_data = new_dataset.X.nnz_data self.assertFalse((self.patterns.nnz_data != new_nnz_data.sort()).any()) # With idx_mapping we should be able to reconstruct original data original = CArray.zeros(self.patterns.shape, sparse=True) original[:, new_dataset.header.idx_mapping] = new_dataset.X self.assertFalse((self.patterns != original).any()) # Cleaning test file try: fm.remove_file(test_file) except (OSError, IOError) as e: if e.errno != 2: raise e
def test_ps_kmedians(self): rule = 'k-medians' self.logger.info("Testing: " + rule + " selector.") ps = CPrototypesSelector.create(rule) ps.verbose = 2 ds_reduced = ps.select(self.dataset, n_prototypes=20, random_state=0) if self.plots is True: self.draw_selection(ds_reduced, rule) # k_means in sklearn > 0.24 returns a different result import sklearn from pkg_resources import parse_version if parse_version(sklearn.__version__) < parse_version("0.24"): idx_path = fm.join(fm.abspath(__file__), "idx_{:}.gz".format(rule)) else: idx_path = fm.join(fm.abspath(__file__), "idx_{:}_sk0-24.gz".format(rule)) self.assert_array_equal(ps.sel_idx, CArray.load(idx_path, dtype=int).ravel())
def _load_mnist(): """Load MNIST 4971 dataset.""" digits = [4, 9, 7, 1] digits_str = "".join(['{:}-'.format(i) for i in digits[:-1]]) digits_str += '{:}'.format(digits[-1]) # FIXME: REMOVE THIS AFTER CDATALOADERS AUTOMATICALLY STORE DS tr_file = fm.join(fm.abspath(__file__), 'mnist_tr_{:}.gz'.format(digits_str)) if not fm.file_exist(tr_file): loader = CDataLoaderMNIST() tr = loader.load('training', digits=digits) pickle_utils.save(tr_file, tr) else: tr = pickle_utils.load(tr_file, encoding='latin1') ts_file = fm.join(fm.abspath(__file__), 'mnist_ts_{:}.gz'.format(digits_str)) if not fm.file_exist(ts_file): loader = CDataLoaderMNIST() ts = loader.load('testing', digits=digits) pickle_utils.save(ts_file, ts) else: ts = pickle_utils.load(ts_file, encoding='latin1') idx = CArray.arange(tr.num_samples) val_dts_idx = CArray.randsample(idx, 200, random_state=0) val_dts = tr[val_dts_idx, :] tr_dts_idx = CArray.randsample(idx, 200, random_state=0) tr = tr[tr_dts_idx, :] idx = CArray.arange(0, ts.num_samples) ts_dts_idx = CArray.randsample(idx, 200, random_state=0) ts = ts[ts_dts_idx, :] tr.X /= 255.0 ts.X /= 255.0 return tr, val_dts, ts, digits, tr.header.img_w, tr.header.img_h
def test_load_img(self): """Testing img dataset loading.""" dl = CDataLoaderImgFolders() self.logger.info("Testing loading rgb dataset...") ds_rgb_path = fm.join(fm.abspath(__file__), "ds_rgb") ds = dl.load(ds_path=ds_rgb_path, img_format='jpeg') self.logger.info( "Loaded {:} images of {:} features, {:} classes".format( ds.num_samples, ds.num_features, ds.num_classes)) self.assertEqual((2, 151875), ds.X.shape) self.assertEqual(2, ds.num_classes) self.assertTrue((ds.header.img_w == 225).all()) self.assertTrue((ds.header.img_h == 225).all()) self.assertTrue((ds.header.img_c == 3).all()) self.logger.info("Testing loading grayscale dataset...") ds_gray_path = fm.join(fm.abspath(__file__), "ds_gray") ds = dl.load(ds_path=ds_gray_path, img_format='jpeg') self.logger.info( "Loaded {:} images of {:} features, {:} classes".format( ds.num_samples, ds.num_features, ds.num_classes)) self.assertEqual((2, 50625), ds.X.shape) self.assertEqual(2, ds.num_classes) self.assertTrue((ds.header.img_w == 225).all()) self.assertTrue((ds.header.img_h == 225).all()) self.assertTrue((ds.header.img_c == 1).all())
def _test_rule(self, rule, n_prototypes=20, random_state=None): """Generic test case for prototype selectors.""" self.logger.info("Testing: " + rule + " selector.") ps = CPrototypesSelector.create(rule) ps.verbose = 2 if random_state is None: ds_reduced = ps.select(self.dataset, n_prototypes=n_prototypes) else: ds_reduced = ps.select(self.dataset, n_prototypes=n_prototypes, random_state=random_state) idx_path = fm.join(fm.abspath(__file__), "idx_{:}.gz".format(rule)) self.assert_array_equal(ps.sel_idx, CArray.load(idx_path, dtype=int).ravel()) if self.plots is True: self.draw_selection(ds_reduced, rule)
def test_ps_kmedians(self): rule = 'k-medians' self.logger.info("Testing: " + rule + " selector.") ps = CPrototypesSelector.create(rule) ps.verbose = 2 ds_reduced = ps.select(self.dataset, n_prototypes=20, random_state=0) # this test will fail with sklearn < 0.22, because of an issue in # random_state setting inside the k-means algorithm import sklearn from pkg_resources import parse_version if not parse_version(sklearn.__version__) < parse_version("0.22"): idx_path = fm.join(fm.abspath(__file__), "idx_{:}.gz".format(rule)) self.assert_array_equal(ps.sel_idx, CArray.load(idx_path, dtype=int).ravel()) if self.plots is True: self.draw_selection(ds_reduced, rule)
def test_save_load(self): """Test save/load of sparse arrays""" self.logger.info("UNITTEST - CSparse - save/load") test_file = fm.join(fm.abspath(__file__), 'test.txt') # Cleaning test file try: fm.remove_file(test_file) except (OSError, IOError) as e: if e.errno != 2: raise e self.logger.info( "UNITTEST - CSparse - Testing save/load for sparse matrix") self.sparse_matrix.save(test_file) self.logger.info( "Saving again with overwrite=False... IOError should be raised.") with self.assertRaises(IOError) as e: self.sparse_matrix.save(test_file) self.logger.info(e.exception) loaded_sparse_matrix = CSparse.load(test_file, dtype=int) self.assertFalse((loaded_sparse_matrix != self.sparse_matrix).any(), "Saved and loaded arrays (matrices) are not equal!") self.logger.info( "UNITTEST - CSparse - Testing save/load for sparse vector") self.sparse_vector.save(test_file, overwrite=True) loaded_sparse_vector = CSparse.load(test_file, dtype=int) self.assertFalse((loaded_sparse_vector != self.sparse_vector).any(), "Saved and loaded arrays (vectors) are not equal!") # Cleaning test file try: fm.remove_file(test_file) except (OSError, IOError) as e: if e.errno != 2: raise e
def test_load_paths(self): """Testing img dataset path loading.""" dl = CDataLoaderImgFolders() self.logger.info("Testing loading paths of rgb dataset...") ds_rgb_path = fm.join(fm.abspath(__file__), "ds_rgb") ds = dl.load(ds_path=ds_rgb_path, img_format='jpeg', load_data=False) self.logger.info( "Loaded {:} images of {:} features, {:} classes".format( ds.num_samples, ds.num_features, ds.num_classes)) # TODO: USE 'U' AFTER TRANSITION TO PYTHON 3 self.assertIn(ds.X.dtype.char, ('S', 'U')) # Checking behavior of `get_labels_ovr` ovr = ds.get_labels_ovr(pos_label='tiger') # Y : ['coyote', 'tiger'] self.assert_array_equal(ovr, CArray([0, 1]))
def test_margin(self): self.logger.info("Testing margin separation of SGD...") # we create 50 separable points dataset = CDLRandomBlobs(n_samples=50, centers=2, random_state=0, cluster_std=0.60).load() # fit the model clf = CClassifierSGD(loss=CLossHinge(), regularizer=CRegularizerL2(), alpha=0.01, max_iter=200, random_state=0) clf.fit(dataset.X, dataset.Y) # plot the line, the points, and the nearest vectors to the plane xx = CArray.linspace(-1, 5, 10) yy = CArray.linspace(-1, 5, 10) X1, X2 = np.meshgrid(xx.tondarray(), yy.tondarray()) Z = CArray.empty(X1.shape) for (i, j), val in np.ndenumerate(X1): x1 = val x2 = X2[i, j] Z[i, j] = clf.decision_function(CArray([x1, x2]), y=1) levels = [-1.0, 0.0, 1.0] linestyles = ['dashed', 'solid', 'dashed'] colors = 'k' fig = CFigure(linewidth=1) fig.sp.contour(X1, X2, Z, levels, colors=colors, linestyles=linestyles) fig.sp.scatter(dataset.X[:, 0].ravel(), dataset.X[:, 1].ravel(), c=dataset.Y, s=40) fig.savefig( fm.join(fm.abspath(__file__), 'figs', 'test_c_classifier_sgd2.pdf'))
def test_margin(self): self.logger.info("Testing margin separation of SVM...") import numpy as np # we create 40 separable points rng = np.random.RandomState(0) n_samples_1 = 1000 n_samples_2 = 100 X = np.r_[1.5 * rng.randn(n_samples_1, 2), 0.5 * rng.randn(n_samples_2, 2) + [2, 2]] y = [0] * (n_samples_1) + [1] * (n_samples_2) dataset = CDataset(X, y) # fit the model clf = CClassifierSVM() clf.fit(dataset.X, dataset.Y) w = clf.w a = -w[0] / w[1] xx = CArray.linspace(-5, 5) yy = a * xx - clf.b / w[1] wclf = CClassifierSVM(class_weight={0: 1, 1: 10}) wclf.fit(dataset.X, dataset.Y) ww = wclf.w wa = -ww[0] / ww[1] wyy = wa * xx - wclf.b / ww[1] fig = CFigure(linewidth=1) fig.sp.plot(xx, yy.ravel(), 'k-', label='no weights') fig.sp.plot(xx, wyy.ravel(), 'k--', label='with weights') fig.sp.scatter(X[:, 0].ravel(), X[:, 1].ravel(), c=y) fig.sp.legend() fig.savefig( fm.join(fm.abspath(__file__), 'figs', 'test_c_classifier_svm.pdf'))
def test_draw(self): """ Compare the classifiers graphically""" self.logger.info("Testing classifiers graphically") # generate 2D synthetic data dataset = CDLRandom(n_features=2, n_redundant=1, n_informative=1, n_clusters_per_class=1).load() dataset.X = CNormalizerMinMax().fit_transform(dataset.X) self.sgds[0].fit(dataset.X, dataset.Y) svm = CClassifierSVM() svm.fit(dataset.X, dataset.Y) fig = CFigure(width=10, markersize=8) fig.subplot(2, 1, 1) # Plot dataset points fig.sp.plot_ds(dataset) # Plot objective function fig.sp.plot_fun(svm.decision_function, grid_limits=dataset.get_bounds(), y=1) fig.sp.title('SVM') fig.subplot(2, 1, 2) # Plot dataset points fig.sp.plot_ds(dataset) # Plot objective function fig.sp.plot_fun(self.sgds[0].decision_function, grid_limits=dataset.get_bounds(), y=1) fig.sp.title('SGD Classifier') fig.savefig( fm.join(fm.abspath(__file__), 'figs', 'test_c_classifier_sgd1.pdf'))
def _test_2D(self, fun, grid_limits=None, levels=None, vmin=None, vmax=None, fun_args=()): """2D plot of the function. Parameters ---------- fun : CFunction grid_limits : list of tuple or None, optional levels : list or None, optional vmin, vmax : scalar or None, optional fun_args : tuple """ fun_name = fun.__class__.__name__ self.logger.info("Plotting 2D of {:}".format(fun_name)) fig = CFigure(width=7) fig.sp.plot_fun(func=fun.fun, plot_levels=True, grid_limits=grid_limits, levels=levels, n_grid_points=50, n_colors=200, vmin=vmin, vmax=vmax, func_args=fun_args) fig.sp.title(fun_name) fig.savefig( fm.join(fm.abspath(__file__), 'test_function_{:}.pdf'.format(fun.class_type)))
def _test_plot(self, c, *points): """Visualize the constraint. Parameters ---------- c : CConstraint *points : CArray A series of point to plot. Each point will be plotted before and after cosntraint projection. """ self.logger.info("Plotting constrain {:}".format(c.class_type)) grid_limits = [(-1, 2.5), (-1, 2.5)] fig = CFigure(height=6, width=6) fig.sp.plot_fun(func=c.constraint, grid_limits=grid_limits, n_grid_points=40, levels=[0], levels_linewidth=1.5) colors = ['g', 'r', 'c', 'm', 'y', 'k', 'w'] for p_i, p in enumerate(points): self.logger.info("Plotting point (color {:}): {:}".format( colors[p_i], p)) fig.sp.scatter(*p, c=colors[p_i], zorder=10) p_proj = c.projection(p) self.logger.info("Plotting point (color {:}): {:}".format( colors[p_i], p_proj)) fig.sp.scatter(*p_proj, c=colors[p_i], zorder=10) filename = "test_constraint_{:}.pdf".format(c.class_type) fig.savefig(fm.join(fm.abspath(__file__), filename))
def save(file_path, obj): """Save object to file using cPickle. This functions stores a generic python object into a compressed gzip file (`*.gz`). Saved objects can be loaded using `.load`. Parameters ---------- file_path : str Path to destination file. obj : object Any python object to save. Returns ------- obj_path : str Full path to the stored object. Notes ----- Objects are stored using **protocol 4** data stream format. For more information see https://docs.python.org/3/library/pickle.html#data-stream-format """ # Adding extension to destination file if user forgot about it... file_ext = fm.splitext(file_path)[1] file_path = file_path + '.gz' if file_ext != '.gz' else file_path # open the reference to target file with gzip.open(file_path, 'wb') as f_ref: pickle.dump(obj, f_ref, protocol=4) return fm.join(fm.abspath(file_path), fm.split(file_path)[1])
def setUp(self): super(TestCArraySaveLoad, self).setUp() self.test_file = fm.join(fm.abspath(__file__), 'test.txt') self.test_file_2 = fm.join(fm.abspath(__file__), 'test2.txt')
def test_save_load(self): self.logger.info("UNITTEST - CDense - save/load matrix") test_file = fm.join(fm.abspath(__file__), 'test.txt') # Cleaning test file try: fm.remove_file(test_file) except (OSError, IOError) as e: if e.errno != 2: raise e a = CDense().zeros((1000, 1000)) with self.timer(): a.save(test_file) with self.timer(): b = CDense().load(test_file, startrow=100, cols=CDense(np.arange(0, 100))) self.assertFalse((a[100:, 0:100] != b).any()) self.logger.info("UNITTEST - CDense - save/load vector") a = CDense().zeros(1000, dtype=int) with self.timer(): a.save(test_file, overwrite=True) with self.timer(): b = CDense().load(test_file, cols=list(range(100, 1000)), dtype=int).ravel() self.assertFalse((a[0, 100] != b).any()) if np.__version__ < '1.18': with self.assertRaises(IndexError) as e: CDense().load(test_file, startrow=10) self.logger.info("Expected error: {:}".format(e.exception)) else: with self.logger.catch_warnings(): self.logger.filterwarnings( "ignore", message="genfromtxt: Empty input file") a = CDense().load(test_file, startrow=10) self.assertEqual(a.size, 0) self.logger.info("UNITTEST - CDense - save/load row vector") a = CDense().zeros((1, 1000)) with self.timer(): a.save(test_file, overwrite=True) with self.timer(): b = CDense().load(test_file, cols=CDense.arange(100, 1000)) self.assertFalse((a[:, 100:] != b).any()) # For some reasons np.genfromtxt does not close the file here # Let's handle the resource warning about unclosed file with self.logger.catch_warnings(): self.logger.filterwarnings("ignore", message="unclosed file") if np.__version__ < '1.18': with self.assertRaises(IndexError) as e: CDense().load(test_file, startrow=10) self.logger.info("Expected error: {:}".format(e.exception)) else: self.logger.filterwarnings( "ignore", message="genfromtxt: Empty input file") a = CDense().load(test_file, startrow=10) self.assertEqual(a.size, 0) self.logger.info("UNITTEST - CDense - save/load negative vector") a = -CDense().zeros(1000) a.save(test_file, overwrite=True) with open(test_file, mode='at+') as fhandle: with self.timer(): a.save(fhandle, overwrite=True) b = CDense().load(test_file) # Simulating double save \w append a = a.atleast_2d().append(a.atleast_2d(), axis=0) self.assertFalse((a != b).any()) a = CDense(['a', 'b']) with self.timer(): a.save(test_file, overwrite=True) b = CDense().load(test_file, dtype=str).ravel() self.assertFalse((a != b).any()) # Cleaning test file try: fm.remove_file(test_file) except (OSError, IOError) as e: if e.errno != 2: raise e
def _plot_optimization(self, solver, x_0, g_min, grid_limits, method=None, vmin=None, vmax=None, label=None): """Plots the optimization problem. Parameters ---------- solver : COptimizer x_0 : CArray Starting point. g_min : CArray Final point (after optimization). grid_limits : list of tuple vmin, vmax : int or None, optional label : str or None, optional """ fig = CFigure(markersize=12) # Plot objective function fig.sp.plot_fun(func=CArray.apply_along_axis, plot_background=True, n_grid_points=30, n_colors=25, grid_limits=grid_limits, levels=[0.5], levels_color='gray', levels_style='--', colorbar=True, func_args=( solver.f.fun, 1, ), vmin=vmin, vmax=vmax) if solver.bounds is not None: # Plot box constraint fig.sp.plot_fun(func=lambda x: solver.bounds.constraint(x), plot_background=False, n_grid_points=20, grid_limits=grid_limits, levels=[0], colorbar=False) if solver.constr is not None: # Plot distance constraint fig.sp.plot_fun(func=lambda x: solver.constr.constraint(x), plot_background=False, n_grid_points=20, grid_limits=grid_limits, levels=[0], colorbar=False) # Plot optimization trace if solver.x_seq is not None: fig.sp.plot_path(solver.x_seq) else: fig.sp.plot_path(x_0.append(g_min, axis=0)) fig.sp.title("{:}(fun={:}) - Glob Min @ {:}".format( solver.class_type, solver.f.class_type, solver.f.global_min_x().round(2).tolist())) test_img_fold_name = 'test_images' test_img_fold_path = fm.join(fm.abspath(__file__), test_img_fold_name) if not fm.folder_exist(test_img_fold_path): fm.make_folder(test_img_fold_path) if method is None: filename = fm.join(test_img_fold_path, solver.class_type + '-' + solver.f.class_type) else: filename = fm.join( test_img_fold_path, solver.class_type + '-' + method + '-' + solver.f.class_type) filename += '-' + label if label is not None else '' test_img_fold_name = 'test_images' if not fm.folder_exist(test_img_fold_name): fm.make_folder(test_img_fold_name) fig.savefig('{:}.pdf'.format(filename))
def test_plot(self): """ Compare the classifiers graphically""" fig = self._test_plot(self.log, self.dataset) fig.savefig(fm.join(fm.abspath(__file__), 'figs', 'test_c_classifier_logistic.pdf'))
def load_model(model_id): """Load a pre-trained classifier. Returns a pre-trained SecML classifier given the id of the model. Check https://gitlab.com/secml/secml-zoo for the list of available models. Parameters ---------- model_id : str Identifier of the pre-trained model to load. Returns ------- CClassifier Desired pre-trained model. """ model_info = _get_models_dict()[model_id] model_path = fm.join(SECML_MODELS_DIR, model_info['model'] + '.py') # Download (if needed) model's script, check md5 and extract it if not fm.file_exist(model_path) or \ model_info['model_md5'] != md5(model_path): model_url_parts = ('models', model_info['model'] + '.py') model_url = '/'.join(s.strip('/') for s in model_url_parts) out_dir = fm.abspath(model_path) # Download requested model from current version's branch first, # then from master branch _dl_data_versioned(model_url, out_dir, model_info['model_md5']) # Check if file has been correctly downloaded if not fm.file_exist(model_path): raise RuntimeError('Something wrong happened while ' 'downloading the model. Please try again.') state_path = fm.join(SECML_MODELS_DIR, model_info['state'] + '.gz') # Download (if needed) state, check md5 and extract it if not fm.file_exist(state_path) or \ model_info['state_md5'] != md5(state_path): state_url_parts = ('models', model_info['state'] + '.gz') state_url = '/'.join(s.strip('/') for s in state_url_parts) out_dir = fm.abspath(state_path) # Download requested model state from current version's branch first, # then from master branch _dl_data_versioned(state_url, out_dir, model_info['state_md5']) # Check if file has been correctly downloaded if not fm.file_exist(state_path): raise RuntimeError('Something wrong happened while ' 'downloading the model. Please try again.') def import_module(full_name, path): """Import a python module from a path.""" from importlib import util spec = util.spec_from_file_location(full_name, path) mod = util.module_from_spec(spec) spec.loader.exec_module(mod) return mod # Name of the function returning the model model_name = model_info["model"].split('/')[-1] # Import the python module containing the function returning the model model_module = import_module(model_name, model_path) # Run the function returning the model model = getattr(model_module, model_name)() # Restore the state of the model from file model.load_state(state_path) return model
from secml.testing import CUnitTest import os import numpy as np from secml.array import CArray from secml.data.loader import CDLRandomBlobs from secml.optim.constraints import \ CConstraintBox, CConstraintL1, CConstraintL2 from secml.ml.features.normalization import CNormalizerMinMax from secml.ml.classifiers import CClassifierSVM, CClassifierDecisionTree from secml.core.type_utils import is_list, is_float from secml.figure import CFigure from secml.utils import fm IMAGES_FOLDER = fm.join(fm.abspath(__file__), 'test_images') if not fm.folder_exist(IMAGES_FOLDER): fm.make_folder(IMAGES_FOLDER) class CAttackEvasionTestCases(CUnitTest): """Unittests interface for CAttackEvasion.""" images_folder = IMAGES_FOLDER make_figures = os.getenv('MAKE_FIGURES', False) # True to produce figures def _load_blobs(self, n_feats, n_clusters, sparse=False, seed=None): """Load Random Blobs dataset. - n_samples = 50 - center_box = (-0.5, 0.5) - cluster_std = 0.5