def main(dataset_names=None): if dataset_names is None: dataset_names = [ 'autos', 'car', 'cleveland', 'dermatology', 'ecoli', 'flare', 'glass', 'led7digit', 'lymphography', 'nursery', 'page-blocks', 'pendigits', 'satimage', 'segment', #'shuttle', 'vehicle', 'vowel', 'yeast', 'zoo', 'auslan' ] seed_num = 42 mc_iterations = 5 n_folds = 2 estimator_type = "svm" # Diary to save the partial and final results diary = Diary(name='results_Krawczyk2015', path='results', overwrite=False, fig_format='svg') # Hyperparameters for this experiment (folds, iterations, seed) diary.add_notebook('parameters', verbose=True) # Summary for each dataset diary.add_notebook('datasets', verbose=False) # Partial results for validation diary.add_notebook('validation', verbose=True) # Final results diary.add_notebook('summary', verbose=True) columns = ['dataset', 'method', 'mc', 'test_fold', 'acc'] df = MyDataFrame(columns=columns) diary.add_entry('parameters', [ 'seed', seed_num, 'mc_it', mc_iterations, 'n_folds', n_folds, 'estimator_type', estimator_type ]) data = Data(dataset_names=dataset_names) for i, (name, dataset) in enumerate(data.datasets.iteritems()): np.random.seed(seed_num) dataset.print_summary() diary.add_entry('datasets', [dataset.__str__()]) accuracies = np.zeros(mc_iterations * n_folds) for mc in np.arange(mc_iterations): skf = StratifiedKFold(dataset.target, n_folds=n_folds, shuffle=True) test_folds = skf.test_folds for test_fold in np.arange(n_folds): x_train, y_train, x_test, y_test = separate_sets( dataset.data, dataset.target, test_fold, test_folds) if estimator_type == "svm": est = OneClassSVM(nu=0.5, gamma=0.5) elif estimator_type == "gmm": est = GMM(n_components=3) bc = BackgroundCheck(estimator=est) oc = OcDecomposition(base_estimator=bc) oc.fit(x_train, y_train) accuracy = oc.accuracy(x_test, y_test) accuracies[mc * n_folds + test_fold] = accuracy diary.add_entry('validation', [ 'dataset', name, 'method', 'our', 'mc', mc, 'test_fold', test_fold, 'acc', accuracy ]) df = df.append_rows([[name, 'our', mc, test_fold, accuracy]]) df = df.convert_objects(convert_numeric=True) table = df.pivot_table(values=['acc'], index=['dataset'], columns=['method'], aggfunc=[np.mean, np.std]) diary.add_entry('summary', [table])
# Classifier of training data model_clas = train_classifier_model(x_train, y_train) # TRAINING SCORES c1_rs = model_rej.predict_proba(r_train) c1_ts = model_rej.predict_proba(x_train) c2_ts = model_clas.predict_proba(x_train) ace1, ace2, cace = compute_cace(c1_rs, c1_ts, c2_ts, y_train) print('TRAIN RESULTS') print('Step1 Average Cross-entropy = {}'.format(ace1)) print('Step2 Average Cross-entropy = {}'.format(ace2)) print('Composite Average Cross-entropy = {}'.format(cace)) diary.add_entry('training', [ 'Example', example, 'Iteration', iteration, 'Step1 ACE', ace1 ]) diary.add_entry('training', [ 'Example', example, 'Iteration', iteration, 'Step2 ACE', ace2 ]) diary.add_entry('training', [ 'Example', example, 'Iteration', iteration, 'Composite ACE', cace ]) step1_reject_scores = c1_rs[:, 1] step1_training_scores = c1_ts[:, 1] step2_training_scores = c2_ts[:, 1] training_labels = y_train print("Step1 Accuracy = {} (prior = {})".format(
def main(dataset_names=None, estimator_type="gmm", mc_iterations=20, n_folds=5, n_ensemble=100, seed_num=42): if dataset_names is None: # All the datasets used in Li2014 datasets_li2014 = [ 'abalone', 'balance-scale', 'credit-approval', 'dermatology', 'ecoli', 'german', 'heart-statlog', 'hepatitis', 'horse', 'ionosphere', 'lung-cancer', 'libras-movement', 'mushroom', 'diabetes', 'landsat-satellite', 'segment', 'spambase', 'wdbc', 'wpbc', 'yeast' ] datasets_hempstalk2008 = [ 'diabetes', 'ecoli', 'glass', 'heart-statlog', 'ionosphere', 'iris', 'letter', 'mfeat-karhunen', 'mfeat-morphological', 'mfeat-zernike', 'optdigits', 'pendigits', 'sonar', 'vehicle', 'waveform-5000' ] datasets_others = [ 'diabetes', 'ecoli', 'glass', 'heart-statlog', 'ionosphere', 'iris', 'letter', 'mfeat-karhunen', 'mfeat-morphological', 'mfeat-zernike', 'optdigits', 'pendigits', 'sonar', 'vehicle', 'waveform-5000', 'scene-classification', 'tic-tac', 'autos', 'car', 'cleveland', 'dermatology', 'flare', 'page-blocks', 'segment', 'shuttle', 'vowel', 'zoo', 'abalone', 'balance-scale', 'credit-approval', 'german', 'hepatitis', 'lung-cancer' ] # Datasets that we can add but need to be reduced datasets_to_add = ['MNIST'] dataset_names = list( set(datasets_li2014 + datasets_hempstalk2008 + datasets_others)) # Diary to save the partial and final results diary = Diary(name='results_Li2014', path='results', overwrite=False, fig_format='svg') # Hyperparameters for this experiment (folds, iterations, seed) diary.add_notebook('parameters', verbose=True) # Summary for each dataset diary.add_notebook('datasets', verbose=False) # Partial results for validation diary.add_notebook('validation', verbose=True) # Final results diary.add_notebook('summary', verbose=True) columns = ['dataset', 'method', 'mc', 'test_fold', 'acc', 'logloss'] df = MyDataFrame(columns=columns) diary.add_entry('parameters', [ 'seed', seed_num, 'mc_it', mc_iterations, 'n_folds', n_folds, 'n_ensemble', n_ensemble, 'estimator_type', estimator_type ]) data = Data(dataset_names=dataset_names) for name, dataset in data.datasets.iteritems(): if name in ['letter', 'shuttle']: dataset.reduce_number_instances(0.1) export_datasets_description_to_latex(data, path=diary.path) for i, (name, dataset) in enumerate(data.datasets.iteritems()): np.random.seed(seed_num) dataset.print_summary() diary.add_entry('datasets', [dataset.__str__()]) for mc in np.arange(mc_iterations): skf = StratifiedKFold(dataset.target, n_folds=n_folds, shuffle=True) test_folds = skf.test_folds for test_fold in np.arange(n_folds): x_train, y_train, x_test, y_test = separate_sets( dataset.data, dataset.target, test_fold, test_folds) # Binary discriminative classifier sv = SVC(kernel='linear', probability=True) # Density estimator for the background check if estimator_type == "svm": gamma = 1.0 / x_train.shape[1] est = OneClassSVM(nu=0.1, gamma=gamma) elif estimator_type == "gmm": est = GMM(n_components=1) elif estimator_type == "gmm3": est = GMM(n_components=3) elif estimator_type == "mymvn": est = MyMultivariateNormal() # Multiclass discriminative model with one-vs-one binary class. ovo = OvoClassifier(base_classifier=sv) classifier = ConfidentClassifier(classifier=ovo, estimator=est, mu=0.5, m=0.5) ensemble = Ensemble(base_classifier=classifier, n_ensemble=n_ensemble) # classifier = ConfidentClassifier(classifier=sv, # estimator=est, mu=0.5, # m=0.5) # ovo = OvoClassifier(base_classifier=classifier) # ensemble = Ensemble(base_classifier=ovo, # n_ensemble=n_ensemble) xs_bootstrap, ys_bootstrap = ensemble.fit(x_train, y_train) accuracy = ensemble.accuracy(x_test, y_test) log_loss = ensemble.log_loss(x_test, y_test) diary.add_entry('validation', [ 'dataset', name, 'method', 'our', 'mc', mc, 'test_fold', test_fold, 'acc', accuracy, 'logloss', log_loss ]) df = df.append_rows( [[name, 'our', mc, test_fold, accuracy, log_loss]]) # Li2014: EP-CC model # The classification confidence is used in learning the weights # of the base classifier as well as in weighted voting. ensemble_li = Ensemble(n_ensemble=n_ensemble, lambd=1e-8) ensemble_li.fit(x_train, y_train, xs=xs_bootstrap, ys=ys_bootstrap) accuracy_li = ensemble_li.accuracy(x_test, y_test) log_loss_li = ensemble_li.log_loss(x_test, y_test) diary.add_entry('validation', [ 'dataset', name, 'method', 'Li2014', 'mc', mc, 'test_fold', test_fold, 'acc', accuracy_li, 'logloss', log_loss_li ]) df = df.append_rows( [[name, 'Li2014', mc, test_fold, accuracy_li, log_loss_li]]) export_summary(df, diary)
inner_batch_size=5000 nb_classes=2 noise_proportion=0.25 score_lin=np.linspace(0,1,100) minibatch_method='lineal' # 'random', 'lineal' n_hidden=[25, 25] output_activation= 'sigmoid' # 'isotonic_regression' # sigmoid if nb_classes == 2: loss='binary_crossentropy' else: loss='categorical_crossentropy' diary = Diary(name='experiment', path='results') diary.add_notebook('hyperparameters') diary.add_entry('hyperparameters', ['train_size', train_size]) diary.add_entry('hyperparameters', ['num_classes', nb_classes]) diary.add_entry('hyperparameters', ['batch_size', batch_size]) diary.add_entry('hyperparameters', ['inner_batch_size', inner_batch_size]) diary.add_entry('hyperparameters', ['minibatch_method', minibatch_method]) diary.add_entry('hyperparameters', ['output_activation', output_activation]) diary.add_entry('hyperparameters', ['loss', loss]) diary.add_entry('hyperparameters', ['optimizer', optimizer.get_config()['name']]) for key, value in optimizer.get_config().iteritems(): diary.add_entry('hyperparameters', [key, value]) diary.add_entry('hyperparameters', ['binarize', binarize]) diary.add_entry('hyperparameters', ['add_noise', add_noise]) diary.add_entry('hyperparameters', ['noise', noise_proportion]) diary.add_notebook('training') diary.add_notebook('validation')
def main(dataset_names=None, estimator_type="kernel", mc_iterations=1, n_folds=10, seed_num=42): if dataset_names is None: dataset_names = ['glass', 'hepatitis', 'ionosphere', 'vowel'] bandwidths_o_norm = { 'glass': 0.09, 'hepatitis': 0.105, 'ionosphere': 0.039, 'vowel': 0.075 } bandwidths_bc = { 'glass': 0.09, 'hepatitis': 0.105, 'ionosphere': 0.039, 'vowel': 0.0145 } bandwidths_t_norm = { 'glass': 0.336, 'hepatitis': 0.015, 'ionosphere': 0.0385, 'vowel': 0.0145 } tuned_mus = { 'glass': [0.094, 0.095, 0.2, 0.0, 0.0, 0.1], 'vowel': [0.0, 0.0, 0.5, 0.5, 0.5, 0.0] } tuned_ms = { 'glass': [1.0, 1.0, 1.0, 1.0, 1.0, 1.0], 'vowel': [1.0, 1.0, 1.0, 1.0, 1.0, 1.0] } bandwidth_o_norm = 0.05 bandwidth_t_norm = 0.05 bandwidth_bc = 0.05 # Diary to save the partial and final results diary = Diary(name='results_Tax2008', path='results', overwrite=False, fig_format='svg') # Hyperparameters for this experiment (folds, iterations, seed) diary.add_notebook('parameters', verbose=True) # Summary for each dataset diary.add_notebook('datasets', verbose=False) # Partial results for validation diary.add_notebook('validation', verbose=True) # Final results diary.add_notebook('summary', verbose=True) columns = ['dataset', 'method', 'mc', 'test_fold', 'acc'] df = MyDataFrame(columns=columns) diary.add_entry('parameters', [ 'seed', seed_num, 'mc_it', mc_iterations, 'n_folds', n_folds, 'estimator_type', estimator_type, 'bw_o', bandwidth_o_norm, 'bw_t', bandwidth_t_norm, 'bw_bc', bandwidth_bc ]) data = Data(dataset_names=dataset_names) for name, dataset in data.datasets.iteritems(): if name in ['letter', 'shuttle']: dataset.reduce_number_instances(0.1) export_datasets_description_to_latex(data, path=diary.path) for i, (name, dataset) in enumerate(data.datasets.iteritems()): np.random.seed(seed_num) dataset.print_summary() diary.add_entry('datasets', [dataset.__str__()]) # accuracies_tuned = np.zeros(mc_iterations * n_folds) # if name in bandwidths_o_norm.keys(): # bandwidth_o_norm = bandwidths_o_norm[name] # bandwidth_t_norm = bandwidths_t_norm[name] # bandwidth_bc = bandwidths_bc[name] # else: # bandwidth_o_norm = np.mean(bandwidths_o_norm.values()) # bandwidth_t_norm = np.mean(bandwidths_t_norm.values()) # bandwidth_bc = np.mean(bandwidths_bc.values()) for mc in np.arange(mc_iterations): skf = StratifiedKFold(dataset.target, n_folds=n_folds, shuffle=True) test_folds = skf.test_folds for test_fold in np.arange(n_folds): x_train, y_train, x_test, y_test = separate_sets( dataset.data, dataset.target, test_fold, test_folds) # if name in ['glass', 'hepatitis', 'ionosphere', 'thyroid', # 'iris', 'heart-statlog', 'diabetes', 'abalone', # 'mushroom', 'spambase']: x_test, y_test = generate_outliers(x_test, y_test) # elif name == 'vowel': # x_train = x_train[y_train <= 5] # y_train = y_train[y_train <= 5] # y_test[y_test > 5] = 6 # elif dataset.n_classes > 2: # x_train = x_train[y_train <= dataset.n_classes/2] # y_train = y_train[y_train <= dataset.n_classes/2] # y_test[y_test > dataset.n_classes/2] = dataset.n_classes+1 # else: # continue if estimator_type == "svm": est = OneClassSVM(nu=0.5, gamma=1.0 / x_train.shape[1]) elif estimator_type == "gmm": est = GMM(n_components=1) elif estimator_type == "gmm3": est = GMM(n_components=3) elif estimator_type == "kernel": est = MyMultivariateKernelDensity(kernel='gaussian', bandwidth=bandwidth_bc) estimators = None bcs = None if estimator_type == "kernel": estimators, bcs = fit_estimators( MyMultivariateKernelDensity(kernel='gaussian', bandwidth=bandwidth_bc), x_train, y_train) # Untuned background check bc = BackgroundCheck(estimator=est, mu=0.0, m=1.0) oc = OcDecomposition(base_estimator=bc) if estimators is None: oc.fit(x_train, y_train) else: oc.set_estimators(bcs, x_train, y_train) accuracy = oc.accuracy(x_test, y_test) diary.add_entry('validation', [ 'dataset', name, 'method', 'BC', 'mc', mc, 'test_fold', test_fold, 'acc', accuracy ]) df = df.append_rows([[name, 'BC', mc, test_fold, accuracy]]) e = MyMultivariateKernelDensity(kernel='gaussian', bandwidth=bandwidth_o_norm) oc_o_norm = OcDecomposition(base_estimator=e, normalization="O-norm") if estimators is None: oc_o_norm.fit(x_train, y_train) else: oc_o_norm.set_estimators(estimators, x_train, y_train) accuracy_o_norm = oc_o_norm.accuracy(x_test, y_test) diary.add_entry('validation', [ 'dataset', name, 'method', 'O-norm', 'mc', mc, 'test_fold', test_fold, 'acc', accuracy_o_norm ]) df = df.append_rows( [[name, 'O-norm', mc, test_fold, accuracy_o_norm]]) e = MyMultivariateKernelDensity(kernel='gaussian', bandwidth=bandwidth_t_norm) oc_t_norm = OcDecomposition(base_estimator=e, normalization="T-norm") if estimators is None: oc_t_norm.fit(x_train, y_train) else: oc_t_norm.set_estimators(estimators, x_train, y_train) accuracy_t_norm = oc_t_norm.accuracy(x_test, y_test) diary.add_entry('validation', [ 'dataset', name, 'method', 'T-norm', 'mc', mc, 'test_fold', test_fold, 'acc', accuracy_t_norm ]) df = df.append_rows( [[name, 'T-norm', mc, test_fold, accuracy_t_norm]]) # Tuned background check # if name in tuned_mus.keys(): # mus = tuned_mus[name] # ms = tuned_ms[name] # else: # mus = None # ms = None # bc = BackgroundCheck(estimator=est, mu=0.0, m=1.0) # oc_tuned = OcDecomposition(base_estimator=bc) # oc_tuned.fit(x_train, y_train, mus=mus, ms=ms) # accuracy_tuned = oc_tuned.accuracy(x_test, y_test, mus=mus, # ms=ms) # accuracies_tuned[mc * n_folds + test_fold] = accuracy_tuned # diary.add_entry('validation', ['dataset', name, # 'method', 'BC-tuned', # 'mc', mc, # 'test_fold', test_fold, # 'acc', accuracy_tuned]) # df = df.append_rows([[name, 'BC-tuned', mc, test_fold, # accuracy_tuned]]) export_summary(df, diary)
def sgd_optimization_gauss(learning_rate=0.13, n_epochs=1000, batch_size=600): """ Demonstrate stochastic gradient descent optimization of a log-linear model :type learning_rate: float :param learning_rate: learning rate used (factor for the stochastic gradient) :type n_epochs: int :param n_epochs: maximal number of epochs to run the optimizer """ #datasets = load_data(dataset) diary = Diary(name='experiment', path='results') diary.add_notebook('training') diary.add_notebook('validation') diary.add_notebook('data') samples=[4000,10000] diary.add_entry('data', ['samples', samples]) diary.add_entry('data', ['num_classes', len(samples)]) diary.add_entry('data', ['batch_size', batch_size]) #means=[[0,0],[5,5]] #cov=[[[1,0],[0,1]],[[3,0],[0,3]]] #diary.add_entry('data', ['means', means]) #diary.add_entry('data', ['covariance', cov]) #datasets = generate_gaussian_data(means, cov, samples) datasets = generate_opposite_cs_data(samples) train_set_x, train_set_y = datasets[0] valid_set_x, valid_set_y = datasets[1] test_set_x, test_set_y = datasets[2] diary.add_entry('data', ['train_size', len(train_set_y.eval())]) diary.add_entry('data', ['valid_size', len(valid_set_y.eval())]) diary.add_entry('data', ['test_size', len(test_set_y.eval())]) pt = PresentationTier() pt.plot_samples(train_set_x.eval(), train_set_y.eval()) # compute number of minibatches for training, validation and testing n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] / batch_size n_test_batches = test_set_x.get_value(borrow=True).shape[0] / batch_size delta = 20 x_min = numpy.min(train_set_x.eval(),axis=0) x_max = numpy.max(train_set_x.eval(),axis=0) x1_lin = numpy.linspace(x_min[0], x_max[0], delta) x2_lin = numpy.linspace(x_min[1], x_max[1], delta) MX1, MX2 = numpy.meshgrid(x1_lin, x2_lin) x_grid = numpy.asarray([MX1.flatten(),MX2.flatten()]).T grid_set_x = theano.shared(numpy.asarray(x_grid, dtype=theano.config.floatX), borrow=True) n_grid_batches = grid_set_x.get_value(borrow=True).shape[0] / batch_size ###################### # BUILD ACTUAL MODEL # ###################### print '... building the model' # allocate symbolic variables for the data index = T.lscalar() # index to a [mini]batch x = T.matrix('x') # the data is presented as rasterized images y = T.ivector('y') # the labels are presented as 1D vector of # [int] labels # construct the logistic regression class # Each MNIST image has size 28*28 n_in = train_set_x.eval().shape[-1] n_out = max(train_set_y.eval()) + 1 classifier = LogisticRegression(input=x, n_in=n_in, n_out=n_out) # the cost we minimize during training is the negative log likelihood of # the model in symbolic format cost = classifier.negative_log_likelihood(y) # compiling a Theano function that computes the mistakes that are made by # the model on a minibatch test_model = theano.function(inputs=[index], outputs=classifier.errors(y), givens={ x: test_set_x[index * batch_size: (index + 1) * batch_size], y: test_set_y[index * batch_size: (index + 1) * batch_size]}) validate_model = theano.function(inputs=[index], outputs=classifier.errors(y), givens={ x: valid_set_x[index * batch_size:(index + 1) * batch_size], y: valid_set_y[index * batch_size:(index + 1) * batch_size]}) # Scores grid_scores_model = theano.function(inputs=[], outputs=classifier.scores(), givens={ x: grid_set_x}) training_scores_model = theano.function( inputs=[index], outputs=classifier.scores(), givens={ x: train_set_x[index * batch_size:(index + 1) * batch_size], } ) validation_scores_model = theano.function( inputs=[index], outputs=classifier.scores(), givens={ x: valid_set_x[index * batch_size:(index + 1) * batch_size], } ) # compute the gradient of cost with respect to theta = (W,b) g_w = T.grad(cost=cost, wrt=classifier.w) g_b = T.grad(cost=cost, wrt=classifier.b) # specify how to update the parameters of the model as a list of # (variable, update expression) pairs. updates = [(classifier.w, classifier.w - learning_rate * g_w), (classifier.b, classifier.b - learning_rate * g_b)] # compiling a Theano function `train_model` that returns the cost, but in # the same time updates the parameter of the model based on the rules # defined in `updates` train_model = theano.function(inputs=[index], outputs=cost, updates=updates, givens={ x: train_set_x[index * batch_size:(index + 1) * batch_size], y: train_set_y[index * batch_size:(index + 1) * batch_size]}) # Accuracy validation_accuracy_model = theano.function( inputs=[index], outputs=classifier.accuracy(y), givens={ x: valid_set_x[index * batch_size:(index + 1) * batch_size], y: valid_set_y[index * batch_size:(index + 1) * batch_size] } ) training_accuracy_model = theano.function( inputs=[index], outputs=classifier.accuracy(y), givens={ x: train_set_x[index * batch_size:(index + 1) * batch_size], y: train_set_y[index * batch_size:(index + 1) * batch_size] } ) # Loss training_error_model = theano.function(inputs=[index], outputs=classifier.errors(y), givens={ x: train_set_x[index * batch_size:(index + 1) * batch_size], y: train_set_y[index * batch_size:(index + 1) * batch_size]}) validation_error_model = theano.function(inputs=[index], outputs=classifier.errors(y), givens={ x: valid_set_x[index * batch_size:(index + 1) * batch_size], y: valid_set_y[index * batch_size:(index + 1) * batch_size]}) ############### # TRAIN MODEL # ############### print '... training the model' # early-stopping parameters patience = 5000 # look as this many examples regardless patience_increase = 2 # wait this much longer when a new best is # found improvement_threshold = 0.995 # a relative improvement of this much is # considered significant validation_frequency = min(n_train_batches, patience / 2) # go through this many # minibatche before checking the network # on the validation set; in this case we # check every epoch print('Creating error and accuracy vectors') error_train = numpy.zeros(n_epochs+1) error_val = numpy.zeros(n_epochs+1) accuracy_train = numpy.zeros(n_epochs+1) accuracy_val = numpy.zeros(n_epochs+1) # Results for Isotonic Regression error_train_ir = numpy.zeros(n_epochs+1) error_val_ir = numpy.zeros(n_epochs+1) accuracy_train_ir = numpy.zeros(n_epochs+1) accuracy_val_ir = numpy.zeros(n_epochs+1) best_params = None best_validation_loss = numpy.inf test_score = 0. start_time = time.clock() ir = IsotonicRegression(increasing=True, out_of_bounds='clip', y_min=0, y_max=1) done_looping = False epoch = 0 CS = None while (epoch < n_epochs) and (not done_looping): epoch = epoch + 1 for minibatch_index in xrange(n_train_batches): minibatch_avg_cost = train_model(minibatch_index) # iteration number iter = (epoch - 1) * n_train_batches + minibatch_index if (iter + 1) % validation_frequency == 0: # compute zero-one loss on validation set validation_losses = [validate_model(i) for i in xrange(n_valid_batches)] this_validation_loss = numpy.mean(validation_losses) print('epoch %i, minibatch %i/%i, validation error %f %%' % \ (epoch, minibatch_index + 1, n_train_batches, this_validation_loss * 100.)) # if we got the best validation score until now if this_validation_loss < best_validation_loss: #improve patience if loss improvement is good enough if this_validation_loss < best_validation_loss * \ improvement_threshold: patience = max(patience, iter * patience_increase) best_validation_loss = this_validation_loss # test it on the test set test_losses = [test_model(i) for i in xrange(n_test_batches)] test_score = numpy.mean(test_losses) print((' epoch %i, minibatch %i/%i, test error of best' ' model %f %%') % (epoch, minibatch_index + 1, n_train_batches, test_score * 100.)) if patience <= iter: done_looping = True break scores_grid = grid_scores_model() fig = pt.update_contourline(grid_set_x.eval(), scores_grid, delta) diary.save_figure(fig, filename='contour_lines', extension='svg') scores_train = numpy.asarray([training_scores_model(i) for i in range(n_train_batches)]).flatten() scores_val = numpy.asarray([validation_scores_model(i) for i in range(n_valid_batches)]).flatten() print('Learning Isotonic Regression from TRAINING set') ir.fit(scores_train, train_set_y.eval()) scores_train_ir = ir.predict(scores_train) print('IR predict validation probabilities') scores_val_ir = ir.predict(scores_val) scores_set = (scores_train, scores_val, scores_train_ir, scores_val_ir) labels_set = (train_set_y.eval(), valid_set_y.eval(), train_set_y.eval(), valid_set_y.eval()) legend = ['train', 'valid', 'iso. train', 'iso. valid'] fig = pt.plot_reliability_diagram(scores_set, labels_set, legend) diary.save_figure(fig, filename='reliability_diagram', extension='svg') # TODO add reliability map scores_set = (scores_train) prob_set = (train_set_y.eval()) fig = pt.plot_reliability_map(scores_set, labels_set, legend) diary.save_figure(fig, filename='reliability_map', extension='svg') fig = pt.plot_histogram_scores(scores_set) diary.save_figure(fig, filename='histogram_scores', extension='svg') # Performance accuracy_train[epoch] = numpy.asarray([training_accuracy_model(i) for i in range(n_train_batches)]).flatten().mean() accuracy_val[epoch] = numpy.asarray([validation_accuracy_model(i) for i in range(n_valid_batches)]).flatten().mean() error_train[epoch] = numpy.asarray([training_error_model(i) for i in range(n_train_batches)]).flatten().mean() error_val[epoch] = numpy.asarray([validation_error_model(i) for i in range(n_valid_batches)]).flatten().mean() accuracy_train_ir[epoch] = compute_accuracy(scores_train_ir, train_set_y.eval()) accuracy_val_ir[epoch] = compute_accuracy(scores_val_ir, valid_set_y.eval()) error_train_ir[epoch] = compute_cross_entropy(scores_train_ir, train_set_y.eval()) error_val_ir[epoch] = compute_cross_entropy(scores_val_ir, valid_set_y.eval()) diary.add_entry('training', [error_train[epoch], accuracy_train[epoch]]) diary.add_entry('validation', [error_val[epoch], accuracy_val[epoch]]) accuracy_set = (accuracy_train[1:epoch], accuracy_val[1:epoch], accuracy_train_ir[1:epoch], accuracy_val_ir[1:epoch]) fig = pt.plot_accuracy(accuracy_set, legend) diary.save_figure(fig, filename='accuracy', extension='svg') error_set = (error_train[1:epoch], error_val[1:epoch], error_train_ir[1:epoch], error_val_ir[1:epoch]) fig = pt.plot_error(error_set, legend, 'cross-entropy') diary.save_figure(fig, filename='error', extension='svg') pt.update_contourline(grid_set_x.eval(), scores_grid, delta, clabel=True) end_time = time.clock() print(('Optimization complete with best validation score of %f %%,' 'with test performance %f %%') % (best_validation_loss * 100., test_score * 100.)) print 'The code run for %d epochs, with %f epochs/sec' % ( epoch, 1. * epoch / (end_time - start_time))