def plot_param_sweep(results, n, params, param_name, param_logscale): """Plot accuracy versus parameter. Parameters ---------- results: list[float] The accuracy or loss for a number of experiments, each of which used different parameters. n: int The number of test samples used for each experiment. params: list[float] The parameter values that changed during each experiment. param_name: str The name of the parameter that varied. param_logscale: bool Whether to plot the parameter axis in log-scale. """ from snntoolbox.utils.utils import wilson_score # Compute confidence intervals of the experiments ci = [wilson_score(q, n) for q in results] if param_logscale: plt.xscale('log', nonposx='clip') plt.errorbar(params, results, yerr=ci, fmt='x-') plt.title('Accuracy vs Hyperparameter') plt.xlabel(param_name) plt.ylabel('accuracy') fac = 0.9 if params[0] < 0: fac += 0.2 plt.xlim(fac * params[0], 1.1 * params[-1]) plt.ylim(0, 1)
def set_spikestats(self): from snntoolbox.utils.utils import wilson_score num_batches = len(os.listdir(self.dirname)) if num_batches == 0: return batch_size, num_timesteps = np.load(os.path.join( self.dirname, '0.npz'))['top1err_b_t'].shape self.time = np.arange(num_timesteps) self.num_samples = num_batches * batch_size e1 = np.empty((self.num_samples, num_timesteps)) e5 = np.empty((self.num_samples, num_timesteps)) # Load operation count operations_d_t = np.empty((self.num_samples, num_timesteps)) for batch_idx in range(num_batches): operations_d_t[batch_idx*batch_size:(batch_idx+1)*batch_size] = \ np.load(os.path.join(self.dirname, str(batch_idx) + '.npz'))[ 'synaptic_operations_b_t'] / self.scale self.mean_computations_t = np.mean(operations_d_t, 0) self.std_computations_t = np.std(operations_d_t, 0) # Load error for batch_idx in range(num_batches): e1[batch_idx * batch_size: (batch_idx + 1) * batch_size] = \ np.multiply(100, np.load(os.path.join( self.dirname, str(batch_idx) + '.npz'))['top1err_b_t']) e5[batch_idx * batch_size: (batch_idx + 1) * batch_size] = \ np.multiply(100, np.load(os.path.join( self.dirname, str(batch_idx) + '.npz'))['top5err_b_t']) self.operations_ann = float(np.load(os.path.join(self.dirname, str( num_batches - 1) + '.npz'))['operations_ann'] / self.scale) self.e1_ann = float(np.load(os.path.join( self.dirname, str(num_batches - 1) + '.npz'))['top1err_ann']) * 100 self.e5_ann = float(np.load(os.path.join( self.dirname, str(num_batches - 1) + '.npz'))['top5err_ann']) * 100 # Averaged across samples, shape (1, num_timesteps) self.e1_mean = np.mean(e1, axis=0) self.e1_std = np.std(e1, axis=0) self.e5_mean = np.mean(e5, axis=0) self.e5_std = np.std(e5, axis=0) self.e1_confidence95 = np.array([wilson_score(1-e/100, self.num_samples) for e in self.e1_mean]) * 100 self.e5_confidence95 = np.array([wilson_score(1-e/100, self.num_samples) for e in self.e5_mean]) * 100 # Get the operation count at which the error is minimal or 1 % above the # min. self.e1_0 = min(self.e1_mean) self.op1_0 = get_op_at_err(self.mean_computations_t, self.e1_mean, self.e1_0) self.e1_1 = min(self.e1_mean) + 1 self.op1_1 = get_op_at_err(self.mean_computations_t, self.e1_mean, self.e1_1) self.e1_1 = get_err_at_op(self.e1_mean, self.mean_computations_t, self.op1_1) self.e1_2 = get_err_at_op(self.e1_mean, self.mean_computations_t, self.operations_ann) self.op1_2 = get_op_at_err(self.mean_computations_t, self.e1_mean, self.e1_2) self.e5_0 = min(self.e5_mean) self.op5_0 = get_op_at_err(self.mean_computations_t, self.e5_mean, self.e5_0) self.e5_1 = min(self.e5_mean) + 1 self.op5_1 = get_op_at_err(self.mean_computations_t, self.e5_mean, self.e5_1) self.e5_1 = get_err_at_op(self.e5_mean, self.mean_computations_t, self.op5_1) self.op5_1 = get_op_at_err(self.mean_computations_t, self.e5_mean, self.e5_1) self.e5_2 = get_err_at_op(self.e5_mean, self.mean_computations_t, self.operations_ann) self.op5_2 = get_op_at_err(self.mean_computations_t, self.e5_mean, self.e5_2) self.e1_std_ann = get_std(self.e1_ann) self.e5_std_ann = get_std(self.e5_ann) self.e1_confidence95_ann = wilson_score(1 - self.e1_ann / 100, self.num_samples) * 100 self.e5_confidence95_ann = wilson_score(1 - self.e5_ann / 100, self.num_samples) * 100 self.e1_optimal, self.op1_optimal = get_minimal_err_and_op( self.mean_computations_t, self.e1_mean) self.e5_optimal, self.op5_optimal = get_minimal_err_and_op( self.mean_computations_t, self.e5_mean)