def test_squared_errors_3(self): """ Tupla obtenida de los test anteriores con un error relativamente bajo ( < 2 ) """ weights = [ 1.0, 0.9999999898725264, -1.0, 0.9999999941226553, 0.7789066176302455, 0.9999999976272673, 0.7685086147538794, 1.0, 0.7789465050493634, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 ] moderate_constant = 0.01 iterations = 30 errors = [] for i in range(iterations): board = experiment_generator() print('Obteniendo traza del juego...') game_trace = get_game_trace_with_random_player(board, weights) print(f'Se obtuvieron {game_trace.__len__()} tuplas') training_examples = get_training_examples(game_trace, weights) print('Ajustando pesos...') weights = gen(training_examples, weights, moderate_constant) errors.append(squared_error(training_examples, weights)) print(f'Pesos obtenidos : {weights}') print(f'Error cuadratico : {errors[-1]}') for i in range(iterations): print('Error {}: {}'.format(i, errors[i])) for i in range(iterations - 1): self.assertGreaterEqual( errors[i], errors[i + 1], f'El error {i} no es mayor o igual que el error {i+1}, los errores deben decrecer' )
def fit(self, X, Y): self.X = X.copy() self.Y = Y.copy() self.N = X.shape[0] self.n_input = X.shape[1] self.n_output = Y.shape[1] # ReLU init self.w0 = np.random.rand(self.n_input, self.n_neurons) * np.sqrt( 2 / self.n_input) self.w1 = np.random.rand(self.n_neurons, self.n_output) * np.sqrt( 2 / self.n_neurons) self.b0 = np.random.rand() * np.sqrt(2 / self.n_input) self.b1 = np.random.rand() * np.sqrt(2 / self.n_input) self.layer0 = np.zeros(self.n_input) self.layer1 = np.zeros(self.n_neurons) self.layer2 = np.zeros(self.n_output) last_sse = np.inf for i in range(self.max_iter): sse = 0 for j in range(X.shape[0]): o = self.__forward_propagate(self.X[j]) self.__back_propagate(self.Y[j]) sse += utils.squared_error(o, self.Y[j]) print(i, ' ', sse) if (last_sse - sse) < self.tol: break last_sse = sse
def train_3(weights, moderate_constant, max_error): board = experiment_generator() game_trace = get_game_trace_with_old_vesion(board, weights, weights) training_examples = get_training_examples(game_trace, weights) weights = gen(training_examples, weights, moderate_constant) error = squared_error(training_examples, weights) return weights, error, error <= max_error
def test_squared_errors_2(self): weights = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] moderate_constant = 0.5 iterations = 5 errors = [] for i in range(iterations): board = experiment_generator() print('Obteniendo traza del juego...') game_trace = get_game_trace_with_random_player(board, weights) print(f'Se obtuvieron {game_trace.__len__()} tuplas') training_examples = get_training_examples(game_trace, weights) print('Ajustando pesos...') weights = gen(training_examples, weights, moderate_constant) errors.append(squared_error(training_examples, weights)) print(f'Pesos obtenidos : {weights}') print(f'Error cuadratico : {errors[-1]}') for i in range(iterations): print('Error {}: {}'.format(i, errors[i])) for i in range(iterations - 1): self.assertGreaterEqual( errors[i], errors[i + 1], f'El error {i} no es mayor o igual que el error {i+1}, los errores deben decrecer' )
def test_squared_errors9(self): logging.basicConfig(filename='./logs/test_with_random_player_mu09.log', level=logging.INFO) logging.info( '------------------------------------------------------------------------------------------------' ) logging.info('Started') moderate_constant = 0.9 iterations = 50 weights = [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] errors = [] won = 0 lost = 0 for i in range(iterations): board = experiment_generator() logging.info('Obteniendo traza del juego...') game_trace = get_game_trace_with_random_player(board, weights) logging.info(f'Se obtuvieron {game_trace.__len__()} tuplas') training_examples = get_training_examples(game_trace, weights) board_features = training_examples[training_examples.__len__() - 1][0] logging.info(training_examples[training_examples.__len__() - 1]) if board_features[6] >= 1: won = won + 1 elif board_features[13] >= 1: lost = lost + 1 logging.info('Ajustando pesos...') weights = gen(training_examples, weights, moderate_constant) errors.append(squared_error(training_examples, weights)) logging.info(weights) # if errors[i -1] < errors[i]: # moderate_constant = max(0.1, moderate_constant - 0.1) # logging.info(moderate_constant) for i in range(iterations): logging.info('Error {}: {}'.format(i, errors[i])) # for i in range(iterations-1): # self.assertGreaterEqual(errors[i], errors[i+1], # f'El error {i} no es mayor o igual que el error {i+1}, los errores deben decrecer') logging.info(f'Ganados: {won}') logging.info(f'Perdidos: {lost}') logging.info('Finished') logging.info( '------------------------------------------------------------------------------------------------' ) return weights
def inf_g(self, p_x, g_gen, x, locations): # Infer abstract location from the combination of [grounded location retrieved from memory by sensory experience] ... if self.hyper['use_p_inf']: # Not in paper, but makes sense from symmetry with f_x: first get g from p by "summing over sensory preferences" g = p * W_repeat^T g_downsampled = [torch.matmul(p_x[f], torch.t(self.hyper['W_repeat'][f])) for f in range(self.hyper['n_f'])] # Then use abstract location after summing over sensory preferences as input to MLP to obtain the inferred abstract location from memory mu_g_mem = self.f_mu_g_mem(g_downsampled) # Not in paper, but this greatly improves zero-shot inference: provide the uncertainty function of the inferred abstract location with measures of memory quality with torch.no_grad(): # For the first measure, use the grounded location inferred from memory to generate an observation x_hat, x_hat_logits = self.gen_x(p_x[0]) # Then calculate the error between the generated observation and the actual observation: if the memory is working well, this error should be small err = utils.squared_error(x, x_hat) # The second measure is the vector norm of the inferred abstract location; good memories should have similar vector norms. Concatenate the two measures as input for the abstract location uncertainty function sigma_g_input = [torch.cat((torch.sum(g ** 2, dim=1, keepdim=True), torch.unsqueeze(err, dim=1)), dim=1) for g in mu_g_mem] # Not in paper, but recommended by James for stability: get final mean of inferred abstract location by clamping activations between -1 and 1 mu_g_mem = self.f_g_clamp(mu_g_mem) # And get standard deviation/uncertainty of inferred abstract location by providing uncertainty function with memory quality measures sigma_g_mem = self.f_sigma_g_mem(sigma_g_input) # ... and [previous abstract location and action (path integration)] mu_g_path = g_gen[0] sigma_g_path = g_gen[1] # Infer abstract location by combining previous abstract location and grounded location retrieved from memory by current sensory experience mu_g, sigma_g = [], [] for f in range(self.hyper['n_f']): if self.hyper['use_p_inf']: # Then get full gaussian distribution of inferred abstract location by calculating precision weighted mean mu, sigma = utils.inv_var_weight([mu_g_path[f], mu_g_mem[f]],[sigma_g_path[f], sigma_g_mem[f]]) else: # Or simply completely ignore the inference memory here, to test if things are working mu, sigma = mu_g_path[f], sigma_g_path[f] # Append mu and sigma to list for all frequency modules mu_g.append(mu) sigma_g.append(sigma) # Finally (though not in paper), also add object vector cell information to inferred abstract location for environments with shiny objects shiny_envs = [location['shiny'] is not None for location in locations] if any(shiny_envs): # Find for which environments the current location has a shiny object shiny_locations = torch.unsqueeze(torch.stack([torch.tensor(location['shiny'], dtype=torch.float) for location in locations if location['shiny'] is not None]), dim=-1) # Get abstract location for environments with shiny objects and feed to each of the object vector cell modules mu_g_shiny = self.f_mu_g_shiny([shiny_locations for _ in range(self.hyper['n_f_g'] if self.hyper['separate_ovc'] else self.hyper['n_f'])]) sigma_g_shiny = self.f_sigma_g_shiny([shiny_locations for _ in range(self.hyper['n_f_g'] if self.hyper['separate_ovc'] else self.hyper['n_f'])]) # Update only object vector modules with shiny-inferred abstract location: start from offset if object vector modules are separate module_start = self.hyper['n_f_g'] if self.hyper['separate_ovc'] else 0 # Inverse variance weighting is associative, so I can just do additional inverse variance weighting to the previously obtained mu and sigma - but only for object vector cell modules! for f in range(module_start, self.hyper['n_f']): # Add inferred abstract location from shiny objects to previously obtained position, only for environments with shiny objects mu, sigma = utils.inv_var_weight([mu_g[f][shiny_envs,:], mu_g_shiny[f - module_start]], [sigma_g[f][shiny_envs,:], sigma_g_shiny[f - module_start]]) # In order to update only the environments with shiny objects, without in-place value assignment, construct a mask of shiny environments mask = torch.zeros_like(mu_g[f], dtype=torch.bool) mask[shiny_envs,:] = True # Use mask to update the shiny environment entries in inferred abstract locations mu_g[f] = mu_g[f].masked_scatter(mask,mu) sigma_g[f] = sigma_g[f].masked_scatter(mask,sigma) # Either sample inferred abstract location from combined (precision weighted) distribution or just take mean g = [mu_g[f] + sigma_g[f] * np.random.randn() if self.hyper['do_sample'] else mu_g[f] for f in range(self.hyper['n_f'])] # Return abstract location inferred from grounded location from memory and previous abstract location return g
def test_squared_errors_4(self): """ Probar : a mayor cantidad de iteraciones, menor constante de entrenamiento """ weights = [0, 1, -1, 2, -1, 3, -1, 2, -1, 1, -1, 1, -1, 1, -2] moderate_constant = 0.001 iterations = 100 errors = [] board = experiment_generator() game_trace = get_game_trace_with_random_player(board, weights) training_examples = get_training_examples(game_trace, weights) weights = gen(training_examples, weights, moderate_constant) best_weights = weights best_game_trace = game_trace error = squared_error(training_examples, best_weights) errors.append(error) for i in range(iterations - 1): board = experiment_generator() game_trace = get_game_trace_with_random_player(board, best_weights) training_examples = get_training_examples(game_trace, best_weights) weights = gen(training_examples, best_weights, moderate_constant) error = squared_error(training_examples, best_weights) print(f'Error : {error}') # Solo actualizo los pesos si la cantidad de jugadas realizadas para ganar # es menor igual que lo que se tiene hasta el momento como minimo. if game_trace.__len__() <= best_game_trace.__len__(): best_weights = weights best_game_trace = game_trace error = squared_error(training_examples, best_weights) errors.append(error) print( f'Tuplas obtenidas en traza de juego : {best_game_trace.__len__()}' ) print(f'Error cuadratico : {error}') for i in range(errors.__len__()): print('Error {}: {}'.format(i, errors[i])) print(f'Pesos obtenidos: {best_weights}')
def predict(self, X): y = np.zeros(X.shape[0], dtype=int) for i in range(X.shape[0]): squared_errors = np.zeros(self.n_clusters) for j in range(self.n_clusters): squared_errors[j] = utils.squared_error( X[i], self.cluster_centers_[j]) y[i] = np.argmin(squared_errors) return y
def orchestrate(self): training_examples = get_training_examples(self.game_trace, self.weights) self.weights = gen(training_examples, self.weights, self.moderate_constant) self.error = squared_error(training_examples, self.weights) for te in training_examples: print(te) print(f'Moderate constant: {self.moderate_constant}') print(f'Game trace: {self.game_trace}') print(f'Pesos obtenidos: {self.weights}') print(f'Error cuadratico: {self.error}')
def loss(self, g_gen, p_gen, x_logits, x, g_inf, p_inf, p_inf_x, M_prev): # Calculate loss function, separately for each component because you might want to reweight contributions later # L_p_gen is squared error loss between inferred grounded location and grounded location retrieved from inferred abstract location L_p_g = torch.sum(torch.stack(utils.squared_error(p_inf, p_gen), dim=0), dim=0) # L_p_inf is squared error loss between inferred grounded location and grounded location retrieved from sensory experience L_p_x = torch.sum(torch.stack(utils.squared_error(p_inf, p_inf_x), dim=0), dim=0) if self.hyper['use_p_inf'] else torch.zeros_like(L_p_g) # L_g is squared error loss between generated abstract location and inferred abstract location L_g = torch.sum(torch.stack(utils.squared_error(g_inf, g_gen), dim=0), dim=0) # L_x is a cross-entropy loss between sensory experience and different model predictions. First get true labels from sensory experience labels = torch.argmax(x, 1) # L_x_gen: losses generated by generative model from g_prev -> g -> p -> x L_x_gen = utils.cross_entropy(x_logits[2], labels) # L_x_g: Losses generated by generative model from g_inf -> p -> x L_x_g = utils.cross_entropy(x_logits[1], labels) # L_x_p: Losses generated by generative model from p_inf -> x L_x_p = utils.cross_entropy(x_logits[0], labels) # L_reg are regularisation losses, L_reg_g on L2 norm of g L_reg_g = torch.sum(torch.stack([torch.sum(g ** 2, dim=1) for g in g_inf], dim=0), dim=0) # And L_reg_p regularisation on L1 norm of p L_reg_p = torch.sum(torch.stack([torch.sum(torch.abs(p), dim=1) for p in p_inf], dim=0), dim=0) # Return total loss as list of losses, so you can possibly reweight them L = [L_p_g, L_p_x, L_x_gen, L_x_g, L_x_p, L_g, L_reg_g, L_reg_p] return L
def run_experiment(est_name, seed, dim, param, ntrain, ntest, nreps, nbags, nfolds, save_all): """ Run a single experiment Parameters ---------- est_name: str Name of the estimator. 'LR' or 'SVM-RBF' seed: int Seed of the experiment dim: int Dimension of the dataset, 1 or 2 param: int, str Extra oarameter for the definition of the problem. If dim==1, this value is the std. If dim=2 this value is an string to indicate if the dataset is the one designed to test HDX ntrain : list List with the number of training examples that must be tested, e.g.,[50, 100, 200] ntest: int Number of testing instances in each bag nreps: int Number of training datasets created nbags: int Number of testing bags created for each training datasets. The total number of experiments will be nreps * nbags nfolds: int Number of folds used to estimate the training distributions by the methods AC, HDy and EDy save_all: bool True if the results of each single experiment must be saved """ # range of testing prevalences low = round(ntest * 0.05) high = round(ntest * 0.95) if est_name == 'LR': estimator = LogisticRegression(C=1, random_state=seed, max_iter=10000, solver='liblinear') else: estimator = SVC(C=1, kernel='rbf', random_state=seed, max_iter=10000, gamma=0.2, probability=True) rng = np.random.RandomState(seed) # methods cc = CC() ac = AC() pac = PAC() sordy = SORDy() # hdys pdfy_hd_4 = DFy(distribution_function='PDF', n_bins=4, distance='HD') pdfy_hd_8 = DFy(distribution_function='PDF', n_bins=8, distance='HD') pdfy_hd_12 = DFy(distribution_function='PDF', n_bins=12, distance='HD') pdfy_hd_16 = DFy(distribution_function='PDF', n_bins=16, distance='HD') # cdf l1 cdfy_l1_8 = DFy(distribution_function='CDF', n_bins=8, distance='L1') cdfy_l1_16 = DFy(distribution_function='CDF', n_bins=16, distance='L1') cdfy_l1_32 = DFy(distribution_function='CDF', n_bins=32, distance='L1') cdfy_l1_64 = DFy(distribution_function='CDF', n_bins=64, distance='L1') # cdf l1 # cdfy_l2_8 = DFy(distribution_function='CDF', n_bins=8, distance='L2') # cdfy_l2_16 = DFy(distribution_function='CDF', n_bins=16, distance='L2') # cdfy_l2_32 = DFy(distribution_function='CDF', n_bins=32, distance='L2') # cdfy_l2_64 = DFy(distribution_function='CDF', n_bins=64, distance='L2') # QUANTy-L1 # quanty_l1_4 = QUANTy(n_quantiles=4, distance=l1) # quanty_l1_10 = QUANTy(n_quantiles=10, distance=l1) # quanty_l1_20 = QUANTy(n_quantiles=20, distance=l1) # quanty_l1_40 = QUANTy(n_quantiles=40, distance=l1) # QUANTy-L2 quanty_l2_4 = QUANTy(n_quantiles=4, distance=l2) quanty_l2_10 = QUANTy(n_quantiles=10, distance=l2) quanty_l2_20 = QUANTy(n_quantiles=20, distance=l2) quanty_l2_40 = QUANTy(n_quantiles=40, distance=l2) # methods methods = [cc, ac, pac, sordy, pdfy_hd_4, pdfy_hd_8, pdfy_hd_12, pdfy_hd_16, cdfy_l1_8, cdfy_l1_16, cdfy_l1_32, cdfy_l1_64, # cdfy_l2_8, cdfy_l2_16, cdfy_l2_32, cdfy_l2_64, # quanty_hd_4, quanty_hd_8, quanty_hd_12, quanty_hd_16, # quanty_l1_4, quanty_l1_8, quanty_l1_12, quanty_l1_16, quanty_l2_4, quanty_l2_10, quanty_l2_20, quanty_l2_40] methods_names = ['CC', 'AC', 'PAC', 'SORDy', 'PDFy_hd_4', 'PDFy_hd_8', 'PDFy_hd_12', 'PDFy_hd_16', 'CDFy_l1_8', 'CDFy_l1_16', 'CDFy_l1_32', 'CDFy_l1_64', # 'CDFy_l2_8', 'CDFy_l2_16', 'CDFy_l2_32', 'CDFy_l2_64', # 'QUANTy_l1_4', 'QUANTy_l1_10', 'QUANTy_l1_20', 'QUANTy_l1_40', 'QUANTy_l2_4', 'QUANTy_l2_10', 'QUANTy_l2_20', 'QUANTy_l2_40'] # to store the results mae_results = np.zeros((len(methods_names), len(ntrain))) sqe_results = np.zeros((len(methods_names), len(ntrain))) mrae_results = np.zeros((len(methods_names), len(ntrain))) classif_results = np.zeros((2, len(ntrain))) std1 = std2 = mu3 = mu4 = cov1 = cov2 = cov3 = cov4 = 0 if dim == 1: # 1D mu1 = -1 std1 = param mu2 = 1 std2 = std1 else: # 2D mu1 = [-1.00, 1.00] mu2 = [1.00, 1.00] mu3 = [1.00, -1.00] cov1 = [[0.4, 0], [0, 0.4]] cov2 = cov1 cov3 = cov1 x1 = rng.multivariate_normal(mu1, cov1, 400) x3 = rng.multivariate_normal(mu3, cov3, 400) plt.scatter(np.vstack((x1[:, 0], x3[:, 0])), np.vstack((x1[:, 1], x3[:, 1])), c='r', marker='+', s=12, label='Class \u2212' + '1') if param == 'HDX': mu4 = [-1.00, -1.00] cov4 = cov1 x2 = rng.multivariate_normal(mu2, cov2, 400) x4 = rng.multivariate_normal(mu4, cov4, 400) plt.scatter(np.vstack((x2[:, 0], x4[:, 0])), np.vstack((x2[:, 1], x4[:, 1])), c='b', marker='x', s=8, label='Class +1') else: x2 = rng.multivariate_normal(mu2, cov2, 800) plt.scatter(x2[:, 0], x2[:, 1], c='b', marker='x', s=8, label='Class +1') plt.xlabel('$x_1$') plt.ylabel('$x_2$') plt.legend(loc='best') plt.savefig('./artificial-2D-' + param + '.png', dpi=300) name_file = 'times-avg-artificial-' + str(dim) + 'D-' + str(param) + '-' + est_name + '-rep' + str(nreps) + \ '-ntest' + str(ntest) + '.txt' file_times = open(name_file, 'w') file_times.write('#examples, ') for index, m in enumerate(methods_names): file_times.write('%s, ' % m) for k in range(len(ntrain)): all_mae_results = np.zeros((len(methods_names), nreps * nbags)) all_sqe_results = np.zeros((len(methods_names), nreps * nbags)) all_mrae_results = np.zeros((len(methods_names), nreps * nbags)) execution_times = np.zeros(len(methods_names)) print() print('#Training examples ', ntrain[k], 'Rep#', end=' ') for rep in range(nreps): print(rep+1, end=' ') if dim == 1: x_train = np.vstack(((std1 * rng.randn(ntrain[k], 1) + mu1), (std2 * rng.randn(ntrain[k], 1) + mu2))) else: if param == 'HDX': x_train = np.vstack((rng.multivariate_normal(mu1, cov1, ntrain[k] // 2), rng.multivariate_normal(mu3, cov3, ntrain[k] - ntrain[k] // 2), rng.multivariate_normal(mu2, cov2, ntrain[k] // 2), rng.multivariate_normal(mu4, cov4, ntrain[k] - ntrain[k] // 2))) else: x_train = np.vstack((rng.multivariate_normal(mu1, cov1, ntrain[k] // 2), rng.multivariate_normal(mu3, cov3, ntrain[k] - ntrain[k] // 2), rng.multivariate_normal(mu2, cov2, ntrain[k]))) y_train = np.hstack((np.zeros(ntrain[k], dtype=int), np.ones(ntrain[k], dtype=int))) skf_train = StratifiedKFold(n_splits=nfolds, shuffle=True, random_state=seed + rep) estimator_train = CV_estimator(estimator=estimator, n_jobs=None, cv=skf_train) estimator_train.fit(x_train, y_train) predictions_train = estimator_train.predict_proba(x_train) for nmethod, method in enumerate(methods): if isinstance(method, UsingClassifiers): method.fit(X=x_train, y=y_train, predictions_train=predictions_train) else: method.fit(X=x_train, y=y_train) #estimator_test = estimator #estimator_test.fit(x_train, y_train) estimator_test = estimator_train for n_bag in range(nbags): ps = rng.randint(low, high, 1) ps = np.append(ps, [0, ntest]) ps = np.diff(np.sort(ps)) if dim == 1: x_test = np.vstack(((std1 * rng.randn(ps[0], 1) + mu1), (std2 * rng.randn(ps[1], 1) + mu2))) else: if param == 'HDX': x_test = np.vstack((rng.multivariate_normal(mu1, cov1, ps[0] // 2), rng.multivariate_normal(mu3, cov3, ps[0] - ps[0] // 2), rng.multivariate_normal(mu2, cov2, ps[1] // 2), rng.multivariate_normal(mu4, cov4, ps[1] - ps[1] // 2))) else: x_test = np.vstack((rng.multivariate_normal(mu1, cov1, ps[0] // 2), rng.multivariate_normal(mu3, cov3, ps[0] - ps[0] // 2), rng.multivariate_normal(mu2, cov2, ps[1]))) y_test = np.hstack((np.zeros(ps[0], dtype=int), np.ones(ps[1], dtype=int))) predictions_test = estimator_test.predict_proba(x_test) # Error classif_results[0, k] = classif_results[0, k] + zero_one_loss(np.array(y_test), np.argmax(predictions_test, axis=1)) # Brier loss classif_results[1, k] = classif_results[1, k] + brier_score_loss(indices_to_one_hot(y_test, 2)[:, 0], predictions_test[:, 0]) prev_true = ps[1] / ntest for nmethod, method in enumerate(methods): t = time.process_time() if isinstance(method, UsingClassifiers): p_predicted = method.predict(X=None, predictions_test=predictions_test)[1] else: p_predicted = method.predict(X=x_test)[1] elapsed_time = time.process_time() execution_times[nmethod] = execution_times[nmethod] + elapsed_time - t all_mae_results[nmethod, rep * nbags + n_bag] = absolute_error(prev_true, p_predicted) all_mrae_results[nmethod, rep * nbags + n_bag] = relative_absolute_error(prev_true, p_predicted) all_sqe_results[nmethod, rep * nbags + n_bag] = squared_error(prev_true, p_predicted) mae_results[nmethod, k] = mae_results[nmethod, k] + all_mae_results[nmethod, rep * nbags + n_bag] mrae_results[nmethod, k] = mrae_results[nmethod, k] + all_mrae_results[nmethod, rep * nbags + n_bag] sqe_results[nmethod, k] = sqe_results[nmethod, k] + all_sqe_results[nmethod, rep * nbags + n_bag] execution_times = execution_times / (nreps * nbags) file_times.write('\n%d, ' % ntrain[k]) for i in execution_times: file_times.write('%.5f, ' % i) if save_all: name_file = 'results-all-mae-artificial-' + str(dim) + 'D-' + str(param) + '-' + est_name + \ '-rep' + str(nreps) + '-value' + str(ntrain[k]) + '-ntest' + str(ntest) + '.txt' file_all = open(name_file, 'w') for method_name in methods_names: file_all.write('%s,' % method_name) file_all.write('\n') for nrep in range(nreps): for n_bag in range(nbags): for n_method in range(len(methods_names)): file_all.write('%.5f, ' % all_mae_results[n_method, nrep * nbags + n_bag]) file_all.write('\n') file_all.close() name_file = 'results-all-mrae-artificial-' + str(dim) + 'D-' + str(param) + '-' + est_name + \ '-rep' + str(nreps) + '-value' + str(ntrain[k]) + '-ntest' + str(ntest) + '.txt' file_all = open(name_file, 'w') for method_name in methods_names: file_all.write('%s,' % method_name) file_all.write('\n') for nrep in range(nreps): for n_bag in range(nbags): for n_method in range(len(methods_names)): file_all.write('%.5f, ' % all_mrae_results[n_method, nrep * nbags + n_bag]) file_all.write('\n') file_all.close() name_file = 'results-all-sqe-artificial-' + str(dim) + 'D-' + str(param) + '-' + est_name + \ '-rep' + str(nreps) + '-value' + str(ntrain[k]) + '-ntest' + str(ntest) + '.txt' file_all = open(name_file, 'w') for method_name in methods_names: file_all.write('%s,' % method_name) file_all.write('\n') for nrep in range(nreps): for n_bag in range(nbags): for n_method in range(len(methods_names)): file_all.write('%.5f, ' % all_sqe_results[n_method, nrep * nbags + n_bag]) file_all.write('\n') file_all.close() file_times.close() mae_results = mae_results / (nreps * nbags) mrae_results = mrae_results / (nreps * nbags) sqe_results = sqe_results / (nreps * nbags) classif_results = classif_results / (nreps * nbags) name_file = 'results-avg-artificial-' + str(dim) + 'D-' + str(param) + '-' + est_name + '-rep' + str(nreps) + \ '-ntest' + str(ntest) + '.txt' file_avg = open(name_file, 'w') file_avg.write('MAE\n') file_avg.write('#examples, Error, ') for index, m in enumerate(methods_names): file_avg.write('%s, ' % m) file_avg.write('BrierLoss') for index, number in enumerate(ntrain): file_avg.write('\n%d, ' % number) # Error file_avg.write('%.5f, ' % classif_results[0, index]) for i in mae_results[:, index]: file_avg.write('%.5f, ' % i) # Brier loss file_avg.write('%.5f' % classif_results[1, index]) file_avg.write('\n\nMRAE\n') file_avg.write('#examples, Error, ') for index, m in enumerate(methods_names): file_avg.write('%s, ' % m) file_avg.write('BrierLoss') for index, number in enumerate(ntrain): file_avg.write('\n%d, ' % number) # Error file_avg.write('%.5f, ' % classif_results[0, index]) for i in mrae_results[:, index]: file_avg.write('%.5f, ' % i) # Brier loss file_avg.write('%.5f' % classif_results[1, index]) file_avg.write('\n\nSQE\n') file_avg.write('#examples, Error, ') for index, m in enumerate(methods_names): file_avg.write('%s, ' % m) file_avg.write('BrierLoss') for index, number in enumerate(ntrain): file_avg.write('\n%d, ' % number) # Error file_avg.write('%.5f, ' % classif_results[0, index]) for i in sqe_results[:, index]: file_avg.write('%.5f, ' % i) # Brier loss file_avg.write('%.5f' % classif_results[1, index]) file_avg.close()