def test_squared_errors_3(self):
"""
Tupla obtenida de los test anteriores con un error relativamente bajo ( < 2 )
"""
weights = [
1.0, 0.9999999898725264, -1.0, 0.9999999941226553,
0.7789066176302455, 0.9999999976272673, 0.7685086147538794, 1.0,
0.7789465050493634, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0
]
moderate_constant = 0.01
iterations = 30
errors = []
for i in range(iterations):
board = experiment_generator()
print('Obteniendo traza del juego...')
game_trace = get_game_trace_with_random_player(board, weights)
print(f'Se obtuvieron {game_trace.__len__()} tuplas')
training_examples = get_training_examples(game_trace, weights)
print('Ajustando pesos...')
weights = gen(training_examples, weights, moderate_constant)
errors.append(squared_error(training_examples, weights))
print(f'Pesos obtenidos : {weights}')
print(f'Error cuadratico : {errors[-1]}')
for i in range(iterations):
print('Error {}: {}'.format(i, errors[i]))
for i in range(iterations - 1):
self.assertGreaterEqual(
errors[i], errors[i + 1],
f'El error {i} no es mayor o igual que el error {i+1}, los errores deben decrecer'
)
def fit(self, X, Y):
self.X = X.copy()
self.Y = Y.copy()
self.N = X.shape[0]
self.n_input = X.shape[1]
self.n_output = Y.shape[1]
# ReLU init
self.w0 = np.random.rand(self.n_input, self.n_neurons) * np.sqrt(
2 / self.n_input)
self.w1 = np.random.rand(self.n_neurons, self.n_output) * np.sqrt(
2 / self.n_neurons)
self.b0 = np.random.rand() * np.sqrt(2 / self.n_input)
self.b1 = np.random.rand() * np.sqrt(2 / self.n_input)
self.layer0 = np.zeros(self.n_input)
self.layer1 = np.zeros(self.n_neurons)
self.layer2 = np.zeros(self.n_output)
last_sse = np.inf
for i in range(self.max_iter):
sse = 0
for j in range(X.shape[0]):
o = self.__forward_propagate(self.X[j])
self.__back_propagate(self.Y[j])
sse += utils.squared_error(o, self.Y[j])
print(i, ' ', sse)
if (last_sse - sse) < self.tol:
break
last_sse = sse
def train_3(weights, moderate_constant, max_error):
board = experiment_generator()
game_trace = get_game_trace_with_old_vesion(board, weights, weights)
training_examples = get_training_examples(game_trace, weights)
weights = gen(training_examples, weights, moderate_constant)
error = squared_error(training_examples, weights)
return weights, error, error <= max_error
def test_squared_errors_2(self):
weights = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
moderate_constant = 0.5
iterations = 5
errors = []
for i in range(iterations):
board = experiment_generator()
print('Obteniendo traza del juego...')
game_trace = get_game_trace_with_random_player(board, weights)
print(f'Se obtuvieron {game_trace.__len__()} tuplas')
training_examples = get_training_examples(game_trace, weights)
print('Ajustando pesos...')
weights = gen(training_examples, weights, moderate_constant)
errors.append(squared_error(training_examples, weights))
print(f'Pesos obtenidos : {weights}')
print(f'Error cuadratico : {errors[-1]}')
for i in range(iterations):
print('Error {}: {}'.format(i, errors[i]))
for i in range(iterations - 1):
self.assertGreaterEqual(
errors[i], errors[i + 1],
f'El error {i} no es mayor o igual que el error {i+1}, los errores deben decrecer'
)
def test_squared_errors9(self):
logging.basicConfig(filename='./logs/test_with_random_player_mu09.log',
level=logging.INFO)
logging.info(
'------------------------------------------------------------------------------------------------'
)
logging.info('Started')
moderate_constant = 0.9
iterations = 50
weights = [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
errors = []
won = 0
lost = 0
for i in range(iterations):
board = experiment_generator()
logging.info('Obteniendo traza del juego...')
game_trace = get_game_trace_with_random_player(board, weights)
logging.info(f'Se obtuvieron {game_trace.__len__()} tuplas')
training_examples = get_training_examples(game_trace, weights)
board_features = training_examples[training_examples.__len__() -
1][0]
logging.info(training_examples[training_examples.__len__() - 1])
if board_features[6] >= 1:
won = won + 1
elif board_features[13] >= 1:
lost = lost + 1
logging.info('Ajustando pesos...')
weights = gen(training_examples, weights, moderate_constant)
errors.append(squared_error(training_examples, weights))
logging.info(weights)
# if errors[i -1] < errors[i]:
# moderate_constant = max(0.1, moderate_constant - 0.1)
# logging.info(moderate_constant)
for i in range(iterations):
logging.info('Error {}: {}'.format(i, errors[i]))
# for i in range(iterations-1):
# self.assertGreaterEqual(errors[i], errors[i+1],
# f'El error {i} no es mayor o igual que el error {i+1}, los errores deben decrecer')
logging.info(f'Ganados: {won}')
logging.info(f'Perdidos: {lost}')
logging.info('Finished')
logging.info(
'------------------------------------------------------------------------------------------------'
)
return weights
def inf_g(self, p_x, g_gen, x, locations):
# Infer abstract location from the combination of [grounded location retrieved from memory by sensory experience] ...
if self.hyper['use_p_inf']:
# Not in paper, but makes sense from symmetry with f_x: first get g from p by "summing over sensory preferences" g = p * W_repeat^T
g_downsampled = [torch.matmul(p_x[f], torch.t(self.hyper['W_repeat'][f])) for f in range(self.hyper['n_f'])]
# Then use abstract location after summing over sensory preferences as input to MLP to obtain the inferred abstract location from memory
mu_g_mem = self.f_mu_g_mem(g_downsampled)
# Not in paper, but this greatly improves zero-shot inference: provide the uncertainty function of the inferred abstract location with measures of memory quality
with torch.no_grad():
# For the first measure, use the grounded location inferred from memory to generate an observation
x_hat, x_hat_logits = self.gen_x(p_x[0])
# Then calculate the error between the generated observation and the actual observation: if the memory is working well, this error should be small
err = utils.squared_error(x, x_hat)
# The second measure is the vector norm of the inferred abstract location; good memories should have similar vector norms. Concatenate the two measures as input for the abstract location uncertainty function
sigma_g_input = [torch.cat((torch.sum(g ** 2, dim=1, keepdim=True), torch.unsqueeze(err, dim=1)), dim=1) for g in mu_g_mem]
# Not in paper, but recommended by James for stability: get final mean of inferred abstract location by clamping activations between -1 and 1
mu_g_mem = self.f_g_clamp(mu_g_mem)
# And get standard deviation/uncertainty of inferred abstract location by providing uncertainty function with memory quality measures
sigma_g_mem = self.f_sigma_g_mem(sigma_g_input)
# ... and [previous abstract location and action (path integration)]
mu_g_path = g_gen[0]
sigma_g_path = g_gen[1]
# Infer abstract location by combining previous abstract location and grounded location retrieved from memory by current sensory experience
mu_g, sigma_g = [], []
for f in range(self.hyper['n_f']):
if self.hyper['use_p_inf']:
# Then get full gaussian distribution of inferred abstract location by calculating precision weighted mean
mu, sigma = utils.inv_var_weight([mu_g_path[f], mu_g_mem[f]],[sigma_g_path[f], sigma_g_mem[f]])
else:
# Or simply completely ignore the inference memory here, to test if things are working
mu, sigma = mu_g_path[f], sigma_g_path[f]
# Append mu and sigma to list for all frequency modules
mu_g.append(mu)
sigma_g.append(sigma)
# Finally (though not in paper), also add object vector cell information to inferred abstract location for environments with shiny objects
shiny_envs = [location['shiny'] is not None for location in locations]
if any(shiny_envs):
# Find for which environments the current location has a shiny object
shiny_locations = torch.unsqueeze(torch.stack([torch.tensor(location['shiny'], dtype=torch.float) for location in locations if location['shiny'] is not None]), dim=-1)
# Get abstract location for environments with shiny objects and feed to each of the object vector cell modules
mu_g_shiny = self.f_mu_g_shiny([shiny_locations for _ in range(self.hyper['n_f_g'] if self.hyper['separate_ovc'] else self.hyper['n_f'])])
sigma_g_shiny = self.f_sigma_g_shiny([shiny_locations for _ in range(self.hyper['n_f_g'] if self.hyper['separate_ovc'] else self.hyper['n_f'])])
# Update only object vector modules with shiny-inferred abstract location: start from offset if object vector modules are separate
module_start = self.hyper['n_f_g'] if self.hyper['separate_ovc'] else 0
# Inverse variance weighting is associative, so I can just do additional inverse variance weighting to the previously obtained mu and sigma - but only for object vector cell modules!
for f in range(module_start, self.hyper['n_f']):
# Add inferred abstract location from shiny objects to previously obtained position, only for environments with shiny objects
mu, sigma = utils.inv_var_weight([mu_g[f][shiny_envs,:], mu_g_shiny[f - module_start]], [sigma_g[f][shiny_envs,:], sigma_g_shiny[f - module_start]])
# In order to update only the environments with shiny objects, without in-place value assignment, construct a mask of shiny environments
mask = torch.zeros_like(mu_g[f], dtype=torch.bool)
mask[shiny_envs,:] = True
# Use mask to update the shiny environment entries in inferred abstract locations
mu_g[f] = mu_g[f].masked_scatter(mask,mu)
sigma_g[f] = sigma_g[f].masked_scatter(mask,sigma)
# Either sample inferred abstract location from combined (precision weighted) distribution or just take mean
g = [mu_g[f] + sigma_g[f] * np.random.randn() if self.hyper['do_sample'] else mu_g[f] for f in range(self.hyper['n_f'])]
# Return abstract location inferred from grounded location from memory and previous abstract location
return g
def test_squared_errors_4(self):
"""
Probar : a mayor cantidad de iteraciones, menor constante de entrenamiento
"""
weights = [0, 1, -1, 2, -1, 3, -1, 2, -1, 1, -1, 1, -1, 1, -2]
moderate_constant = 0.001
iterations = 100
errors = []
board = experiment_generator()
game_trace = get_game_trace_with_random_player(board, weights)
training_examples = get_training_examples(game_trace, weights)
weights = gen(training_examples, weights, moderate_constant)
best_weights = weights
best_game_trace = game_trace
error = squared_error(training_examples, best_weights)
errors.append(error)
for i in range(iterations - 1):
board = experiment_generator()
game_trace = get_game_trace_with_random_player(board, best_weights)
training_examples = get_training_examples(game_trace, best_weights)
weights = gen(training_examples, best_weights, moderate_constant)
error = squared_error(training_examples, best_weights)
print(f'Error : {error}')
# Solo actualizo los pesos si la cantidad de jugadas realizadas para ganar
# es menor igual que lo que se tiene hasta el momento como minimo.
if game_trace.__len__() <= best_game_trace.__len__():
best_weights = weights
best_game_trace = game_trace
error = squared_error(training_examples, best_weights)
errors.append(error)
print(
f'Tuplas obtenidas en traza de juego : {best_game_trace.__len__()}'
)
print(f'Error cuadratico : {error}')
for i in range(errors.__len__()):
print('Error {}: {}'.format(i, errors[i]))
print(f'Pesos obtenidos: {best_weights}')
def predict(self, X):
y = np.zeros(X.shape[0], dtype=int)
for i in range(X.shape[0]):
squared_errors = np.zeros(self.n_clusters)
for j in range(self.n_clusters):
squared_errors[j] = utils.squared_error(
X[i], self.cluster_centers_[j])
y[i] = np.argmin(squared_errors)
return y
def orchestrate(self):
training_examples = get_training_examples(self.game_trace,
self.weights)
self.weights = gen(training_examples, self.weights,
self.moderate_constant)
self.error = squared_error(training_examples, self.weights)
for te in training_examples:
print(te)
print(f'Moderate constant: {self.moderate_constant}')
print(f'Game trace: {self.game_trace}')
print(f'Pesos obtenidos: {self.weights}')
print(f'Error cuadratico: {self.error}')
def loss(self, g_gen, p_gen, x_logits, x, g_inf, p_inf, p_inf_x, M_prev):
# Calculate loss function, separately for each component because you might want to reweight contributions later
# L_p_gen is squared error loss between inferred grounded location and grounded location retrieved from inferred abstract location
L_p_g = torch.sum(torch.stack(utils.squared_error(p_inf, p_gen), dim=0), dim=0)
# L_p_inf is squared error loss between inferred grounded location and grounded location retrieved from sensory experience
L_p_x = torch.sum(torch.stack(utils.squared_error(p_inf, p_inf_x), dim=0), dim=0) if self.hyper['use_p_inf'] else torch.zeros_like(L_p_g)
# L_g is squared error loss between generated abstract location and inferred abstract location
L_g = torch.sum(torch.stack(utils.squared_error(g_inf, g_gen), dim=0), dim=0)
# L_x is a cross-entropy loss between sensory experience and different model predictions. First get true labels from sensory experience
labels = torch.argmax(x, 1)
# L_x_gen: losses generated by generative model from g_prev -> g -> p -> x
L_x_gen = utils.cross_entropy(x_logits[2], labels)
# L_x_g: Losses generated by generative model from g_inf -> p -> x
L_x_g = utils.cross_entropy(x_logits[1], labels)
# L_x_p: Losses generated by generative model from p_inf -> x
L_x_p = utils.cross_entropy(x_logits[0], labels)
# L_reg are regularisation losses, L_reg_g on L2 norm of g
L_reg_g = torch.sum(torch.stack([torch.sum(g ** 2, dim=1) for g in g_inf], dim=0), dim=0)
# And L_reg_p regularisation on L1 norm of p
L_reg_p = torch.sum(torch.stack([torch.sum(torch.abs(p), dim=1) for p in p_inf], dim=0), dim=0)
# Return total loss as list of losses, so you can possibly reweight them
L = [L_p_g, L_p_x, L_x_gen, L_x_g, L_x_p, L_g, L_reg_g, L_reg_p]
return L
def run_experiment(est_name, seed, dim, param, ntrain, ntest, nreps, nbags, nfolds, save_all):
""" Run a single experiment
Parameters
----------
est_name: str
Name of the estimator. 'LR' or 'SVM-RBF'
seed: int
Seed of the experiment
dim: int
Dimension of the dataset, 1 or 2
param: int, str
Extra oarameter for the definition of the problem.
If dim==1, this value is the std.
If dim=2 this value is an string to indicate if the dataset is the one designed to test HDX
ntrain : list
List with the number of training examples that must be tested, e.g.,[50, 100, 200]
ntest: int
Number of testing instances in each bag
nreps: int
Number of training datasets created
nbags: int
Number of testing bags created for each training datasets.
The total number of experiments will be nreps * nbags
nfolds: int
Number of folds used to estimate the training distributions by the methods AC, HDy and EDy
save_all: bool
True if the results of each single experiment must be saved
"""
# range of testing prevalences
low = round(ntest * 0.05)
high = round(ntest * 0.95)
if est_name == 'LR':
estimator = LogisticRegression(C=1, random_state=seed, max_iter=10000, solver='liblinear')
else:
estimator = SVC(C=1, kernel='rbf', random_state=seed, max_iter=10000, gamma=0.2, probability=True)
rng = np.random.RandomState(seed)
# methods
cc = CC()
ac = AC()
pac = PAC()
sordy = SORDy()
# hdys
pdfy_hd_4 = DFy(distribution_function='PDF', n_bins=4, distance='HD')
pdfy_hd_8 = DFy(distribution_function='PDF', n_bins=8, distance='HD')
pdfy_hd_12 = DFy(distribution_function='PDF', n_bins=12, distance='HD')
pdfy_hd_16 = DFy(distribution_function='PDF', n_bins=16, distance='HD')
# cdf l1
cdfy_l1_8 = DFy(distribution_function='CDF', n_bins=8, distance='L1')
cdfy_l1_16 = DFy(distribution_function='CDF', n_bins=16, distance='L1')
cdfy_l1_32 = DFy(distribution_function='CDF', n_bins=32, distance='L1')
cdfy_l1_64 = DFy(distribution_function='CDF', n_bins=64, distance='L1')
# cdf l1
# cdfy_l2_8 = DFy(distribution_function='CDF', n_bins=8, distance='L2')
# cdfy_l2_16 = DFy(distribution_function='CDF', n_bins=16, distance='L2')
# cdfy_l2_32 = DFy(distribution_function='CDF', n_bins=32, distance='L2')
# cdfy_l2_64 = DFy(distribution_function='CDF', n_bins=64, distance='L2')
# QUANTy-L1
# quanty_l1_4 = QUANTy(n_quantiles=4, distance=l1)
# quanty_l1_10 = QUANTy(n_quantiles=10, distance=l1)
# quanty_l1_20 = QUANTy(n_quantiles=20, distance=l1)
# quanty_l1_40 = QUANTy(n_quantiles=40, distance=l1)
# QUANTy-L2
quanty_l2_4 = QUANTy(n_quantiles=4, distance=l2)
quanty_l2_10 = QUANTy(n_quantiles=10, distance=l2)
quanty_l2_20 = QUANTy(n_quantiles=20, distance=l2)
quanty_l2_40 = QUANTy(n_quantiles=40, distance=l2)
# methods
methods = [cc, ac, pac, sordy,
pdfy_hd_4, pdfy_hd_8, pdfy_hd_12, pdfy_hd_16,
cdfy_l1_8, cdfy_l1_16, cdfy_l1_32, cdfy_l1_64,
# cdfy_l2_8, cdfy_l2_16, cdfy_l2_32, cdfy_l2_64,
# quanty_hd_4, quanty_hd_8, quanty_hd_12, quanty_hd_16,
# quanty_l1_4, quanty_l1_8, quanty_l1_12, quanty_l1_16,
quanty_l2_4, quanty_l2_10, quanty_l2_20, quanty_l2_40]
methods_names = ['CC', 'AC', 'PAC', 'SORDy',
'PDFy_hd_4', 'PDFy_hd_8', 'PDFy_hd_12', 'PDFy_hd_16',
'CDFy_l1_8', 'CDFy_l1_16', 'CDFy_l1_32', 'CDFy_l1_64',
# 'CDFy_l2_8', 'CDFy_l2_16', 'CDFy_l2_32', 'CDFy_l2_64',
# 'QUANTy_l1_4', 'QUANTy_l1_10', 'QUANTy_l1_20', 'QUANTy_l1_40',
'QUANTy_l2_4', 'QUANTy_l2_10', 'QUANTy_l2_20', 'QUANTy_l2_40']
# to store the results
mae_results = np.zeros((len(methods_names), len(ntrain)))
sqe_results = np.zeros((len(methods_names), len(ntrain)))
mrae_results = np.zeros((len(methods_names), len(ntrain)))
classif_results = np.zeros((2, len(ntrain)))
std1 = std2 = mu3 = mu4 = cov1 = cov2 = cov3 = cov4 = 0
if dim == 1:
# 1D
mu1 = -1
std1 = param
mu2 = 1
std2 = std1
else:
# 2D
mu1 = [-1.00, 1.00]
mu2 = [1.00, 1.00]
mu3 = [1.00, -1.00]
cov1 = [[0.4, 0],
[0, 0.4]]
cov2 = cov1
cov3 = cov1
x1 = rng.multivariate_normal(mu1, cov1, 400)
x3 = rng.multivariate_normal(mu3, cov3, 400)
plt.scatter(np.vstack((x1[:, 0], x3[:, 0])), np.vstack((x1[:, 1], x3[:, 1])), c='r', marker='+', s=12,
label='Class \u2212' + '1')
if param == 'HDX':
mu4 = [-1.00, -1.00]
cov4 = cov1
x2 = rng.multivariate_normal(mu2, cov2, 400)
x4 = rng.multivariate_normal(mu4, cov4, 400)
plt.scatter(np.vstack((x2[:, 0], x4[:, 0])), np.vstack((x2[:, 1], x4[:, 1])),
c='b', marker='x', s=8, label='Class +1')
else:
x2 = rng.multivariate_normal(mu2, cov2, 800)
plt.scatter(x2[:, 0], x2[:, 1], c='b', marker='x', s=8, label='Class +1')
plt.xlabel('$x_1$')
plt.ylabel('$x_2$')
plt.legend(loc='best')
plt.savefig('./artificial-2D-' + param + '.png', dpi=300)
name_file = 'times-avg-artificial-' + str(dim) + 'D-' + str(param) + '-' + est_name + '-rep' + str(nreps) + \
'-ntest' + str(ntest) + '.txt'
file_times = open(name_file, 'w')
file_times.write('#examples, ')
for index, m in enumerate(methods_names):
file_times.write('%s, ' % m)
for k in range(len(ntrain)):
all_mae_results = np.zeros((len(methods_names), nreps * nbags))
all_sqe_results = np.zeros((len(methods_names), nreps * nbags))
all_mrae_results = np.zeros((len(methods_names), nreps * nbags))
execution_times = np.zeros(len(methods_names))
print()
print('#Training examples ', ntrain[k], 'Rep#', end=' ')
for rep in range(nreps):
print(rep+1, end=' ')
if dim == 1:
x_train = np.vstack(((std1 * rng.randn(ntrain[k], 1) + mu1), (std2 * rng.randn(ntrain[k], 1) + mu2)))
else:
if param == 'HDX':
x_train = np.vstack((rng.multivariate_normal(mu1, cov1, ntrain[k] // 2),
rng.multivariate_normal(mu3, cov3, ntrain[k] - ntrain[k] // 2),
rng.multivariate_normal(mu2, cov2, ntrain[k] // 2),
rng.multivariate_normal(mu4, cov4, ntrain[k] - ntrain[k] // 2)))
else:
x_train = np.vstack((rng.multivariate_normal(mu1, cov1, ntrain[k] // 2),
rng.multivariate_normal(mu3, cov3, ntrain[k] - ntrain[k] // 2),
rng.multivariate_normal(mu2, cov2, ntrain[k])))
y_train = np.hstack((np.zeros(ntrain[k], dtype=int), np.ones(ntrain[k], dtype=int)))
skf_train = StratifiedKFold(n_splits=nfolds, shuffle=True, random_state=seed + rep)
estimator_train = CV_estimator(estimator=estimator, n_jobs=None, cv=skf_train)
estimator_train.fit(x_train, y_train)
predictions_train = estimator_train.predict_proba(x_train)
for nmethod, method in enumerate(methods):
if isinstance(method, UsingClassifiers):
method.fit(X=x_train, y=y_train, predictions_train=predictions_train)
else:
method.fit(X=x_train, y=y_train)
#estimator_test = estimator
#estimator_test.fit(x_train, y_train)
estimator_test = estimator_train
for n_bag in range(nbags):
ps = rng.randint(low, high, 1)
ps = np.append(ps, [0, ntest])
ps = np.diff(np.sort(ps))
if dim == 1:
x_test = np.vstack(((std1 * rng.randn(ps[0], 1) + mu1), (std2 * rng.randn(ps[1], 1) + mu2)))
else:
if param == 'HDX':
x_test = np.vstack((rng.multivariate_normal(mu1, cov1, ps[0] // 2),
rng.multivariate_normal(mu3, cov3, ps[0] - ps[0] // 2),
rng.multivariate_normal(mu2, cov2, ps[1] // 2),
rng.multivariate_normal(mu4, cov4, ps[1] - ps[1] // 2)))
else:
x_test = np.vstack((rng.multivariate_normal(mu1, cov1, ps[0] // 2),
rng.multivariate_normal(mu3, cov3, ps[0] - ps[0] // 2),
rng.multivariate_normal(mu2, cov2, ps[1])))
y_test = np.hstack((np.zeros(ps[0], dtype=int), np.ones(ps[1], dtype=int)))
predictions_test = estimator_test.predict_proba(x_test)
# Error
classif_results[0, k] = classif_results[0, k] + zero_one_loss(np.array(y_test),
np.argmax(predictions_test, axis=1))
# Brier loss
classif_results[1, k] = classif_results[1, k] + brier_score_loss(indices_to_one_hot(y_test, 2)[:, 0],
predictions_test[:, 0])
prev_true = ps[1] / ntest
for nmethod, method in enumerate(methods):
t = time.process_time()
if isinstance(method, UsingClassifiers):
p_predicted = method.predict(X=None, predictions_test=predictions_test)[1]
else:
p_predicted = method.predict(X=x_test)[1]
elapsed_time = time.process_time()
execution_times[nmethod] = execution_times[nmethod] + elapsed_time - t
all_mae_results[nmethod, rep * nbags + n_bag] = absolute_error(prev_true, p_predicted)
all_mrae_results[nmethod, rep * nbags + n_bag] = relative_absolute_error(prev_true, p_predicted)
all_sqe_results[nmethod, rep * nbags + n_bag] = squared_error(prev_true, p_predicted)
mae_results[nmethod, k] = mae_results[nmethod, k] + all_mae_results[nmethod, rep * nbags + n_bag]
mrae_results[nmethod, k] = mrae_results[nmethod, k] + all_mrae_results[nmethod, rep * nbags + n_bag]
sqe_results[nmethod, k] = sqe_results[nmethod, k] + all_sqe_results[nmethod, rep * nbags + n_bag]
execution_times = execution_times / (nreps * nbags)
file_times.write('\n%d, ' % ntrain[k])
for i in execution_times:
file_times.write('%.5f, ' % i)
if save_all:
name_file = 'results-all-mae-artificial-' + str(dim) + 'D-' + str(param) + '-' + est_name + \
'-rep' + str(nreps) + '-value' + str(ntrain[k]) + '-ntest' + str(ntest) + '.txt'
file_all = open(name_file, 'w')
for method_name in methods_names:
file_all.write('%s,' % method_name)
file_all.write('\n')
for nrep in range(nreps):
for n_bag in range(nbags):
for n_method in range(len(methods_names)):
file_all.write('%.5f, ' % all_mae_results[n_method, nrep * nbags + n_bag])
file_all.write('\n')
file_all.close()
name_file = 'results-all-mrae-artificial-' + str(dim) + 'D-' + str(param) + '-' + est_name + \
'-rep' + str(nreps) + '-value' + str(ntrain[k]) + '-ntest' + str(ntest) + '.txt'
file_all = open(name_file, 'w')
for method_name in methods_names:
file_all.write('%s,' % method_name)
file_all.write('\n')
for nrep in range(nreps):
for n_bag in range(nbags):
for n_method in range(len(methods_names)):
file_all.write('%.5f, ' % all_mrae_results[n_method, nrep * nbags + n_bag])
file_all.write('\n')
file_all.close()
name_file = 'results-all-sqe-artificial-' + str(dim) + 'D-' + str(param) + '-' + est_name + \
'-rep' + str(nreps) + '-value' + str(ntrain[k]) + '-ntest' + str(ntest) + '.txt'
file_all = open(name_file, 'w')
for method_name in methods_names:
file_all.write('%s,' % method_name)
file_all.write('\n')
for nrep in range(nreps):
for n_bag in range(nbags):
for n_method in range(len(methods_names)):
file_all.write('%.5f, ' % all_sqe_results[n_method, nrep * nbags + n_bag])
file_all.write('\n')
file_all.close()
file_times.close()
mae_results = mae_results / (nreps * nbags)
mrae_results = mrae_results / (nreps * nbags)
sqe_results = sqe_results / (nreps * nbags)
classif_results = classif_results / (nreps * nbags)
name_file = 'results-avg-artificial-' + str(dim) + 'D-' + str(param) + '-' + est_name + '-rep' + str(nreps) + \
'-ntest' + str(ntest) + '.txt'
file_avg = open(name_file, 'w')
file_avg.write('MAE\n')
file_avg.write('#examples, Error, ')
for index, m in enumerate(methods_names):
file_avg.write('%s, ' % m)
file_avg.write('BrierLoss')
for index, number in enumerate(ntrain):
file_avg.write('\n%d, ' % number)
# Error
file_avg.write('%.5f, ' % classif_results[0, index])
for i in mae_results[:, index]:
file_avg.write('%.5f, ' % i)
# Brier loss
file_avg.write('%.5f' % classif_results[1, index])
file_avg.write('\n\nMRAE\n')
file_avg.write('#examples, Error, ')
for index, m in enumerate(methods_names):
file_avg.write('%s, ' % m)
file_avg.write('BrierLoss')
for index, number in enumerate(ntrain):
file_avg.write('\n%d, ' % number)
# Error
file_avg.write('%.5f, ' % classif_results[0, index])
for i in mrae_results[:, index]:
file_avg.write('%.5f, ' % i)
# Brier loss
file_avg.write('%.5f' % classif_results[1, index])
file_avg.write('\n\nSQE\n')
file_avg.write('#examples, Error, ')
for index, m in enumerate(methods_names):
file_avg.write('%s, ' % m)
file_avg.write('BrierLoss')
for index, number in enumerate(ntrain):
file_avg.write('\n%d, ' % number)
# Error
file_avg.write('%.5f, ' % classif_results[0, index])
for i in sqe_results[:, index]:
file_avg.write('%.5f, ' % i)
# Brier loss
file_avg.write('%.5f' % classif_results[1, index])
file_avg.close()
