def test_read(self):
with patch('os.path.exists', new=MagicMock(return_value=False)):
assert JSONFile.read(self.filename) is None
with patch('os.path.exists', new=MagicMock(return_value=True)):
with patch('__builtin__.open', mock_open(read_data='[]')):
assert JSONFile.read(self.filename) == []
def top_users_papers_selecting_categories(cls,
year,
month,
top_categories=10,
different_papers=20):
"""
Selects only users and papers in the top_categories based on the data generated by the
previous function.
:param year:
:param month:
:param top_categories:
:param different_papers
:return: [ {'paper': (int) number of times seen},
{'user': {'stats': ((int) # entries, (int) # different papers in the top_n papers),
'diff_papers': [str]
}
}
]
"""
categories = JSONFile.read(
cls._name_file_categories(year=year, month=month))
papers_cat = pd.DataFrame.from_records([categories]).transpose()
users_cg, user_cat = cls.assign_categories_to_users(year, month)
user_cat = pd.DataFrame.from_records([user_cat]).transpose()
pap_1 = set(user_cat[0].sort_values().index.values[-top_categories:])
pap_2 = set(papers_cat[0].value_counts().sort_values().index.
values[-top_categories:])
top_cat = pap_1.intersection(pap_2)
full_data = JSONFile.read(cls._name_file_final(year=year, month=month))
papers_or = full_data[0]
papers_new = {}
for paper in papers_or:
cat = categories[paper]
if cat in top_cat:
papers_new[paper] = papers_or[paper]
users_new = {}
for user in full_data[1]:
paper_user = []
for paper in full_data[1][user]['diff_papers']:
cat = categories[paper]
if cat in top_cat:
paper_user.append(paper)
if len(paper_user) > different_papers:
users_new[user] = full_data[1][user]
users_new[user]['diff_papers'] = paper_user
file_name = cls._name_file_final_categ(year=year, month=month)
JSONFile.write([papers_new, users_new], file_name)
logger.info('Number of papers is %d' % len(papers_new))
logger.info('Number of users is %d' % len(users_new))
return [papers_new, users_new]
def plot_histograms_papers_categories(cls, year, month):
data = JSONFile.read(cls._name_file_categories(year=year, month=month))
df = pd.DataFrame.from_records([data])
df = df.transpose()
hist = df[0].value_counts().plot(kind='bar')
fig = hist.get_figure()
hist_file = cls._histogram_papers(year=year, month=month)
fig.savefig(hist_file)
def set_data_from_file(self):
data = JSONFile.read(self.file_path)
if data is None:
return
self.evaluated_points = data['evaluated_points']
self.objective_values = data['objective_values']
self.model_objective_values = data['model_objective_values']
self.standard_deviation_evaluations = data[
'standard_deviation_evaluations']
def assign_categories_date_year(cls, year, month):
"""
:param year: (str)
:param month: (str) e.g. '1', '12'
:return:
"""
file_name = cls._name_file_final(year=year, month=month)
data = JSONFile.read(file_name)
papers = data[0].keys()
papers = cls.assign_categories(papers, year, month)
return papers
def get_training_data(cls, year, month, random_seed=1):
"""
Creates a file with the training data:
[[user_id, paper_id, rating]], where rating is 1 if the paper wasn't seen by the user,
or 2 otherwise.
:param year: str
:param month: str (e.g. '1', '12')
:param random_seed: int
"""
random.seed(random_seed)
file_name = cls._name_file_final_categ(year=year, month=month)
data = JSONFile.read(file_name)
papers = data[0].keys()
users_data = data[1]
users = users_data.keys()
training_data = []
key_paper = {}
for i, paper in enumerate(papers):
key_paper[paper] = i + 1
for i, user in enumerate(users):
for paper in users_data[user]['diff_papers']:
training_data.append([i + 1, key_paper[paper], 2])
other_papers = list(
set(papers) - set(users_data[user]['diff_papers']))
index_papers = range(len(other_papers))
random.shuffle(index_papers)
seen_papers = len(set(users_data[user]['diff_papers']))
dislike_papers = np.random.randint(
int(0.5 * seen_papers),
min(int(1.8 * seen_papers), len(index_papers)), 1)
index = dislike_papers[0]
keep_index_papers = index_papers[0:index]
for index in keep_index_papers:
training_data.append(
[i + 1, key_paper[other_papers[index]], 1])
file_name = cls._name_training_data(year=year, month=month)
logger.info('There are %d training points' % len(training_data))
JSONFile.write(training_data, file_name)
def assign_categories_to_users(cls, year, month):
file_name = cls._name_file_final(year=year, month=month)
full_data = JSONFile.read(file_name)
users = full_data[1]
paper_cat = JSONFile.read(
cls._name_file_categories(year=year, month=month))
users_cg = {}
for user in users:
diff_papers = users[user]['diff_papers']
papers_cat = []
for paper in diff_papers:
papers_cat.append(paper_cat[paper])
users_cg[user] = papers_cat
JSONFile.write(users_cg,
cls._name_file_categories_users(year=year, month=month))
user_cat = {}
for user in users_cg:
papers = users_cg[user]
cat_us = {}
for cat in papers:
if cat not in cat_us:
cat_us[cat] = 0
cat_us[cat] += 1
for cat in cat_us:
if cat_us[cat] >= 0.10 * len(papers):
if cat not in user_cat:
user_cat[cat] = 0
user_cat[cat] += 1
JSONFile.write(
user_cat,
cls._name_file_categories_users_hist(year=year, month=month))
return users_cg, user_cat
def get_points_domain(cls,
n_training,
bounds_domain,
random_seed,
training_name,
problem_name,
type_bounds=None,
simplex_domain=None):
"""
Get random points in the domain.
:param n_training: (int) Number of points
:param bounds_domain: [([float, float] or [float])], the first case is when the bounds are
lower or upper bound of the respective entry; in the second case, it's list of finite
points representing the domain of that entry.
:param random_seed: (int)
:param training_name: (str), prefix used to save the training data.
:param problem_name: str
:param type_bounds: [0 or 1], 0 if the bounds are lower or upper bound of the respective
entry, 1 if the bounds are all the finite options for that entry.
:return: [[float]]
"""
file_name = cls._filename_domain(
problem_name=problem_name,
training_name=training_name,
n_points=n_training,
random_seed=random_seed,
)
training_dir = path.join(PROBLEM_DIR, problem_name, 'data')
training_path = path.join(training_dir, file_name)
points = JSONFile.read(training_path)
if points is not None:
return points
points = DomainService.get_points_domain(n_training,
bounds_domain,
type_bounds=type_bounds,
random_seed=random_seed,
simplex_domain=simplex_domain)
print(points)
JSONFile.write(points, training_path)
return points
def load_discretization(cls, problem_name, bounds_domain_x,
number_points_each_dimension_x):
"""
Try to load discretization for problem_name from file. If the file doesn't exist, will
generate the discretization and store it.
:param problem_name: (str)
:param bounds_domain_x: ([BoundsEntity])
:param number_points_each_dimension_x: ([int])
:return: [[float]]
"""
bounds_str = BoundsEntity.get_bounds_as_lists(bounds_domain_x)
filename = cls._disc_x_filename(
name=problem_name,
bounds=bounds_str,
number_points_each_dimension=number_points_each_dimension_x)
if not os.path.exists(path.join(PROBLEM_DIR, problem_name)):
os.mkdir(path.join(PROBLEM_DIR, problem_name))
domain_dir = path.join(PROBLEM_DIR, problem_name, DOMAIN_DIR)
if not os.path.exists(domain_dir):
os.mkdir(domain_dir)
domain_path = path.join(domain_dir, filename)
discretization_data = JSONFile.read(domain_path)
if discretization_data is not None:
return discretization_data
logger.info('Gnerating discretization of domain_x')
discretization_data = DomainEntity.discretize_domain(
bounds_domain_x, number_points_each_dimension_x)
logger.info('Generated discretization of domain_x')
JSONFile.write(discretization_data, domain_path)
return discretization_data
def cv_data_sets(cls, year, month, n_folds=5, random_seed=1):
"""
Creates n_folds files with pairs of datasets: (training_data, validation_data).
:param year: str
:param month: str (e.g. '1', '12')
:param n_folds: int
:param random_seed: int
"""
random.seed(random_seed)
file_name = cls._name_training_data(year=year, month=month)
data = JSONFile.read(file_name)
indexes_data = range(len(data))
random.shuffle(indexes_data)
n_batch = len(indexes_data) / n_folds
random_indexes = [
indexes_data[i * n_batch:n_batch + i * n_batch]
for i in xrange(n_folds)
]
extra = 0
for j in xrange(len(indexes_data) % n_folds):
random_indexes[j].append(indexes_data[n_batch + extra +
(n_folds - 1) * n_batch])
extra += 1
file_name = cls._name_fold_indexes(year=year, month=month)
JSONFile.write(random_indexes, file_name)
for i in xrange(n_folds):
validation = [data[index] for index in random_indexes[i]]
training_indexes = []
for j in xrange(n_folds):
if j != i:
training_indexes += random_indexes[j]
training = [data[index] for index in training_indexes]
file_name = cls._name_fold_data_training(year=year,
month=month,
fold=i)
JSONFile.write(training, file_name)
file_name = cls._name_fold_data_training_matlab(year=year,
month=month,
fold=i)
sio.savemat(file_name, {'training': training})
file_name = cls._name_fold_data_validation(year=year,
month=month,
fold=i)
JSONFile.write(validation, file_name)
file_name = cls._name_fold_data_validation_matlab(year=year,
month=month,
fold=i)
sio.savemat(file_name, {'validation': validation})
def top_users_papers(cls,
year,
month,
n_entries=100,
different_papers=20,
top_n=5000,
n_users=None,
only_assign_categories=True):
"""
Returns the users that accessed to at least n_entries papers, and at least different_papers
were different and were in the top_n papers in the month of the year.
Returns the top_n papers based on how many times they were seen.
:param year: (str)
:param month: (str) e.g. '1', '12'
:param n_entries: (int)
:param different_papers: int
:param top_n: int
:param n_users: (int) Maximum number of users allowed
:return: [ {'paper': (int) number of times seen},
{'user': {'stats': ((int) # entries, (int) # different papers in the top_n papers),
'diff_papers': [str]
}
}
]
"""
file_name = cls._name_file_(year=year, month=month)
data = JSONFile.read(file_name)
users = data[0]
papers = data[1]
n_papers = []
paper_ls = []
for paper in papers:
paper_ls.append(paper)
n_papers.append(papers[paper]['views'])
index_top_papers = sorted(range(len(n_papers)),
key=lambda k: n_papers[k])
index_top_papers = index_top_papers[-top_n:]
rank_papers = {}
for index in index_top_papers:
rank_papers[paper_ls[index]] = n_papers[index]
paper_ls = rank_papers.keys()
cls.assign_categories(paper_ls)
if only_assign_categories:
return
rank_user = {}
users_ls = []
n_entries_ls = []
for user in users:
users_ls.append(user)
n_entries_ls.append(sum(users[user].values()))
index_top_users = sorted(range(len(n_entries_ls)),
key=lambda k: n_entries_ls[k])
users_ls = [users_ls[i] for i in index_top_users]
n_entries_ls = [n_entries_ls[i] for i in index_top_users]
ind_bis = bisect_left(n_entries_ls, n_entries)
users_ls = users_ls[ind_bis:]
n_entries_ls = n_entries_ls[ind_bis:]
final_users = []
metric_users = []
for user, n in zip(users_ls, n_entries_ls):
diff_papers = set(users[user].keys()).intersection(set(paper_ls))
n_diff = len(diff_papers)
if n_diff < different_papers:
continue
final_users.append(user)
metric_users.append(n_diff)
rank_user[user] = {
'stats': (n, n_diff),
'diff_papers': diff_papers
}
index_top_users = sorted(range(len(final_users)),
key=lambda k: metric_users[k])
if n_users is not None and len(index_top_users) > n_users:
index_top_users = index_top_users[-n_users:]
rank_user_final = {}
for ind in index_top_users:
rank_user_final[final_users[ind]] = rank_user[final_users[ind]]
rank_user = rank_user_final
file_name = cls._name_file_final(year=year, month=month)
JSONFile.write([rank_papers, rank_user], file_name)
logger.info('Number of papers is %d' % len(rank_papers))
logger.info('Number of users is %d' % len(rank_user))
return [rank_papers, rank_user]
from __future__ import absolute_import
from stratified_bayesian_optimization.util.json_file import JSONFile
import numpy as np
read = JSONFile.read(
"problems/test_simulated_gp/simulated_function_with_1000_5")
points = read['points']
function = read['function']
def find_point_in_domain(x, array=np.array(points)):
"""
Find the index of the closest point in array to x
:param x: float
:param array: np.array(float)
:return: int
"""
idx = np.abs(array - x).argmin()
return idx
def toy_example(x):
"""
:param x: [float, int]
:return: [float]
"""
id = find_point_in_domain(x[0])
return [function[str(int(x[1]))][id]]
optimal_value = float(args.optimal_value)
only_plot = bool(int(args.only_plot))
n_iterations_plot = int(args.n_iterations_plot)
np.random.seed(random_seed)
method_ = method
if method == 'lipschitz' or method == 'approx_lipschitz':
method_ = 'real_gradient'
name_model = 'std_%f_rs_%d_lb_%f_ub_%f_lr_%f_%s' % (std, random_seed, lb,
ub, lr, method_)
dir_data = 'data/multi_start/' + problem_name + '/training_results/'
data = JSONFile.read(dir_data + name_model)
name_model = 'std_%f_rs_%d_lb_%f_ub_%f_lr_%f_%s' % (std, random_seed, lb,
ub, lr, method)
if method == 'real_gradient':
data['gradients'] = [-1.0 * np.array(t) for t in data['gradients']]
elif method == 'grad_epoch':
new_grads = {}
for t in data['gradients']:
new_grads[int(t)] = -1.0 * np.array(data['gradients'][t])
data['gradients'] = new_grads
data['values'] = [-1.0 * np.array(t) for t in data['values']]
def get_values(i):
def create_model(args, n_training=3, n_epochs=100, burning=True, point=None):
rs = int(args['rs'])
lb = [float(t) for t in args['lb'] ]
ub = [float(t) for t in args['ub']]
std = float(args['std'])
lr = float(args['lr'])
method = args['method']
problem_name = args['problem_name']
#TODO: ADD THIS AS A PARAMETER
lipschitz_cte = 2.0
method_ = method
if method == 'lipschitz' or method == 'approx_lipschitz':
method_ = 'real_gradient'
if point is None:
name_model = 'std_%f_rs_%d_lb_%f_ub_%f_lr_%f_%s' % (std, rs, lb[0], ub[0], lr, method_)
else:
name_model = 'std_%f_rs_%d_lb_%f_ub_%f_lr_%f_%s_point_%d' % (std, rs, lb[0], ub[0], lr, method_, point)
dir_data = 'data/multi_start/' + problem_name + '/' + 'training_results/'
data = JSONFile.read(dir_data + name_model)
if point is None:
name_model = 'std_%f_rs_%d_lb_%f_ub_%f_lr_%f_%s' % (std, rs, lb[0], ub[0], lr, method)
else:
name_model = 'std_%f_rs_%d_lb_%f_ub_%f_lr_%f_%s_point_%d' % (std, rs, lb[0], ub[0], lr, method, point)
if method == 'real_gradient':
data['gradients'] = [-1.0 * np.array(t) for t in data['gradients']]
elif method == 'grad_epoch':
new_grads = {}
for t in data['gradients']:
new_grads[int(t)] = -1.0 * np.array(data['gradients'][t])
data['gradients'] = new_grads
data['stochastic_gradients'] = [-1.0 * np.array(t) for t in data['stochastic_gradients']]
data['values'] = [-1.0 * np.array(t) for t in data['values']]
data['points'] = [np.array(t) for t in data['points']]
training_data = {'points': data['points'][0:n_training],
'values': data['values'][0:n_training], 'gradients': [],
'stochastic_gradients':data['stochastic_gradients'][0:n_training] }
if method == 'real_gradient':
training_data['gradients'] = data['gradients'][0:n_training]
elif method == 'grad_epoch':
training_data['gradients'] = {}
for j in range(n_training):
if j in data['gradients']:
training_data['gradients'][j] = data['gradients'][j]
points_domain = data['points'][0: n_training]
best_results = np.max(training_data['values'])
functions_get_value = get_values
kwargs = {'data': data, 'method': method}
n_burning = 50
n_batches = 1
total_iterations = n_epochs * n_batches
if method == 'approx_lipschitz' or method == 'lipschitz':
if method == 'approx_lipschitz':
lipschitz_cte = None
model = StatModelLipschitz(
training_data, best_results, n_training, functions_get_value,
points_domain[-1], 0,
n_training, specifications=name_model,problem_name=problem_name,
max_iterations=total_iterations, parametric_mean=False, lower=None, upper=None,
n_burning=n_burning, total_batches=n_batches, type_model=method, lipschitz=lipschitz_cte,
n_thinning=10, kwargs_get_value_next_iteration=kwargs, burning=burning)
else:
model = StatModel(
training_data, best_results, n_training, functions_get_value,
points_domain[-1], 0,
n_training, specifications=name_model, problem_name=problem_name,
max_iterations=total_iterations, parametric_mean=False, lower=None, upper=None,
n_burning=n_burning, total_batches=n_batches,model_gradient=method,
n_thinning=10, kwargs_get_value_next_iteration=kwargs, burning=burning)
return model
def collect_multi_spec_results(cls,
multiple_spec,
total_iterations=None,
sign=True,
sqr=False,
same_random_seeds=False,
rs_lw=0,
rs_up=None):
"""
Writes the files with the aggregated results
:param multiple_spec:
:param total_iterations: (int) Collect results until this iteration
:param sign: (boolean) If true, we multiply the results by -1
:param sqr: (boolean) If true, we take the square root of the results
:param same_random_seeds: (boolean) If true, we use the same random seeds for both problems
:return:
"""
if total_iterations is None:
total_iterations = 10000
n_specs = len(multiple_spec.get('random_seeds'))
results_dict = {}
if sign:
sign = -1.0
else:
sign = 1.0
if sqr:
f = lambda x: x**0.5
else:
f = lambda x: x
if rs_up is not None:
same_random_seeds = True
if same_random_seeds:
random_seeds = {}
for method in set(multiple_spec.get('method_optimizations')):
random_seeds[method] = []
for i in xrange(n_specs):
problem_name = multiple_spec.get('problem_names')[i]
dir = path.join(PROBLEM_DIR, problem_name, PARTIAL_RESULTS)
if not os.path.exists(dir):
continue
training_name = multiple_spec.get('training_names')[i]
n_training = multiple_spec.get('n_trainings')[i]
random_seed = multiple_spec.get('random_seeds')[i]
method = multiple_spec.get('method_optimizations')[i]
n_samples_parameters = multiple_spec.get(
'n_samples_parameterss')[i]
n_iterations = multiple_spec.get('n_iterationss')[i]
file_name = cls._filename_results(
problem_name=problem_name,
training_name=training_name,
n_points=n_training,
random_seed=random_seed,
method=method,
n_samples_parameters=n_samples_parameters,
)
file_path = path.join(dir, file_name)
if not os.path.exists(file_path):
continue
random_seeds[method].append(random_seed)
methods = list(set(multiple_spec.get('method_optimizations')))
random_seeds_check = set(random_seeds[methods[0]])
for i in xrange(1, len(methods)):
random_seeds_check = random_seeds_check.intersection(
random_seeds[methods[i]])
if rs_up is not None:
random_seeds_check = random_seeds_check.intersection(
range(rs_lw, rs_up))
for i in xrange(n_specs):
problem_name = multiple_spec.get('problem_names')[i]
dir = path.join(PROBLEM_DIR, problem_name, PARTIAL_RESULTS)
if not os.path.exists(dir):
continue
training_name = multiple_spec.get('training_names')[i]
n_training = multiple_spec.get('n_trainings')[i]
random_seed = multiple_spec.get('random_seeds')[i]
method = multiple_spec.get('method_optimizations')[i]
n_samples_parameters = multiple_spec.get(
'n_samples_parameterss')[i]
n_iterations = multiple_spec.get('n_iterationss')[i]
if same_random_seeds and random_seed not in random_seeds_check:
continue
file_name = cls._filename_results(
problem_name=problem_name,
training_name=training_name,
n_points=n_training,
random_seed=random_seed,
method=method,
n_samples_parameters=n_samples_parameters,
)
file_path = path.join(dir, file_name)
if not os.path.exists(file_path):
continue
results = JSONFile.read(file_path)
results = results['objective_values']
key_dict = (problem_name, training_name, n_training, method)
if key_dict not in results_dict:
results_dict[key_dict] = \
[[] for _ in range(min(n_iterations + 1, total_iterations))]
for iteration in range(
min(total_iterations, n_iterations + 1, len(results))):
results_dict[key_dict][iteration].append(
f(sign * results[iteration]))
problem_names = list(set(multiple_spec.get('problem_names')))
training_names = set(multiple_spec.get('training_names'))
n_trainings = set(multiple_spec.get('n_trainings'))
methods = set(multiple_spec.get('method_optimizations'))
aggregated_results = {}
for problem in problem_names:
for training in training_names:
for n_training in n_trainings:
for method in methods:
key = (problem, training, n_training, method)
aggregated_results[key] = {}
if key not in results_dict:
continue
results = results_dict[key]
for iteration in xrange(
min(len(results), total_iterations)):
if len(results[iteration]) > 0:
values = results[iteration]
mean = np.mean(values)
std = np.std(values)
n_samples = len(results[iteration])
ci_low = mean - 1.96 * std / np.sqrt(n_samples)
ci_up = mean + 1.96 * std / np.sqrt(n_samples)
aggregated_results[key][iteration] = {}
aggregated_results[key][iteration][
'mean'] = mean
aggregated_results[key][iteration]['std'] = std
aggregated_results[key][iteration][
'n_samples'] = n_samples
aggregated_results[key][iteration][
'ci_low'] = ci_low
aggregated_results[key][iteration][
'ci_up'] = ci_up
else:
break
if len(aggregated_results[key]) > 0:
dir = path.join(PROBLEM_DIR, problem,
AGGREGATED_RESULTS)
if not os.path.exists(dir):
os.mkdir(dir)
file_name = cls._aggregated_results(
problem_name=problem,
training_name=training,
n_points=n_training,
method=method,
)
file_path = path.join(dir, file_name)
JSONFile.write(aggregated_results[key], file_path)
def get_gp(cls,
name_model,
problem_name,
type_kernel,
dimensions,
bounds_domain,
type_bounds=None,
n_training=0,
noise=False,
training_data=None,
points=None,
training_name=None,
mle=True,
thinning=0,
n_burning=0,
max_steps_out=1,
n_samples=None,
random_seed=DEFAULT_RANDOM_SEED,
kernel_values=None,
mean_value=None,
var_noise_value=None,
cache=True,
same_correlation=False,
use_only_training_points=True,
optimization_method=None,
n_samples_parameters=0,
parallel_training=True,
simplex_domain=None,
objective_function=None,
define_samplers=True):
"""
Fetch a GP model from file if it exists, otherwise train a new model and save it locally.
:param name_model: str
:param problem_name: str
:param type_kernel: [(str)] Must be in possible_kernels. If it's a product of kernels it
should be a list as: [PRODUCT_KERNELS_SEPARABLE, NAME_1_KERNEL, NAME_2_KERNEL]
:param dimensions: [int]. It has only the n_tasks for the task_kernels, and for the
PRODUCT_KERNELS_SEPARABLE contains the dimensions of every kernel in the product
:param bounds_domain: [([float, float] or [float])], the first case is when the bounds are
lower or upper bound of the respective entry; in the second case, it's list of finite
points representing the domain of that entry.
:param type_bounds: [0 or 1], 0 if the bounds are lower or upper bound of the respective
entry, 1 if the bounds are all the finite options for that entry.
:param n_training: int
:param noise: (boolean) If true, we get noisy evaluations.
:param training_data: {'points': [[float]], 'evaluations': [float],
'var_noise': [float] or None}
:param points: [[float]]. If training_data is None, we can evaluate the objective
function in these points.
:param training_name: (str), prefix used to save the training data.
:param mle: (boolean) If true, fits the GP by MLE.
:param thinning: (int)
:param n_burning: (int) Number of burnings samples for the MCMC.
:param max_steps_out: (int) Maximum number of steps out for the stepping out or
doubling procedure in slice sampling.
:param n_samples: (int) If the objective is noisy, we take n_samples of the function to
estimate its value.
:param random_seed: (int)
:param kernel_values: [float], contains the default values of the parameters of the kernel
:param mean_value: [float], It contains the value of the mean parameter.
:param var_noise_value: [float], It contains the variance of the noise of the model
:param cache: (boolean) Try to get model from cache
:param same_correlation: (boolean) If true, it uses the same correlations for the task
kernel.
:param use_only_training_points (boolean) If the model is read, and the param is true,
it uses only the training points in data. Otherwise, it also includes new points
previously computed.
:param optimization_method: (str)
:param n_samples_parameters: (int)
:param parallel_training: (boolean)
:param define_samplers: (boolean) If False, samplers for the hyperparameters are not
defined.
:return: (GPFittingGaussian) - An instance of GPFittingGaussian
"""
model_type = cls._model_map[name_model]
if training_name is None:
training_name = 'default_training_data_%d_points_rs_%d' % (
n_training, random_seed)
if use_only_training_points:
f_name = cls._get_filename(model_type, problem_name, type_kernel,
training_name)
f_name_cache = cls._get_filename_modified(model_type, problem_name,
type_kernel,
training_name,
optimization_method,
n_samples_parameters)
else:
f_name = cls._get_filename_modified(model_type, problem_name,
type_kernel, training_name,
optimization_method,
n_samples_parameters)
if not os.path.exists('data'):
os.mkdir('data')
if not os.path.exists(GP_DIR):
os.mkdir(GP_DIR)
gp_dir = path.join(GP_DIR, problem_name)
if not os.path.exists(gp_dir):
os.mkdir(gp_dir)
gp_path = path.join(gp_dir, f_name)
gp_path_cache = path.join(gp_dir, f_name_cache)
if cache:
data = JSONFile.read(gp_path)
data = None
else:
data = None
if data is not None:
return model_type.deserialize(
data, use_only_training_points=use_only_training_points)
if training_data is None or training_data == {}:
training_data = TrainingDataService.get_training_data(
problem_name,
training_name,
bounds_domain,
n_training=n_training,
points=points,
noise=noise,
n_samples=n_samples,
random_seed=random_seed,
type_bounds=type_bounds,
cache=cache,
parallel=parallel_training,
gp_path_cache=gp_path_cache,
simplex_domain=simplex_domain,
objective_function=objective_function)
logger.info("Training %s" % model_type.__name__)
gp_model = model_type.train(type_kernel,
dimensions,
mle,
training_data,
bounds_domain,
thinning=thinning,
n_burning=n_burning,
max_steps_out=max_steps_out,
random_seed=random_seed,
type_bounds=type_bounds,
training_name=training_name,
problem_name=problem_name,
kernel_values=kernel_values,
mean_value=mean_value,
var_noise_value=var_noise_value,
same_correlation=same_correlation,
simplex_domain=simplex_domain,
define_samplers=define_samplers)
JSONFile.write(gp_model.serialize(), gp_path)
return gp_model
# parser.add_argument('method_2', help='real_gradient')
args_ = parser.parse_args()
n_iterations = int(args_.n_iterations)
method = args_.method
problem = args_.problem_name
animation = bool(int(args_.animation))
file_1 = 'data/multi_start/' + problem + '/' + 'greedy_policy/' + method + '.json'
file_2 = 'data/multi_start/' + problem + '/' + 'uniform_policy/' + method + '.json'
n_restarts = int(args_.n_starting_points)
n_training = 3
n = n_iterations
data = JSONFile.read(file_1)
data_2 = JSONFile.read(file_2)
data_list = [data, data_2]
best_values = {}
data_dict = {}
type_1 = 'greedy' #+ method
types = [type_1, 'equal_allocation']
data_dict[type_1] = data
data_dict['equal_allocation'] = data_2
for t in types:
best_values[t] = get_best_values(data_dict[t], n_restarts, n_training)
# num_user = 4815
num_item = 2018
num_user = 2752
# there are 263238 observations
total_obs = 263238
num_batches = int((n_folds - 1) * (float(total_obs) / float(n_folds)) / 500.0)
# num_item = 326
# num_user = 507
# there are 90271 observations
train=[]
validate=[]
file_name = TrainingData._name_fold_indexes(year=year, month=month)
random_indexes = JSONFile.read(file_name)
# file_name = TrainingData._name_training_data(year=year, month=month)
# training_data = JSONFile.read(file_name)
for i in range(n_folds):
file_name = TrainingData._name_fold_data_training(year=year, month=month, fold=i)
training = JSONFile.read(file_name)
train.append(np.array(training))
file_name = TrainingData._name_fold_data_validation(year=year, month=month, fold=i)
validation = JSONFile.read(file_name)
validate.append(np.array(validation))
def toy_example(x):
"""
def get_training_data(cls,
problem_name,
training_name,
bounds_domain,
n_training=5,
points=None,
noise=False,
n_samples=None,
random_seed=DEFAULT_RANDOM_SEED,
parallel=True,
type_bounds=None,
cache=True,
gp_path_cache=None,
simplex_domain=None,
objective_function=None):
"""
:param problem_name: str
:param training_name: (str), prefix used to save the training data.
:param bounds_domain: [([float, float] or [float])], the first case is when the bounds are
lower or upper bound of the respective entry; in the second case, it's list of finite
points representing the domain of that entry.
:param n_training: (int), number of training points if points is None
:param points: [[float]]
:param noise: boolean, true if the evaluations are noisy
:param n_samples: int. If noise is true, we take n_samples of the function to estimate its
value.
:param random_seed: int
:param parallel: (boolean) Train in parallel if it's True.
:param type_bounds: [0 or 1], 0 if the bounds are lower or upper bound of the respective
entry, 1 if the bounds are all the finite options for that entry.
:param cache: (boolean) Try to get model from cache
:return: {'points': [[float]], 'evaluations': [float], 'var_noise': [float] or []}
"""
if cache and gp_path_cache is not None:
data = JSONFile.read(gp_path_cache)
if data is not None:
return data['data']
logger.info("Getting training data")
rs = random_seed
if points is not None and len(points) > 0:
n_training = len(points)
rs = 0
file_name = cls._filename(
problem_name=problem_name,
training_name=training_name,
n_points=n_training,
random_seed=rs,
)
if not os.path.exists(PROBLEM_DIR):
os.mkdir(PROBLEM_DIR)
training_dir = path.join(PROBLEM_DIR, problem_name, 'data')
if not os.path.exists(path.join(PROBLEM_DIR, problem_name)):
os.mkdir(path.join(PROBLEM_DIR, problem_name))
if not os.path.exists(training_dir):
os.mkdir(training_dir)
training_path = path.join(training_dir, file_name)
if cache:
training_data = JSONFile.read(training_path)
else:
training_data = None
if training_data is not None:
return training_data
if n_training == 0:
return {'points': [], 'evaluations': [], 'var_noise': []}
np.random.seed(random_seed)
if points is None or len(points) == 0:
points = cls.get_points_domain(n_training,
bounds_domain,
random_seed,
training_name,
problem_name,
type_bounds,
simplex_domain=simplex_domain)
if objective_function is None:
name_module = cls.get_name_module(problem_name)
module = __import__(name_module, globals(), locals(), -1)
else:
name_module = None
module = None
training_data = {}
training_data['points'] = points
training_data['evaluations'] = []
training_data['var_noise'] = []
if not parallel:
for point in points:
if noise:
if module is not None:
evaluation = cls.evaluate_function(
module, point, n_samples)
else:
evaluation = objective_function(point, n_samples)
training_data['var_noise'].append(evaluation[1])
else:
if module is not None:
evaluation = cls.evaluate_function(module, point)
else:
evaluation = objective_function(point)
training_data['evaluations'].append(evaluation[0])
JSONFile.write(training_data, training_path)
JSONFile.write(training_data, training_path)
return training_data
arguments = convert_list_to_dictionary(points)
if name_module is not None:
kwargs = {
'name_module': name_module,
'cls_': cls,
'n_samples': n_samples
}
else:
kwargs = {
'name_module': None,
'cls_': cls,
'n_samples': n_samples,
'objective_function': objective_function
}
training_points = Parallel.run_function_different_arguments_parallel(
wrapper_evaluate_objective_function, arguments, **kwargs)
training_points = convert_dictionary_to_list(training_points)
training_data['evaluations'] = [value[0] for value in training_points]
if noise:
training_data['var_noise'] = [
value[1] for value in training_points
]
if cache:
JSONFile.write(training_data, training_path)
return training_data
def plot_aggregate_results(multiple_spec,
negative=True,
square=True,
title_plot=None,
y_label=None,
n_iterations=None,
repeat_ei=1):
"""
:param multiple_spec: (multiple_spec entity) Name of the files with the aggregate results
:return:
"""
problem_names = list(set(multiple_spec.get('problem_names')))
training_names = set(multiple_spec.get('training_names'))
n_trainings = set(multiple_spec.get('n_trainings'))
methods = set(multiple_spec.get('method_optimizations'))
results = {}
file_path_plot = None
for problem in problem_names:
dir = path.join(PROBLEM_DIR, problem, AGGREGATED_RESULTS)
if not os.path.exists(dir):
continue
for training in training_names:
for n_training in n_trainings:
file_name = _aggregated_results_plot(
problem_name=problem,
training_name=training,
n_points=n_training,
)
if file_path_plot is None:
logger.info('problem is: %s' % dir)
file_path_plot = path.join(dir, file_name)
for method in methods:
if method in results:
continue
file_name = _aggregated_results(
problem_name=problem,
training_name=training,
n_points=n_training,
method=method,
)
file_path = path.join(dir, file_name)
if not os.path.exists(file_path):
continue
data = JSONFile.read(file_path)
x_axis = list(data.keys())
x_axis = [int(i) for i in x_axis]
x_axis.sort()
if repeat_ei > 1 and method == EI_METHOD:
new_x = []
for i in x_axis:
new_x += range(i * repeat_ei, (i + 1) * repeat_ei)
x_axis = new_x
if n_iterations is not None:
x_axis = x_axis[0:n_iterations]
y_values = []
ci_u = []
ci_l = []
for i in x_axis:
if repeat_ei > 1 and method == EI_METHOD:
j = i / repeat_ei
else:
j = i
y_values.append(data[str(j)]['mean'])
ci_u.append(data[str(j)]['ci_up'])
ci_l.append(data[str(j)]['ci_low'])
results[method] = [x_axis, y_values, ci_u, ci_l]
colors = ['b', 'r', 'g']
plt.figure()
for id, method in enumerate(results):
label = str(method)
if label == SBO_METHOD:
label = 'ibo'
x_axis = results[method][0]
y_values = results[method][1]
ci_u = results[method][2]
ci_l = results[method][3]
col = colors[id]
plt.plot(x_axis, y_values, color=col, linewidth=2.0, label=label)
plt.plot(x_axis, ci_u, '--', color=col, label="95% CI")
plt.plot(x_axis, ci_l, '--', color=col)
if title_plot is None:
title_plot = problem_names[0]
if y_label is None:
y_label = 'Cross Validation Error'
plt.xlabel('Number of Samples', fontsize=22)
plt.ylabel(y_label, fontsize=22)
plt.legend(loc=3, ncol=2, mode="expand", borderaxespad=0.)
plt.title(title_plot, fontsize=22)
plt.subplots_adjust(left=0.13, right=0.99, top=0.92, bottom=0.12)
plt.savefig(file_path_plot)
from stratified_bayesian_optimization.util.json_file import JSONFile
from stratified_bayesian_optimization.initializers.log import SBOLog
logger = SBOLog(__name__)
if __name__ == '__main__':
# Example usage:
# python -m problems.cnn_cifar10.scripts.maximum_runs 500 600
parser = argparse.ArgumentParser()
parser.add_argument('min_rs', help='e.g. 500')
parser.add_argument('max_rs', help='e.g. 600')
args = parser.parse_args()
min_rs = int(args.min_rs)
max_rs = int(args.max_rs)
max_values = []
for i in xrange(min_rs, max_rs):
file_name = 'problems/cnn_cifar10/runs_random_seeds/' + 'rs_%d' % i + '.json'
if not os.path.exists(file_name):
continue
data = JSONFile.read(file_name)
max_values.append(data['test_error_images'])
max = np.max(max_values)
min = np.min(max_values)
logger.info('max is: %f' % max)
logger.info('min is: %f' % min)
def generate_evaluations(self,
problem_name,
model_type,
training_name,
n_training,
random_seed,
iteration,
n_points_by_dimension=None,
n_tasks=0):
"""
Generates evaluations of SBO, and write them in the debug directory.
:param problem_name: (str)
:param model_type: (str)
:param training_name: (str)
:param n_training: (int)
:param random_seed: (int)
:param iteration: (int)
:param n_points_by_dimension: [int] Number of points by dimension
:param n_tasks: (int) n_tasks > 0 if the last element of the domain is a task
"""
if not os.path.exists(DEBUGGING_DIR):
os.mkdir(DEBUGGING_DIR)
debug_dir = path.join(DEBUGGING_DIR, problem_name)
if not os.path.exists(debug_dir):
os.mkdir(debug_dir)
kernel_name = ''
for kernel in self.gp.type_kernel:
kernel_name += kernel + '_'
kernel_name = kernel_name[0:-1]
f_name = self._filename_points_ei_evaluations(
model_type=model_type,
problem_name=problem_name,
type_kernel=kernel_name,
training_name=training_name,
n_training=n_training,
random_seed=random_seed)
debug_path = path.join(debug_dir, f_name)
vectors = JSONFile.read(debug_path)
if vectors is None:
bounds = self.gp.bounds
n_points = n_points_by_dimension
if n_points is None:
n_points = (bounds[0][1] - bounds[0][0]) * 10
if n_tasks > 0:
bounds_x = [bounds[i] for i in xrange(len(bounds) - 1)]
n_points_x = [n_points[i] for i in xrange(len(n_points))]
else:
n_points_x = n_points
bounds_x = bounds
points = []
for bound, number_points in zip(bounds_x, n_points_x):
points.append(np.linspace(bound[0], bound[1], number_points))
vectors = []
for point in itertools.product(*points):
vectors.append(point)
JSONFile.write(vectors, debug_path)
n = len(vectors)
points_ = deepcopy(vectors)
vectors = np.array(vectors)
if n_tasks > 0:
vectors_ = None
for i in xrange(n_tasks):
task_vector = np.zeros(n) + i
task_vector = task_vector.reshape((n, 1))
points_ = np.concatenate((vectors, task_vector), axis=1)
if vectors_ is not None:
vectors_ = np.concatenate((vectors_, points_), axis=0)
else:
vectors_ = points_
vectors = vectors_
# TODO: extend to the case where w can be continuous
n = vectors.shape[0]
points = {}
for i in xrange(n):
points[i] = vectors[i, :]
args = (
False,
None,
False,
0,
self,
)
val = Parallel.run_function_different_arguments_parallel(
wrapper_objective_acquisition_function, points, *args)
values = np.zeros(n)
for i in xrange(n):
values[i] = val.get(i)
f_name = self._filename_ei_evaluations(iteration=iteration,
model_type=model_type,
problem_name=problem_name,
type_kernel=kernel_name,
training_name=training_name,
n_training=n_training,
random_seed=random_seed)
debug_path = path.join(debug_dir, f_name)
JSONFile.write({'points': points_, 'evaluations': values}, debug_path)
return values
lower_random_seed = int(args_.lower_random_seed)
upper_random_seed = int(args_.upper_random_seed)
prefix_file_1 = 'data/multi_start/' + problem + '/' + 'greedy_policy/' + method + '_random_seed_'
prefix_file_2 = 'data/multi_start/' + problem + '/' +'uniform_policy/' + method + '_random_seed_'
# prefix_file_3 = 'data/multi_start/' + problem + '/' + 'random_policy/' + method + '_random_seed_'
prefix_file_3 = 'data/multi_start/' + problem + '/' + 'swersky_greedy_policy/' + 'swersky' + '_random_seed_'
data = {}
data_2 = {}
data_3 = {}
for i in range(lower_random_seed, upper_random_seed):
file_1 = prefix_file_1 + str(i) + '_n_restarts_' + str(n_restarts) + '.json'
try:
data[i] = JSONFile.read(file_1)
except Exception as e:
data[i] = None
file_2 = prefix_file_2 + str(i) + '_n_restarts_' + str(n_restarts)+ '.json'
try:
data_2[i] = JSONFile.read(file_2)
except Exception as e:
data_2[i] = None
file_3 = prefix_file_3 + str(i) + '_n_restarts_' + str(n_restarts)+ '.json'
try:
data_3[i] = JSONFile.read(file_3)
except Exception as e:
data_3[i] = None
parser = argparse.ArgumentParser()
parser.add_argument('starting_point', help='e.g. 0')
args = parser.parse_args()
starting_point_index = int(args.starting_point)
dir_data = 'data/multi_start/neural_networks/training_results/'
n_epochs = 20
n_batches = 60
total_iterations = n_epochs * n_batches
cnn_data = {}
cnn_data[starting_point_index] = JSONFile.read(dir_data +
str(starting_point_index))
for j in cnn_data[starting_point_index]:
cnn_data[starting_point_index][j] = [
t / 100.0 for t in cnn_data[starting_point_index][j]
]
def get_values(i, index):
data = cnn_data[index]
return data[str(i / (n_batches + 1) + 1)][(i - 1) % n_batches]
training_data = {}
best_results = {}
functions_get_value = {}
arguments = {}
n_training = 3
def from_spec(cls, spec):
"""
Construct BGO instance from spec
:param spec: RunSpecEntity
:return: BGO
# TO DO: It now only returns domain
"""
random_seed = spec.get('random_seed')
method_optimization = spec.get('method_optimization')
logger.info("Training GP model")
logger.info("Random seed is: %d" % random_seed)
logger.info("Algorithm used is:")
logger.info(method_optimization)
gp_model = GPFittingService.from_dict(spec)
noise = spec.get('noise')
quadrature = None
acquisition_function = None
domain = DomainService.from_dict(spec)
if method_optimization not in cls._possible_optimization_methods:
raise Exception("Incorrect BGO method")
if method_optimization == SBO_METHOD:
x_domain = spec.get('x_domain')
distribution = spec.get('distribution')
parameters_distribution = spec.get('parameters_distribution')
quadrature = BayesianQuadrature(
gp_model,
x_domain,
distribution,
parameters_distribution=parameters_distribution)
acquisition_function = SBO(
quadrature, np.array(domain.discretization_domain_x))
elif method_optimization == MULTI_TASK_METHOD:
x_domain = spec.get('x_domain')
distribution = spec.get('distribution')
parameters_distribution = spec.get('parameters_distribution')
quadrature = BayesianQuadrature(
gp_model,
x_domain,
distribution,
parameters_distribution=parameters_distribution,
model_only_x=True)
acquisition_function = MultiTasks(
quadrature, quadrature.parameters_distribution.get(TASKS))
elif method_optimization == EI_METHOD:
acquisition_function = EI(gp_model, noisy_evaluations=noise)
elif method_optimization == SDE_METHOD:
x_domain = len(spec.get('x_domain'))
parameters_distribution = spec.get('parameters_distribution')
domain_random = np.array(parameters_distribution['domain_random'])
weights = np.array(parameters_distribution['weights'])
acquisition_function = SDE(gp_model, domain_random, x_domain,
weights)
problem_name = spec.get('problem_name')
training_name = spec.get('training_name')
n_samples = spec.get('n_samples')
minimize = spec.get('minimize')
n_iterations = spec.get('n_iterations')
name_model = spec.get('name_model')
parallel = spec.get('parallel')
n_training = spec.get('n_training')
number_points_each_dimension_debug = spec.get(
'number_points_each_dimension_debug')
n_samples_parameters = spec.get('n_samples_parameters', 0)
use_only_training_points = spec.get('use_only_training_points', True)
n_iterations = n_iterations - (
len(gp_model.training_data['evaluations']) - n_training)
bgo = cls(acquisition_function,
gp_model,
n_iterations,
problem_name,
training_name,
random_seed,
n_training,
name_model,
method_optimization,
minimize=minimize,
n_samples=n_samples,
noise=noise,
quadrature=quadrature,
parallel=parallel,
number_points_each_dimension_debug=
number_points_each_dimension_debug,
n_samples_parameters=n_samples_parameters,
use_only_training_points=use_only_training_points)
if n_training < len(bgo.gp_model.training_data['evaluations']):
extra_iterations = len(
bgo.gp_model.training_data['evaluations']) - n_training
data = JSONFile.read(bgo.objective.file_path)
bgo.objective.evaluated_points = data['evaluated_points'][
0:extra_iterations]
bgo.objective.objective_values = data['objective_values'][
0:extra_iterations]
bgo.objective.model_objective_values = \
data['model_objective_values'][0:extra_iterations]
bgo.objective.standard_deviation_evaluations = data[
'standard_deviation_evaluations']
return bgo